diff options
Diffstat (limited to 'drivers/memory')
52 files changed, 17303 insertions, 5997 deletions
diff --git a/drivers/memory/.gitignore b/drivers/memory/.gitignore index cbca8b028437..5e84bee05ef8 100644 --- a/drivers/memory/.gitignore +++ b/drivers/memory/.gitignore @@ -1 +1,2 @@ -ti-emif-asm-offsets.h +# SPDX-License-Identifier: GPL-2.0-only +/ti-emif-asm-offsets.h diff --git a/drivers/memory/Kconfig b/drivers/memory/Kconfig index 9bddca292330..fac290e48e0b 100644 --- a/drivers/memory/Kconfig +++ b/drivers/memory/Kconfig @@ -5,6 +5,12 @@ menuconfig MEMORY bool "Memory Controller drivers" + help + This option allows to enable specific memory controller drivers, + useful mostly on embedded systems. These could be controllers + for DRAM (SDR, DDR), ROM, SRAM and others. The drivers features + vary from memory tuning and frequency scaling to enabling + access to attached peripherals through memory bus. if MEMORY @@ -26,8 +32,9 @@ config ARM_PL172_MPMC config ATMEL_SDRAMC bool "Atmel (Multi-port DDR-)SDRAM Controller" - default y - depends on ARCH_AT91 && OF + default y if ARCH_AT91 + depends on ARCH_AT91 || COMPILE_TEST + depends on OF help This driver is for Atmel SDRAM Controller or Atmel Multi-port DDR-SDRAM Controller available on Atmel AT91SAM9 and SAMA5 SoCs. @@ -36,8 +43,9 @@ config ATMEL_SDRAMC config ATMEL_EBI bool "Atmel EBI driver" - default y - depends on ARCH_AT91 && OF + default y if ARCH_AT91 + depends on ARCH_AT91 || COMPILE_TEST + depends on OF select MFD_SYSCON select MFD_ATMEL_SMC help @@ -46,9 +54,42 @@ config ATMEL_EBI tree is used. This bus supports NANDs, external ethernet controller, SRAMs, ATA devices, etc. +config BRCMSTB_DPFE + tristate "Broadcom STB DPFE driver" + default ARCH_BRCMSTB + depends on ARCH_BRCMSTB || COMPILE_TEST + help + This driver provides access to the DPFE interface of Broadcom + STB SoCs. The firmware running on the DCPU inside the DDR PHY can + provide current information about the system's RAM, for instance + the DRAM refresh rate. This can be used as an indirect indicator + for the DRAM's temperature. Slower refresh rate means cooler RAM, + higher refresh rate means hotter RAM. + +config BRCMSTB_MEMC + tristate "Broadcom STB MEMC driver" + default ARCH_BRCMSTB + depends on ARCH_BRCMSTB || COMPILE_TEST + help + This driver provides a way to configure the Broadcom STB memory + controller and specifically control the Self Refresh Power Down + (SRPD) inactivity timeout. + +config BT1_L2_CTL + bool "Baikal-T1 CM2 L2-RAM Cache Control Block" + depends on MIPS_BAIKAL_T1 || COMPILE_TEST + select MFD_SYSCON + help + Baikal-T1 CPU is based on the MIPS P5600 Warrior IP-core. The CPU + resides Coherency Manager v2 with embedded 1MB L2-cache. It's + possible to tune the L2 cache performance up by setting the data, + tags and way-select latencies of RAM access. This driver provides a + dt properties-based and sysfs interface for it. + config TI_AEMIF tristate "Texas Instruments AEMIF driver" - depends on (ARCH_DAVINCI || ARCH_KEYSTONE) && OF + depends on ARCH_DAVINCI || ARCH_KEYSTONE || COMPILE_TEST + depends on OF help This driver is for the AEMIF module available in Texas Instruments SoCs. AEMIF stands for Asynchronous External Memory Interface and @@ -59,7 +100,7 @@ config TI_AEMIF config TI_EMIF tristate "Texas Instruments EMIF driver" - depends on ARCH_OMAP2PLUS + depends on ARCH_OMAP2PLUS || COMPILE_TEST select DDR help This driver is for the EMIF module available in Texas Instruments @@ -71,7 +112,9 @@ config TI_EMIF temperature changes config OMAP_GPMC - bool + tristate "Texas Instruments OMAP SoC GPMC driver" + depends on OF_ADDRESS + depends on ARCH_OMAP2PLUS || ARCH_KEYSTONE || ARCH_K3 || COMPILE_TEST select GPIOLIB help This driver is for the General Purpose Memory Controller (GPMC) @@ -95,7 +138,8 @@ config OMAP_GPMC_DEBUG config TI_EMIF_SRAM tristate "Texas Instruments EMIF SRAM driver" - depends on (SOC_AM33XX || SOC_AM43XX) && SRAM + depends on SOC_AM33XX || SOC_AM43XX || (ARM && CPU_V7 && COMPILE_TEST) + depends on SRAM help This driver is for the EMIF module available on Texas Instruments AM33XX and AM43XX SoCs and is required for PM. Certain parts of @@ -103,10 +147,20 @@ config TI_EMIF_SRAM sequence so this driver provides several relocatable PM functions for the SoC PM code to use. +config FPGA_DFL_EMIF + tristate "FPGA DFL EMIF Driver" + depends on FPGA_DFL && HAS_IOMEM + help + This driver is for the EMIF private feature implemented under + FPGA Device Feature List (DFL) framework. It is used to expose + memory interface status information as well as memory clearing + control. + config MVEBU_DEVBUS bool "Marvell EBU Device Bus Controller" - default y - depends on PLAT_ORION && OF + default y if PLAT_ORION + depends on PLAT_ORION || COMPILE_TEST + depends on OF help This driver is for the Device Bus controller available in some Marvell EBU SoCs such as Discovery (mv78xx0), Orion (88f5xxx) and @@ -115,7 +169,7 @@ config MVEBU_DEVBUS config FSL_CORENET_CF tristate "Freescale CoreNet Error Reporting" - depends on FSL_SOC_BOOKE + depends on FSL_SOC_BOOKE || COMPILE_TEST help Say Y for reporting of errors from the Freescale CoreNet Coherency Fabric. Errors reported include accesses to @@ -124,13 +178,12 @@ config FSL_CORENET_CF represents a coherency violation. config FSL_IFC - bool + bool "Freescale IFC driver" if COMPILE_TEST depends on FSL_SOC || ARCH_LAYERSCAPE || SOC_LS1021A || COMPILE_TEST depends on HAS_IOMEM config JZ4780_NEMC bool "Ingenic JZ4780 SoC NEMC driver" - default y depends on MIPS || COMPILE_TEST depends on HAS_IOMEM && OF help @@ -139,7 +192,7 @@ config JZ4780_NEMC memory devices such as NAND and SRAM. config MTK_SMI - bool + tristate "MediaTek SoC Memory Controller driver" if COMPILE_TEST depends on ARCH_MEDIATEK || COMPILE_TEST help This driver is for the Memory Controller module in MediaTek SoCs, @@ -148,7 +201,7 @@ config MTK_SMI config DA8XX_DDRCTL bool "Texas Instruments da8xx DDR2/mDDR driver" - depends on ARCH_DAVINCI_DA8XX + depends on ARCH_DAVINCI_DA8XX || COMPILE_TEST help This driver is for the DDR2/mDDR Memory Controller present on Texas Instruments da8xx SoCs. It's used to tweak various memory @@ -156,13 +209,33 @@ config DA8XX_DDRCTL config PL353_SMC tristate "ARM PL35X Static Memory Controller(SMC) driver" - default y - depends on ARM + default y if ARM + depends on ARM || COMPILE_TEST depends on ARM_AMBA help This driver is for the ARM PL351/PL353 Static Memory Controller(SMC) module. +config RENESAS_RPCIF + tristate "Renesas RPC-IF driver" + depends on ARCH_RENESAS || COMPILE_TEST + select REGMAP_MMIO + select RESET_CONTROLLER + help + This supports Renesas R-Car Gen3 or RZ/G2 RPC-IF which provides + either SPI host or HyperFlash. You'll have to select individual + components under the corresponding menu. + +config STM32_FMC2_EBI + tristate "Support for FMC2 External Bus Interface on STM32MP SoCs" + depends on MACH_STM32MP157 || COMPILE_TEST + select MFD_SYSCON + help + Select this option to enable the STM32 FMC2 External Bus Interface + controller. This driver configures the transactions with external + devices (like SRAM, ethernet adapters, FPGAs, LCD displays, ...) on + SOCs containing the FMC2 External Bus Interface. + source "drivers/memory/samsung/Kconfig" source "drivers/memory/tegra/Kconfig" diff --git a/drivers/memory/Makefile b/drivers/memory/Makefile index 27b493435e61..e148f636c082 100644 --- a/drivers/memory/Makefile +++ b/drivers/memory/Makefile @@ -10,7 +10,9 @@ endif obj-$(CONFIG_ARM_PL172_MPMC) += pl172.o obj-$(CONFIG_ATMEL_SDRAMC) += atmel-sdramc.o obj-$(CONFIG_ATMEL_EBI) += atmel-ebi.o -obj-$(CONFIG_ARCH_BRCMSTB) += brcmstb_dpfe.o +obj-$(CONFIG_BRCMSTB_DPFE) += brcmstb_dpfe.o +obj-$(CONFIG_BRCMSTB_MEMC) += brcmstb_memc.o +obj-$(CONFIG_BT1_L2_CTL) += bt1-l2-ctl.o obj-$(CONFIG_TI_AEMIF) += ti-aemif.o obj-$(CONFIG_TI_EMIF) += emif.o obj-$(CONFIG_OMAP_GPMC) += omap-gpmc.o @@ -21,10 +23,14 @@ obj-$(CONFIG_JZ4780_NEMC) += jz4780-nemc.o obj-$(CONFIG_MTK_SMI) += mtk-smi.o obj-$(CONFIG_DA8XX_DDRCTL) += da8xx-ddrctl.o obj-$(CONFIG_PL353_SMC) += pl353-smc.o +obj-$(CONFIG_RENESAS_RPCIF) += renesas-rpc-if.o +obj-$(CONFIG_STM32_FMC2_EBI) += stm32-fmc2-ebi.o obj-$(CONFIG_SAMSUNG_MC) += samsung/ obj-$(CONFIG_TEGRA_MC) += tegra/ obj-$(CONFIG_TI_EMIF_SRAM) += ti-emif-sram.o +obj-$(CONFIG_FPGA_DFL_EMIF) += dfl-emif.o + ti-emif-sram-objs := ti-emif-pm.o ti-emif-sram-pm.o AFLAGS_ti-emif-sram-pm.o :=-Wa,-march=armv7-a diff --git a/drivers/memory/atmel-ebi.c b/drivers/memory/atmel-ebi.c index 14386d0b5f57..e749dcb3ddea 100644 --- a/drivers/memory/atmel-ebi.c +++ b/drivers/memory/atmel-ebi.c @@ -544,20 +544,27 @@ static int atmel_ebi_probe(struct platform_device *pdev) smc_np = of_parse_phandle(dev->of_node, "atmel,smc", 0); ebi->smc.regmap = syscon_node_to_regmap(smc_np); - if (IS_ERR(ebi->smc.regmap)) - return PTR_ERR(ebi->smc.regmap); + if (IS_ERR(ebi->smc.regmap)) { + ret = PTR_ERR(ebi->smc.regmap); + goto put_node; + } ebi->smc.layout = atmel_hsmc_get_reg_layout(smc_np); - if (IS_ERR(ebi->smc.layout)) - return PTR_ERR(ebi->smc.layout); + if (IS_ERR(ebi->smc.layout)) { + ret = PTR_ERR(ebi->smc.layout); + goto put_node; + } ebi->smc.clk = of_clk_get(smc_np, 0); if (IS_ERR(ebi->smc.clk)) { - if (PTR_ERR(ebi->smc.clk) != -ENOENT) - return PTR_ERR(ebi->smc.clk); + if (PTR_ERR(ebi->smc.clk) != -ENOENT) { + ret = PTR_ERR(ebi->smc.clk); + goto put_node; + } ebi->smc.clk = NULL; } + of_node_put(smc_np); ret = clk_prepare_enable(ebi->smc.clk); if (ret) return ret; @@ -600,12 +607,18 @@ static int atmel_ebi_probe(struct platform_device *pdev) child); ret = atmel_ebi_dev_disable(ebi, child); - if (ret) + if (ret) { + of_node_put(child); return ret; + } } } return of_platform_populate(np, NULL, NULL, dev); + +put_node: + of_node_put(smc_np); + return ret; } static __maybe_unused int atmel_ebi_resume(struct device *dev) diff --git a/drivers/memory/brcmstb_dpfe.c b/drivers/memory/brcmstb_dpfe.c index 82b415be18d1..76c82e9c8fce 100644 --- a/drivers/memory/brcmstb_dpfe.c +++ b/drivers/memory/brcmstb_dpfe.c @@ -23,7 +23,7 @@ * - BE kernel + LE firmware image * - BE kernel + BE firmware image * - * The DPCU always runs in big endian mode. The firwmare image, however, can + * The DPCU always runs in big endian mode. The firmware image, however, can * be in either format. Also, communication between host CPU and DCPU is * always in little endian. */ @@ -188,11 +188,6 @@ struct brcmstb_dpfe_priv { struct mutex lock; }; -static const char *error_text[] = { - "Success", "Header code incorrect", "Unknown command or argument", - "Incorrect checksum", "Malformed command", "Timed out", -}; - /* * Forward declaration of our sysfs attribute functions, so we can declare the * attribute data structures early. @@ -307,6 +302,20 @@ static const struct dpfe_api dpfe_api_v3 = { }, }; +static const char *get_error_text(unsigned int i) +{ + static const char * const error_text[] = { + "Success", "Header code incorrect", + "Unknown command or argument", "Incorrect checksum", + "Malformed command", "Timed out", "Unknown error", + }; + + if (unlikely(i >= ARRAY_SIZE(error_text))) + i = ARRAY_SIZE(error_text) - 1; + + return error_text[i]; +} + static bool is_dcpu_enabled(struct brcmstb_dpfe_priv *priv) { u32 val; @@ -379,9 +388,8 @@ static void __iomem *get_msg_ptr(struct brcmstb_dpfe_priv *priv, u32 response, void __iomem *ptr = NULL; /* There is no need to use this function for API v3 or later. */ - if (unlikely(priv->dpfe_api->version >= 3)) { + if (unlikely(priv->dpfe_api->version >= 3)) return NULL; - } msg_type = (response >> DRAM_MSG_TYPE_OFFSET) & DRAM_MSG_TYPE_MASK; offset = (response >> DRAM_MSG_ADDR_OFFSET) & DRAM_MSG_ADDR_MASK; @@ -416,7 +424,7 @@ static void __finalize_command(struct brcmstb_dpfe_priv *priv) /* * It depends on the API version which MBOX register we have to write to - * to signal we are done. + * signal we are done. */ release_mbox = (priv->dpfe_api->version < 2) ? REG_TO_HOST_MBOX : REG_TO_DCPU_MBOX; @@ -446,7 +454,7 @@ static int __send_command(struct brcmstb_dpfe_priv *priv, unsigned int cmd, } if (resp != 0) { mutex_unlock(&priv->lock); - return -ETIMEDOUT; + return -ffs(DCPU_RET_ERR_TIMEDOUT); } /* Compute checksum over the message */ @@ -648,8 +656,10 @@ static int brcmstb_dpfe_download_firmware(struct brcmstb_dpfe_priv *priv) return (ret == -ENOENT) ? -EPROBE_DEFER : ret; ret = __verify_firmware(&init, fw); - if (ret) - return -EFAULT; + if (ret) { + ret = -EFAULT; + goto release_fw; + } __disable_dcpu(priv); @@ -668,18 +678,20 @@ static int brcmstb_dpfe_download_firmware(struct brcmstb_dpfe_priv *priv) ret = __write_firmware(priv->dmem, dmem, dmem_size, is_big_endian); if (ret) - return ret; + goto release_fw; ret = __write_firmware(priv->imem, imem, imem_size, is_big_endian); if (ret) - return ret; + goto release_fw; ret = __verify_fw_checksum(&init, priv, header, init.chksum); if (ret) - return ret; + goto release_fw; __enable_dcpu(priv); - return 0; +release_fw: + release_firmware(fw); + return ret; } static ssize_t generic_show(unsigned int command, u32 response[], @@ -692,7 +704,7 @@ static ssize_t generic_show(unsigned int command, u32 response[], ret = __send_command(priv, command, response); if (ret < 0) - return sprintf(buf, "ERROR: %s\n", error_text[-ret]); + return sprintf(buf, "ERROR: %s\n", get_error_text(-ret)); return 0; } @@ -845,7 +857,6 @@ static int brcmstb_dpfe_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; struct brcmstb_dpfe_priv *priv; - struct resource *res; int ret; priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL); @@ -857,22 +868,19 @@ static int brcmstb_dpfe_probe(struct platform_device *pdev) mutex_init(&priv->lock); platform_set_drvdata(pdev, priv); - res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "dpfe-cpu"); - priv->regs = devm_ioremap_resource(dev, res); + priv->regs = devm_platform_ioremap_resource_byname(pdev, "dpfe-cpu"); if (IS_ERR(priv->regs)) { dev_err(dev, "couldn't map DCPU registers\n"); return -ENODEV; } - res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "dpfe-dmem"); - priv->dmem = devm_ioremap_resource(dev, res); + priv->dmem = devm_platform_ioremap_resource_byname(pdev, "dpfe-dmem"); if (IS_ERR(priv->dmem)) { dev_err(dev, "Couldn't map DCPU data memory\n"); return -ENOENT; } - res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "dpfe-imem"); - priv->imem = devm_ioremap_resource(dev, res); + priv->imem = devm_platform_ioremap_resource_byname(pdev, "dpfe-imem"); if (IS_ERR(priv->imem)) { dev_err(dev, "Couldn't map DCPU instruction memory\n"); return -ENOENT; @@ -889,11 +897,8 @@ static int brcmstb_dpfe_probe(struct platform_device *pdev) } ret = brcmstb_dpfe_download_firmware(priv); - if (ret) { - if (ret != -EPROBE_DEFER) - dev_err(dev, "Couldn't download firmware -- %d\n", ret); - return ret; - } + if (ret) + return dev_err_probe(dev, ret, "Couldn't download firmware\n"); ret = sysfs_create_groups(&pdev->dev.kobj, priv->dpfe_api->sysfs_attrs); if (!ret) diff --git a/drivers/memory/brcmstb_memc.c b/drivers/memory/brcmstb_memc.c new file mode 100644 index 000000000000..233a53f5bce1 --- /dev/null +++ b/drivers/memory/brcmstb_memc.c @@ -0,0 +1,301 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * DDR Self-Refresh Power Down (SRPD) support for Broadcom STB SoCs + * + */ + +#include <linux/init.h> +#include <linux/io.h> +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/of_device.h> +#include <linux/platform_device.h> + +#define REG_MEMC_CNTRLR_CONFIG 0x00 +#define CNTRLR_CONFIG_LPDDR4_SHIFT 5 +#define CNTRLR_CONFIG_MASK 0xf +#define REG_MEMC_SRPD_CFG_21 0x20 +#define REG_MEMC_SRPD_CFG_20 0x34 +#define REG_MEMC_SRPD_CFG_1x 0x3c +#define INACT_COUNT_SHIFT 0 +#define INACT_COUNT_MASK 0xffff +#define SRPD_EN_SHIFT 16 + +struct brcmstb_memc_data { + u32 srpd_offset; +}; + +struct brcmstb_memc { + struct device *dev; + void __iomem *ddr_ctrl; + unsigned int timeout_cycles; + u32 frequency; + u32 srpd_offset; +}; + +static int brcmstb_memc_uses_lpddr4(struct brcmstb_memc *memc) +{ + void __iomem *config = memc->ddr_ctrl + REG_MEMC_CNTRLR_CONFIG; + u32 reg; + + reg = readl_relaxed(config) & CNTRLR_CONFIG_MASK; + + return reg == CNTRLR_CONFIG_LPDDR4_SHIFT; +} + +static int brcmstb_memc_srpd_config(struct brcmstb_memc *memc, + unsigned int cycles) +{ + void __iomem *cfg = memc->ddr_ctrl + memc->srpd_offset; + u32 val; + + /* Max timeout supported in HW */ + if (cycles > INACT_COUNT_MASK) + return -EINVAL; + + memc->timeout_cycles = cycles; + + val = (cycles << INACT_COUNT_SHIFT) & INACT_COUNT_MASK; + if (cycles) + val |= BIT(SRPD_EN_SHIFT); + + writel_relaxed(val, cfg); + /* Ensure the write is committed to the controller */ + (void)readl_relaxed(cfg); + + return 0; +} + +static ssize_t frequency_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct brcmstb_memc *memc = dev_get_drvdata(dev); + + return sprintf(buf, "%d\n", memc->frequency); +} + +static ssize_t srpd_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct brcmstb_memc *memc = dev_get_drvdata(dev); + + return sprintf(buf, "%d\n", memc->timeout_cycles); +} + +static ssize_t srpd_store(struct device *dev, struct device_attribute *attr, + const char *buf, size_t count) +{ + struct brcmstb_memc *memc = dev_get_drvdata(dev); + unsigned int val; + int ret; + + /* + * Cannot change the inactivity timeout on LPDDR4 chips because the + * dynamic tuning process will also get affected by the inactivity + * timeout, thus making it non functional. + */ + if (brcmstb_memc_uses_lpddr4(memc)) + return -EOPNOTSUPP; + + ret = kstrtouint(buf, 10, &val); + if (ret < 0) + return ret; + + ret = brcmstb_memc_srpd_config(memc, val); + if (ret) + return ret; + + return count; +} + +static DEVICE_ATTR_RO(frequency); +static DEVICE_ATTR_RW(srpd); + +static struct attribute *dev_attrs[] = { + &dev_attr_frequency.attr, + &dev_attr_srpd.attr, + NULL, +}; + +static struct attribute_group dev_attr_group = { + .attrs = dev_attrs, +}; + +static const struct of_device_id brcmstb_memc_of_match[]; + +static int brcmstb_memc_probe(struct platform_device *pdev) +{ + const struct brcmstb_memc_data *memc_data; + const struct of_device_id *of_id; + struct device *dev = &pdev->dev; + struct brcmstb_memc *memc; + int ret; + + memc = devm_kzalloc(dev, sizeof(*memc), GFP_KERNEL); + if (!memc) + return -ENOMEM; + + dev_set_drvdata(dev, memc); + + of_id = of_match_device(brcmstb_memc_of_match, dev); + memc_data = of_id->data; + memc->srpd_offset = memc_data->srpd_offset; + + memc->ddr_ctrl = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(memc->ddr_ctrl)) + return PTR_ERR(memc->ddr_ctrl); + + of_property_read_u32(pdev->dev.of_node, "clock-frequency", + &memc->frequency); + + ret = sysfs_create_group(&dev->kobj, &dev_attr_group); + if (ret) + return ret; + + return 0; +} + +static int brcmstb_memc_remove(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + + sysfs_remove_group(&dev->kobj, &dev_attr_group); + + return 0; +} + +enum brcmstb_memc_hwtype { + BRCMSTB_MEMC_V21, + BRCMSTB_MEMC_V20, + BRCMSTB_MEMC_V1X, +}; + +static const struct brcmstb_memc_data brcmstb_memc_versions[] = { + { .srpd_offset = REG_MEMC_SRPD_CFG_21 }, + { .srpd_offset = REG_MEMC_SRPD_CFG_20 }, + { .srpd_offset = REG_MEMC_SRPD_CFG_1x }, +}; + +static const struct of_device_id brcmstb_memc_of_match[] = { + { + .compatible = "brcm,brcmstb-memc-ddr-rev-b.1.x", + .data = &brcmstb_memc_versions[BRCMSTB_MEMC_V1X] + }, + { + .compatible = "brcm,brcmstb-memc-ddr-rev-b.2.0", + .data = &brcmstb_memc_versions[BRCMSTB_MEMC_V20] + }, + { + .compatible = "brcm,brcmstb-memc-ddr-rev-b.2.1", + .data = &brcmstb_memc_versions[BRCMSTB_MEMC_V21] + }, + { + .compatible = "brcm,brcmstb-memc-ddr-rev-b.2.2", + .data = &brcmstb_memc_versions[BRCMSTB_MEMC_V21] + }, + { + .compatible = "brcm,brcmstb-memc-ddr-rev-b.2.3", + .data = &brcmstb_memc_versions[BRCMSTB_MEMC_V21] + }, + { + .compatible = "brcm,brcmstb-memc-ddr-rev-b.2.5", + .data = &brcmstb_memc_versions[BRCMSTB_MEMC_V21] + }, + { + .compatible = "brcm,brcmstb-memc-ddr-rev-b.2.6", + .data = &brcmstb_memc_versions[BRCMSTB_MEMC_V21] + }, + { + .compatible = "brcm,brcmstb-memc-ddr-rev-b.2.7", + .data = &brcmstb_memc_versions[BRCMSTB_MEMC_V21] + }, + { + .compatible = "brcm,brcmstb-memc-ddr-rev-b.2.8", + .data = &brcmstb_memc_versions[BRCMSTB_MEMC_V21] + }, + { + .compatible = "brcm,brcmstb-memc-ddr-rev-b.3.0", + .data = &brcmstb_memc_versions[BRCMSTB_MEMC_V21] + }, + { + .compatible = "brcm,brcmstb-memc-ddr-rev-b.3.1", + .data = &brcmstb_memc_versions[BRCMSTB_MEMC_V21] + }, + { + .compatible = "brcm,brcmstb-memc-ddr-rev-c.1.0", + .data = &brcmstb_memc_versions[BRCMSTB_MEMC_V21] + }, + { + .compatible = "brcm,brcmstb-memc-ddr-rev-c.1.1", + .data = &brcmstb_memc_versions[BRCMSTB_MEMC_V21] + }, + { + .compatible = "brcm,brcmstb-memc-ddr-rev-c.1.2", + .data = &brcmstb_memc_versions[BRCMSTB_MEMC_V21] + }, + { + .compatible = "brcm,brcmstb-memc-ddr-rev-c.1.3", + .data = &brcmstb_memc_versions[BRCMSTB_MEMC_V21] + }, + { + .compatible = "brcm,brcmstb-memc-ddr-rev-c.1.4", + .data = &brcmstb_memc_versions[BRCMSTB_MEMC_V21] + }, + /* default to the original offset */ + { + .compatible = "brcm,brcmstb-memc-ddr", + .data = &brcmstb_memc_versions[BRCMSTB_MEMC_V1X] + }, + {} +}; + +static int brcmstb_memc_suspend(struct device *dev) +{ + struct brcmstb_memc *memc = dev_get_drvdata(dev); + void __iomem *cfg = memc->ddr_ctrl + memc->srpd_offset; + u32 val; + + if (memc->timeout_cycles == 0) + return 0; + + /* + * Disable SRPD prior to suspending the system since that can + * cause issues with other memory clients managed by the ARM + * trusted firmware to access memory. + */ + val = readl_relaxed(cfg); + val &= ~BIT(SRPD_EN_SHIFT); + writel_relaxed(val, cfg); + /* Ensure the write is committed to the controller */ + (void)readl_relaxed(cfg); + + return 0; +} + +static int brcmstb_memc_resume(struct device *dev) +{ + struct brcmstb_memc *memc = dev_get_drvdata(dev); + + if (memc->timeout_cycles == 0) + return 0; + + return brcmstb_memc_srpd_config(memc, memc->timeout_cycles); +} + +static DEFINE_SIMPLE_DEV_PM_OPS(brcmstb_memc_pm_ops, brcmstb_memc_suspend, + brcmstb_memc_resume); + +static struct platform_driver brcmstb_memc_driver = { + .probe = brcmstb_memc_probe, + .remove = brcmstb_memc_remove, + .driver = { + .name = "brcmstb_memc", + .of_match_table = brcmstb_memc_of_match, + .pm = pm_ptr(&brcmstb_memc_pm_ops), + }, +}; +module_platform_driver(brcmstb_memc_driver); + +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("Broadcom"); +MODULE_DESCRIPTION("DDR SRPD driver for Broadcom STB chips"); diff --git a/drivers/memory/bt1-l2-ctl.c b/drivers/memory/bt1-l2-ctl.c new file mode 100644 index 000000000000..85965fa26e0b --- /dev/null +++ b/drivers/memory/bt1-l2-ctl.c @@ -0,0 +1,324 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2020 BAIKAL ELECTRONICS, JSC + * + * Authors: + * Serge Semin <Sergey.Semin@baikalelectronics.ru> + * + * Baikal-T1 CM2 L2-cache Control Block driver. + */ + +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/bitfield.h> +#include <linux/types.h> +#include <linux/device.h> +#include <linux/platform_device.h> +#include <linux/regmap.h> +#include <linux/mfd/syscon.h> +#include <linux/sysfs.h> +#include <linux/of.h> + +#define L2_CTL_REG 0x028 +#define L2_CTL_DATA_STALL_FLD 0 +#define L2_CTL_DATA_STALL_MASK GENMASK(1, L2_CTL_DATA_STALL_FLD) +#define L2_CTL_TAG_STALL_FLD 2 +#define L2_CTL_TAG_STALL_MASK GENMASK(3, L2_CTL_TAG_STALL_FLD) +#define L2_CTL_WS_STALL_FLD 4 +#define L2_CTL_WS_STALL_MASK GENMASK(5, L2_CTL_WS_STALL_FLD) +#define L2_CTL_SET_CLKRATIO BIT(13) +#define L2_CTL_CLKRATIO_LOCK BIT(31) + +#define L2_CTL_STALL_MIN 0 +#define L2_CTL_STALL_MAX 3 +#define L2_CTL_STALL_SET_DELAY_US 1 +#define L2_CTL_STALL_SET_TOUT_US 1000 + +/* + * struct l2_ctl - Baikal-T1 L2 Control block private data. + * @dev: Pointer to the device structure. + * @sys_regs: Baikal-T1 System Controller registers map. + */ +struct l2_ctl { + struct device *dev; + + struct regmap *sys_regs; +}; + +/* + * enum l2_ctl_stall - Baikal-T1 L2-cache-RAM stall identifier. + * @L2_WSSTALL: Way-select latency. + * @L2_TAGSTALL: Tag latency. + * @L2_DATASTALL: Data latency. + */ +enum l2_ctl_stall { + L2_WS_STALL, + L2_TAG_STALL, + L2_DATA_STALL +}; + +/* + * struct l2_ctl_device_attribute - Baikal-T1 L2-cache device attribute. + * @dev_attr: Actual sysfs device attribute. + * @id: L2-cache stall field identifier. + */ +struct l2_ctl_device_attribute { + struct device_attribute dev_attr; + enum l2_ctl_stall id; +}; + +#define to_l2_ctl_dev_attr(_dev_attr) \ + container_of(_dev_attr, struct l2_ctl_device_attribute, dev_attr) + +#define L2_CTL_ATTR_RW(_name, _prefix, _id) \ + struct l2_ctl_device_attribute l2_ctl_attr_##_name = \ + { __ATTR(_name, 0644, _prefix##_show, _prefix##_store), _id } + +static int l2_ctl_get_latency(struct l2_ctl *l2, enum l2_ctl_stall id, u32 *val) +{ + u32 data = 0; + int ret; + + ret = regmap_read(l2->sys_regs, L2_CTL_REG, &data); + if (ret) + return ret; + + switch (id) { + case L2_WS_STALL: + *val = FIELD_GET(L2_CTL_WS_STALL_MASK, data); + break; + case L2_TAG_STALL: + *val = FIELD_GET(L2_CTL_TAG_STALL_MASK, data); + break; + case L2_DATA_STALL: + *val = FIELD_GET(L2_CTL_DATA_STALL_MASK, data); + break; + default: + return -EINVAL; + } + + return 0; +} + +static int l2_ctl_set_latency(struct l2_ctl *l2, enum l2_ctl_stall id, u32 val) +{ + u32 mask = 0, data = 0; + int ret; + + val = clamp_val(val, L2_CTL_STALL_MIN, L2_CTL_STALL_MAX); + + switch (id) { + case L2_WS_STALL: + data = FIELD_PREP(L2_CTL_WS_STALL_MASK, val); + mask = L2_CTL_WS_STALL_MASK; + break; + case L2_TAG_STALL: + data = FIELD_PREP(L2_CTL_TAG_STALL_MASK, val); + mask = L2_CTL_TAG_STALL_MASK; + break; + case L2_DATA_STALL: + data = FIELD_PREP(L2_CTL_DATA_STALL_MASK, val); + mask = L2_CTL_DATA_STALL_MASK; + break; + default: + return -EINVAL; + } + + data |= L2_CTL_SET_CLKRATIO; + mask |= L2_CTL_SET_CLKRATIO; + + ret = regmap_update_bits(l2->sys_regs, L2_CTL_REG, mask, data); + if (ret) + return ret; + + return regmap_read_poll_timeout(l2->sys_regs, L2_CTL_REG, data, + data & L2_CTL_CLKRATIO_LOCK, + L2_CTL_STALL_SET_DELAY_US, + L2_CTL_STALL_SET_TOUT_US); +} + +static void l2_ctl_clear_data(void *data) +{ + struct l2_ctl *l2 = data; + struct platform_device *pdev = to_platform_device(l2->dev); + + platform_set_drvdata(pdev, NULL); +} + +static struct l2_ctl *l2_ctl_create_data(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct l2_ctl *l2; + int ret; + + l2 = devm_kzalloc(dev, sizeof(*l2), GFP_KERNEL); + if (!l2) + return ERR_PTR(-ENOMEM); + + ret = devm_add_action(dev, l2_ctl_clear_data, l2); + if (ret) { + dev_err(dev, "Can't add L2 CTL data clear action\n"); + return ERR_PTR(ret); + } + + l2->dev = dev; + platform_set_drvdata(pdev, l2); + + return l2; +} + +static int l2_ctl_find_sys_regs(struct l2_ctl *l2) +{ + l2->sys_regs = syscon_node_to_regmap(l2->dev->of_node->parent); + if (IS_ERR(l2->sys_regs)) { + dev_err(l2->dev, "Couldn't get L2 CTL register map\n"); + return PTR_ERR(l2->sys_regs); + } + + return 0; +} + +static int l2_ctl_of_parse_property(struct l2_ctl *l2, enum l2_ctl_stall id, + const char *propname) +{ + int ret = 0; + u32 data; + + if (!of_property_read_u32(l2->dev->of_node, propname, &data)) { + ret = l2_ctl_set_latency(l2, id, data); + if (ret) + dev_err(l2->dev, "Invalid value of '%s'\n", propname); + } + + return ret; +} + +static int l2_ctl_of_parse(struct l2_ctl *l2) +{ + int ret; + + ret = l2_ctl_of_parse_property(l2, L2_WS_STALL, "baikal,l2-ws-latency"); + if (ret) + return ret; + + ret = l2_ctl_of_parse_property(l2, L2_TAG_STALL, "baikal,l2-tag-latency"); + if (ret) + return ret; + + return l2_ctl_of_parse_property(l2, L2_DATA_STALL, + "baikal,l2-data-latency"); +} + +static ssize_t l2_ctl_latency_show(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct l2_ctl_device_attribute *devattr = to_l2_ctl_dev_attr(attr); + struct l2_ctl *l2 = dev_get_drvdata(dev); + u32 data; + int ret; + + ret = l2_ctl_get_latency(l2, devattr->id, &data); + if (ret) + return ret; + + return scnprintf(buf, PAGE_SIZE, "%u\n", data); +} + +static ssize_t l2_ctl_latency_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count) +{ + struct l2_ctl_device_attribute *devattr = to_l2_ctl_dev_attr(attr); + struct l2_ctl *l2 = dev_get_drvdata(dev); + u32 data; + int ret; + + if (kstrtouint(buf, 0, &data) < 0) + return -EINVAL; + + ret = l2_ctl_set_latency(l2, devattr->id, data); + if (ret) + return ret; + + return count; +} + +static L2_CTL_ATTR_RW(l2_ws_latency, l2_ctl_latency, L2_WS_STALL); +static L2_CTL_ATTR_RW(l2_tag_latency, l2_ctl_latency, L2_TAG_STALL); +static L2_CTL_ATTR_RW(l2_data_latency, l2_ctl_latency, L2_DATA_STALL); + +static struct attribute *l2_ctl_sysfs_attrs[] = { + &l2_ctl_attr_l2_ws_latency.dev_attr.attr, + &l2_ctl_attr_l2_tag_latency.dev_attr.attr, + &l2_ctl_attr_l2_data_latency.dev_attr.attr, + NULL +}; +ATTRIBUTE_GROUPS(l2_ctl_sysfs); + +static void l2_ctl_remove_sysfs(void *data) +{ + struct l2_ctl *l2 = data; + + device_remove_groups(l2->dev, l2_ctl_sysfs_groups); +} + +static int l2_ctl_init_sysfs(struct l2_ctl *l2) +{ + int ret; + + ret = device_add_groups(l2->dev, l2_ctl_sysfs_groups); + if (ret) { + dev_err(l2->dev, "Failed to create L2 CTL sysfs nodes\n"); + return ret; + } + + ret = devm_add_action_or_reset(l2->dev, l2_ctl_remove_sysfs, l2); + if (ret) + dev_err(l2->dev, "Can't add L2 CTL sysfs remove action\n"); + + return ret; +} + +static int l2_ctl_probe(struct platform_device *pdev) +{ + struct l2_ctl *l2; + int ret; + + l2 = l2_ctl_create_data(pdev); + if (IS_ERR(l2)) + return PTR_ERR(l2); + + ret = l2_ctl_find_sys_regs(l2); + if (ret) + return ret; + + ret = l2_ctl_of_parse(l2); + if (ret) + return ret; + + ret = l2_ctl_init_sysfs(l2); + if (ret) + return ret; + + return 0; +} + +static const struct of_device_id l2_ctl_of_match[] = { + { .compatible = "baikal,bt1-l2-ctl" }, + { } +}; +MODULE_DEVICE_TABLE(of, l2_ctl_of_match); + +static struct platform_driver l2_ctl_driver = { + .probe = l2_ctl_probe, + .driver = { + .name = "bt1-l2-ctl", + .of_match_table = l2_ctl_of_match + } +}; +module_platform_driver(l2_ctl_driver); + +MODULE_AUTHOR("Serge Semin <Sergey.Semin@baikalelectronics.ru>"); +MODULE_DESCRIPTION("Baikal-T1 L2-cache driver"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/memory/da8xx-ddrctl.c b/drivers/memory/da8xx-ddrctl.c index e8f9b3f461f5..b32005bf269c 100644 --- a/drivers/memory/da8xx-ddrctl.c +++ b/drivers/memory/da8xx-ddrctl.c @@ -102,14 +102,12 @@ static int da8xx_ddrctl_probe(struct platform_device *pdev) { const struct da8xx_ddrctl_config_knob *knob; const struct da8xx_ddrctl_setting *setting; - struct device_node *node; struct resource *res; void __iomem *ddrctl; struct device *dev; u32 reg; dev = &pdev->dev; - node = dev->of_node; setting = da8xx_ddrctl_get_board_settings(); if (!setting) { @@ -117,8 +115,7 @@ static int da8xx_ddrctl_probe(struct platform_device *pdev) return -EINVAL; } - res = platform_get_resource(pdev, IORESOURCE_MEM, 0); - ddrctl = devm_ioremap_resource(dev, res); + ddrctl = devm_platform_get_and_ioremap_resource(pdev, 0, &res); if (IS_ERR(ddrctl)) { dev_err(dev, "unable to map memory controller registers\n"); return PTR_ERR(ddrctl); diff --git a/drivers/memory/dfl-emif.c b/drivers/memory/dfl-emif.c new file mode 100644 index 000000000000..da06cd30a016 --- /dev/null +++ b/drivers/memory/dfl-emif.c @@ -0,0 +1,259 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * DFL device driver for EMIF private feature + * + * Copyright (C) 2020 Intel Corporation, Inc. + * + */ +#include <linux/bitfield.h> +#include <linux/dfl.h> +#include <linux/errno.h> +#include <linux/io.h> +#include <linux/iopoll.h> +#include <linux/io-64-nonatomic-lo-hi.h> +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/spinlock.h> +#include <linux/types.h> + +#define FME_FEATURE_ID_EMIF 0x9 + +#define EMIF_STAT 0x8 +#define EMIF_STAT_INIT_DONE_SFT 0 +#define EMIF_STAT_CALC_FAIL_SFT 8 +#define EMIF_STAT_CLEAR_BUSY_SFT 16 +#define EMIF_CTRL 0x10 +#define EMIF_CTRL_CLEAR_EN_SFT 0 +#define EMIF_CTRL_CLEAR_EN_MSK GENMASK_ULL(7, 0) + +#define EMIF_POLL_INVL 10000 /* us */ +#define EMIF_POLL_TIMEOUT 5000000 /* us */ + +/* + * The Capability Register replaces the Control Register (at the same + * offset) for EMIF feature revisions > 0. The bitmask that indicates + * the presence of memory channels exists in both the Capability Register + * and Control Register definitions. These can be thought of as a C union. + * The Capability Register definitions are used to check for the existence + * of a memory channel, and the Control Register definitions are used for + * managing the memory-clear functionality in revision 0. + */ +#define EMIF_CAPABILITY_BASE 0x10 +#define EMIF_CAPABILITY_CHN_MSK_V0 GENMASK_ULL(3, 0) +#define EMIF_CAPABILITY_CHN_MSK GENMASK_ULL(7, 0) + +struct dfl_emif { + struct device *dev; + void __iomem *base; + spinlock_t lock; /* Serialises access to EMIF_CTRL reg */ +}; + +struct emif_attr { + struct device_attribute attr; + u32 shift; + u32 index; +}; + +#define to_emif_attr(dev_attr) \ + container_of(dev_attr, struct emif_attr, attr) + +static ssize_t emif_state_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct emif_attr *eattr = to_emif_attr(attr); + struct dfl_emif *de = dev_get_drvdata(dev); + u64 val; + + val = readq(de->base + EMIF_STAT); + + return sysfs_emit(buf, "%u\n", + !!(val & BIT_ULL(eattr->shift + eattr->index))); +} + +static ssize_t emif_clear_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count) +{ + struct emif_attr *eattr = to_emif_attr(attr); + struct dfl_emif *de = dev_get_drvdata(dev); + u64 clear_busy_msk, clear_en_msk, val; + void __iomem *base = de->base; + + if (!sysfs_streq(buf, "1")) + return -EINVAL; + + clear_busy_msk = BIT_ULL(EMIF_STAT_CLEAR_BUSY_SFT + eattr->index); + clear_en_msk = BIT_ULL(EMIF_CTRL_CLEAR_EN_SFT + eattr->index); + + spin_lock(&de->lock); + /* The CLEAR_EN field is WO, but other fields are RW */ + val = readq(base + EMIF_CTRL); + val &= ~EMIF_CTRL_CLEAR_EN_MSK; + val |= clear_en_msk; + writeq(val, base + EMIF_CTRL); + spin_unlock(&de->lock); + + if (readq_poll_timeout(base + EMIF_STAT, val, + !(val & clear_busy_msk), + EMIF_POLL_INVL, EMIF_POLL_TIMEOUT)) { + dev_err(de->dev, "timeout, fail to clear\n"); + return -ETIMEDOUT; + } + + return count; +} + +#define emif_state_attr(_name, _shift, _index) \ + static struct emif_attr emif_attr_##inf##_index##_##_name = \ + { .attr = __ATTR(inf##_index##_##_name, 0444, \ + emif_state_show, NULL), \ + .shift = (_shift), .index = (_index) } + +#define emif_clear_attr(_index) \ + static struct emif_attr emif_attr_##inf##_index##_clear = \ + { .attr = __ATTR(inf##_index##_clear, 0200, \ + NULL, emif_clear_store), \ + .index = (_index) } + +emif_state_attr(init_done, EMIF_STAT_INIT_DONE_SFT, 0); +emif_state_attr(init_done, EMIF_STAT_INIT_DONE_SFT, 1); +emif_state_attr(init_done, EMIF_STAT_INIT_DONE_SFT, 2); +emif_state_attr(init_done, EMIF_STAT_INIT_DONE_SFT, 3); +emif_state_attr(init_done, EMIF_STAT_INIT_DONE_SFT, 4); +emif_state_attr(init_done, EMIF_STAT_INIT_DONE_SFT, 5); +emif_state_attr(init_done, EMIF_STAT_INIT_DONE_SFT, 6); +emif_state_attr(init_done, EMIF_STAT_INIT_DONE_SFT, 7); + +emif_state_attr(cal_fail, EMIF_STAT_CALC_FAIL_SFT, 0); +emif_state_attr(cal_fail, EMIF_STAT_CALC_FAIL_SFT, 1); +emif_state_attr(cal_fail, EMIF_STAT_CALC_FAIL_SFT, 2); +emif_state_attr(cal_fail, EMIF_STAT_CALC_FAIL_SFT, 3); +emif_state_attr(cal_fail, EMIF_STAT_CALC_FAIL_SFT, 4); +emif_state_attr(cal_fail, EMIF_STAT_CALC_FAIL_SFT, 5); +emif_state_attr(cal_fail, EMIF_STAT_CALC_FAIL_SFT, 6); +emif_state_attr(cal_fail, EMIF_STAT_CALC_FAIL_SFT, 7); + + +emif_clear_attr(0); +emif_clear_attr(1); +emif_clear_attr(2); +emif_clear_attr(3); +emif_clear_attr(4); +emif_clear_attr(5); +emif_clear_attr(6); +emif_clear_attr(7); + + +static struct attribute *dfl_emif_attrs[] = { + &emif_attr_inf0_init_done.attr.attr, + &emif_attr_inf0_cal_fail.attr.attr, + &emif_attr_inf0_clear.attr.attr, + + &emif_attr_inf1_init_done.attr.attr, + &emif_attr_inf1_cal_fail.attr.attr, + &emif_attr_inf1_clear.attr.attr, + + &emif_attr_inf2_init_done.attr.attr, + &emif_attr_inf2_cal_fail.attr.attr, + &emif_attr_inf2_clear.attr.attr, + + &emif_attr_inf3_init_done.attr.attr, + &emif_attr_inf3_cal_fail.attr.attr, + &emif_attr_inf3_clear.attr.attr, + + &emif_attr_inf4_init_done.attr.attr, + &emif_attr_inf4_cal_fail.attr.attr, + &emif_attr_inf4_clear.attr.attr, + + &emif_attr_inf5_init_done.attr.attr, + &emif_attr_inf5_cal_fail.attr.attr, + &emif_attr_inf5_clear.attr.attr, + + &emif_attr_inf6_init_done.attr.attr, + &emif_attr_inf6_cal_fail.attr.attr, + &emif_attr_inf6_clear.attr.attr, + + &emif_attr_inf7_init_done.attr.attr, + &emif_attr_inf7_cal_fail.attr.attr, + &emif_attr_inf7_clear.attr.attr, + + NULL, +}; + +static umode_t dfl_emif_visible(struct kobject *kobj, + struct attribute *attr, int n) +{ + struct dfl_emif *de = dev_get_drvdata(kobj_to_dev(kobj)); + struct emif_attr *eattr = container_of(attr, struct emif_attr, + attr.attr); + struct dfl_device *ddev = to_dfl_dev(de->dev); + u64 val; + + /* + * This device supports up to 8 memory interfaces, but not all + * interfaces are used on different platforms. The read out value of + * CAPABILITY_CHN_MSK field (which is a bitmap) indicates which + * interfaces are available. + */ + if (ddev->revision > 0 && strstr(attr->name, "_clear")) + return 0; + + if (ddev->revision == 0) + val = FIELD_GET(EMIF_CAPABILITY_CHN_MSK_V0, + readq(de->base + EMIF_CAPABILITY_BASE)); + else + val = FIELD_GET(EMIF_CAPABILITY_CHN_MSK, + readq(de->base + EMIF_CAPABILITY_BASE)); + + return (val & BIT_ULL(eattr->index)) ? attr->mode : 0; +} + +static const struct attribute_group dfl_emif_group = { + .is_visible = dfl_emif_visible, + .attrs = dfl_emif_attrs, +}; + +static const struct attribute_group *dfl_emif_groups[] = { + &dfl_emif_group, + NULL, +}; + +static int dfl_emif_probe(struct dfl_device *ddev) +{ + struct device *dev = &ddev->dev; + struct dfl_emif *de; + + de = devm_kzalloc(dev, sizeof(*de), GFP_KERNEL); + if (!de) + return -ENOMEM; + + de->base = devm_ioremap_resource(dev, &ddev->mmio_res); + if (IS_ERR(de->base)) + return PTR_ERR(de->base); + + de->dev = dev; + spin_lock_init(&de->lock); + dev_set_drvdata(dev, de); + + return 0; +} + +static const struct dfl_device_id dfl_emif_ids[] = { + { FME_ID, FME_FEATURE_ID_EMIF }, + { } +}; +MODULE_DEVICE_TABLE(dfl, dfl_emif_ids); + +static struct dfl_driver dfl_emif_driver = { + .drv = { + .name = "dfl-emif", + .dev_groups = dfl_emif_groups, + }, + .id_table = dfl_emif_ids, + .probe = dfl_emif_probe, +}; +module_dfl_driver(dfl_emif_driver); + +MODULE_DESCRIPTION("DFL EMIF driver"); +MODULE_AUTHOR("Intel Corporation"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/memory/emif-asm-offsets.c b/drivers/memory/emif-asm-offsets.c index db8043019ec6..4b98d1854cd7 100644 --- a/drivers/memory/emif-asm-offsets.c +++ b/drivers/memory/emif-asm-offsets.c @@ -1,16 +1,8 @@ +// SPDX-License-Identifier: GPL-2.0 /* * TI AM33XX EMIF PM Assembly Offsets * * Copyright (C) 2016-2017 Texas Instruments Inc. - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License as - * published by the Free Software Foundation version 2. - * - * This program is distributed "as is" WITHOUT ANY WARRANTY of any - * kind, whether express or implied; without even the implied warranty - * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. */ #include <linux/ti-emif-sram.h> diff --git a/drivers/memory/emif.c b/drivers/memory/emif.c index 9d9127bf2a59..f305643209f0 100644 --- a/drivers/memory/emif.c +++ b/drivers/memory/emif.c @@ -41,7 +41,6 @@ * @node: node in the device list * @base: base address of memory-mapped IO registers. * @dev: device pointer. - * @addressing table with addressing information from the spec * @regs_cache: An array of 'struct emif_regs' that stores * calculated register values for different * frequencies, to avoid re-calculating them on @@ -61,7 +60,6 @@ struct emif_data { unsigned long irq_state; void __iomem *base; struct device *dev; - const struct lpddr2_addressing *addressing; struct emif_regs *regs_cache[EMIF_MAX_NUM_FREQUENCIES]; struct emif_regs *curr_regs; struct emif_platform_data *plat_data; @@ -70,9 +68,8 @@ struct emif_data { }; static struct emif_data *emif1; -static spinlock_t emif_lock; +static DEFINE_SPINLOCK(emif_lock); static unsigned long irq_state; -static u32 t_ck; /* DDR clock period in ps */ static LIST_HEAD(device_list); #ifdef CONFIG_DEBUG_FS @@ -131,16 +128,7 @@ static int emif_regdump_show(struct seq_file *s, void *unused) return 0; } -static int emif_regdump_open(struct inode *inode, struct file *file) -{ - return single_open(file, emif_regdump_show, inode->i_private); -} - -static const struct file_operations emif_regdump_fops = { - .open = emif_regdump_open, - .read = seq_read, - .release = single_release, -}; +DEFINE_SHOW_ATTRIBUTE(emif_regdump); static int emif_mr4_show(struct seq_file *s, void *unused) { @@ -150,48 +138,16 @@ static int emif_mr4_show(struct seq_file *s, void *unused) return 0; } -static int emif_mr4_open(struct inode *inode, struct file *file) -{ - return single_open(file, emif_mr4_show, inode->i_private); -} - -static const struct file_operations emif_mr4_fops = { - .open = emif_mr4_open, - .read = seq_read, - .release = single_release, -}; +DEFINE_SHOW_ATTRIBUTE(emif_mr4); static int __init_or_module emif_debugfs_init(struct emif_data *emif) { - struct dentry *dentry; - int ret; - - dentry = debugfs_create_dir(dev_name(emif->dev), NULL); - if (!dentry) { - ret = -ENOMEM; - goto err0; - } - emif->debugfs_root = dentry; - - dentry = debugfs_create_file("regcache_dump", S_IRUGO, - emif->debugfs_root, emif, &emif_regdump_fops); - if (!dentry) { - ret = -ENOMEM; - goto err1; - } - - dentry = debugfs_create_file("mr4", S_IRUGO, - emif->debugfs_root, emif, &emif_mr4_fops); - if (!dentry) { - ret = -ENOMEM; - goto err1; - } - + emif->debugfs_root = debugfs_create_dir(dev_name(emif->dev), NULL); + debugfs_create_file("regcache_dump", S_IRUGO, emif->debugfs_root, emif, + &emif_regdump_fops); + debugfs_create_file("mr4", S_IRUGO, emif->debugfs_root, emif, + &emif_mr4_fops); return 0; -err1: - debugfs_remove_recursive(emif->debugfs_root); -err0: - return ret; } static void __exit emif_debugfs_exit(struct emif_data *emif) @@ -211,15 +167,6 @@ static inline void __exit emif_debugfs_exit(struct emif_data *emif) #endif /* - * Calculate the period of DDR clock from frequency value - */ -static void set_ddr_clk_period(u32 freq) -{ - /* Divide 10^12 by frequency to get period in ps */ - t_ck = (u32)DIV_ROUND_UP_ULL(1000000000000ull, freq); -} - -/* * Get bus width used by EMIF. Note that this may be different from the * bus width of the DDR devices used. For instance two 16-bit DDR devices * may be connected to a given CS of EMIF. In this case bus width as far @@ -237,19 +184,6 @@ static u32 get_emif_bus_width(struct emif_data *emif) return width; } -/* - * Get the CL from SDRAM_CONFIG register - */ -static u32 get_cl(struct emif_data *emif) -{ - u32 cl; - void __iomem *base = emif->base; - - cl = (readl(base + EMIF_SDRAM_CONFIG) & CL_MASK) >> CL_SHIFT; - - return cl; -} - static void set_lpmode(struct emif_data *emif, u8 lpmode) { u32 temp; @@ -282,10 +216,9 @@ static void set_lpmode(struct emif_data *emif, u8 lpmode) * the EMIF_PWR_MGMT_CTRL[10:8] REG_LP_MODE bit field to 0x4. */ if ((emif->plat_data->ip_rev == EMIF_4D) && - (EMIF_LP_MODE_PWR_DN == lpmode)) { + (lpmode == EMIF_LP_MODE_PWR_DN)) { WARN_ONCE(1, - "REG_LP_MODE = LP_MODE_PWR_DN(4) is prohibited by" - "erratum i743 switch to LP_MODE_SELF_REFRESH(2)\n"); + "REG_LP_MODE = LP_MODE_PWR_DN(4) is prohibited by erratum i743 switch to LP_MODE_SELF_REFRESH(2)\n"); /* rollback LP_MODE to Self-refresh mode */ lpmode = EMIF_LP_MODE_SELF_REFRESH; } @@ -370,203 +303,6 @@ static const struct lpddr2_addressing *get_addressing_table( return &lpddr2_jedec_addressing_table[index]; } -/* - * Find the the right timing table from the array of timing - * tables of the device using DDR clock frequency - */ -static const struct lpddr2_timings *get_timings_table(struct emif_data *emif, - u32 freq) -{ - u32 i, min, max, freq_nearest; - const struct lpddr2_timings *timings = NULL; - const struct lpddr2_timings *timings_arr = emif->plat_data->timings; - struct device *dev = emif->dev; - - /* Start with a very high frequency - 1GHz */ - freq_nearest = 1000000000; - - /* - * Find the timings table such that: - * 1. the frequency range covers the required frequency(safe) AND - * 2. the max_freq is closest to the required frequency(optimal) - */ - for (i = 0; i < emif->plat_data->timings_arr_size; i++) { - max = timings_arr[i].max_freq; - min = timings_arr[i].min_freq; - if ((freq >= min) && (freq <= max) && (max < freq_nearest)) { - freq_nearest = max; - timings = &timings_arr[i]; - } - } - - if (!timings) - dev_err(dev, "%s: couldn't find timings for - %dHz\n", - __func__, freq); - - dev_dbg(dev, "%s: timings table: freq %d, speed bin freq %d\n", - __func__, freq, freq_nearest); - - return timings; -} - -static u32 get_sdram_ref_ctrl_shdw(u32 freq, - const struct lpddr2_addressing *addressing) -{ - u32 ref_ctrl_shdw = 0, val = 0, freq_khz, t_refi; - - /* Scale down frequency and t_refi to avoid overflow */ - freq_khz = freq / 1000; - t_refi = addressing->tREFI_ns / 100; - - /* - * refresh rate to be set is 'tREFI(in us) * freq in MHz - * division by 10000 to account for change in units - */ - val = t_refi * freq_khz / 10000; - ref_ctrl_shdw |= val << REFRESH_RATE_SHIFT; - - return ref_ctrl_shdw; -} - -static u32 get_sdram_tim_1_shdw(const struct lpddr2_timings *timings, - const struct lpddr2_min_tck *min_tck, - const struct lpddr2_addressing *addressing) -{ - u32 tim1 = 0, val = 0; - - val = max(min_tck->tWTR, DIV_ROUND_UP(timings->tWTR, t_ck)) - 1; - tim1 |= val << T_WTR_SHIFT; - - if (addressing->num_banks == B8) - val = DIV_ROUND_UP(timings->tFAW, t_ck*4); - else - val = max(min_tck->tRRD, DIV_ROUND_UP(timings->tRRD, t_ck)); - tim1 |= (val - 1) << T_RRD_SHIFT; - - val = DIV_ROUND_UP(timings->tRAS_min + timings->tRPab, t_ck) - 1; - tim1 |= val << T_RC_SHIFT; - - val = max(min_tck->tRASmin, DIV_ROUND_UP(timings->tRAS_min, t_ck)); - tim1 |= (val - 1) << T_RAS_SHIFT; - - val = max(min_tck->tWR, DIV_ROUND_UP(timings->tWR, t_ck)) - 1; - tim1 |= val << T_WR_SHIFT; - - val = max(min_tck->tRCD, DIV_ROUND_UP(timings->tRCD, t_ck)) - 1; - tim1 |= val << T_RCD_SHIFT; - - val = max(min_tck->tRPab, DIV_ROUND_UP(timings->tRPab, t_ck)) - 1; - tim1 |= val << T_RP_SHIFT; - - return tim1; -} - -static u32 get_sdram_tim_1_shdw_derated(const struct lpddr2_timings *timings, - const struct lpddr2_min_tck *min_tck, - const struct lpddr2_addressing *addressing) -{ - u32 tim1 = 0, val = 0; - - val = max(min_tck->tWTR, DIV_ROUND_UP(timings->tWTR, t_ck)) - 1; - tim1 = val << T_WTR_SHIFT; - - /* - * tFAW is approximately 4 times tRRD. So add 1875*4 = 7500ps - * to tFAW for de-rating - */ - if (addressing->num_banks == B8) { - val = DIV_ROUND_UP(timings->tFAW + 7500, 4 * t_ck) - 1; - } else { - val = DIV_ROUND_UP(timings->tRRD + 1875, t_ck); - val = max(min_tck->tRRD, val) - 1; - } - tim1 |= val << T_RRD_SHIFT; - - val = DIV_ROUND_UP(timings->tRAS_min + timings->tRPab + 1875, t_ck); - tim1 |= (val - 1) << T_RC_SHIFT; - - val = DIV_ROUND_UP(timings->tRAS_min + 1875, t_ck); - val = max(min_tck->tRASmin, val) - 1; - tim1 |= val << T_RAS_SHIFT; - - val = max(min_tck->tWR, DIV_ROUND_UP(timings->tWR, t_ck)) - 1; - tim1 |= val << T_WR_SHIFT; - - val = max(min_tck->tRCD, DIV_ROUND_UP(timings->tRCD + 1875, t_ck)); - tim1 |= (val - 1) << T_RCD_SHIFT; - - val = max(min_tck->tRPab, DIV_ROUND_UP(timings->tRPab + 1875, t_ck)); - tim1 |= (val - 1) << T_RP_SHIFT; - - return tim1; -} - -static u32 get_sdram_tim_2_shdw(const struct lpddr2_timings *timings, - const struct lpddr2_min_tck *min_tck, - const struct lpddr2_addressing *addressing, - u32 type) -{ - u32 tim2 = 0, val = 0; - - val = min_tck->tCKE - 1; - tim2 |= val << T_CKE_SHIFT; - - val = max(min_tck->tRTP, DIV_ROUND_UP(timings->tRTP, t_ck)) - 1; - tim2 |= val << T_RTP_SHIFT; - - /* tXSNR = tRFCab_ps + 10 ns(tRFCab_ps for LPDDR2). */ - val = DIV_ROUND_UP(addressing->tRFCab_ps + 10000, t_ck) - 1; - tim2 |= val << T_XSNR_SHIFT; - - /* XSRD same as XSNR for LPDDR2 */ - tim2 |= val << T_XSRD_SHIFT; - - val = max(min_tck->tXP, DIV_ROUND_UP(timings->tXP, t_ck)) - 1; - tim2 |= val << T_XP_SHIFT; - - return tim2; -} - -static u32 get_sdram_tim_3_shdw(const struct lpddr2_timings *timings, - const struct lpddr2_min_tck *min_tck, - const struct lpddr2_addressing *addressing, - u32 type, u32 ip_rev, u32 derated) -{ - u32 tim3 = 0, val = 0, t_dqsck; - - val = timings->tRAS_max_ns / addressing->tREFI_ns - 1; - val = val > 0xF ? 0xF : val; - tim3 |= val << T_RAS_MAX_SHIFT; - - val = DIV_ROUND_UP(addressing->tRFCab_ps, t_ck) - 1; - tim3 |= val << T_RFC_SHIFT; - - t_dqsck = (derated == EMIF_DERATED_TIMINGS) ? - timings->tDQSCK_max_derated : timings->tDQSCK_max; - if (ip_rev == EMIF_4D5) - val = DIV_ROUND_UP(t_dqsck + 1000, t_ck) - 1; - else - val = DIV_ROUND_UP(t_dqsck, t_ck) - 1; - - tim3 |= val << T_TDQSCKMAX_SHIFT; - - val = DIV_ROUND_UP(timings->tZQCS, t_ck) - 1; - tim3 |= val << ZQ_ZQCS_SHIFT; - - val = DIV_ROUND_UP(timings->tCKESR, t_ck); - val = max(min_tck->tCKESR, val) - 1; - tim3 |= val << T_CKESR_SHIFT; - - if (ip_rev == EMIF_4D5) { - tim3 |= (EMIF_T_CSTA - 1) << T_CSTA_SHIFT; - - val = DIV_ROUND_UP(EMIF_T_PDLL_UL, 128) - 1; - tim3 |= val << T_PDLL_UL_SHIFT; - } - - return tim3; -} - static u32 get_zq_config_reg(const struct lpddr2_addressing *addressing, bool cs1_used, bool cal_resistors_per_cs) { @@ -631,117 +367,6 @@ static u32 get_temp_alert_config(const struct lpddr2_addressing *addressing, return alert; } -static u32 get_read_idle_ctrl_shdw(u8 volt_ramp) -{ - u32 idle = 0, val = 0; - - /* - * Maximum value in normal conditions and increased frequency - * when voltage is ramping - */ - if (volt_ramp) - val = READ_IDLE_INTERVAL_DVFS / t_ck / 64 - 1; - else - val = 0x1FF; - - /* - * READ_IDLE_CTRL register in EMIF4D has same offset and fields - * as DLL_CALIB_CTRL in EMIF4D5, so use the same shifts - */ - idle |= val << DLL_CALIB_INTERVAL_SHIFT; - idle |= EMIF_READ_IDLE_LEN_VAL << ACK_WAIT_SHIFT; - - return idle; -} - -static u32 get_dll_calib_ctrl_shdw(u8 volt_ramp) -{ - u32 calib = 0, val = 0; - - if (volt_ramp == DDR_VOLTAGE_RAMPING) - val = DLL_CALIB_INTERVAL_DVFS / t_ck / 16 - 1; - else - val = 0; /* Disabled when voltage is stable */ - - calib |= val << DLL_CALIB_INTERVAL_SHIFT; - calib |= DLL_CALIB_ACK_WAIT_VAL << ACK_WAIT_SHIFT; - - return calib; -} - -static u32 get_ddr_phy_ctrl_1_attilaphy_4d(const struct lpddr2_timings *timings, - u32 freq, u8 RL) -{ - u32 phy = EMIF_DDR_PHY_CTRL_1_BASE_VAL_ATTILAPHY, val = 0; - - val = RL + DIV_ROUND_UP(timings->tDQSCK_max, t_ck) - 1; - phy |= val << READ_LATENCY_SHIFT_4D; - - if (freq <= 100000000) - val = EMIF_DLL_SLAVE_DLY_CTRL_100_MHZ_AND_LESS_ATTILAPHY; - else if (freq <= 200000000) - val = EMIF_DLL_SLAVE_DLY_CTRL_200_MHZ_ATTILAPHY; - else - val = EMIF_DLL_SLAVE_DLY_CTRL_400_MHZ_ATTILAPHY; - - phy |= val << DLL_SLAVE_DLY_CTRL_SHIFT_4D; - - return phy; -} - -static u32 get_phy_ctrl_1_intelliphy_4d5(u32 freq, u8 cl) -{ - u32 phy = EMIF_DDR_PHY_CTRL_1_BASE_VAL_INTELLIPHY, half_delay; - - /* - * DLL operates at 266 MHz. If DDR frequency is near 266 MHz, - * half-delay is not needed else set half-delay - */ - if (freq >= 265000000 && freq < 267000000) - half_delay = 0; - else - half_delay = 1; - - phy |= half_delay << DLL_HALF_DELAY_SHIFT_4D5; - phy |= ((cl + DIV_ROUND_UP(EMIF_PHY_TOTAL_READ_LATENCY_INTELLIPHY_PS, - t_ck) - 1) << READ_LATENCY_SHIFT_4D5); - - return phy; -} - -static u32 get_ext_phy_ctrl_2_intelliphy_4d5(void) -{ - u32 fifo_we_slave_ratio; - - fifo_we_slave_ratio = DIV_ROUND_CLOSEST( - EMIF_INTELLI_PHY_DQS_GATE_OPENING_DELAY_PS * 256 , t_ck); - - return fifo_we_slave_ratio | fifo_we_slave_ratio << 11 | - fifo_we_slave_ratio << 22; -} - -static u32 get_ext_phy_ctrl_3_intelliphy_4d5(void) -{ - u32 fifo_we_slave_ratio; - - fifo_we_slave_ratio = DIV_ROUND_CLOSEST( - EMIF_INTELLI_PHY_DQS_GATE_OPENING_DELAY_PS * 256 , t_ck); - - return fifo_we_slave_ratio >> 10 | fifo_we_slave_ratio << 1 | - fifo_we_slave_ratio << 12 | fifo_we_slave_ratio << 23; -} - -static u32 get_ext_phy_ctrl_4_intelliphy_4d5(void) -{ - u32 fifo_we_slave_ratio; - - fifo_we_slave_ratio = DIV_ROUND_CLOSEST( - EMIF_INTELLI_PHY_DQS_GATE_OPENING_DELAY_PS * 256 , t_ck); - - return fifo_we_slave_ratio >> 9 | fifo_we_slave_ratio << 2 | - fifo_we_slave_ratio << 13; -} - static u32 get_pwr_mgmt_ctrl(u32 freq, struct emif_data *emif, u32 ip_rev) { u32 pwr_mgmt_ctrl = 0, timeout; @@ -864,51 +489,6 @@ static void get_temperature_level(struct emif_data *emif) } /* - * Program EMIF shadow registers that are not dependent on temperature - * or voltage - */ -static void setup_registers(struct emif_data *emif, struct emif_regs *regs) -{ - void __iomem *base = emif->base; - - writel(regs->sdram_tim2_shdw, base + EMIF_SDRAM_TIMING_2_SHDW); - writel(regs->phy_ctrl_1_shdw, base + EMIF_DDR_PHY_CTRL_1_SHDW); - writel(regs->pwr_mgmt_ctrl_shdw, - base + EMIF_POWER_MANAGEMENT_CTRL_SHDW); - - /* Settings specific for EMIF4D5 */ - if (emif->plat_data->ip_rev != EMIF_4D5) - return; - writel(regs->ext_phy_ctrl_2_shdw, base + EMIF_EXT_PHY_CTRL_2_SHDW); - writel(regs->ext_phy_ctrl_3_shdw, base + EMIF_EXT_PHY_CTRL_3_SHDW); - writel(regs->ext_phy_ctrl_4_shdw, base + EMIF_EXT_PHY_CTRL_4_SHDW); -} - -/* - * When voltage ramps dll calibration and forced read idle should - * happen more often - */ -static void setup_volt_sensitive_regs(struct emif_data *emif, - struct emif_regs *regs, u32 volt_state) -{ - u32 calib_ctrl; - void __iomem *base = emif->base; - - /* - * EMIF_READ_IDLE_CTRL in EMIF4D refers to the same register as - * EMIF_DLL_CALIB_CTRL in EMIF4D5 and dll_calib_ctrl_shadow_* - * is an alias of the respective read_idle_ctrl_shdw_* (members of - * a union). So, the below code takes care of both cases - */ - if (volt_state == DDR_VOLTAGE_RAMPING) - calib_ctrl = regs->dll_calib_ctrl_shdw_volt_ramp; - else - calib_ctrl = regs->dll_calib_ctrl_shdw_normal; - - writel(calib_ctrl, base + EMIF_DLL_CALIB_CTRL_SHDW); -} - -/* * setup_temperature_sensitive_regs() - set the timings for temperature * sensitive registers. This happens once at initialisation time based * on the temperature at boot time and subsequently based on the temperature @@ -975,8 +555,7 @@ static irqreturn_t handle_temp_alert(void __iomem *base, struct emif_data *emif) EMIF_CUSTOM_CONFIG_EXTENDED_TEMP_PART)) { if (emif->temperature_level >= SDRAM_TEMP_HIGH_DERATE_REFRESH) { dev_err(emif->dev, - "%s:NOT Extended temperature capable memory." - "Converting MR4=0x%02x as shutdown event\n", + "%s:NOT Extended temperature capable memory. Converting MR4=0x%02x as shutdown event\n", __func__, emif->temperature_level); /* * Temperature far too high - do kernel_power_off() @@ -1051,7 +630,7 @@ static irqreturn_t emif_threaded_isr(int irq, void *dev_id) dev_emerg(emif->dev, "SDRAM temperature exceeds operating limit.. Needs shut down!!!\n"); /* If we have Power OFF ability, use it, else try restarting */ - if (pm_power_off) { + if (kernel_can_power_off()) { kernel_power_off(); } else { WARN(1, "FIXME: NO pm_power_off!!! trying restart\n"); @@ -1318,9 +897,9 @@ static void __init_or_module of_get_ddr_info(struct device_node *np_emif, if (of_find_property(np_emif, "cal-resistor-per-cs", &len)) dev_info->cal_resistors_per_cs = true; - if (of_device_is_compatible(np_ddr , "jedec,lpddr2-s4")) + if (of_device_is_compatible(np_ddr, "jedec,lpddr2-s4")) dev_info->type = DDR_TYPE_LPDDR2_S4; - else if (of_device_is_compatible(np_ddr , "jedec,lpddr2-s2")) + else if (of_device_is_compatible(np_ddr, "jedec,lpddr2-s2")) dev_info->type = DDR_TYPE_LPDDR2_S2; of_property_read_u32(np_ddr, "density", &density); @@ -1446,10 +1025,8 @@ static struct emif_data *__init_or_module get_device_details( temp = devm_kzalloc(dev, sizeof(*pd), GFP_KERNEL); dev_info = devm_kzalloc(dev, sizeof(*dev_info), GFP_KERNEL); - if (!emif || !pd || !dev_info) { - dev_err(dev, "%s:%d: allocation error\n", __func__, __LINE__); + if (!emif || !temp || !dev_info) goto error; - } memcpy(temp, pd, sizeof(*pd)); pd = temp; @@ -1488,9 +1065,6 @@ static struct emif_data *__init_or_module get_device_details( temp = devm_kzalloc(dev, sizeof(*cust_cfgs), GFP_KERNEL); if (temp) memcpy(temp, cust_cfgs, sizeof(*cust_cfgs)); - else - dev_warn(dev, "%s:%d: allocation error\n", __func__, - __LINE__); pd->custom_configs = temp; } @@ -1505,8 +1079,6 @@ static struct emif_data *__init_or_module get_device_details( memcpy(temp, pd->timings, size); pd->timings = temp; } else { - dev_warn(dev, "%s:%d: allocation error\n", __func__, - __LINE__); get_default_timings(emif); } } else { @@ -1519,8 +1091,6 @@ static struct emif_data *__init_or_module get_device_details( memcpy(temp, pd->min_tck, sizeof(*pd->min_tck)); pd->min_tck = temp; } else { - dev_warn(dev, "%s:%d: allocation error\n", __func__, - __LINE__); pd->min_tck = &lpddr2_jedec_min_tck; } } else { @@ -1537,8 +1107,7 @@ error: static int __init_or_module emif_probe(struct platform_device *pdev) { struct emif_data *emif; - struct resource *res; - int irq; + int irq, ret; if (pdev->dev.of_node) emif = of_get_memory_device_details(pdev->dev.of_node, &pdev->dev); @@ -1551,33 +1120,29 @@ static int __init_or_module emif_probe(struct platform_device *pdev) } list_add(&emif->node, &device_list); - emif->addressing = get_addressing_table(emif->plat_data->device_info); /* Save pointers to each other in emif and device structures */ emif->dev = &pdev->dev; platform_set_drvdata(pdev, emif); - res = platform_get_resource(pdev, IORESOURCE_MEM, 0); - emif->base = devm_ioremap_resource(emif->dev, res); + emif->base = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(emif->base)) goto error; irq = platform_get_irq(pdev, 0); - if (irq < 0) { - dev_err(emif->dev, "%s: error getting IRQ resource - %d\n", - __func__, irq); + if (irq < 0) goto error; - } emif_onetime_settings(emif); emif_debugfs_init(emif); disable_and_clear_all_interrupts(emif); - setup_interrupts(emif, irq); + ret = setup_interrupts(emif, irq); + if (ret) + goto error; /* One-time actions taken on probing the first device */ if (!emif1) { emif1 = emif; - spin_lock_init(&emif_lock); /* * TODO: register notifiers for frequency and voltage @@ -1610,305 +1175,6 @@ static void emif_shutdown(struct platform_device *pdev) disable_and_clear_all_interrupts(emif); } -static int get_emif_reg_values(struct emif_data *emif, u32 freq, - struct emif_regs *regs) -{ - u32 ip_rev, phy_type; - u32 cl, type; - const struct lpddr2_timings *timings; - const struct lpddr2_min_tck *min_tck; - const struct ddr_device_info *device_info; - const struct lpddr2_addressing *addressing; - struct emif_data *emif_for_calc; - struct device *dev; - - dev = emif->dev; - /* - * If the devices on this EMIF instance is duplicate of EMIF1, - * use EMIF1 details for the calculation - */ - emif_for_calc = emif->duplicate ? emif1 : emif; - timings = get_timings_table(emif_for_calc, freq); - addressing = emif_for_calc->addressing; - if (!timings || !addressing) { - dev_err(dev, "%s: not enough data available for %dHz", - __func__, freq); - return -1; - } - - device_info = emif_for_calc->plat_data->device_info; - type = device_info->type; - ip_rev = emif_for_calc->plat_data->ip_rev; - phy_type = emif_for_calc->plat_data->phy_type; - - min_tck = emif_for_calc->plat_data->min_tck; - - set_ddr_clk_period(freq); - - regs->ref_ctrl_shdw = get_sdram_ref_ctrl_shdw(freq, addressing); - regs->sdram_tim1_shdw = get_sdram_tim_1_shdw(timings, min_tck, - addressing); - regs->sdram_tim2_shdw = get_sdram_tim_2_shdw(timings, min_tck, - addressing, type); - regs->sdram_tim3_shdw = get_sdram_tim_3_shdw(timings, min_tck, - addressing, type, ip_rev, EMIF_NORMAL_TIMINGS); - - cl = get_cl(emif); - - if (phy_type == EMIF_PHY_TYPE_ATTILAPHY && ip_rev == EMIF_4D) { - regs->phy_ctrl_1_shdw = get_ddr_phy_ctrl_1_attilaphy_4d( - timings, freq, cl); - } else if (phy_type == EMIF_PHY_TYPE_INTELLIPHY && ip_rev == EMIF_4D5) { - regs->phy_ctrl_1_shdw = get_phy_ctrl_1_intelliphy_4d5(freq, cl); - regs->ext_phy_ctrl_2_shdw = get_ext_phy_ctrl_2_intelliphy_4d5(); - regs->ext_phy_ctrl_3_shdw = get_ext_phy_ctrl_3_intelliphy_4d5(); - regs->ext_phy_ctrl_4_shdw = get_ext_phy_ctrl_4_intelliphy_4d5(); - } else { - return -1; - } - - /* Only timeout values in pwr_mgmt_ctrl_shdw register */ - regs->pwr_mgmt_ctrl_shdw = - get_pwr_mgmt_ctrl(freq, emif_for_calc, ip_rev) & - (CS_TIM_MASK | SR_TIM_MASK | PD_TIM_MASK); - - if (ip_rev & EMIF_4D) { - regs->read_idle_ctrl_shdw_normal = - get_read_idle_ctrl_shdw(DDR_VOLTAGE_STABLE); - - regs->read_idle_ctrl_shdw_volt_ramp = - get_read_idle_ctrl_shdw(DDR_VOLTAGE_RAMPING); - } else if (ip_rev & EMIF_4D5) { - regs->dll_calib_ctrl_shdw_normal = - get_dll_calib_ctrl_shdw(DDR_VOLTAGE_STABLE); - - regs->dll_calib_ctrl_shdw_volt_ramp = - get_dll_calib_ctrl_shdw(DDR_VOLTAGE_RAMPING); - } - - if (type == DDR_TYPE_LPDDR2_S2 || type == DDR_TYPE_LPDDR2_S4) { - regs->ref_ctrl_shdw_derated = get_sdram_ref_ctrl_shdw(freq / 4, - addressing); - - regs->sdram_tim1_shdw_derated = - get_sdram_tim_1_shdw_derated(timings, min_tck, - addressing); - - regs->sdram_tim3_shdw_derated = get_sdram_tim_3_shdw(timings, - min_tck, addressing, type, ip_rev, - EMIF_DERATED_TIMINGS); - } - - regs->freq = freq; - - return 0; -} - -/* - * get_regs() - gets the cached emif_regs structure for a given EMIF instance - * given frequency(freq): - * - * As an optimisation, every EMIF instance other than EMIF1 shares the - * register cache with EMIF1 if the devices connected on this instance - * are same as that on EMIF1(indicated by the duplicate flag) - * - * If we do not have an entry corresponding to the frequency given, we - * allocate a new entry and calculate the values - * - * Upon finding the right reg dump, save it in curr_regs. It can be - * directly used for thermal de-rating and voltage ramping changes. - */ -static struct emif_regs *get_regs(struct emif_data *emif, u32 freq) -{ - int i; - struct emif_regs **regs_cache; - struct emif_regs *regs = NULL; - struct device *dev; - - dev = emif->dev; - if (emif->curr_regs && emif->curr_regs->freq == freq) { - dev_dbg(dev, "%s: using curr_regs - %u Hz", __func__, freq); - return emif->curr_regs; - } - - if (emif->duplicate) - regs_cache = emif1->regs_cache; - else - regs_cache = emif->regs_cache; - - for (i = 0; i < EMIF_MAX_NUM_FREQUENCIES && regs_cache[i]; i++) { - if (regs_cache[i]->freq == freq) { - regs = regs_cache[i]; - dev_dbg(dev, - "%s: reg dump found in reg cache for %u Hz\n", - __func__, freq); - break; - } - } - - /* - * If we don't have an entry for this frequency in the cache create one - * and calculate the values - */ - if (!regs) { - regs = devm_kzalloc(emif->dev, sizeof(*regs), GFP_ATOMIC); - if (!regs) - return NULL; - - if (get_emif_reg_values(emif, freq, regs)) { - devm_kfree(emif->dev, regs); - return NULL; - } - - /* - * Now look for an un-used entry in the cache and save the - * newly created struct. If there are no free entries - * over-write the last entry - */ - for (i = 0; i < EMIF_MAX_NUM_FREQUENCIES && regs_cache[i]; i++) - ; - - if (i >= EMIF_MAX_NUM_FREQUENCIES) { - dev_warn(dev, "%s: regs_cache full - reusing a slot!!\n", - __func__); - i = EMIF_MAX_NUM_FREQUENCIES - 1; - devm_kfree(emif->dev, regs_cache[i]); - } - regs_cache[i] = regs; - } - - return regs; -} - -static void do_volt_notify_handling(struct emif_data *emif, u32 volt_state) -{ - dev_dbg(emif->dev, "%s: voltage notification : %d", __func__, - volt_state); - - if (!emif->curr_regs) { - dev_err(emif->dev, - "%s: volt-notify before registers are ready: %d\n", - __func__, volt_state); - return; - } - - setup_volt_sensitive_regs(emif, emif->curr_regs, volt_state); -} - -/* - * TODO: voltage notify handling should be hooked up to - * regulator framework as soon as the necessary support - * is available in mainline kernel. This function is un-used - * right now. - */ -static void __attribute__((unused)) volt_notify_handling(u32 volt_state) -{ - struct emif_data *emif; - - spin_lock_irqsave(&emif_lock, irq_state); - - list_for_each_entry(emif, &device_list, node) - do_volt_notify_handling(emif, volt_state); - do_freq_update(); - - spin_unlock_irqrestore(&emif_lock, irq_state); -} - -static void do_freq_pre_notify_handling(struct emif_data *emif, u32 new_freq) -{ - struct emif_regs *regs; - - regs = get_regs(emif, new_freq); - if (!regs) - return; - - emif->curr_regs = regs; - - /* - * Update the shadow registers: - * Temperature and voltage-ramp sensitive settings are also configured - * in terms of DDR cycles. So, we need to update them too when there - * is a freq change - */ - dev_dbg(emif->dev, "%s: setting up shadow registers for %uHz", - __func__, new_freq); - setup_registers(emif, regs); - setup_temperature_sensitive_regs(emif, regs); - setup_volt_sensitive_regs(emif, regs, DDR_VOLTAGE_STABLE); - - /* - * Part of workaround for errata i728. See do_freq_update() - * for more details - */ - if (emif->lpmode == EMIF_LP_MODE_SELF_REFRESH) - set_lpmode(emif, EMIF_LP_MODE_DISABLE); -} - -/* - * TODO: frequency notify handling should be hooked up to - * clock framework as soon as the necessary support is - * available in mainline kernel. This function is un-used - * right now. - */ -static void __attribute__((unused)) freq_pre_notify_handling(u32 new_freq) -{ - struct emif_data *emif; - - /* - * NOTE: we are taking the spin-lock here and releases it - * only in post-notifier. This doesn't look good and - * Sparse complains about it, but this seems to be - * un-avoidable. We need to lock a sequence of events - * that is split between EMIF and clock framework. - * - * 1. EMIF driver updates EMIF timings in shadow registers in the - * frequency pre-notify callback from clock framework - * 2. clock framework sets up the registers for the new frequency - * 3. clock framework initiates a hw-sequence that updates - * the frequency EMIF timings synchronously. - * - * All these 3 steps should be performed as an atomic operation - * vis-a-vis similar sequence in the EMIF interrupt handler - * for temperature events. Otherwise, there could be race - * conditions that could result in incorrect EMIF timings for - * a given frequency - */ - spin_lock_irqsave(&emif_lock, irq_state); - - list_for_each_entry(emif, &device_list, node) - do_freq_pre_notify_handling(emif, new_freq); -} - -static void do_freq_post_notify_handling(struct emif_data *emif) -{ - /* - * Part of workaround for errata i728. See do_freq_update() - * for more details - */ - if (emif->lpmode == EMIF_LP_MODE_SELF_REFRESH) - set_lpmode(emif, EMIF_LP_MODE_SELF_REFRESH); -} - -/* - * TODO: frequency notify handling should be hooked up to - * clock framework as soon as the necessary support is - * available in mainline kernel. This function is un-used - * right now. - */ -static void __attribute__((unused)) freq_post_notify_handling(void) -{ - struct emif_data *emif; - - list_for_each_entry(emif, &device_list, node) - do_freq_post_notify_handling(emif); - - /* - * Lock is done in pre-notify handler. See freq_pre_notify_handling() - * for more details - */ - spin_unlock_irqrestore(&emif_lock, irq_state); -} - #if defined(CONFIG_OF) static const struct of_device_id emif_of_match[] = { { .compatible = "ti,emif-4d" }, diff --git a/drivers/memory/fsl-corenet-cf.c b/drivers/memory/fsl-corenet-cf.c index 0b0ed72016da..7fc9f57ae278 100644 --- a/drivers/memory/fsl-corenet-cf.c +++ b/drivers/memory/fsl-corenet-cf.c @@ -172,7 +172,6 @@ out: static int ccf_probe(struct platform_device *pdev) { struct ccf_private *ccf; - struct resource *r; const struct of_device_id *match; u32 errinten; int ret, irq; @@ -185,17 +184,9 @@ static int ccf_probe(struct platform_device *pdev) if (!ccf) return -ENOMEM; - r = platform_get_resource(pdev, IORESOURCE_MEM, 0); - if (!r) { - dev_err(&pdev->dev, "%s: no mem resource\n", __func__); - return -ENXIO; - } - - ccf->regs = devm_ioremap_resource(&pdev->dev, r); - if (IS_ERR(ccf->regs)) { - dev_err(&pdev->dev, "%s: can't map mem resource\n", __func__); + ccf->regs = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(ccf->regs)) return PTR_ERR(ccf->regs); - } ccf->dev = &pdev->dev; ccf->info = match->data; @@ -211,10 +202,8 @@ static int ccf_probe(struct platform_device *pdev) dev_set_drvdata(&pdev->dev, ccf); irq = platform_get_irq(pdev, 0); - if (!irq) { - dev_err(&pdev->dev, "%s: no irq\n", __func__); - return -ENXIO; - } + if (irq < 0) + return irq; ret = devm_request_irq(&pdev->dev, irq, ccf_irq, 0, pdev->name, ccf); if (ret) { diff --git a/drivers/memory/fsl_ifc.c b/drivers/memory/fsl_ifc.c index a2c971743ffe..e83b61c925a4 100644 --- a/drivers/memory/fsl_ifc.c +++ b/drivers/memory/fsl_ifc.c @@ -53,6 +53,7 @@ int fsl_ifc_find(phys_addr_t addr_base) for (i = 0; i < fsl_ifc_ctrl_dev->banks; i++) { u32 cspr = ifc_in32(&fsl_ifc_ctrl_dev->gregs->cspr_cs[i].cspr); + if (cspr & CSPR_V && (cspr & CSPR_BA) == convert_ifc_address(addr_base)) return i; @@ -87,6 +88,7 @@ static int fsl_ifc_ctrl_remove(struct platform_device *dev) { struct fsl_ifc_ctrl *ctrl = dev_get_drvdata(&dev->dev); + of_platform_depopulate(&dev->dev); free_irq(ctrl->nand_irq, ctrl); free_irq(ctrl->irq, ctrl); @@ -96,7 +98,6 @@ static int fsl_ifc_ctrl_remove(struct platform_device *dev) iounmap(ctrl->gregs); dev_set_drvdata(&dev->dev, NULL); - kfree(ctrl); return 0; } @@ -153,8 +154,8 @@ static irqreturn_t fsl_ifc_ctrl_irq(int irqno, void *data) /* read for chip select error */ cs_err = ifc_in32(&ifc->cm_evter_stat); if (cs_err) { - dev_err(ctrl->dev, "transaction sent to IFC is not mapped to" - "any memory bank 0x%08X\n", cs_err); + dev_err(ctrl->dev, "transaction sent to IFC is not mapped to any memory bank 0x%08X\n", + cs_err); /* clear the chip select error */ ifc_out32(IFC_CM_EVTER_STAT_CSER, &ifc->cm_evter_stat); @@ -163,24 +164,24 @@ static irqreturn_t fsl_ifc_ctrl_irq(int irqno, void *data) err_addr = ifc_in32(&ifc->cm_erattr1); if (status & IFC_CM_ERATTR0_ERTYP_READ) - dev_err(ctrl->dev, "Read transaction error" - "CM_ERATTR0 0x%08X\n", status); + dev_err(ctrl->dev, "Read transaction error CM_ERATTR0 0x%08X\n", + status); else - dev_err(ctrl->dev, "Write transaction error" - "CM_ERATTR0 0x%08X\n", status); + dev_err(ctrl->dev, "Write transaction error CM_ERATTR0 0x%08X\n", + status); err_axiid = (status & IFC_CM_ERATTR0_ERAID) >> IFC_CM_ERATTR0_ERAID_SHIFT; - dev_err(ctrl->dev, "AXI ID of the error" - "transaction 0x%08X\n", err_axiid); + dev_err(ctrl->dev, "AXI ID of the error transaction 0x%08X\n", + err_axiid); err_srcid = (status & IFC_CM_ERATTR0_ESRCID) >> IFC_CM_ERATTR0_ESRCID_SHIFT; - dev_err(ctrl->dev, "SRC ID of the error" - "transaction 0x%08X\n", err_srcid); + dev_err(ctrl->dev, "SRC ID of the error transaction 0x%08X\n", + err_srcid); - dev_err(ctrl->dev, "Transaction Address corresponding to error" - "ERADDR 0x%08X\n", err_addr); + dev_err(ctrl->dev, "Transaction Address corresponding to error ERADDR 0x%08X\n", + err_addr); ret = IRQ_HANDLED; } @@ -199,7 +200,7 @@ static irqreturn_t fsl_ifc_ctrl_irq(int irqno, void *data) * the resources needed for the controller only. The * resources for the NAND banks themselves are allocated * in the chip probe function. -*/ + */ static int fsl_ifc_ctrl_probe(struct platform_device *dev) { int ret = 0; @@ -208,7 +209,8 @@ static int fsl_ifc_ctrl_probe(struct platform_device *dev) dev_info(&dev->dev, "Freescale Integrated Flash Controller\n"); - fsl_ifc_ctrl_dev = kzalloc(sizeof(*fsl_ifc_ctrl_dev), GFP_KERNEL); + fsl_ifc_ctrl_dev = devm_kzalloc(&dev->dev, sizeof(*fsl_ifc_ctrl_dev), + GFP_KERNEL); if (!fsl_ifc_ctrl_dev) return -ENOMEM; @@ -218,8 +220,7 @@ static int fsl_ifc_ctrl_probe(struct platform_device *dev) fsl_ifc_ctrl_dev->gregs = of_iomap(dev->dev.of_node, 0); if (!fsl_ifc_ctrl_dev->gregs) { dev_err(&dev->dev, "failed to get memory region\n"); - ret = -ENODEV; - goto err; + return -ENODEV; } if (of_property_read_bool(dev->dev.of_node, "little-endian")) { @@ -250,8 +251,7 @@ static int fsl_ifc_ctrl_probe(struct platform_device *dev) /* get the Controller level irq */ fsl_ifc_ctrl_dev->irq = irq_of_parse_and_map(dev->dev.of_node, 0); if (fsl_ifc_ctrl_dev->irq == 0) { - dev_err(&dev->dev, "failed to get irq resource " - "for IFC\n"); + dev_err(&dev->dev, "failed to get irq resource for IFC\n"); ret = -ENODEV; goto err; } @@ -264,7 +264,7 @@ static int fsl_ifc_ctrl_probe(struct platform_device *dev) ret = fsl_ifc_ctrl_init(fsl_ifc_ctrl_dev); if (ret < 0) - goto err; + goto err_unmap_nandirq; init_waitqueue_head(&fsl_ifc_ctrl_dev->nand_wait); @@ -273,7 +273,7 @@ static int fsl_ifc_ctrl_probe(struct platform_device *dev) if (ret != 0) { dev_err(&dev->dev, "failed to install irq (%d)\n", fsl_ifc_ctrl_dev->irq); - goto err_irq; + goto err_unmap_nandirq; } if (fsl_ifc_ctrl_dev->nand_irq) { @@ -282,19 +282,26 @@ static int fsl_ifc_ctrl_probe(struct platform_device *dev) if (ret != 0) { dev_err(&dev->dev, "failed to install irq (%d)\n", fsl_ifc_ctrl_dev->nand_irq); - goto err_nandirq; + goto err_free_irq; } } + /* legacy dts may still use "simple-bus" compatible */ + ret = of_platform_default_populate(dev->dev.of_node, NULL, &dev->dev); + if (ret) + goto err_free_nandirq; + return 0; -err_nandirq: +err_free_nandirq: free_irq(fsl_ifc_ctrl_dev->nand_irq, fsl_ifc_ctrl_dev); - irq_dispose_mapping(fsl_ifc_ctrl_dev->nand_irq); -err_irq: +err_free_irq: free_irq(fsl_ifc_ctrl_dev->irq, fsl_ifc_ctrl_dev); +err_unmap_nandirq: + irq_dispose_mapping(fsl_ifc_ctrl_dev->nand_irq); irq_dispose_mapping(fsl_ifc_ctrl_dev->irq); err: + iounmap(fsl_ifc_ctrl_dev->gregs); return ret; } diff --git a/drivers/memory/jedec_ddr.h b/drivers/memory/jedec_ddr.h index e59ccbd982d0..6cd508478b14 100644 --- a/drivers/memory/jedec_ddr.h +++ b/drivers/memory/jedec_ddr.h @@ -112,6 +112,26 @@ #define NUM_DDR_ADDR_TABLE_ENTRIES 11 #define NUM_DDR_TIMING_TABLE_ENTRIES 4 +#define LPDDR2_MANID_SAMSUNG 1 +#define LPDDR2_MANID_QIMONDA 2 +#define LPDDR2_MANID_ELPIDA 3 +#define LPDDR2_MANID_ETRON 4 +#define LPDDR2_MANID_NANYA 5 +#define LPDDR2_MANID_HYNIX 6 +#define LPDDR2_MANID_MOSEL 7 +#define LPDDR2_MANID_WINBOND 8 +#define LPDDR2_MANID_ESMT 9 +#define LPDDR2_MANID_SPANSION 11 +#define LPDDR2_MANID_SST 12 +#define LPDDR2_MANID_ZMOS 13 +#define LPDDR2_MANID_INTEL 14 +#define LPDDR2_MANID_NUMONYX 254 +#define LPDDR2_MANID_MICRON 255 + +#define LPDDR2_TYPE_S4 0 +#define LPDDR2_TYPE_S2 1 +#define LPDDR2_TYPE_NVM 2 + /* Structure for DDR addressing info from the JEDEC spec */ struct lpddr2_addressing { u32 num_banks; @@ -170,6 +190,33 @@ extern const struct lpddr2_timings lpddr2_jedec_timings[NUM_DDR_TIMING_TABLE_ENTRIES]; extern const struct lpddr2_min_tck lpddr2_jedec_min_tck; +/* Structure of MR8 */ +union lpddr2_basic_config4 { + u32 value; + + struct { + unsigned int arch_type : 2; + unsigned int density : 4; + unsigned int io_width : 2; + } __packed; +}; + +/* + * Structure for information about LPDDR2 chip. All parameters are + * matching raw values of standard mode register bitfields or set to + * -ENOENT if info unavailable. + */ +struct lpddr2_info { + int arch_type; + int density; + int io_width; + int manufacturer_id; + int revision_id1; + int revision_id2; +}; + +const char *lpddr2_jedec_manufacturer(unsigned int manufacturer_id); + /* * Structure for timings for LPDDR3 based on LPDDR2 plus additional fields. * All parameters are in pico seconds(ps) excluding max_freq, min_freq which diff --git a/drivers/memory/jedec_ddr_data.c b/drivers/memory/jedec_ddr_data.c index ed601d813175..2cca4fa188f9 100644 --- a/drivers/memory/jedec_ddr_data.c +++ b/drivers/memory/jedec_ddr_data.c @@ -131,3 +131,44 @@ const struct lpddr2_min_tck lpddr2_jedec_min_tck = { .tFAW = 8 }; EXPORT_SYMBOL_GPL(lpddr2_jedec_min_tck); + +const char *lpddr2_jedec_manufacturer(unsigned int manufacturer_id) +{ + switch (manufacturer_id) { + case LPDDR2_MANID_SAMSUNG: + return "Samsung"; + case LPDDR2_MANID_QIMONDA: + return "Qimonda"; + case LPDDR2_MANID_ELPIDA: + return "Elpida"; + case LPDDR2_MANID_ETRON: + return "Etron"; + case LPDDR2_MANID_NANYA: + return "Nanya"; + case LPDDR2_MANID_HYNIX: + return "Hynix"; + case LPDDR2_MANID_MOSEL: + return "Mosel"; + case LPDDR2_MANID_WINBOND: + return "Winbond"; + case LPDDR2_MANID_ESMT: + return "ESMT"; + case LPDDR2_MANID_SPANSION: + return "Spansion"; + case LPDDR2_MANID_SST: + return "SST"; + case LPDDR2_MANID_ZMOS: + return "ZMOS"; + case LPDDR2_MANID_INTEL: + return "Intel"; + case LPDDR2_MANID_NUMONYX: + return "Numonyx"; + case LPDDR2_MANID_MICRON: + return "Micron"; + default: + break; + } + + return "invalid"; +} +EXPORT_SYMBOL_GPL(lpddr2_jedec_manufacturer); diff --git a/drivers/memory/jz4780-nemc.c b/drivers/memory/jz4780-nemc.c index b232ed279fc3..555f7ac3b7dd 100644 --- a/drivers/memory/jz4780-nemc.c +++ b/drivers/memory/jz4780-nemc.c @@ -8,6 +8,7 @@ #include <linux/clk.h> #include <linux/init.h> +#include <linux/io.h> #include <linux/math64.h> #include <linux/of.h> #include <linux/of_address.h> @@ -22,6 +23,8 @@ #define NEMC_SMCRn(n) (0x14 + (((n) - 1) * 4)) #define NEMC_NFCSR 0x50 +#define NEMC_REG_LEN 0x54 + #define NEMC_SMCR_SMT BIT(0) #define NEMC_SMCR_BW_SHIFT 6 #define NEMC_SMCR_BW_MASK (0x3 << NEMC_SMCR_BW_SHIFT) @@ -288,10 +291,24 @@ static int jz4780_nemc_probe(struct platform_device *pdev) nemc->dev = dev; res = platform_get_resource(pdev, IORESOURCE_MEM, 0); - nemc->base = devm_ioremap_resource(dev, res); - if (IS_ERR(nemc->base)) { + if (!res) + return -EINVAL; + + /* + * The driver currently only uses the registers up to offset + * NEMC_REG_LEN. Since the EFUSE registers are in the middle of the + * NEMC registers, we only request the registers we will use for now; + * that way the EFUSE driver can probe too. + */ + if (!devm_request_mem_region(dev, res->start, NEMC_REG_LEN, dev_name(dev))) { + dev_err(dev, "unable to request I/O memory region\n"); + return -EBUSY; + } + + nemc->base = devm_ioremap(dev, res->start, NEMC_REG_LEN); + if (!nemc->base) { dev_err(dev, "failed to get I/O memory\n"); - return PTR_ERR(nemc->base); + return -ENOMEM; } writel(0, nemc->base + NEMC_NFCSR); diff --git a/drivers/memory/mtk-smi.c b/drivers/memory/mtk-smi.c index a113e811faab..5a9754442bc7 100644 --- a/drivers/memory/mtk-smi.c +++ b/drivers/memory/mtk-smi.c @@ -3,23 +3,56 @@ * Copyright (c) 2015-2016 MediaTek Inc. * Author: Yong Wu <yong.wu@mediatek.com> */ +#include <linux/arm-smccc.h> #include <linux/clk.h> #include <linux/component.h> #include <linux/device.h> #include <linux/err.h> #include <linux/io.h> +#include <linux/iopoll.h> #include <linux/module.h> #include <linux/of.h> #include <linux/of_platform.h> #include <linux/platform_device.h> #include <linux/pm_runtime.h> +#include <linux/soc/mediatek/mtk_sip_svc.h> #include <soc/mediatek/smi.h> #include <dt-bindings/memory/mt2701-larb-port.h> +#include <dt-bindings/memory/mtk-memory-port.h> -/* mt8173 */ -#define SMI_LARB_MMU_EN 0xf00 +/* SMI COMMON */ +#define SMI_L1LEN 0x100 + +#define SMI_L1_ARB 0x200 +#define SMI_BUS_SEL 0x220 +#define SMI_BUS_LARB_SHIFT(larbid) ((larbid) << 1) +/* All are MMU0 defaultly. Only specialize mmu1 here. */ +#define F_MMU1_LARB(larbid) (0x1 << SMI_BUS_LARB_SHIFT(larbid)) + +#define SMI_READ_FIFO_TH 0x230 +#define SMI_M4U_TH 0x234 +#define SMI_FIFO_TH1 0x238 +#define SMI_FIFO_TH2 0x23c +#define SMI_DCM 0x300 +#define SMI_DUMMY 0x444 + +/* SMI LARB */ +#define SMI_LARB_SLP_CON 0xc +#define SLP_PROT_EN BIT(0) +#define SLP_PROT_RDY BIT(16) -/* mt2701 */ +#define SMI_LARB_CMD_THRT_CON 0x24 +#define SMI_LARB_THRT_RD_NU_LMT_MSK GENMASK(7, 4) +#define SMI_LARB_THRT_RD_NU_LMT (5 << 4) + +#define SMI_LARB_SW_FLAG 0x40 +#define SMI_LARB_SW_FLAG_1 0x1 + +#define SMI_LARB_OSTDL_PORT 0x200 +#define SMI_LARB_OSTDL_PORTx(id) (SMI_LARB_OSTDL_PORT + (((id) & 0x1f) << 2)) + +/* Below are about mmu enable registers, they are different in SoCs */ +/* gen1: mt2701 */ #define REG_SMI_SECUR_CON_BASE 0x5c0 /* every register control 8 port, register offset 0x4 */ @@ -37,108 +70,95 @@ /* mt2701 domain should be set to 3 */ #define SMI_SECUR_CON_VAL_DOMAIN(id) (0x3 << ((((id) & 0x7) << 2) + 1)) -/* mt2712 */ -#define SMI_LARB_NONSEC_CON(id) (0x380 + ((id) * 4)) -#define F_MMU_EN BIT(0) +/* gen2: */ +/* mt8167 */ +#define MT8167_SMI_LARB_MMU_EN 0xfc0 -/* SMI COMMON */ -#define SMI_BUS_SEL 0x220 -#define SMI_BUS_LARB_SHIFT(larbid) ((larbid) << 1) -/* All are MMU0 defaultly. Only specialize mmu1 here. */ -#define F_MMU1_LARB(larbid) (0x1 << SMI_BUS_LARB_SHIFT(larbid)) +/* mt8173 */ +#define MT8173_SMI_LARB_MMU_EN 0xf00 + +/* general */ +#define SMI_LARB_NONSEC_CON(id) (0x380 + ((id) * 4)) +#define F_MMU_EN BIT(0) +#define BANK_SEL(id) ({ \ + u32 _id = (id) & 0x3; \ + (_id << 8 | _id << 10 | _id << 12 | _id << 14); \ +}) + +#define SMI_COMMON_INIT_REGS_NR 6 +#define SMI_LARB_PORT_NR_MAX 32 + +#define MTK_SMI_FLAG_THRT_UPDATE BIT(0) +#define MTK_SMI_FLAG_SW_FLAG BIT(1) +#define MTK_SMI_FLAG_SLEEP_CTL BIT(2) +#define MTK_SMI_FLAG_CFG_PORT_SEC_CTL BIT(3) +#define MTK_SMI_CAPS(flags, _x) (!!((flags) & (_x))) + +struct mtk_smi_reg_pair { + unsigned int offset; + u32 value; +}; -enum mtk_smi_gen { +enum mtk_smi_type { MTK_SMI_GEN1, - MTK_SMI_GEN2 + MTK_SMI_GEN2, /* gen2 smi common */ + MTK_SMI_GEN2_SUB_COMM, /* gen2 smi sub common */ }; +/* larbs: Require apb/smi clocks while gals is optional. */ +static const char * const mtk_smi_larb_clks[] = {"apb", "smi", "gals"}; +#define MTK_SMI_LARB_REQ_CLK_NR 2 +#define MTK_SMI_LARB_OPT_CLK_NR 1 + +/* + * common: Require these four clocks in has_gals case. Otherwise, only apb/smi are required. + * sub common: Require apb/smi/gals0 clocks in has_gals case. Otherwise, only apb/smi are required. + */ +static const char * const mtk_smi_common_clks[] = {"apb", "smi", "gals0", "gals1"}; +#define MTK_SMI_CLK_NR_MAX ARRAY_SIZE(mtk_smi_common_clks) +#define MTK_SMI_COM_REQ_CLK_NR 2 +#define MTK_SMI_COM_GALS_REQ_CLK_NR MTK_SMI_CLK_NR_MAX +#define MTK_SMI_SUB_COM_GALS_REQ_CLK_NR 3 + struct mtk_smi_common_plat { - enum mtk_smi_gen gen; - bool has_gals; - u32 bus_sel; /* Balance some larbs to enter mmu0 or mmu1 */ + enum mtk_smi_type type; + bool has_gals; + u32 bus_sel; /* Balance some larbs to enter mmu0 or mmu1 */ + + const struct mtk_smi_reg_pair *init; }; struct mtk_smi_larb_gen { int port_in_larb[MTK_LARB_NR_MAX + 1]; - void (*config_port)(struct device *); + int (*config_port)(struct device *dev); unsigned int larb_direct_to_common_mask; - bool has_gals; + unsigned int flags_general; + const u8 (*ostd)[SMI_LARB_PORT_NR_MAX]; }; struct mtk_smi { struct device *dev; - struct clk *clk_apb, *clk_smi; - struct clk *clk_gals0, *clk_gals1; + unsigned int clk_num; + struct clk_bulk_data clks[MTK_SMI_CLK_NR_MAX]; struct clk *clk_async; /*only needed by mt2701*/ union { void __iomem *smi_ao_base; /* only for gen1 */ void __iomem *base; /* only for gen2 */ }; + struct device *smi_common_dev; /* for sub common */ const struct mtk_smi_common_plat *plat; }; struct mtk_smi_larb { /* larb: local arbiter */ struct mtk_smi smi; void __iomem *base; - struct device *smi_common_dev; + struct device *smi_common_dev; /* common or sub-common dev */ const struct mtk_smi_larb_gen *larb_gen; int larbid; u32 *mmu; + unsigned char *bank; }; -static int mtk_smi_clk_enable(const struct mtk_smi *smi) -{ - int ret; - - ret = clk_prepare_enable(smi->clk_apb); - if (ret) - return ret; - - ret = clk_prepare_enable(smi->clk_smi); - if (ret) - goto err_disable_apb; - - ret = clk_prepare_enable(smi->clk_gals0); - if (ret) - goto err_disable_smi; - - ret = clk_prepare_enable(smi->clk_gals1); - if (ret) - goto err_disable_gals0; - - return 0; - -err_disable_gals0: - clk_disable_unprepare(smi->clk_gals0); -err_disable_smi: - clk_disable_unprepare(smi->clk_smi); -err_disable_apb: - clk_disable_unprepare(smi->clk_apb); - return ret; -} - -static void mtk_smi_clk_disable(const struct mtk_smi *smi) -{ - clk_disable_unprepare(smi->clk_gals1); - clk_disable_unprepare(smi->clk_gals0); - clk_disable_unprepare(smi->clk_smi); - clk_disable_unprepare(smi->clk_apb); -} - -int mtk_smi_larb_get(struct device *larbdev) -{ - int ret = pm_runtime_get_sync(larbdev); - - return (ret < 0) ? ret : 0; -} -EXPORT_SYMBOL_GPL(mtk_smi_larb_get); - -void mtk_smi_larb_put(struct device *larbdev) -{ - pm_runtime_put_sync(larbdev); -} -EXPORT_SYMBOL_GPL(mtk_smi_larb_put); - static int mtk_smi_larb_bind(struct device *dev, struct device *master, void *data) { @@ -150,36 +170,25 @@ mtk_smi_larb_bind(struct device *dev, struct device *master, void *data) if (dev == larb_mmu[i].dev) { larb->larbid = i; larb->mmu = &larb_mmu[i].mmu; + larb->bank = larb_mmu[i].bank; return 0; } } return -ENODEV; } -static void mtk_smi_larb_config_port_gen2_general(struct device *dev) +static void +mtk_smi_larb_unbind(struct device *dev, struct device *master, void *data) { - struct mtk_smi_larb *larb = dev_get_drvdata(dev); - u32 reg; - int i; - - if (BIT(larb->larbid) & larb->larb_gen->larb_direct_to_common_mask) - return; - - for_each_set_bit(i, (unsigned long *)larb->mmu, 32) { - reg = readl_relaxed(larb->base + SMI_LARB_NONSEC_CON(i)); - reg |= F_MMU_EN; - writel(reg, larb->base + SMI_LARB_NONSEC_CON(i)); - } + /* Do nothing as the iommu is always enabled. */ } -static void mtk_smi_larb_config_port_mt8173(struct device *dev) -{ - struct mtk_smi_larb *larb = dev_get_drvdata(dev); - - writel(*larb->mmu, larb->base + SMI_LARB_MMU_EN); -} +static const struct component_ops mtk_smi_larb_component_ops = { + .bind = mtk_smi_larb_bind, + .unbind = mtk_smi_larb_unbind, +}; -static void mtk_smi_larb_config_port_gen1(struct device *dev) +static int mtk_smi_larb_config_port_gen1(struct device *dev) { struct mtk_smi_larb *larb = dev_get_drvdata(dev); const struct mtk_smi_larb_gen *larb_gen = larb->larb_gen; @@ -208,22 +217,164 @@ static void mtk_smi_larb_config_port_gen1(struct device *dev) common->smi_ao_base + REG_SMI_SECUR_CON_ADDR(m4u_port_id)); } + return 0; } -static void -mtk_smi_larb_unbind(struct device *dev, struct device *master, void *data) +static int mtk_smi_larb_config_port_mt8167(struct device *dev) { - /* Do nothing as the iommu is always enabled. */ + struct mtk_smi_larb *larb = dev_get_drvdata(dev); + + writel(*larb->mmu, larb->base + MT8167_SMI_LARB_MMU_EN); + return 0; } -static const struct component_ops mtk_smi_larb_component_ops = { - .bind = mtk_smi_larb_bind, - .unbind = mtk_smi_larb_unbind, +static int mtk_smi_larb_config_port_mt8173(struct device *dev) +{ + struct mtk_smi_larb *larb = dev_get_drvdata(dev); + + writel(*larb->mmu, larb->base + MT8173_SMI_LARB_MMU_EN); + return 0; +} + +static int mtk_smi_larb_config_port_gen2_general(struct device *dev) +{ + struct mtk_smi_larb *larb = dev_get_drvdata(dev); + u32 reg, flags_general = larb->larb_gen->flags_general; + const u8 *larbostd = larb->larb_gen->ostd ? larb->larb_gen->ostd[larb->larbid] : NULL; + struct arm_smccc_res res; + int i; + + if (BIT(larb->larbid) & larb->larb_gen->larb_direct_to_common_mask) + return 0; + + if (MTK_SMI_CAPS(flags_general, MTK_SMI_FLAG_THRT_UPDATE)) { + reg = readl_relaxed(larb->base + SMI_LARB_CMD_THRT_CON); + reg &= ~SMI_LARB_THRT_RD_NU_LMT_MSK; + reg |= SMI_LARB_THRT_RD_NU_LMT; + writel_relaxed(reg, larb->base + SMI_LARB_CMD_THRT_CON); + } + + if (MTK_SMI_CAPS(flags_general, MTK_SMI_FLAG_SW_FLAG)) + writel_relaxed(SMI_LARB_SW_FLAG_1, larb->base + SMI_LARB_SW_FLAG); + + for (i = 0; i < SMI_LARB_PORT_NR_MAX && larbostd && !!larbostd[i]; i++) + writel_relaxed(larbostd[i], larb->base + SMI_LARB_OSTDL_PORTx(i)); + + /* + * When mmu_en bits are in security world, the bank_sel still is in the + * LARB_NONSEC_CON below. And the mmu_en bits of LARB_NONSEC_CON have no + * effect in this case. + */ + if (MTK_SMI_CAPS(flags_general, MTK_SMI_FLAG_CFG_PORT_SEC_CTL)) { + arm_smccc_smc(MTK_SIP_KERNEL_IOMMU_CONTROL, IOMMU_ATF_CMD_CONFIG_SMI_LARB, + larb->larbid, *larb->mmu, 0, 0, 0, 0, &res); + if (res.a0 != 0) { + dev_err(dev, "Enable iommu fail, ret %ld\n", res.a0); + return -EINVAL; + } + } + + for_each_set_bit(i, (unsigned long *)larb->mmu, 32) { + reg = readl_relaxed(larb->base + SMI_LARB_NONSEC_CON(i)); + reg |= F_MMU_EN; + reg |= BANK_SEL(larb->bank[i]); + writel(reg, larb->base + SMI_LARB_NONSEC_CON(i)); + } + return 0; +} + +static const u8 mtk_smi_larb_mt8188_ostd[][SMI_LARB_PORT_NR_MAX] = { + [0] = {0x02, 0x18, 0x22, 0x22, 0x01, 0x02, 0x0a,}, + [1] = {0x12, 0x02, 0x14, 0x14, 0x01, 0x18, 0x0a,}, + [2] = {0x12, 0x12, 0x12, 0x12, 0x0a,}, + [3] = {0x12, 0x12, 0x12, 0x12, 0x28, 0x28, 0x0a,}, + [4] = {0x06, 0x01, 0x17, 0x06, 0x0a, 0x07, 0x07,}, + [5] = {0x02, 0x01, 0x04, 0x02, 0x06, 0x01, 0x06, 0x0a,}, + [6] = {0x06, 0x01, 0x06, 0x0a,}, + [7] = {0x0c, 0x0c, 0x12,}, + [8] = {0x0c, 0x01, 0x0a, 0x05, 0x02, 0x03, 0x01, 0x01, 0x14, 0x14, + 0x0a, 0x14, 0x1e, 0x01, 0x0c, 0x0a, 0x05, 0x02, 0x02, 0x05, + 0x03, 0x01, 0x1e, 0x01, 0x05,}, + [9] = {0x1e, 0x01, 0x0a, 0x0a, 0x01, 0x01, 0x03, 0x1e, 0x1e, 0x10, + 0x07, 0x01, 0x0a, 0x06, 0x03, 0x03, 0x0e, 0x01, 0x04, 0x28,}, + [10] = {0x03, 0x20, 0x01, 0x20, 0x01, 0x01, 0x14, 0x0a, 0x0a, 0x0c, + 0x0a, 0x05, 0x02, 0x03, 0x02, 0x14, 0x0a, 0x0a, 0x14, 0x14, + 0x14, 0x01, 0x01, 0x14, 0x1e, 0x01, 0x05, 0x03, 0x02, 0x28,}, + [11] = {0x03, 0x20, 0x01, 0x20, 0x01, 0x01, 0x14, 0x0a, 0x0a, 0x0c, + 0x0a, 0x05, 0x02, 0x03, 0x02, 0x14, 0x0a, 0x0a, 0x14, 0x14, + 0x14, 0x01, 0x01, 0x14, 0x1e, 0x01, 0x05, 0x03, 0x02, 0x28,}, + [12] = {0x03, 0x20, 0x01, 0x20, 0x01, 0x01, 0x14, 0x0a, 0x0a, 0x0c, + 0x0a, 0x05, 0x02, 0x03, 0x02, 0x14, 0x0a, 0x0a, 0x14, 0x14, + 0x14, 0x01, 0x01, 0x14, 0x1e, 0x01, 0x05, 0x03, 0x02, 0x28,}, + [13] = {0x07, 0x02, 0x04, 0x02, 0x05, 0x05, 0x05, 0x05, 0x05, 0x05, + 0x07, 0x02, 0x04, 0x02, 0x05, 0x05,}, + [14] = {0x02, 0x02, 0x0c, 0x0c, 0x0c, 0x0c, 0x01, 0x01, 0x02, 0x02, + 0x02, 0x02, 0x0c, 0x0c, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, + 0x02, 0x02, 0x01, 0x01,}, + [15] = {0x0c, 0x0c, 0x02, 0x02, 0x02, 0x02, 0x01, 0x01, 0x0c, 0x0c, + 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x01, 0x02, + 0x0c, 0x01, 0x01,}, + [16] = {0x28, 0x28, 0x03, 0x01, 0x01, 0x03, 0x14, 0x14, 0x0a, 0x0d, + 0x03, 0x05, 0x0e, 0x01, 0x01, 0x05, 0x06, 0x0d, 0x01,}, + [17] = {0x28, 0x02, 0x02, 0x12, 0x02, 0x12, 0x10, 0x02, 0x02, 0x0a, + 0x12, 0x02, 0x02, 0x0a, 0x16, 0x02, 0x04,}, + [18] = {0x28, 0x02, 0x02, 0x12, 0x02, 0x12, 0x10, 0x02, 0x02, 0x0a, + 0x12, 0x02, 0x02, 0x0a, 0x16, 0x02, 0x04,}, + [19] = {0x1a, 0x0e, 0x0a, 0x0a, 0x0c, 0x0e, 0x10,}, + [20] = {0x1a, 0x0e, 0x0a, 0x0a, 0x0c, 0x0e, 0x10,}, + [21] = {0x01, 0x04, 0x01, 0x01, 0x01, 0x01, 0x01, 0x04, 0x04, 0x01, + 0x01, 0x01, 0x04, 0x0a, 0x06, 0x01, 0x01, 0x01, 0x0a, 0x06, + 0x01, 0x01, 0x05, 0x03, 0x03, 0x04, 0x01,}, + [22] = {0x28, 0x19, 0x0c, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x04, + 0x01,}, + [23] = {0x01, 0x01, 0x04, 0x01, 0x01, 0x01, 0x18, 0x01, 0x01,}, + [24] = {0x12, 0x06, 0x12, 0x06,}, + [25] = {0x01}, }; -static const struct mtk_smi_larb_gen mtk_smi_larb_mt8173 = { - /* mt8173 do not need the port in larb */ - .config_port = mtk_smi_larb_config_port_mt8173, +static const u8 mtk_smi_larb_mt8195_ostd[][SMI_LARB_PORT_NR_MAX] = { + [0] = {0x0a, 0xc, 0x22, 0x22, 0x01, 0x0a,}, /* larb0 */ + [1] = {0x0a, 0xc, 0x22, 0x22, 0x01, 0x0a,}, /* larb1 */ + [2] = {0x12, 0x12, 0x12, 0x12, 0x0a,}, /* ... */ + [3] = {0x12, 0x12, 0x12, 0x12, 0x28, 0x28, 0x0a,}, + [4] = {0x06, 0x01, 0x17, 0x06, 0x0a,}, + [5] = {0x06, 0x01, 0x17, 0x06, 0x06, 0x01, 0x06, 0x0a,}, + [6] = {0x06, 0x01, 0x06, 0x0a,}, + [7] = {0x0c, 0x0c, 0x12,}, + [8] = {0x0c, 0x0c, 0x12,}, + [9] = {0x0a, 0x08, 0x04, 0x06, 0x01, 0x01, 0x10, 0x18, 0x11, 0x0a, + 0x08, 0x04, 0x11, 0x06, 0x02, 0x06, 0x01, 0x11, 0x11, 0x06,}, + [10] = {0x18, 0x08, 0x01, 0x01, 0x20, 0x12, 0x18, 0x06, 0x05, 0x10, + 0x08, 0x08, 0x10, 0x08, 0x08, 0x18, 0x0c, 0x09, 0x0b, 0x0d, + 0x0d, 0x06, 0x10, 0x10,}, + [11] = {0x0e, 0x0e, 0x0e, 0x0e, 0x0e, 0x0e, 0x01, 0x01, 0x01, 0x01,}, + [12] = {0x09, 0x09, 0x05, 0x05, 0x0c, 0x18, 0x02, 0x02, 0x04, 0x02,}, + [13] = {0x02, 0x02, 0x12, 0x12, 0x02, 0x02, 0x02, 0x02, 0x08, 0x01,}, + [14] = {0x12, 0x12, 0x02, 0x02, 0x02, 0x02, 0x16, 0x01, 0x16, 0x01, + 0x01, 0x02, 0x02, 0x08, 0x02,}, + [15] = {}, + [16] = {0x28, 0x02, 0x02, 0x12, 0x02, 0x12, 0x10, 0x02, 0x02, 0x0a, + 0x12, 0x02, 0x0a, 0x16, 0x02, 0x04,}, + [17] = {0x1a, 0x0e, 0x0a, 0x0a, 0x0c, 0x0e, 0x10,}, + [18] = {0x12, 0x06, 0x12, 0x06,}, + [19] = {0x01, 0x04, 0x01, 0x01, 0x01, 0x01, 0x01, 0x04, 0x04, 0x01, + 0x01, 0x01, 0x04, 0x0a, 0x06, 0x01, 0x01, 0x01, 0x0a, 0x06, + 0x01, 0x01, 0x05, 0x03, 0x03, 0x04, 0x01,}, + [20] = {0x01, 0x04, 0x01, 0x01, 0x01, 0x01, 0x01, 0x04, 0x04, 0x01, + 0x01, 0x01, 0x04, 0x0a, 0x06, 0x01, 0x01, 0x01, 0x0a, 0x06, + 0x01, 0x01, 0x05, 0x03, 0x03, 0x04, 0x01,}, + [21] = {0x28, 0x19, 0x0c, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x04,}, + [22] = {0x28, 0x19, 0x0c, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x04,}, + [23] = {0x18, 0x01,}, + [24] = {0x01, 0x01, 0x04, 0x01, 0x01, 0x01, 0x01, 0x01, 0x04, 0x01, + 0x01, 0x01,}, + [25] = {0x02, 0x02, 0x02, 0x28, 0x16, 0x02, 0x02, 0x02, 0x12, 0x16, + 0x02, 0x01,}, + [26] = {0x02, 0x02, 0x02, 0x28, 0x16, 0x02, 0x02, 0x02, 0x12, 0x16, + 0x02, 0x01,}, + [27] = {0x02, 0x02, 0x02, 0x28, 0x16, 0x02, 0x02, 0x02, 0x12, 0x16, + 0x02, 0x01,}, + [28] = {0x1a, 0x0e, 0x0a, 0x0a, 0x0c, 0x0e, 0x10,}, }; static const struct mtk_smi_larb_gen mtk_smi_larb_mt2701 = { @@ -239,91 +390,187 @@ static const struct mtk_smi_larb_gen mtk_smi_larb_mt2712 = { .larb_direct_to_common_mask = BIT(8) | BIT(9), /* bdpsys */ }; +static const struct mtk_smi_larb_gen mtk_smi_larb_mt6779 = { + .config_port = mtk_smi_larb_config_port_gen2_general, + .larb_direct_to_common_mask = + BIT(4) | BIT(6) | BIT(11) | BIT(12) | BIT(13), + /* DUMMY | IPU0 | IPU1 | CCU | MDLA */ +}; + +static const struct mtk_smi_larb_gen mtk_smi_larb_mt8167 = { + /* mt8167 do not need the port in larb */ + .config_port = mtk_smi_larb_config_port_mt8167, +}; + +static const struct mtk_smi_larb_gen mtk_smi_larb_mt8173 = { + /* mt8173 do not need the port in larb */ + .config_port = mtk_smi_larb_config_port_mt8173, +}; + static const struct mtk_smi_larb_gen mtk_smi_larb_mt8183 = { - .has_gals = true, .config_port = mtk_smi_larb_config_port_gen2_general, .larb_direct_to_common_mask = BIT(2) | BIT(3) | BIT(7), /* IPU0 | IPU1 | CCU */ }; +static const struct mtk_smi_larb_gen mtk_smi_larb_mt8186 = { + .config_port = mtk_smi_larb_config_port_gen2_general, + .flags_general = MTK_SMI_FLAG_SLEEP_CTL, +}; + +static const struct mtk_smi_larb_gen mtk_smi_larb_mt8188 = { + .config_port = mtk_smi_larb_config_port_gen2_general, + .flags_general = MTK_SMI_FLAG_THRT_UPDATE | MTK_SMI_FLAG_SW_FLAG | + MTK_SMI_FLAG_SLEEP_CTL | MTK_SMI_FLAG_CFG_PORT_SEC_CTL, + .ostd = mtk_smi_larb_mt8188_ostd, +}; + +static const struct mtk_smi_larb_gen mtk_smi_larb_mt8192 = { + .config_port = mtk_smi_larb_config_port_gen2_general, +}; + +static const struct mtk_smi_larb_gen mtk_smi_larb_mt8195 = { + .config_port = mtk_smi_larb_config_port_gen2_general, + .flags_general = MTK_SMI_FLAG_THRT_UPDATE | MTK_SMI_FLAG_SW_FLAG | + MTK_SMI_FLAG_SLEEP_CTL, + .ostd = mtk_smi_larb_mt8195_ostd, +}; + static const struct of_device_id mtk_smi_larb_of_ids[] = { - { - .compatible = "mediatek,mt8173-smi-larb", - .data = &mtk_smi_larb_mt8173 - }, - { - .compatible = "mediatek,mt2701-smi-larb", - .data = &mtk_smi_larb_mt2701 - }, - { - .compatible = "mediatek,mt2712-smi-larb", - .data = &mtk_smi_larb_mt2712 - }, - { - .compatible = "mediatek,mt8183-smi-larb", - .data = &mtk_smi_larb_mt8183 - }, + {.compatible = "mediatek,mt2701-smi-larb", .data = &mtk_smi_larb_mt2701}, + {.compatible = "mediatek,mt2712-smi-larb", .data = &mtk_smi_larb_mt2712}, + {.compatible = "mediatek,mt6779-smi-larb", .data = &mtk_smi_larb_mt6779}, + {.compatible = "mediatek,mt6795-smi-larb", .data = &mtk_smi_larb_mt8173}, + {.compatible = "mediatek,mt8167-smi-larb", .data = &mtk_smi_larb_mt8167}, + {.compatible = "mediatek,mt8173-smi-larb", .data = &mtk_smi_larb_mt8173}, + {.compatible = "mediatek,mt8183-smi-larb", .data = &mtk_smi_larb_mt8183}, + {.compatible = "mediatek,mt8186-smi-larb", .data = &mtk_smi_larb_mt8186}, + {.compatible = "mediatek,mt8188-smi-larb", .data = &mtk_smi_larb_mt8188}, + {.compatible = "mediatek,mt8192-smi-larb", .data = &mtk_smi_larb_mt8192}, + {.compatible = "mediatek,mt8195-smi-larb", .data = &mtk_smi_larb_mt8195}, {} }; +static int mtk_smi_larb_sleep_ctrl_enable(struct mtk_smi_larb *larb) +{ + int ret; + u32 tmp; + + writel_relaxed(SLP_PROT_EN, larb->base + SMI_LARB_SLP_CON); + ret = readl_poll_timeout_atomic(larb->base + SMI_LARB_SLP_CON, + tmp, !!(tmp & SLP_PROT_RDY), 10, 1000); + if (ret) { + /* TODO: Reset this larb if it fails here. */ + dev_err(larb->smi.dev, "sleep ctrl is not ready(0x%x).\n", tmp); + } + return ret; +} + +static void mtk_smi_larb_sleep_ctrl_disable(struct mtk_smi_larb *larb) +{ + writel_relaxed(0, larb->base + SMI_LARB_SLP_CON); +} + +static int mtk_smi_device_link_common(struct device *dev, struct device **com_dev) +{ + struct platform_device *smi_com_pdev; + struct device_node *smi_com_node; + struct device *smi_com_dev; + struct device_link *link; + + smi_com_node = of_parse_phandle(dev->of_node, "mediatek,smi", 0); + if (!smi_com_node) + return -EINVAL; + + smi_com_pdev = of_find_device_by_node(smi_com_node); + of_node_put(smi_com_node); + if (smi_com_pdev) { + /* smi common is the supplier, Make sure it is ready before */ + if (!platform_get_drvdata(smi_com_pdev)) { + put_device(&smi_com_pdev->dev); + return -EPROBE_DEFER; + } + smi_com_dev = &smi_com_pdev->dev; + link = device_link_add(dev, smi_com_dev, + DL_FLAG_PM_RUNTIME | DL_FLAG_STATELESS); + if (!link) { + dev_err(dev, "Unable to link smi-common dev\n"); + put_device(&smi_com_pdev->dev); + return -ENODEV; + } + *com_dev = smi_com_dev; + } else { + dev_err(dev, "Failed to get the smi_common device\n"); + return -EINVAL; + } + return 0; +} + +static int mtk_smi_dts_clk_init(struct device *dev, struct mtk_smi *smi, + const char * const clks[], + unsigned int clk_nr_required, + unsigned int clk_nr_optional) +{ + int i, ret; + + for (i = 0; i < clk_nr_required; i++) + smi->clks[i].id = clks[i]; + ret = devm_clk_bulk_get(dev, clk_nr_required, smi->clks); + if (ret) + return ret; + + for (i = clk_nr_required; i < clk_nr_required + clk_nr_optional; i++) + smi->clks[i].id = clks[i]; + ret = devm_clk_bulk_get_optional(dev, clk_nr_optional, + smi->clks + clk_nr_required); + smi->clk_num = clk_nr_required + clk_nr_optional; + return ret; +} + static int mtk_smi_larb_probe(struct platform_device *pdev) { struct mtk_smi_larb *larb; - struct resource *res; struct device *dev = &pdev->dev; - struct device_node *smi_node; - struct platform_device *smi_pdev; + int ret; larb = devm_kzalloc(dev, sizeof(*larb), GFP_KERNEL); if (!larb) return -ENOMEM; larb->larb_gen = of_device_get_match_data(dev); - res = platform_get_resource(pdev, IORESOURCE_MEM, 0); - larb->base = devm_ioremap_resource(dev, res); + larb->base = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(larb->base)) return PTR_ERR(larb->base); - larb->smi.clk_apb = devm_clk_get(dev, "apb"); - if (IS_ERR(larb->smi.clk_apb)) - return PTR_ERR(larb->smi.clk_apb); - - larb->smi.clk_smi = devm_clk_get(dev, "smi"); - if (IS_ERR(larb->smi.clk_smi)) - return PTR_ERR(larb->smi.clk_smi); - - if (larb->larb_gen->has_gals) { - /* The larbs may still haven't gals even if the SoC support.*/ - larb->smi.clk_gals0 = devm_clk_get(dev, "gals"); - if (PTR_ERR(larb->smi.clk_gals0) == -ENOENT) - larb->smi.clk_gals0 = NULL; - else if (IS_ERR(larb->smi.clk_gals0)) - return PTR_ERR(larb->smi.clk_gals0); - } - larb->smi.dev = dev; + ret = mtk_smi_dts_clk_init(dev, &larb->smi, mtk_smi_larb_clks, + MTK_SMI_LARB_REQ_CLK_NR, MTK_SMI_LARB_OPT_CLK_NR); + if (ret) + return ret; - smi_node = of_parse_phandle(dev->of_node, "mediatek,smi", 0); - if (!smi_node) - return -EINVAL; + larb->smi.dev = dev; - smi_pdev = of_find_device_by_node(smi_node); - of_node_put(smi_node); - if (smi_pdev) { - if (!platform_get_drvdata(smi_pdev)) - return -EPROBE_DEFER; - larb->smi_common_dev = &smi_pdev->dev; - } else { - dev_err(dev, "Failed to get the smi_common device\n"); - return -EINVAL; - } + ret = mtk_smi_device_link_common(dev, &larb->smi_common_dev); + if (ret < 0) + return ret; pm_runtime_enable(dev); platform_set_drvdata(pdev, larb); - return component_add(dev, &mtk_smi_larb_component_ops); + ret = component_add(dev, &mtk_smi_larb_component_ops); + if (ret) + goto err_pm_disable; + return 0; + +err_pm_disable: + pm_runtime_disable(dev); + device_link_remove(dev, larb->smi_common_dev); + return ret; } static int mtk_smi_larb_remove(struct platform_device *pdev) { + struct mtk_smi_larb *larb = platform_get_drvdata(pdev); + + device_link_remove(&pdev->dev, larb->smi_common_dev); pm_runtime_disable(&pdev->dev); component_del(&pdev->dev, &mtk_smi_larb_component_ops); return 0; @@ -335,32 +582,29 @@ static int __maybe_unused mtk_smi_larb_resume(struct device *dev) const struct mtk_smi_larb_gen *larb_gen = larb->larb_gen; int ret; - /* Power on smi-common. */ - ret = pm_runtime_get_sync(larb->smi_common_dev); - if (ret < 0) { - dev_err(dev, "Failed to pm get for smi-common(%d).\n", ret); + ret = clk_bulk_prepare_enable(larb->smi.clk_num, larb->smi.clks); + if (ret) return ret; - } - ret = mtk_smi_clk_enable(&larb->smi); - if (ret < 0) { - dev_err(dev, "Failed to enable clock(%d).\n", ret); - pm_runtime_put_sync(larb->smi_common_dev); - return ret; - } + if (MTK_SMI_CAPS(larb->larb_gen->flags_general, MTK_SMI_FLAG_SLEEP_CTL)) + mtk_smi_larb_sleep_ctrl_disable(larb); /* Configure the basic setting for this larb */ - larb_gen->config_port(dev); - - return 0; + return larb_gen->config_port(dev); } static int __maybe_unused mtk_smi_larb_suspend(struct device *dev) { struct mtk_smi_larb *larb = dev_get_drvdata(dev); + int ret; - mtk_smi_clk_disable(&larb->smi); - pm_runtime_put_sync(larb->smi_common_dev); + if (MTK_SMI_CAPS(larb->larb_gen->flags_general, MTK_SMI_FLAG_SLEEP_CTL)) { + ret = mtk_smi_larb_sleep_ctrl_enable(larb); + if (ret) + return ret; + } + + clk_bulk_disable_unprepare(larb->smi.clk_num, larb->smi.clks); return 0; } @@ -380,38 +624,110 @@ static struct platform_driver mtk_smi_larb_driver = { } }; +static const struct mtk_smi_reg_pair mtk_smi_common_mt6795_init[SMI_COMMON_INIT_REGS_NR] = { + {SMI_L1_ARB, 0x1b}, + {SMI_M4U_TH, 0xce810c85}, + {SMI_FIFO_TH1, 0x43214c8}, + {SMI_READ_FIFO_TH, 0x191f}, +}; + +static const struct mtk_smi_reg_pair mtk_smi_common_mt8195_init[SMI_COMMON_INIT_REGS_NR] = { + {SMI_L1LEN, 0xb}, + {SMI_M4U_TH, 0xe100e10}, + {SMI_FIFO_TH1, 0x506090a}, + {SMI_FIFO_TH2, 0x506090a}, + {SMI_DCM, 0x4f1}, + {SMI_DUMMY, 0x1}, +}; + static const struct mtk_smi_common_plat mtk_smi_common_gen1 = { - .gen = MTK_SMI_GEN1, + .type = MTK_SMI_GEN1, }; static const struct mtk_smi_common_plat mtk_smi_common_gen2 = { - .gen = MTK_SMI_GEN2, + .type = MTK_SMI_GEN2, +}; + +static const struct mtk_smi_common_plat mtk_smi_common_mt6779 = { + .type = MTK_SMI_GEN2, + .has_gals = true, + .bus_sel = F_MMU1_LARB(1) | F_MMU1_LARB(2) | F_MMU1_LARB(4) | + F_MMU1_LARB(5) | F_MMU1_LARB(6) | F_MMU1_LARB(7), +}; + +static const struct mtk_smi_common_plat mtk_smi_common_mt6795 = { + .type = MTK_SMI_GEN2, + .bus_sel = F_MMU1_LARB(0), + .init = mtk_smi_common_mt6795_init, }; static const struct mtk_smi_common_plat mtk_smi_common_mt8183 = { - .gen = MTK_SMI_GEN2, + .type = MTK_SMI_GEN2, .has_gals = true, .bus_sel = F_MMU1_LARB(1) | F_MMU1_LARB(2) | F_MMU1_LARB(5) | F_MMU1_LARB(7), }; +static const struct mtk_smi_common_plat mtk_smi_common_mt8186 = { + .type = MTK_SMI_GEN2, + .has_gals = true, + .bus_sel = F_MMU1_LARB(1) | F_MMU1_LARB(4) | F_MMU1_LARB(7), +}; + +static const struct mtk_smi_common_plat mtk_smi_common_mt8188_vdo = { + .type = MTK_SMI_GEN2, + .bus_sel = F_MMU1_LARB(1) | F_MMU1_LARB(5) | F_MMU1_LARB(7), + .init = mtk_smi_common_mt8195_init, +}; + +static const struct mtk_smi_common_plat mtk_smi_common_mt8188_vpp = { + .type = MTK_SMI_GEN2, + .bus_sel = F_MMU1_LARB(1) | F_MMU1_LARB(2) | F_MMU1_LARB(7), + .init = mtk_smi_common_mt8195_init, +}; + +static const struct mtk_smi_common_plat mtk_smi_common_mt8192 = { + .type = MTK_SMI_GEN2, + .has_gals = true, + .bus_sel = F_MMU1_LARB(1) | F_MMU1_LARB(2) | F_MMU1_LARB(5) | + F_MMU1_LARB(6), +}; + +static const struct mtk_smi_common_plat mtk_smi_common_mt8195_vdo = { + .type = MTK_SMI_GEN2, + .has_gals = true, + .bus_sel = F_MMU1_LARB(1) | F_MMU1_LARB(3) | F_MMU1_LARB(5) | + F_MMU1_LARB(7), + .init = mtk_smi_common_mt8195_init, +}; + +static const struct mtk_smi_common_plat mtk_smi_common_mt8195_vpp = { + .type = MTK_SMI_GEN2, + .has_gals = true, + .bus_sel = F_MMU1_LARB(1) | F_MMU1_LARB(2) | F_MMU1_LARB(7), + .init = mtk_smi_common_mt8195_init, +}; + +static const struct mtk_smi_common_plat mtk_smi_sub_common_mt8195 = { + .type = MTK_SMI_GEN2_SUB_COMM, + .has_gals = true, +}; + static const struct of_device_id mtk_smi_common_of_ids[] = { - { - .compatible = "mediatek,mt8173-smi-common", - .data = &mtk_smi_common_gen2, - }, - { - .compatible = "mediatek,mt2701-smi-common", - .data = &mtk_smi_common_gen1, - }, - { - .compatible = "mediatek,mt2712-smi-common", - .data = &mtk_smi_common_gen2, - }, - { - .compatible = "mediatek,mt8183-smi-common", - .data = &mtk_smi_common_mt8183, - }, + {.compatible = "mediatek,mt2701-smi-common", .data = &mtk_smi_common_gen1}, + {.compatible = "mediatek,mt2712-smi-common", .data = &mtk_smi_common_gen2}, + {.compatible = "mediatek,mt6779-smi-common", .data = &mtk_smi_common_mt6779}, + {.compatible = "mediatek,mt6795-smi-common", .data = &mtk_smi_common_mt6795}, + {.compatible = "mediatek,mt8167-smi-common", .data = &mtk_smi_common_gen2}, + {.compatible = "mediatek,mt8173-smi-common", .data = &mtk_smi_common_gen2}, + {.compatible = "mediatek,mt8183-smi-common", .data = &mtk_smi_common_mt8183}, + {.compatible = "mediatek,mt8186-smi-common", .data = &mtk_smi_common_mt8186}, + {.compatible = "mediatek,mt8188-smi-common-vdo", .data = &mtk_smi_common_mt8188_vdo}, + {.compatible = "mediatek,mt8188-smi-common-vpp", .data = &mtk_smi_common_mt8188_vpp}, + {.compatible = "mediatek,mt8192-smi-common", .data = &mtk_smi_common_mt8192}, + {.compatible = "mediatek,mt8195-smi-common-vdo", .data = &mtk_smi_common_mt8195_vdo}, + {.compatible = "mediatek,mt8195-smi-common-vpp", .data = &mtk_smi_common_mt8195_vpp}, + {.compatible = "mediatek,mt8195-smi-sub-common", .data = &mtk_smi_sub_common_mt8195}, {} }; @@ -419,8 +735,7 @@ static int mtk_smi_common_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; struct mtk_smi *common; - struct resource *res; - int ret; + int ret, clk_required = MTK_SMI_COM_REQ_CLK_NR; common = devm_kzalloc(dev, sizeof(*common), GFP_KERNEL); if (!common) @@ -428,23 +743,15 @@ static int mtk_smi_common_probe(struct platform_device *pdev) common->dev = dev; common->plat = of_device_get_match_data(dev); - common->clk_apb = devm_clk_get(dev, "apb"); - if (IS_ERR(common->clk_apb)) - return PTR_ERR(common->clk_apb); - - common->clk_smi = devm_clk_get(dev, "smi"); - if (IS_ERR(common->clk_smi)) - return PTR_ERR(common->clk_smi); - if (common->plat->has_gals) { - common->clk_gals0 = devm_clk_get(dev, "gals0"); - if (IS_ERR(common->clk_gals0)) - return PTR_ERR(common->clk_gals0); - - common->clk_gals1 = devm_clk_get(dev, "gals1"); - if (IS_ERR(common->clk_gals1)) - return PTR_ERR(common->clk_gals1); + if (common->plat->type == MTK_SMI_GEN2) + clk_required = MTK_SMI_COM_GALS_REQ_CLK_NR; + else if (common->plat->type == MTK_SMI_GEN2_SUB_COMM) + clk_required = MTK_SMI_SUB_COM_GALS_REQ_CLK_NR; } + ret = mtk_smi_dts_clk_init(dev, common, mtk_smi_common_clks, clk_required, 0); + if (ret) + return ret; /* * for mtk smi gen 1, we need to get the ao(always on) base to config @@ -452,9 +759,8 @@ static int mtk_smi_common_probe(struct platform_device *pdev) * clock into emi clock domain, but for mtk smi gen2, there's no smi ao * base. */ - if (common->plat->gen == MTK_SMI_GEN1) { - res = platform_get_resource(pdev, IORESOURCE_MEM, 0); - common->smi_ao_base = devm_ioremap_resource(dev, res); + if (common->plat->type == MTK_SMI_GEN1) { + common->smi_ao_base = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(common->smi_ao_base)) return PTR_ERR(common->smi_ao_base); @@ -466,11 +772,18 @@ static int mtk_smi_common_probe(struct platform_device *pdev) if (ret) return ret; } else { - res = platform_get_resource(pdev, IORESOURCE_MEM, 0); - common->base = devm_ioremap_resource(dev, res); + common->base = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(common->base)) return PTR_ERR(common->base); } + + /* link its smi-common if this is smi-sub-common */ + if (common->plat->type == MTK_SMI_GEN2_SUB_COMM) { + ret = mtk_smi_device_link_common(dev, &common->smi_common_dev); + if (ret < 0) + return ret; + } + pm_runtime_enable(dev); platform_set_drvdata(pdev, common); return 0; @@ -478,6 +791,10 @@ static int mtk_smi_common_probe(struct platform_device *pdev) static int mtk_smi_common_remove(struct platform_device *pdev) { + struct mtk_smi *common = dev_get_drvdata(&pdev->dev); + + if (common->plat->type == MTK_SMI_GEN2_SUB_COMM) + device_link_remove(&pdev->dev, common->smi_common_dev); pm_runtime_disable(&pdev->dev); return 0; } @@ -485,17 +802,21 @@ static int mtk_smi_common_remove(struct platform_device *pdev) static int __maybe_unused mtk_smi_common_resume(struct device *dev) { struct mtk_smi *common = dev_get_drvdata(dev); - u32 bus_sel = common->plat->bus_sel; - int ret; + const struct mtk_smi_reg_pair *init = common->plat->init; + u32 bus_sel = common->plat->bus_sel; /* default is 0 */ + int ret, i; - ret = mtk_smi_clk_enable(common); - if (ret) { - dev_err(common->dev, "Failed to enable clock(%d).\n", ret); + ret = clk_bulk_prepare_enable(common->clk_num, common->clks); + if (ret) return ret; - } - if (common->plat->gen == MTK_SMI_GEN2 && bus_sel) - writel(bus_sel, common->base + SMI_BUS_SEL); + if (common->plat->type != MTK_SMI_GEN2) + return 0; + + for (i = 0; i < SMI_COMMON_INIT_REGS_NR && init && init[i].offset; i++) + writel_relaxed(init[i].value, common->base + init[i].offset); + + writel(bus_sel, common->base + SMI_BUS_SEL); return 0; } @@ -503,7 +824,7 @@ static int __maybe_unused mtk_smi_common_suspend(struct device *dev) { struct mtk_smi *common = dev_get_drvdata(dev); - mtk_smi_clk_disable(common); + clk_bulk_disable_unprepare(common->clk_num, common->clks); return 0; } @@ -523,26 +844,22 @@ static struct platform_driver mtk_smi_common_driver = { } }; +static struct platform_driver * const smidrivers[] = { + &mtk_smi_common_driver, + &mtk_smi_larb_driver, +}; + static int __init mtk_smi_init(void) { - int ret; - - ret = platform_driver_register(&mtk_smi_common_driver); - if (ret != 0) { - pr_err("Failed to register SMI driver\n"); - return ret; - } - - ret = platform_driver_register(&mtk_smi_larb_driver); - if (ret != 0) { - pr_err("Failed to register SMI-LARB driver\n"); - goto err_unreg_smi; - } - return ret; + return platform_register_drivers(smidrivers, ARRAY_SIZE(smidrivers)); +} +module_init(mtk_smi_init); -err_unreg_smi: - platform_driver_unregister(&mtk_smi_common_driver); - return ret; +static void __exit mtk_smi_exit(void) +{ + platform_unregister_drivers(smidrivers, ARRAY_SIZE(smidrivers)); } +module_exit(mtk_smi_exit); -module_init(mtk_smi_init); +MODULE_DESCRIPTION("MediaTek SMI driver"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/memory/mvebu-devbus.c b/drivers/memory/mvebu-devbus.c index 886aea587276..8450638e8670 100644 --- a/drivers/memory/mvebu-devbus.c +++ b/drivers/memory/mvebu-devbus.c @@ -124,32 +124,32 @@ static int devbus_get_timing_params(struct devbus *devbus, * The bus width is encoded into the register as 0 for 8 bits, * and 1 for 16 bits, so we do the necessary conversion here. */ - if (r->bus_width == 8) + if (r->bus_width == 8) { r->bus_width = 0; - else if (r->bus_width == 16) + } else if (r->bus_width == 16) { r->bus_width = 1; - else { + } else { dev_err(devbus->dev, "invalid bus width %d\n", r->bus_width); return -EINVAL; } err = get_timing_param_ps(devbus, node, "devbus,badr-skew-ps", - &r->badr_skew); + &r->badr_skew); if (err < 0) return err; err = get_timing_param_ps(devbus, node, "devbus,turn-off-ps", - &r->turn_off); + &r->turn_off); if (err < 0) return err; err = get_timing_param_ps(devbus, node, "devbus,acc-first-ps", - &r->acc_first); + &r->acc_first); if (err < 0) return err; err = get_timing_param_ps(devbus, node, "devbus,acc-next-ps", - &r->acc_next); + &r->acc_next); if (err < 0) return err; @@ -175,17 +175,17 @@ static int devbus_get_timing_params(struct devbus *devbus, } err = get_timing_param_ps(devbus, node, "devbus,ale-wr-ps", - &w->ale_wr); + &w->ale_wr); if (err < 0) return err; err = get_timing_param_ps(devbus, node, "devbus,wr-low-ps", - &w->wr_low); + &w->wr_low); if (err < 0) return err; err = get_timing_param_ps(devbus, node, "devbus,wr-high-ps", - &w->wr_high); + &w->wr_high); if (err < 0) return err; diff --git a/drivers/memory/of_memory.c b/drivers/memory/of_memory.c index 71f26eac7350..fcd20d85d385 100644 --- a/drivers/memory/of_memory.c +++ b/drivers/memory/of_memory.c @@ -4,11 +4,10 @@ * * Copyright (C) 2012 Texas Instruments, Inc. * Copyright (C) 2019 Samsung Electronics Co., Ltd. + * Copyright (C) 2020 Krzysztof Kozlowski <krzk@kernel.org> */ #include <linux/device.h> -#include <linux/platform_device.h> -#include <linux/list.h> #include <linux/of.h> #include <linux/gfp.h> #include <linux/export.h> @@ -19,7 +18,7 @@ /** * of_get_min_tck() - extract min timing values for ddr * @np: pointer to ddr device tree node - * @device: device requesting for min timing values + * @dev: device requesting for min timing values * * Populates the lpddr2_min_tck structure by extracting data * from device tree node. Returns a pointer to the populated @@ -27,7 +26,7 @@ * default min timings provided by JEDEC. */ const struct lpddr2_min_tck *of_get_min_tck(struct device_node *np, - struct device *dev) + struct device *dev) { int ret = 0; struct lpddr2_min_tck *min; @@ -56,13 +55,13 @@ const struct lpddr2_min_tck *of_get_min_tck(struct device_node *np, return min; default_min_tck: - dev_warn(dev, "%s: using default min-tck values\n", __func__); + dev_warn(dev, "Using default min-tck values\n"); return &lpddr2_jedec_min_tck; } EXPORT_SYMBOL(of_get_min_tck); static int of_do_get_timings(struct device_node *np, - struct lpddr2_timings *tim) + struct lpddr2_timings *tim) { int ret; @@ -84,7 +83,7 @@ static int of_do_get_timings(struct device_node *np, ret |= of_property_read_u32(np, "tZQinit", &tim->tZQinit); ret |= of_property_read_u32(np, "tRAS-max-ns", &tim->tRAS_max_ns); ret |= of_property_read_u32(np, "tDQSCK-max-derated", - &tim->tDQSCK_max_derated); + &tim->tDQSCK_max_derated); return ret; } @@ -103,7 +102,9 @@ static int of_do_get_timings(struct device_node *np, * while populating, returns default timings provided by JEDEC. */ const struct lpddr2_timings *of_get_ddr_timings(struct device_node *np_ddr, - struct device *dev, u32 device_type, u32 *nr_frequencies) + struct device *dev, + u32 device_type, + u32 *nr_frequencies) { struct lpddr2_timings *timings = NULL; u32 arr_sz = 0, i = 0; @@ -116,7 +117,7 @@ const struct lpddr2_timings *of_get_ddr_timings(struct device_node *np_ddr, tim_compat = "jedec,lpddr2-timings"; break; default: - dev_warn(dev, "%s: un-supported memory type\n", __func__); + dev_warn(dev, "Unsupported memory type\n"); } for_each_child_of_node(np_ddr, np_tim) @@ -133,6 +134,7 @@ const struct lpddr2_timings *of_get_ddr_timings(struct device_node *np_ddr, for_each_child_of_node(np_ddr, np_tim) { if (of_device_is_compatible(np_tim, tim_compat)) { if (of_do_get_timings(np_tim, &timings[i])) { + of_node_put(np_tim); devm_kfree(dev, timings); goto default_timings; } @@ -145,7 +147,7 @@ const struct lpddr2_timings *of_get_ddr_timings(struct device_node *np_ddr, return timings; default_timings: - dev_warn(dev, "%s: using default timings\n", __func__); + dev_warn(dev, "Using default memory timings\n"); *nr_frequencies = ARRAY_SIZE(lpddr2_jedec_timings); return lpddr2_jedec_timings; } @@ -154,7 +156,7 @@ EXPORT_SYMBOL(of_get_ddr_timings); /** * of_lpddr3_get_min_tck() - extract min timing values for lpddr3 * @np: pointer to ddr device tree node - * @device: device requesting for min timing values + * @dev: device requesting for min timing values * * Populates the lpddr3_min_tck structure by extracting data * from device tree node. Returns a pointer to the populated @@ -193,8 +195,7 @@ const struct lpddr3_min_tck *of_lpddr3_get_min_tck(struct device_node *np, ret |= of_property_read_u32(np, "tMRD-min-tck", &min->tMRD); if (ret) { - dev_warn(dev, "%s: errors while parsing min-tck values\n", - __func__); + dev_warn(dev, "Errors while parsing min-tck values\n"); devm_kfree(dev, min); goto default_min_tck; } @@ -202,7 +203,7 @@ const struct lpddr3_min_tck *of_lpddr3_get_min_tck(struct device_node *np, return min; default_min_tck: - dev_warn(dev, "%s: using default min-tck values\n", __func__); + dev_warn(dev, "Using default min-tck values\n"); return NULL; } EXPORT_SYMBOL(of_lpddr3_get_min_tck); @@ -212,8 +213,10 @@ static int of_lpddr3_do_get_timings(struct device_node *np, { int ret; - /* The 'reg' param required since DT has changed, used as 'max-freq' */ - ret = of_property_read_u32(np, "reg", &tim->max_freq); + ret = of_property_read_u32(np, "max-freq", &tim->max_freq); + if (ret) + /* Deprecated way of passing max-freq as 'reg' */ + ret = of_property_read_u32(np, "reg", &tim->max_freq); ret |= of_property_read_u32(np, "min-freq", &tim->min_freq); ret |= of_property_read_u32(np, "tRFC", &tim->tRFC); ret |= of_property_read_u32(np, "tRRD", &tim->tRRD); @@ -264,7 +267,7 @@ const struct lpddr3_timings tim_compat = "jedec,lpddr3-timings"; break; default: - dev_warn(dev, "%s: un-supported memory type\n", __func__); + dev_warn(dev, "Unsupported memory type\n"); } for_each_child_of_node(np_ddr, np_tim) @@ -282,6 +285,7 @@ const struct lpddr3_timings if (of_device_is_compatible(np_tim, tim_compat)) { if (of_lpddr3_do_get_timings(np_tim, &timings[i])) { devm_kfree(dev, timings); + of_node_put(np_tim); goto default_timings; } i++; @@ -293,8 +297,102 @@ const struct lpddr3_timings return timings; default_timings: - dev_warn(dev, "%s: failed to get timings\n", __func__); + dev_warn(dev, "Failed to get timings\n"); *nr_frequencies = 0; return NULL; } EXPORT_SYMBOL(of_lpddr3_get_ddr_timings); + +/** + * of_lpddr2_get_info() - extracts information about the lpddr2 chip. + * @np: Pointer to device tree node containing lpddr2 info + * @dev: Device requesting info + * + * Populates lpddr2_info structure by extracting data from device + * tree node. Returns pointer to populated structure. If error + * happened while populating, returns NULL. If property is missing + * in a device-tree, then the corresponding value is set to -ENOENT. + */ +const struct lpddr2_info +*of_lpddr2_get_info(struct device_node *np, struct device *dev) +{ + struct lpddr2_info *ret_info, info = {}; + struct property *prop; + const char *cp; + int err; + u32 revision_id[2]; + + err = of_property_read_u32_array(np, "revision-id", revision_id, 2); + if (!err) { + info.revision_id1 = revision_id[0]; + info.revision_id2 = revision_id[1]; + } else { + err = of_property_read_u32(np, "revision-id1", &info.revision_id1); + if (err) + info.revision_id1 = -ENOENT; + + err = of_property_read_u32(np, "revision-id2", &info.revision_id2); + if (err) + info.revision_id2 = -ENOENT; + } + + err = of_property_read_u32(np, "io-width", &info.io_width); + if (err) + return NULL; + + info.io_width = 32 / info.io_width - 1; + + err = of_property_read_u32(np, "density", &info.density); + if (err) + return NULL; + + info.density = ffs(info.density) - 7; + + if (of_device_is_compatible(np, "jedec,lpddr2-s4")) + info.arch_type = LPDDR2_TYPE_S4; + else if (of_device_is_compatible(np, "jedec,lpddr2-s2")) + info.arch_type = LPDDR2_TYPE_S2; + else if (of_device_is_compatible(np, "jedec,lpddr2-nvm")) + info.arch_type = LPDDR2_TYPE_NVM; + else + return NULL; + + prop = of_find_property(np, "compatible", NULL); + for (cp = of_prop_next_string(prop, NULL); cp; + cp = of_prop_next_string(prop, cp)) { + +#define OF_LPDDR2_VENDOR_CMP(compat, ID) \ + if (!of_compat_cmp(cp, compat ",", strlen(compat ","))) { \ + info.manufacturer_id = LPDDR2_MANID_##ID; \ + break; \ + } + + OF_LPDDR2_VENDOR_CMP("samsung", SAMSUNG) + OF_LPDDR2_VENDOR_CMP("qimonda", QIMONDA) + OF_LPDDR2_VENDOR_CMP("elpida", ELPIDA) + OF_LPDDR2_VENDOR_CMP("etron", ETRON) + OF_LPDDR2_VENDOR_CMP("nanya", NANYA) + OF_LPDDR2_VENDOR_CMP("hynix", HYNIX) + OF_LPDDR2_VENDOR_CMP("mosel", MOSEL) + OF_LPDDR2_VENDOR_CMP("winbond", WINBOND) + OF_LPDDR2_VENDOR_CMP("esmt", ESMT) + OF_LPDDR2_VENDOR_CMP("spansion", SPANSION) + OF_LPDDR2_VENDOR_CMP("sst", SST) + OF_LPDDR2_VENDOR_CMP("zmos", ZMOS) + OF_LPDDR2_VENDOR_CMP("intel", INTEL) + OF_LPDDR2_VENDOR_CMP("numonyx", NUMONYX) + OF_LPDDR2_VENDOR_CMP("micron", MICRON) + +#undef OF_LPDDR2_VENDOR_CMP + } + + if (!info.manufacturer_id) + info.manufacturer_id = -ENOENT; + + ret_info = devm_kzalloc(dev, sizeof(*ret_info), GFP_KERNEL); + if (ret_info) + *ret_info = info; + + return ret_info; +} +EXPORT_SYMBOL(of_lpddr2_get_info); diff --git a/drivers/memory/of_memory.h b/drivers/memory/of_memory.h index e39ecc4c733d..1c4e47fede8a 100644 --- a/drivers/memory/of_memory.h +++ b/drivers/memory/of_memory.h @@ -3,22 +3,26 @@ * OpenFirmware helpers for memory drivers * * Copyright (C) 2012 Texas Instruments, Inc. + * Copyright (C) 2020 Krzysztof Kozlowski <krzk@kernel.org> */ #ifndef __LINUX_MEMORY_OF_REG_H #define __LINUX_MEMORY_OF_REG_H #if defined(CONFIG_OF) && defined(CONFIG_DDR) -extern const struct lpddr2_min_tck *of_get_min_tck(struct device_node *np, - struct device *dev); -extern const struct lpddr2_timings - *of_get_ddr_timings(struct device_node *np_ddr, struct device *dev, - u32 device_type, u32 *nr_frequencies); -extern const struct lpddr3_min_tck - *of_lpddr3_get_min_tck(struct device_node *np, struct device *dev); -extern const struct lpddr3_timings - *of_lpddr3_get_ddr_timings(struct device_node *np_ddr, - struct device *dev, u32 device_type, u32 *nr_frequencies); +const struct lpddr2_min_tck *of_get_min_tck(struct device_node *np, + struct device *dev); +const struct lpddr2_timings *of_get_ddr_timings(struct device_node *np_ddr, + struct device *dev, + u32 device_type, u32 *nr_frequencies); +const struct lpddr3_min_tck *of_lpddr3_get_min_tck(struct device_node *np, + struct device *dev); +const struct lpddr3_timings * +of_lpddr3_get_ddr_timings(struct device_node *np_ddr, + struct device *dev, u32 device_type, u32 *nr_frequencies); + +const struct lpddr2_info *of_lpddr2_get_info(struct device_node *np, + struct device *dev); #else static inline const struct lpddr2_min_tck *of_get_min_tck(struct device_node *np, struct device *dev) @@ -45,6 +49,12 @@ static inline const struct lpddr3_timings { return NULL; } + +static inline const struct lpddr2_info + *of_lpddr2_get_info(struct device_node *np, struct device *dev) +{ + return NULL; +} #endif /* CONFIG_OF && CONFIG_DDR */ #endif /* __LINUX_MEMORY_OF_REG_ */ diff --git a/drivers/memory/omap-gpmc.c b/drivers/memory/omap-gpmc.c index eff26c1b1394..2351f2708da2 100644 --- a/drivers/memory/omap-gpmc.c +++ b/drivers/memory/omap-gpmc.c @@ -9,8 +9,10 @@ * Copyright (C) 2009 Texas Instruments * Added OMAP4 support - Santosh Shilimkar <santosh.shilimkar@ti.com> */ +#include <linux/cpu_pm.h> #include <linux/irq.h> #include <linux/kernel.h> +#include <linux/module.h> #include <linux/init.h> #include <linux/err.h> #include <linux/clk.h> @@ -29,11 +31,10 @@ #include <linux/of_platform.h> #include <linux/omap-gpmc.h> #include <linux/pm_runtime.h> +#include <linux/sizes.h> #include <linux/platform_data/mtd-nand-omap2.h> -#include <asm/mach-types.h> - #define DEVICE_NAME "omap-gpmc" /* GPMC register offsets */ @@ -108,8 +109,8 @@ #define ENABLE_PREFETCH (0x1 << 7) #define DMA_MPU_MODE 2 -#define GPMC_REVISION_MAJOR(l) ((l >> 4) & 0xf) -#define GPMC_REVISION_MINOR(l) (l & 0xf) +#define GPMC_REVISION_MAJOR(l) (((l) >> 4) & 0xf) +#define GPMC_REVISION_MINOR(l) ((l) & 0xf) #define GPMC_HAS_WR_ACCESS 0x1 #define GPMC_HAS_WR_DATA_MUX_BUS 0x2 @@ -140,27 +141,27 @@ #define GPMC_CONFIG1_WRITEMULTIPLE_SUPP (1 << 28) #define GPMC_CONFIG1_WRITETYPE_ASYNC (0 << 27) #define GPMC_CONFIG1_WRITETYPE_SYNC (1 << 27) -#define GPMC_CONFIG1_CLKACTIVATIONTIME(val) ((val & 3) << 25) +#define GPMC_CONFIG1_CLKACTIVATIONTIME(val) (((val) & 3) << 25) /** CLKACTIVATIONTIME Max Ticks */ #define GPMC_CONFIG1_CLKACTIVATIONTIME_MAX 2 -#define GPMC_CONFIG1_PAGE_LEN(val) ((val & 3) << 23) +#define GPMC_CONFIG1_PAGE_LEN(val) (((val) & 3) << 23) /** ATTACHEDDEVICEPAGELENGTH Max Value */ #define GPMC_CONFIG1_ATTACHEDDEVICEPAGELENGTH_MAX 2 #define GPMC_CONFIG1_WAIT_READ_MON (1 << 22) #define GPMC_CONFIG1_WAIT_WRITE_MON (1 << 21) -#define GPMC_CONFIG1_WAIT_MON_TIME(val) ((val & 3) << 18) +#define GPMC_CONFIG1_WAIT_MON_TIME(val) (((val) & 3) << 18) /** WAITMONITORINGTIME Max Ticks */ #define GPMC_CONFIG1_WAITMONITORINGTIME_MAX 2 -#define GPMC_CONFIG1_WAIT_PIN_SEL(val) ((val & 3) << 16) -#define GPMC_CONFIG1_DEVICESIZE(val) ((val & 3) << 12) +#define GPMC_CONFIG1_WAIT_PIN_SEL(val) (((val) & 3) << 16) +#define GPMC_CONFIG1_DEVICESIZE(val) (((val) & 3) << 12) #define GPMC_CONFIG1_DEVICESIZE_16 GPMC_CONFIG1_DEVICESIZE(1) /** DEVICESIZE Max Value */ #define GPMC_CONFIG1_DEVICESIZE_MAX 1 -#define GPMC_CONFIG1_DEVICETYPE(val) ((val & 3) << 10) +#define GPMC_CONFIG1_DEVICETYPE(val) (((val) & 3) << 10) #define GPMC_CONFIG1_DEVICETYPE_NOR GPMC_CONFIG1_DEVICETYPE(0) -#define GPMC_CONFIG1_MUXTYPE(val) ((val & 3) << 8) +#define GPMC_CONFIG1_MUXTYPE(val) (((val) & 3) << 8) #define GPMC_CONFIG1_TIME_PARA_GRAN (1 << 4) -#define GPMC_CONFIG1_FCLK_DIV(val) (val & 3) +#define GPMC_CONFIG1_FCLK_DIV(val) ((val) & 3) #define GPMC_CONFIG1_FCLK_DIV2 (GPMC_CONFIG1_FCLK_DIV(1)) #define GPMC_CONFIG1_FCLK_DIV3 (GPMC_CONFIG1_FCLK_DIV(2)) #define GPMC_CONFIG1_FCLK_DIV4 (GPMC_CONFIG1_FCLK_DIV(3)) @@ -233,7 +234,11 @@ struct gpmc_device { int irq; struct irq_chip irq_chip; struct gpio_chip gpio_chip; + struct notifier_block nb; + struct omap3_gpmc_regs context; int nirqs; + unsigned int is_suspended:1; + struct resource *data; }; static struct irq_domain *gpmc_irq_domain; @@ -244,8 +249,7 @@ static DEFINE_SPINLOCK(gpmc_mem_lock); /* Define chip-selects as reserved by default until probe completes */ static unsigned int gpmc_cs_num = GPMC_CS_NUM; static unsigned int gpmc_nr_waitpins; -static resource_size_t phys_base, mem_size; -static unsigned gpmc_capability; +static unsigned int gpmc_capability; static void __iomem *gpmc_base; static struct clk *gpmc_l3_clk; @@ -291,15 +295,14 @@ static unsigned long gpmc_get_fclk_period(void) /** * gpmc_get_clk_period - get period of selected clock domain in ps - * @cs Chip Select Region. - * @cd Clock Domain. + * @cs: Chip Select Region. + * @cd: Clock Domain. * * GPMC_CS_CONFIG1 GPMCFCLKDIVIDER for cs has to be setup * prior to calling this function with GPMC_CD_CLK. */ static unsigned long gpmc_get_clk_period(int cs, enum gpmc_clk_domain cd) { - unsigned long tick_ps = gpmc_get_fclk_period(); u32 l; int div; @@ -313,13 +316,11 @@ static unsigned long gpmc_get_clk_period(int cs, enum gpmc_clk_domain cd) tick_ps *= div; break; case GPMC_CD_FCLK: - /* FALL-THROUGH */ default: break; } return tick_ps; - } static unsigned int gpmc_ns_to_clk_ticks(unsigned int time_ns, int cs, @@ -411,7 +412,7 @@ static void gpmc_cs_bool_timings(int cs, const struct gpmc_bool_timings *p) * @reg: GPMC_CS_CONFIGn register offset. * @st_bit: Start Bit * @end_bit: End Bit. Must be >= @st_bit. - * @ma:x Maximum parameter value (before optional @shift). + * @max: Maximum parameter value (before optional @shift). * If 0, maximum is as high as @st_bit and @end_bit allow. * @name: DTS node name, w/o "gpmc," * @cd: Clock Domain of timing parameter. @@ -511,7 +512,7 @@ static void gpmc_cs_show_timings(int cs, const char *desc) GPMC_GET_RAW_BOOL(GPMC_CS_CONFIG1, 4, 4, "time-para-granularity"); GPMC_GET_RAW(GPMC_CS_CONFIG1, 8, 9, "mux-add-data"); GPMC_GET_RAW_SHIFT_MAX(GPMC_CS_CONFIG1, 12, 13, 1, - GPMC_CONFIG1_DEVICESIZE_MAX, "device-width"); + GPMC_CONFIG1_DEVICESIZE_MAX, "device-width"); GPMC_GET_RAW(GPMC_CS_CONFIG1, 16, 17, "wait-pin"); GPMC_GET_RAW_BOOL(GPMC_CS_CONFIG1, 21, 21, "wait-on-write"); GPMC_GET_RAW_BOOL(GPMC_CS_CONFIG1, 22, 22, "wait-on-read"); @@ -625,9 +626,8 @@ static int set_gpmc_timing_reg(int cs, int reg, int st_bit, int end_bit, int max l = gpmc_cs_read_reg(cs, reg); #ifdef CONFIG_OMAP_GPMC_DEBUG - pr_info( - "GPMC CS%d: %-17s: %3d ticks, %3lu ns (was %3i ticks) %3d ns\n", - cs, name, ticks, gpmc_get_clk_period(cs, cd) * ticks / 1000, + pr_info("GPMC CS%d: %-17s: %3d ticks, %3lu ns (was %3i ticks) %3d ns\n", + cs, name, ticks, gpmc_get_clk_period(cs, cd) * ticks / 1000, (l >> st_bit) & mask, time); #endif l &= ~(mask << st_bit); @@ -637,14 +637,6 @@ static int set_gpmc_timing_reg(int cs, int reg, int st_bit, int end_bit, int max return 0; } -#define GPMC_SET_ONE_CD_MAX(reg, st, end, max, field, cd) \ - if (set_gpmc_timing_reg(cs, (reg), (st), (end), (max), \ - t->field, (cd), #field) < 0) \ - return -1 - -#define GPMC_SET_ONE(reg, st, end, field) \ - GPMC_SET_ONE_CD_MAX(reg, st, end, 0, field, GPMC_CD_FCLK) - /** * gpmc_calc_waitmonitoring_divider - calculate proper GPMCFCLKDIVIDER based on WAITMONITORINGTIME * WAITMONITORINGTIME will be _at least_ as long as desired, i.e. @@ -662,7 +654,6 @@ static int set_gpmc_timing_reg(int cs, int reg, int st_bit, int end_bit, int max */ static int gpmc_calc_waitmonitoring_divider(unsigned int wait_monitoring) { - int div = gpmc_ns_to_ticks(wait_monitoring); div += GPMC_CONFIG1_WAITMONITORINGTIME_MAX - 1; @@ -674,7 +665,6 @@ static int gpmc_calc_waitmonitoring_divider(unsigned int wait_monitoring) div = 1; return div; - } /** @@ -705,12 +695,12 @@ int gpmc_calc_divider(unsigned int sync_clk) int gpmc_cs_set_timings(int cs, const struct gpmc_timings *t, const struct gpmc_settings *s) { - int div; + int div, ret; u32 l; div = gpmc_calc_divider(t->sync_clk); if (div < 0) - return div; + return -EINVAL; /* * See if we need to change the divider for waitmonitoringtime. @@ -728,64 +718,120 @@ int gpmc_cs_set_timings(int cs, const struct gpmc_timings *t, if (!s->sync_read && !s->sync_write && (s->wait_on_read || s->wait_on_write) ) { - div = gpmc_calc_waitmonitoring_divider(t->wait_monitoring); if (div < 0) { pr_err("%s: waitmonitoringtime %3d ns too large for greatest gpmcfclkdivider.\n", __func__, t->wait_monitoring ); - return -1; + return -ENXIO; } } - GPMC_SET_ONE(GPMC_CS_CONFIG2, 0, 3, cs_on); - GPMC_SET_ONE(GPMC_CS_CONFIG2, 8, 12, cs_rd_off); - GPMC_SET_ONE(GPMC_CS_CONFIG2, 16, 20, cs_wr_off); + ret = 0; + ret |= set_gpmc_timing_reg(cs, GPMC_CS_CONFIG2, 0, 3, 0, t->cs_on, + GPMC_CD_FCLK, "cs_on"); + ret |= set_gpmc_timing_reg(cs, GPMC_CS_CONFIG2, 8, 12, 0, t->cs_rd_off, + GPMC_CD_FCLK, "cs_rd_off"); + ret |= set_gpmc_timing_reg(cs, GPMC_CS_CONFIG2, 16, 20, 0, t->cs_wr_off, + GPMC_CD_FCLK, "cs_wr_off"); + if (ret) + return -ENXIO; + + ret |= set_gpmc_timing_reg(cs, GPMC_CS_CONFIG3, 0, 3, 0, t->adv_on, + GPMC_CD_FCLK, "adv_on"); + ret |= set_gpmc_timing_reg(cs, GPMC_CS_CONFIG3, 8, 12, 0, t->adv_rd_off, + GPMC_CD_FCLK, "adv_rd_off"); + ret |= set_gpmc_timing_reg(cs, GPMC_CS_CONFIG3, 16, 20, 0, t->adv_wr_off, + GPMC_CD_FCLK, "adv_wr_off"); + if (ret) + return -ENXIO; - GPMC_SET_ONE(GPMC_CS_CONFIG3, 0, 3, adv_on); - GPMC_SET_ONE(GPMC_CS_CONFIG3, 8, 12, adv_rd_off); - GPMC_SET_ONE(GPMC_CS_CONFIG3, 16, 20, adv_wr_off); if (gpmc_capability & GPMC_HAS_MUX_AAD) { - GPMC_SET_ONE(GPMC_CS_CONFIG3, 4, 6, adv_aad_mux_on); - GPMC_SET_ONE(GPMC_CS_CONFIG3, 24, 26, adv_aad_mux_rd_off); - GPMC_SET_ONE(GPMC_CS_CONFIG3, 28, 30, adv_aad_mux_wr_off); + ret |= set_gpmc_timing_reg(cs, GPMC_CS_CONFIG3, 4, 6, 0, + t->adv_aad_mux_on, GPMC_CD_FCLK, + "adv_aad_mux_on"); + ret |= set_gpmc_timing_reg(cs, GPMC_CS_CONFIG3, 24, 26, 0, + t->adv_aad_mux_rd_off, GPMC_CD_FCLK, + "adv_aad_mux_rd_off"); + ret |= set_gpmc_timing_reg(cs, GPMC_CS_CONFIG3, 28, 30, 0, + t->adv_aad_mux_wr_off, GPMC_CD_FCLK, + "adv_aad_mux_wr_off"); + if (ret) + return -ENXIO; } - GPMC_SET_ONE(GPMC_CS_CONFIG4, 0, 3, oe_on); - GPMC_SET_ONE(GPMC_CS_CONFIG4, 8, 12, oe_off); + ret |= set_gpmc_timing_reg(cs, GPMC_CS_CONFIG4, 0, 3, 0, t->oe_on, + GPMC_CD_FCLK, "oe_on"); + ret |= set_gpmc_timing_reg(cs, GPMC_CS_CONFIG4, 8, 12, 0, t->oe_off, + GPMC_CD_FCLK, "oe_off"); if (gpmc_capability & GPMC_HAS_MUX_AAD) { - GPMC_SET_ONE(GPMC_CS_CONFIG4, 4, 6, oe_aad_mux_on); - GPMC_SET_ONE(GPMC_CS_CONFIG4, 13, 15, oe_aad_mux_off); + ret |= set_gpmc_timing_reg(cs, GPMC_CS_CONFIG4, 4, 6, 0, + t->oe_aad_mux_on, GPMC_CD_FCLK, + "oe_aad_mux_on"); + ret |= set_gpmc_timing_reg(cs, GPMC_CS_CONFIG4, 13, 15, 0, + t->oe_aad_mux_off, GPMC_CD_FCLK, + "oe_aad_mux_off"); + } + ret |= set_gpmc_timing_reg(cs, GPMC_CS_CONFIG4, 16, 19, 0, t->we_on, + GPMC_CD_FCLK, "we_on"); + ret |= set_gpmc_timing_reg(cs, GPMC_CS_CONFIG4, 24, 28, 0, t->we_off, + GPMC_CD_FCLK, "we_off"); + if (ret) + return -ENXIO; + + ret |= set_gpmc_timing_reg(cs, GPMC_CS_CONFIG5, 0, 4, 0, t->rd_cycle, + GPMC_CD_FCLK, "rd_cycle"); + ret |= set_gpmc_timing_reg(cs, GPMC_CS_CONFIG5, 8, 12, 0, t->wr_cycle, + GPMC_CD_FCLK, "wr_cycle"); + ret |= set_gpmc_timing_reg(cs, GPMC_CS_CONFIG5, 16, 20, 0, t->access, + GPMC_CD_FCLK, "access"); + ret |= set_gpmc_timing_reg(cs, GPMC_CS_CONFIG5, 24, 27, 0, + t->page_burst_access, GPMC_CD_FCLK, + "page_burst_access"); + if (ret) + return -ENXIO; + + ret |= set_gpmc_timing_reg(cs, GPMC_CS_CONFIG6, 0, 3, 0, + t->bus_turnaround, GPMC_CD_FCLK, + "bus_turnaround"); + ret |= set_gpmc_timing_reg(cs, GPMC_CS_CONFIG6, 8, 11, 0, + t->cycle2cycle_delay, GPMC_CD_FCLK, + "cycle2cycle_delay"); + if (ret) + return -ENXIO; + + if (gpmc_capability & GPMC_HAS_WR_DATA_MUX_BUS) { + ret |= set_gpmc_timing_reg(cs, GPMC_CS_CONFIG6, 16, 19, 0, + t->wr_data_mux_bus, GPMC_CD_FCLK, + "wr_data_mux_bus"); + if (ret) + return -ENXIO; + } + if (gpmc_capability & GPMC_HAS_WR_ACCESS) { + ret |= set_gpmc_timing_reg(cs, GPMC_CS_CONFIG6, 24, 28, 0, + t->wr_access, GPMC_CD_FCLK, + "wr_access"); + if (ret) + return -ENXIO; } - GPMC_SET_ONE(GPMC_CS_CONFIG4, 16, 19, we_on); - GPMC_SET_ONE(GPMC_CS_CONFIG4, 24, 28, we_off); - - GPMC_SET_ONE(GPMC_CS_CONFIG5, 0, 4, rd_cycle); - GPMC_SET_ONE(GPMC_CS_CONFIG5, 8, 12, wr_cycle); - GPMC_SET_ONE(GPMC_CS_CONFIG5, 16, 20, access); - - GPMC_SET_ONE(GPMC_CS_CONFIG5, 24, 27, page_burst_access); - - GPMC_SET_ONE(GPMC_CS_CONFIG6, 0, 3, bus_turnaround); - GPMC_SET_ONE(GPMC_CS_CONFIG6, 8, 11, cycle2cycle_delay); - - if (gpmc_capability & GPMC_HAS_WR_DATA_MUX_BUS) - GPMC_SET_ONE(GPMC_CS_CONFIG6, 16, 19, wr_data_mux_bus); - if (gpmc_capability & GPMC_HAS_WR_ACCESS) - GPMC_SET_ONE(GPMC_CS_CONFIG6, 24, 28, wr_access); l = gpmc_cs_read_reg(cs, GPMC_CS_CONFIG1); l &= ~0x03; l |= (div - 1); gpmc_cs_write_reg(cs, GPMC_CS_CONFIG1, l); - GPMC_SET_ONE_CD_MAX(GPMC_CS_CONFIG1, 18, 19, - GPMC_CONFIG1_WAITMONITORINGTIME_MAX, - wait_monitoring, GPMC_CD_CLK); - GPMC_SET_ONE_CD_MAX(GPMC_CS_CONFIG1, 25, 26, - GPMC_CONFIG1_CLKACTIVATIONTIME_MAX, - clk_activation, GPMC_CD_FCLK); + ret = 0; + ret |= set_gpmc_timing_reg(cs, GPMC_CS_CONFIG1, 18, 19, + GPMC_CONFIG1_WAITMONITORINGTIME_MAX, + t->wait_monitoring, GPMC_CD_CLK, + "wait_monitoring"); + ret |= set_gpmc_timing_reg(cs, GPMC_CS_CONFIG1, 25, 26, + GPMC_CONFIG1_CLKACTIVATIONTIME_MAX, + t->clk_activation, GPMC_CD_FCLK, + "clk_activation"); + if (ret) + return -ENXIO; #ifdef CONFIG_OMAP_GPMC_DEBUG pr_info("GPMC CS%d CLK period is %lu ns (div %d)\n", @@ -876,20 +922,6 @@ static bool gpmc_cs_reserved(int cs) return gpmc->flags & GPMC_CS_RESERVED; } -static void gpmc_cs_set_name(int cs, const char *name) -{ - struct gpmc_cs_data *gpmc = &gpmc_cs[cs]; - - gpmc->name = name; -} - -static const char *gpmc_cs_get_name(int cs) -{ - struct gpmc_cs_data *gpmc = &gpmc_cs[cs]; - - return gpmc->name; -} - static unsigned long gpmc_mem_align(unsigned long size) { int order; @@ -935,56 +967,13 @@ static int gpmc_cs_delete_mem(int cs) return r; } -/** - * gpmc_cs_remap - remaps a chip-select physical base address - * @cs: chip-select to remap - * @base: physical base address to re-map chip-select to - * - * Re-maps a chip-select to a new physical base address specified by - * "base". Returns 0 on success and appropriate negative error code - * on failure. - */ -static int gpmc_cs_remap(int cs, u32 base) -{ - int ret; - u32 old_base, size; - - if (cs > gpmc_cs_num) { - pr_err("%s: requested chip-select is disabled\n", __func__); - return -ENODEV; - } - - /* - * Make sure we ignore any device offsets from the GPMC partition - * allocated for the chip select and that the new base confirms - * to the GPMC 16MB minimum granularity. - */ - base &= ~(SZ_16M - 1); - - gpmc_cs_get_memconf(cs, &old_base, &size); - if (base == old_base) - return 0; - - ret = gpmc_cs_delete_mem(cs); - if (ret < 0) - return ret; - - ret = gpmc_cs_insert_mem(cs, base, size); - if (ret < 0) - return ret; - - ret = gpmc_cs_set_memconf(cs, base, size); - - return ret; -} - int gpmc_cs_request(int cs, unsigned long size, unsigned long *base) { struct gpmc_cs_data *gpmc = &gpmc_cs[cs]; struct resource *res = &gpmc->mem; int r = -1; - if (cs > gpmc_cs_num) { + if (cs >= gpmc_cs_num) { pr_err("%s: requested chip-select is disabled\n", __func__); return -ENODEV; } @@ -1026,16 +1015,18 @@ EXPORT_SYMBOL(gpmc_cs_request); void gpmc_cs_free(int cs) { - struct gpmc_cs_data *gpmc = &gpmc_cs[cs]; - struct resource *res = &gpmc->mem; + struct gpmc_cs_data *gpmc; + struct resource *res; spin_lock(&gpmc_mem_lock); if (cs >= gpmc_cs_num || cs < 0 || !gpmc_cs_reserved(cs)) { - printk(KERN_ERR "Trying to free non-reserved GPMC CS%d\n", cs); - BUG(); + WARN(1, "Trying to free non-reserved GPMC CS%d\n", cs); spin_unlock(&gpmc_mem_lock); return; } + gpmc = &gpmc_cs[cs]; + res = &gpmc->mem; + gpmc_cs_disable_mem(cs); if (res->flags) release_resource(res); @@ -1087,7 +1078,7 @@ static struct gpmc_nand_ops nand_ops = { /** * gpmc_omap_get_nand_ops - Get the GPMC NAND interface - * @regs: the GPMC NAND register map exclusive for NAND use. + * @reg: the GPMC NAND register map exclusive for NAND use. * @cs: GPMC chip select number on which the NAND sits. The * register map returned will be specific to this chip select. * @@ -1242,7 +1233,7 @@ int gpmc_omap_onenand_set_timings(struct device *dev, int cs, int freq, } EXPORT_SYMBOL_GPL(gpmc_omap_onenand_set_timings); -int gpmc_get_client_irq(unsigned irq_config) +int gpmc_get_client_irq(unsigned int irq_config) { if (!gpmc_irq_domain) { pr_warn("%s called before GPMC IRQ domain available\n", @@ -1465,15 +1456,20 @@ static void gpmc_mem_exit(void) continue; gpmc_cs_delete_mem(cs); } - } -static void gpmc_mem_init(void) +static void gpmc_mem_init(struct gpmc_device *gpmc) { int cs; - gpmc_mem_root.start = GPMC_MEM_START; - gpmc_mem_root.end = GPMC_MEM_END; + if (!gpmc->data) { + /* All legacy devices have same data IO window */ + gpmc_mem_root.start = GPMC_MEM_START; + gpmc_mem_root.end = GPMC_MEM_END; + } else { + gpmc_mem_root.start = gpmc->data->start; + gpmc_mem_root.end = gpmc->data->end; + } /* Reserve all regions that has been set up by bootloader */ for (cs = 0; cs < gpmc_cs_num; cs++) { @@ -1634,17 +1630,14 @@ static int gpmc_calc_async_read_timings(struct gpmc_timings *gpmc_t, /* oe_on */ temp = dev_t->t_oeasu; if (mux) - temp = max_t(u32, temp, - gpmc_t->adv_rd_off + dev_t->t_aavdh); + temp = max_t(u32, temp, gpmc_t->adv_rd_off + dev_t->t_aavdh); gpmc_t->oe_on = gpmc_round_ps_to_ticks(temp); /* access */ temp = max_t(u32, dev_t->t_iaa, /* XXX: remove t_iaa in async ? */ - gpmc_t->oe_on + dev_t->t_oe); - temp = max_t(u32, temp, - gpmc_t->cs_on + dev_t->t_ce); - temp = max_t(u32, temp, - gpmc_t->adv_on + dev_t->t_aa); + gpmc_t->oe_on + dev_t->t_oe); + temp = max_t(u32, temp, gpmc_t->cs_on + dev_t->t_ce); + temp = max_t(u32, temp, gpmc_t->adv_on + dev_t->t_aa); gpmc_t->access = gpmc_round_ps_to_ticks(temp); gpmc_t->oe_off = gpmc_t->access + gpmc_ticks_to_ps(1); @@ -1753,10 +1746,11 @@ static int gpmc_calc_common_timings(struct gpmc_timings *gpmc_t, return 0; } -/* TODO: remove this function once all peripherals are confirmed to +/* + * TODO: remove this function once all peripherals are confirmed to * work with generic timing. Simultaneously gpmc_cs_set_timings() * has to be modified to handle timings in ps instead of ns -*/ + */ static void gpmc_convert_ps_to_ns(struct gpmc_timings *t) { t->cs_on /= 1000; @@ -1896,14 +1890,62 @@ int gpmc_cs_program_settings(int cs, struct gpmc_settings *p) } #ifdef CONFIG_OF -static const struct of_device_id gpmc_dt_ids[] = { - { .compatible = "ti,omap2420-gpmc" }, - { .compatible = "ti,omap2430-gpmc" }, - { .compatible = "ti,omap3430-gpmc" }, /* omap3430 & omap3630 */ - { .compatible = "ti,omap4430-gpmc" }, /* omap4430 & omap4460 & omap543x */ - { .compatible = "ti,am3352-gpmc" }, /* am335x devices */ - { } -}; +static void gpmc_cs_set_name(int cs, const char *name) +{ + struct gpmc_cs_data *gpmc = &gpmc_cs[cs]; + + gpmc->name = name; +} + +static const char *gpmc_cs_get_name(int cs) +{ + struct gpmc_cs_data *gpmc = &gpmc_cs[cs]; + + return gpmc->name; +} + +/** + * gpmc_cs_remap - remaps a chip-select physical base address + * @cs: chip-select to remap + * @base: physical base address to re-map chip-select to + * + * Re-maps a chip-select to a new physical base address specified by + * "base". Returns 0 on success and appropriate negative error code + * on failure. + */ +static int gpmc_cs_remap(int cs, u32 base) +{ + int ret; + u32 old_base, size; + + if (cs >= gpmc_cs_num) { + pr_err("%s: requested chip-select is disabled\n", __func__); + return -ENODEV; + } + + /* + * Make sure we ignore any device offsets from the GPMC partition + * allocated for the chip select and that the new base confirms + * to the GPMC 16MB minimum granularity. + */ + base &= ~(SZ_16M - 1); + + gpmc_cs_get_memconf(cs, &old_base, &size); + if (base == old_base) + return 0; + + ret = gpmc_cs_delete_mem(cs); + if (ret < 0) + return ret; + + ret = gpmc_cs_insert_mem(cs, base, size); + if (ret < 0) + return ret; + + ret = gpmc_cs_set_memconf(cs, base, size); + + return ret; +} /** * gpmc_read_settings_dt - read gpmc settings from device-tree @@ -2089,7 +2131,7 @@ static int gpmc_probe_generic_child(struct platform_device *pdev, gpmc_cs_disable_mem(cs); /* - * FIXME: gpmc_cs_request() will map the CS to an arbitary + * FIXME: gpmc_cs_request() will map the CS to an arbitrary * location in the gpmc address space. When booting with * device-tree we want the NOR flash to be mapped to the * location specified in the device-tree blob. So remap the @@ -2132,7 +2174,7 @@ static int gpmc_probe_generic_child(struct platform_device *pdev, } } - if (of_device_is_compatible(child, "ti,omap2-nand")) { + if (of_match_node(omap_nand_ids, child)) { /* NAND specific setup */ val = 8; of_property_read_u32(child, "nand-bus-width", &val); @@ -2206,11 +2248,9 @@ no_timings: if (!of_platform_device_create(child, NULL, &pdev->dev)) goto err_child_fail; - /* is child a common bus? */ - if (of_match_node(of_default_bus_match_table, child)) - /* create children and other common bus children */ - if (of_platform_default_populate(child, NULL, &pdev->dev)) - goto err_child_fail; + /* create children and other common bus children */ + if (of_platform_default_populate(child, NULL, &pdev->dev)) + goto err_child_fail; return 0; @@ -2227,6 +2267,8 @@ err: return ret; } +static const struct of_device_id gpmc_dt_ids[]; + static int gpmc_probe_dt(struct platform_device *pdev) { int ret; @@ -2274,6 +2316,10 @@ static void gpmc_probe_dt_children(struct platform_device *pdev) } } #else +void gpmc_read_settings_dt(struct device_node *np, struct gpmc_settings *p) +{ + memset(p, 0, sizeof(*p)); +} static int gpmc_probe_dt(struct platform_device *pdev) { return 0; @@ -2341,6 +2387,106 @@ static int gpmc_gpio_init(struct gpmc_device *gpmc) return 0; } +static void omap3_gpmc_save_context(struct gpmc_device *gpmc) +{ + struct omap3_gpmc_regs *gpmc_context; + int i; + + if (!gpmc || !gpmc_base) + return; + + gpmc_context = &gpmc->context; + + gpmc_context->sysconfig = gpmc_read_reg(GPMC_SYSCONFIG); + gpmc_context->irqenable = gpmc_read_reg(GPMC_IRQENABLE); + gpmc_context->timeout_ctrl = gpmc_read_reg(GPMC_TIMEOUT_CONTROL); + gpmc_context->config = gpmc_read_reg(GPMC_CONFIG); + gpmc_context->prefetch_config1 = gpmc_read_reg(GPMC_PREFETCH_CONFIG1); + gpmc_context->prefetch_config2 = gpmc_read_reg(GPMC_PREFETCH_CONFIG2); + gpmc_context->prefetch_control = gpmc_read_reg(GPMC_PREFETCH_CONTROL); + for (i = 0; i < gpmc_cs_num; i++) { + gpmc_context->cs_context[i].is_valid = gpmc_cs_mem_enabled(i); + if (gpmc_context->cs_context[i].is_valid) { + gpmc_context->cs_context[i].config1 = + gpmc_cs_read_reg(i, GPMC_CS_CONFIG1); + gpmc_context->cs_context[i].config2 = + gpmc_cs_read_reg(i, GPMC_CS_CONFIG2); + gpmc_context->cs_context[i].config3 = + gpmc_cs_read_reg(i, GPMC_CS_CONFIG3); + gpmc_context->cs_context[i].config4 = + gpmc_cs_read_reg(i, GPMC_CS_CONFIG4); + gpmc_context->cs_context[i].config5 = + gpmc_cs_read_reg(i, GPMC_CS_CONFIG5); + gpmc_context->cs_context[i].config6 = + gpmc_cs_read_reg(i, GPMC_CS_CONFIG6); + gpmc_context->cs_context[i].config7 = + gpmc_cs_read_reg(i, GPMC_CS_CONFIG7); + } + } +} + +static void omap3_gpmc_restore_context(struct gpmc_device *gpmc) +{ + struct omap3_gpmc_regs *gpmc_context; + int i; + + if (!gpmc || !gpmc_base) + return; + + gpmc_context = &gpmc->context; + + gpmc_write_reg(GPMC_SYSCONFIG, gpmc_context->sysconfig); + gpmc_write_reg(GPMC_IRQENABLE, gpmc_context->irqenable); + gpmc_write_reg(GPMC_TIMEOUT_CONTROL, gpmc_context->timeout_ctrl); + gpmc_write_reg(GPMC_CONFIG, gpmc_context->config); + gpmc_write_reg(GPMC_PREFETCH_CONFIG1, gpmc_context->prefetch_config1); + gpmc_write_reg(GPMC_PREFETCH_CONFIG2, gpmc_context->prefetch_config2); + gpmc_write_reg(GPMC_PREFETCH_CONTROL, gpmc_context->prefetch_control); + for (i = 0; i < gpmc_cs_num; i++) { + if (gpmc_context->cs_context[i].is_valid) { + gpmc_cs_write_reg(i, GPMC_CS_CONFIG1, + gpmc_context->cs_context[i].config1); + gpmc_cs_write_reg(i, GPMC_CS_CONFIG2, + gpmc_context->cs_context[i].config2); + gpmc_cs_write_reg(i, GPMC_CS_CONFIG3, + gpmc_context->cs_context[i].config3); + gpmc_cs_write_reg(i, GPMC_CS_CONFIG4, + gpmc_context->cs_context[i].config4); + gpmc_cs_write_reg(i, GPMC_CS_CONFIG5, + gpmc_context->cs_context[i].config5); + gpmc_cs_write_reg(i, GPMC_CS_CONFIG6, + gpmc_context->cs_context[i].config6); + gpmc_cs_write_reg(i, GPMC_CS_CONFIG7, + gpmc_context->cs_context[i].config7); + } else { + gpmc_cs_write_reg(i, GPMC_CS_CONFIG7, 0); + } + } +} + +static int omap_gpmc_context_notifier(struct notifier_block *nb, + unsigned long cmd, void *v) +{ + struct gpmc_device *gpmc; + + gpmc = container_of(nb, struct gpmc_device, nb); + if (gpmc->is_suspended || pm_runtime_suspended(gpmc->dev)) + return NOTIFY_OK; + + switch (cmd) { + case CPU_CLUSTER_PM_ENTER: + omap3_gpmc_save_context(gpmc); + break; + case CPU_CLUSTER_PM_ENTER_FAILED: /* No need to restore context */ + break; + case CPU_CLUSTER_PM_EXIT: + omap3_gpmc_restore_context(gpmc); + break; + } + + return NOTIFY_OK; +} + static int gpmc_probe(struct platform_device *pdev) { int rc; @@ -2355,24 +2501,29 @@ static int gpmc_probe(struct platform_device *pdev) gpmc->dev = &pdev->dev; platform_set_drvdata(pdev, gpmc); - res = platform_get_resource(pdev, IORESOURCE_MEM, 0); - if (res == NULL) - return -ENOENT; - - phys_base = res->start; - mem_size = resource_size(res); - - gpmc_base = devm_ioremap_resource(&pdev->dev, res); - if (IS_ERR(gpmc_base)) - return PTR_ERR(gpmc_base); - - res = platform_get_resource(pdev, IORESOURCE_IRQ, 0); + res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "cfg"); if (!res) { - dev_err(&pdev->dev, "Failed to get resource: irq\n"); - return -ENOENT; + /* legacy DT */ + gpmc_base = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(gpmc_base)) + return PTR_ERR(gpmc_base); + } else { + gpmc_base = devm_ioremap_resource(&pdev->dev, res); + if (IS_ERR(gpmc_base)) + return PTR_ERR(gpmc_base); + + res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "data"); + if (!res) { + dev_err(&pdev->dev, "couldn't get data reg resource\n"); + return -ENOENT; + } + + gpmc->data = res; } - gpmc->irq = res->start; + gpmc->irq = platform_get_irq(pdev, 0); + if (gpmc->irq < 0) + return gpmc->irq; gpmc_l3_clk = devm_clk_get(&pdev->dev, "fck"); if (IS_ERR(gpmc_l3_clk)) { @@ -2418,7 +2569,7 @@ static int gpmc_probe(struct platform_device *pdev) dev_info(gpmc->dev, "GPMC revision %d.%d\n", GPMC_REVISION_MAJOR(l), GPMC_REVISION_MINOR(l)); - gpmc_mem_init(); + gpmc_mem_init(gpmc); rc = gpmc_gpio_init(gpmc); if (rc) goto gpio_init_failed; @@ -2432,6 +2583,9 @@ static int gpmc_probe(struct platform_device *pdev) gpmc_probe_dt_children(pdev); + gpmc->nb.notifier_call = omap_gpmc_context_notifier; + cpu_pm_register_notifier(&gpmc->nb); + return 0; gpio_init_failed: @@ -2446,6 +2600,7 @@ static int gpmc_remove(struct platform_device *pdev) { struct gpmc_device *gpmc = platform_get_drvdata(pdev); + cpu_pm_unregister_notifier(&gpmc->nb); gpmc_free_irq(gpmc); gpmc_mem_exit(); pm_runtime_put_sync(&pdev->dev); @@ -2457,21 +2612,42 @@ static int gpmc_remove(struct platform_device *pdev) #ifdef CONFIG_PM_SLEEP static int gpmc_suspend(struct device *dev) { - omap3_gpmc_save_context(); + struct gpmc_device *gpmc = dev_get_drvdata(dev); + + omap3_gpmc_save_context(gpmc); pm_runtime_put_sync(dev); + gpmc->is_suspended = 1; + return 0; } static int gpmc_resume(struct device *dev) { + struct gpmc_device *gpmc = dev_get_drvdata(dev); + pm_runtime_get_sync(dev); - omap3_gpmc_restore_context(); + omap3_gpmc_restore_context(gpmc); + gpmc->is_suspended = 0; + return 0; } #endif static SIMPLE_DEV_PM_OPS(gpmc_pm_ops, gpmc_suspend, gpmc_resume); +#ifdef CONFIG_OF +static const struct of_device_id gpmc_dt_ids[] = { + { .compatible = "ti,omap2420-gpmc" }, + { .compatible = "ti,omap2430-gpmc" }, + { .compatible = "ti,omap3430-gpmc" }, /* omap3430 & omap3630 */ + { .compatible = "ti,omap4430-gpmc" }, /* omap4430 & omap4460 & omap543x */ + { .compatible = "ti,am3352-gpmc" }, /* am335x devices */ + { .compatible = "ti,am64-gpmc" }, + { } +}; +MODULE_DEVICE_TABLE(of, gpmc_dt_ids); +#endif + static struct platform_driver gpmc_driver = { .probe = gpmc_probe, .remove = gpmc_remove, @@ -2482,79 +2658,7 @@ static struct platform_driver gpmc_driver = { }, }; -static __init int gpmc_init(void) -{ - return platform_driver_register(&gpmc_driver); -} -postcore_initcall(gpmc_init); - -static struct omap3_gpmc_regs gpmc_context; - -void omap3_gpmc_save_context(void) -{ - int i; - - if (!gpmc_base) - return; - - gpmc_context.sysconfig = gpmc_read_reg(GPMC_SYSCONFIG); - gpmc_context.irqenable = gpmc_read_reg(GPMC_IRQENABLE); - gpmc_context.timeout_ctrl = gpmc_read_reg(GPMC_TIMEOUT_CONTROL); - gpmc_context.config = gpmc_read_reg(GPMC_CONFIG); - gpmc_context.prefetch_config1 = gpmc_read_reg(GPMC_PREFETCH_CONFIG1); - gpmc_context.prefetch_config2 = gpmc_read_reg(GPMC_PREFETCH_CONFIG2); - gpmc_context.prefetch_control = gpmc_read_reg(GPMC_PREFETCH_CONTROL); - for (i = 0; i < gpmc_cs_num; i++) { - gpmc_context.cs_context[i].is_valid = gpmc_cs_mem_enabled(i); - if (gpmc_context.cs_context[i].is_valid) { - gpmc_context.cs_context[i].config1 = - gpmc_cs_read_reg(i, GPMC_CS_CONFIG1); - gpmc_context.cs_context[i].config2 = - gpmc_cs_read_reg(i, GPMC_CS_CONFIG2); - gpmc_context.cs_context[i].config3 = - gpmc_cs_read_reg(i, GPMC_CS_CONFIG3); - gpmc_context.cs_context[i].config4 = - gpmc_cs_read_reg(i, GPMC_CS_CONFIG4); - gpmc_context.cs_context[i].config5 = - gpmc_cs_read_reg(i, GPMC_CS_CONFIG5); - gpmc_context.cs_context[i].config6 = - gpmc_cs_read_reg(i, GPMC_CS_CONFIG6); - gpmc_context.cs_context[i].config7 = - gpmc_cs_read_reg(i, GPMC_CS_CONFIG7); - } - } -} +module_platform_driver(gpmc_driver); -void omap3_gpmc_restore_context(void) -{ - int i; - - if (!gpmc_base) - return; - - gpmc_write_reg(GPMC_SYSCONFIG, gpmc_context.sysconfig); - gpmc_write_reg(GPMC_IRQENABLE, gpmc_context.irqenable); - gpmc_write_reg(GPMC_TIMEOUT_CONTROL, gpmc_context.timeout_ctrl); - gpmc_write_reg(GPMC_CONFIG, gpmc_context.config); - gpmc_write_reg(GPMC_PREFETCH_CONFIG1, gpmc_context.prefetch_config1); - gpmc_write_reg(GPMC_PREFETCH_CONFIG2, gpmc_context.prefetch_config2); - gpmc_write_reg(GPMC_PREFETCH_CONTROL, gpmc_context.prefetch_control); - for (i = 0; i < gpmc_cs_num; i++) { - if (gpmc_context.cs_context[i].is_valid) { - gpmc_cs_write_reg(i, GPMC_CS_CONFIG1, - gpmc_context.cs_context[i].config1); - gpmc_cs_write_reg(i, GPMC_CS_CONFIG2, - gpmc_context.cs_context[i].config2); - gpmc_cs_write_reg(i, GPMC_CS_CONFIG3, - gpmc_context.cs_context[i].config3); - gpmc_cs_write_reg(i, GPMC_CS_CONFIG4, - gpmc_context.cs_context[i].config4); - gpmc_cs_write_reg(i, GPMC_CS_CONFIG5, - gpmc_context.cs_context[i].config5); - gpmc_cs_write_reg(i, GPMC_CS_CONFIG6, - gpmc_context.cs_context[i].config6); - gpmc_cs_write_reg(i, GPMC_CS_CONFIG7, - gpmc_context.cs_context[i].config7); - } - } -} +MODULE_DESCRIPTION("Texas Instruments GPMC driver"); +MODULE_LICENSE("GPL"); diff --git a/drivers/memory/pl172.c b/drivers/memory/pl172.c index ff57195b4e37..9eb8cc7de494 100644 --- a/drivers/memory/pl172.c +++ b/drivers/memory/pl172.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Memory controller driver for ARM PrimeCell PL172 * PrimeCell MultiPort Memory Controller (PL172) @@ -6,10 +7,6 @@ * * Based on: * TI AEMIF driver, Copyright (C) 2010 - 2013 Texas Instruments Inc. - * - * This file is licensed under the terms of the GNU General Public - * License version 2. This program is licensed "as is" without any - * warranty of any kind, whether express or implied. */ #include <linux/amba/bus.h> @@ -24,7 +21,7 @@ #include <linux/of_platform.h> #include <linux/time.h> -#define MPMC_STATIC_CFG(n) (0x200 + 0x20 * n) +#define MPMC_STATIC_CFG(n) (0x200 + 0x20 * (n)) #define MPMC_STATIC_CFG_MW_8BIT 0x0 #define MPMC_STATIC_CFG_MW_16BIT 0x1 #define MPMC_STATIC_CFG_MW_32BIT 0x2 @@ -34,17 +31,17 @@ #define MPMC_STATIC_CFG_EW BIT(8) #define MPMC_STATIC_CFG_B BIT(19) #define MPMC_STATIC_CFG_P BIT(20) -#define MPMC_STATIC_WAIT_WEN(n) (0x204 + 0x20 * n) +#define MPMC_STATIC_WAIT_WEN(n) (0x204 + 0x20 * (n)) #define MPMC_STATIC_WAIT_WEN_MAX 0x0f -#define MPMC_STATIC_WAIT_OEN(n) (0x208 + 0x20 * n) +#define MPMC_STATIC_WAIT_OEN(n) (0x208 + 0x20 * (n)) #define MPMC_STATIC_WAIT_OEN_MAX 0x0f -#define MPMC_STATIC_WAIT_RD(n) (0x20c + 0x20 * n) +#define MPMC_STATIC_WAIT_RD(n) (0x20c + 0x20 * (n)) #define MPMC_STATIC_WAIT_RD_MAX 0x1f -#define MPMC_STATIC_WAIT_PAGE(n) (0x210 + 0x20 * n) +#define MPMC_STATIC_WAIT_PAGE(n) (0x210 + 0x20 * (n)) #define MPMC_STATIC_WAIT_PAGE_MAX 0x1f -#define MPMC_STATIC_WAIT_WR(n) (0x214 + 0x20 * n) +#define MPMC_STATIC_WAIT_WR(n) (0x214 + 0x20 * (n)) #define MPMC_STATIC_WAIT_WR_MAX 0x1f -#define MPMC_STATIC_WAIT_TURN(n) (0x218 + 0x20 * n) +#define MPMC_STATIC_WAIT_TURN(n) (0x218 + 0x20 * (n)) #define MPMC_STATIC_WAIT_TURN_MAX 0x0f /* Maximum number of static chip selects */ @@ -276,14 +273,12 @@ err_clk_enable: return ret; } -static int pl172_remove(struct amba_device *adev) +static void pl172_remove(struct amba_device *adev) { struct pl172_data *pl172 = amba_get_drvdata(adev); clk_disable_unprepare(pl172->clk); amba_release_regions(adev); - - return 0; } static const struct amba_id pl172_ids[] = { diff --git a/drivers/memory/pl353-smc.c b/drivers/memory/pl353-smc.c index 73bd3023202f..d39ee7d06665 100644 --- a/drivers/memory/pl353-smc.c +++ b/drivers/memory/pl353-smc.c @@ -8,263 +8,22 @@ */ #include <linux/clk.h> -#include <linux/io.h> #include <linux/kernel.h> #include <linux/module.h> #include <linux/of_platform.h> #include <linux/platform_device.h> -#include <linux/slab.h> -#include <linux/pl353-smc.h> #include <linux/amba/bus.h> -/* Register definitions */ -#define PL353_SMC_MEMC_STATUS_OFFS 0 /* Controller status reg, RO */ -#define PL353_SMC_CFG_CLR_OFFS 0xC /* Clear config reg, WO */ -#define PL353_SMC_DIRECT_CMD_OFFS 0x10 /* Direct command reg, WO */ -#define PL353_SMC_SET_CYCLES_OFFS 0x14 /* Set cycles register, WO */ -#define PL353_SMC_SET_OPMODE_OFFS 0x18 /* Set opmode register, WO */ -#define PL353_SMC_ECC_STATUS_OFFS 0x400 /* ECC status register */ -#define PL353_SMC_ECC_MEMCFG_OFFS 0x404 /* ECC mem config reg */ -#define PL353_SMC_ECC_MEMCMD1_OFFS 0x408 /* ECC mem cmd1 reg */ -#define PL353_SMC_ECC_MEMCMD2_OFFS 0x40C /* ECC mem cmd2 reg */ -#define PL353_SMC_ECC_VALUE0_OFFS 0x418 /* ECC value 0 reg */ - -/* Controller status register specific constants */ -#define PL353_SMC_MEMC_STATUS_RAW_INT_1_SHIFT 6 - -/* Clear configuration register specific constants */ -#define PL353_SMC_CFG_CLR_INT_CLR_1 0x10 -#define PL353_SMC_CFG_CLR_ECC_INT_DIS_1 0x40 -#define PL353_SMC_CFG_CLR_INT_DIS_1 0x2 -#define PL353_SMC_CFG_CLR_DEFAULT_MASK (PL353_SMC_CFG_CLR_INT_CLR_1 | \ - PL353_SMC_CFG_CLR_ECC_INT_DIS_1 | \ - PL353_SMC_CFG_CLR_INT_DIS_1) - -/* Set cycles register specific constants */ -#define PL353_SMC_SET_CYCLES_T0_MASK 0xF -#define PL353_SMC_SET_CYCLES_T0_SHIFT 0 -#define PL353_SMC_SET_CYCLES_T1_MASK 0xF -#define PL353_SMC_SET_CYCLES_T1_SHIFT 4 -#define PL353_SMC_SET_CYCLES_T2_MASK 0x7 -#define PL353_SMC_SET_CYCLES_T2_SHIFT 8 -#define PL353_SMC_SET_CYCLES_T3_MASK 0x7 -#define PL353_SMC_SET_CYCLES_T3_SHIFT 11 -#define PL353_SMC_SET_CYCLES_T4_MASK 0x7 -#define PL353_SMC_SET_CYCLES_T4_SHIFT 14 -#define PL353_SMC_SET_CYCLES_T5_MASK 0x7 -#define PL353_SMC_SET_CYCLES_T5_SHIFT 17 -#define PL353_SMC_SET_CYCLES_T6_MASK 0xF -#define PL353_SMC_SET_CYCLES_T6_SHIFT 20 - -/* ECC status register specific constants */ -#define PL353_SMC_ECC_STATUS_BUSY BIT(6) -#define PL353_SMC_ECC_REG_SIZE_OFFS 4 - -/* ECC memory config register specific constants */ -#define PL353_SMC_ECC_MEMCFG_MODE_MASK 0xC -#define PL353_SMC_ECC_MEMCFG_MODE_SHIFT 2 -#define PL353_SMC_ECC_MEMCFG_PGSIZE_MASK 0xC - -#define PL353_SMC_DC_UPT_NAND_REGS ((4 << 23) | /* CS: NAND chip */ \ - (2 << 21)) /* UpdateRegs operation */ - -#define PL353_NAND_ECC_CMD1 ((0x80) | /* Write command */ \ - (0 << 8) | /* Read command */ \ - (0x30 << 16) | /* Read End command */ \ - (1 << 24)) /* Read End command calid */ - -#define PL353_NAND_ECC_CMD2 ((0x85) | /* Write col change cmd */ \ - (5 << 8) | /* Read col change cmd */ \ - (0xE0 << 16) | /* Read col change end cmd */ \ - (1 << 24)) /* Read col change end cmd valid */ -#define PL353_NAND_ECC_BUSY_TIMEOUT (1 * HZ) /** * struct pl353_smc_data - Private smc driver structure * @memclk: Pointer to the peripheral clock - * @aclk: Pointer to the APER clock + * @aclk: Pointer to the AXI peripheral clock */ struct pl353_smc_data { struct clk *memclk; struct clk *aclk; }; -/* SMC virtual register base */ -static void __iomem *pl353_smc_base; - -/** - * pl353_smc_set_buswidth - Set memory buswidth - * @bw: Memory buswidth (8 | 16) - * Return: 0 on success or negative errno. - */ -int pl353_smc_set_buswidth(unsigned int bw) -{ - if (bw != PL353_SMC_MEM_WIDTH_8 && bw != PL353_SMC_MEM_WIDTH_16) - return -EINVAL; - - writel(bw, pl353_smc_base + PL353_SMC_SET_OPMODE_OFFS); - writel(PL353_SMC_DC_UPT_NAND_REGS, pl353_smc_base + - PL353_SMC_DIRECT_CMD_OFFS); - - return 0; -} -EXPORT_SYMBOL_GPL(pl353_smc_set_buswidth); - -/** - * pl353_smc_set_cycles - Set memory timing parameters - * @timings: NAND controller timing parameters - * - * Sets NAND chip specific timing parameters. - */ -void pl353_smc_set_cycles(u32 timings[]) -{ - /* - * Set write pulse timing. This one is easy to extract: - * - * NWE_PULSE = tWP - */ - timings[0] &= PL353_SMC_SET_CYCLES_T0_MASK; - timings[1] = (timings[1] & PL353_SMC_SET_CYCLES_T1_MASK) << - PL353_SMC_SET_CYCLES_T1_SHIFT; - timings[2] = (timings[2] & PL353_SMC_SET_CYCLES_T2_MASK) << - PL353_SMC_SET_CYCLES_T2_SHIFT; - timings[3] = (timings[3] & PL353_SMC_SET_CYCLES_T3_MASK) << - PL353_SMC_SET_CYCLES_T3_SHIFT; - timings[4] = (timings[4] & PL353_SMC_SET_CYCLES_T4_MASK) << - PL353_SMC_SET_CYCLES_T4_SHIFT; - timings[5] = (timings[5] & PL353_SMC_SET_CYCLES_T5_MASK) << - PL353_SMC_SET_CYCLES_T5_SHIFT; - timings[6] = (timings[6] & PL353_SMC_SET_CYCLES_T6_MASK) << - PL353_SMC_SET_CYCLES_T6_SHIFT; - timings[0] |= timings[1] | timings[2] | timings[3] | - timings[4] | timings[5] | timings[6]; - - writel(timings[0], pl353_smc_base + PL353_SMC_SET_CYCLES_OFFS); - writel(PL353_SMC_DC_UPT_NAND_REGS, pl353_smc_base + - PL353_SMC_DIRECT_CMD_OFFS); -} -EXPORT_SYMBOL_GPL(pl353_smc_set_cycles); - -/** - * pl353_smc_ecc_is_busy - Read ecc busy flag - * Return: the ecc_status bit from the ecc_status register. 1 = busy, 0 = idle - */ -bool pl353_smc_ecc_is_busy(void) -{ - return ((readl(pl353_smc_base + PL353_SMC_ECC_STATUS_OFFS) & - PL353_SMC_ECC_STATUS_BUSY) == PL353_SMC_ECC_STATUS_BUSY); -} -EXPORT_SYMBOL_GPL(pl353_smc_ecc_is_busy); - -/** - * pl353_smc_get_ecc_val - Read ecc_valueN registers - * @ecc_reg: Index of the ecc_value reg (0..3) - * Return: the content of the requested ecc_value register. - * - * There are four valid ecc_value registers. The argument is truncated to stay - * within this valid boundary. - */ -u32 pl353_smc_get_ecc_val(int ecc_reg) -{ - u32 addr, reg; - - addr = PL353_SMC_ECC_VALUE0_OFFS + - (ecc_reg * PL353_SMC_ECC_REG_SIZE_OFFS); - reg = readl(pl353_smc_base + addr); - - return reg; -} -EXPORT_SYMBOL_GPL(pl353_smc_get_ecc_val); - -/** - * pl353_smc_get_nand_int_status_raw - Get NAND interrupt status bit - * Return: the raw_int_status1 bit from the memc_status register - */ -int pl353_smc_get_nand_int_status_raw(void) -{ - u32 reg; - - reg = readl(pl353_smc_base + PL353_SMC_MEMC_STATUS_OFFS); - reg >>= PL353_SMC_MEMC_STATUS_RAW_INT_1_SHIFT; - reg &= 1; - - return reg; -} -EXPORT_SYMBOL_GPL(pl353_smc_get_nand_int_status_raw); - -/** - * pl353_smc_clr_nand_int - Clear NAND interrupt - */ -void pl353_smc_clr_nand_int(void) -{ - writel(PL353_SMC_CFG_CLR_INT_CLR_1, - pl353_smc_base + PL353_SMC_CFG_CLR_OFFS); -} -EXPORT_SYMBOL_GPL(pl353_smc_clr_nand_int); - -/** - * pl353_smc_set_ecc_mode - Set SMC ECC mode - * @mode: ECC mode (BYPASS, APB, MEM) - * Return: 0 on success or negative errno. - */ -int pl353_smc_set_ecc_mode(enum pl353_smc_ecc_mode mode) -{ - u32 reg; - int ret = 0; - - switch (mode) { - case PL353_SMC_ECCMODE_BYPASS: - case PL353_SMC_ECCMODE_APB: - case PL353_SMC_ECCMODE_MEM: - - reg = readl(pl353_smc_base + PL353_SMC_ECC_MEMCFG_OFFS); - reg &= ~PL353_SMC_ECC_MEMCFG_MODE_MASK; - reg |= mode << PL353_SMC_ECC_MEMCFG_MODE_SHIFT; - writel(reg, pl353_smc_base + PL353_SMC_ECC_MEMCFG_OFFS); - - break; - default: - ret = -EINVAL; - } - - return ret; -} -EXPORT_SYMBOL_GPL(pl353_smc_set_ecc_mode); - -/** - * pl353_smc_set_ecc_pg_size - Set SMC ECC page size - * @pg_sz: ECC page size - * Return: 0 on success or negative errno. - */ -int pl353_smc_set_ecc_pg_size(unsigned int pg_sz) -{ - u32 reg, sz; - - switch (pg_sz) { - case 0: - sz = 0; - break; - case SZ_512: - sz = 1; - break; - case SZ_1K: - sz = 2; - break; - case SZ_2K: - sz = 3; - break; - default: - return -EINVAL; - } - - reg = readl(pl353_smc_base + PL353_SMC_ECC_MEMCFG_OFFS); - reg &= ~PL353_SMC_ECC_MEMCFG_PGSIZE_MASK; - reg |= sz; - writel(reg, pl353_smc_base + PL353_SMC_ECC_MEMCFG_OFFS); - - return 0; -} -EXPORT_SYMBOL_GPL(pl353_smc_set_ecc_pg_size); - static int __maybe_unused pl353_smc_suspend(struct device *dev) { struct pl353_smc_data *pl353_smc = dev_get_drvdata(dev); @@ -277,8 +36,8 @@ static int __maybe_unused pl353_smc_suspend(struct device *dev) static int __maybe_unused pl353_smc_resume(struct device *dev) { - int ret; struct pl353_smc_data *pl353_smc = dev_get_drvdata(dev); + int ret; ret = clk_enable(pl353_smc->aclk); if (ret) { @@ -296,77 +55,31 @@ static int __maybe_unused pl353_smc_resume(struct device *dev) return ret; } -static struct amba_driver pl353_smc_driver; - static SIMPLE_DEV_PM_OPS(pl353_smc_dev_pm_ops, pl353_smc_suspend, pl353_smc_resume); -/** - * pl353_smc_init_nand_interface - Initialize the NAND interface - * @adev: Pointer to the amba_device struct - * @nand_node: Pointer to the pl353_nand device_node struct - */ -static void pl353_smc_init_nand_interface(struct amba_device *adev, - struct device_node *nand_node) -{ - unsigned long timeout; - - pl353_smc_set_buswidth(PL353_SMC_MEM_WIDTH_8); - writel(PL353_SMC_CFG_CLR_INT_CLR_1, - pl353_smc_base + PL353_SMC_CFG_CLR_OFFS); - writel(PL353_SMC_DC_UPT_NAND_REGS, pl353_smc_base + - PL353_SMC_DIRECT_CMD_OFFS); - - timeout = jiffies + PL353_NAND_ECC_BUSY_TIMEOUT; - /* Wait till the ECC operation is complete */ - do { - if (pl353_smc_ecc_is_busy()) - cpu_relax(); - else - break; - } while (!time_after_eq(jiffies, timeout)); - - if (time_after_eq(jiffies, timeout)) - return; - - writel(PL353_NAND_ECC_CMD1, - pl353_smc_base + PL353_SMC_ECC_MEMCMD1_OFFS); - writel(PL353_NAND_ECC_CMD2, - pl353_smc_base + PL353_SMC_ECC_MEMCMD2_OFFS); -} - static const struct of_device_id pl353_smc_supported_children[] = { { .compatible = "cfi-flash" }, { .compatible = "arm,pl353-nand-r2p1", - .data = pl353_smc_init_nand_interface }, {} }; static int pl353_smc_probe(struct amba_device *adev, const struct amba_id *id) { + struct device_node *of_node = adev->dev.of_node; + const struct of_device_id *match = NULL; struct pl353_smc_data *pl353_smc; struct device_node *child; - struct resource *res; int err; - struct device_node *of_node = adev->dev.of_node; - static void (*init)(struct amba_device *adev, - struct device_node *nand_node); - const struct of_device_id *match = NULL; pl353_smc = devm_kzalloc(&adev->dev, sizeof(*pl353_smc), GFP_KERNEL); if (!pl353_smc) return -ENOMEM; - /* Get the NAND controller virtual address */ - res = &adev->res; - pl353_smc_base = devm_ioremap_resource(&adev->dev, res); - if (IS_ERR(pl353_smc_base)) - return PTR_ERR(pl353_smc_base); - pl353_smc->aclk = devm_clk_get(&adev->dev, "apb_pclk"); if (IS_ERR(pl353_smc->aclk)) { dev_err(&adev->dev, "aclk clock not found.\n"); @@ -388,15 +101,11 @@ static int pl353_smc_probe(struct amba_device *adev, const struct amba_id *id) err = clk_prepare_enable(pl353_smc->memclk); if (err) { dev_err(&adev->dev, "Unable to enable memory clock.\n"); - goto out_clk_dis_aper; + goto disable_axi_clk; } amba_set_drvdata(adev, pl353_smc); - /* clear interrupts */ - writel(PL353_SMC_CFG_CLR_DEFAULT_MASK, - pl353_smc_base + PL353_SMC_CFG_CLR_OFFS); - /* Find compatible children. Only a single child is supported */ for_each_available_child_of_node(of_node, child) { match = of_match_node(pl353_smc_supported_children, child); @@ -407,39 +116,36 @@ static int pl353_smc_probe(struct amba_device *adev, const struct amba_id *id) break; } if (!match) { + err = -ENODEV; dev_err(&adev->dev, "no matching children\n"); - goto out_clk_disable; + goto disable_mem_clk; } - init = match->data; - if (init) - init(adev, child); of_platform_device_create(child, NULL, &adev->dev); + of_node_put(child); return 0; -out_clk_disable: +disable_mem_clk: clk_disable_unprepare(pl353_smc->memclk); -out_clk_dis_aper: +disable_axi_clk: clk_disable_unprepare(pl353_smc->aclk); return err; } -static int pl353_smc_remove(struct amba_device *adev) +static void pl353_smc_remove(struct amba_device *adev) { struct pl353_smc_data *pl353_smc = amba_get_drvdata(adev); clk_disable_unprepare(pl353_smc->memclk); clk_disable_unprepare(pl353_smc->aclk); - - return 0; } static const struct amba_id pl353_ids[] = { { - .id = 0x00041353, - .mask = 0x000fffff, + .id = 0x00041353, + .mask = 0x000fffff, }, { 0, 0 }, }; diff --git a/drivers/memory/renesas-rpc-if.c b/drivers/memory/renesas-rpc-if.c new file mode 100644 index 000000000000..4316988d791a --- /dev/null +++ b/drivers/memory/renesas-rpc-if.c @@ -0,0 +1,734 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Renesas RPC-IF core driver + * + * Copyright (C) 2018-2019 Renesas Solutions Corp. + * Copyright (C) 2019 Macronix International Co., Ltd. + * Copyright (C) 2019-2020 Cogent Embedded, Inc. + */ + +#include <linux/clk.h> +#include <linux/io.h> +#include <linux/module.h> +#include <linux/platform_device.h> +#include <linux/of.h> +#include <linux/of_device.h> +#include <linux/regmap.h> +#include <linux/reset.h> + +#include <memory/renesas-rpc-if.h> + +#define RPCIF_CMNCR 0x0000 /* R/W */ +#define RPCIF_CMNCR_MD BIT(31) +#define RPCIF_CMNCR_MOIIO3(val) (((val) & 0x3) << 22) +#define RPCIF_CMNCR_MOIIO2(val) (((val) & 0x3) << 20) +#define RPCIF_CMNCR_MOIIO1(val) (((val) & 0x3) << 18) +#define RPCIF_CMNCR_MOIIO0(val) (((val) & 0x3) << 16) +#define RPCIF_CMNCR_MOIIO(val) (RPCIF_CMNCR_MOIIO0(val) | RPCIF_CMNCR_MOIIO1(val) | \ + RPCIF_CMNCR_MOIIO2(val) | RPCIF_CMNCR_MOIIO3(val)) +#define RPCIF_CMNCR_IO3FV(val) (((val) & 0x3) << 14) /* documented for RZ/G2L */ +#define RPCIF_CMNCR_IO2FV(val) (((val) & 0x3) << 12) /* documented for RZ/G2L */ +#define RPCIF_CMNCR_IO0FV(val) (((val) & 0x3) << 8) +#define RPCIF_CMNCR_IOFV(val) (RPCIF_CMNCR_IO0FV(val) | RPCIF_CMNCR_IO2FV(val) | \ + RPCIF_CMNCR_IO3FV(val)) +#define RPCIF_CMNCR_BSZ(val) (((val) & 0x3) << 0) + +#define RPCIF_SSLDR 0x0004 /* R/W */ +#define RPCIF_SSLDR_SPNDL(d) (((d) & 0x7) << 16) +#define RPCIF_SSLDR_SLNDL(d) (((d) & 0x7) << 8) +#define RPCIF_SSLDR_SCKDL(d) (((d) & 0x7) << 0) + +#define RPCIF_DRCR 0x000C /* R/W */ +#define RPCIF_DRCR_SSLN BIT(24) +#define RPCIF_DRCR_RBURST(v) ((((v) - 1) & 0x1F) << 16) +#define RPCIF_DRCR_RCF BIT(9) +#define RPCIF_DRCR_RBE BIT(8) +#define RPCIF_DRCR_SSLE BIT(0) + +#define RPCIF_DRCMR 0x0010 /* R/W */ +#define RPCIF_DRCMR_CMD(c) (((c) & 0xFF) << 16) +#define RPCIF_DRCMR_OCMD(c) (((c) & 0xFF) << 0) + +#define RPCIF_DREAR 0x0014 /* R/W */ +#define RPCIF_DREAR_EAV(c) (((c) & 0xF) << 16) +#define RPCIF_DREAR_EAC(c) (((c) & 0x7) << 0) + +#define RPCIF_DROPR 0x0018 /* R/W */ + +#define RPCIF_DRENR 0x001C /* R/W */ +#define RPCIF_DRENR_CDB(o) (u32)((((o) & 0x3) << 30)) +#define RPCIF_DRENR_OCDB(o) (((o) & 0x3) << 28) +#define RPCIF_DRENR_ADB(o) (((o) & 0x3) << 24) +#define RPCIF_DRENR_OPDB(o) (((o) & 0x3) << 20) +#define RPCIF_DRENR_DRDB(o) (((o) & 0x3) << 16) +#define RPCIF_DRENR_DME BIT(15) +#define RPCIF_DRENR_CDE BIT(14) +#define RPCIF_DRENR_OCDE BIT(12) +#define RPCIF_DRENR_ADE(v) (((v) & 0xF) << 8) +#define RPCIF_DRENR_OPDE(v) (((v) & 0xF) << 4) + +#define RPCIF_SMCR 0x0020 /* R/W */ +#define RPCIF_SMCR_SSLKP BIT(8) +#define RPCIF_SMCR_SPIRE BIT(2) +#define RPCIF_SMCR_SPIWE BIT(1) +#define RPCIF_SMCR_SPIE BIT(0) + +#define RPCIF_SMCMR 0x0024 /* R/W */ +#define RPCIF_SMCMR_CMD(c) (((c) & 0xFF) << 16) +#define RPCIF_SMCMR_OCMD(c) (((c) & 0xFF) << 0) + +#define RPCIF_SMADR 0x0028 /* R/W */ + +#define RPCIF_SMOPR 0x002C /* R/W */ +#define RPCIF_SMOPR_OPD3(o) (((o) & 0xFF) << 24) +#define RPCIF_SMOPR_OPD2(o) (((o) & 0xFF) << 16) +#define RPCIF_SMOPR_OPD1(o) (((o) & 0xFF) << 8) +#define RPCIF_SMOPR_OPD0(o) (((o) & 0xFF) << 0) + +#define RPCIF_SMENR 0x0030 /* R/W */ +#define RPCIF_SMENR_CDB(o) (((o) & 0x3) << 30) +#define RPCIF_SMENR_OCDB(o) (((o) & 0x3) << 28) +#define RPCIF_SMENR_ADB(o) (((o) & 0x3) << 24) +#define RPCIF_SMENR_OPDB(o) (((o) & 0x3) << 20) +#define RPCIF_SMENR_SPIDB(o) (((o) & 0x3) << 16) +#define RPCIF_SMENR_DME BIT(15) +#define RPCIF_SMENR_CDE BIT(14) +#define RPCIF_SMENR_OCDE BIT(12) +#define RPCIF_SMENR_ADE(v) (((v) & 0xF) << 8) +#define RPCIF_SMENR_OPDE(v) (((v) & 0xF) << 4) +#define RPCIF_SMENR_SPIDE(v) (((v) & 0xF) << 0) + +#define RPCIF_SMRDR0 0x0038 /* R */ +#define RPCIF_SMRDR1 0x003C /* R */ +#define RPCIF_SMWDR0 0x0040 /* W */ +#define RPCIF_SMWDR1 0x0044 /* W */ + +#define RPCIF_CMNSR 0x0048 /* R */ +#define RPCIF_CMNSR_SSLF BIT(1) +#define RPCIF_CMNSR_TEND BIT(0) + +#define RPCIF_DRDMCR 0x0058 /* R/W */ +#define RPCIF_DMDMCR_DMCYC(v) ((((v) - 1) & 0x1F) << 0) + +#define RPCIF_DRDRENR 0x005C /* R/W */ +#define RPCIF_DRDRENR_HYPE(v) (((v) & 0x7) << 12) +#define RPCIF_DRDRENR_ADDRE BIT(8) +#define RPCIF_DRDRENR_OPDRE BIT(4) +#define RPCIF_DRDRENR_DRDRE BIT(0) + +#define RPCIF_SMDMCR 0x0060 /* R/W */ +#define RPCIF_SMDMCR_DMCYC(v) ((((v) - 1) & 0x1F) << 0) + +#define RPCIF_SMDRENR 0x0064 /* R/W */ +#define RPCIF_SMDRENR_HYPE(v) (((v) & 0x7) << 12) +#define RPCIF_SMDRENR_ADDRE BIT(8) +#define RPCIF_SMDRENR_OPDRE BIT(4) +#define RPCIF_SMDRENR_SPIDRE BIT(0) + +#define RPCIF_PHYADD 0x0070 /* R/W available on R-Car E3/D3/V3M and RZ/G2{E,L} */ +#define RPCIF_PHYWR 0x0074 /* R/W available on R-Car E3/D3/V3M and RZ/G2{E,L} */ + +#define RPCIF_PHYCNT 0x007C /* R/W */ +#define RPCIF_PHYCNT_CAL BIT(31) +#define RPCIF_PHYCNT_OCTA(v) (((v) & 0x3) << 22) +#define RPCIF_PHYCNT_EXDS BIT(21) +#define RPCIF_PHYCNT_OCT BIT(20) +#define RPCIF_PHYCNT_DDRCAL BIT(19) +#define RPCIF_PHYCNT_HS BIT(18) +#define RPCIF_PHYCNT_CKSEL(v) (((v) & 0x3) << 16) /* valid only for RZ/G2L */ +#define RPCIF_PHYCNT_STRTIM(v) (((v) & 0x7) << 15) /* valid for R-Car and RZ/G2{E,H,M,N} */ +#define RPCIF_PHYCNT_WBUF2 BIT(4) +#define RPCIF_PHYCNT_WBUF BIT(2) +#define RPCIF_PHYCNT_PHYMEM(v) (((v) & 0x3) << 0) +#define RPCIF_PHYCNT_PHYMEM_MASK GENMASK(1, 0) + +#define RPCIF_PHYOFFSET1 0x0080 /* R/W */ +#define RPCIF_PHYOFFSET1_DDRTMG(v) (((v) & 0x3) << 28) + +#define RPCIF_PHYOFFSET2 0x0084 /* R/W */ +#define RPCIF_PHYOFFSET2_OCTTMG(v) (((v) & 0x7) << 8) + +#define RPCIF_PHYINT 0x0088 /* R/W */ +#define RPCIF_PHYINT_WPVAL BIT(1) + +static const struct regmap_range rpcif_volatile_ranges[] = { + regmap_reg_range(RPCIF_SMRDR0, RPCIF_SMRDR1), + regmap_reg_range(RPCIF_SMWDR0, RPCIF_SMWDR1), + regmap_reg_range(RPCIF_CMNSR, RPCIF_CMNSR), +}; + +static const struct regmap_access_table rpcif_volatile_table = { + .yes_ranges = rpcif_volatile_ranges, + .n_yes_ranges = ARRAY_SIZE(rpcif_volatile_ranges), +}; + + +/* + * Custom accessor functions to ensure SM[RW]DR[01] are always accessed with + * proper width. Requires rpcif.xfer_size to be correctly set before! + */ +static int rpcif_reg_read(void *context, unsigned int reg, unsigned int *val) +{ + struct rpcif *rpc = context; + + switch (reg) { + case RPCIF_SMRDR0: + case RPCIF_SMWDR0: + switch (rpc->xfer_size) { + case 1: + *val = readb(rpc->base + reg); + return 0; + + case 2: + *val = readw(rpc->base + reg); + return 0; + + case 4: + case 8: + *val = readl(rpc->base + reg); + return 0; + + default: + return -EILSEQ; + } + + case RPCIF_SMRDR1: + case RPCIF_SMWDR1: + if (rpc->xfer_size != 8) + return -EILSEQ; + break; + } + + *val = readl(rpc->base + reg); + return 0; +} + +static int rpcif_reg_write(void *context, unsigned int reg, unsigned int val) +{ + struct rpcif *rpc = context; + + switch (reg) { + case RPCIF_SMWDR0: + switch (rpc->xfer_size) { + case 1: + writeb(val, rpc->base + reg); + return 0; + + case 2: + writew(val, rpc->base + reg); + return 0; + + case 4: + case 8: + writel(val, rpc->base + reg); + return 0; + + default: + return -EILSEQ; + } + + case RPCIF_SMWDR1: + if (rpc->xfer_size != 8) + return -EILSEQ; + break; + + case RPCIF_SMRDR0: + case RPCIF_SMRDR1: + return -EPERM; + } + + writel(val, rpc->base + reg); + return 0; +} + +static const struct regmap_config rpcif_regmap_config = { + .reg_bits = 32, + .val_bits = 32, + .reg_stride = 4, + .reg_read = rpcif_reg_read, + .reg_write = rpcif_reg_write, + .fast_io = true, + .max_register = RPCIF_PHYINT, + .volatile_table = &rpcif_volatile_table, +}; + +int rpcif_sw_init(struct rpcif *rpc, struct device *dev) +{ + struct platform_device *pdev = to_platform_device(dev); + struct resource *res; + + rpc->dev = dev; + + rpc->base = devm_platform_ioremap_resource_byname(pdev, "regs"); + if (IS_ERR(rpc->base)) + return PTR_ERR(rpc->base); + + rpc->regmap = devm_regmap_init(&pdev->dev, NULL, rpc, &rpcif_regmap_config); + if (IS_ERR(rpc->regmap)) { + dev_err(&pdev->dev, + "failed to init regmap for rpcif, error %ld\n", + PTR_ERR(rpc->regmap)); + return PTR_ERR(rpc->regmap); + } + + res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "dirmap"); + rpc->dirmap = devm_ioremap_resource(&pdev->dev, res); + if (IS_ERR(rpc->dirmap)) + return PTR_ERR(rpc->dirmap); + rpc->size = resource_size(res); + + rpc->type = (uintptr_t)of_device_get_match_data(dev); + rpc->rstc = devm_reset_control_get_exclusive(&pdev->dev, NULL); + + return PTR_ERR_OR_ZERO(rpc->rstc); +} +EXPORT_SYMBOL(rpcif_sw_init); + +static void rpcif_rzg2l_timing_adjust_sdr(struct rpcif *rpc) +{ + regmap_write(rpc->regmap, RPCIF_PHYWR, 0xa5390000); + regmap_write(rpc->regmap, RPCIF_PHYADD, 0x80000000); + regmap_write(rpc->regmap, RPCIF_PHYWR, 0x00008080); + regmap_write(rpc->regmap, RPCIF_PHYADD, 0x80000022); + regmap_write(rpc->regmap, RPCIF_PHYWR, 0x00008080); + regmap_write(rpc->regmap, RPCIF_PHYADD, 0x80000024); + regmap_update_bits(rpc->regmap, RPCIF_PHYCNT, RPCIF_PHYCNT_CKSEL(3), + RPCIF_PHYCNT_CKSEL(3)); + regmap_write(rpc->regmap, RPCIF_PHYWR, 0x00000030); + regmap_write(rpc->regmap, RPCIF_PHYADD, 0x80000032); +} + +int rpcif_hw_init(struct rpcif *rpc, bool hyperflash) +{ + u32 dummy; + + pm_runtime_get_sync(rpc->dev); + + if (rpc->type == RPCIF_RZ_G2L) { + int ret; + + ret = reset_control_reset(rpc->rstc); + if (ret) + return ret; + usleep_range(200, 300); + rpcif_rzg2l_timing_adjust_sdr(rpc); + } + + regmap_update_bits(rpc->regmap, RPCIF_PHYCNT, RPCIF_PHYCNT_PHYMEM_MASK, + RPCIF_PHYCNT_PHYMEM(hyperflash ? 3 : 0)); + + if (rpc->type == RPCIF_RCAR_GEN3) + regmap_update_bits(rpc->regmap, RPCIF_PHYCNT, + RPCIF_PHYCNT_STRTIM(7), RPCIF_PHYCNT_STRTIM(7)); + + regmap_update_bits(rpc->regmap, RPCIF_PHYOFFSET1, RPCIF_PHYOFFSET1_DDRTMG(3), + RPCIF_PHYOFFSET1_DDRTMG(3)); + regmap_update_bits(rpc->regmap, RPCIF_PHYOFFSET2, RPCIF_PHYOFFSET2_OCTTMG(7), + RPCIF_PHYOFFSET2_OCTTMG(4)); + + if (hyperflash) + regmap_update_bits(rpc->regmap, RPCIF_PHYINT, + RPCIF_PHYINT_WPVAL, 0); + + if (rpc->type == RPCIF_RCAR_GEN3) + regmap_update_bits(rpc->regmap, RPCIF_CMNCR, + RPCIF_CMNCR_MOIIO(3) | RPCIF_CMNCR_BSZ(3), + RPCIF_CMNCR_MOIIO(3) | + RPCIF_CMNCR_BSZ(hyperflash ? 1 : 0)); + else + regmap_update_bits(rpc->regmap, RPCIF_CMNCR, + RPCIF_CMNCR_MOIIO(3) | RPCIF_CMNCR_IOFV(3) | + RPCIF_CMNCR_BSZ(3), + RPCIF_CMNCR_MOIIO(1) | RPCIF_CMNCR_IOFV(2) | + RPCIF_CMNCR_BSZ(hyperflash ? 1 : 0)); + + /* Set RCF after BSZ update */ + regmap_write(rpc->regmap, RPCIF_DRCR, RPCIF_DRCR_RCF); + /* Dummy read according to spec */ + regmap_read(rpc->regmap, RPCIF_DRCR, &dummy); + regmap_write(rpc->regmap, RPCIF_SSLDR, RPCIF_SSLDR_SPNDL(7) | + RPCIF_SSLDR_SLNDL(7) | RPCIF_SSLDR_SCKDL(7)); + + pm_runtime_put(rpc->dev); + + rpc->bus_size = hyperflash ? 2 : 1; + + return 0; +} +EXPORT_SYMBOL(rpcif_hw_init); + +static int wait_msg_xfer_end(struct rpcif *rpc) +{ + u32 sts; + + return regmap_read_poll_timeout(rpc->regmap, RPCIF_CMNSR, sts, + sts & RPCIF_CMNSR_TEND, 0, + USEC_PER_SEC); +} + +static u8 rpcif_bits_set(struct rpcif *rpc, u32 nbytes) +{ + if (rpc->bus_size == 2) + nbytes /= 2; + nbytes = clamp(nbytes, 1U, 4U); + return GENMASK(3, 4 - nbytes); +} + +static u8 rpcif_bit_size(u8 buswidth) +{ + return buswidth > 4 ? 2 : ilog2(buswidth); +} + +void rpcif_prepare(struct rpcif *rpc, const struct rpcif_op *op, u64 *offs, + size_t *len) +{ + rpc->smcr = 0; + rpc->smadr = 0; + rpc->enable = 0; + rpc->command = 0; + rpc->option = 0; + rpc->dummy = 0; + rpc->ddr = 0; + rpc->xferlen = 0; + + if (op->cmd.buswidth) { + rpc->enable = RPCIF_SMENR_CDE | + RPCIF_SMENR_CDB(rpcif_bit_size(op->cmd.buswidth)); + rpc->command = RPCIF_SMCMR_CMD(op->cmd.opcode); + if (op->cmd.ddr) + rpc->ddr = RPCIF_SMDRENR_HYPE(0x5); + } + if (op->ocmd.buswidth) { + rpc->enable |= RPCIF_SMENR_OCDE | + RPCIF_SMENR_OCDB(rpcif_bit_size(op->ocmd.buswidth)); + rpc->command |= RPCIF_SMCMR_OCMD(op->ocmd.opcode); + } + + if (op->addr.buswidth) { + rpc->enable |= + RPCIF_SMENR_ADB(rpcif_bit_size(op->addr.buswidth)); + if (op->addr.nbytes == 4) + rpc->enable |= RPCIF_SMENR_ADE(0xF); + else + rpc->enable |= RPCIF_SMENR_ADE(GENMASK( + 2, 3 - op->addr.nbytes)); + if (op->addr.ddr) + rpc->ddr |= RPCIF_SMDRENR_ADDRE; + + if (offs && len) + rpc->smadr = *offs; + else + rpc->smadr = op->addr.val; + } + + if (op->dummy.buswidth) { + rpc->enable |= RPCIF_SMENR_DME; + rpc->dummy = RPCIF_SMDMCR_DMCYC(op->dummy.ncycles / + op->dummy.buswidth); + } + + if (op->option.buswidth) { + rpc->enable |= RPCIF_SMENR_OPDE( + rpcif_bits_set(rpc, op->option.nbytes)) | + RPCIF_SMENR_OPDB(rpcif_bit_size(op->option.buswidth)); + if (op->option.ddr) + rpc->ddr |= RPCIF_SMDRENR_OPDRE; + rpc->option = op->option.val; + } + + rpc->dir = op->data.dir; + if (op->data.buswidth) { + u32 nbytes; + + rpc->buffer = op->data.buf.in; + switch (op->data.dir) { + case RPCIF_DATA_IN: + rpc->smcr = RPCIF_SMCR_SPIRE; + break; + case RPCIF_DATA_OUT: + rpc->smcr = RPCIF_SMCR_SPIWE; + break; + default: + break; + } + if (op->data.ddr) + rpc->ddr |= RPCIF_SMDRENR_SPIDRE; + + if (offs && len) + nbytes = *len; + else + nbytes = op->data.nbytes; + rpc->xferlen = nbytes; + + rpc->enable |= RPCIF_SMENR_SPIDB(rpcif_bit_size(op->data.buswidth)); + } +} +EXPORT_SYMBOL(rpcif_prepare); + +int rpcif_manual_xfer(struct rpcif *rpc) +{ + u32 smenr, smcr, pos = 0, max = rpc->bus_size == 2 ? 8 : 4; + int ret = 0; + + pm_runtime_get_sync(rpc->dev); + + regmap_update_bits(rpc->regmap, RPCIF_PHYCNT, + RPCIF_PHYCNT_CAL, RPCIF_PHYCNT_CAL); + regmap_update_bits(rpc->regmap, RPCIF_CMNCR, + RPCIF_CMNCR_MD, RPCIF_CMNCR_MD); + regmap_write(rpc->regmap, RPCIF_SMCMR, rpc->command); + regmap_write(rpc->regmap, RPCIF_SMOPR, rpc->option); + regmap_write(rpc->regmap, RPCIF_SMDMCR, rpc->dummy); + regmap_write(rpc->regmap, RPCIF_SMDRENR, rpc->ddr); + regmap_write(rpc->regmap, RPCIF_SMADR, rpc->smadr); + smenr = rpc->enable; + + switch (rpc->dir) { + case RPCIF_DATA_OUT: + while (pos < rpc->xferlen) { + u32 bytes_left = rpc->xferlen - pos; + u32 nbytes, data[2], *p = data; + + smcr = rpc->smcr | RPCIF_SMCR_SPIE; + + /* nbytes may only be 1, 2, 4, or 8 */ + nbytes = bytes_left >= max ? max : (1 << ilog2(bytes_left)); + if (bytes_left > nbytes) + smcr |= RPCIF_SMCR_SSLKP; + + smenr |= RPCIF_SMENR_SPIDE(rpcif_bits_set(rpc, nbytes)); + regmap_write(rpc->regmap, RPCIF_SMENR, smenr); + rpc->xfer_size = nbytes; + + memcpy(data, rpc->buffer + pos, nbytes); + if (nbytes == 8) + regmap_write(rpc->regmap, RPCIF_SMWDR1, *p++); + regmap_write(rpc->regmap, RPCIF_SMWDR0, *p); + + regmap_write(rpc->regmap, RPCIF_SMCR, smcr); + ret = wait_msg_xfer_end(rpc); + if (ret) + goto err_out; + + pos += nbytes; + smenr = rpc->enable & + ~RPCIF_SMENR_CDE & ~RPCIF_SMENR_ADE(0xF); + } + break; + case RPCIF_DATA_IN: + /* + * RPC-IF spoils the data for the commands without an address + * phase (like RDID) in the manual mode, so we'll have to work + * around this issue by using the external address space read + * mode instead. + */ + if (!(smenr & RPCIF_SMENR_ADE(0xF)) && rpc->dirmap) { + u32 dummy; + + regmap_update_bits(rpc->regmap, RPCIF_CMNCR, + RPCIF_CMNCR_MD, 0); + regmap_write(rpc->regmap, RPCIF_DRCR, + RPCIF_DRCR_RBURST(32) | RPCIF_DRCR_RBE); + regmap_write(rpc->regmap, RPCIF_DRCMR, rpc->command); + regmap_write(rpc->regmap, RPCIF_DREAR, + RPCIF_DREAR_EAC(1)); + regmap_write(rpc->regmap, RPCIF_DROPR, rpc->option); + regmap_write(rpc->regmap, RPCIF_DRENR, + smenr & ~RPCIF_SMENR_SPIDE(0xF)); + regmap_write(rpc->regmap, RPCIF_DRDMCR, rpc->dummy); + regmap_write(rpc->regmap, RPCIF_DRDRENR, rpc->ddr); + memcpy_fromio(rpc->buffer, rpc->dirmap, rpc->xferlen); + regmap_write(rpc->regmap, RPCIF_DRCR, RPCIF_DRCR_RCF); + /* Dummy read according to spec */ + regmap_read(rpc->regmap, RPCIF_DRCR, &dummy); + break; + } + while (pos < rpc->xferlen) { + u32 bytes_left = rpc->xferlen - pos; + u32 nbytes, data[2], *p = data; + + /* nbytes may only be 1, 2, 4, or 8 */ + nbytes = bytes_left >= max ? max : (1 << ilog2(bytes_left)); + + regmap_write(rpc->regmap, RPCIF_SMADR, + rpc->smadr + pos); + smenr &= ~RPCIF_SMENR_SPIDE(0xF); + smenr |= RPCIF_SMENR_SPIDE(rpcif_bits_set(rpc, nbytes)); + regmap_write(rpc->regmap, RPCIF_SMENR, smenr); + regmap_write(rpc->regmap, RPCIF_SMCR, + rpc->smcr | RPCIF_SMCR_SPIE); + rpc->xfer_size = nbytes; + ret = wait_msg_xfer_end(rpc); + if (ret) + goto err_out; + + if (nbytes == 8) + regmap_read(rpc->regmap, RPCIF_SMRDR1, p++); + regmap_read(rpc->regmap, RPCIF_SMRDR0, p); + memcpy(rpc->buffer + pos, data, nbytes); + + pos += nbytes; + } + break; + default: + regmap_write(rpc->regmap, RPCIF_SMENR, rpc->enable); + regmap_write(rpc->regmap, RPCIF_SMCR, + rpc->smcr | RPCIF_SMCR_SPIE); + ret = wait_msg_xfer_end(rpc); + if (ret) + goto err_out; + } + +exit: + pm_runtime_put(rpc->dev); + return ret; + +err_out: + if (reset_control_reset(rpc->rstc)) + dev_err(rpc->dev, "Failed to reset HW\n"); + rpcif_hw_init(rpc, rpc->bus_size == 2); + goto exit; +} +EXPORT_SYMBOL(rpcif_manual_xfer); + +static void memcpy_fromio_readw(void *to, + const void __iomem *from, + size_t count) +{ + const int maxw = (IS_ENABLED(CONFIG_64BIT)) ? 8 : 4; + u8 buf[2]; + + if (count && ((unsigned long)from & 1)) { + *(u16 *)buf = __raw_readw((void __iomem *)((unsigned long)from & ~1)); + *(u8 *)to = buf[1]; + from++; + to++; + count--; + } + while (count >= 2 && !IS_ALIGNED((unsigned long)from, maxw)) { + *(u16 *)to = __raw_readw(from); + from += 2; + to += 2; + count -= 2; + } + while (count >= maxw) { +#ifdef CONFIG_64BIT + *(u64 *)to = __raw_readq(from); +#else + *(u32 *)to = __raw_readl(from); +#endif + from += maxw; + to += maxw; + count -= maxw; + } + while (count >= 2) { + *(u16 *)to = __raw_readw(from); + from += 2; + to += 2; + count -= 2; + } + if (count) { + *(u16 *)buf = __raw_readw(from); + *(u8 *)to = buf[0]; + } +} + +ssize_t rpcif_dirmap_read(struct rpcif *rpc, u64 offs, size_t len, void *buf) +{ + loff_t from = offs & (rpc->size - 1); + size_t size = rpc->size - from; + + if (len > size) + len = size; + + pm_runtime_get_sync(rpc->dev); + + regmap_update_bits(rpc->regmap, RPCIF_CMNCR, RPCIF_CMNCR_MD, 0); + regmap_write(rpc->regmap, RPCIF_DRCR, 0); + regmap_write(rpc->regmap, RPCIF_DRCMR, rpc->command); + regmap_write(rpc->regmap, RPCIF_DREAR, + RPCIF_DREAR_EAV(offs >> 25) | RPCIF_DREAR_EAC(1)); + regmap_write(rpc->regmap, RPCIF_DROPR, rpc->option); + regmap_write(rpc->regmap, RPCIF_DRENR, + rpc->enable & ~RPCIF_SMENR_SPIDE(0xF)); + regmap_write(rpc->regmap, RPCIF_DRDMCR, rpc->dummy); + regmap_write(rpc->regmap, RPCIF_DRDRENR, rpc->ddr); + + if (rpc->bus_size == 2) + memcpy_fromio_readw(buf, rpc->dirmap + from, len); + else + memcpy_fromio(buf, rpc->dirmap + from, len); + + pm_runtime_put(rpc->dev); + + return len; +} +EXPORT_SYMBOL(rpcif_dirmap_read); + +static int rpcif_probe(struct platform_device *pdev) +{ + struct platform_device *vdev; + struct device_node *flash; + const char *name; + int ret; + + flash = of_get_next_child(pdev->dev.of_node, NULL); + if (!flash) { + dev_warn(&pdev->dev, "no flash node found\n"); + return -ENODEV; + } + + if (of_device_is_compatible(flash, "jedec,spi-nor")) { + name = "rpc-if-spi"; + } else if (of_device_is_compatible(flash, "cfi-flash")) { + name = "rpc-if-hyperflash"; + } else { + of_node_put(flash); + dev_warn(&pdev->dev, "unknown flash type\n"); + return -ENODEV; + } + of_node_put(flash); + + vdev = platform_device_alloc(name, pdev->id); + if (!vdev) + return -ENOMEM; + vdev->dev.parent = &pdev->dev; + platform_set_drvdata(pdev, vdev); + + ret = platform_device_add(vdev); + if (ret) { + platform_device_put(vdev); + return ret; + } + + return 0; +} + +static int rpcif_remove(struct platform_device *pdev) +{ + struct platform_device *vdev = platform_get_drvdata(pdev); + + platform_device_unregister(vdev); + + return 0; +} + +static const struct of_device_id rpcif_of_match[] = { + { .compatible = "renesas,rcar-gen3-rpc-if", .data = (void *)RPCIF_RCAR_GEN3 }, + { .compatible = "renesas,rzg2l-rpc-if", .data = (void *)RPCIF_RZ_G2L }, + {}, +}; +MODULE_DEVICE_TABLE(of, rpcif_of_match); + +static struct platform_driver rpcif_driver = { + .probe = rpcif_probe, + .remove = rpcif_remove, + .driver = { + .name = "rpc-if", + .of_match_table = rpcif_of_match, + }, +}; +module_platform_driver(rpcif_driver); + +MODULE_DESCRIPTION("Renesas RPC-IF core driver"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/memory/samsung/Kconfig b/drivers/memory/samsung/Kconfig index 20a8406ce786..7fb70f573031 100644 --- a/drivers/memory/samsung/Kconfig +++ b/drivers/memory/samsung/Kconfig @@ -14,14 +14,22 @@ config EXYNOS5422_DMC depends on DEVFREQ_GOV_SIMPLE_ONDEMAND depends on (PM_DEVFREQ && PM_DEVFREQ_EVENT) help - This adds driver for Exynos5422 DMC (Dynamic Memory Controller). - The driver provides support for Dynamic Voltage and Frequency Scaling in - DMC and DRAM. It also supports changing timings of DRAM running with - different frequency. The timings are calculated based on DT memory - information. + This adds driver for Samsung Exynos5422 SoC DMC (Dynamic Memory + Controller). The driver provides support for Dynamic Voltage and + Frequency Scaling in DMC and DRAM. It also supports changing timings + of DRAM running with different frequency. The timings are calculated + based on DT memory information. + If unsure, say Y on devices with Samsung Exynos SoCs. config EXYNOS_SROM bool "Exynos SROM controller driver" if COMPILE_TEST depends on (ARM && ARCH_EXYNOS) || (COMPILE_TEST && HAS_IOMEM) + help + This adds driver for Samsung Exynos SoC SROM controller. The driver + in basic operation mode only saves and restores SROM registers + during suspend. If however appropriate device tree configuration + is provided, the driver enables support for external memory + or external devices. + If unsure, say Y on devices with Samsung Exynos SoCs. endif diff --git a/drivers/memory/samsung/exynos-srom.c b/drivers/memory/samsung/exynos-srom.c index 6510d7bab217..e73dd330af47 100644 --- a/drivers/memory/samsung/exynos-srom.c +++ b/drivers/memory/samsung/exynos-srom.c @@ -47,9 +47,9 @@ struct exynos_srom { struct exynos_srom_reg_dump *reg_offset; }; -static struct exynos_srom_reg_dump *exynos_srom_alloc_reg_dump( - const unsigned long *rdump, - unsigned long nr_rdump) +static struct exynos_srom_reg_dump * +exynos_srom_alloc_reg_dump(const unsigned long *rdump, + unsigned long nr_rdump) { struct exynos_srom_reg_dump *rd; unsigned int i; @@ -116,7 +116,7 @@ static int exynos_srom_probe(struct platform_device *pdev) } srom = devm_kzalloc(&pdev->dev, - sizeof(struct exynos_srom), GFP_KERNEL); + sizeof(struct exynos_srom), GFP_KERNEL); if (!srom) return -ENOMEM; @@ -130,7 +130,7 @@ static int exynos_srom_probe(struct platform_device *pdev) platform_set_drvdata(pdev, srom); srom->reg_offset = exynos_srom_alloc_reg_dump(exynos_srom_offsets, - ARRAY_SIZE(exynos_srom_offsets)); + ARRAY_SIZE(exynos_srom_offsets)); if (!srom->reg_offset) { iounmap(srom->reg_base); return -ENOMEM; @@ -157,16 +157,16 @@ static int exynos_srom_probe(struct platform_device *pdev) #ifdef CONFIG_PM_SLEEP static void exynos_srom_save(void __iomem *base, - struct exynos_srom_reg_dump *rd, - unsigned int num_regs) + struct exynos_srom_reg_dump *rd, + unsigned int num_regs) { for (; num_regs > 0; --num_regs, ++rd) rd->value = readl(base + rd->offset); } static void exynos_srom_restore(void __iomem *base, - const struct exynos_srom_reg_dump *rd, - unsigned int num_regs) + const struct exynos_srom_reg_dump *rd, + unsigned int num_regs) { for (; num_regs > 0; --num_regs, ++rd) writel(rd->value, base + rd->offset); @@ -177,7 +177,7 @@ static int exynos_srom_suspend(struct device *dev) struct exynos_srom *srom = dev_get_drvdata(dev); exynos_srom_save(srom->reg_base, srom->reg_offset, - ARRAY_SIZE(exynos_srom_offsets)); + ARRAY_SIZE(exynos_srom_offsets)); return 0; } @@ -186,7 +186,7 @@ static int exynos_srom_resume(struct device *dev) struct exynos_srom *srom = dev_get_drvdata(dev); exynos_srom_restore(srom->reg_base, srom->reg_offset, - ARRAY_SIZE(exynos_srom_offsets)); + ARRAY_SIZE(exynos_srom_offsets)); return 0; } #endif diff --git a/drivers/memory/samsung/exynos5422-dmc.c b/drivers/memory/samsung/exynos5422-dmc.c index 81a1b1d01683..c491cd549644 100644 --- a/drivers/memory/samsung/exynos5422-dmc.c +++ b/drivers/memory/samsung/exynos5422-dmc.c @@ -12,6 +12,7 @@ #include <linux/io.h> #include <linux/mfd/syscon.h> #include <linux/module.h> +#include <linux/moduleparam.h> #include <linux/of_device.h> #include <linux/pm_opp.h> #include <linux/platform_device.h> @@ -21,6 +22,10 @@ #include "../jedec_ddr.h" #include "../of_memory.h" +static int irqmode; +module_param(irqmode, int, 0644); +MODULE_PARM_DESC(irqmode, "Enable IRQ mode (0=off [default], 1=on)"); + #define EXYNOS5_DREXI_TIMINGAREF (0x0030) #define EXYNOS5_DREXI_TIMINGROW0 (0x0034) #define EXYNOS5_DREXI_TIMINGDATA0 (0x0038) @@ -93,6 +98,8 @@ /** * struct dmc_opp_table - Operating level desciption + * @freq_hz: target frequency in Hz + * @volt_uv: target voltage in uV * * Covers frequency and voltage settings of the DMC operating mode. */ @@ -103,6 +110,41 @@ struct dmc_opp_table { /** * struct exynos5_dmc - main structure describing DMC device + * @dev: DMC device + * @df: devfreq device structure returned by devfreq framework + * @gov_data: configuration of devfreq governor + * @base_drexi0: DREX0 registers mapping + * @base_drexi1: DREX1 registers mapping + * @clk_regmap: regmap for clock controller registers + * @lock: protects curr_rate and frequency/voltage setting section + * @curr_rate: current frequency + * @curr_volt: current voltage + * @opp: OPP table + * @opp_count: number of 'opp' elements + * @timings_arr_size: number of 'timings' elements + * @timing_row: values for timing row register, for each OPP + * @timing_data: values for timing data register, for each OPP + * @timing_power: balues for timing power register, for each OPP + * @timings: DDR memory timings, from device tree + * @min_tck: DDR memory minimum timing values, from device tree + * @bypass_timing_row: value for timing row register for bypass timings + * @bypass_timing_data: value for timing data register for bypass timings + * @bypass_timing_power: value for timing power register for bypass + * timings + * @vdd_mif: Memory interface regulator + * @fout_spll: clock: SPLL + * @fout_bpll: clock: BPLL + * @mout_spll: clock: mux SPLL + * @mout_bpll: clock: mux BPLL + * @mout_mclk_cdrex: clock: mux mclk_cdrex + * @mout_mx_mspll_ccore: clock: mux mx_mspll_ccore + * @counter: devfreq events + * @num_counters: number of 'counter' elements + * @last_overflow_ts: time (in ns) of last overflow of each DREX + * @load: utilization in percents + * @total: total time between devfreq events + * @in_irq_mode: whether running in interrupt mode (true) + * or polling (false) * * The main structure for the Dynamic Memory Controller which covers clocks, * memory regions, HW information, parameters and current operating mode. @@ -114,12 +156,11 @@ struct exynos5_dmc { void __iomem *base_drexi0; void __iomem *base_drexi1; struct regmap *clk_regmap; + /* Protects curr_rate and frequency/voltage setting section */ struct mutex lock; unsigned long curr_rate; unsigned long curr_volt; - unsigned long bypass_rate; struct dmc_opp_table *opp; - struct dmc_opp_table opp_bypass; int opp_count; u32 timings_arr_size; u32 *timing_row; @@ -137,8 +178,6 @@ struct exynos5_dmc { struct clk *mout_bpll; struct clk *mout_mclk_cdrex; struct clk *mout_mx_mspll_ccore; - struct clk *mx_mspll_ccore_phy; - struct clk *mout_mx_mspll_ccore_phy; struct devfreq_event_dev **counter; int num_counters; u64 last_overflow_ts[2]; @@ -164,7 +203,7 @@ struct timing_reg { unsigned int val; }; -static const struct timing_reg timing_row[] = { +static const struct timing_reg timing_row_reg_fields[] = { TIMING_FIELD("tRFC", 24, 31), TIMING_FIELD("tRRD", 20, 23), TIMING_FIELD("tRP", 16, 19), @@ -173,7 +212,7 @@ static const struct timing_reg timing_row[] = { TIMING_FIELD("tRAS", 0, 5), }; -static const struct timing_reg timing_data[] = { +static const struct timing_reg timing_data_reg_fields[] = { TIMING_FIELD("tWTR", 28, 31), TIMING_FIELD("tWR", 24, 27), TIMING_FIELD("tRTP", 20, 23), @@ -184,7 +223,7 @@ static const struct timing_reg timing_data[] = { TIMING_FIELD("RL", 0, 3), }; -static const struct timing_reg timing_power[] = { +static const struct timing_reg timing_power_reg_fields[] = { TIMING_FIELD("tFAW", 26, 31), TIMING_FIELD("tXSR", 16, 25), TIMING_FIELD("tXP", 8, 15), @@ -192,8 +231,9 @@ static const struct timing_reg timing_power[] = { TIMING_FIELD("tMRD", 0, 3), }; -#define TIMING_COUNT (ARRAY_SIZE(timing_row) + ARRAY_SIZE(timing_data) + \ - ARRAY_SIZE(timing_power)) +#define TIMING_COUNT (ARRAY_SIZE(timing_row_reg_fields) + \ + ARRAY_SIZE(timing_data_reg_fields) + \ + ARRAY_SIZE(timing_power_reg_fields)) static int exynos5_counters_set_event(struct exynos5_dmc *dmc) { @@ -238,7 +278,7 @@ static int exynos5_counters_disable_edev(struct exynos5_dmc *dmc) } /** - * find_target_freq_id() - Finds requested frequency in local DMC configuration + * find_target_freq_idx() - Finds requested frequency in local DMC configuration * @dmc: device for which the information is checked * @target_rate: requested frequency in KHz * @@ -270,12 +310,14 @@ static int find_target_freq_idx(struct exynos5_dmc *dmc, * This function switches between these banks according to the * currently used clock source. */ -static void exynos5_switch_timing_regs(struct exynos5_dmc *dmc, bool set) +static int exynos5_switch_timing_regs(struct exynos5_dmc *dmc, bool set) { unsigned int reg; int ret; ret = regmap_read(dmc->clk_regmap, CDREX_LPDDR3PHY_CON3, ®); + if (ret) + return ret; if (set) reg |= EXYNOS5_TIMING_SET_SWI; @@ -283,6 +325,8 @@ static void exynos5_switch_timing_regs(struct exynos5_dmc *dmc, bool set) reg &= ~EXYNOS5_TIMING_SET_SWI; regmap_write(dmc->clk_regmap, CDREX_LPDDR3PHY_CON3, reg); + + return 0; } /** @@ -299,7 +343,7 @@ static int exynos5_init_freq_table(struct exynos5_dmc *dmc, int idx; unsigned long freq; - ret = dev_pm_opp_of_add_table(dmc->dev); + ret = devm_pm_opp_of_add_table(dmc->dev); if (ret < 0) { dev_err(dmc->dev, "Failed to get OPP table\n"); return ret; @@ -310,7 +354,7 @@ static int exynos5_init_freq_table(struct exynos5_dmc *dmc, dmc->opp = devm_kmalloc_array(dmc->dev, dmc->opp_count, sizeof(struct dmc_opp_table), GFP_KERNEL); if (!dmc->opp) - goto err_opp; + return -ENOMEM; idx = dmc->opp_count - 1; for (i = 0, freq = ULONG_MAX; i < dmc->opp_count; i++, freq--) { @@ -318,7 +362,7 @@ static int exynos5_init_freq_table(struct exynos5_dmc *dmc, opp = dev_pm_opp_find_freq_floor(dmc->dev, &freq); if (IS_ERR(opp)) - goto err_opp; + return PTR_ERR(opp); dmc->opp[idx - i].freq_hz = freq; dmc->opp[idx - i].volt_uv = dev_pm_opp_get_voltage(opp); @@ -327,17 +371,11 @@ static int exynos5_init_freq_table(struct exynos5_dmc *dmc, } return 0; - -err_opp: - dev_pm_opp_of_remove_table(dmc->dev); - - return -EINVAL; } /** * exynos5_set_bypass_dram_timings() - Low-level changes of the DRAM timings * @dmc: device for which the new settings is going to be applied - * @param: DRAM parameters which passes timing data * * Low-level function for changing timings for DRAM memory clocking from * 'bypass' clock source (fixed frequency @400MHz). @@ -444,9 +482,6 @@ static int exynos5_dmc_align_bypass_voltage(struct exynos5_dmc *dmc, unsigned long target_volt) { int ret = 0; - unsigned long bypass_volt = dmc->opp_bypass.volt_uv; - - target_volt = max(bypass_volt, target_volt); if (dmc->curr_volt >= target_volt) return 0; @@ -516,7 +551,7 @@ exynos5_dmc_switch_to_bypass_configuration(struct exynos5_dmc *dmc, /* * Delays are long enough, so use them for the new coming clock. */ - exynos5_switch_timing_regs(dmc, USE_MX_MSPLL_TIMINGS); + ret = exynos5_switch_timing_regs(dmc, USE_MX_MSPLL_TIMINGS); return ret; } @@ -577,7 +612,9 @@ exynos5_dmc_change_freq_and_volt(struct exynos5_dmc *dmc, clk_set_rate(dmc->fout_bpll, target_rate); - exynos5_switch_timing_regs(dmc, USE_BPLL_TIMINGS); + ret = exynos5_switch_timing_regs(dmc, USE_BPLL_TIMINGS); + if (ret) + goto disable_clocks; ret = clk_set_parent(dmc->mout_mclk_cdrex, dmc->mout_bpll); if (ret) @@ -606,6 +643,7 @@ disable_clocks: * requested * @target_volt: returned voltage which corresponds to the returned * frequency + * @flags: devfreq flags provided for this frequency change request * * Function gets requested frequency and checks OPP framework for needed * frequency and voltage. It populates the values 'target_rate' and @@ -897,7 +935,10 @@ static int exynos5_dmc_get_status(struct device *dev, int ret; if (dmc->in_irq_mode) { + mutex_lock(&dmc->lock); stat->current_frequency = dmc->curr_rate; + mutex_unlock(&dmc->lock); + stat->busy_time = dmc->load; stat->total_time = dmc->total; } else { @@ -939,19 +980,20 @@ static int exynos5_dmc_get_cur_freq(struct device *dev, unsigned long *freq) return 0; } -/** +/* * exynos5_dmc_df_profile - Devfreq governor's profile structure * * It provides to the devfreq framework needed functions and polling period. */ static struct devfreq_dev_profile exynos5_dmc_df_profile = { + .timer = DEVFREQ_TIMER_DELAYED, .target = exynos5_dmc_target, .get_dev_status = exynos5_dmc_get_status, .get_cur_freq = exynos5_dmc_get_cur_freq, }; /** - * exynos5_dmc_align_initial_frequency() - Align initial frequency value + * exynos5_dmc_align_init_freq() - Align initial frequency value * @dmc: device for which the frequency is going to be set * @bootloader_init_freq: initial frequency set by the bootloader in KHz * @@ -981,7 +1023,9 @@ exynos5_dmc_align_init_freq(struct exynos5_dmc *dmc, /** * create_timings_aligned() - Create register values and align with standard * @dmc: device for which the frequency is going to be set - * @idx: speed bin in the OPP table + * @reg_timing_row: array to fill with values for timing row register + * @reg_timing_data: array to fill with values for timing data register + * @reg_timing_power: array to fill with values for timing power register * @clk_period_ps: the period of the clock, known as tCK * * The function calculates timings and creates a register value ready for @@ -1006,117 +1050,117 @@ static int create_timings_aligned(struct exynos5_dmc *dmc, u32 *reg_timing_row, val = dmc->timings->tRFC / clk_period_ps; val += dmc->timings->tRFC % clk_period_ps ? 1 : 0; val = max(val, dmc->min_tck->tRFC); - reg = &timing_row[0]; + reg = &timing_row_reg_fields[0]; *reg_timing_row |= TIMING_VAL2REG(reg, val); val = dmc->timings->tRRD / clk_period_ps; val += dmc->timings->tRRD % clk_period_ps ? 1 : 0; val = max(val, dmc->min_tck->tRRD); - reg = &timing_row[1]; + reg = &timing_row_reg_fields[1]; *reg_timing_row |= TIMING_VAL2REG(reg, val); val = dmc->timings->tRPab / clk_period_ps; val += dmc->timings->tRPab % clk_period_ps ? 1 : 0; val = max(val, dmc->min_tck->tRPab); - reg = &timing_row[2]; + reg = &timing_row_reg_fields[2]; *reg_timing_row |= TIMING_VAL2REG(reg, val); val = dmc->timings->tRCD / clk_period_ps; val += dmc->timings->tRCD % clk_period_ps ? 1 : 0; val = max(val, dmc->min_tck->tRCD); - reg = &timing_row[3]; + reg = &timing_row_reg_fields[3]; *reg_timing_row |= TIMING_VAL2REG(reg, val); val = dmc->timings->tRC / clk_period_ps; val += dmc->timings->tRC % clk_period_ps ? 1 : 0; val = max(val, dmc->min_tck->tRC); - reg = &timing_row[4]; + reg = &timing_row_reg_fields[4]; *reg_timing_row |= TIMING_VAL2REG(reg, val); val = dmc->timings->tRAS / clk_period_ps; val += dmc->timings->tRAS % clk_period_ps ? 1 : 0; val = max(val, dmc->min_tck->tRAS); - reg = &timing_row[5]; + reg = &timing_row_reg_fields[5]; *reg_timing_row |= TIMING_VAL2REG(reg, val); /* data related timings */ val = dmc->timings->tWTR / clk_period_ps; val += dmc->timings->tWTR % clk_period_ps ? 1 : 0; val = max(val, dmc->min_tck->tWTR); - reg = &timing_data[0]; + reg = &timing_data_reg_fields[0]; *reg_timing_data |= TIMING_VAL2REG(reg, val); val = dmc->timings->tWR / clk_period_ps; val += dmc->timings->tWR % clk_period_ps ? 1 : 0; val = max(val, dmc->min_tck->tWR); - reg = &timing_data[1]; + reg = &timing_data_reg_fields[1]; *reg_timing_data |= TIMING_VAL2REG(reg, val); val = dmc->timings->tRTP / clk_period_ps; val += dmc->timings->tRTP % clk_period_ps ? 1 : 0; val = max(val, dmc->min_tck->tRTP); - reg = &timing_data[2]; + reg = &timing_data_reg_fields[2]; *reg_timing_data |= TIMING_VAL2REG(reg, val); val = dmc->timings->tW2W_C2C / clk_period_ps; val += dmc->timings->tW2W_C2C % clk_period_ps ? 1 : 0; val = max(val, dmc->min_tck->tW2W_C2C); - reg = &timing_data[3]; + reg = &timing_data_reg_fields[3]; *reg_timing_data |= TIMING_VAL2REG(reg, val); val = dmc->timings->tR2R_C2C / clk_period_ps; val += dmc->timings->tR2R_C2C % clk_period_ps ? 1 : 0; val = max(val, dmc->min_tck->tR2R_C2C); - reg = &timing_data[4]; + reg = &timing_data_reg_fields[4]; *reg_timing_data |= TIMING_VAL2REG(reg, val); val = dmc->timings->tWL / clk_period_ps; val += dmc->timings->tWL % clk_period_ps ? 1 : 0; val = max(val, dmc->min_tck->tWL); - reg = &timing_data[5]; + reg = &timing_data_reg_fields[5]; *reg_timing_data |= TIMING_VAL2REG(reg, val); val = dmc->timings->tDQSCK / clk_period_ps; val += dmc->timings->tDQSCK % clk_period_ps ? 1 : 0; val = max(val, dmc->min_tck->tDQSCK); - reg = &timing_data[6]; + reg = &timing_data_reg_fields[6]; *reg_timing_data |= TIMING_VAL2REG(reg, val); val = dmc->timings->tRL / clk_period_ps; val += dmc->timings->tRL % clk_period_ps ? 1 : 0; val = max(val, dmc->min_tck->tRL); - reg = &timing_data[7]; + reg = &timing_data_reg_fields[7]; *reg_timing_data |= TIMING_VAL2REG(reg, val); /* power related timings */ val = dmc->timings->tFAW / clk_period_ps; val += dmc->timings->tFAW % clk_period_ps ? 1 : 0; - val = max(val, dmc->min_tck->tXP); - reg = &timing_power[0]; + val = max(val, dmc->min_tck->tFAW); + reg = &timing_power_reg_fields[0]; *reg_timing_power |= TIMING_VAL2REG(reg, val); val = dmc->timings->tXSR / clk_period_ps; val += dmc->timings->tXSR % clk_period_ps ? 1 : 0; val = max(val, dmc->min_tck->tXSR); - reg = &timing_power[1]; + reg = &timing_power_reg_fields[1]; *reg_timing_power |= TIMING_VAL2REG(reg, val); val = dmc->timings->tXP / clk_period_ps; val += dmc->timings->tXP % clk_period_ps ? 1 : 0; val = max(val, dmc->min_tck->tXP); - reg = &timing_power[2]; + reg = &timing_power_reg_fields[2]; *reg_timing_power |= TIMING_VAL2REG(reg, val); val = dmc->timings->tCKE / clk_period_ps; val += dmc->timings->tCKE % clk_period_ps ? 1 : 0; val = max(val, dmc->min_tck->tCKE); - reg = &timing_power[3]; + reg = &timing_power_reg_fields[3]; *reg_timing_power |= TIMING_VAL2REG(reg, val); val = dmc->timings->tMRD / clk_period_ps; val += dmc->timings->tMRD % clk_period_ps ? 1 : 0; val = max(val, dmc->min_tck->tMRD); - reg = &timing_power[4]; + reg = &timing_power_reg_fields[4]; *reg_timing_power |= TIMING_VAL2REG(reg, val); return 0; @@ -1143,33 +1187,39 @@ static int of_get_dram_timings(struct exynos5_dmc *dmc) dmc->timing_row = devm_kmalloc_array(dmc->dev, TIMING_COUNT, sizeof(u32), GFP_KERNEL); - if (!dmc->timing_row) - return -ENOMEM; + if (!dmc->timing_row) { + ret = -ENOMEM; + goto put_node; + } dmc->timing_data = devm_kmalloc_array(dmc->dev, TIMING_COUNT, sizeof(u32), GFP_KERNEL); - if (!dmc->timing_data) - return -ENOMEM; + if (!dmc->timing_data) { + ret = -ENOMEM; + goto put_node; + } dmc->timing_power = devm_kmalloc_array(dmc->dev, TIMING_COUNT, sizeof(u32), GFP_KERNEL); - if (!dmc->timing_power) - return -ENOMEM; + if (!dmc->timing_power) { + ret = -ENOMEM; + goto put_node; + } dmc->timings = of_lpddr3_get_ddr_timings(np_ddr, dmc->dev, DDR_TYPE_LPDDR3, &dmc->timings_arr_size); if (!dmc->timings) { - of_node_put(np_ddr); dev_warn(dmc->dev, "could not get timings from DT\n"); - return -EINVAL; + ret = -EINVAL; + goto put_node; } dmc->min_tck = of_lpddr3_get_min_tck(np_ddr, dmc->dev); if (!dmc->min_tck) { - of_node_put(np_ddr); dev_warn(dmc->dev, "could not get tck from DT\n"); - return -EINVAL; + ret = -EINVAL; + goto put_node; } /* Sorted array of OPPs with frequency ascending */ @@ -1183,13 +1233,14 @@ static int of_get_dram_timings(struct exynos5_dmc *dmc) clk_period_ps); } - of_node_put(np_ddr); /* Take the highest frequency's timings as 'bypass' */ dmc->bypass_timing_row = dmc->timing_row[idx - 1]; dmc->bypass_timing_data = dmc->timing_data[idx - 1]; dmc->bypass_timing_power = dmc->timing_power[idx - 1]; +put_node: + of_node_put(np_ddr); return ret; } @@ -1249,9 +1300,9 @@ static int exynos5_dmc_init_clks(struct exynos5_dmc *dmc) dmc->curr_volt = target_volt; - clk_set_parent(dmc->mout_mx_mspll_ccore, dmc->mout_spll); - - dmc->bypass_rate = clk_get_rate(dmc->mout_mx_mspll_ccore); + ret = clk_set_parent(dmc->mout_mx_mspll_ccore, dmc->mout_spll); + if (ret) + return ret; clk_prepare_enable(dmc->fout_bpll); clk_prepare_enable(dmc->mout_bpll); @@ -1278,23 +1329,24 @@ static int exynos5_dmc_init_clks(struct exynos5_dmc *dmc) */ static int exynos5_performance_counters_init(struct exynos5_dmc *dmc) { - int counters_size; int ret, i; - dmc->num_counters = devfreq_event_get_edev_count(dmc->dev); + dmc->num_counters = devfreq_event_get_edev_count(dmc->dev, + "devfreq-events"); if (dmc->num_counters < 0) { dev_err(dmc->dev, "could not get devfreq-event counters\n"); return dmc->num_counters; } - counters_size = sizeof(struct devfreq_event_dev) * dmc->num_counters; - dmc->counter = devm_kzalloc(dmc->dev, counters_size, GFP_KERNEL); + dmc->counter = devm_kcalloc(dmc->dev, dmc->num_counters, + sizeof(*dmc->counter), GFP_KERNEL); if (!dmc->counter) return -ENOMEM; for (i = 0; i < dmc->num_counters; i++) { dmc->counter[i] = - devfreq_event_get_edev_by_phandle(dmc->dev, i); + devfreq_event_get_edev_by_phandle(dmc->dev, + "devfreq-events", i); if (IS_ERR_OR_NULL(dmc->counter[i])) return -EPROBE_DEFER; } @@ -1318,7 +1370,6 @@ static int exynos5_performance_counters_init(struct exynos5_dmc *dmc) /** * exynos5_dmc_set_pause_on_switching() - Controls a pause feature in DMC * @dmc: device which is used for changing this feature - * @set: a boolean state passing enable/disable request * * There is a need of pausing DREX DMC when divider or MUX in clock tree * changes its configuration. In such situation access to the memory is blocked @@ -1346,15 +1397,13 @@ static irqreturn_t dmc_irq_thread(int irq, void *priv) struct exynos5_dmc *dmc = priv; mutex_lock(&dmc->df->lock); - exynos5_dmc_perf_events_check(dmc); - res = update_devfreq(dmc->df); + mutex_unlock(&dmc->df->lock); + if (res) dev_warn(dmc->dev, "devfreq failed with %d\n", res); - mutex_unlock(&dmc->df->lock); - return IRQ_HANDLED; } @@ -1394,7 +1443,7 @@ static int exynos5_dmc_probe(struct platform_device *pdev) return PTR_ERR(dmc->base_drexi1); dmc->clk_regmap = syscon_regmap_lookup_by_phandle(np, - "samsung,syscon-clk"); + "samsung,syscon-clk"); if (IS_ERR(dmc->clk_regmap)) return PTR_ERR(dmc->clk_regmap); @@ -1429,7 +1478,7 @@ static int exynos5_dmc_probe(struct platform_device *pdev) /* There is two modes in which the driver works: polling or IRQ */ irq[0] = platform_get_irq_byname(pdev, "drex_0"); irq[1] = platform_get_irq_byname(pdev, "drex_1"); - if (irq[0] > 0 && irq[1] > 0) { + if (irq[0] > 0 && irq[1] > 0 && irqmode) { ret = devm_request_threaded_irq(dev, irq[0], NULL, dmc_irq_thread, IRQF_ONESHOT, dev_name(dev), dmc); @@ -1467,13 +1516,12 @@ static int exynos5_dmc_probe(struct platform_device *pdev) * Setup default thresholds for the devfreq governor. * The values are chosen based on experiments. */ - dmc->gov_data.upthreshold = 30; + dmc->gov_data.upthreshold = 10; dmc->gov_data.downdifferential = 5; - exynos5_dmc_df_profile.polling_ms = 500; + exynos5_dmc_df_profile.polling_ms = 100; } - dmc->df = devm_devfreq_add_device(dev, &exynos5_dmc_df_profile, DEVFREQ_GOV_SIMPLE_ONDEMAND, &dmc->gov_data); @@ -1486,7 +1534,7 @@ static int exynos5_dmc_probe(struct platform_device *pdev) if (dmc->in_irq_mode) exynos5_dmc_start_perf_events(dmc, PERF_COUNTER_START_VALUE); - dev_info(dev, "DMC initialized\n"); + dev_info(dev, "DMC initialized, in irq mode: %d\n", dmc->in_irq_mode); return 0; @@ -1522,8 +1570,6 @@ static int exynos5_dmc_remove(struct platform_device *pdev) clk_disable_unprepare(dmc->mout_bpll); clk_disable_unprepare(dmc->fout_bpll); - dev_pm_opp_remove_table(dmc->dev); - return 0; } diff --git a/drivers/memory/stm32-fmc2-ebi.c b/drivers/memory/stm32-fmc2-ebi.c new file mode 100644 index 000000000000..ffec26a99313 --- /dev/null +++ b/drivers/memory/stm32-fmc2-ebi.c @@ -0,0 +1,1210 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) STMicroelectronics 2020 + */ + +#include <linux/bitfield.h> +#include <linux/clk.h> +#include <linux/mfd/syscon.h> +#include <linux/module.h> +#include <linux/of_platform.h> +#include <linux/pinctrl/consumer.h> +#include <linux/regmap.h> +#include <linux/reset.h> + +/* FMC2 Controller Registers */ +#define FMC2_BCR1 0x0 +#define FMC2_BTR1 0x4 +#define FMC2_BCR(x) ((x) * 0x8 + FMC2_BCR1) +#define FMC2_BTR(x) ((x) * 0x8 + FMC2_BTR1) +#define FMC2_PCSCNTR 0x20 +#define FMC2_BWTR1 0x104 +#define FMC2_BWTR(x) ((x) * 0x8 + FMC2_BWTR1) + +/* Register: FMC2_BCR1 */ +#define FMC2_BCR1_CCLKEN BIT(20) +#define FMC2_BCR1_FMC2EN BIT(31) + +/* Register: FMC2_BCRx */ +#define FMC2_BCR_MBKEN BIT(0) +#define FMC2_BCR_MUXEN BIT(1) +#define FMC2_BCR_MTYP GENMASK(3, 2) +#define FMC2_BCR_MWID GENMASK(5, 4) +#define FMC2_BCR_FACCEN BIT(6) +#define FMC2_BCR_BURSTEN BIT(8) +#define FMC2_BCR_WAITPOL BIT(9) +#define FMC2_BCR_WAITCFG BIT(11) +#define FMC2_BCR_WREN BIT(12) +#define FMC2_BCR_WAITEN BIT(13) +#define FMC2_BCR_EXTMOD BIT(14) +#define FMC2_BCR_ASYNCWAIT BIT(15) +#define FMC2_BCR_CPSIZE GENMASK(18, 16) +#define FMC2_BCR_CBURSTRW BIT(19) +#define FMC2_BCR_NBLSET GENMASK(23, 22) + +/* Register: FMC2_BTRx/FMC2_BWTRx */ +#define FMC2_BXTR_ADDSET GENMASK(3, 0) +#define FMC2_BXTR_ADDHLD GENMASK(7, 4) +#define FMC2_BXTR_DATAST GENMASK(15, 8) +#define FMC2_BXTR_BUSTURN GENMASK(19, 16) +#define FMC2_BTR_CLKDIV GENMASK(23, 20) +#define FMC2_BTR_DATLAT GENMASK(27, 24) +#define FMC2_BXTR_ACCMOD GENMASK(29, 28) +#define FMC2_BXTR_DATAHLD GENMASK(31, 30) + +/* Register: FMC2_PCSCNTR */ +#define FMC2_PCSCNTR_CSCOUNT GENMASK(15, 0) +#define FMC2_PCSCNTR_CNTBEN(x) BIT((x) + 16) + +#define FMC2_MAX_EBI_CE 4 +#define FMC2_MAX_BANKS 5 + +#define FMC2_BCR_CPSIZE_0 0x0 +#define FMC2_BCR_CPSIZE_128 0x1 +#define FMC2_BCR_CPSIZE_256 0x2 +#define FMC2_BCR_CPSIZE_512 0x3 +#define FMC2_BCR_CPSIZE_1024 0x4 + +#define FMC2_BCR_MWID_8 0x0 +#define FMC2_BCR_MWID_16 0x1 + +#define FMC2_BCR_MTYP_SRAM 0x0 +#define FMC2_BCR_MTYP_PSRAM 0x1 +#define FMC2_BCR_MTYP_NOR 0x2 + +#define FMC2_BXTR_EXTMOD_A 0x0 +#define FMC2_BXTR_EXTMOD_B 0x1 +#define FMC2_BXTR_EXTMOD_C 0x2 +#define FMC2_BXTR_EXTMOD_D 0x3 + +#define FMC2_BCR_NBLSET_MAX 0x3 +#define FMC2_BXTR_ADDSET_MAX 0xf +#define FMC2_BXTR_ADDHLD_MAX 0xf +#define FMC2_BXTR_DATAST_MAX 0xff +#define FMC2_BXTR_BUSTURN_MAX 0xf +#define FMC2_BXTR_DATAHLD_MAX 0x3 +#define FMC2_BTR_CLKDIV_MAX 0xf +#define FMC2_BTR_DATLAT_MAX 0xf +#define FMC2_PCSCNTR_CSCOUNT_MAX 0xff + +enum stm32_fmc2_ebi_bank { + FMC2_EBI1 = 0, + FMC2_EBI2, + FMC2_EBI3, + FMC2_EBI4, + FMC2_NAND +}; + +enum stm32_fmc2_ebi_register_type { + FMC2_REG_BCR = 1, + FMC2_REG_BTR, + FMC2_REG_BWTR, + FMC2_REG_PCSCNTR +}; + +enum stm32_fmc2_ebi_transaction_type { + FMC2_ASYNC_MODE_1_SRAM = 0, + FMC2_ASYNC_MODE_1_PSRAM, + FMC2_ASYNC_MODE_A_SRAM, + FMC2_ASYNC_MODE_A_PSRAM, + FMC2_ASYNC_MODE_2_NOR, + FMC2_ASYNC_MODE_B_NOR, + FMC2_ASYNC_MODE_C_NOR, + FMC2_ASYNC_MODE_D_NOR, + FMC2_SYNC_READ_SYNC_WRITE_PSRAM, + FMC2_SYNC_READ_ASYNC_WRITE_PSRAM, + FMC2_SYNC_READ_SYNC_WRITE_NOR, + FMC2_SYNC_READ_ASYNC_WRITE_NOR +}; + +enum stm32_fmc2_ebi_buswidth { + FMC2_BUSWIDTH_8 = 8, + FMC2_BUSWIDTH_16 = 16 +}; + +enum stm32_fmc2_ebi_cpsize { + FMC2_CPSIZE_0 = 0, + FMC2_CPSIZE_128 = 128, + FMC2_CPSIZE_256 = 256, + FMC2_CPSIZE_512 = 512, + FMC2_CPSIZE_1024 = 1024 +}; + +struct stm32_fmc2_ebi { + struct device *dev; + struct clk *clk; + struct regmap *regmap; + u8 bank_assigned; + + u32 bcr[FMC2_MAX_EBI_CE]; + u32 btr[FMC2_MAX_EBI_CE]; + u32 bwtr[FMC2_MAX_EBI_CE]; + u32 pcscntr; +}; + +/* + * struct stm32_fmc2_prop - STM32 FMC2 EBI property + * @name: the device tree binding name of the property + * @bprop: indicate that it is a boolean property + * @mprop: indicate that it is a mandatory property + * @reg_type: the register that have to be modified + * @reg_mask: the bit that have to be modified in the selected register + * in case of it is a boolean property + * @reset_val: the default value that have to be set in case the property + * has not been defined in the device tree + * @check: this callback ckecks that the property is compliant with the + * transaction type selected + * @calculate: this callback is called to calculate for exemple a timing + * set in nanoseconds in the device tree in clock cycles or in + * clock period + * @set: this callback applies the values in the registers + */ +struct stm32_fmc2_prop { + const char *name; + bool bprop; + bool mprop; + int reg_type; + u32 reg_mask; + u32 reset_val; + int (*check)(struct stm32_fmc2_ebi *ebi, + const struct stm32_fmc2_prop *prop, int cs); + u32 (*calculate)(struct stm32_fmc2_ebi *ebi, int cs, u32 setup); + int (*set)(struct stm32_fmc2_ebi *ebi, + const struct stm32_fmc2_prop *prop, + int cs, u32 setup); +}; + +static int stm32_fmc2_ebi_check_mux(struct stm32_fmc2_ebi *ebi, + const struct stm32_fmc2_prop *prop, + int cs) +{ + u32 bcr; + + regmap_read(ebi->regmap, FMC2_BCR(cs), &bcr); + + if (bcr & FMC2_BCR_MTYP) + return 0; + + return -EINVAL; +} + +static int stm32_fmc2_ebi_check_waitcfg(struct stm32_fmc2_ebi *ebi, + const struct stm32_fmc2_prop *prop, + int cs) +{ + u32 bcr, val = FIELD_PREP(FMC2_BCR_MTYP, FMC2_BCR_MTYP_NOR); + + regmap_read(ebi->regmap, FMC2_BCR(cs), &bcr); + + if ((bcr & FMC2_BCR_MTYP) == val && bcr & FMC2_BCR_BURSTEN) + return 0; + + return -EINVAL; +} + +static int stm32_fmc2_ebi_check_sync_trans(struct stm32_fmc2_ebi *ebi, + const struct stm32_fmc2_prop *prop, + int cs) +{ + u32 bcr; + + regmap_read(ebi->regmap, FMC2_BCR(cs), &bcr); + + if (bcr & FMC2_BCR_BURSTEN) + return 0; + + return -EINVAL; +} + +static int stm32_fmc2_ebi_check_async_trans(struct stm32_fmc2_ebi *ebi, + const struct stm32_fmc2_prop *prop, + int cs) +{ + u32 bcr; + + regmap_read(ebi->regmap, FMC2_BCR(cs), &bcr); + + if (!(bcr & FMC2_BCR_BURSTEN) || !(bcr & FMC2_BCR_CBURSTRW)) + return 0; + + return -EINVAL; +} + +static int stm32_fmc2_ebi_check_cpsize(struct stm32_fmc2_ebi *ebi, + const struct stm32_fmc2_prop *prop, + int cs) +{ + u32 bcr, val = FIELD_PREP(FMC2_BCR_MTYP, FMC2_BCR_MTYP_PSRAM); + + regmap_read(ebi->regmap, FMC2_BCR(cs), &bcr); + + if ((bcr & FMC2_BCR_MTYP) == val && bcr & FMC2_BCR_BURSTEN) + return 0; + + return -EINVAL; +} + +static int stm32_fmc2_ebi_check_address_hold(struct stm32_fmc2_ebi *ebi, + const struct stm32_fmc2_prop *prop, + int cs) +{ + u32 bcr, bxtr, val = FIELD_PREP(FMC2_BXTR_ACCMOD, FMC2_BXTR_EXTMOD_D); + + regmap_read(ebi->regmap, FMC2_BCR(cs), &bcr); + if (prop->reg_type == FMC2_REG_BWTR) + regmap_read(ebi->regmap, FMC2_BWTR(cs), &bxtr); + else + regmap_read(ebi->regmap, FMC2_BTR(cs), &bxtr); + + if ((!(bcr & FMC2_BCR_BURSTEN) || !(bcr & FMC2_BCR_CBURSTRW)) && + ((bxtr & FMC2_BXTR_ACCMOD) == val || bcr & FMC2_BCR_MUXEN)) + return 0; + + return -EINVAL; +} + +static int stm32_fmc2_ebi_check_clk_period(struct stm32_fmc2_ebi *ebi, + const struct stm32_fmc2_prop *prop, + int cs) +{ + u32 bcr, bcr1; + + regmap_read(ebi->regmap, FMC2_BCR(cs), &bcr); + if (cs) + regmap_read(ebi->regmap, FMC2_BCR1, &bcr1); + else + bcr1 = bcr; + + if (bcr & FMC2_BCR_BURSTEN && (!cs || !(bcr1 & FMC2_BCR1_CCLKEN))) + return 0; + + return -EINVAL; +} + +static int stm32_fmc2_ebi_check_cclk(struct stm32_fmc2_ebi *ebi, + const struct stm32_fmc2_prop *prop, + int cs) +{ + if (cs) + return -EINVAL; + + return stm32_fmc2_ebi_check_sync_trans(ebi, prop, cs); +} + +static u32 stm32_fmc2_ebi_ns_to_clock_cycles(struct stm32_fmc2_ebi *ebi, + int cs, u32 setup) +{ + unsigned long hclk = clk_get_rate(ebi->clk); + unsigned long hclkp = NSEC_PER_SEC / (hclk / 1000); + + return DIV_ROUND_UP(setup * 1000, hclkp); +} + +static u32 stm32_fmc2_ebi_ns_to_clk_period(struct stm32_fmc2_ebi *ebi, + int cs, u32 setup) +{ + u32 nb_clk_cycles = stm32_fmc2_ebi_ns_to_clock_cycles(ebi, cs, setup); + u32 bcr, btr, clk_period; + + regmap_read(ebi->regmap, FMC2_BCR1, &bcr); + if (bcr & FMC2_BCR1_CCLKEN || !cs) + regmap_read(ebi->regmap, FMC2_BTR1, &btr); + else + regmap_read(ebi->regmap, FMC2_BTR(cs), &btr); + + clk_period = FIELD_GET(FMC2_BTR_CLKDIV, btr) + 1; + + return DIV_ROUND_UP(nb_clk_cycles, clk_period); +} + +static int stm32_fmc2_ebi_get_reg(int reg_type, int cs, u32 *reg) +{ + switch (reg_type) { + case FMC2_REG_BCR: + *reg = FMC2_BCR(cs); + break; + case FMC2_REG_BTR: + *reg = FMC2_BTR(cs); + break; + case FMC2_REG_BWTR: + *reg = FMC2_BWTR(cs); + break; + case FMC2_REG_PCSCNTR: + *reg = FMC2_PCSCNTR; + break; + default: + return -EINVAL; + } + + return 0; +} + +static int stm32_fmc2_ebi_set_bit_field(struct stm32_fmc2_ebi *ebi, + const struct stm32_fmc2_prop *prop, + int cs, u32 setup) +{ + u32 reg; + int ret; + + ret = stm32_fmc2_ebi_get_reg(prop->reg_type, cs, ®); + if (ret) + return ret; + + regmap_update_bits(ebi->regmap, reg, prop->reg_mask, + setup ? prop->reg_mask : 0); + + return 0; +} + +static int stm32_fmc2_ebi_set_trans_type(struct stm32_fmc2_ebi *ebi, + const struct stm32_fmc2_prop *prop, + int cs, u32 setup) +{ + u32 bcr_mask, bcr = FMC2_BCR_WREN; + u32 btr_mask, btr = 0; + u32 bwtr_mask, bwtr = 0; + + bwtr_mask = FMC2_BXTR_ACCMOD; + btr_mask = FMC2_BXTR_ACCMOD; + bcr_mask = FMC2_BCR_MUXEN | FMC2_BCR_MTYP | FMC2_BCR_FACCEN | + FMC2_BCR_WREN | FMC2_BCR_WAITEN | FMC2_BCR_BURSTEN | + FMC2_BCR_EXTMOD | FMC2_BCR_CBURSTRW; + + switch (setup) { + case FMC2_ASYNC_MODE_1_SRAM: + bcr |= FIELD_PREP(FMC2_BCR_MTYP, FMC2_BCR_MTYP_SRAM); + /* + * MUXEN = 0, MTYP = 0, FACCEN = 0, BURSTEN = 0, WAITEN = 0, + * WREN = 1, EXTMOD = 0, CBURSTRW = 0, ACCMOD = 0 + */ + break; + case FMC2_ASYNC_MODE_1_PSRAM: + /* + * MUXEN = 0, MTYP = 1, FACCEN = 0, BURSTEN = 0, WAITEN = 0, + * WREN = 1, EXTMOD = 0, CBURSTRW = 0, ACCMOD = 0 + */ + bcr |= FIELD_PREP(FMC2_BCR_MTYP, FMC2_BCR_MTYP_PSRAM); + break; + case FMC2_ASYNC_MODE_A_SRAM: + /* + * MUXEN = 0, MTYP = 0, FACCEN = 0, BURSTEN = 0, WAITEN = 0, + * WREN = 1, EXTMOD = 1, CBURSTRW = 0, ACCMOD = 0 + */ + bcr |= FIELD_PREP(FMC2_BCR_MTYP, FMC2_BCR_MTYP_SRAM); + bcr |= FMC2_BCR_EXTMOD; + btr |= FIELD_PREP(FMC2_BXTR_ACCMOD, FMC2_BXTR_EXTMOD_A); + bwtr |= FIELD_PREP(FMC2_BXTR_ACCMOD, FMC2_BXTR_EXTMOD_A); + break; + case FMC2_ASYNC_MODE_A_PSRAM: + /* + * MUXEN = 0, MTYP = 1, FACCEN = 0, BURSTEN = 0, WAITEN = 0, + * WREN = 1, EXTMOD = 1, CBURSTRW = 0, ACCMOD = 0 + */ + bcr |= FIELD_PREP(FMC2_BCR_MTYP, FMC2_BCR_MTYP_PSRAM); + bcr |= FMC2_BCR_EXTMOD; + btr |= FIELD_PREP(FMC2_BXTR_ACCMOD, FMC2_BXTR_EXTMOD_A); + bwtr |= FIELD_PREP(FMC2_BXTR_ACCMOD, FMC2_BXTR_EXTMOD_A); + break; + case FMC2_ASYNC_MODE_2_NOR: + /* + * MUXEN = 0, MTYP = 2, FACCEN = 1, BURSTEN = 0, WAITEN = 0, + * WREN = 1, EXTMOD = 0, CBURSTRW = 0, ACCMOD = 0 + */ + bcr |= FIELD_PREP(FMC2_BCR_MTYP, FMC2_BCR_MTYP_NOR); + bcr |= FMC2_BCR_FACCEN; + break; + case FMC2_ASYNC_MODE_B_NOR: + /* + * MUXEN = 0, MTYP = 2, FACCEN = 1, BURSTEN = 0, WAITEN = 0, + * WREN = 1, EXTMOD = 1, CBURSTRW = 0, ACCMOD = 1 + */ + bcr |= FIELD_PREP(FMC2_BCR_MTYP, FMC2_BCR_MTYP_NOR); + bcr |= FMC2_BCR_FACCEN | FMC2_BCR_EXTMOD; + btr |= FIELD_PREP(FMC2_BXTR_ACCMOD, FMC2_BXTR_EXTMOD_B); + bwtr |= FIELD_PREP(FMC2_BXTR_ACCMOD, FMC2_BXTR_EXTMOD_B); + break; + case FMC2_ASYNC_MODE_C_NOR: + /* + * MUXEN = 0, MTYP = 2, FACCEN = 1, BURSTEN = 0, WAITEN = 0, + * WREN = 1, EXTMOD = 1, CBURSTRW = 0, ACCMOD = 2 + */ + bcr |= FIELD_PREP(FMC2_BCR_MTYP, FMC2_BCR_MTYP_NOR); + bcr |= FMC2_BCR_FACCEN | FMC2_BCR_EXTMOD; + btr |= FIELD_PREP(FMC2_BXTR_ACCMOD, FMC2_BXTR_EXTMOD_C); + bwtr |= FIELD_PREP(FMC2_BXTR_ACCMOD, FMC2_BXTR_EXTMOD_C); + break; + case FMC2_ASYNC_MODE_D_NOR: + /* + * MUXEN = 0, MTYP = 2, FACCEN = 1, BURSTEN = 0, WAITEN = 0, + * WREN = 1, EXTMOD = 1, CBURSTRW = 0, ACCMOD = 3 + */ + bcr |= FIELD_PREP(FMC2_BCR_MTYP, FMC2_BCR_MTYP_NOR); + bcr |= FMC2_BCR_FACCEN | FMC2_BCR_EXTMOD; + btr |= FIELD_PREP(FMC2_BXTR_ACCMOD, FMC2_BXTR_EXTMOD_D); + bwtr |= FIELD_PREP(FMC2_BXTR_ACCMOD, FMC2_BXTR_EXTMOD_D); + break; + case FMC2_SYNC_READ_SYNC_WRITE_PSRAM: + /* + * MUXEN = 0, MTYP = 1, FACCEN = 0, BURSTEN = 1, WAITEN = 0, + * WREN = 1, EXTMOD = 0, CBURSTRW = 1, ACCMOD = 0 + */ + bcr |= FIELD_PREP(FMC2_BCR_MTYP, FMC2_BCR_MTYP_PSRAM); + bcr |= FMC2_BCR_BURSTEN | FMC2_BCR_CBURSTRW; + break; + case FMC2_SYNC_READ_ASYNC_WRITE_PSRAM: + /* + * MUXEN = 0, MTYP = 1, FACCEN = 0, BURSTEN = 1, WAITEN = 0, + * WREN = 1, EXTMOD = 0, CBURSTRW = 0, ACCMOD = 0 + */ + bcr |= FIELD_PREP(FMC2_BCR_MTYP, FMC2_BCR_MTYP_PSRAM); + bcr |= FMC2_BCR_BURSTEN; + break; + case FMC2_SYNC_READ_SYNC_WRITE_NOR: + /* + * MUXEN = 0, MTYP = 2, FACCEN = 1, BURSTEN = 1, WAITEN = 0, + * WREN = 1, EXTMOD = 0, CBURSTRW = 1, ACCMOD = 0 + */ + bcr |= FIELD_PREP(FMC2_BCR_MTYP, FMC2_BCR_MTYP_NOR); + bcr |= FMC2_BCR_FACCEN | FMC2_BCR_BURSTEN | FMC2_BCR_CBURSTRW; + break; + case FMC2_SYNC_READ_ASYNC_WRITE_NOR: + /* + * MUXEN = 0, MTYP = 2, FACCEN = 1, BURSTEN = 1, WAITEN = 0, + * WREN = 1, EXTMOD = 0, CBURSTRW = 0, ACCMOD = 0 + */ + bcr |= FIELD_PREP(FMC2_BCR_MTYP, FMC2_BCR_MTYP_NOR); + bcr |= FMC2_BCR_FACCEN | FMC2_BCR_BURSTEN; + break; + default: + /* Type of transaction not supported */ + return -EINVAL; + } + + if (bcr & FMC2_BCR_EXTMOD) + regmap_update_bits(ebi->regmap, FMC2_BWTR(cs), + bwtr_mask, bwtr); + regmap_update_bits(ebi->regmap, FMC2_BTR(cs), btr_mask, btr); + regmap_update_bits(ebi->regmap, FMC2_BCR(cs), bcr_mask, bcr); + + return 0; +} + +static int stm32_fmc2_ebi_set_buswidth(struct stm32_fmc2_ebi *ebi, + const struct stm32_fmc2_prop *prop, + int cs, u32 setup) +{ + u32 val; + + switch (setup) { + case FMC2_BUSWIDTH_8: + val = FIELD_PREP(FMC2_BCR_MWID, FMC2_BCR_MWID_8); + break; + case FMC2_BUSWIDTH_16: + val = FIELD_PREP(FMC2_BCR_MWID, FMC2_BCR_MWID_16); + break; + default: + /* Buswidth not supported */ + return -EINVAL; + } + + regmap_update_bits(ebi->regmap, FMC2_BCR(cs), FMC2_BCR_MWID, val); + + return 0; +} + +static int stm32_fmc2_ebi_set_cpsize(struct stm32_fmc2_ebi *ebi, + const struct stm32_fmc2_prop *prop, + int cs, u32 setup) +{ + u32 val; + + switch (setup) { + case FMC2_CPSIZE_0: + val = FIELD_PREP(FMC2_BCR_CPSIZE, FMC2_BCR_CPSIZE_0); + break; + case FMC2_CPSIZE_128: + val = FIELD_PREP(FMC2_BCR_CPSIZE, FMC2_BCR_CPSIZE_128); + break; + case FMC2_CPSIZE_256: + val = FIELD_PREP(FMC2_BCR_CPSIZE, FMC2_BCR_CPSIZE_256); + break; + case FMC2_CPSIZE_512: + val = FIELD_PREP(FMC2_BCR_CPSIZE, FMC2_BCR_CPSIZE_512); + break; + case FMC2_CPSIZE_1024: + val = FIELD_PREP(FMC2_BCR_CPSIZE, FMC2_BCR_CPSIZE_1024); + break; + default: + /* Cpsize not supported */ + return -EINVAL; + } + + regmap_update_bits(ebi->regmap, FMC2_BCR(cs), FMC2_BCR_CPSIZE, val); + + return 0; +} + +static int stm32_fmc2_ebi_set_bl_setup(struct stm32_fmc2_ebi *ebi, + const struct stm32_fmc2_prop *prop, + int cs, u32 setup) +{ + u32 val; + + val = min_t(u32, setup, FMC2_BCR_NBLSET_MAX); + val = FIELD_PREP(FMC2_BCR_NBLSET, val); + regmap_update_bits(ebi->regmap, FMC2_BCR(cs), FMC2_BCR_NBLSET, val); + + return 0; +} + +static int stm32_fmc2_ebi_set_address_setup(struct stm32_fmc2_ebi *ebi, + const struct stm32_fmc2_prop *prop, + int cs, u32 setup) +{ + u32 bcr, bxtr, reg; + u32 val = FIELD_PREP(FMC2_BXTR_ACCMOD, FMC2_BXTR_EXTMOD_D); + int ret; + + ret = stm32_fmc2_ebi_get_reg(prop->reg_type, cs, ®); + if (ret) + return ret; + + regmap_read(ebi->regmap, FMC2_BCR(cs), &bcr); + if (prop->reg_type == FMC2_REG_BWTR) + regmap_read(ebi->regmap, FMC2_BWTR(cs), &bxtr); + else + regmap_read(ebi->regmap, FMC2_BTR(cs), &bxtr); + + if ((bxtr & FMC2_BXTR_ACCMOD) == val || bcr & FMC2_BCR_MUXEN) + val = clamp_val(setup, 1, FMC2_BXTR_ADDSET_MAX); + else + val = min_t(u32, setup, FMC2_BXTR_ADDSET_MAX); + val = FIELD_PREP(FMC2_BXTR_ADDSET, val); + regmap_update_bits(ebi->regmap, reg, FMC2_BXTR_ADDSET, val); + + return 0; +} + +static int stm32_fmc2_ebi_set_address_hold(struct stm32_fmc2_ebi *ebi, + const struct stm32_fmc2_prop *prop, + int cs, u32 setup) +{ + u32 val, reg; + int ret; + + ret = stm32_fmc2_ebi_get_reg(prop->reg_type, cs, ®); + if (ret) + return ret; + + val = clamp_val(setup, 1, FMC2_BXTR_ADDHLD_MAX); + val = FIELD_PREP(FMC2_BXTR_ADDHLD, val); + regmap_update_bits(ebi->regmap, reg, FMC2_BXTR_ADDHLD, val); + + return 0; +} + +static int stm32_fmc2_ebi_set_data_setup(struct stm32_fmc2_ebi *ebi, + const struct stm32_fmc2_prop *prop, + int cs, u32 setup) +{ + u32 val, reg; + int ret; + + ret = stm32_fmc2_ebi_get_reg(prop->reg_type, cs, ®); + if (ret) + return ret; + + val = clamp_val(setup, 1, FMC2_BXTR_DATAST_MAX); + val = FIELD_PREP(FMC2_BXTR_DATAST, val); + regmap_update_bits(ebi->regmap, reg, FMC2_BXTR_DATAST, val); + + return 0; +} + +static int stm32_fmc2_ebi_set_bus_turnaround(struct stm32_fmc2_ebi *ebi, + const struct stm32_fmc2_prop *prop, + int cs, u32 setup) +{ + u32 val, reg; + int ret; + + ret = stm32_fmc2_ebi_get_reg(prop->reg_type, cs, ®); + if (ret) + return ret; + + val = setup ? min_t(u32, setup - 1, FMC2_BXTR_BUSTURN_MAX) : 0; + val = FIELD_PREP(FMC2_BXTR_BUSTURN, val); + regmap_update_bits(ebi->regmap, reg, FMC2_BXTR_BUSTURN, val); + + return 0; +} + +static int stm32_fmc2_ebi_set_data_hold(struct stm32_fmc2_ebi *ebi, + const struct stm32_fmc2_prop *prop, + int cs, u32 setup) +{ + u32 val, reg; + int ret; + + ret = stm32_fmc2_ebi_get_reg(prop->reg_type, cs, ®); + if (ret) + return ret; + + if (prop->reg_type == FMC2_REG_BWTR) + val = setup ? min_t(u32, setup - 1, FMC2_BXTR_DATAHLD_MAX) : 0; + else + val = min_t(u32, setup, FMC2_BXTR_DATAHLD_MAX); + val = FIELD_PREP(FMC2_BXTR_DATAHLD, val); + regmap_update_bits(ebi->regmap, reg, FMC2_BXTR_DATAHLD, val); + + return 0; +} + +static int stm32_fmc2_ebi_set_clk_period(struct stm32_fmc2_ebi *ebi, + const struct stm32_fmc2_prop *prop, + int cs, u32 setup) +{ + u32 val; + + val = setup ? clamp_val(setup - 1, 1, FMC2_BTR_CLKDIV_MAX) : 1; + val = FIELD_PREP(FMC2_BTR_CLKDIV, val); + regmap_update_bits(ebi->regmap, FMC2_BTR(cs), FMC2_BTR_CLKDIV, val); + + return 0; +} + +static int stm32_fmc2_ebi_set_data_latency(struct stm32_fmc2_ebi *ebi, + const struct stm32_fmc2_prop *prop, + int cs, u32 setup) +{ + u32 val; + + val = setup > 1 ? min_t(u32, setup - 2, FMC2_BTR_DATLAT_MAX) : 0; + val = FIELD_PREP(FMC2_BTR_DATLAT, val); + regmap_update_bits(ebi->regmap, FMC2_BTR(cs), FMC2_BTR_DATLAT, val); + + return 0; +} + +static int stm32_fmc2_ebi_set_max_low_pulse(struct stm32_fmc2_ebi *ebi, + const struct stm32_fmc2_prop *prop, + int cs, u32 setup) +{ + u32 old_val, new_val, pcscntr; + + if (setup < 1) + return 0; + + regmap_read(ebi->regmap, FMC2_PCSCNTR, &pcscntr); + + /* Enable counter for the bank */ + regmap_update_bits(ebi->regmap, FMC2_PCSCNTR, + FMC2_PCSCNTR_CNTBEN(cs), + FMC2_PCSCNTR_CNTBEN(cs)); + + new_val = min_t(u32, setup - 1, FMC2_PCSCNTR_CSCOUNT_MAX); + old_val = FIELD_GET(FMC2_PCSCNTR_CSCOUNT, pcscntr); + if (old_val && new_val > old_val) + /* Keep current counter value */ + return 0; + + new_val = FIELD_PREP(FMC2_PCSCNTR_CSCOUNT, new_val); + regmap_update_bits(ebi->regmap, FMC2_PCSCNTR, + FMC2_PCSCNTR_CSCOUNT, new_val); + + return 0; +} + +static const struct stm32_fmc2_prop stm32_fmc2_child_props[] = { + /* st,fmc2-ebi-cs-trans-type must be the first property */ + { + .name = "st,fmc2-ebi-cs-transaction-type", + .mprop = true, + .set = stm32_fmc2_ebi_set_trans_type, + }, + { + .name = "st,fmc2-ebi-cs-cclk-enable", + .bprop = true, + .reg_type = FMC2_REG_BCR, + .reg_mask = FMC2_BCR1_CCLKEN, + .check = stm32_fmc2_ebi_check_cclk, + .set = stm32_fmc2_ebi_set_bit_field, + }, + { + .name = "st,fmc2-ebi-cs-mux-enable", + .bprop = true, + .reg_type = FMC2_REG_BCR, + .reg_mask = FMC2_BCR_MUXEN, + .check = stm32_fmc2_ebi_check_mux, + .set = stm32_fmc2_ebi_set_bit_field, + }, + { + .name = "st,fmc2-ebi-cs-buswidth", + .reset_val = FMC2_BUSWIDTH_16, + .set = stm32_fmc2_ebi_set_buswidth, + }, + { + .name = "st,fmc2-ebi-cs-waitpol-high", + .bprop = true, + .reg_type = FMC2_REG_BCR, + .reg_mask = FMC2_BCR_WAITPOL, + .set = stm32_fmc2_ebi_set_bit_field, + }, + { + .name = "st,fmc2-ebi-cs-waitcfg-enable", + .bprop = true, + .reg_type = FMC2_REG_BCR, + .reg_mask = FMC2_BCR_WAITCFG, + .check = stm32_fmc2_ebi_check_waitcfg, + .set = stm32_fmc2_ebi_set_bit_field, + }, + { + .name = "st,fmc2-ebi-cs-wait-enable", + .bprop = true, + .reg_type = FMC2_REG_BCR, + .reg_mask = FMC2_BCR_WAITEN, + .check = stm32_fmc2_ebi_check_sync_trans, + .set = stm32_fmc2_ebi_set_bit_field, + }, + { + .name = "st,fmc2-ebi-cs-asyncwait-enable", + .bprop = true, + .reg_type = FMC2_REG_BCR, + .reg_mask = FMC2_BCR_ASYNCWAIT, + .check = stm32_fmc2_ebi_check_async_trans, + .set = stm32_fmc2_ebi_set_bit_field, + }, + { + .name = "st,fmc2-ebi-cs-cpsize", + .check = stm32_fmc2_ebi_check_cpsize, + .set = stm32_fmc2_ebi_set_cpsize, + }, + { + .name = "st,fmc2-ebi-cs-byte-lane-setup-ns", + .calculate = stm32_fmc2_ebi_ns_to_clock_cycles, + .set = stm32_fmc2_ebi_set_bl_setup, + }, + { + .name = "st,fmc2-ebi-cs-address-setup-ns", + .reg_type = FMC2_REG_BTR, + .reset_val = FMC2_BXTR_ADDSET_MAX, + .check = stm32_fmc2_ebi_check_async_trans, + .calculate = stm32_fmc2_ebi_ns_to_clock_cycles, + .set = stm32_fmc2_ebi_set_address_setup, + }, + { + .name = "st,fmc2-ebi-cs-address-hold-ns", + .reg_type = FMC2_REG_BTR, + .reset_val = FMC2_BXTR_ADDHLD_MAX, + .check = stm32_fmc2_ebi_check_address_hold, + .calculate = stm32_fmc2_ebi_ns_to_clock_cycles, + .set = stm32_fmc2_ebi_set_address_hold, + }, + { + .name = "st,fmc2-ebi-cs-data-setup-ns", + .reg_type = FMC2_REG_BTR, + .reset_val = FMC2_BXTR_DATAST_MAX, + .check = stm32_fmc2_ebi_check_async_trans, + .calculate = stm32_fmc2_ebi_ns_to_clock_cycles, + .set = stm32_fmc2_ebi_set_data_setup, + }, + { + .name = "st,fmc2-ebi-cs-bus-turnaround-ns", + .reg_type = FMC2_REG_BTR, + .reset_val = FMC2_BXTR_BUSTURN_MAX + 1, + .calculate = stm32_fmc2_ebi_ns_to_clock_cycles, + .set = stm32_fmc2_ebi_set_bus_turnaround, + }, + { + .name = "st,fmc2-ebi-cs-data-hold-ns", + .reg_type = FMC2_REG_BTR, + .check = stm32_fmc2_ebi_check_async_trans, + .calculate = stm32_fmc2_ebi_ns_to_clock_cycles, + .set = stm32_fmc2_ebi_set_data_hold, + }, + { + .name = "st,fmc2-ebi-cs-clk-period-ns", + .reset_val = FMC2_BTR_CLKDIV_MAX + 1, + .check = stm32_fmc2_ebi_check_clk_period, + .calculate = stm32_fmc2_ebi_ns_to_clock_cycles, + .set = stm32_fmc2_ebi_set_clk_period, + }, + { + .name = "st,fmc2-ebi-cs-data-latency-ns", + .check = stm32_fmc2_ebi_check_sync_trans, + .calculate = stm32_fmc2_ebi_ns_to_clk_period, + .set = stm32_fmc2_ebi_set_data_latency, + }, + { + .name = "st,fmc2-ebi-cs-write-address-setup-ns", + .reg_type = FMC2_REG_BWTR, + .reset_val = FMC2_BXTR_ADDSET_MAX, + .check = stm32_fmc2_ebi_check_async_trans, + .calculate = stm32_fmc2_ebi_ns_to_clock_cycles, + .set = stm32_fmc2_ebi_set_address_setup, + }, + { + .name = "st,fmc2-ebi-cs-write-address-hold-ns", + .reg_type = FMC2_REG_BWTR, + .reset_val = FMC2_BXTR_ADDHLD_MAX, + .check = stm32_fmc2_ebi_check_address_hold, + .calculate = stm32_fmc2_ebi_ns_to_clock_cycles, + .set = stm32_fmc2_ebi_set_address_hold, + }, + { + .name = "st,fmc2-ebi-cs-write-data-setup-ns", + .reg_type = FMC2_REG_BWTR, + .reset_val = FMC2_BXTR_DATAST_MAX, + .check = stm32_fmc2_ebi_check_async_trans, + .calculate = stm32_fmc2_ebi_ns_to_clock_cycles, + .set = stm32_fmc2_ebi_set_data_setup, + }, + { + .name = "st,fmc2-ebi-cs-write-bus-turnaround-ns", + .reg_type = FMC2_REG_BWTR, + .reset_val = FMC2_BXTR_BUSTURN_MAX + 1, + .calculate = stm32_fmc2_ebi_ns_to_clock_cycles, + .set = stm32_fmc2_ebi_set_bus_turnaround, + }, + { + .name = "st,fmc2-ebi-cs-write-data-hold-ns", + .reg_type = FMC2_REG_BWTR, + .check = stm32_fmc2_ebi_check_async_trans, + .calculate = stm32_fmc2_ebi_ns_to_clock_cycles, + .set = stm32_fmc2_ebi_set_data_hold, + }, + { + .name = "st,fmc2-ebi-cs-max-low-pulse-ns", + .calculate = stm32_fmc2_ebi_ns_to_clock_cycles, + .set = stm32_fmc2_ebi_set_max_low_pulse, + }, +}; + +static int stm32_fmc2_ebi_parse_prop(struct stm32_fmc2_ebi *ebi, + struct device_node *dev_node, + const struct stm32_fmc2_prop *prop, + int cs) +{ + struct device *dev = ebi->dev; + u32 setup = 0; + + if (!prop->set) { + dev_err(dev, "property %s is not well defined\n", prop->name); + return -EINVAL; + } + + if (prop->check && prop->check(ebi, prop, cs)) + /* Skeep this property */ + return 0; + + if (prop->bprop) { + bool bprop; + + bprop = of_property_read_bool(dev_node, prop->name); + if (prop->mprop && !bprop) { + dev_err(dev, "mandatory property %s not defined in the device tree\n", + prop->name); + return -EINVAL; + } + + if (bprop) + setup = 1; + } else { + u32 val; + int ret; + + ret = of_property_read_u32(dev_node, prop->name, &val); + if (prop->mprop && ret) { + dev_err(dev, "mandatory property %s not defined in the device tree\n", + prop->name); + return ret; + } + + if (ret) + setup = prop->reset_val; + else if (prop->calculate) + setup = prop->calculate(ebi, cs, val); + else + setup = val; + } + + return prop->set(ebi, prop, cs, setup); +} + +static void stm32_fmc2_ebi_enable_bank(struct stm32_fmc2_ebi *ebi, int cs) +{ + regmap_update_bits(ebi->regmap, FMC2_BCR(cs), + FMC2_BCR_MBKEN, FMC2_BCR_MBKEN); +} + +static void stm32_fmc2_ebi_disable_bank(struct stm32_fmc2_ebi *ebi, int cs) +{ + regmap_update_bits(ebi->regmap, FMC2_BCR(cs), FMC2_BCR_MBKEN, 0); +} + +static void stm32_fmc2_ebi_save_setup(struct stm32_fmc2_ebi *ebi) +{ + unsigned int cs; + + for (cs = 0; cs < FMC2_MAX_EBI_CE; cs++) { + regmap_read(ebi->regmap, FMC2_BCR(cs), &ebi->bcr[cs]); + regmap_read(ebi->regmap, FMC2_BTR(cs), &ebi->btr[cs]); + regmap_read(ebi->regmap, FMC2_BWTR(cs), &ebi->bwtr[cs]); + } + + regmap_read(ebi->regmap, FMC2_PCSCNTR, &ebi->pcscntr); +} + +static void stm32_fmc2_ebi_set_setup(struct stm32_fmc2_ebi *ebi) +{ + unsigned int cs; + + for (cs = 0; cs < FMC2_MAX_EBI_CE; cs++) { + regmap_write(ebi->regmap, FMC2_BCR(cs), ebi->bcr[cs]); + regmap_write(ebi->regmap, FMC2_BTR(cs), ebi->btr[cs]); + regmap_write(ebi->regmap, FMC2_BWTR(cs), ebi->bwtr[cs]); + } + + regmap_write(ebi->regmap, FMC2_PCSCNTR, ebi->pcscntr); +} + +static void stm32_fmc2_ebi_disable_banks(struct stm32_fmc2_ebi *ebi) +{ + unsigned int cs; + + for (cs = 0; cs < FMC2_MAX_EBI_CE; cs++) { + if (!(ebi->bank_assigned & BIT(cs))) + continue; + + stm32_fmc2_ebi_disable_bank(ebi, cs); + } +} + +/* NWAIT signal can not be connected to EBI controller and NAND controller */ +static bool stm32_fmc2_ebi_nwait_used_by_ctrls(struct stm32_fmc2_ebi *ebi) +{ + unsigned int cs; + u32 bcr; + + for (cs = 0; cs < FMC2_MAX_EBI_CE; cs++) { + if (!(ebi->bank_assigned & BIT(cs))) + continue; + + regmap_read(ebi->regmap, FMC2_BCR(cs), &bcr); + if ((bcr & FMC2_BCR_WAITEN || bcr & FMC2_BCR_ASYNCWAIT) && + ebi->bank_assigned & BIT(FMC2_NAND)) + return true; + } + + return false; +} + +static void stm32_fmc2_ebi_enable(struct stm32_fmc2_ebi *ebi) +{ + regmap_update_bits(ebi->regmap, FMC2_BCR1, + FMC2_BCR1_FMC2EN, FMC2_BCR1_FMC2EN); +} + +static void stm32_fmc2_ebi_disable(struct stm32_fmc2_ebi *ebi) +{ + regmap_update_bits(ebi->regmap, FMC2_BCR1, FMC2_BCR1_FMC2EN, 0); +} + +static int stm32_fmc2_ebi_setup_cs(struct stm32_fmc2_ebi *ebi, + struct device_node *dev_node, + u32 cs) +{ + unsigned int i; + int ret; + + stm32_fmc2_ebi_disable_bank(ebi, cs); + + for (i = 0; i < ARRAY_SIZE(stm32_fmc2_child_props); i++) { + const struct stm32_fmc2_prop *p = &stm32_fmc2_child_props[i]; + + ret = stm32_fmc2_ebi_parse_prop(ebi, dev_node, p, cs); + if (ret) { + dev_err(ebi->dev, "property %s could not be set: %d\n", + p->name, ret); + return ret; + } + } + + stm32_fmc2_ebi_enable_bank(ebi, cs); + + return 0; +} + +static int stm32_fmc2_ebi_parse_dt(struct stm32_fmc2_ebi *ebi) +{ + struct device *dev = ebi->dev; + struct device_node *child; + bool child_found = false; + u32 bank; + int ret; + + for_each_available_child_of_node(dev->of_node, child) { + ret = of_property_read_u32(child, "reg", &bank); + if (ret) { + dev_err(dev, "could not retrieve reg property: %d\n", + ret); + of_node_put(child); + return ret; + } + + if (bank >= FMC2_MAX_BANKS) { + dev_err(dev, "invalid reg value: %d\n", bank); + of_node_put(child); + return -EINVAL; + } + + if (ebi->bank_assigned & BIT(bank)) { + dev_err(dev, "bank already assigned: %d\n", bank); + of_node_put(child); + return -EINVAL; + } + + if (bank < FMC2_MAX_EBI_CE) { + ret = stm32_fmc2_ebi_setup_cs(ebi, child, bank); + if (ret) { + dev_err(dev, "setup chip select %d failed: %d\n", + bank, ret); + of_node_put(child); + return ret; + } + } + + ebi->bank_assigned |= BIT(bank); + child_found = true; + } + + if (!child_found) { + dev_warn(dev, "no subnodes found, disable the driver.\n"); + return -ENODEV; + } + + if (stm32_fmc2_ebi_nwait_used_by_ctrls(ebi)) { + dev_err(dev, "NWAIT signal connected to EBI and NAND controllers\n"); + return -EINVAL; + } + + stm32_fmc2_ebi_enable(ebi); + + return of_platform_populate(dev->of_node, NULL, NULL, dev); +} + +static int stm32_fmc2_ebi_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct stm32_fmc2_ebi *ebi; + struct reset_control *rstc; + int ret; + + ebi = devm_kzalloc(&pdev->dev, sizeof(*ebi), GFP_KERNEL); + if (!ebi) + return -ENOMEM; + + ebi->dev = dev; + + ebi->regmap = device_node_to_regmap(dev->of_node); + if (IS_ERR(ebi->regmap)) + return PTR_ERR(ebi->regmap); + + ebi->clk = devm_clk_get(dev, NULL); + if (IS_ERR(ebi->clk)) + return PTR_ERR(ebi->clk); + + rstc = devm_reset_control_get(dev, NULL); + if (PTR_ERR(rstc) == -EPROBE_DEFER) + return -EPROBE_DEFER; + + ret = clk_prepare_enable(ebi->clk); + if (ret) + return ret; + + if (!IS_ERR(rstc)) { + reset_control_assert(rstc); + reset_control_deassert(rstc); + } + + ret = stm32_fmc2_ebi_parse_dt(ebi); + if (ret) + goto err_release; + + stm32_fmc2_ebi_save_setup(ebi); + platform_set_drvdata(pdev, ebi); + + return 0; + +err_release: + stm32_fmc2_ebi_disable_banks(ebi); + stm32_fmc2_ebi_disable(ebi); + clk_disable_unprepare(ebi->clk); + + return ret; +} + +static int stm32_fmc2_ebi_remove(struct platform_device *pdev) +{ + struct stm32_fmc2_ebi *ebi = platform_get_drvdata(pdev); + + of_platform_depopulate(&pdev->dev); + stm32_fmc2_ebi_disable_banks(ebi); + stm32_fmc2_ebi_disable(ebi); + clk_disable_unprepare(ebi->clk); + + return 0; +} + +static int __maybe_unused stm32_fmc2_ebi_suspend(struct device *dev) +{ + struct stm32_fmc2_ebi *ebi = dev_get_drvdata(dev); + + stm32_fmc2_ebi_disable(ebi); + clk_disable_unprepare(ebi->clk); + pinctrl_pm_select_sleep_state(dev); + + return 0; +} + +static int __maybe_unused stm32_fmc2_ebi_resume(struct device *dev) +{ + struct stm32_fmc2_ebi *ebi = dev_get_drvdata(dev); + int ret; + + pinctrl_pm_select_default_state(dev); + + ret = clk_prepare_enable(ebi->clk); + if (ret) + return ret; + + stm32_fmc2_ebi_set_setup(ebi); + stm32_fmc2_ebi_enable(ebi); + + return 0; +} + +static SIMPLE_DEV_PM_OPS(stm32_fmc2_ebi_pm_ops, stm32_fmc2_ebi_suspend, + stm32_fmc2_ebi_resume); + +static const struct of_device_id stm32_fmc2_ebi_match[] = { + {.compatible = "st,stm32mp1-fmc2-ebi"}, + {} +}; +MODULE_DEVICE_TABLE(of, stm32_fmc2_ebi_match); + +static struct platform_driver stm32_fmc2_ebi_driver = { + .probe = stm32_fmc2_ebi_probe, + .remove = stm32_fmc2_ebi_remove, + .driver = { + .name = "stm32_fmc2_ebi", + .of_match_table = stm32_fmc2_ebi_match, + .pm = &stm32_fmc2_ebi_pm_ops, + }, +}; +module_platform_driver(stm32_fmc2_ebi_driver); + +MODULE_ALIAS("platform:stm32_fmc2_ebi"); +MODULE_AUTHOR("Christophe Kerello <christophe.kerello@st.com>"); +MODULE_DESCRIPTION("STMicroelectronics STM32 FMC2 ebi driver"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/memory/tegra/Kconfig b/drivers/memory/tegra/Kconfig index fbfbaada61a2..3fe83d7c2bf8 100644 --- a/drivers/memory/tegra/Kconfig +++ b/drivers/memory/tegra/Kconfig @@ -2,15 +2,21 @@ config TEGRA_MC bool "NVIDIA Tegra Memory Controller support" default y - depends on ARCH_TEGRA + depends on ARCH_TEGRA || (COMPILE_TEST && COMMON_CLK) + select INTERCONNECT help This driver supports the Memory Controller (MC) hardware found on NVIDIA Tegra SoCs. +if TEGRA_MC + config TEGRA20_EMC - bool "NVIDIA Tegra20 External Memory Controller driver" + tristate "NVIDIA Tegra20 External Memory Controller driver" default y - depends on ARCH_TEGRA_2x_SOC + depends on ARCH_TEGRA_2x_SOC || COMPILE_TEST + select DEVFREQ_GOV_SIMPLE_ONDEMAND + select PM_DEVFREQ + select DDR help This driver is for the External Memory Controller (EMC) found on Tegra20 chips. The EMC controls the external DRAM on the board. @@ -18,9 +24,11 @@ config TEGRA20_EMC external memory. config TEGRA30_EMC - bool "NVIDIA Tegra30 External Memory Controller driver" + tristate "NVIDIA Tegra30 External Memory Controller driver" default y - depends on TEGRA_MC && ARCH_TEGRA_3x_SOC + depends on ARCH_TEGRA_3x_SOC || COMPILE_TEST + select PM_OPP + select DDR help This driver is for the External Memory Controller (EMC) found on Tegra30 chips. The EMC controls the external DRAM on the board. @@ -28,11 +36,29 @@ config TEGRA30_EMC external memory. config TEGRA124_EMC - bool "NVIDIA Tegra124 External Memory Controller driver" + tristate "NVIDIA Tegra124 External Memory Controller driver" default y - depends on TEGRA_MC && ARCH_TEGRA_124_SOC + depends on ARCH_TEGRA_124_SOC || COMPILE_TEST + select TEGRA124_CLK_EMC if ARCH_TEGRA + select PM_OPP help This driver is for the External Memory Controller (EMC) found on Tegra124 chips. The EMC controls the external DRAM on the board. This driver is required to change memory timings / clock rate for external memory. + +config TEGRA210_EMC_TABLE + bool + depends on ARCH_TEGRA_210_SOC || COMPILE_TEST + +config TEGRA210_EMC + tristate "NVIDIA Tegra210 External Memory Controller driver" + depends on ARCH_TEGRA_210_SOC || COMPILE_TEST + select TEGRA210_EMC_TABLE + help + This driver is for the External Memory Controller (EMC) found on + Tegra210 chips. The EMC controls the external DRAM on the board. + This driver is required to change memory timings / clock rate for + external memory. + +endif diff --git a/drivers/memory/tegra/Makefile b/drivers/memory/tegra/Makefile index 529d10bc5650..0750847dac3c 100644 --- a/drivers/memory/tegra/Makefile +++ b/drivers/memory/tegra/Makefile @@ -7,11 +7,19 @@ tegra-mc-$(CONFIG_ARCH_TEGRA_114_SOC) += tegra114.o tegra-mc-$(CONFIG_ARCH_TEGRA_124_SOC) += tegra124.o tegra-mc-$(CONFIG_ARCH_TEGRA_132_SOC) += tegra124.o tegra-mc-$(CONFIG_ARCH_TEGRA_210_SOC) += tegra210.o +tegra-mc-$(CONFIG_ARCH_TEGRA_186_SOC) += tegra186.o +tegra-mc-$(CONFIG_ARCH_TEGRA_194_SOC) += tegra186.o tegra194.o +tegra-mc-$(CONFIG_ARCH_TEGRA_234_SOC) += tegra186.o tegra234.o obj-$(CONFIG_TEGRA_MC) += tegra-mc.o obj-$(CONFIG_TEGRA20_EMC) += tegra20-emc.o obj-$(CONFIG_TEGRA30_EMC) += tegra30-emc.o obj-$(CONFIG_TEGRA124_EMC) += tegra124-emc.o -obj-$(CONFIG_ARCH_TEGRA_186_SOC) += tegra186.o tegra186-emc.o -obj-$(CONFIG_ARCH_TEGRA_194_SOC) += tegra186.o tegra186-emc.o +obj-$(CONFIG_TEGRA210_EMC_TABLE) += tegra210-emc-table.o +obj-$(CONFIG_TEGRA210_EMC) += tegra210-emc.o +obj-$(CONFIG_ARCH_TEGRA_186_SOC) += tegra186-emc.o +obj-$(CONFIG_ARCH_TEGRA_194_SOC) += tegra186-emc.o +obj-$(CONFIG_ARCH_TEGRA_234_SOC) += tegra186-emc.o + +tegra210-emc-y := tegra210-emc-core.o tegra210-emc-cc-r21021.o diff --git a/drivers/memory/tegra/mc.c b/drivers/memory/tegra/mc.c index ec8403557ed4..2f7a58a9df1a 100644 --- a/drivers/memory/tegra/mc.c +++ b/drivers/memory/tegra/mc.c @@ -6,6 +6,7 @@ #include <linux/clk.h> #include <linux/delay.h> #include <linux/dma-mapping.h> +#include <linux/export.h> #include <linux/interrupt.h> #include <linux/kernel.h> #include <linux/module.h> @@ -38,10 +39,74 @@ static const struct of_device_id tegra_mc_of_match[] = { #ifdef CONFIG_ARCH_TEGRA_210_SOC { .compatible = "nvidia,tegra210-mc", .data = &tegra210_mc_soc }, #endif - { } +#ifdef CONFIG_ARCH_TEGRA_186_SOC + { .compatible = "nvidia,tegra186-mc", .data = &tegra186_mc_soc }, +#endif +#ifdef CONFIG_ARCH_TEGRA_194_SOC + { .compatible = "nvidia,tegra194-mc", .data = &tegra194_mc_soc }, +#endif +#ifdef CONFIG_ARCH_TEGRA_234_SOC + { .compatible = "nvidia,tegra234-mc", .data = &tegra234_mc_soc }, +#endif + { /* sentinel */ } }; MODULE_DEVICE_TABLE(of, tegra_mc_of_match); +static void tegra_mc_devm_action_put_device(void *data) +{ + struct tegra_mc *mc = data; + + put_device(mc->dev); +} + +/** + * devm_tegra_memory_controller_get() - get Tegra Memory Controller handle + * @dev: device pointer for the consumer device + * + * This function will search for the Memory Controller node in a device-tree + * and retrieve the Memory Controller handle. + * + * Return: ERR_PTR() on error or a valid pointer to a struct tegra_mc. + */ +struct tegra_mc *devm_tegra_memory_controller_get(struct device *dev) +{ + struct platform_device *pdev; + struct device_node *np; + struct tegra_mc *mc; + int err; + + np = of_parse_phandle(dev->of_node, "nvidia,memory-controller", 0); + if (!np) + return ERR_PTR(-ENOENT); + + pdev = of_find_device_by_node(np); + of_node_put(np); + if (!pdev) + return ERR_PTR(-ENODEV); + + mc = platform_get_drvdata(pdev); + if (!mc) { + put_device(&pdev->dev); + return ERR_PTR(-EPROBE_DEFER); + } + + err = devm_add_action_or_reset(dev, tegra_mc_devm_action_put_device, mc); + if (err) + return ERR_PTR(err); + + return mc; +} +EXPORT_SYMBOL_GPL(devm_tegra_memory_controller_get); + +int tegra_mc_probe_device(struct tegra_mc *mc, struct device *dev) +{ + if (mc->soc->ops && mc->soc->ops->probe_device) + return mc->soc->ops->probe_device(mc, dev); + + return 0; +} +EXPORT_SYMBOL_GPL(tegra_mc_probe_device); + static int tegra_mc_block_dma_common(struct tegra_mc *mc, const struct tegra_mc_reset *rst) { @@ -127,6 +192,13 @@ static int tegra_mc_hotreset_assert(struct reset_controller_dev *rcdev, if (!rst_ops) return -ENODEV; + /* DMA flushing will fail if reset is already asserted */ + if (rst_ops->reset_status) { + /* check whether reset is asserted */ + if (rst_ops->reset_status(mc, rst)) + return 0; + } + if (rst_ops->block_dma) { /* block clients DMA requests */ err = rst_ops->block_dma(mc, rst); @@ -243,38 +315,6 @@ static int tegra_mc_reset_setup(struct tegra_mc *mc) return 0; } -static int tegra_mc_setup_latency_allowance(struct tegra_mc *mc) -{ - unsigned long long tick; - unsigned int i; - u32 value; - - /* compute the number of MC clock cycles per tick */ - tick = (unsigned long long)mc->tick * clk_get_rate(mc->clk); - do_div(tick, NSEC_PER_SEC); - - value = mc_readl(mc, MC_EMEM_ARB_CFG); - value &= ~MC_EMEM_ARB_CFG_CYCLES_PER_UPDATE_MASK; - value |= MC_EMEM_ARB_CFG_CYCLES_PER_UPDATE(tick); - mc_writel(mc, value, MC_EMEM_ARB_CFG); - - /* write latency allowance defaults */ - for (i = 0; i < mc->soc->num_clients; i++) { - const struct tegra_mc_la *la = &mc->soc->clients[i].la; - u32 value; - - value = mc_readl(mc, la->reg); - value &= ~(la->mask << la->shift); - value |= (la->def & la->mask) << la->shift; - mc_writel(mc, value, la->reg); - } - - /* latch new values */ - mc_writel(mc, MC_TIMING_UPDATE, MC_TIMING_CONTROL); - - return 0; -} - int tegra_mc_write_emem_configuration(struct tegra_mc *mc, unsigned long rate) { unsigned int i; @@ -298,6 +338,7 @@ int tegra_mc_write_emem_configuration(struct tegra_mc *mc, unsigned long rate) return 0; } +EXPORT_SYMBOL_GPL(tegra_mc_write_emem_configuration); unsigned int tegra_mc_get_emem_device_count(struct tegra_mc *mc) { @@ -309,6 +350,44 @@ unsigned int tegra_mc_get_emem_device_count(struct tegra_mc *mc) return dram_count; } +EXPORT_SYMBOL_GPL(tegra_mc_get_emem_device_count); + +#if defined(CONFIG_ARCH_TEGRA_3x_SOC) || \ + defined(CONFIG_ARCH_TEGRA_114_SOC) || \ + defined(CONFIG_ARCH_TEGRA_124_SOC) || \ + defined(CONFIG_ARCH_TEGRA_132_SOC) || \ + defined(CONFIG_ARCH_TEGRA_210_SOC) +static int tegra_mc_setup_latency_allowance(struct tegra_mc *mc) +{ + unsigned long long tick; + unsigned int i; + u32 value; + + /* compute the number of MC clock cycles per tick */ + tick = (unsigned long long)mc->tick * clk_get_rate(mc->clk); + do_div(tick, NSEC_PER_SEC); + + value = mc_readl(mc, MC_EMEM_ARB_CFG); + value &= ~MC_EMEM_ARB_CFG_CYCLES_PER_UPDATE_MASK; + value |= MC_EMEM_ARB_CFG_CYCLES_PER_UPDATE(tick); + mc_writel(mc, value, MC_EMEM_ARB_CFG); + + /* write latency allowance defaults */ + for (i = 0; i < mc->soc->num_clients; i++) { + const struct tegra_mc_client *client = &mc->soc->clients[i]; + u32 value; + + value = mc_readl(mc, client->regs.la.reg); + value &= ~(client->regs.la.mask << client->regs.la.shift); + value |= (client->regs.la.def & client->regs.la.mask) << client->regs.la.shift; + mc_writel(mc, value, client->regs.la.reg); + } + + /* latch new values */ + mc_writel(mc, MC_TIMING_UPDATE, MC_TIMING_CONTROL); + + return 0; +} static int load_one_timing(struct tegra_mc *mc, struct tegra_mc_timing *timing, @@ -401,53 +480,153 @@ static int tegra_mc_setup_timings(struct tegra_mc *mc) return 0; } -static const char *const status_names[32] = { - [ 1] = "External interrupt", - [ 6] = "EMEM address decode error", - [ 7] = "GART page fault", - [ 8] = "Security violation", - [ 9] = "EMEM arbitration error", - [10] = "Page fault", - [11] = "Invalid APB ASID update", - [12] = "VPR violation", - [13] = "Secure carveout violation", - [16] = "MTS carveout violation", -}; +int tegra30_mc_probe(struct tegra_mc *mc) +{ + int err; -static const char *const error_names[8] = { - [2] = "EMEM decode error", - [3] = "TrustZone violation", - [4] = "Carveout violation", - [6] = "SMMU translation error", + mc->clk = devm_clk_get_optional(mc->dev, "mc"); + if (IS_ERR(mc->clk)) { + dev_err(mc->dev, "failed to get MC clock: %ld\n", PTR_ERR(mc->clk)); + return PTR_ERR(mc->clk); + } + + /* ensure that debug features are disabled */ + mc_writel(mc, 0x00000000, MC_TIMING_CONTROL_DBG); + + err = tegra_mc_setup_latency_allowance(mc); + if (err < 0) { + dev_err(mc->dev, "failed to setup latency allowance: %d\n", err); + return err; + } + + err = tegra_mc_setup_timings(mc); + if (err < 0) { + dev_err(mc->dev, "failed to setup timings: %d\n", err); + return err; + } + + return 0; +} + +const struct tegra_mc_ops tegra30_mc_ops = { + .probe = tegra30_mc_probe, + .handle_irq = tegra30_mc_handle_irq, }; +#endif -static irqreturn_t tegra_mc_irq(int irq, void *data) +static int mc_global_intstatus_to_channel(const struct tegra_mc *mc, u32 status, + unsigned int *mc_channel) +{ + if ((status & mc->soc->ch_intmask) == 0) + return -EINVAL; + + *mc_channel = __ffs((status & mc->soc->ch_intmask) >> + mc->soc->global_intstatus_channel_shift); + + return 0; +} + +static u32 mc_channel_to_global_intstatus(const struct tegra_mc *mc, + unsigned int channel) +{ + return BIT(channel) << mc->soc->global_intstatus_channel_shift; +} + +irqreturn_t tegra30_mc_handle_irq(int irq, void *data) { struct tegra_mc *mc = data; + unsigned int bit, channel; unsigned long status; - unsigned int bit; - /* mask all interrupts to avoid flooding */ - status = mc_readl(mc, MC_INTSTATUS) & mc->soc->intmask; + if (mc->soc->num_channels) { + u32 global_status; + int err; + + global_status = mc_ch_readl(mc, MC_BROADCAST_CHANNEL, MC_GLOBAL_INTSTATUS); + err = mc_global_intstatus_to_channel(mc, global_status, &channel); + if (err < 0) { + dev_err_ratelimited(mc->dev, "unknown interrupt channel 0x%08x\n", + global_status); + return IRQ_NONE; + } + + /* mask all interrupts to avoid flooding */ + status = mc_ch_readl(mc, channel, MC_INTSTATUS) & mc->soc->intmask; + } else { + status = mc_readl(mc, MC_INTSTATUS) & mc->soc->intmask; + } + if (!status) return IRQ_NONE; for_each_set_bit(bit, &status, 32) { - const char *error = status_names[bit] ?: "unknown"; + const char *error = tegra_mc_status_names[bit] ?: "unknown"; const char *client = "unknown", *desc; const char *direction, *secure; + u32 status_reg, addr_reg; + u32 intmask = BIT(bit); phys_addr_t addr = 0; +#ifdef CONFIG_PHYS_ADDR_T_64BIT + u32 addr_hi_reg = 0; +#endif unsigned int i; char perm[7]; u8 id, type; u32 value; - value = mc_readl(mc, MC_ERR_STATUS); + switch (intmask) { + case MC_INT_DECERR_VPR: + status_reg = MC_ERR_VPR_STATUS; + addr_reg = MC_ERR_VPR_ADR; + break; + + case MC_INT_SECERR_SEC: + status_reg = MC_ERR_SEC_STATUS; + addr_reg = MC_ERR_SEC_ADR; + break; + + case MC_INT_DECERR_MTS: + status_reg = MC_ERR_MTS_STATUS; + addr_reg = MC_ERR_MTS_ADR; + break; + + case MC_INT_DECERR_GENERALIZED_CARVEOUT: + status_reg = MC_ERR_GENERALIZED_CARVEOUT_STATUS; + addr_reg = MC_ERR_GENERALIZED_CARVEOUT_ADR; + break; + + case MC_INT_DECERR_ROUTE_SANITY: + status_reg = MC_ERR_ROUTE_SANITY_STATUS; + addr_reg = MC_ERR_ROUTE_SANITY_ADR; + break; + + default: + status_reg = MC_ERR_STATUS; + addr_reg = MC_ERR_ADR; + +#ifdef CONFIG_PHYS_ADDR_T_64BIT + if (mc->soc->has_addr_hi_reg) + addr_hi_reg = MC_ERR_ADR_HI; +#endif + break; + } + + if (mc->soc->num_channels) + value = mc_ch_readl(mc, channel, status_reg); + else + value = mc_readl(mc, status_reg); #ifdef CONFIG_PHYS_ADDR_T_64BIT if (mc->soc->num_address_bits > 32) { - addr = ((value >> MC_ERR_STATUS_ADR_HI_SHIFT) & - MC_ERR_STATUS_ADR_HI_MASK); + if (addr_hi_reg) { + if (mc->soc->num_channels) + addr = mc_ch_readl(mc, channel, addr_hi_reg); + else + addr = mc_readl(mc, addr_hi_reg); + } else { + addr = ((value >> MC_ERR_STATUS_ADR_HI_SHIFT) & + MC_ERR_STATUS_ADR_HI_MASK); + } addr <<= 32; } #endif @@ -473,7 +652,7 @@ static irqreturn_t tegra_mc_irq(int irq, void *data) type = (value & MC_ERR_STATUS_TYPE_MASK) >> MC_ERR_STATUS_TYPE_SHIFT; - desc = error_names[type]; + desc = tegra_mc_error_names[type]; switch (value & MC_ERR_STATUS_TYPE_MASK) { case MC_ERR_STATUS_TYPE_INVALID_SMMU_PAGE: @@ -504,7 +683,10 @@ static irqreturn_t tegra_mc_irq(int irq, void *data) break; } - value = mc_readl(mc, MC_ERR_ADR); + if (mc->soc->num_channels) + value = mc_ch_readl(mc, channel, addr_reg); + else + value = mc_readl(mc, addr_reg); addr |= value; dev_err_ratelimited(mc->dev, "%s: %s%s @%pa: %s (%s%s)\n", @@ -513,89 +695,129 @@ static irqreturn_t tegra_mc_irq(int irq, void *data) } /* clear interrupts */ - mc_writel(mc, status, MC_INTSTATUS); + if (mc->soc->num_channels) { + mc_ch_writel(mc, channel, status, MC_INTSTATUS); + mc_ch_writel(mc, MC_BROADCAST_CHANNEL, + mc_channel_to_global_intstatus(mc, channel), + MC_GLOBAL_INTSTATUS); + } else { + mc_writel(mc, status, MC_INTSTATUS); + } return IRQ_HANDLED; } -static __maybe_unused irqreturn_t tegra20_mc_irq(int irq, void *data) -{ - struct tegra_mc *mc = data; - unsigned long status; - unsigned int bit; - - /* mask all interrupts to avoid flooding */ - status = mc_readl(mc, MC_INTSTATUS) & mc->soc->intmask; - if (!status) - return IRQ_NONE; - - for_each_set_bit(bit, &status, 32) { - const char *direction = "read", *secure = ""; - const char *error = status_names[bit]; - const char *client, *desc; - phys_addr_t addr; - u32 value, reg; - u8 id, type; - - switch (BIT(bit)) { - case MC_INT_DECERR_EMEM: - reg = MC_DECERR_EMEM_OTHERS_STATUS; - value = mc_readl(mc, reg); +const char *const tegra_mc_status_names[32] = { + [ 1] = "External interrupt", + [ 6] = "EMEM address decode error", + [ 7] = "GART page fault", + [ 8] = "Security violation", + [ 9] = "EMEM arbitration error", + [10] = "Page fault", + [11] = "Invalid APB ASID update", + [12] = "VPR violation", + [13] = "Secure carveout violation", + [16] = "MTS carveout violation", + [17] = "Generalized carveout violation", + [20] = "Route Sanity error", +}; - id = value & mc->soc->client_id_mask; - desc = error_names[2]; +const char *const tegra_mc_error_names[8] = { + [2] = "EMEM decode error", + [3] = "TrustZone violation", + [4] = "Carveout violation", + [6] = "SMMU translation error", +}; - if (value & BIT(31)) - direction = "write"; - break; +/* + * Memory Controller (MC) has few Memory Clients that are issuing memory + * bandwidth allocation requests to the MC interconnect provider. The MC + * provider aggregates the requests and then sends the aggregated request + * up to the External Memory Controller (EMC) interconnect provider which + * re-configures hardware interface to External Memory (EMEM) in accordance + * to the required bandwidth. Each MC interconnect node represents an + * individual Memory Client. + * + * Memory interconnect topology: + * + * +----+ + * +--------+ | | + * | TEXSRD +--->+ | + * +--------+ | | + * | | +-----+ +------+ + * ... | MC +--->+ EMC +--->+ EMEM | + * | | +-----+ +------+ + * +--------+ | | + * | DISP.. +--->+ | + * +--------+ | | + * +----+ + */ +static int tegra_mc_interconnect_setup(struct tegra_mc *mc) +{ + struct icc_node *node; + unsigned int i; + int err; - case MC_INT_INVALID_GART_PAGE: - reg = MC_GART_ERROR_REQ; - value = mc_readl(mc, reg); + /* older device-trees don't have interconnect properties */ + if (!device_property_present(mc->dev, "#interconnect-cells") || + !mc->soc->icc_ops) + return 0; - id = (value >> 1) & mc->soc->client_id_mask; - desc = error_names[2]; + mc->provider.dev = mc->dev; + mc->provider.data = &mc->provider; + mc->provider.set = mc->soc->icc_ops->set; + mc->provider.aggregate = mc->soc->icc_ops->aggregate; + mc->provider.xlate_extended = mc->soc->icc_ops->xlate_extended; - if (value & BIT(0)) - direction = "write"; - break; + err = icc_provider_add(&mc->provider); + if (err) + return err; - case MC_INT_SECURITY_VIOLATION: - reg = MC_SECURITY_VIOLATION_STATUS; - value = mc_readl(mc, reg); + /* create Memory Controller node */ + node = icc_node_create(TEGRA_ICC_MC); + if (IS_ERR(node)) { + err = PTR_ERR(node); + goto del_provider; + } - id = value & mc->soc->client_id_mask; - type = (value & BIT(30)) ? 4 : 3; - desc = error_names[type]; - secure = "secure "; + node->name = "Memory Controller"; + icc_node_add(node, &mc->provider); - if (value & BIT(31)) - direction = "write"; - break; + /* link Memory Controller to External Memory Controller */ + err = icc_link_create(node, TEGRA_ICC_EMC); + if (err) + goto remove_nodes; - default: - continue; + for (i = 0; i < mc->soc->num_clients; i++) { + /* create MC client node */ + node = icc_node_create(mc->soc->clients[i].id); + if (IS_ERR(node)) { + err = PTR_ERR(node); + goto remove_nodes; } - client = mc->soc->clients[id].name; - addr = mc_readl(mc, reg + sizeof(u32)); + node->name = mc->soc->clients[i].name; + icc_node_add(node, &mc->provider); - dev_err_ratelimited(mc->dev, "%s: %s%s @%pa: %s (%s)\n", - client, secure, direction, &addr, error, - desc); + /* link Memory Client to Memory Controller */ + err = icc_link_create(node, TEGRA_ICC_MC); + if (err) + goto remove_nodes; } - /* clear interrupts */ - mc_writel(mc, status, MC_INTSTATUS); + return 0; - return IRQ_HANDLED; +remove_nodes: + icc_nodes_remove(&mc->provider); +del_provider: + icc_provider_del(&mc->provider); + + return err; } static int tegra_mc_probe(struct platform_device *pdev) { - struct resource *res; struct tegra_mc *mc; - void *isr; u64 mask; int err; @@ -619,66 +841,49 @@ static int tegra_mc_probe(struct platform_device *pdev) /* length of MC tick in nanoseconds */ mc->tick = 30; - res = platform_get_resource(pdev, IORESOURCE_MEM, 0); - mc->regs = devm_ioremap_resource(&pdev->dev, res); + mc->regs = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(mc->regs)) return PTR_ERR(mc->regs); - mc->clk = devm_clk_get(&pdev->dev, "mc"); - if (IS_ERR(mc->clk)) { - dev_err(&pdev->dev, "failed to get MC clock: %ld\n", - PTR_ERR(mc->clk)); - return PTR_ERR(mc->clk); + mc->debugfs.root = debugfs_create_dir("mc", NULL); + + if (mc->soc->ops && mc->soc->ops->probe) { + err = mc->soc->ops->probe(mc); + if (err < 0) + return err; } -#ifdef CONFIG_ARCH_TEGRA_2x_SOC - if (mc->soc == &tegra20_mc_soc) { - isr = tegra20_mc_irq; - } else -#endif - { - /* ensure that debug features are disabled */ - mc_writel(mc, 0x00000000, MC_TIMING_CONTROL_DBG); + if (mc->soc->ops && mc->soc->ops->handle_irq) { + mc->irq = platform_get_irq(pdev, 0); + if (mc->irq < 0) + return mc->irq; - err = tegra_mc_setup_latency_allowance(mc); - if (err < 0) { - dev_err(&pdev->dev, - "failed to setup latency allowance: %d\n", - err); - return err; - } + WARN(!mc->soc->client_id_mask, "missing client ID mask for this SoC\n"); - isr = tegra_mc_irq; + if (mc->soc->num_channels) + mc_ch_writel(mc, MC_BROADCAST_CHANNEL, mc->soc->intmask, + MC_INTMASK); + else + mc_writel(mc, mc->soc->intmask, MC_INTMASK); - err = tegra_mc_setup_timings(mc); + err = devm_request_irq(&pdev->dev, mc->irq, mc->soc->ops->handle_irq, 0, + dev_name(&pdev->dev), mc); if (err < 0) { - dev_err(&pdev->dev, "failed to setup timings: %d\n", + dev_err(&pdev->dev, "failed to request IRQ#%u: %d\n", mc->irq, err); return err; } } - mc->irq = platform_get_irq(pdev, 0); - if (mc->irq < 0) { - dev_err(&pdev->dev, "interrupt not specified\n"); - return mc->irq; - } - - WARN(!mc->soc->client_id_mask, "missing client ID mask for this SoC\n"); - - mc_writel(mc, mc->soc->intmask, MC_INTMASK); - - err = devm_request_irq(&pdev->dev, mc->irq, isr, 0, - dev_name(&pdev->dev), mc); - if (err < 0) { - dev_err(&pdev->dev, "failed to request IRQ#%u: %d\n", mc->irq, - err); - return err; + if (mc->soc->reset_ops) { + err = tegra_mc_reset_setup(mc); + if (err < 0) + dev_err(&pdev->dev, "failed to register reset controller: %d\n", err); } - err = tegra_mc_reset_setup(mc); + err = tegra_mc_interconnect_setup(mc); if (err < 0) - dev_err(&pdev->dev, "failed to register reset controller: %d\n", + dev_err(&pdev->dev, "failed to initialize interconnect: %d\n", err); if (IS_ENABLED(CONFIG_TEGRA_IOMMU_SMMU) && mc->soc->smmu) { @@ -702,37 +907,37 @@ static int tegra_mc_probe(struct platform_device *pdev) return 0; } -static int tegra_mc_suspend(struct device *dev) +static int __maybe_unused tegra_mc_suspend(struct device *dev) { struct tegra_mc *mc = dev_get_drvdata(dev); - int err; - if (IS_ENABLED(CONFIG_TEGRA_IOMMU_GART) && mc->gart) { - err = tegra_gart_suspend(mc->gart); - if (err) - return err; - } + if (mc->soc->ops && mc->soc->ops->suspend) + return mc->soc->ops->suspend(mc); return 0; } -static int tegra_mc_resume(struct device *dev) +static int __maybe_unused tegra_mc_resume(struct device *dev) { struct tegra_mc *mc = dev_get_drvdata(dev); - int err; - if (IS_ENABLED(CONFIG_TEGRA_IOMMU_GART) && mc->gart) { - err = tegra_gart_resume(mc->gart); - if (err) - return err; - } + if (mc->soc->ops && mc->soc->ops->resume) + return mc->soc->ops->resume(mc); return 0; } +static void tegra_mc_sync_state(struct device *dev) +{ + struct tegra_mc *mc = dev_get_drvdata(dev); + + /* check whether ICC provider is registered */ + if (mc->provider.dev == dev) + icc_sync_state(dev); +} + static const struct dev_pm_ops tegra_mc_pm_ops = { - .suspend = tegra_mc_suspend, - .resume = tegra_mc_resume, + SET_SYSTEM_SLEEP_PM_OPS(tegra_mc_suspend, tegra_mc_resume) }; static struct platform_driver tegra_mc_driver = { @@ -741,6 +946,7 @@ static struct platform_driver tegra_mc_driver = { .of_match_table = tegra_mc_of_match, .pm = &tegra_mc_pm_ops, .suppress_bind_attrs = true, + .sync_state = tegra_mc_sync_state, }, .prevent_deferred_probe = true, .probe = tegra_mc_probe, diff --git a/drivers/memory/tegra/mc.h b/drivers/memory/tegra/mc.h index 957c6eb74ff9..bc01586b6560 100644 --- a/drivers/memory/tegra/mc.h +++ b/drivers/memory/tegra/mc.h @@ -34,6 +34,7 @@ #define MC_EMEM_ARB_TIMING_W2W 0xbc #define MC_EMEM_ARB_TIMING_R2W 0xc0 #define MC_EMEM_ARB_TIMING_W2R 0xc4 +#define MC_EMEM_ARB_MISC2 0xc8 #define MC_EMEM_ARB_DA_TURNS 0xd0 #define MC_EMEM_ARB_DA_COVERS 0xd4 #define MC_EMEM_ARB_MISC0 0xd8 @@ -42,7 +43,21 @@ #define MC_EMEM_ARB_OVERRIDE 0xe8 #define MC_TIMING_CONTROL_DBG 0xf8 #define MC_TIMING_CONTROL 0xfc - +#define MC_ERR_VPR_STATUS 0x654 +#define MC_ERR_VPR_ADR 0x658 +#define MC_ERR_SEC_STATUS 0x67c +#define MC_ERR_SEC_ADR 0x680 +#define MC_ERR_MTS_STATUS 0x9b0 +#define MC_ERR_MTS_ADR 0x9b4 +#define MC_ERR_ROUTE_SANITY_STATUS 0x9c0 +#define MC_ERR_ROUTE_SANITY_ADR 0x9c4 +#define MC_ERR_GENERALIZED_CARVEOUT_STATUS 0xc00 +#define MC_ERR_GENERALIZED_CARVEOUT_ADR 0xc04 +#define MC_GLOBAL_INTSTATUS 0xf24 +#define MC_ERR_ADR_HI 0x11fc + +#define MC_INT_DECERR_ROUTE_SANITY BIT(20) +#define MC_INT_DECERR_GENERALIZED_CARVEOUT BIT(17) #define MC_INT_DECERR_MTS BIT(16) #define MC_INT_SECERR_SEC BIT(13) #define MC_INT_DECERR_VPR BIT(12) @@ -77,12 +92,52 @@ #define MC_TIMING_UPDATE BIT(0) -static inline u32 mc_readl(struct tegra_mc *mc, unsigned long offset) +#define MC_BROADCAST_CHANNEL ~0 + +static inline u32 tegra_mc_scale_percents(u64 val, unsigned int percents) +{ + val = val * percents; + do_div(val, 100); + + return min_t(u64, val, U32_MAX); +} + +static inline struct tegra_mc * +icc_provider_to_tegra_mc(struct icc_provider *provider) +{ + return container_of(provider, struct tegra_mc, provider); +} + +static inline u32 mc_ch_readl(const struct tegra_mc *mc, int ch, + unsigned long offset) +{ + if (!mc->bcast_ch_regs) + return 0; + + if (ch == MC_BROADCAST_CHANNEL) + return readl_relaxed(mc->bcast_ch_regs + offset); + + return readl_relaxed(mc->ch_regs[ch] + offset); +} + +static inline void mc_ch_writel(const struct tegra_mc *mc, int ch, + u32 value, unsigned long offset) +{ + if (!mc->bcast_ch_regs) + return; + + if (ch == MC_BROADCAST_CHANNEL) + writel_relaxed(value, mc->bcast_ch_regs + offset); + else + writel_relaxed(value, mc->ch_regs[ch] + offset); +} + +static inline u32 mc_readl(const struct tegra_mc *mc, unsigned long offset) { return readl_relaxed(mc->regs + offset); } -static inline void mc_writel(struct tegra_mc *mc, u32 value, +static inline void mc_writel(const struct tegra_mc *mc, u32 value, unsigned long offset) { writel_relaxed(value, mc->regs + offset); @@ -114,4 +169,43 @@ extern const struct tegra_mc_soc tegra132_mc_soc; extern const struct tegra_mc_soc tegra210_mc_soc; #endif +#ifdef CONFIG_ARCH_TEGRA_186_SOC +extern const struct tegra_mc_soc tegra186_mc_soc; +#endif + +#ifdef CONFIG_ARCH_TEGRA_194_SOC +extern const struct tegra_mc_soc tegra194_mc_soc; +#endif + +#ifdef CONFIG_ARCH_TEGRA_234_SOC +extern const struct tegra_mc_soc tegra234_mc_soc; +#endif + +#if defined(CONFIG_ARCH_TEGRA_3x_SOC) || \ + defined(CONFIG_ARCH_TEGRA_114_SOC) || \ + defined(CONFIG_ARCH_TEGRA_124_SOC) || \ + defined(CONFIG_ARCH_TEGRA_132_SOC) || \ + defined(CONFIG_ARCH_TEGRA_210_SOC) +int tegra30_mc_probe(struct tegra_mc *mc); +extern const struct tegra_mc_ops tegra30_mc_ops; +#endif + +#if defined(CONFIG_ARCH_TEGRA_186_SOC) || \ + defined(CONFIG_ARCH_TEGRA_194_SOC) || \ + defined(CONFIG_ARCH_TEGRA_234_SOC) +extern const struct tegra_mc_ops tegra186_mc_ops; +#endif + +irqreturn_t tegra30_mc_handle_irq(int irq, void *data); +extern const char * const tegra_mc_status_names[32]; +extern const char * const tegra_mc_error_names[8]; + +/* + * These IDs are for internal use of Tegra ICC drivers. The ID numbers are + * chosen such that they don't conflict with the device-tree ICC node IDs. + */ +#define TEGRA_ICC_MC 1000 +#define TEGRA_ICC_EMC 1001 +#define TEGRA_ICC_EMEM 1002 + #endif /* MEMORY_TEGRA_MC_H */ diff --git a/drivers/memory/tegra/tegra114.c b/drivers/memory/tegra/tegra114.c index 48ef01c3ff90..41350570c815 100644 --- a/drivers/memory/tegra/tegra114.c +++ b/drivers/memory/tegra/tegra114.c @@ -15,877 +15,1013 @@ static const struct tegra_mc_client tegra114_mc_clients[] = { .id = 0x00, .name = "ptcr", .swgroup = TEGRA_SWGROUP_PTC, + .regs = { + .la = { + .reg = 0x34c, + .shift = 0, + .mask = 0xff, + .def = 0x0, + }, + }, }, { .id = 0x01, .name = "display0a", .swgroup = TEGRA_SWGROUP_DC, - .smmu = { - .reg = 0x228, - .bit = 1, - }, - .la = { - .reg = 0x2e8, - .shift = 0, - .mask = 0xff, - .def = 0x4e, + .regs = { + .smmu = { + .reg = 0x228, + .bit = 1, + }, + .la = { + .reg = 0x2e8, + .shift = 0, + .mask = 0xff, + .def = 0x4e, + }, }, }, { .id = 0x02, .name = "display0ab", .swgroup = TEGRA_SWGROUP_DCB, - .smmu = { - .reg = 0x228, - .bit = 2, - }, - .la = { - .reg = 0x2f4, - .shift = 0, - .mask = 0xff, - .def = 0x4e, + .regs = { + .smmu = { + .reg = 0x228, + .bit = 2, + }, + .la = { + .reg = 0x2f4, + .shift = 0, + .mask = 0xff, + .def = 0x4e, + }, }, }, { .id = 0x03, .name = "display0b", .swgroup = TEGRA_SWGROUP_DC, - .smmu = { - .reg = 0x228, - .bit = 3, - }, - .la = { - .reg = 0x2e8, - .shift = 16, - .mask = 0xff, - .def = 0x4e, + .regs = { + .smmu = { + .reg = 0x228, + .bit = 3, + }, + .la = { + .reg = 0x2e8, + .shift = 16, + .mask = 0xff, + .def = 0x4e, + }, }, }, { .id = 0x04, .name = "display0bb", .swgroup = TEGRA_SWGROUP_DCB, - .smmu = { - .reg = 0x228, - .bit = 4, - }, - .la = { - .reg = 0x2f4, - .shift = 16, - .mask = 0xff, - .def = 0x4e, + .regs = { + .smmu = { + .reg = 0x228, + .bit = 4, + }, + .la = { + .reg = 0x2f4, + .shift = 16, + .mask = 0xff, + .def = 0x4e, + }, }, }, { .id = 0x05, .name = "display0c", .swgroup = TEGRA_SWGROUP_DC, - .smmu = { - .reg = 0x228, - .bit = 5, - }, - .la = { - .reg = 0x2ec, - .shift = 0, - .mask = 0xff, - .def = 0x4e, + .regs = { + .smmu = { + .reg = 0x228, + .bit = 5, + }, + .la = { + .reg = 0x2ec, + .shift = 0, + .mask = 0xff, + .def = 0x4e, + }, }, }, { .id = 0x06, .name = "display0cb", .swgroup = TEGRA_SWGROUP_DCB, - .smmu = { - .reg = 0x228, - .bit = 6, - }, - .la = { - .reg = 0x2f8, - .shift = 0, - .mask = 0xff, - .def = 0x4e, + .regs = { + .smmu = { + .reg = 0x228, + .bit = 6, + }, + .la = { + .reg = 0x2f8, + .shift = 0, + .mask = 0xff, + .def = 0x4e, + }, }, }, { .id = 0x09, .name = "eppup", .swgroup = TEGRA_SWGROUP_EPP, - .smmu = { - .reg = 0x228, - .bit = 9, - }, - .la = { - .reg = 0x300, - .shift = 0, - .mask = 0xff, - .def = 0x33, + .regs = { + .smmu = { + .reg = 0x228, + .bit = 9, + }, + .la = { + .reg = 0x300, + .shift = 0, + .mask = 0xff, + .def = 0x33, + }, }, }, { .id = 0x0a, .name = "g2pr", .swgroup = TEGRA_SWGROUP_G2, - .smmu = { - .reg = 0x228, - .bit = 10, - }, - .la = { - .reg = 0x308, - .shift = 0, - .mask = 0xff, - .def = 0x09, + .regs = { + .smmu = { + .reg = 0x228, + .bit = 10, + }, + .la = { + .reg = 0x308, + .shift = 0, + .mask = 0xff, + .def = 0x09, + }, }, }, { .id = 0x0b, .name = "g2sr", .swgroup = TEGRA_SWGROUP_G2, - .smmu = { - .reg = 0x228, - .bit = 11, - }, - .la = { - .reg = 0x308, - .shift = 16, - .mask = 0xff, - .def = 0x09, + .regs = { + .smmu = { + .reg = 0x228, + .bit = 11, + }, + .la = { + .reg = 0x308, + .shift = 16, + .mask = 0xff, + .def = 0x09, + }, }, }, { .id = 0x0f, .name = "avpcarm7r", .swgroup = TEGRA_SWGROUP_AVPC, - .smmu = { - .reg = 0x228, - .bit = 15, - }, - .la = { - .reg = 0x2e4, - .shift = 0, - .mask = 0xff, - .def = 0x04, + .regs = { + .smmu = { + .reg = 0x228, + .bit = 15, + }, + .la = { + .reg = 0x2e4, + .shift = 0, + .mask = 0xff, + .def = 0x04, + }, }, }, { .id = 0x10, .name = "displayhc", .swgroup = TEGRA_SWGROUP_DC, - .smmu = { - .reg = 0x228, - .bit = 16, - }, - .la = { - .reg = 0x2f0, - .shift = 0, - .mask = 0xff, - .def = 0x68, + .regs = { + .smmu = { + .reg = 0x228, + .bit = 16, + }, + .la = { + .reg = 0x2f0, + .shift = 0, + .mask = 0xff, + .def = 0x68, + }, }, }, { .id = 0x11, .name = "displayhcb", .swgroup = TEGRA_SWGROUP_DCB, - .smmu = { - .reg = 0x228, - .bit = 17, - }, - .la = { - .reg = 0x2fc, - .shift = 0, - .mask = 0xff, - .def = 0x68, + .regs = { + .smmu = { + .reg = 0x228, + .bit = 17, + }, + .la = { + .reg = 0x2fc, + .shift = 0, + .mask = 0xff, + .def = 0x68, + }, }, }, { .id = 0x12, .name = "fdcdrd", .swgroup = TEGRA_SWGROUP_NV, - .smmu = { - .reg = 0x228, - .bit = 18, - }, - .la = { - .reg = 0x334, - .shift = 0, - .mask = 0xff, - .def = 0x0c, + .regs = { + .smmu = { + .reg = 0x228, + .bit = 18, + }, + .la = { + .reg = 0x334, + .shift = 0, + .mask = 0xff, + .def = 0x0c, + }, }, }, { .id = 0x13, .name = "fdcdrd2", .swgroup = TEGRA_SWGROUP_NV, - .smmu = { - .reg = 0x228, - .bit = 19, - }, - .la = { - .reg = 0x33c, - .shift = 0, - .mask = 0xff, - .def = 0x0c, + .regs = { + .smmu = { + .reg = 0x228, + .bit = 19, + }, + .la = { + .reg = 0x33c, + .shift = 0, + .mask = 0xff, + .def = 0x0c, + }, }, }, { .id = 0x14, .name = "g2dr", .swgroup = TEGRA_SWGROUP_G2, - .smmu = { - .reg = 0x228, - .bit = 20, - }, - .la = { - .reg = 0x30c, - .shift = 0, - .mask = 0xff, - .def = 0x0a, + .regs = { + .smmu = { + .reg = 0x228, + .bit = 20, + }, + .la = { + .reg = 0x30c, + .shift = 0, + .mask = 0xff, + .def = 0x0a, + }, }, }, { .id = 0x15, .name = "hdar", .swgroup = TEGRA_SWGROUP_HDA, - .smmu = { - .reg = 0x228, - .bit = 21, - }, - .la = { - .reg = 0x318, - .shift = 0, - .mask = 0xff, - .def = 0xff, + .regs = { + .smmu = { + .reg = 0x228, + .bit = 21, + }, + .la = { + .reg = 0x318, + .shift = 0, + .mask = 0xff, + .def = 0xff, + }, }, }, { .id = 0x16, .name = "host1xdmar", .swgroup = TEGRA_SWGROUP_HC, - .smmu = { - .reg = 0x228, - .bit = 22, - }, - .la = { - .reg = 0x310, - .shift = 0, - .mask = 0xff, - .def = 0x10, + .regs = { + .smmu = { + .reg = 0x228, + .bit = 22, + }, + .la = { + .reg = 0x310, + .shift = 0, + .mask = 0xff, + .def = 0x10, + }, }, }, { .id = 0x17, .name = "host1xr", .swgroup = TEGRA_SWGROUP_HC, - .smmu = { - .reg = 0x228, - .bit = 23, - }, - .la = { - .reg = 0x310, - .shift = 16, - .mask = 0xff, - .def = 0xa5, + .regs = { + .smmu = { + .reg = 0x228, + .bit = 23, + }, + .la = { + .reg = 0x310, + .shift = 16, + .mask = 0xff, + .def = 0xa5, + }, }, }, { .id = 0x18, .name = "idxsrd", .swgroup = TEGRA_SWGROUP_NV, - .smmu = { - .reg = 0x228, - .bit = 24, - }, - .la = { - .reg = 0x334, - .shift = 16, - .mask = 0xff, - .def = 0x0b, + .regs = { + .smmu = { + .reg = 0x228, + .bit = 24, + }, + .la = { + .reg = 0x334, + .shift = 16, + .mask = 0xff, + .def = 0x0b, + }, }, }, { .id = 0x1c, .name = "msencsrd", .swgroup = TEGRA_SWGROUP_MSENC, - .smmu = { - .reg = 0x228, - .bit = 28, - }, - .la = { - .reg = 0x328, - .shift = 0, - .mask = 0xff, - .def = 0x80, + .regs = { + .smmu = { + .reg = 0x228, + .bit = 28, + }, + .la = { + .reg = 0x328, + .shift = 0, + .mask = 0xff, + .def = 0x80, + }, }, }, { .id = 0x1d, .name = "ppcsahbdmar", .swgroup = TEGRA_SWGROUP_PPCS, - .smmu = { - .reg = 0x228, - .bit = 29, - }, - .la = { - .reg = 0x344, - .shift = 0, - .mask = 0xff, - .def = 0x50, + .regs = { + .smmu = { + .reg = 0x228, + .bit = 29, + }, + .la = { + .reg = 0x344, + .shift = 0, + .mask = 0xff, + .def = 0x50, + }, }, }, { .id = 0x1e, .name = "ppcsahbslvr", .swgroup = TEGRA_SWGROUP_PPCS, - .smmu = { - .reg = 0x228, - .bit = 30, - }, - .la = { - .reg = 0x344, - .shift = 16, - .mask = 0xff, - .def = 0xe8, + .regs = { + .smmu = { + .reg = 0x228, + .bit = 30, + }, + .la = { + .reg = 0x344, + .shift = 16, + .mask = 0xff, + .def = 0xe8, + }, }, }, { .id = 0x20, .name = "texl2srd", .swgroup = TEGRA_SWGROUP_NV, - .smmu = { - .reg = 0x22c, - .bit = 0, - }, - .la = { - .reg = 0x338, - .shift = 0, - .mask = 0xff, - .def = 0x0c, + .regs = { + .smmu = { + .reg = 0x22c, + .bit = 0, + }, + .la = { + .reg = 0x338, + .shift = 0, + .mask = 0xff, + .def = 0x0c, + }, }, }, { .id = 0x22, .name = "vdebsevr", .swgroup = TEGRA_SWGROUP_VDE, - .smmu = { - .reg = 0x22c, - .bit = 2, - }, - .la = { - .reg = 0x354, - .shift = 0, - .mask = 0xff, - .def = 0xff, + .regs = { + .smmu = { + .reg = 0x22c, + .bit = 2, + }, + .la = { + .reg = 0x354, + .shift = 0, + .mask = 0xff, + .def = 0xff, + }, }, }, { .id = 0x23, .name = "vdember", .swgroup = TEGRA_SWGROUP_VDE, - .smmu = { - .reg = 0x22c, - .bit = 3, - }, - .la = { - .reg = 0x354, - .shift = 16, - .mask = 0xff, - .def = 0xff, + .regs = { + .smmu = { + .reg = 0x22c, + .bit = 3, + }, + .la = { + .reg = 0x354, + .shift = 16, + .mask = 0xff, + .def = 0xff, + }, }, }, { .id = 0x24, .name = "vdemcer", .swgroup = TEGRA_SWGROUP_VDE, - .smmu = { - .reg = 0x22c, - .bit = 4, - }, - .la = { - .reg = 0x358, - .shift = 0, - .mask = 0xff, - .def = 0xb8, + .regs = { + .smmu = { + .reg = 0x22c, + .bit = 4, + }, + .la = { + .reg = 0x358, + .shift = 0, + .mask = 0xff, + .def = 0xb8, + }, }, }, { .id = 0x25, .name = "vdetper", .swgroup = TEGRA_SWGROUP_VDE, - .smmu = { - .reg = 0x22c, - .bit = 5, - }, - .la = { - .reg = 0x358, - .shift = 16, - .mask = 0xff, - .def = 0xee, + .regs = { + .smmu = { + .reg = 0x22c, + .bit = 5, + }, + .la = { + .reg = 0x358, + .shift = 16, + .mask = 0xff, + .def = 0xee, + }, }, }, { .id = 0x26, .name = "mpcorelpr", .swgroup = TEGRA_SWGROUP_MPCORELP, - .la = { - .reg = 0x324, - .shift = 0, - .mask = 0xff, - .def = 0x04, + .regs = { + .la = { + .reg = 0x324, + .shift = 0, + .mask = 0xff, + .def = 0x04, + }, }, }, { .id = 0x27, .name = "mpcorer", .swgroup = TEGRA_SWGROUP_MPCORE, - .la = { - .reg = 0x320, - .shift = 0, - .mask = 0xff, - .def = 0x04, + .regs = { + .la = { + .reg = 0x320, + .shift = 0, + .mask = 0xff, + .def = 0x04, + }, }, }, { .id = 0x28, .name = "eppu", .swgroup = TEGRA_SWGROUP_EPP, - .smmu = { - .reg = 0x22c, - .bit = 8, - }, - .la = { - .reg = 0x300, - .shift = 16, - .mask = 0xff, - .def = 0x33, + .regs = { + .smmu = { + .reg = 0x22c, + .bit = 8, + }, + .la = { + .reg = 0x300, + .shift = 16, + .mask = 0xff, + .def = 0x33, + }, }, }, { .id = 0x29, .name = "eppv", .swgroup = TEGRA_SWGROUP_EPP, - .smmu = { - .reg = 0x22c, - .bit = 9, - }, - .la = { - .reg = 0x304, - .shift = 0, - .mask = 0xff, - .def = 0x6c, + .regs = { + .smmu = { + .reg = 0x22c, + .bit = 9, + }, + .la = { + .reg = 0x304, + .shift = 0, + .mask = 0xff, + .def = 0x6c, + }, }, }, { .id = 0x2a, .name = "eppy", .swgroup = TEGRA_SWGROUP_EPP, - .smmu = { - .reg = 0x22c, - .bit = 10, - }, - .la = { - .reg = 0x304, - .shift = 16, - .mask = 0xff, - .def = 0x6c, + .regs = { + .smmu = { + .reg = 0x22c, + .bit = 10, + }, + .la = { + .reg = 0x304, + .shift = 16, + .mask = 0xff, + .def = 0x6c, + }, }, }, { .id = 0x2b, .name = "msencswr", .swgroup = TEGRA_SWGROUP_MSENC, - .smmu = { - .reg = 0x22c, - .bit = 11, - }, - .la = { - .reg = 0x328, - .shift = 16, - .mask = 0xff, - .def = 0x80, + .regs = { + .smmu = { + .reg = 0x22c, + .bit = 11, + }, + .la = { + .reg = 0x328, + .shift = 16, + .mask = 0xff, + .def = 0x80, + }, }, }, { .id = 0x2c, .name = "viwsb", .swgroup = TEGRA_SWGROUP_VI, - .smmu = { - .reg = 0x22c, - .bit = 12, - }, - .la = { - .reg = 0x364, - .shift = 0, - .mask = 0xff, - .def = 0x47, + .regs = { + .smmu = { + .reg = 0x22c, + .bit = 12, + }, + .la = { + .reg = 0x364, + .shift = 0, + .mask = 0xff, + .def = 0x47, + }, }, }, { .id = 0x2d, .name = "viwu", .swgroup = TEGRA_SWGROUP_VI, - .smmu = { - .reg = 0x22c, - .bit = 13, - }, - .la = { - .reg = 0x368, - .shift = 0, - .mask = 0xff, - .def = 0xff, + .regs = { + .smmu = { + .reg = 0x22c, + .bit = 13, + }, + .la = { + .reg = 0x368, + .shift = 0, + .mask = 0xff, + .def = 0xff, + }, }, }, { .id = 0x2e, .name = "viwv", .swgroup = TEGRA_SWGROUP_VI, - .smmu = { - .reg = 0x22c, - .bit = 14, - }, - .la = { - .reg = 0x368, - .shift = 16, - .mask = 0xff, - .def = 0xff, + .regs = { + .smmu = { + .reg = 0x22c, + .bit = 14, + }, + .la = { + .reg = 0x368, + .shift = 16, + .mask = 0xff, + .def = 0xff, + }, }, }, { .id = 0x2f, .name = "viwy", .swgroup = TEGRA_SWGROUP_VI, - .smmu = { - .reg = 0x22c, - .bit = 15, - }, - .la = { - .reg = 0x36c, - .shift = 0, - .mask = 0xff, - .def = 0x47, + .regs = { + .smmu = { + .reg = 0x22c, + .bit = 15, + }, + .la = { + .reg = 0x36c, + .shift = 0, + .mask = 0xff, + .def = 0x47, + }, }, }, { .id = 0x30, .name = "g2dw", .swgroup = TEGRA_SWGROUP_G2, - .smmu = { - .reg = 0x22c, - .bit = 16, - }, - .la = { - .reg = 0x30c, - .shift = 16, - .mask = 0xff, - .def = 0x9, + .regs = { + .smmu = { + .reg = 0x22c, + .bit = 16, + }, + .la = { + .reg = 0x30c, + .shift = 16, + .mask = 0xff, + .def = 0x9, + }, }, }, { .id = 0x32, .name = "avpcarm7w", .swgroup = TEGRA_SWGROUP_AVPC, - .smmu = { - .reg = 0x22c, - .bit = 18, - }, - .la = { - .reg = 0x2e4, - .shift = 16, - .mask = 0xff, - .def = 0x0e, + .regs = { + .smmu = { + .reg = 0x22c, + .bit = 18, + }, + .la = { + .reg = 0x2e4, + .shift = 16, + .mask = 0xff, + .def = 0x0e, + }, }, }, { .id = 0x33, .name = "fdcdwr", .swgroup = TEGRA_SWGROUP_NV, - .smmu = { - .reg = 0x22c, - .bit = 19, - }, - .la = { - .reg = 0x338, - .shift = 16, - .mask = 0xff, - .def = 0x10, + .regs = { + .smmu = { + .reg = 0x22c, + .bit = 19, + }, + .la = { + .reg = 0x338, + .shift = 16, + .mask = 0xff, + .def = 0x10, + }, }, }, { .id = 0x34, .name = "fdcdwr2", .swgroup = TEGRA_SWGROUP_NV, - .smmu = { - .reg = 0x22c, - .bit = 20, - }, - .la = { - .reg = 0x340, - .shift = 0, - .mask = 0xff, - .def = 0x10, + .regs = { + .smmu = { + .reg = 0x22c, + .bit = 20, + }, + .la = { + .reg = 0x340, + .shift = 0, + .mask = 0xff, + .def = 0x10, + }, }, }, { .id = 0x35, .name = "hdaw", .swgroup = TEGRA_SWGROUP_HDA, - .smmu = { - .reg = 0x22c, - .bit = 21, - }, - .la = { - .reg = 0x318, - .shift = 16, - .mask = 0xff, - .def = 0xff, + .regs = { + .smmu = { + .reg = 0x22c, + .bit = 21, + }, + .la = { + .reg = 0x318, + .shift = 16, + .mask = 0xff, + .def = 0xff, + }, }, }, { .id = 0x36, .name = "host1xw", .swgroup = TEGRA_SWGROUP_HC, - .smmu = { - .reg = 0x22c, - .bit = 22, - }, - .la = { - .reg = 0x314, - .shift = 0, - .mask = 0xff, - .def = 0x25, + .regs = { + .smmu = { + .reg = 0x22c, + .bit = 22, + }, + .la = { + .reg = 0x314, + .shift = 0, + .mask = 0xff, + .def = 0x25, + }, }, }, { .id = 0x37, .name = "ispw", .swgroup = TEGRA_SWGROUP_ISP, - .smmu = { - .reg = 0x22c, - .bit = 23, - }, - .la = { - .reg = 0x31c, - .shift = 0, - .mask = 0xff, - .def = 0xff, + .regs = { + .smmu = { + .reg = 0x22c, + .bit = 23, + }, + .la = { + .reg = 0x31c, + .shift = 0, + .mask = 0xff, + .def = 0xff, + }, }, }, { .id = 0x38, .name = "mpcorelpw", .swgroup = TEGRA_SWGROUP_MPCORELP, - .la = { - .reg = 0x324, - .shift = 16, - .mask = 0xff, - .def = 0x80, + .regs = { + .la = { + .reg = 0x324, + .shift = 16, + .mask = 0xff, + .def = 0x80, + }, }, }, { .id = 0x39, .name = "mpcorew", .swgroup = TEGRA_SWGROUP_MPCORE, - .la = { - .reg = 0x320, - .shift = 16, - .mask = 0xff, - .def = 0x0e, + .regs = { + .la = { + .reg = 0x320, + .shift = 16, + .mask = 0xff, + .def = 0x0e, + }, }, }, { .id = 0x3b, .name = "ppcsahbdmaw", .swgroup = TEGRA_SWGROUP_PPCS, - .smmu = { - .reg = 0x22c, - .bit = 27, - }, - .la = { - .reg = 0x348, - .shift = 0, - .mask = 0xff, - .def = 0xa5, + .regs = { + .smmu = { + .reg = 0x22c, + .bit = 27, + }, + .la = { + .reg = 0x348, + .shift = 0, + .mask = 0xff, + .def = 0xa5, + }, }, }, { .id = 0x3c, .name = "ppcsahbslvw", .swgroup = TEGRA_SWGROUP_PPCS, - .smmu = { - .reg = 0x22c, - .bit = 28, - }, - .la = { - .reg = 0x348, - .shift = 16, - .mask = 0xff, - .def = 0xe8, + .regs = { + .smmu = { + .reg = 0x22c, + .bit = 28, + }, + .la = { + .reg = 0x348, + .shift = 16, + .mask = 0xff, + .def = 0xe8, + }, }, }, { .id = 0x3e, .name = "vdebsevw", .swgroup = TEGRA_SWGROUP_VDE, - .smmu = { - .reg = 0x22c, - .bit = 30, - }, - .la = { - .reg = 0x35c, - .shift = 0, - .mask = 0xff, - .def = 0xff, + .regs = { + .smmu = { + .reg = 0x22c, + .bit = 30, + }, + .la = { + .reg = 0x35c, + .shift = 0, + .mask = 0xff, + .def = 0xff, + }, }, }, { .id = 0x3f, .name = "vdedbgw", .swgroup = TEGRA_SWGROUP_VDE, - .smmu = { - .reg = 0x22c, - .bit = 31, - }, - .la = { - .reg = 0x35c, - .shift = 16, - .mask = 0xff, - .def = 0xff, + .regs = { + .smmu = { + .reg = 0x22c, + .bit = 31, + }, + .la = { + .reg = 0x35c, + .shift = 16, + .mask = 0xff, + .def = 0xff, + }, }, }, { .id = 0x40, .name = "vdembew", .swgroup = TEGRA_SWGROUP_VDE, - .smmu = { - .reg = 0x230, - .bit = 0, - }, - .la = { - .reg = 0x360, - .shift = 0, - .mask = 0xff, - .def = 0x89, + .regs = { + .smmu = { + .reg = 0x230, + .bit = 0, + }, + .la = { + .reg = 0x360, + .shift = 0, + .mask = 0xff, + .def = 0x89, + }, }, }, { .id = 0x41, .name = "vdetpmw", .swgroup = TEGRA_SWGROUP_VDE, - .smmu = { - .reg = 0x230, - .bit = 1, - }, - .la = { - .reg = 0x360, - .shift = 16, - .mask = 0xff, - .def = 0x59, + .regs = { + .smmu = { + .reg = 0x230, + .bit = 1, + }, + .la = { + .reg = 0x360, + .shift = 16, + .mask = 0xff, + .def = 0x59, + }, }, }, { .id = 0x4a, .name = "xusb_hostr", .swgroup = TEGRA_SWGROUP_XUSB_HOST, - .smmu = { - .reg = 0x230, - .bit = 10, - }, - .la = { - .reg = 0x37c, - .shift = 0, - .mask = 0xff, - .def = 0xa5, + .regs = { + .smmu = { + .reg = 0x230, + .bit = 10, + }, + .la = { + .reg = 0x37c, + .shift = 0, + .mask = 0xff, + .def = 0xa5, + }, }, }, { .id = 0x4b, .name = "xusb_hostw", .swgroup = TEGRA_SWGROUP_XUSB_HOST, - .smmu = { - .reg = 0x230, - .bit = 11, - }, - .la = { - .reg = 0x37c, - .shift = 16, - .mask = 0xff, - .def = 0xa5, + .regs = { + .smmu = { + .reg = 0x230, + .bit = 11, + }, + .la = { + .reg = 0x37c, + .shift = 16, + .mask = 0xff, + .def = 0xa5, + }, }, }, { .id = 0x4c, .name = "xusb_devr", .swgroup = TEGRA_SWGROUP_XUSB_DEV, - .smmu = { - .reg = 0x230, - .bit = 12, - }, - .la = { - .reg = 0x380, - .shift = 0, - .mask = 0xff, - .def = 0xa5, + .regs = { + .smmu = { + .reg = 0x230, + .bit = 12, + }, + .la = { + .reg = 0x380, + .shift = 0, + .mask = 0xff, + .def = 0xa5, + }, }, }, { .id = 0x4d, .name = "xusb_devw", .swgroup = TEGRA_SWGROUP_XUSB_DEV, - .smmu = { - .reg = 0x230, - .bit = 13, - }, - .la = { - .reg = 0x380, - .shift = 16, - .mask = 0xff, - .def = 0xa5, + .regs = { + .smmu = { + .reg = 0x230, + .bit = 13, + }, + .la = { + .reg = 0x380, + .shift = 16, + .mask = 0xff, + .def = 0xa5, + }, }, }, { .id = 0x4e, .name = "fdcdwr3", .swgroup = TEGRA_SWGROUP_NV, - .smmu = { - .reg = 0x230, - .bit = 14, - }, - .la = { - .reg = 0x388, - .shift = 0, - .mask = 0xff, - .def = 0x10, + .regs = { + .smmu = { + .reg = 0x230, + .bit = 14, + }, + .la = { + .reg = 0x388, + .shift = 0, + .mask = 0xff, + .def = 0x10, + }, }, }, { .id = 0x4f, .name = "fdcdrd3", .swgroup = TEGRA_SWGROUP_NV, - .smmu = { - .reg = 0x230, - .bit = 15, - }, - .la = { - .reg = 0x384, - .shift = 0, - .mask = 0xff, - .def = 0x0c, + .regs = { + .smmu = { + .reg = 0x230, + .bit = 15, + }, + .la = { + .reg = 0x384, + .shift = 0, + .mask = 0xff, + .def = 0x0c, + }, }, }, { .id = 0x50, .name = "fdcwr4", .swgroup = TEGRA_SWGROUP_NV, - .smmu = { - .reg = 0x230, - .bit = 16, - }, - .la = { - .reg = 0x388, - .shift = 16, - .mask = 0xff, - .def = 0x10, + .regs = { + .smmu = { + .reg = 0x230, + .bit = 16, + }, + .la = { + .reg = 0x388, + .shift = 16, + .mask = 0xff, + .def = 0x10, + }, }, }, { .id = 0x51, .name = "fdcrd4", .swgroup = TEGRA_SWGROUP_NV, - .smmu = { - .reg = 0x230, - .bit = 17, - }, - .la = { - .reg = 0x384, - .shift = 16, - .mask = 0xff, - .def = 0x0c, + .regs = { + .smmu = { + .reg = 0x230, + .bit = 17, + }, + .la = { + .reg = 0x384, + .shift = 16, + .mask = 0xff, + .def = 0x0c, + }, }, }, { .id = 0x52, .name = "emucifr", .swgroup = TEGRA_SWGROUP_EMUCIF, - .la = { - .reg = 0x38c, - .shift = 0, - .mask = 0xff, - .def = 0x04, + .regs = { + .la = { + .reg = 0x38c, + .shift = 0, + .mask = 0xff, + .def = 0x04, + }, }, }, { .id = 0x53, .name = "emucifw", .swgroup = TEGRA_SWGROUP_EMUCIF, - .la = { - .reg = 0x38c, - .shift = 16, - .mask = 0xff, - .def = 0x0e, + .regs = { + .la = { + .reg = 0x38c, + .shift = 16, + .mask = 0xff, + .def = 0x0e, + }, }, }, { .id = 0x54, .name = "tsecsrd", .swgroup = TEGRA_SWGROUP_TSEC, - .smmu = { - .reg = 0x230, - .bit = 20, - }, - .la = { - .reg = 0x390, - .shift = 0, - .mask = 0xff, - .def = 0x50, + .regs = { + .smmu = { + .reg = 0x230, + .bit = 20, + }, + .la = { + .reg = 0x390, + .shift = 0, + .mask = 0xff, + .def = 0x50, + }, }, }, { .id = 0x55, .name = "tsecswr", .swgroup = TEGRA_SWGROUP_TSEC, - .smmu = { - .reg = 0x230, - .bit = 21, - }, - .la = { - .reg = 0x390, - .shift = 16, - .mask = 0xff, - .def = 0x50, + .regs = { + .smmu = { + .reg = 0x230, + .bit = 21, + }, + .la = { + .reg = 0x390, + .shift = 16, + .mask = 0xff, + .def = 0x50, + }, }, }, }; @@ -977,4 +1113,5 @@ const struct tegra_mc_soc tegra114_mc_soc = { .reset_ops = &tegra_mc_reset_ops_common, .resets = tegra114_mc_resets, .num_resets = ARRAY_SIZE(tegra114_mc_resets), + .ops = &tegra30_mc_ops, }; diff --git a/drivers/memory/tegra/tegra124-emc.c b/drivers/memory/tegra/tegra124-emc.c index 21f05240682b..85bc936c02f9 100644 --- a/drivers/memory/tegra/tegra124-emc.c +++ b/drivers/memory/tegra/tegra124-emc.c @@ -9,22 +9,29 @@ #include <linux/clk-provider.h> #include <linux/clk.h> #include <linux/clkdev.h> +#include <linux/clk/tegra.h> #include <linux/debugfs.h> #include <linux/delay.h> +#include <linux/interconnect-provider.h> #include <linux/io.h> +#include <linux/module.h> +#include <linux/mutex.h> #include <linux/of_address.h> #include <linux/of_platform.h> #include <linux/platform_device.h> +#include <linux/pm_opp.h> #include <linux/sort.h> #include <linux/string.h> -#include <soc/tegra/emc.h> #include <soc/tegra/fuse.h> #include <soc/tegra/mc.h> +#include "mc.h" + #define EMC_FBIO_CFG5 0x104 #define EMC_FBIO_CFG5_DRAM_TYPE_MASK 0x3 #define EMC_FBIO_CFG5_DRAM_TYPE_SHIFT 0 +#define EMC_FBIO_CFG5_DRAM_WIDTH_X64 BIT(4) #define EMC_INTSTATUS 0x0 #define EMC_INTSTATUS_CLKCHANGE_COMPLETE BIT(4) @@ -265,8 +272,8 @@ #define EMC_PUTERM_ADJ 0x574 #define DRAM_DEV_SEL_ALL 0 -#define DRAM_DEV_SEL_0 (2 << 30) -#define DRAM_DEV_SEL_1 (1 << 30) +#define DRAM_DEV_SEL_0 BIT(31) +#define DRAM_DEV_SEL_1 BIT(30) #define EMC_CFG_POWER_FEATURES_MASK \ (EMC_CFG_DYN_SREF | EMC_CFG_DRAM_ACPD | EMC_CFG_DRAM_CLKSTOP_SR | \ @@ -460,6 +467,17 @@ struct emc_timing { u32 emc_zcal_interval; }; +enum emc_rate_request_type { + EMC_RATE_DEBUG, + EMC_RATE_ICC, + EMC_RATE_TYPE_MAX, +}; + +struct emc_rate_request { + unsigned long min_rate; + unsigned long max_rate; +}; + struct tegra_emc { struct device *dev; @@ -470,6 +488,7 @@ struct tegra_emc { struct clk *clk; enum emc_dram_type dram_type; + unsigned int dram_bus_width; unsigned int dram_num; struct emc_timing last_timing; @@ -481,6 +500,17 @@ struct tegra_emc { unsigned long min_rate; unsigned long max_rate; } debugfs; + + struct icc_provider provider; + + /* + * There are multiple sources in the EMC driver which could request + * a min/max clock rate, these rates are contained in this array. + */ + struct emc_rate_request requested_rate[EMC_RATE_TYPE_MAX]; + + /* protect shared rate-change code path */ + struct mutex rate_lock; }; /* Timing change sequence functions */ @@ -562,8 +592,8 @@ static struct emc_timing *tegra_emc_find_timing(struct tegra_emc *emc, return timing; } -int tegra_emc_prepare_timing_change(struct tegra_emc *emc, - unsigned long rate) +static int tegra_emc_prepare_timing_change(struct tegra_emc *emc, + unsigned long rate) { struct emc_timing *timing = tegra_emc_find_timing(emc, rate); struct emc_timing *last = &emc->last_timing; @@ -790,8 +820,8 @@ int tegra_emc_prepare_timing_change(struct tegra_emc *emc, return 0; } -void tegra_emc_complete_timing_change(struct tegra_emc *emc, - unsigned long rate) +static void tegra_emc_complete_timing_change(struct tegra_emc *emc, + unsigned long rate) { struct emc_timing *timing = tegra_emc_find_timing(emc, rate); struct emc_timing *last = &emc->last_timing; @@ -869,6 +899,14 @@ static void emc_read_current_timing(struct tegra_emc *emc, static int emc_init(struct tegra_emc *emc) { emc->dram_type = readl(emc->regs + EMC_FBIO_CFG5); + + if (emc->dram_type & EMC_FBIO_CFG5_DRAM_WIDTH_X64) + emc->dram_bus_width = 64; + else + emc->dram_bus_width = 32; + + dev_info_once(emc->dev, "%ubit DRAM bus\n", emc->dram_bus_width); + emc->dram_type &= EMC_FBIO_CFG5_DRAM_TYPE_MASK; emc->dram_type >>= EMC_FBIO_CFG5_DRAM_TYPE_SHIFT; @@ -984,8 +1022,10 @@ static int tegra_emc_load_timings_from_dt(struct tegra_emc *emc, static const struct of_device_id tegra_emc_of_match[] = { { .compatible = "nvidia,tegra124-emc" }, + { .compatible = "nvidia,tegra132-emc" }, {} }; +MODULE_DEVICE_TABLE(of, tegra_emc_of_match); static struct device_node * tegra_emc_find_node_by_ram_code(struct device_node *node, u32 ram_code) @@ -1006,6 +1046,83 @@ tegra_emc_find_node_by_ram_code(struct device_node *node, u32 ram_code) return NULL; } +static void tegra_emc_rate_requests_init(struct tegra_emc *emc) +{ + unsigned int i; + + for (i = 0; i < EMC_RATE_TYPE_MAX; i++) { + emc->requested_rate[i].min_rate = 0; + emc->requested_rate[i].max_rate = ULONG_MAX; + } +} + +static int emc_request_rate(struct tegra_emc *emc, + unsigned long new_min_rate, + unsigned long new_max_rate, + enum emc_rate_request_type type) +{ + struct emc_rate_request *req = emc->requested_rate; + unsigned long min_rate = 0, max_rate = ULONG_MAX; + unsigned int i; + int err; + + /* select minimum and maximum rates among the requested rates */ + for (i = 0; i < EMC_RATE_TYPE_MAX; i++, req++) { + if (i == type) { + min_rate = max(new_min_rate, min_rate); + max_rate = min(new_max_rate, max_rate); + } else { + min_rate = max(req->min_rate, min_rate); + max_rate = min(req->max_rate, max_rate); + } + } + + if (min_rate > max_rate) { + dev_err_ratelimited(emc->dev, "%s: type %u: out of range: %lu %lu\n", + __func__, type, min_rate, max_rate); + return -ERANGE; + } + + /* + * EMC rate-changes should go via OPP API because it manages voltage + * changes. + */ + err = dev_pm_opp_set_rate(emc->dev, min_rate); + if (err) + return err; + + emc->requested_rate[type].min_rate = new_min_rate; + emc->requested_rate[type].max_rate = new_max_rate; + + return 0; +} + +static int emc_set_min_rate(struct tegra_emc *emc, unsigned long rate, + enum emc_rate_request_type type) +{ + struct emc_rate_request *req = &emc->requested_rate[type]; + int ret; + + mutex_lock(&emc->rate_lock); + ret = emc_request_rate(emc, rate, req->max_rate, type); + mutex_unlock(&emc->rate_lock); + + return ret; +} + +static int emc_set_max_rate(struct tegra_emc *emc, unsigned long rate, + enum emc_rate_request_type type) +{ + struct emc_rate_request *req = &emc->requested_rate[type]; + int ret; + + mutex_lock(&emc->rate_lock); + ret = emc_request_rate(emc, req->min_rate, rate, type); + mutex_unlock(&emc->rate_lock); + + return ret; +} + /* * debugfs interface * @@ -1059,19 +1176,7 @@ static int tegra_emc_debug_available_rates_show(struct seq_file *s, return 0; } -static int tegra_emc_debug_available_rates_open(struct inode *inode, - struct file *file) -{ - return single_open(file, tegra_emc_debug_available_rates_show, - inode->i_private); -} - -static const struct file_operations tegra_emc_debug_available_rates_fops = { - .open = tegra_emc_debug_available_rates_open, - .read = seq_read, - .llseek = seq_lseek, - .release = single_release, -}; +DEFINE_SHOW_ATTRIBUTE(tegra_emc_debug_available_rates); static int tegra_emc_debug_min_rate_get(void *data, u64 *rate) { @@ -1090,7 +1195,7 @@ static int tegra_emc_debug_min_rate_set(void *data, u64 rate) if (!tegra_emc_validate_rate(emc, rate)) return -EINVAL; - err = clk_set_min_rate(emc->clk, rate); + err = emc_set_min_rate(emc, rate, EMC_RATE_DEBUG); if (err < 0) return err; @@ -1099,7 +1204,7 @@ static int tegra_emc_debug_min_rate_set(void *data, u64 rate) return 0; } -DEFINE_SIMPLE_ATTRIBUTE(tegra_emc_debug_min_rate_fops, +DEFINE_DEBUGFS_ATTRIBUTE(tegra_emc_debug_min_rate_fops, tegra_emc_debug_min_rate_get, tegra_emc_debug_min_rate_set, "%llu\n"); @@ -1120,7 +1225,7 @@ static int tegra_emc_debug_max_rate_set(void *data, u64 rate) if (!tegra_emc_validate_rate(emc, rate)) return -EINVAL; - err = clk_set_max_rate(emc->clk, rate); + err = emc_set_max_rate(emc, rate, EMC_RATE_DEBUG); if (err < 0) return err; @@ -1129,7 +1234,7 @@ static int tegra_emc_debug_max_rate_set(void *data, u64 rate) return 0; } -DEFINE_SIMPLE_ATTRIBUTE(tegra_emc_debug_max_rate_fops, +DEFINE_DEBUGFS_ATTRIBUTE(tegra_emc_debug_max_rate_fops, tegra_emc_debug_max_rate_get, tegra_emc_debug_max_rate_set, "%llu\n"); @@ -1138,15 +1243,6 @@ static void emc_debugfs_init(struct device *dev, struct tegra_emc *emc) unsigned int i; int err; - emc->clk = devm_clk_get(dev, "emc"); - if (IS_ERR(emc->clk)) { - if (PTR_ERR(emc->clk) != -ENODEV) { - dev_err(dev, "failed to get EMC clock: %ld\n", - PTR_ERR(emc->clk)); - return; - } - } - emc->debugfs.min_rate = ULONG_MAX; emc->debugfs.max_rate = 0; @@ -1158,6 +1254,11 @@ static void emc_debugfs_init(struct device *dev, struct tegra_emc *emc) emc->debugfs.max_rate = emc->timings[i].rate; } + if (!emc->num_timings) { + emc->debugfs.min_rate = clk_get_rate(emc->clk); + emc->debugfs.max_rate = emc->debugfs.min_rate; + } + err = clk_set_rate_range(emc->clk, emc->debugfs.min_rate, emc->debugfs.max_rate); if (err < 0) { @@ -1168,25 +1269,180 @@ static void emc_debugfs_init(struct device *dev, struct tegra_emc *emc) } emc->debugfs.root = debugfs_create_dir("emc", NULL); - if (!emc->debugfs.root) { - dev_err(dev, "failed to create debugfs directory\n"); - return; - } - debugfs_create_file("available_rates", S_IRUGO, emc->debugfs.root, emc, + debugfs_create_file("available_rates", 0444, emc->debugfs.root, emc, &tegra_emc_debug_available_rates_fops); - debugfs_create_file("min_rate", S_IRUGO | S_IWUSR, emc->debugfs.root, + debugfs_create_file("min_rate", 0644, emc->debugfs.root, emc, &tegra_emc_debug_min_rate_fops); - debugfs_create_file("max_rate", S_IRUGO | S_IWUSR, emc->debugfs.root, + debugfs_create_file("max_rate", 0644, emc->debugfs.root, emc, &tegra_emc_debug_max_rate_fops); } +static inline struct tegra_emc * +to_tegra_emc_provider(struct icc_provider *provider) +{ + return container_of(provider, struct tegra_emc, provider); +} + +static struct icc_node_data * +emc_of_icc_xlate_extended(struct of_phandle_args *spec, void *data) +{ + struct icc_provider *provider = data; + struct icc_node_data *ndata; + struct icc_node *node; + + /* External Memory is the only possible ICC route */ + list_for_each_entry(node, &provider->nodes, node_list) { + if (node->id != TEGRA_ICC_EMEM) + continue; + + ndata = kzalloc(sizeof(*ndata), GFP_KERNEL); + if (!ndata) + return ERR_PTR(-ENOMEM); + + /* + * SRC and DST nodes should have matching TAG in order to have + * it set by default for a requested path. + */ + ndata->tag = TEGRA_MC_ICC_TAG_ISO; + ndata->node = node; + + return ndata; + } + + return ERR_PTR(-EPROBE_DEFER); +} + +static int emc_icc_set(struct icc_node *src, struct icc_node *dst) +{ + struct tegra_emc *emc = to_tegra_emc_provider(dst->provider); + unsigned long long peak_bw = icc_units_to_bps(dst->peak_bw); + unsigned long long avg_bw = icc_units_to_bps(dst->avg_bw); + unsigned long long rate = max(avg_bw, peak_bw); + unsigned int dram_data_bus_width_bytes; + const unsigned int ddr = 2; + int err; + + /* + * Tegra124 EMC runs on a clock rate of SDRAM bus. This means that + * EMC clock rate is twice smaller than the peak data rate because + * data is sampled on both EMC clock edges. + */ + dram_data_bus_width_bytes = emc->dram_bus_width / 8; + do_div(rate, ddr * dram_data_bus_width_bytes); + rate = min_t(u64, rate, U32_MAX); + + err = emc_set_min_rate(emc, rate, EMC_RATE_ICC); + if (err) + return err; + + return 0; +} + +static int tegra_emc_interconnect_init(struct tegra_emc *emc) +{ + const struct tegra_mc_soc *soc = emc->mc->soc; + struct icc_node *node; + int err; + + emc->provider.dev = emc->dev; + emc->provider.set = emc_icc_set; + emc->provider.data = &emc->provider; + emc->provider.aggregate = soc->icc_ops->aggregate; + emc->provider.xlate_extended = emc_of_icc_xlate_extended; + + err = icc_provider_add(&emc->provider); + if (err) + goto err_msg; + + /* create External Memory Controller node */ + node = icc_node_create(TEGRA_ICC_EMC); + if (IS_ERR(node)) { + err = PTR_ERR(node); + goto del_provider; + } + + node->name = "External Memory Controller"; + icc_node_add(node, &emc->provider); + + /* link External Memory Controller to External Memory (DRAM) */ + err = icc_link_create(node, TEGRA_ICC_EMEM); + if (err) + goto remove_nodes; + + /* create External Memory node */ + node = icc_node_create(TEGRA_ICC_EMEM); + if (IS_ERR(node)) { + err = PTR_ERR(node); + goto remove_nodes; + } + + node->name = "External Memory (DRAM)"; + icc_node_add(node, &emc->provider); + + return 0; + +remove_nodes: + icc_nodes_remove(&emc->provider); +del_provider: + icc_provider_del(&emc->provider); +err_msg: + dev_err(emc->dev, "failed to initialize ICC: %d\n", err); + + return err; +} + +static int tegra_emc_opp_table_init(struct tegra_emc *emc) +{ + u32 hw_version = BIT(tegra_sku_info.soc_speedo_id); + int opp_token, err; + + err = dev_pm_opp_set_supported_hw(emc->dev, &hw_version, 1); + if (err < 0) { + dev_err(emc->dev, "failed to set OPP supported HW: %d\n", err); + return err; + } + opp_token = err; + + err = dev_pm_opp_of_add_table(emc->dev); + if (err) { + if (err == -ENODEV) + dev_err(emc->dev, "OPP table not found, please update your device tree\n"); + else + dev_err(emc->dev, "failed to add OPP table: %d\n", err); + + goto put_hw_table; + } + + dev_info_once(emc->dev, "OPP HW ver. 0x%x, current clock rate %lu MHz\n", + hw_version, clk_get_rate(emc->clk) / 1000000); + + /* first dummy rate-set initializes voltage state */ + err = dev_pm_opp_set_rate(emc->dev, clk_get_rate(emc->clk)); + if (err) { + dev_err(emc->dev, "failed to initialize OPP clock: %d\n", err); + goto remove_table; + } + + return 0; + +remove_table: + dev_pm_opp_of_remove_table(emc->dev); +put_hw_table: + dev_pm_opp_put_supported_hw(opp_token); + + return err; +} + +static void devm_tegra_emc_unset_callback(void *data) +{ + tegra124_clk_set_emc_callbacks(NULL, NULL); +} + static int tegra_emc_probe(struct platform_device *pdev) { - struct platform_device *mc; struct device_node *np; struct tegra_emc *emc; - struct resource *res; u32 ram_code; int err; @@ -1194,48 +1450,29 @@ static int tegra_emc_probe(struct platform_device *pdev) if (!emc) return -ENOMEM; + mutex_init(&emc->rate_lock); emc->dev = &pdev->dev; - res = platform_get_resource(pdev, IORESOURCE_MEM, 0); - emc->regs = devm_ioremap_resource(&pdev->dev, res); + emc->regs = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(emc->regs)) return PTR_ERR(emc->regs); - np = of_parse_phandle(pdev->dev.of_node, "nvidia,memory-controller", 0); - if (!np) { - dev_err(&pdev->dev, "could not get memory controller\n"); - return -ENOENT; - } - - mc = of_find_device_by_node(np); - of_node_put(np); - if (!mc) - return -ENOENT; - - emc->mc = platform_get_drvdata(mc); - if (!emc->mc) - return -EPROBE_DEFER; + emc->mc = devm_tegra_memory_controller_get(&pdev->dev); + if (IS_ERR(emc->mc)) + return PTR_ERR(emc->mc); ram_code = tegra_read_ram_code(); np = tegra_emc_find_node_by_ram_code(pdev->dev.of_node, ram_code); - if (!np) { - dev_err(&pdev->dev, - "no memory timings for RAM code %u found in DT\n", - ram_code); - return -ENOENT; - } - - err = tegra_emc_load_timings_from_dt(emc, np); - of_node_put(np); - if (err) - return err; - - if (emc->num_timings == 0) { - dev_err(&pdev->dev, - "no memory timings for RAM code %u registered\n", - ram_code); - return -ENOENT; + if (np) { + err = tegra_emc_load_timings_from_dt(emc, np); + of_node_put(np); + if (err) + return err; + } else { + dev_info_once(&pdev->dev, + "no memory timings for RAM code %u found in DT\n", + ram_code); } err = emc_init(emc); @@ -1246,9 +1483,39 @@ static int tegra_emc_probe(struct platform_device *pdev) platform_set_drvdata(pdev, emc); + tegra124_clk_set_emc_callbacks(tegra_emc_prepare_timing_change, + tegra_emc_complete_timing_change); + + err = devm_add_action_or_reset(&pdev->dev, devm_tegra_emc_unset_callback, + NULL); + if (err) + return err; + + emc->clk = devm_clk_get(&pdev->dev, "emc"); + if (IS_ERR(emc->clk)) { + err = PTR_ERR(emc->clk); + dev_err(&pdev->dev, "failed to get EMC clock: %d\n", err); + return err; + } + + err = tegra_emc_opp_table_init(emc); + if (err) + return err; + + tegra_emc_rate_requests_init(emc); + if (IS_ENABLED(CONFIG_DEBUG_FS)) emc_debugfs_init(&pdev->dev, emc); + tegra_emc_interconnect_init(emc); + + /* + * Don't allow the kernel module to be unloaded. Unloading adds some + * extra complexity which doesn't really worth the effort in a case of + * this driver. + */ + try_module_get(THIS_MODULE); + return 0; }; @@ -1258,11 +1525,11 @@ static struct platform_driver tegra_emc_driver = { .name = "tegra-emc", .of_match_table = tegra_emc_of_match, .suppress_bind_attrs = true, + .sync_state = icc_sync_state, }, }; +module_platform_driver(tegra_emc_driver); -static int tegra_emc_init(void) -{ - return platform_driver_register(&tegra_emc_driver); -} -subsys_initcall(tegra_emc_init); +MODULE_AUTHOR("Mikko Perttunen <mperttunen@nvidia.com>"); +MODULE_DESCRIPTION("NVIDIA Tegra124 EMC driver"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/memory/tegra/tegra124.c b/drivers/memory/tegra/tegra124.c index 493b5dc3a4b3..d780a84241fe 100644 --- a/drivers/memory/tegra/tegra124.c +++ b/drivers/memory/tegra/tegra124.c @@ -4,7 +4,8 @@ */ #include <linux/of.h> -#include <linux/mm.h> +#include <linux/of_device.h> +#include <linux/slab.h> #include <dt-bindings/memory/tegra124-mc.h> @@ -15,915 +16,1055 @@ static const struct tegra_mc_client tegra124_mc_clients[] = { .id = 0x00, .name = "ptcr", .swgroup = TEGRA_SWGROUP_PTC, + .regs = { + .la = { + .reg = 0x34c, + .shift = 0, + .mask = 0xff, + .def = 0x0, + }, + }, }, { .id = 0x01, .name = "display0a", .swgroup = TEGRA_SWGROUP_DC, - .smmu = { - .reg = 0x228, - .bit = 1, - }, - .la = { - .reg = 0x2e8, - .shift = 0, - .mask = 0xff, - .def = 0xc2, + .regs = { + .smmu = { + .reg = 0x228, + .bit = 1, + }, + .la = { + .reg = 0x2e8, + .shift = 0, + .mask = 0xff, + .def = 0xc2, + }, }, }, { .id = 0x02, .name = "display0ab", .swgroup = TEGRA_SWGROUP_DCB, - .smmu = { - .reg = 0x228, - .bit = 2, - }, - .la = { - .reg = 0x2f4, - .shift = 0, - .mask = 0xff, - .def = 0xc6, + .regs = { + .smmu = { + .reg = 0x228, + .bit = 2, + }, + .la = { + .reg = 0x2f4, + .shift = 0, + .mask = 0xff, + .def = 0xc6, + }, }, }, { .id = 0x03, .name = "display0b", .swgroup = TEGRA_SWGROUP_DC, - .smmu = { - .reg = 0x228, - .bit = 3, - }, - .la = { - .reg = 0x2e8, - .shift = 16, - .mask = 0xff, - .def = 0x50, + .regs = { + .smmu = { + .reg = 0x228, + .bit = 3, + }, + .la = { + .reg = 0x2e8, + .shift = 16, + .mask = 0xff, + .def = 0x50, + }, }, }, { .id = 0x04, .name = "display0bb", .swgroup = TEGRA_SWGROUP_DCB, - .smmu = { - .reg = 0x228, - .bit = 4, - }, - .la = { - .reg = 0x2f4, - .shift = 16, - .mask = 0xff, - .def = 0x50, + .regs = { + .smmu = { + .reg = 0x228, + .bit = 4, + }, + .la = { + .reg = 0x2f4, + .shift = 16, + .mask = 0xff, + .def = 0x50, + }, }, }, { .id = 0x05, .name = "display0c", .swgroup = TEGRA_SWGROUP_DC, - .smmu = { - .reg = 0x228, - .bit = 5, - }, - .la = { - .reg = 0x2ec, - .shift = 0, - .mask = 0xff, - .def = 0x50, + .regs = { + .smmu = { + .reg = 0x228, + .bit = 5, + }, + .la = { + .reg = 0x2ec, + .shift = 0, + .mask = 0xff, + .def = 0x50, + }, }, }, { .id = 0x06, .name = "display0cb", .swgroup = TEGRA_SWGROUP_DCB, - .smmu = { - .reg = 0x228, - .bit = 6, - }, - .la = { - .reg = 0x2f8, - .shift = 0, - .mask = 0xff, - .def = 0x50, + .regs = { + .smmu = { + .reg = 0x228, + .bit = 6, + }, + .la = { + .reg = 0x2f8, + .shift = 0, + .mask = 0xff, + .def = 0x50, + }, }, }, { .id = 0x0e, .name = "afir", .swgroup = TEGRA_SWGROUP_AFI, - .smmu = { - .reg = 0x228, - .bit = 14, - }, - .la = { - .reg = 0x2e0, - .shift = 0, - .mask = 0xff, - .def = 0x13, + .regs = { + .smmu = { + .reg = 0x228, + .bit = 14, + }, + .la = { + .reg = 0x2e0, + .shift = 0, + .mask = 0xff, + .def = 0x13, + }, }, }, { .id = 0x0f, .name = "avpcarm7r", .swgroup = TEGRA_SWGROUP_AVPC, - .smmu = { - .reg = 0x228, - .bit = 15, - }, - .la = { - .reg = 0x2e4, - .shift = 0, - .mask = 0xff, - .def = 0x04, + .regs = { + .smmu = { + .reg = 0x228, + .bit = 15, + }, + .la = { + .reg = 0x2e4, + .shift = 0, + .mask = 0xff, + .def = 0x04, + }, }, }, { .id = 0x10, .name = "displayhc", .swgroup = TEGRA_SWGROUP_DC, - .smmu = { - .reg = 0x228, - .bit = 16, - }, - .la = { - .reg = 0x2f0, - .shift = 0, - .mask = 0xff, - .def = 0x50, + .regs = { + .smmu = { + .reg = 0x228, + .bit = 16, + }, + .la = { + .reg = 0x2f0, + .shift = 0, + .mask = 0xff, + .def = 0x50, + }, }, }, { .id = 0x11, .name = "displayhcb", .swgroup = TEGRA_SWGROUP_DCB, - .smmu = { - .reg = 0x228, - .bit = 17, - }, - .la = { - .reg = 0x2fc, - .shift = 0, - .mask = 0xff, - .def = 0x50, + .regs = { + .smmu = { + .reg = 0x228, + .bit = 17, + }, + .la = { + .reg = 0x2fc, + .shift = 0, + .mask = 0xff, + .def = 0x50, + }, }, }, { .id = 0x15, .name = "hdar", .swgroup = TEGRA_SWGROUP_HDA, - .smmu = { - .reg = 0x228, - .bit = 21, - }, - .la = { - .reg = 0x318, - .shift = 0, - .mask = 0xff, - .def = 0x24, + .regs = { + .smmu = { + .reg = 0x228, + .bit = 21, + }, + .la = { + .reg = 0x318, + .shift = 0, + .mask = 0xff, + .def = 0x24, + }, }, }, { .id = 0x16, .name = "host1xdmar", .swgroup = TEGRA_SWGROUP_HC, - .smmu = { - .reg = 0x228, - .bit = 22, - }, - .la = { - .reg = 0x310, - .shift = 0, - .mask = 0xff, - .def = 0x1e, + .regs = { + .smmu = { + .reg = 0x228, + .bit = 22, + }, + .la = { + .reg = 0x310, + .shift = 0, + .mask = 0xff, + .def = 0x1e, + }, }, }, { .id = 0x17, .name = "host1xr", .swgroup = TEGRA_SWGROUP_HC, - .smmu = { - .reg = 0x228, - .bit = 23, - }, - .la = { - .reg = 0x310, - .shift = 16, - .mask = 0xff, - .def = 0x50, + .regs = { + .smmu = { + .reg = 0x228, + .bit = 23, + }, + .la = { + .reg = 0x310, + .shift = 16, + .mask = 0xff, + .def = 0x50, + }, }, }, { .id = 0x1c, .name = "msencsrd", .swgroup = TEGRA_SWGROUP_MSENC, - .smmu = { - .reg = 0x228, - .bit = 28, - }, - .la = { - .reg = 0x328, - .shift = 0, - .mask = 0xff, - .def = 0x23, + .regs = { + .smmu = { + .reg = 0x228, + .bit = 28, + }, + .la = { + .reg = 0x328, + .shift = 0, + .mask = 0xff, + .def = 0x23, + }, }, }, { .id = 0x1d, .name = "ppcsahbdmar", .swgroup = TEGRA_SWGROUP_PPCS, - .smmu = { - .reg = 0x228, - .bit = 29, - }, - .la = { - .reg = 0x344, - .shift = 0, - .mask = 0xff, - .def = 0x49, + .regs = { + .smmu = { + .reg = 0x228, + .bit = 29, + }, + .la = { + .reg = 0x344, + .shift = 0, + .mask = 0xff, + .def = 0x49, + }, }, }, { .id = 0x1e, .name = "ppcsahbslvr", .swgroup = TEGRA_SWGROUP_PPCS, - .smmu = { - .reg = 0x228, - .bit = 30, - }, - .la = { - .reg = 0x344, - .shift = 16, - .mask = 0xff, - .def = 0x1a, + .regs = { + .smmu = { + .reg = 0x228, + .bit = 30, + }, + .la = { + .reg = 0x344, + .shift = 16, + .mask = 0xff, + .def = 0x1a, + }, }, }, { .id = 0x1f, .name = "satar", .swgroup = TEGRA_SWGROUP_SATA, - .smmu = { - .reg = 0x228, - .bit = 31, - }, - .la = { - .reg = 0x350, - .shift = 0, - .mask = 0xff, - .def = 0x65, + .regs = { + .smmu = { + .reg = 0x228, + .bit = 31, + }, + .la = { + .reg = 0x350, + .shift = 0, + .mask = 0xff, + .def = 0x65, + }, }, }, { .id = 0x22, .name = "vdebsevr", .swgroup = TEGRA_SWGROUP_VDE, - .smmu = { - .reg = 0x22c, - .bit = 2, - }, - .la = { - .reg = 0x354, - .shift = 0, - .mask = 0xff, - .def = 0x4f, + .regs = { + .smmu = { + .reg = 0x22c, + .bit = 2, + }, + .la = { + .reg = 0x354, + .shift = 0, + .mask = 0xff, + .def = 0x4f, + }, }, }, { .id = 0x23, .name = "vdember", .swgroup = TEGRA_SWGROUP_VDE, - .smmu = { - .reg = 0x22c, - .bit = 3, - }, - .la = { - .reg = 0x354, - .shift = 16, - .mask = 0xff, - .def = 0x3d, + .regs = { + .smmu = { + .reg = 0x22c, + .bit = 3, + }, + .la = { + .reg = 0x354, + .shift = 16, + .mask = 0xff, + .def = 0x3d, + }, }, }, { .id = 0x24, .name = "vdemcer", .swgroup = TEGRA_SWGROUP_VDE, - .smmu = { - .reg = 0x22c, - .bit = 4, - }, - .la = { - .reg = 0x358, - .shift = 0, - .mask = 0xff, - .def = 0x66, + .regs = { + .smmu = { + .reg = 0x22c, + .bit = 4, + }, + .la = { + .reg = 0x358, + .shift = 0, + .mask = 0xff, + .def = 0x66, + }, }, }, { .id = 0x25, .name = "vdetper", .swgroup = TEGRA_SWGROUP_VDE, - .smmu = { - .reg = 0x22c, - .bit = 5, - }, - .la = { - .reg = 0x358, - .shift = 16, - .mask = 0xff, - .def = 0xa5, + .regs = { + .smmu = { + .reg = 0x22c, + .bit = 5, + }, + .la = { + .reg = 0x358, + .shift = 16, + .mask = 0xff, + .def = 0xa5, + }, }, }, { .id = 0x26, .name = "mpcorelpr", .swgroup = TEGRA_SWGROUP_MPCORELP, - .la = { - .reg = 0x324, - .shift = 0, - .mask = 0xff, - .def = 0x04, + .regs = { + .la = { + .reg = 0x324, + .shift = 0, + .mask = 0xff, + .def = 0x04, + }, }, }, { .id = 0x27, .name = "mpcorer", .swgroup = TEGRA_SWGROUP_MPCORE, - .la = { - .reg = 0x320, - .shift = 0, - .mask = 0xff, - .def = 0x04, + .regs = { + .la = { + .reg = 0x320, + .shift = 0, + .mask = 0xff, + .def = 0x04, + }, }, }, { .id = 0x2b, .name = "msencswr", .swgroup = TEGRA_SWGROUP_MSENC, - .smmu = { - .reg = 0x22c, - .bit = 11, - }, - .la = { - .reg = 0x328, - .shift = 16, - .mask = 0xff, - .def = 0x80, + .regs = { + .smmu = { + .reg = 0x22c, + .bit = 11, + }, + .la = { + .reg = 0x328, + .shift = 16, + .mask = 0xff, + .def = 0x80, + }, }, }, { .id = 0x31, .name = "afiw", .swgroup = TEGRA_SWGROUP_AFI, - .smmu = { - .reg = 0x22c, - .bit = 17, - }, - .la = { - .reg = 0x2e0, - .shift = 16, - .mask = 0xff, - .def = 0x80, + .regs = { + .smmu = { + .reg = 0x22c, + .bit = 17, + }, + .la = { + .reg = 0x2e0, + .shift = 16, + .mask = 0xff, + .def = 0x80, + }, }, }, { .id = 0x32, .name = "avpcarm7w", .swgroup = TEGRA_SWGROUP_AVPC, - .smmu = { - .reg = 0x22c, - .bit = 18, - }, - .la = { - .reg = 0x2e4, - .shift = 16, - .mask = 0xff, - .def = 0x80, + .regs = { + .smmu = { + .reg = 0x22c, + .bit = 18, + }, + .la = { + .reg = 0x2e4, + .shift = 16, + .mask = 0xff, + .def = 0x80, + }, }, }, { .id = 0x35, .name = "hdaw", .swgroup = TEGRA_SWGROUP_HDA, - .smmu = { - .reg = 0x22c, - .bit = 21, - }, - .la = { - .reg = 0x318, - .shift = 16, - .mask = 0xff, - .def = 0x80, + .regs = { + .smmu = { + .reg = 0x22c, + .bit = 21, + }, + .la = { + .reg = 0x318, + .shift = 16, + .mask = 0xff, + .def = 0x80, + }, }, }, { .id = 0x36, .name = "host1xw", .swgroup = TEGRA_SWGROUP_HC, - .smmu = { - .reg = 0x22c, - .bit = 22, - }, - .la = { - .reg = 0x314, - .shift = 0, - .mask = 0xff, - .def = 0x80, + .regs = { + .smmu = { + .reg = 0x22c, + .bit = 22, + }, + .la = { + .reg = 0x314, + .shift = 0, + .mask = 0xff, + .def = 0x80, + }, }, }, { .id = 0x38, .name = "mpcorelpw", .swgroup = TEGRA_SWGROUP_MPCORELP, - .la = { - .reg = 0x324, - .shift = 16, - .mask = 0xff, - .def = 0x80, + .regs = { + .la = { + .reg = 0x324, + .shift = 16, + .mask = 0xff, + .def = 0x80, + }, }, }, { .id = 0x39, .name = "mpcorew", .swgroup = TEGRA_SWGROUP_MPCORE, - .la = { - .reg = 0x320, - .shift = 16, - .mask = 0xff, - .def = 0x80, + .regs = { + .la = { + .reg = 0x320, + .shift = 16, + .mask = 0xff, + .def = 0x80, + }, }, }, { .id = 0x3b, .name = "ppcsahbdmaw", .swgroup = TEGRA_SWGROUP_PPCS, - .smmu = { - .reg = 0x22c, - .bit = 27, - }, - .la = { - .reg = 0x348, - .shift = 0, - .mask = 0xff, - .def = 0x80, + .regs = { + .smmu = { + .reg = 0x22c, + .bit = 27, + }, + .la = { + .reg = 0x348, + .shift = 0, + .mask = 0xff, + .def = 0x80, + }, }, }, { .id = 0x3c, .name = "ppcsahbslvw", .swgroup = TEGRA_SWGROUP_PPCS, - .smmu = { - .reg = 0x22c, - .bit = 28, - }, - .la = { - .reg = 0x348, - .shift = 16, - .mask = 0xff, - .def = 0x80, + .regs = { + .smmu = { + .reg = 0x22c, + .bit = 28, + }, + .la = { + .reg = 0x348, + .shift = 16, + .mask = 0xff, + .def = 0x80, + }, }, }, { .id = 0x3d, .name = "sataw", .swgroup = TEGRA_SWGROUP_SATA, - .smmu = { - .reg = 0x22c, - .bit = 29, - }, - .la = { - .reg = 0x350, - .shift = 16, - .mask = 0xff, - .def = 0x65, + .regs = { + .smmu = { + .reg = 0x22c, + .bit = 29, + }, + .la = { + .reg = 0x350, + .shift = 16, + .mask = 0xff, + .def = 0x65, + }, }, }, { .id = 0x3e, .name = "vdebsevw", .swgroup = TEGRA_SWGROUP_VDE, - .smmu = { - .reg = 0x22c, - .bit = 30, - }, - .la = { - .reg = 0x35c, - .shift = 0, - .mask = 0xff, - .def = 0x80, + .regs = { + .smmu = { + .reg = 0x22c, + .bit = 30, + }, + .la = { + .reg = 0x35c, + .shift = 0, + .mask = 0xff, + .def = 0x80, + }, }, }, { .id = 0x3f, .name = "vdedbgw", .swgroup = TEGRA_SWGROUP_VDE, - .smmu = { - .reg = 0x22c, - .bit = 31, - }, - .la = { - .reg = 0x35c, - .shift = 16, - .mask = 0xff, - .def = 0x80, + .regs = { + .smmu = { + .reg = 0x22c, + .bit = 31, + }, + .la = { + .reg = 0x35c, + .shift = 16, + .mask = 0xff, + .def = 0x80, + }, }, }, { .id = 0x40, .name = "vdembew", .swgroup = TEGRA_SWGROUP_VDE, - .smmu = { - .reg = 0x230, - .bit = 0, - }, - .la = { - .reg = 0x360, - .shift = 0, - .mask = 0xff, - .def = 0x80, + .regs = { + .smmu = { + .reg = 0x230, + .bit = 0, + }, + .la = { + .reg = 0x360, + .shift = 0, + .mask = 0xff, + .def = 0x80, + }, }, }, { .id = 0x41, .name = "vdetpmw", .swgroup = TEGRA_SWGROUP_VDE, - .smmu = { - .reg = 0x230, - .bit = 1, - }, - .la = { - .reg = 0x360, - .shift = 16, - .mask = 0xff, - .def = 0x80, + .regs = { + .smmu = { + .reg = 0x230, + .bit = 1, + }, + .la = { + .reg = 0x360, + .shift = 16, + .mask = 0xff, + .def = 0x80, + }, }, }, { .id = 0x44, .name = "ispra", .swgroup = TEGRA_SWGROUP_ISP2, - .smmu = { - .reg = 0x230, - .bit = 4, - }, - .la = { - .reg = 0x370, - .shift = 0, - .mask = 0xff, - .def = 0x18, + .regs = { + .smmu = { + .reg = 0x230, + .bit = 4, + }, + .la = { + .reg = 0x370, + .shift = 0, + .mask = 0xff, + .def = 0x18, + }, }, }, { .id = 0x46, .name = "ispwa", .swgroup = TEGRA_SWGROUP_ISP2, - .smmu = { - .reg = 0x230, - .bit = 6, - }, - .la = { - .reg = 0x374, - .shift = 0, - .mask = 0xff, - .def = 0x80, + .regs = { + .smmu = { + .reg = 0x230, + .bit = 6, + }, + .la = { + .reg = 0x374, + .shift = 0, + .mask = 0xff, + .def = 0x80, + }, }, }, { .id = 0x47, .name = "ispwb", .swgroup = TEGRA_SWGROUP_ISP2, - .smmu = { - .reg = 0x230, - .bit = 7, - }, - .la = { - .reg = 0x374, - .shift = 16, - .mask = 0xff, - .def = 0x80, + .regs = { + .smmu = { + .reg = 0x230, + .bit = 7, + }, + .la = { + .reg = 0x374, + .shift = 16, + .mask = 0xff, + .def = 0x80, + }, }, }, { .id = 0x4a, .name = "xusb_hostr", .swgroup = TEGRA_SWGROUP_XUSB_HOST, - .smmu = { - .reg = 0x230, - .bit = 10, - }, - .la = { - .reg = 0x37c, - .shift = 0, - .mask = 0xff, - .def = 0x39, + .regs = { + .smmu = { + .reg = 0x230, + .bit = 10, + }, + .la = { + .reg = 0x37c, + .shift = 0, + .mask = 0xff, + .def = 0x39, + }, }, }, { .id = 0x4b, .name = "xusb_hostw", .swgroup = TEGRA_SWGROUP_XUSB_HOST, - .smmu = { - .reg = 0x230, - .bit = 11, - }, - .la = { - .reg = 0x37c, - .shift = 16, - .mask = 0xff, - .def = 0x80, + .regs = { + .smmu = { + .reg = 0x230, + .bit = 11, + }, + .la = { + .reg = 0x37c, + .shift = 16, + .mask = 0xff, + .def = 0x80, + }, }, }, { .id = 0x4c, .name = "xusb_devr", .swgroup = TEGRA_SWGROUP_XUSB_DEV, - .smmu = { - .reg = 0x230, - .bit = 12, - }, - .la = { - .reg = 0x380, - .shift = 0, - .mask = 0xff, - .def = 0x39, + .regs = { + .smmu = { + .reg = 0x230, + .bit = 12, + }, + .la = { + .reg = 0x380, + .shift = 0, + .mask = 0xff, + .def = 0x39, + }, }, }, { .id = 0x4d, .name = "xusb_devw", .swgroup = TEGRA_SWGROUP_XUSB_DEV, - .smmu = { - .reg = 0x230, - .bit = 13, - }, - .la = { - .reg = 0x380, - .shift = 16, - .mask = 0xff, - .def = 0x80, + .regs = { + .smmu = { + .reg = 0x230, + .bit = 13, + }, + .la = { + .reg = 0x380, + .shift = 16, + .mask = 0xff, + .def = 0x80, + }, }, }, { .id = 0x4e, .name = "isprab", .swgroup = TEGRA_SWGROUP_ISP2B, - .smmu = { - .reg = 0x230, - .bit = 14, - }, - .la = { - .reg = 0x384, - .shift = 0, - .mask = 0xff, - .def = 0x18, + .regs = { + .smmu = { + .reg = 0x230, + .bit = 14, + }, + .la = { + .reg = 0x384, + .shift = 0, + .mask = 0xff, + .def = 0x18, + }, }, }, { .id = 0x50, .name = "ispwab", .swgroup = TEGRA_SWGROUP_ISP2B, - .smmu = { - .reg = 0x230, - .bit = 16, - }, - .la = { - .reg = 0x388, - .shift = 0, - .mask = 0xff, - .def = 0x80, + .regs = { + .smmu = { + .reg = 0x230, + .bit = 16, + }, + .la = { + .reg = 0x388, + .shift = 0, + .mask = 0xff, + .def = 0x80, + }, }, }, { .id = 0x51, .name = "ispwbb", .swgroup = TEGRA_SWGROUP_ISP2B, - .smmu = { - .reg = 0x230, - .bit = 17, - }, - .la = { - .reg = 0x388, - .shift = 16, - .mask = 0xff, - .def = 0x80, + .regs = { + .smmu = { + .reg = 0x230, + .bit = 17, + }, + .la = { + .reg = 0x388, + .shift = 16, + .mask = 0xff, + .def = 0x80, + }, }, }, { .id = 0x54, .name = "tsecsrd", .swgroup = TEGRA_SWGROUP_TSEC, - .smmu = { - .reg = 0x230, - .bit = 20, - }, - .la = { - .reg = 0x390, - .shift = 0, - .mask = 0xff, - .def = 0x9b, + .regs = { + .smmu = { + .reg = 0x230, + .bit = 20, + }, + .la = { + .reg = 0x390, + .shift = 0, + .mask = 0xff, + .def = 0x9b, + }, }, }, { .id = 0x55, .name = "tsecswr", .swgroup = TEGRA_SWGROUP_TSEC, - .smmu = { - .reg = 0x230, - .bit = 21, - }, - .la = { - .reg = 0x390, - .shift = 16, - .mask = 0xff, - .def = 0x80, + .regs = { + .smmu = { + .reg = 0x230, + .bit = 21, + }, + .la = { + .reg = 0x390, + .shift = 16, + .mask = 0xff, + .def = 0x80, + }, }, }, { .id = 0x56, .name = "a9avpscr", .swgroup = TEGRA_SWGROUP_A9AVP, - .smmu = { - .reg = 0x230, - .bit = 22, - }, - .la = { - .reg = 0x3a4, - .shift = 0, - .mask = 0xff, - .def = 0x04, + .regs = { + .smmu = { + .reg = 0x230, + .bit = 22, + }, + .la = { + .reg = 0x3a4, + .shift = 0, + .mask = 0xff, + .def = 0x04, + }, }, }, { .id = 0x57, .name = "a9avpscw", .swgroup = TEGRA_SWGROUP_A9AVP, - .smmu = { - .reg = 0x230, - .bit = 23, - }, - .la = { - .reg = 0x3a4, - .shift = 16, - .mask = 0xff, - .def = 0x80, + .regs = { + .smmu = { + .reg = 0x230, + .bit = 23, + }, + .la = { + .reg = 0x3a4, + .shift = 16, + .mask = 0xff, + .def = 0x80, + }, }, }, { .id = 0x58, .name = "gpusrd", .swgroup = TEGRA_SWGROUP_GPU, - .smmu = { - /* read-only */ - .reg = 0x230, - .bit = 24, - }, - .la = { - .reg = 0x3c8, - .shift = 0, - .mask = 0xff, - .def = 0x1a, + .regs = { + .smmu = { + /* read-only */ + .reg = 0x230, + .bit = 24, + }, + .la = { + .reg = 0x3c8, + .shift = 0, + .mask = 0xff, + .def = 0x1a, + }, }, }, { .id = 0x59, .name = "gpuswr", .swgroup = TEGRA_SWGROUP_GPU, - .smmu = { - /* read-only */ - .reg = 0x230, - .bit = 25, - }, - .la = { - .reg = 0x3c8, - .shift = 16, - .mask = 0xff, - .def = 0x80, + .regs = { + .smmu = { + /* read-only */ + .reg = 0x230, + .bit = 25, + }, + .la = { + .reg = 0x3c8, + .shift = 16, + .mask = 0xff, + .def = 0x80, + }, }, }, { .id = 0x5a, .name = "displayt", .swgroup = TEGRA_SWGROUP_DC, - .smmu = { - .reg = 0x230, - .bit = 26, - }, - .la = { - .reg = 0x2f0, - .shift = 16, - .mask = 0xff, - .def = 0x50, + .regs = { + .smmu = { + .reg = 0x230, + .bit = 26, + }, + .la = { + .reg = 0x2f0, + .shift = 16, + .mask = 0xff, + .def = 0x50, + }, }, }, { .id = 0x60, .name = "sdmmcra", .swgroup = TEGRA_SWGROUP_SDMMC1A, - .smmu = { - .reg = 0x234, - .bit = 0, - }, - .la = { - .reg = 0x3b8, - .shift = 0, - .mask = 0xff, - .def = 0x49, + .regs = { + .smmu = { + .reg = 0x234, + .bit = 0, + }, + .la = { + .reg = 0x3b8, + .shift = 0, + .mask = 0xff, + .def = 0x49, + }, }, }, { .id = 0x61, .name = "sdmmcraa", .swgroup = TEGRA_SWGROUP_SDMMC2A, - .smmu = { - .reg = 0x234, - .bit = 1, - }, - .la = { - .reg = 0x3bc, - .shift = 0, - .mask = 0xff, - .def = 0x49, + .regs = { + .smmu = { + .reg = 0x234, + .bit = 1, + }, + .la = { + .reg = 0x3bc, + .shift = 0, + .mask = 0xff, + .def = 0x49, + }, }, }, { .id = 0x62, .name = "sdmmcr", .swgroup = TEGRA_SWGROUP_SDMMC3A, - .smmu = { - .reg = 0x234, - .bit = 2, - }, - .la = { - .reg = 0x3c0, - .shift = 0, - .mask = 0xff, - .def = 0x49, + .regs = { + .smmu = { + .reg = 0x234, + .bit = 2, + }, + .la = { + .reg = 0x3c0, + .shift = 0, + .mask = 0xff, + .def = 0x49, + }, }, }, { .id = 0x63, .swgroup = TEGRA_SWGROUP_SDMMC4A, .name = "sdmmcrab", - .smmu = { - .reg = 0x234, - .bit = 3, - }, - .la = { - .reg = 0x3c4, - .shift = 0, - .mask = 0xff, - .def = 0x49, + .regs = { + .smmu = { + .reg = 0x234, + .bit = 3, + }, + .la = { + .reg = 0x3c4, + .shift = 0, + .mask = 0xff, + .def = 0x49, + }, }, }, { .id = 0x64, .name = "sdmmcwa", .swgroup = TEGRA_SWGROUP_SDMMC1A, - .smmu = { - .reg = 0x234, - .bit = 4, - }, - .la = { - .reg = 0x3b8, - .shift = 16, - .mask = 0xff, - .def = 0x80, + .regs = { + .smmu = { + .reg = 0x234, + .bit = 4, + }, + .la = { + .reg = 0x3b8, + .shift = 16, + .mask = 0xff, + .def = 0x80, + }, }, }, { .id = 0x65, .name = "sdmmcwaa", .swgroup = TEGRA_SWGROUP_SDMMC2A, - .smmu = { - .reg = 0x234, - .bit = 5, - }, - .la = { - .reg = 0x3bc, - .shift = 16, - .mask = 0xff, - .def = 0x80, + .regs = { + .smmu = { + .reg = 0x234, + .bit = 5, + }, + .la = { + .reg = 0x3bc, + .shift = 16, + .mask = 0xff, + .def = 0x80, + }, }, }, { .id = 0x66, .name = "sdmmcw", .swgroup = TEGRA_SWGROUP_SDMMC3A, - .smmu = { - .reg = 0x234, - .bit = 6, - }, - .la = { - .reg = 0x3c0, - .shift = 16, - .mask = 0xff, - .def = 0x80, + .regs = { + .smmu = { + .reg = 0x234, + .bit = 6, + }, + .la = { + .reg = 0x3c0, + .shift = 16, + .mask = 0xff, + .def = 0x80, + }, }, }, { .id = 0x67, .name = "sdmmcwab", .swgroup = TEGRA_SWGROUP_SDMMC4A, - .smmu = { - .reg = 0x234, - .bit = 7, - }, - .la = { - .reg = 0x3c4, - .shift = 16, - .mask = 0xff, - .def = 0x80, + .regs = { + .smmu = { + .reg = 0x234, + .bit = 7, + }, + .la = { + .reg = 0x3c4, + .shift = 16, + .mask = 0xff, + .def = 0x80, + }, }, }, { .id = 0x6c, .name = "vicsrd", .swgroup = TEGRA_SWGROUP_VIC, - .smmu = { - .reg = 0x234, - .bit = 12, - }, - .la = { - .reg = 0x394, - .shift = 0, - .mask = 0xff, - .def = 0x1a, + .regs = { + .smmu = { + .reg = 0x234, + .bit = 12, + }, + .la = { + .reg = 0x394, + .shift = 0, + .mask = 0xff, + .def = 0x1a, + }, }, }, { .id = 0x6d, .name = "vicswr", .swgroup = TEGRA_SWGROUP_VIC, - .smmu = { - .reg = 0x234, - .bit = 13, - }, - .la = { - .reg = 0x394, - .shift = 16, - .mask = 0xff, - .def = 0x80, + .regs = { + .smmu = { + .reg = 0x234, + .bit = 13, + }, + .la = { + .reg = 0x394, + .shift = 16, + .mask = 0xff, + .def = 0x80, + }, }, }, { .id = 0x72, .name = "viw", .swgroup = TEGRA_SWGROUP_VI, - .smmu = { - .reg = 0x234, - .bit = 18, - }, - .la = { - .reg = 0x398, - .shift = 0, - .mask = 0xff, - .def = 0x80, + .regs = { + .smmu = { + .reg = 0x234, + .bit = 18, + }, + .la = { + .reg = 0x398, + .shift = 0, + .mask = 0xff, + .def = 0x80, + }, }, }, { .id = 0x73, .name = "displayd", .swgroup = TEGRA_SWGROUP_DC, - .smmu = { - .reg = 0x234, - .bit = 19, - }, - .la = { - .reg = 0x3c8, - .shift = 0, - .mask = 0xff, - .def = 0x50, + .regs = { + .smmu = { + .reg = 0x234, + .bit = 19, + }, + .la = { + .reg = 0x3c8, + .shift = 0, + .mask = 0xff, + .def = 0x50, + }, }, }, }; @@ -957,7 +1098,6 @@ static const struct tegra_smmu_swgroup tegra124_swgroups[] = { static const unsigned int tegra124_group_drm[] = { TEGRA_SWGROUP_DC, TEGRA_SWGROUP_DCB, - TEGRA_SWGROUP_GPU, TEGRA_SWGROUP_VIC, }; @@ -1005,6 +1145,83 @@ static const struct tegra_mc_reset tegra124_mc_resets[] = { TEGRA124_MC_RESET(GPU, 0x970, 0x974, 2), }; +static int tegra124_mc_icc_set(struct icc_node *src, struct icc_node *dst) +{ + /* TODO: program PTSA */ + return 0; +} + +static int tegra124_mc_icc_aggreate(struct icc_node *node, u32 tag, u32 avg_bw, + u32 peak_bw, u32 *agg_avg, u32 *agg_peak) +{ + /* + * ISO clients need to reserve extra bandwidth up-front because + * there could be high bandwidth pressure during initial filling + * of the client's FIFO buffers. Secondly, we need to take into + * account impurities of the memory subsystem. + */ + if (tag & TEGRA_MC_ICC_TAG_ISO) + peak_bw = tegra_mc_scale_percents(peak_bw, 400); + + *agg_avg += avg_bw; + *agg_peak = max(*agg_peak, peak_bw); + + return 0; +} + +static struct icc_node_data * +tegra124_mc_of_icc_xlate_extended(struct of_phandle_args *spec, void *data) +{ + struct tegra_mc *mc = icc_provider_to_tegra_mc(data); + const struct tegra_mc_client *client; + unsigned int i, idx = spec->args[0]; + struct icc_node_data *ndata; + struct icc_node *node; + + list_for_each_entry(node, &mc->provider.nodes, node_list) { + if (node->id != idx) + continue; + + ndata = kzalloc(sizeof(*ndata), GFP_KERNEL); + if (!ndata) + return ERR_PTR(-ENOMEM); + + client = &mc->soc->clients[idx]; + ndata->node = node; + + switch (client->swgroup) { + case TEGRA_SWGROUP_DC: + case TEGRA_SWGROUP_DCB: + case TEGRA_SWGROUP_PTC: + case TEGRA_SWGROUP_VI: + /* these clients are isochronous by default */ + ndata->tag = TEGRA_MC_ICC_TAG_ISO; + break; + + default: + ndata->tag = TEGRA_MC_ICC_TAG_DEFAULT; + break; + } + + return ndata; + } + + for (i = 0; i < mc->soc->num_clients; i++) { + if (mc->soc->clients[i].id == idx) + return ERR_PTR(-EPROBE_DEFER); + } + + dev_err(mc->dev, "invalid ICC client ID %u\n", idx); + + return ERR_PTR(-EINVAL); +} + +static const struct tegra_mc_icc_ops tegra124_mc_icc_ops = { + .xlate_extended = tegra124_mc_of_icc_xlate_extended, + .aggregate = tegra124_mc_icc_aggreate, + .set = tegra124_mc_icc_set, +}; + #ifdef CONFIG_ARCH_TEGRA_124_SOC static const unsigned long tegra124_mc_emem_regs[] = { MC_EMEM_ARB_CFG, @@ -1056,6 +1273,8 @@ const struct tegra_mc_soc tegra124_mc_soc = { .reset_ops = &tegra_mc_reset_ops_common, .resets = tegra124_mc_resets, .num_resets = ARRAY_SIZE(tegra124_mc_resets), + .icc_ops = &tegra124_mc_icc_ops, + .ops = &tegra30_mc_ops, }; #endif /* CONFIG_ARCH_TEGRA_124_SOC */ @@ -1086,5 +1305,7 @@ const struct tegra_mc_soc tegra132_mc_soc = { .reset_ops = &tegra_mc_reset_ops_common, .resets = tegra124_mc_resets, .num_resets = ARRAY_SIZE(tegra124_mc_resets), + .icc_ops = &tegra124_mc_icc_ops, + .ops = &tegra30_mc_ops, }; #endif /* CONFIG_ARCH_TEGRA_132_SOC */ diff --git a/drivers/memory/tegra/tegra186-emc.c b/drivers/memory/tegra/tegra186-emc.c index 97f26bc77ad4..54b47ca33483 100644 --- a/drivers/memory/tegra/tegra186-emc.c +++ b/drivers/memory/tegra/tegra186-emc.c @@ -125,9 +125,9 @@ static int tegra186_emc_debug_min_rate_set(void *data, u64 rate) return 0; } -DEFINE_SIMPLE_ATTRIBUTE(tegra186_emc_debug_min_rate_fops, - tegra186_emc_debug_min_rate_get, - tegra186_emc_debug_min_rate_set, "%llu\n"); +DEFINE_DEBUGFS_ATTRIBUTE(tegra186_emc_debug_min_rate_fops, + tegra186_emc_debug_min_rate_get, + tegra186_emc_debug_min_rate_set, "%llu\n"); static int tegra186_emc_debug_max_rate_get(void *data, u64 *rate) { @@ -155,9 +155,9 @@ static int tegra186_emc_debug_max_rate_set(void *data, u64 rate) return 0; } -DEFINE_SIMPLE_ATTRIBUTE(tegra186_emc_debug_max_rate_fops, - tegra186_emc_debug_max_rate_get, - tegra186_emc_debug_max_rate_set, "%llu\n"); +DEFINE_DEBUGFS_ATTRIBUTE(tegra186_emc_debug_max_rate_fops, + tegra186_emc_debug_max_rate_get, + tegra186_emc_debug_max_rate_set, "%llu\n"); static int tegra186_emc_probe(struct platform_device *pdev) { @@ -172,20 +172,14 @@ static int tegra186_emc_probe(struct platform_device *pdev) return -ENOMEM; emc->bpmp = tegra_bpmp_get(&pdev->dev); - if (IS_ERR(emc->bpmp)) { - err = PTR_ERR(emc->bpmp); - - if (err != -EPROBE_DEFER) - dev_err(&pdev->dev, "failed to get BPMP: %d\n", err); - - return err; - } + if (IS_ERR(emc->bpmp)) + return dev_err_probe(&pdev->dev, PTR_ERR(emc->bpmp), "failed to get BPMP\n"); emc->clk = devm_clk_get(&pdev->dev, "emc"); if (IS_ERR(emc->clk)) { err = PTR_ERR(emc->clk); dev_err(&pdev->dev, "failed to get EMC clock: %d\n", err); - return err; + goto put_bpmp; } platform_set_drvdata(pdev, emc); @@ -201,7 +195,12 @@ static int tegra186_emc_probe(struct platform_device *pdev) err = tegra_bpmp_transfer(emc->bpmp, &msg); if (err < 0) { dev_err(&pdev->dev, "failed to EMC DVFS pairs: %d\n", err); - return err; + goto put_bpmp; + } + if (msg.rx.ret < 0) { + err = -EINVAL; + dev_err(&pdev->dev, "EMC DVFS MRQ failed: %d (BPMP error code)\n", msg.rx.ret); + goto put_bpmp; } emc->debugfs.min_rate = ULONG_MAX; @@ -211,8 +210,10 @@ static int tegra186_emc_probe(struct platform_device *pdev) emc->dvfs = devm_kmalloc_array(&pdev->dev, emc->num_dvfs, sizeof(*emc->dvfs), GFP_KERNEL); - if (!emc->dvfs) - return -ENOMEM; + if (!emc->dvfs) { + err = -ENOMEM; + goto put_bpmp; + } dev_dbg(&pdev->dev, "%u DVFS pairs:\n", emc->num_dvfs); @@ -237,15 +238,10 @@ static int tegra186_emc_probe(struct platform_device *pdev) "failed to set rate range [%lu-%lu] for %pC\n", emc->debugfs.min_rate, emc->debugfs.max_rate, emc->clk); - return err; + goto put_bpmp; } emc->debugfs.root = debugfs_create_dir("emc", NULL); - if (!emc->debugfs.root) { - dev_err(&pdev->dev, "failed to create debugfs directory\n"); - return 0; - } - debugfs_create_file("available_rates", S_IRUGO, emc->debugfs.root, emc, &tegra186_emc_debug_available_rates_fops); debugfs_create_file("min_rate", S_IRUGO | S_IWUSR, emc->debugfs.root, @@ -254,6 +250,10 @@ static int tegra186_emc_probe(struct platform_device *pdev) emc, &tegra186_emc_debug_max_rate_fops); return 0; + +put_bpmp: + tegra_bpmp_put(emc->bpmp); + return err; } static int tegra186_emc_remove(struct platform_device *pdev) @@ -267,12 +267,15 @@ static int tegra186_emc_remove(struct platform_device *pdev) } static const struct of_device_id tegra186_emc_of_match[] = { -#if defined(CONFIG_ARCH_TEGRA186_SOC) +#if defined(CONFIG_ARCH_TEGRA_186_SOC) { .compatible = "nvidia,tegra186-emc" }, #endif -#if defined(CONFIG_ARCH_TEGRA194_SOC) +#if defined(CONFIG_ARCH_TEGRA_194_SOC) { .compatible = "nvidia,tegra194-emc" }, #endif +#if defined(CONFIG_ARCH_TEGRA_234_SOC) + { .compatible = "nvidia,tegra234-emc" }, +#endif { /* sentinel */ } }; MODULE_DEVICE_TABLE(of, tegra186_emc_of_match); diff --git a/drivers/memory/tegra/tegra186.c b/drivers/memory/tegra/tegra186.c index 5d53f11ca7b6..62477e592bf5 100644 --- a/drivers/memory/tegra/tegra186.c +++ b/drivers/memory/tegra/tegra186.c @@ -1,1605 +1,919 @@ // SPDX-License-Identifier: GPL-2.0-only /* - * Copyright (C) 2017 NVIDIA CORPORATION. All rights reserved. + * Copyright (C) 2017-2021 NVIDIA CORPORATION. All rights reserved. */ #include <linux/io.h> +#include <linux/iommu.h> #include <linux/module.h> #include <linux/mod_devicetable.h> #include <linux/of_device.h> #include <linux/platform_device.h> +#include <soc/tegra/mc.h> + #if defined(CONFIG_ARCH_TEGRA_186_SOC) #include <dt-bindings/memory/tegra186-mc.h> #endif -#if defined(CONFIG_ARCH_TEGRA_194_SOC) -#include <dt-bindings/memory/tegra194-mc.h> -#endif +#include "mc.h" -struct tegra186_mc_client { - const char *name; - unsigned int sid; - struct { - unsigned int override; - unsigned int security; - } regs; -}; - -struct tegra186_mc_soc { - const struct tegra186_mc_client *clients; - unsigned int num_clients; -}; +#define MC_SID_STREAMID_OVERRIDE_MASK GENMASK(7, 0) +#define MC_SID_STREAMID_SECURITY_WRITE_ACCESS_DISABLED BIT(16) +#define MC_SID_STREAMID_SECURITY_OVERRIDE BIT(8) -struct tegra186_mc { - struct device *dev; - void __iomem *regs; - - const struct tegra186_mc_soc *soc; -}; - -static void tegra186_mc_program_sid(struct tegra186_mc *mc) +static void tegra186_mc_program_sid(struct tegra_mc *mc) { unsigned int i; for (i = 0; i < mc->soc->num_clients; i++) { - const struct tegra186_mc_client *client = &mc->soc->clients[i]; + const struct tegra_mc_client *client = &mc->soc->clients[i]; u32 override, security; - override = readl(mc->regs + client->regs.override); - security = readl(mc->regs + client->regs.security); + override = readl(mc->regs + client->regs.sid.override); + security = readl(mc->regs + client->regs.sid.security); dev_dbg(mc->dev, "client %s: override: %x security: %x\n", client->name, override, security); dev_dbg(mc->dev, "setting SID %u for %s\n", client->sid, client->name); - writel(client->sid, mc->regs + client->regs.override); + writel(client->sid, mc->regs + client->regs.sid.override); - override = readl(mc->regs + client->regs.override); - security = readl(mc->regs + client->regs.security); + override = readl(mc->regs + client->regs.sid.override); + security = readl(mc->regs + client->regs.sid.security); dev_dbg(mc->dev, "client %s: override: %x security: %x\n", client->name, override, security); } } +static int tegra186_mc_probe(struct tegra_mc *mc) +{ + struct platform_device *pdev = to_platform_device(mc->dev); + unsigned int i; + char name[8]; + int err; + + mc->bcast_ch_regs = devm_platform_ioremap_resource_byname(pdev, "broadcast"); + if (IS_ERR(mc->bcast_ch_regs)) { + if (PTR_ERR(mc->bcast_ch_regs) == -EINVAL) { + dev_warn(&pdev->dev, + "Broadcast channel is missing, please update your device-tree\n"); + mc->bcast_ch_regs = NULL; + goto populate; + } + + return PTR_ERR(mc->bcast_ch_regs); + } + + mc->ch_regs = devm_kcalloc(mc->dev, mc->soc->num_channels, sizeof(*mc->ch_regs), + GFP_KERNEL); + if (!mc->ch_regs) + return -ENOMEM; + + for (i = 0; i < mc->soc->num_channels; i++) { + snprintf(name, sizeof(name), "ch%u", i); + + mc->ch_regs[i] = devm_platform_ioremap_resource_byname(pdev, name); + if (IS_ERR(mc->ch_regs[i])) + return PTR_ERR(mc->ch_regs[i]); + } + +populate: + err = of_platform_populate(mc->dev->of_node, NULL, NULL, mc->dev); + if (err < 0) + return err; + + tegra186_mc_program_sid(mc); + + return 0; +} + +static void tegra186_mc_remove(struct tegra_mc *mc) +{ + of_platform_depopulate(mc->dev); +} + +static int tegra186_mc_resume(struct tegra_mc *mc) +{ + tegra186_mc_program_sid(mc); + + return 0; +} + +#if IS_ENABLED(CONFIG_IOMMU_API) +static void tegra186_mc_client_sid_override(struct tegra_mc *mc, + const struct tegra_mc_client *client, + unsigned int sid) +{ + u32 value, old; + + value = readl(mc->regs + client->regs.sid.security); + if ((value & MC_SID_STREAMID_SECURITY_OVERRIDE) == 0) { + /* + * If the secure firmware has locked this down the override + * for this memory client, there's nothing we can do here. + */ + if (value & MC_SID_STREAMID_SECURITY_WRITE_ACCESS_DISABLED) + return; + + /* + * Otherwise, try to set the override itself. Typically the + * secure firmware will never have set this configuration. + * Instead, it will either have disabled write access to + * this field, or it will already have set an explicit + * override itself. + */ + WARN_ON((value & MC_SID_STREAMID_SECURITY_OVERRIDE) == 0); + + value |= MC_SID_STREAMID_SECURITY_OVERRIDE; + writel(value, mc->regs + client->regs.sid.security); + } + + value = readl(mc->regs + client->regs.sid.override); + old = value & MC_SID_STREAMID_OVERRIDE_MASK; + + if (old != sid) { + dev_dbg(mc->dev, "overriding SID %x for %s with %x\n", old, + client->name, sid); + writel(sid, mc->regs + client->regs.sid.override); + } +} +#endif + +static int tegra186_mc_probe_device(struct tegra_mc *mc, struct device *dev) +{ +#if IS_ENABLED(CONFIG_IOMMU_API) + struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); + struct of_phandle_args args; + unsigned int i, index = 0; + + while (!of_parse_phandle_with_args(dev->of_node, "interconnects", "#interconnect-cells", + index, &args)) { + if (args.np == mc->dev->of_node && args.args_count != 0) { + for (i = 0; i < mc->soc->num_clients; i++) { + const struct tegra_mc_client *client = &mc->soc->clients[i]; + + if (client->id == args.args[0]) { + u32 sid = fwspec->ids[0] & MC_SID_STREAMID_OVERRIDE_MASK; + + tegra186_mc_client_sid_override(mc, client, sid); + } + } + } + + index++; + } +#endif + + return 0; +} + +const struct tegra_mc_ops tegra186_mc_ops = { + .probe = tegra186_mc_probe, + .remove = tegra186_mc_remove, + .resume = tegra186_mc_resume, + .probe_device = tegra186_mc_probe_device, + .handle_irq = tegra30_mc_handle_irq, +}; + #if defined(CONFIG_ARCH_TEGRA_186_SOC) -static const struct tegra186_mc_client tegra186_mc_clients[] = { +static const struct tegra_mc_client tegra186_mc_clients[] = { { + .id = TEGRA186_MEMORY_CLIENT_PTCR, .name = "ptcr", .sid = TEGRA186_SID_PASSTHROUGH, .regs = { - .override = 0x000, - .security = 0x004, + .sid = { + .override = 0x000, + .security = 0x004, + }, }, }, { + .id = TEGRA186_MEMORY_CLIENT_AFIR, .name = "afir", .sid = TEGRA186_SID_AFI, .regs = { - .override = 0x070, - .security = 0x074, + .sid = { + .override = 0x070, + .security = 0x074, + }, }, }, { + .id = TEGRA186_MEMORY_CLIENT_HDAR, .name = "hdar", .sid = TEGRA186_SID_HDA, .regs = { - .override = 0x0a8, - .security = 0x0ac, + .sid = { + .override = 0x0a8, + .security = 0x0ac, + }, }, }, { + .id = TEGRA186_MEMORY_CLIENT_HOST1XDMAR, .name = "host1xdmar", .sid = TEGRA186_SID_HOST1X, .regs = { - .override = 0x0b0, - .security = 0x0b4, + .sid = { + .override = 0x0b0, + .security = 0x0b4, + }, }, }, { + .id = TEGRA186_MEMORY_CLIENT_NVENCSRD, .name = "nvencsrd", .sid = TEGRA186_SID_NVENC, .regs = { - .override = 0x0e0, - .security = 0x0e4, + .sid = { + .override = 0x0e0, + .security = 0x0e4, + }, }, }, { + .id = TEGRA186_MEMORY_CLIENT_SATAR, .name = "satar", .sid = TEGRA186_SID_SATA, .regs = { - .override = 0x0f8, - .security = 0x0fc, + .sid = { + .override = 0x0f8, + .security = 0x0fc, + }, }, }, { + .id = TEGRA186_MEMORY_CLIENT_MPCORER, .name = "mpcorer", .sid = TEGRA186_SID_PASSTHROUGH, .regs = { - .override = 0x138, - .security = 0x13c, + .sid = { + .override = 0x138, + .security = 0x13c, + }, }, }, { + .id = TEGRA186_MEMORY_CLIENT_NVENCSWR, .name = "nvencswr", .sid = TEGRA186_SID_NVENC, .regs = { - .override = 0x158, - .security = 0x15c, + .sid = { + .override = 0x158, + .security = 0x15c, + }, }, }, { + .id = TEGRA186_MEMORY_CLIENT_AFIW, .name = "afiw", .sid = TEGRA186_SID_AFI, .regs = { - .override = 0x188, - .security = 0x18c, + .sid = { + .override = 0x188, + .security = 0x18c, + }, }, }, { + .id = TEGRA186_MEMORY_CLIENT_HDAW, .name = "hdaw", .sid = TEGRA186_SID_HDA, .regs = { - .override = 0x1a8, - .security = 0x1ac, + .sid = { + .override = 0x1a8, + .security = 0x1ac, + }, }, }, { + .id = TEGRA186_MEMORY_CLIENT_MPCOREW, .name = "mpcorew", .sid = TEGRA186_SID_PASSTHROUGH, .regs = { - .override = 0x1c8, - .security = 0x1cc, + .sid = { + .override = 0x1c8, + .security = 0x1cc, + }, }, }, { + .id = TEGRA186_MEMORY_CLIENT_SATAW, .name = "sataw", .sid = TEGRA186_SID_SATA, .regs = { - .override = 0x1e8, - .security = 0x1ec, + .sid = { + .override = 0x1e8, + .security = 0x1ec, + }, }, }, { + .id = TEGRA186_MEMORY_CLIENT_ISPRA, .name = "ispra", .sid = TEGRA186_SID_ISP, .regs = { - .override = 0x220, - .security = 0x224, + .sid = { + .override = 0x220, + .security = 0x224, + }, }, }, { + .id = TEGRA186_MEMORY_CLIENT_ISPWA, .name = "ispwa", .sid = TEGRA186_SID_ISP, .regs = { - .override = 0x230, - .security = 0x234, + .sid = { + .override = 0x230, + .security = 0x234, + }, }, }, { + .id = TEGRA186_MEMORY_CLIENT_ISPWB, .name = "ispwb", .sid = TEGRA186_SID_ISP, .regs = { - .override = 0x238, - .security = 0x23c, + .sid = { + .override = 0x238, + .security = 0x23c, + }, }, }, { + .id = TEGRA186_MEMORY_CLIENT_XUSB_HOSTR, .name = "xusb_hostr", .sid = TEGRA186_SID_XUSB_HOST, .regs = { - .override = 0x250, - .security = 0x254, + .sid = { + .override = 0x250, + .security = 0x254, + }, }, }, { + .id = TEGRA186_MEMORY_CLIENT_XUSB_HOSTW, .name = "xusb_hostw", .sid = TEGRA186_SID_XUSB_HOST, .regs = { - .override = 0x258, - .security = 0x25c, + .sid = { + .override = 0x258, + .security = 0x25c, + }, }, }, { + .id = TEGRA186_MEMORY_CLIENT_XUSB_DEVR, .name = "xusb_devr", .sid = TEGRA186_SID_XUSB_DEV, .regs = { - .override = 0x260, - .security = 0x264, + .sid = { + .override = 0x260, + .security = 0x264, + }, }, }, { + .id = TEGRA186_MEMORY_CLIENT_XUSB_DEVW, .name = "xusb_devw", .sid = TEGRA186_SID_XUSB_DEV, .regs = { - .override = 0x268, - .security = 0x26c, + .sid = { + .override = 0x268, + .security = 0x26c, + }, }, }, { + .id = TEGRA186_MEMORY_CLIENT_TSECSRD, .name = "tsecsrd", .sid = TEGRA186_SID_TSEC, .regs = { - .override = 0x2a0, - .security = 0x2a4, + .sid = { + .override = 0x2a0, + .security = 0x2a4, + }, }, }, { + .id = TEGRA186_MEMORY_CLIENT_TSECSWR, .name = "tsecswr", .sid = TEGRA186_SID_TSEC, .regs = { - .override = 0x2a8, - .security = 0x2ac, + .sid = { + .override = 0x2a8, + .security = 0x2ac, + }, }, }, { + .id = TEGRA186_MEMORY_CLIENT_GPUSRD, .name = "gpusrd", .sid = TEGRA186_SID_GPU, .regs = { - .override = 0x2c0, - .security = 0x2c4, + .sid = { + .override = 0x2c0, + .security = 0x2c4, + }, }, }, { + .id = TEGRA186_MEMORY_CLIENT_GPUSWR, .name = "gpuswr", .sid = TEGRA186_SID_GPU, .regs = { - .override = 0x2c8, - .security = 0x2cc, + .sid = { + .override = 0x2c8, + .security = 0x2cc, + }, }, }, { + .id = TEGRA186_MEMORY_CLIENT_SDMMCRA, .name = "sdmmcra", .sid = TEGRA186_SID_SDMMC1, .regs = { - .override = 0x300, - .security = 0x304, + .sid = { + .override = 0x300, + .security = 0x304, + }, }, }, { + .id = TEGRA186_MEMORY_CLIENT_SDMMCRAA, .name = "sdmmcraa", .sid = TEGRA186_SID_SDMMC2, .regs = { - .override = 0x308, - .security = 0x30c, + .sid = { + .override = 0x308, + .security = 0x30c, + }, }, }, { + .id = TEGRA186_MEMORY_CLIENT_SDMMCR, .name = "sdmmcr", .sid = TEGRA186_SID_SDMMC3, .regs = { - .override = 0x310, - .security = 0x314, + .sid = { + .override = 0x310, + .security = 0x314, + }, }, }, { + .id = TEGRA186_MEMORY_CLIENT_SDMMCRAB, .name = "sdmmcrab", .sid = TEGRA186_SID_SDMMC4, .regs = { - .override = 0x318, - .security = 0x31c, + .sid = { + .override = 0x318, + .security = 0x31c, + }, }, }, { + .id = TEGRA186_MEMORY_CLIENT_SDMMCWA, .name = "sdmmcwa", .sid = TEGRA186_SID_SDMMC1, .regs = { - .override = 0x320, - .security = 0x324, + .sid = { + .override = 0x320, + .security = 0x324, + }, }, }, { + .id = TEGRA186_MEMORY_CLIENT_SDMMCWAA, .name = "sdmmcwaa", .sid = TEGRA186_SID_SDMMC2, .regs = { - .override = 0x328, - .security = 0x32c, + .sid = { + .override = 0x328, + .security = 0x32c, + }, }, }, { + .id = TEGRA186_MEMORY_CLIENT_SDMMCW, .name = "sdmmcw", .sid = TEGRA186_SID_SDMMC3, .regs = { - .override = 0x330, - .security = 0x334, + .sid = { + .override = 0x330, + .security = 0x334, + }, }, }, { + .id = TEGRA186_MEMORY_CLIENT_SDMMCWAB, .name = "sdmmcwab", .sid = TEGRA186_SID_SDMMC4, .regs = { - .override = 0x338, - .security = 0x33c, + .sid = { + .override = 0x338, + .security = 0x33c, + }, }, }, { + .id = TEGRA186_MEMORY_CLIENT_VICSRD, .name = "vicsrd", .sid = TEGRA186_SID_VIC, .regs = { - .override = 0x360, - .security = 0x364, + .sid = { + .override = 0x360, + .security = 0x364, + }, }, }, { + .id = TEGRA186_MEMORY_CLIENT_VICSWR, .name = "vicswr", .sid = TEGRA186_SID_VIC, .regs = { - .override = 0x368, - .security = 0x36c, + .sid = { + .override = 0x368, + .security = 0x36c, + }, }, }, { + .id = TEGRA186_MEMORY_CLIENT_VIW, .name = "viw", .sid = TEGRA186_SID_VI, .regs = { - .override = 0x390, - .security = 0x394, + .sid = { + .override = 0x390, + .security = 0x394, + }, }, }, { + .id = TEGRA186_MEMORY_CLIENT_NVDECSRD, .name = "nvdecsrd", .sid = TEGRA186_SID_NVDEC, .regs = { - .override = 0x3c0, - .security = 0x3c4, + .sid = { + .override = 0x3c0, + .security = 0x3c4, + }, }, }, { + .id = TEGRA186_MEMORY_CLIENT_NVDECSWR, .name = "nvdecswr", .sid = TEGRA186_SID_NVDEC, .regs = { - .override = 0x3c8, - .security = 0x3cc, + .sid = { + .override = 0x3c8, + .security = 0x3cc, + }, }, }, { + .id = TEGRA186_MEMORY_CLIENT_APER, .name = "aper", .sid = TEGRA186_SID_APE, .regs = { - .override = 0x3d0, - .security = 0x3d4, + .sid = { + .override = 0x3d0, + .security = 0x3d4, + }, }, }, { + .id = TEGRA186_MEMORY_CLIENT_APEW, .name = "apew", .sid = TEGRA186_SID_APE, .regs = { - .override = 0x3d8, - .security = 0x3dc, + .sid = { + .override = 0x3d8, + .security = 0x3dc, + }, }, }, { + .id = TEGRA186_MEMORY_CLIENT_NVJPGSRD, .name = "nvjpgsrd", .sid = TEGRA186_SID_NVJPG, .regs = { - .override = 0x3f0, - .security = 0x3f4, + .sid = { + .override = 0x3f0, + .security = 0x3f4, + }, }, }, { + .id = TEGRA186_MEMORY_CLIENT_NVJPGSWR, .name = "nvjpgswr", .sid = TEGRA186_SID_NVJPG, .regs = { - .override = 0x3f8, - .security = 0x3fc, + .sid = { + .override = 0x3f8, + .security = 0x3fc, + }, }, }, { + .id = TEGRA186_MEMORY_CLIENT_SESRD, .name = "sesrd", .sid = TEGRA186_SID_SE, .regs = { - .override = 0x400, - .security = 0x404, + .sid = { + .override = 0x400, + .security = 0x404, + }, }, }, { + .id = TEGRA186_MEMORY_CLIENT_SESWR, .name = "seswr", .sid = TEGRA186_SID_SE, .regs = { - .override = 0x408, - .security = 0x40c, + .sid = { + .override = 0x408, + .security = 0x40c, + }, }, }, { + .id = TEGRA186_MEMORY_CLIENT_ETRR, .name = "etrr", .sid = TEGRA186_SID_ETR, .regs = { - .override = 0x420, - .security = 0x424, + .sid = { + .override = 0x420, + .security = 0x424, + }, }, }, { + .id = TEGRA186_MEMORY_CLIENT_ETRW, .name = "etrw", .sid = TEGRA186_SID_ETR, .regs = { - .override = 0x428, - .security = 0x42c, + .sid = { + .override = 0x428, + .security = 0x42c, + }, }, }, { + .id = TEGRA186_MEMORY_CLIENT_TSECSRDB, .name = "tsecsrdb", .sid = TEGRA186_SID_TSECB, .regs = { - .override = 0x430, - .security = 0x434, + .sid = { + .override = 0x430, + .security = 0x434, + }, }, }, { + .id = TEGRA186_MEMORY_CLIENT_TSECSWRB, .name = "tsecswrb", .sid = TEGRA186_SID_TSECB, .regs = { - .override = 0x438, - .security = 0x43c, + .sid = { + .override = 0x438, + .security = 0x43c, + }, }, }, { + .id = TEGRA186_MEMORY_CLIENT_GPUSRD2, .name = "gpusrd2", .sid = TEGRA186_SID_GPU, .regs = { - .override = 0x440, - .security = 0x444, + .sid = { + .override = 0x440, + .security = 0x444, + }, }, }, { + .id = TEGRA186_MEMORY_CLIENT_GPUSWR2, .name = "gpuswr2", .sid = TEGRA186_SID_GPU, .regs = { - .override = 0x448, - .security = 0x44c, + .sid = { + .override = 0x448, + .security = 0x44c, + }, }, }, { + .id = TEGRA186_MEMORY_CLIENT_AXISR, .name = "axisr", .sid = TEGRA186_SID_GPCDMA_0, .regs = { - .override = 0x460, - .security = 0x464, + .sid = { + .override = 0x460, + .security = 0x464, + }, }, }, { + .id = TEGRA186_MEMORY_CLIENT_AXISW, .name = "axisw", .sid = TEGRA186_SID_GPCDMA_0, .regs = { - .override = 0x468, - .security = 0x46c, + .sid = { + .override = 0x468, + .security = 0x46c, + }, }, }, { + .id = TEGRA186_MEMORY_CLIENT_EQOSR, .name = "eqosr", .sid = TEGRA186_SID_EQOS, .regs = { - .override = 0x470, - .security = 0x474, + .sid = { + .override = 0x470, + .security = 0x474, + }, }, }, { + .id = TEGRA186_MEMORY_CLIENT_EQOSW, .name = "eqosw", .sid = TEGRA186_SID_EQOS, .regs = { - .override = 0x478, - .security = 0x47c, + .sid = { + .override = 0x478, + .security = 0x47c, + }, }, }, { + .id = TEGRA186_MEMORY_CLIENT_UFSHCR, .name = "ufshcr", .sid = TEGRA186_SID_UFSHC, .regs = { - .override = 0x480, - .security = 0x484, + .sid = { + .override = 0x480, + .security = 0x484, + }, }, }, { + .id = TEGRA186_MEMORY_CLIENT_UFSHCW, .name = "ufshcw", .sid = TEGRA186_SID_UFSHC, .regs = { - .override = 0x488, - .security = 0x48c, + .sid = { + .override = 0x488, + .security = 0x48c, + }, }, }, { + .id = TEGRA186_MEMORY_CLIENT_NVDISPLAYR, .name = "nvdisplayr", .sid = TEGRA186_SID_NVDISPLAY, .regs = { - .override = 0x490, - .security = 0x494, + .sid = { + .override = 0x490, + .security = 0x494, + }, }, }, { + .id = TEGRA186_MEMORY_CLIENT_BPMPR, .name = "bpmpr", .sid = TEGRA186_SID_BPMP, .regs = { - .override = 0x498, - .security = 0x49c, + .sid = { + .override = 0x498, + .security = 0x49c, + }, }, }, { + .id = TEGRA186_MEMORY_CLIENT_BPMPW, .name = "bpmpw", .sid = TEGRA186_SID_BPMP, .regs = { - .override = 0x4a0, - .security = 0x4a4, + .sid = { + .override = 0x4a0, + .security = 0x4a4, + }, }, }, { + .id = TEGRA186_MEMORY_CLIENT_BPMPDMAR, .name = "bpmpdmar", .sid = TEGRA186_SID_BPMP, .regs = { - .override = 0x4a8, - .security = 0x4ac, + .sid = { + .override = 0x4a8, + .security = 0x4ac, + }, }, }, { + .id = TEGRA186_MEMORY_CLIENT_BPMPDMAW, .name = "bpmpdmaw", .sid = TEGRA186_SID_BPMP, .regs = { - .override = 0x4b0, - .security = 0x4b4, + .sid = { + .override = 0x4b0, + .security = 0x4b4, + }, }, }, { + .id = TEGRA186_MEMORY_CLIENT_AONR, .name = "aonr", .sid = TEGRA186_SID_AON, .regs = { - .override = 0x4b8, - .security = 0x4bc, + .sid = { + .override = 0x4b8, + .security = 0x4bc, + }, }, }, { + .id = TEGRA186_MEMORY_CLIENT_AONW, .name = "aonw", .sid = TEGRA186_SID_AON, .regs = { - .override = 0x4c0, - .security = 0x4c4, + .sid = { + .override = 0x4c0, + .security = 0x4c4, + }, }, }, { + .id = TEGRA186_MEMORY_CLIENT_AONDMAR, .name = "aondmar", .sid = TEGRA186_SID_AON, .regs = { - .override = 0x4c8, - .security = 0x4cc, + .sid = { + .override = 0x4c8, + .security = 0x4cc, + }, }, }, { + .id = TEGRA186_MEMORY_CLIENT_AONDMAW, .name = "aondmaw", .sid = TEGRA186_SID_AON, .regs = { - .override = 0x4d0, - .security = 0x4d4, + .sid = { + .override = 0x4d0, + .security = 0x4d4, + }, }, }, { + .id = TEGRA186_MEMORY_CLIENT_SCER, .name = "scer", .sid = TEGRA186_SID_SCE, .regs = { - .override = 0x4d8, - .security = 0x4dc, + .sid = { + .override = 0x4d8, + .security = 0x4dc, + }, }, }, { + .id = TEGRA186_MEMORY_CLIENT_SCEW, .name = "scew", .sid = TEGRA186_SID_SCE, .regs = { - .override = 0x4e0, - .security = 0x4e4, + .sid = { + .override = 0x4e0, + .security = 0x4e4, + }, }, }, { + .id = TEGRA186_MEMORY_CLIENT_SCEDMAR, .name = "scedmar", .sid = TEGRA186_SID_SCE, .regs = { - .override = 0x4e8, - .security = 0x4ec, + .sid = { + .override = 0x4e8, + .security = 0x4ec, + }, }, }, { + .id = TEGRA186_MEMORY_CLIENT_SCEDMAW, .name = "scedmaw", .sid = TEGRA186_SID_SCE, .regs = { - .override = 0x4f0, - .security = 0x4f4, + .sid = { + .override = 0x4f0, + .security = 0x4f4, + }, }, }, { + .id = TEGRA186_MEMORY_CLIENT_APEDMAR, .name = "apedmar", .sid = TEGRA186_SID_APE, .regs = { - .override = 0x4f8, - .security = 0x4fc, + .sid = { + .override = 0x4f8, + .security = 0x4fc, + }, }, }, { + .id = TEGRA186_MEMORY_CLIENT_APEDMAW, .name = "apedmaw", .sid = TEGRA186_SID_APE, .regs = { - .override = 0x500, - .security = 0x504, + .sid = { + .override = 0x500, + .security = 0x504, + }, }, }, { + .id = TEGRA186_MEMORY_CLIENT_NVDISPLAYR1, .name = "nvdisplayr1", .sid = TEGRA186_SID_NVDISPLAY, .regs = { - .override = 0x508, - .security = 0x50c, + .sid = { + .override = 0x508, + .security = 0x50c, + }, }, }, { + .id = TEGRA186_MEMORY_CLIENT_VICSRD1, .name = "vicsrd1", .sid = TEGRA186_SID_VIC, .regs = { - .override = 0x510, - .security = 0x514, + .sid = { + .override = 0x510, + .security = 0x514, + }, }, }, { + .id = TEGRA186_MEMORY_CLIENT_NVDECSRD1, .name = "nvdecsrd1", .sid = TEGRA186_SID_NVDEC, .regs = { - .override = 0x518, - .security = 0x51c, + .sid = { + .override = 0x518, + .security = 0x51c, + }, }, }, }; -static const struct tegra186_mc_soc tegra186_mc_soc = { +const struct tegra_mc_soc tegra186_mc_soc = { .num_clients = ARRAY_SIZE(tegra186_mc_clients), .clients = tegra186_mc_clients, + .num_address_bits = 40, + .num_channels = 4, + .client_id_mask = 0xff, + .intmask = MC_INT_DECERR_GENERALIZED_CARVEOUT | MC_INT_DECERR_MTS | + MC_INT_SECERR_SEC | MC_INT_DECERR_VPR | + MC_INT_SECURITY_VIOLATION | MC_INT_DECERR_EMEM, + .ops = &tegra186_mc_ops, + .ch_intmask = 0x0000000f, + .global_intstatus_channel_shift = 0, }; #endif - -#if defined(CONFIG_ARCH_TEGRA_194_SOC) -static const struct tegra186_mc_client tegra194_mc_clients[] = { - { - .name = "ptcr", - .sid = TEGRA194_SID_PASSTHROUGH, - .regs = { - .override = 0x000, - .security = 0x004, - }, - }, { - .name = "miu7r", - .sid = TEGRA194_SID_MIU, - .regs = { - .override = 0x008, - .security = 0x00c, - }, - }, { - .name = "miu7w", - .sid = TEGRA194_SID_MIU, - .regs = { - .override = 0x010, - .security = 0x014, - }, - }, { - .name = "hdar", - .sid = TEGRA194_SID_HDA, - .regs = { - .override = 0x0a8, - .security = 0x0ac, - }, - }, { - .name = "host1xdmar", - .sid = TEGRA194_SID_HOST1X, - .regs = { - .override = 0x0b0, - .security = 0x0b4, - }, - }, { - .name = "nvencsrd", - .sid = TEGRA194_SID_NVENC, - .regs = { - .override = 0x0e0, - .security = 0x0e4, - }, - }, { - .name = "satar", - .sid = TEGRA194_SID_SATA, - .regs = { - .override = 0x0f8, - .security = 0x0fc, - }, - }, { - .name = "mpcorer", - .sid = TEGRA194_SID_PASSTHROUGH, - .regs = { - .override = 0x138, - .security = 0x13c, - }, - }, { - .name = "nvencswr", - .sid = TEGRA194_SID_NVENC, - .regs = { - .override = 0x158, - .security = 0x15c, - }, - }, { - .name = "hdaw", - .sid = TEGRA194_SID_HDA, - .regs = { - .override = 0x1a8, - .security = 0x1ac, - }, - }, { - .name = "mpcorew", - .sid = TEGRA194_SID_PASSTHROUGH, - .regs = { - .override = 0x1c8, - .security = 0x1cc, - }, - }, { - .name = "sataw", - .sid = TEGRA194_SID_SATA, - .regs = { - .override = 0x1e8, - .security = 0x1ec, - }, - }, { - .name = "ispra", - .sid = TEGRA194_SID_ISP, - .regs = { - .override = 0x220, - .security = 0x224, - }, - }, { - .name = "ispfalr", - .sid = TEGRA194_SID_ISP_FALCON, - .regs = { - .override = 0x228, - .security = 0x22c, - }, - }, { - .name = "ispwa", - .sid = TEGRA194_SID_ISP, - .regs = { - .override = 0x230, - .security = 0x234, - }, - }, { - .name = "ispwb", - .sid = TEGRA194_SID_ISP, - .regs = { - .override = 0x238, - .security = 0x23c, - }, - }, { - .name = "xusb_hostr", - .sid = TEGRA194_SID_XUSB_HOST, - .regs = { - .override = 0x250, - .security = 0x254, - }, - }, { - .name = "xusb_hostw", - .sid = TEGRA194_SID_XUSB_HOST, - .regs = { - .override = 0x258, - .security = 0x25c, - }, - }, { - .name = "xusb_devr", - .sid = TEGRA194_SID_XUSB_DEV, - .regs = { - .override = 0x260, - .security = 0x264, - }, - }, { - .name = "xusb_devw", - .sid = TEGRA194_SID_XUSB_DEV, - .regs = { - .override = 0x268, - .security = 0x26c, - }, - }, { - .name = "sdmmcra", - .sid = TEGRA194_SID_SDMMC1, - .regs = { - .override = 0x300, - .security = 0x304, - }, - }, { - .name = "sdmmcr", - .sid = TEGRA194_SID_SDMMC3, - .regs = { - .override = 0x310, - .security = 0x314, - }, - }, { - .name = "sdmmcrab", - .sid = TEGRA194_SID_SDMMC4, - .regs = { - .override = 0x318, - .security = 0x31c, - }, - }, { - .name = "sdmmcwa", - .sid = TEGRA194_SID_SDMMC1, - .regs = { - .override = 0x320, - .security = 0x324, - }, - }, { - .name = "sdmmcw", - .sid = TEGRA194_SID_SDMMC3, - .regs = { - .override = 0x330, - .security = 0x334, - }, - }, { - .name = "sdmmcwab", - .sid = TEGRA194_SID_SDMMC4, - .regs = { - .override = 0x338, - .security = 0x33c, - }, - }, { - .name = "vicsrd", - .sid = TEGRA194_SID_VIC, - .regs = { - .override = 0x360, - .security = 0x364, - }, - }, { - .name = "vicswr", - .sid = TEGRA194_SID_VIC, - .regs = { - .override = 0x368, - .security = 0x36c, - }, - }, { - .name = "viw", - .sid = TEGRA194_SID_VI, - .regs = { - .override = 0x390, - .security = 0x394, - }, - }, { - .name = "nvdecsrd", - .sid = TEGRA194_SID_NVDEC, - .regs = { - .override = 0x3c0, - .security = 0x3c4, - }, - }, { - .name = "nvdecswr", - .sid = TEGRA194_SID_NVDEC, - .regs = { - .override = 0x3c8, - .security = 0x3cc, - }, - }, { - .name = "aper", - .sid = TEGRA194_SID_APE, - .regs = { - .override = 0x3c0, - .security = 0x3c4, - }, - }, { - .name = "apew", - .sid = TEGRA194_SID_APE, - .regs = { - .override = 0x3d0, - .security = 0x3d4, - }, - }, { - .name = "nvjpgsrd", - .sid = TEGRA194_SID_NVJPG, - .regs = { - .override = 0x3f0, - .security = 0x3f4, - }, - }, { - .name = "nvjpgswr", - .sid = TEGRA194_SID_NVJPG, - .regs = { - .override = 0x3f0, - .security = 0x3f4, - }, - }, { - .name = "axiapr", - .sid = TEGRA194_SID_PASSTHROUGH, - .regs = { - .override = 0x410, - .security = 0x414, - }, - }, { - .name = "axiapw", - .sid = TEGRA194_SID_PASSTHROUGH, - .regs = { - .override = 0x418, - .security = 0x41c, - }, - }, { - .name = "etrr", - .sid = TEGRA194_SID_ETR, - .regs = { - .override = 0x420, - .security = 0x424, - }, - }, { - .name = "etrw", - .sid = TEGRA194_SID_ETR, - .regs = { - .override = 0x428, - .security = 0x42c, - }, - }, { - .name = "axisr", - .sid = TEGRA194_SID_PASSTHROUGH, - .regs = { - .override = 0x460, - .security = 0x464, - }, - }, { - .name = "axisw", - .sid = TEGRA194_SID_PASSTHROUGH, - .regs = { - .override = 0x468, - .security = 0x46c, - }, - }, { - .name = "eqosr", - .sid = TEGRA194_SID_EQOS, - .regs = { - .override = 0x470, - .security = 0x474, - }, - }, { - .name = "eqosw", - .sid = TEGRA194_SID_EQOS, - .regs = { - .override = 0x478, - .security = 0x47c, - }, - }, { - .name = "ufshcr", - .sid = TEGRA194_SID_UFSHC, - .regs = { - .override = 0x480, - .security = 0x484, - }, - }, { - .name = "ufshcw", - .sid = TEGRA194_SID_UFSHC, - .regs = { - .override = 0x488, - .security = 0x48c, - }, - }, { - .name = "nvdisplayr", - .sid = TEGRA194_SID_NVDISPLAY, - .regs = { - .override = 0x490, - .security = 0x494, - }, - }, { - .name = "bpmpr", - .sid = TEGRA194_SID_BPMP, - .regs = { - .override = 0x498, - .security = 0x49c, - }, - }, { - .name = "bpmpw", - .sid = TEGRA194_SID_BPMP, - .regs = { - .override = 0x4a0, - .security = 0x4a4, - }, - }, { - .name = "bpmpdmar", - .sid = TEGRA194_SID_BPMP, - .regs = { - .override = 0x4a8, - .security = 0x4ac, - }, - }, { - .name = "bpmpdmaw", - .sid = TEGRA194_SID_BPMP, - .regs = { - .override = 0x4b0, - .security = 0x4b4, - }, - }, { - .name = "aonr", - .sid = TEGRA194_SID_AON, - .regs = { - .override = 0x4b8, - .security = 0x4bc, - }, - }, { - .name = "aonw", - .sid = TEGRA194_SID_AON, - .regs = { - .override = 0x4c0, - .security = 0x4c4, - }, - }, { - .name = "aondmar", - .sid = TEGRA194_SID_AON, - .regs = { - .override = 0x4c8, - .security = 0x4cc, - }, - }, { - .name = "aondmaw", - .sid = TEGRA194_SID_AON, - .regs = { - .override = 0x4d0, - .security = 0x4d4, - }, - }, { - .name = "scer", - .sid = TEGRA194_SID_SCE, - .regs = { - .override = 0x4d8, - .security = 0x4dc, - }, - }, { - .name = "scew", - .sid = TEGRA194_SID_SCE, - .regs = { - .override = 0x4e0, - .security = 0x4e4, - }, - }, { - .name = "scedmar", - .sid = TEGRA194_SID_SCE, - .regs = { - .override = 0x4e8, - .security = 0x4ec, - }, - }, { - .name = "scedmaw", - .sid = TEGRA194_SID_SCE, - .regs = { - .override = 0x4f0, - .security = 0x4f4, - }, - }, { - .name = "apedmar", - .sid = TEGRA194_SID_APE, - .regs = { - .override = 0x4f8, - .security = 0x4fc, - }, - }, { - .name = "apedmaw", - .sid = TEGRA194_SID_APE, - .regs = { - .override = 0x500, - .security = 0x504, - }, - }, { - .name = "nvdisplayr1", - .sid = TEGRA194_SID_NVDISPLAY, - .regs = { - .override = 0x508, - .security = 0x50c, - }, - }, { - .name = "vicsrd1", - .sid = TEGRA194_SID_VIC, - .regs = { - .override = 0x510, - .security = 0x514, - }, - }, { - .name = "nvdecsrd1", - .sid = TEGRA194_SID_NVDEC, - .regs = { - .override = 0x518, - .security = 0x51c, - }, - }, { - .name = "miu0r", - .sid = TEGRA194_SID_MIU, - .regs = { - .override = 0x530, - .security = 0x534, - }, - }, { - .name = "miu0w", - .sid = TEGRA194_SID_MIU, - .regs = { - .override = 0x538, - .security = 0x53c, - }, - }, { - .name = "miu1r", - .sid = TEGRA194_SID_MIU, - .regs = { - .override = 0x540, - .security = 0x544, - }, - }, { - .name = "miu1w", - .sid = TEGRA194_SID_MIU, - .regs = { - .override = 0x548, - .security = 0x54c, - }, - }, { - .name = "miu2r", - .sid = TEGRA194_SID_MIU, - .regs = { - .override = 0x570, - .security = 0x574, - }, - }, { - .name = "miu2w", - .sid = TEGRA194_SID_MIU, - .regs = { - .override = 0x578, - .security = 0x57c, - }, - }, { - .name = "miu3r", - .sid = TEGRA194_SID_MIU, - .regs = { - .override = 0x580, - .security = 0x584, - }, - }, { - .name = "miu3w", - .sid = TEGRA194_SID_MIU, - .regs = { - .override = 0x588, - .security = 0x58c, - }, - }, { - .name = "miu4r", - .sid = TEGRA194_SID_MIU, - .regs = { - .override = 0x590, - .security = 0x594, - }, - }, { - .name = "miu4w", - .sid = TEGRA194_SID_MIU, - .regs = { - .override = 0x598, - .security = 0x59c, - }, - }, { - .name = "dpmur", - .sid = TEGRA194_SID_PASSTHROUGH, - .regs = { - .override = 0x598, - .security = 0x59c, - }, - }, { - .name = "vifalr", - .sid = TEGRA194_SID_VI_FALCON, - .regs = { - .override = 0x5e0, - .security = 0x5e4, - }, - }, { - .name = "vifalw", - .sid = TEGRA194_SID_VI_FALCON, - .regs = { - .override = 0x5e8, - .security = 0x5ec, - }, - }, { - .name = "dla0rda", - .sid = TEGRA194_SID_NVDLA0, - .regs = { - .override = 0x5f0, - .security = 0x5f4, - }, - }, { - .name = "dla0falrdb", - .sid = TEGRA194_SID_NVDLA0, - .regs = { - .override = 0x5f8, - .security = 0x5fc, - }, - }, { - .name = "dla0wra", - .sid = TEGRA194_SID_NVDLA0, - .regs = { - .override = 0x600, - .security = 0x604, - }, - }, { - .name = "dla0falwrb", - .sid = TEGRA194_SID_NVDLA0, - .regs = { - .override = 0x608, - .security = 0x60c, - }, - }, { - .name = "dla1rda", - .sid = TEGRA194_SID_NVDLA1, - .regs = { - .override = 0x610, - .security = 0x614, - }, - }, { - .name = "dla1falrdb", - .sid = TEGRA194_SID_NVDLA1, - .regs = { - .override = 0x618, - .security = 0x61c, - }, - }, { - .name = "dla1wra", - .sid = TEGRA194_SID_NVDLA1, - .regs = { - .override = 0x620, - .security = 0x624, - }, - }, { - .name = "dla1falwrb", - .sid = TEGRA194_SID_NVDLA1, - .regs = { - .override = 0x628, - .security = 0x62c, - }, - }, { - .name = "pva0rda", - .sid = TEGRA194_SID_PVA0, - .regs = { - .override = 0x630, - .security = 0x634, - }, - }, { - .name = "pva0rdb", - .sid = TEGRA194_SID_PVA0, - .regs = { - .override = 0x638, - .security = 0x63c, - }, - }, { - .name = "pva0rdc", - .sid = TEGRA194_SID_PVA0, - .regs = { - .override = 0x640, - .security = 0x644, - }, - }, { - .name = "pva0wra", - .sid = TEGRA194_SID_PVA0, - .regs = { - .override = 0x648, - .security = 0x64c, - }, - }, { - .name = "pva0wrb", - .sid = TEGRA194_SID_PVA0, - .regs = { - .override = 0x650, - .security = 0x654, - }, - }, { - .name = "pva0wrc", - .sid = TEGRA194_SID_PVA0, - .regs = { - .override = 0x658, - .security = 0x65c, - }, - }, { - .name = "pva1rda", - .sid = TEGRA194_SID_PVA1, - .regs = { - .override = 0x660, - .security = 0x664, - }, - }, { - .name = "pva1rdb", - .sid = TEGRA194_SID_PVA1, - .regs = { - .override = 0x668, - .security = 0x66c, - }, - }, { - .name = "pva1rdc", - .sid = TEGRA194_SID_PVA1, - .regs = { - .override = 0x670, - .security = 0x674, - }, - }, { - .name = "pva1wra", - .sid = TEGRA194_SID_PVA1, - .regs = { - .override = 0x678, - .security = 0x67c, - }, - }, { - .name = "pva1wrb", - .sid = TEGRA194_SID_PVA1, - .regs = { - .override = 0x680, - .security = 0x684, - }, - }, { - .name = "pva1wrc", - .sid = TEGRA194_SID_PVA1, - .regs = { - .override = 0x688, - .security = 0x68c, - }, - }, { - .name = "rcer", - .sid = TEGRA194_SID_RCE, - .regs = { - .override = 0x690, - .security = 0x694, - }, - }, { - .name = "rcew", - .sid = TEGRA194_SID_RCE, - .regs = { - .override = 0x698, - .security = 0x69c, - }, - }, { - .name = "rcedmar", - .sid = TEGRA194_SID_RCE, - .regs = { - .override = 0x6a0, - .security = 0x6a4, - }, - }, { - .name = "rcedmaw", - .sid = TEGRA194_SID_RCE, - .regs = { - .override = 0x6a8, - .security = 0x6ac, - }, - }, { - .name = "nvenc1srd", - .sid = TEGRA194_SID_NVENC1, - .regs = { - .override = 0x6b0, - .security = 0x6b4, - }, - }, { - .name = "nvenc1swr", - .sid = TEGRA194_SID_NVENC1, - .regs = { - .override = 0x6b8, - .security = 0x6bc, - }, - }, { - .name = "pcie0r", - .sid = TEGRA194_SID_PCIE0, - .regs = { - .override = 0x6c0, - .security = 0x6c4, - }, - }, { - .name = "pcie0w", - .sid = TEGRA194_SID_PCIE0, - .regs = { - .override = 0x6c8, - .security = 0x6cc, - }, - }, { - .name = "pcie1r", - .sid = TEGRA194_SID_PCIE1, - .regs = { - .override = 0x6d0, - .security = 0x6d4, - }, - }, { - .name = "pcie1w", - .sid = TEGRA194_SID_PCIE1, - .regs = { - .override = 0x6d8, - .security = 0x6dc, - }, - }, { - .name = "pcie2ar", - .sid = TEGRA194_SID_PCIE2, - .regs = { - .override = 0x6e0, - .security = 0x6e4, - }, - }, { - .name = "pcie2aw", - .sid = TEGRA194_SID_PCIE2, - .regs = { - .override = 0x6e8, - .security = 0x6ec, - }, - }, { - .name = "pcie3r", - .sid = TEGRA194_SID_PCIE3, - .regs = { - .override = 0x6f0, - .security = 0x6f4, - }, - }, { - .name = "pcie3w", - .sid = TEGRA194_SID_PCIE3, - .regs = { - .override = 0x6f8, - .security = 0x6fc, - }, - }, { - .name = "pcie4r", - .sid = TEGRA194_SID_PCIE4, - .regs = { - .override = 0x700, - .security = 0x704, - }, - }, { - .name = "pcie4w", - .sid = TEGRA194_SID_PCIE4, - .regs = { - .override = 0x708, - .security = 0x70c, - }, - }, { - .name = "pcie5r", - .sid = TEGRA194_SID_PCIE5, - .regs = { - .override = 0x710, - .security = 0x714, - }, - }, { - .name = "pcie5w", - .sid = TEGRA194_SID_PCIE5, - .regs = { - .override = 0x718, - .security = 0x71c, - }, - }, { - .name = "ispfalw", - .sid = TEGRA194_SID_ISP_FALCON, - .regs = { - .override = 0x720, - .security = 0x724, - }, - }, { - .name = "dla0rda1", - .sid = TEGRA194_SID_NVDLA0, - .regs = { - .override = 0x748, - .security = 0x74c, - }, - }, { - .name = "dla1rda1", - .sid = TEGRA194_SID_NVDLA1, - .regs = { - .override = 0x750, - .security = 0x754, - }, - }, { - .name = "pva0rda1", - .sid = TEGRA194_SID_PVA0, - .regs = { - .override = 0x758, - .security = 0x75c, - }, - }, { - .name = "pva0rdb1", - .sid = TEGRA194_SID_PVA0, - .regs = { - .override = 0x760, - .security = 0x764, - }, - }, { - .name = "pva1rda1", - .sid = TEGRA194_SID_PVA1, - .regs = { - .override = 0x768, - .security = 0x76c, - }, - }, { - .name = "pva1rdb1", - .sid = TEGRA194_SID_PVA1, - .regs = { - .override = 0x770, - .security = 0x774, - }, - }, { - .name = "pcie5r1", - .sid = TEGRA194_SID_PCIE5, - .regs = { - .override = 0x778, - .security = 0x77c, - }, - }, { - .name = "nvencsrd1", - .sid = TEGRA194_SID_NVENC, - .regs = { - .override = 0x780, - .security = 0x784, - }, - }, { - .name = "nvenc1srd1", - .sid = TEGRA194_SID_NVENC1, - .regs = { - .override = 0x788, - .security = 0x78c, - }, - }, { - .name = "ispra1", - .sid = TEGRA194_SID_ISP, - .regs = { - .override = 0x790, - .security = 0x794, - }, - }, { - .name = "pcie0r1", - .sid = TEGRA194_SID_PCIE0, - .regs = { - .override = 0x798, - .security = 0x79c, - }, - }, { - .name = "nvdec1srd", - .sid = TEGRA194_SID_NVDEC1, - .regs = { - .override = 0x7c8, - .security = 0x7cc, - }, - }, { - .name = "nvdec1srd1", - .sid = TEGRA194_SID_NVDEC1, - .regs = { - .override = 0x7d0, - .security = 0x7d4, - }, - }, { - .name = "nvdec1swr", - .sid = TEGRA194_SID_NVDEC1, - .regs = { - .override = 0x7d8, - .security = 0x7dc, - }, - }, { - .name = "miu5r", - .sid = TEGRA194_SID_MIU, - .regs = { - .override = 0x7e0, - .security = 0x7e4, - }, - }, { - .name = "miu5w", - .sid = TEGRA194_SID_MIU, - .regs = { - .override = 0x7e8, - .security = 0x7ec, - }, - }, { - .name = "miu6r", - .sid = TEGRA194_SID_MIU, - .regs = { - .override = 0x7f0, - .security = 0x7f4, - }, - }, { - .name = "miu6w", - .sid = TEGRA194_SID_MIU, - .regs = { - .override = 0x7f8, - .security = 0x7fc, - }, - }, -}; - -static const struct tegra186_mc_soc tegra194_mc_soc = { - .num_clients = ARRAY_SIZE(tegra194_mc_clients), - .clients = tegra194_mc_clients, -}; -#endif - -static int tegra186_mc_probe(struct platform_device *pdev) -{ - struct tegra186_mc *mc; - struct resource *res; - int err; - - mc = devm_kzalloc(&pdev->dev, sizeof(*mc), GFP_KERNEL); - if (!mc) - return -ENOMEM; - - mc->soc = of_device_get_match_data(&pdev->dev); - - res = platform_get_resource(pdev, IORESOURCE_MEM, 0); - mc->regs = devm_ioremap_resource(&pdev->dev, res); - if (IS_ERR(mc->regs)) - return PTR_ERR(mc->regs); - - mc->dev = &pdev->dev; - - err = of_platform_populate(pdev->dev.of_node, NULL, NULL, &pdev->dev); - if (err < 0) - return err; - - platform_set_drvdata(pdev, mc); - tegra186_mc_program_sid(mc); - - return 0; -} - -static int tegra186_mc_remove(struct platform_device *pdev) -{ - struct tegra186_mc *mc = platform_get_drvdata(pdev); - - of_platform_depopulate(mc->dev); - - return 0; -} - -static const struct of_device_id tegra186_mc_of_match[] = { -#if defined(CONFIG_ARCH_TEGRA_186_SOC) - { .compatible = "nvidia,tegra186-mc", .data = &tegra186_mc_soc }, -#endif -#if defined(CONFIG_ARCH_TEGRA_194_SOC) - { .compatible = "nvidia,tegra194-mc", .data = &tegra194_mc_soc }, -#endif - { /* sentinel */ } -}; -MODULE_DEVICE_TABLE(of, tegra186_mc_of_match); - -static int tegra186_mc_suspend(struct device *dev) -{ - return 0; -} - -static int tegra186_mc_resume(struct device *dev) -{ - struct tegra186_mc *mc = dev_get_drvdata(dev); - - tegra186_mc_program_sid(mc); - - return 0; -} - -static const struct dev_pm_ops tegra186_mc_pm_ops = { - SET_SYSTEM_SLEEP_PM_OPS(tegra186_mc_suspend, tegra186_mc_resume) -}; - -static struct platform_driver tegra186_mc_driver = { - .driver = { - .name = "tegra186-mc", - .of_match_table = tegra186_mc_of_match, - .pm = &tegra186_mc_pm_ops, - .suppress_bind_attrs = true, - }, - .probe = tegra186_mc_probe, - .remove = tegra186_mc_remove, -}; -module_platform_driver(tegra186_mc_driver); - -MODULE_AUTHOR("Thierry Reding <treding@nvidia.com>"); -MODULE_DESCRIPTION("NVIDIA Tegra186 Memory Controller driver"); -MODULE_LICENSE("GPL v2"); diff --git a/drivers/memory/tegra/tegra194.c b/drivers/memory/tegra/tegra194.c new file mode 100644 index 000000000000..b2416ee3ac26 --- /dev/null +++ b/drivers/memory/tegra/tegra194.c @@ -0,0 +1,1360 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2017-2021 NVIDIA CORPORATION. All rights reserved. + */ + +#include <soc/tegra/mc.h> + +#include <dt-bindings/memory/tegra194-mc.h> + +#include "mc.h" + +static const struct tegra_mc_client tegra194_mc_clients[] = { + { + .id = TEGRA194_MEMORY_CLIENT_PTCR, + .name = "ptcr", + .sid = TEGRA194_SID_PASSTHROUGH, + .regs = { + .sid = { + .override = 0x000, + .security = 0x004, + }, + }, + }, { + .id = TEGRA194_MEMORY_CLIENT_MIU7R, + .name = "miu7r", + .sid = TEGRA194_SID_MIU, + .regs = { + .sid = { + .override = 0x008, + .security = 0x00c, + }, + }, + }, { + .id = TEGRA194_MEMORY_CLIENT_MIU7W, + .name = "miu7w", + .sid = TEGRA194_SID_MIU, + .regs = { + .sid = { + .override = 0x010, + .security = 0x014, + }, + }, + }, { + .id = TEGRA194_MEMORY_CLIENT_HDAR, + .name = "hdar", + .sid = TEGRA194_SID_HDA, + .regs = { + .sid = { + .override = 0x0a8, + .security = 0x0ac, + }, + }, + }, { + .id = TEGRA194_MEMORY_CLIENT_HOST1XDMAR, + .name = "host1xdmar", + .sid = TEGRA194_SID_HOST1X, + .regs = { + .sid = { + .override = 0x0b0, + .security = 0x0b4, + }, + }, + }, { + .id = TEGRA194_MEMORY_CLIENT_NVENCSRD, + .name = "nvencsrd", + .sid = TEGRA194_SID_NVENC, + .regs = { + .sid = { + .override = 0x0e0, + .security = 0x0e4, + }, + }, + }, { + .id = TEGRA194_MEMORY_CLIENT_SATAR, + .name = "satar", + .sid = TEGRA194_SID_SATA, + .regs = { + .sid = { + .override = 0x0f8, + .security = 0x0fc, + }, + }, + }, { + .id = TEGRA194_MEMORY_CLIENT_MPCORER, + .name = "mpcorer", + .sid = TEGRA194_SID_PASSTHROUGH, + .regs = { + .sid = { + .override = 0x138, + .security = 0x13c, + }, + }, + }, { + .id = TEGRA194_MEMORY_CLIENT_NVENCSWR, + .name = "nvencswr", + .sid = TEGRA194_SID_NVENC, + .regs = { + .sid = { + .override = 0x158, + .security = 0x15c, + }, + }, + }, { + .id = TEGRA194_MEMORY_CLIENT_HDAW, + .name = "hdaw", + .sid = TEGRA194_SID_HDA, + .regs = { + .sid = { + .override = 0x1a8, + .security = 0x1ac, + }, + }, + }, { + .id = TEGRA194_MEMORY_CLIENT_MPCOREW, + .name = "mpcorew", + .sid = TEGRA194_SID_PASSTHROUGH, + .regs = { + .sid = { + .override = 0x1c8, + .security = 0x1cc, + }, + }, + }, { + .id = TEGRA194_MEMORY_CLIENT_SATAW, + .name = "sataw", + .sid = TEGRA194_SID_SATA, + .regs = { + .sid = { + .override = 0x1e8, + .security = 0x1ec, + }, + }, + }, { + .id = TEGRA194_MEMORY_CLIENT_ISPRA, + .name = "ispra", + .sid = TEGRA194_SID_ISP, + .regs = { + .sid = { + .override = 0x220, + .security = 0x224, + }, + }, + }, { + .id = TEGRA194_MEMORY_CLIENT_ISPFALR, + .name = "ispfalr", + .sid = TEGRA194_SID_ISP_FALCON, + .regs = { + .sid = { + .override = 0x228, + .security = 0x22c, + }, + }, + }, { + .id = TEGRA194_MEMORY_CLIENT_ISPWA, + .name = "ispwa", + .sid = TEGRA194_SID_ISP, + .regs = { + .sid = { + .override = 0x230, + .security = 0x234, + }, + }, + }, { + .id = TEGRA194_MEMORY_CLIENT_ISPWB, + .name = "ispwb", + .sid = TEGRA194_SID_ISP, + .regs = { + .sid = { + .override = 0x238, + .security = 0x23c, + }, + }, + }, { + .id = TEGRA194_MEMORY_CLIENT_XUSB_HOSTR, + .name = "xusb_hostr", + .sid = TEGRA194_SID_XUSB_HOST, + .regs = { + .sid = { + .override = 0x250, + .security = 0x254, + }, + }, + }, { + .id = TEGRA194_MEMORY_CLIENT_XUSB_HOSTW, + .name = "xusb_hostw", + .sid = TEGRA194_SID_XUSB_HOST, + .regs = { + .sid = { + .override = 0x258, + .security = 0x25c, + }, + }, + }, { + .id = TEGRA194_MEMORY_CLIENT_XUSB_DEVR, + .name = "xusb_devr", + .sid = TEGRA194_SID_XUSB_DEV, + .regs = { + .sid = { + .override = 0x260, + .security = 0x264, + }, + }, + }, { + .id = TEGRA194_MEMORY_CLIENT_XUSB_DEVW, + .name = "xusb_devw", + .sid = TEGRA194_SID_XUSB_DEV, + .regs = { + .sid = { + .override = 0x268, + .security = 0x26c, + }, + }, + }, { + .id = TEGRA194_MEMORY_CLIENT_SDMMCRA, + .name = "sdmmcra", + .sid = TEGRA194_SID_SDMMC1, + .regs = { + .sid = { + .override = 0x300, + .security = 0x304, + }, + }, + }, { + .id = TEGRA194_MEMORY_CLIENT_SDMMCR, + .name = "sdmmcr", + .sid = TEGRA194_SID_SDMMC3, + .regs = { + .sid = { + .override = 0x310, + .security = 0x314, + }, + }, + }, { + .id = TEGRA194_MEMORY_CLIENT_SDMMCRAB, + .name = "sdmmcrab", + .sid = TEGRA194_SID_SDMMC4, + .regs = { + .sid = { + .override = 0x318, + .security = 0x31c, + }, + }, + }, { + .id = TEGRA194_MEMORY_CLIENT_SDMMCWA, + .name = "sdmmcwa", + .sid = TEGRA194_SID_SDMMC1, + .regs = { + .sid = { + .override = 0x320, + .security = 0x324, + }, + }, + }, { + .id = TEGRA194_MEMORY_CLIENT_SDMMCW, + .name = "sdmmcw", + .sid = TEGRA194_SID_SDMMC3, + .regs = { + .sid = { + .override = 0x330, + .security = 0x334, + }, + }, + }, { + .id = TEGRA194_MEMORY_CLIENT_SDMMCWAB, + .name = "sdmmcwab", + .sid = TEGRA194_SID_SDMMC4, + .regs = { + .sid = { + .override = 0x338, + .security = 0x33c, + }, + }, + }, { + .id = TEGRA194_MEMORY_CLIENT_VICSRD, + .name = "vicsrd", + .sid = TEGRA194_SID_VIC, + .regs = { + .sid = { + .override = 0x360, + .security = 0x364, + }, + }, + }, { + .id = TEGRA194_MEMORY_CLIENT_VICSWR, + .name = "vicswr", + .sid = TEGRA194_SID_VIC, + .regs = { + .sid = { + .override = 0x368, + .security = 0x36c, + }, + }, + }, { + .id = TEGRA194_MEMORY_CLIENT_VIW, + .name = "viw", + .sid = TEGRA194_SID_VI, + .regs = { + .sid = { + .override = 0x390, + .security = 0x394, + }, + }, + }, { + .id = TEGRA194_MEMORY_CLIENT_NVDECSRD, + .name = "nvdecsrd", + .sid = TEGRA194_SID_NVDEC, + .regs = { + .sid = { + .override = 0x3c0, + .security = 0x3c4, + }, + }, + }, { + .id = TEGRA194_MEMORY_CLIENT_NVDECSWR, + .name = "nvdecswr", + .sid = TEGRA194_SID_NVDEC, + .regs = { + .sid = { + .override = 0x3c8, + .security = 0x3cc, + }, + }, + }, { + .id = TEGRA194_MEMORY_CLIENT_APER, + .name = "aper", + .sid = TEGRA194_SID_APE, + .regs = { + .sid = { + .override = 0x3c0, + .security = 0x3c4, + }, + }, + }, { + .id = TEGRA194_MEMORY_CLIENT_APEW, + .name = "apew", + .sid = TEGRA194_SID_APE, + .regs = { + .sid = { + .override = 0x3d0, + .security = 0x3d4, + }, + }, + }, { + .id = TEGRA194_MEMORY_CLIENT_NVJPGSRD, + .name = "nvjpgsrd", + .sid = TEGRA194_SID_NVJPG, + .regs = { + .sid = { + .override = 0x3f0, + .security = 0x3f4, + }, + }, + }, { + .id = TEGRA194_MEMORY_CLIENT_NVJPGSWR, + .name = "nvjpgswr", + .sid = TEGRA194_SID_NVJPG, + .regs = { + .sid = { + .override = 0x3f0, + .security = 0x3f4, + }, + }, + }, { + .name = "axiapr", + .id = TEGRA194_MEMORY_CLIENT_AXIAPR, + .sid = TEGRA194_SID_PASSTHROUGH, + .regs = { + .sid = { + .override = 0x410, + .security = 0x414, + }, + }, + }, { + .id = TEGRA194_MEMORY_CLIENT_AXIAPW, + .name = "axiapw", + .sid = TEGRA194_SID_PASSTHROUGH, + .regs = { + .sid = { + .override = 0x418, + .security = 0x41c, + }, + }, + }, { + .id = TEGRA194_MEMORY_CLIENT_ETRR, + .name = "etrr", + .sid = TEGRA194_SID_ETR, + .regs = { + .sid = { + .override = 0x420, + .security = 0x424, + }, + }, + }, { + .id = TEGRA194_MEMORY_CLIENT_ETRW, + .name = "etrw", + .sid = TEGRA194_SID_ETR, + .regs = { + .sid = { + .override = 0x428, + .security = 0x42c, + }, + }, + }, { + .id = TEGRA194_MEMORY_CLIENT_AXISR, + .name = "axisr", + .sid = TEGRA194_SID_PASSTHROUGH, + .regs = { + .sid = { + .override = 0x460, + .security = 0x464, + }, + }, + }, { + .id = TEGRA194_MEMORY_CLIENT_AXISW, + .name = "axisw", + .sid = TEGRA194_SID_PASSTHROUGH, + .regs = { + .sid = { + .override = 0x468, + .security = 0x46c, + }, + }, + }, { + .id = TEGRA194_MEMORY_CLIENT_EQOSR, + .name = "eqosr", + .sid = TEGRA194_SID_EQOS, + .regs = { + .sid = { + .override = 0x470, + .security = 0x474, + }, + }, + }, { + .name = "eqosw", + .id = TEGRA194_MEMORY_CLIENT_EQOSW, + .sid = TEGRA194_SID_EQOS, + .regs = { + .sid = { + .override = 0x478, + .security = 0x47c, + }, + }, + }, { + .id = TEGRA194_MEMORY_CLIENT_UFSHCR, + .name = "ufshcr", + .sid = TEGRA194_SID_UFSHC, + .regs = { + .sid = { + .override = 0x480, + .security = 0x484, + }, + }, + }, { + .id = TEGRA194_MEMORY_CLIENT_UFSHCW, + .name = "ufshcw", + .sid = TEGRA194_SID_UFSHC, + .regs = { + .sid = { + .override = 0x488, + .security = 0x48c, + }, + }, + }, { + .id = TEGRA194_MEMORY_CLIENT_NVDISPLAYR, + .name = "nvdisplayr", + .sid = TEGRA194_SID_NVDISPLAY, + .regs = { + .sid = { + .override = 0x490, + .security = 0x494, + }, + }, + }, { + .id = TEGRA194_MEMORY_CLIENT_BPMPR, + .name = "bpmpr", + .sid = TEGRA194_SID_BPMP, + .regs = { + .sid = { + .override = 0x498, + .security = 0x49c, + }, + }, + }, { + .id = TEGRA194_MEMORY_CLIENT_BPMPW, + .name = "bpmpw", + .sid = TEGRA194_SID_BPMP, + .regs = { + .sid = { + .override = 0x4a0, + .security = 0x4a4, + }, + }, + }, { + .id = TEGRA194_MEMORY_CLIENT_BPMPDMAR, + .name = "bpmpdmar", + .sid = TEGRA194_SID_BPMP, + .regs = { + .sid = { + .override = 0x4a8, + .security = 0x4ac, + }, + }, + }, { + .id = TEGRA194_MEMORY_CLIENT_BPMPDMAW, + .name = "bpmpdmaw", + .sid = TEGRA194_SID_BPMP, + .regs = { + .sid = { + .override = 0x4b0, + .security = 0x4b4, + }, + }, + }, { + .id = TEGRA194_MEMORY_CLIENT_AONR, + .name = "aonr", + .sid = TEGRA194_SID_AON, + .regs = { + .sid = { + .override = 0x4b8, + .security = 0x4bc, + }, + }, + }, { + .id = TEGRA194_MEMORY_CLIENT_AONW, + .name = "aonw", + .sid = TEGRA194_SID_AON, + .regs = { + .sid = { + .override = 0x4c0, + .security = 0x4c4, + }, + }, + }, { + .id = TEGRA194_MEMORY_CLIENT_AONDMAR, + .name = "aondmar", + .sid = TEGRA194_SID_AON, + .regs = { + .sid = { + .override = 0x4c8, + .security = 0x4cc, + }, + }, + }, { + .id = TEGRA194_MEMORY_CLIENT_AONDMAW, + .name = "aondmaw", + .sid = TEGRA194_SID_AON, + .regs = { + .sid = { + .override = 0x4d0, + .security = 0x4d4, + }, + }, + }, { + .id = TEGRA194_MEMORY_CLIENT_SCER, + .name = "scer", + .sid = TEGRA194_SID_SCE, + .regs = { + .sid = { + .override = 0x4d8, + .security = 0x4dc, + }, + }, + }, { + .id = TEGRA194_MEMORY_CLIENT_SCEW, + .name = "scew", + .sid = TEGRA194_SID_SCE, + .regs = { + .sid = { + .override = 0x4e0, + .security = 0x4e4, + }, + }, + }, { + .id = TEGRA194_MEMORY_CLIENT_SCEDMAR, + .name = "scedmar", + .sid = TEGRA194_SID_SCE, + .regs = { + .sid = { + .override = 0x4e8, + .security = 0x4ec, + }, + }, + }, { + .id = TEGRA194_MEMORY_CLIENT_SCEDMAW, + .name = "scedmaw", + .sid = TEGRA194_SID_SCE, + .regs = { + .sid = { + .override = 0x4f0, + .security = 0x4f4, + }, + }, + }, { + .id = TEGRA194_MEMORY_CLIENT_APEDMAR, + .name = "apedmar", + .sid = TEGRA194_SID_APE, + .regs = { + .sid = { + .override = 0x4f8, + .security = 0x4fc, + }, + }, + }, { + .id = TEGRA194_MEMORY_CLIENT_APEDMAW, + .name = "apedmaw", + .sid = TEGRA194_SID_APE, + .regs = { + .sid = { + .override = 0x500, + .security = 0x504, + }, + }, + }, { + .id = TEGRA194_MEMORY_CLIENT_NVDISPLAYR1, + .name = "nvdisplayr1", + .sid = TEGRA194_SID_NVDISPLAY, + .regs = { + .sid = { + .override = 0x508, + .security = 0x50c, + }, + }, + }, { + .id = TEGRA194_MEMORY_CLIENT_VICSRD1, + .name = "vicsrd1", + .sid = TEGRA194_SID_VIC, + .regs = { + .sid = { + .override = 0x510, + .security = 0x514, + }, + }, + }, { + .id = TEGRA194_MEMORY_CLIENT_NVDECSRD1, + .name = "nvdecsrd1", + .sid = TEGRA194_SID_NVDEC, + .regs = { + .sid = { + .override = 0x518, + .security = 0x51c, + }, + }, + }, { + .id = TEGRA194_MEMORY_CLIENT_MIU0R, + .name = "miu0r", + .sid = TEGRA194_SID_MIU, + .regs = { + .sid = { + .override = 0x530, + .security = 0x534, + }, + }, + }, { + .name = "miu0w", + .id = TEGRA194_MEMORY_CLIENT_MIU0W, + .sid = TEGRA194_SID_MIU, + .regs = { + .sid = { + .override = 0x538, + .security = 0x53c, + }, + }, + }, { + .id = TEGRA194_MEMORY_CLIENT_MIU1R, + .name = "miu1r", + .sid = TEGRA194_SID_MIU, + .regs = { + .sid = { + .override = 0x540, + .security = 0x544, + }, + }, + }, { + .id = TEGRA194_MEMORY_CLIENT_MIU1W, + .name = "miu1w", + .sid = TEGRA194_SID_MIU, + .regs = { + .sid = { + .override = 0x548, + .security = 0x54c, + }, + }, + }, { + .id = TEGRA194_MEMORY_CLIENT_MIU2R, + .name = "miu2r", + .sid = TEGRA194_SID_MIU, + .regs = { + .sid = { + .override = 0x570, + .security = 0x574, + }, + }, + }, { + .id = TEGRA194_MEMORY_CLIENT_MIU2W, + .name = "miu2w", + .sid = TEGRA194_SID_MIU, + .regs = { + .sid = { + .override = 0x578, + .security = 0x57c, + }, + }, + }, { + .id = TEGRA194_MEMORY_CLIENT_MIU3R, + .name = "miu3r", + .sid = TEGRA194_SID_MIU, + .regs = { + .sid = { + .override = 0x580, + .security = 0x584, + }, + }, + }, { + .id = TEGRA194_MEMORY_CLIENT_MIU3W, + .name = "miu3w", + .sid = TEGRA194_SID_MIU, + .regs = { + .sid = { + .override = 0x588, + .security = 0x58c, + }, + }, + }, { + .id = TEGRA194_MEMORY_CLIENT_MIU4R, + .name = "miu4r", + .sid = TEGRA194_SID_MIU, + .regs = { + .sid = { + .override = 0x590, + .security = 0x594, + }, + }, + }, { + .id = TEGRA194_MEMORY_CLIENT_MIU4W, + .name = "miu4w", + .sid = TEGRA194_SID_MIU, + .regs = { + .sid = { + .override = 0x598, + .security = 0x59c, + }, + }, + }, { + .id = TEGRA194_MEMORY_CLIENT_DPMUR, + .name = "dpmur", + .sid = TEGRA194_SID_PASSTHROUGH, + .regs = { + .sid = { + .override = 0x598, + .security = 0x59c, + }, + }, + }, { + .id = TEGRA194_MEMORY_CLIENT_VIFALR, + .name = "vifalr", + .sid = TEGRA194_SID_VI_FALCON, + .regs = { + .sid = { + .override = 0x5e0, + .security = 0x5e4, + }, + }, + }, { + .id = TEGRA194_MEMORY_CLIENT_VIFALW, + .name = "vifalw", + .sid = TEGRA194_SID_VI_FALCON, + .regs = { + .sid = { + .override = 0x5e8, + .security = 0x5ec, + }, + }, + }, { + .id = TEGRA194_MEMORY_CLIENT_DLA0RDA, + .name = "dla0rda", + .sid = TEGRA194_SID_NVDLA0, + .regs = { + .sid = { + .override = 0x5f0, + .security = 0x5f4, + }, + }, + }, { + .id = TEGRA194_MEMORY_CLIENT_DLA0FALRDB, + .name = "dla0falrdb", + .sid = TEGRA194_SID_NVDLA0, + .regs = { + .sid = { + .override = 0x5f8, + .security = 0x5fc, + }, + }, + }, { + .id = TEGRA194_MEMORY_CLIENT_DLA0WRA, + .name = "dla0wra", + .sid = TEGRA194_SID_NVDLA0, + .regs = { + .sid = { + .override = 0x600, + .security = 0x604, + }, + }, + }, { + .id = TEGRA194_MEMORY_CLIENT_DLA0FALWRB, + .name = "dla0falwrb", + .sid = TEGRA194_SID_NVDLA0, + .regs = { + .sid = { + .override = 0x608, + .security = 0x60c, + }, + }, + }, { + .id = TEGRA194_MEMORY_CLIENT_DLA1RDA, + .name = "dla1rda", + .sid = TEGRA194_SID_NVDLA1, + .regs = { + .sid = { + .override = 0x610, + .security = 0x614, + }, + }, + }, { + .id = TEGRA194_MEMORY_CLIENT_DLA1FALRDB, + .name = "dla1falrdb", + .sid = TEGRA194_SID_NVDLA1, + .regs = { + .sid = { + .override = 0x618, + .security = 0x61c, + }, + }, + }, { + .id = TEGRA194_MEMORY_CLIENT_DLA1WRA, + .name = "dla1wra", + .sid = TEGRA194_SID_NVDLA1, + .regs = { + .sid = { + .override = 0x620, + .security = 0x624, + }, + }, + }, { + .id = TEGRA194_MEMORY_CLIENT_DLA1FALWRB, + .name = "dla1falwrb", + .sid = TEGRA194_SID_NVDLA1, + .regs = { + .sid = { + .override = 0x628, + .security = 0x62c, + }, + }, + }, { + .id = TEGRA194_MEMORY_CLIENT_PVA0RDA, + .name = "pva0rda", + .sid = TEGRA194_SID_PVA0, + .regs = { + .sid = { + .override = 0x630, + .security = 0x634, + }, + }, + }, { + .id = TEGRA194_MEMORY_CLIENT_PVA0RDB, + .name = "pva0rdb", + .sid = TEGRA194_SID_PVA0, + .regs = { + .sid = { + .override = 0x638, + .security = 0x63c, + }, + }, + }, { + .id = TEGRA194_MEMORY_CLIENT_PVA0RDC, + .name = "pva0rdc", + .sid = TEGRA194_SID_PVA0, + .regs = { + .sid = { + .override = 0x640, + .security = 0x644, + }, + }, + }, { + .id = TEGRA194_MEMORY_CLIENT_PVA0WRA, + .name = "pva0wra", + .sid = TEGRA194_SID_PVA0, + .regs = { + .sid = { + .override = 0x648, + .security = 0x64c, + }, + }, + }, { + .id = TEGRA194_MEMORY_CLIENT_PVA0WRB, + .name = "pva0wrb", + .sid = TEGRA194_SID_PVA0, + .regs = { + .sid = { + .override = 0x650, + .security = 0x654, + }, + }, + }, { + .id = TEGRA194_MEMORY_CLIENT_PVA0WRC, + .name = "pva0wrc", + .sid = TEGRA194_SID_PVA0, + .regs = { + .sid = { + .override = 0x658, + .security = 0x65c, + }, + }, + }, { + .id = TEGRA194_MEMORY_CLIENT_PVA1RDA, + .name = "pva1rda", + .sid = TEGRA194_SID_PVA1, + .regs = { + .sid = { + .override = 0x660, + .security = 0x664, + }, + }, + }, { + .id = TEGRA194_MEMORY_CLIENT_PVA1RDB, + .name = "pva1rdb", + .sid = TEGRA194_SID_PVA1, + .regs = { + .sid = { + .override = 0x668, + .security = 0x66c, + }, + }, + }, { + .id = TEGRA194_MEMORY_CLIENT_PVA1RDC, + .name = "pva1rdc", + .sid = TEGRA194_SID_PVA1, + .regs = { + .sid = { + .override = 0x670, + .security = 0x674, + }, + }, + }, { + .id = TEGRA194_MEMORY_CLIENT_PVA1WRA, + .name = "pva1wra", + .sid = TEGRA194_SID_PVA1, + .regs = { + .sid = { + .override = 0x678, + .security = 0x67c, + }, + }, + }, { + .id = TEGRA194_MEMORY_CLIENT_PVA1WRB, + .name = "pva1wrb", + .sid = TEGRA194_SID_PVA1, + .regs = { + .sid = { + .override = 0x680, + .security = 0x684, + }, + }, + }, { + .id = TEGRA194_MEMORY_CLIENT_PVA1WRC, + .name = "pva1wrc", + .sid = TEGRA194_SID_PVA1, + .regs = { + .sid = { + .override = 0x688, + .security = 0x68c, + }, + }, + }, { + .id = TEGRA194_MEMORY_CLIENT_RCER, + .name = "rcer", + .sid = TEGRA194_SID_RCE, + .regs = { + .sid = { + .override = 0x690, + .security = 0x694, + }, + }, + }, { + .id = TEGRA194_MEMORY_CLIENT_RCEW, + .name = "rcew", + .sid = TEGRA194_SID_RCE, + .regs = { + .sid = { + .override = 0x698, + .security = 0x69c, + }, + }, + }, { + .id = TEGRA194_MEMORY_CLIENT_RCEDMAR, + .name = "rcedmar", + .sid = TEGRA194_SID_RCE, + .regs = { + .sid = { + .override = 0x6a0, + .security = 0x6a4, + }, + }, + }, { + .id = TEGRA194_MEMORY_CLIENT_RCEDMAW, + .name = "rcedmaw", + .sid = TEGRA194_SID_RCE, + .regs = { + .sid = { + .override = 0x6a8, + .security = 0x6ac, + }, + }, + }, { + .id = TEGRA194_MEMORY_CLIENT_NVENC1SRD, + .name = "nvenc1srd", + .sid = TEGRA194_SID_NVENC1, + .regs = { + .sid = { + .override = 0x6b0, + .security = 0x6b4, + }, + }, + }, { + .id = TEGRA194_MEMORY_CLIENT_NVENC1SWR, + .name = "nvenc1swr", + .sid = TEGRA194_SID_NVENC1, + .regs = { + .sid = { + .override = 0x6b8, + .security = 0x6bc, + }, + }, + }, { + .id = TEGRA194_MEMORY_CLIENT_PCIE0R, + .name = "pcie0r", + .sid = TEGRA194_SID_PCIE0, + .regs = { + .sid = { + .override = 0x6c0, + .security = 0x6c4, + }, + }, + }, { + .id = TEGRA194_MEMORY_CLIENT_PCIE0W, + .name = "pcie0w", + .sid = TEGRA194_SID_PCIE0, + .regs = { + .sid = { + .override = 0x6c8, + .security = 0x6cc, + }, + }, + }, { + .id = TEGRA194_MEMORY_CLIENT_PCIE1R, + .name = "pcie1r", + .sid = TEGRA194_SID_PCIE1, + .regs = { + .sid = { + .override = 0x6d0, + .security = 0x6d4, + }, + }, + }, { + .id = TEGRA194_MEMORY_CLIENT_PCIE1W, + .name = "pcie1w", + .sid = TEGRA194_SID_PCIE1, + .regs = { + .sid = { + .override = 0x6d8, + .security = 0x6dc, + }, + }, + }, { + .id = TEGRA194_MEMORY_CLIENT_PCIE2AR, + .name = "pcie2ar", + .sid = TEGRA194_SID_PCIE2, + .regs = { + .sid = { + .override = 0x6e0, + .security = 0x6e4, + }, + }, + }, { + .id = TEGRA194_MEMORY_CLIENT_PCIE2AW, + .name = "pcie2aw", + .sid = TEGRA194_SID_PCIE2, + .regs = { + .sid = { + .override = 0x6e8, + .security = 0x6ec, + }, + }, + }, { + .id = TEGRA194_MEMORY_CLIENT_PCIE3R, + .name = "pcie3r", + .sid = TEGRA194_SID_PCIE3, + .regs = { + .sid = { + .override = 0x6f0, + .security = 0x6f4, + }, + }, + }, { + .id = TEGRA194_MEMORY_CLIENT_PCIE3W, + .name = "pcie3w", + .sid = TEGRA194_SID_PCIE3, + .regs = { + .sid = { + .override = 0x6f8, + .security = 0x6fc, + }, + }, + }, { + .id = TEGRA194_MEMORY_CLIENT_PCIE4R, + .name = "pcie4r", + .sid = TEGRA194_SID_PCIE4, + .regs = { + .sid = { + .override = 0x700, + .security = 0x704, + }, + }, + }, { + .id = TEGRA194_MEMORY_CLIENT_PCIE4W, + .name = "pcie4w", + .sid = TEGRA194_SID_PCIE4, + .regs = { + .sid = { + .override = 0x708, + .security = 0x70c, + }, + }, + }, { + .id = TEGRA194_MEMORY_CLIENT_PCIE5R, + .name = "pcie5r", + .sid = TEGRA194_SID_PCIE5, + .regs = { + .sid = { + .override = 0x710, + .security = 0x714, + }, + }, + }, { + .id = TEGRA194_MEMORY_CLIENT_PCIE5W, + .name = "pcie5w", + .sid = TEGRA194_SID_PCIE5, + .regs = { + .sid = { + .override = 0x718, + .security = 0x71c, + }, + }, + }, { + .id = TEGRA194_MEMORY_CLIENT_ISPFALW, + .name = "ispfalw", + .sid = TEGRA194_SID_ISP_FALCON, + .regs = { + .sid = { + .override = 0x720, + .security = 0x724, + }, + }, + }, { + .id = TEGRA194_MEMORY_CLIENT_DLA0RDA1, + .name = "dla0rda1", + .sid = TEGRA194_SID_NVDLA0, + .regs = { + .sid = { + .override = 0x748, + .security = 0x74c, + }, + }, + }, { + .id = TEGRA194_MEMORY_CLIENT_DLA1RDA1, + .name = "dla1rda1", + .sid = TEGRA194_SID_NVDLA1, + .regs = { + .sid = { + .override = 0x750, + .security = 0x754, + }, + }, + }, { + .id = TEGRA194_MEMORY_CLIENT_PVA0RDA1, + .name = "pva0rda1", + .sid = TEGRA194_SID_PVA0, + .regs = { + .sid = { + .override = 0x758, + .security = 0x75c, + }, + }, + }, { + .id = TEGRA194_MEMORY_CLIENT_PVA0RDB1, + .name = "pva0rdb1", + .sid = TEGRA194_SID_PVA0, + .regs = { + .sid = { + .override = 0x760, + .security = 0x764, + }, + }, + }, { + .id = TEGRA194_MEMORY_CLIENT_PVA1RDA1, + .name = "pva1rda1", + .sid = TEGRA194_SID_PVA1, + .regs = { + .sid = { + .override = 0x768, + .security = 0x76c, + }, + }, + }, { + .id = TEGRA194_MEMORY_CLIENT_PVA1RDB1, + .name = "pva1rdb1", + .sid = TEGRA194_SID_PVA1, + .regs = { + .sid = { + .override = 0x770, + .security = 0x774, + }, + }, + }, { + .id = TEGRA194_MEMORY_CLIENT_PCIE5R1, + .name = "pcie5r1", + .sid = TEGRA194_SID_PCIE5, + .regs = { + .sid = { + .override = 0x778, + .security = 0x77c, + }, + }, + }, { + .id = TEGRA194_MEMORY_CLIENT_NVENCSRD1, + .name = "nvencsrd1", + .sid = TEGRA194_SID_NVENC, + .regs = { + .sid = { + .override = 0x780, + .security = 0x784, + }, + }, + }, { + .id = TEGRA194_MEMORY_CLIENT_NVENC1SRD1, + .name = "nvenc1srd1", + .sid = TEGRA194_SID_NVENC1, + .regs = { + .sid = { + .override = 0x788, + .security = 0x78c, + }, + }, + }, { + .id = TEGRA194_MEMORY_CLIENT_ISPRA1, + .name = "ispra1", + .sid = TEGRA194_SID_ISP, + .regs = { + .sid = { + .override = 0x790, + .security = 0x794, + }, + }, + }, { + .id = TEGRA194_MEMORY_CLIENT_PCIE0R1, + .name = "pcie0r1", + .sid = TEGRA194_SID_PCIE0, + .regs = { + .sid = { + .override = 0x798, + .security = 0x79c, + }, + }, + }, { + .id = TEGRA194_MEMORY_CLIENT_NVDEC1SRD, + .name = "nvdec1srd", + .sid = TEGRA194_SID_NVDEC1, + .regs = { + .sid = { + .override = 0x7c8, + .security = 0x7cc, + }, + }, + }, { + .id = TEGRA194_MEMORY_CLIENT_NVDEC1SRD1, + .name = "nvdec1srd1", + .sid = TEGRA194_SID_NVDEC1, + .regs = { + .sid = { + .override = 0x7d0, + .security = 0x7d4, + }, + }, + }, { + .id = TEGRA194_MEMORY_CLIENT_NVDEC1SWR, + .name = "nvdec1swr", + .sid = TEGRA194_SID_NVDEC1, + .regs = { + .sid = { + .override = 0x7d8, + .security = 0x7dc, + }, + }, + }, { + .id = TEGRA194_MEMORY_CLIENT_MIU5R, + .name = "miu5r", + .sid = TEGRA194_SID_MIU, + .regs = { + .sid = { + .override = 0x7e0, + .security = 0x7e4, + }, + }, + }, { + .id = TEGRA194_MEMORY_CLIENT_MIU5W, + .name = "miu5w", + .sid = TEGRA194_SID_MIU, + .regs = { + .sid = { + .override = 0x7e8, + .security = 0x7ec, + }, + }, + }, { + .id = TEGRA194_MEMORY_CLIENT_MIU6R, + .name = "miu6r", + .sid = TEGRA194_SID_MIU, + .regs = { + .sid = { + .override = 0x7f0, + .security = 0x7f4, + }, + }, + }, { + .id = TEGRA194_MEMORY_CLIENT_MIU6W, + .name = "miu6w", + .sid = TEGRA194_SID_MIU, + .regs = { + .sid = { + .override = 0x7f8, + .security = 0x7fc, + }, + }, + }, +}; + +const struct tegra_mc_soc tegra194_mc_soc = { + .num_clients = ARRAY_SIZE(tegra194_mc_clients), + .clients = tegra194_mc_clients, + .num_address_bits = 40, + .num_channels = 16, + .client_id_mask = 0xff, + .intmask = MC_INT_DECERR_ROUTE_SANITY | + MC_INT_DECERR_GENERALIZED_CARVEOUT | MC_INT_DECERR_MTS | + MC_INT_SECERR_SEC | MC_INT_DECERR_VPR | + MC_INT_SECURITY_VIOLATION | MC_INT_DECERR_EMEM, + .has_addr_hi_reg = true, + .ops = &tegra186_mc_ops, + .ch_intmask = 0x00000f00, + .global_intstatus_channel_shift = 8, +}; diff --git a/drivers/memory/tegra/tegra20-emc.c b/drivers/memory/tegra/tegra20-emc.c index 8ae474d9bfb9..25ba3c5e4ad6 100644 --- a/drivers/memory/tegra/tegra20-emc.c +++ b/drivers/memory/tegra/tegra20-emc.c @@ -5,25 +5,38 @@ * Author: Dmitry Osipenko <digetx@gmail.com> */ +#include <linux/bitfield.h> #include <linux/clk.h> #include <linux/clk/tegra.h> -#include <linux/completion.h> #include <linux/debugfs.h> +#include <linux/devfreq.h> #include <linux/err.h> +#include <linux/interconnect-provider.h> #include <linux/interrupt.h> #include <linux/io.h> +#include <linux/iopoll.h> #include <linux/kernel.h> #include <linux/module.h> +#include <linux/mutex.h> #include <linux/of.h> #include <linux/platform_device.h> +#include <linux/pm_opp.h> +#include <linux/slab.h> #include <linux/sort.h> #include <linux/types.h> +#include <soc/tegra/common.h> #include <soc/tegra/fuse.h> +#include "../jedec_ddr.h" +#include "../of_memory.h" + +#include "mc.h" + #define EMC_INTSTATUS 0x000 #define EMC_INTMASK 0x004 #define EMC_DBG 0x008 +#define EMC_ADR_CFG_0 0x010 #define EMC_TIMING_CONTROL 0x028 #define EMC_RC 0x02c #define EMC_RFC 0x030 @@ -60,8 +73,14 @@ #define EMC_QUSE_EXTRA 0x0ac #define EMC_ODT_WRITE 0x0b0 #define EMC_ODT_READ 0x0b4 +#define EMC_MRR 0x0ec #define EMC_FBIO_CFG5 0x104 #define EMC_FBIO_CFG6 0x114 +#define EMC_STAT_CONTROL 0x160 +#define EMC_STAT_LLMC_CONTROL 0x178 +#define EMC_STAT_PWR_CLOCK_LIMIT 0x198 +#define EMC_STAT_PWR_CLOCKS 0x19c +#define EMC_STAT_PWR_COUNT 0x1a0 #define EMC_AUTO_CAL_INTERVAL 0x2a8 #define EMC_CFG_2 0x2b8 #define EMC_CFG_DIG_DLL 0x2bc @@ -81,6 +100,7 @@ #define EMC_REFRESH_OVERFLOW_INT BIT(3) #define EMC_CLKCHANGE_COMPLETE_INT BIT(4) +#define EMC_MRR_DIVLD_INT BIT(5) #define EMC_DBG_READ_MUX_ASSEMBLY BIT(0) #define EMC_DBG_WRITE_MUX_ACTIVE BIT(1) @@ -88,6 +108,26 @@ #define EMC_DBG_READ_DQM_CTRL BIT(9) #define EMC_DBG_CFG_PRIORITY BIT(24) +#define EMC_FBIO_CFG5_DRAM_WIDTH_X16 BIT(4) +#define EMC_FBIO_CFG5_DRAM_TYPE GENMASK(1, 0) + +#define EMC_MRR_DEV_SELECTN GENMASK(31, 30) +#define EMC_MRR_MRR_MA GENMASK(23, 16) +#define EMC_MRR_MRR_DATA GENMASK(15, 0) + +#define EMC_ADR_CFG_0_EMEM_NUMDEV GENMASK(25, 24) + +#define EMC_PWR_GATHER_CLEAR (1 << 8) +#define EMC_PWR_GATHER_DISABLE (2 << 8) +#define EMC_PWR_GATHER_ENABLE (3 << 8) + +enum emc_dram_type { + DRAM_TYPE_RESERVED, + DRAM_TYPE_DDR1, + DRAM_TYPE_LPDDR2, + DRAM_TYPE_DDR2, +}; + static const u16 emc_timing_registers[] = { EMC_RC, EMC_RFC, @@ -142,12 +182,26 @@ struct emc_timing { u32 data[ARRAY_SIZE(emc_timing_registers)]; }; +enum emc_rate_request_type { + EMC_RATE_DEVFREQ, + EMC_RATE_DEBUG, + EMC_RATE_ICC, + EMC_RATE_TYPE_MAX, +}; + +struct emc_rate_request { + unsigned long min_rate; + unsigned long max_rate; +}; + struct tegra_emc { struct device *dev; - struct completion clk_handshake_complete; + struct tegra_mc *mc; + struct icc_provider provider; struct notifier_block clk_nb; struct clk *clk; void __iomem *regs; + unsigned int dram_bus_width; struct emc_timing *timings; unsigned int num_timings; @@ -157,22 +211,37 @@ struct tegra_emc { unsigned long min_rate; unsigned long max_rate; } debugfs; + + /* + * There are multiple sources in the EMC driver which could request + * a min/max clock rate, these rates are contained in this array. + */ + struct emc_rate_request requested_rate[EMC_RATE_TYPE_MAX]; + + /* protect shared rate-change code path */ + struct mutex rate_lock; + + struct devfreq_simple_ondemand_data ondemand_data; + + /* memory chip identity information */ + union lpddr2_basic_config4 basic_conf4; + unsigned int manufacturer_id; + unsigned int revision_id1; + unsigned int revision_id2; + + bool mrr_error; }; static irqreturn_t tegra_emc_isr(int irq, void *data) { struct tegra_emc *emc = data; - u32 intmask = EMC_REFRESH_OVERFLOW_INT | EMC_CLKCHANGE_COMPLETE_INT; + u32 intmask = EMC_REFRESH_OVERFLOW_INT; u32 status; status = readl_relaxed(emc->regs + EMC_INTSTATUS) & intmask; if (!status) return IRQ_NONE; - /* notify about EMC-CAR handshake completion */ - if (status & EMC_CLKCHANGE_COMPLETE_INT) - complete(&emc->clk_handshake_complete); - /* notify about HW problem */ if (status & EMC_REFRESH_OVERFLOW_INT) dev_err_ratelimited(emc->dev, @@ -224,14 +293,13 @@ static int emc_prepare_timing_change(struct tegra_emc *emc, unsigned long rate) /* wait until programming has settled */ readl_relaxed(emc->regs + emc_timing_registers[i - 1]); - reinit_completion(&emc->clk_handshake_complete); - return 0; } static int emc_complete_timing_change(struct tegra_emc *emc, bool flush) { - unsigned long timeout; + int err; + u32 v; dev_dbg(emc->dev, "%s: flush %d\n", __func__, flush); @@ -242,11 +310,12 @@ static int emc_complete_timing_change(struct tegra_emc *emc, bool flush) return 0; } - timeout = wait_for_completion_timeout(&emc->clk_handshake_complete, - msecs_to_jiffies(100)); - if (timeout == 0) { - dev_err(emc->dev, "EMC-CAR handshake failed\n"); - return -EIO; + err = readl_relaxed_poll_timeout_atomic(emc->regs + EMC_INTSTATUS, v, + v & EMC_CLKCHANGE_COMPLETE_INT, + 1, 100); + if (err) { + dev_err(emc->dev, "emc-car handshake timeout: %d\n", err); + return err; } return 0; @@ -357,37 +426,52 @@ static int tegra_emc_load_timings_from_dt(struct tegra_emc *emc, if (!emc->timings) return -ENOMEM; - emc->num_timings = child_count; timing = emc->timings; for_each_child_of_node(node, child) { + if (of_node_name_eq(child, "lpddr2")) + continue; + err = load_one_timing_from_dt(emc, timing++, child); if (err) { of_node_put(child); return err; } + + emc->num_timings++; } sort(emc->timings, emc->num_timings, sizeof(*timing), cmp_timings, NULL); - dev_info(emc->dev, - "got %u timings for RAM code %u (min %luMHz max %luMHz)\n", - emc->num_timings, - tegra_read_ram_code(), - emc->timings[0].rate / 1000000, - emc->timings[emc->num_timings - 1].rate / 1000000); + dev_info_once(emc->dev, + "got %u timings for RAM code %u (min %luMHz max %luMHz)\n", + emc->num_timings, + tegra_read_ram_code(), + emc->timings[0].rate / 1000000, + emc->timings[emc->num_timings - 1].rate / 1000000); return 0; } static struct device_node * -tegra_emc_find_node_by_ram_code(struct device *dev) +tegra_emc_find_node_by_ram_code(struct tegra_emc *emc) { + struct device *dev = emc->dev; struct device_node *np; u32 value, ram_code; int err; + if (emc->mrr_error) { + dev_warn(dev, "memory timings skipped due to MRR error\n"); + return NULL; + } + + if (of_get_child_count(dev->of_node) == 0) { + dev_info_once(dev, "device-tree doesn't have memory timings\n"); + return NULL; + } + if (!of_property_read_bool(dev->of_node, "nvidia,use-ram-code")) return of_node_get(dev->of_node); @@ -397,8 +481,49 @@ tegra_emc_find_node_by_ram_code(struct device *dev) np = of_find_node_by_name(np, "emc-tables")) { err = of_property_read_u32(np, "nvidia,ram-code", &value); if (err || value != ram_code) { - of_node_put(np); - continue; + struct device_node *lpddr2_np; + bool cfg_mismatches = false; + + lpddr2_np = of_find_node_by_name(np, "lpddr2"); + if (lpddr2_np) { + const struct lpddr2_info *info; + + info = of_lpddr2_get_info(lpddr2_np, dev); + if (info) { + if (info->manufacturer_id >= 0 && + info->manufacturer_id != emc->manufacturer_id) + cfg_mismatches = true; + + if (info->revision_id1 >= 0 && + info->revision_id1 != emc->revision_id1) + cfg_mismatches = true; + + if (info->revision_id2 >= 0 && + info->revision_id2 != emc->revision_id2) + cfg_mismatches = true; + + if (info->density != emc->basic_conf4.density) + cfg_mismatches = true; + + if (info->io_width != emc->basic_conf4.io_width) + cfg_mismatches = true; + + if (info->arch_type != emc->basic_conf4.arch_type) + cfg_mismatches = true; + } else { + dev_err(dev, "failed to parse %pOF\n", lpddr2_np); + cfg_mismatches = true; + } + + of_node_put(lpddr2_np); + } else { + cfg_mismatches = true; + } + + if (cfg_mismatches) { + of_node_put(np); + continue; + } } return np; @@ -410,10 +535,72 @@ tegra_emc_find_node_by_ram_code(struct device *dev) return NULL; } +static int emc_read_lpddr_mode_register(struct tegra_emc *emc, + unsigned int emem_dev, + unsigned int register_addr, + unsigned int *register_data) +{ + u32 memory_dev = emem_dev ? 1 : 2; + u32 val, mr_mask = 0xff; + int err; + + /* clear data-valid interrupt status */ + writel_relaxed(EMC_MRR_DIVLD_INT, emc->regs + EMC_INTSTATUS); + + /* issue mode register read request */ + val = FIELD_PREP(EMC_MRR_DEV_SELECTN, memory_dev); + val |= FIELD_PREP(EMC_MRR_MRR_MA, register_addr); + + writel_relaxed(val, emc->regs + EMC_MRR); + + /* wait for the LPDDR2 data-valid interrupt */ + err = readl_relaxed_poll_timeout_atomic(emc->regs + EMC_INTSTATUS, val, + val & EMC_MRR_DIVLD_INT, + 1, 100); + if (err) { + dev_err(emc->dev, "mode register %u read failed: %d\n", + register_addr, err); + emc->mrr_error = true; + return err; + } + + /* read out mode register data */ + val = readl_relaxed(emc->regs + EMC_MRR); + *register_data = FIELD_GET(EMC_MRR_MRR_DATA, val) & mr_mask; + + return 0; +} + +static void emc_read_lpddr_sdram_info(struct tegra_emc *emc, + unsigned int emem_dev, + bool print_out) +{ + /* these registers are standard for all LPDDR JEDEC memory chips */ + emc_read_lpddr_mode_register(emc, emem_dev, 5, &emc->manufacturer_id); + emc_read_lpddr_mode_register(emc, emem_dev, 6, &emc->revision_id1); + emc_read_lpddr_mode_register(emc, emem_dev, 7, &emc->revision_id2); + emc_read_lpddr_mode_register(emc, emem_dev, 8, &emc->basic_conf4.value); + + if (!print_out) + return; + + dev_info(emc->dev, "SDRAM[dev%u]: manufacturer: 0x%x (%s) rev1: 0x%x rev2: 0x%x prefetch: S%u density: %uMbit iowidth: %ubit\n", + emem_dev, emc->manufacturer_id, + lpddr2_jedec_manufacturer(emc->manufacturer_id), + emc->revision_id1, emc->revision_id2, + 4 >> emc->basic_conf4.arch_type, + 64 << emc->basic_conf4.density, + 32 >> emc->basic_conf4.io_width); +} + static int emc_setup_hw(struct tegra_emc *emc) { - u32 intmask = EMC_REFRESH_OVERFLOW_INT | EMC_CLKCHANGE_COMPLETE_INT; - u32 emc_cfg, emc_dbg; + u32 emc_cfg, emc_dbg, emc_fbio, emc_adr_cfg; + u32 intmask = EMC_REFRESH_OVERFLOW_INT; + static bool print_sdram_info_once; + enum emc_dram_type dram_type; + const char *dram_type_str; + unsigned int emem_numdev; emc_cfg = readl_relaxed(emc->regs + EMC_CFG_2); @@ -444,6 +631,44 @@ static int emc_setup_hw(struct tegra_emc *emc) emc_dbg &= ~EMC_DBG_FORCE_UPDATE; writel_relaxed(emc_dbg, emc->regs + EMC_DBG); + emc_fbio = readl_relaxed(emc->regs + EMC_FBIO_CFG5); + + if (emc_fbio & EMC_FBIO_CFG5_DRAM_WIDTH_X16) + emc->dram_bus_width = 16; + else + emc->dram_bus_width = 32; + + dram_type = FIELD_GET(EMC_FBIO_CFG5_DRAM_TYPE, emc_fbio); + + switch (dram_type) { + case DRAM_TYPE_RESERVED: + dram_type_str = "INVALID"; + break; + case DRAM_TYPE_DDR1: + dram_type_str = "DDR1"; + break; + case DRAM_TYPE_LPDDR2: + dram_type_str = "LPDDR2"; + break; + case DRAM_TYPE_DDR2: + dram_type_str = "DDR2"; + break; + } + + emc_adr_cfg = readl_relaxed(emc->regs + EMC_ADR_CFG_0); + emem_numdev = FIELD_GET(EMC_ADR_CFG_0_EMEM_NUMDEV, emc_adr_cfg) + 1; + + dev_info_once(emc->dev, "%ubit DRAM bus, %u %s %s attached\n", + emc->dram_bus_width, emem_numdev, dram_type_str, + emem_numdev == 2 ? "devices" : "device"); + + if (dram_type == DRAM_TYPE_LPDDR2) { + while (emem_numdev--) + emc_read_lpddr_sdram_info(emc, emem_numdev, + !print_sdram_info_once); + print_sdram_info_once = true; + } + return 0; } @@ -456,6 +681,9 @@ static long emc_round_rate(unsigned long rate, struct tegra_emc *emc = arg; unsigned int i; + if (!emc->num_timings) + return clk_get_rate(emc->clk); + min_rate = min(min_rate, emc->timings[emc->num_timings - 1].rate); for (i = 0; i < emc->num_timings; i++) { @@ -485,6 +713,83 @@ static long emc_round_rate(unsigned long rate, return timing->rate; } +static void tegra_emc_rate_requests_init(struct tegra_emc *emc) +{ + unsigned int i; + + for (i = 0; i < EMC_RATE_TYPE_MAX; i++) { + emc->requested_rate[i].min_rate = 0; + emc->requested_rate[i].max_rate = ULONG_MAX; + } +} + +static int emc_request_rate(struct tegra_emc *emc, + unsigned long new_min_rate, + unsigned long new_max_rate, + enum emc_rate_request_type type) +{ + struct emc_rate_request *req = emc->requested_rate; + unsigned long min_rate = 0, max_rate = ULONG_MAX; + unsigned int i; + int err; + + /* select minimum and maximum rates among the requested rates */ + for (i = 0; i < EMC_RATE_TYPE_MAX; i++, req++) { + if (i == type) { + min_rate = max(new_min_rate, min_rate); + max_rate = min(new_max_rate, max_rate); + } else { + min_rate = max(req->min_rate, min_rate); + max_rate = min(req->max_rate, max_rate); + } + } + + if (min_rate > max_rate) { + dev_err_ratelimited(emc->dev, "%s: type %u: out of range: %lu %lu\n", + __func__, type, min_rate, max_rate); + return -ERANGE; + } + + /* + * EMC rate-changes should go via OPP API because it manages voltage + * changes. + */ + err = dev_pm_opp_set_rate(emc->dev, min_rate); + if (err) + return err; + + emc->requested_rate[type].min_rate = new_min_rate; + emc->requested_rate[type].max_rate = new_max_rate; + + return 0; +} + +static int emc_set_min_rate(struct tegra_emc *emc, unsigned long rate, + enum emc_rate_request_type type) +{ + struct emc_rate_request *req = &emc->requested_rate[type]; + int ret; + + mutex_lock(&emc->rate_lock); + ret = emc_request_rate(emc, rate, req->max_rate, type); + mutex_unlock(&emc->rate_lock); + + return ret; +} + +static int emc_set_max_rate(struct tegra_emc *emc, unsigned long rate, + enum emc_rate_request_type type) +{ + struct emc_rate_request *req = &emc->requested_rate[type]; + int ret; + + mutex_lock(&emc->rate_lock); + ret = emc_request_rate(emc, req->min_rate, rate, type); + mutex_unlock(&emc->rate_lock); + + return ret; +} + /* * debugfs interface * @@ -568,7 +873,7 @@ static int tegra_emc_debug_min_rate_set(void *data, u64 rate) if (!tegra_emc_validate_rate(emc, rate)) return -EINVAL; - err = clk_set_min_rate(emc->clk, rate); + err = emc_set_min_rate(emc, rate, EMC_RATE_DEBUG); if (err < 0) return err; @@ -598,7 +903,7 @@ static int tegra_emc_debug_max_rate_set(void *data, u64 rate) if (!tegra_emc_validate_rate(emc, rate)) return -EINVAL; - err = clk_set_max_rate(emc->clk, rate); + err = emc_set_max_rate(emc, rate, EMC_RATE_DEBUG); if (err < 0) return err; @@ -628,6 +933,11 @@ static void tegra_emc_debugfs_init(struct tegra_emc *emc) emc->debugfs.max_rate = emc->timings[i].rate; } + if (!emc->num_timings) { + emc->debugfs.min_rate = clk_get_rate(emc->clk); + emc->debugfs.max_rate = emc->debugfs.min_rate; + } + err = clk_set_rate_range(emc->clk, emc->debugfs.min_rate, emc->debugfs.max_rate); if (err < 0) { @@ -637,61 +947,280 @@ static void tegra_emc_debugfs_init(struct tegra_emc *emc) } emc->debugfs.root = debugfs_create_dir("emc", NULL); - if (!emc->debugfs.root) { - dev_err(emc->dev, "failed to create debugfs directory\n"); - return; - } - debugfs_create_file("available_rates", S_IRUGO, emc->debugfs.root, + debugfs_create_file("available_rates", 0444, emc->debugfs.root, emc, &tegra_emc_debug_available_rates_fops); - debugfs_create_file("min_rate", S_IRUGO | S_IWUSR, emc->debugfs.root, + debugfs_create_file("min_rate", 0644, emc->debugfs.root, emc, &tegra_emc_debug_min_rate_fops); - debugfs_create_file("max_rate", S_IRUGO | S_IWUSR, emc->debugfs.root, + debugfs_create_file("max_rate", 0644, emc->debugfs.root, emc, &tegra_emc_debug_max_rate_fops); } +static inline struct tegra_emc * +to_tegra_emc_provider(struct icc_provider *provider) +{ + return container_of(provider, struct tegra_emc, provider); +} + +static struct icc_node_data * +emc_of_icc_xlate_extended(struct of_phandle_args *spec, void *data) +{ + struct icc_provider *provider = data; + struct icc_node_data *ndata; + struct icc_node *node; + + /* External Memory is the only possible ICC route */ + list_for_each_entry(node, &provider->nodes, node_list) { + if (node->id != TEGRA_ICC_EMEM) + continue; + + ndata = kzalloc(sizeof(*ndata), GFP_KERNEL); + if (!ndata) + return ERR_PTR(-ENOMEM); + + /* + * SRC and DST nodes should have matching TAG in order to have + * it set by default for a requested path. + */ + ndata->tag = TEGRA_MC_ICC_TAG_ISO; + ndata->node = node; + + return ndata; + } + + return ERR_PTR(-EPROBE_DEFER); +} + +static int emc_icc_set(struct icc_node *src, struct icc_node *dst) +{ + struct tegra_emc *emc = to_tegra_emc_provider(dst->provider); + unsigned long long peak_bw = icc_units_to_bps(dst->peak_bw); + unsigned long long avg_bw = icc_units_to_bps(dst->avg_bw); + unsigned long long rate = max(avg_bw, peak_bw); + unsigned int dram_data_bus_width_bytes; + int err; + + /* + * Tegra20 EMC runs on x2 clock rate of SDRAM bus because DDR data + * is sampled on both clock edges. This means that EMC clock rate + * equals to the peak data-rate. + */ + dram_data_bus_width_bytes = emc->dram_bus_width / 8; + do_div(rate, dram_data_bus_width_bytes); + rate = min_t(u64, rate, U32_MAX); + + err = emc_set_min_rate(emc, rate, EMC_RATE_ICC); + if (err) + return err; + + return 0; +} + +static int tegra_emc_interconnect_init(struct tegra_emc *emc) +{ + const struct tegra_mc_soc *soc; + struct icc_node *node; + int err; + + emc->mc = devm_tegra_memory_controller_get(emc->dev); + if (IS_ERR(emc->mc)) + return PTR_ERR(emc->mc); + + soc = emc->mc->soc; + + emc->provider.dev = emc->dev; + emc->provider.set = emc_icc_set; + emc->provider.data = &emc->provider; + emc->provider.aggregate = soc->icc_ops->aggregate; + emc->provider.xlate_extended = emc_of_icc_xlate_extended; + + err = icc_provider_add(&emc->provider); + if (err) + goto err_msg; + + /* create External Memory Controller node */ + node = icc_node_create(TEGRA_ICC_EMC); + if (IS_ERR(node)) { + err = PTR_ERR(node); + goto del_provider; + } + + node->name = "External Memory Controller"; + icc_node_add(node, &emc->provider); + + /* link External Memory Controller to External Memory (DRAM) */ + err = icc_link_create(node, TEGRA_ICC_EMEM); + if (err) + goto remove_nodes; + + /* create External Memory node */ + node = icc_node_create(TEGRA_ICC_EMEM); + if (IS_ERR(node)) { + err = PTR_ERR(node); + goto remove_nodes; + } + + node->name = "External Memory (DRAM)"; + icc_node_add(node, &emc->provider); + + return 0; + +remove_nodes: + icc_nodes_remove(&emc->provider); +del_provider: + icc_provider_del(&emc->provider); +err_msg: + dev_err(emc->dev, "failed to initialize ICC: %d\n", err); + + return err; +} + +static void devm_tegra_emc_unset_callback(void *data) +{ + tegra20_clk_set_emc_round_callback(NULL, NULL); +} + +static void devm_tegra_emc_unreg_clk_notifier(void *data) +{ + struct tegra_emc *emc = data; + + clk_notifier_unregister(emc->clk, &emc->clk_nb); +} + +static int tegra_emc_init_clk(struct tegra_emc *emc) +{ + int err; + + tegra20_clk_set_emc_round_callback(emc_round_rate, emc); + + err = devm_add_action_or_reset(emc->dev, devm_tegra_emc_unset_callback, + NULL); + if (err) + return err; + + emc->clk = devm_clk_get(emc->dev, NULL); + if (IS_ERR(emc->clk)) { + dev_err(emc->dev, "failed to get EMC clock: %pe\n", emc->clk); + return PTR_ERR(emc->clk); + } + + err = clk_notifier_register(emc->clk, &emc->clk_nb); + if (err) { + dev_err(emc->dev, "failed to register clk notifier: %d\n", err); + return err; + } + + err = devm_add_action_or_reset(emc->dev, + devm_tegra_emc_unreg_clk_notifier, emc); + if (err) + return err; + + return 0; +} + +static int tegra_emc_devfreq_target(struct device *dev, unsigned long *freq, + u32 flags) +{ + struct tegra_emc *emc = dev_get_drvdata(dev); + struct dev_pm_opp *opp; + unsigned long rate; + + opp = devfreq_recommended_opp(dev, freq, flags); + if (IS_ERR(opp)) { + dev_err(dev, "failed to find opp for %lu Hz\n", *freq); + return PTR_ERR(opp); + } + + rate = dev_pm_opp_get_freq(opp); + dev_pm_opp_put(opp); + + return emc_set_min_rate(emc, rate, EMC_RATE_DEVFREQ); +} + +static int tegra_emc_devfreq_get_dev_status(struct device *dev, + struct devfreq_dev_status *stat) +{ + struct tegra_emc *emc = dev_get_drvdata(dev); + + /* freeze counters */ + writel_relaxed(EMC_PWR_GATHER_DISABLE, emc->regs + EMC_STAT_CONTROL); + + /* + * busy_time: number of clocks EMC request was accepted + * total_time: number of clocks PWR_GATHER control was set to ENABLE + */ + stat->busy_time = readl_relaxed(emc->regs + EMC_STAT_PWR_COUNT); + stat->total_time = readl_relaxed(emc->regs + EMC_STAT_PWR_CLOCKS); + stat->current_frequency = clk_get_rate(emc->clk); + + /* clear counters and restart */ + writel_relaxed(EMC_PWR_GATHER_CLEAR, emc->regs + EMC_STAT_CONTROL); + writel_relaxed(EMC_PWR_GATHER_ENABLE, emc->regs + EMC_STAT_CONTROL); + + return 0; +} + +static struct devfreq_dev_profile tegra_emc_devfreq_profile = { + .polling_ms = 30, + .target = tegra_emc_devfreq_target, + .get_dev_status = tegra_emc_devfreq_get_dev_status, +}; + +static int tegra_emc_devfreq_init(struct tegra_emc *emc) +{ + struct devfreq *devfreq; + + /* + * PWR_COUNT is 1/2 of PWR_CLOCKS at max, and thus, the up-threshold + * should be less than 50. Secondly, multiple active memory clients + * may cause over 20% of lost clock cycles due to stalls caused by + * competing memory accesses. This means that threshold should be + * set to a less than 30 in order to have a properly working governor. + */ + emc->ondemand_data.upthreshold = 20; + + /* + * Reset statistic gathers state, select global bandwidth for the + * statistics collection mode and set clocks counter saturation + * limit to maximum. + */ + writel_relaxed(0x00000000, emc->regs + EMC_STAT_CONTROL); + writel_relaxed(0x00000000, emc->regs + EMC_STAT_LLMC_CONTROL); + writel_relaxed(0xffffffff, emc->regs + EMC_STAT_PWR_CLOCK_LIMIT); + + devfreq = devm_devfreq_add_device(emc->dev, &tegra_emc_devfreq_profile, + DEVFREQ_GOV_SIMPLE_ONDEMAND, + &emc->ondemand_data); + if (IS_ERR(devfreq)) { + dev_err(emc->dev, "failed to initialize devfreq: %pe", devfreq); + return PTR_ERR(devfreq); + } + + return 0; +} + static int tegra_emc_probe(struct platform_device *pdev) { + struct tegra_core_opp_params opp_params = {}; struct device_node *np; struct tegra_emc *emc; - struct resource *res; int irq, err; - /* driver has nothing to do in a case of memory timing absence */ - if (of_get_child_count(pdev->dev.of_node) == 0) { - dev_info(&pdev->dev, - "EMC device tree node doesn't have memory timings\n"); - return 0; - } - irq = platform_get_irq(pdev, 0); if (irq < 0) { - dev_err(&pdev->dev, "interrupt not specified\n"); dev_err(&pdev->dev, "please update your device tree\n"); return irq; } - np = tegra_emc_find_node_by_ram_code(&pdev->dev); - if (!np) - return -EINVAL; - emc = devm_kzalloc(&pdev->dev, sizeof(*emc), GFP_KERNEL); - if (!emc) { - of_node_put(np); + if (!emc) return -ENOMEM; - } - init_completion(&emc->clk_handshake_complete); + mutex_init(&emc->rate_lock); emc->clk_nb.notifier_call = tegra_emc_clk_change_notify; emc->dev = &pdev->dev; - err = tegra_emc_load_timings_from_dt(emc, np); - of_node_put(np); - if (err) - return err; - - res = platform_get_resource(pdev, IORESOURCE_MEM, 0); - emc->regs = devm_ioremap_resource(&pdev->dev, res); + emc->regs = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(emc->regs)) return PTR_ERR(emc->regs); @@ -699,44 +1228,52 @@ static int tegra_emc_probe(struct platform_device *pdev) if (err) return err; + np = tegra_emc_find_node_by_ram_code(emc); + if (np) { + err = tegra_emc_load_timings_from_dt(emc, np); + of_node_put(np); + if (err) + return err; + } + err = devm_request_irq(&pdev->dev, irq, tegra_emc_isr, 0, dev_name(&pdev->dev), emc); if (err) { - dev_err(&pdev->dev, "failed to request IRQ#%u: %d\n", irq, err); + dev_err(&pdev->dev, "failed to request IRQ: %d\n", err); return err; } - tegra20_clk_set_emc_round_callback(emc_round_rate, emc); + err = tegra_emc_init_clk(emc); + if (err) + return err; - emc->clk = devm_clk_get(&pdev->dev, "emc"); - if (IS_ERR(emc->clk)) { - err = PTR_ERR(emc->clk); - dev_err(&pdev->dev, "failed to get emc clock: %d\n", err); - goto unset_cb; - } + opp_params.init_state = true; - err = clk_notifier_register(emc->clk, &emc->clk_nb); - if (err) { - dev_err(&pdev->dev, "failed to register clk notifier: %d\n", - err); - goto unset_cb; - } + err = devm_tegra_core_dev_init_opp_table(&pdev->dev, &opp_params); + if (err) + return err; platform_set_drvdata(pdev, emc); + tegra_emc_rate_requests_init(emc); tegra_emc_debugfs_init(emc); + tegra_emc_interconnect_init(emc); + tegra_emc_devfreq_init(emc); - return 0; - -unset_cb: - tegra20_clk_set_emc_round_callback(NULL, NULL); + /* + * Don't allow the kernel module to be unloaded. Unloading adds some + * extra complexity which doesn't really worth the effort in a case of + * this driver. + */ + try_module_get(THIS_MODULE); - return err; + return 0; } static const struct of_device_id tegra_emc_of_match[] = { { .compatible = "nvidia,tegra20-emc", }, {}, }; +MODULE_DEVICE_TABLE(of, tegra_emc_of_match); static struct platform_driver tegra_emc_driver = { .probe = tegra_emc_probe, @@ -744,11 +1281,12 @@ static struct platform_driver tegra_emc_driver = { .name = "tegra20-emc", .of_match_table = tegra_emc_of_match, .suppress_bind_attrs = true, + .sync_state = icc_sync_state, }, }; +module_platform_driver(tegra_emc_driver); -static int __init tegra_emc_init(void) -{ - return platform_driver_register(&tegra_emc_driver); -} -subsys_initcall(tegra_emc_init); +MODULE_AUTHOR("Dmitry Osipenko <digetx@gmail.com>"); +MODULE_DESCRIPTION("NVIDIA Tegra20 EMC driver"); +MODULE_SOFTDEP("pre: governor_simpleondemand"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/memory/tegra/tegra20.c b/drivers/memory/tegra/tegra20.c index a8098bff91d9..fcd7738fcb53 100644 --- a/drivers/memory/tegra/tegra20.c +++ b/drivers/memory/tegra/tegra20.c @@ -3,10 +3,90 @@ * Copyright (C) 2012 NVIDIA CORPORATION. All rights reserved. */ +#include <linux/bitfield.h> +#include <linux/delay.h> +#include <linux/mutex.h> +#include <linux/of_device.h> +#include <linux/slab.h> +#include <linux/string.h> + #include <dt-bindings/memory/tegra20-mc.h> #include "mc.h" +#define MC_STAT_CONTROL 0x90 +#define MC_STAT_EMC_CLOCK_LIMIT 0xa0 +#define MC_STAT_EMC_CLOCKS 0xa4 +#define MC_STAT_EMC_CONTROL_0 0xa8 +#define MC_STAT_EMC_CONTROL_1 0xac +#define MC_STAT_EMC_COUNT_0 0xb8 +#define MC_STAT_EMC_COUNT_1 0xbc + +#define MC_STAT_CONTROL_CLIENT_ID GENMASK(13, 8) +#define MC_STAT_CONTROL_EVENT GENMASK(23, 16) +#define MC_STAT_CONTROL_PRI_EVENT GENMASK(25, 24) +#define MC_STAT_CONTROL_FILTER_CLIENT_ENABLE GENMASK(26, 26) +#define MC_STAT_CONTROL_FILTER_PRI GENMASK(29, 28) + +#define MC_STAT_CONTROL_PRI_EVENT_HP 0 +#define MC_STAT_CONTROL_PRI_EVENT_TM 1 +#define MC_STAT_CONTROL_PRI_EVENT_BW 2 + +#define MC_STAT_CONTROL_FILTER_PRI_DISABLE 0 +#define MC_STAT_CONTROL_FILTER_PRI_NO 1 +#define MC_STAT_CONTROL_FILTER_PRI_YES 2 + +#define MC_STAT_CONTROL_EVENT_QUALIFIED 0 +#define MC_STAT_CONTROL_EVENT_ANY_READ 1 +#define MC_STAT_CONTROL_EVENT_ANY_WRITE 2 +#define MC_STAT_CONTROL_EVENT_RD_WR_CHANGE 3 +#define MC_STAT_CONTROL_EVENT_SUCCESSIVE 4 +#define MC_STAT_CONTROL_EVENT_ARB_BANK_AA 5 +#define MC_STAT_CONTROL_EVENT_ARB_BANK_BB 6 +#define MC_STAT_CONTROL_EVENT_PAGE_MISS 7 +#define MC_STAT_CONTROL_EVENT_AUTO_PRECHARGE 8 + +#define EMC_GATHER_RST (0 << 8) +#define EMC_GATHER_CLEAR (1 << 8) +#define EMC_GATHER_DISABLE (2 << 8) +#define EMC_GATHER_ENABLE (3 << 8) + +#define MC_STAT_SAMPLE_TIME_USEC 16000 + +/* we store collected statistics as a fixed point values */ +#define MC_FX_FRAC_SCALE 100 + +static DEFINE_MUTEX(tegra20_mc_stat_lock); + +struct tegra20_mc_stat_gather { + unsigned int pri_filter; + unsigned int pri_event; + unsigned int result; + unsigned int client; + unsigned int event; + bool client_enb; +}; + +struct tegra20_mc_stat { + struct tegra20_mc_stat_gather gather0; + struct tegra20_mc_stat_gather gather1; + unsigned int sample_time_usec; + const struct tegra_mc *mc; +}; + +struct tegra20_mc_client_stat { + unsigned int events; + unsigned int arb_high_prio; + unsigned int arb_timeout; + unsigned int arb_bandwidth; + unsigned int rd_wr_change; + unsigned int successive; + unsigned int page_miss; + unsigned int auto_precharge; + unsigned int arb_bank_aa; + unsigned int arb_bank_bb; +}; + static const struct tegra_mc_client tegra20_mc_clients[] = { { .id = 0x00, @@ -280,6 +360,439 @@ static const struct tegra_mc_reset_ops tegra20_mc_reset_ops = { .reset_status = tegra20_mc_reset_status, }; +static int tegra20_mc_icc_set(struct icc_node *src, struct icc_node *dst) +{ + /* + * It should be possible to tune arbitration knobs here, but the + * default values are known to work well on all devices. Hence + * nothing to do here so far. + */ + return 0; +} + +static int tegra20_mc_icc_aggreate(struct icc_node *node, u32 tag, u32 avg_bw, + u32 peak_bw, u32 *agg_avg, u32 *agg_peak) +{ + /* + * ISO clients need to reserve extra bandwidth up-front because + * there could be high bandwidth pressure during initial filling + * of the client's FIFO buffers. Secondly, we need to take into + * account impurities of the memory subsystem. + */ + if (tag & TEGRA_MC_ICC_TAG_ISO) + peak_bw = tegra_mc_scale_percents(peak_bw, 300); + + *agg_avg += avg_bw; + *agg_peak = max(*agg_peak, peak_bw); + + return 0; +} + +static struct icc_node_data * +tegra20_mc_of_icc_xlate_extended(struct of_phandle_args *spec, void *data) +{ + struct tegra_mc *mc = icc_provider_to_tegra_mc(data); + unsigned int i, idx = spec->args[0]; + struct icc_node_data *ndata; + struct icc_node *node; + + list_for_each_entry(node, &mc->provider.nodes, node_list) { + if (node->id != idx) + continue; + + ndata = kzalloc(sizeof(*ndata), GFP_KERNEL); + if (!ndata) + return ERR_PTR(-ENOMEM); + + ndata->node = node; + + /* these clients are isochronous by default */ + if (strstarts(node->name, "display") || + strstarts(node->name, "vi")) + ndata->tag = TEGRA_MC_ICC_TAG_ISO; + else + ndata->tag = TEGRA_MC_ICC_TAG_DEFAULT; + + return ndata; + } + + for (i = 0; i < mc->soc->num_clients; i++) { + if (mc->soc->clients[i].id == idx) + return ERR_PTR(-EPROBE_DEFER); + } + + dev_err(mc->dev, "invalid ICC client ID %u\n", idx); + + return ERR_PTR(-EINVAL); +} + +static const struct tegra_mc_icc_ops tegra20_mc_icc_ops = { + .xlate_extended = tegra20_mc_of_icc_xlate_extended, + .aggregate = tegra20_mc_icc_aggreate, + .set = tegra20_mc_icc_set, +}; + +static u32 tegra20_mc_stat_gather_control(const struct tegra20_mc_stat_gather *g) +{ + u32 control; + + control = FIELD_PREP(MC_STAT_CONTROL_EVENT, g->event); + control |= FIELD_PREP(MC_STAT_CONTROL_CLIENT_ID, g->client); + control |= FIELD_PREP(MC_STAT_CONTROL_PRI_EVENT, g->pri_event); + control |= FIELD_PREP(MC_STAT_CONTROL_FILTER_PRI, g->pri_filter); + control |= FIELD_PREP(MC_STAT_CONTROL_FILTER_CLIENT_ENABLE, g->client_enb); + + return control; +} + +static void tegra20_mc_stat_gather(struct tegra20_mc_stat *stat) +{ + u32 clocks, count0, count1, control_0, control_1; + const struct tegra_mc *mc = stat->mc; + + control_0 = tegra20_mc_stat_gather_control(&stat->gather0); + control_1 = tegra20_mc_stat_gather_control(&stat->gather1); + + /* + * Reset statistic gathers state, select statistics collection mode + * and set clocks counter saturation limit to maximum. + */ + mc_writel(mc, 0x00000000, MC_STAT_CONTROL); + mc_writel(mc, control_0, MC_STAT_EMC_CONTROL_0); + mc_writel(mc, control_1, MC_STAT_EMC_CONTROL_1); + mc_writel(mc, 0xffffffff, MC_STAT_EMC_CLOCK_LIMIT); + + mc_writel(mc, EMC_GATHER_ENABLE, MC_STAT_CONTROL); + fsleep(stat->sample_time_usec); + mc_writel(mc, EMC_GATHER_DISABLE, MC_STAT_CONTROL); + + count0 = mc_readl(mc, MC_STAT_EMC_COUNT_0); + count1 = mc_readl(mc, MC_STAT_EMC_COUNT_1); + clocks = mc_readl(mc, MC_STAT_EMC_CLOCKS); + clocks = max(clocks / 100 / MC_FX_FRAC_SCALE, 1u); + + stat->gather0.result = DIV_ROUND_UP(count0, clocks); + stat->gather1.result = DIV_ROUND_UP(count1, clocks); +} + +static void tegra20_mc_stat_events(const struct tegra_mc *mc, + const struct tegra_mc_client *client0, + const struct tegra_mc_client *client1, + unsigned int pri_filter, + unsigned int pri_event, + unsigned int event, + unsigned int *result0, + unsigned int *result1) +{ + struct tegra20_mc_stat stat = {}; + + stat.gather0.client = client0 ? client0->id : 0; + stat.gather0.pri_filter = pri_filter; + stat.gather0.client_enb = !!client0; + stat.gather0.pri_event = pri_event; + stat.gather0.event = event; + + stat.gather1.client = client1 ? client1->id : 0; + stat.gather1.pri_filter = pri_filter; + stat.gather1.client_enb = !!client1; + stat.gather1.pri_event = pri_event; + stat.gather1.event = event; + + stat.sample_time_usec = MC_STAT_SAMPLE_TIME_USEC; + stat.mc = mc; + + tegra20_mc_stat_gather(&stat); + + *result0 = stat.gather0.result; + *result1 = stat.gather1.result; +} + +static void tegra20_mc_collect_stats(const struct tegra_mc *mc, + struct tegra20_mc_client_stat *stats) +{ + const struct tegra_mc_client *client0, *client1; + unsigned int i; + + /* collect memory controller utilization percent for each client */ + for (i = 0; i < mc->soc->num_clients; i += 2) { + client0 = &mc->soc->clients[i]; + client1 = &mc->soc->clients[i + 1]; + + if (i + 1 == mc->soc->num_clients) + client1 = NULL; + + tegra20_mc_stat_events(mc, client0, client1, + MC_STAT_CONTROL_FILTER_PRI_DISABLE, + MC_STAT_CONTROL_PRI_EVENT_HP, + MC_STAT_CONTROL_EVENT_QUALIFIED, + &stats[i + 0].events, + &stats[i + 1].events); + } + + /* collect more info from active clients */ + for (i = 0; i < mc->soc->num_clients; i++) { + unsigned int clienta, clientb = mc->soc->num_clients; + + for (client0 = NULL; i < mc->soc->num_clients; i++) { + if (stats[i].events) { + client0 = &mc->soc->clients[i]; + clienta = i++; + break; + } + } + + for (client1 = NULL; i < mc->soc->num_clients; i++) { + if (stats[i].events) { + client1 = &mc->soc->clients[i]; + clientb = i; + break; + } + } + + if (!client0 && !client1) + break; + + tegra20_mc_stat_events(mc, client0, client1, + MC_STAT_CONTROL_FILTER_PRI_YES, + MC_STAT_CONTROL_PRI_EVENT_HP, + MC_STAT_CONTROL_EVENT_QUALIFIED, + &stats[clienta].arb_high_prio, + &stats[clientb].arb_high_prio); + + tegra20_mc_stat_events(mc, client0, client1, + MC_STAT_CONTROL_FILTER_PRI_YES, + MC_STAT_CONTROL_PRI_EVENT_TM, + MC_STAT_CONTROL_EVENT_QUALIFIED, + &stats[clienta].arb_timeout, + &stats[clientb].arb_timeout); + + tegra20_mc_stat_events(mc, client0, client1, + MC_STAT_CONTROL_FILTER_PRI_YES, + MC_STAT_CONTROL_PRI_EVENT_BW, + MC_STAT_CONTROL_EVENT_QUALIFIED, + &stats[clienta].arb_bandwidth, + &stats[clientb].arb_bandwidth); + + tegra20_mc_stat_events(mc, client0, client1, + MC_STAT_CONTROL_FILTER_PRI_DISABLE, + MC_STAT_CONTROL_PRI_EVENT_HP, + MC_STAT_CONTROL_EVENT_RD_WR_CHANGE, + &stats[clienta].rd_wr_change, + &stats[clientb].rd_wr_change); + + tegra20_mc_stat_events(mc, client0, client1, + MC_STAT_CONTROL_FILTER_PRI_DISABLE, + MC_STAT_CONTROL_PRI_EVENT_HP, + MC_STAT_CONTROL_EVENT_SUCCESSIVE, + &stats[clienta].successive, + &stats[clientb].successive); + + tegra20_mc_stat_events(mc, client0, client1, + MC_STAT_CONTROL_FILTER_PRI_DISABLE, + MC_STAT_CONTROL_PRI_EVENT_HP, + MC_STAT_CONTROL_EVENT_PAGE_MISS, + &stats[clienta].page_miss, + &stats[clientb].page_miss); + } +} + +static void tegra20_mc_printf_percents(struct seq_file *s, + const char *fmt, + unsigned int percents_fx) +{ + char percents_str[8]; + + snprintf(percents_str, ARRAY_SIZE(percents_str), "%3u.%02u%%", + percents_fx / MC_FX_FRAC_SCALE, percents_fx % MC_FX_FRAC_SCALE); + + seq_printf(s, fmt, percents_str); +} + +static int tegra20_mc_stats_show(struct seq_file *s, void *unused) +{ + const struct tegra_mc *mc = dev_get_drvdata(s->private); + struct tegra20_mc_client_stat *stats; + unsigned int i; + + stats = kcalloc(mc->soc->num_clients + 1, sizeof(*stats), GFP_KERNEL); + if (!stats) + return -ENOMEM; + + mutex_lock(&tegra20_mc_stat_lock); + + tegra20_mc_collect_stats(mc, stats); + + mutex_unlock(&tegra20_mc_stat_lock); + + seq_puts(s, "Memory client Events Timeout High priority Bandwidth ARB RW change Successive Page miss\n"); + seq_puts(s, "-----------------------------------------------------------------------------------------------------\n"); + + for (i = 0; i < mc->soc->num_clients; i++) { + seq_printf(s, "%-14s ", mc->soc->clients[i].name); + + /* An event is generated when client performs R/W request. */ + tegra20_mc_printf_percents(s, "%-9s", stats[i].events); + + /* + * An event is generated based on the timeout (TM) signal + * accompanying a request for arbitration. + */ + tegra20_mc_printf_percents(s, "%-10s", stats[i].arb_timeout); + + /* + * An event is generated based on the high-priority (HP) signal + * accompanying a request for arbitration. + */ + tegra20_mc_printf_percents(s, "%-16s", stats[i].arb_high_prio); + + /* + * An event is generated based on the bandwidth (BW) signal + * accompanying a request for arbitration. + */ + tegra20_mc_printf_percents(s, "%-16s", stats[i].arb_bandwidth); + + /* + * An event is generated when the memory controller switches + * between making a read request to making a write request. + */ + tegra20_mc_printf_percents(s, "%-12s", stats[i].rd_wr_change); + + /* + * An even generated when the chosen client has wins arbitration + * when it was also the winner at the previous request. If a + * client makes N requests in a row that are honored, SUCCESSIVE + * will be counted (N-1) times. Large values for this event + * imply that if we were patient enough, all of those requests + * could have been coalesced. + */ + tegra20_mc_printf_percents(s, "%-13s", stats[i].successive); + + /* + * An event is generated when the memory controller detects a + * page miss for the current request. + */ + tegra20_mc_printf_percents(s, "%-12s\n", stats[i].page_miss); + } + + kfree(stats); + + return 0; +} + +static int tegra20_mc_probe(struct tegra_mc *mc) +{ + debugfs_create_devm_seqfile(mc->dev, "stats", mc->debugfs.root, + tegra20_mc_stats_show); + + return 0; +} + +static int tegra20_mc_suspend(struct tegra_mc *mc) +{ + int err; + + if (IS_ENABLED(CONFIG_TEGRA_IOMMU_GART) && mc->gart) { + err = tegra_gart_suspend(mc->gart); + if (err < 0) + return err; + } + + return 0; +} + +static int tegra20_mc_resume(struct tegra_mc *mc) +{ + int err; + + if (IS_ENABLED(CONFIG_TEGRA_IOMMU_GART) && mc->gart) { + err = tegra_gart_resume(mc->gart); + if (err < 0) + return err; + } + + return 0; +} + +static irqreturn_t tegra20_mc_handle_irq(int irq, void *data) +{ + struct tegra_mc *mc = data; + unsigned long status; + unsigned int bit; + + /* mask all interrupts to avoid flooding */ + status = mc_readl(mc, MC_INTSTATUS) & mc->soc->intmask; + if (!status) + return IRQ_NONE; + + for_each_set_bit(bit, &status, 32) { + const char *error = tegra_mc_status_names[bit]; + const char *direction = "read", *secure = ""; + const char *client, *desc; + phys_addr_t addr; + u32 value, reg; + u8 id, type; + + switch (BIT(bit)) { + case MC_INT_DECERR_EMEM: + reg = MC_DECERR_EMEM_OTHERS_STATUS; + value = mc_readl(mc, reg); + + id = value & mc->soc->client_id_mask; + desc = tegra_mc_error_names[2]; + + if (value & BIT(31)) + direction = "write"; + break; + + case MC_INT_INVALID_GART_PAGE: + reg = MC_GART_ERROR_REQ; + value = mc_readl(mc, reg); + + id = (value >> 1) & mc->soc->client_id_mask; + desc = tegra_mc_error_names[2]; + + if (value & BIT(0)) + direction = "write"; + break; + + case MC_INT_SECURITY_VIOLATION: + reg = MC_SECURITY_VIOLATION_STATUS; + value = mc_readl(mc, reg); + + id = value & mc->soc->client_id_mask; + type = (value & BIT(30)) ? 4 : 3; + desc = tegra_mc_error_names[type]; + secure = "secure "; + + if (value & BIT(31)) + direction = "write"; + break; + + default: + continue; + } + + client = mc->soc->clients[id].name; + addr = mc_readl(mc, reg + sizeof(u32)); + + dev_err_ratelimited(mc->dev, "%s: %s%s @%pa: %s (%s)\n", + client, secure, direction, &addr, error, + desc); + } + + /* clear interrupts */ + mc_writel(mc, status, MC_INTSTATUS); + + return IRQ_HANDLED; +} + +static const struct tegra_mc_ops tegra20_mc_ops = { + .probe = tegra20_mc_probe, + .suspend = tegra20_mc_suspend, + .resume = tegra20_mc_resume, + .handle_irq = tegra20_mc_handle_irq, +}; + const struct tegra_mc_soc tegra20_mc_soc = { .clients = tegra20_mc_clients, .num_clients = ARRAY_SIZE(tegra20_mc_clients), @@ -290,4 +803,6 @@ const struct tegra_mc_soc tegra20_mc_soc = { .reset_ops = &tegra20_mc_reset_ops, .resets = tegra20_mc_resets, .num_resets = ARRAY_SIZE(tegra20_mc_resets), + .icc_ops = &tegra20_mc_icc_ops, + .ops = &tegra20_mc_ops, }; diff --git a/drivers/memory/tegra/tegra210-emc-cc-r21021.c b/drivers/memory/tegra/tegra210-emc-cc-r21021.c new file mode 100644 index 000000000000..cc76adb8d7e8 --- /dev/null +++ b/drivers/memory/tegra/tegra210-emc-cc-r21021.c @@ -0,0 +1,1774 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2014-2020, NVIDIA CORPORATION. All rights reserved. + */ + +#include <linux/kernel.h> +#include <linux/io.h> +#include <linux/clk.h> +#include <linux/delay.h> +#include <linux/of.h> + +#include <soc/tegra/mc.h> + +#include "tegra210-emc.h" +#include "tegra210-mc.h" + +/* + * Enable flags for specifying verbosity. + */ +#define INFO (1 << 0) +#define STEPS (1 << 1) +#define SUB_STEPS (1 << 2) +#define PRELOCK (1 << 3) +#define PRELOCK_STEPS (1 << 4) +#define ACTIVE_EN (1 << 5) +#define PRAMP_UP (1 << 6) +#define PRAMP_DN (1 << 7) +#define EMA_WRITES (1 << 10) +#define EMA_UPDATES (1 << 11) +#define PER_TRAIN (1 << 16) +#define CC_PRINT (1 << 17) +#define CCFIFO (1 << 29) +#define REGS (1 << 30) +#define REG_LISTS (1 << 31) + +#define emc_dbg(emc, flags, ...) dev_dbg(emc->dev, __VA_ARGS__) + +#define DVFS_CLOCK_CHANGE_VERSION 21021 +#define EMC_PRELOCK_VERSION 2101 + +enum { + DVFS_SEQUENCE = 1, + WRITE_TRAINING_SEQUENCE = 2, + PERIODIC_TRAINING_SEQUENCE = 3, + DVFS_PT1 = 10, + DVFS_UPDATE = 11, + TRAINING_PT1 = 12, + TRAINING_UPDATE = 13, + PERIODIC_TRAINING_UPDATE = 14 +}; + +/* + * PTFV defines - basically just indexes into the per table PTFV array. + */ +#define PTFV_DQSOSC_MOVAVG_C0D0U0_INDEX 0 +#define PTFV_DQSOSC_MOVAVG_C0D0U1_INDEX 1 +#define PTFV_DQSOSC_MOVAVG_C0D1U0_INDEX 2 +#define PTFV_DQSOSC_MOVAVG_C0D1U1_INDEX 3 +#define PTFV_DQSOSC_MOVAVG_C1D0U0_INDEX 4 +#define PTFV_DQSOSC_MOVAVG_C1D0U1_INDEX 5 +#define PTFV_DQSOSC_MOVAVG_C1D1U0_INDEX 6 +#define PTFV_DQSOSC_MOVAVG_C1D1U1_INDEX 7 +#define PTFV_DVFS_SAMPLES_INDEX 9 +#define PTFV_MOVAVG_WEIGHT_INDEX 10 +#define PTFV_CONFIG_CTRL_INDEX 11 + +#define PTFV_CONFIG_CTRL_USE_PREVIOUS_EMA (1 << 0) + +/* + * Do arithmetic in fixed point. + */ +#define MOVAVG_PRECISION_FACTOR 100 + +/* + * The division portion of the average operation. + */ +#define __AVERAGE_PTFV(dev) \ + ({ next->ptfv_list[PTFV_DQSOSC_MOVAVG_ ## dev ## _INDEX] = \ + next->ptfv_list[PTFV_DQSOSC_MOVAVG_ ## dev ## _INDEX] / \ + next->ptfv_list[PTFV_DVFS_SAMPLES_INDEX]; }) + +/* + * Convert val to fixed point and add it to the temporary average. + */ +#define __INCREMENT_PTFV(dev, val) \ + ({ next->ptfv_list[PTFV_DQSOSC_MOVAVG_ ## dev ## _INDEX] += \ + ((val) * MOVAVG_PRECISION_FACTOR); }) + +/* + * Convert a moving average back to integral form and return the value. + */ +#define __MOVAVG_AC(timing, dev) \ + ((timing)->ptfv_list[PTFV_DQSOSC_MOVAVG_ ## dev ## _INDEX] / \ + MOVAVG_PRECISION_FACTOR) + +/* Weighted update. */ +#define __WEIGHTED_UPDATE_PTFV(dev, nval) \ + do { \ + int w = PTFV_MOVAVG_WEIGHT_INDEX; \ + int dqs = PTFV_DQSOSC_MOVAVG_ ## dev ## _INDEX; \ + \ + next->ptfv_list[dqs] = \ + ((nval * MOVAVG_PRECISION_FACTOR) + \ + (next->ptfv_list[dqs] * \ + next->ptfv_list[w])) / \ + (next->ptfv_list[w] + 1); \ + \ + emc_dbg(emc, EMA_UPDATES, "%s: (s=%lu) EMA: %u\n", \ + __stringify(dev), nval, next->ptfv_list[dqs]); \ + } while (0) + +/* Access a particular average. */ +#define __MOVAVG(timing, dev) \ + ((timing)->ptfv_list[PTFV_DQSOSC_MOVAVG_ ## dev ## _INDEX]) + +static u32 update_clock_tree_delay(struct tegra210_emc *emc, int type) +{ + bool periodic_training_update = type == PERIODIC_TRAINING_UPDATE; + struct tegra210_emc_timing *last = emc->last; + struct tegra210_emc_timing *next = emc->next; + u32 last_timing_rate_mhz = last->rate / 1000; + u32 next_timing_rate_mhz = next->rate / 1000; + bool dvfs_update = type == DVFS_UPDATE; + s32 tdel = 0, tmdel = 0, adel = 0; + bool dvfs_pt1 = type == DVFS_PT1; + unsigned long cval = 0; + u32 temp[2][2], value; + unsigned int i; + + /* + * Dev0 MSB. + */ + if (dvfs_pt1 || periodic_training_update) { + value = tegra210_emc_mrr_read(emc, 2, 19); + + for (i = 0; i < emc->num_channels; i++) { + temp[i][0] = (value & 0x00ff) << 8; + temp[i][1] = (value & 0xff00) << 0; + value >>= 16; + } + + /* + * Dev0 LSB. + */ + value = tegra210_emc_mrr_read(emc, 2, 18); + + for (i = 0; i < emc->num_channels; i++) { + temp[i][0] |= (value & 0x00ff) >> 0; + temp[i][1] |= (value & 0xff00) >> 8; + value >>= 16; + } + } + + if (dvfs_pt1 || periodic_training_update) { + cval = tegra210_emc_actual_osc_clocks(last->run_clocks); + cval *= 1000000; + cval /= last_timing_rate_mhz * 2 * temp[0][0]; + } + + if (dvfs_pt1) + __INCREMENT_PTFV(C0D0U0, cval); + else if (dvfs_update) + __AVERAGE_PTFV(C0D0U0); + else if (periodic_training_update) + __WEIGHTED_UPDATE_PTFV(C0D0U0, cval); + + if (dvfs_update || periodic_training_update) { + tdel = next->current_dram_clktree[C0D0U0] - + __MOVAVG_AC(next, C0D0U0); + tmdel = (tdel < 0) ? -1 * tdel : tdel; + adel = tmdel; + + if (tmdel * 128 * next_timing_rate_mhz / 1000000 > + next->tree_margin) + next->current_dram_clktree[C0D0U0] = + __MOVAVG_AC(next, C0D0U0); + } + + if (dvfs_pt1 || periodic_training_update) { + cval = tegra210_emc_actual_osc_clocks(last->run_clocks); + cval *= 1000000; + cval /= last_timing_rate_mhz * 2 * temp[0][1]; + } + + if (dvfs_pt1) + __INCREMENT_PTFV(C0D0U1, cval); + else if (dvfs_update) + __AVERAGE_PTFV(C0D0U1); + else if (periodic_training_update) + __WEIGHTED_UPDATE_PTFV(C0D0U1, cval); + + if (dvfs_update || periodic_training_update) { + tdel = next->current_dram_clktree[C0D0U1] - + __MOVAVG_AC(next, C0D0U1); + tmdel = (tdel < 0) ? -1 * tdel : tdel; + + if (tmdel > adel) + adel = tmdel; + + if (tmdel * 128 * next_timing_rate_mhz / 1000000 > + next->tree_margin) + next->current_dram_clktree[C0D0U1] = + __MOVAVG_AC(next, C0D0U1); + } + + if (emc->num_channels > 1) { + if (dvfs_pt1 || periodic_training_update) { + cval = tegra210_emc_actual_osc_clocks(last->run_clocks); + cval *= 1000000; + cval /= last_timing_rate_mhz * 2 * temp[1][0]; + } + + if (dvfs_pt1) + __INCREMENT_PTFV(C1D0U0, cval); + else if (dvfs_update) + __AVERAGE_PTFV(C1D0U0); + else if (periodic_training_update) + __WEIGHTED_UPDATE_PTFV(C1D0U0, cval); + + if (dvfs_update || periodic_training_update) { + tdel = next->current_dram_clktree[C1D0U0] - + __MOVAVG_AC(next, C1D0U0); + tmdel = (tdel < 0) ? -1 * tdel : tdel; + + if (tmdel > adel) + adel = tmdel; + + if (tmdel * 128 * next_timing_rate_mhz / 1000000 > + next->tree_margin) + next->current_dram_clktree[C1D0U0] = + __MOVAVG_AC(next, C1D0U0); + } + + if (dvfs_pt1 || periodic_training_update) { + cval = tegra210_emc_actual_osc_clocks(last->run_clocks); + cval *= 1000000; + cval /= last_timing_rate_mhz * 2 * temp[1][1]; + } + + if (dvfs_pt1) + __INCREMENT_PTFV(C1D0U1, cval); + else if (dvfs_update) + __AVERAGE_PTFV(C1D0U1); + else if (periodic_training_update) + __WEIGHTED_UPDATE_PTFV(C1D0U1, cval); + + if (dvfs_update || periodic_training_update) { + tdel = next->current_dram_clktree[C1D0U1] - + __MOVAVG_AC(next, C1D0U1); + tmdel = (tdel < 0) ? -1 * tdel : tdel; + + if (tmdel > adel) + adel = tmdel; + + if (tmdel * 128 * next_timing_rate_mhz / 1000000 > + next->tree_margin) + next->current_dram_clktree[C1D0U1] = + __MOVAVG_AC(next, C1D0U1); + } + } + + if (emc->num_devices < 2) + goto done; + + /* + * Dev1 MSB. + */ + if (dvfs_pt1 || periodic_training_update) { + value = tegra210_emc_mrr_read(emc, 1, 19); + + for (i = 0; i < emc->num_channels; i++) { + temp[i][0] = (value & 0x00ff) << 8; + temp[i][1] = (value & 0xff00) << 0; + value >>= 16; + } + + /* + * Dev1 LSB. + */ + value = tegra210_emc_mrr_read(emc, 2, 18); + + for (i = 0; i < emc->num_channels; i++) { + temp[i][0] |= (value & 0x00ff) >> 0; + temp[i][1] |= (value & 0xff00) >> 8; + value >>= 16; + } + } + + if (dvfs_pt1 || periodic_training_update) { + cval = tegra210_emc_actual_osc_clocks(last->run_clocks); + cval *= 1000000; + cval /= last_timing_rate_mhz * 2 * temp[0][0]; + } + + if (dvfs_pt1) + __INCREMENT_PTFV(C0D1U0, cval); + else if (dvfs_update) + __AVERAGE_PTFV(C0D1U0); + else if (periodic_training_update) + __WEIGHTED_UPDATE_PTFV(C0D1U0, cval); + + if (dvfs_update || periodic_training_update) { + tdel = next->current_dram_clktree[C0D1U0] - + __MOVAVG_AC(next, C0D1U0); + tmdel = (tdel < 0) ? -1 * tdel : tdel; + + if (tmdel > adel) + adel = tmdel; + + if (tmdel * 128 * next_timing_rate_mhz / 1000000 > + next->tree_margin) + next->current_dram_clktree[C0D1U0] = + __MOVAVG_AC(next, C0D1U0); + } + + if (dvfs_pt1 || periodic_training_update) { + cval = tegra210_emc_actual_osc_clocks(last->run_clocks); + cval *= 1000000; + cval /= last_timing_rate_mhz * 2 * temp[0][1]; + } + + if (dvfs_pt1) + __INCREMENT_PTFV(C0D1U1, cval); + else if (dvfs_update) + __AVERAGE_PTFV(C0D1U1); + else if (periodic_training_update) + __WEIGHTED_UPDATE_PTFV(C0D1U1, cval); + + if (dvfs_update || periodic_training_update) { + tdel = next->current_dram_clktree[C0D1U1] - + __MOVAVG_AC(next, C0D1U1); + tmdel = (tdel < 0) ? -1 * tdel : tdel; + + if (tmdel > adel) + adel = tmdel; + + if (tmdel * 128 * next_timing_rate_mhz / 1000000 > + next->tree_margin) + next->current_dram_clktree[C0D1U1] = + __MOVAVG_AC(next, C0D1U1); + } + + if (emc->num_channels > 1) { + if (dvfs_pt1 || periodic_training_update) { + cval = tegra210_emc_actual_osc_clocks(last->run_clocks); + cval *= 1000000; + cval /= last_timing_rate_mhz * 2 * temp[1][0]; + } + + if (dvfs_pt1) + __INCREMENT_PTFV(C1D1U0, cval); + else if (dvfs_update) + __AVERAGE_PTFV(C1D1U0); + else if (periodic_training_update) + __WEIGHTED_UPDATE_PTFV(C1D1U0, cval); + + if (dvfs_update || periodic_training_update) { + tdel = next->current_dram_clktree[C1D1U0] - + __MOVAVG_AC(next, C1D1U0); + tmdel = (tdel < 0) ? -1 * tdel : tdel; + + if (tmdel > adel) + adel = tmdel; + + if (tmdel * 128 * next_timing_rate_mhz / 1000000 > + next->tree_margin) + next->current_dram_clktree[C1D1U0] = + __MOVAVG_AC(next, C1D1U0); + } + + if (dvfs_pt1 || periodic_training_update) { + cval = tegra210_emc_actual_osc_clocks(last->run_clocks); + cval *= 1000000; + cval /= last_timing_rate_mhz * 2 * temp[1][1]; + } + + if (dvfs_pt1) + __INCREMENT_PTFV(C1D1U1, cval); + else if (dvfs_update) + __AVERAGE_PTFV(C1D1U1); + else if (periodic_training_update) + __WEIGHTED_UPDATE_PTFV(C1D1U1, cval); + + if (dvfs_update || periodic_training_update) { + tdel = next->current_dram_clktree[C1D1U1] - + __MOVAVG_AC(next, C1D1U1); + tmdel = (tdel < 0) ? -1 * tdel : tdel; + + if (tmdel > adel) + adel = tmdel; + + if (tmdel * 128 * next_timing_rate_mhz / 1000000 > + next->tree_margin) + next->current_dram_clktree[C1D1U1] = + __MOVAVG_AC(next, C1D1U1); + } + } + +done: + return adel; +} + +static u32 periodic_compensation_handler(struct tegra210_emc *emc, u32 type, + struct tegra210_emc_timing *last, + struct tegra210_emc_timing *next) +{ +#define __COPY_EMA(nt, lt, dev) \ + ({ __MOVAVG(nt, dev) = __MOVAVG(lt, dev) * \ + (nt)->ptfv_list[PTFV_DVFS_SAMPLES_INDEX]; }) + + u32 i, adel = 0, samples = next->ptfv_list[PTFV_DVFS_SAMPLES_INDEX]; + u32 delay; + + delay = tegra210_emc_actual_osc_clocks(last->run_clocks); + delay *= 1000; + delay = 2 + (delay / last->rate); + + if (!next->periodic_training) + return 0; + + if (type == DVFS_SEQUENCE) { + if (last->periodic_training && + (next->ptfv_list[PTFV_CONFIG_CTRL_INDEX] & + PTFV_CONFIG_CTRL_USE_PREVIOUS_EMA)) { + /* + * If the previous frequency was using periodic + * calibration then we can reuse the previous + * frequencies EMA data. + */ + __COPY_EMA(next, last, C0D0U0); + __COPY_EMA(next, last, C0D0U1); + __COPY_EMA(next, last, C1D0U0); + __COPY_EMA(next, last, C1D0U1); + __COPY_EMA(next, last, C0D1U0); + __COPY_EMA(next, last, C0D1U1); + __COPY_EMA(next, last, C1D1U0); + __COPY_EMA(next, last, C1D1U1); + } else { + /* Reset the EMA.*/ + __MOVAVG(next, C0D0U0) = 0; + __MOVAVG(next, C0D0U1) = 0; + __MOVAVG(next, C1D0U0) = 0; + __MOVAVG(next, C1D0U1) = 0; + __MOVAVG(next, C0D1U0) = 0; + __MOVAVG(next, C0D1U1) = 0; + __MOVAVG(next, C1D1U0) = 0; + __MOVAVG(next, C1D1U1) = 0; + + for (i = 0; i < samples; i++) { + tegra210_emc_start_periodic_compensation(emc); + udelay(delay); + + /* + * Generate next sample of data. + */ + adel = update_clock_tree_delay(emc, DVFS_PT1); + } + } + + /* + * Seems like it should be part of the + * 'if (last_timing->periodic_training)' conditional + * since is already done for the else clause. + */ + adel = update_clock_tree_delay(emc, DVFS_UPDATE); + } + + if (type == PERIODIC_TRAINING_SEQUENCE) { + tegra210_emc_start_periodic_compensation(emc); + udelay(delay); + + adel = update_clock_tree_delay(emc, PERIODIC_TRAINING_UPDATE); + } + + return adel; +} + +static u32 tegra210_emc_r21021_periodic_compensation(struct tegra210_emc *emc) +{ + u32 emc_cfg, emc_cfg_o, emc_cfg_update, del, value; + static const u32 list[] = { + EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_0, + EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_1, + EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_2, + EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_3, + EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_0, + EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_1, + EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_2, + EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_3, + EMC_DATA_BRLSHFT_0, + EMC_DATA_BRLSHFT_1 + }; + struct tegra210_emc_timing *last = emc->last; + unsigned int items = ARRAY_SIZE(list), i; + unsigned long delay; + + if (last->periodic_training) { + emc_dbg(emc, PER_TRAIN, "Periodic training starting\n"); + + value = emc_readl(emc, EMC_DBG); + emc_cfg_o = emc_readl(emc, EMC_CFG); + emc_cfg = emc_cfg_o & ~(EMC_CFG_DYN_SELF_REF | + EMC_CFG_DRAM_ACPD | + EMC_CFG_DRAM_CLKSTOP_PD); + + + /* + * 1. Power optimizations should be off. + */ + emc_writel(emc, emc_cfg, EMC_CFG); + + /* Does emc_timing_update() for above changes. */ + tegra210_emc_dll_disable(emc); + + for (i = 0; i < emc->num_channels; i++) + tegra210_emc_wait_for_update(emc, i, EMC_EMC_STATUS, + EMC_EMC_STATUS_DRAM_IN_POWERDOWN_MASK, + 0); + + for (i = 0; i < emc->num_channels; i++) + tegra210_emc_wait_for_update(emc, i, EMC_EMC_STATUS, + EMC_EMC_STATUS_DRAM_IN_SELF_REFRESH_MASK, + 0); + + emc_cfg_update = value = emc_readl(emc, EMC_CFG_UPDATE); + value &= ~EMC_CFG_UPDATE_UPDATE_DLL_IN_UPDATE_MASK; + value |= (2 << EMC_CFG_UPDATE_UPDATE_DLL_IN_UPDATE_SHIFT); + emc_writel(emc, value, EMC_CFG_UPDATE); + + /* + * 2. osc kick off - this assumes training and dvfs have set + * correct MR23. + */ + tegra210_emc_start_periodic_compensation(emc); + + /* + * 3. Let dram capture its clock tree delays. + */ + delay = tegra210_emc_actual_osc_clocks(last->run_clocks); + delay *= 1000; + delay /= last->rate + 1; + udelay(delay); + + /* + * 4. Check delta wrt previous values (save value if margin + * exceeds what is set in table). + */ + del = periodic_compensation_handler(emc, + PERIODIC_TRAINING_SEQUENCE, + last, last); + + /* + * 5. Apply compensation w.r.t. trained values (if clock tree + * has drifted more than the set margin). + */ + if (last->tree_margin < ((del * 128 * (last->rate / 1000)) / 1000000)) { + for (i = 0; i < items; i++) { + value = tegra210_emc_compensate(last, list[i]); + emc_dbg(emc, EMA_WRITES, "0x%08x <= 0x%08x\n", + list[i], value); + emc_writel(emc, value, list[i]); + } + } + + emc_writel(emc, emc_cfg_o, EMC_CFG); + + /* + * 6. Timing update actally applies the new trimmers. + */ + tegra210_emc_timing_update(emc); + + /* 6.1. Restore the UPDATE_DLL_IN_UPDATE field. */ + emc_writel(emc, emc_cfg_update, EMC_CFG_UPDATE); + + /* 6.2. Restore the DLL. */ + tegra210_emc_dll_enable(emc); + } + + return 0; +} + +/* + * Do the clock change sequence. + */ +static void tegra210_emc_r21021_set_clock(struct tegra210_emc *emc, u32 clksrc) +{ + /* state variables */ + static bool fsp_for_next_freq; + /* constant configuration parameters */ + const bool save_restore_clkstop_pd = true; + const u32 zqcal_before_cc_cutoff = 2400; + const bool cya_allow_ref_cc = false; + const bool cya_issue_pc_ref = false; + const bool opt_cc_short_zcal = true; + const bool ref_b4_sref_en = false; + const u32 tZQCAL_lpddr4 = 1000000; + const bool opt_short_zcal = true; + const bool opt_do_sw_qrst = true; + const u32 opt_dvfs_mode = MAN_SR; + /* + * This is the timing table for the source frequency. It does _not_ + * necessarily correspond to the actual timing values in the EMC at the + * moment. If the boot BCT differs from the table then this can happen. + * However, we need it for accessing the dram_timings (which are not + * really registers) array for the current frequency. + */ + struct tegra210_emc_timing *fake, *last = emc->last, *next = emc->next; + u32 tRTM, RP_war, R2P_war, TRPab_war, deltaTWATM, W2P_war, tRPST; + u32 mr13_flip_fspwr, mr13_flip_fspop, ramp_up_wait, ramp_down_wait; + u32 zq_wait_long, zq_latch_dvfs_wait_time, tZQCAL_lpddr4_fc_adj; + u32 emc_auto_cal_config, auto_cal_en, emc_cfg, emc_sel_dpd_ctrl; + u32 tFC_lpddr4 = 1000 * next->dram_timings[T_FC_LPDDR4]; + u32 bg_reg_mode_change, enable_bglp_reg, enable_bg_reg; + bool opt_zcal_en_cc = false, is_lpddr3 = false; + bool compensate_trimmer_applicable = false; + u32 emc_dbg, emc_cfg_pipe_clk, emc_pin; + u32 src_clk_period, dst_clk_period; /* in picoseconds */ + bool shared_zq_resistor = false; + u32 value, dram_type; + u32 opt_dll_mode = 0; + unsigned long delay; + unsigned int i; + + emc_dbg(emc, INFO, "Running clock change.\n"); + + /* XXX fake == last */ + fake = tegra210_emc_find_timing(emc, last->rate * 1000UL); + fsp_for_next_freq = !fsp_for_next_freq; + + value = emc_readl(emc, EMC_FBIO_CFG5) & EMC_FBIO_CFG5_DRAM_TYPE_MASK; + dram_type = value >> EMC_FBIO_CFG5_DRAM_TYPE_SHIFT; + + if (last->burst_regs[EMC_ZCAL_WAIT_CNT_INDEX] & BIT(31)) + shared_zq_resistor = true; + + if ((next->burst_regs[EMC_ZCAL_INTERVAL_INDEX] != 0 && + last->burst_regs[EMC_ZCAL_INTERVAL_INDEX] == 0) || + dram_type == DRAM_TYPE_LPDDR4) + opt_zcal_en_cc = true; + + if (dram_type == DRAM_TYPE_DDR3) + opt_dll_mode = tegra210_emc_get_dll_state(next); + + if ((next->burst_regs[EMC_FBIO_CFG5_INDEX] & BIT(25)) && + (dram_type == DRAM_TYPE_LPDDR2)) + is_lpddr3 = true; + + emc_readl(emc, EMC_CFG); + emc_readl(emc, EMC_AUTO_CAL_CONFIG); + + src_clk_period = 1000000000 / last->rate; + dst_clk_period = 1000000000 / next->rate; + + if (dst_clk_period <= zqcal_before_cc_cutoff) + tZQCAL_lpddr4_fc_adj = tZQCAL_lpddr4 - tFC_lpddr4; + else + tZQCAL_lpddr4_fc_adj = tZQCAL_lpddr4; + + tZQCAL_lpddr4_fc_adj /= dst_clk_period; + + emc_dbg = emc_readl(emc, EMC_DBG); + emc_pin = emc_readl(emc, EMC_PIN); + emc_cfg_pipe_clk = emc_readl(emc, EMC_CFG_PIPE_CLK); + + emc_cfg = next->burst_regs[EMC_CFG_INDEX]; + emc_cfg &= ~(EMC_CFG_DYN_SELF_REF | EMC_CFG_DRAM_ACPD | + EMC_CFG_DRAM_CLKSTOP_SR | EMC_CFG_DRAM_CLKSTOP_PD); + emc_sel_dpd_ctrl = next->emc_sel_dpd_ctrl; + emc_sel_dpd_ctrl &= ~(EMC_SEL_DPD_CTRL_CLK_SEL_DPD_EN | + EMC_SEL_DPD_CTRL_CA_SEL_DPD_EN | + EMC_SEL_DPD_CTRL_RESET_SEL_DPD_EN | + EMC_SEL_DPD_CTRL_ODT_SEL_DPD_EN | + EMC_SEL_DPD_CTRL_DATA_SEL_DPD_EN); + + emc_dbg(emc, INFO, "Clock change version: %d\n", + DVFS_CLOCK_CHANGE_VERSION); + emc_dbg(emc, INFO, "DRAM type = %d\n", dram_type); + emc_dbg(emc, INFO, "DRAM dev #: %u\n", emc->num_devices); + emc_dbg(emc, INFO, "Next EMC clksrc: 0x%08x\n", clksrc); + emc_dbg(emc, INFO, "DLL clksrc: 0x%08x\n", next->dll_clk_src); + emc_dbg(emc, INFO, "last rate: %u, next rate %u\n", last->rate, + next->rate); + emc_dbg(emc, INFO, "last period: %u, next period: %u\n", + src_clk_period, dst_clk_period); + emc_dbg(emc, INFO, " shared_zq_resistor: %d\n", !!shared_zq_resistor); + emc_dbg(emc, INFO, " num_channels: %u\n", emc->num_channels); + emc_dbg(emc, INFO, " opt_dll_mode: %d\n", opt_dll_mode); + + /* + * Step 1: + * Pre DVFS SW sequence. + */ + emc_dbg(emc, STEPS, "Step 1\n"); + emc_dbg(emc, STEPS, "Step 1.1: Disable DLL temporarily.\n"); + + value = emc_readl(emc, EMC_CFG_DIG_DLL); + value &= ~EMC_CFG_DIG_DLL_CFG_DLL_EN; + emc_writel(emc, value, EMC_CFG_DIG_DLL); + + tegra210_emc_timing_update(emc); + + for (i = 0; i < emc->num_channels; i++) + tegra210_emc_wait_for_update(emc, i, EMC_CFG_DIG_DLL, + EMC_CFG_DIG_DLL_CFG_DLL_EN, 0); + + emc_dbg(emc, STEPS, "Step 1.2: Disable AUTOCAL temporarily.\n"); + + emc_auto_cal_config = next->emc_auto_cal_config; + auto_cal_en = emc_auto_cal_config & EMC_AUTO_CAL_CONFIG_AUTO_CAL_ENABLE; + emc_auto_cal_config &= ~EMC_AUTO_CAL_CONFIG_AUTO_CAL_START; + emc_auto_cal_config |= EMC_AUTO_CAL_CONFIG_AUTO_CAL_MEASURE_STALL; + emc_auto_cal_config |= EMC_AUTO_CAL_CONFIG_AUTO_CAL_UPDATE_STALL; + emc_auto_cal_config |= auto_cal_en; + emc_writel(emc, emc_auto_cal_config, EMC_AUTO_CAL_CONFIG); + emc_readl(emc, EMC_AUTO_CAL_CONFIG); /* Flush write. */ + + emc_dbg(emc, STEPS, "Step 1.3: Disable other power features.\n"); + + tegra210_emc_set_shadow_bypass(emc, ACTIVE); + emc_writel(emc, emc_cfg, EMC_CFG); + emc_writel(emc, emc_sel_dpd_ctrl, EMC_SEL_DPD_CTRL); + tegra210_emc_set_shadow_bypass(emc, ASSEMBLY); + + if (next->periodic_training) { + tegra210_emc_reset_dram_clktree_values(next); + + for (i = 0; i < emc->num_channels; i++) + tegra210_emc_wait_for_update(emc, i, EMC_EMC_STATUS, + EMC_EMC_STATUS_DRAM_IN_POWERDOWN_MASK, + 0); + + for (i = 0; i < emc->num_channels; i++) + tegra210_emc_wait_for_update(emc, i, EMC_EMC_STATUS, + EMC_EMC_STATUS_DRAM_IN_SELF_REFRESH_MASK, + 0); + + tegra210_emc_start_periodic_compensation(emc); + + delay = 1000 * tegra210_emc_actual_osc_clocks(last->run_clocks); + udelay((delay / last->rate) + 2); + + value = periodic_compensation_handler(emc, DVFS_SEQUENCE, fake, + next); + value = (value * 128 * next->rate / 1000) / 1000000; + + if (next->periodic_training && value > next->tree_margin) + compensate_trimmer_applicable = true; + } + + emc_writel(emc, EMC_INTSTATUS_CLKCHANGE_COMPLETE, EMC_INTSTATUS); + tegra210_emc_set_shadow_bypass(emc, ACTIVE); + emc_writel(emc, emc_cfg, EMC_CFG); + emc_writel(emc, emc_sel_dpd_ctrl, EMC_SEL_DPD_CTRL); + emc_writel(emc, emc_cfg_pipe_clk | EMC_CFG_PIPE_CLK_CLK_ALWAYS_ON, + EMC_CFG_PIPE_CLK); + emc_writel(emc, next->emc_fdpd_ctrl_cmd_no_ramp & + ~EMC_FDPD_CTRL_CMD_NO_RAMP_CMD_DPD_NO_RAMP_ENABLE, + EMC_FDPD_CTRL_CMD_NO_RAMP); + + bg_reg_mode_change = + ((next->burst_regs[EMC_PMACRO_BG_BIAS_CTRL_0_INDEX] & + EMC_PMACRO_BG_BIAS_CTRL_0_BGLP_E_PWRD) ^ + (last->burst_regs[EMC_PMACRO_BG_BIAS_CTRL_0_INDEX] & + EMC_PMACRO_BG_BIAS_CTRL_0_BGLP_E_PWRD)) || + ((next->burst_regs[EMC_PMACRO_BG_BIAS_CTRL_0_INDEX] & + EMC_PMACRO_BG_BIAS_CTRL_0_BG_E_PWRD) ^ + (last->burst_regs[EMC_PMACRO_BG_BIAS_CTRL_0_INDEX] & + EMC_PMACRO_BG_BIAS_CTRL_0_BG_E_PWRD)); + enable_bglp_reg = + (next->burst_regs[EMC_PMACRO_BG_BIAS_CTRL_0_INDEX] & + EMC_PMACRO_BG_BIAS_CTRL_0_BGLP_E_PWRD) == 0; + enable_bg_reg = + (next->burst_regs[EMC_PMACRO_BG_BIAS_CTRL_0_INDEX] & + EMC_PMACRO_BG_BIAS_CTRL_0_BG_E_PWRD) == 0; + + if (bg_reg_mode_change) { + if (enable_bg_reg) + emc_writel(emc, last->burst_regs + [EMC_PMACRO_BG_BIAS_CTRL_0_INDEX] & + ~EMC_PMACRO_BG_BIAS_CTRL_0_BG_E_PWRD, + EMC_PMACRO_BG_BIAS_CTRL_0); + + if (enable_bglp_reg) + emc_writel(emc, last->burst_regs + [EMC_PMACRO_BG_BIAS_CTRL_0_INDEX] & + ~EMC_PMACRO_BG_BIAS_CTRL_0_BGLP_E_PWRD, + EMC_PMACRO_BG_BIAS_CTRL_0); + } + + /* Check if we need to turn on VREF generator. */ + if ((((last->burst_regs[EMC_PMACRO_DATA_PAD_TX_CTRL_INDEX] & + EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQ_E_IVREF) == 0) && + ((next->burst_regs[EMC_PMACRO_DATA_PAD_TX_CTRL_INDEX] & + EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQ_E_IVREF) == 1)) || + (((last->burst_regs[EMC_PMACRO_DATA_PAD_TX_CTRL_INDEX] & + EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQS_E_IVREF) == 0) && + ((next->burst_regs[EMC_PMACRO_DATA_PAD_TX_CTRL_INDEX] & + EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQS_E_IVREF) != 0))) { + u32 pad_tx_ctrl = + next->burst_regs[EMC_PMACRO_DATA_PAD_TX_CTRL_INDEX]; + u32 last_pad_tx_ctrl = + last->burst_regs[EMC_PMACRO_DATA_PAD_TX_CTRL_INDEX]; + u32 next_dq_e_ivref, next_dqs_e_ivref; + + next_dqs_e_ivref = pad_tx_ctrl & + EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQS_E_IVREF; + next_dq_e_ivref = pad_tx_ctrl & + EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQ_E_IVREF; + value = (last_pad_tx_ctrl & + ~EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQ_E_IVREF & + ~EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQS_E_IVREF) | + next_dq_e_ivref | next_dqs_e_ivref; + emc_writel(emc, value, EMC_PMACRO_DATA_PAD_TX_CTRL); + udelay(1); + } else if (bg_reg_mode_change) { + udelay(1); + } + + tegra210_emc_set_shadow_bypass(emc, ASSEMBLY); + + /* + * Step 2: + * Prelock the DLL. + */ + emc_dbg(emc, STEPS, "Step 2\n"); + + if (next->burst_regs[EMC_CFG_DIG_DLL_INDEX] & + EMC_CFG_DIG_DLL_CFG_DLL_EN) { + emc_dbg(emc, INFO, "Prelock enabled for target frequency.\n"); + value = tegra210_emc_dll_prelock(emc, clksrc); + emc_dbg(emc, INFO, "DLL out: 0x%03x\n", value); + } else { + emc_dbg(emc, INFO, "Disabling DLL for target frequency.\n"); + tegra210_emc_dll_disable(emc); + } + + /* + * Step 3: + * Prepare autocal for the clock change. + */ + emc_dbg(emc, STEPS, "Step 3\n"); + + tegra210_emc_set_shadow_bypass(emc, ACTIVE); + emc_writel(emc, next->emc_auto_cal_config2, EMC_AUTO_CAL_CONFIG2); + emc_writel(emc, next->emc_auto_cal_config3, EMC_AUTO_CAL_CONFIG3); + emc_writel(emc, next->emc_auto_cal_config4, EMC_AUTO_CAL_CONFIG4); + emc_writel(emc, next->emc_auto_cal_config5, EMC_AUTO_CAL_CONFIG5); + emc_writel(emc, next->emc_auto_cal_config6, EMC_AUTO_CAL_CONFIG6); + emc_writel(emc, next->emc_auto_cal_config7, EMC_AUTO_CAL_CONFIG7); + emc_writel(emc, next->emc_auto_cal_config8, EMC_AUTO_CAL_CONFIG8); + tegra210_emc_set_shadow_bypass(emc, ASSEMBLY); + + emc_auto_cal_config |= (EMC_AUTO_CAL_CONFIG_AUTO_CAL_COMPUTE_START | + auto_cal_en); + emc_writel(emc, emc_auto_cal_config, EMC_AUTO_CAL_CONFIG); + + /* + * Step 4: + * Update EMC_CFG. (??) + */ + emc_dbg(emc, STEPS, "Step 4\n"); + + if (src_clk_period > 50000 && dram_type == DRAM_TYPE_LPDDR4) + ccfifo_writel(emc, 1, EMC_SELF_REF, 0); + else + emc_writel(emc, next->emc_cfg_2, EMC_CFG_2); + + /* + * Step 5: + * Prepare reference variables for ZQCAL regs. + */ + emc_dbg(emc, STEPS, "Step 5\n"); + + if (dram_type == DRAM_TYPE_LPDDR4) + zq_wait_long = max((u32)1, div_o3(1000000, dst_clk_period)); + else if (dram_type == DRAM_TYPE_LPDDR2 || is_lpddr3) + zq_wait_long = max(next->min_mrs_wait, + div_o3(360000, dst_clk_period)) + 4; + else if (dram_type == DRAM_TYPE_DDR3) + zq_wait_long = max((u32)256, + div_o3(320000, dst_clk_period) + 2); + else + zq_wait_long = 0; + + /* + * Step 6: + * Training code - removed. + */ + emc_dbg(emc, STEPS, "Step 6\n"); + + /* + * Step 7: + * Program FSP reference registers and send MRWs to new FSPWR. + */ + emc_dbg(emc, STEPS, "Step 7\n"); + emc_dbg(emc, SUB_STEPS, "Step 7.1: Bug 200024907 - Patch RP R2P"); + + /* WAR 200024907 */ + if (dram_type == DRAM_TYPE_LPDDR4) { + u32 nRTP = 16; + + if (src_clk_period >= 1000000 / 1866) /* 535.91 ps */ + nRTP = 14; + + if (src_clk_period >= 1000000 / 1600) /* 625.00 ps */ + nRTP = 12; + + if (src_clk_period >= 1000000 / 1333) /* 750.19 ps */ + nRTP = 10; + + if (src_clk_period >= 1000000 / 1066) /* 938.09 ps */ + nRTP = 8; + + deltaTWATM = max_t(u32, div_o3(7500, src_clk_period), 8); + + /* + * Originally there was a + .5 in the tRPST calculation. + * However since we can't do FP in the kernel and the tRTM + * computation was in a floating point ceiling function, adding + * one to tRTP should be ok. There is no other source of non + * integer values, so the result was always going to be + * something for the form: f_ceil(N + .5) = N + 1; + */ + tRPST = (last->emc_mrw & 0x80) >> 7; + tRTM = fake->dram_timings[RL] + div_o3(3600, src_clk_period) + + max_t(u32, div_o3(7500, src_clk_period), 8) + tRPST + + 1 + nRTP; + + emc_dbg(emc, INFO, "tRTM = %u, EMC_RP = %u\n", tRTM, + next->burst_regs[EMC_RP_INDEX]); + + if (last->burst_regs[EMC_RP_INDEX] < tRTM) { + if (tRTM > (last->burst_regs[EMC_R2P_INDEX] + + last->burst_regs[EMC_RP_INDEX])) { + R2P_war = tRTM - last->burst_regs[EMC_RP_INDEX]; + RP_war = last->burst_regs[EMC_RP_INDEX]; + TRPab_war = last->burst_regs[EMC_TRPAB_INDEX]; + + if (R2P_war > 63) { + RP_war = R2P_war + + last->burst_regs[EMC_RP_INDEX] - 63; + + if (TRPab_war < RP_war) + TRPab_war = RP_war; + + R2P_war = 63; + } + } else { + R2P_war = last->burst_regs[EMC_R2P_INDEX]; + RP_war = last->burst_regs[EMC_RP_INDEX]; + TRPab_war = last->burst_regs[EMC_TRPAB_INDEX]; + } + + if (RP_war < deltaTWATM) { + W2P_war = last->burst_regs[EMC_W2P_INDEX] + + deltaTWATM - RP_war; + if (W2P_war > 63) { + RP_war = RP_war + W2P_war - 63; + if (TRPab_war < RP_war) + TRPab_war = RP_war; + W2P_war = 63; + } + } else { + W2P_war = last->burst_regs[ + EMC_W2P_INDEX]; + } + + if ((last->burst_regs[EMC_W2P_INDEX] ^ W2P_war) || + (last->burst_regs[EMC_R2P_INDEX] ^ R2P_war) || + (last->burst_regs[EMC_RP_INDEX] ^ RP_war) || + (last->burst_regs[EMC_TRPAB_INDEX] ^ TRPab_war)) { + emc_writel(emc, RP_war, EMC_RP); + emc_writel(emc, R2P_war, EMC_R2P); + emc_writel(emc, W2P_war, EMC_W2P); + emc_writel(emc, TRPab_war, EMC_TRPAB); + } + + tegra210_emc_timing_update(emc); + } else { + emc_dbg(emc, INFO, "Skipped WAR\n"); + } + } + + if (!fsp_for_next_freq) { + mr13_flip_fspwr = (next->emc_mrw3 & 0xffffff3f) | 0x80; + mr13_flip_fspop = (next->emc_mrw3 & 0xffffff3f) | 0x00; + } else { + mr13_flip_fspwr = (next->emc_mrw3 & 0xffffff3f) | 0x40; + mr13_flip_fspop = (next->emc_mrw3 & 0xffffff3f) | 0xc0; + } + + if (dram_type == DRAM_TYPE_LPDDR4) { + emc_writel(emc, mr13_flip_fspwr, EMC_MRW3); + emc_writel(emc, next->emc_mrw, EMC_MRW); + emc_writel(emc, next->emc_mrw2, EMC_MRW2); + } + + /* + * Step 8: + * Program the shadow registers. + */ + emc_dbg(emc, STEPS, "Step 8\n"); + emc_dbg(emc, SUB_STEPS, "Writing burst_regs\n"); + + for (i = 0; i < next->num_burst; i++) { + const u16 *offsets = emc->offsets->burst; + u16 offset; + + if (!offsets[i]) + continue; + + value = next->burst_regs[i]; + offset = offsets[i]; + + if (dram_type != DRAM_TYPE_LPDDR4 && + (offset == EMC_MRW6 || offset == EMC_MRW7 || + offset == EMC_MRW8 || offset == EMC_MRW9 || + offset == EMC_MRW10 || offset == EMC_MRW11 || + offset == EMC_MRW12 || offset == EMC_MRW13 || + offset == EMC_MRW14 || offset == EMC_MRW15 || + offset == EMC_TRAINING_CTRL)) + continue; + + /* Pain... And suffering. */ + if (offset == EMC_CFG) { + value &= ~EMC_CFG_DRAM_ACPD; + value &= ~EMC_CFG_DYN_SELF_REF; + + if (dram_type == DRAM_TYPE_LPDDR4) { + value &= ~EMC_CFG_DRAM_CLKSTOP_SR; + value &= ~EMC_CFG_DRAM_CLKSTOP_PD; + } + } else if (offset == EMC_MRS_WAIT_CNT && + dram_type == DRAM_TYPE_LPDDR2 && + opt_zcal_en_cc && !opt_cc_short_zcal && + opt_short_zcal) { + value = (value & ~(EMC_MRS_WAIT_CNT_SHORT_WAIT_MASK << + EMC_MRS_WAIT_CNT_SHORT_WAIT_SHIFT)) | + ((zq_wait_long & EMC_MRS_WAIT_CNT_SHORT_WAIT_MASK) << + EMC_MRS_WAIT_CNT_SHORT_WAIT_SHIFT); + } else if (offset == EMC_ZCAL_WAIT_CNT && + dram_type == DRAM_TYPE_DDR3 && opt_zcal_en_cc && + !opt_cc_short_zcal && opt_short_zcal) { + value = (value & ~(EMC_ZCAL_WAIT_CNT_ZCAL_WAIT_CNT_MASK << + EMC_ZCAL_WAIT_CNT_ZCAL_WAIT_CNT_SHIFT)) | + ((zq_wait_long & EMC_ZCAL_WAIT_CNT_ZCAL_WAIT_CNT_MASK) << + EMC_MRS_WAIT_CNT_SHORT_WAIT_SHIFT); + } else if (offset == EMC_ZCAL_INTERVAL && opt_zcal_en_cc) { + value = 0; /* EMC_ZCAL_INTERVAL reset value. */ + } else if (offset == EMC_PMACRO_AUTOCAL_CFG_COMMON) { + value |= EMC_PMACRO_AUTOCAL_CFG_COMMON_E_CAL_BYPASS_DVFS; + } else if (offset == EMC_PMACRO_DATA_PAD_TX_CTRL) { + value &= ~(EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQSP_TX_E_DCC | + EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQSN_TX_E_DCC | + EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQ_TX_E_DCC | + EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_CMD_TX_E_DCC); + } else if (offset == EMC_PMACRO_CMD_PAD_TX_CTRL) { + value |= EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQ_TX_DRVFORCEON; + value &= ~(EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQSP_TX_E_DCC | + EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQSN_TX_E_DCC | + EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQ_TX_E_DCC | + EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_CMD_TX_E_DCC); + } else if (offset == EMC_PMACRO_BRICK_CTRL_RFU1) { + value &= 0xf800f800; + } else if (offset == EMC_PMACRO_COMMON_PAD_TX_CTRL) { + value &= 0xfffffff0; + } + + emc_writel(emc, value, offset); + } + + /* SW addition: do EMC refresh adjustment here. */ + tegra210_emc_adjust_timing(emc, next); + + if (dram_type == DRAM_TYPE_LPDDR4) { + value = (23 << EMC_MRW_MRW_MA_SHIFT) | + (next->run_clocks & EMC_MRW_MRW_OP_MASK); + emc_writel(emc, value, EMC_MRW); + } + + /* Per channel burst registers. */ + emc_dbg(emc, SUB_STEPS, "Writing burst_regs_per_ch\n"); + + for (i = 0; i < next->num_burst_per_ch; i++) { + const struct tegra210_emc_per_channel_regs *burst = + emc->offsets->burst_per_channel; + + if (!burst[i].offset) + continue; + + if (dram_type != DRAM_TYPE_LPDDR4 && + (burst[i].offset == EMC_MRW6 || + burst[i].offset == EMC_MRW7 || + burst[i].offset == EMC_MRW8 || + burst[i].offset == EMC_MRW9 || + burst[i].offset == EMC_MRW10 || + burst[i].offset == EMC_MRW11 || + burst[i].offset == EMC_MRW12 || + burst[i].offset == EMC_MRW13 || + burst[i].offset == EMC_MRW14 || + burst[i].offset == EMC_MRW15)) + continue; + + /* Filter out second channel if not in DUAL_CHANNEL mode. */ + if (emc->num_channels < 2 && burst[i].bank >= 1) + continue; + + emc_dbg(emc, REG_LISTS, "(%u) 0x%08x => 0x%08x\n", i, + next->burst_reg_per_ch[i], burst[i].offset); + emc_channel_writel(emc, burst[i].bank, + next->burst_reg_per_ch[i], + burst[i].offset); + } + + /* Vref regs. */ + emc_dbg(emc, SUB_STEPS, "Writing vref_regs\n"); + + for (i = 0; i < next->vref_num; i++) { + const struct tegra210_emc_per_channel_regs *vref = + emc->offsets->vref_per_channel; + + if (!vref[i].offset) + continue; + + if (emc->num_channels < 2 && vref[i].bank >= 1) + continue; + + emc_dbg(emc, REG_LISTS, "(%u) 0x%08x => 0x%08x\n", i, + next->vref_perch_regs[i], vref[i].offset); + emc_channel_writel(emc, vref[i].bank, next->vref_perch_regs[i], + vref[i].offset); + } + + /* Trimmers. */ + emc_dbg(emc, SUB_STEPS, "Writing trim_regs\n"); + + for (i = 0; i < next->num_trim; i++) { + const u16 *offsets = emc->offsets->trim; + + if (!offsets[i]) + continue; + + if (compensate_trimmer_applicable && + (offsets[i] == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_0 || + offsets[i] == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_1 || + offsets[i] == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_2 || + offsets[i] == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_3 || + offsets[i] == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_0 || + offsets[i] == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_1 || + offsets[i] == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_2 || + offsets[i] == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_3 || + offsets[i] == EMC_DATA_BRLSHFT_0 || + offsets[i] == EMC_DATA_BRLSHFT_1)) { + value = tegra210_emc_compensate(next, offsets[i]); + emc_dbg(emc, REG_LISTS, "(%u) 0x%08x => 0x%08x\n", i, + value, offsets[i]); + emc_dbg(emc, EMA_WRITES, "0x%08x <= 0x%08x\n", + (u32)(u64)offsets[i], value); + emc_writel(emc, value, offsets[i]); + } else { + emc_dbg(emc, REG_LISTS, "(%u) 0x%08x => 0x%08x\n", i, + next->trim_regs[i], offsets[i]); + emc_writel(emc, next->trim_regs[i], offsets[i]); + } + } + + /* Per channel trimmers. */ + emc_dbg(emc, SUB_STEPS, "Writing trim_regs_per_ch\n"); + + for (i = 0; i < next->num_trim_per_ch; i++) { + const struct tegra210_emc_per_channel_regs *trim = + &emc->offsets->trim_per_channel[0]; + unsigned int offset; + + if (!trim[i].offset) + continue; + + if (emc->num_channels < 2 && trim[i].bank >= 1) + continue; + + offset = trim[i].offset; + + if (compensate_trimmer_applicable && + (offset == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_0 || + offset == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_1 || + offset == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_2 || + offset == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_3 || + offset == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_0 || + offset == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_1 || + offset == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_2 || + offset == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_3 || + offset == EMC_DATA_BRLSHFT_0 || + offset == EMC_DATA_BRLSHFT_1)) { + value = tegra210_emc_compensate(next, offset); + emc_dbg(emc, REG_LISTS, "(%u) 0x%08x => 0x%08x\n", i, + value, offset); + emc_dbg(emc, EMA_WRITES, "0x%08x <= 0x%08x\n", offset, + value); + emc_channel_writel(emc, trim[i].bank, value, offset); + } else { + emc_dbg(emc, REG_LISTS, "(%u) 0x%08x => 0x%08x\n", i, + next->trim_perch_regs[i], offset); + emc_channel_writel(emc, trim[i].bank, + next->trim_perch_regs[i], offset); + } + } + + emc_dbg(emc, SUB_STEPS, "Writing burst_mc_regs\n"); + + for (i = 0; i < next->num_mc_regs; i++) { + const u16 *offsets = emc->offsets->burst_mc; + u32 *values = next->burst_mc_regs; + + emc_dbg(emc, REG_LISTS, "(%u) 0x%08x => 0x%08x\n", i, + values[i], offsets[i]); + mc_writel(emc->mc, values[i], offsets[i]); + } + + /* Registers to be programmed on the faster clock. */ + if (next->rate < last->rate) { + const u16 *la = emc->offsets->la_scale; + + emc_dbg(emc, SUB_STEPS, "Writing la_scale_regs\n"); + + for (i = 0; i < next->num_up_down; i++) { + emc_dbg(emc, REG_LISTS, "(%u) 0x%08x => 0x%08x\n", i, + next->la_scale_regs[i], la[i]); + mc_writel(emc->mc, next->la_scale_regs[i], la[i]); + } + } + + /* Flush all the burst register writes. */ + mc_readl(emc->mc, MC_EMEM_ADR_CFG); + + /* + * Step 9: + * LPDDR4 section A. + */ + emc_dbg(emc, STEPS, "Step 9\n"); + + value = next->burst_regs[EMC_ZCAL_WAIT_CNT_INDEX]; + value &= ~EMC_ZCAL_WAIT_CNT_ZCAL_WAIT_CNT_MASK; + + if (dram_type == DRAM_TYPE_LPDDR4) { + emc_writel(emc, 0, EMC_ZCAL_INTERVAL); + emc_writel(emc, value, EMC_ZCAL_WAIT_CNT); + + value = emc_dbg | (EMC_DBG_WRITE_MUX_ACTIVE | + EMC_DBG_WRITE_ACTIVE_ONLY); + + emc_writel(emc, value, EMC_DBG); + emc_writel(emc, 0, EMC_ZCAL_INTERVAL); + emc_writel(emc, emc_dbg, EMC_DBG); + } + + /* + * Step 10: + * LPDDR4 and DDR3 common section. + */ + emc_dbg(emc, STEPS, "Step 10\n"); + + if (opt_dvfs_mode == MAN_SR || dram_type == DRAM_TYPE_LPDDR4) { + if (dram_type == DRAM_TYPE_LPDDR4) + ccfifo_writel(emc, 0x101, EMC_SELF_REF, 0); + else + ccfifo_writel(emc, 0x1, EMC_SELF_REF, 0); + + if (dram_type == DRAM_TYPE_LPDDR4 && + dst_clk_period <= zqcal_before_cc_cutoff) { + ccfifo_writel(emc, mr13_flip_fspwr ^ 0x40, EMC_MRW3, 0); + ccfifo_writel(emc, (next->burst_regs[EMC_MRW6_INDEX] & + 0xFFFF3F3F) | + (last->burst_regs[EMC_MRW6_INDEX] & + 0x0000C0C0), EMC_MRW6, 0); + ccfifo_writel(emc, (next->burst_regs[EMC_MRW14_INDEX] & + 0xFFFF0707) | + (last->burst_regs[EMC_MRW14_INDEX] & + 0x00003838), EMC_MRW14, 0); + + if (emc->num_devices > 1) { + ccfifo_writel(emc, + (next->burst_regs[EMC_MRW7_INDEX] & + 0xFFFF3F3F) | + (last->burst_regs[EMC_MRW7_INDEX] & + 0x0000C0C0), EMC_MRW7, 0); + ccfifo_writel(emc, + (next->burst_regs[EMC_MRW15_INDEX] & + 0xFFFF0707) | + (last->burst_regs[EMC_MRW15_INDEX] & + 0x00003838), EMC_MRW15, 0); + } + + if (opt_zcal_en_cc) { + if (emc->num_devices < 2) + ccfifo_writel(emc, + 2UL << EMC_ZQ_CAL_DEV_SEL_SHIFT + | EMC_ZQ_CAL_ZQ_CAL_CMD, + EMC_ZQ_CAL, 0); + else if (shared_zq_resistor) + ccfifo_writel(emc, + 2UL << EMC_ZQ_CAL_DEV_SEL_SHIFT + | EMC_ZQ_CAL_ZQ_CAL_CMD, + EMC_ZQ_CAL, 0); + else + ccfifo_writel(emc, + EMC_ZQ_CAL_ZQ_CAL_CMD, + EMC_ZQ_CAL, 0); + } + } + } + + if (dram_type == DRAM_TYPE_LPDDR4) { + value = (1000 * fake->dram_timings[T_RP]) / src_clk_period; + ccfifo_writel(emc, mr13_flip_fspop | 0x8, EMC_MRW3, value); + ccfifo_writel(emc, 0, 0, tFC_lpddr4 / src_clk_period); + } + + if (dram_type == DRAM_TYPE_LPDDR4 || opt_dvfs_mode != MAN_SR) { + delay = 30; + + if (cya_allow_ref_cc) { + delay += (1000 * fake->dram_timings[T_RP]) / + src_clk_period; + delay += 4000 * fake->dram_timings[T_RFC]; + } + + ccfifo_writel(emc, emc_pin & ~(EMC_PIN_PIN_CKE_PER_DEV | + EMC_PIN_PIN_CKEB | + EMC_PIN_PIN_CKE), + EMC_PIN, delay); + } + + /* calculate reference delay multiplier */ + value = 1; + + if (ref_b4_sref_en) + value++; + + if (cya_allow_ref_cc) + value++; + + if (cya_issue_pc_ref) + value++; + + if (dram_type != DRAM_TYPE_LPDDR4) { + delay = ((1000 * fake->dram_timings[T_RP] / src_clk_period) + + (1000 * fake->dram_timings[T_RFC] / src_clk_period)); + delay = value * delay + 20; + } else { + delay = 0; + } + + /* + * Step 11: + * Ramp down. + */ + emc_dbg(emc, STEPS, "Step 11\n"); + + ccfifo_writel(emc, 0x0, EMC_CFG_SYNC, delay); + + value = emc_dbg | EMC_DBG_WRITE_MUX_ACTIVE | EMC_DBG_WRITE_ACTIVE_ONLY; + ccfifo_writel(emc, value, EMC_DBG, 0); + + ramp_down_wait = tegra210_emc_dvfs_power_ramp_down(emc, src_clk_period, + 0); + + /* + * Step 12: + * And finally - trigger the clock change. + */ + emc_dbg(emc, STEPS, "Step 12\n"); + + ccfifo_writel(emc, 1, EMC_STALL_THEN_EXE_AFTER_CLKCHANGE, 0); + value &= ~EMC_DBG_WRITE_ACTIVE_ONLY; + ccfifo_writel(emc, value, EMC_DBG, 0); + + /* + * Step 13: + * Ramp up. + */ + emc_dbg(emc, STEPS, "Step 13\n"); + + ramp_up_wait = tegra210_emc_dvfs_power_ramp_up(emc, dst_clk_period, 0); + ccfifo_writel(emc, emc_dbg, EMC_DBG, 0); + + /* + * Step 14: + * Bringup CKE pins. + */ + emc_dbg(emc, STEPS, "Step 14\n"); + + if (dram_type == DRAM_TYPE_LPDDR4) { + value = emc_pin | EMC_PIN_PIN_CKE; + + if (emc->num_devices <= 1) + value &= ~(EMC_PIN_PIN_CKEB | EMC_PIN_PIN_CKE_PER_DEV); + else + value |= EMC_PIN_PIN_CKEB | EMC_PIN_PIN_CKE_PER_DEV; + + ccfifo_writel(emc, value, EMC_PIN, 0); + } + + /* + * Step 15: (two step 15s ??) + * Calculate zqlatch wait time; has dependency on ramping times. + */ + emc_dbg(emc, STEPS, "Step 15\n"); + + if (dst_clk_period <= zqcal_before_cc_cutoff) { + s32 t = (s32)(ramp_up_wait + ramp_down_wait) / + (s32)dst_clk_period; + zq_latch_dvfs_wait_time = (s32)tZQCAL_lpddr4_fc_adj - t; + } else { + zq_latch_dvfs_wait_time = tZQCAL_lpddr4_fc_adj - + div_o3(1000 * next->dram_timings[T_PDEX], + dst_clk_period); + } + + emc_dbg(emc, INFO, "tZQCAL_lpddr4_fc_adj = %u\n", tZQCAL_lpddr4_fc_adj); + emc_dbg(emc, INFO, "dst_clk_period = %u\n", + dst_clk_period); + emc_dbg(emc, INFO, "next->dram_timings[T_PDEX] = %u\n", + next->dram_timings[T_PDEX]); + emc_dbg(emc, INFO, "zq_latch_dvfs_wait_time = %d\n", + max_t(s32, 0, zq_latch_dvfs_wait_time)); + + if (dram_type == DRAM_TYPE_LPDDR4 && opt_zcal_en_cc) { + delay = div_o3(1000 * next->dram_timings[T_PDEX], + dst_clk_period); + + if (emc->num_devices < 2) { + if (dst_clk_period > zqcal_before_cc_cutoff) + ccfifo_writel(emc, + 2UL << EMC_ZQ_CAL_DEV_SEL_SHIFT | + EMC_ZQ_CAL_ZQ_CAL_CMD, EMC_ZQ_CAL, + delay); + + value = (mr13_flip_fspop & 0xfffffff7) | 0x0c000000; + ccfifo_writel(emc, value, EMC_MRW3, delay); + ccfifo_writel(emc, 0, EMC_SELF_REF, 0); + ccfifo_writel(emc, 0, EMC_REF, 0); + ccfifo_writel(emc, 2UL << EMC_ZQ_CAL_DEV_SEL_SHIFT | + EMC_ZQ_CAL_ZQ_LATCH_CMD, + EMC_ZQ_CAL, + max_t(s32, 0, zq_latch_dvfs_wait_time)); + } else if (shared_zq_resistor) { + if (dst_clk_period > zqcal_before_cc_cutoff) + ccfifo_writel(emc, + 2UL << EMC_ZQ_CAL_DEV_SEL_SHIFT | + EMC_ZQ_CAL_ZQ_CAL_CMD, EMC_ZQ_CAL, + delay); + + ccfifo_writel(emc, 2UL << EMC_ZQ_CAL_DEV_SEL_SHIFT | + EMC_ZQ_CAL_ZQ_LATCH_CMD, EMC_ZQ_CAL, + max_t(s32, 0, zq_latch_dvfs_wait_time) + + delay); + ccfifo_writel(emc, 1UL << EMC_ZQ_CAL_DEV_SEL_SHIFT | + EMC_ZQ_CAL_ZQ_LATCH_CMD, + EMC_ZQ_CAL, 0); + + value = (mr13_flip_fspop & 0xfffffff7) | 0x0c000000; + ccfifo_writel(emc, value, EMC_MRW3, 0); + ccfifo_writel(emc, 0, EMC_SELF_REF, 0); + ccfifo_writel(emc, 0, EMC_REF, 0); + + ccfifo_writel(emc, 1UL << EMC_ZQ_CAL_DEV_SEL_SHIFT | + EMC_ZQ_CAL_ZQ_LATCH_CMD, EMC_ZQ_CAL, + tZQCAL_lpddr4 / dst_clk_period); + } else { + if (dst_clk_period > zqcal_before_cc_cutoff) + ccfifo_writel(emc, EMC_ZQ_CAL_ZQ_CAL_CMD, + EMC_ZQ_CAL, delay); + + value = (mr13_flip_fspop & 0xfffffff7) | 0x0c000000; + ccfifo_writel(emc, value, EMC_MRW3, delay); + ccfifo_writel(emc, 0, EMC_SELF_REF, 0); + ccfifo_writel(emc, 0, EMC_REF, 0); + + ccfifo_writel(emc, EMC_ZQ_CAL_ZQ_LATCH_CMD, EMC_ZQ_CAL, + max_t(s32, 0, zq_latch_dvfs_wait_time)); + } + } + + /* WAR: delay for zqlatch */ + ccfifo_writel(emc, 0, 0, 10); + + /* + * Step 16: + * LPDDR4 Conditional Training Kickoff. Removed. + */ + + /* + * Step 17: + * MANSR exit self refresh. + */ + emc_dbg(emc, STEPS, "Step 17\n"); + + if (opt_dvfs_mode == MAN_SR && dram_type != DRAM_TYPE_LPDDR4) + ccfifo_writel(emc, 0, EMC_SELF_REF, 0); + + /* + * Step 18: + * Send MRWs to LPDDR3/DDR3. + */ + emc_dbg(emc, STEPS, "Step 18\n"); + + if (dram_type == DRAM_TYPE_LPDDR2) { + ccfifo_writel(emc, next->emc_mrw2, EMC_MRW2, 0); + ccfifo_writel(emc, next->emc_mrw, EMC_MRW, 0); + if (is_lpddr3) + ccfifo_writel(emc, next->emc_mrw4, EMC_MRW4, 0); + } else if (dram_type == DRAM_TYPE_DDR3) { + if (opt_dll_mode) + ccfifo_writel(emc, next->emc_emrs & + ~EMC_EMRS_USE_EMRS_LONG_CNT, EMC_EMRS, 0); + ccfifo_writel(emc, next->emc_emrs2 & + ~EMC_EMRS2_USE_EMRS2_LONG_CNT, EMC_EMRS2, 0); + ccfifo_writel(emc, next->emc_mrs | + EMC_EMRS_USE_EMRS_LONG_CNT, EMC_MRS, 0); + } + + /* + * Step 19: + * ZQCAL for LPDDR3/DDR3 + */ + emc_dbg(emc, STEPS, "Step 19\n"); + + if (opt_zcal_en_cc) { + if (dram_type == DRAM_TYPE_LPDDR2) { + value = opt_cc_short_zcal ? 90000 : 360000; + value = div_o3(value, dst_clk_period); + value = value << + EMC_MRS_WAIT_CNT2_MRS_EXT2_WAIT_CNT_SHIFT | + value << + EMC_MRS_WAIT_CNT2_MRS_EXT1_WAIT_CNT_SHIFT; + ccfifo_writel(emc, value, EMC_MRS_WAIT_CNT2, 0); + + value = opt_cc_short_zcal ? 0x56 : 0xab; + ccfifo_writel(emc, 2 << EMC_MRW_MRW_DEV_SELECTN_SHIFT | + EMC_MRW_USE_MRW_EXT_CNT | + 10 << EMC_MRW_MRW_MA_SHIFT | + value << EMC_MRW_MRW_OP_SHIFT, + EMC_MRW, 0); + + if (emc->num_devices > 1) { + value = 1 << EMC_MRW_MRW_DEV_SELECTN_SHIFT | + EMC_MRW_USE_MRW_EXT_CNT | + 10 << EMC_MRW_MRW_MA_SHIFT | + value << EMC_MRW_MRW_OP_SHIFT; + ccfifo_writel(emc, value, EMC_MRW, 0); + } + } else if (dram_type == DRAM_TYPE_DDR3) { + value = opt_cc_short_zcal ? 0 : EMC_ZQ_CAL_LONG; + + ccfifo_writel(emc, value | + 2 << EMC_ZQ_CAL_DEV_SEL_SHIFT | + EMC_ZQ_CAL_ZQ_CAL_CMD, EMC_ZQ_CAL, + 0); + + if (emc->num_devices > 1) { + value = value | 1 << EMC_ZQ_CAL_DEV_SEL_SHIFT | + EMC_ZQ_CAL_ZQ_CAL_CMD; + ccfifo_writel(emc, value, EMC_ZQ_CAL, 0); + } + } + } + + if (bg_reg_mode_change) { + tegra210_emc_set_shadow_bypass(emc, ACTIVE); + + if (ramp_up_wait <= 1250000) + delay = (1250000 - ramp_up_wait) / dst_clk_period; + else + delay = 0; + + ccfifo_writel(emc, + next->burst_regs[EMC_PMACRO_BG_BIAS_CTRL_0_INDEX], + EMC_PMACRO_BG_BIAS_CTRL_0, delay); + tegra210_emc_set_shadow_bypass(emc, ASSEMBLY); + } + + /* + * Step 20: + * Issue ref and optional QRST. + */ + emc_dbg(emc, STEPS, "Step 20\n"); + + if (dram_type != DRAM_TYPE_LPDDR4) + ccfifo_writel(emc, 0, EMC_REF, 0); + + if (opt_do_sw_qrst) { + ccfifo_writel(emc, 1, EMC_ISSUE_QRST, 0); + ccfifo_writel(emc, 0, EMC_ISSUE_QRST, 2); + } + + /* + * Step 21: + * Restore ZCAL and ZCAL interval. + */ + emc_dbg(emc, STEPS, "Step 21\n"); + + if (save_restore_clkstop_pd || opt_zcal_en_cc) { + ccfifo_writel(emc, emc_dbg | EMC_DBG_WRITE_MUX_ACTIVE, + EMC_DBG, 0); + if (opt_zcal_en_cc && dram_type != DRAM_TYPE_LPDDR4) + ccfifo_writel(emc, next->burst_regs[EMC_ZCAL_INTERVAL_INDEX], + EMC_ZCAL_INTERVAL, 0); + + if (save_restore_clkstop_pd) + ccfifo_writel(emc, next->burst_regs[EMC_CFG_INDEX] & + ~EMC_CFG_DYN_SELF_REF, + EMC_CFG, 0); + ccfifo_writel(emc, emc_dbg, EMC_DBG, 0); + } + + /* + * Step 22: + * Restore EMC_CFG_PIPE_CLK. + */ + emc_dbg(emc, STEPS, "Step 22\n"); + + ccfifo_writel(emc, emc_cfg_pipe_clk, EMC_CFG_PIPE_CLK, 0); + + if (bg_reg_mode_change) { + if (enable_bg_reg) + emc_writel(emc, + next->burst_regs[EMC_PMACRO_BG_BIAS_CTRL_0_INDEX] & + ~EMC_PMACRO_BG_BIAS_CTRL_0_BGLP_E_PWRD, + EMC_PMACRO_BG_BIAS_CTRL_0); + else + emc_writel(emc, + next->burst_regs[EMC_PMACRO_BG_BIAS_CTRL_0_INDEX] & + ~EMC_PMACRO_BG_BIAS_CTRL_0_BG_E_PWRD, + EMC_PMACRO_BG_BIAS_CTRL_0); + } + + /* + * Step 23: + */ + emc_dbg(emc, STEPS, "Step 23\n"); + + value = emc_readl(emc, EMC_CFG_DIG_DLL); + value |= EMC_CFG_DIG_DLL_CFG_DLL_STALL_ALL_TRAFFIC; + value &= ~EMC_CFG_DIG_DLL_CFG_DLL_STALL_RW_UNTIL_LOCK; + value &= ~EMC_CFG_DIG_DLL_CFG_DLL_STALL_ALL_UNTIL_LOCK; + value &= ~EMC_CFG_DIG_DLL_CFG_DLL_EN; + value = (value & ~EMC_CFG_DIG_DLL_CFG_DLL_MODE_MASK) | + (2 << EMC_CFG_DIG_DLL_CFG_DLL_MODE_SHIFT); + emc_writel(emc, value, EMC_CFG_DIG_DLL); + + tegra210_emc_do_clock_change(emc, clksrc); + + /* + * Step 24: + * Save training results. Removed. + */ + + /* + * Step 25: + * Program MC updown registers. + */ + emc_dbg(emc, STEPS, "Step 25\n"); + + if (next->rate > last->rate) { + for (i = 0; i < next->num_up_down; i++) + mc_writel(emc->mc, next->la_scale_regs[i], + emc->offsets->la_scale[i]); + + tegra210_emc_timing_update(emc); + } + + /* + * Step 26: + * Restore ZCAL registers. + */ + emc_dbg(emc, STEPS, "Step 26\n"); + + if (dram_type == DRAM_TYPE_LPDDR4) { + tegra210_emc_set_shadow_bypass(emc, ACTIVE); + emc_writel(emc, next->burst_regs[EMC_ZCAL_WAIT_CNT_INDEX], + EMC_ZCAL_WAIT_CNT); + emc_writel(emc, next->burst_regs[EMC_ZCAL_INTERVAL_INDEX], + EMC_ZCAL_INTERVAL); + tegra210_emc_set_shadow_bypass(emc, ASSEMBLY); + } + + if (dram_type != DRAM_TYPE_LPDDR4 && opt_zcal_en_cc && + !opt_short_zcal && opt_cc_short_zcal) { + udelay(2); + + tegra210_emc_set_shadow_bypass(emc, ACTIVE); + if (dram_type == DRAM_TYPE_LPDDR2) + emc_writel(emc, next->burst_regs[EMC_MRS_WAIT_CNT_INDEX], + EMC_MRS_WAIT_CNT); + else if (dram_type == DRAM_TYPE_DDR3) + emc_writel(emc, next->burst_regs[EMC_ZCAL_WAIT_CNT_INDEX], + EMC_ZCAL_WAIT_CNT); + tegra210_emc_set_shadow_bypass(emc, ASSEMBLY); + } + + /* + * Step 27: + * Restore EMC_CFG, FDPD registers. + */ + emc_dbg(emc, STEPS, "Step 27\n"); + + tegra210_emc_set_shadow_bypass(emc, ACTIVE); + emc_writel(emc, next->burst_regs[EMC_CFG_INDEX], EMC_CFG); + tegra210_emc_set_shadow_bypass(emc, ASSEMBLY); + emc_writel(emc, next->emc_fdpd_ctrl_cmd_no_ramp, + EMC_FDPD_CTRL_CMD_NO_RAMP); + emc_writel(emc, next->emc_sel_dpd_ctrl, EMC_SEL_DPD_CTRL); + + /* + * Step 28: + * Training recover. Removed. + */ + emc_dbg(emc, STEPS, "Step 28\n"); + + tegra210_emc_set_shadow_bypass(emc, ACTIVE); + emc_writel(emc, + next->burst_regs[EMC_PMACRO_AUTOCAL_CFG_COMMON_INDEX], + EMC_PMACRO_AUTOCAL_CFG_COMMON); + tegra210_emc_set_shadow_bypass(emc, ASSEMBLY); + + /* + * Step 29: + * Power fix WAR. + */ + emc_dbg(emc, STEPS, "Step 29\n"); + + emc_writel(emc, EMC_PMACRO_CFG_PM_GLOBAL_0_DISABLE_CFG_BYTE0 | + EMC_PMACRO_CFG_PM_GLOBAL_0_DISABLE_CFG_BYTE1 | + EMC_PMACRO_CFG_PM_GLOBAL_0_DISABLE_CFG_BYTE2 | + EMC_PMACRO_CFG_PM_GLOBAL_0_DISABLE_CFG_BYTE3 | + EMC_PMACRO_CFG_PM_GLOBAL_0_DISABLE_CFG_BYTE4 | + EMC_PMACRO_CFG_PM_GLOBAL_0_DISABLE_CFG_BYTE5 | + EMC_PMACRO_CFG_PM_GLOBAL_0_DISABLE_CFG_BYTE6 | + EMC_PMACRO_CFG_PM_GLOBAL_0_DISABLE_CFG_BYTE7, + EMC_PMACRO_CFG_PM_GLOBAL_0); + emc_writel(emc, EMC_PMACRO_TRAINING_CTRL_0_CH0_TRAINING_E_WRPTR, + EMC_PMACRO_TRAINING_CTRL_0); + emc_writel(emc, EMC_PMACRO_TRAINING_CTRL_1_CH1_TRAINING_E_WRPTR, + EMC_PMACRO_TRAINING_CTRL_1); + emc_writel(emc, 0, EMC_PMACRO_CFG_PM_GLOBAL_0); + + /* + * Step 30: + * Re-enable autocal. + */ + emc_dbg(emc, STEPS, "Step 30: Re-enable DLL and AUTOCAL\n"); + + if (next->burst_regs[EMC_CFG_DIG_DLL_INDEX] & EMC_CFG_DIG_DLL_CFG_DLL_EN) { + value = emc_readl(emc, EMC_CFG_DIG_DLL); + value |= EMC_CFG_DIG_DLL_CFG_DLL_STALL_ALL_TRAFFIC; + value |= EMC_CFG_DIG_DLL_CFG_DLL_EN; + value &= ~EMC_CFG_DIG_DLL_CFG_DLL_STALL_RW_UNTIL_LOCK; + value &= ~EMC_CFG_DIG_DLL_CFG_DLL_STALL_ALL_UNTIL_LOCK; + value = (value & ~EMC_CFG_DIG_DLL_CFG_DLL_MODE_MASK) | + (2 << EMC_CFG_DIG_DLL_CFG_DLL_MODE_SHIFT); + emc_writel(emc, value, EMC_CFG_DIG_DLL); + tegra210_emc_timing_update(emc); + } + + emc_writel(emc, next->emc_auto_cal_config, EMC_AUTO_CAL_CONFIG); + + /* Done! Yay. */ +} + +const struct tegra210_emc_sequence tegra210_emc_r21021 = { + .revision = 0x7, + .set_clock = tegra210_emc_r21021_set_clock, + .periodic_compensation = tegra210_emc_r21021_periodic_compensation, +}; diff --git a/drivers/memory/tegra/tegra210-emc-core.c b/drivers/memory/tegra/tegra210-emc-core.c new file mode 100644 index 000000000000..cbe1a7723514 --- /dev/null +++ b/drivers/memory/tegra/tegra210-emc-core.c @@ -0,0 +1,2077 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2015-2020, NVIDIA CORPORATION. All rights reserved. + */ + +#include <linux/bitfield.h> +#include <linux/clk.h> +#include <linux/clk/tegra.h> +#include <linux/debugfs.h> +#include <linux/delay.h> +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/of_address.h> +#include <linux/of_platform.h> +#include <linux/of_reserved_mem.h> +#include <linux/slab.h> +#include <linux/thermal.h> +#include <soc/tegra/fuse.h> +#include <soc/tegra/mc.h> + +#include "tegra210-emc.h" +#include "tegra210-mc.h" + +/* CLK_RST_CONTROLLER_CLK_SOURCE_EMC */ +#define EMC_CLK_EMC_2X_CLK_SRC_SHIFT 29 +#define EMC_CLK_EMC_2X_CLK_SRC_MASK \ + (0x7 << EMC_CLK_EMC_2X_CLK_SRC_SHIFT) +#define EMC_CLK_SOURCE_PLLM_LJ 0x4 +#define EMC_CLK_SOURCE_PLLMB_LJ 0x5 +#define EMC_CLK_FORCE_CC_TRIGGER BIT(27) +#define EMC_CLK_MC_EMC_SAME_FREQ BIT(16) +#define EMC_CLK_EMC_2X_CLK_DIVISOR_SHIFT 0 +#define EMC_CLK_EMC_2X_CLK_DIVISOR_MASK \ + (0xff << EMC_CLK_EMC_2X_CLK_DIVISOR_SHIFT) + +/* CLK_RST_CONTROLLER_CLK_SOURCE_EMC_DLL */ +#define DLL_CLK_EMC_DLL_CLK_SRC_SHIFT 29 +#define DLL_CLK_EMC_DLL_CLK_SRC_MASK \ + (0x7 << DLL_CLK_EMC_DLL_CLK_SRC_SHIFT) +#define DLL_CLK_EMC_DLL_DDLL_CLK_SEL_SHIFT 10 +#define DLL_CLK_EMC_DLL_DDLL_CLK_SEL_MASK \ + (0x3 << DLL_CLK_EMC_DLL_DDLL_CLK_SEL_SHIFT) +#define PLLM_VCOA 0 +#define PLLM_VCOB 1 +#define EMC_DLL_SWITCH_OUT 2 +#define DLL_CLK_EMC_DLL_CLK_DIVISOR_SHIFT 0 +#define DLL_CLK_EMC_DLL_CLK_DIVISOR_MASK \ + (0xff << DLL_CLK_EMC_DLL_CLK_DIVISOR_SHIFT) + +/* MC_EMEM_ARB_MISC0 */ +#define MC_EMEM_ARB_MISC0_EMC_SAME_FREQ BIT(27) + +/* EMC_DATA_BRLSHFT_X */ +#define EMC0_EMC_DATA_BRLSHFT_0_INDEX 2 +#define EMC1_EMC_DATA_BRLSHFT_0_INDEX 3 +#define EMC0_EMC_DATA_BRLSHFT_1_INDEX 4 +#define EMC1_EMC_DATA_BRLSHFT_1_INDEX 5 + +#define TRIM_REG(chan, rank, reg, byte) \ + (((EMC_PMACRO_OB_DDLL_LONG_DQ_RANK ## rank ## _ ## reg ## \ + _OB_DDLL_LONG_DQ_RANK ## rank ## _BYTE ## byte ## _MASK & \ + next->trim_regs[EMC_PMACRO_OB_DDLL_LONG_DQ_RANK ## \ + rank ## _ ## reg ## _INDEX]) >> \ + EMC_PMACRO_OB_DDLL_LONG_DQ_RANK ## rank ## _ ## reg ## \ + _OB_DDLL_LONG_DQ_RANK ## rank ## _BYTE ## byte ## _SHIFT) \ + + \ + (((EMC_DATA_BRLSHFT_ ## rank ## _RANK ## rank ## _BYTE ## \ + byte ## _DATA_BRLSHFT_MASK & \ + next->trim_perch_regs[EMC ## chan ## \ + _EMC_DATA_BRLSHFT_ ## rank ## _INDEX]) >> \ + EMC_DATA_BRLSHFT_ ## rank ## _RANK ## rank ## _BYTE ## \ + byte ## _DATA_BRLSHFT_SHIFT) * 64)) + +#define CALC_TEMP(rank, reg, byte1, byte2, n) \ + (((new[n] << EMC_PMACRO_OB_DDLL_LONG_DQ_RANK ## rank ## _ ## \ + reg ## _OB_DDLL_LONG_DQ_RANK ## rank ## _BYTE ## byte1 ## _SHIFT) & \ + EMC_PMACRO_OB_DDLL_LONG_DQ_RANK ## rank ## _ ## reg ## \ + _OB_DDLL_LONG_DQ_RANK ## rank ## _BYTE ## byte1 ## _MASK) \ + | \ + ((new[n + 1] << EMC_PMACRO_OB_DDLL_LONG_DQ_RANK ## rank ## _ ##\ + reg ## _OB_DDLL_LONG_DQ_RANK ## rank ## _BYTE ## byte2 ## _SHIFT) & \ + EMC_PMACRO_OB_DDLL_LONG_DQ_RANK ## rank ## _ ## reg ## \ + _OB_DDLL_LONG_DQ_RANK ## rank ## _BYTE ## byte2 ## _MASK)) + +#define REFRESH_SPEEDUP(value, speedup) \ + (((value) & 0xffff0000) | ((value) & 0xffff) * (speedup)) + +#define LPDDR2_MR4_SRR GENMASK(2, 0) + +static const struct tegra210_emc_sequence *tegra210_emc_sequences[] = { + &tegra210_emc_r21021, +}; + +static const struct tegra210_emc_table_register_offsets +tegra210_emc_table_register_offsets = { + .burst = { + EMC_RC, + EMC_RFC, + EMC_RFCPB, + EMC_REFCTRL2, + EMC_RFC_SLR, + EMC_RAS, + EMC_RP, + EMC_R2W, + EMC_W2R, + EMC_R2P, + EMC_W2P, + EMC_R2R, + EMC_TPPD, + EMC_CCDMW, + EMC_RD_RCD, + EMC_WR_RCD, + EMC_RRD, + EMC_REXT, + EMC_WEXT, + EMC_WDV_CHK, + EMC_WDV, + EMC_WSV, + EMC_WEV, + EMC_WDV_MASK, + EMC_WS_DURATION, + EMC_WE_DURATION, + EMC_QUSE, + EMC_QUSE_WIDTH, + EMC_IBDLY, + EMC_OBDLY, + EMC_EINPUT, + EMC_MRW6, + EMC_EINPUT_DURATION, + EMC_PUTERM_EXTRA, + EMC_PUTERM_WIDTH, + EMC_QRST, + EMC_QSAFE, + EMC_RDV, + EMC_RDV_MASK, + EMC_RDV_EARLY, + EMC_RDV_EARLY_MASK, + EMC_REFRESH, + EMC_BURST_REFRESH_NUM, + EMC_PRE_REFRESH_REQ_CNT, + EMC_PDEX2WR, + EMC_PDEX2RD, + EMC_PCHG2PDEN, + EMC_ACT2PDEN, + EMC_AR2PDEN, + EMC_RW2PDEN, + EMC_CKE2PDEN, + EMC_PDEX2CKE, + EMC_PDEX2MRR, + EMC_TXSR, + EMC_TXSRDLL, + EMC_TCKE, + EMC_TCKESR, + EMC_TPD, + EMC_TFAW, + EMC_TRPAB, + EMC_TCLKSTABLE, + EMC_TCLKSTOP, + EMC_MRW7, + EMC_TREFBW, + EMC_ODT_WRITE, + EMC_FBIO_CFG5, + EMC_FBIO_CFG7, + EMC_CFG_DIG_DLL, + EMC_CFG_DIG_DLL_PERIOD, + EMC_PMACRO_IB_RXRT, + EMC_CFG_PIPE_1, + EMC_CFG_PIPE_2, + EMC_PMACRO_QUSE_DDLL_RANK0_4, + EMC_PMACRO_QUSE_DDLL_RANK0_5, + EMC_PMACRO_QUSE_DDLL_RANK1_4, + EMC_PMACRO_QUSE_DDLL_RANK1_5, + EMC_MRW8, + EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_4, + EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_5, + EMC_PMACRO_OB_DDLL_LONG_DQS_RANK0_0, + EMC_PMACRO_OB_DDLL_LONG_DQS_RANK0_1, + EMC_PMACRO_OB_DDLL_LONG_DQS_RANK0_2, + EMC_PMACRO_OB_DDLL_LONG_DQS_RANK0_3, + EMC_PMACRO_OB_DDLL_LONG_DQS_RANK0_4, + EMC_PMACRO_OB_DDLL_LONG_DQS_RANK0_5, + EMC_PMACRO_OB_DDLL_LONG_DQS_RANK1_0, + EMC_PMACRO_OB_DDLL_LONG_DQS_RANK1_1, + EMC_PMACRO_OB_DDLL_LONG_DQS_RANK1_2, + EMC_PMACRO_OB_DDLL_LONG_DQS_RANK1_3, + EMC_PMACRO_OB_DDLL_LONG_DQS_RANK1_4, + EMC_PMACRO_OB_DDLL_LONG_DQS_RANK1_5, + EMC_PMACRO_DDLL_LONG_CMD_0, + EMC_PMACRO_DDLL_LONG_CMD_1, + EMC_PMACRO_DDLL_LONG_CMD_2, + EMC_PMACRO_DDLL_LONG_CMD_3, + EMC_PMACRO_DDLL_LONG_CMD_4, + EMC_PMACRO_DDLL_SHORT_CMD_0, + EMC_PMACRO_DDLL_SHORT_CMD_1, + EMC_PMACRO_DDLL_SHORT_CMD_2, + EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE0_3, + EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE1_3, + EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE2_3, + EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE3_3, + EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE4_3, + EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE5_3, + EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE6_3, + EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE7_3, + EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD0_3, + EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD1_3, + EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD2_3, + EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD3_3, + EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE0_3, + EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE1_3, + EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE2_3, + EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE3_3, + EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE4_3, + EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE5_3, + EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE6_3, + EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE7_3, + EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_CMD0_0, + EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_CMD0_1, + EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_CMD0_2, + EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_CMD0_3, + EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_CMD1_0, + EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_CMD1_1, + EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_CMD1_2, + EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_CMD1_3, + EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_CMD2_0, + EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_CMD2_1, + EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_CMD2_2, + EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_CMD2_3, + EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_CMD3_0, + EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_CMD3_1, + EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_CMD3_2, + EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_CMD3_3, + EMC_TXDSRVTTGEN, + EMC_FDPD_CTRL_DQ, + EMC_FDPD_CTRL_CMD, + EMC_FBIO_SPARE, + EMC_ZCAL_INTERVAL, + EMC_ZCAL_WAIT_CNT, + EMC_MRS_WAIT_CNT, + EMC_MRS_WAIT_CNT2, + EMC_AUTO_CAL_CHANNEL, + EMC_DLL_CFG_0, + EMC_DLL_CFG_1, + EMC_PMACRO_AUTOCAL_CFG_COMMON, + EMC_PMACRO_ZCTRL, + EMC_CFG, + EMC_CFG_PIPE, + EMC_DYN_SELF_REF_CONTROL, + EMC_QPOP, + EMC_DQS_BRLSHFT_0, + EMC_DQS_BRLSHFT_1, + EMC_CMD_BRLSHFT_2, + EMC_CMD_BRLSHFT_3, + EMC_PMACRO_PAD_CFG_CTRL, + EMC_PMACRO_DATA_PAD_RX_CTRL, + EMC_PMACRO_CMD_PAD_RX_CTRL, + EMC_PMACRO_DATA_RX_TERM_MODE, + EMC_PMACRO_CMD_RX_TERM_MODE, + EMC_PMACRO_CMD_PAD_TX_CTRL, + EMC_PMACRO_DATA_PAD_TX_CTRL, + EMC_PMACRO_COMMON_PAD_TX_CTRL, + EMC_PMACRO_VTTGEN_CTRL_0, + EMC_PMACRO_VTTGEN_CTRL_1, + EMC_PMACRO_VTTGEN_CTRL_2, + EMC_PMACRO_BRICK_CTRL_RFU1, + EMC_PMACRO_CMD_BRICK_CTRL_FDPD, + EMC_PMACRO_BRICK_CTRL_RFU2, + EMC_PMACRO_DATA_BRICK_CTRL_FDPD, + EMC_PMACRO_BG_BIAS_CTRL_0, + EMC_CFG_3, + EMC_PMACRO_TX_PWRD_0, + EMC_PMACRO_TX_PWRD_1, + EMC_PMACRO_TX_PWRD_2, + EMC_PMACRO_TX_PWRD_3, + EMC_PMACRO_TX_PWRD_4, + EMC_PMACRO_TX_PWRD_5, + EMC_CONFIG_SAMPLE_DELAY, + EMC_PMACRO_TX_SEL_CLK_SRC_0, + EMC_PMACRO_TX_SEL_CLK_SRC_1, + EMC_PMACRO_TX_SEL_CLK_SRC_2, + EMC_PMACRO_TX_SEL_CLK_SRC_3, + EMC_PMACRO_TX_SEL_CLK_SRC_4, + EMC_PMACRO_TX_SEL_CLK_SRC_5, + EMC_PMACRO_DDLL_BYPASS, + EMC_PMACRO_DDLL_PWRD_0, + EMC_PMACRO_DDLL_PWRD_1, + EMC_PMACRO_DDLL_PWRD_2, + EMC_PMACRO_CMD_CTRL_0, + EMC_PMACRO_CMD_CTRL_1, + EMC_PMACRO_CMD_CTRL_2, + EMC_TR_TIMING_0, + EMC_TR_DVFS, + EMC_TR_CTRL_1, + EMC_TR_RDV, + EMC_TR_QPOP, + EMC_TR_RDV_MASK, + EMC_MRW14, + EMC_TR_QSAFE, + EMC_TR_QRST, + EMC_TRAINING_CTRL, + EMC_TRAINING_SETTLE, + EMC_TRAINING_VREF_SETTLE, + EMC_TRAINING_CA_FINE_CTRL, + EMC_TRAINING_CA_CTRL_MISC, + EMC_TRAINING_CA_CTRL_MISC1, + EMC_TRAINING_CA_VREF_CTRL, + EMC_TRAINING_QUSE_CORS_CTRL, + EMC_TRAINING_QUSE_FINE_CTRL, + EMC_TRAINING_QUSE_CTRL_MISC, + EMC_TRAINING_QUSE_VREF_CTRL, + EMC_TRAINING_READ_FINE_CTRL, + EMC_TRAINING_READ_CTRL_MISC, + EMC_TRAINING_READ_VREF_CTRL, + EMC_TRAINING_WRITE_FINE_CTRL, + EMC_TRAINING_WRITE_CTRL_MISC, + EMC_TRAINING_WRITE_VREF_CTRL, + EMC_TRAINING_MPC, + EMC_MRW15, + }, + .trim = { + EMC_PMACRO_IB_DDLL_LONG_DQS_RANK0_0, + EMC_PMACRO_IB_DDLL_LONG_DQS_RANK0_1, + EMC_PMACRO_IB_DDLL_LONG_DQS_RANK0_2, + EMC_PMACRO_IB_DDLL_LONG_DQS_RANK0_3, + EMC_PMACRO_IB_DDLL_LONG_DQS_RANK1_0, + EMC_PMACRO_IB_DDLL_LONG_DQS_RANK1_1, + EMC_PMACRO_IB_DDLL_LONG_DQS_RANK1_2, + EMC_PMACRO_IB_DDLL_LONG_DQS_RANK1_3, + EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE0_0, + EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE0_1, + EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE0_2, + EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE1_0, + EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE1_1, + EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE1_2, + EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE2_0, + EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE2_1, + EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE2_2, + EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE3_0, + EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE3_1, + EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE3_2, + EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE4_0, + EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE4_1, + EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE4_2, + EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE5_0, + EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE5_1, + EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE5_2, + EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE6_0, + EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE6_1, + EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE6_2, + EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE7_0, + EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE7_1, + EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE7_2, + EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE0_0, + EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE0_1, + EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE0_2, + EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE1_0, + EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE1_1, + EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE1_2, + EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE2_0, + EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE2_1, + EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE2_2, + EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE3_0, + EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE3_1, + EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE3_2, + EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE4_0, + EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE4_1, + EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE4_2, + EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE5_0, + EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE5_1, + EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE5_2, + EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE6_0, + EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE6_1, + EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE6_2, + EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE7_0, + EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE7_1, + EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE7_2, + EMC_PMACRO_IB_VREF_DQS_0, + EMC_PMACRO_IB_VREF_DQS_1, + EMC_PMACRO_IB_VREF_DQ_0, + EMC_PMACRO_IB_VREF_DQ_1, + EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_0, + EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_1, + EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_2, + EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_3, + EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_4, + EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_5, + EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_0, + EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_1, + EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_2, + EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_3, + EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE0_0, + EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE0_1, + EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE0_2, + EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE1_0, + EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE1_1, + EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE1_2, + EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE2_0, + EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE2_1, + EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE2_2, + EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE3_0, + EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE3_1, + EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE3_2, + EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE4_0, + EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE4_1, + EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE4_2, + EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE5_0, + EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE5_1, + EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE5_2, + EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE6_0, + EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE6_1, + EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE6_2, + EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE7_0, + EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE7_1, + EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE7_2, + EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD0_0, + EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD0_1, + EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD0_2, + EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD1_0, + EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD1_1, + EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD1_2, + EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD2_0, + EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD2_1, + EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD2_2, + EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD3_0, + EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD3_1, + EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD3_2, + EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE0_0, + EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE0_1, + EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE0_2, + EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE1_0, + EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE1_1, + EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE1_2, + EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE2_0, + EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE2_1, + EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE2_2, + EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE3_0, + EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE3_1, + EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE3_2, + EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE4_0, + EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE4_1, + EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE4_2, + EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE5_0, + EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE5_1, + EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE5_2, + EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE6_0, + EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE6_1, + EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE6_2, + EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE7_0, + EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE7_1, + EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE7_2, + EMC_PMACRO_QUSE_DDLL_RANK0_0, + EMC_PMACRO_QUSE_DDLL_RANK0_1, + EMC_PMACRO_QUSE_DDLL_RANK0_2, + EMC_PMACRO_QUSE_DDLL_RANK0_3, + EMC_PMACRO_QUSE_DDLL_RANK1_0, + EMC_PMACRO_QUSE_DDLL_RANK1_1, + EMC_PMACRO_QUSE_DDLL_RANK1_2, + EMC_PMACRO_QUSE_DDLL_RANK1_3 + }, + .burst_mc = { + MC_EMEM_ARB_CFG, + MC_EMEM_ARB_OUTSTANDING_REQ, + MC_EMEM_ARB_REFPB_HP_CTRL, + MC_EMEM_ARB_REFPB_BANK_CTRL, + MC_EMEM_ARB_TIMING_RCD, + MC_EMEM_ARB_TIMING_RP, + MC_EMEM_ARB_TIMING_RC, + MC_EMEM_ARB_TIMING_RAS, + MC_EMEM_ARB_TIMING_FAW, + MC_EMEM_ARB_TIMING_RRD, + MC_EMEM_ARB_TIMING_RAP2PRE, + MC_EMEM_ARB_TIMING_WAP2PRE, + MC_EMEM_ARB_TIMING_R2R, + MC_EMEM_ARB_TIMING_W2W, + MC_EMEM_ARB_TIMING_R2W, + MC_EMEM_ARB_TIMING_CCDMW, + MC_EMEM_ARB_TIMING_W2R, + MC_EMEM_ARB_TIMING_RFCPB, + MC_EMEM_ARB_DA_TURNS, + MC_EMEM_ARB_DA_COVERS, + MC_EMEM_ARB_MISC0, + MC_EMEM_ARB_MISC1, + MC_EMEM_ARB_MISC2, + MC_EMEM_ARB_RING1_THROTTLE, + MC_EMEM_ARB_DHYST_CTRL, + MC_EMEM_ARB_DHYST_TIMEOUT_UTIL_0, + MC_EMEM_ARB_DHYST_TIMEOUT_UTIL_1, + MC_EMEM_ARB_DHYST_TIMEOUT_UTIL_2, + MC_EMEM_ARB_DHYST_TIMEOUT_UTIL_3, + MC_EMEM_ARB_DHYST_TIMEOUT_UTIL_4, + MC_EMEM_ARB_DHYST_TIMEOUT_UTIL_5, + MC_EMEM_ARB_DHYST_TIMEOUT_UTIL_6, + MC_EMEM_ARB_DHYST_TIMEOUT_UTIL_7, + }, + .la_scale = { + MC_MLL_MPCORER_PTSA_RATE, + MC_FTOP_PTSA_RATE, + MC_PTSA_GRANT_DECREMENT, + MC_LATENCY_ALLOWANCE_XUSB_0, + MC_LATENCY_ALLOWANCE_XUSB_1, + MC_LATENCY_ALLOWANCE_TSEC_0, + MC_LATENCY_ALLOWANCE_SDMMCA_0, + MC_LATENCY_ALLOWANCE_SDMMCAA_0, + MC_LATENCY_ALLOWANCE_SDMMC_0, + MC_LATENCY_ALLOWANCE_SDMMCAB_0, + MC_LATENCY_ALLOWANCE_PPCS_0, + MC_LATENCY_ALLOWANCE_PPCS_1, + MC_LATENCY_ALLOWANCE_MPCORE_0, + MC_LATENCY_ALLOWANCE_HC_0, + MC_LATENCY_ALLOWANCE_HC_1, + MC_LATENCY_ALLOWANCE_AVPC_0, + MC_LATENCY_ALLOWANCE_GPU_0, + MC_LATENCY_ALLOWANCE_GPU2_0, + MC_LATENCY_ALLOWANCE_NVENC_0, + MC_LATENCY_ALLOWANCE_NVDEC_0, + MC_LATENCY_ALLOWANCE_VIC_0, + MC_LATENCY_ALLOWANCE_VI2_0, + MC_LATENCY_ALLOWANCE_ISP2_0, + MC_LATENCY_ALLOWANCE_ISP2_1, + }, + .burst_per_channel = { + { .bank = 0, .offset = EMC_MRW10, }, + { .bank = 1, .offset = EMC_MRW10, }, + { .bank = 0, .offset = EMC_MRW11, }, + { .bank = 1, .offset = EMC_MRW11, }, + { .bank = 0, .offset = EMC_MRW12, }, + { .bank = 1, .offset = EMC_MRW12, }, + { .bank = 0, .offset = EMC_MRW13, }, + { .bank = 1, .offset = EMC_MRW13, }, + }, + .trim_per_channel = { + { .bank = 0, .offset = EMC_CMD_BRLSHFT_0, }, + { .bank = 1, .offset = EMC_CMD_BRLSHFT_1, }, + { .bank = 0, .offset = EMC_DATA_BRLSHFT_0, }, + { .bank = 1, .offset = EMC_DATA_BRLSHFT_0, }, + { .bank = 0, .offset = EMC_DATA_BRLSHFT_1, }, + { .bank = 1, .offset = EMC_DATA_BRLSHFT_1, }, + { .bank = 0, .offset = EMC_QUSE_BRLSHFT_0, }, + { .bank = 1, .offset = EMC_QUSE_BRLSHFT_1, }, + { .bank = 0, .offset = EMC_QUSE_BRLSHFT_2, }, + { .bank = 1, .offset = EMC_QUSE_BRLSHFT_3, }, + }, + .vref_per_channel = { + { + .bank = 0, + .offset = EMC_TRAINING_OPT_DQS_IB_VREF_RANK0, + }, { + .bank = 1, + .offset = EMC_TRAINING_OPT_DQS_IB_VREF_RANK0, + }, { + .bank = 0, + .offset = EMC_TRAINING_OPT_DQS_IB_VREF_RANK1, + }, { + .bank = 1, + .offset = EMC_TRAINING_OPT_DQS_IB_VREF_RANK1, + }, + }, +}; + +static void tegra210_emc_train(struct timer_list *timer) +{ + struct tegra210_emc *emc = from_timer(emc, timer, training); + unsigned long flags; + + if (!emc->last) + return; + + spin_lock_irqsave(&emc->lock, flags); + + if (emc->sequence->periodic_compensation) + emc->sequence->periodic_compensation(emc); + + spin_unlock_irqrestore(&emc->lock, flags); + + mod_timer(&emc->training, + jiffies + msecs_to_jiffies(emc->training_interval)); +} + +static void tegra210_emc_training_start(struct tegra210_emc *emc) +{ + mod_timer(&emc->training, + jiffies + msecs_to_jiffies(emc->training_interval)); +} + +static void tegra210_emc_training_stop(struct tegra210_emc *emc) +{ + del_timer(&emc->training); +} + +static unsigned int tegra210_emc_get_temperature(struct tegra210_emc *emc) +{ + unsigned long flags; + u32 value, max = 0; + unsigned int i; + + spin_lock_irqsave(&emc->lock, flags); + + for (i = 0; i < emc->num_devices; i++) { + value = tegra210_emc_mrr_read(emc, i, 4); + + if (value & BIT(7)) + dev_dbg(emc->dev, + "sensor reading changed for device %u: %08x\n", + i, value); + + value = FIELD_GET(LPDDR2_MR4_SRR, value); + if (value > max) + max = value; + } + + spin_unlock_irqrestore(&emc->lock, flags); + + return max; +} + +static void tegra210_emc_poll_refresh(struct timer_list *timer) +{ + struct tegra210_emc *emc = from_timer(emc, timer, refresh_timer); + unsigned int temperature; + + if (!emc->debugfs.temperature) + temperature = tegra210_emc_get_temperature(emc); + else + temperature = emc->debugfs.temperature; + + if (temperature == emc->temperature) + goto reset; + + switch (temperature) { + case 0 ... 3: + /* temperature is fine, using regular refresh */ + dev_dbg(emc->dev, "switching to nominal refresh...\n"); + tegra210_emc_set_refresh(emc, TEGRA210_EMC_REFRESH_NOMINAL); + break; + + case 4: + dev_dbg(emc->dev, "switching to 2x refresh...\n"); + tegra210_emc_set_refresh(emc, TEGRA210_EMC_REFRESH_2X); + break; + + case 5: + dev_dbg(emc->dev, "switching to 4x refresh...\n"); + tegra210_emc_set_refresh(emc, TEGRA210_EMC_REFRESH_4X); + break; + + case 6 ... 7: + dev_dbg(emc->dev, "switching to throttle refresh...\n"); + tegra210_emc_set_refresh(emc, TEGRA210_EMC_REFRESH_THROTTLE); + break; + + default: + WARN(1, "invalid DRAM temperature state %u\n", temperature); + return; + } + + emc->temperature = temperature; + +reset: + if (atomic_read(&emc->refresh_poll) > 0) { + unsigned int interval = emc->refresh_poll_interval; + unsigned int timeout = msecs_to_jiffies(interval); + + mod_timer(&emc->refresh_timer, jiffies + timeout); + } +} + +static void tegra210_emc_poll_refresh_stop(struct tegra210_emc *emc) +{ + atomic_set(&emc->refresh_poll, 0); + del_timer_sync(&emc->refresh_timer); +} + +static void tegra210_emc_poll_refresh_start(struct tegra210_emc *emc) +{ + atomic_set(&emc->refresh_poll, 1); + + mod_timer(&emc->refresh_timer, + jiffies + msecs_to_jiffies(emc->refresh_poll_interval)); +} + +static int tegra210_emc_cd_max_state(struct thermal_cooling_device *cd, + unsigned long *state) +{ + *state = 1; + + return 0; +} + +static int tegra210_emc_cd_get_state(struct thermal_cooling_device *cd, + unsigned long *state) +{ + struct tegra210_emc *emc = cd->devdata; + + *state = atomic_read(&emc->refresh_poll); + + return 0; +} + +static int tegra210_emc_cd_set_state(struct thermal_cooling_device *cd, + unsigned long state) +{ + struct tegra210_emc *emc = cd->devdata; + + if (state == atomic_read(&emc->refresh_poll)) + return 0; + + if (state) + tegra210_emc_poll_refresh_start(emc); + else + tegra210_emc_poll_refresh_stop(emc); + + return 0; +} + +static const struct thermal_cooling_device_ops tegra210_emc_cd_ops = { + .get_max_state = tegra210_emc_cd_max_state, + .get_cur_state = tegra210_emc_cd_get_state, + .set_cur_state = tegra210_emc_cd_set_state, +}; + +static void tegra210_emc_set_clock(struct tegra210_emc *emc, u32 clksrc) +{ + emc->sequence->set_clock(emc, clksrc); + + if (emc->next->periodic_training) + tegra210_emc_training_start(emc); + else + tegra210_emc_training_stop(emc); +} + +static void tegra210_change_dll_src(struct tegra210_emc *emc, + u32 clksrc) +{ + u32 dll_setting = emc->next->dll_clk_src; + u32 emc_clk_src; + u32 emc_clk_div; + + emc_clk_src = (clksrc & EMC_CLK_EMC_2X_CLK_SRC_MASK) >> + EMC_CLK_EMC_2X_CLK_SRC_SHIFT; + emc_clk_div = (clksrc & EMC_CLK_EMC_2X_CLK_DIVISOR_MASK) >> + EMC_CLK_EMC_2X_CLK_DIVISOR_SHIFT; + + dll_setting &= ~(DLL_CLK_EMC_DLL_CLK_SRC_MASK | + DLL_CLK_EMC_DLL_CLK_DIVISOR_MASK); + dll_setting |= emc_clk_src << DLL_CLK_EMC_DLL_CLK_SRC_SHIFT; + dll_setting |= emc_clk_div << DLL_CLK_EMC_DLL_CLK_DIVISOR_SHIFT; + + dll_setting &= ~DLL_CLK_EMC_DLL_DDLL_CLK_SEL_MASK; + if (emc_clk_src == EMC_CLK_SOURCE_PLLMB_LJ) + dll_setting |= (PLLM_VCOB << + DLL_CLK_EMC_DLL_DDLL_CLK_SEL_SHIFT); + else if (emc_clk_src == EMC_CLK_SOURCE_PLLM_LJ) + dll_setting |= (PLLM_VCOA << + DLL_CLK_EMC_DLL_DDLL_CLK_SEL_SHIFT); + else + dll_setting |= (EMC_DLL_SWITCH_OUT << + DLL_CLK_EMC_DLL_DDLL_CLK_SEL_SHIFT); + + tegra210_clk_emc_dll_update_setting(dll_setting); + + if (emc->next->clk_out_enb_x_0_clk_enb_emc_dll) + tegra210_clk_emc_dll_enable(true); + else + tegra210_clk_emc_dll_enable(false); +} + +int tegra210_emc_set_refresh(struct tegra210_emc *emc, + enum tegra210_emc_refresh refresh) +{ + struct tegra210_emc_timing *timings; + unsigned long flags; + + if ((emc->dram_type != DRAM_TYPE_LPDDR2 && + emc->dram_type != DRAM_TYPE_LPDDR4) || + !emc->last) + return -ENODEV; + + if (refresh > TEGRA210_EMC_REFRESH_THROTTLE) + return -EINVAL; + + if (refresh == emc->refresh) + return 0; + + spin_lock_irqsave(&emc->lock, flags); + + if (refresh == TEGRA210_EMC_REFRESH_THROTTLE && emc->derated) + timings = emc->derated; + else + timings = emc->nominal; + + if (timings != emc->timings) { + unsigned int index = emc->last - emc->timings; + u32 clksrc; + + clksrc = emc->provider.configs[index].value | + EMC_CLK_FORCE_CC_TRIGGER; + + emc->next = &timings[index]; + emc->timings = timings; + + tegra210_emc_set_clock(emc, clksrc); + } else { + tegra210_emc_adjust_timing(emc, emc->last); + tegra210_emc_timing_update(emc); + + if (refresh != TEGRA210_EMC_REFRESH_NOMINAL) + emc_writel(emc, EMC_REF_REF_CMD, EMC_REF); + } + + spin_unlock_irqrestore(&emc->lock, flags); + + return 0; +} + +u32 tegra210_emc_mrr_read(struct tegra210_emc *emc, unsigned int chip, + unsigned int address) +{ + u32 value, ret = 0; + unsigned int i; + + value = (chip & EMC_MRR_DEV_SEL_MASK) << EMC_MRR_DEV_SEL_SHIFT | + (address & EMC_MRR_MA_MASK) << EMC_MRR_MA_SHIFT; + emc_writel(emc, value, EMC_MRR); + + for (i = 0; i < emc->num_channels; i++) + WARN(tegra210_emc_wait_for_update(emc, i, EMC_EMC_STATUS, + EMC_EMC_STATUS_MRR_DIVLD, 1), + "Timed out waiting for MRR %u (ch=%u)\n", address, i); + + for (i = 0; i < emc->num_channels; i++) { + value = emc_channel_readl(emc, i, EMC_MRR); + value &= EMC_MRR_DATA_MASK; + + ret = (ret << 16) | value; + } + + return ret; +} + +void tegra210_emc_do_clock_change(struct tegra210_emc *emc, u32 clksrc) +{ + int err; + + mc_readl(emc->mc, MC_EMEM_ADR_CFG); + emc_readl(emc, EMC_INTSTATUS); + + tegra210_clk_emc_update_setting(clksrc); + + err = tegra210_emc_wait_for_update(emc, 0, EMC_INTSTATUS, + EMC_INTSTATUS_CLKCHANGE_COMPLETE, + true); + if (err) + dev_warn(emc->dev, "clock change completion error: %d\n", err); +} + +struct tegra210_emc_timing *tegra210_emc_find_timing(struct tegra210_emc *emc, + unsigned long rate) +{ + unsigned int i; + + for (i = 0; i < emc->num_timings; i++) + if (emc->timings[i].rate * 1000UL == rate) + return &emc->timings[i]; + + return NULL; +} + +int tegra210_emc_wait_for_update(struct tegra210_emc *emc, unsigned int channel, + unsigned int offset, u32 bit_mask, bool state) +{ + unsigned int i; + u32 value; + + for (i = 0; i < EMC_STATUS_UPDATE_TIMEOUT; i++) { + value = emc_channel_readl(emc, channel, offset); + if (!!(value & bit_mask) == state) + return 0; + + udelay(1); + } + + return -ETIMEDOUT; +} + +void tegra210_emc_set_shadow_bypass(struct tegra210_emc *emc, int set) +{ + u32 emc_dbg = emc_readl(emc, EMC_DBG); + + if (set) + emc_writel(emc, emc_dbg | EMC_DBG_WRITE_MUX_ACTIVE, EMC_DBG); + else + emc_writel(emc, emc_dbg & ~EMC_DBG_WRITE_MUX_ACTIVE, EMC_DBG); +} + +u32 tegra210_emc_get_dll_state(struct tegra210_emc_timing *next) +{ + if (next->emc_emrs & 0x1) + return 0; + + return 1; +} + +void tegra210_emc_timing_update(struct tegra210_emc *emc) +{ + unsigned int i; + int err = 0; + + emc_writel(emc, 0x1, EMC_TIMING_CONTROL); + + for (i = 0; i < emc->num_channels; i++) { + err |= tegra210_emc_wait_for_update(emc, i, EMC_EMC_STATUS, + EMC_EMC_STATUS_TIMING_UPDATE_STALLED, + false); + } + + if (err) + dev_warn(emc->dev, "timing update error: %d\n", err); +} + +unsigned long tegra210_emc_actual_osc_clocks(u32 in) +{ + if (in < 0x40) + return in * 16; + else if (in < 0x80) + return 2048; + else if (in < 0xc0) + return 4096; + else + return 8192; +} + +void tegra210_emc_start_periodic_compensation(struct tegra210_emc *emc) +{ + u32 mpc_req = 0x4b; + + emc_writel(emc, mpc_req, EMC_MPC); + mpc_req = emc_readl(emc, EMC_MPC); +} + +u32 tegra210_emc_compensate(struct tegra210_emc_timing *next, u32 offset) +{ + u32 temp = 0, rate = next->rate / 1000; + s32 delta[4], delta_taps[4]; + s32 new[] = { + TRIM_REG(0, 0, 0, 0), + TRIM_REG(0, 0, 0, 1), + TRIM_REG(0, 0, 1, 2), + TRIM_REG(0, 0, 1, 3), + + TRIM_REG(1, 0, 2, 4), + TRIM_REG(1, 0, 2, 5), + TRIM_REG(1, 0, 3, 6), + TRIM_REG(1, 0, 3, 7), + + TRIM_REG(0, 1, 0, 0), + TRIM_REG(0, 1, 0, 1), + TRIM_REG(0, 1, 1, 2), + TRIM_REG(0, 1, 1, 3), + + TRIM_REG(1, 1, 2, 4), + TRIM_REG(1, 1, 2, 5), + TRIM_REG(1, 1, 3, 6), + TRIM_REG(1, 1, 3, 7) + }; + unsigned i; + + switch (offset) { + case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_0: + case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_1: + case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_2: + case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_3: + case EMC_DATA_BRLSHFT_0: + delta[0] = 128 * (next->current_dram_clktree[C0D0U0] - + next->trained_dram_clktree[C0D0U0]); + delta[1] = 128 * (next->current_dram_clktree[C0D0U1] - + next->trained_dram_clktree[C0D0U1]); + delta[2] = 128 * (next->current_dram_clktree[C1D0U0] - + next->trained_dram_clktree[C1D0U0]); + delta[3] = 128 * (next->current_dram_clktree[C1D0U1] - + next->trained_dram_clktree[C1D0U1]); + + delta_taps[0] = (delta[0] * (s32)rate) / 1000000; + delta_taps[1] = (delta[1] * (s32)rate) / 1000000; + delta_taps[2] = (delta[2] * (s32)rate) / 1000000; + delta_taps[3] = (delta[3] * (s32)rate) / 1000000; + + for (i = 0; i < 4; i++) { + if ((delta_taps[i] > next->tree_margin) || + (delta_taps[i] < (-1 * next->tree_margin))) { + new[i * 2] = new[i * 2] + delta_taps[i]; + new[i * 2 + 1] = new[i * 2 + 1] + + delta_taps[i]; + } + } + + if (offset == EMC_DATA_BRLSHFT_0) { + for (i = 0; i < 8; i++) + new[i] = new[i] / 64; + } else { + for (i = 0; i < 8; i++) + new[i] = new[i] % 64; + } + + break; + + case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_0: + case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_1: + case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_2: + case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_3: + case EMC_DATA_BRLSHFT_1: + delta[0] = 128 * (next->current_dram_clktree[C0D1U0] - + next->trained_dram_clktree[C0D1U0]); + delta[1] = 128 * (next->current_dram_clktree[C0D1U1] - + next->trained_dram_clktree[C0D1U1]); + delta[2] = 128 * (next->current_dram_clktree[C1D1U0] - + next->trained_dram_clktree[C1D1U0]); + delta[3] = 128 * (next->current_dram_clktree[C1D1U1] - + next->trained_dram_clktree[C1D1U1]); + + delta_taps[0] = (delta[0] * (s32)rate) / 1000000; + delta_taps[1] = (delta[1] * (s32)rate) / 1000000; + delta_taps[2] = (delta[2] * (s32)rate) / 1000000; + delta_taps[3] = (delta[3] * (s32)rate) / 1000000; + + for (i = 0; i < 4; i++) { + if ((delta_taps[i] > next->tree_margin) || + (delta_taps[i] < (-1 * next->tree_margin))) { + new[8 + i * 2] = new[8 + i * 2] + + delta_taps[i]; + new[8 + i * 2 + 1] = new[8 + i * 2 + 1] + + delta_taps[i]; + } + } + + if (offset == EMC_DATA_BRLSHFT_1) { + for (i = 0; i < 8; i++) + new[i + 8] = new[i + 8] / 64; + } else { + for (i = 0; i < 8; i++) + new[i + 8] = new[i + 8] % 64; + } + + break; + } + + switch (offset) { + case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_0: + temp = CALC_TEMP(0, 0, 0, 1, 0); + break; + + case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_1: + temp = CALC_TEMP(0, 1, 2, 3, 2); + break; + + case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_2: + temp = CALC_TEMP(0, 2, 4, 5, 4); + break; + + case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_3: + temp = CALC_TEMP(0, 3, 6, 7, 6); + break; + + case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_0: + temp = CALC_TEMP(1, 0, 0, 1, 8); + break; + + case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_1: + temp = CALC_TEMP(1, 1, 2, 3, 10); + break; + + case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_2: + temp = CALC_TEMP(1, 2, 4, 5, 12); + break; + + case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_3: + temp = CALC_TEMP(1, 3, 6, 7, 14); + break; + + case EMC_DATA_BRLSHFT_0: + temp = ((new[0] << + EMC_DATA_BRLSHFT_0_RANK0_BYTE0_DATA_BRLSHFT_SHIFT) & + EMC_DATA_BRLSHFT_0_RANK0_BYTE0_DATA_BRLSHFT_MASK) | + ((new[1] << + EMC_DATA_BRLSHFT_0_RANK0_BYTE1_DATA_BRLSHFT_SHIFT) & + EMC_DATA_BRLSHFT_0_RANK0_BYTE1_DATA_BRLSHFT_MASK) | + ((new[2] << + EMC_DATA_BRLSHFT_0_RANK0_BYTE2_DATA_BRLSHFT_SHIFT) & + EMC_DATA_BRLSHFT_0_RANK0_BYTE2_DATA_BRLSHFT_MASK) | + ((new[3] << + EMC_DATA_BRLSHFT_0_RANK0_BYTE3_DATA_BRLSHFT_SHIFT) & + EMC_DATA_BRLSHFT_0_RANK0_BYTE3_DATA_BRLSHFT_MASK) | + ((new[4] << + EMC_DATA_BRLSHFT_0_RANK0_BYTE4_DATA_BRLSHFT_SHIFT) & + EMC_DATA_BRLSHFT_0_RANK0_BYTE4_DATA_BRLSHFT_MASK) | + ((new[5] << + EMC_DATA_BRLSHFT_0_RANK0_BYTE5_DATA_BRLSHFT_SHIFT) & + EMC_DATA_BRLSHFT_0_RANK0_BYTE5_DATA_BRLSHFT_MASK) | + ((new[6] << + EMC_DATA_BRLSHFT_0_RANK0_BYTE6_DATA_BRLSHFT_SHIFT) & + EMC_DATA_BRLSHFT_0_RANK0_BYTE6_DATA_BRLSHFT_MASK) | + ((new[7] << + EMC_DATA_BRLSHFT_0_RANK0_BYTE7_DATA_BRLSHFT_SHIFT) & + EMC_DATA_BRLSHFT_0_RANK0_BYTE7_DATA_BRLSHFT_MASK); + break; + + case EMC_DATA_BRLSHFT_1: + temp = ((new[8] << + EMC_DATA_BRLSHFT_1_RANK1_BYTE0_DATA_BRLSHFT_SHIFT) & + EMC_DATA_BRLSHFT_1_RANK1_BYTE0_DATA_BRLSHFT_MASK) | + ((new[9] << + EMC_DATA_BRLSHFT_1_RANK1_BYTE1_DATA_BRLSHFT_SHIFT) & + EMC_DATA_BRLSHFT_1_RANK1_BYTE1_DATA_BRLSHFT_MASK) | + ((new[10] << + EMC_DATA_BRLSHFT_1_RANK1_BYTE2_DATA_BRLSHFT_SHIFT) & + EMC_DATA_BRLSHFT_1_RANK1_BYTE2_DATA_BRLSHFT_MASK) | + ((new[11] << + EMC_DATA_BRLSHFT_1_RANK1_BYTE3_DATA_BRLSHFT_SHIFT) & + EMC_DATA_BRLSHFT_1_RANK1_BYTE3_DATA_BRLSHFT_MASK) | + ((new[12] << + EMC_DATA_BRLSHFT_1_RANK1_BYTE4_DATA_BRLSHFT_SHIFT) & + EMC_DATA_BRLSHFT_1_RANK1_BYTE4_DATA_BRLSHFT_MASK) | + ((new[13] << + EMC_DATA_BRLSHFT_1_RANK1_BYTE5_DATA_BRLSHFT_SHIFT) & + EMC_DATA_BRLSHFT_1_RANK1_BYTE5_DATA_BRLSHFT_MASK) | + ((new[14] << + EMC_DATA_BRLSHFT_1_RANK1_BYTE6_DATA_BRLSHFT_SHIFT) & + EMC_DATA_BRLSHFT_1_RANK1_BYTE6_DATA_BRLSHFT_MASK) | + ((new[15] << + EMC_DATA_BRLSHFT_1_RANK1_BYTE7_DATA_BRLSHFT_SHIFT) & + EMC_DATA_BRLSHFT_1_RANK1_BYTE7_DATA_BRLSHFT_MASK); + break; + + default: + break; + } + + return temp; +} + +u32 tegra210_emc_dll_prelock(struct tegra210_emc *emc, u32 clksrc) +{ + unsigned int i; + u32 value; + + value = emc_readl(emc, EMC_CFG_DIG_DLL); + value &= ~EMC_CFG_DIG_DLL_CFG_DLL_LOCK_LIMIT_MASK; + value |= (3 << EMC_CFG_DIG_DLL_CFG_DLL_LOCK_LIMIT_SHIFT); + value &= ~EMC_CFG_DIG_DLL_CFG_DLL_EN; + value &= ~EMC_CFG_DIG_DLL_CFG_DLL_MODE_MASK; + value |= (3 << EMC_CFG_DIG_DLL_CFG_DLL_MODE_SHIFT); + value |= EMC_CFG_DIG_DLL_CFG_DLL_STALL_ALL_TRAFFIC; + value &= ~EMC_CFG_DIG_DLL_CFG_DLL_STALL_RW_UNTIL_LOCK; + value &= ~EMC_CFG_DIG_DLL_CFG_DLL_STALL_ALL_UNTIL_LOCK; + emc_writel(emc, value, EMC_CFG_DIG_DLL); + emc_writel(emc, 1, EMC_TIMING_CONTROL); + + for (i = 0; i < emc->num_channels; i++) + tegra210_emc_wait_for_update(emc, i, EMC_EMC_STATUS, + EMC_EMC_STATUS_TIMING_UPDATE_STALLED, + 0); + + for (i = 0; i < emc->num_channels; i++) { + while (true) { + value = emc_channel_readl(emc, i, EMC_CFG_DIG_DLL); + if ((value & EMC_CFG_DIG_DLL_CFG_DLL_EN) == 0) + break; + } + } + + value = emc->next->burst_regs[EMC_DLL_CFG_0_INDEX]; + emc_writel(emc, value, EMC_DLL_CFG_0); + + value = emc_readl(emc, EMC_DLL_CFG_1); + value &= EMC_DLL_CFG_1_DDLLCAL_CTRL_START_TRIM_MASK; + + if (emc->next->rate >= 400000 && emc->next->rate < 600000) + value |= 150; + else if (emc->next->rate >= 600000 && emc->next->rate < 800000) + value |= 100; + else if (emc->next->rate >= 800000 && emc->next->rate < 1000000) + value |= 70; + else if (emc->next->rate >= 1000000 && emc->next->rate < 1200000) + value |= 30; + else + value |= 20; + + emc_writel(emc, value, EMC_DLL_CFG_1); + + tegra210_change_dll_src(emc, clksrc); + + value = emc_readl(emc, EMC_CFG_DIG_DLL); + value |= EMC_CFG_DIG_DLL_CFG_DLL_EN; + emc_writel(emc, value, EMC_CFG_DIG_DLL); + + tegra210_emc_timing_update(emc); + + for (i = 0; i < emc->num_channels; i++) { + while (true) { + value = emc_channel_readl(emc, 0, EMC_CFG_DIG_DLL); + if (value & EMC_CFG_DIG_DLL_CFG_DLL_EN) + break; + } + } + + while (true) { + value = emc_readl(emc, EMC_DIG_DLL_STATUS); + + if ((value & EMC_DIG_DLL_STATUS_DLL_PRIV_UPDATED) == 0) + continue; + + if ((value & EMC_DIG_DLL_STATUS_DLL_LOCK) == 0) + continue; + + break; + } + + value = emc_readl(emc, EMC_DIG_DLL_STATUS); + + return value & EMC_DIG_DLL_STATUS_DLL_OUT_MASK; +} + +u32 tegra210_emc_dvfs_power_ramp_up(struct tegra210_emc *emc, u32 clk, + bool flip_backward) +{ + u32 cmd_pad, dq_pad, rfu1, cfg5, common_tx, ramp_up_wait = 0; + const struct tegra210_emc_timing *timing; + + if (flip_backward) + timing = emc->last; + else + timing = emc->next; + + cmd_pad = timing->burst_regs[EMC_PMACRO_CMD_PAD_TX_CTRL_INDEX]; + dq_pad = timing->burst_regs[EMC_PMACRO_DATA_PAD_TX_CTRL_INDEX]; + rfu1 = timing->burst_regs[EMC_PMACRO_BRICK_CTRL_RFU1_INDEX]; + cfg5 = timing->burst_regs[EMC_FBIO_CFG5_INDEX]; + common_tx = timing->burst_regs[EMC_PMACRO_COMMON_PAD_TX_CTRL_INDEX]; + + cmd_pad |= EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQ_TX_DRVFORCEON; + + if (clk < 1000000 / DVFS_FGCG_MID_SPEED_THRESHOLD) { + ccfifo_writel(emc, common_tx & 0xa, + EMC_PMACRO_COMMON_PAD_TX_CTRL, 0); + ccfifo_writel(emc, common_tx & 0xf, + EMC_PMACRO_COMMON_PAD_TX_CTRL, + (100000 / clk) + 1); + ramp_up_wait += 100000; + } else { + ccfifo_writel(emc, common_tx | 0x8, + EMC_PMACRO_COMMON_PAD_TX_CTRL, 0); + } + + if (clk < 1000000 / DVFS_FGCG_HIGH_SPEED_THRESHOLD) { + if (clk < 1000000 / IOBRICK_DCC_THRESHOLD) { + cmd_pad |= + EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQSP_TX_E_DCC | + EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQSN_TX_E_DCC; + cmd_pad &= + ~(EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQ_TX_E_DCC | + EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_CMD_TX_E_DCC); + ccfifo_writel(emc, cmd_pad, + EMC_PMACRO_CMD_PAD_TX_CTRL, + (100000 / clk) + 1); + ramp_up_wait += 100000; + + dq_pad |= + EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQSP_TX_E_DCC | + EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQSN_TX_E_DCC; + dq_pad &= + ~(EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQ_TX_E_DCC | + EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_CMD_TX_E_DCC); + ccfifo_writel(emc, dq_pad, + EMC_PMACRO_DATA_PAD_TX_CTRL, 0); + ccfifo_writel(emc, rfu1 & 0xfe40fe40, + EMC_PMACRO_BRICK_CTRL_RFU1, 0); + } else { + ccfifo_writel(emc, rfu1 & 0xfe40fe40, + EMC_PMACRO_BRICK_CTRL_RFU1, + (100000 / clk) + 1); + ramp_up_wait += 100000; + } + + ccfifo_writel(emc, rfu1 & 0xfeedfeed, + EMC_PMACRO_BRICK_CTRL_RFU1, (100000 / clk) + 1); + ramp_up_wait += 100000; + + if (clk < 1000000 / IOBRICK_DCC_THRESHOLD) { + cmd_pad |= + EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQSP_TX_E_DCC | + EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQSN_TX_E_DCC | + EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQ_TX_E_DCC | + EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_CMD_TX_E_DCC; + ccfifo_writel(emc, cmd_pad, + EMC_PMACRO_CMD_PAD_TX_CTRL, + (100000 / clk) + 1); + ramp_up_wait += 100000; + + dq_pad |= + EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQSP_TX_E_DCC | + EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQSN_TX_E_DCC | + EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQ_TX_E_DCC | + EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_CMD_TX_E_DCC; + ccfifo_writel(emc, dq_pad, + EMC_PMACRO_DATA_PAD_TX_CTRL, 0); + ccfifo_writel(emc, rfu1, + EMC_PMACRO_BRICK_CTRL_RFU1, 0); + } else { + ccfifo_writel(emc, rfu1, + EMC_PMACRO_BRICK_CTRL_RFU1, + (100000 / clk) + 1); + ramp_up_wait += 100000; + } + + ccfifo_writel(emc, cfg5 & ~EMC_FBIO_CFG5_CMD_TX_DIS, + EMC_FBIO_CFG5, (100000 / clk) + 10); + ramp_up_wait += 100000 + (10 * clk); + } else if (clk < 1000000 / DVFS_FGCG_MID_SPEED_THRESHOLD) { + ccfifo_writel(emc, rfu1 | 0x06000600, + EMC_PMACRO_BRICK_CTRL_RFU1, (100000 / clk) + 1); + ccfifo_writel(emc, cfg5 & ~EMC_FBIO_CFG5_CMD_TX_DIS, + EMC_FBIO_CFG5, (100000 / clk) + 10); + ramp_up_wait += 100000 + 10 * clk; + } else { + ccfifo_writel(emc, rfu1 | 0x00000600, + EMC_PMACRO_BRICK_CTRL_RFU1, 0); + ccfifo_writel(emc, cfg5 & ~EMC_FBIO_CFG5_CMD_TX_DIS, + EMC_FBIO_CFG5, 12); + ramp_up_wait += 12 * clk; + } + + cmd_pad &= ~EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQ_TX_DRVFORCEON; + ccfifo_writel(emc, cmd_pad, EMC_PMACRO_CMD_PAD_TX_CTRL, 5); + + return ramp_up_wait; +} + +u32 tegra210_emc_dvfs_power_ramp_down(struct tegra210_emc *emc, u32 clk, + bool flip_backward) +{ + u32 ramp_down_wait = 0, cmd_pad, dq_pad, rfu1, cfg5, common_tx; + const struct tegra210_emc_timing *entry; + u32 seq_wait; + + if (flip_backward) + entry = emc->next; + else + entry = emc->last; + + cmd_pad = entry->burst_regs[EMC_PMACRO_CMD_PAD_TX_CTRL_INDEX]; + dq_pad = entry->burst_regs[EMC_PMACRO_DATA_PAD_TX_CTRL_INDEX]; + rfu1 = entry->burst_regs[EMC_PMACRO_BRICK_CTRL_RFU1_INDEX]; + cfg5 = entry->burst_regs[EMC_FBIO_CFG5_INDEX]; + common_tx = entry->burst_regs[EMC_PMACRO_COMMON_PAD_TX_CTRL_INDEX]; + + cmd_pad |= EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQ_TX_DRVFORCEON; + + ccfifo_writel(emc, cmd_pad, EMC_PMACRO_CMD_PAD_TX_CTRL, 0); + ccfifo_writel(emc, cfg5 | EMC_FBIO_CFG5_CMD_TX_DIS, + EMC_FBIO_CFG5, 12); + ramp_down_wait = 12 * clk; + + seq_wait = (100000 / clk) + 1; + + if (clk < (1000000 / DVFS_FGCG_HIGH_SPEED_THRESHOLD)) { + if (clk < (1000000 / IOBRICK_DCC_THRESHOLD)) { + cmd_pad &= + ~(EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQ_TX_E_DCC | + EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_CMD_TX_E_DCC); + cmd_pad |= + EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQSP_TX_E_DCC | + EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQSN_TX_E_DCC; + ccfifo_writel(emc, cmd_pad, + EMC_PMACRO_CMD_PAD_TX_CTRL, seq_wait); + ramp_down_wait += 100000; + + dq_pad &= + ~(EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQ_TX_E_DCC | + EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_CMD_TX_E_DCC); + dq_pad |= + EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQSP_TX_E_DCC | + EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQSN_TX_E_DCC; + ccfifo_writel(emc, dq_pad, + EMC_PMACRO_DATA_PAD_TX_CTRL, 0); + ccfifo_writel(emc, rfu1 & ~0x01120112, + EMC_PMACRO_BRICK_CTRL_RFU1, 0); + } else { + ccfifo_writel(emc, rfu1 & ~0x01120112, + EMC_PMACRO_BRICK_CTRL_RFU1, seq_wait); + ramp_down_wait += 100000; + } + + ccfifo_writel(emc, rfu1 & ~0x01bf01bf, + EMC_PMACRO_BRICK_CTRL_RFU1, seq_wait); + ramp_down_wait += 100000; + + if (clk < (1000000 / IOBRICK_DCC_THRESHOLD)) { + cmd_pad &= + ~(EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQ_TX_E_DCC | + EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_CMD_TX_E_DCC | + EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQSP_TX_E_DCC | + EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQSN_TX_E_DCC); + ccfifo_writel(emc, cmd_pad, + EMC_PMACRO_CMD_PAD_TX_CTRL, seq_wait); + ramp_down_wait += 100000; + + dq_pad &= + ~(EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQ_TX_E_DCC | + EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_CMD_TX_E_DCC | + EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQSP_TX_E_DCC | + EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQSN_TX_E_DCC); + ccfifo_writel(emc, dq_pad, + EMC_PMACRO_DATA_PAD_TX_CTRL, 0); + ccfifo_writel(emc, rfu1 & ~0x07ff07ff, + EMC_PMACRO_BRICK_CTRL_RFU1, 0); + } else { + ccfifo_writel(emc, rfu1 & ~0x07ff07ff, + EMC_PMACRO_BRICK_CTRL_RFU1, seq_wait); + ramp_down_wait += 100000; + } + } else { + ccfifo_writel(emc, rfu1 & ~0xffff07ff, + EMC_PMACRO_BRICK_CTRL_RFU1, seq_wait + 19); + ramp_down_wait += 100000 + (20 * clk); + } + + if (clk < (1000000 / DVFS_FGCG_MID_SPEED_THRESHOLD)) { + ramp_down_wait += 100000; + ccfifo_writel(emc, common_tx & ~0x5, + EMC_PMACRO_COMMON_PAD_TX_CTRL, seq_wait); + ramp_down_wait += 100000; + ccfifo_writel(emc, common_tx & ~0xf, + EMC_PMACRO_COMMON_PAD_TX_CTRL, seq_wait); + ramp_down_wait += 100000; + ccfifo_writel(emc, 0, 0, seq_wait); + ramp_down_wait += 100000; + } else { + ccfifo_writel(emc, common_tx & ~0xf, + EMC_PMACRO_COMMON_PAD_TX_CTRL, seq_wait); + } + + return ramp_down_wait; +} + +void tegra210_emc_reset_dram_clktree_values(struct tegra210_emc_timing *timing) +{ + timing->current_dram_clktree[C0D0U0] = + timing->trained_dram_clktree[C0D0U0]; + timing->current_dram_clktree[C0D0U1] = + timing->trained_dram_clktree[C0D0U1]; + timing->current_dram_clktree[C1D0U0] = + timing->trained_dram_clktree[C1D0U0]; + timing->current_dram_clktree[C1D0U1] = + timing->trained_dram_clktree[C1D0U1]; + timing->current_dram_clktree[C1D1U0] = + timing->trained_dram_clktree[C1D1U0]; + timing->current_dram_clktree[C1D1U1] = + timing->trained_dram_clktree[C1D1U1]; +} + +static void update_dll_control(struct tegra210_emc *emc, u32 value, bool state) +{ + unsigned int i; + + emc_writel(emc, value, EMC_CFG_DIG_DLL); + tegra210_emc_timing_update(emc); + + for (i = 0; i < emc->num_channels; i++) + tegra210_emc_wait_for_update(emc, i, EMC_CFG_DIG_DLL, + EMC_CFG_DIG_DLL_CFG_DLL_EN, + state); +} + +void tegra210_emc_dll_disable(struct tegra210_emc *emc) +{ + u32 value; + + value = emc_readl(emc, EMC_CFG_DIG_DLL); + value &= ~EMC_CFG_DIG_DLL_CFG_DLL_EN; + + update_dll_control(emc, value, false); +} + +void tegra210_emc_dll_enable(struct tegra210_emc *emc) +{ + u32 value; + + value = emc_readl(emc, EMC_CFG_DIG_DLL); + value |= EMC_CFG_DIG_DLL_CFG_DLL_EN; + + update_dll_control(emc, value, true); +} + +void tegra210_emc_adjust_timing(struct tegra210_emc *emc, + struct tegra210_emc_timing *timing) +{ + u32 dsr_cntrl = timing->burst_regs[EMC_DYN_SELF_REF_CONTROL_INDEX]; + u32 pre_ref = timing->burst_regs[EMC_PRE_REFRESH_REQ_CNT_INDEX]; + u32 ref = timing->burst_regs[EMC_REFRESH_INDEX]; + + switch (emc->refresh) { + case TEGRA210_EMC_REFRESH_NOMINAL: + case TEGRA210_EMC_REFRESH_THROTTLE: + break; + + case TEGRA210_EMC_REFRESH_2X: + ref = REFRESH_SPEEDUP(ref, 2); + pre_ref = REFRESH_SPEEDUP(pre_ref, 2); + dsr_cntrl = REFRESH_SPEEDUP(dsr_cntrl, 2); + break; + + case TEGRA210_EMC_REFRESH_4X: + ref = REFRESH_SPEEDUP(ref, 4); + pre_ref = REFRESH_SPEEDUP(pre_ref, 4); + dsr_cntrl = REFRESH_SPEEDUP(dsr_cntrl, 4); + break; + + default: + dev_warn(emc->dev, "failed to set refresh: %d\n", emc->refresh); + return; + } + + emc_writel(emc, ref, emc->offsets->burst[EMC_REFRESH_INDEX]); + emc_writel(emc, pre_ref, + emc->offsets->burst[EMC_PRE_REFRESH_REQ_CNT_INDEX]); + emc_writel(emc, dsr_cntrl, + emc->offsets->burst[EMC_DYN_SELF_REF_CONTROL_INDEX]); +} + +static int tegra210_emc_set_rate(struct device *dev, + const struct tegra210_clk_emc_config *config) +{ + struct tegra210_emc *emc = dev_get_drvdata(dev); + struct tegra210_emc_timing *timing = NULL; + unsigned long rate = config->rate; + s64 last_change_delay; + unsigned long flags; + unsigned int i; + + if (rate == emc->last->rate * 1000UL) + return 0; + + for (i = 0; i < emc->num_timings; i++) { + if (emc->timings[i].rate * 1000UL == rate) { + timing = &emc->timings[i]; + break; + } + } + + if (!timing) + return -EINVAL; + + if (rate > 204000000 && !timing->trained) + return -EINVAL; + + emc->next = timing; + last_change_delay = ktime_us_delta(ktime_get(), emc->clkchange_time); + + /* XXX use non-busy-looping sleep? */ + if ((last_change_delay >= 0) && + (last_change_delay < emc->clkchange_delay)) + udelay(emc->clkchange_delay - (int)last_change_delay); + + spin_lock_irqsave(&emc->lock, flags); + tegra210_emc_set_clock(emc, config->value); + emc->clkchange_time = ktime_get(); + emc->last = timing; + spin_unlock_irqrestore(&emc->lock, flags); + + return 0; +} + +/* + * debugfs interface + * + * The memory controller driver exposes some files in debugfs that can be used + * to control the EMC frequency. The top-level directory can be found here: + * + * /sys/kernel/debug/emc + * + * It contains the following files: + * + * - available_rates: This file contains a list of valid, space-separated + * EMC frequencies. + * + * - min_rate: Writing a value to this file sets the given frequency as the + * floor of the permitted range. If this is higher than the currently + * configured EMC frequency, this will cause the frequency to be + * increased so that it stays within the valid range. + * + * - max_rate: Similarily to the min_rate file, writing a value to this file + * sets the given frequency as the ceiling of the permitted range. If + * the value is lower than the currently configured EMC frequency, this + * will cause the frequency to be decreased so that it stays within the + * valid range. + */ + +static bool tegra210_emc_validate_rate(struct tegra210_emc *emc, + unsigned long rate) +{ + unsigned int i; + + for (i = 0; i < emc->num_timings; i++) + if (rate == emc->timings[i].rate * 1000UL) + return true; + + return false; +} + +static int tegra210_emc_debug_available_rates_show(struct seq_file *s, + void *data) +{ + struct tegra210_emc *emc = s->private; + const char *prefix = ""; + unsigned int i; + + for (i = 0; i < emc->num_timings; i++) { + seq_printf(s, "%s%u", prefix, emc->timings[i].rate * 1000); + prefix = " "; + } + + seq_puts(s, "\n"); + + return 0; +} + +static int tegra210_emc_debug_available_rates_open(struct inode *inode, + struct file *file) +{ + return single_open(file, tegra210_emc_debug_available_rates_show, + inode->i_private); +} + +static const struct file_operations tegra210_emc_debug_available_rates_fops = { + .open = tegra210_emc_debug_available_rates_open, + .read = seq_read, + .llseek = seq_lseek, + .release = single_release, +}; + +static int tegra210_emc_debug_min_rate_get(void *data, u64 *rate) +{ + struct tegra210_emc *emc = data; + + *rate = emc->debugfs.min_rate; + + return 0; +} + +static int tegra210_emc_debug_min_rate_set(void *data, u64 rate) +{ + struct tegra210_emc *emc = data; + int err; + + if (!tegra210_emc_validate_rate(emc, rate)) + return -EINVAL; + + err = clk_set_min_rate(emc->clk, rate); + if (err < 0) + return err; + + emc->debugfs.min_rate = rate; + + return 0; +} + +DEFINE_DEBUGFS_ATTRIBUTE(tegra210_emc_debug_min_rate_fops, + tegra210_emc_debug_min_rate_get, + tegra210_emc_debug_min_rate_set, "%llu\n"); + +static int tegra210_emc_debug_max_rate_get(void *data, u64 *rate) +{ + struct tegra210_emc *emc = data; + + *rate = emc->debugfs.max_rate; + + return 0; +} + +static int tegra210_emc_debug_max_rate_set(void *data, u64 rate) +{ + struct tegra210_emc *emc = data; + int err; + + if (!tegra210_emc_validate_rate(emc, rate)) + return -EINVAL; + + err = clk_set_max_rate(emc->clk, rate); + if (err < 0) + return err; + + emc->debugfs.max_rate = rate; + + return 0; +} + +DEFINE_DEBUGFS_ATTRIBUTE(tegra210_emc_debug_max_rate_fops, + tegra210_emc_debug_max_rate_get, + tegra210_emc_debug_max_rate_set, "%llu\n"); + +static int tegra210_emc_debug_temperature_get(void *data, u64 *temperature) +{ + struct tegra210_emc *emc = data; + unsigned int value; + + if (!emc->debugfs.temperature) + value = tegra210_emc_get_temperature(emc); + else + value = emc->debugfs.temperature; + + *temperature = value; + + return 0; +} + +static int tegra210_emc_debug_temperature_set(void *data, u64 temperature) +{ + struct tegra210_emc *emc = data; + + if (temperature > 7) + return -EINVAL; + + emc->debugfs.temperature = temperature; + + return 0; +} + +DEFINE_DEBUGFS_ATTRIBUTE(tegra210_emc_debug_temperature_fops, + tegra210_emc_debug_temperature_get, + tegra210_emc_debug_temperature_set, "%llu\n"); + +static void tegra210_emc_debugfs_init(struct tegra210_emc *emc) +{ + struct device *dev = emc->dev; + unsigned int i; + int err; + + emc->debugfs.min_rate = ULONG_MAX; + emc->debugfs.max_rate = 0; + + for (i = 0; i < emc->num_timings; i++) { + if (emc->timings[i].rate * 1000UL < emc->debugfs.min_rate) + emc->debugfs.min_rate = emc->timings[i].rate * 1000UL; + + if (emc->timings[i].rate * 1000UL > emc->debugfs.max_rate) + emc->debugfs.max_rate = emc->timings[i].rate * 1000UL; + } + + if (!emc->num_timings) { + emc->debugfs.min_rate = clk_get_rate(emc->clk); + emc->debugfs.max_rate = emc->debugfs.min_rate; + } + + err = clk_set_rate_range(emc->clk, emc->debugfs.min_rate, + emc->debugfs.max_rate); + if (err < 0) { + dev_err(dev, "failed to set rate range [%lu-%lu] for %pC\n", + emc->debugfs.min_rate, emc->debugfs.max_rate, + emc->clk); + return; + } + + emc->debugfs.root = debugfs_create_dir("emc", NULL); + + debugfs_create_file("available_rates", 0444, emc->debugfs.root, emc, + &tegra210_emc_debug_available_rates_fops); + debugfs_create_file("min_rate", 0644, emc->debugfs.root, emc, + &tegra210_emc_debug_min_rate_fops); + debugfs_create_file("max_rate", 0644, emc->debugfs.root, emc, + &tegra210_emc_debug_max_rate_fops); + debugfs_create_file("temperature", 0644, emc->debugfs.root, emc, + &tegra210_emc_debug_temperature_fops); +} + +static void tegra210_emc_detect(struct tegra210_emc *emc) +{ + u32 value; + + /* probe the number of connected DRAM devices */ + value = mc_readl(emc->mc, MC_EMEM_ADR_CFG); + + if (value & MC_EMEM_ADR_CFG_EMEM_NUMDEV) + emc->num_devices = 2; + else + emc->num_devices = 1; + + /* probe the type of DRAM */ + value = emc_readl(emc, EMC_FBIO_CFG5); + emc->dram_type = value & 0x3; + + /* probe the number of channels */ + value = emc_readl(emc, EMC_FBIO_CFG7); + + if ((value & EMC_FBIO_CFG7_CH1_ENABLE) && + (value & EMC_FBIO_CFG7_CH0_ENABLE)) + emc->num_channels = 2; + else + emc->num_channels = 1; +} + +static int tegra210_emc_validate_timings(struct tegra210_emc *emc, + struct tegra210_emc_timing *timings, + unsigned int num_timings) +{ + unsigned int i; + + for (i = 0; i < num_timings; i++) { + u32 min_volt = timings[i].min_volt; + u32 rate = timings[i].rate; + + if (!rate) + return -EINVAL; + + if ((i > 0) && ((rate <= timings[i - 1].rate) || + (min_volt < timings[i - 1].min_volt))) + return -EINVAL; + + if (timings[i].revision != timings[0].revision) + continue; + } + + return 0; +} + +static int tegra210_emc_probe(struct platform_device *pdev) +{ + struct thermal_cooling_device *cd; + unsigned long current_rate; + struct tegra210_emc *emc; + struct device_node *np; + unsigned int i; + int err; + + emc = devm_kzalloc(&pdev->dev, sizeof(*emc), GFP_KERNEL); + if (!emc) + return -ENOMEM; + + emc->clk = devm_clk_get(&pdev->dev, "emc"); + if (IS_ERR(emc->clk)) + return PTR_ERR(emc->clk); + + platform_set_drvdata(pdev, emc); + spin_lock_init(&emc->lock); + emc->dev = &pdev->dev; + + emc->mc = devm_tegra_memory_controller_get(&pdev->dev); + if (IS_ERR(emc->mc)) + return PTR_ERR(emc->mc); + + emc->regs = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(emc->regs)) + return PTR_ERR(emc->regs); + + for (i = 0; i < 2; i++) { + emc->channel[i] = devm_platform_ioremap_resource(pdev, 1 + i); + if (IS_ERR(emc->channel[i])) + return PTR_ERR(emc->channel[i]); + + } + + tegra210_emc_detect(emc); + np = pdev->dev.of_node; + + /* attach to the nominal and (optional) derated tables */ + err = of_reserved_mem_device_init_by_name(emc->dev, np, "nominal"); + if (err < 0) { + dev_err(emc->dev, "failed to get nominal EMC table: %d\n", err); + return err; + } + + err = of_reserved_mem_device_init_by_name(emc->dev, np, "derated"); + if (err < 0 && err != -ENODEV) { + dev_err(emc->dev, "failed to get derated EMC table: %d\n", err); + goto release; + } + + /* validate the tables */ + if (emc->nominal) { + err = tegra210_emc_validate_timings(emc, emc->nominal, + emc->num_timings); + if (err < 0) + goto release; + } + + if (emc->derated) { + err = tegra210_emc_validate_timings(emc, emc->derated, + emc->num_timings); + if (err < 0) + goto release; + } + + /* default to the nominal table */ + emc->timings = emc->nominal; + + /* pick the current timing based on the current EMC clock rate */ + current_rate = clk_get_rate(emc->clk) / 1000; + + for (i = 0; i < emc->num_timings; i++) { + if (emc->timings[i].rate == current_rate) { + emc->last = &emc->timings[i]; + break; + } + } + + if (i == emc->num_timings) { + dev_err(emc->dev, "no EMC table entry found for %lu kHz\n", + current_rate); + err = -ENOENT; + goto release; + } + + /* pick a compatible clock change sequence for the EMC table */ + for (i = 0; i < ARRAY_SIZE(tegra210_emc_sequences); i++) { + const struct tegra210_emc_sequence *sequence = + tegra210_emc_sequences[i]; + + if (emc->timings[0].revision == sequence->revision) { + emc->sequence = sequence; + break; + } + } + + if (!emc->sequence) { + dev_err(&pdev->dev, "sequence %u not supported\n", + emc->timings[0].revision); + err = -ENOTSUPP; + goto release; + } + + emc->offsets = &tegra210_emc_table_register_offsets; + emc->refresh = TEGRA210_EMC_REFRESH_NOMINAL; + + emc->provider.owner = THIS_MODULE; + emc->provider.dev = &pdev->dev; + emc->provider.set_rate = tegra210_emc_set_rate; + + emc->provider.configs = devm_kcalloc(&pdev->dev, emc->num_timings, + sizeof(*emc->provider.configs), + GFP_KERNEL); + if (!emc->provider.configs) { + err = -ENOMEM; + goto release; + } + + emc->provider.num_configs = emc->num_timings; + + for (i = 0; i < emc->provider.num_configs; i++) { + struct tegra210_emc_timing *timing = &emc->timings[i]; + struct tegra210_clk_emc_config *config = + &emc->provider.configs[i]; + u32 value; + + config->rate = timing->rate * 1000UL; + config->value = timing->clk_src_emc; + + value = timing->burst_mc_regs[MC_EMEM_ARB_MISC0_INDEX]; + + if ((value & MC_EMEM_ARB_MISC0_EMC_SAME_FREQ) == 0) + config->same_freq = false; + else + config->same_freq = true; + } + + err = tegra210_clk_emc_attach(emc->clk, &emc->provider); + if (err < 0) { + dev_err(&pdev->dev, "failed to attach to EMC clock: %d\n", err); + goto release; + } + + emc->clkchange_delay = 100; + emc->training_interval = 100; + dev_set_drvdata(emc->dev, emc); + + timer_setup(&emc->refresh_timer, tegra210_emc_poll_refresh, + TIMER_DEFERRABLE); + atomic_set(&emc->refresh_poll, 0); + emc->refresh_poll_interval = 1000; + + timer_setup(&emc->training, tegra210_emc_train, 0); + + tegra210_emc_debugfs_init(emc); + + cd = devm_thermal_of_cooling_device_register(emc->dev, np, "emc", emc, + &tegra210_emc_cd_ops); + if (IS_ERR(cd)) { + err = PTR_ERR(cd); + dev_err(emc->dev, "failed to register cooling device: %d\n", + err); + goto detach; + } + + return 0; + +detach: + debugfs_remove_recursive(emc->debugfs.root); + tegra210_clk_emc_detach(emc->clk); +release: + of_reserved_mem_device_release(emc->dev); + + return err; +} + +static int tegra210_emc_remove(struct platform_device *pdev) +{ + struct tegra210_emc *emc = platform_get_drvdata(pdev); + + debugfs_remove_recursive(emc->debugfs.root); + tegra210_clk_emc_detach(emc->clk); + of_reserved_mem_device_release(emc->dev); + + return 0; +} + +static int __maybe_unused tegra210_emc_suspend(struct device *dev) +{ + struct tegra210_emc *emc = dev_get_drvdata(dev); + int err; + + err = clk_rate_exclusive_get(emc->clk); + if (err < 0) { + dev_err(emc->dev, "failed to acquire clock: %d\n", err); + return err; + } + + emc->resume_rate = clk_get_rate(emc->clk); + + clk_set_rate(emc->clk, 204000000); + tegra210_clk_emc_detach(emc->clk); + + dev_dbg(dev, "suspending at %lu Hz\n", clk_get_rate(emc->clk)); + + return 0; +} + +static int __maybe_unused tegra210_emc_resume(struct device *dev) +{ + struct tegra210_emc *emc = dev_get_drvdata(dev); + int err; + + err = tegra210_clk_emc_attach(emc->clk, &emc->provider); + if (err < 0) { + dev_err(dev, "failed to attach to EMC clock: %d\n", err); + return err; + } + + clk_set_rate(emc->clk, emc->resume_rate); + clk_rate_exclusive_put(emc->clk); + + dev_dbg(dev, "resuming at %lu Hz\n", clk_get_rate(emc->clk)); + + return 0; +} + +static const struct dev_pm_ops tegra210_emc_pm_ops = { + SET_SYSTEM_SLEEP_PM_OPS(tegra210_emc_suspend, tegra210_emc_resume) +}; + +static const struct of_device_id tegra210_emc_of_match[] = { + { .compatible = "nvidia,tegra210-emc", }, + { }, +}; +MODULE_DEVICE_TABLE(of, tegra210_emc_of_match); + +static struct platform_driver tegra210_emc_driver = { + .driver = { + .name = "tegra210-emc", + .of_match_table = tegra210_emc_of_match, + .pm = &tegra210_emc_pm_ops, + }, + .probe = tegra210_emc_probe, + .remove = tegra210_emc_remove, +}; + +module_platform_driver(tegra210_emc_driver); + +MODULE_AUTHOR("Thierry Reding <treding@nvidia.com>"); +MODULE_AUTHOR("Joseph Lo <josephl@nvidia.com>"); +MODULE_DESCRIPTION("NVIDIA Tegra210 EMC driver"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/memory/tegra/tegra210-emc-table.c b/drivers/memory/tegra/tegra210-emc-table.c new file mode 100644 index 000000000000..3e0598363b87 --- /dev/null +++ b/drivers/memory/tegra/tegra210-emc-table.c @@ -0,0 +1,90 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. + */ + +#include <linux/of_reserved_mem.h> + +#include "tegra210-emc.h" + +#define TEGRA_EMC_MAX_FREQS 16 + +static int tegra210_emc_table_device_init(struct reserved_mem *rmem, + struct device *dev) +{ + struct tegra210_emc *emc = dev_get_drvdata(dev); + struct tegra210_emc_timing *timings; + unsigned int i, count = 0; + + timings = memremap(rmem->base, rmem->size, MEMREMAP_WB); + if (!timings) { + dev_err(dev, "failed to map EMC table\n"); + return -ENOMEM; + } + + count = 0; + + for (i = 0; i < TEGRA_EMC_MAX_FREQS; i++) { + if (timings[i].revision == 0) + break; + + count++; + } + + /* only the nominal and derated tables are expected */ + if (emc->derated) { + dev_warn(dev, "excess EMC table '%s'\n", rmem->name); + goto out; + } + + if (emc->nominal) { + if (count != emc->num_timings) { + dev_warn(dev, "%u derated vs. %u nominal entries\n", + count, emc->num_timings); + memunmap(timings); + return -EINVAL; + } + + emc->derated = timings; + } else { + emc->num_timings = count; + emc->nominal = timings; + } + +out: + /* keep track of which table this is */ + rmem->priv = timings; + + return 0; +} + +static void tegra210_emc_table_device_release(struct reserved_mem *rmem, + struct device *dev) +{ + struct tegra210_emc_timing *timings = rmem->priv; + struct tegra210_emc *emc = dev_get_drvdata(dev); + + if ((emc->nominal && timings != emc->nominal) && + (emc->derated && timings != emc->derated)) + dev_warn(dev, "trying to release unassigned EMC table '%s'\n", + rmem->name); + + memunmap(timings); +} + +static const struct reserved_mem_ops tegra210_emc_table_ops = { + .device_init = tegra210_emc_table_device_init, + .device_release = tegra210_emc_table_device_release, +}; + +static int tegra210_emc_table_init(struct reserved_mem *rmem) +{ + pr_debug("Tegra210 EMC table at %pa, size %lu bytes\n", &rmem->base, + (unsigned long)rmem->size); + + rmem->ops = &tegra210_emc_table_ops; + + return 0; +} +RESERVEDMEM_OF_DECLARE(tegra210_emc_table, "nvidia,tegra210-emc-table", + tegra210_emc_table_init); diff --git a/drivers/memory/tegra/tegra210-emc.h b/drivers/memory/tegra/tegra210-emc.h new file mode 100644 index 000000000000..8988bcf15290 --- /dev/null +++ b/drivers/memory/tegra/tegra210-emc.h @@ -0,0 +1,1016 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2015-2020, NVIDIA CORPORATION. All rights reserved. + */ + +#ifndef TEGRA210_EMC_H +#define TEGRA210_EMC_H + +#include <linux/clk.h> +#include <linux/clk/tegra.h> +#include <linux/io.h> +#include <linux/platform_device.h> + +#define DVFS_FGCG_HIGH_SPEED_THRESHOLD 1000 +#define IOBRICK_DCC_THRESHOLD 2400 +#define DVFS_FGCG_MID_SPEED_THRESHOLD 600 + +#define EMC_STATUS_UPDATE_TIMEOUT 1000 + +/* register definitions */ +#define EMC_INTSTATUS 0x0 +#define EMC_INTSTATUS_CLKCHANGE_COMPLETE BIT(4) +#define EMC_DBG 0x8 +#define EMC_DBG_WRITE_MUX_ACTIVE BIT(1) +#define EMC_DBG_WRITE_ACTIVE_ONLY BIT(30) +#define EMC_CFG 0xc +#define EMC_CFG_DRAM_CLKSTOP_PD BIT(31) +#define EMC_CFG_DRAM_CLKSTOP_SR BIT(30) +#define EMC_CFG_DRAM_ACPD BIT(29) +#define EMC_CFG_DYN_SELF_REF BIT(28) +#define EMC_PIN 0x24 +#define EMC_PIN_PIN_CKE BIT(0) +#define EMC_PIN_PIN_CKEB BIT(1) +#define EMC_PIN_PIN_CKE_PER_DEV BIT(2) +#define EMC_TIMING_CONTROL 0x28 +#define EMC_RC 0x2c +#define EMC_RFC 0x30 +#define EMC_RAS 0x34 +#define EMC_RP 0x38 +#define EMC_R2W 0x3c +#define EMC_W2R 0x40 +#define EMC_R2P 0x44 +#define EMC_W2P 0x48 +#define EMC_RD_RCD 0x4c +#define EMC_WR_RCD 0x50 +#define EMC_RRD 0x54 +#define EMC_REXT 0x58 +#define EMC_WDV 0x5c +#define EMC_QUSE 0x60 +#define EMC_QRST 0x64 +#define EMC_QSAFE 0x68 +#define EMC_RDV 0x6c +#define EMC_REFRESH 0x70 +#define EMC_BURST_REFRESH_NUM 0x74 +#define EMC_PDEX2WR 0x78 +#define EMC_PDEX2RD 0x7c +#define EMC_PCHG2PDEN 0x80 +#define EMC_ACT2PDEN 0x84 +#define EMC_AR2PDEN 0x88 +#define EMC_RW2PDEN 0x8c +#define EMC_TXSR 0x90 +#define EMC_TCKE 0x94 +#define EMC_TFAW 0x98 +#define EMC_TRPAB 0x9c +#define EMC_TCLKSTABLE 0xa0 +#define EMC_TCLKSTOP 0xa4 +#define EMC_TREFBW 0xa8 +#define EMC_TPPD 0xac +#define EMC_ODT_WRITE 0xb0 +#define EMC_PDEX2MRR 0xb4 +#define EMC_WEXT 0xb8 +#define EMC_RFC_SLR 0xc0 +#define EMC_MRS_WAIT_CNT2 0xc4 +#define EMC_MRS_WAIT_CNT2_MRS_EXT2_WAIT_CNT_SHIFT 16 +#define EMC_MRS_WAIT_CNT2_MRS_EXT1_WAIT_CNT_SHIFT 0 +#define EMC_MRS_WAIT_CNT 0xc8 +#define EMC_MRS_WAIT_CNT_SHORT_WAIT_SHIFT 0 +#define EMC_MRS_WAIT_CNT_SHORT_WAIT_MASK \ + (0x3FF << EMC_MRS_WAIT_CNT_SHORT_WAIT_SHIFT) + +#define EMC_MRS 0xcc +#define EMC_EMRS 0xd0 +#define EMC_EMRS_USE_EMRS_LONG_CNT BIT(26) +#define EMC_REF 0xd4 +#define EMC_REF_REF_CMD BIT(0) +#define EMC_SELF_REF 0xe0 +#define EMC_MRW 0xe8 +#define EMC_MRW_MRW_OP_SHIFT 0 +#define EMC_MRW_MRW_OP_MASK \ + (0xff << EMC_MRW_MRW_OP_SHIFT) +#define EMC_MRW_MRW_MA_SHIFT 16 +#define EMC_MRW_USE_MRW_EXT_CNT 27 +#define EMC_MRW_MRW_DEV_SELECTN_SHIFT 30 + +#define EMC_MRR 0xec +#define EMC_MRR_DEV_SEL_SHIFT 30 +#define EMC_MRR_DEV_SEL_MASK 0x3 +#define EMC_MRR_MA_SHIFT 16 +#define EMC_MRR_MA_MASK 0xff +#define EMC_MRR_DATA_SHIFT 0 +#define EMC_MRR_DATA_MASK 0xffff + +#define EMC_FBIO_SPARE 0x100 +#define EMC_FBIO_CFG5 0x104 +#define EMC_FBIO_CFG5_DRAM_TYPE_SHIFT 0 +#define EMC_FBIO_CFG5_DRAM_TYPE_MASK \ + (0x3 << EMC_FBIO_CFG5_DRAM_TYPE_SHIFT) +#define EMC_FBIO_CFG5_CMD_TX_DIS BIT(8) + +#define EMC_PDEX2CKE 0x118 +#define EMC_CKE2PDEN 0x11c +#define EMC_MPC 0x128 +#define EMC_EMRS2 0x12c +#define EMC_EMRS2_USE_EMRS2_LONG_CNT BIT(26) +#define EMC_MRW2 0x134 +#define EMC_MRW3 0x138 +#define EMC_MRW4 0x13c +#define EMC_R2R 0x144 +#define EMC_EINPUT 0x14c +#define EMC_EINPUT_DURATION 0x150 +#define EMC_PUTERM_EXTRA 0x154 +#define EMC_TCKESR 0x158 +#define EMC_TPD 0x15c +#define EMC_AUTO_CAL_CONFIG 0x2a4 +#define EMC_AUTO_CAL_CONFIG_AUTO_CAL_COMPUTE_START BIT(0) +#define EMC_AUTO_CAL_CONFIG_AUTO_CAL_MEASURE_STALL BIT(9) +#define EMC_AUTO_CAL_CONFIG_AUTO_CAL_UPDATE_STALL BIT(10) +#define EMC_AUTO_CAL_CONFIG_AUTO_CAL_ENABLE BIT(29) +#define EMC_AUTO_CAL_CONFIG_AUTO_CAL_START BIT(31) +#define EMC_EMC_STATUS 0x2b4 +#define EMC_EMC_STATUS_MRR_DIVLD BIT(20) +#define EMC_EMC_STATUS_TIMING_UPDATE_STALLED BIT(23) +#define EMC_EMC_STATUS_DRAM_IN_POWERDOWN_SHIFT 4 +#define EMC_EMC_STATUS_DRAM_IN_POWERDOWN_MASK \ + (0x3 << EMC_EMC_STATUS_DRAM_IN_POWERDOWN_SHIFT) +#define EMC_EMC_STATUS_DRAM_IN_SELF_REFRESH_SHIFT 8 +#define EMC_EMC_STATUS_DRAM_IN_SELF_REFRESH_MASK \ + (0x3 << EMC_EMC_STATUS_DRAM_IN_SELF_REFRESH_SHIFT) + +#define EMC_CFG_2 0x2b8 +#define EMC_CFG_DIG_DLL 0x2bc +#define EMC_CFG_DIG_DLL_CFG_DLL_EN BIT(0) +#define EMC_CFG_DIG_DLL_CFG_DLL_STALL_ALL_UNTIL_LOCK BIT(1) +#define EMC_CFG_DIG_DLL_CFG_DLL_STALL_ALL_TRAFFIC BIT(3) +#define EMC_CFG_DIG_DLL_CFG_DLL_STALL_RW_UNTIL_LOCK BIT(4) +#define EMC_CFG_DIG_DLL_CFG_DLL_MODE_SHIFT 6 +#define EMC_CFG_DIG_DLL_CFG_DLL_MODE_MASK \ + (0x3 << EMC_CFG_DIG_DLL_CFG_DLL_MODE_SHIFT) +#define EMC_CFG_DIG_DLL_CFG_DLL_LOCK_LIMIT_SHIFT 8 +#define EMC_CFG_DIG_DLL_CFG_DLL_LOCK_LIMIT_MASK \ + (0x7 << EMC_CFG_DIG_DLL_CFG_DLL_LOCK_LIMIT_SHIFT) + +#define EMC_CFG_DIG_DLL_PERIOD 0x2c0 +#define EMC_DIG_DLL_STATUS 0x2c4 +#define EMC_DIG_DLL_STATUS_DLL_LOCK BIT(15) +#define EMC_DIG_DLL_STATUS_DLL_PRIV_UPDATED BIT(17) +#define EMC_DIG_DLL_STATUS_DLL_OUT_SHIFT 0 +#define EMC_DIG_DLL_STATUS_DLL_OUT_MASK \ + (0x7ff << EMC_DIG_DLL_STATUS_DLL_OUT_SHIFT) + +#define EMC_CFG_DIG_DLL_1 0x2c8 +#define EMC_RDV_MASK 0x2cc +#define EMC_WDV_MASK 0x2d0 +#define EMC_RDV_EARLY_MASK 0x2d4 +#define EMC_RDV_EARLY 0x2d8 +#define EMC_AUTO_CAL_CONFIG8 0x2dc +#define EMC_ZCAL_INTERVAL 0x2e0 +#define EMC_ZCAL_WAIT_CNT 0x2e4 +#define EMC_ZCAL_WAIT_CNT_ZCAL_WAIT_CNT_MASK 0x7ff +#define EMC_ZCAL_WAIT_CNT_ZCAL_WAIT_CNT_SHIFT 0 + +#define EMC_ZQ_CAL 0x2ec +#define EMC_ZQ_CAL_DEV_SEL_SHIFT 30 +#define EMC_ZQ_CAL_LONG BIT(4) +#define EMC_ZQ_CAL_ZQ_LATCH_CMD BIT(1) +#define EMC_ZQ_CAL_ZQ_CAL_CMD BIT(0) +#define EMC_FDPD_CTRL_DQ 0x310 +#define EMC_FDPD_CTRL_CMD 0x314 +#define EMC_PMACRO_CMD_BRICK_CTRL_FDPD 0x318 +#define EMC_PMACRO_DATA_BRICK_CTRL_FDPD 0x31c +#define EMC_PMACRO_BRICK_CTRL_RFU1 0x330 +#define EMC_PMACRO_BRICK_CTRL_RFU2 0x334 +#define EMC_TR_TIMING_0 0x3b4 +#define EMC_TR_CTRL_1 0x3bc +#define EMC_TR_RDV 0x3c4 +#define EMC_STALL_THEN_EXE_AFTER_CLKCHANGE 0x3cc +#define EMC_SEL_DPD_CTRL 0x3d8 +#define EMC_SEL_DPD_CTRL_DATA_SEL_DPD_EN BIT(8) +#define EMC_SEL_DPD_CTRL_ODT_SEL_DPD_EN BIT(5) +#define EMC_SEL_DPD_CTRL_RESET_SEL_DPD_EN BIT(4) +#define EMC_SEL_DPD_CTRL_CA_SEL_DPD_EN BIT(3) +#define EMC_SEL_DPD_CTRL_CLK_SEL_DPD_EN BIT(2) +#define EMC_PRE_REFRESH_REQ_CNT 0x3dc +#define EMC_DYN_SELF_REF_CONTROL 0x3e0 +#define EMC_TXSRDLL 0x3e4 +#define EMC_CCFIFO_ADDR 0x3e8 +#define EMC_CCFIFO_ADDR_STALL_BY_1 (1 << 31) +#define EMC_CCFIFO_ADDR_STALL(x) (((x) & 0x7fff) << 16) +#define EMC_CCFIFO_ADDR_OFFSET(x) ((x) & 0xffff) +#define EMC_CCFIFO_DATA 0x3ec +#define EMC_TR_QPOP 0x3f4 +#define EMC_TR_RDV_MASK 0x3f8 +#define EMC_TR_QSAFE 0x3fc +#define EMC_TR_QRST 0x400 +#define EMC_ISSUE_QRST 0x428 +#define EMC_AUTO_CAL_CONFIG2 0x458 +#define EMC_AUTO_CAL_CONFIG3 0x45c +#define EMC_TR_DVFS 0x460 +#define EMC_AUTO_CAL_CHANNEL 0x464 +#define EMC_IBDLY 0x468 +#define EMC_OBDLY 0x46c +#define EMC_TXDSRVTTGEN 0x480 +#define EMC_WE_DURATION 0x48c +#define EMC_WS_DURATION 0x490 +#define EMC_WEV 0x494 +#define EMC_WSV 0x498 +#define EMC_CFG_3 0x49c +#define EMC_MRW6 0x4a4 +#define EMC_MRW7 0x4a8 +#define EMC_MRW8 0x4ac +#define EMC_MRW9 0x4b0 +#define EMC_MRW10 0x4b4 +#define EMC_MRW11 0x4b8 +#define EMC_MRW12 0x4bc +#define EMC_MRW13 0x4c0 +#define EMC_MRW14 0x4c4 +#define EMC_MRW15 0x4d0 +#define EMC_CFG_SYNC 0x4d4 +#define EMC_FDPD_CTRL_CMD_NO_RAMP 0x4d8 +#define EMC_FDPD_CTRL_CMD_NO_RAMP_CMD_DPD_NO_RAMP_ENABLE BIT(0) +#define EMC_WDV_CHK 0x4e0 +#define EMC_CFG_PIPE_2 0x554 +#define EMC_CFG_PIPE_CLK 0x558 +#define EMC_CFG_PIPE_CLK_CLK_ALWAYS_ON BIT(0) +#define EMC_CFG_PIPE_1 0x55c +#define EMC_CFG_PIPE 0x560 +#define EMC_QPOP 0x564 +#define EMC_QUSE_WIDTH 0x568 +#define EMC_PUTERM_WIDTH 0x56c +#define EMC_AUTO_CAL_CONFIG7 0x574 +#define EMC_REFCTRL2 0x580 +#define EMC_FBIO_CFG7 0x584 +#define EMC_FBIO_CFG7_CH0_ENABLE BIT(1) +#define EMC_FBIO_CFG7_CH1_ENABLE BIT(2) +#define EMC_DATA_BRLSHFT_0 0x588 +#define EMC_DATA_BRLSHFT_0_RANK0_BYTE7_DATA_BRLSHFT_SHIFT 21 +#define EMC_DATA_BRLSHFT_0_RANK0_BYTE7_DATA_BRLSHFT_MASK \ + (0x7 << EMC_DATA_BRLSHFT_0_RANK0_BYTE7_DATA_BRLSHFT_SHIFT) +#define EMC_DATA_BRLSHFT_0_RANK0_BYTE6_DATA_BRLSHFT_SHIFT 18 +#define EMC_DATA_BRLSHFT_0_RANK0_BYTE6_DATA_BRLSHFT_MASK \ + (0x7 << EMC_DATA_BRLSHFT_0_RANK0_BYTE6_DATA_BRLSHFT_SHIFT) +#define EMC_DATA_BRLSHFT_0_RANK0_BYTE5_DATA_BRLSHFT_SHIFT 15 +#define EMC_DATA_BRLSHFT_0_RANK0_BYTE5_DATA_BRLSHFT_MASK \ + (0x7 << EMC_DATA_BRLSHFT_0_RANK0_BYTE5_DATA_BRLSHFT_SHIFT) +#define EMC_DATA_BRLSHFT_0_RANK0_BYTE4_DATA_BRLSHFT_SHIFT 12 +#define EMC_DATA_BRLSHFT_0_RANK0_BYTE4_DATA_BRLSHFT_MASK \ + (0x7 << EMC_DATA_BRLSHFT_0_RANK0_BYTE4_DATA_BRLSHFT_SHIFT) +#define EMC_DATA_BRLSHFT_0_RANK0_BYTE3_DATA_BRLSHFT_SHIFT 9 +#define EMC_DATA_BRLSHFT_0_RANK0_BYTE3_DATA_BRLSHFT_MASK \ + (0x7 << EMC_DATA_BRLSHFT_0_RANK0_BYTE3_DATA_BRLSHFT_SHIFT) +#define EMC_DATA_BRLSHFT_0_RANK0_BYTE2_DATA_BRLSHFT_SHIFT 6 +#define EMC_DATA_BRLSHFT_0_RANK0_BYTE2_DATA_BRLSHFT_MASK \ + (0x7 << EMC_DATA_BRLSHFT_0_RANK0_BYTE2_DATA_BRLSHFT_SHIFT) +#define EMC_DATA_BRLSHFT_0_RANK0_BYTE1_DATA_BRLSHFT_SHIFT 3 +#define EMC_DATA_BRLSHFT_0_RANK0_BYTE1_DATA_BRLSHFT_MASK \ + (0x7 << EMC_DATA_BRLSHFT_0_RANK0_BYTE1_DATA_BRLSHFT_SHIFT) +#define EMC_DATA_BRLSHFT_0_RANK0_BYTE0_DATA_BRLSHFT_SHIFT 0 +#define EMC_DATA_BRLSHFT_0_RANK0_BYTE0_DATA_BRLSHFT_MASK \ + (0x7 << EMC_DATA_BRLSHFT_0_RANK0_BYTE0_DATA_BRLSHFT_SHIFT) + +#define EMC_DATA_BRLSHFT_1 0x58c +#define EMC_DATA_BRLSHFT_1_RANK1_BYTE7_DATA_BRLSHFT_SHIFT 21 +#define EMC_DATA_BRLSHFT_1_RANK1_BYTE7_DATA_BRLSHFT_MASK \ + (0x7 << EMC_DATA_BRLSHFT_1_RANK1_BYTE7_DATA_BRLSHFT_SHIFT) +#define EMC_DATA_BRLSHFT_1_RANK1_BYTE6_DATA_BRLSHFT_SHIFT 18 +#define EMC_DATA_BRLSHFT_1_RANK1_BYTE6_DATA_BRLSHFT_MASK \ + (0x7 << EMC_DATA_BRLSHFT_1_RANK1_BYTE6_DATA_BRLSHFT_SHIFT) +#define EMC_DATA_BRLSHFT_1_RANK1_BYTE5_DATA_BRLSHFT_SHIFT 15 +#define EMC_DATA_BRLSHFT_1_RANK1_BYTE5_DATA_BRLSHFT_MASK \ + (0x7 << EMC_DATA_BRLSHFT_1_RANK1_BYTE5_DATA_BRLSHFT_SHIFT) +#define EMC_DATA_BRLSHFT_1_RANK1_BYTE4_DATA_BRLSHFT_SHIFT 12 +#define EMC_DATA_BRLSHFT_1_RANK1_BYTE4_DATA_BRLSHFT_MASK \ + (0x7 << EMC_DATA_BRLSHFT_1_RANK1_BYTE4_DATA_BRLSHFT_SHIFT) +#define EMC_DATA_BRLSHFT_1_RANK1_BYTE3_DATA_BRLSHFT_SHIFT 9 +#define EMC_DATA_BRLSHFT_1_RANK1_BYTE3_DATA_BRLSHFT_MASK \ + (0x7 << EMC_DATA_BRLSHFT_1_RANK1_BYTE3_DATA_BRLSHFT_SHIFT) +#define EMC_DATA_BRLSHFT_1_RANK1_BYTE2_DATA_BRLSHFT_SHIFT 6 +#define EMC_DATA_BRLSHFT_1_RANK1_BYTE2_DATA_BRLSHFT_MASK \ + (0x7 << EMC_DATA_BRLSHFT_1_RANK1_BYTE2_DATA_BRLSHFT_SHIFT) +#define EMC_DATA_BRLSHFT_1_RANK1_BYTE1_DATA_BRLSHFT_SHIFT 3 +#define EMC_DATA_BRLSHFT_1_RANK1_BYTE1_DATA_BRLSHFT_MASK \ + (0x7 << EMC_DATA_BRLSHFT_1_RANK1_BYTE1_DATA_BRLSHFT_SHIFT) +#define EMC_DATA_BRLSHFT_1_RANK1_BYTE0_DATA_BRLSHFT_SHIFT 0 +#define EMC_DATA_BRLSHFT_1_RANK1_BYTE0_DATA_BRLSHFT_MASK \ + (0x7 << EMC_DATA_BRLSHFT_1_RANK1_BYTE0_DATA_BRLSHFT_SHIFT) + +#define EMC_RFCPB 0x590 +#define EMC_DQS_BRLSHFT_0 0x594 +#define EMC_DQS_BRLSHFT_1 0x598 +#define EMC_CMD_BRLSHFT_0 0x59c +#define EMC_CMD_BRLSHFT_1 0x5a0 +#define EMC_CMD_BRLSHFT_2 0x5a4 +#define EMC_CMD_BRLSHFT_3 0x5a8 +#define EMC_QUSE_BRLSHFT_0 0x5ac +#define EMC_AUTO_CAL_CONFIG4 0x5b0 +#define EMC_AUTO_CAL_CONFIG5 0x5b4 +#define EMC_QUSE_BRLSHFT_1 0x5b8 +#define EMC_QUSE_BRLSHFT_2 0x5bc +#define EMC_CCDMW 0x5c0 +#define EMC_QUSE_BRLSHFT_3 0x5c4 +#define EMC_AUTO_CAL_CONFIG6 0x5cc +#define EMC_DLL_CFG_0 0x5e4 +#define EMC_DLL_CFG_1 0x5e8 +#define EMC_DLL_CFG_1_DDLLCAL_CTRL_START_TRIM_SHIFT 10 +#define EMC_DLL_CFG_1_DDLLCAL_CTRL_START_TRIM_MASK \ + (0x7ff << EMC_DLL_CFG_1_DDLLCAL_CTRL_START_TRIM_SHIFT) + +#define EMC_CONFIG_SAMPLE_DELAY 0x5f0 +#define EMC_CFG_UPDATE 0x5f4 +#define EMC_CFG_UPDATE_UPDATE_DLL_IN_UPDATE_SHIFT 9 +#define EMC_CFG_UPDATE_UPDATE_DLL_IN_UPDATE_MASK \ + (0x3 << EMC_CFG_UPDATE_UPDATE_DLL_IN_UPDATE_SHIFT) + +#define EMC_PMACRO_QUSE_DDLL_RANK0_0 0x600 +#define EMC_PMACRO_QUSE_DDLL_RANK0_1 0x604 +#define EMC_PMACRO_QUSE_DDLL_RANK0_2 0x608 +#define EMC_PMACRO_QUSE_DDLL_RANK0_3 0x60c +#define EMC_PMACRO_QUSE_DDLL_RANK0_4 0x610 +#define EMC_PMACRO_QUSE_DDLL_RANK0_5 0x614 +#define EMC_PMACRO_QUSE_DDLL_RANK1_0 0x620 +#define EMC_PMACRO_QUSE_DDLL_RANK1_1 0x624 +#define EMC_PMACRO_QUSE_DDLL_RANK1_2 0x628 +#define EMC_PMACRO_QUSE_DDLL_RANK1_3 0x62c +#define EMC_PMACRO_QUSE_DDLL_RANK1_4 0x630 +#define EMC_PMACRO_QUSE_DDLL_RANK1_5 0x634 +#define EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_0 0x640 +#define EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_0_OB_DDLL_LONG_DQ_RANK0_BYTE1_SHIFT \ + 16 +#define EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_0_OB_DDLL_LONG_DQ_RANK0_BYTE1_MASK \ + (0x3ff << \ + EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_0_OB_DDLL_LONG_DQ_RANK0_BYTE1_SHIFT) +#define EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_0_OB_DDLL_LONG_DQ_RANK0_BYTE0_SHIFT \ + 0 +#define EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_0_OB_DDLL_LONG_DQ_RANK0_BYTE0_MASK \ + (0x3ff << \ + EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_0_OB_DDLL_LONG_DQ_RANK0_BYTE0_SHIFT) + +#define EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_1 0x644 +#define EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_1_OB_DDLL_LONG_DQ_RANK0_BYTE3_SHIFT \ + 16 +#define EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_1_OB_DDLL_LONG_DQ_RANK0_BYTE3_MASK \ + (0x3ff << \ + EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_1_OB_DDLL_LONG_DQ_RANK0_BYTE3_SHIFT) +#define EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_1_OB_DDLL_LONG_DQ_RANK0_BYTE2_SHIFT \ + 0 +#define EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_1_OB_DDLL_LONG_DQ_RANK0_BYTE2_MASK \ + (0x3ff << \ + EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_1_OB_DDLL_LONG_DQ_RANK0_BYTE2_SHIFT) + +#define EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_2 0x648 +#define EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_2_OB_DDLL_LONG_DQ_RANK0_BYTE5_SHIFT \ + 16 +#define EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_2_OB_DDLL_LONG_DQ_RANK0_BYTE5_MASK \ + (0x3ff << \ + EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_2_OB_DDLL_LONG_DQ_RANK0_BYTE5_SHIFT) +#define EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_2_OB_DDLL_LONG_DQ_RANK0_BYTE4_SHIFT \ + 0 +#define EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_2_OB_DDLL_LONG_DQ_RANK0_BYTE4_MASK \ + (0x3ff << \ + EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_2_OB_DDLL_LONG_DQ_RANK0_BYTE4_SHIFT) + +#define EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_3 0x64c +#define EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_3_OB_DDLL_LONG_DQ_RANK0_BYTE7_SHIFT \ + 16 +#define EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_3_OB_DDLL_LONG_DQ_RANK0_BYTE7_MASK \ + (0x3ff << \ + EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_3_OB_DDLL_LONG_DQ_RANK0_BYTE7_SHIFT) +#define EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_3_OB_DDLL_LONG_DQ_RANK0_BYTE6_SHIFT \ + 0 +#define EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_3_OB_DDLL_LONG_DQ_RANK0_BYTE6_MASK \ + (0x3ff << \ + EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_3_OB_DDLL_LONG_DQ_RANK0_BYTE6_SHIFT) + +#define EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_4 0x650 +#define EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_5 0x654 +#define EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_0 0x660 +#define EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_0_OB_DDLL_LONG_DQ_RANK1_BYTE1_SHIFT \ + 16 +#define EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_0_OB_DDLL_LONG_DQ_RANK1_BYTE1_MASK \ + (0x3ff << \ + EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_0_OB_DDLL_LONG_DQ_RANK1_BYTE1_SHIFT) +#define EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_0_OB_DDLL_LONG_DQ_RANK1_BYTE0_SHIFT \ + 0 +#define EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_0_OB_DDLL_LONG_DQ_RANK1_BYTE0_MASK \ + (0x3ff << \ + EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_0_OB_DDLL_LONG_DQ_RANK1_BYTE0_SHIFT) + +#define EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_1 0x664 +#define EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_1_OB_DDLL_LONG_DQ_RANK1_BYTE3_SHIFT \ + 16 +#define EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_1_OB_DDLL_LONG_DQ_RANK1_BYTE3_MASK \ + (0x3ff << \ + EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_1_OB_DDLL_LONG_DQ_RANK1_BYTE3_SHIFT) +#define EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_1_OB_DDLL_LONG_DQ_RANK1_BYTE2_SHIFT \ + 0 +#define EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_1_OB_DDLL_LONG_DQ_RANK1_BYTE2_MASK \ + (0x3ff << \ + EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_1_OB_DDLL_LONG_DQ_RANK1_BYTE2_SHIFT) + +#define EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_2 0x668 +#define EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_2_OB_DDLL_LONG_DQ_RANK1_BYTE5_SHIFT \ + 16 +#define EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_2_OB_DDLL_LONG_DQ_RANK1_BYTE5_MASK \ + (0x3ff << \ + EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_2_OB_DDLL_LONG_DQ_RANK1_BYTE5_SHIFT) +#define EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_2_OB_DDLL_LONG_DQ_RANK1_BYTE4_SHIFT \ + 0 +#define EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_2_OB_DDLL_LONG_DQ_RANK1_BYTE4_MASK \ + (0x3ff << \ + EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_2_OB_DDLL_LONG_DQ_RANK1_BYTE4_SHIFT) + +#define EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_3 0x66c +#define EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_3_OB_DDLL_LONG_DQ_RANK1_BYTE7_SHIFT \ + 16 +#define EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_3_OB_DDLL_LONG_DQ_RANK1_BYTE7_MASK \ + (0x3ff << \ + EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_3_OB_DDLL_LONG_DQ_RANK1_BYTE7_SHIFT) +#define EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_3_OB_DDLL_LONG_DQ_RANK1_BYTE6_SHIFT \ + 0 +#define EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_3_OB_DDLL_LONG_DQ_RANK1_BYTE6_MASK \ + (0x3ff << \ + EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_3_OB_DDLL_LONG_DQ_RANK1_BYTE6_SHIFT) + +#define EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_4 0x670 +#define EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_5 0x674 +#define EMC_PMACRO_OB_DDLL_LONG_DQS_RANK0_0 0x680 +#define EMC_PMACRO_OB_DDLL_LONG_DQS_RANK0_1 0x684 +#define EMC_PMACRO_OB_DDLL_LONG_DQS_RANK0_2 0x688 +#define EMC_PMACRO_OB_DDLL_LONG_DQS_RANK0_3 0x68c +#define EMC_PMACRO_OB_DDLL_LONG_DQS_RANK0_4 0x690 +#define EMC_PMACRO_OB_DDLL_LONG_DQS_RANK0_5 0x694 +#define EMC_PMACRO_OB_DDLL_LONG_DQS_RANK1_0 0x6a0 +#define EMC_PMACRO_OB_DDLL_LONG_DQS_RANK1_1 0x6a4 +#define EMC_PMACRO_OB_DDLL_LONG_DQS_RANK1_2 0x6a8 +#define EMC_PMACRO_OB_DDLL_LONG_DQS_RANK1_3 0x6ac +#define EMC_PMACRO_OB_DDLL_LONG_DQS_RANK1_4 0x6b0 +#define EMC_PMACRO_OB_DDLL_LONG_DQS_RANK1_5 0x6b4 +#define EMC_PMACRO_IB_DDLL_LONG_DQS_RANK0_0 0x6c0 +#define EMC_PMACRO_IB_DDLL_LONG_DQS_RANK0_1 0x6c4 +#define EMC_PMACRO_IB_DDLL_LONG_DQS_RANK0_2 0x6c8 +#define EMC_PMACRO_IB_DDLL_LONG_DQS_RANK0_3 0x6cc +#define EMC_PMACRO_IB_DDLL_LONG_DQS_RANK1_0 0x6e0 +#define EMC_PMACRO_IB_DDLL_LONG_DQS_RANK1_1 0x6e4 +#define EMC_PMACRO_IB_DDLL_LONG_DQS_RANK1_2 0x6e8 +#define EMC_PMACRO_IB_DDLL_LONG_DQS_RANK1_3 0x6ec +#define EMC_PMACRO_TX_PWRD_0 0x720 +#define EMC_PMACRO_TX_PWRD_1 0x724 +#define EMC_PMACRO_TX_PWRD_2 0x728 +#define EMC_PMACRO_TX_PWRD_3 0x72c +#define EMC_PMACRO_TX_PWRD_4 0x730 +#define EMC_PMACRO_TX_PWRD_5 0x734 +#define EMC_PMACRO_TX_SEL_CLK_SRC_0 0x740 +#define EMC_PMACRO_TX_SEL_CLK_SRC_1 0x744 +#define EMC_PMACRO_TX_SEL_CLK_SRC_3 0x74c +#define EMC_PMACRO_TX_SEL_CLK_SRC_2 0x748 +#define EMC_PMACRO_TX_SEL_CLK_SRC_4 0x750 +#define EMC_PMACRO_TX_SEL_CLK_SRC_5 0x754 +#define EMC_PMACRO_DDLL_BYPASS 0x760 +#define EMC_PMACRO_DDLL_PWRD_0 0x770 +#define EMC_PMACRO_DDLL_PWRD_1 0x774 +#define EMC_PMACRO_DDLL_PWRD_2 0x778 +#define EMC_PMACRO_CMD_CTRL_0 0x780 +#define EMC_PMACRO_CMD_CTRL_1 0x784 +#define EMC_PMACRO_CMD_CTRL_2 0x788 +#define EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE0_0 0x800 +#define EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE0_1 0x804 +#define EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE0_2 0x808 +#define EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE0_3 0x80c +#define EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE1_0 0x810 +#define EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE1_1 0x814 +#define EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE1_2 0x818 +#define EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE1_3 0x81c +#define EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE2_0 0x820 +#define EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE2_1 0x824 +#define EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE2_2 0x828 +#define EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE2_3 0x82c +#define EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE3_0 0x830 +#define EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE3_1 0x834 +#define EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE3_2 0x838 +#define EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE3_3 0x83c +#define EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE4_0 0x840 +#define EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE4_1 0x844 +#define EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE4_2 0x848 +#define EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE4_3 0x84c +#define EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE5_0 0x850 +#define EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE5_1 0x854 +#define EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE5_2 0x858 +#define EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE5_3 0x85c +#define EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE6_0 0x860 +#define EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE6_1 0x864 +#define EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE6_2 0x868 +#define EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE6_3 0x86c +#define EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE7_0 0x870 +#define EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE7_1 0x874 +#define EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE7_2 0x878 +#define EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE7_3 0x87c +#define EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD0_0 0x880 +#define EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD0_1 0x884 +#define EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD0_2 0x888 +#define EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD0_3 0x88c +#define EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD1_0 0x890 +#define EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD1_1 0x894 +#define EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD1_2 0x898 +#define EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD1_3 0x89c +#define EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD2_0 0x8a0 +#define EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD2_1 0x8a4 +#define EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD2_2 0x8a8 +#define EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD2_3 0x8ac +#define EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD3_0 0x8b0 +#define EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD3_1 0x8b4 +#define EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD3_2 0x8b8 +#define EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD3_3 0x8bc +#define EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE0_0 0x900 +#define EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE0_1 0x904 +#define EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE0_2 0x908 +#define EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE0_3 0x90c +#define EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE1_0 0x910 +#define EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE1_1 0x914 +#define EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE1_2 0x918 +#define EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE1_3 0x91c +#define EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE2_0 0x920 +#define EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE2_1 0x924 +#define EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE2_2 0x928 +#define EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE2_3 0x92c +#define EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE3_0 0x930 +#define EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE3_1 0x934 +#define EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE3_2 0x938 +#define EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE3_3 0x93c +#define EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE4_0 0x940 +#define EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE4_1 0x944 +#define EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE4_2 0x948 +#define EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE4_3 0x94c +#define EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE5_0 0x950 +#define EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE5_1 0x954 +#define EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE5_2 0x958 +#define EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE5_3 0x95c +#define EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE6_0 0x960 +#define EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE6_1 0x964 +#define EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE6_2 0x968 +#define EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE6_3 0x96c +#define EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE7_0 0x970 +#define EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE7_1 0x974 +#define EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE7_2 0x978 +#define EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE7_3 0x97c +#define EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_CMD0_0 0x980 +#define EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_CMD0_1 0x984 +#define EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_CMD0_2 0x988 +#define EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_CMD0_3 0x98c +#define EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_CMD1_0 0x990 +#define EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_CMD1_1 0x994 +#define EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_CMD1_2 0x998 +#define EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_CMD1_3 0x99c +#define EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_CMD2_0 0x9a0 +#define EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_CMD2_1 0x9a4 +#define EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_CMD2_2 0x9a8 +#define EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_CMD2_3 0x9ac +#define EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_CMD3_0 0x9b0 +#define EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_CMD3_1 0x9b4 +#define EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_CMD3_2 0x9b8 +#define EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_CMD3_3 0x9bc +#define EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE0_0 0xa00 +#define EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE0_1 0xa04 +#define EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE0_2 0xa08 +#define EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE1_0 0xa10 +#define EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE1_1 0xa14 +#define EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE1_2 0xa18 +#define EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE2_0 0xa20 +#define EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE2_1 0xa24 +#define EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE2_2 0xa28 +#define EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE3_0 0xa30 +#define EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE3_1 0xa34 +#define EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE3_2 0xa38 +#define EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE4_0 0xa40 +#define EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE4_1 0xa44 +#define EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE4_2 0xa48 +#define EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE5_0 0xa50 +#define EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE5_1 0xa54 +#define EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE5_2 0xa58 +#define EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE6_0 0xa60 +#define EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE6_1 0xa64 +#define EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE6_2 0xa68 +#define EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE7_0 0xa70 +#define EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE7_1 0xa74 +#define EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE7_2 0xa78 +#define EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE0_0 0xb00 +#define EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE0_1 0xb04 +#define EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE0_2 0xb08 +#define EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE1_0 0xb10 +#define EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE1_1 0xb14 +#define EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE1_2 0xb18 +#define EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE2_0 0xb20 +#define EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE2_1 0xb24 +#define EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE2_2 0xb28 +#define EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE3_0 0xb30 +#define EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE3_1 0xb34 +#define EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE3_2 0xb38 +#define EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE4_0 0xb40 +#define EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE4_1 0xb44 +#define EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE4_2 0xb48 +#define EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE5_0 0xb50 +#define EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE5_1 0xb54 +#define EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE5_2 0xb58 +#define EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE6_0 0xb60 +#define EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE6_1 0xb64 +#define EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE6_2 0xb68 +#define EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE7_0 0xb70 +#define EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE7_1 0xb74 +#define EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE7_2 0xb78 +#define EMC_PMACRO_IB_VREF_DQ_0 0xbe0 +#define EMC_PMACRO_IB_VREF_DQ_1 0xbe4 +#define EMC_PMACRO_IB_VREF_DQS_0 0xbf0 +#define EMC_PMACRO_IB_VREF_DQS_1 0xbf4 +#define EMC_PMACRO_DDLL_LONG_CMD_0 0xc00 +#define EMC_PMACRO_DDLL_LONG_CMD_1 0xc04 +#define EMC_PMACRO_DDLL_LONG_CMD_2 0xc08 +#define EMC_PMACRO_DDLL_LONG_CMD_3 0xc0c +#define EMC_PMACRO_DDLL_LONG_CMD_4 0xc10 +#define EMC_PMACRO_DDLL_LONG_CMD_5 0xc14 +#define EMC_PMACRO_DDLL_SHORT_CMD_0 0xc20 +#define EMC_PMACRO_DDLL_SHORT_CMD_1 0xc24 +#define EMC_PMACRO_DDLL_SHORT_CMD_2 0xc28 +#define EMC_PMACRO_CFG_PM_GLOBAL_0 0xc30 +#define EMC_PMACRO_CFG_PM_GLOBAL_0_DISABLE_CFG_BYTE0 BIT(16) +#define EMC_PMACRO_CFG_PM_GLOBAL_0_DISABLE_CFG_BYTE1 BIT(17) +#define EMC_PMACRO_CFG_PM_GLOBAL_0_DISABLE_CFG_BYTE2 BIT(18) +#define EMC_PMACRO_CFG_PM_GLOBAL_0_DISABLE_CFG_BYTE3 BIT(19) +#define EMC_PMACRO_CFG_PM_GLOBAL_0_DISABLE_CFG_BYTE4 BIT(20) +#define EMC_PMACRO_CFG_PM_GLOBAL_0_DISABLE_CFG_BYTE5 BIT(21) +#define EMC_PMACRO_CFG_PM_GLOBAL_0_DISABLE_CFG_BYTE6 BIT(22) +#define EMC_PMACRO_CFG_PM_GLOBAL_0_DISABLE_CFG_BYTE7 BIT(23) +#define EMC_PMACRO_VTTGEN_CTRL_0 0xc34 +#define EMC_PMACRO_VTTGEN_CTRL_1 0xc38 +#define EMC_PMACRO_BG_BIAS_CTRL_0 0xc3c +#define EMC_PMACRO_BG_BIAS_CTRL_0_BG_E_PWRD BIT(0) +#define EMC_PMACRO_BG_BIAS_CTRL_0_BGLP_E_PWRD BIT(2) +#define EMC_PMACRO_PAD_CFG_CTRL 0xc40 +#define EMC_PMACRO_ZCTRL 0xc44 +#define EMC_PMACRO_CMD_PAD_RX_CTRL 0xc50 +#define EMC_PMACRO_DATA_PAD_RX_CTRL 0xc54 +#define EMC_PMACRO_CMD_RX_TERM_MODE 0xc58 +#define EMC_PMACRO_DATA_RX_TERM_MODE 0xc5c +#define EMC_PMACRO_CMD_PAD_TX_CTRL 0xc60 +#define EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQ_TX_E_DCC BIT(1) +#define EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQSP_TX_E_DCC BIT(9) +#define EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQSN_TX_E_DCC BIT(16) +#define EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_CMD_TX_E_DCC BIT(24) +#define EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQ_TX_DRVFORCEON BIT(26) + +#define EMC_PMACRO_DATA_PAD_TX_CTRL 0xc64 +#define EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQ_E_IVREF BIT(0) +#define EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQ_TX_E_DCC BIT(1) +#define EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQS_E_IVREF BIT(8) +#define EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQSP_TX_E_DCC BIT(9) +#define EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQSN_TX_E_DCC BIT(16) +#define EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_CMD_TX_E_DCC BIT(24) + +#define EMC_PMACRO_COMMON_PAD_TX_CTRL 0xc68 +#define EMC_PMACRO_AUTOCAL_CFG_COMMON 0xc78 +#define EMC_PMACRO_AUTOCAL_CFG_COMMON_E_CAL_BYPASS_DVFS BIT(16) +#define EMC_PMACRO_VTTGEN_CTRL_2 0xcf0 +#define EMC_PMACRO_IB_RXRT 0xcf4 +#define EMC_PMACRO_TRAINING_CTRL_0 0xcf8 +#define EMC_PMACRO_TRAINING_CTRL_0_CH0_TRAINING_E_WRPTR BIT(3) +#define EMC_PMACRO_TRAINING_CTRL_1 0xcfc +#define EMC_PMACRO_TRAINING_CTRL_1_CH1_TRAINING_E_WRPTR BIT(3) +#define EMC_TRAINING_CTRL 0xe04 +#define EMC_TRAINING_QUSE_CORS_CTRL 0xe0c +#define EMC_TRAINING_QUSE_FINE_CTRL 0xe10 +#define EMC_TRAINING_QUSE_CTRL_MISC 0xe14 +#define EMC_TRAINING_WRITE_FINE_CTRL 0xe18 +#define EMC_TRAINING_WRITE_CTRL_MISC 0xe1c +#define EMC_TRAINING_WRITE_VREF_CTRL 0xe20 +#define EMC_TRAINING_READ_FINE_CTRL 0xe24 +#define EMC_TRAINING_READ_CTRL_MISC 0xe28 +#define EMC_TRAINING_READ_VREF_CTRL 0xe2c +#define EMC_TRAINING_CA_FINE_CTRL 0xe30 +#define EMC_TRAINING_CA_CTRL_MISC 0xe34 +#define EMC_TRAINING_CA_CTRL_MISC1 0xe38 +#define EMC_TRAINING_CA_VREF_CTRL 0xe3c +#define EMC_TRAINING_SETTLE 0xe44 +#define EMC_TRAINING_MPC 0xe5c +#define EMC_TRAINING_VREF_SETTLE 0xe6c +#define EMC_TRAINING_QUSE_VREF_CTRL 0xed0 +#define EMC_TRAINING_OPT_DQS_IB_VREF_RANK0 0xed4 +#define EMC_TRAINING_OPT_DQS_IB_VREF_RANK1 0xed8 + +#define EMC_COPY_TABLE_PARAM_PERIODIC_FIELDS BIT(0) +#define EMC_COPY_TABLE_PARAM_TRIM_REGS BIT(1) + +enum burst_regs_list { + EMC_RP_INDEX = 6, + EMC_R2P_INDEX = 9, + EMC_W2P_INDEX, + EMC_MRW6_INDEX = 31, + EMC_REFRESH_INDEX = 41, + EMC_PRE_REFRESH_REQ_CNT_INDEX = 43, + EMC_TRPAB_INDEX = 59, + EMC_MRW7_INDEX = 62, + EMC_FBIO_CFG5_INDEX = 65, + EMC_FBIO_CFG7_INDEX, + EMC_CFG_DIG_DLL_INDEX, + EMC_ZCAL_INTERVAL_INDEX = 139, + EMC_ZCAL_WAIT_CNT_INDEX, + EMC_MRS_WAIT_CNT_INDEX = 141, + EMC_DLL_CFG_0_INDEX = 144, + EMC_PMACRO_AUTOCAL_CFG_COMMON_INDEX = 146, + EMC_CFG_INDEX = 148, + EMC_DYN_SELF_REF_CONTROL_INDEX = 150, + EMC_PMACRO_CMD_PAD_TX_CTRL_INDEX = 161, + EMC_PMACRO_DATA_PAD_TX_CTRL_INDEX, + EMC_PMACRO_COMMON_PAD_TX_CTRL_INDEX, + EMC_PMACRO_BRICK_CTRL_RFU1_INDEX = 167, + EMC_PMACRO_BG_BIAS_CTRL_0_INDEX = 171, + EMC_MRW14_INDEX = 199, + EMC_MRW15_INDEX = 220, +}; + +enum trim_regs_list { + EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_0_INDEX = 60, + EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_1_INDEX, + EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_2_INDEX, + EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_3_INDEX, + EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_4_INDEX, + EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_5_INDEX, + EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_0_INDEX, + EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_1_INDEX, + EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_2_INDEX, + EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_3_INDEX, +}; + +enum burst_mc_regs_list { + MC_EMEM_ARB_MISC0_INDEX = 20, +}; + +enum { + T_RP, + T_FC_LPDDR4, + T_RFC, + T_PDEX, + RL, +}; + +enum { + AUTO_PD = 0, + MAN_SR = 2, +}; + +enum { + ASSEMBLY = 0, + ACTIVE, +}; + +enum { + C0D0U0, + C0D0U1, + C0D1U0, + C0D1U1, + C1D0U0, + C1D0U1, + C1D1U0, + C1D1U1, + DRAM_CLKTREE_NUM, +}; + +#define VREF_REGS_PER_CHANNEL_SIZE 4 +#define DRAM_TIMINGS_NUM 5 +#define BURST_REGS_PER_CHANNEL_SIZE 8 +#define TRIM_REGS_PER_CHANNEL_SIZE 10 +#define PTFV_ARRAY_SIZE 12 +#define SAVE_RESTORE_MOD_REGS_SIZE 12 +#define TRAINING_MOD_REGS_SIZE 20 +#define BURST_UP_DOWN_REGS_SIZE 24 +#define BURST_MC_REGS_SIZE 33 +#define TRIM_REGS_SIZE 138 +#define BURST_REGS_SIZE 221 + +struct tegra210_emc_per_channel_regs { + u16 bank; + u16 offset; +}; + +struct tegra210_emc_table_register_offsets { + u16 burst[BURST_REGS_SIZE]; + u16 trim[TRIM_REGS_SIZE]; + u16 burst_mc[BURST_MC_REGS_SIZE]; + u16 la_scale[BURST_UP_DOWN_REGS_SIZE]; + struct tegra210_emc_per_channel_regs burst_per_channel[BURST_REGS_PER_CHANNEL_SIZE]; + struct tegra210_emc_per_channel_regs trim_per_channel[TRIM_REGS_PER_CHANNEL_SIZE]; + struct tegra210_emc_per_channel_regs vref_per_channel[VREF_REGS_PER_CHANNEL_SIZE]; +}; + +struct tegra210_emc_timing { + u32 revision; + const char dvfs_ver[60]; + u32 rate; + u32 min_volt; + u32 gpu_min_volt; + const char clock_src[32]; + u32 clk_src_emc; + u32 needs_training; + u32 training_pattern; + u32 trained; + + u32 periodic_training; + u32 trained_dram_clktree[DRAM_CLKTREE_NUM]; + u32 current_dram_clktree[DRAM_CLKTREE_NUM]; + u32 run_clocks; + u32 tree_margin; + + u32 num_burst; + u32 num_burst_per_ch; + u32 num_trim; + u32 num_trim_per_ch; + u32 num_mc_regs; + u32 num_up_down; + u32 vref_num; + u32 training_mod_num; + u32 dram_timing_num; + + u32 ptfv_list[PTFV_ARRAY_SIZE]; + + u32 burst_regs[BURST_REGS_SIZE]; + u32 burst_reg_per_ch[BURST_REGS_PER_CHANNEL_SIZE]; + u32 shadow_regs_ca_train[BURST_REGS_SIZE]; + u32 shadow_regs_quse_train[BURST_REGS_SIZE]; + u32 shadow_regs_rdwr_train[BURST_REGS_SIZE]; + + u32 trim_regs[TRIM_REGS_SIZE]; + u32 trim_perch_regs[TRIM_REGS_PER_CHANNEL_SIZE]; + + u32 vref_perch_regs[VREF_REGS_PER_CHANNEL_SIZE]; + + u32 dram_timings[DRAM_TIMINGS_NUM]; + u32 training_mod_regs[TRAINING_MOD_REGS_SIZE]; + u32 save_restore_mod_regs[SAVE_RESTORE_MOD_REGS_SIZE]; + u32 burst_mc_regs[BURST_MC_REGS_SIZE]; + u32 la_scale_regs[BURST_UP_DOWN_REGS_SIZE]; + + u32 min_mrs_wait; + u32 emc_mrw; + u32 emc_mrw2; + u32 emc_mrw3; + u32 emc_mrw4; + u32 emc_mrw9; + u32 emc_mrs; + u32 emc_emrs; + u32 emc_emrs2; + u32 emc_auto_cal_config; + u32 emc_auto_cal_config2; + u32 emc_auto_cal_config3; + u32 emc_auto_cal_config4; + u32 emc_auto_cal_config5; + u32 emc_auto_cal_config6; + u32 emc_auto_cal_config7; + u32 emc_auto_cal_config8; + u32 emc_cfg_2; + u32 emc_sel_dpd_ctrl; + u32 emc_fdpd_ctrl_cmd_no_ramp; + u32 dll_clk_src; + u32 clk_out_enb_x_0_clk_enb_emc_dll; + u32 latency; +}; + +enum tegra210_emc_refresh { + TEGRA210_EMC_REFRESH_NOMINAL = 0, + TEGRA210_EMC_REFRESH_2X, + TEGRA210_EMC_REFRESH_4X, + TEGRA210_EMC_REFRESH_THROTTLE, /* 4x Refresh + derating. */ +}; + +#define DRAM_TYPE_DDR3 0 +#define DRAM_TYPE_LPDDR4 1 +#define DRAM_TYPE_LPDDR2 2 +#define DRAM_TYPE_DDR2 3 + +struct tegra210_emc { + struct tegra_mc *mc; + struct device *dev; + struct clk *clk; + + /* nominal EMC frequency table */ + struct tegra210_emc_timing *nominal; + /* derated EMC frequency table */ + struct tegra210_emc_timing *derated; + + /* currently selected table (nominal or derated) */ + struct tegra210_emc_timing *timings; + unsigned int num_timings; + + const struct tegra210_emc_table_register_offsets *offsets; + + const struct tegra210_emc_sequence *sequence; + spinlock_t lock; + + void __iomem *regs, *channel[2]; + unsigned int num_channels; + unsigned int num_devices; + unsigned int dram_type; + + struct tegra210_emc_timing *last; + struct tegra210_emc_timing *next; + + unsigned int training_interval; + struct timer_list training; + + enum tegra210_emc_refresh refresh; + unsigned int refresh_poll_interval; + struct timer_list refresh_timer; + unsigned int temperature; + atomic_t refresh_poll; + + ktime_t clkchange_time; + int clkchange_delay; + + unsigned long resume_rate; + + struct { + struct dentry *root; + unsigned long min_rate; + unsigned long max_rate; + unsigned int temperature; + } debugfs; + + struct tegra210_clk_emc_provider provider; +}; + +struct tegra210_emc_sequence { + u8 revision; + void (*set_clock)(struct tegra210_emc *emc, u32 clksrc); + u32 (*periodic_compensation)(struct tegra210_emc *emc); +}; + +static inline void emc_writel(struct tegra210_emc *emc, u32 value, + unsigned int offset) +{ + writel_relaxed(value, emc->regs + offset); +} + +static inline u32 emc_readl(struct tegra210_emc *emc, unsigned int offset) +{ + return readl_relaxed(emc->regs + offset); +} + +static inline void emc_channel_writel(struct tegra210_emc *emc, + unsigned int channel, + u32 value, unsigned int offset) +{ + writel_relaxed(value, emc->channel[channel] + offset); +} + +static inline u32 emc_channel_readl(struct tegra210_emc *emc, + unsigned int channel, unsigned int offset) +{ + return readl_relaxed(emc->channel[channel] + offset); +} + +static inline void ccfifo_writel(struct tegra210_emc *emc, u32 value, + unsigned int offset, u32 delay) +{ + writel_relaxed(value, emc->regs + EMC_CCFIFO_DATA); + + value = EMC_CCFIFO_ADDR_STALL_BY_1 | EMC_CCFIFO_ADDR_STALL(delay) | + EMC_CCFIFO_ADDR_OFFSET(offset); + writel_relaxed(value, emc->regs + EMC_CCFIFO_ADDR); +} + +static inline u32 div_o3(u32 a, u32 b) +{ + u32 result = a / b; + + if ((b * result) < a) + return result + 1; + + return result; +} + +/* from tegra210-emc-r21021.c */ +extern const struct tegra210_emc_sequence tegra210_emc_r21021; + +int tegra210_emc_set_refresh(struct tegra210_emc *emc, + enum tegra210_emc_refresh refresh); +u32 tegra210_emc_mrr_read(struct tegra210_emc *emc, unsigned int chip, + unsigned int address); +void tegra210_emc_do_clock_change(struct tegra210_emc *emc, u32 clksrc); +void tegra210_emc_set_shadow_bypass(struct tegra210_emc *emc, int set); +void tegra210_emc_timing_update(struct tegra210_emc *emc); +u32 tegra210_emc_get_dll_state(struct tegra210_emc_timing *next); +struct tegra210_emc_timing *tegra210_emc_find_timing(struct tegra210_emc *emc, + unsigned long rate); +void tegra210_emc_adjust_timing(struct tegra210_emc *emc, + struct tegra210_emc_timing *timing); +int tegra210_emc_wait_for_update(struct tegra210_emc *emc, unsigned int channel, + unsigned int offset, u32 bit_mask, bool state); +unsigned long tegra210_emc_actual_osc_clocks(u32 in); +u32 tegra210_emc_compensate(struct tegra210_emc_timing *next, u32 offset); +void tegra210_emc_dll_disable(struct tegra210_emc *emc); +void tegra210_emc_dll_enable(struct tegra210_emc *emc); +u32 tegra210_emc_dll_prelock(struct tegra210_emc *emc, u32 clksrc); +u32 tegra210_emc_dvfs_power_ramp_down(struct tegra210_emc *emc, u32 clk, + bool flip_backward); +u32 tegra210_emc_dvfs_power_ramp_up(struct tegra210_emc *emc, u32 clk, + bool flip_backward); +void tegra210_emc_reset_dram_clktree_values(struct tegra210_emc_timing *timing); +void tegra210_emc_start_periodic_compensation(struct tegra210_emc *emc); + +#endif diff --git a/drivers/memory/tegra/tegra210-mc.h b/drivers/memory/tegra/tegra210-mc.h new file mode 100644 index 000000000000..b9b91ceb4730 --- /dev/null +++ b/drivers/memory/tegra/tegra210-mc.h @@ -0,0 +1,50 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2015-2020, NVIDIA CORPORATION. All rights reserved. + */ + +#ifndef TEGRA210_MC_H +#define TEGRA210_MC_H + +#include "mc.h" + +/* register definitions */ +#define MC_LATENCY_ALLOWANCE_AVPC_0 0x2e4 +#define MC_LATENCY_ALLOWANCE_HC_0 0x310 +#define MC_LATENCY_ALLOWANCE_HC_1 0x314 +#define MC_LATENCY_ALLOWANCE_MPCORE_0 0x320 +#define MC_LATENCY_ALLOWANCE_NVENC_0 0x328 +#define MC_LATENCY_ALLOWANCE_PPCS_0 0x344 +#define MC_LATENCY_ALLOWANCE_PPCS_1 0x348 +#define MC_LATENCY_ALLOWANCE_ISP2_0 0x370 +#define MC_LATENCY_ALLOWANCE_ISP2_1 0x374 +#define MC_LATENCY_ALLOWANCE_XUSB_0 0x37c +#define MC_LATENCY_ALLOWANCE_XUSB_1 0x380 +#define MC_LATENCY_ALLOWANCE_TSEC_0 0x390 +#define MC_LATENCY_ALLOWANCE_VIC_0 0x394 +#define MC_LATENCY_ALLOWANCE_VI2_0 0x398 +#define MC_LATENCY_ALLOWANCE_GPU_0 0x3ac +#define MC_LATENCY_ALLOWANCE_SDMMCA_0 0x3b8 +#define MC_LATENCY_ALLOWANCE_SDMMCAA_0 0x3bc +#define MC_LATENCY_ALLOWANCE_SDMMC_0 0x3c0 +#define MC_LATENCY_ALLOWANCE_SDMMCAB_0 0x3c4 +#define MC_LATENCY_ALLOWANCE_GPU2_0 0x3e8 +#define MC_LATENCY_ALLOWANCE_NVDEC_0 0x3d8 +#define MC_MLL_MPCORER_PTSA_RATE 0x44c +#define MC_FTOP_PTSA_RATE 0x50c +#define MC_EMEM_ARB_TIMING_RFCPB 0x6c0 +#define MC_EMEM_ARB_TIMING_CCDMW 0x6c4 +#define MC_EMEM_ARB_REFPB_HP_CTRL 0x6f0 +#define MC_EMEM_ARB_REFPB_BANK_CTRL 0x6f4 +#define MC_PTSA_GRANT_DECREMENT 0x960 +#define MC_EMEM_ARB_DHYST_CTRL 0xbcc +#define MC_EMEM_ARB_DHYST_TIMEOUT_UTIL_0 0xbd0 +#define MC_EMEM_ARB_DHYST_TIMEOUT_UTIL_1 0xbd4 +#define MC_EMEM_ARB_DHYST_TIMEOUT_UTIL_2 0xbd8 +#define MC_EMEM_ARB_DHYST_TIMEOUT_UTIL_3 0xbdc +#define MC_EMEM_ARB_DHYST_TIMEOUT_UTIL_4 0xbe0 +#define MC_EMEM_ARB_DHYST_TIMEOUT_UTIL_5 0xbe4 +#define MC_EMEM_ARB_DHYST_TIMEOUT_UTIL_6 0xbe8 +#define MC_EMEM_ARB_DHYST_TIMEOUT_UTIL_7 0xbec + +#endif diff --git a/drivers/memory/tegra/tegra210.c b/drivers/memory/tegra/tegra210.c index cc0482434c75..8ab6498dbe7d 100644 --- a/drivers/memory/tegra/tegra210.c +++ b/drivers/memory/tegra/tegra210.c @@ -16,1039 +16,1193 @@ static const struct tegra_mc_client tegra210_mc_clients[] = { .id = 0x01, .name = "display0a", .swgroup = TEGRA_SWGROUP_DC, - .smmu = { - .reg = 0x228, - .bit = 1, - }, - .la = { - .reg = 0x2e8, - .shift = 0, - .mask = 0xff, - .def = 0xc2, + .regs = { + .smmu = { + .reg = 0x228, + .bit = 1, + }, + .la = { + .reg = 0x2e8, + .shift = 0, + .mask = 0xff, + .def = 0x1e, + }, }, }, { .id = 0x02, .name = "display0ab", .swgroup = TEGRA_SWGROUP_DCB, - .smmu = { - .reg = 0x228, - .bit = 2, - }, - .la = { - .reg = 0x2f4, - .shift = 0, - .mask = 0xff, - .def = 0xc6, + .regs = { + .smmu = { + .reg = 0x228, + .bit = 2, + }, + .la = { + .reg = 0x2f4, + .shift = 0, + .mask = 0xff, + .def = 0x1e, + }, }, }, { .id = 0x03, .name = "display0b", .swgroup = TEGRA_SWGROUP_DC, - .smmu = { - .reg = 0x228, - .bit = 3, - }, - .la = { - .reg = 0x2e8, - .shift = 16, - .mask = 0xff, - .def = 0x50, + .regs = { + .smmu = { + .reg = 0x228, + .bit = 3, + }, + .la = { + .reg = 0x2e8, + .shift = 16, + .mask = 0xff, + .def = 0x1e, + }, }, }, { .id = 0x04, .name = "display0bb", .swgroup = TEGRA_SWGROUP_DCB, - .smmu = { - .reg = 0x228, - .bit = 4, - }, - .la = { - .reg = 0x2f4, - .shift = 16, - .mask = 0xff, - .def = 0x50, + .regs = { + .smmu = { + .reg = 0x228, + .bit = 4, + }, + .la = { + .reg = 0x2f4, + .shift = 16, + .mask = 0xff, + .def = 0x1e, + }, }, }, { .id = 0x05, .name = "display0c", .swgroup = TEGRA_SWGROUP_DC, - .smmu = { - .reg = 0x228, - .bit = 5, - }, - .la = { - .reg = 0x2ec, - .shift = 0, - .mask = 0xff, - .def = 0x50, + .regs = { + .smmu = { + .reg = 0x228, + .bit = 5, + }, + .la = { + .reg = 0x2ec, + .shift = 0, + .mask = 0xff, + .def = 0x1e, + }, }, }, { .id = 0x06, .name = "display0cb", .swgroup = TEGRA_SWGROUP_DCB, - .smmu = { - .reg = 0x228, - .bit = 6, - }, - .la = { - .reg = 0x2f8, - .shift = 0, - .mask = 0xff, - .def = 0x50, + .regs = { + .smmu = { + .reg = 0x228, + .bit = 6, + }, + .la = { + .reg = 0x2f8, + .shift = 0, + .mask = 0xff, + .def = 0x1e, + }, }, }, { .id = 0x0e, .name = "afir", .swgroup = TEGRA_SWGROUP_AFI, - .smmu = { - .reg = 0x228, - .bit = 14, - }, - .la = { - .reg = 0x2e0, - .shift = 0, - .mask = 0xff, - .def = 0x13, + .regs = { + .smmu = { + .reg = 0x228, + .bit = 14, + }, + .la = { + .reg = 0x2e0, + .shift = 0, + .mask = 0xff, + .def = 0x2e, + }, }, }, { .id = 0x0f, .name = "avpcarm7r", .swgroup = TEGRA_SWGROUP_AVPC, - .smmu = { - .reg = 0x228, - .bit = 15, - }, - .la = { - .reg = 0x2e4, - .shift = 0, - .mask = 0xff, - .def = 0x04, + .regs = { + .smmu = { + .reg = 0x228, + .bit = 15, + }, + .la = { + .reg = 0x2e4, + .shift = 0, + .mask = 0xff, + .def = 0x04, + }, }, }, { .id = 0x10, .name = "displayhc", .swgroup = TEGRA_SWGROUP_DC, - .smmu = { - .reg = 0x228, - .bit = 16, - }, - .la = { - .reg = 0x2f0, - .shift = 0, - .mask = 0xff, - .def = 0x50, + .regs = { + .smmu = { + .reg = 0x228, + .bit = 16, + }, + .la = { + .reg = 0x2f0, + .shift = 0, + .mask = 0xff, + .def = 0x1e, + }, }, }, { .id = 0x11, .name = "displayhcb", .swgroup = TEGRA_SWGROUP_DCB, - .smmu = { - .reg = 0x228, - .bit = 17, - }, - .la = { - .reg = 0x2fc, - .shift = 0, - .mask = 0xff, - .def = 0x50, + .regs = { + .smmu = { + .reg = 0x228, + .bit = 17, + }, + .la = { + .reg = 0x2fc, + .shift = 0, + .mask = 0xff, + .def = 0x1e, + }, }, }, { .id = 0x15, .name = "hdar", .swgroup = TEGRA_SWGROUP_HDA, - .smmu = { - .reg = 0x228, - .bit = 21, - }, - .la = { - .reg = 0x318, - .shift = 0, - .mask = 0xff, - .def = 0x24, + .regs = { + .smmu = { + .reg = 0x228, + .bit = 21, + }, + .la = { + .reg = 0x318, + .shift = 0, + .mask = 0xff, + .def = 0x24, + }, }, }, { .id = 0x16, .name = "host1xdmar", .swgroup = TEGRA_SWGROUP_HC, - .smmu = { - .reg = 0x228, - .bit = 22, - }, - .la = { - .reg = 0x310, - .shift = 0, - .mask = 0xff, - .def = 0x1e, + .regs = { + .smmu = { + .reg = 0x228, + .bit = 22, + }, + .la = { + .reg = 0x310, + .shift = 0, + .mask = 0xff, + .def = 0x1e, + }, }, }, { .id = 0x17, .name = "host1xr", .swgroup = TEGRA_SWGROUP_HC, - .smmu = { - .reg = 0x228, - .bit = 23, - }, - .la = { - .reg = 0x310, - .shift = 16, - .mask = 0xff, - .def = 0x50, + .regs = { + .smmu = { + .reg = 0x228, + .bit = 23, + }, + .la = { + .reg = 0x310, + .shift = 16, + .mask = 0xff, + .def = 0x50, + }, }, }, { .id = 0x1c, .name = "nvencsrd", .swgroup = TEGRA_SWGROUP_NVENC, - .smmu = { - .reg = 0x228, - .bit = 28, - }, - .la = { - .reg = 0x328, - .shift = 0, - .mask = 0xff, - .def = 0x23, + .regs = { + .smmu = { + .reg = 0x228, + .bit = 28, + }, + .la = { + .reg = 0x328, + .shift = 0, + .mask = 0xff, + .def = 0x23, + }, }, }, { .id = 0x1d, .name = "ppcsahbdmar", .swgroup = TEGRA_SWGROUP_PPCS, - .smmu = { - .reg = 0x228, - .bit = 29, - }, - .la = { - .reg = 0x344, - .shift = 0, - .mask = 0xff, - .def = 0x49, + .regs = { + .smmu = { + .reg = 0x228, + .bit = 29, + }, + .la = { + .reg = 0x344, + .shift = 0, + .mask = 0xff, + .def = 0x49, + }, }, }, { .id = 0x1e, .name = "ppcsahbslvr", .swgroup = TEGRA_SWGROUP_PPCS, - .smmu = { - .reg = 0x228, - .bit = 30, - }, - .la = { - .reg = 0x344, - .shift = 16, - .mask = 0xff, - .def = 0x1a, + .regs = { + .smmu = { + .reg = 0x228, + .bit = 30, + }, + .la = { + .reg = 0x344, + .shift = 16, + .mask = 0xff, + .def = 0x1a, + }, }, }, { .id = 0x1f, .name = "satar", .swgroup = TEGRA_SWGROUP_SATA, - .smmu = { - .reg = 0x228, - .bit = 31, - }, - .la = { - .reg = 0x350, - .shift = 0, - .mask = 0xff, - .def = 0x65, + .regs = { + .smmu = { + .reg = 0x228, + .bit = 31, + }, + .la = { + .reg = 0x350, + .shift = 0, + .mask = 0xff, + .def = 0x65, + }, }, }, { .id = 0x27, .name = "mpcorer", .swgroup = TEGRA_SWGROUP_MPCORE, - .la = { - .reg = 0x320, - .shift = 0, - .mask = 0xff, - .def = 0x04, + .regs = { + .la = { + .reg = 0x320, + .shift = 0, + .mask = 0xff, + .def = 0x04, + }, }, }, { .id = 0x2b, .name = "nvencswr", .swgroup = TEGRA_SWGROUP_NVENC, - .smmu = { - .reg = 0x22c, - .bit = 11, - }, - .la = { - .reg = 0x328, - .shift = 16, - .mask = 0xff, - .def = 0x80, + .regs = { + .smmu = { + .reg = 0x22c, + .bit = 11, + }, + .la = { + .reg = 0x328, + .shift = 16, + .mask = 0xff, + .def = 0x80, + }, }, }, { .id = 0x31, .name = "afiw", .swgroup = TEGRA_SWGROUP_AFI, - .smmu = { - .reg = 0x22c, - .bit = 17, - }, - .la = { - .reg = 0x2e0, - .shift = 16, - .mask = 0xff, - .def = 0x80, + .regs = { + .smmu = { + .reg = 0x22c, + .bit = 17, + }, + .la = { + .reg = 0x2e0, + .shift = 16, + .mask = 0xff, + .def = 0x80, + }, }, }, { .id = 0x32, .name = "avpcarm7w", .swgroup = TEGRA_SWGROUP_AVPC, - .smmu = { - .reg = 0x22c, - .bit = 18, - }, - .la = { - .reg = 0x2e4, - .shift = 16, - .mask = 0xff, - .def = 0x80, + .regs = { + .smmu = { + .reg = 0x22c, + .bit = 18, + }, + .la = { + .reg = 0x2e4, + .shift = 16, + .mask = 0xff, + .def = 0x80, + }, }, }, { .id = 0x35, .name = "hdaw", .swgroup = TEGRA_SWGROUP_HDA, - .smmu = { - .reg = 0x22c, - .bit = 21, - }, - .la = { - .reg = 0x318, - .shift = 16, - .mask = 0xff, - .def = 0x80, + .regs = { + .smmu = { + .reg = 0x22c, + .bit = 21, + }, + .la = { + .reg = 0x318, + .shift = 16, + .mask = 0xff, + .def = 0x80, + }, }, }, { .id = 0x36, .name = "host1xw", .swgroup = TEGRA_SWGROUP_HC, - .smmu = { - .reg = 0x22c, - .bit = 22, - }, - .la = { - .reg = 0x314, - .shift = 0, - .mask = 0xff, - .def = 0x80, + .regs = { + .smmu = { + .reg = 0x22c, + .bit = 22, + }, + .la = { + .reg = 0x314, + .shift = 0, + .mask = 0xff, + .def = 0x80, + }, }, }, { .id = 0x39, .name = "mpcorew", .swgroup = TEGRA_SWGROUP_MPCORE, - .la = { - .reg = 0x320, - .shift = 16, - .mask = 0xff, - .def = 0x80, + .regs = { + .la = { + .reg = 0x320, + .shift = 16, + .mask = 0xff, + .def = 0x80, + }, }, }, { .id = 0x3b, .name = "ppcsahbdmaw", .swgroup = TEGRA_SWGROUP_PPCS, - .smmu = { - .reg = 0x22c, - .bit = 27, - }, - .la = { - .reg = 0x348, - .shift = 0, - .mask = 0xff, - .def = 0x80, + .regs = { + .smmu = { + .reg = 0x22c, + .bit = 27, + }, + .la = { + .reg = 0x348, + .shift = 0, + .mask = 0xff, + .def = 0x80, + }, }, }, { .id = 0x3c, .name = "ppcsahbslvw", .swgroup = TEGRA_SWGROUP_PPCS, - .smmu = { - .reg = 0x22c, - .bit = 28, - }, - .la = { - .reg = 0x348, - .shift = 16, - .mask = 0xff, - .def = 0x80, + .regs = { + .smmu = { + .reg = 0x22c, + .bit = 28, + }, + .la = { + .reg = 0x348, + .shift = 16, + .mask = 0xff, + .def = 0x80, + }, }, }, { .id = 0x3d, .name = "sataw", .swgroup = TEGRA_SWGROUP_SATA, - .smmu = { - .reg = 0x22c, - .bit = 29, - }, - .la = { - .reg = 0x350, - .shift = 16, - .mask = 0xff, - .def = 0x65, + .regs = { + .smmu = { + .reg = 0x22c, + .bit = 29, + }, + .la = { + .reg = 0x350, + .shift = 16, + .mask = 0xff, + .def = 0x80, + }, }, }, { .id = 0x44, .name = "ispra", .swgroup = TEGRA_SWGROUP_ISP2, - .smmu = { - .reg = 0x230, - .bit = 4, - }, - .la = { - .reg = 0x370, - .shift = 0, - .mask = 0xff, - .def = 0x18, + .regs = { + .smmu = { + .reg = 0x230, + .bit = 4, + }, + .la = { + .reg = 0x370, + .shift = 0, + .mask = 0xff, + .def = 0x18, + }, }, }, { .id = 0x46, .name = "ispwa", .swgroup = TEGRA_SWGROUP_ISP2, - .smmu = { - .reg = 0x230, - .bit = 6, - }, - .la = { - .reg = 0x374, - .shift = 0, - .mask = 0xff, - .def = 0x80, + .regs = { + .smmu = { + .reg = 0x230, + .bit = 6, + }, + .la = { + .reg = 0x374, + .shift = 0, + .mask = 0xff, + .def = 0x80, + }, }, }, { .id = 0x47, .name = "ispwb", .swgroup = TEGRA_SWGROUP_ISP2, - .smmu = { - .reg = 0x230, - .bit = 7, - }, - .la = { - .reg = 0x374, - .shift = 16, - .mask = 0xff, - .def = 0x80, + .regs = { + .smmu = { + .reg = 0x230, + .bit = 7, + }, + .la = { + .reg = 0x374, + .shift = 16, + .mask = 0xff, + .def = 0x80, + }, }, }, { .id = 0x4a, .name = "xusb_hostr", .swgroup = TEGRA_SWGROUP_XUSB_HOST, - .smmu = { - .reg = 0x230, - .bit = 10, - }, - .la = { - .reg = 0x37c, - .shift = 0, - .mask = 0xff, - .def = 0x7a, + .regs = { + .smmu = { + .reg = 0x230, + .bit = 10, + }, + .la = { + .reg = 0x37c, + .shift = 0, + .mask = 0xff, + .def = 0x7a, + }, }, }, { .id = 0x4b, .name = "xusb_hostw", .swgroup = TEGRA_SWGROUP_XUSB_HOST, - .smmu = { - .reg = 0x230, - .bit = 11, - }, - .la = { - .reg = 0x37c, - .shift = 16, - .mask = 0xff, - .def = 0x80, + .regs = { + .smmu = { + .reg = 0x230, + .bit = 11, + }, + .la = { + .reg = 0x37c, + .shift = 16, + .mask = 0xff, + .def = 0x80, + }, }, }, { .id = 0x4c, .name = "xusb_devr", .swgroup = TEGRA_SWGROUP_XUSB_DEV, - .smmu = { - .reg = 0x230, - .bit = 12, - }, - .la = { - .reg = 0x380, - .shift = 0, - .mask = 0xff, - .def = 0x39, + .regs = { + .smmu = { + .reg = 0x230, + .bit = 12, + }, + .la = { + .reg = 0x380, + .shift = 0, + .mask = 0xff, + .def = 0x39, + }, }, }, { .id = 0x4d, .name = "xusb_devw", .swgroup = TEGRA_SWGROUP_XUSB_DEV, - .smmu = { - .reg = 0x230, - .bit = 13, - }, - .la = { - .reg = 0x380, - .shift = 16, - .mask = 0xff, - .def = 0x80, + .regs = { + .smmu = { + .reg = 0x230, + .bit = 13, + }, + .la = { + .reg = 0x380, + .shift = 16, + .mask = 0xff, + .def = 0x80, + }, }, }, { .id = 0x4e, .name = "isprab", .swgroup = TEGRA_SWGROUP_ISP2B, - .smmu = { - .reg = 0x230, - .bit = 14, - }, - .la = { - .reg = 0x384, - .shift = 0, - .mask = 0xff, - .def = 0x18, + .regs = { + .smmu = { + .reg = 0x230, + .bit = 14, + }, + .la = { + .reg = 0x384, + .shift = 0, + .mask = 0xff, + .def = 0x18, + }, }, }, { .id = 0x50, .name = "ispwab", .swgroup = TEGRA_SWGROUP_ISP2B, - .smmu = { - .reg = 0x230, - .bit = 16, - }, - .la = { - .reg = 0x388, - .shift = 0, - .mask = 0xff, - .def = 0x80, + .regs = { + .smmu = { + .reg = 0x230, + .bit = 16, + }, + .la = { + .reg = 0x388, + .shift = 0, + .mask = 0xff, + .def = 0x80, + }, }, }, { .id = 0x51, .name = "ispwbb", .swgroup = TEGRA_SWGROUP_ISP2B, - .smmu = { - .reg = 0x230, - .bit = 17, - }, - .la = { - .reg = 0x388, - .shift = 16, - .mask = 0xff, - .def = 0x80, + .regs = { + .smmu = { + .reg = 0x230, + .bit = 17, + }, + .la = { + .reg = 0x388, + .shift = 16, + .mask = 0xff, + .def = 0x80, + }, }, }, { .id = 0x54, .name = "tsecsrd", .swgroup = TEGRA_SWGROUP_TSEC, - .smmu = { - .reg = 0x230, - .bit = 20, - }, - .la = { - .reg = 0x390, - .shift = 0, - .mask = 0xff, - .def = 0x9b, + .regs = { + .smmu = { + .reg = 0x230, + .bit = 20, + }, + .la = { + .reg = 0x390, + .shift = 0, + .mask = 0xff, + .def = 0x9b, + }, }, }, { .id = 0x55, .name = "tsecswr", .swgroup = TEGRA_SWGROUP_TSEC, - .smmu = { - .reg = 0x230, - .bit = 21, - }, - .la = { - .reg = 0x390, - .shift = 16, - .mask = 0xff, - .def = 0x80, + .regs = { + .smmu = { + .reg = 0x230, + .bit = 21, + }, + .la = { + .reg = 0x390, + .shift = 16, + .mask = 0xff, + .def = 0x80, + }, }, }, { .id = 0x56, .name = "a9avpscr", .swgroup = TEGRA_SWGROUP_A9AVP, - .smmu = { - .reg = 0x230, - .bit = 22, - }, - .la = { - .reg = 0x3a4, - .shift = 0, - .mask = 0xff, - .def = 0x04, + .regs = { + .smmu = { + .reg = 0x230, + .bit = 22, + }, + .la = { + .reg = 0x3a4, + .shift = 0, + .mask = 0xff, + .def = 0x04, + }, }, }, { .id = 0x57, .name = "a9avpscw", .swgroup = TEGRA_SWGROUP_A9AVP, - .smmu = { - .reg = 0x230, - .bit = 23, - }, - .la = { - .reg = 0x3a4, - .shift = 16, - .mask = 0xff, - .def = 0x80, + .regs = { + .smmu = { + .reg = 0x230, + .bit = 23, + }, + .la = { + .reg = 0x3a4, + .shift = 16, + .mask = 0xff, + .def = 0x80, + }, }, }, { .id = 0x58, .name = "gpusrd", .swgroup = TEGRA_SWGROUP_GPU, - .smmu = { - /* read-only */ - .reg = 0x230, - .bit = 24, - }, - .la = { - .reg = 0x3c8, - .shift = 0, - .mask = 0xff, - .def = 0x1a, + .regs = { + .smmu = { + /* read-only */ + .reg = 0x230, + .bit = 24, + }, + .la = { + .reg = 0x3c8, + .shift = 0, + .mask = 0xff, + .def = 0x1a, + }, }, }, { .id = 0x59, .name = "gpuswr", .swgroup = TEGRA_SWGROUP_GPU, - .smmu = { - /* read-only */ - .reg = 0x230, - .bit = 25, - }, - .la = { - .reg = 0x3c8, - .shift = 16, - .mask = 0xff, - .def = 0x80, + .regs = { + .smmu = { + /* read-only */ + .reg = 0x230, + .bit = 25, + }, + .la = { + .reg = 0x3c8, + .shift = 16, + .mask = 0xff, + .def = 0x80, + }, }, }, { .id = 0x5a, .name = "displayt", .swgroup = TEGRA_SWGROUP_DC, - .smmu = { - .reg = 0x230, - .bit = 26, - }, - .la = { - .reg = 0x2f0, - .shift = 16, - .mask = 0xff, - .def = 0x50, + .regs = { + .smmu = { + .reg = 0x230, + .bit = 26, + }, + .la = { + .reg = 0x2f0, + .shift = 16, + .mask = 0xff, + .def = 0x1e, + }, }, }, { .id = 0x60, .name = "sdmmcra", .swgroup = TEGRA_SWGROUP_SDMMC1A, - .smmu = { - .reg = 0x234, - .bit = 0, - }, - .la = { - .reg = 0x3b8, - .shift = 0, - .mask = 0xff, - .def = 0x49, + .regs = { + .smmu = { + .reg = 0x234, + .bit = 0, + }, + .la = { + .reg = 0x3b8, + .shift = 0, + .mask = 0xff, + .def = 0x49, + }, }, }, { .id = 0x61, .name = "sdmmcraa", .swgroup = TEGRA_SWGROUP_SDMMC2A, - .smmu = { - .reg = 0x234, - .bit = 1, - }, - .la = { - .reg = 0x3bc, - .shift = 0, - .mask = 0xff, - .def = 0x49, + .regs = { + .smmu = { + .reg = 0x234, + .bit = 1, + }, + .la = { + .reg = 0x3bc, + .shift = 0, + .mask = 0xff, + .def = 0x5a, + }, }, }, { .id = 0x62, .name = "sdmmcr", .swgroup = TEGRA_SWGROUP_SDMMC3A, - .smmu = { - .reg = 0x234, - .bit = 2, - }, - .la = { - .reg = 0x3c0, - .shift = 0, - .mask = 0xff, - .def = 0x49, + .regs = { + .smmu = { + .reg = 0x234, + .bit = 2, + }, + .la = { + .reg = 0x3c0, + .shift = 0, + .mask = 0xff, + .def = 0x49, + }, }, }, { .id = 0x63, .swgroup = TEGRA_SWGROUP_SDMMC4A, .name = "sdmmcrab", - .smmu = { - .reg = 0x234, - .bit = 3, - }, - .la = { - .reg = 0x3c4, - .shift = 0, - .mask = 0xff, - .def = 0x49, + .regs = { + .smmu = { + .reg = 0x234, + .bit = 3, + }, + .la = { + .reg = 0x3c4, + .shift = 0, + .mask = 0xff, + .def = 0x5a, + }, }, }, { .id = 0x64, .name = "sdmmcwa", .swgroup = TEGRA_SWGROUP_SDMMC1A, - .smmu = { - .reg = 0x234, - .bit = 4, - }, - .la = { - .reg = 0x3b8, - .shift = 16, - .mask = 0xff, - .def = 0x80, + .regs = { + .smmu = { + .reg = 0x234, + .bit = 4, + }, + .la = { + .reg = 0x3b8, + .shift = 16, + .mask = 0xff, + .def = 0x80, + }, }, }, { .id = 0x65, .name = "sdmmcwaa", .swgroup = TEGRA_SWGROUP_SDMMC2A, - .smmu = { - .reg = 0x234, - .bit = 5, - }, - .la = { - .reg = 0x3bc, - .shift = 16, - .mask = 0xff, - .def = 0x80, + .regs = { + .smmu = { + .reg = 0x234, + .bit = 5, + }, + .la = { + .reg = 0x3bc, + .shift = 16, + .mask = 0xff, + .def = 0x80, + }, }, }, { .id = 0x66, .name = "sdmmcw", .swgroup = TEGRA_SWGROUP_SDMMC3A, - .smmu = { - .reg = 0x234, - .bit = 6, - }, - .la = { - .reg = 0x3c0, - .shift = 16, - .mask = 0xff, - .def = 0x80, + .regs = { + .smmu = { + .reg = 0x234, + .bit = 6, + }, + .la = { + .reg = 0x3c0, + .shift = 16, + .mask = 0xff, + .def = 0x80, + }, }, }, { .id = 0x67, .name = "sdmmcwab", .swgroup = TEGRA_SWGROUP_SDMMC4A, - .smmu = { - .reg = 0x234, - .bit = 7, - }, - .la = { - .reg = 0x3c4, - .shift = 16, - .mask = 0xff, - .def = 0x80, + .regs = { + .smmu = { + .reg = 0x234, + .bit = 7, + }, + .la = { + .reg = 0x3c4, + .shift = 16, + .mask = 0xff, + .def = 0x80, + }, }, }, { .id = 0x6c, .name = "vicsrd", .swgroup = TEGRA_SWGROUP_VIC, - .smmu = { - .reg = 0x234, - .bit = 12, - }, - .la = { - .reg = 0x394, - .shift = 0, - .mask = 0xff, - .def = 0x1a, + .regs = { + .smmu = { + .reg = 0x234, + .bit = 12, + }, + .la = { + .reg = 0x394, + .shift = 0, + .mask = 0xff, + .def = 0x1a, + }, }, }, { .id = 0x6d, .name = "vicswr", .swgroup = TEGRA_SWGROUP_VIC, - .smmu = { - .reg = 0x234, - .bit = 13, - }, - .la = { - .reg = 0x394, - .shift = 16, - .mask = 0xff, - .def = 0x80, + .regs = { + .smmu = { + .reg = 0x234, + .bit = 13, + }, + .la = { + .reg = 0x394, + .shift = 16, + .mask = 0xff, + .def = 0x80, + }, }, }, { .id = 0x72, .name = "viw", .swgroup = TEGRA_SWGROUP_VI, - .smmu = { - .reg = 0x234, - .bit = 18, - }, - .la = { - .reg = 0x398, - .shift = 0, - .mask = 0xff, - .def = 0x80, + .regs = { + .smmu = { + .reg = 0x234, + .bit = 18, + }, + .la = { + .reg = 0x398, + .shift = 0, + .mask = 0xff, + .def = 0x80, + }, }, }, { .id = 0x73, .name = "displayd", .swgroup = TEGRA_SWGROUP_DC, - .smmu = { - .reg = 0x234, - .bit = 19, - }, - .la = { - .reg = 0x3c8, - .shift = 0, - .mask = 0xff, - .def = 0x50, + .regs = { + .smmu = { + .reg = 0x234, + .bit = 19, + }, + .la = { + .reg = 0x3c8, + .shift = 0, + .mask = 0xff, + .def = 0x50, + }, }, }, { .id = 0x78, .name = "nvdecsrd", .swgroup = TEGRA_SWGROUP_NVDEC, - .smmu = { - .reg = 0x234, - .bit = 24, - }, - .la = { - .reg = 0x3d8, - .shift = 0, - .mask = 0xff, - .def = 0x23, + .regs = { + .smmu = { + .reg = 0x234, + .bit = 24, + }, + .la = { + .reg = 0x3d8, + .shift = 0, + .mask = 0xff, + .def = 0x23, + }, }, }, { .id = 0x79, .name = "nvdecswr", .swgroup = TEGRA_SWGROUP_NVDEC, - .smmu = { - .reg = 0x234, - .bit = 25, - }, - .la = { - .reg = 0x3d8, - .shift = 16, - .mask = 0xff, - .def = 0x80, + .regs = { + .smmu = { + .reg = 0x234, + .bit = 25, + }, + .la = { + .reg = 0x3d8, + .shift = 16, + .mask = 0xff, + .def = 0x80, + }, }, }, { .id = 0x7a, .name = "aper", .swgroup = TEGRA_SWGROUP_APE, - .smmu = { - .reg = 0x234, - .bit = 26, - }, - .la = { - .reg = 0x3dc, - .shift = 0, - .mask = 0xff, - .def = 0xff, + .regs = { + .smmu = { + .reg = 0x234, + .bit = 26, + }, + .la = { + .reg = 0x3dc, + .shift = 0, + .mask = 0xff, + .def = 0xff, + }, }, }, { .id = 0x7b, .name = "apew", .swgroup = TEGRA_SWGROUP_APE, - .smmu = { - .reg = 0x234, - .bit = 27, - }, - .la = { - .reg = 0x3dc, - .shift = 0, - .mask = 0xff, - .def = 0x80, + .regs = { + .smmu = { + .reg = 0x234, + .bit = 27, + }, + .la = { + .reg = 0x3dc, + .shift = 16, + .mask = 0xff, + .def = 0x80, + }, }, }, { .id = 0x7e, .name = "nvjpgsrd", .swgroup = TEGRA_SWGROUP_NVJPG, - .smmu = { - .reg = 0x234, - .bit = 30, - }, - .la = { - .reg = 0x3e4, - .shift = 0, - .mask = 0xff, - .def = 0x23, + .regs = { + .smmu = { + .reg = 0x234, + .bit = 30, + }, + .la = { + .reg = 0x3e4, + .shift = 0, + .mask = 0xff, + .def = 0x23, + }, }, }, { .id = 0x7f, .name = "nvjpgswr", .swgroup = TEGRA_SWGROUP_NVJPG, - .smmu = { - .reg = 0x234, - .bit = 31, - }, - .la = { - .reg = 0x3e4, - .shift = 16, - .mask = 0xff, - .def = 0x80, + .regs = { + .smmu = { + .reg = 0x234, + .bit = 31, + }, + .la = { + .reg = 0x3e4, + .shift = 16, + .mask = 0xff, + .def = 0x80, + }, }, }, { .id = 0x80, .name = "sesrd", .swgroup = TEGRA_SWGROUP_SE, - .smmu = { - .reg = 0xb98, - .bit = 0, - }, - .la = { - .reg = 0x3e0, - .shift = 0, - .mask = 0xff, - .def = 0x2e, + .regs = { + .smmu = { + .reg = 0xb98, + .bit = 0, + }, + .la = { + .reg = 0x3e0, + .shift = 0, + .mask = 0xff, + .def = 0x2e, + }, }, }, { .id = 0x81, .name = "seswr", .swgroup = TEGRA_SWGROUP_SE, - .smmu = { - .reg = 0xb98, - .bit = 1, - }, - .la = { - .reg = 0xb98, - .shift = 16, - .mask = 0xff, - .def = 0x80, + .regs = { + .smmu = { + .reg = 0xb98, + .bit = 1, + }, + .la = { + .reg = 0x3e0, + .shift = 16, + .mask = 0xff, + .def = 0x80, + }, }, }, { .id = 0x82, .name = "axiapr", .swgroup = TEGRA_SWGROUP_AXIAP, - .smmu = { - .reg = 0xb98, - .bit = 2, - }, - .la = { - .reg = 0x3a0, - .shift = 0, - .mask = 0xff, - .def = 0xff, + .regs = { + .smmu = { + .reg = 0xb98, + .bit = 2, + }, + .la = { + .reg = 0x3a0, + .shift = 0, + .mask = 0xff, + .def = 0xff, + }, }, }, { .id = 0x83, .name = "axiapw", .swgroup = TEGRA_SWGROUP_AXIAP, - .smmu = { - .reg = 0xb98, - .bit = 3, - }, - .la = { - .reg = 0x3a0, - .shift = 16, - .mask = 0xff, - .def = 0x80, + .regs = { + .smmu = { + .reg = 0xb98, + .bit = 3, + }, + .la = { + .reg = 0x3a0, + .shift = 16, + .mask = 0xff, + .def = 0x80, + }, }, }, { .id = 0x84, .name = "etrr", .swgroup = TEGRA_SWGROUP_ETR, - .smmu = { - .reg = 0xb98, - .bit = 4, - }, - .la = { - .reg = 0x3ec, - .shift = 0, - .mask = 0xff, - .def = 0xff, + .regs = { + .smmu = { + .reg = 0xb98, + .bit = 4, + }, + .la = { + .reg = 0x3ec, + .shift = 0, + .mask = 0xff, + .def = 0xff, + }, }, }, { .id = 0x85, .name = "etrw", .swgroup = TEGRA_SWGROUP_ETR, - .smmu = { - .reg = 0xb98, - .bit = 5, - }, - .la = { - .reg = 0x3ec, - .shift = 16, - .mask = 0xff, - .def = 0xff, + .regs = { + .smmu = { + .reg = 0xb98, + .bit = 5, + }, + .la = { + .reg = 0x3ec, + .shift = 16, + .mask = 0xff, + .def = 0x80, + }, }, }, { .id = 0x86, .name = "tsecsrdb", .swgroup = TEGRA_SWGROUP_TSECB, - .smmu = { - .reg = 0xb98, - .bit = 6, - }, - .la = { - .reg = 0x3f0, - .shift = 0, - .mask = 0xff, - .def = 0x9b, + .regs = { + .smmu = { + .reg = 0xb98, + .bit = 6, + }, + .la = { + .reg = 0x3f0, + .shift = 0, + .mask = 0xff, + .def = 0x9b, + }, }, }, { .id = 0x87, .name = "tsecswrb", .swgroup = TEGRA_SWGROUP_TSECB, - .smmu = { - .reg = 0xb98, - .bit = 7, - }, - .la = { - .reg = 0x3f0, - .shift = 16, - .mask = 0xff, - .def = 0x80, + .regs = { + .smmu = { + .reg = 0xb98, + .bit = 7, + }, + .la = { + .reg = 0x3f0, + .shift = 16, + .mask = 0xff, + .def = 0x80, + }, }, }, { .id = 0x88, .name = "gpusrd2", .swgroup = TEGRA_SWGROUP_GPU, - .smmu = { - /* read-only */ - .reg = 0xb98, - .bit = 8, - }, - .la = { - .reg = 0x3e8, - .shift = 0, - .mask = 0xff, - .def = 0x1a, + .regs = { + .smmu = { + /* read-only */ + .reg = 0xb98, + .bit = 8, + }, + .la = { + .reg = 0x3e8, + .shift = 0, + .mask = 0xff, + .def = 0x1a, + }, }, }, { .id = 0x89, .name = "gpuswr2", .swgroup = TEGRA_SWGROUP_GPU, - .smmu = { - /* read-only */ - .reg = 0xb98, - .bit = 9, - }, - .la = { - .reg = 0x3e8, - .shift = 16, - .mask = 0xff, - .def = 0x80, + .regs = { + .smmu = { + /* read-only */ + .reg = 0xb98, + .bit = 9, + }, + .la = { + .reg = 0x3e8, + .shift = 16, + .mask = 0xff, + .def = 0x80, + }, }, }, }; static const struct tegra_smmu_swgroup tegra210_swgroups[] = { - { .name = "dc", .swgroup = TEGRA_SWGROUP_DC, .reg = 0x240 }, - { .name = "dcb", .swgroup = TEGRA_SWGROUP_DCB, .reg = 0x244 }, { .name = "afi", .swgroup = TEGRA_SWGROUP_AFI, .reg = 0x238 }, { .name = "avpc", .swgroup = TEGRA_SWGROUP_AVPC, .reg = 0x23c }, - { .name = "hda", .swgroup = TEGRA_SWGROUP_HDA, .reg = 0x254 }, + { .name = "dc", .swgroup = TEGRA_SWGROUP_DC, .reg = 0x240 }, + { .name = "dcb", .swgroup = TEGRA_SWGROUP_DCB, .reg = 0x244 }, { .name = "hc", .swgroup = TEGRA_SWGROUP_HC, .reg = 0x250 }, + { .name = "hda", .swgroup = TEGRA_SWGROUP_HDA, .reg = 0x254 }, + { .name = "isp2", .swgroup = TEGRA_SWGROUP_ISP2, .reg = 0x258 }, { .name = "nvenc", .swgroup = TEGRA_SWGROUP_NVENC, .reg = 0x264 }, + { .name = "nv", .swgroup = TEGRA_SWGROUP_NV, .reg = 0x268 }, + { .name = "nv2", .swgroup = TEGRA_SWGROUP_NV2, .reg = 0x26c }, { .name = "ppcs", .swgroup = TEGRA_SWGROUP_PPCS, .reg = 0x270 }, { .name = "sata", .swgroup = TEGRA_SWGROUP_SATA, .reg = 0x274 }, - { .name = "isp2", .swgroup = TEGRA_SWGROUP_ISP2, .reg = 0x258 }, + { .name = "vi", .swgroup = TEGRA_SWGROUP_VI, .reg = 0x280 }, + { .name = "vic", .swgroup = TEGRA_SWGROUP_VIC, .reg = 0x284 }, { .name = "xusb_host", .swgroup = TEGRA_SWGROUP_XUSB_HOST, .reg = 0x288 }, { .name = "xusb_dev", .swgroup = TEGRA_SWGROUP_XUSB_DEV, .reg = 0x28c }, - { .name = "isp2b", .swgroup = TEGRA_SWGROUP_ISP2B, .reg = 0xaa4 }, - { .name = "tsec", .swgroup = TEGRA_SWGROUP_TSEC, .reg = 0x294 }, { .name = "a9avp", .swgroup = TEGRA_SWGROUP_A9AVP, .reg = 0x290 }, - { .name = "gpu", .swgroup = TEGRA_SWGROUP_GPU, .reg = 0xaac }, + { .name = "tsec", .swgroup = TEGRA_SWGROUP_TSEC, .reg = 0x294 }, + { .name = "ppcs1", .swgroup = TEGRA_SWGROUP_PPCS1, .reg = 0x298 }, + { .name = "dc1", .swgroup = TEGRA_SWGROUP_DC1, .reg = 0xa88 }, { .name = "sdmmc1a", .swgroup = TEGRA_SWGROUP_SDMMC1A, .reg = 0xa94 }, { .name = "sdmmc2a", .swgroup = TEGRA_SWGROUP_SDMMC2A, .reg = 0xa98 }, { .name = "sdmmc3a", .swgroup = TEGRA_SWGROUP_SDMMC3A, .reg = 0xa9c }, { .name = "sdmmc4a", .swgroup = TEGRA_SWGROUP_SDMMC4A, .reg = 0xaa0 }, - { .name = "vic", .swgroup = TEGRA_SWGROUP_VIC, .reg = 0x284 }, - { .name = "vi", .swgroup = TEGRA_SWGROUP_VI, .reg = 0x280 }, + { .name = "isp2b", .swgroup = TEGRA_SWGROUP_ISP2B, .reg = 0xaa4 }, + { .name = "gpu", .swgroup = TEGRA_SWGROUP_GPU, .reg = 0xaac }, + { .name = "ppcs2", .swgroup = TEGRA_SWGROUP_PPCS2, .reg = 0xab0 }, { .name = "nvdec", .swgroup = TEGRA_SWGROUP_NVDEC, .reg = 0xab4 }, { .name = "ape", .swgroup = TEGRA_SWGROUP_APE, .reg = 0xab8 }, - { .name = "nvjpg", .swgroup = TEGRA_SWGROUP_NVJPG, .reg = 0xac0 }, { .name = "se", .swgroup = TEGRA_SWGROUP_SE, .reg = 0xabc }, + { .name = "nvjpg", .swgroup = TEGRA_SWGROUP_NVJPG, .reg = 0xac0 }, + { .name = "hc1", .swgroup = TEGRA_SWGROUP_HC1, .reg = 0xac4 }, + { .name = "se1", .swgroup = TEGRA_SWGROUP_SE1, .reg = 0xac8 }, { .name = "axiap", .swgroup = TEGRA_SWGROUP_AXIAP, .reg = 0xacc }, { .name = "etr", .swgroup = TEGRA_SWGROUP_ETR, .reg = 0xad0 }, { .name = "tsecb", .swgroup = TEGRA_SWGROUP_TSECB, .reg = 0xad4 }, + { .name = "tsec1", .swgroup = TEGRA_SWGROUP_TSEC1, .reg = 0xad8 }, + { .name = "tsecb1", .swgroup = TEGRA_SWGROUP_TSECB1, .reg = 0xadc }, + { .name = "nvdec1", .swgroup = TEGRA_SWGROUP_NVDEC1, .reg = 0xae0 }, }; static const unsigned int tegra210_group_display[] = { @@ -1073,7 +1227,7 @@ static const struct tegra_smmu_soc tegra210_smmu_soc = { .num_groups = ARRAY_SIZE(tegra210_groups), .supports_round_robin_arbitration = true, .supports_request_limit = true, - .num_tlb_lines = 32, + .num_tlb_lines = 48, .num_asids = 128, }; @@ -1132,4 +1286,5 @@ const struct tegra_mc_soc tegra210_mc_soc = { .reset_ops = &tegra_mc_reset_ops_common, .resets = tegra210_mc_resets, .num_resets = ARRAY_SIZE(tegra210_mc_resets), + .ops = &tegra30_mc_ops, }; diff --git a/drivers/memory/tegra/tegra234.c b/drivers/memory/tegra/tegra234.c new file mode 100644 index 000000000000..a9e8fd99730f --- /dev/null +++ b/drivers/memory/tegra/tegra234.c @@ -0,0 +1,190 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2021-2022, NVIDIA CORPORATION. All rights reserved. + */ + +#include <soc/tegra/mc.h> + +#include <dt-bindings/memory/tegra234-mc.h> + +#include "mc.h" + +static const struct tegra_mc_client tegra234_mc_clients[] = { + { + .id = TEGRA234_MEMORY_CLIENT_MGBEARD, + .name = "mgbeard", + .sid = TEGRA234_SID_MGBE, + .regs = { + .sid = { + .override = 0x2c0, + .security = 0x2c4, + }, + }, + }, { + .id = TEGRA234_MEMORY_CLIENT_MGBEBRD, + .name = "mgbebrd", + .sid = TEGRA234_SID_MGBE_VF1, + .regs = { + .sid = { + .override = 0x2c8, + .security = 0x2cc, + }, + }, + }, { + .id = TEGRA234_MEMORY_CLIENT_MGBECRD, + .name = "mgbecrd", + .sid = TEGRA234_SID_MGBE_VF2, + .regs = { + .sid = { + .override = 0x2d0, + .security = 0x2d4, + }, + }, + }, { + .id = TEGRA234_MEMORY_CLIENT_MGBEDRD, + .name = "mgbedrd", + .sid = TEGRA234_SID_MGBE_VF3, + .regs = { + .sid = { + .override = 0x2d8, + .security = 0x2dc, + }, + }, + }, { + .id = TEGRA234_MEMORY_CLIENT_MGBEAWR, + .name = "mgbeawr", + .sid = TEGRA234_SID_MGBE, + .regs = { + .sid = { + .override = 0x2e0, + .security = 0x2e4, + }, + }, + }, { + .id = TEGRA234_MEMORY_CLIENT_MGBEBWR, + .name = "mgbebwr", + .sid = TEGRA234_SID_MGBE_VF1, + .regs = { + .sid = { + .override = 0x2f8, + .security = 0x2fc, + }, + }, + }, { + .id = TEGRA234_MEMORY_CLIENT_MGBECWR, + .name = "mgbecwr", + .sid = TEGRA234_SID_MGBE_VF2, + .regs = { + .sid = { + .override = 0x308, + .security = 0x30c, + }, + }, + }, { + .id = TEGRA234_MEMORY_CLIENT_SDMMCRAB, + .name = "sdmmcrab", + .sid = TEGRA234_SID_SDMMC4, + .regs = { + .sid = { + .override = 0x318, + .security = 0x31c, + }, + }, + }, { + .id = TEGRA234_MEMORY_CLIENT_MGBEDWR, + .name = "mgbedwr", + .sid = TEGRA234_SID_MGBE_VF3, + .regs = { + .sid = { + .override = 0x328, + .security = 0x32c, + }, + }, + }, { + .id = TEGRA234_MEMORY_CLIENT_SDMMCWAB, + .name = "sdmmcwab", + .sid = TEGRA234_SID_SDMMC4, + .regs = { + .sid = { + .override = 0x338, + .security = 0x33c, + }, + }, + }, { + .id = TEGRA234_MEMORY_CLIENT_BPMPR, + .name = "bpmpr", + .sid = TEGRA234_SID_BPMP, + .regs = { + .sid = { + .override = 0x498, + .security = 0x49c, + }, + }, + }, { + .id = TEGRA234_MEMORY_CLIENT_BPMPW, + .name = "bpmpw", + .sid = TEGRA234_SID_BPMP, + .regs = { + .sid = { + .override = 0x4a0, + .security = 0x4a4, + }, + }, + }, { + .id = TEGRA234_MEMORY_CLIENT_BPMPDMAR, + .name = "bpmpdmar", + .sid = TEGRA234_SID_BPMP, + .regs = { + .sid = { + .override = 0x4a8, + .security = 0x4ac, + }, + }, + }, { + .id = TEGRA234_MEMORY_CLIENT_BPMPDMAW, + .name = "bpmpdmaw", + .sid = TEGRA234_SID_BPMP, + .regs = { + .sid = { + .override = 0x4b0, + .security = 0x4b4, + }, + }, + }, { + .id = TEGRA234_MEMORY_CLIENT_APEDMAR, + .name = "apedmar", + .sid = TEGRA234_SID_APE, + .regs = { + .sid = { + .override = 0x4f8, + .security = 0x4fc, + }, + }, + }, { + .id = TEGRA234_MEMORY_CLIENT_APEDMAW, + .name = "apedmaw", + .sid = TEGRA234_SID_APE, + .regs = { + .sid = { + .override = 0x500, + .security = 0x504, + }, + }, + }, +}; + +const struct tegra_mc_soc tegra234_mc_soc = { + .num_clients = ARRAY_SIZE(tegra234_mc_clients), + .clients = tegra234_mc_clients, + .num_address_bits = 40, + .num_channels = 16, + .client_id_mask = 0x1ff, + .intmask = MC_INT_DECERR_ROUTE_SANITY | + MC_INT_DECERR_GENERALIZED_CARVEOUT | MC_INT_DECERR_MTS | + MC_INT_SECERR_SEC | MC_INT_DECERR_VPR | + MC_INT_SECURITY_VIOLATION | MC_INT_DECERR_EMEM, + .has_addr_hi_reg = true, + .ops = &tegra186_mc_ops, + .ch_intmask = 0x0000ff00, + .global_intstatus_channel_shift = 8, +}; diff --git a/drivers/memory/tegra/tegra30-emc.c b/drivers/memory/tegra/tegra30-emc.c index e3efd9529506..9ba2a9e5316b 100644 --- a/drivers/memory/tegra/tegra30-emc.c +++ b/drivers/memory/tegra/tegra30-emc.c @@ -9,29 +9,38 @@ * Copyright (C) 2019 GRATE-DRIVER project */ +#include <linux/bitfield.h> #include <linux/clk.h> #include <linux/clk/tegra.h> -#include <linux/completion.h> #include <linux/debugfs.h> #include <linux/delay.h> #include <linux/err.h> +#include <linux/interconnect-provider.h> #include <linux/interrupt.h> #include <linux/io.h> #include <linux/iopoll.h> #include <linux/kernel.h> #include <linux/module.h> +#include <linux/mutex.h> #include <linux/of_platform.h> #include <linux/platform_device.h> +#include <linux/pm_opp.h> +#include <linux/slab.h> #include <linux/sort.h> #include <linux/types.h> +#include <soc/tegra/common.h> #include <soc/tegra/fuse.h> +#include "../jedec_ddr.h" +#include "../of_memory.h" + #include "mc.h" #define EMC_INTSTATUS 0x000 #define EMC_INTMASK 0x004 #define EMC_DBG 0x008 +#define EMC_ADR_CFG 0x010 #define EMC_CFG 0x00c #define EMC_REFCTRL 0x020 #define EMC_TIMING_CONTROL 0x028 @@ -77,6 +86,7 @@ #define EMC_EMRS 0x0d0 #define EMC_SELF_REF 0x0e0 #define EMC_MRW 0x0e8 +#define EMC_MRR 0x0ec #define EMC_XM2DQSPADCTRL3 0x0f8 #define EMC_FBIO_SPARE 0x100 #define EMC_FBIO_CFG5 0x104 @@ -146,8 +156,8 @@ #define EMC_SELF_REF_CMD_ENABLED BIT(0) #define DRAM_DEV_SEL_ALL (0 << 30) -#define DRAM_DEV_SEL_0 (2 << 30) -#define DRAM_DEV_SEL_1 (1 << 30) +#define DRAM_DEV_SEL_0 BIT(31) +#define DRAM_DEV_SEL_1 BIT(30) #define DRAM_BROADCAST(num) \ ((num) > 1 ? DRAM_DEV_SEL_ALL : DRAM_DEV_SEL_0) @@ -204,6 +214,13 @@ #define EMC_REFRESH_OVERFLOW_INT BIT(3) #define EMC_CLKCHANGE_COMPLETE_INT BIT(4) +#define EMC_MRR_DIVLD_INT BIT(5) + +#define EMC_MRR_DEV_SELECTN GENMASK(31, 30) +#define EMC_MRR_MRR_MA GENMASK(23, 16) +#define EMC_MRR_MRR_DATA GENMASK(15, 0) + +#define EMC_ADR_CFG_EMEM_NUMDEV BIT(0) enum emc_dram_type { DRAM_TYPE_DDR3, @@ -324,10 +341,21 @@ struct emc_timing { bool emc_cfg_dyn_self_ref; }; +enum emc_rate_request_type { + EMC_RATE_DEBUG, + EMC_RATE_ICC, + EMC_RATE_TYPE_MAX, +}; + +struct emc_rate_request { + unsigned long min_rate; + unsigned long max_rate; +}; + struct tegra_emc { struct device *dev; struct tegra_mc *mc; - struct completion clk_handshake_complete; + struct icc_provider provider; struct notifier_block clk_nb; struct clk *clk; void __iomem *regs; @@ -354,6 +382,17 @@ struct tegra_emc { unsigned long min_rate; unsigned long max_rate; } debugfs; + + /* + * There are multiple sources in the EMC driver which could request + * a min/max clock rate, these rates are contained in this array. + */ + struct emc_rate_request requested_rate[EMC_RATE_TYPE_MAX]; + + /* protect shared rate-change code path */ + struct mutex rate_lock; + + bool mrr_error; }; static int emc_seq_update_timing(struct tegra_emc *emc) @@ -374,52 +413,10 @@ static int emc_seq_update_timing(struct tegra_emc *emc) return 0; } -static void emc_complete_clk_change(struct tegra_emc *emc) -{ - struct emc_timing *timing = emc->new_timing; - unsigned int dram_num; - bool failed = false; - int err; - - /* re-enable auto-refresh */ - dram_num = tegra_mc_get_emem_device_count(emc->mc); - writel_relaxed(EMC_REFCTRL_ENABLE_ALL(dram_num), - emc->regs + EMC_REFCTRL); - - /* restore auto-calibration */ - if (emc->vref_cal_toggle) - writel_relaxed(timing->emc_auto_cal_interval, - emc->regs + EMC_AUTO_CAL_INTERVAL); - - /* restore dynamic self-refresh */ - if (timing->emc_cfg_dyn_self_ref) { - emc->emc_cfg |= EMC_CFG_DYN_SREF_ENABLE; - writel_relaxed(emc->emc_cfg, emc->regs + EMC_CFG); - } - - /* set number of clocks to wait after each ZQ command */ - if (emc->zcal_long) - writel_relaxed(timing->emc_zcal_cnt_long, - emc->regs + EMC_ZCAL_WAIT_CNT); - - /* wait for writes to settle */ - udelay(2); - - /* update restored timing */ - err = emc_seq_update_timing(emc); - if (err) - failed = true; - - /* restore early ACK */ - mc_writel(emc->mc, emc->mc_override, MC_EMEM_ARB_OVERRIDE); - - WRITE_ONCE(emc->bad_state, failed); -} - static irqreturn_t tegra_emc_isr(int irq, void *data) { struct tegra_emc *emc = data; - u32 intmask = EMC_REFRESH_OVERFLOW_INT | EMC_CLKCHANGE_COMPLETE_INT; + u32 intmask = EMC_REFRESH_OVERFLOW_INT; u32 status; status = readl_relaxed(emc->regs + EMC_INTSTATUS) & intmask; @@ -434,18 +431,6 @@ static irqreturn_t tegra_emc_isr(int irq, void *data) /* clear interrupts */ writel_relaxed(status, emc->regs + EMC_INTSTATUS); - /* notify about EMC-CAR handshake completion */ - if (status & EMC_CLKCHANGE_COMPLETE_INT) { - if (completion_done(&emc->clk_handshake_complete)) { - dev_err_ratelimited(emc->dev, - "bogus handshake interrupt\n"); - return IRQ_NONE; - } - - emc_complete_clk_change(emc); - complete(&emc->clk_handshake_complete); - } - return IRQ_HANDLED; } @@ -801,29 +786,58 @@ static int emc_prepare_timing_change(struct tegra_emc *emc, unsigned long rate) */ mc_readl(emc->mc, MC_EMEM_ARB_OVERRIDE); - reinit_completion(&emc->clk_handshake_complete); - - emc->new_timing = timing; - return 0; } static int emc_complete_timing_change(struct tegra_emc *emc, unsigned long rate) { - unsigned long timeout; + struct emc_timing *timing = emc_find_timing(emc, rate); + unsigned int dram_num; + int err; + u32 v; - timeout = wait_for_completion_timeout(&emc->clk_handshake_complete, - msecs_to_jiffies(100)); - if (timeout == 0) { - dev_err(emc->dev, "emc-car handshake failed\n"); - return -EIO; + err = readl_relaxed_poll_timeout_atomic(emc->regs + EMC_INTSTATUS, v, + v & EMC_CLKCHANGE_COMPLETE_INT, + 1, 100); + if (err) { + dev_err(emc->dev, "emc-car handshake timeout: %d\n", err); + return err; } - if (READ_ONCE(emc->bad_state)) - return -EIO; + /* re-enable auto-refresh */ + dram_num = tegra_mc_get_emem_device_count(emc->mc); + writel_relaxed(EMC_REFCTRL_ENABLE_ALL(dram_num), + emc->regs + EMC_REFCTRL); - return 0; + /* restore auto-calibration */ + if (emc->vref_cal_toggle) + writel_relaxed(timing->emc_auto_cal_interval, + emc->regs + EMC_AUTO_CAL_INTERVAL); + + /* restore dynamic self-refresh */ + if (timing->emc_cfg_dyn_self_ref) { + emc->emc_cfg |= EMC_CFG_DYN_SREF_ENABLE; + writel_relaxed(emc->emc_cfg, emc->regs + EMC_CFG); + } + + /* set number of clocks to wait after each ZQ command */ + if (emc->zcal_long) + writel_relaxed(timing->emc_zcal_cnt_long, + emc->regs + EMC_ZCAL_WAIT_CNT); + + /* wait for writes to settle */ + udelay(2); + + /* update restored timing */ + err = emc_seq_update_timing(emc); + if (!err) + emc->bad_state = false; + + /* restore early ACK */ + mc_writel(emc->mc, emc->mc_override, MC_EMEM_ARB_OVERRIDE); + + return err; } static int emc_unprepare_timing_change(struct tegra_emc *emc, @@ -999,22 +1013,33 @@ static int emc_load_timings_from_dt(struct tegra_emc *emc, if (err) return err; - dev_info(emc->dev, - "got %u timings for RAM code %u (min %luMHz max %luMHz)\n", - emc->num_timings, - tegra_read_ram_code(), - emc->timings[0].rate / 1000000, - emc->timings[emc->num_timings - 1].rate / 1000000); + dev_info_once(emc->dev, + "got %u timings for RAM code %u (min %luMHz max %luMHz)\n", + emc->num_timings, + tegra_read_ram_code(), + emc->timings[0].rate / 1000000, + emc->timings[emc->num_timings - 1].rate / 1000000); return 0; } -static struct device_node *emc_find_node_by_ram_code(struct device *dev) +static struct device_node *emc_find_node_by_ram_code(struct tegra_emc *emc) { + struct device *dev = emc->dev; struct device_node *np; u32 value, ram_code; int err; + if (emc->mrr_error) { + dev_warn(dev, "memory timings skipped due to MRR error\n"); + return NULL; + } + + if (of_get_child_count(dev->of_node) == 0) { + dev_info_once(dev, "device-tree doesn't have memory timings\n"); + return NULL; + } + ram_code = tegra_read_ram_code(); for_each_child_of_node(dev->of_node, np) { @@ -1031,11 +1056,73 @@ static struct device_node *emc_find_node_by_ram_code(struct device *dev) return NULL; } +static int emc_read_lpddr_mode_register(struct tegra_emc *emc, + unsigned int emem_dev, + unsigned int register_addr, + unsigned int *register_data) +{ + u32 memory_dev = emem_dev ? 1 : 2; + u32 val, mr_mask = 0xff; + int err; + + /* clear data-valid interrupt status */ + writel_relaxed(EMC_MRR_DIVLD_INT, emc->regs + EMC_INTSTATUS); + + /* issue mode register read request */ + val = FIELD_PREP(EMC_MRR_DEV_SELECTN, memory_dev); + val |= FIELD_PREP(EMC_MRR_MRR_MA, register_addr); + + writel_relaxed(val, emc->regs + EMC_MRR); + + /* wait for the LPDDR2 data-valid interrupt */ + err = readl_relaxed_poll_timeout_atomic(emc->regs + EMC_INTSTATUS, val, + val & EMC_MRR_DIVLD_INT, + 1, 100); + if (err) { + dev_err(emc->dev, "mode register %u read failed: %d\n", + register_addr, err); + emc->mrr_error = true; + return err; + } + + /* read out mode register data */ + val = readl_relaxed(emc->regs + EMC_MRR); + *register_data = FIELD_GET(EMC_MRR_MRR_DATA, val) & mr_mask; + + return 0; +} + +static void emc_read_lpddr_sdram_info(struct tegra_emc *emc, + unsigned int emem_dev) +{ + union lpddr2_basic_config4 basic_conf4; + unsigned int manufacturer_id; + unsigned int revision_id1; + unsigned int revision_id2; + + /* these registers are standard for all LPDDR JEDEC memory chips */ + emc_read_lpddr_mode_register(emc, emem_dev, 5, &manufacturer_id); + emc_read_lpddr_mode_register(emc, emem_dev, 6, &revision_id1); + emc_read_lpddr_mode_register(emc, emem_dev, 7, &revision_id2); + emc_read_lpddr_mode_register(emc, emem_dev, 8, &basic_conf4.value); + + dev_info(emc->dev, "SDRAM[dev%u]: manufacturer: 0x%x (%s) rev1: 0x%x rev2: 0x%x prefetch: S%u density: %uMbit iowidth: %ubit\n", + emem_dev, manufacturer_id, + lpddr2_jedec_manufacturer(manufacturer_id), + revision_id1, revision_id2, + 4 >> basic_conf4.arch_type, + 64 << basic_conf4.density, + 32 >> basic_conf4.io_width); +} + static int emc_setup_hw(struct tegra_emc *emc) { - u32 intmask = EMC_REFRESH_OVERFLOW_INT | EMC_CLKCHANGE_COMPLETE_INT; - u32 fbio_cfg5, emc_cfg, emc_dbg; + u32 fbio_cfg5, emc_cfg, emc_dbg, emc_adr_cfg; + u32 intmask = EMC_REFRESH_OVERFLOW_INT; + static bool print_sdram_info_once; enum emc_dram_type dram_type; + const char *dram_type_str; + unsigned int emem_numdev; fbio_cfg5 = readl_relaxed(emc->regs + EMC_FBIO_CFG5); dram_type = fbio_cfg5 & EMC_FBIO_CFG5_DRAM_TYPE_MASK; @@ -1072,6 +1159,34 @@ static int emc_setup_hw(struct tegra_emc *emc) emc_dbg &= ~EMC_DBG_FORCE_UPDATE; writel_relaxed(emc_dbg, emc->regs + EMC_DBG); + switch (dram_type) { + case DRAM_TYPE_DDR1: + dram_type_str = "DDR1"; + break; + case DRAM_TYPE_LPDDR2: + dram_type_str = "LPDDR2"; + break; + case DRAM_TYPE_DDR2: + dram_type_str = "DDR2"; + break; + case DRAM_TYPE_DDR3: + dram_type_str = "DDR3"; + break; + } + + emc_adr_cfg = readl_relaxed(emc->regs + EMC_ADR_CFG); + emem_numdev = FIELD_GET(EMC_ADR_CFG_EMEM_NUMDEV, emc_adr_cfg) + 1; + + dev_info_once(emc->dev, "%u %s %s attached\n", emem_numdev, + dram_type_str, emem_numdev == 2 ? "devices" : "device"); + + if (dram_type == DRAM_TYPE_LPDDR2 && !print_sdram_info_once) { + while (emem_numdev--) + emc_read_lpddr_sdram_info(emc, emem_numdev); + + print_sdram_info_once = true; + } + return 0; } @@ -1084,6 +1199,9 @@ static long emc_round_rate(unsigned long rate, struct tegra_emc *emc = arg; unsigned int i; + if (!emc->num_timings) + return clk_get_rate(emc->clk); + min_rate = min(min_rate, emc->timings[emc->num_timings - 1].rate); for (i = 0; i < emc->num_timings; i++) { @@ -1113,6 +1231,83 @@ static long emc_round_rate(unsigned long rate, return timing->rate; } +static void tegra_emc_rate_requests_init(struct tegra_emc *emc) +{ + unsigned int i; + + for (i = 0; i < EMC_RATE_TYPE_MAX; i++) { + emc->requested_rate[i].min_rate = 0; + emc->requested_rate[i].max_rate = ULONG_MAX; + } +} + +static int emc_request_rate(struct tegra_emc *emc, + unsigned long new_min_rate, + unsigned long new_max_rate, + enum emc_rate_request_type type) +{ + struct emc_rate_request *req = emc->requested_rate; + unsigned long min_rate = 0, max_rate = ULONG_MAX; + unsigned int i; + int err; + + /* select minimum and maximum rates among the requested rates */ + for (i = 0; i < EMC_RATE_TYPE_MAX; i++, req++) { + if (i == type) { + min_rate = max(new_min_rate, min_rate); + max_rate = min(new_max_rate, max_rate); + } else { + min_rate = max(req->min_rate, min_rate); + max_rate = min(req->max_rate, max_rate); + } + } + + if (min_rate > max_rate) { + dev_err_ratelimited(emc->dev, "%s: type %u: out of range: %lu %lu\n", + __func__, type, min_rate, max_rate); + return -ERANGE; + } + + /* + * EMC rate-changes should go via OPP API because it manages voltage + * changes. + */ + err = dev_pm_opp_set_rate(emc->dev, min_rate); + if (err) + return err; + + emc->requested_rate[type].min_rate = new_min_rate; + emc->requested_rate[type].max_rate = new_max_rate; + + return 0; +} + +static int emc_set_min_rate(struct tegra_emc *emc, unsigned long rate, + enum emc_rate_request_type type) +{ + struct emc_rate_request *req = &emc->requested_rate[type]; + int ret; + + mutex_lock(&emc->rate_lock); + ret = emc_request_rate(emc, rate, req->max_rate, type); + mutex_unlock(&emc->rate_lock); + + return ret; +} + +static int emc_set_max_rate(struct tegra_emc *emc, unsigned long rate, + enum emc_rate_request_type type) +{ + struct emc_rate_request *req = &emc->requested_rate[type]; + int ret; + + mutex_lock(&emc->rate_lock); + ret = emc_request_rate(emc, req->min_rate, rate, type); + mutex_unlock(&emc->rate_lock); + + return ret; +} + /* * debugfs interface * @@ -1196,7 +1391,7 @@ static int tegra_emc_debug_min_rate_set(void *data, u64 rate) if (!tegra_emc_validate_rate(emc, rate)) return -EINVAL; - err = clk_set_min_rate(emc->clk, rate); + err = emc_set_min_rate(emc, rate, EMC_RATE_DEBUG); if (err < 0) return err; @@ -1205,7 +1400,7 @@ static int tegra_emc_debug_min_rate_set(void *data, u64 rate) return 0; } -DEFINE_SIMPLE_ATTRIBUTE(tegra_emc_debug_min_rate_fops, +DEFINE_DEBUGFS_ATTRIBUTE(tegra_emc_debug_min_rate_fops, tegra_emc_debug_min_rate_get, tegra_emc_debug_min_rate_set, "%llu\n"); @@ -1226,7 +1421,7 @@ static int tegra_emc_debug_max_rate_set(void *data, u64 rate) if (!tegra_emc_validate_rate(emc, rate)) return -EINVAL; - err = clk_set_max_rate(emc->clk, rate); + err = emc_set_max_rate(emc, rate, EMC_RATE_DEBUG); if (err < 0) return err; @@ -1235,7 +1430,7 @@ static int tegra_emc_debug_max_rate_set(void *data, u64 rate) return 0; } -DEFINE_SIMPLE_ATTRIBUTE(tegra_emc_debug_max_rate_fops, +DEFINE_DEBUGFS_ATTRIBUTE(tegra_emc_debug_max_rate_fops, tegra_emc_debug_max_rate_get, tegra_emc_debug_max_rate_set, "%llu\n"); @@ -1256,6 +1451,11 @@ static void tegra_emc_debugfs_init(struct tegra_emc *emc) emc->debugfs.max_rate = emc->timings[i].rate; } + if (!emc->num_timings) { + emc->debugfs.min_rate = clk_get_rate(emc->clk); + emc->debugfs.max_rate = emc->debugfs.min_rate; + } + err = clk_set_rate_range(emc->clk, emc->debugfs.min_rate, emc->debugfs.max_rate); if (err < 0) { @@ -1265,66 +1465,190 @@ static void tegra_emc_debugfs_init(struct tegra_emc *emc) } emc->debugfs.root = debugfs_create_dir("emc", NULL); - if (!emc->debugfs.root) { - dev_err(emc->dev, "failed to create debugfs directory\n"); - return; - } - debugfs_create_file("available_rates", S_IRUGO, emc->debugfs.root, + debugfs_create_file("available_rates", 0444, emc->debugfs.root, emc, &tegra_emc_debug_available_rates_fops); - debugfs_create_file("min_rate", S_IRUGO | S_IWUSR, emc->debugfs.root, + debugfs_create_file("min_rate", 0644, emc->debugfs.root, emc, &tegra_emc_debug_min_rate_fops); - debugfs_create_file("max_rate", S_IRUGO | S_IWUSR, emc->debugfs.root, + debugfs_create_file("max_rate", 0644, emc->debugfs.root, emc, &tegra_emc_debug_max_rate_fops); } -static int tegra_emc_probe(struct platform_device *pdev) +static inline struct tegra_emc * +to_tegra_emc_provider(struct icc_provider *provider) { - struct platform_device *mc; - struct device_node *np; - struct tegra_emc *emc; + return container_of(provider, struct tegra_emc, provider); +} + +static struct icc_node_data * +emc_of_icc_xlate_extended(struct of_phandle_args *spec, void *data) +{ + struct icc_provider *provider = data; + struct icc_node_data *ndata; + struct icc_node *node; + + /* External Memory is the only possible ICC route */ + list_for_each_entry(node, &provider->nodes, node_list) { + if (node->id != TEGRA_ICC_EMEM) + continue; + + ndata = kzalloc(sizeof(*ndata), GFP_KERNEL); + if (!ndata) + return ERR_PTR(-ENOMEM); + + /* + * SRC and DST nodes should have matching TAG in order to have + * it set by default for a requested path. + */ + ndata->tag = TEGRA_MC_ICC_TAG_ISO; + ndata->node = node; + + return ndata; + } + + return ERR_PTR(-EPROBE_DEFER); +} + +static int emc_icc_set(struct icc_node *src, struct icc_node *dst) +{ + struct tegra_emc *emc = to_tegra_emc_provider(dst->provider); + unsigned long long peak_bw = icc_units_to_bps(dst->peak_bw); + unsigned long long avg_bw = icc_units_to_bps(dst->avg_bw); + unsigned long long rate = max(avg_bw, peak_bw); + const unsigned int dram_data_bus_width_bytes = 4; + const unsigned int ddr = 2; int err; - if (of_get_child_count(pdev->dev.of_node) == 0) { - dev_info(&pdev->dev, - "device-tree node doesn't have memory timings\n"); - return -ENODEV; + /* + * Tegra30 EMC runs on a clock rate of SDRAM bus. This means that + * EMC clock rate is twice smaller than the peak data rate because + * data is sampled on both EMC clock edges. + */ + do_div(rate, ddr * dram_data_bus_width_bytes); + rate = min_t(u64, rate, U32_MAX); + + err = emc_set_min_rate(emc, rate, EMC_RATE_ICC); + if (err) + return err; + + return 0; +} + +static int tegra_emc_interconnect_init(struct tegra_emc *emc) +{ + const struct tegra_mc_soc *soc = emc->mc->soc; + struct icc_node *node; + int err; + + emc->provider.dev = emc->dev; + emc->provider.set = emc_icc_set; + emc->provider.data = &emc->provider; + emc->provider.aggregate = soc->icc_ops->aggregate; + emc->provider.xlate_extended = emc_of_icc_xlate_extended; + + err = icc_provider_add(&emc->provider); + if (err) + goto err_msg; + + /* create External Memory Controller node */ + node = icc_node_create(TEGRA_ICC_EMC); + if (IS_ERR(node)) { + err = PTR_ERR(node); + goto del_provider; } - np = of_parse_phandle(pdev->dev.of_node, "nvidia,memory-controller", 0); - if (!np) { - dev_err(&pdev->dev, "could not get memory controller node\n"); - return -ENOENT; + node->name = "External Memory Controller"; + icc_node_add(node, &emc->provider); + + /* link External Memory Controller to External Memory (DRAM) */ + err = icc_link_create(node, TEGRA_ICC_EMEM); + if (err) + goto remove_nodes; + + /* create External Memory node */ + node = icc_node_create(TEGRA_ICC_EMEM); + if (IS_ERR(node)) { + err = PTR_ERR(node); + goto remove_nodes; } - mc = of_find_device_by_node(np); - of_node_put(np); - if (!mc) - return -ENOENT; + node->name = "External Memory (DRAM)"; + icc_node_add(node, &emc->provider); - np = emc_find_node_by_ram_code(&pdev->dev); - if (!np) - return -EINVAL; + return 0; + +remove_nodes: + icc_nodes_remove(&emc->provider); +del_provider: + icc_provider_del(&emc->provider); +err_msg: + dev_err(emc->dev, "failed to initialize ICC: %d\n", err); + + return err; +} + +static void devm_tegra_emc_unset_callback(void *data) +{ + tegra20_clk_set_emc_round_callback(NULL, NULL); +} + +static void devm_tegra_emc_unreg_clk_notifier(void *data) +{ + struct tegra_emc *emc = data; + + clk_notifier_unregister(emc->clk, &emc->clk_nb); +} + +static int tegra_emc_init_clk(struct tegra_emc *emc) +{ + int err; + + tegra20_clk_set_emc_round_callback(emc_round_rate, emc); + + err = devm_add_action_or_reset(emc->dev, devm_tegra_emc_unset_callback, + NULL); + if (err) + return err; + + emc->clk = devm_clk_get(emc->dev, NULL); + if (IS_ERR(emc->clk)) { + dev_err(emc->dev, "failed to get EMC clock: %pe\n", emc->clk); + return PTR_ERR(emc->clk); + } + + err = clk_notifier_register(emc->clk, &emc->clk_nb); + if (err) { + dev_err(emc->dev, "failed to register clk notifier: %d\n", err); + return err; + } + + err = devm_add_action_or_reset(emc->dev, + devm_tegra_emc_unreg_clk_notifier, emc); + if (err) + return err; + + return 0; +} + +static int tegra_emc_probe(struct platform_device *pdev) +{ + struct tegra_core_opp_params opp_params = {}; + struct device_node *np; + struct tegra_emc *emc; + int err; emc = devm_kzalloc(&pdev->dev, sizeof(*emc), GFP_KERNEL); - if (!emc) { - of_node_put(np); + if (!emc) return -ENOMEM; - } - emc->mc = platform_get_drvdata(mc); - if (!emc->mc) - return -EPROBE_DEFER; + emc->mc = devm_tegra_memory_controller_get(&pdev->dev); + if (IS_ERR(emc->mc)) + return PTR_ERR(emc->mc); - init_completion(&emc->clk_handshake_complete); + mutex_init(&emc->rate_lock); emc->clk_nb.notifier_call = emc_clk_change_notify; emc->dev = &pdev->dev; - err = emc_load_timings_from_dt(emc, np); - of_node_put(np); - if (err) - return err; - emc->regs = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(emc->regs)) return PTR_ERR(emc->regs); @@ -1333,11 +1657,18 @@ static int tegra_emc_probe(struct platform_device *pdev) if (err) return err; + np = emc_find_node_by_ram_code(emc); + if (np) { + err = emc_load_timings_from_dt(emc, np); + of_node_put(np); + if (err) + return err; + } + err = platform_get_irq(pdev, 0); - if (err < 0) { - dev_err(&pdev->dev, "interrupt not specified: %d\n", err); + if (err < 0) return err; - } + emc->irq = err; err = devm_request_irq(&pdev->dev, emc->irq, tegra_emc_isr, 0, @@ -1347,31 +1678,29 @@ static int tegra_emc_probe(struct platform_device *pdev) return err; } - tegra20_clk_set_emc_round_callback(emc_round_rate, emc); + err = tegra_emc_init_clk(emc); + if (err) + return err; - emc->clk = devm_clk_get(&pdev->dev, "emc"); - if (IS_ERR(emc->clk)) { - err = PTR_ERR(emc->clk); - dev_err(&pdev->dev, "failed to get emc clock: %d\n", err); - goto unset_cb; - } + opp_params.init_state = true; - err = clk_notifier_register(emc->clk, &emc->clk_nb); - if (err) { - dev_err(&pdev->dev, "failed to register clk notifier: %d\n", - err); - goto unset_cb; - } + err = devm_tegra_core_dev_init_opp_table(&pdev->dev, &opp_params); + if (err) + return err; platform_set_drvdata(pdev, emc); + tegra_emc_rate_requests_init(emc); tegra_emc_debugfs_init(emc); + tegra_emc_interconnect_init(emc); - return 0; - -unset_cb: - tegra20_clk_set_emc_round_callback(NULL, NULL); + /* + * Don't allow the kernel module to be unloaded. Unloading adds some + * extra complexity which doesn't really worth the effort in a case of + * this driver. + */ + try_module_get(THIS_MODULE); - return err; + return 0; } static int tegra_emc_suspend(struct device *dev) @@ -1416,6 +1745,7 @@ static const struct of_device_id tegra_emc_of_match[] = { { .compatible = "nvidia,tegra30-emc", }, {}, }; +MODULE_DEVICE_TABLE(of, tegra_emc_of_match); static struct platform_driver tegra_emc_driver = { .probe = tegra_emc_probe, @@ -1424,11 +1754,11 @@ static struct platform_driver tegra_emc_driver = { .of_match_table = tegra_emc_of_match, .pm = &tegra_emc_pm_ops, .suppress_bind_attrs = true, + .sync_state = icc_sync_state, }, }; +module_platform_driver(tegra_emc_driver); -static int __init tegra_emc_init(void) -{ - return platform_driver_register(&tegra_emc_driver); -} -subsys_initcall(tegra_emc_init); +MODULE_AUTHOR("Dmitry Osipenko <digetx@gmail.com>"); +MODULE_DESCRIPTION("NVIDIA Tegra30 EMC driver"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/memory/tegra/tegra30.c b/drivers/memory/tegra/tegra30.c index fcdd812eed80..84316357513d 100644 --- a/drivers/memory/tegra/tegra30.c +++ b/drivers/memory/tegra/tegra30.c @@ -4,7 +4,8 @@ */ #include <linux/of.h> -#include <linux/mm.h> +#include <linux/of_device.h> +#include <linux/slab.h> #include <dt-bindings/memory/tegra30-mc.h> @@ -36,900 +37,1104 @@ static const struct tegra_mc_client tegra30_mc_clients[] = { .id = 0x00, .name = "ptcr", .swgroup = TEGRA_SWGROUP_PTC, + .regs = { + .la = { + .reg = 0x34c, + .shift = 0, + .mask = 0xff, + .def = 0x0, + }, + }, + .fifo_size = 16 * 2, }, { .id = 0x01, .name = "display0a", .swgroup = TEGRA_SWGROUP_DC, - .smmu = { - .reg = 0x228, - .bit = 1, - }, - .la = { - .reg = 0x2e8, - .shift = 0, - .mask = 0xff, - .def = 0x4e, - }, + .regs = { + .smmu = { + .reg = 0x228, + .bit = 1, + }, + .la = { + .reg = 0x2e8, + .shift = 0, + .mask = 0xff, + .def = 0x4e, + }, + }, + .fifo_size = 16 * 128, }, { .id = 0x02, .name = "display0ab", .swgroup = TEGRA_SWGROUP_DCB, - .smmu = { - .reg = 0x228, - .bit = 2, - }, - .la = { - .reg = 0x2f4, - .shift = 0, - .mask = 0xff, - .def = 0x4e, - }, + .regs = { + .smmu = { + .reg = 0x228, + .bit = 2, + }, + .la = { + .reg = 0x2f4, + .shift = 0, + .mask = 0xff, + .def = 0x4e, + }, + }, + .fifo_size = 16 * 128, }, { .id = 0x03, .name = "display0b", .swgroup = TEGRA_SWGROUP_DC, - .smmu = { - .reg = 0x228, - .bit = 3, - }, - .la = { - .reg = 0x2e8, - .shift = 16, - .mask = 0xff, - .def = 0x4e, - }, + .regs = { + .smmu = { + .reg = 0x228, + .bit = 3, + }, + .la = { + .reg = 0x2e8, + .shift = 16, + .mask = 0xff, + .def = 0x4e, + }, + }, + .fifo_size = 16 * 64, }, { .id = 0x04, .name = "display0bb", .swgroup = TEGRA_SWGROUP_DCB, - .smmu = { - .reg = 0x228, - .bit = 4, - }, - .la = { - .reg = 0x2f4, - .shift = 16, - .mask = 0xff, - .def = 0x4e, - }, + .regs = { + .smmu = { + .reg = 0x228, + .bit = 4, + }, + .la = { + .reg = 0x2f4, + .shift = 16, + .mask = 0xff, + .def = 0x4e, + }, + }, + .fifo_size = 16 * 64, }, { .id = 0x05, .name = "display0c", .swgroup = TEGRA_SWGROUP_DC, - .smmu = { - .reg = 0x228, - .bit = 5, - }, - .la = { - .reg = 0x2ec, - .shift = 0, - .mask = 0xff, - .def = 0x4e, - }, + .regs = { + .smmu = { + .reg = 0x228, + .bit = 5, + }, + .la = { + .reg = 0x2ec, + .shift = 0, + .mask = 0xff, + .def = 0x4e, + }, + }, + .fifo_size = 16 * 128, }, { .id = 0x06, .name = "display0cb", .swgroup = TEGRA_SWGROUP_DCB, - .smmu = { - .reg = 0x228, - .bit = 6, - }, - .la = { - .reg = 0x2f8, - .shift = 0, - .mask = 0xff, - .def = 0x4e, - }, + .regs = { + .smmu = { + .reg = 0x228, + .bit = 6, + }, + .la = { + .reg = 0x2f8, + .shift = 0, + .mask = 0xff, + .def = 0x4e, + }, + }, + .fifo_size = 16 * 128, }, { .id = 0x07, .name = "display1b", .swgroup = TEGRA_SWGROUP_DC, - .smmu = { - .reg = 0x228, - .bit = 7, - }, - .la = { - .reg = 0x2ec, - .shift = 16, - .mask = 0xff, - .def = 0x4e, - }, + .regs = { + .smmu = { + .reg = 0x228, + .bit = 7, + }, + .la = { + .reg = 0x2ec, + .shift = 16, + .mask = 0xff, + .def = 0x4e, + }, + }, + .fifo_size = 16 * 64, }, { .id = 0x08, .name = "display1bb", .swgroup = TEGRA_SWGROUP_DCB, - .smmu = { - .reg = 0x228, - .bit = 8, - }, - .la = { - .reg = 0x2f8, - .shift = 16, - .mask = 0xff, - .def = 0x4e, - }, + .regs = { + .smmu = { + .reg = 0x228, + .bit = 8, + }, + .la = { + .reg = 0x2f8, + .shift = 16, + .mask = 0xff, + .def = 0x4e, + }, + }, + .fifo_size = 16 * 64, }, { .id = 0x09, .name = "eppup", .swgroup = TEGRA_SWGROUP_EPP, - .smmu = { - .reg = 0x228, - .bit = 9, - }, - .la = { - .reg = 0x300, - .shift = 0, - .mask = 0xff, - .def = 0x17, - }, + .regs = { + .smmu = { + .reg = 0x228, + .bit = 9, + }, + .la = { + .reg = 0x300, + .shift = 0, + .mask = 0xff, + .def = 0x17, + }, + }, + .fifo_size = 16 * 8, }, { .id = 0x0a, .name = "g2pr", .swgroup = TEGRA_SWGROUP_G2, - .smmu = { - .reg = 0x228, - .bit = 10, - }, - .la = { - .reg = 0x308, - .shift = 0, - .mask = 0xff, - .def = 0x09, - }, + .regs = { + .smmu = { + .reg = 0x228, + .bit = 10, + }, + .la = { + .reg = 0x308, + .shift = 0, + .mask = 0xff, + .def = 0x09, + }, + }, + .fifo_size = 16 * 64, }, { .id = 0x0b, .name = "g2sr", .swgroup = TEGRA_SWGROUP_G2, - .smmu = { - .reg = 0x228, - .bit = 11, - }, - .la = { - .reg = 0x308, - .shift = 16, - .mask = 0xff, - .def = 0x09, - }, + .regs = { + .smmu = { + .reg = 0x228, + .bit = 11, + }, + .la = { + .reg = 0x308, + .shift = 16, + .mask = 0xff, + .def = 0x09, + }, + }, + .fifo_size = 16 * 64, }, { .id = 0x0c, .name = "mpeunifbr", .swgroup = TEGRA_SWGROUP_MPE, - .smmu = { - .reg = 0x228, - .bit = 12, - }, - .la = { - .reg = 0x328, - .shift = 0, - .mask = 0xff, - .def = 0x50, - }, + .regs = { + .smmu = { + .reg = 0x228, + .bit = 12, + }, + .la = { + .reg = 0x328, + .shift = 0, + .mask = 0xff, + .def = 0x50, + }, + }, + .fifo_size = 16 * 8, }, { .id = 0x0d, .name = "viruv", .swgroup = TEGRA_SWGROUP_VI, - .smmu = { - .reg = 0x228, - .bit = 13, - }, - .la = { - .reg = 0x364, - .shift = 0, - .mask = 0xff, - .def = 0x2c, - }, + .regs = { + .smmu = { + .reg = 0x228, + .bit = 13, + }, + .la = { + .reg = 0x364, + .shift = 0, + .mask = 0xff, + .def = 0x2c, + }, + }, + .fifo_size = 16 * 8, }, { .id = 0x0e, .name = "afir", .swgroup = TEGRA_SWGROUP_AFI, - .smmu = { - .reg = 0x228, - .bit = 14, - }, - .la = { - .reg = 0x2e0, - .shift = 0, - .mask = 0xff, - .def = 0x10, - }, + .regs = { + .smmu = { + .reg = 0x228, + .bit = 14, + }, + .la = { + .reg = 0x2e0, + .shift = 0, + .mask = 0xff, + .def = 0x10, + }, + }, + .fifo_size = 16 * 32, }, { .id = 0x0f, .name = "avpcarm7r", .swgroup = TEGRA_SWGROUP_AVPC, - .smmu = { - .reg = 0x228, - .bit = 15, - }, - .la = { - .reg = 0x2e4, - .shift = 0, - .mask = 0xff, - .def = 0x04, - }, + .regs = { + .smmu = { + .reg = 0x228, + .bit = 15, + }, + .la = { + .reg = 0x2e4, + .shift = 0, + .mask = 0xff, + .def = 0x04, + }, + }, + .fifo_size = 16 * 2, }, { .id = 0x10, .name = "displayhc", .swgroup = TEGRA_SWGROUP_DC, - .smmu = { - .reg = 0x228, - .bit = 16, - }, - .la = { - .reg = 0x2f0, - .shift = 0, - .mask = 0xff, - .def = 0xff, - }, + .regs = { + .smmu = { + .reg = 0x228, + .bit = 16, + }, + .la = { + .reg = 0x2f0, + .shift = 0, + .mask = 0xff, + .def = 0xff, + }, + }, + .fifo_size = 16 * 2, }, { .id = 0x11, .name = "displayhcb", .swgroup = TEGRA_SWGROUP_DCB, - .smmu = { - .reg = 0x228, - .bit = 17, - }, - .la = { - .reg = 0x2fc, - .shift = 0, - .mask = 0xff, - .def = 0xff, - }, + .regs = { + .smmu = { + .reg = 0x228, + .bit = 17, + }, + .la = { + .reg = 0x2fc, + .shift = 0, + .mask = 0xff, + .def = 0xff, + }, + }, + .fifo_size = 16 * 2, }, { .id = 0x12, .name = "fdcdrd", .swgroup = TEGRA_SWGROUP_NV, - .smmu = { - .reg = 0x228, - .bit = 18, - }, - .la = { - .reg = 0x334, - .shift = 0, - .mask = 0xff, - .def = 0x0a, - }, + .regs = { + .smmu = { + .reg = 0x228, + .bit = 18, + }, + .la = { + .reg = 0x334, + .shift = 0, + .mask = 0xff, + .def = 0x0a, + }, + }, + .fifo_size = 16 * 48, }, { .id = 0x13, .name = "fdcdrd2", .swgroup = TEGRA_SWGROUP_NV2, - .smmu = { - .reg = 0x228, - .bit = 19, - }, - .la = { - .reg = 0x33c, - .shift = 0, - .mask = 0xff, - .def = 0x0a, - }, + .regs = { + .smmu = { + .reg = 0x228, + .bit = 19, + }, + .la = { + .reg = 0x33c, + .shift = 0, + .mask = 0xff, + .def = 0x0a, + }, + }, + .fifo_size = 16 * 48, }, { .id = 0x14, .name = "g2dr", .swgroup = TEGRA_SWGROUP_G2, - .smmu = { - .reg = 0x228, - .bit = 20, - }, - .la = { - .reg = 0x30c, - .shift = 0, - .mask = 0xff, - .def = 0x0a, - }, + .regs = { + .smmu = { + .reg = 0x228, + .bit = 20, + }, + .la = { + .reg = 0x30c, + .shift = 0, + .mask = 0xff, + .def = 0x0a, + }, + }, + .fifo_size = 16 * 48, }, { .id = 0x15, .name = "hdar", .swgroup = TEGRA_SWGROUP_HDA, - .smmu = { - .reg = 0x228, - .bit = 21, - }, - .la = { - .reg = 0x318, - .shift = 0, - .mask = 0xff, - .def = 0xff, - }, + .regs = { + .smmu = { + .reg = 0x228, + .bit = 21, + }, + .la = { + .reg = 0x318, + .shift = 0, + .mask = 0xff, + .def = 0xff, + }, + }, + .fifo_size = 16 * 16, }, { .id = 0x16, .name = "host1xdmar", .swgroup = TEGRA_SWGROUP_HC, - .smmu = { - .reg = 0x228, - .bit = 22, - }, - .la = { - .reg = 0x310, - .shift = 0, - .mask = 0xff, - .def = 0x05, - }, + .regs = { + .smmu = { + .reg = 0x228, + .bit = 22, + }, + .la = { + .reg = 0x310, + .shift = 0, + .mask = 0xff, + .def = 0x05, + }, + }, + .fifo_size = 16 * 16, }, { .id = 0x17, .name = "host1xr", .swgroup = TEGRA_SWGROUP_HC, - .smmu = { - .reg = 0x228, - .bit = 23, - }, - .la = { - .reg = 0x310, - .shift = 16, - .mask = 0xff, - .def = 0x50, - }, + .regs = { + .smmu = { + .reg = 0x228, + .bit = 23, + }, + .la = { + .reg = 0x310, + .shift = 16, + .mask = 0xff, + .def = 0x50, + }, + }, + .fifo_size = 16 * 8, }, { .id = 0x18, .name = "idxsrd", .swgroup = TEGRA_SWGROUP_NV, - .smmu = { - .reg = 0x228, - .bit = 24, - }, - .la = { - .reg = 0x334, - .shift = 16, - .mask = 0xff, - .def = 0x13, - }, + .regs = { + .smmu = { + .reg = 0x228, + .bit = 24, + }, + .la = { + .reg = 0x334, + .shift = 16, + .mask = 0xff, + .def = 0x13, + }, + }, + .fifo_size = 16 * 64, }, { .id = 0x19, .name = "idxsrd2", .swgroup = TEGRA_SWGROUP_NV2, - .smmu = { - .reg = 0x228, - .bit = 25, - }, - .la = { - .reg = 0x33c, - .shift = 16, - .mask = 0xff, - .def = 0x13, - }, + .regs = { + .smmu = { + .reg = 0x228, + .bit = 25, + }, + .la = { + .reg = 0x33c, + .shift = 16, + .mask = 0xff, + .def = 0x13, + }, + }, + .fifo_size = 16 * 64, }, { .id = 0x1a, .name = "mpe_ipred", .swgroup = TEGRA_SWGROUP_MPE, - .smmu = { - .reg = 0x228, - .bit = 26, - }, - .la = { - .reg = 0x328, - .shift = 16, - .mask = 0xff, - .def = 0x80, - }, + .regs = { + .smmu = { + .reg = 0x228, + .bit = 26, + }, + .la = { + .reg = 0x328, + .shift = 16, + .mask = 0xff, + .def = 0x80, + }, + }, + .fifo_size = 16 * 2, }, { .id = 0x1b, .name = "mpeamemrd", .swgroup = TEGRA_SWGROUP_MPE, - .smmu = { - .reg = 0x228, - .bit = 27, - }, - .la = { - .reg = 0x32c, - .shift = 0, - .mask = 0xff, - .def = 0x42, - }, + .regs = { + .smmu = { + .reg = 0x228, + .bit = 27, + }, + .la = { + .reg = 0x32c, + .shift = 0, + .mask = 0xff, + .def = 0x42, + }, + }, + .fifo_size = 16 * 64, }, { .id = 0x1c, .name = "mpecsrd", .swgroup = TEGRA_SWGROUP_MPE, - .smmu = { - .reg = 0x228, - .bit = 28, - }, - .la = { - .reg = 0x32c, - .shift = 16, - .mask = 0xff, - .def = 0xff, - }, + .regs = { + .smmu = { + .reg = 0x228, + .bit = 28, + }, + .la = { + .reg = 0x32c, + .shift = 16, + .mask = 0xff, + .def = 0xff, + }, + }, + .fifo_size = 16 * 8, }, { .id = 0x1d, .name = "ppcsahbdmar", .swgroup = TEGRA_SWGROUP_PPCS, - .smmu = { - .reg = 0x228, - .bit = 29, - }, - .la = { - .reg = 0x344, - .shift = 0, - .mask = 0xff, - .def = 0x10, - }, + .regs = { + .smmu = { + .reg = 0x228, + .bit = 29, + }, + .la = { + .reg = 0x344, + .shift = 0, + .mask = 0xff, + .def = 0x10, + }, + }, + .fifo_size = 16 * 2, }, { .id = 0x1e, .name = "ppcsahbslvr", .swgroup = TEGRA_SWGROUP_PPCS, - .smmu = { - .reg = 0x228, - .bit = 30, - }, - .la = { - .reg = 0x344, - .shift = 16, - .mask = 0xff, - .def = 0x12, - }, + .regs = { + .smmu = { + .reg = 0x228, + .bit = 30, + }, + .la = { + .reg = 0x344, + .shift = 16, + .mask = 0xff, + .def = 0x12, + }, + }, + .fifo_size = 16 * 8, }, { .id = 0x1f, .name = "satar", .swgroup = TEGRA_SWGROUP_SATA, - .smmu = { - .reg = 0x228, - .bit = 31, - }, - .la = { - .reg = 0x350, - .shift = 0, - .mask = 0xff, - .def = 0x33, - }, + .regs = { + .smmu = { + .reg = 0x228, + .bit = 31, + }, + .la = { + .reg = 0x350, + .shift = 0, + .mask = 0xff, + .def = 0x33, + }, + }, + .fifo_size = 16 * 32, }, { .id = 0x20, .name = "texsrd", .swgroup = TEGRA_SWGROUP_NV, - .smmu = { - .reg = 0x22c, - .bit = 0, - }, - .la = { - .reg = 0x338, - .shift = 0, - .mask = 0xff, - .def = 0x13, - }, + .regs = { + .smmu = { + .reg = 0x22c, + .bit = 0, + }, + .la = { + .reg = 0x338, + .shift = 0, + .mask = 0xff, + .def = 0x13, + }, + }, + .fifo_size = 16 * 64, }, { .id = 0x21, .name = "texsrd2", .swgroup = TEGRA_SWGROUP_NV2, - .smmu = { - .reg = 0x22c, - .bit = 1, - }, - .la = { - .reg = 0x340, - .shift = 0, - .mask = 0xff, - .def = 0x13, - }, + .regs = { + .smmu = { + .reg = 0x22c, + .bit = 1, + }, + .la = { + .reg = 0x340, + .shift = 0, + .mask = 0xff, + .def = 0x13, + }, + }, + .fifo_size = 16 * 64, }, { .id = 0x22, .name = "vdebsevr", .swgroup = TEGRA_SWGROUP_VDE, - .smmu = { - .reg = 0x22c, - .bit = 2, - }, - .la = { - .reg = 0x354, - .shift = 0, - .mask = 0xff, - .def = 0xff, - }, + .regs = { + .smmu = { + .reg = 0x22c, + .bit = 2, + }, + .la = { + .reg = 0x354, + .shift = 0, + .mask = 0xff, + .def = 0xff, + }, + }, + .fifo_size = 16 * 8, }, { .id = 0x23, .name = "vdember", .swgroup = TEGRA_SWGROUP_VDE, - .smmu = { - .reg = 0x22c, - .bit = 3, - }, - .la = { - .reg = 0x354, - .shift = 16, - .mask = 0xff, - .def = 0xd0, - }, + .regs = { + .smmu = { + .reg = 0x22c, + .bit = 3, + }, + .la = { + .reg = 0x354, + .shift = 16, + .mask = 0xff, + .def = 0xd0, + }, + }, + .fifo_size = 16 * 4, }, { .id = 0x24, .name = "vdemcer", .swgroup = TEGRA_SWGROUP_VDE, - .smmu = { - .reg = 0x22c, - .bit = 4, - }, - .la = { - .reg = 0x358, - .shift = 0, - .mask = 0xff, - .def = 0x2a, - }, + .regs = { + .smmu = { + .reg = 0x22c, + .bit = 4, + }, + .la = { + .reg = 0x358, + .shift = 0, + .mask = 0xff, + .def = 0x2a, + }, + }, + .fifo_size = 16 * 16, }, { .id = 0x25, .name = "vdetper", .swgroup = TEGRA_SWGROUP_VDE, - .smmu = { - .reg = 0x22c, - .bit = 5, - }, - .la = { - .reg = 0x358, - .shift = 16, - .mask = 0xff, - .def = 0x74, - }, + .regs = { + .smmu = { + .reg = 0x22c, + .bit = 5, + }, + .la = { + .reg = 0x358, + .shift = 16, + .mask = 0xff, + .def = 0x74, + }, + }, + .fifo_size = 16 * 16, }, { .id = 0x26, .name = "mpcorelpr", .swgroup = TEGRA_SWGROUP_MPCORELP, - .la = { - .reg = 0x324, - .shift = 0, - .mask = 0xff, - .def = 0x04, + .regs = { + .la = { + .reg = 0x324, + .shift = 0, + .mask = 0xff, + .def = 0x04, + }, }, + .fifo_size = 16 * 14, }, { .id = 0x27, .name = "mpcorer", .swgroup = TEGRA_SWGROUP_MPCORE, - .la = { - .reg = 0x320, - .shift = 0, - .mask = 0xff, - .def = 0x04, + .regs = { + .la = { + .reg = 0x320, + .shift = 0, + .mask = 0xff, + .def = 0x04, + }, }, + .fifo_size = 16 * 14, }, { .id = 0x28, .name = "eppu", .swgroup = TEGRA_SWGROUP_EPP, - .smmu = { - .reg = 0x22c, - .bit = 8, - }, - .la = { - .reg = 0x300, - .shift = 16, - .mask = 0xff, - .def = 0x6c, - }, + .regs = { + .smmu = { + .reg = 0x22c, + .bit = 8, + }, + .la = { + .reg = 0x300, + .shift = 16, + .mask = 0xff, + .def = 0x6c, + }, + }, + .fifo_size = 16 * 64, }, { .id = 0x29, .name = "eppv", .swgroup = TEGRA_SWGROUP_EPP, - .smmu = { - .reg = 0x22c, - .bit = 9, - }, - .la = { - .reg = 0x304, - .shift = 0, - .mask = 0xff, - .def = 0x6c, - }, + .regs = { + .smmu = { + .reg = 0x22c, + .bit = 9, + }, + .la = { + .reg = 0x304, + .shift = 0, + .mask = 0xff, + .def = 0x6c, + }, + }, + .fifo_size = 16 * 64, }, { .id = 0x2a, .name = "eppy", .swgroup = TEGRA_SWGROUP_EPP, - .smmu = { - .reg = 0x22c, - .bit = 10, - }, - .la = { - .reg = 0x304, - .shift = 16, - .mask = 0xff, - .def = 0x6c, - }, + .regs = { + .smmu = { + .reg = 0x22c, + .bit = 10, + }, + .la = { + .reg = 0x304, + .shift = 16, + .mask = 0xff, + .def = 0x6c, + }, + }, + .fifo_size = 16 * 64, }, { .id = 0x2b, .name = "mpeunifbw", .swgroup = TEGRA_SWGROUP_MPE, - .smmu = { - .reg = 0x22c, - .bit = 11, - }, - .la = { - .reg = 0x330, - .shift = 0, - .mask = 0xff, - .def = 0x13, - }, + .regs = { + .smmu = { + .reg = 0x22c, + .bit = 11, + }, + .la = { + .reg = 0x330, + .shift = 0, + .mask = 0xff, + .def = 0x13, + }, + }, + .fifo_size = 16 * 8, }, { .id = 0x2c, .name = "viwsb", .swgroup = TEGRA_SWGROUP_VI, - .smmu = { - .reg = 0x22c, - .bit = 12, - }, - .la = { - .reg = 0x364, - .shift = 16, - .mask = 0xff, - .def = 0x12, - }, + .regs = { + .smmu = { + .reg = 0x22c, + .bit = 12, + }, + .la = { + .reg = 0x364, + .shift = 16, + .mask = 0xff, + .def = 0x12, + }, + }, + .fifo_size = 16 * 64, }, { .id = 0x2d, .name = "viwu", .swgroup = TEGRA_SWGROUP_VI, - .smmu = { - .reg = 0x22c, - .bit = 13, - }, - .la = { - .reg = 0x368, - .shift = 0, - .mask = 0xff, - .def = 0xb2, - }, + .regs = { + .smmu = { + .reg = 0x22c, + .bit = 13, + }, + .la = { + .reg = 0x368, + .shift = 0, + .mask = 0xff, + .def = 0xb2, + }, + }, + .fifo_size = 16 * 64, }, { .id = 0x2e, .name = "viwv", .swgroup = TEGRA_SWGROUP_VI, - .smmu = { - .reg = 0x22c, - .bit = 14, - }, - .la = { - .reg = 0x368, - .shift = 16, - .mask = 0xff, - .def = 0xb2, - }, + .regs = { + .smmu = { + .reg = 0x22c, + .bit = 14, + }, + .la = { + .reg = 0x368, + .shift = 16, + .mask = 0xff, + .def = 0xb2, + }, + }, + .fifo_size = 16 * 64, }, { .id = 0x2f, .name = "viwy", .swgroup = TEGRA_SWGROUP_VI, - .smmu = { - .reg = 0x22c, - .bit = 15, - }, - .la = { - .reg = 0x36c, - .shift = 0, - .mask = 0xff, - .def = 0x12, - }, + .regs = { + .smmu = { + .reg = 0x22c, + .bit = 15, + }, + .la = { + .reg = 0x36c, + .shift = 0, + .mask = 0xff, + .def = 0x12, + }, + }, + .fifo_size = 16 * 64, }, { .id = 0x30, .name = "g2dw", .swgroup = TEGRA_SWGROUP_G2, - .smmu = { - .reg = 0x22c, - .bit = 16, - }, - .la = { - .reg = 0x30c, - .shift = 16, - .mask = 0xff, - .def = 0x9, - }, + .regs = { + .smmu = { + .reg = 0x22c, + .bit = 16, + }, + .la = { + .reg = 0x30c, + .shift = 16, + .mask = 0xff, + .def = 0x9, + }, + }, + .fifo_size = 16 * 128, }, { .id = 0x31, .name = "afiw", .swgroup = TEGRA_SWGROUP_AFI, - .smmu = { - .reg = 0x22c, - .bit = 17, - }, - .la = { - .reg = 0x2e0, - .shift = 16, - .mask = 0xff, - .def = 0x0c, - }, + .regs = { + .smmu = { + .reg = 0x22c, + .bit = 17, + }, + .la = { + .reg = 0x2e0, + .shift = 16, + .mask = 0xff, + .def = 0x0c, + }, + }, + .fifo_size = 16 * 32, }, { .id = 0x32, .name = "avpcarm7w", .swgroup = TEGRA_SWGROUP_AVPC, - .smmu = { - .reg = 0x22c, - .bit = 18, - }, - .la = { - .reg = 0x2e4, - .shift = 16, - .mask = 0xff, - .def = 0x0e, - }, + .regs = { + .smmu = { + .reg = 0x22c, + .bit = 18, + }, + .la = { + .reg = 0x2e4, + .shift = 16, + .mask = 0xff, + .def = 0x0e, + }, + }, + .fifo_size = 16 * 2, }, { .id = 0x33, .name = "fdcdwr", .swgroup = TEGRA_SWGROUP_NV, - .smmu = { - .reg = 0x22c, - .bit = 19, - }, - .la = { - .reg = 0x338, - .shift = 16, - .mask = 0xff, - .def = 0x0a, - }, + .regs = { + .smmu = { + .reg = 0x22c, + .bit = 19, + }, + .la = { + .reg = 0x338, + .shift = 16, + .mask = 0xff, + .def = 0x0a, + }, + }, + .fifo_size = 16 * 48, }, { .id = 0x34, .name = "fdcdwr2", .swgroup = TEGRA_SWGROUP_NV2, - .smmu = { - .reg = 0x22c, - .bit = 20, - }, - .la = { - .reg = 0x340, - .shift = 16, - .mask = 0xff, - .def = 0x0a, - }, + .regs = { + .smmu = { + .reg = 0x22c, + .bit = 20, + }, + .la = { + .reg = 0x340, + .shift = 16, + .mask = 0xff, + .def = 0x0a, + }, + }, + .fifo_size = 16 * 48, }, { .id = 0x35, .name = "hdaw", .swgroup = TEGRA_SWGROUP_HDA, - .smmu = { - .reg = 0x22c, - .bit = 21, - }, - .la = { - .reg = 0x318, - .shift = 16, - .mask = 0xff, - .def = 0xff, - }, + .regs = { + .smmu = { + .reg = 0x22c, + .bit = 21, + }, + .la = { + .reg = 0x318, + .shift = 16, + .mask = 0xff, + .def = 0xff, + }, + }, + .fifo_size = 16 * 16, }, { .id = 0x36, .name = "host1xw", .swgroup = TEGRA_SWGROUP_HC, - .smmu = { - .reg = 0x22c, - .bit = 22, - }, - .la = { - .reg = 0x314, - .shift = 0, - .mask = 0xff, - .def = 0x10, - }, + .regs = { + .smmu = { + .reg = 0x22c, + .bit = 22, + }, + .la = { + .reg = 0x314, + .shift = 0, + .mask = 0xff, + .def = 0x10, + }, + }, + .fifo_size = 16 * 32, }, { .id = 0x37, .name = "ispw", .swgroup = TEGRA_SWGROUP_ISP, - .smmu = { - .reg = 0x22c, - .bit = 23, - }, - .la = { - .reg = 0x31c, - .shift = 0, - .mask = 0xff, - .def = 0xff, - }, + .regs = { + .smmu = { + .reg = 0x22c, + .bit = 23, + }, + .la = { + .reg = 0x31c, + .shift = 0, + .mask = 0xff, + .def = 0xff, + }, + }, + .fifo_size = 16 * 64, }, { .id = 0x38, .name = "mpcorelpw", .swgroup = TEGRA_SWGROUP_MPCORELP, - .la = { - .reg = 0x324, - .shift = 16, - .mask = 0xff, - .def = 0x0e, + .regs = { + .la = { + .reg = 0x324, + .shift = 16, + .mask = 0xff, + .def = 0x0e, + }, }, + .fifo_size = 16 * 24, }, { .id = 0x39, .name = "mpcorew", .swgroup = TEGRA_SWGROUP_MPCORE, - .la = { - .reg = 0x320, - .shift = 16, - .mask = 0xff, - .def = 0x0e, + .regs = { + .la = { + .reg = 0x320, + .shift = 16, + .mask = 0xff, + .def = 0x0e, + }, }, + .fifo_size = 16 * 24, }, { .id = 0x3a, .name = "mpecswr", .swgroup = TEGRA_SWGROUP_MPE, - .smmu = { - .reg = 0x22c, - .bit = 26, - }, - .la = { - .reg = 0x330, - .shift = 16, - .mask = 0xff, - .def = 0xff, - }, + .regs = { + .smmu = { + .reg = 0x22c, + .bit = 26, + }, + .la = { + .reg = 0x330, + .shift = 16, + .mask = 0xff, + .def = 0xff, + }, + }, + .fifo_size = 16 * 8, }, { .id = 0x3b, .name = "ppcsahbdmaw", .swgroup = TEGRA_SWGROUP_PPCS, - .smmu = { - .reg = 0x22c, - .bit = 27, - }, - .la = { - .reg = 0x348, - .shift = 0, - .mask = 0xff, - .def = 0x10, - }, + .regs = { + .smmu = { + .reg = 0x22c, + .bit = 27, + }, + .la = { + .reg = 0x348, + .shift = 0, + .mask = 0xff, + .def = 0x10, + }, + }, + .fifo_size = 16 * 2, }, { .id = 0x3c, .name = "ppcsahbslvw", .swgroup = TEGRA_SWGROUP_PPCS, - .smmu = { - .reg = 0x22c, - .bit = 28, - }, - .la = { - .reg = 0x348, - .shift = 16, - .mask = 0xff, - .def = 0x06, - }, + .regs = { + .smmu = { + .reg = 0x22c, + .bit = 28, + }, + .la = { + .reg = 0x348, + .shift = 16, + .mask = 0xff, + .def = 0x06, + }, + }, + .fifo_size = 16 * 4, }, { .id = 0x3d, .name = "sataw", .swgroup = TEGRA_SWGROUP_SATA, - .smmu = { - .reg = 0x22c, - .bit = 29, - }, - .la = { - .reg = 0x350, - .shift = 16, - .mask = 0xff, - .def = 0x33, - }, + .regs = { + .smmu = { + .reg = 0x22c, + .bit = 29, + }, + .la = { + .reg = 0x350, + .shift = 16, + .mask = 0xff, + .def = 0x33, + }, + }, + .fifo_size = 16 * 32, }, { .id = 0x3e, .name = "vdebsevw", .swgroup = TEGRA_SWGROUP_VDE, - .smmu = { - .reg = 0x22c, - .bit = 30, - }, - .la = { - .reg = 0x35c, - .shift = 0, - .mask = 0xff, - .def = 0xff, - }, + .regs = { + .smmu = { + .reg = 0x22c, + .bit = 30, + }, + .la = { + .reg = 0x35c, + .shift = 0, + .mask = 0xff, + .def = 0xff, + }, + }, + .fifo_size = 16 * 4, }, { .id = 0x3f, .name = "vdedbgw", .swgroup = TEGRA_SWGROUP_VDE, - .smmu = { - .reg = 0x22c, - .bit = 31, - }, - .la = { - .reg = 0x35c, - .shift = 16, - .mask = 0xff, - .def = 0xff, - }, + .regs = { + .smmu = { + .reg = 0x22c, + .bit = 31, + }, + .la = { + .reg = 0x35c, + .shift = 16, + .mask = 0xff, + .def = 0xff, + }, + }, + .fifo_size = 16 * 16, }, { .id = 0x40, .name = "vdembew", .swgroup = TEGRA_SWGROUP_VDE, - .smmu = { - .reg = 0x230, - .bit = 0, - }, - .la = { - .reg = 0x360, - .shift = 0, - .mask = 0xff, - .def = 0x42, - }, + .regs = { + .smmu = { + .reg = 0x230, + .bit = 0, + }, + .la = { + .reg = 0x360, + .shift = 0, + .mask = 0xff, + .def = 0x42, + }, + }, + .fifo_size = 16 * 2, }, { .id = 0x41, .name = "vdetpmw", .swgroup = TEGRA_SWGROUP_VDE, - .smmu = { - .reg = 0x230, - .bit = 1, - }, - .la = { - .reg = 0x360, - .shift = 16, - .mask = 0xff, - .def = 0x2a, - }, + .regs = { + .smmu = { + .reg = 0x230, + .bit = 1, + }, + .la = { + .reg = 0x360, + .shift = 16, + .mask = 0xff, + .def = 0x2a, + }, + }, + .fifo_size = 16 * 16, }, }; @@ -1011,6 +1216,174 @@ static const struct tegra_mc_reset tegra30_mc_resets[] = { TEGRA30_MC_RESET(VI, 0x200, 0x204, 17), }; +static void tegra30_mc_tune_client_latency(struct tegra_mc *mc, + const struct tegra_mc_client *client, + unsigned int bandwidth_mbytes_sec) +{ + u32 arb_tolerance_compensation_nsec, arb_tolerance_compensation_div; + unsigned int fifo_size = client->fifo_size; + u32 arb_nsec, la_ticks, value; + + /* see 18.4.1 Client Configuration in Tegra3 TRM v03p */ + if (bandwidth_mbytes_sec) + arb_nsec = fifo_size * NSEC_PER_USEC / bandwidth_mbytes_sec; + else + arb_nsec = U32_MAX; + + /* + * Latency allowness should be set with consideration for the module's + * latency tolerance and internal buffering capabilities. + * + * Display memory clients use isochronous transfers and have very low + * tolerance to a belated transfers. Hence we need to compensate the + * memory arbitration imperfection for them in order to prevent FIFO + * underflow condition when memory bus is busy. + * + * VI clients also need a stronger compensation. + */ + switch (client->swgroup) { + case TEGRA_SWGROUP_MPCORE: + case TEGRA_SWGROUP_PTC: + /* + * We always want lower latency for these clients, hence + * don't touch them. + */ + return; + + case TEGRA_SWGROUP_DC: + case TEGRA_SWGROUP_DCB: + arb_tolerance_compensation_nsec = 1050; + arb_tolerance_compensation_div = 2; + break; + + case TEGRA_SWGROUP_VI: + arb_tolerance_compensation_nsec = 1050; + arb_tolerance_compensation_div = 1; + break; + + default: + arb_tolerance_compensation_nsec = 150; + arb_tolerance_compensation_div = 1; + break; + } + + if (arb_nsec > arb_tolerance_compensation_nsec) + arb_nsec -= arb_tolerance_compensation_nsec; + else + arb_nsec = 0; + + arb_nsec /= arb_tolerance_compensation_div; + + /* + * Latency allowance is a number of ticks a request from a particular + * client may wait in the EMEM arbiter before it becomes a high-priority + * request. + */ + la_ticks = arb_nsec / mc->tick; + la_ticks = min(la_ticks, client->regs.la.mask); + + value = mc_readl(mc, client->regs.la.reg); + value &= ~(client->regs.la.mask << client->regs.la.shift); + value |= la_ticks << client->regs.la.shift; + mc_writel(mc, value, client->regs.la.reg); +} + +static int tegra30_mc_icc_set(struct icc_node *src, struct icc_node *dst) +{ + struct tegra_mc *mc = icc_provider_to_tegra_mc(src->provider); + const struct tegra_mc_client *client = &mc->soc->clients[src->id]; + u64 peak_bandwidth = icc_units_to_bps(src->peak_bw); + + /* + * Skip pre-initialization that is done by icc_node_add(), which sets + * bandwidth to maximum for all clients before drivers are loaded. + * + * This doesn't make sense for us because we don't have drivers for all + * clients and it's okay to keep configuration left from bootloader + * during boot, at least for today. + */ + if (src == dst) + return 0; + + /* convert bytes/sec to megabytes/sec */ + do_div(peak_bandwidth, 1000000); + + tegra30_mc_tune_client_latency(mc, client, peak_bandwidth); + + return 0; +} + +static int tegra30_mc_icc_aggreate(struct icc_node *node, u32 tag, u32 avg_bw, + u32 peak_bw, u32 *agg_avg, u32 *agg_peak) +{ + /* + * ISO clients need to reserve extra bandwidth up-front because + * there could be high bandwidth pressure during initial filling + * of the client's FIFO buffers. Secondly, we need to take into + * account impurities of the memory subsystem. + */ + if (tag & TEGRA_MC_ICC_TAG_ISO) + peak_bw = tegra_mc_scale_percents(peak_bw, 400); + + *agg_avg += avg_bw; + *agg_peak = max(*agg_peak, peak_bw); + + return 0; +} + +static struct icc_node_data * +tegra30_mc_of_icc_xlate_extended(struct of_phandle_args *spec, void *data) +{ + struct tegra_mc *mc = icc_provider_to_tegra_mc(data); + const struct tegra_mc_client *client; + unsigned int i, idx = spec->args[0]; + struct icc_node_data *ndata; + struct icc_node *node; + + list_for_each_entry(node, &mc->provider.nodes, node_list) { + if (node->id != idx) + continue; + + ndata = kzalloc(sizeof(*ndata), GFP_KERNEL); + if (!ndata) + return ERR_PTR(-ENOMEM); + + client = &mc->soc->clients[idx]; + ndata->node = node; + + switch (client->swgroup) { + case TEGRA_SWGROUP_DC: + case TEGRA_SWGROUP_DCB: + case TEGRA_SWGROUP_PTC: + case TEGRA_SWGROUP_VI: + /* these clients are isochronous by default */ + ndata->tag = TEGRA_MC_ICC_TAG_ISO; + break; + + default: + ndata->tag = TEGRA_MC_ICC_TAG_DEFAULT; + break; + } + + return ndata; + } + + for (i = 0; i < mc->soc->num_clients; i++) { + if (mc->soc->clients[i].id == idx) + return ERR_PTR(-EPROBE_DEFER); + } + + dev_err(mc->dev, "invalid ICC client ID %u\n", idx); + + return ERR_PTR(-EINVAL); +} + +static const struct tegra_mc_icc_ops tegra30_mc_icc_ops = { + .xlate_extended = tegra30_mc_of_icc_xlate_extended, + .aggregate = tegra30_mc_icc_aggreate, + .set = tegra30_mc_icc_set, +}; + const struct tegra_mc_soc tegra30_mc_soc = { .clients = tegra30_mc_clients, .num_clients = ARRAY_SIZE(tegra30_mc_clients), @@ -1025,4 +1398,6 @@ const struct tegra_mc_soc tegra30_mc_soc = { .reset_ops = &tegra_mc_reset_ops_common, .resets = tegra30_mc_resets, .num_resets = ARRAY_SIZE(tegra30_mc_resets), + .icc_ops = &tegra30_mc_icc_ops, + .ops = &tegra30_mc_ops, }; diff --git a/drivers/memory/ti-aemif.c b/drivers/memory/ti-aemif.c index db526dbf71ee..f81e7df8798a 100644 --- a/drivers/memory/ti-aemif.c +++ b/drivers/memory/ti-aemif.c @@ -27,7 +27,7 @@ #define WSTROBE_SHIFT 20 #define WSETUP_SHIFT 26 #define EW_SHIFT 30 -#define SS_SHIFT 31 +#define SSTROBE_SHIFT 31 #define TA(x) ((x) << TA_SHIFT) #define RHOLD(x) ((x) << RHOLD_SHIFT) @@ -37,7 +37,7 @@ #define WSTROBE(x) ((x) << WSTROBE_SHIFT) #define WSETUP(x) ((x) << WSETUP_SHIFT) #define EW(x) ((x) << EW_SHIFT) -#define SS(x) ((x) << SS_SHIFT) +#define SSTROBE(x) ((x) << SSTROBE_SHIFT) #define ASIZE_MAX 0x1 #define TA_MAX 0x3 @@ -48,7 +48,7 @@ #define WSTROBE_MAX 0x3f #define WSETUP_MAX 0xf #define EW_MAX 0x1 -#define SS_MAX 0x1 +#define SSTROBE_MAX 0x1 #define NUM_CS 4 #define TA_VAL(x) (((x) & TA(TA_MAX)) >> TA_SHIFT) @@ -59,7 +59,7 @@ #define WSTROBE_VAL(x) (((x) & WSTROBE(WSTROBE_MAX)) >> WSTROBE_SHIFT) #define WSETUP_VAL(x) (((x) & WSETUP(WSETUP_MAX)) >> WSETUP_SHIFT) #define EW_VAL(x) (((x) & EW(EW_MAX)) >> EW_SHIFT) -#define SS_VAL(x) (((x) & SS(SS_MAX)) >> SS_SHIFT) +#define SSTROBE_VAL(x) (((x) & SSTROBE(SSTROBE_MAX)) >> SSTROBE_SHIFT) #define NRCSR_OFFSET 0x00 #define AWCCR_OFFSET 0x04 @@ -67,7 +67,7 @@ #define ACR_ASIZE_MASK 0x3 #define ACR_EW_MASK BIT(30) -#define ACR_SS_MASK BIT(31) +#define ACR_SSTROBE_MASK BIT(31) #define ASIZE_16BIT 1 #define CONFIG_MASK (TA(TA_MAX) | \ @@ -77,7 +77,7 @@ WHOLD(WHOLD_MAX) | \ WSTROBE(WSTROBE_MAX) | \ WSETUP(WSETUP_MAX) | \ - EW(EW_MAX) | SS(SS_MAX) | \ + EW(EW_MAX) | SSTROBE(SSTROBE_MAX) | \ ASIZE_MAX) /** @@ -204,7 +204,7 @@ static int aemif_config_abus(struct platform_device *pdev, int csnum) if (data->enable_ew) set |= ACR_EW_MASK; if (data->enable_ss) - set |= ACR_SS_MASK; + set |= ACR_SSTROBE_MASK; val = readl(aemif->base + offset); val &= ~CONFIG_MASK; @@ -246,7 +246,7 @@ static void aemif_get_hw_params(struct platform_device *pdev, int csnum) data->wstrobe = aemif_cycles_to_nsec(WSTROBE_VAL(val), clk_rate); data->wsetup = aemif_cycles_to_nsec(WSETUP_VAL(val), clk_rate); data->enable_ew = EW_VAL(val); - data->enable_ss = SS_VAL(val); + data->enable_ss = SSTROBE_VAL(val); data->asize = val & ASIZE_MAX; } @@ -328,7 +328,6 @@ static int aemif_probe(struct platform_device *pdev) { int i; int ret = -ENODEV; - struct resource *res; struct device *dev = &pdev->dev; struct device_node *np = dev->of_node; struct device_node *child_np; @@ -362,8 +361,7 @@ static int aemif_probe(struct platform_device *pdev) else if (pdata) aemif->cs_offset = pdata->cs_offset; - res = platform_get_resource(pdev, IORESOURCE_MEM, 0); - aemif->base = devm_ioremap_resource(dev, res); + aemif->base = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(aemif->base)) { ret = PTR_ERR(aemif->base); goto error; @@ -378,8 +376,10 @@ static int aemif_probe(struct platform_device *pdev) */ for_each_available_child_of_node(np, child_np) { ret = of_aemif_parse_abus_config(pdev, child_np); - if (ret < 0) + if (ret < 0) { + of_node_put(child_np); goto error; + } } } else if (pdata && pdata->num_abus_data > 0) { for (i = 0; i < pdata->num_abus_data; i++, aemif->num_cs++) { @@ -405,8 +405,10 @@ static int aemif_probe(struct platform_device *pdev) for_each_available_child_of_node(np, child_np) { ret = of_platform_populate(child_np, NULL, dev_lookup, dev); - if (ret < 0) + if (ret < 0) { + of_node_put(child_np); goto error; + } } } else if (pdata) { for (i = 0; i < pdata->num_sub_devices; i++) { diff --git a/drivers/memory/ti-emif-pm.c b/drivers/memory/ti-emif-pm.c index 9c90f815ad3a..31d6266f008c 100644 --- a/drivers/memory/ti-emif-pm.c +++ b/drivers/memory/ti-emif-pm.c @@ -248,7 +248,7 @@ MODULE_DEVICE_TABLE(of, ti_emif_of_match); static int ti_emif_resume(struct device *dev) { unsigned long tmp = - __raw_readl((void *)emif_instance->ti_emif_sram_virt); + __raw_readl((void __iomem *)emif_instance->ti_emif_sram_virt); /* * Check to see if what we are copying is already present in the @@ -290,9 +290,9 @@ static int ti_emif_probe(struct platform_device *pdev) emif_data->pm_data.ti_emif_sram_config = (unsigned long)match->data; - res = platform_get_resource(pdev, IORESOURCE_MEM, 0); - emif_data->pm_data.ti_emif_base_addr_virt = devm_ioremap_resource(dev, - res); + emif_data->pm_data.ti_emif_base_addr_virt = devm_platform_get_and_ioremap_resource(pdev, + 0, + &res); if (IS_ERR(emif_data->pm_data.ti_emif_base_addr_virt)) { ret = PTR_ERR(emif_data->pm_data.ti_emif_base_addr_virt); return ret; @@ -340,7 +340,7 @@ static struct platform_driver ti_emif_driver = { .remove = ti_emif_remove, .driver = { .name = KBUILD_MODNAME, - .of_match_table = of_match_ptr(ti_emif_of_match), + .of_match_table = ti_emif_of_match, .pm = &ti_emif_pm_ops, }, }; diff --git a/drivers/memory/ti-emif-sram-pm.S b/drivers/memory/ti-emif-sram-pm.S index d1c83bd5b98e..9bcac35c3304 100644 --- a/drivers/memory/ti-emif-sram-pm.S +++ b/drivers/memory/ti-emif-sram-pm.S @@ -1,17 +1,9 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ /* * Low level PM code for TI EMIF * * Copyright (C) 2016-2017 Texas Instruments Incorporated - http://www.ti.com/ * Dave Gerlach - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License as - * published by the Free Software Foundation version 2. - * - * This program is distributed "as is" WITHOUT ANY WARRANTY of any - * kind, whether express or implied; without even the implied warranty - * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. */ #include <linux/linkage.h> |