diff options
| author |   Linus Torvalds <torvalds@linux-foundation.org> | 2019-07-11 15:34:05 -0700 |
|---|---|---|
| committer | Linus Torvalds <torvalds@linux-foundation.org> | 2019-07-11 15:34:05 -0700 |
| commit | 97ff4ca46d3279134cec49752de8c5a62dc68460 (patch) | |
| tree | 3b562d5890b2fc358ca55a935395cc7938155406 /drivers/misc | |
| parent | Merge branch 'for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/hid/hid (diff) | |
| parent | Merge tag 'soundwire-5.3-rc1' of git://git.kernel.org/pub/scm/linux/kernel/git/vkoul/soundwire into char-misc-next (diff) | |
| download | linux-dev-97ff4ca46d3279134cec49752de8c5a62dc68460.tar.xz linux-dev-97ff4ca46d3279134cec49752de8c5a62dc68460.zip | |
Merge tag 'char-misc-5.3-rc1' of git://git.kernel.org/pub/scm/linux/kernel/git/gregkh/char-misc
Pull char / misc driver updates from Greg KH:
"Here is the "large" pull request for char and misc and other assorted
smaller driver subsystems for 5.3-rc1.
It seems that this tree is becoming the funnel point of lots of
smaller driver subsystems, which is fine for me, but that's why it is
getting larger over time and does not just contain stuff under
drivers/char/ and drivers/misc.
Lots of small updates all over the place here from different driver
subsystems:
- habana driver updates
- coresight driver updates
- documentation file movements and updates
- Android binder fixes and updates
- extcon driver updates
- google firmware driver updates
- fsi driver updates
- smaller misc and char driver updates
- soundwire driver updates
- nvmem driver updates
- w1 driver fixes
All of these have been in linux-next for a while with no reported
issues"
* tag 'char-misc-5.3-rc1' of git://git.kernel.org/pub/scm/linux/kernel/git/gregkh/char-misc: (188 commits)
coresight: Do not default to CPU0 for missing CPU phandle
dt-bindings: coresight: Change CPU phandle to required property
ocxl: Allow contexts to be attached with a NULL mm
fsi: sbefifo: Don't fail operations when in SBE IPL state
coresight: tmc: Smatch: Fix potential NULL pointer dereference
coresight: etm3x: Smatch: Fix potential NULL pointer dereference
coresight: Potential uninitialized variable in probe()
coresight: etb10: Do not call smp_processor_id from preemptible
coresight: tmc-etf: Do not call smp_processor_id from preemptible
coresight: tmc-etr: alloc_perf_buf: Do not call smp_processor_id from preemptible
coresight: tmc-etr: Do not call smp_processor_id() from preemptible
docs: misc-devices: convert files without extension to ReST
fpga: dfl: fme: align PR buffer size per PR datawidth
fpga: dfl: fme: remove copy_to_user() in ioctl for PR
fpga: dfl-fme-mgr: fix FME_PR_INTFC_ID register address.
intel_th: msu: Start read iterator from a non-empty window
intel_th: msu: Split sgt array and pointer in multiwindow mode
intel_th: msu: Support multipage blocks
intel_th: pci: Add Ice Lake NNPI support
intel_th: msu: Fix single mode with disabled IOMMU
...
Diffstat (limited to 'drivers/misc')
49 files changed, 2265 insertions, 1292 deletions
diff --git a/drivers/misc/Kconfig b/drivers/misc/Kconfig index 85fc77148d19..6abfc8e92fcc 100644 --- a/drivers/misc/Kconfig +++ b/drivers/misc/Kconfig @@ -9,7 +9,6 @@ config SENSORS_LIS3LV02D tristate depends on INPUT select INPUT_POLLDEV - default n config AD525X_DPOT tristate "Analog Devices Digital Potentiometers" @@ -62,7 +61,6 @@ config ATMEL_TCLIB config DUMMY_IRQ tristate "Dummy IRQ handler" - default n ---help--- This module accepts a single 'irq' parameter, which it should register for. The sole purpose of this module is to help with debugging of systems on @@ -118,7 +116,6 @@ config PHANTOM config INTEL_MID_PTI tristate "Parallel Trace Interface for MIPI P1149.7 cJTAG standard" depends on PCI && TTY && (X86_INTEL_MID || COMPILE_TEST) - default n help The PTI (Parallel Trace Interface) driver directs trace data routed from various parts in the system out @@ -194,7 +191,6 @@ config ATMEL_SSC config ENCLOSURE_SERVICES tristate "Enclosure Services" - default n help Provides support for intelligent enclosures (bays which contain storage devices). You also need either a host @@ -218,7 +214,6 @@ config SGI_XP config CS5535_MFGPT tristate "CS5535/CS5536 Geode Multi-Function General Purpose Timer (MFGPT) support" depends on MFD_CS5535 - default n help This driver provides access to MFGPT functionality for other drivers that need timers. MFGPTs are available in the CS5535 and @@ -251,7 +246,6 @@ config CS5535_CLOCK_EVENT_SRC config HP_ILO tristate "Channel interface driver for the HP iLO processor" depends on PCI - default n help The channel interface driver allows applications to communicate with iLO management processors present on HP ProLiant servers. @@ -286,7 +280,6 @@ config QCOM_FASTRPC config SGI_GRU tristate "SGI GRU driver" depends on X86_UV && SMP - default n select MMU_NOTIFIER ---help--- The GRU is a hardware resource located in the system chipset. The GRU @@ -301,7 +294,6 @@ config SGI_GRU config SGI_GRU_DEBUG bool "SGI GRU driver debug" depends on SGI_GRU - default n ---help--- This option enables additional debugging code for the SGI GRU driver. If you are unsure, say N. @@ -359,7 +351,6 @@ config SENSORS_BH1770 config SENSORS_APDS990X tristate "APDS990X combined als and proximity sensors" depends on I2C - default n ---help--- Say Y here if you want to build a driver for Avago APDS990x combined ambient light and proximity sensor chip. @@ -387,7 +378,6 @@ config DS1682 config SPEAR13XX_PCIE_GADGET bool "PCIe gadget support for SPEAr13XX platform" depends on ARCH_SPEAR13XX && BROKEN - default n help This option enables gadget support for PCIe controller. If board file defines any controller as PCIe endpoint then a sysfs @@ -397,6 +387,7 @@ config SPEAR13XX_PCIE_GADGET config VMWARE_BALLOON tristate "VMware Balloon Driver" depends on VMWARE_VMCI && X86 && HYPERVISOR_GUEST + select MEMORY_BALLOON help This is VMware physical memory management driver which acts like a "balloon" that can be inflated to reclaim physical pages @@ -431,15 +422,6 @@ config PCH_PHUB To compile this driver as a module, choose M here: the module will be called pch_phub. -config USB_SWITCH_FSA9480 - tristate "FSA9480 USB Switch" - depends on I2C - help - The FSA9480 is a USB port accessory detector and switch. - The FSA9480 is fully controlled using I2C and enables USB data, - stereo and mono audio, video, microphone and UART data to use - a common connector port. - config LATTICE_ECP3_CONFIG tristate "Lattice ECP3 FPGA bitstream configuration via SPI" depends on SPI && SYSFS @@ -481,6 +463,18 @@ config PCI_ENDPOINT_TEST Enable this configuration option to enable the host side test driver for PCI Endpoint. +config XILINX_SDFEC + tristate "Xilinx SDFEC 16" + help + This option enables support for the Xilinx SDFEC (Soft Decision + Forward Error Correction) driver. This enables a char driver + for the SDFEC. + + You may select this driver if your design instantiates the + SDFEC(16nm) hardened block. To compile this as a module choose M. + + If unsure, say N. + config MISC_RTSX tristate default MISC_RTSX_PCI || MISC_RTSX_USB diff --git a/drivers/misc/Makefile b/drivers/misc/Makefile index b9affcdaa3d6..abd8ae249746 100644 --- a/drivers/misc/Makefile +++ b/drivers/misc/Makefile @@ -42,7 +42,6 @@ obj-$(CONFIG_VMWARE_BALLOON) += vmw_balloon.o obj-$(CONFIG_PCH_PHUB) += pch_phub.o obj-y += ti-st/ obj-y += lis3lv02d/ -obj-$(CONFIG_USB_SWITCH_FSA9480) += fsa9480.o obj-$(CONFIG_ALTERA_STAPL) +=altera-stapl/ obj-$(CONFIG_INTEL_MEI) += mei/ obj-$(CONFIG_VMWARE_VMCI) += vmw_vmci/ @@ -59,3 +58,4 @@ obj-$(CONFIG_OCXL) += ocxl/ obj-y += cardreader/ obj-$(CONFIG_PVPANIC) += pvpanic.o obj-$(CONFIG_HABANA_AI) += habanalabs/ +obj-$(CONFIG_XILINX_SDFEC) += xilinx_sdfec.o diff --git a/drivers/misc/altera-stapl/Kconfig b/drivers/misc/altera-stapl/Kconfig index b34863544365..6c4c6575ec31 100644 --- a/drivers/misc/altera-stapl/Kconfig +++ b/drivers/misc/altera-stapl/Kconfig @@ -5,6 +5,5 @@ comment "Altera FPGA firmware download module (requires I2C)" config ALTERA_STAPL tristate "Altera FPGA firmware download module" depends on I2C - default n help An Altera FPGA module. Say Y when you want to support this tool. diff --git a/drivers/misc/c2port/Kconfig b/drivers/misc/c2port/Kconfig index 192e25094bd4..e20516ffd91e 100644 --- a/drivers/misc/c2port/Kconfig +++ b/drivers/misc/c2port/Kconfig @@ -5,7 +5,6 @@ menuconfig C2PORT tristate "Silicon Labs C2 port support" - default n help This option enables support for Silicon Labs C2 port used to program Silicon micro controller chips (and other 8051 compatible). @@ -24,7 +23,6 @@ if C2PORT config C2PORT_DURAMAR_2150 tristate "C2 port support for Eurotech's Duramar 2150" depends on X86 - default n help This option enables C2 support for the Eurotech's Duramar 2150 on board micro controller. diff --git a/drivers/misc/cb710/Kconfig b/drivers/misc/cb710/Kconfig index 3c7356d55423..a696d7509024 100644 --- a/drivers/misc/cb710/Kconfig +++ b/drivers/misc/cb710/Kconfig @@ -15,7 +15,6 @@ config CB710_CORE config CB710_DEBUG bool "Enable driver debugging" depends on CB710_CORE != n - default n help This is an option for use by developers; most people should say N here. This adds a lot of debugging output to dmesg. diff --git a/drivers/misc/cxl/Kconfig b/drivers/misc/cxl/Kconfig index f1d9a843e361..39eec9031487 100644 --- a/drivers/misc/cxl/Kconfig +++ b/drivers/misc/cxl/Kconfig @@ -5,16 +5,13 @@ config CXL_BASE bool - default n select PPC_COPRO_BASE config CXL_AFU_DRIVER_OPS bool - default n config CXL_LIB bool - default n config CXL tristate "Support for IBM Coherent Accelerators (CXL)" diff --git a/drivers/misc/echo/Kconfig b/drivers/misc/echo/Kconfig index 39656413e70d..be70b263e271 100644 --- a/drivers/misc/echo/Kconfig +++ b/drivers/misc/echo/Kconfig @@ -1,7 +1,6 @@ # SPDX-License-Identifier: GPL-2.0-only config ECHO tristate "Line Echo Canceller support" - default n ---help--- This driver provides line echo cancelling support for mISDN and Zaptel drivers. diff --git a/drivers/misc/eeprom/ee1004.c b/drivers/misc/eeprom/ee1004.c index be3263df278a..6f00c33cfe22 100644 --- a/drivers/misc/eeprom/ee1004.c +++ b/drivers/misc/eeprom/ee1004.c @@ -2,7 +2,7 @@ /* * ee1004 - driver for DDR4 SPD EEPROMs * - * Copyright (C) 2017 Jean Delvare + * Copyright (C) 2017-2019 Jean Delvare * * Based on the at24 driver: * Copyright (C) 2005-2007 David Brownell @@ -53,6 +53,24 @@ MODULE_DEVICE_TABLE(i2c, ee1004_ids); /*-------------------------------------------------------------------------*/ +static int ee1004_get_current_page(void) +{ + int err; + + err = i2c_smbus_read_byte(ee1004_set_page[0]); + if (err == -ENXIO) { + /* Nack means page 1 is selected */ + return 1; + } + if (err < 0) { + /* Anything else is a real error, bail out */ + return err; + } + + /* Ack means page 0 is selected, returned value meaningless */ + return 0; +} + static ssize_t ee1004_eeprom_read(struct i2c_client *client, char *buf, unsigned int offset, size_t count) { @@ -102,6 +120,16 @@ static ssize_t ee1004_read(struct file *filp, struct kobject *kobj, /* Data is ignored */ status = i2c_smbus_write_byte(ee1004_set_page[page], 0x00); + if (status == -ENXIO) { + /* + * Don't give up just yet. Some memory + * modules will select the page but not + * ack the command. Check which page is + * selected now. + */ + if (ee1004_get_current_page() == page) + status = 0; + } if (status < 0) { dev_err(dev, "Failed to select page %d (%d)\n", page, status); @@ -186,17 +214,10 @@ static int ee1004_probe(struct i2c_client *client, } /* Remember current page to avoid unneeded page select */ - err = i2c_smbus_read_byte(ee1004_set_page[0]); - if (err == -ENXIO) { - /* Nack means page 1 is selected */ - ee1004_current_page = 1; - } else if (err < 0) { - /* Anything else is a real error, bail out */ + err = ee1004_get_current_page(); + if (err < 0) goto err_clients; - } else { - /* Ack means page 0 is selected, returned value meaningless */ - ee1004_current_page = 0; - } + ee1004_current_page = err; dev_dbg(&client->dev, "Currently selected page: %d\n", ee1004_current_page); mutex_unlock(&ee1004_bus_lock); diff --git a/drivers/misc/eeprom/idt_89hpesx.c b/drivers/misc/eeprom/idt_89hpesx.c index 8a4659518c33..81c70e5bc168 100644 --- a/drivers/misc/eeprom/idt_89hpesx.c +++ b/drivers/misc/eeprom/idt_89hpesx.c @@ -115,7 +115,6 @@ static struct dentry *csr_dbgdir; * @client: i2c client used to perform IO operations * * @ee_file: EEPROM read/write sysfs-file - * @csr_file: CSR read/write debugfs-node */ struct idt_smb_seq; struct idt_89hpesx_dev { @@ -137,7 +136,6 @@ struct idt_89hpesx_dev { struct bin_attribute *ee_file; struct dentry *csr_dir; - struct dentry *csr_file; }; /* @@ -1378,8 +1376,8 @@ static void idt_create_dbgfs_files(struct idt_89hpesx_dev *pdev) pdev->csr_dir = debugfs_create_dir(fname, csr_dbgdir); /* Create Debugfs file for CSR read/write operations */ - pdev->csr_file = debugfs_create_file(cli->name, 0600, - pdev->csr_dir, pdev, &csr_dbgfs_ops); + debugfs_create_file(cli->name, 0600, pdev->csr_dir, pdev, + &csr_dbgfs_ops); } /* diff --git a/drivers/misc/fsa9480.c b/drivers/misc/fsa9480.c deleted file mode 100644 index fab02f2da077..000000000000 --- a/drivers/misc/fsa9480.c +++ /dev/null @@ -1,547 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * fsa9480.c - FSA9480 micro USB switch device driver - * - * Copyright (C) 2010 Samsung Electronics - * Minkyu Kang <mk7.kang@samsung.com> - * Wonguk Jeong <wonguk.jeong@samsung.com> - */ - -#include <linux/kernel.h> -#include <linux/module.h> -#include <linux/err.h> -#include <linux/i2c.h> -#include <linux/platform_data/fsa9480.h> -#include <linux/irq.h> -#include <linux/interrupt.h> -#include <linux/workqueue.h> -#include <linux/platform_device.h> -#include <linux/slab.h> -#include <linux/pm_runtime.h> - -/* FSA9480 I2C registers */ -#define FSA9480_REG_DEVID 0x01 -#define FSA9480_REG_CTRL 0x02 -#define FSA9480_REG_INT1 0x03 -#define FSA9480_REG_INT2 0x04 -#define FSA9480_REG_INT1_MASK 0x05 -#define FSA9480_REG_INT2_MASK 0x06 -#define FSA9480_REG_ADC 0x07 -#define FSA9480_REG_TIMING1 0x08 -#define FSA9480_REG_TIMING2 0x09 -#define FSA9480_REG_DEV_T1 0x0a -#define FSA9480_REG_DEV_T2 0x0b -#define FSA9480_REG_BTN1 0x0c -#define FSA9480_REG_BTN2 0x0d -#define FSA9480_REG_CK 0x0e -#define FSA9480_REG_CK_INT1 0x0f -#define FSA9480_REG_CK_INT2 0x10 -#define FSA9480_REG_CK_INTMASK1 0x11 -#define FSA9480_REG_CK_INTMASK2 0x12 -#define FSA9480_REG_MANSW1 0x13 -#define FSA9480_REG_MANSW2 0x14 - -/* Control */ -#define CON_SWITCH_OPEN (1 << 4) -#define CON_RAW_DATA (1 << 3) -#define CON_MANUAL_SW (1 << 2) -#define CON_WAIT (1 << 1) -#define CON_INT_MASK (1 << 0) -#define CON_MASK (CON_SWITCH_OPEN | CON_RAW_DATA | \ - CON_MANUAL_SW | CON_WAIT) - -/* Device Type 1 */ -#define DEV_USB_OTG (1 << 7) -#define DEV_DEDICATED_CHG (1 << 6) -#define DEV_USB_CHG (1 << 5) -#define DEV_CAR_KIT (1 << 4) -#define DEV_UART (1 << 3) -#define DEV_USB (1 << 2) -#define DEV_AUDIO_2 (1 << 1) -#define DEV_AUDIO_1 (1 << 0) - -#define DEV_T1_USB_MASK (DEV_USB_OTG | DEV_USB) -#define DEV_T1_UART_MASK (DEV_UART) -#define DEV_T1_CHARGER_MASK (DEV_DEDICATED_CHG | DEV_USB_CHG) - -/* Device Type 2 */ -#define DEV_AV (1 << 6) -#define DEV_TTY (1 << 5) -#define DEV_PPD (1 << 4) -#define DEV_JIG_UART_OFF (1 << 3) -#define DEV_JIG_UART_ON (1 << 2) -#define DEV_JIG_USB_OFF (1 << 1) -#define DEV_JIG_USB_ON (1 << 0) - -#define DEV_T2_USB_MASK (DEV_JIG_USB_OFF | DEV_JIG_USB_ON) -#define DEV_T2_UART_MASK (DEV_JIG_UART_OFF | DEV_JIG_UART_ON) -#define DEV_T2_JIG_MASK (DEV_JIG_USB_OFF | DEV_JIG_USB_ON | \ - DEV_JIG_UART_OFF | DEV_JIG_UART_ON) - -/* - * Manual Switch - * D- [7:5] / D+ [4:2] - * 000: Open all / 001: USB / 010: AUDIO / 011: UART / 100: V_AUDIO - */ -#define SW_VAUDIO ((4 << 5) | (4 << 2)) -#define SW_UART ((3 << 5) | (3 << 2)) -#define SW_AUDIO ((2 << 5) | (2 << 2)) -#define SW_DHOST ((1 << 5) | (1 << 2)) -#define SW_AUTO ((0 << 5) | (0 << 2)) - -/* Interrupt 1 */ -#define INT_DETACH (1 << 1) -#define INT_ATTACH (1 << 0) - -struct fsa9480_usbsw { - struct i2c_client *client; - struct fsa9480_platform_data *pdata; - int dev1; - int dev2; - int mansw; -}; - -static struct fsa9480_usbsw *chip; - -static int fsa9480_write_reg(struct i2c_client *client, - int reg, int value) -{ - int ret; - - ret = i2c_smbus_write_byte_data(client, reg, value); - - if (ret < 0) - dev_err(&client->dev, "%s: err %d\n", __func__, ret); - - return ret; -} - -static int fsa9480_read_reg(struct i2c_client *client, int reg) -{ - int ret; - - ret = i2c_smbus_read_byte_data(client, reg); - - if (ret < 0) - dev_err(&client->dev, "%s: err %d\n", __func__, ret); - - return ret; -} - -static int fsa9480_read_irq(struct i2c_client *client, int *value) -{ - int ret; - - ret = i2c_smbus_read_i2c_block_data(client, - FSA9480_REG_INT1, 2, (u8 *)value); - *value &= 0xffff; - - if (ret < 0) - dev_err(&client->dev, "%s: err %d\n", __func__, ret); - - return ret; -} - -static void fsa9480_set_switch(const char *buf) -{ - struct fsa9480_usbsw *usbsw = chip; - struct i2c_client *client = usbsw->client; - unsigned int value; - unsigned int path = 0; - - value = fsa9480_read_reg(client, FSA9480_REG_CTRL); - - if (!strncmp(buf, "VAUDIO", 6)) { - path = SW_VAUDIO; - value &= ~CON_MANUAL_SW; - } else if (!strncmp(buf, "UART", 4)) { - path = SW_UART; - value &= ~CON_MANUAL_SW; - } else if (!strncmp(buf, "AUDIO", 5)) { - path = SW_AUDIO; - value &= ~CON_MANUAL_SW; - } else if (!strncmp(buf, "DHOST", 5)) { - path = SW_DHOST; - value &= ~CON_MANUAL_SW; - } else if (!strncmp(buf, "AUTO", 4)) { - path = SW_AUTO; - value |= CON_MANUAL_SW; - } else { - printk(KERN_ERR "Wrong command\n"); - return; - } - - usbsw->mansw = path; - fsa9480_write_reg(client, FSA9480_REG_MANSW1, path); - fsa9480_write_reg(client, FSA9480_REG_CTRL, value); -} - -static ssize_t fsa9480_get_switch(char *buf) -{ - struct fsa9480_usbsw *usbsw = chip; - struct i2c_client *client = usbsw->client; - unsigned int value; - - value = fsa9480_read_reg(client, FSA9480_REG_MANSW1); - - if (value == SW_VAUDIO) - return sprintf(buf, "VAUDIO\n"); - else if (value == SW_UART) - return sprintf(buf, "UART\n"); - else if (value == SW_AUDIO) - return sprintf(buf, "AUDIO\n"); - else if (value == SW_DHOST) - return sprintf(buf, "DHOST\n"); - else if (value == SW_AUTO) - return sprintf(buf, "AUTO\n"); - else - return sprintf(buf, "%x", value); -} - -static ssize_t fsa9480_show_device(struct device *dev, - struct device_attribute *attr, - char *buf) -{ - struct fsa9480_usbsw *usbsw = dev_get_drvdata(dev); - struct i2c_client *client = usbsw->client; - int dev1, dev2; - - dev1 = fsa9480_read_reg(client, FSA9480_REG_DEV_T1); - dev2 = fsa9480_read_reg(client, FSA9480_REG_DEV_T2); - - if (!dev1 && !dev2) - return sprintf(buf, "NONE\n"); - - /* USB */ - if (dev1 & DEV_T1_USB_MASK || dev2 & DEV_T2_USB_MASK) - return sprintf(buf, "USB\n"); - - /* UART */ - if (dev1 & DEV_T1_UART_MASK || dev2 & DEV_T2_UART_MASK) - return sprintf(buf, "UART\n"); - - /* CHARGER */ - if (dev1 & DEV_T1_CHARGER_MASK) - return sprintf(buf, "CHARGER\n"); - - /* JIG */ - if (dev2 & DEV_T2_JIG_MASK) - return sprintf(buf, "JIG\n"); - - return sprintf(buf, "UNKNOWN\n"); -} - -static ssize_t fsa9480_show_manualsw(struct device *dev, - struct device_attribute *attr, char *buf) -{ - return fsa9480_get_switch(buf); - -} - -static ssize_t fsa9480_set_manualsw(struct device *dev, - struct device_attribute *attr, - const char *buf, size_t count) -{ - fsa9480_set_switch(buf); - - return count; -} - -static DEVICE_ATTR(device, S_IRUGO, fsa9480_show_device, NULL); -static DEVICE_ATTR(switch, S_IRUGO | S_IWUSR, - fsa9480_show_manualsw, fsa9480_set_manualsw); - -static struct attribute *fsa9480_attributes[] = { - &dev_attr_device.attr, - &dev_attr_switch.attr, - NULL -}; - -static const struct attribute_group fsa9480_group = { - .attrs = fsa9480_attributes, -}; - -static void fsa9480_detect_dev(struct fsa9480_usbsw *usbsw, int intr) -{ - int val1, val2, ctrl; - struct fsa9480_platform_data *pdata = usbsw->pdata; - struct i2c_client *client = usbsw->client; - - val1 = fsa9480_read_reg(client, FSA9480_REG_DEV_T1); - val2 = fsa9480_read_reg(client, FSA9480_REG_DEV_T2); - ctrl = fsa9480_read_reg(client, FSA9480_REG_CTRL); - - dev_info(&client->dev, "intr: 0x%x, dev1: 0x%x, dev2: 0x%x\n", - intr, val1, val2); - - if (!intr) - goto out; - - if (intr & INT_ATTACH) { /* Attached */ - /* USB */ - if (val1 & DEV_T1_USB_MASK || val2 & DEV_T2_USB_MASK) { - if (pdata->usb_cb) - pdata->usb_cb(FSA9480_ATTACHED); - - if (usbsw->mansw) { - fsa9480_write_reg(client, - FSA9480_REG_MANSW1, usbsw->mansw); - } - } - - /* UART */ - if (val1 & DEV_T1_UART_MASK || val2 & DEV_T2_UART_MASK) { - if (pdata->uart_cb) - pdata->uart_cb(FSA9480_ATTACHED); - - if (!(ctrl & CON_MANUAL_SW)) { - fsa9480_write_reg(client, - FSA9480_REG_MANSW1, SW_UART); - } - } - - /* CHARGER */ - if (val1 & DEV_T1_CHARGER_MASK) { - if (pdata->charger_cb) - pdata->charger_cb(FSA9480_ATTACHED); - } - - /* JIG */ - if (val2 & DEV_T2_JIG_MASK) { - if (pdata->jig_cb) - pdata->jig_cb(FSA9480_ATTACHED); - } - } else if (intr & INT_DETACH) { /* Detached */ - /* USB */ - if (usbsw->dev1 & DEV_T1_USB_MASK || - usbsw->dev2 & DEV_T2_USB_MASK) { - if (pdata->usb_cb) - pdata->usb_cb(FSA9480_DETACHED); - } - - /* UART */ - if (usbsw->dev1 & DEV_T1_UART_MASK || - usbsw->dev2 & DEV_T2_UART_MASK) { - if (pdata->uart_cb) - pdata->uart_cb(FSA9480_DETACHED); - } - - /* CHARGER */ - if (usbsw->dev1 & DEV_T1_CHARGER_MASK) { - if (pdata->charger_cb) - pdata->charger_cb(FSA9480_DETACHED); - } - - /* JIG */ - if (usbsw->dev2 & DEV_T2_JIG_MASK) { - if (pdata->jig_cb) - pdata->jig_cb(FSA9480_DETACHED); - } - } - - usbsw->dev1 = val1; - usbsw->dev2 = val2; - -out: - ctrl &= ~CON_INT_MASK; - fsa9480_write_reg(client, FSA9480_REG_CTRL, ctrl); -} - -static irqreturn_t fsa9480_irq_handler(int irq, void *data) -{ - struct fsa9480_usbsw *usbsw = data; - struct i2c_client *client = usbsw->client; - int intr; - - /* clear interrupt */ - fsa9480_read_irq(client, &intr); - - /* device detection */ - fsa9480_detect_dev(usbsw, intr); - - return IRQ_HANDLED; -} - -static int fsa9480_irq_init(struct fsa9480_usbsw *usbsw) -{ - struct fsa9480_platform_data *pdata = usbsw->pdata; - struct i2c_client *client = usbsw->client; - int ret; - int intr; - unsigned int ctrl = CON_MASK; - - /* clear interrupt */ - fsa9480_read_irq(client, &intr); - - /* unmask interrupt (attach/detach only) */ - fsa9480_write_reg(client, FSA9480_REG_INT1_MASK, 0xfc); - fsa9480_write_reg(client, FSA9480_REG_INT2_MASK, 0x1f); - - usbsw->mansw = fsa9480_read_reg(client, FSA9480_REG_MANSW1); - - if (usbsw->mansw) - ctrl &= ~CON_MANUAL_SW; /* Manual Switching Mode */ - - fsa9480_write_reg(client, FSA9480_REG_CTRL, ctrl); - - if (pdata && pdata->cfg_gpio) - pdata->cfg_gpio(); - - if (client->irq) { - ret = request_threaded_irq(client->irq, NULL, - fsa9480_irq_handler, - IRQF_TRIGGER_FALLING | IRQF_ONESHOT, - "fsa9480 micro USB", usbsw); - if (ret) { - dev_err(&client->dev, "failed to request IRQ\n"); - return ret; - } - - if (pdata) - device_init_wakeup(&client->dev, pdata->wakeup); - } - - return 0; -} - -static int fsa9480_probe(struct i2c_client *client, - const struct i2c_device_id *id) -{ - struct i2c_adapter *adapter = to_i2c_adapter(client->dev.parent); - struct fsa9480_usbsw *usbsw; - int ret = 0; - - if (!i2c_check_functionality(adapter, I2C_FUNC_SMBUS_BYTE_DATA)) - return -EIO; - - usbsw = kzalloc(sizeof(struct fsa9480_usbsw), GFP_KERNEL); - if (!usbsw) { - dev_err(&client->dev, "failed to allocate driver data\n"); - return -ENOMEM; - } - - usbsw->client = client; - usbsw->pdata = client->dev.platform_data; - - chip = usbsw; - - i2c_set_clientdata(client, usbsw); - - ret = fsa9480_irq_init(usbsw); - if (ret) - goto fail1; - - ret = sysfs_create_group(&client->dev.kobj, &fsa9480_group); - if (ret) { - dev_err(&client->dev, - "failed to create fsa9480 attribute group\n"); - goto fail2; - } - - /* ADC Detect Time: 500ms */ - fsa9480_write_reg(client, FSA9480_REG_TIMING1, 0x6); - - if (chip->pdata->reset_cb) - chip->pdata->reset_cb(); - - /* device detection */ - fsa9480_detect_dev(usbsw, INT_ATTACH); - - pm_runtime_set_active(&client->dev); - - return 0; - -fail2: - if (client->irq) - free_irq(client->irq, usbsw); -fail1: - kfree(usbsw); - return ret; -} - -static int fsa9480_remove(struct i2c_client *client) -{ - struct fsa9480_usbsw *usbsw = i2c_get_clientdata(client); - - if (client->irq) - free_irq(client->irq, usbsw); - - sysfs_remove_group(&client->dev.kobj, &fsa9480_group); - device_init_wakeup(&client->dev, 0); - kfree(usbsw); - return 0; -} - -#ifdef CONFIG_PM_SLEEP - -static int fsa9480_suspend(struct device *dev) -{ - struct i2c_client *client = to_i2c_client(dev); - struct fsa9480_usbsw *usbsw = i2c_get_clientdata(client); - struct fsa9480_platform_data *pdata = usbsw->pdata; - - if (device_may_wakeup(&client->dev) && client->irq) - enable_irq_wake(client->irq); - - if (pdata->usb_power) - pdata->usb_power(0); - - return 0; -} - -static int fsa9480_resume(struct device *dev) -{ - struct i2c_client *client = to_i2c_client(dev); - struct fsa9480_usbsw *usbsw = i2c_get_clientdata(client); - int dev1, dev2; - - if (device_may_wakeup(&client->dev) && client->irq) - disable_irq_wake(client->irq); - - /* - * Clear Pending interrupt. Note that detect_dev does what - * the interrupt handler does. So, we don't miss pending and - * we reenable interrupt if there is one. - */ - fsa9480_read_reg(client, FSA9480_REG_INT1); - fsa9480_read_reg(client, FSA9480_REG_INT2); - - dev1 = fsa9480_read_reg(client, FSA9480_REG_DEV_T1); - dev2 = fsa9480_read_reg(client, FSA9480_REG_DEV_T2); - - /* device detection */ - fsa9480_detect_dev(usbsw, (dev1 || dev2) ? INT_ATTACH : INT_DETACH); - - return 0; -} - -static SIMPLE_DEV_PM_OPS(fsa9480_pm_ops, fsa9480_suspend, fsa9480_resume); -#define FSA9480_PM_OPS (&fsa9480_pm_ops) - -#else - -#define FSA9480_PM_OPS NULL - -#endif /* CONFIG_PM_SLEEP */ - -static const struct i2c_device_id fsa9480_id[] = { - {"fsa9480", 0}, - {} -}; -MODULE_DEVICE_TABLE(i2c, fsa9480_id); - -static struct i2c_driver fsa9480_i2c_driver = { - .driver = { - .name = "fsa9480", - .pm = FSA9480_PM_OPS, - }, - .probe = fsa9480_probe, - .remove = fsa9480_remove, - .id_table = fsa9480_id, -}; - -module_i2c_driver(fsa9480_i2c_driver); - -MODULE_AUTHOR("Minkyu Kang <mk7.kang@samsung.com>"); -MODULE_DESCRIPTION("FSA9480 USB Switch driver"); -MODULE_LICENSE("GPL"); diff --git a/drivers/misc/genwqe/Kconfig b/drivers/misc/genwqe/Kconfig index a8a608713d26..97f64bcf9fe0 100644 --- a/drivers/misc/genwqe/Kconfig +++ b/drivers/misc/genwqe/Kconfig @@ -7,7 +7,6 @@ menuconfig GENWQE tristate "GenWQE PCIe Accelerator" depends on PCI && 64BIT select CRC_ITU_T - default n help Enables PCIe card driver for IBM GenWQE accelerators. The user-space interface is described in diff --git a/drivers/misc/habanalabs/asid.c b/drivers/misc/habanalabs/asid.c index f54e7971a762..2c01461701a3 100644 --- a/drivers/misc/habanalabs/asid.c +++ b/drivers/misc/habanalabs/asid.c @@ -18,7 +18,7 @@ int hl_asid_init(struct hl_device *hdev) mutex_init(&hdev->asid_mutex); - /* ASID 0 is reserved for KMD */ + /* ASID 0 is reserved for KMD and device CPU */ set_bit(0, hdev->asid_bitmap); return 0; diff --git a/drivers/misc/habanalabs/command_submission.c b/drivers/misc/habanalabs/command_submission.c index 6fe785e26859..6ad83d5ef4b0 100644 --- a/drivers/misc/habanalabs/command_submission.c +++ b/drivers/misc/habanalabs/command_submission.c @@ -682,14 +682,12 @@ int hl_cs_ioctl(struct hl_fpriv *hpriv, void *data) u32 tmp; rc = hl_poll_timeout_memory(hdev, - (u64) (uintptr_t) &ctx->thread_ctx_switch_wait_token, - jiffies_to_usecs(hdev->timeout_jiffies), - &tmp); + &ctx->thread_ctx_switch_wait_token, tmp, (tmp == 1), + 100, jiffies_to_usecs(hdev->timeout_jiffies)); - if (rc || !tmp) { + if (rc == -ETIMEDOUT) { dev_err(hdev->dev, - "context switch phase didn't finish in time\n"); - rc = -ETIMEDOUT; + "context switch phase timeout (%d)\n", tmp); goto out; } } diff --git a/drivers/misc/habanalabs/context.c b/drivers/misc/habanalabs/context.c index f4c92f110a72..8682590e3f6e 100644 --- a/drivers/misc/habanalabs/context.c +++ b/drivers/misc/habanalabs/context.c @@ -31,9 +31,13 @@ static void hl_ctx_fini(struct hl_ctx *ctx) * Coresight might be still working by accessing addresses * related to the stopped engines. Hence stop it explicitly. */ - hdev->asic_funcs->halt_coresight(hdev); + if (hdev->in_debug) + hl_device_set_debug_mode(hdev, false); + hl_vm_ctx_fini(ctx); hl_asid_free(hdev, ctx->asid); + } else { + hl_mmu_ctx_fini(ctx); } } @@ -117,6 +121,11 @@ int hl_ctx_init(struct hl_device *hdev, struct hl_ctx *ctx, bool is_kernel_ctx) if (is_kernel_ctx) { ctx->asid = HL_KERNEL_ASID_ID; /* KMD gets ASID 0 */ + rc = hl_mmu_ctx_init(ctx); + if (rc) { + dev_err(hdev->dev, "Failed to init mmu ctx module\n"); + goto mem_ctx_err; + } } else { ctx->asid = hl_asid_alloc(hdev); if (!ctx->asid) { diff --git a/drivers/misc/habanalabs/debugfs.c b/drivers/misc/habanalabs/debugfs.c index ba418aaa404c..18e499c900c7 100644 --- a/drivers/misc/habanalabs/debugfs.c +++ b/drivers/misc/habanalabs/debugfs.c @@ -355,7 +355,7 @@ static int mmu_show(struct seq_file *s, void *data) struct hl_debugfs_entry *entry = s->private; struct hl_dbg_device_entry *dev_entry = entry->dev_entry; struct hl_device *hdev = dev_entry->hdev; - struct hl_ctx *ctx = hdev->user_ctx; + struct hl_ctx *ctx; u64 hop0_addr = 0, hop0_pte_addr = 0, hop0_pte = 0, hop1_addr = 0, hop1_pte_addr = 0, hop1_pte = 0, @@ -367,6 +367,11 @@ static int mmu_show(struct seq_file *s, void *data) if (!hdev->mmu_enable) return 0; + if (dev_entry->mmu_asid == HL_KERNEL_ASID_ID) + ctx = hdev->kernel_ctx; + else + ctx = hdev->user_ctx; + if (!ctx) { dev_err(hdev->dev, "no ctx available\n"); return 0; @@ -495,6 +500,36 @@ err: return -EINVAL; } +static int engines_show(struct seq_file *s, void *data) +{ + struct hl_debugfs_entry *entry = s->private; + struct hl_dbg_device_entry *dev_entry = entry->dev_entry; + struct hl_device *hdev = dev_entry->hdev; + + hdev->asic_funcs->is_device_idle(hdev, NULL, s); + + return 0; +} + +static bool hl_is_device_va(struct hl_device *hdev, u64 addr) +{ + struct asic_fixed_properties *prop = &hdev->asic_prop; + + if (!hdev->mmu_enable) + goto out; + + if (hdev->dram_supports_virtual_memory && + addr >= prop->va_space_dram_start_address && + addr < prop->va_space_dram_end_address) + return true; + + if (addr >= prop->va_space_host_start_address && + addr < prop->va_space_host_end_address) + return true; +out: + return false; +} + static int device_va_to_pa(struct hl_device *hdev, u64 virt_addr, u64 *phys_addr) { @@ -568,7 +603,6 @@ static ssize_t hl_data_read32(struct file *f, char __user *buf, { struct hl_dbg_device_entry *entry = file_inode(f)->i_private; struct hl_device *hdev = entry->hdev; - struct asic_fixed_properties *prop = &hdev->asic_prop; char tmp_buf[32]; u64 addr = entry->addr; u32 val; @@ -577,11 +611,8 @@ static ssize_t hl_data_read32(struct file *f, char __user *buf, if (*ppos) return 0; - if (addr >= prop->va_space_dram_start_address && - addr < prop->va_space_dram_end_address && - hdev->mmu_enable && - hdev->dram_supports_virtual_memory) { - rc = device_va_to_pa(hdev, entry->addr, &addr); + if (hl_is_device_va(hdev, addr)) { + rc = device_va_to_pa(hdev, addr, &addr); if (rc) return rc; } @@ -602,7 +633,6 @@ static ssize_t hl_data_write32(struct file *f, const char __user *buf, { struct hl_dbg_device_entry *entry = file_inode(f)->i_private; struct hl_device *hdev = entry->hdev; - struct asic_fixed_properties *prop = &hdev->asic_prop; u64 addr = entry->addr; u32 value; ssize_t rc; @@ -611,11 +641,8 @@ static ssize_t hl_data_write32(struct file *f, const char __user *buf, if (rc) return rc; - if (addr >= prop->va_space_dram_start_address && - addr < prop->va_space_dram_end_address && - hdev->mmu_enable && - hdev->dram_supports_virtual_memory) { - rc = device_va_to_pa(hdev, entry->addr, &addr); + if (hl_is_device_va(hdev, addr)) { + rc = device_va_to_pa(hdev, addr, &addr); if (rc) return rc; } @@ -877,6 +904,7 @@ static const struct hl_info_list hl_debugfs_list[] = { {"userptr", userptr_show, NULL}, {"vm", vm_show, NULL}, {"mmu", mmu_show, mmu_write}, + {"engines", engines_show, NULL} }; static int hl_debugfs_open(struct inode *inode, struct file *file) diff --git a/drivers/misc/habanalabs/device.c b/drivers/misc/habanalabs/device.c index 0b19d3eefb98..0c4894dd9c02 100644 --- a/drivers/misc/habanalabs/device.c +++ b/drivers/misc/habanalabs/device.c @@ -231,6 +231,7 @@ static int device_early_init(struct hl_device *hdev) mutex_init(&hdev->fd_open_cnt_lock); mutex_init(&hdev->send_cpu_message_lock); + mutex_init(&hdev->debug_lock); mutex_init(&hdev->mmu_cache_lock); INIT_LIST_HEAD(&hdev->hw_queues_mirror_list); spin_lock_init(&hdev->hw_queues_mirror_lock); @@ -262,6 +263,7 @@ early_fini: static void device_early_fini(struct hl_device *hdev) { mutex_destroy(&hdev->mmu_cache_lock); + mutex_destroy(&hdev->debug_lock); mutex_destroy(&hdev->send_cpu_message_lock); hl_cb_mgr_fini(hdev, &hdev->kernel_cb_mgr); @@ -324,7 +326,15 @@ static int device_late_init(struct hl_device *hdev) { int rc; - INIT_DELAYED_WORK(&hdev->work_freq, set_freq_to_low_job); + if (hdev->asic_funcs->late_init) { + rc = hdev->asic_funcs->late_init(hdev); + if (rc) { + dev_err(hdev->dev, + "failed late initialization for the H/W\n"); + return rc; + } + } + hdev->high_pll = hdev->asic_prop.high_pll; /* force setting to low frequency */ @@ -335,17 +345,9 @@ static int device_late_init(struct hl_device *hdev) else hdev->asic_funcs->set_pll_profile(hdev, PLL_LAST); - if (hdev->asic_funcs->late_init) { - rc = hdev->asic_funcs->late_init(hdev); - if (rc) { - dev_err(hdev->dev, - "failed late initialization for the H/W\n"); - return rc; - } - } - + INIT_DELAYED_WORK(&hdev->work_freq, set_freq_to_low_job); schedule_delayed_work(&hdev->work_freq, - usecs_to_jiffies(HL_PLL_LOW_JOB_FREQ_USEC)); + usecs_to_jiffies(HL_PLL_LOW_JOB_FREQ_USEC)); if (hdev->heartbeat) { INIT_DELAYED_WORK(&hdev->work_heartbeat, hl_device_heartbeat); @@ -420,6 +422,52 @@ int hl_device_set_frequency(struct hl_device *hdev, enum hl_pll_frequency freq) return 1; } +int hl_device_set_debug_mode(struct hl_device *hdev, bool enable) +{ + int rc = 0; + + mutex_lock(&hdev->debug_lock); + + if (!enable) { + if (!hdev->in_debug) { + dev_err(hdev->dev, + "Failed to disable debug mode because device was not in debug mode\n"); + rc = -EFAULT; + goto out; + } + + hdev->asic_funcs->halt_coresight(hdev); + hdev->in_debug = 0; + + goto out; + } + + if (hdev->in_debug) { + dev_err(hdev->dev, + "Failed to enable debug mode because device is already in debug mode\n"); + rc = -EFAULT; + goto out; + } + + mutex_lock(&hdev->fd_open_cnt_lock); + + if (atomic_read(&hdev->fd_open_cnt) > 1) { + dev_err(hdev->dev, + "Failed to enable debug mode. More then a single user is using the device\n"); + rc = -EPERM; + goto unlock_fd_open_lock; + } + + hdev->in_debug = 1; + +unlock_fd_open_lock: + mutex_unlock(&hdev->fd_open_cnt_lock); +out: + mutex_unlock(&hdev->debug_lock); + + return rc; +} + /* * hl_device_suspend - initiate device suspend * @@ -647,13 +695,6 @@ again: hdev->hard_reset_pending = true; - if (!hdev->pdev) { - dev_err(hdev->dev, - "Reset action is NOT supported in simulator\n"); - rc = -EINVAL; - goto out_err; - } - device_reset_work = kzalloc(sizeof(*device_reset_work), GFP_ATOMIC); if (!device_reset_work) { @@ -704,6 +745,7 @@ again: if (hard_reset) { hl_vm_fini(hdev); + hl_mmu_fini(hdev); hl_eq_reset(hdev, &hdev->event_queue); } @@ -731,6 +773,13 @@ again: goto out_err; } + rc = hl_mmu_init(hdev); + if (rc) { + dev_err(hdev->dev, + "Failed to initialize MMU S/W after hard reset\n"); + goto out_err; + } + /* Allocate the kernel context */ hdev->kernel_ctx = kzalloc(sizeof(*hdev->kernel_ctx), GFP_KERNEL); @@ -902,11 +951,18 @@ int hl_device_init(struct hl_device *hdev, struct class *hclass) goto cq_fini; } + /* MMU S/W must be initialized before kernel context is created */ + rc = hl_mmu_init(hdev); + if (rc) { + dev_err(hdev->dev, "Failed to initialize MMU S/W structures\n"); + goto eq_fini; + } + /* Allocate the kernel context */ hdev->kernel_ctx = kzalloc(sizeof(*hdev->kernel_ctx), GFP_KERNEL); if (!hdev->kernel_ctx) { rc = -ENOMEM; - goto eq_fini; + goto mmu_fini; } hdev->user_ctx = NULL; @@ -954,8 +1010,6 @@ int hl_device_init(struct hl_device *hdev, struct class *hclass) goto out_disabled; } - /* After test_queues, KMD can start sending messages to device CPU */ - rc = device_late_init(hdev); if (rc) { dev_err(hdev->dev, "Failed late initialization\n"); @@ -1001,6 +1055,8 @@ release_ctx: "kernel ctx is still alive on initialization failure\n"); free_ctx: kfree(hdev->kernel_ctx); +mmu_fini: + hl_mmu_fini(hdev); eq_fini: hl_eq_fini(hdev, &hdev->event_queue); cq_fini: @@ -1105,6 +1161,8 @@ void hl_device_fini(struct hl_device *hdev) hl_vm_fini(hdev); + hl_mmu_fini(hdev); + hl_eq_fini(hdev, &hdev->event_queue); for (i = 0 ; i < hdev->asic_prop.completion_queues_count ; i++) @@ -1126,95 +1184,6 @@ void hl_device_fini(struct hl_device *hdev) } /* - * hl_poll_timeout_memory - Periodically poll a host memory address - * until it is not zero or a timeout occurs - * @hdev: pointer to habanalabs device structure - * @addr: Address to poll - * @timeout_us: timeout in us - * @val: Variable to read the value into - * - * Returns 0 on success and -ETIMEDOUT upon a timeout. In either - * case, the last read value at @addr is stored in @val. Must not - * be called from atomic context if sleep_us or timeout_us are used. - * - * The function sleeps for 100us with timeout value of - * timeout_us - */ -int hl_poll_timeout_memory(struct hl_device *hdev, u64 addr, - u32 timeout_us, u32 *val) -{ - /* - * address in this function points always to a memory location in the - * host's (server's) memory. That location is updated asynchronously - * either by the direct access of the device or by another core - */ - u32 *paddr = (u32 *) (uintptr_t) addr; - ktime_t timeout; - - /* timeout should be longer when working with simulator */ - if (!hdev->pdev) - timeout_us *= 10; - - timeout = ktime_add_us(ktime_get(), timeout_us); - - might_sleep(); - - for (;;) { - /* - * Flush CPU read/write buffers to make sure we read updates - * done by other cores or by the device - */ - mb(); - *val = *paddr; - if (*val) - break; - if (ktime_compare(ktime_get(), timeout) > 0) { - *val = *paddr; - break; - } - usleep_range((100 >> 2) + 1, 100); - } - - return *val ? 0 : -ETIMEDOUT; -} - -/* - * hl_poll_timeout_devicememory - Periodically poll a device memory address - * until it is not zero or a timeout occurs - * @hdev: pointer to habanalabs device structure - * @addr: Device address to poll - * @timeout_us: timeout in us - * @val: Variable to read the value into - * - * Returns 0 on success and -ETIMEDOUT upon a timeout. In either - * case, the last read value at @addr is stored in @val. Must not - * be called from atomic context if sleep_us or timeout_us are used. - * - * The function sleeps for 100us with timeout value of - * timeout_us - */ -int hl_poll_timeout_device_memory(struct hl_device *hdev, void __iomem *addr, - u32 timeout_us, u32 *val) -{ - ktime_t timeout = ktime_add_us(ktime_get(), timeout_us); - - might_sleep(); - - for (;;) { - *val = readl(addr); - if (*val) - break; - if (ktime_compare(ktime_get(), timeout) > 0) { - *val = readl(addr); - break; - } - usleep_range((100 >> 2) + 1, 100); - } - - return *val ? 0 : -ETIMEDOUT; -} - -/* * MMIO register access helper functions. */ diff --git a/drivers/misc/habanalabs/firmware_if.c b/drivers/misc/habanalabs/firmware_if.c index eda5d7fcb79f..cc8168bacb24 100644 --- a/drivers/misc/habanalabs/firmware_if.c +++ b/drivers/misc/habanalabs/firmware_if.c @@ -29,13 +29,13 @@ int hl_fw_push_fw_to_device(struct hl_device *hdev, const char *fw_name, rc = request_firmware(&fw, fw_name, hdev->dev); if (rc) { - dev_err(hdev->dev, "Failed to request %s\n", fw_name); + dev_err(hdev->dev, "Firmware file %s is not found!\n", fw_name); goto out; } fw_size = fw->size; if ((fw_size % 4) != 0) { - dev_err(hdev->dev, "illegal %s firmware size %zu\n", + dev_err(hdev->dev, "Illegal %s firmware size %zu\n", fw_name, fw_size); rc = -EINVAL; goto out; @@ -85,12 +85,6 @@ int hl_fw_send_cpu_message(struct hl_device *hdev, u32 hw_queue_id, u32 *msg, u32 tmp; int rc = 0; - if (len > HL_CPU_CB_SIZE) { - dev_err(hdev->dev, "Invalid CPU message size of %d bytes\n", - len); - return -ENOMEM; - } - pkt = hdev->asic_funcs->cpu_accessible_dma_pool_alloc(hdev, len, &pkt_dma_addr); if (!pkt) { @@ -117,33 +111,28 @@ int hl_fw_send_cpu_message(struct hl_device *hdev, u32 hw_queue_id, u32 *msg, goto out; } - rc = hl_poll_timeout_memory(hdev, (u64) (uintptr_t) &pkt->fence, - timeout, &tmp); + rc = hl_poll_timeout_memory(hdev, &pkt->fence, tmp, + (tmp == ARMCP_PACKET_FENCE_VAL), 1000, timeout); hl_hw_queue_inc_ci_kernel(hdev, hw_queue_id); if (rc == -ETIMEDOUT) { - dev_err(hdev->dev, "Timeout while waiting for device CPU\n"); + dev_err(hdev->dev, "Device CPU packet timeout (0x%x)\n", tmp); hdev->device_cpu_disabled = true; goto out; } - if (tmp == ARMCP_PACKET_FENCE_VAL) { - u32 ctl = le32_to_cpu(pkt->ctl); + tmp = le32_to_cpu(pkt->ctl); - rc = (ctl & ARMCP_PKT_CTL_RC_MASK) >> ARMCP_PKT_CTL_RC_SHIFT; - if (rc) { - dev_err(hdev->dev, - "F/W ERROR %d for CPU packet %d\n", - rc, (ctl & ARMCP_PKT_CTL_OPCODE_MASK) + rc = (tmp & ARMCP_PKT_CTL_RC_MASK) >> ARMCP_PKT_CTL_RC_SHIFT; + if (rc) { + dev_err(hdev->dev, "F/W ERROR %d for CPU packet %d\n", + rc, + (tmp & ARMCP_PKT_CTL_OPCODE_MASK) >> ARMCP_PKT_CTL_OPCODE_SHIFT); - rc = -EINVAL; - } else if (result) { - *result = (long) le64_to_cpu(pkt->result); - } - } else { - dev_err(hdev->dev, "CPU packet wrong fence value\n"); - rc = -EINVAL; + rc = -EIO; + } else if (result) { + *result = (long) le64_to_cpu(pkt->result); } out: @@ -186,9 +175,6 @@ void *hl_fw_cpu_accessible_dma_pool_alloc(struct hl_device *hdev, size_t size, { u64 kernel_addr; - /* roundup to HL_CPU_PKT_SIZE */ - size = (size + (HL_CPU_PKT_SIZE - 1)) & HL_CPU_PKT_MASK; - kernel_addr = gen_pool_alloc(hdev->cpu_accessible_dma_pool, size); *dma_handle = hdev->cpu_accessible_dma_address + @@ -200,9 +186,6 @@ void *hl_fw_cpu_accessible_dma_pool_alloc(struct hl_device *hdev, size_t size, void hl_fw_cpu_accessible_dma_pool_free(struct hl_device *hdev, size_t size, void *vaddr) { - /* roundup to HL_CPU_PKT_SIZE */ - size = (size + (HL_CPU_PKT_SIZE - 1)) & HL_CPU_PKT_MASK; - gen_pool_free(hdev->cpu_accessible_dma_pool, (u64) (uintptr_t) vaddr, size); } @@ -256,7 +239,7 @@ int hl_fw_armcp_info_get(struct hl_device *hdev) HL_ARMCP_INFO_TIMEOUT_USEC, &result); if (rc) { dev_err(hdev->dev, - "Failed to send armcp info pkt, error %d\n", rc); + "Failed to send ArmCP info pkt, error %d\n", rc); goto out; } @@ -291,7 +274,7 @@ int hl_fw_get_eeprom_data(struct hl_device *hdev, void *data, size_t max_size) max_size, &eeprom_info_dma_addr); if (!eeprom_info_cpu_addr) { dev_err(hdev->dev, - "Failed to allocate DMA memory for EEPROM info packet\n"); + "Failed to allocate DMA memory for ArmCP EEPROM packet\n"); return -ENOMEM; } @@ -307,7 +290,7 @@ int hl_fw_get_eeprom_data(struct hl_device *hdev, void *data, size_t max_size) if (rc) { dev_err(hdev->dev, - "Failed to send armcp EEPROM pkt, error %d\n", rc); + "Failed to send ArmCP EEPROM packet, error %d\n", rc); goto out; } diff --git a/drivers/misc/habanalabs/goya/goya.c b/drivers/misc/habanalabs/goya/goya.c index 02d116b01a1a..75294ec65257 100644 --- a/drivers/misc/habanalabs/goya/goya.c +++ b/drivers/misc/habanalabs/goya/goya.c @@ -14,6 +14,8 @@ #include <linux/genalloc.h> #include <linux/hwmon.h> #include <linux/io-64-nonatomic-lo-hi.h> +#include <linux/iommu.h> +#include <linux/seq_file.h> /* * GOYA security scheme: @@ -89,6 +91,30 @@ #define GOYA_CB_POOL_CB_CNT 512 #define GOYA_CB_POOL_CB_SIZE 0x20000 /* 128KB */ +#define IS_QM_IDLE(engine, qm_glbl_sts0) \ + (((qm_glbl_sts0) & engine##_QM_IDLE_MASK) == engine##_QM_IDLE_MASK) +#define IS_DMA_QM_IDLE(qm_glbl_sts0) IS_QM_IDLE(DMA, qm_glbl_sts0) +#define IS_TPC_QM_IDLE(qm_glbl_sts0) IS_QM_IDLE(TPC, qm_glbl_sts0) +#define IS_MME_QM_IDLE(qm_glbl_sts0) IS_QM_IDLE(MME, qm_glbl_sts0) + +#define IS_CMDQ_IDLE(engine, cmdq_glbl_sts0) \ + (((cmdq_glbl_sts0) & engine##_CMDQ_IDLE_MASK) == \ + engine##_CMDQ_IDLE_MASK) +#define IS_TPC_CMDQ_IDLE(cmdq_glbl_sts0) \ + IS_CMDQ_IDLE(TPC, cmdq_glbl_sts0) +#define IS_MME_CMDQ_IDLE(cmdq_glbl_sts0) \ + IS_CMDQ_IDLE(MME, cmdq_glbl_sts0) + +#define IS_DMA_IDLE(dma_core_sts0) \ + !((dma_core_sts0) & DMA_CH_0_STS0_DMA_BUSY_MASK) + +#define IS_TPC_IDLE(tpc_cfg_sts) \ + (((tpc_cfg_sts) & TPC_CFG_IDLE_MASK) == TPC_CFG_IDLE_MASK) + +#define IS_MME_IDLE(mme_arch_sts) \ + (((mme_arch_sts) & MME_ARCH_IDLE_MASK) == MME_ARCH_IDLE_MASK) + + static const char goya_irq_name[GOYA_MSIX_ENTRIES][GOYA_MAX_STRING_LEN] = { "goya cq 0", "goya cq 1", "goya cq 2", "goya cq 3", "goya cq 4", "goya cpu eq" @@ -297,6 +323,11 @@ static u32 goya_all_events[] = { GOYA_ASYNC_EVENT_ID_DMA_BM_CH4 }; +static int goya_mmu_clear_pgt_range(struct hl_device *hdev); +static int goya_mmu_set_dram_default_page(struct hl_device *hdev); +static int goya_mmu_add_mappings_for_device_cpu(struct hl_device *hdev); +static void goya_mmu_prepare(struct hl_device *hdev, u32 asid); + void goya_get_fixed_properties(struct hl_device *hdev) { struct asic_fixed_properties *prop = &hdev->asic_prop; @@ -467,7 +498,7 @@ static int goya_early_init(struct hl_device *hdev) prop->dram_pci_bar_size = pci_resource_len(pdev, DDR_BAR_ID); - rc = hl_pci_init(hdev, 39); + rc = hl_pci_init(hdev, 48); if (rc) return rc; @@ -539,9 +570,36 @@ int goya_late_init(struct hl_device *hdev) struct asic_fixed_properties *prop = &hdev->asic_prop; int rc; + goya_fetch_psoc_frequency(hdev); + + rc = goya_mmu_clear_pgt_range(hdev); + if (rc) { + dev_err(hdev->dev, + "Failed to clear MMU page tables range %d\n", rc); + return rc; + } + + rc = goya_mmu_set_dram_default_page(hdev); + if (rc) { + dev_err(hdev->dev, "Failed to set DRAM default page %d\n", rc); + return rc; + } + + rc = goya_mmu_add_mappings_for_device_cpu(hdev); + if (rc) + return rc; + + rc = goya_init_cpu_queues(hdev); + if (rc) + return rc; + + rc = goya_test_cpu_queue(hdev); + if (rc) + return rc; + rc = goya_armcp_info_get(hdev); if (rc) { - dev_err(hdev->dev, "Failed to get armcp info\n"); + dev_err(hdev->dev, "Failed to get armcp info %d\n", rc); return rc; } @@ -553,33 +611,15 @@ int goya_late_init(struct hl_device *hdev) rc = hl_fw_send_pci_access_msg(hdev, ARMCP_PACKET_ENABLE_PCI_ACCESS); if (rc) { - dev_err(hdev->dev, "Failed to enable PCI access from CPU\n"); + dev_err(hdev->dev, + "Failed to enable PCI access from CPU %d\n", rc); return rc; } WREG32(mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR, GOYA_ASYNC_EVENT_ID_INTS_REGISTER); - goya_fetch_psoc_frequency(hdev); - - rc = goya_mmu_clear_pgt_range(hdev); - if (rc) { - dev_err(hdev->dev, "Failed to clear MMU page tables range\n"); - goto disable_pci_access; - } - - rc = goya_mmu_set_dram_default_page(hdev); - if (rc) { - dev_err(hdev->dev, "Failed to set DRAM default page\n"); - goto disable_pci_access; - } - return 0; - -disable_pci_access: - hl_fw_send_pci_access_msg(hdev, ARMCP_PACKET_DISABLE_PCI_ACCESS); - - return rc; } /* @@ -655,7 +695,10 @@ static int goya_sw_init(struct hl_device *hdev) goto free_dma_pool; } - hdev->cpu_accessible_dma_pool = gen_pool_create(HL_CPU_PKT_SHIFT, -1); + dev_dbg(hdev->dev, "cpu accessible memory at bus address 0x%llx\n", + hdev->cpu_accessible_dma_address); + + hdev->cpu_accessible_dma_pool = gen_pool_create(ilog2(32), -1); if (!hdev->cpu_accessible_dma_pool) { dev_err(hdev->dev, "Failed to create CPU accessible DMA pool\n"); @@ -786,7 +829,6 @@ static void goya_init_dma_ch(struct hl_device *hdev, int dma_id) else sob_addr = CFG_BASE + mmSYNC_MNGR_SOB_OBJ_1007; - WREG32(mmDMA_CH_0_WR_COMP_ADDR_LO + reg_off, lower_32_bits(sob_addr)); WREG32(mmDMA_CH_0_WR_COMP_ADDR_HI + reg_off, upper_32_bits(sob_addr)); WREG32(mmDMA_CH_0_WR_COMP_WDATA + reg_off, 0x80000001); } @@ -973,9 +1015,9 @@ int goya_init_cpu_queues(struct hl_device *hdev) WREG32(mmPSOC_GLOBAL_CONF_SCRATCHPAD_3, upper_32_bits(eq->bus_address)); WREG32(mmPSOC_GLOBAL_CONF_SCRATCHPAD_8, - lower_32_bits(hdev->cpu_accessible_dma_address)); + lower_32_bits(VA_CPU_ACCESSIBLE_MEM_ADDR)); WREG32(mmPSOC_GLOBAL_CONF_SCRATCHPAD_9, - upper_32_bits(hdev->cpu_accessible_dma_address)); + upper_32_bits(VA_CPU_ACCESSIBLE_MEM_ADDR)); WREG32(mmPSOC_GLOBAL_CONF_SCRATCHPAD_5, HL_QUEUE_SIZE_IN_BYTES); WREG32(mmPSOC_GLOBAL_CONF_SCRATCHPAD_4, HL_EQ_SIZE_IN_BYTES); @@ -1001,7 +1043,7 @@ int goya_init_cpu_queues(struct hl_device *hdev) if (err) { dev_err(hdev->dev, - "Failed to communicate with ARM CPU (ArmCP timeout)\n"); + "Failed to setup communication with device CPU\n"); return -EIO; } @@ -2061,10 +2103,12 @@ static void goya_halt_engines(struct hl_device *hdev, bool hard_reset) goya_disable_external_queues(hdev); goya_disable_internal_queues(hdev); - if (hard_reset) + if (hard_reset) { goya_disable_msix(hdev); - else + goya_mmu_remove_device_cpu_mappings(hdev); + } else { goya_sync_irqs(hdev); + } } /* @@ -2277,14 +2321,14 @@ static int goya_init_cpu(struct hl_device *hdev, u32 cpu_timeout) goya_read_device_fw_version(hdev, FW_COMP_UBOOT); goya_read_device_fw_version(hdev, FW_COMP_PREBOOT); - if (status == CPU_BOOT_STATUS_SRAM_AVAIL) - goto out; - if (!hdev->fw_loading) { dev_info(hdev->dev, "Skip loading FW\n"); goto out; } + if (status == CPU_BOOT_STATUS_SRAM_AVAIL) + goto out; + rc = goya_push_linux_to_device(hdev); if (rc) return rc; @@ -2466,34 +2510,11 @@ static int goya_hw_init(struct hl_device *hdev) if (rc) goto disable_queues; - rc = goya_init_cpu_queues(hdev); - if (rc) { - dev_err(hdev->dev, "failed to initialize CPU H/W queues %d\n", - rc); - goto disable_msix; - } - - /* - * Check if we managed to set the DMA mask to more then 32 bits. If so, - * let's try to increase it again because in Goya we set the initial - * dma mask to less then 39 bits so that the allocation of the memory - * area for the device's cpu will be under 39 bits - */ - if (hdev->dma_mask > 32) { - rc = hl_pci_set_dma_mask(hdev, 48); - if (rc) - goto disable_pci_access; - } - /* Perform read from the device to flush all MSI-X configuration */ val = RREG32(mmPCIE_DBI_DEVICE_ID_VENDOR_ID_REG); return 0; -disable_pci_access: - hl_fw_send_pci_access_msg(hdev, ARMCP_PACKET_DISABLE_PCI_ACCESS); -disable_msix: - goya_disable_msix(hdev); disable_queues: goya_disable_internal_queues(hdev); goya_disable_external_queues(hdev); @@ -2629,7 +2650,6 @@ static int goya_cb_mmap(struct hl_device *hdev, struct vm_area_struct *vma, void goya_ring_doorbell(struct hl_device *hdev, u32 hw_queue_id, u32 pi) { u32 db_reg_offset, db_value; - bool invalid_queue = false; switch (hw_queue_id) { case GOYA_QUEUE_ID_DMA_0: @@ -2653,10 +2673,7 @@ void goya_ring_doorbell(struct hl_device *hdev, u32 hw_queue_id, u32 pi) break; case GOYA_QUEUE_ID_CPU_PQ: - if (hdev->cpu_queues_enable) - db_reg_offset = mmCPU_IF_PF_PQ_PI; - else - invalid_queue = true; + db_reg_offset = mmCPU_IF_PF_PQ_PI; break; case GOYA_QUEUE_ID_MME: @@ -2696,12 +2713,8 @@ void goya_ring_doorbell(struct hl_device *hdev, u32 hw_queue_id, u32 pi) break; default: - invalid_queue = true; - } - - if (invalid_queue) { /* Should never get here */ - dev_err(hdev->dev, "h/w queue %d is invalid. Can't set pi\n", + dev_err(hdev->dev, "H/W queue %d is invalid. Can't set pi\n", hw_queue_id); return; } @@ -2808,7 +2821,6 @@ static int goya_send_job_on_qman0(struct hl_device *hdev, struct hl_cs_job *job) dma_addr_t fence_dma_addr; struct hl_cb *cb; u32 tmp, timeout; - char buf[16] = {}; int rc; if (hdev->pldm) @@ -2816,10 +2828,9 @@ static int goya_send_job_on_qman0(struct hl_device *hdev, struct hl_cs_job *job) else timeout = HL_DEVICE_TIMEOUT_USEC; - if (!hdev->asic_funcs->is_device_idle(hdev, buf, sizeof(buf))) { + if (!hdev->asic_funcs->is_device_idle(hdev, NULL, NULL)) { dev_err_ratelimited(hdev->dev, - "Can't send KMD job on QMAN0 because %s is busy\n", - buf); + "Can't send KMD job on QMAN0 because the device is not idle\n"); return -EBUSY; } @@ -2831,16 +2842,8 @@ static int goya_send_job_on_qman0(struct hl_device *hdev, struct hl_cs_job *job) return -ENOMEM; } - *fence_ptr = 0; - goya_qman0_set_security(hdev, true); - /* - * goya cs parser saves space for 2xpacket_msg_prot at end of CB. For - * synchronized kernel jobs we only need space for 1 packet_msg_prot - */ - job->job_cb_size -= sizeof(struct packet_msg_prot); - cb = job->patched_cb; fence_pkt = (struct packet_msg_prot *) (uintptr_t) (cb->kernel_address + @@ -2860,14 +2863,14 @@ static int goya_send_job_on_qman0(struct hl_device *hdev, struct hl_cs_job *job) goto free_fence_ptr; } - rc = hl_poll_timeout_memory(hdev, (u64) (uintptr_t) fence_ptr, timeout, - &tmp); + rc = hl_poll_timeout_memory(hdev, fence_ptr, tmp, + (tmp == GOYA_QMAN0_FENCE_VAL), 1000, timeout); hl_hw_queue_inc_ci_kernel(hdev, GOYA_QUEUE_ID_DMA_0); - if ((rc) || (tmp != GOYA_QMAN0_FENCE_VAL)) { - dev_err(hdev->dev, "QMAN0 Job hasn't finished in time\n"); - rc = -ETIMEDOUT; + if (rc == -ETIMEDOUT) { + dev_err(hdev->dev, "QMAN0 Job timeout (0x%x)\n", tmp); + goto free_fence_ptr; } free_fence_ptr: @@ -2941,20 +2944,19 @@ int goya_test_queue(struct hl_device *hdev, u32 hw_queue_id) goto free_pkt; } - rc = hl_poll_timeout_memory(hdev, (u64) (uintptr_t) fence_ptr, - GOYA_TEST_QUEUE_WAIT_USEC, &tmp); + rc = hl_poll_timeout_memory(hdev, fence_ptr, tmp, (tmp == fence_val), + 1000, GOYA_TEST_QUEUE_WAIT_USEC); hl_hw_queue_inc_ci_kernel(hdev, hw_queue_id); - if ((!rc) && (tmp == fence_val)) { - dev_info(hdev->dev, - "queue test on H/W queue %d succeeded\n", - hw_queue_id); - } else { + if (rc == -ETIMEDOUT) { dev_err(hdev->dev, "H/W queue %d test failed (scratch(0x%08llX) == 0x%08X)\n", hw_queue_id, (unsigned long long) fence_dma_addr, tmp); - rc = -EINVAL; + rc = -EIO; + } else { + dev_info(hdev->dev, "queue test on H/W queue %d succeeded\n", + hw_queue_id); } free_pkt: @@ -2990,12 +2992,6 @@ int goya_test_queues(struct hl_device *hdev) ret_val = -EINVAL; } - if (hdev->cpu_queues_enable) { - rc = goya_test_cpu_queue(hdev); - if (rc) - ret_val = -EINVAL; - } - return ret_val; } @@ -3028,7 +3024,13 @@ static void goya_dma_pool_free(struct hl_device *hdev, void *vaddr, void *goya_cpu_accessible_dma_pool_alloc(struct hl_device *hdev, size_t size, dma_addr_t *dma_handle) { - return hl_fw_cpu_accessible_dma_pool_alloc(hdev, size, dma_handle); + void *vaddr; + + vaddr = hl_fw_cpu_accessible_dma_pool_alloc(hdev, size, dma_handle); + *dma_handle = (*dma_handle) - hdev->cpu_accessible_dma_address + + VA_CPU_ACCESSIBLE_MEM_ADDR; + + return vaddr; } void goya_cpu_accessible_dma_pool_free(struct hl_device *hdev, size_t size, @@ -3907,8 +3909,8 @@ int goya_cs_parser(struct hl_device *hdev, struct hl_cs_parser *parser) return goya_parse_cb_no_mmu(hdev, parser); } -void goya_add_end_of_cb_packets(u64 kernel_address, u32 len, u64 cq_addr, - u32 cq_val, u32 msix_vec) +void goya_add_end_of_cb_packets(struct hl_device *hdev, u64 kernel_address, + u32 len, u64 cq_addr, u32 cq_val, u32 msix_vec) { struct packet_msg_prot *cq_pkt; u32 tmp; @@ -3939,6 +3941,11 @@ void goya_update_eq_ci(struct hl_device *hdev, u32 val) void goya_restore_phase_topology(struct hl_device *hdev) { + +} + +static void goya_clear_sm_regs(struct hl_device *hdev) +{ int i, num_of_sob_in_longs, num_of_mon_in_longs; num_of_sob_in_longs = @@ -3958,10 +3965,11 @@ void goya_restore_phase_topology(struct hl_device *hdev) } /* - * goya_debugfs_read32 - read a 32bit value from a given device address + * goya_debugfs_read32 - read a 32bit value from a given device or a host mapped + * address. * * @hdev: pointer to hl_device structure - * @addr: address in device + * @addr: device or host mapped address * @val: returned value * * In case of DDR address that is not mapped into the default aperture that @@ -4002,6 +4010,10 @@ static int goya_debugfs_read32(struct hl_device *hdev, u64 addr, u32 *val) } if (ddr_bar_addr == U64_MAX) rc = -EIO; + + } else if (addr >= HOST_PHYS_BASE && !iommu_present(&pci_bus_type)) { + *val = *(u32 *) phys_to_virt(addr - HOST_PHYS_BASE); + } else { rc = -EFAULT; } @@ -4010,10 +4022,11 @@ static int goya_debugfs_read32(struct hl_device *hdev, u64 addr, u32 *val) } /* - * goya_debugfs_write32 - write a 32bit value to a given device address + * goya_debugfs_write32 - write a 32bit value to a given device or a host mapped + * address. * * @hdev: pointer to hl_device structure - * @addr: address in device + * @addr: device or host mapped address * @val: returned value * * In case of DDR address that is not mapped into the default aperture that @@ -4054,6 +4067,10 @@ static int goya_debugfs_write32(struct hl_device *hdev, u64 addr, u32 val) } if (ddr_bar_addr == U64_MAX) rc = -EIO; + + } else if (addr >= HOST_PHYS_BASE && !iommu_present(&pci_bus_type)) { + *(u32 *) phys_to_virt(addr - HOST_PHYS_BASE) = val; + } else { rc = -EFAULT; } @@ -4086,6 +4103,47 @@ static void goya_write_pte(struct hl_device *hdev, u64 addr, u64 val) static const char *_goya_get_event_desc(u16 event_type) { switch (event_type) { + case GOYA_ASYNC_EVENT_ID_PCIE_IF: + return "PCIe_if"; + case GOYA_ASYNC_EVENT_ID_TPC0_ECC: + case GOYA_ASYNC_EVENT_ID_TPC1_ECC: + case GOYA_ASYNC_EVENT_ID_TPC2_ECC: + case GOYA_ASYNC_EVENT_ID_TPC3_ECC: + case GOYA_ASYNC_EVENT_ID_TPC4_ECC: + case GOYA_ASYNC_EVENT_ID_TPC5_ECC: + case GOYA_ASYNC_EVENT_ID_TPC6_ECC: + case GOYA_ASYNC_EVENT_ID_TPC7_ECC: + return "TPC%d_ecc"; + case GOYA_ASYNC_EVENT_ID_MME_ECC: + return "MME_ecc"; + case GOYA_ASYNC_EVENT_ID_MME_ECC_EXT: + return "MME_ecc_ext"; + case GOYA_ASYNC_EVENT_ID_MMU_ECC: + return "MMU_ecc"; + case GOYA_ASYNC_EVENT_ID_DMA_MACRO: + return "DMA_macro"; + case GOYA_ASYNC_EVENT_ID_DMA_ECC: + return "DMA_ecc"; + case GOYA_ASYNC_EVENT_ID_CPU_IF_ECC: + return "CPU_if_ecc"; + case GOYA_ASYNC_EVENT_ID_PSOC_MEM: + return "PSOC_mem"; + case GOYA_ASYNC_EVENT_ID_PSOC_CORESIGHT: + return "PSOC_coresight"; + case GOYA_ASYNC_EVENT_ID_SRAM0 ... GOYA_ASYNC_EVENT_ID_SRAM29: + return "SRAM%d"; + case GOYA_ASYNC_EVENT_ID_GIC500: + return "GIC500"; + case GOYA_ASYNC_EVENT_ID_PLL0 ... GOYA_ASYNC_EVENT_ID_PLL6: + return "PLL%d"; + case GOYA_ASYNC_EVENT_ID_AXI_ECC: + return "AXI_ecc"; + case GOYA_ASYNC_EVENT_ID_L2_RAM_ECC: + return "L2_ram_ecc"; + case GOYA_ASYNC_EVENT_ID_PSOC_GPIO_05_SW_RESET: + return "PSOC_gpio_05_sw_reset"; + case GOYA_ASYNC_EVENT_ID_PSOC_GPIO_10_VRHOT_ICRIT: + return "PSOC_gpio_10_vrhot_icrit"; case GOYA_ASYNC_EVENT_ID_PCIE_DEC: return "PCIe_dec"; case GOYA_ASYNC_EVENT_ID_TPC0_DEC: @@ -4128,6 +4186,17 @@ static const char *_goya_get_event_desc(u16 event_type) return "DMA%d_qm"; case GOYA_ASYNC_EVENT_ID_DMA0_CH ... GOYA_ASYNC_EVENT_ID_DMA4_CH: return "DMA%d_ch"; + case GOYA_ASYNC_EVENT_ID_TPC0_BMON_SPMU: + case GOYA_ASYNC_EVENT_ID_TPC1_BMON_SPMU: + case GOYA_ASYNC_EVENT_ID_TPC2_BMON_SPMU: + case GOYA_ASYNC_EVENT_ID_TPC3_BMON_SPMU: + case GOYA_ASYNC_EVENT_ID_TPC4_BMON_SPMU: + case GOYA_ASYNC_EVENT_ID_TPC5_BMON_SPMU: + case GOYA_ASYNC_EVENT_ID_TPC6_BMON_SPMU: + case GOYA_ASYNC_EVENT_ID_TPC7_BMON_SPMU: + return "TPC%d_bmon_spmu"; + case GOYA_ASYNC_EVENT_ID_DMA_BM_CH0 ... GOYA_ASYNC_EVENT_ID_DMA_BM_CH4: + return "DMA_bm_ch%d"; default: return "N/A"; } @@ -4138,6 +4207,25 @@ static void goya_get_event_desc(u16 event_type, char *desc, size_t size) u8 index; switch (event_type) { + case GOYA_ASYNC_EVENT_ID_TPC0_ECC: + case GOYA_ASYNC_EVENT_ID_TPC1_ECC: + case GOYA_ASYNC_EVENT_ID_TPC2_ECC: + case GOYA_ASYNC_EVENT_ID_TPC3_ECC: + case GOYA_ASYNC_EVENT_ID_TPC4_ECC: + case GOYA_ASYNC_EVENT_ID_TPC5_ECC: + case GOYA_ASYNC_EVENT_ID_TPC6_ECC: + case GOYA_ASYNC_EVENT_ID_TPC7_ECC: + index = (event_type - GOYA_ASYNC_EVENT_ID_TPC0_ECC) / 3; + snprintf(desc, size, _goya_get_event_desc(event_type), index); + break; + case GOYA_ASYNC_EVENT_ID_SRAM0 ... GOYA_ASYNC_EVENT_ID_SRAM29: + index = event_type - GOYA_ASYNC_EVENT_ID_SRAM0; + snprintf(desc, size, _goya_get_event_desc(event_type), index); + break; + case GOYA_ASYNC_EVENT_ID_PLL0 ... GOYA_ASYNC_EVENT_ID_PLL6: + index = event_type - GOYA_ASYNC_EVENT_ID_PLL0; + snprintf(desc, size, _goya_get_event_desc(event_type), index); + break; case GOYA_ASYNC_EVENT_ID_TPC0_DEC: case GOYA_ASYNC_EVENT_ID_TPC1_DEC: case GOYA_ASYNC_EVENT_ID_TPC2_DEC: @@ -4176,6 +4264,21 @@ static void goya_get_event_desc(u16 event_type, char *desc, size_t size) index = event_type - GOYA_ASYNC_EVENT_ID_DMA0_CH; snprintf(desc, size, _goya_get_event_desc(event_type), index); break; + case GOYA_ASYNC_EVENT_ID_TPC0_BMON_SPMU: + case GOYA_ASYNC_EVENT_ID_TPC1_BMON_SPMU: + case GOYA_ASYNC_EVENT_ID_TPC2_BMON_SPMU: + case GOYA_ASYNC_EVENT_ID_TPC3_BMON_SPMU: + case GOYA_ASYNC_EVENT_ID_TPC4_BMON_SPMU: + case GOYA_ASYNC_EVENT_ID_TPC5_BMON_SPMU: + case GOYA_ASYNC_EVENT_ID_TPC6_BMON_SPMU: + case GOYA_ASYNC_EVENT_ID_TPC7_BMON_SPMU: + index = (event_type - GOYA_ASYNC_EVENT_ID_TPC0_BMON_SPMU) / 10; + snprintf(desc, size, _goya_get_event_desc(event_type), index); + break; + case GOYA_ASYNC_EVENT_ID_DMA_BM_CH0 ... GOYA_ASYNC_EVENT_ID_DMA_BM_CH4: + index = event_type - GOYA_ASYNC_EVENT_ID_DMA_BM_CH0; + snprintf(desc, size, _goya_get_event_desc(event_type), index); + break; default: snprintf(desc, size, _goya_get_event_desc(event_type)); break; @@ -4226,7 +4329,8 @@ static void goya_print_mmu_error_info(struct hl_device *hdev) } } -static void goya_print_irq_info(struct hl_device *hdev, u16 event_type) +static void goya_print_irq_info(struct hl_device *hdev, u16 event_type, + bool razwi) { char desc[20] = ""; @@ -4234,8 +4338,10 @@ static void goya_print_irq_info(struct hl_device *hdev, u16 event_type) dev_err(hdev->dev, "Received H/W interrupt %d [\"%s\"]\n", event_type, desc); - goya_print_razwi_info(hdev); - goya_print_mmu_error_info(hdev); + if (razwi) { + goya_print_razwi_info(hdev); + goya_print_mmu_error_info(hdev); + } } static int goya_unmask_irq_arr(struct hl_device *hdev, u32 *irq_arr, @@ -4339,19 +4445,12 @@ void goya_handle_eqe(struct hl_device *hdev, struct hl_eq_entry *eq_entry) case GOYA_ASYNC_EVENT_ID_PSOC_CORESIGHT: case GOYA_ASYNC_EVENT_ID_SRAM0 ... GOYA_ASYNC_EVENT_ID_SRAM29: case GOYA_ASYNC_EVENT_ID_GIC500: - case GOYA_ASYNC_EVENT_ID_PLL0: - case GOYA_ASYNC_EVENT_ID_PLL1: - case GOYA_ASYNC_EVENT_ID_PLL3: - case GOYA_ASYNC_EVENT_ID_PLL4: - case GOYA_ASYNC_EVENT_ID_PLL5: - case GOYA_ASYNC_EVENT_ID_PLL6: + case GOYA_ASYNC_EVENT_ID_PLL0 ... GOYA_ASYNC_EVENT_ID_PLL6: case GOYA_ASYNC_EVENT_ID_AXI_ECC: case GOYA_ASYNC_EVENT_ID_L2_RAM_ECC: case GOYA_ASYNC_EVENT_ID_PSOC_GPIO_05_SW_RESET: case GOYA_ASYNC_EVENT_ID_PSOC_GPIO_10_VRHOT_ICRIT: - dev_err(hdev->dev, - "Received H/W interrupt %d, reset the chip\n", - event_type); + goya_print_irq_info(hdev, event_type, false); hl_device_reset(hdev, true, false); break; @@ -4382,7 +4481,7 @@ void goya_handle_eqe(struct hl_device *hdev, struct hl_eq_entry *eq_entry) case GOYA_ASYNC_EVENT_ID_MME_CMDQ: case GOYA_ASYNC_EVENT_ID_DMA0_QM ... GOYA_ASYNC_EVENT_ID_DMA4_QM: case GOYA_ASYNC_EVENT_ID_DMA0_CH ... GOYA_ASYNC_EVENT_ID_DMA4_CH: - goya_print_irq_info(hdev, event_type); + goya_print_irq_info(hdev, event_type, true); goya_unmask_irq(hdev, event_type); break; @@ -4394,12 +4493,9 @@ void goya_handle_eqe(struct hl_device *hdev, struct hl_eq_entry *eq_entry) case GOYA_ASYNC_EVENT_ID_TPC5_BMON_SPMU: case GOYA_ASYNC_EVENT_ID_TPC6_BMON_SPMU: case GOYA_ASYNC_EVENT_ID_TPC7_BMON_SPMU: - case GOYA_ASYNC_EVENT_ID_DMA_BM_CH0: - case GOYA_ASYNC_EVENT_ID_DMA_BM_CH1: - case GOYA_ASYNC_EVENT_ID_DMA_BM_CH2: - case GOYA_ASYNC_EVENT_ID_DMA_BM_CH3: - case GOYA_ASYNC_EVENT_ID_DMA_BM_CH4: - dev_info(hdev->dev, "Received H/W interrupt %d\n", event_type); + case GOYA_ASYNC_EVENT_ID_DMA_BM_CH0 ... GOYA_ASYNC_EVENT_ID_DMA_BM_CH4: + goya_print_irq_info(hdev, event_type, false); + goya_unmask_irq(hdev, event_type); break; default: @@ -4418,36 +4514,47 @@ void *goya_get_events_stat(struct hl_device *hdev, u32 *size) return goya->events_stat; } -static int goya_memset_device_memory(struct hl_device *hdev, u64 addr, u32 size, +static int goya_memset_device_memory(struct hl_device *hdev, u64 addr, u64 size, u64 val, bool is_dram) { struct packet_lin_dma *lin_dma_pkt; struct hl_cs_job *job; u32 cb_size, ctl; struct hl_cb *cb; - int rc; + int rc, lin_dma_pkts_cnt; - cb = hl_cb_kernel_create(hdev, PAGE_SIZE); + lin_dma_pkts_cnt = DIV_ROUND_UP_ULL(size, SZ_2G); + cb_size = lin_dma_pkts_cnt * sizeof(struct packet_lin_dma) + + sizeof(struct packet_msg_prot); + cb = hl_cb_kernel_create(hdev, cb_size); if (!cb) - return -EFAULT; + return -ENOMEM; lin_dma_pkt = (struct packet_lin_dma *) (uintptr_t) cb->kernel_address; - memset(lin_dma_pkt, 0, sizeof(*lin_dma_pkt)); - cb_size = sizeof(*lin_dma_pkt); - - ctl = ((PACKET_LIN_DMA << GOYA_PKT_CTL_OPCODE_SHIFT) | - (1 << GOYA_PKT_LIN_DMA_CTL_MEMSET_SHIFT) | - (1 << GOYA_PKT_LIN_DMA_CTL_WO_SHIFT) | - (1 << GOYA_PKT_CTL_RB_SHIFT) | - (1 << GOYA_PKT_CTL_MB_SHIFT)); - ctl |= (is_dram ? DMA_HOST_TO_DRAM : DMA_HOST_TO_SRAM) << - GOYA_PKT_LIN_DMA_CTL_DMA_DIR_SHIFT; - lin_dma_pkt->ctl = cpu_to_le32(ctl); + do { + memset(lin_dma_pkt, 0, sizeof(*lin_dma_pkt)); + + ctl = ((PACKET_LIN_DMA << GOYA_PKT_CTL_OPCODE_SHIFT) | + (1 << GOYA_PKT_LIN_DMA_CTL_MEMSET_SHIFT) | + (1 << GOYA_PKT_LIN_DMA_CTL_WO_SHIFT) | + (1 << GOYA_PKT_CTL_RB_SHIFT) | + (1 << GOYA_PKT_CTL_MB_SHIFT)); + ctl |= (is_dram ? DMA_HOST_TO_DRAM : DMA_HOST_TO_SRAM) << + GOYA_PKT_LIN_DMA_CTL_DMA_DIR_SHIFT; + lin_dma_pkt->ctl = cpu_to_le32(ctl); + + lin_dma_pkt->src_addr = cpu_to_le64(val); + lin_dma_pkt->dst_addr = cpu_to_le64(addr); + if (lin_dma_pkts_cnt > 1) + lin_dma_pkt->tsize = cpu_to_le32(SZ_2G); + else + lin_dma_pkt->tsize = cpu_to_le32(size); - lin_dma_pkt->src_addr = cpu_to_le64(val); - lin_dma_pkt->dst_addr = cpu_to_le64(addr); - lin_dma_pkt->tsize = cpu_to_le32(size); + size -= SZ_2G; + addr += SZ_2G; + lin_dma_pkt++; + } while (--lin_dma_pkts_cnt); job = hl_cs_allocate_job(hdev, true); if (!job) { @@ -4462,8 +4569,7 @@ static int goya_memset_device_memory(struct hl_device *hdev, u64 addr, u32 size, job->user_cb_size = cb_size; job->hw_queue_id = GOYA_QUEUE_ID_DMA_0; job->patched_cb = job->user_cb; - job->job_cb_size = job->user_cb_size + - sizeof(struct packet_msg_prot) * 2; + job->job_cb_size = job->user_cb_size; hl_debugfs_add_job(hdev, job); @@ -4485,10 +4591,12 @@ release_cb: int goya_context_switch(struct hl_device *hdev, u32 asid) { struct asic_fixed_properties *prop = &hdev->asic_prop; - u64 addr = prop->sram_base_address; + u64 addr = prop->sram_base_address, sob_addr; u32 size = hdev->pldm ? 0x10000 : prop->sram_size; u64 val = 0x7777777777777777ull; - int rc; + int rc, dma_id; + u32 channel_off = mmDMA_CH_1_WR_COMP_ADDR_LO - + mmDMA_CH_0_WR_COMP_ADDR_LO; rc = goya_memset_device_memory(hdev, addr, size, val, false); if (rc) { @@ -4496,13 +4604,27 @@ int goya_context_switch(struct hl_device *hdev, u32 asid) return rc; } + /* we need to reset registers that the user is allowed to change */ + sob_addr = CFG_BASE + mmSYNC_MNGR_SOB_OBJ_1007; + WREG32(mmDMA_CH_0_WR_COMP_ADDR_LO, lower_32_bits(sob_addr)); + + for (dma_id = 1 ; dma_id < NUMBER_OF_EXT_HW_QUEUES ; dma_id++) { + sob_addr = CFG_BASE + mmSYNC_MNGR_SOB_OBJ_1000 + + (dma_id - 1) * 4; + WREG32(mmDMA_CH_0_WR_COMP_ADDR_LO + channel_off * dma_id, + lower_32_bits(sob_addr)); + } + WREG32(mmTPC_PLL_CLK_RLX_0, 0x200020); + goya_mmu_prepare(hdev, asid); + goya_clear_sm_regs(hdev); + return 0; } -int goya_mmu_clear_pgt_range(struct hl_device *hdev) +static int goya_mmu_clear_pgt_range(struct hl_device *hdev) { struct asic_fixed_properties *prop = &hdev->asic_prop; struct goya_device *goya = hdev->asic_specific; @@ -4516,7 +4638,7 @@ int goya_mmu_clear_pgt_range(struct hl_device *hdev) return goya_memset_device_memory(hdev, addr, size, 0, true); } -int goya_mmu_set_dram_default_page(struct hl_device *hdev) +static int goya_mmu_set_dram_default_page(struct hl_device *hdev) { struct goya_device *goya = hdev->asic_specific; u64 addr = hdev->asic_prop.mmu_dram_default_page_addr; @@ -4529,7 +4651,123 @@ int goya_mmu_set_dram_default_page(struct hl_device *hdev) return goya_memset_device_memory(hdev, addr, size, val, true); } -void goya_mmu_prepare(struct hl_device *hdev, u32 asid) +static int goya_mmu_add_mappings_for_device_cpu(struct hl_device *hdev) +{ + struct asic_fixed_properties *prop = &hdev->asic_prop; + struct goya_device *goya = hdev->asic_specific; + s64 off, cpu_off; + int rc; + + if (!(goya->hw_cap_initialized & HW_CAP_MMU)) + return 0; + + for (off = 0 ; off < CPU_FW_IMAGE_SIZE ; off += PAGE_SIZE_2MB) { + rc = hl_mmu_map(hdev->kernel_ctx, prop->dram_base_address + off, + prop->dram_base_address + off, PAGE_SIZE_2MB); + if (rc) { + dev_err(hdev->dev, "Map failed for address 0x%llx\n", + prop->dram_base_address + off); + goto unmap; + } + } + + if (!(hdev->cpu_accessible_dma_address & (PAGE_SIZE_2MB - 1))) { + rc = hl_mmu_map(hdev->kernel_ctx, VA_CPU_ACCESSIBLE_MEM_ADDR, + hdev->cpu_accessible_dma_address, PAGE_SIZE_2MB); + + if (rc) { + dev_err(hdev->dev, + "Map failed for CPU accessible memory\n"); + off -= PAGE_SIZE_2MB; + goto unmap; + } + } else { + for (cpu_off = 0 ; cpu_off < SZ_2M ; cpu_off += PAGE_SIZE_4KB) { + rc = hl_mmu_map(hdev->kernel_ctx, + VA_CPU_ACCESSIBLE_MEM_ADDR + cpu_off, + hdev->cpu_accessible_dma_address + cpu_off, + PAGE_SIZE_4KB); + if (rc) { + dev_err(hdev->dev, + "Map failed for CPU accessible memory\n"); + cpu_off -= PAGE_SIZE_4KB; + goto unmap_cpu; + } + } + } + + goya_mmu_prepare_reg(hdev, mmCPU_IF_ARUSER_OVR, HL_KERNEL_ASID_ID); + goya_mmu_prepare_reg(hdev, mmCPU_IF_AWUSER_OVR, HL_KERNEL_ASID_ID); + WREG32(mmCPU_IF_ARUSER_OVR_EN, 0x7FF); + WREG32(mmCPU_IF_AWUSER_OVR_EN, 0x7FF); + + /* Make sure configuration is flushed to device */ + RREG32(mmCPU_IF_AWUSER_OVR_EN); + + goya->device_cpu_mmu_mappings_done = true; + + return 0; + +unmap_cpu: + for (; cpu_off >= 0 ; cpu_off -= PAGE_SIZE_4KB) + if (hl_mmu_unmap(hdev->kernel_ctx, + VA_CPU_ACCESSIBLE_MEM_ADDR + cpu_off, + PAGE_SIZE_4KB)) + dev_warn_ratelimited(hdev->dev, + "failed to unmap address 0x%llx\n", + VA_CPU_ACCESSIBLE_MEM_ADDR + cpu_off); +unmap: + for (; off >= 0 ; off -= PAGE_SIZE_2MB) + if (hl_mmu_unmap(hdev->kernel_ctx, + prop->dram_base_address + off, PAGE_SIZE_2MB)) + dev_warn_ratelimited(hdev->dev, + "failed to unmap address 0x%llx\n", + prop->dram_base_address + off); + + return rc; +} + +void goya_mmu_remove_device_cpu_mappings(struct hl_device *hdev) +{ + struct asic_fixed_properties *prop = &hdev->asic_prop; + struct goya_device *goya = hdev->asic_specific; + u32 off, cpu_off; + + if (!(goya->hw_cap_initialized & HW_CAP_MMU)) + return; + + if (!goya->device_cpu_mmu_mappings_done) + return; + + WREG32(mmCPU_IF_ARUSER_OVR_EN, 0); + WREG32(mmCPU_IF_AWUSER_OVR_EN, 0); + + if (!(hdev->cpu_accessible_dma_address & (PAGE_SIZE_2MB - 1))) { + if (hl_mmu_unmap(hdev->kernel_ctx, VA_CPU_ACCESSIBLE_MEM_ADDR, + PAGE_SIZE_2MB)) + dev_warn(hdev->dev, + "Failed to unmap CPU accessible memory\n"); + } else { + for (cpu_off = 0 ; cpu_off < SZ_2M ; cpu_off += PAGE_SIZE_4KB) + if (hl_mmu_unmap(hdev->kernel_ctx, + VA_CPU_ACCESSIBLE_MEM_ADDR + cpu_off, + PAGE_SIZE_4KB)) + dev_warn_ratelimited(hdev->dev, + "failed to unmap address 0x%llx\n", + VA_CPU_ACCESSIBLE_MEM_ADDR + cpu_off); + } + + for (off = 0 ; off < CPU_FW_IMAGE_SIZE ; off += PAGE_SIZE_2MB) + if (hl_mmu_unmap(hdev->kernel_ctx, + prop->dram_base_address + off, PAGE_SIZE_2MB)) + dev_warn_ratelimited(hdev->dev, + "Failed to unmap address 0x%llx\n", + prop->dram_base_address + off); + + goya->device_cpu_mmu_mappings_done = false; +} + +static void goya_mmu_prepare(struct hl_device *hdev, u32 asid) { struct goya_device *goya = hdev->asic_specific; int i; @@ -4676,57 +4914,82 @@ int goya_armcp_info_get(struct hl_device *hdev) return 0; } -static bool goya_is_device_idle(struct hl_device *hdev, char *buf, size_t size) +static bool goya_is_device_idle(struct hl_device *hdev, u32 *mask, + struct seq_file *s) { - u64 offset, dma_qm_reg, tpc_qm_reg, tpc_cmdq_reg, tpc_cfg_reg; + const char *fmt = "%-5d%-9s%#-14x%#-16x%#x\n"; + const char *dma_fmt = "%-5d%-9s%#-14x%#x\n"; + u32 qm_glbl_sts0, cmdq_glbl_sts0, dma_core_sts0, tpc_cfg_sts, + mme_arch_sts; + bool is_idle = true, is_eng_idle; + u64 offset; int i; + if (s) + seq_puts(s, "\nDMA is_idle QM_GLBL_STS0 DMA_CORE_STS0\n" + "--- ------- ------------ -------------\n"); + offset = mmDMA_QM_1_GLBL_STS0 - mmDMA_QM_0_GLBL_STS0; for (i = 0 ; i < DMA_MAX_NUM ; i++) { - dma_qm_reg = mmDMA_QM_0_GLBL_STS0 + i * offset; + qm_glbl_sts0 = RREG32(mmDMA_QM_0_GLBL_STS0 + i * offset); + dma_core_sts0 = RREG32(mmDMA_CH_0_STS0 + i * offset); + is_eng_idle = IS_DMA_QM_IDLE(qm_glbl_sts0) && + IS_DMA_IDLE(dma_core_sts0); + is_idle &= is_eng_idle; - if ((RREG32(dma_qm_reg) & DMA_QM_IDLE_MASK) != - DMA_QM_IDLE_MASK) - return HL_ENG_BUSY(buf, size, "DMA%d_QM", i); + if (mask) + *mask |= !is_eng_idle << (GOYA_ENGINE_ID_DMA_0 + i); + if (s) + seq_printf(s, dma_fmt, i, is_eng_idle ? "Y" : "N", + qm_glbl_sts0, dma_core_sts0); } + if (s) + seq_puts(s, + "\nTPC is_idle QM_GLBL_STS0 CMDQ_GLBL_STS0 CFG_STATUS\n" + "--- ------- ------------ -------------- ----------\n"); + offset = mmTPC1_QM_GLBL_STS0 - mmTPC0_QM_GLBL_STS0; for (i = 0 ; i < TPC_MAX_NUM ; i++) { - tpc_qm_reg = mmTPC0_QM_GLBL_STS0 + i * offset; - tpc_cmdq_reg = mmTPC0_CMDQ_GLBL_STS0 + i * offset; - tpc_cfg_reg = mmTPC0_CFG_STATUS + i * offset; - - if ((RREG32(tpc_qm_reg) & TPC_QM_IDLE_MASK) != - TPC_QM_IDLE_MASK) - return HL_ENG_BUSY(buf, size, "TPC%d_QM", i); - - if ((RREG32(tpc_cmdq_reg) & TPC_CMDQ_IDLE_MASK) != - TPC_CMDQ_IDLE_MASK) - return HL_ENG_BUSY(buf, size, "TPC%d_CMDQ", i); - - if ((RREG32(tpc_cfg_reg) & TPC_CFG_IDLE_MASK) != - TPC_CFG_IDLE_MASK) - return HL_ENG_BUSY(buf, size, "TPC%d_CFG", i); - } - - if ((RREG32(mmMME_QM_GLBL_STS0) & MME_QM_IDLE_MASK) != - MME_QM_IDLE_MASK) - return HL_ENG_BUSY(buf, size, "MME_QM"); - - if ((RREG32(mmMME_CMDQ_GLBL_STS0) & MME_CMDQ_IDLE_MASK) != - MME_CMDQ_IDLE_MASK) - return HL_ENG_BUSY(buf, size, "MME_CMDQ"); - - if ((RREG32(mmMME_ARCH_STATUS) & MME_ARCH_IDLE_MASK) != - MME_ARCH_IDLE_MASK) - return HL_ENG_BUSY(buf, size, "MME_ARCH"); - - if (RREG32(mmMME_SHADOW_0_STATUS) & MME_SHADOW_IDLE_MASK) - return HL_ENG_BUSY(buf, size, "MME"); - - return true; + qm_glbl_sts0 = RREG32(mmTPC0_QM_GLBL_STS0 + i * offset); + cmdq_glbl_sts0 = RREG32(mmTPC0_CMDQ_GLBL_STS0 + i * offset); + tpc_cfg_sts = RREG32(mmTPC0_CFG_STATUS + i * offset); + is_eng_idle = IS_TPC_QM_IDLE(qm_glbl_sts0) && + IS_TPC_CMDQ_IDLE(cmdq_glbl_sts0) && + IS_TPC_IDLE(tpc_cfg_sts); + is_idle &= is_eng_idle; + + if (mask) + *mask |= !is_eng_idle << (GOYA_ENGINE_ID_TPC_0 + i); + if (s) + seq_printf(s, fmt, i, is_eng_idle ? "Y" : "N", + qm_glbl_sts0, cmdq_glbl_sts0, tpc_cfg_sts); + } + + if (s) + seq_puts(s, + "\nMME is_idle QM_GLBL_STS0 CMDQ_GLBL_STS0 ARCH_STATUS\n" + "--- ------- ------------ -------------- -----------\n"); + + qm_glbl_sts0 = RREG32(mmMME_QM_GLBL_STS0); + cmdq_glbl_sts0 = RREG32(mmMME_CMDQ_GLBL_STS0); + mme_arch_sts = RREG32(mmMME_ARCH_STATUS); + is_eng_idle = IS_MME_QM_IDLE(qm_glbl_sts0) && + IS_MME_CMDQ_IDLE(cmdq_glbl_sts0) && + IS_MME_IDLE(mme_arch_sts); + is_idle &= is_eng_idle; + + if (mask) + *mask |= !is_eng_idle << GOYA_ENGINE_ID_MME_0; + if (s) { + seq_printf(s, fmt, 0, is_eng_idle ? "Y" : "N", qm_glbl_sts0, + cmdq_glbl_sts0, mme_arch_sts); + seq_puts(s, "\n"); + } + + return is_idle; } static void goya_hw_queues_lock(struct hl_device *hdev) diff --git a/drivers/misc/habanalabs/goya/goyaP.h b/drivers/misc/habanalabs/goya/goyaP.h index c83cab0d641e..f8c611883dc1 100644 --- a/drivers/misc/habanalabs/goya/goyaP.h +++ b/drivers/misc/habanalabs/goya/goyaP.h @@ -126,6 +126,12 @@ #define VA_DDR_SPACE_SIZE (VA_DDR_SPACE_END - \ VA_DDR_SPACE_START) /* 128GB */ +#if (HL_CPU_ACCESSIBLE_MEM_SIZE != SZ_2M) +#error "HL_CPU_ACCESSIBLE_MEM_SIZE must be exactly 2MB to enable MMU mapping" +#endif + +#define VA_CPU_ACCESSIBLE_MEM_ADDR 0x8000000000ull + #define DMA_MAX_TRANSFER_SIZE U32_MAX #define HW_CAP_PLL 0x00000001 @@ -157,6 +163,7 @@ struct goya_device { u64 ddr_bar_cur_addr; u32 events_stat[GOYA_ASYNC_EVENT_ID_SIZE]; u32 hw_cap_initialized; + u8 device_cpu_mmu_mappings_done; }; void goya_get_fixed_properties(struct hl_device *hdev); @@ -204,18 +211,14 @@ int goya_armcp_info_get(struct hl_device *hdev); int goya_debug_coresight(struct hl_device *hdev, void *data); void goya_halt_coresight(struct hl_device *hdev); -void goya_mmu_prepare(struct hl_device *hdev, u32 asid); -int goya_mmu_clear_pgt_range(struct hl_device *hdev); -int goya_mmu_set_dram_default_page(struct hl_device *hdev); - int goya_suspend(struct hl_device *hdev); int goya_resume(struct hl_device *hdev); void goya_handle_eqe(struct hl_device *hdev, struct hl_eq_entry *eq_entry); void *goya_get_events_stat(struct hl_device *hdev, u32 *size); -void goya_add_end_of_cb_packets(u64 kernel_address, u32 len, u64 cq_addr, - u32 cq_val, u32 msix_vec); +void goya_add_end_of_cb_packets(struct hl_device *hdev, u64 kernel_address, + u32 len, u64 cq_addr, u32 cq_val, u32 msix_vec); int goya_cs_parser(struct hl_device *hdev, struct hl_cs_parser *parser); void *goya_get_int_queue_base(struct hl_device *hdev, u32 queue_id, dma_addr_t *dma_handle, u16 *queue_len); @@ -225,5 +228,6 @@ void *goya_cpu_accessible_dma_pool_alloc(struct hl_device *hdev, size_t size, dma_addr_t *dma_handle); void goya_cpu_accessible_dma_pool_free(struct hl_device *hdev, size_t size, void *vaddr); +void goya_mmu_remove_device_cpu_mappings(struct hl_device *hdev); #endif /* GOYAP_H_ */ diff --git a/drivers/misc/habanalabs/goya/goya_security.c b/drivers/misc/habanalabs/goya/goya_security.c index d95d1b2f860d..d6ec12b3e692 100644 --- a/drivers/misc/habanalabs/goya/goya_security.c +++ b/drivers/misc/habanalabs/goya/goya_security.c @@ -677,6 +677,17 @@ static void goya_init_tpc_protection_bits(struct hl_device *hdev) goya_pb_set_block(hdev, mmTPC0_RD_REGULATOR_BASE); goya_pb_set_block(hdev, mmTPC0_WR_REGULATOR_BASE); + pb_addr = (mmTPC0_CFG_SEMAPHORE & ~0xFFF) + PROT_BITS_OFFS; + word_offset = ((mmTPC0_CFG_SEMAPHORE & PROT_BITS_OFFS) >> 7) << 2; + + mask = 1 << ((mmTPC0_CFG_SEMAPHORE & 0x7F) >> 2); + mask |= 1 << ((mmTPC0_CFG_VFLAGS & 0x7F) >> 2); + mask |= 1 << ((mmTPC0_CFG_SFLAGS & 0x7F) >> 2); + mask |= 1 << ((mmTPC0_CFG_LFSR_POLYNOM & 0x7F) >> 2); + mask |= 1 << ((mmTPC0_CFG_STATUS & 0x7F) >> 2); + + WREG32(pb_addr + word_offset, ~mask); + pb_addr = (mmTPC0_CFG_CFG_BASE_ADDRESS_HIGH & ~0xFFF) + PROT_BITS_OFFS; word_offset = ((mmTPC0_CFG_CFG_BASE_ADDRESS_HIGH & PROT_BITS_OFFS) >> 7) << 2; @@ -684,6 +695,11 @@ static void goya_init_tpc_protection_bits(struct hl_device *hdev) mask |= 1 << ((mmTPC0_CFG_CFG_SUBTRACT_VALUE & 0x7F) >> 2); mask |= 1 << ((mmTPC0_CFG_SM_BASE_ADDRESS_LOW & 0x7F) >> 2); mask |= 1 << ((mmTPC0_CFG_SM_BASE_ADDRESS_HIGH & 0x7F) >> 2); + mask |= 1 << ((mmTPC0_CFG_CFG_SUBTRACT_VALUE & 0x7F) >> 2); + mask |= 1 << ((mmTPC0_CFG_TPC_STALL & 0x7F) >> 2); + mask |= 1 << ((mmTPC0_CFG_MSS_CONFIG & 0x7F) >> 2); + mask |= 1 << ((mmTPC0_CFG_TPC_INTR_CAUSE & 0x7F) >> 2); + mask |= 1 << ((mmTPC0_CFG_TPC_INTR_MASK & 0x7F) >> 2); WREG32(pb_addr + word_offset, ~mask); diff --git a/drivers/misc/habanalabs/habanalabs.h b/drivers/misc/habanalabs/habanalabs.h index adef7d9d7488..10da9940ee0d 100644 --- a/drivers/misc/habanalabs/habanalabs.h +++ b/drivers/misc/habanalabs/habanalabs.h @@ -34,6 +34,8 @@ #define HL_ARMCP_INFO_TIMEOUT_USEC 10000000 /* 10s */ #define HL_ARMCP_EEPROM_TIMEOUT_USEC 10000000 /* 10s */ +#define HL_PCI_ELBI_TIMEOUT_MSEC 10 /* 10ms */ + #define HL_MAX_QUEUES 128 #define HL_MAX_JOBS_PER_CS 64 @@ -123,7 +125,7 @@ enum hl_device_hw_state { /** * struct asic_fixed_properties - ASIC specific immutable properties. * @hw_queues_props: H/W queues properties. - * @armcp_info: received various information from ArmCP regarding the H/W. e.g. + * @armcp_info: received various information from ArmCP regarding the H/W, e.g. * available sensors. * @uboot_ver: F/W U-boot version. * @preboot_ver: F/W Preboot version. @@ -318,18 +320,8 @@ struct hl_cs_job; #define HL_EQ_LENGTH 64 #define HL_EQ_SIZE_IN_BYTES (HL_EQ_LENGTH * HL_EQ_ENTRY_SIZE) -#define HL_CPU_PKT_SHIFT 5 -#define HL_CPU_PKT_SIZE (1 << HL_CPU_PKT_SHIFT) -#define HL_CPU_PKT_MASK (~((1 << HL_CPU_PKT_SHIFT) - 1)) -#define HL_CPU_MAX_PKTS_IN_CB 32 -#define HL_CPU_CB_SIZE (HL_CPU_PKT_SIZE * \ - HL_CPU_MAX_PKTS_IN_CB) -#define HL_CPU_CB_QUEUE_SIZE (HL_QUEUE_LENGTH * HL_CPU_CB_SIZE) - -/* KMD <-> ArmCP shared memory size (EQ + PQ + CPU CB queue) */ -#define HL_CPU_ACCESSIBLE_MEM_SIZE (HL_EQ_SIZE_IN_BYTES + \ - HL_QUEUE_SIZE_IN_BYTES + \ - HL_CPU_CB_QUEUE_SIZE) +/* KMD <-> ArmCP shared memory size */ +#define HL_CPU_ACCESSIBLE_MEM_SIZE SZ_2M /** * struct hl_hw_queue - describes a H/W transport queue. @@ -543,8 +535,9 @@ struct hl_asic_funcs { enum dma_data_direction dir); u32 (*get_dma_desc_list_size)(struct hl_device *hdev, struct sg_table *sgt); - void (*add_end_of_cb_packets)(u64 kernel_address, u32 len, u64 cq_addr, - u32 cq_val, u32 msix_num); + void (*add_end_of_cb_packets)(struct hl_device *hdev, + u64 kernel_address, u32 len, + u64 cq_addr, u32 cq_val, u32 msix_num); void (*update_eq_ci)(struct hl_device *hdev, u32 val); int (*context_switch)(struct hl_device *hdev, u32 asid); void (*restore_phase_topology)(struct hl_device *hdev); @@ -564,7 +557,8 @@ struct hl_asic_funcs { u32 asid, u64 va, u64 size); int (*send_heartbeat)(struct hl_device *hdev); int (*debug_coresight)(struct hl_device *hdev, void *data); - bool (*is_device_idle)(struct hl_device *hdev, char *buf, size_t size); + bool (*is_device_idle)(struct hl_device *hdev, u32 *mask, + struct seq_file *s); int (*soft_reset_late_init)(struct hl_device *hdev); void (*hw_queues_lock)(struct hl_device *hdev); void (*hw_queues_unlock)(struct hl_device *hdev); @@ -1065,12 +1059,59 @@ void hl_wreg(struct hl_device *hdev, u32 reg, u32 val); (cond) ? 0 : -ETIMEDOUT; \ }) +/* + * address in this macro points always to a memory location in the + * host's (server's) memory. That location is updated asynchronously + * either by the direct access of the device or by another core + */ +#define hl_poll_timeout_memory(hdev, addr, val, cond, sleep_us, timeout_us) \ +({ \ + ktime_t __timeout; \ + /* timeout should be longer when working with simulator */ \ + if (hdev->pdev) \ + __timeout = ktime_add_us(ktime_get(), timeout_us); \ + else \ + __timeout = ktime_add_us(ktime_get(), (timeout_us * 10)); \ + might_sleep_if(sleep_us); \ + for (;;) { \ + /* Verify we read updates done by other cores or by device */ \ + mb(); \ + (val) = *((u32 *) (uintptr_t) (addr)); \ + if (cond) \ + break; \ + if (timeout_us && ktime_compare(ktime_get(), __timeout) > 0) { \ + (val) = *((u32 *) (uintptr_t) (addr)); \ + break; \ + } \ + if (sleep_us) \ + usleep_range((sleep_us >> 2) + 1, sleep_us); \ + } \ + (cond) ? 0 : -ETIMEDOUT; \ +}) -#define HL_ENG_BUSY(buf, size, fmt, ...) ({ \ - if (buf) \ - snprintf(buf, size, fmt, ##__VA_ARGS__); \ - false; \ - }) +#define hl_poll_timeout_device_memory(hdev, addr, val, cond, sleep_us, \ + timeout_us) \ +({ \ + ktime_t __timeout; \ + /* timeout should be longer when working with simulator */ \ + if (hdev->pdev) \ + __timeout = ktime_add_us(ktime_get(), timeout_us); \ + else \ + __timeout = ktime_add_us(ktime_get(), (timeout_us * 10)); \ + might_sleep_if(sleep_us); \ + for (;;) { \ + (val) = readl(addr); \ + if (cond) \ + break; \ + if (timeout_us && ktime_compare(ktime_get(), __timeout) > 0) { \ + (val) = readl(addr); \ + break; \ + } \ + if (sleep_us) \ + usleep_range((sleep_us >> 2) + 1, sleep_us); \ + } \ + (cond) ? 0 : -ETIMEDOUT; \ +}) struct hwmon_chip_info; @@ -1117,6 +1158,7 @@ struct hl_device_reset_work { * lock here so we can flush user processes which are opening * the device while we are trying to hard reset it * @send_cpu_message_lock: enforces only one message in KMD <-> ArmCP queue. + * @debug_lock: protects critical section of setting debug mode for device * @asic_prop: ASIC specific immutable properties. * @asic_funcs: ASIC specific functions. * @asic_specific: ASIC specific information to use only from ASIC files. @@ -1159,6 +1201,8 @@ struct hl_device_reset_work { * @mmu_enable: is MMU enabled. * @device_cpu_disabled: is the device CPU disabled (due to timeouts) * @dma_mask: the dma mask that was set for this device + * @in_debug: is device under debug. This, together with fd_open_cnt, enforces + * that only a single user is configuring the debug infrastructure. */ struct hl_device { struct pci_dev *pdev; @@ -1188,6 +1232,7 @@ struct hl_device { /* TODO: remove fd_open_cnt_lock for multiple process support */ struct mutex fd_open_cnt_lock; struct mutex send_cpu_message_lock; + struct mutex debug_lock; struct asic_fixed_properties asic_prop; const struct hl_asic_funcs *asic_funcs; void *asic_specific; @@ -1230,6 +1275,7 @@ struct hl_device { u8 init_done; u8 device_cpu_disabled; u8 dma_mask; + u8 in_debug; /* Parameters for bring-up */ u8 mmu_enable; @@ -1325,13 +1371,10 @@ static inline bool hl_mem_area_crosses_range(u64 address, u32 size, int hl_device_open(struct inode *inode, struct file *filp); bool hl_device_disabled_or_in_reset(struct hl_device *hdev); enum hl_device_status hl_device_status(struct hl_device *hdev); +int hl_device_set_debug_mode(struct hl_device *hdev, bool enable); int create_hdev(struct hl_device **dev, struct pci_dev *pdev, enum hl_asic_type asic_type, int minor); void destroy_hdev(struct hl_device *hdev); -int hl_poll_timeout_memory(struct hl_device *hdev, u64 addr, u32 timeout_us, - u32 *val); -int hl_poll_timeout_device_memory(struct hl_device *hdev, void __iomem *addr, - u32 timeout_us, u32 *val); int hl_hw_queues_create(struct hl_device *hdev); void hl_hw_queues_destroy(struct hl_device *hdev); int hl_hw_queue_send_cb_no_cmpl(struct hl_device *hdev, u32 hw_queue_id, diff --git a/drivers/misc/habanalabs/habanalabs_drv.c b/drivers/misc/habanalabs/habanalabs_drv.c index 5f4d155be767..6f6dbe93f1df 100644 --- a/drivers/misc/habanalabs/habanalabs_drv.c +++ b/drivers/misc/habanalabs/habanalabs_drv.c @@ -105,9 +105,17 @@ int hl_device_open(struct inode *inode, struct file *filp) return -EPERM; } + if (hdev->in_debug) { + dev_err_ratelimited(hdev->dev, + "Can't open %s because it is being debugged by another user\n", + dev_name(hdev->dev)); + mutex_unlock(&hdev->fd_open_cnt_lock); + return -EPERM; + } + if (atomic_read(&hdev->fd_open_cnt)) { dev_info_ratelimited(hdev->dev, - "Device %s is already attached to application\n", + "Can't open %s because another user is working on it\n", dev_name(hdev->dev)); mutex_unlock(&hdev->fd_open_cnt_lock); return -EBUSY; @@ -164,6 +172,17 @@ close_device: return rc; } +static void set_driver_behavior_per_device(struct hl_device *hdev) +{ + hdev->mmu_enable = 1; + hdev->cpu_enable = 1; + hdev->fw_loading = 1; + hdev->cpu_queues_enable = 1; + hdev->heartbeat = 1; + + hdev->reset_pcilink = 0; +} + /* * create_hdev - create habanalabs device instance * @@ -188,29 +207,25 @@ int create_hdev(struct hl_device **dev, struct pci_dev *pdev, if (!hdev) return -ENOMEM; + /* First, we must find out which ASIC are we handling. This is needed + * to configure the behavior of the driver (kernel parameters) + */ + if (pdev) { + hdev->asic_type = get_asic_type(pdev->device); + if (hdev->asic_type == ASIC_INVALID) { + dev_err(&pdev->dev, "Unsupported ASIC\n"); + rc = -ENODEV; + goto free_hdev; + } + } else { + hdev->asic_type = asic_type; + } + hdev->major = hl_major; hdev->reset_on_lockup = reset_on_lockup; - - /* Parameters for bring-up - set them to defaults */ - hdev->mmu_enable = 1; - hdev->cpu_enable = 1; - hdev->reset_pcilink = 0; - hdev->cpu_queues_enable = 1; - hdev->fw_loading = 1; hdev->pldm = 0; - hdev->heartbeat = 1; - - /* If CPU is disabled, no point in loading FW */ - if (!hdev->cpu_enable) - hdev->fw_loading = 0; - /* If we don't load FW, no need to initialize CPU queues */ - if (!hdev->fw_loading) - hdev->cpu_queues_enable = 0; - - /* If CPU queues not enabled, no way to do heartbeat */ - if (!hdev->cpu_queues_enable) - hdev->heartbeat = 0; + set_driver_behavior_per_device(hdev); if (timeout_locked) hdev->timeout_jiffies = msecs_to_jiffies(timeout_locked * 1000); @@ -220,17 +235,6 @@ int create_hdev(struct hl_device **dev, struct pci_dev *pdev, hdev->disabled = true; hdev->pdev = pdev; /* can be NULL in case of simulator device */ - if (pdev) { - hdev->asic_type = get_asic_type(pdev->device); - if (hdev->asic_type == ASIC_INVALID) { - dev_err(&pdev->dev, "Unsupported ASIC\n"); - rc = -ENODEV; - goto free_hdev; - } - } else { - hdev->asic_type = asic_type; - } - /* Set default DMA mask to 32 bits */ hdev->dma_mask = 32; diff --git a/drivers/misc/habanalabs/habanalabs_ioctl.c b/drivers/misc/habanalabs/habanalabs_ioctl.c index b7a0eecf6b6c..07127576b3e8 100644 --- a/drivers/misc/habanalabs/habanalabs_ioctl.c +++ b/drivers/misc/habanalabs/habanalabs_ioctl.c @@ -119,7 +119,8 @@ static int hw_idle(struct hl_device *hdev, struct hl_info_args *args) if ((!max_size) || (!out)) return -EINVAL; - hw_idle.is_idle = hdev->asic_funcs->is_device_idle(hdev, NULL, 0); + hw_idle.is_idle = hdev->asic_funcs->is_device_idle(hdev, + &hw_idle.busy_engines_mask, NULL); return copy_to_user(out, &hw_idle, min((size_t) max_size, sizeof(hw_idle))) ? -EFAULT : 0; @@ -254,10 +255,18 @@ static int hl_debug_ioctl(struct hl_fpriv *hpriv, void *data) case HL_DEBUG_OP_BMON: case HL_DEBUG_OP_SPMU: case HL_DEBUG_OP_TIMESTAMP: + if (!hdev->in_debug) { + dev_err_ratelimited(hdev->dev, + "Rejecting debug configuration request because device not in debug mode\n"); + return -EFAULT; + } args->input_size = min(args->input_size, hl_debug_struct_size[args->op]); rc = debug_coresight(hdev, args); break; + case HL_DEBUG_OP_SET_MODE: + rc = hl_device_set_debug_mode(hdev, (bool) args->enable); + break; default: dev_err(hdev->dev, "Invalid request %d\n", args->op); rc = -ENOTTY; diff --git a/drivers/misc/habanalabs/hw_queue.c b/drivers/misc/habanalabs/hw_queue.c index 2894d8975933..e3b5517897ea 100644 --- a/drivers/misc/habanalabs/hw_queue.c +++ b/drivers/misc/habanalabs/hw_queue.c @@ -265,7 +265,7 @@ static void ext_hw_queue_schedule_job(struct hl_cs_job *job) cq = &hdev->completion_queue[q->hw_queue_id]; cq_addr = cq->bus_address + cq->pi * sizeof(struct hl_cq_entry); - hdev->asic_funcs->add_end_of_cb_packets(cb->kernel_address, len, + hdev->asic_funcs->add_end_of_cb_packets(hdev, cb->kernel_address, len, cq_addr, __le32_to_cpu(cq_pkt.data), q->hw_queue_id); diff --git a/drivers/misc/habanalabs/include/goya/asic_reg/dma_ch_0_masks.h b/drivers/misc/habanalabs/include/goya/asic_reg/dma_ch_0_masks.h new file mode 100644 index 000000000000..028143408401 --- /dev/null +++ b/drivers/misc/habanalabs/include/goya/asic_reg/dma_ch_0_masks.h @@ -0,0 +1,418 @@ +/* SPDX-License-Identifier: GPL-2.0 + * + * Copyright 2016-2018 HabanaLabs, Ltd. + * All Rights Reserved. + * + */ + +/************************************ + ** This is an auto-generated file ** + ** DO NOT EDIT BELOW ** + ************************************/ + +#ifndef ASIC_REG_DMA_CH_0_MASKS_H_ +#define ASIC_REG_DMA_CH_0_MASKS_H_ + +/* + ***************************************** + * DMA_CH_0 (Prototype: DMA_CH) + ***************************************** + */ + +/* DMA_CH_0_CFG0 */ +#define DMA_CH_0_CFG0_RD_MAX_OUTSTAND_SHIFT 0 +#define DMA_CH_0_CFG0_RD_MAX_OUTSTAND_MASK 0x3FF +#define DMA_CH_0_CFG0_WR_MAX_OUTSTAND_SHIFT 16 +#define DMA_CH_0_CFG0_WR_MAX_OUTSTAND_MASK 0xFFF0000 + +/* DMA_CH_0_CFG1 */ +#define DMA_CH_0_CFG1_RD_BUF_MAX_SIZE_SHIFT 0 +#define DMA_CH_0_CFG1_RD_BUF_MAX_SIZE_MASK 0x3FF + +/* DMA_CH_0_ERRMSG_ADDR_LO */ +#define DMA_CH_0_ERRMSG_ADDR_LO_VAL_SHIFT 0 +#define DMA_CH_0_ERRMSG_ADDR_LO_VAL_MASK 0xFFFFFFFF + +/* DMA_CH_0_ERRMSG_ADDR_HI */ +#define DMA_CH_0_ERRMSG_ADDR_HI_VAL_SHIFT 0 +#define DMA_CH_0_ERRMSG_ADDR_HI_VAL_MASK 0xFFFFFFFF + +/* DMA_CH_0_ERRMSG_WDATA */ +#define DMA_CH_0_ERRMSG_WDATA_VAL_SHIFT 0 +#define DMA_CH_0_ERRMSG_WDATA_VAL_MASK 0xFFFFFFFF + +/* DMA_CH_0_RD_COMP_ADDR_LO */ +#define DMA_CH_0_RD_COMP_ADDR_LO_VAL_SHIFT 0 +#define DMA_CH_0_RD_COMP_ADDR_LO_VAL_MASK 0xFFFFFFFF + +/* DMA_CH_0_RD_COMP_ADDR_HI */ +#define DMA_CH_0_RD_COMP_ADDR_HI_VAL_SHIFT 0 +#define DMA_CH_0_RD_COMP_ADDR_HI_VAL_MASK 0xFFFFFFFF + +/* DMA_CH_0_RD_COMP_WDATA */ +#define DMA_CH_0_RD_COMP_WDATA_VAL_SHIFT 0 +#define DMA_CH_0_RD_COMP_WDATA_VAL_MASK 0xFFFFFFFF + +/* DMA_CH_0_WR_COMP_ADDR_LO */ +#define DMA_CH_0_WR_COMP_ADDR_LO_VAL_SHIFT 0 +#define DMA_CH_0_WR_COMP_ADDR_LO_VAL_MASK 0xFFFFFFFF + +/* DMA_CH_0_WR_COMP_ADDR_HI */ +#define DMA_CH_0_WR_COMP_ADDR_HI_VAL_SHIFT 0 +#define DMA_CH_0_WR_COMP_ADDR_HI_VAL_MASK 0xFFFFFFFF + +/* DMA_CH_0_WR_COMP_WDATA */ +#define DMA_CH_0_WR_COMP_WDATA_VAL_SHIFT 0 +#define DMA_CH_0_WR_COMP_WDATA_VAL_MASK 0xFFFFFFFF + +/* DMA_CH_0_LDMA_SRC_ADDR_LO */ +#define DMA_CH_0_LDMA_SRC_ADDR_LO_VAL_SHIFT 0 +#define DMA_CH_0_LDMA_SRC_ADDR_LO_VAL_MASK 0xFFFFFFFF + +/* DMA_CH_0_LDMA_SRC_ADDR_HI */ +#define DMA_CH_0_LDMA_SRC_ADDR_HI_VAL_SHIFT 0 +#define DMA_CH_0_LDMA_SRC_ADDR_HI_VAL_MASK 0xFFFFFFFF + +/* DMA_CH_0_LDMA_DST_ADDR_LO */ +#define DMA_CH_0_LDMA_DST_ADDR_LO_VAL_SHIFT 0 +#define DMA_CH_0_LDMA_DST_ADDR_LO_VAL_MASK 0xFFFFFFFF + +/* DMA_CH_0_LDMA_DST_ADDR_HI */ +#define DMA_CH_0_LDMA_DST_ADDR_HI_VAL_SHIFT 0 +#define DMA_CH_0_LDMA_DST_ADDR_HI_VAL_MASK 0xFFFFFFFF + +/* DMA_CH_0_LDMA_TSIZE */ +#define DMA_CH_0_LDMA_TSIZE_VAL_SHIFT 0 +#define DMA_CH_0_LDMA_TSIZE_VAL_MASK 0xFFFFFFFF + +/* DMA_CH_0_COMIT_TRANSFER */ +#define DMA_CH_0_COMIT_TRANSFER_PCI_UPS_WKORDR_SHIFT 0 +#define DMA_CH_0_COMIT_TRANSFER_PCI_UPS_WKORDR_MASK 0x1 +#define DMA_CH_0_COMIT_TRANSFER_RD_COMP_EN_SHIFT 1 +#define DMA_CH_0_COMIT_TRANSFER_RD_COMP_EN_MASK 0x2 +#define DMA_CH_0_COMIT_TRANSFER_WR_COMP_EN_SHIFT 2 +#define DMA_CH_0_COMIT_TRANSFER_WR_COMP_EN_MASK 0x4 +#define DMA_CH_0_COMIT_TRANSFER_NOSNOOP_SHIFT 3 +#define DMA_CH_0_COMIT_TRANSFER_NOSNOOP_MASK 0x8 +#define DMA_CH_0_COMIT_TRANSFER_SRC_ADDR_INC_DIS_SHIFT 4 +#define DMA_CH_0_COMIT_TRANSFER_SRC_ADDR_INC_DIS_MASK 0x10 +#define DMA_CH_0_COMIT_TRANSFER_DST_ADDR_INC_DIS_SHIFT 5 +#define DMA_CH_0_COMIT_TRANSFER_DST_ADDR_INC_DIS_MASK 0x20 +#define DMA_CH_0_COMIT_TRANSFER_MEM_SET_SHIFT 6 +#define DMA_CH_0_COMIT_TRANSFER_MEM_SET_MASK 0x40 +#define DMA_CH_0_COMIT_TRANSFER_MOD_TENSOR_SHIFT 15 +#define DMA_CH_0_COMIT_TRANSFER_MOD_TENSOR_MASK 0x8000 +#define DMA_CH_0_COMIT_TRANSFER_CTL_SHIFT 16 +#define DMA_CH_0_COMIT_TRANSFER_CTL_MASK 0xFFFF0000 + +/* DMA_CH_0_STS0 */ +#define DMA_CH_0_STS0_DMA_BUSY_SHIFT 0 +#define DMA_CH_0_STS0_DMA_BUSY_MASK 0x1 +#define DMA_CH_0_STS0_RD_STS_CTX_FULL_SHIFT 1 +#define DMA_CH_0_STS0_RD_STS_CTX_FULL_MASK 0x2 +#define DMA_CH_0_STS0_WR_STS_CTX_FULL_SHIFT 2 +#define DMA_CH_0_STS0_WR_STS_CTX_FULL_MASK 0x4 + +/* DMA_CH_0_STS1 */ +#define DMA_CH_0_STS1_RD_STS_CTX_CNT_SHIFT 0 +#define DMA_CH_0_STS1_RD_STS_CTX_CNT_MASK 0xFFFFFFFF + +/* DMA_CH_0_STS2 */ +#define DMA_CH_0_STS2_WR_STS_CTX_CNT_SHIFT 0 +#define DMA_CH_0_STS2_WR_STS_CTX_CNT_MASK 0xFFFFFFFF + +/* DMA_CH_0_STS3 */ +#define DMA_CH_0_STS3_RD_STS_TRN_CNT_SHIFT 0 +#define DMA_CH_0_STS3_RD_STS_TRN_CNT_MASK 0xFFFFFFFF + +/* DMA_CH_0_STS4 */ +#define DMA_CH_0_STS4_WR_STS_TRN_CNT_SHIFT 0 +#define DMA_CH_0_STS4_WR_STS_TRN_CNT_MASK 0xFFFFFFFF + +/* DMA_CH_0_SRC_ADDR_LO_STS */ +#define DMA_CH_0_SRC_ADDR_LO_STS_VAL_SHIFT 0 +#define DMA_CH_0_SRC_ADDR_LO_STS_VAL_MASK 0xFFFFFFFF + +/* DMA_CH_0_SRC_ADDR_HI_STS */ +#define DMA_CH_0_SRC_ADDR_HI_STS_VAL_SHIFT 0 +#define DMA_CH_0_SRC_ADDR_HI_STS_VAL_MASK 0xFFFFFFFF + +/* DMA_CH_0_SRC_TSIZE_STS */ +#define DMA_CH_0_SRC_TSIZE_STS_VAL_SHIFT 0 +#define DMA_CH_0_SRC_TSIZE_STS_VAL_MASK 0xFFFFFFFF + +/* DMA_CH_0_DST_ADDR_LO_STS */ +#define DMA_CH_0_DST_ADDR_LO_STS_VAL_SHIFT 0 +#define DMA_CH_0_DST_ADDR_LO_STS_VAL_MASK 0xFFFFFFFF + +/* DMA_CH_0_DST_ADDR_HI_STS */ +#define DMA_CH_0_DST_ADDR_HI_STS_VAL_SHIFT 0 +#define DMA_CH_0_DST_ADDR_HI_STS_VAL_MASK 0xFFFFFFFF + +/* DMA_CH_0_DST_TSIZE_STS */ +#define DMA_CH_0_DST_TSIZE_STS_VAL_SHIFT 0 +#define DMA_CH_0_DST_TSIZE_STS_VAL_MASK 0xFFFFFFFF + +/* DMA_CH_0_RD_RATE_LIM_EN */ +#define DMA_CH_0_RD_RATE_LIM_EN_VAL_SHIFT 0 +#define DMA_CH_0_RD_RATE_LIM_EN_VAL_MASK 0x1 + +/* DMA_CH_0_RD_RATE_LIM_RST_TOKEN */ +#define DMA_CH_0_RD_RATE_LIM_RST_TOKEN_VAL_SHIFT 0 +#define DMA_CH_0_RD_RATE_LIM_RST_TOKEN_VAL_MASK 0xFFFF + +/* DMA_CH_0_RD_RATE_LIM_SAT */ +#define DMA_CH_0_RD_RATE_LIM_SAT_VAL_SHIFT 0 +#define DMA_CH_0_RD_RATE_LIM_SAT_VAL_MASK 0xFFFF + +/* DMA_CH_0_RD_RATE_LIM_TOUT */ +#define DMA_CH_0_RD_RATE_LIM_TOUT_VAL_SHIFT 0 +#define DMA_CH_0_RD_RATE_LIM_TOUT_VAL_MASK 0x7FFFFFFF + +/* DMA_CH_0_WR_RATE_LIM_EN */ +#define DMA_CH_0_WR_RATE_LIM_EN_VAL_SHIFT 0 +#define DMA_CH_0_WR_RATE_LIM_EN_VAL_MASK 0x1 + +/* DMA_CH_0_WR_RATE_LIM_RST_TOKEN */ +#define DMA_CH_0_WR_RATE_LIM_RST_TOKEN_VAL_SHIFT 0 +#define DMA_CH_0_WR_RATE_LIM_RST_TOKEN_VAL_MASK 0xFFFF + +/* DMA_CH_0_WR_RATE_LIM_SAT */ +#define DMA_CH_0_WR_RATE_LIM_SAT_VAL_SHIFT 0 +#define DMA_CH_0_WR_RATE_LIM_SAT_VAL_MASK 0xFFFF + +/* DMA_CH_0_WR_RATE_LIM_TOUT */ +#define DMA_CH_0_WR_RATE_LIM_TOUT_VAL_SHIFT 0 +#define DMA_CH_0_WR_RATE_LIM_TOUT_VAL_MASK 0x7FFFFFFF + +/* DMA_CH_0_CFG2 */ +#define DMA_CH_0_CFG2_FORCE_WORD_SHIFT 0 +#define DMA_CH_0_CFG2_FORCE_WORD_MASK 0x1 + +/* DMA_CH_0_TDMA_CTL */ +#define DMA_CH_0_TDMA_CTL_DTYPE_SHIFT 0 +#define DMA_CH_0_TDMA_CTL_DTYPE_MASK 0x7 + +/* DMA_CH_0_TDMA_SRC_BASE_ADDR_LO */ +#define DMA_CH_0_TDMA_SRC_BASE_ADDR_LO_VAL_SHIFT 0 +#define DMA_CH_0_TDMA_SRC_BASE_ADDR_LO_VAL_MASK 0xFFFFFFFF + +/* DMA_CH_0_TDMA_SRC_BASE_ADDR_HI */ +#define DMA_CH_0_TDMA_SRC_BASE_ADDR_HI_VAL_SHIFT 0 +#define DMA_CH_0_TDMA_SRC_BASE_ADDR_HI_VAL_MASK 0xFFFFFFFF + +/* DMA_CH_0_TDMA_SRC_ROI_BASE_0 */ +#define DMA_CH_0_TDMA_SRC_ROI_BASE_0_VAL_SHIFT 0 +#define DMA_CH_0_TDMA_SRC_ROI_BASE_0_VAL_MASK 0xFFFFFFFF + +/* DMA_CH_0_TDMA_SRC_ROI_SIZE_0 */ +#define DMA_CH_0_TDMA_SRC_ROI_SIZE_0_VAL_SHIFT 0 +#define DMA_CH_0_TDMA_SRC_ROI_SIZE_0_VAL_MASK 0xFFFFFFFF + +/* DMA_CH_0_TDMA_SRC_VALID_ELEMENTS_0 */ +#define DMA_CH_0_TDMA_SRC_VALID_ELEMENTS_0_VAL_SHIFT 0 +#define DMA_CH_0_TDMA_SRC_VALID_ELEMENTS_0_VAL_MASK 0xFFFFFFFF + +/* DMA_CH_0_TDMA_SRC_START_OFFSET_0 */ +#define DMA_CH_0_TDMA_SRC_START_OFFSET_0_VAL_SHIFT 0 +#define DMA_CH_0_TDMA_SRC_START_OFFSET_0_VAL_MASK 0xFFFFFFFF + +/* DMA_CH_0_TDMA_SRC_STRIDE_0 */ +#define DMA_CH_0_TDMA_SRC_STRIDE_0_VAL_SHIFT 0 +#define DMA_CH_0_TDMA_SRC_STRIDE_0_VAL_MASK 0xFFFFFFFF + +/* DMA_CH_0_TDMA_SRC_ROI_BASE_1 */ +#define DMA_CH_0_TDMA_SRC_ROI_BASE_1_VAL_SHIFT 0 +#define DMA_CH_0_TDMA_SRC_ROI_BASE_1_VAL_MASK 0xFFFFFFFF + +/* DMA_CH_0_TDMA_SRC_ROI_SIZE_1 */ +#define DMA_CH_0_TDMA_SRC_ROI_SIZE_1_VAL_SHIFT 0 +#define DMA_CH_0_TDMA_SRC_ROI_SIZE_1_VAL_MASK 0xFFFFFFFF + +/* DMA_CH_0_TDMA_SRC_VALID_ELEMENTS_1 */ +#define DMA_CH_0_TDMA_SRC_VALID_ELEMENTS_1_VAL_SHIFT 0 +#define DMA_CH_0_TDMA_SRC_VALID_ELEMENTS_1_VAL_MASK 0xFFFFFFFF + +/* DMA_CH_0_TDMA_SRC_START_OFFSET_1 */ +#define DMA_CH_0_TDMA_SRC_START_OFFSET_1_VAL_SHIFT 0 +#define DMA_CH_0_TDMA_SRC_START_OFFSET_1_VAL_MASK 0xFFFFFFFF + +/* DMA_CH_0_TDMA_SRC_STRIDE_1 */ +#define DMA_CH_0_TDMA_SRC_STRIDE_1_VAL_SHIFT 0 +#define DMA_CH_0_TDMA_SRC_STRIDE_1_VAL_MASK 0xFFFFFFFF + +/* DMA_CH_0_TDMA_SRC_ROI_BASE_2 */ +#define DMA_CH_0_TDMA_SRC_ROI_BASE_2_VAL_SHIFT 0 +#define DMA_CH_0_TDMA_SRC_ROI_BASE_2_VAL_MASK 0xFFFFFFFF + +/* DMA_CH_0_TDMA_SRC_ROI_SIZE_2 */ +#define DMA_CH_0_TDMA_SRC_ROI_SIZE_2_VAL_SHIFT 0 +#define DMA_CH_0_TDMA_SRC_ROI_SIZE_2_VAL_MASK 0xFFFFFFFF + +/* DMA_CH_0_TDMA_SRC_VALID_ELEMENTS_2 */ +#define DMA_CH_0_TDMA_SRC_VALID_ELEMENTS_2_VAL_SHIFT 0 +#define DMA_CH_0_TDMA_SRC_VALID_ELEMENTS_2_VAL_MASK 0xFFFFFFFF + +/* DMA_CH_0_TDMA_SRC_START_OFFSET_2 */ +#define DMA_CH_0_TDMA_SRC_START_OFFSET_2_VAL_SHIFT 0 +#define DMA_CH_0_TDMA_SRC_START_OFFSET_2_VAL_MASK 0xFFFFFFFF + +/* DMA_CH_0_TDMA_SRC_STRIDE_2 */ +#define DMA_CH_0_TDMA_SRC_STRIDE_2_VAL_SHIFT 0 +#define DMA_CH_0_TDMA_SRC_STRIDE_2_VAL_MASK 0xFFFFFFFF + +/* DMA_CH_0_TDMA_SRC_ROI_BASE_3 */ +#define DMA_CH_0_TDMA_SRC_ROI_BASE_3_VAL_SHIFT 0 +#define DMA_CH_0_TDMA_SRC_ROI_BASE_3_VAL_MASK 0xFFFFFFFF + +/* DMA_CH_0_TDMA_SRC_ROI_SIZE_3 */ +#define DMA_CH_0_TDMA_SRC_ROI_SIZE_3_VAL_SHIFT 0 +#define DMA_CH_0_TDMA_SRC_ROI_SIZE_3_VAL_MASK 0xFFFFFFFF + +/* DMA_CH_0_TDMA_SRC_VALID_ELEMENTS_3 */ +#define DMA_CH_0_TDMA_SRC_VALID_ELEMENTS_3_VAL_SHIFT 0 +#define DMA_CH_0_TDMA_SRC_VALID_ELEMENTS_3_VAL_MASK 0xFFFFFFFF + +/* DMA_CH_0_TDMA_SRC_START_OFFSET_3 */ +#define DMA_CH_0_TDMA_SRC_START_OFFSET_3_VAL_SHIFT 0 +#define DMA_CH_0_TDMA_SRC_START_OFFSET_3_VAL_MASK 0xFFFFFFFF + +/* DMA_CH_0_TDMA_SRC_STRIDE_3 */ +#define DMA_CH_0_TDMA_SRC_STRIDE_3_VAL_SHIFT 0 +#define DMA_CH_0_TDMA_SRC_STRIDE_3_VAL_MASK 0xFFFFFFFF + +/* DMA_CH_0_TDMA_SRC_ROI_BASE_4 */ +#define DMA_CH_0_TDMA_SRC_ROI_BASE_4_VAL_SHIFT 0 +#define DMA_CH_0_TDMA_SRC_ROI_BASE_4_VAL_MASK 0xFFFFFFFF + +/* DMA_CH_0_TDMA_SRC_ROI_SIZE_4 */ +#define DMA_CH_0_TDMA_SRC_ROI_SIZE_4_VAL_SHIFT 0 +#define DMA_CH_0_TDMA_SRC_ROI_SIZE_4_VAL_MASK 0xFFFFFFFF + +/* DMA_CH_0_TDMA_SRC_VALID_ELEMENTS_4 */ +#define DMA_CH_0_TDMA_SRC_VALID_ELEMENTS_4_VAL_SHIFT 0 +#define DMA_CH_0_TDMA_SRC_VALID_ELEMENTS_4_VAL_MASK 0xFFFFFFFF + +/* DMA_CH_0_TDMA_SRC_START_OFFSET_4 */ +#define DMA_CH_0_TDMA_SRC_START_OFFSET_4_VAL_SHIFT 0 +#define DMA_CH_0_TDMA_SRC_START_OFFSET_4_VAL_MASK 0xFFFFFFFF + +/* DMA_CH_0_TDMA_SRC_STRIDE_4 */ +#define DMA_CH_0_TDMA_SRC_STRIDE_4_VAL_SHIFT 0 +#define DMA_CH_0_TDMA_SRC_STRIDE_4_VAL_MASK 0xFFFFFFFF + +/* DMA_CH_0_TDMA_DST_BASE_ADDR_LO */ +#define DMA_CH_0_TDMA_DST_BASE_ADDR_LO_VAL_SHIFT 0 +#define DMA_CH_0_TDMA_DST_BASE_ADDR_LO_VAL_MASK 0xFFFFFFFF + +/* DMA_CH_0_TDMA_DST_BASE_ADDR_HI */ +#define DMA_CH_0_TDMA_DST_BASE_ADDR_HI_VAL_SHIFT 0 +#define DMA_CH_0_TDMA_DST_BASE_ADDR_HI_VAL_MASK 0xFFFFFFFF + +/* DMA_CH_0_TDMA_DST_ROI_BASE_0 */ +#define DMA_CH_0_TDMA_DST_ROI_BASE_0_VAL_SHIFT 0 +#define DMA_CH_0_TDMA_DST_ROI_BASE_0_VAL_MASK 0xFFFFFFFF + +/* DMA_CH_0_TDMA_DST_ROI_SIZE_0 */ +#define DMA_CH_0_TDMA_DST_ROI_SIZE_0_VAL_SHIFT 0 +#define DMA_CH_0_TDMA_DST_ROI_SIZE_0_VAL_MASK 0xFFFFFFFF + +/* DMA_CH_0_TDMA_DST_VALID_ELEMENTS_0 */ +#define DMA_CH_0_TDMA_DST_VALID_ELEMENTS_0_VAL_SHIFT 0 +#define DMA_CH_0_TDMA_DST_VALID_ELEMENTS_0_VAL_MASK 0xFFFFFFFF + +/* DMA_CH_0_TDMA_DST_START_OFFSET_0 */ +#define DMA_CH_0_TDMA_DST_START_OFFSET_0_VAL_SHIFT 0 +#define DMA_CH_0_TDMA_DST_START_OFFSET_0_VAL_MASK 0xFFFFFFFF + +/* DMA_CH_0_TDMA_DST_STRIDE_0 */ +#define DMA_CH_0_TDMA_DST_STRIDE_0_VAL_SHIFT 0 +#define DMA_CH_0_TDMA_DST_STRIDE_0_VAL_MASK 0xFFFFFFFF + +/* DMA_CH_0_TDMA_DST_ROI_BASE_1 */ +#define DMA_CH_0_TDMA_DST_ROI_BASE_1_VAL_SHIFT 0 +#define DMA_CH_0_TDMA_DST_ROI_BASE_1_VAL_MASK 0xFFFFFFFF + +/* DMA_CH_0_TDMA_DST_ROI_SIZE_1 */ +#define DMA_CH_0_TDMA_DST_ROI_SIZE_1_VAL_SHIFT 0 +#define DMA_CH_0_TDMA_DST_ROI_SIZE_1_VAL_MASK 0xFFFFFFFF + +/* DMA_CH_0_TDMA_DST_VALID_ELEMENTS_1 */ +#define DMA_CH_0_TDMA_DST_VALID_ELEMENTS_1_VAL_SHIFT 0 +#define DMA_CH_0_TDMA_DST_VALID_ELEMENTS_1_VAL_MASK 0xFFFFFFFF + +/* DMA_CH_0_TDMA_DST_START_OFFSET_1 */ +#define DMA_CH_0_TDMA_DST_START_OFFSET_1_VAL_SHIFT 0 +#define DMA_CH_0_TDMA_DST_START_OFFSET_1_VAL_MASK 0xFFFFFFFF + +/* DMA_CH_0_TDMA_DST_STRIDE_1 */ +#define DMA_CH_0_TDMA_DST_STRIDE_1_VAL_SHIFT 0 +#define DMA_CH_0_TDMA_DST_STRIDE_1_VAL_MASK 0xFFFFFFFF + +/* DMA_CH_0_TDMA_DST_ROI_BASE_2 */ +#define DMA_CH_0_TDMA_DST_ROI_BASE_2_VAL_SHIFT 0 +#define DMA_CH_0_TDMA_DST_ROI_BASE_2_VAL_MASK 0xFFFFFFFF + +/* DMA_CH_0_TDMA_DST_ROI_SIZE_2 */ +#define DMA_CH_0_TDMA_DST_ROI_SIZE_2_VAL_SHIFT 0 +#define DMA_CH_0_TDMA_DST_ROI_SIZE_2_VAL_MASK 0xFFFFFFFF + +/* DMA_CH_0_TDMA_DST_VALID_ELEMENTS_2 */ +#define DMA_CH_0_TDMA_DST_VALID_ELEMENTS_2_VAL_SHIFT 0 +#define DMA_CH_0_TDMA_DST_VALID_ELEMENTS_2_VAL_MASK 0xFFFFFFFF + +/* DMA_CH_0_TDMA_DST_START_OFFSET_2 */ +#define DMA_CH_0_TDMA_DST_START_OFFSET_2_VAL_SHIFT 0 +#define DMA_CH_0_TDMA_DST_START_OFFSET_2_VAL_MASK 0xFFFFFFFF + +/* DMA_CH_0_TDMA_DST_STRIDE_2 */ +#define DMA_CH_0_TDMA_DST_STRIDE_2_VAL_SHIFT 0 +#define DMA_CH_0_TDMA_DST_STRIDE_2_VAL_MASK 0xFFFFFFFF + +/* DMA_CH_0_TDMA_DST_ROI_BASE_3 */ +#define DMA_CH_0_TDMA_DST_ROI_BASE_3_VAL_SHIFT 0 +#define DMA_CH_0_TDMA_DST_ROI_BASE_3_VAL_MASK 0xFFFFFFFF + +/* DMA_CH_0_TDMA_DST_ROI_SIZE_3 */ +#define DMA_CH_0_TDMA_DST_ROI_SIZE_3_VAL_SHIFT 0 +#define DMA_CH_0_TDMA_DST_ROI_SIZE_3_VAL_MASK 0xFFFFFFFF + +/* DMA_CH_0_TDMA_DST_VALID_ELEMENTS_3 */ +#define DMA_CH_0_TDMA_DST_VALID_ELEMENTS_3_VAL_SHIFT 0 +#define DMA_CH_0_TDMA_DST_VALID_ELEMENTS_3_VAL_MASK 0xFFFFFFFF + +/* DMA_CH_0_TDMA_DST_START_OFFSET_3 */ +#define DMA_CH_0_TDMA_DST_START_OFFSET_3_VAL_SHIFT 0 +#define DMA_CH_0_TDMA_DST_START_OFFSET_3_VAL_MASK 0xFFFFFFFF + +/* DMA_CH_0_TDMA_DST_STRIDE_3 */ +#define DMA_CH_0_TDMA_DST_STRIDE_3_VAL_SHIFT 0 +#define DMA_CH_0_TDMA_DST_STRIDE_3_VAL_MASK 0xFFFFFFFF + +/* DMA_CH_0_TDMA_DST_ROI_BASE_4 */ +#define DMA_CH_0_TDMA_DST_ROI_BASE_4_VAL_SHIFT 0 +#define DMA_CH_0_TDMA_DST_ROI_BASE_4_VAL_MASK 0xFFFFFFFF + +/* DMA_CH_0_TDMA_DST_ROI_SIZE_4 */ +#define DMA_CH_0_TDMA_DST_ROI_SIZE_4_VAL_SHIFT 0 +#define DMA_CH_0_TDMA_DST_ROI_SIZE_4_VAL_MASK 0xFFFFFFFF + +/* DMA_CH_0_TDMA_DST_VALID_ELEMENTS_4 */ +#define DMA_CH_0_TDMA_DST_VALID_ELEMENTS_4_VAL_SHIFT 0 +#define DMA_CH_0_TDMA_DST_VALID_ELEMENTS_4_VAL_MASK 0xFFFFFFFF + +/* DMA_CH_0_TDMA_DST_START_OFFSET_4 */ +#define DMA_CH_0_TDMA_DST_START_OFFSET_4_VAL_SHIFT 0 +#define DMA_CH_0_TDMA_DST_START_OFFSET_4_VAL_MASK 0xFFFFFFFF + +/* DMA_CH_0_TDMA_DST_STRIDE_4 */ +#define DMA_CH_0_TDMA_DST_STRIDE_4_VAL_SHIFT 0 +#define DMA_CH_0_TDMA_DST_STRIDE_4_VAL_MASK 0xFFFFFFFF + +/* DMA_CH_0_MEM_INIT_BUSY */ +#define DMA_CH_0_MEM_INIT_BUSY_SBC_DATA_SHIFT 0 +#define DMA_CH_0_MEM_INIT_BUSY_SBC_DATA_MASK 0xFF +#define DMA_CH_0_MEM_INIT_BUSY_SBC_MD_SHIFT 8 +#define DMA_CH_0_MEM_INIT_BUSY_SBC_MD_MASK 0x100 + +#endif /* ASIC_REG_DMA_CH_0_MASKS_H_ */ diff --git a/drivers/misc/habanalabs/include/goya/asic_reg/goya_regs.h b/drivers/misc/habanalabs/include/goya/asic_reg/goya_regs.h index 506e71e201e1..19b0f0ef1d0b 100644 --- a/drivers/misc/habanalabs/include/goya/asic_reg/goya_regs.h +++ b/drivers/misc/habanalabs/include/goya/asic_reg/goya_regs.h @@ -88,6 +88,7 @@ #include "psoc_global_conf_masks.h" #include "dma_macro_masks.h" #include "dma_qm_0_masks.h" +#include "dma_ch_0_masks.h" #include "tpc0_qm_masks.h" #include "tpc0_cmdq_masks.h" #include "mme_qm_masks.h" diff --git a/drivers/misc/habanalabs/memory.c b/drivers/misc/habanalabs/memory.c index 693877e37fd8..42d237cae1dc 100644 --- a/drivers/misc/habanalabs/memory.c +++ b/drivers/misc/habanalabs/memory.c @@ -1657,17 +1657,10 @@ int hl_vm_init(struct hl_device *hdev) struct hl_vm *vm = &hdev->vm; int rc; - rc = hl_mmu_init(hdev); - if (rc) { - dev_err(hdev->dev, "Failed to init MMU\n"); - return rc; - } - vm->dram_pg_pool = gen_pool_create(__ffs(prop->dram_page_size), -1); if (!vm->dram_pg_pool) { dev_err(hdev->dev, "Failed to create dram page pool\n"); - rc = -ENOMEM; - goto pool_create_err; + return -ENOMEM; } kref_init(&vm->dram_pg_pool_refcount); @@ -1693,8 +1686,6 @@ int hl_vm_init(struct hl_device *hdev) pool_add_err: gen_pool_destroy(vm->dram_pg_pool); -pool_create_err: - hl_mmu_fini(hdev); return rc; } @@ -1724,7 +1715,5 @@ void hl_vm_fini(struct hl_device *hdev) dev_warn(hdev->dev, "dram_pg_pool was not destroyed on %s\n", __func__); - hl_mmu_fini(hdev); - vm->init_done = false; } diff --git a/drivers/misc/habanalabs/mmu.c b/drivers/misc/habanalabs/mmu.c index 10aee3141444..176c315836f1 100644 --- a/drivers/misc/habanalabs/mmu.c +++ b/drivers/misc/habanalabs/mmu.c @@ -241,8 +241,9 @@ static int dram_default_mapping_init(struct hl_ctx *ctx) hop2_pte_addr, hop3_pte_addr, pte_val; int rc, i, j, hop3_allocated = 0; - if (!hdev->dram_supports_virtual_memory || - !hdev->dram_default_page_mapping) + if ((!hdev->dram_supports_virtual_memory) || + (!hdev->dram_default_page_mapping) || + (ctx->asid == HL_KERNEL_ASID_ID)) return 0; num_of_hop3 = prop->dram_size_for_default_page_mapping; @@ -340,8 +341,9 @@ static void dram_default_mapping_fini(struct hl_ctx *ctx) hop2_pte_addr, hop3_pte_addr; int i, j; - if (!hdev->dram_supports_virtual_memory || - !hdev->dram_default_page_mapping) + if ((!hdev->dram_supports_virtual_memory) || + (!hdev->dram_default_page_mapping) || + (ctx->asid == HL_KERNEL_ASID_ID)) return; num_of_hop3 = prop->dram_size_for_default_page_mapping; @@ -385,12 +387,8 @@ static void dram_default_mapping_fini(struct hl_ctx *ctx) * @hdev: habanalabs device structure. * * This function does the following: - * - Allocate max_asid zeroed hop0 pgts so no mapping is available. - * - Enable MMU in H/W. - * - Invalidate the MMU cache. * - Create a pool of pages for pgt_infos. - * - * This function depends on DMA QMAN to be working! + * - Create a shadow table for pgt * * Return: 0 for success, non-zero for failure. */ @@ -915,6 +913,10 @@ int hl_mmu_map(struct hl_ctx *ctx, u64 virt_addr, u64 phys_addr, u32 page_size) return -EFAULT; } + WARN_ONCE((phys_addr & (real_page_size - 1)), + "Mapping 0x%llx with page size of 0x%x is erroneous! Address must be divisible by page size", + phys_addr, real_page_size); + npages = page_size / real_page_size; real_virt_addr = virt_addr; real_phys_addr = phys_addr; diff --git a/drivers/misc/habanalabs/pci.c b/drivers/misc/habanalabs/pci.c index 0e78a04d63f4..c98d88c7a5c6 100644 --- a/drivers/misc/habanalabs/pci.c +++ b/drivers/misc/habanalabs/pci.c @@ -10,6 +10,8 @@ #include <linux/pci.h> +#define HL_PLDM_PCI_ELBI_TIMEOUT_MSEC (HL_PCI_ELBI_TIMEOUT_MSEC * 10) + /** * hl_pci_bars_map() - Map PCI BARs. * @hdev: Pointer to hl_device structure. @@ -88,8 +90,14 @@ static int hl_pci_elbi_write(struct hl_device *hdev, u64 addr, u32 data) { struct pci_dev *pdev = hdev->pdev; ktime_t timeout; + u64 msec; u32 val; + if (hdev->pldm) + msec = HL_PLDM_PCI_ELBI_TIMEOUT_MSEC; + else + msec = HL_PCI_ELBI_TIMEOUT_MSEC; + /* Clear previous status */ pci_write_config_dword(pdev, mmPCI_CONFIG_ELBI_STS, 0); @@ -98,7 +106,7 @@ static int hl_pci_elbi_write(struct hl_device *hdev, u64 addr, u32 data) pci_write_config_dword(pdev, mmPCI_CONFIG_ELBI_CTRL, PCI_CONFIG_ELBI_CTRL_WRITE); - timeout = ktime_add_ms(ktime_get(), 10); + timeout = ktime_add_ms(ktime_get(), msec); for (;;) { pci_read_config_dword(pdev, mmPCI_CONFIG_ELBI_STS, &val); if (val & PCI_CONFIG_ELBI_STS_MASK) diff --git a/drivers/misc/habanalabs/sysfs.c b/drivers/misc/habanalabs/sysfs.c index c900ab15cceb..25eb46d29d88 100644 --- a/drivers/misc/habanalabs/sysfs.c +++ b/drivers/misc/habanalabs/sysfs.c @@ -328,10 +328,6 @@ static ssize_t pci_addr_show(struct device *dev, struct device_attribute *attr, { struct hl_device *hdev = dev_get_drvdata(dev); - /* Use dummy, fixed address for simulator */ - if (!hdev->pdev) - return sprintf(buf, "0000:%02d:00.0\n", hdev->id); - return sprintf(buf, "%04x:%02x:%02x.%x\n", pci_domain_nr(hdev->pdev->bus), hdev->pdev->bus->number, diff --git a/drivers/misc/isl29003.c b/drivers/misc/isl29003.c index 3431a825f24e..c12406f610d5 100644 --- a/drivers/misc/isl29003.c +++ b/drivers/misc/isl29003.c @@ -3,7 +3,7 @@ * isl29003.c - Linux kernel module for * Intersil ISL29003 ambient light sensor * - * See file:Documentation/misc-devices/isl29003 + * See file:Documentation/misc-devices/isl29003.rst * * Copyright (c) 2009 Daniel Mack <daniel@caiaq.de> * @@ -377,7 +377,7 @@ static int isl29003_init_client(struct i2c_client *client) static int isl29003_probe(struct i2c_client *client, const struct i2c_device_id *id) { - struct i2c_adapter *adapter = to_i2c_adapter(client->dev.parent); + struct i2c_adapter *adapter = client->adapter; struct isl29003_data *data; int err = 0; diff --git a/drivers/misc/lis3lv02d/Kconfig b/drivers/misc/lis3lv02d/Kconfig index 4cfad45229c8..bb2fec4b5880 100644 --- a/drivers/misc/lis3lv02d/Kconfig +++ b/drivers/misc/lis3lv02d/Kconfig @@ -7,7 +7,6 @@ config SENSORS_LIS3_SPI tristate "STMicroeletronics LIS3LV02Dx three-axis digital accelerometer (SPI)" depends on !ACPI && SPI_MASTER && INPUT select SENSORS_LIS3LV02D - default n help This driver provides support for the LIS3LV02Dx accelerometer connected via SPI. The accelerometer data is readable via @@ -24,7 +23,6 @@ config SENSORS_LIS3_I2C tristate "STMicroeletronics LIS3LV02Dx three-axis digital accelerometer (I2C)" depends on I2C && INPUT select SENSORS_LIS3LV02D - default n help This driver provides support for the LIS3LV02Dx accelerometer connected via I2C. The accelerometer data is readable via diff --git a/drivers/misc/lkdtm/Makefile b/drivers/misc/lkdtm/Makefile index 951c984de61a..fb10eafe9bde 100644 --- a/drivers/misc/lkdtm/Makefile +++ b/drivers/misc/lkdtm/Makefile @@ -15,8 +15,7 @@ KCOV_INSTRUMENT_rodata.o := n OBJCOPYFLAGS := OBJCOPYFLAGS_rodata_objcopy.o := \ - --set-section-flags .text=alloc,readonly \ - --rename-section .text=.rodata + --rename-section .text=.rodata,alloc,readonly,load targets += rodata.o rodata_objcopy.o $(obj)/rodata_objcopy.o: $(obj)/rodata.o FORCE $(call if_changed,objcopy) diff --git a/drivers/misc/lkdtm/bugs.c b/drivers/misc/lkdtm/bugs.c index d9fcfd3b5af0..1606658b9b7e 100644 --- a/drivers/misc/lkdtm/bugs.c +++ b/drivers/misc/lkdtm/bugs.c @@ -266,3 +266,69 @@ void lkdtm_STACK_GUARD_PAGE_TRAILING(void) pr_err("FAIL: accessed page after stack!\n"); } + +void lkdtm_UNSET_SMEP(void) +{ +#ifdef CONFIG_X86_64 +#define MOV_CR4_DEPTH 64 + void (*direct_write_cr4)(unsigned long val); + unsigned char *insn; + unsigned long cr4; + int i; + + cr4 = native_read_cr4(); + + if ((cr4 & X86_CR4_SMEP) != X86_CR4_SMEP) { + pr_err("FAIL: SMEP not in use\n"); + return; + } + cr4 &= ~(X86_CR4_SMEP); + + pr_info("trying to clear SMEP normally\n"); + native_write_cr4(cr4); + if (cr4 == native_read_cr4()) { + pr_err("FAIL: pinning SMEP failed!\n"); + cr4 |= X86_CR4_SMEP; + pr_info("restoring SMEP\n"); + native_write_cr4(cr4); + return; + } + pr_info("ok: SMEP did not get cleared\n"); + + /* + * To test the post-write pinning verification we need to call + * directly into the middle of native_write_cr4() where the + * cr4 write happens, skipping any pinning. This searches for + * the cr4 writing instruction. + */ + insn = (unsigned char *)native_write_cr4; + for (i = 0; i < MOV_CR4_DEPTH; i++) { + /* mov %rdi, %cr4 */ + if (insn[i] == 0x0f && insn[i+1] == 0x22 && insn[i+2] == 0xe7) + break; + /* mov %rdi,%rax; mov %rax, %cr4 */ + if (insn[i] == 0x48 && insn[i+1] == 0x89 && + insn[i+2] == 0xf8 && insn[i+3] == 0x0f && + insn[i+4] == 0x22 && insn[i+5] == 0xe0) + break; + } + if (i >= MOV_CR4_DEPTH) { + pr_info("ok: cannot locate cr4 writing call gadget\n"); + return; + } + direct_write_cr4 = (void *)(insn + i); + + pr_info("trying to clear SMEP with call gadget\n"); + direct_write_cr4(cr4); + if (native_read_cr4() & X86_CR4_SMEP) { + pr_info("ok: SMEP removal was reverted\n"); + } else { + pr_err("FAIL: cleared SMEP not detected!\n"); + cr4 |= X86_CR4_SMEP; + pr_info("restoring SMEP\n"); + native_write_cr4(cr4); + } +#else + pr_err("FAIL: this test is x86_64-only\n"); +#endif +} diff --git a/drivers/misc/lkdtm/core.c b/drivers/misc/lkdtm/core.c index bba49abb6750..756794ac8fd8 100644 --- a/drivers/misc/lkdtm/core.c +++ b/drivers/misc/lkdtm/core.c @@ -114,6 +114,7 @@ static const struct crashtype crashtypes[] = { CRASHTYPE(CORRUPT_USER_DS), CRASHTYPE(STACK_GUARD_PAGE_LEADING), CRASHTYPE(STACK_GUARD_PAGE_TRAILING), + CRASHTYPE(UNSET_SMEP), CRASHTYPE(UNALIGNED_LOAD_STORE_WRITE), CRASHTYPE(OVERWRITE_ALLOCATION), CRASHTYPE(WRITE_AFTER_FREE), diff --git a/drivers/misc/lkdtm/lkdtm.h b/drivers/misc/lkdtm/lkdtm.h index 23dc565b4307..bbcd370786d4 100644 --- a/drivers/misc/lkdtm/lkdtm.h +++ b/drivers/misc/lkdtm/lkdtm.h @@ -26,6 +26,7 @@ void lkdtm_CORRUPT_LIST_DEL(void); void lkdtm_CORRUPT_USER_DS(void); void lkdtm_STACK_GUARD_PAGE_LEADING(void); void lkdtm_STACK_GUARD_PAGE_TRAILING(void); +void lkdtm_UNSET_SMEP(void); /* lkdtm_heap.c */ void lkdtm_OVERWRITE_ALLOCATION(void); diff --git a/drivers/misc/mei/debugfs.c b/drivers/misc/mei/debugfs.c index 0970142bcace..47cfd5005e1b 100644 --- a/drivers/misc/mei/debugfs.c +++ b/drivers/misc/mei/debugfs.c @@ -8,6 +8,7 @@ #include <linux/kernel.h> #include <linux/device.h> #include <linux/debugfs.h> +#include <linux/seq_file.h> #include <linux/mei.h> @@ -15,104 +16,56 @@ #include "client.h" #include "hw.h" -static ssize_t mei_dbgfs_read_meclients(struct file *fp, char __user *ubuf, - size_t cnt, loff_t *ppos) +static int mei_dbgfs_meclients_show(struct seq_file *m, void *unused) { - struct mei_device *dev = fp->private_data; + struct mei_device *dev = m->private; struct mei_me_client *me_cl; - size_t bufsz = 1; - char *buf; int i = 0; - int pos = 0; - int ret; -#define HDR \ -" |id|fix| UUID |con|msg len|sb|refc|\n" + if (!dev) + return -ENODEV; down_read(&dev->me_clients_rwsem); - list_for_each_entry(me_cl, &dev->me_clients, list) - bufsz++; - bufsz *= sizeof(HDR) + 1; - buf = kzalloc(bufsz, GFP_KERNEL); - if (!buf) { - up_read(&dev->me_clients_rwsem); - return -ENOMEM; - } - - pos += scnprintf(buf + pos, bufsz - pos, HDR); -#undef HDR + seq_puts(m, " |id|fix| UUID |con|msg len|sb|refc|\n"); /* if the driver is not enabled the list won't be consistent */ if (dev->dev_state != MEI_DEV_ENABLED) goto out; list_for_each_entry(me_cl, &dev->me_clients, list) { - - if (mei_me_cl_get(me_cl)) { - pos += scnprintf(buf + pos, bufsz - pos, - "%2d|%2d|%3d|%pUl|%3d|%7d|%2d|%4d|\n", - i++, me_cl->client_id, - me_cl->props.fixed_address, - &me_cl->props.protocol_name, - me_cl->props.max_number_of_connections, - me_cl->props.max_msg_length, - me_cl->props.single_recv_buf, - kref_read(&me_cl->refcnt)); - - mei_me_cl_put(me_cl); - } + if (!mei_me_cl_get(me_cl)) + continue; + + seq_printf(m, "%2d|%2d|%3d|%pUl|%3d|%7d|%2d|%4d|\n", + i++, me_cl->client_id, + me_cl->props.fixed_address, + &me_cl->props.protocol_name, + me_cl->props.max_number_of_connections, + me_cl->props.max_msg_length, + me_cl->props.single_recv_buf, + kref_read(&me_cl->refcnt)); + mei_me_cl_put(me_cl); } out: up_read(&dev->me_clients_rwsem); - ret = simple_read_from_buffer(ubuf, cnt, ppos, buf, pos); - kfree(buf); - return ret; + return 0; } +DEFINE_SHOW_ATTRIBUTE(mei_dbgfs_meclients); -static const struct file_operations mei_dbgfs_fops_meclients = { - .open = simple_open, - .read = mei_dbgfs_read_meclients, - .llseek = generic_file_llseek, -}; - -static ssize_t mei_dbgfs_read_active(struct file *fp, char __user *ubuf, - size_t cnt, loff_t *ppos) +static int mei_dbgfs_active_show(struct seq_file *m, void *unused) { - struct mei_device *dev = fp->private_data; + struct mei_device *dev = m->private; struct mei_cl *cl; - size_t bufsz = 1; - char *buf; int i = 0; - int pos = 0; - int ret; - -#define HDR " |me|host|state|rd|wr|wrq\n" if (!dev) return -ENODEV; mutex_lock(&dev->device_lock); - /* - * if the driver is not enabled the list won't be consistent, - * we output empty table - */ - if (dev->dev_state == MEI_DEV_ENABLED) - list_for_each_entry(cl, &dev->file_list, link) - bufsz++; - - bufsz *= sizeof(HDR) + 1; - - buf = kzalloc(bufsz, GFP_KERNEL); - if (!buf) { - mutex_unlock(&dev->device_lock); - return -ENOMEM; - } - - pos += scnprintf(buf + pos, bufsz - pos, HDR); -#undef HDR + seq_puts(m, " |me|host|state|rd|wr|wrq\n"); /* if the driver is not enabled the list won't be consistent */ if (dev->dev_state != MEI_DEV_ENABLED) @@ -120,76 +73,44 @@ static ssize_t mei_dbgfs_read_active(struct file *fp, char __user *ubuf, list_for_each_entry(cl, &dev->file_list, link) { - pos += scnprintf(buf + pos, bufsz - pos, - "%3d|%2d|%4d|%5d|%2d|%2d|%3u\n", - i, mei_cl_me_id(cl), cl->host_client_id, cl->state, - !list_empty(&cl->rd_completed), cl->writing_state, - cl->tx_cb_queued); + seq_printf(m, "%3d|%2d|%4d|%5d|%2d|%2d|%3u\n", + i, mei_cl_me_id(cl), cl->host_client_id, cl->state, + !list_empty(&cl->rd_completed), cl->writing_state, + cl->tx_cb_queued); i++; } out: mutex_unlock(&dev->device_lock); - ret = simple_read_from_buffer(ubuf, cnt, ppos, buf, pos); - kfree(buf); - return ret; + return 0; } +DEFINE_SHOW_ATTRIBUTE(mei_dbgfs_active); -static const struct file_operations mei_dbgfs_fops_active = { - .open = simple_open, - .read = mei_dbgfs_read_active, - .llseek = generic_file_llseek, -}; - -static ssize_t mei_dbgfs_read_devstate(struct file *fp, char __user *ubuf, - size_t cnt, loff_t *ppos) +static int mei_dbgfs_devstate_show(struct seq_file *m, void *unused) { - struct mei_device *dev = fp->private_data; - const size_t bufsz = 1024; - char *buf = kzalloc(bufsz, GFP_KERNEL); - int pos = 0; - int ret; - - if (!buf) - return -ENOMEM; + struct mei_device *dev = m->private; - pos += scnprintf(buf + pos, bufsz - pos, "dev: %s\n", - mei_dev_state_str(dev->dev_state)); - pos += scnprintf(buf + pos, bufsz - pos, "hbm: %s\n", - mei_hbm_state_str(dev->hbm_state)); + seq_printf(m, "dev: %s\n", mei_dev_state_str(dev->dev_state)); + seq_printf(m, "hbm: %s\n", mei_hbm_state_str(dev->hbm_state)); if (dev->hbm_state >= MEI_HBM_ENUM_CLIENTS && dev->hbm_state <= MEI_HBM_STARTED) { - pos += scnprintf(buf + pos, bufsz - pos, "hbm features:\n"); - pos += scnprintf(buf + pos, bufsz - pos, "\tPG: %01d\n", - dev->hbm_f_pg_supported); - pos += scnprintf(buf + pos, bufsz - pos, "\tDC: %01d\n", - dev->hbm_f_dc_supported); - pos += scnprintf(buf + pos, bufsz - pos, "\tIE: %01d\n", - dev->hbm_f_ie_supported); - pos += scnprintf(buf + pos, bufsz - pos, "\tDOT: %01d\n", - dev->hbm_f_dot_supported); - pos += scnprintf(buf + pos, bufsz - pos, "\tEV: %01d\n", - dev->hbm_f_ev_supported); - pos += scnprintf(buf + pos, bufsz - pos, "\tFA: %01d\n", - dev->hbm_f_fa_supported); - pos += scnprintf(buf + pos, bufsz - pos, "\tOS: %01d\n", - dev->hbm_f_os_supported); - pos += scnprintf(buf + pos, bufsz - pos, "\tDR: %01d\n", - dev->hbm_f_dr_supported); + seq_puts(m, "hbm features:\n"); + seq_printf(m, "\tPG: %01d\n", dev->hbm_f_pg_supported); + seq_printf(m, "\tDC: %01d\n", dev->hbm_f_dc_supported); + seq_printf(m, "\tIE: %01d\n", dev->hbm_f_ie_supported); + seq_printf(m, "\tDOT: %01d\n", dev->hbm_f_dot_supported); + seq_printf(m, "\tEV: %01d\n", dev->hbm_f_ev_supported); + seq_printf(m, "\tFA: %01d\n", dev->hbm_f_fa_supported); + seq_printf(m, "\tOS: %01d\n", dev->hbm_f_os_supported); + seq_printf(m, "\tDR: %01d\n", dev->hbm_f_dr_supported); } - pos += scnprintf(buf + pos, bufsz - pos, "pg: %s, %s\n", - mei_pg_is_enabled(dev) ? "ENABLED" : "DISABLED", - mei_pg_state_str(mei_pg_state(dev))); - ret = simple_read_from_buffer(ubuf, cnt, ppos, buf, pos); - kfree(buf); - return ret; + seq_printf(m, "pg: %s, %s\n", + mei_pg_is_enabled(dev) ? "ENABLED" : "DISABLED", + mei_pg_state_str(mei_pg_state(dev))); + return 0; } -static const struct file_operations mei_dbgfs_fops_devstate = { - .open = simple_open, - .read = mei_dbgfs_read_devstate, - .llseek = generic_file_llseek, -}; +DEFINE_SHOW_ATTRIBUTE(mei_dbgfs_devstate); static ssize_t mei_dbgfs_write_allow_fa(struct file *file, const char __user *user_buf, @@ -208,7 +129,7 @@ static ssize_t mei_dbgfs_write_allow_fa(struct file *file, return ret; } -static const struct file_operations mei_dbgfs_fops_allow_fa = { +static const struct file_operations mei_dbgfs_allow_fa_fops = { .open = simple_open, .read = debugfs_read_file_bool, .write = mei_dbgfs_write_allow_fa, @@ -247,26 +168,26 @@ int mei_dbgfs_register(struct mei_device *dev, const char *name) dev->dbgfs_dir = dir; f = debugfs_create_file("meclients", S_IRUSR, dir, - dev, &mei_dbgfs_fops_meclients); + dev, &mei_dbgfs_meclients_fops); if (!f) { dev_err(dev->dev, "meclients: registration failed\n"); goto err; } f = debugfs_create_file("active", S_IRUSR, dir, - dev, &mei_dbgfs_fops_active); + dev, &mei_dbgfs_active_fops); if (!f) { dev_err(dev->dev, "active: registration failed\n"); goto err; } f = debugfs_create_file("devstate", S_IRUSR, dir, - dev, &mei_dbgfs_fops_devstate); + dev, &mei_dbgfs_devstate_fops); if (!f) { dev_err(dev->dev, "devstate: registration failed\n"); goto err; } f = debugfs_create_file("allow_fixed_address", S_IRUSR | S_IWUSR, dir, &dev->allow_fixed_address, - &mei_dbgfs_fops_allow_fa); + &mei_dbgfs_allow_fa_fops); if (!f) { dev_err(dev->dev, "allow_fixed_address: registration failed\n"); goto err; @@ -276,4 +197,3 @@ err: mei_dbgfs_deregister(dev); return -ENODEV; } - diff --git a/drivers/misc/mei/hdcp/mei_hdcp.c b/drivers/misc/mei/hdcp/mei_hdcp.c index b07000202d4a..ed816939fb32 100644 --- a/drivers/misc/mei/hdcp/mei_hdcp.c +++ b/drivers/misc/mei/hdcp/mei_hdcp.c @@ -2,7 +2,7 @@ /* * Copyright © 2019 Intel Corporation * - * Mei_hdcp.c: HDCP client driver for mei bus + * mei_hdcp.c: HDCP client driver for mei bus * * Author: * Ramalingam C <ramalingam.c@intel.com> @@ -11,12 +11,9 @@ /** * DOC: MEI_HDCP Client Driver * - * This is a client driver to the mei_bus to make the HDCP2.2 services of - * ME FW available for the interested consumers like I915. - * - * This module will act as a translation layer between HDCP protocol - * implementor(I915) and ME FW by translating HDCP2.2 authentication - * messages to ME FW command payloads and vice versa. + * The mei_hdcp driver acts as a translation layer between HDCP 2.2 + * protocol implementer (I915) and ME FW by translating HDCP2.2 + * negotiation messages to ME FW command payloads and vice versa. */ #include <linux/module.h> diff --git a/drivers/misc/mic/scif/scif_main.c b/drivers/misc/mic/scif/scif_main.c index 490e3bdc1941..e2278bf9f11d 100644 --- a/drivers/misc/mic/scif/scif_main.c +++ b/drivers/misc/mic/scif/scif_main.c @@ -133,6 +133,7 @@ static int scif_setup_scifdev(void) static void scif_destroy_scifdev(void) { kfree(scif_dev); + scif_dev = NULL; } static int scif_probe(struct scif_hw_dev *sdev) diff --git a/drivers/misc/ocxl/Kconfig b/drivers/misc/ocxl/Kconfig index 7fb6d39d4c5a..1916fa65f2f2 100644 --- a/drivers/misc/ocxl/Kconfig +++ b/drivers/misc/ocxl/Kconfig @@ -5,7 +5,6 @@ config OCXL_BASE bool - default n select PPC_COPRO_BASE config OCXL diff --git a/drivers/misc/ocxl/context.c b/drivers/misc/ocxl/context.c index bab9c9364184..994563a078eb 100644 --- a/drivers/misc/ocxl/context.c +++ b/drivers/misc/ocxl/context.c @@ -69,6 +69,7 @@ static void xsl_fault_error(void *data, u64 addr, u64 dsisr) int ocxl_context_attach(struct ocxl_context *ctx, u64 amr, struct mm_struct *mm) { int rc; + unsigned long pidr = 0; // Locks both status & tidr mutex_lock(&ctx->status_mutex); @@ -77,9 +78,11 @@ int ocxl_context_attach(struct ocxl_context *ctx, u64 amr, struct mm_struct *mm) goto out; } - rc = ocxl_link_add_pe(ctx->afu->fn->link, ctx->pasid, - mm->context.id, ctx->tidr, amr, mm, - xsl_fault_error, ctx); + if (mm) + pidr = mm->context.id; + + rc = ocxl_link_add_pe(ctx->afu->fn->link, ctx->pasid, pidr, ctx->tidr, + amr, mm, xsl_fault_error, ctx); if (rc) goto out; diff --git a/drivers/misc/ocxl/link.c b/drivers/misc/ocxl/link.c index cce5b0d64505..58d111afd9f6 100644 --- a/drivers/misc/ocxl/link.c +++ b/drivers/misc/ocxl/link.c @@ -224,6 +224,17 @@ static irqreturn_t xsl_fault_handler(int irq, void *data) ack_irq(spa, ADDRESS_ERROR); return IRQ_HANDLED; } + + if (!pe_data->mm) { + /* + * translation fault from a kernel context - an OpenCAPI + * device tried to access a bad kernel address + */ + rcu_read_unlock(); + pr_warn("Unresolved OpenCAPI xsl fault in kernel context\n"); + ack_irq(spa, ADDRESS_ERROR); + return IRQ_HANDLED; + } WARN_ON(pe_data->mm->context.id != pid); if (mmget_not_zero(pe_data->mm)) { @@ -523,7 +534,13 @@ int ocxl_link_add_pe(void *link_handle, int pasid, u32 pidr, u32 tidr, pe->amr = cpu_to_be64(amr); pe->software_state = cpu_to_be32(SPA_PE_VALID); - mm_context_add_copro(mm); + /* + * For user contexts, register a copro so that TLBIs are seen + * by the nest MMU. If we have a kernel context, TLBIs are + * already global. + */ + if (mm) + mm_context_add_copro(mm); /* * Barrier is to make sure PE is visible in the SPA before it * is used by the device. It also helps with the global TLBI @@ -546,7 +563,8 @@ int ocxl_link_add_pe(void *link_handle, int pasid, u32 pidr, u32 tidr, * have a reference on mm_users. Incrementing mm_count solves * the problem. */ - mmgrab(mm); + if (mm) + mmgrab(mm); trace_ocxl_context_add(current->pid, spa->spa_mem, pasid, pidr, tidr); unlock: mutex_unlock(&spa->spa_lock); @@ -652,8 +670,10 @@ int ocxl_link_remove_pe(void *link_handle, int pasid) if (!pe_data) { WARN(1, "Couldn't find pe data when removing PE\n"); } else { - mm_context_remove_copro(pe_data->mm); - mmdrop(pe_data->mm); + if (pe_data->mm) { + mm_context_remove_copro(pe_data->mm); + mmdrop(pe_data->mm); + } kfree_rcu(pe_data, rcu); } unlock: diff --git a/drivers/misc/sgi-xp/xpc_partition.c b/drivers/misc/sgi-xp/xpc_partition.c index 3eba1c420cc0..782ce95d3f17 100644 --- a/drivers/misc/sgi-xp/xpc_partition.c +++ b/drivers/misc/sgi-xp/xpc_partition.c @@ -70,7 +70,7 @@ xpc_get_rsvd_page_pa(int nasid) unsigned long rp_pa = nasid; /* seed with nasid */ size_t len = 0; size_t buf_len = 0; - void *buf = buf; + void *buf = NULL; void *buf_base = NULL; enum xp_retval (*get_partition_rsvd_page_pa) (void *, u64 *, unsigned long *, size_t *) = diff --git a/drivers/misc/tsl2550.c b/drivers/misc/tsl2550.c index 5b7afd6190fe..09db397df287 100644 --- a/drivers/misc/tsl2550.c +++ b/drivers/misc/tsl2550.c @@ -336,7 +336,7 @@ static struct i2c_driver tsl2550_driver; static int tsl2550_probe(struct i2c_client *client, const struct i2c_device_id *id) { - struct i2c_adapter *adapter = to_i2c_adapter(client->dev.parent); + struct i2c_adapter *adapter = client->adapter; struct tsl2550_data *data; int *opmode, err = 0; diff --git a/drivers/misc/vmw_balloon.c b/drivers/misc/vmw_balloon.c index ad807d5a3141..043eed845246 100644 --- a/drivers/misc/vmw_balloon.c +++ b/drivers/misc/vmw_balloon.c @@ -28,6 +28,8 @@ #include <linux/rwsem.h> #include <linux/slab.h> #include <linux/spinlock.h> +#include <linux/mount.h> +#include <linux/balloon_compaction.h> #include <linux/vmw_vmci_defs.h> #include <linux/vmw_vmci_api.h> #include <asm/hypervisor.h> @@ -38,25 +40,20 @@ MODULE_ALIAS("dmi:*:svnVMware*:*"); MODULE_ALIAS("vmware_vmmemctl"); MODULE_LICENSE("GPL"); -/* - * Use __GFP_HIGHMEM to allow pages from HIGHMEM zone. We don't allow wait - * (__GFP_RECLAIM) for huge page allocations. Use __GFP_NOWARN, to suppress page - * allocation failure warnings. Disallow access to emergency low-memory pools. - */ -#define VMW_HUGE_PAGE_ALLOC_FLAGS (__GFP_HIGHMEM|__GFP_NOWARN| \ - __GFP_NOMEMALLOC) +static bool __read_mostly vmwballoon_shrinker_enable; +module_param(vmwballoon_shrinker_enable, bool, 0444); +MODULE_PARM_DESC(vmwballoon_shrinker_enable, + "Enable non-cooperative out-of-memory protection. Disabled by default as it may degrade performance."); -/* - * Use __GFP_HIGHMEM to allow pages from HIGHMEM zone. We allow lightweight - * reclamation (__GFP_NORETRY). Use __GFP_NOWARN, to suppress page allocation - * failure warnings. Disallow access to emergency low-memory pools. - */ -#define VMW_PAGE_ALLOC_FLAGS (__GFP_HIGHMEM|__GFP_NOWARN| \ - __GFP_NOMEMALLOC|__GFP_NORETRY) +/* Delay in seconds after shrink before inflation. */ +#define VMBALLOON_SHRINK_DELAY (5) /* Maximum number of refused pages we accumulate during inflation cycle */ #define VMW_BALLOON_MAX_REFUSED 16 +/* Magic number for the balloon mount-point */ +#define BALLOON_VMW_MAGIC 0x0ba11007 + /* * Hypervisor communication port definitions. */ @@ -229,29 +226,26 @@ enum vmballoon_stat_general { VMW_BALLOON_STAT_TIMER, VMW_BALLOON_STAT_DOORBELL, VMW_BALLOON_STAT_RESET, - VMW_BALLOON_STAT_LAST = VMW_BALLOON_STAT_RESET + VMW_BALLOON_STAT_SHRINK, + VMW_BALLOON_STAT_SHRINK_FREE, + VMW_BALLOON_STAT_LAST = VMW_BALLOON_STAT_SHRINK_FREE }; #define VMW_BALLOON_STAT_NUM (VMW_BALLOON_STAT_LAST + 1) - static DEFINE_STATIC_KEY_TRUE(vmw_balloon_batching); static DEFINE_STATIC_KEY_FALSE(balloon_stat_enabled); struct vmballoon_ctl { struct list_head pages; struct list_head refused_pages; + struct list_head prealloc_pages; unsigned int n_refused_pages; unsigned int n_pages; enum vmballoon_page_size_type page_size; enum vmballoon_op op; }; -struct vmballoon_page_size { - /* list of reserved physical pages */ - struct list_head pages; -}; - /** * struct vmballoon_batch_entry - a batch entry for lock or unlock. * @@ -266,8 +260,6 @@ struct vmballoon_batch_entry { } __packed; struct vmballoon { - struct vmballoon_page_size page_sizes[VMW_BALLOON_NUM_PAGE_SIZES]; - /** * @max_page_size: maximum supported page size for ballooning. * @@ -340,6 +332,15 @@ struct vmballoon { */ struct page *page; + /** + * @shrink_timeout: timeout until the next inflation. + * + * After an shrink event, indicates the time in jiffies after which + * inflation is allowed again. Can be written concurrently with reads, + * so must use READ_ONCE/WRITE_ONCE when accessing. + */ + unsigned long shrink_timeout; + /* statistics */ struct vmballoon_stats *stats; @@ -348,9 +349,21 @@ struct vmballoon { struct dentry *dbg_entry; #endif + /** + * @b_dev_info: balloon device information descriptor. + */ + struct balloon_dev_info b_dev_info; + struct delayed_work dwork; /** + * @huge_pages - list of the inflated 2MB pages. + * + * Protected by @b_dev_info.pages_lock . + */ + struct list_head huge_pages; + + /** * @vmci_doorbell. * * Protected by @conf_sem. @@ -368,6 +381,20 @@ struct vmballoon { * Lock ordering: @conf_sem -> @comm_lock . */ spinlock_t comm_lock; + + /** + * @shrinker: shrinker interface that is used to avoid over-inflation. + */ + struct shrinker shrinker; + + /** + * @shrinker_registered: whether the shrinker was registered. + * + * The shrinker interface does not handle gracefully the removal of + * shrinker that was not registered before. This indication allows to + * simplify the unregistration process. + */ + bool shrinker_registered; }; static struct vmballoon balloon; @@ -642,15 +669,25 @@ static int vmballoon_alloc_page_list(struct vmballoon *b, unsigned int i; for (i = 0; i < req_n_pages; i++) { - if (ctl->page_size == VMW_BALLOON_2M_PAGE) - page = alloc_pages(VMW_HUGE_PAGE_ALLOC_FLAGS, - VMW_BALLOON_2M_ORDER); - else - page = alloc_page(VMW_PAGE_ALLOC_FLAGS); - - /* Update statistics */ - vmballoon_stats_page_inc(b, VMW_BALLOON_PAGE_STAT_ALLOC, - ctl->page_size); + /* + * First check if we happen to have pages that were allocated + * before. This happens when 2MB page rejected during inflation + * by the hypervisor, and then split into 4KB pages. + */ + if (!list_empty(&ctl->prealloc_pages)) { + page = list_first_entry(&ctl->prealloc_pages, + struct page, lru); + list_del(&page->lru); + } else { + if (ctl->page_size == VMW_BALLOON_2M_PAGE) + page = alloc_pages(__GFP_HIGHMEM|__GFP_NOWARN| + __GFP_NOMEMALLOC, VMW_BALLOON_2M_ORDER); + else + page = balloon_page_alloc(); + + vmballoon_stats_page_inc(b, VMW_BALLOON_PAGE_STAT_ALLOC, + ctl->page_size); + } if (page) { vmballoon_mark_page_offline(page, ctl->page_size); @@ -896,7 +933,8 @@ static void vmballoon_release_page_list(struct list_head *page_list, __free_pages(page, vmballoon_page_order(page_size)); } - *n_pages = 0; + if (n_pages) + *n_pages = 0; } @@ -942,6 +980,10 @@ static int64_t vmballoon_change(struct vmballoon *b) size - target < vmballoon_page_in_frames(VMW_BALLOON_2M_PAGE)) return 0; + /* If an out-of-memory recently occurred, inflation is disallowed. */ + if (target > size && time_before(jiffies, READ_ONCE(b->shrink_timeout))) + return 0; + return target - size; } @@ -961,9 +1003,22 @@ static void vmballoon_enqueue_page_list(struct vmballoon *b, unsigned int *n_pages, enum vmballoon_page_size_type page_size) { - struct vmballoon_page_size *page_size_info = &b->page_sizes[page_size]; + unsigned long flags; + + if (page_size == VMW_BALLOON_4K_PAGE) { + balloon_page_list_enqueue(&b->b_dev_info, pages); + } else { + /* + * Keep the huge pages in a local list which is not available + * for the balloon compaction mechanism. + */ + spin_lock_irqsave(&b->b_dev_info.pages_lock, flags); + list_splice_init(pages, &b->huge_pages); + __count_vm_events(BALLOON_INFLATE, *n_pages * + vmballoon_page_in_frames(VMW_BALLOON_2M_PAGE)); + spin_unlock_irqrestore(&b->b_dev_info.pages_lock, flags); + } - list_splice_init(pages, &page_size_info->pages); *n_pages = 0; } @@ -986,19 +1041,58 @@ static void vmballoon_dequeue_page_list(struct vmballoon *b, enum vmballoon_page_size_type page_size, unsigned int n_req_pages) { - struct vmballoon_page_size *page_size_info = &b->page_sizes[page_size]; struct page *page, *tmp; unsigned int i = 0; + unsigned long flags; + + /* In the case of 4k pages, use the compaction infrastructure */ + if (page_size == VMW_BALLOON_4K_PAGE) { + *n_pages = balloon_page_list_dequeue(&b->b_dev_info, pages, + n_req_pages); + return; + } - list_for_each_entry_safe(page, tmp, &page_size_info->pages, lru) { + /* 2MB pages */ + spin_lock_irqsave(&b->b_dev_info.pages_lock, flags); + list_for_each_entry_safe(page, tmp, &b->huge_pages, lru) { list_move(&page->lru, pages); if (++i == n_req_pages) break; } + + __count_vm_events(BALLOON_DEFLATE, + i * vmballoon_page_in_frames(VMW_BALLOON_2M_PAGE)); + spin_unlock_irqrestore(&b->b_dev_info.pages_lock, flags); *n_pages = i; } /** + * vmballoon_split_refused_pages() - Split the 2MB refused pages to 4k. + * + * If inflation of 2MB pages was denied by the hypervisor, it is likely to be + * due to one or few 4KB pages. These 2MB pages may keep being allocated and + * then being refused. To prevent this case, this function splits the refused + * pages into 4KB pages and adds them into @prealloc_pages list. + * + * @ctl: pointer for the %struct vmballoon_ctl, which defines the operation. + */ +static void vmballoon_split_refused_pages(struct vmballoon_ctl *ctl) +{ + struct page *page, *tmp; + unsigned int i, order; + + order = vmballoon_page_order(ctl->page_size); + + list_for_each_entry_safe(page, tmp, &ctl->refused_pages, lru) { + list_del(&page->lru); + split_page(page, order); + for (i = 0; i < (1 << order); i++) + list_add(&page[i].lru, &ctl->prealloc_pages); + } + ctl->n_refused_pages = 0; +} + +/** * vmballoon_inflate() - Inflate the balloon towards its target size. * * @b: pointer to the balloon. @@ -1009,6 +1103,7 @@ static void vmballoon_inflate(struct vmballoon *b) struct vmballoon_ctl ctl = { .pages = LIST_HEAD_INIT(ctl.pages), .refused_pages = LIST_HEAD_INIT(ctl.refused_pages), + .prealloc_pages = LIST_HEAD_INIT(ctl.prealloc_pages), .page_size = b->max_page_size, .op = VMW_BALLOON_INFLATE }; @@ -1056,10 +1151,10 @@ static void vmballoon_inflate(struct vmballoon *b) break; /* - * Ignore errors from locking as we now switch to 4k - * pages and we might get different errors. + * Split the refused pages to 4k. This will also empty + * the refused pages list. */ - vmballoon_release_refused_pages(b, &ctl); + vmballoon_split_refused_pages(&ctl); ctl.page_size--; } @@ -1073,6 +1168,8 @@ static void vmballoon_inflate(struct vmballoon *b) */ if (ctl.n_refused_pages != 0) vmballoon_release_refused_pages(b, &ctl); + + vmballoon_release_page_list(&ctl.prealloc_pages, NULL, ctl.page_size); } /** @@ -1411,6 +1508,90 @@ static void vmballoon_work(struct work_struct *work) } +/** + * vmballoon_shrinker_scan() - deflate the balloon due to memory pressure. + * @shrinker: pointer to the balloon shrinker. + * @sc: page reclaim information. + * + * Returns: number of pages that were freed during deflation. + */ +static unsigned long vmballoon_shrinker_scan(struct shrinker *shrinker, + struct shrink_control *sc) +{ + struct vmballoon *b = &balloon; + unsigned long deflated_frames; + + pr_debug("%s - size: %llu", __func__, atomic64_read(&b->size)); + + vmballoon_stats_gen_inc(b, VMW_BALLOON_STAT_SHRINK); + + /* + * If the lock is also contended for read, we cannot easily reclaim and + * we bail out. + */ + if (!down_read_trylock(&b->conf_sem)) + return 0; + + deflated_frames = vmballoon_deflate(b, sc->nr_to_scan, true); + + vmballoon_stats_gen_add(b, VMW_BALLOON_STAT_SHRINK_FREE, + deflated_frames); + + /* + * Delay future inflation for some time to mitigate the situations in + * which balloon continuously grows and shrinks. Use WRITE_ONCE() since + * the access is asynchronous. + */ + WRITE_ONCE(b->shrink_timeout, jiffies + HZ * VMBALLOON_SHRINK_DELAY); + + up_read(&b->conf_sem); + + return deflated_frames; +} + +/** + * vmballoon_shrinker_count() - return the number of ballooned pages. + * @shrinker: pointer to the balloon shrinker. + * @sc: page reclaim information. + * + * Returns: number of 4k pages that are allocated for the balloon and can + * therefore be reclaimed under pressure. + */ +static unsigned long vmballoon_shrinker_count(struct shrinker *shrinker, + struct shrink_control *sc) +{ + struct vmballoon *b = &balloon; + + return atomic64_read(&b->size); +} + +static void vmballoon_unregister_shrinker(struct vmballoon *b) +{ + if (b->shrinker_registered) + unregister_shrinker(&b->shrinker); + b->shrinker_registered = false; +} + +static int vmballoon_register_shrinker(struct vmballoon *b) +{ + int r; + + /* Do nothing if the shrinker is not enabled */ + if (!vmwballoon_shrinker_enable) + return 0; + + b->shrinker.scan_objects = vmballoon_shrinker_scan; + b->shrinker.count_objects = vmballoon_shrinker_count; + b->shrinker.seeks = DEFAULT_SEEKS; + + r = register_shrinker(&b->shrinker); + + if (r == 0) + b->shrinker_registered = true; + + return r; +} + /* * DEBUGFS Interface */ @@ -1428,6 +1609,8 @@ static const char * const vmballoon_stat_names[] = { [VMW_BALLOON_STAT_TIMER] = "timer", [VMW_BALLOON_STAT_DOORBELL] = "doorbell", [VMW_BALLOON_STAT_RESET] = "reset", + [VMW_BALLOON_STAT_SHRINK] = "shrink", + [VMW_BALLOON_STAT_SHRINK_FREE] = "shrinkFree" }; static int vmballoon_enable_stats(struct vmballoon *b) @@ -1552,9 +1735,204 @@ static inline void vmballoon_debugfs_exit(struct vmballoon *b) #endif /* CONFIG_DEBUG_FS */ + +#ifdef CONFIG_BALLOON_COMPACTION + +static struct dentry *vmballoon_mount(struct file_system_type *fs_type, + int flags, const char *dev_name, + void *data) +{ + static const struct dentry_operations ops = { + .d_dname = simple_dname, + }; + + return mount_pseudo(fs_type, "balloon-vmware:", NULL, &ops, + BALLOON_VMW_MAGIC); +} + +static struct file_system_type vmballoon_fs = { + .name = "balloon-vmware", + .mount = vmballoon_mount, + .kill_sb = kill_anon_super, +}; + +static struct vfsmount *vmballoon_mnt; + +/** + * vmballoon_migratepage() - migrates a balloon page. + * @b_dev_info: balloon device information descriptor. + * @newpage: the page to which @page should be migrated. + * @page: a ballooned page that should be migrated. + * @mode: migration mode, ignored. + * + * This function is really open-coded, but that is according to the interface + * that balloon_compaction provides. + * + * Return: zero on success, -EAGAIN when migration cannot be performed + * momentarily, and -EBUSY if migration failed and should be retried + * with that specific page. + */ +static int vmballoon_migratepage(struct balloon_dev_info *b_dev_info, + struct page *newpage, struct page *page, + enum migrate_mode mode) +{ + unsigned long status, flags; + struct vmballoon *b; + int ret; + + b = container_of(b_dev_info, struct vmballoon, b_dev_info); + + /* + * If the semaphore is taken, there is ongoing configuration change + * (i.e., balloon reset), so try again. + */ + if (!down_read_trylock(&b->conf_sem)) + return -EAGAIN; + + spin_lock(&b->comm_lock); + /* + * We must start by deflating and not inflating, as otherwise the + * hypervisor may tell us that it has enough memory and the new page is + * not needed. Since the old page is isolated, we cannot use the list + * interface to unlock it, as the LRU field is used for isolation. + * Instead, we use the native interface directly. + */ + vmballoon_add_page(b, 0, page); + status = vmballoon_lock_op(b, 1, VMW_BALLOON_4K_PAGE, + VMW_BALLOON_DEFLATE); + + if (status == VMW_BALLOON_SUCCESS) + status = vmballoon_status_page(b, 0, &page); + + /* + * If a failure happened, let the migration mechanism know that it + * should not retry. + */ + if (status != VMW_BALLOON_SUCCESS) { + spin_unlock(&b->comm_lock); + ret = -EBUSY; + goto out_unlock; + } + + /* + * The page is isolated, so it is safe to delete it without holding + * @pages_lock . We keep holding @comm_lock since we will need it in a + * second. + */ + balloon_page_delete(page); + + put_page(page); + + /* Inflate */ + vmballoon_add_page(b, 0, newpage); + status = vmballoon_lock_op(b, 1, VMW_BALLOON_4K_PAGE, + VMW_BALLOON_INFLATE); + + if (status == VMW_BALLOON_SUCCESS) + status = vmballoon_status_page(b, 0, &newpage); + + spin_unlock(&b->comm_lock); + + if (status != VMW_BALLOON_SUCCESS) { + /* + * A failure happened. While we can deflate the page we just + * inflated, this deflation can also encounter an error. Instead + * we will decrease the size of the balloon to reflect the + * change and report failure. + */ + atomic64_dec(&b->size); + ret = -EBUSY; + } else { + /* + * Success. Take a reference for the page, and we will add it to + * the list after acquiring the lock. + */ + get_page(newpage); + ret = MIGRATEPAGE_SUCCESS; + } + + /* Update the balloon list under the @pages_lock */ + spin_lock_irqsave(&b->b_dev_info.pages_lock, flags); + + /* + * On inflation success, we already took a reference for the @newpage. + * If we succeed just insert it to the list and update the statistics + * under the lock. + */ + if (ret == MIGRATEPAGE_SUCCESS) { + balloon_page_insert(&b->b_dev_info, newpage); + __count_vm_event(BALLOON_MIGRATE); + } + + /* + * We deflated successfully, so regardless to the inflation success, we + * need to reduce the number of isolated_pages. + */ + b->b_dev_info.isolated_pages--; + spin_unlock_irqrestore(&b->b_dev_info.pages_lock, flags); + +out_unlock: + up_read(&b->conf_sem); + return ret; +} + +/** + * vmballoon_compaction_deinit() - removes compaction related data. + * + * @b: pointer to the balloon. + */ +static void vmballoon_compaction_deinit(struct vmballoon *b) +{ + if (!IS_ERR(b->b_dev_info.inode)) + iput(b->b_dev_info.inode); + + b->b_dev_info.inode = NULL; + kern_unmount(vmballoon_mnt); + vmballoon_mnt = NULL; +} + +/** + * vmballoon_compaction_init() - initialized compaction for the balloon. + * + * @b: pointer to the balloon. + * + * If during the initialization a failure occurred, this function does not + * perform cleanup. The caller must call vmballoon_compaction_deinit() in this + * case. + * + * Return: zero on success or error code on failure. + */ +static __init int vmballoon_compaction_init(struct vmballoon *b) +{ + vmballoon_mnt = kern_mount(&vmballoon_fs); + if (IS_ERR(vmballoon_mnt)) + return PTR_ERR(vmballoon_mnt); + + b->b_dev_info.migratepage = vmballoon_migratepage; + b->b_dev_info.inode = alloc_anon_inode(vmballoon_mnt->mnt_sb); + + if (IS_ERR(b->b_dev_info.inode)) + return PTR_ERR(b->b_dev_info.inode); + + b->b_dev_info.inode->i_mapping->a_ops = &balloon_aops; + return 0; +} + +#else /* CONFIG_BALLOON_COMPACTION */ + +static void vmballoon_compaction_deinit(struct vmballoon *b) +{ +} + +static int vmballoon_compaction_init(struct vmballoon *b) +{ + return 0; +} + +#endif /* CONFIG_BALLOON_COMPACTION */ + static int __init vmballoon_init(void) { - enum vmballoon_page_size_type page_size; int error; /* @@ -1564,17 +1942,26 @@ static int __init vmballoon_init(void) if (x86_hyper_type != X86_HYPER_VMWARE) return -ENODEV; - for (page_size = VMW_BALLOON_4K_PAGE; - page_size <= VMW_BALLOON_LAST_SIZE; page_size++) - INIT_LIST_HEAD(&balloon.page_sizes[page_size].pages); - - INIT_DELAYED_WORK(&balloon.dwork, vmballoon_work); + error = vmballoon_register_shrinker(&balloon); + if (error) + goto fail; + error = vmballoon_debugfs_init(&balloon); if (error) - return error; + goto fail; + + /* + * Initialization of compaction must be done after the call to + * balloon_devinfo_init() . + */ + balloon_devinfo_init(&balloon.b_dev_info); + error = vmballoon_compaction_init(&balloon); + if (error) + goto fail; + INIT_LIST_HEAD(&balloon.huge_pages); spin_lock_init(&balloon.comm_lock); init_rwsem(&balloon.conf_sem); balloon.vmci_doorbell = VMCI_INVALID_HANDLE; @@ -1585,6 +1972,10 @@ static int __init vmballoon_init(void) queue_delayed_work(system_freezable_wq, &balloon.dwork, 0); return 0; +fail: + vmballoon_unregister_shrinker(&balloon); + vmballoon_compaction_deinit(&balloon); + return error; } /* @@ -1597,6 +1988,7 @@ late_initcall(vmballoon_init); static void __exit vmballoon_exit(void) { + vmballoon_unregister_shrinker(&balloon); vmballoon_vmci_cleanup(&balloon); cancel_delayed_work_sync(&balloon.dwork); @@ -1609,5 +2001,8 @@ static void __exit vmballoon_exit(void) */ vmballoon_send_start(&balloon, 0); vmballoon_pop(&balloon); + + /* Only once we popped the balloon, compaction can be deinit */ + vmballoon_compaction_deinit(&balloon); } module_exit(vmballoon_exit); diff --git a/drivers/misc/vmw_vmci/vmci_context.c b/drivers/misc/vmw_vmci/vmci_context.c index 300ed69fe2c7..16695366ec92 100644 --- a/drivers/misc/vmw_vmci/vmci_context.c +++ b/drivers/misc/vmw_vmci/vmci_context.c @@ -21,6 +21,9 @@ #include "vmci_driver.h" #include "vmci_event.h" +/* Use a wide upper bound for the maximum contexts. */ +#define VMCI_MAX_CONTEXTS 2000 + /* * List of current VMCI contexts. Contexts can be added by * vmci_ctx_create() and removed via vmci_ctx_destroy(). @@ -117,19 +120,22 @@ struct vmci_ctx *vmci_ctx_create(u32 cid, u32 priv_flags, /* Initialize host-specific VMCI context. */ init_waitqueue_head(&context->host_context.wait_queue); - context->queue_pair_array = vmci_handle_arr_create(0); + context->queue_pair_array = + vmci_handle_arr_create(0, VMCI_MAX_GUEST_QP_COUNT); if (!context->queue_pair_array) { error = -ENOMEM; goto err_free_ctx; } - context->doorbell_array = vmci_handle_arr_create(0); + context->doorbell_array = + vmci_handle_arr_create(0, VMCI_MAX_GUEST_DOORBELL_COUNT); if (!context->doorbell_array) { error = -ENOMEM; goto err_free_qp_array; } - context->pending_doorbell_array = vmci_handle_arr_create(0); + context->pending_doorbell_array = + vmci_handle_arr_create(0, VMCI_MAX_GUEST_DOORBELL_COUNT); if (!context->pending_doorbell_array) { error = -ENOMEM; goto err_free_db_array; @@ -204,7 +210,7 @@ static int ctx_fire_notification(u32 context_id, u32 priv_flags) * We create an array to hold the subscribers we find when * scanning through all contexts. */ - subscriber_array = vmci_handle_arr_create(0); + subscriber_array = vmci_handle_arr_create(0, VMCI_MAX_CONTEXTS); if (subscriber_array == NULL) return VMCI_ERROR_NO_MEM; @@ -623,20 +629,26 @@ int vmci_ctx_add_notification(u32 context_id, u32 remote_cid) spin_lock(&context->lock); - list_for_each_entry(n, &context->notifier_list, node) { - if (vmci_handle_is_equal(n->handle, notifier->handle)) { - exists = true; - break; + if (context->n_notifiers < VMCI_MAX_CONTEXTS) { + list_for_each_entry(n, &context->notifier_list, node) { + if (vmci_handle_is_equal(n->handle, notifier->handle)) { + exists = true; + break; + } } - } - if (exists) { - kfree(notifier); - result = VMCI_ERROR_ALREADY_EXISTS; + if (exists) { + kfree(notifier); + result = VMCI_ERROR_ALREADY_EXISTS; + } else { + list_add_tail_rcu(¬ifier->node, + &context->notifier_list); + context->n_notifiers++; + result = VMCI_SUCCESS; + } } else { - list_add_tail_rcu(¬ifier->node, &context->notifier_list); - context->n_notifiers++; - result = VMCI_SUCCESS; + kfree(notifier); + result = VMCI_ERROR_NO_MEM; } spin_unlock(&context->lock); @@ -721,8 +733,7 @@ static int vmci_ctx_get_chkpt_doorbells(struct vmci_ctx *context, u32 *buf_size, void **pbuf) { struct dbell_cpt_state *dbells; - size_t n_doorbells; - int i; + u32 i, n_doorbells; n_doorbells = vmci_handle_arr_get_size(context->doorbell_array); if (n_doorbells > 0) { @@ -860,7 +871,8 @@ int vmci_ctx_rcv_notifications_get(u32 context_id, spin_lock(&context->lock); *db_handle_array = context->pending_doorbell_array; - context->pending_doorbell_array = vmci_handle_arr_create(0); + context->pending_doorbell_array = + vmci_handle_arr_create(0, VMCI_MAX_GUEST_DOORBELL_COUNT); if (!context->pending_doorbell_array) { context->pending_doorbell_array = *db_handle_array; *db_handle_array = NULL; @@ -942,12 +954,11 @@ int vmci_ctx_dbell_create(u32 context_id, struct vmci_handle handle) return VMCI_ERROR_NOT_FOUND; spin_lock(&context->lock); - if (!vmci_handle_arr_has_entry(context->doorbell_array, handle)) { - vmci_handle_arr_append_entry(&context->doorbell_array, handle); - result = VMCI_SUCCESS; - } else { + if (!vmci_handle_arr_has_entry(context->doorbell_array, handle)) + result = vmci_handle_arr_append_entry(&context->doorbell_array, + handle); + else result = VMCI_ERROR_DUPLICATE_ENTRY; - } spin_unlock(&context->lock); vmci_ctx_put(context); @@ -1083,15 +1094,16 @@ int vmci_ctx_notify_dbell(u32 src_cid, if (!vmci_handle_arr_has_entry( dst_context->pending_doorbell_array, handle)) { - vmci_handle_arr_append_entry( + result = vmci_handle_arr_append_entry( &dst_context->pending_doorbell_array, handle); - - ctx_signal_notify(dst_context); - wake_up(&dst_context->host_context.wait_queue); - + if (result == VMCI_SUCCESS) { + ctx_signal_notify(dst_context); + wake_up(&dst_context->host_context.wait_queue); + } + } else { + result = VMCI_SUCCESS; } - result = VMCI_SUCCESS; } spin_unlock(&dst_context->lock); } @@ -1118,13 +1130,11 @@ int vmci_ctx_qp_create(struct vmci_ctx *context, struct vmci_handle handle) if (context == NULL || vmci_handle_is_invalid(handle)) return VMCI_ERROR_INVALID_ARGS; - if (!vmci_handle_arr_has_entry(context->queue_pair_array, handle)) { - vmci_handle_arr_append_entry(&context->queue_pair_array, - handle); - result = VMCI_SUCCESS; - } else { + if (!vmci_handle_arr_has_entry(context->queue_pair_array, handle)) + result = vmci_handle_arr_append_entry( + &context->queue_pair_array, handle); + else result = VMCI_ERROR_DUPLICATE_ENTRY; - } return result; } diff --git a/drivers/misc/vmw_vmci/vmci_handle_array.c b/drivers/misc/vmw_vmci/vmci_handle_array.c index c527388f5d7b..de7fee7ead1b 100644 --- a/drivers/misc/vmw_vmci/vmci_handle_array.c +++ b/drivers/misc/vmw_vmci/vmci_handle_array.c @@ -8,24 +8,29 @@ #include <linux/slab.h> #include "vmci_handle_array.h" -static size_t handle_arr_calc_size(size_t capacity) +static size_t handle_arr_calc_size(u32 capacity) { - return sizeof(struct vmci_handle_arr) + + return VMCI_HANDLE_ARRAY_HEADER_SIZE + capacity * sizeof(struct vmci_handle); } -struct vmci_handle_arr *vmci_handle_arr_create(size_t capacity) +struct vmci_handle_arr *vmci_handle_arr_create(u32 capacity, u32 max_capacity) { struct vmci_handle_arr *array; + if (max_capacity == 0 || capacity > max_capacity) + return NULL; + if (capacity == 0) - capacity = VMCI_HANDLE_ARRAY_DEFAULT_SIZE; + capacity = min((u32)VMCI_HANDLE_ARRAY_DEFAULT_CAPACITY, + max_capacity); array = kmalloc(handle_arr_calc_size(capacity), GFP_ATOMIC); if (!array) return NULL; array->capacity = capacity; + array->max_capacity = max_capacity; array->size = 0; return array; @@ -36,27 +41,34 @@ void vmci_handle_arr_destroy(struct vmci_handle_arr *array) kfree(array); } -void vmci_handle_arr_append_entry(struct vmci_handle_arr **array_ptr, - struct vmci_handle handle) +int vmci_handle_arr_append_entry(struct vmci_handle_arr **array_ptr, + struct vmci_handle handle) { struct vmci_handle_arr *array = *array_ptr; if (unlikely(array->size >= array->capacity)) { /* reallocate. */ struct vmci_handle_arr *new_array; - size_t new_capacity = array->capacity * VMCI_ARR_CAP_MULT; - size_t new_size = handle_arr_calc_size(new_capacity); + u32 capacity_bump = min(array->max_capacity - array->capacity, + array->capacity); + size_t new_size = handle_arr_calc_size(array->capacity + + capacity_bump); + + if (array->size >= array->max_capacity) + return VMCI_ERROR_NO_MEM; new_array = krealloc(array, new_size, GFP_ATOMIC); if (!new_array) - return; + return VMCI_ERROR_NO_MEM; - new_array->capacity = new_capacity; + new_array->capacity += capacity_bump; *array_ptr = array = new_array; } array->entries[array->size] = handle; array->size++; + + return VMCI_SUCCESS; } /* @@ -66,7 +78,7 @@ struct vmci_handle vmci_handle_arr_remove_entry(struct vmci_handle_arr *array, struct vmci_handle entry_handle) { struct vmci_handle handle = VMCI_INVALID_HANDLE; - size_t i; + u32 i; for (i = 0; i < array->size; i++) { if (vmci_handle_is_equal(array->entries[i], entry_handle)) { @@ -101,7 +113,7 @@ struct vmci_handle vmci_handle_arr_remove_tail(struct vmci_handle_arr *array) * Handle at given index, VMCI_INVALID_HANDLE if invalid index. */ struct vmci_handle -vmci_handle_arr_get_entry(const struct vmci_handle_arr *array, size_t index) +vmci_handle_arr_get_entry(const struct vmci_handle_arr *array, u32 index) { if (unlikely(index >= array->size)) return VMCI_INVALID_HANDLE; @@ -112,7 +124,7 @@ vmci_handle_arr_get_entry(const struct vmci_handle_arr *array, size_t index) bool vmci_handle_arr_has_entry(const struct vmci_handle_arr *array, struct vmci_handle entry_handle) { - size_t i; + u32 i; for (i = 0; i < array->size; i++) if (vmci_handle_is_equal(array->entries[i], entry_handle)) diff --git a/drivers/misc/vmw_vmci/vmci_handle_array.h b/drivers/misc/vmw_vmci/vmci_handle_array.h index bd1559a548e9..96193f85be5b 100644 --- a/drivers/misc/vmw_vmci/vmci_handle_array.h +++ b/drivers/misc/vmw_vmci/vmci_handle_array.h @@ -9,32 +9,41 @@ #define _VMCI_HANDLE_ARRAY_H_ #include <linux/vmw_vmci_defs.h> +#include <linux/limits.h> #include <linux/types.h> -#define VMCI_HANDLE_ARRAY_DEFAULT_SIZE 4 -#define VMCI_ARR_CAP_MULT 2 /* Array capacity multiplier */ - struct vmci_handle_arr { - size_t capacity; - size_t size; + u32 capacity; + u32 max_capacity; + u32 size; + u32 pad; struct vmci_handle entries[]; }; -struct vmci_handle_arr *vmci_handle_arr_create(size_t capacity); +#define VMCI_HANDLE_ARRAY_HEADER_SIZE \ + offsetof(struct vmci_handle_arr, entries) +/* Select a default capacity that results in a 64 byte sized array */ +#define VMCI_HANDLE_ARRAY_DEFAULT_CAPACITY 6 +/* Make sure that the max array size can be expressed by a u32 */ +#define VMCI_HANDLE_ARRAY_MAX_CAPACITY \ + ((U32_MAX - VMCI_HANDLE_ARRAY_HEADER_SIZE - 1) / \ + sizeof(struct vmci_handle)) + +struct vmci_handle_arr *vmci_handle_arr_create(u32 capacity, u32 max_capacity); void vmci_handle_arr_destroy(struct vmci_handle_arr *array); -void vmci_handle_arr_append_entry(struct vmci_handle_arr **array_ptr, - struct vmci_handle handle); +int vmci_handle_arr_append_entry(struct vmci_handle_arr **array_ptr, + struct vmci_handle handle); struct vmci_handle vmci_handle_arr_remove_entry(struct vmci_handle_arr *array, struct vmci_handle entry_handle); struct vmci_handle vmci_handle_arr_remove_tail(struct vmci_handle_arr *array); struct vmci_handle -vmci_handle_arr_get_entry(const struct vmci_handle_arr *array, size_t index); +vmci_handle_arr_get_entry(const struct vmci_handle_arr *array, u32 index); bool vmci_handle_arr_has_entry(const struct vmci_handle_arr *array, struct vmci_handle entry_handle); struct vmci_handle *vmci_handle_arr_get_handles(struct vmci_handle_arr *array); -static inline size_t vmci_handle_arr_get_size( +static inline u32 vmci_handle_arr_get_size( const struct vmci_handle_arr *array) { return array->size; diff --git a/drivers/misc/xilinx_sdfec.c b/drivers/misc/xilinx_sdfec.c new file mode 100644 index 000000000000..f257d3812110 --- /dev/null +++ b/drivers/misc/xilinx_sdfec.c @@ -0,0 +1,345 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Xilinx SDFEC + * + * Copyright (C) 2019 Xilinx, Inc. + * + * Description: + * This driver is developed for SDFEC16 (Soft Decision FEC 16nm) + * IP. It exposes a char device which supports file operations + * like open(), close() and ioctl(). + */ + +#include <linux/miscdevice.h> +#include <linux/io.h> +#include <linux/interrupt.h> +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/of_platform.h> +#include <linux/poll.h> +#include <linux/slab.h> +#include <linux/clk.h> + +#define DEV_NAME_LEN 12 + +static struct idr dev_idr; +static struct mutex dev_idr_lock; + +/** + * struct xsdfec_clks - For managing SD-FEC clocks + * @core_clk: Main processing clock for core + * @axi_clk: AXI4-Lite memory-mapped clock + * @din_words_clk: DIN Words AXI4-Stream Slave clock + * @din_clk: DIN AXI4-Stream Slave clock + * @dout_clk: DOUT Words AXI4-Stream Slave clock + * @dout_words_clk: DOUT AXI4-Stream Slave clock + * @ctrl_clk: Control AXI4-Stream Slave clock + * @status_clk: Status AXI4-Stream Slave clock + */ +struct xsdfec_clks { + struct clk *core_clk; + struct clk *axi_clk; + struct clk *din_words_clk; + struct clk *din_clk; + struct clk *dout_clk; + struct clk *dout_words_clk; + struct clk *ctrl_clk; + struct clk *status_clk; +}; + +/** + * struct xsdfec_dev - Driver data for SDFEC + * @regs: device physical base address + * @dev: pointer to device struct + * @miscdev: Misc device handle + * @error_data_lock: Error counter and states spinlock + * @clks: Clocks managed by the SDFEC driver + * @dev_name: Device name + * @dev_id: Device ID + * + * This structure contains necessary state for SDFEC driver to operate + */ +struct xsdfec_dev { + void __iomem *regs; + struct device *dev; + struct miscdevice miscdev; + /* Spinlock to protect state_updated and stats_updated */ + spinlock_t error_data_lock; + struct xsdfec_clks clks; + char dev_name[DEV_NAME_LEN]; + int dev_id; +}; + +static const struct file_operations xsdfec_fops = { + .owner = THIS_MODULE, +}; + +static int xsdfec_clk_init(struct platform_device *pdev, + struct xsdfec_clks *clks) +{ + int err; + + clks->core_clk = devm_clk_get(&pdev->dev, "core_clk"); + if (IS_ERR(clks->core_clk)) { + dev_err(&pdev->dev, "failed to get core_clk"); + return PTR_ERR(clks->core_clk); + } + + clks->axi_clk = devm_clk_get(&pdev->dev, "s_axi_aclk"); + if (IS_ERR(clks->axi_clk)) { + dev_err(&pdev->dev, "failed to get axi_clk"); + return PTR_ERR(clks->axi_clk); + } + + clks->din_words_clk = devm_clk_get(&pdev->dev, "s_axis_din_words_aclk"); + if (IS_ERR(clks->din_words_clk)) { + if (PTR_ERR(clks->din_words_clk) != -ENOENT) { + err = PTR_ERR(clks->din_words_clk); + return err; + } + clks->din_words_clk = NULL; + } + + clks->din_clk = devm_clk_get(&pdev->dev, "s_axis_din_aclk"); + if (IS_ERR(clks->din_clk)) { + if (PTR_ERR(clks->din_clk) != -ENOENT) { + err = PTR_ERR(clks->din_clk); + return err; + } + clks->din_clk = NULL; + } + + clks->dout_clk = devm_clk_get(&pdev->dev, "m_axis_dout_aclk"); + if (IS_ERR(clks->dout_clk)) { + if (PTR_ERR(clks->dout_clk) != -ENOENT) { + err = PTR_ERR(clks->dout_clk); + return err; + } + clks->dout_clk = NULL; + } + + clks->dout_words_clk = + devm_clk_get(&pdev->dev, "s_axis_dout_words_aclk"); + if (IS_ERR(clks->dout_words_clk)) { + if (PTR_ERR(clks->dout_words_clk) != -ENOENT) { + err = PTR_ERR(clks->dout_words_clk); + return err; + } + clks->dout_words_clk = NULL; + } + + clks->ctrl_clk = devm_clk_get(&pdev->dev, "s_axis_ctrl_aclk"); + if (IS_ERR(clks->ctrl_clk)) { + if (PTR_ERR(clks->ctrl_clk) != -ENOENT) { + err = PTR_ERR(clks->ctrl_clk); + return err; + } + clks->ctrl_clk = NULL; + } + + clks->status_clk = devm_clk_get(&pdev->dev, "m_axis_status_aclk"); + if (IS_ERR(clks->status_clk)) { + if (PTR_ERR(clks->status_clk) != -ENOENT) { + err = PTR_ERR(clks->status_clk); + return err; + } + clks->status_clk = NULL; + } + + err = clk_prepare_enable(clks->core_clk); + if (err) { + dev_err(&pdev->dev, "failed to enable core_clk (%d)", err); + return err; + } + + err = clk_prepare_enable(clks->axi_clk); + if (err) { + dev_err(&pdev->dev, "failed to enable axi_clk (%d)", err); + goto err_disable_core_clk; + } + + err = clk_prepare_enable(clks->din_clk); + if (err) { + dev_err(&pdev->dev, "failed to enable din_clk (%d)", err); + goto err_disable_axi_clk; + } + + err = clk_prepare_enable(clks->din_words_clk); + if (err) { + dev_err(&pdev->dev, "failed to enable din_words_clk (%d)", err); + goto err_disable_din_clk; + } + + err = clk_prepare_enable(clks->dout_clk); + if (err) { + dev_err(&pdev->dev, "failed to enable dout_clk (%d)", err); + goto err_disable_din_words_clk; + } + + err = clk_prepare_enable(clks->dout_words_clk); + if (err) { + dev_err(&pdev->dev, "failed to enable dout_words_clk (%d)", + err); + goto err_disable_dout_clk; + } + + err = clk_prepare_enable(clks->ctrl_clk); + if (err) { + dev_err(&pdev->dev, "failed to enable ctrl_clk (%d)", err); + goto err_disable_dout_words_clk; + } + + err = clk_prepare_enable(clks->status_clk); + if (err) { + dev_err(&pdev->dev, "failed to enable status_clk (%d)\n", err); + goto err_disable_ctrl_clk; + } + + return err; + +err_disable_ctrl_clk: + clk_disable_unprepare(clks->ctrl_clk); +err_disable_dout_words_clk: + clk_disable_unprepare(clks->dout_words_clk); +err_disable_dout_clk: + clk_disable_unprepare(clks->dout_clk); +err_disable_din_words_clk: + clk_disable_unprepare(clks->din_words_clk); +err_disable_din_clk: + clk_disable_unprepare(clks->din_clk); +err_disable_axi_clk: + clk_disable_unprepare(clks->axi_clk); +err_disable_core_clk: + clk_disable_unprepare(clks->core_clk); + + return err; +} + +static void xsdfec_disable_all_clks(struct xsdfec_clks *clks) +{ + clk_disable_unprepare(clks->status_clk); + clk_disable_unprepare(clks->ctrl_clk); + clk_disable_unprepare(clks->dout_words_clk); + clk_disable_unprepare(clks->dout_clk); + clk_disable_unprepare(clks->din_words_clk); + clk_disable_unprepare(clks->din_clk); + clk_disable_unprepare(clks->core_clk); + clk_disable_unprepare(clks->axi_clk); +} + +static void xsdfec_idr_remove(struct xsdfec_dev *xsdfec) +{ + mutex_lock(&dev_idr_lock); + idr_remove(&dev_idr, xsdfec->dev_id); + mutex_unlock(&dev_idr_lock); +} + +static int xsdfec_probe(struct platform_device *pdev) +{ + struct xsdfec_dev *xsdfec; + struct device *dev; + struct resource *res; + int err; + + xsdfec = devm_kzalloc(&pdev->dev, sizeof(*xsdfec), GFP_KERNEL); + if (!xsdfec) + return -ENOMEM; + + xsdfec->dev = &pdev->dev; + spin_lock_init(&xsdfec->error_data_lock); + + err = xsdfec_clk_init(pdev, &xsdfec->clks); + if (err) + return err; + + dev = xsdfec->dev; + res = platform_get_resource(pdev, IORESOURCE_MEM, 0); + xsdfec->regs = devm_ioremap_resource(dev, res); + if (IS_ERR(xsdfec->regs)) { + err = PTR_ERR(xsdfec->regs); + goto err_xsdfec_dev; + } + + /* Save driver private data */ + platform_set_drvdata(pdev, xsdfec); + + mutex_lock(&dev_idr_lock); + err = idr_alloc(&dev_idr, xsdfec->dev_name, 0, 0, GFP_KERNEL); + mutex_unlock(&dev_idr_lock); + if (err < 0) + goto err_xsdfec_dev; + xsdfec->dev_id = err; + + snprintf(xsdfec->dev_name, DEV_NAME_LEN, "xsdfec%d", xsdfec->dev_id); + xsdfec->miscdev.minor = MISC_DYNAMIC_MINOR; + xsdfec->miscdev.name = xsdfec->dev_name; + xsdfec->miscdev.fops = &xsdfec_fops; + xsdfec->miscdev.parent = dev; + err = misc_register(&xsdfec->miscdev); + if (err) { + dev_err(dev, "error:%d. Unable to register device", err); + goto err_xsdfec_idr; + } + return 0; + +err_xsdfec_idr: + xsdfec_idr_remove(xsdfec); +err_xsdfec_dev: + xsdfec_disable_all_clks(&xsdfec->clks); + return err; +} + +static int xsdfec_remove(struct platform_device *pdev) +{ + struct xsdfec_dev *xsdfec; + + xsdfec = platform_get_drvdata(pdev); + misc_deregister(&xsdfec->miscdev); + xsdfec_idr_remove(xsdfec); + xsdfec_disable_all_clks(&xsdfec->clks); + return 0; +} + +static const struct of_device_id xsdfec_of_match[] = { + { + .compatible = "xlnx,sd-fec-1.1", + }, + { /* end of table */ } +}; +MODULE_DEVICE_TABLE(of, xsdfec_of_match); + +static struct platform_driver xsdfec_driver = { + .driver = { + .name = "xilinx-sdfec", + .of_match_table = xsdfec_of_match, + }, + .probe = xsdfec_probe, + .remove = xsdfec_remove, +}; + +static int __init xsdfec_init(void) +{ + int err; + + mutex_init(&dev_idr_lock); + idr_init(&dev_idr); + err = platform_driver_register(&xsdfec_driver); + if (err < 0) { + pr_err("%s Unabled to register SDFEC driver", __func__); + return err; + } + return 0; +} + +static void __exit xsdfec_exit(void) +{ + platform_driver_unregister(&xsdfec_driver); + idr_destroy(&dev_idr); +} + +module_init(xsdfec_init); +module_exit(xsdfec_exit); + +MODULE_AUTHOR("Xilinx, Inc"); +MODULE_DESCRIPTION("Xilinx SD-FEC16 Driver"); +MODULE_LICENSE("GPL"); |