diff options
Diffstat (limited to 'drivers/misc')
73 files changed, 3829 insertions, 1265 deletions
diff --git a/drivers/misc/bcm-vk/bcm_vk_dev.c b/drivers/misc/bcm-vk/bcm_vk_dev.c index 6bfea3210389..ad639ee85b2a 100644 --- a/drivers/misc/bcm-vk/bcm_vk_dev.c +++ b/drivers/misc/bcm-vk/bcm_vk_dev.c @@ -9,6 +9,7 @@ #include <linux/fs.h> #include <linux/idr.h> #include <linux/interrupt.h> +#include <linux/panic_notifier.h> #include <linux/kref.h> #include <linux/module.h> #include <linux/mutex.h> diff --git a/drivers/misc/bcm-vk/bcm_vk_msg.c b/drivers/misc/bcm-vk/bcm_vk_msg.c index f40cf08a6192..066b9ef7fcd7 100644 --- a/drivers/misc/bcm-vk/bcm_vk_msg.c +++ b/drivers/misc/bcm-vk/bcm_vk_msg.c @@ -354,8 +354,7 @@ static void bcm_vk_drain_all_pend(struct device *dev, for (num = 0; num < chan->q_nr; num++) { list_for_each_entry_safe(entry, tmp, &chan->pendq[num], node) { if ((!ctx) || (entry->ctx->idx == ctx->idx)) { - list_del(&entry->node); - list_add_tail(&entry->node, &del_q); + list_move_tail(&entry->node, &del_q); } } } @@ -701,8 +700,7 @@ int bcm_vk_send_shutdown_msg(struct bcm_vk *vk, u32 shut_type, return -EINVAL; } - entry = kzalloc(sizeof(*entry) + - sizeof(struct vk_msg_blk), GFP_KERNEL); + entry = kzalloc(struct_size(entry, to_v_msg, 1), GFP_KERNEL); if (!entry) return -ENOMEM; diff --git a/drivers/misc/bcm-vk/bcm_vk_msg.h b/drivers/misc/bcm-vk/bcm_vk_msg.h index 4eaad84825d6..56784c8896d8 100644 --- a/drivers/misc/bcm-vk/bcm_vk_msg.h +++ b/drivers/misc/bcm-vk/bcm_vk_msg.h @@ -116,7 +116,7 @@ struct bcm_vk_wkent { u32 usr_msg_id; u32 to_v_blks; u32 seq_num; - struct vk_msg_blk to_v_msg[0]; + struct vk_msg_blk to_v_msg[]; }; /* queue stats counters */ diff --git a/drivers/misc/bcm-vk/bcm_vk_tty.c b/drivers/misc/bcm-vk/bcm_vk_tty.c index 4d02692ecfc7..dae9eeed84a2 100644 --- a/drivers/misc/bcm-vk/bcm_vk_tty.c +++ b/drivers/misc/bcm-vk/bcm_vk_tty.c @@ -214,7 +214,7 @@ static int bcm_vk_tty_write(struct tty_struct *tty, return count; } -static int bcm_vk_tty_write_room(struct tty_struct *tty) +static unsigned int bcm_vk_tty_write_room(struct tty_struct *tty) { struct bcm_vk *vk = dev_get_drvdata(tty->dev); diff --git a/drivers/misc/cardreader/alcor_pci.c b/drivers/misc/cardreader/alcor_pci.c index cd402c89189e..de6d44a158bb 100644 --- a/drivers/misc/cardreader/alcor_pci.c +++ b/drivers/misc/cardreader/alcor_pci.c @@ -139,7 +139,13 @@ static void alcor_pci_init_check_aspm(struct alcor_pci_priv *priv) u32 val32; priv->pdev_cap_off = alcor_pci_find_cap_offset(priv, priv->pdev); - priv->parent_cap_off = alcor_pci_find_cap_offset(priv, + /* + * A device might be attached to root complex directly and + * priv->parent_pdev will be NULL. In this case we don't check its + * capability and disable ASPM completely. + */ + if (priv->parent_pdev) + priv->parent_cap_off = alcor_pci_find_cap_offset(priv, priv->parent_pdev); if ((priv->pdev_cap_off == 0) || (priv->parent_cap_off == 0)) { diff --git a/drivers/misc/cxl/file.c b/drivers/misc/cxl/file.c index bd3bd32333c5..3dbdce96fae0 100644 --- a/drivers/misc/cxl/file.c +++ b/drivers/misc/cxl/file.c @@ -569,7 +569,8 @@ static int cxl_add_chardev(struct cxl_afu *afu, dev_t devt, struct cdev *cdev, int rc; cdev_init(cdev, fops); - if ((rc = cdev_add(cdev, devt, 1))) { + rc = cdev_add(cdev, devt, 1); + if (rc) { dev_err(&afu->dev, "Unable to add %s chardev: %i\n", desc, rc); return rc; } @@ -577,8 +578,8 @@ static int cxl_add_chardev(struct cxl_afu *afu, dev_t devt, struct cdev *cdev, dev = device_create(cxl_class, &afu->dev, devt, afu, "afu%i.%i%s", afu->adapter->adapter_num, afu->slice, postfix); if (IS_ERR(dev)) { - dev_err(&afu->dev, "Unable to create %s chardev in sysfs: %i\n", desc, rc); rc = PTR_ERR(dev); + dev_err(&afu->dev, "Unable to create %s chardev in sysfs: %i\n", desc, rc); goto err; } diff --git a/drivers/misc/eeprom/Kconfig b/drivers/misc/eeprom/Kconfig index 0f791bfdc1f5..f0a7531f354c 100644 --- a/drivers/misc/eeprom/Kconfig +++ b/drivers/misc/eeprom/Kconfig @@ -32,12 +32,13 @@ config EEPROM_AT24 will be called at24. config EEPROM_AT25 - tristate "SPI EEPROMs from most vendors" + tristate "SPI EEPROMs (FRAMs) from most vendors" depends on SPI && SYSFS select NVMEM select NVMEM_SYSFS help - Enable this driver to get read/write support to most SPI EEPROMs, + Enable this driver to get read/write support to most SPI EEPROMs + and Cypress FRAMs, after you configure the board init code to know about each eeprom on your target board. diff --git a/drivers/misc/eeprom/at25.c b/drivers/misc/eeprom/at25.c index b76e4901b4a4..4d09b672ac3c 100644 --- a/drivers/misc/eeprom/at25.c +++ b/drivers/misc/eeprom/at25.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: GPL-2.0-or-later /* * at25.c -- support most SPI EEPROMs, such as Atmel AT25 models + * and Cypress FRAMs FM25 models * * Copyright (C) 2006 David Brownell */ @@ -16,6 +17,9 @@ #include <linux/spi/spi.h> #include <linux/spi/eeprom.h> #include <linux/property.h> +#include <linux/of.h> +#include <linux/of_device.h> +#include <linux/math.h> /* * NOTE: this is an *EEPROM* driver. The vagaries of product naming @@ -27,6 +31,7 @@ * AT25M02, AT25128B */ +#define FM25_SN_LEN 8 /* serial number length */ struct at25_data { struct spi_device *spi; struct mutex lock; @@ -34,6 +39,7 @@ struct at25_data { unsigned addrlen; struct nvmem_config nvmem_config; struct nvmem_device *nvmem; + u8 sernum[FM25_SN_LEN]; }; #define AT25_WREN 0x06 /* latch the write enable */ @@ -42,6 +48,9 @@ struct at25_data { #define AT25_WRSR 0x01 /* write status register */ #define AT25_READ 0x03 /* read byte(s) */ #define AT25_WRITE 0x02 /* write byte(s)/sector */ +#define FM25_SLEEP 0xb9 /* enter sleep mode */ +#define FM25_RDID 0x9f /* read device ID */ +#define FM25_RDSN 0xc3 /* read S/N */ #define AT25_SR_nRDY 0x01 /* nRDY = write-in-progress */ #define AT25_SR_WEN 0x02 /* write enable (latched) */ @@ -51,6 +60,8 @@ struct at25_data { #define AT25_INSTR_BIT3 0x08 /* Additional address bit in instr */ +#define FM25_ID_LEN 9 /* ID length */ + #define EE_MAXADDRLEN 3 /* 24 bit addresses, up to 2 MBytes */ /* Specs often allow 5 msec for a page write, sometimes 20 msec; @@ -129,6 +140,51 @@ static int at25_ee_read(void *priv, unsigned int offset, return status; } +/* + * read extra registers as ID or serial number + */ +static int fm25_aux_read(struct at25_data *at25, u8 *buf, uint8_t command, + int len) +{ + int status; + struct spi_transfer t[2]; + struct spi_message m; + + spi_message_init(&m); + memset(t, 0, sizeof(t)); + + t[0].tx_buf = &command; + t[0].len = 1; + spi_message_add_tail(&t[0], &m); + + t[1].rx_buf = buf; + t[1].len = len; + spi_message_add_tail(&t[1], &m); + + mutex_lock(&at25->lock); + + status = spi_sync(at25->spi, &m); + dev_dbg(&at25->spi->dev, "read %d aux bytes --> %d\n", len, status); + + mutex_unlock(&at25->lock); + return status; +} + +static ssize_t sernum_show(struct device *dev, struct device_attribute *attr, char *buf) +{ + struct at25_data *at25; + + at25 = dev_get_drvdata(dev); + return sysfs_emit(buf, "%*ph\n", (int)sizeof(at25->sernum), at25->sernum); +} +static DEVICE_ATTR_RO(sernum); + +static struct attribute *sernum_attrs[] = { + &dev_attr_sernum.attr, + NULL, +}; +ATTRIBUTE_GROUPS(sernum); + static int at25_ee_write(void *priv, unsigned int off, void *val, size_t count) { struct at25_data *at25 = priv; @@ -303,34 +359,39 @@ static int at25_fw_to_chip(struct device *dev, struct spi_eeprom *chip) return 0; } +static const struct of_device_id at25_of_match[] = { + { .compatible = "atmel,at25",}, + { .compatible = "cypress,fm25",}, + { } +}; +MODULE_DEVICE_TABLE(of, at25_of_match); + static int at25_probe(struct spi_device *spi) { struct at25_data *at25 = NULL; struct spi_eeprom chip; int err; int sr; - int addrlen; + u8 id[FM25_ID_LEN]; + u8 sernum[FM25_SN_LEN]; + int i; + const struct of_device_id *match; + bool is_fram = 0; + + match = of_match_device(of_match_ptr(at25_of_match), &spi->dev); + if (match && !strcmp(match->compatible, "cypress,fm25")) + is_fram = 1; /* Chip description */ if (!spi->dev.platform_data) { - err = at25_fw_to_chip(&spi->dev, &chip); - if (err) - return err; + if (!is_fram) { + err = at25_fw_to_chip(&spi->dev, &chip); + if (err) + return err; + } } else chip = *(struct spi_eeprom *)spi->dev.platform_data; - /* For now we only support 8/16/24 bit addressing */ - if (chip.flags & EE_ADDR1) - addrlen = 1; - else if (chip.flags & EE_ADDR2) - addrlen = 2; - else if (chip.flags & EE_ADDR3) - addrlen = 3; - else { - dev_dbg(&spi->dev, "unsupported address type\n"); - return -EINVAL; - } - /* Ping the chip ... the status register is pretty portable, * unlike probing manufacturer IDs. We do expect that system * firmware didn't write it in the past few milliseconds! @@ -349,9 +410,51 @@ static int at25_probe(struct spi_device *spi) at25->chip = chip; at25->spi = spi; spi_set_drvdata(spi, at25); - at25->addrlen = addrlen; - at25->nvmem_config.type = NVMEM_TYPE_EEPROM; + if (is_fram) { + /* Get ID of chip */ + fm25_aux_read(at25, id, FM25_RDID, FM25_ID_LEN); + if (id[6] != 0xc2) { + dev_err(&spi->dev, + "Error: no Cypress FRAM (id %02x)\n", id[6]); + return -ENODEV; + } + /* set size found in ID */ + if (id[7] < 0x21 || id[7] > 0x26) { + dev_err(&spi->dev, "Error: unsupported size (id %02x)\n", id[7]); + return -ENODEV; + } + chip.byte_len = int_pow(2, id[7] - 0x21 + 4) * 1024; + + if (at25->chip.byte_len > 64 * 1024) + at25->chip.flags |= EE_ADDR3; + else + at25->chip.flags |= EE_ADDR2; + + if (id[8]) { + fm25_aux_read(at25, sernum, FM25_RDSN, FM25_SN_LEN); + /* swap byte order */ + for (i = 0; i < FM25_SN_LEN; i++) + at25->sernum[i] = sernum[FM25_SN_LEN - 1 - i]; + } + + at25->chip.page_size = PAGE_SIZE; + strncpy(at25->chip.name, "fm25", sizeof(at25->chip.name)); + } + + /* For now we only support 8/16/24 bit addressing */ + if (at25->chip.flags & EE_ADDR1) + at25->addrlen = 1; + else if (at25->chip.flags & EE_ADDR2) + at25->addrlen = 2; + else if (at25->chip.flags & EE_ADDR3) + at25->addrlen = 3; + else { + dev_dbg(&spi->dev, "unsupported address type\n"); + return -EINVAL; + } + + at25->nvmem_config.type = is_fram ? NVMEM_TYPE_FRAM : NVMEM_TYPE_EEPROM; at25->nvmem_config.name = dev_name(&spi->dev); at25->nvmem_config.dev = &spi->dev; at25->nvmem_config.read_only = chip.flags & EE_READONLY; @@ -370,27 +473,22 @@ static int at25_probe(struct spi_device *spi) if (IS_ERR(at25->nvmem)) return PTR_ERR(at25->nvmem); - dev_info(&spi->dev, "%d %s %s eeprom%s, pagesize %u\n", - (chip.byte_len < 1024) ? chip.byte_len : (chip.byte_len / 1024), - (chip.byte_len < 1024) ? "Byte" : "KByte", - at25->chip.name, - (chip.flags & EE_READONLY) ? " (readonly)" : "", - at25->chip.page_size); + dev_info(&spi->dev, "%d %s %s %s%s, pagesize %u\n", + (chip.byte_len < 1024) ? chip.byte_len : (chip.byte_len / 1024), + (chip.byte_len < 1024) ? "Byte" : "KByte", + at25->chip.name, is_fram ? "fram" : "eeprom", + (chip.flags & EE_READONLY) ? " (readonly)" : "", + at25->chip.page_size); return 0; } /*-------------------------------------------------------------------------*/ -static const struct of_device_id at25_of_match[] = { - { .compatible = "atmel,at25", }, - { } -}; -MODULE_DEVICE_TABLE(of, at25_of_match); - static struct spi_driver at25_driver = { .driver = { .name = "at25", .of_match_table = at25_of_match, + .dev_groups = sernum_groups, }, .probe = at25_probe, }; diff --git a/drivers/misc/eeprom/ee1004.c b/drivers/misc/eeprom/ee1004.c index 252e15ba65e1..bb9c4512c968 100644 --- a/drivers/misc/eeprom/ee1004.c +++ b/drivers/misc/eeprom/ee1004.c @@ -32,16 +32,17 @@ */ #define EE1004_ADDR_SET_PAGE 0x36 -#define EE1004_EEPROM_SIZE 512 +#define EE1004_NUM_PAGES 2 #define EE1004_PAGE_SIZE 256 #define EE1004_PAGE_SHIFT 8 +#define EE1004_EEPROM_SIZE (EE1004_PAGE_SIZE * EE1004_NUM_PAGES) /* * Mutex protects ee1004_set_page and ee1004_dev_count, and must be held * from page selection to end of read. */ static DEFINE_MUTEX(ee1004_bus_lock); -static struct i2c_client *ee1004_set_page[2]; +static struct i2c_client *ee1004_set_page[EE1004_NUM_PAGES]; static unsigned int ee1004_dev_count; static int ee1004_current_page; @@ -71,40 +72,58 @@ static int ee1004_get_current_page(void) return 0; } +static int ee1004_set_current_page(struct device *dev, int page) +{ + int ret; + + if (page == ee1004_current_page) + return 0; + + /* Data is ignored */ + ret = i2c_smbus_write_byte(ee1004_set_page[page], 0x00); + /* + * Don't give up just yet. Some memory modules will select the page + * but not ack the command. Check which page is selected now. + */ + if (ret == -ENXIO && ee1004_get_current_page() == page) + ret = 0; + if (ret < 0) { + dev_err(dev, "Failed to select page %d (%d)\n", page, ret); + return ret; + } + + dev_dbg(dev, "Selected page %d\n", page); + ee1004_current_page = page; + + return 0; +} + static ssize_t ee1004_eeprom_read(struct i2c_client *client, char *buf, unsigned int offset, size_t count) { - int status; + int status, page; + + page = offset >> EE1004_PAGE_SHIFT; + offset &= (1 << EE1004_PAGE_SHIFT) - 1; + + status = ee1004_set_current_page(&client->dev, page); + if (status) + return status; - if (count > I2C_SMBUS_BLOCK_MAX) - count = I2C_SMBUS_BLOCK_MAX; /* Can't cross page boundaries */ - if (unlikely(offset + count > EE1004_PAGE_SIZE)) + if (offset + count > EE1004_PAGE_SIZE) count = EE1004_PAGE_SIZE - offset; - status = i2c_smbus_read_i2c_block_data_or_emulated(client, offset, - count, buf); - dev_dbg(&client->dev, "read %zu@%d --> %d\n", count, offset, status); - - return status; + return i2c_smbus_read_i2c_block_data_or_emulated(client, offset, count, buf); } -static ssize_t ee1004_read(struct file *filp, struct kobject *kobj, +static ssize_t eeprom_read(struct file *filp, struct kobject *kobj, struct bin_attribute *bin_attr, char *buf, loff_t off, size_t count) { - struct device *dev = kobj_to_dev(kobj); - struct i2c_client *client = to_i2c_client(dev); + struct i2c_client *client = kobj_to_i2c_client(kobj); size_t requested = count; - int page; - - if (unlikely(!count)) - return count; - - page = off >> EE1004_PAGE_SHIFT; - if (unlikely(page > 1)) - return 0; - off &= (1 << EE1004_PAGE_SHIFT) - 1; + int ret = 0; /* * Read data from chip, protecting against concurrent access to @@ -113,139 +132,85 @@ static ssize_t ee1004_read(struct file *filp, struct kobject *kobj, mutex_lock(&ee1004_bus_lock); while (count) { - int status; - - /* Select page */ - if (page != ee1004_current_page) { - /* Data is ignored */ - status = i2c_smbus_write_byte(ee1004_set_page[page], - 0x00); - if (status == -ENXIO) { - /* - * Don't give up just yet. Some memory - * modules will select the page but not - * ack the command. Check which page is - * selected now. - */ - if (ee1004_get_current_page() == page) - status = 0; - } - if (status < 0) { - dev_err(dev, "Failed to select page %d (%d)\n", - page, status); - mutex_unlock(&ee1004_bus_lock); - return status; - } - dev_dbg(dev, "Selected page %d\n", page); - ee1004_current_page = page; - } - - status = ee1004_eeprom_read(client, buf, off, count); - if (status < 0) { - mutex_unlock(&ee1004_bus_lock); - return status; - } - buf += status; - off += status; - count -= status; + ret = ee1004_eeprom_read(client, buf, off, count); + if (ret < 0) + goto out; - if (off == EE1004_PAGE_SIZE) { - page++; - off = 0; - } + buf += ret; + off += ret; + count -= ret; } - +out: mutex_unlock(&ee1004_bus_lock); - return requested; + return ret < 0 ? ret : requested; } -static const struct bin_attribute eeprom_attr = { - .attr = { - .name = "eeprom", - .mode = 0444, - }, - .size = EE1004_EEPROM_SIZE, - .read = ee1004_read, +static BIN_ATTR_RO(eeprom, EE1004_EEPROM_SIZE); + +static struct bin_attribute *ee1004_attrs[] = { + &bin_attr_eeprom, + NULL }; -static int ee1004_probe(struct i2c_client *client, - const struct i2c_device_id *id) +BIN_ATTRIBUTE_GROUPS(ee1004); + +static void ee1004_cleanup(int idx) +{ + if (--ee1004_dev_count == 0) + while (--idx >= 0) { + i2c_unregister_device(ee1004_set_page[idx]); + ee1004_set_page[idx] = NULL; + } +} + +static int ee1004_probe(struct i2c_client *client) { int err, cnr = 0; - const char *slow = NULL; /* Make sure we can operate on this adapter */ if (!i2c_check_functionality(client->adapter, - I2C_FUNC_SMBUS_READ_BYTE | - I2C_FUNC_SMBUS_READ_I2C_BLOCK)) { - if (i2c_check_functionality(client->adapter, - I2C_FUNC_SMBUS_READ_BYTE | - I2C_FUNC_SMBUS_READ_WORD_DATA)) - slow = "word"; - else if (i2c_check_functionality(client->adapter, - I2C_FUNC_SMBUS_READ_BYTE | - I2C_FUNC_SMBUS_READ_BYTE_DATA)) - slow = "byte"; - else - return -EPFNOSUPPORT; - } + I2C_FUNC_SMBUS_BYTE | I2C_FUNC_SMBUS_READ_I2C_BLOCK) && + !i2c_check_functionality(client->adapter, + I2C_FUNC_SMBUS_BYTE | I2C_FUNC_SMBUS_READ_BYTE_DATA)) + return -EPFNOSUPPORT; /* Use 2 dummy devices for page select command */ mutex_lock(&ee1004_bus_lock); if (++ee1004_dev_count == 1) { - for (cnr = 0; cnr < 2; cnr++) { - ee1004_set_page[cnr] = i2c_new_dummy_device(client->adapter, - EE1004_ADDR_SET_PAGE + cnr); - if (IS_ERR(ee1004_set_page[cnr])) { - dev_err(&client->dev, - "address 0x%02x unavailable\n", - EE1004_ADDR_SET_PAGE + cnr); - err = PTR_ERR(ee1004_set_page[cnr]); + for (cnr = 0; cnr < EE1004_NUM_PAGES; cnr++) { + struct i2c_client *cl; + + cl = i2c_new_dummy_device(client->adapter, EE1004_ADDR_SET_PAGE + cnr); + if (IS_ERR(cl)) { + err = PTR_ERR(cl); goto err_clients; } + ee1004_set_page[cnr] = cl; } - } else if (i2c_adapter_id(client->adapter) != - i2c_adapter_id(ee1004_set_page[0]->adapter)) { + + /* Remember current page to avoid unneeded page select */ + err = ee1004_get_current_page(); + if (err < 0) + goto err_clients; + dev_dbg(&client->dev, "Currently selected page: %d\n", err); + ee1004_current_page = err; + } else if (client->adapter != ee1004_set_page[0]->adapter) { dev_err(&client->dev, "Driver only supports devices on a single I2C bus\n"); err = -EOPNOTSUPP; goto err_clients; } - - /* Remember current page to avoid unneeded page select */ - err = ee1004_get_current_page(); - if (err < 0) - goto err_clients; - ee1004_current_page = err; - dev_dbg(&client->dev, "Currently selected page: %d\n", - ee1004_current_page); mutex_unlock(&ee1004_bus_lock); - /* Create the sysfs eeprom file */ - err = sysfs_create_bin_file(&client->dev.kobj, &eeprom_attr); - if (err) - goto err_clients_lock; - dev_info(&client->dev, "%u byte EE1004-compliant SPD EEPROM, read-only\n", EE1004_EEPROM_SIZE); - if (slow) - dev_notice(&client->dev, - "Falling back to %s reads, performance will suffer\n", - slow); return 0; - err_clients_lock: - mutex_lock(&ee1004_bus_lock); err_clients: - if (--ee1004_dev_count == 0) { - for (cnr--; cnr >= 0; cnr--) { - i2c_unregister_device(ee1004_set_page[cnr]); - ee1004_set_page[cnr] = NULL; - } - } + ee1004_cleanup(cnr); mutex_unlock(&ee1004_bus_lock); return err; @@ -253,18 +218,9 @@ static int ee1004_probe(struct i2c_client *client, static int ee1004_remove(struct i2c_client *client) { - int i; - - sysfs_remove_bin_file(&client->dev.kobj, &eeprom_attr); - /* Remove page select clients if this is the last device */ mutex_lock(&ee1004_bus_lock); - if (--ee1004_dev_count == 0) { - for (i = 0; i < 2; i++) { - i2c_unregister_device(ee1004_set_page[i]); - ee1004_set_page[i] = NULL; - } - } + ee1004_cleanup(EE1004_NUM_PAGES); mutex_unlock(&ee1004_bus_lock); return 0; @@ -275,8 +231,9 @@ static int ee1004_remove(struct i2c_client *client) static struct i2c_driver ee1004_driver = { .driver = { .name = "ee1004", + .dev_groups = ee1004_groups, }, - .probe = ee1004_probe, + .probe_new = ee1004_probe, .remove = ee1004_remove, .id_table = ee1004_ids, }; diff --git a/drivers/misc/eeprom/eeprom_93xx46.c b/drivers/misc/eeprom/eeprom_93xx46.c index 80114f4c80ad..29d8971ec558 100644 --- a/drivers/misc/eeprom/eeprom_93xx46.c +++ b/drivers/misc/eeprom/eeprom_93xx46.c @@ -9,6 +9,7 @@ #include <linux/device.h> #include <linux/gpio/consumer.h> #include <linux/kernel.h> +#include <linux/log2.h> #include <linux/module.h> #include <linux/mutex.h> #include <linux/of.h> @@ -28,14 +29,29 @@ struct eeprom_93xx46_devtype_data { unsigned int quirks; + unsigned char flags; +}; + +static const struct eeprom_93xx46_devtype_data at93c46_data = { + .flags = EE_SIZE1K, +}; + +static const struct eeprom_93xx46_devtype_data at93c56_data = { + .flags = EE_SIZE2K, +}; + +static const struct eeprom_93xx46_devtype_data at93c66_data = { + .flags = EE_SIZE4K, }; static const struct eeprom_93xx46_devtype_data atmel_at93c46d_data = { + .flags = EE_SIZE1K, .quirks = EEPROM_93XX46_QUIRK_SINGLE_WORD_READ | EEPROM_93XX46_QUIRK_INSTRUCTION_LENGTH, }; static const struct eeprom_93xx46_devtype_data microchip_93lc46b_data = { + .flags = EE_SIZE1K, .quirks = EEPROM_93XX46_QUIRK_EXTRA_READ_CYCLE, }; @@ -70,6 +86,7 @@ static int eeprom_93xx46_read(void *priv, unsigned int off, struct eeprom_93xx46_dev *edev = priv; char *buf = val; int err = 0; + int bits; if (unlikely(off >= edev->size)) return 0; @@ -83,21 +100,21 @@ static int eeprom_93xx46_read(void *priv, unsigned int off, if (edev->pdata->prepare) edev->pdata->prepare(edev); + /* The opcode in front of the address is three bits. */ + bits = edev->addrlen + 3; + while (count) { struct spi_message m; struct spi_transfer t[2] = { { 0 } }; u16 cmd_addr = OP_READ << edev->addrlen; size_t nbytes = count; - int bits; - if (edev->addrlen == 7) { - cmd_addr |= off & 0x7f; - bits = 10; + if (edev->pdata->flags & EE_ADDR8) { + cmd_addr |= off; if (has_quirk_single_word_read(edev)) nbytes = 1; } else { - cmd_addr |= (off >> 1) & 0x3f; - bits = 9; + cmd_addr |= (off >> 1); if (has_quirk_single_word_read(edev)) nbytes = 2; } @@ -152,14 +169,14 @@ static int eeprom_93xx46_ew(struct eeprom_93xx46_dev *edev, int is_on) int bits, ret; u16 cmd_addr; + /* The opcode in front of the address is three bits. */ + bits = edev->addrlen + 3; + cmd_addr = OP_START << edev->addrlen; - if (edev->addrlen == 7) { + if (edev->pdata->flags & EE_ADDR8) cmd_addr |= (is_on ? ADDR_EWEN : ADDR_EWDS) << 1; - bits = 10; - } else { + else cmd_addr |= (is_on ? ADDR_EWEN : ADDR_EWDS); - bits = 9; - } if (has_quirk_instruction_length(edev)) { cmd_addr <<= 2; @@ -205,15 +222,19 @@ eeprom_93xx46_write_word(struct eeprom_93xx46_dev *edev, int bits, data_len, ret; u16 cmd_addr; + if (unlikely(off >= edev->size)) + return -EINVAL; + + /* The opcode in front of the address is three bits. */ + bits = edev->addrlen + 3; + cmd_addr = OP_WRITE << edev->addrlen; - if (edev->addrlen == 7) { - cmd_addr |= off & 0x7f; - bits = 10; + if (edev->pdata->flags & EE_ADDR8) { + cmd_addr |= off; data_len = 1; } else { - cmd_addr |= (off >> 1) & 0x3f; - bits = 9; + cmd_addr |= (off >> 1); data_len = 2; } @@ -253,7 +274,7 @@ static int eeprom_93xx46_write(void *priv, unsigned int off, return count; /* only write even number of bytes on 16-bit devices */ - if (edev->addrlen == 6) { + if (edev->pdata->flags & EE_ADDR16) { step = 2; count &= ~1; } @@ -295,14 +316,14 @@ static int eeprom_93xx46_eral(struct eeprom_93xx46_dev *edev) int bits, ret; u16 cmd_addr; + /* The opcode in front of the address is three bits. */ + bits = edev->addrlen + 3; + cmd_addr = OP_START << edev->addrlen; - if (edev->addrlen == 7) { + if (edev->pdata->flags & EE_ADDR8) cmd_addr |= ADDR_ERAL << 1; - bits = 10; - } else { + else cmd_addr |= ADDR_ERAL; - bits = 9; - } if (has_quirk_instruction_length(edev)) { cmd_addr <<= 2; @@ -375,8 +396,11 @@ static void select_deassert(void *context) } static const struct of_device_id eeprom_93xx46_of_table[] = { - { .compatible = "eeprom-93xx46", }, + { .compatible = "eeprom-93xx46", .data = &at93c46_data, }, + { .compatible = "atmel,at93c46", .data = &at93c46_data, }, { .compatible = "atmel,at93c46d", .data = &atmel_at93c46d_data, }, + { .compatible = "atmel,at93c56", .data = &at93c56_data, }, + { .compatible = "atmel,at93c66", .data = &at93c66_data, }, { .compatible = "microchip,93lc46b", .data = µchip_93lc46b_data, }, {} }; @@ -426,6 +450,7 @@ static int eeprom_93xx46_probe_dt(struct spi_device *spi) const struct eeprom_93xx46_devtype_data *data = of_id->data; pd->quirks = data->quirks; + pd->flags |= data->flags; } spi->dev.platform_data = pd; @@ -455,10 +480,21 @@ static int eeprom_93xx46_probe(struct spi_device *spi) if (!edev) return -ENOMEM; + if (pd->flags & EE_SIZE1K) + edev->size = 128; + else if (pd->flags & EE_SIZE2K) + edev->size = 256; + else if (pd->flags & EE_SIZE4K) + edev->size = 512; + else { + dev_err(&spi->dev, "unspecified size\n"); + return -EINVAL; + } + if (pd->flags & EE_ADDR8) - edev->addrlen = 7; + edev->addrlen = ilog2(edev->size); else if (pd->flags & EE_ADDR16) - edev->addrlen = 6; + edev->addrlen = ilog2(edev->size) - 1; else { dev_err(&spi->dev, "unspecified address type\n"); return -EINVAL; @@ -469,7 +505,6 @@ static int eeprom_93xx46_probe(struct spi_device *spi) edev->spi = spi; edev->pdata = pd; - edev->size = 128; edev->nvmem_config.type = NVMEM_TYPE_EEPROM; edev->nvmem_config.name = dev_name(&spi->dev); edev->nvmem_config.dev = &spi->dev; @@ -489,8 +524,9 @@ static int eeprom_93xx46_probe(struct spi_device *spi) if (IS_ERR(edev->nvmem)) return PTR_ERR(edev->nvmem); - dev_info(&spi->dev, "%d-bit eeprom %s\n", + dev_info(&spi->dev, "%d-bit eeprom containing %d bytes %s\n", (pd->flags & EE_ADDR8) ? 8 : 16, + edev->size, (pd->flags & EE_READONLY) ? "(readonly)" : ""); if (!(pd->flags & EE_READONLY)) { diff --git a/drivers/misc/eeprom/idt_89hpesx.c b/drivers/misc/eeprom/idt_89hpesx.c index 81c70e5bc168..b0cff4b152da 100644 --- a/drivers/misc/eeprom/idt_89hpesx.c +++ b/drivers/misc/eeprom/idt_89hpesx.c @@ -1,38 +1,7 @@ +// SPDX-License-Identifier: GPL-2.0-only /* - * This file is provided under a GPLv2 license. When using or - * redistributing this file, you may do so under that license. - * - * GPL LICENSE SUMMARY - * * Copyright (C) 2016 T-Platforms. All Rights Reserved. * - * This program is free software; you can redistribute it and/or modify it - * under the terms and conditions of the GNU General Public License, - * version 2, as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for - * more details. - * - * You should have received a copy of the GNU General Public License along - * with this program; if not, it can be found <http://www.gnu.org/licenses/>. - * - * The full GNU General Public License is included in this distribution in - * the file called "COPYING". - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * * IDT PCIe-switch NTB Linux driver * * Contact Information: @@ -1126,11 +1095,10 @@ static void idt_get_fw_data(struct idt_89hpesx_dev *pdev) device_for_each_child_node(dev, fwnode) { ee_id = idt_ee_match_id(fwnode); - if (!ee_id) { - dev_warn(dev, "Skip unsupported EEPROM device"); - continue; - } else + if (ee_id) break; + + dev_warn(dev, "Skip unsupported EEPROM device %pfw\n", fwnode); } /* If there is no fwnode EEPROM device, then set zero size */ @@ -1161,6 +1129,7 @@ static void idt_get_fw_data(struct idt_89hpesx_dev *pdev) else /* if (!fwnode_property_read_bool(node, "read-only")) */ pdev->eero = false; + fwnode_handle_put(fwnode); dev_info(dev, "EEPROM of %d bytes found by 0x%x", pdev->eesize, pdev->eeaddr); } diff --git a/drivers/misc/habanalabs/common/command_submission.c b/drivers/misc/habanalabs/common/command_submission.c index af3c497defb1..80c60fb41bbc 100644 --- a/drivers/misc/habanalabs/common/command_submission.c +++ b/drivers/misc/habanalabs/common/command_submission.c @@ -556,6 +556,13 @@ out: else if (!cs->submitted) cs->fence->error = -EBUSY; + if (unlikely(cs->skip_reset_on_timeout)) { + dev_err(hdev->dev, + "Command submission %llu completed after %llu (s)\n", + cs->sequence, + div_u64(jiffies - cs->submission_time_jiffies, HZ)); + } + if (cs->timestamp) cs->fence->timestamp = ktime_get(); complete_all(&cs->fence->completion); @@ -571,6 +578,8 @@ static void cs_timedout(struct work_struct *work) int rc; struct hl_cs *cs = container_of(work, struct hl_cs, work_tdr.work); + bool skip_reset_on_timeout = cs->skip_reset_on_timeout; + rc = cs_get_unless_zero(cs); if (!rc) return; @@ -581,7 +590,8 @@ static void cs_timedout(struct work_struct *work) } /* Mark the CS is timed out so we won't try to cancel its TDR */ - cs->timedout = true; + if (likely(!skip_reset_on_timeout)) + cs->timedout = true; hdev = cs->ctx->hdev; @@ -613,10 +623,12 @@ static void cs_timedout(struct work_struct *work) cs_put(cs); - if (hdev->reset_on_lockup) - hl_device_reset(hdev, 0); - else - hdev->needs_reset = true; + if (likely(!skip_reset_on_timeout)) { + if (hdev->reset_on_lockup) + hl_device_reset(hdev, HL_RESET_TDR); + else + hdev->needs_reset = true; + } } static int allocate_cs(struct hl_device *hdev, struct hl_ctx *ctx, @@ -650,6 +662,10 @@ static int allocate_cs(struct hl_device *hdev, struct hl_ctx *ctx, cs->type = cs_type; cs->timestamp = !!(flags & HL_CS_FLAGS_TIMESTAMP); cs->timeout_jiffies = timeout; + cs->skip_reset_on_timeout = + hdev->skip_reset_on_timeout || + !!(flags & HL_CS_FLAGS_SKIP_RESET_ON_TIMEOUT); + cs->submission_time_jiffies = jiffies; INIT_LIST_HEAD(&cs->job_list); INIT_DELAYED_WORK(&cs->work_tdr, cs_timedout); kref_init(&cs->refcount); @@ -1481,6 +1497,61 @@ out: return rc; } +/* + * hl_cs_signal_sob_wraparound_handler: handle SOB value wrapaound case. + * if the SOB value reaches the max value move to the other SOB reserved + * to the queue. + * Note that this function must be called while hw_queues_lock is taken. + */ +int hl_cs_signal_sob_wraparound_handler(struct hl_device *hdev, u32 q_idx, + struct hl_hw_sob **hw_sob, u32 count) +{ + struct hl_sync_stream_properties *prop; + struct hl_hw_sob *sob = *hw_sob, *other_sob; + u8 other_sob_offset; + + prop = &hdev->kernel_queues[q_idx].sync_stream_prop; + + kref_get(&sob->kref); + + /* check for wraparound */ + if (prop->next_sob_val + count >= HL_MAX_SOB_VAL) { + /* + * Decrement as we reached the max value. + * The release function won't be called here as we've + * just incremented the refcount right before calling this + * function. + */ + kref_put(&sob->kref, hl_sob_reset_error); + + /* + * check the other sob value, if it still in use then fail + * otherwise make the switch + */ + other_sob_offset = (prop->curr_sob_offset + 1) % HL_RSVD_SOBS; + other_sob = &prop->hw_sob[other_sob_offset]; + + if (kref_read(&other_sob->kref) != 1) { + dev_err(hdev->dev, "error: Cannot switch SOBs q_idx: %d\n", + q_idx); + return -EINVAL; + } + + prop->next_sob_val = 1; + + /* only two SOBs are currently in use */ + prop->curr_sob_offset = other_sob_offset; + *hw_sob = other_sob; + + dev_dbg(hdev->dev, "switched to SOB %d, q_idx: %d\n", + prop->curr_sob_offset, q_idx); + } else { + prop->next_sob_val += count; + } + + return 0; +} + static int cs_ioctl_extract_signal_seq(struct hl_device *hdev, struct hl_cs_chunk *chunk, u64 *signal_seq, struct hl_ctx *ctx) { diff --git a/drivers/misc/habanalabs/common/context.c b/drivers/misc/habanalabs/common/context.c index 62d705889ca8..19b6b045219e 100644 --- a/drivers/misc/habanalabs/common/context.c +++ b/drivers/misc/habanalabs/common/context.c @@ -12,7 +12,6 @@ static void hl_ctx_fini(struct hl_ctx *ctx) { struct hl_device *hdev = ctx->hdev; - u64 idle_mask[HL_BUSY_ENGINES_MASK_EXT_SIZE] = {0}; int i; /* Release all allocated pending cb's, those cb's were never @@ -57,14 +56,6 @@ static void hl_ctx_fini(struct hl_ctx *ctx) /* Scrub both SRAM and DRAM */ hdev->asic_funcs->scrub_device_mem(hdev, 0, 0); - - if ((!hdev->pldm) && (hdev->pdev) && - (!hdev->asic_funcs->is_device_idle(hdev, - idle_mask, - HL_BUSY_ENGINES_MASK_EXT_SIZE, NULL))) - dev_notice(hdev->dev, - "device not idle after user context is closed (0x%llx, 0x%llx)\n", - idle_mask[0], idle_mask[1]); } else { dev_dbg(hdev->dev, "closing kernel context\n"); hdev->asic_funcs->ctx_fini(ctx); diff --git a/drivers/misc/habanalabs/common/debugfs.c b/drivers/misc/habanalabs/common/debugfs.c index 8381155578a0..703d79fb6f3f 100644 --- a/drivers/misc/habanalabs/common/debugfs.c +++ b/drivers/misc/habanalabs/common/debugfs.c @@ -1278,6 +1278,11 @@ void hl_debugfs_add_device(struct hl_device *hdev) dev_entry->root, &dev_entry->blob_desc); + debugfs_create_x8("skip_reset_on_timeout", + 0644, + dev_entry->root, + &hdev->skip_reset_on_timeout); + for (i = 0, entry = dev_entry->entry_arr ; i < count ; i++, entry++) { debugfs_create_file(hl_debugfs_list[i].name, 0444, diff --git a/drivers/misc/habanalabs/common/device.c b/drivers/misc/habanalabs/common/device.c index 00e92b678828..ff4cbde289c0 100644 --- a/drivers/misc/habanalabs/common/device.c +++ b/drivers/misc/habanalabs/common/device.c @@ -51,6 +51,8 @@ bool hl_device_operational(struct hl_device *hdev, static void hpriv_release(struct kref *ref) { + u64 idle_mask[HL_BUSY_ENGINES_MASK_EXT_SIZE] = {0}; + bool device_is_idle = true; struct hl_fpriv *hpriv; struct hl_device *hdev; @@ -71,8 +73,20 @@ static void hpriv_release(struct kref *ref) kfree(hpriv); - if (hdev->reset_upon_device_release) - hl_device_reset(hdev, 0); + if ((!hdev->pldm) && (hdev->pdev) && + (!hdev->asic_funcs->is_device_idle(hdev, + idle_mask, + HL_BUSY_ENGINES_MASK_EXT_SIZE, NULL))) { + dev_err(hdev->dev, + "device not idle after user context is closed (0x%llx_%llx)\n", + idle_mask[1], idle_mask[0]); + + device_is_idle = false; + } + + if ((hdev->reset_if_device_not_idle && !device_is_idle) + || hdev->reset_upon_device_release) + hl_device_reset(hdev, HL_RESET_DEVICE_RELEASE); } void hl_hpriv_get(struct hl_fpriv *hpriv) @@ -118,6 +132,9 @@ static int hl_device_release(struct inode *inode, struct file *filp) dev_warn(hdev->dev, "Device is still in use because there are live CS and/or memory mappings\n"); + hdev->last_open_session_duration_jif = + jiffies - hdev->last_successful_open_jif; + return 0; } @@ -868,7 +885,7 @@ static void device_disable_open_processes(struct hl_device *hdev) int hl_device_reset(struct hl_device *hdev, u32 flags) { u64 idle_mask[HL_BUSY_ENGINES_MASK_EXT_SIZE] = {0}; - bool hard_reset, from_hard_reset_thread; + bool hard_reset, from_hard_reset_thread, hard_instead_soft = false; int i, rc; if (!hdev->init_done) { @@ -880,11 +897,28 @@ int hl_device_reset(struct hl_device *hdev, u32 flags) hard_reset = (flags & HL_RESET_HARD) != 0; from_hard_reset_thread = (flags & HL_RESET_FROM_RESET_THREAD) != 0; - if ((!hard_reset) && (!hdev->supports_soft_reset)) { - dev_dbg(hdev->dev, "Doing hard-reset instead of soft-reset\n"); + if (!hard_reset && !hdev->supports_soft_reset) { + hard_instead_soft = true; + hard_reset = true; + } + + if (hdev->reset_upon_device_release && + (flags & HL_RESET_DEVICE_RELEASE)) { + dev_dbg(hdev->dev, + "Perform %s-reset upon device release\n", + hard_reset ? "hard" : "soft"); + goto do_reset; + } + + if (!hard_reset && !hdev->allow_external_soft_reset) { + hard_instead_soft = true; hard_reset = true; } + if (hard_instead_soft) + dev_dbg(hdev->dev, "Doing hard-reset instead of soft-reset\n"); + +do_reset: /* Re-entry of reset thread */ if (from_hard_reset_thread && hdev->process_kill_trial_cnt) goto kill_processes; @@ -901,6 +935,19 @@ int hl_device_reset(struct hl_device *hdev, u32 flags) return 0; /* + * 'reset cause' is being updated here, because getting here + * means that it's the 1st time and the last time we're here + * ('in_reset' makes sure of it). This makes sure that + * 'reset_cause' will continue holding its 1st recorded reason! + */ + if (flags & HL_RESET_HEARTBEAT) + hdev->curr_reset_cause = HL_RESET_CAUSE_HEARTBEAT; + else if (flags & HL_RESET_TDR) + hdev->curr_reset_cause = HL_RESET_CAUSE_TDR; + else + hdev->curr_reset_cause = HL_RESET_CAUSE_UNKNOWN; + + /* * if reset is due to heartbeat, device CPU is no responsive in * which case no point sending PCI disable message to it */ @@ -943,9 +990,8 @@ again: hdev->process_kill_trial_cnt = 0; /* - * Because the reset function can't run from interrupt or - * from heartbeat work, we need to call the reset function - * from a dedicated work + * Because the reset function can't run from heartbeat work, + * we need to call the reset function from a dedicated work. */ queue_delayed_work(hdev->device_reset_work.wq, &hdev->device_reset_work.reset_work, 0); @@ -1096,8 +1142,8 @@ kill_processes: if (!hdev->asic_funcs->is_device_idle(hdev, idle_mask, HL_BUSY_ENGINES_MASK_EXT_SIZE, NULL)) { dev_err(hdev->dev, - "device is not idle (mask %#llx %#llx) after reset\n", - idle_mask[0], idle_mask[1]); + "device is not idle (mask 0x%llx_%llx) after reset\n", + idle_mask[1], idle_mask[0]); rc = -EIO; goto out_err; } @@ -1334,8 +1380,9 @@ int hl_device_init(struct hl_device *hdev, struct class *hclass) } /* - * From this point, in case of an error, add char devices and create - * sysfs nodes as part of the error flow, to allow debugging. + * From this point, override rc (=0) in case of an error to allow + * debugging (by adding char devices and create sysfs nodes as part of + * the error flow). */ add_cdev_sysfs_on_err = true; @@ -1369,7 +1416,7 @@ int hl_device_init(struct hl_device *hdev, struct class *hclass) dev_info(hdev->dev, "Found %s device with %lluGB DRAM\n", hdev->asic_name, - hdev->asic_prop.dram_size / 1024 / 1024 / 1024); + hdev->asic_prop.dram_size / SZ_1G); rc = hl_vm_init(hdev); if (rc) { @@ -1475,6 +1522,7 @@ out_disabled: void hl_device_fini(struct hl_device *hdev) { ktime_t timeout; + u64 reset_sec; int i, rc; dev_info(hdev->dev, "Removing device\n"); @@ -1482,6 +1530,11 @@ void hl_device_fini(struct hl_device *hdev) hdev->device_fini_pending = 1; flush_delayed_work(&hdev->device_reset_work.reset_work); + if (hdev->pldm) + reset_sec = HL_PLDM_HARD_RESET_MAX_TIMEOUT; + else + reset_sec = HL_HARD_RESET_MAX_TIMEOUT; + /* * This function is competing with the reset function, so try to * take the reset atomic and if we are already in middle of reset, @@ -1490,8 +1543,7 @@ void hl_device_fini(struct hl_device *hdev) * ports, the hard reset could take between 10-30 seconds */ - timeout = ktime_add_us(ktime_get(), - HL_HARD_RESET_MAX_TIMEOUT * 1000 * 1000); + timeout = ktime_add_us(ktime_get(), reset_sec * 1000 * 1000); rc = atomic_cmpxchg(&hdev->in_reset, 0, 1); while (rc) { usleep_range(50, 200); diff --git a/drivers/misc/habanalabs/common/firmware_if.c b/drivers/misc/habanalabs/common/firmware_if.c index 0713b2c12d54..2e4d04ec6b53 100644 --- a/drivers/misc/habanalabs/common/firmware_if.c +++ b/drivers/misc/habanalabs/common/firmware_if.c @@ -9,42 +9,63 @@ #include "../include/common/hl_boot_if.h" #include <linux/firmware.h> +#include <linux/crc32.h> #include <linux/slab.h> +#include <linux/ctype.h> -#define FW_FILE_MAX_SIZE 0x1400000 /* maximum size of 20MB */ -/** - * hl_fw_load_fw_to_device() - Load F/W code to device's memory. - * - * @hdev: pointer to hl_device structure. - * @fw_name: the firmware image name - * @dst: IO memory mapped address space to copy firmware to - * @src_offset: offset in src FW to copy from - * @size: amount of bytes to copy (0 to copy the whole binary) - * - * Copy fw code from firmware file to device memory. - * - * Return: 0 on success, non-zero for failure. - */ -int hl_fw_load_fw_to_device(struct hl_device *hdev, const char *fw_name, - void __iomem *dst, u32 src_offset, u32 size) +#define FW_FILE_MAX_SIZE 0x1400000 /* maximum size of 20MB */ + +#define FW_CPU_STATUS_POLL_INTERVAL_USEC 10000 + +static char *extract_fw_ver_from_str(const char *fw_str) +{ + char *str, *fw_ver, *whitespace; + + fw_ver = kmalloc(16, GFP_KERNEL); + if (!fw_ver) + return NULL; + + str = strnstr(fw_str, "fw-", VERSION_MAX_LEN); + if (!str) + goto free_fw_ver; + + /* Skip the fw- part */ + str += 3; + + /* Copy until the next whitespace */ + whitespace = strnstr(str, " ", 15); + if (!whitespace) + goto free_fw_ver; + + strscpy(fw_ver, str, whitespace - str + 1); + + return fw_ver; + +free_fw_ver: + kfree(fw_ver); + return NULL; +} + +static int hl_request_fw(struct hl_device *hdev, + const struct firmware **firmware_p, + const char *fw_name) { - const struct firmware *fw; - const void *fw_data; size_t fw_size; int rc; - rc = request_firmware(&fw, fw_name, hdev->dev); + rc = request_firmware(firmware_p, fw_name, hdev->dev); if (rc) { - dev_err(hdev->dev, "Firmware file %s is not found!\n", fw_name); + dev_err(hdev->dev, "Firmware file %s is not found! (error %d)\n", + fw_name, rc); goto out; } - fw_size = fw->size; + fw_size = (*firmware_p)->size; if ((fw_size % 4) != 0) { dev_err(hdev->dev, "Illegal %s firmware size %zu\n", - fw_name, fw_size); + fw_name, fw_size); rc = -EINVAL; - goto out; + goto release_fw; } dev_dbg(hdev->dev, "%s firmware size == %zu\n", fw_name, fw_size); @@ -54,26 +75,125 @@ int hl_fw_load_fw_to_device(struct hl_device *hdev, const char *fw_name, "FW file size %zu exceeds maximum of %u bytes\n", fw_size, FW_FILE_MAX_SIZE); rc = -EINVAL; - goto out; + goto release_fw; } - if (size - src_offset > fw_size) { + return 0; + +release_fw: + release_firmware(*firmware_p); +out: + return rc; +} + +/** + * hl_release_firmware() - release FW + * + * @fw: fw descriptor + * + * note: this inline function added to serve as a comprehensive mirror for the + * hl_request_fw function. + */ +static inline void hl_release_firmware(const struct firmware *fw) +{ + release_firmware(fw); +} + +/** + * hl_fw_copy_fw_to_device() - copy FW to device + * + * @hdev: pointer to hl_device structure. + * @fw: fw descriptor + * @dst: IO memory mapped address space to copy firmware to + * @src_offset: offset in src FW to copy from + * @size: amount of bytes to copy (0 to copy the whole binary) + * + * actual copy of FW binary data to device, shared by static and dynamic loaders + */ +static int hl_fw_copy_fw_to_device(struct hl_device *hdev, + const struct firmware *fw, void __iomem *dst, + u32 src_offset, u32 size) +{ + const void *fw_data; + + /* size 0 indicates to copy the whole file */ + if (!size) + size = fw->size; + + if (src_offset + size > fw->size) { dev_err(hdev->dev, "size to copy(%u) and offset(%u) are invalid\n", size, src_offset); - rc = -EINVAL; - goto out; + return -EINVAL; } - if (size) - fw_size = size; - fw_data = (const void *) fw->data; - memcpy_toio(dst, fw_data + src_offset, fw_size); + memcpy_toio(dst, fw_data + src_offset, size); + return 0; +} -out: - release_firmware(fw); +/** + * hl_fw_copy_msg_to_device() - copy message to device + * + * @hdev: pointer to hl_device structure. + * @msg: message + * @dst: IO memory mapped address space to copy firmware to + * @src_offset: offset in src message to copy from + * @size: amount of bytes to copy (0 to copy the whole binary) + * + * actual copy of message data to device. + */ +static int hl_fw_copy_msg_to_device(struct hl_device *hdev, + struct lkd_msg_comms *msg, void __iomem *dst, + u32 src_offset, u32 size) +{ + void *msg_data; + + /* size 0 indicates to copy the whole file */ + if (!size) + size = sizeof(struct lkd_msg_comms); + + if (src_offset + size > sizeof(struct lkd_msg_comms)) { + dev_err(hdev->dev, + "size to copy(%u) and offset(%u) are invalid\n", + size, src_offset); + return -EINVAL; + } + + msg_data = (void *) msg; + + memcpy_toio(dst, msg_data + src_offset, size); + + return 0; +} + +/** + * hl_fw_load_fw_to_device() - Load F/W code to device's memory. + * + * @hdev: pointer to hl_device structure. + * @fw_name: the firmware image name + * @dst: IO memory mapped address space to copy firmware to + * @src_offset: offset in src FW to copy from + * @size: amount of bytes to copy (0 to copy the whole binary) + * + * Copy fw code from firmware file to device memory. + * + * Return: 0 on success, non-zero for failure. + */ +int hl_fw_load_fw_to_device(struct hl_device *hdev, const char *fw_name, + void __iomem *dst, u32 src_offset, u32 size) +{ + const struct firmware *fw; + int rc; + + rc = hl_request_fw(hdev, &fw, fw_name); + if (rc) + return rc; + + rc = hl_fw_copy_fw_to_device(hdev, fw, dst, src_offset, size); + + hl_release_firmware(fw); return rc; } @@ -91,6 +211,7 @@ int hl_fw_send_cpu_message(struct hl_device *hdev, u32 hw_queue_id, u32 *msg, u16 len, u32 timeout, u64 *result) { struct hl_hw_queue *queue = &hdev->kernel_queues[hw_queue_id]; + struct asic_fixed_properties *prop = &hdev->asic_prop; struct cpucp_packet *pkt; dma_addr_t pkt_dma_addr; u32 tmp, expected_ack_val; @@ -117,7 +238,7 @@ int hl_fw_send_cpu_message(struct hl_device *hdev, u32 hw_queue_id, u32 *msg, } /* set fence to a non valid value */ - pkt->fence = UINT_MAX; + pkt->fence = cpu_to_le32(UINT_MAX); rc = hl_hw_queue_send_cb_no_cmpl(hdev, hw_queue_id, len, pkt_dma_addr); if (rc) { @@ -125,8 +246,7 @@ int hl_fw_send_cpu_message(struct hl_device *hdev, u32 hw_queue_id, u32 *msg, goto out; } - if (hdev->asic_prop.fw_app_security_map & - CPU_BOOT_DEV_STS0_PKT_PI_ACK_EN) + if (prop->fw_app_cpu_boot_dev_sts0 & CPU_BOOT_DEV_STS0_PKT_PI_ACK_EN) expected_ack_val = queue->pi; else expected_ack_val = CPUCP_PACKET_FENCE_VAL; @@ -272,10 +392,11 @@ void hl_fw_cpu_accessible_dma_pool_free(struct hl_device *hdev, size_t size, int hl_fw_send_heartbeat(struct hl_device *hdev) { - struct cpucp_packet hb_pkt = {}; + struct cpucp_packet hb_pkt; u64 result; int rc; + memset(&hb_pkt, 0, sizeof(hb_pkt)); hb_pkt.ctl = cpu_to_le32(CPUCP_PACKET_TEST << CPUCP_PKT_CTL_OPCODE_SHIFT); hb_pkt.value = cpu_to_le64(CPUCP_PACKET_FENCE_VAL); @@ -284,29 +405,24 @@ int hl_fw_send_heartbeat(struct hl_device *hdev) sizeof(hb_pkt), 0, &result); if ((rc) || (result != CPUCP_PACKET_FENCE_VAL)) + return -EIO; + + if (le32_to_cpu(hb_pkt.status_mask) & + CPUCP_PKT_HB_STATUS_EQ_FAULT_MASK) { + dev_warn(hdev->dev, "FW reported EQ fault during heartbeat\n"); rc = -EIO; + } return rc; } -static int fw_read_errors(struct hl_device *hdev, u32 boot_err0_reg, - u32 cpu_security_boot_status_reg) +static bool fw_report_boot_dev0(struct hl_device *hdev, u32 err_val, + u32 sts_val) { - u32 err_val, security_val; bool err_exists = false; - /* Some of the firmware status codes are deprecated in newer f/w - * versions. In those versions, the errors are reported - * in different registers. Therefore, we need to check those - * registers and print the exact errors. Moreover, there - * may be multiple errors, so we need to report on each error - * separately. Some of the error codes might indicate a state - * that is not an error per-se, but it is an error in production - * environment - */ - err_val = RREG32(boot_err0_reg); if (!(err_val & CPU_BOOT_ERR0_ENABLED)) - return 0; + return false; if (err_val & CPU_BOOT_ERR0_DRAM_INIT_FAIL) { dev_err(hdev->dev, @@ -377,44 +493,120 @@ static int fw_read_errors(struct hl_device *hdev, u32 boot_err0_reg, err_exists = true; } + if (err_val & CPU_BOOT_ERR0_PRI_IMG_VER_FAIL) { + dev_warn(hdev->dev, + "Device boot warning - Failed to load preboot primary image\n"); + /* This is a warning so we don't want it to disable the + * device as we have a secondary preboot image + */ + err_val &= ~CPU_BOOT_ERR0_PRI_IMG_VER_FAIL; + } + + if (err_val & CPU_BOOT_ERR0_SEC_IMG_VER_FAIL) { + dev_err(hdev->dev, "Device boot error - Failed to load preboot secondary image\n"); + err_exists = true; + } + if (err_val & CPU_BOOT_ERR0_PLL_FAIL) { dev_err(hdev->dev, "Device boot error - PLL failure\n"); err_exists = true; } if (err_val & CPU_BOOT_ERR0_DEVICE_UNUSABLE_FAIL) { - dev_err(hdev->dev, - "Device boot error - device unusable\n"); - err_exists = true; + /* Ignore this bit, don't prevent driver loading */ + dev_dbg(hdev->dev, "device unusable status is set\n"); + err_val &= ~CPU_BOOT_ERR0_DEVICE_UNUSABLE_FAIL; } - security_val = RREG32(cpu_security_boot_status_reg); - if (security_val & CPU_BOOT_DEV_STS0_ENABLED) - dev_dbg(hdev->dev, "Device security status %#x\n", - security_val); + if (sts_val & CPU_BOOT_DEV_STS0_ENABLED) + dev_dbg(hdev->dev, "Device status0 %#x\n", sts_val); if (!err_exists && (err_val & ~CPU_BOOT_ERR0_ENABLED)) { dev_err(hdev->dev, - "Device boot error - unknown error 0x%08x\n", - err_val); + "Device boot error - unknown ERR0 error 0x%08x\n", err_val); err_exists = true; } + /* return error only if it's in the predefined mask */ if (err_exists && ((err_val & ~CPU_BOOT_ERR0_ENABLED) & lower_32_bits(hdev->boot_error_status_mask))) + return true; + + return false; +} + +/* placeholder for ERR1 as no errors defined there yet */ +static bool fw_report_boot_dev1(struct hl_device *hdev, u32 err_val, + u32 sts_val) +{ + /* + * keep this variable to preserve the logic of the function. + * this way it would require less modifications when error will be + * added to DEV_ERR1 + */ + bool err_exists = false; + + if (!(err_val & CPU_BOOT_ERR1_ENABLED)) + return false; + + if (sts_val & CPU_BOOT_DEV_STS1_ENABLED) + dev_dbg(hdev->dev, "Device status1 %#x\n", sts_val); + + if (!err_exists && (err_val & ~CPU_BOOT_ERR1_ENABLED)) { + dev_err(hdev->dev, + "Device boot error - unknown ERR1 error 0x%08x\n", + err_val); + err_exists = true; + } + + /* return error only if it's in the predefined mask */ + if (err_exists && ((err_val & ~CPU_BOOT_ERR1_ENABLED) & + upper_32_bits(hdev->boot_error_status_mask))) + return true; + + return false; +} + +static int fw_read_errors(struct hl_device *hdev, u32 boot_err0_reg, + u32 boot_err1_reg, u32 cpu_boot_dev_status0_reg, + u32 cpu_boot_dev_status1_reg) +{ + u32 err_val, status_val; + bool err_exists = false; + + /* Some of the firmware status codes are deprecated in newer f/w + * versions. In those versions, the errors are reported + * in different registers. Therefore, we need to check those + * registers and print the exact errors. Moreover, there + * may be multiple errors, so we need to report on each error + * separately. Some of the error codes might indicate a state + * that is not an error per-se, but it is an error in production + * environment + */ + err_val = RREG32(boot_err0_reg); + status_val = RREG32(cpu_boot_dev_status0_reg); + err_exists = fw_report_boot_dev0(hdev, err_val, status_val); + + err_val = RREG32(boot_err1_reg); + status_val = RREG32(cpu_boot_dev_status1_reg); + err_exists |= fw_report_boot_dev1(hdev, err_val, status_val); + + if (err_exists) return -EIO; return 0; } int hl_fw_cpucp_info_get(struct hl_device *hdev, - u32 cpu_security_boot_status_reg, - u32 boot_err0_reg) + u32 sts_boot_dev_sts0_reg, + u32 sts_boot_dev_sts1_reg, u32 boot_err0_reg, + u32 boot_err1_reg) { struct asic_fixed_properties *prop = &hdev->asic_prop; struct cpucp_packet pkt = {}; - void *cpucp_info_cpu_addr; dma_addr_t cpucp_info_dma_addr; + void *cpucp_info_cpu_addr; + char *kernel_ver; u64 result; int rc; @@ -443,7 +635,8 @@ int hl_fw_cpucp_info_get(struct hl_device *hdev, goto out; } - rc = fw_read_errors(hdev, boot_err0_reg, cpu_security_boot_status_reg); + rc = fw_read_errors(hdev, boot_err0_reg, boot_err1_reg, + sts_boot_dev_sts0_reg, sts_boot_dev_sts1_reg); if (rc) { dev_err(hdev->dev, "Errors in device boot\n"); goto out; @@ -460,10 +653,27 @@ int hl_fw_cpucp_info_get(struct hl_device *hdev, goto out; } + kernel_ver = extract_fw_ver_from_str(prop->cpucp_info.kernel_version); + if (kernel_ver) { + dev_info(hdev->dev, "Linux version %s", kernel_ver); + kfree(kernel_ver); + } + + /* assume EQ code doesn't need to check eqe index */ + hdev->event_queue.check_eqe_index = false; + /* Read FW application security bits again */ - if (hdev->asic_prop.fw_security_status_valid) - hdev->asic_prop.fw_app_security_map = - RREG32(cpu_security_boot_status_reg); + if (hdev->asic_prop.fw_cpu_boot_dev_sts0_valid) { + hdev->asic_prop.fw_app_cpu_boot_dev_sts0 = + RREG32(sts_boot_dev_sts0_reg); + if (hdev->asic_prop.fw_app_cpu_boot_dev_sts0 & + CPU_BOOT_DEV_STS0_EQ_INDEX_EN) + hdev->event_queue.check_eqe_index = true; + } + + if (hdev->asic_prop.fw_cpu_boot_dev_sts1_valid) + hdev->asic_prop.fw_app_cpu_boot_dev_sts1 = + RREG32(sts_boot_dev_sts1_reg); out: hdev->asic_funcs->cpu_accessible_dma_pool_free(hdev, @@ -501,7 +711,8 @@ static int hl_fw_send_msi_info_msg(struct hl_device *hdev) pkt->length = cpu_to_le32(CPUCP_NUM_OF_MSI_TYPES); - hdev->asic_funcs->get_msi_info((u32 *)&pkt->data); + memset((void *) &pkt->data, 0xFF, data_size); + hdev->asic_funcs->get_msi_info(pkt->data); pkt->cpucp_pkt.ctl = cpu_to_le32(CPUCP_PACKET_MSI_INFO_SET << CPUCP_PKT_CTL_OPCODE_SHIFT); @@ -526,13 +737,15 @@ static int hl_fw_send_msi_info_msg(struct hl_device *hdev) } int hl_fw_cpucp_handshake(struct hl_device *hdev, - u32 cpu_security_boot_status_reg, - u32 boot_err0_reg) + u32 sts_boot_dev_sts0_reg, + u32 sts_boot_dev_sts1_reg, u32 boot_err0_reg, + u32 boot_err1_reg) { int rc; - rc = hl_fw_cpucp_info_get(hdev, cpu_security_boot_status_reg, - boot_err0_reg); + rc = hl_fw_cpucp_info_get(hdev, sts_boot_dev_sts0_reg, + sts_boot_dev_sts1_reg, boot_err0_reg, + boot_err1_reg); if (rc) return rc; @@ -667,8 +880,8 @@ int get_used_pll_index(struct hl_device *hdev, u32 input_pll_index, bool dynamic_pll; int fw_pll_idx; - dynamic_pll = prop->fw_security_status_valid && - (prop->fw_app_security_map & CPU_BOOT_DEV_STS0_DYN_PLL_EN); + dynamic_pll = !!(prop->fw_app_cpu_boot_dev_sts0 & + CPU_BOOT_DEV_STS0_DYN_PLL_EN); if (!dynamic_pll) { /* @@ -759,6 +972,47 @@ int hl_fw_cpucp_power_get(struct hl_device *hdev, u64 *power) return rc; } +void hl_fw_ask_hard_reset_without_linux(struct hl_device *hdev) +{ + struct static_fw_load_mgr *static_loader = + &hdev->fw_loader.static_loader; + int rc; + + if (hdev->asic_prop.dynamic_fw_load) { + rc = hl_fw_dynamic_send_protocol_cmd(hdev, &hdev->fw_loader, + COMMS_RST_DEV, 0, false, + hdev->fw_loader.cpu_timeout); + if (rc) + dev_warn(hdev->dev, "Failed sending COMMS_RST_DEV\n"); + } else { + WREG32(static_loader->kmd_msg_to_cpu_reg, KMD_MSG_RST_DEV); + } +} + +void hl_fw_ask_halt_machine_without_linux(struct hl_device *hdev) +{ + struct static_fw_load_mgr *static_loader = + &hdev->fw_loader.static_loader; + int rc; + + if (hdev->device_cpu_is_halted) + return; + + /* Stop device CPU to make sure nothing bad happens */ + if (hdev->asic_prop.dynamic_fw_load) { + rc = hl_fw_dynamic_send_protocol_cmd(hdev, &hdev->fw_loader, + COMMS_GOTO_WFE, 0, true, + hdev->fw_loader.cpu_timeout); + if (rc) + dev_warn(hdev->dev, "Failed sending COMMS_GOTO_WFE\n"); + } else { + WREG32(static_loader->kmd_msg_to_cpu_reg, KMD_MSG_GOTO_WFE); + msleep(static_loader->cpu_reset_wait_msec); + } + + hdev->device_cpu_is_halted = true; +} + static void detect_cpu_boot_status(struct hl_device *hdev, u32 status) { /* Some of the status codes below are deprecated in newer f/w @@ -767,63 +1021,59 @@ static void detect_cpu_boot_status(struct hl_device *hdev, u32 status) switch (status) { case CPU_BOOT_STATUS_NA: dev_err(hdev->dev, - "Device boot error - BTL did NOT run\n"); + "Device boot progress - BTL did NOT run\n"); break; case CPU_BOOT_STATUS_IN_WFE: dev_err(hdev->dev, - "Device boot error - Stuck inside WFE loop\n"); + "Device boot progress - Stuck inside WFE loop\n"); break; case CPU_BOOT_STATUS_IN_BTL: dev_err(hdev->dev, - "Device boot error - Stuck in BTL\n"); + "Device boot progress - Stuck in BTL\n"); break; case CPU_BOOT_STATUS_IN_PREBOOT: dev_err(hdev->dev, - "Device boot error - Stuck in Preboot\n"); + "Device boot progress - Stuck in Preboot\n"); break; case CPU_BOOT_STATUS_IN_SPL: dev_err(hdev->dev, - "Device boot error - Stuck in SPL\n"); + "Device boot progress - Stuck in SPL\n"); break; case CPU_BOOT_STATUS_IN_UBOOT: dev_err(hdev->dev, - "Device boot error - Stuck in u-boot\n"); + "Device boot progress - Stuck in u-boot\n"); break; case CPU_BOOT_STATUS_DRAM_INIT_FAIL: dev_err(hdev->dev, - "Device boot error - DRAM initialization failed\n"); + "Device boot progress - DRAM initialization failed\n"); break; case CPU_BOOT_STATUS_UBOOT_NOT_READY: dev_err(hdev->dev, - "Device boot error - u-boot stopped by user\n"); + "Device boot progress - Cannot boot\n"); break; case CPU_BOOT_STATUS_TS_INIT_FAIL: dev_err(hdev->dev, - "Device boot error - Thermal Sensor initialization failed\n"); + "Device boot progress - Thermal Sensor initialization failed\n"); break; default: dev_err(hdev->dev, - "Device boot error - Invalid status code %d\n", + "Device boot progress - Invalid status code %d\n", status); break; } } -int hl_fw_read_preboot_status(struct hl_device *hdev, u32 cpu_boot_status_reg, - u32 cpu_security_boot_status_reg, u32 boot_err0_reg, - u32 timeout) +static int hl_fw_read_preboot_caps(struct hl_device *hdev, + u32 cpu_boot_status_reg, + u32 sts_boot_dev_sts0_reg, + u32 sts_boot_dev_sts1_reg, + u32 boot_err0_reg, u32 boot_err1_reg, + u32 timeout) { struct asic_fixed_properties *prop = &hdev->asic_prop; - u32 status, security_status; + u32 status, reg_val; int rc; - /* pldm was added for cases in which we use preboot on pldm and want - * to load boot fit, but we can't wait for preboot because it runs - * very slowly - */ - if (!(hdev->fw_components & FW_TYPE_PREBOOT_CPU) || hdev->pldm) - return 0; - /* Need to check two possible scenarios: * * CPU_BOOT_STATUS_WAITING_FOR_BOOT_FIT - for newer firmwares where @@ -842,29 +1092,145 @@ int hl_fw_read_preboot_status(struct hl_device *hdev, u32 cpu_boot_status_reg, (status == CPU_BOOT_STATUS_READY_TO_BOOT) || (status == CPU_BOOT_STATUS_SRAM_AVAIL) || (status == CPU_BOOT_STATUS_WAITING_FOR_BOOT_FIT), - 10000, + FW_CPU_STATUS_POLL_INTERVAL_USEC, timeout); if (rc) { - dev_err(hdev->dev, "Failed to read preboot version\n"); + dev_err(hdev->dev, "CPU boot ready status timeout\n"); detect_cpu_boot_status(hdev, status); /* If we read all FF, then something is totally wrong, no point * of reading specific errors */ if (status != -1) - fw_read_errors(hdev, boot_err0_reg, - cpu_security_boot_status_reg); + fw_read_errors(hdev, boot_err0_reg, boot_err1_reg, + sts_boot_dev_sts0_reg, + sts_boot_dev_sts1_reg); return -EIO; } - rc = hdev->asic_funcs->read_device_fw_version(hdev, FW_COMP_PREBOOT); - if (rc) - return rc; + /* + * the registers DEV_STS* contain FW capabilities/features. + * We can rely on this registers only if bit CPU_BOOT_DEV_STS*_ENABLED + * is set. + * In the first read of this register we store the value of this + * register ONLY if the register is enabled (which will be propagated + * to next stages) and also mark the register as valid. + * In case it is not enabled the stored value will be left 0- all + * caps/features are off + */ + reg_val = RREG32(sts_boot_dev_sts0_reg); + if (reg_val & CPU_BOOT_DEV_STS0_ENABLED) { + prop->fw_cpu_boot_dev_sts0_valid = true; + prop->fw_preboot_cpu_boot_dev_sts0 = reg_val; + } + + reg_val = RREG32(sts_boot_dev_sts1_reg); + if (reg_val & CPU_BOOT_DEV_STS1_ENABLED) { + prop->fw_cpu_boot_dev_sts1_valid = true; + prop->fw_preboot_cpu_boot_dev_sts1 = reg_val; + } - security_status = RREG32(cpu_security_boot_status_reg); + prop->dynamic_fw_load = !!(prop->fw_preboot_cpu_boot_dev_sts0 & + CPU_BOOT_DEV_STS0_FW_LD_COM_EN); + + /* initialize FW loader once we know what load protocol is used */ + hdev->asic_funcs->init_firmware_loader(hdev); + + dev_dbg(hdev->dev, "Attempting %s FW load\n", + prop->dynamic_fw_load ? "dynamic" : "legacy"); + return 0; +} - /* We read security status multiple times during boot: +static int hl_fw_static_read_device_fw_version(struct hl_device *hdev, + enum hl_fw_component fwc) +{ + struct asic_fixed_properties *prop = &hdev->asic_prop; + struct fw_load_mgr *fw_loader = &hdev->fw_loader; + struct static_fw_load_mgr *static_loader; + char *dest, *boot_ver, *preboot_ver; + u32 ver_off, limit; + const char *name; + char btl_ver[32]; + + static_loader = &hdev->fw_loader.static_loader; + + switch (fwc) { + case FW_COMP_BOOT_FIT: + ver_off = RREG32(static_loader->boot_fit_version_offset_reg); + dest = prop->uboot_ver; + name = "Boot-fit"; + limit = static_loader->boot_fit_version_max_off; + break; + case FW_COMP_PREBOOT: + ver_off = RREG32(static_loader->preboot_version_offset_reg); + dest = prop->preboot_ver; + name = "Preboot"; + limit = static_loader->preboot_version_max_off; + break; + default: + dev_warn(hdev->dev, "Undefined FW component: %d\n", fwc); + return -EIO; + } + + ver_off &= static_loader->sram_offset_mask; + + if (ver_off < limit) { + memcpy_fromio(dest, + hdev->pcie_bar[fw_loader->sram_bar_id] + ver_off, + VERSION_MAX_LEN); + } else { + dev_err(hdev->dev, "%s version offset (0x%x) is above SRAM\n", + name, ver_off); + strscpy(dest, "unavailable", VERSION_MAX_LEN); + return -EIO; + } + + if (fwc == FW_COMP_BOOT_FIT) { + boot_ver = extract_fw_ver_from_str(prop->uboot_ver); + if (boot_ver) { + dev_info(hdev->dev, "boot-fit version %s\n", boot_ver); + kfree(boot_ver); + } + } else if (fwc == FW_COMP_PREBOOT) { + preboot_ver = strnstr(prop->preboot_ver, "Preboot", + VERSION_MAX_LEN); + if (preboot_ver && preboot_ver != prop->preboot_ver) { + strscpy(btl_ver, prop->preboot_ver, + min((int) (preboot_ver - prop->preboot_ver), + 31)); + dev_info(hdev->dev, "%s\n", btl_ver); + } + + preboot_ver = extract_fw_ver_from_str(prop->preboot_ver); + if (preboot_ver) { + dev_info(hdev->dev, "preboot version %s\n", + preboot_ver); + kfree(preboot_ver); + } + } + + return 0; +} + +/** + * hl_fw_preboot_update_state - update internal data structures during + * handshake with preboot + * + * + * @hdev: pointer to the habanalabs device structure + * + * @return 0 on success, otherwise non-zero error code + */ +static void hl_fw_preboot_update_state(struct hl_device *hdev) +{ + struct asic_fixed_properties *prop = &hdev->asic_prop; + u32 cpu_boot_dev_sts0, cpu_boot_dev_sts1; + + cpu_boot_dev_sts0 = prop->fw_preboot_cpu_boot_dev_sts0; + cpu_boot_dev_sts1 = prop->fw_preboot_cpu_boot_dev_sts1; + + /* We read boot_dev_sts registers multiple times during boot: * 1. preboot - a. Check whether the security status bits are valid * b. Check whether fw security is enabled * c. Check whether hard reset is done by preboot @@ -874,48 +1240,1121 @@ int hl_fw_read_preboot_status(struct hl_device *hdev, u32 cpu_boot_status_reg, * b. Check whether hard reset is done by fw app * * Preboot: - * Check security status bit (CPU_BOOT_DEV_STS0_ENABLED), if it is set + * Check security status bit (CPU_BOOT_DEV_STS0_ENABLED). If set, then- * check security enabled bit (CPU_BOOT_DEV_STS0_SECURITY_EN) + * If set, then mark GIC controller to be disabled. */ - if (security_status & CPU_BOOT_DEV_STS0_ENABLED) { - prop->fw_security_status_valid = 1; + prop->hard_reset_done_by_fw = + !!(cpu_boot_dev_sts0 & CPU_BOOT_DEV_STS0_FW_HARD_RST_EN); - /* FW security should be derived from PCI ID, we keep this - * check for backward compatibility - */ - if (security_status & CPU_BOOT_DEV_STS0_SECURITY_EN) - prop->fw_security_disabled = false; + dev_dbg(hdev->dev, "Firmware preboot boot device status0 %#x\n", + cpu_boot_dev_sts0); + + dev_dbg(hdev->dev, "Firmware preboot boot device status1 %#x\n", + cpu_boot_dev_sts1); + + dev_dbg(hdev->dev, "Firmware preboot hard-reset is %s\n", + prop->hard_reset_done_by_fw ? "enabled" : "disabled"); + + dev_dbg(hdev->dev, "firmware-level security is %s\n", + prop->fw_security_enabled ? "enabled" : "disabled"); + + dev_dbg(hdev->dev, "GIC controller is %s\n", + prop->gic_interrupts_enable ? "enabled" : "disabled"); +} + +static int hl_fw_static_read_preboot_status(struct hl_device *hdev) +{ + int rc; + + rc = hl_fw_static_read_device_fw_version(hdev, FW_COMP_PREBOOT); + if (rc) + return rc; + + return 0; +} + +int hl_fw_read_preboot_status(struct hl_device *hdev, u32 cpu_boot_status_reg, + u32 sts_boot_dev_sts0_reg, + u32 sts_boot_dev_sts1_reg, u32 boot_err0_reg, + u32 boot_err1_reg, u32 timeout) +{ + int rc; + + /* pldm was added for cases in which we use preboot on pldm and want + * to load boot fit, but we can't wait for preboot because it runs + * very slowly + */ + if (!(hdev->fw_components & FW_TYPE_PREBOOT_CPU) || hdev->pldm) + return 0; + + /* + * In order to determine boot method (static VS dymanic) we need to + * read the boot caps register + */ + rc = hl_fw_read_preboot_caps(hdev, cpu_boot_status_reg, + sts_boot_dev_sts0_reg, + sts_boot_dev_sts1_reg, boot_err0_reg, + boot_err1_reg, timeout); + if (rc) + return rc; + + hl_fw_preboot_update_state(hdev); - if (security_status & CPU_BOOT_DEV_STS0_FW_HARD_RST_EN) + /* no need to read preboot status in dynamic load */ + if (hdev->asic_prop.dynamic_fw_load) + return 0; + + return hl_fw_static_read_preboot_status(hdev); +} + +/* associate string with COMM status */ +static char *hl_dynamic_fw_status_str[COMMS_STS_INVLD_LAST] = { + [COMMS_STS_NOOP] = "NOOP", + [COMMS_STS_ACK] = "ACK", + [COMMS_STS_OK] = "OK", + [COMMS_STS_ERR] = "ERR", + [COMMS_STS_VALID_ERR] = "VALID_ERR", + [COMMS_STS_TIMEOUT_ERR] = "TIMEOUT_ERR", +}; + +/** + * hl_fw_dynamic_report_error_status - report error status + * + * @hdev: pointer to the habanalabs device structure + * @status: value of FW status register + * @expected_status: the expected status + */ +static void hl_fw_dynamic_report_error_status(struct hl_device *hdev, + u32 status, + enum comms_sts expected_status) +{ + enum comms_sts comm_status = + FIELD_GET(COMMS_STATUS_STATUS_MASK, status); + + if (comm_status < COMMS_STS_INVLD_LAST) + dev_err(hdev->dev, "Device status %s, expected status: %s\n", + hl_dynamic_fw_status_str[comm_status], + hl_dynamic_fw_status_str[expected_status]); + else + dev_err(hdev->dev, "Device status unknown %d, expected status: %s\n", + comm_status, + hl_dynamic_fw_status_str[expected_status]); +} + +/** + * hl_fw_dynamic_send_cmd - send LKD to FW cmd + * + * @hdev: pointer to the habanalabs device structure + * @fw_loader: managing structure for loading device's FW + * @cmd: LKD to FW cmd code + * @size: size of next FW component to be loaded (0 if not necessary) + * + * LDK to FW exact command layout is defined at struct comms_command. + * note: the size argument is used only when the next FW component should be + * loaded, otherwise it shall be 0. the size is used by the FW in later + * protocol stages and when sending only indicating the amount of memory + * to be allocated by the FW to receive the next boot component. + */ +static void hl_fw_dynamic_send_cmd(struct hl_device *hdev, + struct fw_load_mgr *fw_loader, + enum comms_cmd cmd, unsigned int size) +{ + struct cpu_dyn_regs *dyn_regs; + u32 val; + + dyn_regs = &fw_loader->dynamic_loader.comm_desc.cpu_dyn_regs; + + val = FIELD_PREP(COMMS_COMMAND_CMD_MASK, cmd); + val |= FIELD_PREP(COMMS_COMMAND_SIZE_MASK, size); + + WREG32(le32_to_cpu(dyn_regs->kmd_msg_to_cpu), val); +} + +/** + * hl_fw_dynamic_extract_fw_response - update the FW response + * + * @hdev: pointer to the habanalabs device structure + * @fw_loader: managing structure for loading device's FW + * @response: FW response + * @status: the status read from CPU status register + * + * @return 0 on success, otherwise non-zero error code + */ +static int hl_fw_dynamic_extract_fw_response(struct hl_device *hdev, + struct fw_load_mgr *fw_loader, + struct fw_response *response, + u32 status) +{ + response->status = FIELD_GET(COMMS_STATUS_STATUS_MASK, status); + response->ram_offset = FIELD_GET(COMMS_STATUS_OFFSET_MASK, status) << + COMMS_STATUS_OFFSET_ALIGN_SHIFT; + response->ram_type = FIELD_GET(COMMS_STATUS_RAM_TYPE_MASK, status); + + if ((response->ram_type != COMMS_SRAM) && + (response->ram_type != COMMS_DRAM)) { + dev_err(hdev->dev, "FW status: invalid RAM type %u\n", + response->ram_type); + return -EIO; + } + + return 0; +} + +/** + * hl_fw_dynamic_wait_for_status - wait for status in dynamic FW load + * + * @hdev: pointer to the habanalabs device structure + * @fw_loader: managing structure for loading device's FW + * @expected_status: expected status to wait for + * @timeout: timeout for status wait + * + * @return 0 on success, otherwise non-zero error code + * + * waiting for status from FW include polling the FW status register until + * expected status is received or timeout occurs (whatever occurs first). + */ +static int hl_fw_dynamic_wait_for_status(struct hl_device *hdev, + struct fw_load_mgr *fw_loader, + enum comms_sts expected_status, + u32 timeout) +{ + struct cpu_dyn_regs *dyn_regs; + u32 status; + int rc; + + dyn_regs = &fw_loader->dynamic_loader.comm_desc.cpu_dyn_regs; + + /* Wait for expected status */ + rc = hl_poll_timeout( + hdev, + le32_to_cpu(dyn_regs->cpu_cmd_status_to_host), + status, + FIELD_GET(COMMS_STATUS_STATUS_MASK, status) == expected_status, + FW_CPU_STATUS_POLL_INTERVAL_USEC, + timeout); + + if (rc) { + hl_fw_dynamic_report_error_status(hdev, status, + expected_status); + return -EIO; + } + + /* + * skip storing FW response for NOOP to preserve the actual desired + * FW status + */ + if (expected_status == COMMS_STS_NOOP) + return 0; + + rc = hl_fw_dynamic_extract_fw_response(hdev, fw_loader, + &fw_loader->dynamic_loader.response, + status); + return rc; +} + +/** + * hl_fw_dynamic_send_clear_cmd - send clear command to FW + * + * @hdev: pointer to the habanalabs device structure + * @fw_loader: managing structure for loading device's FW + * + * @return 0 on success, otherwise non-zero error code + * + * after command cycle between LKD to FW CPU (i.e. LKD got an expected status + * from FW) we need to clear the CPU status register in order to avoid garbage + * between command cycles. + * This is done by sending clear command and polling the CPU to LKD status + * register to hold the status NOOP + */ +static int hl_fw_dynamic_send_clear_cmd(struct hl_device *hdev, + struct fw_load_mgr *fw_loader) +{ + hl_fw_dynamic_send_cmd(hdev, fw_loader, COMMS_CLR_STS, 0); + + return hl_fw_dynamic_wait_for_status(hdev, fw_loader, COMMS_STS_NOOP, + fw_loader->cpu_timeout); +} + +/** + * hl_fw_dynamic_send_protocol_cmd - send LKD to FW cmd and wait for ACK + * + * @hdev: pointer to the habanalabs device structure + * @fw_loader: managing structure for loading device's FW + * @cmd: LKD to FW cmd code + * @size: size of next FW component to be loaded (0 if not necessary) + * @wait_ok: if true also wait for OK response from FW + * @timeout: timeout for status wait + * + * @return 0 on success, otherwise non-zero error code + * + * brief: + * when sending protocol command we have the following steps: + * - send clear (clear command and verify clear status register) + * - send the actual protocol command + * - wait for ACK on the protocol command + * - send clear + * - send NOOP + * if, in addition, the specific protocol command should wait for OK then: + * - wait for OK + * - send clear + * - send NOOP + * + * NOTES: + * send clear: this is necessary in order to clear the status register to avoid + * leftovers between command + * NOOP command: necessary to avoid loop on the clear command by the FW + */ +int hl_fw_dynamic_send_protocol_cmd(struct hl_device *hdev, + struct fw_load_mgr *fw_loader, + enum comms_cmd cmd, unsigned int size, + bool wait_ok, u32 timeout) +{ + int rc; + + /* first send clear command to clean former commands */ + rc = hl_fw_dynamic_send_clear_cmd(hdev, fw_loader); + + /* send the actual command */ + hl_fw_dynamic_send_cmd(hdev, fw_loader, cmd, size); + + /* wait for ACK for the command */ + rc = hl_fw_dynamic_wait_for_status(hdev, fw_loader, COMMS_STS_ACK, + timeout); + if (rc) + return rc; + + /* clear command to prepare for NOOP command */ + rc = hl_fw_dynamic_send_clear_cmd(hdev, fw_loader); + if (rc) + return rc; + + /* send the actual NOOP command */ + hl_fw_dynamic_send_cmd(hdev, fw_loader, COMMS_NOOP, 0); + + if (!wait_ok) + return 0; + + rc = hl_fw_dynamic_wait_for_status(hdev, fw_loader, COMMS_STS_OK, + timeout); + if (rc) + return rc; + + /* clear command to prepare for NOOP command */ + rc = hl_fw_dynamic_send_clear_cmd(hdev, fw_loader); + if (rc) + return rc; + + /* send the actual NOOP command */ + hl_fw_dynamic_send_cmd(hdev, fw_loader, COMMS_NOOP, 0); + + return 0; +} + +/** + * hl_fw_compat_crc32 - CRC compatible with FW + * + * @data: pointer to the data + * @size: size of the data + * + * @return the CRC32 result + * + * NOTE: kernel's CRC32 differ's from standard CRC32 calculation. + * in order to be aligned we need to flip the bits of both the input + * initial CRC and kernel's CRC32 result. + * in addition both sides use initial CRC of 0, + */ +static u32 hl_fw_compat_crc32(u8 *data, size_t size) +{ + return ~crc32_le(~((u32)0), data, size); +} + +/** + * hl_fw_dynamic_validate_memory_bound - validate memory bounds for memory + * transfer (image or descriptor) between + * host and FW + * + * @hdev: pointer to the habanalabs device structure + * @addr: device address of memory transfer + * @size: memory transter size + * @region: PCI memory region + * + * @return 0 on success, otherwise non-zero error code + */ +static int hl_fw_dynamic_validate_memory_bound(struct hl_device *hdev, + u64 addr, size_t size, + struct pci_mem_region *region) +{ + u64 end_addr; + + /* now make sure that the memory transfer is within region's bounds */ + end_addr = addr + size; + if (end_addr >= region->region_base + region->region_size) { + dev_err(hdev->dev, + "dynamic FW load: memory transfer end address out of memory region bounds. addr: %llx\n", + end_addr); + return -EIO; + } + + /* + * now make sure memory transfer is within predefined BAR bounds. + * this is to make sure we do not need to set the bar (e.g. for DRAM + * memory transfers) + */ + if (end_addr >= region->region_base - region->offset_in_bar + + region->bar_size) { + dev_err(hdev->dev, + "FW image beyond PCI BAR bounds\n"); + return -EIO; + } + + return 0; +} + +/** + * hl_fw_dynamic_validate_descriptor - validate FW descriptor + * + * @hdev: pointer to the habanalabs device structure + * @fw_loader: managing structure for loading device's FW + * @fw_desc: the descriptor form FW + * + * @return 0 on success, otherwise non-zero error code + */ +static int hl_fw_dynamic_validate_descriptor(struct hl_device *hdev, + struct fw_load_mgr *fw_loader, + struct lkd_fw_comms_desc *fw_desc) +{ + struct pci_mem_region *region; + enum pci_region region_id; + size_t data_size; + u32 data_crc32; + u8 *data_ptr; + u64 addr; + int rc; + + if (le32_to_cpu(fw_desc->header.magic) != HL_COMMS_DESC_MAGIC) { + dev_err(hdev->dev, "Invalid magic for dynamic FW descriptor (%x)\n", + fw_desc->header.magic); + return -EIO; + } + + if (fw_desc->header.version != HL_COMMS_DESC_VER) { + dev_err(hdev->dev, "Invalid version for dynamic FW descriptor (%x)\n", + fw_desc->header.version); + return -EIO; + } + + /* + * calc CRC32 of data without header. + * note that no alignment/stride address issues here as all structures + * are 64 bit padded + */ + data_size = sizeof(struct lkd_fw_comms_desc) - + sizeof(struct comms_desc_header); + data_ptr = (u8 *)fw_desc + sizeof(struct comms_desc_header); + + if (le16_to_cpu(fw_desc->header.size) != data_size) { + dev_err(hdev->dev, + "Invalid descriptor size 0x%x, expected size 0x%zx\n", + le16_to_cpu(fw_desc->header.size), data_size); + return -EIO; + } + + data_crc32 = hl_fw_compat_crc32(data_ptr, data_size); + + if (data_crc32 != le32_to_cpu(fw_desc->header.crc32)) { + dev_err(hdev->dev, + "CRC32 mismatch for dynamic FW descriptor (%x:%x)\n", + data_crc32, fw_desc->header.crc32); + return -EIO; + } + + /* find memory region to which to copy the image */ + addr = le64_to_cpu(fw_desc->img_addr); + region_id = hl_get_pci_memory_region(hdev, addr); + if ((region_id != PCI_REGION_SRAM) && + ((region_id != PCI_REGION_DRAM))) { + dev_err(hdev->dev, + "Invalid region to copy FW image address=%llx\n", addr); + return -EIO; + } + + region = &hdev->pci_mem_region[region_id]; + + /* store the region for the copy stage */ + fw_loader->dynamic_loader.image_region = region; + + /* + * here we know that the start address is valid, now make sure that the + * image is within region's bounds + */ + rc = hl_fw_dynamic_validate_memory_bound(hdev, addr, + fw_loader->dynamic_loader.fw_image_size, + region); + if (rc) { + dev_err(hdev->dev, + "invalid mem transfer request for FW image\n"); + return rc; + } + + return 0; +} + +static int hl_fw_dynamic_validate_response(struct hl_device *hdev, + struct fw_response *response, + struct pci_mem_region *region) +{ + u64 device_addr; + int rc; + + device_addr = region->region_base + response->ram_offset; + + /* + * validate that the descriptor is within region's bounds + * Note that as the start address was supplied according to the RAM + * type- testing only the end address is enough + */ + rc = hl_fw_dynamic_validate_memory_bound(hdev, device_addr, + sizeof(struct lkd_fw_comms_desc), + region); + return rc; +} + +/** + * hl_fw_dynamic_read_and_validate_descriptor - read and validate FW descriptor + * + * @hdev: pointer to the habanalabs device structure + * @fw_loader: managing structure for loading device's FW + * + * @return 0 on success, otherwise non-zero error code + */ +static int hl_fw_dynamic_read_and_validate_descriptor(struct hl_device *hdev, + struct fw_load_mgr *fw_loader) +{ + struct lkd_fw_comms_desc *fw_desc; + struct pci_mem_region *region; + struct fw_response *response; + enum pci_region region_id; + void __iomem *src; + int rc; + + fw_desc = &fw_loader->dynamic_loader.comm_desc; + response = &fw_loader->dynamic_loader.response; + + region_id = (response->ram_type == COMMS_SRAM) ? + PCI_REGION_SRAM : PCI_REGION_DRAM; + + region = &hdev->pci_mem_region[region_id]; + + rc = hl_fw_dynamic_validate_response(hdev, response, region); + if (rc) { + dev_err(hdev->dev, + "invalid mem transfer request for FW descriptor\n"); + return rc; + } + + /* extract address copy the descriptor from */ + src = hdev->pcie_bar[region->bar_id] + region->offset_in_bar + + response->ram_offset; + memcpy_fromio(fw_desc, src, sizeof(struct lkd_fw_comms_desc)); + + return hl_fw_dynamic_validate_descriptor(hdev, fw_loader, fw_desc); +} + +/** + * hl_fw_dynamic_request_descriptor - handshake with CPU to get FW descriptor + * + * @hdev: pointer to the habanalabs device structure + * @fw_loader: managing structure for loading device's FW + * @next_image_size: size to allocate for next FW component + * + * @return 0 on success, otherwise non-zero error code + */ +static int hl_fw_dynamic_request_descriptor(struct hl_device *hdev, + struct fw_load_mgr *fw_loader, + size_t next_image_size) +{ + int rc; + + rc = hl_fw_dynamic_send_protocol_cmd(hdev, fw_loader, COMMS_PREP_DESC, + next_image_size, true, + fw_loader->cpu_timeout); + if (rc) + return rc; + + return hl_fw_dynamic_read_and_validate_descriptor(hdev, fw_loader); +} + +/** + * hl_fw_dynamic_read_device_fw_version - read FW version to exposed properties + * + * @hdev: pointer to the habanalabs device structure + * @fwc: the firmware component + * @fw_version: fw component's version string + */ +static void hl_fw_dynamic_read_device_fw_version(struct hl_device *hdev, + enum hl_fw_component fwc, + const char *fw_version) +{ + struct asic_fixed_properties *prop = &hdev->asic_prop; + char *preboot_ver, *boot_ver; + char btl_ver[32]; + + switch (fwc) { + case FW_COMP_BOOT_FIT: + strscpy(prop->uboot_ver, fw_version, VERSION_MAX_LEN); + boot_ver = extract_fw_ver_from_str(prop->uboot_ver); + if (boot_ver) { + dev_info(hdev->dev, "boot-fit version %s\n", boot_ver); + kfree(boot_ver); + } + + break; + case FW_COMP_PREBOOT: + strscpy(prop->preboot_ver, fw_version, VERSION_MAX_LEN); + preboot_ver = strnstr(prop->preboot_ver, "Preboot", + VERSION_MAX_LEN); + if (preboot_ver && preboot_ver != prop->preboot_ver) { + strscpy(btl_ver, prop->preboot_ver, + min((int) (preboot_ver - prop->preboot_ver), + 31)); + dev_info(hdev->dev, "%s\n", btl_ver); + } + + preboot_ver = extract_fw_ver_from_str(prop->preboot_ver); + if (preboot_ver) { + dev_info(hdev->dev, "preboot version %s\n", + preboot_ver); + kfree(preboot_ver); + } + + break; + default: + dev_warn(hdev->dev, "Undefined FW component: %d\n", fwc); + return; + } +} + +/** + * hl_fw_dynamic_copy_image - copy image to memory allocated by the FW + * + * @hdev: pointer to the habanalabs device structure + * @fw: fw descriptor + * @fw_loader: managing structure for loading device's FW + */ +static int hl_fw_dynamic_copy_image(struct hl_device *hdev, + const struct firmware *fw, + struct fw_load_mgr *fw_loader) +{ + struct lkd_fw_comms_desc *fw_desc; + struct pci_mem_region *region; + void __iomem *dest; + u64 addr; + int rc; + + fw_desc = &fw_loader->dynamic_loader.comm_desc; + addr = le64_to_cpu(fw_desc->img_addr); + + /* find memory region to which to copy the image */ + region = fw_loader->dynamic_loader.image_region; + + dest = hdev->pcie_bar[region->bar_id] + region->offset_in_bar + + (addr - region->region_base); + + rc = hl_fw_copy_fw_to_device(hdev, fw, dest, + fw_loader->boot_fit_img.src_off, + fw_loader->boot_fit_img.copy_size); + + return rc; +} + +/** + * hl_fw_dynamic_copy_msg - copy msg to memory allocated by the FW + * + * @hdev: pointer to the habanalabs device structure + * @msg: message + * @fw_loader: managing structure for loading device's FW + */ +static int hl_fw_dynamic_copy_msg(struct hl_device *hdev, + struct lkd_msg_comms *msg, struct fw_load_mgr *fw_loader) +{ + struct lkd_fw_comms_desc *fw_desc; + struct pci_mem_region *region; + void __iomem *dest; + u64 addr; + int rc; + + fw_desc = &fw_loader->dynamic_loader.comm_desc; + addr = le64_to_cpu(fw_desc->img_addr); + + /* find memory region to which to copy the image */ + region = fw_loader->dynamic_loader.image_region; + + dest = hdev->pcie_bar[region->bar_id] + region->offset_in_bar + + (addr - region->region_base); + + rc = hl_fw_copy_msg_to_device(hdev, msg, dest, 0, 0); + + return rc; +} + +/** + * hl_fw_boot_fit_update_state - update internal data structures after boot-fit + * is loaded + * + * @hdev: pointer to the habanalabs device structure + * @cpu_boot_dev_sts0_reg: register holding CPU boot dev status 0 + * @cpu_boot_dev_sts1_reg: register holding CPU boot dev status 1 + * + * @return 0 on success, otherwise non-zero error code + */ +static void hl_fw_boot_fit_update_state(struct hl_device *hdev, + u32 cpu_boot_dev_sts0_reg, + u32 cpu_boot_dev_sts1_reg) +{ + struct asic_fixed_properties *prop = &hdev->asic_prop; + + /* Clear reset status since we need to read it again from boot CPU */ + prop->hard_reset_done_by_fw = false; + + /* Read boot_cpu status bits */ + if (prop->fw_preboot_cpu_boot_dev_sts0 & CPU_BOOT_DEV_STS0_ENABLED) { + prop->fw_bootfit_cpu_boot_dev_sts0 = + RREG32(cpu_boot_dev_sts0_reg); + + if (prop->fw_bootfit_cpu_boot_dev_sts0 & + CPU_BOOT_DEV_STS0_FW_HARD_RST_EN) prop->hard_reset_done_by_fw = true; - } else { - prop->fw_security_status_valid = 0; + + dev_dbg(hdev->dev, "Firmware boot CPU status0 %#x\n", + prop->fw_bootfit_cpu_boot_dev_sts0); } - dev_dbg(hdev->dev, "Firmware preboot security status %#x\n", - security_status); + if (prop->fw_cpu_boot_dev_sts1_valid) { + prop->fw_bootfit_cpu_boot_dev_sts1 = + RREG32(cpu_boot_dev_sts1_reg); - dev_dbg(hdev->dev, "Firmware preboot hard-reset is %s\n", + dev_dbg(hdev->dev, "Firmware boot CPU status1 %#x\n", + prop->fw_bootfit_cpu_boot_dev_sts1); + } + + dev_dbg(hdev->dev, "Firmware boot CPU hard-reset is %s\n", prop->hard_reset_done_by_fw ? "enabled" : "disabled"); +} + +static void hl_fw_dynamic_update_linux_interrupt_if(struct hl_device *hdev) +{ + struct cpu_dyn_regs *dyn_regs = + &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs; + + /* Check whether all 3 interrupt interfaces are set, if not use a + * single interface + */ + if (!hdev->asic_prop.gic_interrupts_enable && + !(hdev->asic_prop.fw_app_cpu_boot_dev_sts0 & + CPU_BOOT_DEV_STS0_MULTI_IRQ_POLL_EN)) { + dyn_regs->gic_host_halt_irq = dyn_regs->gic_host_irq_ctrl; + dyn_regs->gic_host_ints_irq = dyn_regs->gic_host_irq_ctrl; + + dev_warn(hdev->dev, + "Using a single interrupt interface towards cpucp"); + } +} +/** + * hl_fw_dynamic_load_image - load FW image using dynamic protocol + * + * @hdev: pointer to the habanalabs device structure + * @fw_loader: managing structure for loading device's FW + * @load_fwc: the FW component to be loaded + * @img_ld_timeout: image load timeout + * + * @return 0 on success, otherwise non-zero error code + */ +static int hl_fw_dynamic_load_image(struct hl_device *hdev, + struct fw_load_mgr *fw_loader, + enum hl_fw_component load_fwc, + u32 img_ld_timeout) +{ + enum hl_fw_component cur_fwc; + const struct firmware *fw; + char *fw_name; + int rc = 0; + + /* + * when loading image we have one of 2 scenarios: + * 1. current FW component is preboot and we want to load boot-fit + * 2. current FW component is boot-fit and we want to load linux + */ + if (load_fwc == FW_COMP_BOOT_FIT) { + cur_fwc = FW_COMP_PREBOOT; + fw_name = fw_loader->boot_fit_img.image_name; + } else { + cur_fwc = FW_COMP_BOOT_FIT; + fw_name = fw_loader->linux_img.image_name; + } + + /* request FW in order to communicate to FW the size to be allocated */ + rc = hl_request_fw(hdev, &fw, fw_name); + if (rc) + return rc; + + /* store the image size for future validation */ + fw_loader->dynamic_loader.fw_image_size = fw->size; + + rc = hl_fw_dynamic_request_descriptor(hdev, fw_loader, fw->size); + if (rc) + goto release_fw; + + /* read preboot version */ + hl_fw_dynamic_read_device_fw_version(hdev, cur_fwc, + fw_loader->dynamic_loader.comm_desc.cur_fw_ver); + + + /* update state according to boot stage */ + if (cur_fwc == FW_COMP_BOOT_FIT) { + struct cpu_dyn_regs *dyn_regs; + + dyn_regs = &fw_loader->dynamic_loader.comm_desc.cpu_dyn_regs; + hl_fw_boot_fit_update_state(hdev, + le32_to_cpu(dyn_regs->cpu_boot_dev_sts0), + le32_to_cpu(dyn_regs->cpu_boot_dev_sts1)); + } + + /* copy boot fit to space allocated by FW */ + rc = hl_fw_dynamic_copy_image(hdev, fw, fw_loader); + if (rc) + goto release_fw; - dev_info(hdev->dev, "firmware-level security is %s\n", - prop->fw_security_disabled ? "disabled" : "enabled"); + rc = hl_fw_dynamic_send_protocol_cmd(hdev, fw_loader, COMMS_DATA_RDY, + 0, true, + fw_loader->cpu_timeout); + if (rc) + goto release_fw; + + rc = hl_fw_dynamic_send_protocol_cmd(hdev, fw_loader, COMMS_EXEC, + 0, false, + img_ld_timeout); +release_fw: + hl_release_firmware(fw); + return rc; +} + +static int hl_fw_dynamic_wait_for_boot_fit_active(struct hl_device *hdev, + struct fw_load_mgr *fw_loader) +{ + struct dynamic_fw_load_mgr *dyn_loader; + u32 status; + int rc; + + dyn_loader = &fw_loader->dynamic_loader; + + /* Make sure CPU boot-loader is running */ + rc = hl_poll_timeout( + hdev, + le32_to_cpu(dyn_loader->comm_desc.cpu_dyn_regs.cpu_boot_status), + status, + (status == CPU_BOOT_STATUS_NIC_FW_RDY) || + (status == CPU_BOOT_STATUS_READY_TO_BOOT), + FW_CPU_STATUS_POLL_INTERVAL_USEC, + dyn_loader->wait_for_bl_timeout); + if (rc) { + dev_err(hdev->dev, "failed to wait for boot\n"); + return rc; + } + + dev_dbg(hdev->dev, "uboot status = %d\n", status); return 0; } -int hl_fw_init_cpu(struct hl_device *hdev, u32 cpu_boot_status_reg, - u32 msg_to_cpu_reg, u32 cpu_msg_status_reg, - u32 cpu_security_boot_status_reg, u32 boot_err0_reg, - bool skip_bmc, u32 cpu_timeout, u32 boot_fit_timeout) +static int hl_fw_dynamic_wait_for_linux_active(struct hl_device *hdev, + struct fw_load_mgr *fw_loader) { - struct asic_fixed_properties *prop = &hdev->asic_prop; + struct dynamic_fw_load_mgr *dyn_loader; u32 status; int rc; + dyn_loader = &fw_loader->dynamic_loader; + + /* Make sure CPU boot-loader is running */ + + rc = hl_poll_timeout( + hdev, + le32_to_cpu(dyn_loader->comm_desc.cpu_dyn_regs.cpu_boot_status), + status, + (status == CPU_BOOT_STATUS_SRAM_AVAIL), + FW_CPU_STATUS_POLL_INTERVAL_USEC, + fw_loader->cpu_timeout); + if (rc) { + dev_err(hdev->dev, "failed to wait for Linux\n"); + return rc; + } + + dev_dbg(hdev->dev, "Boot status = %d\n", status); + return 0; +} + +/** + * hl_fw_linux_update_state - update internal data structures after Linux + * is loaded. + * Note: Linux initialization is comprised mainly + * of two stages - loading kernel (SRAM_AVAIL) + * & loading ARMCP. + * Therefore reading boot device status in any of + * these stages might result in different values. + * + * @hdev: pointer to the habanalabs device structure + * @cpu_boot_dev_sts0_reg: register holding CPU boot dev status 0 + * @cpu_boot_dev_sts1_reg: register holding CPU boot dev status 1 + * + * @return 0 on success, otherwise non-zero error code + */ +static void hl_fw_linux_update_state(struct hl_device *hdev, + u32 cpu_boot_dev_sts0_reg, + u32 cpu_boot_dev_sts1_reg) +{ + struct asic_fixed_properties *prop = &hdev->asic_prop; + + hdev->fw_loader.linux_loaded = true; + + /* Clear reset status since we need to read again from app */ + prop->hard_reset_done_by_fw = false; + + /* Read FW application security bits */ + if (prop->fw_cpu_boot_dev_sts0_valid) { + prop->fw_app_cpu_boot_dev_sts0 = + RREG32(cpu_boot_dev_sts0_reg); + + if (prop->fw_app_cpu_boot_dev_sts0 & + CPU_BOOT_DEV_STS0_FW_HARD_RST_EN) + prop->hard_reset_done_by_fw = true; + + if (prop->fw_app_cpu_boot_dev_sts0 & + CPU_BOOT_DEV_STS0_GIC_PRIVILEGED_EN) + prop->gic_interrupts_enable = false; + + dev_dbg(hdev->dev, + "Firmware application CPU status0 %#x\n", + prop->fw_app_cpu_boot_dev_sts0); + + dev_dbg(hdev->dev, "GIC controller is %s\n", + prop->gic_interrupts_enable ? + "enabled" : "disabled"); + } + + if (prop->fw_cpu_boot_dev_sts1_valid) { + prop->fw_app_cpu_boot_dev_sts1 = + RREG32(cpu_boot_dev_sts1_reg); + + dev_dbg(hdev->dev, + "Firmware application CPU status1 %#x\n", + prop->fw_app_cpu_boot_dev_sts1); + } + + dev_dbg(hdev->dev, "Firmware application CPU hard-reset is %s\n", + prop->hard_reset_done_by_fw ? "enabled" : "disabled"); + + dev_info(hdev->dev, "Successfully loaded firmware to device\n"); +} + +/** + * hl_fw_dynamic_report_reset_cause - send a COMMS message with the cause + * of the newly triggered hard reset + * + * @hdev: pointer to the habanalabs device structure + * @fw_loader: managing structure for loading device's FW + * @reset_cause: enumerated cause for the recent hard reset + * + * @return 0 on success, otherwise non-zero error code + */ +static int hl_fw_dynamic_report_reset_cause(struct hl_device *hdev, + struct fw_load_mgr *fw_loader, + enum comms_reset_cause reset_cause) +{ + struct lkd_msg_comms msg; + int rc; + + memset(&msg, 0, sizeof(msg)); + + /* create message to be sent */ + msg.header.type = HL_COMMS_RESET_CAUSE_TYPE; + msg.header.size = cpu_to_le16(sizeof(struct comms_msg_header)); + msg.header.magic = cpu_to_le32(HL_COMMS_MSG_MAGIC); + + msg.reset_cause = reset_cause; + + rc = hl_fw_dynamic_request_descriptor(hdev, fw_loader, + sizeof(struct lkd_msg_comms)); + if (rc) + return rc; + + /* copy message to space allocated by FW */ + rc = hl_fw_dynamic_copy_msg(hdev, &msg, fw_loader); + if (rc) + return rc; + + rc = hl_fw_dynamic_send_protocol_cmd(hdev, fw_loader, COMMS_DATA_RDY, + 0, true, + fw_loader->cpu_timeout); + if (rc) + return rc; + + rc = hl_fw_dynamic_send_protocol_cmd(hdev, fw_loader, COMMS_EXEC, + 0, true, + fw_loader->cpu_timeout); + if (rc) + return rc; + + return 0; +} + +/** + * hl_fw_dynamic_init_cpu - initialize the device CPU using dynamic protocol + * + * @hdev: pointer to the habanalabs device structure + * @fw_loader: managing structure for loading device's FW + * + * @return 0 on success, otherwise non-zero error code + * + * brief: the dynamic protocol is master (LKD) slave (FW CPU) protocol. + * the communication is done using registers: + * - LKD command register + * - FW status register + * the protocol is race free. this goal is achieved by splitting the requests + * and response to known synchronization points between the LKD and the FW. + * each response to LKD request is known and bound to a predefined timeout. + * in case of timeout expiration without the desired status from FW- the + * protocol (and hence the boot) will fail. + */ +static int hl_fw_dynamic_init_cpu(struct hl_device *hdev, + struct fw_load_mgr *fw_loader) +{ + struct cpu_dyn_regs *dyn_regs; + int rc; + + dev_info(hdev->dev, + "Loading firmware to device, may take some time...\n"); + + dyn_regs = &fw_loader->dynamic_loader.comm_desc.cpu_dyn_regs; + + rc = hl_fw_dynamic_send_protocol_cmd(hdev, fw_loader, COMMS_RST_STATE, + 0, true, + fw_loader->cpu_timeout); + if (rc) + goto protocol_err; + + if (hdev->curr_reset_cause) { + rc = hl_fw_dynamic_report_reset_cause(hdev, fw_loader, + hdev->curr_reset_cause); + if (rc) + goto protocol_err; + + /* Clear current reset cause */ + hdev->curr_reset_cause = HL_RESET_CAUSE_UNKNOWN; + } + + if (!(hdev->fw_components & FW_TYPE_BOOT_CPU)) { + rc = hl_fw_dynamic_request_descriptor(hdev, fw_loader, 0); + if (rc) + goto protocol_err; + + /* read preboot version */ + hl_fw_dynamic_read_device_fw_version(hdev, FW_COMP_PREBOOT, + fw_loader->dynamic_loader.comm_desc.cur_fw_ver); + return 0; + } + + /* load boot fit to FW */ + rc = hl_fw_dynamic_load_image(hdev, fw_loader, FW_COMP_BOOT_FIT, + fw_loader->boot_fit_timeout); + if (rc) { + dev_err(hdev->dev, "failed to load boot fit\n"); + goto protocol_err; + } + + rc = hl_fw_dynamic_wait_for_boot_fit_active(hdev, fw_loader); + if (rc) + goto protocol_err; + + /* Enable DRAM scrambling before Linux boot and after successful + * UBoot + */ + hdev->asic_funcs->init_cpu_scrambler_dram(hdev); + + if (!(hdev->fw_components & FW_TYPE_LINUX)) { + dev_info(hdev->dev, "Skip loading Linux F/W\n"); + return 0; + } + + if (fw_loader->skip_bmc) { + rc = hl_fw_dynamic_send_protocol_cmd(hdev, fw_loader, + COMMS_SKIP_BMC, 0, + true, + fw_loader->cpu_timeout); + if (rc) { + dev_err(hdev->dev, "failed to load boot fit\n"); + goto protocol_err; + } + } + + /* load Linux image to FW */ + rc = hl_fw_dynamic_load_image(hdev, fw_loader, FW_COMP_LINUX, + fw_loader->cpu_timeout); + if (rc) { + dev_err(hdev->dev, "failed to load Linux\n"); + goto protocol_err; + } + + rc = hl_fw_dynamic_wait_for_linux_active(hdev, fw_loader); + if (rc) + goto protocol_err; + + hl_fw_linux_update_state(hdev, le32_to_cpu(dyn_regs->cpu_boot_dev_sts0), + le32_to_cpu(dyn_regs->cpu_boot_dev_sts1)); + + hl_fw_dynamic_update_linux_interrupt_if(hdev); + + return 0; + +protocol_err: + fw_read_errors(hdev, le32_to_cpu(dyn_regs->cpu_boot_err0), + le32_to_cpu(dyn_regs->cpu_boot_err1), + le32_to_cpu(dyn_regs->cpu_boot_dev_sts0), + le32_to_cpu(dyn_regs->cpu_boot_dev_sts1)); + return rc; +} + +/** + * hl_fw_static_init_cpu - initialize the device CPU using static protocol + * + * @hdev: pointer to the habanalabs device structure + * @fw_loader: managing structure for loading device's FW + * + * @return 0 on success, otherwise non-zero error code + */ +static int hl_fw_static_init_cpu(struct hl_device *hdev, + struct fw_load_mgr *fw_loader) +{ + u32 cpu_msg_status_reg, cpu_timeout, msg_to_cpu_reg, status; + u32 cpu_boot_dev_status0_reg, cpu_boot_dev_status1_reg; + struct static_fw_load_mgr *static_loader; + u32 cpu_boot_status_reg; + int rc; + if (!(hdev->fw_components & FW_TYPE_BOOT_CPU)) return 0; + /* init common loader parameters */ + cpu_timeout = fw_loader->cpu_timeout; + + /* init static loader parameters */ + static_loader = &fw_loader->static_loader; + cpu_msg_status_reg = static_loader->cpu_cmd_status_to_host_reg; + msg_to_cpu_reg = static_loader->kmd_msg_to_cpu_reg; + cpu_boot_dev_status0_reg = static_loader->cpu_boot_dev_status0_reg; + cpu_boot_dev_status1_reg = static_loader->cpu_boot_dev_status1_reg; + cpu_boot_status_reg = static_loader->cpu_boot_status_reg; + dev_info(hdev->dev, "Going to wait for device boot (up to %lds)\n", cpu_timeout / USEC_PER_SEC); @@ -925,8 +2364,8 @@ int hl_fw_init_cpu(struct hl_device *hdev, u32 cpu_boot_status_reg, cpu_boot_status_reg, status, status == CPU_BOOT_STATUS_WAITING_FOR_BOOT_FIT, - 10000, - boot_fit_timeout); + FW_CPU_STATUS_POLL_INTERVAL_USEC, + fw_loader->boot_fit_timeout); if (rc) { dev_dbg(hdev->dev, @@ -948,8 +2387,8 @@ int hl_fw_init_cpu(struct hl_device *hdev, u32 cpu_boot_status_reg, cpu_msg_status_reg, status, status == CPU_MSG_OK, - 10000, - boot_fit_timeout); + FW_CPU_STATUS_POLL_INTERVAL_USEC, + fw_loader->boot_fit_timeout); if (rc) { dev_err(hdev->dev, @@ -970,33 +2409,17 @@ int hl_fw_init_cpu(struct hl_device *hdev, u32 cpu_boot_status_reg, (status == CPU_BOOT_STATUS_NIC_FW_RDY) || (status == CPU_BOOT_STATUS_READY_TO_BOOT) || (status == CPU_BOOT_STATUS_SRAM_AVAIL), - 10000, + FW_CPU_STATUS_POLL_INTERVAL_USEC, cpu_timeout); dev_dbg(hdev->dev, "uboot status = %d\n", status); /* Read U-Boot version now in case we will later fail */ - hdev->asic_funcs->read_device_fw_version(hdev, FW_COMP_UBOOT); - - /* Clear reset status since we need to read it again from boot CPU */ - prop->hard_reset_done_by_fw = false; - - /* Read boot_cpu security bits */ - if (prop->fw_security_status_valid) { - prop->fw_boot_cpu_security_map = - RREG32(cpu_security_boot_status_reg); - - if (prop->fw_boot_cpu_security_map & - CPU_BOOT_DEV_STS0_FW_HARD_RST_EN) - prop->hard_reset_done_by_fw = true; - - dev_dbg(hdev->dev, - "Firmware boot CPU security status %#x\n", - prop->fw_boot_cpu_security_map); - } + hl_fw_static_read_device_fw_version(hdev, FW_COMP_BOOT_FIT); - dev_dbg(hdev->dev, "Firmware boot CPU hard-reset is %s\n", - prop->hard_reset_done_by_fw ? "enabled" : "disabled"); + /* update state according to boot stage */ + hl_fw_boot_fit_update_state(hdev, cpu_boot_dev_status0_reg, + cpu_boot_dev_status1_reg); if (rc) { detect_cpu_boot_status(hdev, status); @@ -1004,13 +2427,21 @@ int hl_fw_init_cpu(struct hl_device *hdev, u32 cpu_boot_status_reg, goto out; } + /* Enable DRAM scrambling before Linux boot and after successful + * UBoot + */ + hdev->asic_funcs->init_cpu_scrambler_dram(hdev); + if (!(hdev->fw_components & FW_TYPE_LINUX)) { dev_info(hdev->dev, "Skip loading Linux F/W\n"); + rc = 0; goto out; } - if (status == CPU_BOOT_STATUS_SRAM_AVAIL) + if (status == CPU_BOOT_STATUS_SRAM_AVAIL) { + rc = 0; goto out; + } dev_info(hdev->dev, "Loading firmware to device, may take some time...\n"); @@ -1019,7 +2450,7 @@ int hl_fw_init_cpu(struct hl_device *hdev, u32 cpu_boot_status_reg, if (rc) goto out; - if (skip_bmc) { + if (fw_loader->skip_bmc) { WREG32(msg_to_cpu_reg, KMD_MSG_SKIP_BMC); rc = hl_poll_timeout( @@ -1027,7 +2458,7 @@ int hl_fw_init_cpu(struct hl_device *hdev, u32 cpu_boot_status_reg, cpu_boot_status_reg, status, (status == CPU_BOOT_STATUS_BMC_WAITING_SKIPPED), - 10000, + FW_CPU_STATUS_POLL_INTERVAL_USEC, cpu_timeout); if (rc) { @@ -1047,7 +2478,7 @@ int hl_fw_init_cpu(struct hl_device *hdev, u32 cpu_boot_status_reg, cpu_boot_status_reg, status, (status == CPU_BOOT_STATUS_SRAM_AVAIL), - 10000, + FW_CPU_STATUS_POLL_INTERVAL_USEC, cpu_timeout); /* Clear message */ @@ -1066,36 +2497,43 @@ int hl_fw_init_cpu(struct hl_device *hdev, u32 cpu_boot_status_reg, goto out; } - rc = fw_read_errors(hdev, boot_err0_reg, cpu_security_boot_status_reg); + rc = fw_read_errors(hdev, fw_loader->static_loader.boot_err0_reg, + fw_loader->static_loader.boot_err1_reg, + cpu_boot_dev_status0_reg, + cpu_boot_dev_status1_reg); if (rc) return rc; - /* Clear reset status since we need to read again from app */ - prop->hard_reset_done_by_fw = false; - - /* Read FW application security bits */ - if (prop->fw_security_status_valid) { - prop->fw_app_security_map = - RREG32(cpu_security_boot_status_reg); - - if (prop->fw_app_security_map & - CPU_BOOT_DEV_STS0_FW_HARD_RST_EN) - prop->hard_reset_done_by_fw = true; - - dev_dbg(hdev->dev, - "Firmware application CPU security status %#x\n", - prop->fw_app_security_map); - } - - dev_dbg(hdev->dev, "Firmware application CPU hard-reset is %s\n", - prop->hard_reset_done_by_fw ? "enabled" : "disabled"); - - dev_info(hdev->dev, "Successfully loaded firmware to device\n"); + hl_fw_linux_update_state(hdev, cpu_boot_dev_status0_reg, + cpu_boot_dev_status1_reg); return 0; out: - fw_read_errors(hdev, boot_err0_reg, cpu_security_boot_status_reg); + fw_read_errors(hdev, fw_loader->static_loader.boot_err0_reg, + fw_loader->static_loader.boot_err1_reg, + cpu_boot_dev_status0_reg, + cpu_boot_dev_status1_reg); return rc; } + +/** + * hl_fw_init_cpu - initialize the device CPU + * + * @hdev: pointer to the habanalabs device structure + * + * @return 0 on success, otherwise non-zero error code + * + * perform necessary initializations for device's CPU. takes into account if + * init protocol is static or dynamic. + */ +int hl_fw_init_cpu(struct hl_device *hdev) +{ + struct asic_fixed_properties *prop = &hdev->asic_prop; + struct fw_load_mgr *fw_loader = &hdev->fw_loader; + + return prop->dynamic_fw_load ? + hl_fw_dynamic_init_cpu(hdev, fw_loader) : + hl_fw_static_init_cpu(hdev, fw_loader); +} diff --git a/drivers/misc/habanalabs/common/habanalabs.h b/drivers/misc/habanalabs/common/habanalabs.h index 6579f8767abd..6b3cdd7e068a 100644 --- a/drivers/misc/habanalabs/common/habanalabs.h +++ b/drivers/misc/habanalabs/common/habanalabs.h @@ -48,6 +48,7 @@ #define HL_PENDING_RESET_LONG_SEC 60 #define HL_HARD_RESET_MAX_TIMEOUT 120 +#define HL_PLDM_HARD_RESET_MAX_TIMEOUT (HL_HARD_RESET_MAX_TIMEOUT * 3) #define HL_DEVICE_TIMEOUT_USEC 1000000 /* 1 s */ @@ -115,10 +116,18 @@ enum hl_mmu_page_table_location { * * - HL_RESET_HEARTBEAT * Set if reset is due to heartbeat + * + * - HL_RESET_TDR + * Set if reset is due to TDR + * + * - HL_RESET_DEVICE_RELEASE + * Set if reset is due to device release */ #define HL_RESET_HARD (1 << 0) #define HL_RESET_FROM_RESET_THREAD (1 << 1) #define HL_RESET_HEARTBEAT (1 << 2) +#define HL_RESET_TDR (1 << 3) +#define HL_RESET_DEVICE_RELEASE (1 << 4) #define HL_MAX_SOBS_PER_MONITOR 8 @@ -178,12 +187,14 @@ enum hl_pci_match_mode { /** * enum hl_fw_component - F/W components to read version through registers. - * @FW_COMP_UBOOT: u-boot. + * @FW_COMP_BOOT_FIT: boot fit. * @FW_COMP_PREBOOT: preboot. + * @FW_COMP_LINUX: linux. */ enum hl_fw_component { - FW_COMP_UBOOT, - FW_COMP_PREBOOT + FW_COMP_BOOT_FIT, + FW_COMP_PREBOOT, + FW_COMP_LINUX, }; /** @@ -420,12 +431,24 @@ struct hl_mmu_properties { * @cb_pool_cb_size: size of each CB in the CB pool. * @max_pending_cs: maximum of concurrent pending command submissions * @max_queues: maximum amount of queues in the system - * @fw_boot_cpu_security_map: bitmap representation of boot cpu security status - * reported by FW, bit description can be found in - * CPU_BOOT_DEV_STS* - * @fw_app_security_map: bitmap representation of application security status - * reported by FW, bit description can be found in - * CPU_BOOT_DEV_STS* + * @fw_preboot_cpu_boot_dev_sts0: bitmap representation of preboot cpu + * capabilities reported by FW, bit description + * can be found in CPU_BOOT_DEV_STS0 + * @fw_preboot_cpu_boot_dev_sts1: bitmap representation of preboot cpu + * capabilities reported by FW, bit description + * can be found in CPU_BOOT_DEV_STS1 + * @fw_bootfit_cpu_boot_dev_sts0: bitmap representation of boot cpu security + * status reported by FW, bit description can be + * found in CPU_BOOT_DEV_STS0 + * @fw_bootfit_cpu_boot_dev_sts1: bitmap representation of boot cpu security + * status reported by FW, bit description can be + * found in CPU_BOOT_DEV_STS1 + * @fw_app_cpu_boot_dev_sts0: bitmap representation of application security + * status reported by FW, bit description can be + * found in CPU_BOOT_DEV_STS0 + * @fw_app_cpu_boot_dev_sts1: bitmap representation of application security + * status reported by FW, bit description can be + * found in CPU_BOOT_DEV_STS1 * @collective_first_sob: first sync object available for collective use * @collective_first_mon: first monitor available for collective use * @sync_stream_first_sob: first sync object available for sync stream use @@ -438,14 +461,19 @@ struct hl_mmu_properties { * @user_interrupt_count: number of user interrupts. * @tpc_enabled_mask: which TPCs are enabled. * @completion_queues_count: number of completion queues. - * @fw_security_disabled: true if security measures are disabled in firmware, - * false otherwise - * @fw_security_status_valid: security status bits are valid and can be fetched - * from BOOT_DEV_STS0 + * @fw_security_enabled: true if security measures are enabled in firmware, + * false otherwise + * @fw_cpu_boot_dev_sts0_valid: status bits are valid and can be fetched from + * BOOT_DEV_STS0 + * @fw_cpu_boot_dev_sts1_valid: status bits are valid and can be fetched from + * BOOT_DEV_STS1 * @dram_supports_virtual_memory: is there an MMU towards the DRAM * @hard_reset_done_by_fw: true if firmware is handling hard reset flow * @num_functional_hbms: number of functional HBMs in each DCORE. * @iatu_done_by_fw: true if iATU configuration is being done by FW. + * @dynamic_fw_load: is dynamic FW load is supported. + * @gic_interrupts_enable: true if FW is not blocking GIC controller, + * false otherwise. */ struct asic_fixed_properties { struct hw_queue_properties *hw_queues_props; @@ -491,8 +519,12 @@ struct asic_fixed_properties { u32 cb_pool_cb_size; u32 max_pending_cs; u32 max_queues; - u32 fw_boot_cpu_security_map; - u32 fw_app_security_map; + u32 fw_preboot_cpu_boot_dev_sts0; + u32 fw_preboot_cpu_boot_dev_sts1; + u32 fw_bootfit_cpu_boot_dev_sts0; + u32 fw_bootfit_cpu_boot_dev_sts1; + u32 fw_app_cpu_boot_dev_sts0; + u32 fw_app_cpu_boot_dev_sts1; u16 collective_first_sob; u16 collective_first_mon; u16 sync_stream_first_sob; @@ -504,12 +536,15 @@ struct asic_fixed_properties { u16 user_interrupt_count; u8 tpc_enabled_mask; u8 completion_queues_count; - u8 fw_security_disabled; - u8 fw_security_status_valid; + u8 fw_security_enabled; + u8 fw_cpu_boot_dev_sts0_valid; + u8 fw_cpu_boot_dev_sts1_valid; u8 dram_supports_virtual_memory; u8 hard_reset_done_by_fw; u8 num_functional_hbms; u8 iatu_done_by_fw; + u8 dynamic_fw_load; + u8 gic_interrupts_enable; }; /** @@ -750,12 +785,19 @@ struct hl_user_pending_interrupt { * @kernel_address: holds the queue's kernel virtual address * @bus_address: holds the queue's DMA address * @ci: ci inside the queue + * @prev_eqe_index: the index of the previous event queue entry. The index of + * the current entry's index must be +1 of the previous one. + * @check_eqe_index: do we need to check the index of the current entry vs. the + * previous one. This is for backward compatibility with older + * firmwares */ struct hl_eq { struct hl_device *hdev; void *kernel_address; dma_addr_t bus_address; u32 ci; + u32 prev_eqe_index; + bool check_eqe_index; }; @@ -812,6 +854,132 @@ enum div_select_defs { DIV_SEL_DIVIDED_PLL = 3, }; +enum pci_region { + PCI_REGION_CFG, + PCI_REGION_SRAM, + PCI_REGION_DRAM, + PCI_REGION_SP_SRAM, + PCI_REGION_NUMBER, +}; + +/** + * struct pci_mem_region - describe memory region in a PCI bar + * @region_base: region base address + * @region_size: region size + * @bar_size: size of the BAR + * @offset_in_bar: region offset into the bar + * @bar_id: bar ID of the region + * @used: if used 1, otherwise 0 + */ +struct pci_mem_region { + u64 region_base; + u64 region_size; + u64 bar_size; + u32 offset_in_bar; + u8 bar_id; + u8 used; +}; + +/** + * struct static_fw_load_mgr - static FW load manager + * @preboot_version_max_off: max offset to preboot version + * @boot_fit_version_max_off: max offset to boot fit version + * @kmd_msg_to_cpu_reg: register address for KDM->CPU messages + * @cpu_cmd_status_to_host_reg: register address for CPU command status response + * @cpu_boot_status_reg: boot status register + * @cpu_boot_dev_status0_reg: boot device status register 0 + * @cpu_boot_dev_status1_reg: boot device status register 1 + * @boot_err0_reg: boot error register 0 + * @boot_err1_reg: boot error register 1 + * @preboot_version_offset_reg: SRAM offset to preboot version register + * @boot_fit_version_offset_reg: SRAM offset to boot fit version register + * @sram_offset_mask: mask for getting offset into the SRAM + * @cpu_reset_wait_msec: used when setting WFE via kmd_msg_to_cpu_reg + */ +struct static_fw_load_mgr { + u64 preboot_version_max_off; + u64 boot_fit_version_max_off; + u32 kmd_msg_to_cpu_reg; + u32 cpu_cmd_status_to_host_reg; + u32 cpu_boot_status_reg; + u32 cpu_boot_dev_status0_reg; + u32 cpu_boot_dev_status1_reg; + u32 boot_err0_reg; + u32 boot_err1_reg; + u32 preboot_version_offset_reg; + u32 boot_fit_version_offset_reg; + u32 sram_offset_mask; + u32 cpu_reset_wait_msec; +}; + +/** + * struct fw_response - FW response to LKD command + * @ram_offset: descriptor offset into the RAM + * @ram_type: RAM type containing the descriptor (SRAM/DRAM) + * @status: command status + */ +struct fw_response { + u32 ram_offset; + u8 ram_type; + u8 status; +}; + +/** + * struct dynamic_fw_load_mgr - dynamic FW load manager + * @response: FW to LKD response + * @comm_desc: the communication descriptor with FW + * @image_region: region to copy the FW image to + * @fw_image_size: size of FW image to load + * @wait_for_bl_timeout: timeout for waiting for boot loader to respond + */ +struct dynamic_fw_load_mgr { + struct fw_response response; + struct lkd_fw_comms_desc comm_desc; + struct pci_mem_region *image_region; + size_t fw_image_size; + u32 wait_for_bl_timeout; +}; + +/** + * struct fw_image_props - properties of FW image + * @image_name: name of the image + * @src_off: offset in src FW to copy from + * @copy_size: amount of bytes to copy (0 to copy the whole binary) + */ +struct fw_image_props { + char *image_name; + u32 src_off; + u32 copy_size; +}; + +/** + * struct fw_load_mgr - manager FW loading process + * @dynamic_loader: specific structure for dynamic load + * @static_loader: specific structure for static load + * @boot_fit_img: boot fit image properties + * @linux_img: linux image properties + * @cpu_timeout: CPU response timeout in usec + * @boot_fit_timeout: Boot fit load timeout in usec + * @skip_bmc: should BMC be skipped + * @sram_bar_id: SRAM bar ID + * @dram_bar_id: DRAM bar ID + * @linux_loaded: true if linux was loaded so far + */ +struct fw_load_mgr { + union { + struct dynamic_fw_load_mgr dynamic_loader; + struct static_fw_load_mgr static_loader; + }; + struct fw_image_props boot_fit_img; + struct fw_image_props linux_img; + u32 cpu_timeout; + u32 boot_fit_timeout; + u8 skip_bmc; + u8 sram_bar_id; + u8 dram_bar_id; + u8 linux_loaded; +}; + /** * struct hl_asic_funcs - ASIC specific functions that are can be called from * common code. @@ -901,8 +1069,6 @@ enum div_select_defs { * @ctx_fini: context dependent cleanup. * @get_clk_rate: Retrieve the ASIC current and maximum clock rate in MHz * @get_queue_id_for_cq: Get the H/W queue id related to the given CQ index. - * @read_device_fw_version: read the device's firmware versions that are - * contained in registers * @load_firmware_to_device: load the firmware to the device's memory * @load_boot_fit_to_device: load boot fit to device's memory * @get_signal_cb_size: Get signal CB size. @@ -933,6 +1099,8 @@ enum div_select_defs { * @get_msi_info: Retrieve asic-specific MSI ID of the f/w async event * @map_pll_idx_to_fw_idx: convert driver specific per asic PLL index to * generic f/w compatible PLL Indexes + * @init_firmware_loader: initialize data for FW loader. + * @init_cpu_scrambler_dram: Enable CPU specific DRAM scrambling */ struct hl_asic_funcs { int (*early_init)(struct hl_device *hdev); @@ -1006,7 +1174,7 @@ struct hl_asic_funcs { int (*mmu_invalidate_cache)(struct hl_device *hdev, bool is_hard, u32 flags); int (*mmu_invalidate_cache_range)(struct hl_device *hdev, bool is_hard, - u32 asid, u64 va, u64 size); + u32 flags, u32 asid, u64 va, u64 size); int (*send_heartbeat)(struct hl_device *hdev); void (*set_clock_gating)(struct hl_device *hdev); void (*disable_clock_gating)(struct hl_device *hdev); @@ -1030,8 +1198,6 @@ struct hl_asic_funcs { void (*ctx_fini)(struct hl_ctx *ctx); int (*get_clk_rate)(struct hl_device *hdev, u32 *cur_clk, u32 *max_clk); u32 (*get_queue_id_for_cq)(struct hl_device *hdev, u32 cq_idx); - int (*read_device_fw_version)(struct hl_device *hdev, - enum hl_fw_component fwc); int (*load_firmware_to_device)(struct hl_device *hdev); int (*load_boot_fit_to_device)(struct hl_device *hdev); u32 (*get_signal_cb_size)(struct hl_device *hdev); @@ -1056,8 +1222,10 @@ struct hl_asic_funcs { int (*hw_block_mmap)(struct hl_device *hdev, struct vm_area_struct *vma, u32 block_id, u32 block_size); void (*enable_events_from_fw)(struct hl_device *hdev); - void (*get_msi_info)(u32 *table); + void (*get_msi_info)(__le32 *table); int (*map_pll_idx_to_fw_idx)(u32 pll_idx); + void (*init_firmware_loader)(struct hl_device *hdev); + void (*init_cpu_scrambler_dram)(struct hl_device *hdev); }; @@ -1262,6 +1430,7 @@ struct hl_userptr { * @staged_sequence: the sequence of the staged submission this CS is part of, * relevant only if staged_cs is set. * @timeout_jiffies: cs timeout in jiffies. + * @submission_time_jiffies: submission time of the cs * @type: CS_TYPE_*. * @submitted: true if CS was submitted to H/W. * @completed: true if CS was completed by device. @@ -1274,6 +1443,8 @@ struct hl_userptr { * @staged_first: true if this is the first staged CS and we need to receive * timeout for this CS. * @staged_cs: true if this CS is part of a staged submission. + * @skip_reset_on_timeout: true if we shall not reset the device in case + * timeout occurs (debug scenario). */ struct hl_cs { u16 *jobs_in_queue_cnt; @@ -1291,6 +1462,7 @@ struct hl_cs { u64 sequence; u64 staged_sequence; u64 timeout_jiffies; + u64 submission_time_jiffies; enum hl_cs_type type; u8 submitted; u8 completed; @@ -1301,6 +1473,7 @@ struct hl_cs { u8 staged_last; u8 staged_first; u8 staged_cs; + u8 skip_reset_on_timeout; }; /** @@ -1922,7 +2095,7 @@ struct hl_mmu_funcs { * @kernel_queues: array of hl_hw_queue. * @cs_mirror_list: CS mirror list for TDR. * @cs_mirror_lock: protects cs_mirror_list. - * @kernel_cb_mgr: command buffer manager for creating/destroying/handling CGs. + * @kernel_cb_mgr: command buffer manager for creating/destroying/handling CBs. * @event_queue: event queue for IRQ from CPU-CP. * @dma_pool: DMA pool for small allocations. * @cpu_accessible_dma_mem: Host <-> CPU-CP shared memory CPU address. @@ -1954,6 +2127,8 @@ struct hl_mmu_funcs { * @aggregated_cs_counters: aggregated cs counters among all contexts * @mmu_priv: device-specific MMU data. * @mmu_func: device-related MMU functions. + * @fw_loader: FW loader manager. + * @pci_mem_region: array of memory regions in the PCI * @dram_used_mem: current DRAM memory consumption. * @timeout_jiffies: device CS timeout value. * @max_power: the max power of the device, as configured by the sysadmin. This @@ -1968,6 +2143,11 @@ struct hl_mmu_funcs { * the error will be ignored by the driver during * device initialization. Mainly used to debug and * workaround firmware bugs + * @last_successful_open_jif: timestamp (jiffies) of the last successful + * device open. + * @last_open_session_duration_jif: duration (jiffies) of the last device open + * session. + * @open_counter: number of successful device open operations. * @in_reset: is device in reset flow. * @curr_pll_profile: current PLL profile. * @card_type: Various ASICs have several card types. This indicates the card @@ -2007,6 +2187,8 @@ struct hl_mmu_funcs { * @collective_mon_idx: helper index for collective initialization * @supports_coresight: is CoreSight supported. * @supports_soft_reset: is soft reset supported. + * @allow_external_soft_reset: true if soft reset initiated by user or TDR is + * allowed. * @supports_cb_mapping: is mapping a CB to the device's MMU supported. * @needs_reset: true if reset_on_lockup is false and device should be reset * due to lockup. @@ -2015,6 +2197,14 @@ struct hl_mmu_funcs { * @device_fini_pending: true if device_fini was called and might be * waiting for the reset thread to finish * @supports_staged_submission: true if staged submissions are supported + * @curr_reset_cause: saves an enumerated reset cause when a hard reset is + * triggered, and cleared after it is shared with preboot. + * @skip_reset_on_timeout: Skip device reset if CS has timed out, wait for it to + * complete instead. + * @device_cpu_is_halted: Flag to indicate whether the device CPU was already + * halted. We can't halt it again because the COMMS + * protocol will throw an error. Relevant only for + * cases where Linux was not loaded to device CPU */ struct hl_device { struct pci_dev *pdev; @@ -2079,11 +2269,18 @@ struct hl_device { struct hl_mmu_priv mmu_priv; struct hl_mmu_funcs mmu_func[MMU_NUM_PGT_LOCATIONS]; + struct fw_load_mgr fw_loader; + + struct pci_mem_region pci_mem_region[PCI_REGION_NUMBER]; + atomic64_t dram_used_mem; u64 timeout_jiffies; u64 max_power; u64 clock_gating_mask; u64 boot_error_status_mask; + u64 last_successful_open_jif; + u64 last_open_session_duration_jif; + u64 open_counter; atomic_t in_reset; enum hl_pll_frequency curr_pll_profile; enum cpucp_card_types card_type; @@ -2116,11 +2313,15 @@ struct hl_device { u8 collective_mon_idx; u8 supports_coresight; u8 supports_soft_reset; + u8 allow_external_soft_reset; u8 supports_cb_mapping; u8 needs_reset; u8 process_kill_trial_cnt; u8 device_fini_pending; u8 supports_staged_submission; + u8 curr_reset_cause; + u8 skip_reset_on_timeout; + u8 device_cpu_is_halted; /* Parameters for bring-up */ u64 nic_ports_mask; @@ -2138,6 +2339,7 @@ struct hl_device { u8 rl_enable; u8 reset_on_preboot_fail; u8 reset_upon_device_release; + u8 reset_if_device_not_idle; }; @@ -2384,11 +2586,13 @@ void hl_fw_cpu_accessible_dma_pool_free(struct hl_device *hdev, size_t size, void *vaddr); int hl_fw_send_heartbeat(struct hl_device *hdev); int hl_fw_cpucp_info_get(struct hl_device *hdev, - u32 cpu_security_boot_status_reg, - u32 boot_err0_reg); + u32 sts_boot_dev_sts0_reg, + u32 sts_boot_dev_sts1_reg, u32 boot_err0_reg, + u32 boot_err1_reg); int hl_fw_cpucp_handshake(struct hl_device *hdev, - u32 cpu_security_boot_status_reg, - u32 boot_err0_reg); + u32 sts_boot_dev_sts0_reg, + u32 sts_boot_dev_sts1_reg, u32 boot_err0_reg, + u32 boot_err1_reg); int hl_fw_get_eeprom_data(struct hl_device *hdev, void *data, size_t max_size); int hl_fw_cpucp_pci_counters_get(struct hl_device *hdev, struct hl_info_pci_counters *counters); @@ -2399,14 +2603,17 @@ int get_used_pll_index(struct hl_device *hdev, u32 input_pll_index, int hl_fw_cpucp_pll_info_get(struct hl_device *hdev, u32 pll_index, u16 *pll_freq_arr); int hl_fw_cpucp_power_get(struct hl_device *hdev, u64 *power); -int hl_fw_init_cpu(struct hl_device *hdev, u32 cpu_boot_status_reg, - u32 msg_to_cpu_reg, u32 cpu_msg_status_reg, - u32 cpu_security_boot_status_reg, u32 boot_err0_reg, - bool skip_bmc, u32 cpu_timeout, u32 boot_fit_timeout); +void hl_fw_ask_hard_reset_without_linux(struct hl_device *hdev); +void hl_fw_ask_halt_machine_without_linux(struct hl_device *hdev); +int hl_fw_init_cpu(struct hl_device *hdev); int hl_fw_read_preboot_status(struct hl_device *hdev, u32 cpu_boot_status_reg, - u32 cpu_security_boot_status_reg, u32 boot_err0_reg, - u32 timeout); - + u32 sts_boot_dev_sts0_reg, + u32 sts_boot_dev_sts1_reg, u32 boot_err0_reg, + u32 boot_err1_reg, u32 timeout); +int hl_fw_dynamic_send_protocol_cmd(struct hl_device *hdev, + struct fw_load_mgr *fw_loader, + enum comms_cmd cmd, unsigned int size, + bool wait_ok, u32 timeout); int hl_pci_bars_map(struct hl_device *hdev, const char * const name[3], bool is_wc[3]); int hl_pci_elbi_read(struct hl_device *hdev, u64 addr, u32 *data); @@ -2415,6 +2622,7 @@ int hl_pci_set_inbound_region(struct hl_device *hdev, u8 region, struct hl_inbound_pci_region *pci_region); int hl_pci_set_outbound_region(struct hl_device *hdev, struct hl_outbound_pci_region *pci_region); +enum pci_region hl_get_pci_memory_region(struct hl_device *hdev, u64 addr); int hl_pci_init(struct hl_device *hdev); void hl_pci_fini(struct hl_device *hdev); @@ -2443,6 +2651,8 @@ int hl_set_voltage(struct hl_device *hdev, int hl_set_current(struct hl_device *hdev, int sensor_index, u32 attr, long value); void hl_release_pending_user_interrupts(struct hl_device *hdev); +int hl_cs_signal_sob_wraparound_handler(struct hl_device *hdev, u32 q_idx, + struct hl_hw_sob **hw_sob, u32 count); #ifdef CONFIG_DEBUG_FS diff --git a/drivers/misc/habanalabs/common/habanalabs_drv.c b/drivers/misc/habanalabs/common/habanalabs_drv.c index 64d1530db985..4194cda2d04c 100644 --- a/drivers/misc/habanalabs/common/habanalabs_drv.c +++ b/drivers/misc/habanalabs/common/habanalabs_drv.c @@ -29,7 +29,7 @@ static DEFINE_MUTEX(hl_devs_idr_lock); static int timeout_locked = 30; static int reset_on_lockup = 1; -static int memory_scrub = 1; +static int memory_scrub; static ulong boot_error_status_mask = ULONG_MAX; module_param(timeout_locked, int, 0444); @@ -42,7 +42,7 @@ MODULE_PARM_DESC(reset_on_lockup, module_param(memory_scrub, int, 0444); MODULE_PARM_DESC(memory_scrub, - "Scrub device memory in various states (0 = no, 1 = yes, default yes)"); + "Scrub device memory in various states (0 = no, 1 = yes, default no)"); module_param(boot_error_status_mask, ulong, 0444); MODULE_PARM_DESC(boot_error_status_mask, @@ -187,6 +187,9 @@ int hl_device_open(struct inode *inode, struct file *filp) hl_debugfs_add_file(hpriv); + hdev->open_counter++; + hdev->last_successful_open_jif = jiffies; + return 0; out_err: @@ -264,6 +267,7 @@ static void set_driver_behavior_per_device(struct hl_device *hdev) hdev->bmc_enable = 1; hdev->hard_reset_on_fw_events = 1; hdev->reset_on_preboot_fail = 1; + hdev->reset_if_device_not_idle = 1; hdev->reset_pcilink = 0; hdev->axi_drain = 0; @@ -308,10 +312,10 @@ int create_hdev(struct hl_device **dev, struct pci_dev *pdev, } if (pdev) - hdev->asic_prop.fw_security_disabled = - !is_asic_secured(pdev->device); + hdev->asic_prop.fw_security_enabled = + is_asic_secured(hdev->asic_type); else - hdev->asic_prop.fw_security_disabled = true; + hdev->asic_prop.fw_security_enabled = false; /* Assign status description string */ strncpy(hdev->status[HL_DEVICE_STATUS_MALFUNCTION], @@ -325,11 +329,14 @@ int create_hdev(struct hl_device **dev, struct pci_dev *pdev, hdev->reset_on_lockup = reset_on_lockup; hdev->memory_scrub = memory_scrub; hdev->boot_error_status_mask = boot_error_status_mask; + hdev->stop_on_err = true; hdev->pldm = 0; set_driver_behavior_per_device(hdev); + hdev->curr_reset_cause = HL_RESET_CAUSE_UNKNOWN; + if (timeout_locked) hdev->timeout_jiffies = msecs_to_jiffies(timeout_locked * 1000); else @@ -464,6 +471,7 @@ static int hl_pci_probe(struct pci_dev *pdev, return 0; disable_device: + pci_disable_pcie_error_reporting(pdev); pci_set_drvdata(pdev, NULL); destroy_hdev(hdev); @@ -572,7 +580,11 @@ static struct pci_driver hl_pci_driver = { .probe = hl_pci_probe, .remove = hl_pci_remove, .shutdown = hl_pci_remove, - .driver.pm = &hl_pm_ops, + .driver = { + .name = HL_NAME, + .pm = &hl_pm_ops, + .probe_type = PROBE_PREFER_ASYNCHRONOUS, + }, .err_handler = &hl_pci_err_handler, }; diff --git a/drivers/misc/habanalabs/common/habanalabs_ioctl.c b/drivers/misc/habanalabs/common/habanalabs_ioctl.c index 33841c272eb6..f4dda7b4acdd 100644 --- a/drivers/misc/habanalabs/common/habanalabs_ioctl.c +++ b/drivers/misc/habanalabs/common/habanalabs_ioctl.c @@ -95,7 +95,7 @@ static int hw_ip_info(struct hl_device *hdev, struct hl_info_args *args) hw_ip.first_available_interrupt_id = prop->first_available_user_msix_interrupt; return copy_to_user(out, &hw_ip, - min((size_t)size, sizeof(hw_ip))) ? -EFAULT : 0; + min((size_t) size, sizeof(hw_ip))) ? -EFAULT : 0; } static int hw_events_info(struct hl_device *hdev, bool aggregate, @@ -460,6 +460,24 @@ static int power_info(struct hl_fpriv *hpriv, struct hl_info_args *args) min((size_t) max_size, sizeof(power_info))) ? -EFAULT : 0; } +static int open_stats_info(struct hl_fpriv *hpriv, struct hl_info_args *args) +{ + struct hl_device *hdev = hpriv->hdev; + u32 max_size = args->return_size; + struct hl_open_stats_info open_stats_info = {0}; + void __user *out = (void __user *) (uintptr_t) args->return_pointer; + + if ((!max_size) || (!out)) + return -EINVAL; + + open_stats_info.last_open_period_ms = jiffies64_to_msecs( + hdev->last_open_session_duration_jif); + open_stats_info.open_counter = hdev->open_counter; + + return copy_to_user(out, &open_stats_info, + min((size_t) max_size, sizeof(open_stats_info))) ? -EFAULT : 0; +} + static int _hl_info_ioctl(struct hl_fpriv *hpriv, void *data, struct device *dev) { @@ -543,6 +561,9 @@ static int _hl_info_ioctl(struct hl_fpriv *hpriv, void *data, case HL_INFO_POWER: return power_info(hpriv, args); + case HL_INFO_OPEN_STATS: + return open_stats_info(hpriv, args); + default: dev_err(dev, "Invalid request %d\n", args->op); rc = -ENOTTY; diff --git a/drivers/misc/habanalabs/common/hw_queue.c b/drivers/misc/habanalabs/common/hw_queue.c index 173438461835..bcabfdbf1e01 100644 --- a/drivers/misc/habanalabs/common/hw_queue.c +++ b/drivers/misc/habanalabs/common/hw_queue.c @@ -410,19 +410,20 @@ static void hw_queue_schedule_job(struct hl_cs_job *job) ext_and_hw_queue_submit_bd(hdev, q, ctl, len, ptr); } -static void init_signal_cs(struct hl_device *hdev, +static int init_signal_cs(struct hl_device *hdev, struct hl_cs_job *job, struct hl_cs_compl *cs_cmpl) { struct hl_sync_stream_properties *prop; struct hl_hw_sob *hw_sob; u32 q_idx; + int rc = 0; q_idx = job->hw_queue_id; prop = &hdev->kernel_queues[q_idx].sync_stream_prop; hw_sob = &prop->hw_sob[prop->curr_sob_offset]; cs_cmpl->hw_sob = hw_sob; - cs_cmpl->sob_val = prop->next_sob_val++; + cs_cmpl->sob_val = prop->next_sob_val; dev_dbg(hdev->dev, "generate signal CB, sob_id: %d, sob val: 0x%x, q_idx: %d\n", @@ -434,24 +435,9 @@ static void init_signal_cs(struct hl_device *hdev, hdev->asic_funcs->gen_signal_cb(hdev, job->patched_cb, cs_cmpl->hw_sob->sob_id, 0, true); - kref_get(&hw_sob->kref); + rc = hl_cs_signal_sob_wraparound_handler(hdev, q_idx, &hw_sob, 1); - /* check for wraparound */ - if (prop->next_sob_val == HL_MAX_SOB_VAL) { - /* - * Decrement as we reached the max value. - * The release function won't be called here as we've - * just incremented the refcount. - */ - kref_put(&hw_sob->kref, hl_sob_reset_error); - prop->next_sob_val = 1; - /* only two SOBs are currently in use */ - prop->curr_sob_offset = - (prop->curr_sob_offset + 1) % HL_RSVD_SOBS; - - dev_dbg(hdev->dev, "switched to SOB %d, q_idx: %d\n", - prop->curr_sob_offset, q_idx); - } + return rc; } static void init_wait_cs(struct hl_device *hdev, struct hl_cs *cs, @@ -504,22 +490,25 @@ static void init_wait_cs(struct hl_device *hdev, struct hl_cs *cs, * * H/W queues spinlock should be taken before calling this function */ -static void init_signal_wait_cs(struct hl_cs *cs) +static int init_signal_wait_cs(struct hl_cs *cs) { struct hl_ctx *ctx = cs->ctx; struct hl_device *hdev = ctx->hdev; struct hl_cs_job *job; struct hl_cs_compl *cs_cmpl = container_of(cs->fence, struct hl_cs_compl, base_fence); + int rc = 0; /* There is only one job in a signal/wait CS */ job = list_first_entry(&cs->job_list, struct hl_cs_job, cs_node); if (cs->type & CS_TYPE_SIGNAL) - init_signal_cs(hdev, job, cs_cmpl); + rc = init_signal_cs(hdev, job, cs_cmpl); else if (cs->type & CS_TYPE_WAIT) init_wait_cs(hdev, cs, job, cs_cmpl); + + return rc; } /* @@ -590,11 +579,16 @@ int hl_hw_queue_schedule_cs(struct hl_cs *cs) } } - if ((cs->type == CS_TYPE_SIGNAL) || (cs->type == CS_TYPE_WAIT)) - init_signal_wait_cs(cs); - else if (cs->type == CS_TYPE_COLLECTIVE_WAIT) + if ((cs->type == CS_TYPE_SIGNAL) || (cs->type == CS_TYPE_WAIT)) { + rc = init_signal_wait_cs(cs); + if (rc) { + dev_err(hdev->dev, "Failed to submit signal cs\n"); + goto unroll_cq_resv; + } + } else if (cs->type == CS_TYPE_COLLECTIVE_WAIT) hdev->asic_funcs->collective_wait_init_cs(cs); + spin_lock(&hdev->cs_mirror_lock); /* Verify staged CS exists and add to the staged list */ diff --git a/drivers/misc/habanalabs/common/irq.c b/drivers/misc/habanalabs/common/irq.c index 27129868c711..39b14a933393 100644 --- a/drivers/misc/habanalabs/common/irq.c +++ b/drivers/misc/habanalabs/common/irq.c @@ -207,17 +207,33 @@ irqreturn_t hl_irq_handler_eq(int irq, void *arg) struct hl_eq_entry *eq_entry; struct hl_eq_entry *eq_base; struct hl_eqe_work *handle_eqe_work; + bool entry_ready; + u32 cur_eqe; + u16 cur_eqe_index; eq_base = eq->kernel_address; while (1) { - bool entry_ready = - ((le32_to_cpu(eq_base[eq->ci].hdr.ctl) & - EQ_CTL_READY_MASK) >> EQ_CTL_READY_SHIFT); + cur_eqe = le32_to_cpu(eq_base[eq->ci].hdr.ctl); + entry_ready = !!FIELD_GET(EQ_CTL_READY_MASK, cur_eqe); if (!entry_ready) break; + cur_eqe_index = FIELD_GET(EQ_CTL_INDEX_MASK, cur_eqe); + if ((hdev->event_queue.check_eqe_index) && + (((eq->prev_eqe_index + 1) & EQ_CTL_INDEX_MASK) + != cur_eqe_index)) { + dev_dbg(hdev->dev, + "EQE 0x%x in queue is ready but index does not match %d!=%d", + eq_base[eq->ci].hdr.ctl, + ((eq->prev_eqe_index + 1) & EQ_CTL_INDEX_MASK), + cur_eqe_index); + break; + } + + eq->prev_eqe_index++; + eq_entry = &eq_base[eq->ci]; /* @@ -341,6 +357,7 @@ int hl_eq_init(struct hl_device *hdev, struct hl_eq *q) q->hdev = hdev; q->kernel_address = p; q->ci = 0; + q->prev_eqe_index = 0; return 0; } @@ -365,6 +382,7 @@ void hl_eq_fini(struct hl_device *hdev, struct hl_eq *q) void hl_eq_reset(struct hl_device *hdev, struct hl_eq *q) { q->ci = 0; + q->prev_eqe_index = 0; /* * It's not enough to just reset the PI/CI because the H/W may have diff --git a/drivers/misc/habanalabs/common/memory.c b/drivers/misc/habanalabs/common/memory.c index 2938cbbafbbc..af339ce1ab4f 100644 --- a/drivers/misc/habanalabs/common/memory.c +++ b/drivers/misc/habanalabs/common/memory.c @@ -570,8 +570,10 @@ static u64 get_va_block(struct hl_device *hdev, if ((is_align_pow_2 && (hint_addr & (va_block_align - 1))) || (!is_align_pow_2 && do_div(tmp_hint_addr, va_range->page_size))) { - dev_info(hdev->dev, "Hint address 0x%llx will be ignored\n", - hint_addr); + + dev_dbg(hdev->dev, + "Hint address 0x%llx will be ignored because it is not aligned\n", + hint_addr); hint_addr = 0; } @@ -1117,7 +1119,8 @@ static int map_device_va(struct hl_ctx *ctx, struct hl_mem_in *args, goto map_err; } - rc = hdev->asic_funcs->mmu_invalidate_cache(hdev, false, *vm_type); + rc = hdev->asic_funcs->mmu_invalidate_cache_range(hdev, false, + *vm_type, ctx->asid, ret_vaddr, phys_pg_pack->total_size); mutex_unlock(&ctx->mmu_lock); @@ -1261,8 +1264,9 @@ static int unmap_device_va(struct hl_ctx *ctx, struct hl_mem_in *args, * at the loop end rather than for each iteration */ if (!ctx_free) - rc = hdev->asic_funcs->mmu_invalidate_cache(hdev, true, - *vm_type); + rc = hdev->asic_funcs->mmu_invalidate_cache_range(hdev, true, + *vm_type, ctx->asid, vaddr, + phys_pg_pack->total_size); mutex_unlock(&ctx->mmu_lock); @@ -1369,12 +1373,7 @@ int hl_hw_block_mmap(struct hl_fpriv *hpriv, struct vm_area_struct *vma) /* Driver only allows mapping of a complete HW block */ block_size = vma->vm_end - vma->vm_start; -#ifdef _HAS_TYPE_ARG_IN_ACCESS_OK - if (!access_ok(VERIFY_WRITE, - (void __user *) (uintptr_t) vma->vm_start, block_size)) { -#else if (!access_ok((void __user *) (uintptr_t) vma->vm_start, block_size)) { -#endif dev_err(hdev->dev, "user pointer is invalid - 0x%lx\n", vma->vm_start); @@ -1608,7 +1607,8 @@ static int get_user_memory(struct hl_device *hdev, u64 addr, u64 size, if (rc != npages) { dev_err(hdev->dev, - "Failed to map host memory, user ptr probably wrong\n"); + "Failed (%d) to pin host memory with user ptr 0x%llx, size 0x%llx, npages %d\n", + rc, addr, size, npages); if (rc < 0) goto destroy_pages; npages = rc; diff --git a/drivers/misc/habanalabs/common/mmu/mmu.c b/drivers/misc/habanalabs/common/mmu/mmu.c index b37189956b14..792d25b79ea6 100644 --- a/drivers/misc/habanalabs/common/mmu/mmu.c +++ b/drivers/misc/habanalabs/common/mmu/mmu.c @@ -501,12 +501,20 @@ static void hl_mmu_pa_page_with_offset(struct hl_ctx *ctx, u64 virt_addr, if ((hops->range_type == HL_VA_RANGE_TYPE_DRAM) && !is_power_of_2(prop->dram_page_size)) { - u32 bit; + unsigned long dram_page_size = prop->dram_page_size; u64 page_offset_mask; u64 phys_addr_mask; + u32 bit; - bit = __ffs64((u64)prop->dram_page_size); - page_offset_mask = ((1ull << bit) - 1); + /* + * find last set bit in page_size to cover all bits of page + * offset. note that 1 has to be added to bit index. + * note that the internal ulong variable is used to avoid + * alignment issue. + */ + bit = find_last_bit(&dram_page_size, + sizeof(dram_page_size) * BITS_PER_BYTE) + 1; + page_offset_mask = (BIT_ULL(bit) - 1); phys_addr_mask = ~page_offset_mask; *phys_addr = (tmp_phys_addr & phys_addr_mask) | (virt_addr & page_offset_mask); diff --git a/drivers/misc/habanalabs/common/pci/pci.c b/drivers/misc/habanalabs/common/pci/pci.c index e941b7eef346..d5bedf5ba011 100644 --- a/drivers/misc/habanalabs/common/pci/pci.c +++ b/drivers/misc/habanalabs/common/pci/pci.c @@ -10,7 +10,7 @@ #include <linux/pci.h> -#define HL_PLDM_PCI_ELBI_TIMEOUT_MSEC (HL_PCI_ELBI_TIMEOUT_MSEC * 10) +#define HL_PLDM_PCI_ELBI_TIMEOUT_MSEC (HL_PCI_ELBI_TIMEOUT_MSEC * 100) #define IATU_REGION_CTRL_REGION_EN_MASK BIT(31) #define IATU_REGION_CTRL_MATCH_MODE_MASK BIT(30) @@ -360,6 +360,32 @@ int hl_pci_set_outbound_region(struct hl_device *hdev, } /** + * hl_get_pci_memory_region() - get PCI region for given address + * @hdev: Pointer to hl_device structure. + * @addr: device address + * + * @return region index on success, otherwise PCI_REGION_NUMBER (invalid + * region index) + */ +enum pci_region hl_get_pci_memory_region(struct hl_device *hdev, u64 addr) +{ + int i; + + for (i = 0 ; i < PCI_REGION_NUMBER ; i++) { + struct pci_mem_region *region = &hdev->pci_mem_region[i]; + + if (!region->used) + continue; + + if ((addr >= region->region_base) && + (addr < region->region_base + region->region_size)) + return i; + } + + return PCI_REGION_NUMBER; +} + +/** * hl_pci_init() - PCI initialization code. * @hdev: Pointer to hl_device structure. * @@ -395,6 +421,12 @@ int hl_pci_init(struct hl_device *hdev) goto unmap_pci_bars; } + /* Driver must sleep in order for FW to finish the iATU configuration */ + if (hdev->asic_prop.iatu_done_by_fw) { + usleep_range(2000, 3000); + hdev->asic_funcs->set_dma_mask_from_fw(hdev); + } + rc = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(hdev->dma_mask)); if (rc) { diff --git a/drivers/misc/habanalabs/common/sysfs.c b/drivers/misc/habanalabs/common/sysfs.c index c9f649b31e3a..db72df282ef8 100644 --- a/drivers/misc/habanalabs/common/sysfs.c +++ b/drivers/misc/habanalabs/common/sysfs.c @@ -208,7 +208,7 @@ static ssize_t soft_reset_store(struct device *dev, goto out; } - if (!hdev->supports_soft_reset) { + if (!hdev->allow_external_soft_reset) { dev_err(hdev->dev, "Device does not support soft-reset\n"); goto out; } diff --git a/drivers/misc/habanalabs/gaudi/gaudi.c b/drivers/misc/habanalabs/gaudi/gaudi.c index 9e4a6bb3acd1..aa8a0ca5aca2 100644 --- a/drivers/misc/habanalabs/gaudi/gaudi.c +++ b/drivers/misc/habanalabs/gaudi/gaudi.c @@ -78,6 +78,7 @@ #define GAUDI_PLDM_TPC_KERNEL_WAIT_USEC (HL_DEVICE_TIMEOUT_USEC * 30) #define GAUDI_BOOT_FIT_REQ_TIMEOUT_USEC 1000000 /* 1s */ #define GAUDI_MSG_TO_CPU_TIMEOUT_USEC 4000000 /* 4s */ +#define GAUDI_WAIT_FOR_BL_TIMEOUT_USEC 15000000 /* 15s */ #define GAUDI_QMAN0_FENCE_VAL 0x72E91AB9 @@ -409,7 +410,7 @@ static inline void set_default_power_values(struct hl_device *hdev) } } -static int gaudi_get_fixed_properties(struct hl_device *hdev) +static int gaudi_set_fixed_properties(struct hl_device *hdev) { struct asic_fixed_properties *prop = &hdev->asic_prop; u32 num_sync_stream_queues = 0; @@ -545,8 +546,10 @@ static int gaudi_get_fixed_properties(struct hl_device *hdev) for (i = 0 ; i < HL_MAX_DCORES ; i++) prop->first_available_cq[i] = USHRT_MAX; - prop->fw_security_status_valid = false; + prop->fw_cpu_boot_dev_sts0_valid = false; + prop->fw_cpu_boot_dev_sts1_valid = false; prop->hard_reset_done_by_fw = false; + prop->gic_interrupts_enable = true; return 0; } @@ -577,6 +580,9 @@ static u64 gaudi_set_hbm_bar_base(struct hl_device *hdev, u64 addr) if ((gaudi) && (gaudi->hbm_bar_cur_addr == addr)) return old_addr; + if (hdev->asic_prop.iatu_done_by_fw) + return U64_MAX; + /* Inbound Region 2 - Bar 4 - Point to HBM */ pci_region.mode = PCI_BAR_MATCH_MODE; pci_region.bar = HBM_BAR_ID; @@ -599,10 +605,8 @@ static int gaudi_init_iatu(struct hl_device *hdev) struct hl_outbound_pci_region outbound_region; int rc; - if (hdev->asic_prop.iatu_done_by_fw) { - hdev->asic_funcs->set_dma_mask_from_fw(hdev); + if (hdev->asic_prop.iatu_done_by_fw) return 0; - } /* Inbound Region 0 - Bar 0 - Point to SRAM + CFG */ inbound_region.mode = PCI_BAR_MATCH_MODE; @@ -651,9 +655,9 @@ static int gaudi_early_init(struct hl_device *hdev) u32 fw_boot_status; int rc; - rc = gaudi_get_fixed_properties(hdev); + rc = gaudi_set_fixed_properties(hdev); if (rc) { - dev_err(hdev->dev, "Failed to get fixed properties\n"); + dev_err(hdev->dev, "Failed setting fixed properties\n"); return rc; } @@ -683,8 +687,14 @@ static int gaudi_early_init(struct hl_device *hdev) prop->dram_pci_bar_size = pci_resource_len(pdev, HBM_BAR_ID); /* If FW security is enabled at this point it means no access to ELBI */ - if (!hdev->asic_prop.fw_security_disabled) { + if (hdev->asic_prop.fw_security_enabled) { hdev->asic_prop.iatu_done_by_fw = true; + + /* + * GIC-security-bit can ONLY be set by CPUCP, so in this stage + * decision can only be taken based on PCI ID security. + */ + hdev->asic_prop.gic_interrupts_enable = false; goto pci_init; } @@ -707,8 +717,10 @@ pci_init: * version to determine whether we run with a security-enabled firmware */ rc = hl_fw_read_preboot_status(hdev, mmPSOC_GLOBAL_CONF_CPU_BOOT_STATUS, - mmCPU_BOOT_DEV_STS0, mmCPU_BOOT_ERR0, - GAUDI_BOOT_FIT_REQ_TIMEOUT_USEC); + mmCPU_BOOT_DEV_STS0, + mmCPU_BOOT_DEV_STS1, mmCPU_BOOT_ERR0, + mmCPU_BOOT_ERR1, + GAUDI_BOOT_FIT_REQ_TIMEOUT_USEC); if (rc) { if (hdev->reset_on_preboot_fail) hdev->asic_funcs->hw_fini(hdev, true); @@ -751,7 +763,14 @@ static int gaudi_fetch_psoc_frequency(struct hl_device *hdev) u16 pll_freq_arr[HL_PLL_NUM_OUTPUTS], freq; int rc; - if (hdev->asic_prop.fw_security_disabled) { + if (hdev->asic_prop.fw_security_enabled) { + rc = hl_fw_cpucp_pll_info_get(hdev, HL_GAUDI_CPU_PLL, pll_freq_arr); + + if (rc) + return rc; + + freq = pll_freq_arr[2]; + } else { /* Backward compatibility */ div_fctr = RREG32(mmPSOC_CPU_PLL_DIV_FACTOR_2); div_sel = RREG32(mmPSOC_CPU_PLL_DIV_SEL_2); @@ -779,13 +798,6 @@ static int gaudi_fetch_psoc_frequency(struct hl_device *hdev) div_sel); freq = 0; } - } else { - rc = hl_fw_cpucp_pll_info_get(hdev, HL_GAUDI_CPU_PLL, pll_freq_arr); - - if (rc) - return rc; - - freq = pll_freq_arr[2]; } prop->psoc_timestamp_frequency = freq; @@ -988,9 +1000,27 @@ static void gaudi_sob_group_reset_error(struct kref *ref) hw_sob_group->base_sob_id); } +static void gaudi_collective_mstr_sob_mask_set(struct gaudi_device *gaudi) +{ + struct gaudi_collective_properties *prop; + int i; + + prop = &gaudi->collective_props; + + memset(prop->mstr_sob_mask, 0, sizeof(prop->mstr_sob_mask)); + + for (i = 0 ; i < NIC_NUMBER_OF_ENGINES ; i++) + if (gaudi->hw_cap_initialized & BIT(HW_CAP_NIC_SHIFT + i)) + prop->mstr_sob_mask[i / HL_MAX_SOBS_PER_MONITOR] |= + BIT(i % HL_MAX_SOBS_PER_MONITOR); + /* Set collective engine bit */ + prop->mstr_sob_mask[i / HL_MAX_SOBS_PER_MONITOR] |= + BIT(i % HL_MAX_SOBS_PER_MONITOR); +} + static int gaudi_collective_init(struct hl_device *hdev) { - u32 i, master_monitor_sobs, sob_id, reserved_sobs_per_group; + u32 i, sob_id, reserved_sobs_per_group; struct gaudi_collective_properties *prop; struct gaudi_device *gaudi; @@ -1016,22 +1046,7 @@ static int gaudi_collective_init(struct hl_device *hdev) gaudi_collective_map_sobs(hdev, i); } - prop->mstr_sob_mask[0] = 0; - master_monitor_sobs = HL_MAX_SOBS_PER_MONITOR; - for (i = 0 ; i < master_monitor_sobs ; i++) - if (gaudi->hw_cap_initialized & BIT(HW_CAP_NIC_SHIFT + i)) - prop->mstr_sob_mask[0] |= BIT(i); - - prop->mstr_sob_mask[1] = 0; - master_monitor_sobs = - NIC_NUMBER_OF_ENGINES - HL_MAX_SOBS_PER_MONITOR; - for (i = 0 ; i < master_monitor_sobs; i++) { - if (gaudi->hw_cap_initialized & BIT(HW_CAP_NIC_SHIFT + i)) - prop->mstr_sob_mask[1] |= BIT(i); - } - - /* Set collective engine bit */ - prop->mstr_sob_mask[1] |= BIT(i); + gaudi_collective_mstr_sob_mask_set(gaudi); return 0; } @@ -1513,7 +1528,7 @@ static int gaudi_alloc_cpu_accessible_dma_mem(struct hl_device *hdev) hdev->cpu_pci_msb_addr = GAUDI_CPU_PCI_MSB_ADDR(hdev->cpu_accessible_dma_address); - if (hdev->asic_prop.fw_security_disabled) + if (!hdev->asic_prop.fw_security_enabled) GAUDI_PCI_TO_CPU_ADDR(hdev->cpu_accessible_dma_address); free_dma_mem_arr: @@ -1590,6 +1605,48 @@ free_internal_qmans_pq_mem: return rc; } +static void gaudi_set_pci_memory_regions(struct hl_device *hdev) +{ + struct asic_fixed_properties *prop = &hdev->asic_prop; + struct pci_mem_region *region; + + /* CFG */ + region = &hdev->pci_mem_region[PCI_REGION_CFG]; + region->region_base = CFG_BASE; + region->region_size = CFG_SIZE; + region->offset_in_bar = CFG_BASE - SPI_FLASH_BASE_ADDR; + region->bar_size = CFG_BAR_SIZE; + region->bar_id = CFG_BAR_ID; + region->used = 1; + + /* SRAM */ + region = &hdev->pci_mem_region[PCI_REGION_SRAM]; + region->region_base = SRAM_BASE_ADDR; + region->region_size = SRAM_SIZE; + region->offset_in_bar = 0; + region->bar_size = SRAM_BAR_SIZE; + region->bar_id = SRAM_BAR_ID; + region->used = 1; + + /* DRAM */ + region = &hdev->pci_mem_region[PCI_REGION_DRAM]; + region->region_base = DRAM_PHYS_BASE; + region->region_size = hdev->asic_prop.dram_size; + region->offset_in_bar = 0; + region->bar_size = prop->dram_pci_bar_size; + region->bar_id = HBM_BAR_ID; + region->used = 1; + + /* SP SRAM */ + region = &hdev->pci_mem_region[PCI_REGION_SP_SRAM]; + region->region_base = PSOC_SCRATCHPAD_ADDR; + region->region_size = PSOC_SCRATCHPAD_SIZE; + region->offset_in_bar = PSOC_SCRATCHPAD_ADDR - SPI_FLASH_BASE_ADDR; + region->bar_size = CFG_BAR_SIZE; + region->bar_id = CFG_BAR_ID; + region->used = 1; +} + static int gaudi_sw_init(struct hl_device *hdev) { struct gaudi_device *gaudi; @@ -1664,12 +1721,14 @@ static int gaudi_sw_init(struct hl_device *hdev) hdev->supports_coresight = true; hdev->supports_staged_submission = true; + gaudi_set_pci_memory_regions(hdev); + return 0; free_cpu_accessible_dma_pool: gen_pool_destroy(hdev->cpu_accessible_dma_pool); free_cpu_dma_mem: - if (hdev->asic_prop.fw_security_disabled) + if (!hdev->asic_prop.fw_security_enabled) GAUDI_CPU_TO_PCI_ADDR(hdev->cpu_accessible_dma_address, hdev->cpu_pci_msb_addr); hdev->asic_funcs->asic_dma_free_coherent(hdev, @@ -1691,7 +1750,7 @@ static int gaudi_sw_fini(struct hl_device *hdev) gen_pool_destroy(hdev->cpu_accessible_dma_pool); - if (hdev->asic_prop.fw_security_disabled) + if (!hdev->asic_prop.fw_security_enabled) GAUDI_CPU_TO_PCI_ADDR(hdev->cpu_accessible_dma_address, hdev->cpu_pci_msb_addr); @@ -1879,12 +1938,11 @@ static void gaudi_init_scrambler_sram(struct hl_device *hdev) { struct gaudi_device *gaudi = hdev->asic_specific; - if (!hdev->asic_prop.fw_security_disabled) + if (hdev->asic_prop.fw_security_enabled) return; - if (hdev->asic_prop.fw_security_status_valid && - (hdev->asic_prop.fw_app_security_map & - CPU_BOOT_DEV_STS0_SRAM_SCR_EN)) + if (hdev->asic_prop.fw_app_cpu_boot_dev_sts0 & + CPU_BOOT_DEV_STS0_SRAM_SCR_EN) return; if (gaudi->hw_cap_initialized & HW_CAP_SRAM_SCRAMBLER) @@ -1951,12 +2009,11 @@ static void gaudi_init_scrambler_hbm(struct hl_device *hdev) { struct gaudi_device *gaudi = hdev->asic_specific; - if (!hdev->asic_prop.fw_security_disabled) + if (hdev->asic_prop.fw_security_enabled) return; - if (hdev->asic_prop.fw_security_status_valid && - (hdev->asic_prop.fw_boot_cpu_security_map & - CPU_BOOT_DEV_STS0_DRAM_SCR_EN)) + if (hdev->asic_prop.fw_bootfit_cpu_boot_dev_sts0 & + CPU_BOOT_DEV_STS0_DRAM_SCR_EN) return; if (gaudi->hw_cap_initialized & HW_CAP_HBM_SCRAMBLER) @@ -2021,12 +2078,11 @@ static void gaudi_init_scrambler_hbm(struct hl_device *hdev) static void gaudi_init_e2e(struct hl_device *hdev) { - if (!hdev->asic_prop.fw_security_disabled) + if (hdev->asic_prop.fw_security_enabled) return; - if (hdev->asic_prop.fw_security_status_valid && - (hdev->asic_prop.fw_boot_cpu_security_map & - CPU_BOOT_DEV_STS0_E2E_CRED_EN)) + if (hdev->asic_prop.fw_bootfit_cpu_boot_dev_sts0 & + CPU_BOOT_DEV_STS0_E2E_CRED_EN) return; WREG32(mmSIF_RTR_CTRL_0_E2E_HBM_WR_SIZE, 247 >> 3); @@ -2396,12 +2452,11 @@ static void gaudi_init_hbm_cred(struct hl_device *hdev) { uint32_t hbm0_wr, hbm1_wr, hbm0_rd, hbm1_rd; - if (!hdev->asic_prop.fw_security_disabled) + if (hdev->asic_prop.fw_security_enabled) return; - if (hdev->asic_prop.fw_security_status_valid && - (hdev->asic_prop.fw_boot_cpu_security_map & - CPU_BOOT_DEV_STS0_HBM_CRED_EN)) + if (hdev->asic_prop.fw_bootfit_cpu_boot_dev_sts0 & + CPU_BOOT_DEV_STS0_HBM_CRED_EN) return; hbm0_wr = 0x33333333; @@ -2487,10 +2542,12 @@ static void gaudi_init_golden_registers(struct hl_device *hdev) static void gaudi_init_pci_dma_qman(struct hl_device *hdev, int dma_id, int qman_id, dma_addr_t qman_pq_addr) { + struct cpu_dyn_regs *dyn_regs = + &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs; u32 mtr_base_en_lo, mtr_base_en_hi, mtr_base_ws_lo, mtr_base_ws_hi; u32 so_base_en_lo, so_base_en_hi, so_base_ws_lo, so_base_ws_hi; u32 q_off, dma_qm_offset; - u32 dma_qm_err_cfg; + u32 dma_qm_err_cfg, irq_handler_offset; dma_qm_offset = dma_id * DMA_QMAN_OFFSET; @@ -2539,20 +2596,23 @@ static void gaudi_init_pci_dma_qman(struct hl_device *hdev, int dma_id, /* The following configuration is needed only once per QMAN */ if (qman_id == 0) { + irq_handler_offset = hdev->asic_prop.gic_interrupts_enable ? + mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR : + le32_to_cpu(dyn_regs->gic_dma_qm_irq_ctrl); + /* Configure RAZWI IRQ */ dma_qm_err_cfg = PCI_DMA_QMAN_GLBL_ERR_CFG_MSG_EN_MASK; - if (hdev->stop_on_err) { + if (hdev->stop_on_err) dma_qm_err_cfg |= PCI_DMA_QMAN_GLBL_ERR_CFG_STOP_ON_ERR_EN_MASK; - } WREG32(mmDMA0_QM_GLBL_ERR_CFG + dma_qm_offset, dma_qm_err_cfg); + WREG32(mmDMA0_QM_GLBL_ERR_ADDR_LO + dma_qm_offset, - lower_32_bits(CFG_BASE + - mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR)); + lower_32_bits(CFG_BASE + irq_handler_offset)); WREG32(mmDMA0_QM_GLBL_ERR_ADDR_HI + dma_qm_offset, - upper_32_bits(CFG_BASE + - mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR)); + upper_32_bits(CFG_BASE + irq_handler_offset)); + WREG32(mmDMA0_QM_GLBL_ERR_WDATA + dma_qm_offset, gaudi_irq_map_table[GAUDI_EVENT_DMA0_QM].cpu_id + dma_id); @@ -2573,8 +2633,11 @@ static void gaudi_init_pci_dma_qman(struct hl_device *hdev, int dma_id, static void gaudi_init_dma_core(struct hl_device *hdev, int dma_id) { - u32 dma_offset = dma_id * DMA_CORE_OFFSET; + struct cpu_dyn_regs *dyn_regs = + &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs; u32 dma_err_cfg = 1 << DMA0_CORE_ERR_CFG_ERR_MSG_EN_SHIFT; + u32 dma_offset = dma_id * DMA_CORE_OFFSET; + u32 irq_handler_offset; /* Set to maximum possible according to physical size */ WREG32(mmDMA0_CORE_RD_MAX_OUTSTAND + dma_offset, 0); @@ -2588,10 +2651,16 @@ static void gaudi_init_dma_core(struct hl_device *hdev, int dma_id) dma_err_cfg |= 1 << DMA0_CORE_ERR_CFG_STOP_ON_ERR_SHIFT; WREG32(mmDMA0_CORE_ERR_CFG + dma_offset, dma_err_cfg); + + irq_handler_offset = hdev->asic_prop.gic_interrupts_enable ? + mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR : + le32_to_cpu(dyn_regs->gic_dma_core_irq_ctrl); + WREG32(mmDMA0_CORE_ERRMSG_ADDR_LO + dma_offset, - lower_32_bits(CFG_BASE + mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR)); + lower_32_bits(CFG_BASE + irq_handler_offset)); WREG32(mmDMA0_CORE_ERRMSG_ADDR_HI + dma_offset, - upper_32_bits(CFG_BASE + mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR)); + upper_32_bits(CFG_BASE + irq_handler_offset)); + WREG32(mmDMA0_CORE_ERRMSG_WDATA + dma_offset, gaudi_irq_map_table[GAUDI_EVENT_DMA0_CORE].cpu_id + dma_id); WREG32(mmDMA0_CORE_PROT + dma_offset, @@ -2654,10 +2723,12 @@ static void gaudi_init_pci_dma_qmans(struct hl_device *hdev) static void gaudi_init_hbm_dma_qman(struct hl_device *hdev, int dma_id, int qman_id, u64 qman_base_addr) { + struct cpu_dyn_regs *dyn_regs = + &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs; u32 mtr_base_en_lo, mtr_base_en_hi, mtr_base_ws_lo, mtr_base_ws_hi; u32 so_base_en_lo, so_base_en_hi, so_base_ws_lo, so_base_ws_hi; + u32 dma_qm_err_cfg, irq_handler_offset; u32 q_off, dma_qm_offset; - u32 dma_qm_err_cfg; dma_qm_offset = dma_id * DMA_QMAN_OFFSET; @@ -2697,6 +2768,10 @@ static void gaudi_init_hbm_dma_qman(struct hl_device *hdev, int dma_id, WREG32(mmDMA0_QM_CP_LDMA_DST_BASE_LO_OFFSET_0 + q_off, QMAN_CPDMA_DST_OFFSET); } else { + irq_handler_offset = hdev->asic_prop.gic_interrupts_enable ? + mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR : + le32_to_cpu(dyn_regs->gic_dma_qm_irq_ctrl); + WREG32(mmDMA0_QM_CP_LDMA_TSIZE_OFFSET_0 + q_off, QMAN_LDMA_SIZE_OFFSET); WREG32(mmDMA0_QM_CP_LDMA_SRC_BASE_LO_OFFSET_0 + q_off, @@ -2706,18 +2781,17 @@ static void gaudi_init_hbm_dma_qman(struct hl_device *hdev, int dma_id, /* Configure RAZWI IRQ */ dma_qm_err_cfg = HBM_DMA_QMAN_GLBL_ERR_CFG_MSG_EN_MASK; - if (hdev->stop_on_err) { + if (hdev->stop_on_err) dma_qm_err_cfg |= HBM_DMA_QMAN_GLBL_ERR_CFG_STOP_ON_ERR_EN_MASK; - } + WREG32(mmDMA0_QM_GLBL_ERR_CFG + dma_qm_offset, dma_qm_err_cfg); WREG32(mmDMA0_QM_GLBL_ERR_ADDR_LO + dma_qm_offset, - lower_32_bits(CFG_BASE + - mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR)); + lower_32_bits(CFG_BASE + irq_handler_offset)); WREG32(mmDMA0_QM_GLBL_ERR_ADDR_HI + dma_qm_offset, - upper_32_bits(CFG_BASE + - mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR)); + upper_32_bits(CFG_BASE + irq_handler_offset)); + WREG32(mmDMA0_QM_GLBL_ERR_WDATA + dma_qm_offset, gaudi_irq_map_table[GAUDI_EVENT_DMA0_QM].cpu_id + dma_id); @@ -2792,8 +2866,11 @@ static void gaudi_init_hbm_dma_qmans(struct hl_device *hdev) static void gaudi_init_mme_qman(struct hl_device *hdev, u32 mme_offset, int qman_id, u64 qman_base_addr) { + struct cpu_dyn_regs *dyn_regs = + &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs; u32 mtr_base_lo, mtr_base_hi; u32 so_base_lo, so_base_hi; + u32 irq_handler_offset; u32 q_off, mme_id; u32 mme_qm_err_cfg; @@ -2825,6 +2902,10 @@ static void gaudi_init_mme_qman(struct hl_device *hdev, u32 mme_offset, WREG32(mmMME0_QM_CP_LDMA_DST_BASE_LO_OFFSET_0 + q_off, QMAN_CPDMA_DST_OFFSET); } else { + irq_handler_offset = hdev->asic_prop.gic_interrupts_enable ? + mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR : + le32_to_cpu(dyn_regs->gic_mme_qm_irq_ctrl); + WREG32(mmMME0_QM_CP_LDMA_TSIZE_OFFSET_0 + q_off, QMAN_LDMA_SIZE_OFFSET); WREG32(mmMME0_QM_CP_LDMA_SRC_BASE_LO_OFFSET_0 + q_off, @@ -2834,20 +2915,20 @@ static void gaudi_init_mme_qman(struct hl_device *hdev, u32 mme_offset, /* Configure RAZWI IRQ */ mme_id = mme_offset / - (mmMME1_QM_GLBL_CFG0 - mmMME0_QM_GLBL_CFG0); + (mmMME1_QM_GLBL_CFG0 - mmMME0_QM_GLBL_CFG0) / 2; mme_qm_err_cfg = MME_QMAN_GLBL_ERR_CFG_MSG_EN_MASK; - if (hdev->stop_on_err) { + if (hdev->stop_on_err) mme_qm_err_cfg |= MME_QMAN_GLBL_ERR_CFG_STOP_ON_ERR_EN_MASK; - } + WREG32(mmMME0_QM_GLBL_ERR_CFG + mme_offset, mme_qm_err_cfg); + WREG32(mmMME0_QM_GLBL_ERR_ADDR_LO + mme_offset, - lower_32_bits(CFG_BASE + - mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR)); + lower_32_bits(CFG_BASE + irq_handler_offset)); WREG32(mmMME0_QM_GLBL_ERR_ADDR_HI + mme_offset, - upper_32_bits(CFG_BASE + - mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR)); + upper_32_bits(CFG_BASE + irq_handler_offset)); + WREG32(mmMME0_QM_GLBL_ERR_WDATA + mme_offset, gaudi_irq_map_table[GAUDI_EVENT_MME0_QM].cpu_id + mme_id); @@ -2912,10 +2993,12 @@ static void gaudi_init_mme_qmans(struct hl_device *hdev) static void gaudi_init_tpc_qman(struct hl_device *hdev, u32 tpc_offset, int qman_id, u64 qman_base_addr) { + struct cpu_dyn_regs *dyn_regs = + &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs; u32 mtr_base_en_lo, mtr_base_en_hi, mtr_base_ws_lo, mtr_base_ws_hi; u32 so_base_en_lo, so_base_en_hi, so_base_ws_lo, so_base_ws_hi; + u32 tpc_qm_err_cfg, irq_handler_offset; u32 q_off, tpc_id; - u32 tpc_qm_err_cfg; mtr_base_en_lo = lower_32_bits(CFG_BASE + mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0); @@ -2956,6 +3039,10 @@ static void gaudi_init_tpc_qman(struct hl_device *hdev, u32 tpc_offset, WREG32(mmTPC0_QM_CP_LDMA_DST_BASE_LO_OFFSET_0 + q_off, QMAN_CPDMA_DST_OFFSET); } else { + irq_handler_offset = hdev->asic_prop.gic_interrupts_enable ? + mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR : + le32_to_cpu(dyn_regs->gic_tpc_qm_irq_ctrl); + WREG32(mmTPC0_QM_CP_LDMA_TSIZE_OFFSET_0 + q_off, QMAN_LDMA_SIZE_OFFSET); WREG32(mmTPC0_QM_CP_LDMA_SRC_BASE_LO_OFFSET_0 + q_off, @@ -2965,18 +3052,17 @@ static void gaudi_init_tpc_qman(struct hl_device *hdev, u32 tpc_offset, /* Configure RAZWI IRQ */ tpc_qm_err_cfg = TPC_QMAN_GLBL_ERR_CFG_MSG_EN_MASK; - if (hdev->stop_on_err) { + if (hdev->stop_on_err) tpc_qm_err_cfg |= TPC_QMAN_GLBL_ERR_CFG_STOP_ON_ERR_EN_MASK; - } WREG32(mmTPC0_QM_GLBL_ERR_CFG + tpc_offset, tpc_qm_err_cfg); + WREG32(mmTPC0_QM_GLBL_ERR_ADDR_LO + tpc_offset, - lower_32_bits(CFG_BASE + - mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR)); + lower_32_bits(CFG_BASE + irq_handler_offset)); WREG32(mmTPC0_QM_GLBL_ERR_ADDR_HI + tpc_offset, - upper_32_bits(CFG_BASE + - mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR)); + upper_32_bits(CFG_BASE + irq_handler_offset)); + WREG32(mmTPC0_QM_GLBL_ERR_WDATA + tpc_offset, gaudi_irq_map_table[GAUDI_EVENT_TPC0_QM].cpu_id + tpc_id); @@ -3059,10 +3145,12 @@ static void gaudi_init_tpc_qmans(struct hl_device *hdev) static void gaudi_init_nic_qman(struct hl_device *hdev, u32 nic_offset, int qman_id, u64 qman_base_addr, int nic_id) { + struct cpu_dyn_regs *dyn_regs = + &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs; u32 mtr_base_en_lo, mtr_base_en_hi, mtr_base_ws_lo, mtr_base_ws_hi; u32 so_base_en_lo, so_base_en_hi, so_base_ws_lo, so_base_ws_hi; + u32 nic_qm_err_cfg, irq_handler_offset; u32 q_off; - u32 nic_qm_err_cfg; mtr_base_en_lo = lower_32_bits(CFG_BASE + mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0); @@ -3109,20 +3197,23 @@ static void gaudi_init_nic_qman(struct hl_device *hdev, u32 nic_offset, WREG32(mmNIC0_QM0_CP_MSG_BASE3_ADDR_HI_0 + q_off, so_base_ws_hi); if (qman_id == 0) { + irq_handler_offset = hdev->asic_prop.gic_interrupts_enable ? + mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR : + le32_to_cpu(dyn_regs->gic_nic_qm_irq_ctrl); + /* Configure RAZWI IRQ */ nic_qm_err_cfg = NIC_QMAN_GLBL_ERR_CFG_MSG_EN_MASK; - if (hdev->stop_on_err) { + if (hdev->stop_on_err) nic_qm_err_cfg |= NIC_QMAN_GLBL_ERR_CFG_STOP_ON_ERR_EN_MASK; - } WREG32(mmNIC0_QM0_GLBL_ERR_CFG + nic_offset, nic_qm_err_cfg); + WREG32(mmNIC0_QM0_GLBL_ERR_ADDR_LO + nic_offset, - lower_32_bits(CFG_BASE + - mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR)); + lower_32_bits(CFG_BASE + irq_handler_offset)); WREG32(mmNIC0_QM0_GLBL_ERR_ADDR_HI + nic_offset, - upper_32_bits(CFG_BASE + - mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR)); + upper_32_bits(CFG_BASE + irq_handler_offset)); + WREG32(mmNIC0_QM0_GLBL_ERR_WDATA + nic_offset, gaudi_irq_map_table[GAUDI_EVENT_NIC0_QM0].cpu_id + nic_id); @@ -3475,7 +3566,7 @@ static void gaudi_set_clock_gating(struct hl_device *hdev) if (hdev->in_debug) return; - if (!hdev->asic_prop.fw_security_disabled) + if (hdev->asic_prop.fw_security_enabled) return; for (i = GAUDI_PCI_DMA_1, qman_offset = 0 ; i < GAUDI_HBM_DMA_1 ; i++) { @@ -3535,7 +3626,7 @@ static void gaudi_disable_clock_gating(struct hl_device *hdev) u32 qman_offset; int i; - if (!hdev->asic_prop.fw_security_disabled) + if (hdev->asic_prop.fw_security_enabled) return; for (i = 0, qman_offset = 0 ; i < DMA_NUMBER_OF_CHANNELS ; i++) { @@ -3674,9 +3765,6 @@ static int gaudi_load_firmware_to_device(struct hl_device *hdev) { void __iomem *dst; - /* HBM scrambler must be initialized before pushing F/W to HBM */ - gaudi_init_scrambler_hbm(hdev); - dst = hdev->pcie_bar[HBM_BAR_ID] + LINUX_FW_OFFSET; return hl_fw_load_fw_to_device(hdev, GAUDI_LINUX_FW_FILE, dst, 0, 0); @@ -3691,42 +3779,71 @@ static int gaudi_load_boot_fit_to_device(struct hl_device *hdev) return hl_fw_load_fw_to_device(hdev, GAUDI_BOOT_FIT_FILE, dst, 0, 0); } -static int gaudi_read_device_fw_version(struct hl_device *hdev, - enum hl_fw_component fwc) +static void gaudi_init_dynamic_firmware_loader(struct hl_device *hdev) { - const char *name; - u32 ver_off; - char *dest; + struct dynamic_fw_load_mgr *dynamic_loader; + struct cpu_dyn_regs *dyn_regs; - switch (fwc) { - case FW_COMP_UBOOT: - ver_off = RREG32(mmUBOOT_VER_OFFSET); - dest = hdev->asic_prop.uboot_ver; - name = "U-Boot"; - break; - case FW_COMP_PREBOOT: - ver_off = RREG32(mmPREBOOT_VER_OFFSET); - dest = hdev->asic_prop.preboot_ver; - name = "Preboot"; - break; - default: - dev_warn(hdev->dev, "Undefined FW component: %d\n", fwc); - return -EIO; - } + dynamic_loader = &hdev->fw_loader.dynamic_loader; - ver_off &= ~((u32)SRAM_BASE_ADDR); + /* + * here we update initial values for few specific dynamic regs (as + * before reading the first descriptor from FW those value has to be + * hard-coded) in later stages of the protocol those values will be + * updated automatically by reading the FW descriptor so data there + * will always be up-to-date + */ + dyn_regs = &dynamic_loader->comm_desc.cpu_dyn_regs; + dyn_regs->kmd_msg_to_cpu = + cpu_to_le32(mmPSOC_GLOBAL_CONF_KMD_MSG_TO_CPU); + dyn_regs->cpu_cmd_status_to_host = + cpu_to_le32(mmCPU_CMD_STATUS_TO_HOST); - if (ver_off < SRAM_SIZE - VERSION_MAX_LEN) { - memcpy_fromio(dest, hdev->pcie_bar[SRAM_BAR_ID] + ver_off, - VERSION_MAX_LEN); - } else { - dev_err(hdev->dev, "%s version offset (0x%x) is above SRAM\n", - name, ver_off); - strcpy(dest, "unavailable"); - return -EIO; - } + dynamic_loader->wait_for_bl_timeout = GAUDI_WAIT_FOR_BL_TIMEOUT_USEC; +} - return 0; +static void gaudi_init_static_firmware_loader(struct hl_device *hdev) +{ + struct static_fw_load_mgr *static_loader; + + static_loader = &hdev->fw_loader.static_loader; + + static_loader->preboot_version_max_off = SRAM_SIZE - VERSION_MAX_LEN; + static_loader->boot_fit_version_max_off = SRAM_SIZE - VERSION_MAX_LEN; + static_loader->kmd_msg_to_cpu_reg = mmPSOC_GLOBAL_CONF_KMD_MSG_TO_CPU; + static_loader->cpu_cmd_status_to_host_reg = mmCPU_CMD_STATUS_TO_HOST; + static_loader->cpu_boot_status_reg = mmPSOC_GLOBAL_CONF_CPU_BOOT_STATUS; + static_loader->cpu_boot_dev_status0_reg = mmCPU_BOOT_DEV_STS0; + static_loader->cpu_boot_dev_status1_reg = mmCPU_BOOT_DEV_STS1; + static_loader->boot_err0_reg = mmCPU_BOOT_ERR0; + static_loader->boot_err1_reg = mmCPU_BOOT_ERR1; + static_loader->preboot_version_offset_reg = mmPREBOOT_VER_OFFSET; + static_loader->boot_fit_version_offset_reg = mmUBOOT_VER_OFFSET; + static_loader->sram_offset_mask = ~(lower_32_bits(SRAM_BASE_ADDR)); + static_loader->cpu_reset_wait_msec = hdev->pldm ? + GAUDI_PLDM_RESET_WAIT_MSEC : + GAUDI_CPU_RESET_WAIT_MSEC; +} + +static void gaudi_init_firmware_loader(struct hl_device *hdev) +{ + struct asic_fixed_properties *prop = &hdev->asic_prop; + struct fw_load_mgr *fw_loader = &hdev->fw_loader; + + /* fill common fields */ + fw_loader->linux_loaded = false; + fw_loader->boot_fit_img.image_name = GAUDI_BOOT_FIT_FILE; + fw_loader->linux_img.image_name = GAUDI_LINUX_FW_FILE; + fw_loader->cpu_timeout = GAUDI_CPU_TIMEOUT_USEC; + fw_loader->boot_fit_timeout = GAUDI_BOOT_FIT_REQ_TIMEOUT_USEC; + fw_loader->skip_bmc = !hdev->bmc_enable; + fw_loader->sram_bar_id = SRAM_BAR_ID; + fw_loader->dram_bar_id = HBM_BAR_ID; + + if (prop->dynamic_fw_load) + gaudi_init_dynamic_firmware_loader(hdev); + else + gaudi_init_static_firmware_loader(hdev); } static int gaudi_init_cpu(struct hl_device *hdev) @@ -3744,15 +3861,10 @@ static int gaudi_init_cpu(struct hl_device *hdev) * The device CPU works with 40 bits addresses. * This register sets the extension to 50 bits. */ - if (hdev->asic_prop.fw_security_disabled) + if (!hdev->asic_prop.fw_security_enabled) WREG32(mmCPU_IF_CPU_MSB_ADDR, hdev->cpu_pci_msb_addr); - rc = hl_fw_init_cpu(hdev, mmPSOC_GLOBAL_CONF_CPU_BOOT_STATUS, - mmPSOC_GLOBAL_CONF_KMD_MSG_TO_CPU, - mmCPU_CMD_STATUS_TO_HOST, - mmCPU_BOOT_DEV_STS0, mmCPU_BOOT_ERR0, - !hdev->bmc_enable, GAUDI_CPU_TIMEOUT_USEC, - GAUDI_BOOT_FIT_REQ_TIMEOUT_USEC); + rc = hl_fw_init_cpu(hdev); if (rc) return rc; @@ -3764,10 +3876,12 @@ static int gaudi_init_cpu(struct hl_device *hdev) static int gaudi_init_cpu_queues(struct hl_device *hdev, u32 cpu_timeout) { - struct gaudi_device *gaudi = hdev->asic_specific; + struct cpu_dyn_regs *dyn_regs = + &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs; struct asic_fixed_properties *prop = &hdev->asic_prop; + struct gaudi_device *gaudi = hdev->asic_specific; + u32 status, irq_handler_offset; struct hl_eq *eq; - u32 status; struct hl_hw_queue *cpu_pq = &hdev->kernel_queues[GAUDI_QUEUE_ID_CPU_PQ]; int err; @@ -3806,7 +3920,12 @@ static int gaudi_init_cpu_queues(struct hl_device *hdev, u32 cpu_timeout) WREG32(mmCPU_IF_QUEUE_INIT, PQ_INIT_STATUS_READY_FOR_CP_SINGLE_MSI); - WREG32(mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR, GAUDI_EVENT_PI_UPDATE); + irq_handler_offset = prop->gic_interrupts_enable ? + mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR : + le32_to_cpu(dyn_regs->gic_host_pi_upd_irq); + + WREG32(irq_handler_offset, + gaudi_irq_map_table[GAUDI_EVENT_PI_UPDATE].cpu_id); err = hl_poll_timeout( hdev, @@ -3823,8 +3942,10 @@ static int gaudi_init_cpu_queues(struct hl_device *hdev, u32 cpu_timeout) } /* update FW application security bits */ - if (prop->fw_security_status_valid) - prop->fw_app_security_map = RREG32(mmCPU_BOOT_DEV_STS0); + if (prop->fw_cpu_boot_dev_sts0_valid) + prop->fw_app_cpu_boot_dev_sts0 = RREG32(mmCPU_BOOT_DEV_STS0); + if (prop->fw_cpu_boot_dev_sts1_valid) + prop->fw_app_cpu_boot_dev_sts1 = RREG32(mmCPU_BOOT_DEV_STS1); gaudi->hw_cap_initialized |= HW_CAP_CPU_Q; return 0; @@ -3835,7 +3956,7 @@ static void gaudi_pre_hw_init(struct hl_device *hdev) /* Perform read from the device to make sure device is up */ RREG32(mmHW_STATE); - if (hdev->asic_prop.fw_security_disabled) { + if (!hdev->asic_prop.fw_security_enabled) { /* Set the access through PCI bars (Linux driver only) as * secured */ @@ -3860,13 +3981,27 @@ static void gaudi_pre_hw_init(struct hl_device *hdev) static int gaudi_hw_init(struct hl_device *hdev) { + struct gaudi_device *gaudi = hdev->asic_specific; int rc; gaudi_pre_hw_init(hdev); - gaudi_init_pci_dma_qmans(hdev); + /* If iATU is done by FW, the HBM bar ALWAYS points to DRAM_PHYS_BASE. + * So we set it here and if anyone tries to move it later to + * a different address, there will be an error + */ + if (hdev->asic_prop.iatu_done_by_fw) + gaudi->hbm_bar_cur_addr = DRAM_PHYS_BASE; - gaudi_init_hbm_dma_qmans(hdev); + /* + * Before pushing u-boot/linux to device, need to set the hbm bar to + * base address of dram + */ + if (gaudi_set_hbm_bar_base(hdev, DRAM_PHYS_BASE) == U64_MAX) { + dev_err(hdev->dev, + "failed to map HBM bar to DRAM base address\n"); + return -EIO; + } rc = gaudi_init_cpu(hdev); if (rc) { @@ -3895,6 +4030,10 @@ static int gaudi_hw_init(struct hl_device *hdev) gaudi_init_security(hdev); + gaudi_init_pci_dma_qmans(hdev); + + gaudi_init_hbm_dma_qmans(hdev); + gaudi_init_mme_qmans(hdev); gaudi_init_tpc_qmans(hdev); @@ -3934,8 +4073,11 @@ disable_queues: static void gaudi_hw_fini(struct hl_device *hdev, bool hard_reset) { + struct cpu_dyn_regs *dyn_regs = + &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs; + u32 status, reset_timeout_ms, cpu_timeout_ms, irq_handler_offset; struct gaudi_device *gaudi = hdev->asic_specific; - u32 status, reset_timeout_ms, cpu_timeout_ms; + bool driver_performs_reset; if (!hard_reset) { dev_err(hdev->dev, "GAUDI doesn't support soft-reset\n"); @@ -3950,26 +4092,35 @@ static void gaudi_hw_fini(struct hl_device *hdev, bool hard_reset) cpu_timeout_ms = GAUDI_CPU_RESET_WAIT_MSEC; } + driver_performs_reset = !!(!hdev->asic_prop.fw_security_enabled && + !hdev->asic_prop.hard_reset_done_by_fw); + /* Set device to handle FLR by H/W as we will put the device CPU to * halt mode */ - if (hdev->asic_prop.fw_security_disabled && - !hdev->asic_prop.hard_reset_done_by_fw) + if (driver_performs_reset) WREG32(mmPCIE_AUX_FLR_CTRL, (PCIE_AUX_FLR_CTRL_HW_CTRL_MASK | PCIE_AUX_FLR_CTRL_INT_MASK_MASK)); - /* I don't know what is the state of the CPU so make sure it is - * stopped in any means necessary + /* If linux is loaded in the device CPU we need to communicate with it + * via the GIC. Otherwise, we need to use COMMS or the MSG_TO_CPU + * registers in case of old F/Ws */ - if (hdev->asic_prop.hard_reset_done_by_fw) - WREG32(mmPSOC_GLOBAL_CONF_KMD_MSG_TO_CPU, KMD_MSG_RST_DEV); - else - WREG32(mmPSOC_GLOBAL_CONF_KMD_MSG_TO_CPU, KMD_MSG_GOTO_WFE); + if (hdev->fw_loader.linux_loaded) { + irq_handler_offset = hdev->asic_prop.gic_interrupts_enable ? + mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR : + le32_to_cpu(dyn_regs->gic_host_halt_irq); - WREG32(mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR, GAUDI_EVENT_HALT_MACHINE); + WREG32(irq_handler_offset, + gaudi_irq_map_table[GAUDI_EVENT_HALT_MACHINE].cpu_id); + } else { + if (hdev->asic_prop.hard_reset_done_by_fw) + hl_fw_ask_hard_reset_without_linux(hdev); + else + hl_fw_ask_halt_machine_without_linux(hdev); + } - if (hdev->asic_prop.fw_security_disabled && - !hdev->asic_prop.hard_reset_done_by_fw) { + if (driver_performs_reset) { /* Configure the reset registers. Must be done as early as * possible in case we fail during H/W initialization @@ -4003,8 +4154,7 @@ static void gaudi_hw_fini(struct hl_device *hdev, bool hard_reset) WREG32(mmPREBOOT_PCIE_EN, LKD_HARD_RESET_MAGIC); /* Restart BTL/BLR upon hard-reset */ - if (hdev->asic_prop.fw_security_disabled) - WREG32(mmPSOC_GLOBAL_CONF_BOOT_SEQ_RE_START, 1); + WREG32(mmPSOC_GLOBAL_CONF_BOOT_SEQ_RE_START, 1); WREG32(mmPSOC_GLOBAL_CONF_SW_ALL_RST, 1 << PSOC_GLOBAL_CONF_SW_ALL_RST_IND_SHIFT); @@ -4041,6 +4191,8 @@ static void gaudi_hw_fini(struct hl_device *hdev, bool hard_reset) HW_CAP_CLK_GATE); memset(gaudi->events_stat, 0, sizeof(gaudi->events_stat)); + + hdev->device_cpu_is_halted = false; } } @@ -4078,10 +4230,12 @@ static int gaudi_cb_mmap(struct hl_device *hdev, struct vm_area_struct *vma, static void gaudi_ring_doorbell(struct hl_device *hdev, u32 hw_queue_id, u32 pi) { + struct cpu_dyn_regs *dyn_regs = + &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs; + u32 db_reg_offset, db_value, dma_qm_offset, q_off, irq_handler_offset; struct gaudi_device *gaudi = hdev->asic_specific; - u32 db_reg_offset, db_value, dma_qm_offset, q_off; - int dma_id; bool invalid_queue = false; + int dma_id; switch (hw_queue_id) { case GAUDI_QUEUE_ID_DMA_0_0...GAUDI_QUEUE_ID_DMA_0_3: @@ -4307,164 +4461,84 @@ static void gaudi_ring_doorbell(struct hl_device *hdev, u32 hw_queue_id, u32 pi) db_reg_offset = mmTPC7_QM_PQ_PI_3; break; - case GAUDI_QUEUE_ID_NIC_0_0: - db_reg_offset = mmNIC0_QM0_PQ_PI_0; - break; - - case GAUDI_QUEUE_ID_NIC_0_1: - db_reg_offset = mmNIC0_QM0_PQ_PI_1; - break; - - case GAUDI_QUEUE_ID_NIC_0_2: - db_reg_offset = mmNIC0_QM0_PQ_PI_2; - break; - - case GAUDI_QUEUE_ID_NIC_0_3: - db_reg_offset = mmNIC0_QM0_PQ_PI_3; - break; - - case GAUDI_QUEUE_ID_NIC_1_0: - db_reg_offset = mmNIC0_QM1_PQ_PI_0; - break; - - case GAUDI_QUEUE_ID_NIC_1_1: - db_reg_offset = mmNIC0_QM1_PQ_PI_1; - break; - - case GAUDI_QUEUE_ID_NIC_1_2: - db_reg_offset = mmNIC0_QM1_PQ_PI_2; - break; - - case GAUDI_QUEUE_ID_NIC_1_3: - db_reg_offset = mmNIC0_QM1_PQ_PI_3; - break; - - case GAUDI_QUEUE_ID_NIC_2_0: - db_reg_offset = mmNIC1_QM0_PQ_PI_0; - break; - - case GAUDI_QUEUE_ID_NIC_2_1: - db_reg_offset = mmNIC1_QM0_PQ_PI_1; - break; - - case GAUDI_QUEUE_ID_NIC_2_2: - db_reg_offset = mmNIC1_QM0_PQ_PI_2; - break; - - case GAUDI_QUEUE_ID_NIC_2_3: - db_reg_offset = mmNIC1_QM0_PQ_PI_3; - break; - - case GAUDI_QUEUE_ID_NIC_3_0: - db_reg_offset = mmNIC1_QM1_PQ_PI_0; - break; - - case GAUDI_QUEUE_ID_NIC_3_1: - db_reg_offset = mmNIC1_QM1_PQ_PI_1; - break; - - case GAUDI_QUEUE_ID_NIC_3_2: - db_reg_offset = mmNIC1_QM1_PQ_PI_2; - break; - - case GAUDI_QUEUE_ID_NIC_3_3: - db_reg_offset = mmNIC1_QM1_PQ_PI_3; - break; - - case GAUDI_QUEUE_ID_NIC_4_0: - db_reg_offset = mmNIC2_QM0_PQ_PI_0; - break; - - case GAUDI_QUEUE_ID_NIC_4_1: - db_reg_offset = mmNIC2_QM0_PQ_PI_1; - break; - - case GAUDI_QUEUE_ID_NIC_4_2: - db_reg_offset = mmNIC2_QM0_PQ_PI_2; - break; - - case GAUDI_QUEUE_ID_NIC_4_3: - db_reg_offset = mmNIC2_QM0_PQ_PI_3; - break; - - case GAUDI_QUEUE_ID_NIC_5_0: - db_reg_offset = mmNIC2_QM1_PQ_PI_0; - break; + case GAUDI_QUEUE_ID_NIC_0_0...GAUDI_QUEUE_ID_NIC_0_3: + if (!(gaudi->hw_cap_initialized & HW_CAP_NIC0)) + invalid_queue = true; - case GAUDI_QUEUE_ID_NIC_5_1: - db_reg_offset = mmNIC2_QM1_PQ_PI_1; + q_off = ((hw_queue_id - 1) & 0x3) * 4; + db_reg_offset = mmNIC0_QM0_PQ_PI_0 + q_off; break; - case GAUDI_QUEUE_ID_NIC_5_2: - db_reg_offset = mmNIC2_QM1_PQ_PI_2; - break; + case GAUDI_QUEUE_ID_NIC_1_0...GAUDI_QUEUE_ID_NIC_1_3: + if (!(gaudi->hw_cap_initialized & HW_CAP_NIC1)) + invalid_queue = true; - case GAUDI_QUEUE_ID_NIC_5_3: - db_reg_offset = mmNIC2_QM1_PQ_PI_3; + q_off = ((hw_queue_id - 1) & 0x3) * 4; + db_reg_offset = mmNIC0_QM1_PQ_PI_0 + q_off; break; - case GAUDI_QUEUE_ID_NIC_6_0: - db_reg_offset = mmNIC3_QM0_PQ_PI_0; - break; + case GAUDI_QUEUE_ID_NIC_2_0...GAUDI_QUEUE_ID_NIC_2_3: + if (!(gaudi->hw_cap_initialized & HW_CAP_NIC2)) + invalid_queue = true; - case GAUDI_QUEUE_ID_NIC_6_1: - db_reg_offset = mmNIC3_QM0_PQ_PI_1; + q_off = ((hw_queue_id - 1) & 0x3) * 4; + db_reg_offset = mmNIC1_QM0_PQ_PI_0 + q_off; break; - case GAUDI_QUEUE_ID_NIC_6_2: - db_reg_offset = mmNIC3_QM0_PQ_PI_2; - break; + case GAUDI_QUEUE_ID_NIC_3_0...GAUDI_QUEUE_ID_NIC_3_3: + if (!(gaudi->hw_cap_initialized & HW_CAP_NIC3)) + invalid_queue = true; - case GAUDI_QUEUE_ID_NIC_6_3: - db_reg_offset = mmNIC3_QM0_PQ_PI_3; + q_off = ((hw_queue_id - 1) & 0x3) * 4; + db_reg_offset = mmNIC1_QM1_PQ_PI_0 + q_off; break; - case GAUDI_QUEUE_ID_NIC_7_0: - db_reg_offset = mmNIC3_QM1_PQ_PI_0; - break; + case GAUDI_QUEUE_ID_NIC_4_0...GAUDI_QUEUE_ID_NIC_4_3: + if (!(gaudi->hw_cap_initialized & HW_CAP_NIC4)) + invalid_queue = true; - case GAUDI_QUEUE_ID_NIC_7_1: - db_reg_offset = mmNIC3_QM1_PQ_PI_1; + q_off = ((hw_queue_id - 1) & 0x3) * 4; + db_reg_offset = mmNIC2_QM0_PQ_PI_0 + q_off; break; - case GAUDI_QUEUE_ID_NIC_7_2: - db_reg_offset = mmNIC3_QM1_PQ_PI_2; - break; + case GAUDI_QUEUE_ID_NIC_5_0...GAUDI_QUEUE_ID_NIC_5_3: + if (!(gaudi->hw_cap_initialized & HW_CAP_NIC5)) + invalid_queue = true; - case GAUDI_QUEUE_ID_NIC_7_3: - db_reg_offset = mmNIC3_QM1_PQ_PI_3; + q_off = ((hw_queue_id - 1) & 0x3) * 4; + db_reg_offset = mmNIC2_QM1_PQ_PI_0 + q_off; break; - case GAUDI_QUEUE_ID_NIC_8_0: - db_reg_offset = mmNIC4_QM0_PQ_PI_0; - break; + case GAUDI_QUEUE_ID_NIC_6_0...GAUDI_QUEUE_ID_NIC_6_3: + if (!(gaudi->hw_cap_initialized & HW_CAP_NIC6)) + invalid_queue = true; - case GAUDI_QUEUE_ID_NIC_8_1: - db_reg_offset = mmNIC4_QM0_PQ_PI_1; + q_off = ((hw_queue_id - 1) & 0x3) * 4; + db_reg_offset = mmNIC3_QM0_PQ_PI_0 + q_off; break; - case GAUDI_QUEUE_ID_NIC_8_2: - db_reg_offset = mmNIC4_QM0_PQ_PI_2; - break; + case GAUDI_QUEUE_ID_NIC_7_0...GAUDI_QUEUE_ID_NIC_7_3: + if (!(gaudi->hw_cap_initialized & HW_CAP_NIC7)) + invalid_queue = true; - case GAUDI_QUEUE_ID_NIC_8_3: - db_reg_offset = mmNIC4_QM0_PQ_PI_3; + q_off = ((hw_queue_id - 1) & 0x3) * 4; + db_reg_offset = mmNIC3_QM1_PQ_PI_0 + q_off; break; - case GAUDI_QUEUE_ID_NIC_9_0: - db_reg_offset = mmNIC4_QM1_PQ_PI_0; - break; + case GAUDI_QUEUE_ID_NIC_8_0...GAUDI_QUEUE_ID_NIC_8_3: + if (!(gaudi->hw_cap_initialized & HW_CAP_NIC8)) + invalid_queue = true; - case GAUDI_QUEUE_ID_NIC_9_1: - db_reg_offset = mmNIC4_QM1_PQ_PI_1; + q_off = ((hw_queue_id - 1) & 0x3) * 4; + db_reg_offset = mmNIC4_QM0_PQ_PI_0 + q_off; break; - case GAUDI_QUEUE_ID_NIC_9_2: - db_reg_offset = mmNIC4_QM1_PQ_PI_2; - break; + case GAUDI_QUEUE_ID_NIC_9_0...GAUDI_QUEUE_ID_NIC_9_3: + if (!(gaudi->hw_cap_initialized & HW_CAP_NIC9)) + invalid_queue = true; - case GAUDI_QUEUE_ID_NIC_9_3: - db_reg_offset = mmNIC4_QM1_PQ_PI_3; + q_off = ((hw_queue_id - 1) & 0x3) * 4; + db_reg_offset = mmNIC4_QM1_PQ_PI_0 + q_off; break; default: @@ -4486,8 +4560,13 @@ static void gaudi_ring_doorbell(struct hl_device *hdev, u32 hw_queue_id, u32 pi) if (hw_queue_id == GAUDI_QUEUE_ID_CPU_PQ) { /* make sure device CPU will read latest data from host */ mb(); - WREG32(mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR, - GAUDI_EVENT_PI_UPDATE); + + irq_handler_offset = hdev->asic_prop.gic_interrupts_enable ? + mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR : + le32_to_cpu(dyn_regs->gic_host_pi_upd_irq); + + WREG32(irq_handler_offset, + gaudi_irq_map_table[GAUDI_EVENT_PI_UPDATE].cpu_id); } } @@ -4934,6 +5013,7 @@ already_pinned: return 0; unpin_memory: + list_del(&userptr->job_node); hl_unpin_host_memory(hdev, userptr); free_userptr: kfree(userptr); @@ -6513,7 +6593,7 @@ static void gaudi_mmu_prepare(struct hl_device *hdev, u32 asid) gaudi_mmu_prepare_reg(hdev, mmMME2_ACC_WBC, asid); gaudi_mmu_prepare_reg(hdev, mmMME3_ACC_WBC, asid); - if (hdev->nic_ports_mask & GAUDI_NIC_MASK_NIC0) { + if (gaudi->hw_cap_initialized & HW_CAP_NIC0) { gaudi_mmu_prepare_reg(hdev, mmNIC0_QM0_GLBL_NON_SECURE_PROPS_0, asid); gaudi_mmu_prepare_reg(hdev, mmNIC0_QM0_GLBL_NON_SECURE_PROPS_1, @@ -6526,7 +6606,7 @@ static void gaudi_mmu_prepare(struct hl_device *hdev, u32 asid) asid); } - if (hdev->nic_ports_mask & GAUDI_NIC_MASK_NIC1) { + if (gaudi->hw_cap_initialized & HW_CAP_NIC1) { gaudi_mmu_prepare_reg(hdev, mmNIC0_QM1_GLBL_NON_SECURE_PROPS_0, asid); gaudi_mmu_prepare_reg(hdev, mmNIC0_QM1_GLBL_NON_SECURE_PROPS_1, @@ -6539,7 +6619,7 @@ static void gaudi_mmu_prepare(struct hl_device *hdev, u32 asid) asid); } - if (hdev->nic_ports_mask & GAUDI_NIC_MASK_NIC2) { + if (gaudi->hw_cap_initialized & HW_CAP_NIC2) { gaudi_mmu_prepare_reg(hdev, mmNIC1_QM0_GLBL_NON_SECURE_PROPS_0, asid); gaudi_mmu_prepare_reg(hdev, mmNIC1_QM0_GLBL_NON_SECURE_PROPS_1, @@ -6552,7 +6632,7 @@ static void gaudi_mmu_prepare(struct hl_device *hdev, u32 asid) asid); } - if (hdev->nic_ports_mask & GAUDI_NIC_MASK_NIC3) { + if (gaudi->hw_cap_initialized & HW_CAP_NIC3) { gaudi_mmu_prepare_reg(hdev, mmNIC1_QM1_GLBL_NON_SECURE_PROPS_0, asid); gaudi_mmu_prepare_reg(hdev, mmNIC1_QM1_GLBL_NON_SECURE_PROPS_1, @@ -6565,7 +6645,7 @@ static void gaudi_mmu_prepare(struct hl_device *hdev, u32 asid) asid); } - if (hdev->nic_ports_mask & GAUDI_NIC_MASK_NIC4) { + if (gaudi->hw_cap_initialized & HW_CAP_NIC4) { gaudi_mmu_prepare_reg(hdev, mmNIC2_QM0_GLBL_NON_SECURE_PROPS_0, asid); gaudi_mmu_prepare_reg(hdev, mmNIC2_QM0_GLBL_NON_SECURE_PROPS_1, @@ -6578,7 +6658,7 @@ static void gaudi_mmu_prepare(struct hl_device *hdev, u32 asid) asid); } - if (hdev->nic_ports_mask & GAUDI_NIC_MASK_NIC5) { + if (gaudi->hw_cap_initialized & HW_CAP_NIC5) { gaudi_mmu_prepare_reg(hdev, mmNIC2_QM1_GLBL_NON_SECURE_PROPS_0, asid); gaudi_mmu_prepare_reg(hdev, mmNIC2_QM1_GLBL_NON_SECURE_PROPS_1, @@ -6591,7 +6671,7 @@ static void gaudi_mmu_prepare(struct hl_device *hdev, u32 asid) asid); } - if (hdev->nic_ports_mask & GAUDI_NIC_MASK_NIC6) { + if (gaudi->hw_cap_initialized & HW_CAP_NIC6) { gaudi_mmu_prepare_reg(hdev, mmNIC3_QM0_GLBL_NON_SECURE_PROPS_0, asid); gaudi_mmu_prepare_reg(hdev, mmNIC3_QM0_GLBL_NON_SECURE_PROPS_1, @@ -6604,7 +6684,7 @@ static void gaudi_mmu_prepare(struct hl_device *hdev, u32 asid) asid); } - if (hdev->nic_ports_mask & GAUDI_NIC_MASK_NIC7) { + if (gaudi->hw_cap_initialized & HW_CAP_NIC7) { gaudi_mmu_prepare_reg(hdev, mmNIC3_QM1_GLBL_NON_SECURE_PROPS_0, asid); gaudi_mmu_prepare_reg(hdev, mmNIC3_QM1_GLBL_NON_SECURE_PROPS_1, @@ -6617,7 +6697,7 @@ static void gaudi_mmu_prepare(struct hl_device *hdev, u32 asid) asid); } - if (hdev->nic_ports_mask & GAUDI_NIC_MASK_NIC8) { + if (gaudi->hw_cap_initialized & HW_CAP_NIC8) { gaudi_mmu_prepare_reg(hdev, mmNIC4_QM0_GLBL_NON_SECURE_PROPS_0, asid); gaudi_mmu_prepare_reg(hdev, mmNIC4_QM0_GLBL_NON_SECURE_PROPS_1, @@ -6630,7 +6710,7 @@ static void gaudi_mmu_prepare(struct hl_device *hdev, u32 asid) asid); } - if (hdev->nic_ports_mask & GAUDI_NIC_MASK_NIC9) { + if (gaudi->hw_cap_initialized & HW_CAP_NIC9) { gaudi_mmu_prepare_reg(hdev, mmNIC4_QM1_GLBL_NON_SECURE_PROPS_0, asid); gaudi_mmu_prepare_reg(hdev, mmNIC4_QM1_GLBL_NON_SECURE_PROPS_1, @@ -7044,14 +7124,158 @@ enable_clk_gate: return rc; } +/* + * gaudi_queue_idx_dec - decrement queue index (pi/ci) and handle wrap + * + * @idx: the current pi/ci value + * @q_len: the queue length (power of 2) + * + * @return the cyclically decremented index + */ +static inline u32 gaudi_queue_idx_dec(u32 idx, u32 q_len) +{ + u32 mask = q_len - 1; + + /* + * modular decrement is equivalent to adding (queue_size -1) + * later we take LSBs to make sure the value is in the + * range [0, queue_len - 1] + */ + return (idx + q_len - 1) & mask; +} + +/** + * gaudi_print_sw_config_stream_data - print SW config stream data + * + * @hdev: pointer to the habanalabs device structure + * @stream: the QMAN's stream + * @qman_base: base address of QMAN registers block + */ +static void gaudi_print_sw_config_stream_data(struct hl_device *hdev, u32 stream, + u64 qman_base) +{ + u64 cq_ptr_lo, cq_ptr_hi, cq_tsize, cq_ptr; + u32 cq_ptr_lo_off, size; + + cq_ptr_lo_off = mmTPC0_QM_CQ_PTR_LO_1 - mmTPC0_QM_CQ_PTR_LO_0; + + cq_ptr_lo = qman_base + (mmTPC0_QM_CQ_PTR_LO_0 - mmTPC0_QM_BASE) + + stream * cq_ptr_lo_off; + cq_ptr_hi = cq_ptr_lo + + (mmTPC0_QM_CQ_PTR_HI_0 - mmTPC0_QM_CQ_PTR_LO_0); + cq_tsize = cq_ptr_lo + + (mmTPC0_QM_CQ_TSIZE_0 - mmTPC0_QM_CQ_PTR_LO_0); + + cq_ptr = (((u64) RREG32(cq_ptr_hi)) << 32) | RREG32(cq_ptr_lo); + size = RREG32(cq_tsize); + dev_info(hdev->dev, "stop on err: stream: %u, addr: %#llx, size: %x\n", + stream, cq_ptr, size); +} + +/** + * gaudi_print_last_pqes_on_err - print last PQEs on error + * + * @hdev: pointer to the habanalabs device structure + * @qid_base: first QID of the QMAN (out of 4 streams) + * @stream: the QMAN's stream + * @qman_base: base address of QMAN registers block + * @pr_sw_conf: if true print the SW config stream data (CQ PTR and SIZE) + */ +static void gaudi_print_last_pqes_on_err(struct hl_device *hdev, u32 qid_base, + u32 stream, u64 qman_base, + bool pr_sw_conf) +{ + u32 ci, qm_ci_stream_off, queue_len; + struct hl_hw_queue *q; + u64 pq_ci; + int i; + + q = &hdev->kernel_queues[qid_base + stream]; + + qm_ci_stream_off = mmTPC0_QM_PQ_CI_1 - mmTPC0_QM_PQ_CI_0; + pq_ci = qman_base + (mmTPC0_QM_PQ_CI_0 - mmTPC0_QM_BASE) + + stream * qm_ci_stream_off; + + queue_len = (q->queue_type == QUEUE_TYPE_INT) ? + q->int_queue_len : HL_QUEUE_LENGTH; + + hdev->asic_funcs->hw_queues_lock(hdev); + + if (pr_sw_conf) + gaudi_print_sw_config_stream_data(hdev, stream, qman_base); + + ci = RREG32(pq_ci); + + /* we should start printing form ci -1 */ + ci = gaudi_queue_idx_dec(ci, queue_len); + + for (i = 0; i < PQ_FETCHER_CACHE_SIZE; i++) { + struct hl_bd *bd; + u64 addr; + u32 len; + + bd = q->kernel_address; + bd += ci; + + len = le32_to_cpu(bd->len); + /* len 0 means uninitialized entry- break */ + if (!len) + break; + + addr = le64_to_cpu(bd->ptr); + + dev_info(hdev->dev, "stop on err PQE(stream %u): ci: %u, addr: %#llx, size: %x\n", + stream, ci, addr, len); + + /* get previous ci, wrap if needed */ + ci = gaudi_queue_idx_dec(ci, queue_len); + } + + hdev->asic_funcs->hw_queues_unlock(hdev); +} + +/** + * print_qman_data_on_err - extract QMAN data on error + * + * @hdev: pointer to the habanalabs device structure + * @qid_base: first QID of the QMAN (out of 4 streams) + * @stream: the QMAN's stream + * @qman_base: base address of QMAN registers block + * + * This function attempt to exatract as much data as possible on QMAN error. + * On upper CP print the SW config stream data and last 8 PQEs. + * On lower CP print SW config data and last PQEs of ALL 4 upper CPs + */ +static void print_qman_data_on_err(struct hl_device *hdev, u32 qid_base, + u32 stream, u64 qman_base) +{ + u32 i; + + if (stream != QMAN_STREAMS) { + gaudi_print_last_pqes_on_err(hdev, qid_base, stream, qman_base, + true); + return; + } + + gaudi_print_sw_config_stream_data(hdev, stream, qman_base); + + for (i = 0; i < QMAN_STREAMS; i++) + gaudi_print_last_pqes_on_err(hdev, qid_base, i, qman_base, + false); +} + static void gaudi_handle_qman_err_generic(struct hl_device *hdev, const char *qm_name, - u64 glbl_sts_addr, - u64 arb_err_addr) + u64 qman_base, + u32 qid_base) { u32 i, j, glbl_sts_val, arb_err_val, glbl_sts_clr_val; + u64 glbl_sts_addr, arb_err_addr; char reg_desc[32]; + glbl_sts_addr = qman_base + (mmTPC0_QM_GLBL_STS1_0 - mmTPC0_QM_BASE); + arb_err_addr = qman_base + (mmTPC0_QM_ARB_ERR_CAUSE - mmTPC0_QM_BASE); + /* Iterate through all stream GLBL_STS1 registers + Lower CP */ for (i = 0 ; i < QMAN_STREAMS + 1 ; i++) { glbl_sts_clr_val = 0; @@ -7078,6 +7302,8 @@ static void gaudi_handle_qman_err_generic(struct hl_device *hdev, /* Write 1 clear errors */ if (!hdev->stop_on_err) WREG32(glbl_sts_addr + 4 * i, glbl_sts_clr_val); + else + print_qman_data_on_err(hdev, qid_base, i, qman_base); } arb_err_val = RREG32(arb_err_addr); @@ -7222,90 +7448,88 @@ static void gaudi_handle_ecc_event(struct hl_device *hdev, u16 event_type, static void gaudi_handle_qman_err(struct hl_device *hdev, u16 event_type) { - u64 glbl_sts_addr, arb_err_addr; - u8 index; + u64 qman_base; char desc[32]; + u32 qid_base; + u8 index; switch (event_type) { case GAUDI_EVENT_TPC0_QM ... GAUDI_EVENT_TPC7_QM: index = event_type - GAUDI_EVENT_TPC0_QM; - glbl_sts_addr = - mmTPC0_QM_GLBL_STS1_0 + index * TPC_QMAN_OFFSET; - arb_err_addr = - mmTPC0_QM_ARB_ERR_CAUSE + index * TPC_QMAN_OFFSET; + qid_base = GAUDI_QUEUE_ID_TPC_0_0 + index * QMAN_STREAMS; + qman_base = mmTPC0_QM_BASE + index * TPC_QMAN_OFFSET; snprintf(desc, ARRAY_SIZE(desc), "%s%d", "TPC_QM", index); break; case GAUDI_EVENT_MME0_QM ... GAUDI_EVENT_MME2_QM: index = event_type - GAUDI_EVENT_MME0_QM; - glbl_sts_addr = - mmMME0_QM_GLBL_STS1_0 + index * MME_QMAN_OFFSET; - arb_err_addr = - mmMME0_QM_ARB_ERR_CAUSE + index * MME_QMAN_OFFSET; + qid_base = GAUDI_QUEUE_ID_MME_0_0 + index * QMAN_STREAMS; + qman_base = mmMME0_QM_BASE + index * MME_QMAN_OFFSET; snprintf(desc, ARRAY_SIZE(desc), "%s%d", "MME_QM", index); break; case GAUDI_EVENT_DMA0_QM ... GAUDI_EVENT_DMA7_QM: index = event_type - GAUDI_EVENT_DMA0_QM; - glbl_sts_addr = - mmDMA0_QM_GLBL_STS1_0 + index * DMA_QMAN_OFFSET; - arb_err_addr = - mmDMA0_QM_ARB_ERR_CAUSE + index * DMA_QMAN_OFFSET; + qid_base = GAUDI_QUEUE_ID_DMA_0_0 + index * QMAN_STREAMS; + /* skip GAUDI_QUEUE_ID_CPU_PQ if necessary */ + if (index > 1) + qid_base++; + qman_base = mmDMA0_QM_BASE + index * DMA_QMAN_OFFSET; snprintf(desc, ARRAY_SIZE(desc), "%s%d", "DMA_QM", index); break; case GAUDI_EVENT_NIC0_QM0: - glbl_sts_addr = mmNIC0_QM0_GLBL_STS1_0; - arb_err_addr = mmNIC0_QM0_ARB_ERR_CAUSE; + qid_base = GAUDI_QUEUE_ID_NIC_0_0; + qman_base = mmNIC0_QM0_BASE; snprintf(desc, ARRAY_SIZE(desc), "NIC0_QM0"); break; case GAUDI_EVENT_NIC0_QM1: - glbl_sts_addr = mmNIC0_QM1_GLBL_STS1_0; - arb_err_addr = mmNIC0_QM1_ARB_ERR_CAUSE; + qid_base = GAUDI_QUEUE_ID_NIC_1_0; + qman_base = mmNIC0_QM1_BASE; snprintf(desc, ARRAY_SIZE(desc), "NIC0_QM1"); break; case GAUDI_EVENT_NIC1_QM0: - glbl_sts_addr = mmNIC1_QM0_GLBL_STS1_0; - arb_err_addr = mmNIC1_QM0_ARB_ERR_CAUSE; + qid_base = GAUDI_QUEUE_ID_NIC_2_0; + qman_base = mmNIC1_QM0_BASE; snprintf(desc, ARRAY_SIZE(desc), "NIC1_QM0"); break; case GAUDI_EVENT_NIC1_QM1: - glbl_sts_addr = mmNIC1_QM1_GLBL_STS1_0; - arb_err_addr = mmNIC1_QM1_ARB_ERR_CAUSE; + qid_base = GAUDI_QUEUE_ID_NIC_3_0; + qman_base = mmNIC1_QM1_BASE; snprintf(desc, ARRAY_SIZE(desc), "NIC1_QM1"); break; case GAUDI_EVENT_NIC2_QM0: - glbl_sts_addr = mmNIC2_QM0_GLBL_STS1_0; - arb_err_addr = mmNIC2_QM0_ARB_ERR_CAUSE; + qid_base = GAUDI_QUEUE_ID_NIC_4_0; + qman_base = mmNIC2_QM0_BASE; snprintf(desc, ARRAY_SIZE(desc), "NIC2_QM0"); break; case GAUDI_EVENT_NIC2_QM1: - glbl_sts_addr = mmNIC2_QM1_GLBL_STS1_0; - arb_err_addr = mmNIC2_QM1_ARB_ERR_CAUSE; + qid_base = GAUDI_QUEUE_ID_NIC_5_0; + qman_base = mmNIC2_QM1_BASE; snprintf(desc, ARRAY_SIZE(desc), "NIC2_QM1"); break; case GAUDI_EVENT_NIC3_QM0: - glbl_sts_addr = mmNIC3_QM0_GLBL_STS1_0; - arb_err_addr = mmNIC3_QM0_ARB_ERR_CAUSE; + qid_base = GAUDI_QUEUE_ID_NIC_6_0; + qman_base = mmNIC3_QM0_BASE; snprintf(desc, ARRAY_SIZE(desc), "NIC3_QM0"); break; case GAUDI_EVENT_NIC3_QM1: - glbl_sts_addr = mmNIC3_QM1_GLBL_STS1_0; - arb_err_addr = mmNIC3_QM1_ARB_ERR_CAUSE; + qid_base = GAUDI_QUEUE_ID_NIC_7_0; + qman_base = mmNIC3_QM1_BASE; snprintf(desc, ARRAY_SIZE(desc), "NIC3_QM1"); break; case GAUDI_EVENT_NIC4_QM0: - glbl_sts_addr = mmNIC4_QM0_GLBL_STS1_0; - arb_err_addr = mmNIC4_QM0_ARB_ERR_CAUSE; + qid_base = GAUDI_QUEUE_ID_NIC_8_0; + qman_base = mmNIC4_QM0_BASE; snprintf(desc, ARRAY_SIZE(desc), "NIC4_QM0"); break; case GAUDI_EVENT_NIC4_QM1: - glbl_sts_addr = mmNIC4_QM1_GLBL_STS1_0; - arb_err_addr = mmNIC4_QM1_ARB_ERR_CAUSE; + qid_base = GAUDI_QUEUE_ID_NIC_9_0; + qman_base = mmNIC4_QM1_BASE; snprintf(desc, ARRAY_SIZE(desc), "NIC4_QM1"); break; default: return; } - gaudi_handle_qman_err_generic(hdev, desc, glbl_sts_addr, arb_err_addr); + gaudi_handle_qman_err_generic(hdev, desc, qman_base, qid_base); } static void gaudi_print_irq_info(struct hl_device *hdev, u16 event_type, @@ -7332,6 +7556,16 @@ static void gaudi_print_out_of_sync_info(struct hl_device *hdev, sync_err->pi, sync_err->ci, q->pi, atomic_read(&q->ci)); } +static void gaudi_print_fw_alive_info(struct hl_device *hdev, + struct hl_eq_fw_alive *fw_alive) +{ + dev_err(hdev->dev, + "FW alive report: severity=%s, process_id=%u, thread_id=%u, uptime=%llu seconds\n", + (fw_alive->severity == FW_ALIVE_SEVERITY_MINOR) ? + "Minor" : "Critical", fw_alive->process_id, + fw_alive->thread_id, fw_alive->uptime_seconds); +} + static int gaudi_soft_reset_late_init(struct hl_device *hdev) { struct gaudi_device *gaudi = hdev->asic_specific; @@ -7346,11 +7580,10 @@ static int gaudi_hbm_read_interrupts(struct hl_device *hdev, int device, struct hl_eq_hbm_ecc_data *hbm_ecc_data) { u32 base, val, val2, wr_par, rd_par, ca_par, derr, serr, type, ch; - int err = 0; + int rc = 0; - if (hdev->asic_prop.fw_security_status_valid && - (hdev->asic_prop.fw_app_security_map & - CPU_BOOT_DEV_STS0_HBM_ECC_EN)) { + if (hdev->asic_prop.fw_app_cpu_boot_dev_sts0 & + CPU_BOOT_DEV_STS0_HBM_ECC_EN) { if (!hbm_ecc_data) { dev_err(hdev->dev, "No FW ECC data"); return 0; @@ -7379,13 +7612,10 @@ static int gaudi_hbm_read_interrupts(struct hl_device *hdev, int device, device, ch, hbm_ecc_data->first_addr, type, hbm_ecc_data->sec_cont_cnt, hbm_ecc_data->sec_cnt, hbm_ecc_data->dec_cnt); - - err = 1; - return 0; } - if (!hdev->asic_prop.fw_security_disabled) { + if (hdev->asic_prop.fw_security_enabled) { dev_info(hdev->dev, "Cannot access MC regs for ECC data while security is enabled\n"); return 0; } @@ -7395,7 +7625,7 @@ static int gaudi_hbm_read_interrupts(struct hl_device *hdev, int device, val = RREG32_MASK(base + ch * 0x1000 + 0x06C, 0x0000FFFF); val = (val & 0xFF) | ((val >> 8) & 0xFF); if (val) { - err = 1; + rc = -EIO; dev_err(hdev->dev, "HBM%d pc%d interrupts info: WR_PAR=%d, RD_PAR=%d, CA_PAR=%d, SERR=%d, DERR=%d\n", device, ch * 2, val & 0x1, (val >> 1) & 0x1, @@ -7415,7 +7645,7 @@ static int gaudi_hbm_read_interrupts(struct hl_device *hdev, int device, val = RREG32_MASK(base + ch * 0x1000 + 0x07C, 0x0000FFFF); val = (val & 0xFF) | ((val >> 8) & 0xFF); if (val) { - err = 1; + rc = -EIO; dev_err(hdev->dev, "HBM%d pc%d interrupts info: WR_PAR=%d, RD_PAR=%d, CA_PAR=%d, SERR=%d, DERR=%d\n", device, ch * 2 + 1, val & 0x1, (val >> 1) & 0x1, @@ -7444,7 +7674,7 @@ static int gaudi_hbm_read_interrupts(struct hl_device *hdev, int device, val = RREG32(base + 0x8F30); val2 = RREG32(base + 0x8F34); if (val | val2) { - err = 1; + rc = -EIO; dev_err(hdev->dev, "HBM %d MC SRAM SERR info: Reg 0x8F30=0x%x, Reg 0x8F34=0x%x\n", device, val, val2); @@ -7452,13 +7682,13 @@ static int gaudi_hbm_read_interrupts(struct hl_device *hdev, int device, val = RREG32(base + 0x8F40); val2 = RREG32(base + 0x8F44); if (val | val2) { - err = 1; + rc = -EIO; dev_err(hdev->dev, "HBM %d MC SRAM DERR info: Reg 0x8F40=0x%x, Reg 0x8F44=0x%x\n", device, val, val2); } - return err; + return rc; } static int gaudi_hbm_event_to_dev(u16 hbm_event_type) @@ -7604,6 +7834,7 @@ static void gaudi_handle_eqe(struct hl_device *hdev, case GAUDI_EVENT_DMA_IF0_DERR ... GAUDI_EVENT_DMA_IF3_DERR: case GAUDI_EVENT_HBM_0_DERR ... GAUDI_EVENT_HBM_3_DERR: case GAUDI_EVENT_MMU_DERR: + case GAUDI_EVENT_NIC0_CS_DBG_DERR ... GAUDI_EVENT_NIC4_CS_DBG_DERR: gaudi_print_irq_info(hdev, event_type, true); gaudi_handle_ecc_event(hdev, event_type, &eq_entry->ecc_data); goto reset_device; @@ -7786,6 +8017,11 @@ static void gaudi_handle_eqe(struct hl_device *hdev, gaudi_print_out_of_sync_info(hdev, &eq_entry->pkt_sync_err); goto reset_device; + case GAUDI_EVENT_FW_ALIVE_S: + gaudi_print_irq_info(hdev, event_type, false); + gaudi_print_fw_alive_info(hdev, &eq_entry->fw_alive); + goto reset_device; + default: dev_err(hdev->dev, "Received invalid H/W interrupt %d\n", event_type); @@ -7856,52 +8092,13 @@ static int gaudi_mmu_invalidate_cache(struct hl_device *hdev, bool is_hard, } static int gaudi_mmu_invalidate_cache_range(struct hl_device *hdev, - bool is_hard, u32 asid, u64 va, u64 size) + bool is_hard, u32 flags, + u32 asid, u64 va, u64 size) { - struct gaudi_device *gaudi = hdev->asic_specific; - u32 status, timeout_usec; - u32 inv_data; - u32 pi; - int rc; - - if (!(gaudi->hw_cap_initialized & HW_CAP_MMU) || - hdev->hard_reset_pending) - return 0; - - if (hdev->pldm) - timeout_usec = GAUDI_PLDM_MMU_TIMEOUT_USEC; - else - timeout_usec = MMU_CONFIG_TIMEOUT_USEC; - - /* - * TODO: currently invalidate entire L0 & L1 as in regular hard - * invalidation. Need to apply invalidation of specific cache - * lines with mask of ASID & VA & size. - * Note that L1 with be flushed entirely in any case. + /* Treat as invalidate all because there is no range invalidation + * in Gaudi */ - - /* L0 & L1 invalidation */ - inv_data = RREG32(mmSTLB_CACHE_INV); - /* PI is 8 bit */ - pi = ((inv_data & STLB_CACHE_INV_PRODUCER_INDEX_MASK) + 1) & 0xFF; - WREG32(mmSTLB_CACHE_INV, - (inv_data & STLB_CACHE_INV_INDEX_MASK_MASK) | pi); - - rc = hl_poll_timeout( - hdev, - mmSTLB_INV_CONSUMER_INDEX, - status, - status == pi, - 1000, - timeout_usec); - - if (rc) { - dev_err_ratelimited(hdev->dev, - "MMU cache invalidation timeout\n"); - hl_device_reset(hdev, HL_RESET_HARD); - } - - return rc; + return hdev->asic_funcs->mmu_invalidate_cache(hdev, is_hard, flags); } static int gaudi_mmu_update_asid_hop0_addr(struct hl_device *hdev, @@ -7956,7 +8153,9 @@ static int gaudi_cpucp_info_get(struct hl_device *hdev) if (!(gaudi->hw_cap_initialized & HW_CAP_CPU_Q)) return 0; - rc = hl_fw_cpucp_handshake(hdev, mmCPU_BOOT_DEV_STS0, mmCPU_BOOT_ERR0); + rc = hl_fw_cpucp_handshake(hdev, mmCPU_BOOT_DEV_STS0, + mmCPU_BOOT_DEV_STS1, mmCPU_BOOT_ERR0, + mmCPU_BOOT_ERR1); if (rc) return rc; @@ -8077,7 +8276,7 @@ static bool gaudi_is_device_idle(struct hl_device *hdev, u64 *mask_arr, for (i = 0 ; i < (NIC_NUMBER_OF_ENGINES / 2) ; i++) { offset = i * NIC_MACRO_QMAN_OFFSET; port = 2 * i; - if (hdev->nic_ports_mask & BIT(port)) { + if (gaudi->hw_cap_initialized & BIT(HW_CAP_NIC_SHIFT + port)) { qm_glbl_sts0 = RREG32(mmNIC0_QM0_GLBL_STS0 + offset); qm_cgm_sts = RREG32(mmNIC0_QM0_CGM_STS + offset); is_eng_idle = IS_QM_IDLE(qm_glbl_sts0, qm_cgm_sts); @@ -8092,7 +8291,7 @@ static bool gaudi_is_device_idle(struct hl_device *hdev, u64 *mask_arr, } port = 2 * i + 1; - if (hdev->nic_ports_mask & BIT(port)) { + if (gaudi->hw_cap_initialized & BIT(HW_CAP_NIC_SHIFT + port)) { qm_glbl_sts0 = RREG32(mmNIC0_QM1_GLBL_STS0 + offset); qm_cgm_sts = RREG32(mmNIC0_QM1_CGM_STS + offset); is_eng_idle = IS_QM_IDLE(qm_glbl_sts0, qm_cgm_sts); @@ -8306,8 +8505,10 @@ static int gaudi_internal_cb_pool_init(struct hl_device *hdev, HL_VA_RANGE_TYPE_HOST, HOST_SPACE_INTERNAL_CB_SZ, HL_MMU_VA_ALIGNMENT_NOT_NEEDED); - if (!hdev->internal_cb_va_base) + if (!hdev->internal_cb_va_base) { + rc = -ENOMEM; goto destroy_internal_cb_pool; + } mutex_lock(&ctx->mmu_lock); rc = hl_mmu_map_contiguous(ctx, hdev->internal_cb_va_base, @@ -8749,7 +8950,14 @@ static int gaudi_block_mmap(struct hl_device *hdev, static void gaudi_enable_events_from_fw(struct hl_device *hdev) { - WREG32(mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR, GAUDI_EVENT_INTS_REGISTER); + struct cpu_dyn_regs *dyn_regs = + &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs; + u32 irq_handler_offset = hdev->asic_prop.gic_interrupts_enable ? + mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR : + le32_to_cpu(dyn_regs->gic_host_ints_irq); + + WREG32(irq_handler_offset, + gaudi_irq_map_table[GAUDI_EVENT_INTS_REGISTER].cpu_id); } static int gaudi_map_pll_idx_to_fw_idx(u32 pll_idx) @@ -8834,7 +9042,6 @@ static const struct hl_asic_funcs gaudi_funcs = { .ctx_fini = gaudi_ctx_fini, .get_clk_rate = gaudi_get_clk_rate, .get_queue_id_for_cq = gaudi_get_queue_id_for_cq, - .read_device_fw_version = gaudi_read_device_fw_version, .load_firmware_to_device = gaudi_load_firmware_to_device, .load_boot_fit_to_device = gaudi_load_boot_fit_to_device, .get_signal_cb_size = gaudi_get_signal_cb_size, @@ -8853,7 +9060,9 @@ static const struct hl_asic_funcs gaudi_funcs = { .get_hw_block_id = gaudi_get_hw_block_id, .hw_block_mmap = gaudi_block_mmap, .enable_events_from_fw = gaudi_enable_events_from_fw, - .map_pll_idx_to_fw_idx = gaudi_map_pll_idx_to_fw_idx + .map_pll_idx_to_fw_idx = gaudi_map_pll_idx_to_fw_idx, + .init_firmware_loader = gaudi_init_firmware_loader, + .init_cpu_scrambler_dram = gaudi_init_scrambler_hbm }; /** diff --git a/drivers/misc/habanalabs/gaudi/gaudiP.h b/drivers/misc/habanalabs/gaudi/gaudiP.h index 5929be81ec23..957bf3720f70 100644 --- a/drivers/misc/habanalabs/gaudi/gaudiP.h +++ b/drivers/misc/habanalabs/gaudi/gaudiP.h @@ -82,6 +82,7 @@ QMAN_STREAMS) #define QMAN_STREAMS 4 +#define PQ_FETCHER_CACHE_SIZE 8 #define DMA_QMAN_OFFSET (mmDMA1_QM_BASE - mmDMA0_QM_BASE) #define TPC_QMAN_OFFSET (mmTPC1_QM_BASE - mmTPC0_QM_BASE) diff --git a/drivers/misc/habanalabs/gaudi/gaudi_coresight.c b/drivers/misc/habanalabs/gaudi/gaudi_coresight.c index 6e56fa1c6c69..c2a27ed1c4d1 100644 --- a/drivers/misc/habanalabs/gaudi/gaudi_coresight.c +++ b/drivers/misc/habanalabs/gaudi/gaudi_coresight.c @@ -424,7 +424,7 @@ static int gaudi_config_stm(struct hl_device *hdev, if (frequency == 0) frequency = input->frequency; WREG32(base_reg + 0xE8C, frequency); - WREG32(base_reg + 0xE90, 0x7FF); + WREG32(base_reg + 0xE90, 0x1F00); /* SW-2176 - SW WA for HW bug */ if ((CFG_BASE + base_reg) >= mmDMA_CH_0_CS_STM_BASE && @@ -434,7 +434,7 @@ static int gaudi_config_stm(struct hl_device *hdev, WREG32(base_reg + 0xE6C, 0x0); } - WREG32(base_reg + 0xE80, 0x27 | (input->id << 16)); + WREG32(base_reg + 0xE80, 0x23 | (input->id << 16)); } else { WREG32(base_reg + 0xE80, 4); WREG32(base_reg + 0xD64, 0); @@ -634,7 +634,7 @@ static int gaudi_config_etr(struct hl_device *hdev, WREG32(mmPSOC_ETR_BUFWM, 0x3FFC); WREG32(mmPSOC_ETR_RSZ, input->buffer_size); WREG32(mmPSOC_ETR_MODE, input->sink_mode); - if (hdev->asic_prop.fw_security_disabled) { + if (!hdev->asic_prop.fw_security_enabled) { /* make ETR not privileged */ val = FIELD_PREP( PSOC_ETR_AXICTL_PROTCTRLBIT0_MASK, 0); diff --git a/drivers/misc/habanalabs/gaudi/gaudi_security.c b/drivers/misc/habanalabs/gaudi/gaudi_security.c index 9a706c5980ef..0d3240f1f7d7 100644 --- a/drivers/misc/habanalabs/gaudi/gaudi_security.c +++ b/drivers/misc/habanalabs/gaudi/gaudi_security.c @@ -1448,7 +1448,7 @@ static void gaudi_init_dma_protection_bits(struct hl_device *hdev) u32 pb_addr, mask; u8 word_offset; - if (hdev->asic_prop.fw_security_disabled) { + if (!hdev->asic_prop.fw_security_enabled) { gaudi_pb_set_block(hdev, mmDMA_IF_E_S_BASE); gaudi_pb_set_block(hdev, mmDMA_IF_E_S_DOWN_CH0_BASE); gaudi_pb_set_block(hdev, mmDMA_IF_E_S_DOWN_CH1_BASE); @@ -9135,7 +9135,7 @@ static void gaudi_init_tpc_protection_bits(struct hl_device *hdev) u32 pb_addr, mask; u8 word_offset; - if (hdev->asic_prop.fw_security_disabled) { + if (!hdev->asic_prop.fw_security_enabled) { gaudi_pb_set_block(hdev, mmTPC0_E2E_CRED_BASE); gaudi_pb_set_block(hdev, mmTPC1_E2E_CRED_BASE); gaudi_pb_set_block(hdev, mmTPC2_E2E_CRED_BASE); @@ -12818,7 +12818,7 @@ static void gaudi_init_protection_bits(struct hl_device *hdev) * secured */ - if (hdev->asic_prop.fw_security_disabled) { + if (!hdev->asic_prop.fw_security_enabled) { gaudi_pb_set_block(hdev, mmIF_E_PLL_BASE); gaudi_pb_set_block(hdev, mmMESH_W_PLL_BASE); gaudi_pb_set_block(hdev, mmSRAM_W_PLL_BASE); @@ -13023,7 +13023,7 @@ void gaudi_init_security(struct hl_device *hdev) * property configuration of MME SBAB and ACC to be non-privileged and * non-secured */ - if (hdev->asic_prop.fw_security_disabled) { + if (!hdev->asic_prop.fw_security_enabled) { WREG32(mmMME0_SBAB_PROT, 0x2); WREG32(mmMME0_ACC_PROT, 0x2); WREG32(mmMME1_SBAB_PROT, 0x2); @@ -13032,11 +13032,12 @@ void gaudi_init_security(struct hl_device *hdev) WREG32(mmMME2_ACC_PROT, 0x2); WREG32(mmMME3_SBAB_PROT, 0x2); WREG32(mmMME3_ACC_PROT, 0x2); - } - /* On RAZWI, 0 will be returned from RR and 0xBABA0BAD from PB */ - if (hdev->asic_prop.fw_security_disabled) + /* + * On RAZWI, 0 will be returned from RR and 0xBABA0BAD from PB + */ WREG32(0xC01B28, 0x1); + } gaudi_init_range_registers_lbw(hdev); diff --git a/drivers/misc/habanalabs/goya/goya.c b/drivers/misc/habanalabs/goya/goya.c index e0ad2a269779..755e08cf2ecc 100644 --- a/drivers/misc/habanalabs/goya/goya.c +++ b/drivers/misc/habanalabs/goya/goya.c @@ -87,6 +87,7 @@ #define GOYA_PLDM_QMAN0_TIMEOUT_USEC (HL_DEVICE_TIMEOUT_USEC * 30) #define GOYA_BOOT_FIT_REQ_TIMEOUT_USEC 1000000 /* 1s */ #define GOYA_MSG_TO_CPU_TIMEOUT_USEC 4000000 /* 4s */ +#define GOYA_WAIT_FOR_BL_TIMEOUT_USEC 15000000 /* 15s */ #define GOYA_QMAN0_FENCE_VAL 0xD169B243 @@ -354,7 +355,7 @@ static int goya_mmu_set_dram_default_page(struct hl_device *hdev); static int goya_mmu_add_mappings_for_device_cpu(struct hl_device *hdev); static void goya_mmu_prepare(struct hl_device *hdev, u32 asid); -int goya_get_fixed_properties(struct hl_device *hdev) +int goya_set_fixed_properties(struct hl_device *hdev) { struct asic_fixed_properties *prop = &hdev->asic_prop; int i; @@ -460,8 +461,10 @@ int goya_get_fixed_properties(struct hl_device *hdev) for (i = 0 ; i < HL_MAX_DCORES ; i++) prop->first_available_cq[i] = USHRT_MAX; - prop->fw_security_status_valid = false; + prop->fw_cpu_boot_dev_sts0_valid = false; + prop->fw_cpu_boot_dev_sts1_valid = false; prop->hard_reset_done_by_fw = false; + prop->gic_interrupts_enable = true; return 0; } @@ -531,10 +534,8 @@ static int goya_init_iatu(struct hl_device *hdev) struct hl_outbound_pci_region outbound_region; int rc; - if (hdev->asic_prop.iatu_done_by_fw) { - hdev->asic_funcs->set_dma_mask_from_fw(hdev); + if (hdev->asic_prop.iatu_done_by_fw) return 0; - } /* Inbound Region 0 - Bar 0 - Point to SRAM and CFG */ inbound_region.mode = PCI_BAR_MATCH_MODE; @@ -586,7 +587,7 @@ static int goya_early_init(struct hl_device *hdev) u32 fw_boot_status, val; int rc; - rc = goya_get_fixed_properties(hdev); + rc = goya_set_fixed_properties(hdev); if (rc) { dev_err(hdev->dev, "Failed to get fixed properties\n"); return rc; @@ -618,7 +619,7 @@ static int goya_early_init(struct hl_device *hdev) prop->dram_pci_bar_size = pci_resource_len(pdev, DDR_BAR_ID); /* If FW security is enabled at this point it means no access to ELBI */ - if (!hdev->asic_prop.fw_security_disabled) { + if (hdev->asic_prop.fw_security_enabled) { hdev->asic_prop.iatu_done_by_fw = true; goto pci_init; } @@ -642,8 +643,10 @@ pci_init: * version to determine whether we run with a security-enabled firmware */ rc = hl_fw_read_preboot_status(hdev, mmPSOC_GLOBAL_CONF_CPU_BOOT_STATUS, - mmCPU_BOOT_DEV_STS0, mmCPU_BOOT_ERR0, - GOYA_BOOT_FIT_REQ_TIMEOUT_USEC); + mmCPU_BOOT_DEV_STS0, + mmCPU_BOOT_DEV_STS1, mmCPU_BOOT_ERR0, + mmCPU_BOOT_ERR1, + GOYA_BOOT_FIT_REQ_TIMEOUT_USEC); if (rc) { if (hdev->reset_on_preboot_fail) hdev->asic_funcs->hw_fini(hdev, true); @@ -723,7 +726,15 @@ static void goya_fetch_psoc_frequency(struct hl_device *hdev) u16 pll_freq_arr[HL_PLL_NUM_OUTPUTS], freq; int rc; - if (hdev->asic_prop.fw_security_disabled) { + if (hdev->asic_prop.fw_security_enabled) { + rc = hl_fw_cpucp_pll_info_get(hdev, HL_GOYA_PCI_PLL, + pll_freq_arr); + + if (rc) + return; + + freq = pll_freq_arr[1]; + } else { div_fctr = RREG32(mmPSOC_PCI_PLL_DIV_FACTOR_1); div_sel = RREG32(mmPSOC_PCI_PLL_DIV_SEL_1); nr = RREG32(mmPSOC_PCI_PLL_NR); @@ -750,14 +761,6 @@ static void goya_fetch_psoc_frequency(struct hl_device *hdev) div_sel); freq = 0; } - } else { - rc = hl_fw_cpucp_pll_info_get(hdev, HL_GOYA_PCI_PLL, - pll_freq_arr); - - if (rc) - return; - - freq = pll_freq_arr[1]; } prop->psoc_timestamp_frequency = freq; @@ -849,6 +852,39 @@ void goya_late_fini(struct hl_device *hdev) hdev->hl_chip_info->info = NULL; } +static void goya_set_pci_memory_regions(struct hl_device *hdev) +{ + struct asic_fixed_properties *prop = &hdev->asic_prop; + struct pci_mem_region *region; + + /* CFG */ + region = &hdev->pci_mem_region[PCI_REGION_CFG]; + region->region_base = CFG_BASE; + region->region_size = CFG_SIZE; + region->offset_in_bar = CFG_BASE - SRAM_BASE_ADDR; + region->bar_size = CFG_BAR_SIZE; + region->bar_id = SRAM_CFG_BAR_ID; + region->used = 1; + + /* SRAM */ + region = &hdev->pci_mem_region[PCI_REGION_SRAM]; + region->region_base = SRAM_BASE_ADDR; + region->region_size = SRAM_SIZE; + region->offset_in_bar = 0; + region->bar_size = CFG_BAR_SIZE; + region->bar_id = SRAM_CFG_BAR_ID; + region->used = 1; + + /* DRAM */ + region = &hdev->pci_mem_region[PCI_REGION_DRAM]; + region->region_base = DRAM_PHYS_BASE; + region->region_size = hdev->asic_prop.dram_size; + region->offset_in_bar = 0; + region->bar_size = prop->dram_pci_bar_size; + region->bar_id = DDR_BAR_ID; + region->used = 1; +} + /* * goya_sw_init - Goya software initialization code * @@ -918,6 +954,9 @@ static int goya_sw_init(struct hl_device *hdev) spin_lock_init(&goya->hw_queues_lock); hdev->supports_coresight = true; hdev->supports_soft_reset = true; + hdev->allow_external_soft_reset = true; + + goya_set_pci_memory_regions(hdev); return 0; @@ -1263,8 +1302,11 @@ int goya_init_cpu_queues(struct hl_device *hdev) } /* update FW application security bits */ - if (prop->fw_security_status_valid) - prop->fw_app_security_map = RREG32(mmCPU_BOOT_DEV_STS0); + if (prop->fw_cpu_boot_dev_sts0_valid) + prop->fw_app_cpu_boot_dev_sts0 = RREG32(mmCPU_BOOT_DEV_STS0); + + if (prop->fw_cpu_boot_dev_sts1_valid) + prop->fw_app_cpu_boot_dev_sts1 = RREG32(mmCPU_BOOT_DEV_STS1); goya->hw_cap_initialized |= HW_CAP_CPU_Q; return 0; @@ -2402,47 +2444,67 @@ static int goya_load_boot_fit_to_device(struct hl_device *hdev) return hl_fw_load_fw_to_device(hdev, GOYA_BOOT_FIT_FILE, dst, 0, 0); } -/* - * FW component passes an offset from SRAM_BASE_ADDR in SCRATCHPAD_xx. - * The version string should be located by that offset. - */ -static int goya_read_device_fw_version(struct hl_device *hdev, - enum hl_fw_component fwc) -{ - const char *name; - u32 ver_off; - char *dest; - - switch (fwc) { - case FW_COMP_UBOOT: - ver_off = RREG32(mmUBOOT_VER_OFFSET); - dest = hdev->asic_prop.uboot_ver; - name = "U-Boot"; - break; - case FW_COMP_PREBOOT: - ver_off = RREG32(mmPREBOOT_VER_OFFSET); - dest = hdev->asic_prop.preboot_ver; - name = "Preboot"; - break; - default: - dev_warn(hdev->dev, "Undefined FW component: %d\n", fwc); - return -EIO; - } +static void goya_init_dynamic_firmware_loader(struct hl_device *hdev) +{ + struct dynamic_fw_load_mgr *dynamic_loader; + struct cpu_dyn_regs *dyn_regs; - ver_off &= ~((u32)SRAM_BASE_ADDR); + dynamic_loader = &hdev->fw_loader.dynamic_loader; - if (ver_off < SRAM_SIZE - VERSION_MAX_LEN) { - memcpy_fromio(dest, hdev->pcie_bar[SRAM_CFG_BAR_ID] + ver_off, - VERSION_MAX_LEN); - } else { - dev_err(hdev->dev, "%s version offset (0x%x) is above SRAM\n", - name, ver_off); - strcpy(dest, "unavailable"); + /* + * here we update initial values for few specific dynamic regs (as + * before reading the first descriptor from FW those value has to be + * hard-coded) in later stages of the protocol those values will be + * updated automatically by reading the FW descriptor so data there + * will always be up-to-date + */ + dyn_regs = &dynamic_loader->comm_desc.cpu_dyn_regs; + dyn_regs->kmd_msg_to_cpu = + cpu_to_le32(mmPSOC_GLOBAL_CONF_KMD_MSG_TO_CPU); + dyn_regs->cpu_cmd_status_to_host = + cpu_to_le32(mmCPU_CMD_STATUS_TO_HOST); - return -EIO; - } + dynamic_loader->wait_for_bl_timeout = GOYA_WAIT_FOR_BL_TIMEOUT_USEC; +} - return 0; +static void goya_init_static_firmware_loader(struct hl_device *hdev) +{ + struct static_fw_load_mgr *static_loader; + + static_loader = &hdev->fw_loader.static_loader; + + static_loader->preboot_version_max_off = SRAM_SIZE - VERSION_MAX_LEN; + static_loader->boot_fit_version_max_off = SRAM_SIZE - VERSION_MAX_LEN; + static_loader->kmd_msg_to_cpu_reg = mmPSOC_GLOBAL_CONF_KMD_MSG_TO_CPU; + static_loader->cpu_cmd_status_to_host_reg = mmCPU_CMD_STATUS_TO_HOST; + static_loader->cpu_boot_status_reg = mmPSOC_GLOBAL_CONF_CPU_BOOT_STATUS; + static_loader->cpu_boot_dev_status0_reg = mmCPU_BOOT_DEV_STS0; + static_loader->cpu_boot_dev_status1_reg = mmCPU_BOOT_DEV_STS1; + static_loader->boot_err0_reg = mmCPU_BOOT_ERR0; + static_loader->boot_err1_reg = mmCPU_BOOT_ERR1; + static_loader->preboot_version_offset_reg = mmPREBOOT_VER_OFFSET; + static_loader->boot_fit_version_offset_reg = mmUBOOT_VER_OFFSET; + static_loader->sram_offset_mask = ~(lower_32_bits(SRAM_BASE_ADDR)); +} + +static void goya_init_firmware_loader(struct hl_device *hdev) +{ + struct asic_fixed_properties *prop = &hdev->asic_prop; + struct fw_load_mgr *fw_loader = &hdev->fw_loader; + + /* fill common fields */ + fw_loader->boot_fit_img.image_name = GOYA_BOOT_FIT_FILE; + fw_loader->linux_img.image_name = GOYA_LINUX_FW_FILE; + fw_loader->cpu_timeout = GOYA_CPU_TIMEOUT_USEC; + fw_loader->boot_fit_timeout = GOYA_BOOT_FIT_REQ_TIMEOUT_USEC; + fw_loader->skip_bmc = false; + fw_loader->sram_bar_id = SRAM_CFG_BAR_ID; + fw_loader->dram_bar_id = DDR_BAR_ID; + + if (prop->dynamic_fw_load) + goya_init_dynamic_firmware_loader(hdev); + else + goya_init_static_firmware_loader(hdev); } static int goya_init_cpu(struct hl_device *hdev) @@ -2466,12 +2528,7 @@ static int goya_init_cpu(struct hl_device *hdev) return -EIO; } - rc = hl_fw_init_cpu(hdev, mmPSOC_GLOBAL_CONF_CPU_BOOT_STATUS, - mmPSOC_GLOBAL_CONF_UBOOT_MAGIC, - mmCPU_CMD_STATUS_TO_HOST, - mmCPU_BOOT_DEV_STS0, mmCPU_BOOT_ERR0, - false, GOYA_CPU_TIMEOUT_USEC, - GOYA_BOOT_FIT_REQ_TIMEOUT_USEC); + rc = hl_fw_init_cpu(hdev); if (rc) return rc; @@ -2881,7 +2938,7 @@ void *goya_get_int_queue_base(struct hl_device *hdev, u32 queue_id, *dma_handle = hdev->asic_prop.sram_base_address; - base = (void *) hdev->pcie_bar[SRAM_CFG_BAR_ID]; + base = (__force void *) hdev->pcie_bar[SRAM_CFG_BAR_ID]; switch (queue_id) { case GOYA_QUEUE_ID_MME: @@ -3270,6 +3327,7 @@ already_pinned: return 0; unpin_memory: + list_del(&userptr->job_node); hl_unpin_host_memory(hdev, userptr); free_userptr: kfree(userptr); @@ -5169,54 +5227,13 @@ static int goya_mmu_invalidate_cache(struct hl_device *hdev, bool is_hard, } static int goya_mmu_invalidate_cache_range(struct hl_device *hdev, - bool is_hard, u32 asid, u64 va, u64 size) + bool is_hard, u32 flags, + u32 asid, u64 va, u64 size) { - struct goya_device *goya = hdev->asic_specific; - u32 status, timeout_usec, inv_data, pi; - int rc; - - if (!(goya->hw_cap_initialized & HW_CAP_MMU) || - hdev->hard_reset_pending) - return 0; - - /* no need in L1 only invalidation in Goya */ - if (!is_hard) - return 0; - - if (hdev->pldm) - timeout_usec = GOYA_PLDM_MMU_TIMEOUT_USEC; - else - timeout_usec = MMU_CONFIG_TIMEOUT_USEC; - - /* - * TODO: currently invalidate entire L0 & L1 as in regular hard - * invalidation. Need to apply invalidation of specific cache lines with - * mask of ASID & VA & size. - * Note that L1 with be flushed entirely in any case. + /* Treat as invalidate all because there is no range invalidation + * in Goya */ - - /* L0 & L1 invalidation */ - inv_data = RREG32(mmSTLB_CACHE_INV); - /* PI is 8 bit */ - pi = ((inv_data & STLB_CACHE_INV_PRODUCER_INDEX_MASK) + 1) & 0xFF; - WREG32(mmSTLB_CACHE_INV, - (inv_data & STLB_CACHE_INV_INDEX_MASK_MASK) | pi); - - rc = hl_poll_timeout( - hdev, - mmSTLB_INV_CONSUMER_INDEX, - status, - status == pi, - 1000, - timeout_usec); - - if (rc) { - dev_err_ratelimited(hdev->dev, - "MMU cache invalidation timeout\n"); - hl_device_reset(hdev, HL_RESET_HARD); - } - - return rc; + return hdev->asic_funcs->mmu_invalidate_cache(hdev, is_hard, flags); } int goya_send_heartbeat(struct hl_device *hdev) @@ -5239,7 +5256,9 @@ int goya_cpucp_info_get(struct hl_device *hdev) if (!(goya->hw_cap_initialized & HW_CAP_CPU_Q)) return 0; - rc = hl_fw_cpucp_handshake(hdev, mmCPU_BOOT_DEV_STS0, mmCPU_BOOT_ERR0); + rc = hl_fw_cpucp_handshake(hdev, mmCPU_BOOT_DEV_STS0, + mmCPU_BOOT_DEV_STS1, mmCPU_BOOT_ERR0, + mmCPU_BOOT_ERR1); if (rc) return rc; @@ -5385,6 +5404,11 @@ static int goya_get_eeprom_data(struct hl_device *hdev, void *data, return hl_fw_get_eeprom_data(hdev, data, max_size); } +static void goya_cpu_init_scrambler_dram(struct hl_device *hdev) +{ + +} + static int goya_ctx_init(struct hl_ctx *ctx) { if (ctx->asid != HL_KERNEL_ASID_ID) @@ -5565,7 +5589,6 @@ static const struct hl_asic_funcs goya_funcs = { .ctx_fini = goya_ctx_fini, .get_clk_rate = goya_get_clk_rate, .get_queue_id_for_cq = goya_get_queue_id_for_cq, - .read_device_fw_version = goya_read_device_fw_version, .load_firmware_to_device = goya_load_firmware_to_device, .load_boot_fit_to_device = goya_load_boot_fit_to_device, .get_signal_cb_size = goya_get_signal_cb_size, @@ -5584,7 +5607,9 @@ static const struct hl_asic_funcs goya_funcs = { .get_hw_block_id = goya_get_hw_block_id, .hw_block_mmap = goya_block_mmap, .enable_events_from_fw = goya_enable_events_from_fw, - .map_pll_idx_to_fw_idx = goya_map_pll_idx_to_fw_idx + .map_pll_idx_to_fw_idx = goya_map_pll_idx_to_fw_idx, + .init_firmware_loader = goya_init_firmware_loader, + .init_cpu_scrambler_dram = goya_cpu_init_scrambler_dram }; /* diff --git a/drivers/misc/habanalabs/goya/goyaP.h b/drivers/misc/habanalabs/goya/goyaP.h index ef8c6c8b5c8d..0b05da614729 100644 --- a/drivers/misc/habanalabs/goya/goyaP.h +++ b/drivers/misc/habanalabs/goya/goyaP.h @@ -168,7 +168,7 @@ struct goya_device { u8 device_cpu_mmu_mappings_done; }; -int goya_get_fixed_properties(struct hl_device *hdev); +int goya_set_fixed_properties(struct hl_device *hdev); int goya_mmu_init(struct hl_device *hdev); void goya_init_dma_qmans(struct hl_device *hdev); void goya_init_mme_qmans(struct hl_device *hdev); diff --git a/drivers/misc/habanalabs/goya/goya_coresight.c b/drivers/misc/habanalabs/goya/goya_coresight.c index 6b7445cca580..c55c100fdd24 100644 --- a/drivers/misc/habanalabs/goya/goya_coresight.c +++ b/drivers/misc/habanalabs/goya/goya_coresight.c @@ -434,7 +434,7 @@ static int goya_config_etr(struct hl_device *hdev, WREG32(mmPSOC_ETR_BUFWM, 0x3FFC); WREG32(mmPSOC_ETR_RSZ, input->buffer_size); WREG32(mmPSOC_ETR_MODE, input->sink_mode); - if (hdev->asic_prop.fw_security_disabled) { + if (!hdev->asic_prop.fw_security_enabled) { /* make ETR not privileged */ val = FIELD_PREP(PSOC_ETR_AXICTL_PROTCTRLBIT0_MASK, 0); /* make ETR non-secured (inverted logic) */ diff --git a/drivers/misc/habanalabs/include/common/cpucp_if.h b/drivers/misc/habanalabs/include/common/cpucp_if.h index 27cd0ba99aa3..80b1d5a9d9f1 100644 --- a/drivers/misc/habanalabs/include/common/cpucp_if.h +++ b/drivers/misc/habanalabs/include/common/cpucp_if.h @@ -84,6 +84,20 @@ struct hl_eq_sm_sei_data { __u8 pad[3]; }; +enum hl_fw_alive_severity { + FW_ALIVE_SEVERITY_MINOR, + FW_ALIVE_SEVERITY_CRITICAL +}; + +struct hl_eq_fw_alive { + __le64 uptime_seconds; + __le32 process_id; + __le32 thread_id; + /* enum hl_fw_alive_severity */ + __u8 severity; + __u8 pad[7]; +}; + struct hl_eq_entry { struct hl_eq_header hdr; union { @@ -91,6 +105,7 @@ struct hl_eq_entry { struct hl_eq_hbm_ecc_data hbm_ecc_data; struct hl_eq_sm_sei_data sm_sei_data; struct cpucp_pkt_sync_err pkt_sync_err; + struct hl_eq_fw_alive fw_alive; __le64 data[7]; }; }; @@ -103,11 +118,16 @@ struct hl_eq_entry { #define EQ_CTL_EVENT_TYPE_SHIFT 16 #define EQ_CTL_EVENT_TYPE_MASK 0x03FF0000 +#define EQ_CTL_INDEX_SHIFT 0 +#define EQ_CTL_INDEX_MASK 0x0000FFFF + enum pq_init_status { PQ_INIT_STATUS_NA = 0, PQ_INIT_STATUS_READY_FOR_CP, PQ_INIT_STATUS_READY_FOR_HOST, - PQ_INIT_STATUS_READY_FOR_CP_SINGLE_MSI + PQ_INIT_STATUS_READY_FOR_CP_SINGLE_MSI, + PQ_INIT_STATUS_LEN_NOT_POWER_OF_TWO_ERR, + PQ_INIT_STATUS_ILLEGAL_Q_ADDR_ERR }; /* @@ -384,6 +404,20 @@ enum cpucp_packet_id { #define CPUCP_PKT_RES_PLL_OUT3_SHIFT 48 #define CPUCP_PKT_RES_PLL_OUT3_MASK 0xFFFF000000000000ull +#define CPUCP_PKT_VAL_PFC_IN1_SHIFT 0 +#define CPUCP_PKT_VAL_PFC_IN1_MASK 0x0000000000000001ull +#define CPUCP_PKT_VAL_PFC_IN2_SHIFT 1 +#define CPUCP_PKT_VAL_PFC_IN2_MASK 0x000000000000001Eull + +#define CPUCP_PKT_VAL_LPBK_IN1_SHIFT 0 +#define CPUCP_PKT_VAL_LPBK_IN1_MASK 0x0000000000000001ull +#define CPUCP_PKT_VAL_LPBK_IN2_SHIFT 1 +#define CPUCP_PKT_VAL_LPBK_IN2_MASK 0x000000000000001Eull + +/* heartbeat status bits */ +#define CPUCP_PKT_HB_STATUS_EQ_FAULT_SHIFT 0 +#define CPUCP_PKT_HB_STATUS_EQ_FAULT_MASK 0x00000001 + struct cpucp_packet { union { __le64 value; /* For SET packets */ @@ -425,6 +459,12 @@ struct cpucp_packet { /* For get CpuCP info/EEPROM data/NIC info */ __le32 data_max_size; + + /* + * For any general status bitmask. Shall be used whenever the + * result cannot be used to hold general purpose data. + */ + __le32 status_mask; }; __le32 reserved; @@ -629,6 +669,8 @@ struct cpucp_security_info { * @card_name: card name that will be displayed in HWMON subsystem on the host * @sec_info: security information * @pll_map: Bit map of supported PLLs for current ASIC version. + * @mme_binning_mask: MME binning mask, + * (0 = functional, 1 = binned) */ struct cpucp_info { struct cpucp_sensor sensors[CPUCP_MAX_SENSORS]; @@ -651,6 +693,7 @@ struct cpucp_info { struct cpucp_security_info sec_info; __le32 reserved6; __u8 pll_map[PLL_MAP_LEN]; + __le64 mme_binning_mask; }; struct cpucp_mac_addr { diff --git a/drivers/misc/habanalabs/include/common/hl_boot_if.h b/drivers/misc/habanalabs/include/common/hl_boot_if.h index e0a259e0495c..fa8a5ad2d438 100644 --- a/drivers/misc/habanalabs/include/common/hl_boot_if.h +++ b/drivers/misc/habanalabs/include/common/hl_boot_if.h @@ -8,7 +8,7 @@ #ifndef HL_BOOT_IF_H #define HL_BOOT_IF_H -#define LKD_HARD_RESET_MAGIC 0xED7BD694 +#define LKD_HARD_RESET_MAGIC 0xED7BD694 /* deprecated - do not use */ #define HL_POWER9_HOST_MAGIC 0x1DA30009 #define BOOT_FIT_SRAM_OFFSET 0x200000 @@ -99,6 +99,7 @@ #define CPU_BOOT_ERR0_PLL_FAIL (1 << 12) #define CPU_BOOT_ERR0_DEVICE_UNUSABLE_FAIL (1 << 13) #define CPU_BOOT_ERR0_ENABLED (1 << 31) +#define CPU_BOOT_ERR1_ENABLED (1 << 31) /* * BOOT DEVICE STATUS bits in BOOT_DEVICE_STS registers @@ -190,6 +191,24 @@ * PLLs. * Initialized in: linux * + * CPU_BOOT_DEV_STS0_GIC_PRIVILEGED_EN GIC access permission only from + * previleged entity. FW sets this status + * bit for host. If this bit is set then + * GIC can not be accessed from host. + * Initialized in: linux + * + * CPU_BOOT_DEV_STS0_EQ_INDEX_EN Event Queue (EQ) index is a running + * index for each new event sent to host. + * This is used as a method in host to + * identify that the waiting event in + * queue is actually a new event which + * was not served before. + * Initialized in: linux + * + * CPU_BOOT_DEV_STS0_MULTI_IRQ_POLL_EN Use multiple scratchpad interfaces to + * prevent IRQs overriding each other. + * Initialized in: linux + * * CPU_BOOT_DEV_STS0_ENABLED Device status register enabled. * This is a main indication that the * running FW populates the device status @@ -218,7 +237,11 @@ #define CPU_BOOT_DEV_STS0_FW_LD_COM_EN (1 << 16) #define CPU_BOOT_DEV_STS0_FW_IATU_CONF_EN (1 << 17) #define CPU_BOOT_DEV_STS0_DYN_PLL_EN (1 << 19) +#define CPU_BOOT_DEV_STS0_GIC_PRIVILEGED_EN (1 << 20) +#define CPU_BOOT_DEV_STS0_EQ_INDEX_EN (1 << 21) +#define CPU_BOOT_DEV_STS0_MULTI_IRQ_POLL_EN (1 << 22) #define CPU_BOOT_DEV_STS0_ENABLED (1 << 31) +#define CPU_BOOT_DEV_STS1_ENABLED (1 << 31) enum cpu_boot_status { CPU_BOOT_STATUS_NA = 0, /* Default value after reset of chip */ @@ -264,46 +287,98 @@ enum cpu_msg_status { /* communication registers mapping - consider ABI when changing */ struct cpu_dyn_regs { - uint32_t cpu_pq_base_addr_low; - uint32_t cpu_pq_base_addr_high; - uint32_t cpu_pq_length; - uint32_t cpu_pq_init_status; - uint32_t cpu_eq_base_addr_low; - uint32_t cpu_eq_base_addr_high; - uint32_t cpu_eq_length; - uint32_t cpu_eq_ci; - uint32_t cpu_cq_base_addr_low; - uint32_t cpu_cq_base_addr_high; - uint32_t cpu_cq_length; - uint32_t cpu_pf_pq_pi; - uint32_t cpu_boot_dev_sts0; - uint32_t cpu_boot_dev_sts1; - uint32_t cpu_boot_err0; - uint32_t cpu_boot_err1; - uint32_t cpu_boot_status; - uint32_t fw_upd_sts; - uint32_t fw_upd_cmd; - uint32_t fw_upd_pending_sts; - uint32_t fuse_ver_offset; - uint32_t preboot_ver_offset; - uint32_t uboot_ver_offset; - uint32_t hw_state; - uint32_t kmd_msg_to_cpu; - uint32_t cpu_cmd_status_to_host; - uint32_t reserved1[32]; /* reserve for future use */ + __le32 cpu_pq_base_addr_low; + __le32 cpu_pq_base_addr_high; + __le32 cpu_pq_length; + __le32 cpu_pq_init_status; + __le32 cpu_eq_base_addr_low; + __le32 cpu_eq_base_addr_high; + __le32 cpu_eq_length; + __le32 cpu_eq_ci; + __le32 cpu_cq_base_addr_low; + __le32 cpu_cq_base_addr_high; + __le32 cpu_cq_length; + __le32 cpu_pf_pq_pi; + __le32 cpu_boot_dev_sts0; + __le32 cpu_boot_dev_sts1; + __le32 cpu_boot_err0; + __le32 cpu_boot_err1; + __le32 cpu_boot_status; + __le32 fw_upd_sts; + __le32 fw_upd_cmd; + __le32 fw_upd_pending_sts; + __le32 fuse_ver_offset; + __le32 preboot_ver_offset; + __le32 uboot_ver_offset; + __le32 hw_state; + __le32 kmd_msg_to_cpu; + __le32 cpu_cmd_status_to_host; + union { + __le32 gic_host_irq_ctrl; + __le32 gic_host_pi_upd_irq; + }; + __le32 gic_tpc_qm_irq_ctrl; + __le32 gic_mme_qm_irq_ctrl; + __le32 gic_dma_qm_irq_ctrl; + __le32 gic_nic_qm_irq_ctrl; + __le32 gic_dma_core_irq_ctrl; + __le32 gic_host_halt_irq; + __le32 gic_host_ints_irq; + __le32 reserved1[24]; /* reserve for future use */ }; +/* TODO: remove the desc magic after the code is updated to use message */ /* HCDM - Habana Communications Descriptor Magic */ #define HL_COMMS_DESC_MAGIC 0x4843444D #define HL_COMMS_DESC_VER 1 +/* HCMv - Habana Communications Message + header version */ +#define HL_COMMS_MSG_MAGIC_VALUE 0x48434D00 +#define HL_COMMS_MSG_MAGIC_MASK 0xFFFFFF00 +#define HL_COMMS_MSG_MAGIC_VER_MASK 0xFF + +#define HL_COMMS_MSG_MAGIC_VER(ver) (HL_COMMS_MSG_MAGIC_VALUE | \ + ((ver) & HL_COMMS_MSG_MAGIC_VER_MASK)) +#define HL_COMMS_MSG_MAGIC_V0 HL_COMMS_DESC_MAGIC +#define HL_COMMS_MSG_MAGIC_V1 HL_COMMS_MSG_MAGIC_VER(1) + +#define HL_COMMS_MSG_MAGIC HL_COMMS_MSG_MAGIC_V1 + +#define HL_COMMS_MSG_MAGIC_VALIDATE_MAGIC(magic) \ + (((magic) & HL_COMMS_MSG_MAGIC_MASK) == \ + HL_COMMS_MSG_MAGIC_VALUE) + +#define HL_COMMS_MSG_MAGIC_VALIDATE_VERSION(magic, ver) \ + (((magic) & HL_COMMS_MSG_MAGIC_VER_MASK) >= \ + ((ver) & HL_COMMS_MSG_MAGIC_VER_MASK)) + +#define HL_COMMS_MSG_MAGIC_VALIDATE(magic, ver) \ + (HL_COMMS_MSG_MAGIC_VALIDATE_MAGIC((magic)) && \ + HL_COMMS_MSG_MAGIC_VALIDATE_VERSION((magic), (ver))) + +enum comms_msg_type { + HL_COMMS_DESC_TYPE = 0, + HL_COMMS_RESET_CAUSE_TYPE = 1, +}; + +/* TODO: remove this struct after the code is updated to use message */ /* this is the comms descriptor header - meta data */ struct comms_desc_header { - uint32_t magic; /* magic for validation */ - uint32_t crc32; /* CRC32 of the descriptor w/o header */ - uint16_t size; /* size of the descriptor w/o header */ - uint8_t version; /* descriptor version */ - uint8_t reserved[5]; /* pad to 64 bit */ + __le32 magic; /* magic for validation */ + __le32 crc32; /* CRC32 of the descriptor w/o header */ + __le16 size; /* size of the descriptor w/o header */ + __u8 version; /* descriptor version */ + __u8 reserved[5]; /* pad to 64 bit */ +}; + +/* this is the comms message header - meta data */ +struct comms_msg_header { + __le32 magic; /* magic for validation */ + __le32 crc32; /* CRC32 of the message w/o header */ + __le16 size; /* size of the message w/o header */ + __u8 version; /* message payload version */ + __u8 type; /* message type */ + __u8 reserved[4]; /* pad to 64 bit */ }; /* this is the main FW descriptor - consider ABI when changing */ @@ -314,7 +389,36 @@ struct lkd_fw_comms_desc { char cur_fw_ver[VERSION_MAX_LEN]; /* can be used for 1 more version w/o ABI change */ char reserved0[VERSION_MAX_LEN]; - uint64_t img_addr; /* address for next FW component load */ + __le64 img_addr; /* address for next FW component load */ +}; + +enum comms_reset_cause { + HL_RESET_CAUSE_UNKNOWN = 0, + HL_RESET_CAUSE_HEARTBEAT = 1, + HL_RESET_CAUSE_TDR = 2, +}; + +/* TODO: remove define after struct name is aligned on all projects */ +#define lkd_msg_comms lkd_fw_comms_msg + +/* this is the comms message descriptor */ +struct lkd_fw_comms_msg { + struct comms_msg_header header; + /* union for future expantions of new messages */ + union { + struct { + struct cpu_dyn_regs cpu_dyn_regs; + char fuse_ver[VERSION_MAX_LEN]; + char cur_fw_ver[VERSION_MAX_LEN]; + /* can be used for 1 more version w/o ABI change */ + char reserved0[VERSION_MAX_LEN]; + /* address for next FW component load */ + __le64 img_addr; + }; + struct { + __u8 reset_cause; + }; + }; }; /* @@ -386,11 +490,11 @@ enum comms_cmd { struct comms_command { union { /* bit fields are only for FW use */ struct { - unsigned int size :25; /* 32MB max. */ - unsigned int reserved :2; + u32 size :25; /* 32MB max. */ + u32 reserved :2; enum comms_cmd cmd :5; /* 32 commands */ }; - unsigned int val; + __le32 val; }; }; @@ -449,11 +553,11 @@ enum comms_ram_types { struct comms_status { union { /* bit fields are only for FW use */ struct { - unsigned int offset :26; - unsigned int ram_type :2; + u32 offset :26; + enum comms_ram_types ram_type :2; enum comms_sts status :4; /* 16 statuses */ }; - unsigned int val; + __le32 val; }; }; diff --git a/drivers/misc/habanalabs/include/gaudi/gaudi_async_events.h b/drivers/misc/habanalabs/include/gaudi/gaudi_async_events.h index e8651abf84f2..d966bd4dfea6 100644 --- a/drivers/misc/habanalabs/include/gaudi/gaudi_async_events.h +++ b/drivers/misc/habanalabs/include/gaudi/gaudi_async_events.h @@ -252,10 +252,11 @@ enum gaudi_async_event_id { GAUDI_EVENT_HBM3_SPI_0 = 407, GAUDI_EVENT_HBM3_SPI_1 = 408, GAUDI_EVENT_PSOC_GPIO_U16_0 = 421, - GAUDI_EVENT_PI_UPDATE = 484, - GAUDI_EVENT_HALT_MACHINE = 485, - GAUDI_EVENT_INTS_REGISTER = 486, - GAUDI_EVENT_SOFT_RESET = 487, + GAUDI_EVENT_NIC0_CS_DBG_DERR = 483, + GAUDI_EVENT_NIC1_CS_DBG_DERR = 487, + GAUDI_EVENT_NIC2_CS_DBG_DERR = 491, + GAUDI_EVENT_NIC3_CS_DBG_DERR = 495, + GAUDI_EVENT_NIC4_CS_DBG_DERR = 499, GAUDI_EVENT_RAZWI_OR_ADC = 548, GAUDI_EVENT_TPC0_QM = 572, GAUDI_EVENT_TPC1_QM = 573, @@ -303,6 +304,11 @@ enum gaudi_async_event_id { GAUDI_EVENT_NIC3_QP1 = 619, GAUDI_EVENT_NIC4_QP0 = 620, GAUDI_EVENT_NIC4_QP1 = 621, + GAUDI_EVENT_PI_UPDATE = 635, + GAUDI_EVENT_HALT_MACHINE = 636, + GAUDI_EVENT_INTS_REGISTER = 637, + GAUDI_EVENT_SOFT_RESET = 638, + GAUDI_EVENT_FW_ALIVE_S = 645, GAUDI_EVENT_DEV_RESET_REQ = 646, GAUDI_EVENT_PKT_QUEUE_OUT_SYNC = 647, GAUDI_EVENT_FIX_POWER_ENV_S = 658, diff --git a/drivers/misc/habanalabs/include/gaudi/gaudi_async_ids_map_extended.h b/drivers/misc/habanalabs/include/gaudi/gaudi_async_ids_map_extended.h index 3dc79c131805..479b6b038254 100644 --- a/drivers/misc/habanalabs/include/gaudi/gaudi_async_ids_map_extended.h +++ b/drivers/misc/habanalabs/include/gaudi/gaudi_async_ids_map_extended.h @@ -507,23 +507,28 @@ static struct gaudi_async_events_ids_map gaudi_irq_map_table[] = { { .fc_id = 480, .cpu_id = 329, .valid = 0, .name = "" }, { .fc_id = 481, .cpu_id = 330, .valid = 0, .name = "" }, { .fc_id = 482, .cpu_id = 331, .valid = 0, .name = "" }, - { .fc_id = 483, .cpu_id = 332, .valid = 0, .name = "" }, - { .fc_id = 484, .cpu_id = 333, .valid = 1, .name = "PI_UPDATE" }, - { .fc_id = 485, .cpu_id = 334, .valid = 1, .name = "HALT_MACHINE" }, - { .fc_id = 486, .cpu_id = 335, .valid = 1, .name = "INTS_REGISTER" }, - { .fc_id = 487, .cpu_id = 336, .valid = 1, .name = "SOFT_RESET" }, + { .fc_id = 483, .cpu_id = 332, .valid = 1, + .name = "NIC0_CS_DBG_DERR" }, + { .fc_id = 484, .cpu_id = 333, .valid = 0, .name = "" }, + { .fc_id = 485, .cpu_id = 334, .valid = 0, .name = "" }, + { .fc_id = 486, .cpu_id = 335, .valid = 0, .name = "" }, + { .fc_id = 487, .cpu_id = 336, .valid = 1, + .name = "NIC1_CS_DBG_DERR" }, { .fc_id = 488, .cpu_id = 337, .valid = 0, .name = "" }, { .fc_id = 489, .cpu_id = 338, .valid = 0, .name = "" }, { .fc_id = 490, .cpu_id = 339, .valid = 0, .name = "" }, - { .fc_id = 491, .cpu_id = 340, .valid = 0, .name = "" }, + { .fc_id = 491, .cpu_id = 340, .valid = 1, + .name = "NIC2_CS_DBG_DERR" }, { .fc_id = 492, .cpu_id = 341, .valid = 0, .name = "" }, { .fc_id = 493, .cpu_id = 342, .valid = 0, .name = "" }, { .fc_id = 494, .cpu_id = 343, .valid = 0, .name = "" }, - { .fc_id = 495, .cpu_id = 344, .valid = 0, .name = "" }, + { .fc_id = 495, .cpu_id = 344, .valid = 1, + .name = "NIC3_CS_DBG_DERR" }, { .fc_id = 496, .cpu_id = 345, .valid = 0, .name = "" }, { .fc_id = 497, .cpu_id = 346, .valid = 0, .name = "" }, { .fc_id = 498, .cpu_id = 347, .valid = 0, .name = "" }, - { .fc_id = 499, .cpu_id = 348, .valid = 0, .name = "" }, + { .fc_id = 499, .cpu_id = 348, .valid = 1, + .name = "NIC4_CS_DBG_DERR" }, { .fc_id = 500, .cpu_id = 349, .valid = 0, .name = "" }, { .fc_id = 501, .cpu_id = 350, .valid = 0, .name = "" }, { .fc_id = 502, .cpu_id = 351, .valid = 0, .name = "" }, @@ -659,17 +664,17 @@ static struct gaudi_async_events_ids_map gaudi_irq_map_table[] = { { .fc_id = 632, .cpu_id = 481, .valid = 0, .name = "" }, { .fc_id = 633, .cpu_id = 482, .valid = 0, .name = "" }, { .fc_id = 634, .cpu_id = 483, .valid = 0, .name = "" }, - { .fc_id = 635, .cpu_id = 484, .valid = 0, .name = "" }, - { .fc_id = 636, .cpu_id = 485, .valid = 0, .name = "" }, - { .fc_id = 637, .cpu_id = 486, .valid = 0, .name = "" }, - { .fc_id = 638, .cpu_id = 487, .valid = 0, .name = "" }, + { .fc_id = 635, .cpu_id = 484, .valid = 1, .name = "PI_UPDATE" }, + { .fc_id = 636, .cpu_id = 485, .valid = 1, .name = "HALT_MACHINE" }, + { .fc_id = 637, .cpu_id = 486, .valid = 1, .name = "INTS_REGISTER" }, + { .fc_id = 638, .cpu_id = 487, .valid = 1, .name = "SOFT_RESET" }, { .fc_id = 639, .cpu_id = 488, .valid = 0, .name = "" }, { .fc_id = 640, .cpu_id = 489, .valid = 0, .name = "" }, { .fc_id = 641, .cpu_id = 490, .valid = 0, .name = "" }, { .fc_id = 642, .cpu_id = 491, .valid = 0, .name = "" }, { .fc_id = 643, .cpu_id = 492, .valid = 0, .name = "" }, { .fc_id = 644, .cpu_id = 493, .valid = 0, .name = "" }, - { .fc_id = 645, .cpu_id = 494, .valid = 0, .name = "" }, + { .fc_id = 645, .cpu_id = 494, .valid = 1, .name = "FW_ALIVE_S" }, { .fc_id = 646, .cpu_id = 495, .valid = 1, .name = "DEV_RESET_REQ" }, { .fc_id = 647, .cpu_id = 496, .valid = 1, .name = "PKT_QUEUE_OUT_SYNC" }, diff --git a/drivers/misc/habanalabs/include/gaudi/gaudi_fw_if.h b/drivers/misc/habanalabs/include/gaudi/gaudi_fw_if.h index a9f51f9f9e92..34ca4fe50d91 100644 --- a/drivers/misc/habanalabs/include/gaudi/gaudi_fw_if.h +++ b/drivers/misc/habanalabs/include/gaudi/gaudi_fw_if.h @@ -20,6 +20,9 @@ #define UBOOT_FW_OFFSET 0x100000 /* 1MB in SRAM */ #define LINUX_FW_OFFSET 0x800000 /* 8MB in HBM */ +/* HBM thermal delta in [Deg] added to composite (CTemp) */ +#define HBM_TEMP_ADJUST_COEFF 6 + enum gaudi_nic_axi_error { RXB, RXE, @@ -27,6 +30,7 @@ enum gaudi_nic_axi_error { TXE, QPC_RESP, NON_AXI_ERR, + TMR, }; /* @@ -42,6 +46,48 @@ struct eq_nic_sei_event { __u8 pad[6]; }; +/* + * struct gaudi_nic_status - describes the status of a NIC port. + * @port: NIC port index. + * @bad_format_cnt: e.g. CRC. + * @responder_out_of_sequence_psn_cnt: e.g NAK. + * @high_ber_reinit_cnt: link reinit due to high BER. + * @correctable_err_cnt: e.g. bit-flip. + * @uncorrectable_err_cnt: e.g. MAC errors. + * @retraining_cnt: re-training counter. + * @up: is port up. + * @pcs_link: has PCS link. + * @phy_ready: is PHY ready. + * @auto_neg: is Autoneg enabled. + * @timeout_retransmission_cnt: timeout retransmission events + * @high_ber_cnt: high ber events + */ +struct gaudi_nic_status { + __u32 port; + __u32 bad_format_cnt; + __u32 responder_out_of_sequence_psn_cnt; + __u32 high_ber_reinit; + __u32 correctable_err_cnt; + __u32 uncorrectable_err_cnt; + __u32 retraining_cnt; + __u8 up; + __u8 pcs_link; + __u8 phy_ready; + __u8 auto_neg; + __u32 timeout_retransmission_cnt; + __u32 high_ber_cnt; +}; + +struct gaudi_flops_2_data { + union { + struct { + __u32 spsram_init_done : 1; + __u32 reserved : 31; + }; + __u32 data; + }; +}; + #define GAUDI_PLL_FREQ_LOW 200000000 /* 200 MHz */ #endif /* GAUDI_FW_IF_H */ diff --git a/drivers/misc/habanalabs/include/gaudi/gaudi_masks.h b/drivers/misc/habanalabs/include/gaudi/gaudi_masks.h index b53aeda9a982..9aea7e996654 100644 --- a/drivers/misc/habanalabs/include/gaudi/gaudi_masks.h +++ b/drivers/misc/habanalabs/include/gaudi/gaudi_masks.h @@ -66,7 +66,8 @@ #define PCI_DMA_QMAN_GLBL_ERR_CFG_STOP_ON_ERR_EN_MASK (\ (FIELD_PREP(DMA0_QM_GLBL_ERR_CFG_PQF_STOP_ON_ERR_MASK, 0xF)) | \ (FIELD_PREP(DMA0_QM_GLBL_ERR_CFG_CQF_STOP_ON_ERR_MASK, 0xF)) | \ - (FIELD_PREP(DMA0_QM_GLBL_ERR_CFG_CP_STOP_ON_ERR_MASK, 0xF))) + (FIELD_PREP(DMA0_QM_GLBL_ERR_CFG_CP_STOP_ON_ERR_MASK, 0xF)) | \ + (FIELD_PREP(DMA0_QM_GLBL_ERR_CFG_ARB_STOP_ON_ERR_MASK, 0x1))) #define HBM_DMA_QMAN_GLBL_ERR_CFG_MSG_EN_MASK (\ (FIELD_PREP(DMA0_QM_GLBL_ERR_CFG_PQF_ERR_MSG_EN_MASK, 0xF)) | \ @@ -76,7 +77,8 @@ #define HBM_DMA_QMAN_GLBL_ERR_CFG_STOP_ON_ERR_EN_MASK (\ (FIELD_PREP(DMA0_QM_GLBL_ERR_CFG_PQF_STOP_ON_ERR_MASK, 0xF)) | \ (FIELD_PREP(DMA0_QM_GLBL_ERR_CFG_CQF_STOP_ON_ERR_MASK, 0x1F)) | \ - (FIELD_PREP(DMA0_QM_GLBL_ERR_CFG_CP_STOP_ON_ERR_MASK, 0x1F))) + (FIELD_PREP(DMA0_QM_GLBL_ERR_CFG_CP_STOP_ON_ERR_MASK, 0x1F)) | \ + (FIELD_PREP(DMA0_QM_GLBL_ERR_CFG_ARB_STOP_ON_ERR_MASK, 0x1))) #define TPC_QMAN_GLBL_ERR_CFG_MSG_EN_MASK (\ (FIELD_PREP(TPC0_QM_GLBL_ERR_CFG_PQF_ERR_MSG_EN_MASK, 0xF)) | \ @@ -86,7 +88,8 @@ #define TPC_QMAN_GLBL_ERR_CFG_STOP_ON_ERR_EN_MASK (\ (FIELD_PREP(TPC0_QM_GLBL_ERR_CFG_PQF_STOP_ON_ERR_MASK, 0xF)) | \ (FIELD_PREP(TPC0_QM_GLBL_ERR_CFG_CQF_STOP_ON_ERR_MASK, 0x1F)) | \ - (FIELD_PREP(TPC0_QM_GLBL_ERR_CFG_CP_STOP_ON_ERR_MASK, 0x1F))) + (FIELD_PREP(TPC0_QM_GLBL_ERR_CFG_CP_STOP_ON_ERR_MASK, 0x1F)) | \ + (FIELD_PREP(TPC0_QM_GLBL_ERR_CFG_ARB_STOP_ON_ERR_MASK, 0x1))) #define MME_QMAN_GLBL_ERR_CFG_MSG_EN_MASK (\ (FIELD_PREP(MME0_QM_GLBL_ERR_CFG_PQF_ERR_MSG_EN_MASK, 0xF)) | \ @@ -96,7 +99,8 @@ #define MME_QMAN_GLBL_ERR_CFG_STOP_ON_ERR_EN_MASK (\ (FIELD_PREP(MME0_QM_GLBL_ERR_CFG_PQF_STOP_ON_ERR_MASK, 0xF)) | \ (FIELD_PREP(MME0_QM_GLBL_ERR_CFG_CQF_STOP_ON_ERR_MASK, 0x1F)) | \ - (FIELD_PREP(MME0_QM_GLBL_ERR_CFG_CP_STOP_ON_ERR_MASK, 0x1F))) + (FIELD_PREP(MME0_QM_GLBL_ERR_CFG_CP_STOP_ON_ERR_MASK, 0x1F)) | \ + (FIELD_PREP(MME0_QM_GLBL_ERR_CFG_ARB_STOP_ON_ERR_MASK, 0x1))) #define NIC_QMAN_GLBL_ERR_CFG_MSG_EN_MASK (\ (FIELD_PREP(NIC0_QM0_GLBL_ERR_CFG_PQF_ERR_MSG_EN_MASK, 0xF)) | \ @@ -106,7 +110,8 @@ #define NIC_QMAN_GLBL_ERR_CFG_STOP_ON_ERR_EN_MASK (\ (FIELD_PREP(NIC0_QM0_GLBL_ERR_CFG_PQF_STOP_ON_ERR_MASK, 0xF)) | \ (FIELD_PREP(NIC0_QM0_GLBL_ERR_CFG_CQF_STOP_ON_ERR_MASK, 0xF)) | \ - (FIELD_PREP(NIC0_QM0_GLBL_ERR_CFG_CP_STOP_ON_ERR_MASK, 0xF))) + (FIELD_PREP(NIC0_QM0_GLBL_ERR_CFG_CP_STOP_ON_ERR_MASK, 0xF)) | \ + (FIELD_PREP(NIC0_QM0_GLBL_ERR_CFG_ARB_STOP_ON_ERR_MASK, 0x1))) #define QMAN_CGM1_PWR_GATE_EN (FIELD_PREP(DMA0_QM_CGM_CFG1_MASK_TH_MASK, 0xA)) diff --git a/drivers/misc/habanalabs/include/gaudi/gaudi_reg_map.h b/drivers/misc/habanalabs/include/gaudi/gaudi_reg_map.h index 137afedf5f15..d95d4162ae2c 100644 --- a/drivers/misc/habanalabs/include/gaudi/gaudi_reg_map.h +++ b/drivers/misc/habanalabs/include/gaudi/gaudi_reg_map.h @@ -12,6 +12,16 @@ * PSOC scratch-pad registers */ #define mmHW_STATE mmPSOC_GLOBAL_CONF_SCRATCHPAD_0 +/* TODO: remove mmGIC_HOST_IRQ_CTRL_POLL_REG */ +#define mmGIC_HOST_IRQ_CTRL_POLL_REG mmPSOC_GLOBAL_CONF_SCRATCHPAD_1 +#define mmGIC_HOST_PI_UPD_IRQ_POLL_REG mmPSOC_GLOBAL_CONF_SCRATCHPAD_1 +#define mmGIC_TPC_QM_IRQ_CTRL_POLL_REG mmPSOC_GLOBAL_CONF_SCRATCHPAD_2 +#define mmGIC_MME_QM_IRQ_CTRL_POLL_REG mmPSOC_GLOBAL_CONF_SCRATCHPAD_3 +#define mmGIC_DMA_QM_IRQ_CTRL_POLL_REG mmPSOC_GLOBAL_CONF_SCRATCHPAD_4 +#define mmGIC_NIC_QM_IRQ_CTRL_POLL_REG mmPSOC_GLOBAL_CONF_SCRATCHPAD_5 +#define mmGIC_DMA_CR_IRQ_CTRL_POLL_REG mmPSOC_GLOBAL_CONF_SCRATCHPAD_6 +#define mmGIC_HOST_HALT_IRQ_POLL_REG mmPSOC_GLOBAL_CONF_SCRATCHPAD_7 +#define mmGIC_HOST_INTS_IRQ_POLL_REG mmPSOC_GLOBAL_CONF_SCRATCHPAD_8 #define mmCPU_BOOT_DEV_STS0 mmPSOC_GLOBAL_CONF_SCRATCHPAD_20 #define mmCPU_BOOT_DEV_STS1 mmPSOC_GLOBAL_CONF_SCRATCHPAD_21 #define mmFUSE_VER_OFFSET mmPSOC_GLOBAL_CONF_SCRATCHPAD_22 diff --git a/drivers/misc/hpilo.c b/drivers/misc/hpilo.c index fea3ae9d8686..8d00df9243c4 100644 --- a/drivers/misc/hpilo.c +++ b/drivers/misc/hpilo.c @@ -693,6 +693,8 @@ static int ilo_map_device(struct pci_dev *pdev, struct ilo_hwinfo *hw) { int bar; unsigned long off; + u8 pci_rev_id; + int rc; /* map the memory mapped i/o registers */ hw->mmio_vaddr = pci_iomap(pdev, 1, 0); @@ -702,7 +704,13 @@ static int ilo_map_device(struct pci_dev *pdev, struct ilo_hwinfo *hw) } /* map the adapter shared memory region */ - if (pdev->subsystem_device == 0x00E4) { + rc = pci_read_config_byte(pdev, PCI_REVISION_ID, &pci_rev_id); + if (rc != 0) { + dev_err(&pdev->dev, "Error reading PCI rev id: %d\n", rc); + goto out; + } + + if (pci_rev_id >= PCI_REV_ID_NECHES) { bar = 5; /* Last 8k is reserved for CCBs */ off = pci_resource_len(pdev, bar) - 0x2000; diff --git a/drivers/misc/hpilo.h b/drivers/misc/hpilo.h index f69ff645cac9..d57c34680b09 100644 --- a/drivers/misc/hpilo.h +++ b/drivers/misc/hpilo.h @@ -10,6 +10,9 @@ #define ILO_NAME "hpilo" +/* iLO ASIC PCI revision id */ +#define PCI_REV_ID_NECHES 7 + /* max number of open channel control blocks per device, hw limited to 32 */ #define MAX_CCB 24 /* min number of open channel control blocks per device, hw limited to 32 */ diff --git a/drivers/misc/ibmasm/heartbeat.c b/drivers/misc/ibmasm/heartbeat.c index 4f5f3bdc814d..59c9a0d95659 100644 --- a/drivers/misc/ibmasm/heartbeat.c +++ b/drivers/misc/ibmasm/heartbeat.c @@ -9,6 +9,7 @@ */ #include <linux/notifier.h> +#include <linux/panic_notifier.h> #include "ibmasm.h" #include "dot_command.h" #include "lowlevel.h" diff --git a/drivers/misc/ibmasm/module.c b/drivers/misc/ibmasm/module.c index 4edad6c445d3..dc8a06c06c63 100644 --- a/drivers/misc/ibmasm/module.c +++ b/drivers/misc/ibmasm/module.c @@ -111,7 +111,7 @@ static int ibmasm_init_one(struct pci_dev *pdev, const struct pci_device_id *id) result = ibmasm_init_remote_input_dev(sp); if (result) { dev_err(sp->dev, "Failed to initialize remote queue\n"); - goto error_send_message; + goto error_init_remote; } result = ibmasm_send_driver_vpd(sp); @@ -131,8 +131,9 @@ static int ibmasm_init_one(struct pci_dev *pdev, const struct pci_device_id *id) return 0; error_send_message: - disable_sp_interrupts(sp->base_address); ibmasm_free_remote_input_dev(sp); +error_init_remote: + disable_sp_interrupts(sp->base_address); free_irq(sp->irq, (void *)sp); error_request_irq: iounmap(sp->base_address); diff --git a/drivers/misc/ibmasm/remote.h b/drivers/misc/ibmasm/remote.h index 8d364462aeea..ec4e78ec5a67 100644 --- a/drivers/misc/ibmasm/remote.h +++ b/drivers/misc/ibmasm/remote.h @@ -43,7 +43,7 @@ #define REMOTE_BUTTON_MIDDLE 0x02 #define REMOTE_BUTTON_RIGHT 0x04 -/* size of keysym/keycode translation matricies */ +/* size of keysym/keycode translation matrices */ #define XLATE_SIZE 256 struct mouse_input { diff --git a/drivers/misc/lattice-ecp3-config.c b/drivers/misc/lattice-ecp3-config.c index 5eaf74447ca1..0f54730c7ed5 100644 --- a/drivers/misc/lattice-ecp3-config.c +++ b/drivers/misc/lattice-ecp3-config.c @@ -198,7 +198,7 @@ static int lattice_ecp3_probe(struct spi_device *spi) spi_set_drvdata(spi, data); init_completion(&data->fw_loaded); - err = request_firmware_nowait(THIS_MODULE, FW_ACTION_HOTPLUG, + err = request_firmware_nowait(THIS_MODULE, FW_ACTION_UEVENT, FIRMWARE_NAME, &spi->dev, GFP_KERNEL, spi, firmware_load); if (err) { diff --git a/drivers/misc/lkdtm/bugs.c b/drivers/misc/lkdtm/bugs.c index 0e8254d0cf0b..88c218a9f8b3 100644 --- a/drivers/misc/lkdtm/bugs.c +++ b/drivers/misc/lkdtm/bugs.c @@ -161,6 +161,9 @@ void lkdtm_UNALIGNED_LOAD_STORE_WRITE(void) if (*p == 0) val = 0x87654321; *p = val; + + if (IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS)) + pr_err("XFAIL: arch has CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS\n"); } void lkdtm_SOFTLOCKUP(void) @@ -300,8 +303,10 @@ void lkdtm_CORRUPT_LIST_ADD(void) if (target[0] == NULL && target[1] == NULL) pr_err("Overwrite did not happen, but no BUG?!\n"); - else + else { pr_err("list_add() corruption not detected!\n"); + pr_expected_config(CONFIG_DEBUG_LIST); + } } void lkdtm_CORRUPT_LIST_DEL(void) @@ -325,8 +330,10 @@ void lkdtm_CORRUPT_LIST_DEL(void) if (target[0] == NULL && target[1] == NULL) pr_err("Overwrite did not happen, but no BUG?!\n"); - else + else { pr_err("list_del() corruption not detected!\n"); + pr_expected_config(CONFIG_DEBUG_LIST); + } } /* Test that VMAP_STACK is actually allocating with a leading guard page */ @@ -463,7 +470,7 @@ void lkdtm_DOUBLE_FAULT(void) #ifdef CONFIG_ARM64 static noinline void change_pac_parameters(void) { - if (IS_ENABLED(CONFIG_ARM64_PTR_AUTH)) { + if (IS_ENABLED(CONFIG_ARM64_PTR_AUTH_KERNEL)) { /* Reset the keys of current task */ ptrauth_thread_init_kernel(current); ptrauth_thread_switch_kernel(current); @@ -477,8 +484,8 @@ noinline void lkdtm_CORRUPT_PAC(void) #define CORRUPT_PAC_ITERATE 10 int i; - if (!IS_ENABLED(CONFIG_ARM64_PTR_AUTH)) - pr_err("FAIL: kernel not built with CONFIG_ARM64_PTR_AUTH\n"); + if (!IS_ENABLED(CONFIG_ARM64_PTR_AUTH_KERNEL)) + pr_err("FAIL: kernel not built with CONFIG_ARM64_PTR_AUTH_KERNEL\n"); if (!system_supports_address_auth()) { pr_err("FAIL: CPU lacks pointer authentication feature\n"); diff --git a/drivers/misc/lkdtm/cfi.c b/drivers/misc/lkdtm/cfi.c index e73ebdbfa806..c9aeddef1044 100644 --- a/drivers/misc/lkdtm/cfi.c +++ b/drivers/misc/lkdtm/cfi.c @@ -38,5 +38,6 @@ void lkdtm_CFI_FORWARD_PROTO(void) func = (void *)lkdtm_increment_int; func(&called_count); - pr_info("Fail: survived mismatched prototype function call!\n"); + pr_err("FAIL: survived mismatched prototype function call!\n"); + pr_expected_config(CONFIG_CFI_CLANG); } diff --git a/drivers/misc/lkdtm/core.c b/drivers/misc/lkdtm/core.c index 8024b6a5cc7f..9dda87c6b54a 100644 --- a/drivers/misc/lkdtm/core.c +++ b/drivers/misc/lkdtm/core.c @@ -26,6 +26,7 @@ #include <linux/init.h> #include <linux/slab.h> #include <linux/debugfs.h> +#include <linux/init.h> #define DEFAULT_COUNT 10 @@ -120,11 +121,14 @@ static const struct crashtype crashtypes[] = { CRASHTYPE(UNALIGNED_LOAD_STORE_WRITE), CRASHTYPE(FORTIFY_OBJECT), CRASHTYPE(FORTIFY_SUBOBJECT), - CRASHTYPE(OVERWRITE_ALLOCATION), + CRASHTYPE(SLAB_LINEAR_OVERFLOW), + CRASHTYPE(VMALLOC_LINEAR_OVERFLOW), CRASHTYPE(WRITE_AFTER_FREE), CRASHTYPE(READ_AFTER_FREE), CRASHTYPE(WRITE_BUDDY_AFTER_FREE), CRASHTYPE(READ_BUDDY_AFTER_FREE), + CRASHTYPE(SLAB_INIT_ON_ALLOC), + CRASHTYPE(BUDDY_INIT_ON_ALLOC), CRASHTYPE(SLAB_FREE_DOUBLE), CRASHTYPE(SLAB_FREE_CROSS), CRASHTYPE(SLAB_FREE_PAGE), @@ -177,9 +181,7 @@ static const struct crashtype crashtypes[] = { CRASHTYPE(STACKLEAK_ERASING), CRASHTYPE(CFI_FORWARD_PROTO), CRASHTYPE(FORTIFIED_STRSCPY), -#ifdef CONFIG_X86_32 CRASHTYPE(DOUBLE_FAULT), -#endif #ifdef CONFIG_PPC_BOOK3S_64 CRASHTYPE(PPC_SLB_MULTIHIT), #endif @@ -399,6 +401,56 @@ static ssize_t direct_entry(struct file *f, const char __user *user_buf, return count; } +#ifndef MODULE +/* + * To avoid needing to export parse_args(), just don't use this code + * when LKDTM is built as a module. + */ +struct check_cmdline_args { + const char *param; + int value; +}; + +static int lkdtm_parse_one(char *param, char *val, + const char *unused, void *arg) +{ + struct check_cmdline_args *args = arg; + + /* short circuit if we already found a value. */ + if (args->value != -ESRCH) + return 0; + if (strncmp(param, args->param, strlen(args->param)) == 0) { + bool bool_result; + int ret; + + ret = kstrtobool(val, &bool_result); + if (ret == 0) + args->value = bool_result; + } + return 0; +} + +int lkdtm_check_bool_cmdline(const char *param) +{ + char *command_line; + struct check_cmdline_args args = { + .param = param, + .value = -ESRCH, + }; + + command_line = kstrdup(saved_command_line, GFP_KERNEL); + if (!command_line) + return -ENOMEM; + + parse_args("Setting sysctl args", command_line, + NULL, 0, -1, -1, &args, lkdtm_parse_one); + + kfree(command_line); + + return args.value; +} +#endif + static struct dentry *lkdtm_debugfs_root; static int __init lkdtm_module_init(void) diff --git a/drivers/misc/lkdtm/fortify.c b/drivers/misc/lkdtm/fortify.c index faf29cf04baa..0f51d31b57ca 100644 --- a/drivers/misc/lkdtm/fortify.c +++ b/drivers/misc/lkdtm/fortify.c @@ -76,7 +76,8 @@ void lkdtm_FORTIFIED_STRSCPY(void) */ strscpy(dst, src, strlen(src)); - pr_warn("FAIL: No overflow in above strscpy()\n"); + pr_err("FAIL: strscpy() overflow not detected!\n"); + pr_expected_config(CONFIG_FORTIFY_SOURCE); kfree(src); } diff --git a/drivers/misc/lkdtm/heap.c b/drivers/misc/lkdtm/heap.c index 1323bc16f113..3d9aae5821a0 100644 --- a/drivers/misc/lkdtm/heap.c +++ b/drivers/misc/lkdtm/heap.c @@ -5,6 +5,7 @@ */ #include "lkdtm.h" #include <linux/slab.h> +#include <linux/vmalloc.h> #include <linux/sched.h> static struct kmem_cache *double_free_cache; @@ -12,17 +13,36 @@ static struct kmem_cache *a_cache; static struct kmem_cache *b_cache; /* + * If there aren't guard pages, it's likely that a consecutive allocation will + * let us overflow into the second allocation without overwriting something real. + */ +void lkdtm_VMALLOC_LINEAR_OVERFLOW(void) +{ + char *one, *two; + + one = vzalloc(PAGE_SIZE); + two = vzalloc(PAGE_SIZE); + + pr_info("Attempting vmalloc linear overflow ...\n"); + memset(one, 0xAA, PAGE_SIZE + 1); + + vfree(two); + vfree(one); +} + +/* * This tries to stay within the next largest power-of-2 kmalloc cache * to avoid actually overwriting anything important if it's not detected * correctly. */ -void lkdtm_OVERWRITE_ALLOCATION(void) +void lkdtm_SLAB_LINEAR_OVERFLOW(void) { size_t len = 1020; u32 *data = kmalloc(len, GFP_KERNEL); if (!data) return; + pr_info("Attempting slab linear overflow ...\n"); data[1024 / sizeof(u32)] = 0x12345678; kfree(data); } @@ -89,9 +109,10 @@ void lkdtm_READ_AFTER_FREE(void) if (saw != *val) { /* Good! Poisoning happened, so declare a win. */ pr_info("Memory correctly poisoned (%x)\n", saw); - BUG(); + } else { + pr_err("FAIL: Memory was not poisoned!\n"); + pr_expected_config_param(CONFIG_INIT_ON_FREE_DEFAULT_ON, "init_on_free"); } - pr_info("Memory was not poisoned\n"); kfree(val); } @@ -145,13 +166,79 @@ void lkdtm_READ_BUDDY_AFTER_FREE(void) if (saw != *val) { /* Good! Poisoning happened, so declare a win. */ pr_info("Memory correctly poisoned (%x)\n", saw); - BUG(); + } else { + pr_err("FAIL: Buddy page was not poisoned!\n"); + pr_expected_config_param(CONFIG_INIT_ON_FREE_DEFAULT_ON, "init_on_free"); + } + + kfree(val); +} + +void lkdtm_SLAB_INIT_ON_ALLOC(void) +{ + u8 *first; + u8 *val; + + first = kmalloc(512, GFP_KERNEL); + if (!first) { + pr_info("Unable to allocate 512 bytes the first time.\n"); + return; + } + + memset(first, 0xAB, 512); + kfree(first); + + val = kmalloc(512, GFP_KERNEL); + if (!val) { + pr_info("Unable to allocate 512 bytes the second time.\n"); + return; + } + if (val != first) { + pr_warn("Reallocation missed clobbered memory.\n"); } - pr_info("Buddy page was not poisoned\n"); + if (memchr(val, 0xAB, 512) == NULL) { + pr_info("Memory appears initialized (%x, no earlier values)\n", *val); + } else { + pr_err("FAIL: Slab was not initialized\n"); + pr_expected_config_param(CONFIG_INIT_ON_ALLOC_DEFAULT_ON, "init_on_alloc"); + } kfree(val); } +void lkdtm_BUDDY_INIT_ON_ALLOC(void) +{ + u8 *first; + u8 *val; + + first = (u8 *)__get_free_page(GFP_KERNEL); + if (!first) { + pr_info("Unable to allocate first free page\n"); + return; + } + + memset(first, 0xAB, PAGE_SIZE); + free_page((unsigned long)first); + + val = (u8 *)__get_free_page(GFP_KERNEL); + if (!val) { + pr_info("Unable to allocate second free page\n"); + return; + } + + if (val != first) { + pr_warn("Reallocation missed clobbered memory.\n"); + } + + if (memchr(val, 0xAB, PAGE_SIZE) == NULL) { + pr_info("Memory appears initialized (%x, no earlier values)\n", *val); + } else { + pr_err("FAIL: Slab was not initialized\n"); + pr_expected_config_param(CONFIG_INIT_ON_ALLOC_DEFAULT_ON, "init_on_alloc"); + } + free_page((unsigned long)val); +} + void lkdtm_SLAB_FREE_DOUBLE(void) { int *val; diff --git a/drivers/misc/lkdtm/lkdtm.h b/drivers/misc/lkdtm/lkdtm.h index 99f90d3e5e9c..6a30b60519f3 100644 --- a/drivers/misc/lkdtm/lkdtm.h +++ b/drivers/misc/lkdtm/lkdtm.h @@ -6,6 +6,47 @@ #include <linux/kernel.h> +#define pr_expected_config(kconfig) \ +{ \ + if (IS_ENABLED(kconfig)) \ + pr_err("Unexpected! This kernel was built with " #kconfig "=y\n"); \ + else \ + pr_warn("This is probably expected, since this kernel was built *without* " #kconfig "=y\n"); \ +} + +#ifndef MODULE +int lkdtm_check_bool_cmdline(const char *param); +#define pr_expected_config_param(kconfig, param) \ +{ \ + if (IS_ENABLED(kconfig)) { \ + switch (lkdtm_check_bool_cmdline(param)) { \ + case 0: \ + pr_warn("This is probably expected, since this kernel was built with " #kconfig "=y but booted with '" param "=N'\n"); \ + break; \ + case 1: \ + pr_err("Unexpected! This kernel was built with " #kconfig "=y and booted with '" param "=Y'\n"); \ + break; \ + default: \ + pr_err("Unexpected! This kernel was built with " #kconfig "=y (and booted without '" param "' specified)\n"); \ + } \ + } else { \ + switch (lkdtm_check_bool_cmdline(param)) { \ + case 0: \ + pr_warn("This is probably expected, as kernel was built *without* " #kconfig "=y and booted with '" param "=N'\n"); \ + break; \ + case 1: \ + pr_err("Unexpected! This kernel was built *without* " #kconfig "=y but booted with '" param "=Y'\n"); \ + break; \ + default: \ + pr_err("This is probably expected, since this kernel was built *without* " #kconfig "=y (and booted without '" param "' specified)\n"); \ + break; \ + } \ + } \ +} +#else +#define pr_expected_config_param(kconfig, param) pr_expected_config(kconfig) +#endif + /* bugs.c */ void __init lkdtm_bugs_init(int *recur_param); void lkdtm_PANIC(void); @@ -39,11 +80,14 @@ void lkdtm_FORTIFY_SUBOBJECT(void); /* heap.c */ void __init lkdtm_heap_init(void); void __exit lkdtm_heap_exit(void); -void lkdtm_OVERWRITE_ALLOCATION(void); +void lkdtm_VMALLOC_LINEAR_OVERFLOW(void); +void lkdtm_SLAB_LINEAR_OVERFLOW(void); void lkdtm_WRITE_AFTER_FREE(void); void lkdtm_READ_AFTER_FREE(void); void lkdtm_WRITE_BUDDY_AFTER_FREE(void); void lkdtm_READ_BUDDY_AFTER_FREE(void); +void lkdtm_SLAB_INIT_ON_ALLOC(void); +void lkdtm_BUDDY_INIT_ON_ALLOC(void); void lkdtm_SLAB_FREE_DOUBLE(void); void lkdtm_SLAB_FREE_CROSS(void); void lkdtm_SLAB_FREE_PAGE(void); diff --git a/drivers/misc/lkdtm/stackleak.c b/drivers/misc/lkdtm/stackleak.c index d1a5c0705be3..00db21ff115e 100644 --- a/drivers/misc/lkdtm/stackleak.c +++ b/drivers/misc/lkdtm/stackleak.c @@ -74,8 +74,8 @@ void lkdtm_STACKLEAK_ERASING(void) end: if (test_failed) { - pr_err("FAIL: the thread stack is NOT properly erased\n"); - dump_stack(); + pr_err("FAIL: the thread stack is NOT properly erased!\n"); + pr_expected_config(CONFIG_GCC_PLUGIN_STACKLEAK); } else { pr_info("OK: the rest of the thread stack is properly erased\n"); } diff --git a/drivers/misc/lkdtm/usercopy.c b/drivers/misc/lkdtm/usercopy.c index 15d220ef35a5..9161ce7ed47a 100644 --- a/drivers/misc/lkdtm/usercopy.c +++ b/drivers/misc/lkdtm/usercopy.c @@ -173,6 +173,8 @@ static void do_usercopy_heap_size(bool to_user) goto free_user; } } + pr_err("FAIL: bad usercopy not detected!\n"); + pr_expected_config_param(CONFIG_HARDENED_USERCOPY, "hardened_usercopy"); free_user: vm_munmap(user_addr, PAGE_SIZE); @@ -248,6 +250,8 @@ static void do_usercopy_heap_whitelist(bool to_user) goto free_user; } } + pr_err("FAIL: bad usercopy not detected!\n"); + pr_expected_config_param(CONFIG_HARDENED_USERCOPY, "hardened_usercopy"); free_user: vm_munmap(user_alloc, PAGE_SIZE); @@ -319,7 +323,8 @@ void lkdtm_USERCOPY_KERNEL(void) pr_warn("copy_to_user failed, but lacked Oops\n"); goto free_user; } - pr_err("FAIL: survived bad copy_to_user()\n"); + pr_err("FAIL: bad copy_to_user() not detected!\n"); + pr_expected_config_param(CONFIG_HARDENED_USERCOPY, "hardened_usercopy"); free_user: vm_munmap(user_addr, PAGE_SIZE); diff --git a/drivers/misc/mei/bus-fixup.c b/drivers/misc/mei/bus-fixup.c index d8e760b11ae3..67844089db21 100644 --- a/drivers/misc/mei/bus-fixup.c +++ b/drivers/misc/mei/bus-fixup.c @@ -498,7 +498,7 @@ static struct mei_fixup { }; /** - * mei_cldev_fixup - run fixup handlers + * mei_cl_bus_dev_fixup - run fixup handlers * * @cldev: me client device */ diff --git a/drivers/misc/mei/client.c b/drivers/misc/mei/client.c index 2cc370adb238..96f4e59c32a5 100644 --- a/drivers/misc/mei/client.c +++ b/drivers/misc/mei/client.c @@ -326,7 +326,7 @@ void mei_io_cb_free(struct mei_cl_cb *cb) } /** - * mei_tx_cb_queue - queue tx callback + * mei_tx_cb_enqueue - queue tx callback * * Locking: called under "dev->device_lock" lock * @@ -1726,12 +1726,15 @@ nortpm: return rets; } -static inline u8 mei_ext_hdr_set_vtag(struct mei_ext_hdr *ext, u8 vtag) +static inline u8 mei_ext_hdr_set_vtag(void *ext, u8 vtag) { - ext->type = MEI_EXT_HDR_VTAG; - ext->ext_payload[0] = vtag; - ext->length = mei_data2slots(sizeof(*ext)); - return ext->length; + struct mei_ext_hdr_vtag *vtag_hdr = ext; + + vtag_hdr->hdr.type = MEI_EXT_HDR_VTAG; + vtag_hdr->hdr.length = mei_data2slots(sizeof(*vtag_hdr)); + vtag_hdr->vtag = vtag; + vtag_hdr->reserved = 0; + return vtag_hdr->hdr.length; } /** @@ -1745,7 +1748,6 @@ static struct mei_msg_hdr *mei_msg_hdr_init(const struct mei_cl_cb *cb) { size_t hdr_len; struct mei_ext_meta_hdr *meta; - struct mei_ext_hdr *ext; struct mei_msg_hdr *mei_hdr; bool is_ext, is_vtag; @@ -1764,7 +1766,7 @@ static struct mei_msg_hdr *mei_msg_hdr_init(const struct mei_cl_cb *cb) hdr_len += sizeof(*meta); if (is_vtag) - hdr_len += sizeof(*ext); + hdr_len += sizeof(struct mei_ext_hdr_vtag); setup_hdr: mei_hdr = kzalloc(hdr_len, GFP_KERNEL); @@ -2250,7 +2252,7 @@ static void mei_cl_dma_free(struct mei_cl *cl) } /** - * mei_cl_alloc_and_map - send client dma map request + * mei_cl_dma_alloc_and_map - send client dma map request * * @cl: host client * @fp: pointer to file structure @@ -2349,7 +2351,7 @@ out: } /** - * mei_cl_unmap_and_free - send client dma unmap request + * mei_cl_dma_unmap - send client dma unmap request * * @cl: host client * @fp: pointer to file structure diff --git a/drivers/misc/mei/hbm.c b/drivers/misc/mei/hbm.c index d0277c7fed10..99b5c1ecc444 100644 --- a/drivers/misc/mei/hbm.c +++ b/drivers/misc/mei/hbm.c @@ -853,7 +853,7 @@ out: } /** - * mei_hbm_cl_flow_control_res - flow control response from me + * mei_hbm_cl_tx_flow_ctrl_creds_res - flow control response from me * * @dev: the device structure * @fctrl: flow control response bus message diff --git a/drivers/misc/mei/hdcp/Kconfig b/drivers/misc/mei/hdcp/Kconfig index 95b2d6d37f10..54e1c9526909 100644 --- a/drivers/misc/mei/hdcp/Kconfig +++ b/drivers/misc/mei/hdcp/Kconfig @@ -1,4 +1,3 @@ - # SPDX-License-Identifier: GPL-2.0 # Copyright (c) 2019, Intel Corporation. All rights reserved. # diff --git a/drivers/misc/mei/hw-me.c b/drivers/misc/mei/hw-me.c index cda0829ac589..d3a6c0728645 100644 --- a/drivers/misc/mei/hw-me.c +++ b/drivers/misc/mei/hw-me.c @@ -1380,7 +1380,7 @@ static bool mei_me_fw_type_nm(const struct pci_dev *pdev) .quirk_probe = mei_me_fw_type_nm /** - * mei_me_fw_sku_sps_4() - check for sps 4.0 sku + * mei_me_fw_type_sps_4() - check for sps 4.0 sku * * Read ME FW Status register to check for SPS Firmware. * The SPS FW is only signaled in the PCI function 0. @@ -1405,7 +1405,7 @@ static bool mei_me_fw_type_sps_4(const struct pci_dev *pdev) .quirk_probe = mei_me_fw_type_sps_4 /** - * mei_me_fw_sku_sps() - check for sps sku + * mei_me_fw_type_sps() - check for sps sku * * Read ME FW Status register to check for SPS Firmware. * The SPS FW is only signaled in pci function 0 diff --git a/drivers/misc/mei/hw.h b/drivers/misc/mei/hw.h index b10606550613..dfd60c916da0 100644 --- a/drivers/misc/mei/hw.h +++ b/drivers/misc/mei/hw.h @@ -235,9 +235,8 @@ enum mei_ext_hdr_type { struct mei_ext_hdr { u8 type; u8 length; - u8 ext_payload[2]; - u8 hdr[]; -}; + u8 data[]; +} __packed; /** * struct mei_ext_meta_hdr - extend header meta data @@ -250,8 +249,21 @@ struct mei_ext_meta_hdr { u8 count; u8 size; u8 reserved[2]; - struct mei_ext_hdr hdrs[]; -}; + u8 hdrs[]; +} __packed; + +/** + * struct mei_ext_hdr_vtag - extend header for vtag + * + * @hdr: standard extend header + * @vtag: virtual tag + * @reserved: reserved + */ +struct mei_ext_hdr_vtag { + struct mei_ext_hdr hdr; + u8 vtag; + u8 reserved; +} __packed; /* * Extended header iterator functions @@ -266,7 +278,7 @@ struct mei_ext_meta_hdr { */ static inline struct mei_ext_hdr *mei_ext_begin(struct mei_ext_meta_hdr *meta) { - return meta->hdrs; + return (struct mei_ext_hdr *)meta->hdrs; } /** @@ -284,7 +296,7 @@ static inline bool mei_ext_last(struct mei_ext_meta_hdr *meta, } /** - *mei_ext_next - following extended header on the TLV list + * mei_ext_next - following extended header on the TLV list * * @ext: current extend header * @@ -295,7 +307,7 @@ static inline bool mei_ext_last(struct mei_ext_meta_hdr *meta, */ static inline struct mei_ext_hdr *mei_ext_next(struct mei_ext_hdr *ext) { - return (struct mei_ext_hdr *)(ext->hdr + (ext->length * 4)); + return (struct mei_ext_hdr *)((u8 *)ext + (ext->length * 4)); } /** diff --git a/drivers/misc/mei/interrupt.c b/drivers/misc/mei/interrupt.c index aab3ebfa9fc4..a67f4f2d33a9 100644 --- a/drivers/misc/mei/interrupt.c +++ b/drivers/misc/mei/interrupt.c @@ -123,13 +123,13 @@ static int mei_cl_irq_read_msg(struct mei_cl *cl, if (mei_hdr->extended) { struct mei_ext_hdr *ext; - struct mei_ext_hdr *vtag = NULL; + struct mei_ext_hdr_vtag *vtag_hdr = NULL; ext = mei_ext_begin(meta); do { switch (ext->type) { case MEI_EXT_HDR_VTAG: - vtag = ext; + vtag_hdr = (struct mei_ext_hdr_vtag *)ext; break; case MEI_EXT_HDR_NONE: fallthrough; @@ -141,20 +141,20 @@ static int mei_cl_irq_read_msg(struct mei_cl *cl, ext = mei_ext_next(ext); } while (!mei_ext_last(meta, ext)); - if (!vtag) { + if (!vtag_hdr) { cl_dbg(dev, cl, "vtag not found in extended header.\n"); cb->status = -EPROTO; goto discard; } - cl_dbg(dev, cl, "vtag: %d\n", vtag->ext_payload[0]); - if (cb->vtag && cb->vtag != vtag->ext_payload[0]) { + cl_dbg(dev, cl, "vtag: %d\n", vtag_hdr->vtag); + if (cb->vtag && cb->vtag != vtag_hdr->vtag) { cl_err(dev, cl, "mismatched tag: %d != %d\n", - cb->vtag, vtag->ext_payload[0]); + cb->vtag, vtag_hdr->vtag); cb->status = -EPROTO; goto discard; } - cb->vtag = vtag->ext_payload[0]; + cb->vtag = vtag_hdr->vtag; } if (!mei_cl_is_connected(cl)) { @@ -331,7 +331,6 @@ int mei_irq_read_handler(struct mei_device *dev, struct mei_ext_meta_hdr *meta_hdr = NULL; struct mei_cl *cl; int ret; - u32 ext_meta_hdr_u32; u32 hdr_size_left; u32 hdr_size_ext; int i; @@ -367,14 +366,12 @@ int mei_irq_read_handler(struct mei_device *dev, if (mei_hdr->extended) { if (!dev->rd_msg_hdr[1]) { - ext_meta_hdr_u32 = mei_read_hdr(dev); - dev->rd_msg_hdr[1] = ext_meta_hdr_u32; + dev->rd_msg_hdr[1] = mei_read_hdr(dev); dev->rd_msg_hdr_count++; (*slots)--; - dev_dbg(dev->dev, "extended header is %08x\n", - ext_meta_hdr_u32); + dev_dbg(dev->dev, "extended header is %08x\n", dev->rd_msg_hdr[1]); } - meta_hdr = ((struct mei_ext_meta_hdr *)dev->rd_msg_hdr + 1); + meta_hdr = ((struct mei_ext_meta_hdr *)&dev->rd_msg_hdr[1]); if (check_add_overflow((u32)sizeof(*meta_hdr), mei_slots2data(meta_hdr->size), &hdr_size_ext)) { diff --git a/drivers/misc/mei/main.c b/drivers/misc/mei/main.c index 28937b6e7e0c..786f7c8f7f61 100644 --- a/drivers/misc/mei/main.c +++ b/drivers/misc/mei/main.c @@ -50,8 +50,6 @@ static int mei_open(struct inode *inode, struct file *file) int err; dev = container_of(inode->i_cdev, struct mei_device, cdev); - if (!dev) - return -ENODEV; mutex_lock(&dev->device_lock); @@ -1104,7 +1102,7 @@ static ssize_t dev_state_show(struct device *device, static DEVICE_ATTR_RO(dev_state); /** - * dev_set_devstate: set to new device state and notify sysfs file. + * mei_set_devstate: set to new device state and notify sysfs file. * * @dev: mei_device * @state: new device state diff --git a/drivers/misc/mei/mei-trace.h b/drivers/misc/mei/mei-trace.h index df758033dc93..fe46ff2b9d69 100644 --- a/drivers/misc/mei/mei-trace.h +++ b/drivers/misc/mei/mei-trace.h @@ -26,7 +26,7 @@ TRACE_EVENT(mei_reg_read, __field(u32, val) ), TP_fast_assign( - __assign_str(dev, dev_name(dev)) + __assign_str(dev, dev_name(dev)); __entry->reg = reg; __entry->offs = offs; __entry->val = val; @@ -45,7 +45,7 @@ TRACE_EVENT(mei_reg_write, __field(u32, val) ), TP_fast_assign( - __assign_str(dev, dev_name(dev)) + __assign_str(dev, dev_name(dev)); __entry->reg = reg; __entry->offs = offs; __entry->val = val; @@ -64,7 +64,7 @@ TRACE_EVENT(mei_pci_cfg_read, __field(u32, val) ), TP_fast_assign( - __assign_str(dev, dev_name(dev)) + __assign_str(dev, dev_name(dev)); __entry->reg = reg; __entry->offs = offs; __entry->val = val; diff --git a/drivers/misc/mei/pci-txe.c b/drivers/misc/mei/pci-txe.c index 4bf26ce61044..aec0483b8e72 100644 --- a/drivers/misc/mei/pci-txe.c +++ b/drivers/misc/mei/pci-txe.c @@ -156,7 +156,7 @@ end: } /** - * mei_txe_remove - Device Shutdown Routine + * mei_txe_shutdown- Device Shutdown Routine * * @pdev: PCI device structure * diff --git a/drivers/misc/pvpanic/pvpanic-mmio.c b/drivers/misc/pvpanic/pvpanic-mmio.c index 4c0841776087..be4016084979 100644 --- a/drivers/misc/pvpanic/pvpanic-mmio.c +++ b/drivers/misc/pvpanic/pvpanic-mmio.c @@ -93,7 +93,7 @@ static int pvpanic_mmio_probe(struct platform_device *pdev) return -EINVAL; } - pi = kmalloc(sizeof(*pi), GFP_ATOMIC); + pi = devm_kmalloc(dev, sizeof(*pi), GFP_KERNEL); if (!pi) return -ENOMEM; @@ -104,19 +104,7 @@ static int pvpanic_mmio_probe(struct platform_device *pdev) pi->capability &= ioread8(base); pi->events = pi->capability; - dev_set_drvdata(dev, pi); - - return pvpanic_probe(pi); -} - -static int pvpanic_mmio_remove(struct platform_device *pdev) -{ - struct pvpanic_instance *pi = dev_get_drvdata(&pdev->dev); - - pvpanic_remove(pi); - kfree(pi); - - return 0; + return devm_pvpanic_probe(dev, pi); } static const struct of_device_id pvpanic_mmio_match[] = { @@ -139,6 +127,5 @@ static struct platform_driver pvpanic_mmio_driver = { .dev_groups = pvpanic_mmio_dev_groups, }, .probe = pvpanic_mmio_probe, - .remove = pvpanic_mmio_remove, }; module_platform_driver(pvpanic_mmio_driver); diff --git a/drivers/misc/pvpanic/pvpanic-pci.c b/drivers/misc/pvpanic/pvpanic-pci.c index 9ecc4e8559d5..a43c401017ae 100644 --- a/drivers/misc/pvpanic/pvpanic-pci.c +++ b/drivers/misc/pvpanic/pvpanic-pci.c @@ -73,20 +73,19 @@ ATTRIBUTE_GROUPS(pvpanic_pci_dev); static int pvpanic_pci_probe(struct pci_dev *pdev, const struct pci_device_id *ent) { - struct device *dev = &pdev->dev; struct pvpanic_instance *pi; void __iomem *base; int ret; - ret = pci_enable_device(pdev); + ret = pcim_enable_device(pdev); if (ret < 0) return ret; - base = pci_iomap(pdev, 0, 0); + base = pcim_iomap(pdev, 0, 0); if (!base) return -ENOMEM; - pi = kmalloc(sizeof(*pi), GFP_ATOMIC); + pi = devm_kmalloc(&pdev->dev, sizeof(*pi), GFP_KERNEL); if (!pi) return -ENOMEM; @@ -97,26 +96,13 @@ static int pvpanic_pci_probe(struct pci_dev *pdev, pi->capability &= ioread8(base); pi->events = pi->capability; - dev_set_drvdata(dev, pi); - - return pvpanic_probe(pi); -} - -static void pvpanic_pci_remove(struct pci_dev *pdev) -{ - struct pvpanic_instance *pi = dev_get_drvdata(&pdev->dev); - - pvpanic_remove(pi); - iounmap(pi->base); - kfree(pi); - pci_disable_device(pdev); + return devm_pvpanic_probe(&pdev->dev, pi); } static struct pci_driver pvpanic_pci_driver = { .name = "pvpanic-pci", .id_table = pvpanic_pci_id_tbl, .probe = pvpanic_pci_probe, - .remove = pvpanic_pci_remove, .driver = { .dev_groups = pvpanic_pci_dev_groups, }, diff --git a/drivers/misc/pvpanic/pvpanic.c b/drivers/misc/pvpanic/pvpanic.c index 65f70a4da8c0..02b807c788c9 100644 --- a/drivers/misc/pvpanic/pvpanic.c +++ b/drivers/misc/pvpanic/pvpanic.c @@ -13,6 +13,7 @@ #include <linux/mod_devicetable.h> #include <linux/module.h> #include <linux/platform_device.h> +#include <linux/panic_notifier.h> #include <linux/types.h> #include <linux/cdev.h> #include <linux/list.h> @@ -60,25 +61,10 @@ static struct notifier_block pvpanic_panic_nb = { .priority = 1, /* let this called before broken drm_fb_helper */ }; -int pvpanic_probe(struct pvpanic_instance *pi) -{ - if (!pi || !pi->base) - return -EINVAL; - - spin_lock(&pvpanic_lock); - list_add(&pi->list, &pvpanic_list); - spin_unlock(&pvpanic_lock); - - return 0; -} -EXPORT_SYMBOL_GPL(pvpanic_probe); - -void pvpanic_remove(struct pvpanic_instance *pi) +static void pvpanic_remove(void *param) { struct pvpanic_instance *pi_cur, *pi_next; - - if (!pi) - return; + struct pvpanic_instance *pi = param; spin_lock(&pvpanic_lock); list_for_each_entry_safe(pi_cur, pi_next, &pvpanic_list, list) { @@ -89,7 +75,19 @@ void pvpanic_remove(struct pvpanic_instance *pi) } spin_unlock(&pvpanic_lock); } -EXPORT_SYMBOL_GPL(pvpanic_remove); + +int devm_pvpanic_probe(struct device *dev, struct pvpanic_instance *pi) +{ + if (!pi || !pi->base) + return -EINVAL; + + spin_lock(&pvpanic_lock); + list_add(&pi->list, &pvpanic_list); + spin_unlock(&pvpanic_lock); + + return devm_add_action_or_reset(dev, pvpanic_remove, pi); +} +EXPORT_SYMBOL_GPL(devm_pvpanic_probe); static int pvpanic_init(void) { diff --git a/drivers/misc/pvpanic/pvpanic.h b/drivers/misc/pvpanic/pvpanic.h index 1afccc2e9fec..493545951754 100644 --- a/drivers/misc/pvpanic/pvpanic.h +++ b/drivers/misc/pvpanic/pvpanic.h @@ -15,7 +15,6 @@ struct pvpanic_instance { struct list_head list; }; -int pvpanic_probe(struct pvpanic_instance *pi); -void pvpanic_remove(struct pvpanic_instance *pi); +int devm_pvpanic_probe(struct device *dev, struct pvpanic_instance *pi); #endif /* PVPANIC_H_ */ diff --git a/drivers/misc/sgi-gru/grufault.c b/drivers/misc/sgi-gru/grufault.c index 723825524ea0..d7ef61e602ed 100644 --- a/drivers/misc/sgi-gru/grufault.c +++ b/drivers/misc/sgi-gru/grufault.c @@ -49,8 +49,8 @@ struct vm_area_struct *gru_find_vma(unsigned long vaddr) { struct vm_area_struct *vma; - vma = find_vma(current->mm, vaddr); - if (vma && vma->vm_start <= vaddr && vma->vm_ops == &gru_vm_ops) + vma = vma_lookup(current->mm, vaddr); + if (vma && vma->vm_ops == &gru_vm_ops) return vma; return NULL; } diff --git a/drivers/misc/sram.c b/drivers/misc/sram.c index 202bf951e909..93638ae2753a 100644 --- a/drivers/misc/sram.c +++ b/drivers/misc/sram.c @@ -341,6 +341,7 @@ static int sram_probe(struct platform_device *pdev) { struct sram_dev *sram; int ret; + struct resource *res; int (*init_func)(void); sram = devm_kzalloc(&pdev->dev, sizeof(*sram), GFP_KERNEL); @@ -349,10 +350,11 @@ static int sram_probe(struct platform_device *pdev) sram->dev = &pdev->dev; + res = platform_get_resource(pdev, IORESOURCE_MEM, 0); if (of_property_read_bool(pdev->dev.of_node, "no-memory-wc")) - sram->virt_base = devm_platform_ioremap_resource(pdev, 0); + sram->virt_base = devm_ioremap_resource(&pdev->dev, res); else - sram->virt_base = devm_platform_ioremap_resource_wc(pdev, 0); + sram->virt_base = devm_ioremap_resource_wc(&pdev->dev, res); if (IS_ERR(sram->virt_base)) { dev_err(&pdev->dev, "could not map SRAM registers\n"); return PTR_ERR(sram->virt_base); diff --git a/drivers/misc/ti-st/st_core.c b/drivers/misc/ti-st/st_core.c index 071844b58073..7f6976a9f508 100644 --- a/drivers/misc/ti-st/st_core.c +++ b/drivers/misc/ti-st/st_core.c @@ -52,13 +52,12 @@ static void remove_channel_from_table(struct st_data_s *st_gdata, */ int st_get_uart_wr_room(struct st_data_s *st_gdata) { - struct tty_struct *tty; if (unlikely(st_gdata == NULL || st_gdata->tty == NULL)) { pr_err("tty unavailable to perform write"); return -1; } - tty = st_gdata->tty; - return tty->ops->write_room(tty); + + return tty_write_room(st_gdata->tty); } /* @@ -798,7 +797,7 @@ static void st_tty_close(struct tty_struct *tty) } static void st_tty_receive(struct tty_struct *tty, const unsigned char *data, - char *tty_flags, int count) + const char *tty_flags, int count) { #ifdef VERBOSE print_hex_dump(KERN_DEBUG, ">in>", DUMP_PREFIX_NONE, @@ -845,6 +844,7 @@ static void st_tty_flush_buffer(struct tty_struct *tty) } static struct tty_ldisc_ops st_ldisc_ops = { + .num = N_TI_WL, .name = "n_st", .open = st_tty_open, .close = st_tty_close, @@ -860,7 +860,7 @@ int st_core_init(struct st_data_s **core_data) struct st_data_s *st_gdata; long err; - err = tty_register_ldisc(N_TI_WL, &st_ldisc_ops); + err = tty_register_ldisc(&st_ldisc_ops); if (err) { pr_err("error registering %d line discipline %ld", N_TI_WL, err); @@ -871,11 +871,8 @@ int st_core_init(struct st_data_s **core_data) st_gdata = kzalloc(sizeof(struct st_data_s), GFP_KERNEL); if (!st_gdata) { pr_err("memory allocation failed"); - err = tty_unregister_ldisc(N_TI_WL); - if (err) - pr_err("unable to un-register ldisc %ld", err); err = -ENOMEM; - return err; + goto err_unreg_ldisc; } /* Initialize ST TxQ and Tx waitQ queue head. All BT/FM/GPS module skb's @@ -890,17 +887,18 @@ int st_core_init(struct st_data_s **core_data) err = st_ll_init(st_gdata); if (err) { pr_err("error during st_ll initialization(%ld)", err); - kfree(st_gdata); - err = tty_unregister_ldisc(N_TI_WL); - if (err) - pr_err("unable to un-register ldisc"); - return err; + goto err_free_gdata; } INIT_WORK(&st_gdata->work_write_wakeup, work_fn_write_wakeup); *core_data = st_gdata; return 0; +err_free_gdata: + kfree(st_gdata); +err_unreg_ldisc: + tty_unregister_ldisc(&st_ldisc_ops); + return err; } void st_core_exit(struct st_data_s *st_gdata) @@ -918,9 +916,7 @@ void st_core_exit(struct st_data_s *st_gdata) kfree_skb(st_gdata->rx_skb); kfree_skb(st_gdata->tx_skb); /* TTY ldisc cleanup */ - err = tty_unregister_ldisc(N_TI_WL); - if (err) - pr_err("unable to un-register ldisc %ld", err); + tty_unregister_ldisc(&st_ldisc_ops); /* free the global data pointer */ kfree(st_gdata); } diff --git a/drivers/misc/uacce/uacce.c b/drivers/misc/uacce/uacce.c index bae18ef03dcb..488eeb2811ae 100644 --- a/drivers/misc/uacce/uacce.c +++ b/drivers/misc/uacce/uacce.c @@ -387,15 +387,22 @@ static void uacce_release(struct device *dev) static unsigned int uacce_enable_sva(struct device *parent, unsigned int flags) { + int ret; + if (!(flags & UACCE_DEV_SVA)) return flags; flags &= ~UACCE_DEV_SVA; - if (iommu_dev_enable_feature(parent, IOMMU_DEV_FEAT_IOPF)) + ret = iommu_dev_enable_feature(parent, IOMMU_DEV_FEAT_IOPF); + if (ret) { + dev_err(parent, "failed to enable IOPF feature! ret = %pe\n", ERR_PTR(ret)); return flags; + } - if (iommu_dev_enable_feature(parent, IOMMU_DEV_FEAT_SVA)) { + ret = iommu_dev_enable_feature(parent, IOMMU_DEV_FEAT_SVA); + if (ret) { + dev_err(parent, "failed to enable SVA feature! ret = %pe\n", ERR_PTR(ret)); iommu_dev_disable_feature(parent, IOMMU_DEV_FEAT_IOPF); return flags; } diff --git a/drivers/misc/vmw_vmci/vmci_context.c b/drivers/misc/vmw_vmci/vmci_context.c index 26ff49fdf0f7..c0b5e339d5a1 100644 --- a/drivers/misc/vmw_vmci/vmci_context.c +++ b/drivers/misc/vmw_vmci/vmci_context.c @@ -107,7 +107,7 @@ struct vmci_ctx *vmci_ctx_create(u32 cid, u32 priv_flags, context = kzalloc(sizeof(*context), GFP_KERNEL); if (!context) { pr_warn("Failed to allocate memory for VMCI context\n"); - error = -EINVAL; + error = -ENOMEM; goto err_out; } diff --git a/drivers/misc/xilinx_sdfec.c b/drivers/misc/xilinx_sdfec.c index 23c8448a9c3b..d6e3c650bd11 100644 --- a/drivers/misc/xilinx_sdfec.c +++ b/drivers/misc/xilinx_sdfec.c @@ -1013,9 +1013,6 @@ static __poll_t xsdfec_poll(struct file *file, poll_table *wait) xsdfec = container_of(file->private_data, struct xsdfec_dev, miscdev); - if (!xsdfec) - return EPOLLNVAL | EPOLLHUP; - poll_wait(file, &xsdfec->waitq, wait); /* XSDFEC ISR detected an error */ |