diff options
Diffstat (limited to 'drivers')
399 files changed, 18982 insertions, 4359 deletions
diff --git a/drivers/acpi/acpica/tbfadt.c b/drivers/acpi/acpica/tbfadt.c index 44267a92bce5..3c126c6d306b 100644 --- a/drivers/acpi/acpica/tbfadt.c +++ b/drivers/acpi/acpica/tbfadt.c @@ -315,23 +315,19 @@ void acpi_tb_parse_fadt(void) ACPI_TABLE_ORIGIN_INTERNAL_PHYSICAL, NULL, FALSE, TRUE, &acpi_gbl_dsdt_index); - /* If Hardware Reduced flag is set, there is no FACS */ - - if (!acpi_gbl_reduced_hardware) { - if (acpi_gbl_FADT.facs) { - acpi_tb_install_standard_table((acpi_physical_address) - acpi_gbl_FADT.facs, - ACPI_TABLE_ORIGIN_INTERNAL_PHYSICAL, - NULL, FALSE, TRUE, - &acpi_gbl_facs_index); - } - if (acpi_gbl_FADT.Xfacs) { - acpi_tb_install_standard_table((acpi_physical_address) - acpi_gbl_FADT.Xfacs, - ACPI_TABLE_ORIGIN_INTERNAL_PHYSICAL, - NULL, FALSE, TRUE, - &acpi_gbl_xfacs_index); - } + if (acpi_gbl_FADT.facs) { + acpi_tb_install_standard_table((acpi_physical_address) + acpi_gbl_FADT.facs, + ACPI_TABLE_ORIGIN_INTERNAL_PHYSICAL, + NULL, FALSE, TRUE, + &acpi_gbl_facs_index); + } + if (acpi_gbl_FADT.Xfacs) { + acpi_tb_install_standard_table((acpi_physical_address) + acpi_gbl_FADT.Xfacs, + ACPI_TABLE_ORIGIN_INTERNAL_PHYSICAL, + NULL, FALSE, TRUE, + &acpi_gbl_xfacs_index); } } diff --git a/drivers/acpi/acpica/tbutils.c b/drivers/acpi/acpica/tbutils.c index bb4a56e5673a..15fa68a5ea6e 100644 --- a/drivers/acpi/acpica/tbutils.c +++ b/drivers/acpi/acpica/tbutils.c @@ -36,12 +36,7 @@ acpi_status acpi_tb_initialize_facs(void) { struct acpi_table_facs *facs; - /* If Hardware Reduced flag is set, there is no FACS */ - - if (acpi_gbl_reduced_hardware) { - acpi_gbl_FACS = NULL; - return (AE_OK); - } else if (acpi_gbl_FADT.Xfacs && + if (acpi_gbl_FADT.Xfacs && (!acpi_gbl_FADT.facs || !acpi_gbl_use32_bit_facs_addresses)) { (void)acpi_get_table_by_index(acpi_gbl_xfacs_index, diff --git a/drivers/auxdisplay/Kconfig b/drivers/auxdisplay/Kconfig index 151d95f96b11..69d2138d7efb 100644 --- a/drivers/auxdisplay/Kconfig +++ b/drivers/auxdisplay/Kconfig @@ -16,6 +16,9 @@ menuconfig AUXDISPLAY if AUXDISPLAY +# +# Character LCD section +# config CHARLCD tristate "Character LCD core support" if COMPILE_TEST help @@ -25,12 +28,6 @@ config CHARLCD This is some character LCD core interface that multiple drivers can use. -config LINEDISP - tristate "Character line display core support" if COMPILE_TEST - help - This is the core support for single-line character displays, to be - selected by drivers that use it. - config HD44780_COMMON tristate "Common functions for HD44780 (and compatibles) LCD displays" if COMPILE_TEST select CHARLCD @@ -52,145 +49,6 @@ config HD44780 kernel and started at boot. If you don't understand what all this is about, say N. -config KS0108 - tristate "KS0108 LCD Controller" - depends on PARPORT_PC - default n - help - If you have a LCD controlled by one or more KS0108 - controllers, say Y. You will need also another more specific - driver for your LCD. - - Depends on Parallel Port support. If you say Y at - parport, you will be able to compile this as a module (M) - and built-in as well (Y). - - To compile this as a module, choose M here: - the module will be called ks0108. - - If unsure, say N. - -config KS0108_PORT - hex "Parallel port where the LCD is connected" - depends on KS0108 - default 0x378 - help - The address of the parallel port where the LCD is connected. - - The first standard parallel port address is 0x378. - The second standard parallel port address is 0x278. - The third standard parallel port address is 0x3BC. - - You can specify a different address if you need. - - If you don't know what I'm talking about, load the parport module, - and execute "dmesg" or "cat /proc/ioports". You can see there how - many parallel ports are present and which address each one has. - - Usually you only need to use 0x378. - - If you compile this as a module, you can still override this - using the module parameters. - -config KS0108_DELAY - int "Delay between each control writing (microseconds)" - depends on KS0108 - default "2" - help - Amount of time the ks0108 should wait between each control write - to the parallel port. - - If your LCD seems to miss random writings, increment this. - - If you don't know what I'm talking about, ignore it. - - If you compile this as a module, you can still override this - value using the module parameters. - -config CFAG12864B - tristate "CFAG12864B LCD" - depends on X86 - depends on FB - depends on KS0108 - select FB_SYSMEM_HELPERS - default n - help - If you have a Crystalfontz 128x64 2-color LCD, cfag12864b Series, - say Y. You also need the ks0108 LCD Controller driver. - - For help about how to wire your LCD to the parallel port, - check Documentation/admin-guide/auxdisplay/cfag12864b.rst - - Depends on the x86 arch and the framebuffer support. - - The LCD framebuffer driver can be attached to a console. - It will work fine. However, you can't attach it to the fbdev driver - of the xorg server. - - To compile this as a module, choose M here: - the modules will be called cfag12864b and cfag12864bfb. - - If unsure, say N. - -config CFAG12864B_RATE - int "Refresh rate (hertz)" - depends on CFAG12864B - default "20" - help - Refresh rate of the LCD. - - As the LCD is not memory mapped, the driver has to make the work by - software. This means you should be careful setting this value higher. - If your CPUs are really slow or you feel the system is slowed down, - decrease the value. - - Be careful modifying this value to a very high value: - You can freeze the computer, or the LCD maybe can't draw as fast as you - are requesting. - - If you don't know what I'm talking about, ignore it. - - If you compile this as a module, you can still override this - value using the module parameters. - -config IMG_ASCII_LCD - tristate "Imagination Technologies ASCII LCD Display" - depends on HAS_IOMEM - default y if MIPS_MALTA - select MFD_SYSCON - select LINEDISP - help - Enable this to support the simple ASCII LCD displays found on - development boards such as the MIPS Boston, MIPS Malta & MIPS SEAD3 - from Imagination Technologies. - -config HT16K33 - tristate "Holtek Ht16K33 LED controller with keyscan" - depends on FB && I2C && INPUT - select FB_SYSMEM_HELPERS - select INPUT_MATRIXKMAP - select FB_BACKLIGHT - select NEW_LEDS - select LEDS_CLASS - select LINEDISP - help - Say yes here to add support for Holtek HT16K33, RAM mapping 16*8 - LED controller driver with keyscan. - -config MAX6959 - tristate "Maxim MAX6958/6959 7-segment LED controller" - depends on I2C - select REGMAP_I2C - select LINEDISP - help - If you say yes here you get support for the following Maxim chips - (I2C 7-segment LED display controller): - - MAX6958 - - MAX6959 (input support) - - This driver can also be built as a module. If so, the module - will be called max6959. - config LCD2S tristate "lcd2s 20x4 character display over I2C console" depends on I2C @@ -201,27 +59,6 @@ config LCD2S is a simple single color character display. You have to connect it to an I2C bus. -config ARM_CHARLCD - bool "ARM Ltd. Character LCD Driver" - depends on PLAT_VERSATILE - help - This is a driver for the character LCD found on the ARM Ltd. - Versatile and RealView Platform Baseboards. It doesn't do - very much more than display the text "ARM Linux" on the first - line and the Linux version on the second line, but that's - still useful. - -config SEG_LED_GPIO - tristate "Generic 7-segment LED display" - depends on GPIOLIB || COMPILE_TEST - select LINEDISP - help - This driver supports a generic 7-segment LED display made up - of GPIO pins connected to the individual segments. - - This driver can also be built as a module. If so, the module - will be called seg-led-gpio. - menuconfig PARPORT_PANEL tristate "Parallel port LCD/Keypad Panel support" depends on PARPORT @@ -480,7 +317,6 @@ endif # PARPORT_PANEL config PANEL_CHANGE_MESSAGE bool "Change LCD initialization message ?" depends on CHARLCD - default "n" help This allows you to replace the boot message indicating the kernel version and the driver version with a custom message. This is useful on appliances @@ -529,8 +365,184 @@ choice endchoice +# +# Samsung KS0108 LCD controller section +# +config KS0108 + tristate "KS0108 LCD Controller" + depends on PARPORT_PC + help + If you have a LCD controlled by one or more KS0108 + controllers, say Y. You will need also another more specific + driver for your LCD. + + Depends on Parallel Port support. If you say Y at + parport, you will be able to compile this as a module (M) + and built-in as well (Y). + + To compile this as a module, choose M here: + the module will be called ks0108. + + If unsure, say N. + +config KS0108_PORT + hex "Parallel port where the LCD is connected" + depends on KS0108 + default 0x378 + help + The address of the parallel port where the LCD is connected. + + The first standard parallel port address is 0x378. + The second standard parallel port address is 0x278. + The third standard parallel port address is 0x3BC. + + You can specify a different address if you need. + + If you don't know what I'm talking about, load the parport module, + and execute "dmesg" or "cat /proc/ioports". You can see there how + many parallel ports are present and which address each one has. + + Usually you only need to use 0x378. + + If you compile this as a module, you can still override this + using the module parameters. + +config KS0108_DELAY + int "Delay between each control writing (microseconds)" + depends on KS0108 + default "2" + help + Amount of time the ks0108 should wait between each control write + to the parallel port. + + If your LCD seems to miss random writings, increment this. + + If you don't know what I'm talking about, ignore it. + + If you compile this as a module, you can still override this + value using the module parameters. + +config CFAG12864B + tristate "CFAG12864B LCD" + depends on X86 + depends on FB + depends on KS0108 + select FB_SYSMEM_HELPERS + help + If you have a Crystalfontz 128x64 2-color LCD, cfag12864b Series, + say Y. You also need the ks0108 LCD Controller driver. + + For help about how to wire your LCD to the parallel port, + check Documentation/admin-guide/auxdisplay/cfag12864b.rst + + Depends on the x86 arch and the framebuffer support. + + The LCD framebuffer driver can be attached to a console. + It will work fine. However, you can't attach it to the fbdev driver + of the xorg server. + + To compile this as a module, choose M here: + the modules will be called cfag12864b and cfag12864bfb. + + If unsure, say N. + +config CFAG12864B_RATE + int "Refresh rate (hertz)" + depends on CFAG12864B + default "20" + help + Refresh rate of the LCD. + + As the LCD is not memory mapped, the driver has to make the work by + software. This means you should be careful setting this value higher. + If your CPUs are really slow or you feel the system is slowed down, + decrease the value. + + Be careful modifying this value to a very high value: + You can freeze the computer, or the LCD maybe can't draw as fast as you + are requesting. + + If you don't know what I'm talking about, ignore it. + + If you compile this as a module, you can still override this + value using the module parameters. + +# +# Single character line display section +# +config LINEDISP + tristate "Character line display core support" if COMPILE_TEST + help + This is the core support for single-line character displays, to be + selected by drivers that use it. + +config IMG_ASCII_LCD + tristate "Imagination Technologies ASCII LCD Display" + depends on HAS_IOMEM + default y if MIPS_MALTA + select MFD_SYSCON + select LINEDISP + help + Enable this to support the simple ASCII LCD displays found on + development boards such as the MIPS Boston, MIPS Malta & MIPS SEAD3 + from Imagination Technologies. + +config HT16K33 + tristate "Holtek Ht16K33 LED controller with keyscan" + depends on FB && I2C && INPUT + select FB_SYSMEM_HELPERS + select INPUT_MATRIXKMAP + select FB_BACKLIGHT + select NEW_LEDS + select LEDS_CLASS + select LINEDISP + help + Say yes here to add support for Holtek HT16K33, RAM mapping 16*8 + LED controller driver with keyscan. + +config MAX6959 + tristate "Maxim MAX6958/6959 7-segment LED controller" + depends on I2C + select REGMAP_I2C + select LINEDISP + help + If you say yes here you get support for the following Maxim chips + (I2C 7-segment LED display controller): + - MAX6958 + - MAX6959 (input support) + + This driver can also be built as a module. If so, the module + will be called max6959. + +config SEG_LED_GPIO + tristate "Generic 7-segment LED display" + depends on GPIOLIB || COMPILE_TEST + select LINEDISP + help + This driver supports a generic 7-segment LED display made up + of GPIO pins connected to the individual segments. + + This driver can also be built as a module. If so, the module + will be called seg-led-gpio. + +# +# Character LCD with non-conforming interface section +# +config ARM_CHARLCD + bool "ARM Ltd. Character LCD Driver" + depends on PLAT_VERSATILE + help + This is a driver for the character LCD found on the ARM Ltd. + Versatile and RealView Platform Baseboards. It doesn't do + very much more than display the text "ARM Linux" on the first + line and the Linux version on the second line, but that's + still useful. + endif # AUXDISPLAY +# +# Deprecated options +# config PANEL tristate "Parallel port LCD/Keypad Panel support (OLD OPTION)" depends on PARPORT diff --git a/drivers/auxdisplay/Makefile b/drivers/auxdisplay/Makefile index 4a8ea41b0550..f5c13ed1cd4f 100644 --- a/drivers/auxdisplay/Makefile +++ b/drivers/auxdisplay/Makefile @@ -3,16 +3,16 @@ # Makefile for the kernel auxiliary displays device drivers. # -obj-$(CONFIG_CHARLCD) += charlcd.o -obj-$(CONFIG_HD44780_COMMON) += hd44780_common.o obj-$(CONFIG_ARM_CHARLCD) += arm-charlcd.o -obj-$(CONFIG_KS0108) += ks0108.o obj-$(CONFIG_CFAG12864B) += cfag12864b.o cfag12864bfb.o -obj-$(CONFIG_IMG_ASCII_LCD) += img-ascii-lcd.o +obj-$(CONFIG_CHARLCD) += charlcd.o +obj-$(CONFIG_HD44780_COMMON) += hd44780_common.o obj-$(CONFIG_HD44780) += hd44780.o obj-$(CONFIG_HT16K33) += ht16k33.o -obj-$(CONFIG_PARPORT_PANEL) += panel.o +obj-$(CONFIG_IMG_ASCII_LCD) += img-ascii-lcd.o +obj-$(CONFIG_KS0108) += ks0108.o obj-$(CONFIG_LCD2S) += lcd2s.o obj-$(CONFIG_LINEDISP) += line-display.o obj-$(CONFIG_MAX6959) += max6959.o +obj-$(CONFIG_PARPORT_PANEL) += panel.o obj-$(CONFIG_SEG_LED_GPIO) += seg-led-gpio.o diff --git a/drivers/auxdisplay/charlcd.c b/drivers/auxdisplay/charlcd.c index 6d309e4971b6..bb9463814454 100644 --- a/drivers/auxdisplay/charlcd.c +++ b/drivers/auxdisplay/charlcd.c @@ -17,7 +17,9 @@ #include <linux/uaccess.h> #include <linux/workqueue.h> +#ifndef CONFIG_PANEL_BOOT_MESSAGE #include <generated/utsrelease.h> +#endif #include "charlcd.h" @@ -678,4 +680,5 @@ int charlcd_unregister(struct charlcd *lcd) } EXPORT_SYMBOL_GPL(charlcd_unregister); +MODULE_DESCRIPTION("Character LCD core support"); MODULE_LICENSE("GPL"); diff --git a/drivers/auxdisplay/seg-led-gpio.c b/drivers/auxdisplay/seg-led-gpio.c index 35a8dbb1e9d2..183ab3011cbb 100644 --- a/drivers/auxdisplay/seg-led-gpio.c +++ b/drivers/auxdisplay/seg-led-gpio.c @@ -81,14 +81,12 @@ static int seg_led_probe(struct platform_device *pdev) return linedisp_register(&priv->linedisp, dev, 1, &seg_led_linedisp_ops); } -static int seg_led_remove(struct platform_device *pdev) +static void seg_led_remove(struct platform_device *pdev) { struct seg_led_priv *priv = platform_get_drvdata(pdev); cancel_delayed_work_sync(&priv->work); linedisp_unregister(&priv->linedisp); - - return 0; } static const struct of_device_id seg_led_of_match[] = { @@ -99,7 +97,7 @@ MODULE_DEVICE_TABLE(of, seg_led_of_match); static struct platform_driver seg_led_driver = { .probe = seg_led_probe, - .remove = seg_led_remove, + .remove_new = seg_led_remove, .driver = { .name = "seg-led-gpio", .of_match_table = seg_led_of_match, diff --git a/drivers/base/arch_topology.c b/drivers/base/arch_topology.c index 024b78a0cfc1..0248912ff687 100644 --- a/drivers/base/arch_topology.c +++ b/drivers/base/arch_topology.c @@ -22,7 +22,7 @@ #include <linux/units.h> #define CREATE_TRACE_POINTS -#include <trace/events/thermal_pressure.h> +#include <trace/events/hw_pressure.h> static DEFINE_PER_CPU(struct scale_freq_data __rcu *, sft_data); static struct cpumask scale_freq_counters_mask; @@ -160,26 +160,26 @@ void topology_set_cpu_scale(unsigned int cpu, unsigned long capacity) per_cpu(cpu_scale, cpu) = capacity; } -DEFINE_PER_CPU(unsigned long, thermal_pressure); +DEFINE_PER_CPU(unsigned long, hw_pressure); /** - * topology_update_thermal_pressure() - Update thermal pressure for CPUs + * topology_update_hw_pressure() - Update HW pressure for CPUs * @cpus : The related CPUs for which capacity has been reduced * @capped_freq : The maximum allowed frequency that CPUs can run at * - * Update the value of thermal pressure for all @cpus in the mask. The + * Update the value of HW pressure for all @cpus in the mask. The * cpumask should include all (online+offline) affected CPUs, to avoid * operating on stale data when hot-plug is used for some CPUs. The * @capped_freq reflects the currently allowed max CPUs frequency due to - * thermal capping. It might be also a boost frequency value, which is bigger + * HW capping. It might be also a boost frequency value, which is bigger * than the internal 'capacity_freq_ref' max frequency. In such case the * pressure value should simply be removed, since this is an indication that - * there is no thermal throttling. The @capped_freq must be provided in kHz. + * there is no HW throttling. The @capped_freq must be provided in kHz. */ -void topology_update_thermal_pressure(const struct cpumask *cpus, +void topology_update_hw_pressure(const struct cpumask *cpus, unsigned long capped_freq) { - unsigned long max_capacity, capacity, th_pressure; + unsigned long max_capacity, capacity, hw_pressure; u32 max_freq; int cpu; @@ -189,21 +189,21 @@ void topology_update_thermal_pressure(const struct cpumask *cpus, /* * Handle properly the boost frequencies, which should simply clean - * the thermal pressure value. + * the HW pressure value. */ if (max_freq <= capped_freq) capacity = max_capacity; else capacity = mult_frac(max_capacity, capped_freq, max_freq); - th_pressure = max_capacity - capacity; + hw_pressure = max_capacity - capacity; - trace_thermal_pressure_update(cpu, th_pressure); + trace_hw_pressure_update(cpu, hw_pressure); for_each_cpu(cpu, cpus) - WRITE_ONCE(per_cpu(thermal_pressure, cpu), th_pressure); + WRITE_ONCE(per_cpu(hw_pressure, cpu), hw_pressure); } -EXPORT_SYMBOL_GPL(topology_update_thermal_pressure); +EXPORT_SYMBOL_GPL(topology_update_hw_pressure); static ssize_t cpu_capacity_show(struct device *dev, struct device_attribute *attr, diff --git a/drivers/block/brd.c b/drivers/block/brd.c index e322cef6596b..b900fe9e0030 100644 --- a/drivers/block/brd.c +++ b/drivers/block/brd.c @@ -29,10 +29,7 @@ /* * Each block ramdisk device has a xarray brd_pages of pages that stores - * the pages containing the block device's contents. A brd page's ->index is - * its offset in PAGE_SIZE units. This is similar to, but in no way connected - * with, the kernel's pagecache or buffer cache (which sit above our block - * device). + * the pages containing the block device's contents. */ struct brd_device { int brd_number; @@ -51,15 +48,7 @@ struct brd_device { */ static struct page *brd_lookup_page(struct brd_device *brd, sector_t sector) { - pgoff_t idx; - struct page *page; - - idx = sector >> PAGE_SECTORS_SHIFT; /* sector to page index */ - page = xa_load(&brd->brd_pages, idx); - - BUG_ON(page && page->index != idx); - - return page; + return xa_load(&brd->brd_pages, sector >> PAGE_SECTORS_SHIFT); } /* @@ -67,8 +56,8 @@ static struct page *brd_lookup_page(struct brd_device *brd, sector_t sector) */ static int brd_insert_page(struct brd_device *brd, sector_t sector, gfp_t gfp) { - pgoff_t idx; - struct page *page, *cur; + pgoff_t idx = sector >> PAGE_SECTORS_SHIFT; + struct page *page; int ret = 0; page = brd_lookup_page(brd, sector); @@ -80,23 +69,16 @@ static int brd_insert_page(struct brd_device *brd, sector_t sector, gfp_t gfp) return -ENOMEM; xa_lock(&brd->brd_pages); - - idx = sector >> PAGE_SECTORS_SHIFT; - page->index = idx; - - cur = __xa_cmpxchg(&brd->brd_pages, idx, NULL, page, gfp); - - if (unlikely(cur)) { - __free_page(page); - ret = xa_err(cur); - if (!ret && (cur->index != idx)) - ret = -EIO; - } else { + ret = __xa_insert(&brd->brd_pages, idx, page, gfp); + if (!ret) brd->brd_nr_pages++; - } - xa_unlock(&brd->brd_pages); + if (ret < 0) { + __free_page(page); + if (ret == -EBUSY) + ret = 0; + } return ret; } diff --git a/drivers/block/null_blk/main.c b/drivers/block/null_blk/main.c index ed33cf7192d2..4005a8b685e8 100644 --- a/drivers/block/null_blk/main.c +++ b/drivers/block/null_blk/main.c @@ -225,6 +225,10 @@ static unsigned long g_cache_size; module_param_named(cache_size, g_cache_size, ulong, 0444); MODULE_PARM_DESC(mbps, "Cache size in MiB for memory-backed device. Default: 0 (none)"); +static bool g_fua = true; +module_param_named(fua, g_fua, bool, 0444); +MODULE_PARM_DESC(zoned, "Enable/disable FUA support when cache_size is used. Default: true"); + static unsigned int g_mbps; module_param_named(mbps, g_mbps, uint, 0444); MODULE_PARM_DESC(mbps, "Limit maximum bandwidth (in MiB/s). Default: 0 (no limit)"); @@ -253,6 +257,11 @@ static unsigned int g_zone_max_active; module_param_named(zone_max_active, g_zone_max_active, uint, 0444); MODULE_PARM_DESC(zone_max_active, "Maximum number of active zones when block device is zoned. Default: 0 (no limit)"); +static int g_zone_append_max_sectors = INT_MAX; +module_param_named(zone_append_max_sectors, g_zone_append_max_sectors, int, 0444); +MODULE_PARM_DESC(zone_append_max_sectors, + "Maximum size of a zone append command (in 512B sectors). Specify 0 for zone append emulation"); + static struct nullb_device *null_alloc_dev(void); static void null_free_dev(struct nullb_device *dev); static void null_del_dev(struct nullb *nullb); @@ -436,10 +445,12 @@ NULLB_DEVICE_ATTR(zone_capacity, ulong, NULL); NULLB_DEVICE_ATTR(zone_nr_conv, uint, NULL); NULLB_DEVICE_ATTR(zone_max_open, uint, NULL); NULLB_DEVICE_ATTR(zone_max_active, uint, NULL); +NULLB_DEVICE_ATTR(zone_append_max_sectors, uint, NULL); NULLB_DEVICE_ATTR(virt_boundary, bool, NULL); NULLB_DEVICE_ATTR(no_sched, bool, NULL); NULLB_DEVICE_ATTR(shared_tags, bool, NULL); NULLB_DEVICE_ATTR(shared_tag_bitmap, bool, NULL); +NULLB_DEVICE_ATTR(fua, bool, NULL); static ssize_t nullb_device_power_show(struct config_item *item, char *page) { @@ -580,12 +591,14 @@ static struct configfs_attribute *nullb_device_attrs[] = { &nullb_device_attr_zone_nr_conv, &nullb_device_attr_zone_max_open, &nullb_device_attr_zone_max_active, + &nullb_device_attr_zone_append_max_sectors, &nullb_device_attr_zone_readonly, &nullb_device_attr_zone_offline, &nullb_device_attr_virt_boundary, &nullb_device_attr_no_sched, &nullb_device_attr_shared_tags, &nullb_device_attr_shared_tag_bitmap, + &nullb_device_attr_fua, NULL, }; @@ -664,14 +677,14 @@ nullb_group_drop_item(struct config_group *group, struct config_item *item) static ssize_t memb_group_features_show(struct config_item *item, char *page) { return snprintf(page, PAGE_SIZE, - "badblocks,blocking,blocksize,cache_size," + "badblocks,blocking,blocksize,cache_size,fua," "completion_nsec,discard,home_node,hw_queue_depth," "irqmode,max_sectors,mbps,memory_backed,no_sched," "poll_queues,power,queue_mode,shared_tag_bitmap," "shared_tags,size,submit_queues,use_per_node_hctx," "virt_boundary,zoned,zone_capacity,zone_max_active," "zone_max_open,zone_nr_conv,zone_offline,zone_readonly," - "zone_size\n"); + "zone_size,zone_append_max_sectors\n"); } CONFIGFS_ATTR_RO(memb_group_, features); @@ -751,10 +764,13 @@ static struct nullb_device *null_alloc_dev(void) dev->zone_nr_conv = g_zone_nr_conv; dev->zone_max_open = g_zone_max_open; dev->zone_max_active = g_zone_max_active; + dev->zone_append_max_sectors = g_zone_append_max_sectors; dev->virt_boundary = g_virt_boundary; dev->no_sched = g_no_sched; dev->shared_tags = g_shared_tags; dev->shared_tag_bitmap = g_shared_tag_bitmap; + dev->fua = g_fua; + return dev; } @@ -1151,7 +1167,7 @@ blk_status_t null_handle_discard(struct nullb_device *dev, return BLK_STS_OK; } -static int null_handle_flush(struct nullb *nullb) +static blk_status_t null_handle_flush(struct nullb *nullb) { int err; @@ -1168,7 +1184,7 @@ static int null_handle_flush(struct nullb *nullb) WARN_ON(!radix_tree_empty(&nullb->dev->cache)); spin_unlock_irq(&nullb->lock); - return err; + return errno_to_blk_status(err); } static int null_transfer(struct nullb *nullb, struct page *page, @@ -1206,7 +1222,7 @@ static int null_handle_rq(struct nullb_cmd *cmd) { struct request *rq = blk_mq_rq_from_pdu(cmd); struct nullb *nullb = cmd->nq->dev->nullb; - int err; + int err = 0; unsigned int len; sector_t sector = blk_rq_pos(rq); struct req_iterator iter; @@ -1218,15 +1234,13 @@ static int null_handle_rq(struct nullb_cmd *cmd) err = null_transfer(nullb, bvec.bv_page, len, bvec.bv_offset, op_is_write(req_op(rq)), sector, rq->cmd_flags & REQ_FUA); - if (err) { - spin_unlock_irq(&nullb->lock); - return err; - } + if (err) + break; sector += len >> SECTOR_SHIFT; } spin_unlock_irq(&nullb->lock); - return 0; + return errno_to_blk_status(err); } static inline blk_status_t null_handle_throttled(struct nullb_cmd *cmd) @@ -1273,8 +1287,8 @@ static inline blk_status_t null_handle_memory_backed(struct nullb_cmd *cmd, if (op == REQ_OP_DISCARD) return null_handle_discard(dev, sector, nr_sectors); - return errno_to_blk_status(null_handle_rq(cmd)); + return null_handle_rq(cmd); } static void nullb_zero_read_cmd_buffer(struct nullb_cmd *cmd) @@ -1343,7 +1357,7 @@ static void null_handle_cmd(struct nullb_cmd *cmd, sector_t sector, blk_status_t sts; if (op == REQ_OP_FLUSH) { - cmd->error = errno_to_blk_status(null_handle_flush(nullb)); + cmd->error = null_handle_flush(nullb); goto out; } @@ -1912,7 +1926,7 @@ static int null_add_dev(struct nullb_device *dev) if (dev->cache_size > 0) { set_bit(NULLB_DEV_FL_CACHE, &nullb->dev->flags); - blk_queue_write_cache(nullb->q, true, true); + blk_queue_write_cache(nullb->q, true, dev->fua); } nullb->q->queuedata = nullb; @@ -2113,10 +2127,13 @@ static void __exit null_exit(void) if (tag_set.ops) blk_mq_free_tag_set(&tag_set); + + mutex_destroy(&lock); } module_init(null_init); module_exit(null_exit); MODULE_AUTHOR("Jens Axboe <axboe@kernel.dk>"); +MODULE_DESCRIPTION("multi queue aware block test driver"); MODULE_LICENSE("GPL"); diff --git a/drivers/block/null_blk/null_blk.h b/drivers/block/null_blk/null_blk.h index 477b97746823..3234e6c85eed 100644 --- a/drivers/block/null_blk/null_blk.h +++ b/drivers/block/null_blk/null_blk.h @@ -82,6 +82,7 @@ struct nullb_device { unsigned int zone_nr_conv; /* number of conventional zones */ unsigned int zone_max_open; /* max number of open zones */ unsigned int zone_max_active; /* max number of active zones */ + unsigned int zone_append_max_sectors; /* Max sectors per zone append command */ unsigned int submit_queues; /* number of submission queues */ unsigned int prev_submit_queues; /* number of submission queues before change */ unsigned int poll_queues; /* number of IOPOLL submission queues */ @@ -104,6 +105,7 @@ struct nullb_device { bool no_sched; /* no IO scheduler for the device */ bool shared_tags; /* share tag set between devices for blk-mq */ bool shared_tag_bitmap; /* use hostwide shared tags */ + bool fua; /* Support FUA */ }; struct nullb { diff --git a/drivers/block/null_blk/zoned.c b/drivers/block/null_blk/zoned.c index 1689e2584104..5b5a63adacc1 100644 --- a/drivers/block/null_blk/zoned.c +++ b/drivers/block/null_blk/zoned.c @@ -9,6 +9,8 @@ #undef pr_fmt #define pr_fmt(fmt) "null_blk: " fmt +#define NULL_ZONE_INVALID_WP ((sector_t)-1) + static inline sector_t mb_to_sects(unsigned long mb) { return ((sector_t)mb * SZ_1M) >> SECTOR_SHIFT; @@ -19,18 +21,6 @@ static inline unsigned int null_zone_no(struct nullb_device *dev, sector_t sect) return sect >> ilog2(dev->zone_size_sects); } -static inline void null_lock_zone_res(struct nullb_device *dev) -{ - if (dev->need_zone_res_mgmt) - spin_lock_irq(&dev->zone_res_lock); -} - -static inline void null_unlock_zone_res(struct nullb_device *dev) -{ - if (dev->need_zone_res_mgmt) - spin_unlock_irq(&dev->zone_res_lock); -} - static inline void null_init_zone_lock(struct nullb_device *dev, struct nullb_zone *zone) { @@ -103,6 +93,11 @@ int null_init_zoned_dev(struct nullb_device *dev, dev->zone_nr_conv); } + dev->zone_append_max_sectors = + min(ALIGN_DOWN(dev->zone_append_max_sectors, + dev->blocksize >> SECTOR_SHIFT), + zone_capacity_sects); + /* Max active zones has to be < nbr of seq zones in order to be enforceable */ if (dev->zone_max_active >= dev->nr_zones - dev->zone_nr_conv) { dev->zone_max_active = 0; @@ -154,7 +149,7 @@ int null_init_zoned_dev(struct nullb_device *dev, lim->zoned = true; lim->chunk_sectors = dev->zone_size_sects; - lim->max_zone_append_sectors = dev->zone_size_sects; + lim->max_zone_append_sectors = dev->zone_append_max_sectors; lim->max_open_zones = dev->zone_max_open; lim->max_active_zones = dev->zone_max_active; return 0; @@ -163,11 +158,16 @@ int null_init_zoned_dev(struct nullb_device *dev, int null_register_zoned_dev(struct nullb *nullb) { struct request_queue *q = nullb->q; + struct gendisk *disk = nullb->disk; blk_queue_flag_set(QUEUE_FLAG_ZONE_RESETALL, q); - blk_queue_required_elevator_features(q, ELEVATOR_F_ZBD_SEQ_WRITE); - nullb->disk->nr_zones = bdev_nr_zones(nullb->disk->part0); - return blk_revalidate_disk_zones(nullb->disk, NULL); + disk->nr_zones = bdev_nr_zones(disk->part0); + + pr_info("%s: using %s zone append\n", + disk->disk_name, + queue_emulates_zone_append(q) ? "emulated" : "native"); + + return blk_revalidate_disk_zones(disk); } void null_free_zoned_dev(struct nullb_device *dev) @@ -241,35 +241,6 @@ size_t null_zone_valid_read_len(struct nullb *nullb, return (zone->wp - sector) << SECTOR_SHIFT; } -static blk_status_t __null_close_zone(struct nullb_device *dev, - struct nullb_zone *zone) -{ - switch (zone->cond) { - case BLK_ZONE_COND_CLOSED: - /* close operation on closed is not an error */ - return BLK_STS_OK; - case BLK_ZONE_COND_IMP_OPEN: - dev->nr_zones_imp_open--; - break; - case BLK_ZONE_COND_EXP_OPEN: - dev->nr_zones_exp_open--; - break; - case BLK_ZONE_COND_EMPTY: - case BLK_ZONE_COND_FULL: - default: - return BLK_STS_IOERR; - } - - if (zone->wp == zone->start) { - zone->cond = BLK_ZONE_COND_EMPTY; - } else { - zone->cond = BLK_ZONE_COND_CLOSED; - dev->nr_zones_closed++; - } - - return BLK_STS_OK; -} - static void null_close_imp_open_zone(struct nullb_device *dev) { struct nullb_zone *zone; @@ -286,7 +257,13 @@ static void null_close_imp_open_zone(struct nullb_device *dev) zno = dev->zone_nr_conv; if (zone->cond == BLK_ZONE_COND_IMP_OPEN) { - __null_close_zone(dev, zone); + dev->nr_zones_imp_open--; + if (zone->wp == zone->start) { + zone->cond = BLK_ZONE_COND_EMPTY; + } else { + zone->cond = BLK_ZONE_COND_CLOSED; + dev->nr_zones_closed++; + } dev->imp_close_zone_no = zno; return; } @@ -374,73 +351,73 @@ static blk_status_t null_zone_write(struct nullb_cmd *cmd, sector_t sector, null_lock_zone(dev, zone); - if (zone->cond == BLK_ZONE_COND_FULL || - zone->cond == BLK_ZONE_COND_READONLY || - zone->cond == BLK_ZONE_COND_OFFLINE) { - /* Cannot write to the zone */ - ret = BLK_STS_IOERR; - goto unlock; - } - /* - * Regular writes must be at the write pointer position. - * Zone append writes are automatically issued at the write - * pointer and the position returned using the request or BIO - * sector. + * Regular writes must be at the write pointer position. Zone append + * writes are automatically issued at the write pointer and the position + * returned using the request sector. Note that we do not check the zone + * condition because for FULL, READONLY and OFFLINE zones, the sector + * check against the zone write pointer will always result in failing + * the command. */ if (append) { + if (WARN_ON_ONCE(!dev->zone_append_max_sectors) || + zone->wp == NULL_ZONE_INVALID_WP) { + ret = BLK_STS_IOERR; + goto unlock_zone; + } sector = zone->wp; blk_mq_rq_from_pdu(cmd)->__sector = sector; - } else if (sector != zone->wp) { - ret = BLK_STS_IOERR; - goto unlock; } - if (zone->wp + nr_sectors > zone->start + zone->capacity) { + if (sector != zone->wp || + zone->wp + nr_sectors > zone->start + zone->capacity) { ret = BLK_STS_IOERR; - goto unlock; + goto unlock_zone; } if (zone->cond == BLK_ZONE_COND_CLOSED || zone->cond == BLK_ZONE_COND_EMPTY) { - null_lock_zone_res(dev); + if (dev->need_zone_res_mgmt) { + spin_lock(&dev->zone_res_lock); - ret = null_check_zone_resources(dev, zone); - if (ret != BLK_STS_OK) { - null_unlock_zone_res(dev); - goto unlock; - } - if (zone->cond == BLK_ZONE_COND_CLOSED) { - dev->nr_zones_closed--; - dev->nr_zones_imp_open++; - } else if (zone->cond == BLK_ZONE_COND_EMPTY) { - dev->nr_zones_imp_open++; - } + ret = null_check_zone_resources(dev, zone); + if (ret != BLK_STS_OK) { + spin_unlock(&dev->zone_res_lock); + goto unlock_zone; + } + if (zone->cond == BLK_ZONE_COND_CLOSED) { + dev->nr_zones_closed--; + dev->nr_zones_imp_open++; + } else if (zone->cond == BLK_ZONE_COND_EMPTY) { + dev->nr_zones_imp_open++; + } - if (zone->cond != BLK_ZONE_COND_EXP_OPEN) - zone->cond = BLK_ZONE_COND_IMP_OPEN; + spin_unlock(&dev->zone_res_lock); + } - null_unlock_zone_res(dev); + zone->cond = BLK_ZONE_COND_IMP_OPEN; } ret = null_process_cmd(cmd, REQ_OP_WRITE, sector, nr_sectors); if (ret != BLK_STS_OK) - goto unlock; + goto unlock_zone; zone->wp += nr_sectors; if (zone->wp == zone->start + zone->capacity) { - null_lock_zone_res(dev); - if (zone->cond == BLK_ZONE_COND_EXP_OPEN) - dev->nr_zones_exp_open--; - else if (zone->cond == BLK_ZONE_COND_IMP_OPEN) - dev->nr_zones_imp_open--; + if (dev->need_zone_res_mgmt) { + spin_lock(&dev->zone_res_lock); + if (zone->cond == BLK_ZONE_COND_EXP_OPEN) + dev->nr_zones_exp_open--; + else if (zone->cond == BLK_ZONE_COND_IMP_OPEN) + dev->nr_zones_imp_open--; + spin_unlock(&dev->zone_res_lock); + } zone->cond = BLK_ZONE_COND_FULL; - null_unlock_zone_res(dev); } ret = BLK_STS_OK; -unlock: +unlock_zone: null_unlock_zone(dev, zone); return ret; @@ -454,54 +431,100 @@ static blk_status_t null_open_zone(struct nullb_device *dev, if (zone->type == BLK_ZONE_TYPE_CONVENTIONAL) return BLK_STS_IOERR; - null_lock_zone_res(dev); - switch (zone->cond) { case BLK_ZONE_COND_EXP_OPEN: - /* open operation on exp open is not an error */ - goto unlock; + /* Open operation on exp open is not an error */ + return BLK_STS_OK; case BLK_ZONE_COND_EMPTY: - ret = null_check_zone_resources(dev, zone); - if (ret != BLK_STS_OK) - goto unlock; - break; case BLK_ZONE_COND_IMP_OPEN: - dev->nr_zones_imp_open--; - break; case BLK_ZONE_COND_CLOSED: - ret = null_check_zone_resources(dev, zone); - if (ret != BLK_STS_OK) - goto unlock; - dev->nr_zones_closed--; break; case BLK_ZONE_COND_FULL: default: - ret = BLK_STS_IOERR; - goto unlock; + return BLK_STS_IOERR; } - zone->cond = BLK_ZONE_COND_EXP_OPEN; - dev->nr_zones_exp_open++; + if (dev->need_zone_res_mgmt) { + spin_lock(&dev->zone_res_lock); -unlock: - null_unlock_zone_res(dev); + switch (zone->cond) { + case BLK_ZONE_COND_EMPTY: + ret = null_check_zone_resources(dev, zone); + if (ret != BLK_STS_OK) { + spin_unlock(&dev->zone_res_lock); + return ret; + } + break; + case BLK_ZONE_COND_IMP_OPEN: + dev->nr_zones_imp_open--; + break; + case BLK_ZONE_COND_CLOSED: + ret = null_check_zone_resources(dev, zone); + if (ret != BLK_STS_OK) { + spin_unlock(&dev->zone_res_lock); + return ret; + } + dev->nr_zones_closed--; + break; + default: + break; + } - return ret; + dev->nr_zones_exp_open++; + + spin_unlock(&dev->zone_res_lock); + } + + zone->cond = BLK_ZONE_COND_EXP_OPEN; + + return BLK_STS_OK; } static blk_status_t null_close_zone(struct nullb_device *dev, struct nullb_zone *zone) { - blk_status_t ret; - if (zone->type == BLK_ZONE_TYPE_CONVENTIONAL) return BLK_STS_IOERR; - null_lock_zone_res(dev); - ret = __null_close_zone(dev, zone); - null_unlock_zone_res(dev); + switch (zone->cond) { + case BLK_ZONE_COND_CLOSED: + /* close operation on closed is not an error */ + return BLK_STS_OK; + case BLK_ZONE_COND_IMP_OPEN: + case BLK_ZONE_COND_EXP_OPEN: + break; + case BLK_ZONE_COND_EMPTY: + case BLK_ZONE_COND_FULL: + default: + return BLK_STS_IOERR; + } + + if (dev->need_zone_res_mgmt) { + spin_lock(&dev->zone_res_lock); - return ret; + switch (zone->cond) { + case BLK_ZONE_COND_IMP_OPEN: + dev->nr_zones_imp_open--; + break; + case BLK_ZONE_COND_EXP_OPEN: + dev->nr_zones_exp_open--; + break; + default: + break; + } + + if (zone->wp > zone->start) + dev->nr_zones_closed++; + + spin_unlock(&dev->zone_res_lock); + } + + if (zone->wp == zone->start) + zone->cond = BLK_ZONE_COND_EMPTY; + else + zone->cond = BLK_ZONE_COND_CLOSED; + + return BLK_STS_OK; } static blk_status_t null_finish_zone(struct nullb_device *dev, @@ -512,41 +535,47 @@ static blk_status_t null_finish_zone(struct nullb_device *dev, if (zone->type == BLK_ZONE_TYPE_CONVENTIONAL) return BLK_STS_IOERR; - null_lock_zone_res(dev); + if (dev->need_zone_res_mgmt) { + spin_lock(&dev->zone_res_lock); - switch (zone->cond) { - case BLK_ZONE_COND_FULL: - /* finish operation on full is not an error */ - goto unlock; - case BLK_ZONE_COND_EMPTY: - ret = null_check_zone_resources(dev, zone); - if (ret != BLK_STS_OK) - goto unlock; - break; - case BLK_ZONE_COND_IMP_OPEN: - dev->nr_zones_imp_open--; - break; - case BLK_ZONE_COND_EXP_OPEN: - dev->nr_zones_exp_open--; - break; - case BLK_ZONE_COND_CLOSED: - ret = null_check_zone_resources(dev, zone); - if (ret != BLK_STS_OK) - goto unlock; - dev->nr_zones_closed--; - break; - default: - ret = BLK_STS_IOERR; - goto unlock; + switch (zone->cond) { + case BLK_ZONE_COND_FULL: + /* Finish operation on full is not an error */ + spin_unlock(&dev->zone_res_lock); + return BLK_STS_OK; + case BLK_ZONE_COND_EMPTY: + ret = null_check_zone_resources(dev, zone); + if (ret != BLK_STS_OK) { + spin_unlock(&dev->zone_res_lock); + return ret; + } + break; + case BLK_ZONE_COND_IMP_OPEN: + dev->nr_zones_imp_open--; + break; + case BLK_ZONE_COND_EXP_OPEN: + dev->nr_zones_exp_open--; + break; + case BLK_ZONE_COND_CLOSED: + ret = null_check_zone_resources(dev, zone); + if (ret != BLK_STS_OK) { + spin_unlock(&dev->zone_res_lock); + return ret; + } + dev->nr_zones_closed--; + break; + default: + spin_unlock(&dev->zone_res_lock); + return BLK_STS_IOERR; + } + + spin_unlock(&dev->zone_res_lock); } zone->cond = BLK_ZONE_COND_FULL; zone->wp = zone->start + zone->len; -unlock: - null_unlock_zone_res(dev); - - return ret; + return BLK_STS_OK; } static blk_status_t null_reset_zone(struct nullb_device *dev, @@ -555,34 +584,33 @@ static blk_status_t null_reset_zone(struct nullb_device *dev, if (zone->type == BLK_ZONE_TYPE_CONVENTIONAL) return BLK_STS_IOERR; - null_lock_zone_res(dev); + if (dev->need_zone_res_mgmt) { + spin_lock(&dev->zone_res_lock); - switch (zone->cond) { - case BLK_ZONE_COND_EMPTY: - /* reset operation on empty is not an error */ - null_unlock_zone_res(dev); - return BLK_STS_OK; - case BLK_ZONE_COND_IMP_OPEN: - dev->nr_zones_imp_open--; - break; - case BLK_ZONE_COND_EXP_OPEN: - dev->nr_zones_exp_open--; - break; - case BLK_ZONE_COND_CLOSED: - dev->nr_zones_closed--; - break; - case BLK_ZONE_COND_FULL: - break; - default: - null_unlock_zone_res(dev); - return BLK_STS_IOERR; + switch (zone->cond) { + case BLK_ZONE_COND_IMP_OPEN: + dev->nr_zones_imp_open--; + break; + case BLK_ZONE_COND_EXP_OPEN: + dev->nr_zones_exp_open--; + break; + case BLK_ZONE_COND_CLOSED: + dev->nr_zones_closed--; + break; + case BLK_ZONE_COND_EMPTY: + case BLK_ZONE_COND_FULL: + break; + default: + spin_unlock(&dev->zone_res_lock); + return BLK_STS_IOERR; + } + + spin_unlock(&dev->zone_res_lock); } zone->cond = BLK_ZONE_COND_EMPTY; zone->wp = zone->start; - null_unlock_zone_res(dev); - if (dev->memory_backed) return null_handle_discard(dev, zone->start, zone->len); @@ -711,7 +739,7 @@ static void null_set_zone_cond(struct nullb_device *dev, zone->cond != BLK_ZONE_COND_OFFLINE) null_finish_zone(dev, zone); zone->cond = cond; - zone->wp = (sector_t)-1; + zone->wp = NULL_ZONE_INVALID_WP; } null_unlock_zone(dev, zone); diff --git a/drivers/block/ublk_drv.c b/drivers/block/ublk_drv.c index 374e4efa8759..176657dce3e3 100644 --- a/drivers/block/ublk_drv.c +++ b/drivers/block/ublk_drv.c @@ -221,7 +221,7 @@ static int ublk_get_nr_zones(const struct ublk_device *ub) static int ublk_revalidate_disk_zones(struct ublk_device *ub) { - return blk_revalidate_disk_zones(ub->ub_disk, NULL); + return blk_revalidate_disk_zones(ub->ub_disk); } static int ublk_dev_param_zoned_validate(const struct ublk_device *ub) @@ -249,8 +249,7 @@ static int ublk_dev_param_zoned_validate(const struct ublk_device *ub) static void ublk_dev_param_zoned_apply(struct ublk_device *ub) { blk_queue_flag_set(QUEUE_FLAG_ZONE_RESETALL, ub->ub_disk->queue); - blk_queue_required_elevator_features(ub->ub_disk->queue, - ELEVATOR_F_ZBD_SEQ_WRITE); + ub->ub_disk->nr_zones = ublk_get_nr_zones(ub); } diff --git a/drivers/block/virtio_blk.c b/drivers/block/virtio_blk.c index 42dea7601d87..c1af0a7d56c8 100644 --- a/drivers/block/virtio_blk.c +++ b/drivers/block/virtio_blk.c @@ -1543,7 +1543,7 @@ static int virtblk_probe(struct virtio_device *vdev) */ if (IS_ENABLED(CONFIG_BLK_DEV_ZONED) && lim.zoned) { blk_queue_flag_set(QUEUE_FLAG_ZONE_RESETALL, vblk->disk->queue); - err = blk_revalidate_disk_zones(vblk->disk, NULL); + err = blk_revalidate_disk_zones(vblk->disk); if (err) goto out_cleanup_disk; } diff --git a/drivers/bluetooth/btqca.c b/drivers/bluetooth/btqca.c index 216826c31ee3..638074992c82 100644 --- a/drivers/bluetooth/btqca.c +++ b/drivers/bluetooth/btqca.c @@ -15,8 +15,6 @@ #define VERSION "0.1" -#define QCA_BDADDR_DEFAULT (&(bdaddr_t) {{ 0xad, 0x5a, 0x00, 0x00, 0x00, 0x00 }}) - int qca_read_soc_version(struct hci_dev *hdev, struct qca_btsoc_version *ver, enum qca_btsoc_type soc_type) { @@ -101,7 +99,8 @@ static int qca_read_fw_build_info(struct hci_dev *hdev) { struct sk_buff *skb; struct edl_event_hdr *edl; - char cmd, build_label[QCA_FW_BUILD_VER_LEN]; + char *build_label; + char cmd; int build_lbl_len, err = 0; bt_dev_dbg(hdev, "QCA read fw build info"); @@ -116,6 +115,11 @@ static int qca_read_fw_build_info(struct hci_dev *hdev) return err; } + if (skb->len < sizeof(*edl)) { + err = -EILSEQ; + goto out; + } + edl = (struct edl_event_hdr *)(skb->data); if (!edl) { bt_dev_err(hdev, "QCA read fw build info with no header"); @@ -131,14 +135,25 @@ static int qca_read_fw_build_info(struct hci_dev *hdev) goto out; } + if (skb->len < sizeof(*edl) + 1) { + err = -EILSEQ; + goto out; + } + build_lbl_len = edl->data[0]; - if (build_lbl_len <= QCA_FW_BUILD_VER_LEN - 1) { - memcpy(build_label, edl->data + 1, build_lbl_len); - *(build_label + build_lbl_len) = '\0'; + + if (skb->len < sizeof(*edl) + 1 + build_lbl_len) { + err = -EILSEQ; + goto out; } + build_label = kstrndup(&edl->data[1], build_lbl_len, GFP_KERNEL); + if (!build_label) + goto out; + hci_set_fw_info(hdev, "%s", build_label); + kfree(build_label); out: kfree_skb(skb); return err; @@ -237,6 +252,11 @@ static int qca_read_fw_board_id(struct hci_dev *hdev, u16 *bid) goto out; } + if (skb->len < 3) { + err = -EILSEQ; + goto out; + } + *bid = (edl->data[1] << 8) + edl->data[2]; bt_dev_dbg(hdev, "%s: bid = %x", __func__, *bid); @@ -267,9 +287,10 @@ int qca_send_pre_shutdown_cmd(struct hci_dev *hdev) } EXPORT_SYMBOL_GPL(qca_send_pre_shutdown_cmd); -static void qca_tlv_check_data(struct hci_dev *hdev, +static int qca_tlv_check_data(struct hci_dev *hdev, struct qca_fw_config *config, - u8 *fw_data, enum qca_btsoc_type soc_type) + u8 *fw_data, size_t fw_size, + enum qca_btsoc_type soc_type) { const u8 *data; u32 type_len; @@ -279,12 +300,16 @@ static void qca_tlv_check_data(struct hci_dev *hdev, struct tlv_type_patch *tlv_patch; struct tlv_type_nvm *tlv_nvm; uint8_t nvm_baud_rate = config->user_baud_rate; + u8 type; config->dnld_mode = QCA_SKIP_EVT_NONE; config->dnld_type = QCA_SKIP_EVT_NONE; switch (config->type) { case ELF_TYPE_PATCH: + if (fw_size < 7) + return -EINVAL; + config->dnld_mode = QCA_SKIP_EVT_VSE_CC; config->dnld_type = QCA_SKIP_EVT_VSE_CC; @@ -293,6 +318,9 @@ static void qca_tlv_check_data(struct hci_dev *hdev, bt_dev_dbg(hdev, "File version : 0x%x", fw_data[6]); break; case TLV_TYPE_PATCH: + if (fw_size < sizeof(struct tlv_type_hdr) + sizeof(struct tlv_type_patch)) + return -EINVAL; + tlv = (struct tlv_type_hdr *)fw_data; type_len = le32_to_cpu(tlv->type_len); tlv_patch = (struct tlv_type_patch *)tlv->data; @@ -332,25 +360,64 @@ static void qca_tlv_check_data(struct hci_dev *hdev, break; case TLV_TYPE_NVM: + if (fw_size < sizeof(struct tlv_type_hdr)) + return -EINVAL; + tlv = (struct tlv_type_hdr *)fw_data; type_len = le32_to_cpu(tlv->type_len); - length = (type_len >> 8) & 0x00ffffff; + length = type_len >> 8; + type = type_len & 0xff; - BT_DBG("TLV Type\t\t : 0x%x", type_len & 0x000000ff); + /* Some NVM files have more than one set of tags, only parse + * the first set when it has type 2 for now. When there is + * more than one set there is an enclosing header of type 4. + */ + if (type == 4) { + if (fw_size < 2 * sizeof(struct tlv_type_hdr)) + return -EINVAL; + + tlv++; + + type_len = le32_to_cpu(tlv->type_len); + length = type_len >> 8; + type = type_len & 0xff; + } + + BT_DBG("TLV Type\t\t : 0x%x", type); BT_DBG("Length\t\t : %d bytes", length); + if (type != 2) + break; + + if (fw_size < length + (tlv->data - fw_data)) + return -EINVAL; + idx = 0; data = tlv->data; - while (idx < length) { + while (idx < length - sizeof(struct tlv_type_nvm)) { tlv_nvm = (struct tlv_type_nvm *)(data + idx); tag_id = le16_to_cpu(tlv_nvm->tag_id); tag_len = le16_to_cpu(tlv_nvm->tag_len); + if (length < idx + sizeof(struct tlv_type_nvm) + tag_len) + return -EINVAL; + /* Update NVM tags as needed */ switch (tag_id) { + case EDL_TAG_ID_BD_ADDR: + if (tag_len != sizeof(bdaddr_t)) + return -EINVAL; + + memcpy(&config->bdaddr, tlv_nvm->data, sizeof(bdaddr_t)); + + break; + case EDL_TAG_ID_HCI: + if (tag_len < 3) + return -EINVAL; + /* HCI transport layer parameters * enabling software inband sleep * onto controller side. @@ -366,6 +433,9 @@ static void qca_tlv_check_data(struct hci_dev *hdev, break; case EDL_TAG_ID_DEEP_SLEEP: + if (tag_len < 1) + return -EINVAL; + /* Sleep enable mask * enabling deep sleep feature on controller. */ @@ -374,14 +444,16 @@ static void qca_tlv_check_data(struct hci_dev *hdev, break; } - idx += (sizeof(u16) + sizeof(u16) + 8 + tag_len); + idx += sizeof(struct tlv_type_nvm) + tag_len; } break; default: BT_ERR("Unknown TLV type %d", config->type); - break; + return -EINVAL; } + + return 0; } static int qca_tlv_send_segment(struct hci_dev *hdev, int seg_size, @@ -531,7 +603,9 @@ static int qca_download_firmware(struct hci_dev *hdev, memcpy(data, fw->data, size); release_firmware(fw); - qca_tlv_check_data(hdev, config, data, soc_type); + ret = qca_tlv_check_data(hdev, config, data, size, soc_type); + if (ret) + goto out; segment = data; remain = size; @@ -614,7 +688,7 @@ int qca_set_bdaddr_rome(struct hci_dev *hdev, const bdaddr_t *bdaddr) } EXPORT_SYMBOL_GPL(qca_set_bdaddr_rome); -static int qca_check_bdaddr(struct hci_dev *hdev) +static int qca_check_bdaddr(struct hci_dev *hdev, const struct qca_fw_config *config) { struct hci_rp_read_bd_addr *bda; struct sk_buff *skb; @@ -638,7 +712,7 @@ static int qca_check_bdaddr(struct hci_dev *hdev) } bda = (struct hci_rp_read_bd_addr *)skb->data; - if (!bacmp(&bda->bdaddr, QCA_BDADDR_DEFAULT)) + if (!bacmp(&bda->bdaddr, &config->bdaddr)) set_bit(HCI_QUIRK_USE_BDADDR_PROPERTY, &hdev->quirks); kfree_skb(skb); @@ -667,7 +741,7 @@ int qca_uart_setup(struct hci_dev *hdev, uint8_t baudrate, enum qca_btsoc_type soc_type, struct qca_btsoc_version ver, const char *firmware_name) { - struct qca_fw_config config; + struct qca_fw_config config = {}; int err; u8 rom_ver = 0; u32 soc_ver; @@ -852,7 +926,7 @@ int qca_uart_setup(struct hci_dev *hdev, uint8_t baudrate, break; } - err = qca_check_bdaddr(hdev); + err = qca_check_bdaddr(hdev, &config); if (err) return err; diff --git a/drivers/bluetooth/btqca.h b/drivers/bluetooth/btqca.h index dc31984f71dc..215433fd76a1 100644 --- a/drivers/bluetooth/btqca.h +++ b/drivers/bluetooth/btqca.h @@ -29,6 +29,7 @@ #define EDL_PATCH_CONFIG_RES_EVT (0x00) #define QCA_DISABLE_LOGGING_SUB_OP (0x14) +#define EDL_TAG_ID_BD_ADDR 2 #define EDL_TAG_ID_HCI (17) #define EDL_TAG_ID_DEEP_SLEEP (27) @@ -47,7 +48,6 @@ #define get_soc_ver(soc_id, rom_ver) \ ((le32_to_cpu(soc_id) << 16) | (le16_to_cpu(rom_ver))) -#define QCA_FW_BUILD_VER_LEN 255 #define QCA_HSP_GF_SOC_ID 0x1200 #define QCA_HSP_GF_SOC_MASK 0x0000ff00 @@ -94,6 +94,7 @@ struct qca_fw_config { uint8_t user_baud_rate; enum qca_tlv_dnld_mode dnld_mode; enum qca_tlv_dnld_mode dnld_type; + bdaddr_t bdaddr; }; struct edl_event_hdr { diff --git a/drivers/bus/Kconfig b/drivers/bus/Kconfig index d5e7fa9173a1..64cd2ee03aa3 100644 --- a/drivers/bus/Kconfig +++ b/drivers/bus/Kconfig @@ -163,6 +163,16 @@ config QCOM_SSC_BLOCK_BUS i2c/spi/uart controllers, a hexagon core, and a clock controller which provides clocks for the above. +config STM32_FIREWALL + bool "STM32 Firewall framework" + depends on (ARCH_STM32 || COMPILE_TEST) && OF + select OF_DYNAMIC + help + Say y to enable STM32 firewall framework and its services. Firewall + controllers will be able to register to the framework. Access for + hardware resources linked to a firewall controller can be requested + through this STM32 framework. + config SUN50I_DE2_BUS bool "Allwinner A64 DE2 Bus Driver" default ARM64 diff --git a/drivers/bus/Makefile b/drivers/bus/Makefile index d90eed189a65..cddd4984d6af 100644 --- a/drivers/bus/Makefile +++ b/drivers/bus/Makefile @@ -26,6 +26,7 @@ obj-$(CONFIG_OMAP_INTERCONNECT) += omap_l3_smx.o omap_l3_noc.o obj-$(CONFIG_OMAP_OCP2SCP) += omap-ocp2scp.o obj-$(CONFIG_QCOM_EBI2) += qcom-ebi2.o obj-$(CONFIG_QCOM_SSC_BLOCK_BUS) += qcom-ssc-block-bus.o +obj-$(CONFIG_STM32_FIREWALL) += stm32_firewall.o stm32_rifsc.o stm32_etzpc.o obj-$(CONFIG_SUN50I_DE2_BUS) += sun50i-de2.o obj-$(CONFIG_SUNXI_RSB) += sunxi-rsb.o obj-$(CONFIG_OF) += simple-pm-bus.o diff --git a/drivers/bus/brcmstb_gisb.c b/drivers/bus/brcmstb_gisb.c index 65ae758f3194..ee29162da4ee 100644 --- a/drivers/bus/brcmstb_gisb.c +++ b/drivers/bus/brcmstb_gisb.c @@ -410,6 +410,7 @@ static const struct of_device_id brcmstb_gisb_arb_of_match[] = { { .compatible = "brcm,bcm74165-gisb-arb", .data = gisb_offsets_bcm74165 }, { }, }; +MODULE_DEVICE_TABLE(of, brcmstb_gisb_arb_of_match); static int __init brcmstb_gisb_arb_probe(struct platform_device *pdev) { diff --git a/drivers/bus/stm32_etzpc.c b/drivers/bus/stm32_etzpc.c new file mode 100644 index 000000000000..7fc0f16960be --- /dev/null +++ b/drivers/bus/stm32_etzpc.c @@ -0,0 +1,141 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2023, STMicroelectronics - All Rights Reserved + */ + +#include <linux/bitfield.h> +#include <linux/bits.h> +#include <linux/device.h> +#include <linux/err.h> +#include <linux/init.h> +#include <linux/io.h> +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/of.h> +#include <linux/of_platform.h> +#include <linux/platform_device.h> +#include <linux/types.h> + +#include "stm32_firewall.h" + +/* + * ETZPC registers + */ +#define ETZPC_DECPROT 0x10 +#define ETZPC_HWCFGR 0x3F0 + +/* + * HWCFGR register + */ +#define ETZPC_HWCFGR_NUM_TZMA GENMASK(7, 0) +#define ETZPC_HWCFGR_NUM_PER_SEC GENMASK(15, 8) +#define ETZPC_HWCFGR_NUM_AHB_SEC GENMASK(23, 16) +#define ETZPC_HWCFGR_CHUNKS1N4 GENMASK(31, 24) + +/* + * ETZPC miscellaneous + */ +#define ETZPC_PROT_MASK GENMASK(1, 0) +#define ETZPC_PROT_A7NS 0x3 +#define ETZPC_DECPROT_SHIFT 1 + +#define IDS_PER_DECPROT_REGS 16 + +static int stm32_etzpc_grant_access(struct stm32_firewall_controller *ctrl, u32 firewall_id) +{ + u32 offset, reg_offset, sec_val; + + if (firewall_id >= ctrl->max_entries) { + dev_err(ctrl->dev, "Invalid sys bus ID %u", firewall_id); + return -EINVAL; + } + + /* Check access configuration, 16 peripherals per register */ + reg_offset = ETZPC_DECPROT + 0x4 * (firewall_id / IDS_PER_DECPROT_REGS); + offset = (firewall_id % IDS_PER_DECPROT_REGS) << ETZPC_DECPROT_SHIFT; + + /* Verify peripheral is non-secure and attributed to cortex A7 */ + sec_val = (readl(ctrl->mmio + reg_offset) >> offset) & ETZPC_PROT_MASK; + if (sec_val != ETZPC_PROT_A7NS) { + dev_dbg(ctrl->dev, "Invalid bus configuration: reg_offset %#x, value %d\n", + reg_offset, sec_val); + return -EACCES; + } + + return 0; +} + +static void stm32_etzpc_release_access(struct stm32_firewall_controller *ctrl __maybe_unused, + u32 firewall_id __maybe_unused) +{ +} + +static int stm32_etzpc_probe(struct platform_device *pdev) +{ + struct stm32_firewall_controller *etzpc_controller; + struct device_node *np = pdev->dev.of_node; + u32 nb_per, nb_master; + struct resource *res; + void __iomem *mmio; + int rc; + + etzpc_controller = devm_kzalloc(&pdev->dev, sizeof(*etzpc_controller), GFP_KERNEL); + if (!etzpc_controller) + return -ENOMEM; + + mmio = devm_platform_get_and_ioremap_resource(pdev, 0, &res); + if (IS_ERR(mmio)) + return PTR_ERR(mmio); + + etzpc_controller->dev = &pdev->dev; + etzpc_controller->mmio = mmio; + etzpc_controller->name = dev_driver_string(etzpc_controller->dev); + etzpc_controller->type = STM32_PERIPHERAL_FIREWALL | STM32_MEMORY_FIREWALL; + etzpc_controller->grant_access = stm32_etzpc_grant_access; + etzpc_controller->release_access = stm32_etzpc_release_access; + + /* Get number of etzpc entries*/ + nb_per = FIELD_GET(ETZPC_HWCFGR_NUM_PER_SEC, + readl(etzpc_controller->mmio + ETZPC_HWCFGR)); + nb_master = FIELD_GET(ETZPC_HWCFGR_NUM_AHB_SEC, + readl(etzpc_controller->mmio + ETZPC_HWCFGR)); + etzpc_controller->max_entries = nb_per + nb_master; + + platform_set_drvdata(pdev, etzpc_controller); + + rc = stm32_firewall_controller_register(etzpc_controller); + if (rc) { + dev_err(etzpc_controller->dev, "Couldn't register as a firewall controller: %d", + rc); + return rc; + } + + rc = stm32_firewall_populate_bus(etzpc_controller); + if (rc) { + dev_err(etzpc_controller->dev, "Couldn't populate ETZPC bus: %d", + rc); + return rc; + } + + /* Populate all allowed nodes */ + return of_platform_populate(np, NULL, NULL, &pdev->dev); +} + +static const struct of_device_id stm32_etzpc_of_match[] = { + { .compatible = "st,stm32-etzpc" }, + {} +}; +MODULE_DEVICE_TABLE(of, stm32_etzpc_of_match); + +static struct platform_driver stm32_etzpc_driver = { + .probe = stm32_etzpc_probe, + .driver = { + .name = "stm32-etzpc", + .of_match_table = stm32_etzpc_of_match, + }, +}; +module_platform_driver(stm32_etzpc_driver); + +MODULE_AUTHOR("Gatien Chevallier <gatien.chevallier@foss.st.com>"); +MODULE_DESCRIPTION("STMicroelectronics ETZPC driver"); +MODULE_LICENSE("GPL"); diff --git a/drivers/bus/stm32_firewall.c b/drivers/bus/stm32_firewall.c new file mode 100644 index 000000000000..2fc9761dadec --- /dev/null +++ b/drivers/bus/stm32_firewall.c @@ -0,0 +1,294 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2023, STMicroelectronics - All Rights Reserved + */ + +#include <linux/bitfield.h> +#include <linux/bits.h> +#include <linux/bus/stm32_firewall_device.h> +#include <linux/device.h> +#include <linux/err.h> +#include <linux/init.h> +#include <linux/io.h> +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/of.h> +#include <linux/of_platform.h> +#include <linux/platform_device.h> +#include <linux/types.h> +#include <linux/slab.h> + +#include "stm32_firewall.h" + +/* Corresponds to STM32_FIREWALL_MAX_EXTRA_ARGS + firewall ID */ +#define STM32_FIREWALL_MAX_ARGS (STM32_FIREWALL_MAX_EXTRA_ARGS + 1) + +static LIST_HEAD(firewall_controller_list); +static DEFINE_MUTEX(firewall_controller_list_lock); + +/* Firewall device API */ + +int stm32_firewall_get_firewall(struct device_node *np, struct stm32_firewall *firewall, + unsigned int nb_firewall) +{ + struct stm32_firewall_controller *ctrl; + struct of_phandle_iterator it; + unsigned int i, j = 0; + int err; + + if (!firewall || !nb_firewall) + return -EINVAL; + + /* Parse property with phandle parsed out */ + of_for_each_phandle(&it, err, np, "access-controllers", "#access-controller-cells", 0) { + struct of_phandle_args provider_args; + struct device_node *provider = it.node; + const char *fw_entry; + bool match = false; + + if (err) { + pr_err("Unable to get access-controllers property for node %s\n, err: %d", + np->full_name, err); + of_node_put(provider); + return err; + } + + if (j >= nb_firewall) { + pr_err("Too many firewall controllers"); + of_node_put(provider); + return -EINVAL; + } + + provider_args.args_count = of_phandle_iterator_args(&it, provider_args.args, + STM32_FIREWALL_MAX_ARGS); + + /* Check if the parsed phandle corresponds to a registered firewall controller */ + mutex_lock(&firewall_controller_list_lock); + list_for_each_entry(ctrl, &firewall_controller_list, entry) { + if (ctrl->dev->of_node->phandle == it.phandle) { + match = true; + firewall[j].firewall_ctrl = ctrl; + break; + } + } + mutex_unlock(&firewall_controller_list_lock); + + if (!match) { + firewall[j].firewall_ctrl = NULL; + pr_err("No firewall controller registered for %s\n", np->full_name); + of_node_put(provider); + return -ENODEV; + } + + err = of_property_read_string_index(np, "access-controller-names", j, &fw_entry); + if (err == 0) + firewall[j].entry = fw_entry; + + /* Handle the case when there are no arguments given along with the phandle */ + if (provider_args.args_count < 0 || + provider_args.args_count > STM32_FIREWALL_MAX_ARGS) { + of_node_put(provider); + return -EINVAL; + } else if (provider_args.args_count == 0) { + firewall[j].extra_args_size = 0; + firewall[j].firewall_id = U32_MAX; + j++; + continue; + } + + /* The firewall ID is always the first argument */ + firewall[j].firewall_id = provider_args.args[0]; + + /* Extra args start at the second argument */ + for (i = 0; i < provider_args.args_count - 1; i++) + firewall[j].extra_args[i] = provider_args.args[i + 1]; + + /* Remove the firewall ID arg that is not an extra argument */ + firewall[j].extra_args_size = provider_args.args_count - 1; + + j++; + } + + return 0; +} +EXPORT_SYMBOL_GPL(stm32_firewall_get_firewall); + +int stm32_firewall_grant_access(struct stm32_firewall *firewall) +{ + struct stm32_firewall_controller *firewall_controller; + + if (!firewall || firewall->firewall_id == U32_MAX) + return -EINVAL; + + firewall_controller = firewall->firewall_ctrl; + + if (!firewall_controller) + return -ENODEV; + + return firewall_controller->grant_access(firewall_controller, firewall->firewall_id); +} +EXPORT_SYMBOL_GPL(stm32_firewall_grant_access); + +int stm32_firewall_grant_access_by_id(struct stm32_firewall *firewall, u32 subsystem_id) +{ + struct stm32_firewall_controller *firewall_controller; + + if (!firewall || subsystem_id == U32_MAX || firewall->firewall_id == U32_MAX) + return -EINVAL; + + firewall_controller = firewall->firewall_ctrl; + + if (!firewall_controller) + return -ENODEV; + + return firewall_controller->grant_access(firewall_controller, subsystem_id); +} +EXPORT_SYMBOL_GPL(stm32_firewall_grant_access_by_id); + +void stm32_firewall_release_access(struct stm32_firewall *firewall) +{ + struct stm32_firewall_controller *firewall_controller; + + if (!firewall || firewall->firewall_id == U32_MAX) { + pr_debug("Incorrect arguments when releasing a firewall access\n"); + return; + } + + firewall_controller = firewall->firewall_ctrl; + + if (!firewall_controller) { + pr_debug("No firewall controller to release\n"); + return; + } + + firewall_controller->release_access(firewall_controller, firewall->firewall_id); +} +EXPORT_SYMBOL_GPL(stm32_firewall_release_access); + +void stm32_firewall_release_access_by_id(struct stm32_firewall *firewall, u32 subsystem_id) +{ + struct stm32_firewall_controller *firewall_controller; + + if (!firewall || subsystem_id == U32_MAX || firewall->firewall_id == U32_MAX) { + pr_debug("Incorrect arguments when releasing a firewall access"); + return; + } + + firewall_controller = firewall->firewall_ctrl; + + if (!firewall_controller) { + pr_debug("No firewall controller to release"); + return; + } + + firewall_controller->release_access(firewall_controller, subsystem_id); +} +EXPORT_SYMBOL_GPL(stm32_firewall_release_access_by_id); + +/* Firewall controller API */ + +int stm32_firewall_controller_register(struct stm32_firewall_controller *firewall_controller) +{ + struct stm32_firewall_controller *ctrl; + + if (!firewall_controller) + return -ENODEV; + + pr_info("Registering %s firewall controller\n", firewall_controller->name); + + mutex_lock(&firewall_controller_list_lock); + list_for_each_entry(ctrl, &firewall_controller_list, entry) { + if (ctrl == firewall_controller) { + pr_debug("%s firewall controller already registered\n", + firewall_controller->name); + mutex_unlock(&firewall_controller_list_lock); + return 0; + } + } + list_add_tail(&firewall_controller->entry, &firewall_controller_list); + mutex_unlock(&firewall_controller_list_lock); + + return 0; +} +EXPORT_SYMBOL_GPL(stm32_firewall_controller_register); + +void stm32_firewall_controller_unregister(struct stm32_firewall_controller *firewall_controller) +{ + struct stm32_firewall_controller *ctrl; + bool controller_removed = false; + + if (!firewall_controller) { + pr_debug("Null reference while unregistering firewall controller\n"); + return; + } + + mutex_lock(&firewall_controller_list_lock); + list_for_each_entry(ctrl, &firewall_controller_list, entry) { + if (ctrl == firewall_controller) { + controller_removed = true; + list_del_init(&ctrl->entry); + break; + } + } + mutex_unlock(&firewall_controller_list_lock); + + if (!controller_removed) + pr_debug("There was no firewall controller named %s to unregister\n", + firewall_controller->name); +} +EXPORT_SYMBOL_GPL(stm32_firewall_controller_unregister); + +int stm32_firewall_populate_bus(struct stm32_firewall_controller *firewall_controller) +{ + struct stm32_firewall *firewalls; + struct device_node *child; + struct device *parent; + unsigned int i; + int len; + int err; + + parent = firewall_controller->dev; + + dev_dbg(parent, "Populating %s system bus\n", dev_name(firewall_controller->dev)); + + for_each_available_child_of_node(dev_of_node(parent), child) { + /* The access-controllers property is mandatory for firewall bus devices */ + len = of_count_phandle_with_args(child, "access-controllers", + "#access-controller-cells"); + if (len <= 0) { + of_node_put(child); + return -EINVAL; + } + + firewalls = kcalloc(len, sizeof(*firewalls), GFP_KERNEL); + if (!firewalls) { + of_node_put(child); + return -ENOMEM; + } + + err = stm32_firewall_get_firewall(child, firewalls, (unsigned int)len); + if (err) { + kfree(firewalls); + of_node_put(child); + return err; + } + + for (i = 0; i < len; i++) { + if (firewall_controller->grant_access(firewall_controller, + firewalls[i].firewall_id)) { + /* + * Peripheral access not allowed or not defined. + * Mark the node as populated so platform bus won't probe it + */ + of_detach_node(child); + dev_err(parent, "%s: Device driver will not be probed\n", + child->full_name); + } + } + + kfree(firewalls); + } + + return 0; +} +EXPORT_SYMBOL_GPL(stm32_firewall_populate_bus); diff --git a/drivers/bus/stm32_firewall.h b/drivers/bus/stm32_firewall.h new file mode 100644 index 000000000000..e5fac85fe346 --- /dev/null +++ b/drivers/bus/stm32_firewall.h @@ -0,0 +1,83 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2023, STMicroelectronics - All Rights Reserved + */ + +#ifndef _STM32_FIREWALL_H +#define _STM32_FIREWALL_H + +#include <linux/kernel.h> +#include <linux/list.h> +#include <linux/of.h> +#include <linux/platform_device.h> +#include <linux/types.h> + +/** + * STM32_PERIPHERAL_FIREWALL: This type of firewall protects peripherals + * STM32_MEMORY_FIREWALL: This type of firewall protects memories/subsets of memory + * zones + * STM32_NOTYPE_FIREWALL: Undefined firewall type + */ + +#define STM32_PERIPHERAL_FIREWALL BIT(1) +#define STM32_MEMORY_FIREWALL BIT(2) +#define STM32_NOTYPE_FIREWALL BIT(3) + +/** + * struct stm32_firewall_controller - Information on firewall controller supplying services + * + * @name: Name of the firewall controller + * @dev: Device reference of the firewall controller + * @mmio: Base address of the firewall controller + * @entry: List entry of the firewall controller list + * @type: Type of firewall + * @max_entries: Number of entries covered by the firewall + * @grant_access: Callback used to grant access for a device access against a + * firewall controller + * @release_access: Callback used to release resources taken by a device when access was + * granted + * @grant_memory_range_access: Callback used to grant access for a device to a given memory region + */ +struct stm32_firewall_controller { + const char *name; + struct device *dev; + void __iomem *mmio; + struct list_head entry; + unsigned int type; + unsigned int max_entries; + + int (*grant_access)(struct stm32_firewall_controller *ctrl, u32 id); + void (*release_access)(struct stm32_firewall_controller *ctrl, u32 id); + int (*grant_memory_range_access)(struct stm32_firewall_controller *ctrl, phys_addr_t paddr, + size_t size); +}; + +/** + * stm32_firewall_controller_register - Register a firewall controller to the STM32 firewall + * framework + * @firewall_controller: Firewall controller to register + * + * Returns 0 in case of success or -ENODEV if no controller was given. + */ +int stm32_firewall_controller_register(struct stm32_firewall_controller *firewall_controller); + +/** + * stm32_firewall_controller_unregister - Unregister a firewall controller from the STM32 + * firewall framework + * @firewall_controller: Firewall controller to unregister + */ +void stm32_firewall_controller_unregister(struct stm32_firewall_controller *firewall_controller); + +/** + * stm32_firewall_populate_bus - Populate device tree nodes that have a correct firewall + * configuration. This is used at boot-time only, as a sanity check + * between device tree and firewalls hardware configurations to + * prevent a kernel crash when a device driver is not granted access + * + * @firewall_controller: Firewall controller which nodes will be populated or not + * + * Returns 0 in case of success or appropriate errno code if error occurred. + */ +int stm32_firewall_populate_bus(struct stm32_firewall_controller *firewall_controller); + +#endif /* _STM32_FIREWALL_H */ diff --git a/drivers/bus/stm32_rifsc.c b/drivers/bus/stm32_rifsc.c new file mode 100644 index 000000000000..4cf1b60014b7 --- /dev/null +++ b/drivers/bus/stm32_rifsc.c @@ -0,0 +1,252 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2023, STMicroelectronics - All Rights Reserved + */ + +#include <linux/bitfield.h> +#include <linux/bits.h> +#include <linux/device.h> +#include <linux/err.h> +#include <linux/init.h> +#include <linux/io.h> +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/of.h> +#include <linux/of_platform.h> +#include <linux/platform_device.h> +#include <linux/types.h> + +#include "stm32_firewall.h" + +/* + * RIFSC offset register + */ +#define RIFSC_RISC_SECCFGR0 0x10 +#define RIFSC_RISC_PRIVCFGR0 0x30 +#define RIFSC_RISC_PER0_CIDCFGR 0x100 +#define RIFSC_RISC_PER0_SEMCR 0x104 +#define RIFSC_RISC_HWCFGR2 0xFEC + +/* + * SEMCR register + */ +#define SEMCR_MUTEX BIT(0) + +/* + * HWCFGR2 register + */ +#define HWCFGR2_CONF1_MASK GENMASK(15, 0) +#define HWCFGR2_CONF2_MASK GENMASK(23, 16) +#define HWCFGR2_CONF3_MASK GENMASK(31, 24) + +/* + * RIFSC miscellaneous + */ +#define RIFSC_RISC_CFEN_MASK BIT(0) +#define RIFSC_RISC_SEM_EN_MASK BIT(1) +#define RIFSC_RISC_SCID_MASK GENMASK(6, 4) +#define RIFSC_RISC_SEML_SHIFT 16 +#define RIFSC_RISC_SEMWL_MASK GENMASK(23, 16) +#define RIFSC_RISC_PER_ID_MASK GENMASK(31, 24) + +#define RIFSC_RISC_PERx_CID_MASK (RIFSC_RISC_CFEN_MASK | \ + RIFSC_RISC_SEM_EN_MASK | \ + RIFSC_RISC_SCID_MASK | \ + RIFSC_RISC_SEMWL_MASK) + +#define IDS_PER_RISC_SEC_PRIV_REGS 32 + +/* RIF miscellaneous */ +/* + * CIDCFGR register fields + */ +#define CIDCFGR_CFEN BIT(0) +#define CIDCFGR_SEMEN BIT(1) +#define CIDCFGR_SEMWL(x) BIT(RIFSC_RISC_SEML_SHIFT + (x)) + +#define SEMWL_SHIFT 16 + +/* Compartiment IDs */ +#define RIF_CID0 0x0 +#define RIF_CID1 0x1 + +static bool stm32_rifsc_is_semaphore_available(void __iomem *addr) +{ + return !(readl(addr) & SEMCR_MUTEX); +} + +static int stm32_rif_acquire_semaphore(struct stm32_firewall_controller *stm32_firewall_controller, + int id) +{ + void __iomem *addr = stm32_firewall_controller->mmio + RIFSC_RISC_PER0_SEMCR + 0x8 * id; + + writel(SEMCR_MUTEX, addr); + + /* Check that CID1 has the semaphore */ + if (stm32_rifsc_is_semaphore_available(addr) || + FIELD_GET(RIFSC_RISC_SCID_MASK, readl(addr)) != RIF_CID1) + return -EACCES; + + return 0; +} + +static void stm32_rif_release_semaphore(struct stm32_firewall_controller *stm32_firewall_controller, + int id) +{ + void __iomem *addr = stm32_firewall_controller->mmio + RIFSC_RISC_PER0_SEMCR + 0x8 * id; + + if (stm32_rifsc_is_semaphore_available(addr)) + return; + + writel(SEMCR_MUTEX, addr); + + /* Ok if another compartment takes the semaphore before the check */ + WARN_ON(!stm32_rifsc_is_semaphore_available(addr) && + FIELD_GET(RIFSC_RISC_SCID_MASK, readl(addr)) == RIF_CID1); +} + +static int stm32_rifsc_grant_access(struct stm32_firewall_controller *ctrl, u32 firewall_id) +{ + struct stm32_firewall_controller *rifsc_controller = ctrl; + u32 reg_offset, reg_id, sec_reg_value, cid_reg_value; + int rc; + + if (firewall_id >= rifsc_controller->max_entries) { + dev_err(rifsc_controller->dev, "Invalid sys bus ID %u", firewall_id); + return -EINVAL; + } + + /* + * RIFSC_RISC_PRIVCFGRx and RIFSC_RISC_SECCFGRx both handle configuration access for + * 32 peripherals. On the other hand, there is one _RIFSC_RISC_PERx_CIDCFGR register + * per peripheral + */ + reg_id = firewall_id / IDS_PER_RISC_SEC_PRIV_REGS; + reg_offset = firewall_id % IDS_PER_RISC_SEC_PRIV_REGS; + sec_reg_value = readl(rifsc_controller->mmio + RIFSC_RISC_SECCFGR0 + 0x4 * reg_id); + cid_reg_value = readl(rifsc_controller->mmio + RIFSC_RISC_PER0_CIDCFGR + 0x8 * firewall_id); + + /* First check conditions for semaphore mode, which doesn't take into account static CID. */ + if ((cid_reg_value & CIDCFGR_SEMEN) && (cid_reg_value & CIDCFGR_CFEN)) { + if (cid_reg_value & BIT(RIF_CID1 + SEMWL_SHIFT)) { + /* Static CID is irrelevant if semaphore mode */ + goto skip_cid_check; + } else { + dev_dbg(rifsc_controller->dev, + "Invalid bus semaphore configuration: index %d\n", firewall_id); + return -EACCES; + } + } + + /* + * Skip CID check if CID filtering isn't enabled or filtering is enabled on CID0, which + * corresponds to whatever CID. + */ + if (!(cid_reg_value & CIDCFGR_CFEN) || + FIELD_GET(RIFSC_RISC_SCID_MASK, cid_reg_value) == RIF_CID0) + goto skip_cid_check; + + /* Coherency check with the CID configuration */ + if (FIELD_GET(RIFSC_RISC_SCID_MASK, cid_reg_value) != RIF_CID1) { + dev_dbg(rifsc_controller->dev, "Invalid CID configuration for peripheral: %d\n", + firewall_id); + return -EACCES; + } + +skip_cid_check: + /* Check security configuration */ + if (sec_reg_value & BIT(reg_offset)) { + dev_dbg(rifsc_controller->dev, + "Invalid security configuration for peripheral: %d\n", firewall_id); + return -EACCES; + } + + /* + * If the peripheral is in semaphore mode, take the semaphore so that + * the CID1 has the ownership. + */ + if ((cid_reg_value & CIDCFGR_SEMEN) && (cid_reg_value & CIDCFGR_CFEN)) { + rc = stm32_rif_acquire_semaphore(rifsc_controller, firewall_id); + if (rc) { + dev_err(rifsc_controller->dev, + "Couldn't acquire semaphore for peripheral: %d\n", firewall_id); + return rc; + } + } + + return 0; +} + +static void stm32_rifsc_release_access(struct stm32_firewall_controller *ctrl, u32 firewall_id) +{ + stm32_rif_release_semaphore(ctrl, firewall_id); +} + +static int stm32_rifsc_probe(struct platform_device *pdev) +{ + struct stm32_firewall_controller *rifsc_controller; + struct device_node *np = pdev->dev.of_node; + u32 nb_risup, nb_rimu, nb_risal; + struct resource *res; + void __iomem *mmio; + int rc; + + rifsc_controller = devm_kzalloc(&pdev->dev, sizeof(*rifsc_controller), GFP_KERNEL); + if (!rifsc_controller) + return -ENOMEM; + + mmio = devm_platform_get_and_ioremap_resource(pdev, 0, &res); + if (IS_ERR(mmio)) + return PTR_ERR(mmio); + + rifsc_controller->dev = &pdev->dev; + rifsc_controller->mmio = mmio; + rifsc_controller->name = dev_driver_string(rifsc_controller->dev); + rifsc_controller->type = STM32_PERIPHERAL_FIREWALL | STM32_MEMORY_FIREWALL; + rifsc_controller->grant_access = stm32_rifsc_grant_access; + rifsc_controller->release_access = stm32_rifsc_release_access; + + /* Get number of RIFSC entries*/ + nb_risup = readl(rifsc_controller->mmio + RIFSC_RISC_HWCFGR2) & HWCFGR2_CONF1_MASK; + nb_rimu = readl(rifsc_controller->mmio + RIFSC_RISC_HWCFGR2) & HWCFGR2_CONF2_MASK; + nb_risal = readl(rifsc_controller->mmio + RIFSC_RISC_HWCFGR2) & HWCFGR2_CONF3_MASK; + rifsc_controller->max_entries = nb_risup + nb_rimu + nb_risal; + + platform_set_drvdata(pdev, rifsc_controller); + + rc = stm32_firewall_controller_register(rifsc_controller); + if (rc) { + dev_err(rifsc_controller->dev, "Couldn't register as a firewall controller: %d", + rc); + return rc; + } + + rc = stm32_firewall_populate_bus(rifsc_controller); + if (rc) { + dev_err(rifsc_controller->dev, "Couldn't populate RIFSC bus: %d", + rc); + return rc; + } + + /* Populate all allowed nodes */ + return of_platform_populate(np, NULL, NULL, &pdev->dev); +} + +static const struct of_device_id stm32_rifsc_of_match[] = { + { .compatible = "st,stm32mp25-rifsc" }, + {} +}; +MODULE_DEVICE_TABLE(of, stm32_rifsc_of_match); + +static struct platform_driver stm32_rifsc_driver = { + .probe = stm32_rifsc_probe, + .driver = { + .name = "stm32-rifsc", + .of_match_table = stm32_rifsc_of_match, + }, +}; +module_platform_driver(stm32_rifsc_driver); + +MODULE_AUTHOR("Gatien Chevallier <gatien.chevallier@foss.st.com>"); +MODULE_DESCRIPTION("STMicroelectronics RIFSC driver"); +MODULE_LICENSE("GPL"); diff --git a/drivers/bus/ti-sysc.c b/drivers/bus/ti-sysc.c index 41d33f39efe5..8767e04d6c89 100644 --- a/drivers/bus/ti-sysc.c +++ b/drivers/bus/ti-sysc.c @@ -1,6 +1,17 @@ // SPDX-License-Identifier: GPL-2.0 /* * ti-sysc.c - Texas Instruments sysc interconnect target driver + * + * TI SoCs have an interconnect target wrapper IP for many devices. The wrapper + * IP manages clock gating, resets, and PM capabilities for the connected devices. + * + * Copyright (C) 2017-2024 Texas Instruments Incorporated - https://www.ti.com/ + * + * Many features are based on the earlier omap_hwmod arch code with thanks to all + * the people who developed and debugged the code over the years: + * + * Copyright (C) 2009-2011 Nokia Corporation + * Copyright (C) 2011-2021 Texas Instruments Incorporated - https://www.ti.com/ */ #include <linux/io.h> @@ -1458,8 +1469,7 @@ static int __maybe_unused sysc_noirq_suspend(struct device *dev) ddata = dev_get_drvdata(dev); - if (ddata->cfg.quirks & - (SYSC_QUIRK_LEGACY_IDLE | SYSC_QUIRK_NO_IDLE)) + if (ddata->cfg.quirks & SYSC_QUIRK_NO_IDLE) return 0; if (!ddata->enabled) @@ -1477,8 +1487,7 @@ static int __maybe_unused sysc_noirq_resume(struct device *dev) ddata = dev_get_drvdata(dev); - if (ddata->cfg.quirks & - (SYSC_QUIRK_LEGACY_IDLE | SYSC_QUIRK_NO_IDLE)) + if (ddata->cfg.quirks & SYSC_QUIRK_NO_IDLE) return 0; if (ddata->cfg.quirks & SYSC_QUIRK_REINIT_ON_RESUME) { @@ -1529,19 +1538,6 @@ struct sysc_revision_quirk { } static const struct sysc_revision_quirk sysc_revision_quirks[] = { - /* These drivers need to be fixed to not use pm_runtime_irq_safe() */ - SYSC_QUIRK("uart", 0, 0x50, 0x54, 0x58, 0x00000046, 0xffffffff, - SYSC_QUIRK_SWSUP_SIDLE_ACT | SYSC_QUIRK_LEGACY_IDLE), - SYSC_QUIRK("uart", 0, 0x50, 0x54, 0x58, 0x00000052, 0xffffffff, - SYSC_QUIRK_SWSUP_SIDLE_ACT | SYSC_QUIRK_LEGACY_IDLE), - /* Uarts on omap4 and later */ - SYSC_QUIRK("uart", 0, 0x50, 0x54, 0x58, 0x50411e03, 0xffff00ff, - SYSC_QUIRK_SWSUP_SIDLE_ACT | SYSC_QUIRK_LEGACY_IDLE), - SYSC_QUIRK("uart", 0, 0x50, 0x54, 0x58, 0x47422e03, 0xffffffff, - SYSC_QUIRK_SWSUP_SIDLE_ACT | SYSC_QUIRK_LEGACY_IDLE), - SYSC_QUIRK("uart", 0, 0x50, 0x54, 0x58, 0x47424e03, 0xffffffff, - SYSC_QUIRK_SWSUP_SIDLE_ACT | SYSC_QUIRK_LEGACY_IDLE), - /* Quirks that need to be set based on the module address */ SYSC_QUIRK("mcpdm", 0x40132000, 0, 0x10, -ENODEV, 0x50000800, 0xffffffff, SYSC_QUIRK_EXT_OPT_CLOCK | SYSC_QUIRK_NO_RESET_ON_INIT | @@ -1599,6 +1595,17 @@ static const struct sysc_revision_quirk sysc_revision_quirks[] = { SYSC_QUIRK_SWSUP_SIDLE | SYSC_QUIRK_SWSUP_MSTANDBY), SYSC_QUIRK("sata", 0, 0xfc, 0x1100, -ENODEV, 0x5e412000, 0xffffffff, SYSC_QUIRK_SWSUP_SIDLE | SYSC_QUIRK_SWSUP_MSTANDBY), + SYSC_QUIRK("uart", 0, 0x50, 0x54, 0x58, 0x00000046, 0xffffffff, + SYSC_QUIRK_SWSUP_SIDLE_ACT), + SYSC_QUIRK("uart", 0, 0x50, 0x54, 0x58, 0x00000052, 0xffffffff, + SYSC_QUIRK_SWSUP_SIDLE_ACT), + /* Uarts on omap4 and later */ + SYSC_QUIRK("uart", 0, 0x50, 0x54, 0x58, 0x50411e03, 0xffff00ff, + SYSC_QUIRK_SWSUP_SIDLE_ACT), + SYSC_QUIRK("uart", 0, 0x50, 0x54, 0x58, 0x47422e03, 0xffffffff, + SYSC_QUIRK_SWSUP_SIDLE_ACT), + SYSC_QUIRK("uart", 0, 0x50, 0x54, 0x58, 0x47424e03, 0xffffffff, + SYSC_QUIRK_SWSUP_SIDLE_ACT), SYSC_QUIRK("usb_host_hs", 0, 0, 0x10, 0x14, 0x50700100, 0xffffffff, SYSC_QUIRK_SWSUP_SIDLE | SYSC_QUIRK_SWSUP_MSTANDBY), SYSC_QUIRK("usb_host_hs", 0, 0, 0x10, -ENODEV, 0x50700101, 0xffffffff, @@ -2145,8 +2152,7 @@ static int sysc_reset(struct sysc *ddata) sysc_offset = ddata->offsets[SYSC_SYSCONFIG]; if (ddata->legacy_mode || - ddata->cap->regbits->srst_shift < 0 || - ddata->cfg.quirks & SYSC_QUIRK_NO_RESET_ON_INIT) + ddata->cap->regbits->srst_shift < 0) return 0; sysc_mask = BIT(ddata->cap->regbits->srst_shift); @@ -2240,12 +2246,14 @@ static int sysc_init_module(struct sysc *ddata) goto err_main_clocks; } - error = sysc_reset(ddata); - if (error) - dev_err(ddata->dev, "Reset failed with %d\n", error); + if (!(ddata->cfg.quirks & SYSC_QUIRK_NO_RESET_ON_INIT)) { + error = sysc_reset(ddata); + if (error) + dev_err(ddata->dev, "Reset failed with %d\n", error); - if (error && !ddata->legacy_mode) - sysc_disable_module(ddata->dev); + if (error && !ddata->legacy_mode) + sysc_disable_module(ddata->dev); + } err_main_clocks: if (error) @@ -2447,89 +2455,6 @@ static int __maybe_unused sysc_child_runtime_resume(struct device *dev) return pm_generic_runtime_resume(dev); } -#ifdef CONFIG_PM_SLEEP -static int sysc_child_suspend_noirq(struct device *dev) -{ - struct sysc *ddata; - int error; - - ddata = sysc_child_to_parent(dev); - - dev_dbg(ddata->dev, "%s %s\n", __func__, - ddata->name ? ddata->name : ""); - - error = pm_generic_suspend_noirq(dev); - if (error) { - dev_err(dev, "%s error at %i: %i\n", - __func__, __LINE__, error); - - return error; - } - - if (!pm_runtime_status_suspended(dev)) { - error = pm_generic_runtime_suspend(dev); - if (error) { - dev_dbg(dev, "%s busy at %i: %i\n", - __func__, __LINE__, error); - - return 0; - } - - error = sysc_runtime_suspend(ddata->dev); - if (error) { - dev_err(dev, "%s error at %i: %i\n", - __func__, __LINE__, error); - - return error; - } - - ddata->child_needs_resume = true; - } - - return 0; -} - -static int sysc_child_resume_noirq(struct device *dev) -{ - struct sysc *ddata; - int error; - - ddata = sysc_child_to_parent(dev); - - dev_dbg(ddata->dev, "%s %s\n", __func__, - ddata->name ? ddata->name : ""); - - if (ddata->child_needs_resume) { - ddata->child_needs_resume = false; - - error = sysc_runtime_resume(ddata->dev); - if (error) - dev_err(ddata->dev, - "%s runtime resume error: %i\n", - __func__, error); - - error = pm_generic_runtime_resume(dev); - if (error) - dev_err(ddata->dev, - "%s generic runtime resume: %i\n", - __func__, error); - } - - return pm_generic_resume_noirq(dev); -} -#endif - -static struct dev_pm_domain sysc_child_pm_domain = { - .ops = { - SET_RUNTIME_PM_OPS(sysc_child_runtime_suspend, - sysc_child_runtime_resume, - NULL) - USE_PLATFORM_PM_SLEEP_OPS - SET_NOIRQ_SYSTEM_SLEEP_PM_OPS(sysc_child_suspend_noirq, - sysc_child_resume_noirq) - } -}; - /* Caller needs to take list_lock if ever used outside of cpu_pm */ static void sysc_reinit_modules(struct sysc_soc_info *soc) { @@ -2600,25 +2525,6 @@ out_unlock: mutex_unlock(&sysc_soc->list_lock); } -/** - * sysc_legacy_idle_quirk - handle children in omap_device compatible way - * @ddata: device driver data - * @child: child device driver - * - * Allow idle for child devices as done with _od_runtime_suspend(). - * Otherwise many child devices will not idle because of the permanent - * parent usecount set in pm_runtime_irq_safe(). - * - * Note that the long term solution is to just modify the child device - * drivers to not set pm_runtime_irq_safe() and then this can be just - * dropped. - */ -static void sysc_legacy_idle_quirk(struct sysc *ddata, struct device *child) -{ - if (ddata->cfg.quirks & SYSC_QUIRK_LEGACY_IDLE) - dev_pm_domain_set(child, &sysc_child_pm_domain); -} - static int sysc_notifier_call(struct notifier_block *nb, unsigned long event, void *device) { @@ -2635,7 +2541,6 @@ static int sysc_notifier_call(struct notifier_block *nb, error = sysc_child_add_clocks(ddata, dev); if (error) return error; - sysc_legacy_idle_quirk(ddata, dev); break; default: break; @@ -2859,8 +2764,7 @@ static const struct sysc_capabilities sysc_34xx_sr = { .type = TI_SYSC_OMAP34XX_SR, .sysc_mask = SYSC_OMAP2_CLOCKACTIVITY, .regbits = &sysc_regbits_omap34xx_sr, - .mod_quirks = SYSC_QUIRK_USE_CLOCKACT | SYSC_QUIRK_UNCACHED | - SYSC_QUIRK_LEGACY_IDLE, + .mod_quirks = SYSC_QUIRK_USE_CLOCKACT | SYSC_QUIRK_UNCACHED, }; /* @@ -2881,13 +2785,12 @@ static const struct sysc_capabilities sysc_36xx_sr = { .type = TI_SYSC_OMAP36XX_SR, .sysc_mask = SYSC_OMAP3_SR_ENAWAKEUP, .regbits = &sysc_regbits_omap36xx_sr, - .mod_quirks = SYSC_QUIRK_UNCACHED | SYSC_QUIRK_LEGACY_IDLE, + .mod_quirks = SYSC_QUIRK_UNCACHED, }; static const struct sysc_capabilities sysc_omap4_sr = { .type = TI_SYSC_OMAP4_SR, .regbits = &sysc_regbits_omap36xx_sr, - .mod_quirks = SYSC_QUIRK_LEGACY_IDLE, }; /* diff --git a/drivers/cache/sifive_ccache.c b/drivers/cache/sifive_ccache.c index e9cc8b4786fb..6874b72ec59d 100644 --- a/drivers/cache/sifive_ccache.c +++ b/drivers/cache/sifive_ccache.c @@ -290,7 +290,7 @@ static int __init sifive_ccache_init(void) struct device_node *np; struct resource res; const struct of_device_id *match; - unsigned long quirks; + unsigned long quirks __maybe_unused; int rc; np = of_find_matching_node_and_match(NULL, sifive_ccache_ids, &match); diff --git a/drivers/char/agp/alpha-agp.c b/drivers/char/agp/alpha-agp.c index c9bf2c219841..f0d0c044731c 100644 --- a/drivers/char/agp/alpha-agp.c +++ b/drivers/char/agp/alpha-agp.c @@ -149,7 +149,7 @@ struct agp_bridge_driver alpha_core_agp_driver = { struct agp_bridge_data *alpha_bridge; -int __init +static int __init alpha_core_agp_setup(void) { alpha_agp_info *agp = alpha_mv.agp_info(); diff --git a/drivers/char/hw_random/core.c b/drivers/char/hw_random/core.c index a3bbdd6e60fc..f5c71a617a99 100644 --- a/drivers/char/hw_random/core.c +++ b/drivers/char/hw_random/core.c @@ -382,7 +382,7 @@ static ssize_t rng_current_show(struct device *dev, if (IS_ERR(rng)) return PTR_ERR(rng); - ret = snprintf(buf, PAGE_SIZE, "%s\n", rng ? rng->name : "none"); + ret = sysfs_emit(buf, "%s\n", rng ? rng->name : "none"); put_rng(rng); return ret; diff --git a/drivers/char/hw_random/mxc-rnga.c b/drivers/char/hw_random/mxc-rnga.c index 07ec000e4cd7..94ee18a1120a 100644 --- a/drivers/char/hw_random/mxc-rnga.c +++ b/drivers/char/hw_random/mxc-rnga.c @@ -131,7 +131,7 @@ static void mxc_rnga_cleanup(struct hwrng *rng) __raw_writel(ctrl & ~RNGA_CONTROL_GO, mxc_rng->mem + RNGA_CONTROL); } -static int __init mxc_rnga_probe(struct platform_device *pdev) +static int mxc_rnga_probe(struct platform_device *pdev) { int err; struct mxc_rng *mxc_rng; @@ -176,7 +176,7 @@ err_ioremap: return err; } -static void __exit mxc_rnga_remove(struct platform_device *pdev) +static void mxc_rnga_remove(struct platform_device *pdev) { struct mxc_rng *mxc_rng = platform_get_drvdata(pdev); @@ -197,10 +197,11 @@ static struct platform_driver mxc_rnga_driver = { .name = "mxc_rnga", .of_match_table = mxc_rnga_of_match, }, - .remove_new = __exit_p(mxc_rnga_remove), + .probe = mxc_rnga_probe, + .remove_new = mxc_rnga_remove, }; -module_platform_driver_probe(mxc_rnga_driver, mxc_rnga_probe); +module_platform_driver(mxc_rnga_driver); MODULE_AUTHOR("Freescale Semiconductor, Inc."); MODULE_DESCRIPTION("H/W RNGA driver for i.MX"); diff --git a/drivers/char/hw_random/stm32-rng.c b/drivers/char/hw_random/stm32-rng.c index 379bc245c520..0e903d6e22e3 100644 --- a/drivers/char/hw_random/stm32-rng.c +++ b/drivers/char/hw_random/stm32-rng.c @@ -220,7 +220,8 @@ static int stm32_rng_read(struct hwrng *rng, void *data, size_t max, bool wait) if (err && i > RNG_NB_RECOVER_TRIES) { dev_err((struct device *)priv->rng.priv, "Couldn't recover from seed error\n"); - return -ENOTRECOVERABLE; + retval = -ENOTRECOVERABLE; + goto exit_rpm; } continue; @@ -238,7 +239,8 @@ static int stm32_rng_read(struct hwrng *rng, void *data, size_t max, bool wait) if (err && i > RNG_NB_RECOVER_TRIES) { dev_err((struct device *)priv->rng.priv, "Couldn't recover from seed error"); - return -ENOTRECOVERABLE; + retval = -ENOTRECOVERABLE; + goto exit_rpm; } continue; @@ -250,6 +252,7 @@ static int stm32_rng_read(struct hwrng *rng, void *data, size_t max, bool wait) max -= sizeof(u32); } +exit_rpm: pm_runtime_mark_last_busy((struct device *) priv->rng.priv); pm_runtime_put_sync_autosuspend((struct device *) priv->rng.priv); @@ -353,13 +356,15 @@ static int stm32_rng_init(struct hwrng *rng) err = readl_relaxed_poll_timeout_atomic(priv->base + RNG_SR, reg, reg & RNG_SR_DRDY, 10, 100000); - if (err | (reg & ~RNG_SR_DRDY)) { + if (err || (reg & ~RNG_SR_DRDY)) { clk_disable_unprepare(priv->clk); dev_err((struct device *)priv->rng.priv, "%s: timeout:%x SR: %x!\n", __func__, err, reg); return -EINVAL; } + clk_disable_unprepare(priv->clk); + return 0; } @@ -384,6 +389,11 @@ static int __maybe_unused stm32_rng_runtime_suspend(struct device *dev) static int __maybe_unused stm32_rng_suspend(struct device *dev) { struct stm32_rng_private *priv = dev_get_drvdata(dev); + int err; + + err = clk_prepare_enable(priv->clk); + if (err) + return err; if (priv->data->has_cond_reset) { priv->pm_conf.nscr = readl_relaxed(priv->base + RNG_NSCR); @@ -465,6 +475,8 @@ static int __maybe_unused stm32_rng_resume(struct device *dev) writel_relaxed(reg, priv->base + RNG_CR); } + clk_disable_unprepare(priv->clk); + return 0; } diff --git a/drivers/char/tpm/Kconfig b/drivers/char/tpm/Kconfig index 927088b2c3d3..e63a6a17793c 100644 --- a/drivers/char/tpm/Kconfig +++ b/drivers/char/tpm/Kconfig @@ -27,6 +27,20 @@ menuconfig TCG_TPM if TCG_TPM +config TCG_TPM2_HMAC + bool "Use HMAC and encrypted transactions on the TPM bus" + default y + select CRYPTO_ECDH + select CRYPTO_LIB_AESCFB + select CRYPTO_LIB_SHA256 + help + Setting this causes us to deploy a scheme which uses request + and response HMACs in addition to encryption for + communicating with the TPM to prevent or detect bus snooping + and interposer attacks (see tpm-security.rst). Saying Y + here adds some encryption overhead to all kernel to TPM + transactions. + config HW_RANDOM_TPM bool "TPM HW Random Number Generator support" depends on TCG_TPM && HW_RANDOM && !(TCG_TPM=y && HW_RANDOM=m) @@ -149,6 +163,7 @@ config TCG_NSC config TCG_ATMEL tristate "Atmel TPM Interface" depends on PPC64 || HAS_IOPORT_MAP + depends on HAS_IOPORT help If you have a TPM security chip from Atmel say Yes and it will be accessible from within Linux. To compile this driver @@ -156,7 +171,7 @@ config TCG_ATMEL config TCG_INFINEON tristate "Infineon Technologies TPM Interface" - depends on PNP + depends on PNP || COMPILE_TEST help If you have a TPM security chip from Infineon Technologies (either SLD 9630 TT 1.1 or SLB 9635 TT 1.2) say Yes and it diff --git a/drivers/char/tpm/Makefile b/drivers/char/tpm/Makefile index 0222b1ddb310..4c695b0388f3 100644 --- a/drivers/char/tpm/Makefile +++ b/drivers/char/tpm/Makefile @@ -15,7 +15,9 @@ tpm-y += tpm-sysfs.o tpm-y += eventlog/common.o tpm-y += eventlog/tpm1.o tpm-y += eventlog/tpm2.o +tpm-y += tpm-buf.o +tpm-$(CONFIG_TCG_TPM2_HMAC) += tpm2-sessions.o tpm-$(CONFIG_ACPI) += tpm_ppi.o eventlog/acpi.o tpm-$(CONFIG_EFI) += eventlog/efi.o tpm-$(CONFIG_OF) += eventlog/of.o diff --git a/drivers/char/tpm/eventlog/acpi.c b/drivers/char/tpm/eventlog/acpi.c index bd757d836c5c..69533d0bfb51 100644 --- a/drivers/char/tpm/eventlog/acpi.c +++ b/drivers/char/tpm/eventlog/acpi.c @@ -142,7 +142,6 @@ int tpm_read_log_acpi(struct tpm_chip *chip) log->bios_event_log_end = log->bios_event_log + len; - ret = -EIO; virt = acpi_os_map_iomem(start, len); if (!virt) { dev_warn(&chip->dev, "%s: Failed to map ACPI memory\n", __func__); diff --git a/drivers/char/tpm/tpm-buf.c b/drivers/char/tpm/tpm-buf.c new file mode 100644 index 000000000000..647c6ca92ac3 --- /dev/null +++ b/drivers/char/tpm/tpm-buf.c @@ -0,0 +1,252 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Handling of TPM command and other buffers. + */ + +#include <linux/tpm_command.h> +#include <linux/module.h> +#include <linux/tpm.h> + +/** + * tpm_buf_init() - Allocate and initialize a TPM command + * @buf: A &tpm_buf + * @tag: TPM_TAG_RQU_COMMAND, TPM2_ST_NO_SESSIONS or TPM2_ST_SESSIONS + * @ordinal: A command ordinal + * + * Return: 0 or -ENOMEM + */ +int tpm_buf_init(struct tpm_buf *buf, u16 tag, u32 ordinal) +{ + buf->data = (u8 *)__get_free_page(GFP_KERNEL); + if (!buf->data) + return -ENOMEM; + + tpm_buf_reset(buf, tag, ordinal); + return 0; +} +EXPORT_SYMBOL_GPL(tpm_buf_init); + +/** + * tpm_buf_reset() - Initialize a TPM command + * @buf: A &tpm_buf + * @tag: TPM_TAG_RQU_COMMAND, TPM2_ST_NO_SESSIONS or TPM2_ST_SESSIONS + * @ordinal: A command ordinal + */ +void tpm_buf_reset(struct tpm_buf *buf, u16 tag, u32 ordinal) +{ + struct tpm_header *head = (struct tpm_header *)buf->data; + + WARN_ON(tag != TPM_TAG_RQU_COMMAND && tag != TPM2_ST_NO_SESSIONS && + tag != TPM2_ST_SESSIONS && tag != 0); + + buf->flags = 0; + buf->length = sizeof(*head); + head->tag = cpu_to_be16(tag); + head->length = cpu_to_be32(sizeof(*head)); + head->ordinal = cpu_to_be32(ordinal); + buf->handles = 0; +} +EXPORT_SYMBOL_GPL(tpm_buf_reset); + +/** + * tpm_buf_init_sized() - Allocate and initialize a sized (TPM2B) buffer + * @buf: A @tpm_buf + * + * Return: 0 or -ENOMEM + */ +int tpm_buf_init_sized(struct tpm_buf *buf) +{ + buf->data = (u8 *)__get_free_page(GFP_KERNEL); + if (!buf->data) + return -ENOMEM; + + tpm_buf_reset_sized(buf); + return 0; +} +EXPORT_SYMBOL_GPL(tpm_buf_init_sized); + +/** + * tpm_buf_reset_sized() - Initialize a sized buffer + * @buf: A &tpm_buf + */ +void tpm_buf_reset_sized(struct tpm_buf *buf) +{ + buf->flags = TPM_BUF_TPM2B; + buf->length = 2; + buf->data[0] = 0; + buf->data[1] = 0; +} +EXPORT_SYMBOL_GPL(tpm_buf_reset_sized); + +void tpm_buf_destroy(struct tpm_buf *buf) +{ + free_page((unsigned long)buf->data); +} +EXPORT_SYMBOL_GPL(tpm_buf_destroy); + +/** + * tpm_buf_length() - Return the number of bytes consumed by the data + * @buf: A &tpm_buf + * + * Return: The number of bytes consumed by the buffer + */ +u32 tpm_buf_length(struct tpm_buf *buf) +{ + return buf->length; +} +EXPORT_SYMBOL_GPL(tpm_buf_length); + +/** + * tpm_buf_append() - Append data to an initialized buffer + * @buf: A &tpm_buf + * @new_data: A data blob + * @new_length: Size of the appended data + */ +void tpm_buf_append(struct tpm_buf *buf, const u8 *new_data, u16 new_length) +{ + /* Return silently if overflow has already happened. */ + if (buf->flags & TPM_BUF_OVERFLOW) + return; + + if ((buf->length + new_length) > PAGE_SIZE) { + WARN(1, "tpm_buf: write overflow\n"); + buf->flags |= TPM_BUF_OVERFLOW; + return; + } + + memcpy(&buf->data[buf->length], new_data, new_length); + buf->length += new_length; + + if (buf->flags & TPM_BUF_TPM2B) + ((__be16 *)buf->data)[0] = cpu_to_be16(buf->length - 2); + else + ((struct tpm_header *)buf->data)->length = cpu_to_be32(buf->length); +} +EXPORT_SYMBOL_GPL(tpm_buf_append); + +void tpm_buf_append_u8(struct tpm_buf *buf, const u8 value) +{ + tpm_buf_append(buf, &value, 1); +} +EXPORT_SYMBOL_GPL(tpm_buf_append_u8); + +void tpm_buf_append_u16(struct tpm_buf *buf, const u16 value) +{ + __be16 value2 = cpu_to_be16(value); + + tpm_buf_append(buf, (u8 *)&value2, 2); +} +EXPORT_SYMBOL_GPL(tpm_buf_append_u16); + +void tpm_buf_append_u32(struct tpm_buf *buf, const u32 value) +{ + __be32 value2 = cpu_to_be32(value); + + tpm_buf_append(buf, (u8 *)&value2, 4); +} +EXPORT_SYMBOL_GPL(tpm_buf_append_u32); + +/** + * tpm_buf_read() - Read from a TPM buffer + * @buf: &tpm_buf instance + * @offset: offset within the buffer + * @count: the number of bytes to read + * @output: the output buffer + */ +static void tpm_buf_read(struct tpm_buf *buf, off_t *offset, size_t count, void *output) +{ + off_t next_offset; + + /* Return silently if overflow has already happened. */ + if (buf->flags & TPM_BUF_BOUNDARY_ERROR) + return; + + next_offset = *offset + count; + if (next_offset > buf->length) { + WARN(1, "tpm_buf: read out of boundary\n"); + buf->flags |= TPM_BUF_BOUNDARY_ERROR; + return; + } + + memcpy(output, &buf->data[*offset], count); + *offset = next_offset; +} + +/** + * tpm_buf_read_u8() - Read 8-bit word from a TPM buffer + * @buf: &tpm_buf instance + * @offset: offset within the buffer + * + * Return: next 8-bit word + */ +u8 tpm_buf_read_u8(struct tpm_buf *buf, off_t *offset) +{ + u8 value; + + tpm_buf_read(buf, offset, sizeof(value), &value); + + return value; +} +EXPORT_SYMBOL_GPL(tpm_buf_read_u8); + +/** + * tpm_buf_read_u16() - Read 16-bit word from a TPM buffer + * @buf: &tpm_buf instance + * @offset: offset within the buffer + * + * Return: next 16-bit word + */ +u16 tpm_buf_read_u16(struct tpm_buf *buf, off_t *offset) +{ + u16 value; + + tpm_buf_read(buf, offset, sizeof(value), &value); + + return be16_to_cpu(value); +} +EXPORT_SYMBOL_GPL(tpm_buf_read_u16); + +/** + * tpm_buf_read_u32() - Read 32-bit word from a TPM buffer + * @buf: &tpm_buf instance + * @offset: offset within the buffer + * + * Return: next 32-bit word + */ +u32 tpm_buf_read_u32(struct tpm_buf *buf, off_t *offset) +{ + u32 value; + + tpm_buf_read(buf, offset, sizeof(value), &value); + + return be32_to_cpu(value); +} +EXPORT_SYMBOL_GPL(tpm_buf_read_u32); + +static u16 tpm_buf_tag(struct tpm_buf *buf) +{ + struct tpm_header *head = (struct tpm_header *)buf->data; + + return be16_to_cpu(head->tag); +} + +/** + * tpm_buf_parameters - return the TPM response parameters area of the tpm_buf + * @buf: tpm_buf to use + * + * Where the parameters are located depends on the tag of a TPM + * command (it's immediately after the header for TPM_ST_NO_SESSIONS + * or 4 bytes after for TPM_ST_SESSIONS). Evaluate this and return a + * pointer to the first byte of the parameters area. + * + * @return: pointer to parameters area + */ +u8 *tpm_buf_parameters(struct tpm_buf *buf) +{ + int offset = TPM_HEADER_SIZE; + + if (tpm_buf_tag(buf) == TPM2_ST_SESSIONS) + offset += 4; + + return &buf->data[offset]; +} diff --git a/drivers/char/tpm/tpm-chip.c b/drivers/char/tpm/tpm-chip.c index 42b1062e33cd..854546000c92 100644 --- a/drivers/char/tpm/tpm-chip.c +++ b/drivers/char/tpm/tpm-chip.c @@ -158,6 +158,9 @@ int tpm_try_get_ops(struct tpm_chip *chip) { int rc = -EIO; + if (chip->flags & TPM_CHIP_FLAG_DISABLE) + return rc; + get_device(&chip->dev); down_read(&chip->ops_sem); @@ -275,6 +278,9 @@ static void tpm_dev_release(struct device *dev) kfree(chip->work_space.context_buf); kfree(chip->work_space.session_buf); kfree(chip->allocated_banks); +#ifdef CONFIG_TCG_TPM2_HMAC + kfree(chip->auth); +#endif kfree(chip); } diff --git a/drivers/char/tpm/tpm-interface.c b/drivers/char/tpm/tpm-interface.c index 757336324c90..5da134f12c9a 100644 --- a/drivers/char/tpm/tpm-interface.c +++ b/drivers/char/tpm/tpm-interface.c @@ -232,6 +232,7 @@ ssize_t tpm_transmit_cmd(struct tpm_chip *chip, struct tpm_buf *buf, if (len < min_rsp_body_length + TPM_HEADER_SIZE) return -EFAULT; + buf->length = len; return 0; } EXPORT_SYMBOL_GPL(tpm_transmit_cmd); @@ -342,31 +343,6 @@ out: } EXPORT_SYMBOL_GPL(tpm_pcr_extend); -/** - * tpm_send - send a TPM command - * @chip: a &struct tpm_chip instance, %NULL for the default chip - * @cmd: a TPM command buffer - * @buflen: the length of the TPM command buffer - * - * Return: same as with tpm_transmit_cmd() - */ -int tpm_send(struct tpm_chip *chip, void *cmd, size_t buflen) -{ - struct tpm_buf buf; - int rc; - - chip = tpm_find_get_ops(chip); - if (!chip) - return -ENODEV; - - buf.data = cmd; - rc = tpm_transmit_cmd(chip, &buf, 0, "attempting to a send a command"); - - tpm_put_ops(chip); - return rc; -} -EXPORT_SYMBOL_GPL(tpm_send); - int tpm_auto_startup(struct tpm_chip *chip) { int rc; diff --git a/drivers/char/tpm/tpm-sysfs.c b/drivers/char/tpm/tpm-sysfs.c index 54c71473aa29..94231f052ea7 100644 --- a/drivers/char/tpm/tpm-sysfs.c +++ b/drivers/char/tpm/tpm-sysfs.c @@ -309,6 +309,21 @@ static ssize_t tpm_version_major_show(struct device *dev, } static DEVICE_ATTR_RO(tpm_version_major); +#ifdef CONFIG_TCG_TPM2_HMAC +static ssize_t null_name_show(struct device *dev, struct device_attribute *attr, + char *buf) +{ + struct tpm_chip *chip = to_tpm_chip(dev); + int size = TPM2_NAME_SIZE; + + bin2hex(buf, chip->null_key_name, size); + size *= 2; + buf[size++] = '\n'; + return size; +} +static DEVICE_ATTR_RO(null_name); +#endif + static struct attribute *tpm1_dev_attrs[] = { &dev_attr_pubek.attr, &dev_attr_pcrs.attr, @@ -326,6 +341,9 @@ static struct attribute *tpm1_dev_attrs[] = { static struct attribute *tpm2_dev_attrs[] = { &dev_attr_tpm_version_major.attr, +#ifdef CONFIG_TCG_TPM2_HMAC + &dev_attr_null_name.attr, +#endif NULL }; diff --git a/drivers/char/tpm/tpm.h b/drivers/char/tpm/tpm.h index 61445f1dc46d..6b8b9956ba69 100644 --- a/drivers/char/tpm/tpm.h +++ b/drivers/char/tpm/tpm.h @@ -312,9 +312,23 @@ int tpm2_commit_space(struct tpm_chip *chip, struct tpm_space *space, void *buf, size_t *bufsiz); int tpm_devs_add(struct tpm_chip *chip); void tpm_devs_remove(struct tpm_chip *chip); +int tpm2_save_context(struct tpm_chip *chip, u32 handle, u8 *buf, + unsigned int buf_size, unsigned int *offset); +int tpm2_load_context(struct tpm_chip *chip, u8 *buf, + unsigned int *offset, u32 *handle); void tpm_bios_log_setup(struct tpm_chip *chip); void tpm_bios_log_teardown(struct tpm_chip *chip); int tpm_dev_common_init(void); void tpm_dev_common_exit(void); + +#ifdef CONFIG_TCG_TPM2_HMAC +int tpm2_sessions_init(struct tpm_chip *chip); +#else +static inline int tpm2_sessions_init(struct tpm_chip *chip) +{ + return 0; +} +#endif + #endif diff --git a/drivers/char/tpm/tpm2-cmd.c b/drivers/char/tpm/tpm2-cmd.c index 93545be190a5..0cdf892ec2a7 100644 --- a/drivers/char/tpm/tpm2-cmd.c +++ b/drivers/char/tpm/tpm2-cmd.c @@ -216,13 +216,6 @@ out: return rc; } -struct tpm2_null_auth_area { - __be32 handle; - __be16 nonce_size; - u8 attributes; - __be16 auth_size; -} __packed; - /** * tpm2_pcr_extend() - extend a PCR value * @@ -236,24 +229,22 @@ int tpm2_pcr_extend(struct tpm_chip *chip, u32 pcr_idx, struct tpm_digest *digests) { struct tpm_buf buf; - struct tpm2_null_auth_area auth_area; int rc; int i; - rc = tpm_buf_init(&buf, TPM2_ST_SESSIONS, TPM2_CC_PCR_EXTEND); + rc = tpm2_start_auth_session(chip); if (rc) return rc; - tpm_buf_append_u32(&buf, pcr_idx); + rc = tpm_buf_init(&buf, TPM2_ST_SESSIONS, TPM2_CC_PCR_EXTEND); + if (rc) { + tpm2_end_auth_session(chip); + return rc; + } - auth_area.handle = cpu_to_be32(TPM2_RS_PW); - auth_area.nonce_size = 0; - auth_area.attributes = 0; - auth_area.auth_size = 0; + tpm_buf_append_name(chip, &buf, pcr_idx, NULL); + tpm_buf_append_hmac_session(chip, &buf, 0, NULL, 0); - tpm_buf_append_u32(&buf, sizeof(struct tpm2_null_auth_area)); - tpm_buf_append(&buf, (const unsigned char *)&auth_area, - sizeof(auth_area)); tpm_buf_append_u32(&buf, chip->nr_allocated_banks); for (i = 0; i < chip->nr_allocated_banks; i++) { @@ -262,7 +253,9 @@ int tpm2_pcr_extend(struct tpm_chip *chip, u32 pcr_idx, chip->allocated_banks[i].digest_size); } + tpm_buf_fill_hmac_session(chip, &buf); rc = tpm_transmit_cmd(chip, &buf, 0, "attempting extend a PCR value"); + rc = tpm_buf_check_hmac_response(chip, &buf, rc); tpm_buf_destroy(&buf); @@ -299,25 +292,35 @@ int tpm2_get_random(struct tpm_chip *chip, u8 *dest, size_t max) if (!num_bytes || max > TPM_MAX_RNG_DATA) return -EINVAL; - err = tpm_buf_init(&buf, 0, 0); + err = tpm2_start_auth_session(chip); if (err) return err; + err = tpm_buf_init(&buf, 0, 0); + if (err) { + tpm2_end_auth_session(chip); + return err; + } + do { - tpm_buf_reset(&buf, TPM2_ST_NO_SESSIONS, TPM2_CC_GET_RANDOM); + tpm_buf_reset(&buf, TPM2_ST_SESSIONS, TPM2_CC_GET_RANDOM); + tpm_buf_append_hmac_session_opt(chip, &buf, TPM2_SA_ENCRYPT + | TPM2_SA_CONTINUE_SESSION, + NULL, 0); tpm_buf_append_u16(&buf, num_bytes); + tpm_buf_fill_hmac_session(chip, &buf); err = tpm_transmit_cmd(chip, &buf, offsetof(struct tpm2_get_random_out, buffer), "attempting get random"); + err = tpm_buf_check_hmac_response(chip, &buf, err); if (err) { if (err > 0) err = -EIO; goto out; } - out = (struct tpm2_get_random_out *) - &buf.data[TPM_HEADER_SIZE]; + out = (struct tpm2_get_random_out *)tpm_buf_parameters(&buf); recd = min_t(u32, be16_to_cpu(out->size), num_bytes); if (tpm_buf_length(&buf) < TPM_HEADER_SIZE + @@ -334,9 +337,12 @@ int tpm2_get_random(struct tpm_chip *chip, u8 *dest, size_t max) } while (retries-- && total < max); tpm_buf_destroy(&buf); + tpm2_end_auth_session(chip); + return total ? total : -EIO; out: tpm_buf_destroy(&buf); + tpm2_end_auth_session(chip); return err; } @@ -759,6 +765,11 @@ int tpm2_auto_startup(struct tpm_chip *chip) rc = 0; } + if (rc) + goto out; + + rc = tpm2_sessions_init(chip); + out: /* * Infineon TPM in field upgrade mode will return no data for the number diff --git a/drivers/char/tpm/tpm2-sessions.c b/drivers/char/tpm/tpm2-sessions.c new file mode 100644 index 000000000000..ea8860661876 --- /dev/null +++ b/drivers/char/tpm/tpm2-sessions.c @@ -0,0 +1,1286 @@ +// SPDX-License-Identifier: GPL-2.0 + +/* + * Copyright (C) 2018 James.Bottomley@HansenPartnership.com + * + * Cryptographic helper routines for handling TPM2 sessions for + * authorization HMAC and request response encryption. + * + * The idea is to ensure that every TPM command is HMAC protected by a + * session, meaning in-flight tampering would be detected and in + * addition all sensitive inputs and responses should be encrypted. + * + * The basic way this works is to use a TPM feature called salted + * sessions where a random secret used in session construction is + * encrypted to the public part of a known TPM key. The problem is we + * have no known keys, so initially a primary Elliptic Curve key is + * derived from the NULL seed (we use EC because most TPMs generate + * these keys much faster than RSA ones). The curve used is NIST_P256 + * because that's now mandated to be present in 'TCG TPM v2.0 + * Provisioning Guidance' + * + * Threat problems: the initial TPM2_CreatePrimary is not (and cannot + * be) session protected, so a clever Man in the Middle could return a + * public key they control to this command and from there intercept + * and decode all subsequent session based transactions. The kernel + * cannot mitigate this threat but, after boot, userspace can get + * proof this has not happened by asking the TPM to certify the NULL + * key. This certification would chain back to the TPM Endorsement + * Certificate and prove the NULL seed primary had not been tampered + * with and thus all sessions must have been cryptographically secure. + * To assist with this, the initial NULL seed public key name is made + * available in a sysfs file. + * + * Use of these functions: + * + * The design is all the crypto, hash and hmac gunk is confined in this + * file and never needs to be seen even by the kernel internal user. To + * the user there's an init function tpm2_sessions_init() that needs to + * be called once per TPM which generates the NULL seed primary key. + * + * These are the usage functions: + * + * tpm2_start_auth_session() which allocates the opaque auth structure + * and gets a session from the TPM. This must be called before + * any of the following functions. The session is protected by a + * session_key which is derived from a random salt value + * encrypted to the NULL seed. + * tpm2_end_auth_session() kills the session and frees the resources. + * Under normal operation this function is done by + * tpm_buf_check_hmac_response(), so this is only to be used on + * error legs where the latter is not executed. + * tpm_buf_append_name() to add a handle to the buffer. This must be + * used in place of the usual tpm_buf_append_u32() for adding + * handles because handles have to be processed specially when + * calculating the HMAC. In particular, for NV, volatile and + * permanent objects you now need to provide the name. + * tpm_buf_append_hmac_session() which appends the hmac session to the + * buf in the same way tpm_buf_append_auth does(). + * tpm_buf_fill_hmac_session() This calculates the correct hash and + * places it in the buffer. It must be called after the complete + * command buffer is finalized so it can fill in the correct HMAC + * based on the parameters. + * tpm_buf_check_hmac_response() which checks the session response in + * the buffer and calculates what it should be. If there's a + * mismatch it will log a warning and return an error. If + * tpm_buf_append_hmac_session() did not specify + * TPM_SA_CONTINUE_SESSION then the session will be closed (if it + * hasn't been consumed) and the auth structure freed. + */ + +#include "tpm.h" +#include <linux/random.h> +#include <linux/scatterlist.h> +#include <asm/unaligned.h> +#include <crypto/kpp.h> +#include <crypto/ecdh.h> +#include <crypto/hash.h> +#include <crypto/hmac.h> + +/* maximum number of names the TPM must remember for authorization */ +#define AUTH_MAX_NAMES 3 + +static int tpm2_create_primary(struct tpm_chip *chip, u32 hierarchy, + u32 *handle, u8 *name); + +/* + * This is the structure that carries all the auth information (like + * session handle, nonces, session key and auth) from use to use it is + * designed to be opaque to anything outside. + */ +struct tpm2_auth { + u32 handle; + /* + * This has two meanings: before tpm_buf_fill_hmac_session() + * it marks the offset in the buffer of the start of the + * sessions (i.e. after all the handles). Once the buffer has + * been filled it markes the session number of our auth + * session so we can find it again in the response buffer. + * + * The two cases are distinguished because the first offset + * must always be greater than TPM_HEADER_SIZE and the second + * must be less than or equal to 5. + */ + u32 session; + /* + * the size here is variable and set by the size of our_nonce + * which must be between 16 and the name hash length. we set + * the maximum sha256 size for the greatest protection + */ + u8 our_nonce[SHA256_DIGEST_SIZE]; + u8 tpm_nonce[SHA256_DIGEST_SIZE]; + /* + * the salt is only used across the session command/response + * after that it can be used as a scratch area + */ + union { + u8 salt[EC_PT_SZ]; + /* scratch for key + IV */ + u8 scratch[AES_KEY_BYTES + AES_BLOCK_SIZE]; + }; + /* + * the session key and passphrase are the same size as the + * name digest (sha256 again). The session key is constant + * for the use of the session and the passphrase can change + * with every invocation. + * + * Note: these fields must be adjacent and in this order + * because several HMAC/KDF schemes use the combination of the + * session_key and passphrase. + */ + u8 session_key[SHA256_DIGEST_SIZE]; + u8 passphrase[SHA256_DIGEST_SIZE]; + int passphrase_len; + struct crypto_aes_ctx aes_ctx; + /* saved session attributes: */ + u8 attrs; + __be32 ordinal; + + /* + * memory for three authorization handles. We know them by + * handle, but they are part of the session by name, which + * we must compute and remember + */ + u32 name_h[AUTH_MAX_NAMES]; + u8 name[AUTH_MAX_NAMES][2 + SHA512_DIGEST_SIZE]; +}; + +/* + * Name Size based on TPM algorithm (assumes no hash bigger than 255) + */ +static u8 name_size(const u8 *name) +{ + static u8 size_map[] = { + [TPM_ALG_SHA1] = SHA1_DIGEST_SIZE, + [TPM_ALG_SHA256] = SHA256_DIGEST_SIZE, + [TPM_ALG_SHA384] = SHA384_DIGEST_SIZE, + [TPM_ALG_SHA512] = SHA512_DIGEST_SIZE, + }; + u16 alg = get_unaligned_be16(name); + return size_map[alg] + 2; +} + +/* + * It turns out the crypto hmac(sha256) is hard for us to consume + * because it assumes a fixed key and the TPM seems to change the key + * on every operation, so we weld the hmac init and final functions in + * here to give it the same usage characteristics as a regular hash + */ +static void tpm2_hmac_init(struct sha256_state *sctx, u8 *key, u32 key_len) +{ + u8 pad[SHA256_BLOCK_SIZE]; + int i; + + sha256_init(sctx); + for (i = 0; i < sizeof(pad); i++) { + if (i < key_len) + pad[i] = key[i]; + else + pad[i] = 0; + pad[i] ^= HMAC_IPAD_VALUE; + } + sha256_update(sctx, pad, sizeof(pad)); +} + +static void tpm2_hmac_final(struct sha256_state *sctx, u8 *key, u32 key_len, + u8 *out) +{ + u8 pad[SHA256_BLOCK_SIZE]; + int i; + + for (i = 0; i < sizeof(pad); i++) { + if (i < key_len) + pad[i] = key[i]; + else + pad[i] = 0; + pad[i] ^= HMAC_OPAD_VALUE; + } + + /* collect the final hash; use out as temporary storage */ + sha256_final(sctx, out); + + sha256_init(sctx); + sha256_update(sctx, pad, sizeof(pad)); + sha256_update(sctx, out, SHA256_DIGEST_SIZE); + sha256_final(sctx, out); +} + +/* + * assume hash sha256 and nonces u, v of size SHA256_DIGEST_SIZE but + * otherwise standard tpm2_KDFa. Note output is in bytes not bits. + */ +static void tpm2_KDFa(u8 *key, u32 key_len, const char *label, u8 *u, + u8 *v, u32 bytes, u8 *out) +{ + u32 counter = 1; + const __be32 bits = cpu_to_be32(bytes * 8); + + while (bytes > 0) { + struct sha256_state sctx; + __be32 c = cpu_to_be32(counter); + + tpm2_hmac_init(&sctx, key, key_len); + sha256_update(&sctx, (u8 *)&c, sizeof(c)); + sha256_update(&sctx, label, strlen(label)+1); + sha256_update(&sctx, u, SHA256_DIGEST_SIZE); + sha256_update(&sctx, v, SHA256_DIGEST_SIZE); + sha256_update(&sctx, (u8 *)&bits, sizeof(bits)); + tpm2_hmac_final(&sctx, key, key_len, out); + + bytes -= SHA256_DIGEST_SIZE; + counter++; + out += SHA256_DIGEST_SIZE; + } +} + +/* + * Somewhat of a bastardization of the real KDFe. We're assuming + * we're working with known point sizes for the input parameters and + * the hash algorithm is fixed at sha256. Because we know that the + * point size is 32 bytes like the hash size, there's no need to loop + * in this KDF. + */ +static void tpm2_KDFe(u8 z[EC_PT_SZ], const char *str, u8 *pt_u, u8 *pt_v, + u8 *out) +{ + struct sha256_state sctx; + /* + * this should be an iterative counter, but because we know + * we're only taking 32 bytes for the point using a sha256 + * hash which is also 32 bytes, there's only one loop + */ + __be32 c = cpu_to_be32(1); + + sha256_init(&sctx); + /* counter (BE) */ + sha256_update(&sctx, (u8 *)&c, sizeof(c)); + /* secret value */ + sha256_update(&sctx, z, EC_PT_SZ); + /* string including trailing zero */ + sha256_update(&sctx, str, strlen(str)+1); + sha256_update(&sctx, pt_u, EC_PT_SZ); + sha256_update(&sctx, pt_v, EC_PT_SZ); + sha256_final(&sctx, out); +} + +static void tpm_buf_append_salt(struct tpm_buf *buf, struct tpm_chip *chip) +{ + struct crypto_kpp *kpp; + struct kpp_request *req; + struct scatterlist s[2], d[1]; + struct ecdh p = {0}; + u8 encoded_key[EC_PT_SZ], *x, *y; + unsigned int buf_len; + + /* secret is two sized points */ + tpm_buf_append_u16(buf, (EC_PT_SZ + 2)*2); + /* + * we cheat here and append uninitialized data to form + * the points. All we care about is getting the two + * co-ordinate pointers, which will be used to overwrite + * the uninitialized data + */ + tpm_buf_append_u16(buf, EC_PT_SZ); + x = &buf->data[tpm_buf_length(buf)]; + tpm_buf_append(buf, encoded_key, EC_PT_SZ); + tpm_buf_append_u16(buf, EC_PT_SZ); + y = &buf->data[tpm_buf_length(buf)]; + tpm_buf_append(buf, encoded_key, EC_PT_SZ); + sg_init_table(s, 2); + sg_set_buf(&s[0], x, EC_PT_SZ); + sg_set_buf(&s[1], y, EC_PT_SZ); + + kpp = crypto_alloc_kpp("ecdh-nist-p256", CRYPTO_ALG_INTERNAL, 0); + if (IS_ERR(kpp)) { + dev_err(&chip->dev, "crypto ecdh allocation failed\n"); + return; + } + + buf_len = crypto_ecdh_key_len(&p); + if (sizeof(encoded_key) < buf_len) { + dev_err(&chip->dev, "salt buffer too small needs %d\n", + buf_len); + goto out; + } + crypto_ecdh_encode_key(encoded_key, buf_len, &p); + /* this generates a random private key */ + crypto_kpp_set_secret(kpp, encoded_key, buf_len); + + /* salt is now the public point of this private key */ + req = kpp_request_alloc(kpp, GFP_KERNEL); + if (!req) + goto out; + kpp_request_set_input(req, NULL, 0); + kpp_request_set_output(req, s, EC_PT_SZ*2); + crypto_kpp_generate_public_key(req); + /* + * we're not done: now we have to compute the shared secret + * which is our private key multiplied by the tpm_key public + * point, we actually only take the x point and discard the y + * point and feed it through KDFe to get the final secret salt + */ + sg_set_buf(&s[0], chip->null_ec_key_x, EC_PT_SZ); + sg_set_buf(&s[1], chip->null_ec_key_y, EC_PT_SZ); + kpp_request_set_input(req, s, EC_PT_SZ*2); + sg_init_one(d, chip->auth->salt, EC_PT_SZ); + kpp_request_set_output(req, d, EC_PT_SZ); + crypto_kpp_compute_shared_secret(req); + kpp_request_free(req); + + /* + * pass the shared secret through KDFe for salt. Note salt + * area is used both for input shared secret and output salt. + * This works because KDFe fully consumes the secret before it + * writes the salt + */ + tpm2_KDFe(chip->auth->salt, "SECRET", x, chip->null_ec_key_x, + chip->auth->salt); + + out: + crypto_free_kpp(kpp); +} + +/** + * tpm_buf_append_hmac_session() - Append a TPM session element + * @chip: the TPM chip structure + * @buf: The buffer to be appended + * @attributes: The session attributes + * @passphrase: The session authority (NULL if none) + * @passphrase_len: The length of the session authority (0 if none) + * + * This fills in a session structure in the TPM command buffer, except + * for the HMAC which cannot be computed until the command buffer is + * complete. The type of session is controlled by the @attributes, + * the main ones of which are TPM2_SA_CONTINUE_SESSION which means the + * session won't terminate after tpm_buf_check_hmac_response(), + * TPM2_SA_DECRYPT which means this buffers first parameter should be + * encrypted with a session key and TPM2_SA_ENCRYPT, which means the + * response buffer's first parameter needs to be decrypted (confusing, + * but the defines are written from the point of view of the TPM). + * + * Any session appended by this command must be finalized by calling + * tpm_buf_fill_hmac_session() otherwise the HMAC will be incorrect + * and the TPM will reject the command. + * + * As with most tpm_buf operations, success is assumed because failure + * will be caused by an incorrect programming model and indicated by a + * kernel message. + */ +void tpm_buf_append_hmac_session(struct tpm_chip *chip, struct tpm_buf *buf, + u8 attributes, u8 *passphrase, + int passphrase_len) +{ + u8 nonce[SHA256_DIGEST_SIZE]; + u32 len; + struct tpm2_auth *auth = chip->auth; + + /* + * The Architecture Guide requires us to strip trailing zeros + * before computing the HMAC + */ + while (passphrase && passphrase_len > 0 + && passphrase[passphrase_len - 1] == '\0') + passphrase_len--; + + auth->attrs = attributes; + auth->passphrase_len = passphrase_len; + if (passphrase_len) + memcpy(auth->passphrase, passphrase, passphrase_len); + + if (auth->session != tpm_buf_length(buf)) { + /* we're not the first session */ + len = get_unaligned_be32(&buf->data[auth->session]); + if (4 + len + auth->session != tpm_buf_length(buf)) { + WARN(1, "session length mismatch, cannot append"); + return; + } + + /* add our new session */ + len += 9 + 2 * SHA256_DIGEST_SIZE; + put_unaligned_be32(len, &buf->data[auth->session]); + } else { + tpm_buf_append_u32(buf, 9 + 2 * SHA256_DIGEST_SIZE); + } + + /* random number for our nonce */ + get_random_bytes(nonce, sizeof(nonce)); + memcpy(auth->our_nonce, nonce, sizeof(nonce)); + tpm_buf_append_u32(buf, auth->handle); + /* our new nonce */ + tpm_buf_append_u16(buf, SHA256_DIGEST_SIZE); + tpm_buf_append(buf, nonce, SHA256_DIGEST_SIZE); + tpm_buf_append_u8(buf, auth->attrs); + /* and put a placeholder for the hmac */ + tpm_buf_append_u16(buf, SHA256_DIGEST_SIZE); + tpm_buf_append(buf, nonce, SHA256_DIGEST_SIZE); +} +EXPORT_SYMBOL(tpm_buf_append_hmac_session); + +/** + * tpm_buf_fill_hmac_session() - finalize the session HMAC + * @chip: the TPM chip structure + * @buf: The buffer to be appended + * + * This command must not be called until all of the parameters have + * been appended to @buf otherwise the computed HMAC will be + * incorrect. + * + * This function computes and fills in the session HMAC using the + * session key and, if TPM2_SA_DECRYPT was specified, computes the + * encryption key and encrypts the first parameter of the command + * buffer with it. + * + * As with most tpm_buf operations, success is assumed because failure + * will be caused by an incorrect programming model and indicated by a + * kernel message. + */ +void tpm_buf_fill_hmac_session(struct tpm_chip *chip, struct tpm_buf *buf) +{ + u32 cc, handles, val; + struct tpm2_auth *auth = chip->auth; + int i; + struct tpm_header *head = (struct tpm_header *)buf->data; + off_t offset_s = TPM_HEADER_SIZE, offset_p; + u8 *hmac = NULL; + u32 attrs; + u8 cphash[SHA256_DIGEST_SIZE]; + struct sha256_state sctx; + + /* save the command code in BE format */ + auth->ordinal = head->ordinal; + + cc = be32_to_cpu(head->ordinal); + + i = tpm2_find_cc(chip, cc); + if (i < 0) { + dev_err(&chip->dev, "Command 0x%x not found in TPM\n", cc); + return; + } + attrs = chip->cc_attrs_tbl[i]; + + handles = (attrs >> TPM2_CC_ATTR_CHANDLES) & GENMASK(2, 0); + + /* + * just check the names, it's easy to make mistakes. This + * would happen if someone added a handle via + * tpm_buf_append_u32() instead of tpm_buf_append_name() + */ + for (i = 0; i < handles; i++) { + u32 handle = tpm_buf_read_u32(buf, &offset_s); + + if (auth->name_h[i] != handle) { + dev_err(&chip->dev, "TPM: handle %d wrong for name\n", + i); + return; + } + } + /* point offset_s to the start of the sessions */ + val = tpm_buf_read_u32(buf, &offset_s); + /* point offset_p to the start of the parameters */ + offset_p = offset_s + val; + for (i = 1; offset_s < offset_p; i++) { + u32 handle = tpm_buf_read_u32(buf, &offset_s); + u16 len; + u8 a; + + /* nonce (already in auth) */ + len = tpm_buf_read_u16(buf, &offset_s); + offset_s += len; + + a = tpm_buf_read_u8(buf, &offset_s); + + len = tpm_buf_read_u16(buf, &offset_s); + if (handle == auth->handle && auth->attrs == a) { + hmac = &buf->data[offset_s]; + /* + * save our session number so we know which + * session in the response belongs to us + */ + auth->session = i; + } + + offset_s += len; + } + if (offset_s != offset_p) { + dev_err(&chip->dev, "TPM session length is incorrect\n"); + return; + } + if (!hmac) { + dev_err(&chip->dev, "TPM could not find HMAC session\n"); + return; + } + + /* encrypt before HMAC */ + if (auth->attrs & TPM2_SA_DECRYPT) { + u16 len; + + /* need key and IV */ + tpm2_KDFa(auth->session_key, SHA256_DIGEST_SIZE + + auth->passphrase_len, "CFB", auth->our_nonce, + auth->tpm_nonce, AES_KEY_BYTES + AES_BLOCK_SIZE, + auth->scratch); + + len = tpm_buf_read_u16(buf, &offset_p); + aes_expandkey(&auth->aes_ctx, auth->scratch, AES_KEY_BYTES); + aescfb_encrypt(&auth->aes_ctx, &buf->data[offset_p], + &buf->data[offset_p], len, + auth->scratch + AES_KEY_BYTES); + /* reset p to beginning of parameters for HMAC */ + offset_p -= 2; + } + + sha256_init(&sctx); + /* ordinal is already BE */ + sha256_update(&sctx, (u8 *)&head->ordinal, sizeof(head->ordinal)); + /* add the handle names */ + for (i = 0; i < handles; i++) { + enum tpm2_mso_type mso = tpm2_handle_mso(auth->name_h[i]); + + if (mso == TPM2_MSO_PERSISTENT || + mso == TPM2_MSO_VOLATILE || + mso == TPM2_MSO_NVRAM) { + sha256_update(&sctx, auth->name[i], + name_size(auth->name[i])); + } else { + __be32 h = cpu_to_be32(auth->name_h[i]); + + sha256_update(&sctx, (u8 *)&h, 4); + } + } + if (offset_s != tpm_buf_length(buf)) + sha256_update(&sctx, &buf->data[offset_s], + tpm_buf_length(buf) - offset_s); + sha256_final(&sctx, cphash); + + /* now calculate the hmac */ + tpm2_hmac_init(&sctx, auth->session_key, sizeof(auth->session_key) + + auth->passphrase_len); + sha256_update(&sctx, cphash, sizeof(cphash)); + sha256_update(&sctx, auth->our_nonce, sizeof(auth->our_nonce)); + sha256_update(&sctx, auth->tpm_nonce, sizeof(auth->tpm_nonce)); + sha256_update(&sctx, &auth->attrs, 1); + tpm2_hmac_final(&sctx, auth->session_key, sizeof(auth->session_key) + + auth->passphrase_len, hmac); +} +EXPORT_SYMBOL(tpm_buf_fill_hmac_session); + +static int tpm2_parse_read_public(char *name, struct tpm_buf *buf) +{ + struct tpm_header *head = (struct tpm_header *)buf->data; + off_t offset = TPM_HEADER_SIZE; + u32 tot_len = be32_to_cpu(head->length); + u32 val; + + /* we're starting after the header so adjust the length */ + tot_len -= TPM_HEADER_SIZE; + + /* skip public */ + val = tpm_buf_read_u16(buf, &offset); + if (val > tot_len) + return -EINVAL; + offset += val; + /* name */ + val = tpm_buf_read_u16(buf, &offset); + if (val != name_size(&buf->data[offset])) + return -EINVAL; + memcpy(name, &buf->data[offset], val); + /* forget the rest */ + return 0; +} + +static int tpm2_read_public(struct tpm_chip *chip, u32 handle, char *name) +{ + struct tpm_buf buf; + int rc; + + rc = tpm_buf_init(&buf, TPM2_ST_NO_SESSIONS, TPM2_CC_READ_PUBLIC); + if (rc) + return rc; + + tpm_buf_append_u32(&buf, handle); + rc = tpm_transmit_cmd(chip, &buf, 0, "read public"); + if (rc == TPM2_RC_SUCCESS) + rc = tpm2_parse_read_public(name, &buf); + + tpm_buf_destroy(&buf); + + return rc; +} + +/** + * tpm_buf_append_name() - add a handle area to the buffer + * @chip: the TPM chip structure + * @buf: The buffer to be appended + * @handle: The handle to be appended + * @name: The name of the handle (may be NULL) + * + * In order to compute session HMACs, we need to know the names of the + * objects pointed to by the handles. For most objects, this is simply + * the actual 4 byte handle or an empty buf (in these cases @name + * should be NULL) but for volatile objects, permanent objects and NV + * areas, the name is defined as the hash (according to the name + * algorithm which should be set to sha256) of the public area to + * which the two byte algorithm id has been appended. For these + * objects, the @name pointer should point to this. If a name is + * required but @name is NULL, then TPM2_ReadPublic() will be called + * on the handle to obtain the name. + * + * As with most tpm_buf operations, success is assumed because failure + * will be caused by an incorrect programming model and indicated by a + * kernel message. + */ +void tpm_buf_append_name(struct tpm_chip *chip, struct tpm_buf *buf, + u32 handle, u8 *name) +{ + enum tpm2_mso_type mso = tpm2_handle_mso(handle); + struct tpm2_auth *auth = chip->auth; + int slot; + + slot = (tpm_buf_length(buf) - TPM_HEADER_SIZE)/4; + if (slot >= AUTH_MAX_NAMES) { + dev_err(&chip->dev, "TPM: too many handles\n"); + return; + } + WARN(auth->session != tpm_buf_length(buf), + "name added in wrong place\n"); + tpm_buf_append_u32(buf, handle); + auth->session += 4; + + if (mso == TPM2_MSO_PERSISTENT || + mso == TPM2_MSO_VOLATILE || + mso == TPM2_MSO_NVRAM) { + if (!name) + tpm2_read_public(chip, handle, auth->name[slot]); + } else { + if (name) + dev_err(&chip->dev, "TPM: Handle does not require name but one is specified\n"); + } + + auth->name_h[slot] = handle; + if (name) + memcpy(auth->name[slot], name, name_size(name)); +} +EXPORT_SYMBOL(tpm_buf_append_name); + +/** + * tpm_buf_check_hmac_response() - check the TPM return HMAC for correctness + * @chip: the TPM chip structure + * @buf: the original command buffer (which now contains the response) + * @rc: the return code from tpm_transmit_cmd + * + * If @rc is non zero, @buf may not contain an actual return, so @rc + * is passed through as the return and the session cleaned up and + * de-allocated if required (this is required if + * TPM2_SA_CONTINUE_SESSION was not specified as a session flag). + * + * If @rc is zero, the response HMAC is computed against the returned + * @buf and matched to the TPM one in the session area. If there is a + * mismatch, an error is logged and -EINVAL returned. + * + * The reason for this is that the command issue and HMAC check + * sequence should look like: + * + * rc = tpm_transmit_cmd(...); + * rc = tpm_buf_check_hmac_response(&buf, auth, rc); + * if (rc) + * ... + * + * Which is easily layered into the current contrl flow. + * + * Returns: 0 on success or an error. + */ +int tpm_buf_check_hmac_response(struct tpm_chip *chip, struct tpm_buf *buf, + int rc) +{ + struct tpm_header *head = (struct tpm_header *)buf->data; + struct tpm2_auth *auth = chip->auth; + off_t offset_s, offset_p; + u8 rphash[SHA256_DIGEST_SIZE]; + u32 attrs; + struct sha256_state sctx; + u16 tag = be16_to_cpu(head->tag); + u32 cc = be32_to_cpu(auth->ordinal); + int parm_len, len, i, handles; + + if (auth->session >= TPM_HEADER_SIZE) { + WARN(1, "tpm session not filled correctly\n"); + goto out; + } + + if (rc != 0) + /* pass non success rc through and close the session */ + goto out; + + rc = -EINVAL; + if (tag != TPM2_ST_SESSIONS) { + dev_err(&chip->dev, "TPM: HMAC response check has no sessions tag\n"); + goto out; + } + + i = tpm2_find_cc(chip, cc); + if (i < 0) + goto out; + attrs = chip->cc_attrs_tbl[i]; + handles = (attrs >> TPM2_CC_ATTR_RHANDLE) & 1; + + /* point to area beyond handles */ + offset_s = TPM_HEADER_SIZE + handles * 4; + parm_len = tpm_buf_read_u32(buf, &offset_s); + offset_p = offset_s; + offset_s += parm_len; + /* skip over any sessions before ours */ + for (i = 0; i < auth->session - 1; i++) { + len = tpm_buf_read_u16(buf, &offset_s); + offset_s += len + 1; + len = tpm_buf_read_u16(buf, &offset_s); + offset_s += len; + } + /* TPM nonce */ + len = tpm_buf_read_u16(buf, &offset_s); + if (offset_s + len > tpm_buf_length(buf)) + goto out; + if (len != SHA256_DIGEST_SIZE) + goto out; + memcpy(auth->tpm_nonce, &buf->data[offset_s], len); + offset_s += len; + attrs = tpm_buf_read_u8(buf, &offset_s); + len = tpm_buf_read_u16(buf, &offset_s); + if (offset_s + len != tpm_buf_length(buf)) + goto out; + if (len != SHA256_DIGEST_SIZE) + goto out; + /* + * offset_s points to the HMAC. now calculate comparison, beginning + * with rphash + */ + sha256_init(&sctx); + /* yes, I know this is now zero, but it's what the standard says */ + sha256_update(&sctx, (u8 *)&head->return_code, + sizeof(head->return_code)); + /* ordinal is already BE */ + sha256_update(&sctx, (u8 *)&auth->ordinal, sizeof(auth->ordinal)); + sha256_update(&sctx, &buf->data[offset_p], parm_len); + sha256_final(&sctx, rphash); + + /* now calculate the hmac */ + tpm2_hmac_init(&sctx, auth->session_key, sizeof(auth->session_key) + + auth->passphrase_len); + sha256_update(&sctx, rphash, sizeof(rphash)); + sha256_update(&sctx, auth->tpm_nonce, sizeof(auth->tpm_nonce)); + sha256_update(&sctx, auth->our_nonce, sizeof(auth->our_nonce)); + sha256_update(&sctx, &auth->attrs, 1); + /* we're done with the rphash, so put our idea of the hmac there */ + tpm2_hmac_final(&sctx, auth->session_key, sizeof(auth->session_key) + + auth->passphrase_len, rphash); + if (memcmp(rphash, &buf->data[offset_s], SHA256_DIGEST_SIZE) == 0) { + rc = 0; + } else { + dev_err(&chip->dev, "TPM: HMAC check failed\n"); + goto out; + } + + /* now do response decryption */ + if (auth->attrs & TPM2_SA_ENCRYPT) { + /* need key and IV */ + tpm2_KDFa(auth->session_key, SHA256_DIGEST_SIZE + + auth->passphrase_len, "CFB", auth->tpm_nonce, + auth->our_nonce, AES_KEY_BYTES + AES_BLOCK_SIZE, + auth->scratch); + + len = tpm_buf_read_u16(buf, &offset_p); + aes_expandkey(&auth->aes_ctx, auth->scratch, AES_KEY_BYTES); + aescfb_decrypt(&auth->aes_ctx, &buf->data[offset_p], + &buf->data[offset_p], len, + auth->scratch + AES_KEY_BYTES); + } + + out: + if ((auth->attrs & TPM2_SA_CONTINUE_SESSION) == 0) { + if (rc) + /* manually close the session if it wasn't consumed */ + tpm2_flush_context(chip, auth->handle); + memzero_explicit(auth, sizeof(*auth)); + } else { + /* reset for next use */ + auth->session = TPM_HEADER_SIZE; + } + + return rc; +} +EXPORT_SYMBOL(tpm_buf_check_hmac_response); + +/** + * tpm2_end_auth_session() - kill the allocated auth session + * @chip: the TPM chip structure + * + * ends the session started by tpm2_start_auth_session and frees all + * the resources. Under normal conditions, + * tpm_buf_check_hmac_response() will correctly end the session if + * required, so this function is only for use in error legs that will + * bypass the normal invocation of tpm_buf_check_hmac_response(). + */ +void tpm2_end_auth_session(struct tpm_chip *chip) +{ + tpm2_flush_context(chip, chip->auth->handle); + memzero_explicit(chip->auth, sizeof(*chip->auth)); +} +EXPORT_SYMBOL(tpm2_end_auth_session); + +static int tpm2_parse_start_auth_session(struct tpm2_auth *auth, + struct tpm_buf *buf) +{ + struct tpm_header *head = (struct tpm_header *)buf->data; + u32 tot_len = be32_to_cpu(head->length); + off_t offset = TPM_HEADER_SIZE; + u32 val; + + /* we're starting after the header so adjust the length */ + tot_len -= TPM_HEADER_SIZE; + + /* should have handle plus nonce */ + if (tot_len != 4 + 2 + sizeof(auth->tpm_nonce)) + return -EINVAL; + + auth->handle = tpm_buf_read_u32(buf, &offset); + val = tpm_buf_read_u16(buf, &offset); + if (val != sizeof(auth->tpm_nonce)) + return -EINVAL; + memcpy(auth->tpm_nonce, &buf->data[offset], sizeof(auth->tpm_nonce)); + /* now compute the session key from the nonces */ + tpm2_KDFa(auth->salt, sizeof(auth->salt), "ATH", auth->tpm_nonce, + auth->our_nonce, sizeof(auth->session_key), + auth->session_key); + + return 0; +} + +static int tpm2_load_null(struct tpm_chip *chip, u32 *null_key) +{ + int rc; + unsigned int offset = 0; /* dummy offset for null seed context */ + u8 name[SHA256_DIGEST_SIZE + 2]; + + rc = tpm2_load_context(chip, chip->null_key_context, &offset, + null_key); + if (rc != -EINVAL) + return rc; + + /* an integrity failure may mean the TPM has been reset */ + dev_err(&chip->dev, "NULL key integrity failure!\n"); + /* check the null name against what we know */ + tpm2_create_primary(chip, TPM2_RH_NULL, NULL, name); + if (memcmp(name, chip->null_key_name, sizeof(name)) == 0) + /* name unchanged, assume transient integrity failure */ + return rc; + /* + * Fatal TPM failure: the NULL seed has actually changed, so + * the TPM must have been illegally reset. All in-kernel TPM + * operations will fail because the NULL primary can't be + * loaded to salt the sessions, but disable the TPM anyway so + * userspace programmes can't be compromised by it. + */ + dev_err(&chip->dev, "NULL name has changed, disabling TPM due to interference\n"); + chip->flags |= TPM_CHIP_FLAG_DISABLE; + + return rc; +} + +/** + * tpm2_start_auth_session() - create a HMAC authentication session with the TPM + * @chip: the TPM chip structure to create the session with + * + * This function loads the NULL seed from its saved context and starts + * an authentication session on the null seed, fills in the + * @chip->auth structure to contain all the session details necessary + * for performing the HMAC, encrypt and decrypt operations and + * returns. The NULL seed is flushed before this function returns. + * + * Return: zero on success or actual error encountered. + */ +int tpm2_start_auth_session(struct tpm_chip *chip) +{ + struct tpm_buf buf; + struct tpm2_auth *auth = chip->auth; + int rc; + u32 null_key; + + rc = tpm2_load_null(chip, &null_key); + if (rc) + goto out; + + auth->session = TPM_HEADER_SIZE; + + rc = tpm_buf_init(&buf, TPM2_ST_NO_SESSIONS, TPM2_CC_START_AUTH_SESS); + if (rc) + goto out; + + /* salt key handle */ + tpm_buf_append_u32(&buf, null_key); + /* bind key handle */ + tpm_buf_append_u32(&buf, TPM2_RH_NULL); + /* nonce caller */ + get_random_bytes(auth->our_nonce, sizeof(auth->our_nonce)); + tpm_buf_append_u16(&buf, sizeof(auth->our_nonce)); + tpm_buf_append(&buf, auth->our_nonce, sizeof(auth->our_nonce)); + + /* append encrypted salt and squirrel away unencrypted in auth */ + tpm_buf_append_salt(&buf, chip); + /* session type (HMAC, audit or policy) */ + tpm_buf_append_u8(&buf, TPM2_SE_HMAC); + + /* symmetric encryption parameters */ + /* symmetric algorithm */ + tpm_buf_append_u16(&buf, TPM_ALG_AES); + /* bits for symmetric algorithm */ + tpm_buf_append_u16(&buf, AES_KEY_BITS); + /* symmetric algorithm mode (must be CFB) */ + tpm_buf_append_u16(&buf, TPM_ALG_CFB); + /* hash algorithm for session */ + tpm_buf_append_u16(&buf, TPM_ALG_SHA256); + + rc = tpm_transmit_cmd(chip, &buf, 0, "start auth session"); + tpm2_flush_context(chip, null_key); + + if (rc == TPM2_RC_SUCCESS) + rc = tpm2_parse_start_auth_session(auth, &buf); + + tpm_buf_destroy(&buf); + + if (rc) + goto out; + + out: + return rc; +} +EXPORT_SYMBOL(tpm2_start_auth_session); + +/** + * tpm2_parse_create_primary() - parse the data returned from TPM_CC_CREATE_PRIMARY + * + * @chip: The TPM the primary was created under + * @buf: The response buffer from the chip + * @handle: pointer to be filled in with the return handle of the primary + * @hierarchy: The hierarchy the primary was created for + * @name: pointer to be filled in with the primary key name + * + * Return: + * * 0 - OK + * * -errno - A system error + * * TPM_RC - A TPM error + */ +static int tpm2_parse_create_primary(struct tpm_chip *chip, struct tpm_buf *buf, + u32 *handle, u32 hierarchy, u8 *name) +{ + struct tpm_header *head = (struct tpm_header *)buf->data; + off_t offset_r = TPM_HEADER_SIZE, offset_t; + u16 len = TPM_HEADER_SIZE; + u32 total_len = be32_to_cpu(head->length); + u32 val, param_len, keyhandle; + + keyhandle = tpm_buf_read_u32(buf, &offset_r); + if (handle) + *handle = keyhandle; + else + tpm2_flush_context(chip, keyhandle); + + param_len = tpm_buf_read_u32(buf, &offset_r); + /* + * param_len doesn't include the header, but all the other + * lengths and offsets do, so add it to parm len to make + * the comparisons easier + */ + param_len += TPM_HEADER_SIZE; + + if (param_len + 8 > total_len) + return -EINVAL; + len = tpm_buf_read_u16(buf, &offset_r); + offset_t = offset_r; + if (name) { + /* + * now we have the public area, compute the name of + * the object + */ + put_unaligned_be16(TPM_ALG_SHA256, name); + sha256(&buf->data[offset_r], len, name + 2); + } + + /* validate the public key */ + val = tpm_buf_read_u16(buf, &offset_t); + + /* key type (must be what we asked for) */ + if (val != TPM_ALG_ECC) + return -EINVAL; + val = tpm_buf_read_u16(buf, &offset_t); + + /* name algorithm */ + if (val != TPM_ALG_SHA256) + return -EINVAL; + val = tpm_buf_read_u32(buf, &offset_t); + + /* object properties */ + if (val != TPM2_OA_TMPL) + return -EINVAL; + + /* auth policy (empty) */ + val = tpm_buf_read_u16(buf, &offset_t); + if (val != 0) + return -EINVAL; + + /* symmetric key parameters */ + val = tpm_buf_read_u16(buf, &offset_t); + if (val != TPM_ALG_AES) + return -EINVAL; + + /* symmetric key length */ + val = tpm_buf_read_u16(buf, &offset_t); + if (val != AES_KEY_BITS) + return -EINVAL; + + /* symmetric encryption scheme */ + val = tpm_buf_read_u16(buf, &offset_t); + if (val != TPM_ALG_CFB) + return -EINVAL; + + /* signing scheme */ + val = tpm_buf_read_u16(buf, &offset_t); + if (val != TPM_ALG_NULL) + return -EINVAL; + + /* ECC Curve */ + val = tpm_buf_read_u16(buf, &offset_t); + if (val != TPM2_ECC_NIST_P256) + return -EINVAL; + + /* KDF Scheme */ + val = tpm_buf_read_u16(buf, &offset_t); + if (val != TPM_ALG_NULL) + return -EINVAL; + + /* extract public key (x and y points) */ + val = tpm_buf_read_u16(buf, &offset_t); + if (val != EC_PT_SZ) + return -EINVAL; + memcpy(chip->null_ec_key_x, &buf->data[offset_t], val); + offset_t += val; + val = tpm_buf_read_u16(buf, &offset_t); + if (val != EC_PT_SZ) + return -EINVAL; + memcpy(chip->null_ec_key_y, &buf->data[offset_t], val); + offset_t += val; + + /* original length of the whole TPM2B */ + offset_r += len; + + /* should have exactly consumed the TPM2B public structure */ + if (offset_t != offset_r) + return -EINVAL; + if (offset_r > param_len) + return -EINVAL; + + /* creation data (skip) */ + len = tpm_buf_read_u16(buf, &offset_r); + offset_r += len; + if (offset_r > param_len) + return -EINVAL; + + /* creation digest (must be sha256) */ + len = tpm_buf_read_u16(buf, &offset_r); + offset_r += len; + if (len != SHA256_DIGEST_SIZE || offset_r > param_len) + return -EINVAL; + + /* TPMT_TK_CREATION follows */ + /* tag, must be TPM_ST_CREATION (0x8021) */ + val = tpm_buf_read_u16(buf, &offset_r); + if (val != TPM2_ST_CREATION || offset_r > param_len) + return -EINVAL; + + /* hierarchy */ + val = tpm_buf_read_u32(buf, &offset_r); + if (val != hierarchy || offset_r > param_len) + return -EINVAL; + + /* the ticket digest HMAC (might not be sha256) */ + len = tpm_buf_read_u16(buf, &offset_r); + offset_r += len; + if (offset_r > param_len) + return -EINVAL; + + /* + * finally we have the name, which is a sha256 digest plus a 2 + * byte algorithm type + */ + len = tpm_buf_read_u16(buf, &offset_r); + if (offset_r + len != param_len + 8) + return -EINVAL; + if (len != SHA256_DIGEST_SIZE + 2) + return -EINVAL; + + if (memcmp(chip->null_key_name, &buf->data[offset_r], + SHA256_DIGEST_SIZE + 2) != 0) { + dev_err(&chip->dev, "NULL Seed name comparison failed\n"); + return -EINVAL; + } + + return 0; +} + +/** + * tpm2_create_primary() - create a primary key using a fixed P-256 template + * + * @chip: the TPM chip to create under + * @hierarchy: The hierarchy handle to create under + * @handle: The returned volatile handle on success + * @name: The name of the returned key + * + * For platforms that might not have a persistent primary, this can be + * used to create one quickly on the fly (it uses Elliptic Curve not + * RSA, so even slow TPMs can create one fast). The template uses the + * TCG mandated H one for non-endorsement ECC primaries, i.e. P-256 + * elliptic curve (the only current one all TPM2s are required to + * have) a sha256 name hash and no policy. + * + * Return: + * * 0 - OK + * * -errno - A system error + * * TPM_RC - A TPM error + */ +static int tpm2_create_primary(struct tpm_chip *chip, u32 hierarchy, + u32 *handle, u8 *name) +{ + int rc; + struct tpm_buf buf; + struct tpm_buf template; + + rc = tpm_buf_init(&buf, TPM2_ST_SESSIONS, TPM2_CC_CREATE_PRIMARY); + if (rc) + return rc; + + rc = tpm_buf_init_sized(&template); + if (rc) { + tpm_buf_destroy(&buf); + return rc; + } + + /* + * create the template. Note: in order for userspace to + * verify the security of the system, it will have to create + * and certify this NULL primary, meaning all the template + * parameters will have to be identical, so conform exactly to + * the TCG TPM v2.0 Provisioning Guidance for the SRK ECC + * key H template (H has zero size unique points) + */ + + /* key type */ + tpm_buf_append_u16(&template, TPM_ALG_ECC); + + /* name algorithm */ + tpm_buf_append_u16(&template, TPM_ALG_SHA256); + + /* object properties */ + tpm_buf_append_u32(&template, TPM2_OA_TMPL); + + /* sauth policy (empty) */ + tpm_buf_append_u16(&template, 0); + + /* BEGIN parameters: key specific; for ECC*/ + + /* symmetric algorithm */ + tpm_buf_append_u16(&template, TPM_ALG_AES); + + /* bits for symmetric algorithm */ + tpm_buf_append_u16(&template, AES_KEY_BITS); + + /* algorithm mode (must be CFB) */ + tpm_buf_append_u16(&template, TPM_ALG_CFB); + + /* scheme (NULL means any scheme) */ + tpm_buf_append_u16(&template, TPM_ALG_NULL); + + /* ECC Curve ID */ + tpm_buf_append_u16(&template, TPM2_ECC_NIST_P256); + + /* KDF Scheme */ + tpm_buf_append_u16(&template, TPM_ALG_NULL); + + /* unique: key specific; for ECC it is two zero size points */ + tpm_buf_append_u16(&template, 0); + tpm_buf_append_u16(&template, 0); + + /* END parameters */ + + /* primary handle */ + tpm_buf_append_u32(&buf, hierarchy); + tpm_buf_append_empty_auth(&buf, TPM2_RS_PW); + + /* sensitive create size is 4 for two empty buffers */ + tpm_buf_append_u16(&buf, 4); + + /* sensitive create auth data (empty) */ + tpm_buf_append_u16(&buf, 0); + + /* sensitive create sensitive data (empty) */ + tpm_buf_append_u16(&buf, 0); + + /* the public template */ + tpm_buf_append(&buf, template.data, template.length); + tpm_buf_destroy(&template); + + /* outside info (empty) */ + tpm_buf_append_u16(&buf, 0); + + /* creation PCR (none) */ + tpm_buf_append_u32(&buf, 0); + + rc = tpm_transmit_cmd(chip, &buf, 0, + "attempting to create NULL primary"); + + if (rc == TPM2_RC_SUCCESS) + rc = tpm2_parse_create_primary(chip, &buf, handle, hierarchy, + name); + + tpm_buf_destroy(&buf); + + return rc; +} + +static int tpm2_create_null_primary(struct tpm_chip *chip) +{ + u32 null_key; + int rc; + + rc = tpm2_create_primary(chip, TPM2_RH_NULL, &null_key, + chip->null_key_name); + + if (rc == TPM2_RC_SUCCESS) { + unsigned int offset = 0; /* dummy offset for null key context */ + + rc = tpm2_save_context(chip, null_key, chip->null_key_context, + sizeof(chip->null_key_context), &offset); + tpm2_flush_context(chip, null_key); + } + + return rc; +} + +/** + * tpm2_sessions_init() - start of day initialization for the sessions code + * @chip: TPM chip + * + * Derive and context save the null primary and allocate memory in the + * struct tpm_chip for the authorizations. + */ +int tpm2_sessions_init(struct tpm_chip *chip) +{ + int rc; + + rc = tpm2_create_null_primary(chip); + if (rc) + dev_err(&chip->dev, "TPM: security failed (NULL seed derivation): %d\n", rc); + + chip->auth = kmalloc(sizeof(*chip->auth), GFP_KERNEL); + if (!chip->auth) + return -ENOMEM; + + return rc; +} diff --git a/drivers/char/tpm/tpm2-space.c b/drivers/char/tpm/tpm2-space.c index 363afdd4d1d3..4892d491da8d 100644 --- a/drivers/char/tpm/tpm2-space.c +++ b/drivers/char/tpm/tpm2-space.c @@ -68,8 +68,8 @@ void tpm2_del_space(struct tpm_chip *chip, struct tpm_space *space) kfree(space->session_buf); } -static int tpm2_load_context(struct tpm_chip *chip, u8 *buf, - unsigned int *offset, u32 *handle) +int tpm2_load_context(struct tpm_chip *chip, u8 *buf, + unsigned int *offset, u32 *handle) { struct tpm_buf tbuf; struct tpm2_context *ctx; @@ -105,6 +105,9 @@ static int tpm2_load_context(struct tpm_chip *chip, u8 *buf, *handle = 0; tpm_buf_destroy(&tbuf); return -ENOENT; + } else if (tpm2_rc_value(rc) == TPM2_RC_INTEGRITY) { + tpm_buf_destroy(&tbuf); + return -EINVAL; } else if (rc > 0) { dev_warn(&chip->dev, "%s: failed with a TPM error 0x%04X\n", __func__, rc); @@ -119,8 +122,8 @@ static int tpm2_load_context(struct tpm_chip *chip, u8 *buf, return 0; } -static int tpm2_save_context(struct tpm_chip *chip, u32 handle, u8 *buf, - unsigned int buf_size, unsigned int *offset) +int tpm2_save_context(struct tpm_chip *chip, u32 handle, u8 *buf, + unsigned int buf_size, unsigned int *offset) { struct tpm_buf tbuf; unsigned int body_size; diff --git a/drivers/char/tpm/tpm_infineon.c b/drivers/char/tpm/tpm_infineon.c index 9c924a1440a9..2d2ae37153ba 100644 --- a/drivers/char/tpm/tpm_infineon.c +++ b/drivers/char/tpm/tpm_infineon.c @@ -51,34 +51,40 @@ static struct tpm_inf_dev tpm_dev; static inline void tpm_data_out(unsigned char data, unsigned char offset) { +#ifdef CONFIG_HAS_IOPORT if (tpm_dev.iotype == TPM_INF_IO_PORT) outb(data, tpm_dev.data_regs + offset); else +#endif writeb(data, tpm_dev.mem_base + tpm_dev.data_regs + offset); } static inline unsigned char tpm_data_in(unsigned char offset) { +#ifdef CONFIG_HAS_IOPORT if (tpm_dev.iotype == TPM_INF_IO_PORT) return inb(tpm_dev.data_regs + offset); - else - return readb(tpm_dev.mem_base + tpm_dev.data_regs + offset); +#endif + return readb(tpm_dev.mem_base + tpm_dev.data_regs + offset); } static inline void tpm_config_out(unsigned char data, unsigned char offset) { +#ifdef CONFIG_HAS_IOPORT if (tpm_dev.iotype == TPM_INF_IO_PORT) outb(data, tpm_dev.config_port + offset); else +#endif writeb(data, tpm_dev.mem_base + tpm_dev.index_off + offset); } static inline unsigned char tpm_config_in(unsigned char offset) { +#ifdef CONFIG_HAS_IOPORT if (tpm_dev.iotype == TPM_INF_IO_PORT) return inb(tpm_dev.config_port + offset); - else - return readb(tpm_dev.mem_base + tpm_dev.index_off + offset); +#endif + return readb(tpm_dev.mem_base + tpm_dev.index_off + offset); } /* TPM header definitions */ diff --git a/drivers/char/tpm/tpm_tis_core.c b/drivers/char/tpm/tpm_tis_core.c index 714070ebb6e7..176cd8dbf1db 100644 --- a/drivers/char/tpm/tpm_tis_core.c +++ b/drivers/char/tpm/tpm_tis_core.c @@ -1057,11 +1057,6 @@ static void tpm_tis_clkrun_enable(struct tpm_chip *chip, bool value) clkrun_val &= ~LPC_CLKRUN_EN; iowrite32(clkrun_val, data->ilb_base_addr + LPC_CNTRL_OFFSET); - /* - * Write any random value on port 0x80 which is on LPC, to make - * sure LPC clock is running before sending any TPM command. - */ - outb(0xCC, 0x80); } else { data->clkrun_enabled--; if (data->clkrun_enabled) @@ -1072,13 +1067,15 @@ static void tpm_tis_clkrun_enable(struct tpm_chip *chip, bool value) /* Enable LPC CLKRUN# */ clkrun_val |= LPC_CLKRUN_EN; iowrite32(clkrun_val, data->ilb_base_addr + LPC_CNTRL_OFFSET); - - /* - * Write any random value on port 0x80 which is on LPC, to make - * sure LPC clock is running before sending any TPM command. - */ - outb(0xCC, 0x80); } + +#ifdef CONFIG_HAS_IOPORT + /* + * Write any random value on port 0x80 which is on LPC, to make + * sure LPC clock is running before sending any TPM command. + */ + outb(0xCC, 0x80); +#endif } static const struct tpm_class_ops tpm_tis = { diff --git a/drivers/clk/Kconfig b/drivers/clk/Kconfig index 50af5fc7f570..7517a0dfd15c 100644 --- a/drivers/clk/Kconfig +++ b/drivers/clk/Kconfig @@ -451,8 +451,8 @@ config COMMON_CLK_FIXED_MMIO config COMMON_CLK_K210 bool "Clock driver for the Canaan Kendryte K210 SoC" - depends on OF && RISCV && SOC_CANAAN - default SOC_CANAAN + depends on OF && RISCV && SOC_CANAAN_K210 + default SOC_CANAAN_K210 help Support for the Canaan Kendryte K210 RISC-V SoC clocks. diff --git a/drivers/clocksource/arm_arch_timer.c b/drivers/clocksource/arm_arch_timer.c index 8d4a52056684..5bb43cc1a8df 100644 --- a/drivers/clocksource/arm_arch_timer.c +++ b/drivers/clocksource/arm_arch_timer.c @@ -331,7 +331,7 @@ static u64 notrace hisi_161010101_read_cntvct_el0(void) return __hisi_161010101_read_reg(cntvct_el0); } -static struct ate_acpi_oem_info hisi_161010101_oem_info[] = { +static const struct ate_acpi_oem_info hisi_161010101_oem_info[] = { /* * Note that trailing spaces are required to properly match * the OEM table information. diff --git a/drivers/clocksource/renesas-ostm.c b/drivers/clocksource/renesas-ostm.c index 8da972dc1713..3fcbd02b2483 100644 --- a/drivers/clocksource/renesas-ostm.c +++ b/drivers/clocksource/renesas-ostm.c @@ -210,6 +210,7 @@ static int __init ostm_init(struct device_node *np) pr_info("%pOF: used for clock events\n", np); } + of_node_set_flag(np, OF_POPULATED); return 0; err_cleanup: @@ -224,7 +225,7 @@ err_free: TIMER_OF_DECLARE(ostm, "renesas,ostm", ostm_init); -#ifdef CONFIG_ARCH_RZG2L +#if defined(CONFIG_ARCH_RZG2L) || defined(CONFIG_ARCH_R9A09G057) static int __init ostm_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; diff --git a/drivers/clocksource/timer-ti-dm.c b/drivers/clocksource/timer-ti-dm.c index 56acf2617262..b7a34b1a975e 100644 --- a/drivers/clocksource/timer-ti-dm.c +++ b/drivers/clocksource/timer-ti-dm.c @@ -129,7 +129,6 @@ struct dmtimer { void __iomem *func_base; /* function register base */ atomic_t enabled; - unsigned long rate; unsigned reserved:1; unsigned posted:1; unsigned omap1:1; diff --git a/drivers/cpufreq/cpufreq.c b/drivers/cpufreq/cpufreq.c index 66e10a19d76a..1de8bd105934 100644 --- a/drivers/cpufreq/cpufreq.c +++ b/drivers/cpufreq/cpufreq.c @@ -2582,6 +2582,40 @@ int cpufreq_get_policy(struct cpufreq_policy *policy, unsigned int cpu) } EXPORT_SYMBOL(cpufreq_get_policy); +DEFINE_PER_CPU(unsigned long, cpufreq_pressure); + +/** + * cpufreq_update_pressure() - Update cpufreq pressure for CPUs + * @policy: cpufreq policy of the CPUs. + * + * Update the value of cpufreq pressure for all @cpus in the policy. + */ +static void cpufreq_update_pressure(struct cpufreq_policy *policy) +{ + unsigned long max_capacity, capped_freq, pressure; + u32 max_freq; + int cpu; + + cpu = cpumask_first(policy->related_cpus); + max_freq = arch_scale_freq_ref(cpu); + capped_freq = policy->max; + + /* + * Handle properly the boost frequencies, which should simply clean + * the cpufreq pressure value. + */ + if (max_freq <= capped_freq) { + pressure = 0; + } else { + max_capacity = arch_scale_cpu_capacity(cpu); + pressure = max_capacity - + mult_frac(max_capacity, capped_freq, max_freq); + } + + for_each_cpu(cpu, policy->related_cpus) + WRITE_ONCE(per_cpu(cpufreq_pressure, cpu), pressure); +} + /** * cpufreq_set_policy - Modify cpufreq policy parameters. * @policy: Policy object to modify. @@ -2637,6 +2671,8 @@ static int cpufreq_set_policy(struct cpufreq_policy *policy, policy->max = __resolve_freq(policy, policy->max, CPUFREQ_RELATION_H); trace_cpu_frequency_limits(policy); + cpufreq_update_pressure(policy); + policy->cached_target_freq = UINT_MAX; pr_debug("new min and max freqs are %u - %u kHz\n", diff --git a/drivers/cpufreq/qcom-cpufreq-hw.c b/drivers/cpufreq/qcom-cpufreq-hw.c index 70b0f21968a0..ec8df5496a0c 100644 --- a/drivers/cpufreq/qcom-cpufreq-hw.c +++ b/drivers/cpufreq/qcom-cpufreq-hw.c @@ -347,8 +347,8 @@ static void qcom_lmh_dcvs_notify(struct qcom_cpufreq_data *data) throttled_freq = freq_hz / HZ_PER_KHZ; - /* Update thermal pressure (the boost frequencies are accepted) */ - arch_update_thermal_pressure(policy->related_cpus, throttled_freq); + /* Update HW pressure (the boost frequencies are accepted) */ + arch_update_hw_pressure(policy->related_cpus, throttled_freq); /* * In the unlikely case policy is unregistered do not enable diff --git a/drivers/cpuidle/coupled.c b/drivers/cpuidle/coupled.c index 9acde71558d5..bb8761c8a42e 100644 --- a/drivers/cpuidle/coupled.c +++ b/drivers/cpuidle/coupled.c @@ -439,13 +439,8 @@ static int cpuidle_coupled_clear_pokes(int cpu) static bool cpuidle_coupled_any_pokes_pending(struct cpuidle_coupled *coupled) { - cpumask_t cpus; - int ret; - - cpumask_and(&cpus, cpu_online_mask, &coupled->coupled_cpus); - ret = cpumask_and(&cpus, &cpuidle_coupled_poke_pending, &cpus); - - return ret; + return cpumask_first_and_and(cpu_online_mask, &coupled->coupled_cpus, + &cpuidle_coupled_poke_pending) < nr_cpu_ids; } /** @@ -626,9 +621,7 @@ out: static void cpuidle_coupled_update_online_cpus(struct cpuidle_coupled *coupled) { - cpumask_t cpus; - cpumask_and(&cpus, cpu_online_mask, &coupled->coupled_cpus); - coupled->online_count = cpumask_weight(&cpus); + coupled->online_count = cpumask_weight_and(cpu_online_mask, &coupled->coupled_cpus); } /** diff --git a/drivers/crypto/Kconfig b/drivers/crypto/Kconfig index 3d02702456a5..94f23c6fc93b 100644 --- a/drivers/crypto/Kconfig +++ b/drivers/crypto/Kconfig @@ -67,6 +67,7 @@ config CRYPTO_DEV_GEODE config ZCRYPT tristate "Support for s390 cryptographic adapters" depends on S390 + depends on AP select HW_RANDOM help Select this option if you want to enable support for @@ -74,23 +75,6 @@ config ZCRYPT to 8 in Coprocessor (CEXxC), EP11 Coprocessor (CEXxP) or Accelerator (CEXxA) mode. -config ZCRYPT_DEBUG - bool "Enable debug features for s390 cryptographic adapters" - default n - depends on DEBUG_KERNEL - depends on ZCRYPT - help - Say 'Y' here to enable some additional debug features on the - s390 cryptographic adapters driver. - - There will be some more sysfs attributes displayed for ap cards - and queues and some flags on crypto requests are interpreted as - debugging messages to force error injection. - - Do not enable on production level kernel build. - - If unsure, say N. - config PKEY tristate "Kernel API for protected key handling" depends on S390 @@ -660,6 +644,14 @@ config CRYPTO_DEV_ROCKCHIP_DEBUG This will create /sys/kernel/debug/rk3288_crypto/stats for displaying the number of requests per algorithm and other internal stats. +config CRYPTO_DEV_TEGRA + tristate "Enable Tegra Security Engine" + depends on TEGRA_HOST1X + select CRYPTO_ENGINE + + help + Select this to enable Tegra Security Engine which accelerates various + AES encryption/decryption and HASH algorithms. config CRYPTO_DEV_ZYNQMP_AES tristate "Support for Xilinx ZynqMP AES hw accelerator" diff --git a/drivers/crypto/Makefile b/drivers/crypto/Makefile index 95331bc6456b..ad4ccef67d12 100644 --- a/drivers/crypto/Makefile +++ b/drivers/crypto/Makefile @@ -41,6 +41,7 @@ obj-$(CONFIG_CRYPTO_DEV_SAHARA) += sahara.o obj-$(CONFIG_CRYPTO_DEV_SL3516) += gemini/ obj-y += stm32/ obj-$(CONFIG_CRYPTO_DEV_TALITOS) += talitos.o +obj-$(CONFIG_CRYPTO_DEV_TEGRA) += tegra/ obj-$(CONFIG_CRYPTO_DEV_VIRTIO) += virtio/ #obj-$(CONFIG_CRYPTO_DEV_VMX) += vmx/ obj-$(CONFIG_CRYPTO_DEV_BCM_SPU) += bcm/ diff --git a/drivers/crypto/atmel-i2c.c b/drivers/crypto/atmel-i2c.c index 83a9093eff25..a895e4289efa 100644 --- a/drivers/crypto/atmel-i2c.c +++ b/drivers/crypto/atmel-i2c.c @@ -51,7 +51,7 @@ static void atmel_i2c_checksum(struct atmel_i2c_cmd *cmd) *__crc16 = cpu_to_le16(bitrev16(crc16(0, data, len))); } -void atmel_i2c_init_read_cmd(struct atmel_i2c_cmd *cmd) +void atmel_i2c_init_read_config_cmd(struct atmel_i2c_cmd *cmd) { cmd->word_addr = COMMAND; cmd->opcode = OPCODE_READ; @@ -68,7 +68,31 @@ void atmel_i2c_init_read_cmd(struct atmel_i2c_cmd *cmd) cmd->msecs = MAX_EXEC_TIME_READ; cmd->rxsize = READ_RSP_SIZE; } -EXPORT_SYMBOL(atmel_i2c_init_read_cmd); +EXPORT_SYMBOL(atmel_i2c_init_read_config_cmd); + +int atmel_i2c_init_read_otp_cmd(struct atmel_i2c_cmd *cmd, u16 addr) +{ + if (addr < 0 || addr > OTP_ZONE_SIZE) + return -1; + + cmd->word_addr = COMMAND; + cmd->opcode = OPCODE_READ; + /* + * Read the word from OTP zone that may contain e.g. serial + * numbers or similar if persistently pre-initialized and locked + */ + cmd->param1 = OTP_ZONE; + cmd->param2 = cpu_to_le16(addr); + cmd->count = READ_COUNT; + + atmel_i2c_checksum(cmd); + + cmd->msecs = MAX_EXEC_TIME_READ; + cmd->rxsize = READ_RSP_SIZE; + + return 0; +} +EXPORT_SYMBOL(atmel_i2c_init_read_otp_cmd); void atmel_i2c_init_random_cmd(struct atmel_i2c_cmd *cmd) { @@ -301,7 +325,7 @@ static int device_sanity_check(struct i2c_client *client) if (!cmd) return -ENOMEM; - atmel_i2c_init_read_cmd(cmd); + atmel_i2c_init_read_config_cmd(cmd); ret = atmel_i2c_send_receive(client, cmd); if (ret) diff --git a/drivers/crypto/atmel-i2c.h b/drivers/crypto/atmel-i2c.h index c0bd429ee2c7..72f04c15682f 100644 --- a/drivers/crypto/atmel-i2c.h +++ b/drivers/crypto/atmel-i2c.h @@ -64,6 +64,10 @@ struct atmel_i2c_cmd { /* Definitions for eeprom organization */ #define CONFIGURATION_ZONE 0 +#define OTP_ZONE 1 + +/* Definitions for eeprom zone sizes */ +#define OTP_ZONE_SIZE 64 /* Definitions for Indexes common to all commands */ #define RSP_DATA_IDX 1 /* buffer index of data in response */ @@ -124,6 +128,7 @@ struct atmel_ecc_driver_data { * @wake_token : wake token array of zeros * @wake_token_sz : size in bytes of the wake_token * @tfm_count : number of active crypto transformations on i2c client + * @hwrng : hold the hardware generated rng * * Reads and writes from/to the i2c client are sequential. The first byte * transmitted to the device is treated as the byte size. Any attempt to send @@ -177,7 +182,8 @@ void atmel_i2c_flush_queue(void); int atmel_i2c_send_receive(struct i2c_client *client, struct atmel_i2c_cmd *cmd); -void atmel_i2c_init_read_cmd(struct atmel_i2c_cmd *cmd); +void atmel_i2c_init_read_config_cmd(struct atmel_i2c_cmd *cmd); +int atmel_i2c_init_read_otp_cmd(struct atmel_i2c_cmd *cmd, u16 addr); void atmel_i2c_init_random_cmd(struct atmel_i2c_cmd *cmd); void atmel_i2c_init_genkey_cmd(struct atmel_i2c_cmd *cmd, u16 keyid); int atmel_i2c_init_ecdh_cmd(struct atmel_i2c_cmd *cmd, diff --git a/drivers/crypto/atmel-sha204a.c b/drivers/crypto/atmel-sha204a.c index c77f482d2a97..24ffdf505023 100644 --- a/drivers/crypto/atmel-sha204a.c +++ b/drivers/crypto/atmel-sha204a.c @@ -91,6 +91,62 @@ static int atmel_sha204a_rng_read(struct hwrng *rng, void *data, size_t max, return max; } +static int atmel_sha204a_otp_read(struct i2c_client *client, u16 addr, u8 *otp) +{ + struct atmel_i2c_cmd cmd; + int ret = -1; + + if (atmel_i2c_init_read_otp_cmd(&cmd, addr) < 0) { + dev_err(&client->dev, "failed, invalid otp address %04X\n", + addr); + return ret; + } + + ret = atmel_i2c_send_receive(client, &cmd); + + if (cmd.data[0] == 0xff) { + dev_err(&client->dev, "failed, device not ready\n"); + return -ret; + } + + memcpy(otp, cmd.data+1, 4); + + return ret; +} + +static ssize_t otp_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + u16 addr; + u8 otp[OTP_ZONE_SIZE]; + char *str = buf; + struct i2c_client *client = to_i2c_client(dev); + int i; + + for (addr = 0; addr < OTP_ZONE_SIZE/4; addr++) { + if (atmel_sha204a_otp_read(client, addr, otp + addr * 4) < 0) { + dev_err(dev, "failed to read otp zone\n"); + break; + } + } + + for (i = 0; i < addr*2; i++) + str += sprintf(str, "%02X", otp[i]); + str += sprintf(str, "\n"); + return str - buf; +} +static DEVICE_ATTR_RO(otp); + +static struct attribute *atmel_sha204a_attrs[] = { + &dev_attr_otp.attr, + NULL +}; + +static const struct attribute_group atmel_sha204a_groups = { + .name = "atsha204a", + .attrs = atmel_sha204a_attrs, +}; + static int atmel_sha204a_probe(struct i2c_client *client) { struct atmel_i2c_client_priv *i2c_priv; @@ -111,6 +167,16 @@ static int atmel_sha204a_probe(struct i2c_client *client) if (ret) dev_warn(&client->dev, "failed to register RNG (%d)\n", ret); + /* otp read out */ + if (!i2c_check_functionality(client->adapter, I2C_FUNC_I2C)) + return -ENODEV; + + ret = sysfs_create_group(&client->dev.kobj, &atmel_sha204a_groups); + if (ret) { + dev_err(&client->dev, "failed to register sysfs entry\n"); + return ret; + } + return ret; } @@ -123,6 +189,8 @@ static void atmel_sha204a_remove(struct i2c_client *client) return; } + sysfs_remove_group(&client->dev.kobj, &atmel_sha204a_groups); + kfree((void *)i2c_priv->hwrng.priv); } diff --git a/drivers/crypto/bcm/spu2.c b/drivers/crypto/bcm/spu2.c index 07989bb8c220..3fdc64b5a65e 100644 --- a/drivers/crypto/bcm/spu2.c +++ b/drivers/crypto/bcm/spu2.c @@ -495,7 +495,7 @@ static void spu2_dump_omd(u8 *omd, u16 hash_key_len, u16 ciph_key_len, if (hash_iv_len) { packet_log(" Hash IV Length %u bytes\n", hash_iv_len); packet_dump(" hash IV: ", ptr, hash_iv_len); - ptr += ciph_key_len; + ptr += hash_iv_len; } if (ciph_iv_len) { diff --git a/drivers/crypto/caam/ctrl.c b/drivers/crypto/caam/ctrl.c index bdf367f3f679..bd418dea586d 100644 --- a/drivers/crypto/caam/ctrl.c +++ b/drivers/crypto/caam/ctrl.c @@ -512,6 +512,7 @@ static const struct of_device_id caam_match[] = { MODULE_DEVICE_TABLE(of, caam_match); struct caam_imx_data { + bool page0_access; const struct clk_bulk_data *clks; int num_clks; }; @@ -524,6 +525,7 @@ static const struct clk_bulk_data caam_imx6_clks[] = { }; static const struct caam_imx_data caam_imx6_data = { + .page0_access = true, .clks = caam_imx6_clks, .num_clks = ARRAY_SIZE(caam_imx6_clks), }; @@ -534,6 +536,7 @@ static const struct clk_bulk_data caam_imx7_clks[] = { }; static const struct caam_imx_data caam_imx7_data = { + .page0_access = true, .clks = caam_imx7_clks, .num_clks = ARRAY_SIZE(caam_imx7_clks), }; @@ -545,6 +548,7 @@ static const struct clk_bulk_data caam_imx6ul_clks[] = { }; static const struct caam_imx_data caam_imx6ul_data = { + .page0_access = true, .clks = caam_imx6ul_clks, .num_clks = ARRAY_SIZE(caam_imx6ul_clks), }; @@ -554,15 +558,19 @@ static const struct clk_bulk_data caam_vf610_clks[] = { }; static const struct caam_imx_data caam_vf610_data = { + .page0_access = true, .clks = caam_vf610_clks, .num_clks = ARRAY_SIZE(caam_vf610_clks), }; +static const struct caam_imx_data caam_imx8ulp_data; + static const struct soc_device_attribute caam_imx_soc_table[] = { { .soc_id = "i.MX6UL", .data = &caam_imx6ul_data }, { .soc_id = "i.MX6*", .data = &caam_imx6_data }, { .soc_id = "i.MX7*", .data = &caam_imx7_data }, { .soc_id = "i.MX8M*", .data = &caam_imx7_data }, + { .soc_id = "i.MX8ULP", .data = &caam_imx8ulp_data }, { .soc_id = "VF*", .data = &caam_vf610_data }, { .family = "Freescale i.MX" }, { /* sentinel */ } @@ -860,6 +868,7 @@ static int caam_probe(struct platform_device *pdev) int pg_size; int BLOCK_OFFSET = 0; bool reg_access = true; + const struct caam_imx_data *imx_soc_data; ctrlpriv = devm_kzalloc(&pdev->dev, sizeof(*ctrlpriv), GFP_KERNEL); if (!ctrlpriv) @@ -894,12 +903,20 @@ static int caam_probe(struct platform_device *pdev) return -EINVAL; } + imx_soc_data = imx_soc_match->data; + reg_access = reg_access && imx_soc_data->page0_access; + /* + * CAAM clocks cannot be controlled from kernel. + */ + if (!imx_soc_data->num_clks) + goto iomap_ctrl; + ret = init_clocks(dev, imx_soc_match->data); if (ret) return ret; } - +iomap_ctrl: /* Get configuration properties from device tree */ /* First, get register page */ ctrl = devm_of_iomap(dev, nprop, 0, NULL); diff --git a/drivers/crypto/ccp/sp-platform.c b/drivers/crypto/ccp/sp-platform.c index 473301237760..ff6ceb4feee0 100644 --- a/drivers/crypto/ccp/sp-platform.c +++ b/drivers/crypto/ccp/sp-platform.c @@ -39,44 +39,38 @@ static const struct sp_dev_vdata dev_vdata[] = { }, }; -#ifdef CONFIG_ACPI static const struct acpi_device_id sp_acpi_match[] = { { "AMDI0C00", (kernel_ulong_t)&dev_vdata[0] }, { }, }; MODULE_DEVICE_TABLE(acpi, sp_acpi_match); -#endif -#ifdef CONFIG_OF static const struct of_device_id sp_of_match[] = { { .compatible = "amd,ccp-seattle-v1a", .data = (const void *)&dev_vdata[0] }, { }, }; MODULE_DEVICE_TABLE(of, sp_of_match); -#endif static struct sp_dev_vdata *sp_get_of_version(struct platform_device *pdev) { -#ifdef CONFIG_OF const struct of_device_id *match; match = of_match_node(sp_of_match, pdev->dev.of_node); if (match && match->data) return (struct sp_dev_vdata *)match->data; -#endif + return NULL; } static struct sp_dev_vdata *sp_get_acpi_version(struct platform_device *pdev) { -#ifdef CONFIG_ACPI const struct acpi_device_id *match; match = acpi_match_device(sp_acpi_match, &pdev->dev); if (match && match->driver_data) return (struct sp_dev_vdata *)match->driver_data; -#endif + return NULL; } @@ -212,12 +206,8 @@ static int sp_platform_resume(struct platform_device *pdev) static struct platform_driver sp_platform_driver = { .driver = { .name = "ccp", -#ifdef CONFIG_ACPI .acpi_match_table = sp_acpi_match, -#endif -#ifdef CONFIG_OF .of_match_table = sp_of_match, -#endif }, .probe = sp_platform_probe, .remove_new = sp_platform_remove, diff --git a/drivers/crypto/hisilicon/debugfs.c b/drivers/crypto/hisilicon/debugfs.c index cd67fa348ca7..1b9b7bccdeff 100644 --- a/drivers/crypto/hisilicon/debugfs.c +++ b/drivers/crypto/hisilicon/debugfs.c @@ -13,6 +13,7 @@ #define QM_DFX_COMMON_LEN 0xC3 #define QM_DFX_REGS_LEN 4UL #define QM_DBG_TMP_BUF_LEN 22 +#define QM_XQC_ADDR_MASK GENMASK(31, 0) #define CURRENT_FUN_MASK GENMASK(5, 0) #define CURRENT_Q_MASK GENMASK(31, 16) #define QM_SQE_ADDR_MASK GENMASK(7, 0) @@ -167,7 +168,6 @@ static void dump_show(struct hisi_qm *qm, void *info, static int qm_sqc_dump(struct hisi_qm *qm, char *s, char *name) { struct device *dev = &qm->pdev->dev; - struct qm_sqc *sqc_curr; struct qm_sqc sqc; u32 qp_id; int ret; @@ -183,6 +183,8 @@ static int qm_sqc_dump(struct hisi_qm *qm, char *s, char *name) ret = qm_set_and_get_xqc(qm, QM_MB_CMD_SQC, &sqc, qp_id, 1); if (!ret) { + sqc.base_h = cpu_to_le32(QM_XQC_ADDR_MASK); + sqc.base_l = cpu_to_le32(QM_XQC_ADDR_MASK); dump_show(qm, &sqc, sizeof(struct qm_sqc), name); return 0; @@ -190,9 +192,10 @@ static int qm_sqc_dump(struct hisi_qm *qm, char *s, char *name) down_read(&qm->qps_lock); if (qm->sqc) { - sqc_curr = qm->sqc + qp_id; - - dump_show(qm, sqc_curr, sizeof(*sqc_curr), "SOFT SQC"); + memcpy(&sqc, qm->sqc + qp_id * sizeof(struct qm_sqc), sizeof(struct qm_sqc)); + sqc.base_h = cpu_to_le32(QM_XQC_ADDR_MASK); + sqc.base_l = cpu_to_le32(QM_XQC_ADDR_MASK); + dump_show(qm, &sqc, sizeof(struct qm_sqc), "SOFT SQC"); } up_read(&qm->qps_lock); @@ -202,7 +205,6 @@ static int qm_sqc_dump(struct hisi_qm *qm, char *s, char *name) static int qm_cqc_dump(struct hisi_qm *qm, char *s, char *name) { struct device *dev = &qm->pdev->dev; - struct qm_cqc *cqc_curr; struct qm_cqc cqc; u32 qp_id; int ret; @@ -218,6 +220,8 @@ static int qm_cqc_dump(struct hisi_qm *qm, char *s, char *name) ret = qm_set_and_get_xqc(qm, QM_MB_CMD_CQC, &cqc, qp_id, 1); if (!ret) { + cqc.base_h = cpu_to_le32(QM_XQC_ADDR_MASK); + cqc.base_l = cpu_to_le32(QM_XQC_ADDR_MASK); dump_show(qm, &cqc, sizeof(struct qm_cqc), name); return 0; @@ -225,9 +229,10 @@ static int qm_cqc_dump(struct hisi_qm *qm, char *s, char *name) down_read(&qm->qps_lock); if (qm->cqc) { - cqc_curr = qm->cqc + qp_id; - - dump_show(qm, cqc_curr, sizeof(*cqc_curr), "SOFT CQC"); + memcpy(&cqc, qm->cqc + qp_id * sizeof(struct qm_cqc), sizeof(struct qm_cqc)); + cqc.base_h = cpu_to_le32(QM_XQC_ADDR_MASK); + cqc.base_l = cpu_to_le32(QM_XQC_ADDR_MASK); + dump_show(qm, &cqc, sizeof(struct qm_cqc), "SOFT CQC"); } up_read(&qm->qps_lock); @@ -263,6 +268,10 @@ static int qm_eqc_aeqc_dump(struct hisi_qm *qm, char *s, char *name) if (ret) return ret; + aeqc.base_h = cpu_to_le32(QM_XQC_ADDR_MASK); + aeqc.base_l = cpu_to_le32(QM_XQC_ADDR_MASK); + eqc.base_h = cpu_to_le32(QM_XQC_ADDR_MASK); + eqc.base_l = cpu_to_le32(QM_XQC_ADDR_MASK); dump_show(qm, xeqc, size, name); return ret; @@ -310,27 +319,26 @@ static int q_dump_param_parse(struct hisi_qm *qm, char *s, static int qm_sq_dump(struct hisi_qm *qm, char *s, char *name) { - u16 sq_depth = qm->qp_array->cq_depth; - void *sqe, *sqe_curr; + u16 sq_depth = qm->qp_array->sq_depth; struct hisi_qp *qp; u32 qp_id, sqe_id; + void *sqe; int ret; ret = q_dump_param_parse(qm, s, &sqe_id, &qp_id, sq_depth); if (ret) return ret; - sqe = kzalloc(qm->sqe_size * sq_depth, GFP_KERNEL); + sqe = kzalloc(qm->sqe_size, GFP_KERNEL); if (!sqe) return -ENOMEM; qp = &qm->qp_array[qp_id]; - memcpy(sqe, qp->sqe, qm->sqe_size * sq_depth); - sqe_curr = sqe + (u32)(sqe_id * qm->sqe_size); - memset(sqe_curr + qm->debug.sqe_mask_offset, QM_SQE_ADDR_MASK, + memcpy(sqe, qp->sqe + sqe_id * qm->sqe_size, qm->sqe_size); + memset(sqe + qm->debug.sqe_mask_offset, QM_SQE_ADDR_MASK, qm->debug.sqe_mask_len); - dump_show(qm, sqe_curr, qm->sqe_size, name); + dump_show(qm, sqe, qm->sqe_size, name); kfree(sqe); @@ -809,8 +817,14 @@ static void dfx_regs_uninit(struct hisi_qm *qm, { int i; + if (!dregs) + return; + /* Setting the pointer is NULL to prevent double free */ for (i = 0; i < reg_len; i++) { + if (!dregs[i].regs) + continue; + kfree(dregs[i].regs); dregs[i].regs = NULL; } @@ -860,14 +874,21 @@ alloc_error: static int qm_diff_regs_init(struct hisi_qm *qm, struct dfx_diff_registers *dregs, u32 reg_len) { + int ret; + qm->debug.qm_diff_regs = dfx_regs_init(qm, qm_diff_regs, ARRAY_SIZE(qm_diff_regs)); - if (IS_ERR(qm->debug.qm_diff_regs)) - return PTR_ERR(qm->debug.qm_diff_regs); + if (IS_ERR(qm->debug.qm_diff_regs)) { + ret = PTR_ERR(qm->debug.qm_diff_regs); + qm->debug.qm_diff_regs = NULL; + return ret; + } qm->debug.acc_diff_regs = dfx_regs_init(qm, dregs, reg_len); if (IS_ERR(qm->debug.acc_diff_regs)) { dfx_regs_uninit(qm, qm->debug.qm_diff_regs, ARRAY_SIZE(qm_diff_regs)); - return PTR_ERR(qm->debug.acc_diff_regs); + ret = PTR_ERR(qm->debug.acc_diff_regs); + qm->debug.acc_diff_regs = NULL; + return ret; } return 0; @@ -908,7 +929,9 @@ static int qm_last_regs_init(struct hisi_qm *qm) static void qm_diff_regs_uninit(struct hisi_qm *qm, u32 reg_len) { dfx_regs_uninit(qm, qm->debug.acc_diff_regs, reg_len); + qm->debug.acc_diff_regs = NULL; dfx_regs_uninit(qm, qm->debug.qm_diff_regs, ARRAY_SIZE(qm_diff_regs)); + qm->debug.qm_diff_regs = NULL; } /** @@ -1075,12 +1098,12 @@ static void qm_create_debugfs_file(struct hisi_qm *qm, struct dentry *dir, { struct debugfs_file *file = qm->debug.files + index; - debugfs_create_file(qm_debug_file_name[index], 0600, dir, file, - &qm_debug_fops); - file->index = index; mutex_init(&file->lock); file->debug = &qm->debug; + + debugfs_create_file(qm_debug_file_name[index], 0600, dir, file, + &qm_debug_fops); } static int qm_debugfs_atomic64_set(void *data, u64 val) diff --git a/drivers/crypto/hisilicon/hpre/hpre_main.c b/drivers/crypto/hisilicon/hpre/hpre_main.c index d93aa6630a57..10aa4da93323 100644 --- a/drivers/crypto/hisilicon/hpre/hpre_main.c +++ b/drivers/crypto/hisilicon/hpre/hpre_main.c @@ -106,7 +106,7 @@ #define HPRE_SHAPER_TYPE_RATE 640 #define HPRE_VIA_MSI_DSM 1 #define HPRE_SQE_MASK_OFFSET 8 -#define HPRE_SQE_MASK_LEN 24 +#define HPRE_SQE_MASK_LEN 44 #define HPRE_CTX_Q_NUM_DEF 1 #define HPRE_DFX_BASE 0x301000 @@ -1074,41 +1074,40 @@ static int hpre_debugfs_init(struct hisi_qm *qm) struct device *dev = &qm->pdev->dev; int ret; - qm->debug.debug_root = debugfs_create_dir(dev_name(dev), - hpre_debugfs_root); - - qm->debug.sqe_mask_offset = HPRE_SQE_MASK_OFFSET; - qm->debug.sqe_mask_len = HPRE_SQE_MASK_LEN; ret = hisi_qm_regs_debugfs_init(qm, hpre_diff_regs, ARRAY_SIZE(hpre_diff_regs)); if (ret) { dev_warn(dev, "Failed to init HPRE diff regs!\n"); - goto debugfs_remove; + return ret; } + qm->debug.debug_root = debugfs_create_dir(dev_name(dev), + hpre_debugfs_root); + qm->debug.sqe_mask_offset = HPRE_SQE_MASK_OFFSET; + qm->debug.sqe_mask_len = HPRE_SQE_MASK_LEN; + hisi_qm_debug_init(qm); if (qm->pdev->device == PCI_DEVICE_ID_HUAWEI_HPRE_PF) { ret = hpre_ctrl_debug_init(qm); if (ret) - goto failed_to_create; + goto debugfs_remove; } hpre_dfx_debug_init(qm); return 0; -failed_to_create: - hisi_qm_regs_debugfs_uninit(qm, ARRAY_SIZE(hpre_diff_regs)); debugfs_remove: debugfs_remove_recursive(qm->debug.debug_root); + hisi_qm_regs_debugfs_uninit(qm, ARRAY_SIZE(hpre_diff_regs)); return ret; } static void hpre_debugfs_exit(struct hisi_qm *qm) { - hisi_qm_regs_debugfs_uninit(qm, ARRAY_SIZE(hpre_diff_regs)); - debugfs_remove_recursive(qm->debug.debug_root); + + hisi_qm_regs_debugfs_uninit(qm, ARRAY_SIZE(hpre_diff_regs)); } static int hpre_pre_store_cap_reg(struct hisi_qm *qm) diff --git a/drivers/crypto/hisilicon/qm.c b/drivers/crypto/hisilicon/qm.c index 92f0a1d9b4a6..3dac8d8e8568 100644 --- a/drivers/crypto/hisilicon/qm.c +++ b/drivers/crypto/hisilicon/qm.c @@ -645,6 +645,9 @@ int qm_set_and_get_xqc(struct hisi_qm *qm, u8 cmd, void *xqc, u32 qp_id, bool op tmp_xqc = qm->xqc_buf.aeqc; xqc_dma = qm->xqc_buf.aeqc_dma; break; + default: + dev_err(&qm->pdev->dev, "unknown mailbox cmd %u\n", cmd); + return -EINVAL; } /* Setting xqc will fail if master OOO is blocked. */ @@ -2893,12 +2896,9 @@ void hisi_qm_uninit(struct hisi_qm *qm) hisi_qm_set_state(qm, QM_NOT_READY); up_write(&qm->qps_lock); + qm_remove_uacce(qm); qm_irqs_unregister(qm); hisi_qm_pci_uninit(qm); - if (qm->use_sva) { - uacce_remove(qm->uacce); - qm->uacce = NULL; - } } EXPORT_SYMBOL_GPL(hisi_qm_uninit); diff --git a/drivers/crypto/hisilicon/sec2/sec_crypto.c b/drivers/crypto/hisilicon/sec2/sec_crypto.c index 93a972fcbf63..0558f98e221f 100644 --- a/drivers/crypto/hisilicon/sec2/sec_crypto.c +++ b/drivers/crypto/hisilicon/sec2/sec_crypto.c @@ -481,8 +481,10 @@ static void sec_alg_resource_free(struct sec_ctx *ctx, if (ctx->pbuf_supported) sec_free_pbuf_resource(dev, qp_ctx->res); - if (ctx->alg_type == SEC_AEAD) + if (ctx->alg_type == SEC_AEAD) { sec_free_mac_resource(dev, qp_ctx->res); + sec_free_aiv_resource(dev, qp_ctx->res); + } } static int sec_alloc_qp_ctx_resource(struct sec_ctx *ctx, struct sec_qp_ctx *qp_ctx) diff --git a/drivers/crypto/hisilicon/sec2/sec_main.c b/drivers/crypto/hisilicon/sec2/sec_main.c index c290d8937b19..75aad04ffe5e 100644 --- a/drivers/crypto/hisilicon/sec2/sec_main.c +++ b/drivers/crypto/hisilicon/sec2/sec_main.c @@ -99,8 +99,8 @@ #define SEC_DBGFS_VAL_MAX_LEN 20 #define SEC_SINGLE_PORT_MAX_TRANS 0x2060 -#define SEC_SQE_MASK_OFFSET 64 -#define SEC_SQE_MASK_LEN 48 +#define SEC_SQE_MASK_OFFSET 16 +#define SEC_SQE_MASK_LEN 108 #define SEC_SHAPER_TYPE_RATE 400 #define SEC_DFX_BASE 0x301000 @@ -152,7 +152,7 @@ static const struct hisi_qm_cap_info sec_basic_info[] = { {SEC_CORE_TYPE_NUM_CAP, 0x313c, 16, GENMASK(3, 0), 0x1, 0x1, 0x1}, {SEC_CORE_NUM_CAP, 0x313c, 8, GENMASK(7, 0), 0x4, 0x4, 0x4}, {SEC_CORES_PER_CLUSTER_NUM_CAP, 0x313c, 0, GENMASK(7, 0), 0x4, 0x4, 0x4}, - {SEC_CORE_ENABLE_BITMAP, 0x3140, 32, GENMASK(31, 0), 0x17F, 0x17F, 0xF}, + {SEC_CORE_ENABLE_BITMAP, 0x3140, 0, GENMASK(31, 0), 0x17F, 0x17F, 0xF}, {SEC_DRV_ALG_BITMAP_LOW, 0x3144, 0, GENMASK(31, 0), 0x18050CB, 0x18050CB, 0x18670CF}, {SEC_DRV_ALG_BITMAP_HIGH, 0x3148, 0, GENMASK(31, 0), 0x395C, 0x395C, 0x395C}, {SEC_DEV_ALG_BITMAP_LOW, 0x314c, 0, GENMASK(31, 0), 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF}, @@ -901,37 +901,36 @@ static int sec_debugfs_init(struct hisi_qm *qm) struct device *dev = &qm->pdev->dev; int ret; - qm->debug.debug_root = debugfs_create_dir(dev_name(dev), - sec_debugfs_root); - qm->debug.sqe_mask_offset = SEC_SQE_MASK_OFFSET; - qm->debug.sqe_mask_len = SEC_SQE_MASK_LEN; - ret = hisi_qm_regs_debugfs_init(qm, sec_diff_regs, ARRAY_SIZE(sec_diff_regs)); if (ret) { dev_warn(dev, "Failed to init SEC diff regs!\n"); - goto debugfs_remove; + return ret; } + qm->debug.debug_root = debugfs_create_dir(dev_name(dev), + sec_debugfs_root); + qm->debug.sqe_mask_offset = SEC_SQE_MASK_OFFSET; + qm->debug.sqe_mask_len = SEC_SQE_MASK_LEN; + hisi_qm_debug_init(qm); ret = sec_debug_init(qm); if (ret) - goto failed_to_create; + goto debugfs_remove; return 0; -failed_to_create: - hisi_qm_regs_debugfs_uninit(qm, ARRAY_SIZE(sec_diff_regs)); debugfs_remove: - debugfs_remove_recursive(sec_debugfs_root); + debugfs_remove_recursive(qm->debug.debug_root); + hisi_qm_regs_debugfs_uninit(qm, ARRAY_SIZE(sec_diff_regs)); return ret; } static void sec_debugfs_exit(struct hisi_qm *qm) { - hisi_qm_regs_debugfs_uninit(qm, ARRAY_SIZE(sec_diff_regs)); - debugfs_remove_recursive(qm->debug.debug_root); + + hisi_qm_regs_debugfs_uninit(qm, ARRAY_SIZE(sec_diff_regs)); } static int sec_show_last_regs_init(struct hisi_qm *qm) @@ -1324,7 +1323,8 @@ static struct pci_driver sec_pci_driver = { .probe = sec_probe, .remove = sec_remove, .err_handler = &sec_err_handler, - .sriov_configure = hisi_qm_sriov_configure, + .sriov_configure = IS_ENABLED(CONFIG_PCI_IOV) ? + hisi_qm_sriov_configure : NULL, .shutdown = hisi_qm_dev_shutdown, .driver.pm = &sec_pm_ops, }; diff --git a/drivers/crypto/hisilicon/sgl.c b/drivers/crypto/hisilicon/sgl.c index 0beca257c20b..568acd0aee3f 100644 --- a/drivers/crypto/hisilicon/sgl.c +++ b/drivers/crypto/hisilicon/sgl.c @@ -161,9 +161,6 @@ static struct hisi_acc_hw_sgl *acc_get_sgl(struct hisi_acc_sgl_pool *pool, struct mem_block *block; u32 block_index, offset; - if (!pool || !hw_sgl_dma || index >= pool->count) - return ERR_PTR(-EINVAL); - block = pool->mem_block; block_index = index / pool->sgl_num_per_block; offset = index % pool->sgl_num_per_block; @@ -230,7 +227,7 @@ hisi_acc_sg_buf_map_to_hw_sgl(struct device *dev, struct scatterlist *sg; int sg_n; - if (!dev || !sgl || !pool || !hw_sgl_dma) + if (!dev || !sgl || !pool || !hw_sgl_dma || index >= pool->count) return ERR_PTR(-EINVAL); sg_n = sg_nents(sgl); diff --git a/drivers/crypto/hisilicon/zip/zip_main.c b/drivers/crypto/hisilicon/zip/zip_main.c index c065fd867161..c94a7b20d07e 100644 --- a/drivers/crypto/hisilicon/zip/zip_main.c +++ b/drivers/crypto/hisilicon/zip/zip_main.c @@ -887,36 +887,34 @@ static int hisi_zip_ctrl_debug_init(struct hisi_qm *qm) static int hisi_zip_debugfs_init(struct hisi_qm *qm) { struct device *dev = &qm->pdev->dev; - struct dentry *dev_d; int ret; - dev_d = debugfs_create_dir(dev_name(dev), hzip_debugfs_root); - - qm->debug.sqe_mask_offset = HZIP_SQE_MASK_OFFSET; - qm->debug.sqe_mask_len = HZIP_SQE_MASK_LEN; - qm->debug.debug_root = dev_d; ret = hisi_qm_regs_debugfs_init(qm, hzip_diff_regs, ARRAY_SIZE(hzip_diff_regs)); if (ret) { dev_warn(dev, "Failed to init ZIP diff regs!\n"); - goto debugfs_remove; + return ret; } + qm->debug.sqe_mask_offset = HZIP_SQE_MASK_OFFSET; + qm->debug.sqe_mask_len = HZIP_SQE_MASK_LEN; + qm->debug.debug_root = debugfs_create_dir(dev_name(dev), + hzip_debugfs_root); + hisi_qm_debug_init(qm); if (qm->fun_type == QM_HW_PF) { ret = hisi_zip_ctrl_debug_init(qm); if (ret) - goto failed_to_create; + goto debugfs_remove; } hisi_zip_dfx_debug_init(qm); return 0; -failed_to_create: - hisi_qm_regs_debugfs_uninit(qm, ARRAY_SIZE(hzip_diff_regs)); debugfs_remove: - debugfs_remove_recursive(hzip_debugfs_root); + debugfs_remove_recursive(qm->debug.debug_root); + hisi_qm_regs_debugfs_uninit(qm, ARRAY_SIZE(hzip_diff_regs)); return ret; } @@ -940,10 +938,10 @@ static void hisi_zip_debug_regs_clear(struct hisi_qm *qm) static void hisi_zip_debugfs_exit(struct hisi_qm *qm) { - hisi_qm_regs_debugfs_uninit(qm, ARRAY_SIZE(hzip_diff_regs)); - debugfs_remove_recursive(qm->debug.debug_root); + hisi_qm_regs_debugfs_uninit(qm, ARRAY_SIZE(hzip_diff_regs)); + if (qm->fun_type == QM_HW_PF) { hisi_zip_debug_regs_clear(qm); qm->debug.curr_qm_qp_num = 0; diff --git a/drivers/crypto/intel/iaa/iaa_crypto.h b/drivers/crypto/intel/iaa/iaa_crypto.h index 2524091a5f70..56985e395263 100644 --- a/drivers/crypto/intel/iaa/iaa_crypto.h +++ b/drivers/crypto/intel/iaa/iaa_crypto.h @@ -49,10 +49,10 @@ struct iaa_wq { struct iaa_device *iaa_device; - u64 comp_calls; - u64 comp_bytes; - u64 decomp_calls; - u64 decomp_bytes; + atomic64_t comp_calls; + atomic64_t comp_bytes; + atomic64_t decomp_calls; + atomic64_t decomp_bytes; }; struct iaa_device_compression_mode { @@ -73,10 +73,10 @@ struct iaa_device { int n_wq; struct list_head wqs; - u64 comp_calls; - u64 comp_bytes; - u64 decomp_calls; - u64 decomp_bytes; + atomic64_t comp_calls; + atomic64_t comp_bytes; + atomic64_t decomp_calls; + atomic64_t decomp_bytes; }; struct wq_table_entry { diff --git a/drivers/crypto/intel/iaa/iaa_crypto_main.c b/drivers/crypto/intel/iaa/iaa_crypto_main.c index b2191ade9011..e810d286ee8c 100644 --- a/drivers/crypto/intel/iaa/iaa_crypto_main.c +++ b/drivers/crypto/intel/iaa/iaa_crypto_main.c @@ -347,18 +347,16 @@ int add_iaa_compression_mode(const char *name, goto free; if (ll_table) { - mode->ll_table = kzalloc(ll_table_size, GFP_KERNEL); + mode->ll_table = kmemdup(ll_table, ll_table_size, GFP_KERNEL); if (!mode->ll_table) goto free; - memcpy(mode->ll_table, ll_table, ll_table_size); mode->ll_table_size = ll_table_size; } if (d_table) { - mode->d_table = kzalloc(d_table_size, GFP_KERNEL); + mode->d_table = kmemdup(d_table, d_table_size, GFP_KERNEL); if (!mode->d_table) goto free; - memcpy(mode->d_table, d_table, d_table_size); mode->d_table_size = d_table_size; } @@ -922,7 +920,7 @@ static void rebalance_wq_table(void) for_each_node_with_cpus(node) { node_cpus = cpumask_of_node(node); - for (cpu = 0; cpu < nr_cpus_per_node; cpu++) { + for (cpu = 0; cpu < cpumask_weight(node_cpus); cpu++) { int node_cpu = cpumask_nth(cpu, node_cpus); if (WARN_ON(node_cpu >= nr_cpu_ids)) { @@ -1079,8 +1077,8 @@ static void iaa_desc_complete(struct idxd_desc *idxd_desc, update_total_comp_bytes_out(ctx->req->dlen); update_wq_comp_bytes(iaa_wq->wq, ctx->req->dlen); } else { - update_total_decomp_bytes_in(ctx->req->dlen); - update_wq_decomp_bytes(iaa_wq->wq, ctx->req->dlen); + update_total_decomp_bytes_in(ctx->req->slen); + update_wq_decomp_bytes(iaa_wq->wq, ctx->req->slen); } if (ctx->compress && compression_ctx->verify_compress) { @@ -1498,7 +1496,6 @@ static int iaa_comp_acompress(struct acomp_req *req) u32 compression_crc; struct idxd_wq *wq; struct device *dev; - u64 start_time_ns; int order = -1; compression_ctx = crypto_tfm_ctx(tfm); @@ -1572,10 +1569,8 @@ static int iaa_comp_acompress(struct acomp_req *req) " req->dlen %d, sg_dma_len(sg) %d\n", dst_addr, nr_sgs, req->dst, req->dlen, sg_dma_len(req->dst)); - start_time_ns = iaa_get_ts(); ret = iaa_compress(tfm, req, wq, src_addr, req->slen, dst_addr, &req->dlen, &compression_crc, disable_async); - update_max_comp_delay_ns(start_time_ns); if (ret == -EINPROGRESS) return ret; @@ -1622,7 +1617,6 @@ static int iaa_comp_adecompress_alloc_dest(struct acomp_req *req) struct iaa_wq *iaa_wq; struct device *dev; struct idxd_wq *wq; - u64 start_time_ns; int order = -1; cpu = get_cpu(); @@ -1679,10 +1673,8 @@ alloc_dest: dev_dbg(dev, "dma_map_sg, dst_addr %llx, nr_sgs %d, req->dst %p," " req->dlen %d, sg_dma_len(sg) %d\n", dst_addr, nr_sgs, req->dst, req->dlen, sg_dma_len(req->dst)); - start_time_ns = iaa_get_ts(); ret = iaa_decompress(tfm, req, wq, src_addr, req->slen, dst_addr, &req->dlen, true); - update_max_decomp_delay_ns(start_time_ns); if (ret == -EOVERFLOW) { dma_unmap_sg(dev, req->dst, sg_nents(req->dst), DMA_FROM_DEVICE); req->dlen *= 2; @@ -1713,7 +1705,6 @@ static int iaa_comp_adecompress(struct acomp_req *req) int nr_sgs, cpu, ret = 0; struct iaa_wq *iaa_wq; struct device *dev; - u64 start_time_ns; struct idxd_wq *wq; if (!iaa_crypto_enabled) { @@ -1773,10 +1764,8 @@ static int iaa_comp_adecompress(struct acomp_req *req) " req->dlen %d, sg_dma_len(sg) %d\n", dst_addr, nr_sgs, req->dst, req->dlen, sg_dma_len(req->dst)); - start_time_ns = iaa_get_ts(); ret = iaa_decompress(tfm, req, wq, src_addr, req->slen, dst_addr, &req->dlen, false); - update_max_decomp_delay_ns(start_time_ns); if (ret == -EINPROGRESS) return ret; @@ -2014,7 +2003,7 @@ static int __init iaa_crypto_init_module(void) int ret = 0; int node; - nr_cpus = num_online_cpus(); + nr_cpus = num_possible_cpus(); for_each_node_with_cpus(node) nr_nodes++; if (!nr_nodes) { diff --git a/drivers/crypto/intel/iaa/iaa_crypto_stats.c b/drivers/crypto/intel/iaa/iaa_crypto_stats.c index c9f83af4b307..f5cc3d29ca19 100644 --- a/drivers/crypto/intel/iaa/iaa_crypto_stats.c +++ b/drivers/crypto/intel/iaa/iaa_crypto_stats.c @@ -17,141 +17,117 @@ #include "iaa_crypto.h" #include "iaa_crypto_stats.h" -static u64 total_comp_calls; -static u64 total_decomp_calls; -static u64 total_sw_decomp_calls; -static u64 max_comp_delay_ns; -static u64 max_decomp_delay_ns; -static u64 total_comp_bytes_out; -static u64 total_decomp_bytes_in; -static u64 total_completion_einval_errors; -static u64 total_completion_timeout_errors; -static u64 total_completion_comp_buf_overflow_errors; +static atomic64_t total_comp_calls; +static atomic64_t total_decomp_calls; +static atomic64_t total_sw_decomp_calls; +static atomic64_t total_comp_bytes_out; +static atomic64_t total_decomp_bytes_in; +static atomic64_t total_completion_einval_errors; +static atomic64_t total_completion_timeout_errors; +static atomic64_t total_completion_comp_buf_overflow_errors; static struct dentry *iaa_crypto_debugfs_root; void update_total_comp_calls(void) { - total_comp_calls++; + atomic64_inc(&total_comp_calls); } void update_total_comp_bytes_out(int n) { - total_comp_bytes_out += n; + atomic64_add(n, &total_comp_bytes_out); } void update_total_decomp_calls(void) { - total_decomp_calls++; + atomic64_inc(&total_decomp_calls); } void update_total_sw_decomp_calls(void) { - total_sw_decomp_calls++; + atomic64_inc(&total_sw_decomp_calls); } void update_total_decomp_bytes_in(int n) { - total_decomp_bytes_in += n; + atomic64_add(n, &total_decomp_bytes_in); } void update_completion_einval_errs(void) { - total_completion_einval_errors++; + atomic64_inc(&total_completion_einval_errors); } void update_completion_timeout_errs(void) { - total_completion_timeout_errors++; + atomic64_inc(&total_completion_timeout_errors); } void update_completion_comp_buf_overflow_errs(void) { - total_completion_comp_buf_overflow_errors++; -} - -void update_max_comp_delay_ns(u64 start_time_ns) -{ - u64 time_diff; - - time_diff = ktime_get_ns() - start_time_ns; - - if (time_diff > max_comp_delay_ns) - max_comp_delay_ns = time_diff; -} - -void update_max_decomp_delay_ns(u64 start_time_ns) -{ - u64 time_diff; - - time_diff = ktime_get_ns() - start_time_ns; - - if (time_diff > max_decomp_delay_ns) - max_decomp_delay_ns = time_diff; + atomic64_inc(&total_completion_comp_buf_overflow_errors); } void update_wq_comp_calls(struct idxd_wq *idxd_wq) { struct iaa_wq *wq = idxd_wq_get_private(idxd_wq); - wq->comp_calls++; - wq->iaa_device->comp_calls++; + atomic64_inc(&wq->comp_calls); + atomic64_inc(&wq->iaa_device->comp_calls); } void update_wq_comp_bytes(struct idxd_wq *idxd_wq, int n) { struct iaa_wq *wq = idxd_wq_get_private(idxd_wq); - wq->comp_bytes += n; - wq->iaa_device->comp_bytes += n; + atomic64_add(n, &wq->comp_bytes); + atomic64_add(n, &wq->iaa_device->comp_bytes); } void update_wq_decomp_calls(struct idxd_wq *idxd_wq) { struct iaa_wq *wq = idxd_wq_get_private(idxd_wq); - wq->decomp_calls++; - wq->iaa_device->decomp_calls++; + atomic64_inc(&wq->decomp_calls); + atomic64_inc(&wq->iaa_device->decomp_calls); } void update_wq_decomp_bytes(struct idxd_wq *idxd_wq, int n) { struct iaa_wq *wq = idxd_wq_get_private(idxd_wq); - wq->decomp_bytes += n; - wq->iaa_device->decomp_bytes += n; + atomic64_add(n, &wq->decomp_bytes); + atomic64_add(n, &wq->iaa_device->decomp_bytes); } static void reset_iaa_crypto_stats(void) { - total_comp_calls = 0; - total_decomp_calls = 0; - total_sw_decomp_calls = 0; - max_comp_delay_ns = 0; - max_decomp_delay_ns = 0; - total_comp_bytes_out = 0; - total_decomp_bytes_in = 0; - total_completion_einval_errors = 0; - total_completion_timeout_errors = 0; - total_completion_comp_buf_overflow_errors = 0; + atomic64_set(&total_comp_calls, 0); + atomic64_set(&total_decomp_calls, 0); + atomic64_set(&total_sw_decomp_calls, 0); + atomic64_set(&total_comp_bytes_out, 0); + atomic64_set(&total_decomp_bytes_in, 0); + atomic64_set(&total_completion_einval_errors, 0); + atomic64_set(&total_completion_timeout_errors, 0); + atomic64_set(&total_completion_comp_buf_overflow_errors, 0); } static void reset_wq_stats(struct iaa_wq *wq) { - wq->comp_calls = 0; - wq->comp_bytes = 0; - wq->decomp_calls = 0; - wq->decomp_bytes = 0; + atomic64_set(&wq->comp_calls, 0); + atomic64_set(&wq->comp_bytes, 0); + atomic64_set(&wq->decomp_calls, 0); + atomic64_set(&wq->decomp_bytes, 0); } static void reset_device_stats(struct iaa_device *iaa_device) { struct iaa_wq *iaa_wq; - iaa_device->comp_calls = 0; - iaa_device->comp_bytes = 0; - iaa_device->decomp_calls = 0; - iaa_device->decomp_bytes = 0; + atomic64_set(&iaa_device->comp_calls, 0); + atomic64_set(&iaa_device->comp_bytes, 0); + atomic64_set(&iaa_device->decomp_calls, 0); + atomic64_set(&iaa_device->decomp_bytes, 0); list_for_each_entry(iaa_wq, &iaa_device->wqs, list) reset_wq_stats(iaa_wq); @@ -160,10 +136,14 @@ static void reset_device_stats(struct iaa_device *iaa_device) static void wq_show(struct seq_file *m, struct iaa_wq *iaa_wq) { seq_printf(m, " name: %s\n", iaa_wq->wq->name); - seq_printf(m, " comp_calls: %llu\n", iaa_wq->comp_calls); - seq_printf(m, " comp_bytes: %llu\n", iaa_wq->comp_bytes); - seq_printf(m, " decomp_calls: %llu\n", iaa_wq->decomp_calls); - seq_printf(m, " decomp_bytes: %llu\n\n", iaa_wq->decomp_bytes); + seq_printf(m, " comp_calls: %llu\n", + atomic64_read(&iaa_wq->comp_calls)); + seq_printf(m, " comp_bytes: %llu\n", + atomic64_read(&iaa_wq->comp_bytes)); + seq_printf(m, " decomp_calls: %llu\n", + atomic64_read(&iaa_wq->decomp_calls)); + seq_printf(m, " decomp_bytes: %llu\n\n", + atomic64_read(&iaa_wq->decomp_bytes)); } static void device_stats_show(struct seq_file *m, struct iaa_device *iaa_device) @@ -173,30 +153,41 @@ static void device_stats_show(struct seq_file *m, struct iaa_device *iaa_device) seq_puts(m, "iaa device:\n"); seq_printf(m, " id: %d\n", iaa_device->idxd->id); seq_printf(m, " n_wqs: %d\n", iaa_device->n_wq); - seq_printf(m, " comp_calls: %llu\n", iaa_device->comp_calls); - seq_printf(m, " comp_bytes: %llu\n", iaa_device->comp_bytes); - seq_printf(m, " decomp_calls: %llu\n", iaa_device->decomp_calls); - seq_printf(m, " decomp_bytes: %llu\n", iaa_device->decomp_bytes); + seq_printf(m, " comp_calls: %llu\n", + atomic64_read(&iaa_device->comp_calls)); + seq_printf(m, " comp_bytes: %llu\n", + atomic64_read(&iaa_device->comp_bytes)); + seq_printf(m, " decomp_calls: %llu\n", + atomic64_read(&iaa_device->decomp_calls)); + seq_printf(m, " decomp_bytes: %llu\n", + atomic64_read(&iaa_device->decomp_bytes)); seq_puts(m, " wqs:\n"); list_for_each_entry(iaa_wq, &iaa_device->wqs, list) wq_show(m, iaa_wq); } -static void global_stats_show(struct seq_file *m) +static int global_stats_show(struct seq_file *m, void *v) { seq_puts(m, "global stats:\n"); - seq_printf(m, " total_comp_calls: %llu\n", total_comp_calls); - seq_printf(m, " total_decomp_calls: %llu\n", total_decomp_calls); - seq_printf(m, " total_sw_decomp_calls: %llu\n", total_sw_decomp_calls); - seq_printf(m, " total_comp_bytes_out: %llu\n", total_comp_bytes_out); - seq_printf(m, " total_decomp_bytes_in: %llu\n", total_decomp_bytes_in); + seq_printf(m, " total_comp_calls: %llu\n", + atomic64_read(&total_comp_calls)); + seq_printf(m, " total_decomp_calls: %llu\n", + atomic64_read(&total_decomp_calls)); + seq_printf(m, " total_sw_decomp_calls: %llu\n", + atomic64_read(&total_sw_decomp_calls)); + seq_printf(m, " total_comp_bytes_out: %llu\n", + atomic64_read(&total_comp_bytes_out)); + seq_printf(m, " total_decomp_bytes_in: %llu\n", + atomic64_read(&total_decomp_bytes_in)); seq_printf(m, " total_completion_einval_errors: %llu\n", - total_completion_einval_errors); + atomic64_read(&total_completion_einval_errors)); seq_printf(m, " total_completion_timeout_errors: %llu\n", - total_completion_timeout_errors); + atomic64_read(&total_completion_timeout_errors)); seq_printf(m, " total_completion_comp_buf_overflow_errors: %llu\n\n", - total_completion_comp_buf_overflow_errors); + atomic64_read(&total_completion_comp_buf_overflow_errors)); + + return 0; } static int wq_stats_show(struct seq_file *m, void *v) @@ -205,8 +196,6 @@ static int wq_stats_show(struct seq_file *m, void *v) mutex_lock(&iaa_devices_lock); - global_stats_show(m); - list_for_each_entry(iaa_device, &iaa_devices, list) device_stats_show(m, iaa_device); @@ -243,6 +232,18 @@ static const struct file_operations wq_stats_fops = { .release = single_release, }; +static int global_stats_open(struct inode *inode, struct file *file) +{ + return single_open(file, global_stats_show, file); +} + +static const struct file_operations global_stats_fops = { + .open = global_stats_open, + .read = seq_read, + .llseek = seq_lseek, + .release = single_release, +}; + DEFINE_DEBUGFS_ATTRIBUTE(wq_stats_reset_fops, NULL, iaa_crypto_stats_reset, "%llu\n"); int __init iaa_crypto_debugfs_init(void) @@ -252,20 +253,8 @@ int __init iaa_crypto_debugfs_init(void) iaa_crypto_debugfs_root = debugfs_create_dir("iaa_crypto", NULL); - debugfs_create_u64("max_comp_delay_ns", 0644, - iaa_crypto_debugfs_root, &max_comp_delay_ns); - debugfs_create_u64("max_decomp_delay_ns", 0644, - iaa_crypto_debugfs_root, &max_decomp_delay_ns); - debugfs_create_u64("total_comp_calls", 0644, - iaa_crypto_debugfs_root, &total_comp_calls); - debugfs_create_u64("total_decomp_calls", 0644, - iaa_crypto_debugfs_root, &total_decomp_calls); - debugfs_create_u64("total_sw_decomp_calls", 0644, - iaa_crypto_debugfs_root, &total_sw_decomp_calls); - debugfs_create_u64("total_comp_bytes_out", 0644, - iaa_crypto_debugfs_root, &total_comp_bytes_out); - debugfs_create_u64("total_decomp_bytes_in", 0644, - iaa_crypto_debugfs_root, &total_decomp_bytes_in); + debugfs_create_file("global_stats", 0644, iaa_crypto_debugfs_root, NULL, + &global_stats_fops); debugfs_create_file("wq_stats", 0644, iaa_crypto_debugfs_root, NULL, &wq_stats_fops); debugfs_create_file("stats_reset", 0644, iaa_crypto_debugfs_root, NULL, diff --git a/drivers/crypto/intel/iaa/iaa_crypto_stats.h b/drivers/crypto/intel/iaa/iaa_crypto_stats.h index c916ca83f070..3787a5f507eb 100644 --- a/drivers/crypto/intel/iaa/iaa_crypto_stats.h +++ b/drivers/crypto/intel/iaa/iaa_crypto_stats.h @@ -13,8 +13,6 @@ void update_total_comp_bytes_out(int n); void update_total_decomp_calls(void); void update_total_sw_decomp_calls(void); void update_total_decomp_bytes_in(int n); -void update_max_comp_delay_ns(u64 start_time_ns); -void update_max_decomp_delay_ns(u64 start_time_ns); void update_completion_einval_errs(void); void update_completion_timeout_errs(void); void update_completion_comp_buf_overflow_errs(void); @@ -24,8 +22,6 @@ void update_wq_comp_bytes(struct idxd_wq *idxd_wq, int n); void update_wq_decomp_calls(struct idxd_wq *idxd_wq); void update_wq_decomp_bytes(struct idxd_wq *idxd_wq, int n); -static inline u64 iaa_get_ts(void) { return ktime_get_ns(); } - #else static inline int iaa_crypto_debugfs_init(void) { return 0; } static inline void iaa_crypto_debugfs_cleanup(void) {} @@ -35,8 +31,6 @@ static inline void update_total_comp_bytes_out(int n) {} static inline void update_total_decomp_calls(void) {} static inline void update_total_sw_decomp_calls(void) {} static inline void update_total_decomp_bytes_in(int n) {} -static inline void update_max_comp_delay_ns(u64 start_time_ns) {} -static inline void update_max_decomp_delay_ns(u64 start_time_ns) {} static inline void update_completion_einval_errs(void) {} static inline void update_completion_timeout_errs(void) {} static inline void update_completion_comp_buf_overflow_errs(void) {} @@ -46,8 +40,6 @@ static inline void update_wq_comp_bytes(struct idxd_wq *idxd_wq, int n) {} static inline void update_wq_decomp_calls(struct idxd_wq *idxd_wq) {} static inline void update_wq_decomp_bytes(struct idxd_wq *idxd_wq, int n) {} -static inline u64 iaa_get_ts(void) { return 0; } - #endif // CONFIG_CRYPTO_DEV_IAA_CRYPTO_STATS #endif diff --git a/drivers/crypto/intel/qat/qat_420xx/adf_420xx_hw_data.c b/drivers/crypto/intel/qat/qat_420xx/adf_420xx_hw_data.c index 1102c47f8293..78f0ea49254d 100644 --- a/drivers/crypto/intel/qat/qat_420xx/adf_420xx_hw_data.c +++ b/drivers/crypto/intel/qat/qat_420xx/adf_420xx_hw_data.c @@ -10,12 +10,14 @@ #include <adf_fw_config.h> #include <adf_gen4_config.h> #include <adf_gen4_dc.h> +#include <adf_gen4_hw_csr_data.h> #include <adf_gen4_hw_data.h> #include <adf_gen4_pfvf.h> #include <adf_gen4_pm.h> #include <adf_gen4_ras.h> #include <adf_gen4_timer.h> #include <adf_gen4_tl.h> +#include <adf_gen4_vf_mig.h> #include "adf_420xx_hw_data.h" #include "icp_qat_hw.h" @@ -296,7 +298,7 @@ static const u32 *adf_get_arbiter_mapping(struct adf_accel_dev *accel_dev) { if (adf_gen4_init_thd2arb_map(accel_dev)) dev_warn(&GET_DEV(accel_dev), - "Generate of the thread to arbiter map failed"); + "Failed to generate thread to arbiter mapping"); return GET_HW_DATA(accel_dev)->thd_to_arb_map; } @@ -487,6 +489,7 @@ void adf_init_hw_data_420xx(struct adf_hw_device_data *hw_data, u32 dev_id) adf_gen4_init_dc_ops(&hw_data->dc_ops); adf_gen4_init_ras_ops(&hw_data->ras_ops); adf_gen4_init_tl_data(&hw_data->tl_data); + adf_gen4_init_vf_mig_ops(&hw_data->vfmig_ops); adf_init_rl_data(&hw_data->rl_data); } diff --git a/drivers/crypto/intel/qat/qat_4xxx/adf_4xxx_hw_data.c b/drivers/crypto/intel/qat/qat_4xxx/adf_4xxx_hw_data.c index 927506cf271d..9fd7ec53b9f3 100644 --- a/drivers/crypto/intel/qat/qat_4xxx/adf_4xxx_hw_data.c +++ b/drivers/crypto/intel/qat/qat_4xxx/adf_4xxx_hw_data.c @@ -10,12 +10,14 @@ #include <adf_fw_config.h> #include <adf_gen4_config.h> #include <adf_gen4_dc.h> +#include <adf_gen4_hw_csr_data.h> #include <adf_gen4_hw_data.h> #include <adf_gen4_pfvf.h> #include <adf_gen4_pm.h> #include "adf_gen4_ras.h" #include <adf_gen4_timer.h> #include <adf_gen4_tl.h> +#include <adf_gen4_vf_mig.h> #include "adf_4xxx_hw_data.h" #include "icp_qat_hw.h" @@ -208,7 +210,7 @@ static const u32 *adf_get_arbiter_mapping(struct adf_accel_dev *accel_dev) { if (adf_gen4_init_thd2arb_map(accel_dev)) dev_warn(&GET_DEV(accel_dev), - "Generate of the thread to arbiter map failed"); + "Failed to generate thread to arbiter mapping"); return GET_HW_DATA(accel_dev)->thd_to_arb_map; } @@ -454,6 +456,8 @@ void adf_init_hw_data_4xxx(struct adf_hw_device_data *hw_data, u32 dev_id) hw_data->get_ring_to_svc_map = adf_gen4_get_ring_to_svc_map; hw_data->disable_iov = adf_disable_sriov; hw_data->ring_pair_reset = adf_gen4_ring_pair_reset; + hw_data->bank_state_save = adf_gen4_bank_state_save; + hw_data->bank_state_restore = adf_gen4_bank_state_restore; hw_data->enable_pm = adf_gen4_enable_pm; hw_data->handle_pm_interrupt = adf_gen4_handle_pm_interrupt; hw_data->dev_config = adf_gen4_dev_config; @@ -469,6 +473,7 @@ void adf_init_hw_data_4xxx(struct adf_hw_device_data *hw_data, u32 dev_id) adf_gen4_init_dc_ops(&hw_data->dc_ops); adf_gen4_init_ras_ops(&hw_data->ras_ops); adf_gen4_init_tl_data(&hw_data->tl_data); + adf_gen4_init_vf_mig_ops(&hw_data->vfmig_ops); adf_init_rl_data(&hw_data->rl_data); } diff --git a/drivers/crypto/intel/qat/qat_4xxx/adf_drv.c b/drivers/crypto/intel/qat/qat_4xxx/adf_drv.c index 9762f2bf7727..d26564cebdec 100644 --- a/drivers/crypto/intel/qat/qat_4xxx/adf_drv.c +++ b/drivers/crypto/intel/qat/qat_4xxx/adf_drv.c @@ -197,7 +197,9 @@ module_pci_driver(adf_driver); MODULE_LICENSE("Dual BSD/GPL"); MODULE_AUTHOR("Intel"); MODULE_FIRMWARE(ADF_4XXX_FW); +MODULE_FIRMWARE(ADF_402XX_FW); MODULE_FIRMWARE(ADF_4XXX_MMP); +MODULE_FIRMWARE(ADF_402XX_MMP); MODULE_DESCRIPTION("Intel(R) QuickAssist Technology"); MODULE_VERSION(ADF_DRV_VERSION); MODULE_SOFTDEP("pre: crypto-intel_qat"); diff --git a/drivers/crypto/intel/qat/qat_c3xxx/adf_c3xxx_hw_data.c b/drivers/crypto/intel/qat/qat_c3xxx/adf_c3xxx_hw_data.c index a882e0ea2279..201f9412c582 100644 --- a/drivers/crypto/intel/qat/qat_c3xxx/adf_c3xxx_hw_data.c +++ b/drivers/crypto/intel/qat/qat_c3xxx/adf_c3xxx_hw_data.c @@ -6,6 +6,7 @@ #include <adf_common_drv.h> #include <adf_gen2_config.h> #include <adf_gen2_dc.h> +#include <adf_gen2_hw_csr_data.h> #include <adf_gen2_hw_data.h> #include <adf_gen2_pfvf.h> #include "adf_c3xxx_hw_data.h" diff --git a/drivers/crypto/intel/qat/qat_c3xxxvf/adf_c3xxxvf_hw_data.c b/drivers/crypto/intel/qat/qat_c3xxxvf/adf_c3xxxvf_hw_data.c index 84d9486e04de..a512ca4efd3f 100644 --- a/drivers/crypto/intel/qat/qat_c3xxxvf/adf_c3xxxvf_hw_data.c +++ b/drivers/crypto/intel/qat/qat_c3xxxvf/adf_c3xxxvf_hw_data.c @@ -4,6 +4,7 @@ #include <adf_common_drv.h> #include <adf_gen2_config.h> #include <adf_gen2_dc.h> +#include <adf_gen2_hw_csr_data.h> #include <adf_gen2_hw_data.h> #include <adf_gen2_pfvf.h> #include <adf_pfvf_vf_msg.h> diff --git a/drivers/crypto/intel/qat/qat_c62x/adf_c62x_hw_data.c b/drivers/crypto/intel/qat/qat_c62x/adf_c62x_hw_data.c index 48cf3eb7c734..6b5b0cf9c7c7 100644 --- a/drivers/crypto/intel/qat/qat_c62x/adf_c62x_hw_data.c +++ b/drivers/crypto/intel/qat/qat_c62x/adf_c62x_hw_data.c @@ -6,6 +6,7 @@ #include <adf_common_drv.h> #include <adf_gen2_config.h> #include <adf_gen2_dc.h> +#include <adf_gen2_hw_csr_data.h> #include <adf_gen2_hw_data.h> #include <adf_gen2_pfvf.h> #include "adf_c62x_hw_data.h" diff --git a/drivers/crypto/intel/qat/qat_c62xvf/adf_c62xvf_hw_data.c b/drivers/crypto/intel/qat/qat_c62xvf/adf_c62xvf_hw_data.c index 751d7aa57fc7..4aaaaf921734 100644 --- a/drivers/crypto/intel/qat/qat_c62xvf/adf_c62xvf_hw_data.c +++ b/drivers/crypto/intel/qat/qat_c62xvf/adf_c62xvf_hw_data.c @@ -4,6 +4,7 @@ #include <adf_common_drv.h> #include <adf_gen2_config.h> #include <adf_gen2_dc.h> +#include <adf_gen2_hw_csr_data.h> #include <adf_gen2_hw_data.h> #include <adf_gen2_pfvf.h> #include <adf_pfvf_vf_msg.h> diff --git a/drivers/crypto/intel/qat/qat_common/Makefile b/drivers/crypto/intel/qat/qat_common/Makefile index 5915cde8a7aa..6f9266edc9f1 100644 --- a/drivers/crypto/intel/qat/qat_common/Makefile +++ b/drivers/crypto/intel/qat/qat_common/Makefile @@ -14,16 +14,20 @@ intel_qat-objs := adf_cfg.o \ adf_hw_arbiter.o \ adf_sysfs.o \ adf_sysfs_ras_counters.o \ + adf_gen2_hw_csr_data.o \ adf_gen2_hw_data.o \ adf_gen2_config.o \ adf_gen4_config.o \ + adf_gen4_hw_csr_data.o \ adf_gen4_hw_data.o \ + adf_gen4_vf_mig.o \ adf_gen4_pm.o \ adf_gen2_dc.o \ adf_gen4_dc.o \ adf_gen4_ras.o \ adf_gen4_timer.o \ adf_clock.o \ + adf_mstate_mgr.o \ qat_crypto.o \ qat_compression.o \ qat_comp_algs.o \ @@ -52,6 +56,6 @@ intel_qat-$(CONFIG_DEBUG_FS) += adf_transport_debug.o \ intel_qat-$(CONFIG_PCI_IOV) += adf_sriov.o adf_vf_isr.o adf_pfvf_utils.o \ adf_pfvf_pf_msg.o adf_pfvf_pf_proto.o \ adf_pfvf_vf_msg.o adf_pfvf_vf_proto.o \ - adf_gen2_pfvf.o adf_gen4_pfvf.o + adf_gen2_pfvf.o adf_gen4_pfvf.o qat_mig_dev.o intel_qat-$(CONFIG_CRYPTO_DEV_QAT_ERROR_INJECTION) += adf_heartbeat_inject.o diff --git a/drivers/crypto/intel/qat/qat_common/adf_accel_devices.h b/drivers/crypto/intel/qat/qat_common/adf_accel_devices.h index 08658c3a01e9..7830ecb1a1f1 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_accel_devices.h +++ b/drivers/crypto/intel/qat/qat_common/adf_accel_devices.h @@ -9,6 +9,7 @@ #include <linux/pci.h> #include <linux/ratelimit.h> #include <linux/types.h> +#include <linux/qat/qat_mig_dev.h> #include "adf_cfg_common.h" #include "adf_rl.h" #include "adf_telemetry.h" @@ -140,6 +141,40 @@ struct admin_info { u32 mailbox_offset; }; +struct ring_config { + u64 base; + u32 config; + u32 head; + u32 tail; + u32 reserved0; +}; + +struct bank_state { + u32 ringstat0; + u32 ringstat1; + u32 ringuostat; + u32 ringestat; + u32 ringnestat; + u32 ringnfstat; + u32 ringfstat; + u32 ringcstat0; + u32 ringcstat1; + u32 ringcstat2; + u32 ringcstat3; + u32 iaintflagen; + u32 iaintflagreg; + u32 iaintflagsrcsel0; + u32 iaintflagsrcsel1; + u32 iaintcolen; + u32 iaintcolctl; + u32 iaintflagandcolen; + u32 ringexpstat; + u32 ringexpintenable; + u32 ringsrvarben; + u32 reserved0; + struct ring_config rings[ADF_ETR_MAX_RINGS_PER_BANK]; +}; + struct adf_hw_csr_ops { u64 (*build_csr_ring_base_addr)(dma_addr_t addr, u32 size); u32 (*read_csr_ring_head)(void __iomem *csr_base_addr, u32 bank, @@ -150,22 +185,49 @@ struct adf_hw_csr_ops { u32 ring); void (*write_csr_ring_tail)(void __iomem *csr_base_addr, u32 bank, u32 ring, u32 value); + u32 (*read_csr_stat)(void __iomem *csr_base_addr, u32 bank); + u32 (*read_csr_uo_stat)(void __iomem *csr_base_addr, u32 bank); u32 (*read_csr_e_stat)(void __iomem *csr_base_addr, u32 bank); + u32 (*read_csr_ne_stat)(void __iomem *csr_base_addr, u32 bank); + u32 (*read_csr_nf_stat)(void __iomem *csr_base_addr, u32 bank); + u32 (*read_csr_f_stat)(void __iomem *csr_base_addr, u32 bank); + u32 (*read_csr_c_stat)(void __iomem *csr_base_addr, u32 bank); + u32 (*read_csr_exp_stat)(void __iomem *csr_base_addr, u32 bank); + u32 (*read_csr_exp_int_en)(void __iomem *csr_base_addr, u32 bank); + void (*write_csr_exp_int_en)(void __iomem *csr_base_addr, u32 bank, + u32 value); + u32 (*read_csr_ring_config)(void __iomem *csr_base_addr, u32 bank, + u32 ring); void (*write_csr_ring_config)(void __iomem *csr_base_addr, u32 bank, u32 ring, u32 value); + dma_addr_t (*read_csr_ring_base)(void __iomem *csr_base_addr, u32 bank, + u32 ring); void (*write_csr_ring_base)(void __iomem *csr_base_addr, u32 bank, u32 ring, dma_addr_t addr); + u32 (*read_csr_int_en)(void __iomem *csr_base_addr, u32 bank); + void (*write_csr_int_en)(void __iomem *csr_base_addr, u32 bank, + u32 value); + u32 (*read_csr_int_flag)(void __iomem *csr_base_addr, u32 bank); void (*write_csr_int_flag)(void __iomem *csr_base_addr, u32 bank, u32 value); + u32 (*read_csr_int_srcsel)(void __iomem *csr_base_addr, u32 bank); void (*write_csr_int_srcsel)(void __iomem *csr_base_addr, u32 bank); + void (*write_csr_int_srcsel_w_val)(void __iomem *csr_base_addr, + u32 bank, u32 value); + u32 (*read_csr_int_col_en)(void __iomem *csr_base_addr, u32 bank); void (*write_csr_int_col_en)(void __iomem *csr_base_addr, u32 bank, u32 value); + u32 (*read_csr_int_col_ctl)(void __iomem *csr_base_addr, u32 bank); void (*write_csr_int_col_ctl)(void __iomem *csr_base_addr, u32 bank, u32 value); + u32 (*read_csr_int_flag_and_col)(void __iomem *csr_base_addr, + u32 bank); void (*write_csr_int_flag_and_col)(void __iomem *csr_base_addr, u32 bank, u32 value); + u32 (*read_csr_ring_srv_arb_en)(void __iomem *csr_base_addr, u32 bank); void (*write_csr_ring_srv_arb_en)(void __iomem *csr_base_addr, u32 bank, u32 value); + u32 (*get_int_col_ctl_enable_mask)(void); }; struct adf_cfg_device_data; @@ -197,6 +259,20 @@ struct adf_dc_ops { void (*build_deflate_ctx)(void *ctx); }; +struct qat_migdev_ops { + int (*init)(struct qat_mig_dev *mdev); + void (*cleanup)(struct qat_mig_dev *mdev); + void (*reset)(struct qat_mig_dev *mdev); + int (*open)(struct qat_mig_dev *mdev); + void (*close)(struct qat_mig_dev *mdev); + int (*suspend)(struct qat_mig_dev *mdev); + int (*resume)(struct qat_mig_dev *mdev); + int (*save_state)(struct qat_mig_dev *mdev); + int (*save_setup)(struct qat_mig_dev *mdev); + int (*load_state)(struct qat_mig_dev *mdev); + int (*load_setup)(struct qat_mig_dev *mdev, int size); +}; + struct adf_dev_err_mask { u32 cppagentcmdpar_mask; u32 parerr_ath_cph_mask; @@ -244,6 +320,10 @@ struct adf_hw_device_data { void (*enable_ints)(struct adf_accel_dev *accel_dev); void (*set_ssm_wdtimer)(struct adf_accel_dev *accel_dev); int (*ring_pair_reset)(struct adf_accel_dev *accel_dev, u32 bank_nr); + int (*bank_state_save)(struct adf_accel_dev *accel_dev, u32 bank_number, + struct bank_state *state); + int (*bank_state_restore)(struct adf_accel_dev *accel_dev, + u32 bank_number, struct bank_state *state); void (*reset_device)(struct adf_accel_dev *accel_dev); void (*set_msix_rttable)(struct adf_accel_dev *accel_dev); const char *(*uof_get_name)(struct adf_accel_dev *accel_dev, u32 obj_num); @@ -260,6 +340,7 @@ struct adf_hw_device_data { struct adf_dev_err_mask dev_err_mask; struct adf_rl_hw_data rl_data; struct adf_tl_hw_data tl_data; + struct qat_migdev_ops vfmig_ops; const char *fw_name; const char *fw_mmp_name; u32 fuses; @@ -316,6 +397,7 @@ struct adf_hw_device_data { #define GET_CSR_OPS(accel_dev) (&(accel_dev)->hw_device->csr_ops) #define GET_PFVF_OPS(accel_dev) (&(accel_dev)->hw_device->pfvf_ops) #define GET_DC_OPS(accel_dev) (&(accel_dev)->hw_device->dc_ops) +#define GET_VFMIG_OPS(accel_dev) (&(accel_dev)->hw_device->vfmig_ops) #define GET_TL_DATA(accel_dev) GET_HW_DATA(accel_dev)->tl_data #define accel_to_pci_dev(accel_ptr) accel_ptr->accel_pci_dev.pci_dev @@ -330,11 +412,17 @@ struct adf_fw_loader_data { struct adf_accel_vf_info { struct adf_accel_dev *accel_dev; struct mutex pf2vf_lock; /* protect CSR access for PF2VF messages */ + struct mutex pfvf_mig_lock; /* protects PFVF state for migration */ struct ratelimit_state vf2pf_ratelimit; u32 vf_nr; bool init; bool restarting; u8 vf_compat_ver; + /* + * Private area used for device migration. + * Memory allocation and free is managed by migration driver. + */ + void *mig_priv; }; struct adf_dc_data { diff --git a/drivers/crypto/intel/qat/qat_common/adf_common_drv.h b/drivers/crypto/intel/qat/qat_common/adf_common_drv.h index 57328249c89e..3bec9e20bad0 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_common_drv.h +++ b/drivers/crypto/intel/qat/qat_common/adf_common_drv.h @@ -248,6 +248,16 @@ static inline void __iomem *adf_get_pmisc_base(struct adf_accel_dev *accel_dev) return pmisc->virt_addr; } +static inline void __iomem *adf_get_etr_base(struct adf_accel_dev *accel_dev) +{ + struct adf_hw_device_data *hw_data = accel_dev->hw_device; + struct adf_bar *etr; + + etr = &GET_BARS(accel_dev)[hw_data->get_etr_bar_id(hw_data)]; + + return etr->virt_addr; +} + static inline void __iomem *adf_get_aram_base(struct adf_accel_dev *accel_dev) { struct adf_hw_device_data *hw_data = accel_dev->hw_device; diff --git a/drivers/crypto/intel/qat/qat_common/adf_gen2_hw_csr_data.c b/drivers/crypto/intel/qat/qat_common/adf_gen2_hw_csr_data.c new file mode 100644 index 000000000000..650c9edd8a66 --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/adf_gen2_hw_csr_data.c @@ -0,0 +1,101 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright(c) 2024 Intel Corporation */ +#include <linux/types.h> +#include "adf_gen2_hw_csr_data.h" + +static u64 build_csr_ring_base_addr(dma_addr_t addr, u32 size) +{ + return BUILD_RING_BASE_ADDR(addr, size); +} + +static u32 read_csr_ring_head(void __iomem *csr_base_addr, u32 bank, u32 ring) +{ + return READ_CSR_RING_HEAD(csr_base_addr, bank, ring); +} + +static void write_csr_ring_head(void __iomem *csr_base_addr, u32 bank, u32 ring, + u32 value) +{ + WRITE_CSR_RING_HEAD(csr_base_addr, bank, ring, value); +} + +static u32 read_csr_ring_tail(void __iomem *csr_base_addr, u32 bank, u32 ring) +{ + return READ_CSR_RING_TAIL(csr_base_addr, bank, ring); +} + +static void write_csr_ring_tail(void __iomem *csr_base_addr, u32 bank, u32 ring, + u32 value) +{ + WRITE_CSR_RING_TAIL(csr_base_addr, bank, ring, value); +} + +static u32 read_csr_e_stat(void __iomem *csr_base_addr, u32 bank) +{ + return READ_CSR_E_STAT(csr_base_addr, bank); +} + +static void write_csr_ring_config(void __iomem *csr_base_addr, u32 bank, + u32 ring, u32 value) +{ + WRITE_CSR_RING_CONFIG(csr_base_addr, bank, ring, value); +} + +static void write_csr_ring_base(void __iomem *csr_base_addr, u32 bank, u32 ring, + dma_addr_t addr) +{ + WRITE_CSR_RING_BASE(csr_base_addr, bank, ring, addr); +} + +static void write_csr_int_flag(void __iomem *csr_base_addr, u32 bank, u32 value) +{ + WRITE_CSR_INT_FLAG(csr_base_addr, bank, value); +} + +static void write_csr_int_srcsel(void __iomem *csr_base_addr, u32 bank) +{ + WRITE_CSR_INT_SRCSEL(csr_base_addr, bank); +} + +static void write_csr_int_col_en(void __iomem *csr_base_addr, u32 bank, + u32 value) +{ + WRITE_CSR_INT_COL_EN(csr_base_addr, bank, value); +} + +static void write_csr_int_col_ctl(void __iomem *csr_base_addr, u32 bank, + u32 value) +{ + WRITE_CSR_INT_COL_CTL(csr_base_addr, bank, value); +} + +static void write_csr_int_flag_and_col(void __iomem *csr_base_addr, u32 bank, + u32 value) +{ + WRITE_CSR_INT_FLAG_AND_COL(csr_base_addr, bank, value); +} + +static void write_csr_ring_srv_arb_en(void __iomem *csr_base_addr, u32 bank, + u32 value) +{ + WRITE_CSR_RING_SRV_ARB_EN(csr_base_addr, bank, value); +} + +void adf_gen2_init_hw_csr_ops(struct adf_hw_csr_ops *csr_ops) +{ + csr_ops->build_csr_ring_base_addr = build_csr_ring_base_addr; + csr_ops->read_csr_ring_head = read_csr_ring_head; + csr_ops->write_csr_ring_head = write_csr_ring_head; + csr_ops->read_csr_ring_tail = read_csr_ring_tail; + csr_ops->write_csr_ring_tail = write_csr_ring_tail; + csr_ops->read_csr_e_stat = read_csr_e_stat; + csr_ops->write_csr_ring_config = write_csr_ring_config; + csr_ops->write_csr_ring_base = write_csr_ring_base; + csr_ops->write_csr_int_flag = write_csr_int_flag; + csr_ops->write_csr_int_srcsel = write_csr_int_srcsel; + csr_ops->write_csr_int_col_en = write_csr_int_col_en; + csr_ops->write_csr_int_col_ctl = write_csr_int_col_ctl; + csr_ops->write_csr_int_flag_and_col = write_csr_int_flag_and_col; + csr_ops->write_csr_ring_srv_arb_en = write_csr_ring_srv_arb_en; +} +EXPORT_SYMBOL_GPL(adf_gen2_init_hw_csr_ops); diff --git a/drivers/crypto/intel/qat/qat_common/adf_gen2_hw_csr_data.h b/drivers/crypto/intel/qat/qat_common/adf_gen2_hw_csr_data.h new file mode 100644 index 000000000000..55058b0f9e52 --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/adf_gen2_hw_csr_data.h @@ -0,0 +1,86 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright(c) 2024 Intel Corporation */ +#ifndef ADF_GEN2_HW_CSR_DATA_H_ +#define ADF_GEN2_HW_CSR_DATA_H_ + +#include <linux/bitops.h> +#include "adf_accel_devices.h" + +#define ADF_BANK_INT_SRC_SEL_MASK_0 0x4444444CUL +#define ADF_BANK_INT_SRC_SEL_MASK_X 0x44444444UL +#define ADF_RING_CSR_RING_CONFIG 0x000 +#define ADF_RING_CSR_RING_LBASE 0x040 +#define ADF_RING_CSR_RING_UBASE 0x080 +#define ADF_RING_CSR_RING_HEAD 0x0C0 +#define ADF_RING_CSR_RING_TAIL 0x100 +#define ADF_RING_CSR_E_STAT 0x14C +#define ADF_RING_CSR_INT_FLAG 0x170 +#define ADF_RING_CSR_INT_SRCSEL 0x174 +#define ADF_RING_CSR_INT_SRCSEL_2 0x178 +#define ADF_RING_CSR_INT_COL_EN 0x17C +#define ADF_RING_CSR_INT_COL_CTL 0x180 +#define ADF_RING_CSR_INT_FLAG_AND_COL 0x184 +#define ADF_RING_CSR_INT_COL_CTL_ENABLE 0x80000000 +#define ADF_RING_BUNDLE_SIZE 0x1000 +#define ADF_ARB_REG_SLOT 0x1000 +#define ADF_ARB_RINGSRVARBEN_OFFSET 0x19C + +#define BUILD_RING_BASE_ADDR(addr, size) \ + (((addr) >> 6) & (GENMASK_ULL(63, 0) << (size))) +#define READ_CSR_RING_HEAD(csr_base_addr, bank, ring) \ + ADF_CSR_RD(csr_base_addr, (ADF_RING_BUNDLE_SIZE * (bank)) + \ + ADF_RING_CSR_RING_HEAD + ((ring) << 2)) +#define READ_CSR_RING_TAIL(csr_base_addr, bank, ring) \ + ADF_CSR_RD(csr_base_addr, (ADF_RING_BUNDLE_SIZE * (bank)) + \ + ADF_RING_CSR_RING_TAIL + ((ring) << 2)) +#define READ_CSR_E_STAT(csr_base_addr, bank) \ + ADF_CSR_RD(csr_base_addr, (ADF_RING_BUNDLE_SIZE * (bank)) + \ + ADF_RING_CSR_E_STAT) +#define WRITE_CSR_RING_CONFIG(csr_base_addr, bank, ring, value) \ + ADF_CSR_WR(csr_base_addr, (ADF_RING_BUNDLE_SIZE * (bank)) + \ + ADF_RING_CSR_RING_CONFIG + ((ring) << 2), value) +#define WRITE_CSR_RING_BASE(csr_base_addr, bank, ring, value) \ +do { \ + u32 l_base = 0, u_base = 0; \ + l_base = (u32)((value) & 0xFFFFFFFF); \ + u_base = (u32)(((value) & 0xFFFFFFFF00000000ULL) >> 32); \ + ADF_CSR_WR(csr_base_addr, (ADF_RING_BUNDLE_SIZE * (bank)) + \ + ADF_RING_CSR_RING_LBASE + ((ring) << 2), l_base); \ + ADF_CSR_WR(csr_base_addr, (ADF_RING_BUNDLE_SIZE * (bank)) + \ + ADF_RING_CSR_RING_UBASE + ((ring) << 2), u_base); \ +} while (0) + +#define WRITE_CSR_RING_HEAD(csr_base_addr, bank, ring, value) \ + ADF_CSR_WR(csr_base_addr, (ADF_RING_BUNDLE_SIZE * (bank)) + \ + ADF_RING_CSR_RING_HEAD + ((ring) << 2), value) +#define WRITE_CSR_RING_TAIL(csr_base_addr, bank, ring, value) \ + ADF_CSR_WR(csr_base_addr, (ADF_RING_BUNDLE_SIZE * (bank)) + \ + ADF_RING_CSR_RING_TAIL + ((ring) << 2), value) +#define WRITE_CSR_INT_FLAG(csr_base_addr, bank, value) \ + ADF_CSR_WR(csr_base_addr, (ADF_RING_BUNDLE_SIZE * (bank)) + \ + ADF_RING_CSR_INT_FLAG, value) +#define WRITE_CSR_INT_SRCSEL(csr_base_addr, bank) \ +do { \ + ADF_CSR_WR(csr_base_addr, (ADF_RING_BUNDLE_SIZE * (bank)) + \ + ADF_RING_CSR_INT_SRCSEL, ADF_BANK_INT_SRC_SEL_MASK_0); \ + ADF_CSR_WR(csr_base_addr, (ADF_RING_BUNDLE_SIZE * (bank)) + \ + ADF_RING_CSR_INT_SRCSEL_2, ADF_BANK_INT_SRC_SEL_MASK_X); \ +} while (0) +#define WRITE_CSR_INT_COL_EN(csr_base_addr, bank, value) \ + ADF_CSR_WR(csr_base_addr, (ADF_RING_BUNDLE_SIZE * (bank)) + \ + ADF_RING_CSR_INT_COL_EN, value) +#define WRITE_CSR_INT_COL_CTL(csr_base_addr, bank, value) \ + ADF_CSR_WR(csr_base_addr, (ADF_RING_BUNDLE_SIZE * (bank)) + \ + ADF_RING_CSR_INT_COL_CTL, \ + ADF_RING_CSR_INT_COL_CTL_ENABLE | (value)) +#define WRITE_CSR_INT_FLAG_AND_COL(csr_base_addr, bank, value) \ + ADF_CSR_WR(csr_base_addr, (ADF_RING_BUNDLE_SIZE * (bank)) + \ + ADF_RING_CSR_INT_FLAG_AND_COL, value) + +#define WRITE_CSR_RING_SRV_ARB_EN(csr_addr, index, value) \ + ADF_CSR_WR(csr_addr, ADF_ARB_RINGSRVARBEN_OFFSET + \ + (ADF_ARB_REG_SLOT * (index)), value) + +void adf_gen2_init_hw_csr_ops(struct adf_hw_csr_ops *csr_ops); + +#endif diff --git a/drivers/crypto/intel/qat/qat_common/adf_gen2_hw_data.c b/drivers/crypto/intel/qat/qat_common/adf_gen2_hw_data.c index d1884547b5a1..1f64bf49b221 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_gen2_hw_data.c +++ b/drivers/crypto/intel/qat/qat_common/adf_gen2_hw_data.c @@ -111,103 +111,6 @@ void adf_gen2_enable_ints(struct adf_accel_dev *accel_dev) } EXPORT_SYMBOL_GPL(adf_gen2_enable_ints); -static u64 build_csr_ring_base_addr(dma_addr_t addr, u32 size) -{ - return BUILD_RING_BASE_ADDR(addr, size); -} - -static u32 read_csr_ring_head(void __iomem *csr_base_addr, u32 bank, u32 ring) -{ - return READ_CSR_RING_HEAD(csr_base_addr, bank, ring); -} - -static void write_csr_ring_head(void __iomem *csr_base_addr, u32 bank, u32 ring, - u32 value) -{ - WRITE_CSR_RING_HEAD(csr_base_addr, bank, ring, value); -} - -static u32 read_csr_ring_tail(void __iomem *csr_base_addr, u32 bank, u32 ring) -{ - return READ_CSR_RING_TAIL(csr_base_addr, bank, ring); -} - -static void write_csr_ring_tail(void __iomem *csr_base_addr, u32 bank, u32 ring, - u32 value) -{ - WRITE_CSR_RING_TAIL(csr_base_addr, bank, ring, value); -} - -static u32 read_csr_e_stat(void __iomem *csr_base_addr, u32 bank) -{ - return READ_CSR_E_STAT(csr_base_addr, bank); -} - -static void write_csr_ring_config(void __iomem *csr_base_addr, u32 bank, - u32 ring, u32 value) -{ - WRITE_CSR_RING_CONFIG(csr_base_addr, bank, ring, value); -} - -static void write_csr_ring_base(void __iomem *csr_base_addr, u32 bank, u32 ring, - dma_addr_t addr) -{ - WRITE_CSR_RING_BASE(csr_base_addr, bank, ring, addr); -} - -static void write_csr_int_flag(void __iomem *csr_base_addr, u32 bank, u32 value) -{ - WRITE_CSR_INT_FLAG(csr_base_addr, bank, value); -} - -static void write_csr_int_srcsel(void __iomem *csr_base_addr, u32 bank) -{ - WRITE_CSR_INT_SRCSEL(csr_base_addr, bank); -} - -static void write_csr_int_col_en(void __iomem *csr_base_addr, u32 bank, - u32 value) -{ - WRITE_CSR_INT_COL_EN(csr_base_addr, bank, value); -} - -static void write_csr_int_col_ctl(void __iomem *csr_base_addr, u32 bank, - u32 value) -{ - WRITE_CSR_INT_COL_CTL(csr_base_addr, bank, value); -} - -static void write_csr_int_flag_and_col(void __iomem *csr_base_addr, u32 bank, - u32 value) -{ - WRITE_CSR_INT_FLAG_AND_COL(csr_base_addr, bank, value); -} - -static void write_csr_ring_srv_arb_en(void __iomem *csr_base_addr, u32 bank, - u32 value) -{ - WRITE_CSR_RING_SRV_ARB_EN(csr_base_addr, bank, value); -} - -void adf_gen2_init_hw_csr_ops(struct adf_hw_csr_ops *csr_ops) -{ - csr_ops->build_csr_ring_base_addr = build_csr_ring_base_addr; - csr_ops->read_csr_ring_head = read_csr_ring_head; - csr_ops->write_csr_ring_head = write_csr_ring_head; - csr_ops->read_csr_ring_tail = read_csr_ring_tail; - csr_ops->write_csr_ring_tail = write_csr_ring_tail; - csr_ops->read_csr_e_stat = read_csr_e_stat; - csr_ops->write_csr_ring_config = write_csr_ring_config; - csr_ops->write_csr_ring_base = write_csr_ring_base; - csr_ops->write_csr_int_flag = write_csr_int_flag; - csr_ops->write_csr_int_srcsel = write_csr_int_srcsel; - csr_ops->write_csr_int_col_en = write_csr_int_col_en; - csr_ops->write_csr_int_col_ctl = write_csr_int_col_ctl; - csr_ops->write_csr_int_flag_and_col = write_csr_int_flag_and_col; - csr_ops->write_csr_ring_srv_arb_en = write_csr_ring_srv_arb_en; -} -EXPORT_SYMBOL_GPL(adf_gen2_init_hw_csr_ops); - u32 adf_gen2_get_accel_cap(struct adf_accel_dev *accel_dev) { struct adf_hw_device_data *hw_data = accel_dev->hw_device; diff --git a/drivers/crypto/intel/qat/qat_common/adf_gen2_hw_data.h b/drivers/crypto/intel/qat/qat_common/adf_gen2_hw_data.h index 6bd341061de4..708e9186127b 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_gen2_hw_data.h +++ b/drivers/crypto/intel/qat/qat_common/adf_gen2_hw_data.h @@ -6,78 +6,9 @@ #include "adf_accel_devices.h" #include "adf_cfg_common.h" -/* Transport access */ -#define ADF_BANK_INT_SRC_SEL_MASK_0 0x4444444CUL -#define ADF_BANK_INT_SRC_SEL_MASK_X 0x44444444UL -#define ADF_RING_CSR_RING_CONFIG 0x000 -#define ADF_RING_CSR_RING_LBASE 0x040 -#define ADF_RING_CSR_RING_UBASE 0x080 -#define ADF_RING_CSR_RING_HEAD 0x0C0 -#define ADF_RING_CSR_RING_TAIL 0x100 -#define ADF_RING_CSR_E_STAT 0x14C -#define ADF_RING_CSR_INT_FLAG 0x170 -#define ADF_RING_CSR_INT_SRCSEL 0x174 -#define ADF_RING_CSR_INT_SRCSEL_2 0x178 -#define ADF_RING_CSR_INT_COL_EN 0x17C -#define ADF_RING_CSR_INT_COL_CTL 0x180 -#define ADF_RING_CSR_INT_FLAG_AND_COL 0x184 -#define ADF_RING_CSR_INT_COL_CTL_ENABLE 0x80000000 -#define ADF_RING_BUNDLE_SIZE 0x1000 #define ADF_GEN2_RX_RINGS_OFFSET 8 #define ADF_GEN2_TX_RINGS_MASK 0xFF -#define BUILD_RING_BASE_ADDR(addr, size) \ - (((addr) >> 6) & (GENMASK_ULL(63, 0) << (size))) -#define READ_CSR_RING_HEAD(csr_base_addr, bank, ring) \ - ADF_CSR_RD(csr_base_addr, (ADF_RING_BUNDLE_SIZE * (bank)) + \ - ADF_RING_CSR_RING_HEAD + ((ring) << 2)) -#define READ_CSR_RING_TAIL(csr_base_addr, bank, ring) \ - ADF_CSR_RD(csr_base_addr, (ADF_RING_BUNDLE_SIZE * (bank)) + \ - ADF_RING_CSR_RING_TAIL + ((ring) << 2)) -#define READ_CSR_E_STAT(csr_base_addr, bank) \ - ADF_CSR_RD(csr_base_addr, (ADF_RING_BUNDLE_SIZE * (bank)) + \ - ADF_RING_CSR_E_STAT) -#define WRITE_CSR_RING_CONFIG(csr_base_addr, bank, ring, value) \ - ADF_CSR_WR(csr_base_addr, (ADF_RING_BUNDLE_SIZE * (bank)) + \ - ADF_RING_CSR_RING_CONFIG + ((ring) << 2), value) -#define WRITE_CSR_RING_BASE(csr_base_addr, bank, ring, value) \ -do { \ - u32 l_base = 0, u_base = 0; \ - l_base = (u32)((value) & 0xFFFFFFFF); \ - u_base = (u32)(((value) & 0xFFFFFFFF00000000ULL) >> 32); \ - ADF_CSR_WR(csr_base_addr, (ADF_RING_BUNDLE_SIZE * (bank)) + \ - ADF_RING_CSR_RING_LBASE + ((ring) << 2), l_base); \ - ADF_CSR_WR(csr_base_addr, (ADF_RING_BUNDLE_SIZE * (bank)) + \ - ADF_RING_CSR_RING_UBASE + ((ring) << 2), u_base); \ -} while (0) - -#define WRITE_CSR_RING_HEAD(csr_base_addr, bank, ring, value) \ - ADF_CSR_WR(csr_base_addr, (ADF_RING_BUNDLE_SIZE * (bank)) + \ - ADF_RING_CSR_RING_HEAD + ((ring) << 2), value) -#define WRITE_CSR_RING_TAIL(csr_base_addr, bank, ring, value) \ - ADF_CSR_WR(csr_base_addr, (ADF_RING_BUNDLE_SIZE * (bank)) + \ - ADF_RING_CSR_RING_TAIL + ((ring) << 2), value) -#define WRITE_CSR_INT_FLAG(csr_base_addr, bank, value) \ - ADF_CSR_WR(csr_base_addr, (ADF_RING_BUNDLE_SIZE * (bank)) + \ - ADF_RING_CSR_INT_FLAG, value) -#define WRITE_CSR_INT_SRCSEL(csr_base_addr, bank) \ -do { \ - ADF_CSR_WR(csr_base_addr, (ADF_RING_BUNDLE_SIZE * (bank)) + \ - ADF_RING_CSR_INT_SRCSEL, ADF_BANK_INT_SRC_SEL_MASK_0); \ - ADF_CSR_WR(csr_base_addr, (ADF_RING_BUNDLE_SIZE * (bank)) + \ - ADF_RING_CSR_INT_SRCSEL_2, ADF_BANK_INT_SRC_SEL_MASK_X); \ -} while (0) -#define WRITE_CSR_INT_COL_EN(csr_base_addr, bank, value) \ - ADF_CSR_WR(csr_base_addr, (ADF_RING_BUNDLE_SIZE * (bank)) + \ - ADF_RING_CSR_INT_COL_EN, value) -#define WRITE_CSR_INT_COL_CTL(csr_base_addr, bank, value) \ - ADF_CSR_WR(csr_base_addr, (ADF_RING_BUNDLE_SIZE * (bank)) + \ - ADF_RING_CSR_INT_COL_CTL, \ - ADF_RING_CSR_INT_COL_CTL_ENABLE | (value)) -#define WRITE_CSR_INT_FLAG_AND_COL(csr_base_addr, bank, value) \ - ADF_CSR_WR(csr_base_addr, (ADF_RING_BUNDLE_SIZE * (bank)) + \ - ADF_RING_CSR_INT_FLAG_AND_COL, value) - /* AE to function map */ #define AE2FUNCTION_MAP_A_OFFSET (0x3A400 + 0x190) #define AE2FUNCTION_MAP_B_OFFSET (0x3A400 + 0x310) @@ -106,12 +37,6 @@ do { \ #define ADF_ARB_OFFSET 0x30000 #define ADF_ARB_WRK_2_SER_MAP_OFFSET 0x180 #define ADF_ARB_CONFIG (BIT(31) | BIT(6) | BIT(0)) -#define ADF_ARB_REG_SLOT 0x1000 -#define ADF_ARB_RINGSRVARBEN_OFFSET 0x19C - -#define WRITE_CSR_RING_SRV_ARB_EN(csr_addr, index, value) \ - ADF_CSR_WR(csr_addr, ADF_ARB_RINGSRVARBEN_OFFSET + \ - (ADF_ARB_REG_SLOT * (index)), value) /* Power gating */ #define ADF_POWERGATE_DC BIT(23) @@ -158,7 +83,6 @@ u32 adf_gen2_get_num_aes(struct adf_hw_device_data *self); void adf_gen2_enable_error_correction(struct adf_accel_dev *accel_dev); void adf_gen2_cfg_iov_thds(struct adf_accel_dev *accel_dev, bool enable, int num_a_regs, int num_b_regs); -void adf_gen2_init_hw_csr_ops(struct adf_hw_csr_ops *csr_ops); void adf_gen2_get_admin_info(struct admin_info *admin_csrs_info); void adf_gen2_get_arb_info(struct arb_info *arb_info); void adf_gen2_enable_ints(struct adf_accel_dev *accel_dev); diff --git a/drivers/crypto/intel/qat/qat_common/adf_gen4_hw_csr_data.c b/drivers/crypto/intel/qat/qat_common/adf_gen4_hw_csr_data.c new file mode 100644 index 000000000000..6609c248aaba --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/adf_gen4_hw_csr_data.c @@ -0,0 +1,231 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright(c) 2024 Intel Corporation */ +#include <linux/types.h> +#include "adf_gen4_hw_csr_data.h" + +static u64 build_csr_ring_base_addr(dma_addr_t addr, u32 size) +{ + return BUILD_RING_BASE_ADDR(addr, size); +} + +static u32 read_csr_ring_head(void __iomem *csr_base_addr, u32 bank, u32 ring) +{ + return READ_CSR_RING_HEAD(csr_base_addr, bank, ring); +} + +static void write_csr_ring_head(void __iomem *csr_base_addr, u32 bank, u32 ring, + u32 value) +{ + WRITE_CSR_RING_HEAD(csr_base_addr, bank, ring, value); +} + +static u32 read_csr_ring_tail(void __iomem *csr_base_addr, u32 bank, u32 ring) +{ + return READ_CSR_RING_TAIL(csr_base_addr, bank, ring); +} + +static void write_csr_ring_tail(void __iomem *csr_base_addr, u32 bank, u32 ring, + u32 value) +{ + WRITE_CSR_RING_TAIL(csr_base_addr, bank, ring, value); +} + +static u32 read_csr_stat(void __iomem *csr_base_addr, u32 bank) +{ + return READ_CSR_STAT(csr_base_addr, bank); +} + +static u32 read_csr_uo_stat(void __iomem *csr_base_addr, u32 bank) +{ + return READ_CSR_UO_STAT(csr_base_addr, bank); +} + +static u32 read_csr_e_stat(void __iomem *csr_base_addr, u32 bank) +{ + return READ_CSR_E_STAT(csr_base_addr, bank); +} + +static u32 read_csr_ne_stat(void __iomem *csr_base_addr, u32 bank) +{ + return READ_CSR_NE_STAT(csr_base_addr, bank); +} + +static u32 read_csr_nf_stat(void __iomem *csr_base_addr, u32 bank) +{ + return READ_CSR_NF_STAT(csr_base_addr, bank); +} + +static u32 read_csr_f_stat(void __iomem *csr_base_addr, u32 bank) +{ + return READ_CSR_F_STAT(csr_base_addr, bank); +} + +static u32 read_csr_c_stat(void __iomem *csr_base_addr, u32 bank) +{ + return READ_CSR_C_STAT(csr_base_addr, bank); +} + +static u32 read_csr_exp_stat(void __iomem *csr_base_addr, u32 bank) +{ + return READ_CSR_EXP_STAT(csr_base_addr, bank); +} + +static u32 read_csr_exp_int_en(void __iomem *csr_base_addr, u32 bank) +{ + return READ_CSR_EXP_INT_EN(csr_base_addr, bank); +} + +static void write_csr_exp_int_en(void __iomem *csr_base_addr, u32 bank, + u32 value) +{ + WRITE_CSR_EXP_INT_EN(csr_base_addr, bank, value); +} + +static u32 read_csr_ring_config(void __iomem *csr_base_addr, u32 bank, + u32 ring) +{ + return READ_CSR_RING_CONFIG(csr_base_addr, bank, ring); +} + +static void write_csr_ring_config(void __iomem *csr_base_addr, u32 bank, u32 ring, + u32 value) +{ + WRITE_CSR_RING_CONFIG(csr_base_addr, bank, ring, value); +} + +static dma_addr_t read_csr_ring_base(void __iomem *csr_base_addr, u32 bank, + u32 ring) +{ + return READ_CSR_RING_BASE(csr_base_addr, bank, ring); +} + +static void write_csr_ring_base(void __iomem *csr_base_addr, u32 bank, u32 ring, + dma_addr_t addr) +{ + WRITE_CSR_RING_BASE(csr_base_addr, bank, ring, addr); +} + +static u32 read_csr_int_en(void __iomem *csr_base_addr, u32 bank) +{ + return READ_CSR_INT_EN(csr_base_addr, bank); +} + +static void write_csr_int_en(void __iomem *csr_base_addr, u32 bank, u32 value) +{ + WRITE_CSR_INT_EN(csr_base_addr, bank, value); +} + +static u32 read_csr_int_flag(void __iomem *csr_base_addr, u32 bank) +{ + return READ_CSR_INT_FLAG(csr_base_addr, bank); +} + +static void write_csr_int_flag(void __iomem *csr_base_addr, u32 bank, + u32 value) +{ + WRITE_CSR_INT_FLAG(csr_base_addr, bank, value); +} + +static u32 read_csr_int_srcsel(void __iomem *csr_base_addr, u32 bank) +{ + return READ_CSR_INT_SRCSEL(csr_base_addr, bank); +} + +static void write_csr_int_srcsel(void __iomem *csr_base_addr, u32 bank) +{ + WRITE_CSR_INT_SRCSEL(csr_base_addr, bank); +} + +static void write_csr_int_srcsel_w_val(void __iomem *csr_base_addr, u32 bank, + u32 value) +{ + WRITE_CSR_INT_SRCSEL_W_VAL(csr_base_addr, bank, value); +} + +static u32 read_csr_int_col_en(void __iomem *csr_base_addr, u32 bank) +{ + return READ_CSR_INT_COL_EN(csr_base_addr, bank); +} + +static void write_csr_int_col_en(void __iomem *csr_base_addr, u32 bank, u32 value) +{ + WRITE_CSR_INT_COL_EN(csr_base_addr, bank, value); +} + +static u32 read_csr_int_col_ctl(void __iomem *csr_base_addr, u32 bank) +{ + return READ_CSR_INT_COL_CTL(csr_base_addr, bank); +} + +static void write_csr_int_col_ctl(void __iomem *csr_base_addr, u32 bank, + u32 value) +{ + WRITE_CSR_INT_COL_CTL(csr_base_addr, bank, value); +} + +static u32 read_csr_int_flag_and_col(void __iomem *csr_base_addr, u32 bank) +{ + return READ_CSR_INT_FLAG_AND_COL(csr_base_addr, bank); +} + +static void write_csr_int_flag_and_col(void __iomem *csr_base_addr, u32 bank, + u32 value) +{ + WRITE_CSR_INT_FLAG_AND_COL(csr_base_addr, bank, value); +} + +static u32 read_csr_ring_srv_arb_en(void __iomem *csr_base_addr, u32 bank) +{ + return READ_CSR_RING_SRV_ARB_EN(csr_base_addr, bank); +} + +static void write_csr_ring_srv_arb_en(void __iomem *csr_base_addr, u32 bank, + u32 value) +{ + WRITE_CSR_RING_SRV_ARB_EN(csr_base_addr, bank, value); +} + +static u32 get_int_col_ctl_enable_mask(void) +{ + return ADF_RING_CSR_INT_COL_CTL_ENABLE; +} + +void adf_gen4_init_hw_csr_ops(struct adf_hw_csr_ops *csr_ops) +{ + csr_ops->build_csr_ring_base_addr = build_csr_ring_base_addr; + csr_ops->read_csr_ring_head = read_csr_ring_head; + csr_ops->write_csr_ring_head = write_csr_ring_head; + csr_ops->read_csr_ring_tail = read_csr_ring_tail; + csr_ops->write_csr_ring_tail = write_csr_ring_tail; + csr_ops->read_csr_stat = read_csr_stat; + csr_ops->read_csr_uo_stat = read_csr_uo_stat; + csr_ops->read_csr_e_stat = read_csr_e_stat; + csr_ops->read_csr_ne_stat = read_csr_ne_stat; + csr_ops->read_csr_nf_stat = read_csr_nf_stat; + csr_ops->read_csr_f_stat = read_csr_f_stat; + csr_ops->read_csr_c_stat = read_csr_c_stat; + csr_ops->read_csr_exp_stat = read_csr_exp_stat; + csr_ops->read_csr_exp_int_en = read_csr_exp_int_en; + csr_ops->write_csr_exp_int_en = write_csr_exp_int_en; + csr_ops->read_csr_ring_config = read_csr_ring_config; + csr_ops->write_csr_ring_config = write_csr_ring_config; + csr_ops->read_csr_ring_base = read_csr_ring_base; + csr_ops->write_csr_ring_base = write_csr_ring_base; + csr_ops->read_csr_int_en = read_csr_int_en; + csr_ops->write_csr_int_en = write_csr_int_en; + csr_ops->read_csr_int_flag = read_csr_int_flag; + csr_ops->write_csr_int_flag = write_csr_int_flag; + csr_ops->read_csr_int_srcsel = read_csr_int_srcsel; + csr_ops->write_csr_int_srcsel = write_csr_int_srcsel; + csr_ops->write_csr_int_srcsel_w_val = write_csr_int_srcsel_w_val; + csr_ops->read_csr_int_col_en = read_csr_int_col_en; + csr_ops->write_csr_int_col_en = write_csr_int_col_en; + csr_ops->read_csr_int_col_ctl = read_csr_int_col_ctl; + csr_ops->write_csr_int_col_ctl = write_csr_int_col_ctl; + csr_ops->read_csr_int_flag_and_col = read_csr_int_flag_and_col; + csr_ops->write_csr_int_flag_and_col = write_csr_int_flag_and_col; + csr_ops->read_csr_ring_srv_arb_en = read_csr_ring_srv_arb_en; + csr_ops->write_csr_ring_srv_arb_en = write_csr_ring_srv_arb_en; + csr_ops->get_int_col_ctl_enable_mask = get_int_col_ctl_enable_mask; +} +EXPORT_SYMBOL_GPL(adf_gen4_init_hw_csr_ops); diff --git a/drivers/crypto/intel/qat/qat_common/adf_gen4_hw_csr_data.h b/drivers/crypto/intel/qat/qat_common/adf_gen4_hw_csr_data.h new file mode 100644 index 000000000000..6f33e7c87c2c --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/adf_gen4_hw_csr_data.h @@ -0,0 +1,188 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright(c) 2024 Intel Corporation */ +#ifndef ADF_GEN4_HW_CSR_DATA_H_ +#define ADF_GEN4_HW_CSR_DATA_H_ + +#include <linux/bitops.h> +#include "adf_accel_devices.h" + +#define ADF_BANK_INT_SRC_SEL_MASK 0x44UL +#define ADF_RING_CSR_RING_CONFIG 0x1000 +#define ADF_RING_CSR_RING_LBASE 0x1040 +#define ADF_RING_CSR_RING_UBASE 0x1080 +#define ADF_RING_CSR_RING_HEAD 0x0C0 +#define ADF_RING_CSR_RING_TAIL 0x100 +#define ADF_RING_CSR_STAT 0x140 +#define ADF_RING_CSR_UO_STAT 0x148 +#define ADF_RING_CSR_E_STAT 0x14C +#define ADF_RING_CSR_NE_STAT 0x150 +#define ADF_RING_CSR_NF_STAT 0x154 +#define ADF_RING_CSR_F_STAT 0x158 +#define ADF_RING_CSR_C_STAT 0x15C +#define ADF_RING_CSR_INT_FLAG_EN 0x16C +#define ADF_RING_CSR_INT_FLAG 0x170 +#define ADF_RING_CSR_INT_SRCSEL 0x174 +#define ADF_RING_CSR_INT_COL_EN 0x17C +#define ADF_RING_CSR_INT_COL_CTL 0x180 +#define ADF_RING_CSR_INT_FLAG_AND_COL 0x184 +#define ADF_RING_CSR_EXP_STAT 0x188 +#define ADF_RING_CSR_EXP_INT_EN 0x18C +#define ADF_RING_CSR_INT_COL_CTL_ENABLE 0x80000000 +#define ADF_RING_CSR_ADDR_OFFSET 0x100000 +#define ADF_RING_BUNDLE_SIZE 0x2000 +#define ADF_RING_CSR_RING_SRV_ARB_EN 0x19C + +#define BUILD_RING_BASE_ADDR(addr, size) \ + ((((addr) >> 6) & (GENMASK_ULL(63, 0) << (size))) << 6) +#define READ_CSR_RING_HEAD(csr_base_addr, bank, ring) \ + ADF_CSR_RD((csr_base_addr) + ADF_RING_CSR_ADDR_OFFSET, \ + ADF_RING_BUNDLE_SIZE * (bank) + \ + ADF_RING_CSR_RING_HEAD + ((ring) << 2)) +#define READ_CSR_RING_TAIL(csr_base_addr, bank, ring) \ + ADF_CSR_RD((csr_base_addr) + ADF_RING_CSR_ADDR_OFFSET, \ + ADF_RING_BUNDLE_SIZE * (bank) + \ + ADF_RING_CSR_RING_TAIL + ((ring) << 2)) +#define READ_CSR_STAT(csr_base_addr, bank) \ + ADF_CSR_RD((csr_base_addr) + ADF_RING_CSR_ADDR_OFFSET, \ + ADF_RING_BUNDLE_SIZE * (bank) + ADF_RING_CSR_STAT) +#define READ_CSR_UO_STAT(csr_base_addr, bank) \ + ADF_CSR_RD((csr_base_addr) + ADF_RING_CSR_ADDR_OFFSET, \ + ADF_RING_BUNDLE_SIZE * (bank) + ADF_RING_CSR_UO_STAT) +#define READ_CSR_E_STAT(csr_base_addr, bank) \ + ADF_CSR_RD((csr_base_addr) + ADF_RING_CSR_ADDR_OFFSET, \ + ADF_RING_BUNDLE_SIZE * (bank) + ADF_RING_CSR_E_STAT) +#define READ_CSR_NE_STAT(csr_base_addr, bank) \ + ADF_CSR_RD((csr_base_addr) + ADF_RING_CSR_ADDR_OFFSET, \ + ADF_RING_BUNDLE_SIZE * (bank) + ADF_RING_CSR_NE_STAT) +#define READ_CSR_NF_STAT(csr_base_addr, bank) \ + ADF_CSR_RD((csr_base_addr) + ADF_RING_CSR_ADDR_OFFSET, \ + ADF_RING_BUNDLE_SIZE * (bank) + ADF_RING_CSR_NF_STAT) +#define READ_CSR_F_STAT(csr_base_addr, bank) \ + ADF_CSR_RD((csr_base_addr) + ADF_RING_CSR_ADDR_OFFSET, \ + ADF_RING_BUNDLE_SIZE * (bank) + ADF_RING_CSR_F_STAT) +#define READ_CSR_C_STAT(csr_base_addr, bank) \ + ADF_CSR_RD((csr_base_addr) + ADF_RING_CSR_ADDR_OFFSET, \ + ADF_RING_BUNDLE_SIZE * (bank) + ADF_RING_CSR_C_STAT) +#define READ_CSR_EXP_STAT(csr_base_addr, bank) \ + ADF_CSR_RD((csr_base_addr) + ADF_RING_CSR_ADDR_OFFSET, \ + ADF_RING_BUNDLE_SIZE * (bank) + ADF_RING_CSR_EXP_STAT) +#define READ_CSR_EXP_INT_EN(csr_base_addr, bank) \ + ADF_CSR_RD((csr_base_addr) + ADF_RING_CSR_ADDR_OFFSET, \ + ADF_RING_BUNDLE_SIZE * (bank) + ADF_RING_CSR_EXP_INT_EN) +#define WRITE_CSR_EXP_INT_EN(csr_base_addr, bank, value) \ + ADF_CSR_WR((csr_base_addr) + ADF_RING_CSR_ADDR_OFFSET, \ + ADF_RING_BUNDLE_SIZE * (bank) + \ + ADF_RING_CSR_EXP_INT_EN, value) +#define READ_CSR_RING_CONFIG(csr_base_addr, bank, ring) \ + ADF_CSR_RD((csr_base_addr) + ADF_RING_CSR_ADDR_OFFSET, \ + ADF_RING_BUNDLE_SIZE * (bank) + \ + ADF_RING_CSR_RING_CONFIG + ((ring) << 2)) +#define WRITE_CSR_RING_CONFIG(csr_base_addr, bank, ring, value) \ + ADF_CSR_WR((csr_base_addr) + ADF_RING_CSR_ADDR_OFFSET, \ + ADF_RING_BUNDLE_SIZE * (bank) + \ + ADF_RING_CSR_RING_CONFIG + ((ring) << 2), value) +#define WRITE_CSR_RING_BASE(csr_base_addr, bank, ring, value) \ +do { \ + void __iomem *_csr_base_addr = csr_base_addr; \ + u32 _bank = bank; \ + u32 _ring = ring; \ + dma_addr_t _value = value; \ + u32 l_base = 0, u_base = 0; \ + l_base = lower_32_bits(_value); \ + u_base = upper_32_bits(_value); \ + ADF_CSR_WR((_csr_base_addr) + ADF_RING_CSR_ADDR_OFFSET, \ + ADF_RING_BUNDLE_SIZE * (_bank) + \ + ADF_RING_CSR_RING_LBASE + ((_ring) << 2), l_base); \ + ADF_CSR_WR((_csr_base_addr) + ADF_RING_CSR_ADDR_OFFSET, \ + ADF_RING_BUNDLE_SIZE * (_bank) + \ + ADF_RING_CSR_RING_UBASE + ((_ring) << 2), u_base); \ +} while (0) + +static inline u64 read_base(void __iomem *csr_base_addr, u32 bank, u32 ring) +{ + u32 l_base, u_base; + + /* + * Use special IO wrapper for ring base as LBASE and UBASE are + * not physically contigious + */ + l_base = ADF_CSR_RD(csr_base_addr, (ADF_RING_BUNDLE_SIZE * bank) + + ADF_RING_CSR_RING_LBASE + (ring << 2)); + u_base = ADF_CSR_RD(csr_base_addr, (ADF_RING_BUNDLE_SIZE * bank) + + ADF_RING_CSR_RING_UBASE + (ring << 2)); + + return (u64)u_base << 32 | (u64)l_base; +} + +#define READ_CSR_RING_BASE(csr_base_addr, bank, ring) \ + read_base((csr_base_addr) + ADF_RING_CSR_ADDR_OFFSET, (bank), (ring)) + +#define WRITE_CSR_RING_HEAD(csr_base_addr, bank, ring, value) \ + ADF_CSR_WR((csr_base_addr) + ADF_RING_CSR_ADDR_OFFSET, \ + ADF_RING_BUNDLE_SIZE * (bank) + \ + ADF_RING_CSR_RING_HEAD + ((ring) << 2), value) +#define WRITE_CSR_RING_TAIL(csr_base_addr, bank, ring, value) \ + ADF_CSR_WR((csr_base_addr) + ADF_RING_CSR_ADDR_OFFSET, \ + ADF_RING_BUNDLE_SIZE * (bank) + \ + ADF_RING_CSR_RING_TAIL + ((ring) << 2), value) +#define READ_CSR_INT_EN(csr_base_addr, bank) \ + ADF_CSR_RD((csr_base_addr) + ADF_RING_CSR_ADDR_OFFSET, \ + ADF_RING_BUNDLE_SIZE * (bank) + ADF_RING_CSR_INT_FLAG_EN) +#define WRITE_CSR_INT_EN(csr_base_addr, bank, value) \ + ADF_CSR_WR((csr_base_addr) + ADF_RING_CSR_ADDR_OFFSET, \ + ADF_RING_BUNDLE_SIZE * (bank) + \ + ADF_RING_CSR_INT_FLAG_EN, (value)) +#define READ_CSR_INT_FLAG(csr_base_addr, bank) \ + ADF_CSR_RD((csr_base_addr) + ADF_RING_CSR_ADDR_OFFSET, \ + ADF_RING_BUNDLE_SIZE * (bank) + ADF_RING_CSR_INT_FLAG) +#define WRITE_CSR_INT_FLAG(csr_base_addr, bank, value) \ + ADF_CSR_WR((csr_base_addr) + ADF_RING_CSR_ADDR_OFFSET, \ + ADF_RING_BUNDLE_SIZE * (bank) + \ + ADF_RING_CSR_INT_FLAG, (value)) +#define READ_CSR_INT_SRCSEL(csr_base_addr, bank) \ + ADF_CSR_RD((csr_base_addr) + ADF_RING_CSR_ADDR_OFFSET, \ + ADF_RING_BUNDLE_SIZE * (bank) + ADF_RING_CSR_INT_SRCSEL) +#define WRITE_CSR_INT_SRCSEL(csr_base_addr, bank) \ + ADF_CSR_WR((csr_base_addr) + ADF_RING_CSR_ADDR_OFFSET, \ + ADF_RING_BUNDLE_SIZE * (bank) + \ + ADF_RING_CSR_INT_SRCSEL, ADF_BANK_INT_SRC_SEL_MASK) +#define WRITE_CSR_INT_SRCSEL_W_VAL(csr_base_addr, bank, value) \ + ADF_CSR_WR((csr_base_addr) + ADF_RING_CSR_ADDR_OFFSET, \ + ADF_RING_BUNDLE_SIZE * (bank) + \ + ADF_RING_CSR_INT_SRCSEL, (value)) +#define READ_CSR_INT_COL_EN(csr_base_addr, bank) \ + ADF_CSR_RD((csr_base_addr) + ADF_RING_CSR_ADDR_OFFSET, \ + ADF_RING_BUNDLE_SIZE * (bank) + ADF_RING_CSR_INT_COL_EN) +#define WRITE_CSR_INT_COL_EN(csr_base_addr, bank, value) \ + ADF_CSR_WR((csr_base_addr) + ADF_RING_CSR_ADDR_OFFSET, \ + ADF_RING_BUNDLE_SIZE * (bank) + \ + ADF_RING_CSR_INT_COL_EN, (value)) +#define READ_CSR_INT_COL_CTL(csr_base_addr, bank) \ + ADF_CSR_RD((csr_base_addr) + ADF_RING_CSR_ADDR_OFFSET, \ + ADF_RING_BUNDLE_SIZE * (bank) + ADF_RING_CSR_INT_COL_CTL) +#define WRITE_CSR_INT_COL_CTL(csr_base_addr, bank, value) \ + ADF_CSR_WR((csr_base_addr) + ADF_RING_CSR_ADDR_OFFSET, \ + ADF_RING_BUNDLE_SIZE * (bank) + \ + ADF_RING_CSR_INT_COL_CTL, \ + ADF_RING_CSR_INT_COL_CTL_ENABLE | (value)) +#define READ_CSR_INT_FLAG_AND_COL(csr_base_addr, bank) \ + ADF_CSR_RD((csr_base_addr) + ADF_RING_CSR_ADDR_OFFSET, \ + ADF_RING_BUNDLE_SIZE * (bank) + \ + ADF_RING_CSR_INT_FLAG_AND_COL) +#define WRITE_CSR_INT_FLAG_AND_COL(csr_base_addr, bank, value) \ + ADF_CSR_WR((csr_base_addr) + ADF_RING_CSR_ADDR_OFFSET, \ + ADF_RING_BUNDLE_SIZE * (bank) + \ + ADF_RING_CSR_INT_FLAG_AND_COL, (value)) + +#define READ_CSR_RING_SRV_ARB_EN(csr_base_addr, bank) \ + ADF_CSR_RD((csr_base_addr) + ADF_RING_CSR_ADDR_OFFSET, \ + ADF_RING_BUNDLE_SIZE * (bank) + \ + ADF_RING_CSR_RING_SRV_ARB_EN) +#define WRITE_CSR_RING_SRV_ARB_EN(csr_base_addr, bank, value) \ + ADF_CSR_WR((csr_base_addr) + ADF_RING_CSR_ADDR_OFFSET, \ + ADF_RING_BUNDLE_SIZE * (bank) + \ + ADF_RING_CSR_RING_SRV_ARB_EN, (value)) + +void adf_gen4_init_hw_csr_ops(struct adf_hw_csr_ops *csr_ops); + +#endif diff --git a/drivers/crypto/intel/qat/qat_common/adf_gen4_hw_data.c b/drivers/crypto/intel/qat/qat_common/adf_gen4_hw_data.c index d28e1921940a..41a0979e68c1 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_gen4_hw_data.c +++ b/drivers/crypto/intel/qat/qat_common/adf_gen4_hw_data.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) /* Copyright(c) 2020 Intel Corporation */ #include <linux/iopoll.h> +#include <asm/div64.h> #include "adf_accel_devices.h" #include "adf_cfg_services.h" #include "adf_common_drv.h" @@ -8,103 +9,6 @@ #include "adf_gen4_hw_data.h" #include "adf_gen4_pm.h" -static u64 build_csr_ring_base_addr(dma_addr_t addr, u32 size) -{ - return BUILD_RING_BASE_ADDR(addr, size); -} - -static u32 read_csr_ring_head(void __iomem *csr_base_addr, u32 bank, u32 ring) -{ - return READ_CSR_RING_HEAD(csr_base_addr, bank, ring); -} - -static void write_csr_ring_head(void __iomem *csr_base_addr, u32 bank, u32 ring, - u32 value) -{ - WRITE_CSR_RING_HEAD(csr_base_addr, bank, ring, value); -} - -static u32 read_csr_ring_tail(void __iomem *csr_base_addr, u32 bank, u32 ring) -{ - return READ_CSR_RING_TAIL(csr_base_addr, bank, ring); -} - -static void write_csr_ring_tail(void __iomem *csr_base_addr, u32 bank, u32 ring, - u32 value) -{ - WRITE_CSR_RING_TAIL(csr_base_addr, bank, ring, value); -} - -static u32 read_csr_e_stat(void __iomem *csr_base_addr, u32 bank) -{ - return READ_CSR_E_STAT(csr_base_addr, bank); -} - -static void write_csr_ring_config(void __iomem *csr_base_addr, u32 bank, u32 ring, - u32 value) -{ - WRITE_CSR_RING_CONFIG(csr_base_addr, bank, ring, value); -} - -static void write_csr_ring_base(void __iomem *csr_base_addr, u32 bank, u32 ring, - dma_addr_t addr) -{ - WRITE_CSR_RING_BASE(csr_base_addr, bank, ring, addr); -} - -static void write_csr_int_flag(void __iomem *csr_base_addr, u32 bank, - u32 value) -{ - WRITE_CSR_INT_FLAG(csr_base_addr, bank, value); -} - -static void write_csr_int_srcsel(void __iomem *csr_base_addr, u32 bank) -{ - WRITE_CSR_INT_SRCSEL(csr_base_addr, bank); -} - -static void write_csr_int_col_en(void __iomem *csr_base_addr, u32 bank, u32 value) -{ - WRITE_CSR_INT_COL_EN(csr_base_addr, bank, value); -} - -static void write_csr_int_col_ctl(void __iomem *csr_base_addr, u32 bank, - u32 value) -{ - WRITE_CSR_INT_COL_CTL(csr_base_addr, bank, value); -} - -static void write_csr_int_flag_and_col(void __iomem *csr_base_addr, u32 bank, - u32 value) -{ - WRITE_CSR_INT_FLAG_AND_COL(csr_base_addr, bank, value); -} - -static void write_csr_ring_srv_arb_en(void __iomem *csr_base_addr, u32 bank, - u32 value) -{ - WRITE_CSR_RING_SRV_ARB_EN(csr_base_addr, bank, value); -} - -void adf_gen4_init_hw_csr_ops(struct adf_hw_csr_ops *csr_ops) -{ - csr_ops->build_csr_ring_base_addr = build_csr_ring_base_addr; - csr_ops->read_csr_ring_head = read_csr_ring_head; - csr_ops->write_csr_ring_head = write_csr_ring_head; - csr_ops->read_csr_ring_tail = read_csr_ring_tail; - csr_ops->write_csr_ring_tail = write_csr_ring_tail; - csr_ops->read_csr_e_stat = read_csr_e_stat; - csr_ops->write_csr_ring_config = write_csr_ring_config; - csr_ops->write_csr_ring_base = write_csr_ring_base; - csr_ops->write_csr_int_flag = write_csr_int_flag; - csr_ops->write_csr_int_srcsel = write_csr_int_srcsel; - csr_ops->write_csr_int_col_en = write_csr_int_col_en; - csr_ops->write_csr_int_col_ctl = write_csr_int_col_ctl; - csr_ops->write_csr_int_flag_and_col = write_csr_int_flag_and_col; - csr_ops->write_csr_ring_srv_arb_en = write_csr_ring_srv_arb_en; -} -EXPORT_SYMBOL_GPL(adf_gen4_init_hw_csr_ops); - u32 adf_gen4_get_accel_mask(struct adf_hw_device_data *self) { return ADF_GEN4_ACCELERATORS_MASK; @@ -321,8 +225,7 @@ static int reset_ring_pair(void __iomem *csr, u32 bank_number) int adf_gen4_ring_pair_reset(struct adf_accel_dev *accel_dev, u32 bank_number) { struct adf_hw_device_data *hw_data = accel_dev->hw_device; - u32 etr_bar_id = hw_data->get_etr_bar_id(hw_data); - void __iomem *csr; + void __iomem *csr = adf_get_etr_base(accel_dev); int ret; if (bank_number >= hw_data->num_banks) @@ -331,7 +234,6 @@ int adf_gen4_ring_pair_reset(struct adf_accel_dev *accel_dev, u32 bank_number) dev_dbg(&GET_DEV(accel_dev), "ring pair reset for bank:%d\n", bank_number); - csr = (&GET_BARS(accel_dev)[etr_bar_id])->virt_addr; ret = reset_ring_pair(csr, bank_number); if (ret) dev_err(&GET_DEV(accel_dev), @@ -489,3 +391,281 @@ set_mask: return ring_to_svc_map; } EXPORT_SYMBOL_GPL(adf_gen4_get_ring_to_svc_map); + +/* + * adf_gen4_bank_quiesce_coal_timer() - quiesce bank coalesced interrupt timer + * @accel_dev: Pointer to the device structure + * @bank_idx: Offset to the bank within this device + * @timeout_ms: Timeout in milliseconds for the operation + * + * This function tries to quiesce the coalesced interrupt timer of a bank if + * it has been enabled and triggered. + * + * Returns 0 on success, error code otherwise + * + */ +int adf_gen4_bank_quiesce_coal_timer(struct adf_accel_dev *accel_dev, + u32 bank_idx, int timeout_ms) +{ + struct adf_hw_device_data *hw_data = GET_HW_DATA(accel_dev); + struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(accel_dev); + void __iomem *csr_misc = adf_get_pmisc_base(accel_dev); + void __iomem *csr_etr = adf_get_etr_base(accel_dev); + u32 int_col_ctl, int_col_mask, int_col_en; + u32 e_stat, intsrc; + u64 wait_us; + int ret; + + if (timeout_ms < 0) + return -EINVAL; + + int_col_ctl = csr_ops->read_csr_int_col_ctl(csr_etr, bank_idx); + int_col_mask = csr_ops->get_int_col_ctl_enable_mask(); + if (!(int_col_ctl & int_col_mask)) + return 0; + + int_col_en = csr_ops->read_csr_int_col_en(csr_etr, bank_idx); + int_col_en &= BIT(ADF_WQM_CSR_RP_IDX_RX); + + e_stat = csr_ops->read_csr_e_stat(csr_etr, bank_idx); + if (!(~e_stat & int_col_en)) + return 0; + + wait_us = 2 * ((int_col_ctl & ~int_col_mask) << 8) * USEC_PER_SEC; + do_div(wait_us, hw_data->clock_frequency); + wait_us = min(wait_us, (u64)timeout_ms * USEC_PER_MSEC); + dev_dbg(&GET_DEV(accel_dev), + "wait for bank %d - coalesced timer expires in %llu us (max=%u ms estat=0x%x intcolen=0x%x)\n", + bank_idx, wait_us, timeout_ms, e_stat, int_col_en); + + ret = read_poll_timeout(ADF_CSR_RD, intsrc, intsrc, + ADF_COALESCED_POLL_DELAY_US, wait_us, true, + csr_misc, ADF_WQM_CSR_RPINTSOU(bank_idx)); + if (ret) + dev_warn(&GET_DEV(accel_dev), + "coalesced timer for bank %d expired (%llu us)\n", + bank_idx, wait_us); + + return ret; +} +EXPORT_SYMBOL_GPL(adf_gen4_bank_quiesce_coal_timer); + +static int drain_bank(void __iomem *csr, u32 bank_number, int timeout_us) +{ + u32 status; + + ADF_CSR_WR(csr, ADF_WQM_CSR_RPRESETCTL(bank_number), + ADF_WQM_CSR_RPRESETCTL_DRAIN); + + return read_poll_timeout(ADF_CSR_RD, status, + status & ADF_WQM_CSR_RPRESETSTS_STATUS, + ADF_RPRESET_POLL_DELAY_US, timeout_us, true, + csr, ADF_WQM_CSR_RPRESETSTS(bank_number)); +} + +void adf_gen4_bank_drain_finish(struct adf_accel_dev *accel_dev, + u32 bank_number) +{ + void __iomem *csr = adf_get_etr_base(accel_dev); + + ADF_CSR_WR(csr, ADF_WQM_CSR_RPRESETSTS(bank_number), + ADF_WQM_CSR_RPRESETSTS_STATUS); +} + +int adf_gen4_bank_drain_start(struct adf_accel_dev *accel_dev, + u32 bank_number, int timeout_us) +{ + void __iomem *csr = adf_get_etr_base(accel_dev); + int ret; + + dev_dbg(&GET_DEV(accel_dev), "Drain bank %d\n", bank_number); + + ret = drain_bank(csr, bank_number, timeout_us); + if (ret) + dev_err(&GET_DEV(accel_dev), "Bank drain failed (timeout)\n"); + else + dev_dbg(&GET_DEV(accel_dev), "Bank drain successful\n"); + + return ret; +} + +static void bank_state_save(struct adf_hw_csr_ops *ops, void __iomem *base, + u32 bank, struct bank_state *state, u32 num_rings) +{ + u32 i; + + state->ringstat0 = ops->read_csr_stat(base, bank); + state->ringuostat = ops->read_csr_uo_stat(base, bank); + state->ringestat = ops->read_csr_e_stat(base, bank); + state->ringnestat = ops->read_csr_ne_stat(base, bank); + state->ringnfstat = ops->read_csr_nf_stat(base, bank); + state->ringfstat = ops->read_csr_f_stat(base, bank); + state->ringcstat0 = ops->read_csr_c_stat(base, bank); + state->iaintflagen = ops->read_csr_int_en(base, bank); + state->iaintflagreg = ops->read_csr_int_flag(base, bank); + state->iaintflagsrcsel0 = ops->read_csr_int_srcsel(base, bank); + state->iaintcolen = ops->read_csr_int_col_en(base, bank); + state->iaintcolctl = ops->read_csr_int_col_ctl(base, bank); + state->iaintflagandcolen = ops->read_csr_int_flag_and_col(base, bank); + state->ringexpstat = ops->read_csr_exp_stat(base, bank); + state->ringexpintenable = ops->read_csr_exp_int_en(base, bank); + state->ringsrvarben = ops->read_csr_ring_srv_arb_en(base, bank); + + for (i = 0; i < num_rings; i++) { + state->rings[i].head = ops->read_csr_ring_head(base, bank, i); + state->rings[i].tail = ops->read_csr_ring_tail(base, bank, i); + state->rings[i].config = ops->read_csr_ring_config(base, bank, i); + state->rings[i].base = ops->read_csr_ring_base(base, bank, i); + } +} + +#define CHECK_STAT(op, expect_val, name, args...) \ +({ \ + u32 __expect_val = (expect_val); \ + u32 actual_val = op(args); \ + (__expect_val == actual_val) ? 0 : \ + (pr_err("QAT: Fail to restore %s register. Expected 0x%x, actual 0x%x\n", \ + name, __expect_val, actual_val), -EINVAL); \ +}) + +static int bank_state_restore(struct adf_hw_csr_ops *ops, void __iomem *base, + u32 bank, struct bank_state *state, u32 num_rings, + int tx_rx_gap) +{ + u32 val, tmp_val, i; + int ret; + + for (i = 0; i < num_rings; i++) + ops->write_csr_ring_base(base, bank, i, state->rings[i].base); + + for (i = 0; i < num_rings; i++) + ops->write_csr_ring_config(base, bank, i, state->rings[i].config); + + for (i = 0; i < num_rings / 2; i++) { + int tx = i * (tx_rx_gap + 1); + int rx = tx + tx_rx_gap; + + ops->write_csr_ring_head(base, bank, tx, state->rings[tx].head); + ops->write_csr_ring_tail(base, bank, tx, state->rings[tx].tail); + + /* + * The TX ring head needs to be updated again to make sure that + * the HW will not consider the ring as full when it is empty + * and the correct state flags are set to match the recovered state. + */ + if (state->ringestat & BIT(tx)) { + val = ops->read_csr_int_srcsel(base, bank); + val |= ADF_RP_INT_SRC_SEL_F_RISE_MASK; + ops->write_csr_int_srcsel_w_val(base, bank, val); + ops->write_csr_ring_head(base, bank, tx, state->rings[tx].head); + } + + ops->write_csr_ring_tail(base, bank, rx, state->rings[rx].tail); + val = ops->read_csr_int_srcsel(base, bank); + val |= ADF_RP_INT_SRC_SEL_F_RISE_MASK << ADF_RP_INT_SRC_SEL_RANGE_WIDTH; + ops->write_csr_int_srcsel_w_val(base, bank, val); + + ops->write_csr_ring_head(base, bank, rx, state->rings[rx].head); + val = ops->read_csr_int_srcsel(base, bank); + val |= ADF_RP_INT_SRC_SEL_F_FALL_MASK << ADF_RP_INT_SRC_SEL_RANGE_WIDTH; + ops->write_csr_int_srcsel_w_val(base, bank, val); + + /* + * The RX ring tail needs to be updated again to make sure that + * the HW will not consider the ring as empty when it is full + * and the correct state flags are set to match the recovered state. + */ + if (state->ringfstat & BIT(rx)) + ops->write_csr_ring_tail(base, bank, rx, state->rings[rx].tail); + } + + ops->write_csr_int_flag_and_col(base, bank, state->iaintflagandcolen); + ops->write_csr_int_en(base, bank, state->iaintflagen); + ops->write_csr_int_col_en(base, bank, state->iaintcolen); + ops->write_csr_int_srcsel_w_val(base, bank, state->iaintflagsrcsel0); + ops->write_csr_exp_int_en(base, bank, state->ringexpintenable); + ops->write_csr_int_col_ctl(base, bank, state->iaintcolctl); + ops->write_csr_ring_srv_arb_en(base, bank, state->ringsrvarben); + + /* Check that all ring statuses match the saved state. */ + ret = CHECK_STAT(ops->read_csr_stat, state->ringstat0, "ringstat", + base, bank); + if (ret) + return ret; + + ret = CHECK_STAT(ops->read_csr_e_stat, state->ringestat, "ringestat", + base, bank); + if (ret) + return ret; + + ret = CHECK_STAT(ops->read_csr_ne_stat, state->ringnestat, "ringnestat", + base, bank); + if (ret) + return ret; + + ret = CHECK_STAT(ops->read_csr_nf_stat, state->ringnfstat, "ringnfstat", + base, bank); + if (ret) + return ret; + + ret = CHECK_STAT(ops->read_csr_f_stat, state->ringfstat, "ringfstat", + base, bank); + if (ret) + return ret; + + ret = CHECK_STAT(ops->read_csr_c_stat, state->ringcstat0, "ringcstat", + base, bank); + if (ret) + return ret; + + tmp_val = ops->read_csr_exp_stat(base, bank); + val = state->ringexpstat; + if (tmp_val && !val) { + pr_err("QAT: Bank was restored with exception: 0x%x\n", val); + return -EINVAL; + } + + return 0; +} + +int adf_gen4_bank_state_save(struct adf_accel_dev *accel_dev, u32 bank_number, + struct bank_state *state) +{ + struct adf_hw_device_data *hw_data = GET_HW_DATA(accel_dev); + struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(accel_dev); + void __iomem *csr_base = adf_get_etr_base(accel_dev); + + if (bank_number >= hw_data->num_banks || !state) + return -EINVAL; + + dev_dbg(&GET_DEV(accel_dev), "Saving state of bank %d\n", bank_number); + + bank_state_save(csr_ops, csr_base, bank_number, state, + hw_data->num_rings_per_bank); + + return 0; +} +EXPORT_SYMBOL_GPL(adf_gen4_bank_state_save); + +int adf_gen4_bank_state_restore(struct adf_accel_dev *accel_dev, u32 bank_number, + struct bank_state *state) +{ + struct adf_hw_device_data *hw_data = GET_HW_DATA(accel_dev); + struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(accel_dev); + void __iomem *csr_base = adf_get_etr_base(accel_dev); + int ret; + + if (bank_number >= hw_data->num_banks || !state) + return -EINVAL; + + dev_dbg(&GET_DEV(accel_dev), "Restoring state of bank %d\n", bank_number); + + ret = bank_state_restore(csr_ops, csr_base, bank_number, state, + hw_data->num_rings_per_bank, hw_data->tx_rx_gap); + if (ret) + dev_err(&GET_DEV(accel_dev), + "Unable to restore state of bank %d\n", bank_number); + + return ret; +} +EXPORT_SYMBOL_GPL(adf_gen4_bank_state_restore); diff --git a/drivers/crypto/intel/qat/qat_common/adf_gen4_hw_data.h b/drivers/crypto/intel/qat/qat_common/adf_gen4_hw_data.h index c6e80df5a85a..8b10926cedba 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_gen4_hw_data.h +++ b/drivers/crypto/intel/qat/qat_common/adf_gen4_hw_data.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) */ /* Copyright(c) 2020 Intel Corporation */ -#ifndef ADF_GEN4_HW_CSR_DATA_H_ -#define ADF_GEN4_HW_CSR_DATA_H_ +#ifndef ADF_GEN4_HW_DATA_H_ +#define ADF_GEN4_HW_DATA_H_ #include <linux/units.h> @@ -54,95 +54,6 @@ #define ADF_GEN4_ADMINMSGLR_OFFSET 0x500578 #define ADF_GEN4_MAILBOX_BASE_OFFSET 0x600970 -/* Transport access */ -#define ADF_BANK_INT_SRC_SEL_MASK 0x44UL -#define ADF_RING_CSR_RING_CONFIG 0x1000 -#define ADF_RING_CSR_RING_LBASE 0x1040 -#define ADF_RING_CSR_RING_UBASE 0x1080 -#define ADF_RING_CSR_RING_HEAD 0x0C0 -#define ADF_RING_CSR_RING_TAIL 0x100 -#define ADF_RING_CSR_E_STAT 0x14C -#define ADF_RING_CSR_INT_FLAG 0x170 -#define ADF_RING_CSR_INT_SRCSEL 0x174 -#define ADF_RING_CSR_INT_COL_CTL 0x180 -#define ADF_RING_CSR_INT_FLAG_AND_COL 0x184 -#define ADF_RING_CSR_INT_COL_CTL_ENABLE 0x80000000 -#define ADF_RING_CSR_INT_COL_EN 0x17C -#define ADF_RING_CSR_ADDR_OFFSET 0x100000 -#define ADF_RING_BUNDLE_SIZE 0x2000 - -#define BUILD_RING_BASE_ADDR(addr, size) \ - ((((addr) >> 6) & (GENMASK_ULL(63, 0) << (size))) << 6) -#define READ_CSR_RING_HEAD(csr_base_addr, bank, ring) \ - ADF_CSR_RD((csr_base_addr) + ADF_RING_CSR_ADDR_OFFSET, \ - ADF_RING_BUNDLE_SIZE * (bank) + \ - ADF_RING_CSR_RING_HEAD + ((ring) << 2)) -#define READ_CSR_RING_TAIL(csr_base_addr, bank, ring) \ - ADF_CSR_RD((csr_base_addr) + ADF_RING_CSR_ADDR_OFFSET, \ - ADF_RING_BUNDLE_SIZE * (bank) + \ - ADF_RING_CSR_RING_TAIL + ((ring) << 2)) -#define READ_CSR_E_STAT(csr_base_addr, bank) \ - ADF_CSR_RD((csr_base_addr) + ADF_RING_CSR_ADDR_OFFSET, \ - ADF_RING_BUNDLE_SIZE * (bank) + ADF_RING_CSR_E_STAT) -#define WRITE_CSR_RING_CONFIG(csr_base_addr, bank, ring, value) \ - ADF_CSR_WR((csr_base_addr) + ADF_RING_CSR_ADDR_OFFSET, \ - ADF_RING_BUNDLE_SIZE * (bank) + \ - ADF_RING_CSR_RING_CONFIG + ((ring) << 2), value) -#define WRITE_CSR_RING_BASE(csr_base_addr, bank, ring, value) \ -do { \ - void __iomem *_csr_base_addr = csr_base_addr; \ - u32 _bank = bank; \ - u32 _ring = ring; \ - dma_addr_t _value = value; \ - u32 l_base = 0, u_base = 0; \ - l_base = lower_32_bits(_value); \ - u_base = upper_32_bits(_value); \ - ADF_CSR_WR((_csr_base_addr) + ADF_RING_CSR_ADDR_OFFSET, \ - ADF_RING_BUNDLE_SIZE * (_bank) + \ - ADF_RING_CSR_RING_LBASE + ((_ring) << 2), l_base); \ - ADF_CSR_WR((_csr_base_addr) + ADF_RING_CSR_ADDR_OFFSET, \ - ADF_RING_BUNDLE_SIZE * (_bank) + \ - ADF_RING_CSR_RING_UBASE + ((_ring) << 2), u_base); \ -} while (0) - -#define WRITE_CSR_RING_HEAD(csr_base_addr, bank, ring, value) \ - ADF_CSR_WR((csr_base_addr) + ADF_RING_CSR_ADDR_OFFSET, \ - ADF_RING_BUNDLE_SIZE * (bank) + \ - ADF_RING_CSR_RING_HEAD + ((ring) << 2), value) -#define WRITE_CSR_RING_TAIL(csr_base_addr, bank, ring, value) \ - ADF_CSR_WR((csr_base_addr) + ADF_RING_CSR_ADDR_OFFSET, \ - ADF_RING_BUNDLE_SIZE * (bank) + \ - ADF_RING_CSR_RING_TAIL + ((ring) << 2), value) -#define WRITE_CSR_INT_FLAG(csr_base_addr, bank, value) \ - ADF_CSR_WR((csr_base_addr) + ADF_RING_CSR_ADDR_OFFSET, \ - ADF_RING_BUNDLE_SIZE * (bank) + \ - ADF_RING_CSR_INT_FLAG, (value)) -#define WRITE_CSR_INT_SRCSEL(csr_base_addr, bank) \ - ADF_CSR_WR((csr_base_addr) + ADF_RING_CSR_ADDR_OFFSET, \ - ADF_RING_BUNDLE_SIZE * (bank) + \ - ADF_RING_CSR_INT_SRCSEL, ADF_BANK_INT_SRC_SEL_MASK) -#define WRITE_CSR_INT_COL_EN(csr_base_addr, bank, value) \ - ADF_CSR_WR((csr_base_addr) + ADF_RING_CSR_ADDR_OFFSET, \ - ADF_RING_BUNDLE_SIZE * (bank) + \ - ADF_RING_CSR_INT_COL_EN, (value)) -#define WRITE_CSR_INT_COL_CTL(csr_base_addr, bank, value) \ - ADF_CSR_WR((csr_base_addr) + ADF_RING_CSR_ADDR_OFFSET, \ - ADF_RING_BUNDLE_SIZE * (bank) + \ - ADF_RING_CSR_INT_COL_CTL, \ - ADF_RING_CSR_INT_COL_CTL_ENABLE | (value)) -#define WRITE_CSR_INT_FLAG_AND_COL(csr_base_addr, bank, value) \ - ADF_CSR_WR((csr_base_addr) + ADF_RING_CSR_ADDR_OFFSET, \ - ADF_RING_BUNDLE_SIZE * (bank) + \ - ADF_RING_CSR_INT_FLAG_AND_COL, (value)) - -/* Arbiter configuration */ -#define ADF_RING_CSR_RING_SRV_ARB_EN 0x19C - -#define WRITE_CSR_RING_SRV_ARB_EN(csr_base_addr, bank, value) \ - ADF_CSR_WR((csr_base_addr) + ADF_RING_CSR_ADDR_OFFSET, \ - ADF_RING_BUNDLE_SIZE * (bank) + \ - ADF_RING_CSR_RING_SRV_ARB_EN, (value)) - /* Default ring mapping */ #define ADF_GEN4_DEFAULT_RING_TO_SRV_MAP \ (ASYM << ADF_CFG_SERV_RING_PAIR_0_SHIFT | \ @@ -166,10 +77,20 @@ do { \ #define ADF_RPRESET_POLL_TIMEOUT_US (5 * USEC_PER_SEC) #define ADF_RPRESET_POLL_DELAY_US 20 #define ADF_WQM_CSR_RPRESETCTL_RESET BIT(0) +#define ADF_WQM_CSR_RPRESETCTL_DRAIN BIT(2) #define ADF_WQM_CSR_RPRESETCTL(bank) (0x6000 + ((bank) << 3)) #define ADF_WQM_CSR_RPRESETSTS_STATUS BIT(0) #define ADF_WQM_CSR_RPRESETSTS(bank) (ADF_WQM_CSR_RPRESETCTL(bank) + 4) +/* Ring interrupt */ +#define ADF_RP_INT_SRC_SEL_F_RISE_MASK BIT(2) +#define ADF_RP_INT_SRC_SEL_F_FALL_MASK GENMASK(2, 0) +#define ADF_RP_INT_SRC_SEL_RANGE_WIDTH 4 +#define ADF_COALESCED_POLL_TIMEOUT_US (1 * USEC_PER_SEC) +#define ADF_COALESCED_POLL_DELAY_US 1000 +#define ADF_WQM_CSR_RPINTSOU(bank) (0x200000 + ((bank) << 12)) +#define ADF_WQM_CSR_RP_IDX_RX 1 + /* Error source registers */ #define ADF_GEN4_ERRSOU0 (0x41A200) #define ADF_GEN4_ERRSOU1 (0x41A204) @@ -197,6 +118,19 @@ do { \ /* Arbiter threads mask with error value */ #define ADF_GEN4_ENA_THD_MASK_ERROR GENMASK(ADF_NUM_THREADS_PER_AE, 0) +/* PF2VM communication channel */ +#define ADF_GEN4_PF2VM_OFFSET(i) (0x40B010 + (i) * 0x20) +#define ADF_GEN4_VM2PF_OFFSET(i) (0x40B014 + (i) * 0x20) +#define ADF_GEN4_VINTMSKPF2VM_OFFSET(i) (0x40B00C + (i) * 0x20) +#define ADF_GEN4_VINTSOUPF2VM_OFFSET(i) (0x40B008 + (i) * 0x20) +#define ADF_GEN4_VINTMSK_OFFSET(i) (0x40B004 + (i) * 0x20) +#define ADF_GEN4_VINTSOU_OFFSET(i) (0x40B000 + (i) * 0x20) + +struct adf_gen4_vfmig { + struct adf_mstate_mgr *mstate_mgr; + bool bank_stopped[ADF_GEN4_NUM_BANKS_PER_VF]; +}; + void adf_gen4_set_ssm_wdtimer(struct adf_accel_dev *accel_dev); enum icp_qat_gen4_slice_mask { @@ -230,11 +164,20 @@ u32 adf_gen4_get_num_aes(struct adf_hw_device_data *self); enum dev_sku_info adf_gen4_get_sku(struct adf_hw_device_data *self); u32 adf_gen4_get_sram_bar_id(struct adf_hw_device_data *self); int adf_gen4_init_device(struct adf_accel_dev *accel_dev); -void adf_gen4_init_hw_csr_ops(struct adf_hw_csr_ops *csr_ops); int adf_gen4_ring_pair_reset(struct adf_accel_dev *accel_dev, u32 bank_number); void adf_gen4_set_msix_default_rttable(struct adf_accel_dev *accel_dev); void adf_gen4_set_ssm_wdtimer(struct adf_accel_dev *accel_dev); int adf_gen4_init_thd2arb_map(struct adf_accel_dev *accel_dev); u16 adf_gen4_get_ring_to_svc_map(struct adf_accel_dev *accel_dev); +int adf_gen4_bank_quiesce_coal_timer(struct adf_accel_dev *accel_dev, + u32 bank_idx, int timeout_ms); +int adf_gen4_bank_drain_start(struct adf_accel_dev *accel_dev, + u32 bank_number, int timeout_us); +void adf_gen4_bank_drain_finish(struct adf_accel_dev *accel_dev, + u32 bank_number); +int adf_gen4_bank_state_save(struct adf_accel_dev *accel_dev, u32 bank_number, + struct bank_state *state); +int adf_gen4_bank_state_restore(struct adf_accel_dev *accel_dev, + u32 bank_number, struct bank_state *state); #endif diff --git a/drivers/crypto/intel/qat/qat_common/adf_gen4_pfvf.c b/drivers/crypto/intel/qat/qat_common/adf_gen4_pfvf.c index 8e8efe93f3ee..21474d402d09 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_gen4_pfvf.c +++ b/drivers/crypto/intel/qat/qat_common/adf_gen4_pfvf.c @@ -6,12 +6,10 @@ #include "adf_accel_devices.h" #include "adf_common_drv.h" #include "adf_gen4_pfvf.h" +#include "adf_gen4_hw_data.h" #include "adf_pfvf_pf_proto.h" #include "adf_pfvf_utils.h" -#define ADF_4XXX_PF2VM_OFFSET(i) (0x40B010 + ((i) * 0x20)) -#define ADF_4XXX_VM2PF_OFFSET(i) (0x40B014 + ((i) * 0x20)) - /* VF2PF interrupt source registers */ #define ADF_4XXX_VM2PF_SOU 0x41A180 #define ADF_4XXX_VM2PF_MSK 0x41A1C0 @@ -29,12 +27,12 @@ static const struct pfvf_csr_format csr_gen4_fmt = { static u32 adf_gen4_pf_get_pf2vf_offset(u32 i) { - return ADF_4XXX_PF2VM_OFFSET(i); + return ADF_GEN4_PF2VM_OFFSET(i); } static u32 adf_gen4_pf_get_vf2pf_offset(u32 i) { - return ADF_4XXX_VM2PF_OFFSET(i); + return ADF_GEN4_VM2PF_OFFSET(i); } static void adf_gen4_enable_vf2pf_interrupts(void __iomem *pmisc_addr, u32 vf_mask) diff --git a/drivers/crypto/intel/qat/qat_common/adf_gen4_tl.c b/drivers/crypto/intel/qat/qat_common/adf_gen4_tl.c index 7fc7a77f6aed..c7ad8cf07863 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_gen4_tl.c +++ b/drivers/crypto/intel/qat/qat_common/adf_gen4_tl.c @@ -149,5 +149,6 @@ void adf_gen4_init_tl_data(struct adf_tl_hw_data *tl_data) tl_data->sl_exec_counters = sl_exec_counters; tl_data->rp_counters = rp_counters; tl_data->num_rp_counters = ARRAY_SIZE(rp_counters); + tl_data->max_sl_cnt = ADF_GEN4_TL_MAX_SLICES_PER_TYPE; } EXPORT_SYMBOL_GPL(adf_gen4_init_tl_data); diff --git a/drivers/crypto/intel/qat/qat_common/adf_gen4_vf_mig.c b/drivers/crypto/intel/qat/qat_common/adf_gen4_vf_mig.c new file mode 100644 index 000000000000..a62eb5e8dbe6 --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/adf_gen4_vf_mig.c @@ -0,0 +1,1010 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright(c) 2024 Intel Corporation */ +#include <linux/delay.h> +#include <linux/dev_printk.h> +#include <linux/kernel.h> +#include <linux/slab.h> +#include <linux/string.h> +#include <linux/types.h> +#include <asm/errno.h> + +#include "adf_accel_devices.h" +#include "adf_common_drv.h" +#include "adf_gen4_hw_data.h" +#include "adf_gen4_pfvf.h" +#include "adf_pfvf_utils.h" +#include "adf_mstate_mgr.h" +#include "adf_gen4_vf_mig.h" + +#define ADF_GEN4_VF_MSTATE_SIZE 4096 +#define ADF_GEN4_PFVF_RSP_TIMEOUT_US 5000 + +static int adf_gen4_vfmig_save_setup(struct qat_mig_dev *mdev); +static int adf_gen4_vfmig_load_setup(struct qat_mig_dev *mdev, int len); + +static int adf_gen4_vfmig_init_device(struct qat_mig_dev *mdev) +{ + u8 *state; + + state = kmalloc(ADF_GEN4_VF_MSTATE_SIZE, GFP_KERNEL); + if (!state) + return -ENOMEM; + + mdev->state = state; + mdev->state_size = ADF_GEN4_VF_MSTATE_SIZE; + mdev->setup_size = 0; + mdev->remote_setup_size = 0; + + return 0; +} + +static void adf_gen4_vfmig_cleanup_device(struct qat_mig_dev *mdev) +{ + kfree(mdev->state); + mdev->state = NULL; +} + +static void adf_gen4_vfmig_reset_device(struct qat_mig_dev *mdev) +{ + mdev->setup_size = 0; + mdev->remote_setup_size = 0; +} + +static int adf_gen4_vfmig_open_device(struct qat_mig_dev *mdev) +{ + struct adf_accel_dev *accel_dev = mdev->parent_accel_dev; + struct adf_accel_vf_info *vf_info; + struct adf_gen4_vfmig *vfmig; + + vf_info = &accel_dev->pf.vf_info[mdev->vf_id]; + + vfmig = kzalloc(sizeof(*vfmig), GFP_KERNEL); + if (!vfmig) + return -ENOMEM; + + vfmig->mstate_mgr = adf_mstate_mgr_new(mdev->state, mdev->state_size); + if (!vfmig->mstate_mgr) { + kfree(vfmig); + return -ENOMEM; + } + vf_info->mig_priv = vfmig; + mdev->setup_size = 0; + mdev->remote_setup_size = 0; + + return 0; +} + +static void adf_gen4_vfmig_close_device(struct qat_mig_dev *mdev) +{ + struct adf_accel_dev *accel_dev = mdev->parent_accel_dev; + struct adf_accel_vf_info *vf_info; + struct adf_gen4_vfmig *vfmig; + + vf_info = &accel_dev->pf.vf_info[mdev->vf_id]; + if (vf_info->mig_priv) { + vfmig = vf_info->mig_priv; + adf_mstate_mgr_destroy(vfmig->mstate_mgr); + kfree(vfmig); + vf_info->mig_priv = NULL; + } +} + +static int adf_gen4_vfmig_suspend_device(struct qat_mig_dev *mdev) +{ + struct adf_accel_dev *accel_dev = mdev->parent_accel_dev; + struct adf_hw_device_data *hw_data = accel_dev->hw_device; + struct adf_accel_vf_info *vf_info; + struct adf_gen4_vfmig *vf_mig; + u32 vf_nr = mdev->vf_id; + int ret, i; + + vf_info = &accel_dev->pf.vf_info[vf_nr]; + vf_mig = vf_info->mig_priv; + + /* Stop all inflight jobs */ + for (i = 0; i < hw_data->num_banks_per_vf; i++) { + u32 pf_bank_nr = i + vf_nr * hw_data->num_banks_per_vf; + + ret = adf_gen4_bank_drain_start(accel_dev, pf_bank_nr, + ADF_RPRESET_POLL_TIMEOUT_US); + if (ret) { + dev_err(&GET_DEV(accel_dev), + "Failed to drain bank %d for vf_nr %d\n", i, + vf_nr); + return ret; + } + vf_mig->bank_stopped[i] = true; + + adf_gen4_bank_quiesce_coal_timer(accel_dev, pf_bank_nr, + ADF_COALESCED_POLL_TIMEOUT_US); + } + + return 0; +} + +static int adf_gen4_vfmig_resume_device(struct qat_mig_dev *mdev) +{ + struct adf_accel_dev *accel_dev = mdev->parent_accel_dev; + struct adf_hw_device_data *hw_data = accel_dev->hw_device; + struct adf_accel_vf_info *vf_info; + struct adf_gen4_vfmig *vf_mig; + u32 vf_nr = mdev->vf_id; + int i; + + vf_info = &accel_dev->pf.vf_info[vf_nr]; + vf_mig = vf_info->mig_priv; + + for (i = 0; i < hw_data->num_banks_per_vf; i++) { + u32 pf_bank_nr = i + vf_nr * hw_data->num_banks_per_vf; + + if (vf_mig->bank_stopped[i]) { + adf_gen4_bank_drain_finish(accel_dev, pf_bank_nr); + vf_mig->bank_stopped[i] = false; + } + } + + return 0; +} + +struct adf_vf_bank_info { + struct adf_accel_dev *accel_dev; + u32 vf_nr; + u32 bank_nr; +}; + +struct mig_user_sla { + enum adf_base_services srv; + u64 rp_mask; + u32 cir; + u32 pir; +}; + +static int adf_mstate_sla_check(struct adf_mstate_mgr *sub_mgr, u8 *src_buf, + u32 src_size, void *opaque) +{ + struct adf_mstate_vreginfo _sinfo = { src_buf, src_size }; + struct adf_mstate_vreginfo *sinfo = &_sinfo, *dinfo = opaque; + u32 src_sla_cnt = sinfo->size / sizeof(struct mig_user_sla); + u32 dst_sla_cnt = dinfo->size / sizeof(struct mig_user_sla); + struct mig_user_sla *src_slas = sinfo->addr; + struct mig_user_sla *dst_slas = dinfo->addr; + int i, j; + + for (i = 0; i < src_sla_cnt; i++) { + for (j = 0; j < dst_sla_cnt; j++) { + if (src_slas[i].srv != dst_slas[j].srv || + src_slas[i].rp_mask != dst_slas[j].rp_mask) + continue; + + if (src_slas[i].cir > dst_slas[j].cir || + src_slas[i].pir > dst_slas[j].pir) { + pr_err("QAT: DST VF rate limiting mismatch.\n"); + return -EINVAL; + } + break; + } + + if (j == dst_sla_cnt) { + pr_err("QAT: SRC VF rate limiting mismatch - SRC srv %d and rp_mask 0x%llx.\n", + src_slas[i].srv, src_slas[i].rp_mask); + return -EINVAL; + } + } + + return 0; +} + +static inline int adf_mstate_check_cap_size(u32 src_sz, u32 dst_sz, u32 max_sz) +{ + if (src_sz > max_sz || dst_sz > max_sz) + return -EINVAL; + else + return 0; +} + +static int adf_mstate_compatver_check(struct adf_mstate_mgr *sub_mgr, + u8 *src_buf, u32 src_sz, void *opaque) +{ + struct adf_mstate_vreginfo *info = opaque; + u8 compat = 0; + u8 *pcompat; + + if (src_sz != info->size) { + pr_debug("QAT: State mismatch (compat version size), current %u, expected %u\n", + src_sz, info->size); + return -EINVAL; + } + + memcpy(info->addr, src_buf, info->size); + pcompat = info->addr; + if (*pcompat == 0) { + pr_warn("QAT: Unable to determine the version of VF\n"); + return 0; + } + + compat = adf_vf_compat_checker(*pcompat); + if (compat == ADF_PF2VF_VF_INCOMPATIBLE) { + pr_debug("QAT: SRC VF driver (ver=%u) is incompatible with DST PF driver (ver=%u)\n", + *pcompat, ADF_PFVF_COMPAT_THIS_VERSION); + return -EINVAL; + } + + if (compat == ADF_PF2VF_VF_COMPAT_UNKNOWN) + pr_debug("QAT: SRC VF driver (ver=%u) is newer than DST PF driver (ver=%u)\n", + *pcompat, ADF_PFVF_COMPAT_THIS_VERSION); + + return 0; +} + +/* + * adf_mstate_capmask_compare() - compare QAT device capability mask + * @sinfo: Pointer to source capability info + * @dinfo: Pointer to target capability info + * + * This function compares the capability mask between source VF and target VF + * + * Returns: 0 if target capability mask is identical to source capability mask, + * 1 if target mask can represent all the capabilities represented by source mask, + * -1 if target mask can't represent all the capabilities represented by source + * mask. + */ +static int adf_mstate_capmask_compare(struct adf_mstate_vreginfo *sinfo, + struct adf_mstate_vreginfo *dinfo) +{ + u64 src = 0, dst = 0; + + if (adf_mstate_check_cap_size(sinfo->size, dinfo->size, sizeof(u64))) { + pr_debug("QAT: Unexpected capability size %u %u %zu\n", + sinfo->size, dinfo->size, sizeof(u64)); + return -1; + } + + memcpy(&src, sinfo->addr, sinfo->size); + memcpy(&dst, dinfo->addr, dinfo->size); + + pr_debug("QAT: Check cap compatibility of cap %llu %llu\n", src, dst); + + if (src == dst) + return 0; + + if ((src | dst) == dst) + return 1; + + return -1; +} + +static int adf_mstate_capmask_superset(struct adf_mstate_mgr *sub_mgr, u8 *buf, + u32 size, void *opa) +{ + struct adf_mstate_vreginfo sinfo = { buf, size }; + + if (adf_mstate_capmask_compare(&sinfo, opa) >= 0) + return 0; + + return -EINVAL; +} + +static int adf_mstate_capmask_equal(struct adf_mstate_mgr *sub_mgr, u8 *buf, + u32 size, void *opa) +{ + struct adf_mstate_vreginfo sinfo = { buf, size }; + + if (adf_mstate_capmask_compare(&sinfo, opa) == 0) + return 0; + + return -EINVAL; +} + +static int adf_mstate_set_vreg(struct adf_mstate_mgr *sub_mgr, u8 *buf, + u32 size, void *opa) +{ + struct adf_mstate_vreginfo *info = opa; + + if (size != info->size) { + pr_debug("QAT: Unexpected cap size %u %u\n", size, info->size); + return -EINVAL; + } + memcpy(info->addr, buf, info->size); + + return 0; +} + +static u32 adf_gen4_vfmig_get_slas(struct adf_accel_dev *accel_dev, u32 vf_nr, + struct mig_user_sla *pmig_slas) +{ + struct adf_hw_device_data *hw_data = accel_dev->hw_device; + struct adf_rl *rl_data = accel_dev->rate_limiting; + struct rl_sla **sla_type_arr = NULL; + u64 rp_mask, rp_index; + u32 max_num_sla; + u32 sla_cnt = 0; + int i, j; + + if (!accel_dev->rate_limiting) + return 0; + + rp_index = vf_nr * hw_data->num_banks_per_vf; + max_num_sla = adf_rl_get_sla_arr_of_type(rl_data, RL_LEAF, &sla_type_arr); + + for (i = 0; i < max_num_sla; i++) { + if (!sla_type_arr[i]) + continue; + + rp_mask = 0; + for (j = 0; j < sla_type_arr[i]->ring_pairs_cnt; j++) + rp_mask |= BIT(sla_type_arr[i]->ring_pairs_ids[j]); + + if (rp_mask & GENMASK_ULL(rp_index + 3, rp_index)) { + pmig_slas->rp_mask = rp_mask; + pmig_slas->cir = sla_type_arr[i]->cir; + pmig_slas->pir = sla_type_arr[i]->pir; + pmig_slas->srv = sla_type_arr[i]->srv; + pmig_slas++; + sla_cnt++; + } + } + + return sla_cnt; +} + +static int adf_gen4_vfmig_load_etr_regs(struct adf_mstate_mgr *sub_mgr, + u8 *state, u32 size, void *opa) +{ + struct adf_vf_bank_info *vf_bank_info = opa; + struct adf_accel_dev *accel_dev = vf_bank_info->accel_dev; + struct adf_hw_device_data *hw_data = accel_dev->hw_device; + u32 pf_bank_nr; + int ret; + + pf_bank_nr = vf_bank_info->bank_nr + vf_bank_info->vf_nr * hw_data->num_banks_per_vf; + ret = hw_data->bank_state_restore(accel_dev, pf_bank_nr, + (struct bank_state *)state); + if (ret) { + dev_err(&GET_DEV(accel_dev), + "Failed to load regs for vf%d bank%d\n", + vf_bank_info->vf_nr, vf_bank_info->bank_nr); + return ret; + } + + return 0; +} + +static int adf_gen4_vfmig_load_etr_bank(struct adf_accel_dev *accel_dev, + u32 vf_nr, u32 bank_nr, + struct adf_mstate_mgr *mstate_mgr) +{ + struct adf_vf_bank_info vf_bank_info = {accel_dev, vf_nr, bank_nr}; + struct adf_mstate_sect_h *subsec, *l2_subsec; + struct adf_mstate_mgr sub_sects_mgr; + char bank_ids[ADF_MSTATE_ID_LEN]; + + snprintf(bank_ids, sizeof(bank_ids), ADF_MSTATE_BANK_IDX_IDS "%x", bank_nr); + subsec = adf_mstate_sect_lookup(mstate_mgr, bank_ids, NULL, NULL); + if (!subsec) { + dev_err(&GET_DEV(accel_dev), + "Failed to lookup sec %s for vf%d bank%d\n", + ADF_MSTATE_BANK_IDX_IDS, vf_nr, bank_nr); + return -EINVAL; + } + + adf_mstate_mgr_init_from_psect(&sub_sects_mgr, subsec); + l2_subsec = adf_mstate_sect_lookup(&sub_sects_mgr, ADF_MSTATE_ETR_REGS_IDS, + adf_gen4_vfmig_load_etr_regs, + &vf_bank_info); + if (!l2_subsec) { + dev_err(&GET_DEV(accel_dev), + "Failed to add sec %s for vf%d bank%d\n", + ADF_MSTATE_ETR_REGS_IDS, vf_nr, bank_nr); + return -EINVAL; + } + + return 0; +} + +static int adf_gen4_vfmig_load_etr(struct adf_accel_dev *accel_dev, u32 vf_nr) +{ + struct adf_accel_vf_info *vf_info = &accel_dev->pf.vf_info[vf_nr]; + struct adf_hw_device_data *hw_data = accel_dev->hw_device; + struct adf_gen4_vfmig *vfmig = vf_info->mig_priv; + struct adf_mstate_mgr *mstate_mgr = vfmig->mstate_mgr; + struct adf_mstate_mgr sub_sects_mgr; + struct adf_mstate_sect_h *subsec; + int ret, i; + + subsec = adf_mstate_sect_lookup(mstate_mgr, ADF_MSTATE_ETRB_IDS, NULL, + NULL); + if (!subsec) { + dev_err(&GET_DEV(accel_dev), "Failed to load sec %s\n", + ADF_MSTATE_ETRB_IDS); + return -EINVAL; + } + + adf_mstate_mgr_init_from_psect(&sub_sects_mgr, subsec); + for (i = 0; i < hw_data->num_banks_per_vf; i++) { + ret = adf_gen4_vfmig_load_etr_bank(accel_dev, vf_nr, i, + &sub_sects_mgr); + if (ret) + return ret; + } + + return 0; +} + +static int adf_gen4_vfmig_load_misc(struct adf_accel_dev *accel_dev, u32 vf_nr) +{ + struct adf_accel_vf_info *vf_info = &accel_dev->pf.vf_info[vf_nr]; + struct adf_gen4_vfmig *vfmig = vf_info->mig_priv; + void __iomem *csr = adf_get_pmisc_base(accel_dev); + struct adf_mstate_mgr *mstate_mgr = vfmig->mstate_mgr; + struct adf_mstate_sect_h *subsec, *l2_subsec; + struct adf_mstate_mgr sub_sects_mgr; + struct { + char *id; + u64 ofs; + } misc_states[] = { + {ADF_MSTATE_VINTMSK_IDS, ADF_GEN4_VINTMSK_OFFSET(vf_nr)}, + {ADF_MSTATE_VINTMSK_PF2VM_IDS, ADF_GEN4_VINTMSKPF2VM_OFFSET(vf_nr)}, + {ADF_MSTATE_PF2VM_IDS, ADF_GEN4_PF2VM_OFFSET(vf_nr)}, + {ADF_MSTATE_VM2PF_IDS, ADF_GEN4_VM2PF_OFFSET(vf_nr)}, + }; + int i; + + subsec = adf_mstate_sect_lookup(mstate_mgr, ADF_MSTATE_MISCB_IDS, NULL, + NULL); + if (!subsec) { + dev_err(&GET_DEV(accel_dev), "Failed to load sec %s\n", + ADF_MSTATE_MISCB_IDS); + return -EINVAL; + } + + adf_mstate_mgr_init_from_psect(&sub_sects_mgr, subsec); + for (i = 0; i < ARRAY_SIZE(misc_states); i++) { + struct adf_mstate_vreginfo info; + u32 regv; + + info.addr = ®v; + info.size = sizeof(regv); + l2_subsec = adf_mstate_sect_lookup(&sub_sects_mgr, + misc_states[i].id, + adf_mstate_set_vreg, + &info); + if (!l2_subsec) { + dev_err(&GET_DEV(accel_dev), + "Failed to load sec %s\n", misc_states[i].id); + return -EINVAL; + } + ADF_CSR_WR(csr, misc_states[i].ofs, regv); + } + + return 0; +} + +static int adf_gen4_vfmig_load_generic(struct adf_accel_dev *accel_dev, u32 vf_nr) +{ + struct adf_accel_vf_info *vf_info = &accel_dev->pf.vf_info[vf_nr]; + struct mig_user_sla dst_slas[RL_RP_CNT_PER_LEAF_MAX] = { }; + struct adf_gen4_vfmig *vfmig = vf_info->mig_priv; + struct adf_mstate_mgr *mstate_mgr = vfmig->mstate_mgr; + struct adf_mstate_sect_h *subsec, *l2_subsec; + struct adf_mstate_mgr sub_sects_mgr; + u32 dst_sla_cnt; + struct { + char *id; + int (*action)(struct adf_mstate_mgr *sub_mgr, u8 *buf, u32 size, void *opa); + struct adf_mstate_vreginfo info; + } gen_states[] = { + {ADF_MSTATE_IOV_INIT_IDS, adf_mstate_set_vreg, + {&vf_info->init, sizeof(vf_info->init)}}, + {ADF_MSTATE_COMPAT_VER_IDS, adf_mstate_compatver_check, + {&vf_info->vf_compat_ver, sizeof(vf_info->vf_compat_ver)}}, + {ADF_MSTATE_SLA_IDS, adf_mstate_sla_check, {dst_slas, 0}}, + }; + int i; + + subsec = adf_mstate_sect_lookup(mstate_mgr, ADF_MSTATE_GEN_IDS, NULL, NULL); + if (!subsec) { + dev_err(&GET_DEV(accel_dev), "Failed to load sec %s\n", + ADF_MSTATE_GEN_IDS); + return -EINVAL; + } + + adf_mstate_mgr_init_from_psect(&sub_sects_mgr, subsec); + for (i = 0; i < ARRAY_SIZE(gen_states); i++) { + if (gen_states[i].info.addr == dst_slas) { + dst_sla_cnt = adf_gen4_vfmig_get_slas(accel_dev, vf_nr, dst_slas); + gen_states[i].info.size = dst_sla_cnt * sizeof(struct mig_user_sla); + } + + l2_subsec = adf_mstate_sect_lookup(&sub_sects_mgr, + gen_states[i].id, + gen_states[i].action, + &gen_states[i].info); + if (!l2_subsec) { + dev_err(&GET_DEV(accel_dev), "Failed to load sec %s\n", + gen_states[i].id); + return -EINVAL; + } + } + + return 0; +} + +static int adf_gen4_vfmig_load_config(struct adf_accel_dev *accel_dev, u32 vf_nr) +{ + struct adf_accel_vf_info *vf_info = &accel_dev->pf.vf_info[vf_nr]; + struct adf_hw_device_data *hw_data = accel_dev->hw_device; + struct adf_gen4_vfmig *vfmig = vf_info->mig_priv; + struct adf_mstate_mgr *mstate_mgr = vfmig->mstate_mgr; + struct adf_mstate_sect_h *subsec, *l2_subsec; + struct adf_mstate_mgr sub_sects_mgr; + struct { + char *id; + int (*action)(struct adf_mstate_mgr *sub_mgr, u8 *buf, u32 size, void *opa); + struct adf_mstate_vreginfo info; + } setups[] = { + {ADF_MSTATE_GEN_CAP_IDS, adf_mstate_capmask_superset, + {&hw_data->accel_capabilities_mask, sizeof(hw_data->accel_capabilities_mask)}}, + {ADF_MSTATE_GEN_SVCMAP_IDS, adf_mstate_capmask_equal, + {&hw_data->ring_to_svc_map, sizeof(hw_data->ring_to_svc_map)}}, + {ADF_MSTATE_GEN_EXTDC_IDS, adf_mstate_capmask_superset, + {&hw_data->extended_dc_capabilities, sizeof(hw_data->extended_dc_capabilities)}}, + }; + int i; + + subsec = adf_mstate_sect_lookup(mstate_mgr, ADF_MSTATE_CONFIG_IDS, NULL, NULL); + if (!subsec) { + dev_err(&GET_DEV(accel_dev), "Failed to load sec %s\n", + ADF_MSTATE_CONFIG_IDS); + return -EINVAL; + } + + adf_mstate_mgr_init_from_psect(&sub_sects_mgr, subsec); + for (i = 0; i < ARRAY_SIZE(setups); i++) { + l2_subsec = adf_mstate_sect_lookup(&sub_sects_mgr, setups[i].id, + setups[i].action, &setups[i].info); + if (!l2_subsec) { + dev_err(&GET_DEV(accel_dev), "Failed to load sec %s\n", + setups[i].id); + return -EINVAL; + } + } + + return 0; +} + +static int adf_gen4_vfmig_save_etr_regs(struct adf_mstate_mgr *subs, u8 *state, + u32 size, void *opa) +{ + struct adf_vf_bank_info *vf_bank_info = opa; + struct adf_accel_dev *accel_dev = vf_bank_info->accel_dev; + struct adf_hw_device_data *hw_data = accel_dev->hw_device; + u32 pf_bank_nr; + int ret; + + pf_bank_nr = vf_bank_info->bank_nr; + pf_bank_nr += vf_bank_info->vf_nr * hw_data->num_banks_per_vf; + + ret = hw_data->bank_state_save(accel_dev, pf_bank_nr, + (struct bank_state *)state); + if (ret) { + dev_err(&GET_DEV(accel_dev), + "Failed to save regs for vf%d bank%d\n", + vf_bank_info->vf_nr, vf_bank_info->bank_nr); + return ret; + } + + return sizeof(struct bank_state); +} + +static int adf_gen4_vfmig_save_etr_bank(struct adf_accel_dev *accel_dev, + u32 vf_nr, u32 bank_nr, + struct adf_mstate_mgr *mstate_mgr) +{ + struct adf_mstate_sect_h *subsec, *l2_subsec; + struct adf_vf_bank_info vf_bank_info; + struct adf_mstate_mgr sub_sects_mgr; + char bank_ids[ADF_MSTATE_ID_LEN]; + + snprintf(bank_ids, sizeof(bank_ids), ADF_MSTATE_BANK_IDX_IDS "%x", bank_nr); + + subsec = adf_mstate_sect_add(mstate_mgr, bank_ids, NULL, NULL); + if (!subsec) { + dev_err(&GET_DEV(accel_dev), + "Failed to add sec %s for vf%d bank%d\n", + ADF_MSTATE_BANK_IDX_IDS, vf_nr, bank_nr); + return -EINVAL; + } + + adf_mstate_mgr_init_from_parent(&sub_sects_mgr, mstate_mgr); + vf_bank_info.accel_dev = accel_dev; + vf_bank_info.vf_nr = vf_nr; + vf_bank_info.bank_nr = bank_nr; + l2_subsec = adf_mstate_sect_add(&sub_sects_mgr, ADF_MSTATE_ETR_REGS_IDS, + adf_gen4_vfmig_save_etr_regs, + &vf_bank_info); + if (!l2_subsec) { + dev_err(&GET_DEV(accel_dev), + "Failed to add sec %s for vf%d bank%d\n", + ADF_MSTATE_ETR_REGS_IDS, vf_nr, bank_nr); + return -EINVAL; + } + adf_mstate_sect_update(mstate_mgr, &sub_sects_mgr, subsec); + + return 0; +} + +static int adf_gen4_vfmig_save_etr(struct adf_accel_dev *accel_dev, u32 vf_nr) +{ + struct adf_accel_vf_info *vf_info = &accel_dev->pf.vf_info[vf_nr]; + struct adf_hw_device_data *hw_data = accel_dev->hw_device; + struct adf_gen4_vfmig *vfmig = vf_info->mig_priv; + struct adf_mstate_mgr *mstate_mgr = vfmig->mstate_mgr; + struct adf_mstate_mgr sub_sects_mgr; + struct adf_mstate_sect_h *subsec; + int ret, i; + + subsec = adf_mstate_sect_add(mstate_mgr, ADF_MSTATE_ETRB_IDS, NULL, NULL); + if (!subsec) { + dev_err(&GET_DEV(accel_dev), "Failed to add sec %s\n", + ADF_MSTATE_ETRB_IDS); + return -EINVAL; + } + + adf_mstate_mgr_init_from_parent(&sub_sects_mgr, mstate_mgr); + for (i = 0; i < hw_data->num_banks_per_vf; i++) { + ret = adf_gen4_vfmig_save_etr_bank(accel_dev, vf_nr, i, + &sub_sects_mgr); + if (ret) + return ret; + } + adf_mstate_sect_update(mstate_mgr, &sub_sects_mgr, subsec); + + return 0; +} + +static int adf_gen4_vfmig_save_misc(struct adf_accel_dev *accel_dev, u32 vf_nr) +{ + struct adf_accel_vf_info *vf_info = &accel_dev->pf.vf_info[vf_nr]; + struct adf_gen4_vfmig *vfmig = vf_info->mig_priv; + struct adf_mstate_mgr *mstate_mgr = vfmig->mstate_mgr; + void __iomem *csr = adf_get_pmisc_base(accel_dev); + struct adf_mstate_sect_h *subsec, *l2_subsec; + struct adf_mstate_mgr sub_sects_mgr; + struct { + char *id; + u64 offset; + } misc_states[] = { + {ADF_MSTATE_VINTSRC_IDS, ADF_GEN4_VINTSOU_OFFSET(vf_nr)}, + {ADF_MSTATE_VINTMSK_IDS, ADF_GEN4_VINTMSK_OFFSET(vf_nr)}, + {ADF_MSTATE_VINTSRC_PF2VM_IDS, ADF_GEN4_VINTSOUPF2VM_OFFSET(vf_nr)}, + {ADF_MSTATE_VINTMSK_PF2VM_IDS, ADF_GEN4_VINTMSKPF2VM_OFFSET(vf_nr)}, + {ADF_MSTATE_PF2VM_IDS, ADF_GEN4_PF2VM_OFFSET(vf_nr)}, + {ADF_MSTATE_VM2PF_IDS, ADF_GEN4_VM2PF_OFFSET(vf_nr)}, + }; + ktime_t time_exp; + int i; + + subsec = adf_mstate_sect_add(mstate_mgr, ADF_MSTATE_MISCB_IDS, NULL, NULL); + if (!subsec) { + dev_err(&GET_DEV(accel_dev), "Failed to add sec %s\n", + ADF_MSTATE_MISCB_IDS); + return -EINVAL; + } + + time_exp = ktime_add_us(ktime_get(), ADF_GEN4_PFVF_RSP_TIMEOUT_US); + while (!mutex_trylock(&vf_info->pfvf_mig_lock)) { + if (ktime_after(ktime_get(), time_exp)) { + dev_err(&GET_DEV(accel_dev), "Failed to get pfvf mig lock\n"); + return -ETIMEDOUT; + } + usleep_range(500, 1000); + } + + adf_mstate_mgr_init_from_parent(&sub_sects_mgr, mstate_mgr); + for (i = 0; i < ARRAY_SIZE(misc_states); i++) { + struct adf_mstate_vreginfo info; + u32 regv; + + info.addr = ®v; + info.size = sizeof(regv); + regv = ADF_CSR_RD(csr, misc_states[i].offset); + + l2_subsec = adf_mstate_sect_add_vreg(&sub_sects_mgr, + misc_states[i].id, + &info); + if (!l2_subsec) { + dev_err(&GET_DEV(accel_dev), "Failed to add sec %s\n", + misc_states[i].id); + mutex_unlock(&vf_info->pfvf_mig_lock); + return -EINVAL; + } + } + + mutex_unlock(&vf_info->pfvf_mig_lock); + adf_mstate_sect_update(mstate_mgr, &sub_sects_mgr, subsec); + + return 0; +} + +static int adf_gen4_vfmig_save_generic(struct adf_accel_dev *accel_dev, u32 vf_nr) +{ + struct adf_accel_vf_info *vf_info = &accel_dev->pf.vf_info[vf_nr]; + struct adf_gen4_vfmig *vfmig = vf_info->mig_priv; + struct adf_mstate_mgr *mstate_mgr = vfmig->mstate_mgr; + struct adf_mstate_mgr sub_sects_mgr; + struct adf_mstate_sect_h *subsec, *l2_subsec; + struct mig_user_sla src_slas[RL_RP_CNT_PER_LEAF_MAX] = { }; + u32 src_sla_cnt; + struct { + char *id; + struct adf_mstate_vreginfo info; + } gen_states[] = { + {ADF_MSTATE_IOV_INIT_IDS, + {&vf_info->init, sizeof(vf_info->init)}}, + {ADF_MSTATE_COMPAT_VER_IDS, + {&vf_info->vf_compat_ver, sizeof(vf_info->vf_compat_ver)}}, + {ADF_MSTATE_SLA_IDS, {src_slas, 0}}, + }; + int i; + + subsec = adf_mstate_sect_add(mstate_mgr, ADF_MSTATE_GEN_IDS, NULL, NULL); + if (!subsec) { + dev_err(&GET_DEV(accel_dev), "Failed to add sec %s\n", + ADF_MSTATE_GEN_IDS); + return -EINVAL; + } + + adf_mstate_mgr_init_from_parent(&sub_sects_mgr, mstate_mgr); + for (i = 0; i < ARRAY_SIZE(gen_states); i++) { + if (gen_states[i].info.addr == src_slas) { + src_sla_cnt = adf_gen4_vfmig_get_slas(accel_dev, vf_nr, src_slas); + gen_states[i].info.size = src_sla_cnt * sizeof(struct mig_user_sla); + } + + l2_subsec = adf_mstate_sect_add_vreg(&sub_sects_mgr, + gen_states[i].id, + &gen_states[i].info); + if (!l2_subsec) { + dev_err(&GET_DEV(accel_dev), "Failed to add sec %s\n", + gen_states[i].id); + return -EINVAL; + } + } + adf_mstate_sect_update(mstate_mgr, &sub_sects_mgr, subsec); + + return 0; +} + +static int adf_gen4_vfmig_save_config(struct adf_accel_dev *accel_dev, u32 vf_nr) +{ + struct adf_accel_vf_info *vf_info = &accel_dev->pf.vf_info[vf_nr]; + struct adf_hw_device_data *hw_data = accel_dev->hw_device; + struct adf_gen4_vfmig *vfmig = vf_info->mig_priv; + struct adf_mstate_mgr *mstate_mgr = vfmig->mstate_mgr; + struct adf_mstate_mgr sub_sects_mgr; + struct adf_mstate_sect_h *subsec, *l2_subsec; + struct { + char *id; + struct adf_mstate_vreginfo info; + } setups[] = { + {ADF_MSTATE_GEN_CAP_IDS, + {&hw_data->accel_capabilities_mask, sizeof(hw_data->accel_capabilities_mask)}}, + {ADF_MSTATE_GEN_SVCMAP_IDS, + {&hw_data->ring_to_svc_map, sizeof(hw_data->ring_to_svc_map)}}, + {ADF_MSTATE_GEN_EXTDC_IDS, + {&hw_data->extended_dc_capabilities, sizeof(hw_data->extended_dc_capabilities)}}, + }; + int i; + + subsec = adf_mstate_sect_add(mstate_mgr, ADF_MSTATE_CONFIG_IDS, NULL, NULL); + if (!subsec) { + dev_err(&GET_DEV(accel_dev), "Failed to add sec %s\n", + ADF_MSTATE_CONFIG_IDS); + return -EINVAL; + } + + adf_mstate_mgr_init_from_parent(&sub_sects_mgr, mstate_mgr); + for (i = 0; i < ARRAY_SIZE(setups); i++) { + l2_subsec = adf_mstate_sect_add_vreg(&sub_sects_mgr, setups[i].id, + &setups[i].info); + if (!l2_subsec) { + dev_err(&GET_DEV(accel_dev), "Failed to add sec %s\n", + setups[i].id); + return -EINVAL; + } + } + adf_mstate_sect_update(mstate_mgr, &sub_sects_mgr, subsec); + + return 0; +} + +static int adf_gen4_vfmig_save_state(struct qat_mig_dev *mdev) +{ + struct adf_accel_dev *accel_dev = mdev->parent_accel_dev; + struct adf_accel_vf_info *vf_info; + struct adf_gen4_vfmig *vfmig; + u32 vf_nr = mdev->vf_id; + int ret; + + vf_info = &accel_dev->pf.vf_info[vf_nr]; + vfmig = vf_info->mig_priv; + + ret = adf_gen4_vfmig_save_setup(mdev); + if (ret) { + dev_err(&GET_DEV(accel_dev), + "Failed to save setup for vf_nr %d\n", vf_nr); + return ret; + } + + adf_mstate_mgr_init(vfmig->mstate_mgr, mdev->state + mdev->setup_size, + mdev->state_size - mdev->setup_size); + if (!adf_mstate_preamble_add(vfmig->mstate_mgr)) + return -EINVAL; + + ret = adf_gen4_vfmig_save_generic(accel_dev, vf_nr); + if (ret) { + dev_err(&GET_DEV(accel_dev), + "Failed to save generic state for vf_nr %d\n", vf_nr); + return ret; + } + + ret = adf_gen4_vfmig_save_misc(accel_dev, vf_nr); + if (ret) { + dev_err(&GET_DEV(accel_dev), + "Failed to save misc bar state for vf_nr %d\n", vf_nr); + return ret; + } + + ret = adf_gen4_vfmig_save_etr(accel_dev, vf_nr); + if (ret) { + dev_err(&GET_DEV(accel_dev), + "Failed to save etr bar state for vf_nr %d\n", vf_nr); + return ret; + } + + adf_mstate_preamble_update(vfmig->mstate_mgr); + + return 0; +} + +static int adf_gen4_vfmig_load_state(struct qat_mig_dev *mdev) +{ + struct adf_accel_dev *accel_dev = mdev->parent_accel_dev; + struct adf_accel_vf_info *vf_info; + struct adf_gen4_vfmig *vfmig; + u32 vf_nr = mdev->vf_id; + int ret; + + vf_info = &accel_dev->pf.vf_info[vf_nr]; + vfmig = vf_info->mig_priv; + + ret = adf_gen4_vfmig_load_setup(mdev, mdev->state_size); + if (ret) { + dev_err(&GET_DEV(accel_dev), "Failed to load setup for vf_nr %d\n", + vf_nr); + return ret; + } + + ret = adf_mstate_mgr_init_from_remote(vfmig->mstate_mgr, + mdev->state + mdev->remote_setup_size, + mdev->state_size - mdev->remote_setup_size, + NULL, NULL); + if (ret) { + dev_err(&GET_DEV(accel_dev), "Invalid state for vf_nr %d\n", + vf_nr); + return ret; + } + + ret = adf_gen4_vfmig_load_generic(accel_dev, vf_nr); + if (ret) { + dev_err(&GET_DEV(accel_dev), + "Failed to load general state for vf_nr %d\n", vf_nr); + return ret; + } + + ret = adf_gen4_vfmig_load_misc(accel_dev, vf_nr); + if (ret) { + dev_err(&GET_DEV(accel_dev), + "Failed to load misc bar state for vf_nr %d\n", vf_nr); + return ret; + } + + ret = adf_gen4_vfmig_load_etr(accel_dev, vf_nr); + if (ret) { + dev_err(&GET_DEV(accel_dev), + "Failed to load etr bar state for vf_nr %d\n", vf_nr); + return ret; + } + + return 0; +} + +static int adf_gen4_vfmig_save_setup(struct qat_mig_dev *mdev) +{ + struct adf_accel_dev *accel_dev = mdev->parent_accel_dev; + struct adf_accel_vf_info *vf_info; + struct adf_gen4_vfmig *vfmig; + u32 vf_nr = mdev->vf_id; + int ret; + + vf_info = &accel_dev->pf.vf_info[vf_nr]; + vfmig = vf_info->mig_priv; + + if (mdev->setup_size) + return 0; + + adf_mstate_mgr_init(vfmig->mstate_mgr, mdev->state, mdev->state_size); + if (!adf_mstate_preamble_add(vfmig->mstate_mgr)) + return -EINVAL; + + ret = adf_gen4_vfmig_save_config(accel_dev, mdev->vf_id); + if (ret) + return ret; + + adf_mstate_preamble_update(vfmig->mstate_mgr); + mdev->setup_size = adf_mstate_state_size(vfmig->mstate_mgr); + + return 0; +} + +static int adf_gen4_vfmig_load_setup(struct qat_mig_dev *mdev, int len) +{ + struct adf_accel_dev *accel_dev = mdev->parent_accel_dev; + struct adf_accel_vf_info *vf_info; + struct adf_gen4_vfmig *vfmig; + u32 vf_nr = mdev->vf_id; + u32 setup_size; + int ret; + + vf_info = &accel_dev->pf.vf_info[vf_nr]; + vfmig = vf_info->mig_priv; + + if (mdev->remote_setup_size) + return 0; + + if (len < sizeof(struct adf_mstate_preh)) + return -EAGAIN; + + adf_mstate_mgr_init(vfmig->mstate_mgr, mdev->state, mdev->state_size); + setup_size = adf_mstate_state_size_from_remote(vfmig->mstate_mgr); + if (setup_size > mdev->state_size) + return -EINVAL; + + if (len < setup_size) + return -EAGAIN; + + ret = adf_mstate_mgr_init_from_remote(vfmig->mstate_mgr, mdev->state, + setup_size, NULL, NULL); + if (ret) { + dev_err(&GET_DEV(accel_dev), "Invalid setup for vf_nr %d\n", + vf_nr); + return ret; + } + + mdev->remote_setup_size = setup_size; + + ret = adf_gen4_vfmig_load_config(accel_dev, vf_nr); + if (ret) { + dev_err(&GET_DEV(accel_dev), + "Failed to load config for vf_nr %d\n", vf_nr); + return ret; + } + + return 0; +} + +void adf_gen4_init_vf_mig_ops(struct qat_migdev_ops *vfmig_ops) +{ + vfmig_ops->init = adf_gen4_vfmig_init_device; + vfmig_ops->cleanup = adf_gen4_vfmig_cleanup_device; + vfmig_ops->reset = adf_gen4_vfmig_reset_device; + vfmig_ops->open = adf_gen4_vfmig_open_device; + vfmig_ops->close = adf_gen4_vfmig_close_device; + vfmig_ops->suspend = adf_gen4_vfmig_suspend_device; + vfmig_ops->resume = adf_gen4_vfmig_resume_device; + vfmig_ops->save_state = adf_gen4_vfmig_save_state; + vfmig_ops->load_state = adf_gen4_vfmig_load_state; + vfmig_ops->load_setup = adf_gen4_vfmig_load_setup; + vfmig_ops->save_setup = adf_gen4_vfmig_save_setup; +} +EXPORT_SYMBOL_GPL(adf_gen4_init_vf_mig_ops); diff --git a/drivers/crypto/intel/qat/qat_common/adf_gen4_vf_mig.h b/drivers/crypto/intel/qat/qat_common/adf_gen4_vf_mig.h new file mode 100644 index 000000000000..72216d078ee1 --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/adf_gen4_vf_mig.h @@ -0,0 +1,10 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright(c) 2024 Intel Corporation */ +#ifndef ADF_GEN4_VF_MIG_H_ +#define ADF_GEN4_VF_MIG_H_ + +#include "adf_accel_devices.h" + +void adf_gen4_init_vf_mig_ops(struct qat_migdev_ops *vfmig_ops); + +#endif diff --git a/drivers/crypto/intel/qat/qat_common/adf_mstate_mgr.c b/drivers/crypto/intel/qat/qat_common/adf_mstate_mgr.c new file mode 100644 index 000000000000..41cc763a74aa --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/adf_mstate_mgr.c @@ -0,0 +1,318 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright(c) 2024 Intel Corporation */ + +#include <linux/slab.h> +#include <linux/types.h> +#include "adf_mstate_mgr.h" + +#define ADF_MSTATE_MAGIC 0xADF5CAEA +#define ADF_MSTATE_VERSION 0x1 + +struct adf_mstate_sect_h { + u8 id[ADF_MSTATE_ID_LEN]; + u32 size; + u32 sub_sects; + u8 state[]; +}; + +u32 adf_mstate_state_size(struct adf_mstate_mgr *mgr) +{ + return mgr->state - mgr->buf; +} + +static inline u32 adf_mstate_avail_room(struct adf_mstate_mgr *mgr) +{ + return mgr->buf + mgr->size - mgr->state; +} + +void adf_mstate_mgr_init(struct adf_mstate_mgr *mgr, u8 *buf, u32 size) +{ + mgr->buf = buf; + mgr->state = buf; + mgr->size = size; + mgr->n_sects = 0; +}; + +struct adf_mstate_mgr *adf_mstate_mgr_new(u8 *buf, u32 size) +{ + struct adf_mstate_mgr *mgr; + + mgr = kzalloc(sizeof(*mgr), GFP_KERNEL); + if (!mgr) + return NULL; + + adf_mstate_mgr_init(mgr, buf, size); + + return mgr; +} + +void adf_mstate_mgr_destroy(struct adf_mstate_mgr *mgr) +{ + kfree(mgr); +} + +void adf_mstate_mgr_init_from_parent(struct adf_mstate_mgr *mgr, + struct adf_mstate_mgr *p_mgr) +{ + adf_mstate_mgr_init(mgr, p_mgr->state, + p_mgr->size - adf_mstate_state_size(p_mgr)); +} + +void adf_mstate_mgr_init_from_psect(struct adf_mstate_mgr *mgr, + struct adf_mstate_sect_h *p_sect) +{ + adf_mstate_mgr_init(mgr, p_sect->state, p_sect->size); + mgr->n_sects = p_sect->sub_sects; +} + +static void adf_mstate_preamble_init(struct adf_mstate_preh *preamble) +{ + preamble->magic = ADF_MSTATE_MAGIC; + preamble->version = ADF_MSTATE_VERSION; + preamble->preh_len = sizeof(*preamble); + preamble->size = 0; + preamble->n_sects = 0; +} + +/* default preambles checker */ +static int adf_mstate_preamble_def_checker(struct adf_mstate_preh *preamble, + void *opaque) +{ + struct adf_mstate_mgr *mgr = opaque; + + if (preamble->magic != ADF_MSTATE_MAGIC || + preamble->version > ADF_MSTATE_VERSION || + preamble->preh_len > mgr->size) { + pr_debug("QAT: LM - Invalid state (magic=%#x, version=%#x, hlen=%u), state_size=%u\n", + preamble->magic, preamble->version, preamble->preh_len, + mgr->size); + return -EINVAL; + } + + return 0; +} + +struct adf_mstate_preh *adf_mstate_preamble_add(struct adf_mstate_mgr *mgr) +{ + struct adf_mstate_preh *pre = (struct adf_mstate_preh *)mgr->buf; + + if (adf_mstate_avail_room(mgr) < sizeof(*pre)) { + pr_err("QAT: LM - Not enough space for preamble\n"); + return NULL; + } + + adf_mstate_preamble_init(pre); + mgr->state += pre->preh_len; + + return pre; +} + +int adf_mstate_preamble_update(struct adf_mstate_mgr *mgr) +{ + struct adf_mstate_preh *preamble = (struct adf_mstate_preh *)mgr->buf; + + preamble->size = adf_mstate_state_size(mgr) - preamble->preh_len; + preamble->n_sects = mgr->n_sects; + + return 0; +} + +static void adf_mstate_dump_sect(struct adf_mstate_sect_h *sect, + const char *prefix) +{ + pr_debug("QAT: LM - %s QAT state section %s\n", prefix, sect->id); + print_hex_dump_debug("h-", DUMP_PREFIX_OFFSET, 16, 2, sect, + sizeof(*sect), true); + print_hex_dump_debug("s-", DUMP_PREFIX_OFFSET, 16, 2, sect->state, + sect->size, true); +} + +static inline void __adf_mstate_sect_update(struct adf_mstate_mgr *mgr, + struct adf_mstate_sect_h *sect, + u32 size, + u32 n_subsects) +{ + sect->size += size; + sect->sub_sects += n_subsects; + mgr->n_sects++; + mgr->state += sect->size; + + adf_mstate_dump_sect(sect, "Add"); +} + +void adf_mstate_sect_update(struct adf_mstate_mgr *p_mgr, + struct adf_mstate_mgr *curr_mgr, + struct adf_mstate_sect_h *sect) +{ + __adf_mstate_sect_update(p_mgr, sect, adf_mstate_state_size(curr_mgr), + curr_mgr->n_sects); +} + +static struct adf_mstate_sect_h *adf_mstate_sect_add_header(struct adf_mstate_mgr *mgr, + const char *id) +{ + struct adf_mstate_sect_h *sect = (struct adf_mstate_sect_h *)(mgr->state); + + if (adf_mstate_avail_room(mgr) < sizeof(*sect)) { + pr_debug("QAT: LM - Not enough space for header of QAT state sect %s\n", id); + return NULL; + } + + strscpy(sect->id, id, sizeof(sect->id)); + sect->size = 0; + sect->sub_sects = 0; + mgr->state += sizeof(*sect); + + return sect; +} + +struct adf_mstate_sect_h *adf_mstate_sect_add_vreg(struct adf_mstate_mgr *mgr, + const char *id, + struct adf_mstate_vreginfo *info) +{ + struct adf_mstate_sect_h *sect; + + sect = adf_mstate_sect_add_header(mgr, id); + if (!sect) + return NULL; + + if (adf_mstate_avail_room(mgr) < info->size) { + pr_debug("QAT: LM - Not enough space for QAT state sect %s, requires %u\n", + id, info->size); + return NULL; + } + + memcpy(sect->state, info->addr, info->size); + __adf_mstate_sect_update(mgr, sect, info->size, 0); + + return sect; +} + +struct adf_mstate_sect_h *adf_mstate_sect_add(struct adf_mstate_mgr *mgr, + const char *id, + adf_mstate_populate populate, + void *opaque) +{ + struct adf_mstate_mgr sub_sects_mgr; + struct adf_mstate_sect_h *sect; + int avail_room, size; + + sect = adf_mstate_sect_add_header(mgr, id); + if (!sect) + return NULL; + + if (!populate) + return sect; + + avail_room = adf_mstate_avail_room(mgr); + adf_mstate_mgr_init_from_parent(&sub_sects_mgr, mgr); + + size = (*populate)(&sub_sects_mgr, sect->state, avail_room, opaque); + if (size < 0) + return NULL; + + size += adf_mstate_state_size(&sub_sects_mgr); + if (avail_room < size) { + pr_debug("QAT: LM - Not enough space for QAT state sect %s, requires %u\n", + id, size); + return NULL; + } + __adf_mstate_sect_update(mgr, sect, size, sub_sects_mgr.n_sects); + + return sect; +} + +static int adf_mstate_sect_validate(struct adf_mstate_mgr *mgr) +{ + struct adf_mstate_sect_h *start = (struct adf_mstate_sect_h *)mgr->state; + struct adf_mstate_sect_h *sect = start; + u64 end; + int i; + + end = (uintptr_t)mgr->buf + mgr->size; + for (i = 0; i < mgr->n_sects; i++) { + uintptr_t s_start = (uintptr_t)sect->state; + uintptr_t s_end = s_start + sect->size; + + if (s_end < s_start || s_end > end) { + pr_debug("QAT: LM - Corrupted state section (index=%u, size=%u) in state_mgr (size=%u, secs=%u)\n", + i, sect->size, mgr->size, mgr->n_sects); + return -EINVAL; + } + sect = (struct adf_mstate_sect_h *)s_end; + } + + pr_debug("QAT: LM - Scanned section (last child=%s, size=%lu) in state_mgr (size=%u, secs=%u)\n", + start->id, sizeof(struct adf_mstate_sect_h) * (ulong)(sect - start), + mgr->size, mgr->n_sects); + + return 0; +} + +u32 adf_mstate_state_size_from_remote(struct adf_mstate_mgr *mgr) +{ + struct adf_mstate_preh *preh = (struct adf_mstate_preh *)mgr->buf; + + return preh->preh_len + preh->size; +} + +int adf_mstate_mgr_init_from_remote(struct adf_mstate_mgr *mgr, u8 *buf, u32 size, + adf_mstate_preamble_checker pre_checker, + void *opaque) +{ + struct adf_mstate_preh *pre; + int ret; + + adf_mstate_mgr_init(mgr, buf, size); + pre = (struct adf_mstate_preh *)(mgr->buf); + + pr_debug("QAT: LM - Dump state preambles\n"); + print_hex_dump_debug("", DUMP_PREFIX_OFFSET, 16, 2, pre, pre->preh_len, 0); + + if (pre_checker) + ret = (*pre_checker)(pre, opaque); + else + ret = adf_mstate_preamble_def_checker(pre, mgr); + if (ret) + return ret; + + mgr->state = mgr->buf + pre->preh_len; + mgr->n_sects = pre->n_sects; + + return adf_mstate_sect_validate(mgr); +} + +struct adf_mstate_sect_h *adf_mstate_sect_lookup(struct adf_mstate_mgr *mgr, + const char *id, + adf_mstate_action action, + void *opaque) +{ + struct adf_mstate_sect_h *sect = (struct adf_mstate_sect_h *)mgr->state; + struct adf_mstate_mgr sub_sects_mgr; + int i, ret; + + for (i = 0; i < mgr->n_sects; i++) { + if (!strncmp(sect->id, id, sizeof(sect->id))) + goto found; + + sect = (struct adf_mstate_sect_h *)(sect->state + sect->size); + } + + return NULL; + +found: + adf_mstate_dump_sect(sect, "Found"); + + adf_mstate_mgr_init_from_psect(&sub_sects_mgr, sect); + if (sect->sub_sects && adf_mstate_sect_validate(&sub_sects_mgr)) + return NULL; + + if (!action) + return sect; + + ret = (*action)(&sub_sects_mgr, sect->state, sect->size, opaque); + if (ret) + return NULL; + + return sect; +} diff --git a/drivers/crypto/intel/qat/qat_common/adf_mstate_mgr.h b/drivers/crypto/intel/qat/qat_common/adf_mstate_mgr.h new file mode 100644 index 000000000000..81d263a596c5 --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/adf_mstate_mgr.h @@ -0,0 +1,89 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright(c) 2024 Intel Corporation */ + +#ifndef ADF_MSTATE_MGR_H +#define ADF_MSTATE_MGR_H + +#define ADF_MSTATE_ID_LEN 8 + +#define ADF_MSTATE_ETRB_IDS "ETRBAR" +#define ADF_MSTATE_MISCB_IDS "MISCBAR" +#define ADF_MSTATE_EXTB_IDS "EXTBAR" +#define ADF_MSTATE_GEN_IDS "GENER" +#define ADF_MSTATE_CONFIG_IDS "CONFIG" +#define ADF_MSTATE_SECTION_NUM 5 + +#define ADF_MSTATE_BANK_IDX_IDS "bnk" + +#define ADF_MSTATE_ETR_REGS_IDS "mregs" +#define ADF_MSTATE_VINTSRC_IDS "visrc" +#define ADF_MSTATE_VINTMSK_IDS "vimsk" +#define ADF_MSTATE_SLA_IDS "sla" +#define ADF_MSTATE_IOV_INIT_IDS "iovinit" +#define ADF_MSTATE_COMPAT_VER_IDS "compver" +#define ADF_MSTATE_GEN_CAP_IDS "gencap" +#define ADF_MSTATE_GEN_SVCMAP_IDS "svcmap" +#define ADF_MSTATE_GEN_EXTDC_IDS "extdc" +#define ADF_MSTATE_VINTSRC_PF2VM_IDS "vispv" +#define ADF_MSTATE_VINTMSK_PF2VM_IDS "vimpv" +#define ADF_MSTATE_VM2PF_IDS "vm2pf" +#define ADF_MSTATE_PF2VM_IDS "pf2vm" + +struct adf_mstate_mgr { + u8 *buf; + u8 *state; + u32 size; + u32 n_sects; +}; + +struct adf_mstate_preh { + u32 magic; + u32 version; + u16 preh_len; + u16 n_sects; + u32 size; +}; + +struct adf_mstate_vreginfo { + void *addr; + u32 size; +}; + +struct adf_mstate_sect_h; + +typedef int (*adf_mstate_preamble_checker)(struct adf_mstate_preh *preamble, void *opa); +typedef int (*adf_mstate_populate)(struct adf_mstate_mgr *sub_mgr, u8 *buf, + u32 size, void *opa); +typedef int (*adf_mstate_action)(struct adf_mstate_mgr *sub_mgr, u8 *buf, u32 size, + void *opa); + +struct adf_mstate_mgr *adf_mstate_mgr_new(u8 *buf, u32 size); +void adf_mstate_mgr_destroy(struct adf_mstate_mgr *mgr); +void adf_mstate_mgr_init(struct adf_mstate_mgr *mgr, u8 *buf, u32 size); +void adf_mstate_mgr_init_from_parent(struct adf_mstate_mgr *mgr, + struct adf_mstate_mgr *p_mgr); +void adf_mstate_mgr_init_from_psect(struct adf_mstate_mgr *mgr, + struct adf_mstate_sect_h *p_sect); +int adf_mstate_mgr_init_from_remote(struct adf_mstate_mgr *mgr, + u8 *buf, u32 size, + adf_mstate_preamble_checker checker, + void *opaque); +struct adf_mstate_preh *adf_mstate_preamble_add(struct adf_mstate_mgr *mgr); +int adf_mstate_preamble_update(struct adf_mstate_mgr *mgr); +u32 adf_mstate_state_size(struct adf_mstate_mgr *mgr); +u32 adf_mstate_state_size_from_remote(struct adf_mstate_mgr *mgr); +void adf_mstate_sect_update(struct adf_mstate_mgr *p_mgr, + struct adf_mstate_mgr *curr_mgr, + struct adf_mstate_sect_h *sect); +struct adf_mstate_sect_h *adf_mstate_sect_add_vreg(struct adf_mstate_mgr *mgr, + const char *id, + struct adf_mstate_vreginfo *info); +struct adf_mstate_sect_h *adf_mstate_sect_add(struct adf_mstate_mgr *mgr, + const char *id, + adf_mstate_populate populate, + void *opaque); +struct adf_mstate_sect_h *adf_mstate_sect_lookup(struct adf_mstate_mgr *mgr, + const char *id, + adf_mstate_action action, + void *opaque); +#endif diff --git a/drivers/crypto/intel/qat/qat_common/adf_pfvf_pf_proto.c b/drivers/crypto/intel/qat/qat_common/adf_pfvf_pf_proto.c index 9ab93fbfefde..b9b5e744a3f1 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_pfvf_pf_proto.c +++ b/drivers/crypto/intel/qat/qat_common/adf_pfvf_pf_proto.c @@ -242,13 +242,7 @@ static int adf_handle_vf2pf_msg(struct adf_accel_dev *accel_dev, u8 vf_nr, "VersionRequest received from VF%d (vers %d) to PF (vers %d)\n", vf_nr, vf_compat_ver, ADF_PFVF_COMPAT_THIS_VERSION); - if (vf_compat_ver == 0) - compat = ADF_PF2VF_VF_INCOMPATIBLE; - else if (vf_compat_ver <= ADF_PFVF_COMPAT_THIS_VERSION) - compat = ADF_PF2VF_VF_COMPATIBLE; - else - compat = ADF_PF2VF_VF_COMPAT_UNKNOWN; - + compat = adf_vf_compat_checker(vf_compat_ver); vf_info->vf_compat_ver = vf_compat_ver; resp->type = ADF_PF2VF_MSGTYPE_VERSION_RESP; diff --git a/drivers/crypto/intel/qat/qat_common/adf_pfvf_utils.h b/drivers/crypto/intel/qat/qat_common/adf_pfvf_utils.h index 2be048e2287b..1a044297d873 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_pfvf_utils.h +++ b/drivers/crypto/intel/qat/qat_common/adf_pfvf_utils.h @@ -28,4 +28,15 @@ u32 adf_pfvf_csr_msg_of(struct adf_accel_dev *accel_dev, struct pfvf_message msg struct pfvf_message adf_pfvf_message_of(struct adf_accel_dev *accel_dev, u32 raw_msg, const struct pfvf_csr_format *fmt); +static inline u8 adf_vf_compat_checker(u8 vf_compat_ver) +{ + if (vf_compat_ver == 0) + return ADF_PF2VF_VF_INCOMPATIBLE; + + if (vf_compat_ver <= ADF_PFVF_COMPAT_THIS_VERSION) + return ADF_PF2VF_VF_COMPATIBLE; + + return ADF_PF2VF_VF_COMPAT_UNKNOWN; +} + #endif /* ADF_PFVF_UTILS_H */ diff --git a/drivers/crypto/intel/qat/qat_common/adf_rl.c b/drivers/crypto/intel/qat/qat_common/adf_rl.c index d4f2db3c53d8..346ef8bee99d 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_rl.c +++ b/drivers/crypto/intel/qat/qat_common/adf_rl.c @@ -183,14 +183,14 @@ static enum adf_cfg_service_type srv_to_cfg_svc_type(enum adf_base_services rl_s } /** - * get_sla_arr_of_type() - Returns a pointer to SLA type specific array + * adf_rl_get_sla_arr_of_type() - Returns a pointer to SLA type specific array * @rl_data: pointer to ratelimiting data * @type: SLA type * @sla_arr: pointer to variable where requested pointer will be stored * * Return: Max number of elements allowed for the returned array */ -static u32 get_sla_arr_of_type(struct adf_rl *rl_data, enum rl_node_type type, +u32 adf_rl_get_sla_arr_of_type(struct adf_rl *rl_data, enum rl_node_type type, struct rl_sla ***sla_arr) { switch (type) { @@ -778,7 +778,7 @@ static void clear_sla(struct adf_rl *rl_data, struct rl_sla *sla) rp_in_use[sla->ring_pairs_ids[i]] = false; update_budget(sla, old_cir, true); - get_sla_arr_of_type(rl_data, sla->type, &sla_type_arr); + adf_rl_get_sla_arr_of_type(rl_data, sla->type, &sla_type_arr); assign_node_to_parent(rl_data->accel_dev, sla, true); adf_rl_send_admin_delete_msg(rl_data->accel_dev, node_id, sla->type); mark_rps_usage(sla, rl_data->rp_in_use, false); @@ -875,7 +875,7 @@ static int add_update_sla(struct adf_accel_dev *accel_dev, if (!is_update) { mark_rps_usage(sla, rl_data->rp_in_use, true); - get_sla_arr_of_type(rl_data, sla->type, &sla_type_arr); + adf_rl_get_sla_arr_of_type(rl_data, sla->type, &sla_type_arr); sla_type_arr[sla->node_id] = sla; rl_data->sla[sla->sla_id] = sla; } @@ -1065,7 +1065,7 @@ void adf_rl_remove_sla_all(struct adf_accel_dev *accel_dev, bool incl_default) /* Unregister and remove all SLAs */ for (j = RL_LEAF; j >= end_type; j--) { - max_id = get_sla_arr_of_type(rl_data, j, &sla_type_arr); + max_id = adf_rl_get_sla_arr_of_type(rl_data, j, &sla_type_arr); for (i = 0; i < max_id; i++) { if (!sla_type_arr[i]) @@ -1125,7 +1125,7 @@ int adf_rl_start(struct adf_accel_dev *accel_dev) } if ((fw_caps & RL_CAPABILITY_MASK) != RL_CAPABILITY_VALUE) { - dev_info(&GET_DEV(accel_dev), "not supported\n"); + dev_info(&GET_DEV(accel_dev), "feature not supported by FW\n"); ret = -EOPNOTSUPP; goto ret_free; } diff --git a/drivers/crypto/intel/qat/qat_common/adf_rl.h b/drivers/crypto/intel/qat/qat_common/adf_rl.h index 269c6656fb90..bfe750ea0e83 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_rl.h +++ b/drivers/crypto/intel/qat/qat_common/adf_rl.h @@ -151,6 +151,8 @@ struct rl_sla { u16 ring_pairs_cnt; }; +u32 adf_rl_get_sla_arr_of_type(struct adf_rl *rl_data, enum rl_node_type type, + struct rl_sla ***sla_arr); int adf_rl_add_sla(struct adf_accel_dev *accel_dev, struct adf_rl_sla_input_data *sla_in); int adf_rl_update_sla(struct adf_accel_dev *accel_dev, diff --git a/drivers/crypto/intel/qat/qat_common/adf_sriov.c b/drivers/crypto/intel/qat/qat_common/adf_sriov.c index 87a70c00c41e..8d645e7e04aa 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_sriov.c +++ b/drivers/crypto/intel/qat/qat_common/adf_sriov.c @@ -26,10 +26,12 @@ static void adf_iov_send_resp(struct work_struct *work) u32 vf_nr = vf_info->vf_nr; bool ret; + mutex_lock(&vf_info->pfvf_mig_lock); ret = adf_recv_and_handle_vf2pf_msg(accel_dev, vf_nr); if (ret) /* re-enable interrupt on PF from this VF */ adf_enable_vf2pf_interrupts(accel_dev, 1 << vf_nr); + mutex_unlock(&vf_info->pfvf_mig_lock); kfree(pf2vf_resp); } @@ -62,6 +64,7 @@ static int adf_enable_sriov(struct adf_accel_dev *accel_dev) vf_info->vf_nr = i; mutex_init(&vf_info->pf2vf_lock); + mutex_init(&vf_info->pfvf_mig_lock); ratelimit_state_init(&vf_info->vf2pf_ratelimit, ADF_VF2PF_RATELIMIT_INTERVAL, ADF_VF2PF_RATELIMIT_BURST); @@ -138,8 +141,10 @@ void adf_disable_sriov(struct adf_accel_dev *accel_dev) if (hw_data->configure_iov_threads) hw_data->configure_iov_threads(accel_dev, false); - for (i = 0, vf = accel_dev->pf.vf_info; i < totalvfs; i++, vf++) + for (i = 0, vf = accel_dev->pf.vf_info; i < totalvfs; i++, vf++) { mutex_destroy(&vf->pf2vf_lock); + mutex_destroy(&vf->pfvf_mig_lock); + } if (!test_bit(ADF_STATUS_RESTARTING, &accel_dev->status)) { kfree(accel_dev->pf.vf_info); diff --git a/drivers/crypto/intel/qat/qat_common/adf_telemetry.c b/drivers/crypto/intel/qat/qat_common/adf_telemetry.c index 2ff714d11bd2..74fb0c2ed241 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_telemetry.c +++ b/drivers/crypto/intel/qat/qat_common/adf_telemetry.c @@ -41,6 +41,20 @@ static int validate_tl_data(struct adf_tl_hw_data *tl_data) return 0; } +static int validate_tl_slice_counters(struct icp_qat_fw_init_admin_slice_cnt *slice_count, + u8 max_slices_per_type) +{ + u8 *sl_counter = (u8 *)slice_count; + int i; + + for (i = 0; i < ADF_TL_SL_CNT_COUNT; i++) { + if (sl_counter[i] > max_slices_per_type) + return -EINVAL; + } + + return 0; +} + static int adf_tl_alloc_mem(struct adf_accel_dev *accel_dev) { struct adf_tl_hw_data *tl_data = &GET_TL_DATA(accel_dev); @@ -214,6 +228,13 @@ int adf_tl_run(struct adf_accel_dev *accel_dev, int state) return ret; } + ret = validate_tl_slice_counters(&telemetry->slice_cnt, tl_data->max_sl_cnt); + if (ret) { + dev_err(dev, "invalid value returned by FW\n"); + adf_send_admin_tl_stop(accel_dev); + return ret; + } + telemetry->hbuffs = state; atomic_set(&telemetry->state, state); diff --git a/drivers/crypto/intel/qat/qat_common/adf_telemetry.h b/drivers/crypto/intel/qat/qat_common/adf_telemetry.h index 9be81cd3b886..e54a406cc1b4 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_telemetry.h +++ b/drivers/crypto/intel/qat/qat_common/adf_telemetry.h @@ -40,6 +40,7 @@ struct adf_tl_hw_data { u8 num_dev_counters; u8 num_rp_counters; u8 max_rp; + u8 max_sl_cnt; }; struct adf_telemetry { diff --git a/drivers/crypto/intel/qat/qat_common/adf_transport.c b/drivers/crypto/intel/qat/qat_common/adf_transport.c index 630d0483c4e0..1efdf46490f1 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_transport.c +++ b/drivers/crypto/intel/qat/qat_common/adf_transport.c @@ -474,7 +474,6 @@ err: int adf_init_etr_data(struct adf_accel_dev *accel_dev) { struct adf_etr_data *etr_data; - struct adf_hw_device_data *hw_data = accel_dev->hw_device; void __iomem *csr_addr; u32 size; u32 num_banks = 0; @@ -495,8 +494,7 @@ int adf_init_etr_data(struct adf_accel_dev *accel_dev) } accel_dev->transport = etr_data; - i = hw_data->get_etr_bar_id(hw_data); - csr_addr = accel_dev->accel_pci_dev.pci_bars[i].virt_addr; + csr_addr = adf_get_etr_base(accel_dev); /* accel_dev->debugfs_dir should always be non-NULL here */ etr_data->debug = debugfs_create_dir("transport", diff --git a/drivers/crypto/intel/qat/qat_common/qat_asym_algs.c b/drivers/crypto/intel/qat/qat_common/qat_asym_algs.c index 4128200a9032..85c682e248fb 100644 --- a/drivers/crypto/intel/qat/qat_common/qat_asym_algs.c +++ b/drivers/crypto/intel/qat/qat_common/qat_asym_algs.c @@ -110,6 +110,8 @@ struct qat_dh_ctx { unsigned int p_size; bool g2; struct qat_crypto_instance *inst; + struct crypto_kpp *ftfm; + bool fallback; } __packed __aligned(64); struct qat_asym_request { @@ -381,6 +383,36 @@ unmap_src: return ret; } +static int qat_dh_generate_public_key(struct kpp_request *req) +{ + struct kpp_request *nreq = kpp_request_ctx(req); + struct crypto_kpp *tfm = crypto_kpp_reqtfm(req); + struct qat_dh_ctx *ctx = kpp_tfm_ctx(tfm); + + if (ctx->fallback) { + memcpy(nreq, req, sizeof(*req)); + kpp_request_set_tfm(nreq, ctx->ftfm); + return crypto_kpp_generate_public_key(nreq); + } + + return qat_dh_compute_value(req); +} + +static int qat_dh_compute_shared_secret(struct kpp_request *req) +{ + struct kpp_request *nreq = kpp_request_ctx(req); + struct crypto_kpp *tfm = crypto_kpp_reqtfm(req); + struct qat_dh_ctx *ctx = kpp_tfm_ctx(tfm); + + if (ctx->fallback) { + memcpy(nreq, req, sizeof(*req)); + kpp_request_set_tfm(nreq, ctx->ftfm); + return crypto_kpp_compute_shared_secret(nreq); + } + + return qat_dh_compute_value(req); +} + static int qat_dh_check_params_length(unsigned int p_len) { switch (p_len) { @@ -398,9 +430,6 @@ static int qat_dh_set_params(struct qat_dh_ctx *ctx, struct dh *params) struct qat_crypto_instance *inst = ctx->inst; struct device *dev = &GET_DEV(inst->accel_dev); - if (qat_dh_check_params_length(params->p_size << 3)) - return -EINVAL; - ctx->p_size = params->p_size; ctx->p = dma_alloc_coherent(dev, ctx->p_size, &ctx->dma_p, GFP_KERNEL); if (!ctx->p) @@ -454,6 +483,13 @@ static int qat_dh_set_secret(struct crypto_kpp *tfm, const void *buf, if (crypto_dh_decode_key(buf, len, ¶ms) < 0) return -EINVAL; + if (qat_dh_check_params_length(params.p_size << 3)) { + ctx->fallback = true; + return crypto_kpp_set_secret(ctx->ftfm, buf, len); + } + + ctx->fallback = false; + /* Free old secret if any */ qat_dh_clear_ctx(dev, ctx); @@ -481,6 +517,9 @@ static unsigned int qat_dh_max_size(struct crypto_kpp *tfm) { struct qat_dh_ctx *ctx = kpp_tfm_ctx(tfm); + if (ctx->fallback) + return crypto_kpp_maxsize(ctx->ftfm); + return ctx->p_size; } @@ -489,11 +528,22 @@ static int qat_dh_init_tfm(struct crypto_kpp *tfm) struct qat_dh_ctx *ctx = kpp_tfm_ctx(tfm); struct qat_crypto_instance *inst = qat_crypto_get_instance_node(numa_node_id()); + const char *alg = kpp_alg_name(tfm); + unsigned int reqsize; if (!inst) return -EINVAL; - kpp_set_reqsize(tfm, sizeof(struct qat_asym_request) + 64); + ctx->ftfm = crypto_alloc_kpp(alg, 0, CRYPTO_ALG_NEED_FALLBACK); + if (IS_ERR(ctx->ftfm)) + return PTR_ERR(ctx->ftfm); + + crypto_kpp_set_flags(ctx->ftfm, crypto_kpp_get_flags(tfm)); + + reqsize = max(sizeof(struct qat_asym_request) + 64, + sizeof(struct kpp_request) + crypto_kpp_reqsize(ctx->ftfm)); + + kpp_set_reqsize(tfm, reqsize); ctx->p_size = 0; ctx->g2 = false; @@ -506,6 +556,9 @@ static void qat_dh_exit_tfm(struct crypto_kpp *tfm) struct qat_dh_ctx *ctx = kpp_tfm_ctx(tfm); struct device *dev = &GET_DEV(ctx->inst->accel_dev); + if (ctx->ftfm) + crypto_free_kpp(ctx->ftfm); + qat_dh_clear_ctx(dev, ctx); qat_crypto_put_instance(ctx->inst); } @@ -1265,8 +1318,8 @@ static struct akcipher_alg rsa = { static struct kpp_alg dh = { .set_secret = qat_dh_set_secret, - .generate_public_key = qat_dh_compute_value, - .compute_shared_secret = qat_dh_compute_value, + .generate_public_key = qat_dh_generate_public_key, + .compute_shared_secret = qat_dh_compute_shared_secret, .max_size = qat_dh_max_size, .init = qat_dh_init_tfm, .exit = qat_dh_exit_tfm, @@ -1276,6 +1329,7 @@ static struct kpp_alg dh = { .cra_priority = 1000, .cra_module = THIS_MODULE, .cra_ctxsize = sizeof(struct qat_dh_ctx), + .cra_flags = CRYPTO_ALG_NEED_FALLBACK, }, }; diff --git a/drivers/crypto/intel/qat/qat_common/qat_bl.c b/drivers/crypto/intel/qat/qat_common/qat_bl.c index 76baed0a76c0..338acf29c487 100644 --- a/drivers/crypto/intel/qat/qat_common/qat_bl.c +++ b/drivers/crypto/intel/qat/qat_common/qat_bl.c @@ -81,7 +81,8 @@ static int __qat_bl_sgl_to_bufl(struct adf_accel_dev *accel_dev, if (unlikely(!bufl)) return -ENOMEM; } else { - bufl = &buf->sgl_src.sgl_hdr; + bufl = container_of(&buf->sgl_src.sgl_hdr, + struct qat_alg_buf_list, hdr); memset(bufl, 0, sizeof(struct qat_alg_buf_list)); buf->sgl_src_valid = true; } @@ -139,7 +140,8 @@ static int __qat_bl_sgl_to_bufl(struct adf_accel_dev *accel_dev, if (unlikely(!buflout)) goto err_in; } else { - buflout = &buf->sgl_dst.sgl_hdr; + buflout = container_of(&buf->sgl_dst.sgl_hdr, + struct qat_alg_buf_list, hdr); memset(buflout, 0, sizeof(struct qat_alg_buf_list)); buf->sgl_dst_valid = true; } diff --git a/drivers/crypto/intel/qat/qat_common/qat_bl.h b/drivers/crypto/intel/qat/qat_common/qat_bl.h index d87e4f35ac39..85bc32a9ec0e 100644 --- a/drivers/crypto/intel/qat/qat_common/qat_bl.h +++ b/drivers/crypto/intel/qat/qat_common/qat_bl.h @@ -15,14 +15,17 @@ struct qat_alg_buf { } __packed; struct qat_alg_buf_list { - u64 resrvd; - u32 num_bufs; - u32 num_mapped_bufs; + /* New members must be added within the __struct_group() macro below. */ + __struct_group(qat_alg_buf_list_hdr, hdr, __packed, + u64 resrvd; + u32 num_bufs; + u32 num_mapped_bufs; + ); struct qat_alg_buf buffers[]; } __packed; struct qat_alg_fixed_buf_list { - struct qat_alg_buf_list sgl_hdr; + struct qat_alg_buf_list_hdr sgl_hdr; struct qat_alg_buf descriptors[QAT_MAX_BUFF_DESC]; } __packed __aligned(64); diff --git a/drivers/crypto/intel/qat/qat_common/qat_mig_dev.c b/drivers/crypto/intel/qat/qat_common/qat_mig_dev.c new file mode 100644 index 000000000000..892c2283a50e --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/qat_mig_dev.c @@ -0,0 +1,130 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright(c) 2024 Intel Corporation */ +#include <linux/dev_printk.h> +#include <linux/export.h> +#include <linux/pci.h> +#include <linux/types.h> +#include <linux/qat/qat_mig_dev.h> +#include "adf_accel_devices.h" +#include "adf_common_drv.h" + +struct qat_mig_dev *qat_vfmig_create(struct pci_dev *pdev, int vf_id) +{ + struct adf_accel_dev *accel_dev; + struct qat_migdev_ops *ops; + struct qat_mig_dev *mdev; + + accel_dev = adf_devmgr_pci_to_accel_dev(pdev); + if (!accel_dev) + return ERR_PTR(-ENODEV); + + ops = GET_VFMIG_OPS(accel_dev); + if (!ops || !ops->init || !ops->cleanup || !ops->reset || !ops->open || + !ops->close || !ops->suspend || !ops->resume || !ops->save_state || + !ops->load_state || !ops->save_setup || !ops->load_setup) + return ERR_PTR(-EINVAL); + + mdev = kmalloc(sizeof(*mdev), GFP_KERNEL); + if (!mdev) + return ERR_PTR(-ENOMEM); + + mdev->vf_id = vf_id; + mdev->parent_accel_dev = accel_dev; + + return mdev; +} +EXPORT_SYMBOL_GPL(qat_vfmig_create); + +int qat_vfmig_init(struct qat_mig_dev *mdev) +{ + struct adf_accel_dev *accel_dev = mdev->parent_accel_dev; + + return GET_VFMIG_OPS(accel_dev)->init(mdev); +} +EXPORT_SYMBOL_GPL(qat_vfmig_init); + +void qat_vfmig_cleanup(struct qat_mig_dev *mdev) +{ + struct adf_accel_dev *accel_dev = mdev->parent_accel_dev; + + return GET_VFMIG_OPS(accel_dev)->cleanup(mdev); +} +EXPORT_SYMBOL_GPL(qat_vfmig_cleanup); + +void qat_vfmig_reset(struct qat_mig_dev *mdev) +{ + struct adf_accel_dev *accel_dev = mdev->parent_accel_dev; + + return GET_VFMIG_OPS(accel_dev)->reset(mdev); +} +EXPORT_SYMBOL_GPL(qat_vfmig_reset); + +int qat_vfmig_open(struct qat_mig_dev *mdev) +{ + struct adf_accel_dev *accel_dev = mdev->parent_accel_dev; + + return GET_VFMIG_OPS(accel_dev)->open(mdev); +} +EXPORT_SYMBOL_GPL(qat_vfmig_open); + +void qat_vfmig_close(struct qat_mig_dev *mdev) +{ + struct adf_accel_dev *accel_dev = mdev->parent_accel_dev; + + GET_VFMIG_OPS(accel_dev)->close(mdev); +} +EXPORT_SYMBOL_GPL(qat_vfmig_close); + +int qat_vfmig_suspend(struct qat_mig_dev *mdev) +{ + struct adf_accel_dev *accel_dev = mdev->parent_accel_dev; + + return GET_VFMIG_OPS(accel_dev)->suspend(mdev); +} +EXPORT_SYMBOL_GPL(qat_vfmig_suspend); + +int qat_vfmig_resume(struct qat_mig_dev *mdev) +{ + struct adf_accel_dev *accel_dev = mdev->parent_accel_dev; + + return GET_VFMIG_OPS(accel_dev)->resume(mdev); +} +EXPORT_SYMBOL_GPL(qat_vfmig_resume); + +int qat_vfmig_save_state(struct qat_mig_dev *mdev) +{ + struct adf_accel_dev *accel_dev = mdev->parent_accel_dev; + + return GET_VFMIG_OPS(accel_dev)->save_state(mdev); +} +EXPORT_SYMBOL_GPL(qat_vfmig_save_state); + +int qat_vfmig_save_setup(struct qat_mig_dev *mdev) +{ + struct adf_accel_dev *accel_dev = mdev->parent_accel_dev; + + return GET_VFMIG_OPS(accel_dev)->save_setup(mdev); +} +EXPORT_SYMBOL_GPL(qat_vfmig_save_setup); + +int qat_vfmig_load_state(struct qat_mig_dev *mdev) +{ + struct adf_accel_dev *accel_dev = mdev->parent_accel_dev; + + return GET_VFMIG_OPS(accel_dev)->load_state(mdev); +} +EXPORT_SYMBOL_GPL(qat_vfmig_load_state); + +int qat_vfmig_load_setup(struct qat_mig_dev *mdev, int size) +{ + struct adf_accel_dev *accel_dev = mdev->parent_accel_dev; + + return GET_VFMIG_OPS(accel_dev)->load_setup(mdev, size); +} +EXPORT_SYMBOL_GPL(qat_vfmig_load_setup); + +void qat_vfmig_destroy(struct qat_mig_dev *mdev) +{ + kfree(mdev); +} +EXPORT_SYMBOL_GPL(qat_vfmig_destroy); diff --git a/drivers/crypto/intel/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c b/drivers/crypto/intel/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c index af14090cc4be..6e24d57e6b98 100644 --- a/drivers/crypto/intel/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c +++ b/drivers/crypto/intel/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c @@ -5,6 +5,7 @@ #include <adf_common_drv.h> #include <adf_gen2_config.h> #include <adf_gen2_dc.h> +#include <adf_gen2_hw_csr_data.h> #include <adf_gen2_hw_data.h> #include <adf_gen2_pfvf.h> #include "adf_dh895xcc_hw_data.h" diff --git a/drivers/crypto/intel/qat/qat_dh895xccvf/adf_dh895xccvf_hw_data.c b/drivers/crypto/intel/qat/qat_dh895xccvf/adf_dh895xccvf_hw_data.c index 70e56cc16ece..f4ee4c2e00da 100644 --- a/drivers/crypto/intel/qat/qat_dh895xccvf/adf_dh895xccvf_hw_data.c +++ b/drivers/crypto/intel/qat/qat_dh895xccvf/adf_dh895xccvf_hw_data.c @@ -4,6 +4,7 @@ #include <adf_common_drv.h> #include <adf_gen2_config.h> #include <adf_gen2_dc.h> +#include <adf_gen2_hw_csr_data.h> #include <adf_gen2_hw_data.h> #include <adf_gen2_pfvf.h> #include <adf_pfvf_vf_msg.h> diff --git a/drivers/crypto/marvell/octeontx2/cn10k_cpt.c b/drivers/crypto/marvell/octeontx2/cn10k_cpt.c index 79b4e74804f6..6bfc59e67747 100644 --- a/drivers/crypto/marvell/octeontx2/cn10k_cpt.c +++ b/drivers/crypto/marvell/octeontx2/cn10k_cpt.c @@ -138,6 +138,10 @@ int cn10k_cpt_hw_ctx_init(struct pci_dev *pdev, return -ENOMEM; cptr_dma = dma_map_single(&pdev->dev, hctx, CN10K_CPT_HW_CTX_SIZE, DMA_BIDIRECTIONAL); + if (dma_mapping_error(&pdev->dev, cptr_dma)) { + kfree(hctx); + return -ENOMEM; + } cn10k_cpt_hw_ctx_set(hctx, 1); er_ctx->hw_ctx = hctx; diff --git a/drivers/crypto/mxs-dcp.c b/drivers/crypto/mxs-dcp.c index 2b3ebe0db3a6..057d73c370b7 100644 --- a/drivers/crypto/mxs-dcp.c +++ b/drivers/crypto/mxs-dcp.c @@ -15,6 +15,7 @@ #include <linux/platform_device.h> #include <linux/stmp_device.h> #include <linux/clk.h> +#include <soc/fsl/dcp.h> #include <crypto/aes.h> #include <crypto/sha1.h> @@ -101,6 +102,7 @@ struct dcp_async_ctx { struct crypto_skcipher *fallback; unsigned int key_len; uint8_t key[AES_KEYSIZE_128]; + bool key_referenced; }; struct dcp_aes_req_ctx { @@ -155,6 +157,7 @@ static struct dcp *global_sdcp; #define MXS_DCP_CONTROL0_HASH_TERM (1 << 13) #define MXS_DCP_CONTROL0_HASH_INIT (1 << 12) #define MXS_DCP_CONTROL0_PAYLOAD_KEY (1 << 11) +#define MXS_DCP_CONTROL0_OTP_KEY (1 << 10) #define MXS_DCP_CONTROL0_CIPHER_ENCRYPT (1 << 8) #define MXS_DCP_CONTROL0_CIPHER_INIT (1 << 9) #define MXS_DCP_CONTROL0_ENABLE_HASH (1 << 6) @@ -168,6 +171,8 @@ static struct dcp *global_sdcp; #define MXS_DCP_CONTROL1_CIPHER_MODE_ECB (0 << 4) #define MXS_DCP_CONTROL1_CIPHER_SELECT_AES128 (0 << 0) +#define MXS_DCP_CONTROL1_KEY_SELECT_SHIFT 8 + static int mxs_dcp_start_dma(struct dcp_async_ctx *actx) { int dma_err; @@ -224,13 +229,16 @@ static int mxs_dcp_run_aes(struct dcp_async_ctx *actx, struct dcp *sdcp = global_sdcp; struct dcp_dma_desc *desc = &sdcp->coh->desc[actx->chan]; struct dcp_aes_req_ctx *rctx = skcipher_request_ctx(req); + bool key_referenced = actx->key_referenced; int ret; - key_phys = dma_map_single(sdcp->dev, sdcp->coh->aes_key, - 2 * AES_KEYSIZE_128, DMA_TO_DEVICE); - ret = dma_mapping_error(sdcp->dev, key_phys); - if (ret) - return ret; + if (!key_referenced) { + key_phys = dma_map_single(sdcp->dev, sdcp->coh->aes_key, + 2 * AES_KEYSIZE_128, DMA_TO_DEVICE); + ret = dma_mapping_error(sdcp->dev, key_phys); + if (ret) + return ret; + } src_phys = dma_map_single(sdcp->dev, sdcp->coh->aes_in_buf, DCP_BUF_SZ, DMA_TO_DEVICE); @@ -255,8 +263,12 @@ static int mxs_dcp_run_aes(struct dcp_async_ctx *actx, MXS_DCP_CONTROL0_INTERRUPT | MXS_DCP_CONTROL0_ENABLE_CIPHER; - /* Payload contains the key. */ - desc->control0 |= MXS_DCP_CONTROL0_PAYLOAD_KEY; + if (key_referenced) + /* Set OTP key bit to select the key via KEY_SELECT. */ + desc->control0 |= MXS_DCP_CONTROL0_OTP_KEY; + else + /* Payload contains the key. */ + desc->control0 |= MXS_DCP_CONTROL0_PAYLOAD_KEY; if (rctx->enc) desc->control0 |= MXS_DCP_CONTROL0_CIPHER_ENCRYPT; @@ -270,6 +282,9 @@ static int mxs_dcp_run_aes(struct dcp_async_ctx *actx, else desc->control1 |= MXS_DCP_CONTROL1_CIPHER_MODE_CBC; + if (key_referenced) + desc->control1 |= sdcp->coh->aes_key[0] << MXS_DCP_CONTROL1_KEY_SELECT_SHIFT; + desc->next_cmd_addr = 0; desc->source = src_phys; desc->destination = dst_phys; @@ -284,9 +299,9 @@ aes_done_run: err_dst: dma_unmap_single(sdcp->dev, src_phys, DCP_BUF_SZ, DMA_TO_DEVICE); err_src: - dma_unmap_single(sdcp->dev, key_phys, 2 * AES_KEYSIZE_128, - DMA_TO_DEVICE); - + if (!key_referenced) + dma_unmap_single(sdcp->dev, key_phys, 2 * AES_KEYSIZE_128, + DMA_TO_DEVICE); return ret; } @@ -453,7 +468,7 @@ static int mxs_dcp_aes_enqueue(struct skcipher_request *req, int enc, int ecb) struct dcp_aes_req_ctx *rctx = skcipher_request_ctx(req); int ret; - if (unlikely(actx->key_len != AES_KEYSIZE_128)) + if (unlikely(actx->key_len != AES_KEYSIZE_128 && !actx->key_referenced)) return mxs_dcp_block_fallback(req, enc); rctx->enc = enc; @@ -500,6 +515,7 @@ static int mxs_dcp_aes_setkey(struct crypto_skcipher *tfm, const u8 *key, * there can still be an operation in progress. */ actx->key_len = len; + actx->key_referenced = false; if (len == AES_KEYSIZE_128) { memcpy(actx->key, key, len); return 0; @@ -516,6 +532,32 @@ static int mxs_dcp_aes_setkey(struct crypto_skcipher *tfm, const u8 *key, return crypto_skcipher_setkey(actx->fallback, key, len); } +static int mxs_dcp_aes_setrefkey(struct crypto_skcipher *tfm, const u8 *key, + unsigned int len) +{ + struct dcp_async_ctx *actx = crypto_skcipher_ctx(tfm); + + if (len != DCP_PAES_KEYSIZE) + return -EINVAL; + + switch (key[0]) { + case DCP_PAES_KEY_SLOT0: + case DCP_PAES_KEY_SLOT1: + case DCP_PAES_KEY_SLOT2: + case DCP_PAES_KEY_SLOT3: + case DCP_PAES_KEY_UNIQUE: + case DCP_PAES_KEY_OTP: + memcpy(actx->key, key, len); + actx->key_len = len; + actx->key_referenced = true; + break; + default: + return -EINVAL; + } + + return 0; +} + static int mxs_dcp_aes_fallback_init_tfm(struct crypto_skcipher *tfm) { const char *name = crypto_tfm_alg_name(crypto_skcipher_tfm(tfm)); @@ -539,6 +581,13 @@ static void mxs_dcp_aes_fallback_exit_tfm(struct crypto_skcipher *tfm) crypto_free_skcipher(actx->fallback); } +static int mxs_dcp_paes_init_tfm(struct crypto_skcipher *tfm) +{ + crypto_skcipher_set_reqsize(tfm, sizeof(struct dcp_aes_req_ctx)); + + return 0; +} + /* * Hashing (SHA1/SHA256) */ @@ -889,6 +938,39 @@ static struct skcipher_alg dcp_aes_algs[] = { .ivsize = AES_BLOCK_SIZE, .init = mxs_dcp_aes_fallback_init_tfm, .exit = mxs_dcp_aes_fallback_exit_tfm, + }, { + .base.cra_name = "ecb(paes)", + .base.cra_driver_name = "ecb-paes-dcp", + .base.cra_priority = 401, + .base.cra_alignmask = 15, + .base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_INTERNAL, + .base.cra_blocksize = AES_BLOCK_SIZE, + .base.cra_ctxsize = sizeof(struct dcp_async_ctx), + .base.cra_module = THIS_MODULE, + + .min_keysize = DCP_PAES_KEYSIZE, + .max_keysize = DCP_PAES_KEYSIZE, + .setkey = mxs_dcp_aes_setrefkey, + .encrypt = mxs_dcp_aes_ecb_encrypt, + .decrypt = mxs_dcp_aes_ecb_decrypt, + .init = mxs_dcp_paes_init_tfm, + }, { + .base.cra_name = "cbc(paes)", + .base.cra_driver_name = "cbc-paes-dcp", + .base.cra_priority = 401, + .base.cra_alignmask = 15, + .base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_INTERNAL, + .base.cra_blocksize = AES_BLOCK_SIZE, + .base.cra_ctxsize = sizeof(struct dcp_async_ctx), + .base.cra_module = THIS_MODULE, + + .min_keysize = DCP_PAES_KEYSIZE, + .max_keysize = DCP_PAES_KEYSIZE, + .setkey = mxs_dcp_aes_setrefkey, + .encrypt = mxs_dcp_aes_cbc_encrypt, + .decrypt = mxs_dcp_aes_cbc_decrypt, + .ivsize = AES_BLOCK_SIZE, + .init = mxs_dcp_paes_init_tfm, }, }; diff --git a/drivers/crypto/nx/nx-842.c b/drivers/crypto/nx/nx-842.c index 2ab90ec10e61..82214cde2bcd 100644 --- a/drivers/crypto/nx/nx-842.c +++ b/drivers/crypto/nx/nx-842.c @@ -251,7 +251,9 @@ int nx842_crypto_compress(struct crypto_tfm *tfm, u8 *dst, unsigned int *dlen) { struct nx842_crypto_ctx *ctx = crypto_tfm_ctx(tfm); - struct nx842_crypto_header *hdr = &ctx->header; + struct nx842_crypto_header *hdr = + container_of(&ctx->header, + struct nx842_crypto_header, hdr); struct nx842_crypto_param p; struct nx842_constraints c = *ctx->driver->constraints; unsigned int groups, hdrsize, h; @@ -490,7 +492,7 @@ int nx842_crypto_decompress(struct crypto_tfm *tfm, } memcpy(&ctx->header, src, hdr_len); - hdr = &ctx->header; + hdr = container_of(&ctx->header, struct nx842_crypto_header, hdr); for (n = 0; n < hdr->groups; n++) { /* ignore applies to last group */ diff --git a/drivers/crypto/nx/nx-842.h b/drivers/crypto/nx/nx-842.h index 7590bfb24d79..25fa70b2112c 100644 --- a/drivers/crypto/nx/nx-842.h +++ b/drivers/crypto/nx/nx-842.h @@ -157,9 +157,11 @@ struct nx842_crypto_header_group { } __packed; struct nx842_crypto_header { - __be16 magic; /* NX842_CRYPTO_MAGIC */ - __be16 ignore; /* decompressed end bytes to ignore */ - u8 groups; /* total groups in this header */ + struct_group_tagged(nx842_crypto_header_hdr, hdr, + __be16 magic; /* NX842_CRYPTO_MAGIC */ + __be16 ignore; /* decompressed end bytes to ignore */ + u8 groups; /* total groups in this header */ + ); struct nx842_crypto_header_group group[]; } __packed; @@ -171,7 +173,7 @@ struct nx842_crypto_ctx { u8 *wmem; u8 *sbounce, *dbounce; - struct nx842_crypto_header header; + struct nx842_crypto_header_hdr header; struct nx842_crypto_header_group group[NX842_CRYPTO_GROUP_MAX]; struct nx842_driver *driver; diff --git a/drivers/crypto/sahara.c b/drivers/crypto/sahara.c index 3423b5cde1c7..96d4af5d48a6 100644 --- a/drivers/crypto/sahara.c +++ b/drivers/crypto/sahara.c @@ -559,7 +559,7 @@ static int sahara_aes_process(struct skcipher_request *req) struct sahara_ctx *ctx; struct sahara_aes_reqctx *rctx; int ret; - unsigned long timeout; + unsigned long time_left; /* Request is ready to be dispatched by the device */ dev_dbg(dev->device, @@ -597,15 +597,15 @@ static int sahara_aes_process(struct skcipher_request *req) if (ret) return -EINVAL; - timeout = wait_for_completion_timeout(&dev->dma_completion, - msecs_to_jiffies(SAHARA_TIMEOUT_MS)); + time_left = wait_for_completion_timeout(&dev->dma_completion, + msecs_to_jiffies(SAHARA_TIMEOUT_MS)); dma_unmap_sg(dev->device, dev->out_sg, dev->nb_out_sg, DMA_FROM_DEVICE); dma_unmap_sg(dev->device, dev->in_sg, dev->nb_in_sg, DMA_TO_DEVICE); - if (!timeout) { + if (!time_left) { dev_err(dev->device, "AES timeout\n"); return -ETIMEDOUT; } @@ -931,7 +931,7 @@ static int sahara_sha_process(struct ahash_request *req) struct sahara_dev *dev = dev_ptr; struct sahara_sha_reqctx *rctx = ahash_request_ctx(req); int ret; - unsigned long timeout; + unsigned long time_left; ret = sahara_sha_prepare_request(req); if (!ret) @@ -963,14 +963,14 @@ static int sahara_sha_process(struct ahash_request *req) sahara_write(dev, dev->hw_phys_desc[0], SAHARA_REG_DAR); - timeout = wait_for_completion_timeout(&dev->dma_completion, - msecs_to_jiffies(SAHARA_TIMEOUT_MS)); + time_left = wait_for_completion_timeout(&dev->dma_completion, + msecs_to_jiffies(SAHARA_TIMEOUT_MS)); if (rctx->sg_in_idx) dma_unmap_sg(dev->device, dev->in_sg, dev->nb_in_sg, DMA_TO_DEVICE); - if (!timeout) { + if (!time_left) { dev_err(dev->device, "SHA timeout\n"); return -ETIMEDOUT; } diff --git a/drivers/crypto/starfive/Kconfig b/drivers/crypto/starfive/Kconfig index cb59357b58b2..0fe389e9f932 100644 --- a/drivers/crypto/starfive/Kconfig +++ b/drivers/crypto/starfive/Kconfig @@ -14,6 +14,10 @@ config CRYPTO_DEV_JH7110 select CRYPTO_RSA select CRYPTO_AES select CRYPTO_CCM + select CRYPTO_GCM + select CRYPTO_ECB + select CRYPTO_CBC + select CRYPTO_CTR help Support for StarFive JH7110 crypto hardware acceleration engine. This module provides acceleration for public key algo, diff --git a/drivers/crypto/starfive/jh7110-aes.c b/drivers/crypto/starfive/jh7110-aes.c index 1ac15cc4ef3c..86a1a1fa9f8f 100644 --- a/drivers/crypto/starfive/jh7110-aes.c +++ b/drivers/crypto/starfive/jh7110-aes.c @@ -78,7 +78,7 @@ static inline int is_gcm(struct starfive_cryp_dev *cryp) return (cryp->flags & FLG_MODE_MASK) == STARFIVE_AES_MODE_GCM; } -static inline int is_encrypt(struct starfive_cryp_dev *cryp) +static inline bool is_encrypt(struct starfive_cryp_dev *cryp) { return cryp->flags & FLG_ENCRYPT; } @@ -103,16 +103,6 @@ static void starfive_aes_aead_hw_start(struct starfive_cryp_ctx *ctx, u32 hw_mod } } -static inline void starfive_aes_set_ivlen(struct starfive_cryp_ctx *ctx) -{ - struct starfive_cryp_dev *cryp = ctx->cryp; - - if (is_gcm(cryp)) - writel(GCM_AES_IV_SIZE, cryp->base + STARFIVE_AES_IVLEN); - else - writel(AES_BLOCK_SIZE, cryp->base + STARFIVE_AES_IVLEN); -} - static inline void starfive_aes_set_alen(struct starfive_cryp_ctx *ctx) { struct starfive_cryp_dev *cryp = ctx->cryp; @@ -261,7 +251,6 @@ static int starfive_aes_hw_init(struct starfive_cryp_ctx *ctx) rctx->csr.aes.mode = hw_mode; rctx->csr.aes.cmode = !is_encrypt(cryp); - rctx->csr.aes.ie = 1; rctx->csr.aes.stmode = STARFIVE_AES_MODE_XFB_1; if (cryp->side_chan) { @@ -279,7 +268,7 @@ static int starfive_aes_hw_init(struct starfive_cryp_ctx *ctx) case STARFIVE_AES_MODE_GCM: starfive_aes_set_alen(ctx); starfive_aes_set_mlen(ctx); - starfive_aes_set_ivlen(ctx); + writel(GCM_AES_IV_SIZE, cryp->base + STARFIVE_AES_IVLEN); starfive_aes_aead_hw_start(ctx, hw_mode); starfive_aes_write_iv(ctx, (void *)cryp->req.areq->iv); break; @@ -300,52 +289,49 @@ static int starfive_aes_hw_init(struct starfive_cryp_ctx *ctx) return cryp->err; } -static int starfive_aes_read_authtag(struct starfive_cryp_dev *cryp) +static int starfive_aes_read_authtag(struct starfive_cryp_ctx *ctx) { - int i, start_addr; + struct starfive_cryp_dev *cryp = ctx->cryp; + struct starfive_cryp_request_ctx *rctx = ctx->rctx; + int i; if (starfive_aes_wait_busy(cryp)) return dev_err_probe(cryp->dev, -ETIMEDOUT, "Timeout waiting for tag generation."); - start_addr = STARFIVE_AES_NONCE0; - - if (is_gcm(cryp)) - for (i = 0; i < AES_BLOCK_32; i++, start_addr += 4) - cryp->tag_out[i] = readl(cryp->base + start_addr); - else + if ((cryp->flags & FLG_MODE_MASK) == STARFIVE_AES_MODE_GCM) { + cryp->tag_out[0] = readl(cryp->base + STARFIVE_AES_NONCE0); + cryp->tag_out[1] = readl(cryp->base + STARFIVE_AES_NONCE1); + cryp->tag_out[2] = readl(cryp->base + STARFIVE_AES_NONCE2); + cryp->tag_out[3] = readl(cryp->base + STARFIVE_AES_NONCE3); + } else { for (i = 0; i < AES_BLOCK_32; i++) cryp->tag_out[i] = readl(cryp->base + STARFIVE_AES_AESDIO0R); + } if (is_encrypt(cryp)) { - scatterwalk_copychunks(cryp->tag_out, &cryp->out_walk, cryp->authsize, 1); + scatterwalk_map_and_copy(cryp->tag_out, rctx->out_sg, + cryp->total_in, cryp->authsize, 1); } else { - scatterwalk_copychunks(cryp->tag_in, &cryp->in_walk, cryp->authsize, 0); - if (crypto_memneq(cryp->tag_in, cryp->tag_out, cryp->authsize)) - return dev_err_probe(cryp->dev, -EBADMSG, "Failed tag verification\n"); + return -EBADMSG; } return 0; } -static void starfive_aes_finish_req(struct starfive_cryp_dev *cryp) +static void starfive_aes_finish_req(struct starfive_cryp_ctx *ctx) { - union starfive_aes_csr csr; + struct starfive_cryp_dev *cryp = ctx->cryp; int err = cryp->err; if (!err && cryp->authsize) - err = starfive_aes_read_authtag(cryp); + err = starfive_aes_read_authtag(ctx); if (!err && ((cryp->flags & FLG_MODE_MASK) == STARFIVE_AES_MODE_CBC || (cryp->flags & FLG_MODE_MASK) == STARFIVE_AES_MODE_CTR)) starfive_aes_get_iv(cryp, (void *)cryp->req.sreq->iv); - /* reset irq flags*/ - csr.v = 0; - csr.aesrst = 1; - writel(csr.v, cryp->base + STARFIVE_AES_CSR); - if (cryp->authsize) crypto_finalize_aead_request(cryp->engine, cryp->req.areq, err); else @@ -353,39 +339,6 @@ static void starfive_aes_finish_req(struct starfive_cryp_dev *cryp) err); } -void starfive_aes_done_task(unsigned long param) -{ - struct starfive_cryp_dev *cryp = (struct starfive_cryp_dev *)param; - u32 block[AES_BLOCK_32]; - u32 stat; - int i; - - for (i = 0; i < AES_BLOCK_32; i++) - block[i] = readl(cryp->base + STARFIVE_AES_AESDIO0R); - - scatterwalk_copychunks(block, &cryp->out_walk, min_t(size_t, AES_BLOCK_SIZE, - cryp->total_out), 1); - - cryp->total_out -= min_t(size_t, AES_BLOCK_SIZE, cryp->total_out); - - if (!cryp->total_out) { - starfive_aes_finish_req(cryp); - return; - } - - memset(block, 0, AES_BLOCK_SIZE); - scatterwalk_copychunks(block, &cryp->in_walk, min_t(size_t, AES_BLOCK_SIZE, - cryp->total_in), 0); - cryp->total_in -= min_t(size_t, AES_BLOCK_SIZE, cryp->total_in); - - for (i = 0; i < AES_BLOCK_32; i++) - writel(block[i], cryp->base + STARFIVE_AES_AESDIO0R); - - stat = readl(cryp->base + STARFIVE_IE_MASK_OFFSET); - stat &= ~STARFIVE_IE_MASK_AES_DONE; - writel(stat, cryp->base + STARFIVE_IE_MASK_OFFSET); -} - static int starfive_aes_gcm_write_adata(struct starfive_cryp_ctx *ctx) { struct starfive_cryp_dev *cryp = ctx->cryp; @@ -451,60 +404,165 @@ static int starfive_aes_ccm_write_adata(struct starfive_cryp_ctx *ctx) return 0; } -static int starfive_aes_prepare_req(struct skcipher_request *req, - struct aead_request *areq) +static void starfive_aes_dma_done(void *param) { - struct starfive_cryp_ctx *ctx; - struct starfive_cryp_request_ctx *rctx; - struct starfive_cryp_dev *cryp; + struct starfive_cryp_dev *cryp = param; - if (!req && !areq) - return -EINVAL; + complete(&cryp->dma_done); +} - ctx = req ? crypto_skcipher_ctx(crypto_skcipher_reqtfm(req)) : - crypto_aead_ctx(crypto_aead_reqtfm(areq)); +static void starfive_aes_dma_init(struct starfive_cryp_dev *cryp) +{ + cryp->cfg_in.direction = DMA_MEM_TO_DEV; + cryp->cfg_in.src_addr_width = DMA_SLAVE_BUSWIDTH_16_BYTES; + cryp->cfg_in.dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES; + cryp->cfg_in.src_maxburst = cryp->dma_maxburst; + cryp->cfg_in.dst_maxburst = cryp->dma_maxburst; + cryp->cfg_in.dst_addr = cryp->phys_base + STARFIVE_ALG_FIFO_OFFSET; - cryp = ctx->cryp; - rctx = req ? skcipher_request_ctx(req) : aead_request_ctx(areq); + dmaengine_slave_config(cryp->tx, &cryp->cfg_in); - if (req) { - cryp->req.sreq = req; - cryp->total_in = req->cryptlen; - cryp->total_out = req->cryptlen; - cryp->assoclen = 0; - cryp->authsize = 0; - } else { - cryp->req.areq = areq; - cryp->assoclen = areq->assoclen; - cryp->authsize = crypto_aead_authsize(crypto_aead_reqtfm(areq)); - if (is_encrypt(cryp)) { - cryp->total_in = areq->cryptlen; - cryp->total_out = areq->cryptlen; - } else { - cryp->total_in = areq->cryptlen - cryp->authsize; - cryp->total_out = cryp->total_in; - } - } + cryp->cfg_out.direction = DMA_DEV_TO_MEM; + cryp->cfg_out.src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES; + cryp->cfg_out.dst_addr_width = DMA_SLAVE_BUSWIDTH_16_BYTES; + cryp->cfg_out.src_maxburst = 4; + cryp->cfg_out.dst_maxburst = 4; + cryp->cfg_out.src_addr = cryp->phys_base + STARFIVE_ALG_FIFO_OFFSET; - rctx->in_sg = req ? req->src : areq->src; - scatterwalk_start(&cryp->in_walk, rctx->in_sg); + dmaengine_slave_config(cryp->rx, &cryp->cfg_out); - rctx->out_sg = req ? req->dst : areq->dst; - scatterwalk_start(&cryp->out_walk, rctx->out_sg); + init_completion(&cryp->dma_done); +} - if (cryp->assoclen) { - rctx->adata = kzalloc(cryp->assoclen + AES_BLOCK_SIZE, GFP_KERNEL); - if (!rctx->adata) - return dev_err_probe(cryp->dev, -ENOMEM, - "Failed to alloc memory for adata"); +static int starfive_aes_dma_xfer(struct starfive_cryp_dev *cryp, + struct scatterlist *src, + struct scatterlist *dst, + int len) +{ + struct dma_async_tx_descriptor *in_desc, *out_desc; + union starfive_alg_cr alg_cr; + int ret = 0, in_save, out_save; + + alg_cr.v = 0; + alg_cr.start = 1; + alg_cr.aes_dma_en = 1; + writel(alg_cr.v, cryp->base + STARFIVE_ALG_CR_OFFSET); + + in_save = sg_dma_len(src); + out_save = sg_dma_len(dst); - scatterwalk_copychunks(rctx->adata, &cryp->in_walk, cryp->assoclen, 0); - scatterwalk_copychunks(NULL, &cryp->out_walk, cryp->assoclen, 2); + writel(ALIGN(len, AES_BLOCK_SIZE), cryp->base + STARFIVE_DMA_IN_LEN_OFFSET); + writel(ALIGN(len, AES_BLOCK_SIZE), cryp->base + STARFIVE_DMA_OUT_LEN_OFFSET); + + sg_dma_len(src) = ALIGN(len, AES_BLOCK_SIZE); + sg_dma_len(dst) = ALIGN(len, AES_BLOCK_SIZE); + + out_desc = dmaengine_prep_slave_sg(cryp->rx, dst, 1, DMA_DEV_TO_MEM, + DMA_PREP_INTERRUPT | DMA_CTRL_ACK); + if (!out_desc) { + ret = -EINVAL; + goto dma_err; } - ctx->rctx = rctx; + out_desc->callback = starfive_aes_dma_done; + out_desc->callback_param = cryp; + + reinit_completion(&cryp->dma_done); + dmaengine_submit(out_desc); + dma_async_issue_pending(cryp->rx); + + in_desc = dmaengine_prep_slave_sg(cryp->tx, src, 1, DMA_MEM_TO_DEV, + DMA_PREP_INTERRUPT | DMA_CTRL_ACK); + if (!in_desc) { + ret = -EINVAL; + goto dma_err; + } + + dmaengine_submit(in_desc); + dma_async_issue_pending(cryp->tx); + + if (!wait_for_completion_timeout(&cryp->dma_done, + msecs_to_jiffies(1000))) + ret = -ETIMEDOUT; + +dma_err: + sg_dma_len(src) = in_save; + sg_dma_len(dst) = out_save; + + alg_cr.v = 0; + alg_cr.clear = 1; + writel(alg_cr.v, cryp->base + STARFIVE_ALG_CR_OFFSET); + + return ret; +} + +static int starfive_aes_map_sg(struct starfive_cryp_dev *cryp, + struct scatterlist *src, + struct scatterlist *dst) +{ + struct scatterlist *stsg, *dtsg; + struct scatterlist _src[2], _dst[2]; + unsigned int remain = cryp->total_in; + unsigned int len, src_nents, dst_nents; + int ret; + + if (src == dst) { + for (stsg = src, dtsg = dst; remain > 0; + stsg = sg_next(stsg), dtsg = sg_next(dtsg)) { + src_nents = dma_map_sg(cryp->dev, stsg, 1, DMA_BIDIRECTIONAL); + if (src_nents == 0) + return dev_err_probe(cryp->dev, -ENOMEM, + "dma_map_sg error\n"); + + dst_nents = src_nents; + len = min(sg_dma_len(stsg), remain); + + ret = starfive_aes_dma_xfer(cryp, stsg, dtsg, len); + dma_unmap_sg(cryp->dev, stsg, 1, DMA_BIDIRECTIONAL); + if (ret) + return ret; + + remain -= len; + } + } else { + for (stsg = src, dtsg = dst;;) { + src_nents = dma_map_sg(cryp->dev, stsg, 1, DMA_TO_DEVICE); + if (src_nents == 0) + return dev_err_probe(cryp->dev, -ENOMEM, + "dma_map_sg src error\n"); + + dst_nents = dma_map_sg(cryp->dev, dtsg, 1, DMA_FROM_DEVICE); + if (dst_nents == 0) + return dev_err_probe(cryp->dev, -ENOMEM, + "dma_map_sg dst error\n"); + + len = min(sg_dma_len(stsg), sg_dma_len(dtsg)); + len = min(len, remain); + + ret = starfive_aes_dma_xfer(cryp, stsg, dtsg, len); + dma_unmap_sg(cryp->dev, stsg, 1, DMA_TO_DEVICE); + dma_unmap_sg(cryp->dev, dtsg, 1, DMA_FROM_DEVICE); + if (ret) + return ret; + + remain -= len; + if (remain == 0) + break; + + if (sg_dma_len(stsg) - len) { + stsg = scatterwalk_ffwd(_src, stsg, len); + dtsg = sg_next(dtsg); + } else if (sg_dma_len(dtsg) - len) { + dtsg = scatterwalk_ffwd(_dst, dtsg, len); + stsg = sg_next(stsg); + } else { + stsg = sg_next(stsg); + dtsg = sg_next(dtsg); + } + } + } - return starfive_aes_hw_init(ctx); + return 0; } static int starfive_aes_do_one_req(struct crypto_engine *engine, void *areq) @@ -513,35 +571,42 @@ static int starfive_aes_do_one_req(struct crypto_engine *engine, void *areq) container_of(areq, struct skcipher_request, base); struct starfive_cryp_ctx *ctx = crypto_skcipher_ctx(crypto_skcipher_reqtfm(req)); + struct starfive_cryp_request_ctx *rctx = skcipher_request_ctx(req); struct starfive_cryp_dev *cryp = ctx->cryp; - u32 block[AES_BLOCK_32]; - u32 stat; - int err; - int i; + int ret; - err = starfive_aes_prepare_req(req, NULL); - if (err) - return err; + cryp->req.sreq = req; + cryp->total_in = req->cryptlen; + cryp->total_out = req->cryptlen; + cryp->assoclen = 0; + cryp->authsize = 0; - /* - * Write first plain/ciphertext block to start the module - * then let irq tasklet handle the rest of the data blocks. - */ - scatterwalk_copychunks(block, &cryp->in_walk, min_t(size_t, AES_BLOCK_SIZE, - cryp->total_in), 0); - cryp->total_in -= min_t(size_t, AES_BLOCK_SIZE, cryp->total_in); + rctx->in_sg = req->src; + rctx->out_sg = req->dst; + + ctx->rctx = rctx; + + ret = starfive_aes_hw_init(ctx); + if (ret) + return ret; - for (i = 0; i < AES_BLOCK_32; i++) - writel(block[i], cryp->base + STARFIVE_AES_AESDIO0R); + if (!cryp->total_in) + goto finish_req; - stat = readl(cryp->base + STARFIVE_IE_MASK_OFFSET); - stat &= ~STARFIVE_IE_MASK_AES_DONE; - writel(stat, cryp->base + STARFIVE_IE_MASK_OFFSET); + starfive_aes_dma_init(cryp); + + ret = starfive_aes_map_sg(cryp, rctx->in_sg, rctx->out_sg); + if (ret) + return ret; + +finish_req: + starfive_aes_finish_req(ctx); return 0; } -static int starfive_aes_init_tfm(struct crypto_skcipher *tfm) +static int starfive_aes_init_tfm(struct crypto_skcipher *tfm, + const char *alg_name) { struct starfive_cryp_ctx *ctx = crypto_skcipher_ctx(tfm); @@ -549,12 +614,26 @@ static int starfive_aes_init_tfm(struct crypto_skcipher *tfm) if (!ctx->cryp) return -ENODEV; + ctx->skcipher_fbk = crypto_alloc_skcipher(alg_name, 0, + CRYPTO_ALG_NEED_FALLBACK); + if (IS_ERR(ctx->skcipher_fbk)) + return dev_err_probe(ctx->cryp->dev, PTR_ERR(ctx->skcipher_fbk), + "%s() failed to allocate fallback for %s\n", + __func__, alg_name); + crypto_skcipher_set_reqsize(tfm, sizeof(struct starfive_cryp_request_ctx) + - sizeof(struct skcipher_request)); + crypto_skcipher_reqsize(ctx->skcipher_fbk)); return 0; } +static void starfive_aes_exit_tfm(struct crypto_skcipher *tfm) +{ + struct starfive_cryp_ctx *ctx = crypto_skcipher_ctx(tfm); + + crypto_free_skcipher(ctx->skcipher_fbk); +} + static int starfive_aes_aead_do_one_req(struct crypto_engine *engine, void *areq) { struct aead_request *req = @@ -562,79 +641,99 @@ static int starfive_aes_aead_do_one_req(struct crypto_engine *engine, void *areq struct starfive_cryp_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req)); struct starfive_cryp_dev *cryp = ctx->cryp; - struct starfive_cryp_request_ctx *rctx; - u32 block[AES_BLOCK_32]; - u32 stat; - int err; - int i; + struct starfive_cryp_request_ctx *rctx = aead_request_ctx(req); + struct scatterlist _src[2], _dst[2]; + int ret; + + cryp->req.areq = req; + cryp->assoclen = req->assoclen; + cryp->authsize = crypto_aead_authsize(crypto_aead_reqtfm(req)); + + rctx->in_sg = scatterwalk_ffwd(_src, req->src, cryp->assoclen); + if (req->src == req->dst) + rctx->out_sg = rctx->in_sg; + else + rctx->out_sg = scatterwalk_ffwd(_dst, req->dst, cryp->assoclen); + + if (is_encrypt(cryp)) { + cryp->total_in = req->cryptlen; + cryp->total_out = req->cryptlen; + } else { + cryp->total_in = req->cryptlen - cryp->authsize; + cryp->total_out = cryp->total_in; + scatterwalk_map_and_copy(cryp->tag_in, req->src, + cryp->total_in + cryp->assoclen, + cryp->authsize, 0); + } - err = starfive_aes_prepare_req(NULL, req); - if (err) - return err; + if (cryp->assoclen) { + rctx->adata = kzalloc(cryp->assoclen + AES_BLOCK_SIZE, GFP_KERNEL); + if (!rctx->adata) + return dev_err_probe(cryp->dev, -ENOMEM, + "Failed to alloc memory for adata"); + + if (sg_copy_to_buffer(req->src, sg_nents_for_len(req->src, cryp->assoclen), + rctx->adata, cryp->assoclen) != cryp->assoclen) + return -EINVAL; + } + + if (cryp->total_in) + sg_zero_buffer(rctx->in_sg, sg_nents(rctx->in_sg), + sg_dma_len(rctx->in_sg) - cryp->total_in, + cryp->total_in); - rctx = ctx->rctx; + ctx->rctx = rctx; + + ret = starfive_aes_hw_init(ctx); + if (ret) + return ret; if (!cryp->assoclen) goto write_text; if ((cryp->flags & FLG_MODE_MASK) == STARFIVE_AES_MODE_CCM) - cryp->err = starfive_aes_ccm_write_adata(ctx); + ret = starfive_aes_ccm_write_adata(ctx); else - cryp->err = starfive_aes_gcm_write_adata(ctx); + ret = starfive_aes_gcm_write_adata(ctx); kfree(rctx->adata); - if (cryp->err) - return cryp->err; + if (ret) + return ret; write_text: if (!cryp->total_in) goto finish_req; - /* - * Write first plain/ciphertext block to start the module - * then let irq tasklet handle the rest of the data blocks. - */ - scatterwalk_copychunks(block, &cryp->in_walk, min_t(size_t, AES_BLOCK_SIZE, - cryp->total_in), 0); - cryp->total_in -= min_t(size_t, AES_BLOCK_SIZE, cryp->total_in); - - for (i = 0; i < AES_BLOCK_32; i++) - writel(block[i], cryp->base + STARFIVE_AES_AESDIO0R); - - stat = readl(cryp->base + STARFIVE_IE_MASK_OFFSET); - stat &= ~STARFIVE_IE_MASK_AES_DONE; - writel(stat, cryp->base + STARFIVE_IE_MASK_OFFSET); + starfive_aes_dma_init(cryp); - return 0; + ret = starfive_aes_map_sg(cryp, rctx->in_sg, rctx->out_sg); + if (ret) + return ret; finish_req: - starfive_aes_finish_req(cryp); + starfive_aes_finish_req(ctx); return 0; } -static int starfive_aes_aead_init_tfm(struct crypto_aead *tfm) +static int starfive_aes_aead_init_tfm(struct crypto_aead *tfm, + const char *alg_name) { struct starfive_cryp_ctx *ctx = crypto_aead_ctx(tfm); - struct starfive_cryp_dev *cryp = ctx->cryp; - struct crypto_tfm *aead = crypto_aead_tfm(tfm); - struct crypto_alg *alg = aead->__crt_alg; ctx->cryp = starfive_cryp_find_dev(ctx); if (!ctx->cryp) return -ENODEV; - if (alg->cra_flags & CRYPTO_ALG_NEED_FALLBACK) { - ctx->aead_fbk = crypto_alloc_aead(alg->cra_name, 0, - CRYPTO_ALG_NEED_FALLBACK); - if (IS_ERR(ctx->aead_fbk)) - return dev_err_probe(cryp->dev, PTR_ERR(ctx->aead_fbk), - "%s() failed to allocate fallback for %s\n", - __func__, alg->cra_name); - } + ctx->aead_fbk = crypto_alloc_aead(alg_name, 0, + CRYPTO_ALG_NEED_FALLBACK); + if (IS_ERR(ctx->aead_fbk)) + return dev_err_probe(ctx->cryp->dev, PTR_ERR(ctx->aead_fbk), + "%s() failed to allocate fallback for %s\n", + __func__, alg_name); - crypto_aead_set_reqsize(tfm, sizeof(struct starfive_cryp_ctx) + - sizeof(struct aead_request)); + crypto_aead_set_reqsize(tfm, sizeof(struct starfive_cryp_request_ctx) + + crypto_aead_reqsize(ctx->aead_fbk)); return 0; } @@ -646,6 +745,46 @@ static void starfive_aes_aead_exit_tfm(struct crypto_aead *tfm) crypto_free_aead(ctx->aead_fbk); } +static bool starfive_aes_check_unaligned(struct starfive_cryp_dev *cryp, + struct scatterlist *src, + struct scatterlist *dst) +{ + struct scatterlist *tsg; + int i; + + for_each_sg(src, tsg, sg_nents(src), i) + if (!IS_ALIGNED(tsg->offset, sizeof(u32)) || + (!IS_ALIGNED(tsg->length, AES_BLOCK_SIZE) && + !sg_is_last(tsg))) + return true; + + if (src != dst) + for_each_sg(dst, tsg, sg_nents(dst), i) + if (!IS_ALIGNED(tsg->offset, sizeof(u32)) || + (!IS_ALIGNED(tsg->length, AES_BLOCK_SIZE) && + !sg_is_last(tsg))) + return true; + + return false; +} + +static int starfive_aes_do_fallback(struct skcipher_request *req, bool enc) +{ + struct starfive_cryp_ctx *ctx = + crypto_skcipher_ctx(crypto_skcipher_reqtfm(req)); + struct skcipher_request *subreq = skcipher_request_ctx(req); + + skcipher_request_set_tfm(subreq, ctx->skcipher_fbk); + skcipher_request_set_callback(subreq, req->base.flags, + req->base.complete, + req->base.data); + skcipher_request_set_crypt(subreq, req->src, req->dst, + req->cryptlen, req->iv); + + return enc ? crypto_skcipher_encrypt(subreq) : + crypto_skcipher_decrypt(subreq); +} + static int starfive_aes_crypt(struct skcipher_request *req, unsigned long flags) { struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); @@ -660,32 +799,54 @@ static int starfive_aes_crypt(struct skcipher_request *req, unsigned long flags) if (req->cryptlen & blocksize_align) return -EINVAL; + if (starfive_aes_check_unaligned(cryp, req->src, req->dst)) + return starfive_aes_do_fallback(req, is_encrypt(cryp)); + return crypto_transfer_skcipher_request_to_engine(cryp->engine, req); } +static int starfive_aes_aead_do_fallback(struct aead_request *req, bool enc) +{ + struct starfive_cryp_ctx *ctx = + crypto_aead_ctx(crypto_aead_reqtfm(req)); + struct aead_request *subreq = aead_request_ctx(req); + + aead_request_set_tfm(subreq, ctx->aead_fbk); + aead_request_set_callback(subreq, req->base.flags, + req->base.complete, + req->base.data); + aead_request_set_crypt(subreq, req->src, req->dst, + req->cryptlen, req->iv); + aead_request_set_ad(subreq, req->assoclen); + + return enc ? crypto_aead_encrypt(subreq) : + crypto_aead_decrypt(subreq); +} + static int starfive_aes_aead_crypt(struct aead_request *req, unsigned long flags) { struct starfive_cryp_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req)); struct starfive_cryp_dev *cryp = ctx->cryp; + struct scatterlist *src, *dst, _src[2], _dst[2]; cryp->flags = flags; - /* - * HW engine could not perform CCM tag verification on - * non-blocksize aligned text, use fallback algo instead + /* aes-ccm does not support tag verification for non-aligned text, + * use fallback for ccm decryption instead. */ - if (ctx->aead_fbk && !is_encrypt(cryp)) { - struct aead_request *subreq = aead_request_ctx(req); + if (((cryp->flags & FLG_MODE_MASK) == STARFIVE_AES_MODE_CCM) && + !is_encrypt(cryp)) + return starfive_aes_aead_do_fallback(req, 0); - aead_request_set_tfm(subreq, ctx->aead_fbk); - aead_request_set_callback(subreq, req->base.flags, - req->base.complete, req->base.data); - aead_request_set_crypt(subreq, req->src, - req->dst, req->cryptlen, req->iv); - aead_request_set_ad(subreq, req->assoclen); + src = scatterwalk_ffwd(_src, req->src, req->assoclen); - return crypto_aead_decrypt(subreq); - } + if (req->src == req->dst) + dst = src; + else + dst = scatterwalk_ffwd(_dst, req->dst, req->assoclen); + + if (starfive_aes_check_unaligned(cryp, src, dst)) + return starfive_aes_aead_do_fallback(req, is_encrypt(cryp)); return crypto_transfer_aead_request_to_engine(cryp->engine, req); } @@ -706,7 +867,7 @@ static int starfive_aes_setkey(struct crypto_skcipher *tfm, const u8 *key, memcpy(ctx->key, key, keylen); ctx->keylen = keylen; - return 0; + return crypto_skcipher_setkey(ctx->skcipher_fbk, key, keylen); } static int starfive_aes_aead_setkey(struct crypto_aead *tfm, const u8 *key, @@ -725,16 +886,20 @@ static int starfive_aes_aead_setkey(struct crypto_aead *tfm, const u8 *key, memcpy(ctx->key, key, keylen); ctx->keylen = keylen; - if (ctx->aead_fbk) - return crypto_aead_setkey(ctx->aead_fbk, key, keylen); - - return 0; + return crypto_aead_setkey(ctx->aead_fbk, key, keylen); } static int starfive_aes_gcm_setauthsize(struct crypto_aead *tfm, unsigned int authsize) { - return crypto_gcm_check_authsize(authsize); + struct starfive_cryp_ctx *ctx = crypto_aead_ctx(tfm); + int ret; + + ret = crypto_gcm_check_authsize(authsize); + if (ret) + return ret; + + return crypto_aead_setauthsize(ctx->aead_fbk, authsize); } static int starfive_aes_ccm_setauthsize(struct crypto_aead *tfm, @@ -820,9 +985,35 @@ static int starfive_aes_ccm_decrypt(struct aead_request *req) return starfive_aes_aead_crypt(req, STARFIVE_AES_MODE_CCM); } +static int starfive_aes_ecb_init_tfm(struct crypto_skcipher *tfm) +{ + return starfive_aes_init_tfm(tfm, "ecb(aes-generic)"); +} + +static int starfive_aes_cbc_init_tfm(struct crypto_skcipher *tfm) +{ + return starfive_aes_init_tfm(tfm, "cbc(aes-generic)"); +} + +static int starfive_aes_ctr_init_tfm(struct crypto_skcipher *tfm) +{ + return starfive_aes_init_tfm(tfm, "ctr(aes-generic)"); +} + +static int starfive_aes_ccm_init_tfm(struct crypto_aead *tfm) +{ + return starfive_aes_aead_init_tfm(tfm, "ccm_base(ctr(aes-generic),cbcmac(aes-generic))"); +} + +static int starfive_aes_gcm_init_tfm(struct crypto_aead *tfm) +{ + return starfive_aes_aead_init_tfm(tfm, "gcm_base(ctr(aes-generic),ghash-generic)"); +} + static struct skcipher_engine_alg skcipher_algs[] = { { - .base.init = starfive_aes_init_tfm, + .base.init = starfive_aes_ecb_init_tfm, + .base.exit = starfive_aes_exit_tfm, .base.setkey = starfive_aes_setkey, .base.encrypt = starfive_aes_ecb_encrypt, .base.decrypt = starfive_aes_ecb_decrypt, @@ -832,7 +1023,8 @@ static struct skcipher_engine_alg skcipher_algs[] = { .cra_name = "ecb(aes)", .cra_driver_name = "starfive-ecb-aes", .cra_priority = 200, - .cra_flags = CRYPTO_ALG_ASYNC, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK, .cra_blocksize = AES_BLOCK_SIZE, .cra_ctxsize = sizeof(struct starfive_cryp_ctx), .cra_alignmask = 0xf, @@ -842,7 +1034,8 @@ static struct skcipher_engine_alg skcipher_algs[] = { .do_one_request = starfive_aes_do_one_req, }, }, { - .base.init = starfive_aes_init_tfm, + .base.init = starfive_aes_cbc_init_tfm, + .base.exit = starfive_aes_exit_tfm, .base.setkey = starfive_aes_setkey, .base.encrypt = starfive_aes_cbc_encrypt, .base.decrypt = starfive_aes_cbc_decrypt, @@ -853,7 +1046,8 @@ static struct skcipher_engine_alg skcipher_algs[] = { .cra_name = "cbc(aes)", .cra_driver_name = "starfive-cbc-aes", .cra_priority = 200, - .cra_flags = CRYPTO_ALG_ASYNC, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK, .cra_blocksize = AES_BLOCK_SIZE, .cra_ctxsize = sizeof(struct starfive_cryp_ctx), .cra_alignmask = 0xf, @@ -863,7 +1057,8 @@ static struct skcipher_engine_alg skcipher_algs[] = { .do_one_request = starfive_aes_do_one_req, }, }, { - .base.init = starfive_aes_init_tfm, + .base.init = starfive_aes_ctr_init_tfm, + .base.exit = starfive_aes_exit_tfm, .base.setkey = starfive_aes_setkey, .base.encrypt = starfive_aes_ctr_encrypt, .base.decrypt = starfive_aes_ctr_decrypt, @@ -874,7 +1069,8 @@ static struct skcipher_engine_alg skcipher_algs[] = { .cra_name = "ctr(aes)", .cra_driver_name = "starfive-ctr-aes", .cra_priority = 200, - .cra_flags = CRYPTO_ALG_ASYNC, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK, .cra_blocksize = 1, .cra_ctxsize = sizeof(struct starfive_cryp_ctx), .cra_alignmask = 0xf, @@ -892,7 +1088,7 @@ static struct aead_engine_alg aead_algs[] = { .base.setauthsize = starfive_aes_gcm_setauthsize, .base.encrypt = starfive_aes_gcm_encrypt, .base.decrypt = starfive_aes_gcm_decrypt, - .base.init = starfive_aes_aead_init_tfm, + .base.init = starfive_aes_gcm_init_tfm, .base.exit = starfive_aes_aead_exit_tfm, .base.ivsize = GCM_AES_IV_SIZE, .base.maxauthsize = AES_BLOCK_SIZE, @@ -900,7 +1096,8 @@ static struct aead_engine_alg aead_algs[] = { .cra_name = "gcm(aes)", .cra_driver_name = "starfive-gcm-aes", .cra_priority = 200, - .cra_flags = CRYPTO_ALG_ASYNC, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK, .cra_blocksize = 1, .cra_ctxsize = sizeof(struct starfive_cryp_ctx), .cra_alignmask = 0xf, @@ -914,7 +1111,7 @@ static struct aead_engine_alg aead_algs[] = { .base.setauthsize = starfive_aes_ccm_setauthsize, .base.encrypt = starfive_aes_ccm_encrypt, .base.decrypt = starfive_aes_ccm_decrypt, - .base.init = starfive_aes_aead_init_tfm, + .base.init = starfive_aes_ccm_init_tfm, .base.exit = starfive_aes_aead_exit_tfm, .base.ivsize = AES_BLOCK_SIZE, .base.maxauthsize = AES_BLOCK_SIZE, diff --git a/drivers/crypto/starfive/jh7110-cryp.c b/drivers/crypto/starfive/jh7110-cryp.c index 425fddf3a8ab..e4dfed7ee0b0 100644 --- a/drivers/crypto/starfive/jh7110-cryp.c +++ b/drivers/crypto/starfive/jh7110-cryp.c @@ -89,34 +89,10 @@ static void starfive_dma_cleanup(struct starfive_cryp_dev *cryp) dma_release_channel(cryp->rx); } -static irqreturn_t starfive_cryp_irq(int irq, void *priv) -{ - u32 status; - u32 mask; - struct starfive_cryp_dev *cryp = (struct starfive_cryp_dev *)priv; - - mask = readl(cryp->base + STARFIVE_IE_MASK_OFFSET); - status = readl(cryp->base + STARFIVE_IE_FLAG_OFFSET); - if (status & STARFIVE_IE_FLAG_AES_DONE) { - mask |= STARFIVE_IE_MASK_AES_DONE; - writel(mask, cryp->base + STARFIVE_IE_MASK_OFFSET); - tasklet_schedule(&cryp->aes_done); - } - - if (status & STARFIVE_IE_FLAG_HASH_DONE) { - mask |= STARFIVE_IE_MASK_HASH_DONE; - writel(mask, cryp->base + STARFIVE_IE_MASK_OFFSET); - tasklet_schedule(&cryp->hash_done); - } - - return IRQ_HANDLED; -} - static int starfive_cryp_probe(struct platform_device *pdev) { struct starfive_cryp_dev *cryp; struct resource *res; - int irq; int ret; cryp = devm_kzalloc(&pdev->dev, sizeof(*cryp), GFP_KERNEL); @@ -131,9 +107,6 @@ static int starfive_cryp_probe(struct platform_device *pdev) return dev_err_probe(&pdev->dev, PTR_ERR(cryp->base), "Error remapping memory for platform device\n"); - tasklet_init(&cryp->aes_done, starfive_aes_done_task, (unsigned long)cryp); - tasklet_init(&cryp->hash_done, starfive_hash_done_task, (unsigned long)cryp); - cryp->phys_base = res->start; cryp->dma_maxburst = 32; cryp->side_chan = side_chan; @@ -153,16 +126,6 @@ static int starfive_cryp_probe(struct platform_device *pdev) return dev_err_probe(&pdev->dev, PTR_ERR(cryp->rst), "Error getting hardware reset line\n"); - irq = platform_get_irq(pdev, 0); - if (irq < 0) - return irq; - - ret = devm_request_irq(&pdev->dev, irq, starfive_cryp_irq, 0, pdev->name, - (void *)cryp); - if (ret) - return dev_err_probe(&pdev->dev, ret, - "Failed to register interrupt handler\n"); - clk_prepare_enable(cryp->hclk); clk_prepare_enable(cryp->ahb); reset_control_deassert(cryp->rst); @@ -219,9 +182,6 @@ err_dma_init: clk_disable_unprepare(cryp->ahb); reset_control_assert(cryp->rst); - tasklet_kill(&cryp->aes_done); - tasklet_kill(&cryp->hash_done); - return ret; } @@ -233,9 +193,6 @@ static void starfive_cryp_remove(struct platform_device *pdev) starfive_hash_unregister_algs(); starfive_rsa_unregister_algs(); - tasklet_kill(&cryp->aes_done); - tasklet_kill(&cryp->hash_done); - crypto_engine_stop(cryp->engine); crypto_engine_exit(cryp->engine); diff --git a/drivers/crypto/starfive/jh7110-cryp.h b/drivers/crypto/starfive/jh7110-cryp.h index 6cdf6db5d904..494a74f52706 100644 --- a/drivers/crypto/starfive/jh7110-cryp.h +++ b/drivers/crypto/starfive/jh7110-cryp.h @@ -91,6 +91,7 @@ union starfive_hash_csr { #define STARFIVE_HASH_KEY_DONE BIT(13) u32 key_done :1; u32 key_flag :1; +#define STARFIVE_HASH_HMAC_DONE BIT(15) u32 hmac_done :1; #define STARFIVE_HASH_BUSY BIT(16) u32 busy :1; @@ -168,6 +169,7 @@ struct starfive_cryp_ctx { struct crypto_akcipher *akcipher_fbk; struct crypto_ahash *ahash_fbk; struct crypto_aead *aead_fbk; + struct crypto_skcipher *skcipher_fbk; }; struct starfive_cryp_dev { @@ -185,11 +187,8 @@ struct starfive_cryp_dev { struct dma_chan *rx; struct dma_slave_config cfg_in; struct dma_slave_config cfg_out; - struct scatter_walk in_walk; - struct scatter_walk out_walk; struct crypto_engine *engine; - struct tasklet_struct aes_done; - struct tasklet_struct hash_done; + struct completion dma_done; size_t assoclen; size_t total_in; size_t total_out; @@ -236,7 +235,4 @@ void starfive_rsa_unregister_algs(void); int starfive_aes_register_algs(void); void starfive_aes_unregister_algs(void); - -void starfive_hash_done_task(unsigned long param); -void starfive_aes_done_task(unsigned long param); #endif diff --git a/drivers/crypto/starfive/jh7110-hash.c b/drivers/crypto/starfive/jh7110-hash.c index b6d1808012ca..2c60a1047bc3 100644 --- a/drivers/crypto/starfive/jh7110-hash.c +++ b/drivers/crypto/starfive/jh7110-hash.c @@ -36,15 +36,22 @@ #define STARFIVE_HASH_BUFLEN SHA512_BLOCK_SIZE #define STARFIVE_HASH_RESET 0x2 -static inline int starfive_hash_wait_busy(struct starfive_cryp_ctx *ctx) +static inline int starfive_hash_wait_busy(struct starfive_cryp_dev *cryp) { - struct starfive_cryp_dev *cryp = ctx->cryp; u32 status; return readl_relaxed_poll_timeout(cryp->base + STARFIVE_HASH_SHACSR, status, !(status & STARFIVE_HASH_BUSY), 10, 100000); } +static inline int starfive_hash_wait_hmac_done(struct starfive_cryp_dev *cryp) +{ + u32 status; + + return readl_relaxed_poll_timeout(cryp->base + STARFIVE_HASH_SHACSR, status, + (status & STARFIVE_HASH_HMAC_DONE), 10, 100000); +} + static inline int starfive_hash_wait_key_done(struct starfive_cryp_ctx *ctx) { struct starfive_cryp_dev *cryp = ctx->cryp; @@ -84,64 +91,26 @@ static int starfive_hash_hmac_key(struct starfive_cryp_ctx *ctx) return 0; } -static void starfive_hash_start(void *param) +static void starfive_hash_start(struct starfive_cryp_dev *cryp) { - struct starfive_cryp_ctx *ctx = param; - struct starfive_cryp_request_ctx *rctx = ctx->rctx; - struct starfive_cryp_dev *cryp = ctx->cryp; - union starfive_alg_cr alg_cr; union starfive_hash_csr csr; - u32 stat; - - dma_unmap_sg(cryp->dev, rctx->in_sg, rctx->in_sg_len, DMA_TO_DEVICE); - - alg_cr.v = 0; - alg_cr.clear = 1; - - writel(alg_cr.v, cryp->base + STARFIVE_ALG_CR_OFFSET); csr.v = readl(cryp->base + STARFIVE_HASH_SHACSR); csr.firstb = 0; csr.final = 1; - - stat = readl(cryp->base + STARFIVE_IE_MASK_OFFSET); - stat &= ~STARFIVE_IE_MASK_HASH_DONE; - writel(stat, cryp->base + STARFIVE_IE_MASK_OFFSET); writel(csr.v, cryp->base + STARFIVE_HASH_SHACSR); } -static int starfive_hash_xmit_dma(struct starfive_cryp_ctx *ctx) +static void starfive_hash_dma_callback(void *param) { - struct starfive_cryp_request_ctx *rctx = ctx->rctx; - struct starfive_cryp_dev *cryp = ctx->cryp; - struct dma_async_tx_descriptor *in_desc; - union starfive_alg_cr alg_cr; - int total_len; - int ret; - - if (!rctx->total) { - starfive_hash_start(ctx); - return 0; - } + struct starfive_cryp_dev *cryp = param; - writel(rctx->total, cryp->base + STARFIVE_DMA_IN_LEN_OFFSET); - - total_len = rctx->total; - total_len = (total_len & 0x3) ? (((total_len >> 2) + 1) << 2) : total_len; - sg_dma_len(rctx->in_sg) = total_len; - - alg_cr.v = 0; - alg_cr.start = 1; - alg_cr.hash_dma_en = 1; - - writel(alg_cr.v, cryp->base + STARFIVE_ALG_CR_OFFSET); - - ret = dma_map_sg(cryp->dev, rctx->in_sg, rctx->in_sg_len, DMA_TO_DEVICE); - if (!ret) - return dev_err_probe(cryp->dev, -EINVAL, "dma_map_sg() error\n"); + complete(&cryp->dma_done); +} - cryp->cfg_in.direction = DMA_MEM_TO_DEV; - cryp->cfg_in.src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES; +static void starfive_hash_dma_init(struct starfive_cryp_dev *cryp) +{ + cryp->cfg_in.src_addr_width = DMA_SLAVE_BUSWIDTH_16_BYTES; cryp->cfg_in.dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES; cryp->cfg_in.src_maxburst = cryp->dma_maxburst; cryp->cfg_in.dst_maxburst = cryp->dma_maxburst; @@ -149,50 +118,48 @@ static int starfive_hash_xmit_dma(struct starfive_cryp_ctx *ctx) dmaengine_slave_config(cryp->tx, &cryp->cfg_in); - in_desc = dmaengine_prep_slave_sg(cryp->tx, rctx->in_sg, - ret, DMA_MEM_TO_DEV, - DMA_PREP_INTERRUPT | DMA_CTRL_ACK); - - if (!in_desc) - return -EINVAL; - - in_desc->callback = starfive_hash_start; - in_desc->callback_param = ctx; - - dmaengine_submit(in_desc); - dma_async_issue_pending(cryp->tx); - - return 0; + init_completion(&cryp->dma_done); } -static int starfive_hash_xmit(struct starfive_cryp_ctx *ctx) +static int starfive_hash_dma_xfer(struct starfive_cryp_dev *cryp, + struct scatterlist *sg) { - struct starfive_cryp_request_ctx *rctx = ctx->rctx; - struct starfive_cryp_dev *cryp = ctx->cryp; + struct dma_async_tx_descriptor *in_desc; + union starfive_alg_cr alg_cr; int ret = 0; - rctx->csr.hash.v = 0; - rctx->csr.hash.reset = 1; - writel(rctx->csr.hash.v, cryp->base + STARFIVE_HASH_SHACSR); - - if (starfive_hash_wait_busy(ctx)) - return dev_err_probe(cryp->dev, -ETIMEDOUT, "Error resetting engine.\n"); + alg_cr.v = 0; + alg_cr.start = 1; + alg_cr.hash_dma_en = 1; + writel(alg_cr.v, cryp->base + STARFIVE_ALG_CR_OFFSET); - rctx->csr.hash.v = 0; - rctx->csr.hash.mode = ctx->hash_mode; - rctx->csr.hash.ie = 1; + writel(sg_dma_len(sg), cryp->base + STARFIVE_DMA_IN_LEN_OFFSET); + sg_dma_len(sg) = ALIGN(sg_dma_len(sg), sizeof(u32)); - if (ctx->is_hmac) { - ret = starfive_hash_hmac_key(ctx); - if (ret) - return ret; - } else { - rctx->csr.hash.start = 1; - rctx->csr.hash.firstb = 1; - writel(rctx->csr.hash.v, cryp->base + STARFIVE_HASH_SHACSR); + in_desc = dmaengine_prep_slave_sg(cryp->tx, sg, 1, DMA_MEM_TO_DEV, + DMA_PREP_INTERRUPT | DMA_CTRL_ACK); + if (!in_desc) { + ret = -EINVAL; + goto end; } - return starfive_hash_xmit_dma(ctx); + reinit_completion(&cryp->dma_done); + in_desc->callback = starfive_hash_dma_callback; + in_desc->callback_param = cryp; + + dmaengine_submit(in_desc); + dma_async_issue_pending(cryp->tx); + + if (!wait_for_completion_timeout(&cryp->dma_done, + msecs_to_jiffies(1000))) + ret = -ETIMEDOUT; + +end: + alg_cr.v = 0; + alg_cr.clear = 1; + writel(alg_cr.v, cryp->base + STARFIVE_ALG_CR_OFFSET); + + return ret; } static int starfive_hash_copy_hash(struct ahash_request *req) @@ -215,58 +182,74 @@ static int starfive_hash_copy_hash(struct ahash_request *req) return 0; } -void starfive_hash_done_task(unsigned long param) +static void starfive_hash_done_task(struct starfive_cryp_dev *cryp) { - struct starfive_cryp_dev *cryp = (struct starfive_cryp_dev *)param; int err = cryp->err; if (!err) err = starfive_hash_copy_hash(cryp->req.hreq); - /* Reset to clear hash_done in irq register*/ - writel(STARFIVE_HASH_RESET, cryp->base + STARFIVE_HASH_SHACSR); - crypto_finalize_hash_request(cryp->engine, cryp->req.hreq, err); } -static int starfive_hash_check_aligned(struct scatterlist *sg, size_t total, size_t align) +static int starfive_hash_one_request(struct crypto_engine *engine, void *areq) { - int len = 0; + struct ahash_request *req = container_of(areq, struct ahash_request, + base); + struct starfive_cryp_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(req)); + struct starfive_cryp_request_ctx *rctx = ctx->rctx; + struct starfive_cryp_dev *cryp = ctx->cryp; + struct scatterlist *tsg; + int ret, src_nents, i; - if (!total) - return 0; + writel(STARFIVE_HASH_RESET, cryp->base + STARFIVE_HASH_SHACSR); - if (!IS_ALIGNED(total, align)) - return -EINVAL; + if (starfive_hash_wait_busy(cryp)) + return dev_err_probe(cryp->dev, -ETIMEDOUT, "Error resetting hardware\n"); - while (sg) { - if (!IS_ALIGNED(sg->offset, sizeof(u32))) - return -EINVAL; + rctx->csr.hash.v = 0; + rctx->csr.hash.mode = ctx->hash_mode; - if (!IS_ALIGNED(sg->length, align)) - return -EINVAL; + if (ctx->is_hmac) { + ret = starfive_hash_hmac_key(ctx); + if (ret) + return ret; + } else { + rctx->csr.hash.start = 1; + rctx->csr.hash.firstb = 1; + writel(rctx->csr.hash.v, cryp->base + STARFIVE_HASH_SHACSR); + } + + /* No input message, get digest and end. */ + if (!rctx->total) + goto hash_start; + + starfive_hash_dma_init(cryp); + + for_each_sg(rctx->in_sg, tsg, rctx->in_sg_len, i) { + src_nents = dma_map_sg(cryp->dev, tsg, 1, DMA_TO_DEVICE); + if (src_nents == 0) + return dev_err_probe(cryp->dev, -ENOMEM, + "dma_map_sg error\n"); - len += sg->length; - sg = sg_next(sg); + ret = starfive_hash_dma_xfer(cryp, tsg); + dma_unmap_sg(cryp->dev, tsg, 1, DMA_TO_DEVICE); + if (ret) + return ret; } - if (len != total) - return -EINVAL; +hash_start: + starfive_hash_start(cryp); - return 0; -} + if (starfive_hash_wait_busy(cryp)) + return dev_err_probe(cryp->dev, -ETIMEDOUT, "Error generating digest\n"); -static int starfive_hash_one_request(struct crypto_engine *engine, void *areq) -{ - struct ahash_request *req = container_of(areq, struct ahash_request, - base); - struct starfive_cryp_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(req)); - struct starfive_cryp_dev *cryp = ctx->cryp; + if (ctx->is_hmac) + cryp->err = starfive_hash_wait_hmac_done(cryp); - if (!cryp) - return -ENODEV; + starfive_hash_done_task(cryp); - return starfive_hash_xmit(ctx); + return 0; } static int starfive_hash_init(struct ahash_request *req) @@ -337,22 +320,6 @@ static int starfive_hash_finup(struct ahash_request *req) return crypto_ahash_finup(&rctx->ahash_fbk_req); } -static int starfive_hash_digest_fb(struct ahash_request *req) -{ - struct starfive_cryp_request_ctx *rctx = ahash_request_ctx(req); - struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); - struct starfive_cryp_ctx *ctx = crypto_ahash_ctx(tfm); - - ahash_request_set_tfm(&rctx->ahash_fbk_req, ctx->ahash_fbk); - ahash_request_set_callback(&rctx->ahash_fbk_req, req->base.flags, - req->base.complete, req->base.data); - - ahash_request_set_crypt(&rctx->ahash_fbk_req, req->src, - req->result, req->nbytes); - - return crypto_ahash_digest(&rctx->ahash_fbk_req); -} - static int starfive_hash_digest(struct ahash_request *req) { struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); @@ -370,9 +337,6 @@ static int starfive_hash_digest(struct ahash_request *req) rctx->in_sg_len = sg_nents_for_len(rctx->in_sg, rctx->total); ctx->rctx = rctx; - if (starfive_hash_check_aligned(rctx->in_sg, rctx->total, rctx->blksize)) - return starfive_hash_digest_fb(req); - return crypto_transfer_hash_request_to_engine(cryp->engine, req); } @@ -406,7 +370,8 @@ static int starfive_hash_import(struct ahash_request *req, const void *in) static int starfive_hash_init_tfm(struct crypto_ahash *hash, const char *alg_name, - unsigned int mode) + unsigned int mode, + bool is_hmac) { struct starfive_cryp_ctx *ctx = crypto_ahash_ctx(hash); @@ -426,7 +391,7 @@ static int starfive_hash_init_tfm(struct crypto_ahash *hash, crypto_ahash_set_reqsize(hash, sizeof(struct starfive_cryp_request_ctx) + crypto_ahash_reqsize(ctx->ahash_fbk)); - ctx->keylen = 0; + ctx->is_hmac = is_hmac; ctx->hash_mode = mode; return 0; @@ -529,81 +494,61 @@ static int starfive_hash_setkey(struct crypto_ahash *hash, static int starfive_sha224_init_tfm(struct crypto_ahash *hash) { return starfive_hash_init_tfm(hash, "sha224-generic", - STARFIVE_HASH_SHA224); + STARFIVE_HASH_SHA224, 0); } static int starfive_sha256_init_tfm(struct crypto_ahash *hash) { return starfive_hash_init_tfm(hash, "sha256-generic", - STARFIVE_HASH_SHA256); + STARFIVE_HASH_SHA256, 0); } static int starfive_sha384_init_tfm(struct crypto_ahash *hash) { return starfive_hash_init_tfm(hash, "sha384-generic", - STARFIVE_HASH_SHA384); + STARFIVE_HASH_SHA384, 0); } static int starfive_sha512_init_tfm(struct crypto_ahash *hash) { return starfive_hash_init_tfm(hash, "sha512-generic", - STARFIVE_HASH_SHA512); + STARFIVE_HASH_SHA512, 0); } static int starfive_sm3_init_tfm(struct crypto_ahash *hash) { return starfive_hash_init_tfm(hash, "sm3-generic", - STARFIVE_HASH_SM3); + STARFIVE_HASH_SM3, 0); } static int starfive_hmac_sha224_init_tfm(struct crypto_ahash *hash) { - struct starfive_cryp_ctx *ctx = crypto_ahash_ctx(hash); - - ctx->is_hmac = true; - return starfive_hash_init_tfm(hash, "hmac(sha224-generic)", - STARFIVE_HASH_SHA224); + STARFIVE_HASH_SHA224, 1); } static int starfive_hmac_sha256_init_tfm(struct crypto_ahash *hash) { - struct starfive_cryp_ctx *ctx = crypto_ahash_ctx(hash); - - ctx->is_hmac = true; - return starfive_hash_init_tfm(hash, "hmac(sha256-generic)", - STARFIVE_HASH_SHA256); + STARFIVE_HASH_SHA256, 1); } static int starfive_hmac_sha384_init_tfm(struct crypto_ahash *hash) { - struct starfive_cryp_ctx *ctx = crypto_ahash_ctx(hash); - - ctx->is_hmac = true; - return starfive_hash_init_tfm(hash, "hmac(sha384-generic)", - STARFIVE_HASH_SHA384); + STARFIVE_HASH_SHA384, 1); } static int starfive_hmac_sha512_init_tfm(struct crypto_ahash *hash) { - struct starfive_cryp_ctx *ctx = crypto_ahash_ctx(hash); - - ctx->is_hmac = true; - return starfive_hash_init_tfm(hash, "hmac(sha512-generic)", - STARFIVE_HASH_SHA512); + STARFIVE_HASH_SHA512, 1); } static int starfive_hmac_sm3_init_tfm(struct crypto_ahash *hash) { - struct starfive_cryp_ctx *ctx = crypto_ahash_ctx(hash); - - ctx->is_hmac = true; - return starfive_hash_init_tfm(hash, "hmac(sm3-generic)", - STARFIVE_HASH_SM3); + STARFIVE_HASH_SM3, 1); } static struct ahash_engine_alg algs_sha2_sm3[] = { diff --git a/drivers/crypto/starfive/jh7110-rsa.c b/drivers/crypto/starfive/jh7110-rsa.c index cf8bda7f0855..33093ba4b13a 100644 --- a/drivers/crypto/starfive/jh7110-rsa.c +++ b/drivers/crypto/starfive/jh7110-rsa.c @@ -45,6 +45,9 @@ static inline int starfive_pka_wait_done(struct starfive_cryp_ctx *ctx) static void starfive_rsa_free_key(struct starfive_rsa_key *key) { + if (!key->key_sz) + return; + kfree_sensitive(key->d); kfree_sensitive(key->e); kfree_sensitive(key->n); @@ -273,7 +276,6 @@ static int starfive_rsa_enc_core(struct starfive_cryp_ctx *ctx, int enc) err_rsa_crypt: writel(STARFIVE_RSA_RESET, cryp->base + STARFIVE_PKA_CACR_OFFSET); - kfree(rctx->rsa_data); return ret; } @@ -534,16 +536,14 @@ static int starfive_rsa_init_tfm(struct crypto_akcipher *tfm) { struct starfive_cryp_ctx *ctx = akcipher_tfm_ctx(tfm); + ctx->cryp = starfive_cryp_find_dev(ctx); + if (!ctx->cryp) + return -ENODEV; + ctx->akcipher_fbk = crypto_alloc_akcipher("rsa-generic", 0, 0); if (IS_ERR(ctx->akcipher_fbk)) return PTR_ERR(ctx->akcipher_fbk); - ctx->cryp = starfive_cryp_find_dev(ctx); - if (!ctx->cryp) { - crypto_free_akcipher(ctx->akcipher_fbk); - return -ENODEV; - } - akcipher_set_reqsize(tfm, sizeof(struct starfive_cryp_request_ctx) + sizeof(struct crypto_akcipher) + 32); diff --git a/drivers/crypto/stm32/stm32-hash.c b/drivers/crypto/stm32/stm32-hash.c index 34e0d7e381a8..351827372ea6 100644 --- a/drivers/crypto/stm32/stm32-hash.c +++ b/drivers/crypto/stm32/stm32-hash.c @@ -94,6 +94,7 @@ #define HASH_FLAGS_ERRORS BIT(21) #define HASH_FLAGS_EMPTY BIT(22) #define HASH_FLAGS_HMAC BIT(23) +#define HASH_FLAGS_SGS_COPIED BIT(24) #define HASH_OP_UPDATE 1 #define HASH_OP_FINAL 2 @@ -145,7 +146,7 @@ struct stm32_hash_state { u16 bufcnt; u16 blocklen; - u8 buffer[HASH_BUFLEN] __aligned(4); + u8 buffer[HASH_BUFLEN] __aligned(sizeof(u32)); /* hash state */ u32 hw_context[3 + HASH_CSR_NB_MAX]; @@ -158,8 +159,8 @@ struct stm32_hash_request_ctx { u8 digest[SHA512_DIGEST_SIZE] __aligned(sizeof(u32)); size_t digcnt; - /* DMA */ struct scatterlist *sg; + struct scatterlist sgl[2]; /* scatterlist used to realize alignment */ unsigned int offset; unsigned int total; struct scatterlist sg_key; @@ -184,6 +185,7 @@ struct stm32_hash_pdata { size_t algs_info_size; bool has_sr; bool has_mdmat; + bool context_secured; bool broken_emptymsg; bool ux500; }; @@ -195,6 +197,7 @@ struct stm32_hash_dev { struct reset_control *rst; void __iomem *io_base; phys_addr_t phys_base; + u8 xmit_buf[HASH_BUFLEN] __aligned(sizeof(u32)); u32 dma_mode; bool polled; @@ -220,6 +223,8 @@ static struct stm32_hash_drv stm32_hash = { }; static void stm32_hash_dma_callback(void *param); +static int stm32_hash_prepare_request(struct ahash_request *req); +static void stm32_hash_unprepare_request(struct ahash_request *req); static inline u32 stm32_hash_read(struct stm32_hash_dev *hdev, u32 offset) { @@ -232,6 +237,11 @@ static inline void stm32_hash_write(struct stm32_hash_dev *hdev, writel_relaxed(value, hdev->io_base + offset); } +/** + * stm32_hash_wait_busy - wait until hash processor is available. It return an + * error if the hash core is processing a block of data for more than 10 ms. + * @hdev: the stm32_hash_dev device. + */ static inline int stm32_hash_wait_busy(struct stm32_hash_dev *hdev) { u32 status; @@ -245,6 +255,11 @@ static inline int stm32_hash_wait_busy(struct stm32_hash_dev *hdev) !(status & HASH_SR_BUSY), 10, 10000); } +/** + * stm32_hash_set_nblw - set the number of valid bytes in the last word. + * @hdev: the stm32_hash_dev device. + * @length: the length of the final word. + */ static void stm32_hash_set_nblw(struct stm32_hash_dev *hdev, int length) { u32 reg; @@ -282,6 +297,11 @@ static int stm32_hash_write_key(struct stm32_hash_dev *hdev) return 0; } +/** + * stm32_hash_write_ctrl - Initialize the hash processor, only if + * HASH_FLAGS_INIT is set. + * @hdev: the stm32_hash_dev device + */ static void stm32_hash_write_ctrl(struct stm32_hash_dev *hdev) { struct stm32_hash_request_ctx *rctx = ahash_request_ctx(hdev->req); @@ -469,9 +489,7 @@ static int stm32_hash_update_cpu(struct stm32_hash_dev *hdev) { struct stm32_hash_request_ctx *rctx = ahash_request_ctx(hdev->req); struct stm32_hash_state *state = &rctx->state; - u32 *preg = state->hw_context; int bufcnt, err = 0, final; - int i, swap_reg; dev_dbg(hdev->dev, "%s flags %x\n", __func__, state->flags); @@ -495,34 +513,23 @@ static int stm32_hash_update_cpu(struct stm32_hash_dev *hdev) return stm32_hash_xmit_cpu(hdev, state->buffer, bufcnt, 1); } - if (!(hdev->flags & HASH_FLAGS_INIT)) - return 0; - - if (stm32_hash_wait_busy(hdev)) - return -ETIMEDOUT; - - swap_reg = hash_swap_reg(rctx); - - if (!hdev->pdata->ux500) - *preg++ = stm32_hash_read(hdev, HASH_IMR); - *preg++ = stm32_hash_read(hdev, HASH_STR); - *preg++ = stm32_hash_read(hdev, HASH_CR); - for (i = 0; i < swap_reg; i++) - *preg++ = stm32_hash_read(hdev, HASH_CSR(i)); - - state->flags |= HASH_FLAGS_INIT; - return err; } static int stm32_hash_xmit_dma(struct stm32_hash_dev *hdev, - struct scatterlist *sg, int length, int mdma) + struct scatterlist *sg, int length, int mdmat) { struct dma_async_tx_descriptor *in_desc; dma_cookie_t cookie; u32 reg; int err; + dev_dbg(hdev->dev, "%s mdmat: %x length: %d\n", __func__, mdmat, length); + + /* do not use dma if there is no data to send */ + if (length <= 0) + return 0; + in_desc = dmaengine_prep_slave_sg(hdev->dma_lch, sg, 1, DMA_MEM_TO_DEV, DMA_PREP_INTERRUPT | DMA_CTRL_ACK); @@ -535,13 +542,12 @@ static int stm32_hash_xmit_dma(struct stm32_hash_dev *hdev, in_desc->callback = stm32_hash_dma_callback; in_desc->callback_param = hdev; - hdev->flags |= HASH_FLAGS_FINAL; hdev->flags |= HASH_FLAGS_DMA_ACTIVE; reg = stm32_hash_read(hdev, HASH_CR); if (hdev->pdata->has_mdmat) { - if (mdma) + if (mdmat) reg |= HASH_CR_MDMAT; else reg &= ~HASH_CR_MDMAT; @@ -550,7 +556,6 @@ static int stm32_hash_xmit_dma(struct stm32_hash_dev *hdev, stm32_hash_write(hdev, HASH_CR, reg); - stm32_hash_set_nblw(hdev, length); cookie = dmaengine_submit(in_desc); err = dma_submit_error(cookie); @@ -590,7 +595,7 @@ static int stm32_hash_hmac_dma_send(struct stm32_hash_dev *hdev) struct stm32_hash_ctx *ctx = crypto_ahash_ctx(tfm); int err; - if (ctx->keylen < rctx->state.blocklen || hdev->dma_mode == 1) { + if (ctx->keylen < rctx->state.blocklen || hdev->dma_mode > 0) { err = stm32_hash_write_key(hdev); if (stm32_hash_wait_busy(hdev)) return -ETIMEDOUT; @@ -655,18 +660,20 @@ static int stm32_hash_dma_send(struct stm32_hash_dev *hdev) struct scatterlist sg[1], *tsg; int err = 0, reg, ncp = 0; unsigned int i, len = 0, bufcnt = 0; + bool final = hdev->flags & HASH_FLAGS_FINAL; bool is_last = false; + u32 last_word; - rctx->sg = hdev->req->src; - rctx->total = hdev->req->nbytes; + dev_dbg(hdev->dev, "%s total: %d bufcnt: %d final: %d\n", + __func__, rctx->total, rctx->state.bufcnt, final); - rctx->nents = sg_nents(rctx->sg); if (rctx->nents < 0) return -EINVAL; stm32_hash_write_ctrl(hdev); - if (hdev->flags & HASH_FLAGS_HMAC) { + if (hdev->flags & HASH_FLAGS_HMAC && (!(hdev->flags & HASH_FLAGS_HMAC_KEY))) { + hdev->flags |= HASH_FLAGS_HMAC_KEY; err = stm32_hash_hmac_dma_send(hdev); if (err != -EINPROGRESS) return err; @@ -677,22 +684,36 @@ static int stm32_hash_dma_send(struct stm32_hash_dev *hdev) len = sg->length; if (sg_is_last(sg) || (bufcnt + sg[0].length) >= rctx->total) { - sg->length = rctx->total - bufcnt; - is_last = true; - if (hdev->dma_mode == 1) { - len = (ALIGN(sg->length, 16) - 16); - - ncp = sg_pcopy_to_buffer( - rctx->sg, rctx->nents, - rctx->state.buffer, sg->length - len, - rctx->total - sg->length + len); - - sg->length = len; + if (!final) { + /* Always manually put the last word of a non-final transfer. */ + len -= sizeof(u32); + sg_pcopy_to_buffer(rctx->sg, rctx->nents, &last_word, 4, len); + sg->length -= sizeof(u32); } else { - if (!(IS_ALIGNED(sg->length, sizeof(u32)))) { - len = sg->length; - sg->length = ALIGN(sg->length, - sizeof(u32)); + /* + * In Multiple DMA mode, DMA must be aborted before the final + * transfer. + */ + sg->length = rctx->total - bufcnt; + if (hdev->dma_mode > 0) { + len = (ALIGN(sg->length, 16) - 16); + + ncp = sg_pcopy_to_buffer(rctx->sg, rctx->nents, + rctx->state.buffer, + sg->length - len, + rctx->total - sg->length + len); + + if (!len) + break; + + sg->length = len; + } else { + is_last = true; + if (!(IS_ALIGNED(sg->length, sizeof(u32)))) { + len = sg->length; + sg->length = ALIGN(sg->length, + sizeof(u32)); + } } } } @@ -706,43 +727,67 @@ static int stm32_hash_dma_send(struct stm32_hash_dev *hdev) err = stm32_hash_xmit_dma(hdev, sg, len, !is_last); + /* The last word of a non final transfer is sent manually. */ + if (!final) { + stm32_hash_write(hdev, HASH_DIN, last_word); + len += sizeof(u32); + } + + rctx->total -= len; + bufcnt += sg[0].length; dma_unmap_sg(hdev->dev, sg, 1, DMA_TO_DEVICE); - if (err == -ENOMEM) + if (err == -ENOMEM || err == -ETIMEDOUT) return err; if (is_last) break; } - if (hdev->dma_mode == 1) { - if (stm32_hash_wait_busy(hdev)) - return -ETIMEDOUT; - reg = stm32_hash_read(hdev, HASH_CR); - reg &= ~HASH_CR_DMAE; - reg |= HASH_CR_DMAA; - stm32_hash_write(hdev, HASH_CR, reg); + /* + * When the second last block transfer of 4 words is performed by the DMA, + * the software must set the DMA Abort bit (DMAA) to 1 before completing the + * last transfer of 4 words or less. + */ + if (final) { + if (hdev->dma_mode > 0) { + if (stm32_hash_wait_busy(hdev)) + return -ETIMEDOUT; + reg = stm32_hash_read(hdev, HASH_CR); + reg &= ~HASH_CR_DMAE; + reg |= HASH_CR_DMAA; + stm32_hash_write(hdev, HASH_CR, reg); + + if (ncp) { + memset(buffer + ncp, 0, 4 - DIV_ROUND_UP(ncp, sizeof(u32))); + writesl(hdev->io_base + HASH_DIN, buffer, + DIV_ROUND_UP(ncp, sizeof(u32))); + } - if (ncp) { - memset(buffer + ncp, 0, - DIV_ROUND_UP(ncp, sizeof(u32)) - ncp); - writesl(hdev->io_base + HASH_DIN, buffer, - DIV_ROUND_UP(ncp, sizeof(u32))); + stm32_hash_set_nblw(hdev, ncp); + reg = stm32_hash_read(hdev, HASH_STR); + reg |= HASH_STR_DCAL; + stm32_hash_write(hdev, HASH_STR, reg); + err = -EINPROGRESS; } - stm32_hash_set_nblw(hdev, ncp); - reg = stm32_hash_read(hdev, HASH_STR); - reg |= HASH_STR_DCAL; - stm32_hash_write(hdev, HASH_STR, reg); - err = -EINPROGRESS; - } - if (hdev->flags & HASH_FLAGS_HMAC) { - if (stm32_hash_wait_busy(hdev)) - return -ETIMEDOUT; - err = stm32_hash_hmac_dma_send(hdev); + /* + * The hash processor needs the key to be loaded a second time in order + * to process the HMAC. + */ + if (hdev->flags & HASH_FLAGS_HMAC) { + if (stm32_hash_wait_busy(hdev)) + return -ETIMEDOUT; + err = stm32_hash_hmac_dma_send(hdev); + } + + return err; } - return err; + if (err != -EINPROGRESS) + return err; + + return 0; } static struct stm32_hash_dev *stm32_hash_find_dev(struct stm32_hash_ctx *ctx) @@ -765,33 +810,6 @@ static struct stm32_hash_dev *stm32_hash_find_dev(struct stm32_hash_ctx *ctx) return hdev; } -static bool stm32_hash_dma_aligned_data(struct ahash_request *req) -{ - struct scatterlist *sg; - struct stm32_hash_request_ctx *rctx = ahash_request_ctx(req); - struct stm32_hash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(req)); - struct stm32_hash_dev *hdev = stm32_hash_find_dev(ctx); - int i; - - if (!hdev->dma_lch || req->nbytes <= rctx->state.blocklen) - return false; - - if (sg_nents(req->src) > 1) { - if (hdev->dma_mode == 1) - return false; - for_each_sg(req->src, sg, sg_nents(req->src), i) { - if ((!IS_ALIGNED(sg->length, sizeof(u32))) && - (!sg_is_last(sg))) - return false; - } - } - - if (req->src->offset % 4) - return false; - - return true; -} - static int stm32_hash_init(struct ahash_request *req) { struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); @@ -802,8 +820,10 @@ static int stm32_hash_init(struct ahash_request *req) bool sha3_mode = ctx->flags & HASH_FLAGS_SHA3_MODE; rctx->hdev = hdev; + state->flags = 0; - state->flags = HASH_FLAGS_CPU; + if (!(hdev->dma_lch && hdev->pdata->has_mdmat)) + state->flags |= HASH_FLAGS_CPU; if (sha3_mode) state->flags |= HASH_FLAGS_SHA3_MODE; @@ -857,6 +877,7 @@ static int stm32_hash_init(struct ahash_request *req) dev_err(hdev->dev, "Error, block too large"); return -EINVAL; } + rctx->nents = 0; rctx->total = 0; rctx->offset = 0; rctx->data_type = HASH_DATA_8_BITS; @@ -874,6 +895,9 @@ static int stm32_hash_update_req(struct stm32_hash_dev *hdev) struct stm32_hash_request_ctx *rctx = ahash_request_ctx(hdev->req); struct stm32_hash_state *state = &rctx->state; + dev_dbg(hdev->dev, "update_req: total: %u, digcnt: %zd, final: 0", + rctx->total, rctx->digcnt); + if (!(state->flags & HASH_FLAGS_CPU)) return stm32_hash_dma_send(hdev); @@ -887,6 +911,11 @@ static int stm32_hash_final_req(struct stm32_hash_dev *hdev) struct stm32_hash_state *state = &rctx->state; int buflen = state->bufcnt; + if (!(state->flags & HASH_FLAGS_CPU)) { + hdev->flags |= HASH_FLAGS_FINAL; + return stm32_hash_dma_send(hdev); + } + if (state->flags & HASH_FLAGS_FINUP) return stm32_hash_update_req(hdev); @@ -968,15 +997,21 @@ static int stm32_hash_finish(struct ahash_request *req) static void stm32_hash_finish_req(struct ahash_request *req, int err) { struct stm32_hash_request_ctx *rctx = ahash_request_ctx(req); + struct stm32_hash_state *state = &rctx->state; struct stm32_hash_dev *hdev = rctx->hdev; + if (hdev->flags & HASH_FLAGS_DMA_ACTIVE) + state->flags |= HASH_FLAGS_DMA_ACTIVE; + else + state->flags &= ~HASH_FLAGS_DMA_ACTIVE; + if (!err && (HASH_FLAGS_FINAL & hdev->flags)) { stm32_hash_copy_hash(req); err = stm32_hash_finish(req); } - pm_runtime_mark_last_busy(hdev->dev); - pm_runtime_put_autosuspend(hdev->dev); + /* Finalized request mist be unprepared here */ + stm32_hash_unprepare_request(req); crypto_finalize_hash_request(hdev->engine, req, err); } @@ -1006,6 +1041,10 @@ static int stm32_hash_one_request(struct crypto_engine *engine, void *areq) pm_runtime_get_sync(hdev->dev); + err = stm32_hash_prepare_request(req); + if (err) + return err; + hdev->req = req; hdev->flags = 0; swap_reg = hash_swap_reg(rctx); @@ -1030,6 +1069,12 @@ static int stm32_hash_one_request(struct crypto_engine *engine, void *areq) if (state->flags & HASH_FLAGS_HMAC) hdev->flags |= HASH_FLAGS_HMAC | HASH_FLAGS_HMAC_KEY; + + if (state->flags & HASH_FLAGS_CPU) + hdev->flags |= HASH_FLAGS_CPU; + + if (state->flags & HASH_FLAGS_DMA_ACTIVE) + hdev->flags |= HASH_FLAGS_DMA_ACTIVE; } if (rctx->op == HASH_OP_UPDATE) @@ -1054,6 +1099,284 @@ static int stm32_hash_one_request(struct crypto_engine *engine, void *areq) return 0; } +static int stm32_hash_copy_sgs(struct stm32_hash_request_ctx *rctx, + struct scatterlist *sg, int bs, + unsigned int new_len) +{ + struct stm32_hash_state *state = &rctx->state; + int pages; + void *buf; + + pages = get_order(new_len); + + buf = (void *)__get_free_pages(GFP_ATOMIC, pages); + if (!buf) { + pr_err("Couldn't allocate pages for unaligned cases.\n"); + return -ENOMEM; + } + + if (state->bufcnt) + memcpy(buf, rctx->hdev->xmit_buf, state->bufcnt); + + scatterwalk_map_and_copy(buf + state->bufcnt, sg, rctx->offset, + min(new_len, rctx->total) - state->bufcnt, 0); + sg_init_table(rctx->sgl, 1); + sg_set_buf(rctx->sgl, buf, new_len); + rctx->sg = rctx->sgl; + state->flags |= HASH_FLAGS_SGS_COPIED; + rctx->nents = 1; + rctx->offset += new_len - state->bufcnt; + state->bufcnt = 0; + rctx->total = new_len; + + return 0; +} + +static int stm32_hash_align_sgs(struct scatterlist *sg, + int nbytes, int bs, bool init, bool final, + struct stm32_hash_request_ctx *rctx) +{ + struct stm32_hash_state *state = &rctx->state; + struct stm32_hash_dev *hdev = rctx->hdev; + struct scatterlist *sg_tmp = sg; + int offset = rctx->offset; + int new_len; + int n = 0; + int bufcnt = state->bufcnt; + bool secure_ctx = hdev->pdata->context_secured; + bool aligned = true; + + if (!sg || !sg->length || !nbytes) { + if (bufcnt) { + bufcnt = DIV_ROUND_UP(bufcnt, bs) * bs; + sg_init_table(rctx->sgl, 1); + sg_set_buf(rctx->sgl, rctx->hdev->xmit_buf, bufcnt); + rctx->sg = rctx->sgl; + rctx->nents = 1; + } + + return 0; + } + + new_len = nbytes; + + if (offset) + aligned = false; + + if (final) { + new_len = DIV_ROUND_UP(new_len, bs) * bs; + } else { + new_len = (new_len - 1) / bs * bs; // return n block - 1 block + + /* + * Context save in some version of HASH IP can only be done when the + * FIFO is ready to get a new block. This implies to send n block plus a + * 32 bit word in the first DMA send. + */ + if (init && secure_ctx) { + new_len += sizeof(u32); + if (unlikely(new_len > nbytes)) + new_len -= bs; + } + } + + if (!new_len) + return 0; + + if (nbytes != new_len) + aligned = false; + + while (nbytes > 0 && sg_tmp) { + n++; + + if (bufcnt) { + if (!IS_ALIGNED(bufcnt, bs)) { + aligned = false; + break; + } + nbytes -= bufcnt; + bufcnt = 0; + if (!nbytes) + aligned = false; + + continue; + } + + if (offset < sg_tmp->length) { + if (!IS_ALIGNED(offset + sg_tmp->offset, 4)) { + aligned = false; + break; + } + + if (!IS_ALIGNED(sg_tmp->length - offset, bs)) { + aligned = false; + break; + } + } + + if (offset) { + offset -= sg_tmp->length; + if (offset < 0) { + nbytes += offset; + offset = 0; + } + } else { + nbytes -= sg_tmp->length; + } + + sg_tmp = sg_next(sg_tmp); + + if (nbytes < 0) { + aligned = false; + break; + } + } + + if (!aligned) + return stm32_hash_copy_sgs(rctx, sg, bs, new_len); + + rctx->total = new_len; + rctx->offset += new_len; + rctx->nents = n; + if (state->bufcnt) { + sg_init_table(rctx->sgl, 2); + sg_set_buf(rctx->sgl, rctx->hdev->xmit_buf, state->bufcnt); + sg_chain(rctx->sgl, 2, sg); + rctx->sg = rctx->sgl; + } else { + rctx->sg = sg; + } + + return 0; +} + +static int stm32_hash_prepare_request(struct ahash_request *req) +{ + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct stm32_hash_ctx *ctx = crypto_ahash_ctx(tfm); + struct stm32_hash_request_ctx *rctx = ahash_request_ctx(req); + struct stm32_hash_dev *hdev = stm32_hash_find_dev(ctx); + struct stm32_hash_state *state = &rctx->state; + unsigned int nbytes; + int ret, hash_later, bs; + bool update = rctx->op & HASH_OP_UPDATE; + bool init = !(state->flags & HASH_FLAGS_INIT); + bool finup = state->flags & HASH_FLAGS_FINUP; + bool final = state->flags & HASH_FLAGS_FINAL; + + if (!hdev->dma_lch || state->flags & HASH_FLAGS_CPU) + return 0; + + bs = crypto_ahash_blocksize(tfm); + + nbytes = state->bufcnt; + + /* + * In case of update request nbytes must correspond to the content of the + * buffer + the offset minus the content of the request already in the + * buffer. + */ + if (update || finup) + nbytes += req->nbytes - rctx->offset; + + dev_dbg(hdev->dev, + "%s: nbytes=%d, bs=%d, total=%d, offset=%d, bufcnt=%d\n", + __func__, nbytes, bs, rctx->total, rctx->offset, state->bufcnt); + + if (!nbytes) + return 0; + + rctx->total = nbytes; + + if (update && req->nbytes && (!IS_ALIGNED(state->bufcnt, bs))) { + int len = bs - state->bufcnt % bs; + + if (len > req->nbytes) + len = req->nbytes; + scatterwalk_map_and_copy(state->buffer + state->bufcnt, req->src, + 0, len, 0); + state->bufcnt += len; + rctx->offset = len; + } + + /* copy buffer in a temporary one that is used for sg alignment */ + if (state->bufcnt) + memcpy(hdev->xmit_buf, state->buffer, state->bufcnt); + + ret = stm32_hash_align_sgs(req->src, nbytes, bs, init, final, rctx); + if (ret) + return ret; + + hash_later = nbytes - rctx->total; + if (hash_later < 0) + hash_later = 0; + + if (hash_later && hash_later <= state->blocklen) { + scatterwalk_map_and_copy(state->buffer, + req->src, + req->nbytes - hash_later, + hash_later, 0); + + state->bufcnt = hash_later; + } else { + state->bufcnt = 0; + } + + if (hash_later > state->blocklen) { + /* FIXME: add support of this case */ + pr_err("Buffer contains more than one block.\n"); + return -ENOMEM; + } + + rctx->total = min(nbytes, rctx->total); + + return 0; +} + +static void stm32_hash_unprepare_request(struct ahash_request *req) +{ + struct stm32_hash_request_ctx *rctx = ahash_request_ctx(req); + struct stm32_hash_state *state = &rctx->state; + struct stm32_hash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(req)); + struct stm32_hash_dev *hdev = stm32_hash_find_dev(ctx); + u32 *preg = state->hw_context; + int swap_reg, i; + + if (hdev->dma_lch) + dmaengine_terminate_sync(hdev->dma_lch); + + if (state->flags & HASH_FLAGS_SGS_COPIED) + free_pages((unsigned long)sg_virt(rctx->sg), get_order(rctx->sg->length)); + + rctx->sg = NULL; + rctx->offset = 0; + + state->flags &= ~(HASH_FLAGS_SGS_COPIED); + + if (!(hdev->flags & HASH_FLAGS_INIT)) + goto pm_runtime; + + state->flags |= HASH_FLAGS_INIT; + + if (stm32_hash_wait_busy(hdev)) { + dev_warn(hdev->dev, "Wait busy failed."); + return; + } + + swap_reg = hash_swap_reg(rctx); + + if (!hdev->pdata->ux500) + *preg++ = stm32_hash_read(hdev, HASH_IMR); + *preg++ = stm32_hash_read(hdev, HASH_STR); + *preg++ = stm32_hash_read(hdev, HASH_CR); + for (i = 0; i < swap_reg; i++) + *preg++ = stm32_hash_read(hdev, HASH_CSR(i)); + +pm_runtime: + pm_runtime_mark_last_busy(hdev->dev); + pm_runtime_put_autosuspend(hdev->dev); +} + static int stm32_hash_enqueue(struct ahash_request *req, unsigned int op) { struct stm32_hash_request_ctx *rctx = ahash_request_ctx(req); @@ -1070,16 +1393,26 @@ static int stm32_hash_update(struct ahash_request *req) struct stm32_hash_request_ctx *rctx = ahash_request_ctx(req); struct stm32_hash_state *state = &rctx->state; - if (!req->nbytes || !(state->flags & HASH_FLAGS_CPU)) + if (!req->nbytes) return 0; - rctx->total = req->nbytes; - rctx->sg = req->src; - rctx->offset = 0; - if ((state->bufcnt + rctx->total < state->blocklen)) { - stm32_hash_append_sg(rctx); - return 0; + if (state->flags & HASH_FLAGS_CPU) { + rctx->total = req->nbytes; + rctx->sg = req->src; + rctx->offset = 0; + + if ((state->bufcnt + rctx->total < state->blocklen)) { + stm32_hash_append_sg(rctx); + return 0; + } + } else { /* DMA mode */ + if (state->bufcnt + req->nbytes <= state->blocklen) { + scatterwalk_map_and_copy(state->buffer + state->bufcnt, req->src, + 0, req->nbytes, 0); + state->bufcnt += req->nbytes; + return 0; + } } return stm32_hash_enqueue(req, HASH_OP_UPDATE); @@ -1098,20 +1431,18 @@ static int stm32_hash_final(struct ahash_request *req) static int stm32_hash_finup(struct ahash_request *req) { struct stm32_hash_request_ctx *rctx = ahash_request_ctx(req); - struct stm32_hash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(req)); - struct stm32_hash_dev *hdev = stm32_hash_find_dev(ctx); struct stm32_hash_state *state = &rctx->state; if (!req->nbytes) goto out; state->flags |= HASH_FLAGS_FINUP; - rctx->total = req->nbytes; - rctx->sg = req->src; - rctx->offset = 0; - if (hdev->dma_lch && stm32_hash_dma_aligned_data(req)) - state->flags &= ~HASH_FLAGS_CPU; + if ((state->flags & HASH_FLAGS_CPU)) { + rctx->total = req->nbytes; + rctx->sg = req->src; + rctx->offset = 0; + } out: return stm32_hash_final(req); @@ -1215,7 +1546,6 @@ static int stm32_hash_cra_sha3_hmac_init(struct crypto_tfm *tfm) HASH_FLAGS_HMAC); } - static void stm32_hash_cra_exit(struct crypto_tfm *tfm) { struct stm32_hash_ctx *ctx = crypto_tfm_ctx(tfm); @@ -1228,14 +1558,9 @@ static irqreturn_t stm32_hash_irq_thread(int irq, void *dev_id) { struct stm32_hash_dev *hdev = dev_id; - if (HASH_FLAGS_CPU & hdev->flags) { - if (HASH_FLAGS_OUTPUT_READY & hdev->flags) { - hdev->flags &= ~HASH_FLAGS_OUTPUT_READY; - goto finish; - } - } else if (HASH_FLAGS_DMA_ACTIVE & hdev->flags) { - hdev->flags &= ~HASH_FLAGS_DMA_ACTIVE; - goto finish; + if (HASH_FLAGS_OUTPUT_READY & hdev->flags) { + hdev->flags &= ~HASH_FLAGS_OUTPUT_READY; + goto finish; } return IRQ_HANDLED; @@ -1984,6 +2309,7 @@ static const struct stm32_hash_pdata stm32_hash_pdata_stm32mp13 = { .algs_info_size = ARRAY_SIZE(stm32_hash_algs_info_stm32mp13), .has_sr = true, .has_mdmat = true, + .context_secured = true, }; static const struct of_device_id stm32_hash_of_match[] = { diff --git a/drivers/crypto/tegra/Makefile b/drivers/crypto/tegra/Makefile new file mode 100644 index 000000000000..a32001e58eb2 --- /dev/null +++ b/drivers/crypto/tegra/Makefile @@ -0,0 +1,9 @@ +// SPDX-License-Identifier: GPL-2.0-only +// SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + +tegra-se-objs := tegra-se-key.o tegra-se-main.o + +tegra-se-y += tegra-se-aes.o +tegra-se-y += tegra-se-hash.o + +obj-$(CONFIG_CRYPTO_DEV_TEGRA) += tegra-se.o diff --git a/drivers/crypto/tegra/tegra-se-aes.c b/drivers/crypto/tegra/tegra-se-aes.c new file mode 100644 index 000000000000..ae7a0f8435fc --- /dev/null +++ b/drivers/crypto/tegra/tegra-se-aes.c @@ -0,0 +1,1933 @@ +// SPDX-License-Identifier: GPL-2.0-only +// SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +/* + * Crypto driver to handle block cipher algorithms using NVIDIA Security Engine. + */ + +#include <linux/clk.h> +#include <linux/dma-mapping.h> +#include <linux/module.h> +#include <linux/of_device.h> +#include <linux/platform_device.h> + +#include <crypto/aead.h> +#include <crypto/aes.h> +#include <crypto/engine.h> +#include <crypto/gcm.h> +#include <crypto/scatterwalk.h> +#include <crypto/xts.h> +#include <crypto/internal/aead.h> +#include <crypto/internal/hash.h> +#include <crypto/internal/skcipher.h> + +#include "tegra-se.h" + +struct tegra_aes_ctx { + struct tegra_se *se; + u32 alg; + u32 ivsize; + u32 key1_id; + u32 key2_id; +}; + +struct tegra_aes_reqctx { + struct tegra_se_datbuf datbuf; + bool encrypt; + u32 config; + u32 crypto_config; + u32 len; + u32 *iv; +}; + +struct tegra_aead_ctx { + struct tegra_se *se; + unsigned int authsize; + u32 alg; + u32 keylen; + u32 key_id; +}; + +struct tegra_aead_reqctx { + struct tegra_se_datbuf inbuf; + struct tegra_se_datbuf outbuf; + struct scatterlist *src_sg; + struct scatterlist *dst_sg; + unsigned int assoclen; + unsigned int cryptlen; + unsigned int authsize; + bool encrypt; + u32 config; + u32 crypto_config; + u32 key_id; + u32 iv[4]; + u8 authdata[16]; +}; + +struct tegra_cmac_ctx { + struct tegra_se *se; + unsigned int alg; + u32 key_id; + struct crypto_shash *fallback_tfm; +}; + +struct tegra_cmac_reqctx { + struct scatterlist *src_sg; + struct tegra_se_datbuf datbuf; + struct tegra_se_datbuf residue; + unsigned int total_len; + unsigned int blk_size; + unsigned int task; + u32 crypto_config; + u32 config; + u32 key_id; + u32 *iv; + u32 result[CMAC_RESULT_REG_COUNT]; +}; + +/* increment counter (128-bit int) */ +static void ctr_iv_inc(__u8 *counter, __u8 bits, __u32 nums) +{ + do { + --bits; + nums += counter[bits]; + counter[bits] = nums & 0xff; + nums >>= 8; + } while (bits && nums); +} + +static void tegra_cbc_iv_copyback(struct skcipher_request *req, struct tegra_aes_ctx *ctx) +{ + struct tegra_aes_reqctx *rctx = skcipher_request_ctx(req); + unsigned int offset; + + offset = req->cryptlen - ctx->ivsize; + + if (rctx->encrypt) + memcpy(req->iv, rctx->datbuf.buf + offset, ctx->ivsize); + else + scatterwalk_map_and_copy(req->iv, req->src, offset, ctx->ivsize, 0); +} + +static void tegra_aes_update_iv(struct skcipher_request *req, struct tegra_aes_ctx *ctx) +{ + int num; + + if (ctx->alg == SE_ALG_CBC) { + tegra_cbc_iv_copyback(req, ctx); + } else if (ctx->alg == SE_ALG_CTR) { + num = req->cryptlen / ctx->ivsize; + if (req->cryptlen % ctx->ivsize) + num++; + + ctr_iv_inc(req->iv, ctx->ivsize, num); + } +} + +static int tegra234_aes_crypto_cfg(u32 alg, bool encrypt) +{ + switch (alg) { + case SE_ALG_CMAC: + case SE_ALG_GMAC: + case SE_ALG_GCM: + case SE_ALG_GCM_FINAL: + return 0; + case SE_ALG_CBC: + if (encrypt) + return SE_CRYPTO_CFG_CBC_ENCRYPT; + else + return SE_CRYPTO_CFG_CBC_DECRYPT; + case SE_ALG_ECB: + if (encrypt) + return SE_CRYPTO_CFG_ECB_ENCRYPT; + else + return SE_CRYPTO_CFG_ECB_DECRYPT; + case SE_ALG_XTS: + if (encrypt) + return SE_CRYPTO_CFG_XTS_ENCRYPT; + else + return SE_CRYPTO_CFG_XTS_DECRYPT; + + case SE_ALG_CTR: + return SE_CRYPTO_CFG_CTR; + case SE_ALG_CBC_MAC: + return SE_CRYPTO_CFG_CBC_MAC; + + default: + break; + } + + return -EINVAL; +} + +static int tegra234_aes_cfg(u32 alg, bool encrypt) +{ + switch (alg) { + case SE_ALG_CBC: + case SE_ALG_ECB: + case SE_ALG_XTS: + case SE_ALG_CTR: + if (encrypt) + return SE_CFG_AES_ENCRYPT; + else + return SE_CFG_AES_DECRYPT; + + case SE_ALG_GMAC: + if (encrypt) + return SE_CFG_GMAC_ENCRYPT; + else + return SE_CFG_GMAC_DECRYPT; + + case SE_ALG_GCM: + if (encrypt) + return SE_CFG_GCM_ENCRYPT; + else + return SE_CFG_GCM_DECRYPT; + + case SE_ALG_GCM_FINAL: + if (encrypt) + return SE_CFG_GCM_FINAL_ENCRYPT; + else + return SE_CFG_GCM_FINAL_DECRYPT; + + case SE_ALG_CMAC: + return SE_CFG_CMAC; + + case SE_ALG_CBC_MAC: + return SE_AES_ENC_ALG_AES_ENC | + SE_AES_DST_HASH_REG; + } + return -EINVAL; +} + +static unsigned int tegra_aes_prep_cmd(struct tegra_aes_ctx *ctx, + struct tegra_aes_reqctx *rctx) +{ + unsigned int data_count, res_bits, i = 0, j; + struct tegra_se *se = ctx->se; + u32 *cpuvaddr = se->cmdbuf->addr; + dma_addr_t addr = rctx->datbuf.addr; + + data_count = rctx->len / AES_BLOCK_SIZE; + res_bits = (rctx->len % AES_BLOCK_SIZE) * 8; + + /* + * Hardware processes data_count + 1 blocks. + * Reduce 1 block if there is no residue + */ + if (!res_bits) + data_count--; + + if (rctx->iv) { + cpuvaddr[i++] = host1x_opcode_setpayload(SE_CRYPTO_CTR_REG_COUNT); + cpuvaddr[i++] = se_host1x_opcode_incr_w(se->hw->regs->linear_ctr); + for (j = 0; j < SE_CRYPTO_CTR_REG_COUNT; j++) + cpuvaddr[i++] = rctx->iv[j]; + } + + cpuvaddr[i++] = se_host1x_opcode_nonincr(se->hw->regs->last_blk, 1); + cpuvaddr[i++] = SE_LAST_BLOCK_VAL(data_count) | + SE_LAST_BLOCK_RES_BITS(res_bits); + + cpuvaddr[i++] = se_host1x_opcode_incr(se->hw->regs->config, 6); + cpuvaddr[i++] = rctx->config; + cpuvaddr[i++] = rctx->crypto_config; + + /* Source address setting */ + cpuvaddr[i++] = lower_32_bits(addr); + cpuvaddr[i++] = SE_ADDR_HI_MSB(upper_32_bits(addr)) | SE_ADDR_HI_SZ(rctx->len); + + /* Destination address setting */ + cpuvaddr[i++] = lower_32_bits(addr); + cpuvaddr[i++] = SE_ADDR_HI_MSB(upper_32_bits(addr)) | + SE_ADDR_HI_SZ(rctx->len); + + cpuvaddr[i++] = se_host1x_opcode_nonincr(se->hw->regs->op, 1); + cpuvaddr[i++] = SE_AES_OP_WRSTALL | SE_AES_OP_LASTBUF | + SE_AES_OP_START; + + cpuvaddr[i++] = se_host1x_opcode_nonincr(host1x_uclass_incr_syncpt_r(), 1); + cpuvaddr[i++] = host1x_uclass_incr_syncpt_cond_f(1) | + host1x_uclass_incr_syncpt_indx_f(se->syncpt_id); + + dev_dbg(se->dev, "cfg %#x crypto cfg %#x\n", rctx->config, rctx->crypto_config); + + return i; +} + +static int tegra_aes_do_one_req(struct crypto_engine *engine, void *areq) +{ + struct skcipher_request *req = container_of(areq, struct skcipher_request, base); + struct tegra_aes_ctx *ctx = crypto_skcipher_ctx(crypto_skcipher_reqtfm(req)); + struct tegra_aes_reqctx *rctx = skcipher_request_ctx(req); + struct tegra_se *se = ctx->se; + unsigned int cmdlen; + int ret; + + rctx->datbuf.buf = dma_alloc_coherent(se->dev, SE_AES_BUFLEN, + &rctx->datbuf.addr, GFP_KERNEL); + if (!rctx->datbuf.buf) + return -ENOMEM; + + rctx->datbuf.size = SE_AES_BUFLEN; + rctx->iv = (u32 *)req->iv; + rctx->len = req->cryptlen; + + /* Pad input to AES Block size */ + if (ctx->alg != SE_ALG_XTS) { + if (rctx->len % AES_BLOCK_SIZE) + rctx->len += AES_BLOCK_SIZE - (rctx->len % AES_BLOCK_SIZE); + } + + scatterwalk_map_and_copy(rctx->datbuf.buf, req->src, 0, req->cryptlen, 0); + + /* Prepare the command and submit for execution */ + cmdlen = tegra_aes_prep_cmd(ctx, rctx); + ret = tegra_se_host1x_submit(se, cmdlen); + + /* Copy the result */ + tegra_aes_update_iv(req, ctx); + scatterwalk_map_and_copy(rctx->datbuf.buf, req->dst, 0, req->cryptlen, 1); + + /* Free the buffer */ + dma_free_coherent(ctx->se->dev, SE_AES_BUFLEN, + rctx->datbuf.buf, rctx->datbuf.addr); + + crypto_finalize_skcipher_request(se->engine, req, ret); + + return 0; +} + +static int tegra_aes_cra_init(struct crypto_skcipher *tfm) +{ + struct tegra_aes_ctx *ctx = crypto_skcipher_ctx(tfm); + struct skcipher_alg *alg = crypto_skcipher_alg(tfm); + struct tegra_se_alg *se_alg; + const char *algname; + int ret; + + se_alg = container_of(alg, struct tegra_se_alg, alg.skcipher.base); + + crypto_skcipher_set_reqsize(tfm, sizeof(struct tegra_aes_reqctx)); + + ctx->ivsize = crypto_skcipher_ivsize(tfm); + ctx->se = se_alg->se_dev; + ctx->key1_id = 0; + ctx->key2_id = 0; + + algname = crypto_tfm_alg_name(&tfm->base); + ret = se_algname_to_algid(algname); + if (ret < 0) { + dev_err(ctx->se->dev, "invalid algorithm\n"); + return ret; + } + + ctx->alg = ret; + + return 0; +} + +static void tegra_aes_cra_exit(struct crypto_skcipher *tfm) +{ + struct tegra_aes_ctx *ctx = crypto_tfm_ctx(&tfm->base); + + if (ctx->key1_id) + tegra_key_invalidate(ctx->se, ctx->key1_id, ctx->alg); + + if (ctx->key2_id) + tegra_key_invalidate(ctx->se, ctx->key2_id, ctx->alg); +} + +static int tegra_aes_setkey(struct crypto_skcipher *tfm, + const u8 *key, u32 keylen) +{ + struct tegra_aes_ctx *ctx = crypto_skcipher_ctx(tfm); + + if (aes_check_keylen(keylen)) { + dev_dbg(ctx->se->dev, "invalid key length (%d)\n", keylen); + return -EINVAL; + } + + return tegra_key_submit(ctx->se, key, keylen, ctx->alg, &ctx->key1_id); +} + +static int tegra_xts_setkey(struct crypto_skcipher *tfm, + const u8 *key, u32 keylen) +{ + struct tegra_aes_ctx *ctx = crypto_skcipher_ctx(tfm); + u32 len = keylen / 2; + int ret; + + ret = xts_verify_key(tfm, key, keylen); + if (ret || aes_check_keylen(len)) { + dev_dbg(ctx->se->dev, "invalid key length (%d)\n", keylen); + return -EINVAL; + } + + ret = tegra_key_submit(ctx->se, key, len, + ctx->alg, &ctx->key1_id); + if (ret) + return ret; + + return tegra_key_submit(ctx->se, key + len, len, + ctx->alg, &ctx->key2_id); + + return 0; +} + +static int tegra_aes_kac_manifest(u32 user, u32 alg, u32 keylen) +{ + int manifest; + + manifest = SE_KAC_USER_NS; + + switch (alg) { + case SE_ALG_CBC: + case SE_ALG_ECB: + case SE_ALG_CTR: + manifest |= SE_KAC_ENC; + break; + case SE_ALG_XTS: + manifest |= SE_KAC_XTS; + break; + case SE_ALG_GCM: + manifest |= SE_KAC_GCM; + break; + case SE_ALG_CMAC: + manifest |= SE_KAC_CMAC; + break; + case SE_ALG_CBC_MAC: + manifest |= SE_KAC_ENC; + break; + default: + return -EINVAL; + } + + switch (keylen) { + case AES_KEYSIZE_128: + manifest |= SE_KAC_SIZE_128; + break; + case AES_KEYSIZE_192: + manifest |= SE_KAC_SIZE_192; + break; + case AES_KEYSIZE_256: + manifest |= SE_KAC_SIZE_256; + break; + default: + return -EINVAL; + } + + return manifest; +} + +static int tegra_aes_crypt(struct skcipher_request *req, bool encrypt) + +{ + struct crypto_skcipher *tfm; + struct tegra_aes_ctx *ctx; + struct tegra_aes_reqctx *rctx; + + tfm = crypto_skcipher_reqtfm(req); + ctx = crypto_skcipher_ctx(tfm); + rctx = skcipher_request_ctx(req); + + if (ctx->alg != SE_ALG_XTS) { + if (!IS_ALIGNED(req->cryptlen, crypto_skcipher_blocksize(tfm))) { + dev_dbg(ctx->se->dev, "invalid length (%d)", req->cryptlen); + return -EINVAL; + } + } else if (req->cryptlen < XTS_BLOCK_SIZE) { + dev_dbg(ctx->se->dev, "invalid length (%d)", req->cryptlen); + return -EINVAL; + } + + if (!req->cryptlen) + return 0; + + rctx->encrypt = encrypt; + rctx->config = tegra234_aes_cfg(ctx->alg, encrypt); + rctx->crypto_config = tegra234_aes_crypto_cfg(ctx->alg, encrypt); + rctx->crypto_config |= SE_AES_KEY_INDEX(ctx->key1_id); + + if (ctx->key2_id) + rctx->crypto_config |= SE_AES_KEY2_INDEX(ctx->key2_id); + + return crypto_transfer_skcipher_request_to_engine(ctx->se->engine, req); +} + +static int tegra_aes_encrypt(struct skcipher_request *req) +{ + return tegra_aes_crypt(req, true); +} + +static int tegra_aes_decrypt(struct skcipher_request *req) +{ + return tegra_aes_crypt(req, false); +} + +static struct tegra_se_alg tegra_aes_algs[] = { + { + .alg.skcipher.op.do_one_request = tegra_aes_do_one_req, + .alg.skcipher.base = { + .init = tegra_aes_cra_init, + .exit = tegra_aes_cra_exit, + .setkey = tegra_aes_setkey, + .encrypt = tegra_aes_encrypt, + .decrypt = tegra_aes_decrypt, + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + .base = { + .cra_name = "cbc(aes)", + .cra_driver_name = "cbc-aes-tegra", + .cra_priority = 500, + .cra_flags = CRYPTO_ALG_TYPE_SKCIPHER | CRYPTO_ALG_ASYNC, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct tegra_aes_ctx), + .cra_alignmask = 0xf, + .cra_module = THIS_MODULE, + }, + } + }, { + .alg.skcipher.op.do_one_request = tegra_aes_do_one_req, + .alg.skcipher.base = { + .init = tegra_aes_cra_init, + .exit = tegra_aes_cra_exit, + .setkey = tegra_aes_setkey, + .encrypt = tegra_aes_encrypt, + .decrypt = tegra_aes_decrypt, + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .base = { + .cra_name = "ecb(aes)", + .cra_driver_name = "ecb-aes-tegra", + .cra_priority = 500, + .cra_flags = CRYPTO_ALG_TYPE_SKCIPHER | CRYPTO_ALG_ASYNC, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct tegra_aes_ctx), + .cra_alignmask = 0xf, + .cra_module = THIS_MODULE, + }, + } + }, { + .alg.skcipher.op.do_one_request = tegra_aes_do_one_req, + .alg.skcipher.base = { + .init = tegra_aes_cra_init, + .exit = tegra_aes_cra_exit, + .setkey = tegra_aes_setkey, + .encrypt = tegra_aes_encrypt, + .decrypt = tegra_aes_decrypt, + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + .base = { + .cra_name = "ctr(aes)", + .cra_driver_name = "ctr-aes-tegra", + .cra_priority = 500, + .cra_flags = CRYPTO_ALG_TYPE_SKCIPHER | CRYPTO_ALG_ASYNC, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct tegra_aes_ctx), + .cra_alignmask = 0xf, + .cra_module = THIS_MODULE, + }, + } + }, { + .alg.skcipher.op.do_one_request = tegra_aes_do_one_req, + .alg.skcipher.base = { + .init = tegra_aes_cra_init, + .exit = tegra_aes_cra_exit, + .setkey = tegra_xts_setkey, + .encrypt = tegra_aes_encrypt, + .decrypt = tegra_aes_decrypt, + .min_keysize = 2 * AES_MIN_KEY_SIZE, + .max_keysize = 2 * AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + .base = { + .cra_name = "xts(aes)", + .cra_driver_name = "xts-aes-tegra", + .cra_priority = 500, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct tegra_aes_ctx), + .cra_alignmask = (__alignof__(u64) - 1), + .cra_module = THIS_MODULE, + }, + } + }, +}; + +static unsigned int tegra_gmac_prep_cmd(struct tegra_aead_ctx *ctx, + struct tegra_aead_reqctx *rctx) +{ + unsigned int data_count, res_bits, i = 0; + struct tegra_se *se = ctx->se; + u32 *cpuvaddr = se->cmdbuf->addr; + + data_count = (rctx->assoclen / AES_BLOCK_SIZE); + res_bits = (rctx->assoclen % AES_BLOCK_SIZE) * 8; + + /* + * Hardware processes data_count + 1 blocks. + * Reduce 1 block if there is no residue + */ + if (!res_bits) + data_count--; + + cpuvaddr[i++] = se_host1x_opcode_nonincr(se->hw->regs->last_blk, 1); + cpuvaddr[i++] = SE_LAST_BLOCK_VAL(data_count) | + SE_LAST_BLOCK_RES_BITS(res_bits); + + cpuvaddr[i++] = se_host1x_opcode_incr(se->hw->regs->config, 4); + cpuvaddr[i++] = rctx->config; + cpuvaddr[i++] = rctx->crypto_config; + cpuvaddr[i++] = lower_32_bits(rctx->inbuf.addr); + cpuvaddr[i++] = SE_ADDR_HI_MSB(upper_32_bits(rctx->inbuf.addr)) | + SE_ADDR_HI_SZ(rctx->assoclen); + + cpuvaddr[i++] = se_host1x_opcode_nonincr(se->hw->regs->op, 1); + cpuvaddr[i++] = SE_AES_OP_WRSTALL | SE_AES_OP_FINAL | + SE_AES_OP_INIT | SE_AES_OP_LASTBUF | + SE_AES_OP_START; + + cpuvaddr[i++] = se_host1x_opcode_nonincr(host1x_uclass_incr_syncpt_r(), 1); + cpuvaddr[i++] = host1x_uclass_incr_syncpt_cond_f(1) | + host1x_uclass_incr_syncpt_indx_f(se->syncpt_id); + + return i; +} + +static unsigned int tegra_gcm_crypt_prep_cmd(struct tegra_aead_ctx *ctx, + struct tegra_aead_reqctx *rctx) +{ + unsigned int data_count, res_bits, i = 0, j; + struct tegra_se *se = ctx->se; + u32 *cpuvaddr = se->cmdbuf->addr, op; + + data_count = (rctx->cryptlen / AES_BLOCK_SIZE); + res_bits = (rctx->cryptlen % AES_BLOCK_SIZE) * 8; + op = SE_AES_OP_WRSTALL | SE_AES_OP_FINAL | + SE_AES_OP_LASTBUF | SE_AES_OP_START; + + /* + * If there is no assoc data, + * this will be the init command + */ + if (!rctx->assoclen) + op |= SE_AES_OP_INIT; + + /* + * Hardware processes data_count + 1 blocks. + * Reduce 1 block if there is no residue + */ + if (!res_bits) + data_count--; + + cpuvaddr[i++] = host1x_opcode_setpayload(SE_CRYPTO_CTR_REG_COUNT); + cpuvaddr[i++] = se_host1x_opcode_incr_w(se->hw->regs->linear_ctr); + for (j = 0; j < SE_CRYPTO_CTR_REG_COUNT; j++) + cpuvaddr[i++] = rctx->iv[j]; + + cpuvaddr[i++] = se_host1x_opcode_nonincr(se->hw->regs->last_blk, 1); + cpuvaddr[i++] = SE_LAST_BLOCK_VAL(data_count) | + SE_LAST_BLOCK_RES_BITS(res_bits); + + cpuvaddr[i++] = se_host1x_opcode_incr(se->hw->regs->config, 6); + cpuvaddr[i++] = rctx->config; + cpuvaddr[i++] = rctx->crypto_config; + + /* Source Address */ + cpuvaddr[i++] = lower_32_bits(rctx->inbuf.addr); + cpuvaddr[i++] = SE_ADDR_HI_MSB(upper_32_bits(rctx->inbuf.addr)) | + SE_ADDR_HI_SZ(rctx->cryptlen); + + /* Destination Address */ + cpuvaddr[i++] = lower_32_bits(rctx->outbuf.addr); + cpuvaddr[i++] = SE_ADDR_HI_MSB(upper_32_bits(rctx->outbuf.addr)) | + SE_ADDR_HI_SZ(rctx->cryptlen); + + cpuvaddr[i++] = se_host1x_opcode_nonincr(se->hw->regs->op, 1); + cpuvaddr[i++] = op; + + cpuvaddr[i++] = se_host1x_opcode_nonincr(host1x_uclass_incr_syncpt_r(), 1); + cpuvaddr[i++] = host1x_uclass_incr_syncpt_cond_f(1) | + host1x_uclass_incr_syncpt_indx_f(se->syncpt_id); + + dev_dbg(se->dev, "cfg %#x crypto cfg %#x\n", rctx->config, rctx->crypto_config); + return i; +} + +static int tegra_gcm_prep_final_cmd(struct tegra_se *se, u32 *cpuvaddr, + struct tegra_aead_reqctx *rctx) +{ + unsigned int i = 0, j; + u32 op; + + op = SE_AES_OP_WRSTALL | SE_AES_OP_FINAL | + SE_AES_OP_LASTBUF | SE_AES_OP_START; + + /* + * Set init for zero sized vector + */ + if (!rctx->assoclen && !rctx->cryptlen) + op |= SE_AES_OP_INIT; + + cpuvaddr[i++] = se_host1x_opcode_incr(se->hw->regs->aad_len, 2); + cpuvaddr[i++] = rctx->assoclen * 8; + cpuvaddr[i++] = 0; + + cpuvaddr[i++] = se_host1x_opcode_incr(se->hw->regs->cryp_msg_len, 2); + cpuvaddr[i++] = rctx->cryptlen * 8; + cpuvaddr[i++] = 0; + + cpuvaddr[i++] = host1x_opcode_setpayload(SE_CRYPTO_CTR_REG_COUNT); + cpuvaddr[i++] = se_host1x_opcode_incr_w(se->hw->regs->linear_ctr); + for (j = 0; j < SE_CRYPTO_CTR_REG_COUNT; j++) + cpuvaddr[i++] = rctx->iv[j]; + + cpuvaddr[i++] = se_host1x_opcode_incr(se->hw->regs->config, 6); + cpuvaddr[i++] = rctx->config; + cpuvaddr[i++] = rctx->crypto_config; + cpuvaddr[i++] = 0; + cpuvaddr[i++] = 0; + + /* Destination Address */ + cpuvaddr[i++] = lower_32_bits(rctx->outbuf.addr); + cpuvaddr[i++] = SE_ADDR_HI_MSB(upper_32_bits(rctx->outbuf.addr)) | + SE_ADDR_HI_SZ(0x10); /* HW always generates 128-bit tag */ + + cpuvaddr[i++] = se_host1x_opcode_nonincr(se->hw->regs->op, 1); + cpuvaddr[i++] = op; + + cpuvaddr[i++] = se_host1x_opcode_nonincr(host1x_uclass_incr_syncpt_r(), 1); + cpuvaddr[i++] = host1x_uclass_incr_syncpt_cond_f(1) | + host1x_uclass_incr_syncpt_indx_f(se->syncpt_id); + + dev_dbg(se->dev, "cfg %#x crypto cfg %#x\n", rctx->config, rctx->crypto_config); + + return i; +} + +static int tegra_gcm_do_gmac(struct tegra_aead_ctx *ctx, struct tegra_aead_reqctx *rctx) +{ + struct tegra_se *se = ctx->se; + unsigned int cmdlen; + + scatterwalk_map_and_copy(rctx->inbuf.buf, + rctx->src_sg, 0, rctx->assoclen, 0); + + rctx->config = tegra234_aes_cfg(SE_ALG_GMAC, rctx->encrypt); + rctx->crypto_config = tegra234_aes_crypto_cfg(SE_ALG_GMAC, rctx->encrypt) | + SE_AES_KEY_INDEX(ctx->key_id); + + cmdlen = tegra_gmac_prep_cmd(ctx, rctx); + + return tegra_se_host1x_submit(se, cmdlen); +} + +static int tegra_gcm_do_crypt(struct tegra_aead_ctx *ctx, struct tegra_aead_reqctx *rctx) +{ + struct tegra_se *se = ctx->se; + int cmdlen, ret; + + scatterwalk_map_and_copy(rctx->inbuf.buf, rctx->src_sg, + rctx->assoclen, rctx->cryptlen, 0); + + rctx->config = tegra234_aes_cfg(SE_ALG_GCM, rctx->encrypt); + rctx->crypto_config = tegra234_aes_crypto_cfg(SE_ALG_GCM, rctx->encrypt) | + SE_AES_KEY_INDEX(ctx->key_id); + + /* Prepare command and submit */ + cmdlen = tegra_gcm_crypt_prep_cmd(ctx, rctx); + ret = tegra_se_host1x_submit(se, cmdlen); + if (ret) + return ret; + + /* Copy the result */ + scatterwalk_map_and_copy(rctx->outbuf.buf, rctx->dst_sg, + rctx->assoclen, rctx->cryptlen, 1); + + return 0; +} + +static int tegra_gcm_do_final(struct tegra_aead_ctx *ctx, struct tegra_aead_reqctx *rctx) +{ + struct tegra_se *se = ctx->se; + u32 *cpuvaddr = se->cmdbuf->addr; + int cmdlen, ret, offset; + + rctx->config = tegra234_aes_cfg(SE_ALG_GCM_FINAL, rctx->encrypt); + rctx->crypto_config = tegra234_aes_crypto_cfg(SE_ALG_GCM_FINAL, rctx->encrypt) | + SE_AES_KEY_INDEX(ctx->key_id); + + /* Prepare command and submit */ + cmdlen = tegra_gcm_prep_final_cmd(se, cpuvaddr, rctx); + ret = tegra_se_host1x_submit(se, cmdlen); + if (ret) + return ret; + + if (rctx->encrypt) { + /* Copy the result */ + offset = rctx->assoclen + rctx->cryptlen; + scatterwalk_map_and_copy(rctx->outbuf.buf, rctx->dst_sg, + offset, rctx->authsize, 1); + } + + return 0; +} + +static int tegra_gcm_do_verify(struct tegra_se *se, struct tegra_aead_reqctx *rctx) +{ + unsigned int offset; + u8 mac[16]; + + offset = rctx->assoclen + rctx->cryptlen; + scatterwalk_map_and_copy(mac, rctx->src_sg, offset, rctx->authsize, 0); + + if (crypto_memneq(rctx->outbuf.buf, mac, rctx->authsize)) + return -EBADMSG; + + return 0; +} + +static inline int tegra_ccm_check_iv(const u8 *iv) +{ + /* iv[0] gives value of q-1 + * 2 <= q <= 8 as per NIST 800-38C notation + * 2 <= L <= 8, so 1 <= L' <= 7. as per rfc 3610 notation + */ + if (iv[0] < 1 || iv[0] > 7) { + pr_debug("ccm_check_iv failed %d\n", iv[0]); + return -EINVAL; + } + + return 0; +} + +static unsigned int tegra_cbcmac_prep_cmd(struct tegra_aead_ctx *ctx, + struct tegra_aead_reqctx *rctx) +{ + unsigned int data_count, i = 0; + struct tegra_se *se = ctx->se; + u32 *cpuvaddr = se->cmdbuf->addr; + + data_count = (rctx->inbuf.size / AES_BLOCK_SIZE) - 1; + + cpuvaddr[i++] = se_host1x_opcode_nonincr(se->hw->regs->last_blk, 1); + cpuvaddr[i++] = SE_LAST_BLOCK_VAL(data_count); + + cpuvaddr[i++] = se_host1x_opcode_incr(se->hw->regs->config, 6); + cpuvaddr[i++] = rctx->config; + cpuvaddr[i++] = rctx->crypto_config; + + cpuvaddr[i++] = lower_32_bits(rctx->inbuf.addr); + cpuvaddr[i++] = SE_ADDR_HI_MSB(upper_32_bits(rctx->inbuf.addr)) | + SE_ADDR_HI_SZ(rctx->inbuf.size); + + cpuvaddr[i++] = lower_32_bits(rctx->outbuf.addr); + cpuvaddr[i++] = SE_ADDR_HI_MSB(upper_32_bits(rctx->outbuf.addr)) | + SE_ADDR_HI_SZ(0x10); /* HW always generates 128 bit tag */ + + cpuvaddr[i++] = se_host1x_opcode_nonincr(se->hw->regs->op, 1); + cpuvaddr[i++] = SE_AES_OP_WRSTALL | + SE_AES_OP_LASTBUF | SE_AES_OP_START; + + cpuvaddr[i++] = se_host1x_opcode_nonincr(host1x_uclass_incr_syncpt_r(), 1); + cpuvaddr[i++] = host1x_uclass_incr_syncpt_cond_f(1) | + host1x_uclass_incr_syncpt_indx_f(se->syncpt_id); + + return i; +} + +static unsigned int tegra_ctr_prep_cmd(struct tegra_aead_ctx *ctx, + struct tegra_aead_reqctx *rctx) +{ + unsigned int i = 0, j; + struct tegra_se *se = ctx->se; + u32 *cpuvaddr = se->cmdbuf->addr; + + cpuvaddr[i++] = host1x_opcode_setpayload(SE_CRYPTO_CTR_REG_COUNT); + cpuvaddr[i++] = se_host1x_opcode_incr_w(se->hw->regs->linear_ctr); + for (j = 0; j < SE_CRYPTO_CTR_REG_COUNT; j++) + cpuvaddr[i++] = rctx->iv[j]; + + cpuvaddr[i++] = se_host1x_opcode_nonincr(se->hw->regs->last_blk, 1); + cpuvaddr[i++] = (rctx->inbuf.size / AES_BLOCK_SIZE) - 1; + cpuvaddr[i++] = se_host1x_opcode_incr(se->hw->regs->config, 6); + cpuvaddr[i++] = rctx->config; + cpuvaddr[i++] = rctx->crypto_config; + + /* Source address setting */ + cpuvaddr[i++] = lower_32_bits(rctx->inbuf.addr); + cpuvaddr[i++] = SE_ADDR_HI_MSB(upper_32_bits(rctx->inbuf.addr)) | + SE_ADDR_HI_SZ(rctx->inbuf.size); + + /* Destination address setting */ + cpuvaddr[i++] = lower_32_bits(rctx->outbuf.addr); + cpuvaddr[i++] = SE_ADDR_HI_MSB(upper_32_bits(rctx->outbuf.addr)) | + SE_ADDR_HI_SZ(rctx->inbuf.size); + + cpuvaddr[i++] = se_host1x_opcode_nonincr(se->hw->regs->op, 1); + cpuvaddr[i++] = SE_AES_OP_WRSTALL | SE_AES_OP_LASTBUF | + SE_AES_OP_START; + + cpuvaddr[i++] = se_host1x_opcode_nonincr(host1x_uclass_incr_syncpt_r(), 1); + cpuvaddr[i++] = host1x_uclass_incr_syncpt_cond_f(1) | + host1x_uclass_incr_syncpt_indx_f(se->syncpt_id); + + dev_dbg(se->dev, "cfg %#x crypto cfg %#x\n", + rctx->config, rctx->crypto_config); + + return i; +} + +static int tegra_ccm_do_cbcmac(struct tegra_aead_ctx *ctx, struct tegra_aead_reqctx *rctx) +{ + struct tegra_se *se = ctx->se; + int cmdlen; + + rctx->config = tegra234_aes_cfg(SE_ALG_CBC_MAC, rctx->encrypt); + rctx->crypto_config = tegra234_aes_crypto_cfg(SE_ALG_CBC_MAC, + rctx->encrypt) | + SE_AES_KEY_INDEX(ctx->key_id); + + /* Prepare command and submit */ + cmdlen = tegra_cbcmac_prep_cmd(ctx, rctx); + + return tegra_se_host1x_submit(se, cmdlen); +} + +static int tegra_ccm_set_msg_len(u8 *block, unsigned int msglen, int csize) +{ + __be32 data; + + memset(block, 0, csize); + block += csize; + + if (csize >= 4) + csize = 4; + else if (msglen > (1 << (8 * csize))) + return -EOVERFLOW; + + data = cpu_to_be32(msglen); + memcpy(block - csize, (u8 *)&data + 4 - csize, csize); + + return 0; +} + +static int tegra_ccm_format_nonce(struct tegra_aead_reqctx *rctx, u8 *nonce) +{ + unsigned int q, t; + u8 *q_ptr, *iv = (u8 *)rctx->iv; + + memcpy(nonce, rctx->iv, 16); + + /*** 1. Prepare Flags Octet ***/ + + /* Encode t (mac length) */ + t = rctx->authsize; + nonce[0] |= (((t - 2) / 2) << 3); + + /* Adata */ + if (rctx->assoclen) + nonce[0] |= (1 << 6); + + /*** Encode Q - message length ***/ + q = iv[0] + 1; + q_ptr = nonce + 16 - q; + + return tegra_ccm_set_msg_len(q_ptr, rctx->cryptlen, q); +} + +static int tegra_ccm_format_adata(u8 *adata, unsigned int a) +{ + int len = 0; + + /* add control info for associated data + * RFC 3610 and NIST Special Publication 800-38C + */ + if (a < 65280) { + *(__be16 *)adata = cpu_to_be16(a); + len = 2; + } else { + *(__be16 *)adata = cpu_to_be16(0xfffe); + *(__be32 *)&adata[2] = cpu_to_be32(a); + len = 6; + } + + return len; +} + +static int tegra_ccm_add_padding(u8 *buf, unsigned int len) +{ + unsigned int padlen = 16 - (len % 16); + u8 padding[16] = {0}; + + if (padlen == 16) + return 0; + + memcpy(buf, padding, padlen); + + return padlen; +} + +static int tegra_ccm_format_blocks(struct tegra_aead_reqctx *rctx) +{ + unsigned int alen = 0, offset = 0; + u8 nonce[16], adata[16]; + int ret; + + ret = tegra_ccm_format_nonce(rctx, nonce); + if (ret) + return ret; + + memcpy(rctx->inbuf.buf, nonce, 16); + offset = 16; + + if (rctx->assoclen) { + alen = tegra_ccm_format_adata(adata, rctx->assoclen); + memcpy(rctx->inbuf.buf + offset, adata, alen); + offset += alen; + + scatterwalk_map_and_copy(rctx->inbuf.buf + offset, + rctx->src_sg, 0, rctx->assoclen, 0); + + offset += rctx->assoclen; + offset += tegra_ccm_add_padding(rctx->inbuf.buf + offset, + rctx->assoclen + alen); + } + + return offset; +} + +static int tegra_ccm_mac_result(struct tegra_se *se, struct tegra_aead_reqctx *rctx) +{ + u32 result[16]; + int i, ret; + + /* Read and clear Result */ + for (i = 0; i < CMAC_RESULT_REG_COUNT; i++) + result[i] = readl(se->base + se->hw->regs->result + (i * 4)); + + for (i = 0; i < CMAC_RESULT_REG_COUNT; i++) + writel(0, se->base + se->hw->regs->result + (i * 4)); + + if (rctx->encrypt) { + memcpy(rctx->authdata, result, rctx->authsize); + } else { + ret = crypto_memneq(rctx->authdata, result, rctx->authsize); + if (ret) + return -EBADMSG; + } + + return 0; +} + +static int tegra_ccm_ctr_result(struct tegra_se *se, struct tegra_aead_reqctx *rctx) +{ + /* Copy result */ + scatterwalk_map_and_copy(rctx->outbuf.buf + 16, rctx->dst_sg, + rctx->assoclen, rctx->cryptlen, 1); + + if (rctx->encrypt) + scatterwalk_map_and_copy(rctx->outbuf.buf, rctx->dst_sg, + rctx->assoclen + rctx->cryptlen, + rctx->authsize, 1); + else + memcpy(rctx->authdata, rctx->outbuf.buf, rctx->authsize); + + return 0; +} + +static int tegra_ccm_compute_auth(struct tegra_aead_ctx *ctx, struct tegra_aead_reqctx *rctx) +{ + struct tegra_se *se = ctx->se; + struct scatterlist *sg; + int offset, ret; + + offset = tegra_ccm_format_blocks(rctx); + if (offset < 0) + return -EINVAL; + + /* Copy plain text to the buffer */ + sg = rctx->encrypt ? rctx->src_sg : rctx->dst_sg; + + scatterwalk_map_and_copy(rctx->inbuf.buf + offset, + sg, rctx->assoclen, + rctx->cryptlen, 0); + offset += rctx->cryptlen; + offset += tegra_ccm_add_padding(rctx->inbuf.buf + offset, rctx->cryptlen); + + rctx->inbuf.size = offset; + + ret = tegra_ccm_do_cbcmac(ctx, rctx); + if (ret) + return ret; + + return tegra_ccm_mac_result(se, rctx); +} + +static int tegra_ccm_do_ctr(struct tegra_aead_ctx *ctx, struct tegra_aead_reqctx *rctx) +{ + struct tegra_se *se = ctx->se; + unsigned int cmdlen, offset = 0; + struct scatterlist *sg = rctx->src_sg; + int ret; + + rctx->config = tegra234_aes_cfg(SE_ALG_CTR, rctx->encrypt); + rctx->crypto_config = tegra234_aes_crypto_cfg(SE_ALG_CTR, rctx->encrypt) | + SE_AES_KEY_INDEX(ctx->key_id); + + /* Copy authdata in the top of buffer for encryption/decryption */ + if (rctx->encrypt) + memcpy(rctx->inbuf.buf, rctx->authdata, rctx->authsize); + else + scatterwalk_map_and_copy(rctx->inbuf.buf, sg, + rctx->assoclen + rctx->cryptlen, + rctx->authsize, 0); + + offset += rctx->authsize; + offset += tegra_ccm_add_padding(rctx->inbuf.buf + offset, rctx->authsize); + + /* If there is no cryptlen, proceed to submit the task */ + if (rctx->cryptlen) { + scatterwalk_map_and_copy(rctx->inbuf.buf + offset, sg, + rctx->assoclen, rctx->cryptlen, 0); + offset += rctx->cryptlen; + offset += tegra_ccm_add_padding(rctx->inbuf.buf + offset, rctx->cryptlen); + } + + rctx->inbuf.size = offset; + + /* Prepare command and submit */ + cmdlen = tegra_ctr_prep_cmd(ctx, rctx); + ret = tegra_se_host1x_submit(se, cmdlen); + if (ret) + return ret; + + return tegra_ccm_ctr_result(se, rctx); +} + +static int tegra_ccm_crypt_init(struct aead_request *req, struct tegra_se *se, + struct tegra_aead_reqctx *rctx) +{ + struct crypto_aead *tfm = crypto_aead_reqtfm(req); + u8 *iv = (u8 *)rctx->iv; + int ret, i; + + rctx->src_sg = req->src; + rctx->dst_sg = req->dst; + rctx->assoclen = req->assoclen; + rctx->authsize = crypto_aead_authsize(tfm); + + memcpy(iv, req->iv, 16); + + ret = tegra_ccm_check_iv(iv); + if (ret) + return ret; + + /* Note: rfc 3610 and NIST 800-38C require counter (ctr_0) of + * zero to encrypt auth tag. + * req->iv has the formatted ctr_0 (i.e. Flags || N || 0). + */ + memset(iv + 15 - iv[0], 0, iv[0] + 1); + + /* Clear any previous result */ + for (i = 0; i < CMAC_RESULT_REG_COUNT; i++) + writel(0, se->base + se->hw->regs->result + (i * 4)); + + return 0; +} + +static int tegra_ccm_do_one_req(struct crypto_engine *engine, void *areq) +{ + struct aead_request *req = container_of(areq, struct aead_request, base); + struct tegra_aead_reqctx *rctx = aead_request_ctx(req); + struct crypto_aead *tfm = crypto_aead_reqtfm(req); + struct tegra_aead_ctx *ctx = crypto_aead_ctx(tfm); + struct tegra_se *se = ctx->se; + int ret; + + /* Allocate buffers required */ + rctx->inbuf.buf = dma_alloc_coherent(ctx->se->dev, SE_AES_BUFLEN, + &rctx->inbuf.addr, GFP_KERNEL); + if (!rctx->inbuf.buf) + return -ENOMEM; + + rctx->inbuf.size = SE_AES_BUFLEN; + + rctx->outbuf.buf = dma_alloc_coherent(ctx->se->dev, SE_AES_BUFLEN, + &rctx->outbuf.addr, GFP_KERNEL); + if (!rctx->outbuf.buf) { + ret = -ENOMEM; + goto outbuf_err; + } + + rctx->outbuf.size = SE_AES_BUFLEN; + + ret = tegra_ccm_crypt_init(req, se, rctx); + if (ret) + goto out; + + if (rctx->encrypt) { + rctx->cryptlen = req->cryptlen; + + /* CBC MAC Operation */ + ret = tegra_ccm_compute_auth(ctx, rctx); + if (ret) + goto out; + + /* CTR operation */ + ret = tegra_ccm_do_ctr(ctx, rctx); + if (ret) + goto out; + } else { + rctx->cryptlen = req->cryptlen - ctx->authsize; + if (ret) + goto out; + + /* CTR operation */ + ret = tegra_ccm_do_ctr(ctx, rctx); + if (ret) + goto out; + + /* CBC MAC Operation */ + ret = tegra_ccm_compute_auth(ctx, rctx); + if (ret) + goto out; + } + +out: + dma_free_coherent(ctx->se->dev, SE_AES_BUFLEN, + rctx->outbuf.buf, rctx->outbuf.addr); + +outbuf_err: + dma_free_coherent(ctx->se->dev, SE_AES_BUFLEN, + rctx->inbuf.buf, rctx->inbuf.addr); + + crypto_finalize_aead_request(ctx->se->engine, req, ret); + + return 0; +} + +static int tegra_gcm_do_one_req(struct crypto_engine *engine, void *areq) +{ + struct aead_request *req = container_of(areq, struct aead_request, base); + struct crypto_aead *tfm = crypto_aead_reqtfm(req); + struct tegra_aead_ctx *ctx = crypto_aead_ctx(tfm); + struct tegra_aead_reqctx *rctx = aead_request_ctx(req); + int ret; + + /* Allocate buffers required */ + rctx->inbuf.buf = dma_alloc_coherent(ctx->se->dev, SE_AES_BUFLEN, + &rctx->inbuf.addr, GFP_KERNEL); + if (!rctx->inbuf.buf) + return -ENOMEM; + + rctx->inbuf.size = SE_AES_BUFLEN; + + rctx->outbuf.buf = dma_alloc_coherent(ctx->se->dev, SE_AES_BUFLEN, + &rctx->outbuf.addr, GFP_KERNEL); + if (!rctx->outbuf.buf) { + ret = -ENOMEM; + goto outbuf_err; + } + + rctx->outbuf.size = SE_AES_BUFLEN; + + rctx->src_sg = req->src; + rctx->dst_sg = req->dst; + rctx->assoclen = req->assoclen; + rctx->authsize = crypto_aead_authsize(tfm); + + if (rctx->encrypt) + rctx->cryptlen = req->cryptlen; + else + rctx->cryptlen = req->cryptlen - ctx->authsize; + + memcpy(rctx->iv, req->iv, GCM_AES_IV_SIZE); + rctx->iv[3] = (1 << 24); + + /* If there is associated data perform GMAC operation */ + if (rctx->assoclen) { + ret = tegra_gcm_do_gmac(ctx, rctx); + if (ret) + goto out; + } + + /* GCM Encryption/Decryption operation */ + if (rctx->cryptlen) { + ret = tegra_gcm_do_crypt(ctx, rctx); + if (ret) + goto out; + } + + /* GCM_FINAL operation */ + ret = tegra_gcm_do_final(ctx, rctx); + if (ret) + goto out; + + if (!rctx->encrypt) + ret = tegra_gcm_do_verify(ctx->se, rctx); + +out: + dma_free_coherent(ctx->se->dev, SE_AES_BUFLEN, + rctx->outbuf.buf, rctx->outbuf.addr); + +outbuf_err: + dma_free_coherent(ctx->se->dev, SE_AES_BUFLEN, + rctx->inbuf.buf, rctx->inbuf.addr); + + /* Finalize the request if there are no errors */ + crypto_finalize_aead_request(ctx->se->engine, req, ret); + + return 0; +} + +static int tegra_aead_cra_init(struct crypto_aead *tfm) +{ + struct tegra_aead_ctx *ctx = crypto_aead_ctx(tfm); + struct aead_alg *alg = crypto_aead_alg(tfm); + struct tegra_se_alg *se_alg; + const char *algname; + int ret; + + algname = crypto_tfm_alg_name(&tfm->base); + + se_alg = container_of(alg, struct tegra_se_alg, alg.aead.base); + + crypto_aead_set_reqsize(tfm, sizeof(struct tegra_aead_reqctx)); + + ctx->se = se_alg->se_dev; + ctx->key_id = 0; + + ret = se_algname_to_algid(algname); + if (ret < 0) { + dev_err(ctx->se->dev, "invalid algorithm\n"); + return ret; + } + + ctx->alg = ret; + + return 0; +} + +static int tegra_ccm_setauthsize(struct crypto_aead *tfm, unsigned int authsize) +{ + struct tegra_aead_ctx *ctx = crypto_aead_ctx(tfm); + + switch (authsize) { + case 4: + case 6: + case 8: + case 10: + case 12: + case 14: + case 16: + break; + default: + return -EINVAL; + } + + ctx->authsize = authsize; + + return 0; +} + +static int tegra_gcm_setauthsize(struct crypto_aead *tfm, unsigned int authsize) +{ + struct tegra_aead_ctx *ctx = crypto_aead_ctx(tfm); + int ret; + + ret = crypto_gcm_check_authsize(authsize); + if (ret) + return ret; + + ctx->authsize = authsize; + + return 0; +} + +static void tegra_aead_cra_exit(struct crypto_aead *tfm) +{ + struct tegra_aead_ctx *ctx = crypto_tfm_ctx(&tfm->base); + + if (ctx->key_id) + tegra_key_invalidate(ctx->se, ctx->key_id, ctx->alg); +} + +static int tegra_aead_crypt(struct aead_request *req, bool encrypt) +{ + struct crypto_aead *tfm = crypto_aead_reqtfm(req); + struct tegra_aead_ctx *ctx = crypto_aead_ctx(tfm); + struct tegra_aead_reqctx *rctx = aead_request_ctx(req); + + rctx->encrypt = encrypt; + + return crypto_transfer_aead_request_to_engine(ctx->se->engine, req); +} + +static int tegra_aead_encrypt(struct aead_request *req) +{ + return tegra_aead_crypt(req, true); +} + +static int tegra_aead_decrypt(struct aead_request *req) +{ + return tegra_aead_crypt(req, false); +} + +static int tegra_aead_setkey(struct crypto_aead *tfm, + const u8 *key, u32 keylen) +{ + struct tegra_aead_ctx *ctx = crypto_aead_ctx(tfm); + + if (aes_check_keylen(keylen)) { + dev_dbg(ctx->se->dev, "invalid key length (%d)\n", keylen); + return -EINVAL; + } + + return tegra_key_submit(ctx->se, key, keylen, ctx->alg, &ctx->key_id); +} + +static unsigned int tegra_cmac_prep_cmd(struct tegra_cmac_ctx *ctx, + struct tegra_cmac_reqctx *rctx) +{ + unsigned int data_count, res_bits = 0, i = 0, j; + struct tegra_se *se = ctx->se; + u32 *cpuvaddr = se->cmdbuf->addr, op; + + data_count = (rctx->datbuf.size / AES_BLOCK_SIZE); + + op = SE_AES_OP_WRSTALL | SE_AES_OP_START | SE_AES_OP_LASTBUF; + + if (!(rctx->task & SHA_UPDATE)) { + op |= SE_AES_OP_FINAL; + res_bits = (rctx->datbuf.size % AES_BLOCK_SIZE) * 8; + } + + if (!res_bits && data_count) + data_count--; + + if (rctx->task & SHA_FIRST) { + rctx->task &= ~SHA_FIRST; + + cpuvaddr[i++] = host1x_opcode_setpayload(SE_CRYPTO_CTR_REG_COUNT); + cpuvaddr[i++] = se_host1x_opcode_incr_w(se->hw->regs->linear_ctr); + /* Load 0 IV */ + for (j = 0; j < SE_CRYPTO_CTR_REG_COUNT; j++) + cpuvaddr[i++] = 0; + } + + cpuvaddr[i++] = se_host1x_opcode_nonincr(se->hw->regs->last_blk, 1); + cpuvaddr[i++] = SE_LAST_BLOCK_VAL(data_count) | + SE_LAST_BLOCK_RES_BITS(res_bits); + + cpuvaddr[i++] = se_host1x_opcode_incr(se->hw->regs->config, 6); + cpuvaddr[i++] = rctx->config; + cpuvaddr[i++] = rctx->crypto_config; + + /* Source Address */ + cpuvaddr[i++] = lower_32_bits(rctx->datbuf.addr); + cpuvaddr[i++] = SE_ADDR_HI_MSB(upper_32_bits(rctx->datbuf.addr)) | + SE_ADDR_HI_SZ(rctx->datbuf.size); + cpuvaddr[i++] = 0; + cpuvaddr[i++] = SE_ADDR_HI_SZ(AES_BLOCK_SIZE); + + cpuvaddr[i++] = se_host1x_opcode_nonincr(se->hw->regs->op, 1); + cpuvaddr[i++] = op; + + cpuvaddr[i++] = se_host1x_opcode_nonincr(host1x_uclass_incr_syncpt_r(), 1); + cpuvaddr[i++] = host1x_uclass_incr_syncpt_cond_f(1) | + host1x_uclass_incr_syncpt_indx_f(se->syncpt_id); + + return i; +} + +static void tegra_cmac_copy_result(struct tegra_se *se, struct tegra_cmac_reqctx *rctx) +{ + int i; + + for (i = 0; i < CMAC_RESULT_REG_COUNT; i++) + rctx->result[i] = readl(se->base + se->hw->regs->result + (i * 4)); +} + +static void tegra_cmac_paste_result(struct tegra_se *se, struct tegra_cmac_reqctx *rctx) +{ + int i; + + for (i = 0; i < CMAC_RESULT_REG_COUNT; i++) + writel(rctx->result[i], + se->base + se->hw->regs->result + (i * 4)); +} + +static int tegra_cmac_do_update(struct ahash_request *req) +{ + struct tegra_cmac_reqctx *rctx = ahash_request_ctx(req); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct tegra_cmac_ctx *ctx = crypto_ahash_ctx(tfm); + struct tegra_se *se = ctx->se; + unsigned int nblks, nresidue, cmdlen; + int ret; + + if (!req->nbytes) + return 0; + + nresidue = (req->nbytes + rctx->residue.size) % rctx->blk_size; + nblks = (req->nbytes + rctx->residue.size) / rctx->blk_size; + + /* + * Reserve the last block as residue during final() to process. + */ + if (!nresidue && nblks) { + nresidue += rctx->blk_size; + nblks--; + } + + rctx->src_sg = req->src; + rctx->datbuf.size = (req->nbytes + rctx->residue.size) - nresidue; + rctx->total_len += rctx->datbuf.size; + rctx->config = tegra234_aes_cfg(SE_ALG_CMAC, 0); + rctx->crypto_config = SE_AES_KEY_INDEX(ctx->key_id); + + /* + * Keep one block and residue bytes in residue and + * return. The bytes will be processed in final() + */ + if (nblks < 1) { + scatterwalk_map_and_copy(rctx->residue.buf + rctx->residue.size, + rctx->src_sg, 0, req->nbytes, 0); + + rctx->residue.size += req->nbytes; + return 0; + } + + /* Copy the previous residue first */ + if (rctx->residue.size) + memcpy(rctx->datbuf.buf, rctx->residue.buf, rctx->residue.size); + + scatterwalk_map_and_copy(rctx->datbuf.buf + rctx->residue.size, + rctx->src_sg, 0, req->nbytes - nresidue, 0); + + scatterwalk_map_and_copy(rctx->residue.buf, rctx->src_sg, + req->nbytes - nresidue, nresidue, 0); + + /* Update residue value with the residue after current block */ + rctx->residue.size = nresidue; + + /* + * If this is not the first 'update' call, paste the previous copied + * intermediate results to the registers so that it gets picked up. + * This is to support the import/export functionality. + */ + if (!(rctx->task & SHA_FIRST)) + tegra_cmac_paste_result(ctx->se, rctx); + + cmdlen = tegra_cmac_prep_cmd(ctx, rctx); + + ret = tegra_se_host1x_submit(se, cmdlen); + /* + * If this is not the final update, copy the intermediate results + * from the registers so that it can be used in the next 'update' + * call. This is to support the import/export functionality. + */ + if (!(rctx->task & SHA_FINAL)) + tegra_cmac_copy_result(ctx->se, rctx); + + return ret; +} + +static int tegra_cmac_do_final(struct ahash_request *req) +{ + struct tegra_cmac_reqctx *rctx = ahash_request_ctx(req); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct tegra_cmac_ctx *ctx = crypto_ahash_ctx(tfm); + struct tegra_se *se = ctx->se; + u32 *result = (u32 *)req->result; + int ret = 0, i, cmdlen; + + if (!req->nbytes && !rctx->total_len && ctx->fallback_tfm) { + return crypto_shash_tfm_digest(ctx->fallback_tfm, + rctx->datbuf.buf, 0, req->result); + } + + memcpy(rctx->datbuf.buf, rctx->residue.buf, rctx->residue.size); + rctx->datbuf.size = rctx->residue.size; + rctx->total_len += rctx->residue.size; + rctx->config = tegra234_aes_cfg(SE_ALG_CMAC, 0); + + /* Prepare command and submit */ + cmdlen = tegra_cmac_prep_cmd(ctx, rctx); + ret = tegra_se_host1x_submit(se, cmdlen); + if (ret) + goto out; + + /* Read and clear Result register */ + for (i = 0; i < CMAC_RESULT_REG_COUNT; i++) + result[i] = readl(se->base + se->hw->regs->result + (i * 4)); + + for (i = 0; i < CMAC_RESULT_REG_COUNT; i++) + writel(0, se->base + se->hw->regs->result + (i * 4)); + +out: + dma_free_coherent(se->dev, SE_SHA_BUFLEN, + rctx->datbuf.buf, rctx->datbuf.addr); + dma_free_coherent(se->dev, crypto_ahash_blocksize(tfm) * 2, + rctx->residue.buf, rctx->residue.addr); + return ret; +} + +static int tegra_cmac_do_one_req(struct crypto_engine *engine, void *areq) +{ + struct ahash_request *req = ahash_request_cast(areq); + struct tegra_cmac_reqctx *rctx = ahash_request_ctx(req); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct tegra_cmac_ctx *ctx = crypto_ahash_ctx(tfm); + struct tegra_se *se = ctx->se; + int ret; + + if (rctx->task & SHA_UPDATE) { + ret = tegra_cmac_do_update(req); + rctx->task &= ~SHA_UPDATE; + } + + if (rctx->task & SHA_FINAL) { + ret = tegra_cmac_do_final(req); + rctx->task &= ~SHA_FINAL; + } + + crypto_finalize_hash_request(se->engine, req, ret); + + return 0; +} + +static void tegra_cmac_init_fallback(struct crypto_ahash *tfm, struct tegra_cmac_ctx *ctx, + const char *algname) +{ + unsigned int statesize; + + ctx->fallback_tfm = crypto_alloc_shash(algname, 0, CRYPTO_ALG_NEED_FALLBACK); + + if (IS_ERR(ctx->fallback_tfm)) { + dev_warn(ctx->se->dev, "failed to allocate fallback for %s\n", algname); + ctx->fallback_tfm = NULL; + return; + } + + statesize = crypto_shash_statesize(ctx->fallback_tfm); + + if (statesize > sizeof(struct tegra_cmac_reqctx)) + crypto_ahash_set_statesize(tfm, statesize); +} + +static int tegra_cmac_cra_init(struct crypto_tfm *tfm) +{ + struct tegra_cmac_ctx *ctx = crypto_tfm_ctx(tfm); + struct crypto_ahash *ahash_tfm = __crypto_ahash_cast(tfm); + struct ahash_alg *alg = __crypto_ahash_alg(tfm->__crt_alg); + struct tegra_se_alg *se_alg; + const char *algname; + int ret; + + algname = crypto_tfm_alg_name(tfm); + se_alg = container_of(alg, struct tegra_se_alg, alg.ahash.base); + + crypto_ahash_set_reqsize(ahash_tfm, sizeof(struct tegra_cmac_reqctx)); + + ctx->se = se_alg->se_dev; + ctx->key_id = 0; + + ret = se_algname_to_algid(algname); + if (ret < 0) { + dev_err(ctx->se->dev, "invalid algorithm\n"); + return ret; + } + + ctx->alg = ret; + + tegra_cmac_init_fallback(ahash_tfm, ctx, algname); + + return 0; +} + +static void tegra_cmac_cra_exit(struct crypto_tfm *tfm) +{ + struct tegra_cmac_ctx *ctx = crypto_tfm_ctx(tfm); + + if (ctx->fallback_tfm) + crypto_free_shash(ctx->fallback_tfm); + + tegra_key_invalidate(ctx->se, ctx->key_id, ctx->alg); +} + +static int tegra_cmac_init(struct ahash_request *req) +{ + struct tegra_cmac_reqctx *rctx = ahash_request_ctx(req); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct tegra_cmac_ctx *ctx = crypto_ahash_ctx(tfm); + struct tegra_se *se = ctx->se; + int i; + + rctx->total_len = 0; + rctx->datbuf.size = 0; + rctx->residue.size = 0; + rctx->task = SHA_FIRST; + rctx->blk_size = crypto_ahash_blocksize(tfm); + + rctx->residue.buf = dma_alloc_coherent(se->dev, rctx->blk_size * 2, + &rctx->residue.addr, GFP_KERNEL); + if (!rctx->residue.buf) + goto resbuf_fail; + + rctx->residue.size = 0; + + rctx->datbuf.buf = dma_alloc_coherent(se->dev, SE_SHA_BUFLEN, + &rctx->datbuf.addr, GFP_KERNEL); + if (!rctx->datbuf.buf) + goto datbuf_fail; + + rctx->datbuf.size = 0; + + /* Clear any previous result */ + for (i = 0; i < CMAC_RESULT_REG_COUNT; i++) + writel(0, se->base + se->hw->regs->result + (i * 4)); + + return 0; + +datbuf_fail: + dma_free_coherent(se->dev, rctx->blk_size, rctx->residue.buf, + rctx->residue.addr); +resbuf_fail: + return -ENOMEM; +} + +static int tegra_cmac_setkey(struct crypto_ahash *tfm, const u8 *key, + unsigned int keylen) +{ + struct tegra_cmac_ctx *ctx = crypto_ahash_ctx(tfm); + + if (aes_check_keylen(keylen)) { + dev_dbg(ctx->se->dev, "invalid key length (%d)\n", keylen); + return -EINVAL; + } + + if (ctx->fallback_tfm) + crypto_shash_setkey(ctx->fallback_tfm, key, keylen); + + return tegra_key_submit(ctx->se, key, keylen, ctx->alg, &ctx->key_id); +} + +static int tegra_cmac_update(struct ahash_request *req) +{ + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct tegra_cmac_ctx *ctx = crypto_ahash_ctx(tfm); + struct tegra_cmac_reqctx *rctx = ahash_request_ctx(req); + + rctx->task |= SHA_UPDATE; + + return crypto_transfer_hash_request_to_engine(ctx->se->engine, req); +} + +static int tegra_cmac_final(struct ahash_request *req) +{ + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct tegra_cmac_ctx *ctx = crypto_ahash_ctx(tfm); + struct tegra_cmac_reqctx *rctx = ahash_request_ctx(req); + + rctx->task |= SHA_FINAL; + + return crypto_transfer_hash_request_to_engine(ctx->se->engine, req); +} + +static int tegra_cmac_finup(struct ahash_request *req) +{ + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct tegra_cmac_ctx *ctx = crypto_ahash_ctx(tfm); + struct tegra_cmac_reqctx *rctx = ahash_request_ctx(req); + + rctx->task |= SHA_UPDATE | SHA_FINAL; + + return crypto_transfer_hash_request_to_engine(ctx->se->engine, req); +} + +static int tegra_cmac_digest(struct ahash_request *req) +{ + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct tegra_cmac_ctx *ctx = crypto_ahash_ctx(tfm); + struct tegra_cmac_reqctx *rctx = ahash_request_ctx(req); + + tegra_cmac_init(req); + rctx->task |= SHA_UPDATE | SHA_FINAL; + + return crypto_transfer_hash_request_to_engine(ctx->se->engine, req); +} + +static int tegra_cmac_export(struct ahash_request *req, void *out) +{ + struct tegra_cmac_reqctx *rctx = ahash_request_ctx(req); + + memcpy(out, rctx, sizeof(*rctx)); + + return 0; +} + +static int tegra_cmac_import(struct ahash_request *req, const void *in) +{ + struct tegra_cmac_reqctx *rctx = ahash_request_ctx(req); + + memcpy(rctx, in, sizeof(*rctx)); + + return 0; +} + +static struct tegra_se_alg tegra_aead_algs[] = { + { + .alg.aead.op.do_one_request = tegra_gcm_do_one_req, + .alg.aead.base = { + .init = tegra_aead_cra_init, + .exit = tegra_aead_cra_exit, + .setkey = tegra_aead_setkey, + .setauthsize = tegra_gcm_setauthsize, + .encrypt = tegra_aead_encrypt, + .decrypt = tegra_aead_decrypt, + .maxauthsize = AES_BLOCK_SIZE, + .ivsize = GCM_AES_IV_SIZE, + .base = { + .cra_name = "gcm(aes)", + .cra_driver_name = "gcm-aes-tegra", + .cra_priority = 500, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct tegra_aead_ctx), + .cra_alignmask = 0xf, + .cra_module = THIS_MODULE, + }, + } + }, { + .alg.aead.op.do_one_request = tegra_ccm_do_one_req, + .alg.aead.base = { + .init = tegra_aead_cra_init, + .exit = tegra_aead_cra_exit, + .setkey = tegra_aead_setkey, + .setauthsize = tegra_ccm_setauthsize, + .encrypt = tegra_aead_encrypt, + .decrypt = tegra_aead_decrypt, + .maxauthsize = AES_BLOCK_SIZE, + .ivsize = AES_BLOCK_SIZE, + .chunksize = AES_BLOCK_SIZE, + .base = { + .cra_name = "ccm(aes)", + .cra_driver_name = "ccm-aes-tegra", + .cra_priority = 500, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct tegra_aead_ctx), + .cra_alignmask = 0xf, + .cra_module = THIS_MODULE, + }, + } + } +}; + +static struct tegra_se_alg tegra_cmac_algs[] = { + { + .alg.ahash.op.do_one_request = tegra_cmac_do_one_req, + .alg.ahash.base = { + .init = tegra_cmac_init, + .setkey = tegra_cmac_setkey, + .update = tegra_cmac_update, + .final = tegra_cmac_final, + .finup = tegra_cmac_finup, + .digest = tegra_cmac_digest, + .export = tegra_cmac_export, + .import = tegra_cmac_import, + .halg.digestsize = AES_BLOCK_SIZE, + .halg.statesize = sizeof(struct tegra_cmac_reqctx), + .halg.base = { + .cra_name = "cmac(aes)", + .cra_driver_name = "tegra-se-cmac", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_TYPE_AHASH, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct tegra_cmac_ctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + .cra_init = tegra_cmac_cra_init, + .cra_exit = tegra_cmac_cra_exit, + } + } + } +}; + +int tegra_init_aes(struct tegra_se *se) +{ + struct aead_engine_alg *aead_alg; + struct ahash_engine_alg *ahash_alg; + struct skcipher_engine_alg *sk_alg; + int i, ret; + + se->manifest = tegra_aes_kac_manifest; + + for (i = 0; i < ARRAY_SIZE(tegra_aes_algs); i++) { + sk_alg = &tegra_aes_algs[i].alg.skcipher; + tegra_aes_algs[i].se_dev = se; + + ret = crypto_engine_register_skcipher(sk_alg); + if (ret) { + dev_err(se->dev, "failed to register %s\n", + sk_alg->base.base.cra_name); + goto err_aes; + } + } + + for (i = 0; i < ARRAY_SIZE(tegra_aead_algs); i++) { + aead_alg = &tegra_aead_algs[i].alg.aead; + tegra_aead_algs[i].se_dev = se; + + ret = crypto_engine_register_aead(aead_alg); + if (ret) { + dev_err(se->dev, "failed to register %s\n", + aead_alg->base.base.cra_name); + goto err_aead; + } + } + + for (i = 0; i < ARRAY_SIZE(tegra_cmac_algs); i++) { + ahash_alg = &tegra_cmac_algs[i].alg.ahash; + tegra_cmac_algs[i].se_dev = se; + + ret = crypto_engine_register_ahash(ahash_alg); + if (ret) { + dev_err(se->dev, "failed to register %s\n", + ahash_alg->base.halg.base.cra_name); + goto err_cmac; + } + } + + return 0; + +err_cmac: + while (i--) + crypto_engine_unregister_ahash(&tegra_cmac_algs[i].alg.ahash); + + i = ARRAY_SIZE(tegra_aead_algs); +err_aead: + while (i--) + crypto_engine_unregister_aead(&tegra_aead_algs[i].alg.aead); + + i = ARRAY_SIZE(tegra_aes_algs); +err_aes: + while (i--) + crypto_engine_unregister_skcipher(&tegra_aes_algs[i].alg.skcipher); + + return ret; +} + +void tegra_deinit_aes(struct tegra_se *se) +{ + int i; + + for (i = 0; i < ARRAY_SIZE(tegra_aes_algs); i++) + crypto_engine_unregister_skcipher(&tegra_aes_algs[i].alg.skcipher); + + for (i = 0; i < ARRAY_SIZE(tegra_aead_algs); i++) + crypto_engine_unregister_aead(&tegra_aead_algs[i].alg.aead); + + for (i = 0; i < ARRAY_SIZE(tegra_cmac_algs); i++) + crypto_engine_unregister_ahash(&tegra_cmac_algs[i].alg.ahash); +} diff --git a/drivers/crypto/tegra/tegra-se-hash.c b/drivers/crypto/tegra/tegra-se-hash.c new file mode 100644 index 000000000000..4d4bd727f498 --- /dev/null +++ b/drivers/crypto/tegra/tegra-se-hash.c @@ -0,0 +1,1060 @@ +// SPDX-License-Identifier: GPL-2.0-only +// SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +/* + * Crypto driver to handle HASH algorithms using NVIDIA Security Engine. + */ + +#include <linux/clk.h> +#include <linux/dma-mapping.h> +#include <linux/module.h> +#include <linux/of_device.h> +#include <linux/platform_device.h> + +#include <crypto/aes.h> +#include <crypto/sha1.h> +#include <crypto/sha2.h> +#include <crypto/sha3.h> +#include <crypto/internal/des.h> +#include <crypto/engine.h> +#include <crypto/scatterwalk.h> +#include <crypto/internal/hash.h> + +#include "tegra-se.h" + +struct tegra_sha_ctx { + struct tegra_se *se; + unsigned int alg; + bool fallback; + u32 key_id; + struct crypto_ahash *fallback_tfm; +}; + +struct tegra_sha_reqctx { + struct scatterlist *src_sg; + struct tegra_se_datbuf datbuf; + struct tegra_se_datbuf residue; + struct tegra_se_datbuf digest; + unsigned int alg; + unsigned int config; + unsigned int total_len; + unsigned int blk_size; + unsigned int task; + u32 key_id; + u32 result[HASH_RESULT_REG_COUNT]; + struct ahash_request fallback_req; +}; + +static int tegra_sha_get_config(u32 alg) +{ + int cfg = 0; + + switch (alg) { + case SE_ALG_SHA1: + cfg |= SE_SHA_ENC_ALG_SHA; + cfg |= SE_SHA_ENC_MODE_SHA1; + break; + + case SE_ALG_HMAC_SHA224: + cfg |= SE_SHA_ENC_ALG_HMAC; + fallthrough; + case SE_ALG_SHA224: + cfg |= SE_SHA_ENC_ALG_SHA; + cfg |= SE_SHA_ENC_MODE_SHA224; + break; + + case SE_ALG_HMAC_SHA256: + cfg |= SE_SHA_ENC_ALG_HMAC; + fallthrough; + case SE_ALG_SHA256: + cfg |= SE_SHA_ENC_ALG_SHA; + cfg |= SE_SHA_ENC_MODE_SHA256; + break; + + case SE_ALG_HMAC_SHA384: + cfg |= SE_SHA_ENC_ALG_HMAC; + fallthrough; + case SE_ALG_SHA384: + cfg |= SE_SHA_ENC_ALG_SHA; + cfg |= SE_SHA_ENC_MODE_SHA384; + break; + + case SE_ALG_HMAC_SHA512: + cfg |= SE_SHA_ENC_ALG_HMAC; + fallthrough; + case SE_ALG_SHA512: + cfg |= SE_SHA_ENC_ALG_SHA; + cfg |= SE_SHA_ENC_MODE_SHA512; + break; + + case SE_ALG_SHA3_224: + cfg |= SE_SHA_ENC_ALG_SHA; + cfg |= SE_SHA_ENC_MODE_SHA3_224; + break; + case SE_ALG_SHA3_256: + cfg |= SE_SHA_ENC_ALG_SHA; + cfg |= SE_SHA_ENC_MODE_SHA3_256; + break; + case SE_ALG_SHA3_384: + cfg |= SE_SHA_ENC_ALG_SHA; + cfg |= SE_SHA_ENC_MODE_SHA3_384; + break; + case SE_ALG_SHA3_512: + cfg |= SE_SHA_ENC_ALG_SHA; + cfg |= SE_SHA_ENC_MODE_SHA3_512; + break; + default: + return -EINVAL; + } + + return cfg; +} + +static int tegra_sha_fallback_init(struct ahash_request *req) +{ + struct tegra_sha_reqctx *rctx = ahash_request_ctx(req); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct tegra_sha_ctx *ctx = crypto_ahash_ctx(tfm); + + ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm); + rctx->fallback_req.base.flags = req->base.flags & + CRYPTO_TFM_REQ_MAY_SLEEP; + + return crypto_ahash_init(&rctx->fallback_req); +} + +static int tegra_sha_fallback_update(struct ahash_request *req) +{ + struct tegra_sha_reqctx *rctx = ahash_request_ctx(req); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct tegra_sha_ctx *ctx = crypto_ahash_ctx(tfm); + + ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm); + rctx->fallback_req.base.flags = req->base.flags & + CRYPTO_TFM_REQ_MAY_SLEEP; + rctx->fallback_req.nbytes = req->nbytes; + rctx->fallback_req.src = req->src; + + return crypto_ahash_update(&rctx->fallback_req); +} + +static int tegra_sha_fallback_final(struct ahash_request *req) +{ + struct tegra_sha_reqctx *rctx = ahash_request_ctx(req); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct tegra_sha_ctx *ctx = crypto_ahash_ctx(tfm); + + ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm); + rctx->fallback_req.base.flags = req->base.flags & + CRYPTO_TFM_REQ_MAY_SLEEP; + rctx->fallback_req.result = req->result; + + return crypto_ahash_final(&rctx->fallback_req); +} + +static int tegra_sha_fallback_finup(struct ahash_request *req) +{ + struct tegra_sha_reqctx *rctx = ahash_request_ctx(req); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct tegra_sha_ctx *ctx = crypto_ahash_ctx(tfm); + + ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm); + rctx->fallback_req.base.flags = req->base.flags & + CRYPTO_TFM_REQ_MAY_SLEEP; + + rctx->fallback_req.nbytes = req->nbytes; + rctx->fallback_req.src = req->src; + rctx->fallback_req.result = req->result; + + return crypto_ahash_finup(&rctx->fallback_req); +} + +static int tegra_sha_fallback_digest(struct ahash_request *req) +{ + struct tegra_sha_reqctx *rctx = ahash_request_ctx(req); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct tegra_sha_ctx *ctx = crypto_ahash_ctx(tfm); + + ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm); + rctx->fallback_req.base.flags = req->base.flags & + CRYPTO_TFM_REQ_MAY_SLEEP; + + rctx->fallback_req.nbytes = req->nbytes; + rctx->fallback_req.src = req->src; + rctx->fallback_req.result = req->result; + + return crypto_ahash_digest(&rctx->fallback_req); +} + +static int tegra_sha_fallback_import(struct ahash_request *req, const void *in) +{ + struct tegra_sha_reqctx *rctx = ahash_request_ctx(req); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct tegra_sha_ctx *ctx = crypto_ahash_ctx(tfm); + + ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm); + rctx->fallback_req.base.flags = req->base.flags & + CRYPTO_TFM_REQ_MAY_SLEEP; + + return crypto_ahash_import(&rctx->fallback_req, in); +} + +static int tegra_sha_fallback_export(struct ahash_request *req, void *out) +{ + struct tegra_sha_reqctx *rctx = ahash_request_ctx(req); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct tegra_sha_ctx *ctx = crypto_ahash_ctx(tfm); + + ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm); + rctx->fallback_req.base.flags = req->base.flags & + CRYPTO_TFM_REQ_MAY_SLEEP; + + return crypto_ahash_export(&rctx->fallback_req, out); +} + +static int tegra_sha_prep_cmd(struct tegra_se *se, u32 *cpuvaddr, + struct tegra_sha_reqctx *rctx) +{ + u64 msg_len, msg_left; + int i = 0; + + msg_len = rctx->total_len * 8; + msg_left = rctx->datbuf.size * 8; + + /* + * If IN_ADDR_HI_0.SZ > SHA_MSG_LEFT_[0-3] to the HASH engine, + * HW treats it as the last buffer and process the data. + * Therefore, add an extra byte to msg_left if it is not the + * last buffer. + */ + if (rctx->task & SHA_UPDATE) { + msg_left += 8; + msg_len += 8; + } + + cpuvaddr[i++] = host1x_opcode_setpayload(8); + cpuvaddr[i++] = se_host1x_opcode_incr_w(SE_SHA_MSG_LENGTH); + cpuvaddr[i++] = lower_32_bits(msg_len); + cpuvaddr[i++] = upper_32_bits(msg_len); + cpuvaddr[i++] = 0; + cpuvaddr[i++] = 0; + cpuvaddr[i++] = lower_32_bits(msg_left); + cpuvaddr[i++] = upper_32_bits(msg_left); + cpuvaddr[i++] = 0; + cpuvaddr[i++] = 0; + cpuvaddr[i++] = host1x_opcode_setpayload(6); + cpuvaddr[i++] = se_host1x_opcode_incr_w(SE_SHA_CFG); + cpuvaddr[i++] = rctx->config; + + if (rctx->task & SHA_FIRST) { + cpuvaddr[i++] = SE_SHA_TASK_HASH_INIT; + rctx->task &= ~SHA_FIRST; + } else { + cpuvaddr[i++] = 0; + } + + cpuvaddr[i++] = rctx->datbuf.addr; + cpuvaddr[i++] = (u32)(SE_ADDR_HI_MSB(upper_32_bits(rctx->datbuf.addr)) | + SE_ADDR_HI_SZ(rctx->datbuf.size)); + cpuvaddr[i++] = rctx->digest.addr; + cpuvaddr[i++] = (u32)(SE_ADDR_HI_MSB(upper_32_bits(rctx->digest.addr)) | + SE_ADDR_HI_SZ(rctx->digest.size)); + if (rctx->key_id) { + cpuvaddr[i++] = host1x_opcode_setpayload(1); + cpuvaddr[i++] = se_host1x_opcode_nonincr_w(SE_SHA_CRYPTO_CFG); + cpuvaddr[i++] = SE_AES_KEY_INDEX(rctx->key_id); + } + + cpuvaddr[i++] = host1x_opcode_setpayload(1); + cpuvaddr[i++] = se_host1x_opcode_nonincr_w(SE_SHA_OPERATION); + cpuvaddr[i++] = SE_SHA_OP_WRSTALL | + SE_SHA_OP_START | + SE_SHA_OP_LASTBUF; + cpuvaddr[i++] = se_host1x_opcode_nonincr(host1x_uclass_incr_syncpt_r(), 1); + cpuvaddr[i++] = host1x_uclass_incr_syncpt_cond_f(1) | + host1x_uclass_incr_syncpt_indx_f(se->syncpt_id); + + dev_dbg(se->dev, "msg len %llu msg left %llu cfg %#x", + msg_len, msg_left, rctx->config); + + return i; +} + +static void tegra_sha_copy_hash_result(struct tegra_se *se, struct tegra_sha_reqctx *rctx) +{ + int i; + + for (i = 0; i < HASH_RESULT_REG_COUNT; i++) + rctx->result[i] = readl(se->base + se->hw->regs->result + (i * 4)); +} + +static void tegra_sha_paste_hash_result(struct tegra_se *se, struct tegra_sha_reqctx *rctx) +{ + int i; + + for (i = 0; i < HASH_RESULT_REG_COUNT; i++) + writel(rctx->result[i], + se->base + se->hw->regs->result + (i * 4)); +} + +static int tegra_sha_do_update(struct ahash_request *req) +{ + struct tegra_sha_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(req)); + struct tegra_sha_reqctx *rctx = ahash_request_ctx(req); + unsigned int nblks, nresidue, size, ret; + u32 *cpuvaddr = ctx->se->cmdbuf->addr; + + nresidue = (req->nbytes + rctx->residue.size) % rctx->blk_size; + nblks = (req->nbytes + rctx->residue.size) / rctx->blk_size; + + /* + * If nbytes is a multiple of block size and there is no residue, + * then reserve the last block as residue during final() to process. + */ + if (!nresidue && nblks) { + nresidue = rctx->blk_size; + nblks--; + } + + rctx->src_sg = req->src; + rctx->datbuf.size = (req->nbytes + rctx->residue.size) - nresidue; + rctx->total_len += rctx->datbuf.size; + + /* + * If nbytes are less than a block size, copy it residue and + * return. The bytes will be processed in final() + */ + if (nblks < 1) { + scatterwalk_map_and_copy(rctx->residue.buf + rctx->residue.size, + rctx->src_sg, 0, req->nbytes, 0); + + rctx->residue.size += req->nbytes; + return 0; + } + + /* Copy the previous residue first */ + if (rctx->residue.size) + memcpy(rctx->datbuf.buf, rctx->residue.buf, rctx->residue.size); + + scatterwalk_map_and_copy(rctx->datbuf.buf + rctx->residue.size, + rctx->src_sg, 0, req->nbytes - nresidue, 0); + + scatterwalk_map_and_copy(rctx->residue.buf, rctx->src_sg, + req->nbytes - nresidue, nresidue, 0); + + /* Update residue value with the residue after current block */ + rctx->residue.size = nresidue; + + rctx->config = tegra_sha_get_config(rctx->alg) | + SE_SHA_DST_HASH_REG; + + /* + * If this is not the first 'update' call, paste the previous copied + * intermediate results to the registers so that it gets picked up. + * This is to support the import/export functionality. + */ + if (!(rctx->task & SHA_FIRST)) + tegra_sha_paste_hash_result(ctx->se, rctx); + + size = tegra_sha_prep_cmd(ctx->se, cpuvaddr, rctx); + + ret = tegra_se_host1x_submit(ctx->se, size); + + /* + * If this is not the final update, copy the intermediate results + * from the registers so that it can be used in the next 'update' + * call. This is to support the import/export functionality. + */ + if (!(rctx->task & SHA_FINAL)) + tegra_sha_copy_hash_result(ctx->se, rctx); + + return ret; +} + +static int tegra_sha_do_final(struct ahash_request *req) +{ + struct tegra_sha_reqctx *rctx = ahash_request_ctx(req); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct tegra_sha_ctx *ctx = crypto_ahash_ctx(tfm); + struct tegra_se *se = ctx->se; + u32 *cpuvaddr = se->cmdbuf->addr; + int size, ret = 0; + + memcpy(rctx->datbuf.buf, rctx->residue.buf, rctx->residue.size); + rctx->datbuf.size = rctx->residue.size; + rctx->total_len += rctx->residue.size; + + rctx->config = tegra_sha_get_config(rctx->alg) | + SE_SHA_DST_MEMORY; + + size = tegra_sha_prep_cmd(se, cpuvaddr, rctx); + + ret = tegra_se_host1x_submit(se, size); + if (ret) + goto out; + + /* Copy result */ + memcpy(req->result, rctx->digest.buf, rctx->digest.size); + +out: + dma_free_coherent(se->dev, SE_SHA_BUFLEN, + rctx->datbuf.buf, rctx->datbuf.addr); + dma_free_coherent(se->dev, crypto_ahash_blocksize(tfm), + rctx->residue.buf, rctx->residue.addr); + dma_free_coherent(se->dev, rctx->digest.size, rctx->digest.buf, + rctx->digest.addr); + return ret; +} + +static int tegra_sha_do_one_req(struct crypto_engine *engine, void *areq) +{ + struct ahash_request *req = ahash_request_cast(areq); + struct tegra_sha_reqctx *rctx = ahash_request_ctx(req); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct tegra_sha_ctx *ctx = crypto_ahash_ctx(tfm); + struct tegra_se *se = ctx->se; + int ret = 0; + + if (rctx->task & SHA_UPDATE) { + ret = tegra_sha_do_update(req); + rctx->task &= ~SHA_UPDATE; + } + + if (rctx->task & SHA_FINAL) { + ret = tegra_sha_do_final(req); + rctx->task &= ~SHA_FINAL; + } + + crypto_finalize_hash_request(se->engine, req, ret); + + return 0; +} + +static void tegra_sha_init_fallback(struct crypto_ahash *tfm, struct tegra_sha_ctx *ctx, + const char *algname) +{ + unsigned int statesize; + + ctx->fallback_tfm = crypto_alloc_ahash(algname, 0, CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK); + + if (IS_ERR(ctx->fallback_tfm)) { + dev_warn(ctx->se->dev, + "failed to allocate fallback for %s\n", algname); + ctx->fallback_tfm = NULL; + return; + } + + statesize = crypto_ahash_statesize(ctx->fallback_tfm); + + if (statesize > sizeof(struct tegra_sha_reqctx)) + crypto_ahash_set_statesize(tfm, statesize); + + /* Update reqsize if fallback is added */ + crypto_ahash_set_reqsize(tfm, + sizeof(struct tegra_sha_reqctx) + + crypto_ahash_reqsize(ctx->fallback_tfm)); +} + +static int tegra_sha_cra_init(struct crypto_tfm *tfm) +{ + struct tegra_sha_ctx *ctx = crypto_tfm_ctx(tfm); + struct crypto_ahash *ahash_tfm = __crypto_ahash_cast(tfm); + struct ahash_alg *alg = __crypto_ahash_alg(tfm->__crt_alg); + struct tegra_se_alg *se_alg; + const char *algname; + int ret; + + algname = crypto_tfm_alg_name(tfm); + se_alg = container_of(alg, struct tegra_se_alg, alg.ahash.base); + + crypto_ahash_set_reqsize(ahash_tfm, sizeof(struct tegra_sha_reqctx)); + + ctx->se = se_alg->se_dev; + ctx->fallback = false; + ctx->key_id = 0; + + ret = se_algname_to_algid(algname); + if (ret < 0) { + dev_err(ctx->se->dev, "invalid algorithm\n"); + return ret; + } + + if (se_alg->alg_base) + tegra_sha_init_fallback(ahash_tfm, ctx, algname); + + ctx->alg = ret; + + return 0; +} + +static void tegra_sha_cra_exit(struct crypto_tfm *tfm) +{ + struct tegra_sha_ctx *ctx = crypto_tfm_ctx(tfm); + + if (ctx->fallback_tfm) + crypto_free_ahash(ctx->fallback_tfm); + + tegra_key_invalidate(ctx->se, ctx->key_id, ctx->alg); +} + +static int tegra_sha_init(struct ahash_request *req) +{ + struct tegra_sha_reqctx *rctx = ahash_request_ctx(req); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct tegra_sha_ctx *ctx = crypto_ahash_ctx(tfm); + struct tegra_se *se = ctx->se; + + if (ctx->fallback) + return tegra_sha_fallback_init(req); + + rctx->total_len = 0; + rctx->datbuf.size = 0; + rctx->residue.size = 0; + rctx->key_id = ctx->key_id; + rctx->task = SHA_FIRST; + rctx->alg = ctx->alg; + rctx->blk_size = crypto_ahash_blocksize(tfm); + rctx->digest.size = crypto_ahash_digestsize(tfm); + + rctx->digest.buf = dma_alloc_coherent(se->dev, rctx->digest.size, + &rctx->digest.addr, GFP_KERNEL); + if (!rctx->digest.buf) + goto digbuf_fail; + + rctx->residue.buf = dma_alloc_coherent(se->dev, rctx->blk_size, + &rctx->residue.addr, GFP_KERNEL); + if (!rctx->residue.buf) + goto resbuf_fail; + + rctx->datbuf.buf = dma_alloc_coherent(se->dev, SE_SHA_BUFLEN, + &rctx->datbuf.addr, GFP_KERNEL); + if (!rctx->datbuf.buf) + goto datbuf_fail; + + return 0; + +datbuf_fail: + dma_free_coherent(se->dev, rctx->blk_size, rctx->residue.buf, + rctx->residue.addr); +resbuf_fail: + dma_free_coherent(se->dev, SE_SHA_BUFLEN, rctx->datbuf.buf, + rctx->datbuf.addr); +digbuf_fail: + return -ENOMEM; +} + +static int tegra_hmac_fallback_setkey(struct tegra_sha_ctx *ctx, const u8 *key, + unsigned int keylen) +{ + if (!ctx->fallback_tfm) { + dev_dbg(ctx->se->dev, "invalid key length (%d)\n", keylen); + return -EINVAL; + } + + ctx->fallback = true; + return crypto_ahash_setkey(ctx->fallback_tfm, key, keylen); +} + +static int tegra_hmac_setkey(struct crypto_ahash *tfm, const u8 *key, + unsigned int keylen) +{ + struct tegra_sha_ctx *ctx = crypto_ahash_ctx(tfm); + + if (aes_check_keylen(keylen)) + return tegra_hmac_fallback_setkey(ctx, key, keylen); + + ctx->fallback = false; + + return tegra_key_submit(ctx->se, key, keylen, ctx->alg, &ctx->key_id); +} + +static int tegra_sha_update(struct ahash_request *req) +{ + struct tegra_sha_reqctx *rctx = ahash_request_ctx(req); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct tegra_sha_ctx *ctx = crypto_ahash_ctx(tfm); + + if (ctx->fallback) + return tegra_sha_fallback_update(req); + + rctx->task |= SHA_UPDATE; + + return crypto_transfer_hash_request_to_engine(ctx->se->engine, req); +} + +static int tegra_sha_final(struct ahash_request *req) +{ + struct tegra_sha_reqctx *rctx = ahash_request_ctx(req); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct tegra_sha_ctx *ctx = crypto_ahash_ctx(tfm); + + if (ctx->fallback) + return tegra_sha_fallback_final(req); + + rctx->task |= SHA_FINAL; + + return crypto_transfer_hash_request_to_engine(ctx->se->engine, req); +} + +static int tegra_sha_finup(struct ahash_request *req) +{ + struct tegra_sha_reqctx *rctx = ahash_request_ctx(req); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct tegra_sha_ctx *ctx = crypto_ahash_ctx(tfm); + + if (ctx->fallback) + return tegra_sha_fallback_finup(req); + + rctx->task |= SHA_UPDATE | SHA_FINAL; + + return crypto_transfer_hash_request_to_engine(ctx->se->engine, req); +} + +static int tegra_sha_digest(struct ahash_request *req) +{ + struct tegra_sha_reqctx *rctx = ahash_request_ctx(req); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct tegra_sha_ctx *ctx = crypto_ahash_ctx(tfm); + + if (ctx->fallback) + return tegra_sha_fallback_digest(req); + + tegra_sha_init(req); + rctx->task |= SHA_UPDATE | SHA_FINAL; + + return crypto_transfer_hash_request_to_engine(ctx->se->engine, req); +} + +static int tegra_sha_export(struct ahash_request *req, void *out) +{ + struct tegra_sha_reqctx *rctx = ahash_request_ctx(req); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct tegra_sha_ctx *ctx = crypto_ahash_ctx(tfm); + + if (ctx->fallback) + return tegra_sha_fallback_export(req, out); + + memcpy(out, rctx, sizeof(*rctx)); + + return 0; +} + +static int tegra_sha_import(struct ahash_request *req, const void *in) +{ + struct tegra_sha_reqctx *rctx = ahash_request_ctx(req); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct tegra_sha_ctx *ctx = crypto_ahash_ctx(tfm); + + if (ctx->fallback) + return tegra_sha_fallback_import(req, in); + + memcpy(rctx, in, sizeof(*rctx)); + + return 0; +} + +static struct tegra_se_alg tegra_hash_algs[] = { + { + .alg.ahash.op.do_one_request = tegra_sha_do_one_req, + .alg.ahash.base = { + .init = tegra_sha_init, + .update = tegra_sha_update, + .final = tegra_sha_final, + .finup = tegra_sha_finup, + .digest = tegra_sha_digest, + .export = tegra_sha_export, + .import = tegra_sha_import, + .halg.digestsize = SHA1_DIGEST_SIZE, + .halg.statesize = sizeof(struct tegra_sha_reqctx), + .halg.base = { + .cra_name = "sha1", + .cra_driver_name = "tegra-se-sha1", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_TYPE_AHASH, + .cra_blocksize = SHA1_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct tegra_sha_ctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + .cra_init = tegra_sha_cra_init, + .cra_exit = tegra_sha_cra_exit, + } + } + }, { + .alg.ahash.op.do_one_request = tegra_sha_do_one_req, + .alg.ahash.base = { + .init = tegra_sha_init, + .update = tegra_sha_update, + .final = tegra_sha_final, + .finup = tegra_sha_finup, + .digest = tegra_sha_digest, + .export = tegra_sha_export, + .import = tegra_sha_import, + .halg.digestsize = SHA224_DIGEST_SIZE, + .halg.statesize = sizeof(struct tegra_sha_reqctx), + .halg.base = { + .cra_name = "sha224", + .cra_driver_name = "tegra-se-sha224", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_TYPE_AHASH, + .cra_blocksize = SHA224_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct tegra_sha_ctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + .cra_init = tegra_sha_cra_init, + .cra_exit = tegra_sha_cra_exit, + } + } + }, { + .alg.ahash.op.do_one_request = tegra_sha_do_one_req, + .alg.ahash.base = { + .init = tegra_sha_init, + .update = tegra_sha_update, + .final = tegra_sha_final, + .finup = tegra_sha_finup, + .digest = tegra_sha_digest, + .export = tegra_sha_export, + .import = tegra_sha_import, + .halg.digestsize = SHA256_DIGEST_SIZE, + .halg.statesize = sizeof(struct tegra_sha_reqctx), + .halg.base = { + .cra_name = "sha256", + .cra_driver_name = "tegra-se-sha256", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_TYPE_AHASH, + .cra_blocksize = SHA256_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct tegra_sha_ctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + .cra_init = tegra_sha_cra_init, + .cra_exit = tegra_sha_cra_exit, + } + } + }, { + .alg.ahash.op.do_one_request = tegra_sha_do_one_req, + .alg.ahash.base = { + .init = tegra_sha_init, + .update = tegra_sha_update, + .final = tegra_sha_final, + .finup = tegra_sha_finup, + .digest = tegra_sha_digest, + .export = tegra_sha_export, + .import = tegra_sha_import, + .halg.digestsize = SHA384_DIGEST_SIZE, + .halg.statesize = sizeof(struct tegra_sha_reqctx), + .halg.base = { + .cra_name = "sha384", + .cra_driver_name = "tegra-se-sha384", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_TYPE_AHASH, + .cra_blocksize = SHA384_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct tegra_sha_ctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + .cra_init = tegra_sha_cra_init, + .cra_exit = tegra_sha_cra_exit, + } + } + }, { + .alg.ahash.op.do_one_request = tegra_sha_do_one_req, + .alg.ahash.base = { + .init = tegra_sha_init, + .update = tegra_sha_update, + .final = tegra_sha_final, + .finup = tegra_sha_finup, + .digest = tegra_sha_digest, + .export = tegra_sha_export, + .import = tegra_sha_import, + .halg.digestsize = SHA512_DIGEST_SIZE, + .halg.statesize = sizeof(struct tegra_sha_reqctx), + .halg.base = { + .cra_name = "sha512", + .cra_driver_name = "tegra-se-sha512", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_TYPE_AHASH, + .cra_blocksize = SHA512_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct tegra_sha_ctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + .cra_init = tegra_sha_cra_init, + .cra_exit = tegra_sha_cra_exit, + } + } + }, { + .alg.ahash.op.do_one_request = tegra_sha_do_one_req, + .alg.ahash.base = { + .init = tegra_sha_init, + .update = tegra_sha_update, + .final = tegra_sha_final, + .finup = tegra_sha_finup, + .digest = tegra_sha_digest, + .export = tegra_sha_export, + .import = tegra_sha_import, + .halg.digestsize = SHA3_224_DIGEST_SIZE, + .halg.statesize = sizeof(struct tegra_sha_reqctx), + .halg.base = { + .cra_name = "sha3-224", + .cra_driver_name = "tegra-se-sha3-224", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_TYPE_AHASH, + .cra_blocksize = SHA3_224_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct tegra_sha_ctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + .cra_init = tegra_sha_cra_init, + .cra_exit = tegra_sha_cra_exit, + } + } + }, { + .alg.ahash.op.do_one_request = tegra_sha_do_one_req, + .alg.ahash.base = { + .init = tegra_sha_init, + .update = tegra_sha_update, + .final = tegra_sha_final, + .finup = tegra_sha_finup, + .digest = tegra_sha_digest, + .export = tegra_sha_export, + .import = tegra_sha_import, + .halg.digestsize = SHA3_256_DIGEST_SIZE, + .halg.statesize = sizeof(struct tegra_sha_reqctx), + .halg.base = { + .cra_name = "sha3-256", + .cra_driver_name = "tegra-se-sha3-256", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_TYPE_AHASH, + .cra_blocksize = SHA3_256_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct tegra_sha_ctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + .cra_init = tegra_sha_cra_init, + .cra_exit = tegra_sha_cra_exit, + } + } + }, { + .alg.ahash.op.do_one_request = tegra_sha_do_one_req, + .alg.ahash.base = { + .init = tegra_sha_init, + .update = tegra_sha_update, + .final = tegra_sha_final, + .finup = tegra_sha_finup, + .digest = tegra_sha_digest, + .export = tegra_sha_export, + .import = tegra_sha_import, + .halg.digestsize = SHA3_384_DIGEST_SIZE, + .halg.statesize = sizeof(struct tegra_sha_reqctx), + .halg.base = { + .cra_name = "sha3-384", + .cra_driver_name = "tegra-se-sha3-384", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_TYPE_AHASH, + .cra_blocksize = SHA3_384_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct tegra_sha_ctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + .cra_init = tegra_sha_cra_init, + .cra_exit = tegra_sha_cra_exit, + } + } + }, { + .alg.ahash.op.do_one_request = tegra_sha_do_one_req, + .alg.ahash.base = { + .init = tegra_sha_init, + .update = tegra_sha_update, + .final = tegra_sha_final, + .finup = tegra_sha_finup, + .digest = tegra_sha_digest, + .export = tegra_sha_export, + .import = tegra_sha_import, + .halg.digestsize = SHA3_512_DIGEST_SIZE, + .halg.statesize = sizeof(struct tegra_sha_reqctx), + .halg.base = { + .cra_name = "sha3-512", + .cra_driver_name = "tegra-se-sha3-512", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_TYPE_AHASH, + .cra_blocksize = SHA3_512_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct tegra_sha_ctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + .cra_init = tegra_sha_cra_init, + .cra_exit = tegra_sha_cra_exit, + } + } + }, { + .alg_base = "sha224", + .alg.ahash.op.do_one_request = tegra_sha_do_one_req, + .alg.ahash.base = { + .init = tegra_sha_init, + .update = tegra_sha_update, + .final = tegra_sha_final, + .finup = tegra_sha_finup, + .digest = tegra_sha_digest, + .export = tegra_sha_export, + .import = tegra_sha_import, + .setkey = tegra_hmac_setkey, + .halg.digestsize = SHA224_DIGEST_SIZE, + .halg.statesize = sizeof(struct tegra_sha_reqctx), + .halg.base = { + .cra_name = "hmac(sha224)", + .cra_driver_name = "tegra-se-hmac-sha224", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_TYPE_AHASH | CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = SHA224_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct tegra_sha_ctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + .cra_init = tegra_sha_cra_init, + .cra_exit = tegra_sha_cra_exit, + } + } + }, { + .alg_base = "sha256", + .alg.ahash.op.do_one_request = tegra_sha_do_one_req, + .alg.ahash.base = { + .init = tegra_sha_init, + .update = tegra_sha_update, + .final = tegra_sha_final, + .finup = tegra_sha_finup, + .digest = tegra_sha_digest, + .export = tegra_sha_export, + .import = tegra_sha_import, + .setkey = tegra_hmac_setkey, + .halg.digestsize = SHA256_DIGEST_SIZE, + .halg.statesize = sizeof(struct tegra_sha_reqctx), + .halg.base = { + .cra_name = "hmac(sha256)", + .cra_driver_name = "tegra-se-hmac-sha256", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_TYPE_AHASH | CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = SHA256_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct tegra_sha_ctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + .cra_init = tegra_sha_cra_init, + .cra_exit = tegra_sha_cra_exit, + } + } + }, { + .alg_base = "sha384", + .alg.ahash.op.do_one_request = tegra_sha_do_one_req, + .alg.ahash.base = { + .init = tegra_sha_init, + .update = tegra_sha_update, + .final = tegra_sha_final, + .finup = tegra_sha_finup, + .digest = tegra_sha_digest, + .export = tegra_sha_export, + .import = tegra_sha_import, + .setkey = tegra_hmac_setkey, + .halg.digestsize = SHA384_DIGEST_SIZE, + .halg.statesize = sizeof(struct tegra_sha_reqctx), + .halg.base = { + .cra_name = "hmac(sha384)", + .cra_driver_name = "tegra-se-hmac-sha384", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_TYPE_AHASH | CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = SHA384_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct tegra_sha_ctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + .cra_init = tegra_sha_cra_init, + .cra_exit = tegra_sha_cra_exit, + } + } + }, { + .alg_base = "sha512", + .alg.ahash.op.do_one_request = tegra_sha_do_one_req, + .alg.ahash.base = { + .init = tegra_sha_init, + .update = tegra_sha_update, + .final = tegra_sha_final, + .finup = tegra_sha_finup, + .digest = tegra_sha_digest, + .export = tegra_sha_export, + .import = tegra_sha_import, + .setkey = tegra_hmac_setkey, + .halg.digestsize = SHA512_DIGEST_SIZE, + .halg.statesize = sizeof(struct tegra_sha_reqctx), + .halg.base = { + .cra_name = "hmac(sha512)", + .cra_driver_name = "tegra-se-hmac-sha512", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_TYPE_AHASH | CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = SHA512_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct tegra_sha_ctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + .cra_init = tegra_sha_cra_init, + .cra_exit = tegra_sha_cra_exit, + } + } + } +}; + +static int tegra_hash_kac_manifest(u32 user, u32 alg, u32 keylen) +{ + int manifest; + + manifest = SE_KAC_USER_NS; + + switch (alg) { + case SE_ALG_HMAC_SHA224: + case SE_ALG_HMAC_SHA256: + case SE_ALG_HMAC_SHA384: + case SE_ALG_HMAC_SHA512: + manifest |= SE_KAC_HMAC; + break; + default: + return -EINVAL; + } + + switch (keylen) { + case AES_KEYSIZE_128: + manifest |= SE_KAC_SIZE_128; + break; + case AES_KEYSIZE_192: + manifest |= SE_KAC_SIZE_192; + break; + case AES_KEYSIZE_256: + default: + manifest |= SE_KAC_SIZE_256; + break; + } + + return manifest; +} + +int tegra_init_hash(struct tegra_se *se) +{ + struct ahash_engine_alg *alg; + int i, ret; + + se->manifest = tegra_hash_kac_manifest; + + for (i = 0; i < ARRAY_SIZE(tegra_hash_algs); i++) { + tegra_hash_algs[i].se_dev = se; + alg = &tegra_hash_algs[i].alg.ahash; + + ret = crypto_engine_register_ahash(alg); + if (ret) { + dev_err(se->dev, "failed to register %s\n", + alg->base.halg.base.cra_name); + goto sha_err; + } + } + + return 0; + +sha_err: + while (i--) + crypto_engine_unregister_ahash(&tegra_hash_algs[i].alg.ahash); + + return ret; +} + +void tegra_deinit_hash(struct tegra_se *se) +{ + int i; + + for (i = 0; i < ARRAY_SIZE(tegra_hash_algs); i++) + crypto_engine_unregister_ahash(&tegra_hash_algs[i].alg.ahash); +} diff --git a/drivers/crypto/tegra/tegra-se-key.c b/drivers/crypto/tegra/tegra-se-key.c new file mode 100644 index 000000000000..ac14678dbd30 --- /dev/null +++ b/drivers/crypto/tegra/tegra-se-key.c @@ -0,0 +1,156 @@ +// SPDX-License-Identifier: GPL-2.0-only +// SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +/* + * Crypto driver file to manage keys of NVIDIA Security Engine. + */ + +#include <linux/bitops.h> +#include <linux/module.h> +#include <crypto/aes.h> + +#include "tegra-se.h" + +#define SE_KEY_FULL_MASK GENMASK(SE_MAX_KEYSLOT, 0) + +/* Reserve keyslot 0, 14, 15 */ +#define SE_KEY_RSVD_MASK (BIT(0) | BIT(14) | BIT(15)) +#define SE_KEY_VALID_MASK (SE_KEY_FULL_MASK & ~SE_KEY_RSVD_MASK) + +/* Mutex lock to guard keyslots */ +static DEFINE_MUTEX(kslt_lock); + +/* Keyslot bitmask (0 = available, 1 = in use/not available) */ +static u16 tegra_se_keyslots = SE_KEY_RSVD_MASK; + +static u16 tegra_keyslot_alloc(void) +{ + u16 keyid; + + mutex_lock(&kslt_lock); + /* Check if all key slots are full */ + if (tegra_se_keyslots == GENMASK(SE_MAX_KEYSLOT, 0)) { + mutex_unlock(&kslt_lock); + return 0; + } + + keyid = ffz(tegra_se_keyslots); + tegra_se_keyslots |= BIT(keyid); + + mutex_unlock(&kslt_lock); + + return keyid; +} + +static void tegra_keyslot_free(u16 slot) +{ + mutex_lock(&kslt_lock); + tegra_se_keyslots &= ~(BIT(slot)); + mutex_unlock(&kslt_lock); +} + +static unsigned int tegra_key_prep_ins_cmd(struct tegra_se *se, u32 *cpuvaddr, + const u32 *key, u32 keylen, u16 slot, u32 alg) +{ + int i = 0, j; + + cpuvaddr[i++] = host1x_opcode_setpayload(1); + cpuvaddr[i++] = se_host1x_opcode_incr_w(se->hw->regs->op); + cpuvaddr[i++] = SE_AES_OP_WRSTALL | SE_AES_OP_DUMMY; + + cpuvaddr[i++] = host1x_opcode_setpayload(1); + cpuvaddr[i++] = se_host1x_opcode_incr_w(se->hw->regs->manifest); + cpuvaddr[i++] = se->manifest(se->owner, alg, keylen); + cpuvaddr[i++] = host1x_opcode_setpayload(1); + cpuvaddr[i++] = se_host1x_opcode_incr_w(se->hw->regs->key_dst); + + cpuvaddr[i++] = SE_AES_KEY_DST_INDEX(slot); + + for (j = 0; j < keylen / 4; j++) { + /* Set key address */ + cpuvaddr[i++] = host1x_opcode_setpayload(1); + cpuvaddr[i++] = se_host1x_opcode_incr_w(se->hw->regs->key_addr); + cpuvaddr[i++] = j; + + /* Set key data */ + cpuvaddr[i++] = host1x_opcode_setpayload(1); + cpuvaddr[i++] = se_host1x_opcode_incr_w(se->hw->regs->key_data); + cpuvaddr[i++] = key[j]; + } + + cpuvaddr[i++] = host1x_opcode_setpayload(1); + cpuvaddr[i++] = se_host1x_opcode_incr_w(se->hw->regs->config); + cpuvaddr[i++] = SE_CFG_INS; + + cpuvaddr[i++] = host1x_opcode_setpayload(1); + cpuvaddr[i++] = se_host1x_opcode_incr_w(se->hw->regs->op); + cpuvaddr[i++] = SE_AES_OP_WRSTALL | SE_AES_OP_START | + SE_AES_OP_LASTBUF; + + cpuvaddr[i++] = se_host1x_opcode_nonincr(host1x_uclass_incr_syncpt_r(), 1); + cpuvaddr[i++] = host1x_uclass_incr_syncpt_cond_f(1) | + host1x_uclass_incr_syncpt_indx_f(se->syncpt_id); + + dev_dbg(se->dev, "key-slot %u key-manifest %#x\n", + slot, se->manifest(se->owner, alg, keylen)); + + return i; +} + +static bool tegra_key_in_kslt(u32 keyid) +{ + bool ret; + + if (keyid > SE_MAX_KEYSLOT) + return false; + + mutex_lock(&kslt_lock); + ret = ((BIT(keyid) & SE_KEY_VALID_MASK) && + (BIT(keyid) & tegra_se_keyslots)); + mutex_unlock(&kslt_lock); + + return ret; +} + +static int tegra_key_insert(struct tegra_se *se, const u8 *key, + u32 keylen, u16 slot, u32 alg) +{ + const u32 *keyval = (u32 *)key; + u32 *addr = se->cmdbuf->addr, size; + + size = tegra_key_prep_ins_cmd(se, addr, keyval, keylen, slot, alg); + + return tegra_se_host1x_submit(se, size); +} + +void tegra_key_invalidate(struct tegra_se *se, u32 keyid, u32 alg) +{ + u8 zkey[AES_MAX_KEY_SIZE] = {0}; + + if (!keyid) + return; + + /* Overwrite the key with 0s */ + tegra_key_insert(se, zkey, AES_MAX_KEY_SIZE, keyid, alg); + + tegra_keyslot_free(keyid); +} + +int tegra_key_submit(struct tegra_se *se, const u8 *key, u32 keylen, u32 alg, u32 *keyid) +{ + int ret; + + /* Use the existing slot if it is already allocated */ + if (!tegra_key_in_kslt(*keyid)) { + *keyid = tegra_keyslot_alloc(); + if (!(*keyid)) { + dev_err(se->dev, "failed to allocate key slot\n"); + return -ENOMEM; + } + } + + ret = tegra_key_insert(se, key, keylen, *keyid, alg); + if (ret) + return ret; + + return 0; +} diff --git a/drivers/crypto/tegra/tegra-se-main.c b/drivers/crypto/tegra/tegra-se-main.c new file mode 100644 index 000000000000..9955874b3dc3 --- /dev/null +++ b/drivers/crypto/tegra/tegra-se-main.c @@ -0,0 +1,437 @@ +// SPDX-License-Identifier: GPL-2.0-only +// SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +/* + * Crypto driver for NVIDIA Security Engine in Tegra Chips + */ + +#include <linux/clk.h> +#include <linux/dma-mapping.h> +#include <linux/module.h> +#include <linux/platform_device.h> +#include <linux/mod_devicetable.h> + +#include <crypto/engine.h> + +#include "tegra-se.h" + +static struct host1x_bo *tegra_se_cmdbuf_get(struct host1x_bo *host_bo) +{ + struct tegra_se_cmdbuf *cmdbuf = container_of(host_bo, struct tegra_se_cmdbuf, bo); + + kref_get(&cmdbuf->ref); + + return host_bo; +} + +static void tegra_se_cmdbuf_release(struct kref *ref) +{ + struct tegra_se_cmdbuf *cmdbuf = container_of(ref, struct tegra_se_cmdbuf, ref); + + dma_free_attrs(cmdbuf->dev, cmdbuf->size, cmdbuf->addr, + cmdbuf->iova, 0); + + kfree(cmdbuf); +} + +static void tegra_se_cmdbuf_put(struct host1x_bo *host_bo) +{ + struct tegra_se_cmdbuf *cmdbuf = container_of(host_bo, struct tegra_se_cmdbuf, bo); + + kref_put(&cmdbuf->ref, tegra_se_cmdbuf_release); +} + +static struct host1x_bo_mapping * +tegra_se_cmdbuf_pin(struct device *dev, struct host1x_bo *bo, enum dma_data_direction direction) +{ + struct tegra_se_cmdbuf *cmdbuf = container_of(bo, struct tegra_se_cmdbuf, bo); + struct host1x_bo_mapping *map; + int err; + + map = kzalloc(sizeof(*map), GFP_KERNEL); + if (!map) + return ERR_PTR(-ENOMEM); + + kref_init(&map->ref); + map->bo = host1x_bo_get(bo); + map->direction = direction; + map->dev = dev; + + map->sgt = kzalloc(sizeof(*map->sgt), GFP_KERNEL); + if (!map->sgt) { + err = -ENOMEM; + goto free; + } + + err = dma_get_sgtable(dev, map->sgt, cmdbuf->addr, + cmdbuf->iova, cmdbuf->words * 4); + if (err) + goto free_sgt; + + err = dma_map_sgtable(dev, map->sgt, direction, 0); + if (err) + goto free_sgt; + + map->phys = sg_dma_address(map->sgt->sgl); + map->size = cmdbuf->words * 4; + map->chunks = err; + + return map; + +free_sgt: + sg_free_table(map->sgt); + kfree(map->sgt); +free: + kfree(map); + return ERR_PTR(err); +} + +static void tegra_se_cmdbuf_unpin(struct host1x_bo_mapping *map) +{ + if (!map) + return; + + dma_unmap_sgtable(map->dev, map->sgt, map->direction, 0); + sg_free_table(map->sgt); + kfree(map->sgt); + host1x_bo_put(map->bo); + + kfree(map); +} + +static void *tegra_se_cmdbuf_mmap(struct host1x_bo *host_bo) +{ + struct tegra_se_cmdbuf *cmdbuf = container_of(host_bo, struct tegra_se_cmdbuf, bo); + + return cmdbuf->addr; +} + +static void tegra_se_cmdbuf_munmap(struct host1x_bo *host_bo, void *addr) +{ +} + +static const struct host1x_bo_ops tegra_se_cmdbuf_ops = { + .get = tegra_se_cmdbuf_get, + .put = tegra_se_cmdbuf_put, + .pin = tegra_se_cmdbuf_pin, + .unpin = tegra_se_cmdbuf_unpin, + .mmap = tegra_se_cmdbuf_mmap, + .munmap = tegra_se_cmdbuf_munmap, +}; + +static struct tegra_se_cmdbuf *tegra_se_host1x_bo_alloc(struct tegra_se *se, ssize_t size) +{ + struct tegra_se_cmdbuf *cmdbuf; + struct device *dev = se->dev->parent; + + cmdbuf = kzalloc(sizeof(*cmdbuf), GFP_KERNEL); + if (!cmdbuf) + return NULL; + + cmdbuf->addr = dma_alloc_attrs(dev, size, &cmdbuf->iova, + GFP_KERNEL, 0); + if (!cmdbuf->addr) + return NULL; + + cmdbuf->size = size; + cmdbuf->dev = dev; + + host1x_bo_init(&cmdbuf->bo, &tegra_se_cmdbuf_ops); + kref_init(&cmdbuf->ref); + + return cmdbuf; +} + +int tegra_se_host1x_submit(struct tegra_se *se, u32 size) +{ + struct host1x_job *job; + int ret; + + job = host1x_job_alloc(se->channel, 1, 0, true); + if (!job) { + dev_err(se->dev, "failed to allocate host1x job\n"); + return -ENOMEM; + } + + job->syncpt = host1x_syncpt_get(se->syncpt); + job->syncpt_incrs = 1; + job->client = &se->client; + job->class = se->client.class; + job->serialize = true; + job->engine_fallback_streamid = se->stream_id; + job->engine_streamid_offset = SE_STREAM_ID; + + se->cmdbuf->words = size; + + host1x_job_add_gather(job, &se->cmdbuf->bo, size, 0); + + ret = host1x_job_pin(job, se->dev); + if (ret) { + dev_err(se->dev, "failed to pin host1x job\n"); + goto job_put; + } + + ret = host1x_job_submit(job); + if (ret) { + dev_err(se->dev, "failed to submit host1x job\n"); + goto job_unpin; + } + + ret = host1x_syncpt_wait(job->syncpt, job->syncpt_end, + MAX_SCHEDULE_TIMEOUT, NULL); + if (ret) { + dev_err(se->dev, "host1x job timed out\n"); + return ret; + } + + host1x_job_put(job); + return 0; + +job_unpin: + host1x_job_unpin(job); +job_put: + host1x_job_put(job); + + return ret; +} + +static int tegra_se_client_init(struct host1x_client *client) +{ + struct tegra_se *se = container_of(client, struct tegra_se, client); + int ret; + + se->channel = host1x_channel_request(&se->client); + if (!se->channel) { + dev_err(se->dev, "host1x channel map failed\n"); + return -ENODEV; + } + + se->syncpt = host1x_syncpt_request(&se->client, 0); + if (!se->syncpt) { + dev_err(se->dev, "host1x syncpt allocation failed\n"); + ret = -EINVAL; + goto channel_put; + } + + se->syncpt_id = host1x_syncpt_id(se->syncpt); + + se->cmdbuf = tegra_se_host1x_bo_alloc(se, SZ_4K); + if (!se->cmdbuf) { + ret = -ENOMEM; + goto syncpt_put; + } + + ret = se->hw->init_alg(se); + if (ret) { + dev_err(se->dev, "failed to register algorithms\n"); + goto cmdbuf_put; + } + + return 0; + +cmdbuf_put: + tegra_se_cmdbuf_put(&se->cmdbuf->bo); +syncpt_put: + host1x_syncpt_put(se->syncpt); +channel_put: + host1x_channel_put(se->channel); + + return ret; +} + +static int tegra_se_client_deinit(struct host1x_client *client) +{ + struct tegra_se *se = container_of(client, struct tegra_se, client); + + se->hw->deinit_alg(se); + tegra_se_cmdbuf_put(&se->cmdbuf->bo); + host1x_syncpt_put(se->syncpt); + host1x_channel_put(se->channel); + + return 0; +} + +static const struct host1x_client_ops tegra_se_client_ops = { + .init = tegra_se_client_init, + .exit = tegra_se_client_deinit, +}; + +static int tegra_se_host1x_register(struct tegra_se *se) +{ + INIT_LIST_HEAD(&se->client.list); + se->client.dev = se->dev; + se->client.ops = &tegra_se_client_ops; + se->client.class = se->hw->host1x_class; + se->client.num_syncpts = 1; + + host1x_client_register(&se->client); + + return 0; +} + +static int tegra_se_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct tegra_se *se; + int ret; + + se = devm_kzalloc(dev, sizeof(*se), GFP_KERNEL); + if (!se) + return -ENOMEM; + + se->dev = dev; + se->owner = TEGRA_GPSE_ID; + se->hw = device_get_match_data(&pdev->dev); + + se->base = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(se->base)) + return PTR_ERR(se->base); + + dma_set_mask_and_coherent(dev, DMA_BIT_MASK(39)); + platform_set_drvdata(pdev, se); + + se->clk = devm_clk_get_enabled(se->dev, NULL); + if (IS_ERR(se->clk)) + return dev_err_probe(dev, PTR_ERR(se->clk), + "failed to enable clocks\n"); + + if (!tegra_dev_iommu_get_stream_id(dev, &se->stream_id)) + return dev_err_probe(dev, -ENODEV, + "failed to get IOMMU stream ID\n"); + + writel(se->stream_id, se->base + SE_STREAM_ID); + + se->engine = crypto_engine_alloc_init(dev, 0); + if (!se->engine) + return dev_err_probe(dev, -ENOMEM, "failed to init crypto engine\n"); + + ret = crypto_engine_start(se->engine); + if (ret) { + crypto_engine_exit(se->engine); + return dev_err_probe(dev, ret, "failed to start crypto engine\n"); + } + + ret = tegra_se_host1x_register(se); + if (ret) { + crypto_engine_stop(se->engine); + crypto_engine_exit(se->engine); + return dev_err_probe(dev, ret, "failed to init host1x params\n"); + } + + return 0; +} + +static void tegra_se_remove(struct platform_device *pdev) +{ + struct tegra_se *se = platform_get_drvdata(pdev); + + crypto_engine_stop(se->engine); + crypto_engine_exit(se->engine); + iommu_fwspec_free(se->dev); + host1x_client_unregister(&se->client); +} + +static const struct tegra_se_regs tegra234_aes1_regs = { + .config = SE_AES1_CFG, + .op = SE_AES1_OPERATION, + .last_blk = SE_AES1_LAST_BLOCK, + .linear_ctr = SE_AES1_LINEAR_CTR, + .aad_len = SE_AES1_AAD_LEN, + .cryp_msg_len = SE_AES1_CRYPTO_MSG_LEN, + .manifest = SE_AES1_KEYMANIFEST, + .key_addr = SE_AES1_KEY_ADDR, + .key_data = SE_AES1_KEY_DATA, + .key_dst = SE_AES1_KEY_DST, + .result = SE_AES1_CMAC_RESULT, +}; + +static const struct tegra_se_regs tegra234_hash_regs = { + .config = SE_SHA_CFG, + .op = SE_SHA_OPERATION, + .manifest = SE_SHA_KEYMANIFEST, + .key_addr = SE_SHA_KEY_ADDR, + .key_data = SE_SHA_KEY_DATA, + .key_dst = SE_SHA_KEY_DST, + .result = SE_SHA_HASH_RESULT, +}; + +static const struct tegra_se_hw tegra234_aes_hw = { + .regs = &tegra234_aes1_regs, + .kac_ver = 1, + .host1x_class = 0x3b, + .init_alg = tegra_init_aes, + .deinit_alg = tegra_deinit_aes, +}; + +static const struct tegra_se_hw tegra234_hash_hw = { + .regs = &tegra234_hash_regs, + .kac_ver = 1, + .host1x_class = 0x3d, + .init_alg = tegra_init_hash, + .deinit_alg = tegra_deinit_hash, +}; + +static const struct of_device_id tegra_se_of_match[] = { + { + .compatible = "nvidia,tegra234-se-aes", + .data = &tegra234_aes_hw + }, { + .compatible = "nvidia,tegra234-se-hash", + .data = &tegra234_hash_hw, + }, + { }, +}; +MODULE_DEVICE_TABLE(of, tegra_se_of_match); + +static struct platform_driver tegra_se_driver = { + .driver = { + .name = "tegra-se", + .of_match_table = tegra_se_of_match, + }, + .probe = tegra_se_probe, + .remove_new = tegra_se_remove, +}; + +static int tegra_se_host1x_probe(struct host1x_device *dev) +{ + return host1x_device_init(dev); +} + +static int tegra_se_host1x_remove(struct host1x_device *dev) +{ + host1x_device_exit(dev); + + return 0; +} + +static struct host1x_driver tegra_se_host1x_driver = { + .driver = { + .name = "tegra-se-host1x", + }, + .probe = tegra_se_host1x_probe, + .remove = tegra_se_host1x_remove, + .subdevs = tegra_se_of_match, +}; + +static int __init tegra_se_module_init(void) +{ + int ret; + + ret = host1x_driver_register(&tegra_se_host1x_driver); + if (ret) + return ret; + + return platform_driver_register(&tegra_se_driver); +} + +static void __exit tegra_se_module_exit(void) +{ + host1x_driver_unregister(&tegra_se_host1x_driver); + platform_driver_unregister(&tegra_se_driver); +} + +module_init(tegra_se_module_init); +module_exit(tegra_se_module_exit); + +MODULE_DESCRIPTION("NVIDIA Tegra Security Engine Driver"); +MODULE_AUTHOR("Akhil R <akhilrajeev@nvidia.com>"); +MODULE_LICENSE("GPL"); diff --git a/drivers/crypto/tegra/tegra-se.h b/drivers/crypto/tegra/tegra-se.h new file mode 100644 index 000000000000..b9dd7ceb8783 --- /dev/null +++ b/drivers/crypto/tegra/tegra-se.h @@ -0,0 +1,560 @@ +/* SPDX-License-Identifier: GPL-2.0-only + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * + * Header file for NVIDIA Security Engine driver. + */ + +#ifndef _TEGRA_SE_H +#define _TEGRA_SE_H + +#include <linux/bitfield.h> +#include <linux/iommu.h> +#include <linux/host1x.h> +#include <crypto/aead.h> +#include <crypto/engine.h> +#include <crypto/hash.h> +#include <crypto/sha1.h> +#include <crypto/sha3.h> +#include <crypto/skcipher.h> + +#define SE_OWNERSHIP 0x14 +#define SE_OWNERSHIP_UID(x) FIELD_GET(GENMASK(7, 0), x) +#define TEGRA_GPSE_ID 3 + +#define SE_STREAM_ID 0x90 + +#define SE_SHA_CFG 0x4004 +#define SE_SHA_KEY_ADDR 0x4094 +#define SE_SHA_KEY_DATA 0x4098 +#define SE_SHA_KEYMANIFEST 0x409c +#define SE_SHA_CRYPTO_CFG 0x40a4 +#define SE_SHA_KEY_DST 0x40a8 +#define SE_SHA_SRC_KSLT 0x4180 +#define SE_SHA_TGT_KSLT 0x4184 +#define SE_SHA_MSG_LENGTH 0x401c +#define SE_SHA_OPERATION 0x407c +#define SE_SHA_HASH_RESULT 0x40b0 + +#define SE_SHA_ENC_MODE(x) FIELD_PREP(GENMASK(31, 24), x) +#define SE_SHA_ENC_MODE_SHA1 SE_SHA_ENC_MODE(0) +#define SE_SHA_ENC_MODE_SHA224 SE_SHA_ENC_MODE(4) +#define SE_SHA_ENC_MODE_SHA256 SE_SHA_ENC_MODE(5) +#define SE_SHA_ENC_MODE_SHA384 SE_SHA_ENC_MODE(6) +#define SE_SHA_ENC_MODE_SHA512 SE_SHA_ENC_MODE(7) +#define SE_SHA_ENC_MODE_SHA_CTX_INTEGRITY SE_SHA_ENC_MODE(8) +#define SE_SHA_ENC_MODE_SHA3_224 SE_SHA_ENC_MODE(9) +#define SE_SHA_ENC_MODE_SHA3_256 SE_SHA_ENC_MODE(10) +#define SE_SHA_ENC_MODE_SHA3_384 SE_SHA_ENC_MODE(11) +#define SE_SHA_ENC_MODE_SHA3_512 SE_SHA_ENC_MODE(12) +#define SE_SHA_ENC_MODE_SHAKE128 SE_SHA_ENC_MODE(13) +#define SE_SHA_ENC_MODE_SHAKE256 SE_SHA_ENC_MODE(14) +#define SE_SHA_ENC_MODE_HMAC_SHA256_1KEY SE_SHA_ENC_MODE(0) +#define SE_SHA_ENC_MODE_HMAC_SHA256_2KEY SE_SHA_ENC_MODE(1) +#define SE_SHA_ENC_MODE_SM3_256 SE_SHA_ENC_MODE(0) + +#define SE_SHA_CFG_ENC_ALG(x) FIELD_PREP(GENMASK(15, 12), x) +#define SE_SHA_ENC_ALG_NOP SE_SHA_CFG_ENC_ALG(0) +#define SE_SHA_ENC_ALG_SHA_ENC SE_SHA_CFG_ENC_ALG(1) +#define SE_SHA_ENC_ALG_RNG SE_SHA_CFG_ENC_ALG(2) +#define SE_SHA_ENC_ALG_SHA SE_SHA_CFG_ENC_ALG(3) +#define SE_SHA_ENC_ALG_SM3 SE_SHA_CFG_ENC_ALG(4) +#define SE_SHA_ENC_ALG_HMAC SE_SHA_CFG_ENC_ALG(7) +#define SE_SHA_ENC_ALG_KDF SE_SHA_CFG_ENC_ALG(8) +#define SE_SHA_ENC_ALG_KEY_INVLD SE_SHA_CFG_ENC_ALG(10) +#define SE_SHA_ENC_ALG_KEY_INQUIRE SE_SHA_CFG_ENC_ALG(12) +#define SE_SHA_ENC_ALG_INS SE_SHA_CFG_ENC_ALG(13) + +#define SE_SHA_OP_LASTBUF FIELD_PREP(BIT(16), 1) +#define SE_SHA_OP_WRSTALL FIELD_PREP(BIT(15), 1) + +#define SE_SHA_OP_OP(x) FIELD_PREP(GENMASK(2, 0), x) +#define SE_SHA_OP_START SE_SHA_OP_OP(1) +#define SE_SHA_OP_RESTART_OUT SE_SHA_OP_OP(2) +#define SE_SHA_OP_RESTART_IN SE_SHA_OP_OP(4) +#define SE_SHA_OP_RESTART_INOUT SE_SHA_OP_OP(5) +#define SE_SHA_OP_DUMMY SE_SHA_OP_OP(6) + +#define SE_SHA_CFG_DEC_ALG(x) FIELD_PREP(GENMASK(11, 8), x) +#define SE_SHA_DEC_ALG_NOP SE_SHA_CFG_DEC_ALG(0) +#define SE_SHA_DEC_ALG_AES_DEC SE_SHA_CFG_DEC_ALG(1) +#define SE_SHA_DEC_ALG_HMAC SE_SHA_CFG_DEC_ALG(7) +#define SE_SHA_DEC_ALG_HMAC_VERIFY SE_SHA_CFG_DEC_ALG(9) + +#define SE_SHA_CFG_DST(x) FIELD_PREP(GENMASK(4, 2), x) +#define SE_SHA_DST_MEMORY SE_SHA_CFG_DST(0) +#define SE_SHA_DST_HASH_REG SE_SHA_CFG_DST(1) +#define SE_SHA_DST_KEYTABLE SE_SHA_CFG_DST(2) +#define SE_SHA_DST_SRK SE_SHA_CFG_DST(3) + +#define SE_SHA_TASK_HASH_INIT BIT(0) + +/* AES Configuration */ +#define SE_AES0_CFG 0x1004 +#define SE_AES0_CRYPTO_CONFIG 0x1008 +#define SE_AES0_KEY_DST 0x1030 +#define SE_AES0_OPERATION 0x1038 +#define SE_AES0_LINEAR_CTR 0x101c +#define SE_AES0_LAST_BLOCK 0x102c +#define SE_AES0_KEY_ADDR 0x10bc +#define SE_AES0_KEY_DATA 0x10c0 +#define SE_AES0_CMAC_RESULT 0x10c4 +#define SE_AES0_SRC_KSLT 0x1100 +#define SE_AES0_TGT_KSLT 0x1104 +#define SE_AES0_KEYMANIFEST 0x1114 +#define SE_AES0_AAD_LEN 0x112c +#define SE_AES0_CRYPTO_MSG_LEN 0x1134 + +#define SE_AES1_CFG 0x2004 +#define SE_AES1_CRYPTO_CONFIG 0x2008 +#define SE_AES1_KEY_DST 0x2030 +#define SE_AES1_OPERATION 0x2038 +#define SE_AES1_LINEAR_CTR 0x201c +#define SE_AES1_LAST_BLOCK 0x202c +#define SE_AES1_KEY_ADDR 0x20bc +#define SE_AES1_KEY_DATA 0x20c0 +#define SE_AES1_CMAC_RESULT 0x20c4 +#define SE_AES1_SRC_KSLT 0x2100 +#define SE_AES1_TGT_KSLT 0x2104 +#define SE_AES1_KEYMANIFEST 0x2114 +#define SE_AES1_AAD_LEN 0x212c +#define SE_AES1_CRYPTO_MSG_LEN 0x2134 + +#define SE_AES_CFG_ENC_MODE(x) FIELD_PREP(GENMASK(31, 24), x) +#define SE_AES_ENC_MODE_GMAC SE_AES_CFG_ENC_MODE(3) +#define SE_AES_ENC_MODE_GCM SE_AES_CFG_ENC_MODE(4) +#define SE_AES_ENC_MODE_GCM_FINAL SE_AES_CFG_ENC_MODE(5) +#define SE_AES_ENC_MODE_CMAC SE_AES_CFG_ENC_MODE(7) +#define SE_AES_ENC_MODE_CBC_MAC SE_AES_CFG_ENC_MODE(12) + +#define SE_AES_CFG_DEC_MODE(x) FIELD_PREP(GENMASK(23, 16), x) +#define SE_AES_DEC_MODE_GMAC SE_AES_CFG_DEC_MODE(3) +#define SE_AES_DEC_MODE_GCM SE_AES_CFG_DEC_MODE(4) +#define SE_AES_DEC_MODE_GCM_FINAL SE_AES_CFG_DEC_MODE(5) +#define SE_AES_DEC_MODE_CBC_MAC SE_AES_CFG_DEC_MODE(12) + +#define SE_AES_CFG_ENC_ALG(x) FIELD_PREP(GENMASK(15, 12), x) +#define SE_AES_ENC_ALG_NOP SE_AES_CFG_ENC_ALG(0) +#define SE_AES_ENC_ALG_AES_ENC SE_AES_CFG_ENC_ALG(1) +#define SE_AES_ENC_ALG_RNG SE_AES_CFG_ENC_ALG(2) +#define SE_AES_ENC_ALG_SHA SE_AES_CFG_ENC_ALG(3) +#define SE_AES_ENC_ALG_HMAC SE_AES_CFG_ENC_ALG(7) +#define SE_AES_ENC_ALG_KDF SE_AES_CFG_ENC_ALG(8) +#define SE_AES_ENC_ALG_INS SE_AES_CFG_ENC_ALG(13) + +#define SE_AES_CFG_DEC_ALG(x) FIELD_PREP(GENMASK(11, 8), x) +#define SE_AES_DEC_ALG_NOP SE_AES_CFG_DEC_ALG(0) +#define SE_AES_DEC_ALG_AES_DEC SE_AES_CFG_DEC_ALG(1) + +#define SE_AES_CFG_DST(x) FIELD_PREP(GENMASK(4, 2), x) +#define SE_AES_DST_MEMORY SE_AES_CFG_DST(0) +#define SE_AES_DST_HASH_REG SE_AES_CFG_DST(1) +#define SE_AES_DST_KEYTABLE SE_AES_CFG_DST(2) +#define SE_AES_DST_SRK SE_AES_CFG_DST(3) + +/* AES Crypto Configuration */ +#define SE_AES_KEY2_INDEX(x) FIELD_PREP(GENMASK(31, 28), x) +#define SE_AES_KEY_INDEX(x) FIELD_PREP(GENMASK(27, 24), x) + +#define SE_AES_CRYPTO_CFG_SCC_DIS FIELD_PREP(BIT(20), 1) + +#define SE_AES_CRYPTO_CFG_CTR_CNTN(x) FIELD_PREP(GENMASK(18, 11), x) + +#define SE_AES_CRYPTO_CFG_IV_MODE(x) FIELD_PREP(BIT(10), x) +#define SE_AES_IV_MODE_SWIV SE_AES_CRYPTO_CFG_IV_MODE(0) +#define SE_AES_IV_MODE_HWIV SE_AES_CRYPTO_CFG_IV_MODE(1) + +#define SE_AES_CRYPTO_CFG_CORE_SEL(x) FIELD_PREP(BIT(9), x) +#define SE_AES_CORE_SEL_DECRYPT SE_AES_CRYPTO_CFG_CORE_SEL(0) +#define SE_AES_CORE_SEL_ENCRYPT SE_AES_CRYPTO_CFG_CORE_SEL(1) + +#define SE_AES_CRYPTO_CFG_IV_SEL(x) FIELD_PREP(GENMASK(8, 7), x) +#define SE_AES_IV_SEL_UPDATED SE_AES_CRYPTO_CFG_IV_SEL(1) +#define SE_AES_IV_SEL_REG SE_AES_CRYPTO_CFG_IV_SEL(2) +#define SE_AES_IV_SEL_RANDOM SE_AES_CRYPTO_CFG_IV_SEL(3) + +#define SE_AES_CRYPTO_CFG_VCTRAM_SEL(x) FIELD_PREP(GENMASK(6, 5), x) +#define SE_AES_VCTRAM_SEL_MEMORY SE_AES_CRYPTO_CFG_VCTRAM_SEL(0) +#define SE_AES_VCTRAM_SEL_TWEAK SE_AES_CRYPTO_CFG_VCTRAM_SEL(1) +#define SE_AES_VCTRAM_SEL_AESOUT SE_AES_CRYPTO_CFG_VCTRAM_SEL(2) +#define SE_AES_VCTRAM_SEL_PREV_MEM SE_AES_CRYPTO_CFG_VCTRAM_SEL(3) + +#define SE_AES_CRYPTO_CFG_INPUT_SEL(x) FIELD_PREP(GENMASK(4, 3), x) +#define SE_AES_INPUT_SEL_MEMORY SE_AES_CRYPTO_CFG_INPUT_SEL(0) +#define SE_AES_INPUT_SEL_RANDOM SE_AES_CRYPTO_CFG_INPUT_SEL(1) +#define SE_AES_INPUT_SEL_AESOUT SE_AES_CRYPTO_CFG_INPUT_SEL(2) +#define SE_AES_INPUT_SEL_LINEAR_CTR SE_AES_CRYPTO_CFG_INPUT_SEL(3) +#define SE_AES_INPUT_SEL_REG SE_AES_CRYPTO_CFG_INPUT_SEL(1) + +#define SE_AES_CRYPTO_CFG_XOR_POS(x) FIELD_PREP(GENMASK(2, 1), x) +#define SE_AES_XOR_POS_BYPASS SE_AES_CRYPTO_CFG_XOR_POS(0) +#define SE_AES_XOR_POS_BOTH SE_AES_CRYPTO_CFG_XOR_POS(1) +#define SE_AES_XOR_POS_TOP SE_AES_CRYPTO_CFG_XOR_POS(2) +#define SE_AES_XOR_POS_BOTTOM SE_AES_CRYPTO_CFG_XOR_POS(3) + +#define SE_AES_CRYPTO_CFG_HASH_EN(x) FIELD_PREP(BIT(0), x) +#define SE_AES_HASH_DISABLE SE_AES_CRYPTO_CFG_HASH_EN(0) +#define SE_AES_HASH_ENABLE SE_AES_CRYPTO_CFG_HASH_EN(1) + +#define SE_LAST_BLOCK_VAL(x) FIELD_PREP(GENMASK(19, 0), x) +#define SE_LAST_BLOCK_RES_BITS(x) FIELD_PREP(GENMASK(26, 20), x) + +#define SE_AES_OP_LASTBUF FIELD_PREP(BIT(16), 1) +#define SE_AES_OP_WRSTALL FIELD_PREP(BIT(15), 1) +#define SE_AES_OP_FINAL FIELD_PREP(BIT(5), 1) +#define SE_AES_OP_INIT FIELD_PREP(BIT(4), 1) + +#define SE_AES_OP_OP(x) FIELD_PREP(GENMASK(2, 0), x) +#define SE_AES_OP_START SE_AES_OP_OP(1) +#define SE_AES_OP_RESTART_OUT SE_AES_OP_OP(2) +#define SE_AES_OP_RESTART_IN SE_AES_OP_OP(4) +#define SE_AES_OP_RESTART_INOUT SE_AES_OP_OP(5) +#define SE_AES_OP_DUMMY SE_AES_OP_OP(6) + +#define SE_KAC_SIZE(x) FIELD_PREP(GENMASK(15, 14), x) +#define SE_KAC_SIZE_128 SE_KAC_SIZE(0) +#define SE_KAC_SIZE_192 SE_KAC_SIZE(1) +#define SE_KAC_SIZE_256 SE_KAC_SIZE(2) + +#define SE_KAC_EXPORTABLE FIELD_PREP(BIT(12), 1) + +#define SE_KAC_PURPOSE(x) FIELD_PREP(GENMASK(11, 8), x) +#define SE_KAC_ENC SE_KAC_PURPOSE(0) +#define SE_KAC_CMAC SE_KAC_PURPOSE(1) +#define SE_KAC_HMAC SE_KAC_PURPOSE(2) +#define SE_KAC_GCM_KW SE_KAC_PURPOSE(3) +#define SE_KAC_HMAC_KDK SE_KAC_PURPOSE(6) +#define SE_KAC_HMAC_KDD SE_KAC_PURPOSE(7) +#define SE_KAC_HMAC_KDD_KUW SE_KAC_PURPOSE(8) +#define SE_KAC_XTS SE_KAC_PURPOSE(9) +#define SE_KAC_GCM SE_KAC_PURPOSE(10) + +#define SE_KAC_USER_NS FIELD_PREP(GENMASK(6, 4), 3) + +#define SE_AES_KEY_DST_INDEX(x) FIELD_PREP(GENMASK(11, 8), x) +#define SE_ADDR_HI_MSB(x) FIELD_PREP(GENMASK(31, 24), x) +#define SE_ADDR_HI_SZ(x) FIELD_PREP(GENMASK(23, 0), x) + +#define SE_CFG_AES_ENCRYPT (SE_AES_ENC_ALG_AES_ENC | \ + SE_AES_DEC_ALG_NOP | \ + SE_AES_DST_MEMORY) + +#define SE_CFG_AES_DECRYPT (SE_AES_ENC_ALG_NOP | \ + SE_AES_DEC_ALG_AES_DEC | \ + SE_AES_DST_MEMORY) + +#define SE_CFG_GMAC_ENCRYPT (SE_AES_ENC_ALG_AES_ENC | \ + SE_AES_DEC_ALG_NOP | \ + SE_AES_ENC_MODE_GMAC | \ + SE_AES_DST_MEMORY) + +#define SE_CFG_GMAC_DECRYPT (SE_AES_ENC_ALG_NOP | \ + SE_AES_DEC_ALG_AES_DEC | \ + SE_AES_DEC_MODE_GMAC | \ + SE_AES_DST_MEMORY) + +#define SE_CFG_GCM_ENCRYPT (SE_AES_ENC_ALG_AES_ENC | \ + SE_AES_DEC_ALG_NOP | \ + SE_AES_ENC_MODE_GCM | \ + SE_AES_DST_MEMORY) + +#define SE_CFG_GCM_DECRYPT (SE_AES_ENC_ALG_NOP | \ + SE_AES_DEC_ALG_AES_DEC | \ + SE_AES_DEC_MODE_GCM | \ + SE_AES_DST_MEMORY) + +#define SE_CFG_GCM_FINAL_ENCRYPT (SE_AES_ENC_ALG_AES_ENC | \ + SE_AES_DEC_ALG_NOP | \ + SE_AES_ENC_MODE_GCM_FINAL | \ + SE_AES_DST_MEMORY) + +#define SE_CFG_GCM_FINAL_DECRYPT (SE_AES_ENC_ALG_NOP | \ + SE_AES_DEC_ALG_AES_DEC | \ + SE_AES_DEC_MODE_GCM_FINAL | \ + SE_AES_DST_MEMORY) + +#define SE_CFG_CMAC (SE_AES_ENC_ALG_AES_ENC | \ + SE_AES_ENC_MODE_CMAC | \ + SE_AES_DST_HASH_REG) + +#define SE_CFG_CBC_MAC (SE_AES_ENC_ALG_AES_ENC | \ + SE_AES_ENC_MODE_CBC_MAC) + +#define SE_CFG_INS (SE_AES_ENC_ALG_INS | \ + SE_AES_DEC_ALG_NOP) + +#define SE_CRYPTO_CFG_ECB_ENCRYPT (SE_AES_INPUT_SEL_MEMORY | \ + SE_AES_XOR_POS_BYPASS | \ + SE_AES_CORE_SEL_ENCRYPT) + +#define SE_CRYPTO_CFG_ECB_DECRYPT (SE_AES_INPUT_SEL_MEMORY | \ + SE_AES_XOR_POS_BYPASS | \ + SE_AES_CORE_SEL_DECRYPT) + +#define SE_CRYPTO_CFG_CBC_ENCRYPT (SE_AES_INPUT_SEL_MEMORY | \ + SE_AES_VCTRAM_SEL_AESOUT | \ + SE_AES_XOR_POS_TOP | \ + SE_AES_CORE_SEL_ENCRYPT | \ + SE_AES_IV_SEL_REG) + +#define SE_CRYPTO_CFG_CBC_DECRYPT (SE_AES_INPUT_SEL_MEMORY | \ + SE_AES_VCTRAM_SEL_PREV_MEM | \ + SE_AES_XOR_POS_BOTTOM | \ + SE_AES_CORE_SEL_DECRYPT | \ + SE_AES_IV_SEL_REG) + +#define SE_CRYPTO_CFG_CTR (SE_AES_INPUT_SEL_LINEAR_CTR | \ + SE_AES_VCTRAM_SEL_MEMORY | \ + SE_AES_XOR_POS_BOTTOM | \ + SE_AES_CORE_SEL_ENCRYPT | \ + SE_AES_CRYPTO_CFG_CTR_CNTN(1) | \ + SE_AES_IV_SEL_REG) + +#define SE_CRYPTO_CFG_XTS_ENCRYPT (SE_AES_INPUT_SEL_MEMORY | \ + SE_AES_VCTRAM_SEL_TWEAK | \ + SE_AES_XOR_POS_BOTH | \ + SE_AES_CORE_SEL_ENCRYPT | \ + SE_AES_IV_SEL_REG) + +#define SE_CRYPTO_CFG_XTS_DECRYPT (SE_AES_INPUT_SEL_MEMORY | \ + SE_AES_VCTRAM_SEL_TWEAK | \ + SE_AES_XOR_POS_BOTH | \ + SE_AES_CORE_SEL_DECRYPT | \ + SE_AES_IV_SEL_REG) + +#define SE_CRYPTO_CFG_XTS_DECRYPT (SE_AES_INPUT_SEL_MEMORY | \ + SE_AES_VCTRAM_SEL_TWEAK | \ + SE_AES_XOR_POS_BOTH | \ + SE_AES_CORE_SEL_DECRYPT | \ + SE_AES_IV_SEL_REG) + +#define SE_CRYPTO_CFG_CBC_MAC (SE_AES_INPUT_SEL_MEMORY | \ + SE_AES_VCTRAM_SEL_AESOUT | \ + SE_AES_XOR_POS_TOP | \ + SE_AES_CORE_SEL_ENCRYPT | \ + SE_AES_HASH_ENABLE | \ + SE_AES_IV_SEL_REG) + +#define HASH_RESULT_REG_COUNT 50 +#define CMAC_RESULT_REG_COUNT 4 + +#define SE_CRYPTO_CTR_REG_COUNT 4 +#define SE_MAX_KEYSLOT 15 +#define SE_MAX_MEM_ALLOC SZ_4M +#define SE_AES_BUFLEN 0x8000 +#define SE_SHA_BUFLEN 0x2000 + +#define SHA_FIRST BIT(0) +#define SHA_UPDATE BIT(1) +#define SHA_FINAL BIT(2) + +/* Security Engine operation modes */ +enum se_aes_alg { + SE_ALG_CBC, /* Cipher Block Chaining (CBC) mode */ + SE_ALG_ECB, /* Electronic Codebook (ECB) mode */ + SE_ALG_CTR, /* Counter (CTR) mode */ + SE_ALG_XTS, /* XTS mode */ + SE_ALG_GMAC, /* GMAC mode */ + SE_ALG_GCM, /* GCM mode */ + SE_ALG_GCM_FINAL, /* GCM FINAL mode */ + SE_ALG_CMAC, /* Cipher-based MAC (CMAC) mode */ + SE_ALG_CBC_MAC, /* CBC MAC mode */ +}; + +enum se_hash_alg { + SE_ALG_RNG_DRBG, /* Deterministic Random Bit Generator */ + SE_ALG_SHA1, /* Secure Hash Algorithm-1 (SHA1) mode */ + SE_ALG_SHA224, /* Secure Hash Algorithm-224 (SHA224) mode */ + SE_ALG_SHA256, /* Secure Hash Algorithm-256 (SHA256) mode */ + SE_ALG_SHA384, /* Secure Hash Algorithm-384 (SHA384) mode */ + SE_ALG_SHA512, /* Secure Hash Algorithm-512 (SHA512) mode */ + SE_ALG_SHA3_224, /* Secure Hash Algorithm3-224 (SHA3-224) mode */ + SE_ALG_SHA3_256, /* Secure Hash Algorithm3-256 (SHA3-256) mode */ + SE_ALG_SHA3_384, /* Secure Hash Algorithm3-384 (SHA3-384) mode */ + SE_ALG_SHA3_512, /* Secure Hash Algorithm3-512 (SHA3-512) mode */ + SE_ALG_SHAKE128, /* Secure Hash Algorithm3 (SHAKE128) mode */ + SE_ALG_SHAKE256, /* Secure Hash Algorithm3 (SHAKE256) mode */ + SE_ALG_HMAC_SHA224, /* Hash based MAC (HMAC) - 224 */ + SE_ALG_HMAC_SHA256, /* Hash based MAC (HMAC) - 256 */ + SE_ALG_HMAC_SHA384, /* Hash based MAC (HMAC) - 384 */ + SE_ALG_HMAC_SHA512, /* Hash based MAC (HMAC) - 512 */ +}; + +struct tegra_se_alg { + struct tegra_se *se_dev; + const char *alg_base; + + union { + struct skcipher_engine_alg skcipher; + struct aead_engine_alg aead; + struct ahash_engine_alg ahash; + } alg; +}; + +struct tegra_se_regs { + u32 op; + u32 config; + u32 last_blk; + u32 linear_ctr; + u32 out_addr; + u32 aad_len; + u32 cryp_msg_len; + u32 manifest; + u32 key_addr; + u32 key_data; + u32 key_dst; + u32 result; +}; + +struct tegra_se_hw { + const struct tegra_se_regs *regs; + int (*init_alg)(struct tegra_se *se); + void (*deinit_alg)(struct tegra_se *se); + bool support_sm_alg; + u32 host1x_class; + u32 kac_ver; +}; + +struct tegra_se { + int (*manifest)(u32 user, u32 alg, u32 keylen); + const struct tegra_se_hw *hw; + struct host1x_client client; + struct host1x_channel *channel; + struct tegra_se_cmdbuf *cmdbuf; + struct crypto_engine *engine; + struct host1x_syncpt *syncpt; + struct device *dev; + struct clk *clk; + unsigned int opcode_addr; + unsigned int stream_id; + unsigned int syncpt_id; + void __iomem *base; + u32 owner; +}; + +struct tegra_se_cmdbuf { + dma_addr_t iova; + u32 *addr; + struct device *dev; + struct kref ref; + struct host1x_bo bo; + ssize_t size; + u32 words; +}; + +struct tegra_se_datbuf { + u8 *buf; + dma_addr_t addr; + ssize_t size; +}; + +static inline int se_algname_to_algid(const char *name) +{ + if (!strcmp(name, "cbc(aes)")) + return SE_ALG_CBC; + else if (!strcmp(name, "ecb(aes)")) + return SE_ALG_ECB; + else if (!strcmp(name, "ctr(aes)")) + return SE_ALG_CTR; + else if (!strcmp(name, "xts(aes)")) + return SE_ALG_XTS; + else if (!strcmp(name, "cmac(aes)")) + return SE_ALG_CMAC; + else if (!strcmp(name, "gcm(aes)")) + return SE_ALG_GCM; + else if (!strcmp(name, "ccm(aes)")) + return SE_ALG_CBC_MAC; + + else if (!strcmp(name, "sha1")) + return SE_ALG_SHA1; + else if (!strcmp(name, "sha224")) + return SE_ALG_SHA224; + else if (!strcmp(name, "sha256")) + return SE_ALG_SHA256; + else if (!strcmp(name, "sha384")) + return SE_ALG_SHA384; + else if (!strcmp(name, "sha512")) + return SE_ALG_SHA512; + else if (!strcmp(name, "sha3-224")) + return SE_ALG_SHA3_224; + else if (!strcmp(name, "sha3-256")) + return SE_ALG_SHA3_256; + else if (!strcmp(name, "sha3-384")) + return SE_ALG_SHA3_384; + else if (!strcmp(name, "sha3-512")) + return SE_ALG_SHA3_512; + else if (!strcmp(name, "hmac(sha224)")) + return SE_ALG_HMAC_SHA224; + else if (!strcmp(name, "hmac(sha256)")) + return SE_ALG_HMAC_SHA256; + else if (!strcmp(name, "hmac(sha384)")) + return SE_ALG_HMAC_SHA384; + else if (!strcmp(name, "hmac(sha512)")) + return SE_ALG_HMAC_SHA512; + else + return -EINVAL; +} + +/* Functions */ +int tegra_init_aes(struct tegra_se *se); +int tegra_init_hash(struct tegra_se *se); +void tegra_deinit_aes(struct tegra_se *se); +void tegra_deinit_hash(struct tegra_se *se); +int tegra_key_submit(struct tegra_se *se, const u8 *key, + u32 keylen, u32 alg, u32 *keyid); +void tegra_key_invalidate(struct tegra_se *se, u32 keyid, u32 alg); +int tegra_se_host1x_submit(struct tegra_se *se, u32 size); + +/* HOST1x OPCODES */ +static inline u32 host1x_opcode_setpayload(unsigned int payload) +{ + return (9 << 28) | payload; +} + +static inline u32 host1x_opcode_incr_w(unsigned int offset) +{ + /* 22-bit offset supported */ + return (10 << 28) | offset; +} + +static inline u32 host1x_opcode_nonincr_w(unsigned int offset) +{ + /* 22-bit offset supported */ + return (11 << 28) | offset; +} + +static inline u32 host1x_opcode_incr(unsigned int offset, unsigned int count) +{ + return (1 << 28) | (offset << 16) | count; +} + +static inline u32 host1x_opcode_nonincr(unsigned int offset, unsigned int count) +{ + return (2 << 28) | (offset << 16) | count; +} + +static inline u32 host1x_uclass_incr_syncpt_cond_f(u32 v) +{ + return (v & 0xff) << 10; +} + +static inline u32 host1x_uclass_incr_syncpt_indx_f(u32 v) +{ + return (v & 0x3ff) << 0; +} + +static inline u32 host1x_uclass_wait_syncpt_r(void) +{ + return 0x8; +} + +static inline u32 host1x_uclass_incr_syncpt_r(void) +{ + return 0x0; +} + +#define se_host1x_opcode_incr_w(x) host1x_opcode_incr_w((x) / 4) +#define se_host1x_opcode_nonincr_w(x) host1x_opcode_nonincr_w((x) / 4) +#define se_host1x_opcode_incr(x, y) host1x_opcode_incr((x) / 4, y) +#define se_host1x_opcode_nonincr(x, y) host1x_opcode_nonincr((x) / 4, y) + +#endif /*_TEGRA_SE_H*/ diff --git a/drivers/dax/device.c b/drivers/dax/device.c index 93ebedc5ec8c..c24ef4d3cf31 100644 --- a/drivers/dax/device.c +++ b/drivers/dax/device.c @@ -377,7 +377,7 @@ static const struct file_operations dax_fops = { .release = dax_release, .get_unmapped_area = dax_get_unmapped_area, .mmap = dax_mmap, - .mmap_supported_flags = MAP_SYNC, + .fop_flags = FOP_MMAP_SYNC, }; static void dev_dax_cdev_del(void *cdev) diff --git a/drivers/dma/idxd/cdev.c b/drivers/dma/idxd/cdev.c index c095a2c8f659..39935071174a 100644 --- a/drivers/dma/idxd/cdev.c +++ b/drivers/dma/idxd/cdev.c @@ -400,6 +400,18 @@ static int idxd_cdev_mmap(struct file *filp, struct vm_area_struct *vma) int rc; dev_dbg(&pdev->dev, "%s called\n", __func__); + + /* + * Due to an erratum in some of the devices supported by the driver, + * direct user submission to the device can be unsafe. + * (See the INTEL-SA-01084 security advisory) + * + * For the devices that exhibit this behavior, require that the user + * has CAP_SYS_RAWIO capabilities. + */ + if (!idxd->user_submission_safe && !capable(CAP_SYS_RAWIO)) + return -EPERM; + rc = check_vma(wq, vma, __func__); if (rc < 0) return rc; @@ -414,6 +426,70 @@ static int idxd_cdev_mmap(struct file *filp, struct vm_area_struct *vma) vma->vm_page_prot); } +static int idxd_submit_user_descriptor(struct idxd_user_context *ctx, + struct dsa_hw_desc __user *udesc) +{ + struct idxd_wq *wq = ctx->wq; + struct idxd_dev *idxd_dev = &wq->idxd->idxd_dev; + const uint64_t comp_addr_align = is_dsa_dev(idxd_dev) ? 0x20 : 0x40; + void __iomem *portal = idxd_wq_portal_addr(wq); + struct dsa_hw_desc descriptor __aligned(64); + int rc; + + rc = copy_from_user(&descriptor, udesc, sizeof(descriptor)); + if (rc) + return -EFAULT; + + /* + * DSA devices are capable of indirect ("batch") command submission. + * On devices where direct user submissions are not safe, we cannot + * allow this since there is no good way for us to verify these + * indirect commands. + */ + if (is_dsa_dev(idxd_dev) && descriptor.opcode == DSA_OPCODE_BATCH && + !wq->idxd->user_submission_safe) + return -EINVAL; + /* + * As per the programming specification, the completion address must be + * aligned to 32 or 64 bytes. If this is violated the hardware + * engine can get very confused (security issue). + */ + if (!IS_ALIGNED(descriptor.completion_addr, comp_addr_align)) + return -EINVAL; + + if (wq_dedicated(wq)) + iosubmit_cmds512(portal, &descriptor, 1); + else { + descriptor.priv = 0; + descriptor.pasid = ctx->pasid; + rc = idxd_enqcmds(wq, portal, &descriptor); + if (rc < 0) + return rc; + } + + return 0; +} + +static ssize_t idxd_cdev_write(struct file *filp, const char __user *buf, size_t len, + loff_t *unused) +{ + struct dsa_hw_desc __user *udesc = (struct dsa_hw_desc __user *)buf; + struct idxd_user_context *ctx = filp->private_data; + ssize_t written = 0; + int i; + + for (i = 0; i < len/sizeof(struct dsa_hw_desc); i++) { + int rc = idxd_submit_user_descriptor(ctx, udesc + i); + + if (rc) + return written ? written : rc; + + written += sizeof(struct dsa_hw_desc); + } + + return written; +} + static __poll_t idxd_cdev_poll(struct file *filp, struct poll_table_struct *wait) { @@ -436,6 +512,7 @@ static const struct file_operations idxd_cdev_fops = { .open = idxd_cdev_open, .release = idxd_cdev_release, .mmap = idxd_cdev_mmap, + .write = idxd_cdev_write, .poll = idxd_cdev_poll, }; diff --git a/drivers/dma/idxd/idxd.h b/drivers/dma/idxd/idxd.h index 7b98944135eb..868b724a3b75 100644 --- a/drivers/dma/idxd/idxd.h +++ b/drivers/dma/idxd/idxd.h @@ -288,6 +288,7 @@ struct idxd_driver_data { int evl_cr_off; int cr_status_off; int cr_result_off; + bool user_submission_safe; load_device_defaults_fn_t load_device_defaults; }; @@ -374,6 +375,8 @@ struct idxd_device { struct dentry *dbgfs_dir; struct dentry *dbgfs_evl_file; + + bool user_submission_safe; }; static inline unsigned int evl_ent_size(struct idxd_device *idxd) diff --git a/drivers/dma/idxd/init.c b/drivers/dma/idxd/init.c index 264c4e47d7cc..a7295943fa22 100644 --- a/drivers/dma/idxd/init.c +++ b/drivers/dma/idxd/init.c @@ -47,6 +47,7 @@ static struct idxd_driver_data idxd_driver_data[] = { .align = 32, .dev_type = &dsa_device_type, .evl_cr_off = offsetof(struct dsa_evl_entry, cr), + .user_submission_safe = false, /* See INTEL-SA-01084 security advisory */ .cr_status_off = offsetof(struct dsa_completion_record, status), .cr_result_off = offsetof(struct dsa_completion_record, result), }, @@ -57,6 +58,7 @@ static struct idxd_driver_data idxd_driver_data[] = { .align = 64, .dev_type = &iax_device_type, .evl_cr_off = offsetof(struct iax_evl_entry, cr), + .user_submission_safe = false, /* See INTEL-SA-01084 security advisory */ .cr_status_off = offsetof(struct iax_completion_record, status), .cr_result_off = offsetof(struct iax_completion_record, error_code), .load_device_defaults = idxd_load_iaa_device_defaults, @@ -774,6 +776,8 @@ static int idxd_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) dev_info(&pdev->dev, "Intel(R) Accelerator Device (v%x)\n", idxd->hw.version); + idxd->user_submission_safe = data->user_submission_safe; + return 0; err_dev_register: diff --git a/drivers/dma/idxd/registers.h b/drivers/dma/idxd/registers.h index 315c004f58e4..e16dbf9ab324 100644 --- a/drivers/dma/idxd/registers.h +++ b/drivers/dma/idxd/registers.h @@ -6,9 +6,6 @@ #include <uapi/linux/idxd.h> /* PCI Config */ -#define PCI_DEVICE_ID_INTEL_DSA_SPR0 0x0b25 -#define PCI_DEVICE_ID_INTEL_IAX_SPR0 0x0cfe - #define DEVICE_VERSION_1 0x100 #define DEVICE_VERSION_2 0x200 diff --git a/drivers/dma/idxd/sysfs.c b/drivers/dma/idxd/sysfs.c index 7f28f01be672..f706eae0e76b 100644 --- a/drivers/dma/idxd/sysfs.c +++ b/drivers/dma/idxd/sysfs.c @@ -1197,12 +1197,35 @@ static ssize_t wq_enqcmds_retries_store(struct device *dev, struct device_attrib static struct device_attribute dev_attr_wq_enqcmds_retries = __ATTR(enqcmds_retries, 0644, wq_enqcmds_retries_show, wq_enqcmds_retries_store); +static ssize_t op_cap_show_common(struct device *dev, char *buf, unsigned long *opcap_bmap) +{ + ssize_t pos; + int i; + + pos = 0; + for (i = IDXD_MAX_OPCAP_BITS/64 - 1; i >= 0; i--) { + unsigned long val = opcap_bmap[i]; + + /* On systems where direct user submissions are not safe, we need to clear out + * the BATCH capability from the capability mask in sysfs since we cannot support + * that command on such systems. + */ + if (i == DSA_OPCODE_BATCH/64 && !confdev_to_idxd(dev)->user_submission_safe) + clear_bit(DSA_OPCODE_BATCH % 64, &val); + + pos += sysfs_emit_at(buf, pos, "%*pb", 64, &val); + pos += sysfs_emit_at(buf, pos, "%c", i == 0 ? '\n' : ','); + } + + return pos; +} + static ssize_t wq_op_config_show(struct device *dev, struct device_attribute *attr, char *buf) { struct idxd_wq *wq = confdev_to_wq(dev); - return sysfs_emit(buf, "%*pb\n", IDXD_MAX_OPCAP_BITS, wq->opcap_bmap); + return op_cap_show_common(dev, buf, wq->opcap_bmap); } static int idxd_verify_supported_opcap(struct idxd_device *idxd, unsigned long *opmask) @@ -1455,7 +1478,7 @@ static ssize_t op_cap_show(struct device *dev, { struct idxd_device *idxd = confdev_to_idxd(dev); - return sysfs_emit(buf, "%*pb\n", IDXD_MAX_OPCAP_BITS, idxd->opcap_bmap); + return op_cap_show_common(dev, buf, idxd->opcap_bmap); } static DEVICE_ATTR_RO(op_cap); diff --git a/drivers/edac/altera_edac.c b/drivers/edac/altera_edac.c index ae17ce4d9722..fe89f5c4837f 100644 --- a/drivers/edac/altera_edac.c +++ b/drivers/edac/altera_edac.c @@ -737,8 +737,7 @@ static int altr_edac_device_probe(struct platform_device *pdev) } dci = edac_device_alloc_ctl_info(sizeof(*drvdata), ecc_name, - 1, ecc_name, 1, 0, NULL, 0, - dev_instance++); + 1, ecc_name, 1, 0, dev_instance++); if (!dci) { edac_printk(KERN_ERR, EDAC_DEVICE, @@ -1514,7 +1513,7 @@ static int altr_portb_setup(struct altr_edac_device_dev *device) /* Create the PortB EDAC device */ edac_idx = edac_device_alloc_index(); dci = edac_device_alloc_ctl_info(sizeof(*altdev), ecc_name, 1, - ecc_name, 1, 0, NULL, 0, edac_idx); + ecc_name, 1, 0, edac_idx); if (!dci) { edac_printk(KERN_ERR, EDAC_DEVICE, "%s: Unable to allocate PortB EDAC device\n", @@ -1921,8 +1920,7 @@ static int altr_edac_a10_device_add(struct altr_arria10_edac *edac, edac_idx = edac_device_alloc_index(); dci = edac_device_alloc_ctl_info(sizeof(*altdev), ecc_name, - 1, ecc_name, 1, 0, NULL, 0, - edac_idx); + 1, ecc_name, 1, 0, edac_idx); if (!dci) { edac_printk(KERN_ERR, EDAC_DEVICE, diff --git a/drivers/edac/amd64_edac.h b/drivers/edac/amd64_edac.h index 1665f7932bac..b879b12971e7 100644 --- a/drivers/edac/amd64_edac.h +++ b/drivers/edac/amd64_edac.h @@ -345,7 +345,6 @@ struct amd64_pvt { u32 dchr1; /* DRAM Configuration High DCT1 reg */ u32 nbcap; /* North Bridge Capabilities */ u32 nbcfg; /* F10 North Bridge Configuration */ - u32 ext_nbcfg; /* Extended F10 North Bridge Configuration */ u32 dhar; /* DRAM Hoist reg */ u32 dbam0; /* DRAM Base Address Mapping reg for DCT0 */ u32 dbam1; /* DRAM Base Address Mapping reg for DCT1 */ diff --git a/drivers/edac/amd8111_edac.c b/drivers/edac/amd8111_edac.c index ca718f63fcbc..a6d3013d5823 100644 --- a/drivers/edac/amd8111_edac.c +++ b/drivers/edac/amd8111_edac.c @@ -366,8 +366,7 @@ static int amd8111_dev_probe(struct pci_dev *dev, dev_info->edac_idx = edac_device_alloc_index(); dev_info->edac_dev = edac_device_alloc_ctl_info(0, dev_info->ctl_name, 1, - NULL, 0, 0, - NULL, 0, dev_info->edac_idx); + NULL, 0, 0, dev_info->edac_idx); if (!dev_info->edac_dev) { ret = -ENOMEM; goto err_dev_put; diff --git a/drivers/edac/armada_xp_edac.c b/drivers/edac/armada_xp_edac.c index 25517c99b3ea..589bc81f1249 100644 --- a/drivers/edac/armada_xp_edac.c +++ b/drivers/edac/armada_xp_edac.c @@ -523,7 +523,7 @@ static int aurora_l2_probe(struct platform_device *pdev) dev_warn(&pdev->dev, "data ECC is not enabled\n"); dci = edac_device_alloc_ctl_info(sizeof(*drvdata), - "cpu", 1, "L", 1, 2, NULL, 0, 0); + "cpu", 1, "L", 1, 2, 0); if (!dci) return -ENOMEM; diff --git a/drivers/edac/cpc925_edac.c b/drivers/edac/cpc925_edac.c index 5075dc7526e3..eb702bc3aa29 100644 --- a/drivers/edac/cpc925_edac.c +++ b/drivers/edac/cpc925_edac.c @@ -797,7 +797,7 @@ static void cpc925_add_edac_devices(void __iomem *vbase) dev_info->edac_idx = edac_device_alloc_index(); dev_info->edac_dev = edac_device_alloc_ctl_info(0, dev_info->ctl_name, - 1, NULL, 0, 0, NULL, 0, dev_info->edac_idx); + 1, NULL, 0, 0, dev_info->edac_idx); if (!dev_info->edac_dev) { cpc925_printk(KERN_ERR, "No memory for edac device\n"); goto err1; diff --git a/drivers/edac/edac_device.c b/drivers/edac/edac_device.c index 0689e1510721..621dc2a5d034 100644 --- a/drivers/edac/edac_device.c +++ b/drivers/edac/edac_device.c @@ -56,14 +56,12 @@ static void edac_device_dump_device(struct edac_device_ctl_info *edac_dev) struct edac_device_ctl_info * edac_device_alloc_ctl_info(unsigned pvt_sz, char *dev_name, unsigned nr_instances, char *blk_name, unsigned nr_blocks, unsigned off_val, - struct edac_dev_sysfs_block_attribute *attrib_spec, - unsigned nr_attrib, int device_index) + int device_index) { - struct edac_dev_sysfs_block_attribute *dev_attrib, *attrib_p, *attrib; struct edac_device_block *dev_blk, *blk_p, *blk; struct edac_device_instance *dev_inst, *inst; struct edac_device_ctl_info *dev_ctl; - unsigned instance, block, attr; + unsigned instance, block; void *pvt; int err; @@ -85,15 +83,6 @@ edac_device_alloc_ctl_info(unsigned pvt_sz, char *dev_name, unsigned nr_instance dev_ctl->blocks = dev_blk; - if (nr_attrib) { - dev_attrib = kcalloc(nr_attrib, sizeof(struct edac_dev_sysfs_block_attribute), - GFP_KERNEL); - if (!dev_attrib) - goto free; - - dev_ctl->attribs = dev_attrib; - } - if (pvt_sz) { pvt = kzalloc(pvt_sz, GFP_KERNEL); if (!pvt) @@ -132,44 +121,6 @@ edac_device_alloc_ctl_info(unsigned pvt_sz, char *dev_name, unsigned nr_instance edac_dbg(4, "instance=%d inst_p=%p block=#%d block_p=%p name='%s'\n", instance, inst, block, blk, blk->name); - - /* if there are NO attributes OR no attribute pointer - * then continue on to next block iteration - */ - if ((nr_attrib == 0) || (attrib_spec == NULL)) - continue; - - /* setup the attribute array for this block */ - blk->nr_attribs = nr_attrib; - attrib_p = &dev_attrib[block*nr_instances*nr_attrib]; - blk->block_attributes = attrib_p; - - edac_dbg(4, "THIS BLOCK_ATTRIB=%p\n", - blk->block_attributes); - - /* Initialize every user specified attribute in this - * block with the data the caller passed in - * Each block gets its own copy of pointers, - * and its unique 'value' - */ - for (attr = 0; attr < nr_attrib; attr++) { - attrib = &attrib_p[attr]; - - /* populate the unique per attrib - * with the code pointers and info - */ - attrib->attr = attrib_spec[attr].attr; - attrib->show = attrib_spec[attr].show; - attrib->store = attrib_spec[attr].store; - - attrib->block = blk; /* up link */ - - edac_dbg(4, "alloc-attrib=%p attrib_name='%s' attrib-spec=%p spec-name=%s\n", - attrib, attrib->attr.name, - &attrib_spec[attr], - attrib_spec[attr].attr.name - ); - } } } diff --git a/drivers/edac/edac_device.h b/drivers/edac/edac_device.h index 7db22a4c83ef..034711d71ebf 100644 --- a/drivers/edac/edac_device.h +++ b/drivers/edac/edac_device.h @@ -22,7 +22,6 @@ #ifndef _EDAC_DEVICE_H_ #define _EDAC_DEVICE_H_ -#include <linux/completion.h> #include <linux/device.h> #include <linux/edac.h> #include <linux/kobject.h> @@ -95,22 +94,13 @@ struct edac_dev_sysfs_attribute { * * used in leaf 'block' nodes for adding controls/attributes * - * each block in each instance of the containing control structure - * can have an array of the following. The show and store functions - * will be filled in with the show/store function in the - * low level driver. - * - * The 'value' field will be the actual value field used for - * counting + * each block in each instance of the containing control structure can + * have an array of the following. The show function will be filled in + * with the show function in the low level driver. */ struct edac_dev_sysfs_block_attribute { struct attribute attr; ssize_t (*show)(struct kobject *, struct attribute *, char *); - ssize_t (*store)(struct kobject *, struct attribute *, - const char *, size_t); - struct edac_device_block *block; - - unsigned int value; }; /* device block control structure */ @@ -200,8 +190,6 @@ struct edac_device_ctl_info { unsigned long start_time; /* edac_device load start time (jiffies) */ - struct completion removal_complete; - /* sysfs top name under 'edac' directory * and instance name: * cpu/cpu0/... @@ -217,7 +205,6 @@ struct edac_device_ctl_info { u32 nr_instances; struct edac_device_instance *instances; struct edac_device_block *blocks; - struct edac_dev_sysfs_block_attribute *attribs; /* Event counters for the this whole EDAC Device */ struct edac_device_counter counters; @@ -245,8 +232,6 @@ extern struct edac_device_ctl_info *edac_device_alloc_ctl_info( char *edac_device_name, unsigned nr_instances, char *edac_block_name, unsigned nr_blocks, unsigned offset_value, - struct edac_dev_sysfs_block_attribute *block_attributes, - unsigned nr_attribs, int device_index); /* The offset value can be: @@ -356,7 +341,6 @@ static inline void __edac_device_free_ctl_info(struct edac_device_ctl_info *ci) { if (ci) { kfree(ci->pvt_info); - kfree(ci->attribs); kfree(ci->blocks); kfree(ci->instances); kfree(ci); diff --git a/drivers/edac/edac_device_sysfs.c b/drivers/edac/edac_device_sysfs.c index 237a542e045a..fcebc4ffea26 100644 --- a/drivers/edac/edac_device_sysfs.c +++ b/drivers/edac/edac_device_sysfs.c @@ -457,35 +457,19 @@ static ssize_t edac_dev_block_show(struct kobject *kobj, return -EIO; } -/* Function to 'store' fields into the edac_dev 'block' structure */ -static ssize_t edac_dev_block_store(struct kobject *kobj, - struct attribute *attr, - const char *buffer, size_t count) -{ - struct edac_dev_sysfs_block_attribute *block_attr; - - block_attr = to_block_attr(attr); - - if (block_attr->store) - return block_attr->store(kobj, attr, buffer, count); - return -EIO; -} - /* edac_dev file operations for a 'block' */ static const struct sysfs_ops device_block_ops = { .show = edac_dev_block_show, - .store = edac_dev_block_store }; -#define BLOCK_ATTR(_name,_mode,_show,_store) \ +#define BLOCK_ATTR(_name,_mode,_show) \ static struct edac_dev_sysfs_block_attribute attr_block_##_name = { \ .attr = {.name = __stringify(_name), .mode = _mode }, \ .show = _show, \ - .store = _store, \ }; -BLOCK_ATTR(ce_count, S_IRUGO, block_ce_count_show, NULL); -BLOCK_ATTR(ue_count, S_IRUGO, block_ue_count_show, NULL); +BLOCK_ATTR(ce_count, S_IRUGO, block_ce_count_show); +BLOCK_ATTR(ue_count, S_IRUGO, block_ue_count_show); /* list of edac_dev 'block' attributes */ static struct attribute *device_block_attrs[] = { diff --git a/drivers/edac/edac_mc_sysfs.c b/drivers/edac/edac_mc_sysfs.c index 5116873c3330..4200aec04831 100644 --- a/drivers/edac/edac_mc_sysfs.c +++ b/drivers/edac/edac_mc_sysfs.c @@ -146,7 +146,7 @@ static ssize_t csrow_ue_count_show(struct device *dev, { struct csrow_info *csrow = to_csrow(dev); - return sprintf(data, "%u\n", csrow->ue_count); + return sysfs_emit(data, "%u\n", csrow->ue_count); } static ssize_t csrow_ce_count_show(struct device *dev, @@ -154,7 +154,7 @@ static ssize_t csrow_ce_count_show(struct device *dev, { struct csrow_info *csrow = to_csrow(dev); - return sprintf(data, "%u\n", csrow->ce_count); + return sysfs_emit(data, "%u\n", csrow->ce_count); } static ssize_t csrow_size_show(struct device *dev, @@ -166,7 +166,7 @@ static ssize_t csrow_size_show(struct device *dev, for (i = 0; i < csrow->nr_channels; i++) nr_pages += csrow->channels[i]->dimm->nr_pages; - return sprintf(data, "%u\n", PAGES_TO_MiB(nr_pages)); + return sysfs_emit(data, "%u\n", PAGES_TO_MiB(nr_pages)); } static ssize_t csrow_mem_type_show(struct device *dev, @@ -174,7 +174,7 @@ static ssize_t csrow_mem_type_show(struct device *dev, { struct csrow_info *csrow = to_csrow(dev); - return sprintf(data, "%s\n", edac_mem_types[csrow->channels[0]->dimm->mtype]); + return sysfs_emit(data, "%s\n", edac_mem_types[csrow->channels[0]->dimm->mtype]); } static ssize_t csrow_dev_type_show(struct device *dev, @@ -182,7 +182,7 @@ static ssize_t csrow_dev_type_show(struct device *dev, { struct csrow_info *csrow = to_csrow(dev); - return sprintf(data, "%s\n", dev_types[csrow->channels[0]->dimm->dtype]); + return sysfs_emit(data, "%s\n", dev_types[csrow->channels[0]->dimm->dtype]); } static ssize_t csrow_edac_mode_show(struct device *dev, @@ -191,7 +191,7 @@ static ssize_t csrow_edac_mode_show(struct device *dev, { struct csrow_info *csrow = to_csrow(dev); - return sprintf(data, "%s\n", edac_caps[csrow->channels[0]->dimm->edac_mode]); + return sysfs_emit(data, "%s\n", edac_caps[csrow->channels[0]->dimm->edac_mode]); } /* show/store functions for DIMM Label attributes */ @@ -207,8 +207,7 @@ static ssize_t channel_dimm_label_show(struct device *dev, if (!rank->dimm->label[0]) return 0; - return snprintf(data, sizeof(rank->dimm->label) + 1, "%s\n", - rank->dimm->label); + return sysfs_emit(data, "%s\n", rank->dimm->label); } static ssize_t channel_dimm_label_store(struct device *dev, @@ -243,7 +242,7 @@ static ssize_t channel_ce_count_show(struct device *dev, unsigned int chan = to_channel(mattr); struct rank_info *rank = csrow->channels[chan]; - return sprintf(data, "%u\n", rank->ce_count); + return sysfs_emit(data, "%u\n", rank->ce_count); } /* cwrow<id>/attribute files */ @@ -515,7 +514,7 @@ static ssize_t dimmdev_label_show(struct device *dev, if (!dimm->label[0]) return 0; - return snprintf(data, sizeof(dimm->label) + 1, "%s\n", dimm->label); + return sysfs_emit(data, "%s\n", dimm->label); } static ssize_t dimmdev_label_store(struct device *dev, @@ -546,7 +545,7 @@ static ssize_t dimmdev_size_show(struct device *dev, { struct dimm_info *dimm = to_dimm(dev); - return sprintf(data, "%u\n", PAGES_TO_MiB(dimm->nr_pages)); + return sysfs_emit(data, "%u\n", PAGES_TO_MiB(dimm->nr_pages)); } static ssize_t dimmdev_mem_type_show(struct device *dev, @@ -554,7 +553,7 @@ static ssize_t dimmdev_mem_type_show(struct device *dev, { struct dimm_info *dimm = to_dimm(dev); - return sprintf(data, "%s\n", edac_mem_types[dimm->mtype]); + return sysfs_emit(data, "%s\n", edac_mem_types[dimm->mtype]); } static ssize_t dimmdev_dev_type_show(struct device *dev, @@ -562,7 +561,7 @@ static ssize_t dimmdev_dev_type_show(struct device *dev, { struct dimm_info *dimm = to_dimm(dev); - return sprintf(data, "%s\n", dev_types[dimm->dtype]); + return sysfs_emit(data, "%s\n", dev_types[dimm->dtype]); } static ssize_t dimmdev_edac_mode_show(struct device *dev, @@ -571,7 +570,7 @@ static ssize_t dimmdev_edac_mode_show(struct device *dev, { struct dimm_info *dimm = to_dimm(dev); - return sprintf(data, "%s\n", edac_caps[dimm->edac_mode]); + return sysfs_emit(data, "%s\n", edac_caps[dimm->edac_mode]); } static ssize_t dimmdev_ce_count_show(struct device *dev, @@ -580,7 +579,7 @@ static ssize_t dimmdev_ce_count_show(struct device *dev, { struct dimm_info *dimm = to_dimm(dev); - return sprintf(data, "%u\n", dimm->ce_count); + return sysfs_emit(data, "%u\n", dimm->ce_count); } static ssize_t dimmdev_ue_count_show(struct device *dev, @@ -589,7 +588,7 @@ static ssize_t dimmdev_ue_count_show(struct device *dev, { struct dimm_info *dimm = to_dimm(dev); - return sprintf(data, "%u\n", dimm->ue_count); + return sysfs_emit(data, "%u\n", dimm->ue_count); } /* dimm/rank attribute files */ @@ -758,7 +757,7 @@ static ssize_t mci_sdram_scrub_rate_show(struct device *dev, return bandwidth; } - return sprintf(data, "%d\n", bandwidth); + return sysfs_emit(data, "%d\n", bandwidth); } /* default attribute files for the MCI object */ @@ -768,7 +767,7 @@ static ssize_t mci_ue_count_show(struct device *dev, { struct mem_ctl_info *mci = to_mci(dev); - return sprintf(data, "%u\n", mci->ue_mc); + return sysfs_emit(data, "%u\n", mci->ue_mc); } static ssize_t mci_ce_count_show(struct device *dev, @@ -777,7 +776,7 @@ static ssize_t mci_ce_count_show(struct device *dev, { struct mem_ctl_info *mci = to_mci(dev); - return sprintf(data, "%u\n", mci->ce_mc); + return sysfs_emit(data, "%u\n", mci->ce_mc); } static ssize_t mci_ce_noinfo_show(struct device *dev, @@ -786,7 +785,7 @@ static ssize_t mci_ce_noinfo_show(struct device *dev, { struct mem_ctl_info *mci = to_mci(dev); - return sprintf(data, "%u\n", mci->ce_noinfo_count); + return sysfs_emit(data, "%u\n", mci->ce_noinfo_count); } static ssize_t mci_ue_noinfo_show(struct device *dev, @@ -795,7 +794,7 @@ static ssize_t mci_ue_noinfo_show(struct device *dev, { struct mem_ctl_info *mci = to_mci(dev); - return sprintf(data, "%u\n", mci->ue_noinfo_count); + return sysfs_emit(data, "%u\n", mci->ue_noinfo_count); } static ssize_t mci_seconds_show(struct device *dev, @@ -804,7 +803,7 @@ static ssize_t mci_seconds_show(struct device *dev, { struct mem_ctl_info *mci = to_mci(dev); - return sprintf(data, "%ld\n", (jiffies - mci->start_time) / HZ); + return sysfs_emit(data, "%ld\n", (jiffies - mci->start_time) / HZ); } static ssize_t mci_ctl_name_show(struct device *dev, @@ -813,7 +812,7 @@ static ssize_t mci_ctl_name_show(struct device *dev, { struct mem_ctl_info *mci = to_mci(dev); - return sprintf(data, "%s\n", mci->ctl_name); + return sysfs_emit(data, "%s\n", mci->ctl_name); } static ssize_t mci_size_mb_show(struct device *dev, @@ -833,7 +832,7 @@ static ssize_t mci_size_mb_show(struct device *dev, } } - return sprintf(data, "%u\n", PAGES_TO_MiB(total_pages)); + return sysfs_emit(data, "%u\n", PAGES_TO_MiB(total_pages)); } static ssize_t mci_max_location_show(struct device *dev, diff --git a/drivers/edac/edac_pci.h b/drivers/edac/edac_pci.h index 5175f5724cfa..3f47cd9b2b03 100644 --- a/drivers/edac/edac_pci.h +++ b/drivers/edac/edac_pci.h @@ -22,7 +22,6 @@ #ifndef _EDAC_PCI_H_ #define _EDAC_PCI_H_ -#include <linux/completion.h> #include <linux/device.h> #include <linux/edac.h> #include <linux/kobject.h> @@ -48,8 +47,6 @@ struct edac_pci_ctl_info { int pci_idx; - struct bus_type *edac_subsys; /* pointer to subsystem */ - /* the internal state of this controller instance */ int op_state; /* work struct for this instance */ @@ -72,8 +69,6 @@ struct edac_pci_ctl_info { unsigned long start_time; /* edac_pci load start time (jiffies) */ - struct completion complete; - /* sysfs top name under 'edac' directory * and instance name: * cpu/cpu0/... diff --git a/drivers/edac/highbank_l2_edac.c b/drivers/edac/highbank_l2_edac.c index 5646c049a934..282ca6535f8f 100644 --- a/drivers/edac/highbank_l2_edac.c +++ b/drivers/edac/highbank_l2_edac.c @@ -54,7 +54,7 @@ static int highbank_l2_err_probe(struct platform_device *pdev) int res = 0; dci = edac_device_alloc_ctl_info(sizeof(*drvdata), "cpu", - 1, "L", 1, 2, NULL, 0, 0); + 1, "L", 1, 2, 0); if (!dci) return -ENOMEM; diff --git a/drivers/edac/mpc85xx_edac.c b/drivers/edac/mpc85xx_edac.c index c1bc53f4e184..e8945d4adbad 100644 --- a/drivers/edac/mpc85xx_edac.c +++ b/drivers/edac/mpc85xx_edac.c @@ -496,7 +496,7 @@ static int mpc85xx_l2_err_probe(struct platform_device *op) return -ENOMEM; edac_dev = edac_device_alloc_ctl_info(sizeof(*pdata), - "cpu", 1, "L", 1, 2, NULL, 0, + "cpu", 1, "L", 1, 2, edac_dev_idx); if (!edac_dev) { devres_release_group(&op->dev, mpc85xx_l2_err_probe); diff --git a/drivers/edac/octeon_edac-l2c.c b/drivers/edac/octeon_edac-l2c.c index 4015eb9af6fe..919095d10528 100644 --- a/drivers/edac/octeon_edac-l2c.c +++ b/drivers/edac/octeon_edac-l2c.c @@ -138,7 +138,7 @@ static int octeon_l2c_probe(struct platform_device *pdev) /* 'Tags' are block 0, 'Data' is block 1*/ l2c = edac_device_alloc_ctl_info(0, "l2c", num_tads, "l2c", 2, 0, - NULL, 0, edac_device_alloc_index()); + edac_device_alloc_index()); if (!l2c) return -ENOMEM; diff --git a/drivers/edac/octeon_edac-pc.c b/drivers/edac/octeon_edac-pc.c index ea8a8e337b1e..b8404cc7b65f 100644 --- a/drivers/edac/octeon_edac-pc.c +++ b/drivers/edac/octeon_edac-pc.c @@ -92,7 +92,7 @@ static int co_cache_error_probe(struct platform_device *pdev) platform_set_drvdata(pdev, p); p->ed = edac_device_alloc_ctl_info(0, "cpu", num_possible_cpus(), - "cache", 2, 0, NULL, 0, + "cache", 2, 0, edac_device_alloc_index()); if (!p->ed) goto err; diff --git a/drivers/edac/qcom_edac.c b/drivers/edac/qcom_edac.c index 5539917c01dd..d3cd4cc54ace 100644 --- a/drivers/edac/qcom_edac.c +++ b/drivers/edac/qcom_edac.c @@ -349,7 +349,6 @@ static int qcom_llcc_edac_probe(struct platform_device *pdev) /* Allocate edac control info */ edev_ctl = edac_device_alloc_ctl_info(0, "qcom-llcc", 1, "bank", llcc_driv_data->num_banks, 1, - NULL, 0, edac_device_alloc_index()); if (!edev_ctl) diff --git a/drivers/edac/sifive_edac.c b/drivers/edac/sifive_edac.c index b844e2626fd5..a2b193dc6604 100644 --- a/drivers/edac/sifive_edac.c +++ b/drivers/edac/sifive_edac.c @@ -52,8 +52,7 @@ static int ecc_register(struct platform_device *pdev) platform_set_drvdata(pdev, p); p->dci = edac_device_alloc_ctl_info(0, "sifive_ecc", 1, "sifive_ecc", - 1, 1, NULL, 0, - edac_device_alloc_index()); + 1, 1, edac_device_alloc_index()); if (!p->dci) return -ENOMEM; diff --git a/drivers/edac/skx_common.c b/drivers/edac/skx_common.c index 9c5b6f8bd8bd..27996b7924c8 100644 --- a/drivers/edac/skx_common.c +++ b/drivers/edac/skx_common.c @@ -648,7 +648,7 @@ int skx_mce_check_error(struct notifier_block *nb, unsigned long val, memset(&res, 0, sizeof(res)); res.mce = mce; res.addr = mce->addr & MCI_ADDR_PHYSADDR; - if (!pfn_to_online_page(res.addr >> PAGE_SHIFT)) { + if (!pfn_to_online_page(res.addr >> PAGE_SHIFT) && !arch_is_platform_page(res.addr)) { pr_err("Invalid address 0x%llx in IA32_MC%d_ADDR\n", mce->addr, mce->bank); return NOTIFY_DONE; } diff --git a/drivers/edac/synopsys_edac.c b/drivers/edac/synopsys_edac.c index 5527055b0964..ea7a9a342dd3 100644 --- a/drivers/edac/synopsys_edac.c +++ b/drivers/edac/synopsys_edac.c @@ -9,6 +9,7 @@ #include <linux/edac.h> #include <linux/module.h> #include <linux/platform_device.h> +#include <linux/spinlock.h> #include <linux/interrupt.h> #include <linux/of.h> @@ -299,6 +300,7 @@ struct synps_ecc_status { /** * struct synps_edac_priv - DDR memory controller private instance data. * @baseaddr: Base address of the DDR controller. + * @reglock: Concurrent CSRs access lock. * @message: Buffer for framing the event specific info. * @stat: ECC status information. * @p_data: Platform data. @@ -313,6 +315,7 @@ struct synps_ecc_status { */ struct synps_edac_priv { void __iomem *baseaddr; + spinlock_t reglock; char message[SYNPS_EDAC_MSG_SIZE]; struct synps_ecc_status stat; const struct synps_platform_data *p_data; @@ -408,7 +411,8 @@ out: static int zynqmp_get_error_info(struct synps_edac_priv *priv) { struct synps_ecc_status *p; - u32 regval, clearval = 0; + u32 regval, clearval; + unsigned long flags; void __iomem *base; base = priv->baseaddr; @@ -452,10 +456,14 @@ ue_err: p->ueinfo.blknr = (regval & ECC_CEADDR1_BLKNR_MASK); p->ueinfo.data = readl(base + ECC_UESYND0_OFST); out: - clearval = ECC_CTRL_CLR_CE_ERR | ECC_CTRL_CLR_CE_ERRCNT; - clearval |= ECC_CTRL_CLR_UE_ERR | ECC_CTRL_CLR_UE_ERRCNT; + spin_lock_irqsave(&priv->reglock, flags); + + clearval = readl(base + ECC_CLR_OFST) | + ECC_CTRL_CLR_CE_ERR | ECC_CTRL_CLR_CE_ERRCNT | + ECC_CTRL_CLR_UE_ERR | ECC_CTRL_CLR_UE_ERRCNT; writel(clearval, base + ECC_CLR_OFST); - writel(0x0, base + ECC_CLR_OFST); + + spin_unlock_irqrestore(&priv->reglock, flags); return 0; } @@ -515,24 +523,41 @@ static void handle_error(struct mem_ctl_info *mci, struct synps_ecc_status *p) static void enable_intr(struct synps_edac_priv *priv) { + unsigned long flags; + /* Enable UE/CE Interrupts */ - if (priv->p_data->quirks & DDR_ECC_INTR_SELF_CLEAR) - writel(DDR_UE_MASK | DDR_CE_MASK, - priv->baseaddr + ECC_CLR_OFST); - else + if (!(priv->p_data->quirks & DDR_ECC_INTR_SELF_CLEAR)) { writel(DDR_QOSUE_MASK | DDR_QOSCE_MASK, priv->baseaddr + DDR_QOS_IRQ_EN_OFST); + return; + } + + spin_lock_irqsave(&priv->reglock, flags); + + writel(DDR_UE_MASK | DDR_CE_MASK, + priv->baseaddr + ECC_CLR_OFST); + + spin_unlock_irqrestore(&priv->reglock, flags); } static void disable_intr(struct synps_edac_priv *priv) { + unsigned long flags; + /* Disable UE/CE Interrupts */ - if (priv->p_data->quirks & DDR_ECC_INTR_SELF_CLEAR) - writel(0x0, priv->baseaddr + ECC_CLR_OFST); - else + if (!(priv->p_data->quirks & DDR_ECC_INTR_SELF_CLEAR)) { writel(DDR_QOSUE_MASK | DDR_QOSCE_MASK, priv->baseaddr + DDR_QOS_IRQ_DB_OFST); + + return; + } + + spin_lock_irqsave(&priv->reglock, flags); + + writel(0, priv->baseaddr + ECC_CLR_OFST); + + spin_unlock_irqrestore(&priv->reglock, flags); } /** @@ -576,8 +601,6 @@ static irqreturn_t intr_handler(int irq, void *dev_id) /* v3.0 of the controller does not have this register */ if (!(priv->p_data->quirks & DDR_ECC_INTR_SELF_CLEAR)) writel(regval, priv->baseaddr + DDR_QOS_IRQ_STAT_OFST); - else - enable_intr(priv); return IRQ_HANDLED; } @@ -1357,6 +1380,7 @@ static int mc_probe(struct platform_device *pdev) priv = mci->pvt_info; priv->baseaddr = baseaddr; priv->p_data = p_data; + spin_lock_init(&priv->reglock); mc_init(mci, pdev); diff --git a/drivers/edac/thunderx_edac.c b/drivers/edac/thunderx_edac.c index 90d46e5c4ff0..fab9891e569a 100644 --- a/drivers/edac/thunderx_edac.c +++ b/drivers/edac/thunderx_edac.c @@ -1365,8 +1365,7 @@ static int thunderx_ocx_probe(struct pci_dev *pdev, idx = edac_device_alloc_index(); snprintf(name, sizeof(name), "OCX%d", idx); edac_dev = edac_device_alloc_ctl_info(sizeof(struct thunderx_ocx), - name, 1, "CCPI", 1, - 0, NULL, 0, idx); + name, 1, "CCPI", 1, 0, idx); if (!edac_dev) { dev_err(&pdev->dev, "Cannot allocate EDAC device\n"); return -ENOMEM; @@ -2004,8 +2003,7 @@ static int thunderx_l2c_probe(struct pci_dev *pdev, snprintf(name, sizeof(name), fmt, idx); edac_dev = edac_device_alloc_ctl_info(sizeof(struct thunderx_l2c), - name, 1, "L2C", 1, 0, - NULL, 0, idx); + name, 1, "L2C", 1, 0, idx); if (!edac_dev) { dev_err(&pdev->dev, "Cannot allocate EDAC device\n"); return -ENOMEM; diff --git a/drivers/edac/xgene_edac.c b/drivers/edac/xgene_edac.c index 1b50f8160013..fd87f1b2c145 100644 --- a/drivers/edac/xgene_edac.c +++ b/drivers/edac/xgene_edac.c @@ -913,8 +913,8 @@ static int xgene_edac_pmd_add(struct xgene_edac *edac, struct device_node *np, snprintf(edac_name, sizeof(edac_name), "l2c%d", pmd); edac_dev = edac_device_alloc_ctl_info(sizeof(*ctx), - edac_name, 1, "l2c", 1, 2, NULL, - 0, edac_device_alloc_index()); + edac_name, 1, "l2c", 1, 2, + edac_device_alloc_index()); if (!edac_dev) { rc = -ENOMEM; goto err_group; @@ -1208,8 +1208,7 @@ static int xgene_edac_l3_add(struct xgene_edac *edac, struct device_node *np, edac_idx = edac_device_alloc_index(); edac_dev = edac_device_alloc_ctl_info(sizeof(*ctx), - "l3c", 1, "l3c", 1, 0, NULL, 0, - edac_idx); + "l3c", 1, "l3c", 1, 0, edac_idx); if (!edac_dev) { rc = -ENOMEM; goto err_release_group; @@ -1748,8 +1747,7 @@ static int xgene_edac_soc_add(struct xgene_edac *edac, struct device_node *np, edac_idx = edac_device_alloc_index(); edac_dev = edac_device_alloc_ctl_info(sizeof(*ctx), - "SOC", 1, "SOC", 1, 2, NULL, 0, - edac_idx); + "SOC", 1, "SOC", 1, 2, edac_idx); if (!edac_dev) { rc = -ENOMEM; goto err_release_group; diff --git a/drivers/edac/zynqmp_edac.c b/drivers/edac/zynqmp_edac.c index 2d9a5cfd8931..c9dc78d8c824 100644 --- a/drivers/edac/zynqmp_edac.c +++ b/drivers/edac/zynqmp_edac.c @@ -381,7 +381,7 @@ static int edac_probe(struct platform_device *pdev) } dci = edac_device_alloc_ctl_info(sizeof(*priv), ZYNQMP_OCM_EDAC_STRING, - 1, ZYNQMP_OCM_EDAC_STRING, 1, 0, NULL, 0, + 1, ZYNQMP_OCM_EDAC_STRING, 1, 0, edac_device_alloc_index()); if (!dci) return -ENOMEM; diff --git a/drivers/eisa/Kconfig b/drivers/eisa/Kconfig index c8bbf90209f5..a66b3be502a9 100644 --- a/drivers/eisa/Kconfig +++ b/drivers/eisa/Kconfig @@ -44,17 +44,16 @@ config EISA_PCI_EISA When in doubt, say Y. -# Using EISA_VIRTUAL_ROOT on something other than an Alpha or -# an X86 may lead to crashes... +# Using EISA_VIRTUAL_ROOT on something other than an X86 may lead +# to crashes... config EISA_VIRTUAL_ROOT bool "EISA virtual root device" - depends on EISA && (ALPHA || X86) + depends on EISA && X86 default y help Activate this option if your system only have EISA bus - (no PCI slots). The Alpha Jensen is an example of such - a system. + (no PCI slots). When in doubt, say Y. diff --git a/drivers/eisa/virtual_root.c b/drivers/eisa/virtual_root.c index 37e6dd219c37..cd9515d9d8f0 100644 --- a/drivers/eisa/virtual_root.c +++ b/drivers/eisa/virtual_root.c @@ -13,7 +13,7 @@ #include <linux/moduleparam.h> #include <linux/init.h> -#if defined(CONFIG_ALPHA_JENSEN) || defined(CONFIG_EISA_VLB_PRIMING) +#if defined(CONFIG_EISA_VLB_PRIMING) #define EISA_FORCE_PROBE_DEFAULT 1 #else #define EISA_FORCE_PROBE_DEFAULT 0 diff --git a/drivers/firmware/arm_ffa/driver.c b/drivers/firmware/arm_ffa/driver.c index 9bc2e10381af..1609247cfafc 100644 --- a/drivers/firmware/arm_ffa/driver.c +++ b/drivers/firmware/arm_ffa/driver.c @@ -101,11 +101,12 @@ struct ffa_drv_info { bool bitmap_created; bool notif_enabled; unsigned int sched_recv_irq; + unsigned int notif_pend_irq; unsigned int cpuhp_state; struct ffa_pcpu_irq __percpu *irq_pcpu; struct workqueue_struct *notif_pcpu_wq; struct work_struct notif_pcpu_work; - struct work_struct irq_work; + struct work_struct sched_recv_irq_work; struct xarray partition_info; DECLARE_HASHTABLE(notifier_hash, ilog2(FFA_MAX_NOTIFICATIONS)); struct mutex notify_lock; /* lock to protect notifier hashtable */ @@ -344,6 +345,38 @@ static int ffa_msg_send_direct_req(u16 src_id, u16 dst_id, bool mode_32bit, return -EINVAL; } +static int ffa_msg_send2(u16 src_id, u16 dst_id, void *buf, size_t sz) +{ + u32 src_dst_ids = PACK_TARGET_INFO(src_id, dst_id); + struct ffa_indirect_msg_hdr *msg; + ffa_value_t ret; + int retval = 0; + + if (sz > (RXTX_BUFFER_SIZE - sizeof(*msg))) + return -ERANGE; + + mutex_lock(&drv_info->tx_lock); + + msg = drv_info->tx_buffer; + msg->flags = 0; + msg->res0 = 0; + msg->offset = sizeof(*msg); + msg->send_recv_id = src_dst_ids; + msg->size = sz; + memcpy((u8 *)msg + msg->offset, buf, sz); + + /* flags = 0, sender VMID = 0 works for both physical/virtual NS */ + invoke_ffa_fn((ffa_value_t){ + .a0 = FFA_MSG_SEND2, .a1 = 0, .a2 = 0 + }, &ret); + + if (ret.a0 == FFA_ERROR) + retval = ffa_to_linux_errno((int)ret.a2); + + mutex_unlock(&drv_info->tx_lock); + return retval; +} + static int ffa_mem_first_frag(u32 func_id, phys_addr_t buf, u32 buf_sz, u32 frag_len, u32 len, u64 *handle) { @@ -870,6 +903,11 @@ static int ffa_sync_send_receive(struct ffa_device *dev, dev->mode_32bit, data); } +static int ffa_indirect_msg_send(struct ffa_device *dev, void *buf, size_t sz) +{ + return ffa_msg_send2(drv_info->vm_id, dev->vm_id, buf, sz); +} + static int ffa_memory_share(struct ffa_mem_ops_args *args) { if (drv_info->mem_ops_native) @@ -1108,7 +1146,7 @@ static void handle_notif_callbacks(u64 bitmap, enum notify_type type) } } -static void notif_pcpu_irq_work_fn(struct work_struct *work) +static void notif_get_and_handle(void *unused) { int rc; struct ffa_notify_bitmaps bitmaps; @@ -1131,10 +1169,17 @@ ffa_self_notif_handle(u16 vcpu, bool is_per_vcpu, void *cb_data) struct ffa_drv_info *info = cb_data; if (!is_per_vcpu) - notif_pcpu_irq_work_fn(&info->notif_pcpu_work); + notif_get_and_handle(info); else - queue_work_on(vcpu, info->notif_pcpu_wq, - &info->notif_pcpu_work); + smp_call_function_single(vcpu, notif_get_and_handle, info, 0); +} + +static void notif_pcpu_irq_work_fn(struct work_struct *work) +{ + struct ffa_drv_info *info = container_of(work, struct ffa_drv_info, + notif_pcpu_work); + + ffa_self_notif_handle(smp_processor_id(), true, info); } static const struct ffa_info_ops ffa_drv_info_ops = { @@ -1145,6 +1190,7 @@ static const struct ffa_info_ops ffa_drv_info_ops = { static const struct ffa_msg_ops ffa_drv_msg_ops = { .mode_32bit_set = ffa_mode_32bit_set, .sync_send_receive = ffa_sync_send_receive, + .indirect_send = ffa_indirect_msg_send, }; static const struct ffa_mem_ops ffa_drv_mem_ops = { @@ -1227,6 +1273,8 @@ static int ffa_setup_partitions(void) continue; } + ffa_dev->properties = tpbuf->properties; + if (drv_info->version > FFA_VERSION_1_0 && !(tpbuf->properties & FFA_PARTITION_AARCH64_EXEC)) ffa_mode_32bit_set(ffa_dev); @@ -1291,12 +1339,23 @@ static void ffa_partitions_cleanup(void) #define FFA_FEAT_SCHEDULE_RECEIVER_INT (2) #define FFA_FEAT_MANAGED_EXIT_INT (3) -static irqreturn_t irq_handler(int irq, void *irq_data) +static irqreturn_t ffa_sched_recv_irq_handler(int irq, void *irq_data) +{ + struct ffa_pcpu_irq *pcpu = irq_data; + struct ffa_drv_info *info = pcpu->info; + + queue_work(info->notif_pcpu_wq, &info->sched_recv_irq_work); + + return IRQ_HANDLED; +} + +static irqreturn_t notif_pend_irq_handler(int irq, void *irq_data) { struct ffa_pcpu_irq *pcpu = irq_data; struct ffa_drv_info *info = pcpu->info; - queue_work(info->notif_pcpu_wq, &info->irq_work); + queue_work_on(smp_processor_id(), info->notif_pcpu_wq, + &info->notif_pcpu_work); return IRQ_HANDLED; } @@ -1306,15 +1365,23 @@ static void ffa_sched_recv_irq_work_fn(struct work_struct *work) ffa_notification_info_get(); } -static int ffa_sched_recv_irq_map(void) +static int ffa_irq_map(u32 id) { - int ret, irq, sr_intid; + char *err_str; + int ret, irq, intid; - /* The returned sr_intid is assumed to be SGI donated to NS world */ - ret = ffa_features(FFA_FEAT_SCHEDULE_RECEIVER_INT, 0, &sr_intid, NULL); + if (id == FFA_FEAT_NOTIFICATION_PENDING_INT) + err_str = "Notification Pending Interrupt"; + else if (id == FFA_FEAT_SCHEDULE_RECEIVER_INT) + err_str = "Schedule Receiver Interrupt"; + else + err_str = "Unknown ID"; + + /* The returned intid is assumed to be SGI donated to NS world */ + ret = ffa_features(id, 0, &intid, NULL); if (ret < 0) { if (ret != -EOPNOTSUPP) - pr_err("Failed to retrieve scheduler Rx interrupt\n"); + pr_err("Failed to retrieve FF-A %s %u\n", err_str, id); return ret; } @@ -1329,12 +1396,12 @@ static int ffa_sched_recv_irq_map(void) oirq.np = gic; oirq.args_count = 1; - oirq.args[0] = sr_intid; + oirq.args[0] = intid; irq = irq_create_of_mapping(&oirq); of_node_put(gic); #ifdef CONFIG_ACPI } else { - irq = acpi_register_gsi(NULL, sr_intid, ACPI_EDGE_SENSITIVE, + irq = acpi_register_gsi(NULL, intid, ACPI_EDGE_SENSITIVE, ACPI_ACTIVE_HIGH); #endif } @@ -1347,23 +1414,28 @@ static int ffa_sched_recv_irq_map(void) return irq; } -static void ffa_sched_recv_irq_unmap(void) +static void ffa_irq_unmap(unsigned int irq) { - if (drv_info->sched_recv_irq) { - irq_dispose_mapping(drv_info->sched_recv_irq); - drv_info->sched_recv_irq = 0; - } + if (!irq) + return; + irq_dispose_mapping(irq); } static int ffa_cpuhp_pcpu_irq_enable(unsigned int cpu) { - enable_percpu_irq(drv_info->sched_recv_irq, IRQ_TYPE_NONE); + if (drv_info->sched_recv_irq) + enable_percpu_irq(drv_info->sched_recv_irq, IRQ_TYPE_NONE); + if (drv_info->notif_pend_irq) + enable_percpu_irq(drv_info->notif_pend_irq, IRQ_TYPE_NONE); return 0; } static int ffa_cpuhp_pcpu_irq_disable(unsigned int cpu) { - disable_percpu_irq(drv_info->sched_recv_irq); + if (drv_info->sched_recv_irq) + disable_percpu_irq(drv_info->sched_recv_irq); + if (drv_info->notif_pend_irq) + disable_percpu_irq(drv_info->notif_pend_irq); return 0; } @@ -1382,13 +1454,16 @@ static void ffa_uninit_pcpu_irq(void) if (drv_info->sched_recv_irq) free_percpu_irq(drv_info->sched_recv_irq, drv_info->irq_pcpu); + if (drv_info->notif_pend_irq) + free_percpu_irq(drv_info->notif_pend_irq, drv_info->irq_pcpu); + if (drv_info->irq_pcpu) { free_percpu(drv_info->irq_pcpu); drv_info->irq_pcpu = NULL; } } -static int ffa_init_pcpu_irq(unsigned int irq) +static int ffa_init_pcpu_irq(void) { struct ffa_pcpu_irq __percpu *irq_pcpu; int ret, cpu; @@ -1402,13 +1477,31 @@ static int ffa_init_pcpu_irq(unsigned int irq) drv_info->irq_pcpu = irq_pcpu; - ret = request_percpu_irq(irq, irq_handler, "ARM-FFA", irq_pcpu); - if (ret) { - pr_err("Error registering notification IRQ %d: %d\n", irq, ret); - return ret; + if (drv_info->sched_recv_irq) { + ret = request_percpu_irq(drv_info->sched_recv_irq, + ffa_sched_recv_irq_handler, + "ARM-FFA-SRI", irq_pcpu); + if (ret) { + pr_err("Error registering percpu SRI nIRQ %d : %d\n", + drv_info->sched_recv_irq, ret); + drv_info->sched_recv_irq = 0; + return ret; + } } - INIT_WORK(&drv_info->irq_work, ffa_sched_recv_irq_work_fn); + if (drv_info->notif_pend_irq) { + ret = request_percpu_irq(drv_info->notif_pend_irq, + notif_pend_irq_handler, + "ARM-FFA-NPI", irq_pcpu); + if (ret) { + pr_err("Error registering percpu NPI nIRQ %d : %d\n", + drv_info->notif_pend_irq, ret); + drv_info->notif_pend_irq = 0; + return ret; + } + } + + INIT_WORK(&drv_info->sched_recv_irq_work, ffa_sched_recv_irq_work_fn); INIT_WORK(&drv_info->notif_pcpu_work, notif_pcpu_irq_work_fn); drv_info->notif_pcpu_wq = create_workqueue("ffa_pcpu_irq_notification"); if (!drv_info->notif_pcpu_wq) @@ -1428,7 +1521,10 @@ static int ffa_init_pcpu_irq(unsigned int irq) static void ffa_notifications_cleanup(void) { ffa_uninit_pcpu_irq(); - ffa_sched_recv_irq_unmap(); + ffa_irq_unmap(drv_info->sched_recv_irq); + drv_info->sched_recv_irq = 0; + ffa_irq_unmap(drv_info->notif_pend_irq); + drv_info->notif_pend_irq = 0; if (drv_info->bitmap_created) { ffa_notification_bitmap_destroy(); @@ -1439,30 +1535,31 @@ static void ffa_notifications_cleanup(void) static void ffa_notifications_setup(void) { - int ret, irq; + int ret; ret = ffa_features(FFA_NOTIFICATION_BITMAP_CREATE, 0, NULL, NULL); - if (ret) { - pr_info("Notifications not supported, continuing with it ..\n"); - return; - } + if (!ret) { + ret = ffa_notification_bitmap_create(); + if (ret) { + pr_err("Notification bitmap create error %d\n", ret); + return; + } - ret = ffa_notification_bitmap_create(); - if (ret) { - pr_info("Notification bitmap create error %d\n", ret); - return; + drv_info->bitmap_created = true; } - drv_info->bitmap_created = true; - irq = ffa_sched_recv_irq_map(); - if (irq <= 0) { - ret = irq; - goto cleanup; - } + ret = ffa_irq_map(FFA_FEAT_SCHEDULE_RECEIVER_INT); + if (ret > 0) + drv_info->sched_recv_irq = ret; + + ret = ffa_irq_map(FFA_FEAT_NOTIFICATION_PENDING_INT); + if (ret > 0) + drv_info->notif_pend_irq = ret; - drv_info->sched_recv_irq = irq; + if (!drv_info->sched_recv_irq && !drv_info->notif_pend_irq) + goto cleanup; - ret = ffa_init_pcpu_irq(irq); + ret = ffa_init_pcpu_irq(); if (ret) goto cleanup; diff --git a/drivers/firmware/arm_scmi/Makefile b/drivers/firmware/arm_scmi/Makefile index a7bc4796519c..fd59f58ce8a2 100644 --- a/drivers/firmware/arm_scmi/Makefile +++ b/drivers/firmware/arm_scmi/Makefile @@ -10,7 +10,8 @@ scmi-transport-$(CONFIG_ARM_SCMI_TRANSPORT_SMC) += smc.o scmi-transport-$(CONFIG_ARM_SCMI_HAVE_MSG) += msg.o scmi-transport-$(CONFIG_ARM_SCMI_TRANSPORT_VIRTIO) += virtio.o scmi-transport-$(CONFIG_ARM_SCMI_TRANSPORT_OPTEE) += optee.o -scmi-protocols-y = base.o clock.o perf.o power.o reset.o sensors.o system.o voltage.o powercap.o +scmi-protocols-y := base.o clock.o perf.o power.o reset.o sensors.o system.o voltage.o powercap.o +scmi-protocols-y += pinctrl.o scmi-module-objs := $(scmi-driver-y) $(scmi-protocols-y) $(scmi-transport-y) obj-$(CONFIG_ARM_SCMI_PROTOCOL) += scmi-core.o diff --git a/drivers/firmware/arm_scmi/common.h b/drivers/firmware/arm_scmi/common.h index 6affbfdd1dec..b5ac25dbc1ca 100644 --- a/drivers/firmware/arm_scmi/common.h +++ b/drivers/firmware/arm_scmi/common.h @@ -301,6 +301,17 @@ extern const struct scmi_desc scmi_optee_desc; void scmi_rx_callback(struct scmi_chan_info *cinfo, u32 msg_hdr, void *priv); +enum scmi_bad_msg { + MSG_UNEXPECTED = -1, + MSG_INVALID = -2, + MSG_UNKNOWN = -3, + MSG_NOMEM = -4, + MSG_MBOX_SPURIOUS = -5, +}; + +void scmi_bad_message_trace(struct scmi_chan_info *cinfo, u32 msg_hdr, + enum scmi_bad_msg err); + /* shmem related declarations */ struct scmi_shared_mem; diff --git a/drivers/firmware/arm_scmi/driver.c b/drivers/firmware/arm_scmi/driver.c index 2709598f3008..6b6957f4743f 100644 --- a/drivers/firmware/arm_scmi/driver.c +++ b/drivers/firmware/arm_scmi/driver.c @@ -33,6 +33,7 @@ #include <linux/processor.h> #include <linux/refcount.h> #include <linux/slab.h> +#include <linux/xarray.h> #include "common.h" #include "notify.h" @@ -44,8 +45,7 @@ static DEFINE_IDA(scmi_id); -static DEFINE_IDR(scmi_protocols); -static DEFINE_SPINLOCK(protocol_lock); +static DEFINE_XARRAY(scmi_protocols); /* List of all SCMI devices active in system */ static LIST_HEAD(scmi_list); @@ -194,11 +194,94 @@ struct scmi_info { #define bus_nb_to_scmi_info(nb) container_of(nb, struct scmi_info, bus_nb) #define req_nb_to_scmi_info(nb) container_of(nb, struct scmi_info, dev_req_nb) -static const struct scmi_protocol *scmi_protocol_get(int protocol_id) +static unsigned long +scmi_vendor_protocol_signature(unsigned int protocol_id, char *vendor_id, + char *sub_vendor_id, u32 impl_ver) { - const struct scmi_protocol *proto; + char *signature, *p; + unsigned long hash = 0; - proto = idr_find(&scmi_protocols, protocol_id); + /* vendor_id/sub_vendor_id guaranteed <= SCMI_SHORT_NAME_MAX_SIZE */ + signature = kasprintf(GFP_KERNEL, "%02X|%s|%s|0x%08X", protocol_id, + vendor_id ?: "", sub_vendor_id ?: "", impl_ver); + if (!signature) + return 0; + + p = signature; + while (*p) + hash = partial_name_hash(tolower(*p++), hash); + hash = end_name_hash(hash); + + kfree(signature); + + return hash; +} + +static unsigned long +scmi_protocol_key_calculate(int protocol_id, char *vendor_id, + char *sub_vendor_id, u32 impl_ver) +{ + if (protocol_id < SCMI_PROTOCOL_VENDOR_BASE) + return protocol_id; + else + return scmi_vendor_protocol_signature(protocol_id, vendor_id, + sub_vendor_id, impl_ver); +} + +static const struct scmi_protocol * +__scmi_vendor_protocol_lookup(int protocol_id, char *vendor_id, + char *sub_vendor_id, u32 impl_ver) +{ + unsigned long key; + struct scmi_protocol *proto = NULL; + + key = scmi_protocol_key_calculate(protocol_id, vendor_id, + sub_vendor_id, impl_ver); + if (key) + proto = xa_load(&scmi_protocols, key); + + return proto; +} + +static const struct scmi_protocol * +scmi_vendor_protocol_lookup(int protocol_id, char *vendor_id, + char *sub_vendor_id, u32 impl_ver) +{ + const struct scmi_protocol *proto = NULL; + + /* Searching for closest match ...*/ + proto = __scmi_vendor_protocol_lookup(protocol_id, vendor_id, + sub_vendor_id, impl_ver); + if (proto) + return proto; + + /* Any match just on vendor/sub_vendor ? */ + if (impl_ver) { + proto = __scmi_vendor_protocol_lookup(protocol_id, vendor_id, + sub_vendor_id, 0); + if (proto) + return proto; + } + + /* Any match just on the vendor ? */ + if (sub_vendor_id) + proto = __scmi_vendor_protocol_lookup(protocol_id, vendor_id, + NULL, 0); + return proto; +} + +static const struct scmi_protocol * +scmi_protocol_get(int protocol_id, struct scmi_revision_info *version) +{ + const struct scmi_protocol *proto = NULL; + + if (protocol_id < SCMI_PROTOCOL_VENDOR_BASE) + proto = xa_load(&scmi_protocols, protocol_id); + else + proto = scmi_vendor_protocol_lookup(protocol_id, + version->vendor_id, + version->sub_vendor_id, + version->impl_ver); if (!proto || !try_module_get(proto->owner)) { pr_warn("SCMI Protocol 0x%x not found!\n", protocol_id); return NULL; @@ -206,21 +289,46 @@ static const struct scmi_protocol *scmi_protocol_get(int protocol_id) pr_debug("Found SCMI Protocol 0x%x\n", protocol_id); + if (protocol_id >= SCMI_PROTOCOL_VENDOR_BASE) + pr_info("Loaded SCMI Vendor Protocol 0x%x - %s %s %X\n", + protocol_id, proto->vendor_id ?: "", + proto->sub_vendor_id ?: "", proto->impl_ver); + return proto; } -static void scmi_protocol_put(int protocol_id) +static void scmi_protocol_put(const struct scmi_protocol *proto) { - const struct scmi_protocol *proto; - - proto = idr_find(&scmi_protocols, protocol_id); if (proto) module_put(proto->owner); } +static int scmi_vendor_protocol_check(const struct scmi_protocol *proto) +{ + if (!proto->vendor_id) { + pr_err("missing vendor_id for protocol 0x%x\n", proto->id); + return -EINVAL; + } + + if (strlen(proto->vendor_id) >= SCMI_SHORT_NAME_MAX_SIZE) { + pr_err("malformed vendor_id for protocol 0x%x\n", proto->id); + return -EINVAL; + } + + if (proto->sub_vendor_id && + strlen(proto->sub_vendor_id) >= SCMI_SHORT_NAME_MAX_SIZE) { + pr_err("malformed sub_vendor_id for protocol 0x%x\n", + proto->id); + return -EINVAL; + } + + return 0; +} + int scmi_protocol_register(const struct scmi_protocol *proto) { int ret; + unsigned long key; if (!proto) { pr_err("invalid protocol\n"); @@ -232,12 +340,23 @@ int scmi_protocol_register(const struct scmi_protocol *proto) return -EINVAL; } - spin_lock(&protocol_lock); - ret = idr_alloc(&scmi_protocols, (void *)proto, - proto->id, proto->id + 1, GFP_ATOMIC); - spin_unlock(&protocol_lock); - if (ret != proto->id) { - pr_err("unable to allocate SCMI idr slot for 0x%x - err %d\n", + if (proto->id >= SCMI_PROTOCOL_VENDOR_BASE && + scmi_vendor_protocol_check(proto)) + return -EINVAL; + + /* + * Calculate a protocol key to register this protocol with the core; + * key value 0 is considered invalid. + */ + key = scmi_protocol_key_calculate(proto->id, proto->vendor_id, + proto->sub_vendor_id, + proto->impl_ver); + if (!key) + return -EINVAL; + + ret = xa_insert(&scmi_protocols, key, (void *)proto, GFP_KERNEL); + if (ret) { + pr_err("unable to allocate SCMI protocol slot for 0x%x - err %d\n", proto->id, ret); return ret; } @@ -250,9 +369,15 @@ EXPORT_SYMBOL_GPL(scmi_protocol_register); void scmi_protocol_unregister(const struct scmi_protocol *proto) { - spin_lock(&protocol_lock); - idr_remove(&scmi_protocols, proto->id); - spin_unlock(&protocol_lock); + unsigned long key; + + key = scmi_protocol_key_calculate(proto->id, proto->vendor_id, + proto->sub_vendor_id, + proto->impl_ver); + if (!key) + return; + + xa_erase(&scmi_protocols, key); pr_debug("Unregistered SCMI Protocol 0x%x\n", proto->id); } @@ -697,6 +822,45 @@ scmi_xfer_lookup_unlocked(struct scmi_xfers_info *minfo, u16 xfer_id) } /** + * scmi_bad_message_trace - A helper to trace weird messages + * + * @cinfo: A reference to the channel descriptor on which the message was + * received + * @msg_hdr: Message header to track + * @err: A specific error code used as a status value in traces. + * + * This helper can be used to trace any kind of weird, incomplete, unexpected, + * timed-out message that arrives and as such, can be traced only referring to + * the header content, since the payload is missing/unreliable. + */ +void scmi_bad_message_trace(struct scmi_chan_info *cinfo, u32 msg_hdr, + enum scmi_bad_msg err) +{ + char *tag; + struct scmi_info *info = handle_to_scmi_info(cinfo->handle); + + switch (MSG_XTRACT_TYPE(msg_hdr)) { + case MSG_TYPE_COMMAND: + tag = "!RESP"; + break; + case MSG_TYPE_DELAYED_RESP: + tag = "!DLYD"; + break; + case MSG_TYPE_NOTIFICATION: + tag = "!NOTI"; + break; + default: + tag = "!UNKN"; + break; + } + + trace_scmi_msg_dump(info->id, cinfo->id, + MSG_XTRACT_PROT_ID(msg_hdr), + MSG_XTRACT_ID(msg_hdr), tag, + MSG_XTRACT_TOKEN(msg_hdr), err, NULL, 0); +} + +/** * scmi_msg_response_validate - Validate message type against state of related * xfer * @@ -822,6 +986,9 @@ scmi_xfer_command_acquire(struct scmi_chan_info *cinfo, u32 msg_hdr) "Message for %d type %d is not expected!\n", xfer_id, msg_type); spin_unlock_irqrestore(&minfo->xfer_lock, flags); + + scmi_bad_message_trace(cinfo, msg_hdr, MSG_UNEXPECTED); + return xfer; } refcount_inc(&xfer->users); @@ -846,6 +1013,9 @@ scmi_xfer_command_acquire(struct scmi_chan_info *cinfo, u32 msg_hdr) dev_err(cinfo->dev, "Invalid message type:%d for %d - HDR:0x%X state:%d\n", msg_type, xfer_id, msg_hdr, xfer->state); + + scmi_bad_message_trace(cinfo, msg_hdr, MSG_INVALID); + /* On error the refcount incremented above has to be dropped */ __scmi_xfer_put(minfo, xfer); xfer = ERR_PTR(-EINVAL); @@ -882,6 +1052,9 @@ static void scmi_handle_notification(struct scmi_chan_info *cinfo, if (IS_ERR(xfer)) { dev_err(dev, "failed to get free message slot (%ld)\n", PTR_ERR(xfer)); + + scmi_bad_message_trace(cinfo, msg_hdr, MSG_NOMEM); + scmi_clear_channel(info, cinfo); return; } @@ -1001,6 +1174,7 @@ void scmi_rx_callback(struct scmi_chan_info *cinfo, u32 msg_hdr, void *priv) break; default: WARN_ONCE(1, "received unknown msg_type:%d\n", msg_type); + scmi_bad_message_trace(cinfo, msg_hdr, MSG_UNKNOWN); break; } } @@ -1489,6 +1663,20 @@ out: } /** + * scmi_common_get_max_msg_size - Get maximum message size + * @ph: A protocol handle reference. + * + * Return: Maximum message size for the current protocol. + */ +static int scmi_common_get_max_msg_size(const struct scmi_protocol_handle *ph) +{ + const struct scmi_protocol_instance *pi = ph_to_pi(ph); + struct scmi_info *info = handle_to_scmi_info(pi->handle); + + return info->desc->max_msg_size; +} + +/** * struct scmi_iterator - Iterator descriptor * @msg: A reference to the message TX buffer; filled by @prepare_message with * a proper custom command payload for each multi-part command request. @@ -1799,6 +1987,7 @@ static int scmi_protocol_msg_check(const struct scmi_protocol_handle *ph, static const struct scmi_proto_helpers_ops helpers_ops = { .extended_name_get = scmi_common_extended_name_get, + .get_max_msg_size = scmi_common_get_max_msg_size, .iter_response_init = scmi_iterator_init, .iter_response_run = scmi_iterator_run, .protocol_msg_check = scmi_protocol_msg_check, @@ -1891,7 +2080,7 @@ scmi_alloc_init_protocol_instance(struct scmi_info *info, /* Protocol specific devres group */ gid = devres_open_group(handle->dev, NULL, GFP_KERNEL); if (!gid) { - scmi_protocol_put(proto->id); + scmi_protocol_put(proto); goto out; } @@ -1955,7 +2144,7 @@ scmi_alloc_init_protocol_instance(struct scmi_info *info, clean: /* Take care to put the protocol module's owner before releasing all */ - scmi_protocol_put(proto->id); + scmi_protocol_put(proto); devres_release_group(handle->dev, gid); out: return ERR_PTR(ret); @@ -1989,7 +2178,7 @@ scmi_get_protocol_instance(const struct scmi_handle *handle, u8 protocol_id) const struct scmi_protocol *proto; /* Fails if protocol not registered on bus */ - proto = scmi_protocol_get(protocol_id); + proto = scmi_protocol_get(protocol_id, &info->version); if (proto) pi = scmi_alloc_init_protocol_instance(info, proto); else @@ -2044,7 +2233,7 @@ void scmi_protocol_release(const struct scmi_handle *handle, u8 protocol_id) idr_remove(&info->protocols, protocol_id); - scmi_protocol_put(protocol_id); + scmi_protocol_put(pi->proto); devres_release_group(handle->dev, gid); dev_dbg(handle->dev, "De-Initialized protocol: 0x%X\n", @@ -2491,6 +2680,10 @@ scmi_txrx_setup(struct scmi_info *info, struct device_node *of_node, ret = 0; } + if (ret) + dev_err(info->dev, + "failed to setup channel for protocol:0x%X\n", prot_id); + return ret; } @@ -2760,6 +2953,7 @@ static int scmi_debugfs_raw_mode_setup(struct scmi_info *info) static int scmi_probe(struct platform_device *pdev) { int ret; + char *err_str = "probe failure\n"; struct scmi_handle *handle; const struct scmi_desc *desc; struct scmi_info *info; @@ -2810,27 +3004,37 @@ static int scmi_probe(struct platform_device *pdev) if (desc->ops->link_supplier) { ret = desc->ops->link_supplier(dev); - if (ret) + if (ret) { + err_str = "transport not ready\n"; goto clear_ida; + } } /* Setup all channels described in the DT at first */ ret = scmi_channels_setup(info); - if (ret) + if (ret) { + err_str = "failed to setup channels\n"; goto clear_ida; + } ret = bus_register_notifier(&scmi_bus_type, &info->bus_nb); - if (ret) + if (ret) { + err_str = "failed to register bus notifier\n"; goto clear_txrx_setup; + } ret = blocking_notifier_chain_register(&scmi_requested_devices_nh, &info->dev_req_nb); - if (ret) + if (ret) { + err_str = "failed to register device notifier\n"; goto clear_bus_notifier; + } ret = scmi_xfer_info_init(info); - if (ret) + if (ret) { + err_str = "failed to init xfers pool\n"; goto clear_dev_req_notifier; + } if (scmi_top_dentry) { info->dbg = scmi_debugfs_common_setup(info); @@ -2867,9 +3071,11 @@ static int scmi_probe(struct platform_device *pdev) */ ret = scmi_protocol_acquire(handle, SCMI_PROTOCOL_BASE); if (ret) { - dev_err(dev, "unable to communicate with SCMI\n"); - if (coex) + err_str = "unable to communicate with SCMI\n"; + if (coex) { + dev_err(dev, "%s", err_str); return 0; + } goto notification_exit; } @@ -2923,7 +3129,8 @@ clear_txrx_setup: scmi_cleanup_txrx_channels(info); clear_ida: ida_free(&scmi_id, info->id); - return ret; + + return dev_err_probe(dev, ret, "%s", err_str); } static void scmi_remove(struct platform_device *pdev) @@ -3127,6 +3334,7 @@ static int __init scmi_driver_init(void) scmi_voltage_register(); scmi_system_register(); scmi_powercap_register(); + scmi_pinctrl_register(); return platform_driver_register(&scmi_driver); } @@ -3144,6 +3352,7 @@ static void __exit scmi_driver_exit(void) scmi_voltage_unregister(); scmi_system_unregister(); scmi_powercap_unregister(); + scmi_pinctrl_unregister(); scmi_transports_exit(); diff --git a/drivers/firmware/arm_scmi/mailbox.c b/drivers/firmware/arm_scmi/mailbox.c index b8d470417e8f..615a3b2ad83d 100644 --- a/drivers/firmware/arm_scmi/mailbox.c +++ b/drivers/firmware/arm_scmi/mailbox.c @@ -56,6 +56,9 @@ static void rx_callback(struct mbox_client *cl, void *m) */ if (cl->knows_txdone && !shmem_channel_free(smbox->shmem)) { dev_warn(smbox->cinfo->dev, "Ignoring spurious A2P IRQ !\n"); + scmi_bad_message_trace(smbox->cinfo, + shmem_read_header(smbox->shmem), + MSG_MBOX_SPURIOUS); return; } diff --git a/drivers/firmware/arm_scmi/notify.c b/drivers/firmware/arm_scmi/notify.c index 27c52531194d..e160ecb22948 100644 --- a/drivers/firmware/arm_scmi/notify.c +++ b/drivers/firmware/arm_scmi/notify.c @@ -1513,17 +1513,12 @@ static int scmi_devm_notifier_register(struct scmi_device *sdev, static int scmi_devm_notifier_match(struct device *dev, void *res, void *data) { struct scmi_notifier_devres *dres = res; - struct scmi_notifier_devres *xres = data; + struct notifier_block *nb = data; - if (WARN_ON(!dres || !xres)) + if (WARN_ON(!dres || !nb)) return 0; - return dres->proto_id == xres->proto_id && - dres->evt_id == xres->evt_id && - dres->nb == xres->nb && - ((!dres->src_id && !xres->src_id) || - (dres->src_id && xres->src_id && - dres->__src_id == xres->__src_id)); + return dres->nb == nb; } /** @@ -1531,10 +1526,6 @@ static int scmi_devm_notifier_match(struct device *dev, void *res, void *data) * notifier_block for an event * @sdev: A reference to an scmi_device whose embedded struct device is to * be used for devres accounting. - * @proto_id: Protocol ID - * @evt_id: Event ID - * @src_id: Source ID, when NULL register for events coming form ALL possible - * sources * @nb: A standard notifier block to register for the specified event * * Generic devres managed helper to explicitly un-register a notifier_block @@ -1544,25 +1535,12 @@ static int scmi_devm_notifier_match(struct device *dev, void *res, void *data) * Return: 0 on Success */ static int scmi_devm_notifier_unregister(struct scmi_device *sdev, - u8 proto_id, u8 evt_id, - const u32 *src_id, struct notifier_block *nb) { int ret; - struct scmi_notifier_devres dres; - - dres.handle = sdev->handle; - dres.proto_id = proto_id; - dres.evt_id = evt_id; - if (src_id) { - dres.__src_id = *src_id; - dres.src_id = &dres.__src_id; - } else { - dres.src_id = NULL; - } ret = devres_release(&sdev->dev, scmi_devm_release_notifier, - scmi_devm_notifier_match, &dres); + scmi_devm_notifier_match, nb); WARN_ON(ret); diff --git a/drivers/firmware/arm_scmi/perf.c b/drivers/firmware/arm_scmi/perf.c index 345fff167b52..4b7f1cbb9b04 100644 --- a/drivers/firmware/arm_scmi/perf.c +++ b/drivers/firmware/arm_scmi/perf.c @@ -387,8 +387,8 @@ process_response_opp(struct device *dev, struct perf_dom_info *dom, ret = xa_insert(&dom->opps_by_lvl, opp->perf, opp, GFP_KERNEL); if (ret) - dev_warn(dev, "Failed to add opps_by_lvl at %d - ret:%d\n", - opp->perf, ret); + dev_warn(dev, "Failed to add opps_by_lvl at %d for %s - ret:%d\n", + opp->perf, dom->info.name, ret); } static inline void @@ -405,8 +405,8 @@ process_response_opp_v4(struct device *dev, struct perf_dom_info *dom, ret = xa_insert(&dom->opps_by_lvl, opp->perf, opp, GFP_KERNEL); if (ret) - dev_warn(dev, "Failed to add opps_by_lvl at %d - ret:%d\n", - opp->perf, ret); + dev_warn(dev, "Failed to add opps_by_lvl at %d for %s - ret:%d\n", + opp->perf, dom->info.name, ret); /* Note that PERF v4 reports always five 32-bit words */ opp->indicative_freq = le32_to_cpu(r->opp[loop_idx].indicative_freq); @@ -417,8 +417,8 @@ process_response_opp_v4(struct device *dev, struct perf_dom_info *dom, GFP_KERNEL); if (ret) dev_warn(dev, - "Failed to add opps_by_idx at %d - ret:%d\n", - opp->level_index, ret); + "Failed to add opps_by_idx at %d for %s - ret:%d\n", + opp->level_index, dom->info.name, ret); hash_add(dom->opps_by_freq, &opp->hash, opp->indicative_freq); } @@ -879,7 +879,8 @@ static int scmi_dvfs_device_opps_add(const struct scmi_protocol_handle *ph, ret = dev_pm_opp_add_dynamic(dev, &data); if (ret) { - dev_warn(dev, "failed to add opp %luHz\n", freq); + dev_warn(dev, "[%d][%s]: Failed to add OPP[%d] %lu\n", + domain, dom->info.name, idx, freq); dev_pm_opp_remove_all_dynamic(dev); return ret; } diff --git a/drivers/firmware/arm_scmi/pinctrl.c b/drivers/firmware/arm_scmi/pinctrl.c new file mode 100644 index 000000000000..a2a7f880d6a3 --- /dev/null +++ b/drivers/firmware/arm_scmi/pinctrl.c @@ -0,0 +1,916 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * System Control and Management Interface (SCMI) Pinctrl Protocol + * + * Copyright (C) 2024 EPAM + * Copyright 2024 NXP + */ + +#include <asm/byteorder.h> +#include <linux/bits.h> +#include <linux/bitfield.h> +#include <linux/device.h> +#include <linux/module.h> +#include <linux/scmi_protocol.h> +#include <linux/slab.h> +#include <linux/string.h> +#include <linux/types.h> + +#include "common.h" +#include "protocols.h" + +/* Updated only after ALL the mandatory features for that version are merged */ +#define SCMI_PROTOCOL_SUPPORTED_VERSION 0x10000 + +#define GET_GROUPS_NR(x) le32_get_bits((x), GENMASK(31, 16)) +#define GET_PINS_NR(x) le32_get_bits((x), GENMASK(15, 0)) +#define GET_FUNCTIONS_NR(x) le32_get_bits((x), GENMASK(15, 0)) + +#define EXT_NAME_FLAG(x) le32_get_bits((x), BIT(31)) +#define NUM_ELEMS(x) le32_get_bits((x), GENMASK(15, 0)) + +#define REMAINING(x) le32_get_bits((x), GENMASK(31, 16)) +#define RETURNED(x) le32_get_bits((x), GENMASK(11, 0)) + +#define CONFIG_FLAG_MASK GENMASK(19, 18) +#define SELECTOR_MASK GENMASK(17, 16) +#define SKIP_CONFIGS_MASK GENMASK(15, 8) +#define CONFIG_TYPE_MASK GENMASK(7, 0) + +enum scmi_pinctrl_protocol_cmd { + PINCTRL_ATTRIBUTES = 0x3, + PINCTRL_LIST_ASSOCIATIONS = 0x4, + PINCTRL_SETTINGS_GET = 0x5, + PINCTRL_SETTINGS_CONFIGURE = 0x6, + PINCTRL_REQUEST = 0x7, + PINCTRL_RELEASE = 0x8, + PINCTRL_NAME_GET = 0x9, + PINCTRL_SET_PERMISSIONS = 0xa, +}; + +struct scmi_msg_settings_conf { + __le32 identifier; + __le32 function_id; + __le32 attributes; + __le32 configs[]; +}; + +struct scmi_msg_settings_get { + __le32 identifier; + __le32 attributes; +}; + +struct scmi_resp_settings_get { + __le32 function_selected; + __le32 num_configs; + __le32 configs[]; +}; + +struct scmi_msg_pinctrl_protocol_attributes { + __le32 attributes_low; + __le32 attributes_high; +}; + +struct scmi_msg_pinctrl_attributes { + __le32 identifier; + __le32 flags; +}; + +struct scmi_resp_pinctrl_attributes { + __le32 attributes; + u8 name[SCMI_SHORT_NAME_MAX_SIZE]; +}; + +struct scmi_msg_pinctrl_list_assoc { + __le32 identifier; + __le32 flags; + __le32 index; +}; + +struct scmi_resp_pinctrl_list_assoc { + __le32 flags; + __le16 array[]; +}; + +struct scmi_msg_request { + __le32 identifier; + __le32 flags; +}; + +struct scmi_group_info { + char name[SCMI_MAX_STR_SIZE]; + bool present; + u32 *group_pins; + u32 nr_pins; +}; + +struct scmi_function_info { + char name[SCMI_MAX_STR_SIZE]; + bool present; + u32 *groups; + u32 nr_groups; +}; + +struct scmi_pin_info { + char name[SCMI_MAX_STR_SIZE]; + bool present; +}; + +struct scmi_pinctrl_info { + u32 version; + int nr_groups; + int nr_functions; + int nr_pins; + struct scmi_group_info *groups; + struct scmi_function_info *functions; + struct scmi_pin_info *pins; +}; + +static int scmi_pinctrl_attributes_get(const struct scmi_protocol_handle *ph, + struct scmi_pinctrl_info *pi) +{ + int ret; + struct scmi_xfer *t; + struct scmi_msg_pinctrl_protocol_attributes *attr; + + ret = ph->xops->xfer_get_init(ph, PROTOCOL_ATTRIBUTES, 0, sizeof(*attr), &t); + if (ret) + return ret; + + attr = t->rx.buf; + + ret = ph->xops->do_xfer(ph, t); + if (!ret) { + pi->nr_functions = GET_FUNCTIONS_NR(attr->attributes_high); + pi->nr_groups = GET_GROUPS_NR(attr->attributes_low); + pi->nr_pins = GET_PINS_NR(attr->attributes_low); + if (pi->nr_pins == 0) { + dev_warn(ph->dev, "returned zero pins\n"); + ret = -EINVAL; + } + } + + ph->xops->xfer_put(ph, t); + return ret; +} + +static int scmi_pinctrl_count_get(const struct scmi_protocol_handle *ph, + enum scmi_pinctrl_selector_type type) +{ + struct scmi_pinctrl_info *pi = ph->get_priv(ph); + + switch (type) { + case PIN_TYPE: + return pi->nr_pins; + case GROUP_TYPE: + return pi->nr_groups; + case FUNCTION_TYPE: + return pi->nr_functions; + default: + return -EINVAL; + } +} + +static int scmi_pinctrl_validate_id(const struct scmi_protocol_handle *ph, + u32 selector, + enum scmi_pinctrl_selector_type type) +{ + int value; + + value = scmi_pinctrl_count_get(ph, type); + if (value < 0) + return value; + + if (selector >= value || value == 0) + return -EINVAL; + + return 0; +} + +static int scmi_pinctrl_attributes(const struct scmi_protocol_handle *ph, + enum scmi_pinctrl_selector_type type, + u32 selector, char *name, + u32 *n_elems) +{ + int ret; + struct scmi_xfer *t; + struct scmi_msg_pinctrl_attributes *tx; + struct scmi_resp_pinctrl_attributes *rx; + bool ext_name_flag; + + if (!name) + return -EINVAL; + + ret = scmi_pinctrl_validate_id(ph, selector, type); + if (ret) + return ret; + + ret = ph->xops->xfer_get_init(ph, PINCTRL_ATTRIBUTES, sizeof(*tx), + sizeof(*rx), &t); + if (ret) + return ret; + + tx = t->tx.buf; + rx = t->rx.buf; + tx->identifier = cpu_to_le32(selector); + tx->flags = cpu_to_le32(type); + + ret = ph->xops->do_xfer(ph, t); + if (!ret) { + if (n_elems) + *n_elems = NUM_ELEMS(rx->attributes); + + strscpy(name, rx->name, SCMI_SHORT_NAME_MAX_SIZE); + + ext_name_flag = !!EXT_NAME_FLAG(rx->attributes); + } + + ph->xops->xfer_put(ph, t); + + if (ret) + return ret; + /* + * If supported overwrite short name with the extended one; + * on error just carry on and use already provided short name. + */ + if (ext_name_flag) + ret = ph->hops->extended_name_get(ph, PINCTRL_NAME_GET, + selector, (u32 *)&type, name, + SCMI_MAX_STR_SIZE); + return ret; +} + +struct scmi_pinctrl_ipriv { + u32 selector; + enum scmi_pinctrl_selector_type type; + u32 *array; +}; + +static void iter_pinctrl_assoc_prepare_message(void *message, + u32 desc_index, + const void *priv) +{ + struct scmi_msg_pinctrl_list_assoc *msg = message; + const struct scmi_pinctrl_ipriv *p = priv; + + msg->identifier = cpu_to_le32(p->selector); + msg->flags = cpu_to_le32(p->type); + msg->index = cpu_to_le32(desc_index); +} + +static int iter_pinctrl_assoc_update_state(struct scmi_iterator_state *st, + const void *response, void *priv) +{ + const struct scmi_resp_pinctrl_list_assoc *r = response; + + st->num_returned = RETURNED(r->flags); + st->num_remaining = REMAINING(r->flags); + + return 0; +} + +static int +iter_pinctrl_assoc_process_response(const struct scmi_protocol_handle *ph, + const void *response, + struct scmi_iterator_state *st, void *priv) +{ + const struct scmi_resp_pinctrl_list_assoc *r = response; + struct scmi_pinctrl_ipriv *p = priv; + + p->array[st->desc_index + st->loop_idx] = + le16_to_cpu(r->array[st->loop_idx]); + + return 0; +} + +static int scmi_pinctrl_list_associations(const struct scmi_protocol_handle *ph, + u32 selector, + enum scmi_pinctrl_selector_type type, + u16 size, u32 *array) +{ + int ret; + void *iter; + struct scmi_iterator_ops ops = { + .prepare_message = iter_pinctrl_assoc_prepare_message, + .update_state = iter_pinctrl_assoc_update_state, + .process_response = iter_pinctrl_assoc_process_response, + }; + struct scmi_pinctrl_ipriv ipriv = { + .selector = selector, + .type = type, + .array = array, + }; + + if (!array || !size || type == PIN_TYPE) + return -EINVAL; + + ret = scmi_pinctrl_validate_id(ph, selector, type); + if (ret) + return ret; + + iter = ph->hops->iter_response_init(ph, &ops, size, + PINCTRL_LIST_ASSOCIATIONS, + sizeof(struct scmi_msg_pinctrl_list_assoc), + &ipriv); + if (IS_ERR(iter)) + return PTR_ERR(iter); + + return ph->hops->iter_response_run(iter); +} + +struct scmi_settings_get_ipriv { + u32 selector; + enum scmi_pinctrl_selector_type type; + bool get_all; + unsigned int *nr_configs; + enum scmi_pinctrl_conf_type *config_types; + u32 *config_values; +}; + +static void +iter_pinctrl_settings_get_prepare_message(void *message, u32 desc_index, + const void *priv) +{ + struct scmi_msg_settings_get *msg = message; + const struct scmi_settings_get_ipriv *p = priv; + u32 attributes; + + attributes = FIELD_PREP(SELECTOR_MASK, p->type); + + if (p->get_all) { + attributes |= FIELD_PREP(CONFIG_FLAG_MASK, 1) | + FIELD_PREP(SKIP_CONFIGS_MASK, desc_index); + } else { + attributes |= FIELD_PREP(CONFIG_TYPE_MASK, p->config_types[0]); + } + + msg->attributes = cpu_to_le32(attributes); + msg->identifier = cpu_to_le32(p->selector); +} + +static int +iter_pinctrl_settings_get_update_state(struct scmi_iterator_state *st, + const void *response, void *priv) +{ + const struct scmi_resp_settings_get *r = response; + struct scmi_settings_get_ipriv *p = priv; + + if (p->get_all) { + st->num_returned = le32_get_bits(r->num_configs, GENMASK(7, 0)); + st->num_remaining = le32_get_bits(r->num_configs, GENMASK(31, 24)); + } else { + st->num_returned = 1; + st->num_remaining = 0; + } + + return 0; +} + +static int +iter_pinctrl_settings_get_process_response(const struct scmi_protocol_handle *ph, + const void *response, + struct scmi_iterator_state *st, + void *priv) +{ + const struct scmi_resp_settings_get *r = response; + struct scmi_settings_get_ipriv *p = priv; + u32 type = le32_get_bits(r->configs[st->loop_idx * 2], GENMASK(7, 0)); + u32 val = le32_to_cpu(r->configs[st->loop_idx * 2 + 1]); + + if (p->get_all) { + p->config_types[st->desc_index + st->loop_idx] = type; + } else { + if (p->config_types[0] != type) + return -EINVAL; + } + + p->config_values[st->desc_index + st->loop_idx] = val; + ++*p->nr_configs; + + return 0; +} + +static int +scmi_pinctrl_settings_get(const struct scmi_protocol_handle *ph, u32 selector, + enum scmi_pinctrl_selector_type type, + unsigned int *nr_configs, + enum scmi_pinctrl_conf_type *config_types, + u32 *config_values) +{ + int ret; + void *iter; + unsigned int max_configs = *nr_configs; + struct scmi_iterator_ops ops = { + .prepare_message = iter_pinctrl_settings_get_prepare_message, + .update_state = iter_pinctrl_settings_get_update_state, + .process_response = iter_pinctrl_settings_get_process_response, + }; + struct scmi_settings_get_ipriv ipriv = { + .selector = selector, + .type = type, + .get_all = (max_configs > 1), + .nr_configs = nr_configs, + .config_types = config_types, + .config_values = config_values, + }; + + if (!config_types || !config_values || type == FUNCTION_TYPE) + return -EINVAL; + + ret = scmi_pinctrl_validate_id(ph, selector, type); + if (ret) + return ret; + + /* Prepare to count returned configs */ + *nr_configs = 0; + iter = ph->hops->iter_response_init(ph, &ops, max_configs, + PINCTRL_SETTINGS_GET, + sizeof(struct scmi_msg_settings_get), + &ipriv); + if (IS_ERR(iter)) + return PTR_ERR(iter); + + return ph->hops->iter_response_run(iter); +} + +static int scmi_pinctrl_settings_get_one(const struct scmi_protocol_handle *ph, + u32 selector, + enum scmi_pinctrl_selector_type type, + enum scmi_pinctrl_conf_type config_type, + u32 *config_value) +{ + unsigned int nr_configs = 1; + + return scmi_pinctrl_settings_get(ph, selector, type, &nr_configs, + &config_type, config_value); +} + +static int scmi_pinctrl_settings_get_all(const struct scmi_protocol_handle *ph, + u32 selector, + enum scmi_pinctrl_selector_type type, + unsigned int *nr_configs, + enum scmi_pinctrl_conf_type *config_types, + u32 *config_values) +{ + if (!nr_configs || *nr_configs == 0) + return -EINVAL; + + return scmi_pinctrl_settings_get(ph, selector, type, nr_configs, + config_types, config_values); +} + +static int +scmi_pinctrl_settings_conf(const struct scmi_protocol_handle *ph, + u32 selector, + enum scmi_pinctrl_selector_type type, + u32 nr_configs, + enum scmi_pinctrl_conf_type *config_type, + u32 *config_value) +{ + struct scmi_xfer *t; + struct scmi_msg_settings_conf *tx; + u32 attributes; + int ret, i; + u32 configs_in_chunk, conf_num = 0; + u32 chunk; + int max_msg_size = ph->hops->get_max_msg_size(ph); + + if (!config_type || !config_value || type == FUNCTION_TYPE) + return -EINVAL; + + ret = scmi_pinctrl_validate_id(ph, selector, type); + if (ret) + return ret; + + configs_in_chunk = (max_msg_size - sizeof(*tx)) / (sizeof(__le32) * 2); + while (conf_num < nr_configs) { + chunk = (nr_configs - conf_num > configs_in_chunk) ? + configs_in_chunk : nr_configs - conf_num; + + ret = ph->xops->xfer_get_init(ph, PINCTRL_SETTINGS_CONFIGURE, + sizeof(*tx) + + chunk * 2 * sizeof(__le32), 0, &t); + if (ret) + break; + + tx = t->tx.buf; + tx->identifier = cpu_to_le32(selector); + tx->function_id = cpu_to_le32(0xFFFFFFFF); + attributes = FIELD_PREP(GENMASK(1, 0), type) | + FIELD_PREP(GENMASK(9, 2), chunk); + tx->attributes = cpu_to_le32(attributes); + + for (i = 0; i < chunk; i++) { + tx->configs[i * 2] = + cpu_to_le32(config_type[conf_num + i]); + tx->configs[i * 2 + 1] = + cpu_to_le32(config_value[conf_num + i]); + } + + ret = ph->xops->do_xfer(ph, t); + + ph->xops->xfer_put(ph, t); + + if (ret) + break; + + conf_num += chunk; + } + + return ret; +} + +static int scmi_pinctrl_function_select(const struct scmi_protocol_handle *ph, + u32 group, + enum scmi_pinctrl_selector_type type, + u32 function_id) +{ + int ret; + struct scmi_xfer *t; + struct scmi_msg_settings_conf *tx; + u32 attributes; + + ret = scmi_pinctrl_validate_id(ph, group, type); + if (ret) + return ret; + + ret = ph->xops->xfer_get_init(ph, PINCTRL_SETTINGS_CONFIGURE, + sizeof(*tx), 0, &t); + if (ret) + return ret; + + tx = t->tx.buf; + tx->identifier = cpu_to_le32(group); + tx->function_id = cpu_to_le32(function_id); + attributes = FIELD_PREP(GENMASK(1, 0), type) | BIT(10); + tx->attributes = cpu_to_le32(attributes); + + ret = ph->xops->do_xfer(ph, t); + ph->xops->xfer_put(ph, t); + + return ret; +} + +static int scmi_pinctrl_request_free(const struct scmi_protocol_handle *ph, + u32 identifier, + enum scmi_pinctrl_selector_type type, + enum scmi_pinctrl_protocol_cmd cmd) +{ + int ret; + struct scmi_xfer *t; + struct scmi_msg_request *tx; + + if (type == FUNCTION_TYPE) + return -EINVAL; + + if (cmd != PINCTRL_REQUEST && cmd != PINCTRL_RELEASE) + return -EINVAL; + + ret = scmi_pinctrl_validate_id(ph, identifier, type); + if (ret) + return ret; + + ret = ph->xops->xfer_get_init(ph, cmd, sizeof(*tx), 0, &t); + if (ret) + return ret; + + tx = t->tx.buf; + tx->identifier = cpu_to_le32(identifier); + tx->flags = cpu_to_le32(type); + + ret = ph->xops->do_xfer(ph, t); + ph->xops->xfer_put(ph, t); + + return ret; +} + +static int scmi_pinctrl_pin_request(const struct scmi_protocol_handle *ph, + u32 pin) +{ + return scmi_pinctrl_request_free(ph, pin, PIN_TYPE, PINCTRL_REQUEST); +} + +static int scmi_pinctrl_pin_free(const struct scmi_protocol_handle *ph, u32 pin) +{ + return scmi_pinctrl_request_free(ph, pin, PIN_TYPE, PINCTRL_RELEASE); +} + +static int scmi_pinctrl_get_group_info(const struct scmi_protocol_handle *ph, + u32 selector, + struct scmi_group_info *group) +{ + int ret; + + ret = scmi_pinctrl_attributes(ph, GROUP_TYPE, selector, group->name, + &group->nr_pins); + if (ret) + return ret; + + if (!group->nr_pins) { + dev_err(ph->dev, "Group %d has 0 elements", selector); + return -ENODATA; + } + + group->group_pins = kmalloc_array(group->nr_pins, + sizeof(*group->group_pins), + GFP_KERNEL); + if (!group->group_pins) + return -ENOMEM; + + ret = scmi_pinctrl_list_associations(ph, selector, GROUP_TYPE, + group->nr_pins, group->group_pins); + if (ret) { + kfree(group->group_pins); + return ret; + } + + group->present = true; + return 0; +} + +static int scmi_pinctrl_get_group_name(const struct scmi_protocol_handle *ph, + u32 selector, const char **name) +{ + struct scmi_pinctrl_info *pi = ph->get_priv(ph); + + if (!name) + return -EINVAL; + + if (selector >= pi->nr_groups || pi->nr_groups == 0) + return -EINVAL; + + if (!pi->groups[selector].present) { + int ret; + + ret = scmi_pinctrl_get_group_info(ph, selector, + &pi->groups[selector]); + if (ret) + return ret; + } + + *name = pi->groups[selector].name; + + return 0; +} + +static int scmi_pinctrl_group_pins_get(const struct scmi_protocol_handle *ph, + u32 selector, const u32 **pins, + u32 *nr_pins) +{ + struct scmi_pinctrl_info *pi = ph->get_priv(ph); + + if (!pins || !nr_pins) + return -EINVAL; + + if (selector >= pi->nr_groups || pi->nr_groups == 0) + return -EINVAL; + + if (!pi->groups[selector].present) { + int ret; + + ret = scmi_pinctrl_get_group_info(ph, selector, + &pi->groups[selector]); + if (ret) + return ret; + } + + *pins = pi->groups[selector].group_pins; + *nr_pins = pi->groups[selector].nr_pins; + + return 0; +} + +static int scmi_pinctrl_get_function_info(const struct scmi_protocol_handle *ph, + u32 selector, + struct scmi_function_info *func) +{ + int ret; + + ret = scmi_pinctrl_attributes(ph, FUNCTION_TYPE, selector, func->name, + &func->nr_groups); + if (ret) + return ret; + + if (!func->nr_groups) { + dev_err(ph->dev, "Function %d has 0 elements", selector); + return -ENODATA; + } + + func->groups = kmalloc_array(func->nr_groups, sizeof(*func->groups), + GFP_KERNEL); + if (!func->groups) + return -ENOMEM; + + ret = scmi_pinctrl_list_associations(ph, selector, FUNCTION_TYPE, + func->nr_groups, func->groups); + if (ret) { + kfree(func->groups); + return ret; + } + + func->present = true; + return 0; +} + +static int scmi_pinctrl_get_function_name(const struct scmi_protocol_handle *ph, + u32 selector, const char **name) +{ + struct scmi_pinctrl_info *pi = ph->get_priv(ph); + + if (!name) + return -EINVAL; + + if (selector >= pi->nr_functions || pi->nr_functions == 0) + return -EINVAL; + + if (!pi->functions[selector].present) { + int ret; + + ret = scmi_pinctrl_get_function_info(ph, selector, + &pi->functions[selector]); + if (ret) + return ret; + } + + *name = pi->functions[selector].name; + return 0; +} + +static int +scmi_pinctrl_function_groups_get(const struct scmi_protocol_handle *ph, + u32 selector, u32 *nr_groups, + const u32 **groups) +{ + struct scmi_pinctrl_info *pi = ph->get_priv(ph); + + if (!groups || !nr_groups) + return -EINVAL; + + if (selector >= pi->nr_functions || pi->nr_functions == 0) + return -EINVAL; + + if (!pi->functions[selector].present) { + int ret; + + ret = scmi_pinctrl_get_function_info(ph, selector, + &pi->functions[selector]); + if (ret) + return ret; + } + + *groups = pi->functions[selector].groups; + *nr_groups = pi->functions[selector].nr_groups; + + return 0; +} + +static int scmi_pinctrl_mux_set(const struct scmi_protocol_handle *ph, + u32 selector, u32 group) +{ + return scmi_pinctrl_function_select(ph, group, GROUP_TYPE, selector); +} + +static int scmi_pinctrl_get_pin_info(const struct scmi_protocol_handle *ph, + u32 selector, struct scmi_pin_info *pin) +{ + int ret; + + if (!pin) + return -EINVAL; + + ret = scmi_pinctrl_attributes(ph, PIN_TYPE, selector, pin->name, NULL); + if (ret) + return ret; + + pin->present = true; + return 0; +} + +static int scmi_pinctrl_get_pin_name(const struct scmi_protocol_handle *ph, + u32 selector, const char **name) +{ + struct scmi_pinctrl_info *pi = ph->get_priv(ph); + + if (!name) + return -EINVAL; + + if (selector >= pi->nr_pins) + return -EINVAL; + + if (!pi->pins[selector].present) { + int ret; + + ret = scmi_pinctrl_get_pin_info(ph, selector, &pi->pins[selector]); + if (ret) + return ret; + } + + *name = pi->pins[selector].name; + + return 0; +} + +static int scmi_pinctrl_name_get(const struct scmi_protocol_handle *ph, + u32 selector, + enum scmi_pinctrl_selector_type type, + const char **name) +{ + switch (type) { + case PIN_TYPE: + return scmi_pinctrl_get_pin_name(ph, selector, name); + case GROUP_TYPE: + return scmi_pinctrl_get_group_name(ph, selector, name); + case FUNCTION_TYPE: + return scmi_pinctrl_get_function_name(ph, selector, name); + default: + return -EINVAL; + } +} + +static const struct scmi_pinctrl_proto_ops pinctrl_proto_ops = { + .count_get = scmi_pinctrl_count_get, + .name_get = scmi_pinctrl_name_get, + .group_pins_get = scmi_pinctrl_group_pins_get, + .function_groups_get = scmi_pinctrl_function_groups_get, + .mux_set = scmi_pinctrl_mux_set, + .settings_get_one = scmi_pinctrl_settings_get_one, + .settings_get_all = scmi_pinctrl_settings_get_all, + .settings_conf = scmi_pinctrl_settings_conf, + .pin_request = scmi_pinctrl_pin_request, + .pin_free = scmi_pinctrl_pin_free, +}; + +static int scmi_pinctrl_protocol_init(const struct scmi_protocol_handle *ph) +{ + int ret; + u32 version; + struct scmi_pinctrl_info *pinfo; + + ret = ph->xops->version_get(ph, &version); + if (ret) + return ret; + + dev_dbg(ph->dev, "Pinctrl Version %d.%d\n", + PROTOCOL_REV_MAJOR(version), PROTOCOL_REV_MINOR(version)); + + pinfo = devm_kzalloc(ph->dev, sizeof(*pinfo), GFP_KERNEL); + if (!pinfo) + return -ENOMEM; + + ret = scmi_pinctrl_attributes_get(ph, pinfo); + if (ret) + return ret; + + pinfo->pins = devm_kcalloc(ph->dev, pinfo->nr_pins, + sizeof(*pinfo->pins), GFP_KERNEL); + if (!pinfo->pins) + return -ENOMEM; + + pinfo->groups = devm_kcalloc(ph->dev, pinfo->nr_groups, + sizeof(*pinfo->groups), GFP_KERNEL); + if (!pinfo->groups) + return -ENOMEM; + + pinfo->functions = devm_kcalloc(ph->dev, pinfo->nr_functions, + sizeof(*pinfo->functions), GFP_KERNEL); + if (!pinfo->functions) + return -ENOMEM; + + pinfo->version = version; + + return ph->set_priv(ph, pinfo, version); +} + +static int scmi_pinctrl_protocol_deinit(const struct scmi_protocol_handle *ph) +{ + int i; + struct scmi_pinctrl_info *pi = ph->get_priv(ph); + + /* Free groups_pins allocated in scmi_pinctrl_get_group_info */ + for (i = 0; i < pi->nr_groups; i++) { + if (pi->groups[i].present) { + kfree(pi->groups[i].group_pins); + pi->groups[i].present = false; + } + } + + /* Free groups allocated in scmi_pinctrl_get_function_info */ + for (i = 0; i < pi->nr_functions; i++) { + if (pi->functions[i].present) { + kfree(pi->functions[i].groups); + pi->functions[i].present = false; + } + } + + return 0; +} + +static const struct scmi_protocol scmi_pinctrl = { + .id = SCMI_PROTOCOL_PINCTRL, + .owner = THIS_MODULE, + .instance_init = &scmi_pinctrl_protocol_init, + .instance_deinit = &scmi_pinctrl_protocol_deinit, + .ops = &pinctrl_proto_ops, + .supported_version = SCMI_PROTOCOL_SUPPORTED_VERSION, +}; +DEFINE_SCMI_PROTOCOL_REGISTER_UNREGISTER(pinctrl, scmi_pinctrl) diff --git a/drivers/firmware/arm_scmi/protocols.h b/drivers/firmware/arm_scmi/protocols.h index 317d3fb32676..8e95f53bd7b7 100644 --- a/drivers/firmware/arm_scmi/protocols.h +++ b/drivers/firmware/arm_scmi/protocols.h @@ -29,6 +29,8 @@ #define PROTOCOL_REV_MAJOR(x) ((u16)(FIELD_GET(PROTOCOL_REV_MAJOR_MASK, (x)))) #define PROTOCOL_REV_MINOR(x) ((u16)(FIELD_GET(PROTOCOL_REV_MINOR_MASK, (x)))) +#define SCMI_PROTOCOL_VENDOR_BASE 0x80 + enum scmi_common_cmd { PROTOCOL_VERSION = 0x0, PROTOCOL_ATTRIBUTES = 0x1, @@ -258,6 +260,7 @@ struct scmi_fc_info { * @fastchannel_init: A common helper used to initialize FC descriptors by * gathering FC descriptions from the SCMI platform server. * @fastchannel_db_ring: A common helper to ring a FC doorbell. + * @get_max_msg_size: A common helper to get the maximum message size. */ struct scmi_proto_helpers_ops { int (*extended_name_get)(const struct scmi_protocol_handle *ph, @@ -277,6 +280,7 @@ struct scmi_proto_helpers_ops { struct scmi_fc_db_info **p_db, u32 *rate_limit); void (*fastchannel_db_ring)(struct scmi_fc_db_info *db); + int (*get_max_msg_size)(const struct scmi_protocol_handle *ph); }; /** @@ -323,6 +327,16 @@ typedef int (*scmi_prot_init_ph_fn_t)(const struct scmi_protocol_handle *); * protocol by the agent. Each protocol implementation * in the agent is supposed to downgrade to match the * protocol version supported by the platform. + * @vendor_id: A firmware vendor string for vendor protocols matching. + * Ignored when @id identifies a standard protocol, cannot be NULL + * otherwise. + * @sub_vendor_id: A firmware sub_vendor string for vendor protocols matching. + * Ignored if NULL or when @id identifies a standard protocol. + * @impl_ver: A firmware implementation version for vendor protocols matching. + * Ignored if zero or if @id identifies a standard protocol. + * + * Note that vendor protocols matching at load time is performed by attempting + * the closest match first against the tuple (vendor, sub_vendor, impl_ver) */ struct scmi_protocol { const u8 id; @@ -332,6 +346,9 @@ struct scmi_protocol { const void *ops; const struct scmi_protocol_events *events; unsigned int supported_version; + char *vendor_id; + char *sub_vendor_id; + u32 impl_ver; }; #define DEFINE_SCMI_PROTOCOL_REGISTER_UNREGISTER(name, proto) \ @@ -353,6 +370,7 @@ void __exit scmi_##name##_unregister(void) \ DECLARE_SCMI_REGISTER_UNREGISTER(base); DECLARE_SCMI_REGISTER_UNREGISTER(clock); DECLARE_SCMI_REGISTER_UNREGISTER(perf); +DECLARE_SCMI_REGISTER_UNREGISTER(pinctrl); DECLARE_SCMI_REGISTER_UNREGISTER(power); DECLARE_SCMI_REGISTER_UNREGISTER(reset); DECLARE_SCMI_REGISTER_UNREGISTER(sensors); diff --git a/drivers/firmware/google/cbmem.c b/drivers/firmware/google/cbmem.c index c2bffdc352a3..6f810d720f4d 100644 --- a/drivers/firmware/google/cbmem.c +++ b/drivers/firmware/google/cbmem.c @@ -124,7 +124,6 @@ static struct coreboot_driver cbmem_entry_driver = { .probe = cbmem_entry_probe, .drv = { .name = "cbmem", - .owner = THIS_MODULE, .dev_groups = dev_groups, }, .id_table = cbmem_ids, diff --git a/drivers/firmware/google/coreboot_table.c b/drivers/firmware/google/coreboot_table.c index d4b6e581a6c6..fa7752f6e89b 100644 --- a/drivers/firmware/google/coreboot_table.c +++ b/drivers/firmware/google/coreboot_table.c @@ -85,13 +85,15 @@ static void coreboot_device_release(struct device *dev) kfree(device); } -int coreboot_driver_register(struct coreboot_driver *driver) +int __coreboot_driver_register(struct coreboot_driver *driver, + struct module *owner) { driver->drv.bus = &coreboot_bus_type; + driver->drv.owner = owner; return driver_register(&driver->drv); } -EXPORT_SYMBOL(coreboot_driver_register); +EXPORT_SYMBOL(__coreboot_driver_register); void coreboot_driver_unregister(struct coreboot_driver *driver) { diff --git a/drivers/firmware/google/coreboot_table.h b/drivers/firmware/google/coreboot_table.h index 86427989c57f..bb6f0f7299b4 100644 --- a/drivers/firmware/google/coreboot_table.h +++ b/drivers/firmware/google/coreboot_table.h @@ -97,8 +97,12 @@ struct coreboot_driver { const struct coreboot_device_id *id_table; }; +/* use a macro to avoid include chaining to get THIS_MODULE */ +#define coreboot_driver_register(driver) \ + __coreboot_driver_register(driver, THIS_MODULE) /* Register a driver that uses the data from a coreboot table. */ -int coreboot_driver_register(struct coreboot_driver *driver); +int __coreboot_driver_register(struct coreboot_driver *driver, + struct module *owner); /* Unregister a driver that uses the data from a coreboot table. */ void coreboot_driver_unregister(struct coreboot_driver *driver); diff --git a/drivers/firmware/microchip/mpfs-auto-update.c b/drivers/firmware/microchip/mpfs-auto-update.c index fbeeaee4ac85..835a19a7a3a0 100644 --- a/drivers/firmware/microchip/mpfs-auto-update.c +++ b/drivers/firmware/microchip/mpfs-auto-update.c @@ -206,10 +206,12 @@ static int mpfs_auto_update_verify_image(struct fw_upload *fw_uploader) if (ret | response->resp_status) { dev_warn(priv->dev, "Verification of Upgrade Image failed!\n"); ret = ret ? ret : -EBADMSG; + goto free_message; } dev_info(priv->dev, "Verification of Upgrade Image passed!\n"); +free_message: devm_kfree(priv->dev, message); free_response: devm_kfree(priv->dev, response); @@ -265,7 +267,7 @@ static int mpfs_auto_update_set_image_address(struct mpfs_auto_update_priv *priv AUTO_UPDATE_DIRECTORY_WIDTH); memset(buffer + AUTO_UPDATE_BLANK_DIRECTORY, 0x0, AUTO_UPDATE_DIRECTORY_WIDTH); - dev_info(priv->dev, "Writing the image address (%x) to the flash directory (%llx)\n", + dev_info(priv->dev, "Writing the image address (0x%x) to the flash directory (0x%llx)\n", image_address, directory_address); ret = mtd_write(priv->flash, 0x0, erase_size, &bytes_written, (u_char *)buffer); @@ -313,7 +315,7 @@ static int mpfs_auto_update_write_bitstream(struct fw_upload *fw_uploader, const erase.len = round_up(size, (size_t)priv->flash->erasesize); erase.addr = image_address; - dev_info(priv->dev, "Erasing the flash at address (%x)\n", image_address); + dev_info(priv->dev, "Erasing the flash at address (0x%x)\n", image_address); ret = mtd_erase(priv->flash, &erase); if (ret) goto out; @@ -323,7 +325,7 @@ static int mpfs_auto_update_write_bitstream(struct fw_upload *fw_uploader, const * will do all of that itself - including verifying that the bitstream * is valid. */ - dev_info(priv->dev, "Writing the image to the flash at address (%x)\n", image_address); + dev_info(priv->dev, "Writing the image to the flash at address (0x%x)\n", image_address); ret = mtd_write(priv->flash, (loff_t)image_address, size, &bytes_written, data); if (ret) goto out; diff --git a/drivers/firmware/qcom/qcom_scm.c b/drivers/firmware/qcom/qcom_scm.c index 90283f160a22..68f4df7e6c3c 100644 --- a/drivers/firmware/qcom/qcom_scm.c +++ b/drivers/firmware/qcom/qcom_scm.c @@ -4,6 +4,8 @@ */ #include <linux/arm-smccc.h> +#include <linux/bitfield.h> +#include <linux/bits.h> #include <linux/clk.h> #include <linux/completion.h> #include <linux/cpumask.h> @@ -114,6 +116,10 @@ static const u8 qcom_scm_cpu_warm_bits[QCOM_SCM_BOOT_MAX_CPUS] = { #define QCOM_SMC_WAITQ_FLAG_WAKE_ONE BIT(0) #define QCOM_SMC_WAITQ_FLAG_WAKE_ALL BIT(1) +#define QCOM_DLOAD_MASK GENMASK(5, 4) +#define QCOM_DLOAD_NODUMP 0 +#define QCOM_DLOAD_FULLDUMP 1 + static const char * const qcom_scm_convention_names[] = { [SMC_CONVENTION_UNKNOWN] = "unknown", [SMC_CONVENTION_ARM_32] = "smc arm 32", @@ -163,9 +169,6 @@ static int qcom_scm_bw_enable(void) if (!__scm->path) return 0; - if (IS_ERR(__scm->path)) - return -EINVAL; - mutex_lock(&__scm->scm_bw_lock); if (!__scm->scm_vote_count) { ret = icc_set_bw(__scm->path, 0, UINT_MAX); @@ -183,7 +186,7 @@ err_bw: static void qcom_scm_bw_disable(void) { - if (IS_ERR_OR_NULL(__scm->path)) + if (!__scm->path) return; mutex_lock(&__scm->scm_bw_lock); @@ -496,19 +499,32 @@ static int __qcom_scm_set_dload_mode(struct device *dev, bool enable) return qcom_scm_call_atomic(__scm->dev, &desc, NULL); } +static int qcom_scm_io_rmw(phys_addr_t addr, unsigned int mask, unsigned int val) +{ + unsigned int old; + unsigned int new; + int ret; + + ret = qcom_scm_io_readl(addr, &old); + if (ret) + return ret; + + new = (old & ~mask) | (val & mask); + + return qcom_scm_io_writel(addr, new); +} + static void qcom_scm_set_download_mode(bool enable) { - bool avail; + u32 val = enable ? QCOM_DLOAD_FULLDUMP : QCOM_DLOAD_NODUMP; int ret = 0; - avail = __qcom_scm_is_call_available(__scm->dev, - QCOM_SCM_SVC_BOOT, - QCOM_SCM_BOOT_SET_DLOAD_MODE); - if (avail) { + if (__scm->dload_mode_addr) { + ret = qcom_scm_io_rmw(__scm->dload_mode_addr, QCOM_DLOAD_MASK, + FIELD_PREP(QCOM_DLOAD_MASK, val)); + } else if (__qcom_scm_is_call_available(__scm->dev, QCOM_SCM_SVC_BOOT, + QCOM_SCM_BOOT_SET_DLOAD_MODE)) { ret = __qcom_scm_set_dload_mode(__scm->dev, enable); - } else if (__scm->dload_mode_addr) { - ret = qcom_scm_io_writel(__scm->dload_mode_addr, - enable ? QCOM_SCM_BOOT_SET_DLOAD_MODE : 0); } else { dev_err(__scm->dev, "No available mechanism for setting download mode\n"); @@ -557,10 +573,9 @@ int qcom_scm_pas_init_image(u32 peripheral, const void *metadata, size_t size, */ mdata_buf = dma_alloc_coherent(__scm->dev, size, &mdata_phys, GFP_KERNEL); - if (!mdata_buf) { - dev_err(__scm->dev, "Allocation of metadata buffer failed.\n"); + if (!mdata_buf) return -ENOMEM; - } + memcpy(mdata_buf, metadata, size); ret = qcom_scm_clk_enable(); @@ -569,13 +584,14 @@ int qcom_scm_pas_init_image(u32 peripheral, const void *metadata, size_t size, ret = qcom_scm_bw_enable(); if (ret) - return ret; + goto disable_clk; desc.args[1] = mdata_phys; ret = qcom_scm_call(__scm->dev, &desc, &res); - qcom_scm_bw_disable(); + +disable_clk: qcom_scm_clk_disable(); out: @@ -637,10 +653,12 @@ int qcom_scm_pas_mem_setup(u32 peripheral, phys_addr_t addr, phys_addr_t size) ret = qcom_scm_bw_enable(); if (ret) - return ret; + goto disable_clk; ret = qcom_scm_call(__scm->dev, &desc, &res); qcom_scm_bw_disable(); + +disable_clk: qcom_scm_clk_disable(); return ret ? : res.result[0]; @@ -672,10 +690,12 @@ int qcom_scm_pas_auth_and_reset(u32 peripheral) ret = qcom_scm_bw_enable(); if (ret) - return ret; + goto disable_clk; ret = qcom_scm_call(__scm->dev, &desc, &res); qcom_scm_bw_disable(); + +disable_clk: qcom_scm_clk_disable(); return ret ? : res.result[0]; @@ -706,11 +726,12 @@ int qcom_scm_pas_shutdown(u32 peripheral) ret = qcom_scm_bw_enable(); if (ret) - return ret; + goto disable_clk; ret = qcom_scm_call(__scm->dev, &desc, &res); - qcom_scm_bw_disable(); + +disable_clk: qcom_scm_clk_disable(); return ret ? : res.result[0]; @@ -1624,8 +1645,10 @@ EXPORT_SYMBOL_GPL(qcom_scm_qseecom_app_send); * We do not yet support re-entrant calls via the qseecom interface. To prevent + any potential issues with this, only allow validated machines for now. */ -static const struct of_device_id qcom_scm_qseecom_allowlist[] = { +static const struct of_device_id qcom_scm_qseecom_allowlist[] __maybe_unused = { + { .compatible = "lenovo,flex-5g" }, { .compatible = "lenovo,thinkpad-x13s", }, + { .compatible = "qcom,sc8180x-primus" }, { } }; @@ -1713,7 +1736,7 @@ static int qcom_scm_qseecom_init(struct qcom_scm *scm) */ bool qcom_scm_is_available(void) { - return !!__scm; + return !!READ_ONCE(__scm); } EXPORT_SYMBOL_GPL(qcom_scm_is_available); @@ -1744,7 +1767,7 @@ int qcom_scm_wait_for_wq_completion(u32 wq_ctx) return 0; } -static int qcom_scm_waitq_wakeup(struct qcom_scm *scm, unsigned int wq_ctx) +static int qcom_scm_waitq_wakeup(unsigned int wq_ctx) { int ret; @@ -1776,7 +1799,7 @@ static irqreturn_t qcom_scm_irq_handler(int irq, void *data) goto out; } - ret = qcom_scm_waitq_wakeup(scm, wq_ctx); + ret = qcom_scm_waitq_wakeup(wq_ctx); if (ret) goto out; } while (more_pending); @@ -1794,10 +1817,12 @@ static int qcom_scm_probe(struct platform_device *pdev) if (!scm) return -ENOMEM; + scm->dev = &pdev->dev; ret = qcom_scm_find_dload_address(&pdev->dev, &scm->dload_mode_addr); if (ret < 0) return ret; + init_completion(&scm->waitq_comp); mutex_init(&scm->scm_bw_lock); scm->path = devm_of_icc_get(&pdev->dev, NULL); @@ -1829,10 +1854,8 @@ static int qcom_scm_probe(struct platform_device *pdev) if (ret) return ret; - __scm = scm; - __scm->dev = &pdev->dev; - - init_completion(&__scm->waitq_comp); + /* Let all above stores be available after this */ + smp_store_release(&__scm, scm); irq = platform_get_irq_optional(pdev, 0); if (irq < 0) { diff --git a/drivers/firmware/raspberrypi.c b/drivers/firmware/raspberrypi.c index 322aada20f74..ac34876a97f8 100644 --- a/drivers/firmware/raspberrypi.c +++ b/drivers/firmware/raspberrypi.c @@ -9,6 +9,7 @@ #include <linux/dma-mapping.h> #include <linux/kref.h> #include <linux/mailbox_client.h> +#include <linux/mailbox_controller.h> #include <linux/module.h> #include <linux/of.h> #include <linux/of_platform.h> @@ -97,8 +98,8 @@ int rpi_firmware_property_list(struct rpi_firmware *fw, if (size & 3) return -EINVAL; - buf = dma_alloc_coherent(fw->cl.dev, PAGE_ALIGN(size), &bus_addr, - GFP_ATOMIC); + buf = dma_alloc_coherent(fw->chan->mbox->dev, PAGE_ALIGN(size), + &bus_addr, GFP_ATOMIC); if (!buf) return -ENOMEM; @@ -126,7 +127,7 @@ int rpi_firmware_property_list(struct rpi_firmware *fw, ret = -EINVAL; } - dma_free_coherent(fw->cl.dev, PAGE_ALIGN(size), buf, bus_addr); + dma_free_coherent(fw->chan->mbox->dev, PAGE_ALIGN(size), buf, bus_addr); return ret; } diff --git a/drivers/firmware/ti_sci.c b/drivers/firmware/ti_sci.c index 8b9a2556de16..160968301b1f 100644 --- a/drivers/firmware/ti_sci.c +++ b/drivers/firmware/ti_sci.c @@ -87,7 +87,6 @@ struct ti_sci_desc { * struct ti_sci_info - Structure representing a TI SCI instance * @dev: Device pointer * @desc: SoC description for this instance - * @nb: Reboot Notifier block * @d: Debugfs file entry * @debug_region: Memory region where the debug message are available * @debug_region_size: Debug region size @@ -103,7 +102,6 @@ struct ti_sci_desc { */ struct ti_sci_info { struct device *dev; - struct notifier_block nb; const struct ti_sci_desc *desc; struct dentry *d; void __iomem *debug_region; @@ -122,7 +120,6 @@ struct ti_sci_info { #define cl_to_ti_sci_info(c) container_of(c, struct ti_sci_info, cl) #define handle_to_ti_sci_info(h) container_of(h, struct ti_sci_info, handle) -#define reboot_to_ti_sci_info(n) container_of(n, struct ti_sci_info, nb) #ifdef CONFIG_DEBUG_FS @@ -3254,10 +3251,9 @@ devm_ti_sci_get_resource(const struct ti_sci_handle *handle, struct device *dev, } EXPORT_SYMBOL_GPL(devm_ti_sci_get_resource); -static int tisci_reboot_handler(struct notifier_block *nb, unsigned long mode, - void *cmd) +static int tisci_reboot_handler(struct sys_off_data *data) { - struct ti_sci_info *info = reboot_to_ti_sci_info(nb); + struct ti_sci_info *info = data->cb_data; const struct ti_sci_handle *handle = &info->handle; ti_sci_cmd_core_reboot(handle); @@ -3303,7 +3299,6 @@ static int ti_sci_probe(struct platform_device *pdev) struct mbox_client *cl; int ret = -EINVAL; int i; - int reboot = 0; u32 h_id; desc = device_get_match_data(dev); @@ -3327,8 +3322,6 @@ static int ti_sci_probe(struct platform_device *pdev) } } - reboot = of_property_read_bool(dev->of_node, - "ti,system-reboot-controller"); INIT_LIST_HEAD(&info->node); minfo = &info->minfo; @@ -3399,15 +3392,10 @@ static int ti_sci_probe(struct platform_device *pdev) ti_sci_setup_ops(info); - if (reboot) { - info->nb.notifier_call = tisci_reboot_handler; - info->nb.priority = 128; - - ret = register_restart_handler(&info->nb); - if (ret) { - dev_err(dev, "reboot registration fail(%d)\n", ret); - goto out; - } + ret = devm_register_restart_handler(dev, tisci_reboot_handler, info); + if (ret) { + dev_err(dev, "reboot registration fail(%d)\n", ret); + goto out; } dev_info(dev, "ABI: %d.%d (firmware rev 0x%04x '%s')\n", diff --git a/drivers/gpio/gpiolib-cdev.c b/drivers/gpio/gpiolib-cdev.c index d09c7d728365..9dad67ea2597 100644 --- a/drivers/gpio/gpiolib-cdev.c +++ b/drivers/gpio/gpiolib-cdev.c @@ -1193,6 +1193,8 @@ static int edge_detector_update(struct line *line, struct gpio_v2_line_config *lc, unsigned int line_idx, u64 edflags) { + u64 eflags; + int ret; u64 active_edflags = READ_ONCE(line->edflags); unsigned int debounce_period_us = gpio_v2_line_config_debounce_period(lc, line_idx); @@ -1204,6 +1206,18 @@ static int edge_detector_update(struct line *line, /* sw debounced and still will be...*/ if (debounce_period_us && READ_ONCE(line->sw_debounced)) { line_set_debounce_period(line, debounce_period_us); + /* + * ensure event fifo is initialised if edge detection + * is now enabled. + */ + eflags = edflags & GPIO_V2_LINE_EDGE_FLAGS; + if (eflags && !kfifo_initialized(&line->req->events)) { + ret = kfifo_alloc(&line->req->events, + line->req->event_buffer_size, + GFP_KERNEL); + if (ret) + return ret; + } return 0; } @@ -2351,7 +2365,7 @@ static void gpio_desc_to_lineinfo(struct gpio_desc *desc, dflags = READ_ONCE(desc->flags); - scoped_guard(srcu, &desc->srcu) { + scoped_guard(srcu, &desc->gdev->desc_srcu) { label = gpiod_get_label(desc); if (label && test_bit(FLAG_REQUESTED, &dflags)) strscpy(info->consumer, label, @@ -2799,11 +2813,11 @@ static int gpio_chrdev_release(struct inode *inode, struct file *file) struct gpio_chardev_data *cdev = file->private_data; struct gpio_device *gdev = cdev->gdev; - bitmap_free(cdev->watched_lines); blocking_notifier_chain_unregister(&gdev->device_notifier, &cdev->device_unregistered_nb); blocking_notifier_chain_unregister(&gdev->line_state_notifier, &cdev->lineinfo_changed_nb); + bitmap_free(cdev->watched_lines); gpio_device_put(gdev); kfree(cdev); diff --git a/drivers/gpio/gpiolib.c b/drivers/gpio/gpiolib.c index 94903fc1c145..fa50db0c3605 100644 --- a/drivers/gpio/gpiolib.c +++ b/drivers/gpio/gpiolib.c @@ -101,6 +101,7 @@ static bool gpiolib_initialized; const char *gpiod_get_label(struct gpio_desc *desc) { + struct gpio_desc_label *label; unsigned long flags; flags = READ_ONCE(desc->flags); @@ -108,23 +109,36 @@ const char *gpiod_get_label(struct gpio_desc *desc) !test_bit(FLAG_REQUESTED, &flags)) return "interrupt"; - return test_bit(FLAG_REQUESTED, &flags) ? - srcu_dereference(desc->label, &desc->srcu) : NULL; + if (!test_bit(FLAG_REQUESTED, &flags)) + return NULL; + + label = srcu_dereference_check(desc->label, &desc->gdev->desc_srcu, + srcu_read_lock_held(&desc->gdev->desc_srcu)); + + return label->str; +} + +static void desc_free_label(struct rcu_head *rh) +{ + kfree(container_of(rh, struct gpio_desc_label, rh)); } static int desc_set_label(struct gpio_desc *desc, const char *label) { - const char *new = NULL, *old; + struct gpio_desc_label *new = NULL, *old; if (label) { - new = kstrdup_const(label, GFP_KERNEL); + new = kzalloc(struct_size(new, str, strlen(label) + 1), + GFP_KERNEL); if (!new) return -ENOMEM; + + strcpy(new->str, label); } old = rcu_replace_pointer(desc->label, new, 1); - synchronize_srcu(&desc->srcu); - kfree_const(old); + if (old) + call_srcu(&desc->gdev->desc_srcu, &old->rh, desc_free_label); return 0; } @@ -695,10 +709,10 @@ EXPORT_SYMBOL_GPL(gpiochip_line_is_valid); static void gpiodev_release(struct device *dev) { struct gpio_device *gdev = to_gpio_device(dev); - unsigned int i; - for (i = 0; i < gdev->ngpio; i++) - cleanup_srcu_struct(&gdev->descs[i].srcu); + /* Call pending kfree()s for descriptor labels. */ + synchronize_srcu(&gdev->desc_srcu); + cleanup_srcu_struct(&gdev->desc_srcu); ida_free(&gpio_ida, gdev->id); kfree_const(gdev->label); @@ -975,6 +989,10 @@ int gpiochip_add_data_with_key(struct gpio_chip *gc, void *data, if (ret) goto err_remove_from_list; + ret = init_srcu_struct(&gdev->desc_srcu); + if (ret) + goto err_cleanup_gdev_srcu; + #ifdef CONFIG_PINCTRL INIT_LIST_HEAD(&gdev->pin_ranges); #endif @@ -982,23 +1000,19 @@ int gpiochip_add_data_with_key(struct gpio_chip *gc, void *data, if (gc->names) { ret = gpiochip_set_desc_names(gc); if (ret) - goto err_cleanup_gdev_srcu; + goto err_cleanup_desc_srcu; } ret = gpiochip_set_names(gc); if (ret) - goto err_cleanup_gdev_srcu; + goto err_cleanup_desc_srcu; ret = gpiochip_init_valid_mask(gc); if (ret) - goto err_cleanup_gdev_srcu; + goto err_cleanup_desc_srcu; for (desc_index = 0; desc_index < gc->ngpio; desc_index++) { struct gpio_desc *desc = &gdev->descs[desc_index]; - ret = init_srcu_struct(&desc->srcu); - if (ret) - goto err_cleanup_desc_srcu; - if (gc->get_direction && gpiochip_line_is_valid(gc, desc_index)) { assign_bit(FLAG_IS_OUT, &desc->flags, !gc->get_direction(gc, desc_index)); @@ -1010,7 +1024,7 @@ int gpiochip_add_data_with_key(struct gpio_chip *gc, void *data, ret = of_gpiochip_add(gc); if (ret) - goto err_cleanup_desc_srcu; + goto err_free_valid_mask; ret = gpiochip_add_pin_ranges(gc); if (ret) @@ -1057,10 +1071,10 @@ err_free_hogs: gpiochip_remove_pin_ranges(gc); err_remove_of_chip: of_gpiochip_remove(gc); -err_cleanup_desc_srcu: - while (desc_index--) - cleanup_srcu_struct(&gdev->descs[desc_index].srcu); +err_free_valid_mask: gpiochip_free_valid_mask(gc); +err_cleanup_desc_srcu: + cleanup_srcu_struct(&gdev->desc_srcu); err_cleanup_gdev_srcu: cleanup_srcu_struct(&gdev->srcu); err_remove_from_list: @@ -2390,7 +2404,7 @@ char *gpiochip_dup_line_label(struct gpio_chip *gc, unsigned int offset) if (!test_bit(FLAG_REQUESTED, &desc->flags)) return NULL; - guard(srcu)(&desc->srcu); + guard(srcu)(&desc->gdev->desc_srcu); label = kstrdup(gpiod_get_label(desc), GFP_KERNEL); if (!label) @@ -4781,7 +4795,7 @@ static void gpiolib_dbg_show(struct seq_file *s, struct gpio_device *gdev) } for_each_gpio_desc(gc, desc) { - guard(srcu)(&desc->srcu); + guard(srcu)(&desc->gdev->desc_srcu); if (test_bit(FLAG_REQUESTED, &desc->flags)) { gpiod_get_direction(desc); is_out = test_bit(FLAG_IS_OUT, &desc->flags); diff --git a/drivers/gpio/gpiolib.h b/drivers/gpio/gpiolib.h index f67d5991ab1c..8e0e211ebf08 100644 --- a/drivers/gpio/gpiolib.h +++ b/drivers/gpio/gpiolib.h @@ -31,6 +31,7 @@ * @chip: pointer to the corresponding gpiochip, holding static * data for this device * @descs: array of ngpio descriptors. + * @desc_srcu: ensures consistent state of GPIO descriptors exposed to users * @ngpio: the number of GPIO lines on this GPIO device, equal to the size * of the @descs array. * @can_sleep: indicate whether the GPIO chip driver's callbacks can sleep @@ -61,6 +62,7 @@ struct gpio_device { struct module *owner; struct gpio_chip __rcu *chip; struct gpio_desc *descs; + struct srcu_struct desc_srcu; int base; u16 ngpio; bool can_sleep; @@ -137,6 +139,11 @@ int gpiod_set_transitory(struct gpio_desc *desc, bool transitory); void gpiod_line_state_notify(struct gpio_desc *desc, unsigned long action); +struct gpio_desc_label { + struct rcu_head rh; + char str[]; +}; + /** * struct gpio_desc - Opaque descriptor for a GPIO * @@ -145,7 +152,6 @@ void gpiod_line_state_notify(struct gpio_desc *desc, unsigned long action); * @label: Name of the consumer * @name: Line name * @hog: Pointer to the device node that hogs this line (if any) - * @srcu: SRCU struct protecting the label pointer. * * These are obtained using gpiod_get() and are preferable to the old * integer-based handles. @@ -177,13 +183,12 @@ struct gpio_desc { #define FLAG_EVENT_CLOCK_HTE 19 /* GPIO CDEV reports hardware timestamps in events */ /* Connection label */ - const char __rcu *label; + struct gpio_desc_label __rcu *label; /* Name of the GPIO */ const char *name; #ifdef CONFIG_OF_DYNAMIC struct device_node *hog; #endif - struct srcu_struct srcu; }; #define gpiod_not_found(desc) (IS_ERR(desc) && PTR_ERR(desc) == -ENOENT) @@ -251,7 +256,7 @@ static inline int gpio_chip_hwgpio(const struct gpio_desc *desc) #define gpiod_err(desc, fmt, ...) \ do { \ - scoped_guard(srcu, &desc->srcu) { \ + scoped_guard(srcu, &desc->gdev->desc_srcu) { \ pr_err("gpio-%d (%s): " fmt, desc_to_gpio(desc), \ gpiod_get_label(desc) ? : "?", ##__VA_ARGS__); \ } \ @@ -259,7 +264,7 @@ do { \ #define gpiod_warn(desc, fmt, ...) \ do { \ - scoped_guard(srcu, &desc->srcu) { \ + scoped_guard(srcu, &desc->gdev->desc_srcu) { \ pr_warn("gpio-%d (%s): " fmt, desc_to_gpio(desc), \ gpiod_get_label(desc) ? : "?", ##__VA_ARGS__); \ } \ @@ -267,7 +272,7 @@ do { \ #define gpiod_dbg(desc, fmt, ...) \ do { \ - scoped_guard(srcu, &desc->srcu) { \ + scoped_guard(srcu, &desc->gdev->desc_srcu) { \ pr_debug("gpio-%d (%s): " fmt, desc_to_gpio(desc), \ gpiod_get_label(desc) ? : "?", ##__VA_ARGS__); \ } \ diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c index 109fe557a02b..29c197c00018 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c @@ -427,7 +427,7 @@ bool amdgpu_res_cpu_visible(struct amdgpu_device *adev, amdgpu_res_first(res, 0, res->size, &cursor); while (cursor.remaining) { - if ((cursor.start + cursor.size) >= adev->gmc.visible_vram_size) + if ((cursor.start + cursor.size) > adev->gmc.visible_vram_size) return false; amdgpu_res_next(&cursor, cursor.size); } diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c b/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c index 55aa74cbc532..1e6cc0bfc432 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c @@ -1139,7 +1139,7 @@ static int kfd_ioctl_alloc_memory_of_gpu(struct file *filep, goto err_unlock; } offset = dev->adev->rmmio_remap.bus_addr; - if (!offset) { + if (!offset || (PAGE_SIZE > 4096)) { err = -ENOMEM; goto err_unlock; } @@ -2307,7 +2307,7 @@ static int criu_restore_memory_of_gpu(struct kfd_process_device *pdd, return -EINVAL; } offset = pdd->dev->adev->rmmio_remap.bus_addr; - if (!offset) { + if (!offset || (PAGE_SIZE > 4096)) { pr_err("amdgpu_amdkfd_get_mmio_remap_phys_addr failed\n"); return -ENOMEM; } @@ -3349,6 +3349,9 @@ static int kfd_mmio_mmap(struct kfd_node *dev, struct kfd_process *process, if (vma->vm_end - vma->vm_start != PAGE_SIZE) return -EINVAL; + if (PAGE_SIZE > 4096) + return -EINVAL; + address = dev->adev->rmmio_remap.bus_addr; vm_flags_set(vma, VM_IO | VM_DONTCOPY | VM_DONTEXPAND | VM_NORESERVE | diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_topology.c b/drivers/gpu/drm/amd/amdkfd/kfd_topology.c index c51f131eaa2f..bc9eb847ecfe 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_topology.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_topology.c @@ -1997,9 +1997,8 @@ int kfd_topology_add_device(struct kfd_node *gpu) HSA_CAP_ASIC_REVISION_MASK); dev->node_props.location_id = pci_dev_id(gpu->adev->pdev); - /* On multi-partition nodes, node id = location_id[31:28] */ - if (gpu->kfd->num_nodes > 1) - dev->node_props.location_id |= (dev->gpu->node_id << 28); + if (KFD_GC_VERSION(dev->gpu->kfd) == IP_VERSION(9, 4, 3)) + dev->node_props.location_id |= dev->gpu->node_id; dev->node_props.domain = pci_domain_nr(gpu->adev->pdev->bus); dev->node_props.max_engine_clk_fcompute = diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_mst_types.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_mst_types.c index 941e96f100f4..cb31a699c662 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_mst_types.c +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_mst_types.c @@ -1219,8 +1219,10 @@ static bool is_dsc_need_re_compute( if (dc_link->type != dc_connection_mst_branch) return false; - if (!(dc_link->dpcd_caps.dsc_caps.dsc_basic_caps.fields.dsc_support.DSC_SUPPORT || - dc_link->dpcd_caps.dsc_caps.dsc_basic_caps.fields.dsc_support.DSC_PASSTHROUGH_SUPPORT)) + /* add a check for older MST DSC with no virtual DPCDs */ + if (needs_dsc_aux_workaround(dc_link) && + (!(dc_link->dpcd_caps.dsc_caps.dsc_basic_caps.fields.dsc_support.DSC_SUPPORT || + dc_link->dpcd_caps.dsc_caps.dsc_basic_caps.fields.dsc_support.DSC_PASSTHROUGH_SUPPORT))) return false; for (i = 0; i < MAX_PIPES; i++) @@ -1240,7 +1242,15 @@ static bool is_dsc_need_re_compute( continue; aconnector = (struct amdgpu_dm_connector *) stream->dm_stream_context; - if (!aconnector) + if (!aconnector || !aconnector->dsc_aux) + continue; + + /* + * check if cached virtual MST DSC caps are available and DSC is supported + * as per specifications in their Virtual DPCD registers. + */ + if (!(aconnector->dc_sink->dsc_caps.dsc_dec_caps.is_dsc_supported || + aconnector->dc_link->dpcd_caps.dsc_caps.dsc_basic_caps.fields.dsc_support.DSC_PASSTHROUGH_SUPPORT)) continue; stream_on_link[new_stream_on_link_num] = aconnector; diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn35/dcn35_fpu.c b/drivers/gpu/drm/amd/display/dc/dml/dcn35/dcn35_fpu.c index 21e0eef3269b..53e40d3c48d4 100644 --- a/drivers/gpu/drm/amd/display/dc/dml/dcn35/dcn35_fpu.c +++ b/drivers/gpu/drm/amd/display/dc/dml/dcn35/dcn35_fpu.c @@ -195,9 +195,9 @@ struct _vcs_dpi_soc_bounding_box_st dcn3_5_soc = { .dcn_downspread_percent = 0.5, .gpuvm_min_page_size_bytes = 4096, .hostvm_min_page_size_bytes = 4096, - .do_urgent_latency_adjustment = 0, + .do_urgent_latency_adjustment = 1, .urgent_latency_adjustment_fabric_clock_component_us = 0, - .urgent_latency_adjustment_fabric_clock_reference_mhz = 0, + .urgent_latency_adjustment_fabric_clock_reference_mhz = 3000, }; void dcn35_build_wm_range_table_fpu(struct clk_mgr *clk_mgr) diff --git a/drivers/gpu/drm/amd/display/dc/hwss/dcn35/dcn35_hwseq.c b/drivers/gpu/drm/amd/display/dc/hwss/dcn35/dcn35_hwseq.c index a5560b3fc39b..9067ca78f851 100644 --- a/drivers/gpu/drm/amd/display/dc/hwss/dcn35/dcn35_hwseq.c +++ b/drivers/gpu/drm/amd/display/dc/hwss/dcn35/dcn35_hwseq.c @@ -638,22 +638,43 @@ void dcn35_power_down_on_boot(struct dc *dc) bool dcn35_apply_idle_power_optimizations(struct dc *dc, bool enable) { - struct dc_link *edp_links[MAX_NUM_EDP]; - int i, edp_num; if (dc->debug.dmcub_emulation) return true; if (enable) { - dc_get_edp_links(dc, edp_links, &edp_num); - if (edp_num == 0 || edp_num > 1) - return false; + uint32_t num_active_edp = 0; + int i; for (i = 0; i < dc->current_state->stream_count; ++i) { struct dc_stream_state *stream = dc->current_state->streams[i]; + struct dc_link *link = stream->link; + bool is_psr = link && !link->panel_config.psr.disable_psr && + (link->psr_settings.psr_version == DC_PSR_VERSION_1 || + link->psr_settings.psr_version == DC_PSR_VERSION_SU_1); + bool is_replay = link && link->replay_settings.replay_feature_enabled; + + /* Ignore streams that disabled. */ + if (stream->dpms_off) + continue; + + /* Active external displays block idle optimizations. */ + if (!dc_is_embedded_signal(stream->signal)) + return false; + + /* If not PWRSEQ0 can't enter idle optimizations */ + if (link && link->link_index != 0) + return false; - if (!stream->dpms_off && !dc_is_embedded_signal(stream->signal)) + /* Check for panel power features required for idle optimizations. */ + if (!is_psr && !is_replay) return false; + + num_active_edp += 1; } + + /* If more than one active eDP then disallow. */ + if (num_active_edp > 1) + return false; } // TODO: review other cases when idle optimization is allowed diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_4_ppt.c b/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_4_ppt.c index 949131bd1ecb..4abfcd32747d 100644 --- a/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_4_ppt.c +++ b/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_4_ppt.c @@ -226,7 +226,7 @@ static int smu_v13_0_4_system_features_control(struct smu_context *smu, bool en) struct amdgpu_device *adev = smu->adev; int ret = 0; - if (!en && !adev->in_s0ix) { + if (!en && adev->in_s4) { /* Adds a GFX reset as workaround just before sending the * MP1_UNLOAD message to prevent GC/RLC/PMFW from entering * an invalid state. diff --git a/drivers/gpu/drm/drm_connector.c b/drivers/gpu/drm/drm_connector.c index b0516505f7ae..4d2df7f64dc5 100644 --- a/drivers/gpu/drm/drm_connector.c +++ b/drivers/gpu/drm/drm_connector.c @@ -2940,7 +2940,7 @@ int drm_mode_getconnector(struct drm_device *dev, void *data, dev->mode_config.max_width, dev->mode_config.max_height); else - drm_dbg_kms(dev, "User-space requested a forced probe on [CONNECTOR:%d:%s] but is not the DRM master, demoting to read-only probe", + drm_dbg_kms(dev, "User-space requested a forced probe on [CONNECTOR:%d:%s] but is not the DRM master, demoting to read-only probe\n", connector->base.id, connector->name); } diff --git a/drivers/gpu/drm/i915/display/intel_audio.c b/drivers/gpu/drm/i915/display/intel_audio.c index 07e0c73204f3..ed81e1466c4b 100644 --- a/drivers/gpu/drm/i915/display/intel_audio.c +++ b/drivers/gpu/drm/i915/display/intel_audio.c @@ -76,19 +76,6 @@ struct intel_audio_funcs { struct intel_crtc_state *crtc_state); }; -/* DP N/M table */ -#define LC_810M 810000 -#define LC_540M 540000 -#define LC_270M 270000 -#define LC_162M 162000 - -struct dp_aud_n_m { - int sample_rate; - int clock; - u16 m; - u16 n; -}; - struct hdmi_aud_ncts { int sample_rate; int clock; @@ -96,60 +83,6 @@ struct hdmi_aud_ncts { int cts; }; -/* Values according to DP 1.4 Table 2-104 */ -static const struct dp_aud_n_m dp_aud_n_m[] = { - { 32000, LC_162M, 1024, 10125 }, - { 44100, LC_162M, 784, 5625 }, - { 48000, LC_162M, 512, 3375 }, - { 64000, LC_162M, 2048, 10125 }, - { 88200, LC_162M, 1568, 5625 }, - { 96000, LC_162M, 1024, 3375 }, - { 128000, LC_162M, 4096, 10125 }, - { 176400, LC_162M, 3136, 5625 }, - { 192000, LC_162M, 2048, 3375 }, - { 32000, LC_270M, 1024, 16875 }, - { 44100, LC_270M, 784, 9375 }, - { 48000, LC_270M, 512, 5625 }, - { 64000, LC_270M, 2048, 16875 }, - { 88200, LC_270M, 1568, 9375 }, - { 96000, LC_270M, 1024, 5625 }, - { 128000, LC_270M, 4096, 16875 }, - { 176400, LC_270M, 3136, 9375 }, - { 192000, LC_270M, 2048, 5625 }, - { 32000, LC_540M, 1024, 33750 }, - { 44100, LC_540M, 784, 18750 }, - { 48000, LC_540M, 512, 11250 }, - { 64000, LC_540M, 2048, 33750 }, - { 88200, LC_540M, 1568, 18750 }, - { 96000, LC_540M, 1024, 11250 }, - { 128000, LC_540M, 4096, 33750 }, - { 176400, LC_540M, 3136, 18750 }, - { 192000, LC_540M, 2048, 11250 }, - { 32000, LC_810M, 1024, 50625 }, - { 44100, LC_810M, 784, 28125 }, - { 48000, LC_810M, 512, 16875 }, - { 64000, LC_810M, 2048, 50625 }, - { 88200, LC_810M, 1568, 28125 }, - { 96000, LC_810M, 1024, 16875 }, - { 128000, LC_810M, 4096, 50625 }, - { 176400, LC_810M, 3136, 28125 }, - { 192000, LC_810M, 2048, 16875 }, -}; - -static const struct dp_aud_n_m * -audio_config_dp_get_n_m(const struct intel_crtc_state *crtc_state, int rate) -{ - int i; - - for (i = 0; i < ARRAY_SIZE(dp_aud_n_m); i++) { - if (rate == dp_aud_n_m[i].sample_rate && - crtc_state->port_clock == dp_aud_n_m[i].clock) - return &dp_aud_n_m[i]; - } - - return NULL; -} - static const struct { int clock; u32 config; @@ -387,47 +320,17 @@ hsw_dp_audio_config_update(struct intel_encoder *encoder, const struct intel_crtc_state *crtc_state) { struct drm_i915_private *i915 = to_i915(encoder->base.dev); - struct i915_audio_component *acomp = i915->display.audio.component; enum transcoder cpu_transcoder = crtc_state->cpu_transcoder; - enum port port = encoder->port; - const struct dp_aud_n_m *nm; - int rate; - u32 tmp; - - rate = acomp ? acomp->aud_sample_rate[port] : 0; - nm = audio_config_dp_get_n_m(crtc_state, rate); - if (nm) - drm_dbg_kms(&i915->drm, "using Maud %u, Naud %u\n", nm->m, - nm->n); - else - drm_dbg_kms(&i915->drm, "using automatic Maud, Naud\n"); - - tmp = intel_de_read(i915, HSW_AUD_CFG(cpu_transcoder)); - tmp &= ~AUD_CONFIG_N_VALUE_INDEX; - tmp &= ~AUD_CONFIG_PIXEL_CLOCK_HDMI_MASK; - tmp &= ~AUD_CONFIG_N_PROG_ENABLE; - tmp |= AUD_CONFIG_N_VALUE_INDEX; - if (nm) { - tmp &= ~AUD_CONFIG_N_MASK; - tmp |= AUD_CONFIG_N(nm->n); - tmp |= AUD_CONFIG_N_PROG_ENABLE; - } - - intel_de_write(i915, HSW_AUD_CFG(cpu_transcoder), tmp); - - tmp = intel_de_read(i915, HSW_AUD_M_CTS_ENABLE(cpu_transcoder)); - tmp &= ~AUD_CONFIG_M_MASK; - tmp &= ~AUD_M_CTS_M_VALUE_INDEX; - tmp &= ~AUD_M_CTS_M_PROG_ENABLE; - - if (nm) { - tmp |= nm->m; - tmp |= AUD_M_CTS_M_VALUE_INDEX; - tmp |= AUD_M_CTS_M_PROG_ENABLE; - } + /* Enable time stamps. Let HW calculate Maud/Naud values */ + intel_de_rmw(i915, HSW_AUD_CFG(cpu_transcoder), + AUD_CONFIG_N_VALUE_INDEX | + AUD_CONFIG_PIXEL_CLOCK_HDMI_MASK | + AUD_CONFIG_UPPER_N_MASK | + AUD_CONFIG_LOWER_N_MASK | + AUD_CONFIG_N_PROG_ENABLE, + AUD_CONFIG_N_VALUE_INDEX); - intel_de_write(i915, HSW_AUD_M_CTS_ENABLE(cpu_transcoder), tmp); } static void diff --git a/drivers/gpu/drm/i915/display/intel_bios.c b/drivers/gpu/drm/i915/display/intel_bios.c index 52bd3576835b..7d1e443f97b9 100644 --- a/drivers/gpu/drm/i915/display/intel_bios.c +++ b/drivers/gpu/drm/i915/display/intel_bios.c @@ -1042,22 +1042,11 @@ parse_lfp_backlight(struct drm_i915_private *i915, panel->vbt.backlight.type = INTEL_BACKLIGHT_DISPLAY_DDI; panel->vbt.backlight.controller = 0; if (i915->display.vbt.version >= 191) { - size_t exp_size; + const struct lfp_backlight_control_method *method; - if (i915->display.vbt.version >= 236) - exp_size = sizeof(struct bdb_lfp_backlight_data); - else if (i915->display.vbt.version >= 234) - exp_size = EXP_BDB_LFP_BL_DATA_SIZE_REV_234; - else - exp_size = EXP_BDB_LFP_BL_DATA_SIZE_REV_191; - - if (get_blocksize(backlight_data) >= exp_size) { - const struct lfp_backlight_control_method *method; - - method = &backlight_data->backlight_control[panel_type]; - panel->vbt.backlight.type = method->type; - panel->vbt.backlight.controller = method->controller; - } + method = &backlight_data->backlight_control[panel_type]; + panel->vbt.backlight.type = method->type; + panel->vbt.backlight.controller = method->controller; } panel->vbt.backlight.pwm_freq_hz = entry->pwm_freq_hz; diff --git a/drivers/gpu/drm/i915/display/intel_vbt_defs.h b/drivers/gpu/drm/i915/display/intel_vbt_defs.h index a9f44abfc9fc..b50cd0dcabda 100644 --- a/drivers/gpu/drm/i915/display/intel_vbt_defs.h +++ b/drivers/gpu/drm/i915/display/intel_vbt_defs.h @@ -897,11 +897,6 @@ struct lfp_brightness_level { u16 reserved; } __packed; -#define EXP_BDB_LFP_BL_DATA_SIZE_REV_191 \ - offsetof(struct bdb_lfp_backlight_data, brightness_level) -#define EXP_BDB_LFP_BL_DATA_SIZE_REV_234 \ - offsetof(struct bdb_lfp_backlight_data, brightness_precision_bits) - struct bdb_lfp_backlight_data { u8 entry_size; struct lfp_backlight_data_entry data[16]; diff --git a/drivers/gpu/drm/i915/gt/intel_gt_ccs_mode.c b/drivers/gpu/drm/i915/gt/intel_gt_ccs_mode.c index 044219c5960a..99b71bb7da0a 100644 --- a/drivers/gpu/drm/i915/gt/intel_gt_ccs_mode.c +++ b/drivers/gpu/drm/i915/gt/intel_gt_ccs_mode.c @@ -8,14 +8,14 @@ #include "intel_gt_ccs_mode.h" #include "intel_gt_regs.h" -void intel_gt_apply_ccs_mode(struct intel_gt *gt) +unsigned int intel_gt_apply_ccs_mode(struct intel_gt *gt) { int cslice; u32 mode = 0; int first_ccs = __ffs(CCS_MASK(gt)); if (!IS_DG2(gt->i915)) - return; + return 0; /* Build the value for the fixed CCS load balancing */ for (cslice = 0; cslice < I915_MAX_CCS; cslice++) { @@ -35,5 +35,5 @@ void intel_gt_apply_ccs_mode(struct intel_gt *gt) XEHP_CCS_MODE_CSLICE_MASK); } - intel_uncore_write(gt->uncore, XEHP_CCS_MODE, mode); + return mode; } diff --git a/drivers/gpu/drm/i915/gt/intel_gt_ccs_mode.h b/drivers/gpu/drm/i915/gt/intel_gt_ccs_mode.h index 9e5549caeb26..55547f2ff426 100644 --- a/drivers/gpu/drm/i915/gt/intel_gt_ccs_mode.h +++ b/drivers/gpu/drm/i915/gt/intel_gt_ccs_mode.h @@ -8,6 +8,6 @@ struct intel_gt; -void intel_gt_apply_ccs_mode(struct intel_gt *gt); +unsigned int intel_gt_apply_ccs_mode(struct intel_gt *gt); #endif /* __INTEL_GT_CCS_MODE_H__ */ diff --git a/drivers/gpu/drm/i915/gt/intel_workarounds.c b/drivers/gpu/drm/i915/gt/intel_workarounds.c index 6ec3582c9735..85c860ea9d7c 100644 --- a/drivers/gpu/drm/i915/gt/intel_workarounds.c +++ b/drivers/gpu/drm/i915/gt/intel_workarounds.c @@ -2859,6 +2859,7 @@ add_render_compute_tuning_settings(struct intel_gt *gt, static void ccs_engine_wa_mode(struct intel_engine_cs *engine, struct i915_wa_list *wal) { struct intel_gt *gt = engine->gt; + u32 mode; if (!IS_DG2(gt->i915)) return; @@ -2875,7 +2876,8 @@ static void ccs_engine_wa_mode(struct intel_engine_cs *engine, struct i915_wa_li * After having disabled automatic load balancing we need to * assign all slices to a single CCS. We will call it CCS mode 1 */ - intel_gt_apply_ccs_mode(gt); + mode = intel_gt_apply_ccs_mode(gt); + wa_masked_en(wal, XEHP_CCS_MODE, mode); } /* diff --git a/drivers/gpu/drm/meson/meson_dw_hdmi.c b/drivers/gpu/drm/meson/meson_dw_hdmi.c index 5a9538bc0e26..5565f7777529 100644 --- a/drivers/gpu/drm/meson/meson_dw_hdmi.c +++ b/drivers/gpu/drm/meson/meson_dw_hdmi.c @@ -106,6 +106,8 @@ #define HHI_HDMI_CLK_CNTL 0x1cc /* 0x73 */ #define HHI_HDMI_PHY_CNTL0 0x3a0 /* 0xe8 */ #define HHI_HDMI_PHY_CNTL1 0x3a4 /* 0xe9 */ +#define PHY_CNTL1_INIT 0x03900000 +#define PHY_INVERT BIT(17) #define HHI_HDMI_PHY_CNTL2 0x3a8 /* 0xea */ #define HHI_HDMI_PHY_CNTL3 0x3ac /* 0xeb */ #define HHI_HDMI_PHY_CNTL4 0x3b0 /* 0xec */ @@ -130,6 +132,8 @@ struct meson_dw_hdmi_data { unsigned int addr); void (*dwc_write)(struct meson_dw_hdmi *dw_hdmi, unsigned int addr, unsigned int data); + u32 cntl0_init; + u32 cntl1_init; }; struct meson_dw_hdmi { @@ -384,26 +388,6 @@ static int dw_hdmi_phy_init(struct dw_hdmi *hdmi, void *data, dw_hdmi_bus_fmt_is_420(hdmi)) mode_is_420 = true; - /* Enable clocks */ - regmap_update_bits(priv->hhi, HHI_HDMI_CLK_CNTL, 0xffff, 0x100); - - /* Bring HDMITX MEM output of power down */ - regmap_update_bits(priv->hhi, HHI_MEM_PD_REG0, 0xff << 8, 0); - - /* Bring out of reset */ - dw_hdmi->data->top_write(dw_hdmi, HDMITX_TOP_SW_RESET, 0); - - /* Enable internal pixclk, tmds_clk, spdif_clk, i2s_clk, cecclk */ - dw_hdmi_top_write_bits(dw_hdmi, HDMITX_TOP_CLK_CNTL, - 0x3, 0x3); - - /* Enable cec_clk and hdcp22_tmdsclk_en */ - dw_hdmi_top_write_bits(dw_hdmi, HDMITX_TOP_CLK_CNTL, - 0x3 << 4, 0x3 << 4); - - /* Enable normal output to PHY */ - dw_hdmi->data->top_write(dw_hdmi, HDMITX_TOP_BIST_CNTL, BIT(12)); - /* TMDS pattern setup */ if (mode->clock > 340000 && !mode_is_420) { dw_hdmi->data->top_write(dw_hdmi, HDMITX_TOP_TMDS_CLK_PTTN_01, @@ -425,20 +409,6 @@ static int dw_hdmi_phy_init(struct dw_hdmi *hdmi, void *data, /* Setup PHY parameters */ meson_hdmi_phy_setup_mode(dw_hdmi, mode, mode_is_420); - /* Setup PHY */ - regmap_update_bits(priv->hhi, HHI_HDMI_PHY_CNTL1, - 0xffff << 16, 0x0390 << 16); - - /* BIT_INVERT */ - if (dw_hdmi_is_compatible(dw_hdmi, "amlogic,meson-gxl-dw-hdmi") || - dw_hdmi_is_compatible(dw_hdmi, "amlogic,meson-gxm-dw-hdmi") || - dw_hdmi_is_compatible(dw_hdmi, "amlogic,meson-g12a-dw-hdmi")) - regmap_update_bits(priv->hhi, HHI_HDMI_PHY_CNTL1, - BIT(17), 0); - else - regmap_update_bits(priv->hhi, HHI_HDMI_PHY_CNTL1, - BIT(17), BIT(17)); - /* Disable clock, fifo, fifo_wr */ regmap_update_bits(priv->hhi, HHI_HDMI_PHY_CNTL1, 0xf, 0); @@ -492,7 +462,9 @@ static void dw_hdmi_phy_disable(struct dw_hdmi *hdmi, DRM_DEBUG_DRIVER("\n"); - regmap_write(priv->hhi, HHI_HDMI_PHY_CNTL0, 0); + /* Fallback to init mode */ + regmap_write(priv->hhi, HHI_HDMI_PHY_CNTL1, dw_hdmi->data->cntl1_init); + regmap_write(priv->hhi, HHI_HDMI_PHY_CNTL0, dw_hdmi->data->cntl0_init); } static enum drm_connector_status dw_hdmi_read_hpd(struct dw_hdmi *hdmi, @@ -610,11 +582,22 @@ static const struct regmap_config meson_dw_hdmi_regmap_config = { .fast_io = true, }; -static const struct meson_dw_hdmi_data meson_dw_hdmi_gx_data = { +static const struct meson_dw_hdmi_data meson_dw_hdmi_gxbb_data = { .top_read = dw_hdmi_top_read, .top_write = dw_hdmi_top_write, .dwc_read = dw_hdmi_dwc_read, .dwc_write = dw_hdmi_dwc_write, + .cntl0_init = 0x0, + .cntl1_init = PHY_CNTL1_INIT | PHY_INVERT, +}; + +static const struct meson_dw_hdmi_data meson_dw_hdmi_gxl_data = { + .top_read = dw_hdmi_top_read, + .top_write = dw_hdmi_top_write, + .dwc_read = dw_hdmi_dwc_read, + .dwc_write = dw_hdmi_dwc_write, + .cntl0_init = 0x0, + .cntl1_init = PHY_CNTL1_INIT, }; static const struct meson_dw_hdmi_data meson_dw_hdmi_g12a_data = { @@ -622,6 +605,8 @@ static const struct meson_dw_hdmi_data meson_dw_hdmi_g12a_data = { .top_write = dw_hdmi_g12a_top_write, .dwc_read = dw_hdmi_g12a_dwc_read, .dwc_write = dw_hdmi_g12a_dwc_write, + .cntl0_init = 0x000b4242, /* Bandgap */ + .cntl1_init = PHY_CNTL1_INIT, }; static void meson_dw_hdmi_init(struct meson_dw_hdmi *meson_dw_hdmi) @@ -656,6 +641,13 @@ static void meson_dw_hdmi_init(struct meson_dw_hdmi *meson_dw_hdmi) meson_dw_hdmi->data->top_write(meson_dw_hdmi, HDMITX_TOP_CLK_CNTL, 0xff); + /* Enable normal output to PHY */ + meson_dw_hdmi->data->top_write(meson_dw_hdmi, HDMITX_TOP_BIST_CNTL, BIT(12)); + + /* Setup PHY */ + regmap_write(priv->hhi, HHI_HDMI_PHY_CNTL1, meson_dw_hdmi->data->cntl1_init); + regmap_write(priv->hhi, HHI_HDMI_PHY_CNTL0, meson_dw_hdmi->data->cntl0_init); + /* Enable HDMI-TX Interrupt */ meson_dw_hdmi->data->top_write(meson_dw_hdmi, HDMITX_TOP_INTR_STAT_CLR, HDMITX_TOP_INTR_CORE); @@ -865,11 +857,11 @@ static const struct dev_pm_ops meson_dw_hdmi_pm_ops = { static const struct of_device_id meson_dw_hdmi_of_table[] = { { .compatible = "amlogic,meson-gxbb-dw-hdmi", - .data = &meson_dw_hdmi_gx_data }, + .data = &meson_dw_hdmi_gxbb_data }, { .compatible = "amlogic,meson-gxl-dw-hdmi", - .data = &meson_dw_hdmi_gx_data }, + .data = &meson_dw_hdmi_gxl_data }, { .compatible = "amlogic,meson-gxm-dw-hdmi", - .data = &meson_dw_hdmi_gx_data }, + .data = &meson_dw_hdmi_gxl_data }, { .compatible = "amlogic,meson-g12a-dw-hdmi", .data = &meson_dw_hdmi_g12a_data }, { } diff --git a/drivers/gpu/drm/nouveau/nvkm/core/firmware.c b/drivers/gpu/drm/nouveau/nvkm/core/firmware.c index 141b0a513bf5..adc60b25f8e6 100644 --- a/drivers/gpu/drm/nouveau/nvkm/core/firmware.c +++ b/drivers/gpu/drm/nouveau/nvkm/core/firmware.c @@ -205,9 +205,7 @@ nvkm_firmware_dtor(struct nvkm_firmware *fw) break; case NVKM_FIRMWARE_IMG_DMA: nvkm_memory_unref(&memory); - dma_unmap_single(fw->device->dev, fw->phys, sg_dma_len(&fw->mem.sgl), - DMA_TO_DEVICE); - kfree(fw->img); + dma_free_coherent(fw->device->dev, sg_dma_len(&fw->mem.sgl), fw->img, fw->phys); break; case NVKM_FIRMWARE_IMG_SGT: nvkm_memory_unref(&memory); @@ -237,17 +235,14 @@ nvkm_firmware_ctor(const struct nvkm_firmware_func *func, const char *name, fw->img = kmemdup(src, fw->len, GFP_KERNEL); break; case NVKM_FIRMWARE_IMG_DMA: { - len = ALIGN(fw->len, PAGE_SIZE); + dma_addr_t addr; - fw->img = kmalloc(len, GFP_KERNEL); - if (!fw->img) - return -ENOMEM; + len = ALIGN(fw->len, PAGE_SIZE); - memcpy(fw->img, src, fw->len); - fw->phys = dma_map_single(fw->device->dev, fw->img, len, DMA_TO_DEVICE); - if (dma_mapping_error(fw->device->dev, fw->phys)) { - kfree(fw->img); - return -EFAULT; + fw->img = dma_alloc_coherent(fw->device->dev, len, &addr, GFP_KERNEL); + if (fw->img) { + memcpy(fw->img, src, fw->len); + fw->phys = addr; } sg_init_one(&fw->mem.sgl, fw->img, len); diff --git a/drivers/gpu/drm/qxl/qxl_release.c b/drivers/gpu/drm/qxl/qxl_release.c index 9febc8b73f09..368d26da0d6a 100644 --- a/drivers/gpu/drm/qxl/qxl_release.c +++ b/drivers/gpu/drm/qxl/qxl_release.c @@ -58,56 +58,16 @@ static long qxl_fence_wait(struct dma_fence *fence, bool intr, signed long timeout) { struct qxl_device *qdev; - struct qxl_release *release; - int count = 0, sc = 0; - bool have_drawable_releases; unsigned long cur, end = jiffies + timeout; qdev = container_of(fence->lock, struct qxl_device, release_lock); - release = container_of(fence, struct qxl_release, base); - have_drawable_releases = release->type == QXL_RELEASE_DRAWABLE; - -retry: - sc++; - - if (dma_fence_is_signaled(fence)) - goto signaled; - - qxl_io_notify_oom(qdev); - - for (count = 0; count < 11; count++) { - if (!qxl_queue_garbage_collect(qdev, true)) - break; - - if (dma_fence_is_signaled(fence)) - goto signaled; - } - - if (dma_fence_is_signaled(fence)) - goto signaled; - if (have_drawable_releases || sc < 4) { - if (sc > 2) - /* back off */ - usleep_range(500, 1000); - - if (time_after(jiffies, end)) - return 0; - - if (have_drawable_releases && sc > 300) { - DMA_FENCE_WARN(fence, - "failed to wait on release %llu after spincount %d\n", - fence->context & ~0xf0000000, sc); - goto signaled; - } - goto retry; - } - /* - * yeah, original sync_obj_wait gave up after 3 spins when - * have_drawable_releases is not set. - */ + if (!wait_event_timeout(qdev->release_event, + (dma_fence_is_signaled(fence) || + (qxl_io_notify_oom(qdev), 0)), + timeout)) + return 0; -signaled: cur = jiffies; if (time_after(cur, end)) return 0; diff --git a/drivers/gpu/drm/xe/xe_guc_ads.c b/drivers/gpu/drm/xe/xe_guc_ads.c index 6ad4c1a90a78..5339f8a49004 100644 --- a/drivers/gpu/drm/xe/xe_guc_ads.c +++ b/drivers/gpu/drm/xe/xe_guc_ads.c @@ -100,7 +100,7 @@ struct __guc_ads_blob { struct guc_engine_usage engine_usage; struct guc_um_init_params um_init_params; /* From here on, location is dynamic! Refer to above diagram. */ - struct guc_mmio_reg regset[0]; + struct guc_mmio_reg regset[]; } __packed; #define ads_blob_read(ads_, field_) \ diff --git a/drivers/gpu/drm/xe/xe_guc_ct.c b/drivers/gpu/drm/xe/xe_guc_ct.c index 7f32547f94b2..8bbfa45798e2 100644 --- a/drivers/gpu/drm/xe/xe_guc_ct.c +++ b/drivers/gpu/drm/xe/xe_guc_ct.c @@ -120,6 +120,7 @@ static void guc_ct_fini(struct drm_device *drm, void *arg) { struct xe_guc_ct *ct = arg; + destroy_workqueue(ct->g2h_wq); xa_destroy(&ct->fence_lookup); } @@ -145,13 +146,20 @@ int xe_guc_ct_init(struct xe_guc_ct *ct) xe_assert(xe, !(guc_ct_size() % PAGE_SIZE)); - drmm_mutex_init(&xe->drm, &ct->lock); + ct->g2h_wq = alloc_ordered_workqueue("xe-g2h-wq", 0); + if (!ct->g2h_wq) + return -ENOMEM; + spin_lock_init(&ct->fast_lock); xa_init(&ct->fence_lookup); INIT_WORK(&ct->g2h_worker, g2h_worker_func); init_waitqueue_head(&ct->wq); init_waitqueue_head(&ct->g2h_fence_wq); + err = drmm_mutex_init(&xe->drm, &ct->lock); + if (err) + return err; + primelockdep(ct); bo = xe_managed_bo_create_pin_map(xe, tile, guc_ct_size(), diff --git a/drivers/gpu/drm/xe/xe_guc_ct.h b/drivers/gpu/drm/xe/xe_guc_ct.h index 5083e099064f..105bb8e99a8d 100644 --- a/drivers/gpu/drm/xe/xe_guc_ct.h +++ b/drivers/gpu/drm/xe/xe_guc_ct.h @@ -34,7 +34,7 @@ static inline void xe_guc_ct_irq_handler(struct xe_guc_ct *ct) return; wake_up_all(&ct->wq); - queue_work(system_unbound_wq, &ct->g2h_worker); + queue_work(ct->g2h_wq, &ct->g2h_worker); xe_guc_ct_fast_path(ct); } diff --git a/drivers/gpu/drm/xe/xe_guc_ct_types.h b/drivers/gpu/drm/xe/xe_guc_ct_types.h index d29144c9f20b..fede4c6e93cb 100644 --- a/drivers/gpu/drm/xe/xe_guc_ct_types.h +++ b/drivers/gpu/drm/xe/xe_guc_ct_types.h @@ -120,6 +120,8 @@ struct xe_guc_ct { wait_queue_head_t wq; /** @g2h_fence_wq: wait queue used for G2H fencing */ wait_queue_head_t g2h_fence_wq; + /** @g2h_wq: used to process G2H */ + struct workqueue_struct *g2h_wq; /** @msg: Message buffer */ u32 msg[GUC_CTB_MSG_MAX_LEN]; /** @fast_msg: Message buffer */ diff --git a/drivers/gpu/host1x/dev.c b/drivers/gpu/host1x/dev.c index 89983d7d73ca..3a0aaa68ac8d 100644 --- a/drivers/gpu/host1x/dev.c +++ b/drivers/gpu/host1x/dev.c @@ -216,6 +216,30 @@ static const struct host1x_info host1x07_info = { */ static const struct host1x_sid_entry tegra234_sid_table[] = { { + /* SE2 MMIO */ + .base = 0x1658, + .offset = 0x90, + .limit = 0x90 + }, + { + /* SE4 MMIO */ + .base = 0x1660, + .offset = 0x90, + .limit = 0x90 + }, + { + /* SE2 channel */ + .base = 0x1738, + .offset = 0x90, + .limit = 0x90 + }, + { + /* SE4 channel */ + .base = 0x1740, + .offset = 0x90, + .limit = 0x90 + }, + { /* VIC channel */ .base = 0x17b8, .offset = 0x30, diff --git a/drivers/hwmon/corsair-cpro.c b/drivers/hwmon/corsair-cpro.c index a284a02839fb..3e63666a61bd 100644 --- a/drivers/hwmon/corsair-cpro.c +++ b/drivers/hwmon/corsair-cpro.c @@ -16,6 +16,7 @@ #include <linux/module.h> #include <linux/mutex.h> #include <linux/slab.h> +#include <linux/spinlock.h> #include <linux/types.h> #define USB_VENDOR_ID_CORSAIR 0x1b1c @@ -77,8 +78,11 @@ struct ccp_device { struct hid_device *hdev; struct device *hwmon_dev; + /* For reinitializing the completion below */ + spinlock_t wait_input_report_lock; struct completion wait_input_report; struct mutex mutex; /* whenever buffer is used, lock before send_usb_cmd */ + u8 *cmd_buffer; u8 *buffer; int target[6]; DECLARE_BITMAP(temp_cnct, NUM_TEMP_SENSORS); @@ -111,15 +115,23 @@ static int send_usb_cmd(struct ccp_device *ccp, u8 command, u8 byte1, u8 byte2, unsigned long t; int ret; - memset(ccp->buffer, 0x00, OUT_BUFFER_SIZE); - ccp->buffer[0] = command; - ccp->buffer[1] = byte1; - ccp->buffer[2] = byte2; - ccp->buffer[3] = byte3; - + memset(ccp->cmd_buffer, 0x00, OUT_BUFFER_SIZE); + ccp->cmd_buffer[0] = command; + ccp->cmd_buffer[1] = byte1; + ccp->cmd_buffer[2] = byte2; + ccp->cmd_buffer[3] = byte3; + + /* + * Disable raw event parsing for a moment to safely reinitialize the + * completion. Reinit is done because hidraw could have triggered + * the raw event parsing and marked the ccp->wait_input_report + * completion as done. + */ + spin_lock_bh(&ccp->wait_input_report_lock); reinit_completion(&ccp->wait_input_report); + spin_unlock_bh(&ccp->wait_input_report_lock); - ret = hid_hw_output_report(ccp->hdev, ccp->buffer, OUT_BUFFER_SIZE); + ret = hid_hw_output_report(ccp->hdev, ccp->cmd_buffer, OUT_BUFFER_SIZE); if (ret < 0) return ret; @@ -135,11 +147,12 @@ static int ccp_raw_event(struct hid_device *hdev, struct hid_report *report, u8 struct ccp_device *ccp = hid_get_drvdata(hdev); /* only copy buffer when requested */ - if (completion_done(&ccp->wait_input_report)) - return 0; - - memcpy(ccp->buffer, data, min(IN_BUFFER_SIZE, size)); - complete(&ccp->wait_input_report); + spin_lock(&ccp->wait_input_report_lock); + if (!completion_done(&ccp->wait_input_report)) { + memcpy(ccp->buffer, data, min(IN_BUFFER_SIZE, size)); + complete_all(&ccp->wait_input_report); + } + spin_unlock(&ccp->wait_input_report_lock); return 0; } @@ -492,7 +505,11 @@ static int ccp_probe(struct hid_device *hdev, const struct hid_device_id *id) if (!ccp) return -ENOMEM; - ccp->buffer = devm_kmalloc(&hdev->dev, OUT_BUFFER_SIZE, GFP_KERNEL); + ccp->cmd_buffer = devm_kmalloc(&hdev->dev, OUT_BUFFER_SIZE, GFP_KERNEL); + if (!ccp->cmd_buffer) + return -ENOMEM; + + ccp->buffer = devm_kmalloc(&hdev->dev, IN_BUFFER_SIZE, GFP_KERNEL); if (!ccp->buffer) return -ENOMEM; @@ -510,7 +527,9 @@ static int ccp_probe(struct hid_device *hdev, const struct hid_device_id *id) ccp->hdev = hdev; hid_set_drvdata(hdev, ccp); + mutex_init(&ccp->mutex); + spin_lock_init(&ccp->wait_input_report_lock); init_completion(&ccp->wait_input_report); hid_device_io_start(hdev); diff --git a/drivers/hwmon/pmbus/ucd9000.c b/drivers/hwmon/pmbus/ucd9000.c index 8d9d422450e5..d817c719b90b 100644 --- a/drivers/hwmon/pmbus/ucd9000.c +++ b/drivers/hwmon/pmbus/ucd9000.c @@ -80,11 +80,11 @@ struct ucd9000_debugfs_entry { * It has been observed that the UCD90320 randomly fails register access when * doing another access right on the back of a register write. To mitigate this * make sure that there is a minimum delay between a write access and the - * following access. The 250us is based on experimental data. At a delay of - * 200us the issue seems to go away. Add a bit of extra margin to allow for + * following access. The 500 is based on experimental data. At a delay of + * 350us the issue seems to go away. Add a bit of extra margin to allow for * system to system differences. */ -#define UCD90320_WAIT_DELAY_US 250 +#define UCD90320_WAIT_DELAY_US 500 static inline void ucd90320_wait(const struct ucd9000_data *data) { diff --git a/drivers/infiniband/hw/qib/qib_fs.c b/drivers/infiniband/hw/qib/qib_fs.c index 455e966eeff3..b27791029fa9 100644 --- a/drivers/infiniband/hw/qib/qib_fs.c +++ b/drivers/infiniband/hw/qib/qib_fs.c @@ -439,6 +439,7 @@ static int remove_device_files(struct super_block *sb, return PTR_ERR(dir); } simple_recursive_removal(dir, NULL); + dput(dir); return 0; } diff --git a/drivers/input/serio/i8042-io.h b/drivers/input/serio/i8042-io.h index 64590b86eb37..a8f4b2d70e59 100644 --- a/drivers/input/serio/i8042-io.h +++ b/drivers/input/serio/i8042-io.h @@ -15,10 +15,7 @@ * IRQs. */ -#ifdef __alpha__ -# define I8042_KBD_IRQ 1 -# define I8042_AUX_IRQ (RTC_PORT(0) == 0x170 ? 9 : 12) /* Jensen is special */ -#elif defined(__arm__) +#if defined(__arm__) /* defined in include/asm-arm/arch-xxx/irqs.h */ #include <asm/irq.h> #elif defined(CONFIG_PPC) diff --git a/drivers/iommu/amd/iommu.c b/drivers/iommu/amd/iommu.c index e692217fcb28..fb727f5b0b82 100644 --- a/drivers/iommu/amd/iommu.c +++ b/drivers/iommu/amd/iommu.c @@ -2754,6 +2754,10 @@ static int amd_iommu_def_domain_type(struct device *dev) if (!dev_data) return 0; + /* Always use DMA domain for untrusted device */ + if (dev_is_pci(dev) && to_pci_dev(dev)->untrusted) + return IOMMU_DOMAIN_DMA; + /* * Do not identity map IOMMUv2 capable devices when: * - memory encryption is active, because some of those devices diff --git a/drivers/iommu/arm/arm-smmu/arm-smmu-nvidia.c b/drivers/iommu/arm/arm-smmu/arm-smmu-nvidia.c index 87bf522b9d2e..957d988b6d83 100644 --- a/drivers/iommu/arm/arm-smmu/arm-smmu-nvidia.c +++ b/drivers/iommu/arm/arm-smmu/arm-smmu-nvidia.c @@ -221,11 +221,9 @@ static irqreturn_t nvidia_smmu_context_fault(int irq, void *dev) unsigned int inst; irqreturn_t ret = IRQ_NONE; struct arm_smmu_device *smmu; - struct iommu_domain *domain = dev; - struct arm_smmu_domain *smmu_domain; + struct arm_smmu_domain *smmu_domain = dev; struct nvidia_smmu *nvidia; - smmu_domain = container_of(domain, struct arm_smmu_domain, domain); smmu = smmu_domain->smmu; nvidia = to_nvidia_smmu(smmu); diff --git a/drivers/iommu/intel/irq_remapping.c b/drivers/iommu/intel/irq_remapping.c index 566297bc87dd..712ebfc9870c 100644 --- a/drivers/iommu/intel/irq_remapping.c +++ b/drivers/iommu/intel/irq_remapping.c @@ -19,6 +19,7 @@ #include <asm/cpu.h> #include <asm/irq_remapping.h> #include <asm/pci-direct.h> +#include <asm/posted_intr.h> #include "iommu.h" #include "../irq_remapping.h" @@ -49,6 +50,7 @@ struct irq_2_iommu { u16 sub_handle; u8 irte_mask; enum irq_mode mode; + bool posted_msi; }; struct intel_ir_data { @@ -1118,6 +1120,14 @@ static void prepare_irte(struct irte *irte, int vector, unsigned int dest) irte->redir_hint = 1; } +static void prepare_irte_posted(struct irte *irte) +{ + memset(irte, 0, sizeof(*irte)); + + irte->present = 1; + irte->p_pst = 1; +} + struct irq_remap_ops intel_irq_remap_ops = { .prepare = intel_prepare_irq_remapping, .enable = intel_enable_irq_remapping, @@ -1126,6 +1136,47 @@ struct irq_remap_ops intel_irq_remap_ops = { .enable_faulting = enable_drhd_fault_handling, }; +#ifdef CONFIG_X86_POSTED_MSI + +static phys_addr_t get_pi_desc_addr(struct irq_data *irqd) +{ + int cpu = cpumask_first(irq_data_get_effective_affinity_mask(irqd)); + + if (WARN_ON(cpu >= nr_cpu_ids)) + return 0; + + return __pa(per_cpu_ptr(&posted_msi_pi_desc, cpu)); +} + +static void intel_ir_reconfigure_irte_posted(struct irq_data *irqd) +{ + struct intel_ir_data *ir_data = irqd->chip_data; + struct irte *irte = &ir_data->irte_entry; + struct irte irte_pi; + u64 pid_addr; + + pid_addr = get_pi_desc_addr(irqd); + + if (!pid_addr) { + pr_warn("Failed to setup IRQ %d for posted mode", irqd->irq); + return; + } + + memset(&irte_pi, 0, sizeof(irte_pi)); + + /* The shared IRTE already be set up as posted during alloc_irte */ + dmar_copy_shared_irte(&irte_pi, irte); + + irte_pi.pda_l = (pid_addr >> (32 - PDA_LOW_BIT)) & ~(-1UL << PDA_LOW_BIT); + irte_pi.pda_h = (pid_addr >> 32) & ~(-1UL << PDA_HIGH_BIT); + + modify_irte(&ir_data->irq_2_iommu, &irte_pi); +} + +#else +static inline void intel_ir_reconfigure_irte_posted(struct irq_data *irqd) {} +#endif + static void intel_ir_reconfigure_irte(struct irq_data *irqd, bool force) { struct intel_ir_data *ir_data = irqd->chip_data; @@ -1139,8 +1190,9 @@ static void intel_ir_reconfigure_irte(struct irq_data *irqd, bool force) irte->vector = cfg->vector; irte->dest_id = IRTE_DEST(cfg->dest_apicid); - /* Update the hardware only if the interrupt is in remapped mode. */ - if (force || ir_data->irq_2_iommu.mode == IRQ_REMAPPING) + if (ir_data->irq_2_iommu.posted_msi) + intel_ir_reconfigure_irte_posted(irqd); + else if (force || ir_data->irq_2_iommu.mode == IRQ_REMAPPING) modify_irte(&ir_data->irq_2_iommu, irte); } @@ -1194,7 +1246,7 @@ static int intel_ir_set_vcpu_affinity(struct irq_data *data, void *info) struct intel_ir_data *ir_data = data->chip_data; struct vcpu_data *vcpu_pi_info = info; - /* stop posting interrupts, back to remapping mode */ + /* stop posting interrupts, back to the default mode */ if (!vcpu_pi_info) { modify_irte(&ir_data->irq_2_iommu, &ir_data->irte_entry); } else { @@ -1233,6 +1285,49 @@ static struct irq_chip intel_ir_chip = { .irq_set_vcpu_affinity = intel_ir_set_vcpu_affinity, }; +/* + * With posted MSIs, all vectors are multiplexed into a single notification + * vector. Devices MSIs are then dispatched in a demux loop where + * EOIs can be coalesced as well. + * + * "INTEL-IR-POST" IRQ chip does not do EOI on ACK, thus the dummy irq_ack() + * function. Instead EOI is performed by the posted interrupt notification + * handler. + * + * For the example below, 3 MSIs are coalesced into one CPU notification. Only + * one apic_eoi() is needed. + * + * __sysvec_posted_msi_notification() + * irq_enter(); + * handle_edge_irq() + * irq_chip_ack_parent() + * dummy(); // No EOI + * handle_irq_event() + * driver_handler() + * handle_edge_irq() + * irq_chip_ack_parent() + * dummy(); // No EOI + * handle_irq_event() + * driver_handler() + * handle_edge_irq() + * irq_chip_ack_parent() + * dummy(); // No EOI + * handle_irq_event() + * driver_handler() + * apic_eoi() + * irq_exit() + */ + +static void dummy_ack(struct irq_data *d) { } + +static struct irq_chip intel_ir_chip_post_msi = { + .name = "INTEL-IR-POST", + .irq_ack = dummy_ack, + .irq_set_affinity = intel_ir_set_affinity, + .irq_compose_msi_msg = intel_ir_compose_msi_msg, + .irq_set_vcpu_affinity = intel_ir_set_vcpu_affinity, +}; + static void fill_msi_msg(struct msi_msg *msg, u32 index, u32 subhandle) { memset(msg, 0, sizeof(*msg)); @@ -1274,6 +1369,11 @@ static void intel_irq_remapping_prepare_irte(struct intel_ir_data *data, break; case X86_IRQ_ALLOC_TYPE_PCI_MSI: case X86_IRQ_ALLOC_TYPE_PCI_MSIX: + if (posted_msi_supported()) { + prepare_irte_posted(irte); + data->irq_2_iommu.posted_msi = 1; + } + set_msi_sid(irte, pci_real_dma_dev(msi_desc_to_pci_dev(info->desc))); break; @@ -1361,7 +1461,12 @@ static int intel_irq_remapping_alloc(struct irq_domain *domain, irq_data->hwirq = (index << 16) + i; irq_data->chip_data = ird; - irq_data->chip = &intel_ir_chip; + if (posted_msi_supported() && + ((info->type == X86_IRQ_ALLOC_TYPE_PCI_MSI) || + (info->type == X86_IRQ_ALLOC_TYPE_PCI_MSIX))) + irq_data->chip = &intel_ir_chip_post_msi; + else + irq_data->chip = &intel_ir_chip; intel_irq_remapping_prepare_irte(ird, irq_cfg, info, index, i); irq_set_status_flags(virq + i, IRQ_MOVE_PCNTXT); } diff --git a/drivers/iommu/irq_remapping.c b/drivers/iommu/irq_remapping.c index ee59647c2050..056fec6991bc 100644 --- a/drivers/iommu/irq_remapping.c +++ b/drivers/iommu/irq_remapping.c @@ -24,6 +24,8 @@ int no_x2apic_optout; int disable_irq_post = 0; +bool enable_posted_msi __ro_after_init; + static int disable_irq_remap; static struct irq_remap_ops *remap_ops; @@ -70,7 +72,8 @@ static __init int setup_irqremap(char *str) no_x2apic_optout = 1; else if (!strncmp(str, "nopost", 6)) disable_irq_post = 1; - + else if (IS_ENABLED(CONFIG_X86_POSTED_MSI) && !strncmp(str, "posted_msi", 10)) + enable_posted_msi = true; str += strcspn(str, ","); while (*str == ',') str++; diff --git a/drivers/irqchip/Kconfig b/drivers/irqchip/Kconfig index 72c07a12f5e1..14464716bacb 100644 --- a/drivers/irqchip/Kconfig +++ b/drivers/irqchip/Kconfig @@ -540,6 +540,31 @@ config RISCV_INTC depends on RISCV select IRQ_DOMAIN_HIERARCHY +config RISCV_APLIC + bool + depends on RISCV + select IRQ_DOMAIN_HIERARCHY + +config RISCV_APLIC_MSI + bool + depends on RISCV_APLIC + select GENERIC_MSI_IRQ + default RISCV_APLIC + +config RISCV_IMSIC + bool + depends on RISCV + select IRQ_DOMAIN_HIERARCHY + select GENERIC_IRQ_MATRIX_ALLOCATOR + select GENERIC_MSI_IRQ + +config RISCV_IMSIC_PCI + bool + depends on RISCV_IMSIC + depends on PCI + depends on PCI_MSI + default RISCV_IMSIC + config SIFIVE_PLIC bool depends on RISCV @@ -568,7 +593,7 @@ config IRQ_LOONGARCH_CPU bool select GENERIC_IRQ_CHIP select IRQ_DOMAIN - select GENERIC_IRQ_EFFECTIVE_AFF_MASK + select GENERIC_IRQ_EFFECTIVE_AFF_MASK if SMP select LOONGSON_HTVEC select LOONGSON_LIOINTC select LOONGSON_EIOINTC diff --git a/drivers/irqchip/Makefile b/drivers/irqchip/Makefile index ec4a18380998..d9dc3d99aaa8 100644 --- a/drivers/irqchip/Makefile +++ b/drivers/irqchip/Makefile @@ -95,6 +95,9 @@ obj-$(CONFIG_QCOM_MPM) += irq-qcom-mpm.o obj-$(CONFIG_CSKY_MPINTC) += irq-csky-mpintc.o obj-$(CONFIG_CSKY_APB_INTC) += irq-csky-apb-intc.o obj-$(CONFIG_RISCV_INTC) += irq-riscv-intc.o +obj-$(CONFIG_RISCV_APLIC) += irq-riscv-aplic-main.o irq-riscv-aplic-direct.o +obj-$(CONFIG_RISCV_APLIC_MSI) += irq-riscv-aplic-msi.o +obj-$(CONFIG_RISCV_IMSIC) += irq-riscv-imsic-state.o irq-riscv-imsic-early.o irq-riscv-imsic-platform.o obj-$(CONFIG_SIFIVE_PLIC) += irq-sifive-plic.o obj-$(CONFIG_STARFIVE_JH8100_INTC) += irq-starfive-jh8100-intc.o obj-$(CONFIG_IMX_IRQSTEER) += irq-imx-irqsteer.o diff --git a/drivers/irqchip/irq-alpine-msi.c b/drivers/irqchip/irq-alpine-msi.c index 9c8b1349ee17..a1430ab60a8a 100644 --- a/drivers/irqchip/irq-alpine-msi.c +++ b/drivers/irqchip/irq-alpine-msi.c @@ -165,7 +165,7 @@ static int alpine_msix_middle_domain_alloc(struct irq_domain *domain, return 0; err_sgi: - irq_domain_free_irqs_parent(domain, virq, i - 1); + irq_domain_free_irqs_parent(domain, virq, i); alpine_msix_free_sgi(priv, sgi, nr_irqs); return err; } diff --git a/drivers/irqchip/irq-bcm6345-l1.c b/drivers/irqchip/irq-bcm6345-l1.c index eb02d203c963..90daa274ef23 100644 --- a/drivers/irqchip/irq-bcm6345-l1.c +++ b/drivers/irqchip/irq-bcm6345-l1.c @@ -192,14 +192,10 @@ static int bcm6345_l1_set_affinity(struct irq_data *d, u32 mask = BIT(d->hwirq % IRQS_PER_WORD); unsigned int old_cpu = cpu_for_irq(intc, d); unsigned int new_cpu; - struct cpumask valid; unsigned long flags; bool enabled; - if (!cpumask_and(&valid, &intc->cpumask, dest)) - return -EINVAL; - - new_cpu = cpumask_any_and(&valid, cpu_online_mask); + new_cpu = cpumask_first_and_and(&intc->cpumask, dest, cpu_online_mask); if (new_cpu >= nr_cpu_ids) return -EINVAL; diff --git a/drivers/irqchip/irq-brcmstb-l2.c b/drivers/irqchip/irq-brcmstb-l2.c index 2b0b3175cea0..c988886917f7 100644 --- a/drivers/irqchip/irq-brcmstb-l2.c +++ b/drivers/irqchip/irq-brcmstb-l2.c @@ -118,7 +118,7 @@ out: chained_irq_exit(chip, desc); } -static void brcmstb_l2_intc_suspend(struct irq_data *d) +static void __brcmstb_l2_intc_suspend(struct irq_data *d, bool save) { struct irq_chip_generic *gc = irq_data_get_irq_chip_data(d); struct irq_chip_type *ct = irq_data_get_chip_type(d); @@ -127,7 +127,8 @@ static void brcmstb_l2_intc_suspend(struct irq_data *d) irq_gc_lock_irqsave(gc, flags); /* Save the current mask */ - b->saved_mask = irq_reg_readl(gc, ct->regs.mask); + if (save) + b->saved_mask = irq_reg_readl(gc, ct->regs.mask); if (b->can_wake) { /* Program the wakeup mask */ @@ -137,6 +138,16 @@ static void brcmstb_l2_intc_suspend(struct irq_data *d) irq_gc_unlock_irqrestore(gc, flags); } +static void brcmstb_l2_intc_shutdown(struct irq_data *d) +{ + __brcmstb_l2_intc_suspend(d, false); +} + +static void brcmstb_l2_intc_suspend(struct irq_data *d) +{ + __brcmstb_l2_intc_suspend(d, true); +} + static void brcmstb_l2_intc_resume(struct irq_data *d) { struct irq_chip_generic *gc = irq_data_get_irq_chip_data(d); @@ -252,7 +263,7 @@ static int __init brcmstb_l2_intc_of_init(struct device_node *np, ct->chip.irq_suspend = brcmstb_l2_intc_suspend; ct->chip.irq_resume = brcmstb_l2_intc_resume; - ct->chip.irq_pm_shutdown = brcmstb_l2_intc_suspend; + ct->chip.irq_pm_shutdown = brcmstb_l2_intc_shutdown; if (data->can_wake) { /* This IRQ chip can wake the system, set all child interrupts diff --git a/drivers/irqchip/irq-gic-v3-its.c b/drivers/irqchip/irq-gic-v3-its.c index 5f7d3db3afd8..40ebf1726393 100644 --- a/drivers/irqchip/irq-gic-v3-its.c +++ b/drivers/irqchip/irq-gic-v3-its.c @@ -3832,9 +3832,9 @@ static int its_vpe_set_affinity(struct irq_data *d, bool force) { struct its_vpe *vpe = irq_data_get_irq_chip_data(d); - struct cpumask common, *table_mask; + unsigned int from, cpu = nr_cpu_ids; + struct cpumask *table_mask; unsigned long flags; - int from, cpu; /* * Changing affinity is mega expensive, so let's be as lazy as @@ -3856,10 +3856,15 @@ static int its_vpe_set_affinity(struct irq_data *d, * If we are offered another CPU in the same GICv4.1 ITS * affinity, pick this one. Otherwise, any CPU will do. */ - if (table_mask && cpumask_and(&common, mask_val, table_mask)) - cpu = cpumask_test_cpu(from, &common) ? from : cpumask_first(&common); - else + if (table_mask) + cpu = cpumask_any_and(mask_val, table_mask); + if (cpu < nr_cpu_ids) { + if (cpumask_test_cpu(from, mask_val) && + cpumask_test_cpu(from, table_mask)) + cpu = from; + } else { cpu = cpumask_first(mask_val); + } if (from == cpu) goto out; @@ -4527,8 +4532,6 @@ static int its_vpe_irq_domain_alloc(struct irq_domain *domain, unsigned int virq struct page *vprop_page; int base, nr_ids, i, err = 0; - BUG_ON(!vm); - bitmap = its_lpi_alloc(roundup_pow_of_two(nr_irqs), &base, &nr_ids); if (!bitmap) return -ENOMEM; diff --git a/drivers/irqchip/irq-loongson-eiointc.c b/drivers/irqchip/irq-loongson-eiointc.c index b64cbe3052e8..c7ddebf312ad 100644 --- a/drivers/irqchip/irq-loongson-eiointc.c +++ b/drivers/irqchip/irq-loongson-eiointc.c @@ -59,6 +59,7 @@ static int cpu_to_eio_node(int cpu) return cpu_logical_map(cpu) / CORES_PER_EIO_NODE; } +#ifdef CONFIG_SMP static void eiointc_set_irq_route(int pos, unsigned int cpu, unsigned int mnode, nodemask_t *node_map) { int i, node, cpu_node, route_node; @@ -92,19 +93,15 @@ static int eiointc_set_irq_affinity(struct irq_data *d, const struct cpumask *af unsigned int cpu; unsigned long flags; uint32_t vector, regaddr; - struct cpumask intersect_affinity; struct eiointc_priv *priv = d->domain->host_data; raw_spin_lock_irqsave(&affinity_lock, flags); - cpumask_and(&intersect_affinity, affinity, cpu_online_mask); - cpumask_and(&intersect_affinity, &intersect_affinity, &priv->cpuspan_map); - - if (cpumask_empty(&intersect_affinity)) { + cpu = cpumask_first_and_and(&priv->cpuspan_map, affinity, cpu_online_mask); + if (cpu >= nr_cpu_ids) { raw_spin_unlock_irqrestore(&affinity_lock, flags); return -EINVAL; } - cpu = cpumask_first(&intersect_affinity); vector = d->hwirq; regaddr = EIOINTC_REG_ENABLE + ((vector >> 5) << 2); @@ -126,6 +123,7 @@ static int eiointc_set_irq_affinity(struct irq_data *d, const struct cpumask *af return IRQ_SET_MASK_OK; } +#endif static int eiointc_index(int node) { @@ -238,7 +236,9 @@ static struct irq_chip eiointc_irq_chip = { .irq_ack = eiointc_ack_irq, .irq_mask = eiointc_mask_irq, .irq_unmask = eiointc_unmask_irq, +#ifdef CONFIG_SMP .irq_set_affinity = eiointc_set_irq_affinity, +#endif }; static int eiointc_domain_alloc(struct irq_domain *domain, unsigned int virq, diff --git a/drivers/irqchip/irq-loongson-pch-msi.c b/drivers/irqchip/irq-loongson-pch-msi.c index 6e1e1f011bb2..dd4d699170f4 100644 --- a/drivers/irqchip/irq-loongson-pch-msi.c +++ b/drivers/irqchip/irq-loongson-pch-msi.c @@ -136,7 +136,7 @@ static int pch_msi_middle_domain_alloc(struct irq_domain *domain, err_hwirq: pch_msi_free_hwirq(priv, hwirq, nr_irqs); - irq_domain_free_irqs_parent(domain, virq, i - 1); + irq_domain_free_irqs_parent(domain, virq, i); return err; } diff --git a/drivers/irqchip/irq-loongson-pch-pic.c b/drivers/irqchip/irq-loongson-pch-pic.c index 63db8e2172e0..cbaef65e804c 100644 --- a/drivers/irqchip/irq-loongson-pch-pic.c +++ b/drivers/irqchip/irq-loongson-pch-pic.c @@ -33,6 +33,7 @@ #define PIC_COUNT (PIC_COUNT_PER_REG * PIC_REG_COUNT) #define PIC_REG_IDX(irq_id) ((irq_id) / PIC_COUNT_PER_REG) #define PIC_REG_BIT(irq_id) ((irq_id) % PIC_COUNT_PER_REG) +#define PIC_UNDEF_VECTOR 255 static int nr_pics; @@ -46,12 +47,19 @@ struct pch_pic { u32 saved_vec_en[PIC_REG_COUNT]; u32 saved_vec_pol[PIC_REG_COUNT]; u32 saved_vec_edge[PIC_REG_COUNT]; + u8 table[PIC_COUNT]; + int inuse; }; static struct pch_pic *pch_pic_priv[MAX_IO_PICS]; struct fwnode_handle *pch_pic_handle[MAX_IO_PICS]; +static inline u8 hwirq_to_bit(struct pch_pic *priv, int hirq) +{ + return priv->table[hirq]; +} + static void pch_pic_bitset(struct pch_pic *priv, int offset, int bit) { u32 reg; @@ -80,45 +88,47 @@ static void pch_pic_mask_irq(struct irq_data *d) { struct pch_pic *priv = irq_data_get_irq_chip_data(d); - pch_pic_bitset(priv, PCH_PIC_MASK, d->hwirq); + pch_pic_bitset(priv, PCH_PIC_MASK, hwirq_to_bit(priv, d->hwirq)); irq_chip_mask_parent(d); } static void pch_pic_unmask_irq(struct irq_data *d) { struct pch_pic *priv = irq_data_get_irq_chip_data(d); + int bit = hwirq_to_bit(priv, d->hwirq); - writel(BIT(PIC_REG_BIT(d->hwirq)), - priv->base + PCH_PIC_CLR + PIC_REG_IDX(d->hwirq) * 4); + writel(BIT(PIC_REG_BIT(bit)), + priv->base + PCH_PIC_CLR + PIC_REG_IDX(bit) * 4); irq_chip_unmask_parent(d); - pch_pic_bitclr(priv, PCH_PIC_MASK, d->hwirq); + pch_pic_bitclr(priv, PCH_PIC_MASK, bit); } static int pch_pic_set_type(struct irq_data *d, unsigned int type) { struct pch_pic *priv = irq_data_get_irq_chip_data(d); + int bit = hwirq_to_bit(priv, d->hwirq); int ret = 0; switch (type) { case IRQ_TYPE_EDGE_RISING: - pch_pic_bitset(priv, PCH_PIC_EDGE, d->hwirq); - pch_pic_bitclr(priv, PCH_PIC_POL, d->hwirq); + pch_pic_bitset(priv, PCH_PIC_EDGE, bit); + pch_pic_bitclr(priv, PCH_PIC_POL, bit); irq_set_handler_locked(d, handle_edge_irq); break; case IRQ_TYPE_EDGE_FALLING: - pch_pic_bitset(priv, PCH_PIC_EDGE, d->hwirq); - pch_pic_bitset(priv, PCH_PIC_POL, d->hwirq); + pch_pic_bitset(priv, PCH_PIC_EDGE, bit); + pch_pic_bitset(priv, PCH_PIC_POL, bit); irq_set_handler_locked(d, handle_edge_irq); break; case IRQ_TYPE_LEVEL_HIGH: - pch_pic_bitclr(priv, PCH_PIC_EDGE, d->hwirq); - pch_pic_bitclr(priv, PCH_PIC_POL, d->hwirq); + pch_pic_bitclr(priv, PCH_PIC_EDGE, bit); + pch_pic_bitclr(priv, PCH_PIC_POL, bit); irq_set_handler_locked(d, handle_level_irq); break; case IRQ_TYPE_LEVEL_LOW: - pch_pic_bitclr(priv, PCH_PIC_EDGE, d->hwirq); - pch_pic_bitset(priv, PCH_PIC_POL, d->hwirq); + pch_pic_bitclr(priv, PCH_PIC_EDGE, bit); + pch_pic_bitset(priv, PCH_PIC_POL, bit); irq_set_handler_locked(d, handle_level_irq); break; default: @@ -133,11 +143,12 @@ static void pch_pic_ack_irq(struct irq_data *d) { unsigned int reg; struct pch_pic *priv = irq_data_get_irq_chip_data(d); + int bit = hwirq_to_bit(priv, d->hwirq); - reg = readl(priv->base + PCH_PIC_EDGE + PIC_REG_IDX(d->hwirq) * 4); - if (reg & BIT(PIC_REG_BIT(d->hwirq))) { - writel(BIT(PIC_REG_BIT(d->hwirq)), - priv->base + PCH_PIC_CLR + PIC_REG_IDX(d->hwirq) * 4); + reg = readl(priv->base + PCH_PIC_EDGE + PIC_REG_IDX(bit) * 4); + if (reg & BIT(PIC_REG_BIT(bit))) { + writel(BIT(PIC_REG_BIT(bit)), + priv->base + PCH_PIC_CLR + PIC_REG_IDX(bit) * 4); } irq_chip_ack_parent(d); } @@ -159,6 +170,8 @@ static int pch_pic_domain_translate(struct irq_domain *d, { struct pch_pic *priv = d->host_data; struct device_node *of_node = to_of_node(fwspec->fwnode); + unsigned long flags; + int i; if (of_node) { if (fwspec->param_count < 2) @@ -171,12 +184,33 @@ static int pch_pic_domain_translate(struct irq_domain *d, return -EINVAL; *hwirq = fwspec->param[0] - priv->gsi_base; + if (fwspec->param_count > 1) *type = fwspec->param[1] & IRQ_TYPE_SENSE_MASK; else *type = IRQ_TYPE_NONE; } + raw_spin_lock_irqsave(&priv->pic_lock, flags); + /* Check pic-table to confirm if the hwirq has been assigned */ + for (i = 0; i < priv->inuse; i++) { + if (priv->table[i] == *hwirq) { + *hwirq = i; + break; + } + } + if (i == priv->inuse) { + /* Assign a new hwirq in pic-table */ + if (priv->inuse >= PIC_COUNT) { + pr_err("pch-pic domain has no free vectors\n"); + raw_spin_unlock_irqrestore(&priv->pic_lock, flags); + return -EINVAL; + } + priv->table[priv->inuse] = *hwirq; + *hwirq = priv->inuse++; + } + raw_spin_unlock_irqrestore(&priv->pic_lock, flags); + return 0; } @@ -194,6 +228,9 @@ static int pch_pic_alloc(struct irq_domain *domain, unsigned int virq, if (err) return err; + /* Write vector ID */ + writeb(priv->ht_vec_base + hwirq, priv->base + PCH_INT_HTVEC(hwirq_to_bit(priv, hwirq))); + parent_fwspec.fwnode = domain->parent->fwnode; parent_fwspec.param_count = 1; parent_fwspec.param[0] = hwirq + priv->ht_vec_base; @@ -222,7 +259,7 @@ static void pch_pic_reset(struct pch_pic *priv) for (i = 0; i < PIC_COUNT; i++) { /* Write vector ID */ - writeb(priv->ht_vec_base + i, priv->base + PCH_INT_HTVEC(i)); + writeb(priv->ht_vec_base + i, priv->base + PCH_INT_HTVEC(hwirq_to_bit(priv, i))); /* Hardcode route to HT0 Lo */ writeb(1, priv->base + PCH_INT_ROUTE(i)); } @@ -284,6 +321,7 @@ static int pch_pic_init(phys_addr_t addr, unsigned long size, int vec_base, u32 gsi_base) { struct pch_pic *priv; + int i; priv = kzalloc(sizeof(*priv), GFP_KERNEL); if (!priv) @@ -294,6 +332,10 @@ static int pch_pic_init(phys_addr_t addr, unsigned long size, int vec_base, if (!priv->base) goto free_priv; + priv->inuse = 0; + for (i = 0; i < PIC_COUNT; i++) + priv->table[i] = PIC_UNDEF_VECTOR; + priv->ht_vec_base = vec_base; priv->vec_count = ((readq(priv->base) >> 48) & 0xff) + 1; priv->gsi_base = gsi_base; diff --git a/drivers/irqchip/irq-mxs.c b/drivers/irqchip/irq-mxs.c index be9680645545..d67b5da38982 100644 --- a/drivers/irqchip/irq-mxs.c +++ b/drivers/irqchip/irq-mxs.c @@ -130,7 +130,7 @@ static struct irq_chip asm9260_icoll_chip = { IRQCHIP_SKIP_SET_WAKE, }; -asmlinkage void __exception_irq_entry icoll_handle_irq(struct pt_regs *regs) +static void __exception_irq_entry icoll_handle_irq(struct pt_regs *regs) { u32 irqnr; diff --git a/drivers/irqchip/irq-renesas-rzg2l.c b/drivers/irqchip/irq-renesas-rzg2l.c index ae67fec2ab46..f6484bf15e0b 100644 --- a/drivers/irqchip/irq-renesas-rzg2l.c +++ b/drivers/irqchip/irq-renesas-rzg2l.c @@ -138,7 +138,7 @@ static void rzg2l_irqc_eoi(struct irq_data *d) irq_chip_eoi_parent(d); } -static void rzg2l_irqc_irq_disable(struct irq_data *d) +static void rzg2l_tint_irq_endisable(struct irq_data *d, bool enable) { unsigned int hw_irq = irqd_to_hwirq(d); @@ -151,30 +151,24 @@ static void rzg2l_irqc_irq_disable(struct irq_data *d) raw_spin_lock(&priv->lock); reg = readl_relaxed(priv->base + TSSR(tssr_index)); - reg &= ~(TIEN << TSSEL_SHIFT(tssr_offset)); + if (enable) + reg |= TIEN << TSSEL_SHIFT(tssr_offset); + else + reg &= ~(TIEN << TSSEL_SHIFT(tssr_offset)); writel_relaxed(reg, priv->base + TSSR(tssr_index)); raw_spin_unlock(&priv->lock); } +} + +static void rzg2l_irqc_irq_disable(struct irq_data *d) +{ + rzg2l_tint_irq_endisable(d, false); irq_chip_disable_parent(d); } static void rzg2l_irqc_irq_enable(struct irq_data *d) { - unsigned int hw_irq = irqd_to_hwirq(d); - - if (hw_irq >= IRQC_TINT_START && hw_irq < IRQC_NUM_IRQ) { - struct rzg2l_irqc_priv *priv = irq_data_to_priv(d); - u32 offset = hw_irq - IRQC_TINT_START; - u32 tssr_offset = TSSR_OFFSET(offset); - u8 tssr_index = TSSR_INDEX(offset); - u32 reg; - - raw_spin_lock(&priv->lock); - reg = readl_relaxed(priv->base + TSSR(tssr_index)); - reg |= TIEN << TSSEL_SHIFT(tssr_offset); - writel_relaxed(reg, priv->base + TSSR(tssr_index)); - raw_spin_unlock(&priv->lock); - } + rzg2l_tint_irq_endisable(d, true); irq_chip_enable_parent(d); } diff --git a/drivers/irqchip/irq-riscv-aplic-direct.c b/drivers/irqchip/irq-riscv-aplic-direct.c new file mode 100644 index 000000000000..4a3ffe856d6c --- /dev/null +++ b/drivers/irqchip/irq-riscv-aplic-direct.c @@ -0,0 +1,323 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2021 Western Digital Corporation or its affiliates. + * Copyright (C) 2022 Ventana Micro Systems Inc. + */ + +#include <linux/bitfield.h> +#include <linux/bitops.h> +#include <linux/cpu.h> +#include <linux/interrupt.h> +#include <linux/irqchip.h> +#include <linux/irqchip/chained_irq.h> +#include <linux/irqchip/riscv-aplic.h> +#include <linux/module.h> +#include <linux/of_address.h> +#include <linux/printk.h> +#include <linux/smp.h> + +#include "irq-riscv-aplic-main.h" + +#define APLIC_DISABLE_IDELIVERY 0 +#define APLIC_ENABLE_IDELIVERY 1 +#define APLIC_DISABLE_ITHRESHOLD 1 +#define APLIC_ENABLE_ITHRESHOLD 0 + +struct aplic_direct { + struct aplic_priv priv; + struct irq_domain *irqdomain; + struct cpumask lmask; +}; + +struct aplic_idc { + unsigned int hart_index; + void __iomem *regs; + struct aplic_direct *direct; +}; + +static unsigned int aplic_direct_parent_irq; +static DEFINE_PER_CPU(struct aplic_idc, aplic_idcs); + +static void aplic_direct_irq_eoi(struct irq_data *d) +{ + /* + * The fasteoi_handler requires irq_eoi() callback hence + * provide a dummy handler. + */ +} + +#ifdef CONFIG_SMP +static int aplic_direct_set_affinity(struct irq_data *d, const struct cpumask *mask_val, + bool force) +{ + struct aplic_priv *priv = irq_data_get_irq_chip_data(d); + struct aplic_direct *direct = container_of(priv, struct aplic_direct, priv); + struct aplic_idc *idc; + unsigned int cpu, val; + void __iomem *target; + + if (force) + cpu = cpumask_first_and(&direct->lmask, mask_val); + else + cpu = cpumask_first_and_and(&direct->lmask, mask_val, cpu_online_mask); + + if (cpu >= nr_cpu_ids) + return -EINVAL; + + idc = per_cpu_ptr(&aplic_idcs, cpu); + target = priv->regs + APLIC_TARGET_BASE + (d->hwirq - 1) * sizeof(u32); + val = FIELD_PREP(APLIC_TARGET_HART_IDX, idc->hart_index); + val |= FIELD_PREP(APLIC_TARGET_IPRIO, APLIC_DEFAULT_PRIORITY); + writel(val, target); + + irq_data_update_effective_affinity(d, cpumask_of(cpu)); + + return IRQ_SET_MASK_OK_DONE; +} +#endif + +static struct irq_chip aplic_direct_chip = { + .name = "APLIC-DIRECT", + .irq_mask = aplic_irq_mask, + .irq_unmask = aplic_irq_unmask, + .irq_set_type = aplic_irq_set_type, + .irq_eoi = aplic_direct_irq_eoi, +#ifdef CONFIG_SMP + .irq_set_affinity = aplic_direct_set_affinity, +#endif + .flags = IRQCHIP_SET_TYPE_MASKED | + IRQCHIP_SKIP_SET_WAKE | + IRQCHIP_MASK_ON_SUSPEND, +}; + +static int aplic_direct_irqdomain_translate(struct irq_domain *d, struct irq_fwspec *fwspec, + unsigned long *hwirq, unsigned int *type) +{ + struct aplic_priv *priv = d->host_data; + + return aplic_irqdomain_translate(fwspec, priv->gsi_base, hwirq, type); +} + +static int aplic_direct_irqdomain_alloc(struct irq_domain *domain, unsigned int virq, + unsigned int nr_irqs, void *arg) +{ + struct aplic_priv *priv = domain->host_data; + struct aplic_direct *direct = container_of(priv, struct aplic_direct, priv); + struct irq_fwspec *fwspec = arg; + irq_hw_number_t hwirq; + unsigned int type; + int i, ret; + + ret = aplic_irqdomain_translate(fwspec, priv->gsi_base, &hwirq, &type); + if (ret) + return ret; + + for (i = 0; i < nr_irqs; i++) { + irq_domain_set_info(domain, virq + i, hwirq + i, &aplic_direct_chip, + priv, handle_fasteoi_irq, NULL, NULL); + irq_set_affinity(virq + i, &direct->lmask); + } + + return 0; +} + +static const struct irq_domain_ops aplic_direct_irqdomain_ops = { + .translate = aplic_direct_irqdomain_translate, + .alloc = aplic_direct_irqdomain_alloc, + .free = irq_domain_free_irqs_top, +}; + +/* + * To handle an APLIC direct interrupts, we just read the CLAIMI register + * which will return highest priority pending interrupt and clear the + * pending bit of the interrupt. This process is repeated until CLAIMI + * register return zero value. + */ +static void aplic_direct_handle_irq(struct irq_desc *desc) +{ + struct aplic_idc *idc = this_cpu_ptr(&aplic_idcs); + struct irq_domain *irqdomain = idc->direct->irqdomain; + struct irq_chip *chip = irq_desc_get_chip(desc); + irq_hw_number_t hw_irq; + int irq; + + chained_irq_enter(chip, desc); + + while ((hw_irq = readl(idc->regs + APLIC_IDC_CLAIMI))) { + hw_irq = hw_irq >> APLIC_IDC_TOPI_ID_SHIFT; + irq = irq_find_mapping(irqdomain, hw_irq); + + if (unlikely(irq <= 0)) { + dev_warn_ratelimited(idc->direct->priv.dev, + "hw_irq %lu mapping not found\n", hw_irq); + } else { + generic_handle_irq(irq); + } + } + + chained_irq_exit(chip, desc); +} + +static void aplic_idc_set_delivery(struct aplic_idc *idc, bool en) +{ + u32 de = (en) ? APLIC_ENABLE_IDELIVERY : APLIC_DISABLE_IDELIVERY; + u32 th = (en) ? APLIC_ENABLE_ITHRESHOLD : APLIC_DISABLE_ITHRESHOLD; + + /* Priority must be less than threshold for interrupt triggering */ + writel(th, idc->regs + APLIC_IDC_ITHRESHOLD); + + /* Delivery must be set to 1 for interrupt triggering */ + writel(de, idc->regs + APLIC_IDC_IDELIVERY); +} + +static int aplic_direct_dying_cpu(unsigned int cpu) +{ + if (aplic_direct_parent_irq) + disable_percpu_irq(aplic_direct_parent_irq); + + return 0; +} + +static int aplic_direct_starting_cpu(unsigned int cpu) +{ + if (aplic_direct_parent_irq) { + enable_percpu_irq(aplic_direct_parent_irq, + irq_get_trigger_type(aplic_direct_parent_irq)); + } + + return 0; +} + +static int aplic_direct_parse_parent_hwirq(struct device *dev, u32 index, + u32 *parent_hwirq, unsigned long *parent_hartid) +{ + struct of_phandle_args parent; + int rc; + + /* + * Currently, only OF fwnode is supported so extend this + * function for ACPI support. + */ + if (!is_of_node(dev->fwnode)) + return -EINVAL; + + rc = of_irq_parse_one(to_of_node(dev->fwnode), index, &parent); + if (rc) + return rc; + + rc = riscv_of_parent_hartid(parent.np, parent_hartid); + if (rc) + return rc; + + *parent_hwirq = parent.args[0]; + return 0; +} + +int aplic_direct_setup(struct device *dev, void __iomem *regs) +{ + int i, j, rc, cpu, current_cpu, setup_count = 0; + struct aplic_direct *direct; + struct irq_domain *domain; + struct aplic_priv *priv; + struct aplic_idc *idc; + unsigned long hartid; + u32 v, hwirq; + + direct = devm_kzalloc(dev, sizeof(*direct), GFP_KERNEL); + if (!direct) + return -ENOMEM; + priv = &direct->priv; + + rc = aplic_setup_priv(priv, dev, regs); + if (rc) { + dev_err(dev, "failed to create APLIC context\n"); + return rc; + } + + /* Setup per-CPU IDC and target CPU mask */ + current_cpu = get_cpu(); + for (i = 0; i < priv->nr_idcs; i++) { + rc = aplic_direct_parse_parent_hwirq(dev, i, &hwirq, &hartid); + if (rc) { + dev_warn(dev, "parent irq for IDC%d not found\n", i); + continue; + } + + /* + * Skip interrupts other than external interrupts for + * current privilege level. + */ + if (hwirq != RV_IRQ_EXT) + continue; + + cpu = riscv_hartid_to_cpuid(hartid); + if (cpu < 0) { + dev_warn(dev, "invalid cpuid for IDC%d\n", i); + continue; + } + + cpumask_set_cpu(cpu, &direct->lmask); + + idc = per_cpu_ptr(&aplic_idcs, cpu); + idc->hart_index = i; + idc->regs = priv->regs + APLIC_IDC_BASE + i * APLIC_IDC_SIZE; + idc->direct = direct; + + aplic_idc_set_delivery(idc, true); + + /* + * Boot cpu might not have APLIC hart_index = 0 so check + * and update target registers of all interrupts. + */ + if (cpu == current_cpu && idc->hart_index) { + v = FIELD_PREP(APLIC_TARGET_HART_IDX, idc->hart_index); + v |= FIELD_PREP(APLIC_TARGET_IPRIO, APLIC_DEFAULT_PRIORITY); + for (j = 1; j <= priv->nr_irqs; j++) + writel(v, priv->regs + APLIC_TARGET_BASE + (j - 1) * sizeof(u32)); + } + + setup_count++; + } + put_cpu(); + + /* Find parent domain and register chained handler */ + domain = irq_find_matching_fwnode(riscv_get_intc_hwnode(), + DOMAIN_BUS_ANY); + if (!aplic_direct_parent_irq && domain) { + aplic_direct_parent_irq = irq_create_mapping(domain, RV_IRQ_EXT); + if (aplic_direct_parent_irq) { + irq_set_chained_handler(aplic_direct_parent_irq, + aplic_direct_handle_irq); + + /* + * Setup CPUHP notifier to enable parent + * interrupt on all CPUs + */ + cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, + "irqchip/riscv/aplic:starting", + aplic_direct_starting_cpu, + aplic_direct_dying_cpu); + } + } + + /* Fail if we were not able to setup IDC for any CPU */ + if (!setup_count) + return -ENODEV; + + /* Setup global config and interrupt delivery */ + aplic_init_hw_global(priv, false); + + /* Create irq domain instance for the APLIC */ + direct->irqdomain = irq_domain_create_linear(dev->fwnode, priv->nr_irqs + 1, + &aplic_direct_irqdomain_ops, priv); + if (!direct->irqdomain) { + dev_err(dev, "failed to create direct irq domain\n"); + return -ENOMEM; + } + + /* Advertise the interrupt controller */ + dev_info(dev, "%d interrupts directly connected to %d CPUs\n", + priv->nr_irqs, priv->nr_idcs); + + return 0; +} diff --git a/drivers/irqchip/irq-riscv-aplic-main.c b/drivers/irqchip/irq-riscv-aplic-main.c new file mode 100644 index 000000000000..774a0c97fdab --- /dev/null +++ b/drivers/irqchip/irq-riscv-aplic-main.c @@ -0,0 +1,211 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2021 Western Digital Corporation or its affiliates. + * Copyright (C) 2022 Ventana Micro Systems Inc. + */ + +#include <linux/bitfield.h> +#include <linux/irqchip/riscv-aplic.h> +#include <linux/module.h> +#include <linux/of.h> +#include <linux/of_irq.h> +#include <linux/platform_device.h> +#include <linux/printk.h> + +#include "irq-riscv-aplic-main.h" + +void aplic_irq_unmask(struct irq_data *d) +{ + struct aplic_priv *priv = irq_data_get_irq_chip_data(d); + + writel(d->hwirq, priv->regs + APLIC_SETIENUM); +} + +void aplic_irq_mask(struct irq_data *d) +{ + struct aplic_priv *priv = irq_data_get_irq_chip_data(d); + + writel(d->hwirq, priv->regs + APLIC_CLRIENUM); +} + +int aplic_irq_set_type(struct irq_data *d, unsigned int type) +{ + struct aplic_priv *priv = irq_data_get_irq_chip_data(d); + void __iomem *sourcecfg; + u32 val = 0; + + switch (type) { + case IRQ_TYPE_NONE: + val = APLIC_SOURCECFG_SM_INACTIVE; + break; + case IRQ_TYPE_LEVEL_LOW: + val = APLIC_SOURCECFG_SM_LEVEL_LOW; + break; + case IRQ_TYPE_LEVEL_HIGH: + val = APLIC_SOURCECFG_SM_LEVEL_HIGH; + break; + case IRQ_TYPE_EDGE_FALLING: + val = APLIC_SOURCECFG_SM_EDGE_FALL; + break; + case IRQ_TYPE_EDGE_RISING: + val = APLIC_SOURCECFG_SM_EDGE_RISE; + break; + default: + return -EINVAL; + } + + sourcecfg = priv->regs + APLIC_SOURCECFG_BASE; + sourcecfg += (d->hwirq - 1) * sizeof(u32); + writel(val, sourcecfg); + + return 0; +} + +int aplic_irqdomain_translate(struct irq_fwspec *fwspec, u32 gsi_base, + unsigned long *hwirq, unsigned int *type) +{ + if (WARN_ON(fwspec->param_count < 2)) + return -EINVAL; + if (WARN_ON(!fwspec->param[0])) + return -EINVAL; + + /* For DT, gsi_base is always zero. */ + *hwirq = fwspec->param[0] - gsi_base; + *type = fwspec->param[1] & IRQ_TYPE_SENSE_MASK; + + WARN_ON(*type == IRQ_TYPE_NONE); + + return 0; +} + +void aplic_init_hw_global(struct aplic_priv *priv, bool msi_mode) +{ + u32 val; +#ifdef CONFIG_RISCV_M_MODE + u32 valh; + + if (msi_mode) { + val = lower_32_bits(priv->msicfg.base_ppn); + valh = FIELD_PREP(APLIC_xMSICFGADDRH_BAPPN, upper_32_bits(priv->msicfg.base_ppn)); + valh |= FIELD_PREP(APLIC_xMSICFGADDRH_LHXW, priv->msicfg.lhxw); + valh |= FIELD_PREP(APLIC_xMSICFGADDRH_HHXW, priv->msicfg.hhxw); + valh |= FIELD_PREP(APLIC_xMSICFGADDRH_LHXS, priv->msicfg.lhxs); + valh |= FIELD_PREP(APLIC_xMSICFGADDRH_HHXS, priv->msicfg.hhxs); + writel(val, priv->regs + APLIC_xMSICFGADDR); + writel(valh, priv->regs + APLIC_xMSICFGADDRH); + } +#endif + + /* Setup APLIC domaincfg register */ + val = readl(priv->regs + APLIC_DOMAINCFG); + val |= APLIC_DOMAINCFG_IE; + if (msi_mode) + val |= APLIC_DOMAINCFG_DM; + writel(val, priv->regs + APLIC_DOMAINCFG); + if (readl(priv->regs + APLIC_DOMAINCFG) != val) + dev_warn(priv->dev, "unable to write 0x%x in domaincfg\n", val); +} + +static void aplic_init_hw_irqs(struct aplic_priv *priv) +{ + int i; + + /* Disable all interrupts */ + for (i = 0; i <= priv->nr_irqs; i += 32) + writel(-1U, priv->regs + APLIC_CLRIE_BASE + (i / 32) * sizeof(u32)); + + /* Set interrupt type and default priority for all interrupts */ + for (i = 1; i <= priv->nr_irqs; i++) { + writel(0, priv->regs + APLIC_SOURCECFG_BASE + (i - 1) * sizeof(u32)); + writel(APLIC_DEFAULT_PRIORITY, + priv->regs + APLIC_TARGET_BASE + (i - 1) * sizeof(u32)); + } + + /* Clear APLIC domaincfg */ + writel(0, priv->regs + APLIC_DOMAINCFG); +} + +int aplic_setup_priv(struct aplic_priv *priv, struct device *dev, void __iomem *regs) +{ + struct of_phandle_args parent; + int rc; + + /* + * Currently, only OF fwnode is supported so extend this + * function for ACPI support. + */ + if (!is_of_node(dev->fwnode)) + return -EINVAL; + + /* Save device pointer and register base */ + priv->dev = dev; + priv->regs = regs; + + /* Find out number of interrupt sources */ + rc = of_property_read_u32(to_of_node(dev->fwnode), "riscv,num-sources", + &priv->nr_irqs); + if (rc) { + dev_err(dev, "failed to get number of interrupt sources\n"); + return rc; + } + + /* + * Find out number of IDCs based on parent interrupts + * + * If "msi-parent" property is present then we ignore the + * APLIC IDCs which forces the APLIC driver to use MSI mode. + */ + if (!of_property_present(to_of_node(dev->fwnode), "msi-parent")) { + while (!of_irq_parse_one(to_of_node(dev->fwnode), priv->nr_idcs, &parent)) + priv->nr_idcs++; + } + + /* Setup initial state APLIC interrupts */ + aplic_init_hw_irqs(priv); + + return 0; +} + +static int aplic_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + bool msi_mode = false; + void __iomem *regs; + int rc; + + /* Map the MMIO registers */ + regs = devm_platform_ioremap_resource(pdev, 0); + if (!regs) { + dev_err(dev, "failed map MMIO registers\n"); + return -ENOMEM; + } + + /* + * If msi-parent property is present then setup APLIC MSI + * mode otherwise setup APLIC direct mode. + */ + if (is_of_node(dev->fwnode)) + msi_mode = of_property_present(to_of_node(dev->fwnode), "msi-parent"); + if (msi_mode) + rc = aplic_msi_setup(dev, regs); + else + rc = aplic_direct_setup(dev, regs); + if (rc) + dev_err(dev, "failed to setup APLIC in %s mode\n", msi_mode ? "MSI" : "direct"); + + return rc; +} + +static const struct of_device_id aplic_match[] = { + { .compatible = "riscv,aplic" }, + {} +}; + +static struct platform_driver aplic_driver = { + .driver = { + .name = "riscv-aplic", + .of_match_table = aplic_match, + }, + .probe = aplic_probe, +}; +builtin_platform_driver(aplic_driver); diff --git a/drivers/irqchip/irq-riscv-aplic-main.h b/drivers/irqchip/irq-riscv-aplic-main.h new file mode 100644 index 000000000000..4393927d8c80 --- /dev/null +++ b/drivers/irqchip/irq-riscv-aplic-main.h @@ -0,0 +1,52 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2021 Western Digital Corporation or its affiliates. + * Copyright (C) 2022 Ventana Micro Systems Inc. + */ + +#ifndef _IRQ_RISCV_APLIC_MAIN_H +#define _IRQ_RISCV_APLIC_MAIN_H + +#include <linux/device.h> +#include <linux/io.h> +#include <linux/irq.h> +#include <linux/irqdomain.h> +#include <linux/fwnode.h> + +#define APLIC_DEFAULT_PRIORITY 1 + +struct aplic_msicfg { + phys_addr_t base_ppn; + u32 hhxs; + u32 hhxw; + u32 lhxs; + u32 lhxw; +}; + +struct aplic_priv { + struct device *dev; + u32 gsi_base; + u32 nr_irqs; + u32 nr_idcs; + void __iomem *regs; + struct aplic_msicfg msicfg; +}; + +void aplic_irq_unmask(struct irq_data *d); +void aplic_irq_mask(struct irq_data *d); +int aplic_irq_set_type(struct irq_data *d, unsigned int type); +int aplic_irqdomain_translate(struct irq_fwspec *fwspec, u32 gsi_base, + unsigned long *hwirq, unsigned int *type); +void aplic_init_hw_global(struct aplic_priv *priv, bool msi_mode); +int aplic_setup_priv(struct aplic_priv *priv, struct device *dev, void __iomem *regs); +int aplic_direct_setup(struct device *dev, void __iomem *regs); +#ifdef CONFIG_RISCV_APLIC_MSI +int aplic_msi_setup(struct device *dev, void __iomem *regs); +#else +static inline int aplic_msi_setup(struct device *dev, void __iomem *regs) +{ + return -ENODEV; +} +#endif + +#endif diff --git a/drivers/irqchip/irq-riscv-aplic-msi.c b/drivers/irqchip/irq-riscv-aplic-msi.c new file mode 100644 index 000000000000..028444af48bd --- /dev/null +++ b/drivers/irqchip/irq-riscv-aplic-msi.c @@ -0,0 +1,257 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2021 Western Digital Corporation or its affiliates. + * Copyright (C) 2022 Ventana Micro Systems Inc. + */ + +#include <linux/bitfield.h> +#include <linux/bitops.h> +#include <linux/cpu.h> +#include <linux/interrupt.h> +#include <linux/irqchip.h> +#include <linux/irqchip/riscv-aplic.h> +#include <linux/irqchip/riscv-imsic.h> +#include <linux/module.h> +#include <linux/msi.h> +#include <linux/of_irq.h> +#include <linux/platform_device.h> +#include <linux/printk.h> +#include <linux/smp.h> + +#include "irq-riscv-aplic-main.h" + +static void aplic_msi_irq_mask(struct irq_data *d) +{ + aplic_irq_mask(d); + irq_chip_mask_parent(d); +} + +static void aplic_msi_irq_unmask(struct irq_data *d) +{ + irq_chip_unmask_parent(d); + aplic_irq_unmask(d); +} + +static void aplic_msi_irq_eoi(struct irq_data *d) +{ + struct aplic_priv *priv = irq_data_get_irq_chip_data(d); + + /* + * EOI handling is required only for level-triggered interrupts + * when APLIC is in MSI mode. + */ + + switch (irqd_get_trigger_type(d)) { + case IRQ_TYPE_LEVEL_LOW: + case IRQ_TYPE_LEVEL_HIGH: + /* + * The section "4.9.2 Special consideration for level-sensitive interrupt + * sources" of the RISC-V AIA specification says: + * + * A second option is for the interrupt service routine to write the + * APLIC’s source identity number for the interrupt to the domain’s + * setipnum register just before exiting. This will cause the interrupt’s + * pending bit to be set to one again if the source is still asserting + * an interrupt, but not if the source is not asserting an interrupt. + */ + writel(d->hwirq, priv->regs + APLIC_SETIPNUM_LE); + break; + } +} + +static void aplic_msi_write_msg(struct irq_data *d, struct msi_msg *msg) +{ + unsigned int group_index, hart_index, guest_index, val; + struct aplic_priv *priv = irq_data_get_irq_chip_data(d); + struct aplic_msicfg *mc = &priv->msicfg; + phys_addr_t tppn, tbppn, msg_addr; + void __iomem *target; + + /* For zeroed MSI, simply write zero into the target register */ + if (!msg->address_hi && !msg->address_lo && !msg->data) { + target = priv->regs + APLIC_TARGET_BASE; + target += (d->hwirq - 1) * sizeof(u32); + writel(0, target); + return; + } + + /* Sanity check on message data */ + WARN_ON(msg->data > APLIC_TARGET_EIID_MASK); + + /* Compute target MSI address */ + msg_addr = (((u64)msg->address_hi) << 32) | msg->address_lo; + tppn = msg_addr >> APLIC_xMSICFGADDR_PPN_SHIFT; + + /* Compute target HART Base PPN */ + tbppn = tppn; + tbppn &= ~APLIC_xMSICFGADDR_PPN_HART(mc->lhxs); + tbppn &= ~APLIC_xMSICFGADDR_PPN_LHX(mc->lhxw, mc->lhxs); + tbppn &= ~APLIC_xMSICFGADDR_PPN_HHX(mc->hhxw, mc->hhxs); + WARN_ON(tbppn != mc->base_ppn); + + /* Compute target group and hart indexes */ + group_index = (tppn >> APLIC_xMSICFGADDR_PPN_HHX_SHIFT(mc->hhxs)) & + APLIC_xMSICFGADDR_PPN_HHX_MASK(mc->hhxw); + hart_index = (tppn >> APLIC_xMSICFGADDR_PPN_LHX_SHIFT(mc->lhxs)) & + APLIC_xMSICFGADDR_PPN_LHX_MASK(mc->lhxw); + hart_index |= (group_index << mc->lhxw); + WARN_ON(hart_index > APLIC_TARGET_HART_IDX_MASK); + + /* Compute target guest index */ + guest_index = tppn & APLIC_xMSICFGADDR_PPN_HART(mc->lhxs); + WARN_ON(guest_index > APLIC_TARGET_GUEST_IDX_MASK); + + /* Update IRQ TARGET register */ + target = priv->regs + APLIC_TARGET_BASE; + target += (d->hwirq - 1) * sizeof(u32); + val = FIELD_PREP(APLIC_TARGET_HART_IDX, hart_index); + val |= FIELD_PREP(APLIC_TARGET_GUEST_IDX, guest_index); + val |= FIELD_PREP(APLIC_TARGET_EIID, msg->data); + writel(val, target); +} + +static void aplic_msi_set_desc(msi_alloc_info_t *arg, struct msi_desc *desc) +{ + arg->desc = desc; + arg->hwirq = (u32)desc->data.icookie.value; +} + +static int aplic_msi_translate(struct irq_domain *d, struct irq_fwspec *fwspec, + unsigned long *hwirq, unsigned int *type) +{ + struct msi_domain_info *info = d->host_data; + struct aplic_priv *priv = info->data; + + return aplic_irqdomain_translate(fwspec, priv->gsi_base, hwirq, type); +} + +static const struct msi_domain_template aplic_msi_template = { + .chip = { + .name = "APLIC-MSI", + .irq_mask = aplic_msi_irq_mask, + .irq_unmask = aplic_msi_irq_unmask, + .irq_set_type = aplic_irq_set_type, + .irq_eoi = aplic_msi_irq_eoi, +#ifdef CONFIG_SMP + .irq_set_affinity = irq_chip_set_affinity_parent, +#endif + .irq_write_msi_msg = aplic_msi_write_msg, + .flags = IRQCHIP_SET_TYPE_MASKED | + IRQCHIP_SKIP_SET_WAKE | + IRQCHIP_MASK_ON_SUSPEND, + }, + + .ops = { + .set_desc = aplic_msi_set_desc, + .msi_translate = aplic_msi_translate, + }, + + .info = { + .bus_token = DOMAIN_BUS_WIRED_TO_MSI, + .flags = MSI_FLAG_USE_DEV_FWNODE, + .handler = handle_fasteoi_irq, + .handler_name = "fasteoi", + }, +}; + +int aplic_msi_setup(struct device *dev, void __iomem *regs) +{ + const struct imsic_global_config *imsic_global; + struct aplic_priv *priv; + struct aplic_msicfg *mc; + phys_addr_t pa; + int rc; + + priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL); + if (!priv) + return -ENOMEM; + + rc = aplic_setup_priv(priv, dev, regs); + if (rc) { + dev_err(dev, "failed to create APLIC context\n"); + return rc; + } + mc = &priv->msicfg; + + /* + * The APLIC outgoing MSI config registers assume target MSI + * controller to be RISC-V AIA IMSIC controller. + */ + imsic_global = imsic_get_global_config(); + if (!imsic_global) { + dev_err(dev, "IMSIC global config not found\n"); + return -ENODEV; + } + + /* Find number of guest index bits (LHXS) */ + mc->lhxs = imsic_global->guest_index_bits; + if (APLIC_xMSICFGADDRH_LHXS_MASK < mc->lhxs) { + dev_err(dev, "IMSIC guest index bits big for APLIC LHXS\n"); + return -EINVAL; + } + + /* Find number of HART index bits (LHXW) */ + mc->lhxw = imsic_global->hart_index_bits; + if (APLIC_xMSICFGADDRH_LHXW_MASK < mc->lhxw) { + dev_err(dev, "IMSIC hart index bits big for APLIC LHXW\n"); + return -EINVAL; + } + + /* Find number of group index bits (HHXW) */ + mc->hhxw = imsic_global->group_index_bits; + if (APLIC_xMSICFGADDRH_HHXW_MASK < mc->hhxw) { + dev_err(dev, "IMSIC group index bits big for APLIC HHXW\n"); + return -EINVAL; + } + + /* Find first bit position of group index (HHXS) */ + mc->hhxs = imsic_global->group_index_shift; + if (mc->hhxs < (2 * APLIC_xMSICFGADDR_PPN_SHIFT)) { + dev_err(dev, "IMSIC group index shift should be >= %d\n", + (2 * APLIC_xMSICFGADDR_PPN_SHIFT)); + return -EINVAL; + } + mc->hhxs -= (2 * APLIC_xMSICFGADDR_PPN_SHIFT); + if (APLIC_xMSICFGADDRH_HHXS_MASK < mc->hhxs) { + dev_err(dev, "IMSIC group index shift big for APLIC HHXS\n"); + return -EINVAL; + } + + /* Compute PPN base */ + mc->base_ppn = imsic_global->base_addr >> APLIC_xMSICFGADDR_PPN_SHIFT; + mc->base_ppn &= ~APLIC_xMSICFGADDR_PPN_HART(mc->lhxs); + mc->base_ppn &= ~APLIC_xMSICFGADDR_PPN_LHX(mc->lhxw, mc->lhxs); + mc->base_ppn &= ~APLIC_xMSICFGADDR_PPN_HHX(mc->hhxw, mc->hhxs); + + /* Setup global config and interrupt delivery */ + aplic_init_hw_global(priv, true); + + /* Set the APLIC device MSI domain if not available */ + if (!dev_get_msi_domain(dev)) { + /* + * The device MSI domain for OF devices is only set at the + * time of populating/creating OF device. If the device MSI + * domain is discovered later after the OF device is created + * then we need to set it explicitly before using any platform + * MSI functions. + * + * In case of APLIC device, the parent MSI domain is always + * IMSIC and the IMSIC MSI domains are created later through + * the platform driver probing so we set it explicitly here. + */ + if (is_of_node(dev->fwnode)) + of_msi_configure(dev, to_of_node(dev->fwnode)); + } + + if (!msi_create_device_irq_domain(dev, MSI_DEFAULT_DOMAIN, &aplic_msi_template, + priv->nr_irqs + 1, priv, priv)) { + dev_err(dev, "failed to create MSI irq domain\n"); + return -ENOMEM; + } + + /* Advertise the interrupt controller */ + pa = priv->msicfg.base_ppn << APLIC_xMSICFGADDR_PPN_SHIFT; + dev_info(dev, "%d interrupts forwarded to MSI base %pa\n", priv->nr_irqs, &pa); + + return 0; +} diff --git a/drivers/irqchip/irq-riscv-imsic-early.c b/drivers/irqchip/irq-riscv-imsic-early.c new file mode 100644 index 000000000000..886418ec06cb --- /dev/null +++ b/drivers/irqchip/irq-riscv-imsic-early.c @@ -0,0 +1,201 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2021 Western Digital Corporation or its affiliates. + * Copyright (C) 2022 Ventana Micro Systems Inc. + */ + +#define pr_fmt(fmt) "riscv-imsic: " fmt +#include <linux/cpu.h> +#include <linux/interrupt.h> +#include <linux/io.h> +#include <linux/irq.h> +#include <linux/irqchip.h> +#include <linux/irqchip/chained_irq.h> +#include <linux/module.h> +#include <linux/spinlock.h> +#include <linux/smp.h> + +#include "irq-riscv-imsic-state.h" + +static int imsic_parent_irq; + +#ifdef CONFIG_SMP +static void imsic_ipi_send(unsigned int cpu) +{ + struct imsic_local_config *local = per_cpu_ptr(imsic->global.local, cpu); + + writel_relaxed(IMSIC_IPI_ID, local->msi_va); +} + +static void imsic_ipi_starting_cpu(void) +{ + /* Enable IPIs for current CPU. */ + __imsic_id_set_enable(IMSIC_IPI_ID); +} + +static void imsic_ipi_dying_cpu(void) +{ + /* Disable IPIs for current CPU. */ + __imsic_id_clear_enable(IMSIC_IPI_ID); +} + +static int __init imsic_ipi_domain_init(void) +{ + int virq; + + /* Create IMSIC IPI multiplexing */ + virq = ipi_mux_create(IMSIC_NR_IPI, imsic_ipi_send); + if (virq <= 0) + return virq < 0 ? virq : -ENOMEM; + + /* Set vIRQ range */ + riscv_ipi_set_virq_range(virq, IMSIC_NR_IPI, true); + + /* Announce that IMSIC is providing IPIs */ + pr_info("%pfwP: providing IPIs using interrupt %d\n", imsic->fwnode, IMSIC_IPI_ID); + + return 0; +} +#else +static void imsic_ipi_starting_cpu(void) { } +static void imsic_ipi_dying_cpu(void) { } +static int __init imsic_ipi_domain_init(void) { return 0; } +#endif + +/* + * To handle an interrupt, we read the TOPEI CSR and write zero in one + * instruction. If TOPEI CSR is non-zero then we translate TOPEI.ID to + * Linux interrupt number and let Linux IRQ subsystem handle it. + */ +static void imsic_handle_irq(struct irq_desc *desc) +{ + struct irq_chip *chip = irq_desc_get_chip(desc); + int err, cpu = smp_processor_id(); + struct imsic_vector *vec; + unsigned long local_id; + + chained_irq_enter(chip, desc); + + while ((local_id = csr_swap(CSR_TOPEI, 0))) { + local_id >>= TOPEI_ID_SHIFT; + + if (local_id == IMSIC_IPI_ID) { + if (IS_ENABLED(CONFIG_SMP)) + ipi_mux_process(); + continue; + } + + if (unlikely(!imsic->base_domain)) + continue; + + vec = imsic_vector_from_local_id(cpu, local_id); + if (!vec) { + pr_warn_ratelimited("vector not found for local ID 0x%lx\n", local_id); + continue; + } + + err = generic_handle_domain_irq(imsic->base_domain, vec->hwirq); + if (unlikely(err)) + pr_warn_ratelimited("hwirq 0x%x mapping not found\n", vec->hwirq); + } + + chained_irq_exit(chip, desc); +} + +static int imsic_starting_cpu(unsigned int cpu) +{ + /* Mark per-CPU IMSIC state as online */ + imsic_state_online(); + + /* Enable per-CPU parent interrupt */ + enable_percpu_irq(imsic_parent_irq, irq_get_trigger_type(imsic_parent_irq)); + + /* Setup IPIs */ + imsic_ipi_starting_cpu(); + + /* + * Interrupts identities might have been enabled/disabled while + * this CPU was not running so sync-up local enable/disable state. + */ + imsic_local_sync_all(); + + /* Enable local interrupt delivery */ + imsic_local_delivery(true); + + return 0; +} + +static int imsic_dying_cpu(unsigned int cpu) +{ + /* Cleanup IPIs */ + imsic_ipi_dying_cpu(); + + /* Mark per-CPU IMSIC state as offline */ + imsic_state_offline(); + + return 0; +} + +static int __init imsic_early_probe(struct fwnode_handle *fwnode) +{ + struct irq_domain *domain; + int rc; + + /* Find parent domain and register chained handler */ + domain = irq_find_matching_fwnode(riscv_get_intc_hwnode(), DOMAIN_BUS_ANY); + if (!domain) { + pr_err("%pfwP: Failed to find INTC domain\n", fwnode); + return -ENOENT; + } + imsic_parent_irq = irq_create_mapping(domain, RV_IRQ_EXT); + if (!imsic_parent_irq) { + pr_err("%pfwP: Failed to create INTC mapping\n", fwnode); + return -ENOENT; + } + + /* Initialize IPI domain */ + rc = imsic_ipi_domain_init(); + if (rc) { + pr_err("%pfwP: Failed to initialize IPI domain\n", fwnode); + return rc; + } + + /* Setup chained handler to the parent domain interrupt */ + irq_set_chained_handler(imsic_parent_irq, imsic_handle_irq); + + /* + * Setup cpuhp state (must be done after setting imsic_parent_irq) + * + * Don't disable per-CPU IMSIC file when CPU goes offline + * because this affects IPI and the masking/unmasking of + * virtual IPIs is done via generic IPI-Mux + */ + cpuhp_setup_state(CPUHP_AP_IRQ_RISCV_IMSIC_STARTING, "irqchip/riscv/imsic:starting", + imsic_starting_cpu, imsic_dying_cpu); + + return 0; +} + +static int __init imsic_early_dt_init(struct device_node *node, struct device_node *parent) +{ + struct fwnode_handle *fwnode = &node->fwnode; + int rc; + + /* Setup IMSIC state */ + rc = imsic_setup_state(fwnode); + if (rc) { + pr_err("%pfwP: failed to setup state (error %d)\n", fwnode, rc); + return rc; + } + + /* Do early setup of IPIs */ + rc = imsic_early_probe(fwnode); + if (rc) + return rc; + + /* Ensure that OF platform device gets probed */ + of_node_clear_flag(node, OF_POPULATED); + return 0; +} + +IRQCHIP_DECLARE(riscv_imsic, "riscv,imsics", imsic_early_dt_init); diff --git a/drivers/irqchip/irq-riscv-imsic-platform.c b/drivers/irqchip/irq-riscv-imsic-platform.c new file mode 100644 index 000000000000..11723a763c10 --- /dev/null +++ b/drivers/irqchip/irq-riscv-imsic-platform.c @@ -0,0 +1,375 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2021 Western Digital Corporation or its affiliates. + * Copyright (C) 2022 Ventana Micro Systems Inc. + */ + +#define pr_fmt(fmt) "riscv-imsic: " fmt +#include <linux/bitmap.h> +#include <linux/cpu.h> +#include <linux/interrupt.h> +#include <linux/io.h> +#include <linux/irq.h> +#include <linux/irqchip.h> +#include <linux/irqdomain.h> +#include <linux/module.h> +#include <linux/msi.h> +#include <linux/pci.h> +#include <linux/platform_device.h> +#include <linux/spinlock.h> +#include <linux/smp.h> + +#include "irq-riscv-imsic-state.h" + +static bool imsic_cpu_page_phys(unsigned int cpu, unsigned int guest_index, + phys_addr_t *out_msi_pa) +{ + struct imsic_global_config *global; + struct imsic_local_config *local; + + global = &imsic->global; + local = per_cpu_ptr(global->local, cpu); + + if (BIT(global->guest_index_bits) <= guest_index) + return false; + + if (out_msi_pa) + *out_msi_pa = local->msi_pa + (guest_index * IMSIC_MMIO_PAGE_SZ); + + return true; +} + +static void imsic_irq_mask(struct irq_data *d) +{ + imsic_vector_mask(irq_data_get_irq_chip_data(d)); +} + +static void imsic_irq_unmask(struct irq_data *d) +{ + imsic_vector_unmask(irq_data_get_irq_chip_data(d)); +} + +static int imsic_irq_retrigger(struct irq_data *d) +{ + struct imsic_vector *vec = irq_data_get_irq_chip_data(d); + struct imsic_local_config *local; + + if (WARN_ON(!vec)) + return -ENOENT; + + local = per_cpu_ptr(imsic->global.local, vec->cpu); + writel_relaxed(vec->local_id, local->msi_va); + return 0; +} + +static void imsic_irq_compose_vector_msg(struct imsic_vector *vec, struct msi_msg *msg) +{ + phys_addr_t msi_addr; + + if (WARN_ON(!vec)) + return; + + if (WARN_ON(!imsic_cpu_page_phys(vec->cpu, 0, &msi_addr))) + return; + + msg->address_hi = upper_32_bits(msi_addr); + msg->address_lo = lower_32_bits(msi_addr); + msg->data = vec->local_id; +} + +static void imsic_irq_compose_msg(struct irq_data *d, struct msi_msg *msg) +{ + imsic_irq_compose_vector_msg(irq_data_get_irq_chip_data(d), msg); +} + +#ifdef CONFIG_SMP +static void imsic_msi_update_msg(struct irq_data *d, struct imsic_vector *vec) +{ + struct msi_msg msg = { }; + + imsic_irq_compose_vector_msg(vec, &msg); + irq_data_get_irq_chip(d)->irq_write_msi_msg(d, &msg); +} + +static int imsic_irq_set_affinity(struct irq_data *d, const struct cpumask *mask_val, + bool force) +{ + struct imsic_vector *old_vec, *new_vec; + struct irq_data *pd = d->parent_data; + + old_vec = irq_data_get_irq_chip_data(pd); + if (WARN_ON(!old_vec)) + return -ENOENT; + + /* If old vector cpu belongs to the target cpumask then do nothing */ + if (cpumask_test_cpu(old_vec->cpu, mask_val)) + return IRQ_SET_MASK_OK_DONE; + + /* If move is already in-flight then return failure */ + if (imsic_vector_get_move(old_vec)) + return -EBUSY; + + /* Get a new vector on the desired set of CPUs */ + new_vec = imsic_vector_alloc(old_vec->hwirq, mask_val); + if (!new_vec) + return -ENOSPC; + + /* Point device to the new vector */ + imsic_msi_update_msg(d, new_vec); + + /* Update irq descriptors with the new vector */ + pd->chip_data = new_vec; + + /* Update effective affinity of parent irq data */ + irq_data_update_effective_affinity(pd, cpumask_of(new_vec->cpu)); + + /* Move state of the old vector to the new vector */ + imsic_vector_move(old_vec, new_vec); + + return IRQ_SET_MASK_OK_DONE; +} +#endif + +static struct irq_chip imsic_irq_base_chip = { + .name = "IMSIC", + .irq_mask = imsic_irq_mask, + .irq_unmask = imsic_irq_unmask, + .irq_retrigger = imsic_irq_retrigger, + .irq_compose_msi_msg = imsic_irq_compose_msg, + .flags = IRQCHIP_SKIP_SET_WAKE | + IRQCHIP_MASK_ON_SUSPEND, +}; + +static int imsic_irq_domain_alloc(struct irq_domain *domain, unsigned int virq, + unsigned int nr_irqs, void *args) +{ + struct imsic_vector *vec; + + /* Multi-MSI is not supported yet. */ + if (nr_irqs > 1) + return -EOPNOTSUPP; + + vec = imsic_vector_alloc(virq, cpu_online_mask); + if (!vec) + return -ENOSPC; + + irq_domain_set_info(domain, virq, virq, &imsic_irq_base_chip, vec, + handle_simple_irq, NULL, NULL); + irq_set_noprobe(virq); + irq_set_affinity(virq, cpu_online_mask); + irq_data_update_effective_affinity(irq_get_irq_data(virq), cpumask_of(vec->cpu)); + + return 0; +} + +static void imsic_irq_domain_free(struct irq_domain *domain, unsigned int virq, + unsigned int nr_irqs) +{ + struct irq_data *d = irq_domain_get_irq_data(domain, virq); + + imsic_vector_free(irq_data_get_irq_chip_data(d)); + irq_domain_free_irqs_parent(domain, virq, nr_irqs); +} + +static int imsic_irq_domain_select(struct irq_domain *domain, struct irq_fwspec *fwspec, + enum irq_domain_bus_token bus_token) +{ + const struct msi_parent_ops *ops = domain->msi_parent_ops; + u32 busmask = BIT(bus_token); + + if (fwspec->fwnode != domain->fwnode || fwspec->param_count != 0) + return 0; + + /* Handle pure domain searches */ + if (bus_token == ops->bus_select_token) + return 1; + + return !!(ops->bus_select_mask & busmask); +} + +#ifdef CONFIG_GENERIC_IRQ_DEBUGFS +static void imsic_irq_debug_show(struct seq_file *m, struct irq_domain *d, + struct irq_data *irqd, int ind) +{ + if (!irqd) { + imsic_vector_debug_show_summary(m, ind); + return; + } + + imsic_vector_debug_show(m, irq_data_get_irq_chip_data(irqd), ind); +} +#endif + +static const struct irq_domain_ops imsic_base_domain_ops = { + .alloc = imsic_irq_domain_alloc, + .free = imsic_irq_domain_free, + .select = imsic_irq_domain_select, +#ifdef CONFIG_GENERIC_IRQ_DEBUGFS + .debug_show = imsic_irq_debug_show, +#endif +}; + +#ifdef CONFIG_RISCV_IMSIC_PCI + +static void imsic_pci_mask_irq(struct irq_data *d) +{ + pci_msi_mask_irq(d); + irq_chip_mask_parent(d); +} + +static void imsic_pci_unmask_irq(struct irq_data *d) +{ + irq_chip_unmask_parent(d); + pci_msi_unmask_irq(d); +} + +#define MATCH_PCI_MSI BIT(DOMAIN_BUS_PCI_MSI) + +#else + +#define MATCH_PCI_MSI 0 + +#endif + +static bool imsic_init_dev_msi_info(struct device *dev, + struct irq_domain *domain, + struct irq_domain *real_parent, + struct msi_domain_info *info) +{ + const struct msi_parent_ops *pops = real_parent->msi_parent_ops; + + /* MSI parent domain specific settings */ + switch (real_parent->bus_token) { + case DOMAIN_BUS_NEXUS: + if (WARN_ON_ONCE(domain != real_parent)) + return false; +#ifdef CONFIG_SMP + info->chip->irq_set_affinity = imsic_irq_set_affinity; +#endif + break; + default: + WARN_ON_ONCE(1); + return false; + } + + /* Is the target supported? */ + switch (info->bus_token) { +#ifdef CONFIG_RISCV_IMSIC_PCI + case DOMAIN_BUS_PCI_DEVICE_MSI: + case DOMAIN_BUS_PCI_DEVICE_MSIX: + info->chip->irq_mask = imsic_pci_mask_irq; + info->chip->irq_unmask = imsic_pci_unmask_irq; + break; +#endif + case DOMAIN_BUS_DEVICE_MSI: + /* + * Per-device MSI should never have any MSI feature bits + * set. It's sole purpose is to create a dumb interrupt + * chip which has a device specific irq_write_msi_msg() + * callback. + */ + if (WARN_ON_ONCE(info->flags)) + return false; + + /* Core managed MSI descriptors */ + info->flags |= MSI_FLAG_ALLOC_SIMPLE_MSI_DESCS | + MSI_FLAG_FREE_MSI_DESCS; + break; + case DOMAIN_BUS_WIRED_TO_MSI: + break; + default: + WARN_ON_ONCE(1); + return false; + } + + /* Use hierarchial chip operations re-trigger */ + info->chip->irq_retrigger = irq_chip_retrigger_hierarchy; + + /* + * Mask out the domain specific MSI feature flags which are not + * supported by the real parent. + */ + info->flags &= pops->supported_flags; + + /* Enforce the required flags */ + info->flags |= pops->required_flags; + + return true; +} + +#define MATCH_PLATFORM_MSI BIT(DOMAIN_BUS_PLATFORM_MSI) + +static const struct msi_parent_ops imsic_msi_parent_ops = { + .supported_flags = MSI_GENERIC_FLAGS_MASK | + MSI_FLAG_PCI_MSIX, + .required_flags = MSI_FLAG_USE_DEF_DOM_OPS | + MSI_FLAG_USE_DEF_CHIP_OPS, + .bus_select_token = DOMAIN_BUS_NEXUS, + .bus_select_mask = MATCH_PCI_MSI | MATCH_PLATFORM_MSI, + .init_dev_msi_info = imsic_init_dev_msi_info, +}; + +int imsic_irqdomain_init(void) +{ + struct imsic_global_config *global; + + if (!imsic || !imsic->fwnode) { + pr_err("early driver not probed\n"); + return -ENODEV; + } + + if (imsic->base_domain) { + pr_err("%pfwP: irq domain already created\n", imsic->fwnode); + return -ENODEV; + } + + /* Create Base IRQ domain */ + imsic->base_domain = irq_domain_create_tree(imsic->fwnode, + &imsic_base_domain_ops, imsic); + if (!imsic->base_domain) { + pr_err("%pfwP: failed to create IMSIC base domain\n", imsic->fwnode); + return -ENOMEM; + } + imsic->base_domain->flags |= IRQ_DOMAIN_FLAG_MSI_PARENT; + imsic->base_domain->msi_parent_ops = &imsic_msi_parent_ops; + + irq_domain_update_bus_token(imsic->base_domain, DOMAIN_BUS_NEXUS); + + global = &imsic->global; + pr_info("%pfwP: hart-index-bits: %d, guest-index-bits: %d\n", + imsic->fwnode, global->hart_index_bits, global->guest_index_bits); + pr_info("%pfwP: group-index-bits: %d, group-index-shift: %d\n", + imsic->fwnode, global->group_index_bits, global->group_index_shift); + pr_info("%pfwP: per-CPU IDs %d at base PPN %pa\n", + imsic->fwnode, global->nr_ids, &global->base_addr); + pr_info("%pfwP: total %d interrupts available\n", + imsic->fwnode, num_possible_cpus() * (global->nr_ids - 1)); + + return 0; +} + +static int imsic_platform_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + + if (imsic && imsic->fwnode != dev->fwnode) { + dev_err(dev, "fwnode mismatch\n"); + return -ENODEV; + } + + return imsic_irqdomain_init(); +} + +static const struct of_device_id imsic_platform_match[] = { + { .compatible = "riscv,imsics" }, + {} +}; + +static struct platform_driver imsic_platform_driver = { + .driver = { + .name = "riscv-imsic", + .of_match_table = imsic_platform_match, + }, + .probe = imsic_platform_probe, +}; +builtin_platform_driver(imsic_platform_driver); diff --git a/drivers/irqchip/irq-riscv-imsic-state.c b/drivers/irqchip/irq-riscv-imsic-state.c new file mode 100644 index 000000000000..5479f872e62b --- /dev/null +++ b/drivers/irqchip/irq-riscv-imsic-state.c @@ -0,0 +1,865 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2021 Western Digital Corporation or its affiliates. + * Copyright (C) 2022 Ventana Micro Systems Inc. + */ + +#define pr_fmt(fmt) "riscv-imsic: " fmt +#include <linux/cpu.h> +#include <linux/bitmap.h> +#include <linux/interrupt.h> +#include <linux/irq.h> +#include <linux/module.h> +#include <linux/of.h> +#include <linux/of_address.h> +#include <linux/of_irq.h> +#include <linux/seq_file.h> +#include <linux/spinlock.h> +#include <linux/smp.h> +#include <asm/hwcap.h> + +#include "irq-riscv-imsic-state.h" + +#define IMSIC_DISABLE_EIDELIVERY 0 +#define IMSIC_ENABLE_EIDELIVERY 1 +#define IMSIC_DISABLE_EITHRESHOLD 1 +#define IMSIC_ENABLE_EITHRESHOLD 0 + +static inline void imsic_csr_write(unsigned long reg, unsigned long val) +{ + csr_write(CSR_ISELECT, reg); + csr_write(CSR_IREG, val); +} + +static inline unsigned long imsic_csr_read(unsigned long reg) +{ + csr_write(CSR_ISELECT, reg); + return csr_read(CSR_IREG); +} + +static inline unsigned long imsic_csr_read_clear(unsigned long reg, unsigned long val) +{ + csr_write(CSR_ISELECT, reg); + return csr_read_clear(CSR_IREG, val); +} + +static inline void imsic_csr_set(unsigned long reg, unsigned long val) +{ + csr_write(CSR_ISELECT, reg); + csr_set(CSR_IREG, val); +} + +static inline void imsic_csr_clear(unsigned long reg, unsigned long val) +{ + csr_write(CSR_ISELECT, reg); + csr_clear(CSR_IREG, val); +} + +struct imsic_priv *imsic; + +const struct imsic_global_config *imsic_get_global_config(void) +{ + return imsic ? &imsic->global : NULL; +} +EXPORT_SYMBOL_GPL(imsic_get_global_config); + +static bool __imsic_eix_read_clear(unsigned long id, bool pend) +{ + unsigned long isel, imask; + + isel = id / BITS_PER_LONG; + isel *= BITS_PER_LONG / IMSIC_EIPx_BITS; + isel += pend ? IMSIC_EIP0 : IMSIC_EIE0; + imask = BIT(id & (__riscv_xlen - 1)); + + return !!(imsic_csr_read_clear(isel, imask) & imask); +} + +static inline bool __imsic_id_read_clear_enabled(unsigned long id) +{ + return __imsic_eix_read_clear(id, false); +} + +static inline bool __imsic_id_read_clear_pending(unsigned long id) +{ + return __imsic_eix_read_clear(id, true); +} + +void __imsic_eix_update(unsigned long base_id, unsigned long num_id, bool pend, bool val) +{ + unsigned long id = base_id, last_id = base_id + num_id; + unsigned long i, isel, ireg; + + while (id < last_id) { + isel = id / BITS_PER_LONG; + isel *= BITS_PER_LONG / IMSIC_EIPx_BITS; + isel += pend ? IMSIC_EIP0 : IMSIC_EIE0; + + /* + * Prepare the ID mask to be programmed in the + * IMSIC EIEx and EIPx registers. These registers + * are XLEN-wide and we must not touch IDs which + * are < base_id and >= (base_id + num_id). + */ + ireg = 0; + for (i = id & (__riscv_xlen - 1); id < last_id && i < __riscv_xlen; i++) { + ireg |= BIT(i); + id++; + } + + /* + * The IMSIC EIEx and EIPx registers are indirectly + * accessed via using ISELECT and IREG CSRs so we + * need to access these CSRs without getting preempted. + * + * All existing users of this function call this + * function with local IRQs disabled so we don't + * need to do anything special here. + */ + if (val) + imsic_csr_set(isel, ireg); + else + imsic_csr_clear(isel, ireg); + } +} + +static void __imsic_local_sync(struct imsic_local_priv *lpriv) +{ + struct imsic_local_config *mlocal; + struct imsic_vector *vec, *mvec; + int i; + + lockdep_assert_held(&lpriv->lock); + + for_each_set_bit(i, lpriv->dirty_bitmap, imsic->global.nr_ids + 1) { + if (!i || i == IMSIC_IPI_ID) + goto skip; + vec = &lpriv->vectors[i]; + + if (READ_ONCE(vec->enable)) + __imsic_id_set_enable(i); + else + __imsic_id_clear_enable(i); + + /* + * If the ID was being moved to a new ID on some other CPU + * then we can get a MSI during the movement so check the + * ID pending bit and re-trigger the new ID on other CPU + * using MMIO write. + */ + mvec = READ_ONCE(vec->move); + WRITE_ONCE(vec->move, NULL); + if (mvec && mvec != vec) { + if (__imsic_id_read_clear_pending(i)) { + mlocal = per_cpu_ptr(imsic->global.local, mvec->cpu); + writel_relaxed(mvec->local_id, mlocal->msi_va); + } + + imsic_vector_free(&lpriv->vectors[i]); + } + +skip: + bitmap_clear(lpriv->dirty_bitmap, i, 1); + } +} + +void imsic_local_sync_all(void) +{ + struct imsic_local_priv *lpriv = this_cpu_ptr(imsic->lpriv); + unsigned long flags; + + raw_spin_lock_irqsave(&lpriv->lock, flags); + bitmap_fill(lpriv->dirty_bitmap, imsic->global.nr_ids + 1); + __imsic_local_sync(lpriv); + raw_spin_unlock_irqrestore(&lpriv->lock, flags); +} + +void imsic_local_delivery(bool enable) +{ + if (enable) { + imsic_csr_write(IMSIC_EITHRESHOLD, IMSIC_ENABLE_EITHRESHOLD); + imsic_csr_write(IMSIC_EIDELIVERY, IMSIC_ENABLE_EIDELIVERY); + return; + } + + imsic_csr_write(IMSIC_EIDELIVERY, IMSIC_DISABLE_EIDELIVERY); + imsic_csr_write(IMSIC_EITHRESHOLD, IMSIC_DISABLE_EITHRESHOLD); +} + +#ifdef CONFIG_SMP +static void imsic_local_timer_callback(struct timer_list *timer) +{ + struct imsic_local_priv *lpriv = this_cpu_ptr(imsic->lpriv); + unsigned long flags; + + raw_spin_lock_irqsave(&lpriv->lock, flags); + __imsic_local_sync(lpriv); + raw_spin_unlock_irqrestore(&lpriv->lock, flags); +} + +static void __imsic_remote_sync(struct imsic_local_priv *lpriv, unsigned int cpu) +{ + lockdep_assert_held(&lpriv->lock); + + /* + * The spinlock acquire/release semantics ensure that changes + * to vector enable, vector move and dirty bitmap are visible + * to the target CPU. + */ + + /* + * We schedule a timer on the target CPU if the target CPU is not + * same as the current CPU. An offline CPU will unconditionally + * synchronize IDs through imsic_starting_cpu() when the + * CPU is brought up. + */ + if (cpu_online(cpu)) { + if (cpu == smp_processor_id()) { + __imsic_local_sync(lpriv); + return; + } + + if (!timer_pending(&lpriv->timer)) { + lpriv->timer.expires = jiffies + 1; + add_timer_on(&lpriv->timer, cpu); + } + } +} +#else +static void __imsic_remote_sync(struct imsic_local_priv *lpriv, unsigned int cpu) +{ + lockdep_assert_held(&lpriv->lock); + __imsic_local_sync(lpriv); +} +#endif + +void imsic_vector_mask(struct imsic_vector *vec) +{ + struct imsic_local_priv *lpriv; + + lpriv = per_cpu_ptr(imsic->lpriv, vec->cpu); + if (WARN_ON_ONCE(&lpriv->vectors[vec->local_id] != vec)) + return; + + /* + * This function is called through Linux irq subsystem with + * irqs disabled so no need to save/restore irq flags. + */ + + raw_spin_lock(&lpriv->lock); + + WRITE_ONCE(vec->enable, false); + bitmap_set(lpriv->dirty_bitmap, vec->local_id, 1); + __imsic_remote_sync(lpriv, vec->cpu); + + raw_spin_unlock(&lpriv->lock); +} + +void imsic_vector_unmask(struct imsic_vector *vec) +{ + struct imsic_local_priv *lpriv; + + lpriv = per_cpu_ptr(imsic->lpriv, vec->cpu); + if (WARN_ON_ONCE(&lpriv->vectors[vec->local_id] != vec)) + return; + + /* + * This function is called through Linux irq subsystem with + * irqs disabled so no need to save/restore irq flags. + */ + + raw_spin_lock(&lpriv->lock); + + WRITE_ONCE(vec->enable, true); + bitmap_set(lpriv->dirty_bitmap, vec->local_id, 1); + __imsic_remote_sync(lpriv, vec->cpu); + + raw_spin_unlock(&lpriv->lock); +} + +static bool imsic_vector_move_update(struct imsic_local_priv *lpriv, struct imsic_vector *vec, + bool new_enable, struct imsic_vector *new_move) +{ + unsigned long flags; + bool enabled; + + raw_spin_lock_irqsave(&lpriv->lock, flags); + + /* Update enable and move details */ + enabled = READ_ONCE(vec->enable); + WRITE_ONCE(vec->enable, new_enable); + WRITE_ONCE(vec->move, new_move); + + /* Mark the vector as dirty and synchronize */ + bitmap_set(lpriv->dirty_bitmap, vec->local_id, 1); + __imsic_remote_sync(lpriv, vec->cpu); + + raw_spin_unlock_irqrestore(&lpriv->lock, flags); + + return enabled; +} + +void imsic_vector_move(struct imsic_vector *old_vec, struct imsic_vector *new_vec) +{ + struct imsic_local_priv *old_lpriv, *new_lpriv; + bool enabled; + + if (WARN_ON_ONCE(old_vec->cpu == new_vec->cpu)) + return; + + old_lpriv = per_cpu_ptr(imsic->lpriv, old_vec->cpu); + if (WARN_ON_ONCE(&old_lpriv->vectors[old_vec->local_id] != old_vec)) + return; + + new_lpriv = per_cpu_ptr(imsic->lpriv, new_vec->cpu); + if (WARN_ON_ONCE(&new_lpriv->vectors[new_vec->local_id] != new_vec)) + return; + + /* + * Move and re-trigger the new vector based on the pending + * state of the old vector because we might get a device + * interrupt on the old vector while device was being moved + * to the new vector. + */ + enabled = imsic_vector_move_update(old_lpriv, old_vec, false, new_vec); + imsic_vector_move_update(new_lpriv, new_vec, enabled, new_vec); +} + +#ifdef CONFIG_GENERIC_IRQ_DEBUGFS +void imsic_vector_debug_show(struct seq_file *m, struct imsic_vector *vec, int ind) +{ + struct imsic_local_priv *lpriv; + struct imsic_vector *mvec; + bool is_enabled; + + lpriv = per_cpu_ptr(imsic->lpriv, vec->cpu); + if (WARN_ON_ONCE(&lpriv->vectors[vec->local_id] != vec)) + return; + + is_enabled = imsic_vector_isenabled(vec); + mvec = imsic_vector_get_move(vec); + + seq_printf(m, "%*starget_cpu : %5u\n", ind, "", vec->cpu); + seq_printf(m, "%*starget_local_id : %5u\n", ind, "", vec->local_id); + seq_printf(m, "%*sis_reserved : %5u\n", ind, "", + (vec->local_id <= IMSIC_IPI_ID) ? 1 : 0); + seq_printf(m, "%*sis_enabled : %5u\n", ind, "", is_enabled ? 1 : 0); + seq_printf(m, "%*sis_move_pending : %5u\n", ind, "", mvec ? 1 : 0); + if (mvec) { + seq_printf(m, "%*smove_cpu : %5u\n", ind, "", mvec->cpu); + seq_printf(m, "%*smove_local_id : %5u\n", ind, "", mvec->local_id); + } +} + +void imsic_vector_debug_show_summary(struct seq_file *m, int ind) +{ + irq_matrix_debug_show(m, imsic->matrix, ind); +} +#endif + +struct imsic_vector *imsic_vector_from_local_id(unsigned int cpu, unsigned int local_id) +{ + struct imsic_local_priv *lpriv = per_cpu_ptr(imsic->lpriv, cpu); + + if (!lpriv || imsic->global.nr_ids < local_id) + return NULL; + + return &lpriv->vectors[local_id]; +} + +struct imsic_vector *imsic_vector_alloc(unsigned int hwirq, const struct cpumask *mask) +{ + struct imsic_vector *vec = NULL; + struct imsic_local_priv *lpriv; + unsigned long flags; + unsigned int cpu; + int local_id; + + raw_spin_lock_irqsave(&imsic->matrix_lock, flags); + local_id = irq_matrix_alloc(imsic->matrix, mask, false, &cpu); + raw_spin_unlock_irqrestore(&imsic->matrix_lock, flags); + if (local_id < 0) + return NULL; + + lpriv = per_cpu_ptr(imsic->lpriv, cpu); + vec = &lpriv->vectors[local_id]; + vec->hwirq = hwirq; + vec->enable = false; + vec->move = NULL; + + return vec; +} + +void imsic_vector_free(struct imsic_vector *vec) +{ + unsigned long flags; + + raw_spin_lock_irqsave(&imsic->matrix_lock, flags); + vec->hwirq = UINT_MAX; + irq_matrix_free(imsic->matrix, vec->cpu, vec->local_id, false); + raw_spin_unlock_irqrestore(&imsic->matrix_lock, flags); +} + +static void __init imsic_local_cleanup(void) +{ + struct imsic_local_priv *lpriv; + int cpu; + + for_each_possible_cpu(cpu) { + lpriv = per_cpu_ptr(imsic->lpriv, cpu); + + bitmap_free(lpriv->dirty_bitmap); + kfree(lpriv->vectors); + } + + free_percpu(imsic->lpriv); +} + +static int __init imsic_local_init(void) +{ + struct imsic_global_config *global = &imsic->global; + struct imsic_local_priv *lpriv; + struct imsic_vector *vec; + int cpu, i; + + /* Allocate per-CPU private state */ + imsic->lpriv = alloc_percpu(typeof(*imsic->lpriv)); + if (!imsic->lpriv) + return -ENOMEM; + + /* Setup per-CPU private state */ + for_each_possible_cpu(cpu) { + lpriv = per_cpu_ptr(imsic->lpriv, cpu); + + raw_spin_lock_init(&lpriv->lock); + + /* Allocate dirty bitmap */ + lpriv->dirty_bitmap = bitmap_zalloc(global->nr_ids + 1, GFP_KERNEL); + if (!lpriv->dirty_bitmap) + goto fail_local_cleanup; + +#ifdef CONFIG_SMP + /* Setup lazy timer for synchronization */ + timer_setup(&lpriv->timer, imsic_local_timer_callback, TIMER_PINNED); +#endif + + /* Allocate vector array */ + lpriv->vectors = kcalloc(global->nr_ids + 1, sizeof(*lpriv->vectors), + GFP_KERNEL); + if (!lpriv->vectors) + goto fail_local_cleanup; + + /* Setup vector array */ + for (i = 0; i <= global->nr_ids; i++) { + vec = &lpriv->vectors[i]; + vec->cpu = cpu; + vec->local_id = i; + vec->hwirq = UINT_MAX; + } + } + + return 0; + +fail_local_cleanup: + imsic_local_cleanup(); + return -ENOMEM; +} + +void imsic_state_online(void) +{ + unsigned long flags; + + raw_spin_lock_irqsave(&imsic->matrix_lock, flags); + irq_matrix_online(imsic->matrix); + raw_spin_unlock_irqrestore(&imsic->matrix_lock, flags); +} + +void imsic_state_offline(void) +{ + unsigned long flags; + + raw_spin_lock_irqsave(&imsic->matrix_lock, flags); + irq_matrix_offline(imsic->matrix); + raw_spin_unlock_irqrestore(&imsic->matrix_lock, flags); + +#ifdef CONFIG_SMP + struct imsic_local_priv *lpriv = this_cpu_ptr(imsic->lpriv); + + raw_spin_lock_irqsave(&lpriv->lock, flags); + WARN_ON_ONCE(try_to_del_timer_sync(&lpriv->timer) < 0); + raw_spin_unlock_irqrestore(&lpriv->lock, flags); +#endif +} + +static int __init imsic_matrix_init(void) +{ + struct imsic_global_config *global = &imsic->global; + + raw_spin_lock_init(&imsic->matrix_lock); + imsic->matrix = irq_alloc_matrix(global->nr_ids + 1, + 0, global->nr_ids + 1); + if (!imsic->matrix) + return -ENOMEM; + + /* Reserve ID#0 because it is special and never implemented */ + irq_matrix_assign_system(imsic->matrix, 0, false); + + /* Reserve IPI ID because it is special and used internally */ + irq_matrix_assign_system(imsic->matrix, IMSIC_IPI_ID, false); + + return 0; +} + +static int __init imsic_get_parent_hartid(struct fwnode_handle *fwnode, + u32 index, unsigned long *hartid) +{ + struct of_phandle_args parent; + int rc; + + /* + * Currently, only OF fwnode is supported so extend this + * function for ACPI support. + */ + if (!is_of_node(fwnode)) + return -EINVAL; + + rc = of_irq_parse_one(to_of_node(fwnode), index, &parent); + if (rc) + return rc; + + /* + * Skip interrupts other than external interrupts for + * current privilege level. + */ + if (parent.args[0] != RV_IRQ_EXT) + return -EINVAL; + + return riscv_of_parent_hartid(parent.np, hartid); +} + +static int __init imsic_get_mmio_resource(struct fwnode_handle *fwnode, + u32 index, struct resource *res) +{ + /* + * Currently, only OF fwnode is supported so extend this + * function for ACPI support. + */ + if (!is_of_node(fwnode)) + return -EINVAL; + + return of_address_to_resource(to_of_node(fwnode), index, res); +} + +static int __init imsic_parse_fwnode(struct fwnode_handle *fwnode, + struct imsic_global_config *global, + u32 *nr_parent_irqs, + u32 *nr_mmios) +{ + unsigned long hartid; + struct resource res; + int rc; + u32 i; + + /* + * Currently, only OF fwnode is supported so extend this + * function for ACPI support. + */ + if (!is_of_node(fwnode)) + return -EINVAL; + + *nr_parent_irqs = 0; + *nr_mmios = 0; + + /* Find number of parent interrupts */ + while (!imsic_get_parent_hartid(fwnode, *nr_parent_irqs, &hartid)) + (*nr_parent_irqs)++; + if (!*nr_parent_irqs) { + pr_err("%pfwP: no parent irqs available\n", fwnode); + return -EINVAL; + } + + /* Find number of guest index bits in MSI address */ + rc = of_property_read_u32(to_of_node(fwnode), "riscv,guest-index-bits", + &global->guest_index_bits); + if (rc) + global->guest_index_bits = 0; + + /* Find number of HART index bits */ + rc = of_property_read_u32(to_of_node(fwnode), "riscv,hart-index-bits", + &global->hart_index_bits); + if (rc) { + /* Assume default value */ + global->hart_index_bits = __fls(*nr_parent_irqs); + if (BIT(global->hart_index_bits) < *nr_parent_irqs) + global->hart_index_bits++; + } + + /* Find number of group index bits */ + rc = of_property_read_u32(to_of_node(fwnode), "riscv,group-index-bits", + &global->group_index_bits); + if (rc) + global->group_index_bits = 0; + + /* + * Find first bit position of group index. + * If not specified assumed the default APLIC-IMSIC configuration. + */ + rc = of_property_read_u32(to_of_node(fwnode), "riscv,group-index-shift", + &global->group_index_shift); + if (rc) + global->group_index_shift = IMSIC_MMIO_PAGE_SHIFT * 2; + + /* Find number of interrupt identities */ + rc = of_property_read_u32(to_of_node(fwnode), "riscv,num-ids", + &global->nr_ids); + if (rc) { + pr_err("%pfwP: number of interrupt identities not found\n", fwnode); + return rc; + } + + /* Find number of guest interrupt identities */ + rc = of_property_read_u32(to_of_node(fwnode), "riscv,num-guest-ids", + &global->nr_guest_ids); + if (rc) + global->nr_guest_ids = global->nr_ids; + + /* Sanity check guest index bits */ + i = BITS_PER_LONG - IMSIC_MMIO_PAGE_SHIFT; + if (i < global->guest_index_bits) { + pr_err("%pfwP: guest index bits too big\n", fwnode); + return -EINVAL; + } + + /* Sanity check HART index bits */ + i = BITS_PER_LONG - IMSIC_MMIO_PAGE_SHIFT - global->guest_index_bits; + if (i < global->hart_index_bits) { + pr_err("%pfwP: HART index bits too big\n", fwnode); + return -EINVAL; + } + + /* Sanity check group index bits */ + i = BITS_PER_LONG - IMSIC_MMIO_PAGE_SHIFT - + global->guest_index_bits - global->hart_index_bits; + if (i < global->group_index_bits) { + pr_err("%pfwP: group index bits too big\n", fwnode); + return -EINVAL; + } + + /* Sanity check group index shift */ + i = global->group_index_bits + global->group_index_shift - 1; + if (i >= BITS_PER_LONG) { + pr_err("%pfwP: group index shift too big\n", fwnode); + return -EINVAL; + } + + /* Sanity check number of interrupt identities */ + if (global->nr_ids < IMSIC_MIN_ID || + global->nr_ids >= IMSIC_MAX_ID || + (global->nr_ids & IMSIC_MIN_ID) != IMSIC_MIN_ID) { + pr_err("%pfwP: invalid number of interrupt identities\n", fwnode); + return -EINVAL; + } + + /* Sanity check number of guest interrupt identities */ + if (global->nr_guest_ids < IMSIC_MIN_ID || + global->nr_guest_ids >= IMSIC_MAX_ID || + (global->nr_guest_ids & IMSIC_MIN_ID) != IMSIC_MIN_ID) { + pr_err("%pfwP: invalid number of guest interrupt identities\n", fwnode); + return -EINVAL; + } + + /* Compute base address */ + rc = imsic_get_mmio_resource(fwnode, 0, &res); + if (rc) { + pr_err("%pfwP: first MMIO resource not found\n", fwnode); + return -EINVAL; + } + global->base_addr = res.start; + global->base_addr &= ~(BIT(global->guest_index_bits + + global->hart_index_bits + + IMSIC_MMIO_PAGE_SHIFT) - 1); + global->base_addr &= ~((BIT(global->group_index_bits) - 1) << + global->group_index_shift); + + /* Find number of MMIO register sets */ + while (!imsic_get_mmio_resource(fwnode, *nr_mmios, &res)) + (*nr_mmios)++; + + return 0; +} + +int __init imsic_setup_state(struct fwnode_handle *fwnode) +{ + u32 i, j, index, nr_parent_irqs, nr_mmios, nr_handlers = 0; + struct imsic_global_config *global; + struct imsic_local_config *local; + void __iomem **mmios_va = NULL; + struct resource *mmios = NULL; + unsigned long reloff, hartid; + phys_addr_t base_addr; + int rc, cpu; + + /* + * Only one IMSIC instance allowed in a platform for clean + * implementation of SMP IRQ affinity and per-CPU IPIs. + * + * This means on a multi-socket (or multi-die) platform we + * will have multiple MMIO regions for one IMSIC instance. + */ + if (imsic) { + pr_err("%pfwP: already initialized hence ignoring\n", fwnode); + return -EALREADY; + } + + if (!riscv_isa_extension_available(NULL, SxAIA)) { + pr_err("%pfwP: AIA support not available\n", fwnode); + return -ENODEV; + } + + imsic = kzalloc(sizeof(*imsic), GFP_KERNEL); + if (!imsic) + return -ENOMEM; + imsic->fwnode = fwnode; + global = &imsic->global; + + global->local = alloc_percpu(typeof(*global->local)); + if (!global->local) { + rc = -ENOMEM; + goto out_free_priv; + } + + /* Parse IMSIC fwnode */ + rc = imsic_parse_fwnode(fwnode, global, &nr_parent_irqs, &nr_mmios); + if (rc) + goto out_free_local; + + /* Allocate MMIO resource array */ + mmios = kcalloc(nr_mmios, sizeof(*mmios), GFP_KERNEL); + if (!mmios) { + rc = -ENOMEM; + goto out_free_local; + } + + /* Allocate MMIO virtual address array */ + mmios_va = kcalloc(nr_mmios, sizeof(*mmios_va), GFP_KERNEL); + if (!mmios_va) { + rc = -ENOMEM; + goto out_iounmap; + } + + /* Parse and map MMIO register sets */ + for (i = 0; i < nr_mmios; i++) { + rc = imsic_get_mmio_resource(fwnode, i, &mmios[i]); + if (rc) { + pr_err("%pfwP: unable to parse MMIO regset %d\n", fwnode, i); + goto out_iounmap; + } + + base_addr = mmios[i].start; + base_addr &= ~(BIT(global->guest_index_bits + + global->hart_index_bits + + IMSIC_MMIO_PAGE_SHIFT) - 1); + base_addr &= ~((BIT(global->group_index_bits) - 1) << + global->group_index_shift); + if (base_addr != global->base_addr) { + rc = -EINVAL; + pr_err("%pfwP: address mismatch for regset %d\n", fwnode, i); + goto out_iounmap; + } + + mmios_va[i] = ioremap(mmios[i].start, resource_size(&mmios[i])); + if (!mmios_va[i]) { + rc = -EIO; + pr_err("%pfwP: unable to map MMIO regset %d\n", fwnode, i); + goto out_iounmap; + } + } + + /* Initialize local (or per-CPU )state */ + rc = imsic_local_init(); + if (rc) { + pr_err("%pfwP: failed to initialize local state\n", + fwnode); + goto out_iounmap; + } + + /* Configure handlers for target CPUs */ + for (i = 0; i < nr_parent_irqs; i++) { + rc = imsic_get_parent_hartid(fwnode, i, &hartid); + if (rc) { + pr_warn("%pfwP: hart ID for parent irq%d not found\n", fwnode, i); + continue; + } + + cpu = riscv_hartid_to_cpuid(hartid); + if (cpu < 0) { + pr_warn("%pfwP: invalid cpuid for parent irq%d\n", fwnode, i); + continue; + } + + /* Find MMIO location of MSI page */ + index = nr_mmios; + reloff = i * BIT(global->guest_index_bits) * + IMSIC_MMIO_PAGE_SZ; + for (j = 0; nr_mmios; j++) { + if (reloff < resource_size(&mmios[j])) { + index = j; + break; + } + + /* + * MMIO region size may not be aligned to + * BIT(global->guest_index_bits) * IMSIC_MMIO_PAGE_SZ + * if holes are present. + */ + reloff -= ALIGN(resource_size(&mmios[j]), + BIT(global->guest_index_bits) * IMSIC_MMIO_PAGE_SZ); + } + if (index >= nr_mmios) { + pr_warn("%pfwP: MMIO not found for parent irq%d\n", fwnode, i); + continue; + } + + local = per_cpu_ptr(global->local, cpu); + local->msi_pa = mmios[index].start + reloff; + local->msi_va = mmios_va[index] + reloff; + + nr_handlers++; + } + + /* If no CPU handlers found then can't take interrupts */ + if (!nr_handlers) { + pr_err("%pfwP: No CPU handlers found\n", fwnode); + rc = -ENODEV; + goto out_local_cleanup; + } + + /* Initialize matrix allocator */ + rc = imsic_matrix_init(); + if (rc) { + pr_err("%pfwP: failed to create matrix allocator\n", fwnode); + goto out_local_cleanup; + } + + /* We don't need MMIO arrays anymore so let's free-up */ + kfree(mmios_va); + kfree(mmios); + + return 0; + +out_local_cleanup: + imsic_local_cleanup(); +out_iounmap: + for (i = 0; i < nr_mmios; i++) { + if (mmios_va[i]) + iounmap(mmios_va[i]); + } + kfree(mmios_va); + kfree(mmios); +out_free_local: + free_percpu(imsic->global.local); +out_free_priv: + kfree(imsic); + imsic = NULL; + return rc; +} diff --git a/drivers/irqchip/irq-riscv-imsic-state.h b/drivers/irqchip/irq-riscv-imsic-state.h new file mode 100644 index 000000000000..5ae2f69b035b --- /dev/null +++ b/drivers/irqchip/irq-riscv-imsic-state.h @@ -0,0 +1,108 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2021 Western Digital Corporation or its affiliates. + * Copyright (C) 2022 Ventana Micro Systems Inc. + */ + +#ifndef _IRQ_RISCV_IMSIC_STATE_H +#define _IRQ_RISCV_IMSIC_STATE_H + +#include <linux/irqchip/riscv-imsic.h> +#include <linux/irqdomain.h> +#include <linux/fwnode.h> +#include <linux/timer.h> + +#define IMSIC_IPI_ID 1 +#define IMSIC_NR_IPI 8 + +struct imsic_vector { + /* Fixed details of the vector */ + unsigned int cpu; + unsigned int local_id; + /* Details saved by driver in the vector */ + unsigned int hwirq; + /* Details accessed using local lock held */ + bool enable; + struct imsic_vector *move; +}; + +struct imsic_local_priv { + /* Local lock to protect vector enable/move variables and dirty bitmap */ + raw_spinlock_t lock; + + /* Local dirty bitmap for synchronization */ + unsigned long *dirty_bitmap; + +#ifdef CONFIG_SMP + /* Local timer for synchronization */ + struct timer_list timer; +#endif + + /* Local vector table */ + struct imsic_vector *vectors; +}; + +struct imsic_priv { + /* Device details */ + struct fwnode_handle *fwnode; + + /* Global configuration common for all HARTs */ + struct imsic_global_config global; + + /* Per-CPU state */ + struct imsic_local_priv __percpu *lpriv; + + /* State of IRQ matrix allocator */ + raw_spinlock_t matrix_lock; + struct irq_matrix *matrix; + + /* IRQ domains (created by platform driver) */ + struct irq_domain *base_domain; +}; + +extern struct imsic_priv *imsic; + +void __imsic_eix_update(unsigned long base_id, unsigned long num_id, bool pend, bool val); + +static inline void __imsic_id_set_enable(unsigned long id) +{ + __imsic_eix_update(id, 1, false, true); +} + +static inline void __imsic_id_clear_enable(unsigned long id) +{ + __imsic_eix_update(id, 1, false, false); +} + +void imsic_local_sync_all(void); +void imsic_local_delivery(bool enable); + +void imsic_vector_mask(struct imsic_vector *vec); +void imsic_vector_unmask(struct imsic_vector *vec); + +static inline bool imsic_vector_isenabled(struct imsic_vector *vec) +{ + return READ_ONCE(vec->enable); +} + +static inline struct imsic_vector *imsic_vector_get_move(struct imsic_vector *vec) +{ + return READ_ONCE(vec->move); +} + +void imsic_vector_move(struct imsic_vector *old_vec, struct imsic_vector *new_vec); + +struct imsic_vector *imsic_vector_from_local_id(unsigned int cpu, unsigned int local_id); + +struct imsic_vector *imsic_vector_alloc(unsigned int hwirq, const struct cpumask *mask); +void imsic_vector_free(struct imsic_vector *vector); + +void imsic_vector_debug_show(struct seq_file *m, struct imsic_vector *vec, int ind); +void imsic_vector_debug_show_summary(struct seq_file *m, int ind); + +void imsic_state_online(void); +void imsic_state_offline(void); +int imsic_setup_state(struct fwnode_handle *fwnode); +int imsic_irqdomain_init(void); + +#endif diff --git a/drivers/irqchip/irq-sifive-plic.c b/drivers/irqchip/irq-sifive-plic.c index f3d4cb9e34f7..8fb183ced1e7 100644 --- a/drivers/irqchip/irq-sifive-plic.c +++ b/drivers/irqchip/irq-sifive-plic.c @@ -164,15 +164,12 @@ static int plic_set_affinity(struct irq_data *d, const struct cpumask *mask_val, bool force) { unsigned int cpu; - struct cpumask amask; struct plic_priv *priv = irq_data_get_irq_chip_data(d); - cpumask_and(&amask, &priv->lmask, mask_val); - if (force) - cpu = cpumask_first(&amask); + cpu = cpumask_first_and(&priv->lmask, mask_val); else - cpu = cpumask_any_and(&amask, cpu_online_mask); + cpu = cpumask_first_and_and(&priv->lmask, mask_val, cpu_online_mask); if (cpu >= nr_cpu_ids) return -EINVAL; diff --git a/drivers/irqchip/irq-stm32-exti.c b/drivers/irqchip/irq-stm32-exti.c index 26a5193d0ae4..2cc9f3b7d669 100644 --- a/drivers/irqchip/irq-stm32-exti.c +++ b/drivers/irqchip/irq-stm32-exti.c @@ -19,13 +19,26 @@ #include <linux/of_address.h> #include <linux/of_irq.h> #include <linux/platform_device.h> -#include <linux/syscore_ops.h> +#include <linux/pm.h> #include <dt-bindings/interrupt-controller/arm-gic.h> -#define IRQS_PER_BANK 32 +#define IRQS_PER_BANK 32 -#define HWSPNLCK_TIMEOUT 1000 /* usec */ +#define HWSPNLCK_TIMEOUT 1000 /* usec */ + +#define EXTI_EnCIDCFGR(n) (0x180 + (n) * 4) +#define EXTI_HWCFGR1 0x3f0 + +/* Register: EXTI_EnCIDCFGR(n) */ +#define EXTI_CIDCFGR_CFEN_MASK BIT(0) +#define EXTI_CIDCFGR_CID_MASK GENMASK(6, 4) +#define EXTI_CIDCFGR_CID_SHIFT 4 + +/* Register: EXTI_HWCFGR1 */ +#define EXTI_HWCFGR1_CIDWIDTH_MASK GENMASK(27, 24) + +#define EXTI_CID1 1 struct stm32_exti_bank { u32 imr_ofst; @@ -36,6 +49,7 @@ struct stm32_exti_bank { u32 rpr_ofst; u32 fpr_ofst; u32 trg_ofst; + u32 seccfgr_ofst; }; #define UNDEF_REG ~0 @@ -54,17 +68,18 @@ struct stm32_exti_chip_data { u32 mask_cache; u32 rtsr_cache; u32 ftsr_cache; + u32 event_reserved; }; struct stm32_exti_host_data { void __iomem *base; + struct device *dev; struct stm32_exti_chip_data *chips_data; const struct stm32_exti_drv_data *drv_data; struct hwspinlock *hwlock; + bool dt_has_irqs_desc; /* skip internal desc_irqs array and get it from DT */ }; -static struct stm32_exti_host_data *stm32_host_data; - static const struct stm32_exti_bank stm32f4xx_exti_b1 = { .imr_ofst = 0x00, .emr_ofst = 0x04, @@ -74,6 +89,7 @@ static const struct stm32_exti_bank stm32f4xx_exti_b1 = { .rpr_ofst = 0x14, .fpr_ofst = UNDEF_REG, .trg_ofst = UNDEF_REG, + .seccfgr_ofst = UNDEF_REG, }; static const struct stm32_exti_bank *stm32f4xx_exti_banks[] = { @@ -94,6 +110,7 @@ static const struct stm32_exti_bank stm32h7xx_exti_b1 = { .rpr_ofst = 0x88, .fpr_ofst = UNDEF_REG, .trg_ofst = UNDEF_REG, + .seccfgr_ofst = UNDEF_REG, }; static const struct stm32_exti_bank stm32h7xx_exti_b2 = { @@ -105,6 +122,7 @@ static const struct stm32_exti_bank stm32h7xx_exti_b2 = { .rpr_ofst = 0x98, .fpr_ofst = UNDEF_REG, .trg_ofst = UNDEF_REG, + .seccfgr_ofst = UNDEF_REG, }; static const struct stm32_exti_bank stm32h7xx_exti_b3 = { @@ -116,6 +134,7 @@ static const struct stm32_exti_bank stm32h7xx_exti_b3 = { .rpr_ofst = 0xA8, .fpr_ofst = UNDEF_REG, .trg_ofst = UNDEF_REG, + .seccfgr_ofst = UNDEF_REG, }; static const struct stm32_exti_bank *stm32h7xx_exti_banks[] = { @@ -138,6 +157,7 @@ static const struct stm32_exti_bank stm32mp1_exti_b1 = { .rpr_ofst = 0x0C, .fpr_ofst = 0x10, .trg_ofst = 0x3EC, + .seccfgr_ofst = 0x14, }; static const struct stm32_exti_bank stm32mp1_exti_b2 = { @@ -149,6 +169,7 @@ static const struct stm32_exti_bank stm32mp1_exti_b2 = { .rpr_ofst = 0x2C, .fpr_ofst = 0x30, .trg_ofst = 0x3E8, + .seccfgr_ofst = 0x34, }; static const struct stm32_exti_bank stm32mp1_exti_b3 = { @@ -160,6 +181,7 @@ static const struct stm32_exti_bank stm32mp1_exti_b3 = { .rpr_ofst = 0x4C, .fpr_ofst = 0x50, .trg_ofst = 0x3E4, + .seccfgr_ofst = 0x54, }; static const struct stm32_exti_bank *stm32mp1_exti_banks[] = { @@ -322,7 +344,7 @@ static void stm32_irq_handler(struct irq_desc *desc) while ((pending = stm32_exti_pending(gc))) { for_each_set_bit(n, &pending, IRQS_PER_BANK) generic_handle_domain_irq(domain, irq_base + n); - } + } } chained_irq_exit(chip, desc); @@ -621,50 +643,32 @@ static int stm32_exti_h_set_affinity(struct irq_data *d, return IRQ_SET_MASK_OK_DONE; } -static int __maybe_unused stm32_exti_h_suspend(void) +static int stm32_exti_h_suspend(struct device *dev) { + struct stm32_exti_host_data *host_data = dev_get_drvdata(dev); struct stm32_exti_chip_data *chip_data; int i; - for (i = 0; i < stm32_host_data->drv_data->bank_nr; i++) { - chip_data = &stm32_host_data->chips_data[i]; - raw_spin_lock(&chip_data->rlock); + for (i = 0; i < host_data->drv_data->bank_nr; i++) { + chip_data = &host_data->chips_data[i]; stm32_chip_suspend(chip_data, chip_data->wake_active); - raw_spin_unlock(&chip_data->rlock); } return 0; } -static void __maybe_unused stm32_exti_h_resume(void) +static int stm32_exti_h_resume(struct device *dev) { + struct stm32_exti_host_data *host_data = dev_get_drvdata(dev); struct stm32_exti_chip_data *chip_data; int i; - for (i = 0; i < stm32_host_data->drv_data->bank_nr; i++) { - chip_data = &stm32_host_data->chips_data[i]; - raw_spin_lock(&chip_data->rlock); + for (i = 0; i < host_data->drv_data->bank_nr; i++) { + chip_data = &host_data->chips_data[i]; stm32_chip_resume(chip_data, chip_data->mask_cache); - raw_spin_unlock(&chip_data->rlock); } -} -static struct syscore_ops stm32_exti_h_syscore_ops = { -#ifdef CONFIG_PM_SLEEP - .suspend = stm32_exti_h_suspend, - .resume = stm32_exti_h_resume, -#endif -}; - -static void stm32_exti_h_syscore_init(struct stm32_exti_host_data *host_data) -{ - stm32_host_data = host_data; - register_syscore_ops(&stm32_exti_h_syscore_ops); -} - -static void stm32_exti_h_syscore_deinit(void) -{ - unregister_syscore_ops(&stm32_exti_h_syscore_ops); + return 0; } static int stm32_exti_h_retrigger(struct irq_data *d) @@ -725,12 +729,35 @@ static int stm32_exti_h_domain_alloc(struct irq_domain *dm, bank = hwirq / IRQS_PER_BANK; chip_data = &host_data->chips_data[bank]; + /* Check if event is reserved (Secure) */ + if (chip_data->event_reserved & BIT(hwirq % IRQS_PER_BANK)) { + dev_err(host_data->dev, "event %lu is reserved, secure\n", hwirq); + return -EPERM; + } + event_trg = readl_relaxed(host_data->base + chip_data->reg_bank->trg_ofst); chip = (event_trg & BIT(hwirq % IRQS_PER_BANK)) ? &stm32_exti_h_chip : &stm32_exti_h_chip_direct; irq_domain_set_hwirq_and_chip(dm, virq, hwirq, chip, chip_data); + if (host_data->dt_has_irqs_desc) { + struct of_phandle_args out_irq; + int ret; + + ret = of_irq_parse_one(host_data->dev->of_node, hwirq, &out_irq); + if (ret) + return ret; + /* we only support one parent, so far */ + if (of_node_to_fwnode(out_irq.np) != dm->parent->fwnode) + return -EINVAL; + + of_phandle_args_to_fwspec(out_irq.np, out_irq.args, + out_irq.args_count, &p_fwspec); + + return irq_domain_alloc_irqs_parent(dm, virq, 1, &p_fwspec); + } + if (!host_data->drv_data->desc_irqs) return -EINVAL; @@ -771,8 +798,6 @@ stm32_exti_host_data *stm32_exti_host_init(const struct stm32_exti_drv_data *dd, goto free_chips_data; } - stm32_host_data = host_data; - return host_data; free_chips_data: @@ -807,6 +832,10 @@ stm32_exti_chip_data *stm32_exti_chip_init(struct stm32_exti_host_data *h_data, if (stm32_bank->emr_ofst != UNDEF_REG) writel_relaxed(0, base + stm32_bank->emr_ofst); + /* reserve Secure events */ + if (stm32_bank->seccfgr_ofst != UNDEF_REG) + chip_data->event_reserved = readl_relaxed(base + stm32_bank->seccfgr_ofst); + pr_info("%pOF: bank%d\n", node, bank_idx); return chip_data; @@ -891,6 +920,27 @@ static const struct irq_domain_ops stm32_exti_h_domain_ops = { .xlate = irq_domain_xlate_twocell, }; +static void stm32_exti_check_rif(struct stm32_exti_host_data *host_data) +{ + unsigned int bank, i, event; + u32 cid, cidcfgr, hwcfgr1; + + /* quit on CID not supported */ + hwcfgr1 = readl_relaxed(host_data->base + EXTI_HWCFGR1); + if ((hwcfgr1 & EXTI_HWCFGR1_CIDWIDTH_MASK) == 0) + return; + + for (bank = 0; bank < host_data->drv_data->bank_nr; bank++) { + for (i = 0; i < IRQS_PER_BANK; i++) { + event = bank * IRQS_PER_BANK + i; + cidcfgr = readl_relaxed(host_data->base + EXTI_EnCIDCFGR(event)); + cid = (cidcfgr & EXTI_CIDCFGR_CID_MASK) >> EXTI_CIDCFGR_CID_SHIFT; + if ((cidcfgr & EXTI_CIDCFGR_CFEN_MASK) && cid != EXTI_CID1) + host_data->chips_data[bank].event_reserved |= BIT(i); + } + } +} + static void stm32_exti_remove_irq(void *data) { struct irq_domain *domain = data; @@ -898,11 +948,6 @@ static void stm32_exti_remove_irq(void *data) irq_domain_remove(domain); } -static void stm32_exti_remove(struct platform_device *pdev) -{ - stm32_exti_h_syscore_deinit(); -} - static int stm32_exti_probe(struct platform_device *pdev) { int ret, i; @@ -916,6 +961,9 @@ static int stm32_exti_probe(struct platform_device *pdev) if (!host_data) return -ENOMEM; + dev_set_drvdata(dev, host_data); + host_data->dev = dev; + /* check for optional hwspinlock which may be not available yet */ ret = of_hwspin_lock_get_id(np, 0); if (ret == -EPROBE_DEFER) @@ -955,6 +1003,8 @@ static int stm32_exti_probe(struct platform_device *pdev) for (i = 0; i < drv_data->bank_nr; i++) stm32_exti_chip_init(host_data, i, np); + stm32_exti_check_rif(host_data); + parent_domain = irq_find_host(of_irq_find_parent(np)); if (!parent_domain) { dev_err(dev, "GIC interrupt-parent not found\n"); @@ -975,7 +1025,8 @@ static int stm32_exti_probe(struct platform_device *pdev) if (ret) return ret; - stm32_exti_h_syscore_init(host_data); + if (of_property_read_bool(np, "interrupts-extended")) + host_data->dt_has_irqs_desc = true; return 0; } @@ -988,12 +1039,16 @@ static const struct of_device_id stm32_exti_ids[] = { }; MODULE_DEVICE_TABLE(of, stm32_exti_ids); +static const struct dev_pm_ops stm32_exti_dev_pm_ops = { + NOIRQ_SYSTEM_SLEEP_PM_OPS(stm32_exti_h_suspend, stm32_exti_h_resume) +}; + static struct platform_driver stm32_exti_driver = { .probe = stm32_exti_probe, - .remove_new = stm32_exti_remove, .driver = { .name = "stm32_exti", .of_match_table = stm32_exti_ids, + .pm = &stm32_exti_dev_pm_ops, }, }; diff --git a/drivers/irqchip/irq-sunxi-nmi.c b/drivers/irqchip/irq-sunxi-nmi.c index e760b1278143..bb92fd85e975 100644 --- a/drivers/irqchip/irq-sunxi-nmi.c +++ b/drivers/irqchip/irq-sunxi-nmi.c @@ -192,7 +192,6 @@ static int __init sunxi_sc_nmi_irq_init(struct device_node *node, gc->chip_types[0].regs.type = reg_offs->ctrl; gc->chip_types[1].type = IRQ_TYPE_EDGE_BOTH; - gc->chip_types[1].chip.name = gc->chip_types[0].chip.name; gc->chip_types[1].chip.irq_ack = irq_gc_ack_set_bit; gc->chip_types[1].chip.irq_mask = irq_gc_mask_clr_bit; gc->chip_types[1].chip.irq_unmask = irq_gc_mask_set_bit; diff --git a/drivers/irqchip/irq-tb10x.c b/drivers/irqchip/irq-tb10x.c index 680586354d12..d59bfbe8c6d0 100644 --- a/drivers/irqchip/irq-tb10x.c +++ b/drivers/irqchip/irq-tb10x.c @@ -150,7 +150,6 @@ static int __init of_tb10x_init_irq(struct device_node *ictl, gc->chip_types[0].regs.mask = AB_IRQCTL_INT_ENABLE; gc->chip_types[1].type = IRQ_TYPE_EDGE_BOTH; - gc->chip_types[1].chip.name = gc->chip_types[0].chip.name; gc->chip_types[1].chip.irq_ack = irq_gc_ack_set_bit; gc->chip_types[1].chip.irq_mask = irq_gc_mask_clr_bit; gc->chip_types[1].chip.irq_unmask = irq_gc_mask_set_bit; diff --git a/drivers/macintosh/via-macii.c b/drivers/macintosh/via-macii.c index db9270da5b8e..b6ddf1d47cb4 100644 --- a/drivers/macintosh/via-macii.c +++ b/drivers/macintosh/via-macii.c @@ -140,24 +140,19 @@ static int macii_probe(void) /* Initialize the driver */ static int macii_init(void) { - unsigned long flags; int err; - local_irq_save(flags); - err = macii_init_via(); if (err) - goto out; + return err; err = request_irq(IRQ_MAC_ADB, macii_interrupt, 0, "ADB", macii_interrupt); if (err) - goto out; + return err; macii_state = idle; -out: - local_irq_restore(flags); - return err; + return 0; } /* initialize the hardware */ diff --git a/drivers/md/bcache/bset.c b/drivers/md/bcache/bset.c index 2bba4d6aaaa2..463eb13bd0b2 100644 --- a/drivers/md/bcache/bset.c +++ b/drivers/md/bcache/bset.c @@ -54,7 +54,7 @@ void bch_dump_bucket(struct btree_keys *b) int __bch_count_data(struct btree_keys *b) { unsigned int ret = 0; - struct btree_iter iter; + struct btree_iter_stack iter; struct bkey *k; if (b->ops->is_extents) @@ -67,7 +67,7 @@ void __bch_check_keys(struct btree_keys *b, const char *fmt, ...) { va_list args; struct bkey *k, *p = NULL; - struct btree_iter iter; + struct btree_iter_stack iter; const char *err; for_each_key(b, k, &iter) { @@ -879,7 +879,7 @@ unsigned int bch_btree_insert_key(struct btree_keys *b, struct bkey *k, unsigned int status = BTREE_INSERT_STATUS_NO_INSERT; struct bset *i = bset_tree_last(b)->data; struct bkey *m, *prev = NULL; - struct btree_iter iter; + struct btree_iter_stack iter; struct bkey preceding_key_on_stack = ZERO_KEY; struct bkey *preceding_key_p = &preceding_key_on_stack; @@ -895,9 +895,9 @@ unsigned int bch_btree_insert_key(struct btree_keys *b, struct bkey *k, else preceding_key(k, &preceding_key_p); - m = bch_btree_iter_init(b, &iter, preceding_key_p); + m = bch_btree_iter_stack_init(b, &iter, preceding_key_p); - if (b->ops->insert_fixup(b, k, &iter, replace_key)) + if (b->ops->insert_fixup(b, k, &iter.iter, replace_key)) return status; status = BTREE_INSERT_STATUS_INSERT; @@ -1100,33 +1100,33 @@ void bch_btree_iter_push(struct btree_iter *iter, struct bkey *k, btree_iter_cmp)); } -static struct bkey *__bch_btree_iter_init(struct btree_keys *b, - struct btree_iter *iter, - struct bkey *search, - struct bset_tree *start) +static struct bkey *__bch_btree_iter_stack_init(struct btree_keys *b, + struct btree_iter_stack *iter, + struct bkey *search, + struct bset_tree *start) { struct bkey *ret = NULL; - iter->size = ARRAY_SIZE(iter->data); - iter->used = 0; + iter->iter.size = ARRAY_SIZE(iter->stack_data); + iter->iter.used = 0; #ifdef CONFIG_BCACHE_DEBUG - iter->b = b; + iter->iter.b = b; #endif for (; start <= bset_tree_last(b); start++) { ret = bch_bset_search(b, start, search); - bch_btree_iter_push(iter, ret, bset_bkey_last(start->data)); + bch_btree_iter_push(&iter->iter, ret, bset_bkey_last(start->data)); } return ret; } -struct bkey *bch_btree_iter_init(struct btree_keys *b, - struct btree_iter *iter, +struct bkey *bch_btree_iter_stack_init(struct btree_keys *b, + struct btree_iter_stack *iter, struct bkey *search) { - return __bch_btree_iter_init(b, iter, search, b->set); + return __bch_btree_iter_stack_init(b, iter, search, b->set); } static inline struct bkey *__bch_btree_iter_next(struct btree_iter *iter, @@ -1293,10 +1293,10 @@ void bch_btree_sort_partial(struct btree_keys *b, unsigned int start, struct bset_sort_state *state) { size_t order = b->page_order, keys = 0; - struct btree_iter iter; + struct btree_iter_stack iter; int oldsize = bch_count_data(b); - __bch_btree_iter_init(b, &iter, NULL, &b->set[start]); + __bch_btree_iter_stack_init(b, &iter, NULL, &b->set[start]); if (start) { unsigned int i; @@ -1307,7 +1307,7 @@ void bch_btree_sort_partial(struct btree_keys *b, unsigned int start, order = get_order(__set_bytes(b->set->data, keys)); } - __btree_sort(b, &iter, start, order, false, state); + __btree_sort(b, &iter.iter, start, order, false, state); EBUG_ON(oldsize >= 0 && bch_count_data(b) != oldsize); } @@ -1323,11 +1323,11 @@ void bch_btree_sort_into(struct btree_keys *b, struct btree_keys *new, struct bset_sort_state *state) { uint64_t start_time = local_clock(); - struct btree_iter iter; + struct btree_iter_stack iter; - bch_btree_iter_init(b, &iter, NULL); + bch_btree_iter_stack_init(b, &iter, NULL); - btree_mergesort(b, new->set->data, &iter, false, true); + btree_mergesort(b, new->set->data, &iter.iter, false, true); bch_time_stats_update(&state->time, start_time); diff --git a/drivers/md/bcache/bset.h b/drivers/md/bcache/bset.h index d795c84246b0..011f6062c4c0 100644 --- a/drivers/md/bcache/bset.h +++ b/drivers/md/bcache/bset.h @@ -321,7 +321,14 @@ struct btree_iter { #endif struct btree_iter_set { struct bkey *k, *end; - } data[MAX_BSETS]; + } data[]; +}; + +/* Fixed-size btree_iter that can be allocated on the stack */ + +struct btree_iter_stack { + struct btree_iter iter; + struct btree_iter_set stack_data[MAX_BSETS]; }; typedef bool (*ptr_filter_fn)(struct btree_keys *b, const struct bkey *k); @@ -333,9 +340,9 @@ struct bkey *bch_btree_iter_next_filter(struct btree_iter *iter, void bch_btree_iter_push(struct btree_iter *iter, struct bkey *k, struct bkey *end); -struct bkey *bch_btree_iter_init(struct btree_keys *b, - struct btree_iter *iter, - struct bkey *search); +struct bkey *bch_btree_iter_stack_init(struct btree_keys *b, + struct btree_iter_stack *iter, + struct bkey *search); struct bkey *__bch_bset_search(struct btree_keys *b, struct bset_tree *t, const struct bkey *search); @@ -350,13 +357,14 @@ static inline struct bkey *bch_bset_search(struct btree_keys *b, return search ? __bch_bset_search(b, t, search) : t->data->start; } -#define for_each_key_filter(b, k, iter, filter) \ - for (bch_btree_iter_init((b), (iter), NULL); \ - ((k) = bch_btree_iter_next_filter((iter), (b), filter));) +#define for_each_key_filter(b, k, stack_iter, filter) \ + for (bch_btree_iter_stack_init((b), (stack_iter), NULL); \ + ((k) = bch_btree_iter_next_filter(&((stack_iter)->iter), (b), \ + filter));) -#define for_each_key(b, k, iter) \ - for (bch_btree_iter_init((b), (iter), NULL); \ - ((k) = bch_btree_iter_next(iter));) +#define for_each_key(b, k, stack_iter) \ + for (bch_btree_iter_stack_init((b), (stack_iter), NULL); \ + ((k) = bch_btree_iter_next(&((stack_iter)->iter)));) /* Sorting */ diff --git a/drivers/md/bcache/btree.c b/drivers/md/bcache/btree.c index 196cdacce38f..d011a7154d33 100644 --- a/drivers/md/bcache/btree.c +++ b/drivers/md/bcache/btree.c @@ -1309,7 +1309,7 @@ static bool btree_gc_mark_node(struct btree *b, struct gc_stat *gc) uint8_t stale = 0; unsigned int keys = 0, good_keys = 0; struct bkey *k; - struct btree_iter iter; + struct btree_iter_stack iter; struct bset_tree *t; gc->nodes++; @@ -1570,7 +1570,7 @@ static int btree_gc_rewrite_node(struct btree *b, struct btree_op *op, static unsigned int btree_gc_count_keys(struct btree *b) { struct bkey *k; - struct btree_iter iter; + struct btree_iter_stack iter; unsigned int ret = 0; for_each_key_filter(&b->keys, k, &iter, bch_ptr_bad) @@ -1611,17 +1611,18 @@ static int btree_gc_recurse(struct btree *b, struct btree_op *op, int ret = 0; bool should_rewrite; struct bkey *k; - struct btree_iter iter; + struct btree_iter_stack iter; struct gc_merge_info r[GC_MERGE_NODES]; struct gc_merge_info *i, *last = r + ARRAY_SIZE(r) - 1; - bch_btree_iter_init(&b->keys, &iter, &b->c->gc_done); + bch_btree_iter_stack_init(&b->keys, &iter, &b->c->gc_done); for (i = r; i < r + ARRAY_SIZE(r); i++) i->b = ERR_PTR(-EINTR); while (1) { - k = bch_btree_iter_next_filter(&iter, &b->keys, bch_ptr_bad); + k = bch_btree_iter_next_filter(&iter.iter, &b->keys, + bch_ptr_bad); if (k) { r->b = bch_btree_node_get(b->c, op, k, b->level - 1, true, b); @@ -1911,7 +1912,7 @@ static int bch_btree_check_recurse(struct btree *b, struct btree_op *op) { int ret = 0; struct bkey *k, *p = NULL; - struct btree_iter iter; + struct btree_iter_stack iter; for_each_key_filter(&b->keys, k, &iter, bch_ptr_invalid) bch_initial_mark_key(b->c, b->level, k); @@ -1919,10 +1920,10 @@ static int bch_btree_check_recurse(struct btree *b, struct btree_op *op) bch_initial_mark_key(b->c, b->level + 1, &b->key); if (b->level) { - bch_btree_iter_init(&b->keys, &iter, NULL); + bch_btree_iter_stack_init(&b->keys, &iter, NULL); do { - k = bch_btree_iter_next_filter(&iter, &b->keys, + k = bch_btree_iter_next_filter(&iter.iter, &b->keys, bch_ptr_bad); if (k) { btree_node_prefetch(b, k); @@ -1950,7 +1951,7 @@ static int bch_btree_check_thread(void *arg) struct btree_check_info *info = arg; struct btree_check_state *check_state = info->state; struct cache_set *c = check_state->c; - struct btree_iter iter; + struct btree_iter_stack iter; struct bkey *k, *p; int cur_idx, prev_idx, skip_nr; @@ -1959,8 +1960,8 @@ static int bch_btree_check_thread(void *arg) ret = 0; /* root node keys are checked before thread created */ - bch_btree_iter_init(&c->root->keys, &iter, NULL); - k = bch_btree_iter_next_filter(&iter, &c->root->keys, bch_ptr_bad); + bch_btree_iter_stack_init(&c->root->keys, &iter, NULL); + k = bch_btree_iter_next_filter(&iter.iter, &c->root->keys, bch_ptr_bad); BUG_ON(!k); p = k; @@ -1978,7 +1979,7 @@ static int bch_btree_check_thread(void *arg) skip_nr = cur_idx - prev_idx; while (skip_nr) { - k = bch_btree_iter_next_filter(&iter, + k = bch_btree_iter_next_filter(&iter.iter, &c->root->keys, bch_ptr_bad); if (k) @@ -2051,7 +2052,7 @@ int bch_btree_check(struct cache_set *c) int ret = 0; int i; struct bkey *k = NULL; - struct btree_iter iter; + struct btree_iter_stack iter; struct btree_check_state check_state; /* check and mark root node keys */ @@ -2547,11 +2548,11 @@ static int bch_btree_map_nodes_recurse(struct btree *b, struct btree_op *op, if (b->level) { struct bkey *k; - struct btree_iter iter; + struct btree_iter_stack iter; - bch_btree_iter_init(&b->keys, &iter, from); + bch_btree_iter_stack_init(&b->keys, &iter, from); - while ((k = bch_btree_iter_next_filter(&iter, &b->keys, + while ((k = bch_btree_iter_next_filter(&iter.iter, &b->keys, bch_ptr_bad))) { ret = bcache_btree(map_nodes_recurse, k, b, op, from, fn, flags); @@ -2580,11 +2581,12 @@ int bch_btree_map_keys_recurse(struct btree *b, struct btree_op *op, { int ret = MAP_CONTINUE; struct bkey *k; - struct btree_iter iter; + struct btree_iter_stack iter; - bch_btree_iter_init(&b->keys, &iter, from); + bch_btree_iter_stack_init(&b->keys, &iter, from); - while ((k = bch_btree_iter_next_filter(&iter, &b->keys, bch_ptr_bad))) { + while ((k = bch_btree_iter_next_filter(&iter.iter, &b->keys, + bch_ptr_bad))) { ret = !b->level ? fn(op, b, k) : bcache_btree(map_keys_recurse, k, diff --git a/drivers/md/bcache/super.c b/drivers/md/bcache/super.c index 330bcd9ea4a9..cba09660148a 100644 --- a/drivers/md/bcache/super.c +++ b/drivers/md/bcache/super.c @@ -881,8 +881,8 @@ static void bcache_device_free(struct bcache_device *d) bcache_device_detach(d); if (disk) { - ida_simple_remove(&bcache_device_idx, - first_minor_to_idx(disk->first_minor)); + ida_free(&bcache_device_idx, + first_minor_to_idx(disk->first_minor)); put_disk(disk); } @@ -940,8 +940,8 @@ static int bcache_device_init(struct bcache_device *d, unsigned int block_size, if (!d->full_dirty_stripes) goto out_free_stripe_sectors_dirty; - idx = ida_simple_get(&bcache_device_idx, 0, - BCACHE_DEVICE_IDX_MAX, GFP_KERNEL); + idx = ida_alloc_max(&bcache_device_idx, BCACHE_DEVICE_IDX_MAX - 1, + GFP_KERNEL); if (idx < 0) goto out_free_full_dirty_stripes; @@ -986,7 +986,7 @@ static int bcache_device_init(struct bcache_device *d, unsigned int block_size, out_bioset_exit: bioset_exit(&d->bio_split); out_ida_remove: - ida_simple_remove(&bcache_device_idx, idx); + ida_free(&bcache_device_idx, idx); out_free_full_dirty_stripes: kvfree(d->full_dirty_stripes); out_free_stripe_sectors_dirty: @@ -1914,8 +1914,9 @@ struct cache_set *bch_cache_set_alloc(struct cache_sb *sb) INIT_LIST_HEAD(&c->btree_cache_freed); INIT_LIST_HEAD(&c->data_buckets); - iter_size = ((meta_bucket_pages(sb) * PAGE_SECTORS) / sb->block_size + 1) * - sizeof(struct btree_iter_set); + iter_size = sizeof(struct btree_iter) + + ((meta_bucket_pages(sb) * PAGE_SECTORS) / sb->block_size) * + sizeof(struct btree_iter_set); c->devices = kcalloc(c->nr_uuids, sizeof(void *), GFP_KERNEL); if (!c->devices) diff --git a/drivers/md/bcache/sysfs.c b/drivers/md/bcache/sysfs.c index 6956beb55326..826b14cae4e5 100644 --- a/drivers/md/bcache/sysfs.c +++ b/drivers/md/bcache/sysfs.c @@ -660,7 +660,7 @@ static unsigned int bch_root_usage(struct cache_set *c) unsigned int bytes = 0; struct bkey *k; struct btree *b; - struct btree_iter iter; + struct btree_iter_stack iter; goto lock_root; diff --git a/drivers/md/bcache/writeback.c b/drivers/md/bcache/writeback.c index 8827a6f130ad..792e070ccf38 100644 --- a/drivers/md/bcache/writeback.c +++ b/drivers/md/bcache/writeback.c @@ -908,15 +908,15 @@ static int bch_dirty_init_thread(void *arg) struct dirty_init_thrd_info *info = arg; struct bch_dirty_init_state *state = info->state; struct cache_set *c = state->c; - struct btree_iter iter; + struct btree_iter_stack iter; struct bkey *k, *p; int cur_idx, prev_idx, skip_nr; k = p = NULL; prev_idx = 0; - bch_btree_iter_init(&c->root->keys, &iter, NULL); - k = bch_btree_iter_next_filter(&iter, &c->root->keys, bch_ptr_bad); + bch_btree_iter_stack_init(&c->root->keys, &iter, NULL); + k = bch_btree_iter_next_filter(&iter.iter, &c->root->keys, bch_ptr_bad); BUG_ON(!k); p = k; @@ -930,7 +930,7 @@ static int bch_dirty_init_thread(void *arg) skip_nr = cur_idx - prev_idx; while (skip_nr) { - k = bch_btree_iter_next_filter(&iter, + k = bch_btree_iter_next_filter(&iter.iter, &c->root->keys, bch_ptr_bad); if (k) @@ -979,7 +979,7 @@ void bch_sectors_dirty_init(struct bcache_device *d) int i; struct btree *b = NULL; struct bkey *k = NULL; - struct btree_iter iter; + struct btree_iter_stack iter; struct sectors_dirty_init op; struct cache_set *c = d->c; struct bch_dirty_init_state state; diff --git a/drivers/md/dm-bio-prison-v2.c b/drivers/md/dm-bio-prison-v2.c index fd852981ef9c..cf433b0cf742 100644 --- a/drivers/md/dm-bio-prison-v2.c +++ b/drivers/md/dm-bio-prison-v2.c @@ -321,8 +321,7 @@ static bool __unlock(struct dm_bio_prison_v2 *prison, { BUG_ON(!cell->exclusive_lock); - bio_list_merge(bios, &cell->bios); - bio_list_init(&cell->bios); + bio_list_merge_init(bios, &cell->bios); if (cell->shared_count) { cell->exclusive_lock = false; diff --git a/drivers/md/dm-cache-target.c b/drivers/md/dm-cache-target.c index 911f73f7ebba..0fcbf8603846 100644 --- a/drivers/md/dm-cache-target.c +++ b/drivers/md/dm-cache-target.c @@ -115,8 +115,7 @@ static void __commit(struct work_struct *_ws) */ spin_lock_irq(&b->lock); list_splice_init(&b->work_items, &work_items); - bio_list_merge(&bios, &b->bios); - bio_list_init(&b->bios); + bio_list_merge_init(&bios, &b->bios); b->commit_scheduled = false; spin_unlock_irq(&b->lock); @@ -565,8 +564,7 @@ static void defer_bio(struct cache *cache, struct bio *bio) static void defer_bios(struct cache *cache, struct bio_list *bios) { spin_lock_irq(&cache->lock); - bio_list_merge(&cache->deferred_bios, bios); - bio_list_init(bios); + bio_list_merge_init(&cache->deferred_bios, bios); spin_unlock_irq(&cache->lock); wake_deferred_bio_worker(cache); @@ -1816,8 +1814,7 @@ static void process_deferred_bios(struct work_struct *ws) bio_list_init(&bios); spin_lock_irq(&cache->lock); - bio_list_merge(&bios, &cache->deferred_bios); - bio_list_init(&cache->deferred_bios); + bio_list_merge_init(&bios, &cache->deferred_bios); spin_unlock_irq(&cache->lock); while ((bio = bio_list_pop(&bios))) { @@ -1847,8 +1844,7 @@ static void requeue_deferred_bios(struct cache *cache) struct bio_list bios; bio_list_init(&bios); - bio_list_merge(&bios, &cache->deferred_bios); - bio_list_init(&cache->deferred_bios); + bio_list_merge_init(&bios, &cache->deferred_bios); while ((bio = bio_list_pop(&bios))) { bio->bi_status = BLK_STS_DM_REQUEUE; diff --git a/drivers/md/dm-clone-target.c b/drivers/md/dm-clone-target.c index 94b2fc33f64b..3f68672ab7c9 100644 --- a/drivers/md/dm-clone-target.c +++ b/drivers/md/dm-clone-target.c @@ -1181,8 +1181,7 @@ static void process_deferred_discards(struct clone *clone) struct bio_list discards = BIO_EMPTY_LIST; spin_lock_irq(&clone->lock); - bio_list_merge(&discards, &clone->deferred_discard_bios); - bio_list_init(&clone->deferred_discard_bios); + bio_list_merge_init(&discards, &clone->deferred_discard_bios); spin_unlock_irq(&clone->lock); if (bio_list_empty(&discards)) @@ -1215,8 +1214,7 @@ static void process_deferred_bios(struct clone *clone) struct bio_list bios = BIO_EMPTY_LIST; spin_lock_irq(&clone->lock); - bio_list_merge(&bios, &clone->deferred_bios); - bio_list_init(&clone->deferred_bios); + bio_list_merge_init(&bios, &clone->deferred_bios); spin_unlock_irq(&clone->lock); if (bio_list_empty(&bios)) @@ -1237,11 +1235,9 @@ static void process_deferred_flush_bios(struct clone *clone) * before issuing them or signaling their completion. */ spin_lock_irq(&clone->lock); - bio_list_merge(&bios, &clone->deferred_flush_bios); - bio_list_init(&clone->deferred_flush_bios); - - bio_list_merge(&bio_completions, &clone->deferred_flush_completions); - bio_list_init(&clone->deferred_flush_completions); + bio_list_merge_init(&bios, &clone->deferred_flush_bios); + bio_list_merge_init(&bio_completions, + &clone->deferred_flush_completions); spin_unlock_irq(&clone->lock); if (bio_list_empty(&bios) && bio_list_empty(&bio_completions) && diff --git a/drivers/md/dm-core.h b/drivers/md/dm-core.h index e6757a30dcca..08700bfc3e23 100644 --- a/drivers/md/dm-core.h +++ b/drivers/md/dm-core.h @@ -140,7 +140,7 @@ struct mapped_device { #ifdef CONFIG_BLK_DEV_ZONED unsigned int nr_zones; - unsigned int *zwp_offset; + void *zone_revalidate_map; #endif #ifdef CONFIG_IMA diff --git a/drivers/md/dm-era-target.c b/drivers/md/dm-era-target.c index 6acfa5bf97a4..8f81e597858d 100644 --- a/drivers/md/dm-era-target.c +++ b/drivers/md/dm-era-target.c @@ -1272,8 +1272,7 @@ static void process_deferred_bios(struct era *era) bio_list_init(&marked_bios); spin_lock(&era->deferred_lock); - bio_list_merge(&deferred_bios, &era->deferred_bios); - bio_list_init(&era->deferred_bios); + bio_list_merge_init(&deferred_bios, &era->deferred_bios); spin_unlock(&era->deferred_lock); if (bio_list_empty(&deferred_bios)) diff --git a/drivers/md/dm-mpath.c b/drivers/md/dm-mpath.c index 05d1328d1811..15b681b90153 100644 --- a/drivers/md/dm-mpath.c +++ b/drivers/md/dm-mpath.c @@ -704,8 +704,7 @@ static void process_queued_bios(struct work_struct *work) return; } - bio_list_merge(&bios, &m->queued_bios); - bio_list_init(&m->queued_bios); + bio_list_merge_init(&bios, &m->queued_bios); spin_unlock_irqrestore(&m->lock, flags); diff --git a/drivers/md/dm-table.c b/drivers/md/dm-table.c index 41f1d731ae5a..2c6fbd87363f 100644 --- a/drivers/md/dm-table.c +++ b/drivers/md/dm-table.c @@ -2042,7 +2042,8 @@ int dm_table_set_restrictions(struct dm_table *t, struct request_queue *q, r = dm_set_zones_restrictions(t, q); if (r) return r; - if (!static_key_enabled(&zoned_enabled.key)) + if (blk_queue_is_zoned(q) && + !static_key_enabled(&zoned_enabled.key)) static_branch_enable(&zoned_enabled); } diff --git a/drivers/md/dm-thin.c b/drivers/md/dm-thin.c index 4793ad2aa1f7..f359984c8ef2 100644 --- a/drivers/md/dm-thin.c +++ b/drivers/md/dm-thin.c @@ -592,12 +592,6 @@ struct dm_thin_endio_hook { struct dm_bio_prison_cell *cell; }; -static void __merge_bio_list(struct bio_list *bios, struct bio_list *master) -{ - bio_list_merge(bios, master); - bio_list_init(master); -} - static void error_bio_list(struct bio_list *bios, blk_status_t error) { struct bio *bio; @@ -616,7 +610,7 @@ static void error_thin_bio_list(struct thin_c *tc, struct bio_list *master, bio_list_init(&bios); spin_lock_irq(&tc->lock); - __merge_bio_list(&bios, master); + bio_list_merge_init(&bios, master); spin_unlock_irq(&tc->lock); error_bio_list(&bios, error); @@ -645,8 +639,8 @@ static void requeue_io(struct thin_c *tc) bio_list_init(&bios); spin_lock_irq(&tc->lock); - __merge_bio_list(&bios, &tc->deferred_bio_list); - __merge_bio_list(&bios, &tc->retry_on_resume_list); + bio_list_merge_init(&bios, &tc->deferred_bio_list); + bio_list_merge_init(&bios, &tc->retry_on_resume_list); spin_unlock_irq(&tc->lock); error_bio_list(&bios, BLK_STS_DM_REQUEUE); diff --git a/drivers/md/dm-vdo/data-vio.c b/drivers/md/dm-vdo/data-vio.c index 94f6f1ccfb7d..ab3ea8337809 100644 --- a/drivers/md/dm-vdo/data-vio.c +++ b/drivers/md/dm-vdo/data-vio.c @@ -604,8 +604,7 @@ static void assign_discard_permit(struct limiter *limiter) static void get_waiters(struct limiter *limiter) { - bio_list_merge(&limiter->waiters, &limiter->new_waiters); - bio_list_init(&limiter->new_waiters); + bio_list_merge_init(&limiter->waiters, &limiter->new_waiters); } static inline struct data_vio *get_available_data_vio(struct data_vio_pool *pool) diff --git a/drivers/md/dm-vdo/flush.c b/drivers/md/dm-vdo/flush.c index 57e87f0d7069..dd4fdee2ca0c 100644 --- a/drivers/md/dm-vdo/flush.c +++ b/drivers/md/dm-vdo/flush.c @@ -369,8 +369,7 @@ void vdo_dump_flusher(const struct flusher *flusher) static void initialize_flush(struct vdo_flush *flush, struct vdo *vdo) { bio_list_init(&flush->bios); - bio_list_merge(&flush->bios, &vdo->flusher->waiting_flush_bios); - bio_list_init(&vdo->flusher->waiting_flush_bios); + bio_list_merge_init(&flush->bios, &vdo->flusher->waiting_flush_bios); } static void launch_flush(struct vdo_flush *flush) diff --git a/drivers/md/dm-zone.c b/drivers/md/dm-zone.c index eb9832b22b14..8e6bcb0d786a 100644 --- a/drivers/md/dm-zone.c +++ b/drivers/md/dm-zone.c @@ -60,16 +60,23 @@ int dm_blk_report_zones(struct gendisk *disk, sector_t sector, struct dm_table *map; int srcu_idx, ret; - if (dm_suspended_md(md)) - return -EAGAIN; + if (!md->zone_revalidate_map) { + /* Regular user context */ + if (dm_suspended_md(md)) + return -EAGAIN; - map = dm_get_live_table(md, &srcu_idx); - if (!map) - return -EIO; + map = dm_get_live_table(md, &srcu_idx); + if (!map) + return -EIO; + } else { + /* Zone revalidation during __bind() */ + map = md->zone_revalidate_map; + } ret = dm_blk_do_report_zones(md, map, sector, nr_zones, cb, data); - dm_put_live_table(md, srcu_idx); + if (!md->zone_revalidate_map) + dm_put_live_table(md, srcu_idx); return ret; } @@ -138,80 +145,47 @@ bool dm_is_zone_write(struct mapped_device *md, struct bio *bio) } } -void dm_cleanup_zoned_dev(struct mapped_device *md) +/* + * Count conventional zones of a mapped zoned device. If the device + * only has conventional zones, do not expose it as zoned. + */ +static int dm_check_zoned_cb(struct blk_zone *zone, unsigned int idx, + void *data) { - if (md->disk) { - bitmap_free(md->disk->conv_zones_bitmap); - md->disk->conv_zones_bitmap = NULL; - bitmap_free(md->disk->seq_zones_wlock); - md->disk->seq_zones_wlock = NULL; - } + unsigned int *nr_conv_zones = data; - kvfree(md->zwp_offset); - md->zwp_offset = NULL; - md->nr_zones = 0; -} + if (zone->type == BLK_ZONE_TYPE_CONVENTIONAL) + (*nr_conv_zones)++; -static unsigned int dm_get_zone_wp_offset(struct blk_zone *zone) -{ - switch (zone->cond) { - case BLK_ZONE_COND_IMP_OPEN: - case BLK_ZONE_COND_EXP_OPEN: - case BLK_ZONE_COND_CLOSED: - return zone->wp - zone->start; - case BLK_ZONE_COND_FULL: - return zone->len; - case BLK_ZONE_COND_EMPTY: - case BLK_ZONE_COND_NOT_WP: - case BLK_ZONE_COND_OFFLINE: - case BLK_ZONE_COND_READONLY: - default: - /* - * Conventional, offline and read-only zones do not have a valid - * write pointer. Use 0 as for an empty zone. - */ - return 0; - } + return 0; } -static int dm_zone_revalidate_cb(struct blk_zone *zone, unsigned int idx, - void *data) +static int dm_check_zoned(struct mapped_device *md, struct dm_table *t) { - struct mapped_device *md = data; struct gendisk *disk = md->disk; + unsigned int nr_conv_zones = 0; + int ret; - switch (zone->type) { - case BLK_ZONE_TYPE_CONVENTIONAL: - if (!disk->conv_zones_bitmap) { - disk->conv_zones_bitmap = bitmap_zalloc(disk->nr_zones, - GFP_NOIO); - if (!disk->conv_zones_bitmap) - return -ENOMEM; - } - set_bit(idx, disk->conv_zones_bitmap); - break; - case BLK_ZONE_TYPE_SEQWRITE_REQ: - case BLK_ZONE_TYPE_SEQWRITE_PREF: - if (!disk->seq_zones_wlock) { - disk->seq_zones_wlock = bitmap_zalloc(disk->nr_zones, - GFP_NOIO); - if (!disk->seq_zones_wlock) - return -ENOMEM; - } - if (!md->zwp_offset) { - md->zwp_offset = - kvcalloc(disk->nr_zones, sizeof(unsigned int), - GFP_KERNEL); - if (!md->zwp_offset) - return -ENOMEM; - } - md->zwp_offset[idx] = dm_get_zone_wp_offset(zone); - - break; - default: - DMERR("Invalid zone type 0x%x at sectors %llu", - (int)zone->type, zone->start); - return -ENODEV; + /* Count conventional zones */ + md->zone_revalidate_map = t; + ret = dm_blk_report_zones(disk, 0, UINT_MAX, + dm_check_zoned_cb, &nr_conv_zones); + md->zone_revalidate_map = NULL; + if (ret < 0) { + DMERR("Check zoned failed %d", ret); + return ret; + } + + /* + * If we only have conventional zones, expose the mapped device as + * a regular device. + */ + if (nr_conv_zones >= ret) { + disk->queue->limits.max_open_zones = 0; + disk->queue->limits.max_active_zones = 0; + disk->queue->limits.zoned = false; + clear_bit(DMF_EMULATE_ZONE_APPEND, &md->flags); + disk->nr_zones = 0; } return 0; @@ -226,41 +200,32 @@ static int dm_zone_revalidate_cb(struct blk_zone *zone, unsigned int idx, static int dm_revalidate_zones(struct mapped_device *md, struct dm_table *t) { struct gendisk *disk = md->disk; - unsigned int noio_flag; int ret; - /* - * Check if something changed. If yes, cleanup the current resources - * and reallocate everything. - */ + /* Revalidate only if something changed. */ if (!disk->nr_zones || disk->nr_zones != md->nr_zones) - dm_cleanup_zoned_dev(md); + md->nr_zones = 0; + if (md->nr_zones) return 0; /* - * Scan all zones to initialize everything. Ensure that all vmalloc - * operations in this context are done as if GFP_NOIO was specified. + * Our table is not live yet. So the call to dm_get_live_table() + * in dm_blk_report_zones() will fail. Set a temporary pointer to + * our table for dm_blk_report_zones() to use directly. */ - noio_flag = memalloc_noio_save(); - ret = dm_blk_do_report_zones(md, t, 0, disk->nr_zones, - dm_zone_revalidate_cb, md); - memalloc_noio_restore(noio_flag); - if (ret < 0) - goto err; - if (ret != disk->nr_zones) { - ret = -EIO; - goto err; + md->zone_revalidate_map = t; + ret = blk_revalidate_disk_zones(disk); + md->zone_revalidate_map = NULL; + + if (ret) { + DMERR("Revalidate zones failed %d", ret); + return ret; } md->nr_zones = disk->nr_zones; return 0; - -err: - DMERR("Revalidate zones failed %d", ret); - dm_cleanup_zoned_dev(md); - return ret; } static int device_not_zone_append_capable(struct dm_target *ti, @@ -289,294 +254,40 @@ static bool dm_table_supports_zone_append(struct dm_table *t) int dm_set_zones_restrictions(struct dm_table *t, struct request_queue *q) { struct mapped_device *md = t->md; + int ret; /* - * For a zoned target, the number of zones should be updated for the - * correct value to be exposed in sysfs queue/nr_zones. + * Check if zone append is natively supported, and if not, set the + * mapped device queue as needing zone append emulation. */ WARN_ON_ONCE(queue_is_mq(q)); - md->disk->nr_zones = bdev_nr_zones(md->disk->part0); - - /* Check if zone append is natively supported */ if (dm_table_supports_zone_append(t)) { clear_bit(DMF_EMULATE_ZONE_APPEND, &md->flags); - dm_cleanup_zoned_dev(md); - return 0; + } else { + set_bit(DMF_EMULATE_ZONE_APPEND, &md->flags); + blk_queue_max_zone_append_sectors(q, 0); } - /* - * Mark the mapped device as needing zone append emulation and - * initialize the emulation resources once the capacity is set. - */ - set_bit(DMF_EMULATE_ZONE_APPEND, &md->flags); if (!get_capacity(md->disk)) return 0; - return dm_revalidate_zones(md, t); -} - -static int dm_update_zone_wp_offset_cb(struct blk_zone *zone, unsigned int idx, - void *data) -{ - unsigned int *wp_offset = data; - - *wp_offset = dm_get_zone_wp_offset(zone); - - return 0; -} - -static int dm_update_zone_wp_offset(struct mapped_device *md, unsigned int zno, - unsigned int *wp_ofst) -{ - sector_t sector = zno * bdev_zone_sectors(md->disk->part0); - unsigned int noio_flag; - struct dm_table *t; - int srcu_idx, ret; - - t = dm_get_live_table(md, &srcu_idx); - if (!t) - return -EIO; - - /* - * Ensure that all memory allocations in this context are done as if - * GFP_NOIO was specified. - */ - noio_flag = memalloc_noio_save(); - ret = dm_blk_do_report_zones(md, t, sector, 1, - dm_update_zone_wp_offset_cb, wp_ofst); - memalloc_noio_restore(noio_flag); - - dm_put_live_table(md, srcu_idx); - - if (ret != 1) - return -EIO; - - return 0; -} - -struct orig_bio_details { - enum req_op op; - unsigned int nr_sectors; -}; - -/* - * First phase of BIO mapping for targets with zone append emulation: - * check all BIO that change a zone writer pointer and change zone - * append operations into regular write operations. - */ -static bool dm_zone_map_bio_begin(struct mapped_device *md, - unsigned int zno, struct bio *clone) -{ - sector_t zsectors = bdev_zone_sectors(md->disk->part0); - unsigned int zwp_offset = READ_ONCE(md->zwp_offset[zno]); - - /* - * If the target zone is in an error state, recover by inspecting the - * zone to get its current write pointer position. Note that since the - * target zone is already locked, a BIO issuing context should never - * see the zone write in the DM_ZONE_UPDATING_WP_OFST state. - */ - if (zwp_offset == DM_ZONE_INVALID_WP_OFST) { - if (dm_update_zone_wp_offset(md, zno, &zwp_offset)) - return false; - WRITE_ONCE(md->zwp_offset[zno], zwp_offset); - } - - switch (bio_op(clone)) { - case REQ_OP_ZONE_RESET: - case REQ_OP_ZONE_FINISH: - return true; - case REQ_OP_WRITE_ZEROES: - case REQ_OP_WRITE: - /* Writes must be aligned to the zone write pointer */ - if ((clone->bi_iter.bi_sector & (zsectors - 1)) != zwp_offset) - return false; - break; - case REQ_OP_ZONE_APPEND: - /* - * Change zone append operations into a non-mergeable regular - * writes directed at the current write pointer position of the - * target zone. - */ - clone->bi_opf = REQ_OP_WRITE | REQ_NOMERGE | - (clone->bi_opf & (~REQ_OP_MASK)); - clone->bi_iter.bi_sector += zwp_offset; - break; - default: - DMWARN_LIMIT("Invalid BIO operation"); - return false; - } - - /* Cannot write to a full zone */ - if (zwp_offset >= zsectors) - return false; - - return true; -} - -/* - * Second phase of BIO mapping for targets with zone append emulation: - * update the zone write pointer offset array to account for the additional - * data written to a zone. Note that at this point, the remapped clone BIO - * may already have completed, so we do not touch it. - */ -static blk_status_t dm_zone_map_bio_end(struct mapped_device *md, unsigned int zno, - struct orig_bio_details *orig_bio_details, - unsigned int nr_sectors) -{ - unsigned int zwp_offset = READ_ONCE(md->zwp_offset[zno]); - - /* The clone BIO may already have been completed and failed */ - if (zwp_offset == DM_ZONE_INVALID_WP_OFST) - return BLK_STS_IOERR; - - /* Update the zone wp offset */ - switch (orig_bio_details->op) { - case REQ_OP_ZONE_RESET: - WRITE_ONCE(md->zwp_offset[zno], 0); - return BLK_STS_OK; - case REQ_OP_ZONE_FINISH: - WRITE_ONCE(md->zwp_offset[zno], - bdev_zone_sectors(md->disk->part0)); - return BLK_STS_OK; - case REQ_OP_WRITE_ZEROES: - case REQ_OP_WRITE: - WRITE_ONCE(md->zwp_offset[zno], zwp_offset + nr_sectors); - return BLK_STS_OK; - case REQ_OP_ZONE_APPEND: - /* - * Check that the target did not truncate the write operation - * emulating a zone append. - */ - if (nr_sectors != orig_bio_details->nr_sectors) { - DMWARN_LIMIT("Truncated write for zone append"); - return BLK_STS_IOERR; - } - WRITE_ONCE(md->zwp_offset[zno], zwp_offset + nr_sectors); - return BLK_STS_OK; - default: - DMWARN_LIMIT("Invalid BIO operation"); - return BLK_STS_IOERR; - } -} - -static inline void dm_zone_lock(struct gendisk *disk, unsigned int zno, - struct bio *clone) -{ - if (WARN_ON_ONCE(bio_flagged(clone, BIO_ZONE_WRITE_LOCKED))) - return; - - wait_on_bit_lock_io(disk->seq_zones_wlock, zno, TASK_UNINTERRUPTIBLE); - bio_set_flag(clone, BIO_ZONE_WRITE_LOCKED); -} - -static inline void dm_zone_unlock(struct gendisk *disk, unsigned int zno, - struct bio *clone) -{ - if (!bio_flagged(clone, BIO_ZONE_WRITE_LOCKED)) - return; - - WARN_ON_ONCE(!test_bit(zno, disk->seq_zones_wlock)); - clear_bit_unlock(zno, disk->seq_zones_wlock); - smp_mb__after_atomic(); - wake_up_bit(disk->seq_zones_wlock, zno); - - bio_clear_flag(clone, BIO_ZONE_WRITE_LOCKED); -} - -static bool dm_need_zone_wp_tracking(struct bio *bio) -{ /* - * Special processing is not needed for operations that do not need the - * zone write lock, that is, all operations that target conventional - * zones and all operations that do not modify directly a sequential - * zone write pointer. + * Check that the mapped device will indeed be zoned, that is, that it + * has sequential write required zones. */ - if (op_is_flush(bio->bi_opf) && !bio_sectors(bio)) - return false; - switch (bio_op(bio)) { - case REQ_OP_WRITE_ZEROES: - case REQ_OP_WRITE: - case REQ_OP_ZONE_RESET: - case REQ_OP_ZONE_FINISH: - case REQ_OP_ZONE_APPEND: - return bio_zone_is_seq(bio); - default: - return false; - } -} - -/* - * Special IO mapping for targets needing zone append emulation. - */ -int dm_zone_map_bio(struct dm_target_io *tio) -{ - struct dm_io *io = tio->io; - struct dm_target *ti = tio->ti; - struct mapped_device *md = io->md; - struct bio *clone = &tio->clone; - struct orig_bio_details orig_bio_details; - unsigned int zno; - blk_status_t sts; - int r; - - /* - * IOs that do not change a zone write pointer do not need - * any additional special processing. - */ - if (!dm_need_zone_wp_tracking(clone)) - return ti->type->map(ti, clone); - - /* Lock the target zone */ - zno = bio_zone_no(clone); - dm_zone_lock(md->disk, zno, clone); - - orig_bio_details.nr_sectors = bio_sectors(clone); - orig_bio_details.op = bio_op(clone); + ret = dm_check_zoned(md, t); + if (ret) + return ret; + if (!blk_queue_is_zoned(q)) + return 0; - /* - * Check that the bio and the target zone write pointer offset are - * both valid, and if the bio is a zone append, remap it to a write. - */ - if (!dm_zone_map_bio_begin(md, zno, clone)) { - dm_zone_unlock(md->disk, zno, clone); - return DM_MAPIO_KILL; + if (!md->disk->nr_zones) { + DMINFO("%s using %s zone append", + md->disk->disk_name, + queue_emulates_zone_append(q) ? "emulated" : "native"); } - /* Let the target do its work */ - r = ti->type->map(ti, clone); - switch (r) { - case DM_MAPIO_SUBMITTED: - /* - * The target submitted the clone BIO. The target zone will - * be unlocked on completion of the clone. - */ - sts = dm_zone_map_bio_end(md, zno, &orig_bio_details, - *tio->len_ptr); - break; - case DM_MAPIO_REMAPPED: - /* - * The target only remapped the clone BIO. In case of error, - * unlock the target zone here as the clone will not be - * submitted. - */ - sts = dm_zone_map_bio_end(md, zno, &orig_bio_details, - *tio->len_ptr); - if (sts != BLK_STS_OK) - dm_zone_unlock(md->disk, zno, clone); - break; - case DM_MAPIO_REQUEUE: - case DM_MAPIO_KILL: - default: - dm_zone_unlock(md->disk, zno, clone); - sts = BLK_STS_IOERR; - break; - } - - if (sts != BLK_STS_OK) - return DM_MAPIO_KILL; - - return r; + return dm_revalidate_zones(md, t); } /* @@ -587,61 +298,17 @@ void dm_zone_endio(struct dm_io *io, struct bio *clone) struct mapped_device *md = io->md; struct gendisk *disk = md->disk; struct bio *orig_bio = io->orig_bio; - unsigned int zwp_offset; - unsigned int zno; /* - * For targets that do not emulate zone append, we only need to - * handle native zone-append bios. + * Get the offset within the zone of the written sector + * and add that to the original bio sector position. */ - if (!dm_emulate_zone_append(md)) { - /* - * Get the offset within the zone of the written sector - * and add that to the original bio sector position. - */ - if (clone->bi_status == BLK_STS_OK && - bio_op(clone) == REQ_OP_ZONE_APPEND) { - sector_t mask = - (sector_t)bdev_zone_sectors(disk->part0) - 1; - - orig_bio->bi_iter.bi_sector += - clone->bi_iter.bi_sector & mask; - } - - return; - } + if (clone->bi_status == BLK_STS_OK && + bio_op(clone) == REQ_OP_ZONE_APPEND) { + sector_t mask = bdev_zone_sectors(disk->part0) - 1; - /* - * For targets that do emulate zone append, if the clone BIO does not - * own the target zone write lock, we have nothing to do. - */ - if (!bio_flagged(clone, BIO_ZONE_WRITE_LOCKED)) - return; - - zno = bio_zone_no(orig_bio); - - if (clone->bi_status != BLK_STS_OK) { - /* - * BIOs that modify a zone write pointer may leave the zone - * in an unknown state in case of failure (e.g. the write - * pointer was only partially advanced). In this case, set - * the target zone write pointer as invalid unless it is - * already being updated. - */ - WRITE_ONCE(md->zwp_offset[zno], DM_ZONE_INVALID_WP_OFST); - } else if (bio_op(orig_bio) == REQ_OP_ZONE_APPEND) { - /* - * Get the written sector for zone append operation that were - * emulated using regular write operations. - */ - zwp_offset = READ_ONCE(md->zwp_offset[zno]); - if (WARN_ON_ONCE(zwp_offset < bio_sectors(orig_bio))) - WRITE_ONCE(md->zwp_offset[zno], - DM_ZONE_INVALID_WP_OFST); - else - orig_bio->bi_iter.bi_sector += - zwp_offset - bio_sectors(orig_bio); + orig_bio->bi_iter.bi_sector += clone->bi_iter.bi_sector & mask; } - dm_zone_unlock(disk, zno, clone); + return; } diff --git a/drivers/md/dm.c b/drivers/md/dm.c index 7d0746b37c8e..597dd7a25823 100644 --- a/drivers/md/dm.c +++ b/drivers/md/dm.c @@ -1428,25 +1428,12 @@ static void __map_bio(struct bio *clone) down(&md->swap_bios_semaphore); } - if (static_branch_unlikely(&zoned_enabled)) { - /* - * Check if the IO needs a special mapping due to zone append - * emulation on zoned target. In this case, dm_zone_map_bio() - * calls the target map operation. - */ - if (unlikely(dm_emulate_zone_append(md))) - r = dm_zone_map_bio(tio); - else - goto do_map; - } else { -do_map: - if (likely(ti->type->map == linear_map)) - r = linear_map(ti, clone); - else if (ti->type->map == stripe_map) - r = stripe_map(ti, clone); - else - r = ti->type->map(ti, clone); - } + if (likely(ti->type->map == linear_map)) + r = linear_map(ti, clone); + else if (ti->type->map == stripe_map) + r = stripe_map(ti, clone); + else + r = ti->type->map(ti, clone); switch (r) { case DM_MAPIO_SUBMITTED: @@ -1774,6 +1761,33 @@ static void init_clone_info(struct clone_info *ci, struct dm_io *io, ci->sector_count = 0; } +#ifdef CONFIG_BLK_DEV_ZONED +static inline bool dm_zone_bio_needs_split(struct mapped_device *md, + struct bio *bio) +{ + /* + * For mapped device that need zone append emulation, we must + * split any large BIO that straddles zone boundaries. + */ + return dm_emulate_zone_append(md) && bio_straddles_zones(bio) && + !bio_flagged(bio, BIO_ZONE_WRITE_PLUGGING); +} +static inline bool dm_zone_plug_bio(struct mapped_device *md, struct bio *bio) +{ + return dm_emulate_zone_append(md) && blk_zone_plug_bio(bio, 0); +} +#else +static inline bool dm_zone_bio_needs_split(struct mapped_device *md, + struct bio *bio) +{ + return false; +} +static inline bool dm_zone_plug_bio(struct mapped_device *md, struct bio *bio) +{ + return false; +} +#endif + /* * Entry point to split a bio into clones and submit them to the targets. */ @@ -1783,19 +1797,32 @@ static void dm_split_and_process_bio(struct mapped_device *md, struct clone_info ci; struct dm_io *io; blk_status_t error = BLK_STS_OK; - bool is_abnormal; + bool is_abnormal, need_split; + + need_split = is_abnormal = is_abnormal_io(bio); + if (static_branch_unlikely(&zoned_enabled)) + need_split = is_abnormal || dm_zone_bio_needs_split(md, bio); - is_abnormal = is_abnormal_io(bio); - if (unlikely(is_abnormal)) { + if (unlikely(need_split)) { /* * Use bio_split_to_limits() for abnormal IO (e.g. discard, etc) * otherwise associated queue_limits won't be imposed. + * Also split the BIO for mapped devices needing zone append + * emulation to ensure that the BIO does not cross zone + * boundaries. */ bio = bio_split_to_limits(bio); if (!bio) return; } + /* + * Use the block layer zone write plugging for mapped devices that + * need zone append emulation (e.g. dm-crypt). + */ + if (static_branch_unlikely(&zoned_enabled) && dm_zone_plug_bio(md, bio)) + return; + /* Only support nowait for normal IO */ if (unlikely(bio->bi_opf & REQ_NOWAIT) && !is_abnormal) { io = alloc_io(md, bio, GFP_NOWAIT); @@ -2016,7 +2043,6 @@ static void cleanup_mapped_device(struct mapped_device *md) md->dax_dev = NULL; } - dm_cleanup_zoned_dev(md); if (md->disk) { spin_lock(&_minor_lock); md->disk->private_data = NULL; diff --git a/drivers/md/dm.h b/drivers/md/dm.h index 7f1acbf6bd9e..e0c57f19839b 100644 --- a/drivers/md/dm.h +++ b/drivers/md/dm.h @@ -104,13 +104,11 @@ int dm_setup_md_queue(struct mapped_device *md, struct dm_table *t); int dm_set_zones_restrictions(struct dm_table *t, struct request_queue *q); void dm_zone_endio(struct dm_io *io, struct bio *clone); #ifdef CONFIG_BLK_DEV_ZONED -void dm_cleanup_zoned_dev(struct mapped_device *md); int dm_blk_report_zones(struct gendisk *disk, sector_t sector, unsigned int nr_zones, report_zones_cb cb, void *data); bool dm_is_zone_write(struct mapped_device *md, struct bio *bio); int dm_zone_map_bio(struct dm_target_io *io); #else -static inline void dm_cleanup_zoned_dev(struct mapped_device *md) {} #define dm_blk_report_zones NULL static inline bool dm_is_zone_write(struct mapped_device *md, struct bio *bio) { diff --git a/drivers/md/md-bitmap.c b/drivers/md/md-bitmap.c index 059afc24c08b..0a2d37eb38ef 100644 --- a/drivers/md/md-bitmap.c +++ b/drivers/md/md-bitmap.c @@ -1424,7 +1424,7 @@ __acquires(bitmap->lock) sector_t chunk = offset >> bitmap->chunkshift; unsigned long page = chunk >> PAGE_COUNTER_SHIFT; unsigned long pageoff = (chunk & PAGE_COUNTER_MASK) << COUNTER_BYTE_SHIFT; - sector_t csize; + sector_t csize = ((sector_t)1) << bitmap->chunkshift; int err; if (page >= bitmap->pages) { @@ -1433,6 +1433,7 @@ __acquires(bitmap->lock) * End-of-device while looking for a whole page or * user set a huge number to sysfs bitmap_set_bits. */ + *blocks = csize - (offset & (csize - 1)); return NULL; } err = md_bitmap_checkpage(bitmap, page, create, 0); @@ -1441,8 +1442,7 @@ __acquires(bitmap->lock) bitmap->bp[page].map == NULL) csize = ((sector_t)1) << (bitmap->chunkshift + PAGE_COUNTER_SHIFT); - else - csize = ((sector_t)1) << bitmap->chunkshift; + *blocks = csize - (offset & (csize - 1)); if (err < 0) diff --git a/drivers/md/md.c b/drivers/md/md.c index e575e74aabf5..aff9118ff697 100644 --- a/drivers/md/md.c +++ b/drivers/md/md.c @@ -8087,7 +8087,8 @@ void md_wakeup_thread(struct md_thread __rcu *thread) if (t) { pr_debug("md: waking up MD thread %s.\n", t->tsk->comm); set_bit(THREAD_WAKEUP, &t->flags); - wake_up(&t->wqueue); + if (wq_has_sleeper(&t->wqueue)) + wake_up(&t->wqueue); } rcu_read_unlock(); } @@ -8582,6 +8583,10 @@ static int is_mddev_idle(struct mddev *mddev, int init) rcu_read_lock(); rdev_for_each_rcu(rdev, mddev) { struct gendisk *disk = rdev->bdev->bd_disk; + + if (!init && !blk_queue_io_stat(disk->queue)) + continue; + curr_events = (int)part_stat_read_accum(disk->part0, sectors) - atomic_read(&disk->sync_io); /* sync IO will cause sync_io to increase before the disk_stats diff --git a/drivers/md/md.h b/drivers/md/md.h index 097d9dbd69b8..ca085ecad504 100644 --- a/drivers/md/md.h +++ b/drivers/md/md.h @@ -621,7 +621,8 @@ extern void mddev_unlock(struct mddev *mddev); static inline void md_sync_acct(struct block_device *bdev, unsigned long nr_sectors) { - atomic_add(nr_sectors, &bdev->bd_disk->sync_io); + if (blk_queue_io_stat(bdev->bd_disk->queue)) + atomic_add(nr_sectors, &bdev->bd_disk->sync_io); } static inline void md_sync_acct_bio(struct bio *bio, unsigned long nr_sectors) diff --git a/drivers/md/raid5.c b/drivers/md/raid5.c index d874abfc1836..2bd1ce9b3922 100644 --- a/drivers/md/raid5.c +++ b/drivers/md/raid5.c @@ -36,7 +36,6 @@ */ #include <linux/blkdev.h> -#include <linux/delay.h> #include <linux/kthread.h> #include <linux/raid/pq.h> #include <linux/async_tx.h> @@ -6734,6 +6733,9 @@ static void raid5d(struct md_thread *thread) int batch_size, released; unsigned int offset; + if (test_bit(MD_SB_CHANGE_PENDING, &mddev->sb_flags)) + break; + released = release_stripe_list(conf, conf->temp_inactive_list); if (released) clear_bit(R5_DID_ALLOC, &conf->cache_state); @@ -6770,18 +6772,7 @@ static void raid5d(struct md_thread *thread) spin_unlock_irq(&conf->device_lock); md_check_recovery(mddev); spin_lock_irq(&conf->device_lock); - - /* - * Waiting on MD_SB_CHANGE_PENDING below may deadlock - * seeing md_check_recovery() is needed to clear - * the flag when using mdmon. - */ - continue; } - - wait_event_lock_irq(mddev->sb_wait, - !test_bit(MD_SB_CHANGE_PENDING, &mddev->sb_flags), - conf->device_lock); } pr_debug("%d stripes handled\n", handled); diff --git a/drivers/memory/brcmstb_memc.c b/drivers/memory/brcmstb_memc.c index ea9213f7152e..4f17a93aa028 100644 --- a/drivers/memory/brcmstb_memc.c +++ b/drivers/memory/brcmstb_memc.c @@ -243,6 +243,7 @@ static const struct of_device_id brcmstb_memc_of_match[] = { }, {} }; +MODULE_DEVICE_TABLE(of, brcmstb_memc_of_match); static int brcmstb_memc_suspend(struct device *dev) { diff --git a/drivers/memory/mtk-smi.c b/drivers/memory/mtk-smi.c index 572c7fbdcfd3..fbe52ecc0eca 100644 --- a/drivers/memory/mtk-smi.c +++ b/drivers/memory/mtk-smi.c @@ -450,6 +450,7 @@ static const struct of_device_id mtk_smi_larb_of_ids[] = { {.compatible = "mediatek,mt8195-smi-larb", .data = &mtk_smi_larb_mt8195}, {} }; +MODULE_DEVICE_TABLE(of, mtk_smi_larb_of_ids); static int mtk_smi_larb_sleep_ctrl_enable(struct mtk_smi_larb *larb) { @@ -735,6 +736,7 @@ static const struct of_device_id mtk_smi_common_of_ids[] = { {.compatible = "mediatek,mt8365-smi-common", .data = &mtk_smi_common_mt8365}, {} }; +MODULE_DEVICE_TABLE(of, mtk_smi_common_of_ids); static int mtk_smi_common_probe(struct platform_device *pdev) { diff --git a/drivers/message/fusion/mptsas.c b/drivers/message/fusion/mptsas.c index 300f8e955a53..0f80c840afc3 100644 --- a/drivers/message/fusion/mptsas.c +++ b/drivers/message/fusion/mptsas.c @@ -2964,17 +2964,13 @@ mptsas_exp_repmanufacture_info(MPT_ADAPTER *ioc, goto out_free; manufacture_reply = data_out + sizeof(struct rep_manu_request); - strscpy(edev->vendor_id, manufacture_reply->vendor_id, - sizeof(edev->vendor_id)); - strscpy(edev->product_id, manufacture_reply->product_id, - sizeof(edev->product_id)); - strscpy(edev->product_rev, manufacture_reply->product_rev, - sizeof(edev->product_rev)); + memtostr(edev->vendor_id, manufacture_reply->vendor_id); + memtostr(edev->product_id, manufacture_reply->product_id); + memtostr(edev->product_rev, manufacture_reply->product_rev); edev->level = manufacture_reply->sas_format; if (manufacture_reply->sas_format) { - strscpy(edev->component_vendor_id, - manufacture_reply->component_vendor_id, - sizeof(edev->component_vendor_id)); + memtostr(edev->component_vendor_id, + manufacture_reply->component_vendor_id); tmp = (u8 *)&manufacture_reply->component_id; edev->component_id = tmp[0] << 8 | tmp[1]; edev->component_revision_id = diff --git a/drivers/misc/lkdtm/Makefile b/drivers/misc/lkdtm/Makefile index 95ef971b5e1c..b28701138b4b 100644 --- a/drivers/misc/lkdtm/Makefile +++ b/drivers/misc/lkdtm/Makefile @@ -19,7 +19,7 @@ KASAN_SANITIZE_rodata.o := n KCSAN_SANITIZE_rodata.o := n KCOV_INSTRUMENT_rodata.o := n OBJECT_FILES_NON_STANDARD_rodata.o := y -CFLAGS_REMOVE_rodata.o += $(CC_FLAGS_LTO) $(RETHUNK_CFLAGS) +CFLAGS_REMOVE_rodata.o += $(CC_FLAGS_LTO) $(RETHUNK_CFLAGS) $(CC_FLAGS_CFI) OBJCOPYFLAGS := OBJCOPYFLAGS_rodata_objcopy.o := \ diff --git a/drivers/misc/lkdtm/perms.c b/drivers/misc/lkdtm/perms.c index b93404d65650..5b861dbff27e 100644 --- a/drivers/misc/lkdtm/perms.c +++ b/drivers/misc/lkdtm/perms.c @@ -61,7 +61,7 @@ static void *setup_function_descriptor(func_desc_t *fdesc, void *dst) return fdesc; } -static noinline void execute_location(void *dst, bool write) +static noinline __nocfi void execute_location(void *dst, bool write) { void (*func)(void); func_desc_t fdesc; diff --git a/drivers/net/dsa/mv88e6xxx/chip.c b/drivers/net/dsa/mv88e6xxx/chip.c index 14daf432f30b..0918bd6fa81d 100644 --- a/drivers/net/dsa/mv88e6xxx/chip.c +++ b/drivers/net/dsa/mv88e6xxx/chip.c @@ -637,12 +637,12 @@ static void mv88e6351_phylink_get_caps(struct mv88e6xxx_chip *chip, int port, MAC_1000FD; } -static int mv88e6352_get_port4_serdes_cmode(struct mv88e6xxx_chip *chip) +static int mv88e63xx_get_port_serdes_cmode(struct mv88e6xxx_chip *chip, int port) { u16 reg, val; int err; - err = mv88e6xxx_port_read(chip, 4, MV88E6XXX_PORT_STS, ®); + err = mv88e6xxx_port_read(chip, port, MV88E6XXX_PORT_STS, ®); if (err) return err; @@ -651,16 +651,16 @@ static int mv88e6352_get_port4_serdes_cmode(struct mv88e6xxx_chip *chip) return 0xf; val = reg & ~MV88E6XXX_PORT_STS_PHY_DETECT; - err = mv88e6xxx_port_write(chip, 4, MV88E6XXX_PORT_STS, val); + err = mv88e6xxx_port_write(chip, port, MV88E6XXX_PORT_STS, val); if (err) return err; - err = mv88e6xxx_port_read(chip, 4, MV88E6XXX_PORT_STS, &val); + err = mv88e6xxx_port_read(chip, port, MV88E6XXX_PORT_STS, &val); if (err) return err; /* Restore PHY_DETECT value */ - err = mv88e6xxx_port_write(chip, 4, MV88E6XXX_PORT_STS, reg); + err = mv88e6xxx_port_write(chip, port, MV88E6XXX_PORT_STS, reg); if (err) return err; @@ -688,7 +688,30 @@ static void mv88e6352_phylink_get_caps(struct mv88e6xxx_chip *chip, int port, if (err <= 0) return; - cmode = mv88e6352_get_port4_serdes_cmode(chip); + cmode = mv88e63xx_get_port_serdes_cmode(chip, port); + if (cmode < 0) + dev_err(chip->dev, "p%d: failed to read serdes cmode\n", + port); + else + mv88e6xxx_translate_cmode(cmode, supported); + } +} + +static void mv88e632x_phylink_get_caps(struct mv88e6xxx_chip *chip, int port, + struct phylink_config *config) +{ + unsigned long *supported = config->supported_interfaces; + int cmode; + + /* Translate the default cmode */ + mv88e6xxx_translate_cmode(chip->ports[port].cmode, supported); + + config->mac_capabilities = MAC_SYM_PAUSE | MAC_10 | MAC_100 | + MAC_1000FD; + + /* Port 0/1 are serdes only ports */ + if (port == 0 || port == 1) { + cmode = mv88e63xx_get_port_serdes_cmode(chip, port); if (cmode < 0) dev_err(chip->dev, "p%d: failed to read serdes cmode\n", port); @@ -5093,7 +5116,7 @@ static const struct mv88e6xxx_ops mv88e6320_ops = { .gpio_ops = &mv88e6352_gpio_ops, .avb_ops = &mv88e6352_avb_ops, .ptp_ops = &mv88e6352_ptp_ops, - .phylink_get_caps = mv88e6185_phylink_get_caps, + .phylink_get_caps = mv88e632x_phylink_get_caps, }; static const struct mv88e6xxx_ops mv88e6321_ops = { @@ -5139,7 +5162,7 @@ static const struct mv88e6xxx_ops mv88e6321_ops = { .gpio_ops = &mv88e6352_gpio_ops, .avb_ops = &mv88e6352_avb_ops, .ptp_ops = &mv88e6352_ptp_ops, - .phylink_get_caps = mv88e6185_phylink_get_caps, + .phylink_get_caps = mv88e632x_phylink_get_caps, }; static const struct mv88e6xxx_ops mv88e6341_ops = { diff --git a/drivers/net/ethernet/hisilicon/hns3/hnae3.h b/drivers/net/ethernet/hisilicon/hns3/hnae3.h index f19f1e1d1f9f..133c94646c21 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hnae3.h +++ b/drivers/net/ethernet/hisilicon/hns3/hnae3.h @@ -897,7 +897,7 @@ struct hnae3_handle { struct hnae3_roce_private_info rinfo; }; - u32 numa_node_mask; /* for multi-chip support */ + nodemask_t numa_node_mask; /* for multi-chip support */ enum hnae3_port_base_vlan_state port_base_vlan_state; diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.c b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.c index ff6a2ed23ddb..ce60332d83c3 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.c +++ b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.c @@ -1537,6 +1537,9 @@ static int hclge_configure(struct hclge_dev *hdev) cfg.default_speed, ret); return ret; } + hdev->hw.mac.req_speed = hdev->hw.mac.speed; + hdev->hw.mac.req_autoneg = AUTONEG_ENABLE; + hdev->hw.mac.req_duplex = DUPLEX_FULL; hclge_parse_link_mode(hdev, cfg.speed_ability); @@ -1766,7 +1769,8 @@ static int hclge_vport_setup(struct hclge_vport *vport, u16 num_tqps) nic->pdev = hdev->pdev; nic->ae_algo = &ae_algo; - nic->numa_node_mask = hdev->numa_node_mask; + bitmap_copy(nic->numa_node_mask.bits, hdev->numa_node_mask.bits, + MAX_NUMNODES); nic->kinfo.io_base = hdev->hw.hw.io_base; ret = hclge_knic_setup(vport, num_tqps, @@ -2458,7 +2462,8 @@ static int hclge_init_roce_base_info(struct hclge_vport *vport) roce->pdev = nic->pdev; roce->ae_algo = nic->ae_algo; - roce->numa_node_mask = nic->numa_node_mask; + bitmap_copy(roce->numa_node_mask.bits, nic->numa_node_mask.bits, + MAX_NUMNODES); return 0; } @@ -3342,9 +3347,9 @@ hclge_set_phy_link_ksettings(struct hnae3_handle *handle, return ret; } - hdev->hw.mac.autoneg = cmd->base.autoneg; - hdev->hw.mac.speed = cmd->base.speed; - hdev->hw.mac.duplex = cmd->base.duplex; + hdev->hw.mac.req_autoneg = cmd->base.autoneg; + hdev->hw.mac.req_speed = cmd->base.speed; + hdev->hw.mac.req_duplex = cmd->base.duplex; linkmode_copy(hdev->hw.mac.advertising, cmd->link_modes.advertising); return 0; @@ -3377,9 +3382,9 @@ static int hclge_tp_port_init(struct hclge_dev *hdev) if (!hnae3_dev_phy_imp_supported(hdev)) return 0; - cmd.base.autoneg = hdev->hw.mac.autoneg; - cmd.base.speed = hdev->hw.mac.speed; - cmd.base.duplex = hdev->hw.mac.duplex; + cmd.base.autoneg = hdev->hw.mac.req_autoneg; + cmd.base.speed = hdev->hw.mac.req_speed; + cmd.base.duplex = hdev->hw.mac.req_duplex; linkmode_copy(cmd.link_modes.advertising, hdev->hw.mac.advertising); return hclge_set_phy_link_ksettings(&hdev->vport->nic, &cmd); @@ -7952,8 +7957,7 @@ static void hclge_set_timer_task(struct hnae3_handle *handle, bool enable) /* Set the DOWN flag here to disable link updating */ set_bit(HCLGE_STATE_DOWN, &hdev->state); - /* flush memory to make sure DOWN is seen by service task */ - smp_mb__before_atomic(); + smp_mb__after_atomic(); /* flush memory to make sure DOWN is seen by service task */ hclge_flush_link_update(hdev); } } @@ -9906,6 +9910,7 @@ static int hclge_set_vlan_protocol_type(struct hclge_dev *hdev) static int hclge_init_vlan_filter(struct hclge_dev *hdev) { struct hclge_vport *vport; + bool enable = true; int ret; int i; @@ -9925,8 +9930,12 @@ static int hclge_init_vlan_filter(struct hclge_dev *hdev) vport->cur_vlan_fltr_en = true; } + if (test_bit(HNAE3_DEV_SUPPORT_VLAN_FLTR_MDF_B, hdev->ae_dev->caps) && + !test_bit(HNAE3_DEV_SUPPORT_PORT_VLAN_BYPASS_B, hdev->ae_dev->caps)) + enable = false; + return hclge_set_vlan_filter_ctrl(hdev, HCLGE_FILTER_TYPE_PORT, - HCLGE_FILTER_FE_INGRESS, true, 0); + HCLGE_FILTER_FE_INGRESS, enable, 0); } static int hclge_init_vlan_type(struct hclge_dev *hdev) @@ -11622,16 +11631,10 @@ static int hclge_init_ae_dev(struct hnae3_ae_dev *ae_dev) if (ret) goto out; - ret = hclge_devlink_init(hdev); - if (ret) - goto err_pci_uninit; - - devl_lock(hdev->devlink); - /* Firmware command queue initialize */ ret = hclge_comm_cmd_queue_init(hdev->pdev, &hdev->hw.hw); if (ret) - goto err_devlink_uninit; + goto err_pci_uninit; /* Firmware command initialize */ ret = hclge_comm_cmd_init(hdev->ae_dev, &hdev->hw.hw, &hdev->fw_version, @@ -11759,7 +11762,7 @@ static int hclge_init_ae_dev(struct hnae3_ae_dev *ae_dev) ret = hclge_update_port_info(hdev); if (ret) - goto err_mdiobus_unreg; + goto err_ptp_uninit; INIT_KFIFO(hdev->mac_tnl_log); @@ -11799,6 +11802,10 @@ static int hclge_init_ae_dev(struct hnae3_ae_dev *ae_dev) dev_warn(&pdev->dev, "failed to wake on lan init, ret = %d\n", ret); + ret = hclge_devlink_init(hdev); + if (ret) + goto err_ptp_uninit; + hclge_state_init(hdev); hdev->last_reset_time = jiffies; @@ -11806,10 +11813,10 @@ static int hclge_init_ae_dev(struct hnae3_ae_dev *ae_dev) HCLGE_DRIVER_NAME); hclge_task_schedule(hdev, round_jiffies_relative(HZ)); - - devl_unlock(hdev->devlink); return 0; +err_ptp_uninit: + hclge_ptp_uninit(hdev); err_mdiobus_unreg: if (hdev->hw.mac.phydev) mdiobus_unregister(hdev->hw.mac.mdio_bus); @@ -11819,9 +11826,6 @@ err_msi_uninit: pci_free_irq_vectors(pdev); err_cmd_uninit: hclge_comm_cmd_uninit(hdev->ae_dev, &hdev->hw.hw); -err_devlink_uninit: - devl_unlock(hdev->devlink); - hclge_devlink_uninit(hdev); err_pci_uninit: pcim_iounmap(pdev, hdev->hw.hw.io_base); pci_release_regions(pdev); diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.h b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.h index e821dd2f1528..3a9186457ad8 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.h +++ b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.h @@ -279,11 +279,14 @@ struct hclge_mac { u8 media_type; /* port media type, e.g. fibre/copper/backplane */ u8 mac_addr[ETH_ALEN]; u8 autoneg; + u8 req_autoneg; u8 duplex; + u8 req_duplex; u8 support_autoneg; u8 speed_type; /* 0: sfp speed, 1: active speed */ u8 lane_num; u32 speed; + u32 req_speed; u32 max_speed; u32 speed_ability; /* speed ability supported by current media */ u32 module_type; /* sub media type, e.g. kr/cr/sr/lr */ @@ -891,7 +894,7 @@ struct hclge_dev { u16 fdir_pf_filter_count; /* Num of guaranteed filters for this PF */ u16 num_alloc_vport; /* Num vports this driver supports */ - u32 numa_node_mask; + nodemask_t numa_node_mask; u16 rx_buf_len; u16 num_tx_desc; /* desc num of per tx queue */ u16 num_rx_desc; /* desc num of per rx queue */ diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_mbx.c b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_mbx.c index d4a0e0be7a72..59c863306657 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_mbx.c +++ b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_mbx.c @@ -1077,12 +1077,13 @@ static void hclge_mbx_request_handling(struct hclge_mbx_ops_param *param) hdev = param->vport->back; cmd_func = hclge_mbx_ops_list[param->req->msg.code]; - if (cmd_func) - ret = cmd_func(param); - else + if (!cmd_func) { dev_err(&hdev->pdev->dev, "un-supported mailbox message, code = %u\n", param->req->msg.code); + return; + } + ret = cmd_func(param); /* PF driver should not reply IMP */ if (hnae3_get_bit(param->req->mbx_need_resp, HCLGE_MBX_NEED_RESP_B) && diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3vf/hclgevf_main.c b/drivers/net/ethernet/hisilicon/hns3/hns3vf/hclgevf_main.c index 0aa9beefd1c7..43ee20eb03d1 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hns3vf/hclgevf_main.c +++ b/drivers/net/ethernet/hisilicon/hns3/hns3vf/hclgevf_main.c @@ -412,7 +412,8 @@ static int hclgevf_set_handle_info(struct hclgevf_dev *hdev) nic->ae_algo = &ae_algovf; nic->pdev = hdev->pdev; - nic->numa_node_mask = hdev->numa_node_mask; + bitmap_copy(nic->numa_node_mask.bits, hdev->numa_node_mask.bits, + MAX_NUMNODES); nic->flags |= HNAE3_SUPPORT_VF; nic->kinfo.io_base = hdev->hw.hw.io_base; @@ -2082,8 +2083,8 @@ static int hclgevf_init_roce_base_info(struct hclgevf_dev *hdev) roce->pdev = nic->pdev; roce->ae_algo = nic->ae_algo; - roce->numa_node_mask = nic->numa_node_mask; - + bitmap_copy(roce->numa_node_mask.bits, nic->numa_node_mask.bits, + MAX_NUMNODES); return 0; } @@ -2180,8 +2181,7 @@ static void hclgevf_set_timer_task(struct hnae3_handle *handle, bool enable) } else { set_bit(HCLGEVF_STATE_DOWN, &hdev->state); - /* flush memory to make sure DOWN is seen by service task */ - smp_mb__before_atomic(); + smp_mb__after_atomic(); /* flush memory to make sure DOWN is seen by service task */ hclgevf_flush_link_update(hdev); } } @@ -2845,10 +2845,6 @@ static int hclgevf_init_hdev(struct hclgevf_dev *hdev) if (ret) return ret; - ret = hclgevf_devlink_init(hdev); - if (ret) - goto err_devlink_init; - ret = hclge_comm_cmd_queue_init(hdev->pdev, &hdev->hw.hw); if (ret) goto err_cmd_queue_init; @@ -2941,6 +2937,10 @@ static int hclgevf_init_hdev(struct hclgevf_dev *hdev) hclgevf_init_rxd_adv_layout(hdev); + ret = hclgevf_devlink_init(hdev); + if (ret) + goto err_config; + set_bit(HCLGEVF_STATE_SERVICE_INITED, &hdev->state); hdev->last_reset_time = jiffies; @@ -2960,8 +2960,6 @@ err_misc_irq_init: err_cmd_init: hclge_comm_cmd_uninit(hdev->ae_dev, &hdev->hw.hw); err_cmd_queue_init: - hclgevf_devlink_uninit(hdev); -err_devlink_init: hclgevf_pci_uninit(hdev); clear_bit(HCLGEVF_STATE_IRQ_INITED, &hdev->state); return ret; diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3vf/hclgevf_main.h b/drivers/net/ethernet/hisilicon/hns3/hns3vf/hclgevf_main.h index a73f2bf3a56a..cccef3228461 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hns3vf/hclgevf_main.h +++ b/drivers/net/ethernet/hisilicon/hns3/hns3vf/hclgevf_main.h @@ -236,7 +236,7 @@ struct hclgevf_dev { u16 rss_size_max; /* HW defined max RSS task queue */ u16 num_alloc_vport; /* num vports this driver supports */ - u32 numa_node_mask; + nodemask_t numa_node_mask; u16 rx_buf_len; u16 num_tx_desc; /* desc num of per tx queue */ u16 num_rx_desc; /* desc num of per rx queue */ diff --git a/drivers/net/ethernet/micrel/ks8851_common.c b/drivers/net/ethernet/micrel/ks8851_common.c index d4cdf3d4f552..502518cdb461 100644 --- a/drivers/net/ethernet/micrel/ks8851_common.c +++ b/drivers/net/ethernet/micrel/ks8851_common.c @@ -234,12 +234,13 @@ static void ks8851_dbg_dumpkkt(struct ks8851_net *ks, u8 *rxpkt) /** * ks8851_rx_pkts - receive packets from the host * @ks: The device information. + * @rxq: Queue of packets received in this function. * * This is called from the IRQ work queue when the system detects that there * are packets in the receive queue. Find out how many packets there are and * read them from the FIFO. */ -static void ks8851_rx_pkts(struct ks8851_net *ks) +static void ks8851_rx_pkts(struct ks8851_net *ks, struct sk_buff_head *rxq) { struct sk_buff *skb; unsigned rxfc; @@ -299,7 +300,7 @@ static void ks8851_rx_pkts(struct ks8851_net *ks) ks8851_dbg_dumpkkt(ks, rxpkt); skb->protocol = eth_type_trans(skb, ks->netdev); - __netif_rx(skb); + __skb_queue_tail(rxq, skb); ks->netdev->stats.rx_packets++; ks->netdev->stats.rx_bytes += rxlen; @@ -326,11 +327,11 @@ static void ks8851_rx_pkts(struct ks8851_net *ks) static irqreturn_t ks8851_irq(int irq, void *_ks) { struct ks8851_net *ks = _ks; + struct sk_buff_head rxq; unsigned handled = 0; unsigned long flags; unsigned int status; - - local_bh_disable(); + struct sk_buff *skb; ks8851_lock(ks, &flags); @@ -384,7 +385,8 @@ static irqreturn_t ks8851_irq(int irq, void *_ks) * from the device so do not bother masking just the RX * from the device. */ - ks8851_rx_pkts(ks); + __skb_queue_head_init(&rxq); + ks8851_rx_pkts(ks, &rxq); } /* if something stopped the rx process, probably due to wanting @@ -408,7 +410,9 @@ static irqreturn_t ks8851_irq(int irq, void *_ks) if (status & IRQ_LCI) mii_check_link(&ks->mii); - local_bh_enable(); + if (status & IRQ_RXI) + while ((skb = __skb_dequeue(&rxq))) + netif_rx(skb); return IRQ_HANDLED; } diff --git a/drivers/net/phy/marvell-88q2xxx.c b/drivers/net/phy/marvell-88q2xxx.c index 6b4bd9883304..c812f16eaa3a 100644 --- a/drivers/net/phy/marvell-88q2xxx.c +++ b/drivers/net/phy/marvell-88q2xxx.c @@ -12,6 +12,8 @@ #include <linux/hwmon.h> #define PHY_ID_88Q2220_REVB0 (MARVELL_PHY_ID_88Q2220 | 0x1) +#define PHY_ID_88Q2220_REVB1 (MARVELL_PHY_ID_88Q2220 | 0x2) +#define PHY_ID_88Q2220_REVB2 (MARVELL_PHY_ID_88Q2220 | 0x3) #define MDIO_MMD_AN_MV_STAT 32769 #define MDIO_MMD_AN_MV_STAT_ANEG 0x0100 @@ -129,6 +131,49 @@ static const struct mmd_val mv88q222x_revb0_init_seq1[] = { { MDIO_MMD_PCS, 0xfe05, 0x755c }, }; +static const struct mmd_val mv88q222x_revb1_init_seq0[] = { + { MDIO_MMD_PCS, 0xffe4, 0x0007 }, + { MDIO_MMD_AN, MDIO_AN_T1_CTRL, 0x0 }, + { MDIO_MMD_PCS, 0xffe3, 0x7000 }, + { MDIO_MMD_PMAPMD, MDIO_CTRL1, 0x0840 }, +}; + +static const struct mmd_val mv88q222x_revb2_init_seq0[] = { + { MDIO_MMD_PCS, 0xffe4, 0x0007 }, + { MDIO_MMD_AN, MDIO_AN_T1_CTRL, 0x0 }, + { MDIO_MMD_PMAPMD, MDIO_CTRL1, 0x0840 }, +}; + +static const struct mmd_val mv88q222x_revb1_revb2_init_seq1[] = { + { MDIO_MMD_PCS, 0xfe07, 0x125a }, + { MDIO_MMD_PCS, 0xfe09, 0x1288 }, + { MDIO_MMD_PCS, 0xfe08, 0x2588 }, + { MDIO_MMD_PCS, 0xfe72, 0x042c }, + { MDIO_MMD_PCS, 0xffe4, 0x0071 }, + { MDIO_MMD_PCS, 0xffe4, 0x0001 }, + { MDIO_MMD_PCS, 0xfe1b, 0x0048 }, + { MDIO_MMD_PMAPMD, 0x0000, 0x0000 }, + { MDIO_MMD_PCS, 0x0000, 0x0000 }, + { MDIO_MMD_PCS, 0xffdb, 0xfc10 }, + { MDIO_MMD_PCS, 0xfe1b, 0x58 }, + { MDIO_MMD_PCS, 0xfcad, 0x030c }, + { MDIO_MMD_PCS, 0x8032, 0x6001 }, + { MDIO_MMD_PCS, 0xfdff, 0x05a5 }, + { MDIO_MMD_PCS, 0xfdec, 0xdbaf }, + { MDIO_MMD_PCS, 0xfcab, 0x1054 }, + { MDIO_MMD_PCS, 0xfcac, 0x1483 }, + { MDIO_MMD_PCS, 0x8033, 0xc801 }, + { MDIO_MMD_AN, 0x8032, 0x2020 }, + { MDIO_MMD_AN, 0x8031, 0xa28 }, + { MDIO_MMD_AN, 0x8031, 0xc28 }, + { MDIO_MMD_PCS, 0xfbba, 0x0cb2 }, + { MDIO_MMD_PCS, 0xfbbb, 0x0c4a }, + { MDIO_MMD_PCS, 0xfe5f, 0xe8 }, + { MDIO_MMD_PCS, 0xfe05, 0x755c }, + { MDIO_MMD_PCS, 0xfa20, 0x002a }, + { MDIO_MMD_PCS, 0xfe11, 0x1105 }, +}; + static int mv88q2xxx_soft_reset(struct phy_device *phydev) { int ret; @@ -687,31 +732,72 @@ static int mv88q222x_soft_reset(struct phy_device *phydev) return 0; } -static int mv88q222x_revb0_config_init(struct phy_device *phydev) +static int mv88q222x_write_mmd_vals(struct phy_device *phydev, + const struct mmd_val *vals, size_t len) { - int ret, i; + int ret; - for (i = 0; i < ARRAY_SIZE(mv88q222x_revb0_init_seq0); i++) { - ret = phy_write_mmd(phydev, mv88q222x_revb0_init_seq0[i].devad, - mv88q222x_revb0_init_seq0[i].regnum, - mv88q222x_revb0_init_seq0[i].val); + for (; len; vals++, len--) { + ret = phy_write_mmd(phydev, vals->devad, vals->regnum, + vals->val); if (ret < 0) return ret; } + return 0; +} + +static int mv88q222x_revb0_config_init(struct phy_device *phydev) +{ + int ret; + + ret = mv88q222x_write_mmd_vals(phydev, mv88q222x_revb0_init_seq0, + ARRAY_SIZE(mv88q222x_revb0_init_seq0)); + if (ret < 0) + return ret; + usleep_range(5000, 10000); - for (i = 0; i < ARRAY_SIZE(mv88q222x_revb0_init_seq1); i++) { - ret = phy_write_mmd(phydev, mv88q222x_revb0_init_seq1[i].devad, - mv88q222x_revb0_init_seq1[i].regnum, - mv88q222x_revb0_init_seq1[i].val); - if (ret < 0) - return ret; - } + ret = mv88q222x_write_mmd_vals(phydev, mv88q222x_revb0_init_seq1, + ARRAY_SIZE(mv88q222x_revb0_init_seq1)); + if (ret < 0) + return ret; + + return mv88q2xxx_config_init(phydev); +} + +static int mv88q222x_revb1_revb2_config_init(struct phy_device *phydev) +{ + bool is_rev_b1 = phydev->c45_ids.device_ids[MDIO_MMD_PMAPMD] == PHY_ID_88Q2220_REVB1; + int ret; + + if (is_rev_b1) + ret = mv88q222x_write_mmd_vals(phydev, mv88q222x_revb1_init_seq0, + ARRAY_SIZE(mv88q222x_revb1_init_seq0)); + else + ret = mv88q222x_write_mmd_vals(phydev, mv88q222x_revb2_init_seq0, + ARRAY_SIZE(mv88q222x_revb2_init_seq0)); + if (ret < 0) + return ret; + + usleep_range(3000, 5000); + + ret = mv88q222x_write_mmd_vals(phydev, mv88q222x_revb1_revb2_init_seq1, + ARRAY_SIZE(mv88q222x_revb1_revb2_init_seq1)); + if (ret < 0) + return ret; return mv88q2xxx_config_init(phydev); } +static int mv88q222x_config_init(struct phy_device *phydev) +{ + if (phydev->c45_ids.device_ids[MDIO_MMD_PMAPMD] == PHY_ID_88Q2220_REVB0) + return mv88q222x_revb0_config_init(phydev); + else + return mv88q222x_revb1_revb2_config_init(phydev); +} + static int mv88q222x_cable_test_start(struct phy_device *phydev) { int ret; @@ -810,14 +896,15 @@ static struct phy_driver mv88q2xxx_driver[] = { .get_sqi_max = mv88q2xxx_get_sqi_max, }, { - PHY_ID_MATCH_EXACT(PHY_ID_88Q2220_REVB0), + .phy_id = MARVELL_PHY_ID_88Q2220, + .phy_id_mask = MARVELL_PHY_ID_MASK, .name = "mv88q2220", .flags = PHY_POLL_CABLE_TEST, .probe = mv88q2xxx_probe, .get_features = mv88q2xxx_get_features, .config_aneg = mv88q2xxx_config_aneg, .aneg_done = genphy_c45_aneg_done, - .config_init = mv88q222x_revb0_config_init, + .config_init = mv88q222x_config_init, .read_status = mv88q2xxx_read_status, .soft_reset = mv88q222x_soft_reset, .config_intr = mv88q2xxx_config_intr, @@ -836,7 +923,7 @@ module_phy_driver(mv88q2xxx_driver); static struct mdio_device_id __maybe_unused mv88q2xxx_tbl[] = { { MARVELL_PHY_ID_88Q2110, MARVELL_PHY_ID_MASK }, - { PHY_ID_MATCH_EXACT(PHY_ID_88Q2220_REVB0), }, + { MARVELL_PHY_ID_88Q2220, MARVELL_PHY_ID_MASK }, { /*sentinel*/ } }; MODULE_DEVICE_TABLE(mdio, mv88q2xxx_tbl); diff --git a/drivers/net/tap.c b/drivers/net/tap.c index 9f0495e8df4d..bfdd3875fe86 100644 --- a/drivers/net/tap.c +++ b/drivers/net/tap.c @@ -754,7 +754,7 @@ static ssize_t tap_get_user(struct tap_queue *q, void *msg_control, skb_zcopy_init(skb, msg_control); } else if (msg_control) { struct ubuf_info *uarg = msg_control; - uarg->callback(NULL, uarg, false); + uarg->ops->complete(NULL, uarg, false); } dev_queue_xmit(skb); diff --git a/drivers/net/tun.c b/drivers/net/tun.c index 92da8c03d960..9254bca2813d 100644 --- a/drivers/net/tun.c +++ b/drivers/net/tun.c @@ -1906,7 +1906,7 @@ static ssize_t tun_get_user(struct tun_struct *tun, struct tun_file *tfile, skb_zcopy_init(skb, msg_control); } else if (msg_control) { struct ubuf_info *uarg = msg_control; - uarg->callback(NULL, uarg, false); + uarg->ops->complete(NULL, uarg, false); } skb_reset_network_header(skb); diff --git a/drivers/net/xen-netback/common.h b/drivers/net/xen-netback/common.h index 1fcbd83f7ff2..17421da139f2 100644 --- a/drivers/net/xen-netback/common.h +++ b/drivers/net/xen-netback/common.h @@ -390,9 +390,8 @@ bool xenvif_rx_queue_tail(struct xenvif_queue *queue, struct sk_buff *skb); void xenvif_carrier_on(struct xenvif *vif); -/* Callback from stack when TX packet can be released */ -void xenvif_zerocopy_callback(struct sk_buff *skb, struct ubuf_info *ubuf, - bool zerocopy_success); +/* Callbacks from stack when TX packet can be released */ +extern const struct ubuf_info_ops xenvif_ubuf_ops; static inline pending_ring_idx_t nr_pending_reqs(struct xenvif_queue *queue) { diff --git a/drivers/net/xen-netback/interface.c b/drivers/net/xen-netback/interface.c index 7cff90aa8d24..65db5f14465f 100644 --- a/drivers/net/xen-netback/interface.c +++ b/drivers/net/xen-netback/interface.c @@ -593,7 +593,7 @@ int xenvif_init_queue(struct xenvif_queue *queue) for (i = 0; i < MAX_PENDING_REQS; i++) { queue->pending_tx_info[i].callback_struct = (struct ubuf_info_msgzc) - { { .callback = xenvif_zerocopy_callback }, + { { .ops = &xenvif_ubuf_ops }, { { .ctx = NULL, .desc = i } } }; queue->grant_tx_handle[i] = NETBACK_INVALID_HANDLE; diff --git a/drivers/net/xen-netback/netback.c b/drivers/net/xen-netback/netback.c index ef76850d9bcd..bab7e43ea05b 100644 --- a/drivers/net/xen-netback/netback.c +++ b/drivers/net/xen-netback/netback.c @@ -1156,7 +1156,7 @@ static int xenvif_handle_frag_list(struct xenvif_queue *queue, struct sk_buff *s uarg = skb_shinfo(skb)->destructor_arg; /* increase inflight counter to offset decrement in callback */ atomic_inc(&queue->inflight_packets); - uarg->callback(NULL, uarg, true); + uarg->ops->complete(NULL, uarg, true); skb_shinfo(skb)->destructor_arg = NULL; /* Fill the skb with the new (local) frags. */ @@ -1278,8 +1278,9 @@ static int xenvif_tx_submit(struct xenvif_queue *queue) return work_done; } -void xenvif_zerocopy_callback(struct sk_buff *skb, struct ubuf_info *ubuf_base, - bool zerocopy_success) +static void xenvif_zerocopy_callback(struct sk_buff *skb, + struct ubuf_info *ubuf_base, + bool zerocopy_success) { unsigned long flags; pending_ring_idx_t index; @@ -1312,6 +1313,10 @@ void xenvif_zerocopy_callback(struct sk_buff *skb, struct ubuf_info *ubuf_base, xenvif_skb_zerocopy_complete(queue); } +const struct ubuf_info_ops xenvif_ubuf_ops = { + .complete = xenvif_zerocopy_callback, +}; + static inline void xenvif_tx_dealloc_action(struct xenvif_queue *queue) { struct gnttab_unmap_grant_ref *gop; diff --git a/drivers/nvme/host/core.c b/drivers/nvme/host/core.c index 095f59e7aa93..bf7615cb36ee 100644 --- a/drivers/nvme/host/core.c +++ b/drivers/nvme/host/core.c @@ -2132,7 +2132,7 @@ static int nvme_update_ns_info_block(struct nvme_ns *ns, blk_mq_unfreeze_queue(ns->disk->queue); if (blk_queue_is_zoned(ns->queue)) { - ret = blk_revalidate_disk_zones(ns->disk, NULL); + ret = blk_revalidate_disk_zones(ns->disk); if (ret && !nvme_first_scan(ns->disk)) goto out; } diff --git a/drivers/nvme/host/ioctl.c b/drivers/nvme/host/ioctl.c index 3dfd5ae99ae0..499a8bb7cac7 100644 --- a/drivers/nvme/host/ioctl.c +++ b/drivers/nvme/host/ioctl.c @@ -423,13 +423,20 @@ static enum rq_end_io_ret nvme_uring_cmd_end_io(struct request *req, pdu->result = le64_to_cpu(nvme_req(req)->result.u64); /* - * For iopoll, complete it directly. + * For iopoll, complete it directly. Note that using the uring_cmd + * helper for this is safe only because we check blk_rq_is_poll(). + * As that returns false if we're NOT on a polled queue, then it's + * safe to use the polled completion helper. + * * Otherwise, move the completion to task work. */ - if (blk_rq_is_poll(req)) - nvme_uring_task_cb(ioucmd, IO_URING_F_UNLOCKED); - else + if (blk_rq_is_poll(req)) { + if (pdu->bio) + blk_rq_unmap_user(pdu->bio); + io_uring_cmd_iopoll_done(ioucmd, pdu->result, pdu->status); + } else { io_uring_cmd_do_in_task_lazy(ioucmd, nvme_uring_task_cb); + } return RQ_END_IO_FREE; } diff --git a/drivers/nvme/host/nvme.h b/drivers/nvme/host/nvme.h index b564c5f1450c..05532c281177 100644 --- a/drivers/nvme/host/nvme.h +++ b/drivers/nvme/host/nvme.h @@ -162,6 +162,11 @@ enum nvme_quirks { * Disables simple suspend/resume path. */ NVME_QUIRK_FORCE_NO_SIMPLE_SUSPEND = (1 << 20), + + /* + * MSI (but not MSI-X) interrupts are broken and never fire. + */ + NVME_QUIRK_BROKEN_MSI = (1 << 21), }; /* diff --git a/drivers/nvme/host/pci.c b/drivers/nvme/host/pci.c index e393f6947ce4..710043086dff 100644 --- a/drivers/nvme/host/pci.c +++ b/drivers/nvme/host/pci.c @@ -2224,6 +2224,7 @@ static int nvme_setup_irqs(struct nvme_dev *dev, unsigned int nr_io_queues) .priv = dev, }; unsigned int irq_queues, poll_queues; + unsigned int flags = PCI_IRQ_ALL_TYPES | PCI_IRQ_AFFINITY; /* * Poll queues don't need interrupts, but we need at least one I/O queue @@ -2247,8 +2248,10 @@ static int nvme_setup_irqs(struct nvme_dev *dev, unsigned int nr_io_queues) irq_queues = 1; if (!(dev->ctrl.quirks & NVME_QUIRK_SINGLE_VECTOR)) irq_queues += (nr_io_queues - poll_queues); - return pci_alloc_irq_vectors_affinity(pdev, 1, irq_queues, - PCI_IRQ_ALL_TYPES | PCI_IRQ_AFFINITY, &affd); + if (dev->ctrl.quirks & NVME_QUIRK_BROKEN_MSI) + flags &= ~PCI_IRQ_MSI; + return pci_alloc_irq_vectors_affinity(pdev, 1, irq_queues, flags, + &affd); } static unsigned int nvme_max_io_queues(struct nvme_dev *dev) @@ -2477,6 +2480,7 @@ static int nvme_pci_enable(struct nvme_dev *dev) { int result = -ENOMEM; struct pci_dev *pdev = to_pci_dev(dev->dev); + unsigned int flags = PCI_IRQ_ALL_TYPES; if (pci_enable_device_mem(pdev)) return result; @@ -2493,7 +2497,9 @@ static int nvme_pci_enable(struct nvme_dev *dev) * interrupts. Pre-enable a single MSIX or MSI vec for setup. We'll * adjust this later. */ - result = pci_alloc_irq_vectors(pdev, 1, 1, PCI_IRQ_ALL_TYPES); + if (dev->ctrl.quirks & NVME_QUIRK_BROKEN_MSI) + flags &= ~PCI_IRQ_MSI; + result = pci_alloc_irq_vectors(pdev, 1, 1, flags); if (result < 0) goto disable; @@ -3390,6 +3396,8 @@ static const struct pci_device_id nvme_id_table[] = { .driver_data = NVME_QUIRK_DELAY_BEFORE_CHK_RDY | NVME_QUIRK_DISABLE_WRITE_ZEROES| NVME_QUIRK_IGNORE_DEV_SUBNQN, }, + { PCI_DEVICE(0x15b7, 0x5008), /* Sandisk SN530 */ + .driver_data = NVME_QUIRK_BROKEN_MSI }, { PCI_DEVICE(0x1987, 0x5012), /* Phison E12 */ .driver_data = NVME_QUIRK_BOGUS_NID, }, { PCI_DEVICE(0x1987, 0x5016), /* Phison E16 */ diff --git a/drivers/nvme/target/auth.c b/drivers/nvme/target/auth.c index fb518b00f71f..4f08362aee8b 100644 --- a/drivers/nvme/target/auth.c +++ b/drivers/nvme/target/auth.c @@ -480,7 +480,7 @@ out_free_response: nvme_auth_free_key(transformed_key); out_free_tfm: crypto_free_shash(shash_tfm); - return 0; + return ret; } int nvmet_auth_ctrl_exponential(struct nvmet_req *req, diff --git a/drivers/nvme/target/configfs.c b/drivers/nvme/target/configfs.c index 89a001101a7f..7fda69395c1e 100644 --- a/drivers/nvme/target/configfs.c +++ b/drivers/nvme/target/configfs.c @@ -757,10 +757,9 @@ static struct configfs_attribute *nvmet_ns_attrs[] = { bool nvmet_subsys_nsid_exists(struct nvmet_subsys *subsys, u32 nsid) { struct config_item *ns_item; - char name[4] = {}; + char name[12]; - if (sprintf(name, "%u", nsid) <= 0) - return false; + snprintf(name, sizeof(name), "%u", nsid); mutex_lock(&subsys->namespaces_group.cg_subsys->su_mutex); ns_item = config_group_find_item(&subsys->namespaces_group, name); mutex_unlock(&subsys->namespaces_group.cg_subsys->su_mutex); diff --git a/drivers/nvme/target/core.c b/drivers/nvme/target/core.c index e06013c5dace..2fde22323622 100644 --- a/drivers/nvme/target/core.c +++ b/drivers/nvme/target/core.c @@ -1686,7 +1686,8 @@ static int __init nvmet_init(void) if (!buffered_io_wq) goto out_free_zbd_work_queue; - nvmet_wq = alloc_workqueue("nvmet-wq", WQ_MEM_RECLAIM, 0); + nvmet_wq = alloc_workqueue("nvmet-wq", + WQ_MEM_RECLAIM | WQ_UNBOUND, 0); if (!nvmet_wq) goto out_free_buffered_work_queue; diff --git a/drivers/nvme/target/rdma.c b/drivers/nvme/target/rdma.c index 5b8c63e74639..6e1b4140cde0 100644 --- a/drivers/nvme/target/rdma.c +++ b/drivers/nvme/target/rdma.c @@ -474,12 +474,8 @@ nvmet_rdma_alloc_rsps(struct nvmet_rdma_queue *queue) return 0; out_free: - while (--i >= 0) { - struct nvmet_rdma_rsp *rsp = &queue->rsps[i]; - - list_del(&rsp->free_list); - nvmet_rdma_free_rsp(ndev, rsp); - } + while (--i >= 0) + nvmet_rdma_free_rsp(ndev, &queue->rsps[i]); kfree(queue->rsps); out: return ret; @@ -490,12 +486,8 @@ static void nvmet_rdma_free_rsps(struct nvmet_rdma_queue *queue) struct nvmet_rdma_device *ndev = queue->dev; int i, nr_rsps = queue->recv_queue_size * 2; - for (i = 0; i < nr_rsps; i++) { - struct nvmet_rdma_rsp *rsp = &queue->rsps[i]; - - list_del(&rsp->free_list); - nvmet_rdma_free_rsp(ndev, rsp); - } + for (i = 0; i < nr_rsps; i++) + nvmet_rdma_free_rsp(ndev, &queue->rsps[i]); kfree(queue->rsps); } diff --git a/drivers/nvme/target/zns.c b/drivers/nvme/target/zns.c index 3148d9f1bde6..0021d06041c1 100644 --- a/drivers/nvme/target/zns.c +++ b/drivers/nvme/target/zns.c @@ -52,14 +52,10 @@ bool nvmet_bdev_zns_enable(struct nvmet_ns *ns) if (get_capacity(bd_disk) & (bdev_zone_sectors(ns->bdev) - 1)) return false; /* - * ZNS does not define a conventional zone type. If the underlying - * device has a bitmap set indicating the existence of conventional - * zones, reject the device. Otherwise, use report zones to detect if - * the device has conventional zones. + * ZNS does not define a conventional zone type. Use report zones + * to detect if the device has conventional zones and reject it if + * it does. */ - if (ns->bdev->bd_disk->conv_zones_bitmap) - return false; - ret = blkdev_report_zones(ns->bdev, 0, bdev_nr_zones(ns->bdev), validate_conv_zones_cb, NULL); if (ret < 0) diff --git a/drivers/of/property.c b/drivers/of/property.c index a6358ee99b74..769ae2cf5f3c 100644 --- a/drivers/of/property.c +++ b/drivers/of/property.c @@ -1252,6 +1252,7 @@ DEFINE_SIMPLE_PROP(backlight, "backlight", NULL) DEFINE_SIMPLE_PROP(panel, "panel", NULL) DEFINE_SIMPLE_PROP(msi_parent, "msi-parent", "#msi-cells") DEFINE_SIMPLE_PROP(post_init_providers, "post-init-providers", NULL) +DEFINE_SIMPLE_PROP(access_controllers, "access-controllers", "#access-controller-cells") DEFINE_SUFFIX_PROP(regulators, "-supply", NULL) DEFINE_SUFFIX_PROP(gpio, "-gpio", "#gpio-cells") @@ -1359,6 +1360,7 @@ static const struct supplier_bindings of_supplier_bindings[] = { { .parse_prop = parse_msi_parent, }, { .parse_prop = parse_gpio_compat, }, { .parse_prop = parse_interrupts, }, + { .parse_prop = parse_access_controllers, }, { .parse_prop = parse_regulators, }, { .parse_prop = parse_gpio, }, { .parse_prop = parse_gpios, }, diff --git a/drivers/pci/pcie/aspm.c b/drivers/pci/pcie/aspm.c index 2428d278e015..47761c7ef267 100644 --- a/drivers/pci/pcie/aspm.c +++ b/drivers/pci/pcie/aspm.c @@ -177,8 +177,8 @@ void pci_restore_aspm_l1ss_state(struct pci_dev *pdev) /* Restore L0s/L1 if they were enabled */ if (FIELD_GET(PCI_EXP_LNKCTL_ASPMC, clnkctl) || FIELD_GET(PCI_EXP_LNKCTL_ASPMC, plnkctl)) { - pcie_capability_write_word(parent, PCI_EXP_LNKCTL, clnkctl); - pcie_capability_write_word(pdev, PCI_EXP_LNKCTL, plnkctl); + pcie_capability_write_word(parent, PCI_EXP_LNKCTL, plnkctl); + pcie_capability_write_word(pdev, PCI_EXP_LNKCTL, clnkctl); } } diff --git a/drivers/perf/alibaba_uncore_drw_pmu.c b/drivers/perf/alibaba_uncore_drw_pmu.c index a9277dcf90ce..89dd38343f93 100644 --- a/drivers/perf/alibaba_uncore_drw_pmu.c +++ b/drivers/perf/alibaba_uncore_drw_pmu.c @@ -709,6 +709,7 @@ static int ali_drw_pmu_probe(struct platform_device *pdev) drw_pmu->pmu = (struct pmu) { .module = THIS_MODULE, + .parent = &pdev->dev, .task_ctx_nr = perf_invalid_context, .event_init = ali_drw_pmu_event_init, .add = ali_drw_pmu_add, @@ -746,18 +747,14 @@ static int ali_drw_pmu_offline_cpu(unsigned int cpu, struct hlist_node *node) struct ali_drw_pmu_irq *irq; struct ali_drw_pmu *drw_pmu; unsigned int target; - int ret; - cpumask_t node_online_cpus; irq = hlist_entry_safe(node, struct ali_drw_pmu_irq, node); if (cpu != irq->cpu) return 0; - ret = cpumask_and(&node_online_cpus, - cpumask_of_node(cpu_to_node(cpu)), cpu_online_mask); - if (ret) - target = cpumask_any_but(&node_online_cpus, cpu); - else + target = cpumask_any_and_but(cpumask_of_node(cpu_to_node(cpu)), + cpu_online_mask, cpu); + if (target >= nr_cpu_ids) target = cpumask_any_but(cpu_online_mask, cpu); if (target >= nr_cpu_ids) diff --git a/drivers/perf/amlogic/meson_ddr_pmu_core.c b/drivers/perf/amlogic/meson_ddr_pmu_core.c index bbc7285fd934..07446d784a1a 100644 --- a/drivers/perf/amlogic/meson_ddr_pmu_core.c +++ b/drivers/perf/amlogic/meson_ddr_pmu_core.c @@ -492,6 +492,7 @@ int meson_ddr_pmu_create(struct platform_device *pdev) *pmu = (struct ddr_pmu) { .pmu = { .module = THIS_MODULE, + .parent = &pdev->dev, .capabilities = PERF_PMU_CAP_NO_EXCLUDE, .task_ctx_nr = perf_invalid_context, .attr_groups = attr_groups, diff --git a/drivers/perf/arm-cci.c b/drivers/perf/arm-cci.c index 6be03f81ae5d..a7fd80677919 100644 --- a/drivers/perf/arm-cci.c +++ b/drivers/perf/arm-cci.c @@ -1409,6 +1409,7 @@ static int cci_pmu_init(struct cci_pmu *cci_pmu, struct platform_device *pdev) cci_pmu->pmu = (struct pmu) { .module = THIS_MODULE, + .parent = &pdev->dev, .name = cci_pmu->model->name, .task_ctx_nr = perf_invalid_context, .pmu_enable = cci_pmu_enable, diff --git a/drivers/perf/arm-ccn.c b/drivers/perf/arm-ccn.c index 641471bd5eff..f4495ff6525f 100644 --- a/drivers/perf/arm-ccn.c +++ b/drivers/perf/arm-ccn.c @@ -1265,6 +1265,7 @@ static int arm_ccn_pmu_init(struct arm_ccn *ccn) /* Perf driver registration */ ccn->dt.pmu = (struct pmu) { .module = THIS_MODULE, + .parent = ccn->dev, .attr_groups = arm_ccn_pmu_attr_groups, .task_ctx_nr = perf_invalid_context, .event_init = arm_ccn_pmu_event_init, diff --git a/drivers/perf/arm-cmn.c b/drivers/perf/arm-cmn.c index 7ef9c7e4836b..e26ad1d3ed0b 100644 --- a/drivers/perf/arm-cmn.c +++ b/drivers/perf/arm-cmn.c @@ -1950,20 +1950,20 @@ static int arm_cmn_pmu_offline_cpu(unsigned int cpu, struct hlist_node *cpuhp_no struct arm_cmn *cmn; unsigned int target; int node; - cpumask_t mask; cmn = hlist_entry_safe(cpuhp_node, struct arm_cmn, cpuhp_node); if (cpu != cmn->cpu) return 0; node = dev_to_node(cmn->dev); - if (cpumask_and(&mask, cpumask_of_node(node), cpu_online_mask) && - cpumask_andnot(&mask, &mask, cpumask_of(cpu))) - target = cpumask_any(&mask); - else + + target = cpumask_any_and_but(cpumask_of_node(node), cpu_online_mask, cpu); + if (target >= nr_cpu_ids) target = cpumask_any_but(cpu_online_mask, cpu); + if (target < nr_cpu_ids) arm_cmn_migrate(cmn, target); + return 0; } @@ -2482,6 +2482,7 @@ static int arm_cmn_probe(struct platform_device *pdev) cmn->cpu = cpumask_local_spread(0, dev_to_node(cmn->dev)); cmn->pmu = (struct pmu) { .module = THIS_MODULE, + .parent = cmn->dev, .attr_groups = arm_cmn_attr_groups, .capabilities = PERF_PMU_CAP_NO_EXCLUDE, .task_ctx_nr = perf_invalid_context, diff --git a/drivers/perf/arm_cspmu/arm_cspmu.c b/drivers/perf/arm_cspmu/arm_cspmu.c index b9a252272f1e..ba0cf2f466ef 100644 --- a/drivers/perf/arm_cspmu/arm_cspmu.c +++ b/drivers/perf/arm_cspmu/arm_cspmu.c @@ -1206,6 +1206,7 @@ static int arm_cspmu_register_pmu(struct arm_cspmu *cspmu) cspmu->pmu = (struct pmu){ .task_ctx_nr = perf_invalid_context, .module = cspmu->impl.module, + .parent = cspmu->dev, .pmu_enable = arm_cspmu_enable, .pmu_disable = arm_cspmu_disable, .event_init = arm_cspmu_event_init, @@ -1322,8 +1323,7 @@ static int arm_cspmu_cpu_online(unsigned int cpu, struct hlist_node *node) static int arm_cspmu_cpu_teardown(unsigned int cpu, struct hlist_node *node) { - int dst; - struct cpumask online_supported; + unsigned int dst; struct arm_cspmu *cspmu = hlist_entry_safe(node, struct arm_cspmu, cpuhp_node); @@ -1333,9 +1333,8 @@ static int arm_cspmu_cpu_teardown(unsigned int cpu, struct hlist_node *node) return 0; /* Choose a new CPU to migrate ownership of the PMU to */ - cpumask_and(&online_supported, &cspmu->associated_cpus, - cpu_online_mask); - dst = cpumask_any_but(&online_supported, cpu); + dst = cpumask_any_and_but(&cspmu->associated_cpus, + cpu_online_mask, cpu); if (dst >= nr_cpu_ids) return 0; diff --git a/drivers/perf/arm_dmc620_pmu.c b/drivers/perf/arm_dmc620_pmu.c index 8a81be2dd5ec..2ec96e204c40 100644 --- a/drivers/perf/arm_dmc620_pmu.c +++ b/drivers/perf/arm_dmc620_pmu.c @@ -673,6 +673,7 @@ static int dmc620_pmu_device_probe(struct platform_device *pdev) dmc620_pmu->pmu = (struct pmu) { .module = THIS_MODULE, + .parent = &pdev->dev, .capabilities = PERF_PMU_CAP_NO_EXCLUDE, .task_ctx_nr = perf_invalid_context, .event_init = dmc620_pmu_event_init, diff --git a/drivers/perf/arm_dsu_pmu.c b/drivers/perf/arm_dsu_pmu.c index bae3ca37f846..92248a24a1aa 100644 --- a/drivers/perf/arm_dsu_pmu.c +++ b/drivers/perf/arm_dsu_pmu.c @@ -230,15 +230,6 @@ static const struct attribute_group *dsu_pmu_attr_groups[] = { NULL, }; -static int dsu_pmu_get_online_cpu_any_but(struct dsu_pmu *dsu_pmu, int cpu) -{ - struct cpumask online_supported; - - cpumask_and(&online_supported, - &dsu_pmu->associated_cpus, cpu_online_mask); - return cpumask_any_but(&online_supported, cpu); -} - static inline bool dsu_pmu_counter_valid(struct dsu_pmu *dsu_pmu, u32 idx) { return (idx < dsu_pmu->num_counters) || @@ -751,6 +742,7 @@ static int dsu_pmu_device_probe(struct platform_device *pdev) dsu_pmu->pmu = (struct pmu) { .task_ctx_nr = perf_invalid_context, + .parent = &pdev->dev, .module = THIS_MODULE, .pmu_enable = dsu_pmu_enable, .pmu_disable = dsu_pmu_disable, @@ -827,14 +819,16 @@ static int dsu_pmu_cpu_online(unsigned int cpu, struct hlist_node *node) static int dsu_pmu_cpu_teardown(unsigned int cpu, struct hlist_node *node) { - int dst; - struct dsu_pmu *dsu_pmu = hlist_entry_safe(node, struct dsu_pmu, - cpuhp_node); + struct dsu_pmu *dsu_pmu; + unsigned int dst; + + dsu_pmu = hlist_entry_safe(node, struct dsu_pmu, cpuhp_node); if (!cpumask_test_and_clear_cpu(cpu, &dsu_pmu->active_cpu)) return 0; - dst = dsu_pmu_get_online_cpu_any_but(dsu_pmu, cpu); + dst = cpumask_any_and_but(&dsu_pmu->associated_cpus, + cpu_online_mask, cpu); /* If there are no active CPUs in the DSU, leave IRQ disabled */ if (dst >= nr_cpu_ids) return 0; diff --git a/drivers/perf/arm_pmu_platform.c b/drivers/perf/arm_pmu_platform.c index 3596db36cbff..4b1a9a92ea11 100644 --- a/drivers/perf/arm_pmu_platform.c +++ b/drivers/perf/arm_pmu_platform.c @@ -196,6 +196,7 @@ int arm_pmu_device_probe(struct platform_device *pdev, if (!pmu) return -ENOMEM; + pmu->pmu.parent = &pdev->dev; pmu->plat_device = pdev; ret = pmu_parse_irqs(pmu); diff --git a/drivers/perf/arm_smmuv3_pmu.c b/drivers/perf/arm_smmuv3_pmu.c index 719aa953a1c4..d5fa92ba8373 100644 --- a/drivers/perf/arm_smmuv3_pmu.c +++ b/drivers/perf/arm_smmuv3_pmu.c @@ -860,6 +860,7 @@ static int smmu_pmu_probe(struct platform_device *pdev) smmu_pmu->pmu = (struct pmu) { .module = THIS_MODULE, + .parent = &pdev->dev, .task_ctx_nr = perf_invalid_context, .pmu_enable = smmu_pmu_enable, .pmu_disable = smmu_pmu_disable, diff --git a/drivers/perf/arm_spe_pmu.c b/drivers/perf/arm_spe_pmu.c index 35f0de03416f..9100d82bfabc 100644 --- a/drivers/perf/arm_spe_pmu.c +++ b/drivers/perf/arm_spe_pmu.c @@ -932,6 +932,7 @@ static int arm_spe_pmu_perf_init(struct arm_spe_pmu *spe_pmu) spe_pmu->pmu = (struct pmu) { .module = THIS_MODULE, + .parent = &spe_pmu->pdev->dev, .capabilities = PERF_PMU_CAP_EXCLUSIVE | PERF_PMU_CAP_ITRACE, .attr_groups = arm_spe_pmu_attr_groups, /* diff --git a/drivers/perf/dwc_pcie_pmu.c b/drivers/perf/dwc_pcie_pmu.c index 957058ad0099..c5e328f23841 100644 --- a/drivers/perf/dwc_pcie_pmu.c +++ b/drivers/perf/dwc_pcie_pmu.c @@ -690,9 +690,8 @@ static int dwc_pcie_pmu_offline_cpu(unsigned int cpu, struct hlist_node *cpuhp_n { struct dwc_pcie_pmu *pcie_pmu; struct pci_dev *pdev; - int node; - cpumask_t mask; unsigned int target; + int node; pcie_pmu = hlist_entry_safe(cpuhp_node, struct dwc_pcie_pmu, cpuhp_node); /* Nothing to do if this CPU doesn't own the PMU */ @@ -702,10 +701,9 @@ static int dwc_pcie_pmu_offline_cpu(unsigned int cpu, struct hlist_node *cpuhp_n pcie_pmu->on_cpu = -1; pdev = pcie_pmu->pdev; node = dev_to_node(&pdev->dev); - if (cpumask_and(&mask, cpumask_of_node(node), cpu_online_mask) && - cpumask_andnot(&mask, &mask, cpumask_of(cpu))) - target = cpumask_any(&mask); - else + + target = cpumask_any_and_but(cpumask_of_node(node), cpu_online_mask, cpu); + if (target >= nr_cpu_ids) target = cpumask_any_but(cpu_online_mask, cpu); if (target >= nr_cpu_ids) { diff --git a/drivers/perf/fsl_imx8_ddr_perf.c b/drivers/perf/fsl_imx8_ddr_perf.c index 4e8fa5a48fcf..1bbdb29743c4 100644 --- a/drivers/perf/fsl_imx8_ddr_perf.c +++ b/drivers/perf/fsl_imx8_ddr_perf.c @@ -651,6 +651,7 @@ static int ddr_perf_init(struct ddr_pmu *pmu, void __iomem *base, *pmu = (struct ddr_pmu) { .pmu = (struct pmu) { .module = THIS_MODULE, + .parent = dev, .capabilities = PERF_PMU_CAP_NO_EXCLUDE, .task_ctx_nr = perf_invalid_context, .attr_groups = attr_groups, diff --git a/drivers/perf/hisilicon/hisi_pcie_pmu.c b/drivers/perf/hisilicon/hisi_pcie_pmu.c index 5d1f0e9fdb08..03c506aa3853 100644 --- a/drivers/perf/hisilicon/hisi_pcie_pmu.c +++ b/drivers/perf/hisilicon/hisi_pcie_pmu.c @@ -350,15 +350,27 @@ static bool hisi_pcie_pmu_validate_event_group(struct perf_event *event) return false; for (num = 0; num < counters; num++) { + /* + * If we find a related event, then it's a valid group + * since we don't need to allocate a new counter for it. + */ if (hisi_pcie_pmu_cmp_event(event_group[num], sibling)) break; } + /* + * Otherwise it's a new event but if there's no available counter, + * fail the check since we cannot schedule all the events in + * the group simultaneously. + */ + if (num == HISI_PCIE_MAX_COUNTERS) + return false; + if (num == counters) event_group[counters++] = sibling; } - return counters <= HISI_PCIE_MAX_COUNTERS; + return true; } static int hisi_pcie_pmu_event_init(struct perf_event *event) @@ -673,7 +685,6 @@ static int hisi_pcie_pmu_offline_cpu(unsigned int cpu, struct hlist_node *node) { struct hisi_pcie_pmu *pcie_pmu = hlist_entry_safe(node, struct hisi_pcie_pmu, node); unsigned int target; - cpumask_t mask; int numa_node; /* Nothing to do if this CPU doesn't own the PMU */ @@ -684,10 +695,10 @@ static int hisi_pcie_pmu_offline_cpu(unsigned int cpu, struct hlist_node *node) /* Choose a local CPU from all online cpus. */ numa_node = dev_to_node(&pcie_pmu->pdev->dev); - if (cpumask_and(&mask, cpumask_of_node(numa_node), cpu_online_mask) && - cpumask_andnot(&mask, &mask, cpumask_of(cpu))) - target = cpumask_any(&mask); - else + + target = cpumask_any_and_but(cpumask_of_node(numa_node), + cpu_online_mask, cpu); + if (target >= nr_cpu_ids) target = cpumask_any_but(cpu_online_mask, cpu); if (target >= nr_cpu_ids) { @@ -807,6 +818,7 @@ static int hisi_pcie_alloc_pmu(struct pci_dev *pdev, struct hisi_pcie_pmu *pcie_ pcie_pmu->pmu = (struct pmu) { .name = name, .module = THIS_MODULE, + .parent = &pdev->dev, .event_init = hisi_pcie_pmu_event_init, .pmu_enable = hisi_pcie_pmu_enable, .pmu_disable = hisi_pcie_pmu_disable, diff --git a/drivers/perf/hisilicon/hisi_uncore_pmu.c b/drivers/perf/hisilicon/hisi_uncore_pmu.c index 04031450d5fe..a60e4c966098 100644 --- a/drivers/perf/hisilicon/hisi_uncore_pmu.c +++ b/drivers/perf/hisilicon/hisi_uncore_pmu.c @@ -504,7 +504,6 @@ int hisi_uncore_pmu_offline_cpu(unsigned int cpu, struct hlist_node *node) { struct hisi_pmu *hisi_pmu = hlist_entry_safe(node, struct hisi_pmu, node); - cpumask_t pmu_online_cpus; unsigned int target; if (!cpumask_test_and_clear_cpu(cpu, &hisi_pmu->associated_cpus)) @@ -518,9 +517,8 @@ int hisi_uncore_pmu_offline_cpu(unsigned int cpu, struct hlist_node *node) hisi_pmu->on_cpu = -1; /* Choose a new CPU to migrate ownership of the PMU to */ - cpumask_and(&pmu_online_cpus, &hisi_pmu->associated_cpus, - cpu_online_mask); - target = cpumask_any_but(&pmu_online_cpus, cpu); + target = cpumask_any_and_but(&hisi_pmu->associated_cpus, + cpu_online_mask, cpu); if (target >= nr_cpu_ids) return 0; @@ -538,6 +536,7 @@ void hisi_pmu_init(struct hisi_pmu *hisi_pmu, struct module *module) struct pmu *pmu = &hisi_pmu->pmu; pmu->module = module; + pmu->parent = hisi_pmu->dev; pmu->task_ctx_nr = perf_invalid_context; pmu->event_init = hisi_uncore_pmu_event_init; pmu->pmu_enable = hisi_uncore_pmu_enable; diff --git a/drivers/perf/hisilicon/hns3_pmu.c b/drivers/perf/hisilicon/hns3_pmu.c index 16869bf5bf4c..e900f8e00b18 100644 --- a/drivers/perf/hisilicon/hns3_pmu.c +++ b/drivers/perf/hisilicon/hns3_pmu.c @@ -1085,15 +1085,27 @@ static bool hns3_pmu_validate_event_group(struct perf_event *event) return false; for (num = 0; num < counters; num++) { + /* + * If we find a related event, then it's a valid group + * since we don't need to allocate a new counter for it. + */ if (hns3_pmu_cmp_event(event_group[num], sibling)) break; } + /* + * Otherwise it's a new event but if there's no available counter, + * fail the check since we cannot schedule all the events in + * the group simultaneously. + */ + if (num == HNS3_PMU_MAX_HW_EVENTS) + return false; + if (num == counters) event_group[counters++] = sibling; } - return counters <= HNS3_PMU_MAX_HW_EVENTS; + return true; } static u32 hns3_pmu_get_filter_condition(struct perf_event *event) @@ -1419,6 +1431,7 @@ static int hns3_pmu_alloc_pmu(struct pci_dev *pdev, struct hns3_pmu *hns3_pmu) hns3_pmu->pmu = (struct pmu) { .name = name, .module = THIS_MODULE, + .parent = &pdev->dev, .event_init = hns3_pmu_event_init, .pmu_enable = hns3_pmu_enable, .pmu_disable = hns3_pmu_disable, @@ -1515,7 +1528,7 @@ static int hns3_pmu_irq_register(struct pci_dev *pdev, return ret; } - ret = devm_add_action(&pdev->dev, hns3_pmu_free_irq, pdev); + ret = devm_add_action_or_reset(&pdev->dev, hns3_pmu_free_irq, pdev); if (ret) { pci_err(pdev, "failed to add free irq action, ret = %d.\n", ret); return ret; diff --git a/drivers/perf/qcom_l2_pmu.c b/drivers/perf/qcom_l2_pmu.c index 148df5ae8ef8..980e3051edd7 100644 --- a/drivers/perf/qcom_l2_pmu.c +++ b/drivers/perf/qcom_l2_pmu.c @@ -801,9 +801,8 @@ static int l2cache_pmu_online_cpu(unsigned int cpu, struct hlist_node *node) static int l2cache_pmu_offline_cpu(unsigned int cpu, struct hlist_node *node) { - struct cluster_pmu *cluster; struct l2cache_pmu *l2cache_pmu; - cpumask_t cluster_online_cpus; + struct cluster_pmu *cluster; unsigned int target; l2cache_pmu = hlist_entry_safe(node, struct l2cache_pmu, node); @@ -820,9 +819,8 @@ static int l2cache_pmu_offline_cpu(unsigned int cpu, struct hlist_node *node) cluster->on_cpu = -1; /* Any other CPU for this cluster which is still online */ - cpumask_and(&cluster_online_cpus, &cluster->cluster_cpus, - cpu_online_mask); - target = cpumask_any_but(&cluster_online_cpus, cpu); + target = cpumask_any_and_but(&cluster->cluster_cpus, + cpu_online_mask, cpu); if (target >= nr_cpu_ids) { disable_irq(cluster->irq); return 0; @@ -904,6 +902,7 @@ static int l2_cache_pmu_probe(struct platform_device *pdev) l2cache_pmu->pmu = (struct pmu) { /* suffix is instance id for future use with multiple sockets */ .name = "l2cache_0", + .parent = &pdev->dev, .task_ctx_nr = perf_invalid_context, .pmu_enable = l2_cache_pmu_enable, .pmu_disable = l2_cache_pmu_disable, diff --git a/drivers/perf/qcom_l3_pmu.c b/drivers/perf/qcom_l3_pmu.c index f16783d03db7..37786e88514e 100644 --- a/drivers/perf/qcom_l3_pmu.c +++ b/drivers/perf/qcom_l3_pmu.c @@ -748,6 +748,7 @@ static int qcom_l3_cache_pmu_probe(struct platform_device *pdev) return -ENOMEM; l3pmu->pmu = (struct pmu) { + .parent = &pdev->dev, .task_ctx_nr = perf_invalid_context, .pmu_enable = qcom_l3_cache__pmu_enable, diff --git a/drivers/perf/riscv_pmu_legacy.c b/drivers/perf/riscv_pmu_legacy.c index fa0bccf4edf2..04487ad7fba0 100644 --- a/drivers/perf/riscv_pmu_legacy.c +++ b/drivers/perf/riscv_pmu_legacy.c @@ -136,6 +136,7 @@ static int pmu_legacy_device_probe(struct platform_device *pdev) pmu = riscv_pmu_alloc(); if (!pmu) return -ENOMEM; + pmu->pmu.parent = &pdev->dev; pmu_legacy_init(pmu); return 0; diff --git a/drivers/perf/riscv_pmu_sbi.c b/drivers/perf/riscv_pmu_sbi.c index 8cbe6e5f9c39..82636273d726 100644 --- a/drivers/perf/riscv_pmu_sbi.c +++ b/drivers/perf/riscv_pmu_sbi.c @@ -1043,7 +1043,6 @@ static struct ctl_table sbi_pmu_sysctl_table[] = { .extra1 = SYSCTL_ZERO, .extra2 = SYSCTL_TWO, }, - { } }; static int pmu_sbi_device_probe(struct platform_device *pdev) @@ -1081,6 +1080,7 @@ static int pmu_sbi_device_probe(struct platform_device *pdev) } pmu->pmu.attr_groups = riscv_pmu_attr_groups; + pmu->pmu.parent = &pdev->dev; pmu->cmask = cmask; pmu->ctr_start = pmu_sbi_ctr_start; pmu->ctr_stop = pmu_sbi_ctr_stop; diff --git a/drivers/perf/thunderx2_pmu.c b/drivers/perf/thunderx2_pmu.c index e16d10c763de..faf763d2c95c 100644 --- a/drivers/perf/thunderx2_pmu.c +++ b/drivers/perf/thunderx2_pmu.c @@ -504,24 +504,19 @@ static void tx2_uncore_event_update(struct perf_event *event) static enum tx2_uncore_type get_tx2_pmu_type(struct acpi_device *adev) { - int i = 0; - struct acpi_tx2_pmu_device { - __u8 id[ACPI_ID_LEN]; - enum tx2_uncore_type type; - } devices[] = { + struct acpi_device_id devices[] = { {"CAV901D", PMU_TYPE_L3C}, {"CAV901F", PMU_TYPE_DMC}, {"CAV901E", PMU_TYPE_CCPI2}, - {"", PMU_TYPE_INVALID} + {} }; + const struct acpi_device_id *id; - while (devices[i].type != PMU_TYPE_INVALID) { - if (!strcmp(acpi_device_hid(adev), devices[i].id)) - break; - i++; - } + id = acpi_match_acpi_device(devices, adev); + if (!id) + return PMU_TYPE_INVALID; - return devices[i].type; + return (enum tx2_uncore_type)id->driver_data; } static bool tx2_uncore_validate_event(struct pmu *pmu, @@ -729,6 +724,7 @@ static int tx2_uncore_pmu_register( /* Perf event registration */ tx2_pmu->pmu = (struct pmu) { .module = THIS_MODULE, + .parent = tx2_pmu->dev, .attr_groups = tx2_pmu->attr_groups, .task_ctx_nr = perf_invalid_context, .event_init = tx2_uncore_event_init, @@ -932,9 +928,8 @@ static int tx2_uncore_pmu_online_cpu(unsigned int cpu, static int tx2_uncore_pmu_offline_cpu(unsigned int cpu, struct hlist_node *hpnode) { - int new_cpu; struct tx2_uncore_pmu *tx2_pmu; - struct cpumask cpu_online_mask_temp; + unsigned int new_cpu; tx2_pmu = hlist_entry_safe(hpnode, struct tx2_uncore_pmu, hpnode); @@ -945,11 +940,8 @@ static int tx2_uncore_pmu_offline_cpu(unsigned int cpu, if (tx2_pmu->hrtimer_callback) hrtimer_cancel(&tx2_pmu->hrtimer); - cpumask_copy(&cpu_online_mask_temp, cpu_online_mask); - cpumask_clear_cpu(cpu, &cpu_online_mask_temp); - new_cpu = cpumask_any_and( - cpumask_of_node(tx2_pmu->node), - &cpu_online_mask_temp); + new_cpu = cpumask_any_and_but(cpumask_of_node(tx2_pmu->node), + cpu_online_mask, cpu); tx2_pmu->cpu = new_cpu; if (new_cpu >= nr_cpu_ids) diff --git a/drivers/perf/xgene_pmu.c b/drivers/perf/xgene_pmu.c index 0d49343d704b..8823b4c6b556 100644 --- a/drivers/perf/xgene_pmu.c +++ b/drivers/perf/xgene_pmu.c @@ -1102,6 +1102,7 @@ static int xgene_init_perf(struct xgene_pmu_dev *pmu_dev, char *name) /* Perf driver registration */ pmu_dev->pmu = (struct pmu) { + .parent = pmu_dev->parent->dev, .attr_groups = pmu_dev->attr_groups, .task_ctx_nr = perf_invalid_context, .pmu_enable = xgene_perf_pmu_enable, diff --git a/drivers/pinctrl/Kconfig b/drivers/pinctrl/Kconfig index d45657aa986a..7e4f93a3bc7a 100644 --- a/drivers/pinctrl/Kconfig +++ b/drivers/pinctrl/Kconfig @@ -235,13 +235,13 @@ config PINCTRL_INGENIC config PINCTRL_K210 bool "Pinctrl driver for the Canaan Kendryte K210 SoC" - depends on RISCV && SOC_CANAAN && OF + depends on RISCV && SOC_CANAAN_K210 && OF select GENERIC_PINMUX_FUNCTIONS select GENERIC_PINCONF select GPIOLIB select OF_GPIO select REGMAP_MMIO - default SOC_CANAAN + default SOC_CANAAN_K210 help Add support for the Canaan Kendryte K210 RISC-V SOC Field Programmable IO Array (FPIOA) controller. @@ -450,6 +450,17 @@ config PINCTRL_ROCKCHIP help This support pinctrl and GPIO driver for Rockchip SoCs. +config PINCTRL_SCMI + tristate "Pinctrl driver using SCMI protocol interface" + depends on ARM_SCMI_PROTOCOL || COMPILE_TEST + select PINMUX + select GENERIC_PINCONF + help + This driver provides support for pinctrl which is controlled + by firmware that implements the SCMI interface. + It uses SCMI Message Protocol to interact with the + firmware providing all the pinctrl controls. + config PINCTRL_SINGLE tristate "One-register-per-pin type device tree based pinctrl driver" depends on OF diff --git a/drivers/pinctrl/Makefile b/drivers/pinctrl/Makefile index 2152539b53d5..cc809669405a 100644 --- a/drivers/pinctrl/Makefile +++ b/drivers/pinctrl/Makefile @@ -45,6 +45,7 @@ obj-$(CONFIG_PINCTRL_PIC32) += pinctrl-pic32.o obj-$(CONFIG_PINCTRL_PISTACHIO) += pinctrl-pistachio.o obj-$(CONFIG_PINCTRL_RK805) += pinctrl-rk805.o obj-$(CONFIG_PINCTRL_ROCKCHIP) += pinctrl-rockchip.o +obj-$(CONFIG_PINCTRL_SCMI) += pinctrl-scmi.o obj-$(CONFIG_PINCTRL_SINGLE) += pinctrl-single.o obj-$(CONFIG_PINCTRL_ST) += pinctrl-st.o obj-$(CONFIG_PINCTRL_STMFX) += pinctrl-stmfx.o diff --git a/drivers/pinctrl/pinctrl-scmi.c b/drivers/pinctrl/pinctrl-scmi.c new file mode 100644 index 000000000000..036bc1e3fc6c --- /dev/null +++ b/drivers/pinctrl/pinctrl-scmi.c @@ -0,0 +1,571 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * System Control and Power Interface (SCMI) Protocol based pinctrl driver + * + * Copyright (C) 2024 EPAM + * Copyright 2024 NXP + */ + +#include <linux/device.h> +#include <linux/err.h> +#include <linux/errno.h> +#include <linux/module.h> +#include <linux/mod_devicetable.h> +#include <linux/scmi_protocol.h> +#include <linux/slab.h> +#include <linux/types.h> + +#include <linux/pinctrl/machine.h> +#include <linux/pinctrl/pinconf.h> +#include <linux/pinctrl/pinconf-generic.h> +#include <linux/pinctrl/pinctrl.h> +#include <linux/pinctrl/pinmux.h> + +#include "pinctrl-utils.h" +#include "core.h" +#include "pinconf.h" + +#define DRV_NAME "scmi-pinctrl" + +/* Define num configs, if not large than 4 use stack, else use kcalloc() */ +#define SCMI_NUM_CONFIGS 4 + +static const struct scmi_pinctrl_proto_ops *pinctrl_ops; + +struct scmi_pinctrl { + struct device *dev; + struct scmi_protocol_handle *ph; + struct pinctrl_dev *pctldev; + struct pinctrl_desc pctl_desc; + struct pinfunction *functions; + unsigned int nr_functions; + struct pinctrl_pin_desc *pins; + unsigned int nr_pins; +}; + +static int pinctrl_scmi_get_groups_count(struct pinctrl_dev *pctldev) +{ + struct scmi_pinctrl *pmx = pinctrl_dev_get_drvdata(pctldev); + + return pinctrl_ops->count_get(pmx->ph, GROUP_TYPE); +} + +static const char *pinctrl_scmi_get_group_name(struct pinctrl_dev *pctldev, + unsigned int selector) +{ + int ret; + const char *name; + struct scmi_pinctrl *pmx = pinctrl_dev_get_drvdata(pctldev); + + ret = pinctrl_ops->name_get(pmx->ph, selector, GROUP_TYPE, &name); + if (ret) { + dev_err(pmx->dev, "get name failed with err %d", ret); + return NULL; + } + + return name; +} + +static int pinctrl_scmi_get_group_pins(struct pinctrl_dev *pctldev, + unsigned int selector, + const unsigned int **pins, + unsigned int *num_pins) +{ + struct scmi_pinctrl *pmx = pinctrl_dev_get_drvdata(pctldev); + + return pinctrl_ops->group_pins_get(pmx->ph, selector, pins, num_pins); +} + +static const struct pinctrl_ops pinctrl_scmi_pinctrl_ops = { + .get_groups_count = pinctrl_scmi_get_groups_count, + .get_group_name = pinctrl_scmi_get_group_name, + .get_group_pins = pinctrl_scmi_get_group_pins, +#ifdef CONFIG_OF + .dt_node_to_map = pinconf_generic_dt_node_to_map_all, + .dt_free_map = pinconf_generic_dt_free_map, +#endif +}; + +static int pinctrl_scmi_get_functions_count(struct pinctrl_dev *pctldev) +{ + struct scmi_pinctrl *pmx = pinctrl_dev_get_drvdata(pctldev); + + return pinctrl_ops->count_get(pmx->ph, FUNCTION_TYPE); +} + +static const char *pinctrl_scmi_get_function_name(struct pinctrl_dev *pctldev, + unsigned int selector) +{ + int ret; + const char *name; + struct scmi_pinctrl *pmx = pinctrl_dev_get_drvdata(pctldev); + + ret = pinctrl_ops->name_get(pmx->ph, selector, FUNCTION_TYPE, &name); + if (ret) { + dev_err(pmx->dev, "get name failed with err %d", ret); + return NULL; + } + + return name; +} + +static int pinctrl_scmi_get_function_groups(struct pinctrl_dev *pctldev, + unsigned int selector, + const char * const **p_groups, + unsigned int * const p_num_groups) +{ + struct pinfunction *func; + const unsigned int *group_ids; + unsigned int num_groups; + const char **groups; + int ret, i; + struct scmi_pinctrl *pmx = pinctrl_dev_get_drvdata(pctldev); + + if (!p_groups || !p_num_groups) + return -EINVAL; + + if (selector >= pmx->nr_functions) + return -EINVAL; + + func = &pmx->functions[selector]; + if (func->ngroups) + goto done; + + ret = pinctrl_ops->function_groups_get(pmx->ph, selector, &num_groups, + &group_ids); + if (ret) { + dev_err(pmx->dev, "Unable to get function groups, err %d", ret); + return ret; + } + if (!num_groups) + return -EINVAL; + + groups = kcalloc(num_groups, sizeof(*groups), GFP_KERNEL); + if (!groups) + return -ENOMEM; + + for (i = 0; i < num_groups; i++) { + groups[i] = pinctrl_scmi_get_group_name(pctldev, group_ids[i]); + if (!groups[i]) { + ret = -EINVAL; + goto err_free; + } + } + + func->ngroups = num_groups; + func->groups = groups; +done: + *p_groups = func->groups; + *p_num_groups = func->ngroups; + + return 0; + +err_free: + kfree(groups); + + return ret; +} + +static int pinctrl_scmi_func_set_mux(struct pinctrl_dev *pctldev, + unsigned int selector, unsigned int group) +{ + struct scmi_pinctrl *pmx = pinctrl_dev_get_drvdata(pctldev); + + return pinctrl_ops->mux_set(pmx->ph, selector, group); +} + +static int pinctrl_scmi_request(struct pinctrl_dev *pctldev, + unsigned int offset) +{ + struct scmi_pinctrl *pmx = pinctrl_dev_get_drvdata(pctldev); + + return pinctrl_ops->pin_request(pmx->ph, offset); +} + +static int pinctrl_scmi_free(struct pinctrl_dev *pctldev, unsigned int offset) +{ + struct scmi_pinctrl *pmx = pinctrl_dev_get_drvdata(pctldev); + + return pinctrl_ops->pin_free(pmx->ph, offset); +} + +static const struct pinmux_ops pinctrl_scmi_pinmux_ops = { + .request = pinctrl_scmi_request, + .free = pinctrl_scmi_free, + .get_functions_count = pinctrl_scmi_get_functions_count, + .get_function_name = pinctrl_scmi_get_function_name, + .get_function_groups = pinctrl_scmi_get_function_groups, + .set_mux = pinctrl_scmi_func_set_mux, +}; + +static int pinctrl_scmi_map_pinconf_type(enum pin_config_param param, + enum scmi_pinctrl_conf_type *type) +{ + u32 arg = param; + + switch (arg) { + case PIN_CONFIG_BIAS_BUS_HOLD: + *type = SCMI_PIN_BIAS_BUS_HOLD; + break; + case PIN_CONFIG_BIAS_DISABLE: + *type = SCMI_PIN_BIAS_DISABLE; + break; + case PIN_CONFIG_BIAS_HIGH_IMPEDANCE: + *type = SCMI_PIN_BIAS_HIGH_IMPEDANCE; + break; + case PIN_CONFIG_BIAS_PULL_DOWN: + *type = SCMI_PIN_BIAS_PULL_DOWN; + break; + case PIN_CONFIG_BIAS_PULL_PIN_DEFAULT: + *type = SCMI_PIN_BIAS_PULL_DEFAULT; + break; + case PIN_CONFIG_BIAS_PULL_UP: + *type = SCMI_PIN_BIAS_PULL_UP; + break; + case PIN_CONFIG_DRIVE_OPEN_DRAIN: + *type = SCMI_PIN_DRIVE_OPEN_DRAIN; + break; + case PIN_CONFIG_DRIVE_OPEN_SOURCE: + *type = SCMI_PIN_DRIVE_OPEN_SOURCE; + break; + case PIN_CONFIG_DRIVE_PUSH_PULL: + *type = SCMI_PIN_DRIVE_PUSH_PULL; + break; + case PIN_CONFIG_DRIVE_STRENGTH: + *type = SCMI_PIN_DRIVE_STRENGTH; + break; + case PIN_CONFIG_DRIVE_STRENGTH_UA: + *type = SCMI_PIN_DRIVE_STRENGTH; + break; + case PIN_CONFIG_INPUT_DEBOUNCE: + *type = SCMI_PIN_INPUT_DEBOUNCE; + break; + case PIN_CONFIG_INPUT_ENABLE: + *type = SCMI_PIN_INPUT_MODE; + break; + case PIN_CONFIG_INPUT_SCHMITT: + *type = SCMI_PIN_INPUT_SCHMITT; + break; + case PIN_CONFIG_INPUT_SCHMITT_ENABLE: + *type = SCMI_PIN_INPUT_MODE; + break; + case PIN_CONFIG_MODE_LOW_POWER: + *type = SCMI_PIN_LOW_POWER_MODE; + break; + case PIN_CONFIG_OUTPUT: + *type = SCMI_PIN_OUTPUT_VALUE; + break; + case PIN_CONFIG_OUTPUT_ENABLE: + *type = SCMI_PIN_OUTPUT_MODE; + break; + case PIN_CONFIG_OUTPUT_IMPEDANCE_OHMS: + *type = SCMI_PIN_OUTPUT_VALUE; + break; + case PIN_CONFIG_POWER_SOURCE: + *type = SCMI_PIN_POWER_SOURCE; + break; + case PIN_CONFIG_SLEW_RATE: + *type = SCMI_PIN_SLEW_RATE; + break; + case SCMI_PIN_OEM_START ... SCMI_PIN_OEM_END: + *type = arg; + break; + default: + return -EINVAL; + } + + return 0; +} + +static int pinctrl_scmi_pinconf_get(struct pinctrl_dev *pctldev, + unsigned int pin, unsigned long *config) +{ + int ret; + struct scmi_pinctrl *pmx = pinctrl_dev_get_drvdata(pctldev); + enum pin_config_param config_type; + enum scmi_pinctrl_conf_type type; + u32 config_value; + + if (!config) + return -EINVAL; + + config_type = pinconf_to_config_param(*config); + + ret = pinctrl_scmi_map_pinconf_type(config_type, &type); + if (ret) + return ret; + + ret = pinctrl_ops->settings_get_one(pmx->ph, pin, PIN_TYPE, type, + &config_value); + /* Convert SCMI error code to PINCTRL expected error code */ + if (ret == -EOPNOTSUPP) + return -ENOTSUPP; + if (ret) + return ret; + + *config = pinconf_to_config_packed(config_type, config_value); + + return 0; +} + +static int +pinctrl_scmi_alloc_configs(struct pinctrl_dev *pctldev, u32 num_configs, + u32 **p_config_value, + enum scmi_pinctrl_conf_type **p_config_type) +{ + if (num_configs <= SCMI_NUM_CONFIGS) + return 0; + + *p_config_value = kcalloc(num_configs, sizeof(**p_config_value), GFP_KERNEL); + if (!*p_config_value) + return -ENOMEM; + + *p_config_type = kcalloc(num_configs, sizeof(**p_config_type), GFP_KERNEL); + if (!*p_config_type) { + kfree(*p_config_value); + return -ENOMEM; + } + + return 0; +} + +static void +pinctrl_scmi_free_configs(struct pinctrl_dev *pctldev, u32 num_configs, + u32 **p_config_value, + enum scmi_pinctrl_conf_type **p_config_type) +{ + if (num_configs <= SCMI_NUM_CONFIGS) + return; + + kfree(*p_config_value); + kfree(*p_config_type); +} + +static int pinctrl_scmi_pinconf_set(struct pinctrl_dev *pctldev, + unsigned int pin, + unsigned long *configs, + unsigned int num_configs) +{ + int i, ret; + struct scmi_pinctrl *pmx = pinctrl_dev_get_drvdata(pctldev); + enum scmi_pinctrl_conf_type config_type[SCMI_NUM_CONFIGS]; + u32 config_value[SCMI_NUM_CONFIGS]; + enum scmi_pinctrl_conf_type *p_config_type = config_type; + u32 *p_config_value = config_value; + enum pin_config_param param; + + if (!configs || !num_configs) + return -EINVAL; + + ret = pinctrl_scmi_alloc_configs(pctldev, num_configs, &p_config_type, + &p_config_value); + if (ret) + return ret; + + for (i = 0; i < num_configs; i++) { + param = pinconf_to_config_param(configs[i]); + ret = pinctrl_scmi_map_pinconf_type(param, &p_config_type[i]); + if (ret) { + dev_err(pmx->dev, "Error map pinconf_type %d\n", ret); + goto free_config; + } + p_config_value[i] = pinconf_to_config_argument(configs[i]); + } + + ret = pinctrl_ops->settings_conf(pmx->ph, pin, PIN_TYPE, num_configs, + p_config_type, p_config_value); + if (ret) + dev_err(pmx->dev, "Error parsing config %d\n", ret); + +free_config: + pinctrl_scmi_free_configs(pctldev, num_configs, &p_config_type, + &p_config_value); + return ret; +} + +static int pinctrl_scmi_pinconf_group_set(struct pinctrl_dev *pctldev, + unsigned int group, + unsigned long *configs, + unsigned int num_configs) +{ + int i, ret; + struct scmi_pinctrl *pmx = pinctrl_dev_get_drvdata(pctldev); + enum scmi_pinctrl_conf_type config_type[SCMI_NUM_CONFIGS]; + u32 config_value[SCMI_NUM_CONFIGS]; + enum scmi_pinctrl_conf_type *p_config_type = config_type; + u32 *p_config_value = config_value; + enum pin_config_param param; + + if (!configs || !num_configs) + return -EINVAL; + + ret = pinctrl_scmi_alloc_configs(pctldev, num_configs, &p_config_type, + &p_config_value); + if (ret) + return ret; + + for (i = 0; i < num_configs; i++) { + param = pinconf_to_config_param(configs[i]); + ret = pinctrl_scmi_map_pinconf_type(param, &p_config_type[i]); + if (ret) { + dev_err(pmx->dev, "Error map pinconf_type %d\n", ret); + goto free_config; + } + + p_config_value[i] = pinconf_to_config_argument(configs[i]); + } + + ret = pinctrl_ops->settings_conf(pmx->ph, group, GROUP_TYPE, + num_configs, p_config_type, + p_config_value); + if (ret) + dev_err(pmx->dev, "Error parsing config %d", ret); + +free_config: + pinctrl_scmi_free_configs(pctldev, num_configs, &p_config_type, + &p_config_value); + return ret; +}; + +static int pinctrl_scmi_pinconf_group_get(struct pinctrl_dev *pctldev, + unsigned int group, + unsigned long *config) +{ + int ret; + struct scmi_pinctrl *pmx = pinctrl_dev_get_drvdata(pctldev); + enum pin_config_param config_type; + enum scmi_pinctrl_conf_type type; + u32 config_value; + + if (!config) + return -EINVAL; + + config_type = pinconf_to_config_param(*config); + ret = pinctrl_scmi_map_pinconf_type(config_type, &type); + if (ret) { + dev_err(pmx->dev, "Error map pinconf_type %d\n", ret); + return ret; + } + + ret = pinctrl_ops->settings_get_one(pmx->ph, group, GROUP_TYPE, type, + &config_value); + /* Convert SCMI error code to PINCTRL expected error code */ + if (ret == -EOPNOTSUPP) + return -ENOTSUPP; + if (ret) + return ret; + + *config = pinconf_to_config_packed(config_type, config_value); + + return 0; +} + +static const struct pinconf_ops pinctrl_scmi_pinconf_ops = { + .is_generic = true, + .pin_config_get = pinctrl_scmi_pinconf_get, + .pin_config_set = pinctrl_scmi_pinconf_set, + .pin_config_group_set = pinctrl_scmi_pinconf_group_set, + .pin_config_group_get = pinctrl_scmi_pinconf_group_get, + .pin_config_config_dbg_show = pinconf_generic_dump_config, +}; + +static int pinctrl_scmi_get_pins(struct scmi_pinctrl *pmx, + struct pinctrl_desc *desc) +{ + struct pinctrl_pin_desc *pins; + unsigned int npins; + int ret, i; + + npins = pinctrl_ops->count_get(pmx->ph, PIN_TYPE); + /* + * npins will never be zero, the scmi pinctrl driver has bailed out + * if npins is zero. + */ + pins = devm_kmalloc_array(pmx->dev, npins, sizeof(*pins), GFP_KERNEL); + if (!pins) + return -ENOMEM; + + for (i = 0; i < npins; i++) { + pins[i].number = i; + /* + * The memory for name is handled by the scmi firmware driver, + * no need free here + */ + ret = pinctrl_ops->name_get(pmx->ph, i, PIN_TYPE, &pins[i].name); + if (ret) + return dev_err_probe(pmx->dev, ret, + "Can't get name for pin %d", i); + } + + desc->npins = npins; + desc->pins = pins; + dev_dbg(pmx->dev, "got pins %u", npins); + + return 0; +} + +static int scmi_pinctrl_probe(struct scmi_device *sdev) +{ + int ret; + struct device *dev = &sdev->dev; + struct scmi_pinctrl *pmx; + const struct scmi_handle *handle; + struct scmi_protocol_handle *ph; + + if (!sdev->handle) + return -EINVAL; + + handle = sdev->handle; + + pinctrl_ops = handle->devm_protocol_get(sdev, SCMI_PROTOCOL_PINCTRL, &ph); + if (IS_ERR(pinctrl_ops)) + return PTR_ERR(pinctrl_ops); + + pmx = devm_kzalloc(dev, sizeof(*pmx), GFP_KERNEL); + if (!pmx) + return -ENOMEM; + + pmx->ph = ph; + + pmx->dev = dev; + pmx->pctl_desc.name = DRV_NAME; + pmx->pctl_desc.owner = THIS_MODULE; + pmx->pctl_desc.pctlops = &pinctrl_scmi_pinctrl_ops; + pmx->pctl_desc.pmxops = &pinctrl_scmi_pinmux_ops; + pmx->pctl_desc.confops = &pinctrl_scmi_pinconf_ops; + + ret = pinctrl_scmi_get_pins(pmx, &pmx->pctl_desc); + if (ret) + return ret; + + ret = devm_pinctrl_register_and_init(dev, &pmx->pctl_desc, pmx, + &pmx->pctldev); + if (ret) + return dev_err_probe(dev, ret, "Failed to register pinctrl\n"); + + pmx->nr_functions = pinctrl_scmi_get_functions_count(pmx->pctldev); + pmx->functions = devm_kcalloc(dev, pmx->nr_functions, + sizeof(*pmx->functions), GFP_KERNEL); + if (!pmx->functions) + return -ENOMEM; + + return pinctrl_enable(pmx->pctldev); +} + +static const struct scmi_device_id scmi_id_table[] = { + { SCMI_PROTOCOL_PINCTRL, "pinctrl" }, + { } +}; +MODULE_DEVICE_TABLE(scmi, scmi_id_table); + +static struct scmi_driver scmi_pinctrl_driver = { + .name = DRV_NAME, + .probe = scmi_pinctrl_probe, + .id_table = scmi_id_table, +}; +module_scmi_driver(scmi_pinctrl_driver); + +MODULE_AUTHOR("Oleksii Moisieiev <oleksii_moisieiev@epam.com>"); +MODULE_AUTHOR("Peng Fan <peng.fan@nxp.com>"); +MODULE_DESCRIPTION("ARM SCMI pin controller driver"); +MODULE_LICENSE("GPL"); diff --git a/drivers/platform/chrome/Kconfig b/drivers/platform/chrome/Kconfig index 7a83346bfa53..073616b5b5a0 100644 --- a/drivers/platform/chrome/Kconfig +++ b/drivers/platform/chrome/Kconfig @@ -132,6 +132,7 @@ config CROS_EC_UART config CROS_EC_LPC tristate "ChromeOS Embedded Controller (LPC)" depends on CROS_EC && ACPI && (X86 || COMPILE_TEST) + depends on HAS_IOPORT help If you say Y here, you get support for talking to the ChromeOS EC over an LPC bus, including the LPC Microchip EC (MEC) variant. diff --git a/drivers/platform/chrome/cros_ec.c b/drivers/platform/chrome/cros_ec.c index badc68bbae8c..47d19f7e295a 100644 --- a/drivers/platform/chrome/cros_ec.c +++ b/drivers/platform/chrome/cros_ec.c @@ -432,6 +432,12 @@ static void cros_ec_send_resume_event(struct cros_ec_device *ec_dev) void cros_ec_resume_complete(struct cros_ec_device *ec_dev) { cros_ec_send_resume_event(ec_dev); + + /* + * Let the mfd devices know about events that occur during + * suspend. This way the clients know what to do with them. + */ + cros_ec_report_events_during_suspend(ec_dev); } EXPORT_SYMBOL(cros_ec_resume_complete); @@ -442,12 +448,6 @@ static void cros_ec_enable_irq(struct cros_ec_device *ec_dev) if (ec_dev->wake_enabled) disable_irq_wake(ec_dev->irq); - - /* - * Let the mfd devices know about events that occur during - * suspend. This way the clients know what to do with them. - */ - cros_ec_report_events_during_suspend(ec_dev); } /** @@ -475,8 +475,8 @@ EXPORT_SYMBOL(cros_ec_resume_early); */ int cros_ec_resume(struct cros_ec_device *ec_dev) { - cros_ec_enable_irq(ec_dev); - cros_ec_send_resume_event(ec_dev); + cros_ec_resume_early(ec_dev); + cros_ec_resume_complete(ec_dev); return 0; } EXPORT_SYMBOL(cros_ec_resume); diff --git a/drivers/platform/chrome/cros_ec_chardev.c b/drivers/platform/chrome/cros_ec_chardev.c index 81950bb2c6da..7f034ead7ae4 100644 --- a/drivers/platform/chrome/cros_ec_chardev.c +++ b/drivers/platform/chrome/cros_ec_chardev.c @@ -14,6 +14,7 @@ #include <linux/device.h> #include <linux/fs.h> #include <linux/miscdevice.h> +#include <linux/mod_devicetable.h> #include <linux/module.h> #include <linux/notifier.h> #include <linux/platform_data/cros_ec_chardev.h> @@ -403,17 +404,23 @@ static void cros_ec_chardev_remove(struct platform_device *pdev) misc_deregister(&data->misc); } +static const struct platform_device_id cros_ec_chardev_id[] = { + { DRV_NAME, 0 }, + {} +}; +MODULE_DEVICE_TABLE(platform, cros_ec_chardev_id); + static struct platform_driver cros_ec_chardev_driver = { .driver = { .name = DRV_NAME, }, .probe = cros_ec_chardev_probe, .remove_new = cros_ec_chardev_remove, + .id_table = cros_ec_chardev_id, }; module_platform_driver(cros_ec_chardev_driver); -MODULE_ALIAS("platform:" DRV_NAME); MODULE_AUTHOR("Enric Balletbo i Serra <enric.balletbo@collabora.com>"); MODULE_DESCRIPTION("ChromeOS EC Miscellaneous Character Driver"); MODULE_LICENSE("GPL"); diff --git a/drivers/platform/chrome/cros_ec_debugfs.c b/drivers/platform/chrome/cros_ec_debugfs.c index 6bf6f0e7b597..e1d313246beb 100644 --- a/drivers/platform/chrome/cros_ec_debugfs.c +++ b/drivers/platform/chrome/cros_ec_debugfs.c @@ -7,6 +7,7 @@ #include <linux/debugfs.h> #include <linux/delay.h> #include <linux/fs.h> +#include <linux/mod_devicetable.h> #include <linux/module.h> #include <linux/mutex.h> #include <linux/platform_data/cros_ec_commands.h> @@ -564,6 +565,12 @@ static int __maybe_unused cros_ec_debugfs_resume(struct device *dev) static SIMPLE_DEV_PM_OPS(cros_ec_debugfs_pm_ops, cros_ec_debugfs_suspend, cros_ec_debugfs_resume); +static const struct platform_device_id cros_ec_debugfs_id[] = { + { DRV_NAME, 0 }, + {} +}; +MODULE_DEVICE_TABLE(platform, cros_ec_debugfs_id); + static struct platform_driver cros_ec_debugfs_driver = { .driver = { .name = DRV_NAME, @@ -572,10 +579,10 @@ static struct platform_driver cros_ec_debugfs_driver = { }, .probe = cros_ec_debugfs_probe, .remove_new = cros_ec_debugfs_remove, + .id_table = cros_ec_debugfs_id, }; module_platform_driver(cros_ec_debugfs_driver); MODULE_LICENSE("GPL"); MODULE_DESCRIPTION("Debug logs for ChromeOS EC"); -MODULE_ALIAS("platform:" DRV_NAME); diff --git a/drivers/platform/chrome/cros_ec_lightbar.c b/drivers/platform/chrome/cros_ec_lightbar.c index 6677cc6c4984..1e69f61115a4 100644 --- a/drivers/platform/chrome/cros_ec_lightbar.c +++ b/drivers/platform/chrome/cros_ec_lightbar.c @@ -9,6 +9,7 @@ #include <linux/fs.h> #include <linux/kobject.h> #include <linux/kstrtox.h> +#include <linux/mod_devicetable.h> #include <linux/module.h> #include <linux/platform_data/cros_ec_commands.h> #include <linux/platform_data/cros_ec_proto.h> @@ -594,6 +595,12 @@ static int __maybe_unused cros_ec_lightbar_suspend(struct device *dev) static SIMPLE_DEV_PM_OPS(cros_ec_lightbar_pm_ops, cros_ec_lightbar_suspend, cros_ec_lightbar_resume); +static const struct platform_device_id cros_ec_lightbar_id[] = { + { DRV_NAME, 0 }, + {} +}; +MODULE_DEVICE_TABLE(platform, cros_ec_lightbar_id); + static struct platform_driver cros_ec_lightbar_driver = { .driver = { .name = DRV_NAME, @@ -602,10 +609,10 @@ static struct platform_driver cros_ec_lightbar_driver = { }, .probe = cros_ec_lightbar_probe, .remove_new = cros_ec_lightbar_remove, + .id_table = cros_ec_lightbar_id, }; module_platform_driver(cros_ec_lightbar_driver); MODULE_LICENSE("GPL"); MODULE_DESCRIPTION("Expose the Chromebook Pixel's lightbar to userspace"); -MODULE_ALIAS("platform:" DRV_NAME); diff --git a/drivers/platform/chrome/cros_ec_lpc.c b/drivers/platform/chrome/cros_ec_lpc.c index f0f3d3d56157..ddfbfec44f4c 100644 --- a/drivers/platform/chrome/cros_ec_lpc.c +++ b/drivers/platform/chrome/cros_ec_lpc.c @@ -34,6 +34,32 @@ /* True if ACPI device is present */ static bool cros_ec_lpc_acpi_device_found; +/* + * Indicates that lpc_driver_data.quirk_mmio_memory_base should + * be used as the base port for EC mapped memory. + */ +#define CROS_EC_LPC_QUIRK_REMAP_MEMORY BIT(0) + +/** + * struct lpc_driver_data - driver data attached to a DMI device ID to indicate + * hardware quirks. + * @quirks: a bitfield composed of quirks from CROS_EC_LPC_QUIRK_* + * @quirk_mmio_memory_base: The first I/O port addressing EC mapped memory (used + * when quirk ...REMAP_MEMORY is set.) + */ +struct lpc_driver_data { + u32 quirks; + u16 quirk_mmio_memory_base; +}; + +/** + * struct cros_ec_lpc - LPC device-specific data + * @mmio_memory_base: The first I/O port addressing EC mapped memory. + */ +struct cros_ec_lpc { + u16 mmio_memory_base; +}; + /** * struct lpc_driver_ops - LPC driver operations * @read: Copy length bytes from EC address offset into buffer dest. Returns @@ -290,6 +316,7 @@ done: static int cros_ec_lpc_readmem(struct cros_ec_device *ec, unsigned int offset, unsigned int bytes, void *dest) { + struct cros_ec_lpc *ec_lpc = ec->priv; int i = offset; char *s = dest; int cnt = 0; @@ -299,13 +326,13 @@ static int cros_ec_lpc_readmem(struct cros_ec_device *ec, unsigned int offset, /* fixed length */ if (bytes) { - cros_ec_lpc_ops.read(EC_LPC_ADDR_MEMMAP + offset, bytes, s); + cros_ec_lpc_ops.read(ec_lpc->mmio_memory_base + offset, bytes, s); return bytes; } /* string */ for (; i < EC_MEMMAP_SIZE; i++, s++) { - cros_ec_lpc_ops.read(EC_LPC_ADDR_MEMMAP + i, 1, s); + cros_ec_lpc_ops.read(ec_lpc->mmio_memory_base + i, 1, s); cnt++; if (!*s) break; @@ -353,8 +380,28 @@ static int cros_ec_lpc_probe(struct platform_device *pdev) struct acpi_device *adev; acpi_status status; struct cros_ec_device *ec_dev; + struct cros_ec_lpc *ec_lpc; + struct lpc_driver_data *driver_data; u8 buf[2] = {}; int irq, ret; + u32 quirks; + + ec_lpc = devm_kzalloc(dev, sizeof(*ec_lpc), GFP_KERNEL); + if (!ec_lpc) + return -ENOMEM; + + ec_lpc->mmio_memory_base = EC_LPC_ADDR_MEMMAP; + + driver_data = platform_get_drvdata(pdev); + if (driver_data) { + quirks = driver_data->quirks; + + if (quirks) + dev_info(dev, "loaded with quirks %8.08x\n", quirks); + + if (quirks & CROS_EC_LPC_QUIRK_REMAP_MEMORY) + ec_lpc->mmio_memory_base = driver_data->quirk_mmio_memory_base; + } /* * The Framework Laptop (and possibly other non-ChromeOS devices) @@ -380,7 +427,7 @@ static int cros_ec_lpc_probe(struct platform_device *pdev) cros_ec_lpc_ops.write = cros_ec_lpc_mec_write_bytes; cros_ec_lpc_ops.read(EC_LPC_ADDR_MEMMAP + EC_MEMMAP_ID, 2, buf); if (buf[0] != 'E' || buf[1] != 'C') { - if (!devm_request_region(dev, EC_LPC_ADDR_MEMMAP, EC_MEMMAP_SIZE, + if (!devm_request_region(dev, ec_lpc->mmio_memory_base, EC_MEMMAP_SIZE, dev_name(dev))) { dev_err(dev, "couldn't reserve memmap region\n"); return -EBUSY; @@ -389,7 +436,7 @@ static int cros_ec_lpc_probe(struct platform_device *pdev) /* Re-assign read/write operations for the non MEC variant */ cros_ec_lpc_ops.read = cros_ec_lpc_read_bytes; cros_ec_lpc_ops.write = cros_ec_lpc_write_bytes; - cros_ec_lpc_ops.read(EC_LPC_ADDR_MEMMAP + EC_MEMMAP_ID, 2, + cros_ec_lpc_ops.read(ec_lpc->mmio_memory_base + EC_MEMMAP_ID, 2, buf); if (buf[0] != 'E' || buf[1] != 'C') { dev_err(dev, "EC ID not detected\n"); @@ -423,6 +470,7 @@ static int cros_ec_lpc_probe(struct platform_device *pdev) ec_dev->din_size = sizeof(struct ec_host_response) + sizeof(struct ec_response_get_protocol_info); ec_dev->dout_size = sizeof(struct ec_host_request); + ec_dev->priv = ec_lpc; /* * Some boards do not have an IRQ allotted for cros_ec_lpc, @@ -479,6 +527,11 @@ static const struct acpi_device_id cros_ec_lpc_acpi_device_ids[] = { }; MODULE_DEVICE_TABLE(acpi, cros_ec_lpc_acpi_device_ids); +static const struct lpc_driver_data framework_laptop_amd_lpc_driver_data __initconst = { + .quirks = CROS_EC_LPC_QUIRK_REMAP_MEMORY, + .quirk_mmio_memory_base = 0xE00, +}; + static const struct dmi_system_id cros_ec_lpc_dmi_table[] __initconst = { { /* @@ -533,7 +586,16 @@ static const struct dmi_system_id cros_ec_lpc_dmi_table[] __initconst = { }, /* A small number of non-Chromebook/box machines also use the ChromeOS EC */ { - /* the Framework Laptop */ + /* the Framework Laptop 13 (AMD Ryzen) and 16 (AMD Ryzen) */ + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "Framework"), + DMI_MATCH(DMI_PRODUCT_NAME, "AMD Ryzen"), + DMI_MATCH(DMI_PRODUCT_FAMILY, "Laptop"), + }, + .driver_data = (void *)&framework_laptop_amd_lpc_driver_data, + }, + { + /* the Framework Laptop (Intel 11th, 12th, 13th Generation) */ .matches = { DMI_MATCH(DMI_SYS_VENDOR, "Framework"), DMI_MATCH(DMI_PRODUCT_NAME, "Laptop"), @@ -610,14 +672,16 @@ static int __init cros_ec_lpc_init(void) { int ret; acpi_status status; + const struct dmi_system_id *dmi_match; status = acpi_get_devices(ACPI_DRV_NAME, cros_ec_lpc_parse_device, &cros_ec_lpc_acpi_device_found, NULL); if (ACPI_FAILURE(status)) pr_warn(DRV_NAME ": Looking for %s failed\n", ACPI_DRV_NAME); - if (!cros_ec_lpc_acpi_device_found && - !dmi_check_system(cros_ec_lpc_dmi_table)) { + dmi_match = dmi_first_match(cros_ec_lpc_dmi_table); + + if (!cros_ec_lpc_acpi_device_found && !dmi_match) { pr_err(DRV_NAME ": unsupported system.\n"); return -ENODEV; } @@ -630,6 +694,9 @@ static int __init cros_ec_lpc_init(void) } if (!cros_ec_lpc_acpi_device_found) { + /* Pass the DMI match's driver data down to the platform device */ + platform_set_drvdata(&cros_ec_lpc_device, dmi_match->driver_data); + /* Register the device, and it'll get hooked up automatically */ ret = platform_device_register(&cros_ec_lpc_device); if (ret) { diff --git a/drivers/platform/chrome/cros_ec_proto_test.c b/drivers/platform/chrome/cros_ec_proto_test.c index b6169d6f2467..41378c2ee6a0 100644 --- a/drivers/platform/chrome/cros_ec_proto_test.c +++ b/drivers/platform/chrome/cros_ec_proto_test.c @@ -1543,21 +1543,18 @@ static void cros_ec_proto_test_cmd_xfer_normal(struct kunit *test) struct cros_ec_device *ec_dev = &priv->ec_dev; struct ec_xfer_mock *mock; int ret; - struct { - struct cros_ec_command msg; - u8 data[0x100]; - } __packed buf; + DEFINE_RAW_FLEX(struct cros_ec_command, buf, data, 0x100); ec_dev->max_request = 0xff; ec_dev->max_response = 0xee; ec_dev->max_passthru = 0xdd; - buf.msg.version = 0; - buf.msg.command = EC_CMD_HELLO; - buf.msg.insize = 4; - buf.msg.outsize = 2; - buf.data[0] = 0x55; - buf.data[1] = 0xaa; + buf->version = 0; + buf->command = EC_CMD_HELLO; + buf->insize = 4; + buf->outsize = 2; + buf->data[0] = 0x55; + buf->data[1] = 0xaa; { u8 *data; @@ -1572,7 +1569,7 @@ static void cros_ec_proto_test_cmd_xfer_normal(struct kunit *test) data[3] = 0x33; } - ret = cros_ec_cmd_xfer(ec_dev, &buf.msg); + ret = cros_ec_cmd_xfer(ec_dev, buf); KUNIT_EXPECT_EQ(test, ret, 4); { @@ -1590,10 +1587,10 @@ static void cros_ec_proto_test_cmd_xfer_normal(struct kunit *test) KUNIT_EXPECT_EQ(test, data[0], 0x55); KUNIT_EXPECT_EQ(test, data[1], 0xaa); - KUNIT_EXPECT_EQ(test, buf.data[0], 0xaa); - KUNIT_EXPECT_EQ(test, buf.data[1], 0x55); - KUNIT_EXPECT_EQ(test, buf.data[2], 0xcc); - KUNIT_EXPECT_EQ(test, buf.data[3], 0x33); + KUNIT_EXPECT_EQ(test, buf->data[0], 0xaa); + KUNIT_EXPECT_EQ(test, buf->data[1], 0x55); + KUNIT_EXPECT_EQ(test, buf->data[2], 0xcc); + KUNIT_EXPECT_EQ(test, buf->data[3], 0x33); } } @@ -1603,26 +1600,23 @@ static void cros_ec_proto_test_cmd_xfer_excess_msg_insize(struct kunit *test) struct cros_ec_device *ec_dev = &priv->ec_dev; struct ec_xfer_mock *mock; int ret; - struct { - struct cros_ec_command msg; - u8 data[0x100]; - } __packed buf; + DEFINE_RAW_FLEX(struct cros_ec_command, buf, data, 0x100); ec_dev->max_request = 0xff; ec_dev->max_response = 0xee; ec_dev->max_passthru = 0xdd; - buf.msg.version = 0; - buf.msg.command = EC_CMD_HELLO; - buf.msg.insize = 0xee + 1; - buf.msg.outsize = 2; + buf->version = 0; + buf->command = EC_CMD_HELLO; + buf->insize = 0xee + 1; + buf->outsize = 2; { mock = cros_kunit_ec_xfer_mock_add(test, 0xcc); KUNIT_ASSERT_PTR_NE(test, mock, NULL); } - ret = cros_ec_cmd_xfer(ec_dev, &buf.msg); + ret = cros_ec_cmd_xfer(ec_dev, buf); KUNIT_EXPECT_EQ(test, ret, 0xcc); { @@ -1641,21 +1635,18 @@ static void cros_ec_proto_test_cmd_xfer_excess_msg_outsize_without_passthru(stru struct cros_ec_proto_test_priv *priv = test->priv; struct cros_ec_device *ec_dev = &priv->ec_dev; int ret; - struct { - struct cros_ec_command msg; - u8 data[0x100]; - } __packed buf; + DEFINE_RAW_FLEX(struct cros_ec_command, buf, data, 0x100); ec_dev->max_request = 0xff; ec_dev->max_response = 0xee; ec_dev->max_passthru = 0xdd; - buf.msg.version = 0; - buf.msg.command = EC_CMD_HELLO; - buf.msg.insize = 4; - buf.msg.outsize = 0xff + 1; + buf->version = 0; + buf->command = EC_CMD_HELLO; + buf->insize = 4; + buf->outsize = 0xff + 1; - ret = cros_ec_cmd_xfer(ec_dev, &buf.msg); + ret = cros_ec_cmd_xfer(ec_dev, buf); KUNIT_EXPECT_EQ(test, ret, -EMSGSIZE); } @@ -1664,21 +1655,18 @@ static void cros_ec_proto_test_cmd_xfer_excess_msg_outsize_with_passthru(struct struct cros_ec_proto_test_priv *priv = test->priv; struct cros_ec_device *ec_dev = &priv->ec_dev; int ret; - struct { - struct cros_ec_command msg; - u8 data[0x100]; - } __packed buf; + DEFINE_RAW_FLEX(struct cros_ec_command, buf, data, 0x100); ec_dev->max_request = 0xff; ec_dev->max_response = 0xee; ec_dev->max_passthru = 0xdd; - buf.msg.version = 0; - buf.msg.command = EC_CMD_PASSTHRU_OFFSET(CROS_EC_DEV_PD_INDEX) + EC_CMD_HELLO; - buf.msg.insize = 4; - buf.msg.outsize = 0xdd + 1; + buf->version = 0; + buf->command = EC_CMD_PASSTHRU_OFFSET(CROS_EC_DEV_PD_INDEX) + EC_CMD_HELLO; + buf->insize = 4; + buf->outsize = 0xdd + 1; - ret = cros_ec_cmd_xfer(ec_dev, &buf.msg); + ret = cros_ec_cmd_xfer(ec_dev, buf); KUNIT_EXPECT_EQ(test, ret, -EMSGSIZE); } diff --git a/drivers/platform/chrome/cros_ec_sensorhub.c b/drivers/platform/chrome/cros_ec_sensorhub.c index 31fb8bdaad5a..50cdae67fa32 100644 --- a/drivers/platform/chrome/cros_ec_sensorhub.c +++ b/drivers/platform/chrome/cros_ec_sensorhub.c @@ -8,6 +8,7 @@ #include <linux/init.h> #include <linux/device.h> +#include <linux/mod_devicetable.h> #include <linux/module.h> #include <linux/platform_data/cros_ec_commands.h> #include <linux/platform_data/cros_ec_proto.h> @@ -247,17 +248,23 @@ static SIMPLE_DEV_PM_OPS(cros_ec_sensorhub_pm_ops, cros_ec_sensorhub_suspend, cros_ec_sensorhub_resume); +static const struct platform_device_id cros_ec_sensorhub_id[] = { + { DRV_NAME, 0 }, + {} +}; +MODULE_DEVICE_TABLE(platform, cros_ec_sensorhub_id); + static struct platform_driver cros_ec_sensorhub_driver = { .driver = { .name = DRV_NAME, .pm = &cros_ec_sensorhub_pm_ops, }, .probe = cros_ec_sensorhub_probe, + .id_table = cros_ec_sensorhub_id, }; module_platform_driver(cros_ec_sensorhub_driver); -MODULE_ALIAS("platform:" DRV_NAME); MODULE_AUTHOR("Gwendal Grignou <gwendal@chromium.org>"); MODULE_DESCRIPTION("ChromeOS EC MEMS Sensor Hub Driver"); MODULE_LICENSE("GPL"); diff --git a/drivers/platform/chrome/cros_ec_sysfs.c b/drivers/platform/chrome/cros_ec_sysfs.c index 93e67ab4af06..9c944146ee50 100644 --- a/drivers/platform/chrome/cros_ec_sysfs.c +++ b/drivers/platform/chrome/cros_ec_sysfs.c @@ -8,6 +8,7 @@ #include <linux/device.h> #include <linux/fs.h> #include <linux/kobject.h> +#include <linux/mod_devicetable.h> #include <linux/module.h> #include <linux/platform_data/cros_ec_commands.h> #include <linux/platform_data/cros_ec_proto.h> @@ -347,16 +348,22 @@ static void cros_ec_sysfs_remove(struct platform_device *pd) sysfs_remove_group(&ec_dev->class_dev.kobj, &cros_ec_attr_group); } +static const struct platform_device_id cros_ec_sysfs_id[] = { + { DRV_NAME, 0 }, + {} +}; +MODULE_DEVICE_TABLE(platform, cros_ec_sysfs_id); + static struct platform_driver cros_ec_sysfs_driver = { .driver = { .name = DRV_NAME, }, .probe = cros_ec_sysfs_probe, .remove_new = cros_ec_sysfs_remove, + .id_table = cros_ec_sysfs_id, }; module_platform_driver(cros_ec_sysfs_driver); MODULE_LICENSE("GPL"); MODULE_DESCRIPTION("Expose the ChromeOS EC through sysfs"); -MODULE_ALIAS("platform:" DRV_NAME); diff --git a/drivers/platform/chrome/cros_ec_vbc.c b/drivers/platform/chrome/cros_ec_vbc.c index 274ea0c64b33..787a19db4911 100644 --- a/drivers/platform/chrome/cros_ec_vbc.c +++ b/drivers/platform/chrome/cros_ec_vbc.c @@ -6,6 +6,7 @@ #include <linux/of.h> #include <linux/platform_device.h> +#include <linux/mod_devicetable.h> #include <linux/module.h> #include <linux/platform_data/cros_ec_commands.h> #include <linux/platform_data/cros_ec_proto.h> @@ -133,16 +134,22 @@ static void cros_ec_vbc_remove(struct platform_device *pd) &cros_ec_vbc_attr_group); } +static const struct platform_device_id cros_ec_vbc_id[] = { + { DRV_NAME, 0 }, + {} +}; +MODULE_DEVICE_TABLE(platform, cros_ec_vbc_id); + static struct platform_driver cros_ec_vbc_driver = { .driver = { .name = DRV_NAME, }, .probe = cros_ec_vbc_probe, .remove_new = cros_ec_vbc_remove, + .id_table = cros_ec_vbc_id, }; module_platform_driver(cros_ec_vbc_driver); MODULE_LICENSE("GPL"); MODULE_DESCRIPTION("Expose the vboot context nvram to userspace"); -MODULE_ALIAS("platform:" DRV_NAME); diff --git a/drivers/platform/chrome/cros_hps_i2c.c b/drivers/platform/chrome/cros_hps_i2c.c index b31313080332..dd14957ec39f 100644 --- a/drivers/platform/chrome/cros_hps_i2c.c +++ b/drivers/platform/chrome/cros_hps_i2c.c @@ -126,7 +126,7 @@ static int hps_resume(struct device *dev) hps_set_power(hps, true); return 0; } -static UNIVERSAL_DEV_PM_OPS(hps_pm_ops, hps_suspend, hps_resume, NULL); +static DEFINE_RUNTIME_DEV_PM_OPS(hps_pm_ops, hps_suspend, hps_resume, NULL); static const struct i2c_device_id hps_i2c_id[] = { { "cros-hps", 0 }, @@ -148,7 +148,7 @@ static struct i2c_driver hps_i2c_driver = { .id_table = hps_i2c_id, .driver = { .name = "cros-hps", - .pm = &hps_pm_ops, + .pm = pm_ptr(&hps_pm_ops), .acpi_match_table = ACPI_PTR(hps_acpi_id), }, }; diff --git a/drivers/platform/chrome/cros_kbd_led_backlight.c b/drivers/platform/chrome/cros_kbd_led_backlight.c index 793fd3f1015d..b83e4f328620 100644 --- a/drivers/platform/chrome/cros_kbd_led_backlight.c +++ b/drivers/platform/chrome/cros_kbd_led_backlight.c @@ -9,6 +9,7 @@ #include <linux/init.h> #include <linux/kernel.h> #include <linux/leds.h> +#include <linux/mod_devicetable.h> #include <linux/module.h> #include <linux/of.h> #include <linux/platform_data/cros_ec_commands.h> @@ -247,17 +248,23 @@ static const struct of_device_id keyboard_led_of_match[] = { MODULE_DEVICE_TABLE(of, keyboard_led_of_match); #endif +static const struct platform_device_id keyboard_led_id[] = { + { "cros-keyboard-leds", 0 }, + {} +}; +MODULE_DEVICE_TABLE(platform, keyboard_led_id); + static struct platform_driver keyboard_led_driver = { .driver = { - .name = "chromeos-keyboard-leds", + .name = "cros-keyboard-leds", .acpi_match_table = ACPI_PTR(keyboard_led_acpi_match), .of_match_table = of_match_ptr(keyboard_led_of_match), }, .probe = keyboard_led_probe, + .id_table = keyboard_led_id, }; module_platform_driver(keyboard_led_driver); MODULE_AUTHOR("Simon Que <sque@chromium.org>"); MODULE_DESCRIPTION("ChromeOS Keyboard backlight LED Driver"); MODULE_LICENSE("GPL"); -MODULE_ALIAS("platform:chromeos-keyboard-leds"); diff --git a/drivers/platform/chrome/cros_usbpd_logger.c b/drivers/platform/chrome/cros_usbpd_logger.c index f618757f8b32..930c2f47269f 100644 --- a/drivers/platform/chrome/cros_usbpd_logger.c +++ b/drivers/platform/chrome/cros_usbpd_logger.c @@ -7,6 +7,7 @@ #include <linux/ktime.h> #include <linux/math64.h> +#include <linux/mod_devicetable.h> #include <linux/module.h> #include <linux/platform_data/cros_ec_commands.h> #include <linux/platform_data/cros_ec_proto.h> @@ -249,6 +250,12 @@ static int __maybe_unused cros_usbpd_logger_suspend(struct device *dev) static SIMPLE_DEV_PM_OPS(cros_usbpd_logger_pm_ops, cros_usbpd_logger_suspend, cros_usbpd_logger_resume); +static const struct platform_device_id cros_usbpd_logger_id[] = { + { DRV_NAME, 0 }, + {} +}; +MODULE_DEVICE_TABLE(platform, cros_usbpd_logger_id); + static struct platform_driver cros_usbpd_logger_driver = { .driver = { .name = DRV_NAME, @@ -256,10 +263,10 @@ static struct platform_driver cros_usbpd_logger_driver = { }, .probe = cros_usbpd_logger_probe, .remove_new = cros_usbpd_logger_remove, + .id_table = cros_usbpd_logger_id, }; module_platform_driver(cros_usbpd_logger_driver); MODULE_LICENSE("GPL v2"); MODULE_DESCRIPTION("Logging driver for ChromeOS EC USBPD Charger."); -MODULE_ALIAS("platform:" DRV_NAME); diff --git a/drivers/platform/chrome/cros_usbpd_notify.c b/drivers/platform/chrome/cros_usbpd_notify.c index aacad022f21d..c83f81d86483 100644 --- a/drivers/platform/chrome/cros_usbpd_notify.c +++ b/drivers/platform/chrome/cros_usbpd_notify.c @@ -6,6 +6,7 @@ */ #include <linux/acpi.h> +#include <linux/mod_devicetable.h> #include <linux/module.h> #include <linux/platform_data/cros_ec_proto.h> #include <linux/platform_data/cros_usbpd_notify.h> @@ -218,12 +219,19 @@ static void cros_usbpd_notify_remove_plat(struct platform_device *pdev) &pdnotify->nb); } +static const struct platform_device_id cros_usbpd_notify_id[] = { + { DRV_NAME, 0 }, + {} +}; +MODULE_DEVICE_TABLE(platform, cros_usbpd_notify_id); + static struct platform_driver cros_usbpd_notify_plat_driver = { .driver = { .name = DRV_NAME, }, .probe = cros_usbpd_notify_probe_plat, .remove_new = cros_usbpd_notify_remove_plat, + .id_table = cros_usbpd_notify_id, }; static int __init cros_usbpd_notify_init(void) @@ -258,4 +266,3 @@ module_exit(cros_usbpd_notify_exit); MODULE_LICENSE("GPL"); MODULE_DESCRIPTION("ChromeOS power delivery notifier device"); MODULE_AUTHOR("Jon Flatley <jflat@chromium.org>"); -MODULE_ALIAS("platform:" DRV_NAME); diff --git a/drivers/platform/chrome/wilco_ec/Kconfig b/drivers/platform/chrome/wilco_ec/Kconfig index 49e8530ca0ac..d1648fb099ac 100644 --- a/drivers/platform/chrome/wilco_ec/Kconfig +++ b/drivers/platform/chrome/wilco_ec/Kconfig @@ -3,6 +3,7 @@ config WILCO_EC tristate "ChromeOS Wilco Embedded Controller" depends on X86 || COMPILE_TEST depends on ACPI && CROS_EC_LPC && LEDS_CLASS + depends on HAS_IOPORT help If you say Y here, you get support for talking to the ChromeOS Wilco EC over an eSPI bus. This uses a simple byte-level protocol diff --git a/drivers/platform/chrome/wilco_ec/core.c b/drivers/platform/chrome/wilco_ec/core.c index 9b59a1bed286..3e6b6cd81a9b 100644 --- a/drivers/platform/chrome/wilco_ec/core.c +++ b/drivers/platform/chrome/wilco_ec/core.c @@ -10,6 +10,7 @@ #include <linux/acpi.h> #include <linux/device.h> #include <linux/ioport.h> +#include <linux/mod_devicetable.h> #include <linux/module.h> #include <linux/platform_data/wilco-ec.h> #include <linux/platform_device.h> @@ -150,6 +151,12 @@ static const struct acpi_device_id wilco_ec_acpi_device_ids[] = { }; MODULE_DEVICE_TABLE(acpi, wilco_ec_acpi_device_ids); +static const struct platform_device_id wilco_ec_id[] = { + { DRV_NAME, 0 }, + {} +}; +MODULE_DEVICE_TABLE(platform, wilco_ec_id); + static struct platform_driver wilco_ec_driver = { .driver = { .name = DRV_NAME, @@ -157,6 +164,7 @@ static struct platform_driver wilco_ec_driver = { }, .probe = wilco_ec_probe, .remove_new = wilco_ec_remove, + .id_table = wilco_ec_id, }; module_platform_driver(wilco_ec_driver); @@ -165,4 +173,3 @@ MODULE_AUTHOR("Nick Crews <ncrews@chromium.org>"); MODULE_AUTHOR("Duncan Laurie <dlaurie@chromium.org>"); MODULE_LICENSE("GPL v2"); MODULE_DESCRIPTION("ChromeOS Wilco Embedded Controller driver"); -MODULE_ALIAS("platform:" DRV_NAME); diff --git a/drivers/platform/chrome/wilco_ec/debugfs.c b/drivers/platform/chrome/wilco_ec/debugfs.c index 93c11f81ca45..983f2fa44ba5 100644 --- a/drivers/platform/chrome/wilco_ec/debugfs.c +++ b/drivers/platform/chrome/wilco_ec/debugfs.c @@ -10,6 +10,7 @@ #include <linux/ctype.h> #include <linux/debugfs.h> #include <linux/fs.h> +#include <linux/mod_devicetable.h> #include <linux/module.h> #include <linux/platform_data/wilco-ec.h> #include <linux/platform_device.h> @@ -265,17 +266,23 @@ static void wilco_ec_debugfs_remove(struct platform_device *pdev) debugfs_remove_recursive(debug_info->dir); } +static const struct platform_device_id wilco_ec_debugfs_id[] = { + { DRV_NAME, 0 }, + {} +}; +MODULE_DEVICE_TABLE(platform, wilco_ec_debugfs_id); + static struct platform_driver wilco_ec_debugfs_driver = { .driver = { .name = DRV_NAME, }, .probe = wilco_ec_debugfs_probe, .remove_new = wilco_ec_debugfs_remove, + .id_table = wilco_ec_debugfs_id, }; module_platform_driver(wilco_ec_debugfs_driver); -MODULE_ALIAS("platform:" DRV_NAME); MODULE_AUTHOR("Nick Crews <ncrews@chromium.org>"); MODULE_LICENSE("GPL v2"); MODULE_DESCRIPTION("Wilco EC debugfs driver"); diff --git a/drivers/platform/chrome/wilco_ec/event.c b/drivers/platform/chrome/wilco_ec/event.c index 13291fb4214e..49043c644572 100644 --- a/drivers/platform/chrome/wilco_ec/event.c +++ b/drivers/platform/chrome/wilco_ec/event.c @@ -575,4 +575,3 @@ module_exit(event_module_exit); MODULE_AUTHOR("Nick Crews <ncrews@chromium.org>"); MODULE_DESCRIPTION("Wilco EC ACPI event driver"); MODULE_LICENSE("GPL"); -MODULE_ALIAS("platform:" DRV_NAME); diff --git a/drivers/platform/chrome/wilco_ec/sysfs.c b/drivers/platform/chrome/wilco_ec/sysfs.c index 893c59dde32a..d44c43559621 100644 --- a/drivers/platform/chrome/wilco_ec/sysfs.c +++ b/drivers/platform/chrome/wilco_ec/sysfs.c @@ -192,7 +192,7 @@ static ssize_t usb_charge_show(struct device *dev, if (ret < 0) return ret; - return sprintf(buf, "%d\n", rs.val); + return sysfs_emit(buf, "%d\n", rs.val); } static ssize_t usb_charge_store(struct device *dev, diff --git a/drivers/platform/chrome/wilco_ec/telemetry.c b/drivers/platform/chrome/wilco_ec/telemetry.c index b7c616f3d179..21d4cbbb009a 100644 --- a/drivers/platform/chrome/wilco_ec/telemetry.c +++ b/drivers/platform/chrome/wilco_ec/telemetry.c @@ -30,6 +30,7 @@ #include <linux/cdev.h> #include <linux/device.h> #include <linux/fs.h> +#include <linux/mod_devicetable.h> #include <linux/module.h> #include <linux/platform_data/wilco-ec.h> #include <linux/platform_device.h> @@ -409,12 +410,19 @@ static void telem_device_remove(struct platform_device *pdev) put_device(&dev_data->dev); } +static const struct platform_device_id telem_id[] = { + { DRV_NAME, 0 }, + {} +}; +MODULE_DEVICE_TABLE(platform, telem_id); + static struct platform_driver telem_driver = { .probe = telem_device_probe, .remove_new = telem_device_remove, .driver = { .name = DRV_NAME, }, + .id_table = telem_id, }; static int __init telem_module_init(void) @@ -466,4 +474,3 @@ module_exit(telem_module_exit); MODULE_AUTHOR("Nick Crews <ncrews@chromium.org>"); MODULE_DESCRIPTION("Wilco EC telemetry driver"); MODULE_LICENSE("GPL"); -MODULE_ALIAS("platform:" DRV_NAME); diff --git a/drivers/regulator/core.c b/drivers/regulator/core.c index dabac9772741..2c33653ffdea 100644 --- a/drivers/regulator/core.c +++ b/drivers/regulator/core.c @@ -1911,19 +1911,24 @@ static struct regulator *create_regulator(struct regulator_dev *rdev, } } - if (err != -EEXIST) + if (err != -EEXIST) { regulator->debugfs = debugfs_create_dir(supply_name, rdev->debugfs); - if (IS_ERR(regulator->debugfs)) - rdev_dbg(rdev, "Failed to create debugfs directory\n"); + if (IS_ERR(regulator->debugfs)) { + rdev_dbg(rdev, "Failed to create debugfs directory\n"); + regulator->debugfs = NULL; + } + } - debugfs_create_u32("uA_load", 0444, regulator->debugfs, - ®ulator->uA_load); - debugfs_create_u32("min_uV", 0444, regulator->debugfs, - ®ulator->voltage[PM_SUSPEND_ON].min_uV); - debugfs_create_u32("max_uV", 0444, regulator->debugfs, - ®ulator->voltage[PM_SUSPEND_ON].max_uV); - debugfs_create_file("constraint_flags", 0444, regulator->debugfs, - regulator, &constraint_flags_fops); + if (regulator->debugfs) { + debugfs_create_u32("uA_load", 0444, regulator->debugfs, + ®ulator->uA_load); + debugfs_create_u32("min_uV", 0444, regulator->debugfs, + ®ulator->voltage[PM_SUSPEND_ON].min_uV); + debugfs_create_u32("max_uV", 0444, regulator->debugfs, + ®ulator->voltage[PM_SUSPEND_ON].max_uV); + debugfs_create_file("constraint_flags", 0444, regulator->debugfs, + regulator, &constraint_flags_fops); + } /* * Check now if the regulator is an always on regulator - if diff --git a/drivers/regulator/rtq2208-regulator.c b/drivers/regulator/rtq2208-regulator.c index 2d54844c4226..3079de41ef3f 100644 --- a/drivers/regulator/rtq2208-regulator.c +++ b/drivers/regulator/rtq2208-regulator.c @@ -48,7 +48,7 @@ /* Value */ #define RTQ2208_RAMP_VALUE_MIN_uV 500 -#define RTQ2208_RAMP_VALUE_MAX_uV 64000 +#define RTQ2208_RAMP_VALUE_MAX_uV 16000 #define RTQ2208_BUCK_MASK(uv_irq, ov_irq) (1 << ((uv_irq) % 8) | 1 << ((ov_irq) % 8)) @@ -142,12 +142,11 @@ static int rtq2208_set_ramp_delay(struct regulator_dev *rdev, int ramp_delay) * Because the relation of seleltion and value is like that * * seletion: value - * 000: 64mv - * 001: 32mv + * 010: 16mv * ... * 111: 0.5mv * - * For example, if I would like to select 64mv, the fls(ramp_delay) - 1 will be 0b111, + * For example, if I would like to select 16mv, the fls(ramp_delay) - 1 will be 0b010, * and I need to use 0b111 - sel to do the shifting */ diff --git a/drivers/reset/Kconfig b/drivers/reset/Kconfig index 85b27c42cf65..7112f5932609 100644 --- a/drivers/reset/Kconfig +++ b/drivers/reset/Kconfig @@ -103,9 +103,9 @@ config RESET_INTEL_GW config RESET_K210 bool "Reset controller driver for Canaan Kendryte K210 SoC" - depends on (SOC_CANAAN || COMPILE_TEST) && OF + depends on (SOC_CANAAN_K210 || COMPILE_TEST) && OF select MFD_SYSCON - default SOC_CANAAN + default SOC_CANAAN_K210 help Support for the Canaan Kendryte K210 RISC-V SoC reset controller. Say Y if you want to control reset signals provided by this diff --git a/drivers/s390/char/Makefile b/drivers/s390/char/Makefile index b0f6b3201636..81d6744e1861 100644 --- a/drivers/s390/char/Makefile +++ b/drivers/s390/char/Makefile @@ -32,7 +32,7 @@ obj-$(CONFIG_SCLP_VT220_TTY) += sclp_vt220.o obj-$(CONFIG_PCI) += sclp_pci.o -obj-$(subst m,y,$(CONFIG_ZCRYPT)) += sclp_ap.o +obj-$(subst m,y,$(CONFIG_AP)) += sclp_ap.o obj-$(CONFIG_VMLOGRDR) += vmlogrdr.o obj-$(CONFIG_VMCP) += vmcp.o diff --git a/drivers/s390/cio/chp.c b/drivers/s390/cio/chp.c index 675d7ed82356..a07bbecba61c 100644 --- a/drivers/s390/cio/chp.c +++ b/drivers/s390/cio/chp.c @@ -127,10 +127,9 @@ static int s390_vary_chpid(struct chp_id chpid, int on) /* * Channel measurement related functions */ -static ssize_t chp_measurement_chars_read(struct file *filp, - struct kobject *kobj, - struct bin_attribute *bin_attr, - char *buf, loff_t off, size_t count) +static ssize_t measurement_chars_read(struct file *filp, struct kobject *kobj, + struct bin_attribute *bin_attr, + char *buf, loff_t off, size_t count) { struct channel_path *chp; struct device *device; @@ -143,87 +142,79 @@ static ssize_t chp_measurement_chars_read(struct file *filp, return memory_read_from_buffer(buf, count, &off, &chp->cmg_chars, sizeof(chp->cmg_chars)); } +static BIN_ATTR_ADMIN_RO(measurement_chars, sizeof(struct cmg_chars)); -static const struct bin_attribute chp_measurement_chars_attr = { - .attr = { - .name = "measurement_chars", - .mode = S_IRUSR, - }, - .size = sizeof(struct cmg_chars), - .read = chp_measurement_chars_read, -}; - -static void chp_measurement_copy_block(struct cmg_entry *buf, - struct channel_subsystem *css, - struct chp_id chpid) -{ - void *area; - struct cmg_entry *entry, reference_buf; - int idx; - - if (chpid.id < 128) { - area = css->cub_addr1; - idx = chpid.id; - } else { - area = css->cub_addr2; - idx = chpid.id - 128; - } - entry = area + (idx * sizeof(struct cmg_entry)); - do { - memcpy(buf, entry, sizeof(*entry)); - memcpy(&reference_buf, entry, sizeof(*entry)); - } while (reference_buf.values[0] != buf->values[0]); -} - -static ssize_t chp_measurement_read(struct file *filp, struct kobject *kobj, - struct bin_attribute *bin_attr, - char *buf, loff_t off, size_t count) +static ssize_t chp_measurement_copy_block(void *buf, loff_t off, size_t count, + struct kobject *kobj, bool extended) { struct channel_path *chp; struct channel_subsystem *css; struct device *device; unsigned int size; + void *area, *entry; + int id, idx; device = kobj_to_dev(kobj); chp = to_channelpath(device); css = to_css(chp->dev.parent); + id = chp->chpid.id; - size = sizeof(struct cmg_entry); + if (extended) { + /* Check if extended measurement data is available. */ + if (!chp->extended) + return 0; + + size = sizeof(struct cmg_ext_entry); + area = css->ecub[id / CSS_ECUES_PER_PAGE]; + idx = id % CSS_ECUES_PER_PAGE; + } else { + size = sizeof(struct cmg_entry); + area = css->cub[id / CSS_CUES_PER_PAGE]; + idx = id % CSS_CUES_PER_PAGE; + } + entry = area + (idx * size); /* Only allow single reads. */ if (off || count < size) return 0; - chp_measurement_copy_block((struct cmg_entry *)buf, css, chp->chpid); - count = size; - return count; + + memcpy(buf, entry, size); + + return size; } -static const struct bin_attribute chp_measurement_attr = { - .attr = { - .name = "measurement", - .mode = S_IRUSR, - }, - .size = sizeof(struct cmg_entry), - .read = chp_measurement_read, +static ssize_t measurement_read(struct file *filp, struct kobject *kobj, + struct bin_attribute *bin_attr, + char *buf, loff_t off, size_t count) +{ + return chp_measurement_copy_block(buf, off, count, kobj, false); +} +static BIN_ATTR_ADMIN_RO(measurement, sizeof(struct cmg_entry)); + +static ssize_t ext_measurement_read(struct file *filp, struct kobject *kobj, + struct bin_attribute *bin_attr, + char *buf, loff_t off, size_t count) +{ + return chp_measurement_copy_block(buf, off, count, kobj, true); +} +static BIN_ATTR_ADMIN_RO(ext_measurement, sizeof(struct cmg_ext_entry)); + +static struct bin_attribute *measurement_attrs[] = { + &bin_attr_measurement_chars, + &bin_attr_measurement, + &bin_attr_ext_measurement, + NULL, }; +BIN_ATTRIBUTE_GROUPS(measurement); void chp_remove_cmg_attr(struct channel_path *chp) { - device_remove_bin_file(&chp->dev, &chp_measurement_chars_attr); - device_remove_bin_file(&chp->dev, &chp_measurement_attr); + device_remove_groups(&chp->dev, measurement_groups); } int chp_add_cmg_attr(struct channel_path *chp) { - int ret; - - ret = device_create_bin_file(&chp->dev, &chp_measurement_chars_attr); - if (ret) - return ret; - ret = device_create_bin_file(&chp->dev, &chp_measurement_attr); - if (ret) - device_remove_bin_file(&chp->dev, &chp_measurement_chars_attr); - return ret; + return device_add_groups(&chp->dev, measurement_groups); } /* @@ -401,6 +392,35 @@ static ssize_t chp_esc_show(struct device *dev, } static DEVICE_ATTR(esc, 0444, chp_esc_show, NULL); +static char apply_max_suffix(unsigned long *value, unsigned long base) +{ + static char suffixes[] = { 0, 'K', 'M', 'G', 'T' }; + int i; + + for (i = 0; i < ARRAY_SIZE(suffixes) - 1; i++) { + if (*value < base || *value % base != 0) + break; + *value /= base; + } + + return suffixes[i]; +} + +static ssize_t speed_bps_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct channel_path *chp = to_channelpath(dev); + unsigned long speed = chp->speed; + char suffix; + + suffix = apply_max_suffix(&speed, 1000); + + return suffix ? sysfs_emit(buf, "%lu%c\n", speed, suffix) : + sysfs_emit(buf, "%lu\n", speed); +} + +static DEVICE_ATTR_RO(speed_bps); + static ssize_t util_string_read(struct file *filp, struct kobject *kobj, struct bin_attribute *attr, char *buf, loff_t off, size_t count) @@ -432,6 +452,7 @@ static struct attribute *chp_attrs[] = { &dev_attr_chid.attr, &dev_attr_chid_external.attr, &dev_attr_esc.attr, + &dev_attr_speed_bps.attr, NULL, }; static struct attribute_group chp_attr_group = { diff --git a/drivers/s390/cio/chp.h b/drivers/s390/cio/chp.h index 7ee9eba0abcb..a15324a43aa3 100644 --- a/drivers/s390/cio/chp.h +++ b/drivers/s390/cio/chp.h @@ -51,6 +51,8 @@ struct channel_path { /* Channel-measurement related stuff: */ int cmg; int shared; + int extended; + unsigned long speed; struct cmg_chars cmg_chars; }; diff --git a/drivers/s390/cio/chsc.c b/drivers/s390/cio/chsc.c index 44ea76f9e1de..dcc1e1c34ca2 100644 --- a/drivers/s390/cio/chsc.c +++ b/drivers/s390/cio/chsc.c @@ -24,7 +24,6 @@ #include <asm/crw.h> #include <asm/isc.h> #include <asm/ebcdic.h> -#include <asm/ap.h> #include "css.h" #include "cio.h" @@ -40,6 +39,20 @@ static DEFINE_SPINLOCK(chsc_page_lock); #define SEI_VF_FLA 0xc0 /* VF flag for Full Link Address */ #define SEI_RS_CHPID 0x4 /* 4 in RS field indicates CHPID */ +static BLOCKING_NOTIFIER_HEAD(chsc_notifiers); + +int chsc_notifier_register(struct notifier_block *nb) +{ + return blocking_notifier_chain_register(&chsc_notifiers, nb); +} +EXPORT_SYMBOL(chsc_notifier_register); + +int chsc_notifier_unregister(struct notifier_block *nb) +{ + return blocking_notifier_chain_unregister(&chsc_notifiers, nb); +} +EXPORT_SYMBOL(chsc_notifier_unregister); + /** * chsc_error_from_response() - convert a chsc response to an error * @response: chsc response code @@ -581,7 +594,8 @@ static void chsc_process_sei_ap_cfg_chg(struct chsc_sei_nt0_area *sei_area) if (sei_area->rs != 5) return; - ap_bus_cfg_chg(); + blocking_notifier_call_chain(&chsc_notifiers, + CHSC_NOTIFY_AP_CFG, NULL); } static void chsc_process_sei_fces_event(struct chsc_sei_nt0_area *sei_area) @@ -857,22 +871,22 @@ int __chsc_do_secm(struct channel_subsystem *css, int enable) struct { struct chsc_header request; u32 operation_code : 2; - u32 : 30; + u32 : 1; + u32 e : 1; + u32 : 28; u32 key : 4; u32 : 28; - u32 zeroes1; - dma32_t cub_addr1; - u32 zeroes2; - dma32_t cub_addr2; - u32 reserved[13]; + dma64_t cub[CSS_NUM_CUB_PAGES]; + dma64_t ecub[CSS_NUM_ECUB_PAGES]; + u32 reserved[5]; struct chsc_header response; u32 status : 8; u32 : 4; u32 fmt : 4; u32 : 16; - } *secm_area; + } __packed *secm_area; unsigned long flags; - int ret, ccode; + int ret, ccode, i; spin_lock_irqsave(&chsc_page_lock, flags); memset(chsc_page, 0, PAGE_SIZE); @@ -881,8 +895,12 @@ int __chsc_do_secm(struct channel_subsystem *css, int enable) secm_area->request.code = 0x0016; secm_area->key = PAGE_DEFAULT_KEY >> 4; - secm_area->cub_addr1 = virt_to_dma32(css->cub_addr1); - secm_area->cub_addr2 = virt_to_dma32(css->cub_addr2); + secm_area->e = 1; + + for (i = 0; i < CSS_NUM_CUB_PAGES; i++) + secm_area->cub[i] = (__force dma64_t)virt_to_dma32(css->cub[i]); + for (i = 0; i < CSS_NUM_ECUB_PAGES; i++) + secm_area->ecub[i] = virt_to_dma64(css->ecub[i]); secm_area->operation_code = enable ? 0 : 1; @@ -908,19 +926,47 @@ out: return ret; } +static int cub_alloc(struct channel_subsystem *css) +{ + int i; + + for (i = 0; i < CSS_NUM_CUB_PAGES; i++) { + css->cub[i] = (void *)get_zeroed_page(GFP_KERNEL | GFP_DMA); + if (!css->cub[i]) + return -ENOMEM; + } + for (i = 0; i < CSS_NUM_ECUB_PAGES; i++) { + css->ecub[i] = (void *)get_zeroed_page(GFP_KERNEL); + if (!css->ecub[i]) + return -ENOMEM; + } + + return 0; +} + +static void cub_free(struct channel_subsystem *css) +{ + int i; + + for (i = 0; i < CSS_NUM_CUB_PAGES; i++) { + free_page((unsigned long)css->cub[i]); + css->cub[i] = NULL; + } + for (i = 0; i < CSS_NUM_ECUB_PAGES; i++) { + free_page((unsigned long)css->ecub[i]); + css->ecub[i] = NULL; + } +} + int chsc_secm(struct channel_subsystem *css, int enable) { int ret; if (enable && !css->cm_enabled) { - css->cub_addr1 = (void *)get_zeroed_page(GFP_KERNEL | GFP_DMA); - css->cub_addr2 = (void *)get_zeroed_page(GFP_KERNEL | GFP_DMA); - if (!css->cub_addr1 || !css->cub_addr2) { - free_page((unsigned long)css->cub_addr1); - free_page((unsigned long)css->cub_addr2); - return -ENOMEM; - } + ret = cub_alloc(css); + if (ret) + goto out; } ret = __chsc_do_secm(css, enable); if (!ret) { @@ -934,10 +980,11 @@ chsc_secm(struct channel_subsystem *css, int enable) } else chsc_remove_cmg_attr(css); } - if (!css->cm_enabled) { - free_page((unsigned long)css->cub_addr1); - free_page((unsigned long)css->cub_addr2); - } + +out: + if (!css->cm_enabled) + cub_free(css); + return ret; } @@ -1019,6 +1066,18 @@ chsc_initialize_cmg_chars(struct channel_path *chp, u8 cmcv, } } +static unsigned long scmc_get_speed(u32 s, u32 p) +{ + unsigned long speed = s; + + if (!p) + p = 8; + while (p--) + speed *= 10; + + return speed; +} + int chsc_get_channel_measurement_chars(struct channel_path *chp) { unsigned long flags; @@ -1035,18 +1094,23 @@ int chsc_get_channel_measurement_chars(struct channel_path *chp) u32 zeroes2; u32 not_valid : 1; u32 shared : 1; - u32 : 22; + u32 extended : 1; + u32 : 21; u32 chpid : 8; u32 cmcv : 5; - u32 : 11; + u32 : 7; + u32 cmgp : 4; u32 cmgq : 8; u32 cmg : 8; - u32 zeroes3; + u32 : 16; + u32 cmgs : 16; u32 data[NR_MEASUREMENT_CHARS]; } *scmc_area; chp->shared = -1; chp->cmg = -1; + chp->extended = 0; + chp->speed = 0; if (!css_chsc_characteristics.scmc || !css_chsc_characteristics.secm) return -EINVAL; @@ -1076,10 +1140,8 @@ int chsc_get_channel_measurement_chars(struct channel_path *chp) chp->cmg = scmc_area->cmg; chp->shared = scmc_area->shared; - if (chp->cmg != 2 && chp->cmg != 3) { - /* No cmg-dependent data. */ - goto out; - } + chp->extended = scmc_area->extended; + chp->speed = scmc_get_speed(scmc_area->cmgs, scmc_area->cmgp); chsc_initialize_cmg_chars(chp, scmc_area->cmcv, (struct cmg_chars *) &scmc_area->data); out: diff --git a/drivers/s390/cio/chsc.h b/drivers/s390/cio/chsc.h index 03602295f335..24cd65dbc5a7 100644 --- a/drivers/s390/cio/chsc.h +++ b/drivers/s390/cio/chsc.h @@ -22,6 +22,11 @@ struct cmg_entry { u32 values[NR_MEASUREMENT_ENTRIES]; }; +#define NR_EXT_MEASUREMENT_ENTRIES 16 +struct cmg_ext_entry { + u32 values[NR_EXT_MEASUREMENT_ENTRIES]; +}; + struct channel_path_desc_fmt1 { u8 flags; u8 lsn; diff --git a/drivers/s390/cio/css.c b/drivers/s390/cio/css.c index 1d68db1a3d4e..781f84901256 100644 --- a/drivers/s390/cio/css.c +++ b/drivers/s390/cio/css.c @@ -309,7 +309,7 @@ static ssize_t type_show(struct device *dev, struct device_attribute *attr, { struct subchannel *sch = to_subchannel(dev); - return sprintf(buf, "%01x\n", sch->st); + return sysfs_emit(buf, "%01x\n", sch->st); } static DEVICE_ATTR_RO(type); @@ -319,7 +319,7 @@ static ssize_t modalias_show(struct device *dev, struct device_attribute *attr, { struct subchannel *sch = to_subchannel(dev); - return sprintf(buf, "css:t%01X\n", sch->st); + return sysfs_emit(buf, "css:t%01X\n", sch->st); } static DEVICE_ATTR_RO(modalias); @@ -345,7 +345,7 @@ static ssize_t driver_override_show(struct device *dev, ssize_t len; device_lock(dev); - len = snprintf(buf, PAGE_SIZE, "%s\n", sch->driver_override); + len = sysfs_emit(buf, "%s\n", sch->driver_override); device_unlock(dev); return len; } @@ -396,8 +396,8 @@ static ssize_t pimpampom_show(struct device *dev, struct subchannel *sch = to_subchannel(dev); struct pmcw *pmcw = &sch->schib.pmcw; - return sprintf(buf, "%02x %02x %02x\n", - pmcw->pim, pmcw->pam, pmcw->pom); + return sysfs_emit(buf, "%02x %02x %02x\n", + pmcw->pim, pmcw->pam, pmcw->pom); } static DEVICE_ATTR_RO(pimpampom); @@ -881,7 +881,7 @@ static ssize_t real_cssid_show(struct device *dev, struct device_attribute *a, if (!css->id_valid) return -EINVAL; - return sprintf(buf, "%x\n", css->cssid); + return sysfs_emit(buf, "%x\n", css->cssid); } static DEVICE_ATTR_RO(real_cssid); @@ -904,7 +904,7 @@ static ssize_t cm_enable_show(struct device *dev, struct device_attribute *a, int ret; mutex_lock(&css->mutex); - ret = sprintf(buf, "%x\n", css->cm_enabled); + ret = sysfs_emit(buf, "%x\n", css->cm_enabled); mutex_unlock(&css->mutex); return ret; } diff --git a/drivers/s390/cio/css.h b/drivers/s390/cio/css.h index ea5550554297..c2b175592bb7 100644 --- a/drivers/s390/cio/css.h +++ b/drivers/s390/cio/css.h @@ -35,6 +35,15 @@ #define SNID_STATE3_SINGLE_PATH 0 /* + * Miscellaneous constants + */ + +#define CSS_NUM_CUB_PAGES 2 +#define CSS_CUES_PER_PAGE 128 +#define CSS_NUM_ECUB_PAGES 4 +#define CSS_ECUES_PER_PAGE 64 + +/* * Conditions used to specify which subchannels need evaluation */ enum css_eval_cond { @@ -122,8 +131,8 @@ struct channel_subsystem { struct mutex mutex; /* channel measurement related */ int cm_enabled; - void *cub_addr1; - void *cub_addr2; + void *cub[CSS_NUM_CUB_PAGES]; + void *ecub[CSS_NUM_ECUB_PAGES]; /* for orphaned ccw devices */ struct subchannel *pseudo_subchannel; }; diff --git a/drivers/s390/cio/trace.h b/drivers/s390/cio/trace.h index 86993de25345..a4c5c6736b31 100644 --- a/drivers/s390/cio/trace.h +++ b/drivers/s390/cio/trace.h @@ -50,7 +50,7 @@ DECLARE_EVENT_CLASS(s390_class_schib, __entry->devno = schib->pmcw.dev; __entry->schib = *schib; __entry->pmcw_ena = schib->pmcw.ena; - __entry->pmcw_st = schib->pmcw.ena; + __entry->pmcw_st = schib->pmcw.st; __entry->pmcw_dnv = schib->pmcw.dnv; __entry->pmcw_dev = schib->pmcw.dev; __entry->pmcw_lpm = schib->pmcw.lpm; diff --git a/drivers/s390/crypto/Makefile b/drivers/s390/crypto/Makefile index 0edacd101c12..bd94811fd9f1 100644 --- a/drivers/s390/crypto/Makefile +++ b/drivers/s390/crypto/Makefile @@ -4,7 +4,7 @@ # ap-objs := ap_bus.o ap_card.o ap_queue.o -obj-$(subst m,y,$(CONFIG_ZCRYPT)) += ap.o +obj-$(CONFIG_AP) += ap.o # zcrypt_api.o and zcrypt_msgtype*.o depend on ap.o zcrypt-objs := zcrypt_api.o zcrypt_card.o zcrypt_queue.o zcrypt-objs += zcrypt_msgtype6.o zcrypt_msgtype50.o diff --git a/drivers/s390/crypto/ap_bus.c b/drivers/s390/crypto/ap_bus.c index cce0bafd4c92..c20b45092079 100644 --- a/drivers/s390/crypto/ap_bus.c +++ b/drivers/s390/crypto/ap_bus.c @@ -39,13 +39,15 @@ #include <linux/ctype.h> #include <linux/module.h> #include <asm/uv.h> +#include <asm/chsc.h> #include "ap_bus.h" #include "ap_debug.h" -/* - * Module parameters; note though this file itself isn't modular. - */ +MODULE_AUTHOR("IBM Corporation"); +MODULE_DESCRIPTION("Adjunct Processor Bus driver"); +MODULE_LICENSE("GPL"); + int ap_domain_index = -1; /* Adjunct Processor Domain Index */ static DEFINE_SPINLOCK(ap_domain_lock); module_param_named(domain, ap_domain_index, int, 0440); @@ -90,8 +92,9 @@ static atomic64_t ap_bindings_complete_count = ATOMIC64_INIT(0); /* completion for APQN bindings complete */ static DECLARE_COMPLETION(ap_apqn_bindings_complete); -static struct ap_config_info *ap_qci_info; -static struct ap_config_info *ap_qci_info_old; +static struct ap_config_info qci[2]; +static struct ap_config_info *const ap_qci_info = &qci[0]; +static struct ap_config_info *const ap_qci_info_old = &qci[1]; /* * AP bus related debug feature things. @@ -203,9 +206,7 @@ static int ap_apft_available(void) */ static inline int ap_qact_available(void) { - if (ap_qci_info) - return ap_qci_info->qact; - return 0; + return ap_qci_info->qact; } /* @@ -215,9 +216,7 @@ static inline int ap_qact_available(void) */ int ap_sb_available(void) { - if (ap_qci_info) - return ap_qci_info->apsb; - return 0; + return ap_qci_info->apsb; } /* @@ -229,23 +228,6 @@ bool ap_is_se_guest(void) } EXPORT_SYMBOL(ap_is_se_guest); -/* - * ap_fetch_qci_info(): Fetch cryptographic config info - * - * Returns the ap configuration info fetched via PQAP(QCI). - * On success 0 is returned, on failure a negative errno - * is returned, e.g. if the PQAP(QCI) instruction is not - * available, the return value will be -EOPNOTSUPP. - */ -static inline int ap_fetch_qci_info(struct ap_config_info *info) -{ - if (!ap_qci_available()) - return -EOPNOTSUPP; - if (!info) - return -EINVAL; - return ap_qci(info); -} - /** * ap_init_qci_info(): Allocate and query qci config info. * Does also update the static variables ap_max_domain_id @@ -253,27 +235,12 @@ static inline int ap_fetch_qci_info(struct ap_config_info *info) */ static void __init ap_init_qci_info(void) { - if (!ap_qci_available()) { + if (!ap_qci_available() || + ap_qci(ap_qci_info)) { AP_DBF_INFO("%s QCI not supported\n", __func__); return; } - - ap_qci_info = kzalloc(sizeof(*ap_qci_info), GFP_KERNEL); - if (!ap_qci_info) - return; - ap_qci_info_old = kzalloc(sizeof(*ap_qci_info_old), GFP_KERNEL); - if (!ap_qci_info_old) { - kfree(ap_qci_info); - ap_qci_info = NULL; - return; - } - if (ap_fetch_qci_info(ap_qci_info) != 0) { - kfree(ap_qci_info); - kfree(ap_qci_info_old); - ap_qci_info = NULL; - ap_qci_info_old = NULL; - return; - } + memcpy(ap_qci_info_old, ap_qci_info, sizeof(*ap_qci_info)); AP_DBF_INFO("%s successful fetched initial qci info\n", __func__); if (ap_qci_info->apxa) { @@ -288,8 +255,6 @@ static void __init ap_init_qci_info(void) __func__, ap_max_domain_id); } } - - memcpy(ap_qci_info_old, ap_qci_info, sizeof(*ap_qci_info)); } /* @@ -312,7 +277,7 @@ static inline int ap_test_config_card_id(unsigned int id) { if (id > ap_max_adapter_id) return 0; - if (ap_qci_info) + if (ap_qci_info->flags) return ap_test_config(ap_qci_info->apm, id); return 1; } @@ -329,7 +294,7 @@ int ap_test_config_usage_domain(unsigned int domain) { if (domain > ap_max_domain_id) return 0; - if (ap_qci_info) + if (ap_qci_info->flags) return ap_test_config(ap_qci_info->aqm, domain); return 1; } @@ -1061,22 +1026,24 @@ EXPORT_SYMBOL(ap_bus_force_rescan); /* * A config change has happened, force an ap bus rescan. */ -void ap_bus_cfg_chg(void) +static int ap_bus_cfg_chg(struct notifier_block *nb, + unsigned long action, void *data) { + if (action != CHSC_NOTIFY_AP_CFG) + return NOTIFY_DONE; + pr_debug("%s config change, forcing bus rescan\n", __func__); ap_bus_force_rescan(); + + return NOTIFY_OK; } -/* - * hex2bitmap() - parse hex mask string and set bitmap. - * Valid strings are "0x012345678" with at least one valid hex number. - * Rest of the bitmap to the right is padded with 0. No spaces allowed - * within the string, the leading 0x may be omitted. - * Returns the bitmask with exactly the bits set as given by the hex - * string (both in big endian order). - */ -static int hex2bitmap(const char *str, unsigned long *bitmap, int bits) +static struct notifier_block ap_bus_nb = { + .notifier_call = ap_bus_cfg_chg, +}; + +int ap_hex2bitmap(const char *str, unsigned long *bitmap, int bits) { int i, n, b; @@ -1103,6 +1070,7 @@ static int hex2bitmap(const char *str, unsigned long *bitmap, int bits) return -EINVAL; return 0; } +EXPORT_SYMBOL(ap_hex2bitmap); /* * modify_bitmap() - parse bitmask argument and modify an existing @@ -1168,7 +1136,7 @@ static int ap_parse_bitmap_str(const char *str, unsigned long *bitmap, int bits, rc = modify_bitmap(str, newmap, bits); } else { memset(newmap, 0, size); - rc = hex2bitmap(str, newmap, bits); + rc = ap_hex2bitmap(str, newmap, bits); } return rc; } @@ -1234,7 +1202,7 @@ static BUS_ATTR_RW(ap_domain); static ssize_t ap_control_domain_mask_show(const struct bus_type *bus, char *buf) { - if (!ap_qci_info) /* QCI not supported */ + if (!ap_qci_info->flags) /* QCI not supported */ return sysfs_emit(buf, "not supported\n"); return sysfs_emit(buf, "0x%08x%08x%08x%08x%08x%08x%08x%08x\n", @@ -1248,7 +1216,7 @@ static BUS_ATTR_RO(ap_control_domain_mask); static ssize_t ap_usage_domain_mask_show(const struct bus_type *bus, char *buf) { - if (!ap_qci_info) /* QCI not supported */ + if (!ap_qci_info->flags) /* QCI not supported */ return sysfs_emit(buf, "not supported\n"); return sysfs_emit(buf, "0x%08x%08x%08x%08x%08x%08x%08x%08x\n", @@ -1262,7 +1230,7 @@ static BUS_ATTR_RO(ap_usage_domain_mask); static ssize_t ap_adapter_mask_show(const struct bus_type *bus, char *buf) { - if (!ap_qci_info) /* QCI not supported */ + if (!ap_qci_info->flags) /* QCI not supported */ return sysfs_emit(buf, "not supported\n"); return sysfs_emit(buf, "0x%08x%08x%08x%08x%08x%08x%08x%08x\n", @@ -1595,7 +1563,7 @@ static ssize_t features_show(const struct bus_type *bus, char *buf) { int n = 0; - if (!ap_qci_info) /* QCI not supported */ + if (!ap_qci_info->flags) /* QCI not supported */ return sysfs_emit(buf, "-\n"); if (ap_qci_info->apsc) @@ -2158,11 +2126,11 @@ static inline void ap_scan_adapter(int ap) */ static bool ap_get_configuration(void) { - if (!ap_qci_info) /* QCI not supported */ + if (!ap_qci_info->flags) /* QCI not supported */ return false; memcpy(ap_qci_info_old, ap_qci_info, sizeof(*ap_qci_info)); - ap_fetch_qci_info(ap_qci_info); + ap_qci(ap_qci_info); return memcmp(ap_qci_info, ap_qci_info_old, sizeof(struct ap_config_info)) != 0; @@ -2179,7 +2147,7 @@ static bool ap_config_has_new_aps(void) unsigned long m[BITS_TO_LONGS(AP_DEVICES)]; - if (!ap_qci_info) + if (!ap_qci_info->flags) return false; bitmap_andnot(m, (unsigned long *)ap_qci_info->apm, @@ -2200,7 +2168,7 @@ static bool ap_config_has_new_doms(void) { unsigned long m[BITS_TO_LONGS(AP_DOMAINS)]; - if (!ap_qci_info) + if (!ap_qci_info->flags) return false; bitmap_andnot(m, (unsigned long *)ap_qci_info->aqm, @@ -2310,7 +2278,82 @@ static void ap_scan_bus_wq_callback(struct work_struct *unused) } } -static int __init ap_debug_init(void) +static inline void __exit ap_async_exit(void) +{ + if (ap_thread_flag) + ap_poll_thread_stop(); + chsc_notifier_unregister(&ap_bus_nb); + cancel_work(&ap_scan_bus_work); + hrtimer_cancel(&ap_poll_timer); + timer_delete(&ap_scan_bus_timer); +} + +static inline int __init ap_async_init(void) +{ + int rc; + + /* Setup the AP bus rescan timer. */ + timer_setup(&ap_scan_bus_timer, ap_scan_bus_timer_callback, 0); + + /* + * Setup the high resolution poll timer. + * If we are running under z/VM adjust polling to z/VM polling rate. + */ + if (MACHINE_IS_VM) + poll_high_timeout = 1500000; + hrtimer_init(&ap_poll_timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS); + ap_poll_timer.function = ap_poll_timeout; + + queue_work(system_long_wq, &ap_scan_bus_work); + + rc = chsc_notifier_register(&ap_bus_nb); + if (rc) + goto out; + + /* Start the low priority AP bus poll thread. */ + if (!ap_thread_flag) + return 0; + + rc = ap_poll_thread_start(); + if (rc) + goto out_notifier; + + return 0; + +out_notifier: + chsc_notifier_unregister(&ap_bus_nb); +out: + cancel_work(&ap_scan_bus_work); + hrtimer_cancel(&ap_poll_timer); + timer_delete(&ap_scan_bus_timer); + return rc; +} + +static inline void ap_irq_exit(void) +{ + if (ap_irq_flag) + unregister_adapter_interrupt(&ap_airq); +} + +static inline int __init ap_irq_init(void) +{ + int rc; + + if (!ap_interrupts_available() || !ap_useirq) + return 0; + + rc = register_adapter_interrupt(&ap_airq); + ap_irq_flag = (rc == 0); + + return rc; +} + +static inline void ap_debug_exit(void) +{ + debug_unregister(ap_dbf_info); +} + +static inline int __init ap_debug_init(void) { ap_dbf_info = debug_register("ap", 2, 1, AP_DBF_MAX_SPRINTF_ARGS * sizeof(long)); @@ -2378,12 +2421,6 @@ static int __init ap_module_init(void) ap_domain_index = -1; } - /* enable interrupts if available */ - if (ap_interrupts_available() && ap_useirq) { - rc = register_adapter_interrupt(&ap_airq); - ap_irq_flag = (rc == 0); - } - /* Create /sys/bus/ap. */ rc = bus_register(&ap_bus_type); if (rc) @@ -2396,38 +2433,37 @@ static int __init ap_module_init(void) goto out_bus; ap_root_device->bus = &ap_bus_type; - /* Setup the AP bus rescan timer. */ - timer_setup(&ap_scan_bus_timer, ap_scan_bus_timer_callback, 0); - - /* - * Setup the high resolution poll timer. - * If we are running under z/VM adjust polling to z/VM polling rate. - */ - if (MACHINE_IS_VM) - poll_high_timeout = 1500000; - hrtimer_init(&ap_poll_timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS); - ap_poll_timer.function = ap_poll_timeout; - - /* Start the low priority AP bus poll thread. */ - if (ap_thread_flag) { - rc = ap_poll_thread_start(); - if (rc) - goto out_work; - } + /* enable interrupts if available */ + rc = ap_irq_init(); + if (rc) + goto out_device; - queue_work(system_long_wq, &ap_scan_bus_work); + /* Setup asynchronous work (timers, workqueue, etc). */ + rc = ap_async_init(); + if (rc) + goto out_irq; return 0; -out_work: - hrtimer_cancel(&ap_poll_timer); +out_irq: + ap_irq_exit(); +out_device: root_device_unregister(ap_root_device); out_bus: bus_unregister(&ap_bus_type); out: - if (ap_irq_flag) - unregister_adapter_interrupt(&ap_airq); - kfree(ap_qci_info); + ap_debug_exit(); return rc; } -device_initcall(ap_module_init); + +static void __exit ap_module_exit(void) +{ + ap_async_exit(); + ap_irq_exit(); + root_device_unregister(ap_root_device); + bus_unregister(&ap_bus_type); + ap_debug_exit(); +} + +module_init(ap_module_init); +module_exit(ap_module_exit); diff --git a/drivers/s390/crypto/ap_bus.h b/drivers/s390/crypto/ap_bus.h index 59c7ed49aa02..fdbc6fdfdf57 100644 --- a/drivers/s390/crypto/ap_bus.h +++ b/drivers/s390/crypto/ap_bus.h @@ -344,6 +344,28 @@ int ap_parse_mask_str(const char *str, struct mutex *lock); /* + * ap_hex2bitmap() - Convert a string containing a hexadecimal number (str) + * into a bitmap (bitmap) with bits set that correspond to the bits represented + * by the hex string. Input and output data is in big endian order. + * + * str - Input hex string of format "0x1234abcd". The leading "0x" is optional. + * At least one digit is required. Must be large enough to hold the number of + * bits represented by the bits parameter. + * + * bitmap - Pointer to a bitmap. Upon successful completion of this function, + * this bitmap will have bits set to match the value of str. If bitmap is longer + * than str, then the rightmost bits of bitmap are padded with zeros. Must be + * large enough to hold the number of bits represented by the bits parameter. + * + * bits - Length, in bits, of the bitmap represented by str. Must be a multiple + * of 8. + * + * Returns: 0 On success + * -EINVAL If str format is invalid or bits is not a multiple of 8. + */ +int ap_hex2bitmap(const char *str, unsigned long *bitmap, int bits); + +/* * Interface to wait for the AP bus to have done one initial ap bus * scan and all detected APQNs have been bound to device drivers. * If these both conditions are not fulfilled, this function blocks diff --git a/drivers/s390/crypto/ap_queue.c b/drivers/s390/crypto/ap_queue.c index 6e4e8d324a6d..1f647ffd6f4d 100644 --- a/drivers/s390/crypto/ap_queue.c +++ b/drivers/s390/crypto/ap_queue.c @@ -708,7 +708,7 @@ static ssize_t ap_functions_show(struct device *dev, static DEVICE_ATTR_RO(ap_functions); -#ifdef CONFIG_ZCRYPT_DEBUG +#ifdef CONFIG_AP_DEBUG static ssize_t states_show(struct device *dev, struct device_attribute *attr, char *buf) { @@ -820,7 +820,7 @@ static struct attribute *ap_queue_dev_attrs[] = { &dev_attr_config.attr, &dev_attr_chkstop.attr, &dev_attr_ap_functions.attr, -#ifdef CONFIG_ZCRYPT_DEBUG +#ifdef CONFIG_AP_DEBUG &dev_attr_states.attr, &dev_attr_last_err_rc.attr, #endif diff --git a/drivers/s390/crypto/vfio_ap_ops.c b/drivers/s390/crypto/vfio_ap_ops.c index fc169bc61593..9f76f2d7b66e 100644 --- a/drivers/s390/crypto/vfio_ap_ops.c +++ b/drivers/s390/crypto/vfio_ap_ops.c @@ -794,10 +794,11 @@ err_put_vdev: static void vfio_ap_mdev_link_queue(struct ap_matrix_mdev *matrix_mdev, struct vfio_ap_queue *q) { - if (q) { - q->matrix_mdev = matrix_mdev; - hash_add(matrix_mdev->qtable.queues, &q->mdev_qnode, q->apqn); - } + if (!q || vfio_ap_mdev_get_queue(matrix_mdev, q->apqn)) + return; + + q->matrix_mdev = matrix_mdev; + hash_add(matrix_mdev->qtable.queues, &q->mdev_qnode, q->apqn); } static void vfio_ap_mdev_link_apqn(struct ap_matrix_mdev *matrix_mdev, int apqn) @@ -1118,20 +1119,29 @@ static void vfio_ap_mdev_unlink_adapter(struct ap_matrix_mdev *matrix_mdev, } } -static void vfio_ap_mdev_hot_unplug_adapter(struct ap_matrix_mdev *matrix_mdev, - unsigned long apid) +static void vfio_ap_mdev_hot_unplug_adapters(struct ap_matrix_mdev *matrix_mdev, + unsigned long *apids) { struct vfio_ap_queue *q, *tmpq; struct list_head qlist; + unsigned long apid; + bool apcb_update = false; INIT_LIST_HEAD(&qlist); - vfio_ap_mdev_unlink_adapter(matrix_mdev, apid, &qlist); - if (test_bit_inv(apid, matrix_mdev->shadow_apcb.apm)) { - clear_bit_inv(apid, matrix_mdev->shadow_apcb.apm); - vfio_ap_mdev_update_guest_apcb(matrix_mdev); + for_each_set_bit_inv(apid, apids, AP_DEVICES) { + vfio_ap_mdev_unlink_adapter(matrix_mdev, apid, &qlist); + + if (test_bit_inv(apid, matrix_mdev->shadow_apcb.apm)) { + clear_bit_inv(apid, matrix_mdev->shadow_apcb.apm); + apcb_update = true; + } } + /* Only update apcb if needed to avoid impacting guest */ + if (apcb_update) + vfio_ap_mdev_update_guest_apcb(matrix_mdev); + vfio_ap_mdev_reset_qlist(&qlist); list_for_each_entry_safe(q, tmpq, &qlist, reset_qnode) { @@ -1140,6 +1150,16 @@ static void vfio_ap_mdev_hot_unplug_adapter(struct ap_matrix_mdev *matrix_mdev, } } +static void vfio_ap_mdev_hot_unplug_adapter(struct ap_matrix_mdev *matrix_mdev, + unsigned long apid) +{ + DECLARE_BITMAP(apids, AP_DEVICES); + + bitmap_zero(apids, AP_DEVICES); + set_bit_inv(apid, apids); + vfio_ap_mdev_hot_unplug_adapters(matrix_mdev, apids); +} + /** * unassign_adapter_store - parses the APID from @buf and clears the * corresponding bit in the mediated matrix device's APM @@ -1300,20 +1320,29 @@ static void vfio_ap_mdev_unlink_domain(struct ap_matrix_mdev *matrix_mdev, } } -static void vfio_ap_mdev_hot_unplug_domain(struct ap_matrix_mdev *matrix_mdev, - unsigned long apqi) +static void vfio_ap_mdev_hot_unplug_domains(struct ap_matrix_mdev *matrix_mdev, + unsigned long *apqis) { struct vfio_ap_queue *q, *tmpq; struct list_head qlist; + unsigned long apqi; + bool apcb_update = false; INIT_LIST_HEAD(&qlist); - vfio_ap_mdev_unlink_domain(matrix_mdev, apqi, &qlist); - if (test_bit_inv(apqi, matrix_mdev->shadow_apcb.aqm)) { - clear_bit_inv(apqi, matrix_mdev->shadow_apcb.aqm); - vfio_ap_mdev_update_guest_apcb(matrix_mdev); + for_each_set_bit_inv(apqi, apqis, AP_DOMAINS) { + vfio_ap_mdev_unlink_domain(matrix_mdev, apqi, &qlist); + + if (test_bit_inv(apqi, matrix_mdev->shadow_apcb.aqm)) { + clear_bit_inv(apqi, matrix_mdev->shadow_apcb.aqm); + apcb_update = true; + } } + /* Only update apcb if needed to avoid impacting guest */ + if (apcb_update) + vfio_ap_mdev_update_guest_apcb(matrix_mdev); + vfio_ap_mdev_reset_qlist(&qlist); list_for_each_entry_safe(q, tmpq, &qlist, reset_qnode) { @@ -1322,6 +1351,16 @@ static void vfio_ap_mdev_hot_unplug_domain(struct ap_matrix_mdev *matrix_mdev, } } +static void vfio_ap_mdev_hot_unplug_domain(struct ap_matrix_mdev *matrix_mdev, + unsigned long apqi) +{ + DECLARE_BITMAP(apqis, AP_DOMAINS); + + bitmap_zero(apqis, AP_DEVICES); + set_bit_inv(apqi, apqis); + vfio_ap_mdev_hot_unplug_domains(matrix_mdev, apqis); +} + /** * unassign_domain_store - parses the APQI from @buf and clears the * corresponding bit in the mediated matrix device's AQM @@ -1570,6 +1609,158 @@ static ssize_t guest_matrix_show(struct device *dev, } static DEVICE_ATTR_RO(guest_matrix); +static ssize_t write_ap_bitmap(unsigned long *bitmap, char *buf, int offset, char sep) +{ + return sysfs_emit_at(buf, offset, "0x%016lx%016lx%016lx%016lx%c", + bitmap[0], bitmap[1], bitmap[2], bitmap[3], sep); +} + +static ssize_t ap_config_show(struct device *dev, struct device_attribute *attr, + char *buf) +{ + struct ap_matrix_mdev *matrix_mdev = dev_get_drvdata(dev); + int idx = 0; + + idx += write_ap_bitmap(matrix_mdev->matrix.apm, buf, idx, ','); + idx += write_ap_bitmap(matrix_mdev->matrix.aqm, buf, idx, ','); + idx += write_ap_bitmap(matrix_mdev->matrix.adm, buf, idx, '\n'); + + return idx; +} + +/* Number of characters needed for a complete hex mask representing the bits in .. */ +#define AP_DEVICES_STRLEN (AP_DEVICES / 4 + 3) +#define AP_DOMAINS_STRLEN (AP_DOMAINS / 4 + 3) +#define AP_CONFIG_STRLEN (AP_DEVICES_STRLEN + 2 * AP_DOMAINS_STRLEN) + +static int parse_bitmap(char **strbufptr, unsigned long *bitmap, int nbits) +{ + char *curmask; + + curmask = strsep(strbufptr, ",\n"); + if (!curmask) + return -EINVAL; + + bitmap_clear(bitmap, 0, nbits); + return ap_hex2bitmap(curmask, bitmap, nbits); +} + +static int ap_matrix_overflow_check(struct ap_matrix_mdev *matrix_mdev) +{ + unsigned long bit; + + for_each_set_bit_inv(bit, matrix_mdev->matrix.apm, AP_DEVICES) { + if (bit > matrix_mdev->matrix.apm_max) + return -ENODEV; + } + + for_each_set_bit_inv(bit, matrix_mdev->matrix.aqm, AP_DOMAINS) { + if (bit > matrix_mdev->matrix.aqm_max) + return -ENODEV; + } + + for_each_set_bit_inv(bit, matrix_mdev->matrix.adm, AP_DOMAINS) { + if (bit > matrix_mdev->matrix.adm_max) + return -ENODEV; + } + + return 0; +} + +static void ap_matrix_copy(struct ap_matrix *dst, struct ap_matrix *src) +{ + /* This check works around false positive gcc -Wstringop-overread */ + if (!src) + return; + + bitmap_copy(dst->apm, src->apm, AP_DEVICES); + bitmap_copy(dst->aqm, src->aqm, AP_DOMAINS); + bitmap_copy(dst->adm, src->adm, AP_DOMAINS); +} + +static ssize_t ap_config_store(struct device *dev, struct device_attribute *attr, + const char *buf, size_t count) +{ + struct ap_matrix_mdev *matrix_mdev = dev_get_drvdata(dev); + struct ap_matrix m_new, m_old, m_added, m_removed; + DECLARE_BITMAP(apm_filtered, AP_DEVICES); + unsigned long newbit; + char *newbuf, *rest; + int rc = count; + bool do_update; + + newbuf = kstrndup(buf, AP_CONFIG_STRLEN, GFP_KERNEL); + if (!newbuf) + return -ENOMEM; + rest = newbuf; + + mutex_lock(&ap_perms_mutex); + get_update_locks_for_mdev(matrix_mdev); + + /* Save old state */ + ap_matrix_copy(&m_old, &matrix_mdev->matrix); + if (parse_bitmap(&rest, m_new.apm, AP_DEVICES) || + parse_bitmap(&rest, m_new.aqm, AP_DOMAINS) || + parse_bitmap(&rest, m_new.adm, AP_DOMAINS)) { + rc = -EINVAL; + goto out; + } + + bitmap_andnot(m_removed.apm, m_old.apm, m_new.apm, AP_DEVICES); + bitmap_andnot(m_removed.aqm, m_old.aqm, m_new.aqm, AP_DOMAINS); + bitmap_andnot(m_added.apm, m_new.apm, m_old.apm, AP_DEVICES); + bitmap_andnot(m_added.aqm, m_new.aqm, m_old.aqm, AP_DOMAINS); + + /* Need new bitmaps in matrix_mdev for validation */ + ap_matrix_copy(&matrix_mdev->matrix, &m_new); + + /* Ensure new state is valid, else undo new state */ + rc = vfio_ap_mdev_validate_masks(matrix_mdev); + if (rc) { + ap_matrix_copy(&matrix_mdev->matrix, &m_old); + goto out; + } + rc = ap_matrix_overflow_check(matrix_mdev); + if (rc) { + ap_matrix_copy(&matrix_mdev->matrix, &m_old); + goto out; + } + rc = count; + + /* Need old bitmaps in matrix_mdev for unplug/unlink */ + ap_matrix_copy(&matrix_mdev->matrix, &m_old); + + /* Unlink removed adapters/domains */ + vfio_ap_mdev_hot_unplug_adapters(matrix_mdev, m_removed.apm); + vfio_ap_mdev_hot_unplug_domains(matrix_mdev, m_removed.aqm); + + /* Need new bitmaps in matrix_mdev for linking new adapters/domains */ + ap_matrix_copy(&matrix_mdev->matrix, &m_new); + + /* Link newly added adapters */ + for_each_set_bit_inv(newbit, m_added.apm, AP_DEVICES) + vfio_ap_mdev_link_adapter(matrix_mdev, newbit); + + for_each_set_bit_inv(newbit, m_added.aqm, AP_DOMAINS) + vfio_ap_mdev_link_domain(matrix_mdev, newbit); + + /* filter resources not bound to vfio-ap */ + do_update = vfio_ap_mdev_filter_matrix(matrix_mdev, apm_filtered); + do_update |= vfio_ap_mdev_filter_cdoms(matrix_mdev); + + /* Apply changes to shadow apbc if things changed */ + if (do_update) { + vfio_ap_mdev_update_guest_apcb(matrix_mdev); + reset_queues_for_apids(matrix_mdev, apm_filtered); + } +out: + release_update_locks_for_mdev(matrix_mdev); + mutex_unlock(&ap_perms_mutex); + kfree(newbuf); + return rc; +} +static DEVICE_ATTR_RW(ap_config); + static struct attribute *vfio_ap_mdev_attrs[] = { &dev_attr_assign_adapter.attr, &dev_attr_unassign_adapter.attr, @@ -1577,6 +1768,7 @@ static struct attribute *vfio_ap_mdev_attrs[] = { &dev_attr_unassign_domain.attr, &dev_attr_assign_control_domain.attr, &dev_attr_unassign_control_domain.attr, + &dev_attr_ap_config.attr, &dev_attr_control_domains.attr, &dev_attr_matrix.attr, &dev_attr_guest_matrix.attr, diff --git a/drivers/s390/crypto/vfio_ap_private.h b/drivers/s390/crypto/vfio_ap_private.h index 98d37aa27044..437a161c8659 100644 --- a/drivers/s390/crypto/vfio_ap_private.h +++ b/drivers/s390/crypto/vfio_ap_private.h @@ -75,11 +75,11 @@ extern struct ap_matrix_dev *matrix_dev; */ struct ap_matrix { unsigned long apm_max; - DECLARE_BITMAP(apm, 256); + DECLARE_BITMAP(apm, AP_DEVICES); unsigned long aqm_max; - DECLARE_BITMAP(aqm, 256); + DECLARE_BITMAP(aqm, AP_DOMAINS); unsigned long adm_max; - DECLARE_BITMAP(adm, 256); + DECLARE_BITMAP(adm, AP_DOMAINS); }; /** diff --git a/drivers/scsi/mpi3mr/mpi3mr_transport.c b/drivers/scsi/mpi3mr/mpi3mr_transport.c index d32ad46318cb..231b4d0df85f 100644 --- a/drivers/scsi/mpi3mr/mpi3mr_transport.c +++ b/drivers/scsi/mpi3mr/mpi3mr_transport.c @@ -209,17 +209,13 @@ static int mpi3mr_report_manufacture(struct mpi3mr_ioc *mrioc, goto out; } - strscpy(edev->vendor_id, manufacture_reply->vendor_id, - SAS_EXPANDER_VENDOR_ID_LEN); - strscpy(edev->product_id, manufacture_reply->product_id, - SAS_EXPANDER_PRODUCT_ID_LEN); - strscpy(edev->product_rev, manufacture_reply->product_rev, - SAS_EXPANDER_PRODUCT_REV_LEN); + memtostr(edev->vendor_id, manufacture_reply->vendor_id); + memtostr(edev->product_id, manufacture_reply->product_id); + memtostr(edev->product_rev, manufacture_reply->product_rev); edev->level = manufacture_reply->sas_format & 1; if (edev->level) { - strscpy(edev->component_vendor_id, - manufacture_reply->component_vendor_id, - SAS_EXPANDER_COMPONENT_VENDOR_ID_LEN); + memtostr(edev->component_vendor_id, + manufacture_reply->component_vendor_id); tmp = (u8 *)&manufacture_reply->component_id; edev->component_id = tmp[0] << 8 | tmp[1]; edev->component_revision_id = diff --git a/drivers/scsi/qla2xxx/qla_mr.c b/drivers/scsi/qla2xxx/qla_mr.c index 083f94e43fba..82a7e21ddc83 100644 --- a/drivers/scsi/qla2xxx/qla_mr.c +++ b/drivers/scsi/qla2xxx/qla_mr.c @@ -1909,10 +1909,8 @@ qlafx00_fx_disc(scsi_qla_host_t *vha, fc_port_t *fcport, uint16_t fx_type) if (fx_type == FXDISC_GET_CONFIG_INFO) { struct config_info_data *pinfo = (struct config_info_data *) fdisc->u.fxiocb.rsp_addr; - strscpy(vha->hw->model_number, pinfo->model_num, - ARRAY_SIZE(vha->hw->model_number)); - strscpy(vha->hw->model_desc, pinfo->model_description, - ARRAY_SIZE(vha->hw->model_desc)); + memtostr(vha->hw->model_number, pinfo->model_num); + memtostr(vha->hw->model_desc, pinfo->model_description); memcpy(&vha->hw->mr.symbolic_name, pinfo->symbolic_name, sizeof(vha->hw->mr.symbolic_name)); memcpy(&vha->hw->mr.serial_num, pinfo->serial_num, diff --git a/drivers/scsi/scsi_lib.c b/drivers/scsi/scsi_lib.c index 5b3230ef51fe..967b6d62bb37 100644 --- a/drivers/scsi/scsi_lib.c +++ b/drivers/scsi/scsi_lib.c @@ -1869,7 +1869,6 @@ out_put_budget: case BLK_STS_OK: break; case BLK_STS_RESOURCE: - case BLK_STS_ZONE_RESOURCE: if (scsi_device_blocked(sdev)) ret = BLK_STS_DEV_RESOURCE; break; diff --git a/drivers/scsi/sd.c b/drivers/scsi/sd.c index 65cdc8b77e35..64c5129044b3 100644 --- a/drivers/scsi/sd.c +++ b/drivers/scsi/sd.c @@ -1260,12 +1260,6 @@ static blk_status_t sd_setup_read_write_cmnd(struct scsi_cmnd *cmd) } } - if (req_op(rq) == REQ_OP_ZONE_APPEND) { - ret = sd_zbc_prepare_zone_append(cmd, &lba, nr_blocks); - if (ret) - goto fail; - } - fua = rq->cmd_flags & REQ_FUA ? 0x8 : 0; dix = scsi_prot_sg_count(cmd); dif = scsi_host_dif_capable(cmd->device->host, sdkp->protection_type); @@ -1348,7 +1342,6 @@ static blk_status_t sd_init_command(struct scsi_cmnd *cmd) return sd_setup_flush_cmnd(cmd); case REQ_OP_READ: case REQ_OP_WRITE: - case REQ_OP_ZONE_APPEND: return sd_setup_read_write_cmnd(cmd); case REQ_OP_ZONE_RESET: return sd_zbc_setup_zone_mgmt_cmnd(cmd, ZO_RESET_WRITE_POINTER, @@ -3981,7 +3974,6 @@ static void scsi_disk_release(struct device *dev) struct scsi_disk *sdkp = to_scsi_disk(dev); ida_free(&sd_index_ida, sdkp->index); - sd_zbc_free_zone_info(sdkp); put_device(&sdkp->device->sdev_gendev); free_opal_dev(sdkp->opal_dev); diff --git a/drivers/scsi/sd.h b/drivers/scsi/sd.h index 5c4285a582b2..49dd600bfa48 100644 --- a/drivers/scsi/sd.h +++ b/drivers/scsi/sd.h @@ -104,12 +104,6 @@ struct scsi_disk { * between zone starting LBAs is constant. */ u32 zone_starting_lba_gran; - u32 *zones_wp_offset; - spinlock_t zones_wp_offset_lock; - u32 *rev_wp_offset; - struct mutex rev_mutex; - struct work_struct zone_wp_offset_work; - char *zone_wp_update_buf; #endif atomic_t openers; sector_t capacity; /* size in logical blocks */ @@ -245,7 +239,6 @@ static inline int sd_is_zoned(struct scsi_disk *sdkp) #ifdef CONFIG_BLK_DEV_ZONED -void sd_zbc_free_zone_info(struct scsi_disk *sdkp); int sd_zbc_read_zones(struct scsi_disk *sdkp, u8 buf[SD_BUF_SIZE]); int sd_zbc_revalidate_zones(struct scsi_disk *sdkp); blk_status_t sd_zbc_setup_zone_mgmt_cmnd(struct scsi_cmnd *cmd, @@ -255,13 +248,8 @@ unsigned int sd_zbc_complete(struct scsi_cmnd *cmd, unsigned int good_bytes, int sd_zbc_report_zones(struct gendisk *disk, sector_t sector, unsigned int nr_zones, report_zones_cb cb, void *data); -blk_status_t sd_zbc_prepare_zone_append(struct scsi_cmnd *cmd, sector_t *lba, - unsigned int nr_blocks); - #else /* CONFIG_BLK_DEV_ZONED */ -static inline void sd_zbc_free_zone_info(struct scsi_disk *sdkp) {} - static inline int sd_zbc_read_zones(struct scsi_disk *sdkp, u8 buf[SD_BUF_SIZE]) { return 0; @@ -285,13 +273,6 @@ static inline unsigned int sd_zbc_complete(struct scsi_cmnd *cmd, return good_bytes; } -static inline blk_status_t sd_zbc_prepare_zone_append(struct scsi_cmnd *cmd, - sector_t *lba, - unsigned int nr_blocks) -{ - return BLK_STS_TARGET; -} - #define sd_zbc_report_zones NULL #endif /* CONFIG_BLK_DEV_ZONED */ diff --git a/drivers/scsi/sd_zbc.c b/drivers/scsi/sd_zbc.c index 26af5ab7d7c1..806036e48abe 100644 --- a/drivers/scsi/sd_zbc.c +++ b/drivers/scsi/sd_zbc.c @@ -23,36 +23,6 @@ #define CREATE_TRACE_POINTS #include "sd_trace.h" -/** - * sd_zbc_get_zone_wp_offset - Get zone write pointer offset. - * @zone: Zone for which to return the write pointer offset. - * - * Return: offset of the write pointer from the start of the zone. - */ -static unsigned int sd_zbc_get_zone_wp_offset(struct blk_zone *zone) -{ - if (zone->type == ZBC_ZONE_TYPE_CONV) - return 0; - - switch (zone->cond) { - case BLK_ZONE_COND_IMP_OPEN: - case BLK_ZONE_COND_EXP_OPEN: - case BLK_ZONE_COND_CLOSED: - return zone->wp - zone->start; - case BLK_ZONE_COND_FULL: - return zone->len; - case BLK_ZONE_COND_EMPTY: - case BLK_ZONE_COND_OFFLINE: - case BLK_ZONE_COND_READONLY: - default: - /* - * Offline and read-only zones do not have a valid - * write pointer. Use 0 as for an empty zone. - */ - return 0; - } -} - /* Whether or not a SCSI zone descriptor describes a gap zone. */ static bool sd_zbc_is_gap_zone(const u8 buf[64]) { @@ -121,9 +91,6 @@ static int sd_zbc_parse_report(struct scsi_disk *sdkp, const u8 buf[64], if (ret) return ret; - if (sdkp->rev_wp_offset) - sdkp->rev_wp_offset[idx] = sd_zbc_get_zone_wp_offset(&zone); - return 0; } @@ -347,123 +314,6 @@ static blk_status_t sd_zbc_cmnd_checks(struct scsi_cmnd *cmd) return BLK_STS_OK; } -#define SD_ZBC_INVALID_WP_OFST (~0u) -#define SD_ZBC_UPDATING_WP_OFST (SD_ZBC_INVALID_WP_OFST - 1) - -static int sd_zbc_update_wp_offset_cb(struct blk_zone *zone, unsigned int idx, - void *data) -{ - struct scsi_disk *sdkp = data; - - lockdep_assert_held(&sdkp->zones_wp_offset_lock); - - sdkp->zones_wp_offset[idx] = sd_zbc_get_zone_wp_offset(zone); - - return 0; -} - -/* - * An attempt to append a zone triggered an invalid write pointer error. - * Reread the write pointer of the zone(s) in which the append failed. - */ -static void sd_zbc_update_wp_offset_workfn(struct work_struct *work) -{ - struct scsi_disk *sdkp; - unsigned long flags; - sector_t zno; - int ret; - - sdkp = container_of(work, struct scsi_disk, zone_wp_offset_work); - - spin_lock_irqsave(&sdkp->zones_wp_offset_lock, flags); - for (zno = 0; zno < sdkp->zone_info.nr_zones; zno++) { - if (sdkp->zones_wp_offset[zno] != SD_ZBC_UPDATING_WP_OFST) - continue; - - spin_unlock_irqrestore(&sdkp->zones_wp_offset_lock, flags); - ret = sd_zbc_do_report_zones(sdkp, sdkp->zone_wp_update_buf, - SD_BUF_SIZE, - zno * sdkp->zone_info.zone_blocks, true); - spin_lock_irqsave(&sdkp->zones_wp_offset_lock, flags); - if (!ret) - sd_zbc_parse_report(sdkp, sdkp->zone_wp_update_buf + 64, - zno, sd_zbc_update_wp_offset_cb, - sdkp); - } - spin_unlock_irqrestore(&sdkp->zones_wp_offset_lock, flags); - - scsi_device_put(sdkp->device); -} - -/** - * sd_zbc_prepare_zone_append() - Prepare an emulated ZONE_APPEND command. - * @cmd: the command to setup - * @lba: the LBA to patch - * @nr_blocks: the number of LBAs to be written - * - * Called from sd_setup_read_write_cmnd() for REQ_OP_ZONE_APPEND. - * @sd_zbc_prepare_zone_append() handles the necessary zone wrote locking and - * patching of the lba for an emulated ZONE_APPEND command. - * - * In case the cached write pointer offset is %SD_ZBC_INVALID_WP_OFST it will - * schedule a REPORT ZONES command and return BLK_STS_IOERR. - */ -blk_status_t sd_zbc_prepare_zone_append(struct scsi_cmnd *cmd, sector_t *lba, - unsigned int nr_blocks) -{ - struct request *rq = scsi_cmd_to_rq(cmd); - struct scsi_disk *sdkp = scsi_disk(rq->q->disk); - unsigned int wp_offset, zno = blk_rq_zone_no(rq); - unsigned long flags; - blk_status_t ret; - - ret = sd_zbc_cmnd_checks(cmd); - if (ret != BLK_STS_OK) - return ret; - - if (!blk_rq_zone_is_seq(rq)) - return BLK_STS_IOERR; - - /* Unlock of the write lock will happen in sd_zbc_complete() */ - if (!blk_req_zone_write_trylock(rq)) - return BLK_STS_ZONE_RESOURCE; - - spin_lock_irqsave(&sdkp->zones_wp_offset_lock, flags); - wp_offset = sdkp->zones_wp_offset[zno]; - switch (wp_offset) { - case SD_ZBC_INVALID_WP_OFST: - /* - * We are about to schedule work to update a zone write pointer - * offset, which will cause the zone append command to be - * requeued. So make sure that the scsi device does not go away - * while the work is being processed. - */ - if (scsi_device_get(sdkp->device)) { - ret = BLK_STS_IOERR; - break; - } - sdkp->zones_wp_offset[zno] = SD_ZBC_UPDATING_WP_OFST; - schedule_work(&sdkp->zone_wp_offset_work); - fallthrough; - case SD_ZBC_UPDATING_WP_OFST: - ret = BLK_STS_DEV_RESOURCE; - break; - default: - wp_offset = sectors_to_logical(sdkp->device, wp_offset); - if (wp_offset + nr_blocks > sdkp->zone_info.zone_blocks) { - ret = BLK_STS_IOERR; - break; - } - - trace_scsi_prepare_zone_append(cmd, *lba, wp_offset); - *lba += wp_offset; - } - spin_unlock_irqrestore(&sdkp->zones_wp_offset_lock, flags); - if (ret) - blk_req_zone_write_unlock(rq); - return ret; -} - /** * sd_zbc_setup_zone_mgmt_cmnd - Prepare a zone ZBC_OUT command. The operations * can be RESET WRITE POINTER, OPEN, CLOSE or FINISH. @@ -504,96 +354,6 @@ blk_status_t sd_zbc_setup_zone_mgmt_cmnd(struct scsi_cmnd *cmd, return BLK_STS_OK; } -static bool sd_zbc_need_zone_wp_update(struct request *rq) -{ - switch (req_op(rq)) { - case REQ_OP_ZONE_APPEND: - case REQ_OP_ZONE_FINISH: - case REQ_OP_ZONE_RESET: - case REQ_OP_ZONE_RESET_ALL: - return true; - case REQ_OP_WRITE: - case REQ_OP_WRITE_ZEROES: - return blk_rq_zone_is_seq(rq); - default: - return false; - } -} - -/** - * sd_zbc_zone_wp_update - Update cached zone write pointer upon cmd completion - * @cmd: Completed command - * @good_bytes: Command reply bytes - * - * Called from sd_zbc_complete() to handle the update of the cached zone write - * pointer value in case an update is needed. - */ -static unsigned int sd_zbc_zone_wp_update(struct scsi_cmnd *cmd, - unsigned int good_bytes) -{ - int result = cmd->result; - struct request *rq = scsi_cmd_to_rq(cmd); - struct scsi_disk *sdkp = scsi_disk(rq->q->disk); - unsigned int zno = blk_rq_zone_no(rq); - enum req_op op = req_op(rq); - unsigned long flags; - - /* - * If we got an error for a command that needs updating the write - * pointer offset cache, we must mark the zone wp offset entry as - * invalid to force an update from disk the next time a zone append - * command is issued. - */ - spin_lock_irqsave(&sdkp->zones_wp_offset_lock, flags); - - if (result && op != REQ_OP_ZONE_RESET_ALL) { - if (op == REQ_OP_ZONE_APPEND) { - /* Force complete completion (no retry) */ - good_bytes = 0; - scsi_set_resid(cmd, blk_rq_bytes(rq)); - } - - /* - * Force an update of the zone write pointer offset on - * the next zone append access. - */ - if (sdkp->zones_wp_offset[zno] != SD_ZBC_UPDATING_WP_OFST) - sdkp->zones_wp_offset[zno] = SD_ZBC_INVALID_WP_OFST; - goto unlock_wp_offset; - } - - switch (op) { - case REQ_OP_ZONE_APPEND: - trace_scsi_zone_wp_update(cmd, rq->__sector, - sdkp->zones_wp_offset[zno], good_bytes); - rq->__sector += sdkp->zones_wp_offset[zno]; - fallthrough; - case REQ_OP_WRITE_ZEROES: - case REQ_OP_WRITE: - if (sdkp->zones_wp_offset[zno] < sd_zbc_zone_sectors(sdkp)) - sdkp->zones_wp_offset[zno] += - good_bytes >> SECTOR_SHIFT; - break; - case REQ_OP_ZONE_RESET: - sdkp->zones_wp_offset[zno] = 0; - break; - case REQ_OP_ZONE_FINISH: - sdkp->zones_wp_offset[zno] = sd_zbc_zone_sectors(sdkp); - break; - case REQ_OP_ZONE_RESET_ALL: - memset(sdkp->zones_wp_offset, 0, - sdkp->zone_info.nr_zones * sizeof(unsigned int)); - break; - default: - break; - } - -unlock_wp_offset: - spin_unlock_irqrestore(&sdkp->zones_wp_offset_lock, flags); - - return good_bytes; -} - /** * sd_zbc_complete - ZBC command post processing. * @cmd: Completed command @@ -619,11 +379,7 @@ unsigned int sd_zbc_complete(struct scsi_cmnd *cmd, unsigned int good_bytes, * so be quiet about the error. */ rq->rq_flags |= RQF_QUIET; - } else if (sd_zbc_need_zone_wp_update(rq)) - good_bytes = sd_zbc_zone_wp_update(cmd, good_bytes); - - if (req_op(rq) == REQ_OP_ZONE_APPEND) - blk_req_zone_write_unlock(rq); + } return good_bytes; } @@ -780,46 +536,6 @@ static void sd_zbc_print_zones(struct scsi_disk *sdkp) sdkp->zone_info.zone_blocks); } -static int sd_zbc_init_disk(struct scsi_disk *sdkp) -{ - sdkp->zones_wp_offset = NULL; - spin_lock_init(&sdkp->zones_wp_offset_lock); - sdkp->rev_wp_offset = NULL; - mutex_init(&sdkp->rev_mutex); - INIT_WORK(&sdkp->zone_wp_offset_work, sd_zbc_update_wp_offset_workfn); - sdkp->zone_wp_update_buf = kzalloc(SD_BUF_SIZE, GFP_KERNEL); - if (!sdkp->zone_wp_update_buf) - return -ENOMEM; - - return 0; -} - -void sd_zbc_free_zone_info(struct scsi_disk *sdkp) -{ - if (!sdkp->zone_wp_update_buf) - return; - - /* Serialize against revalidate zones */ - mutex_lock(&sdkp->rev_mutex); - - kvfree(sdkp->zones_wp_offset); - sdkp->zones_wp_offset = NULL; - kfree(sdkp->zone_wp_update_buf); - sdkp->zone_wp_update_buf = NULL; - - sdkp->early_zone_info = (struct zoned_disk_info){ }; - sdkp->zone_info = (struct zoned_disk_info){ }; - - mutex_unlock(&sdkp->rev_mutex); -} - -static void sd_zbc_revalidate_zones_cb(struct gendisk *disk) -{ - struct scsi_disk *sdkp = scsi_disk(disk); - - swap(sdkp->zones_wp_offset, sdkp->rev_wp_offset); -} - /* * Call blk_revalidate_disk_zones() if any of the zoned disk properties have * changed that make it necessary to call that function. Called by @@ -831,18 +547,8 @@ int sd_zbc_revalidate_zones(struct scsi_disk *sdkp) struct request_queue *q = disk->queue; u32 zone_blocks = sdkp->early_zone_info.zone_blocks; unsigned int nr_zones = sdkp->early_zone_info.nr_zones; - int ret = 0; unsigned int flags; - - /* - * For all zoned disks, initialize zone append emulation data if not - * already done. - */ - if (sd_is_zoned(sdkp) && !sdkp->zone_wp_update_buf) { - ret = sd_zbc_init_disk(sdkp); - if (ret) - return ret; - } + int ret; /* * There is nothing to do for regular disks, including host-aware disks @@ -851,50 +557,32 @@ int sd_zbc_revalidate_zones(struct scsi_disk *sdkp) if (!blk_queue_is_zoned(q)) return 0; - /* - * Make sure revalidate zones are serialized to ensure exclusive - * updates of the scsi disk data. - */ - mutex_lock(&sdkp->rev_mutex); - if (sdkp->zone_info.zone_blocks == zone_blocks && sdkp->zone_info.nr_zones == nr_zones && disk->nr_zones == nr_zones) - goto unlock; + return 0; - flags = memalloc_noio_save(); sdkp->zone_info.zone_blocks = zone_blocks; sdkp->zone_info.nr_zones = nr_zones; - sdkp->rev_wp_offset = kvcalloc(nr_zones, sizeof(u32), GFP_KERNEL); - if (!sdkp->rev_wp_offset) { - ret = -ENOMEM; - memalloc_noio_restore(flags); - goto unlock; - } blk_queue_chunk_sectors(q, logical_to_sectors(sdkp->device, zone_blocks)); - blk_queue_max_zone_append_sectors(q, - q->limits.max_segments << PAGE_SECTORS_SHIFT); - ret = blk_revalidate_disk_zones(disk, sd_zbc_revalidate_zones_cb); + /* Enable block layer zone append emulation */ + blk_queue_max_zone_append_sectors(q, 0); + flags = memalloc_noio_save(); + ret = blk_revalidate_disk_zones(disk); memalloc_noio_restore(flags); - kvfree(sdkp->rev_wp_offset); - sdkp->rev_wp_offset = NULL; - if (ret) { sdkp->zone_info = (struct zoned_disk_info){ }; sdkp->capacity = 0; - goto unlock; + return ret; } sd_zbc_print_zones(sdkp); -unlock: - mutex_unlock(&sdkp->rev_mutex); - - return ret; + return 0; } /** @@ -917,10 +605,8 @@ int sd_zbc_read_zones(struct scsi_disk *sdkp, u8 buf[SD_BUF_SIZE]) if (!sd_is_zoned(sdkp)) { /* * Device managed or normal SCSI disk, no special handling - * required. Nevertheless, free the disk zone information in - * case the device type changed. + * required. */ - sd_zbc_free_zone_info(sdkp); return 0; } @@ -941,7 +627,6 @@ int sd_zbc_read_zones(struct scsi_disk *sdkp, u8 buf[SD_BUF_SIZE]) /* The drive satisfies the kernel restrictions: set it up */ blk_queue_flag_set(QUEUE_FLAG_ZONE_RESETALL, q); - blk_queue_required_elevator_features(q, ELEVATOR_F_ZBD_SEQ_WRITE); if (sdkp->zones_max_open == U32_MAX) disk_set_max_open_zones(disk, 0); else diff --git a/drivers/soc/Makefile b/drivers/soc/Makefile index ba8f5b5460e1..fb2bd31387d0 100644 --- a/drivers/soc/Makefile +++ b/drivers/soc/Makefile @@ -7,7 +7,7 @@ obj-y += apple/ obj-y += aspeed/ obj-$(CONFIG_ARCH_AT91) += atmel/ obj-y += bcm/ -obj-$(CONFIG_SOC_CANAAN) += canaan/ +obj-$(CONFIG_ARCH_CANAAN) += canaan/ obj-$(CONFIG_ARCH_DOVE) += dove/ obj-$(CONFIG_MACH_DOVE) += dove/ obj-y += fsl/ diff --git a/drivers/soc/canaan/Kconfig b/drivers/soc/canaan/Kconfig index 43ced2bf8444..3121d351fea6 100644 --- a/drivers/soc/canaan/Kconfig +++ b/drivers/soc/canaan/Kconfig @@ -2,9 +2,9 @@ config SOC_K210_SYSCTL bool "Canaan Kendryte K210 SoC system controller" - depends on RISCV && SOC_CANAAN && OF + depends on RISCV && SOC_CANAAN_K210 && OF depends on COMMON_CLK_K210 - default SOC_CANAAN + default SOC_CANAAN_K210 select PM select MFD_SYSCON help diff --git a/drivers/soc/hisilicon/Kconfig b/drivers/soc/hisilicon/Kconfig index 0ab688af308f..4b0a099b28cc 100644 --- a/drivers/soc/hisilicon/Kconfig +++ b/drivers/soc/hisilicon/Kconfig @@ -6,7 +6,7 @@ menu "Hisilicon SoC drivers" config KUNPENG_HCCS tristate "HCCS driver on Kunpeng SoC" depends on ACPI - depends on MAILBOX + depends on PCC depends on ARM64 || COMPILE_TEST help The Huawei Cache Coherence System (HCCS) is a multi-chip diff --git a/drivers/soc/hisilicon/kunpeng_hccs.c b/drivers/soc/hisilicon/kunpeng_hccs.c index 9ff70b38e5e9..e882a61636ec 100644 --- a/drivers/soc/hisilicon/kunpeng_hccs.c +++ b/drivers/soc/hisilicon/kunpeng_hccs.c @@ -556,6 +556,12 @@ static int hccs_get_all_port_attr(struct hccs_dev *hdev, start_id = rsp_head.next_id; } + if (left_buf_len != 0) { + dev_err(hdev->dev, "failed to get the expected port number(%u) attribute.\n", + size); + return -EINVAL; + } + return 0; } diff --git a/drivers/soc/mediatek/mtk-cmdq-helper.c b/drivers/soc/mediatek/mtk-cmdq-helper.c index b0cd071c4719..046522664dc1 100644 --- a/drivers/soc/mediatek/mtk-cmdq-helper.c +++ b/drivers/soc/mediatek/mtk-cmdq-helper.c @@ -12,9 +12,12 @@ #define CMDQ_WRITE_ENABLE_MASK BIT(0) #define CMDQ_POLL_ENABLE_MASK BIT(0) +/* dedicate the last GPR_R15 to assign the register address to be poll */ +#define CMDQ_POLL_ADDR_GPR (15) #define CMDQ_EOC_IRQ_EN BIT(0) #define CMDQ_REG_TYPE 1 -#define CMDQ_JUMP_RELATIVE 1 +#define CMDQ_JUMP_RELATIVE 0 +#define CMDQ_JUMP_ABSOLUTE 1 struct cmdq_instruction { union { @@ -55,7 +58,7 @@ int cmdq_dev_get_client_reg(struct device *dev, "mediatek,gce-client-reg", 3, idx, &spec); if (err < 0) { - dev_err(dev, + dev_warn(dev, "error %d can't parse gce-client-reg property (%d)", err, idx); @@ -105,22 +108,16 @@ void cmdq_mbox_destroy(struct cmdq_client *client) } EXPORT_SYMBOL(cmdq_mbox_destroy); -struct cmdq_pkt *cmdq_pkt_create(struct cmdq_client *client, size_t size) +int cmdq_pkt_create(struct cmdq_client *client, struct cmdq_pkt *pkt, size_t size) { - struct cmdq_pkt *pkt; struct device *dev; dma_addr_t dma_addr; - pkt = kzalloc(sizeof(*pkt), GFP_KERNEL); - if (!pkt) - return ERR_PTR(-ENOMEM); pkt->va_base = kzalloc(size, GFP_KERNEL); - if (!pkt->va_base) { - kfree(pkt); - return ERR_PTR(-ENOMEM); - } + if (!pkt->va_base) + return -ENOMEM; + pkt->buf_size = size; - pkt->cl = (void *)client; dev = client->chan->mbox->dev; dma_addr = dma_map_single(dev, pkt->va_base, pkt->buf_size, @@ -128,24 +125,20 @@ struct cmdq_pkt *cmdq_pkt_create(struct cmdq_client *client, size_t size) if (dma_mapping_error(dev, dma_addr)) { dev_err(dev, "dma map failed, size=%u\n", (u32)(u64)size); kfree(pkt->va_base); - kfree(pkt); - return ERR_PTR(-ENOMEM); + return -ENOMEM; } pkt->pa_base = dma_addr; - return pkt; + return 0; } EXPORT_SYMBOL(cmdq_pkt_create); -void cmdq_pkt_destroy(struct cmdq_pkt *pkt) +void cmdq_pkt_destroy(struct cmdq_client *client, struct cmdq_pkt *pkt) { - struct cmdq_client *client = (struct cmdq_client *)pkt->cl; - dma_unmap_single(client->chan->mbox->dev, pkt->pa_base, pkt->buf_size, DMA_TO_DEVICE); kfree(pkt->va_base); - kfree(pkt); } EXPORT_SYMBOL(cmdq_pkt_destroy); @@ -299,6 +292,32 @@ int cmdq_pkt_write_s_mask_value(struct cmdq_pkt *pkt, u8 high_addr_reg_idx, } EXPORT_SYMBOL(cmdq_pkt_write_s_mask_value); +int cmdq_pkt_mem_move(struct cmdq_pkt *pkt, dma_addr_t src_addr, dma_addr_t dst_addr) +{ + const u16 high_addr_reg_idx = CMDQ_THR_SPR_IDX0; + const u16 value_reg_idx = CMDQ_THR_SPR_IDX1; + int ret; + + /* read the value of src_addr into high_addr_reg_idx */ + ret = cmdq_pkt_assign(pkt, high_addr_reg_idx, CMDQ_ADDR_HIGH(src_addr)); + if (ret < 0) + return ret; + ret = cmdq_pkt_read_s(pkt, high_addr_reg_idx, CMDQ_ADDR_LOW(src_addr), value_reg_idx); + if (ret < 0) + return ret; + + /* write the value of value_reg_idx into dst_addr */ + ret = cmdq_pkt_assign(pkt, high_addr_reg_idx, CMDQ_ADDR_HIGH(dst_addr)); + if (ret < 0) + return ret; + ret = cmdq_pkt_write_s(pkt, high_addr_reg_idx, CMDQ_ADDR_LOW(dst_addr), value_reg_idx); + if (ret < 0) + return ret; + + return 0; +} +EXPORT_SYMBOL(cmdq_pkt_mem_move); + int cmdq_pkt_wfe(struct cmdq_pkt *pkt, u16 event, bool clear) { struct cmdq_instruction inst = { {0} }; @@ -315,6 +334,21 @@ int cmdq_pkt_wfe(struct cmdq_pkt *pkt, u16 event, bool clear) } EXPORT_SYMBOL(cmdq_pkt_wfe); +int cmdq_pkt_acquire_event(struct cmdq_pkt *pkt, u16 event) +{ + struct cmdq_instruction inst = {}; + + if (event >= CMDQ_MAX_EVENT) + return -EINVAL; + + inst.op = CMDQ_CODE_WFE; + inst.value = CMDQ_WFE_UPDATE | CMDQ_WFE_UPDATE_VALUE | CMDQ_WFE_WAIT; + inst.event = event; + + return cmdq_pkt_append_command(pkt, inst); +} +EXPORT_SYMBOL(cmdq_pkt_acquire_event); + int cmdq_pkt_clear_event(struct cmdq_pkt *pkt, u16 event) { struct cmdq_instruction inst = { {0} }; @@ -380,6 +414,53 @@ int cmdq_pkt_poll_mask(struct cmdq_pkt *pkt, u8 subsys, } EXPORT_SYMBOL(cmdq_pkt_poll_mask); +int cmdq_pkt_poll_addr(struct cmdq_pkt *pkt, dma_addr_t addr, u32 value, u32 mask) +{ + struct cmdq_instruction inst = { {0} }; + u8 use_mask = 0; + int ret; + + /* + * Append an MASK instruction to set the mask for following POLL instruction + * which enables use_mask bit. + */ + if (mask != GENMASK(31, 0)) { + inst.op = CMDQ_CODE_MASK; + inst.mask = ~mask; + ret = cmdq_pkt_append_command(pkt, inst); + if (ret < 0) + return ret; + use_mask = CMDQ_POLL_ENABLE_MASK; + } + + /* + * POLL is an legacy operation in GCE and it does not support SPR and CMDQ_CODE_LOGIC, + * so it can not use cmdq_pkt_assign to keep polling register address to SPR. + * If user wants to poll a register address which doesn't have a subsys id, + * user needs to use GPR and CMDQ_CODE_MASK to move polling register address to GPR. + */ + inst.op = CMDQ_CODE_MASK; + inst.dst_t = CMDQ_REG_TYPE; + inst.sop = CMDQ_POLL_ADDR_GPR; + inst.value = addr; + ret = cmdq_pkt_append_command(pkt, inst); + if (ret < 0) + return ret; + + /* Append POLL instruction to poll the register address assign to GPR previously. */ + inst.op = CMDQ_CODE_POLL; + inst.dst_t = CMDQ_REG_TYPE; + inst.sop = CMDQ_POLL_ADDR_GPR; + inst.offset = use_mask; + inst.value = value; + ret = cmdq_pkt_append_command(pkt, inst); + if (ret < 0) + return ret; + + return 0; +} +EXPORT_SYMBOL(cmdq_pkt_poll_addr); + int cmdq_pkt_assign(struct cmdq_pkt *pkt, u16 reg_idx, u32 value) { struct cmdq_instruction inst = {}; @@ -392,17 +473,36 @@ int cmdq_pkt_assign(struct cmdq_pkt *pkt, u16 reg_idx, u32 value) } EXPORT_SYMBOL(cmdq_pkt_assign); -int cmdq_pkt_jump(struct cmdq_pkt *pkt, dma_addr_t addr) +int cmdq_pkt_jump_abs(struct cmdq_pkt *pkt, dma_addr_t addr, u8 shift_pa) { struct cmdq_instruction inst = {}; inst.op = CMDQ_CODE_JUMP; - inst.offset = CMDQ_JUMP_RELATIVE; - inst.value = addr >> - cmdq_get_shift_pa(((struct cmdq_client *)pkt->cl)->chan); + inst.offset = CMDQ_JUMP_ABSOLUTE; + inst.value = addr >> shift_pa; + return cmdq_pkt_append_command(pkt, inst); +} +EXPORT_SYMBOL(cmdq_pkt_jump_abs); + +int cmdq_pkt_jump_rel(struct cmdq_pkt *pkt, s32 offset, u8 shift_pa) +{ + struct cmdq_instruction inst = { {0} }; + + inst.op = CMDQ_CODE_JUMP; + inst.value = (u32)offset >> shift_pa; + return cmdq_pkt_append_command(pkt, inst); +} +EXPORT_SYMBOL(cmdq_pkt_jump_rel); + +int cmdq_pkt_eoc(struct cmdq_pkt *pkt) +{ + struct cmdq_instruction inst = { {0} }; + + inst.op = CMDQ_CODE_EOC; + inst.value = CMDQ_EOC_IRQ_EN; return cmdq_pkt_append_command(pkt, inst); } -EXPORT_SYMBOL(cmdq_pkt_jump); +EXPORT_SYMBOL(cmdq_pkt_eoc); int cmdq_pkt_finalize(struct cmdq_pkt *pkt) { @@ -426,19 +526,4 @@ int cmdq_pkt_finalize(struct cmdq_pkt *pkt) } EXPORT_SYMBOL(cmdq_pkt_finalize); -int cmdq_pkt_flush_async(struct cmdq_pkt *pkt) -{ - int err; - struct cmdq_client *client = (struct cmdq_client *)pkt->cl; - - err = mbox_send_message(client->chan, pkt); - if (err < 0) - return err; - /* We can send next packet immediately, so just call txdone. */ - mbox_client_txdone(client->chan, 0); - - return 0; -} -EXPORT_SYMBOL(cmdq_pkt_flush_async); - MODULE_LICENSE("GPL v2"); diff --git a/drivers/soc/mediatek/mtk-mutex.c b/drivers/soc/mediatek/mtk-mutex.c index 73c256d3950b..b5af1fb5847e 100644 --- a/drivers/soc/mediatek/mtk-mutex.c +++ b/drivers/soc/mediatek/mtk-mutex.c @@ -496,6 +496,39 @@ static const unsigned int mt8188_mutex_mod[DDP_COMPONENT_ID_MAX] = { [DDP_COMPONENT_MERGE5] = MT8188_MUTEX_MOD_DISP1_VPP_MERGE4, }; +static const unsigned int mt8188_mdp_mutex_table_mod[MUTEX_MOD_IDX_MAX] = { + [MUTEX_MOD_IDX_MDP_RDMA0] = MT8195_MUTEX_MOD_MDP_RDMA0, + [MUTEX_MOD_IDX_MDP_RDMA2] = MT8195_MUTEX_MOD_MDP_RDMA2, + [MUTEX_MOD_IDX_MDP_RDMA3] = MT8195_MUTEX_MOD_MDP_RDMA3, + [MUTEX_MOD_IDX_MDP_FG0] = MT8195_MUTEX_MOD_MDP_FG0, + [MUTEX_MOD_IDX_MDP_FG2] = MT8195_MUTEX_MOD_MDP_FG2, + [MUTEX_MOD_IDX_MDP_FG3] = MT8195_MUTEX_MOD_MDP_FG3, + [MUTEX_MOD_IDX_MDP_HDR0] = MT8195_MUTEX_MOD_MDP_HDR0, + [MUTEX_MOD_IDX_MDP_HDR2] = MT8195_MUTEX_MOD_MDP_HDR2, + [MUTEX_MOD_IDX_MDP_HDR3] = MT8195_MUTEX_MOD_MDP_HDR3, + [MUTEX_MOD_IDX_MDP_AAL0] = MT8195_MUTEX_MOD_MDP_AAL0, + [MUTEX_MOD_IDX_MDP_AAL2] = MT8195_MUTEX_MOD_MDP_AAL2, + [MUTEX_MOD_IDX_MDP_AAL3] = MT8195_MUTEX_MOD_MDP_AAL3, + [MUTEX_MOD_IDX_MDP_RSZ0] = MT8195_MUTEX_MOD_MDP_RSZ0, + [MUTEX_MOD_IDX_MDP_RSZ2] = MT8195_MUTEX_MOD_MDP_RSZ2, + [MUTEX_MOD_IDX_MDP_RSZ3] = MT8195_MUTEX_MOD_MDP_RSZ3, + [MUTEX_MOD_IDX_MDP_MERGE2] = MT8195_MUTEX_MOD_MDP_MERGE2, + [MUTEX_MOD_IDX_MDP_MERGE3] = MT8195_MUTEX_MOD_MDP_MERGE3, + [MUTEX_MOD_IDX_MDP_TDSHP0] = MT8195_MUTEX_MOD_MDP_TDSHP0, + [MUTEX_MOD_IDX_MDP_TDSHP2] = MT8195_MUTEX_MOD_MDP_TDSHP2, + [MUTEX_MOD_IDX_MDP_TDSHP3] = MT8195_MUTEX_MOD_MDP_TDSHP3, + [MUTEX_MOD_IDX_MDP_COLOR0] = MT8195_MUTEX_MOD_MDP_COLOR0, + [MUTEX_MOD_IDX_MDP_COLOR2] = MT8195_MUTEX_MOD_MDP_COLOR2, + [MUTEX_MOD_IDX_MDP_COLOR3] = MT8195_MUTEX_MOD_MDP_COLOR3, + [MUTEX_MOD_IDX_MDP_OVL0] = MT8195_MUTEX_MOD_MDP_OVL0, + [MUTEX_MOD_IDX_MDP_PAD0] = MT8195_MUTEX_MOD_MDP_PAD0, + [MUTEX_MOD_IDX_MDP_PAD2] = MT8195_MUTEX_MOD_MDP_PAD2, + [MUTEX_MOD_IDX_MDP_PAD3] = MT8195_MUTEX_MOD_MDP_PAD3, + [MUTEX_MOD_IDX_MDP_WROT0] = MT8195_MUTEX_MOD_MDP_WROT0, + [MUTEX_MOD_IDX_MDP_WROT2] = MT8195_MUTEX_MOD_MDP_WROT2, + [MUTEX_MOD_IDX_MDP_WROT3] = MT8195_MUTEX_MOD_MDP_WROT3, +}; + static const unsigned int mt8192_mutex_mod[DDP_COMPONENT_ID_MAX] = { [DDP_COMPONENT_AAL0] = MT8192_MUTEX_MOD_DISP_AAL0, [DDP_COMPONENT_CCORR] = MT8192_MUTEX_MOD_DISP_CCORR0, @@ -735,6 +768,13 @@ static const struct mtk_mutex_data mt8188_mutex_driver_data = { .mutex_sof_reg = MT8183_MUTEX0_SOF0, }; +static const struct mtk_mutex_data mt8188_vpp_mutex_driver_data = { + .mutex_sof = mt8188_mutex_sof, + .mutex_mod_reg = MT8183_MUTEX0_MOD0, + .mutex_sof_reg = MT8183_MUTEX0_SOF0, + .mutex_table_mod = mt8188_mdp_mutex_table_mod, +}; + static const struct mtk_mutex_data mt8192_mutex_driver_data = { .mutex_mod = mt8192_mutex_mod, .mutex_sof = mt8183_mutex_sof, @@ -1089,6 +1129,7 @@ static const struct of_device_id mutex_driver_dt_match[] = { { .compatible = "mediatek,mt8186-disp-mutex", .data = &mt8186_mutex_driver_data }, { .compatible = "mediatek,mt8186-mdp3-mutex", .data = &mt8186_mdp_mutex_driver_data }, { .compatible = "mediatek,mt8188-disp-mutex", .data = &mt8188_mutex_driver_data }, + { .compatible = "mediatek,mt8188-vpp-mutex", .data = &mt8188_vpp_mutex_driver_data }, { .compatible = "mediatek,mt8192-disp-mutex", .data = &mt8192_mutex_driver_data }, { .compatible = "mediatek,mt8195-disp-mutex", .data = &mt8195_mutex_driver_data }, { .compatible = "mediatek,mt8195-vpp-mutex", .data = &mt8195_vpp_mutex_driver_data }, diff --git a/drivers/soc/mediatek/mtk-socinfo.c b/drivers/soc/mediatek/mtk-socinfo.c index 42572e8c1520..74672a9d6d13 100644 --- a/drivers/soc/mediatek/mtk-socinfo.c +++ b/drivers/soc/mediatek/mtk-socinfo.c @@ -48,14 +48,15 @@ static struct socinfo_data socinfo_data_table[] = { MTK_SOCINFO_ENTRY("MT8183", "MT8183V/AZA", "Kompanio 500", 0x00010043, 0x00000940), MTK_SOCINFO_ENTRY("MT8186", "MT8186GV/AZA", "Kompanio 520", 0x81861001, CELL_NOT_USED), MTK_SOCINFO_ENTRY("MT8186T", "MT8186TV/AZA", "Kompanio 528", 0x81862001, CELL_NOT_USED), - MTK_SOCINFO_ENTRY("MT8188", "MT8188GV/AZA", "Kompanio 830", 0x81880000, 0x00000010), - MTK_SOCINFO_ENTRY("MT8188", "MT8188GV/HZA", "Kompanio 830", 0x81880000, 0x00000011), + MTK_SOCINFO_ENTRY("MT8188", "MT8188GV/AZA", "Kompanio 838", 0x81880000, 0x00000010), + MTK_SOCINFO_ENTRY("MT8188", "MT8188GV/HZA", "Kompanio 838", 0x81880000, 0x00000011), MTK_SOCINFO_ENTRY("MT8192", "MT8192V/AZA", "Kompanio 820", 0x00001100, 0x00040080), MTK_SOCINFO_ENTRY("MT8192T", "MT8192V/ATZA", "Kompanio 828", 0x00000100, 0x000400C0), MTK_SOCINFO_ENTRY("MT8195", "MT8195GV/EZA", "Kompanio 1200", 0x81950300, CELL_NOT_USED), MTK_SOCINFO_ENTRY("MT8195", "MT8195GV/EHZA", "Kompanio 1200", 0x81950304, CELL_NOT_USED), MTK_SOCINFO_ENTRY("MT8195", "MT8195TV/EZA", "Kompanio 1380", 0x81950400, CELL_NOT_USED), MTK_SOCINFO_ENTRY("MT8195", "MT8195TV/EHZA", "Kompanio 1380", 0x81950404, CELL_NOT_USED), + MTK_SOCINFO_ENTRY("MT8395", "MT8395AV/ZA", "Genio 1200", 0x83950100, CELL_NOT_USED), }; static int mtk_socinfo_create_socinfo_node(struct mtk_socinfo *mtk_socinfop) @@ -144,7 +145,14 @@ static int mtk_socinfo_get_socinfo_data(struct mtk_socinfo *mtk_socinfop) } } - return match_socinfo_index >= 0 ? match_socinfo_index : -ENOENT; + if (match_socinfo_index < 0) { + dev_warn(mtk_socinfop->dev, + "Unknown MediaTek SoC with ID 0x%08x 0x%08x\n", + cell_data[0], cell_data[1]); + return -ENOENT; + } + + return match_socinfo_index; } static int mtk_socinfo_probe(struct platform_device *pdev) diff --git a/drivers/soc/qcom/cmd-db.c b/drivers/soc/qcom/cmd-db.c index a5fd68411bed..d84572662017 100644 --- a/drivers/soc/qcom/cmd-db.c +++ b/drivers/soc/qcom/cmd-db.c @@ -1,6 +1,10 @@ /* SPDX-License-Identifier: GPL-2.0 */ -/* Copyright (c) 2016-2018, 2020, The Linux Foundation. All rights reserved. */ +/* + * Copyright (c) 2016-2018, 2020, The Linux Foundation. All rights reserved. + * Copyright (c) 2024, Qualcomm Innovation Center, Inc. All rights reserved. + */ +#include <linux/bitfield.h> #include <linux/debugfs.h> #include <linux/kernel.h> #include <linux/module.h> @@ -17,6 +21,8 @@ #define MAX_SLV_ID 8 #define SLAVE_ID_MASK 0x7 #define SLAVE_ID_SHIFT 16 +#define SLAVE_ID(addr) FIELD_GET(GENMASK(19, 16), addr) +#define VRM_ADDR(addr) FIELD_GET(GENMASK(19, 4), addr) /** * struct entry_header: header for each entry in cmddb @@ -147,12 +153,7 @@ static int cmd_db_get_header(const char *id, const struct entry_header **eh, if (ret) return ret; - /* - * Pad out query string to same length as in DB. NOTE: the output - * query string is not necessarily '\0' terminated if it bumps up - * against the max size. That's OK and expected. - */ - strncpy(query, id, sizeof(query)); + strtomem_pad(query, id, 0); for (i = 0; i < MAX_SLV_ID; i++) { rsc_hdr = &cmd_db_header->header[i]; @@ -221,6 +222,30 @@ const void *cmd_db_read_aux_data(const char *id, size_t *len) EXPORT_SYMBOL_GPL(cmd_db_read_aux_data); /** + * cmd_db_match_resource_addr() - Compare if both Resource addresses are same + * + * @addr1: Resource address to compare + * @addr2: Resource address to compare + * + * Return: true if two addresses refer to the same resource, false otherwise + */ +bool cmd_db_match_resource_addr(u32 addr1, u32 addr2) +{ + /* + * Each RPMh VRM accelerator resource has 3 or 4 contiguous 4-byte + * aligned addresses associated with it. Ignore the offset to check + * for VRM requests. + */ + if (addr1 == addr2) + return true; + else if (SLAVE_ID(addr1) == CMD_DB_HW_VRM && VRM_ADDR(addr1) == VRM_ADDR(addr2)) + return true; + + return false; +} +EXPORT_SYMBOL_GPL(cmd_db_match_resource_addr); + +/** * cmd_db_read_slave_id - Get the slave ID for a given resource address * * @id: Resource id to query the DB for version @@ -362,7 +387,7 @@ static int __init cmd_db_device_init(void) { return platform_driver_register(&cmd_db_dev_driver); } -arch_initcall(cmd_db_device_init); +core_initcall(cmd_db_device_init); MODULE_DESCRIPTION("Qualcomm Technologies, Inc. Command DB Driver"); MODULE_LICENSE("GPL v2"); diff --git a/drivers/soc/qcom/icc-bwmon.c b/drivers/soc/qcom/icc-bwmon.c index 656706259353..fb323b3364db 100644 --- a/drivers/soc/qcom/icc-bwmon.c +++ b/drivers/soc/qcom/icc-bwmon.c @@ -282,7 +282,7 @@ static const struct regmap_config msm8998_bwmon_regmap_cfg = { * Cache is necessary for using regmap fields with non-readable * registers. */ - .cache_type = REGCACHE_RBTREE, + .cache_type = REGCACHE_MAPLE, }; static const struct regmap_config msm8998_bwmon_global_regmap_cfg = { @@ -301,7 +301,7 @@ static const struct regmap_config msm8998_bwmon_global_regmap_cfg = { * Cache is necessary for using regmap fields with non-readable * registers. */ - .cache_type = REGCACHE_RBTREE, + .cache_type = REGCACHE_MAPLE, }; static const struct reg_field sdm845_cpu_bwmon_reg_fields[] = { @@ -369,7 +369,7 @@ static const struct regmap_config sdm845_cpu_bwmon_regmap_cfg = { * Cache is necessary for using regmap fields with non-readable * registers. */ - .cache_type = REGCACHE_RBTREE, + .cache_type = REGCACHE_MAPLE, }; /* BWMON v5 */ @@ -446,7 +446,7 @@ static const struct regmap_config sdm845_llcc_bwmon_regmap_cfg = { * Cache is necessary for using regmap fields with non-readable * registers. */ - .cache_type = REGCACHE_RBTREE, + .cache_type = REGCACHE_MAPLE, }; static void bwmon_clear_counters(struct icc_bwmon *bwmon, bool clear_all) diff --git a/drivers/soc/qcom/pmic_glink.c b/drivers/soc/qcom/pmic_glink.c index f913e9bd57ed..823fd108fa03 100644 --- a/drivers/soc/qcom/pmic_glink.c +++ b/drivers/soc/qcom/pmic_glink.c @@ -11,6 +11,7 @@ #include <linux/slab.h> #include <linux/soc/qcom/pdr.h> #include <linux/soc/qcom/pmic_glink.h> +#include <linux/spinlock.h> enum { PMIC_GLINK_CLIENT_BATT = 0, @@ -36,7 +37,7 @@ struct pmic_glink { unsigned int pdr_state; /* serializing clients list updates */ - struct mutex client_lock; + spinlock_t client_lock; struct list_head clients; }; @@ -58,10 +59,11 @@ static void _devm_pmic_glink_release_client(struct device *dev, void *res) { struct pmic_glink_client *client = (struct pmic_glink_client *)res; struct pmic_glink *pg = client->pg; + unsigned long flags; - mutex_lock(&pg->client_lock); + spin_lock_irqsave(&pg->client_lock, flags); list_del(&client->node); - mutex_unlock(&pg->client_lock); + spin_unlock_irqrestore(&pg->client_lock, flags); } struct pmic_glink_client *devm_pmic_glink_register_client(struct device *dev, @@ -72,6 +74,7 @@ struct pmic_glink_client *devm_pmic_glink_register_client(struct device *dev, { struct pmic_glink_client *client; struct pmic_glink *pg = dev_get_drvdata(dev->parent); + unsigned long flags; client = devres_alloc(_devm_pmic_glink_release_client, sizeof(*client), GFP_KERNEL); if (!client) @@ -83,9 +86,14 @@ struct pmic_glink_client *devm_pmic_glink_register_client(struct device *dev, client->pdr_notify = pdr; client->priv = priv; - mutex_lock(&pg->client_lock); + mutex_lock(&pg->state_lock); + spin_lock_irqsave(&pg->client_lock, flags); + list_add(&client->node, &pg->clients); - mutex_unlock(&pg->client_lock); + client->pdr_notify(client->priv, pg->client_state); + + spin_unlock_irqrestore(&pg->client_lock, flags); + mutex_unlock(&pg->state_lock); devres_add(dev, client); @@ -107,6 +115,7 @@ static int pmic_glink_rpmsg_callback(struct rpmsg_device *rpdev, void *data, struct pmic_glink_client *client; struct pmic_glink_hdr *hdr; struct pmic_glink *pg = dev_get_drvdata(&rpdev->dev); + unsigned long flags; if (len < sizeof(*hdr)) { dev_warn(pg->dev, "ignoring truncated message\n"); @@ -115,10 +124,12 @@ static int pmic_glink_rpmsg_callback(struct rpmsg_device *rpdev, void *data, hdr = data; + spin_lock_irqsave(&pg->client_lock, flags); list_for_each_entry(client, &pg->clients, node) { if (client->id == le32_to_cpu(hdr->owner)) client->cb(data, len, client->priv); } + spin_unlock_irqrestore(&pg->client_lock, flags); return 0; } @@ -158,6 +169,7 @@ static void pmic_glink_state_notify_clients(struct pmic_glink *pg) { struct pmic_glink_client *client; unsigned int new_state = pg->client_state; + unsigned long flags; if (pg->client_state != SERVREG_SERVICE_STATE_UP) { if (pg->pdr_state == SERVREG_SERVICE_STATE_UP && pg->ept) @@ -168,8 +180,10 @@ static void pmic_glink_state_notify_clients(struct pmic_glink *pg) } if (new_state != pg->client_state) { + spin_lock_irqsave(&pg->client_lock, flags); list_for_each_entry(client, &pg->clients, node) client->pdr_notify(client->priv, new_state); + spin_unlock_irqrestore(&pg->client_lock, flags); pg->client_state = new_state; } } @@ -256,7 +270,7 @@ static int pmic_glink_probe(struct platform_device *pdev) pg->dev = &pdev->dev; INIT_LIST_HEAD(&pg->clients); - mutex_init(&pg->client_lock); + spin_lock_init(&pg->client_lock); mutex_init(&pg->state_lock); match_data = (unsigned long *)of_device_get_match_data(&pdev->dev); diff --git a/drivers/soc/qcom/pmic_pdcharger_ulog.c b/drivers/soc/qcom/pmic_pdcharger_ulog.c index 238cd38589dc..39f412bbf2c1 100644 --- a/drivers/soc/qcom/pmic_pdcharger_ulog.c +++ b/drivers/soc/qcom/pmic_pdcharger_ulog.c @@ -150,6 +150,10 @@ static const struct rpmsg_device_id pmic_pdcharger_ulog_rpmsg_id_match[] = { { "PMIC_LOGS_ADSP_APPS" }, {} }; +/* + * No MODULE_DEVICE_TABLE intentionally: that's a debugging module, to be + * loaded manually only. + */ static struct rpmsg_driver pmic_pdcharger_ulog_rpmsg_driver = { .probe = pmic_pdcharger_ulog_rpmsg_probe, diff --git a/drivers/soc/qcom/qcom_stats.c b/drivers/soc/qcom/qcom_stats.c index 0216fc24f2ca..c429d5154aae 100644 --- a/drivers/soc/qcom/qcom_stats.c +++ b/drivers/soc/qcom/qcom_stats.c @@ -35,11 +35,15 @@ static const struct subsystem_data subsystems[] = { { "wpss", 605, 13 }, { "adsp", 606, 2 }, { "cdsp", 607, 5 }, + { "cdsp1", 607, 12 }, + { "gpdsp0", 607, 17 }, + { "gpdsp1", 607, 18 }, { "slpi", 608, 3 }, { "gpu", 609, 0 }, { "display", 610, 0 }, { "adsp_island", 613, 2 }, { "slpi_island", 613, 3 }, + { "apss", 631, QCOM_SMEM_HOST_ANY }, }; struct stats_config { diff --git a/drivers/soc/qcom/rpm_master_stats.c b/drivers/soc/qcom/rpm_master_stats.c index 9ca13bcf67d3..086fe4ba6707 100644 --- a/drivers/soc/qcom/rpm_master_stats.c +++ b/drivers/soc/qcom/rpm_master_stats.c @@ -148,6 +148,10 @@ static const struct of_device_id rpm_master_table[] = { { .compatible = "qcom,rpm-master-stats" }, { }, }; +/* + * No MODULE_DEVICE_TABLE intentionally: that's a debugging module, to be + * loaded manually only. + */ static struct platform_driver master_stats_driver = { .probe = master_stats_probe, diff --git a/drivers/soc/qcom/rpmh-rsc.c b/drivers/soc/qcom/rpmh-rsc.c index a021dc71807b..561d8037b50a 100644 --- a/drivers/soc/qcom/rpmh-rsc.c +++ b/drivers/soc/qcom/rpmh-rsc.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 /* * Copyright (c) 2016-2018, The Linux Foundation. All rights reserved. + * Copyright (c) 2023-2024, Qualcomm Innovation Center, Inc. All rights reserved. */ #define pr_fmt(fmt) "%s " fmt, KBUILD_MODNAME @@ -557,7 +558,7 @@ static int check_for_req_inflight(struct rsc_drv *drv, struct tcs_group *tcs, for_each_set_bit(j, &curr_enabled, MAX_CMDS_PER_TCS) { addr = read_tcs_cmd(drv, drv->regs[RSC_DRV_CMD_ADDR], i, j); for (k = 0; k < msg->num_cmds; k++) { - if (addr == msg->cmds[k].addr) + if (cmd_db_match_resource_addr(msg->cmds[k].addr, addr)) return -EBUSY; } } @@ -1154,7 +1155,7 @@ static int __init rpmh_driver_init(void) { return platform_driver_register(&rpmh_driver); } -arch_initcall(rpmh_driver_init); +core_initcall(rpmh_driver_init); MODULE_DESCRIPTION("Qualcomm Technologies, Inc. RPMh Driver"); MODULE_LICENSE("GPL v2"); diff --git a/drivers/soc/qcom/socinfo.c b/drivers/soc/qcom/socinfo.c index e8ff9819ac47..277c07a6603d 100644 --- a/drivers/soc/qcom/socinfo.c +++ b/drivers/soc/qcom/socinfo.c @@ -133,6 +133,7 @@ static const char *const pmic_models[] = { [72] = "PMR735D", [73] = "PM8550", [74] = "PMK8550", + [82] = "SMB2360", }; struct socinfo_params { @@ -430,6 +431,7 @@ static const struct soc_id soc_id[] = { { qcom_board_id(QRU1000) }, { qcom_board_id(SM8475_2) }, { qcom_board_id(QDU1000) }, + { qcom_board_id(X1E80100) }, { qcom_board_id(SM8650) }, { qcom_board_id(SM4450) }, { qcom_board_id(QDU1010) }, diff --git a/drivers/soc/renesas/Kconfig b/drivers/soc/renesas/Kconfig index 5deca747fb77..5d94c3f31494 100644 --- a/drivers/soc/renesas/Kconfig +++ b/drivers/soc/renesas/Kconfig @@ -24,6 +24,7 @@ config ARCH_RCAR_GEN2 select RENESAS_IRQC select RST_RCAR select SYS_SUPPORTS_SH_CMT + select SYS_SUPPORTS_SH_TMU config ARCH_RCAR_GEN3 bool @@ -344,6 +345,11 @@ config ARCH_R9A09G011 help This enables support for the Renesas RZ/V2M SoC. +config ARCH_R9A09G057 + bool "ARM64 Platform support for RZ/V2H(P)" + help + This enables support for the Renesas RZ/V2H(P) SoC variants. + endif # ARM64 if RISCV diff --git a/drivers/soc/renesas/renesas-soc.c b/drivers/soc/renesas/renesas-soc.c index 8f9b8d3736dc..172d59e6fbcf 100644 --- a/drivers/soc/renesas/renesas-soc.c +++ b/drivers/soc/renesas/renesas-soc.c @@ -75,6 +75,10 @@ static const struct renesas_family fam_rzg3s __initconst __maybe_unused = { .name = "RZ/G3S", }; +static const struct renesas_family fam_rzv2h __initconst __maybe_unused = { + .name = "RZ/V2H", +}; + static const struct renesas_family fam_rzv2l __initconst __maybe_unused = { .name = "RZ/V2L", }; @@ -177,6 +181,11 @@ static const struct renesas_soc soc_rz_g3s __initconst __maybe_unused = { .id = 0x85e0447, }; +static const struct renesas_soc soc_rz_v2h __initconst __maybe_unused = { + .family = &fam_rzv2h, + .id = 0x847a447, +}; + static const struct renesas_soc soc_rz_v2l __initconst __maybe_unused = { .family = &fam_rzv2l, .id = 0x8447447, @@ -407,6 +416,9 @@ static const struct of_device_id renesas_socs[] __initconst __maybe_unused = { #ifdef CONFIG_ARCH_R9A09G011 { .compatible = "renesas,r9a09g011", .data = &soc_rz_v2m }, #endif +#ifdef CONFIG_ARCH_R9A09G057 + { .compatible = "renesas,r9a09g057", .data = &soc_rz_v2h }, +#endif #ifdef CONFIG_ARCH_SH73A0 { .compatible = "renesas,sh73a0", .data = &soc_shmobile_ag5 }, #endif @@ -432,6 +444,11 @@ static const struct renesas_id id_rzg2l __initconst = { .mask = 0xfffffff, }; +static const struct renesas_id id_rzv2h __initconst = { + .offset = 0x304, + .mask = 0xfffffff, +}; + static const struct renesas_id id_rzv2m __initconst = { .offset = 0x104, .mask = 0xff, @@ -449,6 +466,7 @@ static const struct of_device_id renesas_ids[] __initconst = { { .compatible = "renesas,r9a07g054-sysc", .data = &id_rzg2l }, { .compatible = "renesas,r9a08g045-sysc", .data = &id_rzg2l }, { .compatible = "renesas,r9a09g011-sys", .data = &id_rzv2m }, + { .compatible = "renesas,r9a09g057-sys", .data = &id_rzv2h }, { .compatible = "renesas,prr", .data = &id_prr }, { /* sentinel */ } }; @@ -513,7 +531,7 @@ static int __init renesas_soc_init(void) eslo = product & 0xf; soc_dev_attr->revision = kasprintf(GFP_KERNEL, "ES%u.%u", eshi, eslo); - } else if (id == &id_rzg2l) { + } else if (id == &id_rzg2l || id == &id_rzv2h) { eshi = ((product >> 28) & 0x0f); soc_dev_attr->revision = kasprintf(GFP_KERNEL, "%u", eshi); diff --git a/drivers/soc/tegra/pmc.c b/drivers/soc/tegra/pmc.c index d6bfcea5ee65..91d0ad6ddefc 100644 --- a/drivers/soc/tegra/pmc.c +++ b/drivers/soc/tegra/pmc.c @@ -4074,6 +4074,7 @@ static const char * const tegra194_reset_sources[] = { }; static const struct tegra_wake_event tegra194_wake_events[] = { + TEGRA_WAKE_GPIO("eqos", 20, 0, TEGRA194_MAIN_GPIO(G, 4)), TEGRA_WAKE_IRQ("pmu", 24, 209), TEGRA_WAKE_GPIO("power", 29, 1, TEGRA194_AON_GPIO(EE, 4)), TEGRA_WAKE_IRQ("rtc", 73, 10), @@ -4210,6 +4211,7 @@ static const char * const tegra234_reset_sources[] = { static const struct tegra_wake_event tegra234_wake_events[] = { TEGRA_WAKE_GPIO("sd-wake", 8, 0, TEGRA234_MAIN_GPIO(G, 7)), + TEGRA_WAKE_GPIO("eqos", 20, 0, TEGRA234_MAIN_GPIO(G, 4)), TEGRA_WAKE_IRQ("pmu", 24, 209), TEGRA_WAKE_GPIO("power", 29, 1, TEGRA234_AON_GPIO(EE, 4)), TEGRA_WAKE_GPIO("mgbe", 56, 0, TEGRA234_MAIN_GPIO(Y, 3)), diff --git a/drivers/soc/ti/wkup_m3_ipc.c b/drivers/soc/ti/wkup_m3_ipc.c index 6a1c6b34c414..88f774db9208 100644 --- a/drivers/soc/ti/wkup_m3_ipc.c +++ b/drivers/soc/ti/wkup_m3_ipc.c @@ -16,7 +16,6 @@ #include <linux/irq.h> #include <linux/module.h> #include <linux/of.h> -#include <linux/omap-mailbox.h> #include <linux/platform_device.h> #include <linux/remoteproc.h> #include <linux/suspend.h> @@ -314,7 +313,6 @@ static irqreturn_t wkup_m3_txev_handler(int irq, void *ipc_data) static int wkup_m3_ping(struct wkup_m3_ipc *m3_ipc) { struct device *dev = m3_ipc->dev; - mbox_msg_t dummy_msg = 0; int ret; if (!m3_ipc->mbox) { @@ -330,7 +328,7 @@ static int wkup_m3_ping(struct wkup_m3_ipc *m3_ipc) * the RX callback to avoid multiple interrupts being received * by the CM3. */ - ret = mbox_send_message(m3_ipc->mbox, &dummy_msg); + ret = mbox_send_message(m3_ipc->mbox, NULL); if (ret < 0) { dev_err(dev, "%s: mbox_send_message() failed: %d\n", __func__, ret); @@ -352,7 +350,6 @@ static int wkup_m3_ping(struct wkup_m3_ipc *m3_ipc) static int wkup_m3_ping_noirq(struct wkup_m3_ipc *m3_ipc) { struct device *dev = m3_ipc->dev; - mbox_msg_t dummy_msg = 0; int ret; if (!m3_ipc->mbox) { @@ -361,7 +358,7 @@ static int wkup_m3_ping_noirq(struct wkup_m3_ipc *m3_ipc) return -EIO; } - ret = mbox_send_message(m3_ipc->mbox, &dummy_msg); + ret = mbox_send_message(m3_ipc->mbox, NULL); if (ret < 0) { dev_err(dev, "%s: mbox_send_message() failed: %d\n", __func__, ret); diff --git a/drivers/spi/spi-microchip-core-qspi.c b/drivers/spi/spi-microchip-core-qspi.c index 03d125a71fd9..09f16471c537 100644 --- a/drivers/spi/spi-microchip-core-qspi.c +++ b/drivers/spi/spi-microchip-core-qspi.c @@ -283,6 +283,7 @@ static int mchp_coreqspi_setup_clock(struct mchp_coreqspi *qspi, struct spi_devi } control = readl_relaxed(qspi->regs + REG_CONTROL); + control &= ~CONTROL_CLKRATE_MASK; control |= baud_rate_val << CONTROL_CLKRATE_SHIFT; writel_relaxed(control, qspi->regs + REG_CONTROL); control = readl_relaxed(qspi->regs + REG_CONTROL); diff --git a/drivers/spi/spi-stm32.c b/drivers/spi/spi-stm32.c index e4e7ddb7524a..4a68abcdcc35 100644 --- a/drivers/spi/spi-stm32.c +++ b/drivers/spi/spi-stm32.c @@ -1016,10 +1016,8 @@ end_irq: static irqreturn_t stm32fx_spi_irq_thread(int irq, void *dev_id) { struct spi_controller *ctrl = dev_id; - struct stm32_spi *spi = spi_controller_get_devdata(ctrl); spi_finalize_current_transfer(ctrl); - stm32fx_spi_disable(spi); return IRQ_HANDLED; } @@ -1187,6 +1185,8 @@ static int stm32_spi_prepare_msg(struct spi_controller *ctrl, ~clrb) | setb, spi->base + spi->cfg->regs->cpol.reg); + stm32_spi_enable(spi); + spin_unlock_irqrestore(&spi->lock, flags); return 0; @@ -1204,7 +1204,6 @@ static void stm32fx_spi_dma_tx_cb(void *data) if (spi->cur_comm == SPI_SIMPLEX_TX || spi->cur_comm == SPI_3WIRE_TX) { spi_finalize_current_transfer(spi->ctrl); - stm32fx_spi_disable(spi); } } @@ -1219,7 +1218,6 @@ static void stm32_spi_dma_rx_cb(void *data) struct stm32_spi *spi = data; spi_finalize_current_transfer(spi->ctrl); - spi->cfg->disable(spi); } /** @@ -1307,8 +1305,6 @@ static int stm32fx_spi_transfer_one_irq(struct stm32_spi *spi) stm32_spi_set_bits(spi, STM32FX_SPI_CR2, cr2); - stm32_spi_enable(spi); - /* starting data transfer when buffer is loaded */ if (spi->tx_buf) spi->cfg->write_tx(spi); @@ -1345,8 +1341,6 @@ static int stm32h7_spi_transfer_one_irq(struct stm32_spi *spi) spin_lock_irqsave(&spi->lock, flags); - stm32_spi_enable(spi); - /* Be sure to have data in fifo before starting data transfer */ if (spi->tx_buf) stm32h7_spi_write_txfifo(spi); @@ -1378,8 +1372,6 @@ static void stm32fx_spi_transfer_one_dma_start(struct stm32_spi *spi) */ stm32_spi_set_bits(spi, STM32FX_SPI_CR2, STM32FX_SPI_CR2_ERRIE); } - - stm32_spi_enable(spi); } /** @@ -1413,8 +1405,6 @@ static void stm32h7_spi_transfer_one_dma_start(struct stm32_spi *spi) stm32_spi_set_bits(spi, STM32H7_SPI_IER, ier); - stm32_spi_enable(spi); - if (STM32_SPI_HOST_MODE(spi)) stm32_spi_set_bits(spi, STM32H7_SPI_CR1, STM32H7_SPI_CR1_CSTART); } diff --git a/drivers/tee/Kconfig b/drivers/tee/Kconfig index 73a147202e88..61b507c18780 100644 --- a/drivers/tee/Kconfig +++ b/drivers/tee/Kconfig @@ -15,5 +15,6 @@ if TEE source "drivers/tee/optee/Kconfig" source "drivers/tee/amdtee/Kconfig" +source "drivers/tee/tstee/Kconfig" endif diff --git a/drivers/tee/Makefile b/drivers/tee/Makefile index 68da044afbfa..5488cba30bd2 100644 --- a/drivers/tee/Makefile +++ b/drivers/tee/Makefile @@ -5,3 +5,4 @@ tee-objs += tee_shm.o tee-objs += tee_shm_pool.o obj-$(CONFIG_OPTEE) += optee/ obj-$(CONFIG_AMDTEE) += amdtee/ +obj-$(CONFIG_ARM_TSTEE) += tstee/ diff --git a/drivers/tee/amdtee/amdtee_private.h b/drivers/tee/amdtee/amdtee_private.h index 6d0f7062bb87..d87050033894 100644 --- a/drivers/tee/amdtee/amdtee_private.h +++ b/drivers/tee/amdtee/amdtee_private.h @@ -9,7 +9,7 @@ #include <linux/mutex.h> #include <linux/spinlock.h> -#include <linux/tee_drv.h> +#include <linux/tee_core.h> #include <linux/kref.h> #include <linux/types.h> #include "amdtee_if.h" diff --git a/drivers/tee/amdtee/call.c b/drivers/tee/amdtee/call.c index e9b63dcb3194..4c21b02be4af 100644 --- a/drivers/tee/amdtee/call.c +++ b/drivers/tee/amdtee/call.c @@ -5,7 +5,7 @@ #include <linux/device.h> #include <linux/tee.h> -#include <linux/tee_drv.h> +#include <linux/tee_core.h> #include <linux/psp-tee.h> #include <linux/slab.h> #include <linux/psp.h> diff --git a/drivers/tee/amdtee/core.c b/drivers/tee/amdtee/core.c index 3c15f6a9e91c..e487231d25dc 100644 --- a/drivers/tee/amdtee/core.c +++ b/drivers/tee/amdtee/core.c @@ -9,13 +9,12 @@ #include <linux/slab.h> #include <linux/string.h> #include <linux/device.h> -#include <linux/tee_drv.h> +#include <linux/tee_core.h> #include <linux/types.h> #include <linux/mm.h> #include <linux/uaccess.h> #include <linux/firmware.h> #include "amdtee_private.h" -#include "../tee_private.h" #include <linux/psp-tee.h> static struct amdtee_driver_data *drv_data; diff --git a/drivers/tee/amdtee/shm_pool.c b/drivers/tee/amdtee/shm_pool.c index f0303126f199..6346e0bc8a64 100644 --- a/drivers/tee/amdtee/shm_pool.c +++ b/drivers/tee/amdtee/shm_pool.c @@ -4,7 +4,7 @@ */ #include <linux/slab.h> -#include <linux/tee_drv.h> +#include <linux/tee_core.h> #include <linux/psp.h> #include "amdtee_private.h" diff --git a/drivers/tee/optee/call.c b/drivers/tee/optee/call.c index a91e50be11be..16eb953e14bb 100644 --- a/drivers/tee/optee/call.c +++ b/drivers/tee/optee/call.c @@ -7,7 +7,7 @@ #include <linux/errno.h> #include <linux/mm.h> #include <linux/slab.h> -#include <linux/tee_drv.h> +#include <linux/tee_core.h> #include <linux/types.h> #include "optee_private.h" diff --git a/drivers/tee/optee/core.c b/drivers/tee/optee/core.c index 3aed554bc8d8..39e688d4e974 100644 --- a/drivers/tee/optee/core.c +++ b/drivers/tee/optee/core.c @@ -9,77 +9,13 @@ #include <linux/crash_dump.h> #include <linux/errno.h> #include <linux/io.h> -#include <linux/mm.h> #include <linux/module.h> #include <linux/slab.h> #include <linux/string.h> -#include <linux/tee_drv.h> +#include <linux/tee_core.h> #include <linux/types.h> #include "optee_private.h" -int optee_pool_op_alloc_helper(struct tee_shm_pool *pool, struct tee_shm *shm, - size_t size, size_t align, - int (*shm_register)(struct tee_context *ctx, - struct tee_shm *shm, - struct page **pages, - size_t num_pages, - unsigned long start)) -{ - size_t nr_pages = roundup(size, PAGE_SIZE) / PAGE_SIZE; - struct page **pages; - unsigned int i; - int rc = 0; - - /* - * Ignore alignment since this is already going to be page aligned - * and there's no need for any larger alignment. - */ - shm->kaddr = alloc_pages_exact(nr_pages * PAGE_SIZE, - GFP_KERNEL | __GFP_ZERO); - if (!shm->kaddr) - return -ENOMEM; - - shm->paddr = virt_to_phys(shm->kaddr); - shm->size = nr_pages * PAGE_SIZE; - - pages = kcalloc(nr_pages, sizeof(*pages), GFP_KERNEL); - if (!pages) { - rc = -ENOMEM; - goto err; - } - - for (i = 0; i < nr_pages; i++) - pages[i] = virt_to_page((u8 *)shm->kaddr + i * PAGE_SIZE); - - shm->pages = pages; - shm->num_pages = nr_pages; - - if (shm_register) { - rc = shm_register(shm->ctx, shm, pages, nr_pages, - (unsigned long)shm->kaddr); - if (rc) - goto err; - } - - return 0; -err: - free_pages_exact(shm->kaddr, shm->size); - shm->kaddr = NULL; - return rc; -} - -void optee_pool_op_free_helper(struct tee_shm_pool *pool, struct tee_shm *shm, - int (*shm_unregister)(struct tee_context *ctx, - struct tee_shm *shm)) -{ - if (shm_unregister) - shm_unregister(shm->ctx, shm); - free_pages_exact(shm->kaddr, shm->size); - shm->kaddr = NULL; - kfree(shm->pages); - shm->pages = NULL; -} - static void optee_bus_scan(struct work_struct *work) { WARN_ON(optee_enumerate_devices(PTA_CMD_GET_DEVICES_SUPP)); diff --git a/drivers/tee/optee/device.c b/drivers/tee/optee/device.c index 1892e49a8e6a..d296c70ddfdc 100644 --- a/drivers/tee/optee/device.c +++ b/drivers/tee/optee/device.c @@ -7,7 +7,7 @@ #include <linux/kernel.h> #include <linux/slab.h> -#include <linux/tee_drv.h> +#include <linux/tee_core.h> #include <linux/uuid.h> #include "optee_private.h" diff --git a/drivers/tee/optee/ffa_abi.c b/drivers/tee/optee/ffa_abi.c index ecb5eb079408..3235e1c719e8 100644 --- a/drivers/tee/optee/ffa_abi.c +++ b/drivers/tee/optee/ffa_abi.c @@ -11,7 +11,7 @@ #include <linux/sched.h> #include <linux/slab.h> #include <linux/string.h> -#include <linux/tee_drv.h> +#include <linux/tee_core.h> #include <linux/types.h> #include "optee_private.h" #include "optee_ffa.h" @@ -374,14 +374,14 @@ static int optee_ffa_shm_unregister_supp(struct tee_context *ctx, static int pool_ffa_op_alloc(struct tee_shm_pool *pool, struct tee_shm *shm, size_t size, size_t align) { - return optee_pool_op_alloc_helper(pool, shm, size, align, - optee_ffa_shm_register); + return tee_dyn_shm_alloc_helper(shm, size, align, + optee_ffa_shm_register); } static void pool_ffa_op_free(struct tee_shm_pool *pool, struct tee_shm *shm) { - optee_pool_op_free_helper(pool, shm, optee_ffa_shm_unregister); + tee_dyn_shm_free_helper(shm, optee_ffa_shm_unregister); } static void pool_ffa_op_destroy_pool(struct tee_shm_pool *pool) diff --git a/drivers/tee/optee/notif.c b/drivers/tee/optee/notif.c index 05212842b0a5..0d7878e770cd 100644 --- a/drivers/tee/optee/notif.c +++ b/drivers/tee/optee/notif.c @@ -9,7 +9,7 @@ #include <linux/errno.h> #include <linux/slab.h> #include <linux/spinlock.h> -#include <linux/tee_drv.h> +#include <linux/tee_core.h> #include "optee_private.h" struct notif_entry { diff --git a/drivers/tee/optee/optee_private.h b/drivers/tee/optee/optee_private.h index 7a5243c78b55..429cc20be5cc 100644 --- a/drivers/tee/optee/optee_private.h +++ b/drivers/tee/optee/optee_private.h @@ -9,7 +9,7 @@ #include <linux/arm-smccc.h> #include <linux/rhashtable.h> #include <linux/semaphore.h> -#include <linux/tee_drv.h> +#include <linux/tee_core.h> #include <linux/types.h> #include "optee_msg.h" @@ -283,18 +283,6 @@ int optee_cancel_req(struct tee_context *ctx, u32 cancel_id, u32 session); int optee_enumerate_devices(u32 func); void optee_unregister_devices(void); -int optee_pool_op_alloc_helper(struct tee_shm_pool *pool, struct tee_shm *shm, - size_t size, size_t align, - int (*shm_register)(struct tee_context *ctx, - struct tee_shm *shm, - struct page **pages, - size_t num_pages, - unsigned long start)); -void optee_pool_op_free_helper(struct tee_shm_pool *pool, struct tee_shm *shm, - int (*shm_unregister)(struct tee_context *ctx, - struct tee_shm *shm)); - - void optee_remove_common(struct optee *optee); int optee_open(struct tee_context *ctx, bool cap_memref_null); void optee_release(struct tee_context *ctx); diff --git a/drivers/tee/optee/rpc.c b/drivers/tee/optee/rpc.c index e69bc6380683..f086812f1179 100644 --- a/drivers/tee/optee/rpc.c +++ b/drivers/tee/optee/rpc.c @@ -8,7 +8,7 @@ #include <linux/delay.h> #include <linux/i2c.h> #include <linux/slab.h> -#include <linux/tee_drv.h> +#include <linux/tee_core.h> #include "optee_private.h" #include "optee_rpc_cmd.h" diff --git a/drivers/tee/optee/smc_abi.c b/drivers/tee/optee/smc_abi.c index a37f87087e5c..844285d4f03c 100644 --- a/drivers/tee/optee/smc_abi.c +++ b/drivers/tee/optee/smc_abi.c @@ -23,7 +23,7 @@ #include <linux/sched.h> #include <linux/slab.h> #include <linux/string.h> -#include <linux/tee_drv.h> +#include <linux/tee_core.h> #include <linux/types.h> #include <linux/workqueue.h> #include "optee_private.h" @@ -592,19 +592,18 @@ static int pool_op_alloc(struct tee_shm_pool *pool, * to be registered with OP-TEE. */ if (shm->flags & TEE_SHM_PRIV) - return optee_pool_op_alloc_helper(pool, shm, size, align, NULL); + return tee_dyn_shm_alloc_helper(shm, size, align, NULL); - return optee_pool_op_alloc_helper(pool, shm, size, align, - optee_shm_register); + return tee_dyn_shm_alloc_helper(shm, size, align, optee_shm_register); } static void pool_op_free(struct tee_shm_pool *pool, struct tee_shm *shm) { if (!(shm->flags & TEE_SHM_PRIV)) - optee_pool_op_free_helper(pool, shm, optee_shm_unregister); + tee_dyn_shm_free_helper(shm, optee_shm_unregister); else - optee_pool_op_free_helper(pool, shm, NULL); + tee_dyn_shm_free_helper(shm, NULL); } static void pool_op_destroy_pool(struct tee_shm_pool *pool) @@ -1433,7 +1432,7 @@ static optee_invoke_fn *get_invoke_func(struct device *dev) * optee_remove is called by platform subsystem to alert the driver * that it should release the device */ -static int optee_smc_remove(struct platform_device *pdev) +static void optee_smc_remove(struct platform_device *pdev) { struct optee *optee = platform_get_drvdata(pdev); @@ -1453,8 +1452,6 @@ static int optee_smc_remove(struct platform_device *pdev) memunmap(optee->smc.memremaped_shm); kfree(optee); - - return 0; } /* optee_shutdown - Device Removal Routine @@ -1806,7 +1803,7 @@ MODULE_DEVICE_TABLE(of, optee_dt_match); static struct platform_driver optee_driver = { .probe = optee_probe, - .remove = optee_smc_remove, + .remove_new = optee_smc_remove, .shutdown = optee_shutdown, .driver = { .name = "optee", diff --git a/drivers/tee/tee_core.c b/drivers/tee/tee_core.c index e59c20d74b36..82ad095d2b1c 100644 --- a/drivers/tee/tee_core.c +++ b/drivers/tee/tee_core.c @@ -11,7 +11,7 @@ #include <linux/idr.h> #include <linux/module.h> #include <linux/slab.h> -#include <linux/tee_drv.h> +#include <linux/tee_core.h> #include <linux/uaccess.h> #include <crypto/hash.h> #include <crypto/sha1.h> diff --git a/drivers/tee/tee_private.h b/drivers/tee/tee_private.h index 754e11dcb240..9bc50605227c 100644 --- a/drivers/tee/tee_private.h +++ b/drivers/tee/tee_private.h @@ -12,41 +12,6 @@ #include <linux/mutex.h> #include <linux/types.h> -#define TEE_DEVICE_FLAG_REGISTERED 0x1 -#define TEE_MAX_DEV_NAME_LEN 32 - -/** - * struct tee_device - TEE Device representation - * @name: name of device - * @desc: description of device - * @id: unique id of device - * @flags: represented by TEE_DEVICE_FLAG_REGISTERED above - * @dev: embedded basic device structure - * @cdev: embedded cdev - * @num_users: number of active users of this device - * @c_no_user: completion used when unregistering the device - * @mutex: mutex protecting @num_users and @idr - * @idr: register of user space shared memory objects allocated or - * registered on this device - * @pool: shared memory pool - */ -struct tee_device { - char name[TEE_MAX_DEV_NAME_LEN]; - const struct tee_desc *desc; - int id; - unsigned int flags; - - struct device dev; - struct cdev cdev; - - size_t num_users; - struct completion c_no_users; - struct mutex mutex; /* protects num_users and idr */ - - struct idr idr; - struct tee_shm_pool *pool; -}; - int tee_shm_get_fd(struct tee_shm *shm); bool tee_device_get(struct tee_device *teedev); diff --git a/drivers/tee/tee_shm.c b/drivers/tee/tee_shm.c index 731d9028b67f..daf6e5cfd59a 100644 --- a/drivers/tee/tee_shm.c +++ b/drivers/tee/tee_shm.c @@ -5,10 +5,11 @@ #include <linux/anon_inodes.h> #include <linux/device.h> #include <linux/idr.h> +#include <linux/io.h> #include <linux/mm.h> #include <linux/sched.h> #include <linux/slab.h> -#include <linux/tee_drv.h> +#include <linux/tee_core.h> #include <linux/uaccess.h> #include <linux/uio.h> #include <linux/highmem.h> @@ -202,6 +203,70 @@ struct tee_shm *tee_shm_alloc_priv_buf(struct tee_context *ctx, size_t size) } EXPORT_SYMBOL_GPL(tee_shm_alloc_priv_buf); +int tee_dyn_shm_alloc_helper(struct tee_shm *shm, size_t size, size_t align, + int (*shm_register)(struct tee_context *ctx, + struct tee_shm *shm, + struct page **pages, + size_t num_pages, + unsigned long start)) +{ + size_t nr_pages = roundup(size, PAGE_SIZE) / PAGE_SIZE; + struct page **pages; + unsigned int i; + int rc = 0; + + /* + * Ignore alignment since this is already going to be page aligned + * and there's no need for any larger alignment. + */ + shm->kaddr = alloc_pages_exact(nr_pages * PAGE_SIZE, + GFP_KERNEL | __GFP_ZERO); + if (!shm->kaddr) + return -ENOMEM; + + shm->paddr = virt_to_phys(shm->kaddr); + shm->size = nr_pages * PAGE_SIZE; + + pages = kcalloc(nr_pages, sizeof(*pages), GFP_KERNEL); + if (!pages) { + rc = -ENOMEM; + goto err; + } + + for (i = 0; i < nr_pages; i++) + pages[i] = virt_to_page((u8 *)shm->kaddr + i * PAGE_SIZE); + + shm->pages = pages; + shm->num_pages = nr_pages; + + if (shm_register) { + rc = shm_register(shm->ctx, shm, pages, nr_pages, + (unsigned long)shm->kaddr); + if (rc) + goto err; + } + + return 0; +err: + free_pages_exact(shm->kaddr, shm->size); + shm->kaddr = NULL; + return rc; +} +EXPORT_SYMBOL_GPL(tee_dyn_shm_alloc_helper); + +void tee_dyn_shm_free_helper(struct tee_shm *shm, + int (*shm_unregister)(struct tee_context *ctx, + struct tee_shm *shm)) +{ + if (shm_unregister) + shm_unregister(shm->ctx, shm); + free_pages_exact(shm->kaddr, shm->size); + shm->kaddr = NULL; + kfree(shm->pages); + shm->pages = NULL; +} +EXPORT_SYMBOL_GPL(tee_dyn_shm_free_helper); + static struct tee_shm * register_shm_helper(struct tee_context *ctx, struct iov_iter *iter, u32 flags, int id) diff --git a/drivers/tee/tee_shm_pool.c b/drivers/tee/tee_shm_pool.c index 058bfbac657a..80004b55628d 100644 --- a/drivers/tee/tee_shm_pool.c +++ b/drivers/tee/tee_shm_pool.c @@ -6,7 +6,7 @@ #include <linux/dma-buf.h> #include <linux/genalloc.h> #include <linux/slab.h> -#include <linux/tee_drv.h> +#include <linux/tee_core.h> #include "tee_private.h" static int pool_op_gen_alloc(struct tee_shm_pool *pool, struct tee_shm *shm, diff --git a/drivers/tee/tstee/Kconfig b/drivers/tee/tstee/Kconfig new file mode 100644 index 000000000000..d32f91d47398 --- /dev/null +++ b/drivers/tee/tstee/Kconfig @@ -0,0 +1,11 @@ +# SPDX-License-Identifier: GPL-2.0-only +config ARM_TSTEE + tristate "Arm Trusted Services TEE driver" + depends on ARM_FFA_TRANSPORT + default n + help + The Trusted Services project provides a framework for developing and + deploying device Root of Trust services in FF-A Secure Partitions. + This driver provides an interface to make Trusted Services Secure + Partitions accessible for user space clients, since the FF-A driver + doesn't implement a user space interface directly. diff --git a/drivers/tee/tstee/Makefile b/drivers/tee/tstee/Makefile new file mode 100644 index 000000000000..5227020ebd30 --- /dev/null +++ b/drivers/tee/tstee/Makefile @@ -0,0 +1,3 @@ +# SPDX-License-Identifier: GPL-2.0-only +arm-tstee-objs := core.o +obj-$(CONFIG_ARM_TSTEE) = arm-tstee.o diff --git a/drivers/tee/tstee/core.c b/drivers/tee/tstee/core.c new file mode 100644 index 000000000000..533425e9e9e7 --- /dev/null +++ b/drivers/tee/tstee/core.c @@ -0,0 +1,480 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2023, Arm Limited + */ + +#include <linux/arm_ffa.h> +#include <linux/err.h> +#include <linux/errno.h> +#include <linux/kernel.h> +#include <linux/limits.h> +#include <linux/mm.h> +#include <linux/module.h> +#include <linux/scatterlist.h> +#include <linux/slab.h> +#include <linux/tee_core.h> +#include <linux/types.h> +#include <linux/uuid.h> +#include <linux/xarray.h> +#include "tstee_private.h" + +#define FFA_DIRECT_REQ_ARG_NUM 5 +#define FFA_INVALID_MEM_HANDLE U64_MAX + +static void arg_list_to_ffa_data(const u32 *args, + struct ffa_send_direct_data *data) +{ + data->data0 = args[0]; + data->data1 = args[1]; + data->data2 = args[2]; + data->data3 = args[3]; + data->data4 = args[4]; +} + +static void arg_list_from_ffa_data(const struct ffa_send_direct_data *data, + u32 *args) +{ + args[0] = lower_32_bits(data->data0); + args[1] = lower_32_bits(data->data1); + args[2] = lower_32_bits(data->data2); + args[3] = lower_32_bits(data->data3); + args[4] = lower_32_bits(data->data4); +} + +static void tstee_get_version(struct tee_device *teedev, + struct tee_ioctl_version_data *vers) +{ + struct tstee *tstee = tee_get_drvdata(teedev); + struct tee_ioctl_version_data v = { + .impl_id = TEE_IMPL_ID_TSTEE, + /* FF-A endpoint ID only uses the lower 16 bits */ + .impl_caps = lower_16_bits(tstee->ffa_dev->vm_id), + .gen_caps = 0, + }; + + *vers = v; +} + +static int tstee_open(struct tee_context *ctx) +{ + struct ts_context_data *ctxdata; + + ctxdata = kzalloc(sizeof(*ctxdata), GFP_KERNEL); + if (!ctxdata) + return -ENOMEM; + + xa_init_flags(&ctxdata->sess_list, XA_FLAGS_ALLOC); + + ctx->data = ctxdata; + + return 0; +} + +static void tstee_release(struct tee_context *ctx) +{ + struct ts_context_data *ctxdata = ctx->data; + struct ts_session *sess; + unsigned long idx; + + if (!ctxdata) + return; + + xa_for_each(&ctxdata->sess_list, idx, sess) { + xa_erase(&ctxdata->sess_list, idx); + kfree(sess); + } + + xa_destroy(&ctxdata->sess_list); + + kfree(ctxdata); + ctx->data = NULL; +} + +static int tstee_open_session(struct tee_context *ctx, + struct tee_ioctl_open_session_arg *arg, + struct tee_param *param __always_unused) +{ + struct tstee *tstee = tee_get_drvdata(ctx->teedev); + struct ffa_device *ffa_dev = tstee->ffa_dev; + struct ts_context_data *ctxdata = ctx->data; + struct ffa_send_direct_data ffa_data; + struct ts_session *sess = NULL; + u32 ffa_args[FFA_DIRECT_REQ_ARG_NUM] = {}; + u32 sess_id; + int rc; + + ffa_args[TS_RPC_CTRL_REG] = + TS_RPC_CTRL_PACK_IFACE_OPCODE(TS_RPC_MGMT_IFACE_ID, + TS_RPC_OP_SERVICE_INFO); + + memcpy(ffa_args + TS_RPC_SERVICE_INFO_UUID0, arg->uuid, UUID_SIZE); + + arg_list_to_ffa_data(ffa_args, &ffa_data); + rc = ffa_dev->ops->msg_ops->sync_send_receive(ffa_dev, &ffa_data); + if (rc) + return rc; + + arg_list_from_ffa_data(&ffa_data, ffa_args); + + if (ffa_args[TS_RPC_SERVICE_INFO_RPC_STATUS] != TS_RPC_OK) + return -ENODEV; + + if (ffa_args[TS_RPC_SERVICE_INFO_IFACE] > U8_MAX) + return -EINVAL; + + sess = kzalloc(sizeof(*sess), GFP_KERNEL); + if (!sess) + return -ENOMEM; + + sess->iface_id = ffa_args[TS_RPC_SERVICE_INFO_IFACE]; + + rc = xa_alloc(&ctxdata->sess_list, &sess_id, sess, xa_limit_32b, + GFP_KERNEL); + if (rc) { + kfree(sess); + return rc; + } + + arg->session = sess_id; + arg->ret = 0; + + return 0; +} + +static int tstee_close_session(struct tee_context *ctx, u32 session) +{ + struct ts_context_data *ctxdata = ctx->data; + struct ts_session *sess; + + /* Calls xa_lock() internally */ + sess = xa_erase(&ctxdata->sess_list, session); + if (!sess) + return -EINVAL; + + kfree(sess); + + return 0; +} + +static int tstee_invoke_func(struct tee_context *ctx, + struct tee_ioctl_invoke_arg *arg, + struct tee_param *param) +{ + struct tstee *tstee = tee_get_drvdata(ctx->teedev); + struct ffa_device *ffa_dev = tstee->ffa_dev; + struct ts_context_data *ctxdata = ctx->data; + struct ffa_send_direct_data ffa_data; + struct tee_shm *shm = NULL; + struct ts_session *sess; + u32 req_len, ffa_args[FFA_DIRECT_REQ_ARG_NUM] = {}; + int shm_id, rc; + u8 iface_id; + u64 handle; + u16 opcode; + + xa_lock(&ctxdata->sess_list); + sess = xa_load(&ctxdata->sess_list, arg->session); + + /* + * Do this while holding the lock to make sure that the session wasn't + * closed meanwhile + */ + if (sess) + iface_id = sess->iface_id; + + xa_unlock(&ctxdata->sess_list); + if (!sess) + return -EINVAL; + + opcode = lower_16_bits(arg->func); + shm_id = lower_32_bits(param[0].u.value.a); + req_len = lower_32_bits(param[0].u.value.b); + + if (shm_id != 0) { + shm = tee_shm_get_from_id(ctx, shm_id); + if (IS_ERR(shm)) + return PTR_ERR(shm); + + if (shm->size < req_len) { + dev_err(&ffa_dev->dev, + "request doesn't fit into shared memory buffer\n"); + rc = -EINVAL; + goto out; + } + + handle = shm->sec_world_id; + } else { + handle = FFA_INVALID_MEM_HANDLE; + } + + ffa_args[TS_RPC_CTRL_REG] = TS_RPC_CTRL_PACK_IFACE_OPCODE(iface_id, + opcode); + ffa_args[TS_RPC_SERVICE_MEM_HANDLE_LSW] = lower_32_bits(handle); + ffa_args[TS_RPC_SERVICE_MEM_HANDLE_MSW] = upper_32_bits(handle); + ffa_args[TS_RPC_SERVICE_REQ_LEN] = req_len; + ffa_args[TS_RPC_SERVICE_CLIENT_ID] = 0; + + arg_list_to_ffa_data(ffa_args, &ffa_data); + rc = ffa_dev->ops->msg_ops->sync_send_receive(ffa_dev, &ffa_data); + if (rc) + goto out; + + arg_list_from_ffa_data(&ffa_data, ffa_args); + + if (ffa_args[TS_RPC_SERVICE_RPC_STATUS] != TS_RPC_OK) { + dev_err(&ffa_dev->dev, "invoke_func rpc status: %d\n", + ffa_args[TS_RPC_SERVICE_RPC_STATUS]); + rc = -EINVAL; + goto out; + } + + arg->ret = ffa_args[TS_RPC_SERVICE_STATUS]; + if (shm && shm->size >= ffa_args[TS_RPC_SERVICE_RESP_LEN]) + param[0].u.value.a = ffa_args[TS_RPC_SERVICE_RESP_LEN]; + +out: + if (shm) + tee_shm_put(shm); + + return rc; +} + +static int tstee_shm_register(struct tee_context *ctx, struct tee_shm *shm, + struct page **pages, size_t num_pages, + unsigned long start __always_unused) +{ + struct tstee *tstee = tee_get_drvdata(ctx->teedev); + struct ffa_device *ffa_dev = tstee->ffa_dev; + struct ffa_mem_region_attributes mem_attr = { + .receiver = tstee->ffa_dev->vm_id, + .attrs = FFA_MEM_RW, + .flag = 0, + }; + struct ffa_mem_ops_args mem_args = { + .attrs = &mem_attr, + .use_txbuf = true, + .nattrs = 1, + .flags = 0, + }; + struct ffa_send_direct_data ffa_data; + struct sg_table sgt; + u32 ffa_args[FFA_DIRECT_REQ_ARG_NUM] = {}; + int rc; + + rc = sg_alloc_table_from_pages(&sgt, pages, num_pages, 0, + num_pages * PAGE_SIZE, GFP_KERNEL); + if (rc) + return rc; + + mem_args.sg = sgt.sgl; + rc = ffa_dev->ops->mem_ops->memory_share(&mem_args); + sg_free_table(&sgt); + if (rc) + return rc; + + shm->sec_world_id = mem_args.g_handle; + + ffa_args[TS_RPC_CTRL_REG] = + TS_RPC_CTRL_PACK_IFACE_OPCODE(TS_RPC_MGMT_IFACE_ID, + TS_RPC_OP_RETRIEVE_MEM); + ffa_args[TS_RPC_RETRIEVE_MEM_HANDLE_LSW] = + lower_32_bits(shm->sec_world_id); + ffa_args[TS_RPC_RETRIEVE_MEM_HANDLE_MSW] = + upper_32_bits(shm->sec_world_id); + ffa_args[TS_RPC_RETRIEVE_MEM_TAG_LSW] = 0; + ffa_args[TS_RPC_RETRIEVE_MEM_TAG_MSW] = 0; + + arg_list_to_ffa_data(ffa_args, &ffa_data); + rc = ffa_dev->ops->msg_ops->sync_send_receive(ffa_dev, &ffa_data); + if (rc) { + (void)ffa_dev->ops->mem_ops->memory_reclaim(shm->sec_world_id, + 0); + return rc; + } + + arg_list_from_ffa_data(&ffa_data, ffa_args); + + if (ffa_args[TS_RPC_RETRIEVE_MEM_RPC_STATUS] != TS_RPC_OK) { + dev_err(&ffa_dev->dev, "shm_register rpc status: %d\n", + ffa_args[TS_RPC_RETRIEVE_MEM_RPC_STATUS]); + ffa_dev->ops->mem_ops->memory_reclaim(shm->sec_world_id, 0); + return -EINVAL; + } + + return 0; +} + +static int tstee_shm_unregister(struct tee_context *ctx, struct tee_shm *shm) +{ + struct tstee *tstee = tee_get_drvdata(ctx->teedev); + struct ffa_device *ffa_dev = tstee->ffa_dev; + struct ffa_send_direct_data ffa_data; + u32 ffa_args[FFA_DIRECT_REQ_ARG_NUM] = {}; + int rc; + + ffa_args[TS_RPC_CTRL_REG] = + TS_RPC_CTRL_PACK_IFACE_OPCODE(TS_RPC_MGMT_IFACE_ID, + TS_RPC_OP_RELINQ_MEM); + ffa_args[TS_RPC_RELINQ_MEM_HANDLE_LSW] = + lower_32_bits(shm->sec_world_id); + ffa_args[TS_RPC_RELINQ_MEM_HANDLE_MSW] = + upper_32_bits(shm->sec_world_id); + + arg_list_to_ffa_data(ffa_args, &ffa_data); + rc = ffa_dev->ops->msg_ops->sync_send_receive(ffa_dev, &ffa_data); + if (rc) + return rc; + arg_list_from_ffa_data(&ffa_data, ffa_args); + + if (ffa_args[TS_RPC_RELINQ_MEM_RPC_STATUS] != TS_RPC_OK) { + dev_err(&ffa_dev->dev, "shm_unregister rpc status: %d\n", + ffa_args[TS_RPC_RELINQ_MEM_RPC_STATUS]); + return -EINVAL; + } + + rc = ffa_dev->ops->mem_ops->memory_reclaim(shm->sec_world_id, 0); + + return rc; +} + +static const struct tee_driver_ops tstee_ops = { + .get_version = tstee_get_version, + .open = tstee_open, + .release = tstee_release, + .open_session = tstee_open_session, + .close_session = tstee_close_session, + .invoke_func = tstee_invoke_func, +}; + +static const struct tee_desc tstee_desc = { + .name = "tstee-clnt", + .ops = &tstee_ops, + .owner = THIS_MODULE, +}; + +static int pool_op_alloc(struct tee_shm_pool *pool, struct tee_shm *shm, + size_t size, size_t align) +{ + return tee_dyn_shm_alloc_helper(shm, size, align, tstee_shm_register); +} + +static void pool_op_free(struct tee_shm_pool *pool, struct tee_shm *shm) +{ + tee_dyn_shm_free_helper(shm, tstee_shm_unregister); +} + +static void pool_op_destroy_pool(struct tee_shm_pool *pool) +{ + kfree(pool); +} + +static const struct tee_shm_pool_ops pool_ops = { + .alloc = pool_op_alloc, + .free = pool_op_free, + .destroy_pool = pool_op_destroy_pool, +}; + +static struct tee_shm_pool *tstee_create_shm_pool(void) +{ + struct tee_shm_pool *pool = kzalloc(sizeof(*pool), GFP_KERNEL); + + if (!pool) + return ERR_PTR(-ENOMEM); + + pool->ops = &pool_ops; + + return pool; +} + +static bool tstee_check_rpc_compatible(struct ffa_device *ffa_dev) +{ + struct ffa_send_direct_data ffa_data; + u32 ffa_args[FFA_DIRECT_REQ_ARG_NUM] = {}; + + ffa_args[TS_RPC_CTRL_REG] = + TS_RPC_CTRL_PACK_IFACE_OPCODE(TS_RPC_MGMT_IFACE_ID, + TS_RPC_OP_GET_VERSION); + + arg_list_to_ffa_data(ffa_args, &ffa_data); + if (ffa_dev->ops->msg_ops->sync_send_receive(ffa_dev, &ffa_data)) + return false; + + arg_list_from_ffa_data(&ffa_data, ffa_args); + + return ffa_args[TS_RPC_GET_VERSION_RESP] == TS_RPC_PROTOCOL_VERSION; +} + +static int tstee_probe(struct ffa_device *ffa_dev) +{ + struct tstee *tstee; + int rc; + + ffa_dev->ops->msg_ops->mode_32bit_set(ffa_dev); + + if (!tstee_check_rpc_compatible(ffa_dev)) + return -EINVAL; + + tstee = kzalloc(sizeof(*tstee), GFP_KERNEL); + if (!tstee) + return -ENOMEM; + + tstee->ffa_dev = ffa_dev; + + tstee->pool = tstee_create_shm_pool(); + if (IS_ERR(tstee->pool)) { + rc = PTR_ERR(tstee->pool); + tstee->pool = NULL; + goto err_free_tstee; + } + + tstee->teedev = tee_device_alloc(&tstee_desc, NULL, tstee->pool, tstee); + if (IS_ERR(tstee->teedev)) { + rc = PTR_ERR(tstee->teedev); + tstee->teedev = NULL; + goto err_free_pool; + } + + rc = tee_device_register(tstee->teedev); + if (rc) + goto err_unreg_teedev; + + ffa_dev_set_drvdata(ffa_dev, tstee); + + return 0; + +err_unreg_teedev: + tee_device_unregister(tstee->teedev); +err_free_pool: + tee_shm_pool_free(tstee->pool); +err_free_tstee: + kfree(tstee); + return rc; +} + +static void tstee_remove(struct ffa_device *ffa_dev) +{ + struct tstee *tstee = ffa_dev->dev.driver_data; + + tee_device_unregister(tstee->teedev); + tee_shm_pool_free(tstee->pool); + kfree(tstee); +} + +static const struct ffa_device_id tstee_device_ids[] = { + /* TS RPC protocol UUID: bdcd76d7-825e-4751-963b-86d4f84943ac */ + { TS_RPC_UUID }, + {} +}; + +static struct ffa_driver tstee_driver = { + .name = "arm_tstee", + .probe = tstee_probe, + .remove = tstee_remove, + .id_table = tstee_device_ids, +}; + +module_ffa_driver(tstee_driver); + +MODULE_AUTHOR("Balint Dobszay <balint.dobszay@arm.com>"); +MODULE_DESCRIPTION("Arm Trusted Services TEE driver"); +MODULE_LICENSE("GPL"); diff --git a/drivers/tee/tstee/tstee_private.h b/drivers/tee/tstee/tstee_private.h new file mode 100644 index 000000000000..8e58725b57eb --- /dev/null +++ b/drivers/tee/tstee/tstee_private.h @@ -0,0 +1,92 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) 2023, Arm Limited + */ + +#ifndef TSTEE_PRIVATE_H +#define TSTEE_PRIVATE_H + +#include <linux/arm_ffa.h> +#include <linux/bitops.h> +#include <linux/tee_core.h> +#include <linux/types.h> +#include <linux/uuid.h> +#include <linux/xarray.h> + +/* + * The description of the ABI implemented in this file is available at + * https://trusted-services.readthedocs.io/en/v1.0.0/developer/service-access-protocols.html#abi + */ + +/* UUID of this protocol */ +#define TS_RPC_UUID UUID_INIT(0xbdcd76d7, 0x825e, 0x4751, \ + 0x96, 0x3b, 0x86, 0xd4, 0xf8, 0x49, 0x43, 0xac) + +/* Protocol version*/ +#define TS_RPC_PROTOCOL_VERSION (1) + +/* Status codes */ +#define TS_RPC_OK (0) + +/* RPC control register */ +#define TS_RPC_CTRL_REG (0) +#define OPCODE_MASK GENMASK(15, 0) +#define IFACE_ID_MASK GENMASK(23, 16) +#define TS_RPC_CTRL_OPCODE(x) ((u16)(FIELD_GET(OPCODE_MASK, (x)))) +#define TS_RPC_CTRL_IFACE_ID(x) ((u8)(FIELD_GET(IFACE_ID_MASK, (x)))) +#define TS_RPC_CTRL_PACK_IFACE_OPCODE(i, o) \ + (FIELD_PREP(IFACE_ID_MASK, (i)) | FIELD_PREP(OPCODE_MASK, (o))) +#define TS_RPC_CTRL_SAP_RC BIT(30) +#define TS_RPC_CTRL_SAP_ERR BIT(31) + +/* Interface ID for RPC management operations */ +#define TS_RPC_MGMT_IFACE_ID (0xff) + +/* Management calls */ +#define TS_RPC_OP_GET_VERSION (0x0000) +#define TS_RPC_GET_VERSION_RESP (1) + +#define TS_RPC_OP_RETRIEVE_MEM (0x0001) +#define TS_RPC_RETRIEVE_MEM_HANDLE_LSW (1) +#define TS_RPC_RETRIEVE_MEM_HANDLE_MSW (2) +#define TS_RPC_RETRIEVE_MEM_TAG_LSW (3) +#define TS_RPC_RETRIEVE_MEM_TAG_MSW (4) +#define TS_RPC_RETRIEVE_MEM_RPC_STATUS (1) + +#define TS_RPC_OP_RELINQ_MEM (0x0002) +#define TS_RPC_RELINQ_MEM_HANDLE_LSW (1) +#define TS_RPC_RELINQ_MEM_HANDLE_MSW (2) +#define TS_RPC_RELINQ_MEM_RPC_STATUS (1) + +#define TS_RPC_OP_SERVICE_INFO (0x0003) +#define TS_RPC_SERVICE_INFO_UUID0 (1) +#define TS_RPC_SERVICE_INFO_UUID1 (2) +#define TS_RPC_SERVICE_INFO_UUID2 (3) +#define TS_RPC_SERVICE_INFO_UUID3 (4) +#define TS_RPC_SERVICE_INFO_RPC_STATUS (1) +#define TS_RPC_SERVICE_INFO_IFACE (2) + +/* Service call */ +#define TS_RPC_SERVICE_MEM_HANDLE_LSW (1) +#define TS_RPC_SERVICE_MEM_HANDLE_MSW (2) +#define TS_RPC_SERVICE_REQ_LEN (3) +#define TS_RPC_SERVICE_CLIENT_ID (4) +#define TS_RPC_SERVICE_RPC_STATUS (1) +#define TS_RPC_SERVICE_STATUS (2) +#define TS_RPC_SERVICE_RESP_LEN (3) + +struct tstee { + struct ffa_device *ffa_dev; + struct tee_device *teedev; + struct tee_shm_pool *pool; +}; + +struct ts_session { + u8 iface_id; +}; + +struct ts_context_data { + struct xarray sess_list; +}; + +#endif /* TSTEE_PRIVATE_H */ diff --git a/drivers/thermal/cpufreq_cooling.c b/drivers/thermal/cpufreq_cooling.c index 9d1b1459700d..280071be30b1 100644 --- a/drivers/thermal/cpufreq_cooling.c +++ b/drivers/thermal/cpufreq_cooling.c @@ -477,7 +477,6 @@ static int cpufreq_set_cur_state(struct thermal_cooling_device *cdev, unsigned long state) { struct cpufreq_cooling_device *cpufreq_cdev = cdev->devdata; - struct cpumask *cpus; unsigned int frequency; int ret; @@ -494,8 +493,6 @@ static int cpufreq_set_cur_state(struct thermal_cooling_device *cdev, ret = freq_qos_update_request(&cpufreq_cdev->qos_req, frequency); if (ret >= 0) { cpufreq_cdev->cpufreq_state = state; - cpus = cpufreq_cdev->policy->related_cpus; - arch_update_thermal_pressure(cpus, frequency); ret = 0; } diff --git a/drivers/tty/serial/8250/8250.h b/drivers/tty/serial/8250/8250.h index 1aa3e55c8b47..6a7b286f6f5a 100644 --- a/drivers/tty/serial/8250/8250.h +++ b/drivers/tty/serial/8250/8250.h @@ -293,9 +293,6 @@ static inline int serial8250_in_MCR(struct uart_8250_port *up) return mctrl; } -bool alpha_jensen(void); -void alpha_jensen_set_mctrl(struct uart_port *port, unsigned int mctrl); - #ifdef CONFIG_SERIAL_8250_PNP int serial8250_pnp_init(void); void serial8250_pnp_exit(void); diff --git a/drivers/tty/serial/8250/8250_alpha.c b/drivers/tty/serial/8250/8250_alpha.c deleted file mode 100644 index 58e70328aa4d..000000000000 --- a/drivers/tty/serial/8250/8250_alpha.c +++ /dev/null @@ -1,21 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0+ - -#include <asm/machvec.h> -#include "8250.h" - -bool alpha_jensen(void) -{ - return !strcmp(alpha_mv.vector_name, "Jensen"); -} - -void alpha_jensen_set_mctrl(struct uart_port *port, unsigned int mctrl) -{ - /* - * Digital did something really horribly wrong with the OUT1 and OUT2 - * lines on Alpha Jensen. The failure mode is that if either is - * cleared, the machine locks up with endless interrupts. - */ - mctrl |= TIOCM_OUT1 | TIOCM_OUT2; - - serial8250_do_set_mctrl(port, mctrl); -} diff --git a/drivers/tty/serial/8250/8250_core.c b/drivers/tty/serial/8250/8250_core.c index b62ad9006780..2504e0455875 100644 --- a/drivers/tty/serial/8250/8250_core.c +++ b/drivers/tty/serial/8250/8250_core.c @@ -508,10 +508,6 @@ static struct uart_8250_port *serial8250_setup_port(int index) up->ops = &univ8250_driver_ops; - if (IS_ENABLED(CONFIG_ALPHA_JENSEN) || - (IS_ENABLED(CONFIG_ALPHA_GENERIC) && alpha_jensen())) - up->port.set_mctrl = alpha_jensen_set_mctrl; - serial8250_set_defaults(up); return up; diff --git a/drivers/tty/serial/8250/Makefile b/drivers/tty/serial/8250/Makefile index ea2e81f58eac..69ac00270547 100644 --- a/drivers/tty/serial/8250/Makefile +++ b/drivers/tty/serial/8250/Makefile @@ -5,8 +5,6 @@ obj-$(CONFIG_SERIAL_8250) += 8250.o 8250_base.o 8250-y := 8250_core.o -8250-$(CONFIG_ALPHA_GENERIC) += 8250_alpha.o -8250-$(CONFIG_ALPHA_JENSEN) += 8250_alpha.o 8250-$(CONFIG_SERIAL_8250_PNP) += 8250_pnp.o 8250_base-y := 8250_port.o 8250_base-$(CONFIG_SERIAL_8250_DMA) += 8250_dma.o diff --git a/drivers/vfio/pci/vfio_pci.c b/drivers/vfio/pci/vfio_pci.c index cb5b7f865d58..e727941f589d 100644 --- a/drivers/vfio/pci/vfio_pci.c +++ b/drivers/vfio/pci/vfio_pci.c @@ -71,6 +71,8 @@ static bool vfio_pci_dev_in_denylist(struct pci_dev *pdev) case PCI_DEVICE_ID_INTEL_QAT_C62X_VF: case PCI_DEVICE_ID_INTEL_QAT_DH895XCC: case PCI_DEVICE_ID_INTEL_QAT_DH895XCC_VF: + case PCI_DEVICE_ID_INTEL_DSA_SPR0: + case PCI_DEVICE_ID_INTEL_IAX_SPR0: return true; default: return false; diff --git a/drivers/vhost/net.c b/drivers/vhost/net.c index c64ded183f8d..f16279351db5 100644 --- a/drivers/vhost/net.c +++ b/drivers/vhost/net.c @@ -380,7 +380,7 @@ static void vhost_zerocopy_signal_used(struct vhost_net *net, } } -static void vhost_zerocopy_callback(struct sk_buff *skb, +static void vhost_zerocopy_complete(struct sk_buff *skb, struct ubuf_info *ubuf_base, bool success) { struct ubuf_info_msgzc *ubuf = uarg_to_msgzc(ubuf_base); @@ -408,6 +408,10 @@ static void vhost_zerocopy_callback(struct sk_buff *skb, rcu_read_unlock_bh(); } +static const struct ubuf_info_ops vhost_ubuf_ops = { + .complete = vhost_zerocopy_complete, +}; + static inline unsigned long busy_clock(void) { return local_clock() >> 10; @@ -879,7 +883,7 @@ static void handle_tx_zerocopy(struct vhost_net *net, struct socket *sock) vq->heads[nvq->upend_idx].len = VHOST_DMA_IN_PROGRESS; ubuf->ctx = nvq->ubufs; ubuf->desc = nvq->upend_idx; - ubuf->ubuf.callback = vhost_zerocopy_callback; + ubuf->ubuf.ops = &vhost_ubuf_ops; ubuf->ubuf.flags = SKBFL_ZEROCOPY_FRAG; refcount_set(&ubuf->ubuf.refcnt, 1); msg.msg_control = &ctl; diff --git a/drivers/virt/acrn/ioreq.c b/drivers/virt/acrn/ioreq.c index 29e1ef1915fd..e94358239a4b 100644 --- a/drivers/virt/acrn/ioreq.c +++ b/drivers/virt/acrn/ioreq.c @@ -433,7 +433,7 @@ struct acrn_ioreq_client *acrn_ioreq_client_create(struct acrn_vm *vm, client->priv = priv; client->is_default = is_default; if (name) - strncpy(client->name, name, sizeof(client->name) - 1); + strscpy(client->name, name); rwlock_init(&client->range_lock); INIT_LIST_HEAD(&client->range_list); init_waitqueue_head(&client->wq); diff --git a/drivers/virt/coco/sev-guest/sev-guest.c b/drivers/virt/coco/sev-guest/sev-guest.c index 87f241825bc3..654290a8e1ba 100644 --- a/drivers/virt/coco/sev-guest/sev-guest.c +++ b/drivers/virt/coco/sev-guest/sev-guest.c @@ -59,7 +59,7 @@ struct snp_guest_dev { */ struct snp_guest_msg secret_request, secret_response; - struct snp_secrets_page_layout *layout; + struct snp_secrets_page *secrets; struct snp_req_data input; union { struct snp_report_req report; @@ -743,26 +743,26 @@ static const struct file_operations snp_guest_fops = { .unlocked_ioctl = snp_guest_ioctl, }; -static u8 *get_vmpck(int id, struct snp_secrets_page_layout *layout, u32 **seqno) +static u8 *get_vmpck(int id, struct snp_secrets_page *secrets, u32 **seqno) { u8 *key = NULL; switch (id) { case 0: - *seqno = &layout->os_area.msg_seqno_0; - key = layout->vmpck0; + *seqno = &secrets->os_area.msg_seqno_0; + key = secrets->vmpck0; break; case 1: - *seqno = &layout->os_area.msg_seqno_1; - key = layout->vmpck1; + *seqno = &secrets->os_area.msg_seqno_1; + key = secrets->vmpck1; break; case 2: - *seqno = &layout->os_area.msg_seqno_2; - key = layout->vmpck2; + *seqno = &secrets->os_area.msg_seqno_2; + key = secrets->vmpck2; break; case 3: - *seqno = &layout->os_area.msg_seqno_3; - key = layout->vmpck3; + *seqno = &secrets->os_area.msg_seqno_3; + key = secrets->vmpck3; break; default: break; @@ -897,8 +897,8 @@ static void unregister_sev_tsm(void *data) static int __init sev_guest_probe(struct platform_device *pdev) { - struct snp_secrets_page_layout *layout; struct sev_guest_platform_data *data; + struct snp_secrets_page *secrets; struct device *dev = &pdev->dev; struct snp_guest_dev *snp_dev; struct miscdevice *misc; @@ -916,7 +916,7 @@ static int __init sev_guest_probe(struct platform_device *pdev) if (!mapping) return -ENODEV; - layout = (__force void *)mapping; + secrets = (__force void *)mapping; ret = -ENOMEM; snp_dev = devm_kzalloc(&pdev->dev, sizeof(struct snp_guest_dev), GFP_KERNEL); @@ -924,7 +924,7 @@ static int __init sev_guest_probe(struct platform_device *pdev) goto e_unmap; ret = -EINVAL; - snp_dev->vmpck = get_vmpck(vmpck_id, layout, &snp_dev->os_area_msg_seqno); + snp_dev->vmpck = get_vmpck(vmpck_id, secrets, &snp_dev->os_area_msg_seqno); if (!snp_dev->vmpck) { dev_err(dev, "invalid vmpck id %d\n", vmpck_id); goto e_unmap; @@ -938,7 +938,7 @@ static int __init sev_guest_probe(struct platform_device *pdev) platform_set_drvdata(pdev, snp_dev); snp_dev->dev = dev; - snp_dev->layout = layout; + snp_dev->secrets = secrets; /* Allocate the shared page used for the request and response message. */ snp_dev->request = alloc_shared_pages(dev, sizeof(struct snp_guest_msg)); diff --git a/drivers/zorro/zorro.c b/drivers/zorro/zorro.c index 1b9928648583..2196474ce6ef 100644 --- a/drivers/zorro/zorro.c +++ b/drivers/zorro/zorro.c @@ -117,17 +117,13 @@ static struct resource __init *zorro_find_parent_resource( int i; for (i = 0; i < bridge->num_resources; i++) { - struct resource *r = &bridge->resource[i]; - - if (zorro_resource_start(z) >= r->start && - zorro_resource_end(z) <= r->end) - return r; + if (resource_contains(&bridge->resource[i], &z->resource)) + return &bridge->resource[i]; } + return &iomem_resource; } - - static int __init amiga_zorro_probe(struct platform_device *pdev) { struct zorro_bus *bus; @@ -176,9 +172,7 @@ static int __init amiga_zorro_probe(struct platform_device *pdev) z->slotsize = zi->slotsize; sprintf(z->name, "Zorro device %08x", z->id); zorro_name_device(z); - z->resource.start = zi->boardaddr; - z->resource.end = zi->boardaddr + zi->boardsize - 1; - z->resource.name = z->name; + z->resource = DEFINE_RES_MEM_NAMED(zi->boardaddr, zi->boardsize, z->name); r = zorro_find_parent_resource(pdev, z); error = request_resource(r, &z->resource); if (error && !(z->rom.er_Type & ERTF_MEMLIST)) |