aboutsummaryrefslogtreecommitdiffstatshomepage
diff options
context:
space:
mode:
authorLinus Torvalds <torvalds@linux-foundation.org>2020-06-07 10:59:32 -0700
committerLinus Torvalds <torvalds@linux-foundation.org>2020-06-07 10:59:32 -0700
commit9aa900c8094dba7a60dc805ecec1e9f720744ba1 (patch)
tree3cc09a579f8ea6d3a182076ba722f7c1648e682d
parentMerge tag 'driver-core-5.8-rc1' of git://git.kernel.org/pub/scm/linux/kernel/git/gregkh/driver-core (diff)
parenthabanalabs: correctly cast u64 to void* (diff)
downloadwireguard-linux-9aa900c8094dba7a60dc805ecec1e9f720744ba1.tar.xz
wireguard-linux-9aa900c8094dba7a60dc805ecec1e9f720744ba1.zip
Merge tag 'char-misc-5.8-rc1' of git://git.kernel.org/pub/scm/linux/kernel/git/gregkh/char-misc
Pull char/misc driver updates from Greg KH: "Here is the large set of char/misc driver patches for 5.8-rc1 Included in here are: - habanalabs driver updates, loads - mhi bus driver updates - extcon driver updates - clk driver updates (approved by the clock maintainer) - firmware driver updates - fpga driver updates - gnss driver updates - coresight driver updates - interconnect driver updates - parport driver updates (it's still alive!) - nvmem driver updates - soundwire driver updates - visorbus driver updates - w1 driver updates - various misc driver updates In short, loads of different driver subsystem updates along with the drivers as well. All have been in linux-next for a while with no reported issues" * tag 'char-misc-5.8-rc1' of git://git.kernel.org/pub/scm/linux/kernel/git/gregkh/char-misc: (233 commits) habanalabs: correctly cast u64 to void* habanalabs: initialize variable to default value extcon: arizona: Fix runtime PM imbalance on error extcon: max14577: Add proper dt-compatible strings extcon: adc-jack: Fix an error handling path in 'adc_jack_probe()' extcon: remove redundant assignment to variable idx w1: omap-hdq: print dev_err if irq flags are not cleared w1: omap-hdq: fix interrupt handling which did show spurious timeouts w1: omap-hdq: fix return value to be -1 if there is a timeout w1: omap-hdq: cleanup to add missing newline for some dev_dbg /dev/mem: Revoke mappings when a driver claims the region misc: xilinx-sdfec: convert get_user_pages() --> pin_user_pages() misc: xilinx-sdfec: cleanup return value in xsdfec_table_write() misc: xilinx-sdfec: improve get_user_pages_fast() error handling nvmem: qfprom: remove incorrect write support habanalabs: handle MMU cache invalidation timeout habanalabs: don't allow hard reset with open processes habanalabs: GAUDI does not support soft-reset habanalabs: add print for soft reset due to event habanalabs: improve MMU cache invalidation code ...
-rw-r--r--Documentation/ABI/stable/sysfs-driver-firmware-zynqmp103
-rw-r--r--Documentation/ABI/testing/debugfs-driver-habanalabs17
-rw-r--r--Documentation/ABI/testing/sysfs-bus-event_source-devices-dfl_fme104
-rw-r--r--Documentation/ABI/testing/sysfs-bus-soundwire-master23
-rw-r--r--Documentation/ABI/testing/sysfs-bus-soundwire-slave91
-rw-r--r--Documentation/ABI/testing/sysfs-driver-habanalabs17
-rw-r--r--Documentation/ABI/testing/sysfs-driver-w1_therm116
-rw-r--r--Documentation/devicetree/bindings/firmware/intel,stratix10-svc.txt2
-rw-r--r--Documentation/devicetree/bindings/fpga/intel-stratix10-soc-fpga-mgr.txt3
-rw-r--r--Documentation/devicetree/bindings/interconnect/fsl,imx8m-noc.yaml101
-rw-r--r--Documentation/driver-api/soundwire/stream.rst89
-rw-r--r--Documentation/driver-api/soundwire/summary.rst7
-rw-r--r--Documentation/fpga/dfl.rst84
-rw-r--r--Documentation/trace/coresight/coresight-ect.rst5
-rw-r--r--Documentation/trace/coresight/coresight.rst85
-rw-r--r--Documentation/w1/slaves/w1_therm.rst50
-rw-r--r--arch/arm64/boot/dts/intel/socfpga_agilex.dtsi4
-rw-r--r--drivers/android/binderfs.c4
-rw-r--r--drivers/bus/mhi/core/boot.c75
-rw-r--r--drivers/bus/mhi/core/init.c8
-rw-r--r--drivers/bus/mhi/core/internal.h9
-rw-r--r--drivers/bus/mhi/core/main.c197
-rw-r--r--drivers/bus/mhi/core/pm.c229
-rw-r--r--drivers/char/mem.c101
-rw-r--r--drivers/char/tlclk.c17
-rw-r--r--drivers/clk/zynqmp/clk-gate-zynqmp.c9
-rw-r--r--drivers/clk/zynqmp/clk-mux-zynqmp.c6
-rw-r--r--drivers/clk/zynqmp/clkc.c17
-rw-r--r--drivers/clk/zynqmp/divider.c12
-rw-r--r--drivers/clk/zynqmp/pll.c29
-rw-r--r--drivers/crypto/xilinx/zynqmp-aes-gcm.c22
-rw-r--r--drivers/dca/dca-sysfs.c4
-rw-r--r--drivers/extcon/extcon-adc-jack.c3
-rw-r--r--drivers/extcon/extcon-arizona.c17
-rw-r--r--drivers/extcon/extcon-max14577.c10
-rw-r--r--drivers/extcon/extcon.c2
-rw-r--r--drivers/firmware/stratix10-rsu.c10
-rw-r--r--drivers/firmware/stratix10-svc.c62
-rw-r--r--drivers/firmware/xilinx/zynqmp-debug.c5
-rw-r--r--drivers/firmware/xilinx/zynqmp.c607
-rw-r--r--drivers/fpga/Kconfig2
-rw-r--r--drivers/fpga/Makefile1
-rw-r--r--drivers/fpga/dfl-afu-dma-region.c4
-rw-r--r--drivers/fpga/dfl-afu-main.c35
-rw-r--r--drivers/fpga/dfl-fme-main.c23
-rw-r--r--drivers/fpga/dfl-fme-perf.c1020
-rw-r--r--drivers/fpga/dfl-fme.h2
-rw-r--r--drivers/fpga/dfl.c15
-rw-r--r--drivers/fpga/dfl.h39
-rw-r--r--drivers/fpga/ice40-spi.c10
-rw-r--r--drivers/fpga/machxo2-spi.c12
-rw-r--r--drivers/fpga/stratix10-soc.c28
-rw-r--r--drivers/fpga/zynqmp-fpga.c14
-rw-r--r--drivers/gnss/serial.h2
-rw-r--r--drivers/gnss/sirf.c8
-rw-r--r--drivers/greybus/Kconfig6
-rw-r--r--drivers/greybus/arpc.h2
-rw-r--r--drivers/hwtracing/coresight/Makefile3
-rw-r--r--drivers/hwtracing/coresight/coresight-cti-platform.c11
-rw-r--r--drivers/hwtracing/coresight/coresight-cti-sysfs.c16
-rw-r--r--drivers/hwtracing/coresight/coresight-cti.c232
-rw-r--r--drivers/hwtracing/coresight/coresight-cti.h8
-rw-r--r--drivers/hwtracing/coresight/coresight-etb10.c2
-rw-r--r--drivers/hwtracing/coresight/coresight-etm3x.c2
-rw-r--r--drivers/hwtracing/coresight/coresight-etm4x-sysfs.c2
-rw-r--r--drivers/hwtracing/coresight/coresight-etm4x.c33
-rw-r--r--drivers/hwtracing/coresight/coresight-platform.c91
-rw-r--r--drivers/hwtracing/coresight/coresight-priv.h21
-rw-r--r--drivers/hwtracing/coresight/coresight-sysfs.c204
-rw-r--r--drivers/hwtracing/coresight/coresight-tmc-etf.c16
-rw-r--r--drivers/hwtracing/coresight/coresight-tmc.c2
-rw-r--r--drivers/hwtracing/coresight/coresight.c82
-rw-r--r--drivers/interconnect/Kconfig3
-rw-r--r--drivers/interconnect/Makefile1
-rw-r--r--drivers/interconnect/core.c143
-rw-r--r--drivers/interconnect/imx/Kconfig17
-rw-r--r--drivers/interconnect/imx/Makefile9
-rw-r--r--drivers/interconnect/imx/imx.c284
-rw-r--r--drivers/interconnect/imx/imx.h61
-rw-r--r--drivers/interconnect/imx/imx8mm.c105
-rw-r--r--drivers/interconnect/imx/imx8mn.c94
-rw-r--r--drivers/interconnect/imx/imx8mq.c103
-rw-r--r--drivers/interconnect/internal.h2
-rw-r--r--drivers/misc/cardreader/rts5249.c29
-rw-r--r--drivers/misc/cardreader/rts5260.c26
-rw-r--r--drivers/misc/cardreader/rts5261.c47
-rw-r--r--drivers/misc/cardreader/rtsx_pcr.c43
-rw-r--r--drivers/misc/cardreader/rtsx_pcr.h1
-rw-r--r--drivers/misc/fastrpc.c13
-rw-r--r--drivers/misc/genwqe/card_utils.c42
-rw-r--r--drivers/misc/habanalabs/Makefile3
-rw-r--r--drivers/misc/habanalabs/command_buffer.c28
-rw-r--r--drivers/misc/habanalabs/command_submission.c385
-rw-r--r--drivers/misc/habanalabs/context.c8
-rw-r--r--drivers/misc/habanalabs/debugfs.c116
-rw-r--r--drivers/misc/habanalabs/device.c53
-rw-r--r--drivers/misc/habanalabs/firmware_if.c297
-rw-r--r--drivers/misc/habanalabs/gaudi/Makefile5
-rw-r--r--drivers/misc/habanalabs/gaudi/gaudi.c6748
-rw-r--r--drivers/misc/habanalabs/gaudi/gaudiP.h261
-rw-r--r--drivers/misc/habanalabs/gaudi/gaudi_coresight.c884
-rw-r--r--drivers/misc/habanalabs/gaudi/gaudi_hwmgr.c121
-rw-r--r--drivers/misc/habanalabs/gaudi/gaudi_security.c9090
-rw-r--r--drivers/misc/habanalabs/goya/goya.c345
-rw-r--r--drivers/misc/habanalabs/goya/goyaP.h12
-rw-r--r--drivers/misc/habanalabs/goya/goya_coresight.c2
-rw-r--r--drivers/misc/habanalabs/goya/goya_security.c100
-rw-r--r--drivers/misc/habanalabs/habanalabs.h187
-rw-r--r--drivers/misc/habanalabs/habanalabs_drv.c14
-rw-r--r--drivers/misc/habanalabs/habanalabs_ioctl.c21
-rw-r--r--drivers/misc/habanalabs/hw_queue.c118
-rw-r--r--drivers/misc/habanalabs/hwmon.c75
-rw-r--r--drivers/misc/habanalabs/include/armcp_if.h43
-rw-r--r--drivers/misc/habanalabs/include/gaudi/asic_reg/cpu_if_regs.h174
-rw-r--r--drivers/misc/habanalabs/include/gaudi/asic_reg/dma0_core_masks.h348
-rw-r--r--drivers/misc/habanalabs/include/gaudi/asic_reg/dma0_core_regs.h156
-rw-r--r--drivers/misc/habanalabs/include/gaudi/asic_reg/dma0_qm_masks.h800
-rw-r--r--drivers/misc/habanalabs/include/gaudi/asic_reg/dma0_qm_regs.h834
-rw-r--r--drivers/misc/habanalabs/include/gaudi/asic_reg/dma1_core_regs.h156
-rw-r--r--drivers/misc/habanalabs/include/gaudi/asic_reg/dma1_qm_regs.h834
-rw-r--r--drivers/misc/habanalabs/include/gaudi/asic_reg/dma2_core_regs.h156
-rw-r--r--drivers/misc/habanalabs/include/gaudi/asic_reg/dma2_qm_regs.h834
-rw-r--r--drivers/misc/habanalabs/include/gaudi/asic_reg/dma3_core_regs.h156
-rw-r--r--drivers/misc/habanalabs/include/gaudi/asic_reg/dma3_qm_regs.h834
-rw-r--r--drivers/misc/habanalabs/include/gaudi/asic_reg/dma4_core_regs.h156
-rw-r--r--drivers/misc/habanalabs/include/gaudi/asic_reg/dma4_qm_regs.h834
-rw-r--r--drivers/misc/habanalabs/include/gaudi/asic_reg/dma5_core_regs.h156
-rw-r--r--drivers/misc/habanalabs/include/gaudi/asic_reg/dma5_qm_regs.h834
-rw-r--r--drivers/misc/habanalabs/include/gaudi/asic_reg/dma6_core_regs.h156
-rw-r--r--drivers/misc/habanalabs/include/gaudi/asic_reg/dma6_qm_regs.h834
-rw-r--r--drivers/misc/habanalabs/include/gaudi/asic_reg/dma7_core_regs.h156
-rw-r--r--drivers/misc/habanalabs/include/gaudi/asic_reg/dma7_qm_regs.h834
-rw-r--r--drivers/misc/habanalabs/include/gaudi/asic_reg/dma_if_e_n_down_ch0_regs.h896
-rw-r--r--drivers/misc/habanalabs/include/gaudi/asic_reg/dma_if_e_n_down_ch1_regs.h896
-rw-r--r--drivers/misc/habanalabs/include/gaudi/asic_reg/dma_if_e_n_regs.h860
-rw-r--r--drivers/misc/habanalabs/include/gaudi/asic_reg/dma_if_e_s_down_ch0_regs.h896
-rw-r--r--drivers/misc/habanalabs/include/gaudi/asic_reg/dma_if_e_s_down_ch1_regs.h896
-rw-r--r--drivers/misc/habanalabs/include/gaudi/asic_reg/dma_if_e_s_regs.h860
-rw-r--r--drivers/misc/habanalabs/include/gaudi/asic_reg/dma_if_w_n_down_ch0_regs.h896
-rw-r--r--drivers/misc/habanalabs/include/gaudi/asic_reg/dma_if_w_n_down_ch1_regs.h896
-rw-r--r--drivers/misc/habanalabs/include/gaudi/asic_reg/dma_if_w_n_regs.h860
-rw-r--r--drivers/misc/habanalabs/include/gaudi/asic_reg/dma_if_w_s_down_ch0_regs.h896
-rw-r--r--drivers/misc/habanalabs/include/gaudi/asic_reg/dma_if_w_s_down_ch1_regs.h896
-rw-r--r--drivers/misc/habanalabs/include/gaudi/asic_reg/dma_if_w_s_regs.h860
-rw-r--r--drivers/misc/habanalabs/include/gaudi/asic_reg/gaudi_blocks.h4974
-rw-r--r--drivers/misc/habanalabs/include/gaudi/asic_reg/gaudi_regs.h299
-rw-r--r--drivers/misc/habanalabs/include/gaudi/asic_reg/mme0_ctrl_regs.h1456
-rw-r--r--drivers/misc/habanalabs/include/gaudi/asic_reg/mme0_qm_masks.h800
-rw-r--r--drivers/misc/habanalabs/include/gaudi/asic_reg/mme0_qm_regs.h834
-rw-r--r--drivers/misc/habanalabs/include/gaudi/asic_reg/mme1_ctrl_regs.h1456
-rw-r--r--drivers/misc/habanalabs/include/gaudi/asic_reg/mme2_ctrl_regs.h1456
-rw-r--r--drivers/misc/habanalabs/include/gaudi/asic_reg/mme2_qm_regs.h834
-rw-r--r--drivers/misc/habanalabs/include/gaudi/asic_reg/mme3_ctrl_regs.h1456
-rw-r--r--drivers/misc/habanalabs/include/gaudi/asic_reg/mmu_up_regs.h72
-rw-r--r--drivers/misc/habanalabs/include/gaudi/asic_reg/nif_rtr_ctrl_0_regs.h896
-rw-r--r--drivers/misc/habanalabs/include/gaudi/asic_reg/nif_rtr_ctrl_1_regs.h896
-rw-r--r--drivers/misc/habanalabs/include/gaudi/asic_reg/nif_rtr_ctrl_2_regs.h896
-rw-r--r--drivers/misc/habanalabs/include/gaudi/asic_reg/nif_rtr_ctrl_3_regs.h896
-rw-r--r--drivers/misc/habanalabs/include/gaudi/asic_reg/nif_rtr_ctrl_4_regs.h896
-rw-r--r--drivers/misc/habanalabs/include/gaudi/asic_reg/nif_rtr_ctrl_5_regs.h896
-rw-r--r--drivers/misc/habanalabs/include/gaudi/asic_reg/nif_rtr_ctrl_6_regs.h896
-rw-r--r--drivers/misc/habanalabs/include/gaudi/asic_reg/nif_rtr_ctrl_7_regs.h896
-rw-r--r--drivers/misc/habanalabs/include/gaudi/asic_reg/psoc_etr_regs.h114
-rw-r--r--drivers/misc/habanalabs/include/gaudi/asic_reg/psoc_global_conf_masks.h502
-rw-r--r--drivers/misc/habanalabs/include/gaudi/asic_reg/psoc_global_conf_regs.h1062
-rw-r--r--drivers/misc/habanalabs/include/gaudi/asic_reg/psoc_hbm_pll_regs.h114
-rw-r--r--drivers/misc/habanalabs/include/gaudi/asic_reg/psoc_pci_pll_regs.h114
-rw-r--r--drivers/misc/habanalabs/include/gaudi/asic_reg/psoc_timestamp_regs.h56
-rw-r--r--drivers/misc/habanalabs/include/gaudi/asic_reg/sif_rtr_ctrl_0_regs.h896
-rw-r--r--drivers/misc/habanalabs/include/gaudi/asic_reg/sif_rtr_ctrl_1_regs.h896
-rw-r--r--drivers/misc/habanalabs/include/gaudi/asic_reg/sif_rtr_ctrl_2_regs.h896
-rw-r--r--drivers/misc/habanalabs/include/gaudi/asic_reg/sif_rtr_ctrl_3_regs.h896
-rw-r--r--drivers/misc/habanalabs/include/gaudi/asic_reg/sif_rtr_ctrl_4_regs.h896
-rw-r--r--drivers/misc/habanalabs/include/gaudi/asic_reg/sif_rtr_ctrl_5_regs.h896
-rw-r--r--drivers/misc/habanalabs/include/gaudi/asic_reg/sif_rtr_ctrl_6_regs.h896
-rw-r--r--drivers/misc/habanalabs/include/gaudi/asic_reg/sif_rtr_ctrl_7_regs.h896
-rw-r--r--drivers/misc/habanalabs/include/gaudi/asic_reg/stlb_regs.h82
-rw-r--r--drivers/misc/habanalabs/include/gaudi/asic_reg/tpc0_cfg_masks.h2578
-rw-r--r--drivers/misc/habanalabs/include/gaudi/asic_reg/tpc0_cfg_regs.h1226
-rw-r--r--drivers/misc/habanalabs/include/gaudi/asic_reg/tpc0_qm_masks.h800
-rw-r--r--drivers/misc/habanalabs/include/gaudi/asic_reg/tpc0_qm_regs.h834
-rw-r--r--drivers/misc/habanalabs/include/gaudi/asic_reg/tpc1_cfg_regs.h1226
-rw-r--r--drivers/misc/habanalabs/include/gaudi/asic_reg/tpc1_qm_regs.h834
-rw-r--r--drivers/misc/habanalabs/include/gaudi/asic_reg/tpc2_cfg_regs.h1226
-rw-r--r--drivers/misc/habanalabs/include/gaudi/asic_reg/tpc2_qm_regs.h834
-rw-r--r--drivers/misc/habanalabs/include/gaudi/asic_reg/tpc3_cfg_regs.h1226
-rw-r--r--drivers/misc/habanalabs/include/gaudi/asic_reg/tpc3_qm_regs.h834
-rw-r--r--drivers/misc/habanalabs/include/gaudi/asic_reg/tpc4_cfg_regs.h1226
-rw-r--r--drivers/misc/habanalabs/include/gaudi/asic_reg/tpc4_qm_regs.h834
-rw-r--r--drivers/misc/habanalabs/include/gaudi/asic_reg/tpc5_cfg_regs.h1226
-rw-r--r--drivers/misc/habanalabs/include/gaudi/asic_reg/tpc5_qm_regs.h834
-rw-r--r--drivers/misc/habanalabs/include/gaudi/asic_reg/tpc6_cfg_regs.h1226
-rw-r--r--drivers/misc/habanalabs/include/gaudi/asic_reg/tpc6_qm_regs.h834
-rw-r--r--drivers/misc/habanalabs/include/gaudi/asic_reg/tpc7_cfg_regs.h1226
-rw-r--r--drivers/misc/habanalabs/include/gaudi/asic_reg/tpc7_qm_regs.h834
-rw-r--r--drivers/misc/habanalabs/include/gaudi/gaudi.h59
-rw-r--r--drivers/misc/habanalabs/include/gaudi/gaudi_async_events.h310
-rw-r--r--drivers/misc/habanalabs/include/gaudi/gaudi_async_ids_map_extended.h694
-rw-r--r--drivers/misc/habanalabs/include/gaudi/gaudi_coresight.h367
-rw-r--r--drivers/misc/habanalabs/include/gaudi/gaudi_fw_if.h36
-rw-r--r--drivers/misc/habanalabs/include/gaudi/gaudi_masks.h458
-rw-r--r--drivers/misc/habanalabs/include/gaudi/gaudi_packets.h212
-rw-r--r--drivers/misc/habanalabs/include/gaudi/gaudi_reg_map.h27
-rw-r--r--drivers/misc/habanalabs/include/goya/asic_reg/goya_masks.h3
-rw-r--r--drivers/misc/habanalabs/include/goya/asic_reg/goya_regs.h1
-rw-r--r--drivers/misc/habanalabs/include/goya/asic_reg/psoc_timestamp_regs.h56
-rw-r--r--drivers/misc/habanalabs/include/goya/goya_reg_map.h43
-rw-r--r--drivers/misc/habanalabs/include/hl_boot_if.h58
-rw-r--r--drivers/misc/habanalabs/include/hw_ip/mmu/mmu_general.h2
-rw-r--r--drivers/misc/habanalabs/include/hw_ip/mmu/mmu_v1_1.h16
-rw-r--r--drivers/misc/habanalabs/memory.c37
-rw-r--r--drivers/misc/habanalabs/pci.c63
-rw-r--r--drivers/misc/habanalabs/sysfs.c17
-rw-r--r--drivers/misc/lkdtm/bugs.c2
-rw-r--r--drivers/misc/mic/scif/scif_nodeqp.c2
-rw-r--r--drivers/misc/mic/scif/scif_rma.c26
-rw-r--r--drivers/misc/pci_endpoint_test.c4
-rw-r--r--drivers/misc/sgi-xp/xpc_main.c10
-rw-r--r--drivers/misc/sgi-xp/xpnet.c8
-rw-r--r--drivers/misc/xilinx_sdfec.c61
-rw-r--r--drivers/mmc/host/sdhci-of-arasan.c38
-rw-r--r--drivers/nvmem/core.c104
-rw-r--r--drivers/nvmem/imx-ocotp.c9
-rw-r--r--drivers/nvmem/jz4780-efuse.c4
-rw-r--r--drivers/nvmem/qfprom.c14
-rw-r--r--drivers/nvmem/zynqmp_nvmem.c11
-rw-r--r--drivers/parport/daisy.c29
-rw-r--r--drivers/parport/ieee1284.c94
-rw-r--r--drivers/parport/ieee1284_ops.c70
-rw-r--r--drivers/parport/parport_amiga.c22
-rw-r--r--drivers/parport/parport_atari.c2
-rw-r--r--drivers/parport/parport_cs.c6
-rw-r--r--drivers/parport/parport_gsc.c25
-rw-r--r--drivers/parport/parport_gsc.h21
-rw-r--r--drivers/parport/parport_ip32.c117
-rw-r--r--drivers/parport/parport_mfc3.c21
-rw-r--r--drivers/parport/parport_pc.c263
-rw-r--r--drivers/parport/parport_sunbpp.c2
-rw-r--r--drivers/parport/probe.c34
-rw-r--r--drivers/parport/procfs.c6
-rw-r--r--drivers/parport/share.c292
-rw-r--r--drivers/reset/reset-zynqmp.c26
-rw-r--r--drivers/slimbus/core.c6
-rw-r--r--drivers/slimbus/qcom-ngd-ctrl.c5
-rw-r--r--drivers/soc/xilinx/zynqmp_pm_domains.c26
-rw-r--r--drivers/soc/xilinx/zynqmp_power.c17
-rw-r--r--drivers/soundwire/Makefile8
-rw-r--r--drivers/soundwire/bus.c71
-rw-r--r--drivers/soundwire/bus.h4
-rw-r--r--drivers/soundwire/bus_type.c22
-rw-r--r--drivers/soundwire/cadence_master.c8
-rw-r--r--drivers/soundwire/debugfs.c2
-rw-r--r--drivers/soundwire/intel.c9
-rw-r--r--drivers/soundwire/intel_init.c4
-rw-r--r--drivers/soundwire/master.c172
-rw-r--r--drivers/soundwire/mipi_disco.c11
-rw-r--r--drivers/soundwire/qcom.c34
-rw-r--r--drivers/soundwire/slave.c10
-rw-r--r--drivers/soundwire/sysfs_local.h14
-rw-r--r--drivers/soundwire/sysfs_slave.c214
-rw-r--r--drivers/soundwire/sysfs_slave_dpn.c300
-rw-r--r--drivers/spi/spi-zynqmp-gqspi.c5
-rw-r--r--drivers/uio/uio.c2
-rw-r--r--drivers/uio/uio_dmem_genirq.c3
-rw-r--r--drivers/uio/uio_hv_generic.c1
-rw-r--r--drivers/visorbus/controlvmchannel.h2
-rw-r--r--drivers/visorbus/vbuschannel.h2
-rw-r--r--drivers/visorbus/visorbus_private.h2
-rw-r--r--drivers/w1/masters/omap_hdq.c82
-rw-r--r--drivers/w1/slaves/w1_ds2430.c2
-rw-r--r--drivers/w1/slaves/w1_therm.c1624
-rw-r--r--include/dt-bindings/interconnect/imx8mm.h50
-rw-r--r--include/dt-bindings/interconnect/imx8mn.h41
-rw-r--r--include/dt-bindings/interconnect/imx8mq.h48
-rw-r--r--include/linux/coresight.h32
-rw-r--r--include/linux/cpuhotplug.h1
-rw-r--r--include/linux/firmware/intel/stratix10-smc.h49
-rw-r--r--include/linux/firmware/intel/stratix10-svc-client.h71
-rw-r--r--include/linux/firmware/xlnx-zynqmp.h237
-rw-r--r--include/linux/greybus/greybus_protocols.h44
-rw-r--r--include/linux/interconnect.h25
-rw-r--r--include/linux/ioport.h6
-rw-r--r--include/linux/mhi.h23
-rw-r--r--include/linux/parport.h16
-rw-r--r--include/linux/rtsx_pci.h25
-rw-r--r--include/linux/soundwire/sdw.h32
-rw-r--r--include/linux/soundwire/sdw_type.h9
-rw-r--r--include/uapi/linux/magic.h1
-rw-r--r--include/uapi/misc/habanalabs.h245
-rw-r--r--kernel/resource.c5
290 files changed, 98743 insertions, 2629 deletions
diff --git a/Documentation/ABI/stable/sysfs-driver-firmware-zynqmp b/Documentation/ABI/stable/sysfs-driver-firmware-zynqmp
new file mode 100644
index 000000000000..00fa04c76ff3
--- /dev/null
+++ b/Documentation/ABI/stable/sysfs-driver-firmware-zynqmp
@@ -0,0 +1,103 @@
+What: /sys/devices/platform/firmware\:zynqmp-firmware/ggs*
+Date: March 2020
+KernelVersion: 5.6
+Contact: "Jolly Shah" <jollys@xilinx.com>
+Description:
+ Read/Write PMU global general storage register value,
+ GLOBAL_GEN_STORAGE{0:3}.
+ Global general storage register that can be used
+ by system to pass information between masters.
+
+ The register is reset during system or power-on
+ resets. Three registers are used by the FSBL and
+ other Xilinx software products: GLOBAL_GEN_STORAGE{4:6}.
+
+ Usage:
+ # cat /sys/devices/platform/firmware\:zynqmp-firmware/ggs0
+ # echo <value> > /sys/devices/platform/firmware\:zynqmp-firmware/ggs0
+
+ Example:
+ # cat /sys/devices/platform/firmware\:zynqmp-firmware/ggs0
+ # echo 0x1234ABCD > /sys/devices/platform/firmware\:zynqmp-firmware/ggs0
+
+Users: Xilinx
+
+What: /sys/devices/platform/firmware\:zynqmp-firmware/pggs*
+Date: March 2020
+KernelVersion: 5.6
+Contact: "Jolly Shah" <jollys@xilinx.com>
+Description:
+ Read/Write PMU persistent global general storage register
+ value, PERS_GLOB_GEN_STORAGE{0:3}.
+ Persistent global general storage register that
+ can be used by system to pass information between
+ masters.
+
+ This register is only reset by the power-on reset
+ and maintains its value through a system reset.
+ Four registers are used by the FSBL and other Xilinx
+ software products: PERS_GLOB_GEN_STORAGE{4:7}.
+ Register is reset only by a POR reset.
+
+ Usage:
+ # cat /sys/devices/platform/firmware\:zynqmp-firmware/pggs0
+ # echo <value> > /sys/devices/platform/firmware\:zynqmp-firmware/pggs0
+
+ Example:
+ # cat /sys/devices/platform/firmware\:zynqmp-firmware/pggs0
+ # echo 0x1234ABCD > /sys/devices/platform/firmware\:zynqmp-firmware/pggs0
+
+Users: Xilinx
+
+What: /sys/devices/platform/firmware\:zynqmp-firmware/shutdown_scope
+Date: March 2020
+KernelVersion: 5.6
+Contact: "Jolly Shah" <jollys@xilinx.com>
+Description:
+ This sysfs interface allows to set the shutdown scope for the
+ next shutdown request. When the next shutdown is performed, the
+ platform specific portion of PSCI-system_off can use the chosen
+ shutdown scope.
+
+ Following are available shutdown scopes(subtypes):
+
+ subsystem: Only the APU along with all of its peripherals
+ not used by other processing units will be
+ shut down. This may result in the FPD power
+ domain being shut down provided that no other
+ processing unit uses FPD peripherals or DRAM.
+ ps_only: The complete PS will be shut down, including the
+ RPU, PMU, etc. Only the PL domain (FPGA)
+ remains untouched.
+ system: The complete system/device is shut down.
+
+ Usage:
+ # cat /sys/devices/platform/firmware\:zynqmp-firmware/shutdown_scope
+ # echo <scope> > /sys/devices/platform/firmware\:zynqmp-firmware/shutdown_scope
+
+ Example:
+ # cat /sys/devices/platform/firmware\:zynqmp-firmware/shutdown_scope
+ # echo "subsystem" > /sys/devices/platform/firmware\:zynqmp-firmware/shutdown_scope
+
+Users: Xilinx
+
+What: /sys/devices/platform/firmware\:zynqmp-firmware/health_status
+Date: March 2020
+KernelVersion: 5.6
+Contact: "Jolly Shah" <jollys@xilinx.com>
+Description:
+ This sysfs interface allows to set the health status. If PMUFW
+ is compiled with CHECK_HEALTHY_BOOT, it will check the healthy
+ bit on FPD WDT expiration. If healthy bit is set by a user
+ application running in Linux, PMUFW will do APU only restart. If
+ healthy bit is not set during FPD WDT expiration, PMUFW will do
+ system restart.
+
+ Usage:
+ Set healthy bit
+ # echo 1 > /sys/devices/platform/firmware\:zynqmp-firmware/health_status
+
+ Unset healthy bit
+ # echo 0 > /sys/devices/platform/firmware\:zynqmp-firmware/health_status
+
+Users: Xilinx
diff --git a/Documentation/ABI/testing/debugfs-driver-habanalabs b/Documentation/ABI/testing/debugfs-driver-habanalabs
index a73601c5121e..f6d9c2a8d528 100644
--- a/Documentation/ABI/testing/debugfs-driver-habanalabs
+++ b/Documentation/ABI/testing/debugfs-driver-habanalabs
@@ -8,6 +8,16 @@ Description: Sets the device address to be used for read or write through
only when the IOMMU is disabled.
The acceptable value is a string that starts with "0x"
+What: /sys/kernel/debug/habanalabs/hl<n>/clk_gate
+Date: May 2020
+KernelVersion: 5.8
+Contact: oded.gabbay@gmail.com
+Description: Allow the root user to disable/enable in runtime the clock
+ gating mechanism in Gaudi. Due to how Gaudi is built, the
+ clock gating needs to be disabled in order to access the
+ registers of the TPC and MME engines. This is sometimes needed
+ during debug by the user and hence the user needs this option
+
What: /sys/kernel/debug/habanalabs/hl<n>/command_buffers
Date: Jan 2019
KernelVersion: 5.1
@@ -150,3 +160,10 @@ KernelVersion: 5.1
Contact: oded.gabbay@gmail.com
Description: Displays a list with information about all the active virtual
address mappings per ASID
+
+What: /sys/kernel/debug/habanalabs/hl<n>/stop_on_err
+Date: Mar 2020
+KernelVersion: 5.6
+Contact: oded.gabbay@gmail.com
+Description: Sets the stop-on_error option for the device engines. Value of
+ "0" is for disable, otherwise enable.
diff --git a/Documentation/ABI/testing/sysfs-bus-event_source-devices-dfl_fme b/Documentation/ABI/testing/sysfs-bus-event_source-devices-dfl_fme
new file mode 100644
index 000000000000..c9278a3b3df1
--- /dev/null
+++ b/Documentation/ABI/testing/sysfs-bus-event_source-devices-dfl_fme
@@ -0,0 +1,104 @@
+What: /sys/bus/event_source/devices/dfl_fmeX/format
+Date: April 2020
+KernelVersion: 5.8
+Contact: Wu Hao <hao.wu@intel.com>
+Description: Read-only. Attribute group to describe the magic bits
+ that go into perf_event_attr.config for a particular pmu.
+ (See ABI/testing/sysfs-bus-event_source-devices-format).
+
+ Each attribute under this group defines a bit range of the
+ perf_event_attr.config. All supported attributes are listed
+ below.
+
+ event = "config:0-11" - event ID
+ evtype = "config:12-15" - event type
+ portid = "config:16-23" - event source
+
+ For example,
+
+ fab_mmio_read = "event=0x06,evtype=0x02,portid=0xff"
+
+ It shows this fab_mmio_read is a fabric type (0x02) event with
+ 0x06 local event id for overall monitoring (portid=0xff).
+
+What: /sys/bus/event_source/devices/dfl_fmeX/cpumask
+Date: April 2020
+KernelVersion: 5.8
+Contact: Wu Hao <hao.wu@intel.com>
+Description: Read-only. This file always returns cpu which the PMU is bound
+ for access to all fme pmu performance monitoring events.
+
+What: /sys/bus/event_source/devices/dfl_fmeX/events
+Date: April 2020
+KernelVersion: 5.8
+Contact: Wu Hao <hao.wu@intel.com>
+Description: Read-only. Attribute group to describe performance monitoring
+ events specific to fme. Each attribute in this group describes
+ a single performance monitoring event supported by this fme pmu.
+ The name of the file is the name of the event.
+ (See ABI/testing/sysfs-bus-event_source-devices-events).
+
+ All supported performance monitoring events are listed below.
+
+ Basic events (evtype=0x00)
+
+ clock = "event=0x00,evtype=0x00,portid=0xff"
+
+ Cache events (evtype=0x01)
+
+ cache_read_hit = "event=0x00,evtype=0x01,portid=0xff"
+ cache_read_miss = "event=0x01,evtype=0x01,portid=0xff"
+ cache_write_hit = "event=0x02,evtype=0x01,portid=0xff"
+ cache_write_miss = "event=0x03,evtype=0x01,portid=0xff"
+ cache_hold_request = "event=0x05,evtype=0x01,portid=0xff"
+ cache_data_write_port_contention =
+ "event=0x06,evtype=0x01,portid=0xff"
+ cache_tag_write_port_contention =
+ "event=0x07,evtype=0x01,portid=0xff"
+ cache_tx_req_stall = "event=0x08,evtype=0x01,portid=0xff"
+ cache_rx_req_stall = "event=0x09,evtype=0x01,portid=0xff"
+ cache_eviction = "event=0x0a,evtype=0x01,portid=0xff"
+
+ Fabric events (evtype=0x02)
+
+ fab_pcie0_read = "event=0x00,evtype=0x02,portid=0xff"
+ fab_pcie0_write = "event=0x01,evtype=0x02,portid=0xff"
+ fab_pcie1_read = "event=0x02,evtype=0x02,portid=0xff"
+ fab_pcie1_write = "event=0x03,evtype=0x02,portid=0xff"
+ fab_upi_read = "event=0x04,evtype=0x02,portid=0xff"
+ fab_upi_write = "event=0x05,evtype=0x02,portid=0xff"
+ fab_mmio_read = "event=0x06,evtype=0x02,portid=0xff"
+ fab_mmio_write = "event=0x07,evtype=0x02,portid=0xff"
+ fab_port_pcie0_read = "event=0x00,evtype=0x02,portid=?"
+ fab_port_pcie0_write = "event=0x01,evtype=0x02,portid=?"
+ fab_port_pcie1_read = "event=0x02,evtype=0x02,portid=?"
+ fab_port_pcie1_write = "event=0x03,evtype=0x02,portid=?"
+ fab_port_upi_read = "event=0x04,evtype=0x02,portid=?"
+ fab_port_upi_write = "event=0x05,evtype=0x02,portid=?"
+ fab_port_mmio_read = "event=0x06,evtype=0x02,portid=?"
+ fab_port_mmio_write = "event=0x07,evtype=0x02,portid=?"
+
+ VTD events (evtype=0x03)
+
+ vtd_port_read_transaction = "event=0x00,evtype=0x03,portid=?"
+ vtd_port_write_transaction = "event=0x01,evtype=0x03,portid=?"
+ vtd_port_devtlb_read_hit = "event=0x02,evtype=0x03,portid=?"
+ vtd_port_devtlb_write_hit = "event=0x03,evtype=0x03,portid=?"
+ vtd_port_devtlb_4k_fill = "event=0x04,evtype=0x03,portid=?"
+ vtd_port_devtlb_2m_fill = "event=0x05,evtype=0x03,portid=?"
+ vtd_port_devtlb_1g_fill = "event=0x06,evtype=0x03,portid=?"
+
+ VTD SIP events (evtype=0x04)
+
+ vtd_sip_iotlb_4k_hit = "event=0x00,evtype=0x04,portid=0xff"
+ vtd_sip_iotlb_2m_hit = "event=0x01,evtype=0x04,portid=0xff"
+ vtd_sip_iotlb_1g_hit = "event=0x02,evtype=0x04,portid=0xff"
+ vtd_sip_slpwc_l3_hit = "event=0x03,evtype=0x04,portid=0xff"
+ vtd_sip_slpwc_l4_hit = "event=0x04,evtype=0x04,portid=0xff"
+ vtd_sip_rcc_hit = "event=0x05,evtype=0x04,portid=0xff"
+ vtd_sip_iotlb_4k_miss = "event=0x06,evtype=0x04,portid=0xff"
+ vtd_sip_iotlb_2m_miss = "event=0x07,evtype=0x04,portid=0xff"
+ vtd_sip_iotlb_1g_miss = "event=0x08,evtype=0x04,portid=0xff"
+ vtd_sip_slpwc_l3_miss = "event=0x09,evtype=0x04,portid=0xff"
+ vtd_sip_slpwc_l4_miss = "event=0x0a,evtype=0x04,portid=0xff"
+ vtd_sip_rcc_miss = "event=0x0b,evtype=0x04,portid=0xff"
diff --git a/Documentation/ABI/testing/sysfs-bus-soundwire-master b/Documentation/ABI/testing/sysfs-bus-soundwire-master
new file mode 100644
index 000000000000..46ef038d8722
--- /dev/null
+++ b/Documentation/ABI/testing/sysfs-bus-soundwire-master
@@ -0,0 +1,23 @@
+What: /sys/bus/soundwire/devices/sdw-master-N/revision
+ /sys/bus/soundwire/devices/sdw-master-N/clk_stop_modes
+ /sys/bus/soundwire/devices/sdw-master-N/clk_freq
+ /sys/bus/soundwire/devices/sdw-master-N/clk_gears
+ /sys/bus/soundwire/devices/sdw-master-N/default_col
+ /sys/bus/soundwire/devices/sdw-master-N/default_frame_rate
+ /sys/bus/soundwire/devices/sdw-master-N/default_row
+ /sys/bus/soundwire/devices/sdw-master-N/dynamic_shape
+ /sys/bus/soundwire/devices/sdw-master-N/err_threshold
+ /sys/bus/soundwire/devices/sdw-master-N/max_clk_freq
+
+Date: April 2020
+
+Contact: Pierre-Louis Bossart <pierre-louis.bossart@linux.intel.com>
+ Bard Liao <yung-chuan.liao@linux.intel.com>
+ Vinod Koul <vkoul@kernel.org>
+
+Description: SoundWire Master-N DisCo properties.
+ These properties are defined by MIPI DisCo Specification
+ for SoundWire. They define various properties of the Master
+ and are used by the bus to configure the Master. clk_stop_modes
+ is a bitmask for simplifications and combines the
+ clock-stop-mode0 and clock-stop-mode1 properties.
diff --git a/Documentation/ABI/testing/sysfs-bus-soundwire-slave b/Documentation/ABI/testing/sysfs-bus-soundwire-slave
new file mode 100644
index 000000000000..db4c9511d1aa
--- /dev/null
+++ b/Documentation/ABI/testing/sysfs-bus-soundwire-slave
@@ -0,0 +1,91 @@
+What: /sys/bus/soundwire/devices/sdw:.../dev-properties/mipi_revision
+ /sys/bus/soundwire/devices/sdw:.../dev-properties/wake_capable
+ /sys/bus/soundwire/devices/sdw:.../dev-properties/test_mode_capable
+ /sys/bus/soundwire/devices/sdw:.../dev-properties/clk_stop_mode1
+ /sys/bus/soundwire/devices/sdw:.../dev-properties/simple_clk_stop_capable
+ /sys/bus/soundwire/devices/sdw:.../dev-properties/clk_stop_timeout
+ /sys/bus/soundwire/devices/sdw:.../dev-properties/ch_prep_timeout
+ /sys/bus/soundwire/devices/sdw:.../dev-properties/reset_behave
+ /sys/bus/soundwire/devices/sdw:.../dev-properties/high_PHY_capable
+ /sys/bus/soundwire/devices/sdw:.../dev-properties/paging_support
+ /sys/bus/soundwire/devices/sdw:.../dev-properties/bank_delay_support
+ /sys/bus/soundwire/devices/sdw:.../dev-properties/p15_behave
+ /sys/bus/soundwire/devices/sdw:.../dev-properties/master_count
+ /sys/bus/soundwire/devices/sdw:.../dev-properties/source_ports
+ /sys/bus/soundwire/devices/sdw:.../dev-properties/sink_ports
+
+Date: May 2020
+
+Contact: Pierre-Louis Bossart <pierre-louis.bossart@linux.intel.com>
+ Bard Liao <yung-chuan.liao@linux.intel.com>
+ Vinod Koul <vkoul@kernel.org>
+
+Description: SoundWire Slave DisCo properties.
+ These properties are defined by MIPI DisCo Specification
+ for SoundWire. They define various properties of the
+ SoundWire Slave and are used by the bus to configure
+ the Slave
+
+
+What: /sys/bus/soundwire/devices/sdw:.../dp0/max_word
+ /sys/bus/soundwire/devices/sdw:.../dp0/min_word
+ /sys/bus/soundwire/devices/sdw:.../dp0/words
+ /sys/bus/soundwire/devices/sdw:.../dp0/BRA_flow_controlled
+ /sys/bus/soundwire/devices/sdw:.../dp0/simple_ch_prep_sm
+ /sys/bus/soundwire/devices/sdw:.../dp0/imp_def_interrupts
+
+Date: May 2020
+
+Contact: Pierre-Louis Bossart <pierre-louis.bossart@linux.intel.com>
+ Bard Liao <yung-chuan.liao@linux.intel.com>
+ Vinod Koul <vkoul@kernel.org>
+
+Description: SoundWire Slave Data Port-0 DisCo properties.
+ These properties are defined by MIPI DisCo Specification
+ for the SoundWire. They define various properties of the
+ Data port 0 are used by the bus to configure the Data Port 0.
+
+
+What: /sys/bus/soundwire/devices/sdw:.../dpN_src/max_word
+ /sys/bus/soundwire/devices/sdw:.../dpN_src/min_word
+ /sys/bus/soundwire/devices/sdw:.../dpN_src/words
+ /sys/bus/soundwire/devices/sdw:.../dpN_src/type
+ /sys/bus/soundwire/devices/sdw:.../dpN_src/max_grouping
+ /sys/bus/soundwire/devices/sdw:.../dpN_src/simple_ch_prep_sm
+ /sys/bus/soundwire/devices/sdw:.../dpN_src/ch_prep_timeout
+ /sys/bus/soundwire/devices/sdw:.../dpN_src/imp_def_interrupts
+ /sys/bus/soundwire/devices/sdw:.../dpN_src/min_ch
+ /sys/bus/soundwire/devices/sdw:.../dpN_src/max_ch
+ /sys/bus/soundwire/devices/sdw:.../dpN_src/channels
+ /sys/bus/soundwire/devices/sdw:.../dpN_src/ch_combinations
+ /sys/bus/soundwire/devices/sdw:.../dpN_src/max_async_buffer
+ /sys/bus/soundwire/devices/sdw:.../dpN_src/block_pack_mode
+ /sys/bus/soundwire/devices/sdw:.../dpN_src/port_encoding
+
+ /sys/bus/soundwire/devices/sdw:.../dpN_sink/max_word
+ /sys/bus/soundwire/devices/sdw:.../dpN_sink/min_word
+ /sys/bus/soundwire/devices/sdw:.../dpN_sink/words
+ /sys/bus/soundwire/devices/sdw:.../dpN_sink/type
+ /sys/bus/soundwire/devices/sdw:.../dpN_sink/max_grouping
+ /sys/bus/soundwire/devices/sdw:.../dpN_sink/simple_ch_prep_sm
+ /sys/bus/soundwire/devices/sdw:.../dpN_sink/ch_prep_timeout
+ /sys/bus/soundwire/devices/sdw:.../dpN_sink/imp_def_interrupts
+ /sys/bus/soundwire/devices/sdw:.../dpN_sink/min_ch
+ /sys/bus/soundwire/devices/sdw:.../dpN_sink/max_ch
+ /sys/bus/soundwire/devices/sdw:.../dpN_sink/channels
+ /sys/bus/soundwire/devices/sdw:.../dpN_sink/ch_combinations
+ /sys/bus/soundwire/devices/sdw:.../dpN_sink/max_async_buffer
+ /sys/bus/soundwire/devices/sdw:.../dpN_sink/block_pack_mode
+ /sys/bus/soundwire/devices/sdw:.../dpN_sink/port_encoding
+
+Date: May 2020
+
+Contact: Pierre-Louis Bossart <pierre-louis.bossart@linux.intel.com>
+ Bard Liao <yung-chuan.liao@linux.intel.com>
+ Vinod Koul <vkoul@kernel.org>
+
+Description: SoundWire Slave Data Source/Sink Port-N DisCo properties.
+ These properties are defined by MIPI DisCo Specification
+ for SoundWire. They define various properties of the
+ Source/Sink Data port N and are used by the bus to configure
+ the Data Port N.
diff --git a/Documentation/ABI/testing/sysfs-driver-habanalabs b/Documentation/ABI/testing/sysfs-driver-habanalabs
index 782df74042ed..1a14bf9b22ba 100644
--- a/Documentation/ABI/testing/sysfs-driver-habanalabs
+++ b/Documentation/ABI/testing/sysfs-driver-habanalabs
@@ -10,6 +10,23 @@ KernelVersion: 5.1
Contact: oded.gabbay@gmail.com
Description: Version of the application running on the device's CPU
+What: /sys/class/habanalabs/hl<n>/clk_max_freq_mhz
+Date: Jun 2019
+KernelVersion: not yet upstreamed
+Contact: oded.gabbay@gmail.com
+Description: Allows the user to set the maximum clock frequency, in MHz.
+ The device clock might be set to lower value than the maximum.
+ The user should read the clk_cur_freq_mhz to see the actual
+ frequency value of the device clock. This property is valid
+ only for the Gaudi ASIC family
+
+What: /sys/class/habanalabs/hl<n>/clk_cur_freq_mhz
+Date: Jun 2019
+KernelVersion: not yet upstreamed
+Contact: oded.gabbay@gmail.com
+Description: Displays the current frequency, in MHz, of the device clock.
+ This property is valid only for the Gaudi ASIC family
+
What: /sys/class/habanalabs/hl<n>/cpld_ver
Date: Jan 2019
KernelVersion: 5.1
diff --git a/Documentation/ABI/testing/sysfs-driver-w1_therm b/Documentation/ABI/testing/sysfs-driver-w1_therm
new file mode 100644
index 000000000000..076659d506f2
--- /dev/null
+++ b/Documentation/ABI/testing/sysfs-driver-w1_therm
@@ -0,0 +1,116 @@
+What: /sys/bus/w1/devices/.../alarms
+Date: May 2020
+Contact: Akira Shimahara <akira215corp@gmail.com>
+Description:
+ (RW) read or write TH and TL (Temperature High an Low) alarms.
+ Values shall be space separated and in the device range
+ (typical -55 degC to 125 degC), if not values will be trimmed
+ to device min/max capabilities. Values are integer as they are
+ stored in a 8bit register in the device. Lowest value is
+ automatically put to TL. Once set, alarms could be search at
+ master level, refer to Documentation/w1/w1_generic.rst for
+ detailed information
+Users: any user space application which wants to communicate with
+ w1_term device
+
+
+What: /sys/bus/w1/devices/.../eeprom
+Date: May 2020
+Contact: Akira Shimahara <akira215corp@gmail.com>
+Description:
+ (WO) writing that file will either trigger a save of the
+ device data to its embedded EEPROM, either restore data
+ embedded in device EEPROM. Be aware that devices support
+ limited EEPROM writing cycles (typical 50k)
+ * 'save': save device RAM to EEPROM
+ * 'restore': restore EEPROM data in device RAM
+Users: any user space application which wants to communicate with
+ w1_term device
+
+
+What: /sys/bus/w1/devices/.../ext_power
+Date: May 2020
+Contact: Akira Shimahara <akira215corp@gmail.com>
+Description:
+ (RO) return the power status by asking the device
+ * '0': device parasite powered
+ * '1': device externally powered
+ * '-xx': xx is kernel error when reading power status
+Users: any user space application which wants to communicate with
+ w1_term device
+
+
+What: /sys/bus/w1/devices/.../resolution
+Date: May 2020
+Contact: Akira Shimahara <akira215corp@gmail.com>
+Description:
+ (RW) get or set the device resolution (on supported devices,
+ if not, this entry is not present). Note that the resolution
+ will be changed only in device RAM, so it will be cleared when
+ power is lost. Trigger a 'save' to EEPROM command to keep
+ values after power-on. Read or write are :
+ * '9..12': device resolution in bit
+ or resolution to set in bit
+ * '-xx': xx is kernel error when reading the resolution
+ * Anything else: do nothing
+Users: any user space application which wants to communicate with
+ w1_term device
+
+
+What: /sys/bus/w1/devices/.../temperature
+Date: May 2020
+Contact: Akira Shimahara <akira215corp@gmail.com>
+Description:
+ (RO) return the temperature in 1/1000 degC.
+ * If a bulk read has been triggered, it will directly
+ return the temperature computed when the bulk read
+ occurred, if available. If not yet available, nothing
+ is returned (a debug kernel message is sent), you
+ should retry later on.
+ * If no bulk read has been triggered, it will trigger
+ a conversion and send the result. Note that the
+ conversion duration depend on the resolution (if
+ device support this feature). It takes 94ms in 9bits
+ resolution, 750ms for 12bits.
+Users: any user space application which wants to communicate with
+ w1_term device
+
+
+What: /sys/bus/w1/devices/.../w1_slave
+Date: May 2020
+Contact: Akira Shimahara <akira215corp@gmail.com>
+Description:
+ (RW) return the temperature in 1/1000 degC.
+ *read*: return 2 lines with the hexa output data sent on the
+ bus, return the CRC check and temperature in 1/1000 degC
+ *write* :
+ * '0' : save the 2 or 3 bytes to the device EEPROM
+ (i.e. TH, TL and config register)
+ * '9..12' : set the device resolution in RAM
+ (if supported)
+ * Anything else: do nothing
+ refer to Documentation/w1/slaves/w1_therm.rst for detailed
+ information.
+Users: any user space application which wants to communicate with
+ w1_term device
+
+
+What: /sys/bus/w1/devices/w1_bus_masterXX/therm_bulk_read
+Date: May 2020
+Contact: Akira Shimahara <akira215corp@gmail.com>
+Description:
+ (RW) trigger a bulk read conversion. read the status
+ *read*:
+ * '-1': conversion in progress on at least 1 sensor
+ * '1' : conversion complete but at least one sensor
+ value has not been read yet
+ * '0' : no bulk operation. Reading temperature will
+ trigger a conversion on each device
+ *write*: 'trigger': trigger a bulk read on all supporting
+ devices on the bus
+ Note that if a bulk read is sent but one sensor is not read
+ immediately, the next access to temperature on this device
+ will return the temperature measured at the time of issue
+ of the bulk read command (not the current temperature).
+Users: any user space application which wants to communicate with
+ w1_term device
diff --git a/Documentation/devicetree/bindings/firmware/intel,stratix10-svc.txt b/Documentation/devicetree/bindings/firmware/intel,stratix10-svc.txt
index 1fa66065acc6..6eff1afd8daf 100644
--- a/Documentation/devicetree/bindings/firmware/intel,stratix10-svc.txt
+++ b/Documentation/devicetree/bindings/firmware/intel,stratix10-svc.txt
@@ -23,7 +23,7 @@ Required properties:
The svc node has the following mandatory properties, must be located under
the firmware node.
-- compatible: "intel,stratix10-svc"
+- compatible: "intel,stratix10-svc" or "intel,agilex-svc"
- method: smc or hvc
smc - Secure Monitor Call
hvc - Hypervisor Call
diff --git a/Documentation/devicetree/bindings/fpga/intel-stratix10-soc-fpga-mgr.txt b/Documentation/devicetree/bindings/fpga/intel-stratix10-soc-fpga-mgr.txt
index 6e03f79287fb..0f874137ca46 100644
--- a/Documentation/devicetree/bindings/fpga/intel-stratix10-soc-fpga-mgr.txt
+++ b/Documentation/devicetree/bindings/fpga/intel-stratix10-soc-fpga-mgr.txt
@@ -4,7 +4,8 @@ Required properties:
The fpga_mgr node has the following mandatory property, must be located under
firmware/svc node.
-- compatible : should contain "intel,stratix10-soc-fpga-mgr"
+- compatible : should contain "intel,stratix10-soc-fpga-mgr" or
+ "intel,agilex-soc-fpga-mgr"
Example:
diff --git a/Documentation/devicetree/bindings/interconnect/fsl,imx8m-noc.yaml b/Documentation/devicetree/bindings/interconnect/fsl,imx8m-noc.yaml
new file mode 100644
index 000000000000..ff09550ad959
--- /dev/null
+++ b/Documentation/devicetree/bindings/interconnect/fsl,imx8m-noc.yaml
@@ -0,0 +1,101 @@
+# SPDX-License-Identifier: GPL-2.0
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/interconnect/fsl,imx8m-noc.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Generic i.MX bus frequency device
+
+maintainers:
+ - Leonard Crestez <leonard.crestez@nxp.com>
+
+description: |
+ The i.MX SoC family has multiple buses for which clock frequency (and
+ sometimes voltage) can be adjusted.
+
+ Some of those buses expose register areas mentioned in the memory maps as GPV
+ ("Global Programmers View") but not all. Access to this area might be denied
+ for normal (non-secure) world.
+
+ The buses are based on externally licensed IPs such as ARM NIC-301 and
+ Arteris FlexNOC but DT bindings are specific to the integration of these bus
+ interconnect IPs into imx SOCs.
+
+properties:
+ compatible:
+ oneOf:
+ - items:
+ - enum:
+ - fsl,imx8mn-nic
+ - fsl,imx8mm-nic
+ - fsl,imx8mq-nic
+ - const: fsl,imx8m-nic
+ - items:
+ - enum:
+ - fsl,imx8mn-noc
+ - fsl,imx8mm-noc
+ - fsl,imx8mq-noc
+ - const: fsl,imx8m-noc
+ - const: fsl,imx8m-nic
+
+ reg:
+ maxItems: 1
+
+ clocks:
+ maxItems: 1
+
+ operating-points-v2: true
+ opp-table: true
+
+ fsl,ddrc:
+ $ref: "/schemas/types.yaml#/definitions/phandle"
+ description:
+ Phandle to DDR Controller.
+
+ '#interconnect-cells':
+ description:
+ If specified then also act as an interconnect provider. Should only be
+ set once per soc on the main noc.
+ const: 1
+
+required:
+ - compatible
+ - clocks
+
+additionalProperties: false
+
+examples:
+ - |
+ #include <dt-bindings/clock/imx8mm-clock.h>
+ #include <dt-bindings/interconnect/imx8mm.h>
+ #include <dt-bindings/interrupt-controller/arm-gic.h>
+
+ noc: interconnect@32700000 {
+ compatible = "fsl,imx8mm-noc", "fsl,imx8m-noc";
+ reg = <0x32700000 0x100000>;
+ clocks = <&clk IMX8MM_CLK_NOC>;
+ #interconnect-cells = <1>;
+ fsl,ddrc = <&ddrc>;
+
+ operating-points-v2 = <&noc_opp_table>;
+ noc_opp_table: opp-table {
+ compatible = "operating-points-v2";
+
+ opp-133M {
+ opp-hz = /bits/ 64 <133333333>;
+ };
+ opp-800M {
+ opp-hz = /bits/ 64 <800000000>;
+ };
+ };
+ };
+
+ ddrc: memory-controller@3d400000 {
+ compatible = "fsl,imx8mm-ddrc", "fsl,imx8m-ddrc";
+ reg = <0x3d400000 0x400000>;
+ clock-names = "core", "pll", "alt", "apb";
+ clocks = <&clk IMX8MM_CLK_DRAM_CORE>,
+ <&clk IMX8MM_DRAM_PLL>,
+ <&clk IMX8MM_CLK_DRAM_ALT>,
+ <&clk IMX8MM_CLK_DRAM_APB>;
+ };
diff --git a/Documentation/driver-api/soundwire/stream.rst b/Documentation/driver-api/soundwire/stream.rst
index 8bceece51554..1b386076402c 100644
--- a/Documentation/driver-api/soundwire/stream.rst
+++ b/Documentation/driver-api/soundwire/stream.rst
@@ -75,8 +75,33 @@ Slaves are using single port. ::
| (Data) |
+---------------+
+Example 4: Stereo Stream with L and R channels is rendered by
+Master. Both of the L and R channels are received by two different
+Slaves. Master and both Slaves are using single port handling
+L+R. Each Slave device processes the L + R data locally, typically
+based on static configuration or dynamic orientation, and may drive
+one or more speakers. ::
-Example 4: Stereo Stream with L and R channel is rendered by two different
+ +---------------+ Clock Signal +---------------+
+ | Master +---------+------------------------+ Slave |
+ | Interface | | | Interface |
+ | | | | 1 |
+ | | | Data Signal | |
+ | L + R +---+------------------------------+ L + R |
+ | (Data) | | | Data Direction | (Data) |
+ +---------------+ | | +-------------> +---------------+
+ | |
+ | |
+ | | +---------------+
+ | +----------------------> | Slave |
+ | | Interface |
+ | | 2 |
+ | | |
+ +----------------------------> | L + R |
+ | (Data) |
+ +---------------+
+
+Example 5: Stereo Stream with L and R channel is rendered by two different
Ports of the Master and is received by only single Port of the Slave
interface. ::
@@ -101,7 +126,7 @@ interface. ::
+--------------------+ | |
+----------------+
-Example 5: Stereo Stream with L and R channel is rendered by 2 Masters, each
+Example 6: Stereo Stream with L and R channel is rendered by 2 Masters, each
rendering one channel, and is received by two different Slaves, each
receiving one channel. Both Masters and both Slaves are using single port. ::
@@ -123,12 +148,70 @@ receiving one channel. Both Masters and both Slaves are using single port. ::
| (Data) | Data Direction | (Data) |
+---------------+ +-----------------------> +---------------+
-Note: In multi-link cases like above, to lock, one would acquire a global
+Example 7: Stereo Stream with L and R channel is rendered by 2
+Masters, each rendering both channels. Each Slave receives L + R. This
+is the same application as Example 4 but with Slaves placed on
+separate links. ::
+
+ +---------------+ Clock Signal +---------------+
+ | Master +----------------------------------+ Slave |
+ | Interface | | Interface |
+ | 1 | | 1 |
+ | | Data Signal | |
+ | L + R +----------------------------------+ L + R |
+ | (Data) | Data Direction | (Data) |
+ +---------------+ +-----------------------> +---------------+
+
+ +---------------+ Clock Signal +---------------+
+ | Master +----------------------------------+ Slave |
+ | Interface | | Interface |
+ | 2 | | 2 |
+ | | Data Signal | |
+ | L + R +----------------------------------+ L + R |
+ | (Data) | Data Direction | (Data) |
+ +---------------+ +-----------------------> +---------------+
+
+Example 8: 4-channel Stream is rendered by 2 Masters, each rendering a
+2 channels. Each Slave receives 2 channels. ::
+
+ +---------------+ Clock Signal +---------------+
+ | Master +----------------------------------+ Slave |
+ | Interface | | Interface |
+ | 1 | | 1 |
+ | | Data Signal | |
+ | L1 + R1 +----------------------------------+ L1 + R1 |
+ | (Data) | Data Direction | (Data) |
+ +---------------+ +-----------------------> +---------------+
+
+ +---------------+ Clock Signal +---------------+
+ | Master +----------------------------------+ Slave |
+ | Interface | | Interface |
+ | 2 | | 2 |
+ | | Data Signal | |
+ | L2 + R2 +----------------------------------+ L2 + R2 |
+ | (Data) | Data Direction | (Data) |
+ +---------------+ +-----------------------> +---------------+
+
+Note1: In multi-link cases like above, to lock, one would acquire a global
lock and then go on locking bus instances. But, in this case the caller
framework(ASoC DPCM) guarantees that stream operations on a card are
always serialized. So, there is no race condition and hence no need for
global lock.
+Note2: A Slave device may be configured to receive all channels
+transmitted on a link for a given Stream (Example 4) or just a subset
+of the data (Example 3). The configuration of the Slave device is not
+handled by a SoundWire subsystem API, but instead by the
+snd_soc_dai_set_tdm_slot() API. The platform or machine driver will
+typically configure which of the slots are used. For Example 4, the
+same slots would be used by all Devices, while for Example 3 the Slave
+Device1 would use e.g. Slot 0 and Slave device2 slot 1.
+
+Note3: Multiple Sink ports can extract the same information for the
+same bitSlots in the SoundWire frame, however multiple Source ports
+shall be configured with different bitSlot configurations. This is the
+same limitation as with I2S/PCM TDM usages.
+
SoundWire Stream Management flow
================================
diff --git a/Documentation/driver-api/soundwire/summary.rst b/Documentation/driver-api/soundwire/summary.rst
index 8193125a2bfb..01dcb954f6d7 100644
--- a/Documentation/driver-api/soundwire/summary.rst
+++ b/Documentation/driver-api/soundwire/summary.rst
@@ -101,10 +101,11 @@ Following is the Bus API to register the SoundWire Bus:
.. code-block:: c
- int sdw_add_bus_master(struct sdw_bus *bus)
+ int sdw_bus_master_add(struct sdw_bus *bus,
+ struct device *parent,
+ struct fwnode_handle)
{
- if (!bus->dev)
- return -ENODEV;
+ sdw_master_device_add(bus, parent, fwnode);
mutex_init(&bus->lock);
INIT_LIST_HEAD(&bus->slaves);
diff --git a/Documentation/fpga/dfl.rst b/Documentation/fpga/dfl.rst
index 094fc8aacd8e..978c4af416a4 100644
--- a/Documentation/fpga/dfl.rst
+++ b/Documentation/fpga/dfl.rst
@@ -118,6 +118,11 @@ More functions are exposed through sysfs
management information (current temperature, thresholds, threshold status,
etc.).
+ Performance reporting
+ performance counters are exposed through perf PMU APIs. Standard perf tool
+ can be used to monitor all available perf events. Please see performance
+ counter section below for more detailed information.
+
FIU - PORT
==========
@@ -378,6 +383,85 @@ The device nodes used for ioctl() or mmap() can be referenced through::
/sys/class/fpga_region/<regionX>/<dfl-port.n>/dev
+Performance Counters
+====================
+Performance reporting is one private feature implemented in FME. It could
+supports several independent, system-wide, device counter sets in hardware to
+monitor and count for performance events, including "basic", "cache", "fabric",
+"vtd" and "vtd_sip" counters. Users could use standard perf tool to monitor
+FPGA cache hit/miss rate, transaction number, interface clock counter of AFU
+and other FPGA performance events.
+
+Different FPGA devices may have different counter sets, depending on hardware
+implementation. E.g., some discrete FPGA cards don't have any cache. User could
+use "perf list" to check which perf events are supported by target hardware.
+
+In order to allow user to use standard perf API to access these performance
+counters, driver creates a perf PMU, and related sysfs interfaces in
+/sys/bus/event_source/devices/dfl_fme* to describe available perf events and
+configuration options.
+
+The "format" directory describes the format of the config field of struct
+perf_event_attr. There are 3 bitfields for config: "evtype" defines which type
+the perf event belongs to; "event" is the identity of the event within its
+category; "portid" is introduced to decide counters set to monitor on FPGA
+overall data or a specific port.
+
+The "events" directory describes the configuration templates for all available
+events which can be used with perf tool directly. For example, fab_mmio_read
+has the configuration "event=0x06,evtype=0x02,portid=0xff", which shows this
+event belongs to fabric type (0x02), the local event id is 0x06 and it is for
+overall monitoring (portid=0xff).
+
+Example usage of perf::
+
+ $# perf list |grep dfl_fme
+
+ dfl_fme0/fab_mmio_read/ [Kernel PMU event]
+ <...>
+ dfl_fme0/fab_port_mmio_read,portid=?/ [Kernel PMU event]
+ <...>
+
+ $# perf stat -a -e dfl_fme0/fab_mmio_read/ <command>
+ or
+ $# perf stat -a -e dfl_fme0/event=0x06,evtype=0x02,portid=0xff/ <command>
+ or
+ $# perf stat -a -e dfl_fme0/config=0xff2006/ <command>
+
+Another example, fab_port_mmio_read monitors mmio read of a specific port. So
+its configuration template is "event=0x06,evtype=0x01,portid=?". The portid
+should be explicitly set.
+
+Its usage of perf::
+
+ $# perf stat -a -e dfl_fme0/fab_port_mmio_read,portid=0x0/ <command>
+ or
+ $# perf stat -a -e dfl_fme0/event=0x06,evtype=0x02,portid=0x0/ <command>
+ or
+ $# perf stat -a -e dfl_fme0/config=0x2006/ <command>
+
+Please note for fabric counters, overall perf events (fab_*) and port perf
+events (fab_port_*) actually share one set of counters in hardware, so it can't
+monitor both at the same time. If this set of counters is configured to monitor
+overall data, then per port perf data is not supported. See below example::
+
+ $# perf stat -e dfl_fme0/fab_mmio_read/,dfl_fme0/fab_port_mmio_write,\
+ portid=0/ sleep 1
+
+ Performance counter stats for 'system wide':
+
+ 3 dfl_fme0/fab_mmio_read/
+ <not supported> dfl_fme0/fab_port_mmio_write,portid=0x0/
+
+ 1.001750904 seconds time elapsed
+
+The driver also provides a "cpumask" sysfs attribute, which contains only one
+CPU id used to access these perf events. Counting on multiple CPU is not allowed
+since they are system-wide counters on FPGA device.
+
+The current driver does not support sampling. So "perf record" is unsupported.
+
+
Add new FIUs support
====================
It's possible that developers made some new function blocks (FIUs) under this
diff --git a/Documentation/trace/coresight/coresight-ect.rst b/Documentation/trace/coresight/coresight-ect.rst
index a93e52abcf46..a68732c5c6d6 100644
--- a/Documentation/trace/coresight/coresight-ect.rst
+++ b/Documentation/trace/coresight/coresight-ect.rst
@@ -73,7 +73,7 @@ capable of generating or using trigger signals.::
>$ ls /sys/bus/coresight/devices/etm0/cti_cpu0
channels ctmid enable nr_trigger_cons mgmt power powered regs
- subsystem triggers0 triggers1 uevent
+ connections subsystem triggers0 triggers1 uevent
*Key file items are:-*
* ``enable``: enables/disables the CTI. Read to determine current state.
@@ -89,6 +89,9 @@ capable of generating or using trigger signals.::
* ``channels``: Contains the channel API - CTI main programming interface.
* ``regs``: Gives access to the raw programmable CTI regs.
* ``mgmt``: the standard CoreSight management registers.
+ * ``connections``: Links to connected *CoreSight* devices. The number of
+ links can be 0 to ``nr_trigger_cons``. Actual number given by ``nr_links``
+ in this directory.
triggers<N> directories
diff --git a/Documentation/trace/coresight/coresight.rst b/Documentation/trace/coresight/coresight.rst
index 108600ee1e12..0b73acb44efa 100644
--- a/Documentation/trace/coresight/coresight.rst
+++ b/Documentation/trace/coresight/coresight.rst
@@ -241,6 +241,91 @@ to the newer scheme, to give a confirmation that what you see on your
system is not unexpected. One must use the "names" as they appear on
the system under specified locations.
+Topology Representation
+-----------------------
+
+Each CoreSight component has a ``connections`` directory which will contain
+links to other CoreSight components. This allows the user to explore the trace
+topology and for larger systems, determine the most appropriate sink for a
+given source. The connection information can also be used to establish
+which CTI devices are connected to a given component. This directory contains a
+``nr_links`` attribute detailing the number of links in the directory.
+
+For an ETM source, in this case ``etm0`` on a Juno platform, a typical
+arrangement will be::
+
+ linaro-developer:~# ls - l /sys/bus/coresight/devices/etm0/connections
+ <file details> cti_cpu0 -> ../../../23020000.cti/cti_cpu0
+ <file details> nr_links
+ <file details> out:0 -> ../../../230c0000.funnel/funnel2
+
+Following the out port to ``funnel2``::
+
+ linaro-developer:~# ls -l /sys/bus/coresight/devices/funnel2/connections
+ <file details> in:0 -> ../../../23040000.etm/etm0
+ <file details> in:1 -> ../../../23140000.etm/etm3
+ <file details> in:2 -> ../../../23240000.etm/etm4
+ <file details> in:3 -> ../../../23340000.etm/etm5
+ <file details> nr_links
+ <file details> out:0 -> ../../../20040000.funnel/funnel0
+
+And again to ``funnel0``::
+
+ linaro-developer:~# ls -l /sys/bus/coresight/devices/funnel0/connections
+ <file details> in:0 -> ../../../220c0000.funnel/funnel1
+ <file details> in:1 -> ../../../230c0000.funnel/funnel2
+ <file details> nr_links
+ <file details> out:0 -> ../../../20010000.etf/tmc_etf0
+
+Finding the first sink ``tmc_etf0``. This can be used to collect data
+as a sink, or as a link to propagate further along the chain::
+
+ linaro-developer:~# ls -l /sys/bus/coresight/devices/tmc_etf0/connections
+ <file details> cti_sys0 -> ../../../20020000.cti/cti_sys0
+ <file details> in:0 -> ../../../20040000.funnel/funnel0
+ <file details> nr_links
+ <file details> out:0 -> ../../../20150000.funnel/funnel4
+
+via ``funnel4``::
+
+ linaro-developer:~# ls -l /sys/bus/coresight/devices/funnel4/connections
+ <file details> in:0 -> ../../../20010000.etf/tmc_etf0
+ <file details> in:1 -> ../../../20140000.etf/tmc_etf1
+ <file details> nr_links
+ <file details> out:0 -> ../../../20120000.replicator/replicator0
+
+and a ``replicator0``::
+
+ linaro-developer:~# ls -l /sys/bus/coresight/devices/replicator0/connections
+ <file details> in:0 -> ../../../20150000.funnel/funnel4
+ <file details> nr_links
+ <file details> out:0 -> ../../../20030000.tpiu/tpiu0
+ <file details> out:1 -> ../../../20070000.etr/tmc_etr0
+
+Arriving at the final sink in the chain, ``tmc_etr0``::
+
+ linaro-developer:~# ls -l /sys/bus/coresight/devices/tmc_etr0/connections
+ <file details> cti_sys0 -> ../../../20020000.cti/cti_sys0
+ <file details> in:0 -> ../../../20120000.replicator/replicator0
+ <file details> nr_links
+
+As described below, when using sysfs it is sufficient to enable a sink and
+a source for successful trace. The framework will correctly enable all
+intermediate links as required.
+
+Note: ``cti_sys0`` appears in two of the connections lists above.
+CTIs can connect to multiple devices and are arranged in a star topology
+via the CTM. See (:doc:`coresight-ect`) [#fourth]_ for further details.
+Looking at this device we see 4 connections::
+
+ linaro-developer:~# ls -l /sys/bus/coresight/devices/cti_sys0/connections
+ <file details> nr_links
+ <file details> stm0 -> ../../../20100000.stm/stm0
+ <file details> tmc_etf0 -> ../../../20010000.etf/tmc_etf0
+ <file details> tmc_etr0 -> ../../../20070000.etr/tmc_etr0
+ <file details> tpiu0 -> ../../../20030000.tpiu/tpiu0
+
+
How to use the tracer modules
-----------------------------
diff --git a/Documentation/w1/slaves/w1_therm.rst b/Documentation/w1/slaves/w1_therm.rst
index 90531c340a07..cc4edae17751 100644
--- a/Documentation/w1/slaves/w1_therm.rst
+++ b/Documentation/w1/slaves/w1_therm.rst
@@ -26,20 +26,31 @@ W1_THERM_DS1825 0x3B
W1_THERM_DS28EA00 0x42
==================== ====
-Support is provided through the sysfs w1_slave file. Each open and
+Support is provided through the sysfs w1_slave file. Each open and
read sequence will initiate a temperature conversion then provide two
-lines of ASCII output. The first line contains the nine hex bytes
+lines of ASCII output. The first line contains the nine hex bytes
read along with a calculated crc value and YES or NO if it matched.
-If the crc matched the returned values are retained. The second line
+If the crc matched the returned values are retained. The second line
displays the retained values along with a temperature in millidegrees
Centigrade after t=.
-Parasite powered devices are limited to one slave performing a
-temperature conversion at a time. If none of the devices are parasite
-powered it would be possible to convert all the devices at the same
-time and then go back to read individual sensors. That isn't
-currently supported. The driver also doesn't support reduced
-precision (which would also reduce the conversion time) when reading values.
+Alternatively, temperature can be read using temperature sysfs, it
+return only temperature in millidegrees Centigrade.
+
+A bulk read of all devices on the bus could be done writing 'trigger'
+in the therm_bulk_read sysfs entry at w1_bus_master level. This will
+sent the convert command on all devices on the bus, and if parasite
+powered devices are detected on the bus (and strong pullup is enable
+in the module), it will drive the line high during the longer conversion
+time required by parasited powered device on the line. Reading
+therm_bulk_read will return 0 if no bulk conversion pending,
+-1 if at least one sensor still in conversion, 1 if conversion is complete
+but at least one sensor value has not been read yet. Result temperature is
+then accessed by reading the temperature sysfs entry of each device, which
+may return empty if conversion is still in progress. Note that if a bulk
+read is sent but one sensor is not read immediately, the next access to
+temperature on this device will return the temperature measured at the
+time of issue of the bulk read command (not the current temperature).
Writing a value between 9 and 12 to the sysfs w1_slave file will change the
precision of the sensor for the next readings. This value is in (volatile)
@@ -49,6 +60,27 @@ To store the current precision configuration into EEPROM, the value 0
has to be written to the sysfs w1_slave file. Since the EEPROM has a limited
amount of writes (>50k), this command should be used wisely.
+Alternatively, resolution can be set or read (value from 9 to 12) using the
+dedicated resolution sysfs entry on each device. This sysfs entry is not
+present for devices not supporting this feature. Driver will adjust the
+correct conversion time for each device regarding to its resolution setting.
+In particular, strong pullup will be applied if required during the conversion
+duration.
+
+The write-only sysfs entry eeprom is an alternative for EEPROM operations:
+ * 'save': will save device RAM to EEPROM
+ * 'restore': will restore EEPROM data in device RAM.
+
+ext_power syfs entry allow tho check the power status of each device.
+ * '0': device parasite powered
+ * '1': device externally powered
+
+sysfs alarms allow read or write TH and TL (Temperature High an Low) alarms.
+Values shall be space separated and in the device range (typical -55 degC
+to 125 degC). Values are integer as they are store in a 8bit register in
+the device. Lowest value is automatically put to TL.Once set, alarms could
+be search at master level.
+
The module parameter strong_pullup can be set to 0 to disable the
strong pullup, 1 to enable autodetection or 2 to force strong pullup.
In case of autodetection, the driver will use the "READ POWER SUPPLY"
diff --git a/arch/arm64/boot/dts/intel/socfpga_agilex.dtsi b/arch/arm64/boot/dts/intel/socfpga_agilex.dtsi
index d8c44d3ca15a..f52de8f7806a 100644
--- a/arch/arm64/boot/dts/intel/socfpga_agilex.dtsi
+++ b/arch/arm64/boot/dts/intel/socfpga_agilex.dtsi
@@ -539,12 +539,12 @@
firmware {
svc {
- compatible = "intel,stratix10-svc";
+ compatible = "intel,agilex-svc";
method = "smc";
memory-region = <&service_reserved>;
fpga_mgr: fpga-mgr {
- compatible = "intel,stratix10-soc-fpga-mgr";
+ compatible = "intel,agilex-soc-fpga-mgr";
};
};
};
diff --git a/drivers/android/binderfs.c b/drivers/android/binderfs.c
index 9ecad74183a3..7cf566aafe1f 100644
--- a/drivers/android/binderfs.c
+++ b/drivers/android/binderfs.c
@@ -650,7 +650,7 @@ static int binderfs_fill_super(struct super_block *sb, struct fs_context *fc)
struct binderfs_info *info;
struct binderfs_mount_opts *ctx = fc->fs_private;
struct inode *inode = NULL;
- struct binderfs_device device_info = { 0 };
+ struct binderfs_device device_info = {};
const char *name;
size_t len;
@@ -747,7 +747,7 @@ static const struct fs_context_operations binderfs_fs_context_ops = {
static int binderfs_init_fs_context(struct fs_context *fc)
{
- struct binderfs_mount_opts *ctx = fc->fs_private;
+ struct binderfs_mount_opts *ctx;
ctx = kzalloc(sizeof(struct binderfs_mount_opts), GFP_KERNEL);
if (!ctx)
diff --git a/drivers/bus/mhi/core/boot.c b/drivers/bus/mhi/core/boot.c
index ebad5eb48e5a..0b38014d040e 100644
--- a/drivers/bus/mhi/core/boot.c
+++ b/drivers/bus/mhi/core/boot.c
@@ -43,10 +43,7 @@ void mhi_rddm_prepare(struct mhi_controller *mhi_cntrl,
lower_32_bits(mhi_buf->dma_addr));
mhi_write_reg(mhi_cntrl, base, BHIE_RXVECSIZE_OFFS, mhi_buf->len);
- sequence_id = prandom_u32() & BHIE_RXVECSTATUS_SEQNUM_BMSK;
-
- if (unlikely(!sequence_id))
- sequence_id = 1;
+ sequence_id = MHI_RANDOM_U32_NONZERO(BHIE_RXVECSTATUS_SEQNUM_BMSK);
mhi_write_reg_field(mhi_cntrl, base, BHIE_RXVECDB_OFFS,
BHIE_RXVECDB_SEQNUM_BMSK, BHIE_RXVECDB_SEQNUM_SHFT,
@@ -121,7 +118,8 @@ static int __mhi_download_rddm_in_panic(struct mhi_controller *mhi_cntrl)
ee = mhi_get_exec_env(mhi_cntrl);
}
- dev_dbg(dev, "Waiting for image download completion, current EE: %s\n",
+ dev_dbg(dev,
+ "Waiting for RDDM image download via BHIe, current EE:%s\n",
TO_MHI_EXEC_STR(ee));
while (retry--) {
@@ -152,11 +150,14 @@ static int __mhi_download_rddm_in_panic(struct mhi_controller *mhi_cntrl)
int mhi_download_rddm_img(struct mhi_controller *mhi_cntrl, bool in_panic)
{
void __iomem *base = mhi_cntrl->bhie;
+ struct device *dev = &mhi_cntrl->mhi_dev->dev;
u32 rx_status;
if (in_panic)
return __mhi_download_rddm_in_panic(mhi_cntrl);
+ dev_dbg(dev, "Waiting for RDDM image download via BHIe\n");
+
/* Wait for the image download to complete */
wait_event_timeout(mhi_cntrl->state_event,
mhi_read_reg_field(mhi_cntrl, base,
@@ -174,8 +175,10 @@ static int mhi_fw_load_amss(struct mhi_controller *mhi_cntrl,
const struct mhi_buf *mhi_buf)
{
void __iomem *base = mhi_cntrl->bhie;
+ struct device *dev = &mhi_cntrl->mhi_dev->dev;
rwlock_t *pm_lock = &mhi_cntrl->pm_lock;
u32 tx_status, sequence_id;
+ int ret;
read_lock_bh(pm_lock);
if (!MHI_REG_ACCESS_VALID(mhi_cntrl->pm_state)) {
@@ -183,6 +186,9 @@ static int mhi_fw_load_amss(struct mhi_controller *mhi_cntrl,
return -EIO;
}
+ sequence_id = MHI_RANDOM_U32_NONZERO(BHIE_TXVECSTATUS_SEQNUM_BMSK);
+ dev_dbg(dev, "Starting AMSS download via BHIe. Sequence ID:%u\n",
+ sequence_id);
mhi_write_reg(mhi_cntrl, base, BHIE_TXVECADDR_HIGH_OFFS,
upper_32_bits(mhi_buf->dma_addr));
@@ -191,26 +197,25 @@ static int mhi_fw_load_amss(struct mhi_controller *mhi_cntrl,
mhi_write_reg(mhi_cntrl, base, BHIE_TXVECSIZE_OFFS, mhi_buf->len);
- sequence_id = prandom_u32() & BHIE_TXVECSTATUS_SEQNUM_BMSK;
mhi_write_reg_field(mhi_cntrl, base, BHIE_TXVECDB_OFFS,
BHIE_TXVECDB_SEQNUM_BMSK, BHIE_TXVECDB_SEQNUM_SHFT,
sequence_id);
read_unlock_bh(pm_lock);
/* Wait for the image download to complete */
- wait_event_timeout(mhi_cntrl->state_event,
- MHI_PM_IN_ERROR_STATE(mhi_cntrl->pm_state) ||
- mhi_read_reg_field(mhi_cntrl, base,
- BHIE_TXVECSTATUS_OFFS,
- BHIE_TXVECSTATUS_STATUS_BMSK,
- BHIE_TXVECSTATUS_STATUS_SHFT,
- &tx_status) || tx_status,
- msecs_to_jiffies(mhi_cntrl->timeout_ms));
-
- if (MHI_PM_IN_ERROR_STATE(mhi_cntrl->pm_state))
+ ret = wait_event_timeout(mhi_cntrl->state_event,
+ MHI_PM_IN_ERROR_STATE(mhi_cntrl->pm_state) ||
+ mhi_read_reg_field(mhi_cntrl, base,
+ BHIE_TXVECSTATUS_OFFS,
+ BHIE_TXVECSTATUS_STATUS_BMSK,
+ BHIE_TXVECSTATUS_STATUS_SHFT,
+ &tx_status) || tx_status,
+ msecs_to_jiffies(mhi_cntrl->timeout_ms));
+ if (MHI_PM_IN_ERROR_STATE(mhi_cntrl->pm_state) ||
+ tx_status != BHIE_TXVECSTATUS_STATUS_XFER_COMPL)
return -EIO;
- return (tx_status == BHIE_TXVECSTATUS_STATUS_XFER_COMPL) ? 0 : -EIO;
+ return (!ret) ? -ETIMEDOUT : 0;
}
static int mhi_fw_load_sbl(struct mhi_controller *mhi_cntrl,
@@ -239,14 +244,15 @@ static int mhi_fw_load_sbl(struct mhi_controller *mhi_cntrl,
goto invalid_pm_state;
}
- dev_dbg(dev, "Starting SBL download via BHI\n");
+ session_id = MHI_RANDOM_U32_NONZERO(BHI_TXDB_SEQNUM_BMSK);
+ dev_dbg(dev, "Starting SBL download via BHI. Session ID:%u\n",
+ session_id);
mhi_write_reg(mhi_cntrl, base, BHI_STATUS, 0);
mhi_write_reg(mhi_cntrl, base, BHI_IMGADDR_HIGH,
upper_32_bits(dma_addr));
mhi_write_reg(mhi_cntrl, base, BHI_IMGADDR_LOW,
lower_32_bits(dma_addr));
mhi_write_reg(mhi_cntrl, base, BHI_IMGSIZE, size);
- session_id = prandom_u32() & BHI_TXDB_SEQNUM_BMSK;
mhi_write_reg(mhi_cntrl, base, BHI_IMGTXDB, session_id);
read_unlock_bh(pm_lock);
@@ -377,30 +383,18 @@ static void mhi_firmware_copy(struct mhi_controller *mhi_cntrl,
}
}
-void mhi_fw_load_worker(struct work_struct *work)
+void mhi_fw_load_handler(struct mhi_controller *mhi_cntrl)
{
- struct mhi_controller *mhi_cntrl;
const struct firmware *firmware = NULL;
struct image_info *image_info;
- struct device *dev;
+ struct device *dev = &mhi_cntrl->mhi_dev->dev;
const char *fw_name;
void *buf;
dma_addr_t dma_addr;
size_t size;
int ret;
- mhi_cntrl = container_of(work, struct mhi_controller, fw_worker);
- dev = &mhi_cntrl->mhi_dev->dev;
-
- dev_dbg(dev, "Waiting for device to enter PBL from: %s\n",
- TO_MHI_EXEC_STR(mhi_cntrl->ee));
-
- ret = wait_event_timeout(mhi_cntrl->state_event,
- MHI_IN_PBL(mhi_cntrl->ee) ||
- MHI_PM_IN_ERROR_STATE(mhi_cntrl->pm_state),
- msecs_to_jiffies(mhi_cntrl->timeout_ms));
-
- if (!ret || MHI_PM_IN_ERROR_STATE(mhi_cntrl->pm_state)) {
+ if (MHI_PM_IN_ERROR_STATE(mhi_cntrl->pm_state)) {
dev_err(dev, "Device MHI is not in valid state\n");
return;
}
@@ -446,7 +440,12 @@ void mhi_fw_load_worker(struct work_struct *work)
release_firmware(firmware);
/* Error or in EDL mode, we're done */
- if (ret || mhi_cntrl->ee == MHI_EE_EDL)
+ if (ret) {
+ dev_err(dev, "MHI did not load SBL, ret:%d\n", ret);
+ return;
+ }
+
+ if (mhi_cntrl->ee == MHI_EE_EDL)
return;
write_lock_irq(&mhi_cntrl->pm_lock);
@@ -474,8 +473,10 @@ fw_load_ee_pthru:
if (!mhi_cntrl->fbc_download)
return;
- if (ret)
+ if (ret) {
+ dev_err(dev, "MHI did not enter READY state\n");
goto error_read;
+ }
/* Wait for the SBL event */
ret = wait_event_timeout(mhi_cntrl->state_event,
@@ -493,6 +494,8 @@ fw_load_ee_pthru:
ret = mhi_fw_load_amss(mhi_cntrl,
/* Vector table is the last entry */
&image_info->mhi_buf[image_info->entries - 1]);
+ if (ret)
+ dev_err(dev, "MHI did not load AMSS, ret:%d\n", ret);
release_firmware(firmware);
diff --git a/drivers/bus/mhi/core/init.c b/drivers/bus/mhi/core/init.c
index 1f8c82603179..e43a190a7a36 100644
--- a/drivers/bus/mhi/core/init.c
+++ b/drivers/bus/mhi/core/init.c
@@ -34,6 +34,8 @@ const char * const dev_state_tran_str[DEV_ST_TRANSITION_MAX] = {
[DEV_ST_TRANSITION_READY] = "READY",
[DEV_ST_TRANSITION_SBL] = "SBL",
[DEV_ST_TRANSITION_MISSION_MODE] = "MISSION_MODE",
+ [DEV_ST_TRANSITION_SYS_ERR] = "SYS_ERR",
+ [DEV_ST_TRANSITION_DISABLE] = "DISABLE",
};
const char * const mhi_state_str[MHI_STATE_MAX] = {
@@ -835,8 +837,6 @@ int mhi_register_controller(struct mhi_controller *mhi_cntrl,
spin_lock_init(&mhi_cntrl->transition_lock);
spin_lock_init(&mhi_cntrl->wlock);
INIT_WORK(&mhi_cntrl->st_worker, mhi_pm_st_worker);
- INIT_WORK(&mhi_cntrl->syserr_worker, mhi_pm_sys_err_worker);
- INIT_WORK(&mhi_cntrl->fw_worker, mhi_fw_load_worker);
init_waitqueue_head(&mhi_cntrl->state_event);
mhi_cmd = mhi_cntrl->mhi_cmd;
@@ -864,6 +864,10 @@ int mhi_register_controller(struct mhi_controller *mhi_cntrl,
mutex_init(&mhi_chan->mutex);
init_completion(&mhi_chan->completion);
rwlock_init(&mhi_chan->lock);
+
+ /* used in setting bei field of TRE */
+ mhi_event = &mhi_cntrl->mhi_event[mhi_chan->er_index];
+ mhi_chan->intmod = mhi_event->intmod;
}
if (mhi_cntrl->bounce_buf) {
diff --git a/drivers/bus/mhi/core/internal.h b/drivers/bus/mhi/core/internal.h
index 095d95bc0e37..b1f640b75a94 100644
--- a/drivers/bus/mhi/core/internal.h
+++ b/drivers/bus/mhi/core/internal.h
@@ -386,6 +386,8 @@ enum dev_st_transition {
DEV_ST_TRANSITION_READY,
DEV_ST_TRANSITION_SBL,
DEV_ST_TRANSITION_MISSION_MODE,
+ DEV_ST_TRANSITION_SYS_ERR,
+ DEV_ST_TRANSITION_DISABLE,
DEV_ST_TRANSITION_MAX,
};
@@ -452,6 +454,7 @@ enum mhi_pm_state {
#define PRIMARY_CMD_RING 0
#define MHI_DEV_WAKE_DB 127
#define MHI_MAX_MTU 0xffff
+#define MHI_RANDOM_U32_NONZERO(bmsk) (prandom_u32_max(bmsk) + 1)
enum mhi_er_type {
MHI_ER_TYPE_INVALID = 0x0,
@@ -586,7 +589,7 @@ enum mhi_ee_type mhi_get_exec_env(struct mhi_controller *mhi_cntrl);
int mhi_queue_state_transition(struct mhi_controller *mhi_cntrl,
enum dev_st_transition state);
void mhi_pm_st_worker(struct work_struct *work);
-void mhi_pm_sys_err_worker(struct work_struct *work);
+void mhi_pm_sys_err_handler(struct mhi_controller *mhi_cntrl);
void mhi_fw_load_worker(struct work_struct *work);
int mhi_ready_state_transition(struct mhi_controller *mhi_cntrl);
void mhi_ctrl_ev_task(unsigned long data);
@@ -627,6 +630,7 @@ int mhi_init_irq_setup(struct mhi_controller *mhi_cntrl);
void mhi_deinit_free_irq(struct mhi_controller *mhi_cntrl);
void mhi_rddm_prepare(struct mhi_controller *mhi_cntrl,
struct image_info *img_info);
+void mhi_fw_load_handler(struct mhi_controller *mhi_cntrl);
int mhi_prepare_channel(struct mhi_controller *mhi_cntrl,
struct mhi_chan *mhi_chan);
int mhi_init_chan_ctxt(struct mhi_controller *mhi_cntrl,
@@ -670,8 +674,7 @@ irqreturn_t mhi_intvec_threaded_handler(int irq_number, void *dev);
irqreturn_t mhi_intvec_handler(int irq_number, void *dev);
int mhi_gen_tre(struct mhi_controller *mhi_cntrl, struct mhi_chan *mhi_chan,
- void *buf, void *cb, size_t buf_len, enum mhi_flags flags);
-
+ struct mhi_buf_info *info, enum mhi_flags flags);
int mhi_map_single_no_bb(struct mhi_controller *mhi_cntrl,
struct mhi_buf_info *buf_info);
int mhi_map_single_use_bb(struct mhi_controller *mhi_cntrl,
diff --git a/drivers/bus/mhi/core/main.c b/drivers/bus/mhi/core/main.c
index 97e06cc586e4..1f622ce6be8b 100644
--- a/drivers/bus/mhi/core/main.c
+++ b/drivers/bus/mhi/core/main.c
@@ -258,7 +258,7 @@ int mhi_destroy_device(struct device *dev, void *data)
return 0;
}
-static void mhi_notify(struct mhi_device *mhi_dev, enum mhi_callback cb_reason)
+void mhi_notify(struct mhi_device *mhi_dev, enum mhi_callback cb_reason)
{
struct mhi_driver *mhi_drv;
@@ -270,6 +270,7 @@ static void mhi_notify(struct mhi_device *mhi_dev, enum mhi_callback cb_reason)
if (mhi_drv->status_cb)
mhi_drv->status_cb(mhi_dev, cb_reason);
}
+EXPORT_SYMBOL_GPL(mhi_notify);
/* Bind MHI channels to MHI devices */
void mhi_create_devices(struct mhi_controller *mhi_cntrl)
@@ -368,30 +369,37 @@ irqreturn_t mhi_irq_handler(int irq_number, void *dev)
return IRQ_HANDLED;
}
-irqreturn_t mhi_intvec_threaded_handler(int irq_number, void *dev)
+irqreturn_t mhi_intvec_threaded_handler(int irq_number, void *priv)
{
- struct mhi_controller *mhi_cntrl = dev;
+ struct mhi_controller *mhi_cntrl = priv;
+ struct device *dev = &mhi_cntrl->mhi_dev->dev;
enum mhi_state state = MHI_STATE_MAX;
enum mhi_pm_state pm_state = 0;
enum mhi_ee_type ee = 0;
write_lock_irq(&mhi_cntrl->pm_lock);
- if (MHI_REG_ACCESS_VALID(mhi_cntrl->pm_state)) {
- state = mhi_get_mhi_state(mhi_cntrl);
- ee = mhi_cntrl->ee;
- mhi_cntrl->ee = mhi_get_exec_env(mhi_cntrl);
+ if (!MHI_REG_ACCESS_VALID(mhi_cntrl->pm_state)) {
+ write_unlock_irq(&mhi_cntrl->pm_lock);
+ goto exit_intvec;
}
+ state = mhi_get_mhi_state(mhi_cntrl);
+ ee = mhi_cntrl->ee;
+ mhi_cntrl->ee = mhi_get_exec_env(mhi_cntrl);
+ dev_dbg(dev, "local ee:%s device ee:%s dev_state:%s\n",
+ TO_MHI_EXEC_STR(mhi_cntrl->ee), TO_MHI_EXEC_STR(ee),
+ TO_MHI_STATE_STR(state));
+
if (state == MHI_STATE_SYS_ERR) {
- dev_dbg(&mhi_cntrl->mhi_dev->dev, "System error detected\n");
+ dev_dbg(dev, "System error detected\n");
pm_state = mhi_tryset_pm_state(mhi_cntrl,
MHI_PM_SYS_ERR_DETECT);
}
write_unlock_irq(&mhi_cntrl->pm_lock);
- /* If device in RDDM don't bother processing SYS error */
- if (mhi_cntrl->ee == MHI_EE_RDDM) {
- if (mhi_cntrl->ee != ee) {
+ /* If device supports RDDM don't bother processing SYS error */
+ if (mhi_cntrl->rddm_image) {
+ if (mhi_cntrl->ee == MHI_EE_RDDM && mhi_cntrl->ee != ee) {
mhi_cntrl->status_cb(mhi_cntrl, MHI_CB_EE_RDDM);
wake_up_all(&mhi_cntrl->state_event);
}
@@ -405,7 +413,7 @@ irqreturn_t mhi_intvec_threaded_handler(int irq_number, void *dev)
if (MHI_IN_PBL(ee))
mhi_cntrl->status_cb(mhi_cntrl, MHI_CB_FATAL_ERROR);
else
- schedule_work(&mhi_cntrl->syserr_worker);
+ mhi_pm_sys_err_handler(mhi_cntrl);
}
exit_intvec:
@@ -513,7 +521,10 @@ static int parse_xfer_event(struct mhi_controller *mhi_cntrl,
mhi_cntrl->unmap_single(mhi_cntrl, buf_info);
result.buf_addr = buf_info->cb_buf;
- result.bytes_xferd = xfer_len;
+
+ /* truncate to buf len if xfer_len is larger */
+ result.bytes_xferd =
+ min_t(u16, xfer_len, buf_info->len);
mhi_del_ring_element(mhi_cntrl, buf_ring);
mhi_del_ring_element(mhi_cntrl, tre_ring);
local_rp = tre_ring->rp;
@@ -597,7 +608,9 @@ static int parse_rsc_event(struct mhi_controller *mhi_cntrl,
result.transaction_status = (ev_code == MHI_EV_CC_OVERFLOW) ?
-EOVERFLOW : 0;
- result.bytes_xferd = xfer_len;
+
+ /* truncate to buf len if xfer_len is larger */
+ result.bytes_xferd = min_t(u16, xfer_len, buf_info->len);
result.buf_addr = buf_info->cb_buf;
result.dir = mhi_chan->dir;
@@ -722,13 +735,18 @@ int mhi_process_ctrl_ev_ring(struct mhi_controller *mhi_cntrl,
{
enum mhi_pm_state new_state;
+ /* skip SYS_ERROR handling if RDDM supported */
+ if (mhi_cntrl->ee == MHI_EE_RDDM ||
+ mhi_cntrl->rddm_image)
+ break;
+
dev_dbg(dev, "System error detected\n");
write_lock_irq(&mhi_cntrl->pm_lock);
new_state = mhi_tryset_pm_state(mhi_cntrl,
MHI_PM_SYS_ERR_DETECT);
write_unlock_irq(&mhi_cntrl->pm_lock);
if (new_state == MHI_PM_SYS_ERR_DETECT)
- schedule_work(&mhi_cntrl->syserr_worker);
+ mhi_pm_sys_err_handler(mhi_cntrl);
break;
}
default:
@@ -774,9 +792,18 @@ int mhi_process_ctrl_ev_ring(struct mhi_controller *mhi_cntrl,
}
case MHI_PKT_TYPE_TX_EVENT:
chan = MHI_TRE_GET_EV_CHID(local_rp);
- mhi_chan = &mhi_cntrl->mhi_chan[chan];
- parse_xfer_event(mhi_cntrl, local_rp, mhi_chan);
- event_quota--;
+
+ WARN_ON(chan >= mhi_cntrl->max_chan);
+
+ /*
+ * Only process the event ring elements whose channel
+ * ID is within the maximum supported range.
+ */
+ if (chan < mhi_cntrl->max_chan) {
+ mhi_chan = &mhi_cntrl->mhi_chan[chan];
+ parse_xfer_event(mhi_cntrl, local_rp, mhi_chan);
+ event_quota--;
+ }
break;
default:
dev_err(dev, "Unhandled event type: %d\n", type);
@@ -819,14 +846,23 @@ int mhi_process_data_event_ring(struct mhi_controller *mhi_cntrl,
enum mhi_pkt_type type = MHI_TRE_GET_EV_TYPE(local_rp);
chan = MHI_TRE_GET_EV_CHID(local_rp);
- mhi_chan = &mhi_cntrl->mhi_chan[chan];
-
- if (likely(type == MHI_PKT_TYPE_TX_EVENT)) {
- parse_xfer_event(mhi_cntrl, local_rp, mhi_chan);
- event_quota--;
- } else if (type == MHI_PKT_TYPE_RSC_TX_EVENT) {
- parse_rsc_event(mhi_cntrl, local_rp, mhi_chan);
- event_quota--;
+
+ WARN_ON(chan >= mhi_cntrl->max_chan);
+
+ /*
+ * Only process the event ring elements whose channel
+ * ID is within the maximum supported range.
+ */
+ if (chan < mhi_cntrl->max_chan) {
+ mhi_chan = &mhi_cntrl->mhi_chan[chan];
+
+ if (likely(type == MHI_PKT_TYPE_TX_EVENT)) {
+ parse_xfer_event(mhi_cntrl, local_rp, mhi_chan);
+ event_quota--;
+ } else if (type == MHI_PKT_TYPE_RSC_TX_EVENT) {
+ parse_rsc_event(mhi_cntrl, local_rp, mhi_chan);
+ event_quota--;
+ }
}
mhi_recycle_ev_ring_element(mhi_cntrl, ev_ring);
@@ -896,7 +932,7 @@ void mhi_ctrl_ev_task(unsigned long data)
}
write_unlock_irq(&mhi_cntrl->pm_lock);
if (pm_state == MHI_PM_SYS_ERR_DETECT)
- schedule_work(&mhi_cntrl->syserr_worker);
+ mhi_pm_sys_err_handler(mhi_cntrl);
}
}
@@ -918,9 +954,7 @@ int mhi_queue_skb(struct mhi_device *mhi_dev, enum dma_data_direction dir,
struct mhi_chan *mhi_chan = (dir == DMA_TO_DEVICE) ? mhi_dev->ul_chan :
mhi_dev->dl_chan;
struct mhi_ring *tre_ring = &mhi_chan->tre_ring;
- struct mhi_ring *buf_ring = &mhi_chan->buf_ring;
- struct mhi_buf_info *buf_info;
- struct mhi_tre *mhi_tre;
+ struct mhi_buf_info buf_info = { };
int ret;
/* If MHI host pre-allocates buffers then client drivers cannot queue */
@@ -945,27 +979,15 @@ int mhi_queue_skb(struct mhi_device *mhi_dev, enum dma_data_direction dir,
/* Toggle wake to exit out of M2 */
mhi_cntrl->wake_toggle(mhi_cntrl);
- /* Generate the TRE */
- buf_info = buf_ring->wp;
+ buf_info.v_addr = skb->data;
+ buf_info.cb_buf = skb;
+ buf_info.len = len;
- buf_info->v_addr = skb->data;
- buf_info->cb_buf = skb;
- buf_info->wp = tre_ring->wp;
- buf_info->dir = mhi_chan->dir;
- buf_info->len = len;
- ret = mhi_cntrl->map_single(mhi_cntrl, buf_info);
- if (ret)
- goto map_error;
-
- mhi_tre = tre_ring->wp;
-
- mhi_tre->ptr = MHI_TRE_DATA_PTR(buf_info->p_addr);
- mhi_tre->dword[0] = MHI_TRE_DATA_DWORD0(buf_info->len);
- mhi_tre->dword[1] = MHI_TRE_DATA_DWORD1(1, 1, 0, 0);
-
- /* increment WP */
- mhi_add_ring_element(mhi_cntrl, tre_ring);
- mhi_add_ring_element(mhi_cntrl, buf_ring);
+ ret = mhi_gen_tre(mhi_cntrl, mhi_chan, &buf_info, mflags);
+ if (unlikely(ret)) {
+ read_unlock_bh(&mhi_cntrl->pm_lock);
+ return ret;
+ }
if (mhi_chan->dir == DMA_TO_DEVICE)
atomic_inc(&mhi_cntrl->pending_pkts);
@@ -979,11 +1001,6 @@ int mhi_queue_skb(struct mhi_device *mhi_dev, enum dma_data_direction dir,
read_unlock_bh(&mhi_cntrl->pm_lock);
return 0;
-
-map_error:
- read_unlock_bh(&mhi_cntrl->pm_lock);
-
- return ret;
}
EXPORT_SYMBOL_GPL(mhi_queue_skb);
@@ -995,9 +1012,8 @@ int mhi_queue_dma(struct mhi_device *mhi_dev, enum dma_data_direction dir,
mhi_dev->dl_chan;
struct device *dev = &mhi_cntrl->mhi_dev->dev;
struct mhi_ring *tre_ring = &mhi_chan->tre_ring;
- struct mhi_ring *buf_ring = &mhi_chan->buf_ring;
- struct mhi_buf_info *buf_info;
- struct mhi_tre *mhi_tre;
+ struct mhi_buf_info buf_info = { };
+ int ret;
/* If MHI host pre-allocates buffers then client drivers cannot queue */
if (mhi_chan->pre_alloc)
@@ -1024,25 +1040,16 @@ int mhi_queue_dma(struct mhi_device *mhi_dev, enum dma_data_direction dir,
/* Toggle wake to exit out of M2 */
mhi_cntrl->wake_toggle(mhi_cntrl);
- /* Generate the TRE */
- buf_info = buf_ring->wp;
- WARN_ON(buf_info->used);
- buf_info->p_addr = mhi_buf->dma_addr;
- buf_info->pre_mapped = true;
- buf_info->cb_buf = mhi_buf;
- buf_info->wp = tre_ring->wp;
- buf_info->dir = mhi_chan->dir;
- buf_info->len = len;
-
- mhi_tre = tre_ring->wp;
-
- mhi_tre->ptr = MHI_TRE_DATA_PTR(buf_info->p_addr);
- mhi_tre->dword[0] = MHI_TRE_DATA_DWORD0(buf_info->len);
- mhi_tre->dword[1] = MHI_TRE_DATA_DWORD1(1, 1, 0, 0);
+ buf_info.p_addr = mhi_buf->dma_addr;
+ buf_info.cb_buf = mhi_buf;
+ buf_info.pre_mapped = true;
+ buf_info.len = len;
- /* increment WP */
- mhi_add_ring_element(mhi_cntrl, tre_ring);
- mhi_add_ring_element(mhi_cntrl, buf_ring);
+ ret = mhi_gen_tre(mhi_cntrl, mhi_chan, &buf_info, mflags);
+ if (unlikely(ret)) {
+ read_unlock_bh(&mhi_cntrl->pm_lock);
+ return ret;
+ }
if (mhi_chan->dir == DMA_TO_DEVICE)
atomic_inc(&mhi_cntrl->pending_pkts);
@@ -1060,7 +1067,7 @@ int mhi_queue_dma(struct mhi_device *mhi_dev, enum dma_data_direction dir,
EXPORT_SYMBOL_GPL(mhi_queue_dma);
int mhi_gen_tre(struct mhi_controller *mhi_cntrl, struct mhi_chan *mhi_chan,
- void *buf, void *cb, size_t buf_len, enum mhi_flags flags)
+ struct mhi_buf_info *info, enum mhi_flags flags)
{
struct mhi_ring *buf_ring, *tre_ring;
struct mhi_tre *mhi_tre;
@@ -1072,15 +1079,22 @@ int mhi_gen_tre(struct mhi_controller *mhi_cntrl, struct mhi_chan *mhi_chan,
tre_ring = &mhi_chan->tre_ring;
buf_info = buf_ring->wp;
- buf_info->v_addr = buf;
- buf_info->cb_buf = cb;
+ WARN_ON(buf_info->used);
+ buf_info->pre_mapped = info->pre_mapped;
+ if (info->pre_mapped)
+ buf_info->p_addr = info->p_addr;
+ else
+ buf_info->v_addr = info->v_addr;
+ buf_info->cb_buf = info->cb_buf;
buf_info->wp = tre_ring->wp;
buf_info->dir = mhi_chan->dir;
- buf_info->len = buf_len;
+ buf_info->len = info->len;
- ret = mhi_cntrl->map_single(mhi_cntrl, buf_info);
- if (ret)
- return ret;
+ if (!info->pre_mapped) {
+ ret = mhi_cntrl->map_single(mhi_cntrl, buf_info);
+ if (ret)
+ return ret;
+ }
eob = !!(flags & MHI_EOB);
eot = !!(flags & MHI_EOT);
@@ -1089,7 +1103,7 @@ int mhi_gen_tre(struct mhi_controller *mhi_cntrl, struct mhi_chan *mhi_chan,
mhi_tre = tre_ring->wp;
mhi_tre->ptr = MHI_TRE_DATA_PTR(buf_info->p_addr);
- mhi_tre->dword[0] = MHI_TRE_DATA_DWORD0(buf_len);
+ mhi_tre->dword[0] = MHI_TRE_DATA_DWORD0(info->len);
mhi_tre->dword[1] = MHI_TRE_DATA_DWORD1(bei, eot, eob, chain);
/* increment WP */
@@ -1106,6 +1120,7 @@ int mhi_queue_buf(struct mhi_device *mhi_dev, enum dma_data_direction dir,
struct mhi_chan *mhi_chan = (dir == DMA_TO_DEVICE) ? mhi_dev->ul_chan :
mhi_dev->dl_chan;
struct mhi_ring *tre_ring;
+ struct mhi_buf_info buf_info = { };
unsigned long flags;
int ret;
@@ -1121,7 +1136,11 @@ int mhi_queue_buf(struct mhi_device *mhi_dev, enum dma_data_direction dir,
if (mhi_is_ring_full(mhi_cntrl, tre_ring))
return -ENOMEM;
- ret = mhi_gen_tre(mhi_cntrl, mhi_chan, buf, buf, len, mflags);
+ buf_info.v_addr = buf;
+ buf_info.cb_buf = buf;
+ buf_info.len = len;
+
+ ret = mhi_gen_tre(mhi_cntrl, mhi_chan, &buf_info, mflags);
if (unlikely(ret))
return ret;
@@ -1322,7 +1341,7 @@ int mhi_prepare_channel(struct mhi_controller *mhi_cntrl,
while (nr_el--) {
void *buf;
-
+ struct mhi_buf_info info = { };
buf = kmalloc(len, GFP_KERNEL);
if (!buf) {
ret = -ENOMEM;
@@ -1330,8 +1349,10 @@ int mhi_prepare_channel(struct mhi_controller *mhi_cntrl,
}
/* Prepare transfer descriptors */
- ret = mhi_gen_tre(mhi_cntrl, mhi_chan, buf, buf,
- len, MHI_EOT);
+ info.v_addr = buf;
+ info.cb_buf = buf;
+ info.len = len;
+ ret = mhi_gen_tre(mhi_cntrl, mhi_chan, &info, MHI_EOT);
if (ret) {
kfree(buf);
goto error_pre_alloc;
diff --git a/drivers/bus/mhi/core/pm.c b/drivers/bus/mhi/core/pm.c
index dc83d65f7784..796098078083 100644
--- a/drivers/bus/mhi/core/pm.c
+++ b/drivers/bus/mhi/core/pm.c
@@ -288,14 +288,18 @@ int mhi_pm_m0_transition(struct mhi_controller *mhi_cntrl)
for (i = 0; i < mhi_cntrl->max_chan; i++, mhi_chan++) {
struct mhi_ring *tre_ring = &mhi_chan->tre_ring;
- write_lock_irq(&mhi_chan->lock);
- if (mhi_chan->db_cfg.reset_req)
+ if (mhi_chan->db_cfg.reset_req) {
+ write_lock_irq(&mhi_chan->lock);
mhi_chan->db_cfg.db_mode = true;
+ write_unlock_irq(&mhi_chan->lock);
+ }
+
+ read_lock_irq(&mhi_chan->lock);
/* Only ring DB if ring is not empty */
if (tre_ring->base && tre_ring->wp != tre_ring->rp)
mhi_ring_chan_db(mhi_cntrl, mhi_chan);
- write_unlock_irq(&mhi_chan->lock);
+ read_unlock_irq(&mhi_chan->lock);
}
mhi_cntrl->wake_put(mhi_cntrl, false);
@@ -449,19 +453,8 @@ static void mhi_pm_disable_transition(struct mhi_controller *mhi_cntrl,
to_mhi_pm_state_str(transition_state));
/* We must notify MHI control driver so it can clean up first */
- if (transition_state == MHI_PM_SYS_ERR_PROCESS) {
- /*
- * If controller supports RDDM, we do not process
- * SYS error state, instead we will jump directly
- * to RDDM state
- */
- if (mhi_cntrl->rddm_image) {
- dev_dbg(dev,
- "Controller supports RDDM, so skip SYS_ERR\n");
- return;
- }
+ if (transition_state == MHI_PM_SYS_ERR_PROCESS)
mhi_cntrl->status_cb(mhi_cntrl, MHI_CB_SYS_ERROR);
- }
mutex_lock(&mhi_cntrl->pm_mutex);
write_lock_irq(&mhi_cntrl->pm_lock);
@@ -527,8 +520,6 @@ static void mhi_pm_disable_transition(struct mhi_controller *mhi_cntrl,
mutex_unlock(&mhi_cntrl->pm_mutex);
dev_dbg(dev, "Waiting for all pending threads to complete\n");
wake_up_all(&mhi_cntrl->state_event);
- flush_work(&mhi_cntrl->st_worker);
- flush_work(&mhi_cntrl->fw_worker);
dev_dbg(dev, "Reset all active channels and remove MHI devices\n");
device_for_each_child(mhi_cntrl->cntrl_dev, NULL, mhi_destroy_device);
@@ -608,13 +599,17 @@ int mhi_queue_state_transition(struct mhi_controller *mhi_cntrl,
}
/* SYS_ERR worker */
-void mhi_pm_sys_err_worker(struct work_struct *work)
+void mhi_pm_sys_err_handler(struct mhi_controller *mhi_cntrl)
{
- struct mhi_controller *mhi_cntrl = container_of(work,
- struct mhi_controller,
- syserr_worker);
+ struct device *dev = &mhi_cntrl->mhi_dev->dev;
+
+ /* skip if controller supports RDDM */
+ if (mhi_cntrl->rddm_image) {
+ dev_dbg(dev, "Controller supports RDDM, skip SYS_ERROR\n");
+ return;
+ }
- mhi_pm_disable_transition(mhi_cntrl, MHI_PM_SYS_ERR_PROCESS);
+ mhi_queue_state_transition(mhi_cntrl, DEV_ST_TRANSITION_SYS_ERR);
}
/* Device State Transition worker */
@@ -643,7 +638,7 @@ void mhi_pm_st_worker(struct work_struct *work)
mhi_cntrl->ee = mhi_get_exec_env(mhi_cntrl);
write_unlock_irq(&mhi_cntrl->pm_lock);
if (MHI_IN_PBL(mhi_cntrl->ee))
- wake_up_all(&mhi_cntrl->state_event);
+ mhi_fw_load_handler(mhi_cntrl);
break;
case DEV_ST_TRANSITION_SBL:
write_lock_irq(&mhi_cntrl->pm_lock);
@@ -662,6 +657,14 @@ void mhi_pm_st_worker(struct work_struct *work)
case DEV_ST_TRANSITION_READY:
mhi_ready_state_transition(mhi_cntrl);
break;
+ case DEV_ST_TRANSITION_SYS_ERR:
+ mhi_pm_disable_transition
+ (mhi_cntrl, MHI_PM_SYS_ERR_PROCESS);
+ break;
+ case DEV_ST_TRANSITION_DISABLE:
+ mhi_pm_disable_transition
+ (mhi_cntrl, MHI_PM_SHUTDOWN_PROCESS);
+ break;
default:
break;
}
@@ -669,6 +672,149 @@ void mhi_pm_st_worker(struct work_struct *work)
}
}
+int mhi_pm_suspend(struct mhi_controller *mhi_cntrl)
+{
+ struct mhi_chan *itr, *tmp;
+ struct device *dev = &mhi_cntrl->mhi_dev->dev;
+ enum mhi_pm_state new_state;
+ int ret;
+
+ if (mhi_cntrl->pm_state == MHI_PM_DISABLE)
+ return -EINVAL;
+
+ if (MHI_PM_IN_ERROR_STATE(mhi_cntrl->pm_state))
+ return -EIO;
+
+ /* Return busy if there are any pending resources */
+ if (atomic_read(&mhi_cntrl->dev_wake))
+ return -EBUSY;
+
+ /* Take MHI out of M2 state */
+ read_lock_bh(&mhi_cntrl->pm_lock);
+ mhi_cntrl->wake_get(mhi_cntrl, false);
+ read_unlock_bh(&mhi_cntrl->pm_lock);
+
+ ret = wait_event_timeout(mhi_cntrl->state_event,
+ mhi_cntrl->dev_state == MHI_STATE_M0 ||
+ mhi_cntrl->dev_state == MHI_STATE_M1 ||
+ MHI_PM_IN_ERROR_STATE(mhi_cntrl->pm_state),
+ msecs_to_jiffies(mhi_cntrl->timeout_ms));
+
+ read_lock_bh(&mhi_cntrl->pm_lock);
+ mhi_cntrl->wake_put(mhi_cntrl, false);
+ read_unlock_bh(&mhi_cntrl->pm_lock);
+
+ if (!ret || MHI_PM_IN_ERROR_STATE(mhi_cntrl->pm_state)) {
+ dev_err(dev,
+ "Could not enter M0/M1 state");
+ return -EIO;
+ }
+
+ write_lock_irq(&mhi_cntrl->pm_lock);
+
+ if (atomic_read(&mhi_cntrl->dev_wake)) {
+ write_unlock_irq(&mhi_cntrl->pm_lock);
+ return -EBUSY;
+ }
+
+ dev_info(dev, "Allowing M3 transition\n");
+ new_state = mhi_tryset_pm_state(mhi_cntrl, MHI_PM_M3_ENTER);
+ if (new_state != MHI_PM_M3_ENTER) {
+ write_unlock_irq(&mhi_cntrl->pm_lock);
+ dev_err(dev,
+ "Error setting to PM state: %s from: %s\n",
+ to_mhi_pm_state_str(MHI_PM_M3_ENTER),
+ to_mhi_pm_state_str(mhi_cntrl->pm_state));
+ return -EIO;
+ }
+
+ /* Set MHI to M3 and wait for completion */
+ mhi_set_mhi_state(mhi_cntrl, MHI_STATE_M3);
+ write_unlock_irq(&mhi_cntrl->pm_lock);
+ dev_info(dev, "Wait for M3 completion\n");
+
+ ret = wait_event_timeout(mhi_cntrl->state_event,
+ mhi_cntrl->dev_state == MHI_STATE_M3 ||
+ MHI_PM_IN_ERROR_STATE(mhi_cntrl->pm_state),
+ msecs_to_jiffies(mhi_cntrl->timeout_ms));
+
+ if (!ret || MHI_PM_IN_ERROR_STATE(mhi_cntrl->pm_state)) {
+ dev_err(dev,
+ "Did not enter M3 state, MHI state: %s, PM state: %s\n",
+ TO_MHI_STATE_STR(mhi_cntrl->dev_state),
+ to_mhi_pm_state_str(mhi_cntrl->pm_state));
+ return -EIO;
+ }
+
+ /* Notify clients about entering LPM */
+ list_for_each_entry_safe(itr, tmp, &mhi_cntrl->lpm_chans, node) {
+ mutex_lock(&itr->mutex);
+ if (itr->mhi_dev)
+ mhi_notify(itr->mhi_dev, MHI_CB_LPM_ENTER);
+ mutex_unlock(&itr->mutex);
+ }
+
+ return 0;
+}
+EXPORT_SYMBOL_GPL(mhi_pm_suspend);
+
+int mhi_pm_resume(struct mhi_controller *mhi_cntrl)
+{
+ struct mhi_chan *itr, *tmp;
+ struct device *dev = &mhi_cntrl->mhi_dev->dev;
+ enum mhi_pm_state cur_state;
+ int ret;
+
+ dev_info(dev, "Entered with PM state: %s, MHI state: %s\n",
+ to_mhi_pm_state_str(mhi_cntrl->pm_state),
+ TO_MHI_STATE_STR(mhi_cntrl->dev_state));
+
+ if (mhi_cntrl->pm_state == MHI_PM_DISABLE)
+ return 0;
+
+ if (MHI_PM_IN_ERROR_STATE(mhi_cntrl->pm_state))
+ return -EIO;
+
+ /* Notify clients about exiting LPM */
+ list_for_each_entry_safe(itr, tmp, &mhi_cntrl->lpm_chans, node) {
+ mutex_lock(&itr->mutex);
+ if (itr->mhi_dev)
+ mhi_notify(itr->mhi_dev, MHI_CB_LPM_EXIT);
+ mutex_unlock(&itr->mutex);
+ }
+
+ write_lock_irq(&mhi_cntrl->pm_lock);
+ cur_state = mhi_tryset_pm_state(mhi_cntrl, MHI_PM_M3_EXIT);
+ if (cur_state != MHI_PM_M3_EXIT) {
+ write_unlock_irq(&mhi_cntrl->pm_lock);
+ dev_info(dev,
+ "Error setting to PM state: %s from: %s\n",
+ to_mhi_pm_state_str(MHI_PM_M3_EXIT),
+ to_mhi_pm_state_str(mhi_cntrl->pm_state));
+ return -EIO;
+ }
+
+ /* Set MHI to M0 and wait for completion */
+ mhi_set_mhi_state(mhi_cntrl, MHI_STATE_M0);
+ write_unlock_irq(&mhi_cntrl->pm_lock);
+
+ ret = wait_event_timeout(mhi_cntrl->state_event,
+ mhi_cntrl->dev_state == MHI_STATE_M0 ||
+ MHI_PM_IN_ERROR_STATE(mhi_cntrl->pm_state),
+ msecs_to_jiffies(mhi_cntrl->timeout_ms));
+
+ if (!ret || MHI_PM_IN_ERROR_STATE(mhi_cntrl->pm_state)) {
+ dev_err(dev,
+ "Did not enter M0 state, MHI state: %s, PM state: %s\n",
+ TO_MHI_STATE_STR(mhi_cntrl->dev_state),
+ to_mhi_pm_state_str(mhi_cntrl->pm_state));
+ return -EIO;
+ }
+
+ return 0;
+}
+EXPORT_SYMBOL_GPL(mhi_pm_resume);
+
int __mhi_device_get_sync(struct mhi_controller *mhi_cntrl)
{
int ret;
@@ -760,6 +906,7 @@ static void mhi_deassert_dev_wake(struct mhi_controller *mhi_cntrl,
int mhi_async_power_up(struct mhi_controller *mhi_cntrl)
{
+ enum mhi_state state;
enum mhi_ee_type current_ee;
enum dev_st_transition next_state;
struct device *dev = &mhi_cntrl->mhi_dev->dev;
@@ -829,13 +976,36 @@ int mhi_async_power_up(struct mhi_controller *mhi_cntrl)
goto error_bhi_offset;
}
+ state = mhi_get_mhi_state(mhi_cntrl);
+ if (state == MHI_STATE_SYS_ERR) {
+ mhi_set_mhi_state(mhi_cntrl, MHI_STATE_RESET);
+ ret = wait_event_timeout(mhi_cntrl->state_event,
+ MHI_PM_IN_FATAL_STATE(mhi_cntrl->pm_state) ||
+ mhi_read_reg_field(mhi_cntrl,
+ mhi_cntrl->regs,
+ MHICTRL,
+ MHICTRL_RESET_MASK,
+ MHICTRL_RESET_SHIFT,
+ &val) ||
+ !val,
+ msecs_to_jiffies(mhi_cntrl->timeout_ms));
+ if (ret) {
+ ret = -EIO;
+ dev_info(dev, "Failed to reset MHI due to syserr state\n");
+ goto error_bhi_offset;
+ }
+
+ /*
+ * device cleares INTVEC as part of RESET processing,
+ * re-program it
+ */
+ mhi_write_reg(mhi_cntrl, mhi_cntrl->bhi, BHI_INTVEC, 0);
+ }
+
/* Transition to next state */
next_state = MHI_IN_PBL(current_ee) ?
DEV_ST_TRANSITION_PBL : DEV_ST_TRANSITION_READY;
- if (next_state == DEV_ST_TRANSITION_PBL)
- schedule_work(&mhi_cntrl->fw_worker);
-
mhi_queue_state_transition(mhi_cntrl, next_state);
mutex_unlock(&mhi_cntrl->pm_mutex);
@@ -876,7 +1046,12 @@ void mhi_power_down(struct mhi_controller *mhi_cntrl, bool graceful)
to_mhi_pm_state_str(MHI_PM_LD_ERR_FATAL_DETECT),
to_mhi_pm_state_str(mhi_cntrl->pm_state));
}
- mhi_pm_disable_transition(mhi_cntrl, MHI_PM_SHUTDOWN_PROCESS);
+
+ mhi_queue_state_transition(mhi_cntrl, DEV_ST_TRANSITION_DISABLE);
+
+ /* Wait for shutdown to complete */
+ flush_work(&mhi_cntrl->st_worker);
+
mhi_deinit_free_irq(mhi_cntrl);
if (!mhi_cntrl->pre_init) {
diff --git a/drivers/char/mem.c b/drivers/char/mem.c
index 43dd0891ca1e..31cae88a730b 100644
--- a/drivers/char/mem.c
+++ b/drivers/char/mem.c
@@ -31,11 +31,15 @@
#include <linux/uio.h>
#include <linux/uaccess.h>
#include <linux/security.h>
+#include <linux/pseudo_fs.h>
+#include <uapi/linux/magic.h>
+#include <linux/mount.h>
#ifdef CONFIG_IA64
# include <linux/efi.h>
#endif
+#define DEVMEM_MINOR 1
#define DEVPORT_MINOR 4
static inline unsigned long size_inside_page(unsigned long start,
@@ -805,12 +809,64 @@ static loff_t memory_lseek(struct file *file, loff_t offset, int orig)
return ret;
}
+static struct inode *devmem_inode;
+
+#ifdef CONFIG_IO_STRICT_DEVMEM
+void revoke_devmem(struct resource *res)
+{
+ struct inode *inode = READ_ONCE(devmem_inode);
+
+ /*
+ * Check that the initialization has completed. Losing the race
+ * is ok because it means drivers are claiming resources before
+ * the fs_initcall level of init and prevent /dev/mem from
+ * establishing mappings.
+ */
+ if (!inode)
+ return;
+
+ /*
+ * The expectation is that the driver has successfully marked
+ * the resource busy by this point, so devmem_is_allowed()
+ * should start returning false, however for performance this
+ * does not iterate the entire resource range.
+ */
+ if (devmem_is_allowed(PHYS_PFN(res->start)) &&
+ devmem_is_allowed(PHYS_PFN(res->end))) {
+ /*
+ * *cringe* iomem=relaxed says "go ahead, what's the
+ * worst that can happen?"
+ */
+ return;
+ }
+
+ unmap_mapping_range(inode->i_mapping, res->start, resource_size(res), 1);
+}
+#endif
+
static int open_port(struct inode *inode, struct file *filp)
{
+ int rc;
+
if (!capable(CAP_SYS_RAWIO))
return -EPERM;
- return security_locked_down(LOCKDOWN_DEV_MEM);
+ rc = security_locked_down(LOCKDOWN_DEV_MEM);
+ if (rc)
+ return rc;
+
+ if (iminor(inode) != DEVMEM_MINOR)
+ return 0;
+
+ /*
+ * Use a unified address space to have a single point to manage
+ * revocations when drivers want to take over a /dev/mem mapped
+ * range.
+ */
+ inode->i_mapping = devmem_inode->i_mapping;
+ filp->f_mapping = inode->i_mapping;
+
+ return 0;
}
#define zero_lseek null_lseek
@@ -885,7 +941,7 @@ static const struct memdev {
fmode_t fmode;
} devlist[] = {
#ifdef CONFIG_DEVMEM
- [1] = { "mem", 0, &mem_fops, FMODE_UNSIGNED_OFFSET },
+ [DEVMEM_MINOR] = { "mem", 0, &mem_fops, FMODE_UNSIGNED_OFFSET },
#endif
#ifdef CONFIG_DEVKMEM
[2] = { "kmem", 0, &kmem_fops, FMODE_UNSIGNED_OFFSET },
@@ -939,6 +995,45 @@ static char *mem_devnode(struct device *dev, umode_t *mode)
static struct class *mem_class;
+static int devmem_fs_init_fs_context(struct fs_context *fc)
+{
+ return init_pseudo(fc, DEVMEM_MAGIC) ? 0 : -ENOMEM;
+}
+
+static struct file_system_type devmem_fs_type = {
+ .name = "devmem",
+ .owner = THIS_MODULE,
+ .init_fs_context = devmem_fs_init_fs_context,
+ .kill_sb = kill_anon_super,
+};
+
+static int devmem_init_inode(void)
+{
+ static struct vfsmount *devmem_vfs_mount;
+ static int devmem_fs_cnt;
+ struct inode *inode;
+ int rc;
+
+ rc = simple_pin_fs(&devmem_fs_type, &devmem_vfs_mount, &devmem_fs_cnt);
+ if (rc < 0) {
+ pr_err("Cannot mount /dev/mem pseudo filesystem: %d\n", rc);
+ return rc;
+ }
+
+ inode = alloc_anon_inode(devmem_vfs_mount->mnt_sb);
+ if (IS_ERR(inode)) {
+ rc = PTR_ERR(inode);
+ pr_err("Cannot allocate inode for /dev/mem: %d\n", rc);
+ simple_release_fs(&devmem_vfs_mount, &devmem_fs_cnt);
+ return rc;
+ }
+
+ /* publish /dev/mem initialized */
+ WRITE_ONCE(devmem_inode, inode);
+
+ return 0;
+}
+
static int __init chr_dev_init(void)
{
int minor;
@@ -960,6 +1055,8 @@ static int __init chr_dev_init(void)
*/
if ((minor == DEVPORT_MINOR) && !arch_has_dev_port())
continue;
+ if ((minor == DEVMEM_MINOR) && devmem_init_inode() != 0)
+ continue;
device_create(mem_class, NULL, MKDEV(MEM_MAJOR, minor),
NULL, devlist[minor].name);
diff --git a/drivers/char/tlclk.c b/drivers/char/tlclk.c
index 6d81bb3bb503..896a3550fba9 100644
--- a/drivers/char/tlclk.c
+++ b/drivers/char/tlclk.c
@@ -777,17 +777,21 @@ static int __init tlclk_init(void)
{
int ret;
+ telclk_interrupt = (inb(TLCLK_REG7) & 0x0f);
+
+ alarm_events = kzalloc( sizeof(struct tlclk_alarms), GFP_KERNEL);
+ if (!alarm_events) {
+ ret = -ENOMEM;
+ goto out1;
+ }
+
ret = register_chrdev(tlclk_major, "telco_clock", &tlclk_fops);
if (ret < 0) {
printk(KERN_ERR "tlclk: can't get major %d.\n", tlclk_major);
+ kfree(alarm_events);
return ret;
}
tlclk_major = ret;
- alarm_events = kzalloc( sizeof(struct tlclk_alarms), GFP_KERNEL);
- if (!alarm_events) {
- ret = -ENOMEM;
- goto out1;
- }
/* Read telecom clock IRQ number (Set by BIOS) */
if (!request_region(TLCLK_BASE, 8, "telco_clock")) {
@@ -796,7 +800,6 @@ static int __init tlclk_init(void)
ret = -EBUSY;
goto out2;
}
- telclk_interrupt = (inb(TLCLK_REG7) & 0x0f);
if (0x0F == telclk_interrupt ) { /* not MCPBL0010 ? */
printk(KERN_ERR "telclk_interrupt = 0x%x non-mcpbl0010 hw.\n",
@@ -837,8 +840,8 @@ out3:
release_region(TLCLK_BASE, 8);
out2:
kfree(alarm_events);
-out1:
unregister_chrdev(tlclk_major, "telco_clock");
+out1:
return ret;
}
diff --git a/drivers/clk/zynqmp/clk-gate-zynqmp.c b/drivers/clk/zynqmp/clk-gate-zynqmp.c
index 83b236f20fff..10c9b889324f 100644
--- a/drivers/clk/zynqmp/clk-gate-zynqmp.c
+++ b/drivers/clk/zynqmp/clk-gate-zynqmp.c
@@ -37,9 +37,8 @@ static int zynqmp_clk_gate_enable(struct clk_hw *hw)
const char *clk_name = clk_hw_get_name(hw);
u32 clk_id = gate->clk_id;
int ret;
- const struct zynqmp_eemi_ops *eemi_ops = zynqmp_pm_get_eemi_ops();
- ret = eemi_ops->clock_enable(clk_id);
+ ret = zynqmp_pm_clock_enable(clk_id);
if (ret)
pr_warn_once("%s() clock enabled failed for %s, ret = %d\n",
@@ -58,9 +57,8 @@ static void zynqmp_clk_gate_disable(struct clk_hw *hw)
const char *clk_name = clk_hw_get_name(hw);
u32 clk_id = gate->clk_id;
int ret;
- const struct zynqmp_eemi_ops *eemi_ops = zynqmp_pm_get_eemi_ops();
- ret = eemi_ops->clock_disable(clk_id);
+ ret = zynqmp_pm_clock_disable(clk_id);
if (ret)
pr_warn_once("%s() clock disable failed for %s, ret = %d\n",
@@ -79,9 +77,8 @@ static int zynqmp_clk_gate_is_enabled(struct clk_hw *hw)
const char *clk_name = clk_hw_get_name(hw);
u32 clk_id = gate->clk_id;
int state, ret;
- const struct zynqmp_eemi_ops *eemi_ops = zynqmp_pm_get_eemi_ops();
- ret = eemi_ops->clock_getstate(clk_id, &state);
+ ret = zynqmp_pm_clock_getstate(clk_id, &state);
if (ret) {
pr_warn_once("%s() clock get state failed for %s, ret = %d\n",
__func__, clk_name, ret);
diff --git a/drivers/clk/zynqmp/clk-mux-zynqmp.c b/drivers/clk/zynqmp/clk-mux-zynqmp.c
index 0af8f74c5fa5..06194149be83 100644
--- a/drivers/clk/zynqmp/clk-mux-zynqmp.c
+++ b/drivers/clk/zynqmp/clk-mux-zynqmp.c
@@ -47,9 +47,8 @@ static u8 zynqmp_clk_mux_get_parent(struct clk_hw *hw)
u32 clk_id = mux->clk_id;
u32 val;
int ret;
- const struct zynqmp_eemi_ops *eemi_ops = zynqmp_pm_get_eemi_ops();
- ret = eemi_ops->clock_getparent(clk_id, &val);
+ ret = zynqmp_pm_clock_getparent(clk_id, &val);
if (ret)
pr_warn_once("%s() getparent failed for clock: %s, ret = %d\n",
@@ -71,9 +70,8 @@ static int zynqmp_clk_mux_set_parent(struct clk_hw *hw, u8 index)
const char *clk_name = clk_hw_get_name(hw);
u32 clk_id = mux->clk_id;
int ret;
- const struct zynqmp_eemi_ops *eemi_ops = zynqmp_pm_get_eemi_ops();
- ret = eemi_ops->clock_setparent(clk_id, index);
+ ret = zynqmp_pm_clock_setparent(clk_id, index);
if (ret)
pr_warn_once("%s() set parent failed for clock: %s, ret = %d\n",
diff --git a/drivers/clk/zynqmp/clkc.c b/drivers/clk/zynqmp/clkc.c
index 10e89f23880b..5eed5ce10179 100644
--- a/drivers/clk/zynqmp/clkc.c
+++ b/drivers/clk/zynqmp/clkc.c
@@ -134,7 +134,6 @@ static struct clk_hw *(* const clk_topology[]) (const char *name, u32 clk_id,
static struct zynqmp_clock *clock;
static struct clk_hw_onecell_data *zynqmp_data;
static unsigned int clock_max_idx;
-static const struct zynqmp_eemi_ops *eemi_ops;
/**
* zynqmp_is_valid_clock() - Check whether clock is valid or not
@@ -206,7 +205,7 @@ static int zynqmp_pm_clock_get_num_clocks(u32 *nclocks)
qdata.qid = PM_QID_CLOCK_GET_NUM_CLOCKS;
- ret = eemi_ops->query_data(qdata, ret_payload);
+ ret = zynqmp_pm_query_data(qdata, ret_payload);
*nclocks = ret_payload[1];
return ret;
@@ -231,7 +230,7 @@ static int zynqmp_pm_clock_get_name(u32 clock_id,
qdata.qid = PM_QID_CLOCK_GET_NAME;
qdata.arg1 = clock_id;
- eemi_ops->query_data(qdata, ret_payload);
+ zynqmp_pm_query_data(qdata, ret_payload);
memcpy(response, ret_payload, sizeof(*response));
return 0;
@@ -265,7 +264,7 @@ static int zynqmp_pm_clock_get_topology(u32 clock_id, u32 index,
qdata.arg1 = clock_id;
qdata.arg2 = index;
- ret = eemi_ops->query_data(qdata, ret_payload);
+ ret = zynqmp_pm_query_data(qdata, ret_payload);
memcpy(response, &ret_payload[1], sizeof(*response));
return ret;
@@ -296,7 +295,7 @@ struct clk_hw *zynqmp_clk_register_fixed_factor(const char *name, u32 clk_id,
qdata.qid = PM_QID_CLOCK_GET_FIXEDFACTOR_PARAMS;
qdata.arg1 = clk_id;
- ret = eemi_ops->query_data(qdata, ret_payload);
+ ret = zynqmp_pm_query_data(qdata, ret_payload);
if (ret)
return ERR_PTR(ret);
@@ -339,7 +338,7 @@ static int zynqmp_pm_clock_get_parents(u32 clock_id, u32 index,
qdata.arg1 = clock_id;
qdata.arg2 = index;
- ret = eemi_ops->query_data(qdata, ret_payload);
+ ret = zynqmp_pm_query_data(qdata, ret_payload);
memcpy(response, &ret_payload[1], sizeof(*response));
return ret;
@@ -364,7 +363,7 @@ static int zynqmp_pm_clock_get_attributes(u32 clock_id,
qdata.qid = PM_QID_CLOCK_GET_ATTRIBUTES;
qdata.arg1 = clock_id;
- ret = eemi_ops->query_data(qdata, ret_payload);
+ ret = zynqmp_pm_query_data(qdata, ret_payload);
memcpy(response, &ret_payload[1], sizeof(*response));
return ret;
@@ -738,10 +737,6 @@ static int zynqmp_clock_probe(struct platform_device *pdev)
int ret;
struct device *dev = &pdev->dev;
- eemi_ops = zynqmp_pm_get_eemi_ops();
- if (IS_ERR(eemi_ops))
- return PTR_ERR(eemi_ops);
-
ret = zynqmp_clk_setup(dev->of_node);
return ret;
diff --git a/drivers/clk/zynqmp/divider.c b/drivers/clk/zynqmp/divider.c
index 4be2cc76aa2e..8eed715707e3 100644
--- a/drivers/clk/zynqmp/divider.c
+++ b/drivers/clk/zynqmp/divider.c
@@ -83,9 +83,8 @@ static unsigned long zynqmp_clk_divider_recalc_rate(struct clk_hw *hw,
u32 div_type = divider->div_type;
u32 div, value;
int ret;
- const struct zynqmp_eemi_ops *eemi_ops = zynqmp_pm_get_eemi_ops();
- ret = eemi_ops->clock_getdivider(clk_id, &div);
+ ret = zynqmp_pm_clock_getdivider(clk_id, &div);
if (ret)
pr_warn_once("%s() get divider failed for %s, ret = %d\n",
@@ -163,11 +162,10 @@ static long zynqmp_clk_divider_round_rate(struct clk_hw *hw,
u32 div_type = divider->div_type;
u32 bestdiv;
int ret;
- const struct zynqmp_eemi_ops *eemi_ops = zynqmp_pm_get_eemi_ops();
/* if read only, just return current value */
if (divider->flags & CLK_DIVIDER_READ_ONLY) {
- ret = eemi_ops->clock_getdivider(clk_id, &bestdiv);
+ ret = zynqmp_pm_clock_getdivider(clk_id, &bestdiv);
if (ret)
pr_warn_once("%s() get divider failed for %s, ret = %d\n",
@@ -219,7 +217,6 @@ static int zynqmp_clk_divider_set_rate(struct clk_hw *hw, unsigned long rate,
u32 div_type = divider->div_type;
u32 value, div;
int ret;
- const struct zynqmp_eemi_ops *eemi_ops = zynqmp_pm_get_eemi_ops();
value = zynqmp_divider_get_val(parent_rate, rate, divider->flags);
if (div_type == TYPE_DIV1) {
@@ -233,7 +230,7 @@ static int zynqmp_clk_divider_set_rate(struct clk_hw *hw, unsigned long rate,
if (divider->flags & CLK_DIVIDER_POWER_OF_TWO)
div = __ffs(div);
- ret = eemi_ops->clock_setdivider(clk_id, div);
+ ret = zynqmp_pm_clock_setdivider(clk_id, div);
if (ret)
pr_warn_once("%s() set divider failed for %s, ret = %d\n",
@@ -258,7 +255,6 @@ static const struct clk_ops zynqmp_clk_divider_ops = {
*/
u32 zynqmp_clk_get_max_divisor(u32 clk_id, u32 type)
{
- const struct zynqmp_eemi_ops *eemi_ops = zynqmp_pm_get_eemi_ops();
struct zynqmp_pm_query_data qdata = {0};
u32 ret_payload[PAYLOAD_ARG_CNT];
int ret;
@@ -266,7 +262,7 @@ u32 zynqmp_clk_get_max_divisor(u32 clk_id, u32 type)
qdata.qid = PM_QID_CLOCK_GET_MAX_DIVISOR;
qdata.arg1 = clk_id;
qdata.arg2 = type;
- ret = eemi_ops->query_data(qdata, ret_payload);
+ ret = zynqmp_pm_query_data(qdata, ret_payload);
/*
* To maintain backward compatibility return maximum possible value
* (0xFFFF) if query for max divisor is not successful.
diff --git a/drivers/clk/zynqmp/pll.c b/drivers/clk/zynqmp/pll.c
index 89b599530105..92f449ed38e5 100644
--- a/drivers/clk/zynqmp/pll.c
+++ b/drivers/clk/zynqmp/pll.c
@@ -50,10 +50,8 @@ static inline enum pll_mode zynqmp_pll_get_mode(struct clk_hw *hw)
const char *clk_name = clk_hw_get_name(hw);
u32 ret_payload[PAYLOAD_ARG_CNT];
int ret;
- const struct zynqmp_eemi_ops *eemi_ops = zynqmp_pm_get_eemi_ops();
- ret = eemi_ops->ioctl(0, IOCTL_GET_PLL_FRAC_MODE, clk_id, 0,
- ret_payload);
+ ret = zynqmp_pm_get_pll_frac_mode(clk_id, ret_payload);
if (ret)
pr_warn_once("%s() PLL get frac mode failed for %s, ret = %d\n",
__func__, clk_name, ret);
@@ -73,14 +71,13 @@ static inline void zynqmp_pll_set_mode(struct clk_hw *hw, bool on)
const char *clk_name = clk_hw_get_name(hw);
int ret;
u32 mode;
- const struct zynqmp_eemi_ops *eemi_ops = zynqmp_pm_get_eemi_ops();
if (on)
mode = PLL_MODE_FRAC;
else
mode = PLL_MODE_INT;
- ret = eemi_ops->ioctl(0, IOCTL_SET_PLL_FRAC_MODE, clk_id, mode, NULL);
+ ret = zynqmp_pm_set_pll_frac_mode(clk_id, mode);
if (ret)
pr_warn_once("%s() PLL set frac mode failed for %s, ret = %d\n",
__func__, clk_name, ret);
@@ -139,17 +136,15 @@ static unsigned long zynqmp_pll_recalc_rate(struct clk_hw *hw,
unsigned long rate, frac;
u32 ret_payload[PAYLOAD_ARG_CNT];
int ret;
- const struct zynqmp_eemi_ops *eemi_ops = zynqmp_pm_get_eemi_ops();
- ret = eemi_ops->clock_getdivider(clk_id, &fbdiv);
+ ret = zynqmp_pm_clock_getdivider(clk_id, &fbdiv);
if (ret)
pr_warn_once("%s() get divider failed for %s, ret = %d\n",
__func__, clk_name, ret);
rate = parent_rate * fbdiv;
if (zynqmp_pll_get_mode(hw) == PLL_MODE_FRAC) {
- eemi_ops->ioctl(0, IOCTL_GET_PLL_FRAC_DATA, clk_id, 0,
- ret_payload);
+ zynqmp_pm_get_pll_frac_data(clk_id, ret_payload);
data = ret_payload[1];
frac = (parent_rate * data) / FRAC_DIV;
rate = rate + frac;
@@ -177,7 +172,6 @@ static int zynqmp_pll_set_rate(struct clk_hw *hw, unsigned long rate,
u32 fbdiv;
long rate_div, frac, m, f;
int ret;
- const struct zynqmp_eemi_ops *eemi_ops = zynqmp_pm_get_eemi_ops();
if (zynqmp_pll_get_mode(hw) == PLL_MODE_FRAC) {
rate_div = (rate * FRAC_DIV) / parent_rate;
@@ -187,21 +181,21 @@ static int zynqmp_pll_set_rate(struct clk_hw *hw, unsigned long rate,
rate = parent_rate * m;
frac = (parent_rate * f) / FRAC_DIV;
- ret = eemi_ops->clock_setdivider(clk_id, m);
+ ret = zynqmp_pm_clock_setdivider(clk_id, m);
if (ret == -EUSERS)
WARN(1, "More than allowed devices are using the %s, which is forbidden\n",
clk_name);
else if (ret)
pr_warn_once("%s() set divider failed for %s, ret = %d\n",
__func__, clk_name, ret);
- eemi_ops->ioctl(0, IOCTL_SET_PLL_FRAC_DATA, clk_id, f, NULL);
+ zynqmp_pm_set_pll_frac_data(clk_id, f);
return rate + frac;
}
fbdiv = DIV_ROUND_CLOSEST(rate, parent_rate);
fbdiv = clamp_t(u32, fbdiv, PLL_FBDIV_MIN, PLL_FBDIV_MAX);
- ret = eemi_ops->clock_setdivider(clk_id, fbdiv);
+ ret = zynqmp_pm_clock_setdivider(clk_id, fbdiv);
if (ret)
pr_warn_once("%s() set divider failed for %s, ret = %d\n",
__func__, clk_name, ret);
@@ -222,9 +216,8 @@ static int zynqmp_pll_is_enabled(struct clk_hw *hw)
u32 clk_id = clk->clk_id;
unsigned int state;
int ret;
- const struct zynqmp_eemi_ops *eemi_ops = zynqmp_pm_get_eemi_ops();
- ret = eemi_ops->clock_getstate(clk_id, &state);
+ ret = zynqmp_pm_clock_getstate(clk_id, &state);
if (ret) {
pr_warn_once("%s() clock get state failed for %s, ret = %d\n",
__func__, clk_name, ret);
@@ -246,12 +239,11 @@ static int zynqmp_pll_enable(struct clk_hw *hw)
const char *clk_name = clk_hw_get_name(hw);
u32 clk_id = clk->clk_id;
int ret;
- const struct zynqmp_eemi_ops *eemi_ops = zynqmp_pm_get_eemi_ops();
if (zynqmp_pll_is_enabled(hw))
return 0;
- ret = eemi_ops->clock_enable(clk_id);
+ ret = zynqmp_pm_clock_enable(clk_id);
if (ret)
pr_warn_once("%s() clock enable failed for %s, ret = %d\n",
__func__, clk_name, ret);
@@ -269,12 +261,11 @@ static void zynqmp_pll_disable(struct clk_hw *hw)
const char *clk_name = clk_hw_get_name(hw);
u32 clk_id = clk->clk_id;
int ret;
- const struct zynqmp_eemi_ops *eemi_ops = zynqmp_pm_get_eemi_ops();
if (!zynqmp_pll_is_enabled(hw))
return;
- ret = eemi_ops->clock_disable(clk_id);
+ ret = zynqmp_pm_clock_disable(clk_id);
if (ret)
pr_warn_once("%s() clock disable failed for %s, ret = %d\n",
__func__, clk_name, ret);
diff --git a/drivers/crypto/xilinx/zynqmp-aes-gcm.c b/drivers/crypto/xilinx/zynqmp-aes-gcm.c
index 09f7f468eef8..cd11558893cd 100644
--- a/drivers/crypto/xilinx/zynqmp-aes-gcm.c
+++ b/drivers/crypto/xilinx/zynqmp-aes-gcm.c
@@ -46,7 +46,6 @@ struct zynqmp_aead_drv_ctx {
} alg;
struct device *dev;
struct crypto_engine *engine;
- const struct zynqmp_eemi_ops *eemi_ops;
};
struct zynqmp_aead_hw_req {
@@ -80,21 +79,15 @@ static int zynqmp_aes_aead_cipher(struct aead_request *req)
struct zynqmp_aead_tfm_ctx *tfm_ctx = crypto_aead_ctx(aead);
struct zynqmp_aead_req_ctx *rq_ctx = aead_request_ctx(req);
struct device *dev = tfm_ctx->dev;
- struct aead_alg *alg = crypto_aead_alg(aead);
- struct zynqmp_aead_drv_ctx *drv_ctx;
struct zynqmp_aead_hw_req *hwreq;
dma_addr_t dma_addr_data, dma_addr_hw_req;
unsigned int data_size;
unsigned int status;
+ int ret;
size_t dma_size;
char *kbuf;
int err;
- drv_ctx = container_of(alg, struct zynqmp_aead_drv_ctx, alg.aead);
-
- if (!drv_ctx->eemi_ops->aes)
- return -ENOTSUPP;
-
if (tfm_ctx->keysrc == ZYNQMP_AES_KUP_KEY)
dma_size = req->cryptlen + ZYNQMP_AES_KEY_SIZE
+ GCM_AES_IV_SIZE;
@@ -136,9 +129,12 @@ static int zynqmp_aes_aead_cipher(struct aead_request *req)
hwreq->key = 0;
}
- drv_ctx->eemi_ops->aes(dma_addr_hw_req, &status);
+ ret = zynqmp_pm_aes_engine(dma_addr_hw_req, &status);
- if (status) {
+ if (ret) {
+ dev_err(dev, "ERROR: AES PM API failed\n");
+ err = ret;
+ } else if (status) {
switch (status) {
case ZYNQMP_AES_GCM_TAG_MISMATCH_ERR:
dev_err(dev, "ERROR: Gcm Tag mismatch\n");
@@ -388,12 +384,6 @@ static int zynqmp_aes_aead_probe(struct platform_device *pdev)
else
return -ENODEV;
- aes_drv_ctx.eemi_ops = zynqmp_pm_get_eemi_ops();
- if (IS_ERR(aes_drv_ctx.eemi_ops)) {
- dev_err(dev, "Failed to get ZynqMP EEMI interface\n");
- return PTR_ERR(aes_drv_ctx.eemi_ops);
- }
-
err = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(ZYNQMP_DMA_BIT_MASK));
if (err < 0) {
dev_err(dev, "No usable DMA configuration\n");
diff --git a/drivers/dca/dca-sysfs.c b/drivers/dca/dca-sysfs.c
index eb25627b059d..21ebd0af268b 100644
--- a/drivers/dca/dca-sysfs.c
+++ b/drivers/dca/dca-sysfs.c
@@ -24,9 +24,7 @@ int dca_sysfs_add_req(struct dca_provider *dca, struct device *dev, int slot)
cd = device_create(dca_class, dca->cd, MKDEV(0, slot + 1), NULL,
"requester%d", req_count++);
- if (IS_ERR(cd))
- return PTR_ERR(cd);
- return 0;
+ return PTR_ERR_OR_ZERO(cd);
}
void dca_sysfs_remove_req(struct dca_provider *dca, int slot)
diff --git a/drivers/extcon/extcon-adc-jack.c b/drivers/extcon/extcon-adc-jack.c
index ad02dc6747a4..0317b614b680 100644
--- a/drivers/extcon/extcon-adc-jack.c
+++ b/drivers/extcon/extcon-adc-jack.c
@@ -124,7 +124,7 @@ static int adc_jack_probe(struct platform_device *pdev)
for (i = 0; data->adc_conditions[i].id != EXTCON_NONE; i++);
data->num_conditions = i;
- data->chan = iio_channel_get(&pdev->dev, pdata->consumer_channel);
+ data->chan = devm_iio_channel_get(&pdev->dev, pdata->consumer_channel);
if (IS_ERR(data->chan))
return PTR_ERR(data->chan);
@@ -164,7 +164,6 @@ static int adc_jack_remove(struct platform_device *pdev)
free_irq(data->irq, data);
cancel_work_sync(&data->handler.work);
- iio_channel_release(data->chan);
return 0;
}
diff --git a/drivers/extcon/extcon-arizona.c b/drivers/extcon/extcon-arizona.c
index 7401733db08b..aae82db542a5 100644
--- a/drivers/extcon/extcon-arizona.c
+++ b/drivers/extcon/extcon-arizona.c
@@ -1460,7 +1460,7 @@ static int arizona_extcon_probe(struct platform_device *pdev)
if (!info->input) {
dev_err(arizona->dev, "Can't allocate input dev\n");
ret = -ENOMEM;
- goto err_register;
+ return ret;
}
info->input->name = "Headset";
@@ -1492,7 +1492,7 @@ static int arizona_extcon_probe(struct platform_device *pdev)
if (ret != 0) {
dev_err(arizona->dev, "Failed to request GPIO%d: %d\n",
pdata->micd_pol_gpio, ret);
- goto err_register;
+ return ret;
}
info->micd_pol_gpio = gpio_to_desc(pdata->micd_pol_gpio);
@@ -1515,7 +1515,7 @@ static int arizona_extcon_probe(struct platform_device *pdev)
dev_err(arizona->dev,
"Failed to get microphone polarity GPIO: %d\n",
ret);
- goto err_register;
+ return ret;
}
}
@@ -1672,7 +1672,7 @@ static int arizona_extcon_probe(struct platform_device *pdev)
if (ret != 0) {
dev_err(&pdev->dev, "Failed to get JACKDET rise IRQ: %d\n",
ret);
- goto err_gpio;
+ goto err_pm;
}
ret = arizona_set_irq_wake(arizona, jack_irq_rise, 1);
@@ -1721,14 +1721,14 @@ static int arizona_extcon_probe(struct platform_device *pdev)
dev_warn(arizona->dev, "Failed to set MICVDD to bypass: %d\n",
ret);
- pm_runtime_put(&pdev->dev);
-
ret = input_register_device(info->input);
if (ret) {
dev_err(&pdev->dev, "Can't register input device: %d\n", ret);
goto err_hpdet;
}
+ pm_runtime_put(&pdev->dev);
+
return 0;
err_hpdet:
@@ -1743,10 +1743,11 @@ err_rise_wake:
arizona_set_irq_wake(arizona, jack_irq_rise, 0);
err_rise:
arizona_free_irq(arizona, jack_irq_rise, info);
+err_pm:
+ pm_runtime_put(&pdev->dev);
+ pm_runtime_disable(&pdev->dev);
err_gpio:
gpiod_put(info->micd_pol_gpio);
-err_register:
- pm_runtime_disable(&pdev->dev);
return ret;
}
diff --git a/drivers/extcon/extcon-max14577.c b/drivers/extcon/extcon-max14577.c
index 32f663436e6e..cc47d626095c 100644
--- a/drivers/extcon/extcon-max14577.c
+++ b/drivers/extcon/extcon-max14577.c
@@ -782,9 +782,19 @@ static const struct platform_device_id max14577_muic_id[] = {
};
MODULE_DEVICE_TABLE(platform, max14577_muic_id);
+static const struct of_device_id of_max14577_muic_dt_match[] = {
+ { .compatible = "maxim,max14577-muic",
+ .data = (void *)MAXIM_DEVICE_TYPE_MAX14577, },
+ { .compatible = "maxim,max77836-muic",
+ .data = (void *)MAXIM_DEVICE_TYPE_MAX77836, },
+ { },
+};
+MODULE_DEVICE_TABLE(of, of_max14577_muic_dt_match);
+
static struct platform_driver max14577_muic_driver = {
.driver = {
.name = "max14577-muic",
+ .of_match_table = of_max14577_muic_dt_match,
},
.probe = max14577_muic_probe,
.remove = max14577_muic_remove,
diff --git a/drivers/extcon/extcon.c b/drivers/extcon/extcon.c
index 2dfbfec572f9..0a6438cbb3f3 100644
--- a/drivers/extcon/extcon.c
+++ b/drivers/extcon/extcon.c
@@ -900,7 +900,7 @@ int extcon_register_notifier(struct extcon_dev *edev, unsigned int id,
struct notifier_block *nb)
{
unsigned long flags;
- int ret, idx = -EINVAL;
+ int ret, idx;
if (!edev || !nb)
return -EINVAL;
diff --git a/drivers/firmware/stratix10-rsu.c b/drivers/firmware/stratix10-rsu.c
index f8533338b018..4379475c99ed 100644
--- a/drivers/firmware/stratix10-rsu.c
+++ b/drivers/firmware/stratix10-rsu.c
@@ -72,7 +72,7 @@ static void rsu_status_callback(struct stratix10_svc_client *client,
struct stratix10_rsu_priv *priv = client->priv;
struct arm_smccc_res *res = (struct arm_smccc_res *)data->kaddr1;
- if (data->status == BIT(SVC_STATUS_RSU_OK)) {
+ if (data->status == BIT(SVC_STATUS_OK)) {
priv->status.version = FIELD_GET(RSU_VERSION_MASK,
res->a2);
priv->status.state = FIELD_GET(RSU_STATE_MASK, res->a2);
@@ -108,9 +108,9 @@ static void rsu_command_callback(struct stratix10_svc_client *client,
{
struct stratix10_rsu_priv *priv = client->priv;
- if (data->status == BIT(SVC_STATUS_RSU_NO_SUPPORT))
+ if (data->status == BIT(SVC_STATUS_NO_SUPPORT))
dev_warn(client->dev, "Secure FW doesn't support notify\n");
- else if (data->status == BIT(SVC_STATUS_RSU_ERROR))
+ else if (data->status == BIT(SVC_STATUS_ERROR))
dev_err(client->dev, "Failure, returned status is %lu\n",
BIT(data->status));
@@ -133,9 +133,9 @@ static void rsu_retry_callback(struct stratix10_svc_client *client,
struct stratix10_rsu_priv *priv = client->priv;
unsigned int *counter = (unsigned int *)data->kaddr1;
- if (data->status == BIT(SVC_STATUS_RSU_OK))
+ if (data->status == BIT(SVC_STATUS_OK))
priv->retry_counter = *counter;
- else if (data->status == BIT(SVC_STATUS_RSU_NO_SUPPORT))
+ else if (data->status == BIT(SVC_STATUS_NO_SUPPORT))
dev_warn(client->dev, "Secure FW doesn't support retry\n");
else
dev_err(client->dev, "Failed to get retry counter %lu\n",
diff --git a/drivers/firmware/stratix10-svc.c b/drivers/firmware/stratix10-svc.c
index d5f0769f3761..e0db8dbfc9d1 100644
--- a/drivers/firmware/stratix10-svc.c
+++ b/drivers/firmware/stratix10-svc.c
@@ -214,7 +214,7 @@ static void svc_thread_cmd_data_claim(struct stratix10_svc_controller *ctrl,
complete(&ctrl->complete_status);
break;
}
- cb_data->status = BIT(SVC_STATUS_RECONFIG_BUFFER_DONE);
+ cb_data->status = BIT(SVC_STATUS_BUFFER_DONE);
cb_data->kaddr1 = svc_pa_to_va(res.a1);
cb_data->kaddr2 = (res.a2) ?
svc_pa_to_va(res.a2) : NULL;
@@ -227,7 +227,7 @@ static void svc_thread_cmd_data_claim(struct stratix10_svc_controller *ctrl,
__func__);
}
} while (res.a0 == INTEL_SIP_SMC_STATUS_OK ||
- res.a0 == INTEL_SIP_SMC_FPGA_CONFIG_STATUS_BUSY ||
+ res.a0 == INTEL_SIP_SMC_STATUS_BUSY ||
wait_for_completion_timeout(&ctrl->complete_status, timeout));
}
@@ -250,7 +250,7 @@ static void svc_thread_cmd_config_status(struct stratix10_svc_controller *ctrl,
cb_data->kaddr1 = NULL;
cb_data->kaddr2 = NULL;
cb_data->kaddr3 = NULL;
- cb_data->status = BIT(SVC_STATUS_RECONFIG_ERROR);
+ cb_data->status = BIT(SVC_STATUS_ERROR);
pr_debug("%s: polling config status\n", __func__);
@@ -259,7 +259,7 @@ static void svc_thread_cmd_config_status(struct stratix10_svc_controller *ctrl,
ctrl->invoke_fn(INTEL_SIP_SMC_FPGA_CONFIG_ISDONE,
0, 0, 0, 0, 0, 0, 0, &res);
if ((res.a0 == INTEL_SIP_SMC_STATUS_OK) ||
- (res.a0 == INTEL_SIP_SMC_FPGA_CONFIG_STATUS_ERROR))
+ (res.a0 == INTEL_SIP_SMC_STATUS_ERROR))
break;
/*
@@ -271,7 +271,7 @@ static void svc_thread_cmd_config_status(struct stratix10_svc_controller *ctrl,
}
if (res.a0 == INTEL_SIP_SMC_STATUS_OK && count_in_sec)
- cb_data->status = BIT(SVC_STATUS_RECONFIG_COMPLETED);
+ cb_data->status = BIT(SVC_STATUS_COMPLETED);
p_data->chan->scl->receive_cb(p_data->chan->scl, cb_data);
}
@@ -294,24 +294,18 @@ static void svc_thread_recv_status_ok(struct stratix10_svc_data *p_data,
switch (p_data->command) {
case COMMAND_RECONFIG:
- cb_data->status = BIT(SVC_STATUS_RECONFIG_REQUEST_OK);
+ case COMMAND_RSU_UPDATE:
+ case COMMAND_RSU_NOTIFY:
+ cb_data->status = BIT(SVC_STATUS_OK);
break;
case COMMAND_RECONFIG_DATA_SUBMIT:
- cb_data->status = BIT(SVC_STATUS_RECONFIG_BUFFER_SUBMITTED);
- break;
- case COMMAND_NOOP:
- cb_data->status = BIT(SVC_STATUS_RECONFIG_BUFFER_SUBMITTED);
- cb_data->kaddr1 = svc_pa_to_va(res.a1);
+ cb_data->status = BIT(SVC_STATUS_BUFFER_SUBMITTED);
break;
case COMMAND_RECONFIG_STATUS:
- cb_data->status = BIT(SVC_STATUS_RECONFIG_COMPLETED);
- break;
- case COMMAND_RSU_UPDATE:
- case COMMAND_RSU_NOTIFY:
- cb_data->status = BIT(SVC_STATUS_RSU_OK);
+ cb_data->status = BIT(SVC_STATUS_COMPLETED);
break;
case COMMAND_RSU_RETRY:
- cb_data->status = BIT(SVC_STATUS_RSU_OK);
+ cb_data->status = BIT(SVC_STATUS_OK);
cb_data->kaddr1 = &res.a1;
break;
default:
@@ -430,9 +424,9 @@ static int svc_normal_to_secure_thread(void *data)
if (pdata->command == COMMAND_RSU_STATUS) {
if (res.a0 == INTEL_SIP_SMC_RSU_ERROR)
- cbdata->status = BIT(SVC_STATUS_RSU_ERROR);
+ cbdata->status = BIT(SVC_STATUS_ERROR);
else
- cbdata->status = BIT(SVC_STATUS_RSU_OK);
+ cbdata->status = BIT(SVC_STATUS_OK);
cbdata->kaddr1 = &res;
cbdata->kaddr2 = NULL;
@@ -445,7 +439,7 @@ static int svc_normal_to_secure_thread(void *data)
case INTEL_SIP_SMC_STATUS_OK:
svc_thread_recv_status_ok(pdata, cbdata, res);
break;
- case INTEL_SIP_SMC_FPGA_CONFIG_STATUS_BUSY:
+ case INTEL_SIP_SMC_STATUS_BUSY:
switch (pdata->command) {
case COMMAND_RECONFIG_DATA_SUBMIT:
svc_thread_cmd_data_claim(ctrl,
@@ -460,33 +454,13 @@ static int svc_normal_to_secure_thread(void *data)
break;
}
break;
- case INTEL_SIP_SMC_FPGA_CONFIG_STATUS_REJECTED:
+ case INTEL_SIP_SMC_STATUS_REJECTED:
pr_debug("%s: STATUS_REJECTED\n", __func__);
break;
- case INTEL_SIP_SMC_FPGA_CONFIG_STATUS_ERROR:
+ case INTEL_SIP_SMC_STATUS_ERROR:
case INTEL_SIP_SMC_RSU_ERROR:
pr_err("%s: STATUS_ERROR\n", __func__);
- switch (pdata->command) {
- /* for FPGA mgr */
- case COMMAND_RECONFIG_DATA_CLAIM:
- case COMMAND_RECONFIG:
- case COMMAND_RECONFIG_DATA_SUBMIT:
- case COMMAND_RECONFIG_STATUS:
- cbdata->status =
- BIT(SVC_STATUS_RECONFIG_ERROR);
- break;
-
- /* for RSU */
- case COMMAND_RSU_STATUS:
- case COMMAND_RSU_UPDATE:
- case COMMAND_RSU_NOTIFY:
- case COMMAND_RSU_RETRY:
- cbdata->status =
- BIT(SVC_STATUS_RSU_ERROR);
- break;
- }
-
- cbdata->status = BIT(SVC_STATUS_RECONFIG_ERROR);
+ cbdata->status = BIT(SVC_STATUS_ERROR);
cbdata->kaddr1 = NULL;
cbdata->kaddr2 = NULL;
cbdata->kaddr3 = NULL;
@@ -502,7 +476,7 @@ static int svc_normal_to_secure_thread(void *data)
if ((pdata->command == COMMAND_RSU_RETRY) ||
(pdata->command == COMMAND_RSU_NOTIFY)) {
cbdata->status =
- BIT(SVC_STATUS_RSU_NO_SUPPORT);
+ BIT(SVC_STATUS_NO_SUPPORT);
cbdata->kaddr1 = NULL;
cbdata->kaddr2 = NULL;
cbdata->kaddr3 = NULL;
diff --git a/drivers/firmware/xilinx/zynqmp-debug.c b/drivers/firmware/xilinx/zynqmp-debug.c
index 43bc6cfdab45..99606b34975e 100644
--- a/drivers/firmware/xilinx/zynqmp-debug.c
+++ b/drivers/firmware/xilinx/zynqmp-debug.c
@@ -85,14 +85,13 @@ static int get_pm_api_id(char *pm_api_req, u32 *pm_id)
static int process_api_request(u32 pm_id, u64 *pm_api_arg, u32 *pm_api_ret)
{
- const struct zynqmp_eemi_ops *eemi_ops = zynqmp_pm_get_eemi_ops();
u32 pm_api_version;
int ret;
struct zynqmp_pm_query_data qdata = {0};
switch (pm_id) {
case PM_GET_API_VERSION:
- ret = eemi_ops->get_api_version(&pm_api_version);
+ ret = zynqmp_pm_get_api_version(&pm_api_version);
sprintf(debugfs_buf, "PM-API Version = %d.%d\n",
pm_api_version >> 16, pm_api_version & 0xffff);
break;
@@ -102,7 +101,7 @@ static int process_api_request(u32 pm_id, u64 *pm_api_arg, u32 *pm_api_ret)
qdata.arg2 = pm_api_arg[2];
qdata.arg3 = pm_api_arg[3];
- ret = eemi_ops->query_data(qdata, pm_api_ret);
+ ret = zynqmp_pm_query_data(qdata, pm_api_ret);
if (ret)
break;
diff --git a/drivers/firmware/xilinx/zynqmp.c b/drivers/firmware/xilinx/zynqmp.c
index 41b65164a367..8d1ff2454e2e 100644
--- a/drivers/firmware/xilinx/zynqmp.c
+++ b/drivers/firmware/xilinx/zynqmp.c
@@ -2,7 +2,7 @@
/*
* Xilinx Zynq MPSoC Firmware layer
*
- * Copyright (C) 2014-2018 Xilinx, Inc.
+ * Copyright (C) 2014-2020 Xilinx, Inc.
*
* Michal Simek <michal.simek@xilinx.com>
* Davorin Mista <davorin.mista@aggios.com>
@@ -24,8 +24,6 @@
#include <linux/firmware/xlnx-zynqmp.h>
#include "zynqmp-debug.h"
-static const struct zynqmp_eemi_ops *eemi_ops_tbl;
-
static bool feature_check_enabled;
static u32 zynqmp_pm_features[PM_API_MAX];
@@ -219,7 +217,7 @@ static u32 pm_tz_version;
*
* Return: Returns status, either success or error+reason
*/
-static int zynqmp_pm_get_api_version(u32 *version)
+int zynqmp_pm_get_api_version(u32 *version)
{
u32 ret_payload[PAYLOAD_ARG_CNT];
int ret;
@@ -237,6 +235,7 @@ static int zynqmp_pm_get_api_version(u32 *version)
return ret;
}
+EXPORT_SYMBOL_GPL(zynqmp_pm_get_api_version);
/**
* zynqmp_pm_get_chipid - Get silicon ID registers
@@ -246,7 +245,7 @@ static int zynqmp_pm_get_api_version(u32 *version)
* Return: Returns the status of the operation and the idcode and version
* registers in @idcode and @version.
*/
-static int zynqmp_pm_get_chipid(u32 *idcode, u32 *version)
+int zynqmp_pm_get_chipid(u32 *idcode, u32 *version)
{
u32 ret_payload[PAYLOAD_ARG_CNT];
int ret;
@@ -260,6 +259,7 @@ static int zynqmp_pm_get_chipid(u32 *idcode, u32 *version)
return ret;
}
+EXPORT_SYMBOL_GPL(zynqmp_pm_get_chipid);
/**
* zynqmp_pm_get_trustzone_version() - Get secure trustzone firmware version
@@ -324,7 +324,7 @@ static int get_set_conduit_method(struct device_node *np)
*
* Return: Returns status, either success or error+reason
*/
-static int zynqmp_pm_query_data(struct zynqmp_pm_query_data qdata, u32 *out)
+int zynqmp_pm_query_data(struct zynqmp_pm_query_data qdata, u32 *out)
{
int ret;
@@ -338,6 +338,7 @@ static int zynqmp_pm_query_data(struct zynqmp_pm_query_data qdata, u32 *out)
*/
return qdata.qid == PM_QID_CLOCK_GET_NAME ? 0 : ret;
}
+EXPORT_SYMBOL_GPL(zynqmp_pm_query_data);
/**
* zynqmp_pm_clock_enable() - Enable the clock for given id
@@ -348,10 +349,11 @@ static int zynqmp_pm_query_data(struct zynqmp_pm_query_data qdata, u32 *out)
*
* Return: Returns status, either success or error+reason
*/
-static int zynqmp_pm_clock_enable(u32 clock_id)
+int zynqmp_pm_clock_enable(u32 clock_id)
{
return zynqmp_pm_invoke_fn(PM_CLOCK_ENABLE, clock_id, 0, 0, 0, NULL);
}
+EXPORT_SYMBOL_GPL(zynqmp_pm_clock_enable);
/**
* zynqmp_pm_clock_disable() - Disable the clock for given id
@@ -362,10 +364,11 @@ static int zynqmp_pm_clock_enable(u32 clock_id)
*
* Return: Returns status, either success or error+reason
*/
-static int zynqmp_pm_clock_disable(u32 clock_id)
+int zynqmp_pm_clock_disable(u32 clock_id)
{
return zynqmp_pm_invoke_fn(PM_CLOCK_DISABLE, clock_id, 0, 0, 0, NULL);
}
+EXPORT_SYMBOL_GPL(zynqmp_pm_clock_disable);
/**
* zynqmp_pm_clock_getstate() - Get the clock state for given id
@@ -377,7 +380,7 @@ static int zynqmp_pm_clock_disable(u32 clock_id)
*
* Return: Returns status, either success or error+reason
*/
-static int zynqmp_pm_clock_getstate(u32 clock_id, u32 *state)
+int zynqmp_pm_clock_getstate(u32 clock_id, u32 *state)
{
u32 ret_payload[PAYLOAD_ARG_CNT];
int ret;
@@ -388,6 +391,7 @@ static int zynqmp_pm_clock_getstate(u32 clock_id, u32 *state)
return ret;
}
+EXPORT_SYMBOL_GPL(zynqmp_pm_clock_getstate);
/**
* zynqmp_pm_clock_setdivider() - Set the clock divider for given id
@@ -399,11 +403,12 @@ static int zynqmp_pm_clock_getstate(u32 clock_id, u32 *state)
*
* Return: Returns status, either success or error+reason
*/
-static int zynqmp_pm_clock_setdivider(u32 clock_id, u32 divider)
+int zynqmp_pm_clock_setdivider(u32 clock_id, u32 divider)
{
return zynqmp_pm_invoke_fn(PM_CLOCK_SETDIVIDER, clock_id, divider,
0, 0, NULL);
}
+EXPORT_SYMBOL_GPL(zynqmp_pm_clock_setdivider);
/**
* zynqmp_pm_clock_getdivider() - Get the clock divider for given id
@@ -415,7 +420,7 @@ static int zynqmp_pm_clock_setdivider(u32 clock_id, u32 divider)
*
* Return: Returns status, either success or error+reason
*/
-static int zynqmp_pm_clock_getdivider(u32 clock_id, u32 *divider)
+int zynqmp_pm_clock_getdivider(u32 clock_id, u32 *divider)
{
u32 ret_payload[PAYLOAD_ARG_CNT];
int ret;
@@ -426,6 +431,7 @@ static int zynqmp_pm_clock_getdivider(u32 clock_id, u32 *divider)
return ret;
}
+EXPORT_SYMBOL_GPL(zynqmp_pm_clock_getdivider);
/**
* zynqmp_pm_clock_setrate() - Set the clock rate for given id
@@ -436,13 +442,14 @@ static int zynqmp_pm_clock_getdivider(u32 clock_id, u32 *divider)
*
* Return: Returns status, either success or error+reason
*/
-static int zynqmp_pm_clock_setrate(u32 clock_id, u64 rate)
+int zynqmp_pm_clock_setrate(u32 clock_id, u64 rate)
{
return zynqmp_pm_invoke_fn(PM_CLOCK_SETRATE, clock_id,
lower_32_bits(rate),
upper_32_bits(rate),
0, NULL);
}
+EXPORT_SYMBOL_GPL(zynqmp_pm_clock_setrate);
/**
* zynqmp_pm_clock_getrate() - Get the clock rate for given id
@@ -454,7 +461,7 @@ static int zynqmp_pm_clock_setrate(u32 clock_id, u64 rate)
*
* Return: Returns status, either success or error+reason
*/
-static int zynqmp_pm_clock_getrate(u32 clock_id, u64 *rate)
+int zynqmp_pm_clock_getrate(u32 clock_id, u64 *rate)
{
u32 ret_payload[PAYLOAD_ARG_CNT];
int ret;
@@ -465,6 +472,7 @@ static int zynqmp_pm_clock_getrate(u32 clock_id, u64 *rate)
return ret;
}
+EXPORT_SYMBOL_GPL(zynqmp_pm_clock_getrate);
/**
* zynqmp_pm_clock_setparent() - Set the clock parent for given id
@@ -475,11 +483,12 @@ static int zynqmp_pm_clock_getrate(u32 clock_id, u64 *rate)
*
* Return: Returns status, either success or error+reason
*/
-static int zynqmp_pm_clock_setparent(u32 clock_id, u32 parent_id)
+int zynqmp_pm_clock_setparent(u32 clock_id, u32 parent_id)
{
return zynqmp_pm_invoke_fn(PM_CLOCK_SETPARENT, clock_id,
parent_id, 0, 0, NULL);
}
+EXPORT_SYMBOL_GPL(zynqmp_pm_clock_setparent);
/**
* zynqmp_pm_clock_getparent() - Get the clock parent for given id
@@ -491,7 +500,7 @@ static int zynqmp_pm_clock_setparent(u32 clock_id, u32 parent_id)
*
* Return: Returns status, either success or error+reason
*/
-static int zynqmp_pm_clock_getparent(u32 clock_id, u32 *parent_id)
+int zynqmp_pm_clock_getparent(u32 clock_id, u32 *parent_id)
{
u32 ret_payload[PAYLOAD_ARG_CNT];
int ret;
@@ -502,48 +511,191 @@ static int zynqmp_pm_clock_getparent(u32 clock_id, u32 *parent_id)
return ret;
}
+EXPORT_SYMBOL_GPL(zynqmp_pm_clock_getparent);
/**
- * zynqmp_is_valid_ioctl() - Check whether IOCTL ID is valid or not
- * @ioctl_id: IOCTL ID
+ * zynqmp_pm_set_pll_frac_mode() - PM API for set PLL mode
+ *
+ * @clk_id: PLL clock ID
+ * @mode: PLL mode (PLL_MODE_FRAC/PLL_MODE_INT)
+ *
+ * This function sets PLL mode
*
- * Return: 1 if IOCTL is valid else 0
+ * Return: Returns status, either success or error+reason
*/
-static inline int zynqmp_is_valid_ioctl(u32 ioctl_id)
+int zynqmp_pm_set_pll_frac_mode(u32 clk_id, u32 mode)
{
- switch (ioctl_id) {
- case IOCTL_SD_DLL_RESET:
- case IOCTL_SET_SD_TAPDELAY:
- case IOCTL_SET_PLL_FRAC_MODE:
- case IOCTL_GET_PLL_FRAC_MODE:
- case IOCTL_SET_PLL_FRAC_DATA:
- case IOCTL_GET_PLL_FRAC_DATA:
- return 1;
- default:
- return 0;
- }
+ return zynqmp_pm_invoke_fn(PM_IOCTL, 0, IOCTL_SET_PLL_FRAC_MODE,
+ clk_id, mode, NULL);
}
+EXPORT_SYMBOL_GPL(zynqmp_pm_set_pll_frac_mode);
/**
- * zynqmp_pm_ioctl() - PM IOCTL API for device control and configs
- * @node_id: Node ID of the device
- * @ioctl_id: ID of the requested IOCTL
- * @arg1: Argument 1 to requested IOCTL call
- * @arg2: Argument 2 to requested IOCTL call
- * @out: Returned output value
+ * zynqmp_pm_get_pll_frac_mode() - PM API for get PLL mode
+ *
+ * @clk_id: PLL clock ID
+ * @mode: PLL mode
*
- * This function calls IOCTL to firmware for device control and configuration.
+ * This function return current PLL mode
*
* Return: Returns status, either success or error+reason
*/
-static int zynqmp_pm_ioctl(u32 node_id, u32 ioctl_id, u32 arg1, u32 arg2,
- u32 *out)
+int zynqmp_pm_get_pll_frac_mode(u32 clk_id, u32 *mode)
{
- if (!zynqmp_is_valid_ioctl(ioctl_id))
- return -EINVAL;
+ return zynqmp_pm_invoke_fn(PM_IOCTL, 0, IOCTL_GET_PLL_FRAC_MODE,
+ clk_id, 0, mode);
+}
+EXPORT_SYMBOL_GPL(zynqmp_pm_get_pll_frac_mode);
+
+/**
+ * zynqmp_pm_set_pll_frac_data() - PM API for setting pll fraction data
+ *
+ * @clk_id: PLL clock ID
+ * @data: fraction data
+ *
+ * This function sets fraction data.
+ * It is valid for fraction mode only.
+ *
+ * Return: Returns status, either success or error+reason
+ */
+int zynqmp_pm_set_pll_frac_data(u32 clk_id, u32 data)
+{
+ return zynqmp_pm_invoke_fn(PM_IOCTL, 0, IOCTL_SET_PLL_FRAC_DATA,
+ clk_id, data, NULL);
+}
+EXPORT_SYMBOL_GPL(zynqmp_pm_set_pll_frac_data);
+
+/**
+ * zynqmp_pm_get_pll_frac_data() - PM API for getting pll fraction data
+ *
+ * @clk_id: PLL clock ID
+ * @data: fraction data
+ *
+ * This function returns fraction data value.
+ *
+ * Return: Returns status, either success or error+reason
+ */
+int zynqmp_pm_get_pll_frac_data(u32 clk_id, u32 *data)
+{
+ return zynqmp_pm_invoke_fn(PM_IOCTL, 0, IOCTL_GET_PLL_FRAC_DATA,
+ clk_id, 0, data);
+}
+EXPORT_SYMBOL_GPL(zynqmp_pm_get_pll_frac_data);
+
+/**
+ * zynqmp_pm_set_sd_tapdelay() - Set tap delay for the SD device
+ *
+ * @node_id Node ID of the device
+ * @type Type of tap delay to set (input/output)
+ * @value Value to set fot the tap delay
+ *
+ * This function sets input/output tap delay for the SD device.
+ *
+ * @return Returns status, either success or error+reason
+ */
+int zynqmp_pm_set_sd_tapdelay(u32 node_id, u32 type, u32 value)
+{
+ return zynqmp_pm_invoke_fn(PM_IOCTL, node_id, IOCTL_SET_SD_TAPDELAY,
+ type, value, NULL);
+}
+EXPORT_SYMBOL_GPL(zynqmp_pm_set_sd_tapdelay);
+
+/**
+ * zynqmp_pm_sd_dll_reset() - Reset DLL logic
+ *
+ * @node_id Node ID of the device
+ * @type Reset type
+ *
+ * This function resets DLL logic for the SD device.
+ *
+ * @return Returns status, either success or error+reason
+ */
+int zynqmp_pm_sd_dll_reset(u32 node_id, u32 type)
+{
+ return zynqmp_pm_invoke_fn(PM_IOCTL, node_id, IOCTL_SET_SD_TAPDELAY,
+ type, 0, NULL);
+}
+EXPORT_SYMBOL_GPL(zynqmp_pm_sd_dll_reset);
+
+/**
+ * zynqmp_pm_write_ggs() - PM API for writing global general storage (ggs)
+ * @index GGS register index
+ * @value Register value to be written
+ *
+ * This function writes value to GGS register.
+ *
+ * @return Returns status, either success or error+reason
+ */
+int zynqmp_pm_write_ggs(u32 index, u32 value)
+{
+ return zynqmp_pm_invoke_fn(PM_IOCTL, 0, IOCTL_WRITE_GGS,
+ index, value, NULL);
+}
+EXPORT_SYMBOL_GPL(zynqmp_pm_write_ggs);
+
+/**
+ * zynqmp_pm_write_ggs() - PM API for reading global general storage (ggs)
+ * @index GGS register index
+ * @value Register value to be written
+ *
+ * This function returns GGS register value.
+ *
+ * @return Returns status, either success or error+reason
+ */
+int zynqmp_pm_read_ggs(u32 index, u32 *value)
+{
+ return zynqmp_pm_invoke_fn(PM_IOCTL, 0, IOCTL_READ_GGS,
+ index, 0, value);
+}
+EXPORT_SYMBOL_GPL(zynqmp_pm_read_ggs);
+
+/**
+ * zynqmp_pm_write_pggs() - PM API for writing persistent global general
+ * storage (pggs)
+ * @index PGGS register index
+ * @value Register value to be written
+ *
+ * This function writes value to PGGS register.
+ *
+ * @return Returns status, either success or error+reason
+ */
+int zynqmp_pm_write_pggs(u32 index, u32 value)
+{
+ return zynqmp_pm_invoke_fn(PM_IOCTL, 0, IOCTL_WRITE_PGGS, index, value,
+ NULL);
+}
+EXPORT_SYMBOL_GPL(zynqmp_pm_write_pggs);
+
+/**
+ * zynqmp_pm_write_pggs() - PM API for reading persistent global general
+ * storage (pggs)
+ * @index PGGS register index
+ * @value Register value to be written
+ *
+ * This function returns PGGS register value.
+ *
+ * @return Returns status, either success or error+reason
+ */
+int zynqmp_pm_read_pggs(u32 index, u32 *value)
+{
+ return zynqmp_pm_invoke_fn(PM_IOCTL, 0, IOCTL_READ_PGGS, index, 0,
+ value);
+}
+EXPORT_SYMBOL_GPL(zynqmp_pm_read_pggs);
- return zynqmp_pm_invoke_fn(PM_IOCTL, node_id, ioctl_id,
- arg1, arg2, out);
+/**
+ * zynqmp_pm_set_boot_health_status() - PM API for setting healthy boot status
+ * @value Status value to be written
+ *
+ * This function sets healthy bit value to indicate boot health status
+ * to firmware.
+ *
+ * @return Returns status, either success or error+reason
+ */
+int zynqmp_pm_set_boot_health_status(u32 value)
+{
+ return zynqmp_pm_invoke_fn(PM_IOCTL, 0, IOCTL_SET_BOOT_HEALTH_STATUS,
+ value, 0, NULL);
}
/**
@@ -554,12 +706,13 @@ static int zynqmp_pm_ioctl(u32 node_id, u32 ioctl_id, u32 arg1, u32 arg2,
*
* Return: Returns status, either success or error+reason
*/
-static int zynqmp_pm_reset_assert(const enum zynqmp_pm_reset reset,
- const enum zynqmp_pm_reset_action assert_flag)
+int zynqmp_pm_reset_assert(const enum zynqmp_pm_reset reset,
+ const enum zynqmp_pm_reset_action assert_flag)
{
return zynqmp_pm_invoke_fn(PM_RESET_ASSERT, reset, assert_flag,
0, 0, NULL);
}
+EXPORT_SYMBOL_GPL(zynqmp_pm_reset_assert);
/**
* zynqmp_pm_reset_get_status - Get status of the reset
@@ -568,8 +721,7 @@ static int zynqmp_pm_reset_assert(const enum zynqmp_pm_reset reset,
*
* Return: Returns status, either success or error+reason
*/
-static int zynqmp_pm_reset_get_status(const enum zynqmp_pm_reset reset,
- u32 *status)
+int zynqmp_pm_reset_get_status(const enum zynqmp_pm_reset reset, u32 *status)
{
u32 ret_payload[PAYLOAD_ARG_CNT];
int ret;
@@ -583,6 +735,7 @@ static int zynqmp_pm_reset_get_status(const enum zynqmp_pm_reset reset,
return ret;
}
+EXPORT_SYMBOL_GPL(zynqmp_pm_reset_get_status);
/**
* zynqmp_pm_fpga_load - Perform the fpga load
@@ -597,12 +750,12 @@ static int zynqmp_pm_reset_get_status(const enum zynqmp_pm_reset reset,
*
* Return: Returns status, either success or error+reason
*/
-static int zynqmp_pm_fpga_load(const u64 address, const u32 size,
- const u32 flags)
+int zynqmp_pm_fpga_load(const u64 address, const u32 size, const u32 flags)
{
return zynqmp_pm_invoke_fn(PM_FPGA_LOAD, lower_32_bits(address),
upper_32_bits(address), size, flags, NULL);
}
+EXPORT_SYMBOL_GPL(zynqmp_pm_fpga_load);
/**
* zynqmp_pm_fpga_get_status - Read value from PCAP status register
@@ -613,7 +766,7 @@ static int zynqmp_pm_fpga_load(const u64 address, const u32 size,
*
* Return: Returns status, either success or error+reason
*/
-static int zynqmp_pm_fpga_get_status(u32 *value)
+int zynqmp_pm_fpga_get_status(u32 *value)
{
u32 ret_payload[PAYLOAD_ARG_CNT];
int ret;
@@ -626,6 +779,7 @@ static int zynqmp_pm_fpga_get_status(u32 *value)
return ret;
}
+EXPORT_SYMBOL_GPL(zynqmp_pm_fpga_get_status);
/**
* zynqmp_pm_init_finalize() - PM call to inform firmware that the caller
@@ -636,10 +790,11 @@ static int zynqmp_pm_fpga_get_status(u32 *value)
*
* Return: Returns status, either success or error+reason
*/
-static int zynqmp_pm_init_finalize(void)
+int zynqmp_pm_init_finalize(void)
{
return zynqmp_pm_invoke_fn(PM_PM_INIT_FINALIZE, 0, 0, 0, 0, NULL);
}
+EXPORT_SYMBOL_GPL(zynqmp_pm_init_finalize);
/**
* zynqmp_pm_set_suspend_mode() - Set system suspend mode
@@ -649,10 +804,11 @@ static int zynqmp_pm_init_finalize(void)
*
* Return: Returns status, either success or error+reason
*/
-static int zynqmp_pm_set_suspend_mode(u32 mode)
+int zynqmp_pm_set_suspend_mode(u32 mode)
{
return zynqmp_pm_invoke_fn(PM_SET_SUSPEND_MODE, mode, 0, 0, 0, NULL);
}
+EXPORT_SYMBOL_GPL(zynqmp_pm_set_suspend_mode);
/**
* zynqmp_pm_request_node() - Request a node with specific capabilities
@@ -666,13 +822,13 @@ static int zynqmp_pm_set_suspend_mode(u32 mode)
*
* Return: Returns status, either success or error+reason
*/
-static int zynqmp_pm_request_node(const u32 node, const u32 capabilities,
- const u32 qos,
- const enum zynqmp_pm_request_ack ack)
+int zynqmp_pm_request_node(const u32 node, const u32 capabilities,
+ const u32 qos, const enum zynqmp_pm_request_ack ack)
{
return zynqmp_pm_invoke_fn(PM_REQUEST_NODE, node, capabilities,
qos, ack, NULL);
}
+EXPORT_SYMBOL_GPL(zynqmp_pm_request_node);
/**
* zynqmp_pm_release_node() - Release a node
@@ -684,10 +840,11 @@ static int zynqmp_pm_request_node(const u32 node, const u32 capabilities,
*
* Return: Returns status, either success or error+reason
*/
-static int zynqmp_pm_release_node(const u32 node)
+int zynqmp_pm_release_node(const u32 node)
{
return zynqmp_pm_invoke_fn(PM_RELEASE_NODE, node, 0, 0, 0, NULL);
}
+EXPORT_SYMBOL_GPL(zynqmp_pm_release_node);
/**
* zynqmp_pm_set_requirement() - PM call to set requirement for PM slaves
@@ -701,13 +858,14 @@ static int zynqmp_pm_release_node(const u32 node)
*
* Return: Returns status, either success or error+reason
*/
-static int zynqmp_pm_set_requirement(const u32 node, const u32 capabilities,
- const u32 qos,
- const enum zynqmp_pm_request_ack ack)
+int zynqmp_pm_set_requirement(const u32 node, const u32 capabilities,
+ const u32 qos,
+ const enum zynqmp_pm_request_ack ack)
{
return zynqmp_pm_invoke_fn(PM_SET_REQUIREMENT, node, capabilities,
qos, ack, NULL);
}
+EXPORT_SYMBOL_GPL(zynqmp_pm_set_requirement);
/**
* zynqmp_pm_aes - Access AES hardware to encrypt/decrypt the data using
@@ -717,7 +875,7 @@ static int zynqmp_pm_set_requirement(const u32 node, const u32 capabilities,
*
* Return: Returns status, either success or error code.
*/
-static int zynqmp_pm_aes_engine(const u64 address, u32 *out)
+int zynqmp_pm_aes_engine(const u64 address, u32 *out)
{
u32 ret_payload[PAYLOAD_ARG_CNT];
int ret;
@@ -732,47 +890,304 @@ static int zynqmp_pm_aes_engine(const u64 address, u32 *out)
return ret;
}
+EXPORT_SYMBOL_GPL(zynqmp_pm_aes_engine);
-static const struct zynqmp_eemi_ops eemi_ops = {
- .get_api_version = zynqmp_pm_get_api_version,
- .get_chipid = zynqmp_pm_get_chipid,
- .query_data = zynqmp_pm_query_data,
- .clock_enable = zynqmp_pm_clock_enable,
- .clock_disable = zynqmp_pm_clock_disable,
- .clock_getstate = zynqmp_pm_clock_getstate,
- .clock_setdivider = zynqmp_pm_clock_setdivider,
- .clock_getdivider = zynqmp_pm_clock_getdivider,
- .clock_setrate = zynqmp_pm_clock_setrate,
- .clock_getrate = zynqmp_pm_clock_getrate,
- .clock_setparent = zynqmp_pm_clock_setparent,
- .clock_getparent = zynqmp_pm_clock_getparent,
- .ioctl = zynqmp_pm_ioctl,
- .reset_assert = zynqmp_pm_reset_assert,
- .reset_get_status = zynqmp_pm_reset_get_status,
- .init_finalize = zynqmp_pm_init_finalize,
- .set_suspend_mode = zynqmp_pm_set_suspend_mode,
- .request_node = zynqmp_pm_request_node,
- .release_node = zynqmp_pm_release_node,
- .set_requirement = zynqmp_pm_set_requirement,
- .fpga_load = zynqmp_pm_fpga_load,
- .fpga_get_status = zynqmp_pm_fpga_get_status,
- .aes = zynqmp_pm_aes_engine,
+/**
+ * zynqmp_pm_system_shutdown - PM call to request a system shutdown or restart
+ * @type: Shutdown or restart? 0 for shutdown, 1 for restart
+ * @subtype: Specifies which system should be restarted or shut down
+ *
+ * Return: Returns status, either success or error+reason
+ */
+int zynqmp_pm_system_shutdown(const u32 type, const u32 subtype)
+{
+ return zynqmp_pm_invoke_fn(PM_SYSTEM_SHUTDOWN, type, subtype,
+ 0, 0, NULL);
+}
+
+/**
+ * struct zynqmp_pm_shutdown_scope - Struct for shutdown scope
+ * @subtype: Shutdown subtype
+ * @name: Matching string for scope argument
+ *
+ * This struct encapsulates mapping between shutdown scope ID and string.
+ */
+struct zynqmp_pm_shutdown_scope {
+ const enum zynqmp_pm_shutdown_subtype subtype;
+ const char *name;
+};
+
+static struct zynqmp_pm_shutdown_scope shutdown_scopes[] = {
+ [ZYNQMP_PM_SHUTDOWN_SUBTYPE_SUBSYSTEM] = {
+ .subtype = ZYNQMP_PM_SHUTDOWN_SUBTYPE_SUBSYSTEM,
+ .name = "subsystem",
+ },
+ [ZYNQMP_PM_SHUTDOWN_SUBTYPE_PS_ONLY] = {
+ .subtype = ZYNQMP_PM_SHUTDOWN_SUBTYPE_PS_ONLY,
+ .name = "ps_only",
+ },
+ [ZYNQMP_PM_SHUTDOWN_SUBTYPE_SYSTEM] = {
+ .subtype = ZYNQMP_PM_SHUTDOWN_SUBTYPE_SYSTEM,
+ .name = "system",
+ },
};
+static struct zynqmp_pm_shutdown_scope *selected_scope =
+ &shutdown_scopes[ZYNQMP_PM_SHUTDOWN_SUBTYPE_SYSTEM];
+
/**
- * zynqmp_pm_get_eemi_ops - Get eemi ops functions
+ * zynqmp_pm_is_shutdown_scope_valid - Check if shutdown scope string is valid
+ * @scope_string: Shutdown scope string
*
- * Return: Pointer of eemi_ops structure
+ * Return: Return pointer to matching shutdown scope struct from
+ * array of available options in system if string is valid,
+ * otherwise returns NULL.
*/
-const struct zynqmp_eemi_ops *zynqmp_pm_get_eemi_ops(void)
+static struct zynqmp_pm_shutdown_scope*
+ zynqmp_pm_is_shutdown_scope_valid(const char *scope_string)
+{
+ int count;
+
+ for (count = 0; count < ARRAY_SIZE(shutdown_scopes); count++)
+ if (sysfs_streq(scope_string, shutdown_scopes[count].name))
+ return &shutdown_scopes[count];
+
+ return NULL;
+}
+
+static ssize_t shutdown_scope_show(struct device *device,
+ struct device_attribute *attr,
+ char *buf)
+{
+ int i;
+
+ for (i = 0; i < ARRAY_SIZE(shutdown_scopes); i++) {
+ if (&shutdown_scopes[i] == selected_scope) {
+ strcat(buf, "[");
+ strcat(buf, shutdown_scopes[i].name);
+ strcat(buf, "]");
+ } else {
+ strcat(buf, shutdown_scopes[i].name);
+ }
+ strcat(buf, " ");
+ }
+ strcat(buf, "\n");
+
+ return strlen(buf);
+}
+
+static ssize_t shutdown_scope_store(struct device *device,
+ struct device_attribute *attr,
+ const char *buf, size_t count)
+{
+ int ret;
+ struct zynqmp_pm_shutdown_scope *scope;
+
+ scope = zynqmp_pm_is_shutdown_scope_valid(buf);
+ if (!scope)
+ return -EINVAL;
+
+ ret = zynqmp_pm_system_shutdown(ZYNQMP_PM_SHUTDOWN_TYPE_SETSCOPE_ONLY,
+ scope->subtype);
+ if (ret) {
+ pr_err("unable to set shutdown scope %s\n", buf);
+ return ret;
+ }
+
+ selected_scope = scope;
+
+ return count;
+}
+
+static DEVICE_ATTR_RW(shutdown_scope);
+
+static ssize_t health_status_store(struct device *device,
+ struct device_attribute *attr,
+ const char *buf, size_t count)
+{
+ int ret;
+ unsigned int value;
+
+ ret = kstrtouint(buf, 10, &value);
+ if (ret)
+ return ret;
+
+ ret = zynqmp_pm_set_boot_health_status(value);
+ if (ret) {
+ dev_err(device, "unable to set healthy bit value to %u\n",
+ value);
+ return ret;
+ }
+
+ return count;
+}
+
+static DEVICE_ATTR_WO(health_status);
+
+static ssize_t ggs_show(struct device *device,
+ struct device_attribute *attr,
+ char *buf,
+ u32 reg)
+{
+ int ret;
+ u32 ret_payload[PAYLOAD_ARG_CNT];
+
+ ret = zynqmp_pm_read_ggs(reg, ret_payload);
+ if (ret)
+ return ret;
+
+ return sprintf(buf, "0x%x\n", ret_payload[1]);
+}
+
+static ssize_t ggs_store(struct device *device,
+ struct device_attribute *attr,
+ const char *buf, size_t count,
+ u32 reg)
+{
+ long value;
+ int ret;
+
+ if (reg >= GSS_NUM_REGS)
+ return -EINVAL;
+
+ ret = kstrtol(buf, 16, &value);
+ if (ret) {
+ count = -EFAULT;
+ goto err;
+ }
+
+ ret = zynqmp_pm_write_ggs(reg, value);
+ if (ret)
+ count = -EFAULT;
+err:
+ return count;
+}
+
+/* GGS register show functions */
+#define GGS0_SHOW(N) \
+ ssize_t ggs##N##_show(struct device *device, \
+ struct device_attribute *attr, \
+ char *buf) \
+ { \
+ return ggs_show(device, attr, buf, N); \
+ }
+
+static GGS0_SHOW(0);
+static GGS0_SHOW(1);
+static GGS0_SHOW(2);
+static GGS0_SHOW(3);
+
+/* GGS register store function */
+#define GGS0_STORE(N) \
+ ssize_t ggs##N##_store(struct device *device, \
+ struct device_attribute *attr, \
+ const char *buf, \
+ size_t count) \
+ { \
+ return ggs_store(device, attr, buf, count, N); \
+ }
+
+static GGS0_STORE(0);
+static GGS0_STORE(1);
+static GGS0_STORE(2);
+static GGS0_STORE(3);
+
+static ssize_t pggs_show(struct device *device,
+ struct device_attribute *attr,
+ char *buf,
+ u32 reg)
+{
+ int ret;
+ u32 ret_payload[PAYLOAD_ARG_CNT];
+
+ ret = zynqmp_pm_read_pggs(reg, ret_payload);
+ if (ret)
+ return ret;
+
+ return sprintf(buf, "0x%x\n", ret_payload[1]);
+}
+
+static ssize_t pggs_store(struct device *device,
+ struct device_attribute *attr,
+ const char *buf, size_t count,
+ u32 reg)
{
- if (eemi_ops_tbl)
- return eemi_ops_tbl;
- else
- return ERR_PTR(-EPROBE_DEFER);
+ long value;
+ int ret;
+
+ if (reg >= GSS_NUM_REGS)
+ return -EINVAL;
+ ret = kstrtol(buf, 16, &value);
+ if (ret) {
+ count = -EFAULT;
+ goto err;
+ }
+
+ ret = zynqmp_pm_write_pggs(reg, value);
+ if (ret)
+ count = -EFAULT;
+
+err:
+ return count;
}
-EXPORT_SYMBOL_GPL(zynqmp_pm_get_eemi_ops);
+
+#define PGGS0_SHOW(N) \
+ ssize_t pggs##N##_show(struct device *device, \
+ struct device_attribute *attr, \
+ char *buf) \
+ { \
+ return pggs_show(device, attr, buf, N); \
+ }
+
+#define PGGS0_STORE(N) \
+ ssize_t pggs##N##_store(struct device *device, \
+ struct device_attribute *attr, \
+ const char *buf, \
+ size_t count) \
+ { \
+ return pggs_store(device, attr, buf, count, N); \
+ }
+
+/* PGGS register show functions */
+static PGGS0_SHOW(0);
+static PGGS0_SHOW(1);
+static PGGS0_SHOW(2);
+static PGGS0_SHOW(3);
+
+/* PGGS register store functions */
+static PGGS0_STORE(0);
+static PGGS0_STORE(1);
+static PGGS0_STORE(2);
+static PGGS0_STORE(3);
+
+/* GGS register attributes */
+static DEVICE_ATTR_RW(ggs0);
+static DEVICE_ATTR_RW(ggs1);
+static DEVICE_ATTR_RW(ggs2);
+static DEVICE_ATTR_RW(ggs3);
+
+/* PGGS register attributes */
+static DEVICE_ATTR_RW(pggs0);
+static DEVICE_ATTR_RW(pggs1);
+static DEVICE_ATTR_RW(pggs2);
+static DEVICE_ATTR_RW(pggs3);
+
+static struct attribute *zynqmp_firmware_attrs[] = {
+ &dev_attr_ggs0.attr,
+ &dev_attr_ggs1.attr,
+ &dev_attr_ggs2.attr,
+ &dev_attr_ggs3.attr,
+ &dev_attr_pggs0.attr,
+ &dev_attr_pggs1.attr,
+ &dev_attr_pggs2.attr,
+ &dev_attr_pggs3.attr,
+ &dev_attr_shutdown_scope.attr,
+ &dev_attr_health_status.attr,
+ NULL,
+};
+
+ATTRIBUTE_GROUPS(zynqmp_firmware);
static int zynqmp_firmware_probe(struct platform_device *pdev)
{
@@ -820,11 +1235,6 @@ static int zynqmp_firmware_probe(struct platform_device *pdev)
pr_info("%s Trustzone version v%d.%d\n", __func__,
pm_tz_version >> 16, pm_tz_version & 0xFFFF);
- /* Assign eemi_ops_table */
- eemi_ops_tbl = &eemi_ops;
-
- zynqmp_pm_api_debugfs_init();
-
ret = mfd_add_devices(&pdev->dev, PLATFORM_DEVID_NONE, firmware_devs,
ARRAY_SIZE(firmware_devs), NULL, 0, NULL);
if (ret) {
@@ -832,6 +1242,8 @@ static int zynqmp_firmware_probe(struct platform_device *pdev)
return ret;
}
+ zynqmp_pm_api_debugfs_init();
+
return of_platform_populate(dev->of_node, NULL, NULL, dev);
}
@@ -854,6 +1266,7 @@ static struct platform_driver zynqmp_firmware_driver = {
.driver = {
.name = "zynqmp_firmware",
.of_match_table = zynqmp_firmware_of_match,
+ .dev_groups = zynqmp_firmware_groups,
},
.probe = zynqmp_firmware_probe,
.remove = zynqmp_firmware_remove,
diff --git a/drivers/fpga/Kconfig b/drivers/fpga/Kconfig
index 72380e1d31c7..b2408a710662 100644
--- a/drivers/fpga/Kconfig
+++ b/drivers/fpga/Kconfig
@@ -156,7 +156,7 @@ config FPGA_DFL
config FPGA_DFL_FME
tristate "FPGA DFL FME Driver"
- depends on FPGA_DFL && HWMON
+ depends on FPGA_DFL && HWMON && PERF_EVENTS
help
The FPGA Management Engine (FME) is a feature device implemented
under Device Feature List (DFL) framework. Select this option to
diff --git a/drivers/fpga/Makefile b/drivers/fpga/Makefile
index 4865b74b00a4..d8e21dfc6778 100644
--- a/drivers/fpga/Makefile
+++ b/drivers/fpga/Makefile
@@ -40,6 +40,7 @@ obj-$(CONFIG_FPGA_DFL_FME_REGION) += dfl-fme-region.o
obj-$(CONFIG_FPGA_DFL_AFU) += dfl-afu.o
dfl-fme-objs := dfl-fme-main.o dfl-fme-pr.o dfl-fme-error.o
+dfl-fme-objs += dfl-fme-perf.o
dfl-afu-objs := dfl-afu-main.o dfl-afu-region.o dfl-afu-dma-region.o
dfl-afu-objs += dfl-afu-error.o
diff --git a/drivers/fpga/dfl-afu-dma-region.c b/drivers/fpga/dfl-afu-dma-region.c
index d902acb36d14..02d8cbad1ae2 100644
--- a/drivers/fpga/dfl-afu-dma-region.c
+++ b/drivers/fpga/dfl-afu-dma-region.c
@@ -61,10 +61,10 @@ static int afu_dma_pin_pages(struct dfl_feature_platform_data *pdata,
region->pages);
if (pinned < 0) {
ret = pinned;
- goto put_pages;
+ goto free_pages;
} else if (pinned != npages) {
ret = -EFAULT;
- goto free_pages;
+ goto put_pages;
}
dev_dbg(dev, "%d pages pinned\n", pinned);
diff --git a/drivers/fpga/dfl-afu-main.c b/drivers/fpga/dfl-afu-main.c
index 65437b6a6842..b0c31789a909 100644
--- a/drivers/fpga/dfl-afu-main.c
+++ b/drivers/fpga/dfl-afu-main.c
@@ -561,14 +561,16 @@ static int afu_open(struct inode *inode, struct file *filp)
if (WARN_ON(!pdata))
return -ENODEV;
- ret = dfl_feature_dev_use_begin(pdata);
- if (ret)
- return ret;
-
- dev_dbg(&fdev->dev, "Device File Open\n");
- filp->private_data = fdev;
+ mutex_lock(&pdata->lock);
+ ret = dfl_feature_dev_use_begin(pdata, filp->f_flags & O_EXCL);
+ if (!ret) {
+ dev_dbg(&fdev->dev, "Device File Opened %d Times\n",
+ dfl_feature_dev_use_count(pdata));
+ filp->private_data = fdev;
+ }
+ mutex_unlock(&pdata->lock);
- return 0;
+ return ret;
}
static int afu_release(struct inode *inode, struct file *filp)
@@ -581,12 +583,14 @@ static int afu_release(struct inode *inode, struct file *filp)
pdata = dev_get_platdata(&pdev->dev);
mutex_lock(&pdata->lock);
- __port_reset(pdev);
- afu_dma_region_destroy(pdata);
- mutex_unlock(&pdata->lock);
-
dfl_feature_dev_use_end(pdata);
+ if (!dfl_feature_dev_use_count(pdata)) {
+ __port_reset(pdev);
+ afu_dma_region_destroy(pdata);
+ }
+ mutex_unlock(&pdata->lock);
+
return 0;
}
@@ -746,6 +750,12 @@ static long afu_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)
return -EINVAL;
}
+static const struct vm_operations_struct afu_vma_ops = {
+#ifdef CONFIG_HAVE_IOREMAP_PROT
+ .access = generic_access_phys,
+#endif
+};
+
static int afu_mmap(struct file *filp, struct vm_area_struct *vma)
{
struct platform_device *pdev = filp->private_data;
@@ -775,6 +785,9 @@ static int afu_mmap(struct file *filp, struct vm_area_struct *vma)
!(region.flags & DFL_PORT_REGION_WRITE))
return -EPERM;
+ /* Support debug access to the mapping */
+ vma->vm_ops = &afu_vma_ops;
+
vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
return remap_pfn_range(vma, vma->vm_start,
diff --git a/drivers/fpga/dfl-fme-main.c b/drivers/fpga/dfl-fme-main.c
index 1d4690c99268..fc210d4e1863 100644
--- a/drivers/fpga/dfl-fme-main.c
+++ b/drivers/fpga/dfl-fme-main.c
@@ -580,6 +580,10 @@ static struct dfl_feature_driver fme_feature_drvs[] = {
.ops = &fme_power_mgmt_ops,
},
{
+ .id_table = fme_perf_id_table,
+ .ops = &fme_perf_ops,
+ },
+ {
.ops = NULL,
},
};
@@ -600,14 +604,16 @@ static int fme_open(struct inode *inode, struct file *filp)
if (WARN_ON(!pdata))
return -ENODEV;
- ret = dfl_feature_dev_use_begin(pdata);
- if (ret)
- return ret;
-
- dev_dbg(&fdev->dev, "Device File Open\n");
- filp->private_data = pdata;
+ mutex_lock(&pdata->lock);
+ ret = dfl_feature_dev_use_begin(pdata, filp->f_flags & O_EXCL);
+ if (!ret) {
+ dev_dbg(&fdev->dev, "Device File Opened %d Times\n",
+ dfl_feature_dev_use_count(pdata));
+ filp->private_data = pdata;
+ }
+ mutex_unlock(&pdata->lock);
- return 0;
+ return ret;
}
static int fme_release(struct inode *inode, struct file *filp)
@@ -616,7 +622,10 @@ static int fme_release(struct inode *inode, struct file *filp)
struct platform_device *pdev = pdata->dev;
dev_dbg(&pdev->dev, "Device File Release\n");
+
+ mutex_lock(&pdata->lock);
dfl_feature_dev_use_end(pdata);
+ mutex_unlock(&pdata->lock);
return 0;
}
diff --git a/drivers/fpga/dfl-fme-perf.c b/drivers/fpga/dfl-fme-perf.c
new file mode 100644
index 000000000000..6ce1ed222ea4
--- /dev/null
+++ b/drivers/fpga/dfl-fme-perf.c
@@ -0,0 +1,1020 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Driver for FPGA Management Engine (FME) Global Performance Reporting
+ *
+ * Copyright 2019 Intel Corporation, Inc.
+ *
+ * Authors:
+ * Kang Luwei <luwei.kang@intel.com>
+ * Xiao Guangrong <guangrong.xiao@linux.intel.com>
+ * Wu Hao <hao.wu@intel.com>
+ * Xu Yilun <yilun.xu@intel.com>
+ * Joseph Grecco <joe.grecco@intel.com>
+ * Enno Luebbers <enno.luebbers@intel.com>
+ * Tim Whisonant <tim.whisonant@intel.com>
+ * Ananda Ravuri <ananda.ravuri@intel.com>
+ * Mitchel, Henry <henry.mitchel@intel.com>
+ */
+
+#include <linux/perf_event.h>
+#include "dfl.h"
+#include "dfl-fme.h"
+
+/*
+ * Performance Counter Registers for Cache.
+ *
+ * Cache Events are listed below as CACHE_EVNT_*.
+ */
+#define CACHE_CTRL 0x8
+#define CACHE_RESET_CNTR BIT_ULL(0)
+#define CACHE_FREEZE_CNTR BIT_ULL(8)
+#define CACHE_CTRL_EVNT GENMASK_ULL(19, 16)
+#define CACHE_EVNT_RD_HIT 0x0
+#define CACHE_EVNT_WR_HIT 0x1
+#define CACHE_EVNT_RD_MISS 0x2
+#define CACHE_EVNT_WR_MISS 0x3
+#define CACHE_EVNT_RSVD 0x4
+#define CACHE_EVNT_HOLD_REQ 0x5
+#define CACHE_EVNT_DATA_WR_PORT_CONTEN 0x6
+#define CACHE_EVNT_TAG_WR_PORT_CONTEN 0x7
+#define CACHE_EVNT_TX_REQ_STALL 0x8
+#define CACHE_EVNT_RX_REQ_STALL 0x9
+#define CACHE_EVNT_EVICTIONS 0xa
+#define CACHE_EVNT_MAX CACHE_EVNT_EVICTIONS
+#define CACHE_CHANNEL_SEL BIT_ULL(20)
+#define CACHE_CHANNEL_RD 0
+#define CACHE_CHANNEL_WR 1
+#define CACHE_CNTR0 0x10
+#define CACHE_CNTR1 0x18
+#define CACHE_CNTR_EVNT_CNTR GENMASK_ULL(47, 0)
+#define CACHE_CNTR_EVNT GENMASK_ULL(63, 60)
+
+/*
+ * Performance Counter Registers for Fabric.
+ *
+ * Fabric Events are listed below as FAB_EVNT_*
+ */
+#define FAB_CTRL 0x20
+#define FAB_RESET_CNTR BIT_ULL(0)
+#define FAB_FREEZE_CNTR BIT_ULL(8)
+#define FAB_CTRL_EVNT GENMASK_ULL(19, 16)
+#define FAB_EVNT_PCIE0_RD 0x0
+#define FAB_EVNT_PCIE0_WR 0x1
+#define FAB_EVNT_PCIE1_RD 0x2
+#define FAB_EVNT_PCIE1_WR 0x3
+#define FAB_EVNT_UPI_RD 0x4
+#define FAB_EVNT_UPI_WR 0x5
+#define FAB_EVNT_MMIO_RD 0x6
+#define FAB_EVNT_MMIO_WR 0x7
+#define FAB_EVNT_MAX FAB_EVNT_MMIO_WR
+#define FAB_PORT_ID GENMASK_ULL(21, 20)
+#define FAB_PORT_FILTER BIT_ULL(23)
+#define FAB_PORT_FILTER_DISABLE 0
+#define FAB_PORT_FILTER_ENABLE 1
+#define FAB_CNTR 0x28
+#define FAB_CNTR_EVNT_CNTR GENMASK_ULL(59, 0)
+#define FAB_CNTR_EVNT GENMASK_ULL(63, 60)
+
+/*
+ * Performance Counter Registers for Clock.
+ *
+ * Clock Counter can't be reset or frozen by SW.
+ */
+#define CLK_CNTR 0x30
+#define BASIC_EVNT_CLK 0x0
+#define BASIC_EVNT_MAX BASIC_EVNT_CLK
+
+/*
+ * Performance Counter Registers for IOMMU / VT-D.
+ *
+ * VT-D Events are listed below as VTD_EVNT_* and VTD_SIP_EVNT_*
+ */
+#define VTD_CTRL 0x38
+#define VTD_RESET_CNTR BIT_ULL(0)
+#define VTD_FREEZE_CNTR BIT_ULL(8)
+#define VTD_CTRL_EVNT GENMASK_ULL(19, 16)
+#define VTD_EVNT_AFU_MEM_RD_TRANS 0x0
+#define VTD_EVNT_AFU_MEM_WR_TRANS 0x1
+#define VTD_EVNT_AFU_DEVTLB_RD_HIT 0x2
+#define VTD_EVNT_AFU_DEVTLB_WR_HIT 0x3
+#define VTD_EVNT_DEVTLB_4K_FILL 0x4
+#define VTD_EVNT_DEVTLB_2M_FILL 0x5
+#define VTD_EVNT_DEVTLB_1G_FILL 0x6
+#define VTD_EVNT_MAX VTD_EVNT_DEVTLB_1G_FILL
+#define VTD_CNTR 0x40
+#define VTD_CNTR_EVNT_CNTR GENMASK_ULL(47, 0)
+#define VTD_CNTR_EVNT GENMASK_ULL(63, 60)
+
+#define VTD_SIP_CTRL 0x48
+#define VTD_SIP_RESET_CNTR BIT_ULL(0)
+#define VTD_SIP_FREEZE_CNTR BIT_ULL(8)
+#define VTD_SIP_CTRL_EVNT GENMASK_ULL(19, 16)
+#define VTD_SIP_EVNT_IOTLB_4K_HIT 0x0
+#define VTD_SIP_EVNT_IOTLB_2M_HIT 0x1
+#define VTD_SIP_EVNT_IOTLB_1G_HIT 0x2
+#define VTD_SIP_EVNT_SLPWC_L3_HIT 0x3
+#define VTD_SIP_EVNT_SLPWC_L4_HIT 0x4
+#define VTD_SIP_EVNT_RCC_HIT 0x5
+#define VTD_SIP_EVNT_IOTLB_4K_MISS 0x6
+#define VTD_SIP_EVNT_IOTLB_2M_MISS 0x7
+#define VTD_SIP_EVNT_IOTLB_1G_MISS 0x8
+#define VTD_SIP_EVNT_SLPWC_L3_MISS 0x9
+#define VTD_SIP_EVNT_SLPWC_L4_MISS 0xa
+#define VTD_SIP_EVNT_RCC_MISS 0xb
+#define VTD_SIP_EVNT_MAX VTD_SIP_EVNT_SLPWC_L4_MISS
+#define VTD_SIP_CNTR 0X50
+#define VTD_SIP_CNTR_EVNT_CNTR GENMASK_ULL(47, 0)
+#define VTD_SIP_CNTR_EVNT GENMASK_ULL(63, 60)
+
+#define PERF_TIMEOUT 30
+
+#define PERF_MAX_PORT_NUM 1U
+
+/**
+ * struct fme_perf_priv - priv data structure for fme perf driver
+ *
+ * @dev: parent device.
+ * @ioaddr: mapped base address of mmio region.
+ * @pmu: pmu data structure for fme perf counters.
+ * @id: id of this fme performance report private feature.
+ * @fab_users: current user number on fabric counters.
+ * @fab_port_id: used to indicate current working mode of fabric counters.
+ * @fab_lock: lock to protect fabric counters working mode.
+ * @cpu: active CPU to which the PMU is bound for accesses.
+ * @cpuhp_node: node for CPU hotplug notifier link.
+ * @cpuhp_state: state for CPU hotplug notification;
+ */
+struct fme_perf_priv {
+ struct device *dev;
+ void __iomem *ioaddr;
+ struct pmu pmu;
+ u64 id;
+
+ u32 fab_users;
+ u32 fab_port_id;
+ spinlock_t fab_lock;
+
+ unsigned int cpu;
+ struct hlist_node node;
+ enum cpuhp_state cpuhp_state;
+};
+
+/**
+ * struct fme_perf_event_ops - callbacks for fme perf events
+ *
+ * @event_init: callback invoked during event init.
+ * @event_destroy: callback invoked during event destroy.
+ * @read_counter: callback to read hardware counters.
+ */
+struct fme_perf_event_ops {
+ int (*event_init)(struct fme_perf_priv *priv, u32 event, u32 portid);
+ void (*event_destroy)(struct fme_perf_priv *priv, u32 event,
+ u32 portid);
+ u64 (*read_counter)(struct fme_perf_priv *priv, u32 event, u32 portid);
+};
+
+#define to_fme_perf_priv(_pmu) container_of(_pmu, struct fme_perf_priv, pmu)
+
+static ssize_t cpumask_show(struct device *dev,
+ struct device_attribute *attr, char *buf)
+{
+ struct pmu *pmu = dev_get_drvdata(dev);
+ struct fme_perf_priv *priv;
+
+ priv = to_fme_perf_priv(pmu);
+
+ return cpumap_print_to_pagebuf(true, buf, cpumask_of(priv->cpu));
+}
+static DEVICE_ATTR_RO(cpumask);
+
+static struct attribute *fme_perf_cpumask_attrs[] = {
+ &dev_attr_cpumask.attr,
+ NULL,
+};
+
+static struct attribute_group fme_perf_cpumask_group = {
+ .attrs = fme_perf_cpumask_attrs,
+};
+
+#define FME_EVENT_MASK GENMASK_ULL(11, 0)
+#define FME_EVENT_SHIFT 0
+#define FME_EVTYPE_MASK GENMASK_ULL(15, 12)
+#define FME_EVTYPE_SHIFT 12
+#define FME_EVTYPE_BASIC 0
+#define FME_EVTYPE_CACHE 1
+#define FME_EVTYPE_FABRIC 2
+#define FME_EVTYPE_VTD 3
+#define FME_EVTYPE_VTD_SIP 4
+#define FME_EVTYPE_MAX FME_EVTYPE_VTD_SIP
+#define FME_PORTID_MASK GENMASK_ULL(23, 16)
+#define FME_PORTID_SHIFT 16
+#define FME_PORTID_ROOT (0xffU)
+
+#define get_event(_config) FIELD_GET(FME_EVENT_MASK, _config)
+#define get_evtype(_config) FIELD_GET(FME_EVTYPE_MASK, _config)
+#define get_portid(_config) FIELD_GET(FME_PORTID_MASK, _config)
+
+PMU_FORMAT_ATTR(event, "config:0-11");
+PMU_FORMAT_ATTR(evtype, "config:12-15");
+PMU_FORMAT_ATTR(portid, "config:16-23");
+
+static struct attribute *fme_perf_format_attrs[] = {
+ &format_attr_event.attr,
+ &format_attr_evtype.attr,
+ &format_attr_portid.attr,
+ NULL,
+};
+
+static struct attribute_group fme_perf_format_group = {
+ .name = "format",
+ .attrs = fme_perf_format_attrs,
+};
+
+/*
+ * There are no default events, but we need to create
+ * "events" group (with empty attrs) before updating
+ * it with detected events (using pmu->attr_update).
+ */
+static struct attribute *fme_perf_events_attrs_empty[] = {
+ NULL,
+};
+
+static struct attribute_group fme_perf_events_group = {
+ .name = "events",
+ .attrs = fme_perf_events_attrs_empty,
+};
+
+static const struct attribute_group *fme_perf_groups[] = {
+ &fme_perf_format_group,
+ &fme_perf_cpumask_group,
+ &fme_perf_events_group,
+ NULL,
+};
+
+static bool is_portid_root(u32 portid)
+{
+ return portid == FME_PORTID_ROOT;
+}
+
+static bool is_portid_port(u32 portid)
+{
+ return portid < PERF_MAX_PORT_NUM;
+}
+
+static bool is_portid_root_or_port(u32 portid)
+{
+ return is_portid_root(portid) || is_portid_port(portid);
+}
+
+static u64 fme_read_perf_cntr_reg(void __iomem *addr)
+{
+ u32 low;
+ u64 v;
+
+ /*
+ * For 64bit counter registers, the counter may increases and carries
+ * out of bit [31] between 2 32bit reads. So add extra reads to help
+ * to prevent this issue. This only happens in platforms which don't
+ * support 64bit read - readq is split into 2 readl.
+ */
+ do {
+ v = readq(addr);
+ low = readl(addr);
+ } while (((u32)v) > low);
+
+ return v;
+}
+
+static int basic_event_init(struct fme_perf_priv *priv, u32 event, u32 portid)
+{
+ if (event <= BASIC_EVNT_MAX && is_portid_root(portid))
+ return 0;
+
+ return -EINVAL;
+}
+
+static u64 basic_read_event_counter(struct fme_perf_priv *priv,
+ u32 event, u32 portid)
+{
+ void __iomem *base = priv->ioaddr;
+
+ return fme_read_perf_cntr_reg(base + CLK_CNTR);
+}
+
+static int cache_event_init(struct fme_perf_priv *priv, u32 event, u32 portid)
+{
+ if (priv->id == FME_FEATURE_ID_GLOBAL_IPERF &&
+ event <= CACHE_EVNT_MAX && is_portid_root(portid))
+ return 0;
+
+ return -EINVAL;
+}
+
+static u64 cache_read_event_counter(struct fme_perf_priv *priv,
+ u32 event, u32 portid)
+{
+ void __iomem *base = priv->ioaddr;
+ u64 v, count;
+ u8 channel;
+
+ if (event == CACHE_EVNT_WR_HIT || event == CACHE_EVNT_WR_MISS ||
+ event == CACHE_EVNT_DATA_WR_PORT_CONTEN ||
+ event == CACHE_EVNT_TAG_WR_PORT_CONTEN)
+ channel = CACHE_CHANNEL_WR;
+ else
+ channel = CACHE_CHANNEL_RD;
+
+ /* set channel access type and cache event code. */
+ v = readq(base + CACHE_CTRL);
+ v &= ~(CACHE_CHANNEL_SEL | CACHE_CTRL_EVNT);
+ v |= FIELD_PREP(CACHE_CHANNEL_SEL, channel);
+ v |= FIELD_PREP(CACHE_CTRL_EVNT, event);
+ writeq(v, base + CACHE_CTRL);
+
+ if (readq_poll_timeout_atomic(base + CACHE_CNTR0, v,
+ FIELD_GET(CACHE_CNTR_EVNT, v) == event,
+ 1, PERF_TIMEOUT)) {
+ dev_err(priv->dev, "timeout, unmatched cache event code in counter register.\n");
+ return 0;
+ }
+
+ v = fme_read_perf_cntr_reg(base + CACHE_CNTR0);
+ count = FIELD_GET(CACHE_CNTR_EVNT_CNTR, v);
+ v = fme_read_perf_cntr_reg(base + CACHE_CNTR1);
+ count += FIELD_GET(CACHE_CNTR_EVNT_CNTR, v);
+
+ return count;
+}
+
+static bool is_fabric_event_supported(struct fme_perf_priv *priv, u32 event,
+ u32 portid)
+{
+ if (event > FAB_EVNT_MAX || !is_portid_root_or_port(portid))
+ return false;
+
+ if (priv->id == FME_FEATURE_ID_GLOBAL_DPERF &&
+ (event == FAB_EVNT_PCIE1_RD || event == FAB_EVNT_UPI_RD ||
+ event == FAB_EVNT_PCIE1_WR || event == FAB_EVNT_UPI_WR))
+ return false;
+
+ return true;
+}
+
+static int fabric_event_init(struct fme_perf_priv *priv, u32 event, u32 portid)
+{
+ void __iomem *base = priv->ioaddr;
+ int ret = 0;
+ u64 v;
+
+ if (!is_fabric_event_supported(priv, event, portid))
+ return -EINVAL;
+
+ /*
+ * as fabric counter set only can be in either overall or port mode.
+ * In overall mode, it counts overall data for FPGA, and in port mode,
+ * it is configured to monitor on one individual port.
+ *
+ * so every time, a new event is initialized, driver checks
+ * current working mode and if someone is using this counter set.
+ */
+ spin_lock(&priv->fab_lock);
+ if (priv->fab_users && priv->fab_port_id != portid) {
+ dev_dbg(priv->dev, "conflict fabric event monitoring mode.\n");
+ ret = -EOPNOTSUPP;
+ goto exit;
+ }
+
+ priv->fab_users++;
+
+ /*
+ * skip if current working mode matches, otherwise change the working
+ * mode per input port_id, to monitor overall data or another port.
+ */
+ if (priv->fab_port_id == portid)
+ goto exit;
+
+ priv->fab_port_id = portid;
+
+ v = readq(base + FAB_CTRL);
+ v &= ~(FAB_PORT_FILTER | FAB_PORT_ID);
+
+ if (is_portid_root(portid)) {
+ v |= FIELD_PREP(FAB_PORT_FILTER, FAB_PORT_FILTER_DISABLE);
+ } else {
+ v |= FIELD_PREP(FAB_PORT_FILTER, FAB_PORT_FILTER_ENABLE);
+ v |= FIELD_PREP(FAB_PORT_ID, portid);
+ }
+ writeq(v, base + FAB_CTRL);
+
+exit:
+ spin_unlock(&priv->fab_lock);
+ return ret;
+}
+
+static void fabric_event_destroy(struct fme_perf_priv *priv, u32 event,
+ u32 portid)
+{
+ spin_lock(&priv->fab_lock);
+ priv->fab_users--;
+ spin_unlock(&priv->fab_lock);
+}
+
+static u64 fabric_read_event_counter(struct fme_perf_priv *priv, u32 event,
+ u32 portid)
+{
+ void __iomem *base = priv->ioaddr;
+ u64 v;
+
+ v = readq(base + FAB_CTRL);
+ v &= ~FAB_CTRL_EVNT;
+ v |= FIELD_PREP(FAB_CTRL_EVNT, event);
+ writeq(v, base + FAB_CTRL);
+
+ if (readq_poll_timeout_atomic(base + FAB_CNTR, v,
+ FIELD_GET(FAB_CNTR_EVNT, v) == event,
+ 1, PERF_TIMEOUT)) {
+ dev_err(priv->dev, "timeout, unmatched fab event code in counter register.\n");
+ return 0;
+ }
+
+ v = fme_read_perf_cntr_reg(base + FAB_CNTR);
+ return FIELD_GET(FAB_CNTR_EVNT_CNTR, v);
+}
+
+static int vtd_event_init(struct fme_perf_priv *priv, u32 event, u32 portid)
+{
+ if (priv->id == FME_FEATURE_ID_GLOBAL_IPERF &&
+ event <= VTD_EVNT_MAX && is_portid_port(portid))
+ return 0;
+
+ return -EINVAL;
+}
+
+static u64 vtd_read_event_counter(struct fme_perf_priv *priv, u32 event,
+ u32 portid)
+{
+ void __iomem *base = priv->ioaddr;
+ u64 v;
+
+ event += (portid * (VTD_EVNT_MAX + 1));
+
+ v = readq(base + VTD_CTRL);
+ v &= ~VTD_CTRL_EVNT;
+ v |= FIELD_PREP(VTD_CTRL_EVNT, event);
+ writeq(v, base + VTD_CTRL);
+
+ if (readq_poll_timeout_atomic(base + VTD_CNTR, v,
+ FIELD_GET(VTD_CNTR_EVNT, v) == event,
+ 1, PERF_TIMEOUT)) {
+ dev_err(priv->dev, "timeout, unmatched vtd event code in counter register.\n");
+ return 0;
+ }
+
+ v = fme_read_perf_cntr_reg(base + VTD_CNTR);
+ return FIELD_GET(VTD_CNTR_EVNT_CNTR, v);
+}
+
+static int vtd_sip_event_init(struct fme_perf_priv *priv, u32 event, u32 portid)
+{
+ if (priv->id == FME_FEATURE_ID_GLOBAL_IPERF &&
+ event <= VTD_SIP_EVNT_MAX && is_portid_root(portid))
+ return 0;
+
+ return -EINVAL;
+}
+
+static u64 vtd_sip_read_event_counter(struct fme_perf_priv *priv, u32 event,
+ u32 portid)
+{
+ void __iomem *base = priv->ioaddr;
+ u64 v;
+
+ v = readq(base + VTD_SIP_CTRL);
+ v &= ~VTD_SIP_CTRL_EVNT;
+ v |= FIELD_PREP(VTD_SIP_CTRL_EVNT, event);
+ writeq(v, base + VTD_SIP_CTRL);
+
+ if (readq_poll_timeout_atomic(base + VTD_SIP_CNTR, v,
+ FIELD_GET(VTD_SIP_CNTR_EVNT, v) == event,
+ 1, PERF_TIMEOUT)) {
+ dev_err(priv->dev, "timeout, unmatched vtd sip event code in counter register\n");
+ return 0;
+ }
+
+ v = fme_read_perf_cntr_reg(base + VTD_SIP_CNTR);
+ return FIELD_GET(VTD_SIP_CNTR_EVNT_CNTR, v);
+}
+
+static struct fme_perf_event_ops fme_perf_event_ops[] = {
+ [FME_EVTYPE_BASIC] = {.event_init = basic_event_init,
+ .read_counter = basic_read_event_counter,},
+ [FME_EVTYPE_CACHE] = {.event_init = cache_event_init,
+ .read_counter = cache_read_event_counter,},
+ [FME_EVTYPE_FABRIC] = {.event_init = fabric_event_init,
+ .event_destroy = fabric_event_destroy,
+ .read_counter = fabric_read_event_counter,},
+ [FME_EVTYPE_VTD] = {.event_init = vtd_event_init,
+ .read_counter = vtd_read_event_counter,},
+ [FME_EVTYPE_VTD_SIP] = {.event_init = vtd_sip_event_init,
+ .read_counter = vtd_sip_read_event_counter,},
+};
+
+static ssize_t fme_perf_event_show(struct device *dev,
+ struct device_attribute *attr, char *buf)
+{
+ struct dev_ext_attribute *eattr;
+ unsigned long config;
+ char *ptr = buf;
+
+ eattr = container_of(attr, struct dev_ext_attribute, attr);
+ config = (unsigned long)eattr->var;
+
+ ptr += sprintf(ptr, "event=0x%02x", (unsigned int)get_event(config));
+ ptr += sprintf(ptr, ",evtype=0x%02x", (unsigned int)get_evtype(config));
+
+ if (is_portid_root(get_portid(config)))
+ ptr += sprintf(ptr, ",portid=0x%02x\n", FME_PORTID_ROOT);
+ else
+ ptr += sprintf(ptr, ",portid=?\n");
+
+ return (ssize_t)(ptr - buf);
+}
+
+#define FME_EVENT_ATTR(_name) \
+ __ATTR(_name, 0444, fme_perf_event_show, NULL)
+
+#define FME_PORT_EVENT_CONFIG(_event, _type) \
+ (void *)((((_event) << FME_EVENT_SHIFT) & FME_EVENT_MASK) | \
+ (((_type) << FME_EVTYPE_SHIFT) & FME_EVTYPE_MASK))
+
+#define FME_EVENT_CONFIG(_event, _type) \
+ (void *)((((_event) << FME_EVENT_SHIFT) & FME_EVENT_MASK) | \
+ (((_type) << FME_EVTYPE_SHIFT) & FME_EVTYPE_MASK) | \
+ (FME_PORTID_ROOT << FME_PORTID_SHIFT))
+
+/* FME Perf Basic Events */
+#define FME_EVENT_BASIC(_name, _event) \
+static struct dev_ext_attribute fme_perf_event_##_name = { \
+ .attr = FME_EVENT_ATTR(_name), \
+ .var = FME_EVENT_CONFIG(_event, FME_EVTYPE_BASIC), \
+}
+
+FME_EVENT_BASIC(clock, BASIC_EVNT_CLK);
+
+static struct attribute *fme_perf_basic_events_attrs[] = {
+ &fme_perf_event_clock.attr.attr,
+ NULL,
+};
+
+static const struct attribute_group fme_perf_basic_events_group = {
+ .name = "events",
+ .attrs = fme_perf_basic_events_attrs,
+};
+
+/* FME Perf Cache Events */
+#define FME_EVENT_CACHE(_name, _event) \
+static struct dev_ext_attribute fme_perf_event_cache_##_name = { \
+ .attr = FME_EVENT_ATTR(cache_##_name), \
+ .var = FME_EVENT_CONFIG(_event, FME_EVTYPE_CACHE), \
+}
+
+FME_EVENT_CACHE(read_hit, CACHE_EVNT_RD_HIT);
+FME_EVENT_CACHE(read_miss, CACHE_EVNT_RD_MISS);
+FME_EVENT_CACHE(write_hit, CACHE_EVNT_WR_HIT);
+FME_EVENT_CACHE(write_miss, CACHE_EVNT_WR_MISS);
+FME_EVENT_CACHE(hold_request, CACHE_EVNT_HOLD_REQ);
+FME_EVENT_CACHE(tx_req_stall, CACHE_EVNT_TX_REQ_STALL);
+FME_EVENT_CACHE(rx_req_stall, CACHE_EVNT_RX_REQ_STALL);
+FME_EVENT_CACHE(eviction, CACHE_EVNT_EVICTIONS);
+FME_EVENT_CACHE(data_write_port_contention, CACHE_EVNT_DATA_WR_PORT_CONTEN);
+FME_EVENT_CACHE(tag_write_port_contention, CACHE_EVNT_TAG_WR_PORT_CONTEN);
+
+static struct attribute *fme_perf_cache_events_attrs[] = {
+ &fme_perf_event_cache_read_hit.attr.attr,
+ &fme_perf_event_cache_read_miss.attr.attr,
+ &fme_perf_event_cache_write_hit.attr.attr,
+ &fme_perf_event_cache_write_miss.attr.attr,
+ &fme_perf_event_cache_hold_request.attr.attr,
+ &fme_perf_event_cache_tx_req_stall.attr.attr,
+ &fme_perf_event_cache_rx_req_stall.attr.attr,
+ &fme_perf_event_cache_eviction.attr.attr,
+ &fme_perf_event_cache_data_write_port_contention.attr.attr,
+ &fme_perf_event_cache_tag_write_port_contention.attr.attr,
+ NULL,
+};
+
+static umode_t fme_perf_events_visible(struct kobject *kobj,
+ struct attribute *attr, int n)
+{
+ struct pmu *pmu = dev_get_drvdata(kobj_to_dev(kobj));
+ struct fme_perf_priv *priv = to_fme_perf_priv(pmu);
+
+ return (priv->id == FME_FEATURE_ID_GLOBAL_IPERF) ? attr->mode : 0;
+}
+
+static const struct attribute_group fme_perf_cache_events_group = {
+ .name = "events",
+ .attrs = fme_perf_cache_events_attrs,
+ .is_visible = fme_perf_events_visible,
+};
+
+/* FME Perf Fabric Events */
+#define FME_EVENT_FABRIC(_name, _event) \
+static struct dev_ext_attribute fme_perf_event_fab_##_name = { \
+ .attr = FME_EVENT_ATTR(fab_##_name), \
+ .var = FME_EVENT_CONFIG(_event, FME_EVTYPE_FABRIC), \
+}
+
+#define FME_EVENT_FABRIC_PORT(_name, _event) \
+static struct dev_ext_attribute fme_perf_event_fab_port_##_name = { \
+ .attr = FME_EVENT_ATTR(fab_port_##_name), \
+ .var = FME_PORT_EVENT_CONFIG(_event, FME_EVTYPE_FABRIC), \
+}
+
+FME_EVENT_FABRIC(pcie0_read, FAB_EVNT_PCIE0_RD);
+FME_EVENT_FABRIC(pcie0_write, FAB_EVNT_PCIE0_WR);
+FME_EVENT_FABRIC(pcie1_read, FAB_EVNT_PCIE1_RD);
+FME_EVENT_FABRIC(pcie1_write, FAB_EVNT_PCIE1_WR);
+FME_EVENT_FABRIC(upi_read, FAB_EVNT_UPI_RD);
+FME_EVENT_FABRIC(upi_write, FAB_EVNT_UPI_WR);
+FME_EVENT_FABRIC(mmio_read, FAB_EVNT_MMIO_RD);
+FME_EVENT_FABRIC(mmio_write, FAB_EVNT_MMIO_WR);
+
+FME_EVENT_FABRIC_PORT(pcie0_read, FAB_EVNT_PCIE0_RD);
+FME_EVENT_FABRIC_PORT(pcie0_write, FAB_EVNT_PCIE0_WR);
+FME_EVENT_FABRIC_PORT(pcie1_read, FAB_EVNT_PCIE1_RD);
+FME_EVENT_FABRIC_PORT(pcie1_write, FAB_EVNT_PCIE1_WR);
+FME_EVENT_FABRIC_PORT(upi_read, FAB_EVNT_UPI_RD);
+FME_EVENT_FABRIC_PORT(upi_write, FAB_EVNT_UPI_WR);
+FME_EVENT_FABRIC_PORT(mmio_read, FAB_EVNT_MMIO_RD);
+FME_EVENT_FABRIC_PORT(mmio_write, FAB_EVNT_MMIO_WR);
+
+static struct attribute *fme_perf_fabric_events_attrs[] = {
+ &fme_perf_event_fab_pcie0_read.attr.attr,
+ &fme_perf_event_fab_pcie0_write.attr.attr,
+ &fme_perf_event_fab_pcie1_read.attr.attr,
+ &fme_perf_event_fab_pcie1_write.attr.attr,
+ &fme_perf_event_fab_upi_read.attr.attr,
+ &fme_perf_event_fab_upi_write.attr.attr,
+ &fme_perf_event_fab_mmio_read.attr.attr,
+ &fme_perf_event_fab_mmio_write.attr.attr,
+ &fme_perf_event_fab_port_pcie0_read.attr.attr,
+ &fme_perf_event_fab_port_pcie0_write.attr.attr,
+ &fme_perf_event_fab_port_pcie1_read.attr.attr,
+ &fme_perf_event_fab_port_pcie1_write.attr.attr,
+ &fme_perf_event_fab_port_upi_read.attr.attr,
+ &fme_perf_event_fab_port_upi_write.attr.attr,
+ &fme_perf_event_fab_port_mmio_read.attr.attr,
+ &fme_perf_event_fab_port_mmio_write.attr.attr,
+ NULL,
+};
+
+static umode_t fme_perf_fabric_events_visible(struct kobject *kobj,
+ struct attribute *attr, int n)
+{
+ struct pmu *pmu = dev_get_drvdata(kobj_to_dev(kobj));
+ struct fme_perf_priv *priv = to_fme_perf_priv(pmu);
+ struct dev_ext_attribute *eattr;
+ unsigned long var;
+
+ eattr = container_of(attr, struct dev_ext_attribute, attr.attr);
+ var = (unsigned long)eattr->var;
+
+ if (is_fabric_event_supported(priv, get_event(var), get_portid(var)))
+ return attr->mode;
+
+ return 0;
+}
+
+static const struct attribute_group fme_perf_fabric_events_group = {
+ .name = "events",
+ .attrs = fme_perf_fabric_events_attrs,
+ .is_visible = fme_perf_fabric_events_visible,
+};
+
+/* FME Perf VTD Events */
+#define FME_EVENT_VTD_PORT(_name, _event) \
+static struct dev_ext_attribute fme_perf_event_vtd_port_##_name = { \
+ .attr = FME_EVENT_ATTR(vtd_port_##_name), \
+ .var = FME_PORT_EVENT_CONFIG(_event, FME_EVTYPE_VTD), \
+}
+
+FME_EVENT_VTD_PORT(read_transaction, VTD_EVNT_AFU_MEM_RD_TRANS);
+FME_EVENT_VTD_PORT(write_transaction, VTD_EVNT_AFU_MEM_WR_TRANS);
+FME_EVENT_VTD_PORT(devtlb_read_hit, VTD_EVNT_AFU_DEVTLB_RD_HIT);
+FME_EVENT_VTD_PORT(devtlb_write_hit, VTD_EVNT_AFU_DEVTLB_WR_HIT);
+FME_EVENT_VTD_PORT(devtlb_4k_fill, VTD_EVNT_DEVTLB_4K_FILL);
+FME_EVENT_VTD_PORT(devtlb_2m_fill, VTD_EVNT_DEVTLB_2M_FILL);
+FME_EVENT_VTD_PORT(devtlb_1g_fill, VTD_EVNT_DEVTLB_1G_FILL);
+
+static struct attribute *fme_perf_vtd_events_attrs[] = {
+ &fme_perf_event_vtd_port_read_transaction.attr.attr,
+ &fme_perf_event_vtd_port_write_transaction.attr.attr,
+ &fme_perf_event_vtd_port_devtlb_read_hit.attr.attr,
+ &fme_perf_event_vtd_port_devtlb_write_hit.attr.attr,
+ &fme_perf_event_vtd_port_devtlb_4k_fill.attr.attr,
+ &fme_perf_event_vtd_port_devtlb_2m_fill.attr.attr,
+ &fme_perf_event_vtd_port_devtlb_1g_fill.attr.attr,
+ NULL,
+};
+
+static const struct attribute_group fme_perf_vtd_events_group = {
+ .name = "events",
+ .attrs = fme_perf_vtd_events_attrs,
+ .is_visible = fme_perf_events_visible,
+};
+
+/* FME Perf VTD SIP Events */
+#define FME_EVENT_VTD_SIP(_name, _event) \
+static struct dev_ext_attribute fme_perf_event_vtd_sip_##_name = { \
+ .attr = FME_EVENT_ATTR(vtd_sip_##_name), \
+ .var = FME_EVENT_CONFIG(_event, FME_EVTYPE_VTD_SIP), \
+}
+
+FME_EVENT_VTD_SIP(iotlb_4k_hit, VTD_SIP_EVNT_IOTLB_4K_HIT);
+FME_EVENT_VTD_SIP(iotlb_2m_hit, VTD_SIP_EVNT_IOTLB_2M_HIT);
+FME_EVENT_VTD_SIP(iotlb_1g_hit, VTD_SIP_EVNT_IOTLB_1G_HIT);
+FME_EVENT_VTD_SIP(slpwc_l3_hit, VTD_SIP_EVNT_SLPWC_L3_HIT);
+FME_EVENT_VTD_SIP(slpwc_l4_hit, VTD_SIP_EVNT_SLPWC_L4_HIT);
+FME_EVENT_VTD_SIP(rcc_hit, VTD_SIP_EVNT_RCC_HIT);
+FME_EVENT_VTD_SIP(iotlb_4k_miss, VTD_SIP_EVNT_IOTLB_4K_MISS);
+FME_EVENT_VTD_SIP(iotlb_2m_miss, VTD_SIP_EVNT_IOTLB_2M_MISS);
+FME_EVENT_VTD_SIP(iotlb_1g_miss, VTD_SIP_EVNT_IOTLB_1G_MISS);
+FME_EVENT_VTD_SIP(slpwc_l3_miss, VTD_SIP_EVNT_SLPWC_L3_MISS);
+FME_EVENT_VTD_SIP(slpwc_l4_miss, VTD_SIP_EVNT_SLPWC_L4_MISS);
+FME_EVENT_VTD_SIP(rcc_miss, VTD_SIP_EVNT_RCC_MISS);
+
+static struct attribute *fme_perf_vtd_sip_events_attrs[] = {
+ &fme_perf_event_vtd_sip_iotlb_4k_hit.attr.attr,
+ &fme_perf_event_vtd_sip_iotlb_2m_hit.attr.attr,
+ &fme_perf_event_vtd_sip_iotlb_1g_hit.attr.attr,
+ &fme_perf_event_vtd_sip_slpwc_l3_hit.attr.attr,
+ &fme_perf_event_vtd_sip_slpwc_l4_hit.attr.attr,
+ &fme_perf_event_vtd_sip_rcc_hit.attr.attr,
+ &fme_perf_event_vtd_sip_iotlb_4k_miss.attr.attr,
+ &fme_perf_event_vtd_sip_iotlb_2m_miss.attr.attr,
+ &fme_perf_event_vtd_sip_iotlb_1g_miss.attr.attr,
+ &fme_perf_event_vtd_sip_slpwc_l3_miss.attr.attr,
+ &fme_perf_event_vtd_sip_slpwc_l4_miss.attr.attr,
+ &fme_perf_event_vtd_sip_rcc_miss.attr.attr,
+ NULL,
+};
+
+static const struct attribute_group fme_perf_vtd_sip_events_group = {
+ .name = "events",
+ .attrs = fme_perf_vtd_sip_events_attrs,
+ .is_visible = fme_perf_events_visible,
+};
+
+static const struct attribute_group *fme_perf_events_groups[] = {
+ &fme_perf_basic_events_group,
+ &fme_perf_cache_events_group,
+ &fme_perf_fabric_events_group,
+ &fme_perf_vtd_events_group,
+ &fme_perf_vtd_sip_events_group,
+ NULL,
+};
+
+static struct fme_perf_event_ops *get_event_ops(u32 evtype)
+{
+ if (evtype > FME_EVTYPE_MAX)
+ return NULL;
+
+ return &fme_perf_event_ops[evtype];
+}
+
+static void fme_perf_event_destroy(struct perf_event *event)
+{
+ struct fme_perf_event_ops *ops = get_event_ops(event->hw.event_base);
+ struct fme_perf_priv *priv = to_fme_perf_priv(event->pmu);
+
+ if (ops->event_destroy)
+ ops->event_destroy(priv, event->hw.idx, event->hw.config_base);
+}
+
+static int fme_perf_event_init(struct perf_event *event)
+{
+ struct fme_perf_priv *priv = to_fme_perf_priv(event->pmu);
+ struct hw_perf_event *hwc = &event->hw;
+ struct fme_perf_event_ops *ops;
+ u32 eventid, evtype, portid;
+
+ /* test the event attr type check for PMU enumeration */
+ if (event->attr.type != event->pmu->type)
+ return -ENOENT;
+
+ /*
+ * fme counters are shared across all cores.
+ * Therefore, it does not support per-process mode.
+ * Also, it does not support event sampling mode.
+ */
+ if (is_sampling_event(event) || event->attach_state & PERF_ATTACH_TASK)
+ return -EINVAL;
+
+ if (event->cpu < 0)
+ return -EINVAL;
+
+ if (event->cpu != priv->cpu)
+ return -EINVAL;
+
+ eventid = get_event(event->attr.config);
+ portid = get_portid(event->attr.config);
+ evtype = get_evtype(event->attr.config);
+ if (evtype > FME_EVTYPE_MAX)
+ return -EINVAL;
+
+ hwc->event_base = evtype;
+ hwc->idx = (int)eventid;
+ hwc->config_base = portid;
+
+ event->destroy = fme_perf_event_destroy;
+
+ dev_dbg(priv->dev, "%s event=0x%x, evtype=0x%x, portid=0x%x,\n",
+ __func__, eventid, evtype, portid);
+
+ ops = get_event_ops(evtype);
+ if (ops->event_init)
+ return ops->event_init(priv, eventid, portid);
+
+ return 0;
+}
+
+static void fme_perf_event_update(struct perf_event *event)
+{
+ struct fme_perf_event_ops *ops = get_event_ops(event->hw.event_base);
+ struct fme_perf_priv *priv = to_fme_perf_priv(event->pmu);
+ struct hw_perf_event *hwc = &event->hw;
+ u64 now, prev, delta;
+
+ now = ops->read_counter(priv, (u32)hwc->idx, hwc->config_base);
+ prev = local64_read(&hwc->prev_count);
+ delta = now - prev;
+
+ local64_add(delta, &event->count);
+}
+
+static void fme_perf_event_start(struct perf_event *event, int flags)
+{
+ struct fme_perf_event_ops *ops = get_event_ops(event->hw.event_base);
+ struct fme_perf_priv *priv = to_fme_perf_priv(event->pmu);
+ struct hw_perf_event *hwc = &event->hw;
+ u64 count;
+
+ count = ops->read_counter(priv, (u32)hwc->idx, hwc->config_base);
+ local64_set(&hwc->prev_count, count);
+}
+
+static void fme_perf_event_stop(struct perf_event *event, int flags)
+{
+ fme_perf_event_update(event);
+}
+
+static int fme_perf_event_add(struct perf_event *event, int flags)
+{
+ if (flags & PERF_EF_START)
+ fme_perf_event_start(event, flags);
+
+ return 0;
+}
+
+static void fme_perf_event_del(struct perf_event *event, int flags)
+{
+ fme_perf_event_stop(event, PERF_EF_UPDATE);
+}
+
+static void fme_perf_event_read(struct perf_event *event)
+{
+ fme_perf_event_update(event);
+}
+
+static void fme_perf_setup_hardware(struct fme_perf_priv *priv)
+{
+ void __iomem *base = priv->ioaddr;
+ u64 v;
+
+ /* read and save current working mode for fabric counters */
+ v = readq(base + FAB_CTRL);
+
+ if (FIELD_GET(FAB_PORT_FILTER, v) == FAB_PORT_FILTER_DISABLE)
+ priv->fab_port_id = FME_PORTID_ROOT;
+ else
+ priv->fab_port_id = FIELD_GET(FAB_PORT_ID, v);
+}
+
+static int fme_perf_pmu_register(struct platform_device *pdev,
+ struct fme_perf_priv *priv)
+{
+ struct pmu *pmu = &priv->pmu;
+ char *name;
+ int ret;
+
+ spin_lock_init(&priv->fab_lock);
+
+ fme_perf_setup_hardware(priv);
+
+ pmu->task_ctx_nr = perf_invalid_context;
+ pmu->attr_groups = fme_perf_groups;
+ pmu->attr_update = fme_perf_events_groups;
+ pmu->event_init = fme_perf_event_init;
+ pmu->add = fme_perf_event_add;
+ pmu->del = fme_perf_event_del;
+ pmu->start = fme_perf_event_start;
+ pmu->stop = fme_perf_event_stop;
+ pmu->read = fme_perf_event_read;
+ pmu->capabilities = PERF_PMU_CAP_NO_INTERRUPT |
+ PERF_PMU_CAP_NO_EXCLUDE;
+
+ name = devm_kasprintf(priv->dev, GFP_KERNEL, "dfl_fme%d", pdev->id);
+
+ ret = perf_pmu_register(pmu, name, -1);
+ if (ret)
+ return ret;
+
+ return 0;
+}
+
+static void fme_perf_pmu_unregister(struct fme_perf_priv *priv)
+{
+ perf_pmu_unregister(&priv->pmu);
+}
+
+static int fme_perf_offline_cpu(unsigned int cpu, struct hlist_node *node)
+{
+ struct fme_perf_priv *priv;
+ int target;
+
+ priv = hlist_entry_safe(node, struct fme_perf_priv, node);
+
+ if (cpu != priv->cpu)
+ return 0;
+
+ target = cpumask_any_but(cpu_online_mask, cpu);
+ if (target >= nr_cpu_ids)
+ return 0;
+
+ priv->cpu = target;
+ return 0;
+}
+
+static int fme_perf_init(struct platform_device *pdev,
+ struct dfl_feature *feature)
+{
+ struct fme_perf_priv *priv;
+ int ret;
+
+ priv = devm_kzalloc(&pdev->dev, sizeof(*priv), GFP_KERNEL);
+ if (!priv)
+ return -ENOMEM;
+
+ priv->dev = &pdev->dev;
+ priv->ioaddr = feature->ioaddr;
+ priv->id = feature->id;
+ priv->cpu = raw_smp_processor_id();
+
+ ret = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN,
+ "perf/fpga/dfl_fme:online",
+ NULL, fme_perf_offline_cpu);
+ if (ret < 0)
+ return ret;
+
+ priv->cpuhp_state = ret;
+
+ /* Register the pmu instance for cpu hotplug */
+ ret = cpuhp_state_add_instance_nocalls(priv->cpuhp_state, &priv->node);
+ if (ret)
+ goto cpuhp_instance_err;
+
+ ret = fme_perf_pmu_register(pdev, priv);
+ if (ret)
+ goto pmu_register_err;
+
+ feature->priv = priv;
+ return 0;
+
+pmu_register_err:
+ cpuhp_state_remove_instance_nocalls(priv->cpuhp_state, &priv->node);
+cpuhp_instance_err:
+ cpuhp_remove_multi_state(priv->cpuhp_state);
+ return ret;
+}
+
+static void fme_perf_uinit(struct platform_device *pdev,
+ struct dfl_feature *feature)
+{
+ struct fme_perf_priv *priv = feature->priv;
+
+ fme_perf_pmu_unregister(priv);
+ cpuhp_state_remove_instance_nocalls(priv->cpuhp_state, &priv->node);
+ cpuhp_remove_multi_state(priv->cpuhp_state);
+}
+
+const struct dfl_feature_id fme_perf_id_table[] = {
+ {.id = FME_FEATURE_ID_GLOBAL_IPERF,},
+ {.id = FME_FEATURE_ID_GLOBAL_DPERF,},
+ {0,}
+};
+
+const struct dfl_feature_ops fme_perf_ops = {
+ .init = fme_perf_init,
+ .uinit = fme_perf_uinit,
+};
diff --git a/drivers/fpga/dfl-fme.h b/drivers/fpga/dfl-fme.h
index 6685c8ef965b..4195dd68193e 100644
--- a/drivers/fpga/dfl-fme.h
+++ b/drivers/fpga/dfl-fme.h
@@ -38,5 +38,7 @@ extern const struct dfl_feature_id fme_pr_mgmt_id_table[];
extern const struct dfl_feature_ops fme_global_err_ops;
extern const struct dfl_feature_id fme_global_err_id_table[];
extern const struct attribute_group fme_global_err_group;
+extern const struct dfl_feature_ops fme_perf_ops;
+extern const struct dfl_feature_id fme_perf_id_table[];
#endif /* __DFL_FME_H */
diff --git a/drivers/fpga/dfl.c b/drivers/fpga/dfl.c
index 96a2b8274a33..990994874bf1 100644
--- a/drivers/fpga/dfl.c
+++ b/drivers/fpga/dfl.c
@@ -1079,6 +1079,7 @@ static int __init dfl_fpga_init(void)
*/
int dfl_fpga_cdev_release_port(struct dfl_fpga_cdev *cdev, int port_id)
{
+ struct dfl_feature_platform_data *pdata;
struct platform_device *port_pdev;
int ret = -ENODEV;
@@ -1093,7 +1094,11 @@ int dfl_fpga_cdev_release_port(struct dfl_fpga_cdev *cdev, int port_id)
goto put_dev_exit;
}
- ret = dfl_feature_dev_use_begin(dev_get_platdata(&port_pdev->dev));
+ pdata = dev_get_platdata(&port_pdev->dev);
+
+ mutex_lock(&pdata->lock);
+ ret = dfl_feature_dev_use_begin(pdata, true);
+ mutex_unlock(&pdata->lock);
if (ret)
goto put_dev_exit;
@@ -1120,6 +1125,7 @@ EXPORT_SYMBOL_GPL(dfl_fpga_cdev_release_port);
*/
int dfl_fpga_cdev_assign_port(struct dfl_fpga_cdev *cdev, int port_id)
{
+ struct dfl_feature_platform_data *pdata;
struct platform_device *port_pdev;
int ret = -ENODEV;
@@ -1138,7 +1144,12 @@ int dfl_fpga_cdev_assign_port(struct dfl_fpga_cdev *cdev, int port_id)
if (ret)
goto put_dev_exit;
- dfl_feature_dev_use_end(dev_get_platdata(&port_pdev->dev));
+ pdata = dev_get_platdata(&port_pdev->dev);
+
+ mutex_lock(&pdata->lock);
+ dfl_feature_dev_use_end(pdata);
+ mutex_unlock(&pdata->lock);
+
cdev->released_port_num--;
put_dev_exit:
put_device(&port_pdev->dev);
diff --git a/drivers/fpga/dfl.h b/drivers/fpga/dfl.h
index 9f0e656de720..2f5d3052e36e 100644
--- a/drivers/fpga/dfl.h
+++ b/drivers/fpga/dfl.h
@@ -197,16 +197,16 @@ struct dfl_feature_driver {
* feature dev (platform device)'s reources.
* @ioaddr: mapped mmio resource address.
* @ops: ops of this sub feature.
+ * @priv: priv data of this feature.
*/
struct dfl_feature {
u64 id;
int resource_index;
void __iomem *ioaddr;
const struct dfl_feature_ops *ops;
+ void *priv;
};
-#define DEV_STATUS_IN_USE 0
-
#define FEATURE_DEV_ID_UNUSED (-1)
/**
@@ -219,8 +219,9 @@ struct dfl_feature {
* @dfl_cdev: ptr to container device.
* @id: id used for this feature device.
* @disable_count: count for port disable.
+ * @excl_open: set on feature device exclusive open.
+ * @open_count: count for feature device open.
* @num: number for sub features.
- * @dev_status: dev status (e.g. DEV_STATUS_IN_USE).
* @private: ptr to feature dev private data.
* @features: sub features of this feature dev.
*/
@@ -232,26 +233,46 @@ struct dfl_feature_platform_data {
struct dfl_fpga_cdev *dfl_cdev;
int id;
unsigned int disable_count;
- unsigned long dev_status;
+ bool excl_open;
+ int open_count;
void *private;
int num;
- struct dfl_feature features[0];
+ struct dfl_feature features[];
};
static inline
-int dfl_feature_dev_use_begin(struct dfl_feature_platform_data *pdata)
+int dfl_feature_dev_use_begin(struct dfl_feature_platform_data *pdata,
+ bool excl)
{
- /* Test and set IN_USE flags to ensure file is exclusively used */
- if (test_and_set_bit_lock(DEV_STATUS_IN_USE, &pdata->dev_status))
+ if (pdata->excl_open)
return -EBUSY;
+ if (excl) {
+ if (pdata->open_count)
+ return -EBUSY;
+
+ pdata->excl_open = true;
+ }
+ pdata->open_count++;
+
return 0;
}
static inline
void dfl_feature_dev_use_end(struct dfl_feature_platform_data *pdata)
{
- clear_bit_unlock(DEV_STATUS_IN_USE, &pdata->dev_status);
+ pdata->excl_open = false;
+
+ if (WARN_ON(pdata->open_count <= 0))
+ return;
+
+ pdata->open_count--;
+}
+
+static inline
+int dfl_feature_dev_use_count(struct dfl_feature_platform_data *pdata)
+{
+ return pdata->open_count;
}
static inline
diff --git a/drivers/fpga/ice40-spi.c b/drivers/fpga/ice40-spi.c
index 56e112e14a10..8d689fea0dab 100644
--- a/drivers/fpga/ice40-spi.c
+++ b/drivers/fpga/ice40-spi.c
@@ -46,10 +46,16 @@ static int ice40_fpga_ops_write_init(struct fpga_manager *mgr,
struct spi_message message;
struct spi_transfer assert_cs_then_reset_delay = {
.cs_change = 1,
- .delay_usecs = ICE40_SPI_RESET_DELAY
+ .delay = {
+ .value = ICE40_SPI_RESET_DELAY,
+ .unit = SPI_DELAY_UNIT_USECS
+ }
};
struct spi_transfer housekeeping_delay_then_release_cs = {
- .delay_usecs = ICE40_SPI_HOUSEKEEPING_DELAY
+ .delay = {
+ .value = ICE40_SPI_HOUSEKEEPING_DELAY,
+ .unit = SPI_DELAY_UNIT_USECS
+ }
};
int ret;
diff --git a/drivers/fpga/machxo2-spi.c b/drivers/fpga/machxo2-spi.c
index 4d8a87641587..b316369156fe 100644
--- a/drivers/fpga/machxo2-spi.c
+++ b/drivers/fpga/machxo2-spi.c
@@ -157,7 +157,8 @@ static int machxo2_cleanup(struct fpga_manager *mgr)
spi_message_init(&msg);
tx[1].tx_buf = &refresh;
tx[1].len = sizeof(refresh);
- tx[1].delay_usecs = MACHXO2_REFRESH_USEC;
+ tx[1].delay.value = MACHXO2_REFRESH_USEC;
+ tx[1].delay.unit = SPI_DELAY_UNIT_USECS;
spi_message_add_tail(&tx[1], &msg);
ret = spi_sync(spi, &msg);
if (ret)
@@ -208,7 +209,8 @@ static int machxo2_write_init(struct fpga_manager *mgr,
spi_message_init(&msg);
tx[0].tx_buf = &enable;
tx[0].len = sizeof(enable);
- tx[0].delay_usecs = MACHXO2_LOW_DELAY_USEC;
+ tx[0].delay.value = MACHXO2_LOW_DELAY_USEC;
+ tx[0].delay.unit = SPI_DELAY_UNIT_USECS;
spi_message_add_tail(&tx[0], &msg);
tx[1].tx_buf = &erase;
@@ -269,7 +271,8 @@ static int machxo2_write(struct fpga_manager *mgr, const char *buf,
spi_message_init(&msg);
tx.tx_buf = payload;
tx.len = MACHXO2_BUF_SIZE;
- tx.delay_usecs = MACHXO2_HIGH_DELAY_USEC;
+ tx.delay.value = MACHXO2_HIGH_DELAY_USEC;
+ tx.delay.unit = SPI_DELAY_UNIT_USECS;
spi_message_add_tail(&tx, &msg);
ret = spi_sync(spi, &msg);
if (ret) {
@@ -317,7 +320,8 @@ static int machxo2_write_complete(struct fpga_manager *mgr,
spi_message_init(&msg);
tx[1].tx_buf = &refresh;
tx[1].len = sizeof(refresh);
- tx[1].delay_usecs = MACHXO2_REFRESH_USEC;
+ tx[1].delay.value = MACHXO2_REFRESH_USEC;
+ tx[1].delay.unit = SPI_DELAY_UNIT_USECS;
spi_message_add_tail(&tx[1], &msg);
ret = spi_sync(spi, &msg);
if (ret)
diff --git a/drivers/fpga/stratix10-soc.c b/drivers/fpga/stratix10-soc.c
index 215d33789c74..44b7c569d4dc 100644
--- a/drivers/fpga/stratix10-soc.c
+++ b/drivers/fpga/stratix10-soc.c
@@ -154,11 +154,11 @@ static void s10_receive_callback(struct stratix10_svc_client *client,
* Here we set status bits as we receive them. Elsewhere, we always use
* test_and_clear_bit() to check status in priv->status
*/
- for (i = 0; i <= SVC_STATUS_RECONFIG_ERROR; i++)
+ for (i = 0; i <= SVC_STATUS_ERROR; i++)
if (status & (1 << i))
set_bit(i, &priv->status);
- if (status & BIT(SVC_STATUS_RECONFIG_BUFFER_DONE)) {
+ if (status & BIT(SVC_STATUS_BUFFER_DONE)) {
s10_unlock_bufs(priv, data->kaddr1);
s10_unlock_bufs(priv, data->kaddr2);
s10_unlock_bufs(priv, data->kaddr3);
@@ -209,8 +209,7 @@ static int s10_ops_write_init(struct fpga_manager *mgr,
}
ret = 0;
- if (!test_and_clear_bit(SVC_STATUS_RECONFIG_REQUEST_OK,
- &priv->status)) {
+ if (!test_and_clear_bit(SVC_STATUS_OK, &priv->status)) {
ret = -ETIMEDOUT;
goto init_done;
}
@@ -323,17 +322,15 @@ static int s10_ops_write(struct fpga_manager *mgr, const char *buf,
&priv->status_return_completion,
S10_BUFFER_TIMEOUT);
- if (test_and_clear_bit(SVC_STATUS_RECONFIG_BUFFER_DONE,
- &priv->status) ||
- test_and_clear_bit(SVC_STATUS_RECONFIG_BUFFER_SUBMITTED,
+ if (test_and_clear_bit(SVC_STATUS_BUFFER_DONE, &priv->status) ||
+ test_and_clear_bit(SVC_STATUS_BUFFER_SUBMITTED,
&priv->status)) {
ret = 0;
continue;
}
- if (test_and_clear_bit(SVC_STATUS_RECONFIG_ERROR,
- &priv->status)) {
- dev_err(dev, "ERROR - giving up - SVC_STATUS_RECONFIG_ERROR\n");
+ if (test_and_clear_bit(SVC_STATUS_ERROR, &priv->status)) {
+ dev_err(dev, "ERROR - giving up - SVC_STATUS_ERROR\n");
ret = -EFAULT;
break;
}
@@ -393,13 +390,11 @@ static int s10_ops_write_complete(struct fpga_manager *mgr,
timeout = ret;
ret = 0;
- if (test_and_clear_bit(SVC_STATUS_RECONFIG_COMPLETED,
- &priv->status))
+ if (test_and_clear_bit(SVC_STATUS_COMPLETED, &priv->status))
break;
- if (test_and_clear_bit(SVC_STATUS_RECONFIG_ERROR,
- &priv->status)) {
- dev_err(dev, "ERROR - giving up - SVC_STATUS_RECONFIG_ERROR\n");
+ if (test_and_clear_bit(SVC_STATUS_ERROR, &priv->status)) {
+ dev_err(dev, "ERROR - giving up - SVC_STATUS_ERROR\n");
ret = -EFAULT;
break;
}
@@ -482,7 +477,8 @@ static int s10_remove(struct platform_device *pdev)
}
static const struct of_device_id s10_of_match[] = {
- { .compatible = "intel,stratix10-soc-fpga-mgr", },
+ {.compatible = "intel,stratix10-soc-fpga-mgr"},
+ {.compatible = "intel,agilex-soc-fpga-mgr"},
{},
};
diff --git a/drivers/fpga/zynqmp-fpga.c b/drivers/fpga/zynqmp-fpga.c
index b8a88d21d038..4a1139e05280 100644
--- a/drivers/fpga/zynqmp-fpga.c
+++ b/drivers/fpga/zynqmp-fpga.c
@@ -40,16 +40,12 @@ static int zynqmp_fpga_ops_write_init(struct fpga_manager *mgr,
static int zynqmp_fpga_ops_write(struct fpga_manager *mgr,
const char *buf, size_t size)
{
- const struct zynqmp_eemi_ops *eemi_ops = zynqmp_pm_get_eemi_ops();
struct zynqmp_fpga_priv *priv;
dma_addr_t dma_addr;
u32 eemi_flags = 0;
char *kbuf;
int ret;
- if (IS_ERR_OR_NULL(eemi_ops) || !eemi_ops->fpga_load)
- return -ENXIO;
-
priv = mgr->priv;
kbuf = dma_alloc_coherent(priv->dev, size, &dma_addr, GFP_KERNEL);
@@ -63,7 +59,7 @@ static int zynqmp_fpga_ops_write(struct fpga_manager *mgr,
if (priv->flags & FPGA_MGR_PARTIAL_RECONFIG)
eemi_flags |= XILINX_ZYNQMP_PM_FPGA_PARTIAL;
- ret = eemi_ops->fpga_load(dma_addr, size, eemi_flags);
+ ret = zynqmp_pm_fpga_load(dma_addr, size, eemi_flags);
dma_free_coherent(priv->dev, size, kbuf, dma_addr);
@@ -78,13 +74,9 @@ static int zynqmp_fpga_ops_write_complete(struct fpga_manager *mgr,
static enum fpga_mgr_states zynqmp_fpga_ops_state(struct fpga_manager *mgr)
{
- const struct zynqmp_eemi_ops *eemi_ops = zynqmp_pm_get_eemi_ops();
- u32 status;
-
- if (IS_ERR_OR_NULL(eemi_ops) || !eemi_ops->fpga_get_status)
- return FPGA_MGR_STATE_UNKNOWN;
+ u32 status = 0;
- eemi_ops->fpga_get_status(&status);
+ zynqmp_pm_fpga_get_status(&status);
if (status & IXR_FPGA_DONE_MASK)
return FPGA_MGR_STATE_OPERATING;
diff --git a/drivers/gnss/serial.h b/drivers/gnss/serial.h
index 980ffdc86c2a..621953f7821d 100644
--- a/drivers/gnss/serial.h
+++ b/drivers/gnss/serial.h
@@ -16,7 +16,7 @@ struct gnss_serial {
struct gnss_device *gdev;
speed_t speed;
const struct gnss_serial_ops *ops;
- unsigned long drvdata[0];
+ unsigned long drvdata[];
};
enum gnss_serial_pm_state {
diff --git a/drivers/gnss/sirf.c b/drivers/gnss/sirf.c
index effed3a8d398..2ecb1d3e8eeb 100644
--- a/drivers/gnss/sirf.c
+++ b/drivers/gnss/sirf.c
@@ -439,14 +439,18 @@ static int sirf_probe(struct serdev_device *serdev)
data->on_off = devm_gpiod_get_optional(dev, "sirf,onoff",
GPIOD_OUT_LOW);
- if (IS_ERR(data->on_off))
+ if (IS_ERR(data->on_off)) {
+ ret = PTR_ERR(data->on_off);
goto err_put_device;
+ }
if (data->on_off) {
data->wakeup = devm_gpiod_get_optional(dev, "sirf,wakeup",
GPIOD_IN);
- if (IS_ERR(data->wakeup))
+ if (IS_ERR(data->wakeup)) {
+ ret = PTR_ERR(data->wakeup);
goto err_put_device;
+ }
ret = regulator_enable(data->vcc);
if (ret)
diff --git a/drivers/greybus/Kconfig b/drivers/greybus/Kconfig
index b84fcaf8b105..aeea082f1418 100644
--- a/drivers/greybus/Kconfig
+++ b/drivers/greybus/Kconfig
@@ -3,7 +3,7 @@ menuconfig GREYBUS
tristate "Greybus support"
depends on SYSFS
---help---
- This option enables the Greybus driver core. Greybus is an
+ This option enables the Greybus driver core. Greybus is a
hardware protocol that was designed to provide Unipro with a
sane application layer. It was originally designed for the
ARA project, a module phone system, but has shown up in other
@@ -12,7 +12,7 @@ menuconfig GREYBUS
Say Y here to enable support for these types of drivers.
- To compile this code as a module, chose M here: the module
+ To compile this code as a module, choose M here: the module
will be called greybus.ko
if GREYBUS
@@ -25,7 +25,7 @@ config GREYBUS_ES2
acts as a Greybus "host controller". This device is a bridge
from a USB device to a Unipro network.
- To compile this code as a module, chose M here: the module
+ To compile this code as a module, choose M here: the module
will be called gb-es2.ko
endif # GREYBUS
diff --git a/drivers/greybus/arpc.h b/drivers/greybus/arpc.h
index c8b83c5cfa79..b9ea81b55b29 100644
--- a/drivers/greybus/arpc.h
+++ b/drivers/greybus/arpc.h
@@ -21,7 +21,7 @@ struct arpc_request_message {
__le16 id; /* RPC unique id */
__le16 size; /* Size in bytes of header + payload */
__u8 type; /* RPC type */
- __u8 data[0]; /* ARPC data */
+ __u8 data[]; /* ARPC data */
} __packed;
struct arpc_response_message {
diff --git a/drivers/hwtracing/coresight/Makefile b/drivers/hwtracing/coresight/Makefile
index 0e3e72f0f510..19497d1d92bf 100644
--- a/drivers/hwtracing/coresight/Makefile
+++ b/drivers/hwtracing/coresight/Makefile
@@ -2,7 +2,8 @@
#
# Makefile for CoreSight drivers.
#
-obj-$(CONFIG_CORESIGHT) += coresight.o coresight-etm-perf.o coresight-platform.o
+obj-$(CONFIG_CORESIGHT) += coresight.o coresight-etm-perf.o \
+ coresight-platform.o coresight-sysfs.o
obj-$(CONFIG_CORESIGHT_LINK_AND_SINK_TMC) += coresight-tmc.o \
coresight-tmc-etf.o \
coresight-tmc-etr.o
diff --git a/drivers/hwtracing/coresight/coresight-cti-platform.c b/drivers/hwtracing/coresight/coresight-cti-platform.c
index 2fdaeec80ee5..98f830c6ed50 100644
--- a/drivers/hwtracing/coresight/coresight-cti-platform.c
+++ b/drivers/hwtracing/coresight/coresight-cti-platform.c
@@ -2,11 +2,17 @@
/*
* Copyright (c) 2019, The Linaro Limited. All rights reserved.
*/
+#include <linux/coresight.h>
+#include <linux/device.h>
+#include <linux/err.h>
+#include <linux/of.h>
+#include <linux/property.h>
+#include <linux/slab.h>
#include <dt-bindings/arm/coresight-cti-dt.h>
-#include <linux/of.h>
#include "coresight-cti.h"
+#include "coresight-priv.h"
/* Number of CTI signals in the v8 architecturally defined connection */
#define NR_V8PE_IN_SIGS 2
@@ -429,8 +435,7 @@ static int cti_plat_create_impdef_connections(struct device *dev,
}
/* get the hardware configuration & connection data. */
-int cti_plat_get_hw_data(struct device *dev,
- struct cti_drvdata *drvdata)
+static int cti_plat_get_hw_data(struct device *dev, struct cti_drvdata *drvdata)
{
int rc = 0;
struct cti_device *cti_dev = &drvdata->ctidev;
diff --git a/drivers/hwtracing/coresight/coresight-cti-sysfs.c b/drivers/hwtracing/coresight/coresight-cti-sysfs.c
index 1f8fb7c15e80..392757f3a019 100644
--- a/drivers/hwtracing/coresight/coresight-cti-sysfs.c
+++ b/drivers/hwtracing/coresight/coresight-cti-sysfs.c
@@ -4,7 +4,13 @@
* Author: Mike Leach <mike.leach@linaro.org>
*/
+#include <linux/atomic.h>
#include <linux/coresight.h>
+#include <linux/device.h>
+#include <linux/io.h>
+#include <linux/kernel.h>
+#include <linux/spinlock.h>
+#include <linux/sysfs.h>
#include "coresight-cti.h"
@@ -1036,8 +1042,8 @@ static int cti_create_con_sysfs_attr(struct device *dev,
enum cti_conn_attr_type attr_type,
int attr_idx)
{
- struct dev_ext_attribute *eattr = 0;
- char *name = 0;
+ struct dev_ext_attribute *eattr;
+ char *name;
eattr = devm_kzalloc(dev, sizeof(struct dev_ext_attribute),
GFP_KERNEL);
@@ -1139,7 +1145,7 @@ static int cti_create_con_attr_set(struct device *dev, int con_idx,
}
/* create the array of group pointers for the CTI sysfs groups */
-int cti_create_cons_groups(struct device *dev, struct cti_device *ctidev)
+static int cti_create_cons_groups(struct device *dev, struct cti_device *ctidev)
{
int nr_groups;
@@ -1156,8 +1162,8 @@ int cti_create_cons_groups(struct device *dev, struct cti_device *ctidev)
int cti_create_cons_sysfs(struct device *dev, struct cti_drvdata *drvdata)
{
struct cti_device *ctidev = &drvdata->ctidev;
- int err = 0, con_idx = 0, i;
- struct cti_trig_con *tc = NULL;
+ int err, con_idx = 0, i;
+ struct cti_trig_con *tc;
err = cti_create_cons_groups(dev, ctidev);
if (err)
diff --git a/drivers/hwtracing/coresight/coresight-cti.c b/drivers/hwtracing/coresight/coresight-cti.c
index aa6e0249bd70..40387d58c8e7 100644
--- a/drivers/hwtracing/coresight/coresight-cti.c
+++ b/drivers/hwtracing/coresight/coresight-cti.c
@@ -4,7 +4,22 @@
* Author: Mike Leach <mike.leach@linaro.org>
*/
+#include <linux/amba/bus.h>
+#include <linux/atomic.h>
+#include <linux/bits.h>
+#include <linux/coresight.h>
+#include <linux/cpu_pm.h>
+#include <linux/cpuhotplug.h>
+#include <linux/device.h>
+#include <linux/io.h>
+#include <linux/kernel.h>
+#include <linux/list.h>
+#include <linux/mutex.h>
+#include <linux/pm_runtime.h>
#include <linux/property.h>
+#include <linux/spinlock.h>
+
+#include "coresight-priv.h"
#include "coresight-cti.h"
/**
@@ -19,7 +34,7 @@
*/
/* net of CTI devices connected via CTM */
-LIST_HEAD(ect_net);
+static LIST_HEAD(ect_net);
/* protect the list */
static DEFINE_MUTEX(ect_mutex);
@@ -27,6 +42,12 @@ static DEFINE_MUTEX(ect_mutex);
#define csdev_to_cti_drvdata(csdev) \
dev_get_drvdata(csdev->dev.parent)
+/* power management handling */
+static int nr_cti_cpu;
+
+/* quick lookup list for CPU bound CTIs when power handling */
+static struct cti_drvdata *cti_cpu_drvdata[NR_CPUS];
+
/*
* CTI naming. CTI bound to cores will have the name cti_cpu<N> where
* N is the CPU ID. System CTIs will have the name cti_sys<I> where I
@@ -116,6 +137,35 @@ cti_err_not_enabled:
return rc;
}
+/* re-enable CTI on CPU when using CPU hotplug */
+static void cti_cpuhp_enable_hw(struct cti_drvdata *drvdata)
+{
+ struct cti_config *config = &drvdata->config;
+ struct device *dev = &drvdata->csdev->dev;
+
+ pm_runtime_get_sync(dev->parent);
+ spin_lock(&drvdata->spinlock);
+ config->hw_powered = true;
+
+ /* no need to do anything if no enable request */
+ if (!atomic_read(&drvdata->config.enable_req_count))
+ goto cti_hp_not_enabled;
+
+ /* try to claim the device */
+ if (coresight_claim_device(drvdata->base))
+ goto cti_hp_not_enabled;
+
+ cti_write_all_hw_regs(drvdata);
+ config->hw_enabled = true;
+ spin_unlock(&drvdata->spinlock);
+ return;
+
+ /* did not re-enable due to no claim / no request */
+cti_hp_not_enabled:
+ spin_unlock(&drvdata->spinlock);
+ pm_runtime_put(dev->parent);
+}
+
/* disable hardware */
static int cti_disable_hw(struct cti_drvdata *drvdata)
{
@@ -442,6 +492,34 @@ int cti_channel_setop(struct device *dev, enum cti_chan_set_op op,
return err;
}
+static bool cti_add_sysfs_link(struct cti_drvdata *drvdata,
+ struct cti_trig_con *tc)
+{
+ struct coresight_sysfs_link link_info;
+ int link_err = 0;
+
+ link_info.orig = drvdata->csdev;
+ link_info.orig_name = tc->con_dev_name;
+ link_info.target = tc->con_dev;
+ link_info.target_name = dev_name(&drvdata->csdev->dev);
+
+ link_err = coresight_add_sysfs_link(&link_info);
+ if (link_err)
+ dev_warn(&drvdata->csdev->dev,
+ "Failed to set CTI sysfs link %s<=>%s\n",
+ link_info.orig_name, link_info.target_name);
+ return !link_err;
+}
+
+static void cti_remove_sysfs_link(struct cti_trig_con *tc)
+{
+ struct coresight_sysfs_link link_info;
+
+ link_info.orig_name = tc->con_dev_name;
+ link_info.target = tc->con_dev;
+ coresight_remove_sysfs_link(&link_info);
+}
+
/*
* Look for a matching connection device name in the list of connections.
* If found then swap in the csdev name, set trig con association pointer
@@ -452,6 +530,8 @@ cti_match_fixup_csdev(struct cti_device *ctidev, const char *node_name,
struct coresight_device *csdev)
{
struct cti_trig_con *tc;
+ struct cti_drvdata *drvdata = container_of(ctidev, struct cti_drvdata,
+ ctidev);
list_for_each_entry(tc, &ctidev->trig_cons, node) {
if (tc->con_dev_name) {
@@ -459,7 +539,12 @@ cti_match_fixup_csdev(struct cti_device *ctidev, const char *node_name,
/* match: so swap in csdev name & dev */
tc->con_dev_name = dev_name(&csdev->dev);
tc->con_dev = csdev;
- return true;
+ /* try to set sysfs link */
+ if (cti_add_sysfs_link(drvdata, tc))
+ return true;
+ /* link failed - remove CTI reference */
+ tc->con_dev = NULL;
+ break;
}
}
}
@@ -522,6 +607,7 @@ void cti_remove_assoc_from_csdev(struct coresight_device *csdev)
ctidev = &ctidrv->ctidev;
list_for_each_entry(tc, &ctidev->trig_cons, node) {
if (tc->con_dev == csdev->ect_dev) {
+ cti_remove_sysfs_link(tc);
tc->con_dev = NULL;
break;
}
@@ -543,10 +629,16 @@ static void cti_update_conn_xrefs(struct cti_drvdata *drvdata)
struct cti_device *ctidev = &drvdata->ctidev;
list_for_each_entry(tc, &ctidev->trig_cons, node) {
- if (tc->con_dev)
- /* set tc->con_dev->ect_dev */
- coresight_set_assoc_ectdev_mutex(tc->con_dev,
+ if (tc->con_dev) {
+ /* if we can set the sysfs link */
+ if (cti_add_sysfs_link(drvdata, tc))
+ /* set the CTI/csdev association */
+ coresight_set_assoc_ectdev_mutex(tc->con_dev,
drvdata->csdev);
+ else
+ /* otherwise remove reference from CTI */
+ tc->con_dev = NULL;
+ }
}
}
@@ -559,7 +651,113 @@ static void cti_remove_conn_xrefs(struct cti_drvdata *drvdata)
if (tc->con_dev) {
coresight_set_assoc_ectdev_mutex(tc->con_dev,
NULL);
+ cti_remove_sysfs_link(tc);
+ tc->con_dev = NULL;
+ }
+ }
+}
+
+/** cti PM callbacks **/
+static int cti_cpu_pm_notify(struct notifier_block *nb, unsigned long cmd,
+ void *v)
+{
+ struct cti_drvdata *drvdata;
+ unsigned int cpu = smp_processor_id();
+ int notify_res = NOTIFY_OK;
+
+ if (!cti_cpu_drvdata[cpu])
+ return NOTIFY_OK;
+
+ drvdata = cti_cpu_drvdata[cpu];
+
+ if (WARN_ON_ONCE(drvdata->ctidev.cpu != cpu))
+ return NOTIFY_BAD;
+
+ spin_lock(&drvdata->spinlock);
+
+ switch (cmd) {
+ case CPU_PM_ENTER:
+ /* CTI regs all static - we have a copy & nothing to save */
+ drvdata->config.hw_powered = false;
+ if (drvdata->config.hw_enabled)
+ coresight_disclaim_device(drvdata->base);
+ break;
+
+ case CPU_PM_ENTER_FAILED:
+ drvdata->config.hw_powered = true;
+ if (drvdata->config.hw_enabled) {
+ if (coresight_claim_device(drvdata->base))
+ drvdata->config.hw_enabled = false;
+ }
+ break;
+
+ case CPU_PM_EXIT:
+ /* write hardware registers to re-enable. */
+ drvdata->config.hw_powered = true;
+ drvdata->config.hw_enabled = false;
+
+ /* check enable reference count to enable HW */
+ if (atomic_read(&drvdata->config.enable_req_count)) {
+ /* check we can claim the device as we re-power */
+ if (coresight_claim_device(drvdata->base))
+ goto cti_notify_exit;
+
+ drvdata->config.hw_enabled = true;
+ cti_write_all_hw_regs(drvdata);
+ }
+ break;
+
+ default:
+ notify_res = NOTIFY_DONE;
+ break;
+ }
+
+cti_notify_exit:
+ spin_unlock(&drvdata->spinlock);
+ return notify_res;
+}
+
+static struct notifier_block cti_cpu_pm_nb = {
+ .notifier_call = cti_cpu_pm_notify,
+};
+
+/* CPU HP handlers */
+static int cti_starting_cpu(unsigned int cpu)
+{
+ struct cti_drvdata *drvdata = cti_cpu_drvdata[cpu];
+
+ if (!drvdata)
+ return 0;
+
+ cti_cpuhp_enable_hw(drvdata);
+ return 0;
+}
+
+static int cti_dying_cpu(unsigned int cpu)
+{
+ struct cti_drvdata *drvdata = cti_cpu_drvdata[cpu];
+
+ if (!drvdata)
+ return 0;
+
+ spin_lock(&drvdata->spinlock);
+ drvdata->config.hw_powered = false;
+ coresight_disclaim_device(drvdata->base);
+ spin_unlock(&drvdata->spinlock);
+ return 0;
+}
+
+/* release PM registrations */
+static void cti_pm_release(struct cti_drvdata *drvdata)
+{
+ if (drvdata->ctidev.cpu >= 0) {
+ if (--nr_cti_cpu == 0) {
+ cpu_pm_unregister_notifier(&cti_cpu_pm_nb);
+
+ cpuhp_remove_state_nocalls(
+ CPUHP_AP_ARM_CORESIGHT_CTI_STARTING);
}
+ cti_cpu_drvdata[drvdata->ctidev.cpu] = NULL;
}
}
@@ -578,12 +776,12 @@ int cti_disable(struct coresight_device *csdev)
return cti_disable_hw(drvdata);
}
-const struct coresight_ops_ect cti_ops_ect = {
+static const struct coresight_ops_ect cti_ops_ect = {
.enable = cti_enable,
.disable = cti_disable,
};
-const struct coresight_ops cti_ops = {
+static const struct coresight_ops cti_ops = {
.ect_ops = &cti_ops_ect,
};
@@ -598,6 +796,7 @@ static void cti_device_release(struct device *dev)
mutex_lock(&ect_mutex);
cti_remove_conn_xrefs(drvdata);
+ cti_pm_release(drvdata);
/* remove from the list */
list_for_each_entry_safe(ect_item, ect_tmp, &ect_net, node) {
@@ -673,6 +872,24 @@ static int cti_probe(struct amba_device *adev, const struct amba_id *id)
goto err_out;
}
+ /* setup CPU power management handling for CPU bound CTI devices. */
+ if (drvdata->ctidev.cpu >= 0) {
+ cti_cpu_drvdata[drvdata->ctidev.cpu] = drvdata;
+ if (!nr_cti_cpu++) {
+ cpus_read_lock();
+ ret = cpuhp_setup_state_nocalls_cpuslocked(
+ CPUHP_AP_ARM_CORESIGHT_CTI_STARTING,
+ "arm/coresight_cti:starting",
+ cti_starting_cpu, cti_dying_cpu);
+
+ if (!ret)
+ ret = cpu_pm_register_notifier(&cti_cpu_pm_nb);
+ cpus_read_unlock();
+ if (ret)
+ goto err_out;
+ }
+ }
+
/* create dynamic attributes for connections */
ret = cti_create_cons_sysfs(dev, drvdata);
if (ret) {
@@ -711,6 +928,7 @@ static int cti_probe(struct amba_device *adev, const struct amba_id *id)
return 0;
err_out:
+ cti_pm_release(drvdata);
return ret;
}
diff --git a/drivers/hwtracing/coresight/coresight-cti.h b/drivers/hwtracing/coresight/coresight-cti.h
index 004df3ab9dd0..acf7b545e6b9 100644
--- a/drivers/hwtracing/coresight/coresight-cti.h
+++ b/drivers/hwtracing/coresight/coresight-cti.h
@@ -7,8 +7,14 @@
#ifndef _CORESIGHT_CORESIGHT_CTI_H
#define _CORESIGHT_CORESIGHT_CTI_H
-#include <asm/local.h>
+#include <linux/coresight.h>
+#include <linux/device.h>
+#include <linux/fwnode.h>
+#include <linux/list.h>
#include <linux/spinlock.h>
+#include <linux/sysfs.h>
+#include <linux/types.h>
+
#include "coresight-priv.h"
/*
diff --git a/drivers/hwtracing/coresight/coresight-etb10.c b/drivers/hwtracing/coresight/coresight-etb10.c
index 3810290e6d07..03e3f2590191 100644
--- a/drivers/hwtracing/coresight/coresight-etb10.c
+++ b/drivers/hwtracing/coresight/coresight-etb10.c
@@ -717,7 +717,7 @@ static const struct attribute_group coresight_etb_mgmt_group = {
.name = "mgmt",
};
-const struct attribute_group *coresight_etb_groups[] = {
+static const struct attribute_group *coresight_etb_groups[] = {
&coresight_etb_group,
&coresight_etb_mgmt_group,
NULL,
diff --git a/drivers/hwtracing/coresight/coresight-etm3x.c b/drivers/hwtracing/coresight/coresight-etm3x.c
index e2cb6873c3f2..bf22dcfd3327 100644
--- a/drivers/hwtracing/coresight/coresight-etm3x.c
+++ b/drivers/hwtracing/coresight/coresight-etm3x.c
@@ -504,7 +504,7 @@ static int etm_enable_perf(struct coresight_device *csdev,
static int etm_enable_sysfs(struct coresight_device *csdev)
{
struct etm_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
- struct etm_enable_arg arg = { 0 };
+ struct etm_enable_arg arg = { };
int ret;
spin_lock(&drvdata->spinlock);
diff --git a/drivers/hwtracing/coresight/coresight-etm4x-sysfs.c b/drivers/hwtracing/coresight/coresight-etm4x-sysfs.c
index ce41482431f9..b673e738bc9a 100644
--- a/drivers/hwtracing/coresight/coresight-etm4x-sysfs.c
+++ b/drivers/hwtracing/coresight/coresight-etm4x-sysfs.c
@@ -205,7 +205,7 @@ static ssize_t reset_store(struct device *dev,
* started state. ARM recommends start-stop logic is set before
* each trace run.
*/
- config->vinst_ctrl |= BIT(0);
+ config->vinst_ctrl = BIT(0);
if (drvdata->nr_addr_cmp == true) {
config->mode |= ETM_MODE_VIEWINST_STARTSTOP;
/* SSSTATUS, bit[9] */
diff --git a/drivers/hwtracing/coresight/coresight-etm4x.c b/drivers/hwtracing/coresight/coresight-etm4x.c
index a90d757f7043..747afc875f91 100644
--- a/drivers/hwtracing/coresight/coresight-etm4x.c
+++ b/drivers/hwtracing/coresight/coresight-etm4x.c
@@ -412,7 +412,7 @@ out:
static int etm4_enable_sysfs(struct coresight_device *csdev)
{
struct etmv4_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
- struct etm4_enable_arg arg = { 0 };
+ struct etm4_enable_arg arg = { };
int ret;
spin_lock(&drvdata->spinlock);
@@ -791,7 +791,7 @@ static void etm4_set_default_config(struct etmv4_config *config)
config->ts_ctrl = 0x0;
/* TRCVICTLR::EVENT = 0x01, select the always on logic */
- config->vinst_ctrl |= BIT(0);
+ config->vinst_ctrl = BIT(0);
}
static u64 etm4_get_ns_access_type(struct etmv4_config *config)
@@ -894,17 +894,8 @@ static void etm4_set_start_stop_filter(struct etmv4_config *config,
static void etm4_set_default_filter(struct etmv4_config *config)
{
- u64 start, stop;
-
- /*
- * Configure address range comparator '0' to encompass all
- * possible addresses.
- */
- start = 0x0;
- stop = ~0x0;
-
- etm4_set_comparator_filter(config, start, stop,
- ETM_DEFAULT_ADDR_COMP);
+ /* Trace everything 'default' filter achieved by no filtering */
+ config->viiectlr = 0x0;
/*
* TRCVICTLR::SSSTATUS == 1, the start-stop logic is
@@ -925,11 +916,9 @@ static void etm4_set_default(struct etmv4_config *config)
/*
* Make default initialisation trace everything
*
- * Select the "always true" resource selector on the
- * "Enablign Event" line and configure address range comparator
- * '0' to trace all the possible address range. From there
- * configure the "include/exclude" engine to include address
- * range comparator '0'.
+ * This is done by a minimum default config sufficient to enable
+ * full instruction trace - with a default filter for trace all
+ * achieved by having no filtering.
*/
etm4_set_default_config(config);
etm4_set_default_filter(config);
@@ -1527,6 +1516,7 @@ static int etm4_probe(struct amba_device *adev, const struct amba_id *id)
return 0;
err_arch_supported:
+ etmdrvdata[drvdata->cpu] = NULL;
if (--etm4_count == 0) {
etm4_cpu_pm_unregister();
@@ -1552,10 +1542,13 @@ static const struct amba_id etm4_ids[] = {
CS_AMBA_ID(0x000bb95a), /* Cortex-A72 */
CS_AMBA_ID(0x000bb959), /* Cortex-A73 */
CS_AMBA_UCI_ID(0x000bb9da, uci_id_etm4),/* Cortex-A35 */
+ CS_AMBA_UCI_ID(0x000bbd0c, uci_id_etm4),/* Neoverse N1 */
CS_AMBA_UCI_ID(0x000f0205, uci_id_etm4),/* Qualcomm Kryo */
CS_AMBA_UCI_ID(0x000f0211, uci_id_etm4),/* Qualcomm Kryo */
- CS_AMBA_ID(0x000bb802), /* Qualcomm Kryo 385 Cortex-A55 */
- CS_AMBA_ID(0x000bb803), /* Qualcomm Kryo 385 Cortex-A75 */
+ CS_AMBA_UCI_ID(0x000bb802, uci_id_etm4),/* Qualcomm Kryo 385 Cortex-A55 */
+ CS_AMBA_UCI_ID(0x000bb803, uci_id_etm4),/* Qualcomm Kryo 385 Cortex-A75 */
+ CS_AMBA_UCI_ID(0x000bb805, uci_id_etm4),/* Qualcomm Kryo 4XX Cortex-A55 */
+ CS_AMBA_UCI_ID(0x000bb804, uci_id_etm4),/* Qualcomm Kryo 4XX Cortex-A76 */
CS_AMBA_UCI_ID(0x000cc0af, uci_id_etm4),/* Marvell ThunderX2 */
{},
};
diff --git a/drivers/hwtracing/coresight/coresight-platform.c b/drivers/hwtracing/coresight/coresight-platform.c
index 43418a2126ff..e4912abda3aa 100644
--- a/drivers/hwtracing/coresight/coresight-platform.c
+++ b/drivers/hwtracing/coresight/coresight-platform.c
@@ -87,6 +87,7 @@ static void of_coresight_get_ports_legacy(const struct device_node *node,
int *nr_inport, int *nr_outport)
{
struct device_node *ep = NULL;
+ struct of_endpoint endpoint;
int in = 0, out = 0;
do {
@@ -94,10 +95,16 @@ static void of_coresight_get_ports_legacy(const struct device_node *node,
if (!ep)
break;
- if (of_coresight_legacy_ep_is_input(ep))
- in++;
- else
- out++;
+ if (of_graph_parse_endpoint(ep, &endpoint))
+ continue;
+
+ if (of_coresight_legacy_ep_is_input(ep)) {
+ in = (endpoint.port + 1 > in) ?
+ endpoint.port + 1 : in;
+ } else {
+ out = (endpoint.port + 1) > out ?
+ endpoint.port + 1 : out;
+ }
} while (ep);
@@ -137,9 +144,16 @@ of_coresight_count_ports(struct device_node *port_parent)
{
int i = 0;
struct device_node *ep = NULL;
+ struct of_endpoint endpoint;
+
+ while ((ep = of_graph_get_next_endpoint(port_parent, ep))) {
+ /* Defer error handling to parsing */
+ if (of_graph_parse_endpoint(ep, &endpoint))
+ continue;
+ if (endpoint.port + 1 > i)
+ i = endpoint.port + 1;
+ }
- while ((ep = of_graph_get_next_endpoint(port_parent, ep)))
- i++;
return i;
}
@@ -191,14 +205,12 @@ static int of_coresight_get_cpu(struct device *dev)
* Parses the local port, remote device name and the remote port.
*
* Returns :
- * 1 - If the parsing is successful and a connection record
- * was created for an output connection.
* 0 - If the parsing completed without any fatal errors.
* -Errno - Fatal error, abort the scanning.
*/
static int of_coresight_parse_endpoint(struct device *dev,
struct device_node *ep,
- struct coresight_connection *conn)
+ struct coresight_platform_data *pdata)
{
int ret = 0;
struct of_endpoint endpoint, rendpoint;
@@ -206,6 +218,7 @@ static int of_coresight_parse_endpoint(struct device *dev,
struct device_node *rep = NULL;
struct device *rdev = NULL;
struct fwnode_handle *rdev_fwnode;
+ struct coresight_connection *conn;
do {
/* Parse the local port details */
@@ -232,6 +245,13 @@ static int of_coresight_parse_endpoint(struct device *dev,
break;
}
+ conn = &pdata->conns[endpoint.port];
+ if (conn->child_fwnode) {
+ dev_warn(dev, "Duplicate output port %d\n",
+ endpoint.port);
+ ret = -EINVAL;
+ break;
+ }
conn->outport = endpoint.port;
/*
* Hold the refcount to the target device. This could be
@@ -244,7 +264,6 @@ static int of_coresight_parse_endpoint(struct device *dev,
conn->child_fwnode = fwnode_handle_get(rdev_fwnode);
conn->child_port = rendpoint.port;
/* Connection record updated */
- ret = 1;
} while (0);
of_node_put(rparent);
@@ -258,7 +277,6 @@ static int of_get_coresight_platform_data(struct device *dev,
struct coresight_platform_data *pdata)
{
int ret = 0;
- struct coresight_connection *conn;
struct device_node *ep = NULL;
const struct device_node *parent = NULL;
bool legacy_binding = false;
@@ -287,8 +305,6 @@ static int of_get_coresight_platform_data(struct device *dev,
dev_warn_once(dev, "Uses obsolete Coresight DT bindings\n");
}
- conn = pdata->conns;
-
/* Iterate through each output port to discover topology */
while ((ep = of_graph_get_next_endpoint(parent, ep))) {
/*
@@ -300,15 +316,9 @@ static int of_get_coresight_platform_data(struct device *dev,
if (legacy_binding && of_coresight_legacy_ep_is_input(ep))
continue;
- ret = of_coresight_parse_endpoint(dev, ep, conn);
- switch (ret) {
- case 1:
- conn++; /* Fall through */
- case 0:
- break;
- default:
+ ret = of_coresight_parse_endpoint(dev, ep, pdata);
+ if (ret)
return ret;
- }
}
return 0;
@@ -501,7 +511,7 @@ static inline bool acpi_validate_dsd_graph(const union acpi_object *graph)
}
/* acpi_get_dsd_graph - Find the _DSD Graph property for the given device. */
-const union acpi_object *
+static const union acpi_object *
acpi_get_dsd_graph(struct acpi_device *adev)
{
int i;
@@ -564,7 +574,7 @@ acpi_validate_coresight_graph(const union acpi_object *cs_graph)
* Returns the pointer to the CoreSight Graph Package when found. Otherwise
* returns NULL.
*/
-const union acpi_object *
+static const union acpi_object *
acpi_get_coresight_graph(struct acpi_device *adev)
{
const union acpi_object *graph_list, *graph;
@@ -647,6 +657,16 @@ static int acpi_coresight_parse_link(struct acpi_device *adev,
* coresight_remove_match().
*/
conn->child_fwnode = fwnode_handle_get(&r_adev->fwnode);
+ } else if (dir == ACPI_CORESIGHT_LINK_SLAVE) {
+ /*
+ * We are only interested in the port number
+ * for the input ports at this component.
+ * Store the port number in child_port.
+ */
+ conn->child_port = fields[0].integer.value;
+ } else {
+ /* Invalid direction */
+ return -EINVAL;
}
return dir;
@@ -692,10 +712,20 @@ static int acpi_coresight_parse_graph(struct acpi_device *adev,
return dir;
if (dir == ACPI_CORESIGHT_LINK_MASTER) {
- pdata->nr_outport++;
+ if (ptr->outport > pdata->nr_outport)
+ pdata->nr_outport = ptr->outport;
ptr++;
} else {
- pdata->nr_inport++;
+ WARN_ON(pdata->nr_inport == ptr->child_port);
+ /*
+ * We do not track input port connections for a device.
+ * However we need the highest port number described,
+ * which can be recorded now and reuse this connection
+ * record for an output connection. Hence, do not move
+ * the ptr for input connections
+ */
+ if (ptr->child_port > pdata->nr_inport)
+ pdata->nr_inport = ptr->child_port;
}
}
@@ -704,8 +734,13 @@ static int acpi_coresight_parse_graph(struct acpi_device *adev,
return rc;
/* Copy the connection information to the final location */
- for (i = 0; i < pdata->nr_outport; i++)
- pdata->conns[i] = conns[i];
+ for (i = 0; conns + i < ptr; i++) {
+ int port = conns[i].outport;
+
+ /* Duplicate output port */
+ WARN_ON(pdata->conns[port].child_fwnode);
+ pdata->conns[port] = conns[i];
+ }
devm_kfree(&adev->dev, conns);
return 0;
@@ -822,7 +857,7 @@ coresight_get_platform_data(struct device *dev)
error:
if (!IS_ERR_OR_NULL(pdata))
/* Cleanup the connection information */
- coresight_release_platform_data(pdata);
+ coresight_release_platform_data(NULL, pdata);
return ERR_PTR(ret);
}
EXPORT_SYMBOL_GPL(coresight_get_platform_data);
diff --git a/drivers/hwtracing/coresight/coresight-priv.h b/drivers/hwtracing/coresight/coresight-priv.h
index 890f9a5c97c6..36c943ae94d5 100644
--- a/drivers/hwtracing/coresight/coresight-priv.h
+++ b/drivers/hwtracing/coresight/coresight-priv.h
@@ -153,6 +153,15 @@ struct coresight_device *coresight_get_sink_by_id(u32 id);
struct list_head *coresight_build_path(struct coresight_device *csdev,
struct coresight_device *sink);
void coresight_release_path(struct list_head *path);
+int coresight_add_sysfs_link(struct coresight_sysfs_link *info);
+void coresight_remove_sysfs_link(struct coresight_sysfs_link *info);
+int coresight_create_conns_sysfs_group(struct coresight_device *csdev);
+void coresight_remove_conns_sysfs_group(struct coresight_device *csdev);
+int coresight_make_links(struct coresight_device *orig,
+ struct coresight_connection *conn,
+ struct coresight_device *target);
+void coresight_remove_links(struct coresight_device *orig,
+ struct coresight_connection *conn);
#ifdef CONFIG_CORESIGHT_SOURCE_ETM3X
extern int etm_readl_cp14(u32 off, unsigned int *val);
@@ -206,12 +215,16 @@ cti_remove_assoc_from_csdev(struct coresight_device *csdev) {}
/* extract the data value from a UCI structure given amba_id pointer. */
static inline void *coresight_get_uci_data(const struct amba_id *id)
{
- if (id->data)
- return ((struct amba_cs_uci_id *)(id->data))->data;
- return 0;
+ struct amba_cs_uci_id *uci_id = id->data;
+
+ if (!uci_id)
+ return NULL;
+
+ return uci_id->data;
}
-void coresight_release_platform_data(struct coresight_platform_data *pdata);
+void coresight_release_platform_data(struct coresight_device *csdev,
+ struct coresight_platform_data *pdata);
struct coresight_device *
coresight_find_csdev_by_fwnode(struct fwnode_handle *r_fwnode);
void coresight_set_assoc_ectdev_mutex(struct coresight_device *csdev,
diff --git a/drivers/hwtracing/coresight/coresight-sysfs.c b/drivers/hwtracing/coresight/coresight-sysfs.c
new file mode 100644
index 000000000000..82afeaf2ccc4
--- /dev/null
+++ b/drivers/hwtracing/coresight/coresight-sysfs.c
@@ -0,0 +1,204 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2019, Linaro Limited, All rights reserved.
+ * Author: Mike Leach <mike.leach@linaro.org>
+ */
+
+#include <linux/device.h>
+#include <linux/kernel.h>
+
+#include "coresight-priv.h"
+
+/*
+ * Connections group - links attribute.
+ * Count of created links between coresight components in the group.
+ */
+static ssize_t nr_links_show(struct device *dev,
+ struct device_attribute *attr,
+ char *buf)
+{
+ struct coresight_device *csdev = to_coresight_device(dev);
+
+ return sprintf(buf, "%d\n", csdev->nr_links);
+}
+static DEVICE_ATTR_RO(nr_links);
+
+static struct attribute *coresight_conns_attrs[] = {
+ &dev_attr_nr_links.attr,
+ NULL,
+};
+
+static struct attribute_group coresight_conns_group = {
+ .attrs = coresight_conns_attrs,
+ .name = "connections",
+};
+
+/*
+ * Create connections group for CoreSight devices.
+ * This group will then be used to collate the sysfs links between
+ * devices.
+ */
+int coresight_create_conns_sysfs_group(struct coresight_device *csdev)
+{
+ int ret = 0;
+
+ if (!csdev)
+ return -EINVAL;
+
+ ret = sysfs_create_group(&csdev->dev.kobj, &coresight_conns_group);
+ if (ret)
+ return ret;
+
+ csdev->has_conns_grp = true;
+ return ret;
+}
+
+void coresight_remove_conns_sysfs_group(struct coresight_device *csdev)
+{
+ if (!csdev)
+ return;
+
+ if (csdev->has_conns_grp) {
+ sysfs_remove_group(&csdev->dev.kobj, &coresight_conns_group);
+ csdev->has_conns_grp = false;
+ }
+}
+
+int coresight_add_sysfs_link(struct coresight_sysfs_link *info)
+{
+ int ret = 0;
+
+ if (!info)
+ return -EINVAL;
+ if (!info->orig || !info->target ||
+ !info->orig_name || !info->target_name)
+ return -EINVAL;
+ if (!info->orig->has_conns_grp || !info->target->has_conns_grp)
+ return -EINVAL;
+
+ /* first link orig->target */
+ ret = sysfs_add_link_to_group(&info->orig->dev.kobj,
+ coresight_conns_group.name,
+ &info->target->dev.kobj,
+ info->orig_name);
+ if (ret)
+ return ret;
+
+ /* second link target->orig */
+ ret = sysfs_add_link_to_group(&info->target->dev.kobj,
+ coresight_conns_group.name,
+ &info->orig->dev.kobj,
+ info->target_name);
+
+ /* error in second link - remove first - otherwise inc counts */
+ if (ret) {
+ sysfs_remove_link_from_group(&info->orig->dev.kobj,
+ coresight_conns_group.name,
+ info->orig_name);
+ } else {
+ info->orig->nr_links++;
+ info->target->nr_links++;
+ }
+
+ return ret;
+}
+
+void coresight_remove_sysfs_link(struct coresight_sysfs_link *info)
+{
+ if (!info)
+ return;
+ if (!info->orig || !info->target ||
+ !info->orig_name || !info->target_name)
+ return;
+
+ sysfs_remove_link_from_group(&info->orig->dev.kobj,
+ coresight_conns_group.name,
+ info->orig_name);
+
+ sysfs_remove_link_from_group(&info->target->dev.kobj,
+ coresight_conns_group.name,
+ info->target_name);
+
+ info->orig->nr_links--;
+ info->target->nr_links--;
+}
+
+/*
+ * coresight_make_links: Make a link for a connection from a @orig
+ * device to @target, represented by @conn.
+ *
+ * e.g, for devOrig[output_X] -> devTarget[input_Y] is represented
+ * as two symbolic links :
+ *
+ * /sys/.../devOrig/out:X -> /sys/.../devTarget/
+ * /sys/.../devTarget/in:Y -> /sys/.../devOrig/
+ *
+ * The link names are allocated for a device where it appears. i.e, the
+ * "out" link on the master and "in" link on the slave device.
+ * The link info is stored in the connection record for avoiding
+ * the reconstruction of names for removal.
+ */
+int coresight_make_links(struct coresight_device *orig,
+ struct coresight_connection *conn,
+ struct coresight_device *target)
+{
+ int ret = -ENOMEM;
+ char *outs = NULL, *ins = NULL;
+ struct coresight_sysfs_link *link = NULL;
+
+ do {
+ outs = devm_kasprintf(&orig->dev, GFP_KERNEL,
+ "out:%d", conn->outport);
+ if (!outs)
+ break;
+ ins = devm_kasprintf(&target->dev, GFP_KERNEL,
+ "in:%d", conn->child_port);
+ if (!ins)
+ break;
+ link = devm_kzalloc(&orig->dev,
+ sizeof(struct coresight_sysfs_link),
+ GFP_KERNEL);
+ if (!link)
+ break;
+
+ link->orig = orig;
+ link->target = target;
+ link->orig_name = outs;
+ link->target_name = ins;
+
+ ret = coresight_add_sysfs_link(link);
+ if (ret)
+ break;
+
+ conn->link = link;
+
+ /*
+ * Install the device connection. This also indicates that
+ * the links are operational on both ends.
+ */
+ conn->child_dev = target;
+ return 0;
+ } while (0);
+
+ return ret;
+}
+
+/*
+ * coresight_remove_links: Remove the sysfs links for a given connection @conn,
+ * from @orig device to @target device. See coresight_make_links() for more
+ * details.
+ */
+void coresight_remove_links(struct coresight_device *orig,
+ struct coresight_connection *conn)
+{
+ if (!orig || !conn->link)
+ return;
+
+ coresight_remove_sysfs_link(conn->link);
+
+ devm_kfree(&conn->child_dev->dev, conn->link->target_name);
+ devm_kfree(&orig->dev, conn->link->orig_name);
+ devm_kfree(&orig->dev, conn->link);
+ conn->link = NULL;
+ conn->child_dev = NULL;
+}
diff --git a/drivers/hwtracing/coresight/coresight-tmc-etf.c b/drivers/hwtracing/coresight/coresight-tmc-etf.c
index d0cc3985b72a..36cce2bfb744 100644
--- a/drivers/hwtracing/coresight/coresight-tmc-etf.c
+++ b/drivers/hwtracing/coresight/coresight-tmc-etf.c
@@ -596,13 +596,6 @@ int tmc_read_prepare_etb(struct tmc_drvdata *drvdata)
goto out;
}
- /* There is no point in reading a TMC in HW FIFO mode */
- mode = readl_relaxed(drvdata->base + TMC_MODE);
- if (mode != TMC_MODE_CIRCULAR_BUFFER) {
- ret = -EINVAL;
- goto out;
- }
-
/* Don't interfere if operated from Perf */
if (drvdata->mode == CS_MODE_PERF) {
ret = -EINVAL;
@@ -616,8 +609,15 @@ int tmc_read_prepare_etb(struct tmc_drvdata *drvdata)
}
/* Disable the TMC if need be */
- if (drvdata->mode == CS_MODE_SYSFS)
+ if (drvdata->mode == CS_MODE_SYSFS) {
+ /* There is no point in reading a TMC in HW FIFO mode */
+ mode = readl_relaxed(drvdata->base + TMC_MODE);
+ if (mode != TMC_MODE_CIRCULAR_BUFFER) {
+ ret = -EINVAL;
+ goto out;
+ }
__tmc_etb_disable_hw(drvdata);
+ }
drvdata->reading = true;
out:
diff --git a/drivers/hwtracing/coresight/coresight-tmc.c b/drivers/hwtracing/coresight/coresight-tmc.c
index 1cf82fa58289..39fba1d16e6e 100644
--- a/drivers/hwtracing/coresight/coresight-tmc.c
+++ b/drivers/hwtracing/coresight/coresight-tmc.c
@@ -361,7 +361,7 @@ static const struct attribute_group coresight_tmc_mgmt_group = {
.name = "mgmt",
};
-const struct attribute_group *coresight_tmc_groups[] = {
+static const struct attribute_group *coresight_tmc_groups[] = {
&coresight_tmc_group,
&coresight_tmc_mgmt_group,
NULL,
diff --git a/drivers/hwtracing/coresight/coresight.c b/drivers/hwtracing/coresight/coresight.c
index c71553c09f8e..f3efbb3b2b4d 100644
--- a/drivers/hwtracing/coresight/coresight.c
+++ b/drivers/hwtracing/coresight/coresight.c
@@ -1031,7 +1031,7 @@ static void coresight_device_release(struct device *dev)
static int coresight_orphan_match(struct device *dev, void *data)
{
- int i;
+ int i, ret = 0;
bool still_orphan = false;
struct coresight_device *csdev, *i_csdev;
struct coresight_connection *conn;
@@ -1053,49 +1053,62 @@ static int coresight_orphan_match(struct device *dev, void *data)
for (i = 0; i < i_csdev->pdata->nr_outport; i++) {
conn = &i_csdev->pdata->conns[i];
+ /* Skip the port if FW doesn't describe it */
+ if (!conn->child_fwnode)
+ continue;
/* We have found at least one orphan connection */
if (conn->child_dev == NULL) {
/* Does it match this newly added device? */
- if (conn->child_fwnode == csdev->dev.fwnode)
- conn->child_dev = csdev;
- else
+ if (conn->child_fwnode == csdev->dev.fwnode) {
+ ret = coresight_make_links(i_csdev,
+ conn, csdev);
+ if (ret)
+ return ret;
+ } else {
/* This component still has an orphan */
still_orphan = true;
+ }
}
}
i_csdev->orphan = still_orphan;
/*
- * Returning '0' ensures that all known component on the
- * bus will be checked.
+ * Returning '0' in case we didn't encounter any error,
+ * ensures that all known component on the bus will be checked.
*/
return 0;
}
-static void coresight_fixup_orphan_conns(struct coresight_device *csdev)
+static int coresight_fixup_orphan_conns(struct coresight_device *csdev)
{
- /*
- * No need to check for a return value as orphan connection(s)
- * are hooked-up with each newly added component.
- */
- bus_for_each_dev(&coresight_bustype, NULL,
+ return bus_for_each_dev(&coresight_bustype, NULL,
csdev, coresight_orphan_match);
}
-static void coresight_fixup_device_conns(struct coresight_device *csdev)
+static int coresight_fixup_device_conns(struct coresight_device *csdev)
{
- int i;
+ int i, ret = 0;
for (i = 0; i < csdev->pdata->nr_outport; i++) {
struct coresight_connection *conn = &csdev->pdata->conns[i];
+ if (!conn->child_fwnode)
+ continue;
conn->child_dev =
coresight_find_csdev_by_fwnode(conn->child_fwnode);
- if (!conn->child_dev)
+ if (conn->child_dev) {
+ ret = coresight_make_links(csdev, conn,
+ conn->child_dev);
+ if (ret)
+ break;
+ } else {
csdev->orphan = true;
+ }
}
+
+ return 0;
}
static int coresight_remove_match(struct device *dev, void *data)
@@ -1118,12 +1131,12 @@ static int coresight_remove_match(struct device *dev, void *data)
for (i = 0; i < iterator->pdata->nr_outport; i++) {
conn = &iterator->pdata->conns[i];
- if (conn->child_dev == NULL)
+ if (conn->child_dev == NULL || conn->child_fwnode == NULL)
continue;
if (csdev->dev.fwnode == conn->child_fwnode) {
iterator->orphan = true;
- conn->child_dev = NULL;
+ coresight_remove_links(iterator, conn);
/*
* Drop the reference to the handle for the remote
* device acquired in parsing the connections from
@@ -1213,16 +1226,27 @@ postcore_initcall(coresight_init);
* coresight_release_platform_data: Release references to the devices connected
* to the output port of this device.
*/
-void coresight_release_platform_data(struct coresight_platform_data *pdata)
+void coresight_release_platform_data(struct coresight_device *csdev,
+ struct coresight_platform_data *pdata)
{
int i;
+ struct coresight_connection *conns = pdata->conns;
for (i = 0; i < pdata->nr_outport; i++) {
- if (pdata->conns[i].child_fwnode) {
- fwnode_handle_put(pdata->conns[i].child_fwnode);
+ /* If we have made the links, remove them now */
+ if (csdev && conns[i].child_dev)
+ coresight_remove_links(csdev, &conns[i]);
+ /*
+ * Drop the refcount and clear the handle as this device
+ * is going away
+ */
+ if (conns[i].child_fwnode) {
+ fwnode_handle_put(conns[i].child_fwnode);
pdata->conns[i].child_fwnode = NULL;
}
}
+ if (csdev)
+ coresight_remove_conns_sysfs_group(csdev);
}
struct coresight_device *coresight_register(struct coresight_desc *desc)
@@ -1304,11 +1328,19 @@ struct coresight_device *coresight_register(struct coresight_desc *desc)
mutex_lock(&coresight_mutex);
- coresight_fixup_device_conns(csdev);
- coresight_fixup_orphan_conns(csdev);
- cti_add_assoc_to_csdev(csdev);
+ ret = coresight_create_conns_sysfs_group(csdev);
+ if (!ret)
+ ret = coresight_fixup_device_conns(csdev);
+ if (!ret)
+ ret = coresight_fixup_orphan_conns(csdev);
+ if (!ret)
+ cti_add_assoc_to_csdev(csdev);
mutex_unlock(&coresight_mutex);
+ if (ret) {
+ coresight_unregister(csdev);
+ return ERR_PTR(ret);
+ }
return csdev;
@@ -1316,7 +1348,7 @@ err_free_csdev:
kfree(csdev);
err_out:
/* Cleanup the connection information */
- coresight_release_platform_data(desc->pdata);
+ coresight_release_platform_data(NULL, desc->pdata);
return ERR_PTR(ret);
}
EXPORT_SYMBOL_GPL(coresight_register);
@@ -1326,7 +1358,7 @@ void coresight_unregister(struct coresight_device *csdev)
etm_perf_del_symlink_sink(csdev);
/* Remove references of that device in the topology */
coresight_remove_conns(csdev);
- coresight_release_platform_data(csdev->pdata);
+ coresight_release_platform_data(csdev, csdev->pdata);
device_unregister(&csdev->dev);
}
EXPORT_SYMBOL_GPL(coresight_unregister);
diff --git a/drivers/interconnect/Kconfig b/drivers/interconnect/Kconfig
index bfa4ca3ab7a9..5b7204ee2eb2 100644
--- a/drivers/interconnect/Kconfig
+++ b/drivers/interconnect/Kconfig
@@ -1,6 +1,6 @@
# SPDX-License-Identifier: GPL-2.0-only
menuconfig INTERCONNECT
- tristate "On-Chip Interconnect management support"
+ bool "On-Chip Interconnect management support"
help
Support for management of the on-chip interconnects.
@@ -11,6 +11,7 @@ menuconfig INTERCONNECT
if INTERCONNECT
+source "drivers/interconnect/imx/Kconfig"
source "drivers/interconnect/qcom/Kconfig"
endif
diff --git a/drivers/interconnect/Makefile b/drivers/interconnect/Makefile
index 725029ae7a2c..4825c287ca13 100644
--- a/drivers/interconnect/Makefile
+++ b/drivers/interconnect/Makefile
@@ -4,4 +4,5 @@ CFLAGS_core.o := -I$(src)
icc-core-objs := core.o
obj-$(CONFIG_INTERCONNECT) += icc-core.o
+obj-$(CONFIG_INTERCONNECT_IMX) += imx/
obj-$(CONFIG_INTERCONNECT_QCOM) += qcom/
diff --git a/drivers/interconnect/core.c b/drivers/interconnect/core.c
index 2c6515e3ecf1..ece2a579a9b0 100644
--- a/drivers/interconnect/core.c
+++ b/drivers/interconnect/core.c
@@ -158,6 +158,7 @@ static struct icc_path *path_init(struct device *dev, struct icc_node *dst,
hlist_add_head(&path->reqs[i].req_node, &node->req_list);
path->reqs[i].node = node;
path->reqs[i].dev = dev;
+ path->reqs[i].enabled = true;
/* reference to previous node was saved during path traversal */
node = node->reverse;
}
@@ -249,9 +250,12 @@ static int aggregate_requests(struct icc_node *node)
if (p->pre_aggregate)
p->pre_aggregate(node);
- hlist_for_each_entry(r, &node->req_list, req_node)
+ hlist_for_each_entry(r, &node->req_list, req_node) {
+ if (!r->enabled)
+ continue;
p->aggregate(node, r->tag, r->avg_bw, r->peak_bw,
&node->avg_bw, &node->peak_bw);
+ }
return 0;
}
@@ -350,10 +354,35 @@ static struct icc_node *of_icc_get_from_provider(struct of_phandle_args *spec)
return node;
}
+static void devm_icc_release(struct device *dev, void *res)
+{
+ icc_put(*(struct icc_path **)res);
+}
+
+struct icc_path *devm_of_icc_get(struct device *dev, const char *name)
+{
+ struct icc_path **ptr, *path;
+
+ ptr = devres_alloc(devm_icc_release, sizeof(**ptr), GFP_KERNEL);
+ if (!ptr)
+ return ERR_PTR(-ENOMEM);
+
+ path = of_icc_get(dev, name);
+ if (!IS_ERR(path)) {
+ *ptr = path;
+ devres_add(dev, ptr);
+ } else {
+ devres_free(ptr);
+ }
+
+ return path;
+}
+EXPORT_SYMBOL_GPL(devm_of_icc_get);
+
/**
- * of_icc_get() - get a path handle from a DT node based on name
+ * of_icc_get_by_index() - get a path handle from a DT node based on index
* @dev: device pointer for the consumer device
- * @name: interconnect path name
+ * @idx: interconnect path index
*
* This function will search for a path between two endpoints and return an
* icc_path handle on success. Use icc_put() to release constraints when they
@@ -365,13 +394,12 @@ static struct icc_node *of_icc_get_from_provider(struct of_phandle_args *spec)
* Return: icc_path pointer on success or ERR_PTR() on error. NULL is returned
* when the API is disabled or the "interconnects" DT property is missing.
*/
-struct icc_path *of_icc_get(struct device *dev, const char *name)
+struct icc_path *of_icc_get_by_index(struct device *dev, int idx)
{
- struct icc_path *path = ERR_PTR(-EPROBE_DEFER);
+ struct icc_path *path;
struct icc_node *src_node, *dst_node;
- struct device_node *np = NULL;
+ struct device_node *np;
struct of_phandle_args src_args, dst_args;
- int idx = 0;
int ret;
if (!dev || !dev->of_node)
@@ -391,12 +419,6 @@ struct icc_path *of_icc_get(struct device *dev, const char *name)
* lets support only global ids and extend this in the future if needed
* without breaking DT compatibility.
*/
- if (name) {
- idx = of_property_match_string(np, "interconnect-names", name);
- if (idx < 0)
- return ERR_PTR(idx);
- }
-
ret = of_parse_phandle_with_args(np, "interconnects",
"#interconnect-cells", idx * 2,
&src_args);
@@ -439,12 +461,8 @@ struct icc_path *of_icc_get(struct device *dev, const char *name)
return path;
}
- if (name)
- path->name = kstrdup_const(name, GFP_KERNEL);
- else
- path->name = kasprintf(GFP_KERNEL, "%s-%s",
- src_node->name, dst_node->name);
-
+ path->name = kasprintf(GFP_KERNEL, "%s-%s",
+ src_node->name, dst_node->name);
if (!path->name) {
kfree(path);
return ERR_PTR(-ENOMEM);
@@ -452,6 +470,53 @@ struct icc_path *of_icc_get(struct device *dev, const char *name)
return path;
}
+EXPORT_SYMBOL_GPL(of_icc_get_by_index);
+
+/**
+ * of_icc_get() - get a path handle from a DT node based on name
+ * @dev: device pointer for the consumer device
+ * @name: interconnect path name
+ *
+ * This function will search for a path between two endpoints and return an
+ * icc_path handle on success. Use icc_put() to release constraints when they
+ * are not needed anymore.
+ * If the interconnect API is disabled, NULL is returned and the consumer
+ * drivers will still build. Drivers are free to handle this specifically,
+ * but they don't have to.
+ *
+ * Return: icc_path pointer on success or ERR_PTR() on error. NULL is returned
+ * when the API is disabled or the "interconnects" DT property is missing.
+ */
+struct icc_path *of_icc_get(struct device *dev, const char *name)
+{
+ struct device_node *np;
+ int idx = 0;
+
+ if (!dev || !dev->of_node)
+ return ERR_PTR(-ENODEV);
+
+ np = dev->of_node;
+
+ /*
+ * When the consumer DT node do not have "interconnects" property
+ * return a NULL path to skip setting constraints.
+ */
+ if (!of_find_property(np, "interconnects", NULL))
+ return NULL;
+
+ /*
+ * We use a combination of phandle and specifier for endpoint. For now
+ * lets support only global ids and extend this in the future if needed
+ * without breaking DT compatibility.
+ */
+ if (name) {
+ idx = of_property_match_string(np, "interconnect-names", name);
+ if (idx < 0)
+ return ERR_PTR(idx);
+ }
+
+ return of_icc_get_by_index(dev, idx);
+}
EXPORT_SYMBOL_GPL(of_icc_get);
/**
@@ -546,6 +611,39 @@ int icc_set_bw(struct icc_path *path, u32 avg_bw, u32 peak_bw)
}
EXPORT_SYMBOL_GPL(icc_set_bw);
+static int __icc_enable(struct icc_path *path, bool enable)
+{
+ int i;
+
+ if (!path)
+ return 0;
+
+ if (WARN_ON(IS_ERR(path) || !path->num_nodes))
+ return -EINVAL;
+
+ mutex_lock(&icc_lock);
+
+ for (i = 0; i < path->num_nodes; i++)
+ path->reqs[i].enabled = enable;
+
+ mutex_unlock(&icc_lock);
+
+ return icc_set_bw(path, path->reqs[0].avg_bw,
+ path->reqs[0].peak_bw);
+}
+
+int icc_enable(struct icc_path *path)
+{
+ return __icc_enable(path, true);
+}
+EXPORT_SYMBOL_GPL(icc_enable);
+
+int icc_disable(struct icc_path *path)
+{
+ return __icc_enable(path, false);
+}
+EXPORT_SYMBOL_GPL(icc_disable);
+
/**
* icc_get() - return a handle for path between two endpoints
* @dev: the device requesting the path
@@ -908,12 +1006,7 @@ static int __init icc_init(void)
return 0;
}
-static void __exit icc_exit(void)
-{
- debugfs_remove_recursive(icc_debugfs_dir);
-}
-module_init(icc_init);
-module_exit(icc_exit);
+device_initcall(icc_init);
MODULE_AUTHOR("Georgi Djakov <georgi.djakov@linaro.org>");
MODULE_DESCRIPTION("Interconnect Driver Core");
diff --git a/drivers/interconnect/imx/Kconfig b/drivers/interconnect/imx/Kconfig
new file mode 100644
index 000000000000..be2928362bb7
--- /dev/null
+++ b/drivers/interconnect/imx/Kconfig
@@ -0,0 +1,17 @@
+config INTERCONNECT_IMX
+ tristate "i.MX interconnect drivers"
+ depends on ARCH_MXC || COMPILE_TEST
+ help
+ Generic interconnect drivers for i.MX SOCs
+
+config INTERCONNECT_IMX8MM
+ tristate "i.MX8MM interconnect driver"
+ depends on INTERCONNECT_IMX
+
+config INTERCONNECT_IMX8MN
+ tristate "i.MX8MN interconnect driver"
+ depends on INTERCONNECT_IMX
+
+config INTERCONNECT_IMX8MQ
+ tristate "i.MX8MQ interconnect driver"
+ depends on INTERCONNECT_IMX
diff --git a/drivers/interconnect/imx/Makefile b/drivers/interconnect/imx/Makefile
new file mode 100644
index 000000000000..21fd5233754f
--- /dev/null
+++ b/drivers/interconnect/imx/Makefile
@@ -0,0 +1,9 @@
+imx-interconnect-objs := imx.o
+imx8mm-interconnect-objs := imx8mm.o
+imx8mq-interconnect-objs := imx8mq.o
+imx8mn-interconnect-objs := imx8mn.o
+
+obj-$(CONFIG_INTERCONNECT_IMX) += imx-interconnect.o
+obj-$(CONFIG_INTERCONNECT_IMX8MM) += imx8mm-interconnect.o
+obj-$(CONFIG_INTERCONNECT_IMX8MQ) += imx8mq-interconnect.o
+obj-$(CONFIG_INTERCONNECT_IMX8MN) += imx8mn-interconnect.o
diff --git a/drivers/interconnect/imx/imx.c b/drivers/interconnect/imx/imx.c
new file mode 100644
index 000000000000..ac420f86008e
--- /dev/null
+++ b/drivers/interconnect/imx/imx.c
@@ -0,0 +1,284 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Interconnect framework driver for i.MX SoC
+ *
+ * Copyright (c) 2019, BayLibre
+ * Copyright (c) 2019-2020, NXP
+ * Author: Alexandre Bailon <abailon@baylibre.com>
+ * Author: Leonard Crestez <leonard.crestez@nxp.com>
+ */
+
+#include <linux/device.h>
+#include <linux/interconnect-provider.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/of_platform.h>
+#include <linux/platform_device.h>
+#include <linux/pm_qos.h>
+
+#include "imx.h"
+
+/* private icc_node data */
+struct imx_icc_node {
+ const struct imx_icc_node_desc *desc;
+ struct device *qos_dev;
+ struct dev_pm_qos_request qos_req;
+};
+
+static int imx_icc_node_set(struct icc_node *node)
+{
+ struct device *dev = node->provider->dev;
+ struct imx_icc_node *node_data = node->data;
+ u64 freq;
+
+ if (!node_data->qos_dev)
+ return 0;
+
+ freq = (node->avg_bw + node->peak_bw) * node_data->desc->adj->bw_mul;
+ do_div(freq, node_data->desc->adj->bw_div);
+ dev_dbg(dev, "node %s device %s avg_bw %ukBps peak_bw %ukBps min_freq %llukHz\n",
+ node->name, dev_name(node_data->qos_dev),
+ node->avg_bw, node->peak_bw, freq);
+
+ if (freq > S32_MAX) {
+ dev_err(dev, "%s can't request more than S32_MAX freq\n",
+ node->name);
+ return -ERANGE;
+ }
+
+ dev_pm_qos_update_request(&node_data->qos_req, freq);
+
+ return 0;
+}
+
+static int imx_icc_set(struct icc_node *src, struct icc_node *dst)
+{
+ return imx_icc_node_set(dst);
+}
+
+/* imx_icc_node_destroy() - Destroy an imx icc_node, including private data */
+static void imx_icc_node_destroy(struct icc_node *node)
+{
+ struct imx_icc_node *node_data = node->data;
+ int ret;
+
+ if (dev_pm_qos_request_active(&node_data->qos_req)) {
+ ret = dev_pm_qos_remove_request(&node_data->qos_req);
+ if (ret)
+ dev_warn(node->provider->dev,
+ "failed to remove qos request for %s\n",
+ dev_name(node_data->qos_dev));
+ }
+
+ put_device(node_data->qos_dev);
+ icc_node_del(node);
+ icc_node_destroy(node->id);
+}
+
+static int imx_icc_node_init_qos(struct icc_provider *provider,
+ struct icc_node *node)
+{
+ struct imx_icc_node *node_data = node->data;
+ const struct imx_icc_node_adj_desc *adj = node_data->desc->adj;
+ struct device *dev = provider->dev;
+ struct device_node *dn = NULL;
+ struct platform_device *pdev;
+
+ if (adj->main_noc) {
+ node_data->qos_dev = dev;
+ dev_dbg(dev, "icc node %s[%d] is main noc itself\n",
+ node->name, node->id);
+ } else {
+ dn = of_parse_phandle(dev->of_node, adj->phandle_name, 0);
+ if (!dn) {
+ dev_warn(dev, "Failed to parse %s\n",
+ adj->phandle_name);
+ return -ENODEV;
+ }
+ /* Allow scaling to be disabled on a per-node basis */
+ if (!dn || !of_device_is_available(dn)) {
+ dev_warn(dev, "Missing property %s, skip scaling %s\n",
+ adj->phandle_name, node->name);
+ return 0;
+ }
+
+ pdev = of_find_device_by_node(dn);
+ of_node_put(dn);
+ if (!pdev) {
+ dev_warn(dev, "node %s[%d] missing device for %pOF\n",
+ node->name, node->id, dn);
+ return -EPROBE_DEFER;
+ }
+ node_data->qos_dev = &pdev->dev;
+ dev_dbg(dev, "node %s[%d] has device node %pOF\n",
+ node->name, node->id, dn);
+ }
+
+ return dev_pm_qos_add_request(node_data->qos_dev,
+ &node_data->qos_req,
+ DEV_PM_QOS_MIN_FREQUENCY, 0);
+}
+
+static struct icc_node *imx_icc_node_add(struct icc_provider *provider,
+ const struct imx_icc_node_desc *node_desc)
+{
+ struct device *dev = provider->dev;
+ struct imx_icc_node *node_data;
+ struct icc_node *node;
+ int ret;
+
+ node = icc_node_create(node_desc->id);
+ if (IS_ERR(node)) {
+ dev_err(dev, "failed to create node %d\n", node_desc->id);
+ return node;
+ }
+
+ if (node->data) {
+ dev_err(dev, "already created node %s id=%d\n",
+ node_desc->name, node_desc->id);
+ return ERR_PTR(-EEXIST);
+ }
+
+ node_data = devm_kzalloc(dev, sizeof(*node_data), GFP_KERNEL);
+ if (!node_data) {
+ icc_node_destroy(node->id);
+ return ERR_PTR(-ENOMEM);
+ }
+
+ node->name = node_desc->name;
+ node->data = node_data;
+ node_data->desc = node_desc;
+ icc_node_add(node, provider);
+
+ if (node_desc->adj) {
+ ret = imx_icc_node_init_qos(provider, node);
+ if (ret < 0) {
+ imx_icc_node_destroy(node);
+ return ERR_PTR(ret);
+ }
+ }
+
+ return node;
+}
+
+static void imx_icc_unregister_nodes(struct icc_provider *provider)
+{
+ struct icc_node *node, *tmp;
+
+ list_for_each_entry_safe(node, tmp, &provider->nodes, node_list)
+ imx_icc_node_destroy(node);
+}
+
+static int imx_icc_register_nodes(struct icc_provider *provider,
+ const struct imx_icc_node_desc *descs,
+ int count)
+{
+ struct icc_onecell_data *provider_data = provider->data;
+ int ret;
+ int i;
+
+ for (i = 0; i < count; i++) {
+ struct icc_node *node;
+ const struct imx_icc_node_desc *node_desc = &descs[i];
+ size_t j;
+
+ node = imx_icc_node_add(provider, node_desc);
+ if (IS_ERR(node)) {
+ ret = PTR_ERR(node);
+ if (ret != -EPROBE_DEFER)
+ dev_err(provider->dev, "failed to add %s: %d\n",
+ node_desc->name, ret);
+ goto err;
+ }
+ provider_data->nodes[node->id] = node;
+
+ for (j = 0; j < node_desc->num_links; j++) {
+ ret = icc_link_create(node, node_desc->links[j]);
+ if (ret) {
+ dev_err(provider->dev, "failed to link node %d to %d: %d\n",
+ node->id, node_desc->links[j], ret);
+ goto err;
+ }
+ }
+ }
+
+ return 0;
+
+err:
+ imx_icc_unregister_nodes(provider);
+
+ return ret;
+}
+
+static int get_max_node_id(struct imx_icc_node_desc *nodes, int nodes_count)
+{
+ int i, ret = 0;
+
+ for (i = 0; i < nodes_count; ++i)
+ if (nodes[i].id > ret)
+ ret = nodes[i].id;
+
+ return ret;
+}
+
+int imx_icc_register(struct platform_device *pdev,
+ struct imx_icc_node_desc *nodes, int nodes_count)
+{
+ struct device *dev = &pdev->dev;
+ struct icc_onecell_data *data;
+ struct icc_provider *provider;
+ int max_node_id;
+ int ret;
+
+ /* icc_onecell_data is indexed by node_id, unlike nodes param */
+ max_node_id = get_max_node_id(nodes, nodes_count);
+ data = devm_kzalloc(dev, struct_size(data, nodes, max_node_id),
+ GFP_KERNEL);
+ if (!data)
+ return -ENOMEM;
+ data->num_nodes = max_node_id;
+
+ provider = devm_kzalloc(dev, sizeof(*provider), GFP_KERNEL);
+ if (!provider)
+ return -ENOMEM;
+ provider->set = imx_icc_set;
+ provider->aggregate = icc_std_aggregate;
+ provider->xlate = of_icc_xlate_onecell;
+ provider->data = data;
+ provider->dev = dev->parent;
+ platform_set_drvdata(pdev, provider);
+
+ ret = icc_provider_add(provider);
+ if (ret) {
+ dev_err(dev, "error adding interconnect provider: %d\n", ret);
+ return ret;
+ }
+
+ ret = imx_icc_register_nodes(provider, nodes, nodes_count);
+ if (ret)
+ goto provider_del;
+
+ return 0;
+
+provider_del:
+ icc_provider_del(provider);
+ return ret;
+}
+EXPORT_SYMBOL_GPL(imx_icc_register);
+
+int imx_icc_unregister(struct platform_device *pdev)
+{
+ struct icc_provider *provider = platform_get_drvdata(pdev);
+ int ret;
+
+ imx_icc_unregister_nodes(provider);
+
+ ret = icc_provider_del(provider);
+ if (ret)
+ return ret;
+
+ return 0;
+}
+EXPORT_SYMBOL_GPL(imx_icc_unregister);
+
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/interconnect/imx/imx.h b/drivers/interconnect/imx/imx.h
new file mode 100644
index 000000000000..75da51076c68
--- /dev/null
+++ b/drivers/interconnect/imx/imx.h
@@ -0,0 +1,61 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Interconnect framework driver for i.MX SoC
+ *
+ * Copyright (c) 2019, BayLibre
+ * Copyright (c) 2019-2020, NXP
+ * Author: Alexandre Bailon <abailon@baylibre.com>
+ * Author: Leonard Crestez <leonard.crestez@nxp.com>
+ */
+#ifndef __DRIVERS_INTERCONNECT_IMX_H
+#define __DRIVERS_INTERCONNECT_IMX_H
+
+#include <linux/kernel.h>
+
+#define IMX_ICC_MAX_LINKS 4
+
+/*
+ * struct imx_icc_node_adj - Describe a dynamic adjustable node
+ */
+struct imx_icc_node_adj_desc {
+ unsigned int bw_mul, bw_div;
+ const char *phandle_name;
+ bool main_noc;
+};
+
+/*
+ * struct imx_icc_node - Describe an interconnect node
+ * @name: name of the node
+ * @id: an unique id to identify the node
+ * @links: an array of slaves' node id
+ * @num_links: number of id defined in links
+ */
+struct imx_icc_node_desc {
+ const char *name;
+ u16 id;
+ u16 links[IMX_ICC_MAX_LINKS];
+ u16 num_links;
+ const struct imx_icc_node_adj_desc *adj;
+};
+
+#define DEFINE_BUS_INTERCONNECT(_name, _id, _adj, ...) \
+ { \
+ .id = _id, \
+ .name = _name, \
+ .adj = _adj, \
+ .num_links = ARRAY_SIZE(((int[]){ __VA_ARGS__ })), \
+ .links = { __VA_ARGS__ }, \
+ }
+
+#define DEFINE_BUS_MASTER(_name, _id, _dest_id) \
+ DEFINE_BUS_INTERCONNECT(_name, _id, NULL, _dest_id)
+
+#define DEFINE_BUS_SLAVE(_name, _id, _adj) \
+ DEFINE_BUS_INTERCONNECT(_name, _id, _adj)
+
+int imx_icc_register(struct platform_device *pdev,
+ struct imx_icc_node_desc *nodes,
+ int nodes_count);
+int imx_icc_unregister(struct platform_device *pdev);
+
+#endif /* __DRIVERS_INTERCONNECT_IMX_H */
diff --git a/drivers/interconnect/imx/imx8mm.c b/drivers/interconnect/imx/imx8mm.c
new file mode 100644
index 000000000000..1083490bb391
--- /dev/null
+++ b/drivers/interconnect/imx/imx8mm.c
@@ -0,0 +1,105 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Interconnect framework driver for i.MX8MM SoC
+ *
+ * Copyright (c) 2019, BayLibre
+ * Copyright (c) 2019-2020, NXP
+ * Author: Alexandre Bailon <abailon@baylibre.com>
+ * Author: Leonard Crestez <leonard.crestez@nxp.com>
+ */
+
+#include <linux/module.h>
+#include <linux/platform_device.h>
+#include <dt-bindings/interconnect/imx8mm.h>
+
+#include "imx.h"
+
+static const struct imx_icc_node_adj_desc imx8mm_dram_adj = {
+ .bw_mul = 1,
+ .bw_div = 16,
+ .phandle_name = "fsl,ddrc",
+};
+
+static const struct imx_icc_node_adj_desc imx8mm_noc_adj = {
+ .bw_mul = 1,
+ .bw_div = 16,
+ .main_noc = true,
+};
+
+/*
+ * Describe bus masters, slaves and connections between them
+ *
+ * This is a simplified subset of the bus diagram, there are several other
+ * PL301 nics which are skipped/merged into PL301_MAIN
+ */
+static struct imx_icc_node_desc nodes[] = {
+ DEFINE_BUS_INTERCONNECT("NOC", IMX8MM_ICN_NOC, &imx8mm_noc_adj,
+ IMX8MM_ICS_DRAM, IMX8MM_ICN_MAIN),
+
+ DEFINE_BUS_SLAVE("DRAM", IMX8MM_ICS_DRAM, &imx8mm_dram_adj),
+ DEFINE_BUS_SLAVE("OCRAM", IMX8MM_ICS_OCRAM, NULL),
+ DEFINE_BUS_MASTER("A53", IMX8MM_ICM_A53, IMX8MM_ICN_NOC),
+
+ /* VPUMIX */
+ DEFINE_BUS_MASTER("VPU H1", IMX8MM_ICM_VPU_H1, IMX8MM_ICN_VIDEO),
+ DEFINE_BUS_MASTER("VPU G1", IMX8MM_ICM_VPU_G1, IMX8MM_ICN_VIDEO),
+ DEFINE_BUS_MASTER("VPU G2", IMX8MM_ICM_VPU_G2, IMX8MM_ICN_VIDEO),
+ DEFINE_BUS_INTERCONNECT("PL301_VIDEO", IMX8MM_ICN_VIDEO, NULL, IMX8MM_ICN_NOC),
+
+ /* GPUMIX */
+ DEFINE_BUS_MASTER("GPU 2D", IMX8MM_ICM_GPU2D, IMX8MM_ICN_GPU),
+ DEFINE_BUS_MASTER("GPU 3D", IMX8MM_ICM_GPU3D, IMX8MM_ICN_GPU),
+ DEFINE_BUS_INTERCONNECT("PL301_GPU", IMX8MM_ICN_GPU, NULL, IMX8MM_ICN_NOC),
+
+ /* DISPLAYMIX */
+ DEFINE_BUS_MASTER("CSI", IMX8MM_ICM_CSI, IMX8MM_ICN_MIPI),
+ DEFINE_BUS_MASTER("LCDIF", IMX8MM_ICM_LCDIF, IMX8MM_ICN_MIPI),
+ DEFINE_BUS_INTERCONNECT("PL301_MIPI", IMX8MM_ICN_MIPI, NULL, IMX8MM_ICN_NOC),
+
+ /* HSIO */
+ DEFINE_BUS_MASTER("USB1", IMX8MM_ICM_USB1, IMX8MM_ICN_HSIO),
+ DEFINE_BUS_MASTER("USB2", IMX8MM_ICM_USB2, IMX8MM_ICN_HSIO),
+ DEFINE_BUS_MASTER("PCIE", IMX8MM_ICM_PCIE, IMX8MM_ICN_HSIO),
+ DEFINE_BUS_INTERCONNECT("PL301_HSIO", IMX8MM_ICN_HSIO, NULL, IMX8MM_ICN_NOC),
+
+ /* Audio */
+ DEFINE_BUS_MASTER("SDMA2", IMX8MM_ICM_SDMA2, IMX8MM_ICN_AUDIO),
+ DEFINE_BUS_MASTER("SDMA3", IMX8MM_ICM_SDMA3, IMX8MM_ICN_AUDIO),
+ DEFINE_BUS_INTERCONNECT("PL301_AUDIO", IMX8MM_ICN_AUDIO, NULL, IMX8MM_ICN_MAIN),
+
+ /* Ethernet */
+ DEFINE_BUS_MASTER("ENET", IMX8MM_ICM_ENET, IMX8MM_ICN_ENET),
+ DEFINE_BUS_INTERCONNECT("PL301_ENET", IMX8MM_ICN_ENET, NULL, IMX8MM_ICN_MAIN),
+
+ /* Other */
+ DEFINE_BUS_MASTER("SDMA1", IMX8MM_ICM_SDMA1, IMX8MM_ICN_MAIN),
+ DEFINE_BUS_MASTER("NAND", IMX8MM_ICM_NAND, IMX8MM_ICN_MAIN),
+ DEFINE_BUS_MASTER("USDHC1", IMX8MM_ICM_USDHC1, IMX8MM_ICN_MAIN),
+ DEFINE_BUS_MASTER("USDHC2", IMX8MM_ICM_USDHC2, IMX8MM_ICN_MAIN),
+ DEFINE_BUS_MASTER("USDHC3", IMX8MM_ICM_USDHC3, IMX8MM_ICN_MAIN),
+ DEFINE_BUS_INTERCONNECT("PL301_MAIN", IMX8MM_ICN_MAIN, NULL,
+ IMX8MM_ICN_NOC, IMX8MM_ICS_OCRAM),
+};
+
+static int imx8mm_icc_probe(struct platform_device *pdev)
+{
+ return imx_icc_register(pdev, nodes, ARRAY_SIZE(nodes));
+}
+
+static int imx8mm_icc_remove(struct platform_device *pdev)
+{
+ return imx_icc_unregister(pdev);
+}
+
+static struct platform_driver imx8mm_icc_driver = {
+ .probe = imx8mm_icc_probe,
+ .remove = imx8mm_icc_remove,
+ .driver = {
+ .name = "imx8mm-interconnect",
+ },
+};
+
+module_platform_driver(imx8mm_icc_driver);
+MODULE_AUTHOR("Alexandre Bailon <abailon@baylibre.com>");
+MODULE_LICENSE("GPL v2");
+MODULE_ALIAS("platform:imx8mm-interconnect");
diff --git a/drivers/interconnect/imx/imx8mn.c b/drivers/interconnect/imx/imx8mn.c
new file mode 100644
index 000000000000..ad97e55fd4e5
--- /dev/null
+++ b/drivers/interconnect/imx/imx8mn.c
@@ -0,0 +1,94 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Interconnect framework driver for i.MX8MN SoC
+ *
+ * Copyright (c) 2019-2020, NXP
+ */
+
+#include <linux/module.h>
+#include <linux/platform_device.h>
+#include <dt-bindings/interconnect/imx8mn.h>
+
+#include "imx.h"
+
+static const struct imx_icc_node_adj_desc imx8mn_dram_adj = {
+ .bw_mul = 1,
+ .bw_div = 4,
+ .phandle_name = "fsl,ddrc",
+};
+
+static const struct imx_icc_node_adj_desc imx8mn_noc_adj = {
+ .bw_mul = 1,
+ .bw_div = 4,
+ .main_noc = true,
+};
+
+/*
+ * Describe bus masters, slaves and connections between them
+ *
+ * This is a simplified subset of the bus diagram, there are several other
+ * PL301 nics which are skipped/merged into PL301_MAIN
+ */
+static struct imx_icc_node_desc nodes[] = {
+ DEFINE_BUS_INTERCONNECT("NOC", IMX8MN_ICN_NOC, &imx8mn_noc_adj,
+ IMX8MN_ICS_DRAM, IMX8MN_ICN_MAIN),
+
+ DEFINE_BUS_SLAVE("DRAM", IMX8MN_ICS_DRAM, &imx8mn_dram_adj),
+ DEFINE_BUS_SLAVE("OCRAM", IMX8MN_ICS_OCRAM, NULL),
+ DEFINE_BUS_MASTER("A53", IMX8MN_ICM_A53, IMX8MN_ICN_NOC),
+
+ /* GPUMIX */
+ DEFINE_BUS_MASTER("GPU", IMX8MN_ICM_GPU, IMX8MN_ICN_GPU),
+ DEFINE_BUS_INTERCONNECT("PL301_GPU", IMX8MN_ICN_GPU, NULL, IMX8MN_ICN_NOC),
+
+ /* DISPLAYMIX */
+ DEFINE_BUS_MASTER("CSI1", IMX8MN_ICM_CSI1, IMX8MN_ICN_MIPI),
+ DEFINE_BUS_MASTER("CSI2", IMX8MN_ICM_CSI2, IMX8MN_ICN_MIPI),
+ DEFINE_BUS_MASTER("ISI", IMX8MN_ICM_ISI, IMX8MN_ICN_MIPI),
+ DEFINE_BUS_MASTER("LCDIF", IMX8MN_ICM_LCDIF, IMX8MN_ICN_MIPI),
+ DEFINE_BUS_INTERCONNECT("PL301_MIPI", IMX8MN_ICN_MIPI, NULL, IMX8MN_ICN_NOC),
+
+ /* USB goes straight to NOC */
+ DEFINE_BUS_MASTER("USB", IMX8MN_ICM_USB, IMX8MN_ICN_NOC),
+
+ /* Audio */
+ DEFINE_BUS_MASTER("SDMA2", IMX8MN_ICM_SDMA2, IMX8MN_ICN_AUDIO),
+ DEFINE_BUS_MASTER("SDMA3", IMX8MN_ICM_SDMA3, IMX8MN_ICN_AUDIO),
+ DEFINE_BUS_INTERCONNECT("PL301_AUDIO", IMX8MN_ICN_AUDIO, NULL, IMX8MN_ICN_MAIN),
+
+ /* Ethernet */
+ DEFINE_BUS_MASTER("ENET", IMX8MN_ICM_ENET, IMX8MN_ICN_ENET),
+ DEFINE_BUS_INTERCONNECT("PL301_ENET", IMX8MN_ICN_ENET, NULL, IMX8MN_ICN_MAIN),
+
+ /* Other */
+ DEFINE_BUS_MASTER("SDMA1", IMX8MN_ICM_SDMA1, IMX8MN_ICN_MAIN),
+ DEFINE_BUS_MASTER("NAND", IMX8MN_ICM_NAND, IMX8MN_ICN_MAIN),
+ DEFINE_BUS_MASTER("USDHC1", IMX8MN_ICM_USDHC1, IMX8MN_ICN_MAIN),
+ DEFINE_BUS_MASTER("USDHC2", IMX8MN_ICM_USDHC2, IMX8MN_ICN_MAIN),
+ DEFINE_BUS_MASTER("USDHC3", IMX8MN_ICM_USDHC3, IMX8MN_ICN_MAIN),
+ DEFINE_BUS_INTERCONNECT("PL301_MAIN", IMX8MN_ICN_MAIN, NULL,
+ IMX8MN_ICN_NOC, IMX8MN_ICS_OCRAM),
+};
+
+static int imx8mn_icc_probe(struct platform_device *pdev)
+{
+ return imx_icc_register(pdev, nodes, ARRAY_SIZE(nodes));
+}
+
+static int imx8mn_icc_remove(struct platform_device *pdev)
+{
+ return imx_icc_unregister(pdev);
+}
+
+static struct platform_driver imx8mn_icc_driver = {
+ .probe = imx8mn_icc_probe,
+ .remove = imx8mn_icc_remove,
+ .driver = {
+ .name = "imx8mn-interconnect",
+ },
+};
+
+module_platform_driver(imx8mn_icc_driver);
+MODULE_ALIAS("platform:imx8mn-interconnect");
+MODULE_AUTHOR("Leonard Crestez <leonard.crestez@nxp.com>");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/interconnect/imx/imx8mq.c b/drivers/interconnect/imx/imx8mq.c
new file mode 100644
index 000000000000..ba43a15aefec
--- /dev/null
+++ b/drivers/interconnect/imx/imx8mq.c
@@ -0,0 +1,103 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Interconnect framework driver for i.MX8MQ SoC
+ *
+ * Copyright (c) 2019-2020, NXP
+ */
+
+#include <linux/module.h>
+#include <linux/platform_device.h>
+#include <dt-bindings/interconnect/imx8mq.h>
+
+#include "imx.h"
+
+static const struct imx_icc_node_adj_desc imx8mq_dram_adj = {
+ .bw_mul = 1,
+ .bw_div = 4,
+ .phandle_name = "fsl,ddrc",
+};
+
+static const struct imx_icc_node_adj_desc imx8mq_noc_adj = {
+ .bw_mul = 1,
+ .bw_div = 4,
+ .main_noc = true,
+};
+
+/*
+ * Describe bus masters, slaves and connections between them
+ *
+ * This is a simplified subset of the bus diagram, there are several other
+ * PL301 nics which are skipped/merged into PL301_MAIN
+ */
+static struct imx_icc_node_desc nodes[] = {
+ DEFINE_BUS_INTERCONNECT("NOC", IMX8MQ_ICN_NOC, &imx8mq_noc_adj,
+ IMX8MQ_ICS_DRAM, IMX8MQ_ICN_MAIN),
+
+ DEFINE_BUS_SLAVE("DRAM", IMX8MQ_ICS_DRAM, &imx8mq_dram_adj),
+ DEFINE_BUS_SLAVE("OCRAM", IMX8MQ_ICS_OCRAM, NULL),
+ DEFINE_BUS_MASTER("A53", IMX8MQ_ICM_A53, IMX8MQ_ICN_NOC),
+
+ /* VPUMIX */
+ DEFINE_BUS_MASTER("VPU", IMX8MQ_ICM_VPU, IMX8MQ_ICN_VIDEO),
+ DEFINE_BUS_INTERCONNECT("PL301_VIDEO", IMX8MQ_ICN_VIDEO, NULL, IMX8MQ_ICN_NOC),
+
+ /* GPUMIX */
+ DEFINE_BUS_MASTER("GPU", IMX8MQ_ICM_GPU, IMX8MQ_ICN_GPU),
+ DEFINE_BUS_INTERCONNECT("PL301_GPU", IMX8MQ_ICN_GPU, NULL, IMX8MQ_ICN_NOC),
+
+ /* DISPMIX (only for DCSS) */
+ DEFINE_BUS_MASTER("DC", IMX8MQ_ICM_DCSS, IMX8MQ_ICN_DCSS),
+ DEFINE_BUS_INTERCONNECT("PL301_DC", IMX8MQ_ICN_DCSS, NULL, IMX8MQ_ICN_NOC),
+
+ /* USBMIX */
+ DEFINE_BUS_MASTER("USB1", IMX8MQ_ICM_USB1, IMX8MQ_ICN_USB),
+ DEFINE_BUS_MASTER("USB2", IMX8MQ_ICM_USB2, IMX8MQ_ICN_USB),
+ DEFINE_BUS_INTERCONNECT("PL301_USB", IMX8MQ_ICN_USB, NULL, IMX8MQ_ICN_NOC),
+
+ /* PL301_DISPLAY (IPs other than DCSS, inside SUPERMIX) */
+ DEFINE_BUS_MASTER("CSI1", IMX8MQ_ICM_CSI1, IMX8MQ_ICN_DISPLAY),
+ DEFINE_BUS_MASTER("CSI2", IMX8MQ_ICM_CSI2, IMX8MQ_ICN_DISPLAY),
+ DEFINE_BUS_MASTER("LCDIF", IMX8MQ_ICM_LCDIF, IMX8MQ_ICN_DISPLAY),
+ DEFINE_BUS_INTERCONNECT("PL301_DISPLAY", IMX8MQ_ICN_DISPLAY, NULL, IMX8MQ_ICN_MAIN),
+
+ /* AUDIO */
+ DEFINE_BUS_MASTER("SDMA2", IMX8MQ_ICM_SDMA2, IMX8MQ_ICN_AUDIO),
+ DEFINE_BUS_INTERCONNECT("PL301_AUDIO", IMX8MQ_ICN_AUDIO, NULL, IMX8MQ_ICN_DISPLAY),
+
+ /* ENET */
+ DEFINE_BUS_MASTER("ENET", IMX8MQ_ICM_ENET, IMX8MQ_ICN_ENET),
+ DEFINE_BUS_INTERCONNECT("PL301_ENET", IMX8MQ_ICN_ENET, NULL, IMX8MQ_ICN_MAIN),
+
+ /* OTHER */
+ DEFINE_BUS_MASTER("SDMA1", IMX8MQ_ICM_SDMA1, IMX8MQ_ICN_MAIN),
+ DEFINE_BUS_MASTER("NAND", IMX8MQ_ICM_NAND, IMX8MQ_ICN_MAIN),
+ DEFINE_BUS_MASTER("USDHC1", IMX8MQ_ICM_USDHC1, IMX8MQ_ICN_MAIN),
+ DEFINE_BUS_MASTER("USDHC2", IMX8MQ_ICM_USDHC2, IMX8MQ_ICN_MAIN),
+ DEFINE_BUS_MASTER("PCIE1", IMX8MQ_ICM_PCIE1, IMX8MQ_ICN_MAIN),
+ DEFINE_BUS_MASTER("PCIE2", IMX8MQ_ICM_PCIE2, IMX8MQ_ICN_MAIN),
+ DEFINE_BUS_INTERCONNECT("PL301_MAIN", IMX8MQ_ICN_MAIN, NULL,
+ IMX8MQ_ICN_NOC, IMX8MQ_ICS_OCRAM),
+};
+
+static int imx8mq_icc_probe(struct platform_device *pdev)
+{
+ return imx_icc_register(pdev, nodes, ARRAY_SIZE(nodes));
+}
+
+static int imx8mq_icc_remove(struct platform_device *pdev)
+{
+ return imx_icc_unregister(pdev);
+}
+
+static struct platform_driver imx8mq_icc_driver = {
+ .probe = imx8mq_icc_probe,
+ .remove = imx8mq_icc_remove,
+ .driver = {
+ .name = "imx8mq-interconnect",
+ },
+};
+
+module_platform_driver(imx8mq_icc_driver);
+MODULE_ALIAS("platform:imx8mq-interconnect");
+MODULE_AUTHOR("Leonard Crestez <leonard.crestez@nxp.com>");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/interconnect/internal.h b/drivers/interconnect/internal.h
index bf18cb7239df..f5f82a5c939e 100644
--- a/drivers/interconnect/internal.h
+++ b/drivers/interconnect/internal.h
@@ -14,6 +14,7 @@
* @req_node: entry in list of requests for the particular @node
* @node: the interconnect node to which this constraint applies
* @dev: reference to the device that sets the constraints
+ * @enabled: indicates whether the path with this request is enabled
* @tag: path tag (optional)
* @avg_bw: an integer describing the average bandwidth in kBps
* @peak_bw: an integer describing the peak bandwidth in kBps
@@ -22,6 +23,7 @@ struct icc_req {
struct hlist_node req_node;
struct icc_node *node;
struct device *dev;
+ bool enabled;
u32 tag;
u32 avg_bw;
u32 peak_bw;
diff --git a/drivers/misc/cardreader/rts5249.c b/drivers/misc/cardreader/rts5249.c
index 1a81cda948c1..6c6c9e95a29f 100644
--- a/drivers/misc/cardreader/rts5249.c
+++ b/drivers/misc/cardreader/rts5249.c
@@ -347,31 +347,6 @@ static int rtsx_base_switch_output_voltage(struct rtsx_pcr *pcr, u8 voltage)
return rtsx_pci_send_cmd(pcr, 100);
}
-static void rts5249_set_aspm(struct rtsx_pcr *pcr, bool enable)
-{
- struct rtsx_cr_option *option = &pcr->option;
- u8 val = 0;
-
- if (pcr->aspm_enabled == enable)
- return;
-
- if (option->dev_aspm_mode == DEV_ASPM_DYNAMIC) {
- if (enable)
- val = pcr->aspm_en;
- rtsx_pci_update_cfg_byte(pcr,
- pcr->pcie_cap + PCI_EXP_LNKCTL,
- ASPM_MASK_NEG, val);
- } else if (option->dev_aspm_mode == DEV_ASPM_BACKDOOR) {
- u8 mask = FORCE_ASPM_VAL_MASK | FORCE_ASPM_CTL0;
-
- if (!enable)
- val = FORCE_ASPM_CTL0;
- rtsx_pci_write_register(pcr, ASPM_FORCE_CTL, mask, val);
- }
-
- pcr->aspm_enabled = enable;
-}
-
static const struct pcr_ops rts5249_pcr_ops = {
.fetch_vendor_settings = rtsx_base_fetch_vendor_settings,
.extra_init_hw = rts5249_extra_init_hw,
@@ -384,7 +359,6 @@ static const struct pcr_ops rts5249_pcr_ops = {
.card_power_off = rtsx_base_card_power_off,
.switch_output_voltage = rtsx_base_switch_output_voltage,
.force_power_down = rtsx_base_force_power_down,
- .set_aspm = rts5249_set_aspm,
};
/* SD Pull Control Enable:
@@ -471,7 +445,6 @@ void rts5249_init_params(struct rtsx_pcr *pcr)
option->ltr_active_latency = LTR_ACTIVE_LATENCY_DEF;
option->ltr_idle_latency = LTR_IDLE_LATENCY_DEF;
option->ltr_l1off_latency = LTR_L1OFF_LATENCY_DEF;
- option->dev_aspm_mode = DEV_ASPM_DYNAMIC;
option->l1_snooze_delay = L1_SNOOZE_DELAY_DEF;
option->ltr_l1off_sspwrgate = LTR_L1OFF_SSPWRGATE_5249_DEF;
option->ltr_l1off_snooze_sspwrgate =
@@ -612,7 +585,6 @@ static const struct pcr_ops rts524a_pcr_ops = {
.switch_output_voltage = rtsx_base_switch_output_voltage,
.force_power_down = rtsx_base_force_power_down,
.set_l1off_cfg_sub_d0 = rts5250_set_l1off_cfg_sub_d0,
- .set_aspm = rts5249_set_aspm,
};
void rts524a_init_params(struct rtsx_pcr *pcr)
@@ -728,7 +700,6 @@ static const struct pcr_ops rts525a_pcr_ops = {
.switch_output_voltage = rts525a_switch_output_voltage,
.force_power_down = rtsx_base_force_power_down,
.set_l1off_cfg_sub_d0 = rts5250_set_l1off_cfg_sub_d0,
- .set_aspm = rts5249_set_aspm,
};
void rts525a_init_params(struct rtsx_pcr *pcr)
diff --git a/drivers/misc/cardreader/rts5260.c b/drivers/misc/cardreader/rts5260.c
index 711054ebad74..7a9dbb778e84 100644
--- a/drivers/misc/cardreader/rts5260.c
+++ b/drivers/misc/cardreader/rts5260.c
@@ -570,30 +570,6 @@ static int rts5260_extra_init_hw(struct rtsx_pcr *pcr)
return 0;
}
-static void rts5260_set_aspm(struct rtsx_pcr *pcr, bool enable)
-{
- struct rtsx_cr_option *option = &pcr->option;
- u8 val = 0;
-
- if (pcr->aspm_enabled == enable)
- return;
-
- if (option->dev_aspm_mode == DEV_ASPM_DYNAMIC) {
- if (enable)
- val = pcr->aspm_en;
- rtsx_pci_update_cfg_byte(pcr, pcr->pcie_cap + PCI_EXP_LNKCTL,
- ASPM_MASK_NEG, val);
- } else if (option->dev_aspm_mode == DEV_ASPM_BACKDOOR) {
- u8 mask = FORCE_ASPM_VAL_MASK | FORCE_ASPM_CTL0;
-
- if (!enable)
- val = FORCE_ASPM_CTL0;
- rtsx_pci_write_register(pcr, ASPM_FORCE_CTL, mask, val);
- }
-
- pcr->aspm_enabled = enable;
-}
-
static void rts5260_set_l1off_cfg_sub_d0(struct rtsx_pcr *pcr, int active)
{
struct rtsx_cr_option *option = &pcr->option;
@@ -639,7 +615,6 @@ static const struct pcr_ops rts5260_pcr_ops = {
.switch_output_voltage = rts5260_switch_output_voltage,
.force_power_down = rtsx_base_force_power_down,
.stop_cmd = rts5260_stop_cmd,
- .set_aspm = rts5260_set_aspm,
.set_l1off_cfg_sub_d0 = rts5260_set_l1off_cfg_sub_d0,
.enable_ocp = rts5260_enable_ocp,
.disable_ocp = rts5260_disable_ocp,
@@ -683,7 +658,6 @@ void rts5260_init_params(struct rtsx_pcr *pcr)
option->ltr_active_latency = LTR_ACTIVE_LATENCY_DEF;
option->ltr_idle_latency = LTR_IDLE_LATENCY_DEF;
option->ltr_l1off_latency = LTR_L1OFF_LATENCY_DEF;
- option->dev_aspm_mode = DEV_ASPM_DYNAMIC;
option->l1_snooze_delay = L1_SNOOZE_DELAY_DEF;
option->ltr_l1off_sspwrgate = LTR_L1OFF_SSPWRGATE_5250_DEF;
option->ltr_l1off_snooze_sspwrgate =
diff --git a/drivers/misc/cardreader/rts5261.c b/drivers/misc/cardreader/rts5261.c
index 78c3b1d424c3..195822ec858e 100644
--- a/drivers/misc/cardreader/rts5261.c
+++ b/drivers/misc/cardreader/rts5261.c
@@ -518,51 +518,22 @@ static int rts5261_extra_init_hw(struct rtsx_pcr *pcr)
static void rts5261_enable_aspm(struct rtsx_pcr *pcr, bool enable)
{
- struct rtsx_cr_option *option = &pcr->option;
- u8 val = 0;
-
if (pcr->aspm_enabled == enable)
return;
- if (option->dev_aspm_mode == DEV_ASPM_DYNAMIC) {
- val = pcr->aspm_en;
- rtsx_pci_update_cfg_byte(pcr, pcr->pcie_cap + PCI_EXP_LNKCTL,
- ASPM_MASK_NEG, val);
- } else if (option->dev_aspm_mode == DEV_ASPM_BACKDOOR) {
- u8 mask = FORCE_ASPM_VAL_MASK | FORCE_ASPM_CTL0;
-
- val = FORCE_ASPM_CTL0;
- val |= (pcr->aspm_en & 0x02);
- rtsx_pci_write_register(pcr, ASPM_FORCE_CTL, mask, val);
- val = pcr->aspm_en;
- rtsx_pci_update_cfg_byte(pcr, pcr->pcie_cap + PCI_EXP_LNKCTL,
- ASPM_MASK_NEG, val);
- }
+ pcie_capability_clear_and_set_word(pcr->pci, PCI_EXP_LNKCTL,
+ PCI_EXP_LNKCTL_ASPMC, pcr->aspm_en);
pcr->aspm_enabled = enable;
}
static void rts5261_disable_aspm(struct rtsx_pcr *pcr, bool enable)
{
- struct rtsx_cr_option *option = &pcr->option;
- u8 val = 0;
-
if (pcr->aspm_enabled == enable)
return;
- if (option->dev_aspm_mode == DEV_ASPM_DYNAMIC) {
- val = 0;
- rtsx_pci_update_cfg_byte(pcr, pcr->pcie_cap + PCI_EXP_LNKCTL,
- ASPM_MASK_NEG, val);
- } else if (option->dev_aspm_mode == DEV_ASPM_BACKDOOR) {
- u8 mask = FORCE_ASPM_VAL_MASK | FORCE_ASPM_CTL0;
-
- val = 0;
- rtsx_pci_update_cfg_byte(pcr, pcr->pcie_cap + PCI_EXP_LNKCTL,
- ASPM_MASK_NEG, val);
- val = FORCE_ASPM_CTL0;
- rtsx_pci_write_register(pcr, ASPM_FORCE_CTL, mask, val);
- }
+ pcie_capability_clear_and_set_word(pcr->pci, PCI_EXP_LNKCTL,
+ PCI_EXP_LNKCTL_ASPMC, 0);
rtsx_pci_write_register(pcr, SD_CFG1, SD_ASYNC_FIFO_NOT_RST, 0);
udelay(10);
pcr->aspm_enabled = enable;
@@ -639,8 +610,13 @@ int rts5261_pci_switch_clock(struct rtsx_pcr *pcr, unsigned int card_clock,
if (initial_mode) {
/* We use 250k(around) here, in initial stage */
- clk_divider = SD_CLK_DIVIDE_128;
- card_clock = 30000000;
+ if (is_version(pcr, PID_5261, IC_VER_D)) {
+ clk_divider = SD_CLK_DIVIDE_256;
+ card_clock = 60000000;
+ } else {
+ clk_divider = SD_CLK_DIVIDE_128;
+ card_clock = 30000000;
+ }
} else {
clk_divider = SD_CLK_DIVIDE_0;
}
@@ -784,7 +760,6 @@ void rts5261_init_params(struct rtsx_pcr *pcr)
option->l1_snooze_delay = L1_SNOOZE_DELAY_DEF;
option->ltr_l1off_sspwrgate = 0x7F;
option->ltr_l1off_snooze_sspwrgate = 0x78;
- option->dev_aspm_mode = DEV_ASPM_DYNAMIC;
option->ocp_en = 1;
hw_param->interrupt_en |= SD_OC_INT_EN;
diff --git a/drivers/misc/cardreader/rtsx_pcr.c b/drivers/misc/cardreader/rtsx_pcr.c
index 55da6428ceb0..0d5928bc1b6d 100644
--- a/drivers/misc/cardreader/rtsx_pcr.c
+++ b/drivers/misc/cardreader/rtsx_pcr.c
@@ -55,16 +55,10 @@ static const struct pci_device_id rtsx_pci_ids[] = {
MODULE_DEVICE_TABLE(pci, rtsx_pci_ids);
-static inline void rtsx_pci_enable_aspm(struct rtsx_pcr *pcr)
-{
- rtsx_pci_update_cfg_byte(pcr, pcr->pcie_cap + PCI_EXP_LNKCTL,
- 0xFC, pcr->aspm_en);
-}
-
static inline void rtsx_pci_disable_aspm(struct rtsx_pcr *pcr)
{
- rtsx_pci_update_cfg_byte(pcr, pcr->pcie_cap + PCI_EXP_LNKCTL,
- 0xFC, 0);
+ pcie_capability_clear_and_set_word(pcr->pci, PCI_EXP_LNKCTL,
+ PCI_EXP_LNKCTL_ASPMC, 0);
}
static int rtsx_comm_set_ltr_latency(struct rtsx_pcr *pcr, u32 latency)
@@ -85,32 +79,17 @@ static int rtsx_comm_set_ltr_latency(struct rtsx_pcr *pcr, u32 latency)
int rtsx_set_ltr_latency(struct rtsx_pcr *pcr, u32 latency)
{
- if (pcr->ops->set_ltr_latency)
- return pcr->ops->set_ltr_latency(pcr, latency);
- else
- return rtsx_comm_set_ltr_latency(pcr, latency);
+ return rtsx_comm_set_ltr_latency(pcr, latency);
}
static void rtsx_comm_set_aspm(struct rtsx_pcr *pcr, bool enable)
{
- struct rtsx_cr_option *option = &pcr->option;
-
if (pcr->aspm_enabled == enable)
return;
- if (option->dev_aspm_mode == DEV_ASPM_DYNAMIC) {
- if (enable)
- rtsx_pci_enable_aspm(pcr);
- else
- rtsx_pci_disable_aspm(pcr);
- } else if (option->dev_aspm_mode == DEV_ASPM_BACKDOOR) {
- u8 mask = FORCE_ASPM_VAL_MASK;
- u8 val = 0;
-
- if (enable)
- val = pcr->aspm_en;
- rtsx_pci_write_register(pcr, ASPM_FORCE_CTL, mask, val);
- }
+ pcie_capability_clear_and_set_word(pcr->pci, PCI_EXP_LNKCTL,
+ PCI_EXP_LNKCTL_ASPMC,
+ enable ? pcr->aspm_en : 0);
pcr->aspm_enabled = enable;
}
@@ -154,10 +133,7 @@ static void rtsx_comm_pm_full_on(struct rtsx_pcr *pcr)
static void rtsx_pm_full_on(struct rtsx_pcr *pcr)
{
- if (pcr->ops->full_on)
- pcr->ops->full_on(pcr);
- else
- rtsx_comm_pm_full_on(pcr);
+ rtsx_comm_pm_full_on(pcr);
}
void rtsx_pci_start_run(struct rtsx_pcr *pcr)
@@ -1111,10 +1087,7 @@ static void rtsx_comm_pm_power_saving(struct rtsx_pcr *pcr)
static void rtsx_pm_power_saving(struct rtsx_pcr *pcr)
{
- if (pcr->ops->power_saving)
- pcr->ops->power_saving(pcr);
- else
- rtsx_comm_pm_power_saving(pcr);
+ rtsx_comm_pm_power_saving(pcr);
}
static void rtsx_pci_idle_work(struct work_struct *work)
diff --git a/drivers/misc/cardreader/rtsx_pcr.h b/drivers/misc/cardreader/rtsx_pcr.h
index ed391df52f4f..024cbd998b2a 100644
--- a/drivers/misc/cardreader/rtsx_pcr.h
+++ b/drivers/misc/cardreader/rtsx_pcr.h
@@ -29,7 +29,6 @@
#define LTR_L1OFF_SNOOZE_SSPWRGATE_5249_DEF 0xAC
#define LTR_L1OFF_SNOOZE_SSPWRGATE_5250_DEF 0xF8
#define CMD_TIMEOUT_DEF 100
-#define ASPM_MASK_NEG 0xFC
#define MASK_8_BIT_DEF 0xFF
#define SSC_CLOCK_STABLE_WAIT 130
diff --git a/drivers/misc/fastrpc.c b/drivers/misc/fastrpc.c
index e3e085e33d46..7939c55daceb 100644
--- a/drivers/misc/fastrpc.c
+++ b/drivers/misc/fastrpc.c
@@ -904,6 +904,7 @@ static int fastrpc_invoke_send(struct fastrpc_session_ctx *sctx,
struct fastrpc_channel_ctx *cctx;
struct fastrpc_user *fl = ctx->fl;
struct fastrpc_msg *msg = &ctx->msg;
+ int ret;
cctx = fl->cctx;
msg->pid = fl->tgid;
@@ -919,7 +920,13 @@ static int fastrpc_invoke_send(struct fastrpc_session_ctx *sctx,
msg->size = roundup(ctx->msg_sz, PAGE_SIZE);
fastrpc_context_get(ctx);
- return rpmsg_send(cctx->rpdev->ept, (void *)msg, sizeof(*msg));
+ ret = rpmsg_send(cctx->rpdev->ept, (void *)msg, sizeof(*msg));
+
+ if (ret)
+ fastrpc_context_put(ctx);
+
+ return ret;
+
}
static int fastrpc_internal_invoke(struct fastrpc_user *fl, u32 kernel,
@@ -1613,8 +1620,10 @@ static int fastrpc_rpmsg_probe(struct rpmsg_device *rpdev)
domains[domain_id]);
data->miscdev.fops = &fastrpc_fops;
err = misc_register(&data->miscdev);
- if (err)
+ if (err) {
+ kfree(data);
return err;
+ }
kref_init(&data->refcount);
diff --git a/drivers/misc/genwqe/card_utils.c b/drivers/misc/genwqe/card_utils.c
index 2e1c4d2905e8..60460a053b2d 100644
--- a/drivers/misc/genwqe/card_utils.c
+++ b/drivers/misc/genwqe/card_utils.c
@@ -515,30 +515,6 @@ int genwqe_free_sync_sgl(struct genwqe_dev *cd, struct genwqe_sgl *sgl)
}
/**
- * genwqe_free_user_pages() - Give pinned pages back
- *
- * Documentation of get_user_pages is in mm/gup.c:
- *
- * If the page is written to, set_page_dirty (or set_page_dirty_lock,
- * as appropriate) must be called after the page is finished with, and
- * before put_page is called.
- */
-static int genwqe_free_user_pages(struct page **page_list,
- unsigned int nr_pages, int dirty)
-{
- unsigned int i;
-
- for (i = 0; i < nr_pages; i++) {
- if (page_list[i] != NULL) {
- if (dirty)
- set_page_dirty_lock(page_list[i]);
- put_page(page_list[i]);
- }
- }
- return 0;
-}
-
-/**
* genwqe_user_vmap() - Map user-space memory to virtual kernel memory
* @cd: pointer to genwqe device
* @m: mapping params
@@ -597,18 +573,18 @@ int genwqe_user_vmap(struct genwqe_dev *cd, struct dma_mapping *m, void *uaddr,
m->dma_list = (dma_addr_t *)(m->page_list + m->nr_pages);
/* pin user pages in memory */
- rc = get_user_pages_fast(data & PAGE_MASK, /* page aligned addr */
+ rc = pin_user_pages_fast(data & PAGE_MASK, /* page aligned addr */
m->nr_pages,
m->write ? FOLL_WRITE : 0, /* readable/writable */
m->page_list); /* ptrs to pages */
if (rc < 0)
- goto fail_get_user_pages;
+ goto fail_pin_user_pages;
- /* assumption: get_user_pages can be killed by signals. */
+ /* assumption: pin_user_pages can be killed by signals. */
if (rc < m->nr_pages) {
- genwqe_free_user_pages(m->page_list, rc, m->write);
+ unpin_user_pages_dirty_lock(m->page_list, rc, m->write);
rc = -EFAULT;
- goto fail_get_user_pages;
+ goto fail_pin_user_pages;
}
rc = genwqe_map_pages(cd, m->page_list, m->nr_pages, m->dma_list);
@@ -618,9 +594,9 @@ int genwqe_user_vmap(struct genwqe_dev *cd, struct dma_mapping *m, void *uaddr,
return 0;
fail_free_user_pages:
- genwqe_free_user_pages(m->page_list, m->nr_pages, m->write);
+ unpin_user_pages_dirty_lock(m->page_list, m->nr_pages, m->write);
- fail_get_user_pages:
+ fail_pin_user_pages:
kfree(m->page_list);
m->page_list = NULL;
m->dma_list = NULL;
@@ -650,8 +626,8 @@ int genwqe_user_vunmap(struct genwqe_dev *cd, struct dma_mapping *m)
genwqe_unmap_pages(cd, m->dma_list, m->nr_pages);
if (m->page_list) {
- genwqe_free_user_pages(m->page_list, m->nr_pages, m->write);
-
+ unpin_user_pages_dirty_lock(m->page_list, m->nr_pages,
+ m->write);
kfree(m->page_list);
m->page_list = NULL;
m->dma_list = NULL;
diff --git a/drivers/misc/habanalabs/Makefile b/drivers/misc/habanalabs/Makefile
index 482f6227dbba..421ebd903069 100644
--- a/drivers/misc/habanalabs/Makefile
+++ b/drivers/misc/habanalabs/Makefile
@@ -13,3 +13,6 @@ habanalabs-$(CONFIG_DEBUG_FS) += debugfs.o
include $(src)/goya/Makefile
habanalabs-y += $(HL_GOYA_FILES)
+
+include $(src)/gaudi/Makefile
+habanalabs-y += $(HL_GAUDI_FILES)
diff --git a/drivers/misc/habanalabs/command_buffer.c b/drivers/misc/habanalabs/command_buffer.c
index 53fddbd8e693..02d13f71b1df 100644
--- a/drivers/misc/habanalabs/command_buffer.c
+++ b/drivers/misc/habanalabs/command_buffer.c
@@ -105,10 +105,9 @@ int hl_cb_create(struct hl_device *hdev, struct hl_cb_mgr *mgr,
goto out_err;
}
- if (cb_size > HL_MAX_CB_SIZE) {
- dev_err(hdev->dev,
- "CB size %d must be less then %d\n",
- cb_size, HL_MAX_CB_SIZE);
+ if (cb_size > SZ_2M) {
+ dev_err(hdev->dev, "CB size %d must be less than %d\n",
+ cb_size, SZ_2M);
rc = -EINVAL;
goto out_err;
}
@@ -211,7 +210,7 @@ int hl_cb_ioctl(struct hl_fpriv *hpriv, void *data)
{
union hl_cb_args *args = data;
struct hl_device *hdev = hpriv->hdev;
- u64 handle;
+ u64 handle = 0;
int rc;
if (hl_device_disabled_or_in_reset(hdev)) {
@@ -223,15 +222,26 @@ int hl_cb_ioctl(struct hl_fpriv *hpriv, void *data)
switch (args->in.op) {
case HL_CB_OP_CREATE:
- rc = hl_cb_create(hdev, &hpriv->cb_mgr, args->in.cb_size,
- &handle, hpriv->ctx->asid);
+ if (args->in.cb_size > HL_MAX_CB_SIZE) {
+ dev_err(hdev->dev,
+ "User requested CB size %d must be less than %d\n",
+ args->in.cb_size, HL_MAX_CB_SIZE);
+ rc = -EINVAL;
+ } else {
+ rc = hl_cb_create(hdev, &hpriv->cb_mgr,
+ args->in.cb_size, &handle,
+ hpriv->ctx->asid);
+ }
+
memset(args, 0, sizeof(*args));
args->out.cb_handle = handle;
break;
+
case HL_CB_OP_DESTROY:
rc = hl_cb_destroy(hdev, &hpriv->cb_mgr,
args->in.cb_handle);
break;
+
default:
rc = -ENOTTY;
break;
@@ -278,7 +288,7 @@ int hl_cb_mmap(struct hl_fpriv *hpriv, struct vm_area_struct *vma)
cb = hl_cb_get(hdev, &hpriv->cb_mgr, handle);
if (!cb) {
dev_err(hdev->dev,
- "CB mmap failed, no match to handle %d\n", handle);
+ "CB mmap failed, no match to handle 0x%x\n", handle);
return -EINVAL;
}
@@ -347,7 +357,7 @@ struct hl_cb *hl_cb_get(struct hl_device *hdev, struct hl_cb_mgr *mgr,
if (!cb) {
spin_unlock(&mgr->cb_lock);
dev_warn(hdev->dev,
- "CB get failed, no match to handle %d\n", handle);
+ "CB get failed, no match to handle 0x%x\n", handle);
return NULL;
}
diff --git a/drivers/misc/habanalabs/command_submission.c b/drivers/misc/habanalabs/command_submission.c
index 409276b6374d..f82974a916c3 100644
--- a/drivers/misc/habanalabs/command_submission.c
+++ b/drivers/misc/habanalabs/command_submission.c
@@ -11,11 +11,33 @@
#include <linux/uaccess.h>
#include <linux/slab.h>
+#define HL_CS_FLAGS_SIG_WAIT (HL_CS_FLAGS_SIGNAL | HL_CS_FLAGS_WAIT)
+
static void job_wq_completion(struct work_struct *work);
static long _hl_cs_wait_ioctl(struct hl_device *hdev,
struct hl_ctx *ctx, u64 timeout_us, u64 seq);
static void cs_do_release(struct kref *ref);
+static void hl_sob_reset(struct kref *ref)
+{
+ struct hl_hw_sob *hw_sob = container_of(ref, struct hl_hw_sob,
+ kref);
+ struct hl_device *hdev = hw_sob->hdev;
+
+ hdev->asic_funcs->reset_sob(hdev, hw_sob);
+}
+
+void hl_sob_reset_error(struct kref *ref)
+{
+ struct hl_hw_sob *hw_sob = container_of(ref, struct hl_hw_sob,
+ kref);
+ struct hl_device *hdev = hw_sob->hdev;
+
+ dev_crit(hdev->dev,
+ "SOB release shouldn't be called here, q_idx: %d, sob_id: %d\n",
+ hw_sob->q_idx, hw_sob->sob_id);
+}
+
static const char *hl_fence_get_driver_name(struct dma_fence *fence)
{
return "HabanaLabs";
@@ -23,10 +45,10 @@ static const char *hl_fence_get_driver_name(struct dma_fence *fence)
static const char *hl_fence_get_timeline_name(struct dma_fence *fence)
{
- struct hl_dma_fence *hl_fence =
- container_of(fence, struct hl_dma_fence, base_fence);
+ struct hl_cs_compl *hl_cs_compl =
+ container_of(fence, struct hl_cs_compl, base_fence);
- return dev_name(hl_fence->hdev->dev);
+ return dev_name(hl_cs_compl->hdev->dev);
}
static bool hl_fence_enable_signaling(struct dma_fence *fence)
@@ -36,17 +58,47 @@ static bool hl_fence_enable_signaling(struct dma_fence *fence)
static void hl_fence_release(struct dma_fence *fence)
{
- struct hl_dma_fence *hl_fence =
- container_of(fence, struct hl_dma_fence, base_fence);
+ struct hl_cs_compl *hl_cs_cmpl =
+ container_of(fence, struct hl_cs_compl, base_fence);
+ struct hl_device *hdev = hl_cs_cmpl->hdev;
+
+ if ((hl_cs_cmpl->type == CS_TYPE_SIGNAL) ||
+ (hl_cs_cmpl->type == CS_TYPE_WAIT)) {
- kfree_rcu(hl_fence, base_fence.rcu);
+ dev_dbg(hdev->dev,
+ "CS 0x%llx type %d finished, sob_id: %d, sob_val: 0x%x\n",
+ hl_cs_cmpl->cs_seq,
+ hl_cs_cmpl->type,
+ hl_cs_cmpl->hw_sob->sob_id,
+ hl_cs_cmpl->sob_val);
+
+ /*
+ * A signal CS can get completion while the corresponding wait
+ * for signal CS is on its way to the PQ. The wait for signal CS
+ * will get stuck if the signal CS incremented the SOB to its
+ * max value and there are no pending (submitted) waits on this
+ * SOB.
+ * We do the following to void this situation:
+ * 1. The wait for signal CS must get a ref for the signal CS as
+ * soon as possible in cs_ioctl_signal_wait() and put it
+ * before being submitted to the PQ but after it incremented
+ * the SOB refcnt in init_signal_wait_cs().
+ * 2. Signal/Wait for signal CS will decrement the SOB refcnt
+ * here.
+ * These two measures guarantee that the wait for signal CS will
+ * reset the SOB upon completion rather than the signal CS and
+ * hence the above scenario is avoided.
+ */
+ kref_put(&hl_cs_cmpl->hw_sob->kref, hl_sob_reset);
+ }
+
+ kfree_rcu(hl_cs_cmpl, base_fence.rcu);
}
static const struct dma_fence_ops hl_fence_ops = {
.get_driver_name = hl_fence_get_driver_name,
.get_timeline_name = hl_fence_get_timeline_name,
.enable_signaling = hl_fence_enable_signaling,
- .wait = dma_fence_default_wait,
.release = hl_fence_release
};
@@ -113,6 +165,7 @@ static int cs_parser(struct hl_fpriv *hpriv, struct hl_cs_job *job)
if (!rc) {
job->patched_cb = parser.patched_cb;
job->job_cb_size = parser.patched_cb_size;
+ job->contains_dma_pkt = parser.contains_dma_pkt;
spin_lock(&job->patched_cb->lock);
job->patched_cb->cs_cnt++;
@@ -259,6 +312,12 @@ static void cs_do_release(struct kref *ref)
spin_unlock(&hdev->hw_queues_mirror_lock);
}
+ } else if (cs->type == CS_TYPE_WAIT) {
+ /*
+ * In case the wait for signal CS was submitted, the put occurs
+ * in init_signal_wait_cs() right before hanging on the PQ.
+ */
+ dma_fence_put(cs->signal_fence);
}
/*
@@ -312,9 +371,9 @@ static void cs_timedout(struct work_struct *work)
}
static int allocate_cs(struct hl_device *hdev, struct hl_ctx *ctx,
- struct hl_cs **cs_new)
+ enum hl_cs_type cs_type, struct hl_cs **cs_new)
{
- struct hl_dma_fence *fence;
+ struct hl_cs_compl *cs_cmpl;
struct dma_fence *other = NULL;
struct hl_cs *cs;
int rc;
@@ -326,25 +385,27 @@ static int allocate_cs(struct hl_device *hdev, struct hl_ctx *ctx,
cs->ctx = ctx;
cs->submitted = false;
cs->completed = false;
+ cs->type = cs_type;
INIT_LIST_HEAD(&cs->job_list);
INIT_DELAYED_WORK(&cs->work_tdr, cs_timedout);
kref_init(&cs->refcount);
spin_lock_init(&cs->job_lock);
- fence = kmalloc(sizeof(*fence), GFP_ATOMIC);
- if (!fence) {
+ cs_cmpl = kmalloc(sizeof(*cs_cmpl), GFP_ATOMIC);
+ if (!cs_cmpl) {
rc = -ENOMEM;
goto free_cs;
}
- fence->hdev = hdev;
- spin_lock_init(&fence->lock);
- cs->fence = &fence->base_fence;
+ cs_cmpl->hdev = hdev;
+ cs_cmpl->type = cs->type;
+ spin_lock_init(&cs_cmpl->lock);
+ cs->fence = &cs_cmpl->base_fence;
spin_lock(&ctx->cs_lock);
- fence->cs_seq = ctx->cs_sequence;
- other = ctx->cs_pending[fence->cs_seq & (HL_MAX_PENDING_CS - 1)];
+ cs_cmpl->cs_seq = ctx->cs_sequence;
+ other = ctx->cs_pending[cs_cmpl->cs_seq & (HL_MAX_PENDING_CS - 1)];
if ((other) && (!dma_fence_is_signaled(other))) {
spin_unlock(&ctx->cs_lock);
dev_dbg(hdev->dev,
@@ -353,16 +414,16 @@ static int allocate_cs(struct hl_device *hdev, struct hl_ctx *ctx,
goto free_fence;
}
- dma_fence_init(&fence->base_fence, &hl_fence_ops, &fence->lock,
+ dma_fence_init(&cs_cmpl->base_fence, &hl_fence_ops, &cs_cmpl->lock,
ctx->asid, ctx->cs_sequence);
- cs->sequence = fence->cs_seq;
+ cs->sequence = cs_cmpl->cs_seq;
- ctx->cs_pending[fence->cs_seq & (HL_MAX_PENDING_CS - 1)] =
- &fence->base_fence;
+ ctx->cs_pending[cs_cmpl->cs_seq & (HL_MAX_PENDING_CS - 1)] =
+ &cs_cmpl->base_fence;
ctx->cs_sequence++;
- dma_fence_get(&fence->base_fence);
+ dma_fence_get(&cs_cmpl->base_fence);
dma_fence_put(other);
@@ -373,7 +434,7 @@ static int allocate_cs(struct hl_device *hdev, struct hl_ctx *ctx,
return 0;
free_fence:
- kfree(fence);
+ kfree(cs_cmpl);
free_cs:
kfree(cs);
return rc;
@@ -499,8 +560,8 @@ struct hl_cs_job *hl_cs_allocate_job(struct hl_device *hdev,
return job;
}
-static int _hl_cs_ioctl(struct hl_fpriv *hpriv, void __user *chunks,
- u32 num_chunks, u64 *cs_seq)
+static int cs_ioctl_default(struct hl_fpriv *hpriv, void __user *chunks,
+ u32 num_chunks, u64 *cs_seq)
{
struct hl_device *hdev = hpriv->hdev;
struct hl_cs_chunk *cs_chunk_array;
@@ -538,7 +599,7 @@ static int _hl_cs_ioctl(struct hl_fpriv *hpriv, void __user *chunks,
/* increment refcnt for context */
hl_ctx_get(hdev, hpriv->ctx);
- rc = allocate_cs(hdev, hpriv->ctx, &cs);
+ rc = allocate_cs(hdev, hpriv->ctx, CS_TYPE_DEFAULT, &cs);
if (rc) {
hl_ctx_put(hpriv->ctx);
goto free_cs_chunk_array;
@@ -652,13 +713,230 @@ out:
return rc;
}
+static int cs_ioctl_signal_wait(struct hl_fpriv *hpriv, enum hl_cs_type cs_type,
+ void __user *chunks, u32 num_chunks,
+ u64 *cs_seq)
+{
+ struct hl_device *hdev = hpriv->hdev;
+ struct hl_ctx *ctx = hpriv->ctx;
+ struct hl_cs_chunk *cs_chunk_array, *chunk;
+ struct hw_queue_properties *hw_queue_prop;
+ struct dma_fence *sig_fence = NULL;
+ struct hl_cs_job *job;
+ struct hl_cs *cs;
+ struct hl_cb *cb;
+ u64 *signal_seq_arr = NULL, signal_seq;
+ u32 size_to_copy, q_idx, signal_seq_arr_len, cb_size;
+ int rc;
+
+ *cs_seq = ULLONG_MAX;
+
+ if (num_chunks > HL_MAX_JOBS_PER_CS) {
+ dev_err(hdev->dev,
+ "Number of chunks can NOT be larger than %d\n",
+ HL_MAX_JOBS_PER_CS);
+ rc = -EINVAL;
+ goto out;
+ }
+
+ cs_chunk_array = kmalloc_array(num_chunks, sizeof(*cs_chunk_array),
+ GFP_ATOMIC);
+ if (!cs_chunk_array) {
+ rc = -ENOMEM;
+ goto out;
+ }
+
+ size_to_copy = num_chunks * sizeof(struct hl_cs_chunk);
+ if (copy_from_user(cs_chunk_array, chunks, size_to_copy)) {
+ dev_err(hdev->dev, "Failed to copy cs chunk array from user\n");
+ rc = -EFAULT;
+ goto free_cs_chunk_array;
+ }
+
+ /* currently it is guaranteed to have only one chunk */
+ chunk = &cs_chunk_array[0];
+ q_idx = chunk->queue_index;
+ hw_queue_prop = &hdev->asic_prop.hw_queues_props[q_idx];
+
+ if ((q_idx >= HL_MAX_QUEUES) ||
+ (hw_queue_prop->type != QUEUE_TYPE_EXT)) {
+ dev_err(hdev->dev, "Queue index %d is invalid\n", q_idx);
+ rc = -EINVAL;
+ goto free_cs_chunk_array;
+ }
+
+ if (cs_type == CS_TYPE_WAIT) {
+ struct hl_cs_compl *sig_waitcs_cmpl;
+
+ signal_seq_arr_len = chunk->num_signal_seq_arr;
+
+ /* currently only one signal seq is supported */
+ if (signal_seq_arr_len != 1) {
+ dev_err(hdev->dev,
+ "Wait for signal CS supports only one signal CS seq\n");
+ rc = -EINVAL;
+ goto free_cs_chunk_array;
+ }
+
+ signal_seq_arr = kmalloc_array(signal_seq_arr_len,
+ sizeof(*signal_seq_arr),
+ GFP_ATOMIC);
+ if (!signal_seq_arr) {
+ rc = -ENOMEM;
+ goto free_cs_chunk_array;
+ }
+
+ size_to_copy = chunk->num_signal_seq_arr *
+ sizeof(*signal_seq_arr);
+ if (copy_from_user(signal_seq_arr,
+ u64_to_user_ptr(chunk->signal_seq_arr),
+ size_to_copy)) {
+ dev_err(hdev->dev,
+ "Failed to copy signal seq array from user\n");
+ rc = -EFAULT;
+ goto free_signal_seq_array;
+ }
+
+ /* currently it is guaranteed to have only one signal seq */
+ signal_seq = signal_seq_arr[0];
+ sig_fence = hl_ctx_get_fence(ctx, signal_seq);
+ if (IS_ERR(sig_fence)) {
+ dev_err(hdev->dev,
+ "Failed to get signal CS with seq 0x%llx\n",
+ signal_seq);
+ rc = PTR_ERR(sig_fence);
+ goto free_signal_seq_array;
+ }
+
+ if (!sig_fence) {
+ /* signal CS already finished */
+ rc = 0;
+ goto free_signal_seq_array;
+ }
+
+ sig_waitcs_cmpl =
+ container_of(sig_fence, struct hl_cs_compl, base_fence);
+
+ if (sig_waitcs_cmpl->type != CS_TYPE_SIGNAL) {
+ dev_err(hdev->dev,
+ "CS seq 0x%llx is not of a signal CS\n",
+ signal_seq);
+ dma_fence_put(sig_fence);
+ rc = -EINVAL;
+ goto free_signal_seq_array;
+ }
+
+ if (dma_fence_is_signaled(sig_fence)) {
+ /* signal CS already finished */
+ dma_fence_put(sig_fence);
+ rc = 0;
+ goto free_signal_seq_array;
+ }
+ }
+
+ /* increment refcnt for context */
+ hl_ctx_get(hdev, ctx);
+
+ rc = allocate_cs(hdev, ctx, cs_type, &cs);
+ if (rc) {
+ if (cs_type == CS_TYPE_WAIT)
+ dma_fence_put(sig_fence);
+ hl_ctx_put(ctx);
+ goto free_signal_seq_array;
+ }
+
+ /*
+ * Save the signal CS fence for later initialization right before
+ * hanging the wait CS on the queue.
+ */
+ if (cs->type == CS_TYPE_WAIT)
+ cs->signal_fence = sig_fence;
+
+ hl_debugfs_add_cs(cs);
+
+ *cs_seq = cs->sequence;
+
+ job = hl_cs_allocate_job(hdev, QUEUE_TYPE_EXT, true);
+ if (!job) {
+ dev_err(hdev->dev, "Failed to allocate a new job\n");
+ rc = -ENOMEM;
+ goto put_cs;
+ }
+
+ cb = hl_cb_kernel_create(hdev, PAGE_SIZE);
+ if (!cb) {
+ kfree(job);
+ rc = -EFAULT;
+ goto put_cs;
+ }
+
+ if (cs->type == CS_TYPE_WAIT)
+ cb_size = hdev->asic_funcs->get_wait_cb_size(hdev);
+ else
+ cb_size = hdev->asic_funcs->get_signal_cb_size(hdev);
+
+ job->id = 0;
+ job->cs = cs;
+ job->user_cb = cb;
+ job->user_cb->cs_cnt++;
+ job->user_cb_size = cb_size;
+ job->hw_queue_id = q_idx;
+
+ /*
+ * No need in parsing, user CB is the patched CB.
+ * We call hl_cb_destroy() out of two reasons - we don't need the CB in
+ * the CB idr anymore and to decrement its refcount as it was
+ * incremented inside hl_cb_kernel_create().
+ */
+ job->patched_cb = job->user_cb;
+ job->job_cb_size = job->user_cb_size;
+ hl_cb_destroy(hdev, &hdev->kernel_cb_mgr, cb->id << PAGE_SHIFT);
+
+ cs->jobs_in_queue_cnt[job->hw_queue_id]++;
+
+ list_add_tail(&job->cs_node, &cs->job_list);
+
+ /* increment refcount as for external queues we get completion */
+ cs_get(cs);
+
+ hl_debugfs_add_job(hdev, job);
+
+ rc = hl_hw_queue_schedule_cs(cs);
+ if (rc) {
+ if (rc != -EAGAIN)
+ dev_err(hdev->dev,
+ "Failed to submit CS %d.%llu to H/W queues, error %d\n",
+ ctx->asid, cs->sequence, rc);
+ goto free_cs_object;
+ }
+
+ rc = HL_CS_STATUS_SUCCESS;
+ goto put_cs;
+
+free_cs_object:
+ cs_rollback(hdev, cs);
+ *cs_seq = ULLONG_MAX;
+ /* The path below is both for good and erroneous exits */
+put_cs:
+ /* We finished with the CS in this function, so put the ref */
+ cs_put(cs);
+free_signal_seq_array:
+ if (cs_type == CS_TYPE_WAIT)
+ kfree(signal_seq_arr);
+free_cs_chunk_array:
+ kfree(cs_chunk_array);
+out:
+ return rc;
+}
+
int hl_cs_ioctl(struct hl_fpriv *hpriv, void *data)
{
struct hl_device *hdev = hpriv->hdev;
union hl_cs_args *args = data;
struct hl_ctx *ctx = hpriv->ctx;
void __user *chunks_execute, *chunks_restore;
- u32 num_chunks_execute, num_chunks_restore;
+ enum hl_cs_type cs_type;
+ u32 num_chunks_execute, num_chunks_restore, sig_wait_flags;
u64 cs_seq = ULONG_MAX;
int rc, do_ctx_switch;
bool need_soft_reset = false;
@@ -671,12 +949,44 @@ int hl_cs_ioctl(struct hl_fpriv *hpriv, void *data)
goto out;
}
+ sig_wait_flags = args->in.cs_flags & HL_CS_FLAGS_SIG_WAIT;
+
+ if (unlikely(sig_wait_flags == HL_CS_FLAGS_SIG_WAIT)) {
+ dev_err(hdev->dev,
+ "Signal and wait CS flags are mutually exclusive, context %d\n",
+ ctx->asid);
+ rc = -EINVAL;
+ goto out;
+ }
+
+ if (unlikely((sig_wait_flags & HL_CS_FLAGS_SIG_WAIT) &&
+ (!hdev->supports_sync_stream))) {
+ dev_err(hdev->dev, "Sync stream CS is not supported\n");
+ rc = -EINVAL;
+ goto out;
+ }
+
+ if (args->in.cs_flags & HL_CS_FLAGS_SIGNAL)
+ cs_type = CS_TYPE_SIGNAL;
+ else if (args->in.cs_flags & HL_CS_FLAGS_WAIT)
+ cs_type = CS_TYPE_WAIT;
+ else
+ cs_type = CS_TYPE_DEFAULT;
+
chunks_execute = (void __user *) (uintptr_t) args->in.chunks_execute;
num_chunks_execute = args->in.num_chunks_execute;
- if (!num_chunks_execute) {
+ if (cs_type == CS_TYPE_DEFAULT) {
+ if (!num_chunks_execute) {
+ dev_err(hdev->dev,
+ "Got execute CS with 0 chunks, context %d\n",
+ ctx->asid);
+ rc = -EINVAL;
+ goto out;
+ }
+ } else if (num_chunks_execute != 1) {
dev_err(hdev->dev,
- "Got execute CS with 0 chunks, context %d\n",
+ "Sync stream CS mandates one chunk only, context %d\n",
ctx->asid);
rc = -EINVAL;
goto out;
@@ -722,7 +1032,7 @@ int hl_cs_ioctl(struct hl_fpriv *hpriv, void *data)
"Need to run restore phase but restore CS is empty\n");
rc = 0;
} else {
- rc = _hl_cs_ioctl(hpriv, chunks_restore,
+ rc = cs_ioctl_default(hpriv, chunks_restore,
num_chunks_restore, &cs_seq);
}
@@ -764,7 +1074,12 @@ int hl_cs_ioctl(struct hl_fpriv *hpriv, void *data)
}
}
- rc = _hl_cs_ioctl(hpriv, chunks_execute, num_chunks_execute, &cs_seq);
+ if (cs_type == CS_TYPE_DEFAULT)
+ rc = cs_ioctl_default(hpriv, chunks_execute, num_chunks_execute,
+ &cs_seq);
+ else
+ rc = cs_ioctl_signal_wait(hpriv, cs_type, chunks_execute,
+ num_chunks_execute, &cs_seq);
out:
if (rc != -EAGAIN) {
@@ -796,6 +1111,10 @@ static long _hl_cs_wait_ioctl(struct hl_device *hdev,
fence = hl_ctx_get_fence(ctx, seq);
if (IS_ERR(fence)) {
rc = PTR_ERR(fence);
+ if (rc == -EINVAL)
+ dev_notice_ratelimited(hdev->dev,
+ "Can't wait on seq %llu because current CS is at seq %llu\n",
+ seq, ctx->cs_sequence);
} else if (fence) {
rc = dma_fence_wait_timeout(fence, true, timeout);
if (fence->error == -ETIMEDOUT)
@@ -803,8 +1122,12 @@ static long _hl_cs_wait_ioctl(struct hl_device *hdev,
else if (fence->error == -EIO)
rc = -EIO;
dma_fence_put(fence);
- } else
+ } else {
+ dev_dbg(hdev->dev,
+ "Can't wait on seq %llu because current CS is at seq %llu (Fence is gone)\n",
+ seq, ctx->cs_sequence);
rc = 1;
+ }
hl_ctx_put(ctx);
diff --git a/drivers/misc/habanalabs/context.c b/drivers/misc/habanalabs/context.c
index 2df6fb87e7ff..ec92b3506b1f 100644
--- a/drivers/misc/habanalabs/context.c
+++ b/drivers/misc/habanalabs/context.c
@@ -170,24 +170,16 @@ int hl_ctx_put(struct hl_ctx *ctx)
struct dma_fence *hl_ctx_get_fence(struct hl_ctx *ctx, u64 seq)
{
- struct hl_device *hdev = ctx->hdev;
struct dma_fence *fence;
spin_lock(&ctx->cs_lock);
if (seq >= ctx->cs_sequence) {
- dev_notice_ratelimited(hdev->dev,
- "Can't wait on seq %llu because current CS is at seq %llu\n",
- seq, ctx->cs_sequence);
spin_unlock(&ctx->cs_lock);
return ERR_PTR(-EINVAL);
}
-
if (seq + HL_MAX_PENDING_CS < ctx->cs_sequence) {
- dev_dbg(hdev->dev,
- "Can't wait on seq %llu because current CS is at seq %llu (Fence is gone)\n",
- seq, ctx->cs_sequence);
spin_unlock(&ctx->cs_lock);
return NULL;
}
diff --git a/drivers/misc/habanalabs/debugfs.c b/drivers/misc/habanalabs/debugfs.c
index 756d36ed5d95..3c8dcdfba20c 100644
--- a/drivers/misc/habanalabs/debugfs.c
+++ b/drivers/misc/habanalabs/debugfs.c
@@ -970,6 +970,98 @@ static ssize_t hl_device_write(struct file *f, const char __user *buf,
return count;
}
+static ssize_t hl_clk_gate_read(struct file *f, char __user *buf,
+ size_t count, loff_t *ppos)
+{
+ struct hl_dbg_device_entry *entry = file_inode(f)->i_private;
+ struct hl_device *hdev = entry->hdev;
+ char tmp_buf[200];
+ ssize_t rc;
+
+ if (*ppos)
+ return 0;
+
+ sprintf(tmp_buf, "%d\n", hdev->clock_gating);
+ rc = simple_read_from_buffer(buf, strlen(tmp_buf) + 1, ppos, tmp_buf,
+ strlen(tmp_buf) + 1);
+
+ return rc;
+}
+
+static ssize_t hl_clk_gate_write(struct file *f, const char __user *buf,
+ size_t count, loff_t *ppos)
+{
+ struct hl_dbg_device_entry *entry = file_inode(f)->i_private;
+ struct hl_device *hdev = entry->hdev;
+ u32 value;
+ ssize_t rc;
+
+ if (atomic_read(&hdev->in_reset)) {
+ dev_warn_ratelimited(hdev->dev,
+ "Can't change clock gating during reset\n");
+ return 0;
+ }
+
+ rc = kstrtouint_from_user(buf, count, 10, &value);
+ if (rc)
+ return rc;
+
+ if (value) {
+ hdev->clock_gating = 1;
+ if (hdev->asic_funcs->enable_clock_gating)
+ hdev->asic_funcs->enable_clock_gating(hdev);
+ } else {
+ if (hdev->asic_funcs->disable_clock_gating)
+ hdev->asic_funcs->disable_clock_gating(hdev);
+ hdev->clock_gating = 0;
+ }
+
+ return count;
+}
+
+static ssize_t hl_stop_on_err_read(struct file *f, char __user *buf,
+ size_t count, loff_t *ppos)
+{
+ struct hl_dbg_device_entry *entry = file_inode(f)->i_private;
+ struct hl_device *hdev = entry->hdev;
+ char tmp_buf[200];
+ ssize_t rc;
+
+ if (*ppos)
+ return 0;
+
+ sprintf(tmp_buf, "%d\n", hdev->stop_on_err);
+ rc = simple_read_from_buffer(buf, strlen(tmp_buf) + 1, ppos, tmp_buf,
+ strlen(tmp_buf) + 1);
+
+ return rc;
+}
+
+static ssize_t hl_stop_on_err_write(struct file *f, const char __user *buf,
+ size_t count, loff_t *ppos)
+{
+ struct hl_dbg_device_entry *entry = file_inode(f)->i_private;
+ struct hl_device *hdev = entry->hdev;
+ u32 value;
+ ssize_t rc;
+
+ if (atomic_read(&hdev->in_reset)) {
+ dev_warn_ratelimited(hdev->dev,
+ "Can't change stop on error during reset\n");
+ return 0;
+ }
+
+ rc = kstrtouint_from_user(buf, count, 10, &value);
+ if (rc)
+ return rc;
+
+ hdev->stop_on_err = value ? 1 : 0;
+
+ hl_device_reset(hdev, false, false);
+
+ return count;
+}
+
static const struct file_operations hl_data32b_fops = {
.owner = THIS_MODULE,
.read = hl_data_read32,
@@ -1015,6 +1107,18 @@ static const struct file_operations hl_device_fops = {
.write = hl_device_write
};
+static const struct file_operations hl_clk_gate_fops = {
+ .owner = THIS_MODULE,
+ .read = hl_clk_gate_read,
+ .write = hl_clk_gate_write
+};
+
+static const struct file_operations hl_stop_on_err_fops = {
+ .owner = THIS_MODULE,
+ .read = hl_stop_on_err_read,
+ .write = hl_stop_on_err_write
+};
+
static const struct hl_info_list hl_debugfs_list[] = {
{"command_buffers", command_buffers_show, NULL},
{"command_submission", command_submission_show, NULL},
@@ -1152,6 +1256,18 @@ void hl_debugfs_add_device(struct hl_device *hdev)
dev_entry,
&hl_device_fops);
+ debugfs_create_file("clk_gate",
+ 0200,
+ dev_entry->root,
+ dev_entry,
+ &hl_clk_gate_fops);
+
+ debugfs_create_file("stop_on_err",
+ 0644,
+ dev_entry->root,
+ dev_entry,
+ &hl_stop_on_err_fops);
+
for (i = 0, entry = dev_entry->entry_arr ; i < count ; i++, entry++) {
ent = debugfs_create_file(hl_debugfs_list[i].name,
diff --git a/drivers/misc/habanalabs/device.c b/drivers/misc/habanalabs/device.c
index aef4de36b7aa..2b38a119704c 100644
--- a/drivers/misc/habanalabs/device.c
+++ b/drivers/misc/habanalabs/device.c
@@ -256,6 +256,10 @@ static int device_early_init(struct hl_device *hdev)
goya_set_asic_funcs(hdev);
strlcpy(hdev->asic_name, "GOYA", sizeof(hdev->asic_name));
break;
+ case ASIC_GAUDI:
+ gaudi_set_asic_funcs(hdev);
+ sprintf(hdev->asic_name, "GAUDI");
+ break;
default:
dev_err(hdev->dev, "Unrecognized ASIC type %d\n",
hdev->asic_type);
@@ -603,6 +607,9 @@ int hl_device_set_debug_mode(struct hl_device *hdev, bool enable)
hdev->in_debug = 0;
+ if (!hdev->hard_reset_pending)
+ hdev->asic_funcs->enable_clock_gating(hdev);
+
goto out;
}
@@ -613,6 +620,7 @@ int hl_device_set_debug_mode(struct hl_device *hdev, bool enable)
goto out;
}
+ hdev->asic_funcs->disable_clock_gating(hdev);
hdev->in_debug = 1;
out:
@@ -718,7 +726,7 @@ disable_device:
return rc;
}
-static void device_kill_open_processes(struct hl_device *hdev)
+static int device_kill_open_processes(struct hl_device *hdev)
{
u16 pending_total, pending_cnt;
struct hl_fpriv *hpriv;
@@ -771,9 +779,7 @@ static void device_kill_open_processes(struct hl_device *hdev)
ssleep(1);
}
- if (!list_empty(&hdev->fpriv_list))
- dev_crit(hdev->dev,
- "Going to hard reset with open user contexts\n");
+ return list_empty(&hdev->fpriv_list) ? 0 : -EBUSY;
}
static void device_hard_reset_pending(struct work_struct *work)
@@ -793,6 +799,7 @@ static void device_hard_reset_pending(struct work_struct *work)
* @hdev: pointer to habanalabs device structure
* @hard_reset: should we do hard reset to all engines or just reset the
* compute/dma engines
+ * @from_hard_reset_thread: is the caller the hard-reset thread
*
* Block future CS and wait for pending CS to be enqueued
* Call ASIC H/W fini
@@ -815,6 +822,11 @@ int hl_device_reset(struct hl_device *hdev, bool hard_reset,
return 0;
}
+ if ((!hard_reset) && (!hdev->supports_soft_reset)) {
+ dev_dbg(hdev->dev, "Doing hard-reset instead of soft-reset\n");
+ hard_reset = true;
+ }
+
/*
* Prevent concurrency in this function - only one reset should be
* done at any given time. Only need to perform this if we didn't
@@ -894,7 +906,12 @@ again:
* process can't really exit until all its CSs are done, which
* is what we do in cs rollback
*/
- device_kill_open_processes(hdev);
+ rc = device_kill_open_processes(hdev);
+ if (rc) {
+ dev_crit(hdev->dev,
+ "Failed to kill all open processes, stopping hard reset\n");
+ goto out_err;
+ }
/* Flush the Event queue workers to make sure no other thread is
* reading or writing to registers during the reset
@@ -1062,7 +1079,7 @@ out_err:
*/
int hl_device_init(struct hl_device *hdev, struct class *hclass)
{
- int i, rc, cq_ready_cnt;
+ int i, rc, cq_cnt, cq_ready_cnt;
char *name;
bool add_cdev_sysfs_on_err = false;
@@ -1120,14 +1137,16 @@ int hl_device_init(struct hl_device *hdev, struct class *hclass)
goto sw_fini;
}
+ cq_cnt = hdev->asic_prop.completion_queues_count;
+
/*
* Initialize the completion queues. Must be done before hw_init,
* because there the addresses of the completion queues are being
* passed as arguments to request_irq
*/
- hdev->completion_queue =
- kcalloc(hdev->asic_prop.completion_queues_count,
- sizeof(*hdev->completion_queue), GFP_KERNEL);
+ hdev->completion_queue = kcalloc(cq_cnt,
+ sizeof(*hdev->completion_queue),
+ GFP_KERNEL);
if (!hdev->completion_queue) {
dev_err(hdev->dev, "failed to allocate completion queues\n");
@@ -1135,10 +1154,9 @@ int hl_device_init(struct hl_device *hdev, struct class *hclass)
goto hw_queues_destroy;
}
- for (i = 0, cq_ready_cnt = 0;
- i < hdev->asic_prop.completion_queues_count;
- i++, cq_ready_cnt++) {
- rc = hl_cq_init(hdev, &hdev->completion_queue[i], i);
+ for (i = 0, cq_ready_cnt = 0 ; i < cq_cnt ; i++, cq_ready_cnt++) {
+ rc = hl_cq_init(hdev, &hdev->completion_queue[i],
+ hdev->asic_funcs->get_queue_id_for_cq(hdev, i));
if (rc) {
dev_err(hdev->dev,
"failed to initialize completion queue\n");
@@ -1325,11 +1343,12 @@ void hl_device_fini(struct hl_device *hdev)
* This function is competing with the reset function, so try to
* take the reset atomic and if we are already in middle of reset,
* wait until reset function is finished. Reset function is designed
- * to always finish (could take up to a few seconds in worst case).
+ * to always finish. However, in Gaudi, because of all the network
+ * ports, the hard reset could take between 10-30 seconds
*/
timeout = ktime_add_us(ktime_get(),
- HL_PENDING_RESET_PER_SEC * 1000 * 1000 * 4);
+ HL_HARD_RESET_MAX_TIMEOUT * 1000 * 1000);
rc = atomic_cmpxchg(&hdev->in_reset, 0, 1);
while (rc) {
usleep_range(50, 200);
@@ -1375,7 +1394,9 @@ void hl_device_fini(struct hl_device *hdev)
* can't really exit until all its CSs are done, which is what we
* do in cs rollback
*/
- device_kill_open_processes(hdev);
+ rc = device_kill_open_processes(hdev);
+ if (rc)
+ dev_crit(hdev->dev, "Failed to kill all open processes\n");
hl_cb_pool_fini(hdev);
diff --git a/drivers/misc/habanalabs/firmware_if.c b/drivers/misc/habanalabs/firmware_if.c
index f5bd03171dac..baf790cf4b78 100644
--- a/drivers/misc/habanalabs/firmware_if.c
+++ b/drivers/misc/habanalabs/firmware_if.c
@@ -6,20 +6,22 @@
*/
#include "habanalabs.h"
+#include "include/hl_boot_if.h"
#include <linux/firmware.h>
#include <linux/genalloc.h>
#include <linux/io-64-nonatomic-lo-hi.h>
+#include <linux/slab.h>
/**
- * hl_fw_push_fw_to_device() - Push FW code to device.
+ * hl_fw_load_fw_to_device() - Load F/W code to device's memory.
* @hdev: pointer to hl_device structure.
*
* Copy fw code from firmware file to device memory.
*
* Return: 0 on success, non-zero for failure.
*/
-int hl_fw_push_fw_to_device(struct hl_device *hdev, const char *fw_name,
+int hl_fw_load_fw_to_device(struct hl_device *hdev, const char *fw_name,
void __iomem *dst)
{
const struct firmware *fw;
@@ -129,6 +131,68 @@ out:
return rc;
}
+int hl_fw_unmask_irq(struct hl_device *hdev, u16 event_type)
+{
+ struct armcp_packet pkt;
+ long result;
+ int rc;
+
+ memset(&pkt, 0, sizeof(pkt));
+
+ pkt.ctl = cpu_to_le32(ARMCP_PACKET_UNMASK_RAZWI_IRQ <<
+ ARMCP_PKT_CTL_OPCODE_SHIFT);
+ pkt.value = cpu_to_le64(event_type);
+
+ rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt),
+ HL_DEVICE_TIMEOUT_USEC, &result);
+
+ if (rc)
+ dev_err(hdev->dev, "failed to unmask RAZWI IRQ %d", event_type);
+
+ return rc;
+}
+
+int hl_fw_unmask_irq_arr(struct hl_device *hdev, const u32 *irq_arr,
+ size_t irq_arr_size)
+{
+ struct armcp_unmask_irq_arr_packet *pkt;
+ size_t total_pkt_size;
+ long result;
+ int rc;
+
+ total_pkt_size = sizeof(struct armcp_unmask_irq_arr_packet) +
+ irq_arr_size;
+
+ /* data should be aligned to 8 bytes in order to ArmCP to copy it */
+ total_pkt_size = (total_pkt_size + 0x7) & ~0x7;
+
+ /* total_pkt_size is casted to u16 later on */
+ if (total_pkt_size > USHRT_MAX) {
+ dev_err(hdev->dev, "too many elements in IRQ array\n");
+ return -EINVAL;
+ }
+
+ pkt = kzalloc(total_pkt_size, GFP_KERNEL);
+ if (!pkt)
+ return -ENOMEM;
+
+ pkt->length = cpu_to_le32(irq_arr_size / sizeof(irq_arr[0]));
+ memcpy(&pkt->irqs, irq_arr, irq_arr_size);
+
+ pkt->armcp_pkt.ctl = cpu_to_le32(ARMCP_PACKET_UNMASK_RAZWI_IRQ_ARRAY <<
+ ARMCP_PKT_CTL_OPCODE_SHIFT);
+
+ rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) pkt,
+ total_pkt_size, HL_DEVICE_TIMEOUT_USEC, &result);
+
+ if (rc)
+ dev_err(hdev->dev, "failed to unmask IRQ array\n");
+
+ kfree(pkt);
+
+ return rc;
+}
+
int hl_fw_test_cpu_queue(struct hl_device *hdev)
{
struct armcp_packet test_pkt = {};
@@ -286,3 +350,232 @@ out:
return rc;
}
+
+static void fw_read_errors(struct hl_device *hdev, u32 boot_err0_reg)
+{
+ u32 err_val;
+
+ /* Some of the firmware status codes are deprecated in newer f/w
+ * versions. In those versions, the errors are reported
+ * in different registers. Therefore, we need to check those
+ * registers and print the exact errors. Moreover, there
+ * may be multiple errors, so we need to report on each error
+ * separately. Some of the error codes might indicate a state
+ * that is not an error per-se, but it is an error in production
+ * environment
+ */
+ err_val = RREG32(boot_err0_reg);
+ if (!(err_val & CPU_BOOT_ERR0_ENABLED))
+ return;
+
+ if (err_val & CPU_BOOT_ERR0_DRAM_INIT_FAIL)
+ dev_err(hdev->dev,
+ "Device boot error - DRAM initialization failed\n");
+ if (err_val & CPU_BOOT_ERR0_FIT_CORRUPTED)
+ dev_err(hdev->dev, "Device boot error - FIT image corrupted\n");
+ if (err_val & CPU_BOOT_ERR0_TS_INIT_FAIL)
+ dev_err(hdev->dev,
+ "Device boot error - Thermal Sensor initialization failed\n");
+ if (err_val & CPU_BOOT_ERR0_DRAM_SKIPPED)
+ dev_warn(hdev->dev,
+ "Device boot warning - Skipped DRAM initialization\n");
+ if (err_val & CPU_BOOT_ERR0_BMC_WAIT_SKIPPED)
+ dev_warn(hdev->dev,
+ "Device boot error - Skipped waiting for BMC\n");
+ if (err_val & CPU_BOOT_ERR0_NIC_DATA_NOT_RDY)
+ dev_err(hdev->dev,
+ "Device boot error - Serdes data from BMC not available\n");
+ if (err_val & CPU_BOOT_ERR0_NIC_FW_FAIL)
+ dev_err(hdev->dev,
+ "Device boot error - NIC F/W initialization failed\n");
+}
+
+int hl_fw_init_cpu(struct hl_device *hdev, u32 cpu_boot_status_reg,
+ u32 msg_to_cpu_reg, u32 cpu_msg_status_reg,
+ u32 boot_err0_reg, bool skip_bmc,
+ u32 cpu_timeout, u32 boot_fit_timeout)
+{
+ u32 status;
+ int rc;
+
+ dev_info(hdev->dev, "Going to wait for device boot (up to %lds)\n",
+ cpu_timeout / USEC_PER_SEC);
+
+ /* Wait for boot FIT request */
+ rc = hl_poll_timeout(
+ hdev,
+ cpu_boot_status_reg,
+ status,
+ status == CPU_BOOT_STATUS_WAITING_FOR_BOOT_FIT,
+ 10000,
+ boot_fit_timeout);
+
+ if (rc) {
+ dev_dbg(hdev->dev,
+ "No boot fit request received, resuming boot\n");
+ } else {
+ rc = hdev->asic_funcs->load_boot_fit_to_device(hdev);
+ if (rc)
+ goto out;
+
+ /* Clear device CPU message status */
+ WREG32(cpu_msg_status_reg, CPU_MSG_CLR);
+
+ /* Signal device CPU that boot loader is ready */
+ WREG32(msg_to_cpu_reg, KMD_MSG_FIT_RDY);
+
+ /* Poll for CPU device ack */
+ rc = hl_poll_timeout(
+ hdev,
+ cpu_msg_status_reg,
+ status,
+ status == CPU_MSG_OK,
+ 10000,
+ boot_fit_timeout);
+
+ if (rc) {
+ dev_err(hdev->dev,
+ "Timeout waiting for boot fit load ack\n");
+ goto out;
+ }
+
+ /* Clear message */
+ WREG32(msg_to_cpu_reg, KMD_MSG_NA);
+ }
+
+ /* Make sure CPU boot-loader is running */
+ rc = hl_poll_timeout(
+ hdev,
+ cpu_boot_status_reg,
+ status,
+ (status == CPU_BOOT_STATUS_DRAM_RDY) ||
+ (status == CPU_BOOT_STATUS_NIC_FW_RDY) ||
+ (status == CPU_BOOT_STATUS_READY_TO_BOOT) ||
+ (status == CPU_BOOT_STATUS_SRAM_AVAIL),
+ 10000,
+ cpu_timeout);
+
+ /* Read U-Boot, preboot versions now in case we will later fail */
+ hdev->asic_funcs->read_device_fw_version(hdev, FW_COMP_UBOOT);
+ hdev->asic_funcs->read_device_fw_version(hdev, FW_COMP_PREBOOT);
+
+ /* Some of the status codes below are deprecated in newer f/w
+ * versions but we keep them here for backward compatibility
+ */
+ if (rc) {
+ switch (status) {
+ case CPU_BOOT_STATUS_NA:
+ dev_err(hdev->dev,
+ "Device boot error - BTL did NOT run\n");
+ break;
+ case CPU_BOOT_STATUS_IN_WFE:
+ dev_err(hdev->dev,
+ "Device boot error - Stuck inside WFE loop\n");
+ break;
+ case CPU_BOOT_STATUS_IN_BTL:
+ dev_err(hdev->dev,
+ "Device boot error - Stuck in BTL\n");
+ break;
+ case CPU_BOOT_STATUS_IN_PREBOOT:
+ dev_err(hdev->dev,
+ "Device boot error - Stuck in Preboot\n");
+ break;
+ case CPU_BOOT_STATUS_IN_SPL:
+ dev_err(hdev->dev,
+ "Device boot error - Stuck in SPL\n");
+ break;
+ case CPU_BOOT_STATUS_IN_UBOOT:
+ dev_err(hdev->dev,
+ "Device boot error - Stuck in u-boot\n");
+ break;
+ case CPU_BOOT_STATUS_DRAM_INIT_FAIL:
+ dev_err(hdev->dev,
+ "Device boot error - DRAM initialization failed\n");
+ break;
+ case CPU_BOOT_STATUS_UBOOT_NOT_READY:
+ dev_err(hdev->dev,
+ "Device boot error - u-boot stopped by user\n");
+ break;
+ case CPU_BOOT_STATUS_TS_INIT_FAIL:
+ dev_err(hdev->dev,
+ "Device boot error - Thermal Sensor initialization failed\n");
+ break;
+ default:
+ dev_err(hdev->dev,
+ "Device boot error - Invalid status code %d\n",
+ status);
+ break;
+ }
+
+ rc = -EIO;
+ goto out;
+ }
+
+ if (!hdev->fw_loading) {
+ dev_info(hdev->dev, "Skip loading FW\n");
+ goto out;
+ }
+
+ if (status == CPU_BOOT_STATUS_SRAM_AVAIL)
+ goto out;
+
+ dev_info(hdev->dev,
+ "Loading firmware to device, may take some time...\n");
+
+ rc = hdev->asic_funcs->load_firmware_to_device(hdev);
+ if (rc)
+ goto out;
+
+ if (skip_bmc) {
+ WREG32(msg_to_cpu_reg, KMD_MSG_SKIP_BMC);
+
+ rc = hl_poll_timeout(
+ hdev,
+ cpu_boot_status_reg,
+ status,
+ (status == CPU_BOOT_STATUS_BMC_WAITING_SKIPPED),
+ 10000,
+ cpu_timeout);
+
+ if (rc) {
+ dev_err(hdev->dev,
+ "Failed to get ACK on skipping BMC, %d\n",
+ status);
+ WREG32(msg_to_cpu_reg, KMD_MSG_NA);
+ rc = -EIO;
+ goto out;
+ }
+ }
+
+ WREG32(msg_to_cpu_reg, KMD_MSG_FIT_RDY);
+
+ rc = hl_poll_timeout(
+ hdev,
+ cpu_boot_status_reg,
+ status,
+ (status == CPU_BOOT_STATUS_SRAM_AVAIL),
+ 10000,
+ cpu_timeout);
+
+ /* Clear message */
+ WREG32(msg_to_cpu_reg, KMD_MSG_NA);
+
+ if (rc) {
+ if (status == CPU_BOOT_STATUS_FIT_CORRUPTED)
+ dev_err(hdev->dev,
+ "Device reports FIT image is corrupted\n");
+ else
+ dev_err(hdev->dev,
+ "Device failed to load, %d\n", status);
+
+ rc = -EIO;
+ goto out;
+ }
+
+ dev_info(hdev->dev, "Successfully loaded firmware to device\n");
+
+out:
+ fw_read_errors(hdev, boot_err0_reg);
+
+ return rc;
+}
diff --git a/drivers/misc/habanalabs/gaudi/Makefile b/drivers/misc/habanalabs/gaudi/Makefile
new file mode 100644
index 000000000000..f802cdc980ca
--- /dev/null
+++ b/drivers/misc/habanalabs/gaudi/Makefile
@@ -0,0 +1,5 @@
+# SPDX-License-Identifier: GPL-2.0-only
+subdir-ccflags-y += -I$(src)
+
+HL_GAUDI_FILES := gaudi/gaudi.o gaudi/gaudi_hwmgr.o gaudi/gaudi_security.o \
+ gaudi/gaudi_coresight.o
diff --git a/drivers/misc/habanalabs/gaudi/gaudi.c b/drivers/misc/habanalabs/gaudi/gaudi.c
new file mode 100644
index 000000000000..61f88e9884ce
--- /dev/null
+++ b/drivers/misc/habanalabs/gaudi/gaudi.c
@@ -0,0 +1,6748 @@
+// SPDX-License-Identifier: GPL-2.0
+
+/*
+ * Copyright 2016-2020 HabanaLabs, Ltd.
+ * All Rights Reserved.
+ */
+
+#include "gaudiP.h"
+#include "include/hw_ip/mmu/mmu_general.h"
+#include "include/hw_ip/mmu/mmu_v1_1.h"
+#include "include/gaudi/gaudi_masks.h"
+#include "include/gaudi/gaudi_fw_if.h"
+#include "include/gaudi/gaudi_reg_map.h"
+#include "include/gaudi/gaudi_async_ids_map_extended.h"
+
+#include <linux/module.h>
+#include <linux/pci.h>
+#include <linux/firmware.h>
+#include <linux/hwmon.h>
+#include <linux/genalloc.h>
+#include <linux/io-64-nonatomic-lo-hi.h>
+#include <linux/iommu.h>
+#include <linux/seq_file.h>
+
+/*
+ * Gaudi security scheme:
+ *
+ * 1. Host is protected by:
+ * - Range registers
+ * - MMU
+ *
+ * 2. DDR is protected by:
+ * - Range registers (protect the first 512MB)
+ *
+ * 3. Configuration is protected by:
+ * - Range registers
+ * - Protection bits
+ *
+ * MMU is always enabled.
+ *
+ * QMAN DMA channels 0,1,5 (PCI DMAN):
+ * - DMA is not secured.
+ * - PQ and CQ are secured.
+ * - CP is secured: The driver needs to parse CB but WREG should be allowed
+ * because of TDMA (tensor DMA). Hence, WREG is always not
+ * secured.
+ *
+ * When the driver needs to use DMA it will check that Gaudi is idle, set DMA
+ * channel 0 to be secured, execute the DMA and change it back to not secured.
+ * Currently, the driver doesn't use the DMA while there are compute jobs
+ * running.
+ *
+ * The current use cases for the driver to use the DMA are:
+ * - Clear SRAM on context switch (happens on context switch when device is
+ * idle)
+ * - MMU page tables area clear (happens on init)
+ *
+ * QMAN DMA 2-4,6,7, TPC, MME, NIC:
+ * PQ is secured and is located on the Host (HBM CON TPC3 bug)
+ * CQ, CP and the engine are not secured
+ *
+ */
+
+#define GAUDI_BOOT_FIT_FILE "habanalabs/gaudi/gaudi-boot-fit.itb"
+#define GAUDI_LINUX_FW_FILE "habanalabs/gaudi/gaudi-fit.itb"
+#define GAUDI_TPC_FW_FILE "habanalabs/gaudi/gaudi_tpc.bin"
+
+#define GAUDI_DMA_POOL_BLK_SIZE 0x100 /* 256 bytes */
+
+#define GAUDI_RESET_TIMEOUT_MSEC 1000 /* 1000ms */
+#define GAUDI_RESET_WAIT_MSEC 1 /* 1ms */
+#define GAUDI_CPU_RESET_WAIT_MSEC 200 /* 200ms */
+#define GAUDI_TEST_QUEUE_WAIT_USEC 100000 /* 100ms */
+
+#define GAUDI_PLDM_RESET_WAIT_MSEC 1000 /* 1s */
+#define GAUDI_PLDM_HRESET_TIMEOUT_MSEC 20000 /* 20s */
+#define GAUDI_PLDM_SRESET_TIMEOUT_MSEC 14000 /* 14s */
+#define GAUDI_PLDM_TEST_QUEUE_WAIT_USEC 1000000 /* 1s */
+#define GAUDI_PLDM_MMU_TIMEOUT_USEC (MMU_CONFIG_TIMEOUT_USEC * 100)
+#define GAUDI_PLDM_QMAN0_TIMEOUT_USEC (HL_DEVICE_TIMEOUT_USEC * 30)
+#define GAUDI_PLDM_TPC_KERNEL_WAIT_USEC (HL_DEVICE_TIMEOUT_USEC * 30)
+#define GAUDI_BOOT_FIT_REQ_TIMEOUT_USEC 1000000 /* 1s */
+
+#define GAUDI_QMAN0_FENCE_VAL 0x72E91AB9
+
+#define GAUDI_MAX_STRING_LEN 20
+
+#define GAUDI_CB_POOL_CB_CNT 512
+#define GAUDI_CB_POOL_CB_SIZE 0x20000 /* 128KB */
+
+#define GAUDI_ALLOC_CPU_MEM_RETRY_CNT 3
+
+#define GAUDI_NUM_OF_TPC_INTR_CAUSE 20
+
+#define GAUDI_NUM_OF_QM_ERR_CAUSE 16
+
+#define GAUDI_NUM_OF_QM_ARB_ERR_CAUSE 3
+
+#define GAUDI_ARB_WDT_TIMEOUT 0x400000
+
+static const char gaudi_irq_name[GAUDI_MSI_ENTRIES][GAUDI_MAX_STRING_LEN] = {
+ "gaudi cq 0_0", "gaudi cq 0_1", "gaudi cq 0_2", "gaudi cq 0_3",
+ "gaudi cq 1_0", "gaudi cq 1_1", "gaudi cq 1_2", "gaudi cq 1_3",
+ "gaudi cq 5_0", "gaudi cq 5_1", "gaudi cq 5_2", "gaudi cq 5_3",
+ "gaudi cpu eq"
+};
+
+static const u8 gaudi_dma_assignment[GAUDI_DMA_MAX] = {
+ [GAUDI_PCI_DMA_1] = 0,
+ [GAUDI_PCI_DMA_2] = 1,
+ [GAUDI_PCI_DMA_3] = 5,
+ [GAUDI_HBM_DMA_1] = 2,
+ [GAUDI_HBM_DMA_2] = 3,
+ [GAUDI_HBM_DMA_3] = 4,
+ [GAUDI_HBM_DMA_4] = 6,
+ [GAUDI_HBM_DMA_5] = 7
+};
+
+static const u8 gaudi_cq_assignment[NUMBER_OF_CMPLT_QUEUES] = {
+ [0] = GAUDI_QUEUE_ID_DMA_0_0,
+ [1] = GAUDI_QUEUE_ID_DMA_0_1,
+ [2] = GAUDI_QUEUE_ID_DMA_0_2,
+ [3] = GAUDI_QUEUE_ID_DMA_0_3,
+ [4] = GAUDI_QUEUE_ID_DMA_1_0,
+ [5] = GAUDI_QUEUE_ID_DMA_1_1,
+ [6] = GAUDI_QUEUE_ID_DMA_1_2,
+ [7] = GAUDI_QUEUE_ID_DMA_1_3,
+ [8] = GAUDI_QUEUE_ID_DMA_5_0,
+ [9] = GAUDI_QUEUE_ID_DMA_5_1,
+ [10] = GAUDI_QUEUE_ID_DMA_5_2,
+ [11] = GAUDI_QUEUE_ID_DMA_5_3
+};
+
+static const u16 gaudi_packet_sizes[MAX_PACKET_ID] = {
+ [PACKET_WREG_32] = sizeof(struct packet_wreg32),
+ [PACKET_WREG_BULK] = sizeof(struct packet_wreg_bulk),
+ [PACKET_MSG_LONG] = sizeof(struct packet_msg_long),
+ [PACKET_MSG_SHORT] = sizeof(struct packet_msg_short),
+ [PACKET_CP_DMA] = sizeof(struct packet_cp_dma),
+ [PACKET_REPEAT] = sizeof(struct packet_repeat),
+ [PACKET_MSG_PROT] = sizeof(struct packet_msg_prot),
+ [PACKET_FENCE] = sizeof(struct packet_fence),
+ [PACKET_LIN_DMA] = sizeof(struct packet_lin_dma),
+ [PACKET_NOP] = sizeof(struct packet_nop),
+ [PACKET_STOP] = sizeof(struct packet_stop),
+ [PACKET_ARB_POINT] = sizeof(struct packet_arb_point),
+ [PACKET_WAIT] = sizeof(struct packet_wait),
+ [PACKET_LOAD_AND_EXE] = sizeof(struct packet_load_and_exe)
+};
+
+static const char * const
+gaudi_tpc_interrupts_cause[GAUDI_NUM_OF_TPC_INTR_CAUSE] = {
+ "tpc_address_exceed_slm",
+ "tpc_div_by_0",
+ "tpc_spu_mac_overflow",
+ "tpc_spu_addsub_overflow",
+ "tpc_spu_abs_overflow",
+ "tpc_spu_fp_dst_nan_inf",
+ "tpc_spu_fp_dst_denorm",
+ "tpc_vpu_mac_overflow",
+ "tpc_vpu_addsub_overflow",
+ "tpc_vpu_abs_overflow",
+ "tpc_vpu_fp_dst_nan_inf",
+ "tpc_vpu_fp_dst_denorm",
+ "tpc_assertions",
+ "tpc_illegal_instruction",
+ "tpc_pc_wrap_around",
+ "tpc_qm_sw_err",
+ "tpc_hbw_rresp_err",
+ "tpc_hbw_bresp_err",
+ "tpc_lbw_rresp_err",
+ "tpc_lbw_bresp_err"
+};
+
+static const char * const
+gaudi_qman_error_cause[GAUDI_NUM_OF_QM_ERR_CAUSE] = {
+ "PQ AXI HBW error",
+ "CQ AXI HBW error",
+ "CP AXI HBW error",
+ "CP error due to undefined OPCODE",
+ "CP encountered STOP OPCODE",
+ "CP AXI LBW error",
+ "CP WRREG32 or WRBULK returned error",
+ "N/A",
+ "FENCE 0 inc over max value and clipped",
+ "FENCE 1 inc over max value and clipped",
+ "FENCE 2 inc over max value and clipped",
+ "FENCE 3 inc over max value and clipped",
+ "FENCE 0 dec under min value and clipped",
+ "FENCE 1 dec under min value and clipped",
+ "FENCE 2 dec under min value and clipped",
+ "FENCE 3 dec under min value and clipped"
+};
+
+static const char * const
+gaudi_qman_arb_error_cause[GAUDI_NUM_OF_QM_ARB_ERR_CAUSE] = {
+ "Choice push while full error",
+ "Choice Q watchdog error",
+ "MSG AXI LBW returned with error"
+};
+
+static enum hl_queue_type gaudi_queue_type[GAUDI_QUEUE_ID_SIZE] = {
+ QUEUE_TYPE_EXT, /* GAUDI_QUEUE_ID_DMA_0_0 */
+ QUEUE_TYPE_EXT, /* GAUDI_QUEUE_ID_DMA_0_1 */
+ QUEUE_TYPE_EXT, /* GAUDI_QUEUE_ID_DMA_0_2 */
+ QUEUE_TYPE_EXT, /* GAUDI_QUEUE_ID_DMA_0_3 */
+ QUEUE_TYPE_EXT, /* GAUDI_QUEUE_ID_DMA_1_0 */
+ QUEUE_TYPE_EXT, /* GAUDI_QUEUE_ID_DMA_1_1 */
+ QUEUE_TYPE_EXT, /* GAUDI_QUEUE_ID_DMA_1_2 */
+ QUEUE_TYPE_EXT, /* GAUDI_QUEUE_ID_DMA_1_3 */
+ QUEUE_TYPE_CPU, /* GAUDI_QUEUE_ID_CPU_PQ */
+ QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_DMA_2_0 */
+ QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_DMA_2_1 */
+ QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_DMA_2_2 */
+ QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_DMA_2_3 */
+ QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_DMA_3_0 */
+ QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_DMA_3_1 */
+ QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_DMA_3_2 */
+ QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_DMA_3_3 */
+ QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_DMA_4_0 */
+ QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_DMA_4_1 */
+ QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_DMA_4_2 */
+ QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_DMA_4_3 */
+ QUEUE_TYPE_EXT, /* GAUDI_QUEUE_ID_DMA_5_0 */
+ QUEUE_TYPE_EXT, /* GAUDI_QUEUE_ID_DMA_5_1 */
+ QUEUE_TYPE_EXT, /* GAUDI_QUEUE_ID_DMA_5_2 */
+ QUEUE_TYPE_EXT, /* GAUDI_QUEUE_ID_DMA_5_3 */
+ QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_DMA_6_0 */
+ QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_DMA_6_1 */
+ QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_DMA_6_2 */
+ QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_DMA_6_3 */
+ QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_DMA_7_0 */
+ QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_DMA_7_1 */
+ QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_DMA_7_2 */
+ QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_DMA_7_3 */
+ QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_MME_0_0 */
+ QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_MME_0_1 */
+ QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_MME_0_2 */
+ QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_MME_0_3 */
+ QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_MME_1_0 */
+ QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_MME_1_1 */
+ QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_MME_1_2 */
+ QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_MME_1_3 */
+ QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_TPC_0_0 */
+ QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_TPC_0_1 */
+ QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_TPC_0_2 */
+ QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_TPC_0_3 */
+ QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_TPC_1_0 */
+ QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_TPC_1_1 */
+ QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_TPC_1_2 */
+ QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_TPC_1_3 */
+ QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_TPC_2_0 */
+ QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_TPC_2_1 */
+ QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_TPC_2_2 */
+ QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_TPC_2_3 */
+ QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_TPC_3_0 */
+ QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_TPC_3_1 */
+ QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_TPC_3_2 */
+ QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_TPC_3_3 */
+ QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_TPC_4_0 */
+ QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_TPC_4_1 */
+ QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_TPC_4_2 */
+ QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_TPC_4_3 */
+ QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_TPC_5_0 */
+ QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_TPC_5_1 */
+ QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_TPC_5_2 */
+ QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_TPC_5_3 */
+ QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_TPC_6_0 */
+ QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_TPC_6_1 */
+ QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_TPC_6_2 */
+ QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_TPC_6_3 */
+ QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_TPC_7_0 */
+ QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_TPC_7_1 */
+ QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_TPC_7_2 */
+ QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_TPC_7_3 */
+ QUEUE_TYPE_NA, /* GAUDI_QUEUE_ID_NIC_0_0 */
+ QUEUE_TYPE_NA, /* GAUDI_QUEUE_ID_NIC_0_1 */
+ QUEUE_TYPE_NA, /* GAUDI_QUEUE_ID_NIC_0_2 */
+ QUEUE_TYPE_NA, /* GAUDI_QUEUE_ID_NIC_0_3 */
+ QUEUE_TYPE_NA, /* GAUDI_QUEUE_ID_NIC_1_0 */
+ QUEUE_TYPE_NA, /* GAUDI_QUEUE_ID_NIC_1_1 */
+ QUEUE_TYPE_NA, /* GAUDI_QUEUE_ID_NIC_1_2 */
+ QUEUE_TYPE_NA, /* GAUDI_QUEUE_ID_NIC_1_3 */
+ QUEUE_TYPE_NA, /* GAUDI_QUEUE_ID_NIC_2_0 */
+ QUEUE_TYPE_NA, /* GAUDI_QUEUE_ID_NIC_2_1 */
+ QUEUE_TYPE_NA, /* GAUDI_QUEUE_ID_NIC_2_2 */
+ QUEUE_TYPE_NA, /* GAUDI_QUEUE_ID_NIC_2_3 */
+ QUEUE_TYPE_NA, /* GAUDI_QUEUE_ID_NIC_3_0 */
+ QUEUE_TYPE_NA, /* GAUDI_QUEUE_ID_NIC_3_1 */
+ QUEUE_TYPE_NA, /* GAUDI_QUEUE_ID_NIC_3_2 */
+ QUEUE_TYPE_NA, /* GAUDI_QUEUE_ID_NIC_3_3 */
+ QUEUE_TYPE_NA, /* GAUDI_QUEUE_ID_NIC_4_0 */
+ QUEUE_TYPE_NA, /* GAUDI_QUEUE_ID_NIC_4_1 */
+ QUEUE_TYPE_NA, /* GAUDI_QUEUE_ID_NIC_4_2 */
+ QUEUE_TYPE_NA, /* GAUDI_QUEUE_ID_NIC_4_3 */
+ QUEUE_TYPE_NA, /* GAUDI_QUEUE_ID_NIC_5_0 */
+ QUEUE_TYPE_NA, /* GAUDI_QUEUE_ID_NIC_5_1 */
+ QUEUE_TYPE_NA, /* GAUDI_QUEUE_ID_NIC_5_2 */
+ QUEUE_TYPE_NA, /* GAUDI_QUEUE_ID_NIC_5_3 */
+ QUEUE_TYPE_NA, /* GAUDI_QUEUE_ID_NIC_6_0 */
+ QUEUE_TYPE_NA, /* GAUDI_QUEUE_ID_NIC_6_1 */
+ QUEUE_TYPE_NA, /* GAUDI_QUEUE_ID_NIC_6_2 */
+ QUEUE_TYPE_NA, /* GAUDI_QUEUE_ID_NIC_6_3 */
+ QUEUE_TYPE_NA, /* GAUDI_QUEUE_ID_NIC_7_0 */
+ QUEUE_TYPE_NA, /* GAUDI_QUEUE_ID_NIC_7_1 */
+ QUEUE_TYPE_NA, /* GAUDI_QUEUE_ID_NIC_7_2 */
+ QUEUE_TYPE_NA, /* GAUDI_QUEUE_ID_NIC_7_3 */
+ QUEUE_TYPE_NA, /* GAUDI_QUEUE_ID_NIC_8_0 */
+ QUEUE_TYPE_NA, /* GAUDI_QUEUE_ID_NIC_8_1 */
+ QUEUE_TYPE_NA, /* GAUDI_QUEUE_ID_NIC_8_2 */
+ QUEUE_TYPE_NA, /* GAUDI_QUEUE_ID_NIC_8_3 */
+ QUEUE_TYPE_NA, /* GAUDI_QUEUE_ID_NIC_9_0 */
+ QUEUE_TYPE_NA, /* GAUDI_QUEUE_ID_NIC_9_1 */
+ QUEUE_TYPE_NA, /* GAUDI_QUEUE_ID_NIC_9_2 */
+ QUEUE_TYPE_NA, /* GAUDI_QUEUE_ID_NIC_9_3 */
+};
+
+static int gaudi_mmu_update_asid_hop0_addr(struct hl_device *hdev, u32 asid,
+ u64 phys_addr);
+static int gaudi_send_job_on_qman0(struct hl_device *hdev,
+ struct hl_cs_job *job);
+static int gaudi_memset_device_memory(struct hl_device *hdev, u64 addr,
+ u32 size, u64 val);
+static int gaudi_run_tpc_kernel(struct hl_device *hdev, u64 tpc_kernel,
+ u32 tpc_id);
+static int gaudi_mmu_clear_pgt_range(struct hl_device *hdev);
+static int gaudi_armcp_info_get(struct hl_device *hdev);
+static void gaudi_disable_clock_gating(struct hl_device *hdev);
+static void gaudi_mmu_prepare(struct hl_device *hdev, u32 asid);
+
+static int gaudi_get_fixed_properties(struct hl_device *hdev)
+{
+ struct asic_fixed_properties *prop = &hdev->asic_prop;
+ int i;
+
+ if (GAUDI_QUEUE_ID_SIZE >= HL_MAX_QUEUES) {
+ dev_err(hdev->dev,
+ "Number of H/W queues must be smaller than %d\n",
+ HL_MAX_QUEUES);
+ return -EFAULT;
+ }
+
+ for (i = 0 ; i < GAUDI_QUEUE_ID_SIZE ; i++) {
+ if (gaudi_queue_type[i] == QUEUE_TYPE_EXT) {
+ prop->hw_queues_props[i].type = QUEUE_TYPE_EXT;
+ prop->hw_queues_props[i].driver_only = 0;
+ prop->hw_queues_props[i].requires_kernel_cb = 1;
+ } else if (gaudi_queue_type[i] == QUEUE_TYPE_CPU) {
+ prop->hw_queues_props[i].type = QUEUE_TYPE_CPU;
+ prop->hw_queues_props[i].driver_only = 1;
+ prop->hw_queues_props[i].requires_kernel_cb = 0;
+ } else if (gaudi_queue_type[i] == QUEUE_TYPE_INT) {
+ prop->hw_queues_props[i].type = QUEUE_TYPE_INT;
+ prop->hw_queues_props[i].driver_only = 0;
+ prop->hw_queues_props[i].requires_kernel_cb = 0;
+ } else if (gaudi_queue_type[i] == QUEUE_TYPE_NA) {
+ prop->hw_queues_props[i].type = QUEUE_TYPE_NA;
+ prop->hw_queues_props[i].driver_only = 0;
+ prop->hw_queues_props[i].requires_kernel_cb = 0;
+ }
+ }
+
+ for (; i < HL_MAX_QUEUES; i++)
+ prop->hw_queues_props[i].type = QUEUE_TYPE_NA;
+
+ prop->completion_queues_count = NUMBER_OF_CMPLT_QUEUES;
+
+ prop->dram_base_address = DRAM_PHYS_BASE;
+ prop->dram_size = GAUDI_HBM_SIZE_32GB;
+ prop->dram_end_address = prop->dram_base_address +
+ prop->dram_size;
+ prop->dram_user_base_address = DRAM_BASE_ADDR_USER;
+
+ prop->sram_base_address = SRAM_BASE_ADDR;
+ prop->sram_size = SRAM_SIZE;
+ prop->sram_end_address = prop->sram_base_address +
+ prop->sram_size;
+ prop->sram_user_base_address = prop->sram_base_address +
+ SRAM_USER_BASE_OFFSET;
+
+ prop->mmu_pgt_addr = MMU_PAGE_TABLES_ADDR;
+ if (hdev->pldm)
+ prop->mmu_pgt_size = 0x800000; /* 8MB */
+ else
+ prop->mmu_pgt_size = MMU_PAGE_TABLES_SIZE;
+ prop->mmu_pte_size = HL_PTE_SIZE;
+ prop->mmu_hop_table_size = HOP_TABLE_SIZE;
+ prop->mmu_hop0_tables_total_size = HOP0_TABLES_TOTAL_SIZE;
+ prop->dram_page_size = PAGE_SIZE_2MB;
+
+ prop->pmmu.hop0_shift = HOP0_SHIFT;
+ prop->pmmu.hop1_shift = HOP1_SHIFT;
+ prop->pmmu.hop2_shift = HOP2_SHIFT;
+ prop->pmmu.hop3_shift = HOP3_SHIFT;
+ prop->pmmu.hop4_shift = HOP4_SHIFT;
+ prop->pmmu.hop0_mask = HOP0_MASK;
+ prop->pmmu.hop1_mask = HOP1_MASK;
+ prop->pmmu.hop2_mask = HOP2_MASK;
+ prop->pmmu.hop3_mask = HOP3_MASK;
+ prop->pmmu.hop4_mask = HOP4_MASK;
+ prop->pmmu.start_addr = VA_HOST_SPACE_START;
+ prop->pmmu.end_addr =
+ (VA_HOST_SPACE_START + VA_HOST_SPACE_SIZE / 2) - 1;
+ prop->pmmu.page_size = PAGE_SIZE_4KB;
+
+ /* PMMU and HPMMU are the same except of page size */
+ memcpy(&prop->pmmu_huge, &prop->pmmu, sizeof(prop->pmmu));
+ prop->pmmu_huge.page_size = PAGE_SIZE_2MB;
+
+ /* shifts and masks are the same in PMMU and DMMU */
+ memcpy(&prop->dmmu, &prop->pmmu, sizeof(prop->pmmu));
+ prop->dmmu.start_addr = (VA_HOST_SPACE_START + VA_HOST_SPACE_SIZE / 2);
+ prop->dmmu.end_addr = VA_HOST_SPACE_END;
+ prop->dmmu.page_size = PAGE_SIZE_2MB;
+
+ prop->cfg_size = CFG_SIZE;
+ prop->max_asid = MAX_ASID;
+ prop->num_of_events = GAUDI_EVENT_SIZE;
+ prop->tpc_enabled_mask = TPC_ENABLED_MASK;
+
+ prop->max_power_default = MAX_POWER_DEFAULT;
+
+ prop->cb_pool_cb_cnt = GAUDI_CB_POOL_CB_CNT;
+ prop->cb_pool_cb_size = GAUDI_CB_POOL_CB_SIZE;
+
+ prop->pcie_dbi_base_address = mmPCIE_DBI_BASE;
+ prop->pcie_aux_dbi_reg_addr = CFG_BASE + mmPCIE_AUX_DBI;
+
+ strncpy(prop->armcp_info.card_name, GAUDI_DEFAULT_CARD_NAME,
+ CARD_NAME_MAX_LEN);
+
+ return 0;
+}
+
+static int gaudi_pci_bars_map(struct hl_device *hdev)
+{
+ static const char * const name[] = {"SRAM", "CFG", "HBM"};
+ bool is_wc[3] = {false, false, true};
+ int rc;
+
+ rc = hl_pci_bars_map(hdev, name, is_wc);
+ if (rc)
+ return rc;
+
+ hdev->rmmio = hdev->pcie_bar[CFG_BAR_ID] +
+ (CFG_BASE - SPI_FLASH_BASE_ADDR);
+
+ return 0;
+}
+
+static u64 gaudi_set_hbm_bar_base(struct hl_device *hdev, u64 addr)
+{
+ struct gaudi_device *gaudi = hdev->asic_specific;
+ u64 old_addr = addr;
+ int rc;
+
+ if ((gaudi) && (gaudi->hbm_bar_cur_addr == addr))
+ return old_addr;
+
+ /* Inbound Region 2 - Bar 4 - Point to HBM */
+ rc = hl_pci_set_dram_bar_base(hdev, 2, 4, addr);
+ if (rc)
+ return U64_MAX;
+
+ if (gaudi) {
+ old_addr = gaudi->hbm_bar_cur_addr;
+ gaudi->hbm_bar_cur_addr = addr;
+ }
+
+ return old_addr;
+}
+
+static int gaudi_init_iatu(struct hl_device *hdev)
+{
+ int rc = 0;
+
+ /* Inbound Region 1 - Bar 2 - Point to SPI FLASH */
+ rc = hl_pci_iatu_write(hdev, 0x314,
+ lower_32_bits(SPI_FLASH_BASE_ADDR));
+ rc |= hl_pci_iatu_write(hdev, 0x318,
+ upper_32_bits(SPI_FLASH_BASE_ADDR));
+ rc |= hl_pci_iatu_write(hdev, 0x300, 0);
+ /* Enable + Bar match + match enable */
+ rc |= hl_pci_iatu_write(hdev, 0x304, 0xC0080200);
+
+ if (rc)
+ return -EIO;
+
+ return hl_pci_init_iatu(hdev, SRAM_BASE_ADDR, DRAM_PHYS_BASE,
+ HOST_PHYS_BASE, HOST_PHYS_SIZE);
+}
+
+static int gaudi_early_init(struct hl_device *hdev)
+{
+ struct asic_fixed_properties *prop = &hdev->asic_prop;
+ struct pci_dev *pdev = hdev->pdev;
+ int rc;
+
+ rc = gaudi_get_fixed_properties(hdev);
+ if (rc) {
+ dev_err(hdev->dev, "Failed to get fixed properties\n");
+ return rc;
+ }
+
+ /* Check BAR sizes */
+ if (pci_resource_len(pdev, SRAM_BAR_ID) != SRAM_BAR_SIZE) {
+ dev_err(hdev->dev,
+ "Not " HL_NAME "? BAR %d size %llu, expecting %llu\n",
+ SRAM_BAR_ID,
+ (unsigned long long) pci_resource_len(pdev,
+ SRAM_BAR_ID),
+ SRAM_BAR_SIZE);
+ return -ENODEV;
+ }
+
+ if (pci_resource_len(pdev, CFG_BAR_ID) != CFG_BAR_SIZE) {
+ dev_err(hdev->dev,
+ "Not " HL_NAME "? BAR %d size %llu, expecting %llu\n",
+ CFG_BAR_ID,
+ (unsigned long long) pci_resource_len(pdev,
+ CFG_BAR_ID),
+ CFG_BAR_SIZE);
+ return -ENODEV;
+ }
+
+ prop->dram_pci_bar_size = pci_resource_len(pdev, HBM_BAR_ID);
+
+ rc = hl_pci_init(hdev);
+ if (rc)
+ return rc;
+
+ return 0;
+}
+
+static int gaudi_early_fini(struct hl_device *hdev)
+{
+ hl_pci_fini(hdev);
+
+ return 0;
+}
+
+/**
+ * gaudi_fetch_psoc_frequency - Fetch PSOC frequency values
+ *
+ * @hdev: pointer to hl_device structure
+ *
+ */
+static void gaudi_fetch_psoc_frequency(struct hl_device *hdev)
+{
+ struct asic_fixed_properties *prop = &hdev->asic_prop;
+
+ prop->psoc_pci_pll_nr = RREG32(mmPSOC_PCI_PLL_NR);
+ prop->psoc_pci_pll_nf = RREG32(mmPSOC_PCI_PLL_NF);
+ prop->psoc_pci_pll_od = RREG32(mmPSOC_PCI_PLL_OD);
+ prop->psoc_pci_pll_div_factor = RREG32(mmPSOC_PCI_PLL_DIV_FACTOR_1);
+}
+
+static int _gaudi_init_tpc_mem(struct hl_device *hdev,
+ dma_addr_t tpc_kernel_src_addr, u32 tpc_kernel_size)
+{
+ struct asic_fixed_properties *prop = &hdev->asic_prop;
+ struct packet_lin_dma *init_tpc_mem_pkt;
+ struct hl_cs_job *job;
+ struct hl_cb *cb;
+ u64 dst_addr;
+ u32 cb_size, ctl;
+ u8 tpc_id;
+ int rc;
+
+ cb = hl_cb_kernel_create(hdev, PAGE_SIZE);
+ if (!cb)
+ return -EFAULT;
+
+ init_tpc_mem_pkt = (struct packet_lin_dma *) (uintptr_t)
+ cb->kernel_address;
+ cb_size = sizeof(*init_tpc_mem_pkt);
+ memset(init_tpc_mem_pkt, 0, cb_size);
+
+ init_tpc_mem_pkt->tsize = cpu_to_le32(tpc_kernel_size);
+
+ ctl = ((PACKET_LIN_DMA << GAUDI_PKT_CTL_OPCODE_SHIFT) |
+ (1 << GAUDI_PKT_LIN_DMA_CTL_LIN_SHIFT) |
+ (1 << GAUDI_PKT_CTL_RB_SHIFT) |
+ (1 << GAUDI_PKT_CTL_MB_SHIFT));
+
+ init_tpc_mem_pkt->ctl = cpu_to_le32(ctl);
+
+ init_tpc_mem_pkt->src_addr = cpu_to_le64(tpc_kernel_src_addr);
+ dst_addr = (prop->sram_user_base_address &
+ GAUDI_PKT_LIN_DMA_DST_ADDR_MASK) >>
+ GAUDI_PKT_LIN_DMA_DST_ADDR_SHIFT;
+ init_tpc_mem_pkt->dst_addr |= cpu_to_le64(dst_addr);
+
+ job = hl_cs_allocate_job(hdev, QUEUE_TYPE_EXT, true);
+ if (!job) {
+ dev_err(hdev->dev, "Failed to allocate a new job\n");
+ rc = -ENOMEM;
+ goto release_cb;
+ }
+
+ job->id = 0;
+ job->user_cb = cb;
+ job->user_cb->cs_cnt++;
+ job->user_cb_size = cb_size;
+ job->hw_queue_id = GAUDI_QUEUE_ID_DMA_0_0;
+ job->patched_cb = job->user_cb;
+ job->job_cb_size = job->user_cb_size + sizeof(struct packet_msg_prot);
+
+ hl_debugfs_add_job(hdev, job);
+
+ rc = gaudi_send_job_on_qman0(hdev, job);
+
+ if (rc)
+ goto free_job;
+
+ for (tpc_id = 0 ; tpc_id < TPC_NUMBER_OF_ENGINES ; tpc_id++) {
+ rc = gaudi_run_tpc_kernel(hdev, dst_addr, tpc_id);
+ if (rc)
+ break;
+ }
+
+free_job:
+ hl_userptr_delete_list(hdev, &job->userptr_list);
+ hl_debugfs_remove_job(hdev, job);
+ kfree(job);
+ cb->cs_cnt--;
+
+release_cb:
+ hl_cb_put(cb);
+ hl_cb_destroy(hdev, &hdev->kernel_cb_mgr, cb->id << PAGE_SHIFT);
+
+ return rc;
+}
+
+/*
+ * gaudi_init_tpc_mem() - Initialize TPC memories.
+ * @hdev: Pointer to hl_device structure.
+ *
+ * Copy TPC kernel fw from firmware file and run it to initialize TPC memories.
+ *
+ * Return: 0 for success, negative value for error.
+ */
+static int gaudi_init_tpc_mem(struct hl_device *hdev)
+{
+ const struct firmware *fw;
+ size_t fw_size;
+ void *cpu_addr;
+ dma_addr_t dma_handle;
+ int rc;
+
+ rc = request_firmware(&fw, GAUDI_TPC_FW_FILE, hdev->dev);
+ if (rc) {
+ dev_err(hdev->dev, "Firmware file %s is not found!\n",
+ GAUDI_TPC_FW_FILE);
+ goto out;
+ }
+
+ fw_size = fw->size;
+ cpu_addr = hdev->asic_funcs->asic_dma_alloc_coherent(hdev, fw_size,
+ &dma_handle, GFP_KERNEL | __GFP_ZERO);
+ if (!cpu_addr) {
+ dev_err(hdev->dev,
+ "Failed to allocate %zu of dma memory for TPC kernel\n",
+ fw_size);
+ rc = -ENOMEM;
+ goto out;
+ }
+
+ memcpy(cpu_addr, fw->data, fw_size);
+
+ rc = _gaudi_init_tpc_mem(hdev, dma_handle, fw_size);
+
+ hdev->asic_funcs->asic_dma_free_coherent(hdev, fw->size, cpu_addr,
+ dma_handle);
+
+out:
+ release_firmware(fw);
+ return rc;
+}
+
+static int gaudi_late_init(struct hl_device *hdev)
+{
+ struct gaudi_device *gaudi = hdev->asic_specific;
+ int rc;
+
+ rc = gaudi->armcp_info_get(hdev);
+ if (rc) {
+ dev_err(hdev->dev, "Failed to get armcp info\n");
+ return rc;
+ }
+
+ rc = hl_fw_send_pci_access_msg(hdev, ARMCP_PACKET_ENABLE_PCI_ACCESS);
+ if (rc) {
+ dev_err(hdev->dev, "Failed to enable PCI access from CPU\n");
+ return rc;
+ }
+
+ WREG32(mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR, GAUDI_EVENT_INTS_REGISTER);
+
+ gaudi_fetch_psoc_frequency(hdev);
+
+ rc = gaudi_mmu_clear_pgt_range(hdev);
+ if (rc) {
+ dev_err(hdev->dev, "Failed to clear MMU page tables range\n");
+ goto disable_pci_access;
+ }
+
+ rc = gaudi_init_tpc_mem(hdev);
+ if (rc) {
+ dev_err(hdev->dev, "Failed to initialize TPC memories\n");
+ goto disable_pci_access;
+ }
+
+ return 0;
+
+disable_pci_access:
+ hl_fw_send_pci_access_msg(hdev, ARMCP_PACKET_DISABLE_PCI_ACCESS);
+
+ return rc;
+}
+
+static void gaudi_late_fini(struct hl_device *hdev)
+{
+ const struct hwmon_channel_info **channel_info_arr;
+ int i = 0;
+
+ if (!hdev->hl_chip_info->info)
+ return;
+
+ channel_info_arr = hdev->hl_chip_info->info;
+
+ while (channel_info_arr[i]) {
+ kfree(channel_info_arr[i]->config);
+ kfree(channel_info_arr[i]);
+ i++;
+ }
+
+ kfree(channel_info_arr);
+
+ hdev->hl_chip_info->info = NULL;
+}
+
+static int gaudi_alloc_cpu_accessible_dma_mem(struct hl_device *hdev)
+{
+ dma_addr_t dma_addr_arr[GAUDI_ALLOC_CPU_MEM_RETRY_CNT] = {}, end_addr;
+ void *virt_addr_arr[GAUDI_ALLOC_CPU_MEM_RETRY_CNT] = {};
+ int i, j, rc = 0;
+
+ /*
+ * The device CPU works with 40-bits addresses, while bit 39 must be set
+ * to '1' when accessing the host.
+ * Bits 49:39 of the full host address are saved for a later
+ * configuration of the HW to perform extension to 50 bits.
+ * Because there is a single HW register that holds the extension bits,
+ * these bits must be identical in all allocated range.
+ */
+
+ for (i = 0 ; i < GAUDI_ALLOC_CPU_MEM_RETRY_CNT ; i++) {
+ virt_addr_arr[i] =
+ hdev->asic_funcs->asic_dma_alloc_coherent(hdev,
+ HL_CPU_ACCESSIBLE_MEM_SIZE,
+ &dma_addr_arr[i],
+ GFP_KERNEL | __GFP_ZERO);
+ if (!virt_addr_arr[i]) {
+ rc = -ENOMEM;
+ goto free_dma_mem_arr;
+ }
+
+ end_addr = dma_addr_arr[i] + HL_CPU_ACCESSIBLE_MEM_SIZE - 1;
+ if (GAUDI_CPU_PCI_MSB_ADDR(dma_addr_arr[i]) ==
+ GAUDI_CPU_PCI_MSB_ADDR(end_addr))
+ break;
+ }
+
+ if (i == GAUDI_ALLOC_CPU_MEM_RETRY_CNT) {
+ dev_err(hdev->dev,
+ "MSB of CPU accessible DMA memory are not identical in all range\n");
+ rc = -EFAULT;
+ goto free_dma_mem_arr;
+ }
+
+ hdev->cpu_accessible_dma_mem = virt_addr_arr[i];
+ hdev->cpu_accessible_dma_address = dma_addr_arr[i];
+ hdev->cpu_pci_msb_addr =
+ GAUDI_CPU_PCI_MSB_ADDR(hdev->cpu_accessible_dma_address);
+
+ GAUDI_PCI_TO_CPU_ADDR(hdev->cpu_accessible_dma_address);
+
+free_dma_mem_arr:
+ for (j = 0 ; j < i ; j++)
+ hdev->asic_funcs->asic_dma_free_coherent(hdev,
+ HL_CPU_ACCESSIBLE_MEM_SIZE,
+ virt_addr_arr[j],
+ dma_addr_arr[j]);
+
+ return rc;
+}
+
+static void gaudi_free_internal_qmans_pq_mem(struct hl_device *hdev)
+{
+ struct gaudi_device *gaudi = hdev->asic_specific;
+ struct gaudi_internal_qman_info *q;
+ u32 i;
+
+ for (i = 0 ; i < GAUDI_QUEUE_ID_SIZE ; i++) {
+ q = &gaudi->internal_qmans[i];
+ if (!q->pq_kernel_addr)
+ continue;
+ hdev->asic_funcs->asic_dma_free_coherent(hdev, q->pq_size,
+ q->pq_kernel_addr,
+ q->pq_dma_addr);
+ }
+}
+
+static int gaudi_alloc_internal_qmans_pq_mem(struct hl_device *hdev)
+{
+ struct gaudi_device *gaudi = hdev->asic_specific;
+ struct gaudi_internal_qman_info *q;
+ int rc, i;
+
+ for (i = 0 ; i < GAUDI_QUEUE_ID_SIZE ; i++) {
+ if (gaudi_queue_type[i] != QUEUE_TYPE_INT)
+ continue;
+
+ q = &gaudi->internal_qmans[i];
+
+ switch (i) {
+ case GAUDI_QUEUE_ID_DMA_2_0 ... GAUDI_QUEUE_ID_DMA_4_3:
+ case GAUDI_QUEUE_ID_DMA_6_0 ... GAUDI_QUEUE_ID_DMA_7_3:
+ q->pq_size = HBM_DMA_QMAN_SIZE_IN_BYTES;
+ break;
+ case GAUDI_QUEUE_ID_MME_0_0 ... GAUDI_QUEUE_ID_MME_1_3:
+ q->pq_size = MME_QMAN_SIZE_IN_BYTES;
+ break;
+ case GAUDI_QUEUE_ID_TPC_0_0 ... GAUDI_QUEUE_ID_TPC_7_3:
+ q->pq_size = TPC_QMAN_SIZE_IN_BYTES;
+ break;
+ default:
+ dev_err(hdev->dev, "Bad internal queue index %d", i);
+ rc = -EINVAL;
+ goto free_internal_qmans_pq_mem;
+ }
+
+ q->pq_kernel_addr = hdev->asic_funcs->asic_dma_alloc_coherent(
+ hdev, q->pq_size,
+ &q->pq_dma_addr,
+ GFP_KERNEL | __GFP_ZERO);
+ if (!q->pq_kernel_addr) {
+ rc = -ENOMEM;
+ goto free_internal_qmans_pq_mem;
+ }
+ }
+
+ return 0;
+
+free_internal_qmans_pq_mem:
+ gaudi_free_internal_qmans_pq_mem(hdev);
+ return rc;
+}
+
+static int gaudi_sw_init(struct hl_device *hdev)
+{
+ struct gaudi_device *gaudi;
+ u32 i, event_id = 0;
+ int rc;
+
+ /* Allocate device structure */
+ gaudi = kzalloc(sizeof(*gaudi), GFP_KERNEL);
+ if (!gaudi)
+ return -ENOMEM;
+
+ for (i = 0 ; i < ARRAY_SIZE(gaudi_irq_map_table) ; i++) {
+ if (gaudi_irq_map_table[i].valid) {
+ if (event_id == GAUDI_EVENT_SIZE) {
+ dev_err(hdev->dev,
+ "Event array exceeds the limit of %u events\n",
+ GAUDI_EVENT_SIZE);
+ rc = -EINVAL;
+ goto free_gaudi_device;
+ }
+
+ gaudi->events[event_id++] =
+ gaudi_irq_map_table[i].fc_id;
+ }
+ }
+
+ gaudi->armcp_info_get = gaudi_armcp_info_get;
+
+ gaudi->max_freq_value = GAUDI_MAX_CLK_FREQ;
+
+ hdev->asic_specific = gaudi;
+
+ /* Create DMA pool for small allocations */
+ hdev->dma_pool = dma_pool_create(dev_name(hdev->dev),
+ &hdev->pdev->dev, GAUDI_DMA_POOL_BLK_SIZE, 8, 0);
+ if (!hdev->dma_pool) {
+ dev_err(hdev->dev, "failed to create DMA pool\n");
+ rc = -ENOMEM;
+ goto free_gaudi_device;
+ }
+
+ rc = gaudi_alloc_cpu_accessible_dma_mem(hdev);
+ if (rc)
+ goto free_dma_pool;
+
+ hdev->cpu_accessible_dma_pool = gen_pool_create(ilog2(32), -1);
+ if (!hdev->cpu_accessible_dma_pool) {
+ dev_err(hdev->dev,
+ "Failed to create CPU accessible DMA pool\n");
+ rc = -ENOMEM;
+ goto free_cpu_dma_mem;
+ }
+
+ rc = gen_pool_add(hdev->cpu_accessible_dma_pool,
+ (uintptr_t) hdev->cpu_accessible_dma_mem,
+ HL_CPU_ACCESSIBLE_MEM_SIZE, -1);
+ if (rc) {
+ dev_err(hdev->dev,
+ "Failed to add memory to CPU accessible DMA pool\n");
+ rc = -EFAULT;
+ goto free_cpu_accessible_dma_pool;
+ }
+
+ rc = gaudi_alloc_internal_qmans_pq_mem(hdev);
+ if (rc)
+ goto free_cpu_accessible_dma_pool;
+
+ spin_lock_init(&gaudi->hw_queues_lock);
+ mutex_init(&gaudi->clk_gate_mutex);
+
+ hdev->supports_sync_stream = true;
+ hdev->supports_coresight = true;
+
+ return 0;
+
+free_cpu_accessible_dma_pool:
+ gen_pool_destroy(hdev->cpu_accessible_dma_pool);
+free_cpu_dma_mem:
+ GAUDI_CPU_TO_PCI_ADDR(hdev->cpu_accessible_dma_address,
+ hdev->cpu_pci_msb_addr);
+ hdev->asic_funcs->asic_dma_free_coherent(hdev,
+ HL_CPU_ACCESSIBLE_MEM_SIZE,
+ hdev->cpu_accessible_dma_mem,
+ hdev->cpu_accessible_dma_address);
+free_dma_pool:
+ dma_pool_destroy(hdev->dma_pool);
+free_gaudi_device:
+ kfree(gaudi);
+ return rc;
+}
+
+static int gaudi_sw_fini(struct hl_device *hdev)
+{
+ struct gaudi_device *gaudi = hdev->asic_specific;
+
+ gaudi_free_internal_qmans_pq_mem(hdev);
+
+ gen_pool_destroy(hdev->cpu_accessible_dma_pool);
+
+ GAUDI_CPU_TO_PCI_ADDR(hdev->cpu_accessible_dma_address,
+ hdev->cpu_pci_msb_addr);
+ hdev->asic_funcs->asic_dma_free_coherent(hdev,
+ HL_CPU_ACCESSIBLE_MEM_SIZE,
+ hdev->cpu_accessible_dma_mem,
+ hdev->cpu_accessible_dma_address);
+
+ dma_pool_destroy(hdev->dma_pool);
+
+ mutex_destroy(&gaudi->clk_gate_mutex);
+
+ kfree(gaudi);
+
+ return 0;
+}
+
+static irqreturn_t gaudi_irq_handler_single(int irq, void *arg)
+{
+ struct hl_device *hdev = arg;
+ int i;
+
+ if (hdev->disabled)
+ return IRQ_HANDLED;
+
+ for (i = 0 ; i < hdev->asic_prop.completion_queues_count ; i++)
+ hl_irq_handler_cq(irq, &hdev->completion_queue[i]);
+
+ hl_irq_handler_eq(irq, &hdev->event_queue);
+
+ return IRQ_HANDLED;
+}
+
+/*
+ * For backward compatibility, new MSI interrupts should be set after the
+ * existing CPU and NIC interrupts.
+ */
+static int gaudi_pci_irq_vector(struct hl_device *hdev, unsigned int nr,
+ bool cpu_eq)
+{
+ int msi_vec;
+
+ if ((nr != GAUDI_EVENT_QUEUE_MSI_IDX) && (cpu_eq))
+ dev_crit(hdev->dev, "CPU EQ must use IRQ %d\n",
+ GAUDI_EVENT_QUEUE_MSI_IDX);
+
+ msi_vec = ((nr < GAUDI_EVENT_QUEUE_MSI_IDX) || (cpu_eq)) ? nr :
+ (nr + NIC_NUMBER_OF_ENGINES + 1);
+
+ return pci_irq_vector(hdev->pdev, msi_vec);
+}
+
+static int gaudi_enable_msi_single(struct hl_device *hdev)
+{
+ int rc, irq;
+
+ dev_info(hdev->dev, "Working in single MSI IRQ mode\n");
+
+ irq = gaudi_pci_irq_vector(hdev, 0, false);
+ rc = request_irq(irq, gaudi_irq_handler_single, 0,
+ "gaudi single msi", hdev);
+ if (rc)
+ dev_err(hdev->dev,
+ "Failed to request single MSI IRQ\n");
+
+ return rc;
+}
+
+static int gaudi_enable_msi_multi(struct hl_device *hdev)
+{
+ int cq_cnt = hdev->asic_prop.completion_queues_count;
+ int rc, i, irq_cnt_init, irq;
+
+ for (i = 0, irq_cnt_init = 0 ; i < cq_cnt ; i++, irq_cnt_init++) {
+ irq = gaudi_pci_irq_vector(hdev, i, false);
+ rc = request_irq(irq, hl_irq_handler_cq, 0, gaudi_irq_name[i],
+ &hdev->completion_queue[i]);
+ if (rc) {
+ dev_err(hdev->dev, "Failed to request IRQ %d", irq);
+ goto free_irqs;
+ }
+ }
+
+ irq = gaudi_pci_irq_vector(hdev, GAUDI_EVENT_QUEUE_MSI_IDX, true);
+ rc = request_irq(irq, hl_irq_handler_eq, 0, gaudi_irq_name[cq_cnt],
+ &hdev->event_queue);
+ if (rc) {
+ dev_err(hdev->dev, "Failed to request IRQ %d", irq);
+ goto free_irqs;
+ }
+
+ return 0;
+
+free_irqs:
+ for (i = 0 ; i < irq_cnt_init ; i++)
+ free_irq(gaudi_pci_irq_vector(hdev, i, false),
+ &hdev->completion_queue[i]);
+ return rc;
+}
+
+static int gaudi_enable_msi(struct hl_device *hdev)
+{
+ struct gaudi_device *gaudi = hdev->asic_specific;
+ int rc;
+
+ if (gaudi->hw_cap_initialized & HW_CAP_MSI)
+ return 0;
+
+ rc = pci_alloc_irq_vectors(hdev->pdev, 1, GAUDI_MSI_ENTRIES,
+ PCI_IRQ_MSI);
+ if (rc < 0) {
+ dev_err(hdev->dev, "MSI: Failed to enable support %d\n", rc);
+ return rc;
+ }
+
+ if (rc < NUMBER_OF_INTERRUPTS) {
+ gaudi->multi_msi_mode = false;
+ rc = gaudi_enable_msi_single(hdev);
+ } else {
+ gaudi->multi_msi_mode = true;
+ rc = gaudi_enable_msi_multi(hdev);
+ }
+
+ if (rc)
+ goto free_pci_irq_vectors;
+
+ gaudi->hw_cap_initialized |= HW_CAP_MSI;
+
+ return 0;
+
+free_pci_irq_vectors:
+ pci_free_irq_vectors(hdev->pdev);
+ return rc;
+}
+
+static void gaudi_sync_irqs(struct hl_device *hdev)
+{
+ struct gaudi_device *gaudi = hdev->asic_specific;
+ int i, cq_cnt = hdev->asic_prop.completion_queues_count;
+
+ if (!(gaudi->hw_cap_initialized & HW_CAP_MSI))
+ return;
+
+ /* Wait for all pending IRQs to be finished */
+ if (gaudi->multi_msi_mode) {
+ for (i = 0 ; i < cq_cnt ; i++)
+ synchronize_irq(gaudi_pci_irq_vector(hdev, i, false));
+
+ synchronize_irq(gaudi_pci_irq_vector(hdev,
+ GAUDI_EVENT_QUEUE_MSI_IDX,
+ true));
+ } else {
+ synchronize_irq(gaudi_pci_irq_vector(hdev, 0, false));
+ }
+}
+
+static void gaudi_disable_msi(struct hl_device *hdev)
+{
+ struct gaudi_device *gaudi = hdev->asic_specific;
+ int i, irq, cq_cnt = hdev->asic_prop.completion_queues_count;
+
+ if (!(gaudi->hw_cap_initialized & HW_CAP_MSI))
+ return;
+
+ gaudi_sync_irqs(hdev);
+
+ if (gaudi->multi_msi_mode) {
+ irq = gaudi_pci_irq_vector(hdev, GAUDI_EVENT_QUEUE_MSI_IDX,
+ true);
+ free_irq(irq, &hdev->event_queue);
+
+ for (i = 0 ; i < cq_cnt ; i++) {
+ irq = gaudi_pci_irq_vector(hdev, i, false);
+ free_irq(irq, &hdev->completion_queue[i]);
+ }
+ } else {
+ free_irq(gaudi_pci_irq_vector(hdev, 0, false), hdev);
+ }
+
+ pci_free_irq_vectors(hdev->pdev);
+
+ gaudi->hw_cap_initialized &= ~HW_CAP_MSI;
+}
+
+static void gaudi_init_scrambler_sram(struct hl_device *hdev)
+{
+ struct gaudi_device *gaudi = hdev->asic_specific;
+
+ if (gaudi->hw_cap_initialized & HW_CAP_SRAM_SCRAMBLER)
+ return;
+
+ if (!hdev->sram_scrambler_enable)
+ return;
+
+ WREG32(mmNIF_RTR_CTRL_0_SCRAM_SRAM_EN,
+ 1 << IF_RTR_CTRL_SCRAM_SRAM_EN_VAL_SHIFT);
+ WREG32(mmNIF_RTR_CTRL_1_SCRAM_SRAM_EN,
+ 1 << IF_RTR_CTRL_SCRAM_SRAM_EN_VAL_SHIFT);
+ WREG32(mmNIF_RTR_CTRL_2_SCRAM_SRAM_EN,
+ 1 << IF_RTR_CTRL_SCRAM_SRAM_EN_VAL_SHIFT);
+ WREG32(mmNIF_RTR_CTRL_3_SCRAM_SRAM_EN,
+ 1 << IF_RTR_CTRL_SCRAM_SRAM_EN_VAL_SHIFT);
+ WREG32(mmNIF_RTR_CTRL_4_SCRAM_SRAM_EN,
+ 1 << IF_RTR_CTRL_SCRAM_SRAM_EN_VAL_SHIFT);
+ WREG32(mmNIF_RTR_CTRL_5_SCRAM_SRAM_EN,
+ 1 << IF_RTR_CTRL_SCRAM_SRAM_EN_VAL_SHIFT);
+ WREG32(mmNIF_RTR_CTRL_6_SCRAM_SRAM_EN,
+ 1 << IF_RTR_CTRL_SCRAM_SRAM_EN_VAL_SHIFT);
+ WREG32(mmNIF_RTR_CTRL_7_SCRAM_SRAM_EN,
+ 1 << IF_RTR_CTRL_SCRAM_SRAM_EN_VAL_SHIFT);
+
+ WREG32(mmSIF_RTR_CTRL_0_SCRAM_SRAM_EN,
+ 1 << IF_RTR_CTRL_SCRAM_SRAM_EN_VAL_SHIFT);
+ WREG32(mmSIF_RTR_CTRL_1_SCRAM_SRAM_EN,
+ 1 << IF_RTR_CTRL_SCRAM_SRAM_EN_VAL_SHIFT);
+ WREG32(mmSIF_RTR_CTRL_2_SCRAM_SRAM_EN,
+ 1 << IF_RTR_CTRL_SCRAM_SRAM_EN_VAL_SHIFT);
+ WREG32(mmSIF_RTR_CTRL_3_SCRAM_SRAM_EN,
+ 1 << IF_RTR_CTRL_SCRAM_SRAM_EN_VAL_SHIFT);
+ WREG32(mmSIF_RTR_CTRL_4_SCRAM_SRAM_EN,
+ 1 << IF_RTR_CTRL_SCRAM_SRAM_EN_VAL_SHIFT);
+ WREG32(mmSIF_RTR_CTRL_5_SCRAM_SRAM_EN,
+ 1 << IF_RTR_CTRL_SCRAM_SRAM_EN_VAL_SHIFT);
+ WREG32(mmSIF_RTR_CTRL_6_SCRAM_SRAM_EN,
+ 1 << IF_RTR_CTRL_SCRAM_SRAM_EN_VAL_SHIFT);
+ WREG32(mmSIF_RTR_CTRL_7_SCRAM_SRAM_EN,
+ 1 << IF_RTR_CTRL_SCRAM_SRAM_EN_VAL_SHIFT);
+
+ WREG32(mmDMA_IF_E_N_DOWN_CH0_SCRAM_SRAM_EN,
+ 1 << DMA_IF_DOWN_CHX_SCRAM_SRAM_EN_VAL_SHIFT);
+ WREG32(mmDMA_IF_E_N_DOWN_CH1_SCRAM_SRAM_EN,
+ 1 << DMA_IF_DOWN_CHX_SCRAM_SRAM_EN_VAL_SHIFT);
+ WREG32(mmDMA_IF_E_S_DOWN_CH0_SCRAM_SRAM_EN,
+ 1 << DMA_IF_DOWN_CHX_SCRAM_SRAM_EN_VAL_SHIFT);
+ WREG32(mmDMA_IF_E_S_DOWN_CH1_SCRAM_SRAM_EN,
+ 1 << DMA_IF_DOWN_CHX_SCRAM_SRAM_EN_VAL_SHIFT);
+ WREG32(mmDMA_IF_W_N_DOWN_CH0_SCRAM_SRAM_EN,
+ 1 << DMA_IF_DOWN_CHX_SCRAM_SRAM_EN_VAL_SHIFT);
+ WREG32(mmDMA_IF_W_N_DOWN_CH1_SCRAM_SRAM_EN,
+ 1 << DMA_IF_DOWN_CHX_SCRAM_SRAM_EN_VAL_SHIFT);
+ WREG32(mmDMA_IF_W_S_DOWN_CH0_SCRAM_SRAM_EN,
+ 1 << DMA_IF_DOWN_CHX_SCRAM_SRAM_EN_VAL_SHIFT);
+ WREG32(mmDMA_IF_W_S_DOWN_CH1_SCRAM_SRAM_EN,
+ 1 << DMA_IF_DOWN_CHX_SCRAM_SRAM_EN_VAL_SHIFT);
+
+ gaudi->hw_cap_initialized |= HW_CAP_SRAM_SCRAMBLER;
+}
+
+static void gaudi_init_scrambler_hbm(struct hl_device *hdev)
+{
+ struct gaudi_device *gaudi = hdev->asic_specific;
+
+ if (gaudi->hw_cap_initialized & HW_CAP_HBM_SCRAMBLER)
+ return;
+
+ if (!hdev->dram_scrambler_enable)
+ return;
+
+ WREG32(mmNIF_RTR_CTRL_0_SCRAM_HBM_EN,
+ 1 << IF_RTR_CTRL_SCRAM_HBM_EN_VAL_SHIFT);
+ WREG32(mmNIF_RTR_CTRL_1_SCRAM_HBM_EN,
+ 1 << IF_RTR_CTRL_SCRAM_HBM_EN_VAL_SHIFT);
+ WREG32(mmNIF_RTR_CTRL_2_SCRAM_HBM_EN,
+ 1 << IF_RTR_CTRL_SCRAM_HBM_EN_VAL_SHIFT);
+ WREG32(mmNIF_RTR_CTRL_3_SCRAM_HBM_EN,
+ 1 << IF_RTR_CTRL_SCRAM_HBM_EN_VAL_SHIFT);
+ WREG32(mmNIF_RTR_CTRL_4_SCRAM_HBM_EN,
+ 1 << IF_RTR_CTRL_SCRAM_HBM_EN_VAL_SHIFT);
+ WREG32(mmNIF_RTR_CTRL_5_SCRAM_HBM_EN,
+ 1 << IF_RTR_CTRL_SCRAM_HBM_EN_VAL_SHIFT);
+ WREG32(mmNIF_RTR_CTRL_6_SCRAM_HBM_EN,
+ 1 << IF_RTR_CTRL_SCRAM_HBM_EN_VAL_SHIFT);
+ WREG32(mmNIF_RTR_CTRL_7_SCRAM_HBM_EN,
+ 1 << IF_RTR_CTRL_SCRAM_HBM_EN_VAL_SHIFT);
+
+ WREG32(mmSIF_RTR_CTRL_0_SCRAM_HBM_EN,
+ 1 << IF_RTR_CTRL_SCRAM_HBM_EN_VAL_SHIFT);
+ WREG32(mmSIF_RTR_CTRL_1_SCRAM_HBM_EN,
+ 1 << IF_RTR_CTRL_SCRAM_HBM_EN_VAL_SHIFT);
+ WREG32(mmSIF_RTR_CTRL_2_SCRAM_HBM_EN,
+ 1 << IF_RTR_CTRL_SCRAM_HBM_EN_VAL_SHIFT);
+ WREG32(mmSIF_RTR_CTRL_3_SCRAM_HBM_EN,
+ 1 << IF_RTR_CTRL_SCRAM_HBM_EN_VAL_SHIFT);
+ WREG32(mmSIF_RTR_CTRL_4_SCRAM_HBM_EN,
+ 1 << IF_RTR_CTRL_SCRAM_HBM_EN_VAL_SHIFT);
+ WREG32(mmSIF_RTR_CTRL_5_SCRAM_HBM_EN,
+ 1 << IF_RTR_CTRL_SCRAM_HBM_EN_VAL_SHIFT);
+ WREG32(mmSIF_RTR_CTRL_6_SCRAM_HBM_EN,
+ 1 << IF_RTR_CTRL_SCRAM_HBM_EN_VAL_SHIFT);
+ WREG32(mmSIF_RTR_CTRL_7_SCRAM_HBM_EN,
+ 1 << IF_RTR_CTRL_SCRAM_HBM_EN_VAL_SHIFT);
+
+ WREG32(mmDMA_IF_E_N_DOWN_CH0_SCRAM_HBM_EN,
+ 1 << DMA_IF_DOWN_CHX_SCRAM_HBM_EN_VAL_SHIFT);
+ WREG32(mmDMA_IF_E_N_DOWN_CH1_SCRAM_HBM_EN,
+ 1 << DMA_IF_DOWN_CHX_SCRAM_HBM_EN_VAL_SHIFT);
+ WREG32(mmDMA_IF_E_S_DOWN_CH0_SCRAM_HBM_EN,
+ 1 << DMA_IF_DOWN_CHX_SCRAM_HBM_EN_VAL_SHIFT);
+ WREG32(mmDMA_IF_E_S_DOWN_CH1_SCRAM_HBM_EN,
+ 1 << DMA_IF_DOWN_CHX_SCRAM_HBM_EN_VAL_SHIFT);
+ WREG32(mmDMA_IF_W_N_DOWN_CH0_SCRAM_HBM_EN,
+ 1 << DMA_IF_DOWN_CHX_SCRAM_HBM_EN_VAL_SHIFT);
+ WREG32(mmDMA_IF_W_N_DOWN_CH1_SCRAM_HBM_EN,
+ 1 << DMA_IF_DOWN_CHX_SCRAM_HBM_EN_VAL_SHIFT);
+ WREG32(mmDMA_IF_W_S_DOWN_CH0_SCRAM_HBM_EN,
+ 1 << DMA_IF_DOWN_CHX_SCRAM_HBM_EN_VAL_SHIFT);
+ WREG32(mmDMA_IF_W_S_DOWN_CH1_SCRAM_HBM_EN,
+ 1 << DMA_IF_DOWN_CHX_SCRAM_HBM_EN_VAL_SHIFT);
+
+ gaudi->hw_cap_initialized |= HW_CAP_HBM_SCRAMBLER;
+}
+
+static void gaudi_init_e2e(struct hl_device *hdev)
+{
+ WREG32(mmSIF_RTR_CTRL_0_E2E_HBM_WR_SIZE, 247 >> 3);
+ WREG32(mmSIF_RTR_CTRL_0_E2E_HBM_RD_SIZE, 785 >> 3);
+ WREG32(mmSIF_RTR_CTRL_0_E2E_PCI_WR_SIZE, 49);
+ WREG32(mmSIF_RTR_CTRL_0_E2E_PCI_RD_SIZE, 101);
+
+ WREG32(mmSIF_RTR_CTRL_1_E2E_HBM_WR_SIZE, 275 >> 3);
+ WREG32(mmSIF_RTR_CTRL_1_E2E_HBM_RD_SIZE, 614 >> 3);
+ WREG32(mmSIF_RTR_CTRL_1_E2E_PCI_WR_SIZE, 1);
+ WREG32(mmSIF_RTR_CTRL_1_E2E_PCI_RD_SIZE, 39);
+
+ WREG32(mmSIF_RTR_CTRL_2_E2E_HBM_WR_SIZE, 1);
+ WREG32(mmSIF_RTR_CTRL_2_E2E_HBM_RD_SIZE, 1);
+ WREG32(mmSIF_RTR_CTRL_2_E2E_PCI_WR_SIZE, 1);
+ WREG32(mmSIF_RTR_CTRL_2_E2E_PCI_RD_SIZE, 32);
+
+ WREG32(mmSIF_RTR_CTRL_3_E2E_HBM_WR_SIZE, 176 >> 3);
+ WREG32(mmSIF_RTR_CTRL_3_E2E_HBM_RD_SIZE, 32 >> 3);
+ WREG32(mmSIF_RTR_CTRL_3_E2E_PCI_WR_SIZE, 19);
+ WREG32(mmSIF_RTR_CTRL_3_E2E_PCI_RD_SIZE, 32);
+
+ WREG32(mmSIF_RTR_CTRL_4_E2E_HBM_WR_SIZE, 176 >> 3);
+ WREG32(mmSIF_RTR_CTRL_4_E2E_HBM_RD_SIZE, 32 >> 3);
+ WREG32(mmSIF_RTR_CTRL_4_E2E_PCI_WR_SIZE, 19);
+ WREG32(mmSIF_RTR_CTRL_4_E2E_PCI_RD_SIZE, 32);
+
+ WREG32(mmSIF_RTR_CTRL_5_E2E_HBM_WR_SIZE, 1);
+ WREG32(mmSIF_RTR_CTRL_5_E2E_HBM_RD_SIZE, 1);
+ WREG32(mmSIF_RTR_CTRL_5_E2E_PCI_WR_SIZE, 1);
+ WREG32(mmSIF_RTR_CTRL_5_E2E_PCI_RD_SIZE, 32);
+
+ WREG32(mmSIF_RTR_CTRL_6_E2E_HBM_WR_SIZE, 275 >> 3);
+ WREG32(mmSIF_RTR_CTRL_6_E2E_HBM_RD_SIZE, 614 >> 3);
+ WREG32(mmSIF_RTR_CTRL_6_E2E_PCI_WR_SIZE, 1);
+ WREG32(mmSIF_RTR_CTRL_6_E2E_PCI_RD_SIZE, 39);
+
+ WREG32(mmSIF_RTR_CTRL_7_E2E_HBM_WR_SIZE, 297 >> 3);
+ WREG32(mmSIF_RTR_CTRL_7_E2E_HBM_RD_SIZE, 908 >> 3);
+ WREG32(mmSIF_RTR_CTRL_7_E2E_PCI_WR_SIZE, 19);
+ WREG32(mmSIF_RTR_CTRL_7_E2E_PCI_RD_SIZE, 19);
+
+ WREG32(mmNIF_RTR_CTRL_0_E2E_HBM_WR_SIZE, 318 >> 3);
+ WREG32(mmNIF_RTR_CTRL_0_E2E_HBM_RD_SIZE, 956 >> 3);
+ WREG32(mmNIF_RTR_CTRL_0_E2E_PCI_WR_SIZE, 79);
+ WREG32(mmNIF_RTR_CTRL_0_E2E_PCI_RD_SIZE, 163);
+
+ WREG32(mmNIF_RTR_CTRL_1_E2E_HBM_WR_SIZE, 275 >> 3);
+ WREG32(mmNIF_RTR_CTRL_1_E2E_HBM_RD_SIZE, 614 >> 3);
+ WREG32(mmNIF_RTR_CTRL_1_E2E_PCI_WR_SIZE, 1);
+ WREG32(mmNIF_RTR_CTRL_1_E2E_PCI_RD_SIZE, 39);
+
+ WREG32(mmNIF_RTR_CTRL_2_E2E_HBM_WR_SIZE, 1);
+ WREG32(mmNIF_RTR_CTRL_2_E2E_HBM_RD_SIZE, 1);
+ WREG32(mmNIF_RTR_CTRL_2_E2E_PCI_WR_SIZE, 1);
+ WREG32(mmNIF_RTR_CTRL_2_E2E_PCI_RD_SIZE, 32);
+
+ WREG32(mmNIF_RTR_CTRL_3_E2E_HBM_WR_SIZE, 176 >> 3);
+ WREG32(mmNIF_RTR_CTRL_3_E2E_HBM_RD_SIZE, 32 >> 3);
+ WREG32(mmNIF_RTR_CTRL_3_E2E_PCI_WR_SIZE, 19);
+ WREG32(mmNIF_RTR_CTRL_3_E2E_PCI_RD_SIZE, 32);
+
+ WREG32(mmNIF_RTR_CTRL_4_E2E_HBM_WR_SIZE, 176 >> 3);
+ WREG32(mmNIF_RTR_CTRL_4_E2E_HBM_RD_SIZE, 32 >> 3);
+ WREG32(mmNIF_RTR_CTRL_4_E2E_PCI_WR_SIZE, 19);
+ WREG32(mmNIF_RTR_CTRL_4_E2E_PCI_RD_SIZE, 32);
+
+ WREG32(mmNIF_RTR_CTRL_5_E2E_HBM_WR_SIZE, 1);
+ WREG32(mmNIF_RTR_CTRL_5_E2E_HBM_RD_SIZE, 1);
+ WREG32(mmNIF_RTR_CTRL_5_E2E_PCI_WR_SIZE, 1);
+ WREG32(mmNIF_RTR_CTRL_5_E2E_PCI_RD_SIZE, 32);
+
+ WREG32(mmNIF_RTR_CTRL_6_E2E_HBM_WR_SIZE, 275 >> 3);
+ WREG32(mmNIF_RTR_CTRL_6_E2E_HBM_RD_SIZE, 614 >> 3);
+ WREG32(mmNIF_RTR_CTRL_6_E2E_PCI_WR_SIZE, 1);
+ WREG32(mmNIF_RTR_CTRL_6_E2E_PCI_RD_SIZE, 39);
+
+ WREG32(mmNIF_RTR_CTRL_7_E2E_HBM_WR_SIZE, 318 >> 3);
+ WREG32(mmNIF_RTR_CTRL_7_E2E_HBM_RD_SIZE, 956 >> 3);
+ WREG32(mmNIF_RTR_CTRL_7_E2E_PCI_WR_SIZE, 79);
+ WREG32(mmNIF_RTR_CTRL_7_E2E_PCI_RD_SIZE, 79);
+
+ WREG32(mmDMA_IF_E_N_DOWN_CH0_E2E_HBM_WR_SIZE, 344 >> 3);
+ WREG32(mmDMA_IF_E_N_DOWN_CH0_E2E_HBM_RD_SIZE, 1000 >> 3);
+ WREG32(mmDMA_IF_E_N_DOWN_CH0_E2E_PCI_WR_SIZE, 162);
+ WREG32(mmDMA_IF_E_N_DOWN_CH0_E2E_PCI_RD_SIZE, 338);
+
+ WREG32(mmDMA_IF_E_N_DOWN_CH1_E2E_HBM_WR_SIZE, 344 >> 3);
+ WREG32(mmDMA_IF_E_N_DOWN_CH1_E2E_HBM_RD_SIZE, 1000 >> 3);
+ WREG32(mmDMA_IF_E_N_DOWN_CH1_E2E_PCI_WR_SIZE, 162);
+ WREG32(mmDMA_IF_E_N_DOWN_CH1_E2E_PCI_RD_SIZE, 338);
+
+ WREG32(mmDMA_IF_E_S_DOWN_CH0_E2E_HBM_WR_SIZE, 344 >> 3);
+ WREG32(mmDMA_IF_E_S_DOWN_CH0_E2E_HBM_RD_SIZE, 1000 >> 3);
+ WREG32(mmDMA_IF_E_S_DOWN_CH0_E2E_PCI_WR_SIZE, 162);
+ WREG32(mmDMA_IF_E_S_DOWN_CH0_E2E_PCI_RD_SIZE, 338);
+
+ WREG32(mmDMA_IF_E_S_DOWN_CH1_E2E_HBM_WR_SIZE, 344 >> 3);
+ WREG32(mmDMA_IF_E_S_DOWN_CH1_E2E_HBM_RD_SIZE, 1000 >> 3);
+ WREG32(mmDMA_IF_E_S_DOWN_CH1_E2E_PCI_WR_SIZE, 162);
+ WREG32(mmDMA_IF_E_S_DOWN_CH1_E2E_PCI_RD_SIZE, 338);
+
+ WREG32(mmDMA_IF_W_N_DOWN_CH0_E2E_HBM_WR_SIZE, 344 >> 3);
+ WREG32(mmDMA_IF_W_N_DOWN_CH0_E2E_HBM_RD_SIZE, 1000 >> 3);
+ WREG32(mmDMA_IF_W_N_DOWN_CH0_E2E_PCI_WR_SIZE, 162);
+ WREG32(mmDMA_IF_W_N_DOWN_CH0_E2E_PCI_RD_SIZE, 338);
+
+ WREG32(mmDMA_IF_W_N_DOWN_CH1_E2E_HBM_WR_SIZE, 344 >> 3);
+ WREG32(mmDMA_IF_W_N_DOWN_CH1_E2E_HBM_RD_SIZE, 1000 >> 3);
+ WREG32(mmDMA_IF_W_N_DOWN_CH1_E2E_PCI_WR_SIZE, 162);
+ WREG32(mmDMA_IF_W_N_DOWN_CH1_E2E_PCI_RD_SIZE, 338);
+
+ WREG32(mmDMA_IF_W_S_DOWN_CH0_E2E_HBM_WR_SIZE, 344 >> 3);
+ WREG32(mmDMA_IF_W_S_DOWN_CH0_E2E_HBM_RD_SIZE, 1000 >> 3);
+ WREG32(mmDMA_IF_W_S_DOWN_CH0_E2E_PCI_WR_SIZE, 162);
+ WREG32(mmDMA_IF_W_S_DOWN_CH0_E2E_PCI_RD_SIZE, 338);
+
+ WREG32(mmDMA_IF_W_S_DOWN_CH1_E2E_HBM_WR_SIZE, 344 >> 3);
+ WREG32(mmDMA_IF_W_S_DOWN_CH1_E2E_HBM_RD_SIZE, 1000 >> 3);
+ WREG32(mmDMA_IF_W_S_DOWN_CH1_E2E_PCI_WR_SIZE, 162);
+ WREG32(mmDMA_IF_W_S_DOWN_CH1_E2E_PCI_RD_SIZE, 338);
+
+ if (!hdev->dram_scrambler_enable) {
+ WREG32(mmSIF_RTR_CTRL_0_NL_HBM_SEL_0, 0x21);
+ WREG32(mmSIF_RTR_CTRL_0_NL_HBM_SEL_1, 0x22);
+ WREG32(mmSIF_RTR_CTRL_0_NL_HBM_OFFSET_18, 0x1F);
+ WREG32(mmSIF_RTR_CTRL_0_NL_HBM_PC_SEL_3, 0x20);
+
+ WREG32(mmSIF_RTR_CTRL_1_NL_HBM_SEL_0, 0x21);
+ WREG32(mmSIF_RTR_CTRL_1_NL_HBM_SEL_1, 0x22);
+ WREG32(mmSIF_RTR_CTRL_1_NL_HBM_OFFSET_18, 0x1F);
+ WREG32(mmSIF_RTR_CTRL_1_NL_HBM_PC_SEL_3, 0x20);
+
+ WREG32(mmSIF_RTR_CTRL_2_NL_HBM_SEL_0, 0x21);
+ WREG32(mmSIF_RTR_CTRL_2_NL_HBM_SEL_1, 0x22);
+ WREG32(mmSIF_RTR_CTRL_2_NL_HBM_OFFSET_18, 0x1F);
+ WREG32(mmSIF_RTR_CTRL_2_NL_HBM_PC_SEL_3, 0x20);
+
+ WREG32(mmSIF_RTR_CTRL_3_NL_HBM_SEL_0, 0x21);
+ WREG32(mmSIF_RTR_CTRL_3_NL_HBM_SEL_1, 0x22);
+ WREG32(mmSIF_RTR_CTRL_3_NL_HBM_OFFSET_18, 0x1F);
+ WREG32(mmSIF_RTR_CTRL_3_NL_HBM_PC_SEL_3, 0x20);
+
+ WREG32(mmSIF_RTR_CTRL_4_NL_HBM_SEL_0, 0x21);
+ WREG32(mmSIF_RTR_CTRL_4_NL_HBM_SEL_1, 0x22);
+ WREG32(mmSIF_RTR_CTRL_4_NL_HBM_OFFSET_18, 0x1F);
+ WREG32(mmSIF_RTR_CTRL_4_NL_HBM_PC_SEL_3, 0x20);
+
+ WREG32(mmSIF_RTR_CTRL_5_NL_HBM_SEL_0, 0x21);
+ WREG32(mmSIF_RTR_CTRL_5_NL_HBM_SEL_1, 0x22);
+ WREG32(mmSIF_RTR_CTRL_5_NL_HBM_OFFSET_18, 0x1F);
+ WREG32(mmSIF_RTR_CTRL_5_NL_HBM_PC_SEL_3, 0x20);
+
+ WREG32(mmSIF_RTR_CTRL_6_NL_HBM_SEL_0, 0x21);
+ WREG32(mmSIF_RTR_CTRL_6_NL_HBM_SEL_1, 0x22);
+ WREG32(mmSIF_RTR_CTRL_6_NL_HBM_OFFSET_18, 0x1F);
+ WREG32(mmSIF_RTR_CTRL_6_NL_HBM_PC_SEL_3, 0x20);
+
+ WREG32(mmSIF_RTR_CTRL_7_NL_HBM_SEL_0, 0x21);
+ WREG32(mmSIF_RTR_CTRL_7_NL_HBM_SEL_1, 0x22);
+ WREG32(mmSIF_RTR_CTRL_7_NL_HBM_OFFSET_18, 0x1F);
+ WREG32(mmSIF_RTR_CTRL_7_NL_HBM_PC_SEL_3, 0x20);
+
+ WREG32(mmNIF_RTR_CTRL_0_NL_HBM_SEL_0, 0x21);
+ WREG32(mmNIF_RTR_CTRL_0_NL_HBM_SEL_1, 0x22);
+ WREG32(mmNIF_RTR_CTRL_0_NL_HBM_OFFSET_18, 0x1F);
+ WREG32(mmNIF_RTR_CTRL_0_NL_HBM_PC_SEL_3, 0x20);
+
+ WREG32(mmNIF_RTR_CTRL_1_NL_HBM_SEL_0, 0x21);
+ WREG32(mmNIF_RTR_CTRL_1_NL_HBM_SEL_1, 0x22);
+ WREG32(mmNIF_RTR_CTRL_1_NL_HBM_OFFSET_18, 0x1F);
+ WREG32(mmNIF_RTR_CTRL_1_NL_HBM_PC_SEL_3, 0x20);
+
+ WREG32(mmNIF_RTR_CTRL_2_NL_HBM_SEL_0, 0x21);
+ WREG32(mmNIF_RTR_CTRL_2_NL_HBM_SEL_1, 0x22);
+ WREG32(mmNIF_RTR_CTRL_2_NL_HBM_OFFSET_18, 0x1F);
+ WREG32(mmNIF_RTR_CTRL_2_NL_HBM_PC_SEL_3, 0x20);
+
+ WREG32(mmNIF_RTR_CTRL_3_NL_HBM_SEL_0, 0x21);
+ WREG32(mmNIF_RTR_CTRL_3_NL_HBM_SEL_1, 0x22);
+ WREG32(mmNIF_RTR_CTRL_3_NL_HBM_OFFSET_18, 0x1F);
+ WREG32(mmNIF_RTR_CTRL_3_NL_HBM_PC_SEL_3, 0x20);
+
+ WREG32(mmNIF_RTR_CTRL_4_NL_HBM_SEL_0, 0x21);
+ WREG32(mmNIF_RTR_CTRL_4_NL_HBM_SEL_1, 0x22);
+ WREG32(mmNIF_RTR_CTRL_4_NL_HBM_OFFSET_18, 0x1F);
+ WREG32(mmNIF_RTR_CTRL_4_NL_HBM_PC_SEL_3, 0x20);
+
+ WREG32(mmNIF_RTR_CTRL_5_NL_HBM_SEL_0, 0x21);
+ WREG32(mmNIF_RTR_CTRL_5_NL_HBM_SEL_1, 0x22);
+ WREG32(mmNIF_RTR_CTRL_5_NL_HBM_OFFSET_18, 0x1F);
+ WREG32(mmNIF_RTR_CTRL_5_NL_HBM_PC_SEL_3, 0x20);
+
+ WREG32(mmNIF_RTR_CTRL_6_NL_HBM_SEL_0, 0x21);
+ WREG32(mmNIF_RTR_CTRL_6_NL_HBM_SEL_1, 0x22);
+ WREG32(mmNIF_RTR_CTRL_6_NL_HBM_OFFSET_18, 0x1F);
+ WREG32(mmNIF_RTR_CTRL_6_NL_HBM_PC_SEL_3, 0x20);
+
+ WREG32(mmNIF_RTR_CTRL_7_NL_HBM_SEL_0, 0x21);
+ WREG32(mmNIF_RTR_CTRL_7_NL_HBM_SEL_1, 0x22);
+ WREG32(mmNIF_RTR_CTRL_7_NL_HBM_OFFSET_18, 0x1F);
+ WREG32(mmNIF_RTR_CTRL_7_NL_HBM_PC_SEL_3, 0x20);
+
+ WREG32(mmDMA_IF_E_N_DOWN_CH0_NL_HBM_SEL_0, 0x21);
+ WREG32(mmDMA_IF_E_N_DOWN_CH0_NL_HBM_SEL_1, 0x22);
+ WREG32(mmDMA_IF_E_N_DOWN_CH0_NL_HBM_OFFSET_18, 0x1F);
+ WREG32(mmDMA_IF_E_N_DOWN_CH0_NL_HBM_PC_SEL_3, 0x20);
+
+ WREG32(mmDMA_IF_E_N_DOWN_CH1_NL_HBM_SEL_0, 0x21);
+ WREG32(mmDMA_IF_E_N_DOWN_CH1_NL_HBM_SEL_1, 0x22);
+ WREG32(mmDMA_IF_E_N_DOWN_CH1_NL_HBM_OFFSET_18, 0x1F);
+ WREG32(mmDMA_IF_E_N_DOWN_CH1_NL_HBM_PC_SEL_3, 0x20);
+
+ WREG32(mmDMA_IF_E_S_DOWN_CH0_NL_HBM_SEL_0, 0x21);
+ WREG32(mmDMA_IF_E_S_DOWN_CH0_NL_HBM_SEL_1, 0x22);
+ WREG32(mmDMA_IF_E_S_DOWN_CH0_NL_HBM_OFFSET_18, 0x1F);
+ WREG32(mmDMA_IF_E_S_DOWN_CH0_NL_HBM_PC_SEL_3, 0x20);
+
+ WREG32(mmDMA_IF_E_S_DOWN_CH1_NL_HBM_SEL_0, 0x21);
+ WREG32(mmDMA_IF_E_S_DOWN_CH1_NL_HBM_SEL_1, 0x22);
+ WREG32(mmDMA_IF_E_S_DOWN_CH1_NL_HBM_OFFSET_18, 0x1F);
+ WREG32(mmDMA_IF_E_S_DOWN_CH1_NL_HBM_PC_SEL_3, 0x20);
+
+ WREG32(mmDMA_IF_W_N_DOWN_CH0_NL_HBM_SEL_0, 0x21);
+ WREG32(mmDMA_IF_W_N_DOWN_CH0_NL_HBM_SEL_1, 0x22);
+ WREG32(mmDMA_IF_W_N_DOWN_CH0_NL_HBM_OFFSET_18, 0x1F);
+ WREG32(mmDMA_IF_W_N_DOWN_CH0_NL_HBM_PC_SEL_3, 0x20);
+
+ WREG32(mmDMA_IF_W_N_DOWN_CH1_NL_HBM_SEL_0, 0x21);
+ WREG32(mmDMA_IF_W_N_DOWN_CH1_NL_HBM_SEL_1, 0x22);
+ WREG32(mmDMA_IF_W_N_DOWN_CH1_NL_HBM_OFFSET_18, 0x1F);
+ WREG32(mmDMA_IF_W_N_DOWN_CH1_NL_HBM_PC_SEL_3, 0x20);
+
+ WREG32(mmDMA_IF_W_S_DOWN_CH0_NL_HBM_SEL_0, 0x21);
+ WREG32(mmDMA_IF_W_S_DOWN_CH0_NL_HBM_SEL_1, 0x22);
+ WREG32(mmDMA_IF_W_S_DOWN_CH0_NL_HBM_OFFSET_18, 0x1F);
+ WREG32(mmDMA_IF_W_S_DOWN_CH0_NL_HBM_PC_SEL_3, 0x20);
+
+ WREG32(mmDMA_IF_W_S_DOWN_CH1_NL_HBM_SEL_0, 0x21);
+ WREG32(mmDMA_IF_W_S_DOWN_CH1_NL_HBM_SEL_1, 0x22);
+ WREG32(mmDMA_IF_W_S_DOWN_CH1_NL_HBM_OFFSET_18, 0x1F);
+ WREG32(mmDMA_IF_W_S_DOWN_CH1_NL_HBM_PC_SEL_3, 0x20);
+ }
+
+ WREG32(mmSIF_RTR_CTRL_0_E2E_HBM_EN,
+ 1 << IF_RTR_CTRL_E2E_HBM_EN_VAL_SHIFT);
+ WREG32(mmSIF_RTR_CTRL_0_E2E_PCI_EN,
+ 1 << IF_RTR_CTRL_E2E_PCI_EN_VAL_SHIFT);
+
+ WREG32(mmSIF_RTR_CTRL_1_E2E_HBM_EN,
+ 1 << IF_RTR_CTRL_E2E_HBM_EN_VAL_SHIFT);
+ WREG32(mmSIF_RTR_CTRL_1_E2E_PCI_EN,
+ 1 << IF_RTR_CTRL_E2E_PCI_EN_VAL_SHIFT);
+
+ WREG32(mmSIF_RTR_CTRL_2_E2E_HBM_EN,
+ 1 << IF_RTR_CTRL_E2E_HBM_EN_VAL_SHIFT);
+ WREG32(mmSIF_RTR_CTRL_2_E2E_PCI_EN,
+ 1 << IF_RTR_CTRL_E2E_PCI_EN_VAL_SHIFT);
+
+ WREG32(mmSIF_RTR_CTRL_3_E2E_HBM_EN,
+ 1 << IF_RTR_CTRL_E2E_HBM_EN_VAL_SHIFT);
+ WREG32(mmSIF_RTR_CTRL_3_E2E_PCI_EN,
+ 1 << IF_RTR_CTRL_E2E_PCI_EN_VAL_SHIFT);
+
+ WREG32(mmSIF_RTR_CTRL_4_E2E_HBM_EN,
+ 1 << IF_RTR_CTRL_E2E_HBM_EN_VAL_SHIFT);
+ WREG32(mmSIF_RTR_CTRL_4_E2E_PCI_EN,
+ 1 << IF_RTR_CTRL_E2E_PCI_EN_VAL_SHIFT);
+
+ WREG32(mmSIF_RTR_CTRL_5_E2E_HBM_EN,
+ 1 << IF_RTR_CTRL_E2E_HBM_EN_VAL_SHIFT);
+ WREG32(mmSIF_RTR_CTRL_5_E2E_PCI_EN,
+ 1 << IF_RTR_CTRL_E2E_PCI_EN_VAL_SHIFT);
+
+ WREG32(mmSIF_RTR_CTRL_6_E2E_HBM_EN,
+ 1 << IF_RTR_CTRL_E2E_HBM_EN_VAL_SHIFT);
+ WREG32(mmSIF_RTR_CTRL_6_E2E_PCI_EN,
+ 1 << IF_RTR_CTRL_E2E_PCI_EN_VAL_SHIFT);
+
+ WREG32(mmSIF_RTR_CTRL_7_E2E_HBM_EN,
+ 1 << IF_RTR_CTRL_E2E_HBM_EN_VAL_SHIFT);
+ WREG32(mmSIF_RTR_CTRL_7_E2E_PCI_EN,
+ 1 << IF_RTR_CTRL_E2E_PCI_EN_VAL_SHIFT);
+
+ WREG32(mmNIF_RTR_CTRL_0_E2E_HBM_EN,
+ 1 << IF_RTR_CTRL_E2E_HBM_EN_VAL_SHIFT);
+ WREG32(mmNIF_RTR_CTRL_0_E2E_PCI_EN,
+ 1 << IF_RTR_CTRL_E2E_PCI_EN_VAL_SHIFT);
+
+ WREG32(mmNIF_RTR_CTRL_1_E2E_HBM_EN,
+ 1 << IF_RTR_CTRL_E2E_HBM_EN_VAL_SHIFT);
+ WREG32(mmNIF_RTR_CTRL_1_E2E_PCI_EN,
+ 1 << IF_RTR_CTRL_E2E_PCI_EN_VAL_SHIFT);
+
+ WREG32(mmNIF_RTR_CTRL_2_E2E_HBM_EN,
+ 1 << IF_RTR_CTRL_E2E_HBM_EN_VAL_SHIFT);
+ WREG32(mmNIF_RTR_CTRL_2_E2E_PCI_EN,
+ 1 << IF_RTR_CTRL_E2E_PCI_EN_VAL_SHIFT);
+
+ WREG32(mmNIF_RTR_CTRL_3_E2E_HBM_EN,
+ 1 << IF_RTR_CTRL_E2E_HBM_EN_VAL_SHIFT);
+ WREG32(mmNIF_RTR_CTRL_3_E2E_PCI_EN,
+ 1 << IF_RTR_CTRL_E2E_PCI_EN_VAL_SHIFT);
+
+ WREG32(mmNIF_RTR_CTRL_4_E2E_HBM_EN,
+ 1 << IF_RTR_CTRL_E2E_HBM_EN_VAL_SHIFT);
+ WREG32(mmNIF_RTR_CTRL_4_E2E_PCI_EN,
+ 1 << IF_RTR_CTRL_E2E_PCI_EN_VAL_SHIFT);
+
+ WREG32(mmNIF_RTR_CTRL_5_E2E_HBM_EN,
+ 1 << IF_RTR_CTRL_E2E_HBM_EN_VAL_SHIFT);
+ WREG32(mmNIF_RTR_CTRL_5_E2E_PCI_EN,
+ 1 << IF_RTR_CTRL_E2E_PCI_EN_VAL_SHIFT);
+
+ WREG32(mmNIF_RTR_CTRL_6_E2E_HBM_EN,
+ 1 << IF_RTR_CTRL_E2E_HBM_EN_VAL_SHIFT);
+ WREG32(mmNIF_RTR_CTRL_6_E2E_PCI_EN,
+ 1 << IF_RTR_CTRL_E2E_PCI_EN_VAL_SHIFT);
+
+ WREG32(mmNIF_RTR_CTRL_7_E2E_HBM_EN,
+ 1 << IF_RTR_CTRL_E2E_HBM_EN_VAL_SHIFT);
+ WREG32(mmNIF_RTR_CTRL_7_E2E_PCI_EN,
+ 1 << IF_RTR_CTRL_E2E_PCI_EN_VAL_SHIFT);
+
+ WREG32(mmDMA_IF_E_N_DOWN_CH0_E2E_HBM_EN,
+ 1 << DMA_IF_DOWN_CHX_E2E_HBM_EN_VAL_SHIFT);
+ WREG32(mmDMA_IF_E_N_DOWN_CH0_E2E_PCI_EN,
+ 1 << DMA_IF_DOWN_CHX_E2E_PCI_EN_VAL_SHIFT);
+
+ WREG32(mmDMA_IF_E_N_DOWN_CH1_E2E_HBM_EN,
+ 1 << DMA_IF_DOWN_CHX_E2E_HBM_EN_VAL_SHIFT);
+ WREG32(mmDMA_IF_E_N_DOWN_CH1_E2E_PCI_EN,
+ 1 << DMA_IF_DOWN_CHX_E2E_PCI_EN_VAL_SHIFT);
+
+ WREG32(mmDMA_IF_E_S_DOWN_CH0_E2E_HBM_EN,
+ 1 << DMA_IF_DOWN_CHX_E2E_HBM_EN_VAL_SHIFT);
+ WREG32(mmDMA_IF_E_S_DOWN_CH0_E2E_PCI_EN,
+ 1 << DMA_IF_DOWN_CHX_E2E_PCI_EN_VAL_SHIFT);
+
+ WREG32(mmDMA_IF_E_S_DOWN_CH1_E2E_HBM_EN,
+ 1 << DMA_IF_DOWN_CHX_E2E_HBM_EN_VAL_SHIFT);
+ WREG32(mmDMA_IF_E_S_DOWN_CH1_E2E_PCI_EN,
+ 1 << DMA_IF_DOWN_CHX_E2E_PCI_EN_VAL_SHIFT);
+
+ WREG32(mmDMA_IF_W_N_DOWN_CH0_E2E_HBM_EN,
+ 1 << DMA_IF_DOWN_CHX_E2E_HBM_EN_VAL_SHIFT);
+ WREG32(mmDMA_IF_W_N_DOWN_CH0_E2E_PCI_EN,
+ 1 << DMA_IF_DOWN_CHX_E2E_PCI_EN_VAL_SHIFT);
+
+ WREG32(mmDMA_IF_W_N_DOWN_CH1_E2E_HBM_EN,
+ 1 << DMA_IF_DOWN_CHX_E2E_HBM_EN_VAL_SHIFT);
+ WREG32(mmDMA_IF_W_N_DOWN_CH1_E2E_PCI_EN,
+ 1 << DMA_IF_DOWN_CHX_E2E_PCI_EN_VAL_SHIFT);
+
+ WREG32(mmDMA_IF_W_S_DOWN_CH0_E2E_HBM_EN,
+ 1 << DMA_IF_DOWN_CHX_E2E_HBM_EN_VAL_SHIFT);
+ WREG32(mmDMA_IF_W_S_DOWN_CH0_E2E_PCI_EN,
+ 1 << DMA_IF_DOWN_CHX_E2E_PCI_EN_VAL_SHIFT);
+
+ WREG32(mmDMA_IF_W_S_DOWN_CH1_E2E_HBM_EN,
+ 1 << DMA_IF_DOWN_CHX_E2E_HBM_EN_VAL_SHIFT);
+ WREG32(mmDMA_IF_W_S_DOWN_CH1_E2E_PCI_EN,
+ 1 << DMA_IF_DOWN_CHX_E2E_PCI_EN_VAL_SHIFT);
+}
+
+static void gaudi_init_hbm_cred(struct hl_device *hdev)
+{
+ uint32_t hbm0_wr, hbm1_wr, hbm0_rd, hbm1_rd;
+
+ hbm0_wr = 0x33333333;
+ hbm1_wr = 0x33333333;
+ hbm0_rd = 0x77777777;
+ hbm1_rd = 0xDDDDDDDD;
+
+ WREG32(mmDMA_IF_E_N_HBM0_WR_CRED_CNT, hbm0_wr);
+ WREG32(mmDMA_IF_E_N_HBM1_WR_CRED_CNT, hbm1_wr);
+ WREG32(mmDMA_IF_E_N_HBM0_RD_CRED_CNT, hbm0_rd);
+ WREG32(mmDMA_IF_E_N_HBM1_RD_CRED_CNT, hbm1_rd);
+
+ WREG32(mmDMA_IF_E_S_HBM0_WR_CRED_CNT, hbm0_wr);
+ WREG32(mmDMA_IF_E_S_HBM1_WR_CRED_CNT, hbm1_wr);
+ WREG32(mmDMA_IF_E_S_HBM0_RD_CRED_CNT, hbm0_rd);
+ WREG32(mmDMA_IF_E_S_HBM1_RD_CRED_CNT, hbm1_rd);
+
+ WREG32(mmDMA_IF_W_N_HBM0_WR_CRED_CNT, hbm0_wr);
+ WREG32(mmDMA_IF_W_N_HBM1_WR_CRED_CNT, hbm1_wr);
+ WREG32(mmDMA_IF_W_N_HBM0_RD_CRED_CNT, hbm0_rd);
+ WREG32(mmDMA_IF_W_N_HBM1_RD_CRED_CNT, hbm1_rd);
+
+ WREG32(mmDMA_IF_W_S_HBM0_WR_CRED_CNT, hbm0_wr);
+ WREG32(mmDMA_IF_W_S_HBM1_WR_CRED_CNT, hbm1_wr);
+ WREG32(mmDMA_IF_W_S_HBM0_RD_CRED_CNT, hbm0_rd);
+ WREG32(mmDMA_IF_W_S_HBM1_RD_CRED_CNT, hbm1_rd);
+
+ WREG32(mmDMA_IF_E_N_HBM_CRED_EN_0,
+ (1 << DMA_IF_HBM_CRED_EN_READ_CREDIT_EN_SHIFT) |
+ (1 << DMA_IF_HBM_CRED_EN_WRITE_CREDIT_EN_SHIFT));
+ WREG32(mmDMA_IF_E_S_HBM_CRED_EN_0,
+ (1 << DMA_IF_HBM_CRED_EN_READ_CREDIT_EN_SHIFT) |
+ (1 << DMA_IF_HBM_CRED_EN_WRITE_CREDIT_EN_SHIFT));
+ WREG32(mmDMA_IF_W_N_HBM_CRED_EN_0,
+ (1 << DMA_IF_HBM_CRED_EN_READ_CREDIT_EN_SHIFT) |
+ (1 << DMA_IF_HBM_CRED_EN_WRITE_CREDIT_EN_SHIFT));
+ WREG32(mmDMA_IF_W_S_HBM_CRED_EN_0,
+ (1 << DMA_IF_HBM_CRED_EN_READ_CREDIT_EN_SHIFT) |
+ (1 << DMA_IF_HBM_CRED_EN_WRITE_CREDIT_EN_SHIFT));
+
+ WREG32(mmDMA_IF_E_N_HBM_CRED_EN_1,
+ (1 << DMA_IF_HBM_CRED_EN_READ_CREDIT_EN_SHIFT) |
+ (1 << DMA_IF_HBM_CRED_EN_WRITE_CREDIT_EN_SHIFT));
+ WREG32(mmDMA_IF_E_S_HBM_CRED_EN_1,
+ (1 << DMA_IF_HBM_CRED_EN_READ_CREDIT_EN_SHIFT) |
+ (1 << DMA_IF_HBM_CRED_EN_WRITE_CREDIT_EN_SHIFT));
+ WREG32(mmDMA_IF_W_N_HBM_CRED_EN_1,
+ (1 << DMA_IF_HBM_CRED_EN_READ_CREDIT_EN_SHIFT) |
+ (1 << DMA_IF_HBM_CRED_EN_WRITE_CREDIT_EN_SHIFT));
+ WREG32(mmDMA_IF_W_S_HBM_CRED_EN_1,
+ (1 << DMA_IF_HBM_CRED_EN_READ_CREDIT_EN_SHIFT) |
+ (1 << DMA_IF_HBM_CRED_EN_WRITE_CREDIT_EN_SHIFT));
+}
+
+static void gaudi_init_rate_limiter(struct hl_device *hdev)
+{
+ u32 nr, nf, od, sat, rst, timeout;
+ u64 freq;
+
+ nr = RREG32(mmPSOC_HBM_PLL_NR);
+ nf = RREG32(mmPSOC_HBM_PLL_NF);
+ od = RREG32(mmPSOC_HBM_PLL_OD);
+ freq = (50 * (nf + 1)) / ((nr + 1) * (od + 1));
+
+ dev_dbg(hdev->dev, "HBM frequency is %lluMHz\n", freq);
+
+ /* Configuration is for five (5) DDMA channels */
+ if (freq == 800) {
+ sat = 4;
+ rst = 11;
+ timeout = 15;
+ } else if (freq == 900) {
+ sat = 4;
+ rst = 15;
+ timeout = 16;
+ } else if (freq == 950) {
+ sat = 4;
+ rst = 15;
+ timeout = 15;
+ } else {
+ dev_warn(hdev->dev,
+ "unsupported HBM frequency %lluMHz, no rate-limiters\n",
+ freq);
+ return;
+ }
+
+ WREG32(mmDMA_IF_W_S_DOWN_RSP_MID_WGHT_0, 0x111);
+ WREG32(mmDMA_IF_W_S_DOWN_RSP_MID_WGHT_1, 0x111);
+ WREG32(mmDMA_IF_E_S_DOWN_RSP_MID_WGHT_0, 0x111);
+ WREG32(mmDMA_IF_E_S_DOWN_RSP_MID_WGHT_1, 0x111);
+ WREG32(mmDMA_IF_W_N_DOWN_RSP_MID_WGHT_0, 0x111);
+ WREG32(mmDMA_IF_W_N_DOWN_RSP_MID_WGHT_1, 0x111);
+ WREG32(mmDMA_IF_E_N_DOWN_RSP_MID_WGHT_0, 0x111);
+ WREG32(mmDMA_IF_E_N_DOWN_RSP_MID_WGHT_1, 0x111);
+
+ if (!hdev->rl_enable) {
+ dev_info(hdev->dev, "Rate limiters disabled\n");
+ return;
+ }
+
+ WREG32(mmDMA_IF_W_S_DOWN_CH0_RL_HBM_SAT, sat);
+ WREG32(mmDMA_IF_W_S_DOWN_CH1_RL_HBM_SAT, sat);
+ WREG32(mmDMA_IF_E_S_DOWN_CH0_RL_HBM_SAT, sat);
+ WREG32(mmDMA_IF_E_S_DOWN_CH1_RL_HBM_SAT, sat);
+ WREG32(mmDMA_IF_W_N_DOWN_CH0_RL_HBM_SAT, sat);
+ WREG32(mmDMA_IF_W_N_DOWN_CH1_RL_HBM_SAT, sat);
+ WREG32(mmDMA_IF_E_N_DOWN_CH0_RL_HBM_SAT, sat);
+ WREG32(mmDMA_IF_E_N_DOWN_CH1_RL_HBM_SAT, sat);
+
+ WREG32(mmDMA_IF_W_S_DOWN_CH0_RL_HBM_RST, rst);
+ WREG32(mmDMA_IF_W_S_DOWN_CH1_RL_HBM_RST, rst);
+ WREG32(mmDMA_IF_E_S_DOWN_CH0_RL_HBM_RST, rst);
+ WREG32(mmDMA_IF_E_S_DOWN_CH1_RL_HBM_RST, rst);
+ WREG32(mmDMA_IF_W_N_DOWN_CH0_RL_HBM_RST, rst);
+ WREG32(mmDMA_IF_W_N_DOWN_CH1_RL_HBM_RST, rst);
+ WREG32(mmDMA_IF_E_N_DOWN_CH0_RL_HBM_RST, rst);
+ WREG32(mmDMA_IF_E_N_DOWN_CH1_RL_HBM_RST, rst);
+
+ WREG32(mmDMA_IF_W_S_DOWN_CH0_RL_HBM_TIMEOUT, timeout);
+ WREG32(mmDMA_IF_W_S_DOWN_CH1_RL_HBM_TIMEOUT, timeout);
+ WREG32(mmDMA_IF_E_S_DOWN_CH0_RL_HBM_TIMEOUT, timeout);
+ WREG32(mmDMA_IF_E_S_DOWN_CH1_RL_HBM_TIMEOUT, timeout);
+ WREG32(mmDMA_IF_W_N_DOWN_CH0_RL_HBM_TIMEOUT, timeout);
+ WREG32(mmDMA_IF_W_N_DOWN_CH1_RL_HBM_TIMEOUT, timeout);
+ WREG32(mmDMA_IF_E_N_DOWN_CH0_RL_HBM_TIMEOUT, timeout);
+ WREG32(mmDMA_IF_E_N_DOWN_CH1_RL_HBM_TIMEOUT, timeout);
+
+ WREG32(mmDMA_IF_W_S_DOWN_CH0_RL_HBM_EN, 1);
+ WREG32(mmDMA_IF_W_S_DOWN_CH1_RL_HBM_EN, 1);
+ WREG32(mmDMA_IF_E_S_DOWN_CH0_RL_HBM_EN, 1);
+ WREG32(mmDMA_IF_E_S_DOWN_CH1_RL_HBM_EN, 1);
+ WREG32(mmDMA_IF_W_N_DOWN_CH0_RL_HBM_EN, 1);
+ WREG32(mmDMA_IF_W_N_DOWN_CH1_RL_HBM_EN, 1);
+ WREG32(mmDMA_IF_E_N_DOWN_CH0_RL_HBM_EN, 1);
+ WREG32(mmDMA_IF_E_N_DOWN_CH1_RL_HBM_EN, 1);
+
+ WREG32(mmDMA_IF_W_S_DOWN_CH0_RL_SRAM_SAT, sat);
+ WREG32(mmDMA_IF_W_S_DOWN_CH1_RL_SRAM_SAT, sat);
+ WREG32(mmDMA_IF_E_S_DOWN_CH0_RL_SRAM_SAT, sat);
+ WREG32(mmDMA_IF_E_S_DOWN_CH1_RL_SRAM_SAT, sat);
+ WREG32(mmDMA_IF_W_N_DOWN_CH0_RL_SRAM_SAT, sat);
+ WREG32(mmDMA_IF_W_N_DOWN_CH1_RL_SRAM_SAT, sat);
+ WREG32(mmDMA_IF_E_N_DOWN_CH0_RL_SRAM_SAT, sat);
+ WREG32(mmDMA_IF_E_N_DOWN_CH1_RL_SRAM_SAT, sat);
+
+ WREG32(mmDMA_IF_W_S_DOWN_CH0_RL_SRAM_RST, rst);
+ WREG32(mmDMA_IF_W_S_DOWN_CH1_RL_SRAM_RST, rst);
+ WREG32(mmDMA_IF_E_S_DOWN_CH0_RL_SRAM_RST, rst);
+ WREG32(mmDMA_IF_E_S_DOWN_CH1_RL_SRAM_RST, rst);
+ WREG32(mmDMA_IF_W_N_DOWN_CH0_RL_SRAM_RST, rst);
+ WREG32(mmDMA_IF_W_N_DOWN_CH1_RL_SRAM_RST, rst);
+ WREG32(mmDMA_IF_E_N_DOWN_CH0_RL_SRAM_RST, rst);
+ WREG32(mmDMA_IF_E_N_DOWN_CH1_RL_SRAM_RST, rst);
+
+ WREG32(mmDMA_IF_W_S_DOWN_CH0_RL_SRAM_TIMEOUT, timeout);
+ WREG32(mmDMA_IF_W_S_DOWN_CH1_RL_SRAM_TIMEOUT, timeout);
+ WREG32(mmDMA_IF_E_S_DOWN_CH0_RL_SRAM_TIMEOUT, timeout);
+ WREG32(mmDMA_IF_E_S_DOWN_CH1_RL_SRAM_TIMEOUT, timeout);
+ WREG32(mmDMA_IF_W_N_DOWN_CH0_RL_SRAM_TIMEOUT, timeout);
+ WREG32(mmDMA_IF_W_N_DOWN_CH1_RL_SRAM_TIMEOUT, timeout);
+ WREG32(mmDMA_IF_E_N_DOWN_CH0_RL_SRAM_TIMEOUT, timeout);
+ WREG32(mmDMA_IF_E_N_DOWN_CH1_RL_SRAM_TIMEOUT, timeout);
+
+ WREG32(mmDMA_IF_W_S_DOWN_CH0_RL_SRAM_EN, 1);
+ WREG32(mmDMA_IF_W_S_DOWN_CH1_RL_SRAM_EN, 1);
+ WREG32(mmDMA_IF_E_S_DOWN_CH0_RL_SRAM_EN, 1);
+ WREG32(mmDMA_IF_E_S_DOWN_CH1_RL_SRAM_EN, 1);
+ WREG32(mmDMA_IF_W_N_DOWN_CH0_RL_SRAM_EN, 1);
+ WREG32(mmDMA_IF_W_N_DOWN_CH1_RL_SRAM_EN, 1);
+ WREG32(mmDMA_IF_E_N_DOWN_CH0_RL_SRAM_EN, 1);
+ WREG32(mmDMA_IF_E_N_DOWN_CH1_RL_SRAM_EN, 1);
+}
+
+static void gaudi_init_golden_registers(struct hl_device *hdev)
+{
+ u32 tpc_offset;
+ int tpc_id, i;
+
+ gaudi_init_e2e(hdev);
+
+ gaudi_init_hbm_cred(hdev);
+
+ gaudi_init_rate_limiter(hdev);
+
+ gaudi_disable_clock_gating(hdev);
+
+ for (tpc_id = 0, tpc_offset = 0;
+ tpc_id < TPC_NUMBER_OF_ENGINES;
+ tpc_id++, tpc_offset += TPC_CFG_OFFSET) {
+ /* Mask all arithmetic interrupts from TPC */
+ WREG32(mmTPC0_CFG_TPC_INTR_MASK + tpc_offset, 0x8FFF);
+ /* Set 16 cache lines */
+ WREG32_FIELD(TPC0_CFG_MSS_CONFIG, tpc_offset,
+ ICACHE_FETCH_LINE_NUM, 2);
+ }
+
+ /* Make sure 1st 128 bytes in SRAM are 0 for Tensor DMA */
+ for (i = 0 ; i < 128 ; i += 8)
+ writeq(0, hdev->pcie_bar[SRAM_BAR_ID] + i);
+
+ WREG32(mmMME0_CTRL_EUS_ROLLUP_CNT_ADD, 3);
+ WREG32(mmMME1_CTRL_EUS_ROLLUP_CNT_ADD, 3);
+ WREG32(mmMME2_CTRL_EUS_ROLLUP_CNT_ADD, 3);
+ WREG32(mmMME3_CTRL_EUS_ROLLUP_CNT_ADD, 3);
+
+ /* WA for H3-2081 */
+ WREG32(mmPCIE_WRAP_MAX_OUTSTAND, 0x10ff);
+}
+
+static void gaudi_init_pci_dma_qman(struct hl_device *hdev, int dma_id,
+ int qman_id, dma_addr_t qman_pq_addr)
+{
+ u32 mtr_base_en_lo, mtr_base_en_hi, mtr_base_ws_lo, mtr_base_ws_hi;
+ u32 so_base_en_lo, so_base_en_hi, so_base_ws_lo, so_base_ws_hi;
+ u32 q_off, dma_qm_offset;
+ u32 dma_qm_err_cfg;
+
+ dma_qm_offset = dma_id * DMA_QMAN_OFFSET;
+
+ mtr_base_en_lo = lower_32_bits(CFG_BASE +
+ mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0);
+ mtr_base_en_hi = upper_32_bits(CFG_BASE +
+ mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0);
+ so_base_en_lo = lower_32_bits(CFG_BASE +
+ mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_SOB_OBJ_0);
+ so_base_en_hi = upper_32_bits(CFG_BASE +
+ mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_SOB_OBJ_0);
+ mtr_base_ws_lo = lower_32_bits(CFG_BASE +
+ mmSYNC_MNGR_W_S_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0);
+ mtr_base_ws_hi = upper_32_bits(CFG_BASE +
+ mmSYNC_MNGR_W_S_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0);
+ so_base_ws_lo = lower_32_bits(CFG_BASE +
+ mmSYNC_MNGR_W_S_SYNC_MNGR_OBJS_SOB_OBJ_0);
+ so_base_ws_hi = upper_32_bits(CFG_BASE +
+ mmSYNC_MNGR_W_S_SYNC_MNGR_OBJS_SOB_OBJ_0);
+
+ q_off = dma_qm_offset + qman_id * 4;
+
+ WREG32(mmDMA0_QM_PQ_BASE_LO_0 + q_off, lower_32_bits(qman_pq_addr));
+ WREG32(mmDMA0_QM_PQ_BASE_HI_0 + q_off, upper_32_bits(qman_pq_addr));
+
+ WREG32(mmDMA0_QM_PQ_SIZE_0 + q_off, ilog2(HL_QUEUE_LENGTH));
+ WREG32(mmDMA0_QM_PQ_PI_0 + q_off, 0);
+ WREG32(mmDMA0_QM_PQ_CI_0 + q_off, 0);
+
+ WREG32(mmDMA0_QM_CP_LDMA_TSIZE_OFFSET_0 + q_off, 0x74);
+ WREG32(mmDMA0_QM_CP_LDMA_SRC_BASE_LO_OFFSET_0 + q_off, 0x14);
+ WREG32(mmDMA0_QM_CP_LDMA_DST_BASE_LO_OFFSET_0 + q_off, 0x1C);
+
+ WREG32(mmDMA0_QM_CP_MSG_BASE0_ADDR_LO_0 + q_off, mtr_base_en_lo);
+ WREG32(mmDMA0_QM_CP_MSG_BASE0_ADDR_HI_0 + q_off, mtr_base_en_hi);
+ WREG32(mmDMA0_QM_CP_MSG_BASE1_ADDR_LO_0 + q_off, so_base_en_lo);
+ WREG32(mmDMA0_QM_CP_MSG_BASE1_ADDR_HI_0 + q_off, so_base_en_hi);
+ WREG32(mmDMA0_QM_CP_MSG_BASE2_ADDR_LO_0 + q_off, mtr_base_ws_lo);
+ WREG32(mmDMA0_QM_CP_MSG_BASE2_ADDR_HI_0 + q_off, mtr_base_ws_hi);
+ WREG32(mmDMA0_QM_CP_MSG_BASE3_ADDR_LO_0 + q_off, so_base_ws_lo);
+ WREG32(mmDMA0_QM_CP_MSG_BASE3_ADDR_HI_0 + q_off, so_base_ws_hi);
+
+ /* The following configuration is needed only once per QMAN */
+ if (qman_id == 0) {
+ /* Configure RAZWI IRQ */
+ dma_qm_err_cfg = PCI_DMA_QMAN_GLBL_ERR_CFG_MSG_EN_MASK;
+ if (hdev->stop_on_err) {
+ dma_qm_err_cfg |=
+ PCI_DMA_QMAN_GLBL_ERR_CFG_STOP_ON_ERR_EN_MASK;
+ }
+
+ WREG32(mmDMA0_QM_GLBL_ERR_CFG + dma_qm_offset, dma_qm_err_cfg);
+ WREG32(mmDMA0_QM_GLBL_ERR_ADDR_LO + dma_qm_offset,
+ lower_32_bits(CFG_BASE +
+ mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR));
+ WREG32(mmDMA0_QM_GLBL_ERR_ADDR_HI + dma_qm_offset,
+ upper_32_bits(CFG_BASE +
+ mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR));
+ WREG32(mmDMA0_QM_GLBL_ERR_WDATA + dma_qm_offset,
+ gaudi_irq_map_table[GAUDI_EVENT_DMA0_QM].cpu_id +
+ dma_id);
+
+ WREG32(mmDMA0_QM_ARB_ERR_MSG_EN + dma_qm_offset,
+ QM_ARB_ERR_MSG_EN_MASK);
+
+ /* Increase ARB WDT to support streams architecture */
+ WREG32(mmDMA0_QM_ARB_SLV_CHOISE_WDT + dma_qm_offset,
+ GAUDI_ARB_WDT_TIMEOUT);
+
+ WREG32(mmDMA0_QM_GLBL_PROT + dma_qm_offset,
+ QMAN_EXTERNAL_MAKE_TRUSTED);
+
+ WREG32(mmDMA0_QM_GLBL_CFG1 + dma_qm_offset, 0);
+ }
+}
+
+static void gaudi_init_dma_core(struct hl_device *hdev, int dma_id)
+{
+ u32 dma_offset = dma_id * DMA_CORE_OFFSET;
+ u32 dma_err_cfg = 1 << DMA0_CORE_ERR_CFG_ERR_MSG_EN_SHIFT;
+
+ /* Set to maximum possible according to physical size */
+ WREG32(mmDMA0_CORE_RD_MAX_OUTSTAND + dma_offset, 0);
+ WREG32(mmDMA0_CORE_RD_MAX_SIZE + dma_offset, 0);
+
+ /* STOP_ON bit implies no completion to operation in case of RAZWI */
+ if (hdev->stop_on_err)
+ dma_err_cfg |= 1 << DMA0_CORE_ERR_CFG_STOP_ON_ERR_SHIFT;
+
+ WREG32(mmDMA0_CORE_ERR_CFG + dma_offset, dma_err_cfg);
+ WREG32(mmDMA0_CORE_ERRMSG_ADDR_LO + dma_offset,
+ lower_32_bits(CFG_BASE + mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR));
+ WREG32(mmDMA0_CORE_ERRMSG_ADDR_HI + dma_offset,
+ upper_32_bits(CFG_BASE + mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR));
+ WREG32(mmDMA0_CORE_ERRMSG_WDATA + dma_offset,
+ gaudi_irq_map_table[GAUDI_EVENT_DMA0_CORE].cpu_id + dma_id);
+ WREG32(mmDMA0_CORE_PROT + dma_offset,
+ 1 << DMA0_CORE_PROT_ERR_VAL_SHIFT);
+ /* If the channel is secured, it should be in MMU bypass mode */
+ WREG32(mmDMA0_CORE_SECURE_PROPS + dma_offset,
+ 1 << DMA0_CORE_SECURE_PROPS_MMBP_SHIFT);
+ WREG32(mmDMA0_CORE_CFG_0 + dma_offset, 1 << DMA0_CORE_CFG_0_EN_SHIFT);
+}
+
+static void gaudi_enable_qman(struct hl_device *hdev, int dma_id,
+ u32 enable_mask)
+{
+ u32 dma_qm_offset = dma_id * DMA_QMAN_OFFSET;
+
+ WREG32(mmDMA0_QM_GLBL_CFG0 + dma_qm_offset, enable_mask);
+}
+
+static void gaudi_init_pci_dma_qmans(struct hl_device *hdev)
+{
+ struct gaudi_device *gaudi = hdev->asic_specific;
+ struct hl_hw_queue *q;
+ int i, j, dma_id, cpu_skip, nic_skip, cq_id = 0, q_idx, msi_vec = 0;
+
+ if (gaudi->hw_cap_initialized & HW_CAP_PCI_DMA)
+ return;
+
+ for (i = 0 ; i < PCI_DMA_NUMBER_OF_CHNLS ; i++) {
+ dma_id = gaudi_dma_assignment[i];
+ /*
+ * For queues after the CPU Q need to add 1 to get the correct
+ * queue. In addition, need to add the CPU EQ and NIC IRQs in
+ * order to get the correct MSI register.
+ */
+ if (dma_id > 1) {
+ cpu_skip = 1;
+ nic_skip = NIC_NUMBER_OF_ENGINES;
+ } else {
+ cpu_skip = 0;
+ nic_skip = 0;
+ }
+
+ for (j = 0 ; j < QMAN_STREAMS ; j++) {
+ q_idx = 4 * dma_id + j + cpu_skip;
+ q = &hdev->kernel_queues[q_idx];
+ q->cq_id = cq_id++;
+ q->msi_vec = nic_skip + cpu_skip + msi_vec++;
+ gaudi_init_pci_dma_qman(hdev, dma_id, j,
+ q->bus_address);
+ }
+
+ gaudi_init_dma_core(hdev, dma_id);
+
+ gaudi_enable_qman(hdev, dma_id, PCI_DMA_QMAN_ENABLE);
+ }
+
+ gaudi->hw_cap_initialized |= HW_CAP_PCI_DMA;
+}
+
+static void gaudi_init_hbm_dma_qman(struct hl_device *hdev, int dma_id,
+ int qman_id, u64 qman_base_addr)
+{
+ u32 mtr_base_lo, mtr_base_hi;
+ u32 so_base_lo, so_base_hi;
+ u32 q_off, dma_qm_offset;
+ u32 dma_qm_err_cfg;
+
+ dma_qm_offset = dma_id * DMA_QMAN_OFFSET;
+
+ mtr_base_lo = lower_32_bits(CFG_BASE +
+ mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0);
+ mtr_base_hi = upper_32_bits(CFG_BASE +
+ mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0);
+ so_base_lo = lower_32_bits(CFG_BASE +
+ mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_SOB_OBJ_0);
+ so_base_hi = upper_32_bits(CFG_BASE +
+ mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_SOB_OBJ_0);
+
+ q_off = dma_qm_offset + qman_id * 4;
+
+ if (qman_id < 4) {
+ WREG32(mmDMA0_QM_PQ_BASE_LO_0 + q_off,
+ lower_32_bits(qman_base_addr));
+ WREG32(mmDMA0_QM_PQ_BASE_HI_0 + q_off,
+ upper_32_bits(qman_base_addr));
+
+ WREG32(mmDMA0_QM_PQ_SIZE_0 + q_off, ilog2(HBM_DMA_QMAN_LENGTH));
+ WREG32(mmDMA0_QM_PQ_PI_0 + q_off, 0);
+ WREG32(mmDMA0_QM_PQ_CI_0 + q_off, 0);
+
+ WREG32(mmDMA0_QM_CP_LDMA_TSIZE_OFFSET_0 + q_off, 0x81BC);
+ WREG32(mmDMA0_QM_CP_LDMA_SRC_BASE_LO_OFFSET_0 + q_off, 0x81B4);
+ WREG32(mmDMA0_QM_CP_LDMA_DST_BASE_LO_OFFSET_0 + q_off, 0x1C);
+ } else {
+ WREG32(mmDMA0_QM_CP_LDMA_TSIZE_OFFSET_0 + q_off, 0x74);
+ WREG32(mmDMA0_QM_CP_LDMA_SRC_BASE_LO_OFFSET_0 + q_off, 0x14);
+ WREG32(mmDMA0_QM_CP_LDMA_DST_BASE_LO_OFFSET_0 + q_off, 0x1C);
+
+ /* Configure RAZWI IRQ */
+ dma_qm_err_cfg = HBM_DMA_QMAN_GLBL_ERR_CFG_MSG_EN_MASK;
+ if (hdev->stop_on_err) {
+ dma_qm_err_cfg |=
+ HBM_DMA_QMAN_GLBL_ERR_CFG_STOP_ON_ERR_EN_MASK;
+ }
+ WREG32(mmDMA0_QM_GLBL_ERR_CFG + dma_qm_offset, dma_qm_err_cfg);
+
+ WREG32(mmDMA0_QM_GLBL_ERR_ADDR_LO + dma_qm_offset,
+ lower_32_bits(CFG_BASE +
+ mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR));
+ WREG32(mmDMA0_QM_GLBL_ERR_ADDR_HI + dma_qm_offset,
+ upper_32_bits(CFG_BASE +
+ mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR));
+ WREG32(mmDMA0_QM_GLBL_ERR_WDATA + dma_qm_offset,
+ gaudi_irq_map_table[GAUDI_EVENT_DMA0_QM].cpu_id +
+ dma_id);
+
+ WREG32(mmDMA0_QM_ARB_ERR_MSG_EN + dma_qm_offset,
+ QM_ARB_ERR_MSG_EN_MASK);
+
+ /* Increase ARB WDT to support streams architecture */
+ WREG32(mmDMA0_QM_ARB_SLV_CHOISE_WDT + dma_qm_offset,
+ GAUDI_ARB_WDT_TIMEOUT);
+
+ WREG32(mmDMA0_QM_GLBL_CFG1 + dma_qm_offset, 0);
+ WREG32(mmDMA0_QM_GLBL_PROT + dma_qm_offset,
+ QMAN_INTERNAL_MAKE_TRUSTED);
+ }
+
+ WREG32(mmDMA0_QM_CP_MSG_BASE0_ADDR_LO_0 + q_off, mtr_base_lo);
+ WREG32(mmDMA0_QM_CP_MSG_BASE0_ADDR_HI_0 + q_off, mtr_base_hi);
+ WREG32(mmDMA0_QM_CP_MSG_BASE1_ADDR_LO_0 + q_off, so_base_lo);
+ WREG32(mmDMA0_QM_CP_MSG_BASE1_ADDR_HI_0 + q_off, so_base_hi);
+}
+
+static void gaudi_init_hbm_dma_qmans(struct hl_device *hdev)
+{
+ struct gaudi_device *gaudi = hdev->asic_specific;
+ struct gaudi_internal_qman_info *q;
+ u64 qman_base_addr;
+ int i, j, dma_id, internal_q_index;
+
+ if (gaudi->hw_cap_initialized & HW_CAP_HBM_DMA)
+ return;
+
+ for (i = 0 ; i < HBM_DMA_NUMBER_OF_CHNLS ; i++) {
+ dma_id = gaudi_dma_assignment[GAUDI_HBM_DMA_1 + i];
+
+ for (j = 0 ; j < QMAN_STREAMS ; j++) {
+ /*
+ * Add the CPU queue in order to get the correct queue
+ * number as all internal queue are placed after it
+ */
+ internal_q_index = dma_id * QMAN_STREAMS + j + 1;
+
+ q = &gaudi->internal_qmans[internal_q_index];
+ qman_base_addr = (u64) q->pq_dma_addr;
+ gaudi_init_hbm_dma_qman(hdev, dma_id, j,
+ qman_base_addr);
+ }
+
+ /* Initializing lower CP for HBM DMA QMAN */
+ gaudi_init_hbm_dma_qman(hdev, dma_id, 4, 0);
+
+ gaudi_init_dma_core(hdev, dma_id);
+
+ gaudi_enable_qman(hdev, dma_id, HBM_DMA_QMAN_ENABLE);
+ }
+
+ gaudi->hw_cap_initialized |= HW_CAP_HBM_DMA;
+}
+
+static void gaudi_init_mme_qman(struct hl_device *hdev, u32 mme_offset,
+ int qman_id, u64 qman_base_addr)
+{
+ u32 mtr_base_lo, mtr_base_hi;
+ u32 so_base_lo, so_base_hi;
+ u32 q_off, mme_id;
+ u32 mme_qm_err_cfg;
+
+ mtr_base_lo = lower_32_bits(CFG_BASE +
+ mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0);
+ mtr_base_hi = upper_32_bits(CFG_BASE +
+ mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0);
+ so_base_lo = lower_32_bits(CFG_BASE +
+ mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_SOB_OBJ_0);
+ so_base_hi = upper_32_bits(CFG_BASE +
+ mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_SOB_OBJ_0);
+
+ q_off = mme_offset + qman_id * 4;
+
+ if (qman_id < 4) {
+ WREG32(mmMME0_QM_PQ_BASE_LO_0 + q_off,
+ lower_32_bits(qman_base_addr));
+ WREG32(mmMME0_QM_PQ_BASE_HI_0 + q_off,
+ upper_32_bits(qman_base_addr));
+
+ WREG32(mmMME0_QM_PQ_SIZE_0 + q_off, ilog2(MME_QMAN_LENGTH));
+ WREG32(mmMME0_QM_PQ_PI_0 + q_off, 0);
+ WREG32(mmMME0_QM_PQ_CI_0 + q_off, 0);
+
+ WREG32(mmMME0_QM_CP_LDMA_TSIZE_OFFSET_0 + q_off, 0x81BC);
+ WREG32(mmMME0_QM_CP_LDMA_SRC_BASE_LO_OFFSET_0 + q_off, 0x81B4);
+ WREG32(mmMME0_QM_CP_LDMA_DST_BASE_LO_OFFSET_0 + q_off, 0x1C);
+ } else {
+ WREG32(mmMME0_QM_CP_LDMA_TSIZE_OFFSET_0 + q_off, 0x74);
+ WREG32(mmMME0_QM_CP_LDMA_SRC_BASE_LO_OFFSET_0 + q_off, 0x14);
+ WREG32(mmMME0_QM_CP_LDMA_DST_BASE_LO_OFFSET_0 + q_off, 0x1C);
+
+ /* Configure RAZWI IRQ */
+ mme_id = mme_offset /
+ (mmMME1_QM_GLBL_CFG0 - mmMME0_QM_GLBL_CFG0);
+
+ mme_qm_err_cfg = MME_QMAN_GLBL_ERR_CFG_MSG_EN_MASK;
+ if (hdev->stop_on_err) {
+ mme_qm_err_cfg |=
+ MME_QMAN_GLBL_ERR_CFG_STOP_ON_ERR_EN_MASK;
+ }
+ WREG32(mmMME0_QM_GLBL_ERR_CFG + mme_offset, mme_qm_err_cfg);
+ WREG32(mmMME0_QM_GLBL_ERR_ADDR_LO + mme_offset,
+ lower_32_bits(CFG_BASE +
+ mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR));
+ WREG32(mmMME0_QM_GLBL_ERR_ADDR_HI + mme_offset,
+ upper_32_bits(CFG_BASE +
+ mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR));
+ WREG32(mmMME0_QM_GLBL_ERR_WDATA + mme_offset,
+ gaudi_irq_map_table[GAUDI_EVENT_MME0_QM].cpu_id +
+ mme_id);
+
+ WREG32(mmMME0_QM_ARB_ERR_MSG_EN + mme_offset,
+ QM_ARB_ERR_MSG_EN_MASK);
+
+ /* Increase ARB WDT to support streams architecture */
+ WREG32(mmMME0_QM_ARB_SLV_CHOISE_WDT + mme_offset,
+ GAUDI_ARB_WDT_TIMEOUT);
+
+ WREG32(mmMME0_QM_GLBL_CFG1 + mme_offset, 0);
+ WREG32(mmMME0_QM_GLBL_PROT + mme_offset,
+ QMAN_INTERNAL_MAKE_TRUSTED);
+ }
+
+ WREG32(mmMME0_QM_CP_MSG_BASE0_ADDR_LO_0 + q_off, mtr_base_lo);
+ WREG32(mmMME0_QM_CP_MSG_BASE0_ADDR_HI_0 + q_off, mtr_base_hi);
+ WREG32(mmMME0_QM_CP_MSG_BASE1_ADDR_LO_0 + q_off, so_base_lo);
+ WREG32(mmMME0_QM_CP_MSG_BASE1_ADDR_HI_0 + q_off, so_base_hi);
+}
+
+static void gaudi_init_mme_qmans(struct hl_device *hdev)
+{
+ struct gaudi_device *gaudi = hdev->asic_specific;
+ struct gaudi_internal_qman_info *q;
+ u64 qman_base_addr;
+ u32 mme_offset;
+ int i, internal_q_index;
+
+ if (gaudi->hw_cap_initialized & HW_CAP_MME)
+ return;
+
+ /*
+ * map GAUDI_QUEUE_ID_MME_0_X to the N_W_MME (mmMME2_QM_BASE)
+ * and GAUDI_QUEUE_ID_MME_1_X to the S_W_MME (mmMME0_QM_BASE)
+ */
+
+ mme_offset = mmMME2_QM_GLBL_CFG0 - mmMME0_QM_GLBL_CFG0;
+
+ for (i = 0 ; i < MME_NUMBER_OF_QMANS ; i++) {
+ internal_q_index = GAUDI_QUEUE_ID_MME_0_0 + i;
+ q = &gaudi->internal_qmans[internal_q_index];
+ qman_base_addr = (u64) q->pq_dma_addr;
+ gaudi_init_mme_qman(hdev, mme_offset, (i & 0x3),
+ qman_base_addr);
+ if (i == 3)
+ mme_offset = 0;
+ }
+
+ /* Initializing lower CP for MME QMANs */
+ mme_offset = mmMME2_QM_GLBL_CFG0 - mmMME0_QM_GLBL_CFG0;
+ gaudi_init_mme_qman(hdev, mme_offset, 4, 0);
+ gaudi_init_mme_qman(hdev, 0, 4, 0);
+
+ WREG32(mmMME2_QM_GLBL_CFG0, QMAN_MME_ENABLE);
+ WREG32(mmMME0_QM_GLBL_CFG0, QMAN_MME_ENABLE);
+
+ gaudi->hw_cap_initialized |= HW_CAP_MME;
+}
+
+static void gaudi_init_tpc_qman(struct hl_device *hdev, u32 tpc_offset,
+ int qman_id, u64 qman_base_addr)
+{
+ u32 mtr_base_lo, mtr_base_hi;
+ u32 so_base_lo, so_base_hi;
+ u32 q_off, tpc_id;
+ u32 tpc_qm_err_cfg;
+
+ mtr_base_lo = lower_32_bits(CFG_BASE +
+ mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0);
+ mtr_base_hi = upper_32_bits(CFG_BASE +
+ mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0);
+ so_base_lo = lower_32_bits(CFG_BASE +
+ mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_SOB_OBJ_0);
+ so_base_hi = upper_32_bits(CFG_BASE +
+ mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_SOB_OBJ_0);
+
+ q_off = tpc_offset + qman_id * 4;
+
+ if (qman_id < 4) {
+ WREG32(mmTPC0_QM_PQ_BASE_LO_0 + q_off,
+ lower_32_bits(qman_base_addr));
+ WREG32(mmTPC0_QM_PQ_BASE_HI_0 + q_off,
+ upper_32_bits(qman_base_addr));
+
+ WREG32(mmTPC0_QM_PQ_SIZE_0 + q_off, ilog2(TPC_QMAN_LENGTH));
+ WREG32(mmTPC0_QM_PQ_PI_0 + q_off, 0);
+ WREG32(mmTPC0_QM_PQ_CI_0 + q_off, 0);
+
+ WREG32(mmTPC0_QM_CP_LDMA_TSIZE_OFFSET_0 + q_off, 0x81BC);
+ WREG32(mmTPC0_QM_CP_LDMA_SRC_BASE_LO_OFFSET_0 + q_off, 0x81B4);
+ WREG32(mmTPC0_QM_CP_LDMA_DST_BASE_LO_OFFSET_0 + q_off, 0x1C);
+ } else {
+ WREG32(mmTPC0_QM_CP_LDMA_TSIZE_OFFSET_0 + q_off, 0x74);
+ WREG32(mmTPC0_QM_CP_LDMA_SRC_BASE_LO_OFFSET_0 + q_off, 0x14);
+ WREG32(mmTPC0_QM_CP_LDMA_DST_BASE_LO_OFFSET_0 + q_off, 0x1C);
+
+ /* Configure RAZWI IRQ */
+ tpc_id = tpc_offset /
+ (mmTPC1_QM_GLBL_CFG0 - mmTPC0_QM_GLBL_CFG0);
+
+ tpc_qm_err_cfg = TPC_QMAN_GLBL_ERR_CFG_MSG_EN_MASK;
+ if (hdev->stop_on_err) {
+ tpc_qm_err_cfg |=
+ TPC_QMAN_GLBL_ERR_CFG_STOP_ON_ERR_EN_MASK;
+ }
+
+ WREG32(mmTPC0_QM_GLBL_ERR_CFG + tpc_offset, tpc_qm_err_cfg);
+ WREG32(mmTPC0_QM_GLBL_ERR_ADDR_LO + tpc_offset,
+ lower_32_bits(CFG_BASE +
+ mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR));
+ WREG32(mmTPC0_QM_GLBL_ERR_ADDR_HI + tpc_offset,
+ upper_32_bits(CFG_BASE +
+ mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR));
+ WREG32(mmTPC0_QM_GLBL_ERR_WDATA + tpc_offset,
+ gaudi_irq_map_table[GAUDI_EVENT_TPC0_QM].cpu_id +
+ tpc_id);
+
+ WREG32(mmTPC0_QM_ARB_ERR_MSG_EN + tpc_offset,
+ QM_ARB_ERR_MSG_EN_MASK);
+
+ /* Increase ARB WDT to support streams architecture */
+ WREG32(mmTPC0_QM_ARB_SLV_CHOISE_WDT + tpc_offset,
+ GAUDI_ARB_WDT_TIMEOUT);
+
+ WREG32(mmTPC0_QM_GLBL_CFG1 + tpc_offset, 0);
+ WREG32(mmTPC0_QM_GLBL_PROT + tpc_offset,
+ QMAN_INTERNAL_MAKE_TRUSTED);
+ }
+
+ WREG32(mmTPC0_QM_CP_MSG_BASE0_ADDR_LO_0 + q_off, mtr_base_lo);
+ WREG32(mmTPC0_QM_CP_MSG_BASE0_ADDR_HI_0 + q_off, mtr_base_hi);
+ WREG32(mmTPC0_QM_CP_MSG_BASE1_ADDR_LO_0 + q_off, so_base_lo);
+ WREG32(mmTPC0_QM_CP_MSG_BASE1_ADDR_HI_0 + q_off, so_base_hi);
+}
+
+static void gaudi_init_tpc_qmans(struct hl_device *hdev)
+{
+ struct gaudi_device *gaudi = hdev->asic_specific;
+ struct gaudi_internal_qman_info *q;
+ u64 qman_base_addr;
+ u32 so_base_hi, tpc_offset = 0;
+ u32 tpc_delta = mmTPC1_CFG_SM_BASE_ADDRESS_HIGH -
+ mmTPC0_CFG_SM_BASE_ADDRESS_HIGH;
+ int i, tpc_id, internal_q_index;
+
+ if (gaudi->hw_cap_initialized & HW_CAP_TPC_MASK)
+ return;
+
+ so_base_hi = upper_32_bits(CFG_BASE +
+ mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_SOB_OBJ_0);
+
+ for (tpc_id = 0 ; tpc_id < TPC_NUMBER_OF_ENGINES ; tpc_id++) {
+ for (i = 0 ; i < QMAN_STREAMS ; i++) {
+ internal_q_index = GAUDI_QUEUE_ID_TPC_0_0 +
+ tpc_id * QMAN_STREAMS + i;
+ q = &gaudi->internal_qmans[internal_q_index];
+ qman_base_addr = (u64) q->pq_dma_addr;
+ gaudi_init_tpc_qman(hdev, tpc_offset, i,
+ qman_base_addr);
+
+ if (i == 3) {
+ /* Initializing lower CP for TPC QMAN */
+ gaudi_init_tpc_qman(hdev, tpc_offset, 4, 0);
+
+ /* Enable the QMAN and TPC channel */
+ WREG32(mmTPC0_QM_GLBL_CFG0 + tpc_offset,
+ QMAN_TPC_ENABLE);
+ }
+ }
+
+ WREG32(mmTPC0_CFG_SM_BASE_ADDRESS_HIGH + tpc_id * tpc_delta,
+ so_base_hi);
+
+ tpc_offset += mmTPC1_QM_GLBL_CFG0 - mmTPC0_QM_GLBL_CFG0;
+
+ gaudi->hw_cap_initialized |= 1 << (HW_CAP_TPC_SHIFT + tpc_id);
+ }
+}
+
+static void gaudi_disable_pci_dma_qmans(struct hl_device *hdev)
+{
+ struct gaudi_device *gaudi = hdev->asic_specific;
+
+ if (!(gaudi->hw_cap_initialized & HW_CAP_PCI_DMA))
+ return;
+
+ WREG32(mmDMA0_QM_GLBL_CFG0, 0);
+ WREG32(mmDMA1_QM_GLBL_CFG0, 0);
+ WREG32(mmDMA5_QM_GLBL_CFG0, 0);
+}
+
+static void gaudi_disable_hbm_dma_qmans(struct hl_device *hdev)
+{
+ struct gaudi_device *gaudi = hdev->asic_specific;
+
+ if (!(gaudi->hw_cap_initialized & HW_CAP_HBM_DMA))
+ return;
+
+ WREG32(mmDMA2_QM_GLBL_CFG0, 0);
+ WREG32(mmDMA3_QM_GLBL_CFG0, 0);
+ WREG32(mmDMA4_QM_GLBL_CFG0, 0);
+ WREG32(mmDMA6_QM_GLBL_CFG0, 0);
+ WREG32(mmDMA7_QM_GLBL_CFG0, 0);
+}
+
+static void gaudi_disable_mme_qmans(struct hl_device *hdev)
+{
+ struct gaudi_device *gaudi = hdev->asic_specific;
+
+ if (!(gaudi->hw_cap_initialized & HW_CAP_MME))
+ return;
+
+ WREG32(mmMME2_QM_GLBL_CFG0, 0);
+ WREG32(mmMME0_QM_GLBL_CFG0, 0);
+}
+
+static void gaudi_disable_tpc_qmans(struct hl_device *hdev)
+{
+ struct gaudi_device *gaudi = hdev->asic_specific;
+ u32 tpc_offset = 0;
+ int tpc_id;
+
+ if (!(gaudi->hw_cap_initialized & HW_CAP_TPC_MASK))
+ return;
+
+ for (tpc_id = 0 ; tpc_id < TPC_NUMBER_OF_ENGINES ; tpc_id++) {
+ WREG32(mmTPC0_QM_GLBL_CFG0 + tpc_offset, 0);
+ tpc_offset += mmTPC1_QM_GLBL_CFG0 - mmTPC0_QM_GLBL_CFG0;
+ }
+}
+
+static void gaudi_stop_pci_dma_qmans(struct hl_device *hdev)
+{
+ struct gaudi_device *gaudi = hdev->asic_specific;
+
+ if (!(gaudi->hw_cap_initialized & HW_CAP_PCI_DMA))
+ return;
+
+ /* Stop upper CPs of QMANs 0.0 to 1.3 and 5.0 to 5.3 */
+ WREG32(mmDMA0_QM_GLBL_CFG1, 0xF << DMA0_QM_GLBL_CFG1_CP_STOP_SHIFT);
+ WREG32(mmDMA1_QM_GLBL_CFG1, 0xF << DMA0_QM_GLBL_CFG1_CP_STOP_SHIFT);
+ WREG32(mmDMA5_QM_GLBL_CFG1, 0xF << DMA0_QM_GLBL_CFG1_CP_STOP_SHIFT);
+}
+
+static void gaudi_stop_hbm_dma_qmans(struct hl_device *hdev)
+{
+ struct gaudi_device *gaudi = hdev->asic_specific;
+
+ if (!(gaudi->hw_cap_initialized & HW_CAP_HBM_DMA))
+ return;
+
+ /* Stop CPs of HBM DMA QMANs */
+
+ WREG32(mmDMA2_QM_GLBL_CFG1, 0x1F << DMA0_QM_GLBL_CFG1_CP_STOP_SHIFT);
+ WREG32(mmDMA3_QM_GLBL_CFG1, 0x1F << DMA0_QM_GLBL_CFG1_CP_STOP_SHIFT);
+ WREG32(mmDMA4_QM_GLBL_CFG1, 0x1F << DMA0_QM_GLBL_CFG1_CP_STOP_SHIFT);
+ WREG32(mmDMA6_QM_GLBL_CFG1, 0x1F << DMA0_QM_GLBL_CFG1_CP_STOP_SHIFT);
+ WREG32(mmDMA7_QM_GLBL_CFG1, 0x1F << DMA0_QM_GLBL_CFG1_CP_STOP_SHIFT);
+}
+
+static void gaudi_stop_mme_qmans(struct hl_device *hdev)
+{
+ struct gaudi_device *gaudi = hdev->asic_specific;
+
+ if (!(gaudi->hw_cap_initialized & HW_CAP_MME))
+ return;
+
+ /* Stop CPs of MME QMANs */
+ WREG32(mmMME2_QM_GLBL_CFG1, 0x1F << MME0_QM_GLBL_CFG1_CP_STOP_SHIFT);
+ WREG32(mmMME0_QM_GLBL_CFG1, 0x1F << MME0_QM_GLBL_CFG1_CP_STOP_SHIFT);
+}
+
+static void gaudi_stop_tpc_qmans(struct hl_device *hdev)
+{
+ struct gaudi_device *gaudi = hdev->asic_specific;
+
+ if (!(gaudi->hw_cap_initialized & HW_CAP_TPC_MASK))
+ return;
+
+ WREG32(mmTPC0_QM_GLBL_CFG1, 0x1F << TPC0_QM_GLBL_CFG1_CP_STOP_SHIFT);
+ WREG32(mmTPC1_QM_GLBL_CFG1, 0x1F << TPC0_QM_GLBL_CFG1_CP_STOP_SHIFT);
+ WREG32(mmTPC2_QM_GLBL_CFG1, 0x1F << TPC0_QM_GLBL_CFG1_CP_STOP_SHIFT);
+ WREG32(mmTPC3_QM_GLBL_CFG1, 0x1F << TPC0_QM_GLBL_CFG1_CP_STOP_SHIFT);
+ WREG32(mmTPC4_QM_GLBL_CFG1, 0x1F << TPC0_QM_GLBL_CFG1_CP_STOP_SHIFT);
+ WREG32(mmTPC5_QM_GLBL_CFG1, 0x1F << TPC0_QM_GLBL_CFG1_CP_STOP_SHIFT);
+ WREG32(mmTPC6_QM_GLBL_CFG1, 0x1F << TPC0_QM_GLBL_CFG1_CP_STOP_SHIFT);
+ WREG32(mmTPC7_QM_GLBL_CFG1, 0x1F << TPC0_QM_GLBL_CFG1_CP_STOP_SHIFT);
+}
+
+static void gaudi_pci_dma_stall(struct hl_device *hdev)
+{
+ struct gaudi_device *gaudi = hdev->asic_specific;
+
+ if (!(gaudi->hw_cap_initialized & HW_CAP_PCI_DMA))
+ return;
+
+ WREG32(mmDMA0_CORE_CFG_1, 1 << DMA0_CORE_CFG_1_HALT_SHIFT);
+ WREG32(mmDMA1_CORE_CFG_1, 1 << DMA0_CORE_CFG_1_HALT_SHIFT);
+ WREG32(mmDMA5_CORE_CFG_1, 1 << DMA0_CORE_CFG_1_HALT_SHIFT);
+}
+
+static void gaudi_hbm_dma_stall(struct hl_device *hdev)
+{
+ struct gaudi_device *gaudi = hdev->asic_specific;
+
+ if (!(gaudi->hw_cap_initialized & HW_CAP_HBM_DMA))
+ return;
+
+ WREG32(mmDMA2_CORE_CFG_1, 1 << DMA0_CORE_CFG_1_HALT_SHIFT);
+ WREG32(mmDMA3_CORE_CFG_1, 1 << DMA0_CORE_CFG_1_HALT_SHIFT);
+ WREG32(mmDMA4_CORE_CFG_1, 1 << DMA0_CORE_CFG_1_HALT_SHIFT);
+ WREG32(mmDMA6_CORE_CFG_1, 1 << DMA0_CORE_CFG_1_HALT_SHIFT);
+ WREG32(mmDMA7_CORE_CFG_1, 1 << DMA0_CORE_CFG_1_HALT_SHIFT);
+}
+
+static void gaudi_mme_stall(struct hl_device *hdev)
+{
+ struct gaudi_device *gaudi = hdev->asic_specific;
+
+ if (!(gaudi->hw_cap_initialized & HW_CAP_MME))
+ return;
+
+ /* WA for H3-1800 bug: do ACC and SBAB writes twice */
+ WREG32(mmMME0_ACC_ACC_STALL, 1 << MME_ACC_ACC_STALL_R_SHIFT);
+ WREG32(mmMME0_ACC_ACC_STALL, 1 << MME_ACC_ACC_STALL_R_SHIFT);
+ WREG32(mmMME0_SBAB_SB_STALL, 1 << MME_SBAB_SB_STALL_R_SHIFT);
+ WREG32(mmMME0_SBAB_SB_STALL, 1 << MME_SBAB_SB_STALL_R_SHIFT);
+ WREG32(mmMME1_ACC_ACC_STALL, 1 << MME_ACC_ACC_STALL_R_SHIFT);
+ WREG32(mmMME1_ACC_ACC_STALL, 1 << MME_ACC_ACC_STALL_R_SHIFT);
+ WREG32(mmMME1_SBAB_SB_STALL, 1 << MME_SBAB_SB_STALL_R_SHIFT);
+ WREG32(mmMME1_SBAB_SB_STALL, 1 << MME_SBAB_SB_STALL_R_SHIFT);
+ WREG32(mmMME2_ACC_ACC_STALL, 1 << MME_ACC_ACC_STALL_R_SHIFT);
+ WREG32(mmMME2_ACC_ACC_STALL, 1 << MME_ACC_ACC_STALL_R_SHIFT);
+ WREG32(mmMME2_SBAB_SB_STALL, 1 << MME_SBAB_SB_STALL_R_SHIFT);
+ WREG32(mmMME2_SBAB_SB_STALL, 1 << MME_SBAB_SB_STALL_R_SHIFT);
+ WREG32(mmMME3_ACC_ACC_STALL, 1 << MME_ACC_ACC_STALL_R_SHIFT);
+ WREG32(mmMME3_ACC_ACC_STALL, 1 << MME_ACC_ACC_STALL_R_SHIFT);
+ WREG32(mmMME3_SBAB_SB_STALL, 1 << MME_SBAB_SB_STALL_R_SHIFT);
+ WREG32(mmMME3_SBAB_SB_STALL, 1 << MME_SBAB_SB_STALL_R_SHIFT);
+}
+
+static void gaudi_tpc_stall(struct hl_device *hdev)
+{
+ struct gaudi_device *gaudi = hdev->asic_specific;
+
+ if (!(gaudi->hw_cap_initialized & HW_CAP_TPC_MASK))
+ return;
+
+ WREG32(mmTPC0_CFG_TPC_STALL, 1 << TPC0_CFG_TPC_STALL_V_SHIFT);
+ WREG32(mmTPC1_CFG_TPC_STALL, 1 << TPC0_CFG_TPC_STALL_V_SHIFT);
+ WREG32(mmTPC2_CFG_TPC_STALL, 1 << TPC0_CFG_TPC_STALL_V_SHIFT);
+ WREG32(mmTPC3_CFG_TPC_STALL, 1 << TPC0_CFG_TPC_STALL_V_SHIFT);
+ WREG32(mmTPC4_CFG_TPC_STALL, 1 << TPC0_CFG_TPC_STALL_V_SHIFT);
+ WREG32(mmTPC5_CFG_TPC_STALL, 1 << TPC0_CFG_TPC_STALL_V_SHIFT);
+ WREG32(mmTPC6_CFG_TPC_STALL, 1 << TPC0_CFG_TPC_STALL_V_SHIFT);
+ WREG32(mmTPC7_CFG_TPC_STALL, 1 << TPC0_CFG_TPC_STALL_V_SHIFT);
+}
+
+static void gaudi_enable_clock_gating(struct hl_device *hdev)
+{
+ struct gaudi_device *gaudi = hdev->asic_specific;
+ u32 qman_offset;
+ int i;
+
+ if (!hdev->clock_gating)
+ return;
+
+ if (gaudi->hw_cap_initialized & HW_CAP_CLK_GATE)
+ return;
+
+ /* In case we are during debug session, don't enable the clock gate
+ * as it may interfere
+ */
+ if (hdev->in_debug)
+ return;
+
+ for (i = 0, qman_offset = 0 ; i < PCI_DMA_NUMBER_OF_CHNLS ; i++) {
+ qman_offset = gaudi_dma_assignment[i] * DMA_QMAN_OFFSET;
+ WREG32(mmDMA0_QM_CGM_CFG1 + qman_offset, QMAN_CGM1_PWR_GATE_EN);
+ WREG32(mmDMA0_QM_CGM_CFG + qman_offset,
+ QMAN_UPPER_CP_CGM_PWR_GATE_EN);
+ }
+
+ for (; i < HBM_DMA_NUMBER_OF_CHNLS ; i++) {
+ qman_offset = gaudi_dma_assignment[i] * DMA_QMAN_OFFSET;
+ WREG32(mmDMA0_QM_CGM_CFG1 + qman_offset, QMAN_CGM1_PWR_GATE_EN);
+ WREG32(mmDMA0_QM_CGM_CFG + qman_offset,
+ QMAN_COMMON_CP_CGM_PWR_GATE_EN);
+ }
+
+ WREG32(mmMME0_QM_CGM_CFG1, QMAN_CGM1_PWR_GATE_EN);
+ WREG32(mmMME0_QM_CGM_CFG,
+ QMAN_COMMON_CP_CGM_PWR_GATE_EN);
+ WREG32(mmMME2_QM_CGM_CFG1, QMAN_CGM1_PWR_GATE_EN);
+ WREG32(mmMME2_QM_CGM_CFG,
+ QMAN_COMMON_CP_CGM_PWR_GATE_EN);
+
+ for (i = 0, qman_offset = 0 ; i < TPC_NUMBER_OF_ENGINES ; i++) {
+ WREG32(mmTPC0_QM_CGM_CFG1 + qman_offset,
+ QMAN_CGM1_PWR_GATE_EN);
+ WREG32(mmTPC0_QM_CGM_CFG + qman_offset,
+ QMAN_COMMON_CP_CGM_PWR_GATE_EN);
+
+ qman_offset += TPC_QMAN_OFFSET;
+ }
+
+ gaudi->hw_cap_initialized |= HW_CAP_CLK_GATE;
+}
+
+static void gaudi_disable_clock_gating(struct hl_device *hdev)
+{
+ struct gaudi_device *gaudi = hdev->asic_specific;
+ u32 qman_offset;
+ int i;
+
+ if (!(gaudi->hw_cap_initialized & HW_CAP_CLK_GATE))
+ return;
+
+ for (i = 0, qman_offset = 0 ; i < DMA_NUMBER_OF_CHANNELS ; i++) {
+ WREG32(mmDMA0_QM_CGM_CFG + qman_offset, 0);
+ WREG32(mmDMA0_QM_CGM_CFG1 + qman_offset, 0);
+
+ qman_offset += (mmDMA1_QM_CGM_CFG - mmDMA0_QM_CGM_CFG);
+ }
+
+ WREG32(mmMME0_QM_CGM_CFG, 0);
+ WREG32(mmMME0_QM_CGM_CFG1, 0);
+ WREG32(mmMME2_QM_CGM_CFG, 0);
+ WREG32(mmMME2_QM_CGM_CFG1, 0);
+
+ for (i = 0, qman_offset = 0 ; i < TPC_NUMBER_OF_ENGINES ; i++) {
+ WREG32(mmTPC0_QM_CGM_CFG + qman_offset, 0);
+ WREG32(mmTPC0_QM_CGM_CFG1 + qman_offset, 0);
+
+ qman_offset += (mmTPC1_QM_CGM_CFG - mmTPC0_QM_CGM_CFG);
+ }
+
+ gaudi->hw_cap_initialized &= ~(HW_CAP_CLK_GATE);
+}
+
+static void gaudi_enable_timestamp(struct hl_device *hdev)
+{
+ /* Disable the timestamp counter */
+ WREG32(mmPSOC_TIMESTAMP_BASE - CFG_BASE, 0);
+
+ /* Zero the lower/upper parts of the 64-bit counter */
+ WREG32(mmPSOC_TIMESTAMP_BASE - CFG_BASE + 0xC, 0);
+ WREG32(mmPSOC_TIMESTAMP_BASE - CFG_BASE + 0x8, 0);
+
+ /* Enable the counter */
+ WREG32(mmPSOC_TIMESTAMP_BASE - CFG_BASE, 1);
+}
+
+static void gaudi_disable_timestamp(struct hl_device *hdev)
+{
+ /* Disable the timestamp counter */
+ WREG32(mmPSOC_TIMESTAMP_BASE - CFG_BASE, 0);
+}
+
+static void gaudi_halt_engines(struct hl_device *hdev, bool hard_reset)
+{
+ u32 wait_timeout_ms, cpu_timeout_ms;
+
+ dev_info(hdev->dev,
+ "Halting compute engines and disabling interrupts\n");
+
+ if (hdev->pldm) {
+ wait_timeout_ms = GAUDI_PLDM_RESET_WAIT_MSEC;
+ cpu_timeout_ms = GAUDI_PLDM_RESET_WAIT_MSEC;
+ } else {
+ wait_timeout_ms = GAUDI_RESET_WAIT_MSEC;
+ cpu_timeout_ms = GAUDI_CPU_RESET_WAIT_MSEC;
+ }
+
+ if (hard_reset) {
+ /*
+ * I don't know what is the state of the CPU so make sure it is
+ * stopped in any means necessary
+ */
+ WREG32(mmPSOC_GLOBAL_CONF_KMD_MSG_TO_CPU, KMD_MSG_GOTO_WFE);
+ WREG32(mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR,
+ GAUDI_EVENT_HALT_MACHINE);
+ msleep(cpu_timeout_ms);
+ }
+
+ gaudi_stop_mme_qmans(hdev);
+ gaudi_stop_tpc_qmans(hdev);
+ gaudi_stop_hbm_dma_qmans(hdev);
+ gaudi_stop_pci_dma_qmans(hdev);
+
+ gaudi_disable_clock_gating(hdev);
+
+ msleep(wait_timeout_ms);
+
+ gaudi_pci_dma_stall(hdev);
+ gaudi_hbm_dma_stall(hdev);
+ gaudi_tpc_stall(hdev);
+ gaudi_mme_stall(hdev);
+
+ msleep(wait_timeout_ms);
+
+ gaudi_disable_mme_qmans(hdev);
+ gaudi_disable_tpc_qmans(hdev);
+ gaudi_disable_hbm_dma_qmans(hdev);
+ gaudi_disable_pci_dma_qmans(hdev);
+
+ gaudi_disable_timestamp(hdev);
+
+ if (hard_reset)
+ gaudi_disable_msi(hdev);
+ else
+ gaudi_sync_irqs(hdev);
+}
+
+static int gaudi_mmu_init(struct hl_device *hdev)
+{
+ struct asic_fixed_properties *prop = &hdev->asic_prop;
+ struct gaudi_device *gaudi = hdev->asic_specific;
+ u64 hop0_addr;
+ int rc, i;
+
+ if (!hdev->mmu_enable)
+ return 0;
+
+ if (gaudi->hw_cap_initialized & HW_CAP_MMU)
+ return 0;
+
+ hdev->dram_supports_virtual_memory = false;
+
+ for (i = 0 ; i < prop->max_asid ; i++) {
+ hop0_addr = prop->mmu_pgt_addr +
+ (i * prop->mmu_hop_table_size);
+
+ rc = gaudi_mmu_update_asid_hop0_addr(hdev, i, hop0_addr);
+ if (rc) {
+ dev_err(hdev->dev,
+ "failed to set hop0 addr for asid %d\n", i);
+ goto err;
+ }
+ }
+
+ /* init MMU cache manage page */
+ WREG32(mmSTLB_CACHE_INV_BASE_39_8, MMU_CACHE_MNG_ADDR >> 8);
+ WREG32(mmSTLB_CACHE_INV_BASE_49_40, MMU_CACHE_MNG_ADDR >> 40);
+
+ hdev->asic_funcs->mmu_invalidate_cache(hdev, true,
+ VM_TYPE_USERPTR | VM_TYPE_PHYS_PACK);
+
+ WREG32(mmMMU_UP_MMU_ENABLE, 1);
+ WREG32(mmMMU_UP_SPI_MASK, 0xF);
+
+ WREG32(mmSTLB_HOP_CONFIGURATION,
+ hdev->mmu_huge_page_opt ? 0x30440 : 0x40440);
+
+ gaudi->hw_cap_initialized |= HW_CAP_MMU;
+
+ return 0;
+
+err:
+ return rc;
+}
+
+static int gaudi_load_firmware_to_device(struct hl_device *hdev)
+{
+ void __iomem *dst;
+
+ /* HBM scrambler must be initialized before pushing F/W to HBM */
+ gaudi_init_scrambler_hbm(hdev);
+
+ dst = hdev->pcie_bar[HBM_BAR_ID] + LINUX_FW_OFFSET;
+
+ return hl_fw_load_fw_to_device(hdev, GAUDI_LINUX_FW_FILE, dst);
+}
+
+static int gaudi_load_boot_fit_to_device(struct hl_device *hdev)
+{
+ void __iomem *dst;
+
+ dst = hdev->pcie_bar[SRAM_BAR_ID] + BOOT_FIT_SRAM_OFFSET;
+
+ return hl_fw_load_fw_to_device(hdev, GAUDI_BOOT_FIT_FILE, dst);
+}
+
+static void gaudi_read_device_fw_version(struct hl_device *hdev,
+ enum hl_fw_component fwc)
+{
+ const char *name;
+ u32 ver_off;
+ char *dest;
+
+ switch (fwc) {
+ case FW_COMP_UBOOT:
+ ver_off = RREG32(mmUBOOT_VER_OFFSET);
+ dest = hdev->asic_prop.uboot_ver;
+ name = "U-Boot";
+ break;
+ case FW_COMP_PREBOOT:
+ ver_off = RREG32(mmPREBOOT_VER_OFFSET);
+ dest = hdev->asic_prop.preboot_ver;
+ name = "Preboot";
+ break;
+ default:
+ dev_warn(hdev->dev, "Undefined FW component: %d\n", fwc);
+ return;
+ }
+
+ ver_off &= ~((u32)SRAM_BASE_ADDR);
+
+ if (ver_off < SRAM_SIZE - VERSION_MAX_LEN) {
+ memcpy_fromio(dest, hdev->pcie_bar[SRAM_BAR_ID] + ver_off,
+ VERSION_MAX_LEN);
+ } else {
+ dev_err(hdev->dev, "%s version offset (0x%x) is above SRAM\n",
+ name, ver_off);
+ strcpy(dest, "unavailable");
+ }
+}
+
+static int gaudi_init_cpu(struct hl_device *hdev)
+{
+ struct gaudi_device *gaudi = hdev->asic_specific;
+ int rc;
+
+ if (!hdev->cpu_enable)
+ return 0;
+
+ if (gaudi->hw_cap_initialized & HW_CAP_CPU)
+ return 0;
+
+ /*
+ * The device CPU works with 40 bits addresses.
+ * This register sets the extension to 50 bits.
+ */
+ WREG32(mmCPU_IF_CPU_MSB_ADDR, hdev->cpu_pci_msb_addr);
+
+ rc = hl_fw_init_cpu(hdev, mmPSOC_GLOBAL_CONF_CPU_BOOT_STATUS,
+ mmPSOC_GLOBAL_CONF_KMD_MSG_TO_CPU,
+ mmCPU_CMD_STATUS_TO_HOST,
+ mmCPU_BOOT_ERR0,
+ !hdev->bmc_enable, GAUDI_CPU_TIMEOUT_USEC,
+ GAUDI_BOOT_FIT_REQ_TIMEOUT_USEC);
+
+ if (rc)
+ return rc;
+
+ gaudi->hw_cap_initialized |= HW_CAP_CPU;
+
+ return 0;
+}
+
+static int gaudi_init_cpu_queues(struct hl_device *hdev, u32 cpu_timeout)
+{
+ struct gaudi_device *gaudi = hdev->asic_specific;
+ struct hl_eq *eq;
+ u32 status;
+ struct hl_hw_queue *cpu_pq =
+ &hdev->kernel_queues[GAUDI_QUEUE_ID_CPU_PQ];
+ int err;
+
+ if (!hdev->cpu_queues_enable)
+ return 0;
+
+ if (gaudi->hw_cap_initialized & HW_CAP_CPU_Q)
+ return 0;
+
+ eq = &hdev->event_queue;
+
+ WREG32(mmCPU_IF_PQ_BASE_ADDR_LOW, lower_32_bits(cpu_pq->bus_address));
+ WREG32(mmCPU_IF_PQ_BASE_ADDR_HIGH, upper_32_bits(cpu_pq->bus_address));
+
+ WREG32(mmCPU_IF_EQ_BASE_ADDR_LOW, lower_32_bits(eq->bus_address));
+ WREG32(mmCPU_IF_EQ_BASE_ADDR_HIGH, upper_32_bits(eq->bus_address));
+
+ WREG32(mmCPU_IF_CQ_BASE_ADDR_LOW,
+ lower_32_bits(hdev->cpu_accessible_dma_address));
+ WREG32(mmCPU_IF_CQ_BASE_ADDR_HIGH,
+ upper_32_bits(hdev->cpu_accessible_dma_address));
+
+ WREG32(mmCPU_IF_PQ_LENGTH, HL_QUEUE_SIZE_IN_BYTES);
+ WREG32(mmCPU_IF_EQ_LENGTH, HL_EQ_SIZE_IN_BYTES);
+ WREG32(mmCPU_IF_CQ_LENGTH, HL_CPU_ACCESSIBLE_MEM_SIZE);
+
+ /* Used for EQ CI */
+ WREG32(mmCPU_IF_EQ_RD_OFFS, 0);
+
+ WREG32(mmCPU_IF_PF_PQ_PI, 0);
+
+ if (gaudi->multi_msi_mode)
+ WREG32(mmCPU_IF_QUEUE_INIT, PQ_INIT_STATUS_READY_FOR_CP);
+ else
+ WREG32(mmCPU_IF_QUEUE_INIT,
+ PQ_INIT_STATUS_READY_FOR_CP_SINGLE_MSI);
+
+ WREG32(mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR, GAUDI_EVENT_PI_UPDATE);
+
+ err = hl_poll_timeout(
+ hdev,
+ mmCPU_IF_QUEUE_INIT,
+ status,
+ (status == PQ_INIT_STATUS_READY_FOR_HOST),
+ 1000,
+ cpu_timeout);
+
+ if (err) {
+ dev_err(hdev->dev,
+ "Failed to communicate with ARM CPU (ArmCP timeout)\n");
+ return -EIO;
+ }
+
+ gaudi->hw_cap_initialized |= HW_CAP_CPU_Q;
+ return 0;
+}
+
+static void gaudi_pre_hw_init(struct hl_device *hdev)
+{
+ /* Perform read from the device to make sure device is up */
+ RREG32(mmPCIE_DBI_DEVICE_ID_VENDOR_ID_REG);
+
+ /*
+ * Let's mark in the H/W that we have reached this point. We check
+ * this value in the reset_before_init function to understand whether
+ * we need to reset the chip before doing H/W init. This register is
+ * cleared by the H/W upon H/W reset
+ */
+ WREG32(mmHW_STATE, HL_DEVICE_HW_STATE_DIRTY);
+
+ /* Set the access through PCI bars (Linux driver only) as secured */
+ WREG32(mmPCIE_WRAP_LBW_PROT_OVR, (PCIE_WRAP_LBW_PROT_OVR_RD_EN_MASK |
+ PCIE_WRAP_LBW_PROT_OVR_WR_EN_MASK));
+
+ /* Perform read to flush the waiting writes to ensure configuration
+ * was set in the device
+ */
+ RREG32(mmPCIE_WRAP_LBW_PROT_OVR);
+
+ if (hdev->axi_drain) {
+ WREG32(mmPCIE_WRAP_LBW_DRAIN_CFG,
+ 1 << PCIE_WRAP_LBW_DRAIN_CFG_EN_SHIFT);
+ WREG32(mmPCIE_WRAP_HBW_DRAIN_CFG,
+ 1 << PCIE_WRAP_HBW_DRAIN_CFG_EN_SHIFT);
+
+ /* Perform read to flush the DRAIN cfg */
+ RREG32(mmPCIE_WRAP_HBW_DRAIN_CFG);
+ } else {
+ WREG32(mmPCIE_WRAP_LBW_DRAIN_CFG, 0);
+ WREG32(mmPCIE_WRAP_HBW_DRAIN_CFG, 0);
+
+ /* Perform read to flush the DRAIN cfg */
+ RREG32(mmPCIE_WRAP_HBW_DRAIN_CFG);
+ }
+
+ /* Configure the reset registers. Must be done as early as possible
+ * in case we fail during H/W initialization
+ */
+ WREG32(mmPSOC_GLOBAL_CONF_SOFT_RST_CFG_H,
+ (CFG_RST_H_DMA_MASK |
+ CFG_RST_H_MME_MASK |
+ CFG_RST_H_SM_MASK |
+ CFG_RST_H_TPC_MASK));
+
+ WREG32(mmPSOC_GLOBAL_CONF_SOFT_RST_CFG_L, CFG_RST_L_TPC_MASK);
+
+ WREG32(mmPSOC_GLOBAL_CONF_SW_ALL_RST_CFG_H,
+ (CFG_RST_H_HBM_MASK |
+ CFG_RST_H_TPC_MASK |
+ CFG_RST_H_NIC_MASK |
+ CFG_RST_H_SM_MASK |
+ CFG_RST_H_DMA_MASK |
+ CFG_RST_H_MME_MASK |
+ CFG_RST_H_CPU_MASK |
+ CFG_RST_H_MMU_MASK));
+
+ WREG32(mmPSOC_GLOBAL_CONF_SW_ALL_RST_CFG_L,
+ (CFG_RST_L_IF_MASK |
+ CFG_RST_L_PSOC_MASK |
+ CFG_RST_L_TPC_MASK));
+}
+
+static int gaudi_hw_init(struct hl_device *hdev)
+{
+ int rc;
+
+ dev_info(hdev->dev, "Starting initialization of H/W\n");
+
+ gaudi_pre_hw_init(hdev);
+
+ gaudi_init_pci_dma_qmans(hdev);
+
+ gaudi_init_hbm_dma_qmans(hdev);
+
+ /*
+ * Before pushing u-boot/linux to device, need to set the hbm bar to
+ * base address of dram
+ */
+ if (gaudi_set_hbm_bar_base(hdev, DRAM_PHYS_BASE) == U64_MAX) {
+ dev_err(hdev->dev,
+ "failed to map HBM bar to DRAM base address\n");
+ return -EIO;
+ }
+
+ rc = gaudi_init_cpu(hdev);
+ if (rc) {
+ dev_err(hdev->dev, "failed to initialize CPU\n");
+ return rc;
+ }
+
+ /* SRAM scrambler must be initialized after CPU is running from HBM */
+ gaudi_init_scrambler_sram(hdev);
+
+ /* This is here just in case we are working without CPU */
+ gaudi_init_scrambler_hbm(hdev);
+
+ gaudi_init_golden_registers(hdev);
+
+ rc = gaudi_mmu_init(hdev);
+ if (rc)
+ return rc;
+
+ gaudi_init_security(hdev);
+
+ gaudi_init_mme_qmans(hdev);
+
+ gaudi_init_tpc_qmans(hdev);
+
+ gaudi_enable_clock_gating(hdev);
+
+ gaudi_enable_timestamp(hdev);
+
+ /* MSI must be enabled before CPU queues are initialized */
+ rc = gaudi_enable_msi(hdev);
+ if (rc)
+ goto disable_queues;
+
+ /* must be called after MSI was enabled */
+ rc = gaudi_init_cpu_queues(hdev, GAUDI_CPU_TIMEOUT_USEC);
+ if (rc) {
+ dev_err(hdev->dev, "failed to initialize CPU H/W queues %d\n",
+ rc);
+ goto disable_msi;
+ }
+
+ /* Perform read from the device to flush all configuration */
+ RREG32(mmPCIE_DBI_DEVICE_ID_VENDOR_ID_REG);
+
+ return 0;
+
+disable_msi:
+ gaudi_disable_msi(hdev);
+disable_queues:
+ gaudi_disable_mme_qmans(hdev);
+ gaudi_disable_pci_dma_qmans(hdev);
+
+ return rc;
+}
+
+static void gaudi_hw_fini(struct hl_device *hdev, bool hard_reset)
+{
+ struct gaudi_device *gaudi = hdev->asic_specific;
+ u32 status, reset_timeout_ms, boot_strap = 0;
+
+ if (hdev->pldm) {
+ if (hard_reset)
+ reset_timeout_ms = GAUDI_PLDM_HRESET_TIMEOUT_MSEC;
+ else
+ reset_timeout_ms = GAUDI_PLDM_SRESET_TIMEOUT_MSEC;
+ } else {
+ reset_timeout_ms = GAUDI_RESET_TIMEOUT_MSEC;
+ }
+
+ if (hard_reset) {
+ /* Tell ASIC not to re-initialize PCIe */
+ WREG32(mmPREBOOT_PCIE_EN, LKD_HARD_RESET_MAGIC);
+
+ boot_strap = RREG32(mmPSOC_GLOBAL_CONF_BOOT_STRAP_PINS);
+ /* H/W bug WA:
+ * rdata[31:0] = strap_read_val;
+ * wdata[31:0] = rdata[30:21],1'b0,rdata[20:0]
+ */
+ boot_strap = (((boot_strap & 0x7FE00000) << 1) |
+ (boot_strap & 0x001FFFFF));
+ WREG32(mmPSOC_GLOBAL_CONF_BOOT_STRAP_PINS, boot_strap & ~0x2);
+
+ /* Restart BTL/BLR upon hard-reset */
+ WREG32(mmPSOC_GLOBAL_CONF_BOOT_SEQ_RE_START, 1);
+
+ WREG32(mmPSOC_GLOBAL_CONF_SW_ALL_RST,
+ 1 << PSOC_GLOBAL_CONF_SW_ALL_RST_IND_SHIFT);
+ dev_info(hdev->dev,
+ "Issued HARD reset command, going to wait %dms\n",
+ reset_timeout_ms);
+ } else {
+ /* Don't restart BTL/BLR upon soft-reset */
+ WREG32(mmPSOC_GLOBAL_CONF_BOOT_SEQ_RE_START, 0);
+
+ WREG32(mmPSOC_GLOBAL_CONF_SOFT_RST,
+ 1 << PSOC_GLOBAL_CONF_SOFT_RST_IND_SHIFT);
+ dev_info(hdev->dev,
+ "Issued SOFT reset command, going to wait %dms\n",
+ reset_timeout_ms);
+ }
+
+ /*
+ * After hard reset, we can't poll the BTM_FSM register because the PSOC
+ * itself is in reset. Need to wait until the reset is deasserted
+ */
+ msleep(reset_timeout_ms);
+
+ status = RREG32(mmPSOC_GLOBAL_CONF_BTM_FSM);
+ if (status & PSOC_GLOBAL_CONF_BTM_FSM_STATE_MASK)
+ dev_err(hdev->dev,
+ "Timeout while waiting for device to reset 0x%x\n",
+ status);
+
+ if (!hard_reset) {
+ gaudi->hw_cap_initialized &= ~(HW_CAP_PCI_DMA | HW_CAP_MME |
+ HW_CAP_TPC_MASK |
+ HW_CAP_HBM_DMA);
+
+ WREG32(mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR,
+ GAUDI_EVENT_SOFT_RESET);
+ return;
+ }
+
+ /* We continue here only for hard-reset */
+
+ WREG32(mmPSOC_GLOBAL_CONF_BOOT_STRAP_PINS, boot_strap);
+
+ gaudi->hw_cap_initialized &= ~(HW_CAP_CPU | HW_CAP_CPU_Q |
+ HW_CAP_HBM | HW_CAP_PCI_DMA |
+ HW_CAP_MME | HW_CAP_TPC_MASK |
+ HW_CAP_HBM_DMA | HW_CAP_PLL |
+ HW_CAP_MMU |
+ HW_CAP_SRAM_SCRAMBLER |
+ HW_CAP_HBM_SCRAMBLER);
+ memset(gaudi->events_stat, 0, sizeof(gaudi->events_stat));
+}
+
+static int gaudi_suspend(struct hl_device *hdev)
+{
+ int rc;
+
+ rc = hl_fw_send_pci_access_msg(hdev, ARMCP_PACKET_DISABLE_PCI_ACCESS);
+ if (rc)
+ dev_err(hdev->dev, "Failed to disable PCI access from CPU\n");
+
+ return rc;
+}
+
+static int gaudi_resume(struct hl_device *hdev)
+{
+ return gaudi_init_iatu(hdev);
+}
+
+static int gaudi_cb_mmap(struct hl_device *hdev, struct vm_area_struct *vma,
+ u64 kaddress, phys_addr_t paddress, u32 size)
+{
+ int rc;
+
+ vma->vm_flags |= VM_IO | VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP |
+ VM_DONTCOPY | VM_NORESERVE;
+
+ rc = remap_pfn_range(vma, vma->vm_start, paddress >> PAGE_SHIFT,
+ size, vma->vm_page_prot);
+ if (rc)
+ dev_err(hdev->dev, "remap_pfn_range error %d", rc);
+
+ return rc;
+}
+
+static void gaudi_ring_doorbell(struct hl_device *hdev, u32 hw_queue_id, u32 pi)
+{
+ struct gaudi_device *gaudi = hdev->asic_specific;
+ u32 db_reg_offset, db_value, dma_qm_offset, q_off;
+ int dma_id;
+ bool invalid_queue = false;
+
+ switch (hw_queue_id) {
+ case GAUDI_QUEUE_ID_DMA_0_0...GAUDI_QUEUE_ID_DMA_0_3:
+ dma_id = gaudi_dma_assignment[GAUDI_PCI_DMA_1];
+ dma_qm_offset = dma_id * DMA_QMAN_OFFSET;
+ q_off = dma_qm_offset + (hw_queue_id & 0x3) * 4;
+ db_reg_offset = mmDMA0_QM_PQ_PI_0 + q_off;
+ break;
+
+ case GAUDI_QUEUE_ID_DMA_1_0...GAUDI_QUEUE_ID_DMA_1_3:
+ dma_id = gaudi_dma_assignment[GAUDI_PCI_DMA_2];
+ dma_qm_offset = dma_id * DMA_QMAN_OFFSET;
+ q_off = dma_qm_offset + (hw_queue_id & 0x3) * 4;
+ db_reg_offset = mmDMA0_QM_PQ_PI_0 + q_off;
+ break;
+
+ case GAUDI_QUEUE_ID_DMA_2_0...GAUDI_QUEUE_ID_DMA_2_3:
+ dma_id = gaudi_dma_assignment[GAUDI_HBM_DMA_1];
+ dma_qm_offset = dma_id * DMA_QMAN_OFFSET;
+ q_off = dma_qm_offset + ((hw_queue_id - 1) & 0x3) * 4;
+ db_reg_offset = mmDMA0_QM_PQ_PI_0 + q_off;
+ break;
+
+ case GAUDI_QUEUE_ID_DMA_3_0...GAUDI_QUEUE_ID_DMA_3_3:
+ dma_id = gaudi_dma_assignment[GAUDI_HBM_DMA_2];
+ dma_qm_offset = dma_id * DMA_QMAN_OFFSET;
+ q_off = dma_qm_offset + ((hw_queue_id - 1) & 0x3) * 4;
+ db_reg_offset = mmDMA0_QM_PQ_PI_0 + q_off;
+ break;
+
+ case GAUDI_QUEUE_ID_DMA_4_0...GAUDI_QUEUE_ID_DMA_4_3:
+ dma_id = gaudi_dma_assignment[GAUDI_HBM_DMA_3];
+ dma_qm_offset = dma_id * DMA_QMAN_OFFSET;
+ q_off = dma_qm_offset + ((hw_queue_id - 1) & 0x3) * 4;
+ db_reg_offset = mmDMA0_QM_PQ_PI_0 + q_off;
+ break;
+
+ case GAUDI_QUEUE_ID_DMA_5_0...GAUDI_QUEUE_ID_DMA_5_3:
+ dma_id = gaudi_dma_assignment[GAUDI_PCI_DMA_3];
+ dma_qm_offset = dma_id * DMA_QMAN_OFFSET;
+ q_off = dma_qm_offset + ((hw_queue_id - 1) & 0x3) * 4;
+ db_reg_offset = mmDMA0_QM_PQ_PI_0 + q_off;
+ break;
+
+ case GAUDI_QUEUE_ID_DMA_6_0...GAUDI_QUEUE_ID_DMA_6_3:
+ dma_id = gaudi_dma_assignment[GAUDI_HBM_DMA_4];
+ dma_qm_offset = dma_id * DMA_QMAN_OFFSET;
+ q_off = dma_qm_offset + ((hw_queue_id - 1) & 0x3) * 4;
+ db_reg_offset = mmDMA0_QM_PQ_PI_0 + q_off;
+ break;
+
+ case GAUDI_QUEUE_ID_DMA_7_0...GAUDI_QUEUE_ID_DMA_7_3:
+ dma_id = gaudi_dma_assignment[GAUDI_HBM_DMA_5];
+ dma_qm_offset = dma_id * DMA_QMAN_OFFSET;
+ q_off = dma_qm_offset + ((hw_queue_id - 1) & 0x3) * 4;
+ db_reg_offset = mmDMA0_QM_PQ_PI_0 + q_off;
+ break;
+
+ case GAUDI_QUEUE_ID_CPU_PQ:
+ if (gaudi->hw_cap_initialized & HW_CAP_CPU_Q)
+ db_reg_offset = mmCPU_IF_PF_PQ_PI;
+ else
+ invalid_queue = true;
+ break;
+
+ case GAUDI_QUEUE_ID_MME_0_0:
+ db_reg_offset = mmMME2_QM_PQ_PI_0;
+ break;
+
+ case GAUDI_QUEUE_ID_MME_0_1:
+ db_reg_offset = mmMME2_QM_PQ_PI_1;
+ break;
+
+ case GAUDI_QUEUE_ID_MME_0_2:
+ db_reg_offset = mmMME2_QM_PQ_PI_2;
+ break;
+
+ case GAUDI_QUEUE_ID_MME_0_3:
+ db_reg_offset = mmMME2_QM_PQ_PI_3;
+ break;
+
+ case GAUDI_QUEUE_ID_MME_1_0:
+ db_reg_offset = mmMME0_QM_PQ_PI_0;
+ break;
+
+ case GAUDI_QUEUE_ID_MME_1_1:
+ db_reg_offset = mmMME0_QM_PQ_PI_1;
+ break;
+
+ case GAUDI_QUEUE_ID_MME_1_2:
+ db_reg_offset = mmMME0_QM_PQ_PI_2;
+ break;
+
+ case GAUDI_QUEUE_ID_MME_1_3:
+ db_reg_offset = mmMME0_QM_PQ_PI_3;
+ break;
+
+ case GAUDI_QUEUE_ID_TPC_0_0:
+ db_reg_offset = mmTPC0_QM_PQ_PI_0;
+ break;
+
+ case GAUDI_QUEUE_ID_TPC_0_1:
+ db_reg_offset = mmTPC0_QM_PQ_PI_1;
+ break;
+
+ case GAUDI_QUEUE_ID_TPC_0_2:
+ db_reg_offset = mmTPC0_QM_PQ_PI_2;
+ break;
+
+ case GAUDI_QUEUE_ID_TPC_0_3:
+ db_reg_offset = mmTPC0_QM_PQ_PI_3;
+ break;
+
+ case GAUDI_QUEUE_ID_TPC_1_0:
+ db_reg_offset = mmTPC1_QM_PQ_PI_0;
+ break;
+
+ case GAUDI_QUEUE_ID_TPC_1_1:
+ db_reg_offset = mmTPC1_QM_PQ_PI_1;
+ break;
+
+ case GAUDI_QUEUE_ID_TPC_1_2:
+ db_reg_offset = mmTPC1_QM_PQ_PI_2;
+ break;
+
+ case GAUDI_QUEUE_ID_TPC_1_3:
+ db_reg_offset = mmTPC1_QM_PQ_PI_3;
+ break;
+
+ case GAUDI_QUEUE_ID_TPC_2_0:
+ db_reg_offset = mmTPC2_QM_PQ_PI_0;
+ break;
+
+ case GAUDI_QUEUE_ID_TPC_2_1:
+ db_reg_offset = mmTPC2_QM_PQ_PI_1;
+ break;
+
+ case GAUDI_QUEUE_ID_TPC_2_2:
+ db_reg_offset = mmTPC2_QM_PQ_PI_2;
+ break;
+
+ case GAUDI_QUEUE_ID_TPC_2_3:
+ db_reg_offset = mmTPC2_QM_PQ_PI_3;
+ break;
+
+ case GAUDI_QUEUE_ID_TPC_3_0:
+ db_reg_offset = mmTPC3_QM_PQ_PI_0;
+ break;
+
+ case GAUDI_QUEUE_ID_TPC_3_1:
+ db_reg_offset = mmTPC3_QM_PQ_PI_1;
+ break;
+
+ case GAUDI_QUEUE_ID_TPC_3_2:
+ db_reg_offset = mmTPC3_QM_PQ_PI_2;
+ break;
+
+ case GAUDI_QUEUE_ID_TPC_3_3:
+ db_reg_offset = mmTPC3_QM_PQ_PI_3;
+ break;
+
+ case GAUDI_QUEUE_ID_TPC_4_0:
+ db_reg_offset = mmTPC4_QM_PQ_PI_0;
+ break;
+
+ case GAUDI_QUEUE_ID_TPC_4_1:
+ db_reg_offset = mmTPC4_QM_PQ_PI_1;
+ break;
+
+ case GAUDI_QUEUE_ID_TPC_4_2:
+ db_reg_offset = mmTPC4_QM_PQ_PI_2;
+ break;
+
+ case GAUDI_QUEUE_ID_TPC_4_3:
+ db_reg_offset = mmTPC4_QM_PQ_PI_3;
+ break;
+
+ case GAUDI_QUEUE_ID_TPC_5_0:
+ db_reg_offset = mmTPC5_QM_PQ_PI_0;
+ break;
+
+ case GAUDI_QUEUE_ID_TPC_5_1:
+ db_reg_offset = mmTPC5_QM_PQ_PI_1;
+ break;
+
+ case GAUDI_QUEUE_ID_TPC_5_2:
+ db_reg_offset = mmTPC5_QM_PQ_PI_2;
+ break;
+
+ case GAUDI_QUEUE_ID_TPC_5_3:
+ db_reg_offset = mmTPC5_QM_PQ_PI_3;
+ break;
+
+ case GAUDI_QUEUE_ID_TPC_6_0:
+ db_reg_offset = mmTPC6_QM_PQ_PI_0;
+ break;
+
+ case GAUDI_QUEUE_ID_TPC_6_1:
+ db_reg_offset = mmTPC6_QM_PQ_PI_1;
+ break;
+
+ case GAUDI_QUEUE_ID_TPC_6_2:
+ db_reg_offset = mmTPC6_QM_PQ_PI_2;
+ break;
+
+ case GAUDI_QUEUE_ID_TPC_6_3:
+ db_reg_offset = mmTPC6_QM_PQ_PI_3;
+ break;
+
+ case GAUDI_QUEUE_ID_TPC_7_0:
+ db_reg_offset = mmTPC7_QM_PQ_PI_0;
+ break;
+
+ case GAUDI_QUEUE_ID_TPC_7_1:
+ db_reg_offset = mmTPC7_QM_PQ_PI_1;
+ break;
+
+ case GAUDI_QUEUE_ID_TPC_7_2:
+ db_reg_offset = mmTPC7_QM_PQ_PI_2;
+ break;
+
+ case GAUDI_QUEUE_ID_TPC_7_3:
+ db_reg_offset = mmTPC7_QM_PQ_PI_3;
+ break;
+
+ default:
+ invalid_queue = true;
+ }
+
+ if (invalid_queue) {
+ /* Should never get here */
+ dev_err(hdev->dev, "h/w queue %d is invalid. Can't set pi\n",
+ hw_queue_id);
+ return;
+ }
+
+ db_value = pi;
+
+ /* ring the doorbell */
+ WREG32(db_reg_offset, db_value);
+
+ if (hw_queue_id == GAUDI_QUEUE_ID_CPU_PQ)
+ WREG32(mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR,
+ GAUDI_EVENT_PI_UPDATE);
+}
+
+static void gaudi_pqe_write(struct hl_device *hdev, __le64 *pqe,
+ struct hl_bd *bd)
+{
+ __le64 *pbd = (__le64 *) bd;
+
+ /* The QMANs are on the host memory so a simple copy suffice */
+ pqe[0] = pbd[0];
+ pqe[1] = pbd[1];
+}
+
+static void *gaudi_dma_alloc_coherent(struct hl_device *hdev, size_t size,
+ dma_addr_t *dma_handle, gfp_t flags)
+{
+ void *kernel_addr = dma_alloc_coherent(&hdev->pdev->dev, size,
+ dma_handle, flags);
+
+ /* Shift to the device's base physical address of host memory */
+ if (kernel_addr)
+ *dma_handle += HOST_PHYS_BASE;
+
+ return kernel_addr;
+}
+
+static void gaudi_dma_free_coherent(struct hl_device *hdev, size_t size,
+ void *cpu_addr, dma_addr_t dma_handle)
+{
+ /* Cancel the device's base physical address of host memory */
+ dma_addr_t fixed_dma_handle = dma_handle - HOST_PHYS_BASE;
+
+ dma_free_coherent(&hdev->pdev->dev, size, cpu_addr, fixed_dma_handle);
+}
+
+static void *gaudi_get_int_queue_base(struct hl_device *hdev,
+ u32 queue_id, dma_addr_t *dma_handle,
+ u16 *queue_len)
+{
+ struct gaudi_device *gaudi = hdev->asic_specific;
+ struct gaudi_internal_qman_info *q;
+
+ if (queue_id >= GAUDI_QUEUE_ID_SIZE ||
+ gaudi_queue_type[queue_id] != QUEUE_TYPE_INT) {
+ dev_err(hdev->dev, "Got invalid queue id %d\n", queue_id);
+ return NULL;
+ }
+
+ q = &gaudi->internal_qmans[queue_id];
+ *dma_handle = q->pq_dma_addr;
+ *queue_len = q->pq_size / QMAN_PQ_ENTRY_SIZE;
+
+ return q->pq_kernel_addr;
+}
+
+static int gaudi_send_cpu_message(struct hl_device *hdev, u32 *msg,
+ u16 len, u32 timeout, long *result)
+{
+ struct gaudi_device *gaudi = hdev->asic_specific;
+
+ if (!(gaudi->hw_cap_initialized & HW_CAP_CPU_Q)) {
+ if (result)
+ *result = 0;
+ return 0;
+ }
+
+ return hl_fw_send_cpu_message(hdev, GAUDI_QUEUE_ID_CPU_PQ, msg, len,
+ timeout, result);
+}
+
+static int gaudi_test_queue(struct hl_device *hdev, u32 hw_queue_id)
+{
+ struct packet_msg_prot *fence_pkt;
+ dma_addr_t pkt_dma_addr;
+ u32 fence_val, tmp, timeout_usec;
+ dma_addr_t fence_dma_addr;
+ u32 *fence_ptr;
+ int rc;
+
+ if (hdev->pldm)
+ timeout_usec = GAUDI_PLDM_TEST_QUEUE_WAIT_USEC;
+ else
+ timeout_usec = GAUDI_TEST_QUEUE_WAIT_USEC;
+
+ fence_val = GAUDI_QMAN0_FENCE_VAL;
+
+ fence_ptr = hdev->asic_funcs->asic_dma_pool_zalloc(hdev, 4, GFP_KERNEL,
+ &fence_dma_addr);
+ if (!fence_ptr) {
+ dev_err(hdev->dev,
+ "Failed to allocate memory for queue testing\n");
+ return -ENOMEM;
+ }
+
+ *fence_ptr = 0;
+
+ fence_pkt = hdev->asic_funcs->asic_dma_pool_zalloc(hdev,
+ sizeof(struct packet_msg_prot),
+ GFP_KERNEL, &pkt_dma_addr);
+ if (!fence_pkt) {
+ dev_err(hdev->dev,
+ "Failed to allocate packet for queue testing\n");
+ rc = -ENOMEM;
+ goto free_fence_ptr;
+ }
+
+ tmp = (PACKET_MSG_PROT << GAUDI_PKT_CTL_OPCODE_SHIFT) |
+ (1 << GAUDI_PKT_CTL_EB_SHIFT) |
+ (1 << GAUDI_PKT_CTL_MB_SHIFT);
+ fence_pkt->ctl = cpu_to_le32(tmp);
+ fence_pkt->value = cpu_to_le32(fence_val);
+ fence_pkt->addr = cpu_to_le64(fence_dma_addr);
+
+ rc = hl_hw_queue_send_cb_no_cmpl(hdev, hw_queue_id,
+ sizeof(struct packet_msg_prot),
+ pkt_dma_addr);
+ if (rc) {
+ dev_err(hdev->dev,
+ "Failed to send fence packet\n");
+ goto free_pkt;
+ }
+
+ rc = hl_poll_timeout_memory(hdev, fence_ptr, tmp, (tmp == fence_val),
+ 1000, timeout_usec, true);
+
+ hl_hw_queue_inc_ci_kernel(hdev, hw_queue_id);
+
+ if (rc == -ETIMEDOUT) {
+ dev_err(hdev->dev,
+ "H/W queue %d test failed (scratch(0x%08llX) == 0x%08X)\n",
+ hw_queue_id, (unsigned long long) fence_dma_addr, tmp);
+ rc = -EIO;
+ }
+
+free_pkt:
+ hdev->asic_funcs->asic_dma_pool_free(hdev, (void *) fence_pkt,
+ pkt_dma_addr);
+free_fence_ptr:
+ hdev->asic_funcs->asic_dma_pool_free(hdev, (void *) fence_ptr,
+ fence_dma_addr);
+ return rc;
+}
+
+static int gaudi_test_cpu_queue(struct hl_device *hdev)
+{
+ struct gaudi_device *gaudi = hdev->asic_specific;
+
+ /*
+ * check capability here as send_cpu_message() won't update the result
+ * value if no capability
+ */
+ if (!(gaudi->hw_cap_initialized & HW_CAP_CPU_Q))
+ return 0;
+
+ return hl_fw_test_cpu_queue(hdev);
+}
+
+static int gaudi_test_queues(struct hl_device *hdev)
+{
+ int i, rc, ret_val = 0;
+
+ for (i = 0 ; i < HL_MAX_QUEUES ; i++) {
+ if (hdev->asic_prop.hw_queues_props[i].type == QUEUE_TYPE_EXT) {
+ rc = gaudi_test_queue(hdev, i);
+ if (rc)
+ ret_val = -EINVAL;
+ }
+ }
+
+ rc = gaudi_test_cpu_queue(hdev);
+ if (rc)
+ ret_val = -EINVAL;
+
+ return ret_val;
+}
+
+static void *gaudi_dma_pool_zalloc(struct hl_device *hdev, size_t size,
+ gfp_t mem_flags, dma_addr_t *dma_handle)
+{
+ void *kernel_addr;
+
+ if (size > GAUDI_DMA_POOL_BLK_SIZE)
+ return NULL;
+
+ kernel_addr = dma_pool_zalloc(hdev->dma_pool, mem_flags, dma_handle);
+
+ /* Shift to the device's base physical address of host memory */
+ if (kernel_addr)
+ *dma_handle += HOST_PHYS_BASE;
+
+ return kernel_addr;
+}
+
+static void gaudi_dma_pool_free(struct hl_device *hdev, void *vaddr,
+ dma_addr_t dma_addr)
+{
+ /* Cancel the device's base physical address of host memory */
+ dma_addr_t fixed_dma_addr = dma_addr - HOST_PHYS_BASE;
+
+ dma_pool_free(hdev->dma_pool, vaddr, fixed_dma_addr);
+}
+
+static void *gaudi_cpu_accessible_dma_pool_alloc(struct hl_device *hdev,
+ size_t size, dma_addr_t *dma_handle)
+{
+ return hl_fw_cpu_accessible_dma_pool_alloc(hdev, size, dma_handle);
+}
+
+static void gaudi_cpu_accessible_dma_pool_free(struct hl_device *hdev,
+ size_t size, void *vaddr)
+{
+ hl_fw_cpu_accessible_dma_pool_free(hdev, size, vaddr);
+}
+
+static int gaudi_dma_map_sg(struct hl_device *hdev, struct scatterlist *sgl,
+ int nents, enum dma_data_direction dir)
+{
+ struct scatterlist *sg;
+ int i;
+
+ if (!dma_map_sg(&hdev->pdev->dev, sgl, nents, dir))
+ return -ENOMEM;
+
+ /* Shift to the device's base physical address of host memory */
+ for_each_sg(sgl, sg, nents, i)
+ sg->dma_address += HOST_PHYS_BASE;
+
+ return 0;
+}
+
+static void gaudi_dma_unmap_sg(struct hl_device *hdev, struct scatterlist *sgl,
+ int nents, enum dma_data_direction dir)
+{
+ struct scatterlist *sg;
+ int i;
+
+ /* Cancel the device's base physical address of host memory */
+ for_each_sg(sgl, sg, nents, i)
+ sg->dma_address -= HOST_PHYS_BASE;
+
+ dma_unmap_sg(&hdev->pdev->dev, sgl, nents, dir);
+}
+
+static u32 gaudi_get_dma_desc_list_size(struct hl_device *hdev,
+ struct sg_table *sgt)
+{
+ struct scatterlist *sg, *sg_next_iter;
+ u32 count, dma_desc_cnt;
+ u64 len, len_next;
+ dma_addr_t addr, addr_next;
+
+ dma_desc_cnt = 0;
+
+ for_each_sg(sgt->sgl, sg, sgt->nents, count) {
+
+ len = sg_dma_len(sg);
+ addr = sg_dma_address(sg);
+
+ if (len == 0)
+ break;
+
+ while ((count + 1) < sgt->nents) {
+ sg_next_iter = sg_next(sg);
+ len_next = sg_dma_len(sg_next_iter);
+ addr_next = sg_dma_address(sg_next_iter);
+
+ if (len_next == 0)
+ break;
+
+ if ((addr + len == addr_next) &&
+ (len + len_next <= DMA_MAX_TRANSFER_SIZE)) {
+ len += len_next;
+ count++;
+ sg = sg_next_iter;
+ } else {
+ break;
+ }
+ }
+
+ dma_desc_cnt++;
+ }
+
+ return dma_desc_cnt * sizeof(struct packet_lin_dma);
+}
+
+static int gaudi_pin_memory_before_cs(struct hl_device *hdev,
+ struct hl_cs_parser *parser,
+ struct packet_lin_dma *user_dma_pkt,
+ u64 addr, enum dma_data_direction dir)
+{
+ struct hl_userptr *userptr;
+ int rc;
+
+ if (hl_userptr_is_pinned(hdev, addr, le32_to_cpu(user_dma_pkt->tsize),
+ parser->job_userptr_list, &userptr))
+ goto already_pinned;
+
+ userptr = kzalloc(sizeof(*userptr), GFP_ATOMIC);
+ if (!userptr)
+ return -ENOMEM;
+
+ rc = hl_pin_host_memory(hdev, addr, le32_to_cpu(user_dma_pkt->tsize),
+ userptr);
+ if (rc)
+ goto free_userptr;
+
+ list_add_tail(&userptr->job_node, parser->job_userptr_list);
+
+ rc = hdev->asic_funcs->asic_dma_map_sg(hdev, userptr->sgt->sgl,
+ userptr->sgt->nents, dir);
+ if (rc) {
+ dev_err(hdev->dev, "failed to map sgt with DMA region\n");
+ goto unpin_memory;
+ }
+
+ userptr->dma_mapped = true;
+ userptr->dir = dir;
+
+already_pinned:
+ parser->patched_cb_size +=
+ gaudi_get_dma_desc_list_size(hdev, userptr->sgt);
+
+ return 0;
+
+unpin_memory:
+ hl_unpin_host_memory(hdev, userptr);
+free_userptr:
+ kfree(userptr);
+ return rc;
+}
+
+static int gaudi_validate_dma_pkt_host(struct hl_device *hdev,
+ struct hl_cs_parser *parser,
+ struct packet_lin_dma *user_dma_pkt,
+ bool src_in_host)
+{
+ enum dma_data_direction dir;
+ bool skip_host_mem_pin = false, user_memset;
+ u64 addr;
+ int rc = 0;
+
+ user_memset = (le32_to_cpu(user_dma_pkt->ctl) &
+ GAUDI_PKT_LIN_DMA_CTL_MEMSET_MASK) >>
+ GAUDI_PKT_LIN_DMA_CTL_MEMSET_SHIFT;
+
+ if (src_in_host) {
+ if (user_memset)
+ skip_host_mem_pin = true;
+
+ dev_dbg(hdev->dev, "DMA direction is HOST --> DEVICE\n");
+ dir = DMA_TO_DEVICE;
+ addr = le64_to_cpu(user_dma_pkt->src_addr);
+ } else {
+ dev_dbg(hdev->dev, "DMA direction is DEVICE --> HOST\n");
+ dir = DMA_FROM_DEVICE;
+ addr = (le64_to_cpu(user_dma_pkt->dst_addr) &
+ GAUDI_PKT_LIN_DMA_DST_ADDR_MASK) >>
+ GAUDI_PKT_LIN_DMA_DST_ADDR_SHIFT;
+ }
+
+ if (skip_host_mem_pin)
+ parser->patched_cb_size += sizeof(*user_dma_pkt);
+ else
+ rc = gaudi_pin_memory_before_cs(hdev, parser, user_dma_pkt,
+ addr, dir);
+
+ return rc;
+}
+
+static int gaudi_validate_dma_pkt_no_mmu(struct hl_device *hdev,
+ struct hl_cs_parser *parser,
+ struct packet_lin_dma *user_dma_pkt)
+{
+ bool src_in_host = false;
+ u64 dst_addr = (le64_to_cpu(user_dma_pkt->dst_addr) &
+ GAUDI_PKT_LIN_DMA_DST_ADDR_MASK) >>
+ GAUDI_PKT_LIN_DMA_DST_ADDR_SHIFT;
+
+ dev_dbg(hdev->dev, "DMA packet details:\n");
+ dev_dbg(hdev->dev, "source == 0x%llx\n",
+ le64_to_cpu(user_dma_pkt->src_addr));
+ dev_dbg(hdev->dev, "destination == 0x%llx\n", dst_addr);
+ dev_dbg(hdev->dev, "size == %u\n", le32_to_cpu(user_dma_pkt->tsize));
+
+ /*
+ * Special handling for DMA with size 0. Bypass all validations
+ * because no transactions will be done except for WR_COMP, which
+ * is not a security issue
+ */
+ if (!le32_to_cpu(user_dma_pkt->tsize)) {
+ parser->patched_cb_size += sizeof(*user_dma_pkt);
+ return 0;
+ }
+
+ if (parser->hw_queue_id <= GAUDI_QUEUE_ID_DMA_0_3)
+ src_in_host = true;
+
+ return gaudi_validate_dma_pkt_host(hdev, parser, user_dma_pkt,
+ src_in_host);
+}
+
+static int gaudi_validate_cb(struct hl_device *hdev,
+ struct hl_cs_parser *parser, bool is_mmu)
+{
+ u32 cb_parsed_length = 0;
+ int rc = 0;
+
+ parser->patched_cb_size = 0;
+
+ /* cb_user_size is more than 0 so loop will always be executed */
+ while (cb_parsed_length < parser->user_cb_size) {
+ enum packet_id pkt_id;
+ u16 pkt_size;
+ struct gaudi_packet *user_pkt;
+
+ user_pkt = (struct gaudi_packet *) (uintptr_t)
+ (parser->user_cb->kernel_address + cb_parsed_length);
+
+ pkt_id = (enum packet_id) (
+ (le64_to_cpu(user_pkt->header) &
+ PACKET_HEADER_PACKET_ID_MASK) >>
+ PACKET_HEADER_PACKET_ID_SHIFT);
+
+ pkt_size = gaudi_packet_sizes[pkt_id];
+ cb_parsed_length += pkt_size;
+ if (cb_parsed_length > parser->user_cb_size) {
+ dev_err(hdev->dev,
+ "packet 0x%x is out of CB boundary\n", pkt_id);
+ rc = -EINVAL;
+ break;
+ }
+
+ switch (pkt_id) {
+ case PACKET_MSG_PROT:
+ dev_err(hdev->dev,
+ "User not allowed to use MSG_PROT\n");
+ rc = -EPERM;
+ break;
+
+ case PACKET_CP_DMA:
+ dev_err(hdev->dev, "User not allowed to use CP_DMA\n");
+ rc = -EPERM;
+ break;
+
+ case PACKET_STOP:
+ dev_err(hdev->dev, "User not allowed to use STOP\n");
+ rc = -EPERM;
+ break;
+
+ case PACKET_LIN_DMA:
+ parser->contains_dma_pkt = true;
+ if (is_mmu)
+ parser->patched_cb_size += pkt_size;
+ else
+ rc = gaudi_validate_dma_pkt_no_mmu(hdev, parser,
+ (struct packet_lin_dma *) user_pkt);
+ break;
+
+ case PACKET_WREG_32:
+ case PACKET_WREG_BULK:
+ case PACKET_MSG_LONG:
+ case PACKET_MSG_SHORT:
+ case PACKET_REPEAT:
+ case PACKET_FENCE:
+ case PACKET_NOP:
+ case PACKET_ARB_POINT:
+ case PACKET_LOAD_AND_EXE:
+ parser->patched_cb_size += pkt_size;
+ break;
+
+ default:
+ dev_err(hdev->dev, "Invalid packet header 0x%x\n",
+ pkt_id);
+ rc = -EINVAL;
+ break;
+ }
+
+ if (rc)
+ break;
+ }
+
+ /*
+ * The new CB should have space at the end for two MSG_PROT packets:
+ * 1. A packet that will act as a completion packet
+ * 2. A packet that will generate MSI-X interrupt
+ */
+ parser->patched_cb_size += sizeof(struct packet_msg_prot) * 2;
+
+ return rc;
+}
+
+static int gaudi_patch_dma_packet(struct hl_device *hdev,
+ struct hl_cs_parser *parser,
+ struct packet_lin_dma *user_dma_pkt,
+ struct packet_lin_dma *new_dma_pkt,
+ u32 *new_dma_pkt_size)
+{
+ struct hl_userptr *userptr;
+ struct scatterlist *sg, *sg_next_iter;
+ u32 count, dma_desc_cnt, user_wrcomp_en_mask, ctl;
+ u64 len, len_next;
+ dma_addr_t dma_addr, dma_addr_next;
+ u64 device_memory_addr, addr;
+ enum dma_data_direction dir;
+ struct sg_table *sgt;
+ bool src_in_host = false;
+ bool skip_host_mem_pin = false;
+ bool user_memset;
+
+ ctl = le32_to_cpu(user_dma_pkt->ctl);
+
+ if (parser->hw_queue_id <= GAUDI_QUEUE_ID_DMA_0_3)
+ src_in_host = true;
+
+ user_memset = (ctl & GAUDI_PKT_LIN_DMA_CTL_MEMSET_MASK) >>
+ GAUDI_PKT_LIN_DMA_CTL_MEMSET_SHIFT;
+
+ if (src_in_host) {
+ addr = le64_to_cpu(user_dma_pkt->src_addr);
+ device_memory_addr = le64_to_cpu(user_dma_pkt->dst_addr);
+ dir = DMA_TO_DEVICE;
+ if (user_memset)
+ skip_host_mem_pin = true;
+ } else {
+ addr = le64_to_cpu(user_dma_pkt->dst_addr);
+ device_memory_addr = le64_to_cpu(user_dma_pkt->src_addr);
+ dir = DMA_FROM_DEVICE;
+ }
+
+ if ((!skip_host_mem_pin) &&
+ (!hl_userptr_is_pinned(hdev, addr,
+ le32_to_cpu(user_dma_pkt->tsize),
+ parser->job_userptr_list, &userptr))) {
+ dev_err(hdev->dev, "Userptr 0x%llx + 0x%x NOT mapped\n",
+ addr, user_dma_pkt->tsize);
+ return -EFAULT;
+ }
+
+ if ((user_memset) && (dir == DMA_TO_DEVICE)) {
+ memcpy(new_dma_pkt, user_dma_pkt, sizeof(*user_dma_pkt));
+ *new_dma_pkt_size = sizeof(*user_dma_pkt);
+ return 0;
+ }
+
+ user_wrcomp_en_mask = ctl & GAUDI_PKT_LIN_DMA_CTL_WRCOMP_EN_MASK;
+
+ sgt = userptr->sgt;
+ dma_desc_cnt = 0;
+
+ for_each_sg(sgt->sgl, sg, sgt->nents, count) {
+ len = sg_dma_len(sg);
+ dma_addr = sg_dma_address(sg);
+
+ if (len == 0)
+ break;
+
+ while ((count + 1) < sgt->nents) {
+ sg_next_iter = sg_next(sg);
+ len_next = sg_dma_len(sg_next_iter);
+ dma_addr_next = sg_dma_address(sg_next_iter);
+
+ if (len_next == 0)
+ break;
+
+ if ((dma_addr + len == dma_addr_next) &&
+ (len + len_next <= DMA_MAX_TRANSFER_SIZE)) {
+ len += len_next;
+ count++;
+ sg = sg_next_iter;
+ } else {
+ break;
+ }
+ }
+
+ new_dma_pkt->ctl = user_dma_pkt->ctl;
+
+ ctl = le32_to_cpu(user_dma_pkt->ctl);
+ if (likely(dma_desc_cnt))
+ ctl &= ~GAUDI_PKT_CTL_EB_MASK;
+ ctl &= ~GAUDI_PKT_LIN_DMA_CTL_WRCOMP_EN_MASK;
+ new_dma_pkt->ctl = cpu_to_le32(ctl);
+ new_dma_pkt->tsize = cpu_to_le32(len);
+
+ if (dir == DMA_TO_DEVICE) {
+ new_dma_pkt->src_addr = cpu_to_le64(dma_addr);
+ new_dma_pkt->dst_addr = cpu_to_le64(device_memory_addr);
+ } else {
+ new_dma_pkt->src_addr = cpu_to_le64(device_memory_addr);
+ new_dma_pkt->dst_addr = cpu_to_le64(dma_addr);
+ }
+
+ if (!user_memset)
+ device_memory_addr += len;
+ dma_desc_cnt++;
+ new_dma_pkt++;
+ }
+
+ if (!dma_desc_cnt) {
+ dev_err(hdev->dev,
+ "Error of 0 SG entries when patching DMA packet\n");
+ return -EFAULT;
+ }
+
+ /* Fix the last dma packet - wrcomp must be as user set it */
+ new_dma_pkt--;
+ new_dma_pkt->ctl |= cpu_to_le32(user_wrcomp_en_mask);
+
+ *new_dma_pkt_size = dma_desc_cnt * sizeof(struct packet_lin_dma);
+
+ return 0;
+}
+
+static int gaudi_patch_cb(struct hl_device *hdev,
+ struct hl_cs_parser *parser)
+{
+ u32 cb_parsed_length = 0;
+ u32 cb_patched_cur_length = 0;
+ int rc = 0;
+
+ /* cb_user_size is more than 0 so loop will always be executed */
+ while (cb_parsed_length < parser->user_cb_size) {
+ enum packet_id pkt_id;
+ u16 pkt_size;
+ u32 new_pkt_size = 0;
+ struct gaudi_packet *user_pkt, *kernel_pkt;
+
+ user_pkt = (struct gaudi_packet *) (uintptr_t)
+ (parser->user_cb->kernel_address + cb_parsed_length);
+ kernel_pkt = (struct gaudi_packet *) (uintptr_t)
+ (parser->patched_cb->kernel_address +
+ cb_patched_cur_length);
+
+ pkt_id = (enum packet_id) (
+ (le64_to_cpu(user_pkt->header) &
+ PACKET_HEADER_PACKET_ID_MASK) >>
+ PACKET_HEADER_PACKET_ID_SHIFT);
+
+ pkt_size = gaudi_packet_sizes[pkt_id];
+ cb_parsed_length += pkt_size;
+ if (cb_parsed_length > parser->user_cb_size) {
+ dev_err(hdev->dev,
+ "packet 0x%x is out of CB boundary\n", pkt_id);
+ rc = -EINVAL;
+ break;
+ }
+
+ switch (pkt_id) {
+ case PACKET_LIN_DMA:
+ rc = gaudi_patch_dma_packet(hdev, parser,
+ (struct packet_lin_dma *) user_pkt,
+ (struct packet_lin_dma *) kernel_pkt,
+ &new_pkt_size);
+ cb_patched_cur_length += new_pkt_size;
+ break;
+
+ case PACKET_MSG_PROT:
+ dev_err(hdev->dev,
+ "User not allowed to use MSG_PROT\n");
+ rc = -EPERM;
+ break;
+
+ case PACKET_CP_DMA:
+ dev_err(hdev->dev, "User not allowed to use CP_DMA\n");
+ rc = -EPERM;
+ break;
+
+ case PACKET_STOP:
+ dev_err(hdev->dev, "User not allowed to use STOP\n");
+ rc = -EPERM;
+ break;
+
+ case PACKET_WREG_32:
+ case PACKET_WREG_BULK:
+ case PACKET_MSG_LONG:
+ case PACKET_MSG_SHORT:
+ case PACKET_REPEAT:
+ case PACKET_FENCE:
+ case PACKET_NOP:
+ case PACKET_ARB_POINT:
+ case PACKET_LOAD_AND_EXE:
+ memcpy(kernel_pkt, user_pkt, pkt_size);
+ cb_patched_cur_length += pkt_size;
+ break;
+
+ default:
+ dev_err(hdev->dev, "Invalid packet header 0x%x\n",
+ pkt_id);
+ rc = -EINVAL;
+ break;
+ }
+
+ if (rc)
+ break;
+ }
+
+ return rc;
+}
+
+static int gaudi_parse_cb_mmu(struct hl_device *hdev,
+ struct hl_cs_parser *parser)
+{
+ u64 patched_cb_handle;
+ u32 patched_cb_size;
+ struct hl_cb *user_cb;
+ int rc;
+
+ /*
+ * The new CB should have space at the end for two MSG_PROT pkt:
+ * 1. A packet that will act as a completion packet
+ * 2. A packet that will generate MSI interrupt
+ */
+ parser->patched_cb_size = parser->user_cb_size +
+ sizeof(struct packet_msg_prot) * 2;
+
+ rc = hl_cb_create(hdev, &hdev->kernel_cb_mgr,
+ parser->patched_cb_size,
+ &patched_cb_handle, HL_KERNEL_ASID_ID);
+
+ if (rc) {
+ dev_err(hdev->dev,
+ "Failed to allocate patched CB for DMA CS %d\n",
+ rc);
+ return rc;
+ }
+
+ patched_cb_handle >>= PAGE_SHIFT;
+ parser->patched_cb = hl_cb_get(hdev, &hdev->kernel_cb_mgr,
+ (u32) patched_cb_handle);
+ /* hl_cb_get should never fail here so use kernel WARN */
+ WARN(!parser->patched_cb, "DMA CB handle invalid 0x%x\n",
+ (u32) patched_cb_handle);
+ if (!parser->patched_cb) {
+ rc = -EFAULT;
+ goto out;
+ }
+
+ /*
+ * The check that parser->user_cb_size <= parser->user_cb->size was done
+ * in validate_queue_index().
+ */
+ memcpy((void *) (uintptr_t) parser->patched_cb->kernel_address,
+ (void *) (uintptr_t) parser->user_cb->kernel_address,
+ parser->user_cb_size);
+
+ patched_cb_size = parser->patched_cb_size;
+
+ /* Validate patched CB instead of user CB */
+ user_cb = parser->user_cb;
+ parser->user_cb = parser->patched_cb;
+ rc = gaudi_validate_cb(hdev, parser, true);
+ parser->user_cb = user_cb;
+
+ if (rc) {
+ hl_cb_put(parser->patched_cb);
+ goto out;
+ }
+
+ if (patched_cb_size != parser->patched_cb_size) {
+ dev_err(hdev->dev, "user CB size mismatch\n");
+ hl_cb_put(parser->patched_cb);
+ rc = -EINVAL;
+ goto out;
+ }
+
+out:
+ /*
+ * Always call cb destroy here because we still have 1 reference
+ * to it by calling cb_get earlier. After the job will be completed,
+ * cb_put will release it, but here we want to remove it from the
+ * idr
+ */
+ hl_cb_destroy(hdev, &hdev->kernel_cb_mgr,
+ patched_cb_handle << PAGE_SHIFT);
+
+ return rc;
+}
+
+static int gaudi_parse_cb_no_mmu(struct hl_device *hdev,
+ struct hl_cs_parser *parser)
+{
+ u64 patched_cb_handle;
+ int rc;
+
+ rc = gaudi_validate_cb(hdev, parser, false);
+
+ if (rc)
+ goto free_userptr;
+
+ rc = hl_cb_create(hdev, &hdev->kernel_cb_mgr,
+ parser->patched_cb_size,
+ &patched_cb_handle, HL_KERNEL_ASID_ID);
+ if (rc) {
+ dev_err(hdev->dev,
+ "Failed to allocate patched CB for DMA CS %d\n", rc);
+ goto free_userptr;
+ }
+
+ patched_cb_handle >>= PAGE_SHIFT;
+ parser->patched_cb = hl_cb_get(hdev, &hdev->kernel_cb_mgr,
+ (u32) patched_cb_handle);
+ /* hl_cb_get should never fail here so use kernel WARN */
+ WARN(!parser->patched_cb, "DMA CB handle invalid 0x%x\n",
+ (u32) patched_cb_handle);
+ if (!parser->patched_cb) {
+ rc = -EFAULT;
+ goto out;
+ }
+
+ rc = gaudi_patch_cb(hdev, parser);
+
+ if (rc)
+ hl_cb_put(parser->patched_cb);
+
+out:
+ /*
+ * Always call cb destroy here because we still have 1 reference
+ * to it by calling cb_get earlier. After the job will be completed,
+ * cb_put will release it, but here we want to remove it from the
+ * idr
+ */
+ hl_cb_destroy(hdev, &hdev->kernel_cb_mgr,
+ patched_cb_handle << PAGE_SHIFT);
+
+free_userptr:
+ if (rc)
+ hl_userptr_delete_list(hdev, parser->job_userptr_list);
+ return rc;
+}
+
+static int gaudi_parse_cb_no_ext_queue(struct hl_device *hdev,
+ struct hl_cs_parser *parser)
+{
+ struct asic_fixed_properties *asic_prop = &hdev->asic_prop;
+
+ /* For internal queue jobs just check if CB address is valid */
+ if (hl_mem_area_inside_range((u64) (uintptr_t) parser->user_cb,
+ parser->user_cb_size,
+ asic_prop->sram_user_base_address,
+ asic_prop->sram_end_address))
+ return 0;
+
+ if (hl_mem_area_inside_range((u64) (uintptr_t) parser->user_cb,
+ parser->user_cb_size,
+ asic_prop->dram_user_base_address,
+ asic_prop->dram_end_address))
+ return 0;
+
+ /* PMMU and HPMMU addresses are equal, check only one of them */
+ if (hl_mem_area_inside_range((u64) (uintptr_t) parser->user_cb,
+ parser->user_cb_size,
+ asic_prop->pmmu.start_addr,
+ asic_prop->pmmu.end_addr))
+ return 0;
+
+ dev_err(hdev->dev,
+ "CB address 0x%px + 0x%x for internal QMAN is not valid\n",
+ parser->user_cb, parser->user_cb_size);
+
+ return -EFAULT;
+}
+
+static int gaudi_cs_parser(struct hl_device *hdev, struct hl_cs_parser *parser)
+{
+ struct gaudi_device *gaudi = hdev->asic_specific;
+
+ if (parser->queue_type == QUEUE_TYPE_INT)
+ return gaudi_parse_cb_no_ext_queue(hdev, parser);
+
+ if (gaudi->hw_cap_initialized & HW_CAP_MMU)
+ return gaudi_parse_cb_mmu(hdev, parser);
+ else
+ return gaudi_parse_cb_no_mmu(hdev, parser);
+}
+
+static void gaudi_add_end_of_cb_packets(struct hl_device *hdev,
+ u64 kernel_address, u32 len,
+ u64 cq_addr, u32 cq_val, u32 msi_vec,
+ bool eb)
+{
+ struct gaudi_device *gaudi = hdev->asic_specific;
+ struct packet_msg_prot *cq_pkt;
+ u32 tmp;
+
+ cq_pkt = (struct packet_msg_prot *) (uintptr_t)
+ (kernel_address + len - (sizeof(struct packet_msg_prot) * 2));
+
+ tmp = (PACKET_MSG_PROT << GAUDI_PKT_CTL_OPCODE_SHIFT) |
+ (1 << GAUDI_PKT_CTL_MB_SHIFT);
+
+ if (eb)
+ tmp |= (1 << GAUDI_PKT_CTL_EB_SHIFT);
+
+ cq_pkt->ctl = cpu_to_le32(tmp);
+ cq_pkt->value = cpu_to_le32(cq_val);
+ cq_pkt->addr = cpu_to_le64(cq_addr);
+
+ cq_pkt++;
+
+ tmp = (PACKET_MSG_PROT << GAUDI_PKT_CTL_OPCODE_SHIFT) |
+ (1 << GAUDI_PKT_CTL_MB_SHIFT);
+ cq_pkt->ctl = cpu_to_le32(tmp);
+ cq_pkt->value = cpu_to_le32(1);
+
+ if (!gaudi->multi_msi_mode)
+ msi_vec = 0;
+
+ cq_pkt->addr = cpu_to_le64(CFG_BASE + mmPCIE_MSI_INTR_0 + msi_vec * 4);
+}
+
+static void gaudi_update_eq_ci(struct hl_device *hdev, u32 val)
+{
+ WREG32(mmCPU_IF_EQ_RD_OFFS, val);
+}
+
+static int gaudi_memset_device_memory(struct hl_device *hdev, u64 addr,
+ u32 size, u64 val)
+{
+ struct packet_lin_dma *lin_dma_pkt;
+ struct hl_cs_job *job;
+ u32 cb_size, ctl;
+ struct hl_cb *cb;
+ int rc;
+
+ cb = hl_cb_kernel_create(hdev, PAGE_SIZE);
+ if (!cb)
+ return -EFAULT;
+
+ lin_dma_pkt = (struct packet_lin_dma *) (uintptr_t) cb->kernel_address;
+ memset(lin_dma_pkt, 0, sizeof(*lin_dma_pkt));
+ cb_size = sizeof(*lin_dma_pkt);
+
+ ctl = ((PACKET_LIN_DMA << GAUDI_PKT_CTL_OPCODE_SHIFT) |
+ (1 << GAUDI_PKT_LIN_DMA_CTL_MEMSET_SHIFT) |
+ (1 << GAUDI_PKT_LIN_DMA_CTL_LIN_SHIFT) |
+ (1 << GAUDI_PKT_CTL_RB_SHIFT) |
+ (1 << GAUDI_PKT_CTL_MB_SHIFT));
+ lin_dma_pkt->ctl = cpu_to_le32(ctl);
+ lin_dma_pkt->src_addr = cpu_to_le64(val);
+ lin_dma_pkt->dst_addr |= cpu_to_le64(addr);
+ lin_dma_pkt->tsize = cpu_to_le32(size);
+
+ job = hl_cs_allocate_job(hdev, QUEUE_TYPE_EXT, true);
+ if (!job) {
+ dev_err(hdev->dev, "Failed to allocate a new job\n");
+ rc = -ENOMEM;
+ goto release_cb;
+ }
+
+ job->id = 0;
+ job->user_cb = cb;
+ job->user_cb->cs_cnt++;
+ job->user_cb_size = cb_size;
+ job->hw_queue_id = GAUDI_QUEUE_ID_DMA_0_0;
+ job->patched_cb = job->user_cb;
+ job->job_cb_size = job->user_cb_size + sizeof(struct packet_msg_prot);
+
+ hl_debugfs_add_job(hdev, job);
+
+ rc = gaudi_send_job_on_qman0(hdev, job);
+
+ hl_debugfs_remove_job(hdev, job);
+ kfree(job);
+ cb->cs_cnt--;
+
+release_cb:
+ hl_cb_put(cb);
+ hl_cb_destroy(hdev, &hdev->kernel_cb_mgr, cb->id << PAGE_SHIFT);
+
+ return rc;
+}
+
+static void gaudi_restore_sm_registers(struct hl_device *hdev)
+{
+ int i;
+
+ for (i = 0 ; i < NUM_OF_SOB_IN_BLOCK << 2 ; i += 4) {
+ WREG32(mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_SOB_OBJ_0 + i, 0);
+ WREG32(mmSYNC_MNGR_E_S_SYNC_MNGR_OBJS_SOB_OBJ_0 + i, 0);
+ WREG32(mmSYNC_MNGR_W_N_SYNC_MNGR_OBJS_SOB_OBJ_0 + i, 0);
+ }
+
+ for (i = 0 ; i < NUM_OF_MONITORS_IN_BLOCK << 2 ; i += 4) {
+ WREG32(mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_MON_STATUS_0 + i, 0);
+ WREG32(mmSYNC_MNGR_E_S_SYNC_MNGR_OBJS_MON_STATUS_0 + i, 0);
+ WREG32(mmSYNC_MNGR_W_N_SYNC_MNGR_OBJS_MON_STATUS_0 + i, 0);
+ }
+
+ i = GAUDI_FIRST_AVAILABLE_W_S_SYNC_OBJECT * 4;
+
+ for (; i < NUM_OF_SOB_IN_BLOCK << 2 ; i += 4)
+ WREG32(mmSYNC_MNGR_W_S_SYNC_MNGR_OBJS_SOB_OBJ_0 + i, 0);
+
+ i = GAUDI_FIRST_AVAILABLE_W_S_MONITOR * 4;
+
+ for (; i < NUM_OF_MONITORS_IN_BLOCK << 2 ; i += 4)
+ WREG32(mmSYNC_MNGR_W_S_SYNC_MNGR_OBJS_MON_STATUS_0 + i, 0);
+}
+
+static void gaudi_restore_dma_registers(struct hl_device *hdev)
+{
+ u32 sob_delta = mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_SOB_OBJ_1 -
+ mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_SOB_OBJ_0;
+ int i;
+
+ for (i = 0 ; i < DMA_NUMBER_OF_CHANNELS ; i++) {
+ u64 sob_addr = CFG_BASE +
+ mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_SOB_OBJ_0 +
+ (i * sob_delta);
+ u32 dma_offset = i * DMA_CORE_OFFSET;
+
+ WREG32(mmDMA0_CORE_WR_COMP_ADDR_LO + dma_offset,
+ lower_32_bits(sob_addr));
+ WREG32(mmDMA0_CORE_WR_COMP_ADDR_HI + dma_offset,
+ upper_32_bits(sob_addr));
+ WREG32(mmDMA0_CORE_WR_COMP_WDATA + dma_offset, 0x80000001);
+
+ /* For DMAs 2-7, need to restore WR_AWUSER_31_11 as it can be
+ * modified by the user for SRAM reduction
+ */
+ if (i > 1)
+ WREG32(mmDMA0_CORE_WR_AWUSER_31_11 + dma_offset,
+ 0x00000001);
+ }
+}
+
+static void gaudi_restore_qm_registers(struct hl_device *hdev)
+{
+ u32 qman_offset;
+ int i;
+
+ for (i = 0 ; i < DMA_NUMBER_OF_CHANNELS ; i++) {
+ qman_offset = i * DMA_QMAN_OFFSET;
+ WREG32(mmDMA0_QM_ARB_CFG_0 + qman_offset, 0);
+ }
+
+ for (i = 0 ; i < MME_NUMBER_OF_MASTER_ENGINES ; i++) {
+ qman_offset = i * (mmMME2_QM_BASE - mmMME0_QM_BASE);
+ WREG32(mmMME0_QM_ARB_CFG_0 + qman_offset, 0);
+ }
+
+ for (i = 0 ; i < TPC_NUMBER_OF_ENGINES ; i++) {
+ qman_offset = i * TPC_QMAN_OFFSET;
+ WREG32(mmTPC0_QM_ARB_CFG_0 + qman_offset, 0);
+ }
+}
+
+static void gaudi_restore_user_registers(struct hl_device *hdev)
+{
+ gaudi_restore_sm_registers(hdev);
+ gaudi_restore_dma_registers(hdev);
+ gaudi_restore_qm_registers(hdev);
+}
+
+static int gaudi_context_switch(struct hl_device *hdev, u32 asid)
+{
+ struct asic_fixed_properties *prop = &hdev->asic_prop;
+ u64 addr = prop->sram_user_base_address;
+ u32 size = hdev->pldm ? 0x10000 :
+ (prop->sram_size - SRAM_USER_BASE_OFFSET);
+ u64 val = 0x7777777777777777ull;
+ int rc;
+
+ rc = gaudi_memset_device_memory(hdev, addr, size, val);
+ if (rc) {
+ dev_err(hdev->dev, "Failed to clear SRAM in context switch\n");
+ return rc;
+ }
+
+ gaudi_mmu_prepare(hdev, asid);
+
+ gaudi_restore_user_registers(hdev);
+
+ return 0;
+}
+
+static int gaudi_mmu_clear_pgt_range(struct hl_device *hdev)
+{
+ struct asic_fixed_properties *prop = &hdev->asic_prop;
+ struct gaudi_device *gaudi = hdev->asic_specific;
+ u64 addr = prop->mmu_pgt_addr;
+ u32 size = prop->mmu_pgt_size + MMU_CACHE_MNG_SIZE;
+
+ if (!(gaudi->hw_cap_initialized & HW_CAP_MMU))
+ return 0;
+
+ return gaudi_memset_device_memory(hdev, addr, size, 0);
+}
+
+static void gaudi_restore_phase_topology(struct hl_device *hdev)
+{
+
+}
+
+static int gaudi_debugfs_read32(struct hl_device *hdev, u64 addr, u32 *val)
+{
+ struct asic_fixed_properties *prop = &hdev->asic_prop;
+ struct gaudi_device *gaudi = hdev->asic_specific;
+ u64 hbm_bar_addr;
+ int rc = 0;
+
+ if ((addr >= CFG_BASE) && (addr < CFG_BASE + CFG_SIZE)) {
+ if (gaudi->hw_cap_initialized & HW_CAP_CLK_GATE) {
+ dev_err_ratelimited(hdev->dev,
+ "Can't read register - clock gating is enabled!\n");
+ rc = -EFAULT;
+ } else {
+ *val = RREG32(addr - CFG_BASE);
+ }
+ } else if ((addr >= SRAM_BASE_ADDR) &&
+ (addr < SRAM_BASE_ADDR + SRAM_BAR_SIZE)) {
+ *val = readl(hdev->pcie_bar[SRAM_BAR_ID] +
+ (addr - SRAM_BASE_ADDR));
+ } else if (addr < DRAM_PHYS_BASE + hdev->asic_prop.dram_size) {
+ u64 bar_base_addr = DRAM_PHYS_BASE +
+ (addr & ~(prop->dram_pci_bar_size - 0x1ull));
+
+ hbm_bar_addr = gaudi_set_hbm_bar_base(hdev, bar_base_addr);
+ if (hbm_bar_addr != U64_MAX) {
+ *val = readl(hdev->pcie_bar[HBM_BAR_ID] +
+ (addr - bar_base_addr));
+
+ hbm_bar_addr = gaudi_set_hbm_bar_base(hdev,
+ hbm_bar_addr);
+ }
+ if (hbm_bar_addr == U64_MAX)
+ rc = -EIO;
+ } else if (addr >= HOST_PHYS_BASE && !iommu_present(&pci_bus_type)) {
+ *val = *(u32 *) phys_to_virt(addr - HOST_PHYS_BASE);
+ } else {
+ rc = -EFAULT;
+ }
+
+ return rc;
+}
+
+static int gaudi_debugfs_write32(struct hl_device *hdev, u64 addr, u32 val)
+{
+ struct asic_fixed_properties *prop = &hdev->asic_prop;
+ struct gaudi_device *gaudi = hdev->asic_specific;
+ u64 hbm_bar_addr;
+ int rc = 0;
+
+ if ((addr >= CFG_BASE) && (addr < CFG_BASE + CFG_SIZE)) {
+ if (gaudi->hw_cap_initialized & HW_CAP_CLK_GATE) {
+ dev_err_ratelimited(hdev->dev,
+ "Can't write register - clock gating is enabled!\n");
+ rc = -EFAULT;
+ } else {
+ WREG32(addr - CFG_BASE, val);
+ }
+ } else if ((addr >= SRAM_BASE_ADDR) &&
+ (addr < SRAM_BASE_ADDR + SRAM_BAR_SIZE)) {
+ writel(val, hdev->pcie_bar[SRAM_BAR_ID] +
+ (addr - SRAM_BASE_ADDR));
+ } else if (addr < DRAM_PHYS_BASE + hdev->asic_prop.dram_size) {
+ u64 bar_base_addr = DRAM_PHYS_BASE +
+ (addr & ~(prop->dram_pci_bar_size - 0x1ull));
+
+ hbm_bar_addr = gaudi_set_hbm_bar_base(hdev, bar_base_addr);
+ if (hbm_bar_addr != U64_MAX) {
+ writel(val, hdev->pcie_bar[HBM_BAR_ID] +
+ (addr - bar_base_addr));
+
+ hbm_bar_addr = gaudi_set_hbm_bar_base(hdev,
+ hbm_bar_addr);
+ }
+ if (hbm_bar_addr == U64_MAX)
+ rc = -EIO;
+ } else if (addr >= HOST_PHYS_BASE && !iommu_present(&pci_bus_type)) {
+ *(u32 *) phys_to_virt(addr - HOST_PHYS_BASE) = val;
+ } else {
+ rc = -EFAULT;
+ }
+
+ return rc;
+}
+
+static int gaudi_debugfs_read64(struct hl_device *hdev, u64 addr, u64 *val)
+{
+ struct asic_fixed_properties *prop = &hdev->asic_prop;
+ struct gaudi_device *gaudi = hdev->asic_specific;
+ u64 hbm_bar_addr;
+ int rc = 0;
+
+ if ((addr >= CFG_BASE) && (addr <= CFG_BASE + CFG_SIZE - sizeof(u64))) {
+ if (gaudi->hw_cap_initialized & HW_CAP_CLK_GATE) {
+ dev_err_ratelimited(hdev->dev,
+ "Can't read register - clock gating is enabled!\n");
+ rc = -EFAULT;
+ } else {
+ u32 val_l = RREG32(addr - CFG_BASE);
+ u32 val_h = RREG32(addr + sizeof(u32) - CFG_BASE);
+
+ *val = (((u64) val_h) << 32) | val_l;
+ }
+ } else if ((addr >= SRAM_BASE_ADDR) &&
+ (addr <= SRAM_BASE_ADDR + SRAM_BAR_SIZE - sizeof(u64))) {
+ *val = readq(hdev->pcie_bar[SRAM_BAR_ID] +
+ (addr - SRAM_BASE_ADDR));
+ } else if (addr <=
+ DRAM_PHYS_BASE + hdev->asic_prop.dram_size - sizeof(u64)) {
+ u64 bar_base_addr = DRAM_PHYS_BASE +
+ (addr & ~(prop->dram_pci_bar_size - 0x1ull));
+
+ hbm_bar_addr = gaudi_set_hbm_bar_base(hdev, bar_base_addr);
+ if (hbm_bar_addr != U64_MAX) {
+ *val = readq(hdev->pcie_bar[HBM_BAR_ID] +
+ (addr - bar_base_addr));
+
+ hbm_bar_addr = gaudi_set_hbm_bar_base(hdev,
+ hbm_bar_addr);
+ }
+ if (hbm_bar_addr == U64_MAX)
+ rc = -EIO;
+ } else if (addr >= HOST_PHYS_BASE && !iommu_present(&pci_bus_type)) {
+ *val = *(u64 *) phys_to_virt(addr - HOST_PHYS_BASE);
+ } else {
+ rc = -EFAULT;
+ }
+
+ return rc;
+}
+
+static int gaudi_debugfs_write64(struct hl_device *hdev, u64 addr, u64 val)
+{
+ struct asic_fixed_properties *prop = &hdev->asic_prop;
+ struct gaudi_device *gaudi = hdev->asic_specific;
+ u64 hbm_bar_addr;
+ int rc = 0;
+
+ if ((addr >= CFG_BASE) && (addr <= CFG_BASE + CFG_SIZE - sizeof(u64))) {
+ if (gaudi->hw_cap_initialized & HW_CAP_CLK_GATE) {
+ dev_err_ratelimited(hdev->dev,
+ "Can't write register - clock gating is enabled!\n");
+ rc = -EFAULT;
+ } else {
+ WREG32(addr - CFG_BASE, lower_32_bits(val));
+ WREG32(addr + sizeof(u32) - CFG_BASE,
+ upper_32_bits(val));
+ }
+ } else if ((addr >= SRAM_BASE_ADDR) &&
+ (addr <= SRAM_BASE_ADDR + SRAM_BAR_SIZE - sizeof(u64))) {
+ writeq(val, hdev->pcie_bar[SRAM_BAR_ID] +
+ (addr - SRAM_BASE_ADDR));
+ } else if (addr <=
+ DRAM_PHYS_BASE + hdev->asic_prop.dram_size - sizeof(u64)) {
+ u64 bar_base_addr = DRAM_PHYS_BASE +
+ (addr & ~(prop->dram_pci_bar_size - 0x1ull));
+
+ hbm_bar_addr = gaudi_set_hbm_bar_base(hdev, bar_base_addr);
+ if (hbm_bar_addr != U64_MAX) {
+ writeq(val, hdev->pcie_bar[HBM_BAR_ID] +
+ (addr - bar_base_addr));
+
+ hbm_bar_addr = gaudi_set_hbm_bar_base(hdev,
+ hbm_bar_addr);
+ }
+ if (hbm_bar_addr == U64_MAX)
+ rc = -EIO;
+ } else if (addr >= HOST_PHYS_BASE && !iommu_present(&pci_bus_type)) {
+ *(u64 *) phys_to_virt(addr - HOST_PHYS_BASE) = val;
+ } else {
+ rc = -EFAULT;
+ }
+
+ return rc;
+}
+
+static u64 gaudi_read_pte(struct hl_device *hdev, u64 addr)
+{
+ struct gaudi_device *gaudi = hdev->asic_specific;
+
+ if (hdev->hard_reset_pending)
+ return U64_MAX;
+
+ return readq(hdev->pcie_bar[HBM_BAR_ID] +
+ (addr - gaudi->hbm_bar_cur_addr));
+}
+
+static void gaudi_write_pte(struct hl_device *hdev, u64 addr, u64 val)
+{
+ struct gaudi_device *gaudi = hdev->asic_specific;
+
+ if (hdev->hard_reset_pending)
+ return;
+
+ writeq(val, hdev->pcie_bar[HBM_BAR_ID] +
+ (addr - gaudi->hbm_bar_cur_addr));
+}
+
+static void gaudi_mmu_prepare_reg(struct hl_device *hdev, u64 reg, u32 asid)
+{
+ /* mask to zero the MMBP and ASID bits */
+ WREG32_AND(reg, ~0x7FF);
+ WREG32_OR(reg, asid);
+}
+
+static void gaudi_mmu_prepare(struct hl_device *hdev, u32 asid)
+{
+ struct gaudi_device *gaudi = hdev->asic_specific;
+
+ if (!(gaudi->hw_cap_initialized & HW_CAP_MMU))
+ return;
+
+ if (asid & ~DMA0_QM_GLBL_NON_SECURE_PROPS_0_ASID_MASK) {
+ WARN(1, "asid %u is too big\n", asid);
+ return;
+ }
+
+ mutex_lock(&gaudi->clk_gate_mutex);
+
+ hdev->asic_funcs->disable_clock_gating(hdev);
+
+ gaudi_mmu_prepare_reg(hdev, mmDMA0_QM_GLBL_NON_SECURE_PROPS_0, asid);
+ gaudi_mmu_prepare_reg(hdev, mmDMA0_QM_GLBL_NON_SECURE_PROPS_1, asid);
+ gaudi_mmu_prepare_reg(hdev, mmDMA0_QM_GLBL_NON_SECURE_PROPS_2, asid);
+ gaudi_mmu_prepare_reg(hdev, mmDMA0_QM_GLBL_NON_SECURE_PROPS_3, asid);
+ gaudi_mmu_prepare_reg(hdev, mmDMA0_QM_GLBL_NON_SECURE_PROPS_4, asid);
+
+ gaudi_mmu_prepare_reg(hdev, mmDMA1_QM_GLBL_NON_SECURE_PROPS_0, asid);
+ gaudi_mmu_prepare_reg(hdev, mmDMA1_QM_GLBL_NON_SECURE_PROPS_1, asid);
+ gaudi_mmu_prepare_reg(hdev, mmDMA1_QM_GLBL_NON_SECURE_PROPS_2, asid);
+ gaudi_mmu_prepare_reg(hdev, mmDMA1_QM_GLBL_NON_SECURE_PROPS_3, asid);
+ gaudi_mmu_prepare_reg(hdev, mmDMA1_QM_GLBL_NON_SECURE_PROPS_4, asid);
+
+ gaudi_mmu_prepare_reg(hdev, mmDMA2_QM_GLBL_NON_SECURE_PROPS_0, asid);
+ gaudi_mmu_prepare_reg(hdev, mmDMA2_QM_GLBL_NON_SECURE_PROPS_1, asid);
+ gaudi_mmu_prepare_reg(hdev, mmDMA2_QM_GLBL_NON_SECURE_PROPS_2, asid);
+ gaudi_mmu_prepare_reg(hdev, mmDMA2_QM_GLBL_NON_SECURE_PROPS_3, asid);
+ gaudi_mmu_prepare_reg(hdev, mmDMA2_QM_GLBL_NON_SECURE_PROPS_4, asid);
+
+ gaudi_mmu_prepare_reg(hdev, mmDMA3_QM_GLBL_NON_SECURE_PROPS_0, asid);
+ gaudi_mmu_prepare_reg(hdev, mmDMA3_QM_GLBL_NON_SECURE_PROPS_1, asid);
+ gaudi_mmu_prepare_reg(hdev, mmDMA3_QM_GLBL_NON_SECURE_PROPS_2, asid);
+ gaudi_mmu_prepare_reg(hdev, mmDMA3_QM_GLBL_NON_SECURE_PROPS_3, asid);
+ gaudi_mmu_prepare_reg(hdev, mmDMA3_QM_GLBL_NON_SECURE_PROPS_4, asid);
+
+ gaudi_mmu_prepare_reg(hdev, mmDMA4_QM_GLBL_NON_SECURE_PROPS_0, asid);
+ gaudi_mmu_prepare_reg(hdev, mmDMA4_QM_GLBL_NON_SECURE_PROPS_1, asid);
+ gaudi_mmu_prepare_reg(hdev, mmDMA4_QM_GLBL_NON_SECURE_PROPS_2, asid);
+ gaudi_mmu_prepare_reg(hdev, mmDMA4_QM_GLBL_NON_SECURE_PROPS_3, asid);
+ gaudi_mmu_prepare_reg(hdev, mmDMA4_QM_GLBL_NON_SECURE_PROPS_4, asid);
+
+ gaudi_mmu_prepare_reg(hdev, mmDMA5_QM_GLBL_NON_SECURE_PROPS_0, asid);
+ gaudi_mmu_prepare_reg(hdev, mmDMA5_QM_GLBL_NON_SECURE_PROPS_1, asid);
+ gaudi_mmu_prepare_reg(hdev, mmDMA5_QM_GLBL_NON_SECURE_PROPS_2, asid);
+ gaudi_mmu_prepare_reg(hdev, mmDMA5_QM_GLBL_NON_SECURE_PROPS_3, asid);
+ gaudi_mmu_prepare_reg(hdev, mmDMA5_QM_GLBL_NON_SECURE_PROPS_4, asid);
+
+ gaudi_mmu_prepare_reg(hdev, mmDMA6_QM_GLBL_NON_SECURE_PROPS_0, asid);
+ gaudi_mmu_prepare_reg(hdev, mmDMA6_QM_GLBL_NON_SECURE_PROPS_1, asid);
+ gaudi_mmu_prepare_reg(hdev, mmDMA6_QM_GLBL_NON_SECURE_PROPS_2, asid);
+ gaudi_mmu_prepare_reg(hdev, mmDMA6_QM_GLBL_NON_SECURE_PROPS_3, asid);
+ gaudi_mmu_prepare_reg(hdev, mmDMA6_QM_GLBL_NON_SECURE_PROPS_4, asid);
+
+ gaudi_mmu_prepare_reg(hdev, mmDMA7_QM_GLBL_NON_SECURE_PROPS_0, asid);
+ gaudi_mmu_prepare_reg(hdev, mmDMA7_QM_GLBL_NON_SECURE_PROPS_1, asid);
+ gaudi_mmu_prepare_reg(hdev, mmDMA7_QM_GLBL_NON_SECURE_PROPS_2, asid);
+ gaudi_mmu_prepare_reg(hdev, mmDMA7_QM_GLBL_NON_SECURE_PROPS_3, asid);
+ gaudi_mmu_prepare_reg(hdev, mmDMA7_QM_GLBL_NON_SECURE_PROPS_4, asid);
+
+ gaudi_mmu_prepare_reg(hdev, mmDMA0_CORE_NON_SECURE_PROPS, asid);
+ gaudi_mmu_prepare_reg(hdev, mmDMA1_CORE_NON_SECURE_PROPS, asid);
+ gaudi_mmu_prepare_reg(hdev, mmDMA2_CORE_NON_SECURE_PROPS, asid);
+ gaudi_mmu_prepare_reg(hdev, mmDMA3_CORE_NON_SECURE_PROPS, asid);
+ gaudi_mmu_prepare_reg(hdev, mmDMA4_CORE_NON_SECURE_PROPS, asid);
+ gaudi_mmu_prepare_reg(hdev, mmDMA5_CORE_NON_SECURE_PROPS, asid);
+ gaudi_mmu_prepare_reg(hdev, mmDMA6_CORE_NON_SECURE_PROPS, asid);
+ gaudi_mmu_prepare_reg(hdev, mmDMA7_CORE_NON_SECURE_PROPS, asid);
+
+ gaudi_mmu_prepare_reg(hdev, mmTPC0_QM_GLBL_NON_SECURE_PROPS_0, asid);
+ gaudi_mmu_prepare_reg(hdev, mmTPC0_QM_GLBL_NON_SECURE_PROPS_1, asid);
+ gaudi_mmu_prepare_reg(hdev, mmTPC0_QM_GLBL_NON_SECURE_PROPS_2, asid);
+ gaudi_mmu_prepare_reg(hdev, mmTPC0_QM_GLBL_NON_SECURE_PROPS_3, asid);
+ gaudi_mmu_prepare_reg(hdev, mmTPC0_QM_GLBL_NON_SECURE_PROPS_4, asid);
+ gaudi_mmu_prepare_reg(hdev, mmTPC0_CFG_ARUSER_LO, asid);
+ gaudi_mmu_prepare_reg(hdev, mmTPC0_CFG_AWUSER_LO, asid);
+
+ gaudi_mmu_prepare_reg(hdev, mmTPC1_QM_GLBL_NON_SECURE_PROPS_0, asid);
+ gaudi_mmu_prepare_reg(hdev, mmTPC1_QM_GLBL_NON_SECURE_PROPS_1, asid);
+ gaudi_mmu_prepare_reg(hdev, mmTPC1_QM_GLBL_NON_SECURE_PROPS_2, asid);
+ gaudi_mmu_prepare_reg(hdev, mmTPC1_QM_GLBL_NON_SECURE_PROPS_3, asid);
+ gaudi_mmu_prepare_reg(hdev, mmTPC1_QM_GLBL_NON_SECURE_PROPS_4, asid);
+ gaudi_mmu_prepare_reg(hdev, mmTPC1_CFG_ARUSER_LO, asid);
+ gaudi_mmu_prepare_reg(hdev, mmTPC1_CFG_AWUSER_LO, asid);
+
+ gaudi_mmu_prepare_reg(hdev, mmTPC2_QM_GLBL_NON_SECURE_PROPS_0, asid);
+ gaudi_mmu_prepare_reg(hdev, mmTPC2_QM_GLBL_NON_SECURE_PROPS_1, asid);
+ gaudi_mmu_prepare_reg(hdev, mmTPC2_QM_GLBL_NON_SECURE_PROPS_2, asid);
+ gaudi_mmu_prepare_reg(hdev, mmTPC2_QM_GLBL_NON_SECURE_PROPS_3, asid);
+ gaudi_mmu_prepare_reg(hdev, mmTPC2_QM_GLBL_NON_SECURE_PROPS_4, asid);
+ gaudi_mmu_prepare_reg(hdev, mmTPC2_CFG_ARUSER_LO, asid);
+ gaudi_mmu_prepare_reg(hdev, mmTPC2_CFG_AWUSER_LO, asid);
+
+ gaudi_mmu_prepare_reg(hdev, mmTPC3_QM_GLBL_NON_SECURE_PROPS_0, asid);
+ gaudi_mmu_prepare_reg(hdev, mmTPC3_QM_GLBL_NON_SECURE_PROPS_1, asid);
+ gaudi_mmu_prepare_reg(hdev, mmTPC3_QM_GLBL_NON_SECURE_PROPS_2, asid);
+ gaudi_mmu_prepare_reg(hdev, mmTPC3_QM_GLBL_NON_SECURE_PROPS_3, asid);
+ gaudi_mmu_prepare_reg(hdev, mmTPC3_QM_GLBL_NON_SECURE_PROPS_4, asid);
+ gaudi_mmu_prepare_reg(hdev, mmTPC3_CFG_ARUSER_LO, asid);
+ gaudi_mmu_prepare_reg(hdev, mmTPC3_CFG_AWUSER_LO, asid);
+
+ gaudi_mmu_prepare_reg(hdev, mmTPC4_QM_GLBL_NON_SECURE_PROPS_0, asid);
+ gaudi_mmu_prepare_reg(hdev, mmTPC4_QM_GLBL_NON_SECURE_PROPS_1, asid);
+ gaudi_mmu_prepare_reg(hdev, mmTPC4_QM_GLBL_NON_SECURE_PROPS_2, asid);
+ gaudi_mmu_prepare_reg(hdev, mmTPC4_QM_GLBL_NON_SECURE_PROPS_3, asid);
+ gaudi_mmu_prepare_reg(hdev, mmTPC4_QM_GLBL_NON_SECURE_PROPS_4, asid);
+ gaudi_mmu_prepare_reg(hdev, mmTPC4_CFG_ARUSER_LO, asid);
+ gaudi_mmu_prepare_reg(hdev, mmTPC4_CFG_AWUSER_LO, asid);
+
+ gaudi_mmu_prepare_reg(hdev, mmTPC5_QM_GLBL_NON_SECURE_PROPS_0, asid);
+ gaudi_mmu_prepare_reg(hdev, mmTPC5_QM_GLBL_NON_SECURE_PROPS_1, asid);
+ gaudi_mmu_prepare_reg(hdev, mmTPC5_QM_GLBL_NON_SECURE_PROPS_2, asid);
+ gaudi_mmu_prepare_reg(hdev, mmTPC5_QM_GLBL_NON_SECURE_PROPS_3, asid);
+ gaudi_mmu_prepare_reg(hdev, mmTPC5_QM_GLBL_NON_SECURE_PROPS_4, asid);
+ gaudi_mmu_prepare_reg(hdev, mmTPC5_CFG_ARUSER_LO, asid);
+ gaudi_mmu_prepare_reg(hdev, mmTPC5_CFG_AWUSER_LO, asid);
+
+ gaudi_mmu_prepare_reg(hdev, mmTPC6_QM_GLBL_NON_SECURE_PROPS_0, asid);
+ gaudi_mmu_prepare_reg(hdev, mmTPC6_QM_GLBL_NON_SECURE_PROPS_1, asid);
+ gaudi_mmu_prepare_reg(hdev, mmTPC6_QM_GLBL_NON_SECURE_PROPS_2, asid);
+ gaudi_mmu_prepare_reg(hdev, mmTPC6_QM_GLBL_NON_SECURE_PROPS_3, asid);
+ gaudi_mmu_prepare_reg(hdev, mmTPC6_QM_GLBL_NON_SECURE_PROPS_4, asid);
+ gaudi_mmu_prepare_reg(hdev, mmTPC6_CFG_ARUSER_LO, asid);
+ gaudi_mmu_prepare_reg(hdev, mmTPC6_CFG_AWUSER_LO, asid);
+
+ gaudi_mmu_prepare_reg(hdev, mmTPC7_QM_GLBL_NON_SECURE_PROPS_0, asid);
+ gaudi_mmu_prepare_reg(hdev, mmTPC7_QM_GLBL_NON_SECURE_PROPS_1, asid);
+ gaudi_mmu_prepare_reg(hdev, mmTPC7_QM_GLBL_NON_SECURE_PROPS_2, asid);
+ gaudi_mmu_prepare_reg(hdev, mmTPC7_QM_GLBL_NON_SECURE_PROPS_3, asid);
+ gaudi_mmu_prepare_reg(hdev, mmTPC7_QM_GLBL_NON_SECURE_PROPS_4, asid);
+ gaudi_mmu_prepare_reg(hdev, mmTPC7_CFG_ARUSER_LO, asid);
+ gaudi_mmu_prepare_reg(hdev, mmTPC7_CFG_AWUSER_LO, asid);
+
+ gaudi_mmu_prepare_reg(hdev, mmMME0_QM_GLBL_NON_SECURE_PROPS_0, asid);
+ gaudi_mmu_prepare_reg(hdev, mmMME0_QM_GLBL_NON_SECURE_PROPS_1, asid);
+ gaudi_mmu_prepare_reg(hdev, mmMME0_QM_GLBL_NON_SECURE_PROPS_2, asid);
+ gaudi_mmu_prepare_reg(hdev, mmMME0_QM_GLBL_NON_SECURE_PROPS_3, asid);
+ gaudi_mmu_prepare_reg(hdev, mmMME0_QM_GLBL_NON_SECURE_PROPS_4, asid);
+ gaudi_mmu_prepare_reg(hdev, mmMME2_QM_GLBL_NON_SECURE_PROPS_0, asid);
+ gaudi_mmu_prepare_reg(hdev, mmMME2_QM_GLBL_NON_SECURE_PROPS_1, asid);
+ gaudi_mmu_prepare_reg(hdev, mmMME2_QM_GLBL_NON_SECURE_PROPS_2, asid);
+ gaudi_mmu_prepare_reg(hdev, mmMME2_QM_GLBL_NON_SECURE_PROPS_3, asid);
+ gaudi_mmu_prepare_reg(hdev, mmMME2_QM_GLBL_NON_SECURE_PROPS_4, asid);
+
+ gaudi_mmu_prepare_reg(hdev, mmMME0_SBAB_ARUSER0, asid);
+ gaudi_mmu_prepare_reg(hdev, mmMME0_SBAB_ARUSER1, asid);
+ gaudi_mmu_prepare_reg(hdev, mmMME1_SBAB_ARUSER0, asid);
+ gaudi_mmu_prepare_reg(hdev, mmMME1_SBAB_ARUSER1, asid);
+ gaudi_mmu_prepare_reg(hdev, mmMME2_SBAB_ARUSER0, asid);
+ gaudi_mmu_prepare_reg(hdev, mmMME2_SBAB_ARUSER1, asid);
+ gaudi_mmu_prepare_reg(hdev, mmMME3_SBAB_ARUSER0, asid);
+ gaudi_mmu_prepare_reg(hdev, mmMME3_SBAB_ARUSER1, asid);
+ gaudi_mmu_prepare_reg(hdev, mmMME0_ACC_WBC, asid);
+ gaudi_mmu_prepare_reg(hdev, mmMME1_ACC_WBC, asid);
+ gaudi_mmu_prepare_reg(hdev, mmMME2_ACC_WBC, asid);
+ gaudi_mmu_prepare_reg(hdev, mmMME3_ACC_WBC, asid);
+
+ gaudi_mmu_prepare_reg(hdev, mmPSOC_GLOBAL_CONF_TRACE_ARUSER, asid);
+ gaudi_mmu_prepare_reg(hdev, mmPSOC_GLOBAL_CONF_TRACE_AWUSER, asid);
+
+ hdev->asic_funcs->enable_clock_gating(hdev);
+
+ mutex_unlock(&gaudi->clk_gate_mutex);
+}
+
+static int gaudi_send_job_on_qman0(struct hl_device *hdev,
+ struct hl_cs_job *job)
+{
+ struct packet_msg_prot *fence_pkt;
+ u32 *fence_ptr;
+ dma_addr_t fence_dma_addr;
+ struct hl_cb *cb;
+ u32 tmp, timeout, dma_offset;
+ int rc;
+
+ if (hdev->pldm)
+ timeout = GAUDI_PLDM_QMAN0_TIMEOUT_USEC;
+ else
+ timeout = HL_DEVICE_TIMEOUT_USEC;
+
+ if (!hdev->asic_funcs->is_device_idle(hdev, NULL, NULL)) {
+ dev_err_ratelimited(hdev->dev,
+ "Can't send driver job on QMAN0 because the device is not idle\n");
+ return -EBUSY;
+ }
+
+ fence_ptr = hdev->asic_funcs->asic_dma_pool_zalloc(hdev, 4, GFP_KERNEL,
+ &fence_dma_addr);
+ if (!fence_ptr) {
+ dev_err(hdev->dev,
+ "Failed to allocate fence memory for QMAN0\n");
+ return -ENOMEM;
+ }
+
+ cb = job->patched_cb;
+
+ fence_pkt = (struct packet_msg_prot *) (uintptr_t) (cb->kernel_address +
+ job->job_cb_size - sizeof(struct packet_msg_prot));
+
+ tmp = (PACKET_MSG_PROT << GAUDI_PKT_CTL_OPCODE_SHIFT) |
+ (1 << GAUDI_PKT_CTL_EB_SHIFT) |
+ (1 << GAUDI_PKT_CTL_MB_SHIFT);
+ fence_pkt->ctl = cpu_to_le32(tmp);
+ fence_pkt->value = cpu_to_le32(GAUDI_QMAN0_FENCE_VAL);
+ fence_pkt->addr = cpu_to_le64(fence_dma_addr);
+
+ dma_offset = gaudi_dma_assignment[GAUDI_PCI_DMA_1] * DMA_CORE_OFFSET;
+
+ WREG32_OR(mmDMA0_CORE_PROT + dma_offset, BIT(DMA0_CORE_PROT_VAL_SHIFT));
+
+ rc = hl_hw_queue_send_cb_no_cmpl(hdev, GAUDI_QUEUE_ID_DMA_0_0,
+ job->job_cb_size, cb->bus_address);
+ if (rc) {
+ dev_err(hdev->dev, "Failed to send CB on QMAN0, %d\n", rc);
+ goto free_fence_ptr;
+ }
+
+ rc = hl_poll_timeout_memory(hdev, fence_ptr, tmp,
+ (tmp == GAUDI_QMAN0_FENCE_VAL), 1000,
+ timeout, true);
+
+ hl_hw_queue_inc_ci_kernel(hdev, GAUDI_QUEUE_ID_DMA_0_0);
+
+ if (rc == -ETIMEDOUT) {
+ dev_err(hdev->dev, "QMAN0 Job timeout (0x%x)\n", tmp);
+ goto free_fence_ptr;
+ }
+
+free_fence_ptr:
+ WREG32_AND(mmDMA0_CORE_PROT + dma_offset,
+ ~BIT(DMA0_CORE_PROT_VAL_SHIFT));
+
+ hdev->asic_funcs->asic_dma_pool_free(hdev, (void *) fence_ptr,
+ fence_dma_addr);
+ return rc;
+}
+
+static void gaudi_get_event_desc(u16 event_type, char *desc, size_t size)
+{
+ if (event_type >= GAUDI_EVENT_SIZE)
+ goto event_not_supported;
+
+ if (!gaudi_irq_map_table[event_type].valid)
+ goto event_not_supported;
+
+ snprintf(desc, size, gaudi_irq_map_table[event_type].name);
+
+ return;
+
+event_not_supported:
+ snprintf(desc, size, "N/A");
+}
+
+static const char *gaudi_get_razwi_initiator_dma_name(struct hl_device *hdev,
+ u32 x_y, bool is_write)
+{
+ u32 dma_id[2], dma_offset, err_cause[2], mask, i;
+
+ mask = is_write ? DMA0_CORE_ERR_CAUSE_HBW_WR_ERR_MASK :
+ DMA0_CORE_ERR_CAUSE_HBW_RD_ERR_MASK;
+
+ switch (x_y) {
+ case RAZWI_INITIATOR_ID_X_Y_DMA_IF_W_S_0:
+ case RAZWI_INITIATOR_ID_X_Y_DMA_IF_W_S_1:
+ dma_id[0] = 0;
+ dma_id[1] = 2;
+ break;
+ case RAZWI_INITIATOR_ID_X_Y_DMA_IF_E_S_0:
+ case RAZWI_INITIATOR_ID_X_Y_DMA_IF_E_S_1:
+ dma_id[0] = 1;
+ dma_id[1] = 3;
+ break;
+ case RAZWI_INITIATOR_ID_X_Y_DMA_IF_W_N_0:
+ case RAZWI_INITIATOR_ID_X_Y_DMA_IF_W_N_1:
+ dma_id[0] = 4;
+ dma_id[1] = 6;
+ break;
+ case RAZWI_INITIATOR_ID_X_Y_DMA_IF_E_N_0:
+ case RAZWI_INITIATOR_ID_X_Y_DMA_IF_E_N_1:
+ dma_id[0] = 5;
+ dma_id[1] = 7;
+ break;
+ default:
+ goto unknown_initiator;
+ }
+
+ for (i = 0 ; i < 2 ; i++) {
+ dma_offset = dma_id[i] * DMA_CORE_OFFSET;
+ err_cause[i] = RREG32(mmDMA0_CORE_ERR_CAUSE + dma_offset);
+ }
+
+ switch (x_y) {
+ case RAZWI_INITIATOR_ID_X_Y_DMA_IF_W_S_0:
+ case RAZWI_INITIATOR_ID_X_Y_DMA_IF_W_S_1:
+ if ((err_cause[0] & mask) && !(err_cause[1] & mask))
+ return "DMA0";
+ else if (!(err_cause[0] & mask) && (err_cause[1] & mask))
+ return "DMA2";
+ else
+ return "DMA0 or DMA2";
+ case RAZWI_INITIATOR_ID_X_Y_DMA_IF_E_S_0:
+ case RAZWI_INITIATOR_ID_X_Y_DMA_IF_E_S_1:
+ if ((err_cause[0] & mask) && !(err_cause[1] & mask))
+ return "DMA1";
+ else if (!(err_cause[0] & mask) && (err_cause[1] & mask))
+ return "DMA3";
+ else
+ return "DMA1 or DMA3";
+ case RAZWI_INITIATOR_ID_X_Y_DMA_IF_W_N_0:
+ case RAZWI_INITIATOR_ID_X_Y_DMA_IF_W_N_1:
+ if ((err_cause[0] & mask) && !(err_cause[1] & mask))
+ return "DMA4";
+ else if (!(err_cause[0] & mask) && (err_cause[1] & mask))
+ return "DMA6";
+ else
+ return "DMA4 or DMA6";
+ case RAZWI_INITIATOR_ID_X_Y_DMA_IF_E_N_0:
+ case RAZWI_INITIATOR_ID_X_Y_DMA_IF_E_N_1:
+ if ((err_cause[0] & mask) && !(err_cause[1] & mask))
+ return "DMA5";
+ else if (!(err_cause[0] & mask) && (err_cause[1] & mask))
+ return "DMA7";
+ else
+ return "DMA5 or DMA7";
+ }
+
+unknown_initiator:
+ return "unknown initiator";
+}
+
+static const char *gaudi_get_razwi_initiator_name(struct hl_device *hdev,
+ bool is_write)
+{
+ u32 val, x_y, axi_id;
+
+ val = is_write ? RREG32(mmMMU_UP_RAZWI_WRITE_ID) :
+ RREG32(mmMMU_UP_RAZWI_READ_ID);
+ x_y = val & ((RAZWI_INITIATOR_Y_MASK << RAZWI_INITIATOR_Y_SHIFT) |
+ (RAZWI_INITIATOR_X_MASK << RAZWI_INITIATOR_X_SHIFT));
+ axi_id = val & (RAZWI_INITIATOR_AXI_ID_MASK <<
+ RAZWI_INITIATOR_AXI_ID_SHIFT);
+
+ switch (x_y) {
+ case RAZWI_INITIATOR_ID_X_Y_TPC0_NIC0:
+ if (axi_id == RAZWI_INITIATOR_ID_AXI_ID(AXI_ID_TPC))
+ return "TPC0";
+ if (axi_id == RAZWI_INITIATOR_ID_AXI_ID(AXI_ID_NIC))
+ return "NIC0";
+ break;
+ case RAZWI_INITIATOR_ID_X_Y_TPC1:
+ return "TPC1";
+ case RAZWI_INITIATOR_ID_X_Y_MME0_0:
+ case RAZWI_INITIATOR_ID_X_Y_MME0_1:
+ return "MME0";
+ case RAZWI_INITIATOR_ID_X_Y_MME1_0:
+ case RAZWI_INITIATOR_ID_X_Y_MME1_1:
+ return "MME1";
+ case RAZWI_INITIATOR_ID_X_Y_TPC2:
+ return "TPC2";
+ case RAZWI_INITIATOR_ID_X_Y_TPC3_PCI_CPU_PSOC:
+ if (axi_id == RAZWI_INITIATOR_ID_AXI_ID(AXI_ID_TPC))
+ return "TPC3";
+ if (axi_id == RAZWI_INITIATOR_ID_AXI_ID(AXI_ID_PCI))
+ return "PCI";
+ if (axi_id == RAZWI_INITIATOR_ID_AXI_ID(AXI_ID_CPU))
+ return "CPU";
+ if (axi_id == RAZWI_INITIATOR_ID_AXI_ID(AXI_ID_PSOC))
+ return "PSOC";
+ break;
+ case RAZWI_INITIATOR_ID_X_Y_DMA_IF_W_S_0:
+ case RAZWI_INITIATOR_ID_X_Y_DMA_IF_W_S_1:
+ case RAZWI_INITIATOR_ID_X_Y_DMA_IF_E_S_0:
+ case RAZWI_INITIATOR_ID_X_Y_DMA_IF_E_S_1:
+ case RAZWI_INITIATOR_ID_X_Y_DMA_IF_W_N_0:
+ case RAZWI_INITIATOR_ID_X_Y_DMA_IF_W_N_1:
+ case RAZWI_INITIATOR_ID_X_Y_DMA_IF_E_N_0:
+ case RAZWI_INITIATOR_ID_X_Y_DMA_IF_E_N_1:
+ return gaudi_get_razwi_initiator_dma_name(hdev, x_y, is_write);
+ case RAZWI_INITIATOR_ID_X_Y_TPC4_NIC1_NIC2:
+ if (axi_id == RAZWI_INITIATOR_ID_AXI_ID(AXI_ID_TPC))
+ return "TPC4";
+ if (axi_id == RAZWI_INITIATOR_ID_AXI_ID(AXI_ID_NIC))
+ return "NIC1";
+ if (axi_id == RAZWI_INITIATOR_ID_AXI_ID(AXI_ID_NIC_FT))
+ return "NIC2";
+ break;
+ case RAZWI_INITIATOR_ID_X_Y_TPC5:
+ return "TPC5";
+ case RAZWI_INITIATOR_ID_X_Y_MME2_0:
+ case RAZWI_INITIATOR_ID_X_Y_MME2_1:
+ return "MME2";
+ case RAZWI_INITIATOR_ID_X_Y_MME3_0:
+ case RAZWI_INITIATOR_ID_X_Y_MME3_1:
+ return "MME3";
+ case RAZWI_INITIATOR_ID_X_Y_TPC6:
+ return "TPC6";
+ case RAZWI_INITIATOR_ID_X_Y_TPC7_NIC4_NIC5:
+ if (axi_id == RAZWI_INITIATOR_ID_AXI_ID(AXI_ID_TPC))
+ return "TPC7";
+ if (axi_id == RAZWI_INITIATOR_ID_AXI_ID(AXI_ID_NIC))
+ return "NIC4";
+ if (axi_id == RAZWI_INITIATOR_ID_AXI_ID(AXI_ID_NIC_FT))
+ return "NIC5";
+ break;
+ default:
+ break;
+ }
+
+ dev_err(hdev->dev,
+ "Unknown RAZWI initiator ID 0x%x [Y=%d, X=%d, AXI_ID=%d]\n",
+ val,
+ (val >> RAZWI_INITIATOR_Y_SHIFT) & RAZWI_INITIATOR_Y_MASK,
+ (val >> RAZWI_INITIATOR_X_SHIFT) & RAZWI_INITIATOR_X_MASK,
+ (val >> RAZWI_INITIATOR_AXI_ID_SHIFT) &
+ RAZWI_INITIATOR_AXI_ID_MASK);
+
+ return "unknown initiator";
+}
+
+static void gaudi_print_razwi_info(struct hl_device *hdev)
+{
+ if (RREG32(mmMMU_UP_RAZWI_WRITE_VLD)) {
+ dev_err_ratelimited(hdev->dev,
+ "RAZWI event caused by illegal write of %s\n",
+ gaudi_get_razwi_initiator_name(hdev, true));
+ WREG32(mmMMU_UP_RAZWI_WRITE_VLD, 0);
+ }
+
+ if (RREG32(mmMMU_UP_RAZWI_READ_VLD)) {
+ dev_err_ratelimited(hdev->dev,
+ "RAZWI event caused by illegal read of %s\n",
+ gaudi_get_razwi_initiator_name(hdev, false));
+ WREG32(mmMMU_UP_RAZWI_READ_VLD, 0);
+ }
+}
+
+static void gaudi_print_mmu_error_info(struct hl_device *hdev)
+{
+ struct gaudi_device *gaudi = hdev->asic_specific;
+ u64 addr;
+ u32 val;
+
+ if (!(gaudi->hw_cap_initialized & HW_CAP_MMU))
+ return;
+
+ val = RREG32(mmMMU_UP_PAGE_ERROR_CAPTURE);
+ if (val & MMU_UP_PAGE_ERROR_CAPTURE_ENTRY_VALID_MASK) {
+ addr = val & MMU_UP_PAGE_ERROR_CAPTURE_VA_49_32_MASK;
+ addr <<= 32;
+ addr |= RREG32(mmMMU_UP_PAGE_ERROR_CAPTURE_VA);
+
+ dev_err_ratelimited(hdev->dev, "MMU page fault on va 0x%llx\n",
+ addr);
+
+ WREG32(mmMMU_UP_PAGE_ERROR_CAPTURE, 0);
+ }
+
+ val = RREG32(mmMMU_UP_ACCESS_ERROR_CAPTURE);
+ if (val & MMU_UP_ACCESS_ERROR_CAPTURE_ENTRY_VALID_MASK) {
+ addr = val & MMU_UP_ACCESS_ERROR_CAPTURE_VA_49_32_MASK;
+ addr <<= 32;
+ addr |= RREG32(mmMMU_UP_ACCESS_ERROR_CAPTURE_VA);
+
+ dev_err_ratelimited(hdev->dev,
+ "MMU access error on va 0x%llx\n", addr);
+
+ WREG32(mmMMU_UP_ACCESS_ERROR_CAPTURE, 0);
+ }
+}
+
+/*
+ * +-------------------+------------------------------------------------------+
+ * | Configuration Reg | Description |
+ * | Address | |
+ * +-------------------+------------------------------------------------------+
+ * | 0xF30 - 0xF3F |ECC single error indication (1 bit per memory wrapper)|
+ * | |0xF30 memory wrappers 31:0 (MSB to LSB) |
+ * | |0xF34 memory wrappers 63:32 |
+ * | |0xF38 memory wrappers 95:64 |
+ * | |0xF3C memory wrappers 127:96 |
+ * +-------------------+------------------------------------------------------+
+ * | 0xF40 - 0xF4F |ECC double error indication (1 bit per memory wrapper)|
+ * | |0xF40 memory wrappers 31:0 (MSB to LSB) |
+ * | |0xF44 memory wrappers 63:32 |
+ * | |0xF48 memory wrappers 95:64 |
+ * | |0xF4C memory wrappers 127:96 |
+ * +-------------------+------------------------------------------------------+
+ */
+static void gaudi_print_ecc_info_generic(struct hl_device *hdev,
+ const char *block_name,
+ u64 block_address, int num_memories,
+ bool derr, bool disable_clock_gating)
+{
+ struct gaudi_device *gaudi = hdev->asic_specific;
+ int num_mem_regs = num_memories / 32 + ((num_memories % 32) ? 1 : 0);
+
+ if (block_address >= CFG_BASE)
+ block_address -= CFG_BASE;
+
+ if (derr)
+ block_address += GAUDI_ECC_DERR0_OFFSET;
+ else
+ block_address += GAUDI_ECC_SERR0_OFFSET;
+
+ if (disable_clock_gating) {
+ mutex_lock(&gaudi->clk_gate_mutex);
+ hdev->asic_funcs->disable_clock_gating(hdev);
+ }
+
+ switch (num_mem_regs) {
+ case 1:
+ dev_err(hdev->dev,
+ "%s ECC indication: 0x%08x\n",
+ block_name, RREG32(block_address));
+ break;
+ case 2:
+ dev_err(hdev->dev,
+ "%s ECC indication: 0x%08x 0x%08x\n",
+ block_name,
+ RREG32(block_address), RREG32(block_address + 4));
+ break;
+ case 3:
+ dev_err(hdev->dev,
+ "%s ECC indication: 0x%08x 0x%08x 0x%08x\n",
+ block_name,
+ RREG32(block_address), RREG32(block_address + 4),
+ RREG32(block_address + 8));
+ break;
+ case 4:
+ dev_err(hdev->dev,
+ "%s ECC indication: 0x%08x 0x%08x 0x%08x 0x%08x\n",
+ block_name,
+ RREG32(block_address), RREG32(block_address + 4),
+ RREG32(block_address + 8), RREG32(block_address + 0xc));
+ break;
+ default:
+ break;
+
+ }
+
+ if (disable_clock_gating) {
+ hdev->asic_funcs->enable_clock_gating(hdev);
+ mutex_unlock(&gaudi->clk_gate_mutex);
+ }
+}
+
+static void gaudi_handle_qman_err_generic(struct hl_device *hdev,
+ const char *qm_name,
+ u64 glbl_sts_addr,
+ u64 arb_err_addr)
+{
+ u32 i, j, glbl_sts_val, arb_err_val, glbl_sts_clr_val;
+ char reg_desc[32];
+
+ /* Iterate through all stream GLBL_STS1 registers + Lower CP */
+ for (i = 0 ; i < QMAN_STREAMS + 1 ; i++) {
+ glbl_sts_clr_val = 0;
+ glbl_sts_val = RREG32(glbl_sts_addr + 4 * i);
+
+ if (!glbl_sts_val)
+ continue;
+
+ if (i == QMAN_STREAMS)
+ snprintf(reg_desc, ARRAY_SIZE(reg_desc), "LowerCP");
+ else
+ snprintf(reg_desc, ARRAY_SIZE(reg_desc), "stream%u", i);
+
+ for (j = 0 ; j < GAUDI_NUM_OF_QM_ERR_CAUSE ; j++) {
+ if (glbl_sts_val & BIT(j)) {
+ dev_err_ratelimited(hdev->dev,
+ "%s %s. err cause: %s\n",
+ qm_name, reg_desc,
+ gaudi_qman_error_cause[j]);
+ glbl_sts_clr_val |= BIT(j);
+ }
+ }
+
+ /* Write 1 clear errors */
+ WREG32(glbl_sts_addr + 4 * i, glbl_sts_clr_val);
+ }
+
+ arb_err_val = RREG32(arb_err_addr);
+
+ if (!arb_err_val)
+ return;
+
+ for (j = 0 ; j < GAUDI_NUM_OF_QM_ARB_ERR_CAUSE ; j++) {
+ if (arb_err_val & BIT(j)) {
+ dev_err_ratelimited(hdev->dev,
+ "%s ARB_ERR. err cause: %s\n",
+ qm_name,
+ gaudi_qman_arb_error_cause[j]);
+ }
+ }
+}
+
+static void gaudi_print_ecc_info(struct hl_device *hdev, u16 event_type)
+{
+ u64 block_address;
+ u8 index;
+ int num_memories;
+ char desc[32];
+ bool derr;
+ bool disable_clock_gating;
+
+ switch (event_type) {
+ case GAUDI_EVENT_PCIE_CORE_SERR:
+ snprintf(desc, ARRAY_SIZE(desc), "%s", "PCIE_CORE");
+ block_address = mmPCIE_CORE_BASE;
+ num_memories = 51;
+ derr = false;
+ disable_clock_gating = false;
+ break;
+ case GAUDI_EVENT_PCIE_CORE_DERR:
+ snprintf(desc, ARRAY_SIZE(desc), "%s", "PCIE_CORE");
+ block_address = mmPCIE_CORE_BASE;
+ num_memories = 51;
+ derr = true;
+ disable_clock_gating = false;
+ break;
+ case GAUDI_EVENT_PCIE_IF_SERR:
+ snprintf(desc, ARRAY_SIZE(desc), "%s", "PCIE_WRAP");
+ block_address = mmPCIE_WRAP_BASE;
+ num_memories = 11;
+ derr = false;
+ disable_clock_gating = false;
+ break;
+ case GAUDI_EVENT_PCIE_IF_DERR:
+ snprintf(desc, ARRAY_SIZE(desc), "%s", "PCIE_WRAP");
+ block_address = mmPCIE_WRAP_BASE;
+ num_memories = 11;
+ derr = true;
+ disable_clock_gating = false;
+ break;
+ case GAUDI_EVENT_PCIE_PHY_SERR:
+ snprintf(desc, ARRAY_SIZE(desc), "%s", "PCIE_PHY");
+ block_address = mmPCIE_PHY_BASE;
+ num_memories = 4;
+ derr = false;
+ disable_clock_gating = false;
+ break;
+ case GAUDI_EVENT_PCIE_PHY_DERR:
+ snprintf(desc, ARRAY_SIZE(desc), "%s", "PCIE_PHY");
+ block_address = mmPCIE_PHY_BASE;
+ num_memories = 4;
+ derr = true;
+ disable_clock_gating = false;
+ break;
+ case GAUDI_EVENT_TPC0_SERR ... GAUDI_EVENT_TPC7_SERR:
+ index = event_type - GAUDI_EVENT_TPC0_SERR;
+ block_address = mmTPC0_CFG_BASE + index * TPC_CFG_OFFSET;
+ snprintf(desc, ARRAY_SIZE(desc), "%s%d", "TPC", index);
+ num_memories = 90;
+ derr = false;
+ disable_clock_gating = true;
+ break;
+ case GAUDI_EVENT_TPC0_DERR ... GAUDI_EVENT_TPC7_DERR:
+ index = event_type - GAUDI_EVENT_TPC0_DERR;
+ block_address =
+ mmTPC0_CFG_BASE + index * TPC_CFG_OFFSET;
+ snprintf(desc, ARRAY_SIZE(desc), "%s%d", "TPC", index);
+ num_memories = 90;
+ derr = true;
+ disable_clock_gating = true;
+ break;
+ case GAUDI_EVENT_MME0_ACC_SERR:
+ case GAUDI_EVENT_MME1_ACC_SERR:
+ case GAUDI_EVENT_MME2_ACC_SERR:
+ case GAUDI_EVENT_MME3_ACC_SERR:
+ index = (event_type - GAUDI_EVENT_MME0_ACC_SERR) / 4;
+ block_address = mmMME0_ACC_BASE + index * MME_ACC_OFFSET;
+ snprintf(desc, ARRAY_SIZE(desc), "MME%d_ACC", index);
+ num_memories = 128;
+ derr = false;
+ disable_clock_gating = true;
+ break;
+ case GAUDI_EVENT_MME0_ACC_DERR:
+ case GAUDI_EVENT_MME1_ACC_DERR:
+ case GAUDI_EVENT_MME2_ACC_DERR:
+ case GAUDI_EVENT_MME3_ACC_DERR:
+ index = (event_type - GAUDI_EVENT_MME0_ACC_DERR) / 4;
+ block_address = mmMME0_ACC_BASE + index * MME_ACC_OFFSET;
+ snprintf(desc, ARRAY_SIZE(desc), "MME%d_ACC", index);
+ num_memories = 128;
+ derr = true;
+ disable_clock_gating = true;
+ break;
+ case GAUDI_EVENT_MME0_SBAB_SERR:
+ case GAUDI_EVENT_MME1_SBAB_SERR:
+ case GAUDI_EVENT_MME2_SBAB_SERR:
+ case GAUDI_EVENT_MME3_SBAB_SERR:
+ index = (event_type - GAUDI_EVENT_MME0_SBAB_SERR) / 4;
+ block_address = mmMME0_SBAB_BASE + index * MME_ACC_OFFSET;
+ snprintf(desc, ARRAY_SIZE(desc), "MME%d_SBAB", index);
+ num_memories = 33;
+ derr = false;
+ disable_clock_gating = true;
+ break;
+ case GAUDI_EVENT_MME0_SBAB_DERR:
+ case GAUDI_EVENT_MME1_SBAB_DERR:
+ case GAUDI_EVENT_MME2_SBAB_DERR:
+ case GAUDI_EVENT_MME3_SBAB_DERR:
+ index = (event_type - GAUDI_EVENT_MME0_SBAB_DERR) / 4;
+ block_address = mmMME0_SBAB_BASE + index * MME_ACC_OFFSET;
+ snprintf(desc, ARRAY_SIZE(desc), "MME%d_SBAB", index);
+ num_memories = 33;
+ derr = true;
+ disable_clock_gating = true;
+ break;
+ case GAUDI_EVENT_DMA0_SERR_ECC ... GAUDI_EVENT_DMA7_SERR_ECC:
+ index = event_type - GAUDI_EVENT_DMA0_SERR_ECC;
+ block_address = mmDMA0_CORE_BASE + index * DMA_CORE_OFFSET;
+ snprintf(desc, ARRAY_SIZE(desc), "DMA%d_CORE", index);
+ num_memories = 16;
+ derr = false;
+ disable_clock_gating = false;
+ break;
+ case GAUDI_EVENT_DMA0_DERR_ECC ... GAUDI_EVENT_DMA7_DERR_ECC:
+ index = event_type - GAUDI_EVENT_DMA0_DERR_ECC;
+ block_address = mmDMA0_CORE_BASE + index * DMA_CORE_OFFSET;
+ snprintf(desc, ARRAY_SIZE(desc), "DMA%d_CORE", index);
+ num_memories = 16;
+ derr = true;
+ disable_clock_gating = false;
+ break;
+ case GAUDI_EVENT_CPU_IF_ECC_SERR:
+ block_address = mmCPU_IF_BASE;
+ snprintf(desc, ARRAY_SIZE(desc), "%s", "CPU");
+ num_memories = 4;
+ derr = false;
+ disable_clock_gating = false;
+ break;
+ case GAUDI_EVENT_CPU_IF_ECC_DERR:
+ block_address = mmCPU_IF_BASE;
+ snprintf(desc, ARRAY_SIZE(desc), "%s", "CPU");
+ num_memories = 4;
+ derr = true;
+ disable_clock_gating = false;
+ break;
+ case GAUDI_EVENT_PSOC_MEM_SERR:
+ block_address = mmPSOC_GLOBAL_CONF_BASE;
+ snprintf(desc, ARRAY_SIZE(desc), "%s", "CPU");
+ num_memories = 4;
+ derr = false;
+ disable_clock_gating = false;
+ break;
+ case GAUDI_EVENT_PSOC_MEM_DERR:
+ block_address = mmPSOC_GLOBAL_CONF_BASE;
+ snprintf(desc, ARRAY_SIZE(desc), "%s", "CPU");
+ num_memories = 4;
+ derr = true;
+ disable_clock_gating = false;
+ break;
+ case GAUDI_EVENT_PSOC_CORESIGHT_SERR:
+ block_address = mmPSOC_CS_TRACE_BASE;
+ snprintf(desc, ARRAY_SIZE(desc), "%s", "CPU");
+ num_memories = 2;
+ derr = false;
+ disable_clock_gating = false;
+ break;
+ case GAUDI_EVENT_PSOC_CORESIGHT_DERR:
+ block_address = mmPSOC_CS_TRACE_BASE;
+ snprintf(desc, ARRAY_SIZE(desc), "%s", "CPU");
+ num_memories = 2;
+ derr = true;
+ disable_clock_gating = false;
+ break;
+ case GAUDI_EVENT_SRAM0_SERR ... GAUDI_EVENT_SRAM28_SERR:
+ index = event_type - GAUDI_EVENT_SRAM0_SERR;
+ block_address =
+ mmSRAM_Y0_X0_BANK_BASE + index * SRAM_BANK_OFFSET;
+ snprintf(desc, ARRAY_SIZE(desc), "SRAM%d", index);
+ num_memories = 2;
+ derr = false;
+ disable_clock_gating = false;
+ break;
+ case GAUDI_EVENT_SRAM0_DERR ... GAUDI_EVENT_SRAM28_DERR:
+ index = event_type - GAUDI_EVENT_SRAM0_DERR;
+ block_address =
+ mmSRAM_Y0_X0_BANK_BASE + index * SRAM_BANK_OFFSET;
+ snprintf(desc, ARRAY_SIZE(desc), "SRAM%d", index);
+ num_memories = 2;
+ derr = true;
+ disable_clock_gating = false;
+ break;
+ case GAUDI_EVENT_DMA_IF0_SERR ... GAUDI_EVENT_DMA_IF3_SERR:
+ index = event_type - GAUDI_EVENT_DMA_IF0_SERR;
+ block_address = mmDMA_IF_W_S_BASE +
+ index * (mmDMA_IF_E_S_BASE - mmDMA_IF_W_S_BASE);
+ snprintf(desc, ARRAY_SIZE(desc), "DMA_IF%d", index);
+ num_memories = 60;
+ derr = false;
+ disable_clock_gating = false;
+ break;
+ case GAUDI_EVENT_DMA_IF0_DERR ... GAUDI_EVENT_DMA_IF3_DERR:
+ index = event_type - GAUDI_EVENT_DMA_IF0_DERR;
+ block_address = mmDMA_IF_W_S_BASE +
+ index * (mmDMA_IF_E_S_BASE - mmDMA_IF_W_S_BASE);
+ snprintf(desc, ARRAY_SIZE(desc), "DMA_IF%d", index);
+ derr = true;
+ num_memories = 60;
+ disable_clock_gating = false;
+ break;
+ case GAUDI_EVENT_HBM_0_SERR ... GAUDI_EVENT_HBM_3_SERR:
+ index = event_type - GAUDI_EVENT_HBM_0_SERR;
+ /* HBM Registers are at different offsets */
+ block_address = mmHBM0_BASE + 0x8000 +
+ index * (mmHBM1_BASE - mmHBM0_BASE);
+ snprintf(desc, ARRAY_SIZE(desc), "HBM%d", index);
+ derr = false;
+ num_memories = 64;
+ disable_clock_gating = false;
+ break;
+ case GAUDI_EVENT_HBM_0_DERR ... GAUDI_EVENT_HBM_3_DERR:
+ index = event_type - GAUDI_EVENT_HBM_0_SERR;
+ /* HBM Registers are at different offsets */
+ block_address = mmHBM0_BASE + 0x8000 +
+ index * (mmHBM1_BASE - mmHBM0_BASE);
+ snprintf(desc, ARRAY_SIZE(desc), "HBM%d", index);
+ derr = true;
+ num_memories = 64;
+ disable_clock_gating = false;
+ break;
+ default:
+ return;
+ }
+
+ gaudi_print_ecc_info_generic(hdev, desc, block_address, num_memories,
+ derr, disable_clock_gating);
+}
+
+static void gaudi_handle_qman_err(struct hl_device *hdev, u16 event_type)
+{
+ u64 glbl_sts_addr, arb_err_addr;
+ u8 index;
+ char desc[32];
+
+ switch (event_type) {
+ case GAUDI_EVENT_TPC0_QM ... GAUDI_EVENT_TPC7_QM:
+ index = event_type - GAUDI_EVENT_TPC0_QM;
+ glbl_sts_addr =
+ mmTPC0_QM_GLBL_STS1_0 + index * TPC_QMAN_OFFSET;
+ arb_err_addr =
+ mmTPC0_QM_ARB_ERR_CAUSE + index * TPC_QMAN_OFFSET;
+ snprintf(desc, ARRAY_SIZE(desc), "%s%d", "TPC_QM", index);
+ break;
+ case GAUDI_EVENT_MME0_QM ... GAUDI_EVENT_MME2_QM:
+ index = event_type - GAUDI_EVENT_MME0_QM;
+ glbl_sts_addr =
+ mmMME0_QM_GLBL_STS1_0 + index * MME_QMAN_OFFSET;
+ arb_err_addr =
+ mmMME0_QM_ARB_ERR_CAUSE + index * MME_QMAN_OFFSET;
+ snprintf(desc, ARRAY_SIZE(desc), "%s%d", "MME_QM", index);
+ break;
+ case GAUDI_EVENT_DMA0_QM ... GAUDI_EVENT_DMA7_QM:
+ index = event_type - GAUDI_EVENT_DMA0_QM;
+ glbl_sts_addr =
+ mmDMA0_QM_GLBL_STS1_0 + index * DMA_QMAN_OFFSET;
+ arb_err_addr =
+ mmDMA0_QM_ARB_ERR_CAUSE + index * DMA_QMAN_OFFSET;
+ snprintf(desc, ARRAY_SIZE(desc), "%s%d", "DMA_QM", index);
+ break;
+ default:
+ return;
+ }
+
+ gaudi_handle_qman_err_generic(hdev, desc, glbl_sts_addr, arb_err_addr);
+}
+
+static void gaudi_print_irq_info(struct hl_device *hdev, u16 event_type,
+ bool razwi)
+{
+ char desc[64] = "";
+
+ gaudi_get_event_desc(event_type, desc, sizeof(desc));
+ dev_err_ratelimited(hdev->dev, "Received H/W interrupt %d [\"%s\"]\n",
+ event_type, desc);
+
+ gaudi_print_ecc_info(hdev, event_type);
+
+ if (razwi) {
+ gaudi_print_razwi_info(hdev);
+ gaudi_print_mmu_error_info(hdev);
+ }
+}
+
+static int gaudi_soft_reset_late_init(struct hl_device *hdev)
+{
+ struct gaudi_device *gaudi = hdev->asic_specific;
+
+ /* Unmask all IRQs since some could have been received
+ * during the soft reset
+ */
+ return hl_fw_unmask_irq_arr(hdev, gaudi->events, sizeof(gaudi->events));
+}
+
+static int gaudi_hbm_read_interrupts(struct hl_device *hdev, int device)
+{
+ int ch, err = 0;
+ u32 base, val, val2;
+
+ base = GAUDI_HBM_CFG_BASE + device * GAUDI_HBM_CFG_OFFSET;
+ for (ch = 0 ; ch < GAUDI_HBM_CHANNELS ; ch++) {
+ val = RREG32_MASK(base + ch * 0x1000 + 0x06C, 0x0000FFFF);
+ val = (val & 0xFF) | ((val >> 8) & 0xFF);
+ if (val) {
+ err = 1;
+ dev_err(hdev->dev,
+ "HBM%d pc%d interrupts info: WR_PAR=%d, RD_PAR=%d, CA_PAR=%d, SERR=%d, DERR=%d\n",
+ device, ch * 2, val & 0x1, (val >> 1) & 0x1,
+ (val >> 2) & 0x1, (val >> 3) & 0x1,
+ (val >> 4) & 0x1);
+
+ val2 = RREG32(base + ch * 0x1000 + 0x060);
+ dev_err(hdev->dev,
+ "HBM%d pc%d ECC info: 1ST_ERR_ADDR=0x%x, 1ST_ERR_TYPE=%d, SEC_CONT_CNT=%d, SEC_CNT=%d, DED_CNT=%d\n",
+ device, ch * 2,
+ RREG32(base + ch * 0x1000 + 0x064),
+ (val2 & 0x200) >> 9, (val2 & 0xFC00) >> 10,
+ (val2 & 0xFF0000) >> 16,
+ (val2 & 0xFF000000) >> 24);
+ }
+
+ val = RREG32_MASK(base + ch * 0x1000 + 0x07C, 0x0000FFFF);
+ val = (val & 0xFF) | ((val >> 8) & 0xFF);
+ if (val) {
+ err = 1;
+ dev_err(hdev->dev,
+ "HBM%d pc%d interrupts info: WR_PAR=%d, RD_PAR=%d, CA_PAR=%d, SERR=%d, DERR=%d\n",
+ device, ch * 2 + 1, val & 0x1, (val >> 1) & 0x1,
+ (val >> 2) & 0x1, (val >> 3) & 0x1,
+ (val >> 4) & 0x1);
+
+ val2 = RREG32(base + ch * 0x1000 + 0x070);
+ dev_err(hdev->dev,
+ "HBM%d pc%d ECC info: 1ST_ERR_ADDR=0x%x, 1ST_ERR_TYPE=%d, SEC_CONT_CNT=%d, SEC_CNT=%d, DED_CNT=%d\n",
+ device, ch * 2 + 1,
+ RREG32(base + ch * 0x1000 + 0x074),
+ (val2 & 0x200) >> 9, (val2 & 0xFC00) >> 10,
+ (val2 & 0xFF0000) >> 16,
+ (val2 & 0xFF000000) >> 24);
+ }
+
+ /* Clear interrupts */
+ RMWREG32(base + (ch * 0x1000) + 0x060, 0x1C8, 0x1FF);
+ RMWREG32(base + (ch * 0x1000) + 0x070, 0x1C8, 0x1FF);
+ WREG32(base + (ch * 0x1000) + 0x06C, 0x1F1F);
+ WREG32(base + (ch * 0x1000) + 0x07C, 0x1F1F);
+ RMWREG32(base + (ch * 0x1000) + 0x060, 0x0, 0xF);
+ RMWREG32(base + (ch * 0x1000) + 0x070, 0x0, 0xF);
+ }
+
+ val = RREG32(base + 0x8F30);
+ val2 = RREG32(base + 0x8F34);
+ if (val | val2) {
+ err = 1;
+ dev_err(hdev->dev,
+ "HBM %d MC SRAM SERR info: Reg 0x8F30=0x%x, Reg 0x8F34=0x%x\n",
+ device, val, val2);
+ }
+ val = RREG32(base + 0x8F40);
+ val2 = RREG32(base + 0x8F44);
+ if (val | val2) {
+ err = 1;
+ dev_err(hdev->dev,
+ "HBM %d MC SRAM DERR info: Reg 0x8F40=0x%x, Reg 0x8F44=0x%x\n",
+ device, val, val2);
+ }
+
+ return err;
+}
+
+static int gaudi_hbm_event_to_dev(u16 hbm_event_type)
+{
+ switch (hbm_event_type) {
+ case GAUDI_EVENT_HBM0_SPI_0:
+ case GAUDI_EVENT_HBM0_SPI_1:
+ return 0;
+ case GAUDI_EVENT_HBM1_SPI_0:
+ case GAUDI_EVENT_HBM1_SPI_1:
+ return 1;
+ case GAUDI_EVENT_HBM2_SPI_0:
+ case GAUDI_EVENT_HBM2_SPI_1:
+ return 2;
+ case GAUDI_EVENT_HBM3_SPI_0:
+ case GAUDI_EVENT_HBM3_SPI_1:
+ return 3;
+ default:
+ break;
+ }
+
+ /* Should never happen */
+ return 0;
+}
+
+static bool gaudi_tpc_read_interrupts(struct hl_device *hdev, u8 tpc_id,
+ char *interrupt_name)
+{
+ struct gaudi_device *gaudi = hdev->asic_specific;
+ u32 tpc_offset = tpc_id * TPC_CFG_OFFSET, tpc_interrupts_cause, i;
+ bool soft_reset_required = false;
+
+ /* Accessing the TPC_INTR_CAUSE registers requires disabling the clock
+ * gating, and thus cannot be done in ArmCP and should be done instead
+ * by the driver.
+ */
+
+ mutex_lock(&gaudi->clk_gate_mutex);
+
+ hdev->asic_funcs->disable_clock_gating(hdev);
+
+ tpc_interrupts_cause = RREG32(mmTPC0_CFG_TPC_INTR_CAUSE + tpc_offset) &
+ TPC0_CFG_TPC_INTR_CAUSE_CAUSE_MASK;
+
+ for (i = 0 ; i < GAUDI_NUM_OF_TPC_INTR_CAUSE ; i++)
+ if (tpc_interrupts_cause & BIT(i)) {
+ dev_err_ratelimited(hdev->dev,
+ "TPC%d_%s interrupt cause: %s\n",
+ tpc_id, interrupt_name,
+ gaudi_tpc_interrupts_cause[i]);
+ /* If this is QM error, we need to soft-reset */
+ if (i == 15)
+ soft_reset_required = true;
+ }
+
+ /* Clear interrupts */
+ WREG32(mmTPC0_CFG_TPC_INTR_CAUSE + tpc_offset, 0);
+
+ hdev->asic_funcs->enable_clock_gating(hdev);
+
+ mutex_unlock(&gaudi->clk_gate_mutex);
+
+ return soft_reset_required;
+}
+
+static int tpc_dec_event_to_tpc_id(u16 tpc_dec_event_type)
+{
+ return (tpc_dec_event_type - GAUDI_EVENT_TPC0_DEC) >> 1;
+}
+
+static int tpc_krn_event_to_tpc_id(u16 tpc_dec_event_type)
+{
+ return (tpc_dec_event_type - GAUDI_EVENT_TPC0_KRN_ERR) / 6;
+}
+
+static void gaudi_print_clk_change_info(struct hl_device *hdev,
+ u16 event_type)
+{
+ switch (event_type) {
+ case GAUDI_EVENT_FIX_POWER_ENV_S:
+ dev_info_ratelimited(hdev->dev,
+ "Clock throttling due to power consumption\n");
+ break;
+
+ case GAUDI_EVENT_FIX_POWER_ENV_E:
+ dev_info_ratelimited(hdev->dev,
+ "Power envelop is safe, back to optimal clock\n");
+ break;
+
+ case GAUDI_EVENT_FIX_THERMAL_ENV_S:
+ dev_info_ratelimited(hdev->dev,
+ "Clock throttling due to overheating\n");
+ break;
+
+ case GAUDI_EVENT_FIX_THERMAL_ENV_E:
+ dev_info_ratelimited(hdev->dev,
+ "Thermal envelop is safe, back to optimal clock\n");
+ break;
+
+ default:
+ dev_err(hdev->dev, "Received invalid clock change event %d\n",
+ event_type);
+ break;
+ }
+}
+
+static void gaudi_handle_eqe(struct hl_device *hdev,
+ struct hl_eq_entry *eq_entry)
+{
+ struct gaudi_device *gaudi = hdev->asic_specific;
+ u32 ctl = le32_to_cpu(eq_entry->hdr.ctl);
+ u16 event_type = ((ctl & EQ_CTL_EVENT_TYPE_MASK)
+ >> EQ_CTL_EVENT_TYPE_SHIFT);
+ u8 cause;
+ bool reset_required;
+
+ gaudi->events_stat[event_type]++;
+ gaudi->events_stat_aggregate[event_type]++;
+
+ switch (event_type) {
+ case GAUDI_EVENT_PCIE_CORE_DERR:
+ case GAUDI_EVENT_PCIE_IF_DERR:
+ case GAUDI_EVENT_PCIE_PHY_DERR:
+ case GAUDI_EVENT_TPC0_DERR ... GAUDI_EVENT_TPC7_DERR:
+ case GAUDI_EVENT_MME0_ACC_DERR:
+ case GAUDI_EVENT_MME0_SBAB_DERR:
+ case GAUDI_EVENT_MME1_ACC_DERR:
+ case GAUDI_EVENT_MME1_SBAB_DERR:
+ case GAUDI_EVENT_MME2_ACC_DERR:
+ case GAUDI_EVENT_MME2_SBAB_DERR:
+ case GAUDI_EVENT_MME3_ACC_DERR:
+ case GAUDI_EVENT_MME3_SBAB_DERR:
+ case GAUDI_EVENT_DMA0_DERR_ECC ... GAUDI_EVENT_DMA7_DERR_ECC:
+ fallthrough;
+ case GAUDI_EVENT_CPU_IF_ECC_DERR:
+ case GAUDI_EVENT_PSOC_MEM_DERR:
+ case GAUDI_EVENT_PSOC_CORESIGHT_DERR:
+ case GAUDI_EVENT_SRAM0_DERR ... GAUDI_EVENT_SRAM28_DERR:
+ case GAUDI_EVENT_DMA_IF0_DERR ... GAUDI_EVENT_DMA_IF3_DERR:
+ fallthrough;
+ case GAUDI_EVENT_GIC500:
+ case GAUDI_EVENT_HBM_0_DERR ... GAUDI_EVENT_HBM_3_DERR:
+ case GAUDI_EVENT_MMU_DERR:
+ case GAUDI_EVENT_AXI_ECC:
+ case GAUDI_EVENT_L2_RAM_ECC:
+ case GAUDI_EVENT_PLL0 ... GAUDI_EVENT_PLL17:
+ gaudi_print_irq_info(hdev, event_type, false);
+ if (hdev->hard_reset_on_fw_events)
+ hl_device_reset(hdev, true, false);
+ break;
+
+ case GAUDI_EVENT_HBM0_SPI_0:
+ case GAUDI_EVENT_HBM1_SPI_0:
+ case GAUDI_EVENT_HBM2_SPI_0:
+ case GAUDI_EVENT_HBM3_SPI_0:
+ gaudi_print_irq_info(hdev, event_type, false);
+ gaudi_hbm_read_interrupts(hdev,
+ gaudi_hbm_event_to_dev(event_type));
+ if (hdev->hard_reset_on_fw_events)
+ hl_device_reset(hdev, true, false);
+ break;
+
+ case GAUDI_EVENT_HBM0_SPI_1:
+ case GAUDI_EVENT_HBM1_SPI_1:
+ case GAUDI_EVENT_HBM2_SPI_1:
+ case GAUDI_EVENT_HBM3_SPI_1:
+ gaudi_print_irq_info(hdev, event_type, false);
+ gaudi_hbm_read_interrupts(hdev,
+ gaudi_hbm_event_to_dev(event_type));
+ break;
+
+ case GAUDI_EVENT_TPC0_DEC:
+ case GAUDI_EVENT_TPC1_DEC:
+ case GAUDI_EVENT_TPC2_DEC:
+ case GAUDI_EVENT_TPC3_DEC:
+ case GAUDI_EVENT_TPC4_DEC:
+ case GAUDI_EVENT_TPC5_DEC:
+ case GAUDI_EVENT_TPC6_DEC:
+ case GAUDI_EVENT_TPC7_DEC:
+ gaudi_print_irq_info(hdev, event_type, true);
+ reset_required = gaudi_tpc_read_interrupts(hdev,
+ tpc_dec_event_to_tpc_id(event_type),
+ "AXI_SLV_DEC_Error");
+ if (reset_required) {
+ dev_err(hdev->dev, "hard reset required due to %s\n",
+ gaudi_irq_map_table[event_type].name);
+
+ if (hdev->hard_reset_on_fw_events)
+ hl_device_reset(hdev, true, false);
+ } else {
+ hl_fw_unmask_irq(hdev, event_type);
+ }
+ break;
+
+ case GAUDI_EVENT_TPC0_KRN_ERR:
+ case GAUDI_EVENT_TPC1_KRN_ERR:
+ case GAUDI_EVENT_TPC2_KRN_ERR:
+ case GAUDI_EVENT_TPC3_KRN_ERR:
+ case GAUDI_EVENT_TPC4_KRN_ERR:
+ case GAUDI_EVENT_TPC5_KRN_ERR:
+ case GAUDI_EVENT_TPC6_KRN_ERR:
+ case GAUDI_EVENT_TPC7_KRN_ERR:
+ gaudi_print_irq_info(hdev, event_type, true);
+ reset_required = gaudi_tpc_read_interrupts(hdev,
+ tpc_krn_event_to_tpc_id(event_type),
+ "KRN_ERR");
+ if (reset_required) {
+ dev_err(hdev->dev, "hard reset required due to %s\n",
+ gaudi_irq_map_table[event_type].name);
+
+ if (hdev->hard_reset_on_fw_events)
+ hl_device_reset(hdev, true, false);
+ } else {
+ hl_fw_unmask_irq(hdev, event_type);
+ }
+ break;
+
+ case GAUDI_EVENT_PCIE_CORE_SERR:
+ case GAUDI_EVENT_PCIE_IF_SERR:
+ case GAUDI_EVENT_PCIE_PHY_SERR:
+ case GAUDI_EVENT_TPC0_SERR ... GAUDI_EVENT_TPC7_SERR:
+ case GAUDI_EVENT_MME0_ACC_SERR:
+ case GAUDI_EVENT_MME0_SBAB_SERR:
+ case GAUDI_EVENT_MME1_ACC_SERR:
+ case GAUDI_EVENT_MME1_SBAB_SERR:
+ case GAUDI_EVENT_MME2_ACC_SERR:
+ case GAUDI_EVENT_MME2_SBAB_SERR:
+ case GAUDI_EVENT_MME3_ACC_SERR:
+ case GAUDI_EVENT_MME3_SBAB_SERR:
+ case GAUDI_EVENT_DMA0_SERR_ECC ... GAUDI_EVENT_DMA7_SERR_ECC:
+ case GAUDI_EVENT_CPU_IF_ECC_SERR:
+ case GAUDI_EVENT_PSOC_MEM_SERR:
+ case GAUDI_EVENT_PSOC_CORESIGHT_SERR:
+ case GAUDI_EVENT_SRAM0_SERR ... GAUDI_EVENT_SRAM28_SERR:
+ case GAUDI_EVENT_DMA_IF0_SERR ... GAUDI_EVENT_DMA_IF3_SERR:
+ case GAUDI_EVENT_HBM_0_SERR ... GAUDI_EVENT_HBM_3_SERR:
+ fallthrough;
+ case GAUDI_EVENT_MMU_SERR:
+ case GAUDI_EVENT_PCIE_DEC:
+ case GAUDI_EVENT_MME0_WBC_RSP:
+ case GAUDI_EVENT_MME0_SBAB0_RSP:
+ case GAUDI_EVENT_MME1_WBC_RSP:
+ case GAUDI_EVENT_MME1_SBAB0_RSP:
+ case GAUDI_EVENT_MME2_WBC_RSP:
+ case GAUDI_EVENT_MME2_SBAB0_RSP:
+ case GAUDI_EVENT_MME3_WBC_RSP:
+ case GAUDI_EVENT_MME3_SBAB0_RSP:
+ case GAUDI_EVENT_CPU_AXI_SPLITTER:
+ case GAUDI_EVENT_PSOC_AXI_DEC:
+ case GAUDI_EVENT_PSOC_PRSTN_FALL:
+ case GAUDI_EVENT_MMU_PAGE_FAULT:
+ case GAUDI_EVENT_MMU_WR_PERM:
+ case GAUDI_EVENT_RAZWI_OR_ADC:
+ case GAUDI_EVENT_TPC0_QM ... GAUDI_EVENT_TPC7_QM:
+ case GAUDI_EVENT_MME0_QM ... GAUDI_EVENT_MME2_QM:
+ case GAUDI_EVENT_DMA0_QM ... GAUDI_EVENT_DMA7_QM:
+ fallthrough;
+ case GAUDI_EVENT_DMA0_CORE ... GAUDI_EVENT_DMA7_CORE:
+ gaudi_print_irq_info(hdev, event_type, true);
+ gaudi_handle_qman_err(hdev, event_type);
+ hl_fw_unmask_irq(hdev, event_type);
+ break;
+
+ case GAUDI_EVENT_RAZWI_OR_ADC_SW:
+ gaudi_print_irq_info(hdev, event_type, true);
+ if (hdev->hard_reset_on_fw_events)
+ hl_device_reset(hdev, true, false);
+ break;
+
+ case GAUDI_EVENT_TPC0_BMON_SPMU:
+ case GAUDI_EVENT_TPC1_BMON_SPMU:
+ case GAUDI_EVENT_TPC2_BMON_SPMU:
+ case GAUDI_EVENT_TPC3_BMON_SPMU:
+ case GAUDI_EVENT_TPC4_BMON_SPMU:
+ case GAUDI_EVENT_TPC5_BMON_SPMU:
+ case GAUDI_EVENT_TPC6_BMON_SPMU:
+ case GAUDI_EVENT_TPC7_BMON_SPMU:
+ case GAUDI_EVENT_DMA_BM_CH0 ... GAUDI_EVENT_DMA_BM_CH7:
+ gaudi_print_irq_info(hdev, event_type, false);
+ hl_fw_unmask_irq(hdev, event_type);
+ break;
+
+ case GAUDI_EVENT_FIX_POWER_ENV_S ... GAUDI_EVENT_FIX_THERMAL_ENV_E:
+ gaudi_print_clk_change_info(hdev, event_type);
+ hl_fw_unmask_irq(hdev, event_type);
+ break;
+
+ case GAUDI_EVENT_PSOC_GPIO_U16_0:
+ cause = le64_to_cpu(eq_entry->data[0]) & 0xFF;
+ dev_err(hdev->dev,
+ "Received high temp H/W interrupt %d (cause %d)\n",
+ event_type, cause);
+ break;
+
+ default:
+ dev_err(hdev->dev, "Received invalid H/W interrupt %d\n",
+ event_type);
+ break;
+ }
+}
+
+static void *gaudi_get_events_stat(struct hl_device *hdev, bool aggregate,
+ u32 *size)
+{
+ struct gaudi_device *gaudi = hdev->asic_specific;
+
+ if (aggregate) {
+ *size = (u32) sizeof(gaudi->events_stat_aggregate);
+ return gaudi->events_stat_aggregate;
+ }
+
+ *size = (u32) sizeof(gaudi->events_stat);
+ return gaudi->events_stat;
+}
+
+static int gaudi_mmu_invalidate_cache(struct hl_device *hdev, bool is_hard,
+ u32 flags)
+{
+ struct gaudi_device *gaudi = hdev->asic_specific;
+ u32 status, timeout_usec;
+ int rc;
+
+ if (!(gaudi->hw_cap_initialized & HW_CAP_MMU) ||
+ hdev->hard_reset_pending)
+ return 0;
+
+ if (hdev->pldm)
+ timeout_usec = GAUDI_PLDM_MMU_TIMEOUT_USEC;
+ else
+ timeout_usec = MMU_CONFIG_TIMEOUT_USEC;
+
+ mutex_lock(&hdev->mmu_cache_lock);
+
+ /* L0 & L1 invalidation */
+ WREG32(mmSTLB_INV_PS, 2);
+
+ rc = hl_poll_timeout(
+ hdev,
+ mmSTLB_INV_PS,
+ status,
+ !status,
+ 1000,
+ timeout_usec);
+
+ WREG32(mmSTLB_INV_SET, 0);
+
+ mutex_unlock(&hdev->mmu_cache_lock);
+
+ if (rc) {
+ dev_err_ratelimited(hdev->dev,
+ "MMU cache invalidation timeout\n");
+ hl_device_reset(hdev, true, false);
+ }
+
+ return rc;
+}
+
+static int gaudi_mmu_invalidate_cache_range(struct hl_device *hdev,
+ bool is_hard, u32 asid, u64 va, u64 size)
+{
+ struct gaudi_device *gaudi = hdev->asic_specific;
+ u32 status, timeout_usec;
+ u32 inv_data;
+ u32 pi;
+ int rc;
+
+ if (!(gaudi->hw_cap_initialized & HW_CAP_MMU) ||
+ hdev->hard_reset_pending)
+ return 0;
+
+ mutex_lock(&hdev->mmu_cache_lock);
+
+ if (hdev->pldm)
+ timeout_usec = GAUDI_PLDM_MMU_TIMEOUT_USEC;
+ else
+ timeout_usec = MMU_CONFIG_TIMEOUT_USEC;
+
+ /*
+ * TODO: currently invalidate entire L0 & L1 as in regular hard
+ * invalidation. Need to apply invalidation of specific cache
+ * lines with mask of ASID & VA & size.
+ * Note that L1 with be flushed entirely in any case.
+ */
+
+ /* L0 & L1 invalidation */
+ inv_data = RREG32(mmSTLB_CACHE_INV);
+ /* PI is 8 bit */
+ pi = ((inv_data & STLB_CACHE_INV_PRODUCER_INDEX_MASK) + 1) & 0xFF;
+ WREG32(mmSTLB_CACHE_INV,
+ (inv_data & STLB_CACHE_INV_INDEX_MASK_MASK) | pi);
+
+ rc = hl_poll_timeout(
+ hdev,
+ mmSTLB_INV_CONSUMER_INDEX,
+ status,
+ status == pi,
+ 1000,
+ timeout_usec);
+
+ mutex_unlock(&hdev->mmu_cache_lock);
+
+ if (rc) {
+ dev_err_ratelimited(hdev->dev,
+ "MMU cache invalidation timeout\n");
+ hl_device_reset(hdev, true, false);
+ }
+
+ return rc;
+}
+
+static int gaudi_mmu_update_asid_hop0_addr(struct hl_device *hdev,
+ u32 asid, u64 phys_addr)
+{
+ u32 status, timeout_usec;
+ int rc;
+
+ if (hdev->pldm)
+ timeout_usec = GAUDI_PLDM_MMU_TIMEOUT_USEC;
+ else
+ timeout_usec = MMU_CONFIG_TIMEOUT_USEC;
+
+ WREG32(MMU_ASID, asid);
+ WREG32(MMU_HOP0_PA43_12, phys_addr >> MMU_HOP0_PA43_12_SHIFT);
+ WREG32(MMU_HOP0_PA49_44, phys_addr >> MMU_HOP0_PA49_44_SHIFT);
+ WREG32(MMU_BUSY, 0x80000000);
+
+ rc = hl_poll_timeout(
+ hdev,
+ MMU_BUSY,
+ status,
+ !(status & 0x80000000),
+ 1000,
+ timeout_usec);
+
+ if (rc) {
+ dev_err(hdev->dev,
+ "Timeout during MMU hop0 config of asid %d\n", asid);
+ return rc;
+ }
+
+ return 0;
+}
+
+static int gaudi_send_heartbeat(struct hl_device *hdev)
+{
+ struct gaudi_device *gaudi = hdev->asic_specific;
+
+ if (!(gaudi->hw_cap_initialized & HW_CAP_CPU_Q))
+ return 0;
+
+ return hl_fw_send_heartbeat(hdev);
+}
+
+static int gaudi_armcp_info_get(struct hl_device *hdev)
+{
+ struct gaudi_device *gaudi = hdev->asic_specific;
+ struct asic_fixed_properties *prop = &hdev->asic_prop;
+ int rc;
+
+ if (!(gaudi->hw_cap_initialized & HW_CAP_CPU_Q))
+ return 0;
+
+ rc = hl_fw_armcp_info_get(hdev);
+ if (rc)
+ return rc;
+
+ if (!strlen(prop->armcp_info.card_name))
+ strncpy(prop->armcp_info.card_name, GAUDI_DEFAULT_CARD_NAME,
+ CARD_NAME_MAX_LEN);
+
+ return 0;
+}
+
+static bool gaudi_is_device_idle(struct hl_device *hdev, u32 *mask,
+ struct seq_file *s)
+{
+ struct gaudi_device *gaudi = hdev->asic_specific;
+ const char *fmt = "%-5d%-9s%#-14x%#-12x%#x\n";
+ const char *mme_slave_fmt = "%-5d%-9s%-14s%-12s%#x\n";
+ u32 qm_glbl_sts0, qm_cgm_sts, dma_core_sts0, tpc_cfg_sts, mme_arch_sts;
+ bool is_idle = true, is_eng_idle, is_slave;
+ u64 offset;
+ int i, dma_id;
+
+ mutex_lock(&gaudi->clk_gate_mutex);
+
+ hdev->asic_funcs->disable_clock_gating(hdev);
+
+ if (s)
+ seq_puts(s,
+ "\nDMA is_idle QM_GLBL_STS0 QM_CGM_STS DMA_CORE_STS0\n"
+ "--- ------- ------------ ---------- -------------\n");
+
+ for (i = 0 ; i < DMA_NUMBER_OF_CHNLS ; i++) {
+ dma_id = gaudi_dma_assignment[i];
+ offset = dma_id * DMA_QMAN_OFFSET;
+
+ qm_glbl_sts0 = RREG32(mmDMA0_QM_GLBL_STS0 + offset);
+ qm_cgm_sts = RREG32(mmDMA0_QM_CGM_STS + offset);
+ dma_core_sts0 = RREG32(mmDMA0_CORE_STS0 + offset);
+ is_eng_idle = IS_QM_IDLE(qm_glbl_sts0, qm_cgm_sts) &&
+ IS_DMA_IDLE(dma_core_sts0);
+ is_idle &= is_eng_idle;
+
+ if (mask)
+ *mask |= !is_eng_idle <<
+ (GAUDI_ENGINE_ID_DMA_0 + dma_id);
+ if (s)
+ seq_printf(s, fmt, dma_id,
+ is_eng_idle ? "Y" : "N", qm_glbl_sts0,
+ qm_cgm_sts, dma_core_sts0);
+ }
+
+ if (s)
+ seq_puts(s,
+ "\nTPC is_idle QM_GLBL_STS0 QM_CGM_STS CFG_STATUS\n"
+ "--- ------- ------------ ---------- ----------\n");
+
+ for (i = 0 ; i < TPC_NUMBER_OF_ENGINES ; i++) {
+ offset = i * TPC_QMAN_OFFSET;
+ qm_glbl_sts0 = RREG32(mmTPC0_QM_GLBL_STS0 + offset);
+ qm_cgm_sts = RREG32(mmTPC0_QM_CGM_STS + offset);
+ tpc_cfg_sts = RREG32(mmTPC0_CFG_STATUS + offset);
+ is_eng_idle = IS_QM_IDLE(qm_glbl_sts0, qm_cgm_sts) &&
+ IS_TPC_IDLE(tpc_cfg_sts);
+ is_idle &= is_eng_idle;
+
+ if (mask)
+ *mask |= !is_eng_idle << (GAUDI_ENGINE_ID_TPC_0 + i);
+ if (s)
+ seq_printf(s, fmt, i,
+ is_eng_idle ? "Y" : "N",
+ qm_glbl_sts0, qm_cgm_sts, tpc_cfg_sts);
+ }
+
+ if (s)
+ seq_puts(s,
+ "\nMME is_idle QM_GLBL_STS0 QM_CGM_STS ARCH_STATUS\n"
+ "--- ------- ------------ ---------- -----------\n");
+
+ for (i = 0 ; i < MME_NUMBER_OF_ENGINES ; i++) {
+ offset = i * MME_QMAN_OFFSET;
+ mme_arch_sts = RREG32(mmMME0_CTRL_ARCH_STATUS + offset);
+ is_eng_idle = IS_MME_IDLE(mme_arch_sts);
+
+ /* MME 1 & 3 are slaves, no need to check their QMANs */
+ is_slave = i % 2;
+ if (!is_slave) {
+ qm_glbl_sts0 = RREG32(mmMME0_QM_GLBL_STS0 + offset);
+ qm_cgm_sts = RREG32(mmMME0_QM_CGM_STS + offset);
+ is_eng_idle &= IS_QM_IDLE(qm_glbl_sts0, qm_cgm_sts);
+ }
+
+ is_idle &= is_eng_idle;
+
+ if (mask)
+ *mask |= !is_eng_idle << (GAUDI_ENGINE_ID_MME_0 + i);
+ if (s) {
+ if (!is_slave)
+ seq_printf(s, fmt, i,
+ is_eng_idle ? "Y" : "N",
+ qm_glbl_sts0, qm_cgm_sts, mme_arch_sts);
+ else
+ seq_printf(s, mme_slave_fmt, i,
+ is_eng_idle ? "Y" : "N", "-",
+ "-", mme_arch_sts);
+ }
+ }
+
+ if (s)
+ seq_puts(s, "\n");
+
+ hdev->asic_funcs->enable_clock_gating(hdev);
+
+ mutex_unlock(&gaudi->clk_gate_mutex);
+
+ return is_idle;
+}
+
+static void gaudi_hw_queues_lock(struct hl_device *hdev)
+ __acquires(&gaudi->hw_queues_lock)
+{
+ struct gaudi_device *gaudi = hdev->asic_specific;
+
+ spin_lock(&gaudi->hw_queues_lock);
+}
+
+static void gaudi_hw_queues_unlock(struct hl_device *hdev)
+ __releases(&gaudi->hw_queues_lock)
+{
+ struct gaudi_device *gaudi = hdev->asic_specific;
+
+ spin_unlock(&gaudi->hw_queues_lock);
+}
+
+static u32 gaudi_get_pci_id(struct hl_device *hdev)
+{
+ return hdev->pdev->device;
+}
+
+static int gaudi_get_eeprom_data(struct hl_device *hdev, void *data,
+ size_t max_size)
+{
+ struct gaudi_device *gaudi = hdev->asic_specific;
+
+ if (!(gaudi->hw_cap_initialized & HW_CAP_CPU_Q))
+ return 0;
+
+ return hl_fw_get_eeprom_data(hdev, data, max_size);
+}
+
+/*
+ * this function should be used only during initialization and/or after reset,
+ * when there are no active users.
+ */
+static int gaudi_run_tpc_kernel(struct hl_device *hdev, u64 tpc_kernel,
+ u32 tpc_id)
+{
+ struct gaudi_device *gaudi = hdev->asic_specific;
+ u64 kernel_timeout;
+ u32 status, offset;
+ int rc;
+
+ offset = tpc_id * (mmTPC1_CFG_STATUS - mmTPC0_CFG_STATUS);
+
+ if (hdev->pldm)
+ kernel_timeout = GAUDI_PLDM_TPC_KERNEL_WAIT_USEC;
+ else
+ kernel_timeout = HL_DEVICE_TIMEOUT_USEC;
+
+ mutex_lock(&gaudi->clk_gate_mutex);
+
+ hdev->asic_funcs->disable_clock_gating(hdev);
+
+ WREG32(mmTPC0_CFG_QM_KERNEL_BASE_ADDRESS_LOW + offset,
+ lower_32_bits(tpc_kernel));
+ WREG32(mmTPC0_CFG_QM_KERNEL_BASE_ADDRESS_HIGH + offset,
+ upper_32_bits(tpc_kernel));
+
+ WREG32(mmTPC0_CFG_ICACHE_BASE_ADDERESS_LOW + offset,
+ lower_32_bits(tpc_kernel));
+ WREG32(mmTPC0_CFG_ICACHE_BASE_ADDERESS_HIGH + offset,
+ upper_32_bits(tpc_kernel));
+ /* set a valid LUT pointer, content is of no significance */
+ WREG32(mmTPC0_CFG_LUT_FUNC256_BASE_ADDR_LO + offset,
+ lower_32_bits(tpc_kernel));
+ WREG32(mmTPC0_CFG_LUT_FUNC256_BASE_ADDR_HI + offset,
+ upper_32_bits(tpc_kernel));
+
+ WREG32(mmTPC0_CFG_QM_SYNC_OBJECT_ADDR + offset,
+ lower_32_bits(CFG_BASE +
+ mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_SOB_OBJ_0));
+
+ WREG32(mmTPC0_CFG_TPC_CMD + offset,
+ (1 << TPC0_CFG_TPC_CMD_ICACHE_INVALIDATE_SHIFT |
+ 1 << TPC0_CFG_TPC_CMD_ICACHE_PREFETCH_64KB_SHIFT));
+ /* wait a bit for the engine to start executing */
+ usleep_range(1000, 1500);
+
+ /* wait until engine has finished executing */
+ rc = hl_poll_timeout(
+ hdev,
+ mmTPC0_CFG_STATUS + offset,
+ status,
+ (status & TPC0_CFG_STATUS_VECTOR_PIPE_EMPTY_MASK) ==
+ TPC0_CFG_STATUS_VECTOR_PIPE_EMPTY_MASK,
+ 1000,
+ kernel_timeout);
+
+ if (rc) {
+ dev_err(hdev->dev,
+ "Timeout while waiting for TPC%d icache prefetch\n",
+ tpc_id);
+ hdev->asic_funcs->enable_clock_gating(hdev);
+ mutex_unlock(&gaudi->clk_gate_mutex);
+ return -EIO;
+ }
+
+ WREG32(mmTPC0_CFG_TPC_EXECUTE + offset,
+ 1 << TPC0_CFG_TPC_EXECUTE_V_SHIFT);
+
+ /* wait a bit for the engine to start executing */
+ usleep_range(1000, 1500);
+
+ /* wait until engine has finished executing */
+ rc = hl_poll_timeout(
+ hdev,
+ mmTPC0_CFG_STATUS + offset,
+ status,
+ (status & TPC0_CFG_STATUS_VECTOR_PIPE_EMPTY_MASK) ==
+ TPC0_CFG_STATUS_VECTOR_PIPE_EMPTY_MASK,
+ 1000,
+ kernel_timeout);
+
+ rc = hl_poll_timeout(
+ hdev,
+ mmTPC0_CFG_WQ_INFLIGHT_CNTR + offset,
+ status,
+ (status == 0),
+ 1000,
+ kernel_timeout);
+
+ hdev->asic_funcs->enable_clock_gating(hdev);
+ mutex_unlock(&gaudi->clk_gate_mutex);
+
+ if (rc) {
+ dev_err(hdev->dev,
+ "Timeout while waiting for TPC%d kernel to execute\n",
+ tpc_id);
+ return -EIO;
+ }
+
+ return 0;
+}
+
+static enum hl_device_hw_state gaudi_get_hw_state(struct hl_device *hdev)
+{
+ return RREG32(mmHW_STATE);
+}
+
+static u32 gaudi_get_queue_id_for_cq(struct hl_device *hdev, u32 cq_idx)
+{
+ return gaudi_cq_assignment[cq_idx];
+}
+
+static void gaudi_ext_queue_init(struct hl_device *hdev, u32 q_idx)
+{
+ struct gaudi_device *gaudi = hdev->asic_specific;
+ struct hl_hw_queue *hw_queue = &hdev->kernel_queues[q_idx];
+ struct hl_hw_sob *hw_sob;
+ int sob, ext_idx = gaudi->ext_queue_idx++;
+
+ /*
+ * The external queues might not sit sequentially, hence use the
+ * real external queue index for the SOB/MON base id.
+ */
+ hw_queue->base_sob_id = ext_idx * HL_RSVD_SOBS;
+ hw_queue->base_mon_id = ext_idx * HL_RSVD_MONS;
+ hw_queue->next_sob_val = 1;
+ hw_queue->curr_sob_offset = 0;
+
+ for (sob = 0 ; sob < HL_RSVD_SOBS ; sob++) {
+ hw_sob = &hw_queue->hw_sob[sob];
+ hw_sob->hdev = hdev;
+ hw_sob->sob_id = hw_queue->base_sob_id + sob;
+ hw_sob->q_idx = q_idx;
+ kref_init(&hw_sob->kref);
+ }
+}
+
+static void gaudi_ext_queue_reset(struct hl_device *hdev, u32 q_idx)
+{
+ struct hl_hw_queue *hw_queue = &hdev->kernel_queues[q_idx];
+
+ /*
+ * In case we got here due to a stuck CS, the refcnt might be bigger
+ * than 1 and therefore we reset it.
+ */
+ kref_init(&hw_queue->hw_sob[hw_queue->curr_sob_offset].kref);
+ hw_queue->curr_sob_offset = 0;
+ hw_queue->next_sob_val = 1;
+}
+
+static u32 gaudi_get_signal_cb_size(struct hl_device *hdev)
+{
+ return sizeof(struct packet_msg_short) +
+ sizeof(struct packet_msg_prot) * 2;
+}
+
+static u32 gaudi_get_wait_cb_size(struct hl_device *hdev)
+{
+ return sizeof(struct packet_msg_short) * 4 +
+ sizeof(struct packet_fence) +
+ sizeof(struct packet_msg_prot) * 2;
+}
+
+static void gaudi_gen_signal_cb(struct hl_device *hdev, void *data, u16 sob_id)
+{
+ struct hl_cb *cb = (struct hl_cb *) data;
+ struct packet_msg_short *pkt;
+ u32 value, ctl;
+
+ pkt = (struct packet_msg_short *) (uintptr_t) cb->kernel_address;
+ memset(pkt, 0, sizeof(*pkt));
+
+ value = 1 << GAUDI_PKT_SHORT_VAL_SOB_SYNC_VAL_SHIFT; /* inc by 1 */
+ value |= 1 << GAUDI_PKT_SHORT_VAL_SOB_MOD_SHIFT; /* add mode */
+
+ ctl = (sob_id * 4) << GAUDI_PKT_SHORT_CTL_ADDR_SHIFT; /* SOB id */
+ ctl |= 0 << GAUDI_PKT_SHORT_CTL_OP_SHIFT; /* write the value */
+ ctl |= 3 << GAUDI_PKT_SHORT_CTL_BASE_SHIFT; /* W_S SOB base */
+ ctl |= PACKET_MSG_SHORT << GAUDI_PKT_SHORT_CTL_OPCODE_SHIFT;
+ ctl |= 1 << GAUDI_PKT_SHORT_CTL_EB_SHIFT;
+ ctl |= 1 << GAUDI_PKT_SHORT_CTL_RB_SHIFT;
+ ctl |= 1 << GAUDI_PKT_SHORT_CTL_MB_SHIFT;
+
+ pkt->value = cpu_to_le32(value);
+ pkt->ctl = cpu_to_le32(ctl);
+}
+
+static u32 gaudi_add_mon_msg_short(struct packet_msg_short *pkt, u32 value,
+ u16 addr)
+{
+ u32 ctl, pkt_size = sizeof(*pkt);
+
+ memset(pkt, 0, pkt_size);
+
+ ctl = addr << GAUDI_PKT_SHORT_CTL_ADDR_SHIFT;
+ ctl |= 2 << GAUDI_PKT_SHORT_CTL_BASE_SHIFT; /* W_S MON base */
+ ctl |= PACKET_MSG_SHORT << GAUDI_PKT_SHORT_CTL_OPCODE_SHIFT;
+ ctl |= 0 << GAUDI_PKT_SHORT_CTL_EB_SHIFT;
+ ctl |= 1 << GAUDI_PKT_SHORT_CTL_RB_SHIFT;
+ ctl |= 0 << GAUDI_PKT_SHORT_CTL_MB_SHIFT; /* only last pkt needs MB */
+
+ pkt->value = cpu_to_le32(value);
+ pkt->ctl = cpu_to_le32(ctl);
+
+ return pkt_size;
+}
+
+static u32 gaudi_add_arm_monitor_pkt(struct packet_msg_short *pkt, u16 sob_id,
+ u16 sob_val, u16 addr)
+{
+ u32 ctl, value, pkt_size = sizeof(*pkt);
+ u8 mask = ~(1 << (sob_id & 0x7));
+
+ memset(pkt, 0, pkt_size);
+
+ value = (sob_id / 8) << GAUDI_PKT_SHORT_VAL_MON_SYNC_GID_SHIFT;
+ value |= sob_val << GAUDI_PKT_SHORT_VAL_MON_SYNC_VAL_SHIFT;
+ value |= 0 << GAUDI_PKT_SHORT_VAL_MON_MODE_SHIFT; /* GREATER_OR_EQUAL */
+ value |= mask << GAUDI_PKT_SHORT_VAL_MON_MASK_SHIFT;
+
+ ctl = addr << GAUDI_PKT_SHORT_CTL_ADDR_SHIFT;
+ ctl |= 0 << GAUDI_PKT_SHORT_CTL_OP_SHIFT; /* write the value */
+ ctl |= 2 << GAUDI_PKT_SHORT_CTL_BASE_SHIFT; /* W_S MON base */
+ ctl |= PACKET_MSG_SHORT << GAUDI_PKT_SHORT_CTL_OPCODE_SHIFT;
+ ctl |= 0 << GAUDI_PKT_SHORT_CTL_EB_SHIFT;
+ ctl |= 1 << GAUDI_PKT_SHORT_CTL_RB_SHIFT;
+ ctl |= 1 << GAUDI_PKT_SHORT_CTL_MB_SHIFT;
+
+ pkt->value = cpu_to_le32(value);
+ pkt->ctl = cpu_to_le32(ctl);
+
+ return pkt_size;
+}
+
+static u32 gaudi_add_fence_pkt(struct packet_fence *pkt)
+{
+ u32 ctl, cfg, pkt_size = sizeof(*pkt);
+
+ memset(pkt, 0, pkt_size);
+
+ cfg = 1 << GAUDI_PKT_FENCE_CFG_DEC_VAL_SHIFT;
+ cfg |= 1 << GAUDI_PKT_FENCE_CFG_TARGET_VAL_SHIFT;
+ cfg |= 2 << GAUDI_PKT_FENCE_CFG_ID_SHIFT;
+
+ ctl = 0 << GAUDI_PKT_FENCE_CTL_PRED_SHIFT;
+ ctl |= PACKET_FENCE << GAUDI_PKT_FENCE_CTL_OPCODE_SHIFT;
+ ctl |= 0 << GAUDI_PKT_FENCE_CTL_EB_SHIFT;
+ ctl |= 1 << GAUDI_PKT_FENCE_CTL_RB_SHIFT;
+ ctl |= 1 << GAUDI_PKT_FENCE_CTL_MB_SHIFT;
+
+ pkt->cfg = cpu_to_le32(cfg);
+ pkt->ctl = cpu_to_le32(ctl);
+
+ return pkt_size;
+}
+
+static void gaudi_gen_wait_cb(struct hl_device *hdev, void *data, u16 sob_id,
+ u16 sob_val, u16 mon_id, u32 q_idx)
+{
+ struct hl_cb *cb = (struct hl_cb *) data;
+ void *buf = (void *) (uintptr_t) cb->kernel_address;
+ u64 monitor_base, fence_addr = 0;
+ u32 size = 0;
+ u16