diff options
Diffstat (limited to 'drivers/misc/habanalabs/common/habanalabs.h')
| -rw-r--r-- | drivers/misc/habanalabs/common/habanalabs.h | 415 |
1 files changed, 255 insertions, 160 deletions
diff --git a/drivers/misc/habanalabs/common/habanalabs.h b/drivers/misc/habanalabs/common/habanalabs.h index 1edaf6ab67bd..b0b0f3f89865 100644 --- a/drivers/misc/habanalabs/common/habanalabs.h +++ b/drivers/misc/habanalabs/common/habanalabs.h @@ -21,6 +21,7 @@ #include <linux/hashtable.h> #include <linux/debugfs.h> #include <linux/rwsem.h> +#include <linux/eventfd.h> #include <linux/bitfield.h> #include <linux/genalloc.h> #include <linux/sched/signal.h> @@ -61,8 +62,10 @@ #define HL_CPUCP_INFO_TIMEOUT_USEC 10000000 /* 10s */ #define HL_CPUCP_EEPROM_TIMEOUT_USEC 10000000 /* 10s */ +#define HL_CPUCP_MON_DUMP_TIMEOUT_USEC 10000000 /* 10s */ #define HL_FW_STATUS_POLL_INTERVAL_USEC 10000 /* 10ms */ +#define HL_FW_COMMS_STATUS_PLDM_POLL_INTERVAL_USEC 1000000 /* 1s */ #define HL_PCI_ELBI_TIMEOUT_MSEC 10 /* 10ms */ @@ -394,18 +397,8 @@ enum hl_device_hw_state { * struct hl_mmu_properties - ASIC specific MMU address translation properties. * @start_addr: virtual start address of the memory region. * @end_addr: virtual end address of the memory region. - * @hop0_shift: shift of hop 0 mask. - * @hop1_shift: shift of hop 1 mask. - * @hop2_shift: shift of hop 2 mask. - * @hop3_shift: shift of hop 3 mask. - * @hop4_shift: shift of hop 4 mask. - * @hop5_shift: shift of hop 5 mask. - * @hop0_mask: mask to get the PTE address in hop 0. - * @hop1_mask: mask to get the PTE address in hop 1. - * @hop2_mask: mask to get the PTE address in hop 2. - * @hop3_mask: mask to get the PTE address in hop 3. - * @hop4_mask: mask to get the PTE address in hop 4. - * @hop5_mask: mask to get the PTE address in hop 5. + * @hop_shifts: array holds HOPs shifts. + * @hop_masks: array holds HOPs masks. * @last_mask: mask to get the bit indicating this is the last hop. * @pgt_size: size for page tables. * @page_size: default page size used to allocate memory. @@ -418,18 +411,8 @@ enum hl_device_hw_state { struct hl_mmu_properties { u64 start_addr; u64 end_addr; - u64 hop0_shift; - u64 hop1_shift; - u64 hop2_shift; - u64 hop3_shift; - u64 hop4_shift; - u64 hop5_shift; - u64 hop0_mask; - u64 hop1_mask; - u64 hop2_mask; - u64 hop3_mask; - u64 hop4_mask; - u64 hop5_mask; + u64 hop_shifts[MMU_HOP_MAX]; + u64 hop_masks[MMU_HOP_MAX]; u64 last_mask; u64 pgt_size; u32 page_size; @@ -486,8 +469,10 @@ struct hl_hints_range { * the device's MMU. * @dram_hints_align_mask: dram va hint addresses alignment mask which is used * for hints validity check. - * device_dma_offset_for_host_access: the offset to add to host DMA addresses - * to enable the device to access them. + * @device_dma_offset_for_host_access: the offset to add to host DMA addresses + * to enable the device to access them. + * @host_base_address: host physical start address for host DMA from device + * @host_end_address: host physical end address for host DMA from device * @max_freq_value: current max clk frequency. * @clk_pll_index: clock PLL index that specify which PLL determines the clock * we display to the user @@ -528,6 +513,10 @@ struct hl_hints_range { * @fw_app_cpu_boot_dev_sts1: bitmap representation of application security * status reported by FW, bit description can be * found in CPU_BOOT_DEV_STS1 + * @device_mem_alloc_default_page_size: may be different than dram_page_size only for ASICs for + * which the property supports_user_set_page_size is true + * (i.e. the DRAM supports multiple page sizes), otherwise + * it will shall be equal to dram_page_size. * @collective_first_sob: first sync object available for collective use * @collective_first_mon: first monitor available for collective use * @sync_stream_first_sob: first sync object available for sync stream use @@ -568,6 +557,7 @@ struct hl_hints_range { * @configurable_stop_on_err: is stop-on-error option configurable via debugfs. * @set_max_power_on_device_init: true if need to set max power in F/W on device init. * @supports_user_set_page_size: true if user can set the allocation page size. + * @dma_mask: the dma mask to be set for this device */ struct asic_fixed_properties { struct hw_queue_properties *hw_queues_props; @@ -599,6 +589,8 @@ struct asic_fixed_properties { u64 cb_va_end_addr; u64 dram_hints_align_mask; u64 device_dma_offset_for_host_access; + u64 host_base_address; + u64 host_end_address; u64 max_freq_value; u32 clk_pll_index; u32 mmu_pgt_size; @@ -626,6 +618,7 @@ struct asic_fixed_properties { u32 fw_bootfit_cpu_boot_dev_sts1; u32 fw_app_cpu_boot_dev_sts0; u32 fw_app_cpu_boot_dev_sts1; + u32 device_mem_alloc_default_page_size; u16 collective_first_sob; u16 collective_first_mon; u16 sync_stream_first_sob; @@ -654,6 +647,7 @@ struct asic_fixed_properties { u8 configurable_stop_on_err; u8 set_max_power_on_device_init; u8 supports_user_set_page_size; + u8 dma_mask; }; /** @@ -711,85 +705,102 @@ struct hl_cs_compl { */ /** - * struct hl_cb_mgr - describes a Command Buffer Manager. - * @cb_lock: protects cb_handles. - * @cb_handles: an idr to hold all command buffer handles. - */ -struct hl_cb_mgr { - spinlock_t cb_lock; - struct idr cb_handles; /* protected by cb_lock */ -}; - -/** - * struct hl_ts_mgr - describes the timestamp registration memory manager. - * @ts_lock: protects ts_handles. - * @ts_handles: an idr to hold all ts bufferes handles. - */ -struct hl_ts_mgr { - spinlock_t ts_lock; - struct idr ts_handles; -}; - -/** * struct hl_ts_buff - describes a timestamp buffer. - * @refcount: reference counter for usage of the buffer. - * @hdev: pointer to device this buffer belongs to. - * @mmap: true if the buff is currently mapped to user. * @kernel_buff_address: Holds the internal buffer's kernel virtual address. * @user_buff_address: Holds the user buffer's kernel virtual address. - * @id: the buffer ID. - * @mmap_size: Holds the buffer size that was mmaped. * @kernel_buff_size: Holds the internal kernel buffer size. - * @user_buff_size: Holds the user buffer size. */ struct hl_ts_buff { - struct kref refcount; - struct hl_device *hdev; - atomic_t mmap; void *kernel_buff_address; void *user_buff_address; - u32 id; - u32 mmap_size; u32 kernel_buff_size; - u32 user_buff_size; +}; + +struct hl_mmap_mem_buf; + +/** + * struct hl_mem_mgr - describes unified memory manager for mappable memory chunks. + * @dev: back pointer to the owning device + * @lock: protects handles + * @handles: an idr holding all active handles to the memory buffers in the system. + */ +struct hl_mem_mgr { + struct device *dev; + spinlock_t lock; + struct idr handles; +}; + +/** + * struct hl_mmap_mem_buf_behavior - describes unified memory manager buffer behavior + * @topic: string identifier used for logging + * @mem_id: memory type identifier, embedded in the handle and used to identify + * the memory type by handle. + * @alloc: callback executed on buffer allocation, shall allocate the memory, + * set it under buffer private, and set mappable size. + * @mmap: callback executed on mmap, must map the buffer to vma + * @release: callback executed on release, must free the resources used by the buffer + */ +struct hl_mmap_mem_buf_behavior { + const char *topic; + u64 mem_id; + + int (*alloc)(struct hl_mmap_mem_buf *buf, gfp_t gfp, void *args); + int (*mmap)(struct hl_mmap_mem_buf *buf, struct vm_area_struct *vma, void *args); + void (*release)(struct hl_mmap_mem_buf *buf); +}; + +/** + * struct hl_mmap_mem_buf - describes a single unified memory buffer + * @behavior: buffer behavior + * @mmg: back pointer to the unified memory manager + * @refcount: reference counter for buffer users + * @private: pointer to buffer behavior private data + * @mmap: atomic boolean indicating whether or not the buffer is mapped right now + * @real_mapped_size: the actual size of buffer mapped, after part of it may be released, + * may change at runtime. + * @mappable_size: the original mappable size of the buffer, does not change after + * the allocation. + * @handle: the buffer id in mmg handles store + */ +struct hl_mmap_mem_buf { + struct hl_mmap_mem_buf_behavior *behavior; + struct hl_mem_mgr *mmg; + struct kref refcount; + void *private; + atomic_t mmap; + u64 real_mapped_size; + u64 mappable_size; + u64 handle; }; /** * struct hl_cb - describes a Command Buffer. - * @refcount: reference counter for usage of the CB. * @hdev: pointer to device this CB belongs to. * @ctx: pointer to the CB owner's context. - * @lock: spinlock to protect mmap flows. + * @buf: back pointer to the parent mappable memory buffer * @debugfs_list: node in debugfs list of command buffers. * @pool_list: node in pool list of command buffers. * @va_block_list: list of virtual addresses blocks of the CB if it is mapped to * the device's MMU. - * @id: the CB's ID. * @kernel_address: Holds the CB's kernel virtual address. * @bus_address: Holds the CB's DMA address. - * @mmap_size: Holds the CB's size that was mmaped. * @size: holds the CB's size. * @cs_cnt: holds number of CS that this CB participates in. - * @mmap: true if the CB is currently mmaped to user. * @is_pool: true if CB was acquired from the pool, false otherwise. * @is_internal: internaly allocated * @is_mmu_mapped: true if the CB is mapped to the device's MMU. */ struct hl_cb { - struct kref refcount; struct hl_device *hdev; struct hl_ctx *ctx; - spinlock_t lock; + struct hl_mmap_mem_buf *buf; struct list_head debugfs_list; struct list_head pool_list; struct list_head va_block_list; - u64 id; void *kernel_address; dma_addr_t bus_address; - u32 mmap_size; u32 size; atomic_t cs_cnt; - u8 mmap; u8 is_pool; u8 is_internal; u8 is_mmu_mapped; @@ -935,12 +946,12 @@ struct hl_user_interrupt { * struct timestamp_reg_free_node - holds the timestamp registration free objects node * @free_objects_node: node in the list free_obj_jobs * @cq_cb: pointer to cq command buffer to be freed - * @ts_buff: pointer to timestamp buffer to be freed + * @buf: pointer to timestamp buffer to be freed */ struct timestamp_reg_free_node { struct list_head free_objects_node; struct hl_cb *cq_cb; - struct hl_ts_buff *ts_buff; + struct hl_mmap_mem_buf *buf; }; /* struct timestamp_reg_work_obj - holds the timestamp registration free objects job @@ -957,8 +968,8 @@ struct timestamp_reg_work_obj { }; /* struct timestamp_reg_info - holds the timestamp registration related data. - * @ts_buff: pointer to the timestamp buffer which include both user/kernel buffers. - * relevant only when doing timestamps records registration. + * @buf: pointer to the timestamp buffer which include both user/kernel buffers. + * relevant only when doing timestamps records registration. * @cq_cb: pointer to CQ counter CB. * @timestamp_kernel_addr: timestamp handle address, where to set timestamp * relevant only when doing timestamps records @@ -969,7 +980,7 @@ struct timestamp_reg_work_obj { * allocating records dynamically. */ struct timestamp_reg_info { - struct hl_ts_buff *ts_buff; + struct hl_mmap_mem_buf *buf; struct hl_cb *cq_cb; u64 *timestamp_kernel_addr; u8 in_use; @@ -1068,6 +1079,15 @@ enum div_select_defs { DIV_SEL_DIVIDED_PLL = 3, }; +enum debugfs_access_type { + DEBUGFS_READ8, + DEBUGFS_WRITE8, + DEBUGFS_READ32, + DEBUGFS_WRITE32, + DEBUGFS_READ64, + DEBUGFS_WRITE64, +}; + enum pci_region { PCI_REGION_CFG, PCI_REGION_SRAM, @@ -1229,6 +1249,7 @@ struct fw_load_mgr { * its implementation is not trivial when the driver * is loaded in simulation mode (not upstreamed). * @scrub_device_mem: Scrub device memory given an address and size + * @scrub_device_dram: Scrub the dram memory of the device. * @get_int_queue_base: get the internal queue base address. * @test_queues: run simple test on all queues for sanity check. * @asic_dma_pool_zalloc: small DMA allocation of coherent memory from DMA pool. @@ -1236,18 +1257,14 @@ struct fw_load_mgr { * @asic_dma_pool_free: free small DMA allocation from pool. * @cpu_accessible_dma_pool_alloc: allocate CPU PQ packet from DMA pool. * @cpu_accessible_dma_pool_free: free CPU PQ packet from DMA pool. - * @hl_dma_unmap_sg: DMA unmap scatter-gather list. + * @hl_dma_unmap_sgtable: DMA unmap scatter-gather table. * @cs_parser: parse Command Submission. - * @asic_dma_map_sg: DMA map scatter-gather list. + * @asic_dma_map_sgtable: DMA map scatter-gather table. * @get_dma_desc_list_size: get number of LIN_DMA packets required for CB. * @add_end_of_cb_packets: Add packets to the end of CB, if device requires it. * @update_eq_ci: update event queue CI. * @context_switch: called upon ASID context switch. * @restore_phase_topology: clear all SOBs amd MONs. - * @debugfs_read32: debug interface for reading u32 from DRAM/SRAM/Host memory. - * @debugfs_write32: debug interface for writing u32 to DRAM/SRAM/Host memory. - * @debugfs_read64: debug interface for reading u64 from DRAM/SRAM/Host memory. - * @debugfs_write64: debug interface for writing u64 to DRAM/SRAM/Host memory. * @debugfs_read_dma: debug interface for reading up to 2MB from the device's * internal memory via DMA engine. * @add_device_attr: add ASIC specific device attributes. @@ -1257,8 +1274,8 @@ struct fw_load_mgr { * @write_pte: write MMU page table entry to DRAM. * @mmu_invalidate_cache: flush MMU STLB host/DRAM cache, either with soft * (L1 only) or hard (L0 & L1) flush. - * @mmu_invalidate_cache_range: flush specific MMU STLB cache lines with - * ASID-VA-size mask. + * @mmu_invalidate_cache_range: flush specific MMU STLB cache lines with ASID-VA-size mask. + * @mmu_prefetch_cache_range: pre-fetch specific MMU STLB cache lines with ASID-VA-size mask. * @send_heartbeat: send is-alive packet to CPU-CP and verify response. * @debug_coresight: perform certain actions on Coresight for debugging. * @is_device_idle: return true if device is idle, false otherwise. @@ -1267,6 +1284,7 @@ struct fw_load_mgr { * @hw_queues_unlock: release H/W queues lock. * @get_pci_id: retrieve PCI ID. * @get_eeprom_data: retrieve EEPROM data from F/W. + * @get_monitor_dump: retrieve monitor registers dump from F/W. * @send_cpu_message: send message to F/W. If the message is timedout, the * driver will eventually reset the device. The timeout can * be determined by the calling function or it can be 0 and @@ -1289,8 +1307,6 @@ struct fw_load_mgr { * @gen_wait_cb: Generate a wait CB. * @reset_sob: Reset a SOB. * @reset_sob_group: Reset SOB group - * @set_dma_mask_from_fw: set the DMA mask in the driver according to the - * firmware configuration * @get_device_time: Get the device time. * @collective_wait_init_cs: Generate collective master/slave packets * and place them in the relevant cs jobs @@ -1319,6 +1335,9 @@ struct fw_load_mgr { * @get_stream_master_qid_arr: get pointer to stream masters QID array * @is_valid_dram_page_size: return true if page size is supported in device * memory allocation, otherwise false. + * @get_valid_dram_page_orders: get valid device memory allocation page orders + * @access_dev_mem: access device memory + * @set_dram_bar_base: set the base of the DRAM BAR */ struct hl_asic_funcs { int (*early_init)(struct hl_device *hdev); @@ -1342,6 +1361,7 @@ struct hl_asic_funcs { void (*asic_dma_free_coherent)(struct hl_device *hdev, size_t size, void *cpu_addr, dma_addr_t dma_handle); int (*scrub_device_mem)(struct hl_device *hdev, u64 addr, u64 size); + int (*scrub_device_dram)(struct hl_device *hdev, u64 val); void* (*get_int_queue_base)(struct hl_device *hdev, u32 queue_id, dma_addr_t *dma_handle, u16 *queue_len); int (*test_queues)(struct hl_device *hdev); @@ -1353,12 +1373,11 @@ struct hl_asic_funcs { size_t size, dma_addr_t *dma_handle); void (*cpu_accessible_dma_pool_free)(struct hl_device *hdev, size_t size, void *vaddr); - void (*hl_dma_unmap_sg)(struct hl_device *hdev, - struct scatterlist *sgl, int nents, + void (*hl_dma_unmap_sgtable)(struct hl_device *hdev, + struct sg_table *sgt, enum dma_data_direction dir); int (*cs_parser)(struct hl_device *hdev, struct hl_cs_parser *parser); - int (*asic_dma_map_sg)(struct hl_device *hdev, - struct scatterlist *sgl, int nents, + int (*asic_dma_map_sgtable)(struct hl_device *hdev, struct sg_table *sgt, enum dma_data_direction dir); u32 (*get_dma_desc_list_size)(struct hl_device *hdev, struct sg_table *sgt); @@ -1369,14 +1388,6 @@ struct hl_asic_funcs { void (*update_eq_ci)(struct hl_device *hdev, u32 val); int (*context_switch)(struct hl_device *hdev, u32 asid); void (*restore_phase_topology)(struct hl_device *hdev); - int (*debugfs_read32)(struct hl_device *hdev, u64 addr, - bool user_address, u32 *val); - int (*debugfs_write32)(struct hl_device *hdev, u64 addr, - bool user_address, u32 val); - int (*debugfs_read64)(struct hl_device *hdev, u64 addr, - bool user_address, u64 *val); - int (*debugfs_write64)(struct hl_device *hdev, u64 addr, - bool user_address, u64 val); int (*debugfs_read_dma)(struct hl_device *hdev, u64 addr, u32 size, void *blob_addr); void (*add_device_attr)(struct hl_device *hdev, struct attribute_group *dev_clk_attr_grp, @@ -1391,6 +1402,7 @@ struct hl_asic_funcs { u32 flags); int (*mmu_invalidate_cache_range)(struct hl_device *hdev, bool is_hard, u32 flags, u32 asid, u64 va, u64 size); + int (*mmu_prefetch_cache_range)(struct hl_ctx *ctx, u32 flags, u32 asid, u64 va, u64 size); int (*send_heartbeat)(struct hl_device *hdev); int (*debug_coresight)(struct hl_device *hdev, struct hl_ctx *ctx, void *data); bool (*is_device_idle)(struct hl_device *hdev, u64 *mask_arr, @@ -1399,8 +1411,8 @@ struct hl_asic_funcs { void (*hw_queues_lock)(struct hl_device *hdev); void (*hw_queues_unlock)(struct hl_device *hdev); u32 (*get_pci_id)(struct hl_device *hdev); - int (*get_eeprom_data)(struct hl_device *hdev, void *data, - size_t max_size); + int (*get_eeprom_data)(struct hl_device *hdev, void *data, size_t max_size); + int (*get_monitor_dump)(struct hl_device *hdev, void *data); int (*send_cpu_message)(struct hl_device *hdev, u32 *msg, u16 len, u32 timeout, u64 *result); int (*pci_bars_map)(struct hl_device *hdev); @@ -1421,7 +1433,6 @@ struct hl_asic_funcs { struct hl_gen_wait_properties *prop); void (*reset_sob)(struct hl_device *hdev, void *data); void (*reset_sob_group)(struct hl_device *hdev, u16 sob_group); - void (*set_dma_mask_from_fw)(struct hl_device *hdev); u64 (*get_device_time)(struct hl_device *hdev); int (*collective_wait_init_cs)(struct hl_cs *cs); int (*collective_wait_create_jobs)(struct hl_device *hdev, @@ -1445,6 +1456,12 @@ struct hl_asic_funcs { void (*set_pci_memory_regions)(struct hl_device *hdev); u32* (*get_stream_master_qid_arr)(void); bool (*is_valid_dram_page_size)(u32 page_size); + int (*mmu_get_real_page_size)(struct hl_device *hdev, struct hl_mmu_properties *mmu_prop, + u32 page_size, u32 *real_page_size, bool is_dram_addr); + void (*get_valid_dram_page_orders)(struct hl_info_dev_memalloc_page_sizes *info); + int (*access_dev_mem)(struct hl_device *hdev, struct pci_mem_region *region, + enum pci_region region_type, u64 addr, u64 *val, enum debugfs_access_type acc_type); + u64 (*set_dram_bar_base)(struct hl_device *hdev, u64 addr); }; @@ -1915,6 +1932,18 @@ struct hl_debug_params { bool enable; }; +/** + * struct hl_notifier_event - holds the notifier data structure + * @eventfd: the event file descriptor to raise the notifications + * @lock: mutex lock to protect the notifier data flows + * @events_mask: indicates the bitmap events + */ +struct hl_notifier_event { + struct eventfd_ctx *eventfd; + struct mutex lock; + u64 events_mask; +}; + /* * FILE PRIVATE STRUCTURE */ @@ -1926,25 +1955,25 @@ struct hl_debug_params { * @taskpid: current process ID. * @ctx: current executing context. TODO: remove for multiple ctx per process * @ctx_mgr: context manager to handle multiple context for this FD. - * @cb_mgr: command buffer manager to handle multiple buffers for this FD. - * @ts_mem_mgr: timestamp registration manager for alloc/free/map timestamp buffers. + * @mem_mgr: manager descriptor for memory exportable via mmap + * @notifier_event: notifier eventfd towards user process * @debugfs_list: list of relevant ASIC debugfs. * @dev_node: node in the device list of file private data * @refcount: number of related contexts. * @restore_phase_mutex: lock for context switch and restore phase. */ struct hl_fpriv { - struct hl_device *hdev; - struct file *filp; - struct pid *taskpid; - struct hl_ctx *ctx; - struct hl_ctx_mgr ctx_mgr; - struct hl_cb_mgr cb_mgr; - struct hl_ts_mgr ts_mem_mgr; - struct list_head debugfs_list; - struct list_head dev_node; - struct kref refcount; - struct mutex restore_phase_mutex; + struct hl_device *hdev; + struct file *filp; + struct pid *taskpid; + struct hl_ctx *ctx; + struct hl_ctx_mgr ctx_mgr; + struct hl_mem_mgr mem_mgr; + struct hl_notifier_event notifier_event; + struct list_head debugfs_list; + struct list_head dev_node; + struct kref refcount; + struct mutex restore_phase_mutex; }; @@ -1992,12 +2021,14 @@ struct hl_debugfs_entry { * @userptr_spinlock: protects userptr_list. * @ctx_mem_hash_list: list of available contexts with MMU mappings. * @ctx_mem_hash_spinlock: protects cb_list. - * @blob_desc: descriptor of blob + * @data_dma_blob_desc: data DMA descriptor of blob. + * @mon_dump_blob_desc: monitor dump descriptor of blob. * @state_dump: data of the system states in case of a bad cs. * @state_dump_sem: protects state_dump. * @addr: next address to read/write from/to in read/write32. * @mmu_addr: next virtual address to translate to physical address in mmu_show. * @userptr_lookup: the target user ptr to look up for on demand. + * @memory_scrub_val: the value to which the dram will be scrubbed to using cb scrub_device_dram * @mmu_asid: ASID to use while translating in mmu_show. * @state_dump_head: index of the latest state dump * @i2c_bus: generic u8 debugfs file for bus value to use in i2c_data_read. @@ -2021,12 +2052,14 @@ struct hl_dbg_device_entry { spinlock_t userptr_spinlock; struct list_head ctx_mem_hash_list; spinlock_t ctx_mem_hash_spinlock; - struct debugfs_blob_wrapper blob_desc; + struct debugfs_blob_wrapper data_dma_blob_desc; + struct debugfs_blob_wrapper mon_dump_blob_desc; char *state_dump[HL_STATE_DUMP_HIST_LEN]; struct rw_semaphore state_dump_sem; u64 addr; u64 mmu_addr; u64 userptr_lookup; + u64 memory_scrub_val; u32 mmu_asid; u32 state_dump_head; u8 i2c_bus; @@ -2442,6 +2475,24 @@ struct hl_mmu_funcs { }; /** + * struct hl_prefetch_work - prefetch work structure handler + * @pf_work: actual work struct. + * @ctx: compute context. + * @va: virtual address to pre-fetch. + * @size: pre-fetch size. + * @flags: operation flags. + * @asid: ASID for maintenance operation. + */ +struct hl_prefetch_work { + struct work_struct pf_work; + struct hl_ctx *ctx; + u64 va; + u64 size; + u32 flags; + u32 asid; +}; + +/* * number of user contexts allowed to call wait_for_multi_cs ioctl in * parallel */ @@ -2517,37 +2568,50 @@ struct hl_clk_throttle { }; /** - * struct last_error_session_info - info about last session in which CS timeout or - * razwi error occurred. - * @open_dev_timestamp: device open timestamp. - * @cs_timeout_timestamp: CS timeout timestamp. - * @razwi_timestamp: razwi timestamp. - * @cs_write_disable: if set writing to CS parameters in the structure is disabled so the - * first (root cause) CS timeout will not be overwritten. - * @razwi_write_disable: if set writing to razwi parameters in the structure is disabled so the - * first (root cause) razwi will not be overwritten. - * @cs_timeout_seq: CS timeout sequence number. - * @razwi_addr: address that caused razwi. - * @razwi_engine_id_1: engine id of the razwi initiator, if it was initiated by engine that does - * not have engine id it will be set to U16_MAX. - * @razwi_engine_id_2: second engine id of razwi initiator. Might happen that razwi have 2 possible - * engines which one them caused the razwi. In that case, it will contain the - * second possible engine id, otherwise it will be set to U16_MAX. - * @razwi_non_engine_initiator: in case the initiator of the razwi does not have engine id. - * @razwi_type: cause of razwi, page fault or access error, otherwise it will be set to U8_MAX. + * struct cs_timeout_info - info of last CS timeout occurred. + * @timestamp: CS timeout timestamp. + * @write_disable: if set writing to CS parameters in the structure is disabled so, + * the first (root cause) CS timeout will not be overwritten. + * @seq: CS timeout sequence number. + */ +struct cs_timeout_info { + ktime_t timestamp; + atomic_t write_disable; + u64 seq; +}; + +/** + * struct razwi_info - info about last razwi error occurred. + * @timestamp: razwi timestamp. + * @write_disable: if set writing to razwi parameters in the structure is disabled so the + * first (root cause) razwi will not be overwritten. + * @addr: address that caused razwi. + * @engine_id_1: engine id of the razwi initiator, if it was initiated by engine that does + * not have engine id it will be set to U16_MAX. + * @engine_id_2: second engine id of razwi initiator. Might happen that razwi have 2 possible + * engines which one them caused the razwi. In that case, it will contain the + * second possible engine id, otherwise it will be set to U16_MAX. + * @non_engine_initiator: in case the initiator of the razwi does not have engine id. + * @type: cause of razwi, page fault or access error, otherwise it will be set to U8_MAX. + */ +struct razwi_info { + ktime_t timestamp; + atomic_t write_disable; + u64 addr; + u16 engine_id_1; + u16 engine_id_2; + u8 non_engine_initiator; + u8 type; +}; + +/** + * struct last_error_session_info - info about last session errors occurred. + * @cs_timeout: CS timeout error last information. + * @razwi: razwi last information. */ struct last_error_session_info { - ktime_t open_dev_timestamp; - ktime_t cs_timeout_timestamp; - ktime_t razwi_timestamp; - atomic_t cs_write_disable; - atomic_t razwi_write_disable; - u64 cs_timeout_seq; - u64 razwi_addr; - u16 razwi_engine_id_1; - u16 razwi_engine_id_2; - u8 razwi_non_engine_initiator; - u8 razwi_type; + struct cs_timeout_info cs_timeout; + struct razwi_info razwi; }; /** @@ -2614,11 +2678,12 @@ struct hl_reset_info { * context. * @eq_wq: work queue of event queue for executing work in process context. * @ts_free_obj_wq: work queue for timestamp registration objects release. + * @pf_wq: work queue for MMU pre-fetch operations. * @kernel_ctx: Kernel driver context structure. * @kernel_queues: array of hl_hw_queue. * @cs_mirror_list: CS mirror list for TDR. * @cs_mirror_lock: protects cs_mirror_list. - * @kernel_cb_mgr: command buffer manager for creating/destroying/handling CBs. + * @kernel_mem_mgr: memory manager for memory buffers with lifespan of driver. * @event_queue: event queue for IRQ from CPU-CP. * @dma_pool: DMA pool for small allocations. * @cpu_accessible_dma_mem: Host <-> CPU-CP shared memory CPU address. @@ -2656,9 +2721,10 @@ struct hl_reset_info { * @state_dump_specs: constants and dictionaries needed to dump system state. * @multi_cs_completion: array of multi-CS completion. * @clk_throttling: holds information about current/previous clock throttling events - * @reset_info: holds current device reset information. * @last_error: holds information about last session in which CS timeout or razwi error occurred. + * @reset_info: holds current device reset information. * @stream_master_qid_arr: pointer to array with QIDs of master streams. + * @fw_major_version: major version of current loaded preboot * @dram_used_mem: current DRAM memory consumption. * @timeout_jiffies: device CS timeout value. * @max_power: the max power of the device, as configured by the sysadmin. This @@ -2678,6 +2744,9 @@ struct hl_reset_info { * session. * @open_counter: number of successful device open operations. * @fw_poll_interval_usec: FW status poll interval in usec. + * used for CPU boot status + * @fw_comms_poll_interval_usec: FW comms/protocol poll interval in usec. + * used for COMMs protocols cmds(COMMS_STS_*) * @card_type: Various ASICs have several card types. This indicates the card * type of the current device. * @major: habanalabs kernel driver major. @@ -2686,6 +2755,7 @@ struct hl_reset_info { * @id_control: minor of the control device * @cpu_pci_msb_addr: 50-bit extension bits for the device CPU's 40-bit * addresses. + * @is_in_dram_scrub: true if dram scrub operation is on going. * @disabled: is device disabled. * @late_init_done: is late init stage was done during initialization. * @hwmon_initialized: is H/W monitor sensors was initialized. @@ -2699,7 +2769,6 @@ struct hl_reset_info { * huge pages. * @init_done: is the initialization of the device done. * @device_cpu_disabled: is the device CPU disabled (due to timeouts) - * @dma_mask: the dma mask that was set for this device * @in_debug: whether the device is in a state where the profiling/tracing infrastructure * can be used. This indication is needed because in some ASICs we need to do * specific operations to enable that infrastructure. @@ -2721,6 +2790,8 @@ struct hl_reset_info { * cases where Linux was not loaded to device CPU * @supports_wait_for_multi_cs: true if wait for multi CS is supported * @is_compute_ctx_active: Whether there is an active compute context executing. + * @compute_ctx_in_release: true if the current compute context is being released. + * @supports_mmu_prefetch: true if prefetch is supported, otherwise false. */ struct hl_device { struct pci_dev *pdev; @@ -2742,11 +2813,12 @@ struct hl_device { struct workqueue_struct **cq_wq; struct workqueue_struct *eq_wq; struct workqueue_struct *ts_free_obj_wq; + struct workqueue_struct *pf_wq; struct hl_ctx *kernel_ctx; struct hl_hw_queue *kernel_queues; struct list_head cs_mirror_list; spinlock_t cs_mirror_lock; - struct hl_cb_mgr kernel_cb_mgr; + struct hl_mem_mgr kernel_mem_mgr; struct hl_eq event_queue; struct dma_pool *dma_pool; void *cpu_accessible_dma_mem; @@ -2797,6 +2869,7 @@ struct hl_device { struct hl_reset_info reset_info; u32 *stream_master_qid_arr; + u32 fw_major_version; atomic64_t dram_used_mem; u64 timeout_jiffies; u64 max_power; @@ -2807,12 +2880,15 @@ struct hl_device { u64 open_counter; u64 fw_poll_interval_usec; ktime_t last_successful_open_ktime; + u64 fw_comms_poll_interval_usec; + enum cpucp_card_types card_type; u32 major; u32 high_pll; u16 id; u16 id_control; u16 cpu_pci_msb_addr; + u8 is_in_dram_scrub; u8 disabled; u8 late_init_done; u8 hwmon_initialized; @@ -2823,7 +2899,6 @@ struct hl_device { u8 pmmu_huge_range; u8 init_done; u8 device_cpu_disabled; - u8 dma_mask; u8 in_debug; u8 cdev_sysfs_created; u8 stop_on_err; @@ -2839,6 +2914,8 @@ struct hl_device { u8 supports_wait_for_multi_cs; u8 stream_master_qid_arr_size; u8 is_compute_ctx_active; + u8 compute_ctx_in_release; + u8 supports_mmu_prefetch; /* Parameters for bring-up */ u64 nic_ports_mask; @@ -2971,6 +3048,14 @@ static inline bool hl_mem_area_crosses_range(u64 address, u32 size, return ((address <= range_end_address) && (range_start_address <= end_address)); } +uint64_t hl_set_dram_bar_default(struct hl_device *hdev, u64 addr); +int hl_dma_map_sgtable(struct hl_device *hdev, struct sg_table *sgt, enum dma_data_direction dir); +void hl_dma_unmap_sgtable(struct hl_device *hdev, struct sg_table *sgt, + enum dma_data_direction dir); +int hl_access_cfg_region(struct hl_device *hdev, u64 addr, u64 *val, + enum debugfs_access_type acc_type); +int hl_access_dev_mem(struct hl_device *hdev, struct pci_mem_region *region, + enum pci_region region_type, u64 addr, u64 *val, enum debugfs_access_type acc_type); int hl_device_open(struct inode *inode, struct file *filp); int hl_device_open_ctrl(struct inode *inode, struct file *filp); bool hl_device_operational(struct hl_device *hdev, @@ -3013,7 +3098,7 @@ int hl_ctx_create(struct hl_device *hdev, struct hl_fpriv *hpriv); void hl_ctx_free(struct hl_device *hdev, struct hl_ctx *ctx); int hl_ctx_init(struct hl_device *hdev, struct hl_ctx *ctx, bool is_kernel_ctx); void hl_ctx_do_release(struct kref *ref); -void hl_ctx_get(struct hl_device *hdev, struct hl_ctx *ctx); +void hl_ctx_get(struct hl_ctx *ctx); int hl_ctx_put(struct hl_ctx *ctx); struct hl_ctx *hl_get_compute_ctx(struct hl_device *hdev); struct hl_fence *hl_ctx_get_fence(struct hl_ctx *ctx, u64 seq); @@ -3034,23 +3119,21 @@ int hl_device_utilization(struct hl_device *hdev, u32 *utilization); int hl_build_hwmon_channel_info(struct hl_device *hdev, struct cpucp_sensor *sensors_arr); +void hl_notifier_event_send_all(struct hl_device *hdev, u64 event); + int hl_sysfs_init(struct hl_device *hdev); void hl_sysfs_fini(struct hl_device *hdev); int hl_hwmon_init(struct hl_device *hdev); void hl_hwmon_fini(struct hl_device *hdev); -int hl_cb_create(struct hl_device *hdev, struct hl_cb_mgr *mgr, +int hl_cb_create(struct hl_device *hdev, struct hl_mem_mgr *mmg, struct hl_ctx *ctx, u32 cb_size, bool internal_cb, bool map_cb, u64 *handle); -int hl_cb_destroy(struct hl_device *hdev, struct hl_cb_mgr *mgr, u64 cb_handle); -int hl_cb_mmap(struct hl_fpriv *hpriv, struct vm_area_struct *vma); +int hl_cb_destroy(struct hl_mem_mgr *mmg, u64 cb_handle); int hl_hw_block_mmap(struct hl_fpriv *hpriv, struct vm_area_struct *vma); -struct hl_cb *hl_cb_get(struct hl_device *hdev, struct hl_cb_mgr *mgr, - u32 handle); +struct hl_cb *hl_cb_get(struct hl_mem_mgr *mmg, u64 handle); void hl_cb_put(struct hl_cb *cb); -void hl_cb_mgr_init(struct hl_cb_mgr *mgr); -void hl_cb_mgr_fini(struct hl_device *hdev, struct hl_cb_mgr *mgr); struct hl_cb *hl_cb_kernel_create(struct hl_device *hdev, u32 cb_size, bool internal_cb); int hl_cb_pool_init(struct hl_device *hdev); @@ -3104,6 +3187,8 @@ int hl_mmu_ctx_init(struct hl_ctx *ctx); void hl_mmu_ctx_fini(struct hl_ctx *ctx); int hl_mmu_map_page(struct hl_ctx *ctx, u64 virt_addr, u64 phys_addr, u32 page_size, bool flush_pte); +int hl_mmu_get_real_page_size(struct hl_device *hdev, struct hl_mmu_properties *mmu_prop, + u32 page_size, u32 *real_page_size, bool is_dram_addr); int hl_mmu_unmap_page(struct hl_ctx *ctx, u64 virt_addr, u32 page_size, bool flush_pte); int hl_mmu_map_contiguous(struct hl_ctx *ctx, u64 virt_addr, @@ -3112,6 +3197,7 @@ int hl_mmu_unmap_contiguous(struct hl_ctx *ctx, u64 virt_addr, u32 size); int hl_mmu_invalidate_cache(struct hl_device *hdev, bool is_hard, u32 flags); int hl_mmu_invalidate_cache_range(struct hl_device *hdev, bool is_hard, u32 flags, u32 asid, u64 va, u64 size); +int hl_mmu_prefetch_cache_range(struct hl_ctx *ctx, u32 flags, u32 asid, u64 va, u64 size); u64 hl_mmu_get_next_hop_addr(struct hl_ctx *ctx, u64 curr_pte); u64 hl_mmu_get_hop_pte_phys_addr(struct hl_ctx *ctx, struct hl_mmu_properties *mmu_prop, u8 hop_idx, u64 hop_addr, u64 virt_addr); @@ -3149,6 +3235,7 @@ int hl_fw_cpucp_handshake(struct hl_device *hdev, u32 sts_boot_dev_sts1_reg, u32 boot_err0_reg, u32 boot_err1_reg); int hl_fw_get_eeprom_data(struct hl_device *hdev, void *data, size_t max_size); +int hl_fw_get_monitor_dump(struct hl_device *hdev, void *data); int hl_fw_cpucp_pci_counters_get(struct hl_device *hdev, struct hl_info_pci_counters *counters); int hl_fw_cpucp_total_energy_get(struct hl_device *hdev, @@ -3224,11 +3311,19 @@ __printf(4, 5) int hl_snprintf_resize(char **buf, size_t *size, size_t *offset, const char *format, ...); char *hl_format_as_binary(char *buf, size_t buf_len, u32 n); const char *hl_sync_engine_to_string(enum hl_sync_engine_type engine_type); -void hl_ts_mgr_init(struct hl_ts_mgr *mgr); -void hl_ts_mgr_fini(struct hl_device *hdev, struct hl_ts_mgr *mgr); -int hl_ts_mmap(struct hl_fpriv *hpriv, struct vm_area_struct *vma); -struct hl_ts_buff *hl_ts_get(struct hl_device *hdev, struct hl_ts_mgr *mgr, u32 handle); -void hl_ts_put(struct hl_ts_buff *buff); + +void hl_mem_mgr_init(struct device *dev, struct hl_mem_mgr *mmg); +void hl_mem_mgr_fini(struct hl_mem_mgr *mmg); +int hl_mem_mgr_mmap(struct hl_mem_mgr *mmg, struct vm_area_struct *vma, + void *args); +struct hl_mmap_mem_buf *hl_mmap_mem_buf_get(struct hl_mem_mgr *mmg, + u64 handle); +int hl_mmap_mem_buf_put_handle(struct hl_mem_mgr *mmg, u64 handle); +int hl_mmap_mem_buf_put(struct hl_mmap_mem_buf *buf); +struct hl_mmap_mem_buf * +hl_mmap_mem_buf_alloc(struct hl_mem_mgr *mmg, + struct hl_mmap_mem_buf_behavior *behavior, gfp_t gfp, + void *args); #ifdef CONFIG_DEBUG_FS |