// SPDX-License-Identifier: GPL-2.0 /* * Copyright 2016-2021 HabanaLabs, Ltd. * All Rights Reserved. */ #include #include "habanalabs.h" #include "../include/hw_ip/mmu/mmu_general.h" #include #include #include #include MODULE_IMPORT_NS(DMA_BUF); #define HL_MMU_DEBUG 0 /* use small pages for supporting non-pow2 (32M/40M/48M) DRAM phys page sizes */ #define DRAM_POOL_PAGE_SIZE SZ_8M static int allocate_timestamps_buffers(struct hl_fpriv *hpriv, struct hl_mem_in *args, u64 *handle); static int set_alloc_page_size(struct hl_device *hdev, struct hl_mem_in *args, u32 *page_size) { struct asic_fixed_properties *prop = &hdev->asic_prop; u32 psize; /* * for ASIC that supports setting the allocation page size by user we will address * user's choice only if it is not 0 (as 0 means taking the default page size) */ if (prop->supports_user_set_page_size && args->alloc.page_size) { psize = args->alloc.page_size; if (!hdev->asic_funcs->is_valid_dram_page_size(psize)) { dev_err(hdev->dev, "user page size (%#x) is not valid\n", psize); return -EINVAL; } } else { psize = prop->device_mem_alloc_default_page_size; } *page_size = psize; return 0; } /* * The va ranges in context object contain a list with the available chunks of * device virtual memory. * There is one range for host allocations and one for DRAM allocations. * * On initialization each range contains one chunk of all of its available * virtual range which is a half of the total device virtual range. * * On each mapping of physical pages, a suitable virtual range chunk (with a * minimum size) is selected from the list. If the chunk size equals the * requested size, the chunk is returned. Otherwise, the chunk is split into * two chunks - one to return as result and a remainder to stay in the list. * * On each Unmapping of a virtual address, the relevant virtual chunk is * returned to the list. The chunk is added to the list and if its edges match * the edges of the adjacent chunks (means a contiguous chunk can be created), * the chunks are merged. * * On finish, the list is checked to have only one chunk of all the relevant * virtual range (which is a half of the device total virtual range). * If not (means not all mappings were unmapped), a warning is printed. */ /* * alloc_device_memory() - allocate device memory. * @ctx: pointer to the context structure. * @args: host parameters containing the requested size. * @ret_handle: result handle. * * This function does the following: * - Allocate the requested size rounded up to 'dram_page_size' pages. * - Return unique handle for later map/unmap/free. */ static int alloc_device_memory(struct hl_ctx *ctx, struct hl_mem_in *args, u32 *ret_handle) { struct hl_device *hdev = ctx->hdev; struct hl_vm *vm = &hdev->vm; struct hl_vm_phys_pg_pack *phys_pg_pack; u64 paddr = 0, total_size, num_pgs, i; u32 num_curr_pgs, page_size; bool contiguous; int handle, rc; num_curr_pgs = 0; rc = set_alloc_page_size(hdev, args, &page_size); if (rc) return rc; num_pgs = DIV_ROUND_UP_ULL(args->alloc.mem_size, page_size); total_size = num_pgs * page_size; if (!total_size) { dev_err(hdev->dev, "Cannot allocate 0 bytes\n"); return -EINVAL; } contiguous = args->flags & HL_MEM_CONTIGUOUS; if (contiguous) { if (is_power_of_2(page_size)) paddr = (uintptr_t) gen_pool_dma_alloc_align(vm->dram_pg_pool, total_size, NULL, page_size); else paddr = gen_pool_alloc(vm->dram_pg_pool, total_size); if (!paddr) { dev_err(hdev->dev, "Cannot allocate %llu contiguous pages with total size of %llu\n", num_pgs, total_size); return -ENOMEM; } } phys_pg_pack = kzalloc(sizeof(*phys_pg_pack), GFP_KERNEL); if (!phys_pg_pack) { rc = -ENOMEM; goto pages_pack_err; } phys_pg_pack->vm_type = VM_TYPE_PHYS_PACK; phys_pg_pack->asid = ctx->asid; phys_pg_pack->npages = num_pgs; phys_pg_pack->page_size = page_size; phys_pg_pack->total_size = total_size; phys_pg_pack->flags = args->flags; phys_pg_pack->contiguous = contiguous; phys_pg_pack->pages = kvmalloc_array(num_pgs, sizeof(u64), GFP_KERNEL); if (ZERO_OR_NULL_PTR(phys_pg_pack->pages)) { rc = -ENOMEM; goto pages_arr_err; } if (phys_pg_pack->contiguous) { for (i = 0 ; i < num_pgs ; i++) phys_pg_pack->pages[i] = paddr + i * page_size; } else { for (i = 0 ; i < num_pgs ; i++) { if (is_power_of_2(page_size)) phys_pg_pack->pages[i] = (uintptr_t)gen_pool_dma_alloc_align(vm->dram_pg_pool, page_size, NULL, page_size); else phys_pg_pack->pages[i] = gen_pool_alloc(vm->dram_pg_pool, page_size); if (!phys_pg_pack->pages[i]) { dev_err(hdev->dev, "Cannot allocate device memory (out of memory)\n"); rc = -ENOMEM; goto page_err; } num_curr_pgs++; } } spin_lock(&vm->idr_lock); handle = idr_alloc(&vm->phys_pg_pack_handles, phys_pg_pack, 1, 0, GFP_ATOMIC); spin_unlock(&vm->idr_lock); if (handle < 0) { dev_err(hdev->dev, "Failed to get handle for page\n"); rc = -EFAULT; goto idr_err; } for (i = 0 ; i < num_pgs ; i++) kref_get(&vm->dram_pg_pool_refcount); phys_pg_pack->handle = handle; atomic64_add(phys_pg_pack->total_size, &ctx->dram_phys_mem); atomic64_add(phys_pg_pack->total_size, &hdev->dram_used_mem); *ret_handle = handle; return 0; idr_err: page_err: if (!phys_pg_pack->contiguous) for (i = 0 ; i < num_curr_pgs ; i++) gen_pool_free(vm->dram_pg_pool, phys_pg_pack->pages[i], page_size); kvfree(phys_pg_pack->pages); pages_arr_err: kfree(phys_pg_pack); pages_pack_err: if (contiguous) gen_pool_free(vm->dram_pg_pool, paddr, total_size); return rc; } /** * dma_map_host_va() - DMA mapping of the given host virtual address. * @hdev: habanalabs device structure. * @addr: the host virtual address of the memory area. * @size: the size of the memory area. * @p_userptr: pointer to result userptr structure. * * This function does the following: * - Allocate userptr structure. * - Pin the given host memory using the userptr structure. * - Perform DMA mapping to have the DMA addresses of the pages. */ static int dma_map_host_va(struct hl_device *hdev, u64 addr, u64 size, struct hl_userptr **p_userptr) { struct hl_userptr *userptr; int rc; userptr = kzalloc(sizeof(*userptr), GFP_KERNEL); if (!userptr) { rc = -ENOMEM; goto userptr_err; } rc = hl_pin_host_memory(hdev, addr, size, userptr); if (rc) { dev_err(hdev->dev, "Failed to pin host memory\n"); goto pin_err; } userptr->dma_mapped = true; userptr->dir = DMA_BIDIRECTIONAL; userptr->vm_type = VM_TYPE_USERPTR; *p_userptr = userptr; rc = hdev->asic_funcs->asic_dma_map_sgtable(hdev, userptr->sgt, DMA_BIDIRECTIONAL); if (rc) { dev_err(hdev->dev, "failed to map sgt with DMA region\n"); goto dma_map_err; } return 0; dma_map_err: hl_unpin_host_memory(hdev, userptr); pin_err: kfree(userptr); userptr_err: return rc; } /** * dma_unmap_host_va() - DMA unmapping of the given host virtual address. * @hdev: habanalabs device structure. * @userptr: userptr to free. * * This function does the following: * - Unpins the physical pages. * - Frees the userptr structure. */ static void dma_unmap_host_va(struct hl_device *hdev, struct hl_userptr *userptr) { hl_unpin_host_memory(hdev, userptr); kfree(userptr); } /** * dram_pg_pool_do_release() - free DRAM pages pool * @ref: pointer to reference object. * * This function does the following: * - Frees the idr structure of physical pages handles. * - Frees the generic pool of DRAM physical pages. */ static void dram_pg_pool_do_release(struct kref *ref) { struct hl_vm *vm = container_of(ref, struct hl_vm, dram_pg_pool_refcount); /* * free the idr here as only here we know for sure that there are no * allocated physical pages and hence there are no handles in use */ idr_destroy(&vm->phys_pg_pack_handles); gen_pool_destroy(vm->dram_pg_pool); } /** * free_phys_pg_pack() - free physical page pack. * @hdev: habanalabs device structure. * @phys_pg_pack: physical page pack to free. * * This function does the following: * - For DRAM memory only * - iterate over the pack, scrub and free each physical block structure by * returning it to the general pool. * In case of error during scrubbing, initiate hard reset. * Once hard reset is triggered, scrubbing is bypassed while freeing the * memory continues. * - Free the hl_vm_phys_pg_pack structure. */ static int free_phys_pg_pack(struct hl_device *hdev, struct hl_vm_phys_pg_pack *phys_pg_pack) { struct hl_vm *vm = &hdev->vm; u64 i; int rc = 0; if (phys_pg_pack->created_from_userptr) goto end; if (phys_pg_pack->contiguous) { if (hdev->memory_scrub && !hdev->disabled) { rc = hdev->asic_funcs->scrub_device_mem(hdev, phys_pg_pack->pages[0], phys_pg_pack->total_size); if (rc) dev_err(hdev->dev, "Failed to scrub contiguous device memory\n"); } gen_pool_free(vm->dram_pg_pool, phys_pg_pack->pages[0], phys_pg_pack->total_size); for (i = 0; i < phys_pg_pack->npages ; i++) kref_put(&vm->dram_pg_pool_refcount, dram_pg_pool_do_release); } else { for (i = 0 ; i < phys_pg_pack->npages ; i++) { if (hdev->memory_scrub && !hdev->disabled && rc == 0) { rc = hdev->asic_funcs->scrub_device_mem( hdev, phys_pg_pack->pages[i], phys_pg_pack->page_size); if (rc) dev_err(hdev->dev, "Failed to scrub device memory\n"); } gen_pool_free(vm->dram_pg_pool, phys_pg_pack->pages[i], phys_pg_pack->page_size); kref_put(&vm->dram_pg_pool_refcount, dram_pg_pool_do_release); } } if (rc && !hdev->disabled) hl_device_reset(hdev, HL_DRV_RESET_HARD); end: kvfree(phys_pg_pack->pages); kfree(phys_pg_pack); return rc; } /** * free_device_memory() - free device memory. * @ctx: pointer to the context structure. * @args: host parameters containing the requested size. * * This function does the following: * - Free the device memory related to the given handle. */ static int free_device_memory(struct hl_ctx *ctx, struct hl_mem_in *args) { struct hl_device *hdev = ctx->hdev; struct hl_vm *vm = &hdev->vm; struct hl_vm_phys_pg_pack *phys_pg_pack; u32 handle = args->free.handle; spin_lock(&vm->idr_lock); phys_pg_pack = idr_find(&vm->phys_pg_pack_handles, handle); if (phys_pg_pack) { if (atomic_read(&phys_pg_pack->mapping_cnt) > 0) { dev_err(hdev->dev, "handle %u is mapped, cannot free\n", handle); spin_unlock(&vm->idr_lock); return -EINVAL; } if (phys_pg_pack->exporting_cnt) { dev_dbg(hdev->dev, "handle %u is exported, cannot free\n", handle); spin_unlock(&vm->idr_lock); return -EINVAL; } /* * must remove from idr before the freeing of the physical * pages as the refcount of the pool is also the trigger of the * idr destroy */ idr_remove(&vm->phys_pg_pack_handles, handle); spin_unlock(&vm->idr_lock); atomic64_sub(phys_pg_pack->total_size, &ctx->dram_phys_mem); atomic64_sub(phys_pg_pack->total_size, &hdev->dram_used_mem); return free_phys_pg_pack(hdev, phys_pg_pack); } else { spin_unlock(&vm->idr_lock); dev_err(hdev->dev, "free device memory failed, no match for handle %u\n", handle); return -EINVAL; } return 0; } /** * clear_va_list_locked() - free virtual addresses list. * @hdev: habanalabs device structure. * @va_list: list of virtual addresses to free. * * This function does the following: * - Iterate over the list and free each virtual addresses block. * * This function should be called only when va_list lock is taken. */ static void clear_va_list_locked(struct hl_device *hdev, struct list_head *va_list) { struct hl_vm_va_block *va_block, *tmp; list_for_each_entry_safe(va_block, tmp, va_list, node) { list_del(&va_block->node); kfree(va_block); } } /** * print_va_list_locked() - print virtual addresses list. * @hdev: habanalabs device structure. * @va_list: list of virtual addresses to print. * * This function does the following: * - Iterate over the list and print each virtual addresses block. * * This function should be called only when va_list lock is taken. */ static void print_va_list_locked(struct hl_device *hdev, struct list_head *va_list) { #if HL_MMU_DEBUG struct hl_vm_va_block *va_block; dev_dbg(hdev->dev, "print va list:\n"); list_for_each_entry(va_block, va_list, node) dev_dbg(hdev->dev, "va block, start: 0x%llx, end: 0x%llx, size: %llu\n", va_block->start, va_block->end, va_block->size); #endif } /** * merge_va_blocks_locked() - merge a virtual block if possible. * @hdev: pointer to the habanalabs device structure. * @va_list: pointer to the virtual addresses block list. * @va_block: virtual block to merge with adjacent blocks. * * This function does the following: * - Merge the given blocks with the adjacent blocks if their virtual ranges * create a contiguous virtual range. * * This Function should be called only when va_list lock is taken. */ static void merge_va_blocks_locked(struct hl_device *hdev, struct list_head *va_list, struct hl_vm_va_block *va_block) { struct hl_vm_va_block *prev, *next; prev = list_prev_entry(va_block, node); if (&prev->node != va_list && prev->end + 1 == va_block->start) { prev->end = va_block->end; prev->size = prev->end - prev->start; list_del(&va_block->node); kfree(va_block); va_block = prev; } next = list_next_entry(va_block, node); if (&next->node != va_list && va_block->end + 1 == next->start) { next->start = va_block->start; next->size = next->end - next->start; list_del(&va_block->node); kfree(va_block); } } /** * add_va_block_locked() - add a virtual block to the virtual addresses list. * @hdev: pointer to the habanalabs device structure. * @va_list: pointer to the virtual addresses block list. * @start: start virtual address. * @end: end virtual address. * * This function does the following: * - Add the given block to the virtual blocks list and merge with other blocks * if a contiguous virtual block can be created. * * This Function should be called only when va_list lock is taken. */ static int add_va_block_locked(struct hl_device *hdev, struct list_head *va_list, u64 start, u64 end) { struct hl_vm_va_block *va_block, *res = NULL; u64 size = end - start + 1; print_va_list_locked(hdev, va_list); list_for_each_entry(va_block, va_list, node) { /* TODO: remove upon matureness */ if (hl_mem_area_crosses_range(start, size, va_block->start, va_block->end)) { dev_err(hdev->dev, "block crossing ranges at start 0x%llx, end 0x%llx\n", va_block->start, va_block->end); return -EINVAL; } if (va_block->end < start) res = va_block; } va_block = kmalloc(sizeof(*va_block), GFP_KERNEL); if (!va_block) return -ENOMEM; va_block->start = start; va_block->end = end; va_block->size = size; if (!res) list_add(&va_block->node, va_list); else list_add(&va_block->node, &res->node); merge_va_blocks_locked(hdev, va_list, va_block); print_va_list_locked(hdev, va_list); return 0; } /** * add_va_block() - wrapper for add_va_block_locked. * @hdev: pointer to the habanalabs device structure. * @va_range: pointer to the virtual addresses range object. * @start: start virtual address. * @end: end virtual address. * * This function does the following: * - Takes the list lock and calls add_va_block_locked. */ static inline int add_va_block(struct hl_device *hdev, struct hl_va_range *va_range, u64 start, u64 end) { int rc; mutex_lock(&va_range->lock); rc = add_va_block_locked(hdev, &va_range->list, start, end); mutex_unlock(&va_range->lock); return rc; } /** * is_hint_crossing_range() - check if hint address crossing specified reserved. * @range_type: virtual space range type. * @start_addr: start virtual address. * @size: block size. * @prop: asic properties structure to retrieve reserved ranges from. */ static inline bool is_hint_crossing_range(enum hl_va_range_type range_type, u64 start_addr, u32 size, struct asic_fixed_properties *prop) { bool range_cross; if (range_type == HL_VA_RANGE_TYPE_DRAM) range_cross = hl_mem_area_crosses_range(start_addr, size, prop->hints_dram_reserved_va_range.start_addr, prop->hints_dram_reserved_va_range.end_addr); else if (range_type == HL_VA_RANGE_TYPE_HOST) range_cross = hl_mem_area_crosses_range(start_addr, size, prop->hints_host_reserved_va_range.start_addr, prop->hints_host_reserved_va_range.end_addr); else range_cross = hl_mem_area_crosses_range(start_addr, size, prop->hints_host_hpage_reserved_va_range.start_addr, prop->hints_host_hpage_reserved_va_range.end_addr); return range_cross; } /** * get_va_block() - get a virtual block for the given size and alignment. * * @hdev: pointer to the habanalabs device structure. * @va_range: pointer to the virtual addresses range. * @size: requested block size. * @hint_addr: hint for requested address by the user. * @va_block_align: required alignment of the virtual block start address. * @range_type: va range type (host, dram) * @flags: additional memory flags, currently only uses HL_MEM_FORCE_HINT * * This function does the following: * - Iterate on the virtual block list to find a suitable virtual block for the * given size, hint address and alignment. * - Reserve the requested block and update the list. * - Return the start address of the virtual block. */ static u64 get_va_block(struct hl_device *hdev, struct hl_va_range *va_range, u64 size, u64 hint_addr, u32 va_block_align, enum hl_va_range_type range_type, u32 flags) { struct hl_vm_va_block *va_block, *new_va_block = NULL; struct asic_fixed_properties *prop = &hdev->asic_prop; u64 tmp_hint_addr, valid_start, valid_size, prev_start, prev_end, align_mask, reserved_valid_start = 0, reserved_valid_size = 0, dram_hint_mask = prop->dram_hints_align_mask; bool add_prev = false; bool is_align_pow_2 = is_power_of_2(va_range->page_size); bool is_hint_dram_addr = hl_is_dram_va(hdev, hint_addr); bool force_hint = flags & HL_MEM_FORCE_HINT; if (is_align_pow_2) align_mask = ~((u64)va_block_align - 1); else /* * with non-power-of-2 range we work only with page granularity * and the start address is page aligned, * so no need for alignment checking. */ size = DIV_ROUND_UP_ULL(size, va_range->page_size) * va_range->page_size; tmp_hint_addr = hint_addr & ~dram_hint_mask; /* Check if we need to ignore hint address */ if ((is_align_pow_2 && (hint_addr & (va_block_align - 1))) || (!is_align_pow_2 && is_hint_dram_addr && do_div(tmp_hint_addr, va_range->page_size))) { if (force_hint) { /* Hint must be respected, so here we just fail */ dev_err(hdev->dev, "Hint address 0x%llx is not page aligned - cannot be respected\n", hint_addr); return 0; } dev_dbg(hdev->dev, "Hint address 0x%llx will be ignored because it is not aligned\n", hint_addr); hint_addr = 0; } mutex_lock(&va_range->lock); print_va_list_locked(hdev, &va_range->list); list_for_each_entry(va_block, &va_range->list, node) { /* Calc the first possible aligned addr */ valid_start = va_block->start; if (is_align_pow_2 && (valid_start & (va_block_align - 1))) { valid_start &= align_mask; valid_start += va_block_align; if (valid_start > va_block->end) continue; } valid_size = va_block->end - valid_start + 1; if (valid_size < size) continue; /* * In case hint address is 0, and hints_range_reservation * property enabled, then avoid allocating va blocks from the * range reserved for hint addresses */ if (prop->hints_range_reservation && !hint_addr) if (is_hint_crossing_range(range_type, valid_start, size, prop)) continue; /* Pick the minimal length block which has the required size */ if (!new_va_block || (valid_size < reserved_valid_size)) { new_va_block = va_block; reserved_valid_start = valid_start; reserved_valid_size = valid_size; } if (hint_addr && hint_addr >= valid_start && (hint_addr + size) <= va_block->end) { new_va_block = va_block; reserved_valid_start = hint_addr; reserved_valid_size = valid_size; break; } } if (!new_va_block) { dev_err(hdev->dev, "no available va block for size %llu\n", size); goto out; } if (force_hint && reserved_valid_start != hint_addr) { /* Hint address must be respected. If we are here - this means * we could not respect it. */ dev_err(hdev->dev, "Hint address 0x%llx could not be respected\n", hint_addr); reserved_valid_start = 0; goto out; } /* * Check if there is some leftover range due to reserving the new * va block, then return it to the main virtual addresses list. */ if (reserved_valid_start > new_va_block->start) { prev_start = new_va_block->start; prev_end = reserved_valid_start - 1; new_va_block->start = reserved_valid_start; new_va_block->size = reserved_valid_size; add_prev = true; } if (new_va_block->size > size) { new_va_block->start += size; new_va_block->size = new_va_block->end - new_va_block->start + 1; } else { list_del(&new_va_block->node); kfree(new_va_block); } if (add_prev) add_va_block_locked(hdev, &va_range->list, prev_start, prev_end); print_va_list_locked(hdev, &va_range->list); out: mutex_unlock(&va_range->lock); return reserved_valid_start; } /* * hl_reserve_va_block() - reserve a virtual block of a given size. * @hdev: pointer to the habanalabs device structure. * @ctx: current context * @type: virtual addresses range type. * @size: requested block size. * @alignment: required alignment in bytes of the virtual block start address, * 0 means no alignment. * * This function does the following: * - Iterate on the virtual block list to find a suitable virtual block for the * given size and alignment. * - Reserve the requested block and update the list. * - Return the start address of the virtual block. */ u64 hl_reserve_va_block(struct hl_device *hdev, struct hl_ctx *ctx, enum hl_va_range_type type, u32 size, u32 alignment) { return get_va_block(hdev, ctx->va_range[type], size, 0, max(alignment, ctx->va_range[type]->page_size), type, 0); } /** * hl_get_va_range_type() - get va_range type for the given address and size. * @ctx: context to fetch va_range from. * @address: the start address of the area we want to validate. * @size: the size in bytes of the area we want to validate. * @type: returned va_range type. * * Return: true if the area is inside a valid range, false otherwise. */ static int hl_get_va_range_type(struct hl_ctx *ctx, u64 address, u64 size, enum hl_va_range_type *type) { int i; for (i = 0 ; i < HL_VA_RANGE_TYPE_MAX; i++) { if (hl_mem_area_inside_range(address, size, ctx->va_range[i]->start_addr, ctx->va_range[i]->end_addr)) { *type = i; return 0; } } return -EINVAL; } /** * hl_unreserve_va_block() - wrapper for add_va_block to unreserve a va block. * @hdev: pointer to the habanalabs device structure * @ctx: pointer to the context structure. * @start_addr: start virtual address. * @size: number of bytes to unreserve. * * This function does the following: * - Takes the list lock and calls add_va_block_locked. */ int hl_unreserve_va_block(struct hl_device *hdev, struct hl_ctx *ctx, u64 start_addr, u64 size) { enum hl_va_range_type type; int rc; rc = hl_get_va_range_type(ctx, start_addr, size, &type); if (rc) { dev_err(hdev->dev, "cannot find va_range for va %#llx size %llu", start_addr, size); return rc; } rc = add_va_block(hdev, ctx->va_range[type], start_addr, start_addr + size - 1); if (rc) dev_warn(hdev->dev, "add va block failed for vaddr: 0x%llx\n", start_addr); return rc; } /** * init_phys_pg_pack_from_userptr() - initialize physical page pack from host * memory * @ctx: pointer to the context structure. * @userptr: userptr to initialize from. * @pphys_pg_pack: result pointer. * @force_regular_page: tell the function to ignore huge page optimization, * even if possible. Needed for cases where the device VA * is allocated before we know the composition of the * physical pages * * This function does the following: * - Pin the physical pages related to the given virtual block. * - Create a physical page pack from the physical pages related to the given * virtual block. */ static int init_phys_pg_pack_from_userptr(struct hl_ctx *ctx, struct hl_userptr *userptr, struct hl_vm_phys_pg_pack **pphys_pg_pack, bool force_regular_page) { u32 npages, page_size = PAGE_SIZE, huge_page_size = ctx->hdev->asic_prop.pmmu_huge.page_size; u32 pgs_in_huge_page = huge_page_size >> __ffs(page_size); struct hl_vm_phys_pg_pack *phys_pg_pack; bool first = true, is_huge_page_opt; u64 page_mask, total_npages; struct scatterlist *sg; dma_addr_t dma_addr; int rc, i, j; phys_pg_pack = kzalloc(sizeof(*phys_pg_pack), GFP_KERNEL); if (!phys_pg_pack) return -ENOMEM; phys_pg_pack->vm_type = userptr->vm_type; phys_pg_pack->created_from_userptr = true; phys_pg_pack->asid = ctx->asid; atomic_set(&phys_pg_pack->mapping_cnt, 1); is_huge_page_opt = (force_regular_page ? false : true); /* Only if all dma_addrs are aligned to 2MB and their * sizes is at least 2MB, we can use huge page mapping. * We limit the 2MB optimization to this condition, * since later on we acquire the related VA range as one * consecutive block. */ total_npages = 0; for_each_sgtable_dma_sg(userptr->sgt, sg, i) { npages = hl_get_sg_info(sg, &dma_addr); total_npages += npages; if ((npages % pgs_in_huge_page) || (dma_addr & (huge_page_size - 1))) is_huge_page_opt = false; } if (is_huge_page_opt) { page_size = huge_page_size; do_div(total_npages, pgs_in_huge_page); } page_mask = ~(((u64) page_size) - 1); phys_pg_pack->pages = kvmalloc_array(total_npages, sizeof(u64), GFP_KERNEL); if (ZERO_OR_NULL_PTR(phys_pg_pack->pages)) { rc = -ENOMEM; goto page_pack_arr_mem_err; } phys_pg_pack->npages = total_npages; phys_pg_pack->page_size = page_size; phys_pg_pack->total_size = total_npages * page_size; j = 0; for_each_sgtable_dma_sg(userptr->sgt, sg, i) { npages = hl_get_sg_info(sg, &dma_addr); /* align down to physical page size and save the offset */ if (first) { first = false; phys_pg_pack->offset = dma_addr & (page_size - 1); dma_addr &= page_mask; } while (npages) { phys_pg_pack->pages[j++] = dma_addr; dma_addr += page_size; if (is_huge_page_opt) npages -= pgs_in_huge_page; else npages--; } } *pphys_pg_pack = phys_pg_pack; return 0; page_pack_arr_mem_err: kfree(phys_pg_pack); return rc; } /** * map_phys_pg_pack() - maps the physical page pack.. * @ctx: pointer to the context structure. * @vaddr: start address of the virtual area to map from. * @phys_pg_pack: the pack of physical pages to map to. * * This function does the following: * - Maps each chunk of virtual memory to matching physical chunk. * - Stores number of successful mappings in the given argument. * - Returns 0 on success, error code otherwise. */ static int map_phys_pg_pack(struct hl_ctx *ctx, u64 vaddr, struct hl_vm_phys_pg_pack *phys_pg_pack) { struct hl_device *hdev = ctx->hdev; u64 next_vaddr = vaddr, paddr, mapped_pg_cnt = 0, i; u32 page_size = phys_pg_pack->page_size; int rc = 0; bool is_host_addr; for (i = 0 ; i < phys_pg_pack->npages ; i++) { paddr = phys_pg_pack->pages[i]; rc = hl_mmu_map_page(ctx, next_vaddr, paddr, page_size, (i + 1) == phys_pg_pack->npages); if (rc) { dev_err(hdev->dev, "map failed for handle %u, npages: %llu, mapped: %llu", phys_pg_pack->handle, phys_pg_pack->npages, mapped_pg_cnt); goto err; } mapped_pg_cnt++; next_vaddr += page_size; } return 0; err: is_host_addr = !hl_is_dram_va(hdev, vaddr); next_vaddr = vaddr; for (i = 0 ; i < mapped_pg_cnt ; i++) { if (hl_mmu_unmap_page(ctx, next_vaddr, page_size, (i + 1) == mapped_pg_cnt)) dev_warn_ratelimited(hdev->dev, "failed to unmap handle %u, va: 0x%llx, pa: 0x%llx, page size: %u\n", phys_pg_pack->handle, next_vaddr, phys_pg_pack->pages[i], page_size); next_vaddr += page_size; /* * unmapping on Palladium can be really long, so avoid a CPU * soft lockup bug by sleeping a little between unmapping pages * * In addition, on host num of pages could be huge, * because page size could be 4KB, so when unmapping host * pages sleep every 32K pages to avoid soft lockup */ if (hdev->pldm || (is_host_addr && (i & 0x7FFF) == 0)) usleep_range(50, 200); } return rc; } /** * unmap_phys_pg_pack() - unmaps the physical page pack. * @ctx: pointer to the context structure. * @vaddr: start address of the virtual area to unmap. * @phys_pg_pack: the pack of physical pages to unmap. */ static void unmap_phys_pg_pack(struct hl_ctx *ctx, u64 vaddr, struct hl_vm_phys_pg_pack *phys_pg_pack) { struct hl_device *hdev = ctx->hdev; u64 next_vaddr, i; bool is_host_addr; u32 page_size; is_host_addr = !hl_is_dram_va(hdev, vaddr); page_size = phys_pg_pack->page_size; next_vaddr = vaddr; for (i = 0 ; i < phys_pg_pack->npages ; i++, next_vaddr += page_size) { if (hl_mmu_unmap_page(ctx, next_vaddr, page_size, (i + 1) == phys_pg_pack->npages)) dev_warn_ratelimited(hdev->dev, "unmap failed for vaddr: 0x%llx\n", next_vaddr); /* * unmapping on Palladium can be really long, so avoid a CPU * soft lockup bug by sleeping a little between unmapping pages * * In addition, on host num of pages could be huge, * because page size could be 4KB, so when unmapping host * pages sleep every 32K pages to avoid soft lockup */ if (hdev->pldm || (is_host_addr && (i & 0x7FFF) == 0)) usleep_range(50, 200); } } static int get_paddr_from_handle(struct hl_ctx *ctx, struct hl_mem_in *args, u64 *paddr) { struct hl_device *hdev = ctx->hdev; struct hl_vm *vm = &hdev->vm; struct hl_vm_phys_pg_pack *phys_pg_pack; u32 handle; handle = lower_32_bits(args->map_device.handle); spin_lock(&vm->idr_lock); phys_pg_pack = idr_find(&vm->phys_pg_pack_handles, handle); if (!phys_pg_pack) { spin_unlock(&vm->idr_lock); dev_err(hdev->dev, "no match for handle %u\n", handle); return -EINVAL; } *paddr = phys_pg_pack->pages[0]; spin_unlock(&vm->idr_lock); return 0; } /** * map_device_va() - map the given memory. * @ctx: pointer to the context structure. * @args: host parameters with handle/host virtual address. * @device_addr: pointer to result device virtual address. * * This function does the following: * - If given a physical device memory handle, map to a device virtual block * and return the start address of this block. * - If given a host virtual address and size, find the related physical pages, * map a device virtual block to this pages and return the start address of * this block. */ static int map_device_va(struct hl_ctx *ctx, struct hl_mem_in *args, u64 *device_addr) { struct hl_device *hdev = ctx->hdev; struct hl_vm *vm = &hdev->vm; struct hl_vm_phys_pg_pack *phys_pg_pack; struct hl_userptr *userptr = NULL; struct hl_vm_hash_node *hnode; struct hl_va_range *va_range; enum vm_type *vm_type; u64 ret_vaddr, hint_addr; u32 handle = 0, va_block_align; int rc; bool is_userptr = args->flags & HL_MEM_USERPTR; enum hl_va_range_type va_range_type = 0; /* Assume failure */ *device_addr = 0; if (is_userptr) { u64 addr = args->map_host.host_virt_addr, size = args->map_host.mem_size; u32 page_size = hdev->asic_prop.pmmu.page_size, huge_page_size = hdev->asic_prop.pmmu_huge.page_size; rc = dma_map_host_va(hdev, addr, size, &userptr); if (rc) { dev_err(hdev->dev, "failed to get userptr from va\n"); return rc; } rc = init_phys_pg_pack_from_userptr(ctx, userptr, &phys_pg_pack, false); if (rc) { dev_err(hdev->dev, "unable to init page pack for vaddr 0x%llx\n", addr); goto init_page_pack_err; } vm_type = (enum vm_type *) userptr; hint_addr = args->map_host.hint_addr; handle = phys_pg_pack->handle; /* get required alignment */ if (phys_pg_pack->page_size == page_size) { va_range = ctx->va_range[HL_VA_RANGE_TYPE_HOST]; va_range_type = HL_VA_RANGE_TYPE_HOST; /* * huge page alignment may be needed in case of regular * page mapping, depending on the host VA alignment */ if (addr & (huge_page_size - 1)) va_block_align = page_size; else va_block_align = huge_page_size; } else { /* * huge page alignment is needed in case of huge page * mapping */ va_range = ctx->va_range[HL_VA_RANGE_TYPE_HOST_HUGE]; va_range_type = HL_VA_RANGE_TYPE_HOST_HUGE; va_block_align = huge_page_size; } } else { handle = lower_32_bits(args->map_device.handle); spin_lock(&vm->idr_lock); phys_pg_pack = idr_find(&vm->phys_pg_pack_handles, handle); if (!phys_pg_pack) { spin_unlock(&vm->idr_lock); dev_err(hdev->dev, "no match for handle %u\n", handle); return -EINVAL; } /* increment now to avoid freeing device memory while mapping */ atomic_inc(&phys_pg_pack->mapping_cnt); spin_unlock(&vm->idr_lock); vm_type = (enum vm_type *) phys_pg_pack; hint_addr = args->map_device.hint_addr; /* DRAM VA alignment is the same as the MMU page size */ va_range = ctx->va_range[HL_VA_RANGE_TYPE_DRAM]; va_range_type = HL_VA_RANGE_TYPE_DRAM; va_block_align = hdev->asic_prop.dmmu.page_size; } /* * relevant for mapping device physical memory only, as host memory is * implicitly shared */ if (!is_userptr && !(phys_pg_pack->flags & HL_MEM_SHARED) && phys_pg_pack->asid != ctx->asid) { dev_err(hdev->dev, "Failed to map memory, handle %u is not shared\n", handle); rc = -EPERM; goto shared_err; } hnode = kzalloc(sizeof(*hnode), GFP_KERNEL); if (!hnode) { rc = -ENOMEM; goto hnode_err; } if (hint_addr && phys_pg_pack->offset) { if (args->flags & HL_MEM_FORCE_HINT) { /* Fail if hint must be respected but it can't be */ dev_err(hdev->dev, "Hint address 0x%llx cannot be respected because source memory is not aligned 0x%x\n", hint_addr, phys_pg_pack->offset); rc = -EINVAL; goto va_block_err; } dev_dbg(hdev->dev, "Hint address 0x%llx will be ignored because source memory is not aligned 0x%x\n", hint_addr, phys_pg_pack->offset); } ret_vaddr = get_va_block(hdev, va_range, phys_pg_pack->total_size, hint_addr, va_block_align, va_range_type, args->flags); if (!ret_vaddr) { dev_err(hdev->dev, "no available va block for handle %u\n", handle); rc = -ENOMEM; goto va_block_err; } mutex_lock(&ctx->mmu_lock); rc = map_phys_pg_pack(ctx, ret_vaddr, phys_pg_pack); if (rc) { dev_err(hdev->dev, "mapping page pack failed for handle %u\n", handle); goto map_err; } rc = hl_mmu_invalidate_cache_range(hdev, false, *vm_type | MMU_OP_SKIP_LOW_CACHE_INV, ctx->asid, ret_vaddr, phys_pg_pack->total_size); if (rc) goto map_err; /* already prefetch the relevant translations to the cache */ rc = hl_mmu_prefetch_cache_range(hdev, *vm_type, ctx->asid, ret_vaddr, phys_pg_pack->total_size); if (rc) goto map_err; mutex_unlock(&ctx->mmu_lock); ret_vaddr += phys_pg_pack->offset; hnode->ptr = vm_type; hnode->vaddr = ret_vaddr; mutex_lock(&ctx->mem_hash_lock); hash_add(ctx->mem_hash, &hnode->node, ret_vaddr); mutex_unlock(&ctx->mem_hash_lock); *device_addr = ret_vaddr; if (is_userptr) rc = free_phys_pg_pack(hdev, phys_pg_pack); return rc; map_err: mutex_unlock(&ctx->mmu_lock); if (add_va_block(hdev, va_range, ret_vaddr, ret_vaddr + phys_pg_pack->total_size - 1)) dev_warn(hdev->dev, "release va block failed for handle 0x%x, vaddr: 0x%llx\n", handle, ret_vaddr); va_block_err: kfree(hnode); hnode_err: shared_err: atomic_dec(&phys_pg_pack->mapping_cnt); if (is_userptr) free_phys_pg_pack(hdev, phys_pg_pack); init_page_pack_err: if (is_userptr) dma_unmap_host_va(hdev, userptr); return rc; } /** * unmap_device_va() - unmap the given device virtual address. * @ctx: pointer to the context structure. * @args: host parameters with device virtual address to unmap. * @ctx_free: true if in context free flow, false otherwise. * * This function does the following: * - unmap the physical pages related to the given virtual address. * - return the device virtual block to the virtual block list. */ static int unmap_device_va(struct hl_ctx *ctx, struct hl_mem_in *args, bool ctx_free) { struct hl_vm_phys_pg_pack *phys_pg_pack = NULL; u64 vaddr = args->unmap.device_virt_addr; struct hl_vm_hash_node *hnode = NULL; struct asic_fixed_properties *prop; struct hl_device *hdev = ctx->hdev; struct hl_userptr *userptr = NULL; struct hl_va_range *va_range; enum vm_type *vm_type; bool is_userptr; int rc = 0; prop = &hdev->asic_prop; /* protect from double entrance */ mutex_lock(&ctx->mem_hash_lock); hash_for_each_possible(ctx->mem_hash, hnode, node, (unsigned long)vaddr) if (vaddr == hnode->vaddr) break; if (!hnode) { mutex_unlock(&ctx->mem_hash_lock); dev_err(hdev->dev, "unmap failed, no mem hnode for vaddr 0x%llx\n", vaddr); return -EINVAL; } hash_del(&hnode->node); mutex_unlock(&ctx->mem_hash_lock); vm_type = hnode->ptr; if (*vm_type == VM_TYPE_USERPTR) { is_userptr = true; userptr = hnode->ptr; rc = init_phys_pg_pack_from_userptr(ctx, userptr, &phys_pg_pack, false); if (rc) { dev_err(hdev->dev, "unable to init page pack for vaddr 0x%llx\n", vaddr); goto vm_type_err; } if (phys_pg_pack->page_size == hdev->asic_prop.pmmu.page_size) va_range = ctx->va_range[HL_VA_RANGE_TYPE_HOST]; else va_range = ctx->va_range[HL_VA_RANGE_TYPE_HOST_HUGE]; } else if (*vm_type == VM_TYPE_PHYS_PACK) { is_userptr = false; va_range = ctx->va_range[HL_VA_RANGE_TYPE_DRAM]; phys_pg_pack = hnode->ptr; } else { dev_warn(hdev->dev, "unmap failed, unknown vm desc for vaddr 0x%llx\n", vaddr); rc = -EFAULT; goto vm_type_err; } if (atomic_read(&phys_pg_pack->mapping_cnt) == 0) { dev_err(hdev->dev, "vaddr 0x%llx is not mapped\n", vaddr); rc = -EINVAL; goto mapping_cnt_err; } if (!is_userptr && !is_power_of_2(phys_pg_pack->page_size)) vaddr = prop->dram_base_address + DIV_ROUND_DOWN_ULL(vaddr - prop->dram_base_address, phys_pg_pack->page_size) * phys_pg_pack->page_size; else vaddr &= ~(((u64) phys_pg_pack->page_size) - 1); mutex_lock(&ctx->mmu_lock); unmap_phys_pg_pack(ctx, vaddr, phys_pg_pack); /* * During context free this function is called in a loop to clean all * the context mappings. Hence the cache invalidation can be called once * at the loop end rather than for each iteration */ if (!ctx_free) rc = hl_mmu_invalidate_cache_range(hdev, true, *vm_type, ctx->asid, vaddr, phys_pg_pack->total_size); mutex_unlock(&ctx->mmu_lock); /* * If the context is closing we don't need to check for the MMU cache * invalidation return code and update the VA free list as in this flow * we invalidate the MMU cache outside of this unmap function and the VA * free list will be freed anyway. */ if (!ctx_free) { int tmp_rc; tmp_rc = add_va_block(hdev, va_range, vaddr, vaddr + phys_pg_pack->total_size - 1); if (tmp_rc) { dev_warn(hdev->dev, "add va block failed for vaddr: 0x%llx\n", vaddr); if (!rc) rc = tmp_rc; } } atomic_dec(&phys_pg_pack->mapping_cnt); kfree(hnode); if (is_userptr) { free_phys_pg_pack(hdev, phys_pg_pack); dma_unmap_host_va(hdev, userptr); } return rc; mapping_cnt_err: if (is_userptr) free_phys_pg_pack(hdev, phys_pg_pack); vm_type_err: mutex_lock(&ctx->mem_hash_lock); hash_add(ctx->mem_hash, &hnode->node, vaddr); mutex_unlock(&ctx->mem_hash_lock); return rc; } static int map_block(struct hl_device *hdev, u64 address, u64 *handle, u32 *size) { u32 block_id = 0; int rc; rc = hdev->asic_funcs->get_hw_block_id(hdev, address, size, &block_id); *handle = block_id | HL_MMAP_TYPE_BLOCK; *handle <<= PAGE_SHIFT; return rc; } static void hw_block_vm_close(struct vm_area_struct *vma) { struct hl_vm_hw_block_list_node *lnode = (struct hl_vm_hw_block_list_node *) vma->vm_private_data; struct hl_ctx *ctx = lnode->ctx; mutex_lock(&ctx->hw_block_list_lock); list_del(&lnode->node); mutex_unlock(&ctx->hw_block_list_lock); hl_ctx_put(ctx); kfree(lnode); vma->vm_private_data = NULL; } static const struct vm_operations_struct hw_block_vm_ops = { .close = hw_block_vm_close }; /** * hl_hw_block_mmap() - mmap a hw block to user. * @hpriv: pointer to the private data of the fd * @vma: pointer to vm_area_struct of the process * * Driver increments context reference for every HW block mapped in order * to prevent user from closing FD without unmapping first */ int hl_hw_block_mmap(struct hl_fpriv *hpriv, struct vm_area_struct *vma) { struct hl_vm_hw_block_list_node *lnode; struct hl_device *hdev = hpriv->hdev; struct hl_ctx *ctx = hpriv->ctx; u32 block_id, block_size; int rc; /* We use the page offset to hold the block id and thus we need to clear * it before doing the mmap itself */ block_id = vma->vm_pgoff; vma->vm_pgoff = 0; /* Driver only allows mapping of a complete HW block */ block_size = vma->vm_end - vma->vm_start; if (!access_ok((void __user *) (uintptr_t) vma->vm_start, block_size)) { dev_err(hdev->dev, "user pointer is invalid - 0x%lx\n", vma->vm_start); return -EINVAL; } lnode = kzalloc(sizeof(*lnode), GFP_KERNEL); if (!lnode) return -ENOMEM; vma->vm_ops = &hw_block_vm_ops; vma->vm_private_data = lnode; hl_ctx_get(hdev, ctx); rc = hdev->asic_funcs->hw_block_mmap(hdev, vma, block_id, block_size); if (rc) { hl_ctx_put(ctx); kfree(lnode); return rc; } lnode->ctx = ctx; lnode->vaddr = vma->vm_start; lnode->size = block_size; lnode->id = block_id; mutex_lock(&ctx->hw_block_list_lock); list_add_tail(&lnode->node, &ctx->hw_block_mem_list); mutex_unlock(&ctx->hw_block_list_lock); vma->vm_pgoff = block_id; return 0; } static int set_dma_sg(struct scatterlist *sg, u64 bar_address, u64 chunk_size, struct device *dev, enum dma_data_direction dir) { dma_addr_t addr; int rc; addr = dma_map_resource(dev, bar_address, chunk_size, dir, DMA_ATTR_SKIP_CPU_SYNC); rc = dma_mapping_error(dev, addr); if (rc) return rc; sg_set_page(sg, NULL, chunk_size, 0); sg_dma_address(sg) = addr; sg_dma_len(sg) = chunk_size; return 0; } static struct sg_table *alloc_sgt_from_device_pages(struct hl_device *hdev, u64 *pages, u64 npages, u64 page_size, struct device *dev, enum dma_data_direction dir) { u64 chunk_size, bar_address, dma_max_seg_size; struct asic_fixed_properties *prop; int rc, i, j, nents, cur_page; struct scatterlist *sg; struct sg_table *sgt; prop = &hdev->asic_prop; dma_max_seg_size = dma_get_max_seg_size(dev); /* We would like to align the max segment size to PAGE_SIZE, so the * SGL will contain aligned addresses that can be easily mapped to * an MMU */ dma_max_seg_size = ALIGN_DOWN(dma_max_seg_size, PAGE_SIZE); if (dma_max_seg_size < PAGE_SIZE) { dev_err_ratelimited(hdev->dev, "dma_max_seg_size %llu can't be smaller than PAGE_SIZE\n", dma_max_seg_size); return ERR_PTR(-EINVAL); } sgt = kzalloc(sizeof(*sgt), GFP_KERNEL); if (!sgt) return ERR_PTR(-ENOMEM); /* If the size of each page is larger than the dma max segment size, * then we can't combine pages and the number of entries in the SGL * will just be the * * */ if (page_size > dma_max_seg_size) nents = npages * DIV_ROUND_UP_ULL(page_size, dma_max_seg_size); else /* Get number of non-contiguous chunks */ for (i = 1, nents = 1, chunk_size = page_size ; i < npages ; i++) { if (pages[i - 1] + page_size != pages[i] || chunk_size + page_size > dma_max_seg_size) { nents++; chunk_size = page_size; continue; } chunk_size += page_size; } rc = sg_alloc_table(sgt, nents, GFP_KERNEL | __GFP_ZERO); if (rc) goto error_free; cur_page = 0; if (page_size > dma_max_seg_size) { u64 size_left, cur_device_address = 0; size_left = page_size; /* Need to split each page into the number of chunks of * dma_max_seg_size */ for_each_sgtable_dma_sg(sgt, sg, i) { if (size_left == page_size) cur_device_address = pages[cur_page] - prop->dram_base_address; else cur_device_address += dma_max_seg_size; chunk_size = min(size_left, dma_max_seg_size); bar_address = hdev->dram_pci_bar_start + cur_device_address; rc = set_dma_sg(sg, bar_address, chunk_size, dev, dir); if (rc) goto error_unmap; if (size_left > dma_max_seg_size) { size_left -= dma_max_seg_size; } else { cur_page++; size_left = page_size; } } } else { /* Merge pages and put them into the scatterlist */ for_each_sgtable_dma_sg(sgt, sg, i) { chunk_size = page_size; for (j = cur_page + 1 ; j < npages ; j++) { if (pages[j - 1] + page_size != pages[j] || chunk_size + page_size > dma_max_seg_size) break; chunk_size += page_size; } bar_address = hdev->dram_pci_bar_start + (pages[cur_page] - prop->dram_base_address); rc = set_dma_sg(sg, bar_address, chunk_size, dev, dir); if (rc) goto error_unmap; cur_page = j; } } /* Because we are not going to include a CPU list we want to have some * chance that other users will detect this by setting the orig_nents * to 0 and using only nents (length of DMA list) when going over the * sgl */ sgt->orig_nents = 0; return sgt; error_unmap: for_each_sgtable_dma_sg(sgt, sg, i) { if (!sg_dma_len(sg)) continue; dma_unmap_resource(dev, sg_dma_address(sg), sg_dma_len(sg), dir, DMA_ATTR_SKIP_CPU_SYNC); } sg_free_table(sgt); error_free: kfree(sgt); return ERR_PTR(rc); } static int hl_dmabuf_attach(struct dma_buf *dmabuf, struct dma_buf_attachment *attachment) { struct hl_dmabuf_priv *hl_dmabuf; struct hl_device *hdev; int rc; hl_dmabuf = dmabuf->priv; hdev = hl_dmabuf->ctx->hdev; rc = pci_p2pdma_distance_many(hdev->pdev, &attachment->dev, 1, true); if (rc < 0) attachment->peer2peer = false; return 0; } static struct sg_table *hl_map_dmabuf(struct dma_buf_attachment *attachment, enum dma_data_direction dir) { struct dma_buf *dma_buf = attachment->dmabuf; struct hl_vm_phys_pg_pack *phys_pg_pack; struct hl_dmabuf_priv *hl_dmabuf; struct hl_device *hdev; struct sg_table *sgt; hl_dmabuf = dma_buf->priv; hdev = hl_dmabuf->ctx->hdev; phys_pg_pack = hl_dmabuf->phys_pg_pack; if (!attachment->peer2peer) { dev_dbg(hdev->dev, "Failed to map dmabuf because p2p is disabled\n"); return ERR_PTR(-EPERM); } if (phys_pg_pack) sgt = alloc_sgt_from_device_pages(hdev, phys_pg_pack->pages, phys_pg_pack->npages, phys_pg_pack->page_size, attachment->dev, dir); else sgt = alloc_sgt_from_device_pages(hdev, &hl_dmabuf->device_address, 1, hl_dmabuf->dmabuf->size, attachment->dev, dir); if (IS_ERR(sgt)) dev_err(hdev->dev, "failed (%ld) to initialize sgt for dmabuf\n", PTR_ERR(sgt)); return sgt; } static void hl_unmap_dmabuf(struct dma_buf_attachment *attachment, struct sg_table *sgt, enum dma_data_direction dir) { struct scatterlist *sg; int i; /* The memory behind the dma-buf has *always* resided on the device itself, i.e. it lives * only in the 'device' domain (after all, it maps a PCI bar address which points to the * device memory). * * Therefore, it was never in the 'CPU' domain and hence, there is no need to perform * a sync of the memory to the CPU's cache, as it never resided inside that cache. */ for_each_sgtable_dma_sg(sgt, sg, i) dma_unmap_resource(attachment->dev, sg_dma_address(sg), sg_dma_len(sg), dir, DMA_ATTR_SKIP_CPU_SYNC); /* Need to restore orig_nents because sg_free_table use that field */ sgt->orig_nents = sgt->nents; sg_free_table(sgt); kfree(sgt); } static void hl_release_dmabuf(struct dma_buf *dmabuf) { struct hl_dmabuf_priv *hl_dmabuf = dmabuf->priv; struct hl_ctx *ctx = hl_dmabuf->ctx; struct hl_device *hdev = ctx->hdev; struct hl_vm *vm = &hdev->vm; if (hl_dmabuf->phys_pg_pack) { spin_lock(&vm->idr_lock); hl_dmabuf->phys_pg_pack->exporting_cnt--; spin_unlock(&vm->idr_lock); } hl_ctx_put(hl_dmabuf->ctx); kfree(hl_dmabuf); } static const struct dma_buf_ops habanalabs_dmabuf_ops = { .attach = hl_dmabuf_attach, .map_dma_buf = hl_map_dmabuf, .unmap_dma_buf = hl_unmap_dmabuf, .release = hl_release_dmabuf, }; static int export_dmabuf_common(struct hl_ctx *ctx, struct hl_dmabuf_priv *hl_dmabuf, u64 total_size, int flags, int *dmabuf_fd) { DEFINE_DMA_BUF_EXPORT_INFO(exp_info); struct hl_device *hdev = ctx->hdev; int rc, fd; exp_info.ops = &habanalabs_dmabuf_ops; exp_info.size = total_size; exp_info.flags = flags; exp_info.priv = hl_dmabuf; hl_dmabuf->dmabuf = dma_buf_export(&exp_info); if (IS_ERR(hl_dmabuf->dmabuf)) { dev_err(hdev->dev, "failed to export dma-buf\n"); return PTR_ERR(hl_dmabuf->dmabuf); } fd = dma_buf_fd(hl_dmabuf->dmabuf, flags); if (fd < 0) { dev_err(hdev->dev, "failed to get a file descriptor for a dma-buf\n"); rc = fd; goto err_dma_buf_put; } hl_dmabuf->ctx = ctx; hl_ctx_get(hdev, hl_dmabuf->ctx); *dmabuf_fd = fd; return 0; err_dma_buf_put: dma_buf_put(hl_dmabuf->dmabuf); return rc; } /** * export_dmabuf_from_addr() - export a dma-buf object for the given memory * address and size. * @ctx: pointer to the context structure. * @device_addr: device memory physical address. * @size: size of device memory. * @flags: DMA-BUF file/FD flags. * @dmabuf_fd: pointer to result FD that represents the dma-buf object. * * Create and export a dma-buf object for an existing memory allocation inside * the device memory, and return a FD which is associated with the dma-buf * object. * * Return: 0 on success, non-zero for failure. */ static int export_dmabuf_from_addr(struct hl_ctx *ctx, u64 device_addr, u64 size, int flags, int *dmabuf_fd) { struct hl_dmabuf_priv *hl_dmabuf; struct hl_device *hdev = ctx->hdev; struct asic_fixed_properties *prop; u64 bar_address; int rc; prop = &hdev->asic_prop; if (!IS_ALIGNED(device_addr, PAGE_SIZE)) { dev_dbg(hdev->dev, "exported device memory address 0x%llx should be aligned to 0x%lx\n", device_addr, PAGE_SIZE); return -EINVAL; } if (size < PAGE_SIZE) { dev_dbg(hdev->dev, "exported device memory size %llu should be equal to or greater than %lu\n", size, PAGE_SIZE); return -EINVAL; } if (device_addr < prop->dram_user_base_address || device_addr + size > prop->dram_end_address || device_addr + size < device_addr) { dev_dbg(hdev->dev, "DRAM memory range 0x%llx (+0x%llx) is outside of DRAM boundaries\n", device_addr, size); return -EINVAL; } bar_address = hdev->dram_pci_bar_start + (device_addr - prop->dram_base_address); if (bar_address + size > hdev->dram_pci_bar_start + prop->dram_pci_bar_size || bar_address + size < bar_address) { dev_dbg(hdev->dev, "DRAM memory range 0x%llx (+0x%llx) is outside of PCI BAR boundaries\n", device_addr, size); return -EINVAL; } hl_dmabuf = kzalloc(sizeof(*hl_dmabuf), GFP_KERNEL); if (!hl_dmabuf) return -ENOMEM; hl_dmabuf->device_address = device_addr; rc = export_dmabuf_common(ctx, hl_dmabuf, size, flags, dmabuf_fd); if (rc) goto err_free_dmabuf_wrapper; return 0; err_free_dmabuf_wrapper: kfree(hl_dmabuf); return rc; } /** * export_dmabuf_from_handle() - export a dma-buf object for the given memory * handle. * @ctx: pointer to the context structure. * @handle: device memory allocation handle. * @flags: DMA-BUF file/FD flags. * @dmabuf_fd: pointer to result FD that represents the dma-buf object. * * Create and export a dma-buf object for an existing memory allocation inside * the device memory, and return a FD which is associated with the dma-buf * object. * * Return: 0 on success, non-zero for failure. */ static int export_dmabuf_from_handle(struct hl_ctx *ctx, u64 handle, int flags, int *dmabuf_fd) { struct hl_vm_phys_pg_pack *phys_pg_pack; struct hl_dmabuf_priv *hl_dmabuf; struct hl_device *hdev = ctx->hdev; struct asic_fixed_properties *prop; struct hl_vm *vm = &hdev->vm; u64 bar_address; int rc, i; prop = &hdev->asic_prop; if (upper_32_bits(handle)) { dev_dbg(hdev->dev, "no match for handle 0x%llx\n", handle); return -EINVAL; } spin_lock(&vm->idr_lock); phys_pg_pack = idr_find(&vm->phys_pg_pack_handles, (u32) handle); if (!phys_pg_pack) { spin_unlock(&vm->idr_lock); dev_dbg(hdev->dev, "no match for handle 0x%x\n", (u32) handle); return -EINVAL; } /* increment now to avoid freeing device memory while exporting */ phys_pg_pack->exporting_cnt++; spin_unlock(&vm->idr_lock); if (phys_pg_pack->vm_type != VM_TYPE_PHYS_PACK) { dev_dbg(hdev->dev, "handle 0x%llx does not represent DRAM memory\n", handle); rc = -EINVAL; goto err_dec_exporting_cnt; } for (i = 0 ; i < phys_pg_pack->npages ; i++) { bar_address = hdev->dram_pci_bar_start + (phys_pg_pack->pages[i] - prop->dram_base_address); if (bar_address + phys_pg_pack->page_size > hdev->dram_pci_bar_start + prop->dram_pci_bar_size || bar_address + phys_pg_pack->page_size < bar_address) { dev_dbg(hdev->dev, "DRAM memory range 0x%llx (+0x%x) is outside of PCI BAR boundaries\n", phys_pg_pack->pages[i], phys_pg_pack->page_size); rc = -EINVAL; goto err_dec_exporting_cnt; } } hl_dmabuf = kzalloc(sizeof(*hl_dmabuf), GFP_KERNEL); if (!hl_dmabuf) { rc = -ENOMEM; goto err_dec_exporting_cnt; } hl_dmabuf->phys_pg_pack = phys_pg_pack; rc = export_dmabuf_common(ctx, hl_dmabuf, phys_pg_pack->total_size, flags, dmabuf_fd); if (rc) goto err_free_dmabuf_wrapper; return 0; err_free_dmabuf_wrapper: kfree(hl_dmabuf); err_dec_exporting_cnt: spin_lock(&vm->idr_lock); phys_pg_pack->exporting_cnt--; spin_unlock(&vm->idr_lock); return rc; } static int mem_ioctl_no_mmu(struct hl_fpriv *hpriv, union hl_mem_args *args) { struct hl_device *hdev = hpriv->hdev; u64 block_handle, device_addr = 0; struct hl_ctx *ctx = hpriv->ctx; u32 handle = 0, block_size; int rc; switch (args->in.op) { case HL_MEM_OP_ALLOC: if (args->in.alloc.mem_size == 0) { dev_err(hdev->dev, "alloc size must be larger than 0\n"); rc = -EINVAL; goto out; } /* Force contiguous as there are no real MMU * translations to overcome physical memory gaps */ args->in.flags |= HL_MEM_CONTIGUOUS; rc = alloc_device_memory(ctx, &args->in, &handle); memset(args, 0, sizeof(*args)); args->out.handle = (__u64) handle; break; case HL_MEM_OP_FREE: rc = free_device_memory(ctx, &args->in); break; case HL_MEM_OP_MAP: if (args->in.flags & HL_MEM_USERPTR) { dev_err(hdev->dev, "Failed to map host memory when MMU is disabled\n"); rc = -EPERM; } else { rc = get_paddr_from_handle(ctx, &args->in, &device_addr); memset(args, 0, sizeof(*args)); args->out.device_virt_addr = device_addr; } break; case HL_MEM_OP_UNMAP: rc = 0; break; case HL_MEM_OP_MAP_BLOCK: rc = map_block(hdev, args->in.map_block.block_addr, &block_handle, &block_size); args->out.block_handle = block_handle; args->out.block_size = block_size; break; case HL_MEM_OP_EXPORT_DMABUF_FD: dev_err(hdev->dev, "Failed to export dma-buf object when MMU is disabled\n"); rc = -EPERM; break; case HL_MEM_OP_TS_ALLOC: rc = allocate_timestamps_buffers(hpriv, &args->in, &args->out.handle); break; default: dev_err(hdev->dev, "Unknown opcode for memory IOCTL\n"); rc = -EINVAL; break; } out: return rc; } static void ts_buff_release(struct hl_mmap_mem_buf *buf) { struct hl_ts_buff *ts_buff = buf->private; vfree(ts_buff->kernel_buff_address); vfree(ts_buff->user_buff_address); kfree(ts_buff); } static int hl_ts_mmap(struct hl_mmap_mem_buf *buf, struct vm_area_struct *vma, void *args) { struct hl_ts_buff *ts_buff = buf->private; vma->vm_flags |= VM_DONTEXPAND | VM_DONTDUMP | VM_DONTCOPY | VM_NORESERVE; return remap_vmalloc_range(vma, ts_buff->user_buff_address, 0); } static int hl_ts_alloc_buf(struct hl_mmap_mem_buf *buf, gfp_t gfp, void *args) { struct hl_ts_buff *ts_buff = NULL; u32 size, num_elements; void *p; num_elements = *(u32 *)args; ts_buff = kzalloc(sizeof(*ts_buff), GFP_KERNEL); if (!ts_buff) return -ENOMEM; /* Allocate the user buffer */ size = num_elements * sizeof(u64); p = vmalloc_user(size); if (!p) goto free_mem; ts_buff->user_buff_address = p; buf->mappable_size = size; /* Allocate the internal kernel buffer */ size = num_elements * sizeof(struct hl_user_pending_interrupt); p = vmalloc(size); if (!p) goto free_user_buff; ts_buff->kernel_buff_address = p; ts_buff->kernel_buff_size = size; buf->private = ts_buff; return 0; free_user_buff: vfree(ts_buff->user_buff_address); free_mem: kfree(ts_buff); return -ENOMEM; } static struct hl_mmap_mem_buf_ops hl_ts_behavior = { .mmap = hl_ts_mmap, .alloc = hl_ts_alloc_buf, .release = ts_buff_release, }; /** * allocate_timestamps_buffers() - allocate timestamps buffers * This function will allocate ts buffer that will later on be mapped to the user * in order to be able to read the timestamp. * in additon it'll allocate an extra buffer for registration management. * since we cannot fail during registration for out-of-memory situation, so * we'll prepare a pool which will be used as user interrupt nodes and instead * of dynamically allocating nodes while registration we'll pick the node from * this pool. in addtion it'll add node to the mapping hash which will be used * to map user ts buffer to the internal kernel ts buffer. * @hpriv: pointer to the private data of the fd * @args: ioctl input * @handle: user timestamp buffer handle as an output */ static int allocate_timestamps_buffers(struct hl_fpriv *hpriv, struct hl_mem_in *args, u64 *handle) { struct hl_mem_mgr *mmg = &hpriv->mem_mgr; struct hl_mmap_mem_buf *buf; if (args->num_of_elements > TS_MAX_ELEMENTS_NUM) { dev_err(mmg->dev, "Num of elements exceeds Max allowed number (0x%x > 0x%x)\n", args->num_of_elements, TS_MAX_ELEMENTS_NUM); return -EINVAL; } buf = hl_mmap_mem_buf_alloc(mmg, &hl_ts_behavior, GFP_KERNEL, &args->num_of_elements); if (!buf) return -ENOMEM; /* TODO: * Remove HL_MMAP_TYPE_TS_BUFF. * Embedding type in handle will no longer be needed as soon as we * switch to using a single memory manager for all memory types. */ *handle = ((u64)buf->handle | HL_MMAP_TYPE_TS_BUFF) << PAGE_SHIFT; return 0; } int hl_mem_ioctl(struct hl_fpriv *hpriv, void *data) { enum hl_device_status status; union hl_mem_args *args = data; struct hl_device *hdev = hpriv->hdev; struct hl_ctx *ctx = hpriv->ctx; u64 block_handle, device_addr = 0; u32 handle = 0, block_size; int rc, dmabuf_fd = -EBADF; if (!hl_device_operational(hdev, &status)) { dev_warn_ratelimited(hdev->dev, "Device is %s. Can't execute MEMORY IOCTL\n", hdev->status[status]); return -EBUSY; } if (!hdev->mmu_enable) return mem_ioctl_no_mmu(hpriv, args); switch (args->in.op) { case HL_MEM_OP_ALLOC: if (args->in.alloc.mem_size == 0) { dev_err(hdev->dev, "alloc size must be larger than 0\n"); rc = -EINVAL; goto out; } /* If DRAM does not support virtual memory the driver won't * handle the allocation/freeing of that memory. However, for * system administration/monitoring purposes, the driver will * keep track of the amount of DRAM memory that is allocated * and freed by the user. Because this code totally relies on * the user's input, the driver can't ensure the validity * of this accounting. */ if (!hdev->asic_prop.dram_supports_virtual_memory) { atomic64_add(args->in.alloc.mem_size, &ctx->dram_phys_mem); atomic64_add(args->in.alloc.mem_size, &hdev->dram_used_mem); dev_dbg(hdev->dev, "DRAM alloc is not supported\n"); rc = 0; memset(args, 0, sizeof(*args)); args->out.handle = 0; goto out; } rc = alloc_device_memory(ctx, &args->in, &handle); memset(args, 0, sizeof(*args)); args->out.handle = (__u64) handle; break; case HL_MEM_OP_FREE: /* If DRAM does not support virtual memory the driver won't * handle the allocation/freeing of that memory. However, for * system administration/monitoring purposes, the driver will * keep track of the amount of DRAM memory that is allocated * and freed by the user. Because this code totally relies on * the user's input, the driver can't ensure the validity * of this accounting. */ if (!hdev->asic_prop.dram_supports_virtual_memory) { atomic64_sub(args->in.alloc.mem_size, &ctx->dram_phys_mem); atomic64_sub(args->in.alloc.mem_size, &hdev->dram_used_mem); dev_dbg(hdev->dev, "DRAM alloc is not supported\n"); rc = 0; goto out; } rc = free_device_memory(ctx, &args->in); break; case HL_MEM_OP_MAP: rc = map_device_va(ctx, &args->in, &device_addr); memset(args, 0, sizeof(*args)); args->out.device_virt_addr = device_addr; break; case HL_MEM_OP_UNMAP: rc = unmap_device_va(ctx, &args->in, false); break; case HL_MEM_OP_MAP_BLOCK: rc = map_block(hdev, args->in.map_block.block_addr, &block_handle, &block_size); args->out.block_handle = block_handle; args->out.block_size = block_size; break; case HL_MEM_OP_EXPORT_DMABUF_FD: if (hdev->asic_prop.dram_supports_virtual_memory) rc = export_dmabuf_from_handle(ctx, args->in.export_dmabuf_fd.handle, args->in.flags, &dmabuf_fd); else rc = export_dmabuf_from_addr(ctx, args->in.export_dmabuf_fd.handle, args->in.export_dmabuf_fd.mem_size, args->in.flags, &dmabuf_fd); memset(args, 0, sizeof(*args)); args->out.fd = dmabuf_fd; break; case HL_MEM_OP_TS_ALLOC: rc = allocate_timestamps_buffers(hpriv, &args->in, &args->out.handle); break; default: dev_err(hdev->dev, "Unknown opcode for memory IOCTL\n"); rc = -EINVAL; break; } out: return rc; } static int get_user_memory(struct hl_device *hdev, u64 addr, u64 size, u32 npages, u64 start, u32 offset, struct hl_userptr *userptr) { int rc; if (!access_ok((void __user *) (uintptr_t) addr, size)) { dev_err(hdev->dev, "user pointer is invalid - 0x%llx\n", addr); return -EFAULT; } userptr->pages = kvmalloc_array(npages, sizeof(*userptr->pages), GFP_KERNEL); if (!userptr->pages) return -ENOMEM; rc = pin_user_pages_fast(start, npages, FOLL_FORCE | FOLL_WRITE | FOLL_LONGTERM, userptr->pages); if (rc != npages) { dev_err(hdev->dev, "Failed (%d) to pin host memory with user ptr 0x%llx, size 0x%llx, npages %d\n", rc, addr, size, npages); if (rc < 0) goto destroy_pages; npages = rc; rc = -EFAULT; goto put_pages; } userptr->npages = npages; rc = sg_alloc_table_from_pages(userptr->sgt, userptr->pages, npages, offset, size, GFP_KERNEL); if (rc < 0) { dev_err(hdev->dev, "failed to create SG table from pages\n"); goto put_pages; } return 0; put_pages: unpin_user_pages(userptr->pages, npages); destroy_pages: kvfree(userptr->pages); return rc; } /** * hl_pin_host_memory() - pins a chunk of host memory. * @hdev: pointer to the habanalabs device structure. * @addr: the host virtual address of the memory area. * @size: the size of the memory area. * @userptr: pointer to hl_userptr structure. * * This function does the following: * - Pins the physical pages. * - Create an SG list from those pages. */ int hl_pin_host_memory(struct hl_device *hdev, u64 addr, u64 size, struct hl_userptr *userptr) { u64 start, end; u32 npages, offset; int rc; if (!size) { dev_err(hdev->dev, "size to pin is invalid - %llu\n", size); return -EINVAL; } /* * If the combination of the address and size requested for this memory * region causes an integer overflow, return error. */ if (((addr + size) < addr) || PAGE_ALIGN(addr + size) < (addr + size)) { dev_err(hdev->dev, "user pointer 0x%llx + %llu causes integer overflow\n", addr, size); return -EINVAL; } userptr->pid = current->pid; userptr->sgt = kzalloc(sizeof(*userptr->sgt), GFP_KERNEL); if (!userptr->sgt) return -ENOMEM; start = addr & PAGE_MASK; offset = addr & ~PAGE_MASK; end = PAGE_ALIGN(addr + size); npages = (end - start) >> PAGE_SHIFT; userptr->size = size; userptr->addr = addr; userptr->dma_mapped = false; INIT_LIST_HEAD(&userptr->job_node); rc = get_user_memory(hdev, addr, size, npages, start, offset, userptr); if (rc) { dev_err(hdev->dev, "failed to get user memory for address 0x%llx\n", addr); goto free_sgt; } hl_debugfs_add_userptr(hdev, userptr); return 0; free_sgt: kfree(userptr->sgt); return rc; } /* * hl_unpin_host_memory - unpins a chunk of host memory. * @hdev: pointer to the habanalabs device structure * @userptr: pointer to hl_userptr structure * * This function does the following: * - Unpins the physical pages related to the host memory * - Free the SG list */ void hl_unpin_host_memory(struct hl_device *hdev, struct hl_userptr *userptr) { hl_debugfs_remove_userptr(hdev, userptr); if (userptr->dma_mapped) hdev->asic_funcs->hl_dma_unmap_sgtable(hdev, userptr->sgt, userptr->dir); unpin_user_pages_dirty_lock(userptr->pages, userptr->npages, true); kvfree(userptr->pages); list_del(&userptr->job_node); sg_free_table(userptr->sgt); kfree(userptr->sgt); } /** * hl_userptr_delete_list() - clear userptr list. * @hdev: pointer to the habanalabs device structure. * @userptr_list: pointer to the list to clear. * * This function does the following: * - Iterates over the list and unpins the host memory and frees the userptr * structure. */ void hl_userptr_delete_list(struct hl_device *hdev, struct list_head *userptr_list) { struct hl_userptr *userptr, *tmp; list_for_each_entry_safe(userptr, tmp, userptr_list, job_node) { hl_unpin_host_memory(hdev, userptr); kfree(userptr); } INIT_LIST_HEAD(userptr_list); } /** * hl_userptr_is_pinned() - returns whether the given userptr is pinned. * @hdev: pointer to the habanalabs device structure. * @addr: user address to check. * @size: user block size to check. * @userptr_list: pointer to the list to clear. * @userptr: pointer to userptr to check. * * This function does the following: * - Iterates over the list and checks if the given userptr is in it, means is * pinned. If so, returns true, otherwise returns false. */ bool hl_userptr_is_pinned(struct hl_device *hdev, u64 addr, u32 size, struct list_head *userptr_list, struct hl_userptr **userptr) { list_for_each_entry((*userptr), userptr_list, job_node) { if ((addr == (*userptr)->addr) && (size == (*userptr)->size)) return true; } return false; } /** * va_range_init() - initialize virtual addresses range. * @hdev: pointer to the habanalabs device structure. * @va_ranges: pointer to va_ranges array. * @start: range start address. * @end: range end address. * @page_size: page size for this va_range. * * This function does the following: * - Initializes the virtual addresses list of the given range with the given * addresses. */ static int va_range_init(struct hl_device *hdev, struct hl_va_range *va_range, u64 start, u64 end, u32 page_size) { int rc; INIT_LIST_HEAD(&va_range->list); /* * PAGE_SIZE alignment * it is the callers responsibility to align the addresses if the * page size is not a power of 2 */ if (is_power_of_2(page_size)) { if (start & (PAGE_SIZE - 1)) { start &= PAGE_MASK; start += PAGE_SIZE; } /* * The end of the range is inclusive, hence we need to align it * to the end of the last full page in the range. For example if * end = 0x3ff5 with page size 0x1000, we need to align it to * 0x2fff. The remainig 0xff5 bytes do not form a full page. */ if ((end + 1) & (PAGE_SIZE - 1)) end = ((end + 1) & PAGE_MASK) - 1; } if (start >= end) { dev_err(hdev->dev, "too small vm range for va list\n"); return -EFAULT; } rc = add_va_block(hdev, va_range, start, end); if (rc) { dev_err(hdev->dev, "Failed to init host va list\n"); return rc; } va_range->start_addr = start; va_range->end_addr = end; va_range->page_size = page_size; return 0; } /** * va_range_fini() - clear a virtual addresses range. * @hdev: pointer to the habanalabs structure. * @va_range: pointer to virtual addresses range. * * This function does the following: * - Frees the virtual addresses block list and its lock. */ static void va_range_fini(struct hl_device *hdev, struct hl_va_range *va_range) { mutex_lock(&va_range->lock); clear_va_list_locked(hdev, &va_range->list); mutex_unlock(&va_range->lock); mutex_destroy(&va_range->lock); kfree(va_range); } /** * vm_ctx_init_with_ranges() - initialize virtual memory for context. * @ctx: pointer to the habanalabs context structure. * @host_range_start: host virtual addresses range start. * @host_range_end: host virtual addresses range end. * @host_page_size: host page size. * @host_huge_range_start: host virtual addresses range start for memory * allocated with huge pages. * @host_huge_range_end: host virtual addresses range end for memory allocated * with huge pages. * @host_huge_page_size: host huge page size. * @dram_range_start: dram virtual addresses range start. * @dram_range_end: dram virtual addresses range end. * @dram_page_size: dram page size. * * This function initializes the following: * - MMU for context. * - Virtual address to area descriptor hashtable. * - Virtual block list of available virtual memory. */ static int vm_ctx_init_with_ranges(struct hl_ctx *ctx, u64 host_range_start, u64 host_range_end, u32 host_page_size, u64 host_huge_range_start, u64 host_huge_range_end, u32 host_huge_page_size, u64 dram_range_start, u64 dram_range_end, u32 dram_page_size) { struct hl_device *hdev = ctx->hdev; int i, rc; for (i = 0 ; i < HL_VA_RANGE_TYPE_MAX ; i++) { ctx->va_range[i] = kzalloc(sizeof(struct hl_va_range), GFP_KERNEL); if (!ctx->va_range[i]) { rc = -ENOMEM; goto free_va_range; } } rc = hl_mmu_ctx_init(ctx); if (rc) { dev_err(hdev->dev, "failed to init context %d\n", ctx->asid); goto free_va_range; } mutex_init(&ctx->mem_hash_lock); hash_init(ctx->mem_hash); mutex_init(&ctx->va_range[HL_VA_RANGE_TYPE_HOST]->lock); rc = va_range_init(hdev, ctx->va_range[HL_VA_RANGE_TYPE_HOST], host_range_start, host_range_end, host_page_size); if (rc) { dev_err(hdev->dev, "failed to init host vm range\n"); goto mmu_ctx_fini; } if (hdev->pmmu_huge_range) { mutex_init(&ctx->va_range[HL_VA_RANGE_TYPE_HOST_HUGE]->lock); rc = va_range_init(hdev, ctx->va_range[HL_VA_RANGE_TYPE_HOST_HUGE], host_huge_range_start, host_huge_range_end, host_huge_page_size); if (rc) { dev_err(hdev->dev, "failed to init host huge vm range\n"); goto clear_host_va_range; } } else { kfree(ctx->va_range[HL_VA_RANGE_TYPE_HOST_HUGE]); ctx->va_range[HL_VA_RANGE_TYPE_HOST_HUGE] = ctx->va_range[HL_VA_RANGE_TYPE_HOST]; } mutex_init(&ctx->va_range[HL_VA_RANGE_TYPE_DRAM]->lock); rc = va_range_init(hdev, ctx->va_range[HL_VA_RANGE_TYPE_DRAM], dram_range_start, dram_range_end, dram_page_size); if (rc) { dev_err(hdev->dev, "failed to init dram vm range\n"); goto clear_host_huge_va_range; } hl_debugfs_add_ctx_mem_hash(hdev, ctx); return 0; clear_host_huge_va_range: mutex_destroy(&ctx->va_range[HL_VA_RANGE_TYPE_DRAM]->lock); if (hdev->pmmu_huge_range) { mutex_lock(&ctx->va_range[HL_VA_RANGE_TYPE_HOST_HUGE]->lock); clear_va_list_locked(hdev, &ctx->va_range[HL_VA_RANGE_TYPE_HOST_HUGE]->list); mutex_unlock(&ctx->va_range[HL_VA_RANGE_TYPE_HOST_HUGE]->lock); } clear_host_va_range: if (hdev->pmmu_huge_range) mutex_destroy(&ctx->va_range[HL_VA_RANGE_TYPE_HOST_HUGE]->lock); mutex_lock(&ctx->va_range[HL_VA_RANGE_TYPE_HOST]->lock); clear_va_list_locked(hdev, &ctx->va_range[HL_VA_RANGE_TYPE_HOST]->list); mutex_unlock(&ctx->va_range[HL_VA_RANGE_TYPE_HOST]->lock); mmu_ctx_fini: mutex_destroy(&ctx->va_range[HL_VA_RANGE_TYPE_HOST]->lock); mutex_destroy(&ctx->mem_hash_lock); hl_mmu_ctx_fini(ctx); free_va_range: for (i = 0 ; i < HL_VA_RANGE_TYPE_MAX ; i++) kfree(ctx->va_range[i]); return rc; } int hl_vm_ctx_init(struct hl_ctx *ctx) { struct asic_fixed_properties *prop = &ctx->hdev->asic_prop; u64 host_range_start, host_range_end, host_huge_range_start, host_huge_range_end, dram_range_start, dram_range_end; u32 host_page_size, host_huge_page_size, dram_page_size; atomic64_set(&ctx->dram_phys_mem, 0); /* * - If MMU is enabled, init the ranges as usual. * - If MMU is disabled, in case of host mapping, the returned address * is the given one. * In case of DRAM mapping, the returned address is the physical * address of the memory related to the given handle. */ if (!ctx->hdev->mmu_enable) return 0; dram_range_start = prop->dmmu.start_addr; dram_range_end = prop->dmmu.end_addr - 1; dram_page_size = prop->dram_page_size ? prop->dram_page_size : prop->dmmu.page_size; host_range_start = prop->pmmu.start_addr; host_range_end = prop->pmmu.end_addr - 1; host_page_size = prop->pmmu.page_size; host_huge_range_start = prop->pmmu_huge.start_addr; host_huge_range_end = prop->pmmu_huge.end_addr - 1; host_huge_page_size = prop->pmmu_huge.page_size; return vm_ctx_init_with_ranges(ctx, host_range_start, host_range_end, host_page_size, host_huge_range_start, host_huge_range_end, host_huge_page_size, dram_range_start, dram_range_end, dram_page_size); } /** * hl_vm_ctx_fini() - virtual memory teardown of context. * @ctx: pointer to the habanalabs context structure. * * This function perform teardown the following: * - Virtual block list of available virtual memory. * - Virtual address to area descriptor hashtable. * - MMU for context. * * In addition this function does the following: * - Unmaps the existing hashtable nodes if the hashtable is not empty. The * hashtable should be empty as no valid mappings should exist at this * point. * - Frees any existing physical page list from the idr which relates to the * current context asid. * - This function checks the virtual block list for correctness. At this point * the list should contain one element which describes the whole virtual * memory range of the context. Otherwise, a warning is printed. */ void hl_vm_ctx_fini(struct hl_ctx *ctx) { struct hl_vm_phys_pg_pack *phys_pg_list, *tmp_phys_node; struct hl_device *hdev = ctx->hdev; struct hl_vm_hash_node *hnode; struct hl_vm *vm = &hdev->vm; struct hlist_node *tmp_node; struct list_head free_list; struct hl_mem_in args; int i; if (!hdev->mmu_enable) return; hl_debugfs_remove_ctx_mem_hash(hdev, ctx); /* * Clearly something went wrong on hard reset so no point in printing * another side effect error */ if (!hdev->reset_info.hard_reset_pending && !hash_empty(ctx->mem_hash)) dev_dbg(hdev->dev, "user released device without removing its memory mappings\n"); hash_for_each_safe(ctx->mem_hash, i, tmp_node, hnode, node) { dev_dbg(hdev->dev, "hl_mem_hash_node of vaddr 0x%llx of asid %d is still alive\n", hnode->vaddr, ctx->asid); args.unmap.device_virt_addr = hnode->vaddr; unmap_device_va(ctx, &args, true); } mutex_lock(&ctx->mmu_lock); /* invalidate the cache once after the unmapping loop */ hl_mmu_invalidate_cache(hdev, true, MMU_OP_USERPTR); hl_mmu_invalidate_cache(hdev, true, MMU_OP_PHYS_PACK); mutex_unlock(&ctx->mmu_lock); INIT_LIST_HEAD(&free_list); spin_lock(&vm->idr_lock); idr_for_each_entry(&vm->phys_pg_pack_handles, phys_pg_list, i) if (phys_pg_list->asid == ctx->asid) { dev_dbg(hdev->dev, "page list 0x%px of asid %d is still alive\n", phys_pg_list, ctx->asid); atomic64_sub(phys_pg_list->total_size, &hdev->dram_used_mem); idr_remove(&vm->phys_pg_pack_handles, i); list_add(&phys_pg_list->node, &free_list); } spin_unlock(&vm->idr_lock); list_for_each_entry_safe(phys_pg_list, tmp_phys_node, &free_list, node) free_phys_pg_pack(hdev, phys_pg_list); va_range_fini(hdev, ctx->va_range[HL_VA_RANGE_TYPE_DRAM]); va_range_fini(hdev, ctx->va_range[HL_VA_RANGE_TYPE_HOST]); if (hdev->pmmu_huge_range) va_range_fini(hdev, ctx->va_range[HL_VA_RANGE_TYPE_HOST_HUGE]); mutex_destroy(&ctx->mem_hash_lock); hl_mmu_ctx_fini(ctx); /* In this case we need to clear the global accounting of DRAM usage * because the user notifies us on allocations. If the user is no more, * all DRAM is available */ if (ctx->asid != HL_KERNEL_ASID_ID && !hdev->asic_prop.dram_supports_virtual_memory) atomic64_set(&hdev->dram_used_mem, 0); } /** * hl_vm_init() - initialize virtual memory module. * @hdev: pointer to the habanalabs device structure. * * This function initializes the following: * - MMU module. * - DRAM physical pages pool of 2MB. * - Idr for device memory allocation handles. */ int hl_vm_init(struct hl_device *hdev) { struct asic_fixed_properties *prop = &hdev->asic_prop; struct hl_vm *vm = &hdev->vm; int rc; if (is_power_of_2(prop->dram_page_size)) vm->dram_pg_pool = gen_pool_create(__ffs(prop->dram_page_size), -1); else vm->dram_pg_pool = gen_pool_create(__ffs(DRAM_POOL_PAGE_SIZE), -1); if (!vm->dram_pg_pool) { dev_err(hdev->dev, "Failed to create dram page pool\n"); return -ENOMEM; } kref_init(&vm->dram_pg_pool_refcount); rc = gen_pool_add(vm->dram_pg_pool, prop->dram_user_base_address, prop->dram_end_address - prop->dram_user_base_address, -1); if (rc) { dev_err(hdev->dev, "Failed to add memory to dram page pool %d\n", rc); goto pool_add_err; } spin_lock_init(&vm->idr_lock); idr_init(&vm->phys_pg_pack_handles); atomic64_set(&hdev->dram_used_mem, 0); vm->init_done = true; return 0; pool_add_err: gen_pool_destroy(vm->dram_pg_pool); return rc; } /** * hl_vm_fini() - virtual memory module teardown. * @hdev: pointer to the habanalabs device structure. * * This function perform teardown to the following: * - Idr for device memory allocation handles. * - DRAM physical pages pool of 2MB. * - MMU module. */ void hl_vm_fini(struct hl_device *hdev) { struct hl_vm *vm = &hdev->vm; if (!vm->init_done) return; /* * At this point all the contexts should be freed and hence no DRAM * memory should be in use. Hence the DRAM pool should be freed here. */ if (kref_put(&vm->dram_pg_pool_refcount, dram_pg_pool_do_release) != 1) dev_warn(hdev->dev, "dram_pg_pool was not destroyed on %s\n", __func__); vm->init_done = false; } /** * hl_hw_block_mem_init() - HW block memory initialization. * @ctx: pointer to the habanalabs context structure. * * This function initializes the HW block virtual mapped addresses list and * it's lock. */ void hl_hw_block_mem_init(struct hl_ctx *ctx) { mutex_init(&ctx->hw_block_list_lock); INIT_LIST_HEAD(&ctx->hw_block_mem_list); } /** * hl_hw_block_mem_fini() - HW block memory teardown. * @ctx: pointer to the habanalabs context structure. * * This function clears the HW block virtual mapped addresses list and destroys * it's lock. */ void hl_hw_block_mem_fini(struct hl_ctx *ctx) { struct hl_vm_hw_block_list_node *lnode, *tmp; if (!list_empty(&ctx->hw_block_mem_list)) dev_crit(ctx->hdev->dev, "HW block mem list isn't empty\n"); list_for_each_entry_safe(lnode, tmp, &ctx->hw_block_mem_list, node) { list_del(&lnode->node); kfree(lnode); } mutex_destroy(&ctx->hw_block_list_lock); }