diff options
Diffstat (limited to 'drivers/gpu/drm/etnaviv')
| -rw-r--r-- | drivers/gpu/drm/etnaviv/Kconfig | 3 | ||||
| -rw-r--r-- | drivers/gpu/drm/etnaviv/Makefile | 1 | ||||
| -rw-r--r-- | drivers/gpu/drm/etnaviv/etnaviv_buffer.c | 14 | ||||
| -rw-r--r-- | drivers/gpu/drm/etnaviv/etnaviv_cmd_parser.c | 6 | ||||
| -rw-r--r-- | drivers/gpu/drm/etnaviv/etnaviv_cmdbuf.c | 153 | ||||
| -rw-r--r-- | drivers/gpu/drm/etnaviv/etnaviv_cmdbuf.h | 58 | ||||
| -rw-r--r-- | drivers/gpu/drm/etnaviv/etnaviv_drv.c | 24 | ||||
| -rw-r--r-- | drivers/gpu/drm/etnaviv/etnaviv_drv.h | 2 | ||||
| -rw-r--r-- | drivers/gpu/drm/etnaviv/etnaviv_dump.c | 6 | ||||
| -rw-r--r-- | drivers/gpu/drm/etnaviv/etnaviv_gem.c | 3 | ||||
| -rw-r--r-- | drivers/gpu/drm/etnaviv/etnaviv_gem_submit.c | 8 | ||||
| -rw-r--r-- | drivers/gpu/drm/etnaviv/etnaviv_gpu.c | 95 | ||||
| -rw-r--r-- | drivers/gpu/drm/etnaviv/etnaviv_gpu.h | 28 | ||||
| -rw-r--r-- | drivers/gpu/drm/etnaviv/etnaviv_iommu.c | 2 | ||||
| -rw-r--r-- | drivers/gpu/drm/etnaviv/etnaviv_iommu_v2.c | 6 | ||||
| -rw-r--r-- | drivers/gpu/drm/etnaviv/etnaviv_mmu.c | 76 | ||||
| -rw-r--r-- | drivers/gpu/drm/etnaviv/etnaviv_mmu.h | 10 |
17 files changed, 351 insertions, 144 deletions
diff --git a/drivers/gpu/drm/etnaviv/Kconfig b/drivers/gpu/drm/etnaviv/Kconfig index 2cde7a5442fb..cc1731c5289c 100644 --- a/drivers/gpu/drm/etnaviv/Kconfig +++ b/drivers/gpu/drm/etnaviv/Kconfig @@ -2,7 +2,8 @@ config DRM_ETNAVIV tristate "ETNAVIV (DRM support for Vivante GPU IP cores)" depends on DRM - depends on ARCH_MXC || ARCH_DOVE + depends on ARCH_MXC || ARCH_DOVE || (ARM && COMPILE_TEST) + depends on MMU select SHMEM select TMPFS select IOMMU_API diff --git a/drivers/gpu/drm/etnaviv/Makefile b/drivers/gpu/drm/etnaviv/Makefile index 1086e9876f91..4f76c992043f 100644 --- a/drivers/gpu/drm/etnaviv/Makefile +++ b/drivers/gpu/drm/etnaviv/Makefile @@ -1,6 +1,7 @@ etnaviv-y := \ etnaviv_buffer.o \ etnaviv_cmd_parser.o \ + etnaviv_cmdbuf.o \ etnaviv_drv.o \ etnaviv_dump.o \ etnaviv_gem_prime.o \ diff --git a/drivers/gpu/drm/etnaviv/etnaviv_buffer.c b/drivers/gpu/drm/etnaviv/etnaviv_buffer.c index d9230132dfbc..ed9588f36bc9 100644 --- a/drivers/gpu/drm/etnaviv/etnaviv_buffer.c +++ b/drivers/gpu/drm/etnaviv/etnaviv_buffer.c @@ -15,6 +15,7 @@ * this program. If not, see <http://www.gnu.org/licenses/>. */ +#include "etnaviv_cmdbuf.h" #include "etnaviv_gpu.h" #include "etnaviv_gem.h" #include "etnaviv_mmu.h" @@ -125,7 +126,7 @@ static void etnaviv_buffer_dump(struct etnaviv_gpu *gpu, u32 *ptr = buf->vaddr + off; dev_info(gpu->dev, "virt %p phys 0x%08x free 0x%08x\n", - ptr, etnaviv_iommu_get_cmdbuf_va(gpu, buf) + off, size - len * 4 - off); + ptr, etnaviv_cmdbuf_get_va(buf) + off, size - len * 4 - off); print_hex_dump(KERN_INFO, "cmd ", DUMP_PREFIX_OFFSET, 16, 4, ptr, len * 4, 0); @@ -158,7 +159,7 @@ static u32 etnaviv_buffer_reserve(struct etnaviv_gpu *gpu, if (buffer->user_size + cmd_dwords * sizeof(u64) > buffer->size) buffer->user_size = 0; - return etnaviv_iommu_get_cmdbuf_va(gpu, buffer) + buffer->user_size; + return etnaviv_cmdbuf_get_va(buffer) + buffer->user_size; } u16 etnaviv_buffer_init(struct etnaviv_gpu *gpu) @@ -169,7 +170,7 @@ u16 etnaviv_buffer_init(struct etnaviv_gpu *gpu) buffer->user_size = 0; CMD_WAIT(buffer); - CMD_LINK(buffer, 2, etnaviv_iommu_get_cmdbuf_va(gpu, buffer) + + CMD_LINK(buffer, 2, etnaviv_cmdbuf_get_va(buffer) + buffer->user_size - 4); return buffer->user_size / 8; @@ -261,7 +262,7 @@ void etnaviv_buffer_queue(struct etnaviv_gpu *gpu, unsigned int event, if (drm_debug & DRM_UT_DRIVER) etnaviv_buffer_dump(gpu, buffer, 0, 0x50); - link_target = etnaviv_iommu_get_cmdbuf_va(gpu, cmdbuf); + link_target = etnaviv_cmdbuf_get_va(cmdbuf); link_dwords = cmdbuf->size / 8; /* @@ -355,12 +356,13 @@ void etnaviv_buffer_queue(struct etnaviv_gpu *gpu, unsigned int event, CMD_LOAD_STATE(buffer, VIVS_GL_EVENT, VIVS_GL_EVENT_EVENT_ID(event) | VIVS_GL_EVENT_FROM_PE); CMD_WAIT(buffer); - CMD_LINK(buffer, 2, etnaviv_iommu_get_cmdbuf_va(gpu, buffer) + + CMD_LINK(buffer, 2, etnaviv_cmdbuf_get_va(buffer) + buffer->user_size - 4); if (drm_debug & DRM_UT_DRIVER) pr_info("stream link to 0x%08x @ 0x%08x %p\n", - return_target, etnaviv_iommu_get_cmdbuf_va(gpu, cmdbuf), cmdbuf->vaddr); + return_target, etnaviv_cmdbuf_get_va(cmdbuf), + cmdbuf->vaddr); if (drm_debug & DRM_UT_DRIVER) { print_hex_dump(KERN_INFO, "cmd ", DUMP_PREFIX_OFFSET, 16, 4, diff --git a/drivers/gpu/drm/etnaviv/etnaviv_cmd_parser.c b/drivers/gpu/drm/etnaviv/etnaviv_cmd_parser.c index 2a2e5e366ab7..6e3bbcf24160 100644 --- a/drivers/gpu/drm/etnaviv/etnaviv_cmd_parser.c +++ b/drivers/gpu/drm/etnaviv/etnaviv_cmd_parser.c @@ -56,6 +56,8 @@ static const struct { ST(0x0644, 1), ST(0x064c, 1), ST(0x0680, 8), + ST(0x086c, 1), + ST(0x1028, 1), ST(0x1410, 1), ST(0x1430, 1), ST(0x1458, 1), @@ -73,8 +75,12 @@ static const struct { ST(0x16c0, 8), ST(0x16e0, 8), ST(0x1740, 8), + ST(0x17c0, 8), + ST(0x17e0, 8), ST(0x2400, 14 * 16), ST(0x10800, 32 * 16), + ST(0x14600, 16), + ST(0x14800, 8 * 8), #undef ST }; diff --git a/drivers/gpu/drm/etnaviv/etnaviv_cmdbuf.c b/drivers/gpu/drm/etnaviv/etnaviv_cmdbuf.c new file mode 100644 index 000000000000..633e0f07cbac --- /dev/null +++ b/drivers/gpu/drm/etnaviv/etnaviv_cmdbuf.c @@ -0,0 +1,153 @@ +/* + * Copyright (C) 2017 Etnaviv Project + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 as published by + * the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * this program. If not, see <http://www.gnu.org/licenses/>. + */ + +#include <drm/drm_mm.h> + +#include "etnaviv_cmdbuf.h" +#include "etnaviv_gpu.h" +#include "etnaviv_mmu.h" + +#define SUBALLOC_SIZE SZ_256K +#define SUBALLOC_GRANULE SZ_4K +#define SUBALLOC_GRANULES (SUBALLOC_SIZE / SUBALLOC_GRANULE) + +struct etnaviv_cmdbuf_suballoc { + /* suballocated dma buffer properties */ + struct etnaviv_gpu *gpu; + void *vaddr; + dma_addr_t paddr; + + /* GPU mapping */ + u32 iova; + struct drm_mm_node vram_node; /* only used on MMUv2 */ + + /* allocation management */ + struct mutex lock; + DECLARE_BITMAP(granule_map, SUBALLOC_GRANULES); + int free_space; + wait_queue_head_t free_event; +}; + +struct etnaviv_cmdbuf_suballoc * +etnaviv_cmdbuf_suballoc_new(struct etnaviv_gpu * gpu) +{ + struct etnaviv_cmdbuf_suballoc *suballoc; + int ret; + + suballoc = kzalloc(sizeof(*suballoc), GFP_KERNEL); + if (!suballoc) + return ERR_PTR(-ENOMEM); + + suballoc->gpu = gpu; + mutex_init(&suballoc->lock); + init_waitqueue_head(&suballoc->free_event); + + suballoc->vaddr = dma_alloc_wc(gpu->dev, SUBALLOC_SIZE, + &suballoc->paddr, GFP_KERNEL); + if (!suballoc->vaddr) + goto free_suballoc; + + ret = etnaviv_iommu_get_suballoc_va(gpu, suballoc->paddr, + &suballoc->vram_node, SUBALLOC_SIZE, + &suballoc->iova); + if (ret) + goto free_dma; + + return suballoc; + +free_dma: + dma_free_wc(gpu->dev, SUBALLOC_SIZE, suballoc->vaddr, suballoc->paddr); +free_suballoc: + kfree(suballoc); + + return NULL; +} + +void etnaviv_cmdbuf_suballoc_destroy(struct etnaviv_cmdbuf_suballoc *suballoc) +{ + etnaviv_iommu_put_suballoc_va(suballoc->gpu, &suballoc->vram_node, + SUBALLOC_SIZE, suballoc->iova); + dma_free_wc(suballoc->gpu->dev, SUBALLOC_SIZE, suballoc->vaddr, + suballoc->paddr); + kfree(suballoc); +} + +struct etnaviv_cmdbuf * +etnaviv_cmdbuf_new(struct etnaviv_cmdbuf_suballoc *suballoc, u32 size, + size_t nr_bos) +{ + struct etnaviv_cmdbuf *cmdbuf; + size_t sz = size_vstruct(nr_bos, sizeof(cmdbuf->bo_map[0]), + sizeof(*cmdbuf)); + int granule_offs, order, ret; + + cmdbuf = kzalloc(sz, GFP_KERNEL); + if (!cmdbuf) + return NULL; + + cmdbuf->suballoc = suballoc; + cmdbuf->size = size; + + order = order_base_2(ALIGN(size, SUBALLOC_GRANULE) / SUBALLOC_GRANULE); +retry: + mutex_lock(&suballoc->lock); + granule_offs = bitmap_find_free_region(suballoc->granule_map, + SUBALLOC_GRANULES, order); + if (granule_offs < 0) { + suballoc->free_space = 0; + mutex_unlock(&suballoc->lock); + ret = wait_event_interruptible_timeout(suballoc->free_event, + suballoc->free_space, + msecs_to_jiffies(10 * 1000)); + if (!ret) { + dev_err(suballoc->gpu->dev, + "Timeout waiting for cmdbuf space\n"); + return NULL; + } + goto retry; + } + mutex_unlock(&suballoc->lock); + cmdbuf->suballoc_offset = granule_offs * SUBALLOC_GRANULE; + cmdbuf->vaddr = suballoc->vaddr + cmdbuf->suballoc_offset; + + return cmdbuf; +} + +void etnaviv_cmdbuf_free(struct etnaviv_cmdbuf *cmdbuf) +{ + struct etnaviv_cmdbuf_suballoc *suballoc = cmdbuf->suballoc; + int order = order_base_2(ALIGN(cmdbuf->size, SUBALLOC_GRANULE) / + SUBALLOC_GRANULE); + + mutex_lock(&suballoc->lock); + bitmap_release_region(suballoc->granule_map, + cmdbuf->suballoc_offset / SUBALLOC_GRANULE, + order); + suballoc->free_space = 1; + mutex_unlock(&suballoc->lock); + wake_up_all(&suballoc->free_event); + kfree(cmdbuf); +} + +u32 etnaviv_cmdbuf_get_va(struct etnaviv_cmdbuf *buf) +{ + return buf->suballoc->iova + buf->suballoc_offset; +} + +dma_addr_t etnaviv_cmdbuf_get_pa(struct etnaviv_cmdbuf *buf) +{ + return buf->suballoc->paddr + buf->suballoc_offset; +} diff --git a/drivers/gpu/drm/etnaviv/etnaviv_cmdbuf.h b/drivers/gpu/drm/etnaviv/etnaviv_cmdbuf.h new file mode 100644 index 000000000000..80d78076c679 --- /dev/null +++ b/drivers/gpu/drm/etnaviv/etnaviv_cmdbuf.h @@ -0,0 +1,58 @@ +/* + * Copyright (C) 2017 Etnaviv Project + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 as published by + * the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * this program. If not, see <http://www.gnu.org/licenses/>. + */ + +#ifndef __ETNAVIV_CMDBUF_H__ +#define __ETNAVIV_CMDBUF_H__ + +#include <linux/types.h> + +struct etnaviv_gpu; +struct etnaviv_cmdbuf_suballoc; + +struct etnaviv_cmdbuf { + /* suballocator this cmdbuf is allocated from */ + struct etnaviv_cmdbuf_suballoc *suballoc; + /* user context key, must be unique between all active users */ + struct etnaviv_file_private *ctx; + /* cmdbuf properties */ + int suballoc_offset; + void *vaddr; + u32 size; + u32 user_size; + /* fence after which this buffer is to be disposed */ + struct dma_fence *fence; + /* target exec state */ + u32 exec_state; + /* per GPU in-flight list */ + struct list_head node; + /* BOs attached to this command buffer */ + unsigned int nr_bos; + struct etnaviv_vram_mapping *bo_map[0]; +}; + +struct etnaviv_cmdbuf_suballoc * +etnaviv_cmdbuf_suballoc_new(struct etnaviv_gpu * gpu); +void etnaviv_cmdbuf_suballoc_destroy(struct etnaviv_cmdbuf_suballoc *suballoc); + +struct etnaviv_cmdbuf * +etnaviv_cmdbuf_new(struct etnaviv_cmdbuf_suballoc *suballoc, u32 size, + size_t nr_bos); +void etnaviv_cmdbuf_free(struct etnaviv_cmdbuf *cmdbuf); + +u32 etnaviv_cmdbuf_get_va(struct etnaviv_cmdbuf *buf); +dma_addr_t etnaviv_cmdbuf_get_pa(struct etnaviv_cmdbuf *buf); + +#endif /* __ETNAVIV_CMDBUF_H__ */ diff --git a/drivers/gpu/drm/etnaviv/etnaviv_drv.c b/drivers/gpu/drm/etnaviv/etnaviv_drv.c index 00368b14d08d..587e45043542 100644 --- a/drivers/gpu/drm/etnaviv/etnaviv_drv.c +++ b/drivers/gpu/drm/etnaviv/etnaviv_drv.c @@ -18,11 +18,11 @@ #include <linux/of_platform.h> #include <drm/drm_of.h> +#include "etnaviv_cmdbuf.h" #include "etnaviv_drv.h" #include "etnaviv_gpu.h" #include "etnaviv_gem.h" #include "etnaviv_mmu.h" -#include "etnaviv_gem.h" #ifdef CONFIG_DRM_ETNAVIV_REGISTER_LOGGING static bool reglog; @@ -147,21 +147,23 @@ static int etnaviv_gem_show(struct drm_device *dev, struct seq_file *m) static int etnaviv_mm_show(struct drm_device *dev, struct seq_file *m) { - int ret; + struct drm_printer p = drm_seq_file_printer(m); read_lock(&dev->vma_offset_manager->vm_lock); - ret = drm_mm_dump_table(m, &dev->vma_offset_manager->vm_addr_space_mm); + drm_mm_print(&dev->vma_offset_manager->vm_addr_space_mm, &p); read_unlock(&dev->vma_offset_manager->vm_lock); - return ret; + return 0; } static int etnaviv_mmu_show(struct etnaviv_gpu *gpu, struct seq_file *m) { + struct drm_printer p = drm_seq_file_printer(m); + seq_printf(m, "Active Objects (%s):\n", dev_name(gpu->dev)); mutex_lock(&gpu->mmu->lock); - drm_mm_dump_table(m, &gpu->mmu->mm); + drm_mm_print(&gpu->mmu->mm, &p); mutex_unlock(&gpu->mmu->lock); return 0; @@ -175,7 +177,8 @@ static void etnaviv_buffer_dump(struct etnaviv_gpu *gpu, struct seq_file *m) u32 i; seq_printf(m, "virt %p - phys 0x%llx - free 0x%08x\n", - buf->vaddr, (u64)buf->paddr, size - buf->user_size); + buf->vaddr, (u64)etnaviv_cmdbuf_get_pa(buf), + size - buf->user_size); for (i = 0; i < size / 4; i++) { if (i && !(i % 4)) @@ -256,12 +259,6 @@ static int etnaviv_debugfs_init(struct drm_minor *minor) return ret; } - -static void etnaviv_debugfs_cleanup(struct drm_minor *minor) -{ - drm_debugfs_remove_files(etnaviv_debugfs_list, - ARRAY_SIZE(etnaviv_debugfs_list), minor); -} #endif /* @@ -507,7 +504,6 @@ static struct drm_driver etnaviv_drm_driver = { .gem_prime_mmap = etnaviv_gem_prime_mmap, #ifdef CONFIG_DEBUG_FS .debugfs_init = etnaviv_debugfs_init, - .debugfs_cleanup = etnaviv_debugfs_cleanup, #endif .ioctls = etnaviv_ioctls, .num_ioctls = DRM_ETNAVIV_NUM_IOCTLS, @@ -592,7 +588,7 @@ static void etnaviv_unbind(struct device *dev) drm->dev_private = NULL; kfree(priv); - drm_put_dev(drm); + drm_dev_unref(drm); } static const struct component_master_ops etnaviv_master_ops = { diff --git a/drivers/gpu/drm/etnaviv/etnaviv_drv.h b/drivers/gpu/drm/etnaviv/etnaviv_drv.h index c255eda40526..e41f38667c1c 100644 --- a/drivers/gpu/drm/etnaviv/etnaviv_drv.h +++ b/drivers/gpu/drm/etnaviv/etnaviv_drv.h @@ -73,7 +73,7 @@ int etnaviv_ioctl_gem_submit(struct drm_device *dev, void *data, struct drm_file *file); int etnaviv_gem_mmap(struct file *filp, struct vm_area_struct *vma); -int etnaviv_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf); +int etnaviv_gem_fault(struct vm_fault *vmf); int etnaviv_gem_mmap_offset(struct drm_gem_object *obj, u64 *offset); struct sg_table *etnaviv_gem_prime_get_sg_table(struct drm_gem_object *obj); void *etnaviv_gem_prime_vmap(struct drm_gem_object *obj); diff --git a/drivers/gpu/drm/etnaviv/etnaviv_dump.c b/drivers/gpu/drm/etnaviv/etnaviv_dump.c index af65491a78e2..d019b5e311cc 100644 --- a/drivers/gpu/drm/etnaviv/etnaviv_dump.c +++ b/drivers/gpu/drm/etnaviv/etnaviv_dump.c @@ -15,6 +15,7 @@ */ #include <linux/devcoredump.h> +#include "etnaviv_cmdbuf.h" #include "etnaviv_dump.h" #include "etnaviv_gem.h" #include "etnaviv_gpu.h" @@ -177,12 +178,11 @@ void etnaviv_core_dump(struct etnaviv_gpu *gpu) etnaviv_core_dump_mmu(&iter, gpu, mmu_size); etnaviv_core_dump_mem(&iter, ETDUMP_BUF_RING, gpu->buffer->vaddr, gpu->buffer->size, - etnaviv_iommu_get_cmdbuf_va(gpu, gpu->buffer)); + etnaviv_cmdbuf_get_va(gpu->buffer)); list_for_each_entry(cmd, &gpu->active_cmd_list, node) etnaviv_core_dump_mem(&iter, ETDUMP_BUF_CMD, cmd->vaddr, - cmd->size, - etnaviv_iommu_get_cmdbuf_va(gpu, cmd)); + cmd->size, etnaviv_cmdbuf_get_va(cmd)); /* Reserve space for the bomap */ if (n_bomap_pages) { diff --git a/drivers/gpu/drm/etnaviv/etnaviv_gem.c b/drivers/gpu/drm/etnaviv/etnaviv_gem.c index aa6e35ddc87f..e78f1406885d 100644 --- a/drivers/gpu/drm/etnaviv/etnaviv_gem.c +++ b/drivers/gpu/drm/etnaviv/etnaviv_gem.c @@ -175,8 +175,9 @@ int etnaviv_gem_mmap(struct file *filp, struct vm_area_struct *vma) return obj->ops->mmap(obj, vma); } -int etnaviv_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf) +int etnaviv_gem_fault(struct vm_fault *vmf) { + struct vm_area_struct *vma = vmf->vma; struct drm_gem_object *obj = vma->vm_private_data; struct etnaviv_gem_object *etnaviv_obj = to_etnaviv_bo(obj); struct page **pages, *page; diff --git a/drivers/gpu/drm/etnaviv/etnaviv_gem_submit.c b/drivers/gpu/drm/etnaviv/etnaviv_gem_submit.c index afdd55ddf821..726090d7a6ac 100644 --- a/drivers/gpu/drm/etnaviv/etnaviv_gem_submit.c +++ b/drivers/gpu/drm/etnaviv/etnaviv_gem_submit.c @@ -15,6 +15,7 @@ */ #include <linux/reservation.h> +#include "etnaviv_cmdbuf.h" #include "etnaviv_drv.h" #include "etnaviv_gpu.h" #include "etnaviv_gem.h" @@ -332,8 +333,9 @@ int etnaviv_ioctl_gem_submit(struct drm_device *dev, void *data, bos = drm_malloc_ab(args->nr_bos, sizeof(*bos)); relocs = drm_malloc_ab(args->nr_relocs, sizeof(*relocs)); stream = drm_malloc_ab(1, args->stream_size); - cmdbuf = etnaviv_gpu_cmdbuf_new(gpu, ALIGN(args->stream_size, 8) + 8, - args->nr_bos); + cmdbuf = etnaviv_cmdbuf_new(gpu->cmdbuf_suballoc, + ALIGN(args->stream_size, 8) + 8, + args->nr_bos); if (!bos || !relocs || !stream || !cmdbuf) { ret = -ENOMEM; goto err_submit_cmds; @@ -422,7 +424,7 @@ err_submit_objects: err_submit_cmds: /* if we still own the cmdbuf */ if (cmdbuf) - etnaviv_gpu_cmdbuf_free(cmdbuf); + etnaviv_cmdbuf_free(cmdbuf); if (stream) drm_free_large(stream); if (bos) diff --git a/drivers/gpu/drm/etnaviv/etnaviv_gpu.c b/drivers/gpu/drm/etnaviv/etnaviv_gpu.c index 0a67124bb2a4..130d7d517a19 100644 --- a/drivers/gpu/drm/etnaviv/etnaviv_gpu.c +++ b/drivers/gpu/drm/etnaviv/etnaviv_gpu.c @@ -18,6 +18,8 @@ #include <linux/dma-fence.h> #include <linux/moduleparam.h> #include <linux/of_device.h> + +#include "etnaviv_cmdbuf.h" #include "etnaviv_dump.h" #include "etnaviv_gpu.h" #include "etnaviv_gem.h" @@ -546,6 +548,37 @@ void etnaviv_gpu_start_fe(struct etnaviv_gpu *gpu, u32 address, u16 prefetch) VIVS_FE_COMMAND_CONTROL_PREFETCH(prefetch)); } +static void etnaviv_gpu_setup_pulse_eater(struct etnaviv_gpu *gpu) +{ + /* + * Base value for VIVS_PM_PULSE_EATER register on models where it + * cannot be read, extracted from vivante kernel driver. + */ + u32 pulse_eater = 0x01590880; + + if (etnaviv_is_model_rev(gpu, GC4000, 0x5208) || + etnaviv_is_model_rev(gpu, GC4000, 0x5222)) { + pulse_eater |= BIT(23); + + } + + if (etnaviv_is_model_rev(gpu, GC1000, 0x5039) || + etnaviv_is_model_rev(gpu, GC1000, 0x5040)) { + pulse_eater &= ~BIT(16); + pulse_eater |= BIT(17); + } + + if ((gpu->identity.revision > 0x5420) && + (gpu->identity.features & chipFeatures_PIPE_3D)) + { + /* Performance fix: disable internal DFS */ + pulse_eater = gpu_read(gpu, VIVS_PM_PULSE_EATER); + pulse_eater |= BIT(18); + } + + gpu_write(gpu, VIVS_PM_PULSE_EATER, pulse_eater); +} + static void etnaviv_gpu_hw_init(struct etnaviv_gpu *gpu) { u16 prefetch; @@ -586,6 +619,9 @@ static void etnaviv_gpu_hw_init(struct etnaviv_gpu *gpu) gpu_write(gpu, VIVS_MC_BUS_CONFIG, bus_config); } + /* setup the pulse eater */ + etnaviv_gpu_setup_pulse_eater(gpu); + /* setup the MMU */ etnaviv_iommu_restore(gpu); @@ -593,7 +629,7 @@ static void etnaviv_gpu_hw_init(struct etnaviv_gpu *gpu) prefetch = etnaviv_buffer_init(gpu); gpu_write(gpu, VIVS_HI_INTR_ENBL, ~0U); - etnaviv_gpu_start_fe(gpu, etnaviv_iommu_get_cmdbuf_va(gpu, gpu->buffer), + etnaviv_gpu_start_fe(gpu, etnaviv_cmdbuf_get_va(gpu->buffer), prefetch); } @@ -658,8 +694,15 @@ int etnaviv_gpu_init(struct etnaviv_gpu *gpu) goto fail; } + gpu->cmdbuf_suballoc = etnaviv_cmdbuf_suballoc_new(gpu); + if (IS_ERR(gpu->cmdbuf_suballoc)) { + dev_err(gpu->dev, "Failed to create cmdbuf suballocator\n"); + ret = PTR_ERR(gpu->cmdbuf_suballoc); + goto fail; + } + /* Create buffer: */ - gpu->buffer = etnaviv_gpu_cmdbuf_new(gpu, PAGE_SIZE, 0); + gpu->buffer = etnaviv_cmdbuf_new(gpu->cmdbuf_suballoc, PAGE_SIZE, 0); if (!gpu->buffer) { ret = -ENOMEM; dev_err(gpu->dev, "could not create command buffer\n"); @@ -667,7 +710,7 @@ int etnaviv_gpu_init(struct etnaviv_gpu *gpu) } if (gpu->mmu->version == ETNAVIV_IOMMU_V1 && - gpu->buffer->paddr - gpu->memory_base > 0x80000000) { + etnaviv_cmdbuf_get_va(gpu->buffer) > 0x80000000) { ret = -EINVAL; dev_err(gpu->dev, "command buffer outside valid memory window\n"); @@ -694,7 +737,7 @@ int etnaviv_gpu_init(struct etnaviv_gpu *gpu) return 0; free_buffer: - etnaviv_gpu_cmdbuf_free(gpu->buffer); + etnaviv_cmdbuf_free(gpu->buffer); gpu->buffer = NULL; destroy_iommu: etnaviv_iommu_destroy(gpu->mmu); @@ -1117,41 +1160,6 @@ static void event_free(struct etnaviv_gpu *gpu, unsigned int event) * Cmdstream submission/retirement: */ -struct etnaviv_cmdbuf *etnaviv_gpu_cmdbuf_new(struct etnaviv_gpu *gpu, u32 size, - size_t nr_bos) -{ - struct etnaviv_cmdbuf *cmdbuf; - size_t sz = size_vstruct(nr_bos, sizeof(cmdbuf->bo_map[0]), - sizeof(*cmdbuf)); - - cmdbuf = kzalloc(sz, GFP_KERNEL); - if (!cmdbuf) - return NULL; - - if (gpu->mmu->version == ETNAVIV_IOMMU_V2) - size = ALIGN(size, SZ_4K); - - cmdbuf->vaddr = dma_alloc_wc(gpu->dev, size, &cmdbuf->paddr, - GFP_KERNEL); - if (!cmdbuf->vaddr) { - kfree(cmdbuf); - return NULL; - } - - cmdbuf->gpu = gpu; - cmdbuf->size = size; - - return cmdbuf; -} - -void etnaviv_gpu_cmdbuf_free(struct etnaviv_cmdbuf *cmdbuf) -{ - etnaviv_iommu_put_cmdbuf_va(cmdbuf->gpu, cmdbuf); - dma_free_wc(cmdbuf->gpu->dev, cmdbuf->size, cmdbuf->vaddr, - cmdbuf->paddr); - kfree(cmdbuf); -} - static void retire_worker(struct work_struct *work) { struct etnaviv_gpu *gpu = container_of(work, struct etnaviv_gpu, @@ -1177,7 +1185,7 @@ static void retire_worker(struct work_struct *work) etnaviv_gem_mapping_unreference(mapping); } - etnaviv_gpu_cmdbuf_free(cmdbuf); + etnaviv_cmdbuf_free(cmdbuf); /* * We need to balance the runtime PM count caused by * each submission. Upon submission, we increment @@ -1593,10 +1601,15 @@ static void etnaviv_gpu_unbind(struct device *dev, struct device *master, #endif if (gpu->buffer) { - etnaviv_gpu_cmdbuf_free(gpu->buffer); + etnaviv_cmdbuf_free(gpu->buffer); gpu->buffer = NULL; } + if (gpu->cmdbuf_suballoc) { + etnaviv_cmdbuf_suballoc_destroy(gpu->cmdbuf_suballoc); + gpu->cmdbuf_suballoc = NULL; + } + if (gpu->mmu) { etnaviv_iommu_destroy(gpu->mmu); gpu->mmu = NULL; diff --git a/drivers/gpu/drm/etnaviv/etnaviv_gpu.h b/drivers/gpu/drm/etnaviv/etnaviv_gpu.h index 8c6b824e9d0a..1c0606ea7d5e 100644 --- a/drivers/gpu/drm/etnaviv/etnaviv_gpu.h +++ b/drivers/gpu/drm/etnaviv/etnaviv_gpu.h @@ -92,6 +92,7 @@ struct etnaviv_event { struct dma_fence *fence; }; +struct etnaviv_cmdbuf_suballoc; struct etnaviv_cmdbuf; struct etnaviv_gpu { @@ -135,6 +136,7 @@ struct etnaviv_gpu { int irq; struct etnaviv_iommu *mmu; + struct etnaviv_cmdbuf_suballoc *cmdbuf_suballoc; /* Power Control: */ struct clk *clk_bus; @@ -150,29 +152,6 @@ struct etnaviv_gpu { struct work_struct recover_work; }; -struct etnaviv_cmdbuf { - /* device this cmdbuf is allocated for */ - struct etnaviv_gpu *gpu; - /* user context key, must be unique between all active users */ - struct etnaviv_file_private *ctx; - /* cmdbuf properties */ - void *vaddr; - dma_addr_t paddr; - u32 size; - u32 user_size; - /* vram node used if the cmdbuf is mapped through the MMUv2 */ - struct drm_mm_node vram_node; - /* fence after which this buffer is to be disposed */ - struct dma_fence *fence; - /* target exec state */ - u32 exec_state; - /* per GPU in-flight list */ - struct list_head node; - /* BOs attached to this command buffer */ - unsigned int nr_bos; - struct etnaviv_vram_mapping *bo_map[0]; -}; - static inline void gpu_write(struct etnaviv_gpu *gpu, u32 reg, u32 data) { etnaviv_writel(data, gpu->mmio + reg); @@ -211,9 +190,6 @@ int etnaviv_gpu_wait_obj_inactive(struct etnaviv_gpu *gpu, struct etnaviv_gem_object *etnaviv_obj, struct timespec *timeout); int etnaviv_gpu_submit(struct etnaviv_gpu *gpu, struct etnaviv_gem_submit *submit, struct etnaviv_cmdbuf *cmdbuf); -struct etnaviv_cmdbuf *etnaviv_gpu_cmdbuf_new(struct etnaviv_gpu *gpu, - u32 size, size_t nr_bos); -void etnaviv_gpu_cmdbuf_free(struct etnaviv_cmdbuf *cmdbuf); int etnaviv_gpu_pm_get_sync(struct etnaviv_gpu *gpu); void etnaviv_gpu_pm_put(struct etnaviv_gpu *gpu); int etnaviv_gpu_wait_idle(struct etnaviv_gpu *gpu, unsigned int timeout_ms); diff --git a/drivers/gpu/drm/etnaviv/etnaviv_iommu.c b/drivers/gpu/drm/etnaviv/etnaviv_iommu.c index 81f1583a7946..7a7c97f599d7 100644 --- a/drivers/gpu/drm/etnaviv/etnaviv_iommu.c +++ b/drivers/gpu/drm/etnaviv/etnaviv_iommu.c @@ -184,7 +184,7 @@ static void etnaviv_iommuv1_dump(struct iommu_domain *domain, void *buf) memcpy(buf, etnaviv_domain->pgtable.pgtable, PT_SIZE); } -static struct etnaviv_iommu_ops etnaviv_iommu_ops = { +static const struct etnaviv_iommu_ops etnaviv_iommu_ops = { .ops = { .domain_free = etnaviv_domain_free, .map = etnaviv_iommuv1_map, diff --git a/drivers/gpu/drm/etnaviv/etnaviv_iommu_v2.c b/drivers/gpu/drm/etnaviv/etnaviv_iommu_v2.c index 7e9c4d210a84..cbe447ac5974 100644 --- a/drivers/gpu/drm/etnaviv/etnaviv_iommu_v2.c +++ b/drivers/gpu/drm/etnaviv/etnaviv_iommu_v2.c @@ -21,6 +21,7 @@ #include <linux/dma-mapping.h> #include <linux/bitops.h> +#include "etnaviv_cmdbuf.h" #include "etnaviv_gpu.h" #include "etnaviv_mmu.h" #include "etnaviv_iommu.h" @@ -229,7 +230,7 @@ static void etnaviv_iommuv2_dump(struct iommu_domain *domain, void *buf) memcpy(buf, etnaviv_domain->stlb_cpu[i], SZ_4K); } -static struct etnaviv_iommu_ops etnaviv_iommu_ops = { +static const struct etnaviv_iommu_ops etnaviv_iommu_ops = { .ops = { .domain_free = etnaviv_iommuv2_domain_free, .map = etnaviv_iommuv2_map, @@ -254,7 +255,8 @@ void etnaviv_iommuv2_restore(struct etnaviv_gpu *gpu) prefetch = etnaviv_buffer_config_mmuv2(gpu, (u32)etnaviv_domain->mtlb_dma, (u32)etnaviv_domain->bad_page_dma); - etnaviv_gpu_start_fe(gpu, gpu->buffer->paddr, prefetch); + etnaviv_gpu_start_fe(gpu, (u32)etnaviv_cmdbuf_get_pa(gpu->buffer), + prefetch); etnaviv_gpu_wait_idle(gpu, 100); gpu_write(gpu, VIVS_MMUv2_CONTROL, VIVS_MMUv2_CONTROL_ENABLE); diff --git a/drivers/gpu/drm/etnaviv/etnaviv_mmu.c b/drivers/gpu/drm/etnaviv/etnaviv_mmu.c index fe0e85b41310..f103e787de94 100644 --- a/drivers/gpu/drm/etnaviv/etnaviv_mmu.c +++ b/drivers/gpu/drm/etnaviv/etnaviv_mmu.c @@ -15,6 +15,7 @@ */ #include "common.xml.h" +#include "etnaviv_cmdbuf.h" #include "etnaviv_drv.h" #include "etnaviv_gem.h" #include "etnaviv_gpu.h" @@ -107,24 +108,21 @@ static int etnaviv_iommu_find_iova(struct etnaviv_iommu *mmu, struct drm_mm_node *node, size_t size) { struct etnaviv_vram_mapping *free = NULL; + enum drm_mm_insert_mode mode = DRM_MM_INSERT_LOW; int ret; lockdep_assert_held(&mmu->lock); while (1) { struct etnaviv_vram_mapping *m, *n; + struct drm_mm_scan scan; struct list_head list; bool found; - /* - * XXX: The DRM_MM_SEARCH_BELOW is really a hack to trick - * drm_mm into giving out a low IOVA after address space - * rollover. This needs a proper fix. - */ ret = drm_mm_insert_node_in_range(&mmu->mm, node, - size, 0, mmu->last_iova, ~0UL, - mmu->last_iova ? DRM_MM_SEARCH_DEFAULT : DRM_MM_SEARCH_BELOW); - + size, 0, 0, + mmu->last_iova, U64_MAX, + mode); if (ret != -ENOSPC) break; @@ -139,7 +137,7 @@ static int etnaviv_iommu_find_iova(struct etnaviv_iommu *mmu, } /* Try to retire some entries */ - drm_mm_init_scan(&mmu->mm, size, 0, 0); + drm_mm_scan_init(&scan, &mmu->mm, size, 0, 0, mode); found = 0; INIT_LIST_HEAD(&list); @@ -156,7 +154,7 @@ static int etnaviv_iommu_find_iova(struct etnaviv_iommu *mmu, continue; list_add(&free->scan_node, &list); - if (drm_mm_scan_add_block(&free->vram_node)) { + if (drm_mm_scan_add_block(&scan, &free->vram_node)) { found = true; break; } @@ -165,7 +163,7 @@ static int etnaviv_iommu_find_iova(struct etnaviv_iommu *mmu, if (!found) { /* Nothing found, clean up and fail */ list_for_each_entry_safe(m, n, &list, scan_node) - BUG_ON(drm_mm_scan_remove_block(&m->vram_node)); + BUG_ON(drm_mm_scan_remove_block(&scan, &m->vram_node)); break; } @@ -176,7 +174,7 @@ static int etnaviv_iommu_find_iova(struct etnaviv_iommu *mmu, * can leave the block pinned. */ list_for_each_entry_safe(m, n, &list, scan_node) - if (!drm_mm_scan_remove_block(&m->vram_node)) + if (!drm_mm_scan_remove_block(&scan, &m->vram_node)) list_del_init(&m->scan_node); /* @@ -191,13 +189,12 @@ static int etnaviv_iommu_find_iova(struct etnaviv_iommu *mmu, list_del_init(&m->scan_node); } + mode = DRM_MM_INSERT_EVICT; + /* * We removed enough mappings so that the new allocation will - * succeed. Ensure that the MMU will be flushed before the - * associated commit requesting this mapping, and retry the - * allocation one more time. + * succeed, retry the allocation one more time. */ - mmu->need_flush = true; } return ret; @@ -249,6 +246,7 @@ int etnaviv_iommu_map_gem(struct etnaviv_iommu *mmu, } list_add_tail(&mapping->mmu_node, &mmu->mappings); + mmu->need_flush = true; mutex_unlock(&mmu->lock); return ret; @@ -266,6 +264,7 @@ void etnaviv_iommu_unmap_gem(struct etnaviv_iommu *mmu, etnaviv_iommu_remove_mapping(mmu, mapping); list_del(&mapping->mmu_node); + mmu->need_flush = true; mutex_unlock(&mmu->lock); } @@ -321,55 +320,50 @@ void etnaviv_iommu_restore(struct etnaviv_gpu *gpu) etnaviv_iommuv2_restore(gpu); } -u32 etnaviv_iommu_get_cmdbuf_va(struct etnaviv_gpu *gpu, - struct etnaviv_cmdbuf *buf) +int etnaviv_iommu_get_suballoc_va(struct etnaviv_gpu *gpu, dma_addr_t paddr, + struct drm_mm_node *vram_node, size_t size, + u32 *iova) { struct etnaviv_iommu *mmu = gpu->mmu; if (mmu->version == ETNAVIV_IOMMU_V1) { - return buf->paddr - gpu->memory_base; + *iova = paddr - gpu->memory_base; + return 0; } else { int ret; - if (buf->vram_node.allocated) - return (u32)buf->vram_node.start; - mutex_lock(&mmu->lock); - ret = etnaviv_iommu_find_iova(mmu, &buf->vram_node, - buf->size + SZ_64K); + ret = etnaviv_iommu_find_iova(mmu, vram_node, size); if (ret < 0) { mutex_unlock(&mmu->lock); - return 0; + return ret; } - ret = iommu_map(mmu->domain, buf->vram_node.start, buf->paddr, - buf->size, IOMMU_READ); + ret = iommu_map(mmu->domain, vram_node->start, paddr, size, + IOMMU_READ); if (ret < 0) { - drm_mm_remove_node(&buf->vram_node); + drm_mm_remove_node(vram_node); mutex_unlock(&mmu->lock); - return 0; + return ret; } - /* - * At least on GC3000 the FE MMU doesn't properly flush old TLB - * entries. Make sure to space the command buffers out in a way - * that the FE MMU prefetch won't load invalid entries. - */ - mmu->last_iova = buf->vram_node.start + buf->size + SZ_64K; + mmu->last_iova = vram_node->start + size; gpu->mmu->need_flush = true; mutex_unlock(&mmu->lock); - return (u32)buf->vram_node.start; + *iova = (u32)vram_node->start; + return 0; } } -void etnaviv_iommu_put_cmdbuf_va(struct etnaviv_gpu *gpu, - struct etnaviv_cmdbuf *buf) +void etnaviv_iommu_put_suballoc_va(struct etnaviv_gpu *gpu, + struct drm_mm_node *vram_node, size_t size, + u32 iova) { struct etnaviv_iommu *mmu = gpu->mmu; - if (mmu->version == ETNAVIV_IOMMU_V2 && buf->vram_node.allocated) { + if (mmu->version == ETNAVIV_IOMMU_V2) { mutex_lock(&mmu->lock); - iommu_unmap(mmu->domain, buf->vram_node.start, buf->size); - drm_mm_remove_node(&buf->vram_node); + iommu_unmap(mmu->domain,iova, size); + drm_mm_remove_node(vram_node); mutex_unlock(&mmu->lock); } } diff --git a/drivers/gpu/drm/etnaviv/etnaviv_mmu.h b/drivers/gpu/drm/etnaviv/etnaviv_mmu.h index e787e49c9693..54be289e5981 100644 --- a/drivers/gpu/drm/etnaviv/etnaviv_mmu.h +++ b/drivers/gpu/drm/etnaviv/etnaviv_mmu.h @@ -62,10 +62,12 @@ void etnaviv_iommu_unmap_gem(struct etnaviv_iommu *mmu, struct etnaviv_vram_mapping *mapping); void etnaviv_iommu_destroy(struct etnaviv_iommu *iommu); -u32 etnaviv_iommu_get_cmdbuf_va(struct etnaviv_gpu *gpu, - struct etnaviv_cmdbuf *buf); -void etnaviv_iommu_put_cmdbuf_va(struct etnaviv_gpu *gpu, - struct etnaviv_cmdbuf *buf); +int etnaviv_iommu_get_suballoc_va(struct etnaviv_gpu *gpu, dma_addr_t paddr, + struct drm_mm_node *vram_node, size_t size, + u32 *iova); +void etnaviv_iommu_put_suballoc_va(struct etnaviv_gpu *gpu, + struct drm_mm_node *vram_node, size_t size, + u32 iova); size_t etnaviv_iommu_dump_size(struct etnaviv_iommu *iommu); void etnaviv_iommu_dump(struct etnaviv_iommu *iommu, void *buf); |