From eecbb3cdcccac24e1e67e92fec24f9035fe19784 Mon Sep 17 00:00:00 2001 From: Rob Herring Date: Mon, 1 Jul 2019 17:05:43 -0600 Subject: drm/panfrost: Split panfrost_mmu_map SG list mapping to its own function In preparation to create partial GPU mappings of BOs on page faults, split out the SG list handling of panfrost_mmu_map(). Cc: Tomeu Vizoso Cc: Boris Brezillon Cc: Robin Murphy Reviewed: Steven Price Acked-by: Alyssa Rosenzweig Signed-off-by: Rob Herring Link: https://patchwork.freedesktop.org/patch/msgid/20190808222200.13176-5-robh@kernel.org --- drivers/gpu/drm/panfrost/panfrost_mmu.c | 52 ++++++++++++++++++++------------- 1 file changed, 31 insertions(+), 21 deletions(-) (limited to 'drivers/gpu/drm/panfrost/panfrost_mmu.c') diff --git a/drivers/gpu/drm/panfrost/panfrost_mmu.c b/drivers/gpu/drm/panfrost/panfrost_mmu.c index 92ac995dd9c6..b4ac149b2399 100644 --- a/drivers/gpu/drm/panfrost/panfrost_mmu.c +++ b/drivers/gpu/drm/panfrost/panfrost_mmu.c @@ -145,27 +145,13 @@ static size_t get_pgsize(u64 addr, size_t size) return SZ_2M; } -int panfrost_mmu_map(struct panfrost_gem_object *bo) +static int mmu_map_sg(struct panfrost_device *pfdev, u64 iova, + int prot, struct sg_table *sgt) { - struct drm_gem_object *obj = &bo->base.base; - struct panfrost_device *pfdev = to_panfrost_device(obj->dev); - struct io_pgtable_ops *ops = pfdev->mmu->pgtbl_ops; - u64 iova = bo->node.start << PAGE_SHIFT; unsigned int count; struct scatterlist *sgl; - struct sg_table *sgt; - int ret; - - if (WARN_ON(bo->is_mapped)) - return 0; - - sgt = drm_gem_shmem_get_pages_sgt(obj); - if (WARN_ON(IS_ERR(sgt))) - return PTR_ERR(sgt); - - ret = pm_runtime_get_sync(pfdev->dev); - if (ret < 0) - return ret; + struct io_pgtable_ops *ops = pfdev->mmu->pgtbl_ops; + u64 start_iova = iova; mutex_lock(&pfdev->mmu->lock); @@ -178,18 +164,42 @@ int panfrost_mmu_map(struct panfrost_gem_object *bo) while (len) { size_t pgsize = get_pgsize(iova | paddr, len); - ops->map(ops, iova, paddr, pgsize, IOMMU_WRITE | IOMMU_READ); + ops->map(ops, iova, paddr, pgsize, prot); iova += pgsize; paddr += pgsize; len -= pgsize; } } - mmu_hw_do_operation(pfdev, 0, bo->node.start << PAGE_SHIFT, - bo->node.size << PAGE_SHIFT, AS_COMMAND_FLUSH_PT); + mmu_hw_do_operation(pfdev, 0, start_iova, iova - start_iova, + AS_COMMAND_FLUSH_PT); mutex_unlock(&pfdev->mmu->lock); + return 0; +} + +int panfrost_mmu_map(struct panfrost_gem_object *bo) +{ + struct drm_gem_object *obj = &bo->base.base; + struct panfrost_device *pfdev = to_panfrost_device(obj->dev); + struct sg_table *sgt; + int ret; + int prot = IOMMU_READ | IOMMU_WRITE; + + if (WARN_ON(bo->is_mapped)) + return 0; + + sgt = drm_gem_shmem_get_pages_sgt(obj); + if (WARN_ON(IS_ERR(sgt))) + return PTR_ERR(sgt); + + ret = pm_runtime_get_sync(pfdev->dev); + if (ret < 0) + return ret; + + mmu_map_sg(pfdev, bo->node.start << PAGE_SHIFT, prot, sgt); + pm_runtime_mark_last_busy(pfdev->dev); pm_runtime_put_autosuspend(pfdev->dev); bo->is_mapped = true; -- cgit v1.2.3-59-g8ed1b From 203270c025be02d52feca729d9193860317c2e82 Mon Sep 17 00:00:00 2001 From: Rob Herring Date: Thu, 11 Jul 2019 15:56:14 -0600 Subject: drm/panfrost: Add a no execute flag for BO allocations Executable buffers have an alignment restriction that they can't cross 16MB boundary as the GPU program counter is 24-bits. This restriction is currently not handled and we just get lucky. As current userspace assumes all BOs are executable, that has to remain the default. So add a new PANFROST_BO_NOEXEC flag to allow userspace to indicate which BOs are not executable. There is also a restriction that executable buffers cannot start or end on a 4GB boundary. This is mostly avoided as there is only 4GB of space currently and the beginning is already blocked out for NULL ptr detection. Add support to handle this restriction fully regardless of the current constraints. For existing userspace, all created BOs remain executable, but the GPU VA alignment will be increased to the size of the BO. This shouldn't matter as there is plenty of GPU VA space. Cc: Tomeu Vizoso Cc: Boris Brezillon Cc: Robin Murphy Reviewed-by: Steven Price Acked-by: Alyssa Rosenzweig Signed-off-by: Rob Herring Link: https://patchwork.freedesktop.org/patch/msgid/20190808222200.13176-6-robh@kernel.org --- drivers/gpu/drm/panfrost/panfrost_drv.c | 42 ++++++++++++++++++++++----- drivers/gpu/drm/panfrost/panfrost_gem.c | 50 +++++++++++++++++++++++++++++++-- drivers/gpu/drm/panfrost/panfrost_gem.h | 9 +++++- drivers/gpu/drm/panfrost/panfrost_mmu.c | 3 ++ include/uapi/drm/panfrost_drm.h | 2 ++ 5 files changed, 96 insertions(+), 10 deletions(-) (limited to 'drivers/gpu/drm/panfrost/panfrost_mmu.c') diff --git a/drivers/gpu/drm/panfrost/panfrost_drv.c b/drivers/gpu/drm/panfrost/panfrost_drv.c index 2894cfbbce2b..f070f2dd9f84 100644 --- a/drivers/gpu/drm/panfrost/panfrost_drv.c +++ b/drivers/gpu/drm/panfrost/panfrost_drv.c @@ -78,18 +78,19 @@ static int panfrost_ioctl_get_param(struct drm_device *ddev, void *data, struct static int panfrost_ioctl_create_bo(struct drm_device *dev, void *data, struct drm_file *file) { - struct drm_gem_shmem_object *shmem; + struct panfrost_gem_object *bo; struct drm_panfrost_create_bo *args = data; - if (!args->size || args->flags || args->pad) + if (!args->size || args->pad || + (args->flags & ~PANFROST_BO_NOEXEC)) return -EINVAL; - shmem = drm_gem_shmem_create_with_handle(file, dev, args->size, - &args->handle); - if (IS_ERR(shmem)) - return PTR_ERR(shmem); + bo = panfrost_gem_create_with_handle(file, dev, args->size, args->flags, + &args->handle); + if (IS_ERR(bo)) + return PTR_ERR(bo); - args->offset = to_panfrost_bo(&shmem->base)->node.start << PAGE_SHIFT; + args->offset = bo->node.start << PAGE_SHIFT; return 0; } @@ -364,6 +365,32 @@ int panfrost_unstable_ioctl_check(void) return 0; } +#define PFN_4G (SZ_4G >> PAGE_SHIFT) +#define PFN_4G_MASK (PFN_4G - 1) +#define PFN_16M (SZ_16M >> PAGE_SHIFT) + +static void panfrost_drm_mm_color_adjust(const struct drm_mm_node *node, + unsigned long color, + u64 *start, u64 *end) +{ + /* Executable buffers can't start or end on a 4GB boundary */ + if (!(color & PANFROST_BO_NOEXEC)) { + u64 next_seg; + + if ((*start & PFN_4G_MASK) == 0) + (*start)++; + + if ((*end & PFN_4G_MASK) == 0) + (*end)--; + + next_seg = ALIGN(*start, PFN_4G); + if (next_seg - *start <= PFN_16M) + *start = next_seg + 1; + + *end = min(*end, ALIGN(*start, PFN_4G) - 1); + } +} + static int panfrost_open(struct drm_device *dev, struct drm_file *file) { @@ -461,6 +488,7 @@ static int panfrost_probe(struct platform_device *pdev) /* 4G enough for now. can be 48-bit */ drm_mm_init(&pfdev->mm, SZ_32M >> PAGE_SHIFT, (SZ_4G - SZ_32M) >> PAGE_SHIFT); + pfdev->mm.color_adjust = panfrost_drm_mm_color_adjust; pm_runtime_use_autosuspend(pfdev->dev); pm_runtime_set_autosuspend_delay(pfdev->dev, 50); /* ~3 frames */ diff --git a/drivers/gpu/drm/panfrost/panfrost_gem.c b/drivers/gpu/drm/panfrost/panfrost_gem.c index d0b0f5f2270e..1a3bde3dc662 100644 --- a/drivers/gpu/drm/panfrost/panfrost_gem.c +++ b/drivers/gpu/drm/panfrost/panfrost_gem.c @@ -31,13 +31,25 @@ static int panfrost_gem_open(struct drm_gem_object *obj, struct drm_file *file_p { int ret; size_t size = obj->size; - u64 align = size >= SZ_2M ? SZ_2M >> PAGE_SHIFT : 0; + u64 align; struct panfrost_gem_object *bo = to_panfrost_bo(obj); struct panfrost_device *pfdev = obj->dev->dev_private; + unsigned long color = bo->noexec ? PANFROST_BO_NOEXEC : 0; + + /* + * Executable buffers cannot cross a 16MB boundary as the program + * counter is 24-bits. We assume executable buffers will be less than + * 16MB and aligning executable buffers to their size will avoid + * crossing a 16MB boundary. + */ + if (!bo->noexec) + align = size >> PAGE_SHIFT; + else + align = size >= SZ_2M ? SZ_2M >> PAGE_SHIFT : 0; spin_lock(&pfdev->mm_lock); ret = drm_mm_insert_node_generic(&pfdev->mm, &bo->node, - size >> PAGE_SHIFT, align, 0, 0); + size >> PAGE_SHIFT, align, color, 0); if (ret) goto out; @@ -98,16 +110,50 @@ struct drm_gem_object *panfrost_gem_create_object(struct drm_device *dev, size_t return &obj->base.base; } +struct panfrost_gem_object * +panfrost_gem_create_with_handle(struct drm_file *file_priv, + struct drm_device *dev, size_t size, + u32 flags, + uint32_t *handle) +{ + int ret; + struct drm_gem_shmem_object *shmem; + struct panfrost_gem_object *bo; + + shmem = drm_gem_shmem_create(dev, size); + if (IS_ERR(shmem)) + return ERR_CAST(shmem); + + bo = to_panfrost_bo(&shmem->base); + bo->noexec = !!(flags & PANFROST_BO_NOEXEC); + + /* + * Allocate an id of idr table where the obj is registered + * and handle has the id what user can see. + */ + ret = drm_gem_handle_create(file_priv, &shmem->base, handle); + /* drop reference from allocate - handle holds it now. */ + drm_gem_object_put_unlocked(&shmem->base); + if (ret) + return ERR_PTR(ret); + + return bo; +} + struct drm_gem_object * panfrost_gem_prime_import_sg_table(struct drm_device *dev, struct dma_buf_attachment *attach, struct sg_table *sgt) { struct drm_gem_object *obj; + struct panfrost_gem_object *bo; obj = drm_gem_shmem_prime_import_sg_table(dev, attach, sgt); if (IS_ERR(obj)) return ERR_CAST(obj); + bo = to_panfrost_bo(obj); + bo->noexec = true; + return obj; } diff --git a/drivers/gpu/drm/panfrost/panfrost_gem.h b/drivers/gpu/drm/panfrost/panfrost_gem.h index 5f51f881ea3f..d4c7aa1790a7 100644 --- a/drivers/gpu/drm/panfrost/panfrost_gem.h +++ b/drivers/gpu/drm/panfrost/panfrost_gem.h @@ -11,7 +11,8 @@ struct panfrost_gem_object { struct drm_gem_shmem_object base; struct drm_mm_node node; - bool is_mapped; + bool is_mapped :1; + bool noexec :1; }; static inline @@ -27,6 +28,12 @@ panfrost_gem_prime_import_sg_table(struct drm_device *dev, struct dma_buf_attachment *attach, struct sg_table *sgt); +struct panfrost_gem_object * +panfrost_gem_create_with_handle(struct drm_file *file_priv, + struct drm_device *dev, size_t size, + u32 flags, + uint32_t *handle); + void panfrost_gem_shrinker_init(struct drm_device *dev); void panfrost_gem_shrinker_cleanup(struct drm_device *dev); diff --git a/drivers/gpu/drm/panfrost/panfrost_mmu.c b/drivers/gpu/drm/panfrost/panfrost_mmu.c index b4ac149b2399..eba6ce785ef0 100644 --- a/drivers/gpu/drm/panfrost/panfrost_mmu.c +++ b/drivers/gpu/drm/panfrost/panfrost_mmu.c @@ -190,6 +190,9 @@ int panfrost_mmu_map(struct panfrost_gem_object *bo) if (WARN_ON(bo->is_mapped)) return 0; + if (bo->noexec) + prot |= IOMMU_NOEXEC; + sgt = drm_gem_shmem_get_pages_sgt(obj); if (WARN_ON(IS_ERR(sgt))) return PTR_ERR(sgt); diff --git a/include/uapi/drm/panfrost_drm.h b/include/uapi/drm/panfrost_drm.h index 1e547f9692e9..b80c20d17dec 100644 --- a/include/uapi/drm/panfrost_drm.h +++ b/include/uapi/drm/panfrost_drm.h @@ -84,6 +84,8 @@ struct drm_panfrost_wait_bo { __s64 timeout_ns; /* absolute */ }; +#define PANFROST_BO_NOEXEC 1 + /** * struct drm_panfrost_create_bo - ioctl argument for creating Panfrost BOs. * -- cgit v1.2.3-59-g8ed1b From 73e467f60acdabd480d1b377a623ba13db0e5dd2 Mon Sep 17 00:00:00 2001 From: Rob Herring Date: Thu, 8 Aug 2019 14:30:39 -0600 Subject: drm/panfrost: Consolidate reset handling Runtime PM resume and job timeouts both call the same sequence of functions, so consolidate them to a common function. This will make changing the reset related code easier. The MMU also needs some re-initialization on reset, so rework its call. In the process, we hide the address space details within the MMU code in preparation to support multiple address spaces. Cc: Tomeu Vizoso Cc: David Airlie Cc: Daniel Vetter Cc: Robin Murphy Cc: Alyssa Rosenzweig Reviewed-by: Steven Price Signed-off-by: Rob Herring Link: https://patchwork.freedesktop.org/patch/msgid/20190808222200.13176-7-robh@kernel.org --- drivers/gpu/drm/panfrost/panfrost_device.c | 16 ++++++++++------ drivers/gpu/drm/panfrost/panfrost_device.h | 1 + drivers/gpu/drm/panfrost/panfrost_job.c | 7 +------ drivers/gpu/drm/panfrost/panfrost_mmu.c | 16 +++++++++------- drivers/gpu/drm/panfrost/panfrost_mmu.h | 3 +-- 5 files changed, 22 insertions(+), 21 deletions(-) (limited to 'drivers/gpu/drm/panfrost/panfrost_mmu.c') diff --git a/drivers/gpu/drm/panfrost/panfrost_device.c b/drivers/gpu/drm/panfrost/panfrost_device.c index 8a111d7c0200..9814f4ccbd26 100644 --- a/drivers/gpu/drm/panfrost/panfrost_device.c +++ b/drivers/gpu/drm/panfrost/panfrost_device.c @@ -254,18 +254,22 @@ const char *panfrost_exception_name(struct panfrost_device *pfdev, u32 exception return "UNKNOWN"; } +void panfrost_device_reset(struct panfrost_device *pfdev) +{ + panfrost_gpu_soft_reset(pfdev); + + panfrost_gpu_power_on(pfdev); + panfrost_mmu_reset(pfdev); + panfrost_job_enable_interrupts(pfdev); +} + #ifdef CONFIG_PM int panfrost_device_resume(struct device *dev) { struct platform_device *pdev = to_platform_device(dev); struct panfrost_device *pfdev = platform_get_drvdata(pdev); - panfrost_gpu_soft_reset(pfdev); - - /* TODO: Re-enable all other address spaces */ - panfrost_gpu_power_on(pfdev); - panfrost_mmu_enable(pfdev, 0); - panfrost_job_enable_interrupts(pfdev); + panfrost_device_reset(pfdev); panfrost_devfreq_resume(pfdev); return 0; diff --git a/drivers/gpu/drm/panfrost/panfrost_device.h b/drivers/gpu/drm/panfrost/panfrost_device.h index 038b32c62484..4e5641db9c7e 100644 --- a/drivers/gpu/drm/panfrost/panfrost_device.h +++ b/drivers/gpu/drm/panfrost/panfrost_device.h @@ -132,6 +132,7 @@ int panfrost_unstable_ioctl_check(void); int panfrost_device_init(struct panfrost_device *pfdev); void panfrost_device_fini(struct panfrost_device *pfdev); +void panfrost_device_reset(struct panfrost_device *pfdev); int panfrost_device_resume(struct device *dev); int panfrost_device_suspend(struct device *dev); diff --git a/drivers/gpu/drm/panfrost/panfrost_job.c b/drivers/gpu/drm/panfrost/panfrost_job.c index 9bb9260d9181..d567ce98494c 100644 --- a/drivers/gpu/drm/panfrost/panfrost_job.c +++ b/drivers/gpu/drm/panfrost/panfrost_job.c @@ -395,12 +395,7 @@ static void panfrost_job_timedout(struct drm_sched_job *sched_job) /* panfrost_core_dump(pfdev); */ panfrost_devfreq_record_transition(pfdev, js); - panfrost_gpu_soft_reset(pfdev); - - /* TODO: Re-enable all other address spaces */ - panfrost_mmu_enable(pfdev, 0); - panfrost_gpu_power_on(pfdev); - panfrost_job_enable_interrupts(pfdev); + panfrost_device_reset(pfdev); for (i = 0; i < NUM_JOB_SLOTS; i++) drm_sched_resubmit_jobs(&pfdev->js->queue[i].sched); diff --git a/drivers/gpu/drm/panfrost/panfrost_mmu.c b/drivers/gpu/drm/panfrost/panfrost_mmu.c index eba6ce785ef0..13757427b886 100644 --- a/drivers/gpu/drm/panfrost/panfrost_mmu.c +++ b/drivers/gpu/drm/panfrost/panfrost_mmu.c @@ -105,15 +105,12 @@ static int mmu_hw_do_operation(struct panfrost_device *pfdev, u32 as_nr, return ret; } -void panfrost_mmu_enable(struct panfrost_device *pfdev, u32 as_nr) +static void panfrost_mmu_enable(struct panfrost_device *pfdev, u32 as_nr) { struct io_pgtable_cfg *cfg = &pfdev->mmu->pgtbl_cfg; u64 transtab = cfg->arm_mali_lpae_cfg.transtab; u64 memattr = cfg->arm_mali_lpae_cfg.memattr; - mmu_write(pfdev, MMU_INT_CLEAR, ~0); - mmu_write(pfdev, MMU_INT_MASK, ~0); - mmu_write(pfdev, AS_TRANSTAB_LO(as_nr), transtab & 0xffffffffUL); mmu_write(pfdev, AS_TRANSTAB_HI(as_nr), transtab >> 32); @@ -137,6 +134,14 @@ static void mmu_disable(struct panfrost_device *pfdev, u32 as_nr) write_cmd(pfdev, as_nr, AS_COMMAND_UPDATE); } +void panfrost_mmu_reset(struct panfrost_device *pfdev) +{ + panfrost_mmu_enable(pfdev, 0); + + mmu_write(pfdev, MMU_INT_CLEAR, ~0); + mmu_write(pfdev, MMU_INT_MASK, ~0); +} + static size_t get_pgsize(u64 addr, size_t size) { if (addr & (SZ_2M - 1) || size < SZ_2M) @@ -375,9 +380,6 @@ int panfrost_mmu_init(struct panfrost_device *pfdev) dev_err(pfdev->dev, "failed to request mmu irq"); return err; } - mmu_write(pfdev, MMU_INT_CLEAR, ~0); - mmu_write(pfdev, MMU_INT_MASK, ~0); - pfdev->mmu->pgtbl_cfg = (struct io_pgtable_cfg) { .pgsize_bitmap = SZ_4K | SZ_2M, .ias = FIELD_GET(0xff, pfdev->features.mmu_features), diff --git a/drivers/gpu/drm/panfrost/panfrost_mmu.h b/drivers/gpu/drm/panfrost/panfrost_mmu.h index f5878d86a5ce..d5f9b24537db 100644 --- a/drivers/gpu/drm/panfrost/panfrost_mmu.h +++ b/drivers/gpu/drm/panfrost/panfrost_mmu.h @@ -11,7 +11,6 @@ void panfrost_mmu_unmap(struct panfrost_gem_object *bo); int panfrost_mmu_init(struct panfrost_device *pfdev); void panfrost_mmu_fini(struct panfrost_device *pfdev); - -void panfrost_mmu_enable(struct panfrost_device *pfdev, u32 as_nr); +void panfrost_mmu_reset(struct panfrost_device *pfdev); #endif -- cgit v1.2.3-59-g8ed1b From b31bdd1389fc765c07ab3d5b341092cb16807d29 Mon Sep 17 00:00:00 2001 From: Rob Herring Date: Fri, 26 Jul 2019 16:06:57 -0600 Subject: drm/panfrost: Convert MMU IRQ handler to threaded handler In preparation to handle mapping of page faults, we need the MMU handler to be threaded as code paths take a mutex. As the IRQ may be shared, we can't use the default handler and must disable the MMU interrupts locally. Cc: Tomeu Vizoso Cc: Boris Brezillon Cc: Robin Murphy Reviewed-by: Steven Price Acked-by: Alyssa Rosenzweig Signed-off-by: Rob Herring Link: https://patchwork.freedesktop.org/patch/msgid/20190808222200.13176-8-robh@kernel.org --- drivers/gpu/drm/panfrost/panfrost_mmu.c | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) (limited to 'drivers/gpu/drm/panfrost/panfrost_mmu.c') diff --git a/drivers/gpu/drm/panfrost/panfrost_mmu.c b/drivers/gpu/drm/panfrost/panfrost_mmu.c index 13757427b886..b609ee55a872 100644 --- a/drivers/gpu/drm/panfrost/panfrost_mmu.c +++ b/drivers/gpu/drm/panfrost/panfrost_mmu.c @@ -305,12 +305,20 @@ static const char *access_type_name(struct panfrost_device *pfdev, static irqreturn_t panfrost_mmu_irq_handler(int irq, void *data) { struct panfrost_device *pfdev = data; - u32 status = mmu_read(pfdev, MMU_INT_STAT); - int i; - if (!status) + if (!mmu_read(pfdev, MMU_INT_STAT)) return IRQ_NONE; + mmu_write(pfdev, MMU_INT_MASK, 0); + return IRQ_WAKE_THREAD; +} + +static irqreturn_t panfrost_mmu_irq_handler_thread(int irq, void *data) +{ + struct panfrost_device *pfdev = data; + u32 status = mmu_read(pfdev, MMU_INT_RAWSTAT); + int i; + dev_err(pfdev->dev, "mmu irq status=%x\n", status); for (i = 0; status; i++) { @@ -355,6 +363,7 @@ static irqreturn_t panfrost_mmu_irq_handler(int irq, void *data) status &= ~mask; } + mmu_write(pfdev, MMU_INT_MASK, ~0); return IRQ_HANDLED; }; @@ -373,8 +382,9 @@ int panfrost_mmu_init(struct panfrost_device *pfdev) if (irq <= 0) return -ENODEV; - err = devm_request_irq(pfdev->dev, irq, panfrost_mmu_irq_handler, - IRQF_SHARED, "mmu", pfdev); + err = devm_request_threaded_irq(pfdev->dev, irq, panfrost_mmu_irq_handler, + panfrost_mmu_irq_handler_thread, + IRQF_SHARED, "mmu", pfdev); if (err) { dev_err(pfdev->dev, "failed to request mmu irq"); -- cgit v1.2.3-59-g8ed1b From 187d2929206e6b098312c174ea873e4cedf5420d Mon Sep 17 00:00:00 2001 From: Rob Herring Date: Fri, 26 Jul 2019 16:09:43 -0600 Subject: drm/panfrost: Add support for GPU heap allocations The midgard/bifrost GPUs need to allocate GPU heap memory which is allocated on GPU page faults and not pinned in memory. The vendor driver calls this functionality GROW_ON_GPF. This implementation assumes that BOs allocated with the PANFROST_BO_NOEXEC flag are never mmapped or exported. Both of those may actually work, but I'm unsure if there's some interaction there. It would cause the whole object to be pinned in memory which would defeat the point of this. On faults, we map in 2MB at a time in order to utilize huge pages (if enabled). Currently, once we've mapped pages in, they are only unmapped if the BO is freed. Once we add shrinker support, we can unmap pages with the shrinker. Cc: Tomeu Vizoso Cc: Boris Brezillon Cc: Robin Murphy Acked-by: Alyssa Rosenzweig Reviewed-by: Steven Price Signed-off-by: Rob Herring Link: https://patchwork.freedesktop.org/patch/msgid/20190808222200.13176-9-robh@kernel.org --- drivers/gpu/drm/panfrost/TODO | 2 - drivers/gpu/drm/panfrost/panfrost_drv.c | 11 ++- drivers/gpu/drm/panfrost/panfrost_gem.c | 38 ++++++++-- drivers/gpu/drm/panfrost/panfrost_gem.h | 8 ++ drivers/gpu/drm/panfrost/panfrost_mmu.c | 129 +++++++++++++++++++++++++++++--- include/uapi/drm/panfrost_drm.h | 1 + 6 files changed, 172 insertions(+), 17 deletions(-) (limited to 'drivers/gpu/drm/panfrost/panfrost_mmu.c') diff --git a/drivers/gpu/drm/panfrost/TODO b/drivers/gpu/drm/panfrost/TODO index d4c7fb7e7d13..e7727b292355 100644 --- a/drivers/gpu/drm/panfrost/TODO +++ b/drivers/gpu/drm/panfrost/TODO @@ -10,8 +10,6 @@ The hard part is handling when more address spaces are needed than what the h/w provides. -- Support pinning pages on demand (GPU page faults). - - Support userspace controlled GPU virtual addresses. Needed for Vulkan. (Tomeu) - Compute job support. So called 'compute only' jobs need to be plumbed up to diff --git a/drivers/gpu/drm/panfrost/panfrost_drv.c b/drivers/gpu/drm/panfrost/panfrost_drv.c index f070f2dd9f84..994dbc37e4d0 100644 --- a/drivers/gpu/drm/panfrost/panfrost_drv.c +++ b/drivers/gpu/drm/panfrost/panfrost_drv.c @@ -82,7 +82,12 @@ static int panfrost_ioctl_create_bo(struct drm_device *dev, void *data, struct drm_panfrost_create_bo *args = data; if (!args->size || args->pad || - (args->flags & ~PANFROST_BO_NOEXEC)) + (args->flags & ~(PANFROST_BO_NOEXEC | PANFROST_BO_HEAP))) + return -EINVAL; + + /* Heaps should never be executable */ + if ((args->flags & PANFROST_BO_HEAP) && + !(args->flags & PANFROST_BO_NOEXEC)) return -EINVAL; bo = panfrost_gem_create_with_handle(file, dev, args->size, args->flags, @@ -297,6 +302,10 @@ static int panfrost_ioctl_mmap_bo(struct drm_device *dev, void *data, return -ENOENT; } + /* Don't allow mmapping of heap objects as pages are not pinned. */ + if (to_panfrost_bo(gem_obj)->is_heap) + return -EINVAL; + ret = drm_gem_create_mmap_offset(gem_obj); if (ret == 0) args->offset = drm_vma_node_offset_addr(&gem_obj->vma_node); diff --git a/drivers/gpu/drm/panfrost/panfrost_gem.c b/drivers/gpu/drm/panfrost/panfrost_gem.c index 1a3bde3dc662..e71f27c4041e 100644 --- a/drivers/gpu/drm/panfrost/panfrost_gem.c +++ b/drivers/gpu/drm/panfrost/panfrost_gem.c @@ -19,6 +19,20 @@ static void panfrost_gem_free_object(struct drm_gem_object *obj) struct panfrost_gem_object *bo = to_panfrost_bo(obj); struct panfrost_device *pfdev = obj->dev->dev_private; + if (bo->sgts) { + int i; + int n_sgt = bo->base.base.size / SZ_2M; + + for (i = 0; i < n_sgt; i++) { + if (bo->sgts[i].sgl) { + dma_unmap_sg(pfdev->dev, bo->sgts[i].sgl, + bo->sgts[i].nents, DMA_BIDIRECTIONAL); + sg_free_table(&bo->sgts[i]); + } + } + kfree(bo->sgts); + } + mutex_lock(&pfdev->shrinker_lock); if (!list_empty(&bo->base.madv_list)) list_del(&bo->base.madv_list); @@ -53,10 +67,11 @@ static int panfrost_gem_open(struct drm_gem_object *obj, struct drm_file *file_p if (ret) goto out; - ret = panfrost_mmu_map(bo); - if (ret) - drm_mm_remove_node(&bo->node); - + if (!bo->is_heap) { + ret = panfrost_mmu_map(bo); + if (ret) + drm_mm_remove_node(&bo->node); + } out: spin_unlock(&pfdev->mm_lock); return ret; @@ -76,12 +91,20 @@ static void panfrost_gem_close(struct drm_gem_object *obj, struct drm_file *file spin_unlock(&pfdev->mm_lock); } +static int panfrost_gem_pin(struct drm_gem_object *obj) +{ + if (to_panfrost_bo(obj)->is_heap) + return -EINVAL; + + return drm_gem_shmem_pin(obj); +} + static const struct drm_gem_object_funcs panfrost_gem_funcs = { .free = panfrost_gem_free_object, .open = panfrost_gem_open, .close = panfrost_gem_close, .print_info = drm_gem_shmem_print_info, - .pin = drm_gem_shmem_pin, + .pin = panfrost_gem_pin, .unpin = drm_gem_shmem_unpin, .get_sg_table = drm_gem_shmem_get_sg_table, .vmap = drm_gem_shmem_vmap, @@ -120,12 +143,17 @@ panfrost_gem_create_with_handle(struct drm_file *file_priv, struct drm_gem_shmem_object *shmem; struct panfrost_gem_object *bo; + /* Round up heap allocations to 2MB to keep fault handling simple */ + if (flags & PANFROST_BO_HEAP) + size = roundup(size, SZ_2M); + shmem = drm_gem_shmem_create(dev, size); if (IS_ERR(shmem)) return ERR_CAST(shmem); bo = to_panfrost_bo(&shmem->base); bo->noexec = !!(flags & PANFROST_BO_NOEXEC); + bo->is_heap = !!(flags & PANFROST_BO_HEAP); /* * Allocate an id of idr table where the obj is registered diff --git a/drivers/gpu/drm/panfrost/panfrost_gem.h b/drivers/gpu/drm/panfrost/panfrost_gem.h index d4c7aa1790a7..e10f58316915 100644 --- a/drivers/gpu/drm/panfrost/panfrost_gem.h +++ b/drivers/gpu/drm/panfrost/panfrost_gem.h @@ -9,10 +9,12 @@ struct panfrost_gem_object { struct drm_gem_shmem_object base; + struct sg_table *sgts; struct drm_mm_node node; bool is_mapped :1; bool noexec :1; + bool is_heap :1; }; static inline @@ -21,6 +23,12 @@ struct panfrost_gem_object *to_panfrost_bo(struct drm_gem_object *obj) return container_of(to_drm_gem_shmem_obj(obj), struct panfrost_gem_object, base); } +static inline +struct panfrost_gem_object *drm_mm_node_to_panfrost_bo(struct drm_mm_node *node) +{ + return container_of(node, struct panfrost_gem_object, node); +} + struct drm_gem_object *panfrost_gem_create_object(struct drm_device *dev, size_t size); struct drm_gem_object * diff --git a/drivers/gpu/drm/panfrost/panfrost_mmu.c b/drivers/gpu/drm/panfrost/panfrost_mmu.c index b609ee55a872..2ed411f09d80 100644 --- a/drivers/gpu/drm/panfrost/panfrost_mmu.c +++ b/drivers/gpu/drm/panfrost/panfrost_mmu.c @@ -2,6 +2,7 @@ /* Copyright 2019 Linaro, Ltd, Rob Herring */ #include #include +#include #include #include #include @@ -9,6 +10,7 @@ #include #include #include +#include #include #include "panfrost_device.h" @@ -240,12 +242,12 @@ void panfrost_mmu_unmap(struct panfrost_gem_object *bo) size_t unmapped_page; size_t pgsize = get_pgsize(iova, len - unmapped_len); - unmapped_page = ops->unmap(ops, iova, pgsize); - if (!unmapped_page) - break; - - iova += unmapped_page; - unmapped_len += unmapped_page; + if (ops->iova_to_phys(ops, iova)) { + unmapped_page = ops->unmap(ops, iova, pgsize); + WARN_ON(unmapped_page != pgsize); + } + iova += pgsize; + unmapped_len += pgsize; } mmu_hw_do_operation(pfdev, 0, bo->node.start << PAGE_SHIFT, @@ -281,6 +283,105 @@ static const struct iommu_gather_ops mmu_tlb_ops = { .tlb_sync = mmu_tlb_sync_context, }; +static struct drm_mm_node *addr_to_drm_mm_node(struct panfrost_device *pfdev, int as, u64 addr) +{ + struct drm_mm_node *node; + u64 offset = addr >> PAGE_SHIFT; + + drm_mm_for_each_node(node, &pfdev->mm) { + if (offset >= node->start && offset < (node->start + node->size)) + return node; + } + return NULL; +} + +#define NUM_FAULT_PAGES (SZ_2M / PAGE_SIZE) + +int panfrost_mmu_map_fault_addr(struct panfrost_device *pfdev, int as, u64 addr) +{ + int ret, i; + struct drm_mm_node *node; + struct panfrost_gem_object *bo; + struct address_space *mapping; + pgoff_t page_offset; + struct sg_table *sgt; + struct page **pages; + + node = addr_to_drm_mm_node(pfdev, as, addr); + if (!node) + return -ENOENT; + + bo = drm_mm_node_to_panfrost_bo(node); + if (!bo->is_heap) { + dev_WARN(pfdev->dev, "matching BO is not heap type (GPU VA = %llx)", + node->start << PAGE_SHIFT); + return -EINVAL; + } + /* Assume 2MB alignment and size multiple */ + addr &= ~((u64)SZ_2M - 1); + page_offset = addr >> PAGE_SHIFT; + page_offset -= node->start; + + mutex_lock(&bo->base.pages_lock); + + if (!bo->base.pages) { + bo->sgts = kvmalloc_array(bo->base.base.size / SZ_2M, + sizeof(struct sg_table), GFP_KERNEL | __GFP_ZERO); + if (!bo->sgts) + return -ENOMEM; + + pages = kvmalloc_array(bo->base.base.size >> PAGE_SHIFT, + sizeof(struct page *), GFP_KERNEL | __GFP_ZERO); + if (!pages) { + kfree(bo->sgts); + bo->sgts = NULL; + return -ENOMEM; + } + bo->base.pages = pages; + bo->base.pages_use_count = 1; + } else + pages = bo->base.pages; + + mapping = bo->base.base.filp->f_mapping; + mapping_set_unevictable(mapping); + + for (i = page_offset; i < page_offset + NUM_FAULT_PAGES; i++) { + pages[i] = shmem_read_mapping_page(mapping, i); + if (IS_ERR(pages[i])) { + mutex_unlock(&bo->base.pages_lock); + ret = PTR_ERR(pages[i]); + goto err_pages; + } + } + + mutex_unlock(&bo->base.pages_lock); + + sgt = &bo->sgts[page_offset / (SZ_2M / PAGE_SIZE)]; + ret = sg_alloc_table_from_pages(sgt, pages + page_offset, + NUM_FAULT_PAGES, 0, SZ_2M, GFP_KERNEL); + if (ret) + goto err_pages; + + if (!dma_map_sg(pfdev->dev, sgt->sgl, sgt->nents, DMA_BIDIRECTIONAL)) { + ret = -EINVAL; + goto err_map; + } + + mmu_map_sg(pfdev, addr, IOMMU_WRITE | IOMMU_READ | IOMMU_NOEXEC, sgt); + + bo->is_mapped = true; + + dev_dbg(pfdev->dev, "mapped page fault @ %llx", addr); + + return 0; + +err_map: + sg_free_table(sgt); +err_pages: + drm_gem_shmem_put_pages(&bo->base); + return ret; +} + static const char *access_type_name(struct panfrost_device *pfdev, u32 fault_status) { @@ -317,9 +418,7 @@ static irqreturn_t panfrost_mmu_irq_handler_thread(int irq, void *data) { struct panfrost_device *pfdev = data; u32 status = mmu_read(pfdev, MMU_INT_RAWSTAT); - int i; - - dev_err(pfdev->dev, "mmu irq status=%x\n", status); + int i, ret; for (i = 0; status; i++) { u32 mask = BIT(i) | BIT(i + 16); @@ -341,6 +440,18 @@ static irqreturn_t panfrost_mmu_irq_handler_thread(int irq, void *data) access_type = (fault_status >> 8) & 0x3; source_id = (fault_status >> 16); + /* Page fault only */ + if ((status & mask) == BIT(i)) { + WARN_ON(exception_type < 0xC1 || exception_type > 0xC4); + + ret = panfrost_mmu_map_fault_addr(pfdev, i, addr); + if (!ret) { + mmu_write(pfdev, MMU_INT_CLEAR, BIT(i)); + status &= ~mask; + continue; + } + } + /* terminal fault, print info about the fault */ dev_err(pfdev->dev, "Unhandled Page fault in AS%d at VA 0x%016llX\n" diff --git a/include/uapi/drm/panfrost_drm.h b/include/uapi/drm/panfrost_drm.h index b80c20d17dec..ec19db1eead8 100644 --- a/include/uapi/drm/panfrost_drm.h +++ b/include/uapi/drm/panfrost_drm.h @@ -85,6 +85,7 @@ struct drm_panfrost_wait_bo { }; #define PANFROST_BO_NOEXEC 1 +#define PANFROST_BO_HEAP 2 /** * struct drm_panfrost_create_bo - ioctl argument for creating Panfrost BOs. -- cgit v1.2.3-59-g8ed1b From 3efdf83ca0f9d3149f8c2201dad86a74fd952f91 Mon Sep 17 00:00:00 2001 From: Wei Yongjun Date: Wed, 14 Aug 2019 04:48:14 +0000 Subject: drm/panfrost: Fix missing unlock on error in panfrost_mmu_map_fault_addr() Add the missing unlock before return from function panfrost_mmu_map_fault_addr() in the error handling case. Fixes: 187d2929206e ("drm/panfrost: Add support for GPU heap allocations") Signed-off-by: Wei Yongjun Reviewed-by: Steven Price Signed-off-by: Rob Herring Link: https://patchwork.freedesktop.org/patch/msgid/20190814044814.102294-1-weiyongjun1@huawei.com --- drivers/gpu/drm/panfrost/panfrost_mmu.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'drivers/gpu/drm/panfrost/panfrost_mmu.c') diff --git a/drivers/gpu/drm/panfrost/panfrost_mmu.c b/drivers/gpu/drm/panfrost/panfrost_mmu.c index 2ed411f09d80..06f1a563e940 100644 --- a/drivers/gpu/drm/panfrost/panfrost_mmu.c +++ b/drivers/gpu/drm/panfrost/panfrost_mmu.c @@ -327,14 +327,17 @@ int panfrost_mmu_map_fault_addr(struct panfrost_device *pfdev, int as, u64 addr) if (!bo->base.pages) { bo->sgts = kvmalloc_array(bo->base.base.size / SZ_2M, sizeof(struct sg_table), GFP_KERNEL | __GFP_ZERO); - if (!bo->sgts) + if (!bo->sgts) { + mutex_unlock(&bo->base.pages_lock); return -ENOMEM; + } pages = kvmalloc_array(bo->base.base.size >> PAGE_SHIFT, sizeof(struct page *), GFP_KERNEL | __GFP_ZERO); if (!pages) { kfree(bo->sgts); bo->sgts = NULL; + mutex_unlock(&bo->base.pages_lock); return -ENOMEM; } bo->base.pages = pages; -- cgit v1.2.3-59-g8ed1b From 7282f7645d06bf0afe0a3c11ab92d9392528b819 Mon Sep 17 00:00:00 2001 From: Rob Herring Date: Tue, 13 Aug 2019 09:01:15 -0600 Subject: drm/panfrost: Implement per FD address spaces Up until now, a single shared GPU address space was used. This is not ideal as there's no protection between processes and doesn't work for supporting the same GPU/CPU VA feature. Most importantly, this will hopefully mitigate Alyssa's fear of WebGL, whatever that is. Most of the changes here are moving struct drm_mm and struct panfrost_mmu objects from the per device struct to the per FD struct. The critical function is panfrost_mmu_as_get() which handles allocating and switching the h/w address spaces. There's 3 states an AS can be in: free, allocated, and in use. When a job runs, it requests an address space and then marks it not in use when job is complete(but stays assigned). The first time thru, we find a free AS in the alloc_mask and assign the AS to the FD. Then the next time thru, we most likely already have our AS and we just mark it in use with a ref count. We need a ref count because we have multiple job slots. If the job/FD doesn't have an AS assigned and there are no free ones, then we pick an allocated one not in use from our LRU list and switch the AS from the old FD to the new one. Cc: Tomeu Vizoso Cc: David Airlie Cc: Daniel Vetter Cc: Robin Murphy Cc: Steven Price Signed-off-by: Rob Herring Acked-by: Alyssa Rosenzweig Reviewed-by: Steven Price Link: https://patchwork.freedesktop.org/patch/msgid/20190813150115.30338-1-robh@kernel.org --- drivers/gpu/drm/panfrost/TODO | 4 - drivers/gpu/drm/panfrost/panfrost_device.c | 2 + drivers/gpu/drm/panfrost/panfrost_device.h | 24 +++- drivers/gpu/drm/panfrost/panfrost_drv.c | 31 +++- drivers/gpu/drm/panfrost/panfrost_gem.c | 15 +- drivers/gpu/drm/panfrost/panfrost_gem.h | 3 + drivers/gpu/drm/panfrost/panfrost_job.c | 14 +- drivers/gpu/drm/panfrost/panfrost_mmu.c | 222 +++++++++++++++++++++-------- drivers/gpu/drm/panfrost/panfrost_mmu.h | 8 ++ 9 files changed, 236 insertions(+), 87 deletions(-) (limited to 'drivers/gpu/drm/panfrost/panfrost_mmu.c') diff --git a/drivers/gpu/drm/panfrost/TODO b/drivers/gpu/drm/panfrost/TODO index e7727b292355..536a0d4f8d29 100644 --- a/drivers/gpu/drm/panfrost/TODO +++ b/drivers/gpu/drm/panfrost/TODO @@ -6,10 +6,6 @@ - Bifrost specific feature and issue handling - Coherent DMA support -- Per FD address space support. The h/w supports multiple addresses spaces. - The hard part is handling when more address spaces are needed than what - the h/w provides. - - Support userspace controlled GPU virtual addresses. Needed for Vulkan. (Tomeu) - Compute job support. So called 'compute only' jobs need to be plumbed up to diff --git a/drivers/gpu/drm/panfrost/panfrost_device.c b/drivers/gpu/drm/panfrost/panfrost_device.c index 9814f4ccbd26..4da71bb56c20 100644 --- a/drivers/gpu/drm/panfrost/panfrost_device.c +++ b/drivers/gpu/drm/panfrost/panfrost_device.c @@ -123,8 +123,10 @@ int panfrost_device_init(struct panfrost_device *pfdev) mutex_init(&pfdev->sched_lock); mutex_init(&pfdev->reset_lock); INIT_LIST_HEAD(&pfdev->scheduled_jobs); + INIT_LIST_HEAD(&pfdev->as_lru_list); spin_lock_init(&pfdev->hwaccess_lock); + spin_lock_init(&pfdev->as_lock); err = panfrost_clk_init(pfdev); if (err) { diff --git a/drivers/gpu/drm/panfrost/panfrost_device.h b/drivers/gpu/drm/panfrost/panfrost_device.h index 4e5641db9c7e..f503c566e99f 100644 --- a/drivers/gpu/drm/panfrost/panfrost_device.h +++ b/drivers/gpu/drm/panfrost/panfrost_device.h @@ -5,6 +5,8 @@ #ifndef __PANFROST_DEVICE_H__ #define __PANFROST_DEVICE_H__ +#include +#include #include #include #include @@ -63,9 +65,6 @@ struct panfrost_device { spinlock_t hwaccess_lock; - struct drm_mm mm; - spinlock_t mm_lock; - void __iomem *iomem; struct clk *clock; struct clk *bus_clock; @@ -74,7 +73,11 @@ struct panfrost_device { struct panfrost_features features; - struct panfrost_mmu *mmu; + spinlock_t as_lock; + unsigned long as_in_use_mask; + unsigned long as_alloc_mask; + struct list_head as_lru_list; + struct panfrost_job_slot *js; struct panfrost_job *jobs[NUM_JOB_SLOTS]; @@ -98,10 +101,23 @@ struct panfrost_device { } devfreq; }; +struct panfrost_mmu { + struct io_pgtable_cfg pgtbl_cfg; + struct io_pgtable_ops *pgtbl_ops; + struct mutex lock; + int as; + atomic_t as_count; + struct list_head list; +}; + struct panfrost_file_priv { struct panfrost_device *pfdev; struct drm_sched_entity sched_entity[NUM_JOB_SLOTS]; + + struct panfrost_mmu mmu; + struct drm_mm mm; + spinlock_t mm_lock; }; static inline struct panfrost_device *to_panfrost_device(struct drm_device *ddev) diff --git a/drivers/gpu/drm/panfrost/panfrost_drv.c b/drivers/gpu/drm/panfrost/panfrost_drv.c index b41754658681..ae70200be44a 100644 --- a/drivers/gpu/drm/panfrost/panfrost_drv.c +++ b/drivers/gpu/drm/panfrost/panfrost_drv.c @@ -403,6 +403,7 @@ static void panfrost_drm_mm_color_adjust(const struct drm_mm_node *node, static int panfrost_open(struct drm_device *dev, struct drm_file *file) { + int ret; struct panfrost_device *pfdev = dev->dev_private; struct panfrost_file_priv *panfrost_priv; @@ -413,7 +414,28 @@ panfrost_open(struct drm_device *dev, struct drm_file *file) panfrost_priv->pfdev = pfdev; file->driver_priv = panfrost_priv; - return panfrost_job_open(panfrost_priv); + spin_lock_init(&panfrost_priv->mm_lock); + + /* 4G enough for now. can be 48-bit */ + drm_mm_init(&panfrost_priv->mm, SZ_32M >> PAGE_SHIFT, (SZ_4G - SZ_32M) >> PAGE_SHIFT); + panfrost_priv->mm.color_adjust = panfrost_drm_mm_color_adjust; + + ret = panfrost_mmu_pgtable_alloc(panfrost_priv); + if (ret) + goto err_pgtable; + + ret = panfrost_job_open(panfrost_priv); + if (ret) + goto err_job; + + return 0; + +err_job: + panfrost_mmu_pgtable_free(panfrost_priv); +err_pgtable: + drm_mm_takedown(&panfrost_priv->mm); + kfree(panfrost_priv); + return ret; } static void @@ -424,6 +446,8 @@ panfrost_postclose(struct drm_device *dev, struct drm_file *file) panfrost_perfcnt_close(panfrost_priv); panfrost_job_close(panfrost_priv); + panfrost_mmu_pgtable_free(panfrost_priv); + drm_mm_takedown(&panfrost_priv->mm); kfree(panfrost_priv); } @@ -496,14 +520,9 @@ static int panfrost_probe(struct platform_device *pdev) ddev->dev_private = pfdev; pfdev->ddev = ddev; - spin_lock_init(&pfdev->mm_lock); mutex_init(&pfdev->shrinker_lock); INIT_LIST_HEAD(&pfdev->shrinker_list); - /* 4G enough for now. can be 48-bit */ - drm_mm_init(&pfdev->mm, SZ_32M >> PAGE_SHIFT, (SZ_4G - SZ_32M) >> PAGE_SHIFT); - pfdev->mm.color_adjust = panfrost_drm_mm_color_adjust; - pm_runtime_use_autosuspend(pfdev->dev); pm_runtime_set_autosuspend_delay(pfdev->dev, 50); /* ~3 frames */ pm_runtime_enable(pfdev->dev); diff --git a/drivers/gpu/drm/panfrost/panfrost_gem.c b/drivers/gpu/drm/panfrost/panfrost_gem.c index e71f27c4041e..e084bc4e9083 100644 --- a/drivers/gpu/drm/panfrost/panfrost_gem.c +++ b/drivers/gpu/drm/panfrost/panfrost_gem.c @@ -47,8 +47,8 @@ static int panfrost_gem_open(struct drm_gem_object *obj, struct drm_file *file_p size_t size = obj->size; u64 align; struct panfrost_gem_object *bo = to_panfrost_bo(obj); - struct panfrost_device *pfdev = obj->dev->dev_private; unsigned long color = bo->noexec ? PANFROST_BO_NOEXEC : 0; + struct panfrost_file_priv *priv = file_priv->driver_priv; /* * Executable buffers cannot cross a 16MB boundary as the program @@ -61,8 +61,9 @@ static int panfrost_gem_open(struct drm_gem_object *obj, struct drm_file *file_p else align = size >= SZ_2M ? SZ_2M >> PAGE_SHIFT : 0; - spin_lock(&pfdev->mm_lock); - ret = drm_mm_insert_node_generic(&pfdev->mm, &bo->node, + bo->mmu = &priv->mmu; + spin_lock(&priv->mm_lock); + ret = drm_mm_insert_node_generic(&priv->mm, &bo->node, size >> PAGE_SHIFT, align, color, 0); if (ret) goto out; @@ -73,22 +74,22 @@ static int panfrost_gem_open(struct drm_gem_object *obj, struct drm_file *file_p drm_mm_remove_node(&bo->node); } out: - spin_unlock(&pfdev->mm_lock); + spin_unlock(&priv->mm_lock); return ret; } static void panfrost_gem_close(struct drm_gem_object *obj, struct drm_file *file_priv) { struct panfrost_gem_object *bo = to_panfrost_bo(obj); - struct panfrost_device *pfdev = obj->dev->dev_private; + struct panfrost_file_priv *priv = file_priv->driver_priv; if (bo->is_mapped) panfrost_mmu_unmap(bo); - spin_lock(&pfdev->mm_lock); + spin_lock(&priv->mm_lock); if (drm_mm_node_allocated(&bo->node)) drm_mm_remove_node(&bo->node); - spin_unlock(&pfdev->mm_lock); + spin_unlock(&priv->mm_lock); } static int panfrost_gem_pin(struct drm_gem_object *obj) diff --git a/drivers/gpu/drm/panfrost/panfrost_gem.h b/drivers/gpu/drm/panfrost/panfrost_gem.h index e10f58316915..50920819cc16 100644 --- a/drivers/gpu/drm/panfrost/panfrost_gem.h +++ b/drivers/gpu/drm/panfrost/panfrost_gem.h @@ -7,10 +7,13 @@ #include #include +struct panfrost_mmu; + struct panfrost_gem_object { struct drm_gem_shmem_object base; struct sg_table *sgts; + struct panfrost_mmu *mmu; struct drm_mm_node node; bool is_mapped :1; bool noexec :1; diff --git a/drivers/gpu/drm/panfrost/panfrost_job.c b/drivers/gpu/drm/panfrost/panfrost_job.c index 0fc4539fd08d..05c85f45a0de 100644 --- a/drivers/gpu/drm/panfrost/panfrost_job.c +++ b/drivers/gpu/drm/panfrost/panfrost_job.c @@ -153,6 +153,8 @@ static void panfrost_job_hw_submit(struct panfrost_job *job, int js) if (WARN_ON(job_read(pfdev, JS_COMMAND_NEXT(js)))) goto end; + cfg = panfrost_mmu_as_get(pfdev, &job->file_priv->mmu); + panfrost_devfreq_record_transition(pfdev, js); spin_lock_irqsave(&pfdev->hwaccess_lock, flags); @@ -163,8 +165,7 @@ static void panfrost_job_hw_submit(struct panfrost_job *job, int js) /* start MMU, medium priority, cache clean/flush on end, clean/flush on * start */ - /* TODO: different address spaces */ - cfg = JS_CONFIG_THREAD_PRI(8) | + cfg |= JS_CONFIG_THREAD_PRI(8) | JS_CONFIG_START_FLUSH_CLEAN_INVALIDATE | JS_CONFIG_END_FLUSH_CLEAN_INVALIDATE; @@ -377,8 +378,9 @@ static void panfrost_job_timedout(struct drm_sched_job *sched_job) if (dma_fence_is_signaled(job->done_fence)) return; - dev_err(pfdev->dev, "gpu sched timeout, js=%d, status=0x%x, head=0x%x, tail=0x%x, sched_job=%p", + dev_err(pfdev->dev, "gpu sched timeout, js=%d, config=0x%x, status=0x%x, head=0x%x, tail=0x%x, sched_job=%p", js, + job_read(pfdev, JS_CONFIG(js)), job_read(pfdev, JS_STATUS(js)), job_read(pfdev, JS_HEAD_LO(js)), job_read(pfdev, JS_TAIL_LO(js)), @@ -448,8 +450,12 @@ static irqreturn_t panfrost_job_irq_handler(int irq, void *data) } if (status & JOB_INT_MASK_DONE(j)) { + struct panfrost_job *job = pfdev->jobs[j]; + + pfdev->jobs[j] = NULL; + panfrost_mmu_as_put(pfdev, &job->file_priv->mmu); panfrost_devfreq_record_transition(pfdev, j); - dma_fence_signal(pfdev->jobs[j]->done_fence); + dma_fence_signal(job->done_fence); } status &= ~mask; diff --git a/drivers/gpu/drm/panfrost/panfrost_mmu.c b/drivers/gpu/drm/panfrost/panfrost_mmu.c index 06f1a563e940..842bdd7cf6be 100644 --- a/drivers/gpu/drm/panfrost/panfrost_mmu.c +++ b/drivers/gpu/drm/panfrost/panfrost_mmu.c @@ -1,5 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 /* Copyright 2019 Linaro, Ltd, Rob Herring */ +#include #include #include #include @@ -22,12 +23,6 @@ #define mmu_write(dev, reg, data) writel(data, dev->iomem + reg) #define mmu_read(dev, reg) readl(dev->iomem + reg) -struct panfrost_mmu { - struct io_pgtable_cfg pgtbl_cfg; - struct io_pgtable_ops *pgtbl_ops; - struct mutex lock; -}; - static int wait_ready(struct panfrost_device *pfdev, u32 as_nr) { int ret; @@ -85,13 +80,19 @@ static void lock_region(struct panfrost_device *pfdev, u32 as_nr, } -static int mmu_hw_do_operation(struct panfrost_device *pfdev, u32 as_nr, - u64 iova, size_t size, u32 op) +static int mmu_hw_do_operation(struct panfrost_device *pfdev, + struct panfrost_mmu *mmu, + u64 iova, size_t size, u32 op) { - unsigned long flags; - int ret; + int ret, as_nr; - spin_lock_irqsave(&pfdev->hwaccess_lock, flags); + spin_lock(&pfdev->as_lock); + as_nr = mmu->as; + + if (as_nr < 0) { + spin_unlock(&pfdev->as_lock); + return 0; + } if (op != AS_COMMAND_UNLOCK) lock_region(pfdev, as_nr, iova, size); @@ -102,14 +103,15 @@ static int mmu_hw_do_operation(struct panfrost_device *pfdev, u32 as_nr, /* Wait for the flush to complete */ ret = wait_ready(pfdev, as_nr); - spin_unlock_irqrestore(&pfdev->hwaccess_lock, flags); + spin_unlock(&pfdev->as_lock); return ret; } -static void panfrost_mmu_enable(struct panfrost_device *pfdev, u32 as_nr) +static void panfrost_mmu_enable(struct panfrost_device *pfdev, struct panfrost_mmu *mmu) { - struct io_pgtable_cfg *cfg = &pfdev->mmu->pgtbl_cfg; + int as_nr = mmu->as; + struct io_pgtable_cfg *cfg = &mmu->pgtbl_cfg; u64 transtab = cfg->arm_mali_lpae_cfg.transtab; u64 memattr = cfg->arm_mali_lpae_cfg.memattr; @@ -136,9 +138,75 @@ static void mmu_disable(struct panfrost_device *pfdev, u32 as_nr) write_cmd(pfdev, as_nr, AS_COMMAND_UPDATE); } +u32 panfrost_mmu_as_get(struct panfrost_device *pfdev, struct panfrost_mmu *mmu) +{ + int as; + + spin_lock(&pfdev->as_lock); + + as = mmu->as; + if (as >= 0) { + int en = atomic_inc_return(&mmu->as_count); + WARN_ON(en >= NUM_JOB_SLOTS); + + list_move(&mmu->list, &pfdev->as_lru_list); + goto out; + } + + /* Check for a free AS */ + as = ffz(pfdev->as_alloc_mask); + if (!(BIT(as) & pfdev->features.as_present)) { + struct panfrost_mmu *lru_mmu; + + list_for_each_entry_reverse(lru_mmu, &pfdev->as_lru_list, list) { + if (!atomic_read(&lru_mmu->as_count)) + break; + } + WARN_ON(&lru_mmu->list == &pfdev->as_lru_list); + + list_del_init(&lru_mmu->list); + as = lru_mmu->as; + + WARN_ON(as < 0); + lru_mmu->as = -1; + } + + /* Assign the free or reclaimed AS to the FD */ + mmu->as = as; + set_bit(as, &pfdev->as_alloc_mask); + atomic_set(&mmu->as_count, 1); + list_add(&mmu->list, &pfdev->as_lru_list); + + dev_dbg(pfdev->dev, "Assigned AS%d to mmu %p, alloc_mask=%lx", as, mmu, pfdev->as_alloc_mask); + + panfrost_mmu_enable(pfdev, mmu); + +out: + spin_unlock(&pfdev->as_lock); + return as; +} + +void panfrost_mmu_as_put(struct panfrost_device *pfdev, struct panfrost_mmu *mmu) +{ + atomic_dec(&mmu->as_count); + WARN_ON(atomic_read(&mmu->as_count) < 0); +} + void panfrost_mmu_reset(struct panfrost_device *pfdev) { - panfrost_mmu_enable(pfdev, 0); + struct panfrost_mmu *mmu, *mmu_tmp; + + spin_lock(&pfdev->as_lock); + + pfdev->as_alloc_mask = 0; + + list_for_each_entry_safe(mmu, mmu_tmp, &pfdev->as_lru_list, list) { + mmu->as = -1; + atomic_set(&mmu->as_count, 0); + list_del_init(&mmu->list); + } + + spin_unlock(&pfdev->as_lock); mmu_write(pfdev, MMU_INT_CLEAR, ~0); mmu_write(pfdev, MMU_INT_MASK, ~0); @@ -152,21 +220,21 @@ static size_t get_pgsize(u64 addr, size_t size) return SZ_2M; } -static int mmu_map_sg(struct panfrost_device *pfdev, u64 iova, - int prot, struct sg_table *sgt) +static int mmu_map_sg(struct panfrost_device *pfdev, struct panfrost_mmu *mmu, + u64 iova, int prot, struct sg_table *sgt) { unsigned int count; struct scatterlist *sgl; - struct io_pgtable_ops *ops = pfdev->mmu->pgtbl_ops; + struct io_pgtable_ops *ops = mmu->pgtbl_ops; u64 start_iova = iova; - mutex_lock(&pfdev->mmu->lock); + mutex_lock(&mmu->lock); for_each_sg(sgt->sgl, sgl, sgt->nents, count) { unsigned long paddr = sg_dma_address(sgl); size_t len = sg_dma_len(sgl); - dev_dbg(pfdev->dev, "map: iova=%llx, paddr=%lx, len=%zx", iova, paddr, len); + dev_dbg(pfdev->dev, "map: as=%d, iova=%llx, paddr=%lx, len=%zx", mmu->as, iova, paddr, len); while (len) { size_t pgsize = get_pgsize(iova | paddr, len); @@ -178,10 +246,10 @@ static int mmu_map_sg(struct panfrost_device *pfdev, u64 iova, } } - mmu_hw_do_operation(pfdev, 0, start_iova, iova - start_iova, + mmu_hw_do_operation(pfdev, mmu, start_iova, iova - start_iova, AS_COMMAND_FLUSH_PT); - mutex_unlock(&pfdev->mmu->lock); + mutex_unlock(&mmu->lock); return 0; } @@ -208,7 +276,7 @@ int panfrost_mmu_map(struct panfrost_gem_object *bo) if (ret < 0) return ret; - mmu_map_sg(pfdev, bo->node.start << PAGE_SHIFT, prot, sgt); + mmu_map_sg(pfdev, bo->mmu, bo->node.start << PAGE_SHIFT, prot, sgt); pm_runtime_mark_last_busy(pfdev->dev); pm_runtime_put_autosuspend(pfdev->dev); @@ -221,7 +289,7 @@ void panfrost_mmu_unmap(struct panfrost_gem_object *bo) { struct drm_gem_object *obj = &bo->base.base; struct panfrost_device *pfdev = to_panfrost_device(obj->dev); - struct io_pgtable_ops *ops = pfdev->mmu->pgtbl_ops; + struct io_pgtable_ops *ops = bo->mmu->pgtbl_ops; u64 iova = bo->node.start << PAGE_SHIFT; size_t len = bo->node.size << PAGE_SHIFT; size_t unmapped_len = 0; @@ -230,13 +298,13 @@ void panfrost_mmu_unmap(struct panfrost_gem_object *bo) if (WARN_ON(!bo->is_mapped)) return; - dev_dbg(pfdev->dev, "unmap: iova=%llx, len=%zx", iova, len); + dev_dbg(pfdev->dev, "unmap: as=%d, iova=%llx, len=%zx", bo->mmu->as, iova, len); ret = pm_runtime_get_sync(pfdev->dev); if (ret < 0) return; - mutex_lock(&pfdev->mmu->lock); + mutex_lock(&bo->mmu->lock); while (unmapped_len < len) { size_t unmapped_page; @@ -250,10 +318,10 @@ void panfrost_mmu_unmap(struct panfrost_gem_object *bo) unmapped_len += pgsize; } - mmu_hw_do_operation(pfdev, 0, bo->node.start << PAGE_SHIFT, + mmu_hw_do_operation(pfdev, bo->mmu, bo->node.start << PAGE_SHIFT, bo->node.size << PAGE_SHIFT, AS_COMMAND_FLUSH_PT); - mutex_unlock(&pfdev->mmu->lock); + mutex_unlock(&bo->mmu->lock); pm_runtime_mark_last_busy(pfdev->dev); pm_runtime_put_autosuspend(pfdev->dev); @@ -262,9 +330,9 @@ void panfrost_mmu_unmap(struct panfrost_gem_object *bo) static void mmu_tlb_inv_context_s1(void *cookie) { - struct panfrost_device *pfdev = cookie; + struct panfrost_file_priv *priv = cookie; - mmu_hw_do_operation(pfdev, 0, 0, ~0UL, AS_COMMAND_FLUSH_MEM); + mmu_hw_do_operation(priv->pfdev, &priv->mmu, 0, ~0UL, AS_COMMAND_FLUSH_MEM); } static void mmu_tlb_inv_range_nosync(unsigned long iova, size_t size, @@ -283,16 +351,69 @@ static const struct iommu_gather_ops mmu_tlb_ops = { .tlb_sync = mmu_tlb_sync_context, }; +int panfrost_mmu_pgtable_alloc(struct panfrost_file_priv *priv) +{ + struct panfrost_mmu *mmu = &priv->mmu; + struct panfrost_device *pfdev = priv->pfdev; + + mutex_init(&mmu->lock); + INIT_LIST_HEAD(&mmu->list); + mmu->as = -1; + + mmu->pgtbl_cfg = (struct io_pgtable_cfg) { + .pgsize_bitmap = SZ_4K | SZ_2M, + .ias = FIELD_GET(0xff, pfdev->features.mmu_features), + .oas = FIELD_GET(0xff00, pfdev->features.mmu_features), + .tlb = &mmu_tlb_ops, + .iommu_dev = pfdev->dev, + }; + + mmu->pgtbl_ops = alloc_io_pgtable_ops(ARM_MALI_LPAE, &mmu->pgtbl_cfg, + priv); + if (!mmu->pgtbl_ops) + return -EINVAL; + + return 0; +} + +void panfrost_mmu_pgtable_free(struct panfrost_file_priv *priv) +{ + struct panfrost_device *pfdev = priv->pfdev; + struct panfrost_mmu *mmu = &priv->mmu; + + spin_lock(&pfdev->as_lock); + if (mmu->as >= 0) { + clear_bit(mmu->as, &pfdev->as_alloc_mask); + clear_bit(mmu->as, &pfdev->as_in_use_mask); + list_del(&mmu->list); + } + spin_unlock(&pfdev->as_lock); + + free_io_pgtable_ops(mmu->pgtbl_ops); +} + static struct drm_mm_node *addr_to_drm_mm_node(struct panfrost_device *pfdev, int as, u64 addr) { - struct drm_mm_node *node; + struct drm_mm_node *node = NULL; u64 offset = addr >> PAGE_SHIFT; + struct panfrost_mmu *mmu; - drm_mm_for_each_node(node, &pfdev->mm) { - if (offset >= node->start && offset < (node->start + node->size)) - return node; + spin_lock(&pfdev->as_lock); + list_for_each_entry(mmu, &pfdev->as_lru_list, list) { + struct panfrost_file_priv *priv; + if (as != mmu->as) + continue; + + priv = container_of(mmu, struct panfrost_file_priv, mmu); + drm_mm_for_each_node(node, &priv->mm) { + if (offset >= node->start && offset < (node->start + node->size)) + goto out; + } } - return NULL; + +out: + spin_unlock(&pfdev->as_lock); + return node; } #define NUM_FAULT_PAGES (SZ_2M / PAGE_SIZE) @@ -317,6 +438,8 @@ int panfrost_mmu_map_fault_addr(struct panfrost_device *pfdev, int as, u64 addr) node->start << PAGE_SHIFT); return -EINVAL; } + WARN_ON(bo->mmu->as != as); + /* Assume 2MB alignment and size multiple */ addr &= ~((u64)SZ_2M - 1); page_offset = addr >> PAGE_SHIFT; @@ -370,11 +493,11 @@ int panfrost_mmu_map_fault_addr(struct panfrost_device *pfdev, int as, u64 addr) goto err_map; } - mmu_map_sg(pfdev, addr, IOMMU_WRITE | IOMMU_READ | IOMMU_NOEXEC, sgt); + mmu_map_sg(pfdev, bo->mmu, addr, IOMMU_WRITE | IOMMU_READ | IOMMU_NOEXEC, sgt); bo->is_mapped = true; - dev_dbg(pfdev->dev, "mapped page fault @ %llx", addr); + dev_dbg(pfdev->dev, "mapped page fault @ AS%d %llx", as, addr); return 0; @@ -483,15 +606,8 @@ static irqreturn_t panfrost_mmu_irq_handler_thread(int irq, void *data) int panfrost_mmu_init(struct panfrost_device *pfdev) { - struct io_pgtable_ops *pgtbl_ops; int err, irq; - pfdev->mmu = devm_kzalloc(pfdev->dev, sizeof(*pfdev->mmu), GFP_KERNEL); - if (!pfdev->mmu) - return -ENOMEM; - - mutex_init(&pfdev->mmu->lock); - irq = platform_get_irq_byname(to_platform_device(pfdev->dev), "mmu"); if (irq <= 0) return -ENODEV; @@ -504,22 +620,6 @@ int panfrost_mmu_init(struct panfrost_device *pfdev) dev_err(pfdev->dev, "failed to request mmu irq"); return err; } - pfdev->mmu->pgtbl_cfg = (struct io_pgtable_cfg) { - .pgsize_bitmap = SZ_4K | SZ_2M, - .ias = FIELD_GET(0xff, pfdev->features.mmu_features), - .oas = FIELD_GET(0xff00, pfdev->features.mmu_features), - .tlb = &mmu_tlb_ops, - .iommu_dev = pfdev->dev, - }; - - pgtbl_ops = alloc_io_pgtable_ops(ARM_MALI_LPAE, &pfdev->mmu->pgtbl_cfg, - pfdev); - if (!pgtbl_ops) - return -ENOMEM; - - pfdev->mmu->pgtbl_ops = pgtbl_ops; - - panfrost_mmu_enable(pfdev, 0); return 0; } @@ -528,6 +628,4 @@ void panfrost_mmu_fini(struct panfrost_device *pfdev) { mmu_write(pfdev, MMU_INT_MASK, 0); mmu_disable(pfdev, 0); - - free_io_pgtable_ops(pfdev->mmu->pgtbl_ops); } diff --git a/drivers/gpu/drm/panfrost/panfrost_mmu.h b/drivers/gpu/drm/panfrost/panfrost_mmu.h index d5f9b24537db..7c5b6775ae23 100644 --- a/drivers/gpu/drm/panfrost/panfrost_mmu.h +++ b/drivers/gpu/drm/panfrost/panfrost_mmu.h @@ -5,6 +5,8 @@ #define __PANFROST_MMU_H__ struct panfrost_gem_object; +struct panfrost_file_priv; +struct panfrost_mmu; int panfrost_mmu_map(struct panfrost_gem_object *bo); void panfrost_mmu_unmap(struct panfrost_gem_object *bo); @@ -13,4 +15,10 @@ int panfrost_mmu_init(struct panfrost_device *pfdev); void panfrost_mmu_fini(struct panfrost_device *pfdev); void panfrost_mmu_reset(struct panfrost_device *pfdev); +u32 panfrost_mmu_as_get(struct panfrost_device *pfdev, struct panfrost_mmu *mmu); +void panfrost_mmu_as_put(struct panfrost_device *pfdev, struct panfrost_mmu *mmu); + +int panfrost_mmu_pgtable_alloc(struct panfrost_file_priv *priv); +void panfrost_mmu_pgtable_free(struct panfrost_file_priv *priv); + #endif -- cgit v1.2.3-59-g8ed1b From e316f08f1abf5f1116118e25dce7bc3e9ab03246 Mon Sep 17 00:00:00 2001 From: Rob Herring Date: Mon, 26 Aug 2019 17:33:12 -0500 Subject: drm/panfrost: Remove unnecessary mmu->lock mutex There's no need to serialize io-pgtable calls and the as_lock is sufficient to serialize flush operations, so we can remove the per page table lock. Fixes: 7282f7645d06 ("drm/panfrost: Implement per FD address spaces") Suggested-by: Robin Murphy Cc: Tomeu Vizoso Cc: David Airlie Cc: Daniel Vetter Signed-off-by: Rob Herring Acked-by: Alyssa Rosenzweig Reviewed-by: Robin Murphy Reviewed-by: Steven Price Link: https://patchwork.freedesktop.org/patch/msgid/20190826223317.28509-4-robh@kernel.org --- drivers/gpu/drm/panfrost/panfrost_device.h | 1 - drivers/gpu/drm/panfrost/panfrost_mmu.c | 9 --------- 2 files changed, 10 deletions(-) (limited to 'drivers/gpu/drm/panfrost/panfrost_mmu.c') diff --git a/drivers/gpu/drm/panfrost/panfrost_device.h b/drivers/gpu/drm/panfrost/panfrost_device.h index f503c566e99f..b7fa08ed3a23 100644 --- a/drivers/gpu/drm/panfrost/panfrost_device.h +++ b/drivers/gpu/drm/panfrost/panfrost_device.h @@ -104,7 +104,6 @@ struct panfrost_device { struct panfrost_mmu { struct io_pgtable_cfg pgtbl_cfg; struct io_pgtable_ops *pgtbl_ops; - struct mutex lock; int as; atomic_t as_count; struct list_head list; diff --git a/drivers/gpu/drm/panfrost/panfrost_mmu.c b/drivers/gpu/drm/panfrost/panfrost_mmu.c index 842bdd7cf6be..3a8bcfa7e7b6 100644 --- a/drivers/gpu/drm/panfrost/panfrost_mmu.c +++ b/drivers/gpu/drm/panfrost/panfrost_mmu.c @@ -228,8 +228,6 @@ static int mmu_map_sg(struct panfrost_device *pfdev, struct panfrost_mmu *mmu, struct io_pgtable_ops *ops = mmu->pgtbl_ops; u64 start_iova = iova; - mutex_lock(&mmu->lock); - for_each_sg(sgt->sgl, sgl, sgt->nents, count) { unsigned long paddr = sg_dma_address(sgl); size_t len = sg_dma_len(sgl); @@ -249,8 +247,6 @@ static int mmu_map_sg(struct panfrost_device *pfdev, struct panfrost_mmu *mmu, mmu_hw_do_operation(pfdev, mmu, start_iova, iova - start_iova, AS_COMMAND_FLUSH_PT); - mutex_unlock(&mmu->lock); - return 0; } @@ -304,8 +300,6 @@ void panfrost_mmu_unmap(struct panfrost_gem_object *bo) if (ret < 0) return; - mutex_lock(&bo->mmu->lock); - while (unmapped_len < len) { size_t unmapped_page; size_t pgsize = get_pgsize(iova, len - unmapped_len); @@ -321,8 +315,6 @@ void panfrost_mmu_unmap(struct panfrost_gem_object *bo) mmu_hw_do_operation(pfdev, bo->mmu, bo->node.start << PAGE_SHIFT, bo->node.size << PAGE_SHIFT, AS_COMMAND_FLUSH_PT); - mutex_unlock(&bo->mmu->lock); - pm_runtime_mark_last_busy(pfdev->dev); pm_runtime_put_autosuspend(pfdev->dev); bo->is_mapped = false; @@ -356,7 +348,6 @@ int panfrost_mmu_pgtable_alloc(struct panfrost_file_priv *priv) struct panfrost_mmu *mmu = &priv->mmu; struct panfrost_device *pfdev = priv->pfdev; - mutex_init(&mmu->lock); INIT_LIST_HEAD(&mmu->list); mmu->as = -1; -- cgit v1.2.3-59-g8ed1b From ec7eba47da867e1dd0ff7d442a8717d70c064658 Mon Sep 17 00:00:00 2001 From: Rob Herring Date: Mon, 26 Aug 2019 17:33:13 -0500 Subject: drm/panfrost: Rework page table flushing and runtime PM interaction There is no point in resuming the h/w just to do flush operations and doing so takes several locks which cause lockdep issues with the shrinker. Rework the flush operations to only happen when the h/w is already awake. This avoids taking any locks associated with resuming which trigger lockdep warnings. Fixes: 013b65101315 ("drm/panfrost: Add madvise and shrinker support") Cc: Tomeu Vizoso Cc: David Airlie Cc: Daniel Vetter Signed-off-by: Rob Herring Acked-by: Alyssa Rosenzweig Reviewed-by: Robin Murphy Reviewed-by: Steven Price Link: https://patchwork.freedesktop.org/patch/msgid/20190826223317.28509-5-robh@kernel.org --- drivers/gpu/drm/panfrost/panfrost_mmu.c | 38 ++++++++++++++++----------------- 1 file changed, 18 insertions(+), 20 deletions(-) (limited to 'drivers/gpu/drm/panfrost/panfrost_mmu.c') diff --git a/drivers/gpu/drm/panfrost/panfrost_mmu.c b/drivers/gpu/drm/panfrost/panfrost_mmu.c index 3a8bcfa7e7b6..2204e60f7808 100644 --- a/drivers/gpu/drm/panfrost/panfrost_mmu.c +++ b/drivers/gpu/drm/panfrost/panfrost_mmu.c @@ -220,6 +220,22 @@ static size_t get_pgsize(u64 addr, size_t size) return SZ_2M; } +void panfrost_mmu_flush_range(struct panfrost_device *pfdev, + struct panfrost_mmu *mmu, + u64 iova, size_t size) +{ + if (mmu->as < 0) + return; + + pm_runtime_get_noresume(pfdev->dev); + + /* Flush the PTs only if we're already awake */ + if (pm_runtime_active(pfdev->dev)) + mmu_hw_do_operation(pfdev, mmu, iova, size, AS_COMMAND_FLUSH_PT); + + pm_runtime_put_sync_autosuspend(pfdev->dev); +} + static int mmu_map_sg(struct panfrost_device *pfdev, struct panfrost_mmu *mmu, u64 iova, int prot, struct sg_table *sgt) { @@ -244,8 +260,7 @@ static int mmu_map_sg(struct panfrost_device *pfdev, struct panfrost_mmu *mmu, } } - mmu_hw_do_operation(pfdev, mmu, start_iova, iova - start_iova, - AS_COMMAND_FLUSH_PT); + panfrost_mmu_flush_range(pfdev, mmu, start_iova, iova - start_iova); return 0; } @@ -255,7 +270,6 @@ int panfrost_mmu_map(struct panfrost_gem_object *bo) struct drm_gem_object *obj = &bo->base.base; struct panfrost_device *pfdev = to_panfrost_device(obj->dev); struct sg_table *sgt; - int ret; int prot = IOMMU_READ | IOMMU_WRITE; if (WARN_ON(bo->is_mapped)) @@ -268,14 +282,7 @@ int panfrost_mmu_map(struct panfrost_gem_object *bo) if (WARN_ON(IS_ERR(sgt))) return PTR_ERR(sgt); - ret = pm_runtime_get_sync(pfdev->dev); - if (ret < 0) - return ret; - mmu_map_sg(pfdev, bo->mmu, bo->node.start << PAGE_SHIFT, prot, sgt); - - pm_runtime_mark_last_busy(pfdev->dev); - pm_runtime_put_autosuspend(pfdev->dev); bo->is_mapped = true; return 0; @@ -289,17 +296,12 @@ void panfrost_mmu_unmap(struct panfrost_gem_object *bo) u64 iova = bo->node.start << PAGE_SHIFT; size_t len = bo->node.size << PAGE_SHIFT; size_t unmapped_len = 0; - int ret; if (WARN_ON(!bo->is_mapped)) return; dev_dbg(pfdev->dev, "unmap: as=%d, iova=%llx, len=%zx", bo->mmu->as, iova, len); - ret = pm_runtime_get_sync(pfdev->dev); - if (ret < 0) - return; - while (unmapped_len < len) { size_t unmapped_page; size_t pgsize = get_pgsize(iova, len - unmapped_len); @@ -312,11 +314,7 @@ void panfrost_mmu_unmap(struct panfrost_gem_object *bo) unmapped_len += pgsize; } - mmu_hw_do_operation(pfdev, bo->mmu, bo->node.start << PAGE_SHIFT, - bo->node.size << PAGE_SHIFT, AS_COMMAND_FLUSH_PT); - - pm_runtime_mark_last_busy(pfdev->dev); - pm_runtime_put_autosuspend(pfdev->dev); + panfrost_mmu_flush_range(pfdev, bo->mmu, bo->node.start << PAGE_SHIFT, len); bo->is_mapped = false; } -- cgit v1.2.3-59-g8ed1b From 86df65f39b009e00cf2826eac72c5a969b878065 Mon Sep 17 00:00:00 2001 From: Rob Herring Date: Mon, 26 Aug 2019 17:33:14 -0500 Subject: drm/panfrost: Split mmu_hw_do_operation into locked and unlocked version In preparation to call mmu_hw_do_operation with the as_lock already held, Add a mmu_hw_do_operation_locked function. Fixes: 7282f7645d06 ("drm/panfrost: Implement per FD address spaces") Cc: Tomeu Vizoso Cc: David Airlie Cc: Daniel Vetter Signed-off-by: Rob Herring Acked-by: Alyssa Rosenzweig Reviewed-by: Steven Price Link: https://patchwork.freedesktop.org/patch/msgid/20190826223317.28509-6-robh@kernel.org --- drivers/gpu/drm/panfrost/panfrost_mmu.c | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) (limited to 'drivers/gpu/drm/panfrost/panfrost_mmu.c') diff --git a/drivers/gpu/drm/panfrost/panfrost_mmu.c b/drivers/gpu/drm/panfrost/panfrost_mmu.c index 2204e60f7808..3407b00d0a3a 100644 --- a/drivers/gpu/drm/panfrost/panfrost_mmu.c +++ b/drivers/gpu/drm/panfrost/panfrost_mmu.c @@ -80,19 +80,11 @@ static void lock_region(struct panfrost_device *pfdev, u32 as_nr, } -static int mmu_hw_do_operation(struct panfrost_device *pfdev, - struct panfrost_mmu *mmu, - u64 iova, size_t size, u32 op) +static int mmu_hw_do_operation_locked(struct panfrost_device *pfdev, int as_nr, + u64 iova, size_t size, u32 op) { - int ret, as_nr; - - spin_lock(&pfdev->as_lock); - as_nr = mmu->as; - - if (as_nr < 0) { - spin_unlock(&pfdev->as_lock); + if (as_nr < 0) return 0; - } if (op != AS_COMMAND_UNLOCK) lock_region(pfdev, as_nr, iova, size); @@ -101,10 +93,18 @@ static int mmu_hw_do_operation(struct panfrost_device *pfdev, write_cmd(pfdev, as_nr, op); /* Wait for the flush to complete */ - ret = wait_ready(pfdev, as_nr); + return wait_ready(pfdev, as_nr); +} - spin_unlock(&pfdev->as_lock); +static int mmu_hw_do_operation(struct panfrost_device *pfdev, + struct panfrost_mmu *mmu, + u64 iova, size_t size, u32 op) +{ + int ret; + spin_lock(&pfdev->as_lock); + ret = mmu_hw_do_operation_locked(pfdev, mmu->as, iova, size, op); + spin_unlock(&pfdev->as_lock); return ret; } -- cgit v1.2.3-59-g8ed1b From 5924d40958dfc2b8996fbf788a9d58e411a6db71 Mon Sep 17 00:00:00 2001 From: Rob Herring Date: Mon, 26 Aug 2019 17:33:15 -0500 Subject: drm/panfrost: Add cache/TLB flush before switching address space It's not entirely clear if this is required, but add a flush of GPU caches and TLBs before we change an address space to new page tables. Fixes: 7282f7645d06 ("drm/panfrost: Implement per FD address spaces") Cc: Tomeu Vizoso Cc: David Airlie Cc: Daniel Vetter Signed-off-by: Rob Herring Acked-by: Alyssa Rosenzweig Reviewed-by: Robin Murphy Reviewed-by: Steven Price Link: https://patchwork.freedesktop.org/patch/msgid/20190826223317.28509-7-robh@kernel.org --- drivers/gpu/drm/panfrost/panfrost_mmu.c | 2 ++ 1 file changed, 2 insertions(+) (limited to 'drivers/gpu/drm/panfrost/panfrost_mmu.c') diff --git a/drivers/gpu/drm/panfrost/panfrost_mmu.c b/drivers/gpu/drm/panfrost/panfrost_mmu.c index 3407b00d0a3a..d1ebde3327fe 100644 --- a/drivers/gpu/drm/panfrost/panfrost_mmu.c +++ b/drivers/gpu/drm/panfrost/panfrost_mmu.c @@ -115,6 +115,8 @@ static void panfrost_mmu_enable(struct panfrost_device *pfdev, struct panfrost_m u64 transtab = cfg->arm_mali_lpae_cfg.transtab; u64 memattr = cfg->arm_mali_lpae_cfg.memattr; + mmu_hw_do_operation_locked(pfdev, as_nr, 0, ~0UL, AS_COMMAND_FLUSH_MEM); + mmu_write(pfdev, AS_TRANSTAB_LO(as_nr), transtab & 0xffffffffUL); mmu_write(pfdev, AS_TRANSTAB_HI(as_nr), transtab >> 32); -- cgit v1.2.3-59-g8ed1b From 62f1089f3cbe7d99ced92bf96a8158813b75e5e8 Mon Sep 17 00:00:00 2001 From: Rob Herring Date: Mon, 26 Aug 2019 17:33:16 -0500 Subject: drm/panfrost: Flush and disable address space when freeing page tables Currently, page tables are freed without disabling the address space first. This probably is fine as we'll switch to new page tables when the address space is allocated again and runtime PM suspend will reset the GPU clearing the registers. However, it's better to clean up after ourselves. There is also a problem that we could be accessing the h/w in tlb_inv_context() when suspended. Rework the disable code to make sure we flush caches/TLBs and disable the address space before freeing the page tables if we are not suspended. As the tlb_inv_context() hook is only called when freeing the page tables and we do a flush before disabling the AS, lets remove the flush from tlb_inv_context and avoid any runtime PM issues. Fixes: 7282f7645d06 ("drm/panfrost: Implement per FD address spaces") Cc: Tomeu Vizoso Cc: David Airlie Cc: Daniel Vetter Signed-off-by: Rob Herring Acked-by: Alyssa Rosenzweig Reviewed-by: Robin Murphy Reviewed-by: Steven Price Link: https://patchwork.freedesktop.org/patch/msgid/20190826223317.28509-8-robh@kernel.org --- drivers/gpu/drm/panfrost/panfrost_mmu.c | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) (limited to 'drivers/gpu/drm/panfrost/panfrost_mmu.c') diff --git a/drivers/gpu/drm/panfrost/panfrost_mmu.c b/drivers/gpu/drm/panfrost/panfrost_mmu.c index d1ebde3327fe..387d830cb7cf 100644 --- a/drivers/gpu/drm/panfrost/panfrost_mmu.c +++ b/drivers/gpu/drm/panfrost/panfrost_mmu.c @@ -129,8 +129,10 @@ static void panfrost_mmu_enable(struct panfrost_device *pfdev, struct panfrost_m write_cmd(pfdev, as_nr, AS_COMMAND_UPDATE); } -static void mmu_disable(struct panfrost_device *pfdev, u32 as_nr) +static void panfrost_mmu_disable(struct panfrost_device *pfdev, u32 as_nr) { + mmu_hw_do_operation_locked(pfdev, as_nr, 0, ~0UL, AS_COMMAND_FLUSH_MEM); + mmu_write(pfdev, AS_TRANSTAB_LO(as_nr), 0); mmu_write(pfdev, AS_TRANSTAB_HI(as_nr), 0); @@ -321,11 +323,7 @@ void panfrost_mmu_unmap(struct panfrost_gem_object *bo) } static void mmu_tlb_inv_context_s1(void *cookie) -{ - struct panfrost_file_priv *priv = cookie; - - mmu_hw_do_operation(priv->pfdev, &priv->mmu, 0, ~0UL, AS_COMMAND_FLUSH_MEM); -} +{} static void mmu_tlb_inv_range_nosync(unsigned long iova, size_t size, size_t granule, bool leaf, void *cookie) @@ -374,6 +372,11 @@ void panfrost_mmu_pgtable_free(struct panfrost_file_priv *priv) spin_lock(&pfdev->as_lock); if (mmu->as >= 0) { + pm_runtime_get_noresume(pfdev->dev); + if (pm_runtime_active(pfdev->dev)) + panfrost_mmu_disable(pfdev, mmu->as); + pm_runtime_put_autosuspend(pfdev->dev); + clear_bit(mmu->as, &pfdev->as_alloc_mask); clear_bit(mmu->as, &pfdev->as_in_use_mask); list_del(&mmu->list); @@ -618,5 +621,4 @@ int panfrost_mmu_init(struct panfrost_device *pfdev) void panfrost_mmu_fini(struct panfrost_device *pfdev) { mmu_write(pfdev, MMU_INT_MASK, 0); - mmu_disable(pfdev, 0); } -- cgit v1.2.3-59-g8ed1b