diff options
Diffstat (limited to 'drivers/gpu/drm/nouveau/nouveau_bo.c')
| -rw-r--r-- | drivers/gpu/drm/nouveau/nouveau_bo.c | 196 |
1 files changed, 169 insertions, 27 deletions
diff --git a/drivers/gpu/drm/nouveau/nouveau_bo.c b/drivers/gpu/drm/nouveau/nouveau_bo.c index 3d474ac03f88..21ec561edc99 100644 --- a/drivers/gpu/drm/nouveau/nouveau_bo.c +++ b/drivers/gpu/drm/nouveau/nouveau_bo.c @@ -214,6 +214,9 @@ nouveau_bo_new(struct drm_device *dev, int size, int align, nvbo->tile_flags = tile_flags; nvbo->bo.bdev = &drm->ttm.bdev; + if (!nv_device_is_cpu_coherent(nvkm_device(&drm->device))) + nvbo->force_coherent = flags & TTM_PL_FLAG_UNCACHED; + nvbo->page_shift = 12; if (drm->client.vm) { if (!(flags & TTM_PL_FLAG_TT) && size > 256 * 1024) @@ -291,8 +294,9 @@ void nouveau_bo_placement_set(struct nouveau_bo *nvbo, uint32_t type, uint32_t busy) { struct ttm_placement *pl = &nvbo->placement; - uint32_t flags = TTM_PL_MASK_CACHING | - (nvbo->pin_refcnt ? TTM_PL_FLAG_NO_EVICT : 0); + uint32_t flags = (nvbo->force_coherent ? TTM_PL_FLAG_UNCACHED : + TTM_PL_MASK_CACHING) | + (nvbo->pin_refcnt ? TTM_PL_FLAG_NO_EVICT : 0); pl->placement = nvbo->placements; set_placement_list(nvbo->placements, &pl->num_placement, @@ -306,42 +310,75 @@ nouveau_bo_placement_set(struct nouveau_bo *nvbo, uint32_t type, uint32_t busy) } int -nouveau_bo_pin(struct nouveau_bo *nvbo, uint32_t memtype) +nouveau_bo_pin(struct nouveau_bo *nvbo, uint32_t memtype, bool contig) { struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev); struct ttm_buffer_object *bo = &nvbo->bo; + bool force = false, evict = false; int ret; ret = ttm_bo_reserve(bo, false, false, false, NULL); if (ret) - goto out; + return ret; - if (nvbo->pin_refcnt && !(memtype & (1 << bo->mem.mem_type))) { - NV_ERROR(drm, "bo %p pinned elsewhere: 0x%08x vs 0x%08x\n", bo, - 1 << bo->mem.mem_type, memtype); - ret = -EINVAL; - goto out; + if (drm->device.info.family >= NV_DEVICE_INFO_V0_TESLA && + memtype == TTM_PL_FLAG_VRAM && contig) { + if (nvbo->tile_flags & NOUVEAU_GEM_TILE_NONCONTIG) { + if (bo->mem.mem_type == TTM_PL_VRAM) { + struct nouveau_mem *mem = bo->mem.mm_node; + if (!list_is_singular(&mem->regions)) + evict = true; + } + nvbo->tile_flags &= ~NOUVEAU_GEM_TILE_NONCONTIG; + force = true; + } } - if (nvbo->pin_refcnt++) + if (nvbo->pin_refcnt) { + if (!(memtype & (1 << bo->mem.mem_type)) || evict) { + NV_ERROR(drm, "bo %p pinned elsewhere: " + "0x%08x vs 0x%08x\n", bo, + 1 << bo->mem.mem_type, memtype); + ret = -EBUSY; + } + nvbo->pin_refcnt++; goto out; + } + if (evict) { + nouveau_bo_placement_set(nvbo, TTM_PL_FLAG_TT, 0); + ret = nouveau_bo_validate(nvbo, false, false); + if (ret) + goto out; + } + + nvbo->pin_refcnt++; nouveau_bo_placement_set(nvbo, memtype, 0); + /* drop pin_refcnt temporarily, so we don't trip the assertion + * in nouveau_bo_move() that makes sure we're not trying to + * move a pinned buffer + */ + nvbo->pin_refcnt--; ret = nouveau_bo_validate(nvbo, false, false); - if (ret == 0) { - switch (bo->mem.mem_type) { - case TTM_PL_VRAM: - drm->gem.vram_available -= bo->mem.size; - break; - case TTM_PL_TT: - drm->gem.gart_available -= bo->mem.size; - break; - default: - break; - } + if (ret) + goto out; + nvbo->pin_refcnt++; + + switch (bo->mem.mem_type) { + case TTM_PL_VRAM: + drm->gem.vram_available -= bo->mem.size; + break; + case TTM_PL_TT: + drm->gem.gart_available -= bo->mem.size; + break; + default: + break; } + out: + if (force && ret) + nvbo->tile_flags |= NOUVEAU_GEM_TILE_NONCONTIG; ttm_bo_unreserve(bo); return ret; } @@ -392,7 +429,14 @@ nouveau_bo_map(struct nouveau_bo *nvbo) if (ret) return ret; - ret = ttm_bo_kmap(&nvbo->bo, 0, nvbo->bo.mem.num_pages, &nvbo->kmap); + /* + * TTM buffers allocated using the DMA API already have a mapping, let's + * use it instead. + */ + if (!nvbo->force_coherent) + ret = ttm_bo_kmap(&nvbo->bo, 0, nvbo->bo.mem.num_pages, + &nvbo->kmap); + ttm_bo_unreserve(&nvbo->bo); return ret; } @@ -400,10 +444,57 @@ nouveau_bo_map(struct nouveau_bo *nvbo) void nouveau_bo_unmap(struct nouveau_bo *nvbo) { - if (nvbo) + if (!nvbo) + return; + + /* + * TTM buffers allocated using the DMA API already had a coherent + * mapping which we used, no need to unmap. + */ + if (!nvbo->force_coherent) ttm_bo_kunmap(&nvbo->kmap); } +void +nouveau_bo_sync_for_device(struct nouveau_bo *nvbo) +{ + struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev); + struct nouveau_device *device = nvkm_device(&drm->device); + struct ttm_dma_tt *ttm_dma = (struct ttm_dma_tt *)nvbo->bo.ttm; + int i; + + if (!ttm_dma) + return; + + /* Don't waste time looping if the object is coherent */ + if (nvbo->force_coherent) + return; + + for (i = 0; i < ttm_dma->ttm.num_pages; i++) + dma_sync_single_for_device(nv_device_base(device), + ttm_dma->dma_address[i], PAGE_SIZE, DMA_TO_DEVICE); +} + +void +nouveau_bo_sync_for_cpu(struct nouveau_bo *nvbo) +{ + struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev); + struct nouveau_device *device = nvkm_device(&drm->device); + struct ttm_dma_tt *ttm_dma = (struct ttm_dma_tt *)nvbo->bo.ttm; + int i; + + if (!ttm_dma) + return; + + /* Don't waste time looping if the object is coherent */ + if (nvbo->force_coherent) + return; + + for (i = 0; i < ttm_dma->ttm.num_pages; i++) + dma_sync_single_for_cpu(nv_device_base(device), + ttm_dma->dma_address[i], PAGE_SIZE, DMA_FROM_DEVICE); +} + int nouveau_bo_validate(struct nouveau_bo *nvbo, bool interruptible, bool no_wait_gpu) @@ -415,15 +506,41 @@ nouveau_bo_validate(struct nouveau_bo *nvbo, bool interruptible, if (ret) return ret; + nouveau_bo_sync_for_device(nvbo); + return 0; } +static inline void * +_nouveau_bo_mem_index(struct nouveau_bo *nvbo, unsigned index, void *mem, u8 sz) +{ + struct ttm_dma_tt *dma_tt; + u8 *m = mem; + + index *= sz; + + if (m) { + /* kmap'd address, return the corresponding offset */ + m += index; + } else { + /* DMA-API mapping, lookup the right address */ + dma_tt = (struct ttm_dma_tt *)nvbo->bo.ttm; + m = dma_tt->cpu_address[index / PAGE_SIZE]; + m += index % PAGE_SIZE; + } + + return m; +} +#define nouveau_bo_mem_index(o, i, m) _nouveau_bo_mem_index(o, i, m, sizeof(*m)) + u16 nouveau_bo_rd16(struct nouveau_bo *nvbo, unsigned index) { bool is_iomem; u16 *mem = ttm_kmap_obj_virtual(&nvbo->kmap, &is_iomem); - mem = &mem[index]; + + mem = nouveau_bo_mem_index(nvbo, index, mem); + if (is_iomem) return ioread16_native((void __force __iomem *)mem); else @@ -435,7 +552,9 @@ nouveau_bo_wr16(struct nouveau_bo *nvbo, unsigned index, u16 val) { bool is_iomem; u16 *mem = ttm_kmap_obj_virtual(&nvbo->kmap, &is_iomem); - mem = &mem[index]; + + mem = nouveau_bo_mem_index(nvbo, index, mem); + if (is_iomem) iowrite16_native(val, (void __force __iomem *)mem); else @@ -447,7 +566,9 @@ nouveau_bo_rd32(struct nouveau_bo *nvbo, unsigned index) { bool is_iomem; u32 *mem = ttm_kmap_obj_virtual(&nvbo->kmap, &is_iomem); - mem = &mem[index]; + + mem = nouveau_bo_mem_index(nvbo, index, mem); + if (is_iomem) return ioread32_native((void __force __iomem *)mem); else @@ -459,7 +580,9 @@ nouveau_bo_wr32(struct nouveau_bo *nvbo, unsigned index, u32 val) { bool is_iomem; u32 *mem = ttm_kmap_obj_virtual(&nvbo->kmap, &is_iomem); - mem = &mem[index]; + + mem = nouveau_bo_mem_index(nvbo, index, mem); + if (is_iomem) iowrite32_native(val, (void __force __iomem *)mem); else @@ -1184,6 +1307,9 @@ nouveau_bo_move(struct ttm_buffer_object *bo, bool evict, bool intr, struct nouveau_drm_tile *new_tile = NULL; int ret = 0; + if (nvbo->pin_refcnt) + NV_WARN(drm, "Moving pinned object %p!\n", nvbo); + if (drm->device.info.family < NV_DEVICE_INFO_V0_TESLA) { ret = nouveau_bo_vm_bind(bo, new_mem, &new_tile); if (ret) @@ -1376,6 +1502,14 @@ nouveau_ttm_tt_populate(struct ttm_tt *ttm) dev = drm->dev; pdev = nv_device_base(device); + /* + * Objects matching this condition have been marked as force_coherent, + * so use the DMA API for them. + */ + if (!nv_device_is_cpu_coherent(device) && + ttm->caching_state == tt_uncached) + return ttm_dma_populate(ttm_dma, dev->dev); + #if __OS_HAS_AGP if (drm->agp.stat == ENABLED) { return ttm_agp_tt_populate(ttm); @@ -1433,6 +1567,14 @@ nouveau_ttm_tt_unpopulate(struct ttm_tt *ttm) dev = drm->dev; pdev = nv_device_base(device); + /* + * Objects matching this condition have been marked as force_coherent, + * so use the DMA API for them. + */ + if (!nv_device_is_cpu_coherent(device) && + ttm->caching_state == tt_uncached) + ttm_dma_unpopulate(ttm_dma, dev->dev); + #if __OS_HAS_AGP if (drm->agp.stat == ENABLED) { ttm_agp_tt_unpopulate(ttm); |