diff options
Diffstat (limited to 'drivers/gpu/drm/nouveau/nvkm/subdev')
59 files changed, 1891 insertions, 270 deletions
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/Kbuild b/drivers/gpu/drm/nouveau/nvkm/subdev/Kbuild index cfdffef1afb9..a339fe03d423 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/Kbuild +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/Kbuild @@ -7,6 +7,7 @@ include $(src)/nvkm/subdev/fault/Kbuild include $(src)/nvkm/subdev/fb/Kbuild include $(src)/nvkm/subdev/fuse/Kbuild include $(src)/nvkm/subdev/gpio/Kbuild +include $(src)/nvkm/subdev/gsp/Kbuild include $(src)/nvkm/subdev/i2c/Kbuild include $(src)/nvkm/subdev/ibus/Kbuild include $(src)/nvkm/subdev/iccsense/Kbuild diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/Kbuild b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/Kbuild index e5830453813d..dc300600c019 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/Kbuild +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/Kbuild @@ -5,3 +5,4 @@ nvkm-y += nvkm/subdev/bar/gf100.o nvkm-y += nvkm/subdev/bar/gk20a.o nvkm-y += nvkm/subdev/bar/gm107.o nvkm-y += nvkm/subdev/bar/gm20b.o +nvkm-y += nvkm/subdev/bar/tu102.o diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/base.c b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/base.c index 243f0a5c8a62..209a6a40834a 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/base.c @@ -36,6 +36,16 @@ nvkm_bar_bar1_vmm(struct nvkm_device *device) return device->bar->func->bar1.vmm(device->bar); } +void +nvkm_bar_bar1_reset(struct nvkm_device *device) +{ + struct nvkm_bar *bar = device->bar; + if (bar) { + bar->func->bar1.init(bar); + bar->func->bar1.wait(bar); + } +} + struct nvkm_vmm * nvkm_bar_bar2_vmm(struct nvkm_device *device) { @@ -49,6 +59,16 @@ nvkm_bar_bar2_vmm(struct nvkm_device *device) } void +nvkm_bar_bar2_reset(struct nvkm_device *device) +{ + struct nvkm_bar *bar = device->bar; + if (bar && bar->bar2) { + bar->func->bar2.init(bar); + bar->func->bar2.wait(bar); + } +} + +void nvkm_bar_bar2_fini(struct nvkm_device *device) { struct nvkm_bar *bar = device->bar; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/tu102.c b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/tu102.c new file mode 100644 index 000000000000..798f65ec3a86 --- /dev/null +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/tu102.c @@ -0,0 +1,98 @@ +/* + * Copyright 2018 Red Hat Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ +#include "gf100.h" + +#include <core/memory.h> +#include <subdev/timer.h> + +static void +tu102_bar_bar2_wait(struct nvkm_bar *bar) +{ + struct nvkm_device *device = bar->subdev.device; + nvkm_msec(device, 2000, + if (!(nvkm_rd32(device, 0xb80f50) & 0x0000000c)) + break; + ); +} + +static void +tu102_bar_bar2_fini(struct nvkm_bar *bar) +{ + nvkm_mask(bar->subdev.device, 0xb80f48, 0x80000000, 0x00000000); +} + +static void +tu102_bar_bar2_init(struct nvkm_bar *base) +{ + struct nvkm_device *device = base->subdev.device; + struct gf100_bar *bar = gf100_bar(base); + u32 addr = nvkm_memory_addr(bar->bar[0].inst) >> 12; + if (bar->bar2_halve) + addr |= 0x40000000; + nvkm_wr32(device, 0xb80f48, 0x80000000 | addr); +} + +static void +tu102_bar_bar1_wait(struct nvkm_bar *bar) +{ + struct nvkm_device *device = bar->subdev.device; + nvkm_msec(device, 2000, + if (!(nvkm_rd32(device, 0xb80f50) & 0x00000003)) + break; + ); +} + +static void +tu102_bar_bar1_fini(struct nvkm_bar *bar) +{ + nvkm_mask(bar->subdev.device, 0xb80f40, 0x80000000, 0x00000000); +} + +static void +tu102_bar_bar1_init(struct nvkm_bar *base) +{ + struct nvkm_device *device = base->subdev.device; + struct gf100_bar *bar = gf100_bar(base); + const u32 addr = nvkm_memory_addr(bar->bar[1].inst) >> 12; + nvkm_wr32(device, 0xb80f40, 0x80000000 | addr); +} + +static const struct nvkm_bar_func +tu102_bar = { + .dtor = gf100_bar_dtor, + .oneinit = gf100_bar_oneinit, + .bar1.init = tu102_bar_bar1_init, + .bar1.fini = tu102_bar_bar1_fini, + .bar1.wait = tu102_bar_bar1_wait, + .bar1.vmm = gf100_bar_bar1_vmm, + .bar2.init = tu102_bar_bar2_init, + .bar2.fini = tu102_bar_bar2_fini, + .bar2.wait = tu102_bar_bar2_wait, + .bar2.vmm = gf100_bar_bar2_vmm, + .flush = g84_bar_flush, +}; + +int +tu102_bar_new(struct nvkm_device *device, int index, struct nvkm_bar **pbar) +{ + return gf100_bar_new_(&tu102_bar, device, index, pbar); +} diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/bios/dp.c b/drivers/gpu/drm/nouveau/nvkm/subdev/bios/dp.c index 3133b28f849c..b099d1209be8 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/bios/dp.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/bios/dp.c @@ -212,7 +212,7 @@ nvbios_dpcfg_match(struct nvkm_bios *bios, u16 outp, u8 pc, u8 vs, u8 pe, u16 data; if (*ver >= 0x30) { - const u8 vsoff[] = { 0, 4, 7, 9 }; + static const u8 vsoff[] = { 0, 4, 7, 9 }; idx = (pc * 10) + vsoff[vs] + pe; if (*ver >= 0x40 && *ver <= 0x41 && *hdr >= 0x12) idx += nvbios_rd08(bios, outp + 0x11) * 40; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/bios/init.c b/drivers/gpu/drm/nouveau/nvkm/subdev/bios/init.c index 9cc10e438b3d..ec0e9f7224b5 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/bios/init.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/bios/init.c @@ -806,12 +806,12 @@ init_generic_condition(struct nvbios_init *init) init->offset += 3; switch (cond) { - case 0: + case 0: /* CONDITION_ID_INT_DP. */ if (init_conn(init) != DCB_CONNECTOR_eDP) init_exec_set(init, false); break; - case 1: - case 2: + case 1: /* CONDITION_ID_USE_SPPLL0. */ + case 2: /* CONDITION_ID_USE_SPPLL1. */ if ( init->outp && (data = nvbios_dpout_match(bios, DCB_OUTPUT_DP, (init->outp->or << 0) | @@ -826,10 +826,13 @@ init_generic_condition(struct nvbios_init *init) if (init_exec(init)) warn("script needs dp output table data\n"); break; - case 5: + case 5: /* CONDITION_ID_ASSR_SUPPORT. */ if (!(init_rdauxr(init, 0x0d) & 1)) init_exec_set(init, false); break; + case 7: /* CONDITION_ID_NO_PANEL_SEQ_DELAYS. */ + init_exec_set(init, false); + break; default: warn("INIT_GENERIC_CONDITON: unknown 0x%02x\n", cond); init->offset += size; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/devinit/Kbuild b/drivers/gpu/drm/nouveau/nvkm/subdev/devinit/Kbuild index 50a436926484..f3c388932b6f 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/devinit/Kbuild +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/devinit/Kbuild @@ -13,3 +13,4 @@ nvkm-y += nvkm/subdev/devinit/gf100.o nvkm-y += nvkm/subdev/devinit/gm107.o nvkm-y += nvkm/subdev/devinit/gm200.o nvkm-y += nvkm/subdev/devinit/gv100.o +nvkm-y += nvkm/subdev/devinit/tu102.o diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/devinit/gm200.c b/drivers/gpu/drm/nouveau/nvkm/subdev/devinit/gm200.c index 17235e940ca9..59940dacc2ba 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/devinit/gm200.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/devinit/gm200.c @@ -105,6 +105,15 @@ pmu_load(struct nv50_devinit *init, u8 type, bool post, return pmu_exec(init, pmu.init_addr_pmu), 0; } +void +gm200_devinit_preos(struct nv50_devinit *init, bool post) +{ + /* Optional: Execute PRE_OS application on PMU, which should at + * least take care of fans until a full PMU has been loaded. + */ + pmu_load(init, 0x01, post, NULL, NULL); +} + int gm200_devinit_post(struct nvkm_devinit *base, bool post) { @@ -156,10 +165,7 @@ gm200_devinit_post(struct nvkm_devinit *base, bool post) return -ETIMEDOUT; } - /* Optional: Execute PRE_OS application on PMU, which should at - * least take care of fans until a full PMU has been loaded. - */ - pmu_load(init, 0x01, post, NULL, NULL); + gm200_devinit_preos(init, post); return 0; } diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/devinit/nv50.h b/drivers/gpu/drm/nouveau/nvkm/subdev/devinit/nv50.h index 9b9f0dc1e192..72d130bb7f7c 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/devinit/nv50.h +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/devinit/nv50.h @@ -26,4 +26,5 @@ void gf100_devinit_preinit(struct nvkm_devinit *); u64 gm107_devinit_disable(struct nvkm_devinit *); int gm200_devinit_post(struct nvkm_devinit *, bool); +void gm200_devinit_preos(struct nv50_devinit *, bool); #endif diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/devinit/tu102.c b/drivers/gpu/drm/nouveau/nvkm/subdev/devinit/tu102.c new file mode 100644 index 000000000000..397670e72fff --- /dev/null +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/devinit/tu102.c @@ -0,0 +1,89 @@ +/* + * Copyright 2018 Red Hat Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ +#include "nv50.h" + +#include <subdev/bios.h> +#include <subdev/bios/pll.h> +#include <subdev/clk/pll.h> + +static int +tu102_devinit_pll_set(struct nvkm_devinit *init, u32 type, u32 freq) +{ + struct nvkm_subdev *subdev = &init->subdev; + struct nvkm_device *device = subdev->device; + struct nvbios_pll info; + int head = type - PLL_VPLL0; + int N, fN, M, P; + int ret; + + ret = nvbios_pll_parse(device->bios, type, &info); + if (ret) + return ret; + + ret = gt215_pll_calc(subdev, &info, freq, &N, &fN, &M, &P); + if (ret < 0) + return ret; + + switch (info.type) { + case PLL_VPLL0: + case PLL_VPLL1: + case PLL_VPLL2: + case PLL_VPLL3: + nvkm_wr32(device, 0x00ef10 + (head * 0x40), fN << 16); + nvkm_wr32(device, 0x00ef04 + (head * 0x40), (P << 16) | + (N << 8) | + (M << 0)); + /*XXX*/ + nvkm_wr32(device, 0x00ef0c + (head * 0x40), 0x00000900); + nvkm_wr32(device, 0x00ef00 + (head * 0x40), 0x02000014); + break; + default: + nvkm_warn(subdev, "%08x/%dKhz unimplemented\n", type, freq); + ret = -EINVAL; + break; + } + + return ret; +} + +static int +tu102_devinit_post(struct nvkm_devinit *base, bool post) +{ + struct nv50_devinit *init = nv50_devinit(base); + gm200_devinit_preos(init, post); + return 0; +} + +static const struct nvkm_devinit_func +tu102_devinit = { + .init = nv50_devinit_init, + .post = tu102_devinit_post, + .pll_set = tu102_devinit_pll_set, + .disable = gm107_devinit_disable, +}; + +int +tu102_devinit_new(struct nvkm_device *device, int index, + struct nvkm_devinit **pinit) +{ + return nv50_devinit_new_(&tu102_devinit, device, index, pinit); +} diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fault/Kbuild b/drivers/gpu/drm/nouveau/nvkm/subdev/fault/Kbuild index 45bb46fb0929..42586267fc08 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/fault/Kbuild +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fault/Kbuild @@ -1,3 +1,5 @@ nvkm-y += nvkm/subdev/fault/base.o +nvkm-y += nvkm/subdev/fault/user.o nvkm-y += nvkm/subdev/fault/gp100.o nvkm-y += nvkm/subdev/fault/gv100.o +nvkm-y += nvkm/subdev/fault/tu102.o diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fault/base.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fault/base.c index 16ad91c91a7b..ca251560d3e0 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/fault/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fault/base.c @@ -23,21 +23,19 @@ #include <core/memory.h> #include <core/notify.h> -#include <subdev/bar.h> -#include <subdev/mmu.h> static void nvkm_fault_ntfy_fini(struct nvkm_event *event, int type, int index) { struct nvkm_fault *fault = container_of(event, typeof(*fault), event); - fault->func->buffer.fini(fault->buffer[index]); + fault->func->buffer.intr(fault->buffer[index], false); } static void nvkm_fault_ntfy_init(struct nvkm_event *event, int type, int index) { struct nvkm_fault *fault = container_of(event, typeof(*fault), event); - fault->func->buffer.init(fault->buffer[index]); + fault->func->buffer.intr(fault->buffer[index], true); } static int @@ -91,7 +89,6 @@ nvkm_fault_oneinit_buffer(struct nvkm_fault *fault, int id) { struct nvkm_subdev *subdev = &fault->subdev; struct nvkm_device *device = subdev->device; - struct nvkm_vmm *bar2 = nvkm_bar_bar2_vmm(device); struct nvkm_fault_buffer *buffer; int ret; @@ -99,7 +96,7 @@ nvkm_fault_oneinit_buffer(struct nvkm_fault *fault, int id) return -ENOMEM; buffer->fault = fault; buffer->id = id; - buffer->entries = fault->func->buffer.entries(buffer); + fault->func->buffer.info(buffer); fault->buffer[id] = buffer; nvkm_debug(subdev, "buffer %d: %d entries\n", id, buffer->entries); @@ -110,12 +107,12 @@ nvkm_fault_oneinit_buffer(struct nvkm_fault *fault, int id) if (ret) return ret; - ret = nvkm_vmm_get(bar2, 12, nvkm_memory_size(buffer->mem), - &buffer->vma); - if (ret) - return ret; + /* Pin fault buffer in BAR2. */ + buffer->addr = nvkm_memory_bar2(buffer->mem); + if (buffer->addr == ~0ULL) + return -EFAULT; - return nvkm_memory_map(buffer->mem, 0, bar2, buffer->vma, NULL, 0); + return 0; } static int @@ -146,7 +143,6 @@ nvkm_fault_oneinit(struct nvkm_subdev *subdev) static void * nvkm_fault_dtor(struct nvkm_subdev *subdev) { - struct nvkm_vmm *bar2 = nvkm_bar_bar2_vmm(subdev->device); struct nvkm_fault *fault = nvkm_fault(subdev); int i; @@ -154,7 +150,6 @@ nvkm_fault_dtor(struct nvkm_subdev *subdev) for (i = 0; i < fault->buffer_nr; i++) { if (fault->buffer[i]) { - nvkm_vmm_put(bar2, &fault->buffer[i]->vma); nvkm_memory_unref(&fault->buffer[i]->mem); kfree(fault->buffer[i]); } @@ -181,5 +176,7 @@ nvkm_fault_new_(const struct nvkm_fault_func *func, struct nvkm_device *device, return -ENOMEM; nvkm_subdev_ctor(&nvkm_fault, device, index, &fault->subdev); fault->func = func; + fault->user.ctor = nvkm_ufault_new; + fault->user.base = func->user.base; return 0; } diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fault/gp100.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fault/gp100.c index 5e71db2e8d75..4f3c4e091117 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/fault/gp100.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fault/gp100.c @@ -21,7 +21,16 @@ */ #include "priv.h" -#include <subdev/mmu.h> +#include <subdev/mc.h> + +#include <nvif/class.h> + +static void +gp100_fault_buffer_intr(struct nvkm_fault_buffer *buffer, bool enable) +{ + struct nvkm_device *device = buffer->fault->subdev.device; + nvkm_mc_intr_mask(device, NVKM_SUBDEV_FAULT, enable); +} static void gp100_fault_buffer_fini(struct nvkm_fault_buffer *buffer) @@ -34,15 +43,17 @@ static void gp100_fault_buffer_init(struct nvkm_fault_buffer *buffer) { struct nvkm_device *device = buffer->fault->subdev.device; - nvkm_wr32(device, 0x002a74, upper_32_bits(buffer->vma->addr)); - nvkm_wr32(device, 0x002a70, lower_32_bits(buffer->vma->addr)); + nvkm_wr32(device, 0x002a74, upper_32_bits(buffer->addr)); + nvkm_wr32(device, 0x002a70, lower_32_bits(buffer->addr)); nvkm_mask(device, 0x002a70, 0x00000001, 0x00000001); } -static u32 -gp100_fault_buffer_entries(struct nvkm_fault_buffer *buffer) +static void +gp100_fault_buffer_info(struct nvkm_fault_buffer *buffer) { - return nvkm_rd32(buffer->fault->subdev.device, 0x002a78); + buffer->entries = nvkm_rd32(buffer->fault->subdev.device, 0x002a78); + buffer->get = 0x002a7c; + buffer->put = 0x002a80; } static void @@ -56,9 +67,11 @@ gp100_fault = { .intr = gp100_fault_intr, .buffer.nr = 1, .buffer.entry_size = 32, - .buffer.entries = gp100_fault_buffer_entries, + .buffer.info = gp100_fault_buffer_info, .buffer.init = gp100_fault_buffer_init, .buffer.fini = gp100_fault_buffer_fini, + .buffer.intr = gp100_fault_buffer_intr, + .user = { { 0, 0, MAXWELL_FAULT_BUFFER_A }, 0 }, }; int diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fault/gv100.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fault/gv100.c index 3cd610d7deb5..6747f09c2dc3 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/fault/gv100.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fault/gv100.c @@ -25,14 +25,15 @@ #include <subdev/mmu.h> #include <engine/fifo.h> +#include <nvif/class.h> + static void gv100_fault_buffer_process(struct nvkm_fault_buffer *buffer) { struct nvkm_device *device = buffer->fault->subdev.device; struct nvkm_memory *mem = buffer->mem; - const u32 foff = buffer->id * 0x14; - u32 get = nvkm_rd32(device, 0x100e2c + foff); - u32 put = nvkm_rd32(device, 0x100e30 + foff); + u32 get = nvkm_rd32(device, buffer->get); + u32 put = nvkm_rd32(device, buffer->put); if (put == get) return; @@ -51,7 +52,7 @@ gv100_fault_buffer_process(struct nvkm_fault_buffer *buffer) if (++get == buffer->entries) get = 0; - nvkm_wr32(device, 0x100e2c + foff, get); + nvkm_wr32(device, buffer->get, get); info.addr = ((u64)addrhi << 32) | addrlo; info.inst = ((u64)insthi << 32) | instlo; @@ -70,13 +71,21 @@ gv100_fault_buffer_process(struct nvkm_fault_buffer *buffer) } static void -gv100_fault_buffer_fini(struct nvkm_fault_buffer *buffer) +gv100_fault_buffer_intr(struct nvkm_fault_buffer *buffer, bool enable) { struct nvkm_device *device = buffer->fault->subdev.device; const u32 intr = buffer->id ? 0x08000000 : 0x20000000; - const u32 foff = buffer->id * 0x14; + if (enable) + nvkm_mask(device, 0x100a2c, intr, intr); + else + nvkm_mask(device, 0x100a34, intr, intr); +} - nvkm_mask(device, 0x100a34, intr, intr); +static void +gv100_fault_buffer_fini(struct nvkm_fault_buffer *buffer) +{ + struct nvkm_device *device = buffer->fault->subdev.device; + const u32 foff = buffer->id * 0x14; nvkm_mask(device, 0x100e34 + foff, 0x80000000, 0x00000000); } @@ -84,23 +93,25 @@ static void gv100_fault_buffer_init(struct nvkm_fault_buffer *buffer) { struct nvkm_device *device = buffer->fault->subdev.device; - const u32 intr = buffer->id ? 0x08000000 : 0x20000000; const u32 foff = buffer->id * 0x14; nvkm_mask(device, 0x100e34 + foff, 0xc0000000, 0x40000000); - nvkm_wr32(device, 0x100e28 + foff, upper_32_bits(buffer->vma->addr)); - nvkm_wr32(device, 0x100e24 + foff, lower_32_bits(buffer->vma->addr)); + nvkm_wr32(device, 0x100e28 + foff, upper_32_bits(buffer->addr)); + nvkm_wr32(device, 0x100e24 + foff, lower_32_bits(buffer->addr)); nvkm_mask(device, 0x100e34 + foff, 0x80000000, 0x80000000); - nvkm_mask(device, 0x100a2c, intr, intr); } -static u32 -gv100_fault_buffer_entries(struct nvkm_fault_buffer *buffer) +static void +gv100_fault_buffer_info(struct nvkm_fault_buffer *buffer) { struct nvkm_device *device = buffer->fault->subdev.device; const u32 foff = buffer->id * 0x14; + nvkm_mask(device, 0x100e34 + foff, 0x40000000, 0x40000000); - return nvkm_rd32(device, 0x100e34 + foff) & 0x000fffff; + + buffer->entries = nvkm_rd32(device, 0x100e34 + foff) & 0x000fffff; + buffer->get = 0x100e2c + foff; + buffer->put = 0x100e30 + foff; } static int @@ -157,6 +168,13 @@ gv100_fault_intr(struct nvkm_fault *fault) } } + if (stat & 0x08000000) { + if (fault->buffer[1]) { + nvkm_event_send(&fault->event, 1, 1, NULL, 0); + stat &= ~0x08000000; + } + } + if (stat) { nvkm_debug(subdev, "intr %08x\n", stat); } @@ -166,6 +184,8 @@ static void gv100_fault_fini(struct nvkm_fault *fault) { nvkm_notify_put(&fault->nrpfb); + if (fault->buffer[0]) + fault->func->buffer.fini(fault->buffer[0]); nvkm_mask(fault->subdev.device, 0x100a34, 0x80000000, 0x80000000); } @@ -173,14 +193,15 @@ static void gv100_fault_init(struct nvkm_fault *fault) { nvkm_mask(fault->subdev.device, 0x100a2c, 0x80000000, 0x80000000); + fault->func->buffer.init(fault->buffer[0]); nvkm_notify_get(&fault->nrpfb); } -static int +int gv100_fault_oneinit(struct nvkm_fault *fault) { return nvkm_notify_init(&fault->buffer[0]->object, &fault->event, - gv100_fault_ntfy_nrpfb, false, NULL, 0, 0, + gv100_fault_ntfy_nrpfb, true, NULL, 0, 0, &fault->nrpfb); } @@ -192,9 +213,17 @@ gv100_fault = { .intr = gv100_fault_intr, .buffer.nr = 2, .buffer.entry_size = 32, - .buffer.entries = gv100_fault_buffer_entries, + .buffer.info = gv100_fault_buffer_info, .buffer.init = gv100_fault_buffer_init, .buffer.fini = gv100_fault_buffer_fini, + .buffer.intr = gv100_fault_buffer_intr, + /*TODO: Figure out how to expose non-replayable fault buffer, which, + * for some reason, is where recoverable CE faults appear... + * + * It's a bit tricky, as both NVKM and SVM will need access to + * the non-replayable fault buffer. + */ + .user = { { 0, 0, VOLTA_FAULT_BUFFER_A }, 1 }, }; int diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fault/priv.h b/drivers/gpu/drm/nouveau/nvkm/subdev/fault/priv.h index e4d2f5234fd1..975e66ac6344 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/fault/priv.h +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fault/priv.h @@ -12,8 +12,10 @@ struct nvkm_fault_buffer { struct nvkm_fault *fault; int id; int entries; + u32 get; + u32 put; struct nvkm_memory *mem; - struct nvkm_vma *vma; + u64 addr; }; int nvkm_fault_new_(const struct nvkm_fault_func *, struct nvkm_device *, @@ -27,9 +29,19 @@ struct nvkm_fault_func { struct { int nr; u32 entry_size; - u32 (*entries)(struct nvkm_fault_buffer *); + void (*info)(struct nvkm_fault_buffer *); void (*init)(struct nvkm_fault_buffer *); void (*fini)(struct nvkm_fault_buffer *); + void (*intr)(struct nvkm_fault_buffer *, bool enable); } buffer; + struct { + struct nvkm_sclass base; + int rp; + } user; }; + +int gv100_fault_oneinit(struct nvkm_fault *); + +int nvkm_ufault_new(struct nvkm_device *, const struct nvkm_oclass *, + void *, u32, struct nvkm_object **); #endif diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fault/tu102.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fault/tu102.c new file mode 100644 index 000000000000..fa1dfe5692b0 --- /dev/null +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fault/tu102.c @@ -0,0 +1,168 @@ +/* + * Copyright 2018 Red Hat Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ +#include "priv.h" + +#include <core/memory.h> +#include <subdev/mmu.h> +#include <engine/fifo.h> + +#include <nvif/class.h> + +static void +tu102_fault_buffer_intr(struct nvkm_fault_buffer *buffer, bool enable) +{ + /*XXX: Earlier versions of RM touched the old regs on Turing, + * which don't appear to actually work anymore, but newer + * versions of RM don't appear to touch anything at all.. + */ +} + +static void +tu102_fault_buffer_fini(struct nvkm_fault_buffer *buffer) +{ + struct nvkm_device *device = buffer->fault->subdev.device; + const u32 foff = buffer->id * 0x20; + nvkm_mask(device, 0xb83010 + foff, 0x80000000, 0x00000000); +} + +static void +tu102_fault_buffer_init(struct nvkm_fault_buffer *buffer) +{ + struct nvkm_device *device = buffer->fault->subdev.device; + const u32 foff = buffer->id * 0x20; + + nvkm_mask(device, 0xb83010 + foff, 0xc0000000, 0x40000000); + nvkm_wr32(device, 0xb83004 + foff, upper_32_bits(buffer->addr)); + nvkm_wr32(device, 0xb83000 + foff, lower_32_bits(buffer->addr)); + nvkm_mask(device, 0xb83010 + foff, 0x80000000, 0x80000000); +} + +static void +tu102_fault_buffer_info(struct nvkm_fault_buffer *buffer) +{ + struct nvkm_device *device = buffer->fault->subdev.device; + const u32 foff = buffer->id * 0x20; + + nvkm_mask(device, 0xb83010 + foff, 0x40000000, 0x40000000); + + buffer->entries = nvkm_rd32(device, 0xb83010 + foff) & 0x000fffff; + buffer->get = 0xb83008 + foff; + buffer->put = 0xb8300c + foff; +} + +static void +tu102_fault_intr_fault(struct nvkm_fault *fault) +{ + struct nvkm_subdev *subdev = &fault->subdev; + struct nvkm_device *device = subdev->device; + struct nvkm_fault_data info; + const u32 addrlo = nvkm_rd32(device, 0xb83080); + const u32 addrhi = nvkm_rd32(device, 0xb83084); + const u32 info0 = nvkm_rd32(device, 0xb83088); + const u32 insthi = nvkm_rd32(device, 0xb8308c); + const u32 info1 = nvkm_rd32(device, 0xb83090); + + info.addr = ((u64)addrhi << 32) | addrlo; + info.inst = ((u64)insthi << 32) | (info0 & 0xfffff000); + info.time = 0; + info.engine = (info0 & 0x000000ff); + info.valid = (info1 & 0x80000000) >> 31; + info.gpc = (info1 & 0x1f000000) >> 24; + info.hub = (info1 & 0x00100000) >> 20; + info.access = (info1 & 0x000f0000) >> 16; + info.client = (info1 & 0x00007f00) >> 8; + info.reason = (info1 & 0x0000001f); + + nvkm_fifo_fault(device->fifo, &info); +} + +static void +tu102_fault_intr(struct nvkm_fault *fault) +{ + struct nvkm_subdev *subdev = &fault->subdev; + struct nvkm_device *device = subdev->device; + u32 stat = nvkm_rd32(device, 0xb83094); + + if (stat & 0x80000000) { + tu102_fault_intr_fault(fault); + nvkm_wr32(device, 0xb83094, 0x80000000); + stat &= ~0x80000000; + } + + if (stat & 0x00000200) { + if (fault->buffer[0]) { + nvkm_event_send(&fault->event, 1, 0, NULL, 0); + stat &= ~0x00000200; + } + } + + /*XXX: guess, can't confirm until we get fw... */ + if (stat & 0x00000100) { + if (fault->buffer[1]) { + nvkm_event_send(&fault->event, 1, 1, NULL, 0); + stat &= ~0x00000100; + } + } + + if (stat) { + nvkm_debug(subdev, "intr %08x\n", stat); + } +} + +static void +tu102_fault_fini(struct nvkm_fault *fault) +{ + nvkm_notify_put(&fault->nrpfb); + if (fault->buffer[0]) + fault->func->buffer.fini(fault->buffer[0]); + /*XXX: disable priv faults */ +} + +static void +tu102_fault_init(struct nvkm_fault *fault) +{ + /*XXX: enable priv faults */ + fault->func->buffer.init(fault->buffer[0]); + nvkm_notify_get(&fault->nrpfb); +} + +static const struct nvkm_fault_func +tu102_fault = { + .oneinit = gv100_fault_oneinit, + .init = tu102_fault_init, + .fini = tu102_fault_fini, + .intr = tu102_fault_intr, + .buffer.nr = 2, + .buffer.entry_size = 32, + .buffer.info = tu102_fault_buffer_info, + .buffer.init = tu102_fault_buffer_init, + .buffer.fini = tu102_fault_buffer_fini, + .buffer.intr = tu102_fault_buffer_intr, + .user = { { 0, 0, VOLTA_FAULT_BUFFER_A }, 1 }, +}; + +int +tu102_fault_new(struct nvkm_device *device, int index, + struct nvkm_fault **pfault) +{ + return nvkm_fault_new_(&tu102_fault, device, index, pfault); +} diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fault/user.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fault/user.c new file mode 100644 index 000000000000..ac835c9582fd --- /dev/null +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fault/user.c @@ -0,0 +1,106 @@ +/* + * Copyright 2018 Red Hat Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ +#include "priv.h" + +#include <core/memory.h> +#include <subdev/mmu.h> + +#include <nvif/clb069.h> +#include <nvif/unpack.h> + +static int +nvkm_ufault_map(struct nvkm_object *object, void *argv, u32 argc, + enum nvkm_object_map *type, u64 *addr, u64 *size) +{ + struct nvkm_fault_buffer *buffer = nvkm_fault_buffer(object); + struct nvkm_device *device = buffer->fault->subdev.device; + *type = NVKM_OBJECT_MAP_IO; + *addr = device->func->resource_addr(device, 3) + buffer->addr; + *size = nvkm_memory_size(buffer->mem); + return 0; +} + +static int +nvkm_ufault_ntfy(struct nvkm_object *object, u32 type, + struct nvkm_event **pevent) +{ + struct nvkm_fault_buffer *buffer = nvkm_fault_buffer(object); + if (type == NVB069_V0_NTFY_FAULT) { + *pevent = &buffer->fault->event; + return 0; + } + return -EINVAL; +} + +static int +nvkm_ufault_fini(struct nvkm_object *object, bool suspend) +{ + struct nvkm_fault_buffer *buffer = nvkm_fault_buffer(object); + buffer->fault->func->buffer.fini(buffer); + return 0; +} + +static int +nvkm_ufault_init(struct nvkm_object *object) +{ + struct nvkm_fault_buffer *buffer = nvkm_fault_buffer(object); + buffer->fault->func->buffer.init(buffer); + return 0; +} + +static void * +nvkm_ufault_dtor(struct nvkm_object *object) +{ + return NULL; +} + +static const struct nvkm_object_func +nvkm_ufault = { + .dtor = nvkm_ufault_dtor, + .init = nvkm_ufault_init, + .fini = nvkm_ufault_fini, + .ntfy = nvkm_ufault_ntfy, + .map = nvkm_ufault_map, +}; + +int +nvkm_ufault_new(struct nvkm_device *device, const struct nvkm_oclass *oclass, + void *argv, u32 argc, struct nvkm_object **pobject) +{ + union { + struct nvif_clb069_v0 v0; + } *args = argv; + struct nvkm_fault *fault = device->fault; + struct nvkm_fault_buffer *buffer = fault->buffer[fault->func->user.rp]; + int ret = -ENOSYS; + + if (!(ret = nvif_unpack(ret, &argv, &argc, args->v0, 0, 0, false))) { + args->v0.entries = buffer->entries; + args->v0.get = buffer->get; + args->v0.put = buffer->put; + } else + return ret; + + nvkm_object_ctor(&nvkm_ufault, oclass, &buffer->object); + *pobject = &buffer->object; + return 0; +} diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/base.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/base.c index 434d2fc5bb1c..b2bb5a3ccb02 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/base.c @@ -68,10 +68,13 @@ nvkm_fb_bios_memtype(struct nvkm_bios *bios) if (nvbios_M0203Em(bios, ramcfg, &ver, &hdr, &M0203E)) { switch (M0203E.type) { - case M0203E_TYPE_DDR2 : return NVKM_RAM_TYPE_DDR2; - case M0203E_TYPE_DDR3 : return NVKM_RAM_TYPE_DDR3; - case M0203E_TYPE_GDDR3: return NVKM_RAM_TYPE_GDDR3; - case M0203E_TYPE_GDDR5: return NVKM_RAM_TYPE_GDDR5; + case M0203E_TYPE_DDR2 : return NVKM_RAM_TYPE_DDR2; + case M0203E_TYPE_DDR3 : return NVKM_RAM_TYPE_DDR3; + case M0203E_TYPE_GDDR3 : return NVKM_RAM_TYPE_GDDR3; + case M0203E_TYPE_GDDR5 : return NVKM_RAM_TYPE_GDDR5; + case M0203E_TYPE_GDDR5X: return NVKM_RAM_TYPE_GDDR5X; + case M0203E_TYPE_GDDR6 : return NVKM_RAM_TYPE_GDDR6; + case M0203E_TYPE_HBM2 : return NVKM_RAM_TYPE_HBM2; default: nvkm_warn(subdev, "M0203E type %02x\n", M0203E.type); return NVKM_RAM_TYPE_UNKNOWN; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gddr3.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gddr3.c index 60ece0a8a2e1..1d2d6bae73cd 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gddr3.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gddr3.c @@ -87,7 +87,7 @@ nvkm_gddr3_calc(struct nvkm_ram *ram) WR = (ram->next->bios.timing[2] & 0x007f0000) >> 16; /* XXX: Get these values from the VBIOS instead */ DLL = !(ram->mr[1] & 0x1); - RON = !(ram->mr[1] & 0x300) >> 8; + RON = !((ram->mr[1] & 0x300) >> 8); break; default: return -ENOSYS; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ram.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ram.c index 24c7bd505731..b11867f682cb 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ram.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ram.c @@ -184,6 +184,9 @@ nvkm_ram_ctor(const struct nvkm_ram_func *func, struct nvkm_fb *fb, [NVKM_RAM_TYPE_GDDR3 ] = "GDDR3", [NVKM_RAM_TYPE_GDDR4 ] = "GDDR4", [NVKM_RAM_TYPE_GDDR5 ] = "GDDR5", + [NVKM_RAM_TYPE_GDDR5X ] = "GDDR5X", + [NVKM_RAM_TYPE_GDDR6 ] = "GDDR6", + [NVKM_RAM_TYPE_HBM2 ] = "HBM2", }; struct nvkm_subdev *subdev = &fb->subdev; int ret; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/Kbuild b/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/Kbuild new file mode 100644 index 000000000000..26fc6feb807e --- /dev/null +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/Kbuild @@ -0,0 +1 @@ +nvkm-y += nvkm/subdev/gsp/gv100.o diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/gv100.c b/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/gv100.c new file mode 100644 index 000000000000..dccfaf1162e2 --- /dev/null +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/gv100.c @@ -0,0 +1,62 @@ +/* + * Copyright 2019 Red Hat Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ +#include <subdev/gsp.h> +#include <subdev/top.h> +#include <engine/falcon.h> + +static int +gv100_gsp_oneinit(struct nvkm_subdev *subdev) +{ + struct nvkm_gsp *gsp = nvkm_gsp(subdev); + + gsp->addr = nvkm_top_addr(subdev->device, subdev->index); + if (!gsp->addr) + return -EINVAL; + + return nvkm_falcon_v1_new(subdev, "GSP", gsp->addr, &gsp->falcon); +} + +static void * +gv100_gsp_dtor(struct nvkm_subdev *subdev) +{ + struct nvkm_gsp *gsp = nvkm_gsp(subdev); + nvkm_falcon_del(&gsp->falcon); + return gsp; +} + +static const struct nvkm_subdev_func +gv100_gsp = { + .dtor = gv100_gsp_dtor, + .oneinit = gv100_gsp_oneinit, +}; + +int +gv100_gsp_new(struct nvkm_device *device, int index, struct nvkm_gsp **pgsp) +{ + struct nvkm_gsp *gsp; + + if (!(gsp = *pgsp = kzalloc(sizeof(*gsp), GFP_KERNEL))) + return -ENOMEM; + + nvkm_subdev_ctor(&gv100_gsp, device, index, &gsp->subdev); + return 0; +} diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/nv50.c b/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/nv50.c index db48a1daca0c..02c4eb28cef4 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/nv50.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/nv50.c @@ -288,6 +288,19 @@ nv50_instobj_addr(struct nvkm_memory *memory) return nvkm_memory_addr(nv50_instobj(memory)->ram); } +static u64 +nv50_instobj_bar2(struct nvkm_memory *memory) +{ + struct nv50_instobj *iobj = nv50_instobj(memory); + u64 addr = ~0ULL; + if (nv50_instobj_acquire(&iobj->base.memory)) { + iobj->lru.next = NULL; /* Exclude from eviction. */ + addr = iobj->bar->addr; + } + nv50_instobj_release(&iobj->base.memory); + return addr; +} + static enum nvkm_memory_target nv50_instobj_target(struct nvkm_memory *memory) { @@ -325,8 +338,9 @@ static const struct nvkm_memory_func nv50_instobj_func = { .dtor = nv50_instobj_dtor, .target = nv50_instobj_target, - .size = nv50_instobj_size, + .bar2 = nv50_instobj_bar2, .addr = nv50_instobj_addr, + .size = nv50_instobj_size, .boot = nv50_instobj_boot, .acquire = nv50_instobj_acquire, .release = nv50_instobj_release, diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mc/Kbuild b/drivers/gpu/drm/nouveau/nvkm/subdev/mc/Kbuild index 2befbe36dc28..c64e399326b3 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mc/Kbuild +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mc/Kbuild @@ -12,3 +12,4 @@ nvkm-y += nvkm/subdev/mc/gk104.o nvkm-y += nvkm/subdev/mc/gk20a.o nvkm-y += nvkm/subdev/mc/gp100.o nvkm-y += nvkm/subdev/mc/gp10b.o +nvkm-y += nvkm/subdev/mc/tu102.o diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mc/base.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mc/base.c index 09f669ac6630..0e57ab2a709f 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mc/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mc/base.c @@ -108,6 +108,9 @@ nvkm_mc_intr(struct nvkm_device *device, bool *handled) if (stat) nvkm_error(&mc->subdev, "intr %08x\n", stat); *handled = intr != 0; + + if (mc->func->intr_hack) + mc->func->intr_hack(mc, handled); } static u32 diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mc/priv.h b/drivers/gpu/drm/nouveau/nvkm/subdev/mc/priv.h index d9e3691d45b7..eb91a4cf452b 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mc/priv.h +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mc/priv.h @@ -26,6 +26,7 @@ struct nvkm_mc_func { void (*intr_mask)(struct nvkm_mc *, u32 mask, u32 stat); /* retrieve pending interrupt mask (NV_PMC_INTR) */ u32 (*intr_stat)(struct nvkm_mc *); + void (*intr_hack)(struct nvkm_mc *, bool *handled); const struct nvkm_mc_map *reset; void (*unk260)(struct nvkm_mc *, u32); }; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mc/tu102.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mc/tu102.c new file mode 100644 index 000000000000..d098c44a4fcb --- /dev/null +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mc/tu102.c @@ -0,0 +1,55 @@ +/* + * Copyright 2018 Red Hat Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ +#include "priv.h" + +static void +tu102_mc_intr_hack(struct nvkm_mc *mc, bool *handled) +{ + struct nvkm_device *device = mc->subdev.device; + u32 stat = nvkm_rd32(device, 0xb81010); + if (stat & 0x00000050) { + struct nvkm_subdev *subdev = + nvkm_device_subdev(device, NVKM_SUBDEV_FAULT); + nvkm_wr32(device, 0xb81010, stat & 0x00000050); + if (subdev) + nvkm_subdev_intr(subdev); + *handled = true; + } +} + +static const struct nvkm_mc_func +tu102_mc = { + .init = nv50_mc_init, + .intr = gp100_mc_intr, + .intr_unarm = gp100_mc_intr_unarm, + .intr_rearm = gp100_mc_intr_rearm, + .intr_mask = gp100_mc_intr_mask, + .intr_stat = gf100_mc_intr_stat, + .intr_hack = tu102_mc_intr_hack, + .reset = gk104_mc_reset, +}; + +int +tu102_mc_new(struct nvkm_device *device, int index, struct nvkm_mc **pmc) +{ + return gp100_mc_new_(&tu102_mc, device, index, pmc); +} diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/Kbuild b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/Kbuild index 58a24e3a0598..db9c56028f21 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/Kbuild +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/Kbuild @@ -13,6 +13,7 @@ nvkm-y += nvkm/subdev/mmu/gm20b.o nvkm-y += nvkm/subdev/mmu/gp100.o nvkm-y += nvkm/subdev/mmu/gp10b.o nvkm-y += nvkm/subdev/mmu/gv100.o +nvkm-y += nvkm/subdev/mmu/tu102.o nvkm-y += nvkm/subdev/mmu/mem.o nvkm-y += nvkm/subdev/mmu/memnv04.o @@ -33,6 +34,7 @@ nvkm-y += nvkm/subdev/mmu/vmmgm20b.o nvkm-y += nvkm/subdev/mmu/vmmgp100.o nvkm-y += nvkm/subdev/mmu/vmmgp10b.o nvkm-y += nvkm/subdev/mmu/vmmgv100.o +nvkm-y += nvkm/subdev/mmu/vmmtu102.o nvkm-y += nvkm/subdev/mmu/umem.o nvkm-y += nvkm/subdev/mmu/ummu.o diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gp100.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gp100.c index 651b8805c67c..65cb9d28e60e 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gp100.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gp100.c @@ -31,7 +31,7 @@ gp100_mmu = { .dma_bits = 47, .mmu = {{ -1, -1, NVIF_CLASS_MMU_GF100}}, .mem = {{ -1, 0, NVIF_CLASS_MEM_GF100}, gf100_mem_new, gf100_mem_map }, - .vmm = {{ -1, -1, NVIF_CLASS_VMM_GP100}, gp100_vmm_new }, + .vmm = {{ -1, 0, NVIF_CLASS_VMM_GP100}, gp100_vmm_new }, .kind = gm200_mmu_kind, .kind_sys = true, }; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gp10b.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gp10b.c index 3bd3db31e0bb..0a50be9a785a 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gp10b.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gp10b.c @@ -31,7 +31,7 @@ gp10b_mmu = { .dma_bits = 47, .mmu = {{ -1, -1, NVIF_CLASS_MMU_GF100}}, .mem = {{ -1, -1, NVIF_CLASS_MEM_GF100}, .umap = gf100_mem_map }, - .vmm = {{ -1, -1, NVIF_CLASS_VMM_GP100}, gp10b_vmm_new }, + .vmm = {{ -1, 0, NVIF_CLASS_VMM_GP100}, gp10b_vmm_new }, .kind = gm200_mmu_kind, .kind_sys = true, }; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gv100.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gv100.c index f666cb57f69e..e0997eedd6d9 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gv100.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gv100.c @@ -31,7 +31,7 @@ gv100_mmu = { .dma_bits = 47, .mmu = {{ -1, -1, NVIF_CLASS_MMU_GF100}}, .mem = {{ -1, 0, NVIF_CLASS_MEM_GF100}, gf100_mem_new, gf100_mem_map }, - .vmm = {{ -1, -1, NVIF_CLASS_VMM_GP100}, gv100_vmm_new }, + .vmm = {{ -1, 0, NVIF_CLASS_VMM_GP100}, gv100_vmm_new }, .kind = gm200_mmu_kind, .kind_sys = true, }; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/priv.h b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/priv.h index 948a48c21be4..2ad1102a4e31 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/priv.h +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/priv.h @@ -28,7 +28,7 @@ struct nvkm_mmu_func { struct { struct nvkm_sclass user; - int (*ctor)(struct nvkm_mmu *, u64 addr, u64 size, + int (*ctor)(struct nvkm_mmu *, bool managed, u64 addr, u64 size, void *argv, u32 argc, struct lock_class_key *, const char *name, struct nvkm_vmm **); bool global; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/tu102.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/tu102.c new file mode 100644 index 000000000000..c0db0ce10cba --- /dev/null +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/tu102.c @@ -0,0 +1,43 @@ +/* + * Copyright 2018 Red Hat Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ +#include "mem.h" +#include "vmm.h" + +#include <core/option.h> + +#include <nvif/class.h> + +static const struct nvkm_mmu_func +tu102_mmu = { + .dma_bits = 47, + .mmu = {{ -1, -1, NVIF_CLASS_MMU_GF100}}, + .mem = {{ -1, 0, NVIF_CLASS_MEM_GF100}, gf100_mem_new, gf100_mem_map }, + .vmm = {{ -1, 0, NVIF_CLASS_VMM_GP100}, tu102_vmm_new }, + .kind = gm200_mmu_kind, + .kind_sys = true, +}; + +int +tu102_mmu_new(struct nvkm_device *device, int index, struct nvkm_mmu **pmmu) +{ + return nvkm_mmu_new_(&tu102_mmu, device, index, pmmu); +} diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/uvmm.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/uvmm.c index 37b201b95f15..c43b8248c682 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/uvmm.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/uvmm.c @@ -43,6 +43,69 @@ nvkm_uvmm_search(struct nvkm_client *client, u64 handle) } static int +nvkm_uvmm_mthd_pfnclr(struct nvkm_uvmm *uvmm, void *argv, u32 argc) +{ + struct nvkm_client *client = uvmm->object.client; + union { + struct nvif_vmm_pfnclr_v0 v0; + } *args = argv; + struct nvkm_vmm *vmm = uvmm->vmm; + int ret = -ENOSYS; + u64 addr, size; + + if (!(ret = nvif_unpack(ret, &argv, &argc, args->v0, 0, 0, false))) { + addr = args->v0.addr; + size = args->v0.size; + } else + return ret; + + if (!client->super) + return -ENOENT; + + if (size) { + mutex_lock(&vmm->mutex); + ret = nvkm_vmm_pfn_unmap(vmm, addr, size); + mutex_unlock(&vmm->mutex); + } + + return ret; +} + +static int +nvkm_uvmm_mthd_pfnmap(struct nvkm_uvmm *uvmm, void *argv, u32 argc) +{ + struct nvkm_client *client = uvmm->object.client; + union { + struct nvif_vmm_pfnmap_v0 v0; + } *args = argv; + struct nvkm_vmm *vmm = uvmm->vmm; + int ret = -ENOSYS; + u64 addr, size, *phys; + u8 page; + + if (!(ret = nvif_unpack(ret, &argv, &argc, args->v0, 0, 0, true))) { + page = args->v0.page; + addr = args->v0.addr; + size = args->v0.size; + phys = args->v0.phys; + if (argc != (size >> page) * sizeof(args->v0.phys[0])) + return -EINVAL; + } else + return ret; + + if (!client->super) + return -ENOENT; + + if (size) { + mutex_lock(&vmm->mutex); + ret = nvkm_vmm_pfn_map(vmm, page, addr, size, phys); + mutex_unlock(&vmm->mutex); + } + + return ret; +} + +static int nvkm_uvmm_mthd_unmap(struct nvkm_uvmm *uvmm, void *argv, u32 argc) { struct nvkm_client *client = uvmm->object.client; @@ -78,7 +141,7 @@ nvkm_uvmm_mthd_unmap(struct nvkm_uvmm *uvmm, void *argv, u32 argc) goto done; } - nvkm_vmm_unmap_locked(vmm, vma); + nvkm_vmm_unmap_locked(vmm, vma, false); ret = 0; done: mutex_unlock(&vmm->mutex); @@ -124,6 +187,11 @@ nvkm_uvmm_mthd_map(struct nvkm_uvmm *uvmm, void *argv, u32 argc) goto fail; } + if (ret = -EINVAL, vma->mapped && !vma->memory) { + VMM_DEBUG(vmm, "pfnmap %016llx", addr); + goto fail; + } + if (ret = -EINVAL, vma->addr != addr || vma->size != size) { if (addr + size > vma->addr + vma->size || vma->memory || (vma->refd == NVKM_VMA_PAGE_NONE && !vma->mapref)) { @@ -134,23 +202,10 @@ nvkm_uvmm_mthd_map(struct nvkm_uvmm *uvmm, void *argv, u32 argc) goto fail; } - if (vma->addr != addr) { - const u64 tail = vma->size + vma->addr - addr; - if (ret = -ENOMEM, !(vma = nvkm_vma_tail(vma, tail))) - goto fail; - vma->part = true; - nvkm_vmm_node_insert(vmm, vma); - } - - if (vma->size != size) { - const u64 tail = vma->size - size; - struct nvkm_vma *tmp; - if (ret = -ENOMEM, !(tmp = nvkm_vma_tail(vma, tail))) { - nvkm_vmm_unmap_region(vmm, vma); - goto fail; - } - tmp->part = true; - nvkm_vmm_node_insert(vmm, tmp); + vma = nvkm_vmm_node_split(vmm, vma, addr, size); + if (!vma) { + ret = -ENOMEM; + goto fail; } } vma->busy = true; @@ -284,6 +339,15 @@ nvkm_uvmm_mthd(struct nvkm_object *object, u32 mthd, void *argv, u32 argc) case NVIF_VMM_V0_PUT : return nvkm_uvmm_mthd_put (uvmm, argv, argc); case NVIF_VMM_V0_MAP : return nvkm_uvmm_mthd_map (uvmm, argv, argc); case NVIF_VMM_V0_UNMAP : return nvkm_uvmm_mthd_unmap (uvmm, argv, argc); + case NVIF_VMM_V0_PFNMAP: return nvkm_uvmm_mthd_pfnmap(uvmm, argv, argc); + case NVIF_VMM_V0_PFNCLR: return nvkm_uvmm_mthd_pfnclr(uvmm, argv, argc); + case NVIF_VMM_V0_MTHD(0x00) ... NVIF_VMM_V0_MTHD(0x7f): + if (uvmm->vmm->func->mthd) { + return uvmm->vmm->func->mthd(uvmm->vmm, + uvmm->object.client, + mthd, argv, argc); + } + break; default: break; } @@ -317,8 +381,10 @@ nvkm_uvmm_new(const struct nvkm_oclass *oclass, void *argv, u32 argc, struct nvkm_uvmm *uvmm; int ret = -ENOSYS; u64 addr, size; + bool managed; if (!(ret = nvif_unpack(ret, &argv, &argc, args->v0, 0, 0, more))) { + managed = args->v0.managed != 0; addr = args->v0.addr; size = args->v0.size; } else @@ -330,7 +396,7 @@ nvkm_uvmm_new(const struct nvkm_oclass *oclass, void *argv, u32 argc, *pobject = &uvmm->object; if (!mmu->vmm) { - ret = mmu->func->vmm.ctor(mmu, addr, size, argv, argc, + ret = mmu->func->vmm.ctor(mmu, managed, addr, size, argv, argc, NULL, "user", &uvmm->vmm); if (ret) return ret; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c index 7459def78d50..fa93f964e6a4 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c @@ -255,11 +255,23 @@ nvkm_vmm_unref_sptes(struct nvkm_vmm_iter *it, struct nvkm_vmm_pt *pgt, } static bool -nvkm_vmm_unref_ptes(struct nvkm_vmm_iter *it, u32 ptei, u32 ptes) +nvkm_vmm_unref_ptes(struct nvkm_vmm_iter *it, bool pfn, u32 ptei, u32 ptes) { const struct nvkm_vmm_desc *desc = it->desc; const int type = desc->type == SPT; struct nvkm_vmm_pt *pgt = it->pt[0]; + bool dma; + + if (pfn) { + /* Need to clear PTE valid bits before we dma_unmap_page(). */ + dma = desc->func->pfn_clear(it->vmm, pgt->pt[type], ptei, ptes); + if (dma) { + /* GPU may have cached the PT, flush before unmap. */ + nvkm_vmm_flush_mark(it); + nvkm_vmm_flush(it); + desc->func->pfn_unmap(it->vmm, pgt->pt[type], ptei, ptes); + } + } /* Drop PTE references. */ pgt->refs[type] -= ptes; @@ -349,7 +361,7 @@ nvkm_vmm_ref_sptes(struct nvkm_vmm_iter *it, struct nvkm_vmm_pt *pgt, } static bool -nvkm_vmm_ref_ptes(struct nvkm_vmm_iter *it, u32 ptei, u32 ptes) +nvkm_vmm_ref_ptes(struct nvkm_vmm_iter *it, bool pfn, u32 ptei, u32 ptes) { const struct nvkm_vmm_desc *desc = it->desc; const int type = desc->type == SPT; @@ -379,7 +391,7 @@ nvkm_vmm_sparse_ptes(const struct nvkm_vmm_desc *desc, } static bool -nvkm_vmm_sparse_unref_ptes(struct nvkm_vmm_iter *it, u32 ptei, u32 ptes) +nvkm_vmm_sparse_unref_ptes(struct nvkm_vmm_iter *it, bool pfn, u32 ptei, u32 ptes) { struct nvkm_vmm_pt *pt = it->pt[0]; if (it->desc->type == PGD) @@ -387,14 +399,14 @@ nvkm_vmm_sparse_unref_ptes(struct nvkm_vmm_iter *it, u32 ptei, u32 ptes) else if (it->desc->type == LPT) memset(&pt->pte[ptei], 0x00, sizeof(pt->pte[0]) * ptes); - return nvkm_vmm_unref_ptes(it, ptei, ptes); + return nvkm_vmm_unref_ptes(it, pfn, ptei, ptes); } static bool -nvkm_vmm_sparse_ref_ptes(struct nvkm_vmm_iter *it, u32 ptei, u32 ptes) +nvkm_vmm_sparse_ref_ptes(struct nvkm_vmm_iter *it, bool pfn, u32 ptei, u32 ptes) { nvkm_vmm_sparse_ptes(it->desc, it->pt[0], ptei, ptes); - return nvkm_vmm_ref_ptes(it, ptei, ptes); + return nvkm_vmm_ref_ptes(it, pfn, ptei, ptes); } static bool @@ -487,8 +499,8 @@ nvkm_vmm_ref_swpt(struct nvkm_vmm_iter *it, struct nvkm_vmm_pt *pgd, u32 pdei) static inline u64 nvkm_vmm_iter(struct nvkm_vmm *vmm, const struct nvkm_vmm_page *page, - u64 addr, u64 size, const char *name, bool ref, - bool (*REF_PTES)(struct nvkm_vmm_iter *, u32, u32), + u64 addr, u64 size, const char *name, bool ref, bool pfn, + bool (*REF_PTES)(struct nvkm_vmm_iter *, bool pfn, u32, u32), nvkm_vmm_pte_func MAP_PTES, struct nvkm_vmm_map *map, nvkm_vmm_pxe_func CLR_PTES) { @@ -548,7 +560,7 @@ nvkm_vmm_iter(struct nvkm_vmm *vmm, const struct nvkm_vmm_page *page, } /* Handle PTE updates. */ - if (!REF_PTES || REF_PTES(&it, ptei, ptes)) { + if (!REF_PTES || REF_PTES(&it, pfn, ptei, ptes)) { struct nvkm_mmu_pt *pt = pgt->pt[type]; if (MAP_PTES || CLR_PTES) { if (MAP_PTES) @@ -590,7 +602,7 @@ static void nvkm_vmm_ptes_sparse_put(struct nvkm_vmm *vmm, const struct nvkm_vmm_page *page, u64 addr, u64 size) { - nvkm_vmm_iter(vmm, page, addr, size, "sparse unref", false, + nvkm_vmm_iter(vmm, page, addr, size, "sparse unref", false, false, nvkm_vmm_sparse_unref_ptes, NULL, NULL, page->desc->func->invalid ? page->desc->func->invalid : page->desc->func->unmap); @@ -602,8 +614,8 @@ nvkm_vmm_ptes_sparse_get(struct nvkm_vmm *vmm, const struct nvkm_vmm_page *page, { if ((page->type & NVKM_VMM_PAGE_SPARSE)) { u64 fail = nvkm_vmm_iter(vmm, page, addr, size, "sparse ref", - true, nvkm_vmm_sparse_ref_ptes, NULL, - NULL, page->desc->func->sparse); + true, false, nvkm_vmm_sparse_ref_ptes, + NULL, NULL, page->desc->func->sparse); if (fail != ~0ULL) { if ((size = fail - addr)) nvkm_vmm_ptes_sparse_put(vmm, page, addr, size); @@ -666,11 +678,11 @@ nvkm_vmm_ptes_sparse(struct nvkm_vmm *vmm, u64 addr, u64 size, bool ref) static void nvkm_vmm_ptes_unmap_put(struct nvkm_vmm *vmm, const struct nvkm_vmm_page *page, - u64 addr, u64 size, bool sparse) + u64 addr, u64 size, bool sparse, bool pfn) { const struct nvkm_vmm_desc_func *func = page->desc->func; nvkm_vmm_iter(vmm, page, addr, size, "unmap + unref", - false, nvkm_vmm_unref_ptes, NULL, NULL, + false, pfn, nvkm_vmm_unref_ptes, NULL, NULL, sparse ? func->sparse : func->invalid ? func->invalid : func->unmap); } @@ -681,10 +693,10 @@ nvkm_vmm_ptes_get_map(struct nvkm_vmm *vmm, const struct nvkm_vmm_page *page, nvkm_vmm_pte_func func) { u64 fail = nvkm_vmm_iter(vmm, page, addr, size, "ref + map", true, - nvkm_vmm_ref_ptes, func, map, NULL); + false, nvkm_vmm_ref_ptes, func, map, NULL); if (fail != ~0ULL) { if ((size = fail - addr)) - nvkm_vmm_ptes_unmap_put(vmm, page, addr, size, false); + nvkm_vmm_ptes_unmap_put(vmm, page, addr, size, false, false); return -ENOMEM; } return 0; @@ -692,10 +704,11 @@ nvkm_vmm_ptes_get_map(struct nvkm_vmm *vmm, const struct nvkm_vmm_page *page, static void nvkm_vmm_ptes_unmap(struct nvkm_vmm *vmm, const struct nvkm_vmm_page *page, - u64 addr, u64 size, bool sparse) + u64 addr, u64 size, bool sparse, bool pfn) { const struct nvkm_vmm_desc_func *func = page->desc->func; - nvkm_vmm_iter(vmm, page, addr, size, "unmap", false, NULL, NULL, NULL, + nvkm_vmm_iter(vmm, page, addr, size, "unmap", false, pfn, + NULL, NULL, NULL, sparse ? func->sparse : func->invalid ? func->invalid : func->unmap); } @@ -705,7 +718,7 @@ nvkm_vmm_ptes_map(struct nvkm_vmm *vmm, const struct nvkm_vmm_page *page, u64 addr, u64 size, struct nvkm_vmm_map *map, nvkm_vmm_pte_func func) { - nvkm_vmm_iter(vmm, page, addr, size, "map", false, + nvkm_vmm_iter(vmm, page, addr, size, "map", false, false, NULL, func, map, NULL); } @@ -713,7 +726,7 @@ static void nvkm_vmm_ptes_put(struct nvkm_vmm *vmm, const struct nvkm_vmm_page *page, u64 addr, u64 size) { - nvkm_vmm_iter(vmm, page, addr, size, "unref", false, + nvkm_vmm_iter(vmm, page, addr, size, "unref", false, false, nvkm_vmm_unref_ptes, NULL, NULL, NULL); } @@ -721,7 +734,7 @@ static int nvkm_vmm_ptes_get(struct nvkm_vmm *vmm, const struct nvkm_vmm_page *page, u64 addr, u64 size) { - u64 fail = nvkm_vmm_iter(vmm, page, addr, size, "ref", true, + u64 fail = nvkm_vmm_iter(vmm, page, addr, size, "ref", true, false, nvkm_vmm_ref_ptes, NULL, NULL, NULL); if (fail != ~0ULL) { if (fail != addr) @@ -763,10 +776,25 @@ nvkm_vma_tail(struct nvkm_vma *vma, u64 tail) new->part = vma->part; new->user = vma->user; new->busy = vma->busy; + new->mapped = vma->mapped; list_add(&new->head, &vma->head); return new; } +static inline void +nvkm_vmm_free_remove(struct nvkm_vmm *vmm, struct nvkm_vma *vma) +{ + rb_erase(&vma->tree, &vmm->free); +} + +static inline void +nvkm_vmm_free_delete(struct nvkm_vmm *vmm, struct nvkm_vma *vma) +{ + nvkm_vmm_free_remove(vmm, vma); + list_del(&vma->head); + kfree(vma); +} + static void nvkm_vmm_free_insert(struct nvkm_vmm *vmm, struct nvkm_vma *vma) { @@ -795,7 +823,21 @@ nvkm_vmm_free_insert(struct nvkm_vmm *vmm, struct nvkm_vma *vma) rb_insert_color(&vma->tree, &vmm->free); } -void +static inline void +nvkm_vmm_node_remove(struct nvkm_vmm *vmm, struct nvkm_vma *vma) +{ + rb_erase(&vma->tree, &vmm->root); +} + +static inline void +nvkm_vmm_node_delete(struct nvkm_vmm *vmm, struct nvkm_vma *vma) +{ + nvkm_vmm_node_remove(vmm, vma); + list_del(&vma->head); + kfree(vma); +} + +static void nvkm_vmm_node_insert(struct nvkm_vmm *vmm, struct nvkm_vma *vma) { struct rb_node **ptr = &vmm->root.rb_node; @@ -834,12 +876,113 @@ nvkm_vmm_node_search(struct nvkm_vmm *vmm, u64 addr) return NULL; } +#define node(root, dir) (((root)->head.dir == &vmm->list) ? NULL : \ + list_entry((root)->head.dir, struct nvkm_vma, head)) + +static struct nvkm_vma * +nvkm_vmm_node_merge(struct nvkm_vmm *vmm, struct nvkm_vma *prev, + struct nvkm_vma *vma, struct nvkm_vma *next, u64 size) +{ + if (next) { + if (vma->size == size) { + vma->size += next->size; + nvkm_vmm_node_delete(vmm, next); + if (prev) { + prev->size += vma->size; + nvkm_vmm_node_delete(vmm, vma); + return prev; + } + return vma; + } + BUG_ON(prev); + + nvkm_vmm_node_remove(vmm, next); + vma->size -= size; + next->addr -= size; + next->size += size; + nvkm_vmm_node_insert(vmm, next); + return next; + } + + if (prev) { + if (vma->size != size) { + nvkm_vmm_node_remove(vmm, vma); + prev->size += size; + vma->addr += size; + vma->size -= size; + nvkm_vmm_node_insert(vmm, vma); + } else { + prev->size += vma->size; + nvkm_vmm_node_delete(vmm, vma); + } + return prev; + } + + return vma; +} + +struct nvkm_vma * +nvkm_vmm_node_split(struct nvkm_vmm *vmm, + struct nvkm_vma *vma, u64 addr, u64 size) +{ + struct nvkm_vma *prev = NULL; + + if (vma->addr != addr) { + prev = vma; + if (!(vma = nvkm_vma_tail(vma, vma->size + vma->addr - addr))) + return NULL; + vma->part = true; + nvkm_vmm_node_insert(vmm, vma); + } + + if (vma->size != size) { + struct nvkm_vma *tmp; + if (!(tmp = nvkm_vma_tail(vma, vma->size - size))) { + nvkm_vmm_node_merge(vmm, prev, vma, NULL, vma->size); + return NULL; + } + tmp->part = true; + nvkm_vmm_node_insert(vmm, tmp); + } + + return vma; +} + +static void +nvkm_vma_dump(struct nvkm_vma *vma) +{ + printk(KERN_ERR "%016llx %016llx %c%c%c%c%c%c%c%c%c %p\n", + vma->addr, (u64)vma->size, + vma->used ? '-' : 'F', + vma->mapref ? 'R' : '-', + vma->sparse ? 'S' : '-', + vma->page != NVKM_VMA_PAGE_NONE ? '0' + vma->page : '-', + vma->refd != NVKM_VMA_PAGE_NONE ? '0' + vma->refd : '-', + vma->part ? 'P' : '-', + vma->user ? 'U' : '-', + vma->busy ? 'B' : '-', + vma->mapped ? 'M' : '-', + vma->memory); +} + +static void +nvkm_vmm_dump(struct nvkm_vmm *vmm) +{ + struct nvkm_vma *vma; + list_for_each_entry(vma, &vmm->list, head) { + nvkm_vma_dump(vma); + } +} + static void nvkm_vmm_dtor(struct nvkm_vmm *vmm) { struct nvkm_vma *vma; struct rb_node *node; + if (0) + nvkm_vmm_dump(vmm); + while ((node = rb_first(&vmm->root))) { struct nvkm_vma *vma = rb_entry(node, typeof(*vma), tree); nvkm_vmm_put(vmm, &vma); @@ -872,16 +1015,32 @@ nvkm_vmm_dtor(struct nvkm_vmm *vmm) } } +static int +nvkm_vmm_ctor_managed(struct nvkm_vmm *vmm, u64 addr, u64 size) +{ + struct nvkm_vma *vma; + if (!(vma = nvkm_vma_new(addr, size))) + return -ENOMEM; + vma->mapref = true; + vma->sparse = false; + vma->used = true; + vma->user = true; + nvkm_vmm_node_insert(vmm, vma); + list_add_tail(&vma->head, &vmm->list); + return 0; +} + int nvkm_vmm_ctor(const struct nvkm_vmm_func *func, struct nvkm_mmu *mmu, - u32 pd_header, u64 addr, u64 size, struct lock_class_key *key, - const char *name, struct nvkm_vmm *vmm) + u32 pd_header, bool managed, u64 addr, u64 size, + struct lock_class_key *key, const char *name, + struct nvkm_vmm *vmm) { static struct lock_class_key _key; const struct nvkm_vmm_page *page = func->page; const struct nvkm_vmm_desc *desc; struct nvkm_vma *vma; - int levels, bits = 0; + int levels, bits = 0, ret; vmm->func = func; vmm->mmu = mmu; @@ -909,11 +1068,6 @@ nvkm_vmm_ctor(const struct nvkm_vmm_func *func, struct nvkm_mmu *mmu, if (WARN_ON(levels > NVKM_VMM_LEVELS_MAX)) return -EINVAL; - vmm->start = addr; - vmm->limit = size ? (addr + size) : (1ULL << bits); - if (vmm->start > vmm->limit || vmm->limit > (1ULL << bits)) - return -EINVAL; - /* Allocate top-level page table. */ vmm->pd = nvkm_vmm_pt_new(desc, false, NULL); if (!vmm->pd) @@ -936,67 +1090,273 @@ nvkm_vmm_ctor(const struct nvkm_vmm_func *func, struct nvkm_mmu *mmu, vmm->free = RB_ROOT; vmm->root = RB_ROOT; - if (!(vma = nvkm_vma_new(vmm->start, vmm->limit - vmm->start))) - return -ENOMEM; + if (managed) { + /* Address-space will be managed by the client for the most + * part, except for a specified area where NVKM allocations + * are allowed to be placed. + */ + vmm->start = 0; + vmm->limit = 1ULL << bits; + if (addr + size < addr || addr + size > vmm->limit) + return -EINVAL; + + /* Client-managed area before the NVKM-managed area. */ + if (addr && (ret = nvkm_vmm_ctor_managed(vmm, 0, addr))) + return ret; + + /* NVKM-managed area. */ + if (size) { + if (!(vma = nvkm_vma_new(addr, size))) + return -ENOMEM; + nvkm_vmm_free_insert(vmm, vma); + list_add_tail(&vma->head, &vmm->list); + } + + /* Client-managed area after the NVKM-managed area. */ + addr = addr + size; + size = vmm->limit - addr; + if (size && (ret = nvkm_vmm_ctor_managed(vmm, addr, size))) + return ret; + } else { + /* Address-space fully managed by NVKM, requiring calls to + * nvkm_vmm_get()/nvkm_vmm_put() to allocate address-space. + */ + vmm->start = addr; + vmm->limit = size ? (addr + size) : (1ULL << bits); + if (vmm->start > vmm->limit || vmm->limit > (1ULL << bits)) + return -EINVAL; + + if (!(vma = nvkm_vma_new(vmm->start, vmm->limit - vmm->start))) + return -ENOMEM; + + nvkm_vmm_free_insert(vmm, vma); + list_add(&vma->head, &vmm->list); + } - nvkm_vmm_free_insert(vmm, vma); - list_add(&vma->head, &vmm->list); return 0; } int nvkm_vmm_new_(const struct nvkm_vmm_func *func, struct nvkm_mmu *mmu, - u32 hdr, u64 addr, u64 size, struct lock_class_key *key, - const char *name, struct nvkm_vmm **pvmm) + u32 hdr, bool managed, u64 addr, u64 size, + struct lock_class_key *key, const char *name, + struct nvkm_vmm **pvmm) { if (!(*pvmm = kzalloc(sizeof(**pvmm), GFP_KERNEL))) return -ENOMEM; - return nvkm_vmm_ctor(func, mmu, hdr, addr, size, key, name, *pvmm); + return nvkm_vmm_ctor(func, mmu, hdr, managed, addr, size, key, name, *pvmm); } -#define node(root, dir) ((root)->head.dir == &vmm->list) ? NULL : \ - list_entry((root)->head.dir, struct nvkm_vma, head) +static struct nvkm_vma * +nvkm_vmm_pfn_split_merge(struct nvkm_vmm *vmm, struct nvkm_vma *vma, + u64 addr, u64 size, u8 page, bool map) +{ + struct nvkm_vma *prev = NULL; + struct nvkm_vma *next = NULL; -void -nvkm_vmm_unmap_region(struct nvkm_vmm *vmm, struct nvkm_vma *vma) + if (vma->addr == addr && vma->part && (prev = node(vma, prev))) { + if (prev->memory || prev->mapped != map) + prev = NULL; + } + + if (vma->addr + vma->size == addr + size && (next = node(vma, next))) { + if (!next->part || + next->memory || next->mapped != map) + next = NULL; + } + + if (prev || next) + return nvkm_vmm_node_merge(vmm, prev, vma, next, size); + return nvkm_vmm_node_split(vmm, vma, addr, size); +} + +int +nvkm_vmm_pfn_unmap(struct nvkm_vmm *vmm, u64 addr, u64 size) { + struct nvkm_vma *vma = nvkm_vmm_node_search(vmm, addr); struct nvkm_vma *next; + u64 limit = addr + size; + u64 start = addr; - nvkm_memory_tags_put(vma->memory, vmm->mmu->subdev.device, &vma->tags); - nvkm_memory_unref(&vma->memory); + if (!vma) + return -EINVAL; - if (vma->part) { - struct nvkm_vma *prev = node(vma, prev); - if (!prev->memory) { - prev->size += vma->size; - rb_erase(&vma->tree, &vmm->root); - list_del(&vma->head); - kfree(vma); - vma = prev; + do { + if (!vma->mapped || vma->memory) + continue; + + size = min(limit - start, vma->size - (start - vma->addr)); + + nvkm_vmm_ptes_unmap_put(vmm, &vmm->func->page[vma->refd], + start, size, false, true); + + next = nvkm_vmm_pfn_split_merge(vmm, vma, start, size, 0, false); + if (!WARN_ON(!next)) { + vma = next; + vma->refd = NVKM_VMA_PAGE_NONE; + vma->mapped = false; } + } while ((vma = node(vma, next)) && (start = vma->addr) < limit); + + return 0; +} + +/*TODO: + * - Avoid PT readback (for dma_unmap etc), this might end up being dealt + * with inside HMM, which would be a lot nicer for us to deal with. + * - Multiple page sizes (particularly for huge page support). + * - Support for systems without a 4KiB page size. + */ +int +nvkm_vmm_pfn_map(struct nvkm_vmm *vmm, u8 shift, u64 addr, u64 size, u64 *pfn) +{ + const struct nvkm_vmm_page *page = vmm->func->page; + struct nvkm_vma *vma, *tmp; + u64 limit = addr + size; + u64 start = addr; + int pm = size >> shift; + int pi = 0; + + /* Only support mapping where the page size of the incoming page + * array matches a page size available for direct mapping. + */ + while (page->shift && page->shift != shift && + page->desc->func->pfn == NULL) + page++; + + if (!page->shift || !IS_ALIGNED(addr, 1ULL << shift) || + !IS_ALIGNED(size, 1ULL << shift) || + addr + size < addr || addr + size > vmm->limit) { + VMM_DEBUG(vmm, "paged map %d %d %016llx %016llx\n", + shift, page->shift, addr, size); + return -EINVAL; } - next = node(vma, next); - if (next && next->part) { - if (!next->memory) { - vma->size += next->size; - rb_erase(&next->tree, &vmm->root); - list_del(&next->head); - kfree(next); + if (!(vma = nvkm_vmm_node_search(vmm, addr))) + return -ENOENT; + + do { + bool map = !!(pfn[pi] & NVKM_VMM_PFN_V); + bool mapped = vma->mapped; + u64 size = limit - start; + u64 addr = start; + int pn, ret = 0; + + /* Narrow the operation window to cover a single action (page + * should be mapped or not) within a single VMA. + */ + for (pn = 0; pi + pn < pm; pn++) { + if (map != !!(pfn[pi + pn] & NVKM_VMM_PFN_V)) + break; } - } + size = min_t(u64, size, pn << page->shift); + size = min_t(u64, size, vma->size + vma->addr - addr); + + /* Reject any operation to unmanaged regions, and areas that + * have nvkm_memory objects mapped in them already. + */ + if (!vma->mapref || vma->memory) { + ret = -EINVAL; + goto next; + } + + /* In order to both properly refcount GPU page tables, and + * prevent "normal" mappings and these direct mappings from + * interfering with each other, we need to track contiguous + * ranges that have been mapped with this interface. + * + * Here we attempt to either split an existing VMA so we're + * able to flag the region as either unmapped/mapped, or to + * merge with adjacent VMAs that are already compatible. + * + * If the region is already compatible, nothing is required. + */ + if (map != mapped) { + tmp = nvkm_vmm_pfn_split_merge(vmm, vma, addr, size, + page - + vmm->func->page, map); + if (WARN_ON(!tmp)) { + ret = -ENOMEM; + goto next; + } + + if ((tmp->mapped = map)) + tmp->refd = page - vmm->func->page; + else + tmp->refd = NVKM_VMA_PAGE_NONE; + vma = tmp; + } + + /* Update HW page tables. */ + if (map) { + struct nvkm_vmm_map args; + args.page = page; + args.pfn = &pfn[pi]; + + if (!mapped) { + ret = nvkm_vmm_ptes_get_map(vmm, page, addr, + size, &args, page-> + desc->func->pfn); + } else { + nvkm_vmm_ptes_map(vmm, page, addr, size, &args, + page->desc->func->pfn); + } + } else { + if (mapped) { + nvkm_vmm_ptes_unmap_put(vmm, page, addr, size, + false, true); + } + } + +next: + /* Iterate to next operation. */ + if (vma->addr + vma->size == addr + size) + vma = node(vma, next); + start += size; + + if (ret) { + /* Failure is signalled by clearing the valid bit on + * any PFN that couldn't be modified as requested. + */ + while (size) { + pfn[pi++] = NVKM_VMM_PFN_NONE; + size -= 1 << page->shift; + } + } else { + pi += size >> page->shift; + } + } while (vma && start < limit); + + return 0; } void -nvkm_vmm_unmap_locked(struct nvkm_vmm *vmm, struct nvkm_vma *vma) +nvkm_vmm_unmap_region(struct nvkm_vmm *vmm, struct nvkm_vma *vma) +{ + struct nvkm_vma *prev = NULL; + struct nvkm_vma *next; + + nvkm_memory_tags_put(vma->memory, vmm->mmu->subdev.device, &vma->tags); + nvkm_memory_unref(&vma->memory); + vma->mapped = false; + + if (vma->part && (prev = node(vma, prev)) && prev->mapped) + prev = NULL; + if ((next = node(vma, next)) && (!next->part || next->mapped)) + next = NULL; + nvkm_vmm_node_merge(vmm, prev, vma, next, vma->size); +} + +void +nvkm_vmm_unmap_locked(struct nvkm_vmm *vmm, struct nvkm_vma *vma, bool pfn) { const struct nvkm_vmm_page *page = &vmm->func->page[vma->refd]; if (vma->mapref) { - nvkm_vmm_ptes_unmap_put(vmm, page, vma->addr, vma->size, vma->sparse); + nvkm_vmm_ptes_unmap_put(vmm, page, vma->addr, vma->size, vma->sparse, pfn); vma->refd = NVKM_VMA_PAGE_NONE; } else { - nvkm_vmm_ptes_unmap(vmm, page, vma->addr, vma->size, vma->sparse); + nvkm_vmm_ptes_unmap(vmm, page, vma->addr, vma->size, vma->sparse, pfn); } nvkm_vmm_unmap_region(vmm, vma); @@ -1007,7 +1367,7 @@ nvkm_vmm_unmap(struct nvkm_vmm *vmm, struct nvkm_vma *vma) { if (vma->memory) { mutex_lock(&vmm->mutex); - nvkm_vmm_unmap_locked(vmm, vma); + nvkm_vmm_unmap_locked(vmm, vma, false); mutex_unlock(&vmm->mutex); } } @@ -1141,6 +1501,7 @@ nvkm_vmm_map_locked(struct nvkm_vmm *vmm, struct nvkm_vma *vma, nvkm_memory_tags_put(vma->memory, vmm->mmu->subdev.device, &vma->tags); nvkm_memory_unref(&vma->memory); vma->memory = nvkm_memory_ref(map->memory); + vma->mapped = true; vma->tags = map->tags; return 0; } @@ -1163,18 +1524,14 @@ nvkm_vmm_put_region(struct nvkm_vmm *vmm, struct nvkm_vma *vma) struct nvkm_vma *prev, *next; if ((prev = node(vma, prev)) && !prev->used) { - rb_erase(&prev->tree, &vmm->free); - list_del(&prev->head); vma->addr = prev->addr; vma->size += prev->size; - kfree(prev); + nvkm_vmm_free_delete(vmm, prev); } if ((next = node(vma, next)) && !next->used) { - rb_erase(&next->tree, &vmm->free); - list_del(&next->head); vma->size += next->size; - kfree(next); + nvkm_vmm_free_delete(vmm, next); } nvkm_vmm_free_insert(vmm, vma); @@ -1190,14 +1547,16 @@ nvkm_vmm_put_locked(struct nvkm_vmm *vmm, struct nvkm_vma *vma) if (vma->mapref || !vma->sparse) { do { - const bool map = next->memory != NULL; + const bool mem = next->memory != NULL; + const bool map = next->mapped; const u8 refd = next->refd; const u64 addr = next->addr; u64 size = next->size; /* Merge regions that are in the same state. */ while ((next = node(next, next)) && next->part && - (next->memory != NULL) == map && + (next->mapped == map) && + (next->memory != NULL) == mem && (next->refd == refd)) size += next->size; @@ -1207,7 +1566,8 @@ nvkm_vmm_put_locked(struct nvkm_vmm *vmm, struct nvkm_vma *vma) * the page tree. */ nvkm_vmm_ptes_unmap_put(vmm, &page[refd], addr, - size, vma->sparse); + size, vma->sparse, + !mem); } else if (refd != NVKM_VMA_PAGE_NONE) { /* Drop allocation-time PTE references. */ @@ -1222,7 +1582,7 @@ nvkm_vmm_put_locked(struct nvkm_vmm *vmm, struct nvkm_vma *vma) */ next = vma; do { - if (next->memory) + if (next->mapped) nvkm_vmm_unmap_region(vmm, next); } while ((next = node(vma, next)) && next->part); @@ -1250,7 +1610,7 @@ nvkm_vmm_put_locked(struct nvkm_vmm *vmm, struct nvkm_vma *vma) } /* Remove VMA from the list of allocated nodes. */ - rb_erase(&vma->tree, &vmm->root); + nvkm_vmm_node_remove(vmm, vma); /* Merge VMA back into the free list. */ vma->page = NVKM_VMA_PAGE_NONE; @@ -1357,7 +1717,7 @@ nvkm_vmm_get_locked(struct nvkm_vmm *vmm, bool getref, bool mapref, bool sparse, tail = ALIGN_DOWN(tail, vmm->func->page_block); if (addr <= tail && tail - addr >= size) { - rb_erase(&this->tree, &vmm->free); + nvkm_vmm_free_remove(vmm, this); vma = this; break; } @@ -1443,7 +1803,7 @@ nvkm_vmm_join(struct nvkm_vmm *vmm, struct nvkm_memory *inst) } static bool -nvkm_vmm_boot_ptes(struct nvkm_vmm_iter *it, u32 ptei, u32 ptes) +nvkm_vmm_boot_ptes(struct nvkm_vmm_iter *it, bool pfn, u32 ptei, u32 ptes) { const struct nvkm_vmm_desc *desc = it->desc; const int type = desc->type == SPT; @@ -1465,7 +1825,7 @@ nvkm_vmm_boot(struct nvkm_vmm *vmm) if (ret) return ret; - nvkm_vmm_iter(vmm, page, vmm->start, limit, "bootstrap", false, + nvkm_vmm_iter(vmm, page, vmm->start, limit, "bootstrap", false, false, nvkm_vmm_boot_ptes, NULL, NULL, NULL); vmm->bootstrapped = true; return 0; @@ -1505,7 +1865,8 @@ nvkm_vmm_new(struct nvkm_device *device, u64 addr, u64 size, void *argv, struct nvkm_mmu *mmu = device->mmu; struct nvkm_vmm *vmm = NULL; int ret; - ret = mmu->func->vmm.ctor(mmu, addr, size, argv, argc, key, name, &vmm); + ret = mmu->func->vmm.ctor(mmu, false, addr, size, argv, argc, + key, name, &vmm); if (ret) nvkm_vmm_unref(&vmm); *pvmm = vmm; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.h b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.h index 1a3b0a3724ca..5e55ecbd8005 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.h +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.h @@ -67,6 +67,10 @@ struct nvkm_vmm_desc_func { nvkm_vmm_pte_func mem; nvkm_vmm_pte_func dma; nvkm_vmm_pte_func sgl; + + nvkm_vmm_pte_func pfn; + bool (*pfn_clear)(struct nvkm_vmm *, struct nvkm_mmu_pt *, u32 ptei, u32 ptes); + nvkm_vmm_pxe_func pfn_unmap; }; extern const struct nvkm_vmm_desc_func gf100_vmm_pgd; @@ -141,6 +145,11 @@ struct nvkm_vmm_func { struct nvkm_vmm_map *); void (*flush)(struct nvkm_vmm *, int depth); + int (*mthd)(struct nvkm_vmm *, struct nvkm_client *, + u32 mthd, void *argv, u32 argc); + + void (*invalidate_pdb)(struct nvkm_vmm *, u64 addr); + u64 page_block; const struct nvkm_vmm_page page[]; }; @@ -151,24 +160,38 @@ struct nvkm_vmm_join { }; int nvkm_vmm_new_(const struct nvkm_vmm_func *, struct nvkm_mmu *, - u32 pd_header, u64 addr, u64 size, struct lock_class_key *, - const char *name, struct nvkm_vmm **); + u32 pd_header, bool managed, u64 addr, u64 size, + struct lock_class_key *, const char *name, + struct nvkm_vmm **); int nvkm_vmm_ctor(const struct nvkm_vmm_func *, struct nvkm_mmu *, - u32 pd_header, u64 addr, u64 size, struct lock_class_key *, - const char *name, struct nvkm_vmm *); + u32 pd_header, bool managed, u64 addr, u64 size, + struct lock_class_key *, const char *name, struct nvkm_vmm *); struct nvkm_vma *nvkm_vmm_node_search(struct nvkm_vmm *, u64 addr); +struct nvkm_vma *nvkm_vmm_node_split(struct nvkm_vmm *, struct nvkm_vma *, + u64 addr, u64 size); int nvkm_vmm_get_locked(struct nvkm_vmm *, bool getref, bool mapref, bool sparse, u8 page, u8 align, u64 size, struct nvkm_vma **pvma); void nvkm_vmm_put_locked(struct nvkm_vmm *, struct nvkm_vma *); -void nvkm_vmm_unmap_locked(struct nvkm_vmm *, struct nvkm_vma *); -void nvkm_vmm_unmap_region(struct nvkm_vmm *vmm, struct nvkm_vma *vma); +void nvkm_vmm_unmap_locked(struct nvkm_vmm *, struct nvkm_vma *, bool pfn); +void nvkm_vmm_unmap_region(struct nvkm_vmm *, struct nvkm_vma *); + +#define NVKM_VMM_PFN_ADDR 0xfffffffffffff000ULL +#define NVKM_VMM_PFN_ADDR_SHIFT 12 +#define NVKM_VMM_PFN_APER 0x00000000000000f0ULL +#define NVKM_VMM_PFN_HOST 0x0000000000000000ULL +#define NVKM_VMM_PFN_VRAM 0x0000000000000010ULL +#define NVKM_VMM_PFN_W 0x0000000000000002ULL +#define NVKM_VMM_PFN_V 0x0000000000000001ULL +#define NVKM_VMM_PFN_NONE 0x0000000000000000ULL + +int nvkm_vmm_pfn_map(struct nvkm_vmm *, u8 page, u64 addr, u64 size, u64 *pfn); +int nvkm_vmm_pfn_unmap(struct nvkm_vmm *, u64 addr, u64 size); struct nvkm_vma *nvkm_vma_tail(struct nvkm_vma *, u64 tail); -void nvkm_vmm_node_insert(struct nvkm_vmm *, struct nvkm_vma *); int nv04_vmm_new_(const struct nvkm_vmm_func *, struct nvkm_mmu *, u32, - u64, u64, void *, u32, struct lock_class_key *, + bool, u64, u64, void *, u32, struct lock_class_key *, const char *, struct nvkm_vmm **); int nv04_vmm_valid(struct nvkm_vmm *, void *, u32, struct nvkm_vmm_map *); @@ -178,65 +201,76 @@ int nv50_vmm_valid(struct nvkm_vmm *, void *, u32, struct nvkm_vmm_map *); void nv50_vmm_flush(struct nvkm_vmm *, int); int gf100_vmm_new_(const struct nvkm_vmm_func *, const struct nvkm_vmm_func *, - struct nvkm_mmu *, u64, u64, void *, u32, + struct nvkm_mmu *, bool, u64, u64, void *, u32, struct lock_class_key *, const char *, struct nvkm_vmm **); int gf100_vmm_join_(struct nvkm_vmm *, struct nvkm_memory *, u64 base); int gf100_vmm_join(struct nvkm_vmm *, struct nvkm_memory *); void gf100_vmm_part(struct nvkm_vmm *, struct nvkm_memory *); int gf100_vmm_aper(enum nvkm_memory_target); int gf100_vmm_valid(struct nvkm_vmm *, void *, u32, struct nvkm_vmm_map *); -void gf100_vmm_flush_(struct nvkm_vmm *, int); void gf100_vmm_flush(struct nvkm_vmm *, int); +void gf100_vmm_invalidate(struct nvkm_vmm *, u32 type); +void gf100_vmm_invalidate_pdb(struct nvkm_vmm *, u64 addr); int gk20a_vmm_aper(enum nvkm_memory_target); int gm200_vmm_new_(const struct nvkm_vmm_func *, const struct nvkm_vmm_func *, - struct nvkm_mmu *, u64, u64, void *, u32, + struct nvkm_mmu *, bool, u64, u64, void *, u32, struct lock_class_key *, const char *, struct nvkm_vmm **); int gm200_vmm_join_(struct nvkm_vmm *, struct nvkm_memory *, u64 base); int gm200_vmm_join(struct nvkm_vmm *, struct nvkm_memory *); +int gp100_vmm_new_(const struct nvkm_vmm_func *, + struct nvkm_mmu *, bool, u64, u64, void *, u32, + struct lock_class_key *, const char *, struct nvkm_vmm **); int gp100_vmm_join(struct nvkm_vmm *, struct nvkm_memory *); int gp100_vmm_valid(struct nvkm_vmm *, void *, u32, struct nvkm_vmm_map *); void gp100_vmm_flush(struct nvkm_vmm *, int); +int gp100_vmm_mthd(struct nvkm_vmm *, struct nvkm_client *, u32, void *, u32); +void gp100_vmm_invalidate_pdb(struct nvkm_vmm *, u64 addr); + +int gv100_vmm_join(struct nvkm_vmm *, struct nvkm_memory *); -int nv04_vmm_new(struct nvkm_mmu *, u64, u64, void *, u32, +int nv04_vmm_new(struct nvkm_mmu *, bool, u64, u64, void *, u32, struct lock_class_key *, const char *, struct nvkm_vmm **); -int nv41_vmm_new(struct nvkm_mmu *, u64, u64, void *, u32, +int nv41_vmm_new(struct nvkm_mmu *, bool, u64, u64, void *, u32, struct lock_class_key *, const char *, struct nvkm_vmm **); -int nv44_vmm_new(struct nvkm_mmu *, u64, u64, void *, u32, +int nv44_vmm_new(struct nvkm_mmu *, bool, u64, u64, void *, u32, struct lock_class_key *, const char *, struct nvkm_vmm **); -int nv50_vmm_new(struct nvkm_mmu *, u64, u64, void *, u32, +int nv50_vmm_new(struct nvkm_mmu *, bool, u64, u64, void *, u32, struct lock_class_key *, const char *, struct nvkm_vmm **); -int mcp77_vmm_new(struct nvkm_mmu *, u64, u64, void *, u32, +int mcp77_vmm_new(struct nvkm_mmu *, bool, u64, u64, void *, u32, struct lock_class_key *, const char *, struct nvkm_vmm **); -int g84_vmm_new(struct nvkm_mmu *, u64, u64, void *, u32, +int g84_vmm_new(struct nvkm_mmu *, bool, u64, u64, void *, u32, struct lock_class_key *, const char *, struct nvkm_vmm **); -int gf100_vmm_new(struct nvkm_mmu *, u64, u64, void *, u32, +int gf100_vmm_new(struct nvkm_mmu *, bool, u64, u64, void *, u32, struct lock_class_key *, const char *, struct nvkm_vmm **); -int gk104_vmm_new(struct nvkm_mmu *, u64, u64, void *, u32, +int gk104_vmm_new(struct nvkm_mmu *, bool, u64, u64, void *, u32, struct lock_class_key *, const char *, struct nvkm_vmm **); -int gk20a_vmm_new(struct nvkm_mmu *, u64, u64, void *, u32, +int gk20a_vmm_new(struct nvkm_mmu *, bool, u64, u64, void *, u32, struct lock_class_key *, const char *, struct nvkm_vmm **); -int gm200_vmm_new_fixed(struct nvkm_mmu *, u64, u64, void *, u32, +int gm200_vmm_new_fixed(struct nvkm_mmu *, bool, u64, u64, void *, u32, struct lock_class_key *, const char *, struct nvkm_vmm **); -int gm200_vmm_new(struct nvkm_mmu *, u64, u64, void *, u32, +int gm200_vmm_new(struct nvkm_mmu *, bool, u64, u64, void *, u32, struct lock_class_key *, const char *, struct nvkm_vmm **); -int gm20b_vmm_new_fixed(struct nvkm_mmu *, u64, u64, void *, u32, +int gm20b_vmm_new_fixed(struct nvkm_mmu *, bool, u64, u64, void *, u32, struct lock_class_key *, const char *, struct nvkm_vmm **); -int gm20b_vmm_new(struct nvkm_mmu *, u64, u64, void *, u32, +int gm20b_vmm_new(struct nvkm_mmu *, bool, u64, u64, void *, u32, + struct lock_class_key *, const char *, + struct nvkm_vmm **); +int gp100_vmm_new(struct nvkm_mmu *, bool, u64, u64, void *, u32, struct lock_class_key *, const char *, struct nvkm_vmm **); -int gp100_vmm_new(struct nvkm_mmu *, u64, u64, void *, u32, +int gp10b_vmm_new(struct nvkm_mmu *, bool, u64, u64, void *, u32, struct lock_class_key *, const char *, struct nvkm_vmm **); -int gp10b_vmm_new(struct nvkm_mmu *, u64, u64, void *, u32, +int gv100_vmm_new(struct nvkm_mmu *, bool, u64, u64, void *, u32, struct lock_class_key *, const char *, struct nvkm_vmm **); -int gv100_vmm_new(struct nvkm_mmu *, u64, u64, void *, u32, +int tu102_vmm_new(struct nvkm_mmu *, bool, u64, u64, void *, u32, struct lock_class_key *, const char *, struct nvkm_vmm **); diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgf100.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgf100.c index faf5a7e9265e..ab6424faf84c 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgf100.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgf100.c @@ -178,15 +178,19 @@ gf100_vmm_desc_16_16[] = { }; void -gf100_vmm_flush_(struct nvkm_vmm *vmm, int depth) +gf100_vmm_invalidate_pdb(struct nvkm_vmm *vmm, u64 addr) +{ + struct nvkm_device *device = vmm->mmu->subdev.device; + nvkm_wr32(device, 0x100cb8, addr); +} + +void +gf100_vmm_invalidate(struct nvkm_vmm *vmm, u32 type) { struct nvkm_subdev *subdev = &vmm->mmu->subdev; struct nvkm_device *device = subdev->device; - u32 type = depth << 24; - - type = 0x00000001; /* PAGE_ALL */ - if (atomic_read(&vmm->engref[NVKM_SUBDEV_BAR])) - type |= 0x00000004; /* HUB_ONLY */ + struct nvkm_mmu_pt *pd = vmm->pd->pt[0]; + u64 addr = 0; mutex_lock(&subdev->mutex); /* Looks like maybe a "free flush slots" counter, the @@ -197,7 +201,20 @@ gf100_vmm_flush_(struct nvkm_vmm *vmm, int depth) break; ); - nvkm_wr32(device, 0x100cb8, vmm->pd->pt[0]->addr >> 8); + if (!(type & 0x00000002) /* ALL_PDB. */) { + switch (nvkm_memory_target(pd->memory)) { + case NVKM_MEM_TARGET_VRAM: addr |= 0x00000000; break; + case NVKM_MEM_TARGET_HOST: addr |= 0x00000002; break; + case NVKM_MEM_TARGET_NCOH: addr |= 0x00000003; break; + default: + WARN_ON(1); + break; + } + addr |= (vmm->pd->pt[0]->addr >> 12) << 4; + + vmm->func->invalidate_pdb(vmm, addr); + } + nvkm_wr32(device, 0x100cbc, 0x80000000 | type); /* Wait for flush to be queued? */ @@ -211,7 +228,10 @@ gf100_vmm_flush_(struct nvkm_vmm *vmm, int depth) void gf100_vmm_flush(struct nvkm_vmm *vmm, int depth) { - gf100_vmm_flush_(vmm, 0); + u32 type = 0x00000001; /* PAGE_ALL */ + if (atomic_read(&vmm->engref[NVKM_SUBDEV_BAR])) + type |= 0x00000004; /* HUB_ONLY */ + gf100_vmm_invalidate(vmm, type); } int @@ -354,6 +374,7 @@ gf100_vmm_17 = { .aper = gf100_vmm_aper, .valid = gf100_vmm_valid, .flush = gf100_vmm_flush, + .invalidate_pdb = gf100_vmm_invalidate_pdb, .page = { { 17, &gf100_vmm_desc_17_17[0], NVKM_VMM_PAGE_xVxC }, { 12, &gf100_vmm_desc_17_12[0], NVKM_VMM_PAGE_xVHx }, @@ -368,6 +389,7 @@ gf100_vmm_16 = { .aper = gf100_vmm_aper, .valid = gf100_vmm_valid, .flush = gf100_vmm_flush, + .invalidate_pdb = gf100_vmm_invalidate_pdb, .page = { { 16, &gf100_vmm_desc_16_16[0], NVKM_VMM_PAGE_xVxC }, { 12, &gf100_vmm_desc_16_12[0], NVKM_VMM_PAGE_xVHx }, @@ -378,14 +400,14 @@ gf100_vmm_16 = { int gf100_vmm_new_(const struct nvkm_vmm_func *func_16, const struct nvkm_vmm_func *func_17, - struct nvkm_mmu *mmu, u64 addr, u64 size, void *argv, u32 argc, - struct lock_class_key *key, const char *name, - struct nvkm_vmm **pvmm) + struct nvkm_mmu *mmu, bool managed, u64 addr, u64 size, + void *argv, u32 argc, struct lock_class_key *key, + const char *name, struct nvkm_vmm **pvmm) { switch (mmu->subdev.device->fb->page) { - case 16: return nv04_vmm_new_(func_16, mmu, 0, addr, size, + case 16: return nv04_vmm_new_(func_16, mmu, 0, managed, addr, size, argv, argc, key, name, pvmm); - case 17: return nv04_vmm_new_(func_17, mmu, 0, addr, size, + case 17: return nv04_vmm_new_(func_17, mmu, 0, managed, addr, size, argv, argc, key, name, pvmm); default: WARN_ON(1); @@ -394,10 +416,10 @@ gf100_vmm_new_(const struct nvkm_vmm_func *func_16, } int -gf100_vmm_new(struct nvkm_mmu *mmu, u64 addr, u64 size, void *argv, u32 argc, - struct lock_class_key *key, const char *name, - struct nvkm_vmm **pvmm) +gf100_vmm_new(struct nvkm_mmu *mmu, bool managed, u64 addr, u64 size, + void *argv, u32 argc, struct lock_class_key *key, + const char *name, struct nvkm_vmm **pvmm) { - return gf100_vmm_new_(&gf100_vmm_16, &gf100_vmm_17, mmu, addr, + return gf100_vmm_new_(&gf100_vmm_16, &gf100_vmm_17, mmu, managed, addr, size, argv, argc, key, name, pvmm); } diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgk104.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgk104.c index 0ebb7bccfcd2..0b59c01fd146 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgk104.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgk104.c @@ -71,6 +71,7 @@ gk104_vmm_17 = { .aper = gf100_vmm_aper, .valid = gf100_vmm_valid, .flush = gf100_vmm_flush, + .invalidate_pdb = gf100_vmm_invalidate_pdb, .page = { { 17, &gk104_vmm_desc_17_17[0], NVKM_VMM_PAGE_xVxC }, { 12, &gk104_vmm_desc_17_12[0], NVKM_VMM_PAGE_xVHx }, @@ -85,6 +86,7 @@ gk104_vmm_16 = { .aper = gf100_vmm_aper, .valid = gf100_vmm_valid, .flush = gf100_vmm_flush, + .invalidate_pdb = gf100_vmm_invalidate_pdb, .page = { { 16, &gk104_vmm_desc_16_16[0], NVKM_VMM_PAGE_xVxC }, { 12, &gk104_vmm_desc_16_12[0], NVKM_VMM_PAGE_xVHx }, @@ -93,10 +95,10 @@ gk104_vmm_16 = { }; int -gk104_vmm_new(struct nvkm_mmu *mmu, u64 addr, u64 size, void *argv, u32 argc, - struct lock_class_key *key, const char *name, - struct nvkm_vmm **pvmm) +gk104_vmm_new(struct nvkm_mmu *mmu, bool managed, u64 addr, u64 size, + void *argv, u32 argc, struct lock_class_key *key, + const char *name, struct nvkm_vmm **pvmm) { - return gf100_vmm_new_(&gk104_vmm_16, &gk104_vmm_17, mmu, addr, + return gf100_vmm_new_(&gk104_vmm_16, &gk104_vmm_17, mmu, managed, addr, size, argv, argc, key, name, pvmm); } diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgk20a.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgk20a.c index 8086994a0446..5a9582dce970 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgk20a.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgk20a.c @@ -40,6 +40,7 @@ gk20a_vmm_17 = { .aper = gf100_vmm_aper, .valid = gf100_vmm_valid, .flush = gf100_vmm_flush, + .invalidate_pdb = gf100_vmm_invalidate_pdb, .page = { { 17, &gk104_vmm_desc_17_17[0], NVKM_VMM_PAGE_xxHC }, { 12, &gk104_vmm_desc_17_12[0], NVKM_VMM_PAGE_xxHx }, @@ -54,6 +55,7 @@ gk20a_vmm_16 = { .aper = gf100_vmm_aper, .valid = gf100_vmm_valid, .flush = gf100_vmm_flush, + .invalidate_pdb = gf100_vmm_invalidate_pdb, .page = { { 16, &gk104_vmm_desc_16_16[0], NVKM_VMM_PAGE_xxHC }, { 12, &gk104_vmm_desc_16_12[0], NVKM_VMM_PAGE_xxHx }, @@ -62,10 +64,10 @@ gk20a_vmm_16 = { }; int -gk20a_vmm_new(struct nvkm_mmu *mmu, u64 addr, u64 size, void *argv, u32 argc, - struct lock_class_key *key, const char *name, - struct nvkm_vmm **pvmm) +gk20a_vmm_new(struct nvkm_mmu *mmu, bool managed, u64 addr, u64 size, + void *argv, u32 argc, struct lock_class_key *key, + const char *name, struct nvkm_vmm **pvmm) { - return gf100_vmm_new_(&gk20a_vmm_16, &gk20a_vmm_17, mmu, addr, + return gf100_vmm_new_(&gk20a_vmm_16, &gk20a_vmm_17, mmu, managed, addr, size, argv, argc, key, name, pvmm); } diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgm200.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgm200.c index a1676a4644fe..2e61af02d4d8 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgm200.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgm200.c @@ -113,6 +113,7 @@ gm200_vmm_17 = { .aper = gf100_vmm_aper, .valid = gf100_vmm_valid, .flush = gf100_vmm_flush, + .invalidate_pdb = gf100_vmm_invalidate_pdb, .page = { { 27, &gm200_vmm_desc_17_17[1], NVKM_VMM_PAGE_Sxxx }, { 17, &gm200_vmm_desc_17_17[0], NVKM_VMM_PAGE_SVxC }, @@ -128,6 +129,7 @@ gm200_vmm_16 = { .aper = gf100_vmm_aper, .valid = gf100_vmm_valid, .flush = gf100_vmm_flush, + .invalidate_pdb = gf100_vmm_invalidate_pdb, .page = { { 27, &gm200_vmm_desc_16_16[1], NVKM_VMM_PAGE_Sxxx }, { 16, &gm200_vmm_desc_16_16[0], NVKM_VMM_PAGE_SVxC }, @@ -139,9 +141,9 @@ gm200_vmm_16 = { int gm200_vmm_new_(const struct nvkm_vmm_func *func_16, const struct nvkm_vmm_func *func_17, - struct nvkm_mmu *mmu, u64 addr, u64 size, void *argv, u32 argc, - struct lock_class_key *key, const char *name, - struct nvkm_vmm **pvmm) + struct nvkm_mmu *mmu, bool managed, u64 addr, u64 size, + void *argv, u32 argc, struct lock_class_key *key, + const char *name, struct nvkm_vmm **pvmm) { const struct nvkm_vmm_func *func; union { @@ -163,23 +165,23 @@ gm200_vmm_new_(const struct nvkm_vmm_func *func_16, } else return ret; - return nvkm_vmm_new_(func, mmu, 0, addr, size, key, name, pvmm); + return nvkm_vmm_new_(func, mmu, 0, managed, addr, size, key, name, pvmm); } int -gm200_vmm_new(struct nvkm_mmu *mmu, u64 addr, u64 size, void *argv, u32 argc, - struct lock_class_key *key, const char *name, - struct nvkm_vmm **pvmm) +gm200_vmm_new(struct nvkm_mmu *mmu, bool managed, u64 addr, u64 size, + void *argv, u32 argc, struct lock_class_key *key, + const char *name, struct nvkm_vmm **pvmm) { - return gm200_vmm_new_(&gm200_vmm_16, &gm200_vmm_17, mmu, addr, + return gm200_vmm_new_(&gm200_vmm_16, &gm200_vmm_17, mmu, managed, addr, size, argv, argc, key, name, pvmm); } int -gm200_vmm_new_fixed(struct nvkm_mmu *mmu, u64 addr, u64 size, +gm200_vmm_new_fixed(struct nvkm_mmu *mmu, bool managed, u64 addr, u64 size, void *argv, u32 argc, struct lock_class_key *key, const char *name, struct nvkm_vmm **pvmm) { - return gf100_vmm_new_(&gm200_vmm_16, &gm200_vmm_17, mmu, addr, + return gf100_vmm_new_(&gm200_vmm_16, &gm200_vmm_17, mmu, managed, addr, size, argv, argc, key, name, pvmm); } diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgm20b.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgm20b.c index 64d4b6cff8dd..96b759695dd8 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgm20b.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgm20b.c @@ -28,6 +28,7 @@ gm20b_vmm_17 = { .aper = gk20a_vmm_aper, .valid = gf100_vmm_valid, .flush = gf100_vmm_flush, + .invalidate_pdb = gf100_vmm_invalidate_pdb, .page = { { 27, &gm200_vmm_desc_17_17[1], NVKM_VMM_PAGE_Sxxx }, { 17, &gm200_vmm_desc_17_17[0], NVKM_VMM_PAGE_SxHC }, @@ -43,6 +44,7 @@ gm20b_vmm_16 = { .aper = gk20a_vmm_aper, .valid = gf100_vmm_valid, .flush = gf100_vmm_flush, + .invalidate_pdb = gf100_vmm_invalidate_pdb, .page = { { 27, &gm200_vmm_desc_16_16[1], NVKM_VMM_PAGE_Sxxx }, { 16, &gm200_vmm_desc_16_16[0], NVKM_VMM_PAGE_SxHC }, @@ -52,19 +54,19 @@ gm20b_vmm_16 = { }; int -gm20b_vmm_new(struct nvkm_mmu *mmu, u64 addr, u64 size, void *argv, u32 argc, - struct lock_class_key *key, const char *name, - struct nvkm_vmm **pvmm) +gm20b_vmm_new(struct nvkm_mmu *mmu, bool managed, u64 addr, u64 size, + void *argv, u32 argc, struct lock_class_key *key, + const char *name, struct nvkm_vmm **pvmm) { - return gm200_vmm_new_(&gm20b_vmm_16, &gm20b_vmm_17, mmu, addr, + return gm200_vmm_new_(&gm20b_vmm_16, &gm20b_vmm_17, mmu, managed, addr, size, argv, argc, key, name, pvmm); } int -gm20b_vmm_new_fixed(struct nvkm_mmu *mmu, u64 addr, u64 size, +gm20b_vmm_new_fixed(struct nvkm_mmu *mmu, bool managed, u64 addr, u64 size, void *argv, u32 argc, struct lock_class_key *key, const char *name, struct nvkm_vmm **pvmm) { - return gf100_vmm_new_(&gm20b_vmm_16, &gm20b_vmm_17, mmu, addr, + return gf100_vmm_new_(&gm20b_vmm_16, &gm20b_vmm_17, mmu, managed, addr, size, argv, argc, key, name, pvmm); } diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgp100.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgp100.c index 059fafe0e771..b4f519768d5e 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgp100.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgp100.c @@ -21,12 +21,90 @@ */ #include "vmm.h" +#include <core/client.h> #include <subdev/fb.h> #include <subdev/ltc.h> +#include <subdev/timer.h> +#include <engine/gr.h> #include <nvif/ifc00d.h> #include <nvif/unpack.h> +static void +gp100_vmm_pfn_unmap(struct nvkm_vmm *vmm, + struct nvkm_mmu_pt *pt, u32 ptei, u32 ptes) +{ + struct device *dev = vmm->mmu->subdev.device->dev; + dma_addr_t addr; + + nvkm_kmap(pt->memory); + while (ptes--) { + u32 datalo = nvkm_ro32(pt->memory, pt->base + ptei * 8 + 0); + u32 datahi = nvkm_ro32(pt->memory, pt->base + ptei * 8 + 4); + u64 data = (u64)datahi << 32 | datalo; + if ((data & (3ULL << 1)) != 0) { + addr = (data >> 8) << 12; + dma_unmap_page(dev, addr, PAGE_SIZE, DMA_BIDIRECTIONAL); + } + ptei++; + } + nvkm_done(pt->memory); +} + +static bool +gp100_vmm_pfn_clear(struct nvkm_vmm *vmm, + struct nvkm_mmu_pt *pt, u32 ptei, u32 ptes) +{ + bool dma = false; + nvkm_kmap(pt->memory); + while (ptes--) { + u32 datalo = nvkm_ro32(pt->memory, pt->base + ptei * 8 + 0); + u32 datahi = nvkm_ro32(pt->memory, pt->base + ptei * 8 + 4); + u64 data = (u64)datahi << 32 | datalo; + if ((data & BIT_ULL(0)) && (data & (3ULL << 1)) != 0) { + VMM_WO064(pt, vmm, ptei * 8, data & ~BIT_ULL(0)); + dma = true; + } + ptei++; + } + nvkm_done(pt->memory); + return dma; +} + +static void +gp100_vmm_pgt_pfn(struct nvkm_vmm *vmm, struct nvkm_mmu_pt *pt, + u32 ptei, u32 ptes, struct nvkm_vmm_map *map) +{ + struct device *dev = vmm->mmu->subdev.device->dev; + dma_addr_t addr; + + nvkm_kmap(pt->memory); + while (ptes--) { + u64 data = 0; + if (!(*map->pfn & NVKM_VMM_PFN_W)) + data |= BIT_ULL(6); /* RO. */ + + if (!(*map->pfn & NVKM_VMM_PFN_VRAM)) { + addr = *map->pfn >> NVKM_VMM_PFN_ADDR_SHIFT; + addr = dma_map_page(dev, pfn_to_page(addr), 0, + PAGE_SIZE, DMA_BIDIRECTIONAL); + if (!WARN_ON(dma_mapping_error(dev, addr))) { + data |= addr >> 4; + data |= 2ULL << 1; /* SYSTEM_COHERENT_MEMORY. */ + data |= BIT_ULL(3); /* VOL. */ + data |= BIT_ULL(0); /* VALID. */ + } + } else { + data |= (*map->pfn & NVKM_VMM_PFN_ADDR) >> 4; + data |= BIT_ULL(0); /* VALID. */ + } + + VMM_WO064(pt, vmm, ptei++ * 8, data); + map->pfn++; + } + nvkm_done(pt->memory); +} + static inline void gp100_vmm_pgt_pte(struct nvkm_vmm *vmm, struct nvkm_mmu_pt *pt, u32 ptei, u32 ptes, struct nvkm_vmm_map *map, u64 addr) @@ -89,6 +167,9 @@ gp100_vmm_desc_spt = { .mem = gp100_vmm_pgt_mem, .dma = gp100_vmm_pgt_dma, .sgl = gp100_vmm_pgt_sgl, + .pfn = gp100_vmm_pgt_pfn, + .pfn_clear = gp100_vmm_pfn_clear, + .pfn_unmap = gp100_vmm_pfn_unmap, }; static void @@ -306,16 +387,100 @@ gp100_vmm_valid(struct nvkm_vmm *vmm, void *argv, u32 argc, return 0; } +static int +gp100_vmm_fault_cancel(struct nvkm_vmm *vmm, void *argv, u32 argc) +{ + struct nvkm_device *device = vmm->mmu->subdev.device; + union { + struct gp100_vmm_fault_cancel_v0 v0; + } *args = argv; + int ret = -ENOSYS; + u32 inst, aper; + + if ((ret = nvif_unpack(ret, &argv, &argc, args->v0, 0, 0, false))) + return ret; + + /* Translate MaxwellFaultBufferA instance pointer to the same + * format as the NV_GR_FECS_CURRENT_CTX register. + */ + aper = (args->v0.inst >> 8) & 3; + args->v0.inst >>= 12; + args->v0.inst |= aper << 28; + args->v0.inst |= 0x80000000; + + if (!WARN_ON(nvkm_gr_ctxsw_pause(device))) { + if ((inst = nvkm_gr_ctxsw_inst(device)) == args->v0.inst) { + gf100_vmm_invalidate(vmm, 0x0000001b + /* CANCEL_TARGETED. */ | + (args->v0.hub << 20) | + (args->v0.gpc << 15) | + (args->v0.client << 9)); + } + WARN_ON(nvkm_gr_ctxsw_resume(device)); + } + + return 0; +} + +static int +gp100_vmm_fault_replay(struct nvkm_vmm *vmm, void *argv, u32 argc) +{ + union { + struct gp100_vmm_fault_replay_vn vn; + } *args = argv; + int ret = -ENOSYS; + + if (!(ret = nvif_unvers(ret, &argv, &argc, args->vn))) { + gf100_vmm_invalidate(vmm, 0x0000000b); /* REPLAY_GLOBAL. */ + } + + return ret; +} + +int +gp100_vmm_mthd(struct nvkm_vmm *vmm, + struct nvkm_client *client, u32 mthd, void *argv, u32 argc) +{ + if (client->super) { + switch (mthd) { + case GP100_VMM_VN_FAULT_REPLAY: + return gp100_vmm_fault_replay(vmm, argv, argc); + case GP100_VMM_VN_FAULT_CANCEL: + return gp100_vmm_fault_cancel(vmm, argv, argc); + default: + break; + } + } + return -EINVAL; +} + +void +gp100_vmm_invalidate_pdb(struct nvkm_vmm *vmm, u64 addr) +{ + struct nvkm_device *device = vmm->mmu->subdev.device; + nvkm_wr32(device, 0x100cb8, lower_32_bits(addr)); + nvkm_wr32(device, 0x100cec, upper_32_bits(addr)); +} + void gp100_vmm_flush(struct nvkm_vmm *vmm, int depth) { - gf100_vmm_flush_(vmm, 5 /* CACHE_LEVEL_UP_TO_PDE3 */ - depth); + u32 type = (5 /* CACHE_LEVEL_UP_TO_PDE3 */ - depth) << 24; + type = 0; /*XXX: need to confirm stuff works with depth enabled... */ + if (atomic_read(&vmm->engref[NVKM_SUBDEV_BAR])) + type |= 0x00000004; /* HUB_ONLY */ + type |= 0x00000001; /* PAGE_ALL */ + gf100_vmm_invalidate(vmm, type); } int gp100_vmm_join(struct nvkm_vmm *vmm, struct nvkm_memory *inst) { - const u64 base = BIT_ULL(10) /* VER2 */ | BIT_ULL(11); /* 64KiB */ + u64 base = BIT_ULL(10) /* VER2 */ | BIT_ULL(11) /* 64KiB */; + if (vmm->replay) { + base |= BIT_ULL(4); /* FAULT_REPLAY_TEX */ + base |= BIT_ULL(5); /* FAULT_REPLAY_GCC */ + } return gf100_vmm_join_(vmm, inst, base); } @@ -326,6 +491,8 @@ gp100_vmm = { .aper = gf100_vmm_aper, .valid = gp100_vmm_valid, .flush = gp100_vmm_flush, + .mthd = gp100_vmm_mthd, + .invalidate_pdb = gp100_vmm_invalidate_pdb, .page = { { 47, &gp100_vmm_desc_16[4], NVKM_VMM_PAGE_Sxxx }, { 38, &gp100_vmm_desc_16[3], NVKM_VMM_PAGE_Sxxx }, @@ -338,10 +505,39 @@ gp100_vmm = { }; int -gp100_vmm_new(struct nvkm_mmu *mmu, u64 addr, u64 size, void *argv, u32 argc, - struct lock_class_key *key, const char *name, - struct nvkm_vmm **pvmm) +gp100_vmm_new_(const struct nvkm_vmm_func *func, + struct nvkm_mmu *mmu, bool managed, u64 addr, u64 size, + void *argv, u32 argc, struct lock_class_key *key, + const char *name, struct nvkm_vmm **pvmm) +{ + union { + struct gp100_vmm_vn vn; + struct gp100_vmm_v0 v0; + } *args = argv; + int ret = -ENOSYS; + bool replay; + + if (!(ret = nvif_unpack(ret, &argv, &argc, args->v0, 0, 0, false))) { + replay = args->v0.fault_replay != 0; + } else + if (!(ret = nvif_unvers(ret, &argv, &argc, args->vn))) { + replay = false; + } else + return ret; + + ret = nvkm_vmm_new_(func, mmu, 0, managed, addr, size, key, name, pvmm); + if (ret) + return ret; + + (*pvmm)->replay = replay; + return 0; +} + +int +gp100_vmm_new(struct nvkm_mmu *mmu, bool managed, u64 addr, u64 size, + void *argv, u32 argc, struct lock_class_key *key, + const char *name, struct nvkm_vmm **pvmm) { - return nv04_vmm_new_(&gp100_vmm, mmu, 0, addr, size, - argv, argc, key, name, pvmm); + return gp100_vmm_new_(&gp100_vmm, mmu, managed, addr, size, + argv, argc, key, name, pvmm); } diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgp10b.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgp10b.c index 3dcc6bddb32f..e081239afe58 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgp10b.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgp10b.c @@ -28,6 +28,8 @@ gp10b_vmm = { .aper = gk20a_vmm_aper, .valid = gp100_vmm_valid, .flush = gp100_vmm_flush, + .mthd = gp100_vmm_mthd, + .invalidate_pdb = gp100_vmm_invalidate_pdb, .page = { { 47, &gp100_vmm_desc_16[4], NVKM_VMM_PAGE_Sxxx }, { 38, &gp100_vmm_desc_16[3], NVKM_VMM_PAGE_Sxxx }, @@ -40,10 +42,10 @@ gp10b_vmm = { }; int -gp10b_vmm_new(struct nvkm_mmu *mmu, u64 addr, u64 size, void *argv, u32 argc, - struct lock_class_key *key, const char *name, - struct nvkm_vmm **pvmm) +gp10b_vmm_new(struct nvkm_mmu *mmu, bool managed, u64 addr, u64 size, + void *argv, u32 argc, struct lock_class_key *key, + const char *name, struct nvkm_vmm **pvmm) { - return nv04_vmm_new_(&gp10b_vmm, mmu, 0, addr, size, - argv, argc, key, name, pvmm); + return gp100_vmm_new_(&gp10b_vmm, mmu, managed, addr, size, + argv, argc, key, name, pvmm); } diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgv100.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgv100.c index 2fa40c16e6d2..f0e21f63253a 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgv100.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgv100.c @@ -66,6 +66,8 @@ gv100_vmm = { .aper = gf100_vmm_aper, .valid = gp100_vmm_valid, .flush = gp100_vmm_flush, + .mthd = gp100_vmm_mthd, + .invalidate_pdb = gp100_vmm_invalidate_pdb, .page = { { 47, &gp100_vmm_desc_16[4], NVKM_VMM_PAGE_Sxxx }, { 38, &gp100_vmm_desc_16[3], NVKM_VMM_PAGE_Sxxx }, @@ -78,10 +80,10 @@ gv100_vmm = { }; int -gv100_vmm_new(struct nvkm_mmu *mmu, u64 addr, u64 size, void *argv, u32 argc, - struct lock_class_key *key, const char *name, - struct nvkm_vmm **pvmm) +gv100_vmm_new(struct nvkm_mmu *mmu, bool managed, u64 addr, u64 size, + void *argv, u32 argc, struct lock_class_key *key, + const char *name, struct nvkm_vmm **pvmm) { - return nv04_vmm_new_(&gv100_vmm, mmu, 0, addr, size, - argv, argc, key, name, pvmm); + return gp100_vmm_new_(&gv100_vmm, mmu, managed, addr, size, + argv, argc, key, name, pvmm); } diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmmcp77.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmmcp77.c index e63d984cbfd4..bdddd99f5877 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmmcp77.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmmcp77.c @@ -36,10 +36,10 @@ mcp77_vmm = { }; int -mcp77_vmm_new(struct nvkm_mmu *mmu, u64 addr, u64 size, void *argv, u32 argc, - struct lock_class_key *key, const char *name, - struct nvkm_vmm **pvmm) +mcp77_vmm_new(struct nvkm_mmu *mmu, bool managed, u64 addr, u64 size, + void *argv, u32 argc, struct lock_class_key *key, + const char *name, struct nvkm_vmm **pvmm) { - return nv04_vmm_new_(&mcp77_vmm, mmu, 0, addr, size, + return nv04_vmm_new_(&mcp77_vmm, mmu, 0, managed, addr, size, argv, argc, key, name, pvmm); } diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmnv04.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmnv04.c index 0cab1ffc9f64..4c6b3b7d221f 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmnv04.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmnv04.c @@ -100,16 +100,17 @@ nv04_vmm = { int nv04_vmm_new_(const struct nvkm_vmm_func *func, struct nvkm_mmu *mmu, - u32 pd_header, u64 addr, u64 size, void *argv, u32 argc, - struct lock_class_key *key, const char *name, - struct nvkm_vmm **pvmm) + u32 pd_header, bool managed, u64 addr, u64 size, + void *argv, u32 argc, struct lock_class_key *key, + const char *name, struct nvkm_vmm **pvmm) { union { struct nv04_vmm_vn vn; } *args = argv; int ret; - ret = nvkm_vmm_new_(func, mmu, pd_header, addr, size, key, name, pvmm); + ret = nvkm_vmm_new_(func, mmu, pd_header, managed, addr, size, + key, name, pvmm); if (ret) return ret; @@ -117,15 +118,15 @@ nv04_vmm_new_(const struct nvkm_vmm_func *func, struct nvkm_mmu *mmu, } int -nv04_vmm_new(struct nvkm_mmu *mmu, u64 addr, u64 size, void *argv, u32 argc, - struct lock_class_key *key, const char *name, +nv04_vmm_new(struct nvkm_mmu *mmu, bool managed, u64 addr, u64 size, + void *argv, u32 argc, struct lock_class_key *key, const char *name, struct nvkm_vmm **pvmm) { struct nvkm_memory *mem; struct nvkm_vmm *vmm; int ret; - ret = nv04_vmm_new_(&nv04_vmm, mmu, 8, addr, size, + ret = nv04_vmm_new_(&nv04_vmm, mmu, 8, managed, addr, size, argv, argc, key, name, &vmm); *pvmm = vmm; if (ret) diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmnv41.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmnv41.c index b595f130e573..1d3369683a21 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmnv41.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmnv41.c @@ -104,10 +104,10 @@ nv41_vmm = { }; int -nv41_vmm_new(struct nvkm_mmu *mmu, u64 addr, u64 size, void *argv, u32 argc, - struct lock_class_key *key, const char *name, +nv41_vmm_new(struct nvkm_mmu *mmu, bool managed, u64 addr, u64 size, + void *argv, u32 argc, struct lock_class_key *key, const char *name, struct nvkm_vmm **pvmm) { - return nv04_vmm_new_(&nv41_vmm, mmu, 0, addr, size, + return nv04_vmm_new_(&nv41_vmm, mmu, 0, managed, addr, size, argv, argc, key, name, pvmm); } diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmnv44.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmnv44.c index b834e4352334..a82936ba9890 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmnv44.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmnv44.c @@ -205,15 +205,15 @@ nv44_vmm = { }; int -nv44_vmm_new(struct nvkm_mmu *mmu, u64 addr, u64 size, void *argv, u32 argc, - struct lock_class_key *key, const char *name, +nv44_vmm_new(struct nvkm_mmu *mmu, bool managed, u64 addr, u64 size, + void *argv, u32 argc, struct lock_class_key *key, const char *name, struct nvkm_vmm **pvmm) { struct nvkm_subdev *subdev = &mmu->subdev; struct nvkm_vmm *vmm; int ret; - ret = nv04_vmm_new_(&nv44_vmm, mmu, 0, addr, size, + ret = nv04_vmm_new_(&nv44_vmm, mmu, 0, managed, addr, size, argv, argc, key, name, &vmm); *pvmm = vmm; if (ret) diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmnv50.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmnv50.c index 64f75d906202..c98afe3134ee 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmnv50.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmnv50.c @@ -376,10 +376,10 @@ nv50_vmm = { }; int -nv50_vmm_new(struct nvkm_mmu *mmu, u64 addr, u64 size, void *argv, u32 argc, - struct lock_class_key *key, const char *name, +nv50_vmm_new(struct nvkm_mmu *mmu, bool managed, u64 addr, u64 size, + void *argv, u32 argc, struct lock_class_key *key, const char *name, struct nvkm_vmm **pvmm) { - return nv04_vmm_new_(&nv50_vmm, mmu, 0, addr, size, + return nv04_vmm_new_(&nv50_vmm, mmu, 0, managed, addr, size, argv, argc, key, name, pvmm); } diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmtu102.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmtu102.c new file mode 100644 index 000000000000..be91cffc3b52 --- /dev/null +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmtu102.c @@ -0,0 +1,78 @@ +/* + * Copyright 2018 Red Hat Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ +#include "vmm.h" + +#include <subdev/timer.h> + +static void +tu102_vmm_flush(struct nvkm_vmm *vmm, int depth) +{ + struct nvkm_subdev *subdev = &vmm->mmu->subdev; + struct nvkm_device *device = subdev->device; + u32 type = depth << 24; /*XXX: not confirmed */ + + type = 0x00000001; /* PAGE_ALL */ + if (atomic_read(&vmm->engref[NVKM_SUBDEV_BAR])) + type |= 0x00000004; /* HUB_ONLY */ + + mutex_lock(&subdev->mutex); + + nvkm_wr32(device, 0xb830a0, vmm->pd->pt[0]->addr >> 8); + nvkm_wr32(device, 0xb830a4, 0x00000000); + nvkm_wr32(device, 0x100e68, 0x00000000); + nvkm_wr32(device, 0xb830b0, 0x80000000 | type); + + nvkm_msec(device, 2000, + if (!(nvkm_rd32(device, 0xb830b0) & 0x80000000)) + break; + ); + + mutex_unlock(&subdev->mutex); +} + +static const struct nvkm_vmm_func +tu102_vmm = { + .join = gv100_vmm_join, + .part = gf100_vmm_part, + .aper = gf100_vmm_aper, + .valid = gp100_vmm_valid, + .flush = tu102_vmm_flush, + .mthd = gp100_vmm_mthd, + .page = { + { 47, &gp100_vmm_desc_16[4], NVKM_VMM_PAGE_Sxxx }, + { 38, &gp100_vmm_desc_16[3], NVKM_VMM_PAGE_Sxxx }, + { 29, &gp100_vmm_desc_16[2], NVKM_VMM_PAGE_Sxxx }, + { 21, &gp100_vmm_desc_16[1], NVKM_VMM_PAGE_SVxC }, + { 16, &gp100_vmm_desc_16[0], NVKM_VMM_PAGE_SVxC }, + { 12, &gp100_vmm_desc_12[0], NVKM_VMM_PAGE_SVHx }, + {} + } +}; + +int +tu102_vmm_new(struct nvkm_mmu *mmu, bool managed, u64 addr, u64 size, + void *argv, u32 argc, struct lock_class_key *key, + const char *name, struct nvkm_vmm **pvmm) +{ + return gp100_vmm_new_(&tu102_vmm, mmu, managed, addr, size, + argv, argc, key, name, pvmm); +} diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/memx.c b/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/memx.c index 11b28b086a06..7b052879af72 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/memx.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/memx.c @@ -88,10 +88,10 @@ nvkm_memx_fini(struct nvkm_memx **pmemx, bool exec) if (exec) { nvkm_pmu_send(pmu, reply, PROC_MEMX, MEMX_MSG_EXEC, memx->base, finish); + nvkm_debug(subdev, "Exec took %uns, PMU_IN %08x\n", + reply[0], reply[1]); } - nvkm_debug(subdev, "Exec took %uns, PMU_IN %08x\n", - reply[0], reply[1]); kfree(memx); return 0; } diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/secboot/acr_r352.c b/drivers/gpu/drm/nouveau/nvkm/subdev/secboot/acr_r352.c index 5c14d6ac855d..1df09ed6fe6d 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/secboot/acr_r352.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/secboot/acr_r352.c @@ -853,7 +853,7 @@ acr_r352_shutdown(struct acr_r352 *acr, struct nvkm_secboot *sb) * and the expected behavior on RM as well */ if (ret && ret != 0x1d) { - nvkm_error(subdev, "HS unload failed, ret 0x%08x", ret); + nvkm_error(subdev, "HS unload failed, ret 0x%08x\n", ret); return -EINVAL; } nvkm_debug(subdev, "HS unload blob completed\n"); @@ -922,7 +922,7 @@ acr_r352_bootstrap(struct acr_r352 *acr, struct nvkm_secboot *sb) if (ret < 0) { return ret; } else if (ret > 0) { - nvkm_error(subdev, "HS load failed, ret 0x%08x", ret); + nvkm_error(subdev, "HS load failed, ret 0x%08x\n", ret); return -EINVAL; } nvkm_debug(subdev, "HS load blob completed\n"); diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/secboot/gp102.c b/drivers/gpu/drm/nouveau/nvkm/subdev/secboot/gp102.c index 1f7a3c1a7f50..84a2f243ed9b 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/secboot/gp102.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/secboot/gp102.c @@ -59,10 +59,10 @@ gp102_run_secure_scrub(struct nvkm_secboot *sb) nvkm_debug(subdev, "running VPR scrubber binary on NVDEC...\n"); - engine = nvkm_engine_ref(&device->nvdec->engine); + engine = nvkm_engine_ref(&device->nvdec[0]->engine); if (IS_ERR(engine)) return PTR_ERR(engine); - falcon = device->nvdec->falcon; + falcon = device->nvdec[0]->falcon; nvkm_falcon_get(falcon, &sb->subdev); diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/therm/base.c b/drivers/gpu/drm/nouveau/nvkm/subdev/therm/base.c index 3695cde669f8..07914e36939e 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/therm/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/therm/base.c @@ -132,11 +132,12 @@ nvkm_therm_update(struct nvkm_therm *therm, int mode) duty = nvkm_therm_update_linear(therm); break; case NVBIOS_THERM_FAN_OTHER: - if (therm->cstate) + if (therm->cstate) { duty = therm->cstate; - else + poll = false; + } else { duty = nvkm_therm_update_linear_fallback(therm); - poll = false; + } break; } immd = false; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/timer/base.c b/drivers/gpu/drm/nouveau/nvkm/subdev/timer/base.c index 36de23d12ae4..dd922033628c 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/timer/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/timer/base.c @@ -23,6 +23,42 @@ */ #include "priv.h" +s64 +nvkm_timer_wait_test(struct nvkm_timer_wait *wait) +{ + struct nvkm_subdev *subdev = &wait->tmr->subdev; + u64 time = nvkm_timer_read(wait->tmr); + + if (wait->reads == 0) { + wait->time0 = time; + wait->time1 = time; + } + + if (wait->time1 == time) { + if (wait->reads++ == 16) { + nvkm_fatal(subdev, "stalled at %016llx\n", time); + return -ETIMEDOUT; + } + } else { + wait->time1 = time; + wait->reads = 1; + } + + if (wait->time1 - wait->time0 > wait->limit) + return -ETIMEDOUT; + + return wait->time1 - wait->time0; +} + +void +nvkm_timer_wait_init(struct nvkm_device *device, u64 nsec, + struct nvkm_timer_wait *wait) +{ + wait->tmr = device->timer; + wait->limit = nsec; + wait->reads = 0; +} + u64 nvkm_timer_read(struct nvkm_timer *tmr) { diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/top/base.c b/drivers/gpu/drm/nouveau/nvkm/subdev/top/base.c index 67ada1d9a28c..cce6e4e90ebf 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/top/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/top/base.c @@ -41,6 +41,22 @@ nvkm_top_device_new(struct nvkm_top *top) } u32 +nvkm_top_addr(struct nvkm_device *device, enum nvkm_devidx index) +{ + struct nvkm_top *top = device->top; + struct nvkm_top_device *info; + + if (top) { + list_for_each_entry(info, &top->device, head) { + if (info->index == index) + return info->addr; + } + } + + return 0; +} + +u32 nvkm_top_reset(struct nvkm_device *device, enum nvkm_devidx index) { struct nvkm_top *top = device->top; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/top/gk104.c b/drivers/gpu/drm/nouveau/nvkm/subdev/top/gk104.c index 4f1f3e890650..e01746ce9fc4 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/top/gk104.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/top/gk104.c @@ -73,6 +73,7 @@ gk104_top_oneinit(struct nvkm_top *top) #define A_(A) if (inst == 0) info->index = NVKM_ENGINE_##A #define B_(A) if (inst + NVKM_ENGINE_##A##0 < NVKM_ENGINE_##A##_LAST + 1) \ info->index = NVKM_ENGINE_##A##0 + inst +#define C_(A) if (inst == 0) info->index = NVKM_SUBDEV_##A switch (type) { case 0x00000000: A_(GR ); break; case 0x00000001: A_(CE0 ); break; @@ -86,8 +87,9 @@ gk104_top_oneinit(struct nvkm_top *top) case 0x0000000d: A_(SEC2 ); break; case 0x0000000e: B_(NVENC ); break; case 0x0000000f: A_(NVENC1); break; - case 0x00000010: A_(NVDEC ); break; + case 0x00000010: B_(NVDEC ); break; case 0x00000013: B_(CE ); break; + case 0x00000014: C_(GSP ); break; break; default: break; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/volt/Kbuild b/drivers/gpu/drm/nouveau/nvkm/subdev/volt/Kbuild index bcd179ba11d0..146adcdd316a 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/volt/Kbuild +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/volt/Kbuild @@ -2,6 +2,7 @@ nvkm-y += nvkm/subdev/volt/base.o nvkm-y += nvkm/subdev/volt/gpio.o nvkm-y += nvkm/subdev/volt/nv40.o nvkm-y += nvkm/subdev/volt/gf100.o +nvkm-y += nvkm/subdev/volt/gf117.o nvkm-y += nvkm/subdev/volt/gk104.o nvkm-y += nvkm/subdev/volt/gk20a.o nvkm-y += nvkm/subdev/volt/gm20b.o diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/volt/gf117.c b/drivers/gpu/drm/nouveau/nvkm/subdev/volt/gf117.c new file mode 100644 index 000000000000..547a58f0aeac --- /dev/null +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/volt/gf117.c @@ -0,0 +1,60 @@ +/* + * Copyright 2019 Ilia Mirkin + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * Authors: Ilia Mirkin + */ +#include "priv.h" + +#include <subdev/fuse.h> + +static int +gf117_volt_speedo_read(struct nvkm_volt *volt) +{ + struct nvkm_device *device = volt->subdev.device; + struct nvkm_fuse *fuse = device->fuse; + + if (!fuse) + return -EINVAL; + + return nvkm_fuse_read(fuse, 0x3a8); +} + +static const struct nvkm_volt_func +gf117_volt = { + .oneinit = gf100_volt_oneinit, + .vid_get = nvkm_voltgpio_get, + .vid_set = nvkm_voltgpio_set, + .speedo_read = gf117_volt_speedo_read, +}; + +int +gf117_volt_new(struct nvkm_device *device, int index, struct nvkm_volt **pvolt) +{ + struct nvkm_volt *volt; + int ret; + + ret = nvkm_volt_new_(&gf117_volt, device, index, &volt); + *pvolt = volt; + if (ret) + return ret; + + return nvkm_voltgpio_init(volt); +} |