diff options
Diffstat (limited to 'drivers/gpu/drm/nouveau/nvkm')
112 files changed, 3256 insertions, 489 deletions
diff --git a/drivers/gpu/drm/nouveau/nvkm/core/subdev.c b/drivers/gpu/drm/nouveau/nvkm/core/subdev.c index 03f676c18aad..245990de1e90 100644 --- a/drivers/gpu/drm/nouveau/nvkm/core/subdev.c +++ b/drivers/gpu/drm/nouveau/nvkm/core/subdev.c @@ -39,6 +39,7 @@ nvkm_subdev_name[NVKM_SUBDEV_NR] = { [NVKM_SUBDEV_FB ] = "fb", [NVKM_SUBDEV_FUSE ] = "fuse", [NVKM_SUBDEV_GPIO ] = "gpio", + [NVKM_SUBDEV_GSP ] = "gsp", [NVKM_SUBDEV_I2C ] = "i2c", [NVKM_SUBDEV_IBUS ] = "priv", [NVKM_SUBDEV_ICCSENSE] = "iccsense", @@ -79,7 +80,9 @@ nvkm_subdev_name[NVKM_SUBDEV_NR] = { [NVKM_ENGINE_NVENC0 ] = "nvenc0", [NVKM_ENGINE_NVENC1 ] = "nvenc1", [NVKM_ENGINE_NVENC2 ] = "nvenc2", - [NVKM_ENGINE_NVDEC ] = "nvdec", + [NVKM_ENGINE_NVDEC0 ] = "nvdec0", + [NVKM_ENGINE_NVDEC1 ] = "nvdec1", + [NVKM_ENGINE_NVDEC2 ] = "nvdec2", [NVKM_ENGINE_PM ] = "pm", [NVKM_ENGINE_SEC ] = "sec", [NVKM_ENGINE_SEC2 ] = "sec2", diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/ce/Kbuild b/drivers/gpu/drm/nouveau/nvkm/engine/ce/Kbuild index 80d784441904..9211663239af 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/ce/Kbuild +++ b/drivers/gpu/drm/nouveau/nvkm/engine/ce/Kbuild @@ -6,3 +6,4 @@ nvkm-y += nvkm/engine/ce/gm200.o nvkm-y += nvkm/engine/ce/gp100.o nvkm-y += nvkm/engine/ce/gp102.o nvkm-y += nvkm/engine/ce/gv100.o +nvkm-y += nvkm/engine/ce/tu102.o diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/ce/tu102.c b/drivers/gpu/drm/nouveau/nvkm/engine/ce/tu102.c new file mode 100644 index 000000000000..b4308e2d8c75 --- /dev/null +++ b/drivers/gpu/drm/nouveau/nvkm/engine/ce/tu102.c @@ -0,0 +1,40 @@ +/* + * Copyright 2018 Red Hat Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ +#include "priv.h" + +#include <nvif/class.h> + +static const struct nvkm_engine_func +tu102_ce = { + .intr = gp100_ce_intr, + .sclass = { + { -1, -1, TURING_DMA_COPY_A }, + {} + } +}; + +int +tu102_ce_new(struct nvkm_device *device, int index, + struct nvkm_engine **pengine) +{ + return nvkm_engine_new_(&tu102_ce, device, index, true, pengine); +} diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/device/base.c b/drivers/gpu/drm/nouveau/nvkm/engine/device/base.c index e294013426ce..7971096b6767 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/device/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/device/base.c @@ -1613,7 +1613,7 @@ nvd7_chipset = { .pci = gf106_pci_new, .therm = gf119_therm_new, .timer = nv41_timer_new, - .volt = gf100_volt_new, + .volt = gf117_volt_new, .ce[0] = gf100_ce_new, .disp = gf119_disp_new, .dma = gf119_dma_new, @@ -2221,7 +2221,7 @@ nv132_chipset = { .dma = gf119_dma_new, .fifo = gp100_fifo_new, .gr = gp102_gr_new, - .nvdec = gp102_nvdec_new, + .nvdec[0] = gp102_nvdec_new, .sec2 = gp102_sec2_new, .sw = gf100_sw_new, }; @@ -2257,7 +2257,7 @@ nv134_chipset = { .dma = gf119_dma_new, .fifo = gp100_fifo_new, .gr = gp104_gr_new, - .nvdec = gp102_nvdec_new, + .nvdec[0] = gp102_nvdec_new, .sec2 = gp102_sec2_new, .sw = gf100_sw_new, }; @@ -2293,7 +2293,7 @@ nv136_chipset = { .dma = gf119_dma_new, .fifo = gp100_fifo_new, .gr = gp104_gr_new, - .nvdec = gp102_nvdec_new, + .nvdec[0] = gp102_nvdec_new, .sec2 = gp102_sec2_new, .sw = gf100_sw_new, }; @@ -2329,7 +2329,7 @@ nv137_chipset = { .dma = gf119_dma_new, .fifo = gp100_fifo_new, .gr = gp107_gr_new, - .nvdec = gp102_nvdec_new, + .nvdec[0] = gp102_nvdec_new, .sec2 = gp102_sec2_new, .sw = gf100_sw_new, }; @@ -2365,7 +2365,7 @@ nv138_chipset = { .dma = gf119_dma_new, .fifo = gp100_fifo_new, .gr = gp107_gr_new, - .nvdec = gp102_nvdec_new, + .nvdec[0] = gp102_nvdec_new, .sec2 = gp102_sec2_new, .sw = gf100_sw_new, }; @@ -2405,6 +2405,7 @@ nv140_chipset = { .fb = gv100_fb_new, .fuse = gm107_fuse_new, .gpio = gk104_gpio_new, + .gsp = gv100_gsp_new, .i2c = gm200_i2c_new, .ibus = gm200_ibus_new, .imem = nv50_instmem_new, @@ -2430,10 +2431,115 @@ nv140_chipset = { .dma = gv100_dma_new, .fifo = gv100_fifo_new, .gr = gv100_gr_new, - .nvdec = gp102_nvdec_new, + .nvdec[0] = gp102_nvdec_new, .sec2 = gp102_sec2_new, }; +static const struct nvkm_device_chip +nv162_chipset = { + .name = "TU102", + .bar = tu102_bar_new, + .bios = nvkm_bios_new, + .bus = gf100_bus_new, + .devinit = tu102_devinit_new, + .fault = tu102_fault_new, + .fb = gv100_fb_new, + .fuse = gm107_fuse_new, + .gpio = gk104_gpio_new, + .gsp = gv100_gsp_new, + .i2c = gm200_i2c_new, + .ibus = gm200_ibus_new, + .imem = nv50_instmem_new, + .ltc = gp102_ltc_new, + .mc = tu102_mc_new, + .mmu = tu102_mmu_new, + .pci = gp100_pci_new, + .pmu = gp102_pmu_new, + .therm = gp100_therm_new, + .timer = gk20a_timer_new, + .top = gk104_top_new, + .ce[0] = tu102_ce_new, + .ce[1] = tu102_ce_new, + .ce[2] = tu102_ce_new, + .ce[3] = tu102_ce_new, + .ce[4] = tu102_ce_new, + .disp = tu102_disp_new, + .dma = gv100_dma_new, + .fifo = tu102_fifo_new, + .nvdec[0] = gp102_nvdec_new, + .sec2 = tu102_sec2_new, +}; + +static const struct nvkm_device_chip +nv164_chipset = { + .name = "TU104", + .bar = tu102_bar_new, + .bios = nvkm_bios_new, + .bus = gf100_bus_new, + .devinit = tu102_devinit_new, + .fault = tu102_fault_new, + .fb = gv100_fb_new, + .fuse = gm107_fuse_new, + .gpio = gk104_gpio_new, + .gsp = gv100_gsp_new, + .i2c = gm200_i2c_new, + .ibus = gm200_ibus_new, + .imem = nv50_instmem_new, + .ltc = gp102_ltc_new, + .mc = tu102_mc_new, + .mmu = tu102_mmu_new, + .pci = gp100_pci_new, + .pmu = gp102_pmu_new, + .therm = gp100_therm_new, + .timer = gk20a_timer_new, + .top = gk104_top_new, + .ce[0] = tu102_ce_new, + .ce[1] = tu102_ce_new, + .ce[2] = tu102_ce_new, + .ce[3] = tu102_ce_new, + .ce[4] = tu102_ce_new, + .disp = tu102_disp_new, + .dma = gv100_dma_new, + .fifo = tu102_fifo_new, + .nvdec[0] = gp102_nvdec_new, + .sec2 = tu102_sec2_new, +}; + +static const struct nvkm_device_chip +nv166_chipset = { + .name = "TU106", + .bar = tu102_bar_new, + .bios = nvkm_bios_new, + .bus = gf100_bus_new, + .devinit = tu102_devinit_new, + .fault = tu102_fault_new, + .fb = gv100_fb_new, + .fuse = gm107_fuse_new, + .gpio = gk104_gpio_new, + .gsp = gv100_gsp_new, + .i2c = gm200_i2c_new, + .ibus = gm200_ibus_new, + .imem = nv50_instmem_new, + .ltc = gp102_ltc_new, + .mc = tu102_mc_new, + .mmu = tu102_mmu_new, + .pci = gp100_pci_new, + .pmu = gp102_pmu_new, + .therm = gp100_therm_new, + .timer = gk20a_timer_new, + .top = gk104_top_new, + .ce[0] = tu102_ce_new, + .ce[1] = tu102_ce_new, + .ce[2] = tu102_ce_new, + .ce[3] = tu102_ce_new, + .ce[4] = tu102_ce_new, + .disp = tu102_disp_new, + .dma = gv100_dma_new, + .fifo = tu102_fifo_new, + .nvdec[0] = gp102_nvdec_new, + .sec2 = tu102_sec2_new, +}; + static int nvkm_device_event_ctor(struct nvkm_object *object, void *data, u32 size, struct nvkm_notify *notify) @@ -2471,6 +2577,7 @@ nvkm_device_subdev(struct nvkm_device *device, int index) _(FB , device->fb , &device->fb->subdev); _(FUSE , device->fuse , &device->fuse->subdev); _(GPIO , device->gpio , &device->gpio->subdev); + _(GSP , device->gsp , &device->gsp->subdev); _(I2C , device->i2c , &device->i2c->subdev); _(IBUS , device->ibus , device->ibus); _(ICCSENSE, device->iccsense, &device->iccsense->subdev); @@ -2529,7 +2636,9 @@ nvkm_device_engine(struct nvkm_device *device, int index) _(NVENC0 , device->nvenc[0], device->nvenc[0]); _(NVENC1 , device->nvenc[1], device->nvenc[1]); _(NVENC2 , device->nvenc[2], device->nvenc[2]); - _(NVDEC , device->nvdec , &device->nvdec->engine); + _(NVDEC0 , device->nvdec[0], &device->nvdec[0]->engine); + _(NVDEC1 , device->nvdec[1], &device->nvdec[1]->engine); + _(NVDEC2 , device->nvdec[2], &device->nvdec[2]->engine); _(PM , device->pm , &device->pm->engine); _(SEC , device->sec , device->sec); _(SEC2 , device->sec2 , &device->sec2->engine); @@ -2791,6 +2900,7 @@ nvkm_device_ctor(const struct nvkm_device_func *func, case 0x120: device->card_type = GM100; break; case 0x130: device->card_type = GP100; break; case 0x140: device->card_type = GV100; break; + case 0x160: device->card_type = TU100; break; default: break; } @@ -2883,6 +2993,9 @@ nvkm_device_ctor(const struct nvkm_device_func *func, case 0x138: device->chip = &nv138_chipset; break; case 0x13b: device->chip = &nv13b_chipset; break; case 0x140: device->chip = &nv140_chipset; break; + case 0x162: device->chip = &nv162_chipset; break; + case 0x164: device->chip = &nv164_chipset; break; + case 0x166: device->chip = &nv166_chipset; break; default: nvdev_error(device, "unknown chipset (%08x)\n", boot0); goto done; @@ -2948,6 +3061,7 @@ nvkm_device_ctor(const struct nvkm_device_func *func, _(NVKM_SUBDEV_FB , fb); _(NVKM_SUBDEV_FUSE , fuse); _(NVKM_SUBDEV_GPIO , gpio); + _(NVKM_SUBDEV_GSP , gsp); _(NVKM_SUBDEV_I2C , i2c); _(NVKM_SUBDEV_IBUS , ibus); _(NVKM_SUBDEV_ICCSENSE, iccsense); @@ -2988,7 +3102,9 @@ nvkm_device_ctor(const struct nvkm_device_func *func, _(NVKM_ENGINE_NVENC0 , nvenc[0]); _(NVKM_ENGINE_NVENC1 , nvenc[1]); _(NVKM_ENGINE_NVENC2 , nvenc[2]); - _(NVKM_ENGINE_NVDEC , nvdec); + _(NVKM_ENGINE_NVDEC0 , nvdec[0]); + _(NVKM_ENGINE_NVDEC1 , nvdec[1]); + _(NVKM_ENGINE_NVDEC2 , nvdec[2]); _(NVKM_ENGINE_PM , pm); _(NVKM_ENGINE_SEC , sec); _(NVKM_ENGINE_SEC2 , sec2); diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/device/priv.h b/drivers/gpu/drm/nouveau/nvkm/engine/device/priv.h index 253ab914a8ef..2a53e37dfa7a 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/device/priv.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/device/priv.h @@ -12,6 +12,7 @@ #include <subdev/fb.h> #include <subdev/fuse.h> #include <subdev/gpio.h> +#include <subdev/gsp.h> #include <subdev/i2c.h> #include <subdev/ibus.h> #include <subdev/iccsense.h> diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/device/user.c b/drivers/gpu/drm/nouveau/nvkm/engine/device/user.c index dde6bbafa709..03c6d9aef075 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/device/user.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/device/user.c @@ -91,7 +91,7 @@ nvkm_udevice_info_v1(struct nvkm_device *device, case ENGINE_A(MSENC ); break; case ENGINE_A(VIC ); break; case ENGINE_A(SEC2 ); break; - case ENGINE_A(NVDEC ); break; + case ENGINE_B(NVDEC ); break; case ENGINE_B(NVENC ); break; default: args->mthd = NV_DEVICE_INFO_INVALID; @@ -175,6 +175,7 @@ nvkm_udevice_info(struct nvkm_udevice *udev, void *data, u32 size) case GM100: args->v0.family = NV_DEVICE_INFO_V0_MAXWELL; break; case GP100: args->v0.family = NV_DEVICE_INFO_V0_PASCAL; break; case GV100: args->v0.family = NV_DEVICE_INFO_V0_VOLTA; break; + case TU100: args->v0.family = NV_DEVICE_INFO_V0_TURING; break; default: args->v0.family = 0; break; @@ -364,16 +365,15 @@ nvkm_udevice_child_get(struct nvkm_object *object, int index, } if (!sclass) { - switch (index) { - case 0: sclass = &nvkm_control_oclass; break; - case 1: - if (!device->mmu) - return -EINVAL; + if (index-- == 0) + sclass = &nvkm_control_oclass; + else if (device->mmu && index-- == 0) sclass = &device->mmu->user; - break; - default: + else if (device->fault && index-- == 0) + sclass = &device->fault->user; + else return -EINVAL; - } + oclass->base = sclass->base; } diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/Kbuild b/drivers/gpu/drm/nouveau/nvkm/engine/disp/Kbuild index 8089ac9a12e2..2c28a5e747cc 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/Kbuild +++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/Kbuild @@ -15,6 +15,7 @@ nvkm-y += nvkm/engine/disp/gm200.o nvkm-y += nvkm/engine/disp/gp100.o nvkm-y += nvkm/engine/disp/gp102.o nvkm-y += nvkm/engine/disp/gv100.o +nvkm-y += nvkm/engine/disp/tu102.o nvkm-y += nvkm/engine/disp/vga.o nvkm-y += nvkm/engine/disp/head.o @@ -38,6 +39,7 @@ nvkm-y += nvkm/engine/disp/sorgk104.o nvkm-y += nvkm/engine/disp/sorgm107.o nvkm-y += nvkm/engine/disp/sorgm200.o nvkm-y += nvkm/engine/disp/sorgv100.o +nvkm-y += nvkm/engine/disp/sortu102.o nvkm-y += nvkm/engine/disp/outp.o nvkm-y += nvkm/engine/disp/dp.o @@ -69,6 +71,7 @@ nvkm-y += nvkm/engine/disp/rootgm200.o nvkm-y += nvkm/engine/disp/rootgp100.o nvkm-y += nvkm/engine/disp/rootgp102.o nvkm-y += nvkm/engine/disp/rootgv100.o +nvkm-y += nvkm/engine/disp/roottu102.o nvkm-y += nvkm/engine/disp/channv50.o nvkm-y += nvkm/engine/disp/changf119.o diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/gf119.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/gf119.c index 794e90982641..e675d9b9d5d7 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/gf119.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/gf119.c @@ -91,15 +91,21 @@ gf119_disp_intr_error(struct nv50_disp *disp, int chid) { struct nvkm_subdev *subdev = &disp->base.engine.subdev; struct nvkm_device *device = subdev->device; - u32 mthd = nvkm_rd32(device, 0x6101f0 + (chid * 12)); + u32 stat = nvkm_rd32(device, 0x6101f0 + (chid * 12)); + u32 type = (stat & 0x00007000) >> 12; + u32 mthd = (stat & 0x00000ffc); u32 data = nvkm_rd32(device, 0x6101f4 + (chid * 12)); - u32 unkn = nvkm_rd32(device, 0x6101f8 + (chid * 12)); + u32 code = nvkm_rd32(device, 0x6101f8 + (chid * 12)); + const struct nvkm_enum *reason = + nvkm_enum_find(nv50_disp_intr_error_type, type); - nvkm_error(subdev, "chid %d mthd %04x data %08x %08x %08x\n", - chid, (mthd & 0x0000ffc), data, mthd, unkn); + nvkm_error(subdev, "chid %d stat %08x reason %d [%s] mthd %04x " + "data %08x code %08x\n", + chid, stat, type, reason ? reason->name : "", + mthd, data, code); if (chid < ARRAY_SIZE(disp->chan)) { - switch (mthd & 0xffc) { + switch (mthd) { case 0x0080: nv50_disp_chan_mthd(disp->chan[chid], NV_DBG_ERROR); break; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/gv100.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/gv100.c index d0a7e3456da1..892be6c9b76c 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/gv100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/gv100.c @@ -28,7 +28,7 @@ #include <core/gpuobj.h> #include <subdev/timer.h> -static int +int gv100_disp_wndw_cnt(struct nvkm_disp *disp, unsigned long *pmask) { struct nvkm_device *device = disp->engine.subdev.device; @@ -36,7 +36,7 @@ gv100_disp_wndw_cnt(struct nvkm_disp *disp, unsigned long *pmask) return (nvkm_rd32(device, 0x610074) & 0x03f00000) >> 20; } -static void +void gv100_disp_super(struct work_struct *work) { struct nv50_disp *disp = @@ -103,10 +103,13 @@ gv100_disp_exception(struct nv50_disp *disp, int chid) u32 mthd = (stat & 0x00000fff) << 2; u32 data = nvkm_rd32(device, 0x611024 + (chid * 12)); u32 code = nvkm_rd32(device, 0x611028 + (chid * 12)); + const struct nvkm_enum *reason = + nvkm_enum_find(nv50_disp_intr_error_type, type); - nvkm_error(subdev, "chid %d %08x [type %d mthd %04x] " + nvkm_error(subdev, "chid %d stat %08x reason %d [%s] mthd %04x " "data %08x code %08x\n", - chid, stat, type, mthd, data, code); + chid, stat, type, reason ? reason->name : "", + mthd, data, code); if (chid < ARRAY_SIZE(disp->chan) && disp->chan[chid]) { switch (mthd) { @@ -257,7 +260,7 @@ gv100_disp_intr_head_timing(struct nv50_disp *disp, int head) } } -static void +void gv100_disp_intr(struct nv50_disp *disp) { struct nvkm_subdev *subdev = &disp->base.engine.subdev; @@ -297,7 +300,7 @@ gv100_disp_intr(struct nv50_disp *disp) nvkm_warn(subdev, "intr %08x\n", stat); } -static void +void gv100_disp_fini(struct nv50_disp *disp) { struct nvkm_device *device = disp->base.engine.subdev.device; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/ior.h b/drivers/gpu/drm/nouveau/nvkm/engine/disp/ior.h index 0f0c86c32ec3..1681ddccd298 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/ior.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/ior.h @@ -144,6 +144,11 @@ void gm200_sor_route_set(struct nvkm_outp *, struct nvkm_ior *); int gm200_sor_route_get(struct nvkm_outp *, int *); void gm200_sor_dp_drive(struct nvkm_ior *, int, int, int, int, int); +void gv100_sor_state(struct nvkm_ior *, struct nvkm_ior_state *); +void gv100_sor_dp_audio(struct nvkm_ior *, int, bool); +void gv100_sor_dp_audio_sym(struct nvkm_ior *, int, u16, u32); +void gv100_sor_dp_watermark(struct nvkm_ior *, int, u8); + void g84_hdmi_ctrl(struct nvkm_ior *, int, bool, u8, u8, u8 *, u8 , u8 *, u8); void gt215_hdmi_ctrl(struct nvkm_ior *, int, bool, u8, u8, u8 *, u8 , u8 *, u8); void gf119_hdmi_ctrl(struct nvkm_ior *, int, bool, u8, u8, u8 *, u8 , u8 *, u8); @@ -195,4 +200,6 @@ int gm200_sor_new(struct nvkm_disp *, int); int gv100_sor_cnt(struct nvkm_disp *, unsigned long *); int gv100_sor_new(struct nvkm_disp *, int); + +int tu102_sor_new(struct nvkm_disp *, int); #endif diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/nv50.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/nv50.c index def005dd5fda..e21556bf2cb1 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/nv50.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/nv50.c @@ -28,7 +28,6 @@ #include "rootnv50.h" #include <core/client.h> -#include <core/enum.h> #include <core/ramht.h> #include <subdev/bios.h> #include <subdev/bios/disp.h> @@ -593,12 +592,15 @@ nv50_disp_super(struct work_struct *work) nvkm_wr32(device, 0x610030, 0x80000000); } -static const struct nvkm_enum +const struct nvkm_enum nv50_disp_intr_error_type[] = { - { 3, "ILLEGAL_MTHD" }, - { 4, "INVALID_VALUE" }, + { 0, "NONE" }, + { 1, "PUSHBUFFER_ERR" }, + { 2, "TRAP" }, + { 3, "RESERVED_METHOD" }, + { 4, "INVALID_ARG" }, { 5, "INVALID_STATE" }, - { 7, "INVALID_HANDLE" }, + { 7, "UNRESOLVABLE_HANDLE" }, {} }; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/nv50.h b/drivers/gpu/drm/nouveau/nvkm/engine/disp/nv50.h index 8580382ab248..e5d00f478bb1 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/nv50.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/nv50.h @@ -5,6 +5,8 @@ #include "priv.h" struct nvkm_head; +#include <core/enum.h> + struct nv50_disp { const struct nv50_disp_func *func; struct nvkm_disp base; @@ -71,6 +73,7 @@ int nv50_disp_init(struct nv50_disp *); void nv50_disp_fini(struct nv50_disp *); void nv50_disp_intr(struct nv50_disp *); void nv50_disp_super(struct work_struct *); +extern const struct nvkm_enum nv50_disp_intr_error_type[]; int gf119_disp_init(struct nv50_disp *); void gf119_disp_fini(struct nv50_disp *); @@ -78,6 +81,11 @@ void gf119_disp_intr(struct nv50_disp *); void gf119_disp_super(struct work_struct *); void gf119_disp_intr_error(struct nv50_disp *, int); +void gv100_disp_fini(struct nv50_disp *); +void gv100_disp_intr(struct nv50_disp *); +void gv100_disp_super(struct work_struct *); +int gv100_disp_wndw_cnt(struct nvkm_disp *, unsigned long *); + void nv50_disp_dptmds_war_2(struct nv50_disp *, struct dcb_output *); void nv50_disp_dptmds_war_3(struct nv50_disp *, struct dcb_output *); void nv50_disp_update_sppll1(struct nv50_disp *); diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/rootnv50.h b/drivers/gpu/drm/nouveau/nvkm/engine/disp/rootnv50.h index 6ca4f9184b51..aee9822a7a87 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/rootnv50.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/rootnv50.h @@ -37,4 +37,5 @@ extern const struct nvkm_disp_oclass gm200_disp_root_oclass; extern const struct nvkm_disp_oclass gp100_disp_root_oclass; extern const struct nvkm_disp_oclass gp102_disp_root_oclass; extern const struct nvkm_disp_oclass gv100_disp_root_oclass; +extern const struct nvkm_disp_oclass tu102_disp_root_oclass; #endif diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/roottu102.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/roottu102.c new file mode 100644 index 000000000000..579a5d02308a --- /dev/null +++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/roottu102.c @@ -0,0 +1,52 @@ +/* + * Copyright 2018 Red Hat Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ +#include "rootnv50.h" +#include "channv50.h" + +#include <nvif/class.h> + +static const struct nv50_disp_root_func +tu102_disp_root = { + .user = { + {{0,0,TU102_DISP_CURSOR }, gv100_disp_curs_new }, + {{0,0,TU102_DISP_WINDOW_IMM_CHANNEL_DMA}, gv100_disp_wimm_new }, + {{0,0,TU102_DISP_CORE_CHANNEL_DMA }, gv100_disp_core_new }, + {{0,0,TU102_DISP_WINDOW_CHANNEL_DMA }, gv100_disp_wndw_new }, + {} + }, +}; + +static int +tu102_disp_root_new(struct nvkm_disp *disp, const struct nvkm_oclass *oclass, + void *data, u32 size, struct nvkm_object **pobject) +{ + return nv50_disp_root_new_(&tu102_disp_root, disp, oclass, + data, size, pobject); +} + +const struct nvkm_disp_oclass +tu102_disp_root_oclass = { + .base.oclass = TU102_DISP, + .base.minver = -1, + .base.maxver = -1, + .ctor = tu102_disp_root_new, +}; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/sorgv100.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/sorgv100.c index 8ba881a729ee..b0597ff9a714 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/sorgv100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/sorgv100.c @@ -23,7 +23,7 @@ #include <subdev/timer.h> -static void +void gv100_sor_dp_watermark(struct nvkm_ior *sor, int head, u8 watermark) { struct nvkm_device *device = sor->disp->engine.subdev.device; @@ -31,7 +31,7 @@ gv100_sor_dp_watermark(struct nvkm_ior *sor, int head, u8 watermark) nvkm_mask(device, 0x616550 + hoff, 0x0c00003f, 0x08000000 | watermark); } -static void +void gv100_sor_dp_audio_sym(struct nvkm_ior *sor, int head, u16 h, u32 v) { struct nvkm_device *device = sor->disp->engine.subdev.device; @@ -40,7 +40,7 @@ gv100_sor_dp_audio_sym(struct nvkm_ior *sor, int head, u16 h, u32 v) nvkm_mask(device, 0x61656c + hoff, 0x00ffffff, v); } -static void +void gv100_sor_dp_audio(struct nvkm_ior *sor, int head, bool enable) { struct nvkm_device *device = sor->disp->engine.subdev.device; @@ -54,7 +54,7 @@ gv100_sor_dp_audio(struct nvkm_ior *sor, int head, bool enable) ); } -static void +void gv100_sor_state(struct nvkm_ior *sor, struct nvkm_ior_state *state) { struct nvkm_device *device = sor->disp->engine.subdev.device; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/sortu102.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/sortu102.c new file mode 100644 index 000000000000..d57b73ada89e --- /dev/null +++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/sortu102.c @@ -0,0 +1,97 @@ +/* + * Copyright 2018 Red Hat Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ +#include "ior.h" + +#include <subdev/timer.h> + +static void +tu102_sor_dp_vcpi(struct nvkm_ior *sor, int head, + u8 slot, u8 slot_nr, u16 pbn, u16 aligned) +{ + struct nvkm_device *device = sor->disp->engine.subdev.device; + const u32 hoff = head * 0x800; + + nvkm_mask(device, 0x61657c + hoff, 0xffffffff, (aligned << 16) | pbn); + nvkm_mask(device, 0x616578 + hoff, 0x00003f3f, (slot_nr << 8) | slot); +} + +static int +tu102_sor_dp_links(struct nvkm_ior *sor, struct nvkm_i2c_aux *aux) +{ + struct nvkm_device *device = sor->disp->engine.subdev.device; + const u32 soff = nv50_ior_base(sor); + const u32 loff = nv50_sor_link(sor); + u32 dpctrl = 0x00000000; + u32 clksor = 0x00000000; + + clksor |= sor->dp.bw << 18; + dpctrl |= ((1 << sor->dp.nr) - 1) << 16; + if (sor->dp.mst) + dpctrl |= 0x40000000; + if (sor->dp.ef) + dpctrl |= 0x00004000; + + nvkm_mask(device, 0x612300 + soff, 0x007c0000, clksor); + + /*XXX*/ + nvkm_msec(device, 40, NVKM_DELAY); + nvkm_mask(device, 0x612300 + soff, 0x00030000, 0x00010000); + nvkm_mask(device, 0x61c10c + loff, 0x00000003, 0x00000001); + + nvkm_mask(device, 0x61c10c + loff, 0x401f4000, dpctrl); + return 0; +} + +static const struct nvkm_ior_func +tu102_sor = { + .route = { + .get = gm200_sor_route_get, + .set = gm200_sor_route_set, + }, + .state = gv100_sor_state, + .power = nv50_sor_power, + .clock = gf119_sor_clock, + .hdmi = { + .ctrl = gv100_hdmi_ctrl, + }, + .dp = { + .lanes = { 0, 1, 2, 3 }, + .links = tu102_sor_dp_links, + .power = g94_sor_dp_power, + .pattern = gm107_sor_dp_pattern, + .drive = gm200_sor_dp_drive, + .vcpi = tu102_sor_dp_vcpi, + .audio = gv100_sor_dp_audio, + .audio_sym = gv100_sor_dp_audio_sym, + .watermark = gv100_sor_dp_watermark, + }, + .hda = { + .hpd = gf119_hda_hpd, + .eld = gf119_hda_eld, + }, +}; + +int +tu102_sor_new(struct nvkm_disp *disp, int id) +{ + return nvkm_ior_new_(&tu102_sor, disp, SOR, id); +} diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/tu102.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/tu102.c new file mode 100644 index 000000000000..883ae4151ff8 --- /dev/null +++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/tu102.c @@ -0,0 +1,152 @@ +/* + * Copyright 2018 Red Hat Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ +#include "nv50.h" +#include "head.h" +#include "ior.h" +#include "channv50.h" +#include "rootnv50.h" + +#include <core/gpuobj.h> +#include <subdev/timer.h> + +static int +tu102_disp_init(struct nv50_disp *disp) +{ + struct nvkm_device *device = disp->base.engine.subdev.device; + struct nvkm_head *head; + int i, j; + u32 tmp; + + /* Claim ownership of display. */ + if (nvkm_rd32(device, 0x6254e8) & 0x00000002) { + nvkm_mask(device, 0x6254e8, 0x00000001, 0x00000000); + if (nvkm_msec(device, 2000, + if (!(nvkm_rd32(device, 0x6254e8) & 0x00000002)) + break; + ) < 0) + return -EBUSY; + } + + /* Lock pin capabilities. */ + tmp = 0x00000021; /*XXX*/ + nvkm_wr32(device, 0x640008, tmp); + + /* SOR capabilities. */ + for (i = 0; i < disp->sor.nr; i++) { + tmp = nvkm_rd32(device, 0x61c000 + (i * 0x800)); + nvkm_mask(device, 0x640000, 0x00000100 << i, 0x00000100 << i); + nvkm_wr32(device, 0x640144 + (i * 0x08), tmp); + } + + /* Head capabilities. */ + list_for_each_entry(head, &disp->base.head, head) { + const int id = head->id; + + /* RG. */ + tmp = nvkm_rd32(device, 0x616300 + (id * 0x800)); + nvkm_wr32(device, 0x640048 + (id * 0x020), tmp); + + /* POSTCOMP. */ + for (j = 0; j < 5 * 4; j += 4) { + tmp = nvkm_rd32(device, 0x616140 + (id * 0x800) + j); + nvkm_wr32(device, 0x640680 + (id * 0x20) + j, tmp); + } + } + + /* Window capabilities. */ + for (i = 0; i < disp->wndw.nr; i++) { + nvkm_mask(device, 0x640004, 1 << i, 1 << i); + for (j = 0; j < 6 * 4; j += 4) { + tmp = nvkm_rd32(device, 0x630100 + (i * 0x800) + j); + nvkm_mask(device, 0x640780 + (i * 0x20) + j, 0xffffffff, tmp); + } + nvkm_mask(device, 0x64000c, 0x00000100, 0x00000100); + } + + /* IHUB capabilities. */ + for (i = 0; i < 3; i++) { + tmp = nvkm_rd32(device, 0x62e000 + (i * 0x04)); + nvkm_wr32(device, 0x640010 + (i * 0x04), tmp); + } + + nvkm_mask(device, 0x610078, 0x00000001, 0x00000001); + + /* Setup instance memory. */ + switch (nvkm_memory_target(disp->inst->memory)) { + case NVKM_MEM_TARGET_VRAM: tmp = 0x00000001; break; + case NVKM_MEM_TARGET_NCOH: tmp = 0x00000002; break; + case NVKM_MEM_TARGET_HOST: tmp = 0x00000003; break; + default: + break; + } + nvkm_wr32(device, 0x610010, 0x00000008 | tmp); + nvkm_wr32(device, 0x610014, disp->inst->addr >> 16); + + /* CTRL_DISP: AWAKEN, ERROR, SUPERVISOR[1-3]. */ + nvkm_wr32(device, 0x611cf0, 0x00000187); /* MSK. */ + nvkm_wr32(device, 0x611db0, 0x00000187); /* EN. */ + + /* EXC_OTHER: CURSn, CORE. */ + nvkm_wr32(device, 0x611cec, disp->head.mask << 16 | + 0x00000001); /* MSK. */ + nvkm_wr32(device, 0x611dac, 0x00000000); /* EN. */ + + /* EXC_WINIM. */ + nvkm_wr32(device, 0x611ce8, disp->wndw.mask); /* MSK. */ + nvkm_wr32(device, 0x611da8, 0x00000000); /* EN. */ + + /* EXC_WIN. */ + nvkm_wr32(device, 0x611ce4, disp->wndw.mask); /* MSK. */ + nvkm_wr32(device, 0x611da4, 0x00000000); /* EN. */ + + /* HEAD_TIMING(n): VBLANK. */ + list_for_each_entry(head, &disp->base.head, head) { + const u32 hoff = head->id * 4; + nvkm_wr32(device, 0x611cc0 + hoff, 0x00000004); /* MSK. */ + nvkm_wr32(device, 0x611d80 + hoff, 0x00000000); /* EN. */ + } + + /* OR. */ + nvkm_wr32(device, 0x611cf4, 0x00000000); /* MSK. */ + nvkm_wr32(device, 0x611db4, 0x00000000); /* EN. */ + return 0; +} + +static const struct nv50_disp_func +tu102_disp = { + .init = tu102_disp_init, + .fini = gv100_disp_fini, + .intr = gv100_disp_intr, + .uevent = &gv100_disp_chan_uevent, + .super = gv100_disp_super, + .root = &tu102_disp_root_oclass, + .wndw = { .cnt = gv100_disp_wndw_cnt }, + .head = { .cnt = gv100_head_cnt, .new = gv100_head_new }, + .sor = { .cnt = gv100_sor_cnt, .new = tu102_sor_new }, + .ramht_size = 0x2000, +}; + +int +tu102_disp_new(struct nvkm_device *device, int index, struct nvkm_disp **pdisp) +{ + return nv50_disp_new_(&tu102_disp, device, index, pdisp); +} diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/wndwgv100.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/wndwgv100.c index 98911805aabf..5d3b641dbb14 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/wndwgv100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/wndwgv100.c @@ -118,7 +118,7 @@ gv100_disp_wndw_mthd_base = { const struct nv50_disp_chan_mthd gv100_disp_wndw_mthd = { - .name = "Base", + .name = "Window", .addr = 0x001000, .prev = 0x000800, .data = { diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/falcon.c b/drivers/gpu/drm/nouveau/nvkm/engine/falcon.c index 816ccaedfc73..8675613e142b 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/falcon.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/falcon.c @@ -22,6 +22,7 @@ #include <engine/falcon.h> #include <core/gpuobj.h> +#include <subdev/mc.h> #include <subdev/timer.h> #include <engine/fifo.h> @@ -107,8 +108,10 @@ nvkm_falcon_fini(struct nvkm_engine *engine, bool suspend) } } - nvkm_mask(device, base + 0x048, 0x00000003, 0x00000000); - nvkm_wr32(device, base + 0x014, 0xffffffff); + if (nvkm_mc_enabled(device, engine->subdev.index)) { + nvkm_mask(device, base + 0x048, 0x00000003, 0x00000000); + nvkm_wr32(device, base + 0x014, 0xffffffff); + } return 0; } diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/Kbuild b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/Kbuild index f00408577a6a..05aada541ea5 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/Kbuild +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/Kbuild @@ -16,6 +16,7 @@ nvkm-y += nvkm/engine/fifo/gm20b.o nvkm-y += nvkm/engine/fifo/gp100.o nvkm-y += nvkm/engine/fifo/gp10b.o nvkm-y += nvkm/engine/fifo/gv100.o +nvkm-y += nvkm/engine/fifo/tu102.o nvkm-y += nvkm/engine/fifo/chan.o nvkm-y += nvkm/engine/fifo/channv50.o @@ -33,5 +34,7 @@ nvkm-y += nvkm/engine/fifo/gpfifog84.o nvkm-y += nvkm/engine/fifo/gpfifogf100.o nvkm-y += nvkm/engine/fifo/gpfifogk104.o nvkm-y += nvkm/engine/fifo/gpfifogv100.o +nvkm-y += nvkm/engine/fifo/gpfifotu102.o nvkm-y += nvkm/engine/fifo/usergv100.o +nvkm-y += nvkm/engine/fifo/usertu102.o diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.h index 3ffef236189e..2c7c5afc1ea5 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.h @@ -17,6 +17,7 @@ struct nvkm_fifo_chan_func { bool suspend); int (*object_ctor)(struct nvkm_fifo_chan *, struct nvkm_object *); void (*object_dtor)(struct nvkm_fifo_chan *, int); + u32 (*submit_token)(struct nvkm_fifo_chan *); }; int nvkm_fifo_chan_ctor(const struct nvkm_fifo_chan_func *, struct nvkm_fifo *, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/changk104.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/changk104.h index 8e28ba6b2307..f8557cdfbd81 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/changk104.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/changk104.h @@ -14,6 +14,8 @@ struct gk104_fifo_chan { struct list_head head; bool killed; + struct nvkm_memory *mthd; + struct { struct nvkm_gpuobj *inst; struct nvkm_vma *vma; @@ -36,4 +38,15 @@ int gk104_fifo_gpfifo_kick_locked(struct gk104_fifo_chan *); int gv100_fifo_gpfifo_new(struct gk104_fifo *, const struct nvkm_oclass *, void *data, u32 size, struct nvkm_object **); +int gv100_fifo_gpfifo_new_(const struct nvkm_fifo_chan_func *, + struct gk104_fifo *, u64 *, u16 *, u64, u64, u64, + u64 *, bool, u32 *, const struct nvkm_oclass *, + struct nvkm_object **); +int gv100_fifo_gpfifo_engine_init(struct nvkm_fifo_chan *, + struct nvkm_engine *); +int gv100_fifo_gpfifo_engine_fini(struct nvkm_fifo_chan *, + struct nvkm_engine *, bool); + +int tu102_fifo_gpfifo_new(struct gk104_fifo *, const struct nvkm_oclass *, + void *data, u32 size, struct nvkm_object **); #endif diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c index f69576868164..10a2e7039a75 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c @@ -346,10 +346,10 @@ gf100_fifo_intr_fault(struct gf100_fifo *fifo, int unit) if (eu && eu->data2) { switch (eu->data2) { case NVKM_SUBDEV_BAR: - nvkm_mask(device, 0x001704, 0x00000000, 0x00000000); + nvkm_bar_bar1_reset(device); break; case NVKM_SUBDEV_INSTMEM: - nvkm_mask(device, 0x001714, 0x00000000, 0x00000000); + nvkm_bar_bar2_reset(device); break; case NVKM_ENGINE_IFB: nvkm_mask(device, 0x001718, 0x00000000, 0x00000000); diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c index afccf9721cf0..1053fe796466 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c @@ -149,16 +149,41 @@ gk104_fifo_uevent_init(struct nvkm_fifo *fifo) } void -gk104_fifo_runlist_commit(struct gk104_fifo *fifo, int runl) +gk104_fifo_runlist_commit(struct gk104_fifo *fifo, int runl, + struct nvkm_memory *mem, int nr) +{ + struct nvkm_subdev *subdev = &fifo->base.engine.subdev; + struct nvkm_device *device = subdev->device; + int target; + + switch (nvkm_memory_target(mem)) { + case NVKM_MEM_TARGET_VRAM: target = 0; break; + case NVKM_MEM_TARGET_NCOH: target = 3; break; + default: + WARN_ON(1); + return; + } + + nvkm_wr32(device, 0x002270, (nvkm_memory_addr(mem) >> 12) | + (target << 28)); + nvkm_wr32(device, 0x002274, (runl << 20) | nr); + + if (nvkm_msec(device, 2000, + if (!(nvkm_rd32(device, 0x002284 + (runl * 0x08)) & 0x00100000)) + break; + ) < 0) + nvkm_error(subdev, "runlist %d update timeout\n", runl); +} + +void +gk104_fifo_runlist_update(struct gk104_fifo *fifo, int runl) { const struct gk104_fifo_runlist_func *func = fifo->func->runlist; struct gk104_fifo_chan *chan; struct nvkm_subdev *subdev = &fifo->base.engine.subdev; - struct nvkm_device *device = subdev->device; struct nvkm_memory *mem; struct nvkm_fifo_cgrp *cgrp; int nr = 0; - int target; mutex_lock(&subdev->mutex); mem = fifo->runlist[runl].mem[fifo->runlist[runl].next]; @@ -177,24 +202,7 @@ gk104_fifo_runlist_commit(struct gk104_fifo *fifo, int runl) } nvkm_done(mem); - switch (nvkm_memory_target(mem)) { - case NVKM_MEM_TARGET_VRAM: target = 0; break; - case NVKM_MEM_TARGET_NCOH: target = 3; break; - default: - WARN_ON(1); - goto unlock; - } - - nvkm_wr32(device, 0x002270, (nvkm_memory_addr(mem) >> 12) | - (target << 28)); - nvkm_wr32(device, 0x002274, (runl << 20) | nr); - - if (nvkm_msec(device, 2000, - if (!(nvkm_rd32(device, 0x002284 + (runl * 0x08)) & 0x00100000)) - break; - ) < 0) - nvkm_error(subdev, "runlist %d update timeout\n", runl); -unlock: + func->commit(fifo, runl, mem, nr); mutex_unlock(&subdev->mutex); } @@ -238,6 +246,29 @@ const struct gk104_fifo_runlist_func gk104_fifo_runlist = { .size = 8, .chan = gk104_fifo_runlist_chan, + .commit = gk104_fifo_runlist_commit, +}; + +void +gk104_fifo_pbdma_init(struct gk104_fifo *fifo) +{ + struct nvkm_device *device = fifo->base.engine.subdev.device; + nvkm_wr32(device, 0x000204, (1 << fifo->pbdma_nr) - 1); +} + +int +gk104_fifo_pbdma_nr(struct gk104_fifo *fifo) +{ + struct nvkm_device *device = fifo->base.engine.subdev.device; + /* Determine number of PBDMAs by checking valid enable bits. */ + nvkm_wr32(device, 0x000204, 0xffffffff); + return hweight32(nvkm_rd32(device, 0x000204)); +} + +const struct gk104_fifo_pbdma_func +gk104_fifo_pbdma = { + .nr = gk104_fifo_pbdma_nr, + .init = gk104_fifo_pbdma_init, }; static void @@ -267,7 +298,7 @@ gk104_fifo_recover_work(struct work_struct *w) } for (todo = runm; runl = __ffs(todo), todo; todo &= ~BIT(runl)) - gk104_fifo_runlist_commit(fifo, runl); + gk104_fifo_runlist_update(fifo, runl); nvkm_wr32(device, 0x00262c, runm); nvkm_mask(device, 0x002630, runm, 0x00000000); @@ -456,10 +487,10 @@ gk104_fifo_fault(struct nvkm_fifo *base, struct nvkm_fault_data *info) if (ee && ee->data2) { switch (ee->data2) { case NVKM_SUBDEV_BAR: - nvkm_mask(device, 0x001704, 0x00000000, 0x00000000); + nvkm_bar_bar1_reset(device); break; case NVKM_SUBDEV_INSTMEM: - nvkm_mask(device, 0x001714, 0x00000000, 0x00000000); + nvkm_bar_bar2_reset(device); break; case NVKM_ENGINE_IFB: nvkm_mask(device, 0x001718, 0x00000000, 0x00000000); @@ -904,9 +935,7 @@ gk104_fifo_oneinit(struct nvkm_fifo *base) enum nvkm_devidx engidx; u32 *map; - /* Determine number of PBDMAs by checking valid enable bits. */ - nvkm_wr32(device, 0x000204, 0xffffffff); - fifo->pbdma_nr = hweight32(nvkm_rd32(device, 0x000204)); + fifo->pbdma_nr = fifo->func->pbdma->nr(fifo); nvkm_debug(subdev, "%d PBDMA(s)\n", fifo->pbdma_nr); /* Read PBDMA->runlist(s) mapping from HW. */ @@ -978,7 +1007,7 @@ gk104_fifo_init(struct nvkm_fifo *base) int i; /* Enable PBDMAs. */ - nvkm_wr32(device, 0x000204, (1 << fifo->pbdma_nr) - 1); + fifo->func->pbdma->init(fifo); /* PBDMA[n] */ for (i = 0; i < fifo->pbdma_nr; i++) { @@ -995,8 +1024,8 @@ gk104_fifo_init(struct nvkm_fifo *base) nvkm_wr32(device, 0x002254, 0x10000000 | fifo->user.bar->addr >> 12); - if (fifo->func->init_pbdma_timeout) - fifo->func->init_pbdma_timeout(fifo); + if (fifo->func->pbdma->init_timeout) + fifo->func->pbdma->init_timeout(fifo); nvkm_wr32(device, 0x002100, 0xffffffff); nvkm_wr32(device, 0x002140, 0x7fffffff); @@ -1175,6 +1204,7 @@ gk104_fifo_fault_gpcclient[] = { static const struct gk104_fifo_func gk104_fifo = { + .pbdma = &gk104_fifo_pbdma, .fault.access = gk104_fifo_fault_access, .fault.engine = gk104_fifo_fault_engine, .fault.reason = gk104_fifo_fault_reason, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.h index d295b81e18d6..d4e565658f46 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.h @@ -45,7 +45,11 @@ struct gk104_fifo { }; struct gk104_fifo_func { - void (*init_pbdma_timeout)(struct gk104_fifo *); + const struct gk104_fifo_pbdma_func { + int (*nr)(struct gk104_fifo *); + void (*init)(struct gk104_fifo *); + void (*init_timeout)(struct gk104_fifo *); + } *pbdma; struct { const struct nvkm_enum *access; @@ -61,6 +65,8 @@ struct gk104_fifo_func { struct nvkm_memory *, u32 offset); void (*chan)(struct gk104_fifo_chan *, struct nvkm_memory *, u32 offset); + void (*commit)(struct gk104_fifo *, int runl, + struct nvkm_memory *, int entries); } *runlist; struct gk104_fifo_user_user { @@ -81,8 +87,11 @@ int gk104_fifo_new_(const struct gk104_fifo_func *, struct nvkm_device *, int index, int nr, struct nvkm_fifo **); void gk104_fifo_runlist_insert(struct gk104_fifo *, struct gk104_fifo_chan *); void gk104_fifo_runlist_remove(struct gk104_fifo *, struct gk104_fifo_chan *); -void gk104_fifo_runlist_commit(struct gk104_fifo *, int runl); +void gk104_fifo_runlist_update(struct gk104_fifo *, int runl); +extern const struct gk104_fifo_pbdma_func gk104_fifo_pbdma; +int gk104_fifo_pbdma_nr(struct gk104_fifo *); +void gk104_fifo_pbdma_init(struct gk104_fifo *); extern const struct nvkm_enum gk104_fifo_fault_access[]; extern const struct nvkm_enum gk104_fifo_fault_engine[]; extern const struct nvkm_enum gk104_fifo_fault_reason[]; @@ -91,15 +100,30 @@ extern const struct nvkm_enum gk104_fifo_fault_gpcclient[]; extern const struct gk104_fifo_runlist_func gk104_fifo_runlist; void gk104_fifo_runlist_chan(struct gk104_fifo_chan *, struct nvkm_memory *, u32); +void gk104_fifo_runlist_commit(struct gk104_fifo *, int runl, + struct nvkm_memory *, int); extern const struct gk104_fifo_runlist_func gk110_fifo_runlist; void gk110_fifo_runlist_cgrp(struct nvkm_fifo_cgrp *, struct nvkm_memory *, u32); -void gk208_fifo_init_pbdma_timeout(struct gk104_fifo *); +extern const struct gk104_fifo_pbdma_func gk208_fifo_pbdma; +void gk208_fifo_pbdma_init_timeout(struct gk104_fifo *); extern const struct nvkm_enum gm107_fifo_fault_engine[]; extern const struct gk104_fifo_runlist_func gm107_fifo_runlist; +extern const struct gk104_fifo_pbdma_func gm200_fifo_pbdma; +int gm200_fifo_pbdma_nr(struct gk104_fifo *); + extern const struct nvkm_enum gp100_fifo_fault_engine[]; + +extern const struct nvkm_enum gv100_fifo_fault_access[]; +extern const struct nvkm_enum gv100_fifo_fault_reason[]; +extern const struct nvkm_enum gv100_fifo_fault_hubclient[]; +extern const struct nvkm_enum gv100_fifo_fault_gpcclient[]; +void gv100_fifo_runlist_cgrp(struct nvkm_fifo_cgrp *, + struct nvkm_memory *, u32); +void gv100_fifo_runlist_chan(struct gk104_fifo_chan *, + struct nvkm_memory *, u32); #endif diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk110.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk110.c index ac7655a130fb..8adfa6b182cb 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk110.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk110.c @@ -43,10 +43,12 @@ gk110_fifo_runlist = { .size = 8, .cgrp = gk110_fifo_runlist_cgrp, .chan = gk104_fifo_runlist_chan, + .commit = gk104_fifo_runlist_commit, }; static const struct gk104_fifo_func gk110_fifo = { + .pbdma = &gk104_fifo_pbdma, .fault.access = gk104_fifo_fault_access, .fault.engine = gk104_fifo_fault_engine, .fault.reason = gk104_fifo_fault_reason, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk208.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk208.c index 5ea7e452cc66..9553fb4af601 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk208.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk208.c @@ -27,7 +27,7 @@ #include <nvif/class.h> void -gk208_fifo_init_pbdma_timeout(struct gk104_fifo *fifo) +gk208_fifo_pbdma_init_timeout(struct gk104_fifo *fifo) { struct nvkm_device *device = fifo->base.engine.subdev.device; int i; @@ -36,9 +36,16 @@ gk208_fifo_init_pbdma_timeout(struct gk104_fifo *fifo) nvkm_wr32(device, 0x04012c + (i * 0x2000), 0x0000ffff); } +const struct gk104_fifo_pbdma_func +gk208_fifo_pbdma = { + .nr = gk104_fifo_pbdma_nr, + .init = gk104_fifo_pbdma_init, + .init_timeout = gk208_fifo_pbdma_init_timeout, +}; + static const struct gk104_fifo_func gk208_fifo = { - .init_pbdma_timeout = gk208_fifo_init_pbdma_timeout, + .pbdma = &gk208_fifo_pbdma, .fault.access = gk104_fifo_fault_access, .fault.engine = gk104_fifo_fault_engine, .fault.reason = gk104_fifo_fault_reason, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk20a.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk20a.c index 535a0eb67a5f..a4c6ac3cd6c7 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk20a.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk20a.c @@ -26,7 +26,7 @@ static const struct gk104_fifo_func gk20a_fifo = { - .init_pbdma_timeout = gk208_fifo_init_pbdma_timeout, + .pbdma = &gk208_fifo_pbdma, .fault.access = gk104_fifo_fault_access, .fault.engine = gk104_fifo_fault_engine, .fault.reason = gk104_fifo_fault_reason, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm107.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm107.c index 79ae19b1db67..acf230764cb0 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm107.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm107.c @@ -41,6 +41,7 @@ gm107_fifo_runlist = { .size = 8, .cgrp = gk110_fifo_runlist_cgrp, .chan = gm107_fifo_runlist_chan, + .commit = gk104_fifo_runlist_commit, }; const struct nvkm_enum @@ -68,7 +69,7 @@ gm107_fifo_fault_engine[] = { static const struct gk104_fifo_func gm107_fifo = { - .init_pbdma_timeout = gk208_fifo_init_pbdma_timeout, + .pbdma = &gk208_fifo_pbdma, .fault.access = gk104_fifo_fault_access, .fault.engine = gm107_fifo_fault_engine, .fault.reason = gk104_fifo_fault_reason, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm200.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm200.c index 49565faa854d..b96c1c5d6577 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm200.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm200.c @@ -26,9 +26,23 @@ #include <nvif/class.h> +int +gm200_fifo_pbdma_nr(struct gk104_fifo *fifo) +{ + struct nvkm_device *device = fifo->base.engine.subdev.device; + return nvkm_rd32(device, 0x002004) & 0x000000ff; +} + +const struct gk104_fifo_pbdma_func +gm200_fifo_pbdma = { + .nr = gm200_fifo_pbdma_nr, + .init = gk104_fifo_pbdma_init, + .init_timeout = gk208_fifo_pbdma_init_timeout, +}; + static const struct gk104_fifo_func gm200_fifo = { - .init_pbdma_timeout = gk208_fifo_init_pbdma_timeout, + .pbdma = &gm200_fifo_pbdma, .fault.access = gk104_fifo_fault_access, .fault.engine = gm107_fifo_fault_engine, .fault.reason = gk104_fifo_fault_reason, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm20b.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm20b.c index 46736513bd11..a49539b9e4ec 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm20b.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm20b.c @@ -26,7 +26,7 @@ static const struct gk104_fifo_func gm20b_fifo = { - .init_pbdma_timeout = gk208_fifo_init_pbdma_timeout, + .pbdma = &gm200_fifo_pbdma, .fault.access = gk104_fifo_fault_access, .fault.engine = gm107_fifo_fault_engine, .fault.reason = gk104_fifo_fault_reason, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gp100.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gp100.c index e2f8f9087d7c..54377e0f6a88 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gp100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gp100.c @@ -52,7 +52,7 @@ gp100_fifo_fault_engine[] = { static const struct gk104_fifo_func gp100_fifo = { - .init_pbdma_timeout = gk208_fifo_init_pbdma_timeout, + .pbdma = &gm200_fifo_pbdma, .fault.access = gk104_fifo_fault_access, .fault.engine = gp100_fifo_fault_engine, .fault.reason = gk104_fifo_fault_reason, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gp10b.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gp10b.c index 7733bf7c6545..778ba7e46fb3 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gp10b.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gp10b.c @@ -26,7 +26,7 @@ static const struct gk104_fifo_func gp10b_fifo = { - .init_pbdma_timeout = gk208_fifo_init_pbdma_timeout, + .pbdma = &gm200_fifo_pbdma, .fault.access = gk104_fifo_fault_access, .fault.engine = gp100_fifo_fault_engine, .fault.reason = gk104_fifo_fault_reason, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogk104.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogk104.c index 118b37aea318..728a1edbf98c 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogk104.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogk104.c @@ -85,7 +85,7 @@ gk104_fifo_gpfifo_engine_addr(struct nvkm_engine *engine) case NVKM_ENGINE_MSVLD : return 0x0270; case NVKM_ENGINE_VIC : return 0x0280; case NVKM_ENGINE_MSENC : return 0x0290; - case NVKM_ENGINE_NVDEC : return 0x02100270; + case NVKM_ENGINE_NVDEC0: return 0x02100270; case NVKM_ENGINE_NVENC0: return 0x02100290; case NVKM_ENGINE_NVENC1: return 0x0210; default: @@ -192,7 +192,7 @@ gk104_fifo_gpfifo_fini(struct nvkm_fifo_chan *base) gk104_fifo_runlist_remove(fifo, chan); nvkm_mask(device, 0x800004 + coff, 0x00000800, 0x00000800); gk104_fifo_gpfifo_kick(chan); - gk104_fifo_runlist_commit(fifo, chan->runl); + gk104_fifo_runlist_update(fifo, chan->runl); } nvkm_wr32(device, 0x800000 + coff, 0x00000000); @@ -213,7 +213,7 @@ gk104_fifo_gpfifo_init(struct nvkm_fifo_chan *base) if (list_empty(&chan->head) && !chan->killed) { gk104_fifo_runlist_insert(fifo, chan); nvkm_mask(device, 0x800004 + coff, 0x00000400, 0x00000400); - gk104_fifo_runlist_commit(fifo, chan->runl); + gk104_fifo_runlist_update(fifo, chan->runl); nvkm_mask(device, 0x800004 + coff, 0x00000400, 0x00000400); } } @@ -222,6 +222,7 @@ void * gk104_fifo_gpfifo_dtor(struct nvkm_fifo_chan *base) { struct gk104_fifo_chan *chan = gk104_fifo_chan(base); + nvkm_memory_unref(&chan->mthd); kfree(chan->cgrp); return chan; } @@ -240,7 +241,7 @@ gk104_fifo_gpfifo_func = { static int gk104_fifo_gpfifo_new_(struct gk104_fifo *fifo, u64 *runlists, u16 *chid, - u64 vmm, u64 ioffset, u64 ilength, + u64 vmm, u64 ioffset, u64 ilength, u64 *inst, bool priv, const struct nvkm_oclass *oclass, struct nvkm_object **pobject) { @@ -279,6 +280,7 @@ gk104_fifo_gpfifo_new_(struct gk104_fifo *fifo, u64 *runlists, u16 *chid, return ret; *chid = chan->base.chid; + *inst = chan->base.inst->addr; /* Hack to support GPUs where even individual channels should be * part of a channel group. @@ -315,6 +317,7 @@ gk104_fifo_gpfifo_new_(struct gk104_fifo *fifo, u64 *runlists, u16 *chid, nvkm_wo32(chan->base.inst, 0x94, 0x30000001); nvkm_wo32(chan->base.inst, 0x9c, 0x00000100); nvkm_wo32(chan->base.inst, 0xac, 0x0000001f); + nvkm_wo32(chan->base.inst, 0xe4, priv ? 0x00000020 : 0x00000000); nvkm_wo32(chan->base.inst, 0xe8, chan->base.chid); nvkm_wo32(chan->base.inst, 0xb8, 0xf8000000); nvkm_wo32(chan->base.inst, 0xf8, 0x10003080); /* 0x002310 */ @@ -337,15 +340,19 @@ gk104_fifo_gpfifo_new(struct gk104_fifo *fifo, const struct nvkm_oclass *oclass, if (!(ret = nvif_unpack(ret, &data, &size, args->v0, 0, 0, false))) { nvif_ioctl(parent, "create channel gpfifo vers %d vmm %llx " "ioffset %016llx ilength %08x " - "runlist %016llx\n", + "runlist %016llx priv %d\n", args->v0.version, args->v0.vmm, args->v0.ioffset, - args->v0.ilength, args->v0.runlist); + args->v0.ilength, args->v0.runlist, args->v0.priv); + if (args->v0.priv && !oclass->client->super) + return -EINVAL; return gk104_fifo_gpfifo_new_(fifo, &args->v0.runlist, &args->v0.chid, args->v0.vmm, args->v0.ioffset, args->v0.ilength, + &args->v0.inst, + args->v0.priv, oclass, pobject); } diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogv100.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogv100.c index 9598853ced56..a7462cf59d65 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogv100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogv100.c @@ -25,9 +25,15 @@ #include <core/client.h> #include <core/gpuobj.h> -#include <nvif/cla06f.h> +#include <nvif/clc36f.h> #include <nvif/unpack.h> +static u32 +gv100_fifo_gpfifo_submit_token(struct nvkm_fifo_chan *chan) +{ + return chan->chid; +} + static int gv100_fifo_gpfifo_engine_valid(struct gk104_fifo_chan *chan, bool ce, bool valid) { @@ -56,7 +62,7 @@ gv100_fifo_gpfifo_engine_valid(struct gk104_fifo_chan *chan, bool ce, bool valid return ret; } -static int +int gv100_fifo_gpfifo_engine_fini(struct nvkm_fifo_chan *base, struct nvkm_engine *engine, bool suspend) { @@ -79,7 +85,7 @@ gv100_fifo_gpfifo_engine_fini(struct nvkm_fifo_chan *base, return ret; } -static int +int gv100_fifo_gpfifo_engine_init(struct nvkm_fifo_chan *base, struct nvkm_engine *engine) { @@ -100,8 +106,8 @@ gv100_fifo_gpfifo_engine_init(struct nvkm_fifo_chan *base, return gv100_fifo_gpfifo_engine_valid(chan, false, true); } -const struct nvkm_fifo_chan_func -gv100_fifo_gpfifo_func = { +static const struct nvkm_fifo_chan_func +gv100_fifo_gpfifo = { .dtor = gk104_fifo_gpfifo_dtor, .init = gk104_fifo_gpfifo_init, .fini = gk104_fifo_gpfifo_fini, @@ -110,19 +116,23 @@ gv100_fifo_gpfifo_func = { .engine_dtor = gk104_fifo_gpfifo_engine_dtor, .engine_init = gv100_fifo_gpfifo_engine_init, .engine_fini = gv100_fifo_gpfifo_engine_fini, + .submit_token = gv100_fifo_gpfifo_submit_token, }; -static int -gv100_fifo_gpfifo_new_(struct gk104_fifo *fifo, u64 *runlists, u16 *chid, - u64 vmm, u64 ioffset, u64 ilength, - const struct nvkm_oclass *oclass, +int +gv100_fifo_gpfifo_new_(const struct nvkm_fifo_chan_func *func, + struct gk104_fifo *fifo, u64 *runlists, u16 *chid, + u64 vmm, u64 ioffset, u64 ilength, u64 *inst, bool priv, + u32 *token, const struct nvkm_oclass *oclass, struct nvkm_object **pobject) { + struct nvkm_device *device = fifo->base.engine.subdev.device; struct gk104_fifo_chan *chan; int runlist = ffs(*runlists) -1, ret, i; unsigned long engm; u64 subdevs = 0; - u64 usermem; + u64 usermem, mthd; + u32 size; if (!vmm || runlist < 0 || runlist >= fifo->runlist_nr) return -EINVAL; @@ -142,14 +152,15 @@ gv100_fifo_gpfifo_new_(struct gk104_fifo *fifo, u64 *runlists, u16 *chid, chan->runl = runlist; INIT_LIST_HEAD(&chan->head); - ret = nvkm_fifo_chan_ctor(&gv100_fifo_gpfifo_func, &fifo->base, - 0x1000, 0x1000, true, vmm, 0, subdevs, - 1, fifo->user.bar->addr, 0x200, + ret = nvkm_fifo_chan_ctor(func, &fifo->base, 0x1000, 0x1000, true, vmm, + 0, subdevs, 1, fifo->user.bar->addr, 0x200, oclass, &chan->base); if (ret) return ret; *chid = chan->base.chid; + *inst = chan->base.inst->addr; + *token = chan->base.func->submit_token(&chan->base); /* Hack to support GPUs where even individual channels should be * part of a channel group. @@ -173,6 +184,20 @@ gv100_fifo_gpfifo_new_(struct gk104_fifo *fifo, u64 *runlists, u16 *chid, nvkm_done(fifo->user.mem); usermem = nvkm_memory_addr(fifo->user.mem) + usermem; + /* Allocate fault method buffer (magics come from nvgpu). */ + size = nvkm_rd32(device, 0x104028); /* NV_PCE_PCE_MAP */ + size = 27 * 5 * (((9 + 1 + 3) * hweight32(size)) + 2); + size = roundup(size, PAGE_SIZE); + + ret = nvkm_memory_new(device, NVKM_MEM_TARGET_INST, size, 0x1000, true, + &chan->mthd); + if (ret) + return ret; + + mthd = nvkm_memory_bar2(chan->mthd); + if (mthd == ~0ULL) + return -EFAULT; + /* RAMFC */ nvkm_kmap(chan->base.inst); nvkm_wo32(chan->base.inst, 0x008, lower_32_bits(usermem)); @@ -184,13 +209,13 @@ gv100_fifo_gpfifo_new_(struct gk104_fifo *fifo, u64 *runlists, u16 *chid, (ilength << 16)); nvkm_wo32(chan->base.inst, 0x084, 0x20400000); nvkm_wo32(chan->base.inst, 0x094, 0x30000001); - nvkm_wo32(chan->base.inst, 0x0e4, 0x00000020); + nvkm_wo32(chan->base.inst, 0x0e4, priv ? 0x00000020 : 0x00000000); nvkm_wo32(chan->base.inst, 0x0e8, chan->base.chid); - nvkm_wo32(chan->base.inst, 0x0f4, 0x00001100); + nvkm_wo32(chan->base.inst, 0x0f4, 0x00001000); nvkm_wo32(chan->base.inst, 0x0f8, 0x10003080); nvkm_mo32(chan->base.inst, 0x218, 0x00000000, 0x00000000); - nvkm_wo32(chan->base.inst, 0x220, 0x020a1000); - nvkm_wo32(chan->base.inst, 0x224, 0x00000000); + nvkm_wo32(chan->base.inst, 0x220, lower_32_bits(mthd)); + nvkm_wo32(chan->base.inst, 0x224, upper_32_bits(mthd)); nvkm_done(chan->base.inst); return gv100_fifo_gpfifo_engine_valid(chan, true, true); } @@ -201,7 +226,7 @@ gv100_fifo_gpfifo_new(struct gk104_fifo *fifo, const struct nvkm_oclass *oclass, { struct nvkm_object *parent = oclass->parent; union { - struct kepler_channel_gpfifo_a_v0 v0; + struct volta_channel_gpfifo_a_v0 v0; } *args = data; int ret = -ENOSYS; @@ -209,15 +234,20 @@ gv100_fifo_gpfifo_new(struct gk104_fifo *fifo, const struct nvkm_oclass *oclass, if (!(ret = nvif_unpack(ret, &data, &size, args->v0, 0, 0, false))) { nvif_ioctl(parent, "create channel gpfifo vers %d vmm %llx " "ioffset %016llx ilength %08x " - "runlist %016llx\n", + "runlist %016llx priv %d\n", args->v0.version, args->v0.vmm, args->v0.ioffset, - args->v0.ilength, args->v0.runlist); - return gv100_fifo_gpfifo_new_(fifo, + args->v0.ilength, args->v0.runlist, args->v0.priv); + if (args->v0.priv && !oclass->client->super) + return -EINVAL; + return gv100_fifo_gpfifo_new_(&gv100_fifo_gpfifo, fifo, &args->v0.runlist, &args->v0.chid, args->v0.vmm, args->v0.ioffset, args->v0.ilength, + &args->v0.inst, + args->v0.priv, + &args->v0.token, oclass, pobject); } diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifotu102.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifotu102.c new file mode 100644 index 000000000000..abef7fb6e2d3 --- /dev/null +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifotu102.c @@ -0,0 +1,83 @@ +/* + * Copyright 2018 Red Hat Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ +#include "changk104.h" +#include "cgrp.h" + +#include <core/client.h> +#include <core/gpuobj.h> + +#include <nvif/clc36f.h> +#include <nvif/unpack.h> + +static u32 +tu102_fifo_gpfifo_submit_token(struct nvkm_fifo_chan *base) +{ + struct gk104_fifo_chan *chan = gk104_fifo_chan(base); + return (chan->runl << 16) | chan->base.chid; +} + +static const struct nvkm_fifo_chan_func +tu102_fifo_gpfifo = { + .dtor = gk104_fifo_gpfifo_dtor, + .init = gk104_fifo_gpfifo_init, + .fini = gk104_fifo_gpfifo_fini, + .ntfy = gf100_fifo_chan_ntfy, + .engine_ctor = gk104_fifo_gpfifo_engine_ctor, + .engine_dtor = gk104_fifo_gpfifo_engine_dtor, + .engine_init = gv100_fifo_gpfifo_engine_init, + .engine_fini = gv100_fifo_gpfifo_engine_fini, + .submit_token = tu102_fifo_gpfifo_submit_token, +}; + +int +tu102_fifo_gpfifo_new(struct gk104_fifo *fifo, const struct nvkm_oclass *oclass, + void *data, u32 size, struct nvkm_object **pobject) +{ + struct nvkm_object *parent = oclass->parent; + union { + struct volta_channel_gpfifo_a_v0 v0; + } *args = data; + int ret = -ENOSYS; + + nvif_ioctl(parent, "create channel gpfifo size %d\n", size); + if (!(ret = nvif_unpack(ret, &data, &size, args->v0, 0, 0, false))) { + nvif_ioctl(parent, "create channel gpfifo vers %d vmm %llx " + "ioffset %016llx ilength %08x " + "runlist %016llx priv %d\n", + args->v0.version, args->v0.vmm, args->v0.ioffset, + args->v0.ilength, args->v0.runlist, args->v0.priv); + if (args->v0.priv && !oclass->client->super) + return -EINVAL; + return gv100_fifo_gpfifo_new_(&tu102_fifo_gpfifo, fifo, + &args->v0.runlist, + &args->v0.chid, + args->v0.vmm, + args->v0.ioffset, + args->v0.ilength, + &args->v0.inst, + args->v0.priv, + &args->v0.token, + oclass, pobject); + } + + return ret; +} diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gv100.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gv100.c index 4e1d159c0ae7..6ee1bb32a071 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gv100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gv100.c @@ -28,7 +28,7 @@ #include <nvif/class.h> -static void +void gv100_fifo_runlist_chan(struct gk104_fifo_chan *chan, struct nvkm_memory *memory, u32 offset) { @@ -42,7 +42,7 @@ gv100_fifo_runlist_chan(struct gk104_fifo_chan *chan, nvkm_wo32(memory, offset + 0xc, upper_32_bits(inst)); } -static void +void gv100_fifo_runlist_cgrp(struct nvkm_fifo_cgrp *cgrp, struct nvkm_memory *memory, u32 offset) { @@ -57,9 +57,10 @@ gv100_fifo_runlist = { .size = 16, .cgrp = gv100_fifo_runlist_cgrp, .chan = gv100_fifo_runlist_chan, + .commit = gk104_fifo_runlist_commit, }; -static const struct nvkm_enum +const struct nvkm_enum gv100_fifo_fault_gpcclient[] = { { 0x00, "T1_0" }, { 0x01, "T1_1" }, @@ -161,7 +162,7 @@ gv100_fifo_fault_gpcclient[] = { {} }; -static const struct nvkm_enum +const struct nvkm_enum gv100_fifo_fault_hubclient[] = { { 0x00, "VIP" }, { 0x01, "CE0" }, @@ -223,7 +224,7 @@ gv100_fifo_fault_hubclient[] = { {} }; -static const struct nvkm_enum +const struct nvkm_enum gv100_fifo_fault_reason[] = { { 0x00, "PDE" }, { 0x01, "PDE_SIZE" }, @@ -271,7 +272,7 @@ gv100_fifo_fault_engine[] = { {} }; -static const struct nvkm_enum +const struct nvkm_enum gv100_fifo_fault_access[] = { { 0x0, "VIRT_READ" }, { 0x1, "VIRT_WRITE" }, @@ -287,7 +288,7 @@ gv100_fifo_fault_access[] = { static const struct gk104_fifo_func gv100_fifo = { - .init_pbdma_timeout = gk208_fifo_init_pbdma_timeout, + .pbdma = &gm200_fifo_pbdma, .fault.access = gv100_fifo_fault_access, .fault.engine = gv100_fifo_fault_engine, .fault.reason = gv100_fifo_fault_reason, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/tu102.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/tu102.c new file mode 100644 index 000000000000..005f3e1729b9 --- /dev/null +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/tu102.c @@ -0,0 +1,116 @@ +/* + * Copyright 2018 Red Hat Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ +#include "gk104.h" +#include "cgrp.h" +#include "changk104.h" +#include "user.h" + +#include <core/gpuobj.h> + +#include <nvif/class.h> + +static void +tu102_fifo_runlist_commit(struct gk104_fifo *fifo, int runl, + struct nvkm_memory *mem, int nr) +{ + struct nvkm_device *device = fifo->base.engine.subdev.device; + u64 addr = nvkm_memory_addr(mem); + /*XXX: target? */ + + nvkm_wr32(device, 0x002b00 + (runl * 0x10), lower_32_bits(addr)); + nvkm_wr32(device, 0x002b04 + (runl * 0x10), upper_32_bits(addr)); + nvkm_wr32(device, 0x002b08 + (runl * 0x10), nr); + + /*XXX: how to wait? can you even wait? */ +} + +const struct gk104_fifo_runlist_func +tu102_fifo_runlist = { + .size = 16, + .cgrp = gv100_fifo_runlist_cgrp, + .chan = gv100_fifo_runlist_chan, + .commit = tu102_fifo_runlist_commit, +}; + +static const struct nvkm_enum +tu102_fifo_fault_engine[] = { + { 0x01, "DISPLAY" }, + { 0x03, "PTP" }, + { 0x06, "PWR_PMU" }, + { 0x08, "IFB", NULL, NVKM_ENGINE_IFB }, + { 0x09, "PERF" }, + { 0x1f, "PHYSICAL" }, + { 0x20, "HOST0" }, + { 0x21, "HOST1" }, + { 0x22, "HOST2" }, + { 0x23, "HOST3" }, + { 0x24, "HOST4" }, + { 0x25, "HOST5" }, + { 0x26, "HOST6" }, + { 0x27, "HOST7" }, + { 0x28, "HOST8" }, + { 0x29, "HOST9" }, + { 0x2a, "HOST10" }, + { 0x2b, "HOST11" }, + { 0x2c, "HOST12" }, + { 0x2d, "HOST13" }, + { 0x2e, "HOST14" }, + { 0x80, "BAR1", NULL, NVKM_SUBDEV_BAR }, + { 0xc0, "BAR2", NULL, NVKM_SUBDEV_INSTMEM }, + {} +}; + +static void +tu102_fifo_pbdma_init(struct gk104_fifo *fifo) +{ + struct nvkm_device *device = fifo->base.engine.subdev.device; + const u32 mask = (1 << fifo->pbdma_nr) - 1; + /*XXX: this is a bit of a guess at this point in time. */ + nvkm_mask(device, 0xb65000, 0x80000fff, 0x80000000 | mask); +} + +static const struct gk104_fifo_pbdma_func +tu102_fifo_pbdma = { + .nr = gm200_fifo_pbdma_nr, + .init = tu102_fifo_pbdma_init, + .init_timeout = gk208_fifo_pbdma_init_timeout, +}; + +static const struct gk104_fifo_func +tu102_fifo = { + .pbdma = &tu102_fifo_pbdma, + .fault.access = gv100_fifo_fault_access, + .fault.engine = tu102_fifo_fault_engine, + .fault.reason = gv100_fifo_fault_reason, + .fault.hubclient = gv100_fifo_fault_hubclient, + .fault.gpcclient = gv100_fifo_fault_gpcclient, + .runlist = &tu102_fifo_runlist, + .user = {{-1,-1,VOLTA_USERMODE_A }, tu102_fifo_user_new }, + .chan = {{ 0, 0,TURING_CHANNEL_GPFIFO_A}, tu102_fifo_gpfifo_new }, + .cgrp_force = true, +}; + +int +tu102_fifo_new(struct nvkm_device *device, int index, struct nvkm_fifo **pfifo) +{ + return gk104_fifo_new_(&tu102_fifo, device, index, 4096, pfifo); +} diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/user.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/user.h index ed840921ebe8..54a3a3092cc0 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/user.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/user.h @@ -3,4 +3,6 @@ #include "priv.h" int gv100_fifo_user_new(const struct nvkm_oclass *, void *, u32, struct nvkm_object **); +int tu102_fifo_user_new(const struct nvkm_oclass *, void *, u32, + struct nvkm_object **); #endif diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/usertu102.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/usertu102.c new file mode 100644 index 000000000000..217268f8ccad --- /dev/null +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/usertu102.c @@ -0,0 +1,45 @@ +/* + * Copyright 2018 Red Hat Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ +#include "user.h" + +static int +tu102_fifo_user_map(struct nvkm_object *object, void *argv, u32 argc, + enum nvkm_object_map *type, u64 *addr, u64 *size) +{ + struct nvkm_device *device = object->engine->subdev.device; + *addr = 0xbb0000 + device->func->resource_addr(device, 0); + *size = 0x010000; + *type = NVKM_OBJECT_MAP_IO; + return 0; +} + +static const struct nvkm_object_func +tu102_fifo_user = { + .map = tu102_fifo_user_map, +}; + +int +tu102_fifo_user_new(const struct nvkm_oclass *oclass, void *argv, u32 argc, + struct nvkm_object **pobject) +{ + return nvkm_object_new_(&tu102_fifo_user, oclass, argv, argc, pobject); +} diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/base.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/base.c index cd8cf6f7024c..d41fb94524e9 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/base.c @@ -25,6 +25,33 @@ #include <engine/fifo.h> +u32 +nvkm_gr_ctxsw_inst(struct nvkm_device *device) +{ + struct nvkm_gr *gr = device->gr; + if (gr && gr->func->ctxsw.inst) + return gr->func->ctxsw.inst(gr); + return 0; +} + +int +nvkm_gr_ctxsw_resume(struct nvkm_device *device) +{ + struct nvkm_gr *gr = device->gr; + if (gr && gr->func->ctxsw.resume) + return gr->func->ctxsw.resume(gr); + return 0; +} + +int +nvkm_gr_ctxsw_pause(struct nvkm_device *device) +{ + struct nvkm_gr *gr = device->gr; + if (gr && gr->func->ctxsw.pause) + return gr->func->ctxsw.pause(gr); + return 0; +} + static bool nvkm_gr_chsw_load(struct nvkm_engine *engine) { diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf100.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf100.c index e813a3f8ea93..85f2d1e950e8 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf100.c @@ -1523,13 +1523,9 @@ gf100_grctx_generate(struct gf100_gr *gr) /* Make channel current. */ addr = nvkm_memory_addr(inst) >> 12; if (gr->firmware) { - nvkm_wr32(device, 0x409840, 0x00000030); - nvkm_wr32(device, 0x409500, 0x80000000 | addr); - nvkm_wr32(device, 0x409504, 0x00000003); - nvkm_msec(device, 2000, - if (nvkm_rd32(device, 0x409800) & 0x00000010) - break; - ); + ret = gf100_gr_fecs_bind_pointer(gr, 0x80000000 | addr); + if (ret) + goto done; nvkm_kmap(data); nvkm_wo32(data, 0x1c, 1); diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.c index 70d3d41e616c..81a13cf9a292 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.c @@ -715,6 +715,211 @@ gf100_gr_pack_mmio[] = { * PGRAPH engine/subdev functions ******************************************************************************/ +static u32 +gf100_gr_ctxsw_inst(struct nvkm_gr *gr) +{ + return nvkm_rd32(gr->engine.subdev.device, 0x409b00); +} + +static int +gf100_gr_fecs_ctrl_ctxsw(struct gf100_gr *gr, u32 mthd) +{ + struct nvkm_device *device = gr->base.engine.subdev.device; + + nvkm_wr32(device, 0x409804, 0xffffffff); + nvkm_wr32(device, 0x409840, 0xffffffff); + nvkm_wr32(device, 0x409500, 0xffffffff); + nvkm_wr32(device, 0x409504, mthd); + nvkm_msec(device, 2000, + u32 stat = nvkm_rd32(device, 0x409804); + if (stat == 0x00000002) + return -EIO; + if (stat == 0x00000001) + return 0; + ); + + return -ETIMEDOUT; +} + +int +gf100_gr_fecs_start_ctxsw(struct nvkm_gr *base) +{ + struct gf100_gr *gr = gf100_gr(base); + int ret = 0; + + mutex_lock(&gr->fecs.mutex); + if (!--gr->fecs.disable) { + if (WARN_ON(ret = gf100_gr_fecs_ctrl_ctxsw(gr, 0x39))) + gr->fecs.disable++; + } + mutex_unlock(&gr->fecs.mutex); + return ret; +} + +int +gf100_gr_fecs_stop_ctxsw(struct nvkm_gr *base) +{ + struct gf100_gr *gr = gf100_gr(base); + int ret = 0; + + mutex_lock(&gr->fecs.mutex); + if (!gr->fecs.disable++) { + if (WARN_ON(ret = gf100_gr_fecs_ctrl_ctxsw(gr, 0x38))) + gr->fecs.disable--; + } + mutex_unlock(&gr->fecs.mutex); + return ret; +} + +int +gf100_gr_fecs_bind_pointer(struct gf100_gr *gr, u32 inst) +{ + struct nvkm_device *device = gr->base.engine.subdev.device; + + nvkm_wr32(device, 0x409840, 0x00000030); + nvkm_wr32(device, 0x409500, inst); + nvkm_wr32(device, 0x409504, 0x00000003); + nvkm_msec(device, 2000, + u32 stat = nvkm_rd32(device, 0x409800); + if (stat & 0x00000020) + return -EIO; + if (stat & 0x00000010) + return 0; + ); + + return -ETIMEDOUT; +} + +static int +gf100_gr_fecs_set_reglist_virtual_address(struct gf100_gr *gr, u64 addr) +{ + struct nvkm_device *device = gr->base.engine.subdev.device; + + nvkm_wr32(device, 0x409810, addr >> 8); + nvkm_wr32(device, 0x409800, 0x00000000); + nvkm_wr32(device, 0x409500, 0x00000001); + nvkm_wr32(device, 0x409504, 0x00000032); + nvkm_msec(device, 2000, + if (nvkm_rd32(device, 0x409800) == 0x00000001) + return 0; + ); + + return -ETIMEDOUT; +} + +static int +gf100_gr_fecs_set_reglist_bind_instance(struct gf100_gr *gr, u32 inst) +{ + struct nvkm_device *device = gr->base.engine.subdev.device; + + nvkm_wr32(device, 0x409810, inst); + nvkm_wr32(device, 0x409800, 0x00000000); + nvkm_wr32(device, 0x409500, 0x00000001); + nvkm_wr32(device, 0x409504, 0x00000031); + nvkm_msec(device, 2000, + if (nvkm_rd32(device, 0x409800) == 0x00000001) + return 0; + ); + + return -ETIMEDOUT; +} + +static int +gf100_gr_fecs_discover_reglist_image_size(struct gf100_gr *gr, u32 *psize) +{ + struct nvkm_device *device = gr->base.engine.subdev.device; + + nvkm_wr32(device, 0x409800, 0x00000000); + nvkm_wr32(device, 0x409500, 0x00000001); + nvkm_wr32(device, 0x409504, 0x00000030); + nvkm_msec(device, 2000, + if ((*psize = nvkm_rd32(device, 0x409800))) + return 0; + ); + + return -ETIMEDOUT; +} + +static int +gf100_gr_fecs_elpg_bind(struct gf100_gr *gr) +{ + u32 size; + int ret; + + ret = gf100_gr_fecs_discover_reglist_image_size(gr, &size); + if (ret) + return ret; + + /*XXX: We need to allocate + map the above into PMU's inst block, + * which which means we probably need a proper PMU before we + * even bother. + */ + + ret = gf100_gr_fecs_set_reglist_bind_instance(gr, 0); + if (ret) + return ret; + + return gf100_gr_fecs_set_reglist_virtual_address(gr, 0); +} + +static int +gf100_gr_fecs_discover_pm_image_size(struct gf100_gr *gr, u32 *psize) +{ + struct nvkm_device *device = gr->base.engine.subdev.device; + + nvkm_wr32(device, 0x409840, 0xffffffff); + nvkm_wr32(device, 0x409500, 0x00000000); + nvkm_wr32(device, 0x409504, 0x00000025); + nvkm_msec(device, 2000, + if ((*psize = nvkm_rd32(device, 0x409800))) + return 0; + ); + + return -ETIMEDOUT; +} + +static int +gf100_gr_fecs_discover_zcull_image_size(struct gf100_gr *gr, u32 *psize) +{ + struct nvkm_device *device = gr->base.engine.subdev.device; + + nvkm_wr32(device, 0x409840, 0xffffffff); + nvkm_wr32(device, 0x409500, 0x00000000); + nvkm_wr32(device, 0x409504, 0x00000016); + nvkm_msec(device, 2000, + if ((*psize = nvkm_rd32(device, 0x409800))) + return 0; + ); + + return -ETIMEDOUT; +} + +static int +gf100_gr_fecs_discover_image_size(struct gf100_gr *gr, u32 *psize) +{ + struct nvkm_device *device = gr->base.engine.subdev.device; + + nvkm_wr32(device, 0x409840, 0xffffffff); + nvkm_wr32(device, 0x409500, 0x00000000); + nvkm_wr32(device, 0x409504, 0x00000010); + nvkm_msec(device, 2000, + if ((*psize = nvkm_rd32(device, 0x409800))) + return 0; + ); + + return -ETIMEDOUT; +} + +static void +gf100_gr_fecs_set_watchdog_timeout(struct gf100_gr *gr, u32 timeout) +{ + struct nvkm_device *device = gr->base.engine.subdev.device; + + nvkm_wr32(device, 0x409840, 0xffffffff); + nvkm_wr32(device, 0x409500, timeout); + nvkm_wr32(device, 0x409504, 0x00000021); +} + static bool gf100_gr_chsw_load(struct nvkm_gr *base) { @@ -1487,6 +1692,7 @@ gf100_gr_init_ctxctl_ext(struct gf100_gr *gr) struct nvkm_device *device = subdev->device; struct nvkm_secboot *sb = device->secboot; u32 secboot_mask = 0; + int ret; /* load fuc microcode */ nvkm_mc_unk260(device, 0); @@ -1495,12 +1701,12 @@ gf100_gr_init_ctxctl_ext(struct gf100_gr *gr) if (nvkm_secboot_is_managed(sb, NVKM_SECBOOT_FALCON_FECS)) secboot_mask |= BIT(NVKM_SECBOOT_FALCON_FECS); else - gf100_gr_init_fw(gr->fecs, &gr->fuc409c, &gr->fuc409d); + gf100_gr_init_fw(gr->fecs.falcon, &gr->fuc409c, &gr->fuc409d); if (nvkm_secboot_is_managed(sb, NVKM_SECBOOT_FALCON_GPCCS)) secboot_mask |= BIT(NVKM_SECBOOT_FALCON_GPCCS); else - gf100_gr_init_fw(gr->gpccs, &gr->fuc41ac, &gr->fuc41ad); + gf100_gr_init_fw(gr->gpccs.falcon, &gr->fuc41ac, &gr->fuc41ad); if (secboot_mask != 0) { int ret = nvkm_secboot_reset(sb, secboot_mask); @@ -1515,8 +1721,8 @@ gf100_gr_init_ctxctl_ext(struct gf100_gr *gr) nvkm_wr32(device, 0x41a10c, 0x00000000); nvkm_wr32(device, 0x40910c, 0x00000000); - nvkm_falcon_start(gr->gpccs); - nvkm_falcon_start(gr->fecs); + nvkm_falcon_start(gr->gpccs.falcon); + nvkm_falcon_start(gr->fecs.falcon); if (nvkm_msec(device, 2000, if (nvkm_rd32(device, 0x409800) & 0x00000001) @@ -1524,72 +1730,36 @@ gf100_gr_init_ctxctl_ext(struct gf100_gr *gr) ) < 0) return -EBUSY; - nvkm_wr32(device, 0x409840, 0xffffffff); - nvkm_wr32(device, 0x409500, 0x7fffffff); - nvkm_wr32(device, 0x409504, 0x00000021); + gf100_gr_fecs_set_watchdog_timeout(gr, 0x7fffffff); - nvkm_wr32(device, 0x409840, 0xffffffff); - nvkm_wr32(device, 0x409500, 0x00000000); - nvkm_wr32(device, 0x409504, 0x00000010); - if (nvkm_msec(device, 2000, - if ((gr->size = nvkm_rd32(device, 0x409800))) - break; - ) < 0) - return -EBUSY; - - nvkm_wr32(device, 0x409840, 0xffffffff); - nvkm_wr32(device, 0x409500, 0x00000000); - nvkm_wr32(device, 0x409504, 0x00000016); - if (nvkm_msec(device, 2000, - if (nvkm_rd32(device, 0x409800)) - break; - ) < 0) - return -EBUSY; + /* Determine how much memory is required to store main context image. */ + ret = gf100_gr_fecs_discover_image_size(gr, &gr->size); + if (ret) + return ret; - nvkm_wr32(device, 0x409840, 0xffffffff); - nvkm_wr32(device, 0x409500, 0x00000000); - nvkm_wr32(device, 0x409504, 0x00000025); - if (nvkm_msec(device, 2000, - if (nvkm_rd32(device, 0x409800)) - break; - ) < 0) - return -EBUSY; + /* Determine how much memory is required to store ZCULL image. */ + ret = gf100_gr_fecs_discover_zcull_image_size(gr, &gr->size_zcull); + if (ret) + return ret; - if (device->chipset >= 0xe0) { - nvkm_wr32(device, 0x409800, 0x00000000); - nvkm_wr32(device, 0x409500, 0x00000001); - nvkm_wr32(device, 0x409504, 0x00000030); - if (nvkm_msec(device, 2000, - if (nvkm_rd32(device, 0x409800)) - break; - ) < 0) - return -EBUSY; - - nvkm_wr32(device, 0x409810, 0xb00095c8); - nvkm_wr32(device, 0x409800, 0x00000000); - nvkm_wr32(device, 0x409500, 0x00000001); - nvkm_wr32(device, 0x409504, 0x00000031); - if (nvkm_msec(device, 2000, - if (nvkm_rd32(device, 0x409800)) - break; - ) < 0) - return -EBUSY; - - nvkm_wr32(device, 0x409810, 0x00080420); - nvkm_wr32(device, 0x409800, 0x00000000); - nvkm_wr32(device, 0x409500, 0x00000001); - nvkm_wr32(device, 0x409504, 0x00000032); - if (nvkm_msec(device, 2000, - if (nvkm_rd32(device, 0x409800)) - break; - ) < 0) - return -EBUSY; + /* Determine how much memory is required to store PerfMon image. */ + ret = gf100_gr_fecs_discover_pm_image_size(gr, &gr->size_pm); + if (ret) + return ret; - nvkm_wr32(device, 0x409614, 0x00000070); - nvkm_wr32(device, 0x409614, 0x00000770); - nvkm_wr32(device, 0x40802c, 0x00000001); + /*XXX: We (likely) require PMU support to even bother with this. + * + * Also, it seems like not all GPUs support ELPG. Traces I + * have here show RM enabling it on Kepler/Turing, but none + * of the GPUs between those. NVGPU decides this by PCIID. + */ + if (0) { + ret = gf100_gr_fecs_elpg_bind(gr); + if (ret) + return ret; } + /* Generate golden context image. */ if (gr->data == NULL) { int ret = gf100_grctx_generate(gr); if (ret) { @@ -1614,15 +1784,19 @@ gf100_gr_init_ctxctl_int(struct gf100_gr *gr) /* load HUB microcode */ nvkm_mc_unk260(device, 0); - nvkm_falcon_load_dmem(gr->fecs, gr->func->fecs.ucode->data.data, 0x0, + nvkm_falcon_load_dmem(gr->fecs.falcon, + gr->func->fecs.ucode->data.data, 0x0, gr->func->fecs.ucode->data.size, 0); - nvkm_falcon_load_imem(gr->fecs, gr->func->fecs.ucode->code.data, 0x0, + nvkm_falcon_load_imem(gr->fecs.falcon, + gr->func->fecs.ucode->code.data, 0x0, gr->func->fecs.ucode->code.size, 0, 0, false); /* load GPC microcode */ - nvkm_falcon_load_dmem(gr->gpccs, gr->func->gpccs.ucode->data.data, 0x0, + nvkm_falcon_load_dmem(gr->gpccs.falcon, + gr->func->gpccs.ucode->data.data, 0x0, gr->func->gpccs.ucode->data.size, 0); - nvkm_falcon_load_imem(gr->gpccs, gr->func->gpccs.ucode->code.data, 0x0, + nvkm_falcon_load_imem(gr->gpccs.falcon, + gr->func->gpccs.ucode->code.data, 0x0, gr->func->gpccs.ucode->code.size, 0, 0, false); nvkm_mc_unk260(device, 1); @@ -1769,11 +1943,13 @@ gf100_gr_oneinit(struct nvkm_gr *base) int i, j; int ret; - ret = nvkm_falcon_v1_new(subdev, "FECS", 0x409000, &gr->fecs); + ret = nvkm_falcon_v1_new(subdev, "FECS", 0x409000, &gr->fecs.falcon); if (ret) return ret; - ret = nvkm_falcon_v1_new(subdev, "GPCCS", 0x41a000, &gr->gpccs); + mutex_init(&gr->fecs.mutex); + + ret = nvkm_falcon_v1_new(subdev, "GPCCS", 0x41a000, &gr->gpccs.falcon); if (ret) return ret; @@ -1816,11 +1992,11 @@ gf100_gr_init_(struct nvkm_gr *base) nvkm_pmu_pgob(gr->base.engine.subdev.device->pmu, false); - ret = nvkm_falcon_get(gr->fecs, subdev); + ret = nvkm_falcon_get(gr->fecs.falcon, subdev); if (ret) return ret; - ret = nvkm_falcon_get(gr->gpccs, subdev); + ret = nvkm_falcon_get(gr->gpccs.falcon, subdev); if (ret) return ret; @@ -1832,8 +2008,8 @@ gf100_gr_fini_(struct nvkm_gr *base, bool suspend) { struct gf100_gr *gr = gf100_gr(base); struct nvkm_subdev *subdev = &gr->base.engine.subdev; - nvkm_falcon_put(gr->gpccs, subdev); - nvkm_falcon_put(gr->fecs, subdev); + nvkm_falcon_put(gr->gpccs.falcon, subdev); + nvkm_falcon_put(gr->fecs.falcon, subdev); return 0; } @@ -1859,8 +2035,8 @@ gf100_gr_dtor(struct nvkm_gr *base) gr->func->dtor(gr); kfree(gr->data); - nvkm_falcon_del(&gr->gpccs); - nvkm_falcon_del(&gr->fecs); + nvkm_falcon_del(&gr->gpccs.falcon); + nvkm_falcon_del(&gr->fecs.falcon); gf100_gr_dtor_fw(&gr->fuc409c); gf100_gr_dtor_fw(&gr->fuc409d); @@ -1886,6 +2062,9 @@ gf100_gr_ = { .chan_new = gf100_gr_chan_new, .object_get = gf100_gr_object_get, .chsw_load = gf100_gr_chsw_load, + .ctxsw.pause = gf100_gr_fecs_stop_ctxsw, + .ctxsw.resume = gf100_gr_fecs_start_ctxsw, + .ctxsw.inst = gf100_gr_ctxsw_inst, }; int diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.h b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.h index dc46cf0131db..fafdd0bbea9b 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.h @@ -82,8 +82,16 @@ struct gf100_gr { const struct gf100_gr_func *func; struct nvkm_gr base; - struct nvkm_falcon *fecs; - struct nvkm_falcon *gpccs; + struct { + struct nvkm_falcon *falcon; + struct mutex mutex; + u32 disable; + } fecs; + + struct { + struct nvkm_falcon *falcon; + } gpccs; + struct gf100_gr_fuc fuc409c; struct gf100_gr_fuc fuc409d; struct gf100_gr_fuc fuc41ac; @@ -128,6 +136,8 @@ struct gf100_gr { struct gf100_gr_mmio mmio_list[4096/8]; u32 size; u32 *data; + u32 size_zcull; + u32 size_pm; }; int gf100_gr_ctor(const struct gf100_gr_func *, struct nvkm_device *, @@ -136,6 +146,8 @@ int gf100_gr_new_(const struct gf100_gr_func *, struct nvkm_device *, int, struct nvkm_gr **); void *gf100_gr_dtor(struct nvkm_gr *); +int gf100_gr_fecs_bind_pointer(struct gf100_gr *, u32 inst); + struct gf100_gr_func_zbc { void (*clear_color)(struct gf100_gr *, int zbc); void (*clear_depth)(struct gf100_gr *, int zbc); diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/priv.h b/drivers/gpu/drm/nouveau/nvkm/engine/gr/priv.h index 66359c23cbce..d4d5601c51e7 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/priv.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/priv.h @@ -27,6 +27,11 @@ struct nvkm_gr_func { */ u64 (*units)(struct nvkm_gr *); bool (*chsw_load)(struct nvkm_gr *); + struct { + int (*pause)(struct nvkm_gr *); + int (*resume)(struct nvkm_gr *); + u32 (*inst)(struct nvkm_gr *); + } ctxsw; struct nvkm_sclass sclass[]; }; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/nvdec/base.c b/drivers/gpu/drm/nouveau/nvkm/engine/nvdec/base.c index 4807021fd990..4a63581bdd5e 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/nvdec/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/nvdec/base.c @@ -21,13 +21,21 @@ */ #include "priv.h" +#include <subdev/top.h> #include <engine/falcon.h> static int nvkm_nvdec_oneinit(struct nvkm_engine *engine) { struct nvkm_nvdec *nvdec = nvkm_nvdec(engine); - return nvkm_falcon_v1_new(&nvdec->engine.subdev, "NVDEC", 0x84000, + struct nvkm_subdev *subdev = &nvdec->engine.subdev; + + nvdec->addr = nvkm_top_addr(subdev->device, subdev->index); + if (!nvdec->addr) + return -EINVAL; + + /*XXX: fix naming of this when adding support for multiple-NVDEC */ + return nvkm_falcon_v1_new(subdev, "NVDEC", nvdec->addr, &nvdec->falcon); } diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/sec2/Kbuild b/drivers/gpu/drm/nouveau/nvkm/engine/sec2/Kbuild index 4b17254cfbd0..d9cdea7d9353 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/sec2/Kbuild +++ b/drivers/gpu/drm/nouveau/nvkm/engine/sec2/Kbuild @@ -1,2 +1,3 @@ nvkm-y += nvkm/engine/sec2/base.o nvkm-y += nvkm/engine/sec2/gp102.o +nvkm-y += nvkm/engine/sec2/tu102.o diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/sec2/base.c b/drivers/gpu/drm/nouveau/nvkm/engine/sec2/base.c index f865d2a3e184..1b49e5b6717f 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/sec2/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/sec2/base.c @@ -22,6 +22,7 @@ #include "priv.h" #include <core/msgqueue.h> +#include <subdev/top.h> #include <engine/falcon.h> static void * @@ -39,18 +40,18 @@ nvkm_sec2_intr(struct nvkm_engine *engine) struct nvkm_sec2 *sec2 = nvkm_sec2(engine); struct nvkm_subdev *subdev = &engine->subdev; struct nvkm_device *device = subdev->device; - u32 disp = nvkm_rd32(device, 0x8701c); - u32 intr = nvkm_rd32(device, 0x87008) & disp & ~(disp >> 16); + u32 disp = nvkm_rd32(device, sec2->addr + 0x01c); + u32 intr = nvkm_rd32(device, sec2->addr + 0x008) & disp & ~(disp >> 16); if (intr & 0x00000040) { schedule_work(&sec2->work); - nvkm_wr32(device, 0x87004, 0x00000040); + nvkm_wr32(device, sec2->addr + 0x004, 0x00000040); intr &= ~0x00000040; } if (intr) { nvkm_error(subdev, "unhandled intr %08x\n", intr); - nvkm_wr32(device, 0x87004, intr); + nvkm_wr32(device, sec2->addr + 0x004, intr); } } @@ -74,8 +75,15 @@ static int nvkm_sec2_oneinit(struct nvkm_engine *engine) { struct nvkm_sec2 *sec2 = nvkm_sec2(engine); - return nvkm_falcon_v1_new(&sec2->engine.subdev, "SEC2", 0x87000, - &sec2->falcon); + struct nvkm_subdev *subdev = &sec2->engine.subdev; + + if (!sec2->addr) { + sec2->addr = nvkm_top_addr(subdev->device, subdev->index); + if (WARN_ON(!sec2->addr)) + return -EINVAL; + } + + return nvkm_falcon_v1_new(subdev, "SEC2", sec2->addr, &sec2->falcon); } static int @@ -95,13 +103,14 @@ nvkm_sec2 = { }; int -nvkm_sec2_new_(struct nvkm_device *device, int index, +nvkm_sec2_new_(struct nvkm_device *device, int index, u32 addr, struct nvkm_sec2 **psec2) { struct nvkm_sec2 *sec2; if (!(sec2 = *psec2 = kzalloc(sizeof(*sec2), GFP_KERNEL))) return -ENOMEM; + sec2->addr = addr; INIT_WORK(&sec2->work, nvkm_sec2_recv); return nvkm_engine_ctor(&nvkm_sec2, device, index, true, &sec2->engine); diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/sec2/gp102.c b/drivers/gpu/drm/nouveau/nvkm/engine/sec2/gp102.c index 9be1524c08f5..858cf27fa010 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/sec2/gp102.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/sec2/gp102.c @@ -26,5 +26,5 @@ int gp102_sec2_new(struct nvkm_device *device, int index, struct nvkm_sec2 **psec2) { - return nvkm_sec2_new_(device, index, psec2); + return nvkm_sec2_new_(device, index, 0, psec2); } diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/sec2/priv.h b/drivers/gpu/drm/nouveau/nvkm/engine/sec2/priv.h index 2f97c806a79d..ab0165e2d1a3 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/sec2/priv.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/sec2/priv.h @@ -5,6 +5,5 @@ #define nvkm_sec2(p) container_of((p), struct nvkm_sec2, engine) -int nvkm_sec2_new_(struct nvkm_device *, int, struct nvkm_sec2 **); - +int nvkm_sec2_new_(struct nvkm_device *, int, u32 addr, struct nvkm_sec2 **); #endif diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/sec2/tu102.c b/drivers/gpu/drm/nouveau/nvkm/engine/sec2/tu102.c new file mode 100644 index 000000000000..d655576164b1 --- /dev/null +++ b/drivers/gpu/drm/nouveau/nvkm/engine/sec2/tu102.c @@ -0,0 +1,33 @@ +/* + * Copyright 2019 Red Hat Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#include "priv.h" + +int +tu102_sec2_new(struct nvkm_device *device, int index, + struct nvkm_sec2 **psec2) +{ + /* TOP info wasn't updated on Turing to reflect the PRI + * address change for some reason. We override it here. + */ + return nvkm_sec2_new_(device, index, 0x840000, psec2); +} diff --git a/drivers/gpu/drm/nouveau/nvkm/falcon/base.c b/drivers/gpu/drm/nouveau/nvkm/falcon/base.c index 14be41f24155..366c87de6e72 100644 --- a/drivers/gpu/drm/nouveau/nvkm/falcon/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/falcon/base.c @@ -197,13 +197,16 @@ nvkm_falcon_ctor(const struct nvkm_falcon_func *func, case NVKM_SUBDEV_PMU: debug_reg = 0xc08; break; - case NVKM_ENGINE_NVDEC: + case NVKM_ENGINE_NVDEC0: debug_reg = 0xd00; break; case NVKM_ENGINE_SEC2: debug_reg = 0x408; falcon->has_emem = true; break; + case NVKM_SUBDEV_GSP: + debug_reg = 0x0; /*XXX*/ + break; default: nvkm_warn(subdev, "unsupported falcon %s!\n", nvkm_subdev_name[subdev->index]); diff --git a/drivers/gpu/drm/nouveau/nvkm/falcon/msgqueue.c b/drivers/gpu/drm/nouveau/nvkm/falcon/msgqueue.c index 771e16a16267..a8bee1e046aa 100644 --- a/drivers/gpu/drm/nouveau/nvkm/falcon/msgqueue.c +++ b/drivers/gpu/drm/nouveau/nvkm/falcon/msgqueue.c @@ -269,7 +269,7 @@ cmd_write(struct nvkm_msgqueue *priv, struct nvkm_msgqueue_hdr *cmd, commit = false; } - cmd_queue_close(priv, queue, commit); + cmd_queue_close(priv, queue, commit); return ret; } @@ -347,7 +347,7 @@ nvkm_msgqueue_post(struct nvkm_msgqueue *priv, enum msgqueue_msg_priority prio, ret = cmd_write(priv, cmd, queue); if (ret) { seq->state = SEQ_STATE_PENDING; - msgqueue_seq_release(priv, seq); + msgqueue_seq_release(priv, seq); } return ret; @@ -373,7 +373,7 @@ msgqueue_msg_handle(struct nvkm_msgqueue *priv, struct nvkm_msgqueue_hdr *hdr) if (seq->completion) complete(seq->completion); - msgqueue_seq_release(priv, seq); + msgqueue_seq_release(priv, seq); return 0; } diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/Kbuild b/drivers/gpu/drm/nouveau/nvkm/subdev/Kbuild index cfdffef1afb9..a339fe03d423 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/Kbuild +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/Kbuild @@ -7,6 +7,7 @@ include $(src)/nvkm/subdev/fault/Kbuild include $(src)/nvkm/subdev/fb/Kbuild include $(src)/nvkm/subdev/fuse/Kbuild include $(src)/nvkm/subdev/gpio/Kbuild +include $(src)/nvkm/subdev/gsp/Kbuild include $(src)/nvkm/subdev/i2c/Kbuild include $(src)/nvkm/subdev/ibus/Kbuild include $(src)/nvkm/subdev/iccsense/Kbuild diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/Kbuild b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/Kbuild index e5830453813d..dc300600c019 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/Kbuild +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/Kbuild @@ -5,3 +5,4 @@ nvkm-y += nvkm/subdev/bar/gf100.o nvkm-y += nvkm/subdev/bar/gk20a.o nvkm-y += nvkm/subdev/bar/gm107.o nvkm-y += nvkm/subdev/bar/gm20b.o +nvkm-y += nvkm/subdev/bar/tu102.o diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/base.c b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/base.c index 243f0a5c8a62..209a6a40834a 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/base.c @@ -36,6 +36,16 @@ nvkm_bar_bar1_vmm(struct nvkm_device *device) return device->bar->func->bar1.vmm(device->bar); } +void +nvkm_bar_bar1_reset(struct nvkm_device *device) +{ + struct nvkm_bar *bar = device->bar; + if (bar) { + bar->func->bar1.init(bar); + bar->func->bar1.wait(bar); + } +} + struct nvkm_vmm * nvkm_bar_bar2_vmm(struct nvkm_device *device) { @@ -49,6 +59,16 @@ nvkm_bar_bar2_vmm(struct nvkm_device *device) } void +nvkm_bar_bar2_reset(struct nvkm_device *device) +{ + struct nvkm_bar *bar = device->bar; + if (bar && bar->bar2) { + bar->func->bar2.init(bar); + bar->func->bar2.wait(bar); + } +} + +void nvkm_bar_bar2_fini(struct nvkm_device *device) { struct nvkm_bar *bar = device->bar; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/tu102.c b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/tu102.c new file mode 100644 index 000000000000..798f65ec3a86 --- /dev/null +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/tu102.c @@ -0,0 +1,98 @@ +/* + * Copyright 2018 Red Hat Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ +#include "gf100.h" + +#include <core/memory.h> +#include <subdev/timer.h> + +static void +tu102_bar_bar2_wait(struct nvkm_bar *bar) +{ + struct nvkm_device *device = bar->subdev.device; + nvkm_msec(device, 2000, + if (!(nvkm_rd32(device, 0xb80f50) & 0x0000000c)) + break; + ); +} + +static void +tu102_bar_bar2_fini(struct nvkm_bar *bar) +{ + nvkm_mask(bar->subdev.device, 0xb80f48, 0x80000000, 0x00000000); +} + +static void +tu102_bar_bar2_init(struct nvkm_bar *base) +{ + struct nvkm_device *device = base->subdev.device; + struct gf100_bar *bar = gf100_bar(base); + u32 addr = nvkm_memory_addr(bar->bar[0].inst) >> 12; + if (bar->bar2_halve) + addr |= 0x40000000; + nvkm_wr32(device, 0xb80f48, 0x80000000 | addr); +} + +static void +tu102_bar_bar1_wait(struct nvkm_bar *bar) +{ + struct nvkm_device *device = bar->subdev.device; + nvkm_msec(device, 2000, + if (!(nvkm_rd32(device, 0xb80f50) & 0x00000003)) + break; + ); +} + +static void +tu102_bar_bar1_fini(struct nvkm_bar *bar) +{ + nvkm_mask(bar->subdev.device, 0xb80f40, 0x80000000, 0x00000000); +} + +static void +tu102_bar_bar1_init(struct nvkm_bar *base) +{ + struct nvkm_device *device = base->subdev.device; + struct gf100_bar *bar = gf100_bar(base); + const u32 addr = nvkm_memory_addr(bar->bar[1].inst) >> 12; + nvkm_wr32(device, 0xb80f40, 0x80000000 | addr); +} + +static const struct nvkm_bar_func +tu102_bar = { + .dtor = gf100_bar_dtor, + .oneinit = gf100_bar_oneinit, + .bar1.init = tu102_bar_bar1_init, + .bar1.fini = tu102_bar_bar1_fini, + .bar1.wait = tu102_bar_bar1_wait, + .bar1.vmm = gf100_bar_bar1_vmm, + .bar2.init = tu102_bar_bar2_init, + .bar2.fini = tu102_bar_bar2_fini, + .bar2.wait = tu102_bar_bar2_wait, + .bar2.vmm = gf100_bar_bar2_vmm, + .flush = g84_bar_flush, +}; + +int +tu102_bar_new(struct nvkm_device *device, int index, struct nvkm_bar **pbar) +{ + return gf100_bar_new_(&tu102_bar, device, index, pbar); +} diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/bios/dp.c b/drivers/gpu/drm/nouveau/nvkm/subdev/bios/dp.c index 3133b28f849c..b099d1209be8 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/bios/dp.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/bios/dp.c @@ -212,7 +212,7 @@ nvbios_dpcfg_match(struct nvkm_bios *bios, u16 outp, u8 pc, u8 vs, u8 pe, u16 data; if (*ver >= 0x30) { - const u8 vsoff[] = { 0, 4, 7, 9 }; + static const u8 vsoff[] = { 0, 4, 7, 9 }; idx = (pc * 10) + vsoff[vs] + pe; if (*ver >= 0x40 && *ver <= 0x41 && *hdr >= 0x12) idx += nvbios_rd08(bios, outp + 0x11) * 40; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/bios/init.c b/drivers/gpu/drm/nouveau/nvkm/subdev/bios/init.c index 9cc10e438b3d..ec0e9f7224b5 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/bios/init.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/bios/init.c @@ -806,12 +806,12 @@ init_generic_condition(struct nvbios_init *init) init->offset += 3; switch (cond) { - case 0: + case 0: /* CONDITION_ID_INT_DP. */ if (init_conn(init) != DCB_CONNECTOR_eDP) init_exec_set(init, false); break; - case 1: - case 2: + case 1: /* CONDITION_ID_USE_SPPLL0. */ + case 2: /* CONDITION_ID_USE_SPPLL1. */ if ( init->outp && (data = nvbios_dpout_match(bios, DCB_OUTPUT_DP, (init->outp->or << 0) | @@ -826,10 +826,13 @@ init_generic_condition(struct nvbios_init *init) if (init_exec(init)) warn("script needs dp output table data\n"); break; - case 5: + case 5: /* CONDITION_ID_ASSR_SUPPORT. */ if (!(init_rdauxr(init, 0x0d) & 1)) init_exec_set(init, false); break; + case 7: /* CONDITION_ID_NO_PANEL_SEQ_DELAYS. */ + init_exec_set(init, false); + break; default: warn("INIT_GENERIC_CONDITON: unknown 0x%02x\n", cond); init->offset += size; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/devinit/Kbuild b/drivers/gpu/drm/nouveau/nvkm/subdev/devinit/Kbuild index 50a436926484..f3c388932b6f 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/devinit/Kbuild +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/devinit/Kbuild @@ -13,3 +13,4 @@ nvkm-y += nvkm/subdev/devinit/gf100.o nvkm-y += nvkm/subdev/devinit/gm107.o nvkm-y += nvkm/subdev/devinit/gm200.o nvkm-y += nvkm/subdev/devinit/gv100.o +nvkm-y += nvkm/subdev/devinit/tu102.o diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/devinit/gm200.c b/drivers/gpu/drm/nouveau/nvkm/subdev/devinit/gm200.c index 17235e940ca9..59940dacc2ba 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/devinit/gm200.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/devinit/gm200.c @@ -105,6 +105,15 @@ pmu_load(struct nv50_devinit *init, u8 type, bool post, return pmu_exec(init, pmu.init_addr_pmu), 0; } +void +gm200_devinit_preos(struct nv50_devinit *init, bool post) +{ + /* Optional: Execute PRE_OS application on PMU, which should at + * least take care of fans until a full PMU has been loaded. + */ + pmu_load(init, 0x01, post, NULL, NULL); +} + int gm200_devinit_post(struct nvkm_devinit *base, bool post) { @@ -156,10 +165,7 @@ gm200_devinit_post(struct nvkm_devinit *base, bool post) return -ETIMEDOUT; } - /* Optional: Execute PRE_OS application on PMU, which should at - * least take care of fans until a full PMU has been loaded. - */ - pmu_load(init, 0x01, post, NULL, NULL); + gm200_devinit_preos(init, post); return 0; } diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/devinit/nv50.h b/drivers/gpu/drm/nouveau/nvkm/subdev/devinit/nv50.h index 9b9f0dc1e192..72d130bb7f7c 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/devinit/nv50.h +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/devinit/nv50.h @@ -26,4 +26,5 @@ void gf100_devinit_preinit(struct nvkm_devinit *); u64 gm107_devinit_disable(struct nvkm_devinit *); int gm200_devinit_post(struct nvkm_devinit *, bool); +void gm200_devinit_preos(struct nv50_devinit *, bool); #endif diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/devinit/tu102.c b/drivers/gpu/drm/nouveau/nvkm/subdev/devinit/tu102.c new file mode 100644 index 000000000000..397670e72fff --- /dev/null +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/devinit/tu102.c @@ -0,0 +1,89 @@ +/* + * Copyright 2018 Red Hat Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ +#include "nv50.h" + +#include <subdev/bios.h> +#include <subdev/bios/pll.h> +#include <subdev/clk/pll.h> + +static int +tu102_devinit_pll_set(struct nvkm_devinit *init, u32 type, u32 freq) +{ + struct nvkm_subdev *subdev = &init->subdev; + struct nvkm_device *device = subdev->device; + struct nvbios_pll info; + int head = type - PLL_VPLL0; + int N, fN, M, P; + int ret; + + ret = nvbios_pll_parse(device->bios, type, &info); + if (ret) + return ret; + + ret = gt215_pll_calc(subdev, &info, freq, &N, &fN, &M, &P); + if (ret < 0) + return ret; + + switch (info.type) { + case PLL_VPLL0: + case PLL_VPLL1: + case PLL_VPLL2: + case PLL_VPLL3: + nvkm_wr32(device, 0x00ef10 + (head * 0x40), fN << 16); + nvkm_wr32(device, 0x00ef04 + (head * 0x40), (P << 16) | + (N << 8) | + (M << 0)); + /*XXX*/ + nvkm_wr32(device, 0x00ef0c + (head * 0x40), 0x00000900); + nvkm_wr32(device, 0x00ef00 + (head * 0x40), 0x02000014); + break; + default: + nvkm_warn(subdev, "%08x/%dKhz unimplemented\n", type, freq); + ret = -EINVAL; + break; + } + + return ret; +} + +static int +tu102_devinit_post(struct nvkm_devinit *base, bool post) +{ + struct nv50_devinit *init = nv50_devinit(base); + gm200_devinit_preos(init, post); + return 0; +} + +static const struct nvkm_devinit_func +tu102_devinit = { + .init = nv50_devinit_init, + .post = tu102_devinit_post, + .pll_set = tu102_devinit_pll_set, + .disable = gm107_devinit_disable, +}; + +int +tu102_devinit_new(struct nvkm_device *device, int index, + struct nvkm_devinit **pinit) +{ + return nv50_devinit_new_(&tu102_devinit, device, index, pinit); +} diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fault/Kbuild b/drivers/gpu/drm/nouveau/nvkm/subdev/fault/Kbuild index 45bb46fb0929..42586267fc08 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/fault/Kbuild +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fault/Kbuild @@ -1,3 +1,5 @@ nvkm-y += nvkm/subdev/fault/base.o +nvkm-y += nvkm/subdev/fault/user.o nvkm-y += nvkm/subdev/fault/gp100.o nvkm-y += nvkm/subdev/fault/gv100.o +nvkm-y += nvkm/subdev/fault/tu102.o diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fault/base.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fault/base.c index 16ad91c91a7b..ca251560d3e0 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/fault/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fault/base.c @@ -23,21 +23,19 @@ #include <core/memory.h> #include <core/notify.h> -#include <subdev/bar.h> -#include <subdev/mmu.h> static void nvkm_fault_ntfy_fini(struct nvkm_event *event, int type, int index) { struct nvkm_fault *fault = container_of(event, typeof(*fault), event); - fault->func->buffer.fini(fault->buffer[index]); + fault->func->buffer.intr(fault->buffer[index], false); } static void nvkm_fault_ntfy_init(struct nvkm_event *event, int type, int index) { struct nvkm_fault *fault = container_of(event, typeof(*fault), event); - fault->func->buffer.init(fault->buffer[index]); + fault->func->buffer.intr(fault->buffer[index], true); } static int @@ -91,7 +89,6 @@ nvkm_fault_oneinit_buffer(struct nvkm_fault *fault, int id) { struct nvkm_subdev *subdev = &fault->subdev; struct nvkm_device *device = subdev->device; - struct nvkm_vmm *bar2 = nvkm_bar_bar2_vmm(device); struct nvkm_fault_buffer *buffer; int ret; @@ -99,7 +96,7 @@ nvkm_fault_oneinit_buffer(struct nvkm_fault *fault, int id) return -ENOMEM; buffer->fault = fault; buffer->id = id; - buffer->entries = fault->func->buffer.entries(buffer); + fault->func->buffer.info(buffer); fault->buffer[id] = buffer; nvkm_debug(subdev, "buffer %d: %d entries\n", id, buffer->entries); @@ -110,12 +107,12 @@ nvkm_fault_oneinit_buffer(struct nvkm_fault *fault, int id) if (ret) return ret; - ret = nvkm_vmm_get(bar2, 12, nvkm_memory_size(buffer->mem), - &buffer->vma); - if (ret) - return ret; + /* Pin fault buffer in BAR2. */ + buffer->addr = nvkm_memory_bar2(buffer->mem); + if (buffer->addr == ~0ULL) + return -EFAULT; - return nvkm_memory_map(buffer->mem, 0, bar2, buffer->vma, NULL, 0); + return 0; } static int @@ -146,7 +143,6 @@ nvkm_fault_oneinit(struct nvkm_subdev *subdev) static void * nvkm_fault_dtor(struct nvkm_subdev *subdev) { - struct nvkm_vmm *bar2 = nvkm_bar_bar2_vmm(subdev->device); struct nvkm_fault *fault = nvkm_fault(subdev); int i; @@ -154,7 +150,6 @@ nvkm_fault_dtor(struct nvkm_subdev *subdev) for (i = 0; i < fault->buffer_nr; i++) { if (fault->buffer[i]) { - nvkm_vmm_put(bar2, &fault->buffer[i]->vma); nvkm_memory_unref(&fault->buffer[i]->mem); kfree(fault->buffer[i]); } @@ -181,5 +176,7 @@ nvkm_fault_new_(const struct nvkm_fault_func *func, struct nvkm_device *device, return -ENOMEM; nvkm_subdev_ctor(&nvkm_fault, device, index, &fault->subdev); fault->func = func; + fault->user.ctor = nvkm_ufault_new; + fault->user.base = func->user.base; return 0; } diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fault/gp100.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fault/gp100.c index 5e71db2e8d75..4f3c4e091117 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/fault/gp100.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fault/gp100.c @@ -21,7 +21,16 @@ */ #include "priv.h" -#include <subdev/mmu.h> +#include <subdev/mc.h> + +#include <nvif/class.h> + +static void +gp100_fault_buffer_intr(struct nvkm_fault_buffer *buffer, bool enable) +{ + struct nvkm_device *device = buffer->fault->subdev.device; + nvkm_mc_intr_mask(device, NVKM_SUBDEV_FAULT, enable); +} static void gp100_fault_buffer_fini(struct nvkm_fault_buffer *buffer) @@ -34,15 +43,17 @@ static void gp100_fault_buffer_init(struct nvkm_fault_buffer *buffer) { struct nvkm_device *device = buffer->fault->subdev.device; - nvkm_wr32(device, 0x002a74, upper_32_bits(buffer->vma->addr)); - nvkm_wr32(device, 0x002a70, lower_32_bits(buffer->vma->addr)); + nvkm_wr32(device, 0x002a74, upper_32_bits(buffer->addr)); + nvkm_wr32(device, 0x002a70, lower_32_bits(buffer->addr)); nvkm_mask(device, 0x002a70, 0x00000001, 0x00000001); } -static u32 -gp100_fault_buffer_entries(struct nvkm_fault_buffer *buffer) +static void +gp100_fault_buffer_info(struct nvkm_fault_buffer *buffer) { - return nvkm_rd32(buffer->fault->subdev.device, 0x002a78); + buffer->entries = nvkm_rd32(buffer->fault->subdev.device, 0x002a78); + buffer->get = 0x002a7c; + buffer->put = 0x002a80; } static void @@ -56,9 +67,11 @@ gp100_fault = { .intr = gp100_fault_intr, .buffer.nr = 1, .buffer.entry_size = 32, - .buffer.entries = gp100_fault_buffer_entries, + .buffer.info = gp100_fault_buffer_info, .buffer.init = gp100_fault_buffer_init, .buffer.fini = gp100_fault_buffer_fini, + .buffer.intr = gp100_fault_buffer_intr, + .user = { { 0, 0, MAXWELL_FAULT_BUFFER_A }, 0 }, }; int diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fault/gv100.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fault/gv100.c index 3cd610d7deb5..6747f09c2dc3 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/fault/gv100.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fault/gv100.c @@ -25,14 +25,15 @@ #include <subdev/mmu.h> #include <engine/fifo.h> +#include <nvif/class.h> + static void gv100_fault_buffer_process(struct nvkm_fault_buffer *buffer) { struct nvkm_device *device = buffer->fault->subdev.device; struct nvkm_memory *mem = buffer->mem; - const u32 foff = buffer->id * 0x14; - u32 get = nvkm_rd32(device, 0x100e2c + foff); - u32 put = nvkm_rd32(device, 0x100e30 + foff); + u32 get = nvkm_rd32(device, buffer->get); + u32 put = nvkm_rd32(device, buffer->put); if (put == get) return; @@ -51,7 +52,7 @@ gv100_fault_buffer_process(struct nvkm_fault_buffer *buffer) if (++get == buffer->entries) get = 0; - nvkm_wr32(device, 0x100e2c + foff, get); + nvkm_wr32(device, buffer->get, get); info.addr = ((u64)addrhi << 32) | addrlo; info.inst = ((u64)insthi << 32) | instlo; @@ -70,13 +71,21 @@ gv100_fault_buffer_process(struct nvkm_fault_buffer *buffer) } static void -gv100_fault_buffer_fini(struct nvkm_fault_buffer *buffer) +gv100_fault_buffer_intr(struct nvkm_fault_buffer *buffer, bool enable) { struct nvkm_device *device = buffer->fault->subdev.device; const u32 intr = buffer->id ? 0x08000000 : 0x20000000; - const u32 foff = buffer->id * 0x14; + if (enable) + nvkm_mask(device, 0x100a2c, intr, intr); + else + nvkm_mask(device, 0x100a34, intr, intr); +} - nvkm_mask(device, 0x100a34, intr, intr); +static void +gv100_fault_buffer_fini(struct nvkm_fault_buffer *buffer) +{ + struct nvkm_device *device = buffer->fault->subdev.device; + const u32 foff = buffer->id * 0x14; nvkm_mask(device, 0x100e34 + foff, 0x80000000, 0x00000000); } @@ -84,23 +93,25 @@ static void gv100_fault_buffer_init(struct nvkm_fault_buffer *buffer) { struct nvkm_device *device = buffer->fault->subdev.device; - const u32 intr = buffer->id ? 0x08000000 : 0x20000000; const u32 foff = buffer->id * 0x14; nvkm_mask(device, 0x100e34 + foff, 0xc0000000, 0x40000000); - nvkm_wr32(device, 0x100e28 + foff, upper_32_bits(buffer->vma->addr)); - nvkm_wr32(device, 0x100e24 + foff, lower_32_bits(buffer->vma->addr)); + nvkm_wr32(device, 0x100e28 + foff, upper_32_bits(buffer->addr)); + nvkm_wr32(device, 0x100e24 + foff, lower_32_bits(buffer->addr)); nvkm_mask(device, 0x100e34 + foff, 0x80000000, 0x80000000); - nvkm_mask(device, 0x100a2c, intr, intr); } -static u32 -gv100_fault_buffer_entries(struct nvkm_fault_buffer *buffer) +static void +gv100_fault_buffer_info(struct nvkm_fault_buffer *buffer) { struct nvkm_device *device = buffer->fault->subdev.device; const u32 foff = buffer->id * 0x14; + nvkm_mask(device, 0x100e34 + foff, 0x40000000, 0x40000000); - return nvkm_rd32(device, 0x100e34 + foff) & 0x000fffff; + + buffer->entries = nvkm_rd32(device, 0x100e34 + foff) & 0x000fffff; + buffer->get = 0x100e2c + foff; + buffer->put = 0x100e30 + foff; } static int @@ -157,6 +168,13 @@ gv100_fault_intr(struct nvkm_fault *fault) } } + if (stat & 0x08000000) { + if (fault->buffer[1]) { + nvkm_event_send(&fault->event, 1, 1, NULL, 0); + stat &= ~0x08000000; + } + } + if (stat) { nvkm_debug(subdev, "intr %08x\n", stat); } @@ -166,6 +184,8 @@ static void gv100_fault_fini(struct nvkm_fault *fault) { nvkm_notify_put(&fault->nrpfb); + if (fault->buffer[0]) + fault->func->buffer.fini(fault->buffer[0]); nvkm_mask(fault->subdev.device, 0x100a34, 0x80000000, 0x80000000); } @@ -173,14 +193,15 @@ static void gv100_fault_init(struct nvkm_fault *fault) { nvkm_mask(fault->subdev.device, 0x100a2c, 0x80000000, 0x80000000); + fault->func->buffer.init(fault->buffer[0]); nvkm_notify_get(&fault->nrpfb); } -static int +int gv100_fault_oneinit(struct nvkm_fault *fault) { return nvkm_notify_init(&fault->buffer[0]->object, &fault->event, - gv100_fault_ntfy_nrpfb, false, NULL, 0, 0, + gv100_fault_ntfy_nrpfb, true, NULL, 0, 0, &fault->nrpfb); } @@ -192,9 +213,17 @@ gv100_fault = { .intr = gv100_fault_intr, .buffer.nr = 2, .buffer.entry_size = 32, - .buffer.entries = gv100_fault_buffer_entries, + .buffer.info = gv100_fault_buffer_info, .buffer.init = gv100_fault_buffer_init, .buffer.fini = gv100_fault_buffer_fini, + .buffer.intr = gv100_fault_buffer_intr, + /*TODO: Figure out how to expose non-replayable fault buffer, which, + * for some reason, is where recoverable CE faults appear... + * + * It's a bit tricky, as both NVKM and SVM will need access to + * the non-replayable fault buffer. + */ + .user = { { 0, 0, VOLTA_FAULT_BUFFER_A }, 1 }, }; int diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fault/priv.h b/drivers/gpu/drm/nouveau/nvkm/subdev/fault/priv.h index e4d2f5234fd1..975e66ac6344 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/fault/priv.h +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fault/priv.h @@ -12,8 +12,10 @@ struct nvkm_fault_buffer { struct nvkm_fault *fault; int id; int entries; + u32 get; + u32 put; struct nvkm_memory *mem; - struct nvkm_vma *vma; + u64 addr; }; int nvkm_fault_new_(const struct nvkm_fault_func *, struct nvkm_device *, @@ -27,9 +29,19 @@ struct nvkm_fault_func { struct { int nr; u32 entry_size; - u32 (*entries)(struct nvkm_fault_buffer *); + void (*info)(struct nvkm_fault_buffer *); void (*init)(struct nvkm_fault_buffer *); void (*fini)(struct nvkm_fault_buffer *); + void (*intr)(struct nvkm_fault_buffer *, bool enable); } buffer; + struct { + struct nvkm_sclass base; + int rp; + } user; }; + +int gv100_fault_oneinit(struct nvkm_fault *); + +int nvkm_ufault_new(struct nvkm_device *, const struct nvkm_oclass *, + void *, u32, struct nvkm_object **); #endif diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fault/tu102.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fault/tu102.c new file mode 100644 index 000000000000..fa1dfe5692b0 --- /dev/null +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fault/tu102.c @@ -0,0 +1,168 @@ +/* + * Copyright 2018 Red Hat Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ +#include "priv.h" + +#include <core/memory.h> +#include <subdev/mmu.h> +#include <engine/fifo.h> + +#include <nvif/class.h> + +static void +tu102_fault_buffer_intr(struct nvkm_fault_buffer *buffer, bool enable) +{ + /*XXX: Earlier versions of RM touched the old regs on Turing, + * which don't appear to actually work anymore, but newer + * versions of RM don't appear to touch anything at all.. + */ +} + +static void +tu102_fault_buffer_fini(struct nvkm_fault_buffer *buffer) +{ + struct nvkm_device *device = buffer->fault->subdev.device; + const u32 foff = buffer->id * 0x20; + nvkm_mask(device, 0xb83010 + foff, 0x80000000, 0x00000000); +} + +static void +tu102_fault_buffer_init(struct nvkm_fault_buffer *buffer) +{ + struct nvkm_device *device = buffer->fault->subdev.device; + const u32 foff = buffer->id * 0x20; + + nvkm_mask(device, 0xb83010 + foff, 0xc0000000, 0x40000000); + nvkm_wr32(device, 0xb83004 + foff, upper_32_bits(buffer->addr)); + nvkm_wr32(device, 0xb83000 + foff, lower_32_bits(buffer->addr)); + nvkm_mask(device, 0xb83010 + foff, 0x80000000, 0x80000000); +} + +static void +tu102_fault_buffer_info(struct nvkm_fault_buffer *buffer) +{ + struct nvkm_device *device = buffer->fault->subdev.device; + const u32 foff = buffer->id * 0x20; + + nvkm_mask(device, 0xb83010 + foff, 0x40000000, 0x40000000); + + buffer->entries = nvkm_rd32(device, 0xb83010 + foff) & 0x000fffff; + buffer->get = 0xb83008 + foff; + buffer->put = 0xb8300c + foff; +} + +static void +tu102_fault_intr_fault(struct nvkm_fault *fault) +{ + struct nvkm_subdev *subdev = &fault->subdev; + struct nvkm_device *device = subdev->device; + struct nvkm_fault_data info; + const u32 addrlo = nvkm_rd32(device, 0xb83080); + const u32 addrhi = nvkm_rd32(device, 0xb83084); + const u32 info0 = nvkm_rd32(device, 0xb83088); + const u32 insthi = nvkm_rd32(device, 0xb8308c); + const u32 info1 = nvkm_rd32(device, 0xb83090); + + info.addr = ((u64)addrhi << 32) | addrlo; + info.inst = ((u64)insthi << 32) | (info0 & 0xfffff000); + info.time = 0; + info.engine = (info0 & 0x000000ff); + info.valid = (info1 & 0x80000000) >> 31; + info.gpc = (info1 & 0x1f000000) >> 24; + info.hub = (info1 & 0x00100000) >> 20; + info.access = (info1 & 0x000f0000) >> 16; + info.client = (info1 & 0x00007f00) >> 8; + info.reason = (info1 & 0x0000001f); + + nvkm_fifo_fault(device->fifo, &info); +} + +static void +tu102_fault_intr(struct nvkm_fault *fault) +{ + struct nvkm_subdev *subdev = &fault->subdev; + struct nvkm_device *device = subdev->device; + u32 stat = nvkm_rd32(device, 0xb83094); + + if (stat & 0x80000000) { + tu102_fault_intr_fault(fault); + nvkm_wr32(device, 0xb83094, 0x80000000); + stat &= ~0x80000000; + } + + if (stat & 0x00000200) { + if (fault->buffer[0]) { + nvkm_event_send(&fault->event, 1, 0, NULL, 0); + stat &= ~0x00000200; + } + } + + /*XXX: guess, can't confirm until we get fw... */ + if (stat & 0x00000100) { + if (fault->buffer[1]) { + nvkm_event_send(&fault->event, 1, 1, NULL, 0); + stat &= ~0x00000100; + } + } + + if (stat) { + nvkm_debug(subdev, "intr %08x\n", stat); + } +} + +static void +tu102_fault_fini(struct nvkm_fault *fault) +{ + nvkm_notify_put(&fault->nrpfb); + if (fault->buffer[0]) + fault->func->buffer.fini(fault->buffer[0]); + /*XXX: disable priv faults */ +} + +static void +tu102_fault_init(struct nvkm_fault *fault) +{ + /*XXX: enable priv faults */ + fault->func->buffer.init(fault->buffer[0]); + nvkm_notify_get(&fault->nrpfb); +} + +static const struct nvkm_fault_func +tu102_fault = { + .oneinit = gv100_fault_oneinit, + .init = tu102_fault_init, + .fini = tu102_fault_fini, + .intr = tu102_fault_intr, + .buffer.nr = 2, + .buffer.entry_size = 32, + .buffer.info = tu102_fault_buffer_info, + .buffer.init = tu102_fault_buffer_init, + .buffer.fini = tu102_fault_buffer_fini, + .buffer.intr = tu102_fault_buffer_intr, + .user = { { 0, 0, VOLTA_FAULT_BUFFER_A }, 1 }, +}; + +int +tu102_fault_new(struct nvkm_device *device, int index, + struct nvkm_fault **pfault) +{ + return nvkm_fault_new_(&tu102_fault, device, index, pfault); +} diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fault/user.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fault/user.c new file mode 100644 index 000000000000..ac835c9582fd --- /dev/null +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fault/user.c @@ -0,0 +1,106 @@ +/* + * Copyright 2018 Red Hat Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ +#include "priv.h" + +#include <core/memory.h> +#include <subdev/mmu.h> + +#include <nvif/clb069.h> +#include <nvif/unpack.h> + +static int +nvkm_ufault_map(struct nvkm_object *object, void *argv, u32 argc, + enum nvkm_object_map *type, u64 *addr, u64 *size) +{ + struct nvkm_fault_buffer *buffer = nvkm_fault_buffer(object); + struct nvkm_device *device = buffer->fault->subdev.device; + *type = NVKM_OBJECT_MAP_IO; + *addr = device->func->resource_addr(device, 3) + buffer->addr; + *size = nvkm_memory_size(buffer->mem); + return 0; +} + +static int +nvkm_ufault_ntfy(struct nvkm_object *object, u32 type, + struct nvkm_event **pevent) +{ + struct nvkm_fault_buffer *buffer = nvkm_fault_buffer(object); + if (type == NVB069_V0_NTFY_FAULT) { + *pevent = &buffer->fault->event; + return 0; + } + return -EINVAL; +} + +static int +nvkm_ufault_fini(struct nvkm_object *object, bool suspend) +{ + struct nvkm_fault_buffer *buffer = nvkm_fault_buffer(object); + buffer->fault->func->buffer.fini(buffer); + return 0; +} + +static int +nvkm_ufault_init(struct nvkm_object *object) +{ + struct nvkm_fault_buffer *buffer = nvkm_fault_buffer(object); + buffer->fault->func->buffer.init(buffer); + return 0; +} + +static void * +nvkm_ufault_dtor(struct nvkm_object *object) +{ + return NULL; +} + +static const struct nvkm_object_func +nvkm_ufault = { + .dtor = nvkm_ufault_dtor, + .init = nvkm_ufault_init, + .fini = nvkm_ufault_fini, + .ntfy = nvkm_ufault_ntfy, + .map = nvkm_ufault_map, +}; + +int +nvkm_ufault_new(struct nvkm_device *device, const struct nvkm_oclass *oclass, + void *argv, u32 argc, struct nvkm_object **pobject) +{ + union { + struct nvif_clb069_v0 v0; + } *args = argv; + struct nvkm_fault *fault = device->fault; + struct nvkm_fault_buffer *buffer = fault->buffer[fault->func->user.rp]; + int ret = -ENOSYS; + + if (!(ret = nvif_unpack(ret, &argv, &argc, args->v0, 0, 0, false))) { + args->v0.entries = buffer->entries; + args->v0.get = buffer->get; + args->v0.put = buffer->put; + } else + return ret; + + nvkm_object_ctor(&nvkm_ufault, oclass, &buffer->object); + *pobject = &buffer->object; + return 0; +} diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/base.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/base.c index 434d2fc5bb1c..b2bb5a3ccb02 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/base.c @@ -68,10 +68,13 @@ nvkm_fb_bios_memtype(struct nvkm_bios *bios) if (nvbios_M0203Em(bios, ramcfg, &ver, &hdr, &M0203E)) { switch (M0203E.type) { - case M0203E_TYPE_DDR2 : return NVKM_RAM_TYPE_DDR2; - case M0203E_TYPE_DDR3 : return NVKM_RAM_TYPE_DDR3; - case M0203E_TYPE_GDDR3: return NVKM_RAM_TYPE_GDDR3; - case M0203E_TYPE_GDDR5: return NVKM_RAM_TYPE_GDDR5; + case M0203E_TYPE_DDR2 : return NVKM_RAM_TYPE_DDR2; + case M0203E_TYPE_DDR3 : return NVKM_RAM_TYPE_DDR3; + case M0203E_TYPE_GDDR3 : return NVKM_RAM_TYPE_GDDR3; + case M0203E_TYPE_GDDR5 : return NVKM_RAM_TYPE_GDDR5; + case M0203E_TYPE_GDDR5X: return NVKM_RAM_TYPE_GDDR5X; + case M0203E_TYPE_GDDR6 : return NVKM_RAM_TYPE_GDDR6; + case M0203E_TYPE_HBM2 : return NVKM_RAM_TYPE_HBM2; default: nvkm_warn(subdev, "M0203E type %02x\n", M0203E.type); return NVKM_RAM_TYPE_UNKNOWN; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gddr3.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gddr3.c index 60ece0a8a2e1..1d2d6bae73cd 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gddr3.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gddr3.c @@ -87,7 +87,7 @@ nvkm_gddr3_calc(struct nvkm_ram *ram) WR = (ram->next->bios.timing[2] & 0x007f0000) >> 16; /* XXX: Get these values from the VBIOS instead */ DLL = !(ram->mr[1] & 0x1); - RON = !(ram->mr[1] & 0x300) >> 8; + RON = !((ram->mr[1] & 0x300) >> 8); break; default: return -ENOSYS; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ram.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ram.c index 24c7bd505731..b11867f682cb 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ram.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ram.c @@ -184,6 +184,9 @@ nvkm_ram_ctor(const struct nvkm_ram_func *func, struct nvkm_fb *fb, [NVKM_RAM_TYPE_GDDR3 ] = "GDDR3", [NVKM_RAM_TYPE_GDDR4 ] = "GDDR4", [NVKM_RAM_TYPE_GDDR5 ] = "GDDR5", + [NVKM_RAM_TYPE_GDDR5X ] = "GDDR5X", + [NVKM_RAM_TYPE_GDDR6 ] = "GDDR6", + [NVKM_RAM_TYPE_HBM2 ] = "HBM2", }; struct nvkm_subdev *subdev = &fb->subdev; int ret; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/Kbuild b/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/Kbuild new file mode 100644 index 000000000000..26fc6feb807e --- /dev/null +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/Kbuild @@ -0,0 +1 @@ +nvkm-y += nvkm/subdev/gsp/gv100.o diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/gv100.c b/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/gv100.c new file mode 100644 index 000000000000..dccfaf1162e2 --- /dev/null +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/gv100.c @@ -0,0 +1,62 @@ +/* + * Copyright 2019 Red Hat Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ +#include <subdev/gsp.h> +#include <subdev/top.h> +#include <engine/falcon.h> + +static int +gv100_gsp_oneinit(struct nvkm_subdev *subdev) +{ + struct nvkm_gsp *gsp = nvkm_gsp(subdev); + + gsp->addr = nvkm_top_addr(subdev->device, subdev->index); + if (!gsp->addr) + return -EINVAL; + + return nvkm_falcon_v1_new(subdev, "GSP", gsp->addr, &gsp->falcon); +} + +static void * +gv100_gsp_dtor(struct nvkm_subdev *subdev) +{ + struct nvkm_gsp *gsp = nvkm_gsp(subdev); + nvkm_falcon_del(&gsp->falcon); + return gsp; +} + +static const struct nvkm_subdev_func +gv100_gsp = { + .dtor = gv100_gsp_dtor, + .oneinit = gv100_gsp_oneinit, +}; + +int +gv100_gsp_new(struct nvkm_device *device, int index, struct nvkm_gsp **pgsp) +{ + struct nvkm_gsp *gsp; + + if (!(gsp = *pgsp = kzalloc(sizeof(*gsp), GFP_KERNEL))) + return -ENOMEM; + + nvkm_subdev_ctor(&gv100_gsp, device, index, &gsp->subdev); + return 0; +} diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/nv50.c b/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/nv50.c index db48a1daca0c..02c4eb28cef4 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/nv50.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/nv50.c @@ -288,6 +288,19 @@ nv50_instobj_addr(struct nvkm_memory *memory) return nvkm_memory_addr(nv50_instobj(memory)->ram); } +static u64 +nv50_instobj_bar2(struct nvkm_memory *memory) +{ + struct nv50_instobj *iobj = nv50_instobj(memory); + u64 addr = ~0ULL; + if (nv50_instobj_acquire(&iobj->base.memory)) { + iobj->lru.next = NULL; /* Exclude from eviction. */ + addr = iobj->bar->addr; + } + nv50_instobj_release(&iobj->base.memory); + return addr; +} + static enum nvkm_memory_target nv50_instobj_target(struct nvkm_memory *memory) { @@ -325,8 +338,9 @@ static const struct nvkm_memory_func nv50_instobj_func = { .dtor = nv50_instobj_dtor, .target = nv50_instobj_target, - .size = nv50_instobj_size, + .bar2 = nv50_instobj_bar2, .addr = nv50_instobj_addr, + .size = nv50_instobj_size, .boot = nv50_instobj_boot, .acquire = nv50_instobj_acquire, .release = nv50_instobj_release, diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mc/Kbuild b/drivers/gpu/drm/nouveau/nvkm/subdev/mc/Kbuild index 2befbe36dc28..c64e399326b3 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mc/Kbuild +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mc/Kbuild @@ -12,3 +12,4 @@ nvkm-y += nvkm/subdev/mc/gk104.o nvkm-y += nvkm/subdev/mc/gk20a.o nvkm-y += nvkm/subdev/mc/gp100.o nvkm-y += nvkm/subdev/mc/gp10b.o +nvkm-y += nvkm/subdev/mc/tu102.o diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mc/base.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mc/base.c index 09f669ac6630..0e57ab2a709f 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mc/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mc/base.c @@ -108,6 +108,9 @@ nvkm_mc_intr(struct nvkm_device *device, bool *handled) if (stat) nvkm_error(&mc->subdev, "intr %08x\n", stat); *handled = intr != 0; + + if (mc->func->intr_hack) + mc->func->intr_hack(mc, handled); } static u32 diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mc/priv.h b/drivers/gpu/drm/nouveau/nvkm/subdev/mc/priv.h index d9e3691d45b7..eb91a4cf452b 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mc/priv.h +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mc/priv.h @@ -26,6 +26,7 @@ struct nvkm_mc_func { void (*intr_mask)(struct nvkm_mc *, u32 mask, u32 stat); /* retrieve pending interrupt mask (NV_PMC_INTR) */ u32 (*intr_stat)(struct nvkm_mc *); + void (*intr_hack)(struct nvkm_mc *, bool *handled); const struct nvkm_mc_map *reset; void (*unk260)(struct nvkm_mc *, u32); }; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mc/tu102.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mc/tu102.c new file mode 100644 index 000000000000..d098c44a4fcb --- /dev/null +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mc/tu102.c @@ -0,0 +1,55 @@ +/* + * Copyright 2018 Red Hat Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ +#include "priv.h" + +static void +tu102_mc_intr_hack(struct nvkm_mc *mc, bool *handled) +{ + struct nvkm_device *device = mc->subdev.device; + u32 stat = nvkm_rd32(device, 0xb81010); + if (stat & 0x00000050) { + struct nvkm_subdev *subdev = + nvkm_device_subdev(device, NVKM_SUBDEV_FAULT); + nvkm_wr32(device, 0xb81010, stat & 0x00000050); + if (subdev) + nvkm_subdev_intr(subdev); + *handled = true; + } +} + +static const struct nvkm_mc_func +tu102_mc = { + .init = nv50_mc_init, + .intr = gp100_mc_intr, + .intr_unarm = gp100_mc_intr_unarm, + .intr_rearm = gp100_mc_intr_rearm, + .intr_mask = gp100_mc_intr_mask, + .intr_stat = gf100_mc_intr_stat, + .intr_hack = tu102_mc_intr_hack, + .reset = gk104_mc_reset, +}; + +int +tu102_mc_new(struct nvkm_device *device, int index, struct nvkm_mc **pmc) +{ + return gp100_mc_new_(&tu102_mc, device, index, pmc); +} diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/Kbuild b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/Kbuild index 58a24e3a0598..db9c56028f21 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/Kbuild +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/Kbuild @@ -13,6 +13,7 @@ nvkm-y += nvkm/subdev/mmu/gm20b.o nvkm-y += nvkm/subdev/mmu/gp100.o nvkm-y += nvkm/subdev/mmu/gp10b.o nvkm-y += nvkm/subdev/mmu/gv100.o +nvkm-y += nvkm/subdev/mmu/tu102.o nvkm-y += nvkm/subdev/mmu/mem.o nvkm-y += nvkm/subdev/mmu/memnv04.o @@ -33,6 +34,7 @@ nvkm-y += nvkm/subdev/mmu/vmmgm20b.o nvkm-y += nvkm/subdev/mmu/vmmgp100.o nvkm-y += nvkm/subdev/mmu/vmmgp10b.o nvkm-y += nvkm/subdev/mmu/vmmgv100.o +nvkm-y += nvkm/subdev/mmu/vmmtu102.o nvkm-y += nvkm/subdev/mmu/umem.o nvkm-y += nvkm/subdev/mmu/ummu.o diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gp100.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gp100.c index 651b8805c67c..65cb9d28e60e 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gp100.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gp100.c @@ -31,7 +31,7 @@ gp100_mmu = { .dma_bits = 47, .mmu = {{ -1, -1, NVIF_CLASS_MMU_GF100}}, .mem = {{ -1, 0, NVIF_CLASS_MEM_GF100}, gf100_mem_new, gf100_mem_map }, - .vmm = {{ -1, -1, NVIF_CLASS_VMM_GP100}, gp100_vmm_new }, + .vmm = {{ -1, 0, NVIF_CLASS_VMM_GP100}, gp100_vmm_new }, .kind = gm200_mmu_kind, .kind_sys = true, }; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gp10b.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gp10b.c index 3bd3db31e0bb..0a50be9a785a 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gp10b.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gp10b.c @@ -31,7 +31,7 @@ gp10b_mmu = { .dma_bits = 47, .mmu = {{ -1, -1, NVIF_CLASS_MMU_GF100}}, .mem = {{ -1, -1, NVIF_CLASS_MEM_GF100}, .umap = gf100_mem_map }, - .vmm = {{ -1, -1, NVIF_CLASS_VMM_GP100}, gp10b_vmm_new }, + .vmm = {{ -1, 0, NVIF_CLASS_VMM_GP100}, gp10b_vmm_new }, .kind = gm200_mmu_kind, .kind_sys = true, }; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gv100.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gv100.c index f666cb57f69e..e0997eedd6d9 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gv100.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gv100.c @@ -31,7 +31,7 @@ gv100_mmu = { .dma_bits = 47, .mmu = {{ -1, -1, NVIF_CLASS_MMU_GF100}}, .mem = {{ -1, 0, NVIF_CLASS_MEM_GF100}, gf100_mem_new, gf100_mem_map }, - .vmm = {{ -1, -1, NVIF_CLASS_VMM_GP100}, gv100_vmm_new }, + .vmm = {{ -1, 0, NVIF_CLASS_VMM_GP100}, gv100_vmm_new }, .kind = gm200_mmu_kind, .kind_sys = true, }; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/priv.h b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/priv.h index 948a48c21be4..2ad1102a4e31 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/priv.h +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/priv.h @@ -28,7 +28,7 @@ struct nvkm_mmu_func { struct { struct nvkm_sclass user; - int (*ctor)(struct nvkm_mmu *, u64 addr, u64 size, + int (*ctor)(struct nvkm_mmu *, bool managed, u64 addr, u64 size, void *argv, u32 argc, struct lock_class_key *, const char *name, struct nvkm_vmm **); bool global; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/tu102.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/tu102.c new file mode 100644 index 000000000000..c0db0ce10cba --- /dev/null +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/tu102.c @@ -0,0 +1,43 @@ +/* + * Copyright 2018 Red Hat Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ +#include "mem.h" +#include "vmm.h" + +#include <core/option.h> + +#include <nvif/class.h> + +static const struct nvkm_mmu_func +tu102_mmu = { + .dma_bits = 47, + .mmu = {{ -1, -1, NVIF_CLASS_MMU_GF100}}, + .mem = {{ -1, 0, NVIF_CLASS_MEM_GF100}, gf100_mem_new, gf100_mem_map }, + .vmm = {{ -1, 0, NVIF_CLASS_VMM_GP100}, tu102_vmm_new }, + .kind = gm200_mmu_kind, + .kind_sys = true, +}; + +int +tu102_mmu_new(struct nvkm_device *device, int index, struct nvkm_mmu **pmmu) +{ + return nvkm_mmu_new_(&tu102_mmu, device, index, pmmu); +} diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/uvmm.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/uvmm.c index 37b201b95f15..c43b8248c682 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/uvmm.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/uvmm.c @@ -43,6 +43,69 @@ nvkm_uvmm_search(struct nvkm_client *client, u64 handle) } static int +nvkm_uvmm_mthd_pfnclr(struct nvkm_uvmm *uvmm, void *argv, u32 argc) +{ + struct nvkm_client *client = uvmm->object.client; + union { + struct nvif_vmm_pfnclr_v0 v0; + } *args = argv; + struct nvkm_vmm *vmm = uvmm->vmm; + int ret = -ENOSYS; + u64 addr, size; + + if (!(ret = nvif_unpack(ret, &argv, &argc, args->v0, 0, 0, false))) { + addr = args->v0.addr; + size = args->v0.size; + } else + return ret; + + if (!client->super) + return -ENOENT; + + if (size) { + mutex_lock(&vmm->mutex); + ret = nvkm_vmm_pfn_unmap(vmm, addr, size); + mutex_unlock(&vmm->mutex); + } + + return ret; +} + +static int +nvkm_uvmm_mthd_pfnmap(struct nvkm_uvmm *uvmm, void *argv, u32 argc) +{ + struct nvkm_client *client = uvmm->object.client; + union { + struct nvif_vmm_pfnmap_v0 v0; + } *args = argv; + struct nvkm_vmm *vmm = uvmm->vmm; + int ret = -ENOSYS; + u64 addr, size, *phys; + u8 page; + + if (!(ret = nvif_unpack(ret, &argv, &argc, args->v0, 0, 0, true))) { + page = args->v0.page; + addr = args->v0.addr; + size = args->v0.size; + phys = args->v0.phys; + if (argc != (size >> page) * sizeof(args->v0.phys[0])) + return -EINVAL; + } else + return ret; + + if (!client->super) + return -ENOENT; + + if (size) { + mutex_lock(&vmm->mutex); + ret = nvkm_vmm_pfn_map(vmm, page, addr, size, phys); + mutex_unlock(&vmm->mutex); + } + + return ret; +} + +static int nvkm_uvmm_mthd_unmap(struct nvkm_uvmm *uvmm, void *argv, u32 argc) { struct nvkm_client *client = uvmm->object.client; @@ -78,7 +141,7 @@ nvkm_uvmm_mthd_unmap(struct nvkm_uvmm *uvmm, void *argv, u32 argc) goto done; } - nvkm_vmm_unmap_locked(vmm, vma); + nvkm_vmm_unmap_locked(vmm, vma, false); ret = 0; done: mutex_unlock(&vmm->mutex); @@ -124,6 +187,11 @@ nvkm_uvmm_mthd_map(struct nvkm_uvmm *uvmm, void *argv, u32 argc) goto fail; } + if (ret = -EINVAL, vma->mapped && !vma->memory) { + VMM_DEBUG(vmm, "pfnmap %016llx", addr); + goto fail; + } + if (ret = -EINVAL, vma->addr != addr || vma->size != size) { if (addr + size > vma->addr + vma->size || vma->memory || (vma->refd == NVKM_VMA_PAGE_NONE && !vma->mapref)) { @@ -134,23 +202,10 @@ nvkm_uvmm_mthd_map(struct nvkm_uvmm *uvmm, void *argv, u32 argc) goto fail; } - if (vma->addr != addr) { - const u64 tail = vma->size + vma->addr - addr; - if (ret = -ENOMEM, !(vma = nvkm_vma_tail(vma, tail))) - goto fail; - vma->part = true; - nvkm_vmm_node_insert(vmm, vma); - } - - if (vma->size != size) { - const u64 tail = vma->size - size; - struct nvkm_vma *tmp; - if (ret = -ENOMEM, !(tmp = nvkm_vma_tail(vma, tail))) { - nvkm_vmm_unmap_region(vmm, vma); - goto fail; - } - tmp->part = true; - nvkm_vmm_node_insert(vmm, tmp); + vma = nvkm_vmm_node_split(vmm, vma, addr, size); + if (!vma) { + ret = -ENOMEM; + goto fail; } } vma->busy = true; @@ -284,6 +339,15 @@ nvkm_uvmm_mthd(struct nvkm_object *object, u32 mthd, void *argv, u32 argc) case NVIF_VMM_V0_PUT : return nvkm_uvmm_mthd_put (uvmm, argv, argc); case NVIF_VMM_V0_MAP : return nvkm_uvmm_mthd_map (uvmm, argv, argc); case NVIF_VMM_V0_UNMAP : return nvkm_uvmm_mthd_unmap (uvmm, argv, argc); + case NVIF_VMM_V0_PFNMAP: return nvkm_uvmm_mthd_pfnmap(uvmm, argv, argc); + case NVIF_VMM_V0_PFNCLR: return nvkm_uvmm_mthd_pfnclr(uvmm, argv, argc); + case NVIF_VMM_V0_MTHD(0x00) ... NVIF_VMM_V0_MTHD(0x7f): + if (uvmm->vmm->func->mthd) { + return uvmm->vmm->func->mthd(uvmm->vmm, + uvmm->object.client, + mthd, argv, argc); + } + break; default: break; } @@ -317,8 +381,10 @@ nvkm_uvmm_new(const struct nvkm_oclass *oclass, void *argv, u32 argc, struct nvkm_uvmm *uvmm; int ret = -ENOSYS; u64 addr, size; + bool managed; if (!(ret = nvif_unpack(ret, &argv, &argc, args->v0, 0, 0, more))) { + managed = args->v0.managed != 0; addr = args->v0.addr; size = args->v0.size; } else @@ -330,7 +396,7 @@ nvkm_uvmm_new(const struct nvkm_oclass *oclass, void *argv, u32 argc, *pobject = &uvmm->object; if (!mmu->vmm) { - ret = mmu->func->vmm.ctor(mmu, addr, size, argv, argc, + ret = mmu->func->vmm.ctor(mmu, managed, addr, size, argv, argc, NULL, "user", &uvmm->vmm); if (ret) return ret; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c index 7459def78d50..fa93f964e6a4 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c @@ -255,11 +255,23 @@ nvkm_vmm_unref_sptes(struct nvkm_vmm_iter *it, struct nvkm_vmm_pt *pgt, } static bool -nvkm_vmm_unref_ptes(struct nvkm_vmm_iter *it, u32 ptei, u32 ptes) +nvkm_vmm_unref_ptes(struct nvkm_vmm_iter *it, bool pfn, u32 ptei, u32 ptes) { const struct nvkm_vmm_desc *desc = it->desc; const int type = desc->type == SPT; struct nvkm_vmm_pt *pgt = it->pt[0]; + bool dma; + + if (pfn) { + /* Need to clear PTE valid bits before we dma_unmap_page(). */ + dma = desc->func->pfn_clear(it->vmm, pgt->pt[type], ptei, ptes); + if (dma) { + /* GPU may have cached the PT, flush before unmap. */ + nvkm_vmm_flush_mark(it); + nvkm_vmm_flush(it); + desc->func->pfn_unmap(it->vmm, pgt->pt[type], ptei, ptes); + } + } /* Drop PTE references. */ pgt->refs[type] -= ptes; @@ -349,7 +361,7 @@ nvkm_vmm_ref_sptes(struct nvkm_vmm_iter *it, struct nvkm_vmm_pt *pgt, } static bool -nvkm_vmm_ref_ptes(struct nvkm_vmm_iter *it, u32 ptei, u32 ptes) +nvkm_vmm_ref_ptes(struct nvkm_vmm_iter *it, bool pfn, u32 ptei, u32 ptes) { const struct nvkm_vmm_desc *desc = it->desc; const int type = desc->type == SPT; @@ -379,7 +391,7 @@ nvkm_vmm_sparse_ptes(const struct nvkm_vmm_desc *desc, } static bool -nvkm_vmm_sparse_unref_ptes(struct nvkm_vmm_iter *it, u32 ptei, u32 ptes) +nvkm_vmm_sparse_unref_ptes(struct nvkm_vmm_iter *it, bool pfn, u32 ptei, u32 ptes) { struct nvkm_vmm_pt *pt = it->pt[0]; if (it->desc->type == PGD) @@ -387,14 +399,14 @@ nvkm_vmm_sparse_unref_ptes(struct nvkm_vmm_iter *it, u32 ptei, u32 ptes) else if (it->desc->type == LPT) memset(&pt->pte[ptei], 0x00, sizeof(pt->pte[0]) * ptes); - return nvkm_vmm_unref_ptes(it, ptei, ptes); + return nvkm_vmm_unref_ptes(it, pfn, ptei, ptes); } static bool -nvkm_vmm_sparse_ref_ptes(struct nvkm_vmm_iter *it, u32 ptei, u32 ptes) +nvkm_vmm_sparse_ref_ptes(struct nvkm_vmm_iter *it, bool pfn, u32 ptei, u32 ptes) { nvkm_vmm_sparse_ptes(it->desc, it->pt[0], ptei, ptes); - return nvkm_vmm_ref_ptes(it, ptei, ptes); + return nvkm_vmm_ref_ptes(it, pfn, ptei, ptes); } static bool @@ -487,8 +499,8 @@ nvkm_vmm_ref_swpt(struct nvkm_vmm_iter *it, struct nvkm_vmm_pt *pgd, u32 pdei) static inline u64 nvkm_vmm_iter(struct nvkm_vmm *vmm, const struct nvkm_vmm_page *page, - u64 addr, u64 size, const char *name, bool ref, - bool (*REF_PTES)(struct nvkm_vmm_iter *, u32, u32), + u64 addr, u64 size, const char *name, bool ref, bool pfn, + bool (*REF_PTES)(struct nvkm_vmm_iter *, bool pfn, u32, u32), nvkm_vmm_pte_func MAP_PTES, struct nvkm_vmm_map *map, nvkm_vmm_pxe_func CLR_PTES) { @@ -548,7 +560,7 @@ nvkm_vmm_iter(struct nvkm_vmm *vmm, const struct nvkm_vmm_page *page, } /* Handle PTE updates. */ - if (!REF_PTES || REF_PTES(&it, ptei, ptes)) { + if (!REF_PTES || REF_PTES(&it, pfn, ptei, ptes)) { struct nvkm_mmu_pt *pt = pgt->pt[type]; if (MAP_PTES || CLR_PTES) { if (MAP_PTES) @@ -590,7 +602,7 @@ static void nvkm_vmm_ptes_sparse_put(struct nvkm_vmm *vmm, const struct nvkm_vmm_page *page, u64 addr, u64 size) { - nvkm_vmm_iter(vmm, page, addr, size, "sparse unref", false, + nvkm_vmm_iter(vmm, page, addr, size, "sparse unref", false, false, nvkm_vmm_sparse_unref_ptes, NULL, NULL, page->desc->func->invalid ? page->desc->func->invalid : page->desc->func->unmap); @@ -602,8 +614,8 @@ nvkm_vmm_ptes_sparse_get(struct nvkm_vmm *vmm, const struct nvkm_vmm_page *page, { if ((page->type & NVKM_VMM_PAGE_SPARSE)) { u64 fail = nvkm_vmm_iter(vmm, page, addr, size, "sparse ref", - true, nvkm_vmm_sparse_ref_ptes, NULL, - NULL, page->desc->func->sparse); + true, false, nvkm_vmm_sparse_ref_ptes, + NULL, NULL, page->desc->func->sparse); if (fail != ~0ULL) { if ((size = fail - addr)) nvkm_vmm_ptes_sparse_put(vmm, page, addr, size); @@ -666,11 +678,11 @@ nvkm_vmm_ptes_sparse(struct nvkm_vmm *vmm, u64 addr, u64 size, bool ref) static void nvkm_vmm_ptes_unmap_put(struct nvkm_vmm *vmm, const struct nvkm_vmm_page *page, - u64 addr, u64 size, bool sparse) + u64 addr, u64 size, bool sparse, bool pfn) { const struct nvkm_vmm_desc_func *func = page->desc->func; nvkm_vmm_iter(vmm, page, addr, size, "unmap + unref", - false, nvkm_vmm_unref_ptes, NULL, NULL, + false, pfn, nvkm_vmm_unref_ptes, NULL, NULL, sparse ? func->sparse : func->invalid ? func->invalid : func->unmap); } @@ -681,10 +693,10 @@ nvkm_vmm_ptes_get_map(struct nvkm_vmm *vmm, const struct nvkm_vmm_page *page, nvkm_vmm_pte_func func) { u64 fail = nvkm_vmm_iter(vmm, page, addr, size, "ref + map", true, - nvkm_vmm_ref_ptes, func, map, NULL); + false, nvkm_vmm_ref_ptes, func, map, NULL); if (fail != ~0ULL) { if ((size = fail - addr)) - nvkm_vmm_ptes_unmap_put(vmm, page, addr, size, false); + nvkm_vmm_ptes_unmap_put(vmm, page, addr, size, false, false); return -ENOMEM; } return 0; @@ -692,10 +704,11 @@ nvkm_vmm_ptes_get_map(struct nvkm_vmm *vmm, const struct nvkm_vmm_page *page, static void nvkm_vmm_ptes_unmap(struct nvkm_vmm *vmm, const struct nvkm_vmm_page *page, - u64 addr, u64 size, bool sparse) + u64 addr, u64 size, bool sparse, bool pfn) { const struct nvkm_vmm_desc_func *func = page->desc->func; - nvkm_vmm_iter(vmm, page, addr, size, "unmap", false, NULL, NULL, NULL, + nvkm_vmm_iter(vmm, page, addr, size, "unmap", false, pfn, + NULL, NULL, NULL, sparse ? func->sparse : func->invalid ? func->invalid : func->unmap); } @@ -705,7 +718,7 @@ nvkm_vmm_ptes_map(struct nvkm_vmm *vmm, const struct nvkm_vmm_page *page, u64 addr, u64 size, struct nvkm_vmm_map *map, nvkm_vmm_pte_func func) { - nvkm_vmm_iter(vmm, page, addr, size, "map", false, + nvkm_vmm_iter(vmm, page, addr, size, "map", false, false, NULL, func, map, NULL); } @@ -713,7 +726,7 @@ static void nvkm_vmm_ptes_put(struct nvkm_vmm *vmm, const struct nvkm_vmm_page *page, u64 addr, u64 size) { - nvkm_vmm_iter(vmm, page, addr, size, "unref", false, + nvkm_vmm_iter(vmm, page, addr, size, "unref", false, false, nvkm_vmm_unref_ptes, NULL, NULL, NULL); } @@ -721,7 +734,7 @@ static int nvkm_vmm_ptes_get(struct nvkm_vmm *vmm, const struct nvkm_vmm_page *page, u64 addr, u64 size) { - u64 fail = nvkm_vmm_iter(vmm, page, addr, size, "ref", true, + u64 fail = nvkm_vmm_iter(vmm, page, addr, size, "ref", true, false, nvkm_vmm_ref_ptes, NULL, NULL, NULL); if (fail != ~0ULL) { if (fail != addr) @@ -763,10 +776,25 @@ nvkm_vma_tail(struct nvkm_vma *vma, u64 tail) new->part = vma->part; new->user = vma->user; new->busy = vma->busy; + new->mapped = vma->mapped; list_add(&new->head, &vma->head); return new; } +static inline void +nvkm_vmm_free_remove(struct nvkm_vmm *vmm, struct nvkm_vma *vma) +{ + rb_erase(&vma->tree, &vmm->free); +} + +static inline void +nvkm_vmm_free_delete(struct nvkm_vmm *vmm, struct nvkm_vma *vma) +{ + nvkm_vmm_free_remove(vmm, vma); + list_del(&vma->head); + kfree(vma); +} + static void nvkm_vmm_free_insert(struct nvkm_vmm *vmm, struct nvkm_vma *vma) { @@ -795,7 +823,21 @@ nvkm_vmm_free_insert(struct nvkm_vmm *vmm, struct nvkm_vma *vma) rb_insert_color(&vma->tree, &vmm->free); } -void +static inline void +nvkm_vmm_node_remove(struct nvkm_vmm *vmm, struct nvkm_vma *vma) +{ + rb_erase(&vma->tree, &vmm->root); +} + +static inline void +nvkm_vmm_node_delete(struct nvkm_vmm *vmm, struct nvkm_vma *vma) +{ + nvkm_vmm_node_remove(vmm, vma); + list_del(&vma->head); + kfree(vma); +} + +static void nvkm_vmm_node_insert(struct nvkm_vmm *vmm, struct nvkm_vma *vma) { struct rb_node **ptr = &vmm->root.rb_node; @@ -834,12 +876,113 @@ nvkm_vmm_node_search(struct nvkm_vmm *vmm, u64 addr) return NULL; } +#define node(root, dir) (((root)->head.dir == &vmm->list) ? NULL : \ + list_entry((root)->head.dir, struct nvkm_vma, head)) + +static struct nvkm_vma * +nvkm_vmm_node_merge(struct nvkm_vmm *vmm, struct nvkm_vma *prev, + struct nvkm_vma *vma, struct nvkm_vma *next, u64 size) +{ + if (next) { + if (vma->size == size) { + vma->size += next->size; + nvkm_vmm_node_delete(vmm, next); + if (prev) { + prev->size += vma->size; + nvkm_vmm_node_delete(vmm, vma); + return prev; + } + return vma; + } + BUG_ON(prev); + + nvkm_vmm_node_remove(vmm, next); + vma->size -= size; + next->addr -= size; + next->size += size; + nvkm_vmm_node_insert(vmm, next); + return next; + } + + if (prev) { + if (vma->size != size) { + nvkm_vmm_node_remove(vmm, vma); + prev->size += size; + vma->addr += size; + vma->size -= size; + nvkm_vmm_node_insert(vmm, vma); + } else { + prev->size += vma->size; + nvkm_vmm_node_delete(vmm, vma); + } + return prev; + } + + return vma; +} + +struct nvkm_vma * +nvkm_vmm_node_split(struct nvkm_vmm *vmm, + struct nvkm_vma *vma, u64 addr, u64 size) +{ + struct nvkm_vma *prev = NULL; + + if (vma->addr != addr) { + prev = vma; + if (!(vma = nvkm_vma_tail(vma, vma->size + vma->addr - addr))) + return NULL; + vma->part = true; + nvkm_vmm_node_insert(vmm, vma); + } + + if (vma->size != size) { + struct nvkm_vma *tmp; + if (!(tmp = nvkm_vma_tail(vma, vma->size - size))) { + nvkm_vmm_node_merge(vmm, prev, vma, NULL, vma->size); + return NULL; + } + tmp->part = true; + nvkm_vmm_node_insert(vmm, tmp); + } + + return vma; +} + +static void +nvkm_vma_dump(struct nvkm_vma *vma) +{ + printk(KERN_ERR "%016llx %016llx %c%c%c%c%c%c%c%c%c %p\n", + vma->addr, (u64)vma->size, + vma->used ? '-' : 'F', + vma->mapref ? 'R' : '-', + vma->sparse ? 'S' : '-', + vma->page != NVKM_VMA_PAGE_NONE ? '0' + vma->page : '-', + vma->refd != NVKM_VMA_PAGE_NONE ? '0' + vma->refd : '-', + vma->part ? 'P' : '-', + vma->user ? 'U' : '-', + vma->busy ? 'B' : '-', + vma->mapped ? 'M' : '-', + vma->memory); +} + +static void +nvkm_vmm_dump(struct nvkm_vmm *vmm) +{ + struct nvkm_vma *vma; + list_for_each_entry(vma, &vmm->list, head) { + nvkm_vma_dump(vma); + } +} + static void nvkm_vmm_dtor(struct nvkm_vmm *vmm) { struct nvkm_vma *vma; struct rb_node *node; + if (0) + nvkm_vmm_dump(vmm); + while ((node = rb_first(&vmm->root))) { struct nvkm_vma *vma = rb_entry(node, typeof(*vma), tree); nvkm_vmm_put(vmm, &vma); @@ -872,16 +1015,32 @@ nvkm_vmm_dtor(struct nvkm_vmm *vmm) } } +static int +nvkm_vmm_ctor_managed(struct nvkm_vmm *vmm, u64 addr, u64 size) +{ + struct nvkm_vma *vma; + if (!(vma = nvkm_vma_new(addr, size))) + return -ENOMEM; + vma->mapref = true; + vma->sparse = false; + vma->used = true; + vma->user = true; + nvkm_vmm_node_insert(vmm, vma); + list_add_tail(&vma->head, &vmm->list); + return 0; +} + int nvkm_vmm_ctor(const struct nvkm_vmm_func *func, struct nvkm_mmu *mmu, - u32 pd_header, u64 addr, u64 size, struct lock_class_key *key, - const char *name, struct nvkm_vmm *vmm) + u32 pd_header, bool managed, u64 addr, u64 size, + struct lock_class_key *key, const char *name, + struct nvkm_vmm *vmm) { static struct lock_class_key _key; const struct nvkm_vmm_page *page = func->page; const struct nvkm_vmm_desc *desc; struct nvkm_vma *vma; - int levels, bits = 0; + int levels, bits = 0, ret; vmm->func = func; vmm->mmu = mmu; @@ -909,11 +1068,6 @@ nvkm_vmm_ctor(const struct nvkm_vmm_func *func, struct nvkm_mmu *mmu, if (WARN_ON(levels > NVKM_VMM_LEVELS_MAX)) return -EINVAL; - vmm->start = addr; - vmm->limit = size ? (addr + size) : (1ULL << bits); - if (vmm->start > vmm->limit || vmm->limit > (1ULL << bits)) - return -EINVAL; - /* Allocate top-level page table. */ vmm->pd = nvkm_vmm_pt_new(desc, false, NULL); if (!vmm->pd) @@ -936,67 +1090,273 @@ nvkm_vmm_ctor(const struct nvkm_vmm_func *func, struct nvkm_mmu *mmu, vmm->free = RB_ROOT; vmm->root = RB_ROOT; - if (!(vma = nvkm_vma_new(vmm->start, vmm->limit - vmm->start))) - return -ENOMEM; + if (managed) { + /* Address-space will be managed by the client for the most + * part, except for a specified area where NVKM allocations + * are allowed to be placed. + */ + vmm->start = 0; + vmm->limit = 1ULL << bits; + if (addr + size < addr || addr + size > vmm->limit) + return -EINVAL; + + /* Client-managed area before the NVKM-managed area. */ + if (addr && (ret = nvkm_vmm_ctor_managed(vmm, 0, addr))) + return ret; + + /* NVKM-managed area. */ + if (size) { + if (!(vma = nvkm_vma_new(addr, size))) + return -ENOMEM; + nvkm_vmm_free_insert(vmm, vma); + list_add_tail(&vma->head, &vmm->list); + } + + /* Client-managed area after the NVKM-managed area. */ + addr = addr + size; + size = vmm->limit - addr; + if (size && (ret = nvkm_vmm_ctor_managed(vmm, addr, size))) + return ret; + } else { + /* Address-space fully managed by NVKM, requiring calls to + * nvkm_vmm_get()/nvkm_vmm_put() to allocate address-space. + */ + vmm->start = addr; + vmm->limit = size ? (addr + size) : (1ULL << bits); + if (vmm->start > vmm->limit || vmm->limit > (1ULL << bits)) + return -EINVAL; + + if (!(vma = nvkm_vma_new(vmm->start, vmm->limit - vmm->start))) + return -ENOMEM; + + nvkm_vmm_free_insert(vmm, vma); + list_add(&vma->head, &vmm->list); + } - nvkm_vmm_free_insert(vmm, vma); - list_add(&vma->head, &vmm->list); return 0; } int nvkm_vmm_new_(const struct nvkm_vmm_func *func, struct nvkm_mmu *mmu, - u32 hdr, u64 addr, u64 size, struct lock_class_key *key, - const char *name, struct nvkm_vmm **pvmm) + u32 hdr, bool managed, u64 addr, u64 size, + struct lock_class_key *key, const char *name, + struct nvkm_vmm **pvmm) { if (!(*pvmm = kzalloc(sizeof(**pvmm), GFP_KERNEL))) return -ENOMEM; - return nvkm_vmm_ctor(func, mmu, hdr, addr, size, key, name, *pvmm); + return nvkm_vmm_ctor(func, mmu, hdr, managed, addr, size, key, name, *pvmm); } -#define node(root, dir) ((root)->head.dir == &vmm->list) ? NULL : \ - list_entry((root)->head.dir, struct nvkm_vma, head) +static struct nvkm_vma * +nvkm_vmm_pfn_split_merge(struct nvkm_vmm *vmm, struct nvkm_vma *vma, + u64 addr, u64 size, u8 page, bool map) +{ + struct nvkm_vma *prev = NULL; + struct nvkm_vma *next = NULL; -void -nvkm_vmm_unmap_region(struct nvkm_vmm *vmm, struct nvkm_vma *vma) + if (vma->addr == addr && vma->part && (prev = node(vma, prev))) { + if (prev->memory || prev->mapped != map) + prev = NULL; + } + + if (vma->addr + vma->size == addr + size && (next = node(vma, next))) { + if (!next->part || + next->memory || next->mapped != map) + next = NULL; + } + + if (prev || next) + return nvkm_vmm_node_merge(vmm, prev, vma, next, size); + return nvkm_vmm_node_split(vmm, vma, addr, size); +} + +int +nvkm_vmm_pfn_unmap(struct nvkm_vmm *vmm, u64 addr, u64 size) { + struct nvkm_vma *vma = nvkm_vmm_node_search(vmm, addr); struct nvkm_vma *next; + u64 limit = addr + size; + u64 start = addr; - nvkm_memory_tags_put(vma->memory, vmm->mmu->subdev.device, &vma->tags); - nvkm_memory_unref(&vma->memory); + if (!vma) + return -EINVAL; - if (vma->part) { - struct nvkm_vma *prev = node(vma, prev); - if (!prev->memory) { - prev->size += vma->size; - rb_erase(&vma->tree, &vmm->root); - list_del(&vma->head); - kfree(vma); - vma = prev; + do { + if (!vma->mapped || vma->memory) + continue; + + size = min(limit - start, vma->size - (start - vma->addr)); + + nvkm_vmm_ptes_unmap_put(vmm, &vmm->func->page[vma->refd], + start, size, false, true); + + next = nvkm_vmm_pfn_split_merge(vmm, vma, start, size, 0, false); + if (!WARN_ON(!next)) { + vma = next; + vma->refd = NVKM_VMA_PAGE_NONE; + vma->mapped = false; } + } while ((vma = node(vma, next)) && (start = vma->addr) < limit); + + return 0; +} + +/*TODO: + * - Avoid PT readback (for dma_unmap etc), this might end up being dealt + * with inside HMM, which would be a lot nicer for us to deal with. + * - Multiple page sizes (particularly for huge page support). + * - Support for systems without a 4KiB page size. + */ +int +nvkm_vmm_pfn_map(struct nvkm_vmm *vmm, u8 shift, u64 addr, u64 size, u64 *pfn) +{ + const struct nvkm_vmm_page *page = vmm->func->page; + struct nvkm_vma *vma, *tmp; + u64 limit = addr + size; + u64 start = addr; + int pm = size >> shift; + int pi = 0; + + /* Only support mapping where the page size of the incoming page + * array matches a page size available for direct mapping. + */ + while (page->shift && page->shift != shift && + page->desc->func->pfn == NULL) + page++; + + if (!page->shift || !IS_ALIGNED(addr, 1ULL << shift) || + !IS_ALIGNED(size, 1ULL << shift) || + addr + size < addr || addr + size > vmm->limit) { + VMM_DEBUG(vmm, "paged map %d %d %016llx %016llx\n", + shift, page->shift, addr, size); + return -EINVAL; } - next = node(vma, next); - if (next && next->part) { - if (!next->memory) { - vma->size += next->size; - rb_erase(&next->tree, &vmm->root); - list_del(&next->head); - kfree(next); + if (!(vma = nvkm_vmm_node_search(vmm, addr))) + return -ENOENT; + + do { + bool map = !!(pfn[pi] & NVKM_VMM_PFN_V); + bool mapped = vma->mapped; + u64 size = limit - start; + u64 addr = start; + int pn, ret = 0; + + /* Narrow the operation window to cover a single action (page + * should be mapped or not) within a single VMA. + */ + for (pn = 0; pi + pn < pm; pn++) { + if (map != !!(pfn[pi + pn] & NVKM_VMM_PFN_V)) + break; } - } + size = min_t(u64, size, pn << page->shift); + size = min_t(u64, size, vma->size + vma->addr - addr); + + /* Reject any operation to unmanaged regions, and areas that + * have nvkm_memory objects mapped in them already. + */ + if (!vma->mapref || vma->memory) { + ret = -EINVAL; + goto next; + } + + /* In order to both properly refcount GPU page tables, and + * prevent "normal" mappings and these direct mappings from + * interfering with each other, we need to track contiguous + * ranges that have been mapped with this interface. + * + * Here we attempt to either split an existing VMA so we're + * able to flag the region as either unmapped/mapped, or to + * merge with adjacent VMAs that are already compatible. + * + * If the region is already compatible, nothing is required. + */ + if (map != mapped) { + tmp = nvkm_vmm_pfn_split_merge(vmm, vma, addr, size, + page - + vmm->func->page, map); + if (WARN_ON(!tmp)) { + ret = -ENOMEM; + goto next; + } + + if ((tmp->mapped = map)) + tmp->refd = page - vmm->func->page; + else + tmp->refd = NVKM_VMA_PAGE_NONE; + vma = tmp; + } + + /* Update HW page tables. */ + if (map) { + struct nvkm_vmm_map args; + args.page = page; + args.pfn = &pfn[pi]; + + if (!mapped) { + ret = nvkm_vmm_ptes_get_map(vmm, page, addr, + size, &args, page-> + desc->func->pfn); + } else { + nvkm_vmm_ptes_map(vmm, page, addr, size, &args, + page->desc->func->pfn); + } + } else { + if (mapped) { + nvkm_vmm_ptes_unmap_put(vmm, page, addr, size, + false, true); + } + } + +next: + /* Iterate to next operation. */ + if (vma->addr + vma->size == addr + size) + vma = node(vma, next); + start += size; + + if (ret) { + /* Failure is signalled by clearing the valid bit on + * any PFN that couldn't be modified as requested. + */ + while (size) { + pfn[pi++] = NVKM_VMM_PFN_NONE; + size -= 1 << page->shift; + } + } else { + pi += size >> page->shift; + } + } while (vma && start < limit); + + return 0; } void -nvkm_vmm_unmap_locked(struct nvkm_vmm *vmm, struct nvkm_vma *vma) +nvkm_vmm_unmap_region(struct nvkm_vmm *vmm, struct nvkm_vma *vma) +{ + struct nvkm_vma *prev = NULL; + struct nvkm_vma *next; + + nvkm_memory_tags_put(vma->memory, vmm->mmu->subdev.device, &vma->tags); + nvkm_memory_unref(&vma->memory); + vma->mapped = false; + + if (vma->part && (prev = node(vma, prev)) && prev->mapped) + prev = NULL; + if ((next = node(vma, next)) && (!next->part || next->mapped)) + next = NULL; + nvkm_vmm_node_merge(vmm, prev, vma, next, vma->size); +} + +void +nvkm_vmm_unmap_locked(struct nvkm_vmm *vmm, struct nvkm_vma *vma, bool pfn) { const struct nvkm_vmm_page *page = &vmm->func->page[vma->refd]; if (vma->mapref) { - nvkm_vmm_ptes_unmap_put(vmm, page, vma->addr, vma->size, vma->sparse); + nvkm_vmm_ptes_unmap_put(vmm, page, vma->addr, vma->size, vma->sparse, pfn); vma->refd = NVKM_VMA_PAGE_NONE; } else { - nvkm_vmm_ptes_unmap(vmm, page, vma->addr, vma->size, vma->sparse); + nvkm_vmm_ptes_unmap(vmm, page, vma->addr, vma->size, vma->sparse, pfn); } nvkm_vmm_unmap_region(vmm, vma); @@ -1007,7 +1367,7 @@ nvkm_vmm_unmap(struct nvkm_vmm *vmm, struct nvkm_vma *vma) { if (vma->memory) { mutex_lock(&vmm->mutex); - nvkm_vmm_unmap_locked(vmm, vma); + nvkm_vmm_unmap_locked(vmm, vma, false); mutex_unlock(&vmm->mutex); } } @@ -1141,6 +1501,7 @@ nvkm_vmm_map_locked(struct nvkm_vmm *vmm, struct nvkm_vma *vma, nvkm_memory_tags_put(vma->memory, vmm->mmu->subdev.device, &vma->tags); nvkm_memory_unref(&vma->memory); vma->memory = nvkm_memory_ref(map->memory); + vma->mapped = true; vma->tags = map->tags; return 0; } @@ -1163,18 +1524,14 @@ nvkm_vmm_put_region(struct nvkm_vmm *vmm, struct nvkm_vma *vma) struct nvkm_vma *prev, *next; if ((prev = node(vma, prev)) && !prev->used) { - rb_erase(&prev->tree, &vmm->free); - list_del(&prev->head); vma->addr = prev->addr; vma->size += prev->size; - kfree(prev); + nvkm_vmm_free_delete(vmm, prev); } if ((next = node(vma, next)) && !next->used) { - rb_erase(&next->tree, &vmm->free); - list_del(&next->head); vma->size += next->size; - kfree(next); + nvkm_vmm_free_delete(vmm, next); } nvkm_vmm_free_insert(vmm, vma); @@ -1190,14 +1547,16 @@ nvkm_vmm_put_locked(struct nvkm_vmm *vmm, struct nvkm_vma *vma) if (vma->mapref || !vma->sparse) { do { - const bool map = next->memory != NULL; + const bool mem = next->memory != NULL; + const bool map = next->mapped; const u8 refd = next->refd; const u64 addr = next->addr; u64 size = next->size; /* Merge regions that are in the same state. */ while ((next = node(next, next)) && next->part && - (next->memory != NULL) == map && + (next->mapped == map) && + (next->memory != NULL) == mem && (next->refd == refd)) size += next->size; @@ -1207,7 +1566,8 @@ nvkm_vmm_put_locked(struct nvkm_vmm *vmm, struct nvkm_vma *vma) * the page tree. */ nvkm_vmm_ptes_unmap_put(vmm, &page[refd], addr, - size, vma->sparse); + size, vma->sparse, + !mem); } else if (refd != NVKM_VMA_PAGE_NONE) { /* Drop allocation-time PTE references. */ @@ -1222,7 +1582,7 @@ nvkm_vmm_put_locked(struct nvkm_vmm *vmm, struct nvkm_vma *vma) */ next = vma; do { - if (next->memory) + if (next->mapped) nvkm_vmm_unmap_region(vmm, next); } while ((next = node(vma, next)) && next->part); @@ -1250,7 +1610,7 @@ nvkm_vmm_put_locked(struct nvkm_vmm *vmm, struct nvkm_vma *vma) } /* Remove VMA from the list of allocated nodes. */ - rb_erase(&vma->tree, &vmm->root); + nvkm_vmm_node_remove(vmm, vma); /* Merge VMA back into the free list. */ vma->page = NVKM_VMA_PAGE_NONE; @@ -1357,7 +1717,7 @@ nvkm_vmm_get_locked(struct nvkm_vmm *vmm, bool getref, bool mapref, bool sparse, tail = ALIGN_DOWN(tail, vmm->func->page_block); if (addr <= tail && tail - addr >= size) { - rb_erase(&this->tree, &vmm->free); + nvkm_vmm_free_remove(vmm, this); vma = this; break; } @@ -1443,7 +1803,7 @@ nvkm_vmm_join(struct nvkm_vmm *vmm, struct nvkm_memory *inst) } static bool -nvkm_vmm_boot_ptes(struct nvkm_vmm_iter *it, u32 ptei, u32 ptes) +nvkm_vmm_boot_ptes(struct nvkm_vmm_iter *it, bool pfn, u32 ptei, u32 ptes) { const struct nvkm_vmm_desc *desc = it->desc; const int type = desc->type == SPT; @@ -1465,7 +1825,7 @@ nvkm_vmm_boot(struct nvkm_vmm *vmm) if (ret) return ret; - nvkm_vmm_iter(vmm, page, vmm->start, limit, "bootstrap", false, + nvkm_vmm_iter(vmm, page, vmm->start, limit, "bootstrap", false, false, nvkm_vmm_boot_ptes, NULL, NULL, NULL); vmm->bootstrapped = true; return 0; @@ -1505,7 +1865,8 @@ nvkm_vmm_new(struct nvkm_device *device, u64 addr, u64 size, void *argv, struct nvkm_mmu *mmu = device->mmu; struct nvkm_vmm *vmm = NULL; int ret; - ret = mmu->func->vmm.ctor(mmu, addr, size, argv, argc, key, name, &vmm); + ret = mmu->func->vmm.ctor(mmu, false, addr, size, argv, argc, + key, name, &vmm); if (ret) nvkm_vmm_unref(&vmm); *pvmm = vmm; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.h b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.h index 1a3b0a3724ca..5e55ecbd8005 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.h +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.h @@ -67,6 +67,10 @@ struct nvkm_vmm_desc_func { nvkm_vmm_pte_func mem; nvkm_vmm_pte_func dma; nvkm_vmm_pte_func sgl; + + nvkm_vmm_pte_func pfn; + bool (*pfn_clear)(struct nvkm_vmm *, struct nvkm_mmu_pt *, u32 ptei, u32 ptes); + nvkm_vmm_pxe_func pfn_unmap; }; extern const struct nvkm_vmm_desc_func gf100_vmm_pgd; @@ -141,6 +145,11 @@ struct nvkm_vmm_func { struct nvkm_vmm_map *); void (*flush)(struct nvkm_vmm *, int depth); + int (*mthd)(struct nvkm_vmm *, struct nvkm_client *, + u32 mthd, void *argv, u32 argc); + + void (*invalidate_pdb)(struct nvkm_vmm *, u64 addr); + u64 page_block; const struct nvkm_vmm_page page[]; }; @@ -151,24 +160,38 @@ struct nvkm_vmm_join { }; int nvkm_vmm_new_(const struct nvkm_vmm_func *, struct nvkm_mmu *, - u32 pd_header, u64 addr, u64 size, struct lock_class_key *, - const char *name, struct nvkm_vmm **); + u32 pd_header, bool managed, u64 addr, u64 size, + struct lock_class_key *, const char *name, + struct nvkm_vmm **); int nvkm_vmm_ctor(const struct nvkm_vmm_func *, struct nvkm_mmu *, - u32 pd_header, u64 addr, u64 size, struct lock_class_key *, - const char *name, struct nvkm_vmm *); + u32 pd_header, bool managed, u64 addr, u64 size, + struct lock_class_key *, const char *name, struct nvkm_vmm *); struct nvkm_vma *nvkm_vmm_node_search(struct nvkm_vmm *, u64 addr); +struct nvkm_vma *nvkm_vmm_node_split(struct nvkm_vmm *, struct nvkm_vma *, + u64 addr, u64 size); int nvkm_vmm_get_locked(struct nvkm_vmm *, bool getref, bool mapref, bool sparse, u8 page, u8 align, u64 size, struct nvkm_vma **pvma); void nvkm_vmm_put_locked(struct nvkm_vmm *, struct nvkm_vma *); -void nvkm_vmm_unmap_locked(struct nvkm_vmm *, struct nvkm_vma *); -void nvkm_vmm_unmap_region(struct nvkm_vmm *vmm, struct nvkm_vma *vma); +void nvkm_vmm_unmap_locked(struct nvkm_vmm *, struct nvkm_vma *, bool pfn); +void nvkm_vmm_unmap_region(struct nvkm_vmm *, struct nvkm_vma *); + +#define NVKM_VMM_PFN_ADDR 0xfffffffffffff000ULL +#define NVKM_VMM_PFN_ADDR_SHIFT 12 +#define NVKM_VMM_PFN_APER 0x00000000000000f0ULL +#define NVKM_VMM_PFN_HOST 0x0000000000000000ULL +#define NVKM_VMM_PFN_VRAM 0x0000000000000010ULL +#define NVKM_VMM_PFN_W 0x0000000000000002ULL +#define NVKM_VMM_PFN_V 0x0000000000000001ULL +#define NVKM_VMM_PFN_NONE 0x0000000000000000ULL + +int nvkm_vmm_pfn_map(struct nvkm_vmm *, u8 page, u64 addr, u64 size, u64 *pfn); +int nvkm_vmm_pfn_unmap(struct nvkm_vmm *, u64 addr, u64 size); struct nvkm_vma *nvkm_vma_tail(struct nvkm_vma *, u64 tail); -void nvkm_vmm_node_insert(struct nvkm_vmm *, struct nvkm_vma *); int nv04_vmm_new_(const struct nvkm_vmm_func *, struct nvkm_mmu *, u32, - u64, u64, void *, u32, struct lock_class_key *, + bool, u64, u64, void *, u32, struct lock_class_key *, const char *, struct nvkm_vmm **); int nv04_vmm_valid(struct nvkm_vmm *, void *, u32, struct nvkm_vmm_map *); @@ -178,65 +201,76 @@ int nv50_vmm_valid(struct nvkm_vmm *, void *, u32, struct nvkm_vmm_map *); void nv50_vmm_flush(struct nvkm_vmm *, int); int gf100_vmm_new_(const struct nvkm_vmm_func *, const struct nvkm_vmm_func *, - struct nvkm_mmu *, u64, u64, void *, u32, + struct nvkm_mmu *, bool, u64, u64, void *, u32, struct lock_class_key *, const char *, struct nvkm_vmm **); int gf100_vmm_join_(struct nvkm_vmm *, struct nvkm_memory *, u64 base); int gf100_vmm_join(struct nvkm_vmm *, struct nvkm_memory *); void gf100_vmm_part(struct nvkm_vmm *, struct nvkm_memory *); int gf100_vmm_aper(enum nvkm_memory_target); int gf100_vmm_valid(struct nvkm_vmm *, void *, u32, struct nvkm_vmm_map *); -void gf100_vmm_flush_(struct nvkm_vmm *, int); void gf100_vmm_flush(struct nvkm_vmm *, int); +void gf100_vmm_invalidate(struct nvkm_vmm *, u32 type); +void gf100_vmm_invalidate_pdb(struct nvkm_vmm *, u64 addr); int gk20a_vmm_aper(enum nvkm_memory_target); int gm200_vmm_new_(const struct nvkm_vmm_func *, const struct nvkm_vmm_func *, - struct nvkm_mmu *, u64, u64, void *, u32, + struct nvkm_mmu *, bool, u64, u64, void *, u32, struct lock_class_key *, const char *, struct nvkm_vmm **); int gm200_vmm_join_(struct nvkm_vmm *, struct nvkm_memory *, u64 base); int gm200_vmm_join(struct nvkm_vmm *, struct nvkm_memory *); +int gp100_vmm_new_(const struct nvkm_vmm_func *, + struct nvkm_mmu *, bool, u64, u64, void *, u32, + struct lock_class_key *, const char *, struct nvkm_vmm **); int gp100_vmm_join(struct nvkm_vmm *, struct nvkm_memory *); int gp100_vmm_valid(struct nvkm_vmm *, void *, u32, struct nvkm_vmm_map *); void gp100_vmm_flush(struct nvkm_vmm *, int); +int gp100_vmm_mthd(struct nvkm_vmm *, struct nvkm_client *, u32, void *, u32); +void gp100_vmm_invalidate_pdb(struct nvkm_vmm *, u64 addr); + +int gv100_vmm_join(struct nvkm_vmm *, struct nvkm_memory *); -int nv04_vmm_new(struct nvkm_mmu *, u64, u64, void *, u32, +int nv04_vmm_new(struct nvkm_mmu *, bool, u64, u64, void *, u32, struct lock_class_key *, const char *, struct nvkm_vmm **); -int nv41_vmm_new(struct nvkm_mmu *, u64, u64, void *, u32, +int nv41_vmm_new(struct nvkm_mmu *, bool, u64, u64, void *, u32, struct lock_class_key *, const char *, struct nvkm_vmm **); -int nv44_vmm_new(struct nvkm_mmu *, u64, u64, void *, u32, +int nv44_vmm_new(struct nvkm_mmu *, bool, u64, u64, void *, u32, struct lock_class_key *, const char *, struct nvkm_vmm **); -int nv50_vmm_new(struct nvkm_mmu *, u64, u64, void *, u32, +int nv50_vmm_new(struct nvkm_mmu *, bool, u64, u64, void *, u32, struct lock_class_key *, const char *, struct nvkm_vmm **); -int mcp77_vmm_new(struct nvkm_mmu *, u64, u64, void *, u32, +int mcp77_vmm_new(struct nvkm_mmu *, bool, u64, u64, void *, u32, struct lock_class_key *, const char *, struct nvkm_vmm **); -int g84_vmm_new(struct nvkm_mmu *, u64, u64, void *, u32, +int g84_vmm_new(struct nvkm_mmu *, bool, u64, u64, void *, u32, struct lock_class_key *, const char *, struct nvkm_vmm **); -int gf100_vmm_new(struct nvkm_mmu *, u64, u64, void *, u32, +int gf100_vmm_new(struct nvkm_mmu *, bool, u64, u64, void *, u32, struct lock_class_key *, const char *, struct nvkm_vmm **); -int gk104_vmm_new(struct nvkm_mmu *, u64, u64, void *, u32, +int gk104_vmm_new(struct nvkm_mmu *, bool, u64, u64, void *, u32, struct lock_class_key *, const char *, struct nvkm_vmm **); -int gk20a_vmm_new(struct nvkm_mmu *, u64, u64, void *, u32, +int gk20a_vmm_new(struct nvkm_mmu *, bool, u64, u64, void *, u32, struct lock_class_key *, const char *, struct nvkm_vmm **); -int gm200_vmm_new_fixed(struct nvkm_mmu *, u64, u64, void *, u32, +int gm200_vmm_new_fixed(struct nvkm_mmu *, bool, u64, u64, void *, u32, struct lock_class_key *, const char *, struct nvkm_vmm **); -int gm200_vmm_new(struct nvkm_mmu *, u64, u64, void *, u32, +int gm200_vmm_new(struct nvkm_mmu *, bool, u64, u64, void *, u32, struct lock_class_key *, const char *, struct nvkm_vmm **); -int gm20b_vmm_new_fixed(struct nvkm_mmu *, u64, u64, void *, u32, +int gm20b_vmm_new_fixed(struct nvkm_mmu *, bool, u64, u64, void *, u32, struct lock_class_key *, const char *, struct nvkm_vmm **); -int gm20b_vmm_new(struct nvkm_mmu *, u64, u64, void *, u32, +int gm20b_vmm_new(struct nvkm_mmu *, bool, u64, u64, void *, u32, + struct lock_class_key *, const char *, + struct nvkm_vmm **); +int gp100_vmm_new(struct nvkm_mmu *, bool, u64, u64, void *, u32, struct lock_class_key *, const char *, struct nvkm_vmm **); -int gp100_vmm_new(struct nvkm_mmu *, u64, u64, void *, u32, +int gp10b_vmm_new(struct nvkm_mmu *, bool, u64, u64, void *, u32, struct lock_class_key *, const char *, struct nvkm_vmm **); -int gp10b_vmm_new(struct nvkm_mmu *, u64, u64, void *, u32, +int gv100_vmm_new(struct nvkm_mmu *, bool, u64, u64, void *, u32, struct lock_class_key *, const char *, struct nvkm_vmm **); -int gv100_vmm_new(struct nvkm_mmu *, u64, u64, void *, u32, +int tu102_vmm_new(struct nvkm_mmu *, bool, u64, u64, void *, u32, struct lock_class_key *, const char *, struct nvkm_vmm **); diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgf100.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgf100.c index faf5a7e9265e..ab6424faf84c 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgf100.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgf100.c @@ -178,15 +178,19 @@ gf100_vmm_desc_16_16[] = { }; void -gf100_vmm_flush_(struct nvkm_vmm *vmm, int depth) +gf100_vmm_invalidate_pdb(struct nvkm_vmm *vmm, u64 addr) +{ + struct nvkm_device *device = vmm->mmu->subdev.device; + nvkm_wr32(device, 0x100cb8, addr); +} + +void +gf100_vmm_invalidate(struct nvkm_vmm *vmm, u32 type) { struct nvkm_subdev *subdev = &vmm->mmu->subdev; struct nvkm_device *device = subdev->device; - u32 type = depth << 24; - - type = 0x00000001; /* PAGE_ALL */ - if (atomic_read(&vmm->engref[NVKM_SUBDEV_BAR])) - type |= 0x00000004; /* HUB_ONLY */ + struct nvkm_mmu_pt *pd = vmm->pd->pt[0]; + u64 addr = 0; mutex_lock(&subdev->mutex); /* Looks like maybe a "free flush slots" counter, the @@ -197,7 +201,20 @@ gf100_vmm_flush_(struct nvkm_vmm *vmm, int depth) break; ); - nvkm_wr32(device, 0x100cb8, vmm->pd->pt[0]->addr >> 8); + if (!(type & 0x00000002) /* ALL_PDB. */) { + switch (nvkm_memory_target(pd->memory)) { + case NVKM_MEM_TARGET_VRAM: addr |= 0x00000000; break; + case NVKM_MEM_TARGET_HOST: addr |= 0x00000002; break; + case NVKM_MEM_TARGET_NCOH: addr |= 0x00000003; break; + default: + WARN_ON(1); + break; + } + addr |= (vmm->pd->pt[0]->addr >> 12) << 4; + + vmm->func->invalidate_pdb(vmm, addr); + } + nvkm_wr32(device, 0x100cbc, 0x80000000 | type); /* Wait for flush to be queued? */ @@ -211,7 +228,10 @@ gf100_vmm_flush_(struct nvkm_vmm *vmm, int depth) void gf100_vmm_flush(struct nvkm_vmm *vmm, int depth) { - gf100_vmm_flush_(vmm, 0); + u32 type = 0x00000001; /* PAGE_ALL */ + if (atomic_read(&vmm->engref[NVKM_SUBDEV_BAR])) + type |= 0x00000004; /* HUB_ONLY */ + gf100_vmm_invalidate(vmm, type); } int @@ -354,6 +374,7 @@ gf100_vmm_17 = { .aper = gf100_vmm_aper, .valid = gf100_vmm_valid, .flush = gf100_vmm_flush, + .invalidate_pdb = gf100_vmm_invalidate_pdb, .page = { { 17, &gf100_vmm_desc_17_17[0], NVKM_VMM_PAGE_xVxC }, { 12, &gf100_vmm_desc_17_12[0], NVKM_VMM_PAGE_xVHx }, @@ -368,6 +389,7 @@ gf100_vmm_16 = { .aper = gf100_vmm_aper, .valid = gf100_vmm_valid, .flush = gf100_vmm_flush, + .invalidate_pdb = gf100_vmm_invalidate_pdb, .page = { { 16, &gf100_vmm_desc_16_16[0], NVKM_VMM_PAGE_xVxC }, { 12, &gf100_vmm_desc_16_12[0], NVKM_VMM_PAGE_xVHx }, @@ -378,14 +400,14 @@ gf100_vmm_16 = { int gf100_vmm_new_(const struct nvkm_vmm_func *func_16, const struct nvkm_vmm_func *func_17, - struct nvkm_mmu *mmu, u64 addr, u64 size, void *argv, u32 argc, - struct lock_class_key *key, const char *name, - struct nvkm_vmm **pvmm) + struct nvkm_mmu *mmu, bool managed, u64 addr, u64 size, + void *argv, u32 argc, struct lock_class_key *key, + const char *name, struct nvkm_vmm **pvmm) { switch (mmu->subdev.device->fb->page) { - case 16: return nv04_vmm_new_(func_16, mmu, 0, addr, size, + case 16: return nv04_vmm_new_(func_16, mmu, 0, managed, addr, size, argv, argc, key, name, pvmm); - case 17: return nv04_vmm_new_(func_17, mmu, 0, addr, size, + case 17: return nv04_vmm_new_(func_17, mmu, 0, managed, addr, size, argv, argc, key, name, pvmm); default: WARN_ON(1); @@ -394,10 +416,10 @@ gf100_vmm_new_(const struct nvkm_vmm_func *func_16, } int -gf100_vmm_new(struct nvkm_mmu *mmu, u64 addr, u64 size, void *argv, u32 argc, - struct lock_class_key *key, const char *name, - struct nvkm_vmm **pvmm) +gf100_vmm_new(struct nvkm_mmu *mmu, bool managed, u64 addr, u64 size, + void *argv, u32 argc, struct lock_class_key *key, + const char *name, struct nvkm_vmm **pvmm) { - return gf100_vmm_new_(&gf100_vmm_16, &gf100_vmm_17, mmu, addr, + return gf100_vmm_new_(&gf100_vmm_16, &gf100_vmm_17, mmu, managed, addr, size, argv, argc, key, name, pvmm); } diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgk104.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgk104.c index 0ebb7bccfcd2..0b59c01fd146 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgk104.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgk104.c @@ -71,6 +71,7 @@ gk104_vmm_17 = { .aper = gf100_vmm_aper, .valid = gf100_vmm_valid, .flush = gf100_vmm_flush, + .invalidate_pdb = gf100_vmm_invalidate_pdb, .page = { { 17, &gk104_vmm_desc_17_17[0], NVKM_VMM_PAGE_xVxC }, { 12, &gk104_vmm_desc_17_12[0], NVKM_VMM_PAGE_xVHx }, @@ -85,6 +86,7 @@ gk104_vmm_16 = { .aper = gf100_vmm_aper, .valid = gf100_vmm_valid, .flush = gf100_vmm_flush, + .invalidate_pdb = gf100_vmm_invalidate_pdb, .page = { { 16, &gk104_vmm_desc_16_16[0], NVKM_VMM_PAGE_xVxC }, { 12, &gk104_vmm_desc_16_12[0], NVKM_VMM_PAGE_xVHx }, @@ -93,10 +95,10 @@ gk104_vmm_16 = { }; int -gk104_vmm_new(struct nvkm_mmu *mmu, u64 addr, u64 size, void *argv, u32 argc, - struct lock_class_key *key, const char *name, - struct nvkm_vmm **pvmm) +gk104_vmm_new(struct nvkm_mmu *mmu, bool managed, u64 addr, u64 size, + void *argv, u32 argc, struct lock_class_key *key, + const char *name, struct nvkm_vmm **pvmm) { - return gf100_vmm_new_(&gk104_vmm_16, &gk104_vmm_17, mmu, addr, + return gf100_vmm_new_(&gk104_vmm_16, &gk104_vmm_17, mmu, managed, addr, size, argv, argc, key, name, pvmm); } diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgk20a.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgk20a.c index 8086994a0446..5a9582dce970 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgk20a.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgk20a.c @@ -40,6 +40,7 @@ gk20a_vmm_17 = { .aper = gf100_vmm_aper, .valid = gf100_vmm_valid, .flush = gf100_vmm_flush, + .invalidate_pdb = gf100_vmm_invalidate_pdb, .page = { { 17, &gk104_vmm_desc_17_17[0], NVKM_VMM_PAGE_xxHC }, { 12, &gk104_vmm_desc_17_12[0], NVKM_VMM_PAGE_xxHx }, @@ -54,6 +55,7 @@ gk20a_vmm_16 = { .aper = gf100_vmm_aper, .valid = gf100_vmm_valid, .flush = gf100_vmm_flush, + .invalidate_pdb = gf100_vmm_invalidate_pdb, .page = { { 16, &gk104_vmm_desc_16_16[0], NVKM_VMM_PAGE_xxHC }, { 12, &gk104_vmm_desc_16_12[0], NVKM_VMM_PAGE_xxHx }, @@ -62,10 +64,10 @@ gk20a_vmm_16 = { }; int -gk20a_vmm_new(struct nvkm_mmu *mmu, u64 addr, u64 size, void *argv, u32 argc, - struct lock_class_key *key, const char *name, - struct nvkm_vmm **pvmm) +gk20a_vmm_new(struct nvkm_mmu *mmu, bool managed, u64 addr, u64 size, + void *argv, u32 argc, struct lock_class_key *key, + const char *name, struct nvkm_vmm **pvmm) { - return gf100_vmm_new_(&gk20a_vmm_16, &gk20a_vmm_17, mmu, addr, + return gf100_vmm_new_(&gk20a_vmm_16, &gk20a_vmm_17, mmu, managed, addr, size, argv, argc, key, name, pvmm); } diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgm200.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgm200.c index a1676a4644fe..2e61af02d4d8 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgm200.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgm200.c @@ -113,6 +113,7 @@ gm200_vmm_17 = { .aper = gf100_vmm_aper, .valid = gf100_vmm_valid, .flush = gf100_vmm_flush, + .invalidate_pdb = gf100_vmm_invalidate_pdb, .page = { { 27, &gm200_vmm_desc_17_17[1], NVKM_VMM_PAGE_Sxxx }, { 17, &gm200_vmm_desc_17_17[0], NVKM_VMM_PAGE_SVxC }, @@ -128,6 +129,7 @@ gm200_vmm_16 = { .aper = gf100_vmm_aper, .valid = gf100_vmm_valid, .flush = gf100_vmm_flush, + .invalidate_pdb = gf100_vmm_invalidate_pdb, .page = { { 27, &gm200_vmm_desc_16_16[1], NVKM_VMM_PAGE_Sxxx }, { 16, &gm200_vmm_desc_16_16[0], NVKM_VMM_PAGE_SVxC }, @@ -139,9 +141,9 @@ gm200_vmm_16 = { int gm200_vmm_new_(const struct nvkm_vmm_func *func_16, const struct nvkm_vmm_func *func_17, - struct nvkm_mmu *mmu, u64 addr, u64 size, void *argv, u32 argc, - struct lock_class_key *key, const char *name, - struct nvkm_vmm **pvmm) + struct nvkm_mmu *mmu, bool managed, u64 addr, u64 size, + void *argv, u32 argc, struct lock_class_key *key, + const char *name, struct nvkm_vmm **pvmm) { const struct nvkm_vmm_func *func; union { @@ -163,23 +165,23 @@ gm200_vmm_new_(const struct nvkm_vmm_func *func_16, } else return ret; - return nvkm_vmm_new_(func, mmu, 0, addr, size, key, name, pvmm); + return nvkm_vmm_new_(func, mmu, 0, managed, addr, size, key, name, pvmm); } int -gm200_vmm_new(struct nvkm_mmu *mmu, u64 addr, u64 size, void *argv, u32 argc, - struct lock_class_key *key, const char *name, - struct nvkm_vmm **pvmm) +gm200_vmm_new(struct nvkm_mmu *mmu, bool managed, u64 addr, u64 size, + void *argv, u32 argc, struct lock_class_key *key, + const char *name, struct nvkm_vmm **pvmm) { - return gm200_vmm_new_(&gm200_vmm_16, &gm200_vmm_17, mmu, addr, + return gm200_vmm_new_(&gm200_vmm_16, &gm200_vmm_17, mmu, managed, addr, size, argv, argc, key, name, pvmm); } int -gm200_vmm_new_fixed(struct nvkm_mmu *mmu, u64 addr, u64 size, +gm200_vmm_new_fixed(struct nvkm_mmu *mmu, bool managed, u64 addr, u64 size, void *argv, u32 argc, struct lock_class_key *key, const char *name, struct nvkm_vmm **pvmm) { - return gf100_vmm_new_(&gm200_vmm_16, &gm200_vmm_17, mmu, addr, + return gf100_vmm_new_(&gm200_vmm_16, &gm200_vmm_17, mmu, managed, addr, size, argv, argc, key, name, pvmm); } diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgm20b.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgm20b.c index 64d4b6cff8dd..96b759695dd8 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgm20b.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgm20b.c @@ -28,6 +28,7 @@ gm20b_vmm_17 = { .aper = gk20a_vmm_aper, .valid = gf100_vmm_valid, .flush = gf100_vmm_flush, + .invalidate_pdb = gf100_vmm_invalidate_pdb, .page = { { 27, &gm200_vmm_desc_17_17[1], NVKM_VMM_PAGE_Sxxx }, { 17, &gm200_vmm_desc_17_17[0], NVKM_VMM_PAGE_SxHC }, @@ -43,6 +44,7 @@ gm20b_vmm_16 = { .aper = gk20a_vmm_aper, .valid = gf100_vmm_valid, .flush = gf100_vmm_flush, + .invalidate_pdb = gf100_vmm_invalidate_pdb, .page = { { 27, &gm200_vmm_desc_16_16[1], NVKM_VMM_PAGE_Sxxx }, { 16, &gm200_vmm_desc_16_16[0], NVKM_VMM_PAGE_SxHC }, @@ -52,19 +54,19 @@ gm20b_vmm_16 = { }; int -gm20b_vmm_new(struct nvkm_mmu *mmu, u64 addr, u64 size, void *argv, u32 argc, - struct lock_class_key *key, const char *name, - struct nvkm_vmm **pvmm) +gm20b_vmm_new(struct nvkm_mmu *mmu, bool managed, u64 addr, u64 size, + void *argv, u32 argc, struct lock_class_key *key, + const char *name, struct nvkm_vmm **pvmm) { - return gm200_vmm_new_(&gm20b_vmm_16, &gm20b_vmm_17, mmu, addr, + return gm200_vmm_new_(&gm20b_vmm_16, &gm20b_vmm_17, mmu, managed, addr, size, argv, argc, key, name, pvmm); } int -gm20b_vmm_new_fixed(struct nvkm_mmu *mmu, u64 addr, u64 size, +gm20b_vmm_new_fixed(struct nvkm_mmu *mmu, bool managed, u64 addr, u64 size, void *argv, u32 argc, struct lock_class_key *key, const char *name, struct nvkm_vmm **pvmm) { - return gf100_vmm_new_(&gm20b_vmm_16, &gm20b_vmm_17, mmu, addr, + return gf100_vmm_new_(&gm20b_vmm_16, &gm20b_vmm_17, mmu, managed, addr, size, argv, argc, key, name, pvmm); } diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgp100.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgp100.c index 059fafe0e771..b4f519768d5e 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgp100.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgp100.c @@ -21,12 +21,90 @@ */ #include "vmm.h" +#include <core/client.h> #include <subdev/fb.h> #include <subdev/ltc.h> +#include <subdev/timer.h> +#include <engine/gr.h> #include <nvif/ifc00d.h> #include <nvif/unpack.h> +static void +gp100_vmm_pfn_unmap(struct nvkm_vmm *vmm, + struct nvkm_mmu_pt *pt, u32 ptei, u32 ptes) +{ + struct device *dev = vmm->mmu->subdev.device->dev; + dma_addr_t addr; + + nvkm_kmap(pt->memory); + while (ptes--) { + u32 datalo = nvkm_ro32(pt->memory, pt->base + ptei * 8 + 0); + u32 datahi = nvkm_ro32(pt->memory, pt->base + ptei * 8 + 4); + u64 data = (u64)datahi << 32 | datalo; + if ((data & (3ULL << 1)) != 0) { + addr = (data >> 8) << 12; + dma_unmap_page(dev, addr, PAGE_SIZE, DMA_BIDIRECTIONAL); + } + ptei++; + } + nvkm_done(pt->memory); +} + +static bool +gp100_vmm_pfn_clear(struct nvkm_vmm *vmm, + struct nvkm_mmu_pt *pt, u32 ptei, u32 ptes) +{ + bool dma = false; + nvkm_kmap(pt->memory); + while (ptes--) { + u32 datalo = nvkm_ro32(pt->memory, pt->base + ptei * 8 + 0); + u32 datahi = nvkm_ro32(pt->memory, pt->base + ptei * 8 + 4); + u64 data = (u64)datahi << 32 | datalo; + if ((data & BIT_ULL(0)) && (data & (3ULL << 1)) != 0) { + VMM_WO064(pt, vmm, ptei * 8, data & ~BIT_ULL(0)); + dma = true; + } + ptei++; + } + nvkm_done(pt->memory); + return dma; +} + +static void +gp100_vmm_pgt_pfn(struct nvkm_vmm *vmm, struct nvkm_mmu_pt *pt, + u32 ptei, u32 ptes, struct nvkm_vmm_map *map) +{ + struct device *dev = vmm->mmu->subdev.device->dev; + dma_addr_t addr; + + nvkm_kmap(pt->memory); + while (ptes--) { + u64 data = 0; + if (!(*map->pfn & NVKM_VMM_PFN_W)) + data |= BIT_ULL(6); /* RO. */ + + if (!(*map->pfn & NVKM_VMM_PFN_VRAM)) { + addr = *map->pfn >> NVKM_VMM_PFN_ADDR_SHIFT; + addr = dma_map_page(dev, pfn_to_page(addr), 0, + PAGE_SIZE, DMA_BIDIRECTIONAL); + if (!WARN_ON(dma_mapping_error(dev, addr))) { + data |= addr >> 4; + data |= 2ULL << 1; /* SYSTEM_COHERENT_MEMORY. */ + data |= BIT_ULL(3); /* VOL. */ + data |= BIT_ULL(0); /* VALID. */ + } + } else { + data |= (*map->pfn & NVKM_VMM_PFN_ADDR) >> 4; + data |= BIT_ULL(0); /* VALID. */ + } + + VMM_WO064(pt, vmm, ptei++ * 8, data); + map->pfn++; + } + nvkm_done(pt->memory); +} + static inline void gp100_vmm_pgt_pte(struct nvkm_vmm *vmm, struct nvkm_mmu_pt *pt, u32 ptei, u32 ptes, struct nvkm_vmm_map *map, u64 addr) @@ -89,6 +167,9 @@ gp100_vmm_desc_spt = { .mem = gp100_vmm_pgt_mem, .dma = gp100_vmm_pgt_dma, .sgl = gp100_vmm_pgt_sgl, + .pfn = gp100_vmm_pgt_pfn, + .pfn_clear = gp100_vmm_pfn_clear, + .pfn_unmap = gp100_vmm_pfn_unmap, }; static void @@ -306,16 +387,100 @@ gp100_vmm_valid(struct nvkm_vmm *vmm, void *argv, u32 argc, return 0; } +static int +gp100_vmm_fault_cancel(struct nvkm_vmm *vmm, void *argv, u32 argc) +{ + struct nvkm_device *device = vmm->mmu->subdev.device; + union { + struct gp100_vmm_fault_cancel_v0 v0; + } *args = argv; + int ret = -ENOSYS; + u32 inst, aper; + + if ((ret = nvif_unpack(ret, &argv, &argc, args->v0, 0, 0, false))) + return ret; + + /* Translate MaxwellFaultBufferA instance pointer to the same + * format as the NV_GR_FECS_CURRENT_CTX register. + */ + aper = (args->v0.inst >> 8) & 3; + args->v0.inst >>= 12; + args->v0.inst |= aper << 28; + args->v0.inst |= 0x80000000; + + if (!WARN_ON(nvkm_gr_ctxsw_pause(device))) { + if ((inst = nvkm_gr_ctxsw_inst(device)) == args->v0.inst) { + gf100_vmm_invalidate(vmm, 0x0000001b + /* CANCEL_TARGETED. */ | + (args->v0.hub << 20) | + (args->v0.gpc << 15) | + (args->v0.client << 9)); + } + WARN_ON(nvkm_gr_ctxsw_resume(device)); + } + + return 0; +} + +static int +gp100_vmm_fault_replay(struct nvkm_vmm *vmm, void *argv, u32 argc) +{ + union { + struct gp100_vmm_fault_replay_vn vn; + } *args = argv; + int ret = -ENOSYS; + + if (!(ret = nvif_unvers(ret, &argv, &argc, args->vn))) { + gf100_vmm_invalidate(vmm, 0x0000000b); /* REPLAY_GLOBAL. */ + } + + return ret; +} + +int +gp100_vmm_mthd(struct nvkm_vmm *vmm, + struct nvkm_client *client, u32 mthd, void *argv, u32 argc) +{ + if (client->super) { + switch (mthd) { + case GP100_VMM_VN_FAULT_REPLAY: + return gp100_vmm_fault_replay(vmm, argv, argc); + case GP100_VMM_VN_FAULT_CANCEL: + return gp100_vmm_fault_cancel(vmm, argv, argc); + default: + break; + } + } + return -EINVAL; +} + +void +gp100_vmm_invalidate_pdb(struct nvkm_vmm *vmm, u64 addr) +{ + struct nvkm_device *device = vmm->mmu->subdev.device; + nvkm_wr32(device, 0x100cb8, lower_32_bits(addr)); + nvkm_wr32(device, 0x100cec, upper_32_bits(addr)); +} + void gp100_vmm_flush(struct nvkm_vmm *vmm, int depth) { - gf100_vmm_flush_(vmm, 5 /* CACHE_LEVEL_UP_TO_PDE3 */ - depth); + u32 type = (5 /* CACHE_LEVEL_UP_TO_PDE3 */ - depth) << 24; + type = 0; /*XXX: need to confirm stuff works with depth enabled... */ + if (atomic_read(&vmm->engref[NVKM_SUBDEV_BAR])) + type |= 0x00000004; /* HUB_ONLY */ + type |= 0x00000001; /* PAGE_ALL */ + gf100_vmm_invalidate(vmm, type); } int gp100_vmm_join(struct nvkm_vmm *vmm, struct nvkm_memory *inst) { - const u64 base = BIT_ULL(10) /* VER2 */ | BIT_ULL(11); /* 64KiB */ + u64 base = BIT_ULL(10) /* VER2 */ | BIT_ULL(11) /* 64KiB */; + if (vmm->replay) { + base |= BIT_ULL(4); /* FAULT_REPLAY_TEX */ + base |= BIT_ULL(5); /* FAULT_REPLAY_GCC */ + } return gf100_vmm_join_(vmm, inst, base); } @@ -326,6 +491,8 @@ gp100_vmm = { .aper = gf100_vmm_aper, .valid = gp100_vmm_valid, .flush = gp100_vmm_flush, + .mthd = gp100_vmm_mthd, + .invalidate_pdb = gp100_vmm_invalidate_pdb, .page = { { 47, &gp100_vmm_desc_16[4], NVKM_VMM_PAGE_Sxxx }, { 38, &gp100_vmm_desc_16[3], NVKM_VMM_PAGE_Sxxx }, @@ -338,10 +505,39 @@ gp100_vmm = { }; int -gp100_vmm_new(struct nvkm_mmu *mmu, u64 addr, u64 size, void *argv, u32 argc, - struct lock_class_key *key, const char *name, - struct nvkm_vmm **pvmm) +gp100_vmm_new_(const struct nvkm_vmm_func *func, + struct nvkm_mmu *mmu, bool managed, u64 addr, u64 size, + void *argv, u32 argc, struct lock_class_key *key, + const char *name, struct nvkm_vmm **pvmm) +{ + union { + struct gp100_vmm_vn vn; + struct gp100_vmm_v0 v0; + } *args = argv; + int ret = -ENOSYS; + bool replay; + + if (!(ret = nvif_unpack(ret, &argv, &argc, args->v0, 0, 0, false))) { + replay = args->v0.fault_replay != 0; + } else + if (!(ret = nvif_unvers(ret, &argv, &argc, args->vn))) { + replay = false; + } else + return ret; + + ret = nvkm_vmm_new_(func, mmu, 0, managed, addr, size, key, name, pvmm); + if (ret) + return ret; + + (*pvmm)->replay = replay; + return 0; +} + +int +gp100_vmm_new(struct nvkm_mmu *mmu, bool managed, u64 addr, u64 size, + void *argv, u32 argc, struct lock_class_key *key, + const char *name, struct nvkm_vmm **pvmm) { - return nv04_vmm_new_(&gp100_vmm, mmu, 0, addr, size, - argv, argc, key, name, pvmm); + return gp100_vmm_new_(&gp100_vmm, mmu, managed, addr, size, + argv, argc, key, name, pvmm); } diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgp10b.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgp10b.c index 3dcc6bddb32f..e081239afe58 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgp10b.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgp10b.c @@ -28,6 +28,8 @@ gp10b_vmm = { .aper = gk20a_vmm_aper, .valid = gp100_vmm_valid, .flush = gp100_vmm_flush, + .mthd = gp100_vmm_mthd, + .invalidate_pdb = gp100_vmm_invalidate_pdb, .page = { { 47, &gp100_vmm_desc_16[4], NVKM_VMM_PAGE_Sxxx }, { 38, &gp100_vmm_desc_16[3], NVKM_VMM_PAGE_Sxxx }, @@ -40,10 +42,10 @@ gp10b_vmm = { }; int -gp10b_vmm_new(struct nvkm_mmu *mmu, u64 addr, u64 size, void *argv, u32 argc, - struct lock_class_key *key, const char *name, - struct nvkm_vmm **pvmm) +gp10b_vmm_new(struct nvkm_mmu *mmu, bool managed, u64 addr, u64 size, + void *argv, u32 argc, struct lock_class_key *key, + const char *name, struct nvkm_vmm **pvmm) { - return nv04_vmm_new_(&gp10b_vmm, mmu, 0, addr, size, - argv, argc, key, name, pvmm); + return gp100_vmm_new_(&gp10b_vmm, mmu, managed, addr, size, + argv, argc, key, name, pvmm); } diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgv100.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgv100.c index 2fa40c16e6d2..f0e21f63253a 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgv100.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgv100.c @@ -66,6 +66,8 @@ gv100_vmm = { .aper = gf100_vmm_aper, .valid = gp100_vmm_valid, .flush = gp100_vmm_flush, + .mthd = gp100_vmm_mthd, + .invalidate_pdb = gp100_vmm_invalidate_pdb, .page = { { 47, &gp100_vmm_desc_16[4], NVKM_VMM_PAGE_Sxxx }, { 38, &gp100_vmm_desc_16[3], NVKM_VMM_PAGE_Sxxx }, @@ -78,10 +80,10 @@ gv100_vmm = { }; int -gv100_vmm_new(struct nvkm_mmu *mmu, u64 addr, u64 size, void *argv, u32 argc, - struct lock_class_key *key, const char *name, - struct nvkm_vmm **pvmm) +gv100_vmm_new(struct nvkm_mmu *mmu, bool managed, u64 addr, u64 size, + void *argv, u32 argc, struct lock_class_key *key, + const char *name, struct nvkm_vmm **pvmm) { - return nv04_vmm_new_(&gv100_vmm, mmu, 0, addr, size, - argv, argc, key, name, pvmm); + return gp100_vmm_new_(&gv100_vmm, mmu, managed, addr, size, + argv, argc, key, name, pvmm); } diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmmcp77.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmmcp77.c index e63d984cbfd4..bdddd99f5877 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmmcp77.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmmcp77.c @@ -36,10 +36,10 @@ mcp77_vmm = { }; int -mcp77_vmm_new(struct nvkm_mmu *mmu, u64 addr, u64 size, void *argv, u32 argc, - struct lock_class_key *key, const char *name, - struct nvkm_vmm **pvmm) +mcp77_vmm_new(struct nvkm_mmu *mmu, bool managed, u64 addr, u64 size, + void *argv, u32 argc, struct lock_class_key *key, + const char *name, struct nvkm_vmm **pvmm) { - return nv04_vmm_new_(&mcp77_vmm, mmu, 0, addr, size, + return nv04_vmm_new_(&mcp77_vmm, mmu, 0, managed, addr, size, argv, argc, key, name, pvmm); } diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmnv04.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmnv04.c index 0cab1ffc9f64..4c6b3b7d221f 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmnv04.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmnv04.c @@ -100,16 +100,17 @@ nv04_vmm = { int nv04_vmm_new_(const struct nvkm_vmm_func *func, struct nvkm_mmu *mmu, - u32 pd_header, u64 addr, u64 size, void *argv, u32 argc, - struct lock_class_key *key, const char *name, - struct nvkm_vmm **pvmm) + u32 pd_header, bool managed, u64 addr, u64 size, + void *argv, u32 argc, struct lock_class_key *key, + const char *name, struct nvkm_vmm **pvmm) { union { struct nv04_vmm_vn vn; } *args = argv; int ret; - ret = nvkm_vmm_new_(func, mmu, pd_header, addr, size, key, name, pvmm); + ret = nvkm_vmm_new_(func, mmu, pd_header, managed, addr, size, + key, name, pvmm); if (ret) return ret; @@ -117,15 +118,15 @@ nv04_vmm_new_(const struct nvkm_vmm_func *func, struct nvkm_mmu *mmu, } int -nv04_vmm_new(struct nvkm_mmu *mmu, u64 addr, u64 size, void *argv, u32 argc, - struct lock_class_key *key, const char *name, +nv04_vmm_new(struct nvkm_mmu *mmu, bool managed, u64 addr, u64 size, + void *argv, u32 argc, struct lock_class_key *key, const char *name, struct nvkm_vmm **pvmm) { struct nvkm_memory *mem; struct nvkm_vmm *vmm; int ret; - ret = nv04_vmm_new_(&nv04_vmm, mmu, 8, addr, size, + ret = nv04_vmm_new_(&nv04_vmm, mmu, 8, managed, addr, size, argv, argc, key, name, &vmm); *pvmm = vmm; if (ret) diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmnv41.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmnv41.c index b595f130e573..1d3369683a21 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmnv41.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmnv41.c @@ -104,10 +104,10 @@ nv41_vmm = { }; int -nv41_vmm_new(struct nvkm_mmu *mmu, u64 addr, u64 size, void *argv, u32 argc, - struct lock_class_key *key, const char *name, +nv41_vmm_new(struct nvkm_mmu *mmu, bool managed, u64 addr, u64 size, + void *argv, u32 argc, struct lock_class_key *key, const char *name, struct nvkm_vmm **pvmm) { - return nv04_vmm_new_(&nv41_vmm, mmu, 0, addr, size, + return nv04_vmm_new_(&nv41_vmm, mmu, 0, managed, addr, size, argv, argc, key, name, pvmm); } diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmnv44.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmnv44.c index b834e4352334..a82936ba9890 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmnv44.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmnv44.c @@ -205,15 +205,15 @@ nv44_vmm = { }; int -nv44_vmm_new(struct nvkm_mmu *mmu, u64 addr, u64 size, void *argv, u32 argc, - struct lock_class_key *key, const char *name, +nv44_vmm_new(struct nvkm_mmu *mmu, bool managed, u64 addr, u64 size, + void *argv, u32 argc, struct lock_class_key *key, const char *name, struct nvkm_vmm **pvmm) { struct nvkm_subdev *subdev = &mmu->subdev; struct nvkm_vmm *vmm; int ret; - ret = nv04_vmm_new_(&nv44_vmm, mmu, 0, addr, size, + ret = nv04_vmm_new_(&nv44_vmm, mmu, 0, managed, addr, size, argv, argc, key, name, &vmm); *pvmm = vmm; if (ret) diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmnv50.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmnv50.c index 64f75d906202..c98afe3134ee 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmnv50.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmnv50.c @@ -376,10 +376,10 @@ nv50_vmm = { }; int -nv50_vmm_new(struct nvkm_mmu *mmu, u64 addr, u64 size, void *argv, u32 argc, - struct lock_class_key *key, const char *name, +nv50_vmm_new(struct nvkm_mmu *mmu, bool managed, u64 addr, u64 size, + void *argv, u32 argc, struct lock_class_key *key, const char *name, struct nvkm_vmm **pvmm) { - return nv04_vmm_new_(&nv50_vmm, mmu, 0, addr, size, + return nv04_vmm_new_(&nv50_vmm, mmu, 0, managed, addr, size, argv, argc, key, name, pvmm); } diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmtu102.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmtu102.c new file mode 100644 index 000000000000..be91cffc3b52 --- /dev/null +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmtu102.c @@ -0,0 +1,78 @@ +/* + * Copyright 2018 Red Hat Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ +#include "vmm.h" + +#include <subdev/timer.h> + +static void +tu102_vmm_flush(struct nvkm_vmm *vmm, int depth) +{ + struct nvkm_subdev *subdev = &vmm->mmu->subdev; + struct nvkm_device *device = subdev->device; + u32 type = depth << 24; /*XXX: not confirmed */ + + type = 0x00000001; /* PAGE_ALL */ + if (atomic_read(&vmm->engref[NVKM_SUBDEV_BAR])) + type |= 0x00000004; /* HUB_ONLY */ + + mutex_lock(&subdev->mutex); + + nvkm_wr32(device, 0xb830a0, vmm->pd->pt[0]->addr >> 8); + nvkm_wr32(device, 0xb830a4, 0x00000000); + nvkm_wr32(device, 0x100e68, 0x00000000); + nvkm_wr32(device, 0xb830b0, 0x80000000 | type); + + nvkm_msec(device, 2000, + if (!(nvkm_rd32(device, 0xb830b0) & 0x80000000)) + break; + ); + + mutex_unlock(&subdev->mutex); +} + +static const struct nvkm_vmm_func +tu102_vmm = { + .join = gv100_vmm_join, + .part = gf100_vmm_part, + .aper = gf100_vmm_aper, + .valid = gp100_vmm_valid, + .flush = tu102_vmm_flush, + .mthd = gp100_vmm_mthd, + .page = { + { 47, &gp100_vmm_desc_16[4], NVKM_VMM_PAGE_Sxxx }, + { 38, &gp100_vmm_desc_16[3], NVKM_VMM_PAGE_Sxxx }, + { 29, &gp100_vmm_desc_16[2], NVKM_VMM_PAGE_Sxxx }, + { 21, &gp100_vmm_desc_16[1], NVKM_VMM_PAGE_SVxC }, + { 16, &gp100_vmm_desc_16[0], NVKM_VMM_PAGE_SVxC }, + { 12, &gp100_vmm_desc_12[0], NVKM_VMM_PAGE_SVHx }, + {} + } +}; + +int +tu102_vmm_new(struct nvkm_mmu *mmu, bool managed, u64 addr, u64 size, + void *argv, u32 argc, struct lock_class_key *key, + const char *name, struct nvkm_vmm **pvmm) +{ + return gp100_vmm_new_(&tu102_vmm, mmu, managed, addr, size, + argv, argc, key, name, pvmm); +} diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/memx.c b/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/memx.c index 11b28b086a06..7b052879af72 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/memx.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/memx.c @@ -88,10 +88,10 @@ nvkm_memx_fini(struct nvkm_memx **pmemx, bool exec) if (exec) { nvkm_pmu_send(pmu, reply, PROC_MEMX, MEMX_MSG_EXEC, memx->base, finish); + nvkm_debug(subdev, "Exec took %uns, PMU_IN %08x\n", + reply[0], reply[1]); } - nvkm_debug(subdev, "Exec took %uns, PMU_IN %08x\n", - reply[0], reply[1]); kfree(memx); return 0; } diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/secboot/acr_r352.c b/drivers/gpu/drm/nouveau/nvkm/subdev/secboot/acr_r352.c index 5c14d6ac855d..1df09ed6fe6d 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/secboot/acr_r352.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/secboot/acr_r352.c @@ -853,7 +853,7 @@ acr_r352_shutdown(struct acr_r352 *acr, struct nvkm_secboot *sb) * and the expected behavior on RM as well */ if (ret && ret != 0x1d) { - nvkm_error(subdev, "HS unload failed, ret 0x%08x", ret); + nvkm_error(subdev, "HS unload failed, ret 0x%08x\n", ret); return -EINVAL; } nvkm_debug(subdev, "HS unload blob completed\n"); @@ -922,7 +922,7 @@ acr_r352_bootstrap(struct acr_r352 *acr, struct nvkm_secboot *sb) if (ret < 0) { return ret; } else if (ret > 0) { - nvkm_error(subdev, "HS load failed, ret 0x%08x", ret); + nvkm_error(subdev, "HS load failed, ret 0x%08x\n", ret); return -EINVAL; } nvkm_debug(subdev, "HS load blob completed\n"); diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/secboot/gp102.c b/drivers/gpu/drm/nouveau/nvkm/subdev/secboot/gp102.c index 1f7a3c1a7f50..84a2f243ed9b 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/secboot/gp102.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/secboot/gp102.c @@ -59,10 +59,10 @@ gp102_run_secure_scrub(struct nvkm_secboot *sb) nvkm_debug(subdev, "running VPR scrubber binary on NVDEC...\n"); - engine = nvkm_engine_ref(&device->nvdec->engine); + engine = nvkm_engine_ref(&device->nvdec[0]->engine); if (IS_ERR(engine)) return PTR_ERR(engine); - falcon = device->nvdec->falcon; + falcon = device->nvdec[0]->falcon; nvkm_falcon_get(falcon, &sb->subdev); diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/therm/base.c b/drivers/gpu/drm/nouveau/nvkm/subdev/therm/base.c index 3695cde669f8..07914e36939e 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/therm/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/therm/base.c @@ -132,11 +132,12 @@ nvkm_therm_update(struct nvkm_therm *therm, int mode) duty = nvkm_therm_update_linear(therm); break; case NVBIOS_THERM_FAN_OTHER: - if (therm->cstate) + if (therm->cstate) { duty = therm->cstate; - else + poll = false; + } else { duty = nvkm_therm_update_linear_fallback(therm); - poll = false; + } break; } immd = false; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/timer/base.c b/drivers/gpu/drm/nouveau/nvkm/subdev/timer/base.c index 36de23d12ae4..dd922033628c 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/timer/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/timer/base.c @@ -23,6 +23,42 @@ */ #include "priv.h" +s64 +nvkm_timer_wait_test(struct nvkm_timer_wait *wait) +{ + struct nvkm_subdev *subdev = &wait->tmr->subdev; + u64 time = nvkm_timer_read(wait->tmr); + + if (wait->reads == 0) { + wait->time0 = time; + wait->time1 = time; + } + + if (wait->time1 == time) { + if (wait->reads++ == 16) { + nvkm_fatal(subdev, "stalled at %016llx\n", time); + return -ETIMEDOUT; + } + } else { + wait->time1 = time; + wait->reads = 1; + } + + if (wait->time1 - wait->time0 > wait->limit) + return -ETIMEDOUT; + + return wait->time1 - wait->time0; +} + +void +nvkm_timer_wait_init(struct nvkm_device *device, u64 nsec, + struct nvkm_timer_wait *wait) +{ + wait->tmr = device->timer; + wait->limit = nsec; + wait->reads = 0; +} + u64 nvkm_timer_read(struct nvkm_timer *tmr) { diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/top/base.c b/drivers/gpu/drm/nouveau/nvkm/subdev/top/base.c index 67ada1d9a28c..cce6e4e90ebf 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/top/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/top/base.c @@ -41,6 +41,22 @@ nvkm_top_device_new(struct nvkm_top *top) } u32 +nvkm_top_addr(struct nvkm_device *device, enum nvkm_devidx index) +{ + struct nvkm_top *top = device->top; + struct nvkm_top_device *info; + + if (top) { + list_for_each_entry(info, &top->device, head) { + if (info->index == index) + return info->addr; + } + } + + return 0; +} + +u32 nvkm_top_reset(struct nvkm_device *device, enum nvkm_devidx index) { struct nvkm_top *top = device->top; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/top/gk104.c b/drivers/gpu/drm/nouveau/nvkm/subdev/top/gk104.c index 4f1f3e890650..e01746ce9fc4 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/top/gk104.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/top/gk104.c @@ -73,6 +73,7 @@ gk104_top_oneinit(struct nvkm_top *top) #define A_(A) if (inst == 0) info->index = NVKM_ENGINE_##A #define B_(A) if (inst + NVKM_ENGINE_##A##0 < NVKM_ENGINE_##A##_LAST + 1) \ info->index = NVKM_ENGINE_##A##0 + inst +#define C_(A) if (inst == 0) info->index = NVKM_SUBDEV_##A switch (type) { case 0x00000000: A_(GR ); break; case 0x00000001: A_(CE0 ); break; @@ -86,8 +87,9 @@ gk104_top_oneinit(struct nvkm_top *top) case 0x0000000d: A_(SEC2 ); break; case 0x0000000e: B_(NVENC ); break; case 0x0000000f: A_(NVENC1); break; - case 0x00000010: A_(NVDEC ); break; + case 0x00000010: B_(NVDEC ); break; case 0x00000013: B_(CE ); break; + case 0x00000014: C_(GSP ); break; break; default: break; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/volt/Kbuild b/drivers/gpu/drm/nouveau/nvkm/subdev/volt/Kbuild index bcd179ba11d0..146adcdd316a 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/volt/Kbuild +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/volt/Kbuild @@ -2,6 +2,7 @@ nvkm-y += nvkm/subdev/volt/base.o nvkm-y += nvkm/subdev/volt/gpio.o nvkm-y += nvkm/subdev/volt/nv40.o nvkm-y += nvkm/subdev/volt/gf100.o +nvkm-y += nvkm/subdev/volt/gf117.o nvkm-y += nvkm/subdev/volt/gk104.o nvkm-y += nvkm/subdev/volt/gk20a.o nvkm-y += nvkm/subdev/volt/gm20b.o diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/volt/gf117.c b/drivers/gpu/drm/nouveau/nvkm/subdev/volt/gf117.c new file mode 100644 index 000000000000..547a58f0aeac --- /dev/null +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/volt/gf117.c @@ -0,0 +1,60 @@ +/* + * Copyright 2019 Ilia Mirkin + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * Authors: Ilia Mirkin + */ +#include "priv.h" + +#include <subdev/fuse.h> + +static int +gf117_volt_speedo_read(struct nvkm_volt *volt) +{ + struct nvkm_device *device = volt->subdev.device; + struct nvkm_fuse *fuse = device->fuse; + + if (!fuse) + return -EINVAL; + + return nvkm_fuse_read(fuse, 0x3a8); +} + +static const struct nvkm_volt_func +gf117_volt = { + .oneinit = gf100_volt_oneinit, + .vid_get = nvkm_voltgpio_get, + .vid_set = nvkm_voltgpio_set, + .speedo_read = gf117_volt_speedo_read, +}; + +int +gf117_volt_new(struct nvkm_device *device, int index, struct nvkm_volt **pvolt) +{ + struct nvkm_volt *volt; + int ret; + + ret = nvkm_volt_new_(&gf117_volt, device, index, &volt); + *pvolt = volt; + if (ret) + return ret; + + return nvkm_voltgpio_init(volt); +} |