summaryrefslogtreecommitdiff
path: root/drivers/gpu/drm/nouveau/nvkm
diff options
context:
space:
mode:
Diffstat (limited to 'drivers/gpu/drm/nouveau/nvkm')
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/core/client.c2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/core/engine.c10
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/core/gpuobj.c47
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/core/ioctl.c15
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/core/memory.c100
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/core/mm.c5
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/core/object.c14
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/core/oproxy.c13
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/core/ramht.c1
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/bsp/g84.c2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/device/base.c98
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/device/ctrl.h2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/device/pci.c24
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/device/tegra.c6
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/device/user.c9
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/disp/channv50.c4
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/disp/channv50.h1
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/disp/ior.h2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/dma/usernv04.c6
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/falcon.c2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c8
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.c43
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/fifo/chang84.c13
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/fifo/changf100.h5
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/fifo/changk104.h5
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/fifo/channv50.c14
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/fifo/channv50.h5
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/fifo/dmag84.c6
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/fifo/dmanv04.c2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/fifo/dmanv50.c6
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c18
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.h2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c19
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.h2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifog84.c6
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogf100.c45
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogk104.c51
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifonv50.c6
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv50.c4
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf100.c123
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf100.h2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf108.c3
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf117.c3
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgk104.c6
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgm107.c9
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgp100.c6
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgp102.c3
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.c83
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.h9
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/gr/gf104.c1
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/gr/gf108.c1
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/gr/gf110.c1
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/gr/gf117.c1
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/gr/gf119.c1
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/gr/gk104.c13
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/gr/gk110.c1
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/gr/gk110b.c1
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/gr/gk208.c1
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/gr/nv20.c4
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/gr/nv20.h1
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/gr/nv40.h1
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/gr/nv50.h1
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/mpeg/nv31.h1
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/mpeg/nv50.c1
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/pm/priv.h1
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/sw/chan.h4
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/sw/nvsw.h2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/xtensa.c2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/falcon/base.c2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/falcon/v1.c6
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/bar/Kbuild2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/bar/base.c60
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/bar/g84.c10
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/bar/gf100.c151
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/bar/gf100.h16
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/bar/gk20a.c6
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/bar/gm107.c65
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/bar/gm20b.c42
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/bar/nv50.c174
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/bar/nv50.h16
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/bar/priv.h20
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/bios/iccsense.c4
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/bios/init.c7
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/bios/timing.c5
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/devinit/nv04.c12
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/fb/base.c35
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/fb/g84.c1
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/fb/gf100.c24
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/fb/gf100.h2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/fb/gf108.c2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/fb/gk104.c2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/fb/gk20a.c2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/fb/gm107.c2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/fb/gm200.c18
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/fb/gm20b.c2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/fb/gp100.c1
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/fb/gp102.c1
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/fb/gp10b.c1
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/fb/gt215.c1
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv04.c9
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv10.c1
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv1a.c1
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv20.c13
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv25.c4
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv30.c4
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv35.c4
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv36.c4
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv40.c4
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv41.c2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv44.c1
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv46.c1
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv47.c2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv49.c2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv4e.c1
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv50.c30
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv50.h2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/fb/priv.h13
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/fb/ram.c144
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/fb/ram.h11
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramgf100.c94
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramgf108.c2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramgk104.c2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramgm107.c2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramgm200.c2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramgp100.c2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramgt215.c20
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/fb/rammcp77.c7
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramnv04.c2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramnv10.c2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramnv1a.c2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramnv20.c3
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramnv40.c7
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramnv40.h2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramnv41.c3
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramnv44.c2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramnv49.c3
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramnv4e.c2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramnv50.c98
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/instmem/base.c266
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/instmem/gk20a.c61
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/instmem/nv04.c70
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/instmem/nv40.c71
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/instmem/nv50.c329
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/instmem/priv.h14
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/ltc/base.c25
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/ltc/gf100.c20
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/ltc/gp100.c2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/mmu/Kbuild29
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/mmu/base.c694
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/mmu/g84.c41
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gf100.c228
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gk104.c41
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gk20a.c41
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gm200.c97
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gm20b.c55
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gp100.c45
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gp10b.0
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gp10b.c45
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/mmu/mem.c242
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/mmu/mem.h23
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/mmu/memgf100.c94
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/mmu/memnv04.c69
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/mmu/memnv50.c88
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/mmu/nv04.c122
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/mmu/nv04.h18
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/mmu/nv41.c104
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/mmu/nv44.c190
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/mmu/nv50.c231
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/mmu/priv.h70
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/mmu/umem.c192
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/mmu/umem.h26
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/mmu/ummu.c178
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/mmu/ummu.h14
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/mmu/uvmm.c352
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/mmu/uvmm.h14
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c1513
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.h310
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgf100.c403
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgk104.c102
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgk20a.c71
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgm200.c185
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgm20b.c70
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgp100.c347
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgp10b.c49
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmnv04.c140
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmnv41.c113
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmnv44.c230
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmnv50.c385
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/pci/base.c2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/secboot/gm200.c43
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/secboot/gm200.h5
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/secboot/ls_ucode_msgqueue.c2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/secboot/priv.h1
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/therm/Kbuild1
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/therm/base.c3
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/therm/gp100.c56
196 files changed, 7798 insertions, 2545 deletions
diff --git a/drivers/gpu/drm/nouveau/nvkm/core/client.c b/drivers/gpu/drm/nouveau/nvkm/core/client.c
index 0d3a896892b4..ac671202919e 100644
--- a/drivers/gpu/drm/nouveau/nvkm/core/client.c
+++ b/drivers/gpu/drm/nouveau/nvkm/core/client.c
@@ -301,5 +301,7 @@ nvkm_client_new(const char *name, u64 device, const char *cfg,
client->debug = nvkm_dbgopt(dbg, "CLIENT");
client->objroot = RB_ROOT;
client->ntfy = ntfy;
+ INIT_LIST_HEAD(&client->umem);
+ spin_lock_init(&client->lock);
return 0;
}
diff --git a/drivers/gpu/drm/nouveau/nvkm/core/engine.c b/drivers/gpu/drm/nouveau/nvkm/core/engine.c
index b6c916954a10..657231c3c098 100644
--- a/drivers/gpu/drm/nouveau/nvkm/core/engine.c
+++ b/drivers/gpu/drm/nouveau/nvkm/core/engine.c
@@ -126,6 +126,15 @@ nvkm_engine_init(struct nvkm_subdev *subdev)
return ret;
}
+static int
+nvkm_engine_preinit(struct nvkm_subdev *subdev)
+{
+ struct nvkm_engine *engine = nvkm_engine(subdev);
+ if (engine->func->preinit)
+ engine->func->preinit(engine);
+ return 0;
+}
+
static void *
nvkm_engine_dtor(struct nvkm_subdev *subdev)
{
@@ -138,6 +147,7 @@ nvkm_engine_dtor(struct nvkm_subdev *subdev)
static const struct nvkm_subdev_func
nvkm_engine_func = {
.dtor = nvkm_engine_dtor,
+ .preinit = nvkm_engine_preinit,
.init = nvkm_engine_init,
.fini = nvkm_engine_fini,
.intr = nvkm_engine_intr,
diff --git a/drivers/gpu/drm/nouveau/nvkm/core/gpuobj.c b/drivers/gpu/drm/nouveau/nvkm/core/gpuobj.c
index a7bd22706b2a..d6de2b3ed2c3 100644
--- a/drivers/gpu/drm/nouveau/nvkm/core/gpuobj.c
+++ b/drivers/gpu/drm/nouveau/nvkm/core/gpuobj.c
@@ -42,6 +42,14 @@ nvkm_gpuobj_wr32_fast(struct nvkm_gpuobj *gpuobj, u32 offset, u32 data)
}
/* accessor functions for gpuobjs allocated directly from instmem */
+static int
+nvkm_gpuobj_heap_map(struct nvkm_gpuobj *gpuobj, u64 offset,
+ struct nvkm_vmm *vmm, struct nvkm_vma *vma,
+ void *argv, u32 argc)
+{
+ return nvkm_memory_map(gpuobj->memory, offset, vmm, vma, argv, argc);
+}
+
static u32
nvkm_gpuobj_heap_rd32(struct nvkm_gpuobj *gpuobj, u32 offset)
{
@@ -67,6 +75,7 @@ nvkm_gpuobj_heap_fast = {
.release = nvkm_gpuobj_heap_release,
.rd32 = nvkm_gpuobj_rd32_fast,
.wr32 = nvkm_gpuobj_wr32_fast,
+ .map = nvkm_gpuobj_heap_map,
};
static const struct nvkm_gpuobj_func
@@ -74,6 +83,7 @@ nvkm_gpuobj_heap_slow = {
.release = nvkm_gpuobj_heap_release,
.rd32 = nvkm_gpuobj_heap_rd32,
.wr32 = nvkm_gpuobj_heap_wr32,
+ .map = nvkm_gpuobj_heap_map,
};
static void *
@@ -90,9 +100,19 @@ nvkm_gpuobj_heap_acquire(struct nvkm_gpuobj *gpuobj)
static const struct nvkm_gpuobj_func
nvkm_gpuobj_heap = {
.acquire = nvkm_gpuobj_heap_acquire,
+ .map = nvkm_gpuobj_heap_map,
};
/* accessor functions for gpuobjs sub-allocated from a parent gpuobj */
+static int
+nvkm_gpuobj_map(struct nvkm_gpuobj *gpuobj, u64 offset,
+ struct nvkm_vmm *vmm, struct nvkm_vma *vma,
+ void *argv, u32 argc)
+{
+ return nvkm_memory_map(gpuobj->parent, gpuobj->node->offset + offset,
+ vmm, vma, argv, argc);
+}
+
static u32
nvkm_gpuobj_rd32(struct nvkm_gpuobj *gpuobj, u32 offset)
{
@@ -118,6 +138,7 @@ nvkm_gpuobj_fast = {
.release = nvkm_gpuobj_release,
.rd32 = nvkm_gpuobj_rd32_fast,
.wr32 = nvkm_gpuobj_wr32_fast,
+ .map = nvkm_gpuobj_map,
};
static const struct nvkm_gpuobj_func
@@ -125,6 +146,7 @@ nvkm_gpuobj_slow = {
.release = nvkm_gpuobj_release,
.rd32 = nvkm_gpuobj_rd32,
.wr32 = nvkm_gpuobj_wr32,
+ .map = nvkm_gpuobj_map,
};
static void *
@@ -143,6 +165,7 @@ nvkm_gpuobj_acquire(struct nvkm_gpuobj *gpuobj)
static const struct nvkm_gpuobj_func
nvkm_gpuobj_func = {
.acquire = nvkm_gpuobj_acquire,
+ .map = nvkm_gpuobj_map,
};
static int
@@ -185,7 +208,7 @@ nvkm_gpuobj_ctor(struct nvkm_device *device, u32 size, int align, bool zero,
gpuobj->size = nvkm_memory_size(gpuobj->memory);
}
- return nvkm_mm_init(&gpuobj->heap, 0, gpuobj->size, 1);
+ return nvkm_mm_init(&gpuobj->heap, 0, 0, gpuobj->size, 1);
}
void
@@ -196,7 +219,7 @@ nvkm_gpuobj_del(struct nvkm_gpuobj **pgpuobj)
if (gpuobj->parent)
nvkm_mm_free(&gpuobj->parent->heap, &gpuobj->node);
nvkm_mm_fini(&gpuobj->heap);
- nvkm_memory_del(&gpuobj->memory);
+ nvkm_memory_unref(&gpuobj->memory);
kfree(*pgpuobj);
*pgpuobj = NULL;
}
@@ -218,26 +241,6 @@ nvkm_gpuobj_new(struct nvkm_device *device, u32 size, int align, bool zero,
return ret;
}
-int
-nvkm_gpuobj_map(struct nvkm_gpuobj *gpuobj, struct nvkm_vm *vm,
- u32 access, struct nvkm_vma *vma)
-{
- struct nvkm_memory *memory = gpuobj->memory;
- int ret = nvkm_vm_get(vm, gpuobj->size, 12, access, vma);
- if (ret == 0)
- nvkm_memory_map(memory, vma, 0);
- return ret;
-}
-
-void
-nvkm_gpuobj_unmap(struct nvkm_vma *vma)
-{
- if (vma->node) {
- nvkm_vm_unmap(vma);
- nvkm_vm_put(vma);
- }
-}
-
/* the below is basically only here to support sharing the paged dma object
* for PCI(E)GART on <=nv4x chipsets, and should *not* be expected to work
* anywhere else.
diff --git a/drivers/gpu/drm/nouveau/nvkm/core/ioctl.c b/drivers/gpu/drm/nouveau/nvkm/core/ioctl.c
index be19bbe56bba..d777df5a64e6 100644
--- a/drivers/gpu/drm/nouveau/nvkm/core/ioctl.c
+++ b/drivers/gpu/drm/nouveau/nvkm/core/ioctl.c
@@ -53,7 +53,7 @@ nvkm_ioctl_sclass(struct nvkm_client *client,
union {
struct nvif_ioctl_sclass_v0 v0;
} *args = data;
- struct nvkm_oclass oclass;
+ struct nvkm_oclass oclass = { .client = client };
int ret = -ENOSYS, i = 0;
nvif_ioctl(object, "sclass size %d\n", size);
@@ -257,13 +257,19 @@ nvkm_ioctl_map(struct nvkm_client *client,
union {
struct nvif_ioctl_map_v0 v0;
} *args = data;
+ enum nvkm_object_map type;
int ret = -ENOSYS;
nvif_ioctl(object, "map size %d\n", size);
- if (!(ret = nvif_unpack(ret, &data, &size, args->v0, 0, 0, false))) {
+ if (!(ret = nvif_unpack(ret, &data, &size, args->v0, 0, 0, true))) {
nvif_ioctl(object, "map vers %d\n", args->v0.version);
- ret = nvkm_object_map(object, &args->v0.handle,
- &args->v0.length);
+ ret = nvkm_object_map(object, data, size, &type,
+ &args->v0.handle,
+ &args->v0.length);
+ if (type == NVKM_OBJECT_MAP_IO)
+ args->v0.type = NVIF_IOCTL_MAP_V0_IO;
+ else
+ args->v0.type = NVIF_IOCTL_MAP_V0_VA;
}
return ret;
@@ -281,6 +287,7 @@ nvkm_ioctl_unmap(struct nvkm_client *client,
nvif_ioctl(object, "unmap size %d\n", size);
if (!(ret = nvif_unvers(ret, &data, &size, args->none))) {
nvif_ioctl(object, "unmap\n");
+ ret = nvkm_object_unmap(object);
}
return ret;
diff --git a/drivers/gpu/drm/nouveau/nvkm/core/memory.c b/drivers/gpu/drm/nouveau/nvkm/core/memory.c
index 8903c04c977e..e85a08ecd9da 100644
--- a/drivers/gpu/drm/nouveau/nvkm/core/memory.c
+++ b/drivers/gpu/drm/nouveau/nvkm/core/memory.c
@@ -22,27 +22,117 @@
* Authors: Ben Skeggs <bskeggs@redhat.com>
*/
#include <core/memory.h>
+#include <core/mm.h>
+#include <subdev/fb.h>
#include <subdev/instmem.h>
void
+nvkm_memory_tags_put(struct nvkm_memory *memory, struct nvkm_device *device,
+ struct nvkm_tags **ptags)
+{
+ struct nvkm_fb *fb = device->fb;
+ struct nvkm_tags *tags = *ptags;
+ if (tags) {
+ mutex_lock(&fb->subdev.mutex);
+ if (refcount_dec_and_test(&tags->refcount)) {
+ nvkm_mm_free(&fb->tags, &tags->mn);
+ kfree(memory->tags);
+ memory->tags = NULL;
+ }
+ mutex_unlock(&fb->subdev.mutex);
+ *ptags = NULL;
+ }
+}
+
+int
+nvkm_memory_tags_get(struct nvkm_memory *memory, struct nvkm_device *device,
+ u32 nr, void (*clr)(struct nvkm_device *, u32, u32),
+ struct nvkm_tags **ptags)
+{
+ struct nvkm_fb *fb = device->fb;
+ struct nvkm_tags *tags;
+
+ mutex_lock(&fb->subdev.mutex);
+ if ((tags = memory->tags)) {
+ /* If comptags exist for the memory, but a different amount
+ * than requested, the buffer is being mapped with settings
+ * that are incompatible with existing mappings.
+ */
+ if (tags->mn && tags->mn->length != nr) {
+ mutex_unlock(&fb->subdev.mutex);
+ return -EINVAL;
+ }
+
+ refcount_inc(&tags->refcount);
+ mutex_unlock(&fb->subdev.mutex);
+ *ptags = tags;
+ return 0;
+ }
+
+ if (!(tags = kmalloc(sizeof(*tags), GFP_KERNEL))) {
+ mutex_unlock(&fb->subdev.mutex);
+ return -ENOMEM;
+ }
+
+ if (!nvkm_mm_head(&fb->tags, 0, 1, nr, nr, 1, &tags->mn)) {
+ if (clr)
+ clr(device, tags->mn->offset, tags->mn->length);
+ } else {
+ /* Failure to allocate HW comptags is not an error, the
+ * caller should fall back to an uncompressed map.
+ *
+ * As memory can be mapped in multiple places, we still
+ * need to track the allocation failure and ensure that
+ * any additional mappings remain uncompressed.
+ *
+ * This is handled by returning an empty nvkm_tags.
+ */
+ tags->mn = NULL;
+ }
+
+ refcount_set(&tags->refcount, 1);
+ mutex_unlock(&fb->subdev.mutex);
+ *ptags = tags;
+ return 0;
+}
+
+void
nvkm_memory_ctor(const struct nvkm_memory_func *func,
struct nvkm_memory *memory)
{
memory->func = func;
+ kref_init(&memory->kref);
+}
+
+static void
+nvkm_memory_del(struct kref *kref)
+{
+ struct nvkm_memory *memory = container_of(kref, typeof(*memory), kref);
+ if (!WARN_ON(!memory->func)) {
+ if (memory->func->dtor)
+ memory = memory->func->dtor(memory);
+ kfree(memory);
+ }
}
void
-nvkm_memory_del(struct nvkm_memory **pmemory)
+nvkm_memory_unref(struct nvkm_memory **pmemory)
{
struct nvkm_memory *memory = *pmemory;
- if (memory && !WARN_ON(!memory->func)) {
- if (memory->func->dtor)
- *pmemory = memory->func->dtor(memory);
- kfree(*pmemory);
+ if (memory) {
+ kref_put(&memory->kref, nvkm_memory_del);
*pmemory = NULL;
}
}
+struct nvkm_memory *
+nvkm_memory_ref(struct nvkm_memory *memory)
+{
+ if (memory)
+ kref_get(&memory->kref);
+ return memory;
+}
+
int
nvkm_memory_new(struct nvkm_device *device, enum nvkm_memory_target target,
u64 size, u32 align, bool zero,
diff --git a/drivers/gpu/drm/nouveau/nvkm/core/mm.c b/drivers/gpu/drm/nouveau/nvkm/core/mm.c
index 5c7891234eea..f78a06a6b2f1 100644
--- a/drivers/gpu/drm/nouveau/nvkm/core/mm.c
+++ b/drivers/gpu/drm/nouveau/nvkm/core/mm.c
@@ -237,7 +237,7 @@ nvkm_mm_tail(struct nvkm_mm *mm, u8 heap, u8 type, u32 size_max, u32 size_min,
}
int
-nvkm_mm_init(struct nvkm_mm *mm, u32 offset, u32 length, u32 block)
+nvkm_mm_init(struct nvkm_mm *mm, u8 heap, u32 offset, u32 length, u32 block)
{
struct nvkm_mm_node *node, *prev;
u32 next;
@@ -274,7 +274,8 @@ nvkm_mm_init(struct nvkm_mm *mm, u32 offset, u32 length, u32 block)
list_add_tail(&node->nl_entry, &mm->nodes);
list_add_tail(&node->fl_entry, &mm->free);
- node->heap = ++mm->heap_nodes;
+ node->heap = heap;
+ mm->heap_nodes++;
return 0;
}
diff --git a/drivers/gpu/drm/nouveau/nvkm/core/object.c b/drivers/gpu/drm/nouveau/nvkm/core/object.c
index acd76fd4f6d8..301a5e5b5f7f 100644
--- a/drivers/gpu/drm/nouveau/nvkm/core/object.c
+++ b/drivers/gpu/drm/nouveau/nvkm/core/object.c
@@ -102,10 +102,19 @@ nvkm_object_ntfy(struct nvkm_object *object, u32 mthd,
}
int
-nvkm_object_map(struct nvkm_object *object, u64 *addr, u32 *size)
+nvkm_object_map(struct nvkm_object *object, void *argv, u32 argc,
+ enum nvkm_object_map *type, u64 *addr, u64 *size)
{
if (likely(object->func->map))
- return object->func->map(object, addr, size);
+ return object->func->map(object, argv, argc, type, addr, size);
+ return -ENODEV;
+}
+
+int
+nvkm_object_unmap(struct nvkm_object *object)
+{
+ if (likely(object->func->unmap))
+ return object->func->unmap(object);
return -ENODEV;
}
@@ -259,6 +268,7 @@ nvkm_object_dtor(struct nvkm_object *object)
}
nvif_debug(object, "destroy running...\n");
+ nvkm_object_unmap(object);
if (object->func->dtor)
data = object->func->dtor(object);
nvkm_engine_unref(&object->engine);
diff --git a/drivers/gpu/drm/nouveau/nvkm/core/oproxy.c b/drivers/gpu/drm/nouveau/nvkm/core/oproxy.c
index e31a0479add0..16299837a296 100644
--- a/drivers/gpu/drm/nouveau/nvkm/core/oproxy.c
+++ b/drivers/gpu/drm/nouveau/nvkm/core/oproxy.c
@@ -37,9 +37,17 @@ nvkm_oproxy_ntfy(struct nvkm_object *object, u32 mthd,
}
static int
-nvkm_oproxy_map(struct nvkm_object *object, u64 *addr, u32 *size)
+nvkm_oproxy_map(struct nvkm_object *object, void *argv, u32 argc,
+ enum nvkm_object_map *type, u64 *addr, u64 *size)
{
- return nvkm_object_map(nvkm_oproxy(object)->object, addr, size);
+ struct nvkm_oproxy *oproxy = nvkm_oproxy(object);
+ return nvkm_object_map(oproxy->object, argv, argc, type, addr, size);
+}
+
+static int
+nvkm_oproxy_unmap(struct nvkm_object *object)
+{
+ return nvkm_object_unmap(nvkm_oproxy(object)->object);
}
static int
@@ -171,6 +179,7 @@ nvkm_oproxy_func = {
.mthd = nvkm_oproxy_mthd,
.ntfy = nvkm_oproxy_ntfy,
.map = nvkm_oproxy_map,
+ .unmap = nvkm_oproxy_unmap,
.rd08 = nvkm_oproxy_rd08,
.rd16 = nvkm_oproxy_rd16,
.rd32 = nvkm_oproxy_rd32,
diff --git a/drivers/gpu/drm/nouveau/nvkm/core/ramht.c b/drivers/gpu/drm/nouveau/nvkm/core/ramht.c
index 89da47234016..ccba4ae73cc5 100644
--- a/drivers/gpu/drm/nouveau/nvkm/core/ramht.c
+++ b/drivers/gpu/drm/nouveau/nvkm/core/ramht.c
@@ -21,6 +21,7 @@
*/
#include <core/ramht.h>
#include <core/engine.h>
+#include <core/object.h>
static u32
nvkm_ramht_hash(struct nvkm_ramht *ramht, int chid, u32 handle)
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/bsp/g84.c b/drivers/gpu/drm/nouveau/nvkm/engine/bsp/g84.c
index 8e2e24a74774..44e116f7880d 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/bsp/g84.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/bsp/g84.c
@@ -39,5 +39,5 @@ int
g84_bsp_new(struct nvkm_device *device, int index, struct nvkm_engine **pengine)
{
return nvkm_xtensa_new_(&g84_bsp, device, index,
- true, 0x103000, pengine);
+ device->chipset != 0x92, 0x103000, pengine);
}
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/device/base.c b/drivers/gpu/drm/nouveau/nvkm/engine/device/base.c
index e096a5d9c292..e14643615698 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/device/base.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/device/base.c
@@ -927,7 +927,7 @@ nv84_chipset = {
.i2c = nv50_i2c_new,
.imem = nv50_instmem_new,
.mc = g84_mc_new,
- .mmu = nv50_mmu_new,
+ .mmu = g84_mmu_new,
.mxm = nv50_mxm_new,
.pci = g84_pci_new,
.therm = g84_therm_new,
@@ -959,7 +959,7 @@ nv86_chipset = {
.i2c = nv50_i2c_new,
.imem = nv50_instmem_new,
.mc = g84_mc_new,
- .mmu = nv50_mmu_new,
+ .mmu = g84_mmu_new,
.mxm = nv50_mxm_new,
.pci = g84_pci_new,
.therm = g84_therm_new,
@@ -991,7 +991,7 @@ nv92_chipset = {
.i2c = nv50_i2c_new,
.imem = nv50_instmem_new,
.mc = g84_mc_new,
- .mmu = nv50_mmu_new,
+ .mmu = g84_mmu_new,
.mxm = nv50_mxm_new,
.pci = g92_pci_new,
.therm = g84_therm_new,
@@ -1023,7 +1023,7 @@ nv94_chipset = {
.i2c = g94_i2c_new,
.imem = nv50_instmem_new,
.mc = g84_mc_new,
- .mmu = nv50_mmu_new,
+ .mmu = g84_mmu_new,
.mxm = nv50_mxm_new,
.pci = g94_pci_new,
.therm = g84_therm_new,
@@ -1055,7 +1055,7 @@ nv96_chipset = {
.i2c = g94_i2c_new,
.imem = nv50_instmem_new,
.mc = g84_mc_new,
- .mmu = nv50_mmu_new,
+ .mmu = g84_mmu_new,
.mxm = nv50_mxm_new,
.pci = g94_pci_new,
.therm = g84_therm_new,
@@ -1087,7 +1087,7 @@ nv98_chipset = {
.i2c = g94_i2c_new,
.imem = nv50_instmem_new,
.mc = g98_mc_new,
- .mmu = nv50_mmu_new,
+ .mmu = g84_mmu_new,
.mxm = nv50_mxm_new,
.pci = g94_pci_new,
.therm = g84_therm_new,
@@ -1119,7 +1119,7 @@ nva0_chipset = {
.i2c = nv50_i2c_new,
.imem = nv50_instmem_new,
.mc = g84_mc_new,
- .mmu = nv50_mmu_new,
+ .mmu = g84_mmu_new,
.mxm = nv50_mxm_new,
.pci = g94_pci_new,
.therm = g84_therm_new,
@@ -1151,7 +1151,7 @@ nva3_chipset = {
.i2c = g94_i2c_new,
.imem = nv50_instmem_new,
.mc = gt215_mc_new,
- .mmu = nv50_mmu_new,
+ .mmu = g84_mmu_new,
.mxm = nv50_mxm_new,
.pci = g94_pci_new,
.pmu = gt215_pmu_new,
@@ -1185,7 +1185,7 @@ nva5_chipset = {
.i2c = g94_i2c_new,
.imem = nv50_instmem_new,
.mc = gt215_mc_new,
- .mmu = nv50_mmu_new,
+ .mmu = g84_mmu_new,
.mxm = nv50_mxm_new,
.pci = g94_pci_new,
.pmu = gt215_pmu_new,
@@ -1218,7 +1218,7 @@ nva8_chipset = {
.i2c = g94_i2c_new,
.imem = nv50_instmem_new,
.mc = gt215_mc_new,
- .mmu = nv50_mmu_new,
+ .mmu = g84_mmu_new,
.mxm = nv50_mxm_new,
.pci = g94_pci_new,
.pmu = gt215_pmu_new,
@@ -1251,7 +1251,7 @@ nvaa_chipset = {
.i2c = g94_i2c_new,
.imem = nv50_instmem_new,
.mc = g98_mc_new,
- .mmu = nv50_mmu_new,
+ .mmu = g84_mmu_new,
.mxm = nv50_mxm_new,
.pci = g94_pci_new,
.therm = g84_therm_new,
@@ -1283,7 +1283,7 @@ nvac_chipset = {
.i2c = g94_i2c_new,
.imem = nv50_instmem_new,
.mc = g98_mc_new,
- .mmu = nv50_mmu_new,
+ .mmu = g84_mmu_new,
.mxm = nv50_mxm_new,
.pci = g94_pci_new,
.therm = g84_therm_new,
@@ -1315,7 +1315,7 @@ nvaf_chipset = {
.i2c = g94_i2c_new,
.imem = nv50_instmem_new,
.mc = gt215_mc_new,
- .mmu = nv50_mmu_new,
+ .mmu = g84_mmu_new,
.mxm = nv50_mxm_new,
.pci = g94_pci_new,
.pmu = gt215_pmu_new,
@@ -1678,7 +1678,7 @@ nve4_chipset = {
.imem = nv50_instmem_new,
.ltc = gk104_ltc_new,
.mc = gk104_mc_new,
- .mmu = gf100_mmu_new,
+ .mmu = gk104_mmu_new,
.mxm = nv50_mxm_new,
.pci = gk104_pci_new,
.pmu = gk104_pmu_new,
@@ -1717,7 +1717,7 @@ nve6_chipset = {
.imem = nv50_instmem_new,
.ltc = gk104_ltc_new,
.mc = gk104_mc_new,
- .mmu = gf100_mmu_new,
+ .mmu = gk104_mmu_new,
.mxm = nv50_mxm_new,
.pci = gk104_pci_new,
.pmu = gk104_pmu_new,
@@ -1756,7 +1756,7 @@ nve7_chipset = {
.imem = nv50_instmem_new,
.ltc = gk104_ltc_new,
.mc = gk104_mc_new,
- .mmu = gf100_mmu_new,
+ .mmu = gk104_mmu_new,
.mxm = nv50_mxm_new,
.pci = gk104_pci_new,
.pmu = gk104_pmu_new,
@@ -1790,7 +1790,7 @@ nvea_chipset = {
.imem = gk20a_instmem_new,
.ltc = gk104_ltc_new,
.mc = gk20a_mc_new,
- .mmu = gf100_mmu_new,
+ .mmu = gk20a_mmu_new,
.pmu = gk20a_pmu_new,
.timer = gk20a_timer_new,
.top = gk104_top_new,
@@ -1820,7 +1820,7 @@ nvf0_chipset = {
.imem = nv50_instmem_new,
.ltc = gk104_ltc_new,
.mc = gk104_mc_new,
- .mmu = gf100_mmu_new,
+ .mmu = gk104_mmu_new,
.mxm = nv50_mxm_new,
.pci = gk104_pci_new,
.pmu = gk110_pmu_new,
@@ -1858,7 +1858,7 @@ nvf1_chipset = {
.imem = nv50_instmem_new,
.ltc = gk104_ltc_new,
.mc = gk104_mc_new,
- .mmu = gf100_mmu_new,
+ .mmu = gk104_mmu_new,
.mxm = nv50_mxm_new,
.pci = gk104_pci_new,
.pmu = gk110_pmu_new,
@@ -1896,7 +1896,7 @@ nv106_chipset = {
.imem = nv50_instmem_new,
.ltc = gk104_ltc_new,
.mc = gk20a_mc_new,
- .mmu = gf100_mmu_new,
+ .mmu = gk104_mmu_new,
.mxm = nv50_mxm_new,
.pci = gk104_pci_new,
.pmu = gk208_pmu_new,
@@ -1934,7 +1934,7 @@ nv108_chipset = {
.imem = nv50_instmem_new,
.ltc = gk104_ltc_new,
.mc = gk20a_mc_new,
- .mmu = gf100_mmu_new,
+ .mmu = gk104_mmu_new,
.mxm = nv50_mxm_new,
.pci = gk104_pci_new,
.pmu = gk208_pmu_new,
@@ -1958,7 +1958,7 @@ nv108_chipset = {
static const struct nvkm_device_chip
nv117_chipset = {
.name = "GM107",
- .bar = gf100_bar_new,
+ .bar = gm107_bar_new,
.bios = nvkm_bios_new,
.bus = gf100_bus_new,
.clk = gk104_clk_new,
@@ -1972,7 +1972,7 @@ nv117_chipset = {
.imem = nv50_instmem_new,
.ltc = gm107_ltc_new,
.mc = gk20a_mc_new,
- .mmu = gf100_mmu_new,
+ .mmu = gk104_mmu_new,
.mxm = nv50_mxm_new,
.pci = gk104_pci_new,
.pmu = gm107_pmu_new,
@@ -1992,7 +1992,7 @@ nv117_chipset = {
static const struct nvkm_device_chip
nv118_chipset = {
.name = "GM108",
- .bar = gf100_bar_new,
+ .bar = gm107_bar_new,
.bios = nvkm_bios_new,
.bus = gf100_bus_new,
.clk = gk104_clk_new,
@@ -2006,7 +2006,7 @@ nv118_chipset = {
.imem = nv50_instmem_new,
.ltc = gm107_ltc_new,
.mc = gk20a_mc_new,
- .mmu = gf100_mmu_new,
+ .mmu = gk104_mmu_new,
.mxm = nv50_mxm_new,
.pci = gk104_pci_new,
.pmu = gm107_pmu_new,
@@ -2026,7 +2026,7 @@ nv118_chipset = {
static const struct nvkm_device_chip
nv120_chipset = {
.name = "GM200",
- .bar = gf100_bar_new,
+ .bar = gm107_bar_new,
.bios = nvkm_bios_new,
.bus = gf100_bus_new,
.devinit = gm200_devinit_new,
@@ -2039,7 +2039,7 @@ nv120_chipset = {
.imem = nv50_instmem_new,
.ltc = gm200_ltc_new,
.mc = gk20a_mc_new,
- .mmu = gf100_mmu_new,
+ .mmu = gm200_mmu_new,
.mxm = nv50_mxm_new,
.pci = gk104_pci_new,
.pmu = gm107_pmu_new,
@@ -2061,7 +2061,7 @@ nv120_chipset = {
static const struct nvkm_device_chip
nv124_chipset = {
.name = "GM204",
- .bar = gf100_bar_new,
+ .bar = gm107_bar_new,
.bios = nvkm_bios_new,
.bus = gf100_bus_new,
.devinit = gm200_devinit_new,
@@ -2074,7 +2074,7 @@ nv124_chipset = {
.imem = nv50_instmem_new,
.ltc = gm200_ltc_new,
.mc = gk20a_mc_new,
- .mmu = gf100_mmu_new,
+ .mmu = gm200_mmu_new,
.mxm = nv50_mxm_new,
.pci = gk104_pci_new,
.pmu = gm107_pmu_new,
@@ -2096,7 +2096,7 @@ nv124_chipset = {
static const struct nvkm_device_chip
nv126_chipset = {
.name = "GM206",
- .bar = gf100_bar_new,
+ .bar = gm107_bar_new,
.bios = nvkm_bios_new,
.bus = gf100_bus_new,
.devinit = gm200_devinit_new,
@@ -2109,7 +2109,7 @@ nv126_chipset = {
.imem = nv50_instmem_new,
.ltc = gm200_ltc_new,
.mc = gk20a_mc_new,
- .mmu = gf100_mmu_new,
+ .mmu = gm200_mmu_new,
.mxm = nv50_mxm_new,
.pci = gk104_pci_new,
.pmu = gm107_pmu_new,
@@ -2131,7 +2131,7 @@ nv126_chipset = {
static const struct nvkm_device_chip
nv12b_chipset = {
.name = "GM20B",
- .bar = gk20a_bar_new,
+ .bar = gm20b_bar_new,
.bus = gf100_bus_new,
.clk = gm20b_clk_new,
.fb = gm20b_fb_new,
@@ -2140,7 +2140,7 @@ nv12b_chipset = {
.imem = gk20a_instmem_new,
.ltc = gm200_ltc_new,
.mc = gk20a_mc_new,
- .mmu = gf100_mmu_new,
+ .mmu = gm20b_mmu_new,
.pmu = gm20b_pmu_new,
.secboot = gm20b_secboot_new,
.timer = gk20a_timer_new,
@@ -2156,7 +2156,7 @@ nv12b_chipset = {
static const struct nvkm_device_chip
nv130_chipset = {
.name = "GP100",
- .bar = gf100_bar_new,
+ .bar = gm107_bar_new,
.bios = nvkm_bios_new,
.bus = gf100_bus_new,
.devinit = gm200_devinit_new,
@@ -2168,7 +2168,8 @@ nv130_chipset = {
.imem = nv50_instmem_new,
.ltc = gp100_ltc_new,
.mc = gp100_mc_new,
- .mmu = gf100_mmu_new,
+ .mmu = gp100_mmu_new,
+ .therm = gp100_therm_new,
.secboot = gm200_secboot_new,
.pci = gp100_pci_new,
.pmu = gp100_pmu_new,
@@ -2190,7 +2191,7 @@ nv130_chipset = {
static const struct nvkm_device_chip
nv132_chipset = {
.name = "GP102",
- .bar = gf100_bar_new,
+ .bar = gm107_bar_new,
.bios = nvkm_bios_new,
.bus = gf100_bus_new,
.devinit = gm200_devinit_new,
@@ -2202,7 +2203,8 @@ nv132_chipset = {
.imem = nv50_instmem_new,
.ltc = gp100_ltc_new,
.mc = gp100_mc_new,
- .mmu = gf100_mmu_new,
+ .mmu = gp100_mmu_new,
+ .therm = gp100_therm_new,
.secboot = gp102_secboot_new,
.pci = gp100_pci_new,
.pmu = gp102_pmu_new,
@@ -2224,7 +2226,7 @@ nv132_chipset = {
static const struct nvkm_device_chip
nv134_chipset = {
.name = "GP104",
- .bar = gf100_bar_new,
+ .bar = gm107_bar_new,
.bios = nvkm_bios_new,
.bus = gf100_bus_new,
.devinit = gm200_devinit_new,
@@ -2236,7 +2238,8 @@ nv134_chipset = {
.imem = nv50_instmem_new,
.ltc = gp100_ltc_new,
.mc = gp100_mc_new,
- .mmu = gf100_mmu_new,
+ .mmu = gp100_mmu_new,
+ .therm = gp100_therm_new,
.secboot = gp102_secboot_new,
.pci = gp100_pci_new,
.pmu = gp102_pmu_new,
@@ -2258,7 +2261,7 @@ nv134_chipset = {
static const struct nvkm_device_chip
nv136_chipset = {
.name = "GP106",
- .bar = gf100_bar_new,
+ .bar = gm107_bar_new,
.bios = nvkm_bios_new,
.bus = gf100_bus_new,
.devinit = gm200_devinit_new,
@@ -2270,7 +2273,8 @@ nv136_chipset = {
.imem = nv50_instmem_new,
.ltc = gp100_ltc_new,
.mc = gp100_mc_new,
- .mmu = gf100_mmu_new,
+ .mmu = gp100_mmu_new,
+ .therm = gp100_therm_new,
.secboot = gp102_secboot_new,
.pci = gp100_pci_new,
.pmu = gp102_pmu_new,
@@ -2292,7 +2296,7 @@ nv136_chipset = {
static const struct nvkm_device_chip
nv137_chipset = {
.name = "GP107",
- .bar = gf100_bar_new,
+ .bar = gm107_bar_new,
.bios = nvkm_bios_new,
.bus = gf100_bus_new,
.devinit = gm200_devinit_new,
@@ -2304,7 +2308,8 @@ nv137_chipset = {
.imem = nv50_instmem_new,
.ltc = gp100_ltc_new,
.mc = gp100_mc_new,
- .mmu = gf100_mmu_new,
+ .mmu = gp100_mmu_new,
+ .therm = gp100_therm_new,
.secboot = gp102_secboot_new,
.pci = gp100_pci_new,
.pmu = gp102_pmu_new,
@@ -2326,7 +2331,7 @@ nv137_chipset = {
static const struct nvkm_device_chip
nv138_chipset = {
.name = "GP108",
- .bar = gf100_bar_new,
+ .bar = gm107_bar_new,
.bios = nvkm_bios_new,
.bus = gf100_bus_new,
.devinit = gm200_devinit_new,
@@ -2338,7 +2343,8 @@ nv138_chipset = {
.imem = nv50_instmem_new,
.ltc = gp100_ltc_new,
.mc = gp100_mc_new,
- .mmu = gf100_mmu_new,
+ .mmu = gp100_mmu_new,
+ .therm = gp100_therm_new,
.pci = gp100_pci_new,
.pmu = gp102_pmu_new,
.timer = gk20a_timer_new,
@@ -2355,7 +2361,7 @@ nv138_chipset = {
static const struct nvkm_device_chip
nv13b_chipset = {
.name = "GP10B",
- .bar = gk20a_bar_new,
+ .bar = gm20b_bar_new,
.bus = gf100_bus_new,
.fb = gp10b_fb_new,
.fuse = gm107_fuse_new,
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/device/ctrl.h b/drivers/gpu/drm/nouveau/nvkm/engine/device/ctrl.h
index 20249d8e444d..2c3c3ee3c494 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/device/ctrl.h
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/device/ctrl.h
@@ -1,7 +1,7 @@
#ifndef __NVKM_DEVICE_CTRL_H__
#define __NVKM_DEVICE_CTRL_H__
#define nvkm_control(p) container_of((p), struct nvkm_control, object)
-#include <core/device.h>
+#include <core/object.h>
struct nvkm_control {
struct nvkm_object object;
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/device/pci.c b/drivers/gpu/drm/nouveau/nvkm/engine/device/pci.c
index 74a1ffa425f7..f302d2b5782a 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/device/pci.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/device/pci.c
@@ -1627,7 +1627,7 @@ nvkm_device_pci_new(struct pci_dev *pci_dev, const char *cfg, const char *dbg,
const struct nvkm_device_pci_vendor *pciv;
const char *name = NULL;
struct nvkm_device_pci *pdev;
- int ret;
+ int ret, bits;
ret = pci_enable_device(pci_dev);
if (ret)
@@ -1679,17 +1679,17 @@ nvkm_device_pci_new(struct pci_dev *pci_dev, const char *cfg, const char *dbg,
if (ret)
return ret;
- /*
- * Set a preliminary DMA mask based on the .dma_bits member of the
- * MMU subdevice. This allows other subdevices to create DMA mappings
- * in their init() or oneinit() methods, which may be called before the
- * TTM layer sets the DMA mask definitively.
- * This is necessary for platforms where the default DMA mask of 32
- * does not cover any system memory, i.e., when all RAM is > 4 GB.
- */
- if (pdev->device.mmu)
- dma_set_mask_and_coherent(&pci_dev->dev,
- DMA_BIT_MASK(pdev->device.mmu->dma_bits));
+ /* Set DMA mask based on capabilities reported by the MMU subdev. */
+ if (pdev->device.mmu && !pdev->device.pci->agp.bridge)
+ bits = pdev->device.mmu->dma_bits;
+ else
+ bits = 32;
+
+ ret = dma_set_mask_and_coherent(&pci_dev->dev, DMA_BIT_MASK(bits));
+ if (ret && bits != 32) {
+ dma_set_mask_and_coherent(&pci_dev->dev, DMA_BIT_MASK(32));
+ pdev->device.mmu->dma_bits = 32;
+ }
return 0;
}
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/device/tegra.c b/drivers/gpu/drm/nouveau/nvkm/engine/device/tegra.c
index 189ed80e21ff..78597da6313a 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/device/tegra.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/device/tegra.c
@@ -136,7 +136,7 @@ nvkm_device_tegra_probe_iommu(struct nvkm_device_tegra *tdev)
if (ret)
goto free_domain;
- ret = nvkm_mm_init(&tdev->iommu.mm, 0,
+ ret = nvkm_mm_init(&tdev->iommu.mm, 0, 0,
(1ULL << tdev->func->iommu_bit) >>
tdev->iommu.pgshift, 1);
if (ret)
@@ -216,7 +216,7 @@ nvkm_device_tegra_fini(struct nvkm_device *device, bool suspend)
if (tdev->irq) {
free_irq(tdev->irq, tdev);
tdev->irq = 0;
- };
+ }
}
static int
@@ -309,8 +309,6 @@ nvkm_device_tegra_new(const struct nvkm_device_tegra_func *func,
/**
* The IOMMU bit defines the upper limit of the GPU-addressable space.
- * This will be refined in nouveau_ttm_init but we need to do it early
- * for instmem to behave properly
*/
ret = dma_set_mask(&pdev->dev, DMA_BIT_MASK(tdev->func->iommu_bit));
if (ret)
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/device/user.c b/drivers/gpu/drm/nouveau/nvkm/engine/device/user.c
index 513ee6b79553..17adcb4e8854 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/device/user.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/device/user.c
@@ -206,10 +206,12 @@ nvkm_udevice_wr32(struct nvkm_object *object, u64 addr, u32 data)
}
static int
-nvkm_udevice_map(struct nvkm_object *object, u64 *addr, u32 *size)
+nvkm_udevice_map(struct nvkm_object *object, void *argv, u32 argc,
+ enum nvkm_object_map *type, u64 *addr, u64 *size)
{
struct nvkm_udevice *udev = nvkm_udevice(object);
struct nvkm_device *device = udev->device;
+ *type = NVKM_OBJECT_MAP_IO;
*addr = device->func->resource_addr(device, 0);
*size = device->func->resource_size(device, 0);
return 0;
@@ -292,6 +294,11 @@ nvkm_udevice_child_get(struct nvkm_object *object, int index,
if (!sclass) {
switch (index) {
case 0: sclass = &nvkm_control_oclass; break;
+ case 1:
+ if (!device->mmu)
+ return -EINVAL;
+ sclass = &device->mmu->user;
+ break;
default:
return -EINVAL;
}
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/channv50.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/channv50.c
index 0c0310498afd..723dcbde2ac2 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/channv50.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/channv50.c
@@ -191,11 +191,13 @@ nv50_disp_chan_ntfy(struct nvkm_object *object, u32 type,
}
static int
-nv50_disp_chan_map(struct nvkm_object *object, u64 *addr, u32 *size)
+nv50_disp_chan_map(struct nvkm_object *object, void *argv, u32 argc,
+ enum nvkm_object_map *type, u64 *addr, u64 *size)
{
struct nv50_disp_chan *chan = nv50_disp_chan(object);
struct nv50_disp *disp = chan->root->disp;
struct nvkm_device *device = disp->base.engine.subdev.device;
+ *type = NVKM_OBJECT_MAP_IO;
*addr = device->func->resource_addr(device, 0) +
0x640000 + (chan->chid.user * 0x1000);
*size = 0x001000;
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/channv50.h b/drivers/gpu/drm/nouveau/nvkm/engine/disp/channv50.h
index 737b38f6fbd2..9bb4ad5b0e57 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/channv50.h
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/channv50.h
@@ -1,6 +1,7 @@
#ifndef __NV50_DISP_CHAN_H__
#define __NV50_DISP_CHAN_H__
#define nv50_disp_chan(p) container_of((p), struct nv50_disp_chan, object)
+#include <core/object.h>
#include "nv50.h"
struct nv50_disp_chan {
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/ior.h b/drivers/gpu/drm/nouveau/nvkm/engine/disp/ior.h
index a1e8bf48b778..c9e0a8f7b5d5 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/ior.h
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/ior.h
@@ -147,7 +147,7 @@ void gf119_hda_eld(struct nvkm_ior *, u8 *, u8);
#define IOR_MSG(i,l,f,a...) do { \
struct nvkm_ior *_ior = (i); \
- nvkm_##l(&_ior->disp->engine.subdev, "%s: "f, _ior->name, ##a); \
+ nvkm_##l(&_ior->disp->engine.subdev, "%s: "f"\n", _ior->name, ##a); \
} while(0)
#define IOR_WARN(i,f,a...) IOR_MSG((i), warn, f, ##a)
#define IOR_DBG(i,f,a...) IOR_MSG((i), debug, f, ##a)
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/dma/usernv04.c b/drivers/gpu/drm/nouveau/nvkm/engine/dma/usernv04.c
index c95942ef8216..49ef7e57aad4 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/dma/usernv04.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/dma/usernv04.c
@@ -26,7 +26,7 @@
#include <core/gpuobj.h>
#include <subdev/fb.h>
-#include <subdev/mmu/nv04.h>
+#include <subdev/mmu/vmm.h>
#include <nvif/class.h>
@@ -49,8 +49,8 @@ nv04_dmaobj_bind(struct nvkm_dmaobj *base, struct nvkm_gpuobj *parent,
int ret;
if (dmaobj->clone) {
- struct nv04_mmu *mmu = nv04_mmu(device->mmu);
- struct nvkm_memory *pgt = mmu->vm->pgt[0].mem[0];
+ struct nvkm_memory *pgt =
+ device->mmu->vmm->pd->pt[0]->memory;
if (!dmaobj->base.start)
return nvkm_gpuobj_wrap(pgt, pgpuobj);
nvkm_kmap(pgt);
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/falcon.c b/drivers/gpu/drm/nouveau/nvkm/engine/falcon.c
index 2e7b4e2105ef..816ccaedfc73 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/falcon.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/falcon.c
@@ -99,7 +99,7 @@ nvkm_falcon_fini(struct nvkm_engine *engine, bool suspend)
const u32 base = falcon->addr;
if (!suspend) {
- nvkm_memory_del(&falcon->core);
+ nvkm_memory_unref(&falcon->core);
if (falcon->external) {
vfree(falcon->data.data);
vfree(falcon->code.data);
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c
index 660ca7aa95ea..64f6b7654a08 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c
@@ -27,6 +27,7 @@
#include <core/client.h>
#include <core/gpuobj.h>
#include <core/notify.h>
+#include <subdev/mc.h>
#include <nvif/event.h>
#include <nvif/unpack.h>
@@ -278,6 +279,12 @@ nvkm_fifo_oneinit(struct nvkm_engine *engine)
return 0;
}
+static void
+nvkm_fifo_preinit(struct nvkm_engine *engine)
+{
+ nvkm_mc_reset(engine->subdev.device, NVKM_ENGINE_FIFO);
+}
+
static int
nvkm_fifo_init(struct nvkm_engine *engine)
{
@@ -302,6 +309,7 @@ nvkm_fifo_dtor(struct nvkm_engine *engine)
static const struct nvkm_engine_func
nvkm_fifo = {
.dtor = nvkm_fifo_dtor,
+ .preinit = nvkm_fifo_preinit,
.oneinit = nvkm_fifo_oneinit,
.init = nvkm_fifo_init,
.fini = nvkm_fifo_fini,
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.c
index fab760ae922f..d83485385934 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.c
@@ -117,8 +117,8 @@ nvkm_fifo_chan_child_del(struct nvkm_oproxy *base)
if (chan->func->engine_dtor)
chan->func->engine_dtor(chan, engine);
nvkm_object_del(&engn->object);
- if (chan->vm)
- atomic_dec(&chan->vm->engref[engine->subdev.index]);
+ if (chan->vmm)
+ atomic_dec(&chan->vmm->engref[engine->subdev.index]);
}
}
@@ -151,8 +151,8 @@ nvkm_fifo_chan_child_new(const struct nvkm_oclass *oclass, void *data, u32 size,
.engine = oclass->engine,
};
- if (chan->vm)
- atomic_inc(&chan->vm->engref[engine->subdev.index]);
+ if (chan->vmm)
+ atomic_inc(&chan->vmm->engref[engine->subdev.index]);
if (engine->func->fifo.cclass) {
ret = engine->func->fifo.cclass(chan, &cclass,
@@ -253,9 +253,11 @@ nvkm_fifo_chan_ntfy(struct nvkm_object *object, u32 type,
}
static int
-nvkm_fifo_chan_map(struct nvkm_object *object, u64 *addr, u32 *size)
+nvkm_fifo_chan_map(struct nvkm_object *object, void *argv, u32 argc,
+ enum nvkm_object_map *type, u64 *addr, u64 *size)
{
struct nvkm_fifo_chan *chan = nvkm_fifo_chan(object);
+ *type = NVKM_OBJECT_MAP_IO;
*addr = chan->addr;
*size = chan->size;
return 0;
@@ -325,7 +327,10 @@ nvkm_fifo_chan_dtor(struct nvkm_object *object)
if (chan->user)
iounmap(chan->user);
- nvkm_vm_ref(NULL, &chan->vm, NULL);
+ if (chan->vmm) {
+ nvkm_vmm_part(chan->vmm, chan->inst->memory);
+ nvkm_vmm_unref(&chan->vmm);
+ }
nvkm_gpuobj_del(&chan->push);
nvkm_gpuobj_del(&chan->inst);
@@ -347,13 +352,12 @@ nvkm_fifo_chan_func = {
int
nvkm_fifo_chan_ctor(const struct nvkm_fifo_chan_func *func,
struct nvkm_fifo *fifo, u32 size, u32 align, bool zero,
- u64 vm, u64 push, u64 engines, int bar, u32 base, u32 user,
- const struct nvkm_oclass *oclass,
+ u64 hvmm, u64 push, u64 engines, int bar, u32 base,
+ u32 user, const struct nvkm_oclass *oclass,
struct nvkm_fifo_chan *chan)
{
struct nvkm_client *client = oclass->client;
struct nvkm_device *device = fifo->engine.subdev.device;
- struct nvkm_mmu *mmu = device->mmu;
struct nvkm_dmaobj *dmaobj;
unsigned long flags;
int ret;
@@ -382,16 +386,19 @@ nvkm_fifo_chan_ctor(const struct nvkm_fifo_chan_func *func,
}
/* channel address space */
- if (!vm && mmu) {
- if (!client->vm || client->vm->mmu == mmu) {
- ret = nvkm_vm_ref(client->vm, &chan->vm, NULL);
- if (ret)
- return ret;
- } else {
+ if (hvmm) {
+ struct nvkm_vmm *vmm = nvkm_uvmm_search(client, hvmm);
+ if (IS_ERR(vmm))
+ return PTR_ERR(vmm);
+
+ if (vmm->mmu != device->mmu)
return -EINVAL;
- }
- } else {
- return -ENOENT;
+
+ ret = nvkm_vmm_join(vmm, chan->inst->memory);
+ if (ret)
+ return ret;
+
+ chan->vmm = nvkm_vmm_ref(vmm);
}
/* allocate channel id */
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chang84.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chang84.c
index 61797c4dd07a..a5c998fe4485 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chang84.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chang84.c
@@ -229,15 +229,18 @@ g84_fifo_chan_func = {
};
int
-g84_fifo_chan_ctor(struct nv50_fifo *fifo, u64 vm, u64 push,
+g84_fifo_chan_ctor(struct nv50_fifo *fifo, u64 vmm, u64 push,
const struct nvkm_oclass *oclass,
struct nv50_fifo_chan *chan)
{
struct nvkm_device *device = fifo->base.engine.subdev.device;
int ret;
+ if (!vmm)
+ return -EINVAL;
+
ret = nvkm_fifo_chan_ctor(&g84_fifo_chan_func, &fifo->base,
- 0x10000, 0x1000, false, vm, push,
+ 0x10000, 0x1000, false, vmm, push,
(1ULL << NVKM_ENGINE_BSP) |
(1ULL << NVKM_ENGINE_CE0) |
(1ULL << NVKM_ENGINE_CIPHER) |
@@ -277,9 +280,5 @@ g84_fifo_chan_ctor(struct nv50_fifo *fifo, u64 vm, u64 push,
if (ret)
return ret;
- ret = nvkm_ramht_new(device, 0x8000, 16, chan->base.inst, &chan->ramht);
- if (ret)
- return ret;
-
- return nvkm_vm_ref(chan->base.vm, &chan->vm, chan->pgd);
+ return nvkm_ramht_new(device, 0x8000, 16, chan->base.inst, &chan->ramht);
}
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/changf100.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/changf100.h
index 7d697e2dce1a..fc1142af02cf 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/changf100.h
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/changf100.h
@@ -11,12 +11,9 @@ struct gf100_fifo_chan {
struct list_head head;
bool killed;
- struct nvkm_gpuobj *pgd;
- struct nvkm_vm *vm;
-
struct {
struct nvkm_gpuobj *inst;
- struct nvkm_vma vma;
+ struct nvkm_vma *vma;
} engn[NVKM_SUBDEV_NR];
};
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/changk104.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/changk104.h
index 230f64e5f731..5beb5c628473 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/changk104.h
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/changk104.h
@@ -12,12 +12,9 @@ struct gk104_fifo_chan {
struct list_head head;
bool killed;
- struct nvkm_gpuobj *pgd;
- struct nvkm_vm *vm;
-
struct {
struct nvkm_gpuobj *inst;
- struct nvkm_vma vma;
+ struct nvkm_vma *vma;
} engn[NVKM_SUBDEV_NR];
};
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/channv50.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/channv50.c
index 25b60aff40e4..85f7dbf53c99 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/channv50.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/channv50.c
@@ -206,7 +206,6 @@ void *
nv50_fifo_chan_dtor(struct nvkm_fifo_chan *base)
{
struct nv50_fifo_chan *chan = nv50_fifo_chan(base);
- nvkm_vm_ref(NULL, &chan->vm, chan->pgd);
nvkm_ramht_del(&chan->ramht);
nvkm_gpuobj_del(&chan->pgd);
nvkm_gpuobj_del(&chan->eng);
@@ -229,15 +228,18 @@ nv50_fifo_chan_func = {
};
int
-nv50_fifo_chan_ctor(struct nv50_fifo *fifo, u64 vm, u64 push,
+nv50_fifo_chan_ctor(struct nv50_fifo *fifo, u64 vmm, u64 push,
const struct nvkm_oclass *oclass,
struct nv50_fifo_chan *chan)
{
struct nvkm_device *device = fifo->base.engine.subdev.device;
int ret;
+ if (!vmm)
+ return -EINVAL;
+
ret = nvkm_fifo_chan_ctor(&nv50_fifo_chan_func, &fifo->base,
- 0x10000, 0x1000, false, vm, push,
+ 0x10000, 0x1000, false, vmm, push,
(1ULL << NVKM_ENGINE_DMAOBJ) |
(1ULL << NVKM_ENGINE_SW) |
(1ULL << NVKM_ENGINE_GR) |
@@ -262,9 +264,5 @@ nv50_fifo_chan_ctor(struct nv50_fifo *fifo, u64 vm, u64 push,
if (ret)
return ret;
- ret = nvkm_ramht_new(device, 0x8000, 16, chan->base.inst, &chan->ramht);
- if (ret)
- return ret;
-
- return nvkm_vm_ref(chan->base.vm, &chan->vm, chan->pgd);
+ return nvkm_ramht_new(device, 0x8000, 16, chan->base.inst, &chan->ramht);
}
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/channv50.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/channv50.h
index 4b9da469b704..d853056e040b 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/channv50.h
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/channv50.h
@@ -13,19 +13,18 @@ struct nv50_fifo_chan {
struct nvkm_gpuobj *eng;
struct nvkm_gpuobj *pgd;
struct nvkm_ramht *ramht;
- struct nvkm_vm *vm;
struct nvkm_gpuobj *engn[NVKM_SUBDEV_NR];
};
-int nv50_fifo_chan_ctor(struct nv50_fifo *, u64 vm, u64 push,
+int nv50_fifo_chan_ctor(struct nv50_fifo *, u64 vmm, u64 push,
const struct nvkm_oclass *, struct nv50_fifo_chan *);
void *nv50_fifo_chan_dtor(struct nvkm_fifo_chan *);
void nv50_fifo_chan_fini(struct nvkm_fifo_chan *);
void nv50_fifo_chan_engine_dtor(struct nvkm_fifo_chan *, struct nvkm_engine *);
void nv50_fifo_chan_object_dtor(struct nvkm_fifo_chan *, int);
-int g84_fifo_chan_ctor(struct nv50_fifo *, u64 vm, u64 push,
+int g84_fifo_chan_ctor(struct nv50_fifo *, u64 vmm, u64 push,
const struct nvkm_oclass *, struct nv50_fifo_chan *);
extern const struct nvkm_fifo_chan_oclass nv50_fifo_dma_oclass;
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/dmag84.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/dmag84.c
index caa914074752..fc34cddcd2f5 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/dmag84.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/dmag84.c
@@ -44,9 +44,9 @@ g84_fifo_dma_new(struct nvkm_fifo *base, const struct nvkm_oclass *oclass,
nvif_ioctl(parent, "create channel dma size %d\n", size);
if (!(ret = nvif_unpack(ret, &data, &size, args->v0, 0, 0, false))) {
- nvif_ioctl(parent, "create channel dma vers %d vm %llx "
+ nvif_ioctl(parent, "create channel dma vers %d vmm %llx "
"pushbuf %llx offset %016llx\n",
- args->v0.version, args->v0.vm, args->v0.pushbuf,
+ args->v0.version, args->v0.vmm, args->v0.pushbuf,
args->v0.offset);
if (!args->v0.pushbuf)
return -EINVAL;
@@ -57,7 +57,7 @@ g84_fifo_dma_new(struct nvkm_fifo *base, const struct nvkm_oclass *oclass,
return -ENOMEM;
*pobject = &chan->base.object;
- ret = g84_fifo_chan_ctor(fifo, args->v0.vm, args->v0.pushbuf,
+ ret = g84_fifo_chan_ctor(fifo, args->v0.vmm, args->v0.pushbuf,
oclass, chan);
if (ret)
return ret;
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/dmanv04.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/dmanv04.c
index 0a7b6ed5ed28..c213122cf088 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/dmanv04.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/dmanv04.c
@@ -95,6 +95,7 @@ nv04_fifo_dma_fini(struct nvkm_fifo_chan *base)
nvkm_mask(device, NV04_PFIFO_CACHE1_PULL0, 0x00000001, 0);
c = fifo->ramfc;
+ nvkm_kmap(fctx);
do {
u32 rm = ((1ULL << c->bits) - 1) << c->regs;
u32 cm = ((1ULL << c->bits) - 1) << c->ctxs;
@@ -102,6 +103,7 @@ nv04_fifo_dma_fini(struct nvkm_fifo_chan *base)
u32 cv = (nvkm_ro32(fctx, c->ctxp + data) & ~cm);
nvkm_wo32(fctx, c->ctxp + data, cv | (rv << c->ctxs));
} while ((++c)->bits);
+ nvkm_done(fctx);
c = fifo->ramfc;
do {
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/dmanv50.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/dmanv50.c
index 480bc3777be5..8043718ad150 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/dmanv50.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/dmanv50.c
@@ -44,9 +44,9 @@ nv50_fifo_dma_new(struct nvkm_fifo *base, const struct nvkm_oclass *oclass,
nvif_ioctl(parent, "create channel dma size %d\n", size);
if (!(ret = nvif_unpack(ret, &data, &size, args->v0, 0, 0, false))) {
- nvif_ioctl(parent, "create channel dma vers %d vm %llx "
+ nvif_ioctl(parent, "create channel dma vers %d vmm %llx "
"pushbuf %llx offset %016llx\n",
- args->v0.version, args->v0.vm, args->v0.pushbuf,
+ args->v0.version, args->v0.vmm, args->v0.pushbuf,
args->v0.offset);
if (!args->v0.pushbuf)
return -EINVAL;
@@ -57,7 +57,7 @@ nv50_fifo_dma_new(struct nvkm_fifo *base, const struct nvkm_oclass *oclass,
return -ENOMEM;
*pobject = &chan->base.object;
- ret = nv50_fifo_chan_ctor(fifo, args->v0.vm, args->v0.pushbuf,
+ ret = nv50_fifo_chan_ctor(fifo, args->v0.vmm, args->v0.pushbuf,
oclass, chan);
if (ret)
return ret;
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c
index cd468ab1db12..f69576868164 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c
@@ -559,6 +559,7 @@ gf100_fifo_oneinit(struct nvkm_fifo *base)
struct gf100_fifo *fifo = gf100_fifo(base);
struct nvkm_subdev *subdev = &fifo->base.engine.subdev;
struct nvkm_device *device = subdev->device;
+ struct nvkm_vmm *bar = nvkm_bar_bar1_vmm(device);
int ret;
/* Determine number of PBDMAs by checking valid enable bits. */
@@ -584,12 +585,12 @@ gf100_fifo_oneinit(struct nvkm_fifo *base)
if (ret)
return ret;
- ret = nvkm_bar_umap(device->bar, 128 * 0x1000, 12, &fifo->user.bar);
+ ret = nvkm_vmm_get(bar, 12, nvkm_memory_size(fifo->user.mem),
+ &fifo->user.bar);
if (ret)
return ret;
- nvkm_memory_map(fifo->user.mem, &fifo->user.bar, 0);
- return 0;
+ return nvkm_memory_map(fifo->user.mem, 0, bar, fifo->user.bar, NULL, 0);
}
static void
@@ -628,7 +629,7 @@ gf100_fifo_init(struct nvkm_fifo *base)
}
nvkm_mask(device, 0x002200, 0x00000001, 0x00000001);
- nvkm_wr32(device, 0x002254, 0x10000000 | fifo->user.bar.offset >> 12);
+ nvkm_wr32(device, 0x002254, 0x10000000 | fifo->user.bar->addr >> 12);
nvkm_wr32(device, 0x002100, 0xffffffff);
nvkm_wr32(device, 0x002140, 0x7fffffff);
@@ -639,10 +640,11 @@ static void *
gf100_fifo_dtor(struct nvkm_fifo *base)
{
struct gf100_fifo *fifo = gf100_fifo(base);
- nvkm_vm_put(&fifo->user.bar);
- nvkm_memory_del(&fifo->user.mem);
- nvkm_memory_del(&fifo->runlist.mem[0]);
- nvkm_memory_del(&fifo->runlist.mem[1]);
+ struct nvkm_device *device = fifo->base.engine.subdev.device;
+ nvkm_vmm_put(nvkm_bar_bar1_vmm(device), &fifo->user.bar);
+ nvkm_memory_unref(&fifo->user.mem);
+ nvkm_memory_unref(&fifo->runlist.mem[0]);
+ nvkm_memory_unref(&fifo->runlist.mem[1]);
return fifo;
}
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.h
index 70db58eab9c3..b81a2ad48aa4 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.h
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.h
@@ -26,7 +26,7 @@ struct gf100_fifo {
struct {
struct nvkm_memory *mem;
- struct nvkm_vma bar;
+ struct nvkm_vma *bar;
} user;
};
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c
index a7e55c422501..84bd703dd897 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c
@@ -771,6 +771,7 @@ gk104_fifo_oneinit(struct nvkm_fifo *base)
struct gk104_fifo *fifo = gk104_fifo(base);
struct nvkm_subdev *subdev = &fifo->base.engine.subdev;
struct nvkm_device *device = subdev->device;
+ struct nvkm_vmm *bar = nvkm_bar_bar1_vmm(device);
int engn, runl, pbid, ret, i, j;
enum nvkm_devidx engidx;
u32 *map;
@@ -834,13 +835,12 @@ gk104_fifo_oneinit(struct nvkm_fifo *base)
if (ret)
return ret;
- ret = nvkm_bar_umap(device->bar, fifo->base.nr * 0x200, 12,
- &fifo->user.bar);
+ ret = nvkm_vmm_get(bar, 12, nvkm_memory_size(fifo->user.mem),
+ &fifo->user.bar);
if (ret)
return ret;
- nvkm_memory_map(fifo->user.mem, &fifo->user.bar, 0);
- return 0;
+ return nvkm_memory_map(fifo->user.mem, 0, bar, fifo->user.bar, NULL, 0);
}
static void
@@ -866,7 +866,7 @@ gk104_fifo_init(struct nvkm_fifo *base)
nvkm_wr32(device, 0x04014c + (i * 0x2000), 0xffffffff); /* INTREN */
}
- nvkm_wr32(device, 0x002254, 0x10000000 | fifo->user.bar.offset >> 12);
+ nvkm_wr32(device, 0x002254, 0x10000000 | fifo->user.bar->addr >> 12);
nvkm_wr32(device, 0x002100, 0xffffffff);
nvkm_wr32(device, 0x002140, 0x7fffffff);
@@ -876,14 +876,15 @@ static void *
gk104_fifo_dtor(struct nvkm_fifo *base)
{
struct gk104_fifo *fifo = gk104_fifo(base);
+ struct nvkm_device *device = fifo->base.engine.subdev.device;
int i;
- nvkm_vm_put(&fifo->user.bar);
- nvkm_memory_del(&fifo->user.mem);
+ nvkm_vmm_put(nvkm_bar_bar1_vmm(device), &fifo->user.bar);
+ nvkm_memory_unref(&fifo->user.mem);
for (i = 0; i < fifo->runlist_nr; i++) {
- nvkm_memory_del(&fifo->runlist[i].mem[1]);
- nvkm_memory_del(&fifo->runlist[i].mem[0]);
+ nvkm_memory_unref(&fifo->runlist[i].mem[1]);
+ nvkm_memory_unref(&fifo->runlist[i].mem[0]);
}
return fifo;
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.h
index 44bff98d6725..466f1051f91a 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.h
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.h
@@ -37,7 +37,7 @@ struct gk104_fifo {
struct {
struct nvkm_memory *mem;
- struct nvkm_vma bar;
+ struct nvkm_vma *bar;
} user;
};
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifog84.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifog84.c
index 77c2f2a28bf3..2121f517b1dd 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifog84.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifog84.c
@@ -45,10 +45,10 @@ g84_fifo_gpfifo_new(struct nvkm_fifo *base, const struct nvkm_oclass *oclass,
nvif_ioctl(parent, "create channel gpfifo size %d\n", size);
if (!(ret = nvif_unpack(ret, &data, &size, args->v0, 0, 0, false))) {
- nvif_ioctl(parent, "create channel gpfifo vers %d vm %llx "
+ nvif_ioctl(parent, "create channel gpfifo vers %d vmm %llx "
"pushbuf %llx ioffset %016llx "
"ilength %08x\n",
- args->v0.version, args->v0.vm, args->v0.pushbuf,
+ args->v0.version, args->v0.vmm, args->v0.pushbuf,
args->v0.ioffset, args->v0.ilength);
if (!args->v0.pushbuf)
return -EINVAL;
@@ -59,7 +59,7 @@ g84_fifo_gpfifo_new(struct nvkm_fifo *base, const struct nvkm_oclass *oclass,
return -ENOMEM;
*pobject = &chan->base.object;
- ret = g84_fifo_chan_ctor(fifo, args->v0.vm, args->v0.pushbuf,
+ ret = g84_fifo_chan_ctor(fifo, args->v0.vmm, args->v0.pushbuf,
oclass, chan);
if (ret)
return ret;
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogf100.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogf100.c
index f9e0377d3d24..75f9632789b3 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogf100.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogf100.c
@@ -111,7 +111,7 @@ gf100_fifo_gpfifo_engine_init(struct nvkm_fifo_chan *base,
struct nvkm_gpuobj *inst = chan->base.inst;
if (offset) {
- u64 addr = chan->engn[engine->subdev.index].vma.offset;
+ u64 addr = chan->engn[engine->subdev.index].vma->addr;
nvkm_kmap(inst);
nvkm_wo32(inst, offset + 0x00, lower_32_bits(addr) | 4);
nvkm_wo32(inst, offset + 0x04, upper_32_bits(addr));
@@ -126,7 +126,7 @@ gf100_fifo_gpfifo_engine_dtor(struct nvkm_fifo_chan *base,
struct nvkm_engine *engine)
{
struct gf100_fifo_chan *chan = gf100_fifo_chan(base);
- nvkm_gpuobj_unmap(&chan->engn[engine->subdev.index].vma);
+ nvkm_vmm_put(chan->base.vmm, &chan->engn[engine->subdev.index].vma);
nvkm_gpuobj_del(&chan->engn[engine->subdev.index].inst);
}
@@ -146,8 +146,13 @@ gf100_fifo_gpfifo_engine_ctor(struct nvkm_fifo_chan *base,
if (ret)
return ret;
- return nvkm_gpuobj_map(chan->engn[engn].inst, chan->vm,
- NV_MEM_ACCESS_RW, &chan->engn[engn].vma);
+ ret = nvkm_vmm_get(chan->base.vmm, 12, chan->engn[engn].inst->size,
+ &chan->engn[engn].vma);
+ if (ret)
+ return ret;
+
+ return nvkm_memory_map(chan->engn[engn].inst, 0, chan->base.vmm,
+ chan->engn[engn].vma, NULL, 0);
}
static void
@@ -190,10 +195,7 @@ gf100_fifo_gpfifo_init(struct nvkm_fifo_chan *base)
static void *
gf100_fifo_gpfifo_dtor(struct nvkm_fifo_chan *base)
{
- struct gf100_fifo_chan *chan = gf100_fifo_chan(base);
- nvkm_vm_ref(NULL, &chan->vm, chan->pgd);
- nvkm_gpuobj_del(&chan->pgd);
- return chan;
+ return gf100_fifo_chan(base);
}
static const struct nvkm_fifo_chan_func
@@ -216,7 +218,6 @@ gf100_fifo_gpfifo_new(struct nvkm_fifo *base, const struct nvkm_oclass *oclass,
struct fermi_channel_gpfifo_v0 v0;
} *args = data;
struct gf100_fifo *fifo = gf100_fifo(base);
- struct nvkm_device *device = fifo->base.engine.subdev.device;
struct nvkm_object *parent = oclass->parent;
struct gf100_fifo_chan *chan;
u64 usermem, ioffset, ilength;
@@ -224,10 +225,12 @@ gf100_fifo_gpfifo_new(struct nvkm_fifo *base, const struct nvkm_oclass *oclass,
nvif_ioctl(parent, "create channel gpfifo size %d\n", size);
if (!(ret = nvif_unpack(ret, &data, &size, args->v0, 0, 0, false))) {
- nvif_ioctl(parent, "create channel gpfifo vers %d vm %llx "
+ nvif_ioctl(parent, "create channel gpfifo vers %d vmm %llx "
"ioffset %016llx ilength %08x\n",
- args->v0.version, args->v0.vm, args->v0.ioffset,
+ args->v0.version, args->v0.vmm, args->v0.ioffset,
args->v0.ilength);
+ if (!args->v0.vmm)
+ return -EINVAL;
} else
return ret;
@@ -239,7 +242,7 @@ gf100_fifo_gpfifo_new(struct nvkm_fifo *base, const struct nvkm_oclass *oclass,
INIT_LIST_HEAD(&chan->head);
ret = nvkm_fifo_chan_ctor(&gf100_fifo_gpfifo_func, &fifo->base,
- 0x1000, 0x1000, true, args->v0.vm, 0,
+ 0x1000, 0x1000, true, args->v0.vmm, 0,
(1ULL << NVKM_ENGINE_CE0) |
(1ULL << NVKM_ENGINE_CE1) |
(1ULL << NVKM_ENGINE_GR) |
@@ -247,29 +250,13 @@ gf100_fifo_gpfifo_new(struct nvkm_fifo *base, const struct nvkm_oclass *oclass,
(1ULL << NVKM_ENGINE_MSPPP) |
(1ULL << NVKM_ENGINE_MSVLD) |
(1ULL << NVKM_ENGINE_SW),
- 1, fifo->user.bar.offset, 0x1000,
+ 1, fifo->user.bar->addr, 0x1000,
oclass, &chan->base);
if (ret)
return ret;
args->v0.chid = chan->base.chid;
- /* page directory */
- ret = nvkm_gpuobj_new(device, 0x10000, 0x1000, false, NULL, &chan->pgd);
- if (ret)
- return ret;
-
- nvkm_kmap(chan->base.inst);
- nvkm_wo32(chan->base.inst, 0x0200, lower_32_bits(chan->pgd->addr));
- nvkm_wo32(chan->base.inst, 0x0204, upper_32_bits(chan->pgd->addr));
- nvkm_wo32(chan->base.inst, 0x0208, 0xffffffff);
- nvkm_wo32(chan->base.inst, 0x020c, 0x000000ff);
- nvkm_done(chan->base.inst);
-
- ret = nvkm_vm_ref(chan->base.vm, &chan->vm, chan->pgd);
- if (ret)
- return ret;
-
/* clear channel control registers */
usermem = chan->base.chid * 0x1000;
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogk104.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogk104.c
index 8abf6f8ef445..80c87521bebe 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogk104.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogk104.c
@@ -117,7 +117,7 @@ gk104_fifo_gpfifo_engine_init(struct nvkm_fifo_chan *base,
u32 offset = gk104_fifo_gpfifo_engine_addr(engine);
if (offset) {
- u64 addr = chan->engn[engine->subdev.index].vma.offset;
+ u64 addr = chan->engn[engine->subdev.index].vma->addr;
u32 datalo = lower_32_bits(addr) | 0x00000004;
u32 datahi = upper_32_bits(addr);
nvkm_kmap(inst);
@@ -138,7 +138,7 @@ gk104_fifo_gpfifo_engine_dtor(struct nvkm_fifo_chan *base,
struct nvkm_engine *engine)
{
struct gk104_fifo_chan *chan = gk104_fifo_chan(base);
- nvkm_gpuobj_unmap(&chan->engn[engine->subdev.index].vma);
+ nvkm_vmm_put(chan->base.vmm, &chan->engn[engine->subdev.index].vma);
nvkm_gpuobj_del(&chan->engn[engine->subdev.index].inst);
}
@@ -158,8 +158,13 @@ gk104_fifo_gpfifo_engine_ctor(struct nvkm_fifo_chan *base,
if (ret)
return ret;
- return nvkm_gpuobj_map(chan->engn[engn].inst, chan->vm,
- NV_MEM_ACCESS_RW, &chan->engn[engn].vma);
+ ret = nvkm_vmm_get(chan->base.vmm, 12, chan->engn[engn].inst->size,
+ &chan->engn[engn].vma);
+ if (ret)
+ return ret;
+
+ return nvkm_memory_map(chan->engn[engn].inst, 0, chan->base.vmm,
+ chan->engn[engn].vma, NULL, 0);
}
static void
@@ -203,10 +208,7 @@ gk104_fifo_gpfifo_init(struct nvkm_fifo_chan *base)
static void *
gk104_fifo_gpfifo_dtor(struct nvkm_fifo_chan *base)
{
- struct gk104_fifo_chan *chan = gk104_fifo_chan(base);
- nvkm_vm_ref(NULL, &chan->vm, chan->pgd);
- nvkm_gpuobj_del(&chan->pgd);
- return chan;
+ return gk104_fifo_chan(base);
}
static const struct nvkm_fifo_chan_func
@@ -229,17 +231,19 @@ struct gk104_fifo_chan_func {
static int
gk104_fifo_gpfifo_new_(const struct gk104_fifo_chan_func *func,
struct gk104_fifo *fifo, u32 *engmask, u16 *chid,
- u64 vm, u64 ioffset, u64 ilength,
+ u64 vmm, u64 ioffset, u64 ilength,
const struct nvkm_oclass *oclass,
struct nvkm_object **pobject)
{
- struct nvkm_device *device = fifo->base.engine.subdev.device;
struct gk104_fifo_chan *chan;
int runlist = -1, ret = -ENOSYS, i, j;
u32 engines = 0, present = 0;
u64 subdevs = 0;
u64 usermem;
+ if (!vmm)
+ return -EINVAL;
+
/* Determine which downstream engines are present */
for (i = 0; i < fifo->engine_nr; i++) {
struct nvkm_engine *engine = fifo->engine[i].engine;
@@ -285,30 +289,14 @@ gk104_fifo_gpfifo_new_(const struct gk104_fifo_chan_func *func,
INIT_LIST_HEAD(&chan->head);
ret = nvkm_fifo_chan_ctor(&gk104_fifo_gpfifo_func, &fifo->base,
- 0x1000, 0x1000, true, vm, 0, subdevs,
- 1, fifo->user.bar.offset, 0x200,
+ 0x1000, 0x1000, true, vmm, 0, subdevs,
+ 1, fifo->user.bar->addr, 0x200,
oclass, &chan->base);
if (ret)
return ret;
*chid = chan->base.chid;
- /* Page directory. */
- ret = nvkm_gpuobj_new(device, 0x10000, 0x1000, false, NULL, &chan->pgd);
- if (ret)
- return ret;
-
- nvkm_kmap(chan->base.inst);
- nvkm_wo32(chan->base.inst, 0x0200, lower_32_bits(chan->pgd->addr));
- nvkm_wo32(chan->base.inst, 0x0204, upper_32_bits(chan->pgd->addr));
- nvkm_wo32(chan->base.inst, 0x0208, 0xffffffff);
- nvkm_wo32(chan->base.inst, 0x020c, 0x000000ff);
- nvkm_done(chan->base.inst);
-
- ret = nvkm_vm_ref(chan->base.vm, &chan->vm, chan->pgd);
- if (ret)
- return ret;
-
/* Clear channel control registers. */
usermem = chan->base.chid * 0x200;
ilength = order_base_2(ilength / 8);
@@ -373,18 +361,17 @@ gk104_fifo_gpfifo_new(struct nvkm_fifo *base, const struct nvkm_oclass *oclass,
nvif_ioctl(parent, "create channel gpfifo size %d\n", size);
if (!(ret = nvif_unpack(ret, &data, &size, args->v0, 0, 0, false))) {
- nvif_ioctl(parent, "create channel gpfifo vers %d vm %llx "
+ nvif_ioctl(parent, "create channel gpfifo vers %d vmm %llx "
"ioffset %016llx ilength %08x engine %08x\n",
- args->v0.version, args->v0.vm, args->v0.ioffset,
+ args->v0.version, args->v0.vmm, args->v0.ioffset,
args->v0.ilength, args->v0.engines);
return gk104_fifo_gpfifo_new_(gk104_fifo_gpfifo, fifo,
&args->v0.engines,
&args->v0.chid,
- args->v0.vm,
+ args->v0.vmm,
args->v0.ioffset,
args->v0.ilength,
oclass, pobject);
-
}
return ret;
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifonv50.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifonv50.c
index c5a7de9db259..d8f28ec1e4a8 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifonv50.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifonv50.c
@@ -45,10 +45,10 @@ nv50_fifo_gpfifo_new(struct nvkm_fifo *base, const struct nvkm_oclass *oclass,
nvif_ioctl(parent, "create channel gpfifo size %d\n", size);
if (!(ret = nvif_unpack(ret, &data, &size, args->v0, 0, 0, false))) {
- nvif_ioctl(parent, "create channel gpfifo vers %d vm %llx "
+ nvif_ioctl(parent, "create channel gpfifo vers %d vmm %llx "
"pushbuf %llx ioffset %016llx "
"ilength %08x\n",
- args->v0.version, args->v0.vm, args->v0.pushbuf,
+ args->v0.version, args->v0.vmm, args->v0.pushbuf,
args->v0.ioffset, args->v0.ilength);
if (!args->v0.pushbuf)
return -EINVAL;
@@ -59,7 +59,7 @@ nv50_fifo_gpfifo_new(struct nvkm_fifo *base, const struct nvkm_oclass *oclass,
return -ENOMEM;
*pobject = &chan->base.object;
- ret = nv50_fifo_chan_ctor(fifo, args->v0.vm, args->v0.pushbuf,
+ ret = nv50_fifo_chan_ctor(fifo, args->v0.vmm, args->v0.pushbuf,
oclass, chan);
if (ret)
return ret;
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv50.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv50.c
index 66eb12c2b5ba..fa6e094d8068 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv50.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv50.c
@@ -100,8 +100,8 @@ void *
nv50_fifo_dtor(struct nvkm_fifo *base)
{
struct nv50_fifo *fifo = nv50_fifo(base);
- nvkm_memory_del(&fifo->runlist[1]);
- nvkm_memory_del(&fifo->runlist[0]);
+ nvkm_memory_unref(&fifo->runlist[1]);
+ nvkm_memory_unref(&fifo->runlist[0]);
return fifo;
}
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf100.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf100.c
index bc77eea351a5..881015080d83 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf100.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf100.c
@@ -986,14 +986,14 @@ gf100_grctx_pack_tpc[] = {
******************************************************************************/
int
-gf100_grctx_mmio_data(struct gf100_grctx *info, u32 size, u32 align, u32 access)
+gf100_grctx_mmio_data(struct gf100_grctx *info, u32 size, u32 align, bool priv)
{
if (info->data) {
info->buffer[info->buffer_nr] = round_up(info->addr, align);
info->addr = info->buffer[info->buffer_nr] + size;
info->data->size = size;
info->data->align = align;
- info->data->access = access;
+ info->data->priv = priv;
info->data++;
return info->buffer_nr++;
}
@@ -1028,9 +1028,8 @@ void
gf100_grctx_generate_bundle(struct gf100_grctx *info)
{
const struct gf100_grctx_func *grctx = info->gr->func->grctx;
- const u32 access = NV_MEM_ACCESS_RW | NV_MEM_ACCESS_SYS;
const int s = 8;
- const int b = mmio_vram(info, grctx->bundle_size, (1 << s), access);
+ const int b = mmio_vram(info, grctx->bundle_size, (1 << s), true);
mmio_refn(info, 0x408004, 0x00000000, s, b);
mmio_wr32(info, 0x408008, 0x80000000 | (grctx->bundle_size >> s));
mmio_refn(info, 0x418808, 0x00000000, s, b);
@@ -1041,9 +1040,8 @@ void
gf100_grctx_generate_pagepool(struct gf100_grctx *info)
{
const struct gf100_grctx_func *grctx = info->gr->func->grctx;
- const u32 access = NV_MEM_ACCESS_RW | NV_MEM_ACCESS_SYS;
const int s = 8;
- const int b = mmio_vram(info, grctx->pagepool_size, (1 << s), access);
+ const int b = mmio_vram(info, grctx->pagepool_size, (1 << s), true);
mmio_refn(info, 0x40800c, 0x00000000, s, b);
mmio_wr32(info, 0x408010, 0x80000000);
mmio_refn(info, 0x419004, 0x00000000, s, b);
@@ -1057,9 +1055,8 @@ gf100_grctx_generate_attrib(struct gf100_grctx *info)
const struct gf100_grctx_func *grctx = gr->func->grctx;
const u32 attrib = grctx->attrib_nr;
const u32 size = 0x20 * (grctx->attrib_nr_max + grctx->alpha_nr_max);
- const u32 access = NV_MEM_ACCESS_RW;
const int s = 12;
- const int b = mmio_vram(info, size * gr->tpc_total, (1 << s), access);
+ const int b = mmio_vram(info, size * gr->tpc_total, (1 << s), false);
int gpc, tpc;
u32 bo = 0;
@@ -1267,85 +1264,87 @@ gf100_grctx_generate_main(struct gf100_gr *gr, struct gf100_grctx *info)
nvkm_mc_unk260(device, 1);
}
+#define CB_RESERVED 0x80000
+
int
gf100_grctx_generate(struct gf100_gr *gr)
{
const struct gf100_grctx_func *grctx = gr->func->grctx;
struct nvkm_subdev *subdev = &gr->base.engine.subdev;
struct nvkm_device *device = subdev->device;
- struct nvkm_memory *chan;
+ struct nvkm_memory *inst = NULL;
+ struct nvkm_memory *data = NULL;
+ struct nvkm_vmm *vmm = NULL;
+ struct nvkm_vma *ctx = NULL;
struct gf100_grctx info;
int ret, i;
u64 addr;
- /* allocate memory to for a "channel", which we'll use to generate
- * the default context values
+ /* Allocate memory to for a "channel", which we'll use to generate
+ * the default context values.
*/
- ret = nvkm_memory_new(device, NVKM_MEM_TARGET_INST, 0x80000 + gr->size,
- 0x1000, true, &chan);
- if (ret) {
- nvkm_error(subdev, "failed to allocate chan memory, %d\n", ret);
- return ret;
- }
+ ret = nvkm_memory_new(device, NVKM_MEM_TARGET_INST,
+ 0x1000, 0x1000, true, &inst);
+ if (ret)
+ goto done;
- addr = nvkm_memory_addr(chan);
+ ret = nvkm_vmm_new(device, 0, 0, NULL, 0, NULL, "grctx", &vmm);
+ if (ret)
+ goto done;
- /* PGD pointer */
- nvkm_kmap(chan);
- nvkm_wo32(chan, 0x0200, lower_32_bits(addr + 0x1000));
- nvkm_wo32(chan, 0x0204, upper_32_bits(addr + 0x1000));
- nvkm_wo32(chan, 0x0208, 0xffffffff);
- nvkm_wo32(chan, 0x020c, 0x000000ff);
+ vmm->debug = subdev->debug;
- /* PGT[0] pointer */
- nvkm_wo32(chan, 0x1000, 0x00000000);
- nvkm_wo32(chan, 0x1004, 0x00000001 | (addr + 0x2000) >> 8);
+ ret = nvkm_vmm_join(vmm, inst);
+ if (ret)
+ goto done;
- /* identity-map the whole "channel" into its own vm */
- for (i = 0; i < nvkm_memory_size(chan) / 4096; i++) {
- u64 addr = ((nvkm_memory_addr(chan) + (i * 4096)) >> 8) | 1;
- nvkm_wo32(chan, 0x2000 + (i * 8), lower_32_bits(addr));
- nvkm_wo32(chan, 0x2004 + (i * 8), upper_32_bits(addr));
- }
+ ret = nvkm_memory_new(device, NVKM_MEM_TARGET_INST,
+ CB_RESERVED + gr->size, 0, true, &data);
+ if (ret)
+ goto done;
- /* context pointer (virt) */
- nvkm_wo32(chan, 0x0210, 0x00080004);
- nvkm_wo32(chan, 0x0214, 0x00000000);
- nvkm_done(chan);
+ ret = nvkm_vmm_get(vmm, 0, nvkm_memory_size(data), &ctx);
+ if (ret)
+ goto done;
- nvkm_wr32(device, 0x100cb8, (addr + 0x1000) >> 8);
- nvkm_wr32(device, 0x100cbc, 0x80000001);
- nvkm_msec(device, 2000,
- if (nvkm_rd32(device, 0x100c80) & 0x00008000)
- break;
- );
+ ret = nvkm_memory_map(data, 0, vmm, ctx, NULL, 0);
+ if (ret)
+ goto done;
+
+
+ /* Setup context pointer. */
+ nvkm_kmap(inst);
+ nvkm_wo32(inst, 0x0210, lower_32_bits(ctx->addr + CB_RESERVED) | 4);
+ nvkm_wo32(inst, 0x0214, upper_32_bits(ctx->addr + CB_RESERVED));
+ nvkm_done(inst);
- /* setup default state for mmio list construction */
+ /* Setup default state for mmio list construction. */
info.gr = gr;
info.data = gr->mmio_data;
info.mmio = gr->mmio_list;
- info.addr = 0x2000 + (i * 8);
+ info.addr = ctx->addr;
info.buffer_nr = 0;
- /* make channel current */
+ /* Make channel current. */
+ addr = nvkm_memory_addr(inst) >> 12;
if (gr->firmware) {
nvkm_wr32(device, 0x409840, 0x00000030);
- nvkm_wr32(device, 0x409500, 0x80000000 | addr >> 12);
+ nvkm_wr32(device, 0x409500, 0x80000000 | addr);
nvkm_wr32(device, 0x409504, 0x00000003);
nvkm_msec(device, 2000,
if (nvkm_rd32(device, 0x409800) & 0x00000010)
break;
);
- nvkm_kmap(chan);
- nvkm_wo32(chan, 0x8001c, 1);
- nvkm_wo32(chan, 0x80020, 0);
- nvkm_wo32(chan, 0x80028, 0);
- nvkm_wo32(chan, 0x8002c, 0);
- nvkm_done(chan);
+ nvkm_kmap(data);
+ nvkm_wo32(data, 0x1c, 1);
+ nvkm_wo32(data, 0x20, 0);
+ nvkm_wo32(data, 0x28, 0);
+ nvkm_wo32(data, 0x2c, 0);
+ nvkm_done(data);
} else {
nvkm_wr32(device, 0x409840, 0x80000000);
- nvkm_wr32(device, 0x409500, 0x80000000 | addr >> 12);
+ nvkm_wr32(device, 0x409500, 0x80000000 | addr);
nvkm_wr32(device, 0x409504, 0x00000001);
nvkm_msec(device, 2000,
if (nvkm_rd32(device, 0x409800) & 0x80000000)
@@ -1355,8 +1354,8 @@ gf100_grctx_generate(struct gf100_gr *gr)
grctx->main(gr, &info);
- /* trigger a context unload by unsetting the "next channel valid" bit
- * and faking a context switch interrupt
+ /* Trigger a context unload by unsetting the "next channel valid" bit
+ * and faking a context switch interrupt.
*/
nvkm_mask(device, 0x409b04, 0x80000000, 0x00000000);
nvkm_wr32(device, 0x409000, 0x00000100);
@@ -1370,17 +1369,21 @@ gf100_grctx_generate(struct gf100_gr *gr)
gr->data = kmalloc(gr->size, GFP_KERNEL);
if (gr->data) {
- nvkm_kmap(chan);
+ nvkm_kmap(data);
for (i = 0; i < gr->size; i += 4)
- gr->data[i / 4] = nvkm_ro32(chan, 0x80000 + i);
- nvkm_done(chan);
+ gr->data[i / 4] = nvkm_ro32(data, CB_RESERVED + i);
+ nvkm_done(data);
ret = 0;
} else {
ret = -ENOMEM;
}
done:
- nvkm_memory_del(&chan);
+ nvkm_vmm_put(vmm, &ctx);
+ nvkm_memory_unref(&data);
+ nvkm_vmm_part(vmm, inst);
+ nvkm_vmm_unref(&vmm);
+ nvkm_memory_unref(&inst);
return ret;
}
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf100.h b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf100.h
index 017180d147cf..4731e56fbb11 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf100.h
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf100.h
@@ -11,7 +11,7 @@ struct gf100_grctx {
u64 addr;
};
-int gf100_grctx_mmio_data(struct gf100_grctx *, u32 size, u32 align, u32 access);
+int gf100_grctx_mmio_data(struct gf100_grctx *, u32 size, u32 align, bool priv);
void gf100_grctx_mmio_item(struct gf100_grctx *, u32 addr, u32 data, int s, int);
#define mmio_vram(a,b,c,d) gf100_grctx_mmio_data((a), (b), (c), (d))
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf108.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf108.c
index 505cdcbfc085..82f71b10c06e 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf108.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf108.c
@@ -735,9 +735,8 @@ gf108_grctx_generate_attrib(struct gf100_grctx *info)
const u32 alpha = grctx->alpha_nr;
const u32 beta = grctx->attrib_nr;
const u32 size = 0x20 * (grctx->attrib_nr_max + grctx->alpha_nr_max);
- const u32 access = NV_MEM_ACCESS_RW;
const int s = 12;
- const int b = mmio_vram(info, size * gr->tpc_total, (1 << s), access);
+ const int b = mmio_vram(info, size * gr->tpc_total, (1 << s), false);
const int timeslice_mode = 1;
const int max_batches = 0xffff;
u32 bo = 0;
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf117.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf117.c
index 74a64e3fd59a..19301d88577d 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf117.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf117.c
@@ -187,9 +187,8 @@ gf117_grctx_generate_attrib(struct gf100_grctx *info)
const u32 alpha = grctx->alpha_nr;
const u32 beta = grctx->attrib_nr;
const u32 size = 0x20 * (grctx->attrib_nr_max + grctx->alpha_nr_max);
- const u32 access = NV_MEM_ACCESS_RW;
const int s = 12;
- const int b = mmio_vram(info, size * gr->tpc_total, (1 << s), access);
+ const int b = mmio_vram(info, size * gr->tpc_total, (1 << s), false);
const int timeslice_mode = 1;
const int max_batches = 0xffff;
u32 bo = 0;
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgk104.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgk104.c
index c46b3fdf7203..825c8fd500bc 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgk104.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgk104.c
@@ -847,9 +847,8 @@ gk104_grctx_generate_bundle(struct gf100_grctx *info)
const u32 state_limit = min(grctx->bundle_min_gpm_fifo_depth,
grctx->bundle_size / 0x20);
const u32 token_limit = grctx->bundle_token_limit;
- const u32 access = NV_MEM_ACCESS_RW | NV_MEM_ACCESS_SYS;
const int s = 8;
- const int b = mmio_vram(info, grctx->bundle_size, (1 << s), access);
+ const int b = mmio_vram(info, grctx->bundle_size, (1 << s), true);
mmio_refn(info, 0x408004, 0x00000000, s, b);
mmio_wr32(info, 0x408008, 0x80000000 | (grctx->bundle_size >> s));
mmio_refn(info, 0x418808, 0x00000000, s, b);
@@ -861,9 +860,8 @@ void
gk104_grctx_generate_pagepool(struct gf100_grctx *info)
{
const struct gf100_grctx_func *grctx = info->gr->func->grctx;
- const u32 access = NV_MEM_ACCESS_RW | NV_MEM_ACCESS_SYS;
const int s = 8;
- const int b = mmio_vram(info, grctx->pagepool_size, (1 << s), access);
+ const int b = mmio_vram(info, grctx->pagepool_size, (1 << s), true);
mmio_refn(info, 0x40800c, 0x00000000, s, b);
mmio_wr32(info, 0x408010, 0x80000000);
mmio_refn(info, 0x419004, 0x00000000, s, b);
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgm107.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgm107.c
index 4c4b5ab6e46d..9b43d4ce3eaa 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgm107.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgm107.c
@@ -867,9 +867,8 @@ gm107_grctx_generate_bundle(struct gf100_grctx *info)
const u32 state_limit = min(grctx->bundle_min_gpm_fifo_depth,
grctx->bundle_size / 0x20);
const u32 token_limit = grctx->bundle_token_limit;
- const u32 access = NV_MEM_ACCESS_RW | NV_MEM_ACCESS_SYS;
const int s = 8;
- const int b = mmio_vram(info, grctx->bundle_size, (1 << s), access);
+ const int b = mmio_vram(info, grctx->bundle_size, (1 << s), true);
mmio_refn(info, 0x408004, 0x00000000, s, b);
mmio_wr32(info, 0x408008, 0x80000000 | (grctx->bundle_size >> s));
mmio_refn(info, 0x418e24, 0x00000000, s, b);
@@ -881,9 +880,8 @@ void
gm107_grctx_generate_pagepool(struct gf100_grctx *info)
{
const struct gf100_grctx_func *grctx = info->gr->func->grctx;
- const u32 access = NV_MEM_ACCESS_RW | NV_MEM_ACCESS_SYS;
const int s = 8;
- const int b = mmio_vram(info, grctx->pagepool_size, (1 << s), access);
+ const int b = mmio_vram(info, grctx->pagepool_size, (1 << s), true);
mmio_refn(info, 0x40800c, 0x00000000, s, b);
mmio_wr32(info, 0x408010, 0x80000000);
mmio_refn(info, 0x419004, 0x00000000, s, b);
@@ -900,9 +898,8 @@ gm107_grctx_generate_attrib(struct gf100_grctx *info)
const u32 alpha = grctx->alpha_nr;
const u32 attrib = grctx->attrib_nr;
const u32 size = 0x20 * (grctx->attrib_nr_max + grctx->alpha_nr_max);
- const u32 access = NV_MEM_ACCESS_RW;
const int s = 12;
- const int b = mmio_vram(info, size * gr->tpc_total, (1 << s), access);
+ const int b = mmio_vram(info, size * gr->tpc_total, (1 << s), false);
const int max_batches = 0xffff;
u32 bo = 0;
u32 ao = bo + grctx->attrib_nr_max * gr->tpc_total;
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgp100.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgp100.c
index 7833bc777a29..88ea322d956c 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgp100.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgp100.c
@@ -33,9 +33,8 @@ void
gp100_grctx_generate_pagepool(struct gf100_grctx *info)
{
const struct gf100_grctx_func *grctx = info->gr->func->grctx;
- const u32 access = NV_MEM_ACCESS_RW | NV_MEM_ACCESS_SYS;
const int s = 8;
- const int b = mmio_vram(info, grctx->pagepool_size, (1 << s), access);
+ const int b = mmio_vram(info, grctx->pagepool_size, (1 << s), true);
mmio_refn(info, 0x40800c, 0x00000000, s, b);
mmio_wr32(info, 0x408010, 0x80000000);
mmio_refn(info, 0x419004, 0x00000000, s, b);
@@ -51,9 +50,8 @@ gp100_grctx_generate_attrib(struct gf100_grctx *info)
const u32 attrib = grctx->attrib_nr;
const u32 pertpc = 0x20 * (grctx->attrib_nr_max + grctx->alpha_nr_max);
const u32 size = roundup(gr->tpc_total * pertpc, 0x80);
- const u32 access = NV_MEM_ACCESS_RW;
const int s = 12;
- const int b = mmio_vram(info, size, (1 << s), access);
+ const int b = mmio_vram(info, size, (1 << s), false);
const int max_batches = 0xffff;
u32 ao = 0;
u32 bo = ao + grctx->alpha_nr_max * gr->tpc_total;
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgp102.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgp102.c
index 80b7ab0bee3a..7a66b4c2eb18 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgp102.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgp102.c
@@ -38,9 +38,8 @@ gp102_grctx_generate_attrib(struct gf100_grctx *info)
const u32 attrib = grctx->attrib_nr;
const u32 pertpc = 0x20 * (grctx->attrib_nr_max + grctx->alpha_nr_max);
const u32 size = roundup(gr->tpc_total * pertpc, 0x80);
- const u32 access = NV_MEM_ACCESS_RW;
const int s = 12;
- const int b = mmio_vram(info, size, (1 << s), access);
+ const int b = mmio_vram(info, size, (1 << s), false);
const int max_batches = 0xffff;
u32 ao = 0;
u32 bo = ao + grctx->alpha_nr_max * gr->tpc_total;
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.c
index 99689f4de502..2f8dc107047d 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.c
@@ -37,6 +37,7 @@
#include <nvif/class.h>
#include <nvif/cl9097.h>
+#include <nvif/if900d.h>
#include <nvif/unpack.h>
/*******************************************************************************
@@ -327,13 +328,13 @@ gf100_gr_chan_bind(struct nvkm_object *object, struct nvkm_gpuobj *parent,
if (!gr->firmware) {
nvkm_wo32(*pgpuobj, 0x00, chan->mmio_nr / 2);
- nvkm_wo32(*pgpuobj, 0x04, chan->mmio_vma.offset >> 8);
+ nvkm_wo32(*pgpuobj, 0x04, chan->mmio_vma->addr >> 8);
} else {
nvkm_wo32(*pgpuobj, 0xf4, 0);
nvkm_wo32(*pgpuobj, 0xf8, 0);
nvkm_wo32(*pgpuobj, 0x10, chan->mmio_nr / 2);
- nvkm_wo32(*pgpuobj, 0x14, lower_32_bits(chan->mmio_vma.offset));
- nvkm_wo32(*pgpuobj, 0x18, upper_32_bits(chan->mmio_vma.offset));
+ nvkm_wo32(*pgpuobj, 0x14, lower_32_bits(chan->mmio_vma->addr));
+ nvkm_wo32(*pgpuobj, 0x18, upper_32_bits(chan->mmio_vma->addr));
nvkm_wo32(*pgpuobj, 0x1c, 1);
nvkm_wo32(*pgpuobj, 0x20, 0);
nvkm_wo32(*pgpuobj, 0x28, 0);
@@ -350,18 +351,13 @@ gf100_gr_chan_dtor(struct nvkm_object *object)
int i;
for (i = 0; i < ARRAY_SIZE(chan->data); i++) {
- if (chan->data[i].vma.node) {
- nvkm_vm_unmap(&chan->data[i].vma);
- nvkm_vm_put(&chan->data[i].vma);
- }
- nvkm_memory_del(&chan->data[i].mem);
+ nvkm_vmm_put(chan->vmm, &chan->data[i].vma);
+ nvkm_memory_unref(&chan->data[i].mem);
}
- if (chan->mmio_vma.node) {
- nvkm_vm_unmap(&chan->mmio_vma);
- nvkm_vm_put(&chan->mmio_vma);
- }
- nvkm_memory_del(&chan->mmio);
+ nvkm_vmm_put(chan->vmm, &chan->mmio_vma);
+ nvkm_memory_unref(&chan->mmio);
+ nvkm_vmm_unref(&chan->vmm);
return chan;
}
@@ -380,6 +376,7 @@ gf100_gr_chan_new(struct nvkm_gr *base, struct nvkm_fifo_chan *fifoch,
struct gf100_gr_data *data = gr->mmio_data;
struct gf100_gr_mmio *mmio = gr->mmio_list;
struct gf100_gr_chan *chan;
+ struct gf100_vmm_map_v0 args = { .priv = 1 };
struct nvkm_device *device = gr->base.engine.subdev.device;
int ret, i;
@@ -387,6 +384,7 @@ gf100_gr_chan_new(struct nvkm_gr *base, struct nvkm_fifo_chan *fifoch,
return -ENOMEM;
nvkm_object_ctor(&gf100_gr_chan, oclass, &chan->object);
chan->gr = gr;
+ chan->vmm = nvkm_vmm_ref(fifoch->vmm);
*pobject = &chan->object;
/* allocate memory for a "mmio list" buffer that's used by the HUB
@@ -398,12 +396,14 @@ gf100_gr_chan_new(struct nvkm_gr *base, struct nvkm_fifo_chan *fifoch,
if (ret)
return ret;
- ret = nvkm_vm_get(fifoch->vm, 0x1000, 12, NV_MEM_ACCESS_RW |
- NV_MEM_ACCESS_SYS, &chan->mmio_vma);
+ ret = nvkm_vmm_get(fifoch->vmm, 12, 0x1000, &chan->mmio_vma);
if (ret)
return ret;
- nvkm_memory_map(chan->mmio, &chan->mmio_vma, 0);
+ ret = nvkm_memory_map(chan->mmio, 0, fifoch->vmm,
+ chan->mmio_vma, &args, sizeof(args));
+ if (ret)
+ return ret;
/* allocate buffers referenced by mmio list */
for (i = 0; data->size && i < ARRAY_SIZE(gr->mmio_data); i++) {
@@ -413,13 +413,19 @@ gf100_gr_chan_new(struct nvkm_gr *base, struct nvkm_fifo_chan *fifoch,
if (ret)
return ret;
- ret = nvkm_vm_get(fifoch->vm,
- nvkm_memory_size(chan->data[i].mem), 12,
- data->access, &chan->data[i].vma);
+ ret = nvkm_vmm_get(fifoch->vmm, 12,
+ nvkm_memory_size(chan->data[i].mem),
+ &chan->data[i].vma);
+ if (ret)
+ return ret;
+
+ args.priv = data->priv;
+
+ ret = nvkm_memory_map(chan->data[i].mem, 0, chan->vmm,
+ chan->data[i].vma, &args, sizeof(args));
if (ret)
return ret;
- nvkm_memory_map(chan->data[i].mem, &chan->data[i].vma, 0);
data++;
}
@@ -430,7 +436,7 @@ gf100_gr_chan_new(struct nvkm_gr *base, struct nvkm_fifo_chan *fifoch,
u32 data = mmio->data;
if (mmio->buffer >= 0) {
- u64 info = chan->data[mmio->buffer].vma.offset;
+ u64 info = chan->data[mmio->buffer].vma->addr;
data |= info >> mmio->shift;
}
@@ -1855,8 +1861,12 @@ gf100_gr_ctor_fw(struct gf100_gr *gr, const char *fwname,
int ret;
ret = nvkm_firmware_get(device, fwname, &fw);
- if (ret)
- return gf100_gr_ctor_fw_legacy(gr, fwname, fuc, ret);
+ if (ret) {
+ ret = gf100_gr_ctor_fw_legacy(gr, fwname, fuc, ret);
+ if (ret)
+ return -ENODEV;
+ return 0;
+ }
fuc->size = fw->size;
fuc->data = kmemdup(fw->data, fuc->size, GFP_KERNEL);
@@ -1903,25 +1913,33 @@ gf100_gr_new_(const struct gf100_gr_func *func, struct nvkm_device *device,
return 0;
}
+void
+gf100_gr_init_gpc_mmu(struct gf100_gr *gr)
+{
+ struct nvkm_device *device = gr->base.engine.subdev.device;
+ struct nvkm_fb *fb = device->fb;
+
+ nvkm_wr32(device, 0x418880, nvkm_rd32(device, 0x100c80) & 0x00000001);
+ nvkm_wr32(device, 0x4188a4, 0x00000000);
+ nvkm_wr32(device, 0x418888, 0x00000000);
+ nvkm_wr32(device, 0x41888c, 0x00000000);
+ nvkm_wr32(device, 0x418890, 0x00000000);
+ nvkm_wr32(device, 0x418894, 0x00000000);
+ nvkm_wr32(device, 0x4188b4, nvkm_memory_addr(fb->mmu_wr) >> 8);
+ nvkm_wr32(device, 0x4188b8, nvkm_memory_addr(fb->mmu_rd) >> 8);
+}
+
int
gf100_gr_init(struct gf100_gr *gr)
{
struct nvkm_device *device = gr->base.engine.subdev.device;
- struct nvkm_fb *fb = device->fb;
const u32 magicgpc918 = DIV_ROUND_UP(0x00800000, gr->tpc_total);
u32 data[TPC_MAX / 8] = {};
u8 tpcnr[GPC_MAX];
int gpc, tpc, rop;
int i;
- nvkm_wr32(device, GPC_BCAST(0x0880), 0x00000000);
- nvkm_wr32(device, GPC_BCAST(0x08a4), 0x00000000);
- nvkm_wr32(device, GPC_BCAST(0x0888), 0x00000000);
- nvkm_wr32(device, GPC_BCAST(0x088c), 0x00000000);
- nvkm_wr32(device, GPC_BCAST(0x0890), 0x00000000);
- nvkm_wr32(device, GPC_BCAST(0x0894), 0x00000000);
- nvkm_wr32(device, GPC_BCAST(0x08b4), nvkm_memory_addr(fb->mmu_wr) >> 8);
- nvkm_wr32(device, GPC_BCAST(0x08b8), nvkm_memory_addr(fb->mmu_rd) >> 8);
+ gr->func->init_gpc_mmu(gr);
gf100_gr_mmio(gr, gr->func->mmio);
@@ -2036,6 +2054,7 @@ gf100_gr_gpccs_ucode = {
static const struct gf100_gr_func
gf100_gr = {
.init = gf100_gr_init,
+ .init_gpc_mmu = gf100_gr_init_gpc_mmu,
.mmio = gf100_gr_pack_mmio,
.fecs.ucode = &gf100_gr_fecs_ucode,
.gpccs.ucode = &gf100_gr_gpccs_ucode,
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.h b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.h
index a36e45a4a635..d7c2adb9b543 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.h
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.h
@@ -45,7 +45,7 @@
struct gf100_gr_data {
u32 size;
u32 align;
- u32 access;
+ bool priv;
};
struct gf100_gr_mmio {
@@ -156,18 +156,20 @@ int gp100_gr_init(struct gf100_gr *);
void gp100_gr_init_rop_active_fbps(struct gf100_gr *);
#define gf100_gr_chan(p) container_of((p), struct gf100_gr_chan, object)
+#include <core/object.h>
struct gf100_gr_chan {
struct nvkm_object object;
struct gf100_gr *gr;
+ struct nvkm_vmm *vmm;
struct nvkm_memory *mmio;
- struct nvkm_vma mmio_vma;
+ struct nvkm_vma *mmio_vma;
int mmio_nr;
struct {
struct nvkm_memory *mem;
- struct nvkm_vma vma;
+ struct nvkm_vma *vma;
} data[4];
};
@@ -253,6 +255,7 @@ extern const struct gf100_gr_init gf100_gr_init_mpc_0[];
extern const struct gf100_gr_init gf100_gr_init_be_0[];
extern const struct gf100_gr_init gf100_gr_init_fe_1[];
extern const struct gf100_gr_init gf100_gr_init_pe_1[];
+void gf100_gr_init_gpc_mmu(struct gf100_gr *);
extern const struct gf100_gr_init gf104_gr_init_ds_0[];
extern const struct gf100_gr_init gf104_gr_init_tex_0[];
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf104.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf104.c
index d736dcd55ea2..ec0f11983b23 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf104.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf104.c
@@ -115,6 +115,7 @@ gf104_gr_pack_mmio[] = {
static const struct gf100_gr_func
gf104_gr = {
.init = gf100_gr_init,
+ .init_gpc_mmu = gf100_gr_init_gpc_mmu,
.mmio = gf104_gr_pack_mmio,
.fecs.ucode = &gf100_gr_fecs_ucode,
.gpccs.ucode = &gf100_gr_gpccs_ucode,
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf108.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf108.c
index 2f0d24498427..cc152eb74123 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf108.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf108.c
@@ -106,6 +106,7 @@ gf108_gr_pack_mmio[] = {
static const struct gf100_gr_func
gf108_gr = {
.init = gf100_gr_init,
+ .init_gpc_mmu = gf100_gr_init_gpc_mmu,
.mmio = gf108_gr_pack_mmio,
.fecs.ucode = &gf100_gr_fecs_ucode,
.gpccs.ucode = &gf100_gr_gpccs_ucode,
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf110.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf110.c
index d1d942eb86af..10d2d73ca8c3 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf110.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf110.c
@@ -87,6 +87,7 @@ gf110_gr_pack_mmio[] = {
static const struct gf100_gr_func
gf110_gr = {
.init = gf100_gr_init,
+ .init_gpc_mmu = gf100_gr_init_gpc_mmu,
.mmio = gf110_gr_pack_mmio,
.fecs.ucode = &gf100_gr_fecs_ucode,
.gpccs.ucode = &gf100_gr_gpccs_ucode,
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf117.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf117.c
index 0124e468086e..ac09a07c4150 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf117.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf117.c
@@ -123,6 +123,7 @@ gf117_gr_gpccs_ucode = {
static const struct gf100_gr_func
gf117_gr = {
.init = gf100_gr_init,
+ .init_gpc_mmu = gf100_gr_init_gpc_mmu,
.mmio = gf117_gr_pack_mmio,
.fecs.ucode = &gf117_gr_fecs_ucode,
.gpccs.ucode = &gf117_gr_gpccs_ucode,
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf119.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf119.c
index 8d8e4cafe28f..7f449ec6f760 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf119.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf119.c
@@ -178,6 +178,7 @@ gf119_gr_pack_mmio[] = {
static const struct gf100_gr_func
gf119_gr = {
.init = gf100_gr_init,
+ .init_gpc_mmu = gf100_gr_init_gpc_mmu,
.mmio = gf119_gr_pack_mmio,
.fecs.ucode = &gf100_gr_fecs_ucode,
.gpccs.ucode = &gf100_gr_gpccs_ucode,
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gk104.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gk104.c
index ec22da6c99fc..5e82f94c2245 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gk104.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gk104.c
@@ -24,8 +24,6 @@
#include "gf100.h"
#include "ctxgf100.h"
-#include <subdev/fb.h>
-
#include <nvif/class.h>
/*******************************************************************************
@@ -207,21 +205,13 @@ int
gk104_gr_init(struct gf100_gr *gr)
{
struct nvkm_device *device = gr->base.engine.subdev.device;
- struct nvkm_fb *fb = device->fb;
const u32 magicgpc918 = DIV_ROUND_UP(0x00800000, gr->tpc_total);
u32 data[TPC_MAX / 8] = {};
u8 tpcnr[GPC_MAX];
int gpc, tpc, rop;
int i;
- nvkm_wr32(device, GPC_BCAST(0x0880), 0x00000000);
- nvkm_wr32(device, GPC_BCAST(0x08a4), 0x00000000);
- nvkm_wr32(device, GPC_BCAST(0x0888), 0x00000000);
- nvkm_wr32(device, GPC_BCAST(0x088c), 0x00000000);
- nvkm_wr32(device, GPC_BCAST(0x0890), 0x00000000);
- nvkm_wr32(device, GPC_BCAST(0x0894), 0x00000000);
- nvkm_wr32(device, GPC_BCAST(0x08b4), nvkm_memory_addr(fb->mmu_wr) >> 8);
- nvkm_wr32(device, GPC_BCAST(0x08b8), nvkm_memory_addr(fb->mmu_rd) >> 8);
+ gr->func->init_gpc_mmu(gr);
gf100_gr_mmio(gr, gr->func->mmio);
@@ -339,6 +329,7 @@ gk104_gr_gpccs_ucode = {
static const struct gf100_gr_func
gk104_gr = {
.init = gk104_gr_init,
+ .init_gpc_mmu = gf100_gr_init_gpc_mmu,
.init_rop_active_fbps = gk104_gr_init_rop_active_fbps,
.init_ppc_exceptions = gk104_gr_init_ppc_exceptions,
.mmio = gk104_gr_pack_mmio,
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gk110.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gk110.c
index f31b171a4102..a38e19b61c1d 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gk110.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gk110.c
@@ -183,6 +183,7 @@ gk110_gr_gpccs_ucode = {
static const struct gf100_gr_func
gk110_gr = {
.init = gk104_gr_init,
+ .init_gpc_mmu = gf100_gr_init_gpc_mmu,
.init_rop_active_fbps = gk104_gr_init_rop_active_fbps,
.init_ppc_exceptions = gk104_gr_init_ppc_exceptions,
.mmio = gk110_gr_pack_mmio,
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gk110b.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gk110b.c
index d76dd178007f..1912c0bfd7ee 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gk110b.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gk110b.c
@@ -103,6 +103,7 @@ gk110b_gr_pack_mmio[] = {
static const struct gf100_gr_func
gk110b_gr = {
.init = gk104_gr_init,
+ .init_gpc_mmu = gf100_gr_init_gpc_mmu,
.init_rop_active_fbps = gk104_gr_init_rop_active_fbps,
.init_ppc_exceptions = gk104_gr_init_ppc_exceptions,
.mmio = gk110b_gr_pack_mmio,
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gk208.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gk208.c
index 14bbe6ed02a9..1fc258163f25 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gk208.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gk208.c
@@ -162,6 +162,7 @@ gk208_gr_gpccs_ucode = {
static const struct gf100_gr_func
gk208_gr = {
.init = gk104_gr_init,
+ .init_gpc_mmu = gf100_gr_init_gpc_mmu,
.init_rop_active_fbps = gk104_gr_init_rop_active_fbps,
.init_ppc_exceptions = gk104_gr_init_ppc_exceptions,
.mmio = gk208_gr_pack_mmio,
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/nv20.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/nv20.c
index d1dc92999dc0..d6840dc81a29 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/nv20.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/nv20.c
@@ -59,7 +59,7 @@ void *
nv20_gr_chan_dtor(struct nvkm_object *object)
{
struct nv20_gr_chan *chan = nv20_gr_chan(object);
- nvkm_memory_del(&chan->inst);
+ nvkm_memory_unref(&chan->inst);
return chan;
}
@@ -323,7 +323,7 @@ void *
nv20_gr_dtor(struct nvkm_gr *base)
{
struct nv20_gr *gr = nv20_gr(base);
- nvkm_memory_del(&gr->ctxtab);
+ nvkm_memory_unref(&gr->ctxtab);
return gr;
}
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/nv20.h b/drivers/gpu/drm/nouveau/nvkm/engine/gr/nv20.h
index cdf4501e3798..d0cb2b8846ec 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/nv20.h
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/nv20.h
@@ -19,6 +19,7 @@ void nv20_gr_tile(struct nvkm_gr *, int, struct nvkm_fb_tile *);
int nv30_gr_init(struct nvkm_gr *);
#define nv20_gr_chan(p) container_of((p), struct nv20_gr_chan, object)
+#include <core/object.h>
struct nv20_gr_chan {
struct nvkm_object object;
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/nv40.h b/drivers/gpu/drm/nouveau/nvkm/engine/gr/nv40.h
index 2812ed11f877..bee8ef2d5697 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/nv40.h
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/nv40.h
@@ -16,6 +16,7 @@ void nv40_gr_intr(struct nvkm_gr *);
u64 nv40_gr_units(struct nvkm_gr *);
#define nv40_gr_chan(p) container_of((p), struct nv40_gr_chan, object)
+#include <core/object.h>
struct nv40_gr_chan {
struct nvkm_object object;
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/nv50.h b/drivers/gpu/drm/nouveau/nvkm/engine/gr/nv50.h
index 45eec83a5969..1ab6ea436b70 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/nv50.h
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/nv50.h
@@ -19,6 +19,7 @@ u64 nv50_gr_units(struct nvkm_gr *);
int g84_gr_tlb_flush(struct nvkm_gr *);
#define nv50_gr_chan(p) container_of((p), struct nv50_gr_chan, object)
+#include <core/object.h>
struct nv50_gr_chan {
struct nvkm_object object;
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/mpeg/nv31.h b/drivers/gpu/drm/nouveau/nvkm/engine/mpeg/nv31.h
index d3bb34fcdebf..f0d35beb58df 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/mpeg/nv31.h
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/mpeg/nv31.h
@@ -18,6 +18,7 @@ struct nv31_mpeg_func {
};
#define nv31_mpeg_chan(p) container_of((p), struct nv31_mpeg_chan, object)
+#include <core/object.h>
struct nv31_mpeg_chan {
struct nvkm_object object;
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/mpeg/nv50.c b/drivers/gpu/drm/nouveau/nvkm/engine/mpeg/nv50.c
index 4e528851e9c0..6df880a39019 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/mpeg/nv50.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/mpeg/nv50.c
@@ -24,6 +24,7 @@
#include "priv.h"
#include <core/gpuobj.h>
+#include <core/object.h>
#include <subdev/timer.h>
#include <nvif/class.h>
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/pm/priv.h b/drivers/gpu/drm/nouveau/nvkm/engine/pm/priv.h
index d7b81cbf82b5..4ff0475e776c 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/pm/priv.h
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/pm/priv.h
@@ -67,6 +67,7 @@ struct nvkm_specdom {
};
#define nvkm_perfdom(p) container_of((p), struct nvkm_perfdom, object)
+#include <core/object.h>
struct nvkm_perfdom {
struct nvkm_object object;
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/sw/chan.h b/drivers/gpu/drm/nouveau/nvkm/engine/sw/chan.h
index 6608bf6c6842..b5be49f0ac56 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/sw/chan.h
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/sw/chan.h
@@ -1,9 +1,11 @@
#ifndef __NVKM_SW_CHAN_H__
#define __NVKM_SW_CHAN_H__
#define nvkm_sw_chan(p) container_of((p), struct nvkm_sw_chan, object)
-#include "priv.h"
+#include <core/object.h>
#include <core/event.h>
+#include "priv.h"
+
struct nvkm_sw_chan {
const struct nvkm_sw_chan_func *func;
struct nvkm_object object;
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/sw/nvsw.h b/drivers/gpu/drm/nouveau/nvkm/engine/sw/nvsw.h
index 943ef4c10091..bcfff62131fe 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/sw/nvsw.h
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/sw/nvsw.h
@@ -1,7 +1,7 @@
#ifndef __NVKM_NVSW_H__
#define __NVKM_NVSW_H__
#define nvkm_nvsw(p) container_of((p), struct nvkm_nvsw, object)
-#include "priv.h"
+#include <core/object.h>
struct nvkm_nvsw {
struct nvkm_object object;
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/xtensa.c b/drivers/gpu/drm/nouveau/nvkm/engine/xtensa.c
index 06bdb67a0205..70549381e082 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/xtensa.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/xtensa.c
@@ -86,7 +86,7 @@ nvkm_xtensa_fini(struct nvkm_engine *engine, bool suspend)
nvkm_wr32(device, base + 0xd94, 0); /* FIFO_CTRL */
if (!suspend)
- nvkm_memory_del(&xtensa->gpu_fw);
+ nvkm_memory_unref(&xtensa->gpu_fw);
return 0;
}
diff --git a/drivers/gpu/drm/nouveau/nvkm/falcon/base.c b/drivers/gpu/drm/nouveau/nvkm/falcon/base.c
index 1b7f48efd8b1..14be41f24155 100644
--- a/drivers/gpu/drm/nouveau/nvkm/falcon/base.c
+++ b/drivers/gpu/drm/nouveau/nvkm/falcon/base.c
@@ -60,7 +60,7 @@ nvkm_falcon_read_dmem(struct nvkm_falcon *falcon, u32 start, u32 size, u8 port,
}
void
-nvkm_falcon_bind_context(struct nvkm_falcon *falcon, struct nvkm_gpuobj *inst)
+nvkm_falcon_bind_context(struct nvkm_falcon *falcon, struct nvkm_memory *inst)
{
if (!falcon->func->bind_context) {
nvkm_error(falcon->user,
diff --git a/drivers/gpu/drm/nouveau/nvkm/falcon/v1.c b/drivers/gpu/drm/nouveau/nvkm/falcon/v1.c
index 669c24028470..9def926f24d4 100644
--- a/drivers/gpu/drm/nouveau/nvkm/falcon/v1.c
+++ b/drivers/gpu/drm/nouveau/nvkm/falcon/v1.c
@@ -180,7 +180,7 @@ nvkm_falcon_v1_read_dmem(struct nvkm_falcon *falcon, u32 start, u32 size,
}
static void
-nvkm_falcon_v1_bind_context(struct nvkm_falcon *falcon, struct nvkm_gpuobj *ctx)
+nvkm_falcon_v1_bind_context(struct nvkm_falcon *falcon, struct nvkm_memory *ctx)
{
u32 inst_loc;
u32 fbif;
@@ -216,7 +216,7 @@ nvkm_falcon_v1_bind_context(struct nvkm_falcon *falcon, struct nvkm_gpuobj *ctx)
nvkm_falcon_wr32(falcon, fbif + 4 * FALCON_DMAIDX_PHYS_SYS_NCOH, 0x6);
/* Set context */
- switch (nvkm_memory_target(ctx->memory)) {
+ switch (nvkm_memory_target(ctx)) {
case NVKM_MEM_TARGET_VRAM: inst_loc = 0; break;
case NVKM_MEM_TARGET_HOST: inst_loc = 2; break;
case NVKM_MEM_TARGET_NCOH: inst_loc = 3; break;
@@ -228,7 +228,7 @@ nvkm_falcon_v1_bind_context(struct nvkm_falcon *falcon, struct nvkm_gpuobj *ctx)
/* Enable context */
nvkm_falcon_mask(falcon, 0x048, 0x1, 0x1);
nvkm_falcon_wr32(falcon, 0x054,
- ((ctx->addr >> 12) & 0xfffffff) |
+ ((nvkm_memory_addr(ctx) >> 12) & 0xfffffff) |
(inst_loc << 28) | (1 << 30));
nvkm_falcon_mask(falcon, 0x090, 0x10000, 0x10000);
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/Kbuild b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/Kbuild
index 1e138b337955..e5830453813d 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/Kbuild
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/Kbuild
@@ -3,3 +3,5 @@ nvkm-y += nvkm/subdev/bar/nv50.o
nvkm-y += nvkm/subdev/bar/g84.o
nvkm-y += nvkm/subdev/bar/gf100.o
nvkm-y += nvkm/subdev/bar/gk20a.o
+nvkm-y += nvkm/subdev/bar/gm107.o
+nvkm-y += nvkm/subdev/bar/gm20b.o
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/base.c b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/base.c
index c561d148cebc..9646adec57cb 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/base.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/base.c
@@ -30,39 +30,76 @@ nvkm_bar_flush(struct nvkm_bar *bar)
bar->func->flush(bar);
}
-struct nvkm_vm *
-nvkm_bar_kmap(struct nvkm_bar *bar)
+struct nvkm_vmm *
+nvkm_bar_bar1_vmm(struct nvkm_device *device)
{
- /* disallow kmap() until after vm has been bootstrapped */
- if (bar && bar->func->kmap && bar->subdev.oneinit)
- return bar->func->kmap(bar);
+ return device->bar->func->bar1.vmm(device->bar);
+}
+
+struct nvkm_vmm *
+nvkm_bar_bar2_vmm(struct nvkm_device *device)
+{
+ /* Denies access to BAR2 when it's not initialised, used by INSTMEM
+ * to know when object access needs to go through the BAR0 window.
+ */
+ struct nvkm_bar *bar = device->bar;
+ if (bar && bar->bar2)
+ return bar->func->bar2.vmm(bar);
return NULL;
}
-int
-nvkm_bar_umap(struct nvkm_bar *bar, u64 size, int type, struct nvkm_vma *vma)
+void
+nvkm_bar_bar2_fini(struct nvkm_device *device)
{
- return bar->func->umap(bar, size, type, vma);
+ struct nvkm_bar *bar = device->bar;
+ if (bar && bar->bar2) {
+ bar->func->bar2.fini(bar);
+ bar->bar2 = false;
+ }
+}
+
+void
+nvkm_bar_bar2_init(struct nvkm_device *device)
+{
+ struct nvkm_bar *bar = device->bar;
+ if (bar && bar->subdev.oneinit && !bar->bar2 && bar->func->bar2.init) {
+ bar->func->bar2.init(bar);
+ bar->func->bar2.wait(bar);
+ bar->bar2 = true;
+ }
}
static int
-nvkm_bar_oneinit(struct nvkm_subdev *subdev)
+nvkm_bar_fini(struct nvkm_subdev *subdev, bool suspend)
{
struct nvkm_bar *bar = nvkm_bar(subdev);
- return bar->func->oneinit(bar);
+ bar->func->bar1.fini(bar);
+ return 0;
}
static int
nvkm_bar_init(struct nvkm_subdev *subdev)
{
struct nvkm_bar *bar = nvkm_bar(subdev);
- return bar->func->init(bar);
+ bar->func->bar1.init(bar);
+ bar->func->bar1.wait(bar);
+ if (bar->func->init)
+ bar->func->init(bar);
+ return 0;
+}
+
+static int
+nvkm_bar_oneinit(struct nvkm_subdev *subdev)
+{
+ struct nvkm_bar *bar = nvkm_bar(subdev);
+ return bar->func->oneinit(bar);
}
static void *
nvkm_bar_dtor(struct nvkm_subdev *subdev)
{
struct nvkm_bar *bar = nvkm_bar(subdev);
+ nvkm_bar_bar2_fini(subdev->device);
return bar->func->dtor(bar);
}
@@ -71,6 +108,7 @@ nvkm_bar = {
.dtor = nvkm_bar_dtor,
.oneinit = nvkm_bar_oneinit,
.init = nvkm_bar_init,
+ .fini = nvkm_bar_fini,
};
void
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/g84.c b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/g84.c
index ef717136c838..87f26f54b481 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/g84.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/g84.c
@@ -44,8 +44,14 @@ g84_bar_func = {
.dtor = nv50_bar_dtor,
.oneinit = nv50_bar_oneinit,
.init = nv50_bar_init,
- .kmap = nv50_bar_kmap,
- .umap = nv50_bar_umap,
+ .bar1.init = nv50_bar_bar1_init,
+ .bar1.fini = nv50_bar_bar1_fini,
+ .bar1.wait = nv50_bar_bar1_wait,
+ .bar1.vmm = nv50_bar_bar1_vmm,
+ .bar2.init = nv50_bar_bar2_init,
+ .bar2.fini = nv50_bar_bar2_fini,
+ .bar2.wait = nv50_bar_bar1_wait,
+ .bar2.vmm = nv50_bar_bar2_vmm,
.flush = g84_bar_flush,
};
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/gf100.c b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/gf100.c
index 676c167c95b9..a3ba7f50198b 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/gf100.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/gf100.c
@@ -23,39 +23,73 @@
*/
#include "gf100.h"
-#include <core/gpuobj.h>
+#include <core/memory.h>
#include <core/option.h>
#include <subdev/fb.h>
#include <subdev/mmu.h>
-static struct nvkm_vm *
-gf100_bar_kmap(struct nvkm_bar *base)
+struct nvkm_vmm *
+gf100_bar_bar1_vmm(struct nvkm_bar *base)
{
- return gf100_bar(base)->bar[0].vm;
+ return gf100_bar(base)->bar[1].vmm;
}
-int
-gf100_bar_umap(struct nvkm_bar *base, u64 size, int type, struct nvkm_vma *vma)
+void
+gf100_bar_bar1_wait(struct nvkm_bar *base)
+{
+ /* NFI why it's twice. */
+ nvkm_bar_flush(base);
+ nvkm_bar_flush(base);
+}
+
+void
+gf100_bar_bar1_fini(struct nvkm_bar *bar)
{
+ nvkm_mask(bar->subdev.device, 0x001704, 0x80000000, 0x00000000);
+}
+
+void
+gf100_bar_bar1_init(struct nvkm_bar *base)
+{
+ struct nvkm_device *device = base->subdev.device;
struct gf100_bar *bar = gf100_bar(base);
- return nvkm_vm_get(bar->bar[1].vm, size, type, NV_MEM_ACCESS_RW, vma);
+ const u32 addr = nvkm_memory_addr(bar->bar[1].inst) >> 12;
+ nvkm_wr32(device, 0x001704, 0x80000000 | addr);
+}
+
+struct nvkm_vmm *
+gf100_bar_bar2_vmm(struct nvkm_bar *base)
+{
+ return gf100_bar(base)->bar[0].vmm;
+}
+
+void
+gf100_bar_bar2_fini(struct nvkm_bar *bar)
+{
+ nvkm_mask(bar->subdev.device, 0x001714, 0x80000000, 0x00000000);
+}
+
+void
+gf100_bar_bar2_init(struct nvkm_bar *base)
+{
+ struct nvkm_device *device = base->subdev.device;
+ struct gf100_bar *bar = gf100_bar(base);
+ u32 addr = nvkm_memory_addr(bar->bar[0].inst) >> 12;
+ if (bar->bar2_halve)
+ addr |= 0x40000000;
+ nvkm_wr32(device, 0x001714, 0x80000000 | addr);
}
static int
-gf100_bar_ctor_vm(struct gf100_bar *bar, struct gf100_bar_vm *bar_vm,
- struct lock_class_key *key, int bar_nr)
+gf100_bar_oneinit_bar(struct gf100_bar *bar, struct gf100_barN *bar_vm,
+ struct lock_class_key *key, int bar_nr)
{
struct nvkm_device *device = bar->base.subdev.device;
- struct nvkm_vm *vm;
resource_size_t bar_len;
int ret;
ret = nvkm_memory_new(device, NVKM_MEM_TARGET_INST, 0x1000, 0, false,
- &bar_vm->mem);
- if (ret)
- return ret;
-
- ret = nvkm_gpuobj_new(device, 0x8000, 0, false, NULL, &bar_vm->pgd);
+ &bar_vm->inst);
if (ret)
return ret;
@@ -63,98 +97,64 @@ gf100_bar_ctor_vm(struct gf100_bar *bar, struct gf100_bar_vm *bar_vm,
if (bar_nr == 3 && bar->bar2_halve)
bar_len >>= 1;
- ret = nvkm_vm_new(device, 0, bar_len, 0, key, &vm);
+ ret = nvkm_vmm_new(device, 0, bar_len, NULL, 0, key,
+ (bar_nr == 3) ? "bar2" : "bar1", &bar_vm->vmm);
if (ret)
return ret;
- atomic_inc(&vm->engref[NVKM_SUBDEV_BAR]);
+ atomic_inc(&bar_vm->vmm->engref[NVKM_SUBDEV_BAR]);
+ bar_vm->vmm->debug = bar->base.subdev.debug;
/*
* Bootstrap page table lookup.
*/
if (bar_nr == 3) {
- ret = nvkm_vm_boot(vm, bar_len);
- if (ret) {
- nvkm_vm_ref(NULL, &vm, NULL);
+ ret = nvkm_vmm_boot(bar_vm->vmm);
+ if (ret)
return ret;
- }
}
- ret = nvkm_vm_ref(vm, &bar_vm->vm, bar_vm->pgd);
- nvkm_vm_ref(NULL, &vm, NULL);
- if (ret)
- return ret;
-
- nvkm_kmap(bar_vm->mem);
- nvkm_wo32(bar_vm->mem, 0x0200, lower_32_bits(bar_vm->pgd->addr));
- nvkm_wo32(bar_vm->mem, 0x0204, upper_32_bits(bar_vm->pgd->addr));
- nvkm_wo32(bar_vm->mem, 0x0208, lower_32_bits(bar_len - 1));
- nvkm_wo32(bar_vm->mem, 0x020c, upper_32_bits(bar_len - 1));
- nvkm_done(bar_vm->mem);
- return 0;
+ return nvkm_vmm_join(bar_vm->vmm, bar_vm->inst);
}
int
gf100_bar_oneinit(struct nvkm_bar *base)
{
static struct lock_class_key bar1_lock;
- static struct lock_class_key bar3_lock;
+ static struct lock_class_key bar2_lock;
struct gf100_bar *bar = gf100_bar(base);
int ret;
- /* BAR3 */
- if (bar->base.func->kmap) {
- ret = gf100_bar_ctor_vm(bar, &bar->bar[0], &bar3_lock, 3);
+ /* BAR2 */
+ if (bar->base.func->bar2.init) {
+ ret = gf100_bar_oneinit_bar(bar, &bar->bar[0], &bar2_lock, 3);
if (ret)
return ret;
+
+ bar->base.subdev.oneinit = true;
+ nvkm_bar_bar2_init(bar->base.subdev.device);
}
/* BAR1 */
- ret = gf100_bar_ctor_vm(bar, &bar->bar[1], &bar1_lock, 1);
+ ret = gf100_bar_oneinit_bar(bar, &bar->bar[1], &bar1_lock, 1);
if (ret)
return ret;
return 0;
}
-int
-gf100_bar_init(struct nvkm_bar *base)
-{
- struct gf100_bar *bar = gf100_bar(base);
- struct nvkm_device *device = bar->base.subdev.device;
- u32 addr;
-
- nvkm_mask(device, 0x000200, 0x00000100, 0x00000000);
- nvkm_mask(device, 0x000200, 0x00000100, 0x00000100);
-
- addr = nvkm_memory_addr(bar->bar[1].mem) >> 12;
- nvkm_wr32(device, 0x001704, 0x80000000 | addr);
-
- if (bar->bar[0].mem) {
- addr = nvkm_memory_addr(bar->bar[0].mem) >> 12;
- if (bar->bar2_halve)
- addr |= 0x40000000;
- nvkm_wr32(device, 0x001714, 0x80000000 | addr);
- }
-
- return 0;
-}
-
void *
gf100_bar_dtor(struct nvkm_bar *base)
{
struct gf100_bar *bar = gf100_bar(base);
- nvkm_vm_ref(NULL, &bar->bar[1].vm, bar->bar[1].pgd);
- nvkm_gpuobj_del(&bar->bar[1].pgd);
- nvkm_memory_del(&bar->bar[1].mem);
+ nvkm_vmm_part(bar->bar[1].vmm, bar->bar[1].inst);
+ nvkm_vmm_unref(&bar->bar[1].vmm);
+ nvkm_memory_unref(&bar->bar[1].inst);
- if (bar->bar[0].vm) {
- nvkm_memory_del(&bar->bar[0].vm->pgt[0].mem[0]);
- nvkm_vm_ref(NULL, &bar->bar[0].vm, bar->bar[0].pgd);
- }
- nvkm_gpuobj_del(&bar->bar[0].pgd);
- nvkm_memory_del(&bar->bar[0].mem);
+ nvkm_vmm_part(bar->bar[0].vmm, bar->bar[0].inst);
+ nvkm_vmm_unref(&bar->bar[0].vmm);
+ nvkm_memory_unref(&bar->bar[0].inst);
return bar;
}
@@ -175,9 +175,14 @@ static const struct nvkm_bar_func
gf100_bar_func = {
.dtor = gf100_bar_dtor,
.oneinit = gf100_bar_oneinit,
- .init = gf100_bar_init,
- .kmap = gf100_bar_kmap,
- .umap = gf100_bar_umap,
+ .bar1.init = gf100_bar_bar1_init,
+ .bar1.fini = gf100_bar_bar1_fini,
+ .bar1.wait = gf100_bar_bar1_wait,
+ .bar1.vmm = gf100_bar_bar1_vmm,
+ .bar2.init = gf100_bar_bar2_init,
+ .bar2.fini = gf100_bar_bar2_fini,
+ .bar2.wait = gf100_bar_bar1_wait,
+ .bar2.vmm = gf100_bar_bar2_vmm,
.flush = g84_bar_flush,
};
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/gf100.h b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/gf100.h
index 20a5255362ba..e4da39139e95 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/gf100.h
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/gf100.h
@@ -3,22 +3,24 @@
#define gf100_bar(p) container_of((p), struct gf100_bar, base)
#include "priv.h"
-struct gf100_bar_vm {
- struct nvkm_memory *mem;
- struct nvkm_gpuobj *pgd;
- struct nvkm_vm *vm;
+struct gf100_barN {
+ struct nvkm_memory *inst;
+ struct nvkm_vmm *vmm;
};
struct gf100_bar {
struct nvkm_bar base;
bool bar2_halve;
- struct gf100_bar_vm bar[2];
+ struct gf100_barN bar[2];
};
int gf100_bar_new_(const struct nvkm_bar_func *, struct nvkm_device *,
int, struct nvkm_bar **);
void *gf100_bar_dtor(struct nvkm_bar *);
int gf100_bar_oneinit(struct nvkm_bar *);
-int gf100_bar_init(struct nvkm_bar *);
-int gf100_bar_umap(struct nvkm_bar *, u64, int, struct nvkm_vma *);
+void gf100_bar_bar1_init(struct nvkm_bar *);
+void gf100_bar_bar1_wait(struct nvkm_bar *);
+struct nvkm_vmm *gf100_bar_bar1_vmm(struct nvkm_bar *);
+void gf100_bar_bar2_init(struct nvkm_bar *);
+struct nvkm_vmm *gf100_bar_bar2_vmm(struct nvkm_bar *);
#endif
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/gk20a.c b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/gk20a.c
index 9232fab4274c..b10077d38839 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/gk20a.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/gk20a.c
@@ -25,8 +25,10 @@ static const struct nvkm_bar_func
gk20a_bar_func = {
.dtor = gf100_bar_dtor,
.oneinit = gf100_bar_oneinit,
- .init = gf100_bar_init,
- .umap = gf100_bar_umap,
+ .bar1.init = gf100_bar_bar1_init,
+ .bar1.fini = gf100_bar_bar1_fini,
+ .bar1.wait = gf100_bar_bar1_wait,
+ .bar1.vmm = gf100_bar_bar1_vmm,
.flush = g84_bar_flush,
};
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/gm107.c b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/gm107.c
new file mode 100644
index 000000000000..3ddf9222d935
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/gm107.c
@@ -0,0 +1,65 @@
+/*
+ * Copyright 2017 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+#include "gf100.h"
+
+#include <subdev/timer.h>
+
+void
+gm107_bar_bar1_wait(struct nvkm_bar *bar)
+{
+ struct nvkm_device *device = bar->subdev.device;
+ nvkm_msec(device, 2000,
+ if (!(nvkm_rd32(device, 0x001710) & 0x00000003))
+ break;
+ );
+}
+
+static void
+gm107_bar_bar2_wait(struct nvkm_bar *bar)
+{
+ struct nvkm_device *device = bar->subdev.device;
+ nvkm_msec(device, 2000,
+ if (!(nvkm_rd32(device, 0x001710) & 0x0000000c))
+ break;
+ );
+}
+
+static const struct nvkm_bar_func
+gm107_bar_func = {
+ .dtor = gf100_bar_dtor,
+ .oneinit = gf100_bar_oneinit,
+ .bar1.init = gf100_bar_bar1_init,
+ .bar1.fini = gf100_bar_bar1_fini,
+ .bar1.wait = gm107_bar_bar1_wait,
+ .bar1.vmm = gf100_bar_bar1_vmm,
+ .bar2.init = gf100_bar_bar2_init,
+ .bar2.fini = gf100_bar_bar2_fini,
+ .bar2.wait = gm107_bar_bar2_wait,
+ .bar2.vmm = gf100_bar_bar2_vmm,
+ .flush = g84_bar_flush,
+};
+
+int
+gm107_bar_new(struct nvkm_device *device, int index, struct nvkm_bar **pbar)
+{
+ return gf100_bar_new_(&gm107_bar_func, device, index, pbar);
+}
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/gm20b.c b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/gm20b.c
new file mode 100644
index 000000000000..950bff1955ad
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/gm20b.c
@@ -0,0 +1,42 @@
+/*
+ * Copyright 2017 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+#include "gf100.h"
+
+static const struct nvkm_bar_func
+gm20b_bar_func = {
+ .dtor = gf100_bar_dtor,
+ .oneinit = gf100_bar_oneinit,
+ .bar1.init = gf100_bar_bar1_init,
+ .bar1.fini = gf100_bar_bar1_fini,
+ .bar1.wait = gm107_bar_bar1_wait,
+ .bar1.vmm = gf100_bar_bar1_vmm,
+ .flush = g84_bar_flush,
+};
+
+int
+gm20b_bar_new(struct nvkm_device *device, int index, struct nvkm_bar **pbar)
+{
+ int ret = gf100_bar_new_(&gm20b_bar_func, device, index, pbar);
+ if (ret == 0)
+ (*pbar)->iomap_uncached = true;
+ return ret;
+}
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/nv50.c b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/nv50.c
index 6eff637ac301..157b076a1272 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/nv50.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/nv50.c
@@ -28,19 +28,6 @@
#include <subdev/mmu.h>
#include <subdev/timer.h>
-struct nvkm_vm *
-nv50_bar_kmap(struct nvkm_bar *base)
-{
- return nv50_bar(base)->bar3_vm;
-}
-
-int
-nv50_bar_umap(struct nvkm_bar *base, u64 size, int type, struct nvkm_vma *vma)
-{
- struct nv50_bar *bar = nv50_bar(base);
- return nvkm_vm_get(bar->bar1_vm, size, type, NV_MEM_ACCESS_RW, vma);
-}
-
static void
nv50_bar_flush(struct nvkm_bar *base)
{
@@ -56,14 +43,72 @@ nv50_bar_flush(struct nvkm_bar *base)
spin_unlock_irqrestore(&bar->base.lock, flags);
}
+struct nvkm_vmm *
+nv50_bar_bar1_vmm(struct nvkm_bar *base)
+{
+ return nv50_bar(base)->bar1_vmm;
+}
+
+void
+nv50_bar_bar1_wait(struct nvkm_bar *base)
+{
+ nvkm_bar_flush(base);
+}
+
+void
+nv50_bar_bar1_fini(struct nvkm_bar *bar)
+{
+ nvkm_wr32(bar->subdev.device, 0x001708, 0x00000000);
+}
+
+void
+nv50_bar_bar1_init(struct nvkm_bar *base)
+{
+ struct nvkm_device *device = base->subdev.device;
+ struct nv50_bar *bar = nv50_bar(base);
+ nvkm_wr32(device, 0x001708, 0x80000000 | bar->bar1->node->offset >> 4);
+}
+
+struct nvkm_vmm *
+nv50_bar_bar2_vmm(struct nvkm_bar *base)
+{
+ return nv50_bar(base)->bar2_vmm;
+}
+
+void
+nv50_bar_bar2_fini(struct nvkm_bar *bar)
+{
+ nvkm_wr32(bar->subdev.device, 0x00170c, 0x00000000);
+}
+
+void
+nv50_bar_bar2_init(struct nvkm_bar *base)
+{
+ struct nvkm_device *device = base->subdev.device;
+ struct nv50_bar *bar = nv50_bar(base);
+ nvkm_wr32(device, 0x001704, 0x00000000 | bar->mem->addr >> 12);
+ nvkm_wr32(device, 0x001704, 0x40000000 | bar->mem->addr >> 12);
+ nvkm_wr32(device, 0x00170c, 0x80000000 | bar->bar2->node->offset >> 4);
+}
+
+void
+nv50_bar_init(struct nvkm_bar *base)
+{
+ struct nv50_bar *bar = nv50_bar(base);
+ struct nvkm_device *device = bar->base.subdev.device;
+ int i;
+
+ for (i = 0; i < 8; i++)
+ nvkm_wr32(device, 0x001900 + (i * 4), 0x00000000);
+}
+
int
nv50_bar_oneinit(struct nvkm_bar *base)
{
struct nv50_bar *bar = nv50_bar(base);
struct nvkm_device *device = bar->base.subdev.device;
static struct lock_class_key bar1_lock;
- static struct lock_class_key bar3_lock;
- struct nvkm_vm *vm;
+ static struct lock_class_key bar2_lock;
u64 start, limit;
int ret;
@@ -80,51 +125,54 @@ nv50_bar_oneinit(struct nvkm_bar *base)
if (ret)
return ret;
- /* BAR3 */
+ /* BAR2 */
start = 0x0100000000ULL;
limit = start + device->func->resource_size(device, 3);
- ret = nvkm_vm_new(device, start, limit - start, start, &bar3_lock, &vm);
+ ret = nvkm_vmm_new(device, start, limit-- - start, NULL, 0,
+ &bar2_lock, "bar2", &bar->bar2_vmm);
if (ret)
return ret;
- atomic_inc(&vm->engref[NVKM_SUBDEV_BAR]);
+ atomic_inc(&bar->bar2_vmm->engref[NVKM_SUBDEV_BAR]);
+ bar->bar2_vmm->debug = bar->base.subdev.debug;
- ret = nvkm_vm_boot(vm, limit-- - start);
+ ret = nvkm_vmm_boot(bar->bar2_vmm);
if (ret)
return ret;
- ret = nvkm_vm_ref(vm, &bar->bar3_vm, bar->pgd);
- nvkm_vm_ref(NULL, &vm, NULL);
+ ret = nvkm_vmm_join(bar->bar2_vmm, bar->mem->memory);
if (ret)
return ret;
- ret = nvkm_gpuobj_new(device, 24, 16, false, bar->mem, &bar->bar3);
+ ret = nvkm_gpuobj_new(device, 24, 16, false, bar->mem, &bar->bar2);
if (ret)
return ret;
- nvkm_kmap(bar->bar3);
- nvkm_wo32(bar->bar3, 0x00, 0x7fc00000);
- nvkm_wo32(bar->bar3, 0x04, lower_32_bits(limit));
- nvkm_wo32(bar->bar3, 0x08, lower_32_bits(start));
- nvkm_wo32(bar->bar3, 0x0c, upper_32_bits(limit) << 24 |
+ nvkm_kmap(bar->bar2);
+ nvkm_wo32(bar->bar2, 0x00, 0x7fc00000);
+ nvkm_wo32(bar->bar2, 0x04, lower_32_bits(limit));
+ nvkm_wo32(bar->bar2, 0x08, lower_32_bits(start));
+ nvkm_wo32(bar->bar2, 0x0c, upper_32_bits(limit) << 24 |
upper_32_bits(start));
- nvkm_wo32(bar->bar3, 0x10, 0x00000000);
- nvkm_wo32(bar->bar3, 0x14, 0x00000000);
- nvkm_done(bar->bar3);
+ nvkm_wo32(bar->bar2, 0x10, 0x00000000);
+ nvkm_wo32(bar->bar2, 0x14, 0x00000000);
+ nvkm_done(bar->bar2);
+
+ bar->base.subdev.oneinit = true;
+ nvkm_bar_bar2_init(device);
/* BAR1 */
start = 0x0000000000ULL;
limit = start + device->func->resource_size(device, 1);
- ret = nvkm_vm_new(device, start, limit-- - start, start, &bar1_lock, &vm);
- if (ret)
- return ret;
+ ret = nvkm_vmm_new(device, start, limit-- - start, NULL, 0,
+ &bar1_lock, "bar1", &bar->bar1_vmm);
- atomic_inc(&vm->engref[NVKM_SUBDEV_BAR]);
+ atomic_inc(&bar->bar1_vmm->engref[NVKM_SUBDEV_BAR]);
+ bar->bar1_vmm->debug = bar->base.subdev.debug;
- ret = nvkm_vm_ref(vm, &bar->bar1_vm, bar->pgd);
- nvkm_vm_ref(NULL, &vm, NULL);
+ ret = nvkm_vmm_join(bar->bar1_vmm, bar->mem->memory);
if (ret)
return ret;
@@ -144,45 +192,21 @@ nv50_bar_oneinit(struct nvkm_bar *base)
return 0;
}
-int
-nv50_bar_init(struct nvkm_bar *base)
-{
- struct nv50_bar *bar = nv50_bar(base);
- struct nvkm_device *device = bar->base.subdev.device;
- int i;
-
- nvkm_mask(device, 0x000200, 0x00000100, 0x00000000);
- nvkm_mask(device, 0x000200, 0x00000100, 0x00000100);
- nvkm_wr32(device, 0x100c80, 0x00060001);
- if (nvkm_msec(device, 2000,
- if (!(nvkm_rd32(device, 0x100c80) & 0x00000001))
- break;
- ) < 0)
- return -EBUSY;
-
- nvkm_wr32(device, 0x001704, 0x00000000 | bar->mem->addr >> 12);
- nvkm_wr32(device, 0x001704, 0x40000000 | bar->mem->addr >> 12);
- nvkm_wr32(device, 0x001708, 0x80000000 | bar->bar1->node->offset >> 4);
- nvkm_wr32(device, 0x00170c, 0x80000000 | bar->bar3->node->offset >> 4);
- for (i = 0; i < 8; i++)
- nvkm_wr32(device, 0x001900 + (i * 4), 0x00000000);
- return 0;
-}
-
void *
nv50_bar_dtor(struct nvkm_bar *base)
{
struct nv50_bar *bar = nv50_bar(base);
- nvkm_gpuobj_del(&bar->bar1);
- nvkm_vm_ref(NULL, &bar->bar1_vm, bar->pgd);
- nvkm_gpuobj_del(&bar->bar3);
- if (bar->bar3_vm) {
- nvkm_memory_del(&bar->bar3_vm->pgt[0].mem[0]);
- nvkm_vm_ref(NULL, &bar->bar3_vm, bar->pgd);
+ if (bar->mem) {
+ nvkm_gpuobj_del(&bar->bar1);
+ nvkm_vmm_part(bar->bar1_vmm, bar->mem->memory);
+ nvkm_vmm_unref(&bar->bar1_vmm);
+ nvkm_gpuobj_del(&bar->bar2);
+ nvkm_vmm_part(bar->bar2_vmm, bar->mem->memory);
+ nvkm_vmm_unref(&bar->bar2_vmm);
+ nvkm_gpuobj_del(&bar->pgd);
+ nvkm_gpuobj_del(&bar->pad);
+ nvkm_gpuobj_del(&bar->mem);
}
- nvkm_gpuobj_del(&bar->pgd);
- nvkm_gpuobj_del(&bar->pad);
- nvkm_gpuobj_del(&bar->mem);
return bar;
}
@@ -204,8 +228,14 @@ nv50_bar_func = {
.dtor = nv50_bar_dtor,
.oneinit = nv50_bar_oneinit,
.init = nv50_bar_init,
- .kmap = nv50_bar_kmap,
- .umap = nv50_bar_umap,
+ .bar1.init = nv50_bar_bar1_init,
+ .bar1.fini = nv50_bar_bar1_fini,
+ .bar1.wait = nv50_bar_bar1_wait,
+ .bar1.vmm = nv50_bar_bar1_vmm,
+ .bar2.init = nv50_bar_bar2_init,
+ .bar2.fini = nv50_bar_bar2_fini,
+ .bar2.wait = nv50_bar_bar1_wait,
+ .bar2.vmm = nv50_bar_bar2_vmm,
.flush = nv50_bar_flush,
};
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/nv50.h b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/nv50.h
index 1eb764f22a49..140b76f588b6 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/nv50.h
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/nv50.h
@@ -9,18 +9,20 @@ struct nv50_bar {
struct nvkm_gpuobj *mem;
struct nvkm_gpuobj *pad;
struct nvkm_gpuobj *pgd;
- struct nvkm_vm *bar1_vm;
+ struct nvkm_vmm *bar1_vmm;
struct nvkm_gpuobj *bar1;
- struct nvkm_vm *bar3_vm;
- struct nvkm_gpuobj *bar3;
+ struct nvkm_vmm *bar2_vmm;
+ struct nvkm_gpuobj *bar2;
};
int nv50_bar_new_(const struct nvkm_bar_func *, struct nvkm_device *,
int, u32 pgd_addr, struct nvkm_bar **);
void *nv50_bar_dtor(struct nvkm_bar *);
int nv50_bar_oneinit(struct nvkm_bar *);
-int nv50_bar_init(struct nvkm_bar *);
-struct nvkm_vm *nv50_bar_kmap(struct nvkm_bar *);
-int nv50_bar_umap(struct nvkm_bar *, u64, int, struct nvkm_vma *);
-void nv50_bar_unmap(struct nvkm_bar *, struct nvkm_vma *);
+void nv50_bar_init(struct nvkm_bar *);
+void nv50_bar_bar1_init(struct nvkm_bar *);
+void nv50_bar_bar1_wait(struct nvkm_bar *);
+struct nvkm_vmm *nv50_bar_bar1_vmm(struct nvkm_bar *);
+void nv50_bar_bar2_init(struct nvkm_bar *);
+struct nvkm_vmm *nv50_bar_bar2_vmm(struct nvkm_bar *);
#endif
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/priv.h b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/priv.h
index d834ef20db5b..14398e2dbdf9 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/priv.h
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/priv.h
@@ -9,11 +9,25 @@ void nvkm_bar_ctor(const struct nvkm_bar_func *, struct nvkm_device *,
struct nvkm_bar_func {
void *(*dtor)(struct nvkm_bar *);
int (*oneinit)(struct nvkm_bar *);
- int (*init)(struct nvkm_bar *);
- struct nvkm_vm *(*kmap)(struct nvkm_bar *);
- int (*umap)(struct nvkm_bar *, u64 size, int type, struct nvkm_vma *);
+ void (*init)(struct nvkm_bar *);
+
+ struct {
+ void (*init)(struct nvkm_bar *);
+ void (*fini)(struct nvkm_bar *);
+ void (*wait)(struct nvkm_bar *);
+ struct nvkm_vmm *(*vmm)(struct nvkm_bar *);
+ } bar1, bar2;
+
void (*flush)(struct nvkm_bar *);
};
+void nv50_bar_bar1_fini(struct nvkm_bar *);
+void nv50_bar_bar2_fini(struct nvkm_bar *);
+
void g84_bar_flush(struct nvkm_bar *);
+
+void gf100_bar_bar1_fini(struct nvkm_bar *);
+void gf100_bar_bar2_fini(struct nvkm_bar *);
+
+void gm107_bar_bar1_wait(struct nvkm_bar *);
#endif
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/bios/iccsense.c b/drivers/gpu/drm/nouveau/nvkm/subdev/bios/iccsense.c
index 23caef8df17f..73e463ed55c3 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/bios/iccsense.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/bios/iccsense.c
@@ -99,7 +99,7 @@ nvbios_iccsense_parse(struct nvkm_bios *bios, struct nvbios_iccsense *iccsense)
rail->extdev_id = nvbios_rd08(bios, entry + 0x1);
res_start = 0x5;
break;
- };
+ }
if (nvbios_extdev_parse(bios, rail->extdev_id, &extdev))
continue;
@@ -115,7 +115,7 @@ nvbios_iccsense_parse(struct nvkm_bios *bios, struct nvbios_iccsense *iccsense)
default:
rail->resistor_count = 0;
break;
- };
+ }
for (r = 0; r < rail->resistor_count; ++r) {
rail->resistors[r].mohm = nvbios_rd08(bios, entry + res_start + r * 2);
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/bios/init.c b/drivers/gpu/drm/nouveau/nvkm/subdev/bios/init.c
index b58ee99f7bfc..9cc10e438b3d 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/bios/init.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/bios/init.c
@@ -36,6 +36,8 @@
#include <subdev/i2c.h>
#include <subdev/vga.h>
+#include <linux/kernel.h>
+
#define bioslog(lvl, fmt, args...) do { \
nvkm_printk(init->subdev, lvl, info, "0x%08x[%c]: "fmt, \
init->offset, init_exec(init) ? \
@@ -2271,8 +2273,6 @@ static struct nvbios_init_opcode {
[0xaa] = { init_reserved },
};
-#define init_opcode_nr (sizeof(init_opcode) / sizeof(init_opcode[0]))
-
int
nvbios_exec(struct nvbios_init *init)
{
@@ -2281,7 +2281,8 @@ nvbios_exec(struct nvbios_init *init)
init->nested++;
while (init->offset) {
u8 opcode = nvbios_rd08(bios, init->offset);
- if (opcode >= init_opcode_nr || !init_opcode[opcode].exec) {
+ if (opcode >= ARRAY_SIZE(init_opcode) ||
+ !init_opcode[opcode].exec) {
error("unknown opcode 0x%02x\n", opcode);
return -EINVAL;
}
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/bios/timing.c b/drivers/gpu/drm/nouveau/nvkm/subdev/bios/timing.c
index 7e83c3985020..20ff5173cf8f 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/bios/timing.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/bios/timing.c
@@ -115,16 +115,21 @@ nvbios_timingEp(struct nvkm_bios *bios, int idx,
switch (min_t(u8, *hdr, 25)) {
case 25:
p->timing_10_24 = nvbios_rd08(bios, data + 0x18);
+ /* fall through */
case 24:
case 23:
case 22:
p->timing_10_21 = nvbios_rd08(bios, data + 0x15);
+ /* fall through */
case 21:
p->timing_10_20 = nvbios_rd08(bios, data + 0x14);
+ /* fall through */
case 20:
p->timing_10_CWL = nvbios_rd08(bios, data + 0x13);
+ /* fall through */
case 19:
p->timing_10_18 = nvbios_rd08(bios, data + 0x12);
+ /* fall through */
case 18:
case 17:
p->timing_10_16 = nvbios_rd08(bios, data + 0x10);
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/devinit/nv04.c b/drivers/gpu/drm/nouveau/nvkm/subdev/devinit/nv04.c
index 158977f8a6e6..c3dae05348eb 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/devinit/nv04.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/devinit/nv04.c
@@ -119,11 +119,11 @@ powerctrl_1_shift(int chip_version, int reg)
switch (reg) {
case 0x680520:
- shift += 4;
+ shift += 4; /* fall through */
case 0x680508:
- shift += 4;
+ shift += 4; /* fall through */
case 0x680504:
- shift += 4;
+ shift += 4; /* fall through */
case 0x680500:
shift += 4;
}
@@ -245,11 +245,11 @@ setPLL_double_highregs(struct nvkm_devinit *init, u32 reg1,
switch (reg1) {
case 0x680504:
- shift_c040 += 2;
+ shift_c040 += 2; /* fall through */
case 0x680500:
- shift_c040 += 2;
+ shift_c040 += 2; /* fall through */
case 0x680520:
- shift_c040 += 2;
+ shift_c040 += 2; /* fall through */
case 0x680508:
shift_c040 += 2;
}
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/base.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/base.c
index a7049c041594..73b5d46104bd 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/base.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/base.c
@@ -31,12 +31,6 @@
#include <engine/gr.h>
#include <engine/mpeg.h>
-bool
-nvkm_fb_memtype_valid(struct nvkm_fb *fb, u32 memtype)
-{
- return fb->func->memtype_valid(fb, memtype);
-}
-
void
nvkm_fb_tile_fini(struct nvkm_fb *fb, int region, struct nvkm_fb_tile *tile)
{
@@ -100,6 +94,7 @@ static int
nvkm_fb_oneinit(struct nvkm_subdev *subdev)
{
struct nvkm_fb *fb = nvkm_fb(subdev);
+ u32 tags = 0;
if (fb->func->ram_new) {
int ret = fb->func->ram_new(fb, &fb->ram);
@@ -115,7 +110,16 @@ nvkm_fb_oneinit(struct nvkm_subdev *subdev)
return ret;
}
- return 0;
+ /* Initialise compression tag allocator.
+ *
+ * LTC oneinit() will override this on Fermi and newer.
+ */
+ if (fb->func->tags) {
+ tags = fb->func->tags(fb);
+ nvkm_debug(subdev, "%d comptags\n", tags);
+ }
+
+ return nvkm_mm_init(&fb->tags, 0, 0, tags, 1);
}
static int
@@ -135,8 +139,13 @@ nvkm_fb_init(struct nvkm_subdev *subdev)
if (fb->func->init)
fb->func->init(fb);
- if (fb->func->init_page)
- fb->func->init_page(fb);
+
+ if (fb->func->init_page) {
+ ret = fb->func->init_page(fb);
+ if (WARN_ON(ret))
+ return ret;
+ }
+
if (fb->func->init_unkn)
fb->func->init_unkn(fb);
return 0;
@@ -148,12 +157,13 @@ nvkm_fb_dtor(struct nvkm_subdev *subdev)
struct nvkm_fb *fb = nvkm_fb(subdev);
int i;
- nvkm_memory_del(&fb->mmu_wr);
- nvkm_memory_del(&fb->mmu_rd);
+ nvkm_memory_unref(&fb->mmu_wr);
+ nvkm_memory_unref(&fb->mmu_rd);
for (i = 0; i < fb->tile.regions; i++)
fb->func->tile.fini(fb, i, &fb->tile.region[i]);
+ nvkm_mm_fini(&fb->tags);
nvkm_ram_del(&fb->ram);
if (fb->func->dtor)
@@ -176,7 +186,8 @@ nvkm_fb_ctor(const struct nvkm_fb_func *func, struct nvkm_device *device,
nvkm_subdev_ctor(&nvkm_fb, device, index, &fb->subdev);
fb->func = func;
fb->tile.regions = fb->func->tile.regions;
- fb->page = nvkm_longopt(device->cfgopt, "NvFbBigPage", 0);
+ fb->page = nvkm_longopt(device->cfgopt, "NvFbBigPage",
+ fb->func->default_bigpage);
}
int
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/g84.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/g84.c
index 9c28392d07e4..06bf95c0c549 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/g84.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/g84.c
@@ -27,6 +27,7 @@
static const struct nv50_fb_func
g84_fb = {
.ram_new = nv50_ram_new,
+ .tags = nv20_fb_tags,
.trap = 0x001d07ff,
};
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gf100.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gf100.c
index a239e73562c8..47d28c279707 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gf100.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gf100.c
@@ -27,15 +27,6 @@
#include <core/memory.h>
#include <core/option.h>
-extern const u8 gf100_pte_storage_type_map[256];
-
-bool
-gf100_fb_memtype_valid(struct nvkm_fb *fb, u32 tile_flags)
-{
- u8 memtype = (tile_flags & 0x0000ff00) >> 8;
- return likely((gf100_pte_storage_type_map[memtype] != 0xff));
-}
-
void
gf100_fb_intr(struct nvkm_fb *base)
{
@@ -80,20 +71,17 @@ gf100_fb_oneinit(struct nvkm_fb *base)
return 0;
}
-void
+int
gf100_fb_init_page(struct nvkm_fb *fb)
{
struct nvkm_device *device = fb->subdev.device;
switch (fb->page) {
- case 16:
- nvkm_mask(device, 0x100c80, 0x00000001, 0x00000001);
- break;
- case 17:
+ case 16: nvkm_mask(device, 0x100c80, 0x00000001, 0x00000001); break;
+ case 17: nvkm_mask(device, 0x100c80, 0x00000001, 0x00000000); break;
default:
- nvkm_mask(device, 0x100c80, 0x00000001, 0x00000000);
- fb->page = 17;
- break;
+ return -EINVAL;
}
+ return 0;
}
void
@@ -143,7 +131,7 @@ gf100_fb = {
.init_page = gf100_fb_init_page,
.intr = gf100_fb_intr,
.ram_new = gf100_ram_new,
- .memtype_valid = gf100_fb_memtype_valid,
+ .default_bigpage = 17,
};
int
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gf100.h b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gf100.h
index 412eb89834e8..e3cf0515bb70 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gf100.h
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gf100.h
@@ -17,7 +17,5 @@ void gf100_fb_intr(struct nvkm_fb *);
void gp100_fb_init(struct nvkm_fb *);
-void gm200_fb_init_page(struct nvkm_fb *fb);
void gm200_fb_init(struct nvkm_fb *base);
-
#endif
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gf108.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gf108.c
index 56af84aa333b..4a9f463745b5 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gf108.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gf108.c
@@ -32,7 +32,7 @@ gf108_fb = {
.init_page = gf100_fb_init_page,
.intr = gf100_fb_intr,
.ram_new = gf108_ram_new,
- .memtype_valid = gf100_fb_memtype_valid,
+ .default_bigpage = 17,
};
int
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gk104.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gk104.c
index 4245e2e6e604..0a6e8eaad42c 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gk104.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gk104.c
@@ -32,7 +32,7 @@ gk104_fb = {
.init_page = gf100_fb_init_page,
.intr = gf100_fb_intr,
.ram_new = gk104_ram_new,
- .memtype_valid = gf100_fb_memtype_valid,
+ .default_bigpage = 17,
};
int
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gk20a.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gk20a.c
index 5d34d6136616..a7e29b125094 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gk20a.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gk20a.c
@@ -30,7 +30,7 @@ gk20a_fb = {
.init = gf100_fb_init,
.init_page = gf100_fb_init_page,
.intr = gf100_fb_intr,
- .memtype_valid = gf100_fb_memtype_valid,
+ .default_bigpage = 17,
};
int
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gm107.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gm107.c
index db699025f546..69c876d5d1c1 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gm107.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gm107.c
@@ -32,7 +32,7 @@ gm107_fb = {
.init_page = gf100_fb_init_page,
.intr = gf100_fb_intr,
.ram_new = gm107_ram_new,
- .memtype_valid = gf100_fb_memtype_valid,
+ .default_bigpage = 17,
};
int
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gm200.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gm200.c
index d83da5ddbc1e..8137e19d3292 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gm200.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gm200.c
@@ -26,22 +26,18 @@
#include <core/memory.h>
-void
+int
gm200_fb_init_page(struct nvkm_fb *fb)
{
struct nvkm_device *device = fb->subdev.device;
switch (fb->page) {
- case 16:
- nvkm_mask(device, 0x100c80, 0x00000801, 0x00000001);
- break;
- case 17:
- nvkm_mask(device, 0x100c80, 0x00000801, 0x00000000);
- break;
+ case 16: nvkm_mask(device, 0x100c80, 0x00001801, 0x00001001); break;
+ case 17: nvkm_mask(device, 0x100c80, 0x00001801, 0x00000000); break;
+ case 0: nvkm_mask(device, 0x100c80, 0x00001800, 0x00001800); break;
default:
- nvkm_mask(device, 0x100c80, 0x00000800, 0x00000800);
- fb->page = 0;
- break;
+ return -EINVAL;
}
+ return 0;
}
void
@@ -69,7 +65,7 @@ gm200_fb = {
.init_page = gm200_fb_init_page,
.intr = gf100_fb_intr,
.ram_new = gm200_ram_new,
- .memtype_valid = gf100_fb_memtype_valid,
+ .default_bigpage = 0 /* per-instance. */,
};
int
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gm20b.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gm20b.c
index b87c233bcd6d..12db61e31128 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gm20b.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gm20b.c
@@ -30,7 +30,7 @@ gm20b_fb = {
.init = gm200_fb_init,
.init_page = gm200_fb_init_page,
.intr = gf100_fb_intr,
- .memtype_valid = gf100_fb_memtype_valid,
+ .default_bigpage = 0 /* per-instance. */,
};
int
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gp100.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gp100.c
index 98474aec1921..147f69b30cd8 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gp100.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gp100.c
@@ -59,7 +59,6 @@ gp100_fb = {
.init_page = gm200_fb_init_page,
.init_unkn = gp100_fb_init_unkn,
.ram_new = gp100_ram_new,
- .memtype_valid = gf100_fb_memtype_valid,
};
int
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gp102.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gp102.c
index 73b4ae1c73dc..b84b9861ef26 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gp102.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gp102.c
@@ -33,7 +33,6 @@ gp102_fb = {
.init = gp100_fb_init,
.init_page = gm200_fb_init_page,
.ram_new = gp100_ram_new,
- .memtype_valid = gf100_fb_memtype_valid,
};
int
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gp10b.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gp10b.c
index f2b1fbf428d5..af8e43979dc1 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gp10b.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gp10b.c
@@ -28,7 +28,6 @@ gp10b_fb = {
.init = gm200_fb_init,
.init_page = gm200_fb_init_page,
.intr = gf100_fb_intr,
- .memtype_valid = gf100_fb_memtype_valid,
};
int
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gt215.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gt215.c
index ebb30608d5ef..9266559b45f9 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gt215.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gt215.c
@@ -27,6 +27,7 @@
static const struct nv50_fb_func
gt215_fb = {
.ram_new = gt215_ram_new,
+ .tags = nv20_fb_tags,
.trap = 0x000d0fff,
};
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv04.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv04.c
index 8ff2e5db4571..c886664533c8 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv04.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv04.c
@@ -25,14 +25,6 @@
#include "ram.h"
#include "regsnv04.h"
-bool
-nv04_fb_memtype_valid(struct nvkm_fb *fb, u32 tile_flags)
-{
- if (!(tile_flags & 0xff00))
- return true;
- return false;
-}
-
static void
nv04_fb_init(struct nvkm_fb *fb)
{
@@ -49,7 +41,6 @@ static const struct nvkm_fb_func
nv04_fb = {
.init = nv04_fb_init,
.ram_new = nv04_ram_new,
- .memtype_valid = nv04_fb_memtype_valid,
};
int
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv10.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv10.c
index e8c44f5a3d84..c998b7e96aa3 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv10.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv10.c
@@ -61,7 +61,6 @@ nv10_fb = {
.tile.fini = nv10_fb_tile_fini,
.tile.prog = nv10_fb_tile_prog,
.ram_new = nv10_ram_new,
- .memtype_valid = nv04_fb_memtype_valid,
};
int
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv1a.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv1a.c
index 2ae0beb87567..7b9f04f44af8 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv1a.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv1a.c
@@ -33,7 +33,6 @@ nv1a_fb = {
.tile.fini = nv10_fb_tile_fini,
.tile.prog = nv10_fb_tile_prog,
.ram_new = nv1a_ram_new,
- .memtype_valid = nv04_fb_memtype_valid,
};
int
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv20.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv20.c
index 126865dfe777..a021d21ff153 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv20.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv20.c
@@ -45,7 +45,7 @@ nv20_fb_tile_comp(struct nvkm_fb *fb, int i, u32 size, u32 flags,
{
u32 tiles = DIV_ROUND_UP(size, 0x40);
u32 tags = round_up(tiles / fb->ram->parts, 0x40);
- if (!nvkm_mm_head(&fb->ram->tags, 0, 1, tags, tags, 1, &tile->tag)) {
+ if (!nvkm_mm_head(&fb->tags, 0, 1, tags, tags, 1, &tile->tag)) {
if (!(flags & 2)) tile->zcomp = 0x00000000; /* Z16 */
else tile->zcomp = 0x04000000; /* Z24S8 */
tile->zcomp |= tile->tag->offset;
@@ -63,7 +63,7 @@ nv20_fb_tile_fini(struct nvkm_fb *fb, int i, struct nvkm_fb_tile *tile)
tile->limit = 0;
tile->pitch = 0;
tile->zcomp = 0;
- nvkm_mm_free(&fb->ram->tags, &tile->tag);
+ nvkm_mm_free(&fb->tags, &tile->tag);
}
void
@@ -77,15 +77,22 @@ nv20_fb_tile_prog(struct nvkm_fb *fb, int i, struct nvkm_fb_tile *tile)
nvkm_wr32(device, 0x100300 + (i * 0x04), tile->zcomp);
}
+u32
+nv20_fb_tags(struct nvkm_fb *fb)
+{
+ const u32 tags = nvkm_rd32(fb->subdev.device, 0x100320);
+ return tags ? tags + 1 : 0;
+}
+
static const struct nvkm_fb_func
nv20_fb = {
+ .tags = nv20_fb_tags,
.tile.regions = 8,
.tile.init = nv20_fb_tile_init,
.tile.comp = nv20_fb_tile_comp,
.tile.fini = nv20_fb_tile_fini,
.tile.prog = nv20_fb_tile_prog,
.ram_new = nv20_ram_new,
- .memtype_valid = nv04_fb_memtype_valid,
};
int
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv25.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv25.c
index c56746d2a502..7709f5fe9a45 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv25.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv25.c
@@ -32,7 +32,7 @@ nv25_fb_tile_comp(struct nvkm_fb *fb, int i, u32 size, u32 flags,
{
u32 tiles = DIV_ROUND_UP(size, 0x40);
u32 tags = round_up(tiles / fb->ram->parts, 0x40);
- if (!nvkm_mm_head(&fb->ram->tags, 0, 1, tags, tags, 1, &tile->tag)) {
+ if (!nvkm_mm_head(&fb->tags, 0, 1, tags, tags, 1, &tile->tag)) {
if (!(flags & 2)) tile->zcomp = 0x00100000; /* Z16 */
else tile->zcomp = 0x00200000; /* Z24S8 */
tile->zcomp |= tile->tag->offset;
@@ -44,13 +44,13 @@ nv25_fb_tile_comp(struct nvkm_fb *fb, int i, u32 size, u32 flags,
static const struct nvkm_fb_func
nv25_fb = {
+ .tags = nv20_fb_tags,
.tile.regions = 8,
.tile.init = nv20_fb_tile_init,
.tile.comp = nv25_fb_tile_comp,
.tile.fini = nv20_fb_tile_fini,
.tile.prog = nv20_fb_tile_prog,
.ram_new = nv20_ram_new,
- .memtype_valid = nv04_fb_memtype_valid,
};
int
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv30.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv30.c
index 2a7c4831b821..8aa782666507 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv30.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv30.c
@@ -51,7 +51,7 @@ nv30_fb_tile_comp(struct nvkm_fb *fb, int i, u32 size, u32 flags,
{
u32 tiles = DIV_ROUND_UP(size, 0x40);
u32 tags = round_up(tiles / fb->ram->parts, 0x40);
- if (!nvkm_mm_head(&fb->ram->tags, 0, 1, tags, tags, 1, &tile->tag)) {
+ if (!nvkm_mm_head(&fb->tags, 0, 1, tags, tags, 1, &tile->tag)) {
if (flags & 2) tile->zcomp |= 0x01000000; /* Z16 */
else tile->zcomp |= 0x02000000; /* Z24S8 */
tile->zcomp |= ((tile->tag->offset ) >> 6);
@@ -116,6 +116,7 @@ nv30_fb_init(struct nvkm_fb *fb)
static const struct nvkm_fb_func
nv30_fb = {
+ .tags = nv20_fb_tags,
.init = nv30_fb_init,
.tile.regions = 8,
.tile.init = nv30_fb_tile_init,
@@ -123,7 +124,6 @@ nv30_fb = {
.tile.fini = nv20_fb_tile_fini,
.tile.prog = nv20_fb_tile_prog,
.ram_new = nv20_ram_new,
- .memtype_valid = nv04_fb_memtype_valid,
};
int
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv35.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv35.c
index 1604b3789ad1..6e83dcff72e0 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv35.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv35.c
@@ -32,7 +32,7 @@ nv35_fb_tile_comp(struct nvkm_fb *fb, int i, u32 size, u32 flags,
{
u32 tiles = DIV_ROUND_UP(size, 0x40);
u32 tags = round_up(tiles / fb->ram->parts, 0x40);
- if (!nvkm_mm_head(&fb->ram->tags, 0, 1, tags, tags, 1, &tile->tag)) {
+ if (!nvkm_mm_head(&fb->tags, 0, 1, tags, tags, 1, &tile->tag)) {
if (flags & 2) tile->zcomp |= 0x04000000; /* Z16 */
else tile->zcomp |= 0x08000000; /* Z24S8 */
tile->zcomp |= ((tile->tag->offset ) >> 6);
@@ -45,6 +45,7 @@ nv35_fb_tile_comp(struct nvkm_fb *fb, int i, u32 size, u32 flags,
static const struct nvkm_fb_func
nv35_fb = {
+ .tags = nv20_fb_tags,
.init = nv30_fb_init,
.tile.regions = 8,
.tile.init = nv30_fb_tile_init,
@@ -52,7 +53,6 @@ nv35_fb = {
.tile.fini = nv20_fb_tile_fini,
.tile.prog = nv20_fb_tile_prog,
.ram_new = nv20_ram_new,
- .memtype_valid = nv04_fb_memtype_valid,
};
int
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv36.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv36.c
index 80cc0a6e3416..2a07617bb44c 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv36.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv36.c
@@ -32,7 +32,7 @@ nv36_fb_tile_comp(struct nvkm_fb *fb, int i, u32 size, u32 flags,
{
u32 tiles = DIV_ROUND_UP(size, 0x40);
u32 tags = round_up(tiles / fb->ram->parts, 0x40);
- if (!nvkm_mm_head(&fb->ram->tags, 0, 1, tags, tags, 1, &tile->tag)) {
+ if (!nvkm_mm_head(&fb->tags, 0, 1, tags, tags, 1, &tile->tag)) {
if (flags & 2) tile->zcomp |= 0x10000000; /* Z16 */
else tile->zcomp |= 0x20000000; /* Z24S8 */
tile->zcomp |= ((tile->tag->offset ) >> 6);
@@ -45,6 +45,7 @@ nv36_fb_tile_comp(struct nvkm_fb *fb, int i, u32 size, u32 flags,
static const struct nvkm_fb_func
nv36_fb = {
+ .tags = nv20_fb_tags,
.init = nv30_fb_init,
.tile.regions = 8,
.tile.init = nv30_fb_tile_init,
@@ -52,7 +53,6 @@ nv36_fb = {
.tile.fini = nv20_fb_tile_fini,
.tile.prog = nv20_fb_tile_prog,
.ram_new = nv20_ram_new,
- .memtype_valid = nv04_fb_memtype_valid,
};
int
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv40.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv40.c
index deec46a310f8..955160778b5b 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv40.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv40.c
@@ -33,7 +33,7 @@ nv40_fb_tile_comp(struct nvkm_fb *fb, int i, u32 size, u32 flags,
u32 tiles = DIV_ROUND_UP(size, 0x80);
u32 tags = round_up(tiles / fb->ram->parts, 0x100);
if ( (flags & 2) &&
- !nvkm_mm_head(&fb->ram->tags, 0, 1, tags, tags, 1, &tile->tag)) {
+ !nvkm_mm_head(&fb->tags, 0, 1, tags, tags, 1, &tile->tag)) {
tile->zcomp = 0x28000000; /* Z24S8_SPLIT_GRAD */
tile->zcomp |= ((tile->tag->offset ) >> 8);
tile->zcomp |= ((tile->tag->offset + tags - 1) >> 8) << 13;
@@ -51,6 +51,7 @@ nv40_fb_init(struct nvkm_fb *fb)
static const struct nvkm_fb_func
nv40_fb = {
+ .tags = nv20_fb_tags,
.init = nv40_fb_init,
.tile.regions = 8,
.tile.init = nv30_fb_tile_init,
@@ -58,7 +59,6 @@ nv40_fb = {
.tile.fini = nv20_fb_tile_fini,
.tile.prog = nv20_fb_tile_prog,
.ram_new = nv40_ram_new,
- .memtype_valid = nv04_fb_memtype_valid,
};
int
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv41.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv41.c
index 79e57dd5a00f..b77f08d34cc3 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv41.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv41.c
@@ -45,6 +45,7 @@ nv41_fb_init(struct nvkm_fb *fb)
static const struct nvkm_fb_func
nv41_fb = {
+ .tags = nv20_fb_tags,
.init = nv41_fb_init,
.tile.regions = 12,
.tile.init = nv30_fb_tile_init,
@@ -52,7 +53,6 @@ nv41_fb = {
.tile.fini = nv20_fb_tile_fini,
.tile.prog = nv41_fb_tile_prog,
.ram_new = nv41_ram_new,
- .memtype_valid = nv04_fb_memtype_valid,
};
int
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv44.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv44.c
index 06246cce5ec4..b59dc486083d 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv44.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv44.c
@@ -62,7 +62,6 @@ nv44_fb = {
.tile.fini = nv20_fb_tile_fini,
.tile.prog = nv44_fb_tile_prog,
.ram_new = nv44_ram_new,
- .memtype_valid = nv04_fb_memtype_valid,
};
int
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv46.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv46.c
index 3598a1aa65be..cab7d20fa039 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv46.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv46.c
@@ -48,7 +48,6 @@ nv46_fb = {
.tile.fini = nv20_fb_tile_fini,
.tile.prog = nv44_fb_tile_prog,
.ram_new = nv44_ram_new,
- .memtype_valid = nv04_fb_memtype_valid,
};
int
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv47.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv47.c
index c505e4429314..a8b0ad4c871d 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv47.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv47.c
@@ -28,6 +28,7 @@
static const struct nvkm_fb_func
nv47_fb = {
+ .tags = nv20_fb_tags,
.init = nv41_fb_init,
.tile.regions = 15,
.tile.init = nv30_fb_tile_init,
@@ -35,7 +36,6 @@ nv47_fb = {
.tile.fini = nv20_fb_tile_fini,
.tile.prog = nv41_fb_tile_prog,
.ram_new = nv41_ram_new,
- .memtype_valid = nv04_fb_memtype_valid,
};
int
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv49.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv49.c
index 7b91b9f170e5..d0b317bb0252 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv49.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv49.c
@@ -28,6 +28,7 @@
static const struct nvkm_fb_func
nv49_fb = {
+ .tags = nv20_fb_tags,
.init = nv41_fb_init,
.tile.regions = 15,
.tile.init = nv30_fb_tile_init,
@@ -35,7 +36,6 @@ nv49_fb = {
.tile.fini = nv20_fb_tile_fini,
.tile.prog = nv41_fb_tile_prog,
.ram_new = nv49_ram_new,
- .memtype_valid = nv04_fb_memtype_valid,
};
int
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv4e.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv4e.c
index 4e98210c1b1c..6a6f0c086071 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv4e.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv4e.c
@@ -34,7 +34,6 @@ nv4e_fb = {
.tile.fini = nv20_fb_tile_fini,
.tile.prog = nv44_fb_tile_prog,
.ram_new = nv44_ram_new,
- .memtype_valid = nv04_fb_memtype_valid,
};
int
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv50.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv50.c
index 0595e0722bfc..b2f5bf8144ea 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv50.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv50.c
@@ -28,18 +28,6 @@
#include <core/enum.h>
#include <engine/fifo.h>
-int
-nv50_fb_memtype[0x80] = {
- 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- 1, 1, 1, 1, 0, 0, 0, 0, 2, 2, 2, 2, 0, 0, 0, 0,
- 1, 1, 1, 1, 1, 1, 1, 0, 2, 2, 2, 2, 2, 2, 2, 0,
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 0, 0,
- 0, 0, 0, 0, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 2, 2, 2, 2,
- 1, 0, 2, 0, 1, 0, 2, 0, 1, 1, 2, 2, 1, 1, 0, 0
-};
-
static int
nv50_fb_ram_new(struct nvkm_fb *base, struct nvkm_ram **pram)
{
@@ -47,12 +35,6 @@ nv50_fb_ram_new(struct nvkm_fb *base, struct nvkm_ram **pram)
return fb->func->ram_new(&fb->base, pram);
}
-static bool
-nv50_fb_memtype_valid(struct nvkm_fb *fb, u32 memtype)
-{
- return nv50_fb_memtype[(memtype & 0xff00) >> 8] != 0;
-}
-
static const struct nvkm_enum vm_dispatch_subclients[] = {
{ 0x00000000, "GRCTX" },
{ 0x00000001, "NOTIFY" },
@@ -244,6 +226,15 @@ nv50_fb_init(struct nvkm_fb *base)
nvkm_wr32(device, 0x100c90, fb->func->trap);
}
+static u32
+nv50_fb_tags(struct nvkm_fb *base)
+{
+ struct nv50_fb *fb = nv50_fb(base);
+ if (fb->func->tags)
+ return fb->func->tags(&fb->base);
+ return 0;
+}
+
static void *
nv50_fb_dtor(struct nvkm_fb *base)
{
@@ -262,11 +253,11 @@ nv50_fb_dtor(struct nvkm_fb *base)
static const struct nvkm_fb_func
nv50_fb_ = {
.dtor = nv50_fb_dtor,
+ .tags = nv50_fb_tags,
.oneinit = nv50_fb_oneinit,
.init = nv50_fb_init,
.intr = nv50_fb_intr,
.ram_new = nv50_fb_ram_new,
- .memtype_valid = nv50_fb_memtype_valid,
};
int
@@ -287,6 +278,7 @@ nv50_fb_new_(const struct nv50_fb_func *func, struct nvkm_device *device,
static const struct nv50_fb_func
nv50_fb = {
.ram_new = nv50_ram_new,
+ .tags = nv20_fb_tags,
.trap = 0x000707ff,
};
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv50.h b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv50.h
index faa88c8c66fe..13231d4b00d9 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv50.h
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv50.h
@@ -12,10 +12,10 @@ struct nv50_fb {
struct nv50_fb_func {
int (*ram_new)(struct nvkm_fb *, struct nvkm_ram **);
+ u32 (*tags)(struct nvkm_fb *);
u32 trap;
};
int nv50_fb_new_(const struct nv50_fb_func *, struct nvkm_device *, int index,
struct nvkm_fb **pfb);
-extern int nv50_fb_memtype[0x80];
#endif
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/priv.h b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/priv.h
index e905d44fa1d5..e05d95240e85 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/priv.h
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/priv.h
@@ -6,9 +6,10 @@ struct nvkm_bios;
struct nvkm_fb_func {
void *(*dtor)(struct nvkm_fb *);
+ u32 (*tags)(struct nvkm_fb *);
int (*oneinit)(struct nvkm_fb *);
void (*init)(struct nvkm_fb *);
- void (*init_page)(struct nvkm_fb *);
+ int (*init_page)(struct nvkm_fb *);
void (*init_unkn)(struct nvkm_fb *);
void (*intr)(struct nvkm_fb *);
@@ -24,7 +25,7 @@ struct nvkm_fb_func {
int (*ram_new)(struct nvkm_fb *, struct nvkm_ram **);
- bool (*memtype_valid)(struct nvkm_fb *, u32 memtype);
+ u8 default_bigpage;
};
void nvkm_fb_ctor(const struct nvkm_fb_func *, struct nvkm_device *device,
@@ -33,13 +34,12 @@ int nvkm_fb_new_(const struct nvkm_fb_func *, struct nvkm_device *device,
int index, struct nvkm_fb **);
int nvkm_fb_bios_memtype(struct nvkm_bios *);
-bool nv04_fb_memtype_valid(struct nvkm_fb *, u32 memtype);
-
void nv10_fb_tile_init(struct nvkm_fb *, int i, u32 addr, u32 size,
u32 pitch, u32 flags, struct nvkm_fb_tile *);
void nv10_fb_tile_fini(struct nvkm_fb *, int i, struct nvkm_fb_tile *);
void nv10_fb_tile_prog(struct nvkm_fb *, int, struct nvkm_fb_tile *);
+u32 nv20_fb_tags(struct nvkm_fb *);
void nv20_fb_tile_init(struct nvkm_fb *, int i, u32 addr, u32 size,
u32 pitch, u32 flags, struct nvkm_fb_tile *);
void nv20_fb_tile_fini(struct nvkm_fb *, int i, struct nvkm_fb_tile *);
@@ -62,8 +62,7 @@ void nv46_fb_tile_init(struct nvkm_fb *, int i, u32 addr, u32 size,
u32 pitch, u32 flags, struct nvkm_fb_tile *);
int gf100_fb_oneinit(struct nvkm_fb *);
-void gf100_fb_init_page(struct nvkm_fb *);
-bool gf100_fb_memtype_valid(struct nvkm_fb *, u32);
+int gf100_fb_init_page(struct nvkm_fb *);
-void gm200_fb_init_page(struct nvkm_fb *);
+int gm200_fb_init_page(struct nvkm_fb *);
#endif
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ram.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ram.c
index c17d559dbfbe..24c7bd505731 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ram.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ram.c
@@ -21,8 +21,132 @@
*
* Authors: Ben Skeggs <bskeggs@redhat.com>
*/
+#define nvkm_vram(p) container_of((p), struct nvkm_vram, memory)
#include "ram.h"
+#include <core/memory.h>
+#include <subdev/mmu.h>
+
+struct nvkm_vram {
+ struct nvkm_memory memory;
+ struct nvkm_ram *ram;
+ u8 page;
+ struct nvkm_mm_node *mn;
+};
+
+static int
+nvkm_vram_map(struct nvkm_memory *memory, u64 offset, struct nvkm_vmm *vmm,
+ struct nvkm_vma *vma, void *argv, u32 argc)
+{
+ struct nvkm_vram *vram = nvkm_vram(memory);
+ struct nvkm_vmm_map map = {
+ .memory = &vram->memory,
+ .offset = offset,
+ .mem = vram->mn,
+ };
+
+ return nvkm_vmm_map(vmm, vma, argv, argc, &map);
+}
+
+static u64
+nvkm_vram_size(struct nvkm_memory *memory)
+{
+ return (u64)nvkm_mm_size(nvkm_vram(memory)->mn) << NVKM_RAM_MM_SHIFT;
+}
+
+static u64
+nvkm_vram_addr(struct nvkm_memory *memory)
+{
+ struct nvkm_vram *vram = nvkm_vram(memory);
+ if (!nvkm_mm_contiguous(vram->mn))
+ return ~0ULL;
+ return (u64)nvkm_mm_addr(vram->mn) << NVKM_RAM_MM_SHIFT;
+}
+
+static u8
+nvkm_vram_page(struct nvkm_memory *memory)
+{
+ return nvkm_vram(memory)->page;
+}
+
+static enum nvkm_memory_target
+nvkm_vram_target(struct nvkm_memory *memory)
+{
+ return NVKM_MEM_TARGET_VRAM;
+}
+
+static void *
+nvkm_vram_dtor(struct nvkm_memory *memory)
+{
+ struct nvkm_vram *vram = nvkm_vram(memory);
+ struct nvkm_mm_node *next = vram->mn;
+ struct nvkm_mm_node *node;
+ mutex_lock(&vram->ram->fb->subdev.mutex);
+ while ((node = next)) {
+ next = node->next;
+ nvkm_mm_free(&vram->ram->vram, &node);
+ }
+ mutex_unlock(&vram->ram->fb->subdev.mutex);
+ return vram;
+}
+
+static const struct nvkm_memory_func
+nvkm_vram = {
+ .dtor = nvkm_vram_dtor,
+ .target = nvkm_vram_target,
+ .page = nvkm_vram_page,
+ .addr = nvkm_vram_addr,
+ .size = nvkm_vram_size,
+ .map = nvkm_vram_map,
+};
+
+int
+nvkm_ram_get(struct nvkm_device *device, u8 heap, u8 type, u8 rpage, u64 size,
+ bool contig, bool back, struct nvkm_memory **pmemory)
+{
+ struct nvkm_ram *ram;
+ struct nvkm_mm *mm;
+ struct nvkm_mm_node **node, *r;
+ struct nvkm_vram *vram;
+ u8 page = max(rpage, (u8)NVKM_RAM_MM_SHIFT);
+ u32 align = (1 << page) >> NVKM_RAM_MM_SHIFT;
+ u32 max = ALIGN(size, 1 << page) >> NVKM_RAM_MM_SHIFT;
+ u32 min = contig ? max : align;
+ int ret;
+
+ if (!device->fb || !(ram = device->fb->ram))
+ return -ENODEV;
+ ram = device->fb->ram;
+ mm = &ram->vram;
+
+ if (!(vram = kzalloc(sizeof(*vram), GFP_KERNEL)))
+ return -ENOMEM;
+ nvkm_memory_ctor(&nvkm_vram, &vram->memory);
+ vram->ram = ram;
+ vram->page = page;
+ *pmemory = &vram->memory;
+
+ mutex_lock(&ram->fb->subdev.mutex);
+ node = &vram->mn;
+ do {
+ if (back)
+ ret = nvkm_mm_tail(mm, heap, type, max, min, align, &r);
+ else
+ ret = nvkm_mm_head(mm, heap, type, max, min, align, &r);
+ if (ret) {
+ mutex_unlock(&ram->fb->subdev.mutex);
+ nvkm_memory_unref(pmemory);
+ return ret;
+ }
+
+ *node = r;
+ node = &r->next;
+ max -= r->length;
+ } while (max);
+ mutex_unlock(&ram->fb->subdev.mutex);
+ return 0;
+}
+
int
nvkm_ram_init(struct nvkm_ram *ram)
{
@@ -38,7 +162,6 @@ nvkm_ram_del(struct nvkm_ram **pram)
if (ram && !WARN_ON(!ram->func)) {
if (ram->func->dtor)
*pram = ram->func->dtor(ram);
- nvkm_mm_fini(&ram->tags);
nvkm_mm_fini(&ram->vram);
kfree(*pram);
*pram = NULL;
@@ -47,8 +170,7 @@ nvkm_ram_del(struct nvkm_ram **pram)
int
nvkm_ram_ctor(const struct nvkm_ram_func *func, struct nvkm_fb *fb,
- enum nvkm_ram_type type, u64 size, u32 tags,
- struct nvkm_ram *ram)
+ enum nvkm_ram_type type, u64 size, struct nvkm_ram *ram)
{
static const char *name[] = {
[NVKM_RAM_TYPE_UNKNOWN] = "of unknown memory type",
@@ -73,28 +195,20 @@ nvkm_ram_ctor(const struct nvkm_ram_func *func, struct nvkm_fb *fb,
ram->size = size;
if (!nvkm_mm_initialised(&ram->vram)) {
- ret = nvkm_mm_init(&ram->vram, 0, size >> NVKM_RAM_MM_SHIFT, 1);
+ ret = nvkm_mm_init(&ram->vram, NVKM_RAM_MM_NORMAL, 0,
+ size >> NVKM_RAM_MM_SHIFT, 1);
if (ret)
return ret;
}
- if (!nvkm_mm_initialised(&ram->tags)) {
- ret = nvkm_mm_init(&ram->tags, 0, tags ? ++tags : 0, 1);
- if (ret)
- return ret;
-
- nvkm_debug(subdev, "%d compression tags\n", tags);
- }
-
return 0;
}
int
nvkm_ram_new_(const struct nvkm_ram_func *func, struct nvkm_fb *fb,
- enum nvkm_ram_type type, u64 size, u32 tags,
- struct nvkm_ram **pram)
+ enum nvkm_ram_type type, u64 size, struct nvkm_ram **pram)
{
if (!(*pram = kzalloc(sizeof(**pram), GFP_KERNEL)))
return -ENOMEM;
- return nvkm_ram_ctor(func, fb, type, size, tags, *pram);
+ return nvkm_ram_ctor(func, fb, type, size, *pram);
}
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ram.h b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ram.h
index fac7e73c3ddf..70fd59dcd06d 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ram.h
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ram.h
@@ -3,11 +3,9 @@
#include "priv.h"
int nvkm_ram_ctor(const struct nvkm_ram_func *, struct nvkm_fb *,
- enum nvkm_ram_type, u64 size, u32 tags,
- struct nvkm_ram *);
+ enum nvkm_ram_type, u64 size, struct nvkm_ram *);
int nvkm_ram_new_(const struct nvkm_ram_func *, struct nvkm_fb *,
- enum nvkm_ram_type, u64 size, u32 tags,
- struct nvkm_ram **);
+ enum nvkm_ram_type, u64 size, struct nvkm_ram **);
void nvkm_ram_del(struct nvkm_ram **);
int nvkm_ram_init(struct nvkm_ram *);
@@ -15,9 +13,6 @@ extern const struct nvkm_ram_func nv04_ram_func;
int nv50_ram_ctor(const struct nvkm_ram_func *, struct nvkm_fb *,
struct nvkm_ram *);
-int nv50_ram_get(struct nvkm_ram *, u64, u32, u32, u32, struct nvkm_mem **);
-void nv50_ram_put(struct nvkm_ram *, struct nvkm_mem **);
-void __nv50_ram_put(struct nvkm_ram *, struct nvkm_mem *);
int gf100_ram_new_(const struct nvkm_ram_func *, struct nvkm_fb *,
struct nvkm_ram **);
@@ -28,8 +23,6 @@ u32 gf100_ram_probe_fbp(const struct nvkm_ram_func *,
u32 gf100_ram_probe_fbp_amount(const struct nvkm_ram_func *, u32,
struct nvkm_device *, int, int *);
u32 gf100_ram_probe_fbpa_amount(struct nvkm_device *, int);
-int gf100_ram_get(struct nvkm_ram *, u64, u32, u32, u32, struct nvkm_mem **);
-void gf100_ram_put(struct nvkm_ram *, struct nvkm_mem **);
int gf100_ram_init(struct nvkm_ram *);
int gf100_ram_calc(struct nvkm_ram *, u32);
int gf100_ram_prog(struct nvkm_ram *);
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramgf100.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramgf100.c
index 4a9bd4f1cb93..ac87a3b6b7c9 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramgf100.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramgf100.c
@@ -32,7 +32,6 @@
#include <subdev/bios/timing.h>
#include <subdev/clk.h>
#include <subdev/clk/pll.h>
-#include <subdev/ltc.h>
struct gf100_ramfuc {
struct ramfuc base;
@@ -420,86 +419,6 @@ gf100_ram_tidy(struct nvkm_ram *base)
ram_exec(&ram->fuc, false);
}
-void
-gf100_ram_put(struct nvkm_ram *ram, struct nvkm_mem **pmem)
-{
- struct nvkm_ltc *ltc = ram->fb->subdev.device->ltc;
- struct nvkm_mem *mem = *pmem;
-
- *pmem = NULL;
- if (unlikely(mem == NULL))
- return;
-
- mutex_lock(&ram->fb->subdev.mutex);
- if (mem->tag)
- nvkm_ltc_tags_free(ltc, &mem->tag);
- __nv50_ram_put(ram, mem);
- mutex_unlock(&ram->fb->subdev.mutex);
-
- kfree(mem);
-}
-
-int
-gf100_ram_get(struct nvkm_ram *ram, u64 size, u32 align, u32 ncmin,
- u32 memtype, struct nvkm_mem **pmem)
-{
- struct nvkm_ltc *ltc = ram->fb->subdev.device->ltc;
- struct nvkm_mm *mm = &ram->vram;
- struct nvkm_mm_node **node, *r;
- struct nvkm_mem *mem;
- int type = (memtype & 0x0ff);
- int back = (memtype & 0x800);
- const bool comp = gf100_pte_storage_type_map[type] != type;
- int ret;
-
- size >>= NVKM_RAM_MM_SHIFT;
- align >>= NVKM_RAM_MM_SHIFT;
- ncmin >>= NVKM_RAM_MM_SHIFT;
- if (!ncmin)
- ncmin = size;
-
- mem = kzalloc(sizeof(*mem), GFP_KERNEL);
- if (!mem)
- return -ENOMEM;
-
- mem->size = size;
-
- mutex_lock(&ram->fb->subdev.mutex);
- if (comp) {
- /* compression only works with lpages */
- if (align == (1 << (17 - NVKM_RAM_MM_SHIFT))) {
- int n = size >> 5;
- nvkm_ltc_tags_alloc(ltc, n, &mem->tag);
- }
-
- if (unlikely(!mem->tag))
- type = gf100_pte_storage_type_map[type];
- }
- mem->memtype = type;
-
- node = &mem->mem;
- do {
- if (back)
- ret = nvkm_mm_tail(mm, 0, 1, size, ncmin, align, &r);
- else
- ret = nvkm_mm_head(mm, 0, 1, size, ncmin, align, &r);
- if (ret) {
- mutex_unlock(&ram->fb->subdev.mutex);
- ram->func->put(ram, &mem);
- return ret;
- }
-
- *node = r;
- node = &r->next;
- size -= r->length;
- } while (size);
- mutex_unlock(&ram->fb->subdev.mutex);
-
- mem->offset = (u64)mem->mem->offset << NVKM_RAM_MM_SHIFT;
- *pmem = mem;
- return 0;
-}
-
int
gf100_ram_init(struct nvkm_ram *base)
{
@@ -604,7 +523,7 @@ gf100_ram_ctor(const struct nvkm_ram_func *func, struct nvkm_fb *fb,
nvkm_debug(subdev, "Upper: %4lld MiB @ %010llx\n", usize >> 20, ubase);
nvkm_debug(subdev, "Total: %4lld MiB\n", total >> 20);
- ret = nvkm_ram_ctor(func, fb, type, total, 0, ram);
+ ret = nvkm_ram_ctor(func, fb, type, total, ram);
if (ret)
return ret;
@@ -617,7 +536,8 @@ gf100_ram_ctor(const struct nvkm_ram_func *func, struct nvkm_fb *fb,
*/
if (lower != total) {
/* The common memory amount is addressed normally. */
- ret = nvkm_mm_init(&ram->vram, rsvd_head >> NVKM_RAM_MM_SHIFT,
+ ret = nvkm_mm_init(&ram->vram, NVKM_RAM_MM_NORMAL,
+ rsvd_head >> NVKM_RAM_MM_SHIFT,
(lower - rsvd_head) >> NVKM_RAM_MM_SHIFT, 1);
if (ret)
return ret;
@@ -625,13 +545,15 @@ gf100_ram_ctor(const struct nvkm_ram_func *func, struct nvkm_fb *fb,
/* And the rest is much higher in the physical address
* space, and may not be usable for certain operations.
*/
- ret = nvkm_mm_init(&ram->vram, ubase >> NVKM_RAM_MM_SHIFT,
+ ret = nvkm_mm_init(&ram->vram, NVKM_RAM_MM_MIXED,
+ ubase >> NVKM_RAM_MM_SHIFT,
(usize - rsvd_tail) >> NVKM_RAM_MM_SHIFT, 1);
if (ret)
return ret;
} else {
/* GPUs without mixed-memory are a lot nicer... */
- ret = nvkm_mm_init(&ram->vram, rsvd_head >> NVKM_RAM_MM_SHIFT,
+ ret = nvkm_mm_init(&ram->vram, NVKM_RAM_MM_NORMAL,
+ rsvd_head >> NVKM_RAM_MM_SHIFT,
(total - rsvd_head - rsvd_tail) >>
NVKM_RAM_MM_SHIFT, 1);
if (ret)
@@ -738,8 +660,6 @@ gf100_ram = {
.probe_fbp_amount = gf100_ram_probe_fbp_amount,
.probe_fbpa_amount = gf100_ram_probe_fbpa_amount,
.init = gf100_ram_init,
- .get = gf100_ram_get,
- .put = gf100_ram_put,
.calc = gf100_ram_calc,
.prog = gf100_ram_prog,
.tidy = gf100_ram_tidy,
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramgf108.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramgf108.c
index 985ec64cf369..70a06e3cd55a 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramgf108.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramgf108.c
@@ -48,8 +48,6 @@ gf108_ram = {
.probe_fbp_amount = gf108_ram_probe_fbp_amount,
.probe_fbpa_amount = gf100_ram_probe_fbpa_amount,
.init = gf100_ram_init,
- .get = gf100_ram_get,
- .put = gf100_ram_put,
.calc = gf100_ram_calc,
.prog = gf100_ram_prog,
.tidy = gf100_ram_tidy,
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramgk104.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramgk104.c
index 75814f15eb53..8bcb7e79a0cb 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramgk104.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramgk104.c
@@ -1704,8 +1704,6 @@ gk104_ram = {
.probe_fbpa_amount = gf100_ram_probe_fbpa_amount,
.dtor = gk104_ram_dtor,
.init = gk104_ram_init,
- .get = gf100_ram_get,
- .put = gf100_ram_put,
.calc = gk104_ram_calc,
.prog = gk104_ram_prog,
.tidy = gk104_ram_tidy,
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramgm107.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramgm107.c
index 3f0b56347291..27c68e3f9772 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramgm107.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramgm107.c
@@ -39,8 +39,6 @@ gm107_ram = {
.probe_fbpa_amount = gf100_ram_probe_fbpa_amount,
.dtor = gk104_ram_dtor,
.init = gk104_ram_init,
- .get = gf100_ram_get,
- .put = gf100_ram_put,
.calc = gk104_ram_calc,
.prog = gk104_ram_prog,
.tidy = gk104_ram_tidy,
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramgm200.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramgm200.c
index fd8facf90476..6b0cac1fe7b4 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramgm200.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramgm200.c
@@ -54,8 +54,6 @@ gm200_ram = {
.probe_fbpa_amount = gf100_ram_probe_fbpa_amount,
.dtor = gk104_ram_dtor,
.init = gk104_ram_init,
- .get = gf100_ram_get,
- .put = gf100_ram_put,
.calc = gk104_ram_calc,
.prog = gk104_ram_prog,
.tidy = gk104_ram_tidy,
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramgp100.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramgp100.c
index df8a87333b67..adb62a6beb63 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramgp100.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramgp100.c
@@ -84,8 +84,6 @@ gp100_ram = {
.probe_fbp_amount = gm200_ram_probe_fbp_amount,
.probe_fbpa_amount = gp100_ram_probe_fbpa,
.init = gp100_ram_init,
- .get = gf100_ram_get,
- .put = gf100_ram_put,
};
int
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramgt215.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramgt215.c
index f10664372161..920b3d347803 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramgt215.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramgt215.c
@@ -26,6 +26,7 @@
#include "ram.h"
#include "ramfuc.h"
+#include <core/memory.h>
#include <core/option.h>
#include <subdev/bios.h>
#include <subdev/bios/M0205.h>
@@ -86,7 +87,7 @@ struct gt215_ltrain {
u32 r_100720;
u32 r_1111e0;
u32 r_111400;
- struct nvkm_mem *mem;
+ struct nvkm_memory *memory;
};
struct gt215_ram {
@@ -279,10 +280,10 @@ gt215_link_train_init(struct gt215_ram *ram)
struct gt215_ltrain *train = &ram->ltrain;
struct nvkm_device *device = ram->base.fb->subdev.device;
struct nvkm_bios *bios = device->bios;
- struct nvkm_mem *mem;
struct nvbios_M0205E M0205E;
u8 ver, hdr, cnt, len;
u32 r001700;
+ u64 addr;
int ret, i = 0;
train->state = NVA3_TRAIN_UNSUPPORTED;
@@ -297,14 +298,14 @@ gt215_link_train_init(struct gt215_ram *ram)
train->state = NVA3_TRAIN_ONCE;
- ret = ram->base.func->get(&ram->base, 0x8000, 0x10000, 0, 0x800,
- &ram->ltrain.mem);
+ ret = nvkm_ram_get(device, NVKM_RAM_MM_NORMAL, 0x01, 16, 0x8000,
+ true, true, &ram->ltrain.memory);
if (ret)
return ret;
- mem = ram->ltrain.mem;
+ addr = nvkm_memory_addr(ram->ltrain.memory);
- nvkm_wr32(device, 0x100538, 0x10000000 | (mem->offset >> 16));
+ nvkm_wr32(device, 0x100538, 0x10000000 | (addr >> 16));
nvkm_wr32(device, 0x1005a8, 0x0000ffff);
nvkm_mask(device, 0x10f800, 0x00000001, 0x00000001);
@@ -320,7 +321,7 @@ gt215_link_train_init(struct gt215_ram *ram)
/* And upload the pattern */
r001700 = nvkm_rd32(device, 0x1700);
- nvkm_wr32(device, 0x1700, mem->offset >> 16);
+ nvkm_wr32(device, 0x1700, addr >> 16);
for (i = 0; i < 16; i++)
nvkm_wr32(device, 0x700000 + (i << 2), pattern[i]);
for (i = 0; i < 16; i++)
@@ -336,8 +337,7 @@ gt215_link_train_init(struct gt215_ram *ram)
static void
gt215_link_train_fini(struct gt215_ram *ram)
{
- if (ram->ltrain.mem)
- ram->base.func->put(&ram->base, &ram->ltrain.mem);
+ nvkm_memory_unref(&ram->ltrain.memory);
}
/*
@@ -931,8 +931,6 @@ static const struct nvkm_ram_func
gt215_ram_func = {
.dtor = gt215_ram_dtor,
.init = gt215_ram_init,
- .get = nv50_ram_get,
- .put = nv50_ram_put,
.calc = gt215_ram_calc,
.prog = gt215_ram_prog,
.tidy = gt215_ram_tidy,
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/rammcp77.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/rammcp77.c
index 017a91de74a0..7de18e53ef45 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/rammcp77.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/rammcp77.c
@@ -53,8 +53,6 @@ mcp77_ram_init(struct nvkm_ram *base)
static const struct nvkm_ram_func
mcp77_ram_func = {
.init = mcp77_ram_init,
- .get = nv50_ram_get,
- .put = nv50_ram_put,
};
int
@@ -73,7 +71,7 @@ mcp77_ram_new(struct nvkm_fb *fb, struct nvkm_ram **pram)
*pram = &ram->base;
ret = nvkm_ram_ctor(&mcp77_ram_func, fb, NVKM_RAM_TYPE_STOLEN,
- size, 0, &ram->base);
+ size, &ram->base);
if (ret)
return ret;
@@ -81,7 +79,8 @@ mcp77_ram_new(struct nvkm_fb *fb, struct nvkm_ram **pram)
ram->base.stolen = base;
nvkm_mm_fini(&ram->base.vram);
- return nvkm_mm_init(&ram->base.vram, rsvd_head >> NVKM_RAM_MM_SHIFT,
+ return nvkm_mm_init(&ram->base.vram, NVKM_RAM_MM_NORMAL,
+ rsvd_head >> NVKM_RAM_MM_SHIFT,
(size - rsvd_head - rsvd_tail) >>
NVKM_RAM_MM_SHIFT, 1);
}
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramnv04.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramnv04.c
index 6f053a03d61c..cc764a93f1a3 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramnv04.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramnv04.c
@@ -61,5 +61,5 @@ nv04_ram_new(struct nvkm_fb *fb, struct nvkm_ram **pram)
else
type = NVKM_RAM_TYPE_SDRAM;
- return nvkm_ram_new_(&nv04_ram_func, fb, type, size, 0, pram);
+ return nvkm_ram_new_(&nv04_ram_func, fb, type, size, pram);
}
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramnv10.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramnv10.c
index dfd155c98dbb..afe54e323b18 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramnv10.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramnv10.c
@@ -36,5 +36,5 @@ nv10_ram_new(struct nvkm_fb *fb, struct nvkm_ram **pram)
else
type = NVKM_RAM_TYPE_SDRAM;
- return nvkm_ram_new_(&nv04_ram_func, fb, type, size, 0, pram);
+ return nvkm_ram_new_(&nv04_ram_func, fb, type, size, pram);
}
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramnv1a.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramnv1a.c
index 3c6a8710e812..4c07d10bb976 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramnv1a.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramnv1a.c
@@ -44,5 +44,5 @@ nv1a_ram_new(struct nvkm_fb *fb, struct nvkm_ram **pram)
}
return nvkm_ram_new_(&nv04_ram_func, fb, NVKM_RAM_TYPE_STOLEN,
- mib * 1024 * 1024, 0, pram);
+ mib * 1024 * 1024, pram);
}
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramnv20.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramnv20.c
index 747e47c10cc7..71d63d7daa75 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramnv20.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramnv20.c
@@ -29,7 +29,6 @@ nv20_ram_new(struct nvkm_fb *fb, struct nvkm_ram **pram)
struct nvkm_device *device = fb->subdev.device;
u32 pbus1218 = nvkm_rd32(device, 0x001218);
u32 size = (nvkm_rd32(device, 0x10020c) & 0xff000000);
- u32 tags = nvkm_rd32(device, 0x100320);
enum nvkm_ram_type type = NVKM_RAM_TYPE_UNKNOWN;
int ret;
@@ -40,7 +39,7 @@ nv20_ram_new(struct nvkm_fb *fb, struct nvkm_ram **pram)
case 0x00000300: type = NVKM_RAM_TYPE_GDDR2; break;
}
- ret = nvkm_ram_new_(&nv04_ram_func, fb, type, size, tags, pram);
+ ret = nvkm_ram_new_(&nv04_ram_func, fb, type, size, pram);
if (ret)
return ret;
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramnv40.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramnv40.c
index 70c63535d56b..2b12e388f47a 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramnv40.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramnv40.c
@@ -187,13 +187,13 @@ nv40_ram_func = {
int
nv40_ram_new_(struct nvkm_fb *fb, enum nvkm_ram_type type, u64 size,
- u32 tags, struct nvkm_ram **pram)
+ struct nvkm_ram **pram)
{
struct nv40_ram *ram;
if (!(ram = kzalloc(sizeof(*ram), GFP_KERNEL)))
return -ENOMEM;
*pram = &ram->base;
- return nvkm_ram_ctor(&nv40_ram_func, fb, type, size, tags, &ram->base);
+ return nvkm_ram_ctor(&nv40_ram_func, fb, type, size, &ram->base);
}
int
@@ -202,7 +202,6 @@ nv40_ram_new(struct nvkm_fb *fb, struct nvkm_ram **pram)
struct nvkm_device *device = fb->subdev.device;
u32 pbus1218 = nvkm_rd32(device, 0x001218);
u32 size = nvkm_rd32(device, 0x10020c) & 0xff000000;
- u32 tags = nvkm_rd32(device, 0x100320);
enum nvkm_ram_type type = NVKM_RAM_TYPE_UNKNOWN;
int ret;
@@ -213,7 +212,7 @@ nv40_ram_new(struct nvkm_fb *fb, struct nvkm_ram **pram)
case 0x00000300: type = NVKM_RAM_TYPE_DDR2 ; break;
}
- ret = nv40_ram_new_(fb, type, size, tags, pram);
+ ret = nv40_ram_new_(fb, type, size, pram);
if (ret)
return ret;
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramnv40.h b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramnv40.h
index 8a0524566b48..ec5dcbfcaea8 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramnv40.h
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramnv40.h
@@ -9,6 +9,6 @@ struct nv40_ram {
u32 coef;
};
-int nv40_ram_new_(struct nvkm_fb *fb, enum nvkm_ram_type, u64, u32,
+int nv40_ram_new_(struct nvkm_fb *fb, enum nvkm_ram_type, u64,
struct nvkm_ram **);
#endif
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramnv41.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramnv41.c
index 114828be292e..d3fea3726461 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramnv41.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramnv41.c
@@ -28,7 +28,6 @@ nv41_ram_new(struct nvkm_fb *fb, struct nvkm_ram **pram)
{
struct nvkm_device *device = fb->subdev.device;
u32 size = nvkm_rd32(device, 0x10020c) & 0xff000000;
- u32 tags = nvkm_rd32(device, 0x100320);
u32 fb474 = nvkm_rd32(device, 0x100474);
enum nvkm_ram_type type = NVKM_RAM_TYPE_UNKNOWN;
int ret;
@@ -40,7 +39,7 @@ nv41_ram_new(struct nvkm_fb *fb, struct nvkm_ram **pram)
if (fb474 & 0x00000001)
type = NVKM_RAM_TYPE_DDR1;
- ret = nv40_ram_new_(fb, type, size, tags, pram);
+ ret = nv40_ram_new_(fb, type, size, pram);
if (ret)
return ret;
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramnv44.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramnv44.c
index bc56fbf1c788..ab2630e5e6fb 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramnv44.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramnv44.c
@@ -38,5 +38,5 @@ nv44_ram_new(struct nvkm_fb *fb, struct nvkm_ram **pram)
if (fb474 & 0x00000001)
type = NVKM_RAM_TYPE_DDR1;
- return nv40_ram_new_(fb, type, size, 0, pram);
+ return nv40_ram_new_(fb, type, size, pram);
}
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramnv49.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramnv49.c
index c01f4b1022b8..946ca7c2e0b6 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramnv49.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramnv49.c
@@ -28,7 +28,6 @@ nv49_ram_new(struct nvkm_fb *fb, struct nvkm_ram **pram)
{
struct nvkm_device *device = fb->subdev.device;
u32 size = nvkm_rd32(device, 0x10020c) & 0xff000000;
- u32 tags = nvkm_rd32(device, 0x100320);
u32 fb914 = nvkm_rd32(device, 0x100914);
enum nvkm_ram_type type = NVKM_RAM_TYPE_UNKNOWN;
int ret;
@@ -40,7 +39,7 @@ nv49_ram_new(struct nvkm_fb *fb, struct nvkm_ram **pram)
case 0x00000003: break;
}
- ret = nv40_ram_new_(fb, type, size, tags, pram);
+ ret = nv40_ram_new_(fb, type, size, pram);
if (ret)
return ret;
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramnv4e.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramnv4e.c
index fa3c2e06203d..02b8bdbc819f 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramnv4e.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramnv4e.c
@@ -29,5 +29,5 @@ nv4e_ram_new(struct nvkm_fb *fb, struct nvkm_ram **pram)
struct nvkm_device *device = fb->subdev.device;
u32 size = nvkm_rd32(device, 0x10020c) & 0xff000000;
return nvkm_ram_new_(&nv04_ram_func, fb, NVKM_RAM_TYPE_UNKNOWN,
- size, 0, pram);
+ size, pram);
}
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramnv50.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramnv50.c
index 6549b0588309..2ccb4b6be153 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramnv50.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramnv50.c
@@ -493,100 +493,8 @@ nv50_ram_tidy(struct nvkm_ram *base)
ram_exec(&ram->hwsq, false);
}
-void
-__nv50_ram_put(struct nvkm_ram *ram, struct nvkm_mem *mem)
-{
- struct nvkm_mm_node *next = mem->mem;
- struct nvkm_mm_node *node;
- while ((node = next)) {
- next = node->next;
- nvkm_mm_free(&ram->vram, &node);
- }
- nvkm_mm_free(&ram->tags, &mem->tag);
-}
-
-void
-nv50_ram_put(struct nvkm_ram *ram, struct nvkm_mem **pmem)
-{
- struct nvkm_mem *mem = *pmem;
-
- *pmem = NULL;
- if (unlikely(mem == NULL))
- return;
-
- mutex_lock(&ram->fb->subdev.mutex);
- __nv50_ram_put(ram, mem);
- mutex_unlock(&ram->fb->subdev.mutex);
-
- kfree(mem);
-}
-
-int
-nv50_ram_get(struct nvkm_ram *ram, u64 size, u32 align, u32 ncmin,
- u32 memtype, struct nvkm_mem **pmem)
-{
- struct nvkm_mm *heap = &ram->vram;
- struct nvkm_mm *tags = &ram->tags;
- struct nvkm_mm_node **node, *r;
- struct nvkm_mem *mem;
- int comp = (memtype & 0x300) >> 8;
- int type = (memtype & 0x07f);
- int back = (memtype & 0x800);
- int min, max, ret;
-
- max = (size >> NVKM_RAM_MM_SHIFT);
- min = ncmin ? (ncmin >> NVKM_RAM_MM_SHIFT) : max;
- align >>= NVKM_RAM_MM_SHIFT;
-
- mem = kzalloc(sizeof(*mem), GFP_KERNEL);
- if (!mem)
- return -ENOMEM;
-
- mutex_lock(&ram->fb->subdev.mutex);
- if (comp) {
- if (align == (1 << (16 - NVKM_RAM_MM_SHIFT))) {
- int n = (max >> 4) * comp;
-
- ret = nvkm_mm_head(tags, 0, 1, n, n, 1, &mem->tag);
- if (ret)
- mem->tag = NULL;
- }
-
- if (unlikely(!mem->tag))
- comp = 0;
- }
-
- mem->memtype = (comp << 7) | type;
- mem->size = max;
-
- type = nv50_fb_memtype[type];
- node = &mem->mem;
- do {
- if (back)
- ret = nvkm_mm_tail(heap, 0, type, max, min, align, &r);
- else
- ret = nvkm_mm_head(heap, 0, type, max, min, align, &r);
- if (ret) {
- mutex_unlock(&ram->fb->subdev.mutex);
- ram->func->put(ram, &mem);
- return ret;
- }
-
- *node = r;
- node = &r->next;
- max -= r->length;
- } while (max);
- mutex_unlock(&ram->fb->subdev.mutex);
-
- mem->offset = (u64)mem->mem->offset << NVKM_RAM_MM_SHIFT;
- *pmem = mem;
- return 0;
-}
-
static const struct nvkm_ram_func
nv50_ram_func = {
- .get = nv50_ram_get,
- .put = nv50_ram_put,
.calc = nv50_ram_calc,
.prog = nv50_ram_prog,
.tidy = nv50_ram_tidy,
@@ -639,7 +547,6 @@ nv50_ram_ctor(const struct nvkm_ram_func *func,
const u32 rsvd_head = ( 256 * 1024); /* vga memory */
const u32 rsvd_tail = (1024 * 1024); /* vbios etc */
u64 size = nvkm_rd32(device, 0x10020c);
- u32 tags = nvkm_rd32(device, 0x100320);
enum nvkm_ram_type type = NVKM_RAM_TYPE_UNKNOWN;
int ret;
@@ -660,7 +567,7 @@ nv50_ram_ctor(const struct nvkm_ram_func *func,
size = (size & 0x000000ff) << 32 | (size & 0xffffff00);
- ret = nvkm_ram_ctor(func, fb, type, size, tags, ram);
+ ret = nvkm_ram_ctor(func, fb, type, size, ram);
if (ret)
return ret;
@@ -669,7 +576,8 @@ nv50_ram_ctor(const struct nvkm_ram_func *func,
ram->ranks = (nvkm_rd32(device, 0x100200) & 0x4) ? 2 : 1;
nvkm_mm_fini(&ram->vram);
- return nvkm_mm_init(&ram->vram, rsvd_head >> NVKM_RAM_MM_SHIFT,
+ return nvkm_mm_init(&ram->vram, NVKM_RAM_MM_NORMAL,
+ rsvd_head >> NVKM_RAM_MM_SHIFT,
(size - rsvd_head - rsvd_tail) >> NVKM_RAM_MM_SHIFT,
nv50_fb_vram_rblock(ram) >> NVKM_RAM_MM_SHIFT);
}
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/base.c b/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/base.c
index 10c987a654ec..364ea4492acc 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/base.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/base.c
@@ -23,181 +23,90 @@
*/
#include "priv.h"
-#include <core/memory.h>
#include <subdev/bar.h>
/******************************************************************************
* instmem object base implementation
*****************************************************************************/
-#define nvkm_instobj(p) container_of((p), struct nvkm_instobj, memory)
-
-struct nvkm_instobj {
- struct nvkm_memory memory;
- struct nvkm_memory *parent;
- struct nvkm_instmem *imem;
- struct list_head head;
- u32 *suspend;
- void __iomem *map;
-};
-
-static enum nvkm_memory_target
-nvkm_instobj_target(struct nvkm_memory *memory)
-{
- memory = nvkm_instobj(memory)->parent;
- return nvkm_memory_target(memory);
-}
-
-static u64
-nvkm_instobj_addr(struct nvkm_memory *memory)
-{
- memory = nvkm_instobj(memory)->parent;
- return nvkm_memory_addr(memory);
-}
-
-static u64
-nvkm_instobj_size(struct nvkm_memory *memory)
-{
- memory = nvkm_instobj(memory)->parent;
- return nvkm_memory_size(memory);
-}
-
static void
-nvkm_instobj_release(struct nvkm_memory *memory)
+nvkm_instobj_load(struct nvkm_instobj *iobj)
{
- struct nvkm_instobj *iobj = nvkm_instobj(memory);
- nvkm_bar_flush(iobj->imem->subdev.device->bar);
-}
-
-static void __iomem *
-nvkm_instobj_acquire(struct nvkm_memory *memory)
-{
- return nvkm_instobj(memory)->map;
-}
-
-static u32
-nvkm_instobj_rd32(struct nvkm_memory *memory, u64 offset)
-{
- return ioread32_native(nvkm_instobj(memory)->map + offset);
-}
-
-static void
-nvkm_instobj_wr32(struct nvkm_memory *memory, u64 offset, u32 data)
-{
- iowrite32_native(data, nvkm_instobj(memory)->map + offset);
-}
+ struct nvkm_memory *memory = &iobj->memory;
+ const u64 size = nvkm_memory_size(memory);
+ void __iomem *map;
+ int i;
-static void
-nvkm_instobj_map(struct nvkm_memory *memory, struct nvkm_vma *vma, u64 offset)
-{
- memory = nvkm_instobj(memory)->parent;
- nvkm_memory_map(memory, vma, offset);
-}
+ if (!(map = nvkm_kmap(memory))) {
+ for (i = 0; i < size; i += 4)
+ nvkm_wo32(memory, i, iobj->suspend[i / 4]);
+ } else {
+ memcpy_toio(map, iobj->suspend, size);
+ }
+ nvkm_done(memory);
-static void *
-nvkm_instobj_dtor(struct nvkm_memory *memory)
-{
- struct nvkm_instobj *iobj = nvkm_instobj(memory);
- spin_lock(&iobj->imem->lock);
- list_del(&iobj->head);
- spin_unlock(&iobj->imem->lock);
- nvkm_memory_del(&iobj->parent);
- return iobj;
+ kvfree(iobj->suspend);
+ iobj->suspend = NULL;
}
-static const struct nvkm_memory_func
-nvkm_instobj_func = {
- .dtor = nvkm_instobj_dtor,
- .target = nvkm_instobj_target,
- .addr = nvkm_instobj_addr,
- .size = nvkm_instobj_size,
- .acquire = nvkm_instobj_acquire,
- .release = nvkm_instobj_release,
- .rd32 = nvkm_instobj_rd32,
- .wr32 = nvkm_instobj_wr32,
- .map = nvkm_instobj_map,
-};
-
-static void
-nvkm_instobj_boot(struct nvkm_memory *memory, struct nvkm_vm *vm)
+static int
+nvkm_instobj_save(struct nvkm_instobj *iobj)
{
- memory = nvkm_instobj(memory)->parent;
- nvkm_memory_boot(memory, vm);
-}
+ struct nvkm_memory *memory = &iobj->memory;
+ const u64 size = nvkm_memory_size(memory);
+ void __iomem *map;
+ int i;
-static void
-nvkm_instobj_release_slow(struct nvkm_memory *memory)
-{
- struct nvkm_instobj *iobj = nvkm_instobj(memory);
- nvkm_instobj_release(memory);
- nvkm_done(iobj->parent);
-}
+ iobj->suspend = kvmalloc(size, GFP_KERNEL);
+ if (!iobj->suspend)
+ return -ENOMEM;
-static void __iomem *
-nvkm_instobj_acquire_slow(struct nvkm_memory *memory)
-{
- struct nvkm_instobj *iobj = nvkm_instobj(memory);
- iobj->map = nvkm_kmap(iobj->parent);
- if (iobj->map)
- memory->func = &nvkm_instobj_func;
- return iobj->map;
+ if (!(map = nvkm_kmap(memory))) {
+ for (i = 0; i < size; i += 4)
+ iobj->suspend[i / 4] = nvkm_ro32(memory, i);
+ } else {
+ memcpy_fromio(iobj->suspend, map, size);
+ }
+ nvkm_done(memory);
+ return 0;
}
-static u32
-nvkm_instobj_rd32_slow(struct nvkm_memory *memory, u64 offset)
+void
+nvkm_instobj_dtor(struct nvkm_instmem *imem, struct nvkm_instobj *iobj)
{
- struct nvkm_instobj *iobj = nvkm_instobj(memory);
- return nvkm_ro32(iobj->parent, offset);
+ spin_lock(&imem->lock);
+ list_del(&iobj->head);
+ spin_unlock(&imem->lock);
}
-static void
-nvkm_instobj_wr32_slow(struct nvkm_memory *memory, u64 offset, u32 data)
+void
+nvkm_instobj_ctor(const struct nvkm_memory_func *func,
+ struct nvkm_instmem *imem, struct nvkm_instobj *iobj)
{
- struct nvkm_instobj *iobj = nvkm_instobj(memory);
- return nvkm_wo32(iobj->parent, offset, data);
+ nvkm_memory_ctor(func, &iobj->memory);
+ iobj->suspend = NULL;
+ spin_lock(&imem->lock);
+ list_add_tail(&iobj->head, &imem->list);
+ spin_unlock(&imem->lock);
}
-static const struct nvkm_memory_func
-nvkm_instobj_func_slow = {
- .dtor = nvkm_instobj_dtor,
- .target = nvkm_instobj_target,
- .addr = nvkm_instobj_addr,
- .size = nvkm_instobj_size,
- .boot = nvkm_instobj_boot,
- .acquire = nvkm_instobj_acquire_slow,
- .release = nvkm_instobj_release_slow,
- .rd32 = nvkm_instobj_rd32_slow,
- .wr32 = nvkm_instobj_wr32_slow,
- .map = nvkm_instobj_map,
-};
-
int
nvkm_instobj_new(struct nvkm_instmem *imem, u32 size, u32 align, bool zero,
struct nvkm_memory **pmemory)
{
+ struct nvkm_subdev *subdev = &imem->subdev;
struct nvkm_memory *memory = NULL;
- struct nvkm_instobj *iobj;
u32 offset;
int ret;
ret = imem->func->memory_new(imem, size, align, zero, &memory);
- if (ret)
+ if (ret) {
+ nvkm_error(subdev, "OOM: %08x %08x %d\n", size, align, ret);
goto done;
-
- if (!imem->func->persistent) {
- if (!(iobj = kzalloc(sizeof(*iobj), GFP_KERNEL))) {
- ret = -ENOMEM;
- goto done;
- }
-
- nvkm_memory_ctor(&nvkm_instobj_func_slow, &iobj->memory);
- iobj->parent = memory;
- iobj->imem = imem;
- spin_lock(&iobj->imem->lock);
- list_add_tail(&iobj->head, &imem->list);
- spin_unlock(&iobj->imem->lock);
- memory = &iobj->memory;
}
+ nvkm_trace(subdev, "new %08x %08x %d: %010llx %010llx\n", size, align,
+ zero, nvkm_memory_addr(memory), nvkm_memory_size(memory));
+
if (!imem->func->zero && zero) {
void __iomem *map = nvkm_kmap(memory);
if (unlikely(!map)) {
@@ -211,7 +120,7 @@ nvkm_instobj_new(struct nvkm_instmem *imem, u32 size, u32 align, bool zero,
done:
if (ret)
- nvkm_memory_del(&memory);
+ nvkm_memory_unref(&memory);
*pmemory = memory;
return ret;
}
@@ -232,39 +141,46 @@ nvkm_instmem_wr32(struct nvkm_instmem *imem, u32 addr, u32 data)
return imem->func->wr32(imem, addr, data);
}
+void
+nvkm_instmem_boot(struct nvkm_instmem *imem)
+{
+ /* Separate bootstrapped objects from normal list, as we need
+ * to make sure they're accessed with the slowpath on suspend
+ * and resume.
+ */
+ struct nvkm_instobj *iobj, *itmp;
+ spin_lock(&imem->lock);
+ list_for_each_entry_safe(iobj, itmp, &imem->list, head) {
+ list_move_tail(&iobj->head, &imem->boot);
+ }
+ spin_unlock(&imem->lock);
+}
+
static int
nvkm_instmem_fini(struct nvkm_subdev *subdev, bool suspend)
{
struct nvkm_instmem *imem = nvkm_instmem(subdev);
struct nvkm_instobj *iobj;
- int i;
-
- if (imem->func->fini)
- imem->func->fini(imem);
if (suspend) {
list_for_each_entry(iobj, &imem->list, head) {
- struct nvkm_memory *memory = iobj->parent;
- u64 size = nvkm_memory_size(memory);
+ int ret = nvkm_instobj_save(iobj);
+ if (ret)
+ return ret;
+ }
- iobj->suspend = vmalloc(size);
- if (!iobj->suspend)
- return -ENOMEM;
+ nvkm_bar_bar2_fini(subdev->device);
- for (i = 0; i < size; i += 4)
- iobj->suspend[i / 4] = nvkm_ro32(memory, i);
+ list_for_each_entry(iobj, &imem->boot, head) {
+ int ret = nvkm_instobj_save(iobj);
+ if (ret)
+ return ret;
}
}
- return 0;
-}
+ if (imem->func->fini)
+ imem->func->fini(imem);
-static int
-nvkm_instmem_oneinit(struct nvkm_subdev *subdev)
-{
- struct nvkm_instmem *imem = nvkm_instmem(subdev);
- if (imem->func->oneinit)
- return imem->func->oneinit(imem);
return 0;
}
@@ -273,22 +189,31 @@ nvkm_instmem_init(struct nvkm_subdev *subdev)
{
struct nvkm_instmem *imem = nvkm_instmem(subdev);
struct nvkm_instobj *iobj;
- int i;
+
+ list_for_each_entry(iobj, &imem->boot, head) {
+ if (iobj->suspend)
+ nvkm_instobj_load(iobj);
+ }
+
+ nvkm_bar_bar2_init(subdev->device);
list_for_each_entry(iobj, &imem->list, head) {
- if (iobj->suspend) {
- struct nvkm_memory *memory = iobj->parent;
- u64 size = nvkm_memory_size(memory);
- for (i = 0; i < size; i += 4)
- nvkm_wo32(memory, i, iobj->suspend[i / 4]);
- vfree(iobj->suspend);
- iobj->suspend = NULL;
- }
+ if (iobj->suspend)
+ nvkm_instobj_load(iobj);
}
return 0;
}
+static int
+nvkm_instmem_oneinit(struct nvkm_subdev *subdev)
+{
+ struct nvkm_instmem *imem = nvkm_instmem(subdev);
+ if (imem->func->oneinit)
+ return imem->func->oneinit(imem);
+ return 0;
+}
+
static void *
nvkm_instmem_dtor(struct nvkm_subdev *subdev)
{
@@ -315,4 +240,5 @@ nvkm_instmem_ctor(const struct nvkm_instmem_func *func,
imem->func = func;
spin_lock_init(&imem->lock);
INIT_LIST_HEAD(&imem->list);
+ INIT_LIST_HEAD(&imem->boot);
}
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/gk20a.c b/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/gk20a.c
index cd5adbec5e57..985f2990ab0d 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/gk20a.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/gk20a.c
@@ -44,14 +44,13 @@
#include "priv.h"
#include <core/memory.h>
-#include <core/mm.h>
#include <core/tegra.h>
-#include <subdev/fb.h>
#include <subdev/ltc.h>
+#include <subdev/mmu.h>
struct gk20a_instobj {
struct nvkm_memory memory;
- struct nvkm_mem mem;
+ struct nvkm_mm_node *mn;
struct gk20a_instmem *imem;
/* CPU mapping */
@@ -119,16 +118,22 @@ gk20a_instobj_target(struct nvkm_memory *memory)
return NVKM_MEM_TARGET_NCOH;
}
+static u8
+gk20a_instobj_page(struct nvkm_memory *memory)
+{
+ return 12;
+}
+
static u64
gk20a_instobj_addr(struct nvkm_memory *memory)
{
- return gk20a_instobj(memory)->mem.offset;
+ return (u64)gk20a_instobj(memory)->mn->offset << 12;
}
static u64
gk20a_instobj_size(struct nvkm_memory *memory)
{
- return (u64)gk20a_instobj(memory)->mem.size << 12;
+ return (u64)gk20a_instobj(memory)->mn->length << 12;
}
/*
@@ -272,12 +277,18 @@ gk20a_instobj_wr32(struct nvkm_memory *memory, u64 offset, u32 data)
node->vaddr[offset / 4] = data;
}
-static void
-gk20a_instobj_map(struct nvkm_memory *memory, struct nvkm_vma *vma, u64 offset)
+static int
+gk20a_instobj_map(struct nvkm_memory *memory, u64 offset, struct nvkm_vmm *vmm,
+ struct nvkm_vma *vma, void *argv, u32 argc)
{
struct gk20a_instobj *node = gk20a_instobj(memory);
+ struct nvkm_vmm_map map = {
+ .memory = &node->memory,
+ .offset = offset,
+ .mem = node->mn,
+ };
- nvkm_vm_map_at(vma, offset, &node->mem);
+ return nvkm_vmm_map(vmm, vma, argv, argc, &map);
}
static void *
@@ -290,8 +301,8 @@ gk20a_instobj_dtor_dma(struct nvkm_memory *memory)
if (unlikely(!node->base.vaddr))
goto out;
- dma_free_attrs(dev, node->base.mem.size << PAGE_SHIFT, node->base.vaddr,
- node->handle, imem->attrs);
+ dma_free_attrs(dev, (u64)node->base.mn->length << PAGE_SHIFT,
+ node->base.vaddr, node->handle, imem->attrs);
out:
return node;
@@ -303,7 +314,7 @@ gk20a_instobj_dtor_iommu(struct nvkm_memory *memory)
struct gk20a_instobj_iommu *node = gk20a_instobj_iommu(memory);
struct gk20a_instmem *imem = node->base.imem;
struct device *dev = imem->base.subdev.device->dev;
- struct nvkm_mm_node *r = node->base.mem.mem;
+ struct nvkm_mm_node *r = node->base.mn;
int i;
if (unlikely(!r))
@@ -321,7 +332,7 @@ gk20a_instobj_dtor_iommu(struct nvkm_memory *memory)
r->offset &= ~BIT(imem->iommu_bit - imem->iommu_pgshift);
/* Unmap pages from GPU address space and free them */
- for (i = 0; i < node->base.mem.size; i++) {
+ for (i = 0; i < node->base.mn->length; i++) {
iommu_unmap(imem->domain,
(r->offset + i) << imem->iommu_pgshift, PAGE_SIZE);
dma_unmap_page(dev, node->dma_addrs[i], PAGE_SIZE,
@@ -342,12 +353,11 @@ static const struct nvkm_memory_func
gk20a_instobj_func_dma = {
.dtor = gk20a_instobj_dtor_dma,
.target = gk20a_instobj_target,
+ .page = gk20a_instobj_page,
.addr = gk20a_instobj_addr,
.size = gk20a_instobj_size,
.acquire = gk20a_instobj_acquire_dma,
.release = gk20a_instobj_release_dma,
- .rd32 = gk20a_instobj_rd32,
- .wr32 = gk20a_instobj_wr32,
.map = gk20a_instobj_map,
};
@@ -355,13 +365,18 @@ static const struct nvkm_memory_func
gk20a_instobj_func_iommu = {
.dtor = gk20a_instobj_dtor_iommu,
.target = gk20a_instobj_target,
+ .page = gk20a_instobj_page,
.addr = gk20a_instobj_addr,
.size = gk20a_instobj_size,
.acquire = gk20a_instobj_acquire_iommu,
.release = gk20a_instobj_release_iommu,
+ .map = gk20a_instobj_map,
+};
+
+static const struct nvkm_memory_ptrs
+gk20a_instobj_ptrs = {
.rd32 = gk20a_instobj_rd32,
.wr32 = gk20a_instobj_wr32,
- .map = gk20a_instobj_map,
};
static int
@@ -377,6 +392,7 @@ gk20a_instobj_ctor_dma(struct gk20a_instmem *imem, u32 npages, u32 align,
*_node = &node->base;
nvkm_memory_ctor(&gk20a_instobj_func_dma, &node->base.memory);
+ node->base.memory.ptrs = &gk20a_instobj_ptrs;
node->base.vaddr = dma_alloc_attrs(dev, npages << PAGE_SHIFT,
&node->handle, GFP_KERNEL,
@@ -397,8 +413,7 @@ gk20a_instobj_ctor_dma(struct gk20a_instmem *imem, u32 npages, u32 align,
node->r.offset = node->handle >> 12;
node->r.length = (npages << PAGE_SHIFT) >> 12;
- node->base.mem.offset = node->handle;
- node->base.mem.mem = &node->r;
+ node->base.mn = &node->r;
return 0;
}
@@ -424,6 +439,7 @@ gk20a_instobj_ctor_iommu(struct gk20a_instmem *imem, u32 npages, u32 align,
node->dma_addrs = (void *)(node->pages + npages);
nvkm_memory_ctor(&gk20a_instobj_func_iommu, &node->base.memory);
+ node->base.memory.ptrs = &gk20a_instobj_ptrs;
/* Allocate backing memory */
for (i = 0; i < npages; i++) {
@@ -474,8 +490,7 @@ gk20a_instobj_ctor_iommu(struct gk20a_instmem *imem, u32 npages, u32 align,
/* IOMMU bit tells that an address is to be resolved through the IOMMU */
r->offset |= BIT(imem->iommu_bit - imem->iommu_pgshift);
- node->base.mem.offset = ((u64)r->offset) << imem->iommu_pgshift;
- node->base.mem.mem = r;
+ node->base.mn = r;
return 0;
release_area:
@@ -523,13 +538,8 @@ gk20a_instobj_new(struct nvkm_instmem *base, u32 size, u32 align, bool zero,
node->imem = imem;
- /* present memory for being mapped using small pages */
- node->mem.size = size >> 12;
- node->mem.memtype = 0;
- node->mem.page_shift = 12;
-
nvkm_debug(subdev, "alloc size: 0x%x, align: 0x%x, gaddr: 0x%llx\n",
- size, align, node->mem.offset);
+ size, align, (u64)node->mn->offset << 12);
return 0;
}
@@ -554,7 +564,6 @@ static const struct nvkm_instmem_func
gk20a_instmem = {
.dtor = gk20a_instmem_dtor,
.memory_new = gk20a_instobj_new,
- .persistent = true,
.zero = false,
};
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/nv04.c b/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/nv04.c
index 6133c8bb2d42..6bf0dad46919 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/nv04.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/nv04.c
@@ -24,7 +24,6 @@
#define nv04_instmem(p) container_of((p), struct nv04_instmem, base)
#include "priv.h"
-#include <core/memory.h>
#include <core/ramht.h>
struct nv04_instmem {
@@ -35,30 +34,39 @@ struct nv04_instmem {
/******************************************************************************
* instmem object implementation
*****************************************************************************/
-#define nv04_instobj(p) container_of((p), struct nv04_instobj, memory)
+#define nv04_instobj(p) container_of((p), struct nv04_instobj, base.memory)
struct nv04_instobj {
- struct nvkm_memory memory;
+ struct nvkm_instobj base;
struct nv04_instmem *imem;
struct nvkm_mm_node *node;
};
-static enum nvkm_memory_target
-nv04_instobj_target(struct nvkm_memory *memory)
+static void
+nv04_instobj_wr32(struct nvkm_memory *memory, u64 offset, u32 data)
{
- return NVKM_MEM_TARGET_INST;
+ struct nv04_instobj *iobj = nv04_instobj(memory);
+ struct nvkm_device *device = iobj->imem->base.subdev.device;
+ nvkm_wr32(device, 0x700000 + iobj->node->offset + offset, data);
}
-static u64
-nv04_instobj_addr(struct nvkm_memory *memory)
+static u32
+nv04_instobj_rd32(struct nvkm_memory *memory, u64 offset)
{
- return nv04_instobj(memory)->node->offset;
+ struct nv04_instobj *iobj = nv04_instobj(memory);
+ struct nvkm_device *device = iobj->imem->base.subdev.device;
+ return nvkm_rd32(device, 0x700000 + iobj->node->offset + offset);
}
-static u64
-nv04_instobj_size(struct nvkm_memory *memory)
+static const struct nvkm_memory_ptrs
+nv04_instobj_ptrs = {
+ .rd32 = nv04_instobj_rd32,
+ .wr32 = nv04_instobj_wr32,
+};
+
+static void
+nv04_instobj_release(struct nvkm_memory *memory)
{
- return nv04_instobj(memory)->node->length;
}
static void __iomem *
@@ -69,25 +77,22 @@ nv04_instobj_acquire(struct nvkm_memory *memory)
return device->pri + 0x700000 + iobj->node->offset;
}
-static void
-nv04_instobj_release(struct nvkm_memory *memory)
+static u64
+nv04_instobj_size(struct nvkm_memory *memory)
{
+ return nv04_instobj(memory)->node->length;
}
-static u32
-nv04_instobj_rd32(struct nvkm_memory *memory, u64 offset)
+static u64
+nv04_instobj_addr(struct nvkm_memory *memory)
{
- struct nv04_instobj *iobj = nv04_instobj(memory);
- struct nvkm_device *device = iobj->imem->base.subdev.device;
- return nvkm_rd32(device, 0x700000 + iobj->node->offset + offset);
+ return nv04_instobj(memory)->node->offset;
}
-static void
-nv04_instobj_wr32(struct nvkm_memory *memory, u64 offset, u32 data)
+static enum nvkm_memory_target
+nv04_instobj_target(struct nvkm_memory *memory)
{
- struct nv04_instobj *iobj = nv04_instobj(memory);
- struct nvkm_device *device = iobj->imem->base.subdev.device;
- nvkm_wr32(device, 0x700000 + iobj->node->offset + offset, data);
+ return NVKM_MEM_TARGET_INST;
}
static void *
@@ -97,6 +102,7 @@ nv04_instobj_dtor(struct nvkm_memory *memory)
mutex_lock(&iobj->imem->base.subdev.mutex);
nvkm_mm_free(&iobj->imem->heap, &iobj->node);
mutex_unlock(&iobj->imem->base.subdev.mutex);
+ nvkm_instobj_dtor(&iobj->imem->base, &iobj->base);
return iobj;
}
@@ -108,8 +114,6 @@ nv04_instobj_func = {
.addr = nv04_instobj_addr,
.acquire = nv04_instobj_acquire,
.release = nv04_instobj_release,
- .rd32 = nv04_instobj_rd32,
- .wr32 = nv04_instobj_wr32,
};
static int
@@ -122,9 +126,10 @@ nv04_instobj_new(struct nvkm_instmem *base, u32 size, u32 align, bool zero,
if (!(iobj = kzalloc(sizeof(*iobj), GFP_KERNEL)))
return -ENOMEM;
- *pmemory = &iobj->memory;
+ *pmemory = &iobj->base.memory;
- nvkm_memory_ctor(&nv04_instobj_func, &iobj->memory);
+ nvkm_instobj_ctor(&nv04_instobj_func, &imem->base, &iobj->base);
+ iobj->base.memory.ptrs = &nv04_instobj_ptrs;
iobj->imem = imem;
mutex_lock(&imem->base.subdev.mutex);
@@ -160,7 +165,7 @@ nv04_instmem_oneinit(struct nvkm_instmem *base)
/* PRAMIN aperture maps over the end of VRAM, reserve it */
imem->base.reserved = 512 * 1024;
- ret = nvkm_mm_init(&imem->heap, 0, imem->base.reserved, 1);
+ ret = nvkm_mm_init(&imem->heap, 0, 0, imem->base.reserved, 1);
if (ret)
return ret;
@@ -194,10 +199,10 @@ static void *
nv04_instmem_dtor(struct nvkm_instmem *base)
{
struct nv04_instmem *imem = nv04_instmem(base);
- nvkm_memory_del(&imem->base.ramfc);
- nvkm_memory_del(&imem->base.ramro);
+ nvkm_memory_unref(&imem->base.ramfc);
+ nvkm_memory_unref(&imem->base.ramro);
nvkm_ramht_del(&imem->base.ramht);
- nvkm_memory_del(&imem->base.vbios);
+ nvkm_memory_unref(&imem->base.vbios);
nvkm_mm_fini(&imem->heap);
return imem;
}
@@ -209,7 +214,6 @@ nv04_instmem = {
.rd32 = nv04_instmem_rd32,
.wr32 = nv04_instmem_wr32,
.memory_new = nv04_instobj_new,
- .persistent = false,
.zero = false,
};
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/nv40.c b/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/nv40.c
index c0543875e490..086c118488ef 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/nv40.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/nv40.c
@@ -24,7 +24,6 @@
#define nv40_instmem(p) container_of((p), struct nv40_instmem, base)
#include "priv.h"
-#include <core/memory.h>
#include <core/ramht.h>
#include <engine/gr/nv40.h>
@@ -37,30 +36,38 @@ struct nv40_instmem {
/******************************************************************************
* instmem object implementation
*****************************************************************************/
-#define nv40_instobj(p) container_of((p), struct nv40_instobj, memory)
+#define nv40_instobj(p) container_of((p), struct nv40_instobj, base.memory)
struct nv40_instobj {
- struct nvkm_memory memory;
+ struct nvkm_instobj base;
struct nv40_instmem *imem;
struct nvkm_mm_node *node;
};
-static enum nvkm_memory_target
-nv40_instobj_target(struct nvkm_memory *memory)
+static void
+nv40_instobj_wr32(struct nvkm_memory *memory, u64 offset, u32 data)
{
- return NVKM_MEM_TARGET_INST;
+ struct nv40_instobj *iobj = nv40_instobj(memory);
+ iowrite32_native(data, iobj->imem->iomem + iobj->node->offset + offset);
}
-static u64
-nv40_instobj_addr(struct nvkm_memory *memory)
+static u32
+nv40_instobj_rd32(struct nvkm_memory *memory, u64 offset)
{
- return nv40_instobj(memory)->node->offset;
+ struct nv40_instobj *iobj = nv40_instobj(memory);
+ return ioread32_native(iobj->imem->iomem + iobj->node->offset + offset);
}
-static u64
-nv40_instobj_size(struct nvkm_memory *memory)
+static const struct nvkm_memory_ptrs
+nv40_instobj_ptrs = {
+ .rd32 = nv40_instobj_rd32,
+ .wr32 = nv40_instobj_wr32,
+};
+
+static void
+nv40_instobj_release(struct nvkm_memory *memory)
{
- return nv40_instobj(memory)->node->length;
+ wmb();
}
static void __iomem *
@@ -70,23 +77,22 @@ nv40_instobj_acquire(struct nvkm_memory *memory)
return iobj->imem->iomem + iobj->node->offset;
}
-static void
-nv40_instobj_release(struct nvkm_memory *memory)
+static u64
+nv40_instobj_size(struct nvkm_memory *memory)
{
+ return nv40_instobj(memory)->node->length;
}
-static u32
-nv40_instobj_rd32(struct nvkm_memory *memory, u64 offset)
+static u64
+nv40_instobj_addr(struct nvkm_memory *memory)
{
- struct nv40_instobj *iobj = nv40_instobj(memory);
- return ioread32_native(iobj->imem->iomem + iobj->node->offset + offset);
+ return nv40_instobj(memory)->node->offset;
}
-static void
-nv40_instobj_wr32(struct nvkm_memory *memory, u64 offset, u32 data)
+static enum nvkm_memory_target
+nv40_instobj_target(struct nvkm_memory *memory)
{
- struct nv40_instobj *iobj = nv40_instobj(memory);
- iowrite32_native(data, iobj->imem->iomem + iobj->node->offset + offset);
+ return NVKM_MEM_TARGET_INST;
}
static void *
@@ -96,6 +102,7 @@ nv40_instobj_dtor(struct nvkm_memory *memory)
mutex_lock(&iobj->imem->base.subdev.mutex);
nvkm_mm_free(&iobj->imem->heap, &iobj->node);
mutex_unlock(&iobj->imem->base.subdev.mutex);
+ nvkm_instobj_dtor(&iobj->imem->base, &iobj->base);
return iobj;
}
@@ -107,8 +114,6 @@ nv40_instobj_func = {
.addr = nv40_instobj_addr,
.acquire = nv40_instobj_acquire,
.release = nv40_instobj_release,
- .rd32 = nv40_instobj_rd32,
- .wr32 = nv40_instobj_wr32,
};
static int
@@ -121,9 +126,10 @@ nv40_instobj_new(struct nvkm_instmem *base, u32 size, u32 align, bool zero,
if (!(iobj = kzalloc(sizeof(*iobj), GFP_KERNEL)))
return -ENOMEM;
- *pmemory = &iobj->memory;
+ *pmemory = &iobj->base.memory;
- nvkm_memory_ctor(&nv40_instobj_func, &iobj->memory);
+ nvkm_instobj_ctor(&nv40_instobj_func, &imem->base, &iobj->base);
+ iobj->base.memory.ptrs = &nv40_instobj_ptrs;
iobj->imem = imem;
mutex_lock(&imem->base.subdev.mutex);
@@ -171,7 +177,7 @@ nv40_instmem_oneinit(struct nvkm_instmem *base)
imem->base.reserved += 512 * 1024; /* object storage */
imem->base.reserved = round_up(imem->base.reserved, 4096);
- ret = nvkm_mm_init(&imem->heap, 0, imem->base.reserved, 1);
+ ret = nvkm_mm_init(&imem->heap, 0, 0, imem->base.reserved, 1);
if (ret)
return ret;
@@ -209,10 +215,10 @@ static void *
nv40_instmem_dtor(struct nvkm_instmem *base)
{
struct nv40_instmem *imem = nv40_instmem(base);
- nvkm_memory_del(&imem->base.ramfc);
- nvkm_memory_del(&imem->base.ramro);
+ nvkm_memory_unref(&imem->base.ramfc);
+ nvkm_memory_unref(&imem->base.ramro);
nvkm_ramht_del(&imem->base.ramht);
- nvkm_memory_del(&imem->base.vbios);
+ nvkm_memory_unref(&imem->base.vbios);
nvkm_mm_fini(&imem->heap);
if (imem->iomem)
iounmap(imem->iomem);
@@ -226,7 +232,6 @@ nv40_instmem = {
.rd32 = nv40_instmem_rd32,
.wr32 = nv40_instmem_wr32,
.memory_new = nv40_instobj_new,
- .persistent = false,
.zero = false,
};
@@ -248,8 +253,8 @@ nv40_instmem_new(struct nvkm_device *device, int index,
else
bar = 3;
- imem->iomem = ioremap(device->func->resource_addr(device, bar),
- device->func->resource_size(device, bar));
+ imem->iomem = ioremap_wc(device->func->resource_addr(device, bar),
+ device->func->resource_size(device, bar));
if (!imem->iomem) {
nvkm_error(&imem->base.subdev, "unable to map PRAMIN BAR\n");
return -EFAULT;
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/nv50.c b/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/nv50.c
index 6d512c062ae3..1ba7289684aa 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/nv50.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/nv50.c
@@ -31,147 +31,293 @@
struct nv50_instmem {
struct nvkm_instmem base;
- unsigned long lock_flags;
- spinlock_t lock;
u64 addr;
+
+ /* Mappings that can be evicted when BAR2 space has been exhausted. */
+ struct list_head lru;
};
/******************************************************************************
* instmem object implementation
*****************************************************************************/
-#define nv50_instobj(p) container_of((p), struct nv50_instobj, memory)
+#define nv50_instobj(p) container_of((p), struct nv50_instobj, base.memory)
struct nv50_instobj {
- struct nvkm_memory memory;
+ struct nvkm_instobj base;
struct nv50_instmem *imem;
- struct nvkm_mem *mem;
- struct nvkm_vma bar;
+ struct nvkm_memory *ram;
+ struct nvkm_vma *bar;
+ refcount_t maps;
void *map;
+ struct list_head lru;
};
-static enum nvkm_memory_target
-nv50_instobj_target(struct nvkm_memory *memory)
+static void
+nv50_instobj_wr32_slow(struct nvkm_memory *memory, u64 offset, u32 data)
{
- return NVKM_MEM_TARGET_VRAM;
+ struct nv50_instobj *iobj = nv50_instobj(memory);
+ struct nv50_instmem *imem = iobj->imem;
+ struct nvkm_device *device = imem->base.subdev.device;
+ u64 base = (nvkm_memory_addr(iobj->ram) + offset) & 0xffffff00000ULL;
+ u64 addr = (nvkm_memory_addr(iobj->ram) + offset) & 0x000000fffffULL;
+ unsigned long flags;
+
+ spin_lock_irqsave(&imem->base.lock, flags);
+ if (unlikely(imem->addr != base)) {
+ nvkm_wr32(device, 0x001700, base >> 16);
+ imem->addr = base;
+ }
+ nvkm_wr32(device, 0x700000 + addr, data);
+ spin_unlock_irqrestore(&imem->base.lock, flags);
}
-static u64
-nv50_instobj_addr(struct nvkm_memory *memory)
+static u32
+nv50_instobj_rd32_slow(struct nvkm_memory *memory, u64 offset)
{
- return nv50_instobj(memory)->mem->offset;
+ struct nv50_instobj *iobj = nv50_instobj(memory);
+ struct nv50_instmem *imem = iobj->imem;
+ struct nvkm_device *device = imem->base.subdev.device;
+ u64 base = (nvkm_memory_addr(iobj->ram) + offset) & 0xffffff00000ULL;
+ u64 addr = (nvkm_memory_addr(iobj->ram) + offset) & 0x000000fffffULL;
+ u32 data;
+ unsigned long flags;
+
+ spin_lock_irqsave(&imem->base.lock, flags);
+ if (unlikely(imem->addr != base)) {
+ nvkm_wr32(device, 0x001700, base >> 16);
+ imem->addr = base;
+ }
+ data = nvkm_rd32(device, 0x700000 + addr);
+ spin_unlock_irqrestore(&imem->base.lock, flags);
+ return data;
}
-static u64
-nv50_instobj_size(struct nvkm_memory *memory)
+static const struct nvkm_memory_ptrs
+nv50_instobj_slow = {
+ .rd32 = nv50_instobj_rd32_slow,
+ .wr32 = nv50_instobj_wr32_slow,
+};
+
+static void
+nv50_instobj_wr32(struct nvkm_memory *memory, u64 offset, u32 data)
{
- return (u64)nv50_instobj(memory)->mem->size << NVKM_RAM_MM_SHIFT;
+ iowrite32_native(data, nv50_instobj(memory)->map + offset);
}
+static u32
+nv50_instobj_rd32(struct nvkm_memory *memory, u64 offset)
+{
+ return ioread32_native(nv50_instobj(memory)->map + offset);
+}
+
+static const struct nvkm_memory_ptrs
+nv50_instobj_fast = {
+ .rd32 = nv50_instobj_rd32,
+ .wr32 = nv50_instobj_wr32,
+};
+
static void
-nv50_instobj_boot(struct nvkm_memory *memory, struct nvkm_vm *vm)
+nv50_instobj_kmap(struct nv50_instobj *iobj, struct nvkm_vmm *vmm)
{
- struct nv50_instobj *iobj = nv50_instobj(memory);
- struct nvkm_subdev *subdev = &iobj->imem->base.subdev;
+ struct nv50_instmem *imem = iobj->imem;
+ struct nv50_instobj *eobj;
+ struct nvkm_memory *memory = &iobj->base.memory;
+ struct nvkm_subdev *subdev = &imem->base.subdev;
struct nvkm_device *device = subdev->device;
+ struct nvkm_vma *bar = NULL, *ebar;
u64 size = nvkm_memory_size(memory);
- void __iomem *map;
+ void *emap;
int ret;
- iobj->map = ERR_PTR(-ENOMEM);
-
- ret = nvkm_vm_get(vm, size, 12, NV_MEM_ACCESS_RW, &iobj->bar);
- if (ret == 0) {
- map = ioremap(device->func->resource_addr(device, 3) +
- (u32)iobj->bar.offset, size);
- if (map) {
- nvkm_memory_map(memory, &iobj->bar, 0);
- iobj->map = map;
- } else {
- nvkm_warn(subdev, "PRAMIN ioremap failed\n");
- nvkm_vm_put(&iobj->bar);
+ /* Attempt to allocate BAR2 address-space and map the object
+ * into it. The lock has to be dropped while doing this due
+ * to the possibility of recursion for page table allocation.
+ */
+ mutex_unlock(&subdev->mutex);
+ while ((ret = nvkm_vmm_get(vmm, 12, size, &bar))) {
+ /* Evict unused mappings, and keep retrying until we either
+ * succeed,or there's no more objects left on the LRU.
+ */
+ mutex_lock(&subdev->mutex);
+ eobj = list_first_entry_or_null(&imem->lru, typeof(*eobj), lru);
+ if (eobj) {
+ nvkm_debug(subdev, "evict %016llx %016llx @ %016llx\n",
+ nvkm_memory_addr(&eobj->base.memory),
+ nvkm_memory_size(&eobj->base.memory),
+ eobj->bar->addr);
+ list_del_init(&eobj->lru);
+ ebar = eobj->bar;
+ eobj->bar = NULL;
+ emap = eobj->map;
+ eobj->map = NULL;
}
- } else {
- nvkm_warn(subdev, "PRAMIN exhausted\n");
+ mutex_unlock(&subdev->mutex);
+ if (!eobj)
+ break;
+ iounmap(emap);
+ nvkm_vmm_put(vmm, &ebar);
}
+
+ if (ret == 0)
+ ret = nvkm_memory_map(memory, 0, vmm, bar, NULL, 0);
+ mutex_lock(&subdev->mutex);
+ if (ret || iobj->bar) {
+ /* We either failed, or another thread beat us. */
+ mutex_unlock(&subdev->mutex);
+ nvkm_vmm_put(vmm, &bar);
+ mutex_lock(&subdev->mutex);
+ return;
+ }
+
+ /* Make the mapping visible to the host. */
+ iobj->bar = bar;
+ iobj->map = ioremap_wc(device->func->resource_addr(device, 3) +
+ (u32)iobj->bar->addr, size);
+ if (!iobj->map) {
+ nvkm_warn(subdev, "PRAMIN ioremap failed\n");
+ nvkm_vmm_put(vmm, &iobj->bar);
+ }
+}
+
+static int
+nv50_instobj_map(struct nvkm_memory *memory, u64 offset, struct nvkm_vmm *vmm,
+ struct nvkm_vma *vma, void *argv, u32 argc)
+{
+ memory = nv50_instobj(memory)->ram;
+ return nvkm_memory_map(memory, offset, vmm, vma, argv, argc);
}
static void
nv50_instobj_release(struct nvkm_memory *memory)
{
- struct nv50_instmem *imem = nv50_instobj(memory)->imem;
- spin_unlock_irqrestore(&imem->lock, imem->lock_flags);
+ struct nv50_instobj *iobj = nv50_instobj(memory);
+ struct nv50_instmem *imem = iobj->imem;
+ struct nvkm_subdev *subdev = &imem->base.subdev;
+
+ wmb();
+ nvkm_bar_flush(subdev->device->bar);
+
+ if (refcount_dec_and_mutex_lock(&iobj->maps, &subdev->mutex)) {
+ /* Add the now-unused mapping to the LRU instead of directly
+ * unmapping it here, in case we need to map it again later.
+ */
+ if (likely(iobj->lru.next) && iobj->map) {
+ BUG_ON(!list_empty(&iobj->lru));
+ list_add_tail(&iobj->lru, &imem->lru);
+ }
+
+ /* Switch back to NULL accessors when last map is gone. */
+ iobj->base.memory.ptrs = NULL;
+ mutex_unlock(&subdev->mutex);
+ }
}
static void __iomem *
nv50_instobj_acquire(struct nvkm_memory *memory)
{
struct nv50_instobj *iobj = nv50_instobj(memory);
- struct nv50_instmem *imem = iobj->imem;
- struct nvkm_bar *bar = imem->base.subdev.device->bar;
- struct nvkm_vm *vm;
- unsigned long flags;
+ struct nvkm_instmem *imem = &iobj->imem->base;
+ struct nvkm_vmm *vmm;
+ void __iomem *map = NULL;
- if (!iobj->map && (vm = nvkm_bar_kmap(bar)))
- nvkm_memory_boot(memory, vm);
- if (!IS_ERR_OR_NULL(iobj->map))
+ /* Already mapped? */
+ if (refcount_inc_not_zero(&iobj->maps))
return iobj->map;
- spin_lock_irqsave(&imem->lock, flags);
- imem->lock_flags = flags;
- return NULL;
-}
+ /* Take the lock, and re-check that another thread hasn't
+ * already mapped the object in the meantime.
+ */
+ mutex_lock(&imem->subdev.mutex);
+ if (refcount_inc_not_zero(&iobj->maps)) {
+ mutex_unlock(&imem->subdev.mutex);
+ return iobj->map;
+ }
-static u32
-nv50_instobj_rd32(struct nvkm_memory *memory, u64 offset)
-{
- struct nv50_instobj *iobj = nv50_instobj(memory);
- struct nv50_instmem *imem = iobj->imem;
- struct nvkm_device *device = imem->base.subdev.device;
- u64 base = (iobj->mem->offset + offset) & 0xffffff00000ULL;
- u64 addr = (iobj->mem->offset + offset) & 0x000000fffffULL;
- u32 data;
+ /* Attempt to get a direct CPU mapping of the object. */
+ if ((vmm = nvkm_bar_bar2_vmm(imem->subdev.device))) {
+ if (!iobj->map)
+ nv50_instobj_kmap(iobj, vmm);
+ map = iobj->map;
+ }
- if (unlikely(imem->addr != base)) {
- nvkm_wr32(device, 0x001700, base >> 16);
- imem->addr = base;
+ if (!refcount_inc_not_zero(&iobj->maps)) {
+ /* Exclude object from eviction while it's being accessed. */
+ if (likely(iobj->lru.next))
+ list_del_init(&iobj->lru);
+
+ if (map)
+ iobj->base.memory.ptrs = &nv50_instobj_fast;
+ else
+ iobj->base.memory.ptrs = &nv50_instobj_slow;
+ refcount_inc(&iobj->maps);
}
- data = nvkm_rd32(device, 0x700000 + addr);
- return data;
+
+ mutex_unlock(&imem->subdev.mutex);
+ return map;
}
static void
-nv50_instobj_wr32(struct nvkm_memory *memory, u64 offset, u32 data)
+nv50_instobj_boot(struct nvkm_memory *memory, struct nvkm_vmm *vmm)
{
struct nv50_instobj *iobj = nv50_instobj(memory);
- struct nv50_instmem *imem = iobj->imem;
- struct nvkm_device *device = imem->base.subdev.device;
- u64 base = (iobj->mem->offset + offset) & 0xffffff00000ULL;
- u64 addr = (iobj->mem->offset + offset) & 0x000000fffffULL;
-
- if (unlikely(imem->addr != base)) {
- nvkm_wr32(device, 0x001700, base >> 16);
- imem->addr = base;
+ struct nvkm_instmem *imem = &iobj->imem->base;
+
+ /* Exclude bootstrapped objects (ie. the page tables for the
+ * instmem BAR itself) from eviction.
+ */
+ mutex_lock(&imem->subdev.mutex);
+ if (likely(iobj->lru.next)) {
+ list_del_init(&iobj->lru);
+ iobj->lru.next = NULL;
}
- nvkm_wr32(device, 0x700000 + addr, data);
+
+ nv50_instobj_kmap(iobj, vmm);
+ nvkm_instmem_boot(imem);
+ mutex_unlock(&imem->subdev.mutex);
}
-static void
-nv50_instobj_map(struct nvkm_memory *memory, struct nvkm_vma *vma, u64 offset)
+static u64
+nv50_instobj_size(struct nvkm_memory *memory)
{
- struct nv50_instobj *iobj = nv50_instobj(memory);
- nvkm_vm_map_at(vma, offset, iobj->mem);
+ return nvkm_memory_size(nv50_instobj(memory)->ram);
+}
+
+static u64
+nv50_instobj_addr(struct nvkm_memory *memory)
+{
+ return nvkm_memory_addr(nv50_instobj(memory)->ram);
+}
+
+static enum nvkm_memory_target
+nv50_instobj_target(struct nvkm_memory *memory)
+{
+ return nvkm_memory_target(nv50_instobj(memory)->ram);
}
static void *
nv50_instobj_dtor(struct nvkm_memory *memory)
{
struct nv50_instobj *iobj = nv50_instobj(memory);
- struct nvkm_ram *ram = iobj->imem->base.subdev.device->fb->ram;
- if (!IS_ERR_OR_NULL(iobj->map)) {
- nvkm_vm_put(&iobj->bar);
- iounmap(iobj->map);
+ struct nvkm_instmem *imem = &iobj->imem->base;
+ struct nvkm_vma *bar;
+ void *map = map;
+
+ mutex_lock(&imem->subdev.mutex);
+ if (likely(iobj->lru.next))
+ list_del(&iobj->lru);
+ map = iobj->map;
+ bar = iobj->bar;
+ mutex_unlock(&imem->subdev.mutex);
+
+ if (map) {
+ struct nvkm_vmm *vmm = nvkm_bar_bar2_vmm(imem->subdev.device);
+ iounmap(map);
+ if (likely(vmm)) /* Can be NULL during BAR destructor. */
+ nvkm_vmm_put(vmm, &bar);
}
- ram->func->put(ram, &iobj->mem);
+
+ nvkm_memory_unref(&iobj->ram);
+ nvkm_instobj_dtor(imem, &iobj->base);
return iobj;
}
@@ -184,8 +330,6 @@ nv50_instobj_func = {
.boot = nv50_instobj_boot,
.acquire = nv50_instobj_acquire,
.release = nv50_instobj_release,
- .rd32 = nv50_instobj_rd32,
- .wr32 = nv50_instobj_wr32,
.map = nv50_instobj_map,
};
@@ -195,25 +339,19 @@ nv50_instobj_new(struct nvkm_instmem *base, u32 size, u32 align, bool zero,
{
struct nv50_instmem *imem = nv50_instmem(base);
struct nv50_instobj *iobj;
- struct nvkm_ram *ram = imem->base.subdev.device->fb->ram;
- int ret;
+ struct nvkm_device *device = imem->base.subdev.device;
+ u8 page = max(order_base_2(align), 12);
if (!(iobj = kzalloc(sizeof(*iobj), GFP_KERNEL)))
return -ENOMEM;
- *pmemory = &iobj->memory;
+ *pmemory = &iobj->base.memory;
- nvkm_memory_ctor(&nv50_instobj_func, &iobj->memory);
+ nvkm_instobj_ctor(&nv50_instobj_func, &imem->base, &iobj->base);
iobj->imem = imem;
+ refcount_set(&iobj->maps, 0);
+ INIT_LIST_HEAD(&iobj->lru);
- size = max((size + 4095) & ~4095, (u32)4096);
- align = max((align + 4095) & ~4095, (u32)4096);
-
- ret = ram->func->get(ram, size, align, 0, 0x800, &iobj->mem);
- if (ret)
- return ret;
-
- iobj->mem->page_shift = 12;
- return 0;
+ return nvkm_ram_get(device, 0, 1, page, size, true, true, &iobj->ram);
}
/******************************************************************************
@@ -230,7 +368,6 @@ static const struct nvkm_instmem_func
nv50_instmem = {
.fini = nv50_instmem_fini,
.memory_new = nv50_instobj_new,
- .persistent = false,
.zero = false,
};
@@ -243,7 +380,7 @@ nv50_instmem_new(struct nvkm_device *device, int index,
if (!(imem = kzalloc(sizeof(*imem), GFP_KERNEL)))
return -ENOMEM;
nvkm_instmem_ctor(&nv50_instmem, device, index, &imem->base);
- spin_lock_init(&imem->lock);
+ INIT_LIST_HEAD(&imem->lru);
*pimem = &imem->base;
return 0;
}
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/priv.h b/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/priv.h
index ace4471864a3..44651ca42d52 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/priv.h
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/priv.h
@@ -11,10 +11,22 @@ struct nvkm_instmem_func {
void (*wr32)(struct nvkm_instmem *, u32 addr, u32 data);
int (*memory_new)(struct nvkm_instmem *, u32 size, u32 align,
bool zero, struct nvkm_memory **);
- bool persistent;
bool zero;
};
void nvkm_instmem_ctor(const struct nvkm_instmem_func *, struct nvkm_device *,
int index, struct nvkm_instmem *);
+void nvkm_instmem_boot(struct nvkm_instmem *);
+
+#include <core/memory.h>
+
+struct nvkm_instobj {
+ struct nvkm_memory memory;
+ struct list_head head;
+ u32 *suspend;
+};
+
+void nvkm_instobj_ctor(const struct nvkm_memory_func *func,
+ struct nvkm_instmem *, struct nvkm_instobj *);
+void nvkm_instobj_dtor(struct nvkm_instmem *, struct nvkm_instobj *);
#endif
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/ltc/base.c b/drivers/gpu/drm/nouveau/nvkm/subdev/ltc/base.c
index 0c7ef250dcaf..1f185274d3e6 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/ltc/base.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/ltc/base.c
@@ -23,26 +23,12 @@
*/
#include "priv.h"
-#include <subdev/fb.h>
-
-int
-nvkm_ltc_tags_alloc(struct nvkm_ltc *ltc, u32 n, struct nvkm_mm_node **pnode)
-{
- int ret = nvkm_mm_head(&ltc->tags, 0, 1, n, n, 1, pnode);
- if (ret)
- *pnode = NULL;
- return ret;
-}
-
-void
-nvkm_ltc_tags_free(struct nvkm_ltc *ltc, struct nvkm_mm_node **pnode)
-{
- nvkm_mm_free(&ltc->tags, pnode);
-}
+#include <core/memory.h>
void
-nvkm_ltc_tags_clear(struct nvkm_ltc *ltc, u32 first, u32 count)
+nvkm_ltc_tags_clear(struct nvkm_device *device, u32 first, u32 count)
{
+ struct nvkm_ltc *ltc = device->ltc;
const u32 limit = first + count - 1;
BUG_ON((first > limit) || (limit >= ltc->num_tags));
@@ -116,10 +102,7 @@ static void *
nvkm_ltc_dtor(struct nvkm_subdev *subdev)
{
struct nvkm_ltc *ltc = nvkm_ltc(subdev);
- struct nvkm_ram *ram = ltc->subdev.device->fb->ram;
- nvkm_mm_fini(&ltc->tags);
- if (ram)
- nvkm_mm_free(&ram->vram, &ltc->tag_ram);
+ nvkm_memory_unref(&ltc->tag_ram);
return ltc;
}
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/ltc/gf100.c b/drivers/gpu/drm/nouveau/nvkm/subdev/ltc/gf100.c
index 4a0fa0a9b802..a21ef45b8572 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/ltc/gf100.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/ltc/gf100.c
@@ -23,6 +23,7 @@
*/
#include "priv.h"
+#include <core/memory.h>
#include <subdev/fb.h>
#include <subdev/timer.h>
@@ -152,7 +153,10 @@ gf100_ltc_flush(struct nvkm_ltc *ltc)
int
gf100_ltc_oneinit_tag_ram(struct nvkm_ltc *ltc)
{
- struct nvkm_ram *ram = ltc->subdev.device->fb->ram;
+ struct nvkm_device *device = ltc->subdev.device;
+ struct nvkm_fb *fb = device->fb;
+ struct nvkm_ram *ram = fb->ram;
+ u32 bits = (nvkm_rd32(device, 0x100c80) & 0x00001000) ? 16 : 17;
u32 tag_size, tag_margin, tag_align;
int ret;
@@ -164,8 +168,8 @@ gf100_ltc_oneinit_tag_ram(struct nvkm_ltc *ltc)
/* tags for 1/4 of VRAM should be enough (8192/4 per GiB of VRAM) */
ltc->num_tags = (ram->size >> 17) / 4;
- if (ltc->num_tags > (1 << 17))
- ltc->num_tags = 1 << 17; /* we have 17 bits in PTE */
+ if (ltc->num_tags > (1 << bits))
+ ltc->num_tags = 1 << bits; /* we have 16/17 bits in PTE */
ltc->num_tags = (ltc->num_tags + 63) & ~63; /* round up to 64 */
tag_align = ltc->ltc_nr * 0x800;
@@ -181,14 +185,13 @@ gf100_ltc_oneinit_tag_ram(struct nvkm_ltc *ltc)
*/
tag_size = (ltc->num_tags / 64) * 0x6000 + tag_margin;
tag_size += tag_align;
- tag_size = (tag_size + 0xfff) >> 12; /* round up */
- ret = nvkm_mm_tail(&ram->vram, 1, 1, tag_size, tag_size, 1,
- &ltc->tag_ram);
+ ret = nvkm_ram_get(device, NVKM_RAM_MM_NORMAL, 0x01, 12, tag_size,
+ true, true, &ltc->tag_ram);
if (ret) {
ltc->num_tags = 0;
} else {
- u64 tag_base = ((u64)ltc->tag_ram->offset << 12) + tag_margin;
+ u64 tag_base = nvkm_memory_addr(ltc->tag_ram) + tag_margin;
tag_base += tag_align - 1;
do_div(tag_base, tag_align);
@@ -197,7 +200,8 @@ gf100_ltc_oneinit_tag_ram(struct nvkm_ltc *ltc)
}
mm_init:
- return nvkm_mm_init(&ltc->tags, 0, ltc->num_tags, 1);
+ nvkm_mm_fini(&fb->tags);
+ return nvkm_mm_init(&fb->tags, 0, 0, ltc->num_tags, 1);
}
int
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/ltc/gp100.c b/drivers/gpu/drm/nouveau/nvkm/subdev/ltc/gp100.c
index 0bdfb2f40266..e34d42108019 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/ltc/gp100.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/ltc/gp100.c
@@ -45,7 +45,7 @@ gp100_ltc_oneinit(struct nvkm_ltc *ltc)
ltc->ltc_nr = nvkm_rd32(device, 0x12006c);
ltc->lts_nr = nvkm_rd32(device, 0x17e280) >> 28;
/*XXX: tagram allocation - TBD */
- return nvkm_mm_init(&ltc->tags, 0, 0, 1);
+ return 0;
}
static void
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/Kbuild b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/Kbuild
index 012c9db687b2..352a65f9371c 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/Kbuild
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/Kbuild
@@ -3,4 +3,33 @@ nvkm-y += nvkm/subdev/mmu/nv04.o
nvkm-y += nvkm/subdev/mmu/nv41.o
nvkm-y += nvkm/subdev/mmu/nv44.o
nvkm-y += nvkm/subdev/mmu/nv50.o
+nvkm-y += nvkm/subdev/mmu/g84.o
nvkm-y += nvkm/subdev/mmu/gf100.o
+nvkm-y += nvkm/subdev/mmu/gk104.o
+nvkm-y += nvkm/subdev/mmu/gk20a.o
+nvkm-y += nvkm/subdev/mmu/gm200.o
+nvkm-y += nvkm/subdev/mmu/gm20b.o
+nvkm-y += nvkm/subdev/mmu/gp100.o
+nvkm-y += nvkm/subdev/mmu/gp10b.o
+
+nvkm-y += nvkm/subdev/mmu/mem.o
+nvkm-y += nvkm/subdev/mmu/memnv04.o
+nvkm-y += nvkm/subdev/mmu/memnv50.o
+nvkm-y += nvkm/subdev/mmu/memgf100.o
+
+nvkm-y += nvkm/subdev/mmu/vmm.o
+nvkm-y += nvkm/subdev/mmu/vmmnv04.o
+nvkm-y += nvkm/subdev/mmu/vmmnv41.o
+nvkm-y += nvkm/subdev/mmu/vmmnv44.o
+nvkm-y += nvkm/subdev/mmu/vmmnv50.o
+nvkm-y += nvkm/subdev/mmu/vmmgf100.o
+nvkm-y += nvkm/subdev/mmu/vmmgk104.o
+nvkm-y += nvkm/subdev/mmu/vmmgk20a.o
+nvkm-y += nvkm/subdev/mmu/vmmgm200.o
+nvkm-y += nvkm/subdev/mmu/vmmgm20b.o
+nvkm-y += nvkm/subdev/mmu/vmmgp100.o
+nvkm-y += nvkm/subdev/mmu/vmmgp10b.o
+
+nvkm-y += nvkm/subdev/mmu/umem.o
+nvkm-y += nvkm/subdev/mmu/ummu.o
+nvkm-y += nvkm/subdev/mmu/uvmm.o
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/base.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/base.c
index d06ad2c372bf..ee11ccaf0563 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/base.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/base.c
@@ -21,478 +21,367 @@
*
* Authors: Ben Skeggs
*/
-#include "priv.h"
+#include "ummu.h"
+#include "vmm.h"
-#include <core/gpuobj.h>
+#include <subdev/bar.h>
#include <subdev/fb.h>
-void
-nvkm_vm_map_at(struct nvkm_vma *vma, u64 delta, struct nvkm_mem *node)
-{
- struct nvkm_vm *vm = vma->vm;
- struct nvkm_mmu *mmu = vm->mmu;
- struct nvkm_mm_node *r = node->mem;
- int big = vma->node->type != mmu->func->spg_shift;
- u32 offset = vma->node->offset + (delta >> 12);
- u32 bits = vma->node->type - 12;
- u32 pde = (offset >> mmu->func->pgt_bits) - vm->fpde;
- u32 pte = (offset & ((1 << mmu->func->pgt_bits) - 1)) >> bits;
- u32 max = 1 << (mmu->func->pgt_bits - bits);
- u32 end, len;
-
- delta = 0;
- while (r) {
- u64 phys = (u64)r->offset << 12;
- u32 num = r->length >> bits;
-
- while (num) {
- struct nvkm_memory *pgt = vm->pgt[pde].mem[big];
-
- end = (pte + num);
- if (unlikely(end >= max))
- end = max;
- len = end - pte;
-
- mmu->func->map(vma, pgt, node, pte, len, phys, delta);
-
- num -= len;
- pte += len;
- if (unlikely(end >= max)) {
- phys += len << (bits + 12);
- pde++;
- pte = 0;
- }
-
- delta += (u64)len << vma->node->type;
- }
- r = r->next;
- };
+#include <nvif/if500d.h>
+#include <nvif/if900d.h>
- mmu->func->flush(vm);
-}
+struct nvkm_mmu_ptp {
+ struct nvkm_mmu_pt *pt;
+ struct list_head head;
+ u8 shift;
+ u16 mask;
+ u16 free;
+};
static void
-nvkm_vm_map_sg_table(struct nvkm_vma *vma, u64 delta, u64 length,
- struct nvkm_mem *mem)
+nvkm_mmu_ptp_put(struct nvkm_mmu *mmu, bool force, struct nvkm_mmu_pt *pt)
{
- struct nvkm_vm *vm = vma->vm;
- struct nvkm_mmu *mmu = vm->mmu;
- int big = vma->node->type != mmu->func->spg_shift;
- u32 offset = vma->node->offset + (delta >> 12);
- u32 bits = vma->node->type - 12;
- u32 num = length >> vma->node->type;
- u32 pde = (offset >> mmu->func->pgt_bits) - vm->fpde;
- u32 pte = (offset & ((1 << mmu->func->pgt_bits) - 1)) >> bits;
- u32 max = 1 << (mmu->func->pgt_bits - bits);
- unsigned m, sglen;
- u32 end, len;
- int i;
- struct scatterlist *sg;
-
- for_each_sg(mem->sg->sgl, sg, mem->sg->nents, i) {
- struct nvkm_memory *pgt = vm->pgt[pde].mem[big];
- sglen = sg_dma_len(sg) >> PAGE_SHIFT;
-
- end = pte + sglen;
- if (unlikely(end >= max))
- end = max;
- len = end - pte;
-
- for (m = 0; m < len; m++) {
- dma_addr_t addr = sg_dma_address(sg) + (m << PAGE_SHIFT);
-
- mmu->func->map_sg(vma, pgt, mem, pte, 1, &addr);
- num--;
- pte++;
-
- if (num == 0)
- goto finish;
- }
- if (unlikely(end >= max)) {
- pde++;
- pte = 0;
- }
- if (m < sglen) {
- for (; m < sglen; m++) {
- dma_addr_t addr = sg_dma_address(sg) + (m << PAGE_SHIFT);
-
- mmu->func->map_sg(vma, pgt, mem, pte, 1, &addr);
- num--;
- pte++;
- if (num == 0)
- goto finish;
- }
- }
-
+ const int slot = pt->base >> pt->ptp->shift;
+ struct nvkm_mmu_ptp *ptp = pt->ptp;
+
+ /* If there were no free slots in the parent allocation before,
+ * there will be now, so return PTP to the cache.
+ */
+ if (!ptp->free)
+ list_add(&ptp->head, &mmu->ptp.list);
+ ptp->free |= BIT(slot);
+
+ /* If there's no more sub-allocations, destroy PTP. */
+ if (ptp->free == ptp->mask) {
+ nvkm_mmu_ptc_put(mmu, force, &ptp->pt);
+ list_del(&ptp->head);
+ kfree(ptp);
}
-finish:
- mmu->func->flush(vm);
+
+ kfree(pt);
}
-static void
-nvkm_vm_map_sg(struct nvkm_vma *vma, u64 delta, u64 length,
- struct nvkm_mem *mem)
+struct nvkm_mmu_pt *
+nvkm_mmu_ptp_get(struct nvkm_mmu *mmu, u32 size, bool zero)
{
- struct nvkm_vm *vm = vma->vm;
- struct nvkm_mmu *mmu = vm->mmu;
- dma_addr_t *list = mem->pages;
- int big = vma->node->type != mmu->func->spg_shift;
- u32 offset = vma->node->offset + (delta >> 12);
- u32 bits = vma->node->type - 12;
- u32 num = length >> vma->node->type;
- u32 pde = (offset >> mmu->func->pgt_bits) - vm->fpde;
- u32 pte = (offset & ((1 << mmu->func->pgt_bits) - 1)) >> bits;
- u32 max = 1 << (mmu->func->pgt_bits - bits);
- u32 end, len;
-
- while (num) {
- struct nvkm_memory *pgt = vm->pgt[pde].mem[big];
-
- end = (pte + num);
- if (unlikely(end >= max))
- end = max;
- len = end - pte;
-
- mmu->func->map_sg(vma, pgt, mem, pte, len, list);
-
- num -= len;
- pte += len;
- list += len;
- if (unlikely(end >= max)) {
- pde++;
- pte = 0;
+ struct nvkm_mmu_pt *pt;
+ struct nvkm_mmu_ptp *ptp;
+ int slot;
+
+ if (!(pt = kzalloc(sizeof(*pt), GFP_KERNEL)))
+ return NULL;
+
+ ptp = list_first_entry_or_null(&mmu->ptp.list, typeof(*ptp), head);
+ if (!ptp) {
+ /* Need to allocate a new parent to sub-allocate from. */
+ if (!(ptp = kmalloc(sizeof(*ptp), GFP_KERNEL))) {
+ kfree(pt);
+ return NULL;
}
- }
- mmu->func->flush(vm);
-}
+ ptp->pt = nvkm_mmu_ptc_get(mmu, 0x1000, 0x1000, false);
+ if (!ptp->pt) {
+ kfree(ptp);
+ kfree(pt);
+ return NULL;
+ }
-void
-nvkm_vm_map(struct nvkm_vma *vma, struct nvkm_mem *node)
-{
- if (node->sg)
- nvkm_vm_map_sg_table(vma, 0, node->size << 12, node);
- else
- if (node->pages)
- nvkm_vm_map_sg(vma, 0, node->size << 12, node);
- else
- nvkm_vm_map_at(vma, 0, node);
+ ptp->shift = order_base_2(size);
+ slot = nvkm_memory_size(ptp->pt->memory) >> ptp->shift;
+ ptp->mask = (1 << slot) - 1;
+ ptp->free = ptp->mask;
+ list_add(&ptp->head, &mmu->ptp.list);
+ }
+ pt->ptp = ptp;
+ pt->sub = true;
+
+ /* Sub-allocate from parent object, removing PTP from cache
+ * if there's no more free slots left.
+ */
+ slot = __ffs(ptp->free);
+ ptp->free &= ~BIT(slot);
+ if (!ptp->free)
+ list_del(&ptp->head);
+
+ pt->memory = pt->ptp->pt->memory;
+ pt->base = slot << ptp->shift;
+ pt->addr = pt->ptp->pt->addr + pt->base;
+ return pt;
}
-void
-nvkm_vm_unmap_at(struct nvkm_vma *vma, u64 delta, u64 length)
+struct nvkm_mmu_ptc {
+ struct list_head head;
+ struct list_head item;
+ u32 size;
+ u32 refs;
+};
+
+static inline struct nvkm_mmu_ptc *
+nvkm_mmu_ptc_find(struct nvkm_mmu *mmu, u32 size)
{
- struct nvkm_vm *vm = vma->vm;
- struct nvkm_mmu *mmu = vm->mmu;
- int big = vma->node->type != mmu->func->spg_shift;
- u32 offset = vma->node->offset + (delta >> 12);
- u32 bits = vma->node->type - 12;
- u32 num = length >> vma->node->type;
- u32 pde = (offset >> mmu->func->pgt_bits) - vm->fpde;
- u32 pte = (offset & ((1 << mmu->func->pgt_bits) - 1)) >> bits;
- u32 max = 1 << (mmu->func->pgt_bits - bits);
- u32 end, len;
-
- while (num) {
- struct nvkm_memory *pgt = vm->pgt[pde].mem[big];
-
- end = (pte + num);
- if (unlikely(end >= max))
- end = max;
- len = end - pte;
-
- mmu->func->unmap(vma, pgt, pte, len);
-
- num -= len;
- pte += len;
- if (unlikely(end >= max)) {
- pde++;
- pte = 0;
- }
+ struct nvkm_mmu_ptc *ptc;
+
+ list_for_each_entry(ptc, &mmu->ptc.list, head) {
+ if (ptc->size == size)
+ return ptc;
}
- mmu->func->flush(vm);
-}
+ ptc = kmalloc(sizeof(*ptc), GFP_KERNEL);
+ if (ptc) {
+ INIT_LIST_HEAD(&ptc->item);
+ ptc->size = size;
+ ptc->refs = 0;
+ list_add(&ptc->head, &mmu->ptc.list);
+ }
-void
-nvkm_vm_unmap(struct nvkm_vma *vma)
-{
- nvkm_vm_unmap_at(vma, 0, (u64)vma->node->length << 12);
+ return ptc;
}
-static void
-nvkm_vm_unmap_pgt(struct nvkm_vm *vm, int big, u32 fpde, u32 lpde)
+void
+nvkm_mmu_ptc_put(struct nvkm_mmu *mmu, bool force, struct nvkm_mmu_pt **ppt)
{
- struct nvkm_mmu *mmu = vm->mmu;
- struct nvkm_vm_pgd *vpgd;
- struct nvkm_vm_pgt *vpgt;
- struct nvkm_memory *pgt;
- u32 pde;
-
- for (pde = fpde; pde <= lpde; pde++) {
- vpgt = &vm->pgt[pde - vm->fpde];
- if (--vpgt->refcount[big])
- continue;
-
- pgt = vpgt->mem[big];
- vpgt->mem[big] = NULL;
-
- list_for_each_entry(vpgd, &vm->pgd_list, head) {
- mmu->func->map_pgt(vpgd->obj, pde, vpgt->mem);
+ struct nvkm_mmu_pt *pt = *ppt;
+ if (pt) {
+ /* Handle sub-allocated page tables. */
+ if (pt->sub) {
+ mutex_lock(&mmu->ptp.mutex);
+ nvkm_mmu_ptp_put(mmu, force, pt);
+ mutex_unlock(&mmu->ptp.mutex);
+ return;
}
- nvkm_memory_del(&pgt);
+ /* Either cache or free the object. */
+ mutex_lock(&mmu->ptc.mutex);
+ if (pt->ptc->refs < 8 /* Heuristic. */ && !force) {
+ list_add_tail(&pt->head, &pt->ptc->item);
+ pt->ptc->refs++;
+ } else {
+ nvkm_memory_unref(&pt->memory);
+ kfree(pt);
+ }
+ mutex_unlock(&mmu->ptc.mutex);
}
}
-static int
-nvkm_vm_map_pgt(struct nvkm_vm *vm, u32 pde, u32 type)
+struct nvkm_mmu_pt *
+nvkm_mmu_ptc_get(struct nvkm_mmu *mmu, u32 size, u32 align, bool zero)
{
- struct nvkm_mmu *mmu = vm->mmu;
- struct nvkm_vm_pgt *vpgt = &vm->pgt[pde - vm->fpde];
- struct nvkm_vm_pgd *vpgd;
- int big = (type != mmu->func->spg_shift);
- u32 pgt_size;
+ struct nvkm_mmu_ptc *ptc;
+ struct nvkm_mmu_pt *pt;
int ret;
- pgt_size = (1 << (mmu->func->pgt_bits + 12)) >> type;
- pgt_size *= 8;
-
- ret = nvkm_memory_new(mmu->subdev.device, NVKM_MEM_TARGET_INST,
- pgt_size, 0x1000, true, &vpgt->mem[big]);
- if (unlikely(ret))
- return ret;
-
- list_for_each_entry(vpgd, &vm->pgd_list, head) {
- mmu->func->map_pgt(vpgd->obj, pde, vpgt->mem);
+ /* Sub-allocated page table (ie. GP100 LPT). */
+ if (align < 0x1000) {
+ mutex_lock(&mmu->ptp.mutex);
+ pt = nvkm_mmu_ptp_get(mmu, align, zero);
+ mutex_unlock(&mmu->ptp.mutex);
+ return pt;
}
- vpgt->refcount[big]++;
- return 0;
-}
-
-int
-nvkm_vm_get(struct nvkm_vm *vm, u64 size, u32 page_shift, u32 access,
- struct nvkm_vma *vma)
-{
- struct nvkm_mmu *mmu = vm->mmu;
- u32 align = (1 << page_shift) >> 12;
- u32 msize = size >> 12;
- u32 fpde, lpde, pde;
- int ret;
-
- mutex_lock(&vm->mutex);
- ret = nvkm_mm_head(&vm->mm, 0, page_shift, msize, msize, align,
- &vma->node);
- if (unlikely(ret != 0)) {
- mutex_unlock(&vm->mutex);
- return ret;
+ /* Lookup cache for this page table size. */
+ mutex_lock(&mmu->ptc.mutex);
+ ptc = nvkm_mmu_ptc_find(mmu, size);
+ if (!ptc) {
+ mutex_unlock(&mmu->ptc.mutex);
+ return NULL;
}
- fpde = (vma->node->offset >> mmu->func->pgt_bits);
- lpde = (vma->node->offset + vma->node->length - 1) >> mmu->func->pgt_bits;
-
- for (pde = fpde; pde <= lpde; pde++) {
- struct nvkm_vm_pgt *vpgt = &vm->pgt[pde - vm->fpde];
- int big = (vma->node->type != mmu->func->spg_shift);
+ /* If there's a free PT in the cache, reuse it. */
+ pt = list_first_entry_or_null(&ptc->item, typeof(*pt), head);
+ if (pt) {
+ if (zero)
+ nvkm_fo64(pt->memory, 0, 0, size >> 3);
+ list_del(&pt->head);
+ ptc->refs--;
+ mutex_unlock(&mmu->ptc.mutex);
+ return pt;
+ }
+ mutex_unlock(&mmu->ptc.mutex);
- if (likely(vpgt->refcount[big])) {
- vpgt->refcount[big]++;
- continue;
- }
+ /* No such luck, we need to allocate. */
+ if (!(pt = kmalloc(sizeof(*pt), GFP_KERNEL)))
+ return NULL;
+ pt->ptc = ptc;
+ pt->sub = false;
- ret = nvkm_vm_map_pgt(vm, pde, vma->node->type);
- if (ret) {
- if (pde != fpde)
- nvkm_vm_unmap_pgt(vm, big, fpde, pde - 1);
- nvkm_mm_free(&vm->mm, &vma->node);
- mutex_unlock(&vm->mutex);
- return ret;
- }
+ ret = nvkm_memory_new(mmu->subdev.device, NVKM_MEM_TARGET_INST,
+ size, align, zero, &pt->memory);
+ if (ret) {
+ kfree(pt);
+ return NULL;
}
- mutex_unlock(&vm->mutex);
- vma->vm = NULL;
- nvkm_vm_ref(vm, &vma->vm, NULL);
- vma->offset = (u64)vma->node->offset << 12;
- vma->access = access;
- return 0;
+ pt->base = 0;
+ pt->addr = nvkm_memory_addr(pt->memory);
+ return pt;
}
void
-nvkm_vm_put(struct nvkm_vma *vma)
+nvkm_mmu_ptc_dump(struct nvkm_mmu *mmu)
{
- struct nvkm_mmu *mmu;
- struct nvkm_vm *vm;
- u32 fpde, lpde;
-
- if (unlikely(vma->node == NULL))
- return;
- vm = vma->vm;
- mmu = vm->mmu;
-
- fpde = (vma->node->offset >> mmu->func->pgt_bits);
- lpde = (vma->node->offset + vma->node->length - 1) >> mmu->func->pgt_bits;
-
- mutex_lock(&vm->mutex);
- nvkm_vm_unmap_pgt(vm, vma->node->type != mmu->func->spg_shift, fpde, lpde);
- nvkm_mm_free(&vm->mm, &vma->node);
- mutex_unlock(&vm->mutex);
-
- nvkm_vm_ref(NULL, &vma->vm, NULL);
-}
-
-int
-nvkm_vm_boot(struct nvkm_vm *vm, u64 size)
-{
- struct nvkm_mmu *mmu = vm->mmu;
- struct nvkm_memory *pgt;
- int ret;
-
- ret = nvkm_memory_new(mmu->subdev.device, NVKM_MEM_TARGET_INST,
- (size >> mmu->func->spg_shift) * 8, 0x1000, true, &pgt);
- if (ret == 0) {
- vm->pgt[0].refcount[0] = 1;
- vm->pgt[0].mem[0] = pgt;
- nvkm_memory_boot(pgt, vm);
+ struct nvkm_mmu_ptc *ptc;
+ list_for_each_entry(ptc, &mmu->ptc.list, head) {
+ struct nvkm_mmu_pt *pt, *tt;
+ list_for_each_entry_safe(pt, tt, &ptc->item, head) {
+ nvkm_memory_unref(&pt->memory);
+ list_del(&pt->head);
+ kfree(pt);
+ }
}
-
- return ret;
}
-int
-nvkm_vm_create(struct nvkm_mmu *mmu, u64 offset, u64 length, u64 mm_offset,
- u32 block, struct lock_class_key *key, struct nvkm_vm **pvm)
+static void
+nvkm_mmu_ptc_fini(struct nvkm_mmu *mmu)
{
- static struct lock_class_key _key;
- struct nvkm_vm *vm;
- u64 mm_length = (offset + length) - mm_offset;
- int ret;
-
- vm = kzalloc(sizeof(*vm), GFP_KERNEL);
- if (!vm)
- return -ENOMEM;
-
- __mutex_init(&vm->mutex, "&vm->mutex", key ? key : &_key);
- INIT_LIST_HEAD(&vm->pgd_list);
- vm->mmu = mmu;
- kref_init(&vm->refcount);
- vm->fpde = offset >> (mmu->func->pgt_bits + 12);
- vm->lpde = (offset + length - 1) >> (mmu->func->pgt_bits + 12);
-
- vm->pgt = vzalloc((vm->lpde - vm->fpde + 1) * sizeof(*vm->pgt));
- if (!vm->pgt) {
- kfree(vm);
- return -ENOMEM;
- }
+ struct nvkm_mmu_ptc *ptc, *ptct;
- ret = nvkm_mm_init(&vm->mm, mm_offset >> 12, mm_length >> 12,
- block >> 12);
- if (ret) {
- vfree(vm->pgt);
- kfree(vm);
- return ret;
+ list_for_each_entry_safe(ptc, ptct, &mmu->ptc.list, head) {
+ WARN_ON(!list_empty(&ptc->item));
+ list_del(&ptc->head);
+ kfree(ptc);
}
-
- *pvm = vm;
-
- return 0;
}
-int
-nvkm_vm_new(struct nvkm_device *device, u64 offset, u64 length, u64 mm_offset,
- struct lock_class_key *key, struct nvkm_vm **pvm)
+static void
+nvkm_mmu_ptc_init(struct nvkm_mmu *mmu)
{
- struct nvkm_mmu *mmu = device->mmu;
- if (!mmu->func->create)
- return -EINVAL;
- return mmu->func->create(mmu, offset, length, mm_offset, key, pvm);
+ mutex_init(&mmu->ptc.mutex);
+ INIT_LIST_HEAD(&mmu->ptc.list);
+ mutex_init(&mmu->ptp.mutex);
+ INIT_LIST_HEAD(&mmu->ptp.list);
}
-static int
-nvkm_vm_link(struct nvkm_vm *vm, struct nvkm_gpuobj *pgd)
+static void
+nvkm_mmu_type(struct nvkm_mmu *mmu, int heap, u8 type)
{
- struct nvkm_mmu *mmu = vm->mmu;
- struct nvkm_vm_pgd *vpgd;
- int i;
-
- if (!pgd)
- return 0;
-
- vpgd = kzalloc(sizeof(*vpgd), GFP_KERNEL);
- if (!vpgd)
- return -ENOMEM;
-
- vpgd->obj = pgd;
-
- mutex_lock(&vm->mutex);
- for (i = vm->fpde; i <= vm->lpde; i++)
- mmu->func->map_pgt(pgd, i, vm->pgt[i - vm->fpde].mem);
- list_add(&vpgd->head, &vm->pgd_list);
- mutex_unlock(&vm->mutex);
- return 0;
+ if (heap >= 0 && !WARN_ON(mmu->type_nr == ARRAY_SIZE(mmu->type))) {
+ mmu->type[mmu->type_nr].type = type | mmu->heap[heap].type;
+ mmu->type[mmu->type_nr].heap = heap;
+ mmu->type_nr++;
+ }
}
-static void
-nvkm_vm_unlink(struct nvkm_vm *vm, struct nvkm_gpuobj *mpgd)
+static int
+nvkm_mmu_heap(struct nvkm_mmu *mmu, u8 type, u64 size)
{
- struct nvkm_vm_pgd *vpgd, *tmp;
-
- if (!mpgd)
- return;
-
- mutex_lock(&vm->mutex);
- list_for_each_entry_safe(vpgd, tmp, &vm->pgd_list, head) {
- if (vpgd->obj == mpgd) {
- list_del(&vpgd->head);
- kfree(vpgd);
- break;
+ if (size) {
+ if (!WARN_ON(mmu->heap_nr == ARRAY_SIZE(mmu->heap))) {
+ mmu->heap[mmu->heap_nr].type = type;
+ mmu->heap[mmu->heap_nr].size = size;
+ return mmu->heap_nr++;
}
}
- mutex_unlock(&vm->mutex);
+ return -EINVAL;
}
static void
-nvkm_vm_del(struct kref *kref)
+nvkm_mmu_host(struct nvkm_mmu *mmu)
{
- struct nvkm_vm *vm = container_of(kref, typeof(*vm), refcount);
- struct nvkm_vm_pgd *vpgd, *tmp;
-
- list_for_each_entry_safe(vpgd, tmp, &vm->pgd_list, head) {
- nvkm_vm_unlink(vm, vpgd->obj);
- }
-
- nvkm_mm_fini(&vm->mm);
- vfree(vm->pgt);
- kfree(vm);
+ struct nvkm_device *device = mmu->subdev.device;
+ u8 type = NVKM_MEM_KIND * !!mmu->func->kind_sys;
+ int heap;
+
+ /* Non-mappable system memory. */
+ heap = nvkm_mmu_heap(mmu, NVKM_MEM_HOST, ~0ULL);
+ nvkm_mmu_type(mmu, heap, type);
+
+ /* Non-coherent, cached, system memory.
+ *
+ * Block-linear mappings of system memory must be done through
+ * BAR1, and cannot be supported on systems where we're unable
+ * to map BAR1 with write-combining.
+ */
+ type |= NVKM_MEM_MAPPABLE;
+ if (!device->bar || device->bar->iomap_uncached)
+ nvkm_mmu_type(mmu, heap, type & ~NVKM_MEM_KIND);
+ else
+ nvkm_mmu_type(mmu, heap, type);
+
+ /* Coherent, cached, system memory.
+ *
+ * Unsupported on systems that aren't able to support snooped
+ * mappings, and also for block-linear mappings which must be
+ * done through BAR1.
+ */
+ type |= NVKM_MEM_COHERENT;
+ if (device->func->cpu_coherent)
+ nvkm_mmu_type(mmu, heap, type & ~NVKM_MEM_KIND);
+
+ /* Uncached system memory. */
+ nvkm_mmu_type(mmu, heap, type |= NVKM_MEM_UNCACHED);
}
-int
-nvkm_vm_ref(struct nvkm_vm *ref, struct nvkm_vm **ptr, struct nvkm_gpuobj *pgd)
+static void
+nvkm_mmu_vram(struct nvkm_mmu *mmu)
{
- if (ref) {
- int ret = nvkm_vm_link(ref, pgd);
- if (ret)
- return ret;
-
- kref_get(&ref->refcount);
- }
+ struct nvkm_device *device = mmu->subdev.device;
+ struct nvkm_mm *mm = &device->fb->ram->vram;
+ const u32 sizeN = nvkm_mm_heap_size(mm, NVKM_RAM_MM_NORMAL);
+ const u32 sizeU = nvkm_mm_heap_size(mm, NVKM_RAM_MM_NOMAP);
+ const u32 sizeM = nvkm_mm_heap_size(mm, NVKM_RAM_MM_MIXED);
+ u8 type = NVKM_MEM_KIND * !!mmu->func->kind;
+ u8 heap = NVKM_MEM_VRAM;
+ int heapM, heapN, heapU;
+
+ /* Mixed-memory doesn't support compression or display. */
+ heapM = nvkm_mmu_heap(mmu, heap, sizeM << NVKM_RAM_MM_SHIFT);
+
+ heap |= NVKM_MEM_COMP;
+ heap |= NVKM_MEM_DISP;
+ heapN = nvkm_mmu_heap(mmu, heap, sizeN << NVKM_RAM_MM_SHIFT);
+ heapU = nvkm_mmu_heap(mmu, heap, sizeU << NVKM_RAM_MM_SHIFT);
+
+ /* Add non-mappable VRAM types first so that they're preferred
+ * over anything else. Mixed-memory will be slower than other
+ * heaps, it's prioritised last.
+ */
+ nvkm_mmu_type(mmu, heapU, type);
+ nvkm_mmu_type(mmu, heapN, type);
+ nvkm_mmu_type(mmu, heapM, type);
+
+ /* Add host memory types next, under the assumption that users
+ * wanting mappable memory want to use them as staging buffers
+ * or the like.
+ */
+ nvkm_mmu_host(mmu);
+
+ /* Mappable VRAM types go last, as they're basically the worst
+ * possible type to ask for unless there's no other choice.
+ */
+ if (device->bar) {
+ /* Write-combined BAR1 access. */
+ type |= NVKM_MEM_MAPPABLE;
+ if (!device->bar->iomap_uncached) {
+ nvkm_mmu_type(mmu, heapN, type);
+ nvkm_mmu_type(mmu, heapM, type);
+ }
- if (*ptr) {
- nvkm_vm_unlink(*ptr, pgd);
- kref_put(&(*ptr)->refcount, nvkm_vm_del);
+ /* Uncached BAR1 access. */
+ type |= NVKM_MEM_COHERENT;
+ type |= NVKM_MEM_UNCACHED;
+ nvkm_mmu_type(mmu, heapN, type);
+ nvkm_mmu_type(mmu, heapM, type);
}
-
- *ptr = ref;
- return 0;
}
static int
nvkm_mmu_oneinit(struct nvkm_subdev *subdev)
{
struct nvkm_mmu *mmu = nvkm_mmu(subdev);
- if (mmu->func->oneinit)
- return mmu->func->oneinit(mmu);
+
+ /* Determine available memory types. */
+ if (mmu->subdev.device->fb && mmu->subdev.device->fb->ram)
+ nvkm_mmu_vram(mmu);
+ else
+ nvkm_mmu_host(mmu);
+
+ if (mmu->func->vmm.global) {
+ int ret = nvkm_vmm_new(subdev->device, 0, 0, NULL, 0, NULL,
+ "gart", &mmu->vmm);
+ if (ret)
+ return ret;
+ }
+
return 0;
}
@@ -509,8 +398,10 @@ static void *
nvkm_mmu_dtor(struct nvkm_subdev *subdev)
{
struct nvkm_mmu *mmu = nvkm_mmu(subdev);
- if (mmu->func->dtor)
- return mmu->func->dtor(mmu);
+
+ nvkm_vmm_unref(&mmu->vmm);
+
+ nvkm_mmu_ptc_fini(mmu);
return mmu;
}
@@ -527,9 +418,10 @@ nvkm_mmu_ctor(const struct nvkm_mmu_func *func, struct nvkm_device *device,
{
nvkm_subdev_ctor(&nvkm_mmu, device, index, &mmu->subdev);
mmu->func = func;
- mmu->limit = func->limit;
mmu->dma_bits = func->dma_bits;
- mmu->lpg_shift = func->lpg_shift;
+ nvkm_mmu_ptc_init(mmu);
+ mmu->user.ctor = nvkm_ummu_new;
+ mmu->user.base = func->mmu.user;
}
int
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/g84.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/g84.c
new file mode 100644
index 000000000000..8accda5a772b
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/g84.c
@@ -0,0 +1,41 @@
+/*
+ * Copyright 2017 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+#include "mem.h"
+#include "vmm.h"
+
+#include <nvif/class.h>
+
+static const struct nvkm_mmu_func
+g84_mmu = {
+ .dma_bits = 40,
+ .mmu = {{ -1, -1, NVIF_CLASS_MMU_NV50}},
+ .mem = {{ -1, 0, NVIF_CLASS_MEM_NV50}, nv50_mem_new, nv50_mem_map },
+ .vmm = {{ -1, -1, NVIF_CLASS_VMM_NV50}, nv50_vmm_new, false, 0x0200 },
+ .kind = nv50_mmu_kind,
+ .kind_sys = true,
+};
+
+int
+g84_mmu_new(struct nvkm_device *device, int index, struct nvkm_mmu **pmmu)
+{
+ return nvkm_mmu_new_(&g84_mmu, device, index, pmmu);
+}
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gf100.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gf100.c
index 7ac507c927bb..2d075246dc46 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gf100.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gf100.c
@@ -21,197 +21,65 @@
*
* Authors: Ben Skeggs
*/
-#include "priv.h"
+#include "mem.h"
+#include "vmm.h"
-#include <subdev/fb.h>
-#include <subdev/ltc.h>
-#include <subdev/timer.h>
-
-#include <core/gpuobj.h>
+#include <nvif/class.h>
/* Map from compressed to corresponding uncompressed storage type.
* The value 0xff represents an invalid storage type.
*/
-const u8 gf100_pte_storage_type_map[256] =
-{
- 0x00, 0x01, 0x01, 0x01, 0x01, 0x01, 0xff, 0x01, /* 0x00 */
- 0x01, 0x01, 0x01, 0xff, 0xff, 0xff, 0xff, 0xff,
- 0xff, 0x11, 0xff, 0xff, 0xff, 0xff, 0xff, 0x11, /* 0x10 */
- 0x11, 0x11, 0x11, 0xff, 0xff, 0xff, 0xff, 0xff,
- 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x26, 0x27, /* 0x20 */
- 0x28, 0x29, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
- 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x30 */
- 0xff, 0xff, 0x26, 0x27, 0x28, 0x29, 0x26, 0x27,
- 0x28, 0x29, 0xff, 0xff, 0xff, 0xff, 0x46, 0xff, /* 0x40 */
- 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
- 0xff, 0x46, 0x46, 0x46, 0x46, 0xff, 0xff, 0xff, /* 0x50 */
- 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
- 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x60 */
- 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
- 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x70 */
- 0xff, 0xff, 0xff, 0x7b, 0xff, 0xff, 0xff, 0xff,
- 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7b, 0x7b, /* 0x80 */
- 0x7b, 0x7b, 0xff, 0x8b, 0x8c, 0x8d, 0x8e, 0xff,
- 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x90 */
- 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
- 0xff, 0xff, 0xff, 0x8b, 0x8c, 0x8d, 0x8e, 0xa7, /* 0xa0 */
- 0xa8, 0xa9, 0xaa, 0xff, 0xff, 0xff, 0xff, 0xff,
- 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xb0 */
- 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xa7,
- 0xa8, 0xa9, 0xaa, 0xc3, 0xff, 0xff, 0xff, 0xff, /* 0xc0 */
- 0xff, 0xff, 0xff, 0xff, 0xfe, 0xfe, 0xc3, 0xc3,
- 0xc3, 0xc3, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xd0 */
- 0xfe, 0xff, 0xff, 0xfe, 0xff, 0xfe, 0xff, 0xfe,
- 0xfe, 0xff, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xff, /* 0xe0 */
- 0xff, 0xfe, 0xff, 0xfe, 0xff, 0xfe, 0xfe, 0xff,
- 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, /* 0xf0 */
- 0xfe, 0xfe, 0xfe, 0xfe, 0xff, 0xfd, 0xfe, 0xff
-};
-
-
-static void
-gf100_vm_map_pgt(struct nvkm_gpuobj *pgd, u32 index, struct nvkm_memory *pgt[2])
-{
- u32 pde[2] = { 0, 0 };
-
- if (pgt[0])
- pde[1] = 0x00000001 | (nvkm_memory_addr(pgt[0]) >> 8);
- if (pgt[1])
- pde[0] = 0x00000001 | (nvkm_memory_addr(pgt[1]) >> 8);
-
- nvkm_kmap(pgd);
- nvkm_wo32(pgd, (index * 8) + 0, pde[0]);
- nvkm_wo32(pgd, (index * 8) + 4, pde[1]);
- nvkm_done(pgd);
-}
-
-static inline u64
-gf100_vm_addr(struct nvkm_vma *vma, u64 phys, u32 memtype, u32 target)
-{
- phys >>= 8;
-
- phys |= 0x00000001; /* present */
- if (vma->access & NV_MEM_ACCESS_SYS)
- phys |= 0x00000002;
-
- phys |= ((u64)target << 32);
- phys |= ((u64)memtype << 36);
- return phys;
-}
-
-static void
-gf100_vm_map(struct nvkm_vma *vma, struct nvkm_memory *pgt,
- struct nvkm_mem *mem, u32 pte, u32 cnt, u64 phys, u64 delta)
-{
- u64 next = 1 << (vma->node->type - 8);
-
- phys = gf100_vm_addr(vma, phys, mem->memtype, 0);
- pte <<= 3;
-
- if (mem->tag) {
- struct nvkm_ltc *ltc = vma->vm->mmu->subdev.device->ltc;
- u32 tag = mem->tag->offset + (delta >> 17);
- phys |= (u64)tag << (32 + 12);
- next |= (u64)1 << (32 + 12);
- nvkm_ltc_tags_clear(ltc, tag, cnt);
- }
-
- nvkm_kmap(pgt);
- while (cnt--) {
- nvkm_wo32(pgt, pte + 0, lower_32_bits(phys));
- nvkm_wo32(pgt, pte + 4, upper_32_bits(phys));
- phys += next;
- pte += 8;
- }
- nvkm_done(pgt);
-}
-
-static void
-gf100_vm_map_sg(struct nvkm_vma *vma, struct nvkm_memory *pgt,
- struct nvkm_mem *mem, u32 pte, u32 cnt, dma_addr_t *list)
-{
- u32 target = (vma->access & NV_MEM_ACCESS_NOSNOOP) ? 7 : 5;
- /* compressed storage types are invalid for system memory */
- u32 memtype = gf100_pte_storage_type_map[mem->memtype & 0xff];
-
- nvkm_kmap(pgt);
- pte <<= 3;
- while (cnt--) {
- u64 phys = gf100_vm_addr(vma, *list++, memtype, target);
- nvkm_wo32(pgt, pte + 0, lower_32_bits(phys));
- nvkm_wo32(pgt, pte + 4, upper_32_bits(phys));
- pte += 8;
- }
- nvkm_done(pgt);
-}
-
-static void
-gf100_vm_unmap(struct nvkm_vma *vma, struct nvkm_memory *pgt, u32 pte, u32 cnt)
-{
- nvkm_kmap(pgt);
- pte <<= 3;
- while (cnt--) {
- nvkm_wo32(pgt, pte + 0, 0x00000000);
- nvkm_wo32(pgt, pte + 4, 0x00000000);
- pte += 8;
- }
- nvkm_done(pgt);
-}
-
-static void
-gf100_vm_flush(struct nvkm_vm *vm)
-{
- struct nvkm_mmu *mmu = vm->mmu;
- struct nvkm_device *device = mmu->subdev.device;
- struct nvkm_vm_pgd *vpgd;
- u32 type;
-
- type = 0x00000001; /* PAGE_ALL */
- if (atomic_read(&vm->engref[NVKM_SUBDEV_BAR]))
- type |= 0x00000004; /* HUB_ONLY */
-
- mutex_lock(&mmu->subdev.mutex);
- list_for_each_entry(vpgd, &vm->pgd_list, head) {
- /* looks like maybe a "free flush slots" counter, the
- * faster you write to 0x100cbc to more it decreases
- */
- nvkm_msec(device, 2000,
- if (nvkm_rd32(device, 0x100c80) & 0x00ff0000)
- break;
- );
-
- nvkm_wr32(device, 0x100cb8, vpgd->obj->addr >> 8);
- nvkm_wr32(device, 0x100cbc, 0x80000000 | type);
-
- /* wait for flush to be queued? */
- nvkm_msec(device, 2000,
- if (nvkm_rd32(device, 0x100c80) & 0x00008000)
- break;
- );
- }
- mutex_unlock(&mmu->subdev.mutex);
-}
-
-static int
-gf100_vm_create(struct nvkm_mmu *mmu, u64 offset, u64 length, u64 mm_offset,
- struct lock_class_key *key, struct nvkm_vm **pvm)
+const u8 *
+gf100_mmu_kind(struct nvkm_mmu *mmu, int *count)
{
- return nvkm_vm_create(mmu, offset, length, mm_offset, 4096, key, pvm);
+ static const u8
+ kind[256] = {
+ 0x00, 0x01, 0x01, 0x01, 0x01, 0x01, 0xff, 0x01, /* 0x00 */
+ 0x01, 0x01, 0x01, 0xff, 0xff, 0xff, 0xff, 0xff,
+ 0xff, 0x11, 0xff, 0xff, 0xff, 0xff, 0xff, 0x11, /* 0x10 */
+ 0x11, 0x11, 0x11, 0xff, 0xff, 0xff, 0xff, 0xff,
+ 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x26, 0x27, /* 0x20 */
+ 0x28, 0x29, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+ 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x30 */
+ 0xff, 0xff, 0x26, 0x27, 0x28, 0x29, 0x26, 0x27,
+ 0x28, 0x29, 0xff, 0xff, 0xff, 0xff, 0x46, 0xff, /* 0x40 */
+ 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+ 0xff, 0x46, 0x46, 0x46, 0x46, 0xff, 0xff, 0xff, /* 0x50 */
+ 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+ 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x60 */
+ 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+ 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x70 */
+ 0xff, 0xff, 0xff, 0x7b, 0xff, 0xff, 0xff, 0xff,
+ 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7b, 0x7b, /* 0x80 */
+ 0x7b, 0x7b, 0xff, 0x8b, 0x8c, 0x8d, 0x8e, 0xff,
+ 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x90 */
+ 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+ 0xff, 0xff, 0xff, 0x8b, 0x8c, 0x8d, 0x8e, 0xa7, /* 0xa0 */
+ 0xa8, 0xa9, 0xaa, 0xff, 0xff, 0xff, 0xff, 0xff,
+ 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xb0 */
+ 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xa7,
+ 0xa8, 0xa9, 0xaa, 0xc3, 0xff, 0xff, 0xff, 0xff, /* 0xc0 */
+ 0xff, 0xff, 0xff, 0xff, 0xfe, 0xfe, 0xc3, 0xc3,
+ 0xc3, 0xc3, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xd0 */
+ 0xfe, 0xff, 0xff, 0xfe, 0xff, 0xfe, 0xff, 0xfe,
+ 0xfe, 0xff, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xff, /* 0xe0 */
+ 0xff, 0xfe, 0xff, 0xfe, 0xff, 0xfe, 0xfe, 0xff,
+ 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, /* 0xf0 */
+ 0xfe, 0xfe, 0xfe, 0xfe, 0xff, 0xfd, 0xfe, 0xff
+ };
+
+ *count = ARRAY_SIZE(kind);
+ return kind;
}
static const struct nvkm_mmu_func
gf100_mmu = {
- .limit = (1ULL << 40),
.dma_bits = 40,
- .pgt_bits = 27 - 12,
- .spg_shift = 12,
- .lpg_shift = 17,
- .create = gf100_vm_create,
- .map_pgt = gf100_vm_map_pgt,
- .map = gf100_vm_map,
- .map_sg = gf100_vm_map_sg,
- .unmap = gf100_vm_unmap,
- .flush = gf100_vm_flush,
+ .mmu = {{ -1, -1, NVIF_CLASS_MMU_GF100}},
+ .mem = {{ -1, 0, NVIF_CLASS_MEM_GF100}, gf100_mem_new, gf100_mem_map },
+ .vmm = {{ -1, -1, NVIF_CLASS_VMM_GF100}, gf100_vmm_new },
+ .kind = gf100_mmu_kind,
+ .kind_sys = true,
};
int
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gk104.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gk104.c
new file mode 100644
index 000000000000..3d7d1eb1cff9
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gk104.c
@@ -0,0 +1,41 @@
+/*
+ * Copyright 2017 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+#include "mem.h"
+#include "vmm.h"
+
+#include <nvif/class.h>
+
+static const struct nvkm_mmu_func
+gk104_mmu = {
+ .dma_bits = 40,
+ .mmu = {{ -1, -1, NVIF_CLASS_MMU_GF100}},
+ .mem = {{ -1, 0, NVIF_CLASS_MEM_GF100}, gf100_mem_new, gf100_mem_map },
+ .vmm = {{ -1, -1, NVIF_CLASS_VMM_GF100}, gk104_vmm_new },
+ .kind = gf100_mmu_kind,
+ .kind_sys = true,
+};
+
+int
+gk104_mmu_new(struct nvkm_device *device, int index, struct nvkm_mmu **pmmu)
+{
+ return nvkm_mmu_new_(&gk104_mmu, device, index, pmmu);
+}
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gk20a.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gk20a.c
new file mode 100644
index 000000000000..ac74965a60d4
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gk20a.c
@@ -0,0 +1,41 @@
+/*
+ * Copyright 2017 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+#include "mem.h"
+#include "vmm.h"
+
+#include <nvif/class.h>
+
+static const struct nvkm_mmu_func
+gk20a_mmu = {
+ .dma_bits = 40,
+ .mmu = {{ -1, -1, NVIF_CLASS_MMU_GF100}},
+ .mem = {{ -1, -1, NVIF_CLASS_MEM_GF100}, .umap = gf100_mem_map },
+ .vmm = {{ -1, -1, NVIF_CLASS_VMM_GF100}, gk20a_vmm_new },
+ .kind = gf100_mmu_kind,
+ .kind_sys = true,
+};
+
+int
+gk20a_mmu_new(struct nvkm_device *device, int index, struct nvkm_mmu **pmmu)
+{
+ return nvkm_mmu_new_(&gk20a_mmu, device, index, pmmu);
+}
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gm200.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gm200.c
new file mode 100644
index 000000000000..dbf644ebac97
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gm200.c
@@ -0,0 +1,97 @@
+/*
+ * Copyright 2017 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+#include "mem.h"
+#include "vmm.h"
+
+#include <subdev/fb.h>
+
+#include <nvif/class.h>
+
+const u8 *
+gm200_mmu_kind(struct nvkm_mmu *mmu, int *count)
+{
+ static const u8
+ kind[256] = {
+ 0x00, 0x01, 0x01, 0x01, 0x01, 0x01, 0xff, 0x01, /* 0x00 */
+ 0x01, 0x01, 0x01, 0xff, 0xff, 0xff, 0xff, 0xff,
+ 0xff, 0x11, 0xff, 0xff, 0xff, 0xff, 0xff, 0x11, /* 0x10 */
+ 0x11, 0x11, 0x11, 0xff, 0xff, 0xff, 0xff, 0xff,
+ 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x26, 0x27, /* 0x20 */
+ 0x28, 0x29, 0x2a, 0x2b, 0xff, 0xff, 0xff, 0xff,
+ 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x30 */
+ 0xff, 0xff, 0x26, 0x27, 0x28, 0x29, 0x26, 0x27,
+ 0x28, 0x29, 0xff, 0xff, 0xff, 0xff, 0x46, 0xff, /* 0x40 */
+ 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+ 0xff, 0x46, 0x46, 0x46, 0x46, 0xff, 0xff, 0xff, /* 0x50 */
+ 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+ 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x60 */
+ 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+ 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x70 */
+ 0xff, 0xff, 0xff, 0x7b, 0xff, 0xff, 0xff, 0xff,
+ 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7b, 0x7b, /* 0x80 */
+ 0x7b, 0x7b, 0xff, 0x8b, 0x8c, 0x8d, 0x8e, 0xff,
+ 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x90 */
+ 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+ 0xff, 0xff, 0xff, 0x8b, 0x8c, 0x8d, 0x8e, 0xa7, /* 0xa0 */
+ 0xa8, 0xa9, 0xaa, 0xff, 0xff, 0xff, 0xff, 0xff,
+ 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xb0 */
+ 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xa7,
+ 0xa8, 0xa9, 0xaa, 0xc3, 0xff, 0xff, 0xff, 0xff, /* 0xc0 */
+ 0xff, 0xff, 0xff, 0xff, 0xfe, 0xfe, 0xc3, 0xc3,
+ 0xc3, 0xc3, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xd0 */
+ 0xfe, 0xff, 0xff, 0xfe, 0xff, 0xfe, 0xff, 0xfe,
+ 0xfe, 0xff, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xff, /* 0xe0 */
+ 0xff, 0xfe, 0xff, 0xfe, 0xff, 0xfe, 0xfe, 0xff,
+ 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, /* 0xf0 */
+ 0xfe, 0xfe, 0xfe, 0xfe, 0xff, 0xfd, 0xfe, 0xff
+ };
+ *count = ARRAY_SIZE(kind);
+ return kind;
+}
+
+static const struct nvkm_mmu_func
+gm200_mmu = {
+ .dma_bits = 40,
+ .mmu = {{ -1, -1, NVIF_CLASS_MMU_GF100}},
+ .mem = {{ -1, 0, NVIF_CLASS_MEM_GF100}, gf100_mem_new, gf100_mem_map },
+ .vmm = {{ -1, 0, NVIF_CLASS_VMM_GM200}, gm200_vmm_new },
+ .kind = gm200_mmu_kind,
+ .kind_sys = true,
+};
+
+static const struct nvkm_mmu_func
+gm200_mmu_fixed = {
+ .dma_bits = 40,
+ .mmu = {{ -1, -1, NVIF_CLASS_MMU_GF100}},
+ .mem = {{ -1, 0, NVIF_CLASS_MEM_GF100}, gf100_mem_new, gf100_mem_map },
+ .vmm = {{ -1, -1, NVIF_CLASS_VMM_GM200}, gm200_vmm_new_fixed },
+ .kind = gm200_mmu_kind,
+ .kind_sys = true,
+};
+
+int
+gm200_mmu_new(struct nvkm_device *device, int index, struct nvkm_mmu **pmmu)
+{
+ if (device->fb->page)
+ return nvkm_mmu_new_(&gm200_mmu_fixed, device, index, pmmu);
+ return nvkm_mmu_new_(&gm200_mmu, device, index, pmmu);
+}
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gm20b.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gm20b.c
new file mode 100644
index 000000000000..7353a94b4091
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gm20b.c
@@ -0,0 +1,55 @@
+/*
+ * Copyright 2017 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+#include "mem.h"
+#include "vmm.h"
+
+#include <subdev/fb.h>
+
+#include <nvif/class.h>
+
+static const struct nvkm_mmu_func
+gm20b_mmu = {
+ .dma_bits = 40,
+ .mmu = {{ -1, -1, NVIF_CLASS_MMU_GF100}},
+ .mem = {{ -1, -1, NVIF_CLASS_MEM_GF100}, .umap = gf100_mem_map },
+ .vmm = {{ -1, 0, NVIF_CLASS_VMM_GM200}, gm20b_vmm_new },
+ .kind = gm200_mmu_kind,
+ .kind_sys = true,
+};
+
+static const struct nvkm_mmu_func
+gm20b_mmu_fixed = {
+ .dma_bits = 40,
+ .mmu = {{ -1, -1, NVIF_CLASS_MMU_GF100}},
+ .mem = {{ -1, -1, NVIF_CLASS_MEM_GF100}, .umap = gf100_mem_map },
+ .vmm = {{ -1, -1, NVIF_CLASS_VMM_GM200}, gm20b_vmm_new_fixed },
+ .kind = gm200_mmu_kind,
+ .kind_sys = true,
+};
+
+int
+gm20b_mmu_new(struct nvkm_device *device, int index, struct nvkm_mmu **pmmu)
+{
+ if (device->fb->page)
+ return nvkm_mmu_new_(&gm20b_mmu_fixed, device, index, pmmu);
+ return nvkm_mmu_new_(&gm20b_mmu, device, index, pmmu);
+}
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gp100.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gp100.c
new file mode 100644
index 000000000000..651b8805c67c
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gp100.c
@@ -0,0 +1,45 @@
+/*
+ * Copyright 2017 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+#include "mem.h"
+#include "vmm.h"
+
+#include <core/option.h>
+
+#include <nvif/class.h>
+
+static const struct nvkm_mmu_func
+gp100_mmu = {
+ .dma_bits = 47,
+ .mmu = {{ -1, -1, NVIF_CLASS_MMU_GF100}},
+ .mem = {{ -1, 0, NVIF_CLASS_MEM_GF100}, gf100_mem_new, gf100_mem_map },
+ .vmm = {{ -1, -1, NVIF_CLASS_VMM_GP100}, gp100_vmm_new },
+ .kind = gm200_mmu_kind,
+ .kind_sys = true,
+};
+
+int
+gp100_mmu_new(struct nvkm_device *device, int index, struct nvkm_mmu **pmmu)
+{
+ if (!nvkm_boolopt(device->cfgopt, "GP100MmuLayout", true))
+ return gm200_mmu_new(device, index, pmmu);
+ return nvkm_mmu_new_(&gp100_mmu, device, index, pmmu);
+}
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gp10b. b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gp10b.
new file mode 100644
index 000000000000..e69de29bb2d1
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gp10b.
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gp10b.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gp10b.c
new file mode 100644
index 000000000000..3bd3db31e0bb
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gp10b.c
@@ -0,0 +1,45 @@
+/*
+ * Copyright 2017 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+#include "mem.h"
+#include "vmm.h"
+
+#include <core/option.h>
+
+#include <nvif/class.h>
+
+static const struct nvkm_mmu_func
+gp10b_mmu = {
+ .dma_bits = 47,
+ .mmu = {{ -1, -1, NVIF_CLASS_MMU_GF100}},
+ .mem = {{ -1, -1, NVIF_CLASS_MEM_GF100}, .umap = gf100_mem_map },
+ .vmm = {{ -1, -1, NVIF_CLASS_VMM_GP100}, gp10b_vmm_new },
+ .kind = gm200_mmu_kind,
+ .kind_sys = true,
+};
+
+int
+gp10b_mmu_new(struct nvkm_device *device, int index, struct nvkm_mmu **pmmu)
+{
+ if (!nvkm_boolopt(device->cfgopt, "GP100MmuLayout", true))
+ return gm20b_mmu_new(device, index, pmmu);
+ return nvkm_mmu_new_(&gp10b_mmu, device, index, pmmu);
+}
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/mem.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/mem.c
new file mode 100644
index 000000000000..39808489f21d
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/mem.c
@@ -0,0 +1,242 @@
+/*
+ * Copyright 2017 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+#define nvkm_mem(p) container_of((p), struct nvkm_mem, memory)
+#include "mem.h"
+
+#include <core/memory.h>
+
+#include <nvif/if000a.h>
+#include <nvif/unpack.h>
+
+struct nvkm_mem {
+ struct nvkm_memory memory;
+ enum nvkm_memory_target target;
+ struct nvkm_mmu *mmu;
+ u64 pages;
+ struct page **mem;
+ union {
+ struct scatterlist *sgl;
+ dma_addr_t *dma;
+ };
+};
+
+static enum nvkm_memory_target
+nvkm_mem_target(struct nvkm_memory *memory)
+{
+ return nvkm_mem(memory)->target;
+}
+
+static u8
+nvkm_mem_page(struct nvkm_memory *memory)
+{
+ return PAGE_SHIFT;
+}
+
+static u64
+nvkm_mem_addr(struct nvkm_memory *memory)
+{
+ struct nvkm_mem *mem = nvkm_mem(memory);
+ if (mem->pages == 1 && mem->mem)
+ return mem->dma[0];
+ return ~0ULL;
+}
+
+static u64
+nvkm_mem_size(struct nvkm_memory *memory)
+{
+ return nvkm_mem(memory)->pages << PAGE_SHIFT;
+}
+
+static int
+nvkm_mem_map_dma(struct nvkm_memory *memory, u64 offset, struct nvkm_vmm *vmm,
+ struct nvkm_vma *vma, void *argv, u32 argc)
+{
+ struct nvkm_mem *mem = nvkm_mem(memory);
+ struct nvkm_vmm_map map = {
+ .memory = &mem->memory,
+ .offset = offset,
+ .dma = mem->dma,
+ };
+ return nvkm_vmm_map(vmm, vma, argv, argc, &map);
+}
+
+static void *
+nvkm_mem_dtor(struct nvkm_memory *memory)
+{
+ struct nvkm_mem *mem = nvkm_mem(memory);
+ if (mem->mem) {
+ while (mem->pages--) {
+ dma_unmap_page(mem->mmu->subdev.device->dev,
+ mem->dma[mem->pages], PAGE_SIZE,
+ DMA_BIDIRECTIONAL);
+ __free_page(mem->mem[mem->pages]);
+ }
+ kvfree(mem->dma);
+ kvfree(mem->mem);
+ }
+ return mem;
+}
+
+static const struct nvkm_memory_func
+nvkm_mem_dma = {
+ .dtor = nvkm_mem_dtor,
+ .target = nvkm_mem_target,
+ .page = nvkm_mem_page,
+ .addr = nvkm_mem_addr,
+ .size = nvkm_mem_size,
+ .map = nvkm_mem_map_dma,
+};
+
+static int
+nvkm_mem_map_sgl(struct nvkm_memory *memory, u64 offset, struct nvkm_vmm *vmm,
+ struct nvkm_vma *vma, void *argv, u32 argc)
+{
+ struct nvkm_mem *mem = nvkm_mem(memory);
+ struct nvkm_vmm_map map = {
+ .memory = &mem->memory,
+ .offset = offset,
+ .sgl = mem->sgl,
+ };
+ return nvkm_vmm_map(vmm, vma, argv, argc, &map);
+}
+
+static const struct nvkm_memory_func
+nvkm_mem_sgl = {
+ .dtor = nvkm_mem_dtor,
+ .target = nvkm_mem_target,
+ .page = nvkm_mem_page,
+ .addr = nvkm_mem_addr,
+ .size = nvkm_mem_size,
+ .map = nvkm_mem_map_sgl,
+};
+
+int
+nvkm_mem_map_host(struct nvkm_memory *memory, void **pmap)
+{
+ struct nvkm_mem *mem = nvkm_mem(memory);
+ if (mem->mem) {
+ *pmap = vmap(mem->mem, mem->pages, VM_MAP, PAGE_KERNEL);
+ return *pmap ? 0 : -EFAULT;
+ }
+ return -EINVAL;
+}
+
+static int
+nvkm_mem_new_host(struct nvkm_mmu *mmu, int type, u8 page, u64 size,
+ void *argv, u32 argc, struct nvkm_memory **pmemory)
+{
+ struct device *dev = mmu->subdev.device->dev;
+ union {
+ struct nvif_mem_ram_vn vn;
+ struct nvif_mem_ram_v0 v0;
+ } *args = argv;
+ int ret = -ENOSYS;
+ enum nvkm_memory_target target;
+ struct nvkm_mem *mem;
+ gfp_t gfp = GFP_USER | __GFP_ZERO;
+
+ if ( (mmu->type[type].type & NVKM_MEM_COHERENT) &&
+ !(mmu->type[type].type & NVKM_MEM_UNCACHED))
+ target = NVKM_MEM_TARGET_HOST;
+ else
+ target = NVKM_MEM_TARGET_NCOH;
+
+ if (page != PAGE_SHIFT)
+ return -EINVAL;
+
+ if (!(mem = kzalloc(sizeof(*mem), GFP_KERNEL)))
+ return -ENOMEM;
+ mem->target = target;
+ mem->mmu = mmu;
+ *pmemory = &mem->memory;
+
+ if (!(ret = nvif_unpack(ret, &argv, &argc, args->v0, 0, 0, false))) {
+ if (args->v0.dma) {
+ nvkm_memory_ctor(&nvkm_mem_dma, &mem->memory);
+ mem->dma = args->v0.dma;
+ } else {
+ nvkm_memory_ctor(&nvkm_mem_sgl, &mem->memory);
+ mem->sgl = args->v0.sgl;
+ }
+
+ if (!IS_ALIGNED(size, PAGE_SIZE))
+ return -EINVAL;
+ mem->pages = size >> PAGE_SHIFT;
+ return 0;
+ } else
+ if ( (ret = nvif_unvers(ret, &argv, &argc, args->vn))) {
+ kfree(mem);
+ return ret;
+ }
+
+ nvkm_memory_ctor(&nvkm_mem_dma, &mem->memory);
+ size = ALIGN(size, PAGE_SIZE) >> PAGE_SHIFT;
+
+ if (!(mem->mem = kvmalloc(sizeof(*mem->mem) * size, GFP_KERNEL)))
+ return -ENOMEM;
+ if (!(mem->dma = kvmalloc(sizeof(*mem->dma) * size, GFP_KERNEL)))
+ return -ENOMEM;
+
+ if (mmu->dma_bits > 32)
+ gfp |= GFP_HIGHUSER;
+ else
+ gfp |= GFP_DMA32;
+
+ for (mem->pages = 0; size; size--, mem->pages++) {
+ struct page *p = alloc_page(gfp);
+ if (!p)
+ return -ENOMEM;
+
+ mem->dma[mem->pages] = dma_map_page(mmu->subdev.device->dev,
+ p, 0, PAGE_SIZE,
+ DMA_BIDIRECTIONAL);
+ if (dma_mapping_error(dev, mem->dma[mem->pages])) {
+ __free_page(p);
+ return -ENOMEM;
+ }
+
+ mem->mem[mem->pages] = p;
+ }
+
+ return 0;
+}
+
+int
+nvkm_mem_new_type(struct nvkm_mmu *mmu, int type, u8 page, u64 size,
+ void *argv, u32 argc, struct nvkm_memory **pmemory)
+{
+ struct nvkm_memory *memory = NULL;
+ int ret;
+
+ if (mmu->type[type].type & NVKM_MEM_VRAM) {
+ ret = mmu->func->mem.vram(mmu, type, page, size,
+ argv, argc, &memory);
+ } else {
+ ret = nvkm_mem_new_host(mmu, type, page, size,
+ argv, argc, &memory);
+ }
+
+ if (ret)
+ nvkm_memory_unref(&memory);
+ *pmemory = memory;
+ return ret;
+}
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/mem.h b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/mem.h
new file mode 100644
index 000000000000..234267e1b215
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/mem.h
@@ -0,0 +1,23 @@
+#ifndef __NVKM_MEM_H__
+#define __NVKM_MEM_H__
+#include "priv.h"
+
+int nvkm_mem_new_type(struct nvkm_mmu *, int type, u8 page, u64 size,
+ void *argv, u32 argc, struct nvkm_memory **);
+int nvkm_mem_map_host(struct nvkm_memory *, void **pmap);
+
+int nv04_mem_new(struct nvkm_mmu *, int, u8, u64, void *, u32,
+ struct nvkm_memory **);
+int nv04_mem_map(struct nvkm_mmu *, struct nvkm_memory *, void *, u32,
+ u64 *, u64 *, struct nvkm_vma **);
+
+int nv50_mem_new(struct nvkm_mmu *, int, u8, u64, void *, u32,
+ struct nvkm_memory **);
+int nv50_mem_map(struct nvkm_mmu *, struct nvkm_memory *, void *, u32,
+ u64 *, u64 *, struct nvkm_vma **);
+
+int gf100_mem_new(struct nvkm_mmu *, int, u8, u64, void *, u32,
+ struct nvkm_memory **);
+int gf100_mem_map(struct nvkm_mmu *, struct nvkm_memory *, void *, u32,
+ u64 *, u64 *, struct nvkm_vma **);
+#endif
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/memgf100.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/memgf100.c
new file mode 100644
index 000000000000..d9c9bee45222
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/memgf100.c
@@ -0,0 +1,94 @@
+/*
+ * Copyright 2017 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+#include "mem.h"
+
+#include <core/memory.h>
+#include <subdev/bar.h>
+#include <subdev/fb.h>
+
+#include <nvif/class.h>
+#include <nvif/if900b.h>
+#include <nvif/if900d.h>
+#include <nvif/unpack.h>
+
+int
+gf100_mem_map(struct nvkm_mmu *mmu, struct nvkm_memory *memory, void *argv,
+ u32 argc, u64 *paddr, u64 *psize, struct nvkm_vma **pvma)
+{
+ struct gf100_vmm_map_v0 uvmm = {};
+ union {
+ struct gf100_mem_map_vn vn;
+ struct gf100_mem_map_v0 v0;
+ } *args = argv;
+ struct nvkm_device *device = mmu->subdev.device;
+ struct nvkm_vmm *bar = nvkm_bar_bar1_vmm(device);
+ int ret = -ENOSYS;
+
+ if (!(ret = nvif_unpack(ret, &argv, &argc, args->v0, 0, 0, false))) {
+ uvmm.ro = args->v0.ro;
+ uvmm.kind = args->v0.kind;
+ } else
+ if (!(ret = nvif_unvers(ret, &argv, &argc, args->vn))) {
+ } else
+ return ret;
+
+ ret = nvkm_vmm_get(bar, nvkm_memory_page(memory),
+ nvkm_memory_size(memory), pvma);
+ if (ret)
+ return ret;
+
+ ret = nvkm_memory_map(memory, 0, bar, *pvma, &uvmm, sizeof(uvmm));
+ if (ret)
+ return ret;
+
+ *paddr = device->func->resource_addr(device, 1) + (*pvma)->addr;
+ *psize = (*pvma)->size;
+ return 0;
+}
+
+int
+gf100_mem_new(struct nvkm_mmu *mmu, int type, u8 page, u64 size,
+ void *argv, u32 argc, struct nvkm_memory **pmemory)
+{
+ union {
+ struct gf100_mem_vn vn;
+ struct gf100_mem_v0 v0;
+ } *args = argv;
+ int ret = -ENOSYS;
+ bool contig;
+
+ if (!(ret = nvif_unpack(ret, &argv, &argc, args->v0, 0, 0, false))) {
+ contig = args->v0.contig;
+ } else
+ if (!(ret = nvif_unvers(ret, &argv, &argc, args->vn))) {
+ contig = false;
+ } else
+ return ret;
+
+ if (mmu->type[type].type & (NVKM_MEM_DISP | NVKM_MEM_COMP))
+ type = NVKM_RAM_MM_NORMAL;
+ else
+ type = NVKM_RAM_MM_MIXED;
+
+ return nvkm_ram_get(mmu->subdev.device, type, 0x01, page,
+ size, contig, false, pmemory);
+}
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/memnv04.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/memnv04.c
new file mode 100644
index 000000000000..79a3b0cc9f5b
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/memnv04.c
@@ -0,0 +1,69 @@
+/*
+ * Copyright 2017 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+#include "mem.h"
+
+#include <core/memory.h>
+#include <subdev/fb.h>
+
+#include <nvif/if000b.h>
+#include <nvif/unpack.h>
+
+int
+nv04_mem_map(struct nvkm_mmu *mmu, struct nvkm_memory *memory, void *argv,
+ u32 argc, u64 *paddr, u64 *psize, struct nvkm_vma **pvma)
+{
+ union {
+ struct nv04_mem_map_vn vn;
+ } *args = argv;
+ struct nvkm_device *device = mmu->subdev.device;
+ const u64 addr = nvkm_memory_addr(memory);
+ int ret = -ENOSYS;
+
+ if ((ret = nvif_unvers(ret, &argv, &argc, args->vn)))
+ return ret;
+
+ *paddr = device->func->resource_addr(device, 1) + addr;
+ *psize = nvkm_memory_size(memory);
+ *pvma = ERR_PTR(-ENODEV);
+ return 0;
+}
+
+int
+nv04_mem_new(struct nvkm_mmu *mmu, int type, u8 page, u64 size,
+ void *argv, u32 argc, struct nvkm_memory **pmemory)
+{
+ union {
+ struct nv04_mem_vn vn;
+ } *args = argv;
+ int ret = -ENOSYS;
+
+ if ((ret = nvif_unvers(ret, &argv, &argc, args->vn)))
+ return ret;
+
+ if (mmu->type[type].type & NVKM_MEM_MAPPABLE)
+ type = NVKM_RAM_MM_NORMAL;
+ else
+ type = NVKM_RAM_MM_NOMAP;
+
+ return nvkm_ram_get(mmu->subdev.device, type, 0x01, page,
+ size, true, false, pmemory);
+}
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/memnv50.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/memnv50.c
new file mode 100644
index 000000000000..46759b89fc1f
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/memnv50.c
@@ -0,0 +1,88 @@
+/*
+ * Copyright 2017 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+#include "mem.h"
+
+#include <core/memory.h>
+#include <subdev/bar.h>
+#include <subdev/fb.h>
+
+#include <nvif/class.h>
+#include <nvif/if500b.h>
+#include <nvif/if500d.h>
+#include <nvif/unpack.h>
+
+int
+nv50_mem_map(struct nvkm_mmu *mmu, struct nvkm_memory *memory, void *argv,
+ u32 argc, u64 *paddr, u64 *psize, struct nvkm_vma **pvma)
+{
+ struct nv50_vmm_map_v0 uvmm = {};
+ union {
+ struct nv50_mem_map_vn vn;
+ struct nv50_mem_map_v0 v0;
+ } *args = argv;
+ struct nvkm_device *device = mmu->subdev.device;
+ struct nvkm_vmm *bar = nvkm_bar_bar1_vmm(device);
+ u64 size = nvkm_memory_size(memory);
+ int ret = -ENOSYS;
+
+ if (!(ret = nvif_unpack(ret, &argv, &argc, args->v0, 0, 0, false))) {
+ uvmm.ro = args->v0.ro;
+ uvmm.kind = args->v0.kind;
+ uvmm.comp = args->v0.comp;
+ } else
+ if (!(ret = nvif_unvers(ret, &argv, &argc, args->vn))) {
+ } else
+ return ret;
+
+ ret = nvkm_vmm_get(bar, 12, size, pvma);
+ if (ret)
+ return ret;
+
+ *paddr = device->func->resource_addr(device, 1) + (*pvma)->addr;
+ *psize = (*pvma)->size;
+ return nvkm_memory_map(memory, 0, bar, *pvma, &uvmm, sizeof(uvmm));
+}
+
+int
+nv50_mem_new(struct nvkm_mmu *mmu, int type, u8 page, u64 size,
+ void *argv, u32 argc, struct nvkm_memory **pmemory)
+{
+ union {
+ struct nv50_mem_vn vn;
+ struct nv50_mem_v0 v0;
+ } *args = argv;
+ int ret = -ENOSYS;
+ bool contig;
+
+ if (!(ret = nvif_unpack(ret, &argv, &argc, args->v0, 0, 0, false))) {
+ type = args->v0.bankswz ? 0x02 : 0x01;
+ contig = args->v0.contig;
+ } else
+ if (!(ret = nvif_unvers(ret, &argv, &argc, args->vn))) {
+ type = 0x01;
+ contig = false;
+ } else
+ return -ENOSYS;
+
+ return nvkm_ram_get(mmu->subdev.device, NVKM_RAM_MM_NORMAL, type,
+ page, size, contig, false, pmemory);
+}
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/nv04.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/nv04.c
index 37927c3fdc3e..d201c887c2cd 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/nv04.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/nv04.c
@@ -21,129 +21,21 @@
*
* Authors: Ben Skeggs
*/
-#include "nv04.h"
+#include "mem.h"
+#include "vmm.h"
-#include <core/gpuobj.h>
-
-#define NV04_PDMA_SIZE (128 * 1024 * 1024)
-#define NV04_PDMA_PAGE ( 4 * 1024)
-
-/*******************************************************************************
- * VM map/unmap callbacks
- ******************************************************************************/
-
-static void
-nv04_vm_map_sg(struct nvkm_vma *vma, struct nvkm_memory *pgt,
- struct nvkm_mem *mem, u32 pte, u32 cnt, dma_addr_t *list)
-{
- pte = 0x00008 + (pte * 4);
- nvkm_kmap(pgt);
- while (cnt) {
- u32 page = PAGE_SIZE / NV04_PDMA_PAGE;
- u32 phys = (u32)*list++;
- while (cnt && page--) {
- nvkm_wo32(pgt, pte, phys | 3);
- phys += NV04_PDMA_PAGE;
- pte += 4;
- cnt -= 1;
- }
- }
- nvkm_done(pgt);
-}
-
-static void
-nv04_vm_unmap(struct nvkm_vma *vma, struct nvkm_memory *pgt, u32 pte, u32 cnt)
-{
- pte = 0x00008 + (pte * 4);
- nvkm_kmap(pgt);
- while (cnt--) {
- nvkm_wo32(pgt, pte, 0x00000000);
- pte += 4;
- }
- nvkm_done(pgt);
-}
-
-static void
-nv04_vm_flush(struct nvkm_vm *vm)
-{
-}
-
-/*******************************************************************************
- * MMU subdev
- ******************************************************************************/
-
-static int
-nv04_mmu_oneinit(struct nvkm_mmu *base)
-{
- struct nv04_mmu *mmu = nv04_mmu(base);
- struct nvkm_device *device = mmu->base.subdev.device;
- struct nvkm_memory *dma;
- int ret;
-
- ret = nvkm_vm_create(&mmu->base, 0, NV04_PDMA_SIZE, 0, 4096, NULL,
- &mmu->vm);
- if (ret)
- return ret;
-
- ret = nvkm_memory_new(device, NVKM_MEM_TARGET_INST,
- (NV04_PDMA_SIZE / NV04_PDMA_PAGE) * 4 + 8,
- 16, true, &dma);
- mmu->vm->pgt[0].mem[0] = dma;
- mmu->vm->pgt[0].refcount[0] = 1;
- if (ret)
- return ret;
-
- nvkm_kmap(dma);
- nvkm_wo32(dma, 0x00000, 0x0002103d); /* PCI, RW, PT, !LN */
- nvkm_wo32(dma, 0x00004, NV04_PDMA_SIZE - 1);
- nvkm_done(dma);
- return 0;
-}
-
-void *
-nv04_mmu_dtor(struct nvkm_mmu *base)
-{
- struct nv04_mmu *mmu = nv04_mmu(base);
- struct nvkm_device *device = mmu->base.subdev.device;
- if (mmu->vm) {
- nvkm_memory_del(&mmu->vm->pgt[0].mem[0]);
- nvkm_vm_ref(NULL, &mmu->vm, NULL);
- }
- if (mmu->nullp) {
- dma_free_coherent(device->dev, 16 * 1024,
- mmu->nullp, mmu->null);
- }
- return mmu;
-}
-
-int
-nv04_mmu_new_(const struct nvkm_mmu_func *func, struct nvkm_device *device,
- int index, struct nvkm_mmu **pmmu)
-{
- struct nv04_mmu *mmu;
- if (!(mmu = kzalloc(sizeof(*mmu), GFP_KERNEL)))
- return -ENOMEM;
- *pmmu = &mmu->base;
- nvkm_mmu_ctor(func, device, index, &mmu->base);
- return 0;
-}
+#include <nvif/class.h>
const struct nvkm_mmu_func
nv04_mmu = {
- .oneinit = nv04_mmu_oneinit,
- .dtor = nv04_mmu_dtor,
- .limit = NV04_PDMA_SIZE,
.dma_bits = 32,
- .pgt_bits = 32 - 12,
- .spg_shift = 12,
- .lpg_shift = 12,
- .map_sg = nv04_vm_map_sg,
- .unmap = nv04_vm_unmap,
- .flush = nv04_vm_flush,
+ .mmu = {{ -1, -1, NVIF_CLASS_MMU_NV04}},
+ .mem = {{ -1, -1, NVIF_CLASS_MEM_NV04}, nv04_mem_new, nv04_mem_map },
+ .vmm = {{ -1, -1, NVIF_CLASS_VMM_NV04}, nv04_vmm_new, true },
};
int
nv04_mmu_new(struct nvkm_device *device, int index, struct nvkm_mmu **pmmu)
{
- return nv04_mmu_new_(&nv04_mmu, device, index, pmmu);
+ return nvkm_mmu_new_(&nv04_mmu, device, index, pmmu);
}
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/nv04.h b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/nv04.h
deleted file mode 100644
index 363e33b296d5..000000000000
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/nv04.h
+++ /dev/null
@@ -1,18 +0,0 @@
-#ifndef __NV04_MMU_PRIV__
-#define __NV04_MMU_PRIV__
-#define nv04_mmu(p) container_of((p), struct nv04_mmu, base)
-#include "priv.h"
-
-struct nv04_mmu {
- struct nvkm_mmu base;
- struct nvkm_vm *vm;
- dma_addr_t null;
- void *nullp;
-};
-
-int nv04_mmu_new_(const struct nvkm_mmu_func *, struct nvkm_device *,
- int index, struct nvkm_mmu **);
-void *nv04_mmu_dtor(struct nvkm_mmu *);
-
-extern const struct nvkm_mmu_func nv04_mmu;
-#endif
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/nv41.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/nv41.c
index c6a26f907009..adca81895c09 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/nv41.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/nv41.c
@@ -21,113 +21,29 @@
*
* Authors: Ben Skeggs
*/
-#include "nv04.h"
+#include "mem.h"
+#include "vmm.h"
-#include <core/gpuobj.h>
#include <core/option.h>
-#include <subdev/timer.h>
-#define NV41_GART_SIZE (512 * 1024 * 1024)
-#define NV41_GART_PAGE ( 4 * 1024)
-
-/*******************************************************************************
- * VM map/unmap callbacks
- ******************************************************************************/
-
-static void
-nv41_vm_map_sg(struct nvkm_vma *vma, struct nvkm_memory *pgt,
- struct nvkm_mem *mem, u32 pte, u32 cnt, dma_addr_t *list)
-{
- pte = pte * 4;
- nvkm_kmap(pgt);
- while (cnt) {
- u32 page = PAGE_SIZE / NV41_GART_PAGE;
- u64 phys = (u64)*list++;
- while (cnt && page--) {
- nvkm_wo32(pgt, pte, (phys >> 7) | 1);
- phys += NV41_GART_PAGE;
- pte += 4;
- cnt -= 1;
- }
- }
- nvkm_done(pgt);
-}
-
-static void
-nv41_vm_unmap(struct nvkm_vma *vma, struct nvkm_memory *pgt, u32 pte, u32 cnt)
-{
- pte = pte * 4;
- nvkm_kmap(pgt);
- while (cnt--) {
- nvkm_wo32(pgt, pte, 0x00000000);
- pte += 4;
- }
- nvkm_done(pgt);
-}
-
-static void
-nv41_vm_flush(struct nvkm_vm *vm)
-{
- struct nv04_mmu *mmu = nv04_mmu(vm->mmu);
- struct nvkm_device *device = mmu->base.subdev.device;
-
- mutex_lock(&mmu->base.subdev.mutex);
- nvkm_wr32(device, 0x100810, 0x00000022);
- nvkm_msec(device, 2000,
- if (nvkm_rd32(device, 0x100810) & 0x00000020)
- break;
- );
- nvkm_wr32(device, 0x100810, 0x00000000);
- mutex_unlock(&mmu->base.subdev.mutex);
-}
-
-/*******************************************************************************
- * MMU subdev
- ******************************************************************************/
-
-static int
-nv41_mmu_oneinit(struct nvkm_mmu *base)
-{
- struct nv04_mmu *mmu = nv04_mmu(base);
- struct nvkm_device *device = mmu->base.subdev.device;
- int ret;
-
- ret = nvkm_vm_create(&mmu->base, 0, NV41_GART_SIZE, 0, 4096, NULL,
- &mmu->vm);
- if (ret)
- return ret;
-
- ret = nvkm_memory_new(device, NVKM_MEM_TARGET_INST,
- (NV41_GART_SIZE / NV41_GART_PAGE) * 4, 16, true,
- &mmu->vm->pgt[0].mem[0]);
- mmu->vm->pgt[0].refcount[0] = 1;
- return ret;
-}
+#include <nvif/class.h>
static void
-nv41_mmu_init(struct nvkm_mmu *base)
+nv41_mmu_init(struct nvkm_mmu *mmu)
{
- struct nv04_mmu *mmu = nv04_mmu(base);
- struct nvkm_device *device = mmu->base.subdev.device;
- struct nvkm_memory *dma = mmu->vm->pgt[0].mem[0];
- nvkm_wr32(device, 0x100800, 0x00000002 | nvkm_memory_addr(dma));
+ struct nvkm_device *device = mmu->subdev.device;
+ nvkm_wr32(device, 0x100800, 0x00000002 | mmu->vmm->pd->pt[0]->addr);
nvkm_mask(device, 0x10008c, 0x00000100, 0x00000100);
nvkm_wr32(device, 0x100820, 0x00000000);
}
static const struct nvkm_mmu_func
nv41_mmu = {
- .dtor = nv04_mmu_dtor,
- .oneinit = nv41_mmu_oneinit,
.init = nv41_mmu_init,
- .limit = NV41_GART_SIZE,
.dma_bits = 39,
- .pgt_bits = 32 - 12,
- .spg_shift = 12,
- .lpg_shift = 12,
- .map_sg = nv41_vm_map_sg,
- .unmap = nv41_vm_unmap,
- .flush = nv41_vm_flush,
+ .mmu = {{ -1, -1, NVIF_CLASS_MMU_NV04}},
+ .mem = {{ -1, -1, NVIF_CLASS_MEM_NV04}, nv04_mem_new, nv04_mem_map },
+ .vmm = {{ -1, -1, NVIF_CLASS_VMM_NV04}, nv41_vmm_new, true },
};
int
@@ -137,5 +53,5 @@ nv41_mmu_new(struct nvkm_device *device, int index, struct nvkm_mmu **pmmu)
!nvkm_boolopt(device->cfgopt, "NvPCIE", true))
return nv04_mmu_new(device, index, pmmu);
- return nv04_mmu_new_(&nv41_mmu, device, index, pmmu);
+ return nvkm_mmu_new_(&nv41_mmu, device, index, pmmu);
}
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/nv44.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/nv44.c
index a648c2395545..598c53a27bde 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/nv44.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/nv44.c
@@ -21,176 +21,18 @@
*
* Authors: Ben Skeggs
*/
-#include "nv04.h"
+#include "mem.h"
+#include "vmm.h"
-#include <core/gpuobj.h>
#include <core/option.h>
-#include <subdev/timer.h>
-#define NV44_GART_SIZE (512 * 1024 * 1024)
-#define NV44_GART_PAGE ( 4 * 1024)
-
-/*******************************************************************************
- * VM map/unmap callbacks
- ******************************************************************************/
-
-static void
-nv44_vm_fill(struct nvkm_memory *pgt, dma_addr_t null,
- dma_addr_t *list, u32 pte, u32 cnt)
-{
- u32 base = (pte << 2) & ~0x0000000f;
- u32 tmp[4];
-
- tmp[0] = nvkm_ro32(pgt, base + 0x0);
- tmp[1] = nvkm_ro32(pgt, base + 0x4);
- tmp[2] = nvkm_ro32(pgt, base + 0x8);
- tmp[3] = nvkm_ro32(pgt, base + 0xc);
-
- while (cnt--) {
- u32 addr = list ? (*list++ >> 12) : (null >> 12);
- switch (pte++ & 0x3) {
- case 0:
- tmp[0] &= ~0x07ffffff;
- tmp[0] |= addr;
- break;
- case 1:
- tmp[0] &= ~0xf8000000;
- tmp[0] |= addr << 27;
- tmp[1] &= ~0x003fffff;
- tmp[1] |= addr >> 5;
- break;
- case 2:
- tmp[1] &= ~0xffc00000;
- tmp[1] |= addr << 22;
- tmp[2] &= ~0x0001ffff;
- tmp[2] |= addr >> 10;
- break;
- case 3:
- tmp[2] &= ~0xfffe0000;
- tmp[2] |= addr << 17;
- tmp[3] &= ~0x00000fff;
- tmp[3] |= addr >> 15;
- break;
- }
- }
-
- nvkm_wo32(pgt, base + 0x0, tmp[0]);
- nvkm_wo32(pgt, base + 0x4, tmp[1]);
- nvkm_wo32(pgt, base + 0x8, tmp[2]);
- nvkm_wo32(pgt, base + 0xc, tmp[3] | 0x40000000);
-}
-
-static void
-nv44_vm_map_sg(struct nvkm_vma *vma, struct nvkm_memory *pgt,
- struct nvkm_mem *mem, u32 pte, u32 cnt, dma_addr_t *list)
-{
- struct nv04_mmu *mmu = nv04_mmu(vma->vm->mmu);
- u32 tmp[4];
- int i;
-
- nvkm_kmap(pgt);
- if (pte & 3) {
- u32 max = 4 - (pte & 3);
- u32 part = (cnt > max) ? max : cnt;
- nv44_vm_fill(pgt, mmu->null, list, pte, part);
- pte += part;
- list += part;
- cnt -= part;
- }
-
- while (cnt >= 4) {
- for (i = 0; i < 4; i++)
- tmp[i] = *list++ >> 12;
- nvkm_wo32(pgt, pte++ * 4, tmp[0] >> 0 | tmp[1] << 27);
- nvkm_wo32(pgt, pte++ * 4, tmp[1] >> 5 | tmp[2] << 22);
- nvkm_wo32(pgt, pte++ * 4, tmp[2] >> 10 | tmp[3] << 17);
- nvkm_wo32(pgt, pte++ * 4, tmp[3] >> 15 | 0x40000000);
- cnt -= 4;
- }
-
- if (cnt)
- nv44_vm_fill(pgt, mmu->null, list, pte, cnt);
- nvkm_done(pgt);
-}
-
-static void
-nv44_vm_unmap(struct nvkm_vma *vma, struct nvkm_memory *pgt, u32 pte, u32 cnt)
-{
- struct nv04_mmu *mmu = nv04_mmu(vma->vm->mmu);
-
- nvkm_kmap(pgt);
- if (pte & 3) {
- u32 max = 4 - (pte & 3);
- u32 part = (cnt > max) ? max : cnt;
- nv44_vm_fill(pgt, mmu->null, NULL, pte, part);
- pte += part;
- cnt -= part;
- }
-
- while (cnt >= 4) {
- nvkm_wo32(pgt, pte++ * 4, 0x00000000);
- nvkm_wo32(pgt, pte++ * 4, 0x00000000);
- nvkm_wo32(pgt, pte++ * 4, 0x00000000);
- nvkm_wo32(pgt, pte++ * 4, 0x00000000);
- cnt -= 4;
- }
-
- if (cnt)
- nv44_vm_fill(pgt, mmu->null, NULL, pte, cnt);
- nvkm_done(pgt);
-}
-
-static void
-nv44_vm_flush(struct nvkm_vm *vm)
-{
- struct nv04_mmu *mmu = nv04_mmu(vm->mmu);
- struct nvkm_device *device = mmu->base.subdev.device;
- nvkm_wr32(device, 0x100814, mmu->base.limit - NV44_GART_PAGE);
- nvkm_wr32(device, 0x100808, 0x00000020);
- nvkm_msec(device, 2000,
- if (nvkm_rd32(device, 0x100808) & 0x00000001)
- break;
- );
- nvkm_wr32(device, 0x100808, 0x00000000);
-}
-
-/*******************************************************************************
- * MMU subdev
- ******************************************************************************/
-
-static int
-nv44_mmu_oneinit(struct nvkm_mmu *base)
-{
- struct nv04_mmu *mmu = nv04_mmu(base);
- struct nvkm_device *device = mmu->base.subdev.device;
- int ret;
-
- mmu->nullp = dma_alloc_coherent(device->dev, 16 * 1024,
- &mmu->null, GFP_KERNEL);
- if (!mmu->nullp) {
- nvkm_warn(&mmu->base.subdev, "unable to allocate dummy pages\n");
- mmu->null = 0;
- }
-
- ret = nvkm_vm_create(&mmu->base, 0, NV44_GART_SIZE, 0, 4096, NULL,
- &mmu->vm);
- if (ret)
- return ret;
-
- ret = nvkm_memory_new(device, NVKM_MEM_TARGET_INST,
- (NV44_GART_SIZE / NV44_GART_PAGE) * 4,
- 512 * 1024, true,
- &mmu->vm->pgt[0].mem[0]);
- mmu->vm->pgt[0].refcount[0] = 1;
- return ret;
-}
+#include <nvif/class.h>
static void
-nv44_mmu_init(struct nvkm_mmu *base)
+nv44_mmu_init(struct nvkm_mmu *mmu)
{
- struct nv04_mmu *mmu = nv04_mmu(base);
- struct nvkm_device *device = mmu->base.subdev.device;
- struct nvkm_memory *gart = mmu->vm->pgt[0].mem[0];
+ struct nvkm_device *device = mmu->subdev.device;
+ struct nvkm_memory *pt = mmu->vmm->pd->pt[0]->memory;
u32 addr;
/* calculate vram address of this PRAMIN block, object must be
@@ -198,11 +40,11 @@ nv44_mmu_init(struct nvkm_mmu *base)
* of 512KiB for this to work correctly
*/
addr = nvkm_rd32(device, 0x10020c);
- addr -= ((nvkm_memory_addr(gart) >> 19) + 1) << 19;
+ addr -= ((nvkm_memory_addr(pt) >> 19) + 1) << 19;
nvkm_wr32(device, 0x100850, 0x80000000);
- nvkm_wr32(device, 0x100818, mmu->null);
- nvkm_wr32(device, 0x100804, NV44_GART_SIZE);
+ nvkm_wr32(device, 0x100818, mmu->vmm->null);
+ nvkm_wr32(device, 0x100804, (nvkm_memory_size(pt) / 4) * 4096);
nvkm_wr32(device, 0x100850, 0x00008000);
nvkm_mask(device, 0x10008c, 0x00000200, 0x00000200);
nvkm_wr32(device, 0x100820, 0x00000000);
@@ -212,17 +54,11 @@ nv44_mmu_init(struct nvkm_mmu *base)
static const struct nvkm_mmu_func
nv44_mmu = {
- .dtor = nv04_mmu_dtor,
- .oneinit = nv44_mmu_oneinit,
.init = nv44_mmu_init,
- .limit = NV44_GART_SIZE,
.dma_bits = 39,
- .pgt_bits = 32 - 12,
- .spg_shift = 12,
- .lpg_shift = 12,
- .map_sg = nv44_vm_map_sg,
- .unmap = nv44_vm_unmap,
- .flush = nv44_vm_flush,
+ .mmu = {{ -1, -1, NVIF_CLASS_MMU_NV04}},
+ .mem = {{ -1, -1, NVIF_CLASS_MEM_NV04}, nv04_mem_new, nv04_mem_map },
+ .vmm = {{ -1, -1, NVIF_CLASS_VMM_NV04}, nv44_vmm_new, true },
};
int
@@ -232,5 +68,5 @@ nv44_mmu_new(struct nvkm_device *device, int index, struct nvkm_mmu **pmmu)
!nvkm_boolopt(device->cfgopt, "NvPCIE", true))
return nv04_mmu_new(device, index, pmmu);
- return nv04_mmu_new_(&nv44_mmu, device, index, pmmu);
+ return nvkm_mmu_new_(&nv44_mmu, device, index, pmmu);
}
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/nv50.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/nv50.c
index a1f8d65f0276..db3dfbbb2aa0 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/nv50.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/nv50.c
@@ -21,207 +21,52 @@
*
* Authors: Ben Skeggs
*/
-#include "priv.h"
+#include "mem.h"
+#include "vmm.h"
-#include <core/gpuobj.h>
-#include <subdev/fb.h>
-#include <subdev/timer.h>
-#include <engine/gr.h>
+#include <nvif/class.h>
-static void
-nv50_vm_map_pgt(struct nvkm_gpuobj *pgd, u32 pde, struct nvkm_memory *pgt[2])
+const u8 *
+nv50_mmu_kind(struct nvkm_mmu *base, int *count)
{
- u64 phys = 0xdeadcafe00000000ULL;
- u32 coverage = 0;
-
- if (pgt[0]) {
- /* present, 4KiB pages */
- phys = 0x00000003 | nvkm_memory_addr(pgt[0]);
- coverage = (nvkm_memory_size(pgt[0]) >> 3) << 12;
- } else
- if (pgt[1]) {
- /* present, 64KiB pages */
- phys = 0x00000001 | nvkm_memory_addr(pgt[1]);
- coverage = (nvkm_memory_size(pgt[1]) >> 3) << 16;
- }
-
- if (phys & 1) {
- if (coverage <= 32 * 1024 * 1024)
- phys |= 0x60;
- else if (coverage <= 64 * 1024 * 1024)
- phys |= 0x40;
- else if (coverage <= 128 * 1024 * 1024)
- phys |= 0x20;
- }
-
- nvkm_kmap(pgd);
- nvkm_wo32(pgd, (pde * 8) + 0, lower_32_bits(phys));
- nvkm_wo32(pgd, (pde * 8) + 4, upper_32_bits(phys));
- nvkm_done(pgd);
-}
-
-static inline u64
-vm_addr(struct nvkm_vma *vma, u64 phys, u32 memtype, u32 target)
-{
- phys |= 1; /* present */
- phys |= (u64)memtype << 40;
- phys |= target << 4;
- if (vma->access & NV_MEM_ACCESS_SYS)
- phys |= (1 << 6);
- if (!(vma->access & NV_MEM_ACCESS_WO))
- phys |= (1 << 3);
- return phys;
-}
-
-static void
-nv50_vm_map(struct nvkm_vma *vma, struct nvkm_memory *pgt,
- struct nvkm_mem *mem, u32 pte, u32 cnt, u64 phys, u64 delta)
-{
- struct nvkm_ram *ram = vma->vm->mmu->subdev.device->fb->ram;
- u32 comp = (mem->memtype & 0x180) >> 7;
- u32 block, target;
- int i;
-
- /* IGPs don't have real VRAM, re-target to stolen system memory */
- target = 0;
- if (ram->stolen) {
- phys += ram->stolen;
- target = 3;
- }
-
- phys = vm_addr(vma, phys, mem->memtype, target);
- pte <<= 3;
- cnt <<= 3;
-
- nvkm_kmap(pgt);
- while (cnt) {
- u32 offset_h = upper_32_bits(phys);
- u32 offset_l = lower_32_bits(phys);
-
- for (i = 7; i >= 0; i--) {
- block = 1 << (i + 3);
- if (cnt >= block && !(pte & (block - 1)))
- break;
- }
- offset_l |= (i << 7);
-
- phys += block << (vma->node->type - 3);
- cnt -= block;
- if (comp) {
- u32 tag = mem->tag->offset + ((delta >> 16) * comp);
- offset_h |= (tag << 17);
- delta += block << (vma->node->type - 3);
- }
-
- while (block) {
- nvkm_wo32(pgt, pte + 0, offset_l);
- nvkm_wo32(pgt, pte + 4, offset_h);
- pte += 8;
- block -= 8;
- }
- }
- nvkm_done(pgt);
-}
-
-static void
-nv50_vm_map_sg(struct nvkm_vma *vma, struct nvkm_memory *pgt,
- struct nvkm_mem *mem, u32 pte, u32 cnt, dma_addr_t *list)
-{
- u32 target = (vma->access & NV_MEM_ACCESS_NOSNOOP) ? 3 : 2;
- pte <<= 3;
- nvkm_kmap(pgt);
- while (cnt--) {
- u64 phys = vm_addr(vma, (u64)*list++, mem->memtype, target);
- nvkm_wo32(pgt, pte + 0, lower_32_bits(phys));
- nvkm_wo32(pgt, pte + 4, upper_32_bits(phys));
- pte += 8;
- }
- nvkm_done(pgt);
-}
-
-static void
-nv50_vm_unmap(struct nvkm_vma *vma, struct nvkm_memory *pgt, u32 pte, u32 cnt)
-{
- pte <<= 3;
- nvkm_kmap(pgt);
- while (cnt--) {
- nvkm_wo32(pgt, pte + 0, 0x00000000);
- nvkm_wo32(pgt, pte + 4, 0x00000000);
- pte += 8;
- }
- nvkm_done(pgt);
-}
-
-static void
-nv50_vm_flush(struct nvkm_vm *vm)
-{
- struct nvkm_mmu *mmu = vm->mmu;
- struct nvkm_subdev *subdev = &mmu->subdev;
- struct nvkm_device *device = subdev->device;
- int i, vme;
-
- mutex_lock(&subdev->mutex);
- for (i = 0; i < NVKM_SUBDEV_NR; i++) {
- if (!atomic_read(&vm->engref[i]))
- continue;
-
- /* unfortunate hw bug workaround... */
- if (i == NVKM_ENGINE_GR && device->gr) {
- int ret = nvkm_gr_tlb_flush(device->gr);
- if (ret != -ENODEV)
- continue;
- }
-
- switch (i) {
- case NVKM_ENGINE_GR : vme = 0x00; break;
- case NVKM_ENGINE_VP :
- case NVKM_ENGINE_MSPDEC: vme = 0x01; break;
- case NVKM_SUBDEV_BAR : vme = 0x06; break;
- case NVKM_ENGINE_MSPPP :
- case NVKM_ENGINE_MPEG : vme = 0x08; break;
- case NVKM_ENGINE_BSP :
- case NVKM_ENGINE_MSVLD : vme = 0x09; break;
- case NVKM_ENGINE_CIPHER:
- case NVKM_ENGINE_SEC : vme = 0x0a; break;
- case NVKM_ENGINE_CE0 : vme = 0x0d; break;
- default:
- continue;
- }
-
- nvkm_wr32(device, 0x100c80, (vme << 16) | 1);
- if (nvkm_msec(device, 2000,
- if (!(nvkm_rd32(device, 0x100c80) & 0x00000001))
- break;
- ) < 0)
- nvkm_error(subdev, "vm flush timeout: engine %d\n", vme);
- }
- mutex_unlock(&subdev->mutex);
-}
-
-static int
-nv50_vm_create(struct nvkm_mmu *mmu, u64 offset, u64 length, u64 mm_offset,
- struct lock_class_key *key, struct nvkm_vm **pvm)
-{
- u32 block = (1 << (mmu->func->pgt_bits + 12));
- if (block > length)
- block = length;
-
- return nvkm_vm_create(mmu, offset, length, mm_offset, block, key, pvm);
+ /* 0x01: no bank swizzle
+ * 0x02: bank swizzled
+ * 0x7f: invalid
+ *
+ * 0x01/0x02 are values understood by the VRAM allocator,
+ * and are required to avoid mixing the two types within
+ * a certain range.
+ */
+ static const u8
+ kind[128] = {
+ 0x01, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, /* 0x00 */
+ 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f,
+ 0x01, 0x01, 0x01, 0x01, 0x7f, 0x7f, 0x7f, 0x7f, /* 0x10 */
+ 0x02, 0x02, 0x02, 0x02, 0x7f, 0x7f, 0x7f, 0x7f,
+ 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x7f, /* 0x20 */
+ 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x7f,
+ 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, /* 0x30 */
+ 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f,
+ 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x02, /* 0x40 */
+ 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x7f, 0x7f,
+ 0x7f, 0x7f, 0x7f, 0x7f, 0x01, 0x01, 0x01, 0x7f, /* 0x50 */
+ 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f,
+ 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x7f, /* 0x60 */
+ 0x01, 0x01, 0x01, 0x01, 0x02, 0x02, 0x02, 0x02,
+ 0x01, 0x7f, 0x02, 0x7f, 0x01, 0x7f, 0x02, 0x7f, /* 0x70 */
+ 0x01, 0x01, 0x02, 0x02, 0x01, 0x01, 0x7f, 0x7f
+ };
+ *count = ARRAY_SIZE(kind);
+ return kind;
}
static const struct nvkm_mmu_func
nv50_mmu = {
- .limit = (1ULL << 40),
.dma_bits = 40,
- .pgt_bits = 29 - 12,
- .spg_shift = 12,
- .lpg_shift = 16,
- .create = nv50_vm_create,
- .map_pgt = nv50_vm_map_pgt,
- .map = nv50_vm_map,
- .map_sg = nv50_vm_map_sg,
- .unmap = nv50_vm_unmap,
- .flush = nv50_vm_flush,
+ .mmu = {{ -1, -1, NVIF_CLASS_MMU_NV50}},
+ .mem = {{ -1, 0, NVIF_CLASS_MEM_NV50}, nv50_mem_new, nv50_mem_map },
+ .vmm = {{ -1, -1, NVIF_CLASS_VMM_NV50}, nv50_vmm_new, false, 0x1400 },
+ .kind = nv50_mmu_kind,
};
int
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/priv.h b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/priv.h
index 27cedc60b507..d024d8055fcb 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/priv.h
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/priv.h
@@ -9,31 +9,57 @@ int nvkm_mmu_new_(const struct nvkm_mmu_func *, struct nvkm_device *,
int index, struct nvkm_mmu **);
struct nvkm_mmu_func {
- void *(*dtor)(struct nvkm_mmu *);
- int (*oneinit)(struct nvkm_mmu *);
void (*init)(struct nvkm_mmu *);
- u64 limit;
u8 dma_bits;
- u32 pgt_bits;
- u8 spg_shift;
- u8 lpg_shift;
-
- int (*create)(struct nvkm_mmu *, u64 offset, u64 length, u64 mm_offset,
- struct lock_class_key *, struct nvkm_vm **);
-
- void (*map_pgt)(struct nvkm_gpuobj *pgd, u32 pde,
- struct nvkm_memory *pgt[2]);
- void (*map)(struct nvkm_vma *, struct nvkm_memory *,
- struct nvkm_mem *, u32 pte, u32 cnt,
- u64 phys, u64 delta);
- void (*map_sg)(struct nvkm_vma *, struct nvkm_memory *,
- struct nvkm_mem *, u32 pte, u32 cnt, dma_addr_t *);
- void (*unmap)(struct nvkm_vma *, struct nvkm_memory *pgt,
- u32 pte, u32 cnt);
- void (*flush)(struct nvkm_vm *);
+
+ struct {
+ struct nvkm_sclass user;
+ } mmu;
+
+ struct {
+ struct nvkm_sclass user;
+ int (*vram)(struct nvkm_mmu *, int type, u8 page, u64 size,
+ void *argv, u32 argc, struct nvkm_memory **);
+ int (*umap)(struct nvkm_mmu *, struct nvkm_memory *, void *argv,
+ u32 argc, u64 *addr, u64 *size, struct nvkm_vma **);
+ } mem;
+
+ struct {
+ struct nvkm_sclass user;
+ int (*ctor)(struct nvkm_mmu *, u64 addr, u64 size,
+ void *argv, u32 argc, struct lock_class_key *,
+ const char *name, struct nvkm_vmm **);
+ bool global;
+ u32 pd_offset;
+ } vmm;
+
+ const u8 *(*kind)(struct nvkm_mmu *, int *count);
+ bool kind_sys;
+};
+
+extern const struct nvkm_mmu_func nv04_mmu;
+
+const u8 *nv50_mmu_kind(struct nvkm_mmu *, int *count);
+
+const u8 *gf100_mmu_kind(struct nvkm_mmu *, int *count);
+
+const u8 *gm200_mmu_kind(struct nvkm_mmu *, int *);
+
+struct nvkm_mmu_pt {
+ union {
+ struct nvkm_mmu_ptc *ptc;
+ struct nvkm_mmu_ptp *ptp;
+ };
+ struct nvkm_memory *memory;
+ bool sub;
+ u16 base;
+ u64 addr;
+ struct list_head head;
};
-int nvkm_vm_create(struct nvkm_mmu *, u64, u64, u64, u32,
- struct lock_class_key *, struct nvkm_vm **);
+void nvkm_mmu_ptc_dump(struct nvkm_mmu *);
+struct nvkm_mmu_pt *
+nvkm_mmu_ptc_get(struct nvkm_mmu *, u32 size, u32 align, bool zero);
+void nvkm_mmu_ptc_put(struct nvkm_mmu *, bool force, struct nvkm_mmu_pt **);
#endif
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/umem.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/umem.c
new file mode 100644
index 000000000000..fac2f9a45ea6
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/umem.c
@@ -0,0 +1,192 @@
+/*
+ * Copyright 2017 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+#include "umem.h"
+#include "ummu.h"
+
+#include <core/client.h>
+#include <core/memory.h>
+#include <subdev/bar.h>
+
+#include <nvif/class.h>
+#include <nvif/if000a.h>
+#include <nvif/unpack.h>
+
+static const struct nvkm_object_func nvkm_umem;
+struct nvkm_memory *
+nvkm_umem_search(struct nvkm_client *client, u64 handle)
+{
+ struct nvkm_client *master = client->object.client;
+ struct nvkm_memory *memory = NULL;
+ struct nvkm_object *object;
+ struct nvkm_umem *umem;
+
+ object = nvkm_object_search(client, handle, &nvkm_umem);
+ if (IS_ERR(object)) {
+ if (client->super && client != master) {
+ spin_lock(&master->lock);
+ list_for_each_entry(umem, &master->umem, head) {
+ if (umem->object.object == handle) {
+ memory = nvkm_memory_ref(umem->memory);
+ break;
+ }
+ }
+ spin_unlock(&master->lock);
+ }
+ } else {
+ umem = nvkm_umem(object);
+ if (!umem->priv || client->super)
+ memory = nvkm_memory_ref(umem->memory);
+ }
+
+ return memory ? memory : ERR_PTR(-ENOENT);
+}
+
+static int
+nvkm_umem_unmap(struct nvkm_object *object)
+{
+ struct nvkm_umem *umem = nvkm_umem(object);
+
+ if (!umem->map)
+ return -EEXIST;
+
+ if (umem->io) {
+ if (!IS_ERR(umem->bar)) {
+ struct nvkm_device *device = umem->mmu->subdev.device;
+ nvkm_vmm_put(nvkm_bar_bar1_vmm(device), &umem->bar);
+ } else {
+ umem->bar = NULL;
+ }
+ } else {
+ vunmap(umem->map);
+ umem->map = NULL;
+ }
+
+ return 0;
+}
+
+static int
+nvkm_umem_map(struct nvkm_object *object, void *argv, u32 argc,
+ enum nvkm_object_map *type, u64 *handle, u64 *length)
+{
+ struct nvkm_umem *umem = nvkm_umem(object);
+ struct nvkm_mmu *mmu = umem->mmu;
+
+ if (!umem->mappable)
+ return -EINVAL;
+ if (umem->map)
+ return -EEXIST;
+
+ if ((umem->type & NVKM_MEM_HOST) && !argc) {
+ int ret = nvkm_mem_map_host(umem->memory, &umem->map);
+ if (ret)
+ return ret;
+
+ *handle = (unsigned long)(void *)umem->map;
+ *length = nvkm_memory_size(umem->memory);
+ *type = NVKM_OBJECT_MAP_VA;
+ return 0;
+ } else
+ if ((umem->type & NVKM_MEM_VRAM) ||
+ (umem->type & NVKM_MEM_KIND)) {
+ int ret = mmu->func->mem.umap(mmu, umem->memory, argv, argc,
+ handle, length, &umem->bar);
+ if (ret)
+ return ret;
+
+ *type = NVKM_OBJECT_MAP_IO;
+ } else {
+ return -EINVAL;
+ }
+
+ umem->io = (*type == NVKM_OBJECT_MAP_IO);
+ return 0;
+}
+
+static void *
+nvkm_umem_dtor(struct nvkm_object *object)
+{
+ struct nvkm_umem *umem = nvkm_umem(object);
+ spin_lock(&umem->object.client->lock);
+ list_del_init(&umem->head);
+ spin_unlock(&umem->object.client->lock);
+ nvkm_memory_unref(&umem->memory);
+ return umem;
+}
+
+static const struct nvkm_object_func
+nvkm_umem = {
+ .dtor = nvkm_umem_dtor,
+ .map = nvkm_umem_map,
+ .unmap = nvkm_umem_unmap,
+};
+
+int
+nvkm_umem_new(const struct nvkm_oclass *oclass, void *argv, u32 argc,
+ struct nvkm_object **pobject)
+{
+ struct nvkm_mmu *mmu = nvkm_ummu(oclass->parent)->mmu;
+ union {
+ struct nvif_mem_v0 v0;
+ } *args = argv;
+ struct nvkm_umem *umem;
+ int type, ret = -ENOSYS;
+ u8 page;
+ u64 size;
+
+ if (!(ret = nvif_unpack(ret, &argv, &argc, args->v0, 0, 0, true))) {
+ type = args->v0.type;
+ page = args->v0.page;
+ size = args->v0.size;
+ } else
+ return ret;
+
+ if (type >= mmu->type_nr)
+ return -EINVAL;
+
+ if (!(umem = kzalloc(sizeof(*umem), GFP_KERNEL)))
+ return -ENOMEM;
+ nvkm_object_ctor(&nvkm_umem, oclass, &umem->object);
+ umem->mmu = mmu;
+ umem->type = mmu->type[type].type;
+ umem->priv = oclass->client->super;
+ INIT_LIST_HEAD(&umem->head);
+ *pobject = &umem->object;
+
+ if (mmu->type[type].type & NVKM_MEM_MAPPABLE) {
+ page = max_t(u8, page, PAGE_SHIFT);
+ umem->mappable = true;
+ }
+
+ ret = nvkm_mem_new_type(mmu, type, page, size, argv, argc,
+ &umem->memory);
+ if (ret)
+ return ret;
+
+ spin_lock(&umem->object.client->lock);
+ list_add(&umem->head, &umem->object.client->umem);
+ spin_unlock(&umem->object.client->lock);
+
+ args->v0.page = nvkm_memory_page(umem->memory);
+ args->v0.addr = nvkm_memory_addr(umem->memory);
+ args->v0.size = nvkm_memory_size(umem->memory);
+ return 0;
+}
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/umem.h b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/umem.h
new file mode 100644
index 000000000000..85cf692d620a
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/umem.h
@@ -0,0 +1,26 @@
+#ifndef __NVKM_UMEM_H__
+#define __NVKM_UMEM_H__
+#define nvkm_umem(p) container_of((p), struct nvkm_umem, object)
+#include <core/object.h>
+#include "mem.h"
+
+struct nvkm_umem {
+ struct nvkm_object object;
+ struct nvkm_mmu *mmu;
+ u8 type:8;
+ bool priv:1;
+ bool mappable:1;
+ bool io:1;
+
+ struct nvkm_memory *memory;
+ struct list_head head;
+
+ union {
+ struct nvkm_vma *bar;
+ void *map;
+ };
+};
+
+int nvkm_umem_new(const struct nvkm_oclass *, void *argv, u32 argc,
+ struct nvkm_object **);
+#endif
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/ummu.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/ummu.c
new file mode 100644
index 000000000000..353f10f92b77
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/ummu.c
@@ -0,0 +1,178 @@
+/*
+ * Copyright 2017 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+#include "ummu.h"
+#include "umem.h"
+#include "uvmm.h"
+
+#include <core/client.h>
+
+#include <nvif/if0008.h>
+#include <nvif/unpack.h>
+
+static int
+nvkm_ummu_sclass(struct nvkm_object *object, int index,
+ struct nvkm_oclass *oclass)
+{
+ struct nvkm_mmu *mmu = nvkm_ummu(object)->mmu;
+
+ if (mmu->func->mem.user.oclass && oclass->client->super) {
+ if (index-- == 0) {
+ oclass->base = mmu->func->mem.user;
+ oclass->ctor = nvkm_umem_new;
+ return 0;
+ }
+ }
+
+ if (mmu->func->vmm.user.oclass) {
+ if (index-- == 0) {
+ oclass->base = mmu->func->vmm.user;
+ oclass->ctor = nvkm_uvmm_new;
+ return 0;
+ }
+ }
+
+ return -EINVAL;
+}
+
+static int
+nvkm_ummu_heap(struct nvkm_ummu *ummu, void *argv, u32 argc)
+{
+ struct nvkm_mmu *mmu = ummu->mmu;
+ union {
+ struct nvif_mmu_heap_v0 v0;
+ } *args = argv;
+ int ret = -ENOSYS;
+ u8 index;
+
+ if (!(ret = nvif_unpack(ret, &argv, &argc, args->v0, 0, 0, false))) {
+ if ((index = args->v0.index) >= mmu->heap_nr)
+ return -EINVAL;
+ args->v0.size = mmu->heap[index].size;
+ } else
+ return ret;
+
+ return 0;
+}
+
+static int
+nvkm_ummu_type(struct nvkm_ummu *ummu, void *argv, u32 argc)
+{
+ struct nvkm_mmu *mmu = ummu->mmu;
+ union {
+ struct nvif_mmu_type_v0 v0;
+ } *args = argv;
+ int ret = -ENOSYS;
+ u8 type, index;
+
+ if (!(ret = nvif_unpack(ret, &argv, &argc, args->v0, 0, 0, false))) {
+ if ((index = args->v0.index) >= mmu->type_nr)
+ return -EINVAL;
+ type = mmu->type[index].type;
+ args->v0.heap = mmu->type[index].heap;
+ args->v0.vram = !!(type & NVKM_MEM_VRAM);
+ args->v0.host = !!(type & NVKM_MEM_HOST);
+ args->v0.comp = !!(type & NVKM_MEM_COMP);
+ args->v0.disp = !!(type & NVKM_MEM_DISP);
+ args->v0.kind = !!(type & NVKM_MEM_KIND);
+ args->v0.mappable = !!(type & NVKM_MEM_MAPPABLE);
+ args->v0.coherent = !!(type & NVKM_MEM_COHERENT);
+ args->v0.uncached = !!(type & NVKM_MEM_UNCACHED);
+ } else
+ return ret;
+
+ return 0;
+}
+
+static int
+nvkm_ummu_kind(struct nvkm_ummu *ummu, void *argv, u32 argc)
+{
+ struct nvkm_mmu *mmu = ummu->mmu;
+ union {
+ struct nvif_mmu_kind_v0 v0;
+ } *args = argv;
+ const u8 *kind = NULL;
+ int ret = -ENOSYS, count = 0;
+
+ if (mmu->func->kind)
+ kind = mmu->func->kind(mmu, &count);
+
+ if (!(ret = nvif_unpack(ret, &argv, &argc, args->v0, 0, 0, true))) {
+ if (argc != args->v0.count * sizeof(*args->v0.data))
+ return -EINVAL;
+ if (args->v0.count > count)
+ return -EINVAL;
+ memcpy(args->v0.data, kind, args->v0.count);
+ } else
+ return ret;
+
+ return 0;
+}
+
+static int
+nvkm_ummu_mthd(struct nvkm_object *object, u32 mthd, void *argv, u32 argc)
+{
+ struct nvkm_ummu *ummu = nvkm_ummu(object);
+ switch (mthd) {
+ case NVIF_MMU_V0_HEAP: return nvkm_ummu_heap(ummu, argv, argc);
+ case NVIF_MMU_V0_TYPE: return nvkm_ummu_type(ummu, argv, argc);
+ case NVIF_MMU_V0_KIND: return nvkm_ummu_kind(ummu, argv, argc);
+ default:
+ break;
+ }
+ return -EINVAL;
+}
+
+static const struct nvkm_object_func
+nvkm_ummu = {
+ .mthd = nvkm_ummu_mthd,
+ .sclass = nvkm_ummu_sclass,
+};
+
+int
+nvkm_ummu_new(struct nvkm_device *device, const struct nvkm_oclass *oclass,
+ void *argv, u32 argc, struct nvkm_object **pobject)
+{
+ union {
+ struct nvif_mmu_v0 v0;
+ } *args = argv;
+ struct nvkm_mmu *mmu = device->mmu;
+ struct nvkm_ummu *ummu;
+ int ret = -ENOSYS, kinds = 0;
+
+ if (mmu->func->kind)
+ mmu->func->kind(mmu, &kinds);
+
+ if (!(ret = nvif_unpack(ret, &argv, &argc, args->v0, 0, 0, false))) {
+ args->v0.dmabits = mmu->dma_bits;
+ args->v0.heap_nr = mmu->heap_nr;
+ args->v0.type_nr = mmu->type_nr;
+ args->v0.kind_nr = kinds;
+ } else
+ return ret;
+
+ if (!(ummu = kzalloc(sizeof(*ummu), GFP_KERNEL)))
+ return -ENOMEM;
+ nvkm_object_ctor(&nvkm_ummu, oclass, &ummu->object);
+ ummu->mmu = mmu;
+ *pobject = &ummu->object;
+ return 0;
+}
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/ummu.h b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/ummu.h
new file mode 100644
index 000000000000..0cd510dcfc68
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/ummu.h
@@ -0,0 +1,14 @@
+#ifndef __NVKM_UMMU_H__
+#define __NVKM_UMMU_H__
+#define nvkm_ummu(p) container_of((p), struct nvkm_ummu, object)
+#include <core/object.h>
+#include "priv.h"
+
+struct nvkm_ummu {
+ struct nvkm_object object;
+ struct nvkm_mmu *mmu;
+};
+
+int nvkm_ummu_new(struct nvkm_device *, const struct nvkm_oclass *,
+ void *argv, u32 argc, struct nvkm_object **);
+#endif
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/uvmm.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/uvmm.c
new file mode 100644
index 000000000000..fa81d0c1ba41
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/uvmm.c
@@ -0,0 +1,352 @@
+/*
+ * Copyright 2017 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+#include "uvmm.h"
+#include "umem.h"
+#include "ummu.h"
+
+#include <core/client.h>
+#include <core/memory.h>
+
+#include <nvif/if000c.h>
+#include <nvif/unpack.h>
+
+static const struct nvkm_object_func nvkm_uvmm;
+struct nvkm_vmm *
+nvkm_uvmm_search(struct nvkm_client *client, u64 handle)
+{
+ struct nvkm_object *object;
+
+ object = nvkm_object_search(client, handle, &nvkm_uvmm);
+ if (IS_ERR(object))
+ return (void *)object;
+
+ return nvkm_uvmm(object)->vmm;
+}
+
+static int
+nvkm_uvmm_mthd_unmap(struct nvkm_uvmm *uvmm, void *argv, u32 argc)
+{
+ struct nvkm_client *client = uvmm->object.client;
+ union {
+ struct nvif_vmm_unmap_v0 v0;
+ } *args = argv;
+ struct nvkm_vmm *vmm = uvmm->vmm;
+ struct nvkm_vma *vma;
+ int ret = -ENOSYS;
+ u64 addr;
+
+ if (!(ret = nvif_unpack(ret, &argv, &argc, args->v0, 0, 0, false))) {
+ addr = args->v0.addr;
+ } else
+ return ret;
+
+ mutex_lock(&vmm->mutex);
+ vma = nvkm_vmm_node_search(vmm, addr);
+ if (ret = -ENOENT, !vma || vma->addr != addr) {
+ VMM_DEBUG(vmm, "lookup %016llx: %016llx",
+ addr, vma ? vma->addr : ~0ULL);
+ goto done;
+ }
+
+ if (ret = -ENOENT, (!vma->user && !client->super) || vma->busy) {
+ VMM_DEBUG(vmm, "denied %016llx: %d %d %d", addr,
+ vma->user, !client->super, vma->busy);
+ goto done;
+ }
+
+ if (ret = -EINVAL, !vma->memory) {
+ VMM_DEBUG(vmm, "unmapped");
+ goto done;
+ }
+
+ nvkm_vmm_unmap_locked(vmm, vma);
+ ret = 0;
+done:
+ mutex_unlock(&vmm->mutex);
+ return ret;
+}
+
+static int
+nvkm_uvmm_mthd_map(struct nvkm_uvmm *uvmm, void *argv, u32 argc)
+{
+ struct nvkm_client *client = uvmm->object.client;
+ union {
+ struct nvif_vmm_map_v0 v0;
+ } *args = argv;
+ u64 addr, size, handle, offset;
+ struct nvkm_vmm *vmm = uvmm->vmm;
+ struct nvkm_vma *vma;
+ struct nvkm_memory *memory;
+ int ret = -ENOSYS;
+
+ if (!(ret = nvif_unpack(ret, &argv, &argc, args->v0, 0, 0, true))) {
+ addr = args->v0.addr;
+ size = args->v0.size;
+ handle = args->v0.memory;
+ offset = args->v0.offset;
+ } else
+ return ret;
+
+ if (IS_ERR((memory = nvkm_umem_search(client, handle)))) {
+ VMM_DEBUG(vmm, "memory %016llx %ld\n", handle, PTR_ERR(memory));
+ return PTR_ERR(memory);
+ }
+
+ mutex_lock(&vmm->mutex);
+ if (ret = -ENOENT, !(vma = nvkm_vmm_node_search(vmm, addr))) {
+ VMM_DEBUG(vmm, "lookup %016llx", addr);
+ goto fail;
+ }
+
+ if (ret = -ENOENT, (!vma->user && !client->super) || vma->busy) {
+ VMM_DEBUG(vmm, "denied %016llx: %d %d %d", addr,
+ vma->user, !client->super, vma->busy);
+ goto fail;
+ }
+
+ if (ret = -EINVAL, vma->addr != addr || vma->size != size) {
+ if (addr + size > vma->addr + vma->size || vma->memory ||
+ (vma->refd == NVKM_VMA_PAGE_NONE && !vma->mapref)) {
+ VMM_DEBUG(vmm, "split %d %d %d "
+ "%016llx %016llx %016llx %016llx",
+ !!vma->memory, vma->refd, vma->mapref,
+ addr, size, vma->addr, (u64)vma->size);
+ goto fail;
+ }
+
+ if (vma->addr != addr) {
+ const u64 tail = vma->size + vma->addr - addr;
+ if (ret = -ENOMEM, !(vma = nvkm_vma_tail(vma, tail)))
+ goto fail;
+ vma->part = true;
+ nvkm_vmm_node_insert(vmm, vma);
+ }
+
+ if (vma->size != size) {
+ const u64 tail = vma->size - size;
+ struct nvkm_vma *tmp;
+ if (ret = -ENOMEM, !(tmp = nvkm_vma_tail(vma, tail))) {
+ nvkm_vmm_unmap_region(vmm, vma);
+ goto fail;
+ }
+ tmp->part = true;
+ nvkm_vmm_node_insert(vmm, tmp);
+ }
+ }
+ vma->busy = true;
+ mutex_unlock(&vmm->mutex);
+
+ ret = nvkm_memory_map(memory, offset, vmm, vma, argv, argc);
+ if (ret == 0) {
+ /* Successful map will clear vma->busy. */
+ nvkm_memory_unref(&memory);
+ return 0;
+ }
+
+ mutex_lock(&vmm->mutex);
+ vma->busy = false;
+ nvkm_vmm_unmap_region(vmm, vma);
+fail:
+ mutex_unlock(&vmm->mutex);
+ nvkm_memory_unref(&memory);
+ return ret;
+}
+
+static int
+nvkm_uvmm_mthd_put(struct nvkm_uvmm *uvmm, void *argv, u32 argc)
+{
+ struct nvkm_client *client = uvmm->object.client;
+ union {
+ struct nvif_vmm_put_v0 v0;
+ } *args = argv;
+ struct nvkm_vmm *vmm = uvmm->vmm;
+ struct nvkm_vma *vma;
+ int ret = -ENOSYS;
+ u64 addr;
+
+ if (!(ret = nvif_unpack(ret, &argv, &argc, args->v0, 0, 0, false))) {
+ addr = args->v0.addr;
+ } else
+ return ret;
+
+ mutex_lock(&vmm->mutex);
+ vma = nvkm_vmm_node_search(vmm, args->v0.addr);
+ if (ret = -ENOENT, !vma || vma->addr != addr || vma->part) {
+ VMM_DEBUG(vmm, "lookup %016llx: %016llx %d", addr,
+ vma ? vma->addr : ~0ULL, vma ? vma->part : 0);
+ goto done;
+ }
+
+ if (ret = -ENOENT, (!vma->user && !client->super) || vma->busy) {
+ VMM_DEBUG(vmm, "denied %016llx: %d %d %d", addr,
+ vma->user, !client->super, vma->busy);
+ goto done;
+ }
+
+ nvkm_vmm_put_locked(vmm, vma);
+ ret = 0;
+done:
+ mutex_unlock(&vmm->mutex);
+ return ret;
+}
+
+static int
+nvkm_uvmm_mthd_get(struct nvkm_uvmm *uvmm, void *argv, u32 argc)
+{
+ struct nvkm_client *client = uvmm->object.client;
+ union {
+ struct nvif_vmm_get_v0 v0;
+ } *args = argv;
+ struct nvkm_vmm *vmm = uvmm->vmm;
+ struct nvkm_vma *vma;
+ int ret = -ENOSYS;
+ bool getref, mapref, sparse;
+ u8 page, align;
+ u64 size;
+
+ if (!(ret = nvif_unpack(ret, &argv, &argc, args->v0, 0, 0, false))) {
+ getref = args->v0.type == NVIF_VMM_GET_V0_PTES;
+ mapref = args->v0.type == NVIF_VMM_GET_V0_ADDR;
+ sparse = args->v0.sparse;
+ page = args->v0.page;
+ align = args->v0.align;
+ size = args->v0.size;
+ } else
+ return ret;
+
+ mutex_lock(&vmm->mutex);
+ ret = nvkm_vmm_get_locked(vmm, getref, mapref, sparse,
+ page, align, size, &vma);
+ mutex_unlock(&vmm->mutex);
+ if (ret)
+ return ret;
+
+ args->v0.addr = vma->addr;
+ vma->user = !client->super;
+ return ret;
+}
+
+static int
+nvkm_uvmm_mthd_page(struct nvkm_uvmm *uvmm, void *argv, u32 argc)
+{
+ union {
+ struct nvif_vmm_page_v0 v0;
+ } *args = argv;
+ const struct nvkm_vmm_page *page;
+ int ret = -ENOSYS;
+ u8 type, index, nr;
+
+ page = uvmm->vmm->func->page;
+ for (nr = 0; page[nr].shift; nr++);
+
+ if (!(ret = nvif_unpack(ret, &argv, &argc, args->v0, 0, 0, false))) {
+ if ((index = args->v0.index) >= nr)
+ return -EINVAL;
+ type = page[index].type;
+ args->v0.shift = page[index].shift;
+ args->v0.sparse = !!(type & NVKM_VMM_PAGE_SPARSE);
+ args->v0.vram = !!(type & NVKM_VMM_PAGE_VRAM);
+ args->v0.host = !!(type & NVKM_VMM_PAGE_HOST);
+ args->v0.comp = !!(type & NVKM_VMM_PAGE_COMP);
+ } else
+ return -ENOSYS;
+
+ return 0;
+}
+
+static int
+nvkm_uvmm_mthd(struct nvkm_object *object, u32 mthd, void *argv, u32 argc)
+{
+ struct nvkm_uvmm *uvmm = nvkm_uvmm(object);
+ switch (mthd) {
+ case NVIF_VMM_V0_PAGE : return nvkm_uvmm_mthd_page (uvmm, argv, argc);
+ case NVIF_VMM_V0_GET : return nvkm_uvmm_mthd_get (uvmm, argv, argc);
+ case NVIF_VMM_V0_PUT : return nvkm_uvmm_mthd_put (uvmm, argv, argc);
+ case NVIF_VMM_V0_MAP : return nvkm_uvmm_mthd_map (uvmm, argv, argc);
+ case NVIF_VMM_V0_UNMAP : return nvkm_uvmm_mthd_unmap (uvmm, argv, argc);
+ default:
+ break;
+ }
+ return -EINVAL;
+}
+
+static void *
+nvkm_uvmm_dtor(struct nvkm_object *object)
+{
+ struct nvkm_uvmm *uvmm = nvkm_uvmm(object);
+ nvkm_vmm_unref(&uvmm->vmm);
+ return uvmm;
+}
+
+static const struct nvkm_object_func
+nvkm_uvmm = {
+ .dtor = nvkm_uvmm_dtor,
+ .mthd = nvkm_uvmm_mthd,
+};
+
+int
+nvkm_uvmm_new(const struct nvkm_oclass *oclass, void *argv, u32 argc,
+ struct nvkm_object **pobject)
+{
+ struct nvkm_mmu *mmu = nvkm_ummu(oclass->parent)->mmu;
+ const bool more = oclass->base.maxver >= 0;
+ union {
+ struct nvif_vmm_v0 v0;
+ } *args = argv;
+ const struct nvkm_vmm_page *page;
+ struct nvkm_uvmm *uvmm;
+ int ret = -ENOSYS;
+ u64 addr, size;
+
+ if (!(ret = nvif_unpack(ret, &argv, &argc, args->v0, 0, 0, more))) {
+ addr = args->v0.addr;
+ size = args->v0.size;
+ } else
+ return ret;
+
+ if (!(uvmm = kzalloc(sizeof(*uvmm), GFP_KERNEL)))
+ return -ENOMEM;
+ nvkm_object_ctor(&nvkm_uvmm, oclass, &uvmm->object);
+ *pobject = &uvmm->object;
+
+ if (!mmu->vmm) {
+ ret = mmu->func->vmm.ctor(mmu, addr, size, argv, argc,
+ NULL, "user", &uvmm->vmm);
+ if (ret)
+ return ret;
+
+ uvmm->vmm->debug = max(uvmm->vmm->debug, oclass->client->debug);
+ } else {
+ if (size)
+ return -EINVAL;
+
+ uvmm->vmm = nvkm_vmm_ref(mmu->vmm);
+ }
+
+ page = uvmm->vmm->func->page;
+ args->v0.page_nr = 0;
+ while (page && (page++)->shift)
+ args->v0.page_nr++;
+ args->v0.addr = uvmm->vmm->start;
+ args->v0.size = uvmm->vmm->limit;
+ return 0;
+}
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/uvmm.h b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/uvmm.h
new file mode 100644
index 000000000000..71dab55e18a9
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/uvmm.h
@@ -0,0 +1,14 @@
+#ifndef __NVKM_UVMM_H__
+#define __NVKM_UVMM_H__
+#define nvkm_uvmm(p) container_of((p), struct nvkm_uvmm, object)
+#include <core/object.h>
+#include "vmm.h"
+
+struct nvkm_uvmm {
+ struct nvkm_object object;
+ struct nvkm_vmm *vmm;
+};
+
+int nvkm_uvmm_new(const struct nvkm_oclass *, void *argv, u32 argc,
+ struct nvkm_object **);
+#endif
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c
new file mode 100644
index 000000000000..e35d3e17cd7c
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c
@@ -0,0 +1,1513 @@
+/*
+ * Copyright 2017 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+#define NVKM_VMM_LEVELS_MAX 5
+#include "vmm.h"
+
+#include <subdev/fb.h>
+
+static void
+nvkm_vmm_pt_del(struct nvkm_vmm_pt **ppgt)
+{
+ struct nvkm_vmm_pt *pgt = *ppgt;
+ if (pgt) {
+ kvfree(pgt->pde);
+ kfree(pgt);
+ *ppgt = NULL;
+ }
+}
+
+
+static struct nvkm_vmm_pt *
+nvkm_vmm_pt_new(const struct nvkm_vmm_desc *desc, bool sparse,
+ const struct nvkm_vmm_page *page)
+{
+ const u32 pten = 1 << desc->bits;
+ struct nvkm_vmm_pt *pgt;
+ u32 lpte = 0;
+
+ if (desc->type > PGT) {
+ if (desc->type == SPT) {
+ const struct nvkm_vmm_desc *pair = page[-1].desc;
+ lpte = pten >> (desc->bits - pair->bits);
+ } else {
+ lpte = pten;
+ }
+ }
+
+ if (!(pgt = kzalloc(sizeof(*pgt) + lpte, GFP_KERNEL)))
+ return NULL;
+ pgt->page = page ? page->shift : 0;
+ pgt->sparse = sparse;
+
+ if (desc->type == PGD) {
+ pgt->pde = kvzalloc(sizeof(*pgt->pde) * pten, GFP_KERNEL);
+ if (!pgt->pde) {
+ kfree(pgt);
+ return NULL;
+ }
+ }
+
+ return pgt;
+}
+
+struct nvkm_vmm_iter {
+ const struct nvkm_vmm_page *page;
+ const struct nvkm_vmm_desc *desc;
+ struct nvkm_vmm *vmm;
+ u64 cnt;
+ u16 max, lvl;
+ u32 pte[NVKM_VMM_LEVELS_MAX];
+ struct nvkm_vmm_pt *pt[NVKM_VMM_LEVELS_MAX];
+ int flush;
+};
+
+#ifdef CONFIG_NOUVEAU_DEBUG_MMU
+static const char *
+nvkm_vmm_desc_type(const struct nvkm_vmm_desc *desc)
+{
+ switch (desc->type) {
+ case PGD: return "PGD";
+ case PGT: return "PGT";
+ case SPT: return "SPT";
+ case LPT: return "LPT";
+ default:
+ return "UNKNOWN";
+ }
+}
+
+static void
+nvkm_vmm_trace(struct nvkm_vmm_iter *it, char *buf)
+{
+ int lvl;
+ for (lvl = it->max; lvl >= 0; lvl--) {
+ if (lvl >= it->lvl)
+ buf += sprintf(buf, "%05x:", it->pte[lvl]);
+ else
+ buf += sprintf(buf, "xxxxx:");
+ }
+}
+
+#define TRA(i,f,a...) do { \
+ char _buf[NVKM_VMM_LEVELS_MAX * 7]; \
+ struct nvkm_vmm_iter *_it = (i); \
+ nvkm_vmm_trace(_it, _buf); \
+ VMM_TRACE(_it->vmm, "%s "f, _buf, ##a); \
+} while(0)
+#else
+#define TRA(i,f,a...)
+#endif
+
+static inline void
+nvkm_vmm_flush_mark(struct nvkm_vmm_iter *it)
+{
+ it->flush = min(it->flush, it->max - it->lvl);
+}
+
+static inline void
+nvkm_vmm_flush(struct nvkm_vmm_iter *it)
+{
+ if (it->flush != NVKM_VMM_LEVELS_MAX) {
+ if (it->vmm->func->flush) {
+ TRA(it, "flush: %d", it->flush);
+ it->vmm->func->flush(it->vmm, it->flush);
+ }
+ it->flush = NVKM_VMM_LEVELS_MAX;
+ }
+}
+
+static void
+nvkm_vmm_unref_pdes(struct nvkm_vmm_iter *it)
+{
+ const struct nvkm_vmm_desc *desc = it->desc;
+ const int type = desc[it->lvl].type == SPT;
+ struct nvkm_vmm_pt *pgd = it->pt[it->lvl + 1];
+ struct nvkm_vmm_pt *pgt = it->pt[it->lvl];
+ struct nvkm_mmu_pt *pt = pgt->pt[type];
+ struct nvkm_vmm *vmm = it->vmm;
+ u32 pdei = it->pte[it->lvl + 1];
+
+ /* Recurse up the tree, unreferencing/destroying unneeded PDs. */
+ it->lvl++;
+ if (--pgd->refs[0]) {
+ const struct nvkm_vmm_desc_func *func = desc[it->lvl].func;
+ /* PD has other valid PDEs, so we need a proper update. */
+ TRA(it, "PDE unmap %s", nvkm_vmm_desc_type(&desc[it->lvl - 1]));
+ pgt->pt[type] = NULL;
+ if (!pgt->refs[!type]) {
+ /* PDE no longer required. */
+ if (pgd->pt[0]) {
+ if (pgt->sparse) {
+ func->sparse(vmm, pgd->pt[0], pdei, 1);
+ pgd->pde[pdei] = NVKM_VMM_PDE_SPARSE;
+ } else {
+ func->unmap(vmm, pgd->pt[0], pdei, 1);
+ pgd->pde[pdei] = NULL;
+ }
+ } else {
+ /* Special handling for Tesla-class GPUs,
+ * where there's no central PD, but each
+ * instance has its own embedded PD.
+ */
+ func->pde(vmm, pgd, pdei);
+ pgd->pde[pdei] = NULL;
+ }
+ } else {
+ /* PDE was pointing at dual-PTs and we're removing
+ * one of them, leaving the other in place.
+ */
+ func->pde(vmm, pgd, pdei);
+ }
+
+ /* GPU may have cached the PTs, flush before freeing. */
+ nvkm_vmm_flush_mark(it);
+ nvkm_vmm_flush(it);
+ } else {
+ /* PD has no valid PDEs left, so we can just destroy it. */
+ nvkm_vmm_unref_pdes(it);
+ }
+
+ /* Destroy PD/PT. */
+ TRA(it, "PDE free %s", nvkm_vmm_desc_type(&desc[it->lvl - 1]));
+ nvkm_mmu_ptc_put(vmm->mmu, vmm->bootstrapped, &pt);
+ if (!pgt->refs[!type])
+ nvkm_vmm_pt_del(&pgt);
+ it->lvl--;
+}
+
+static void
+nvkm_vmm_unref_sptes(struct nvkm_vmm_iter *it, struct nvkm_vmm_pt *pgt,
+ const struct nvkm_vmm_desc *desc, u32 ptei, u32 ptes)
+{
+ const struct nvkm_vmm_desc *pair = it->page[-1].desc;
+ const u32 sptb = desc->bits - pair->bits;
+ const u32 sptn = 1 << sptb;
+ struct nvkm_vmm *vmm = it->vmm;
+ u32 spti = ptei & (sptn - 1), lpti, pteb;
+
+ /* Determine how many SPTEs are being touched under each LPTE,
+ * and drop reference counts.
+ */
+ for (lpti = ptei >> sptb; ptes; spti = 0, lpti++) {
+ const u32 pten = min(sptn - spti, ptes);
+ pgt->pte[lpti] -= pten;
+ ptes -= pten;
+ }
+
+ /* We're done here if there's no corresponding LPT. */
+ if (!pgt->refs[0])
+ return;
+
+ for (ptei = pteb = ptei >> sptb; ptei < lpti; pteb = ptei) {
+ /* Skip over any LPTEs that still have valid SPTEs. */
+ if (pgt->pte[pteb] & NVKM_VMM_PTE_SPTES) {
+ for (ptes = 1, ptei++; ptei < lpti; ptes++, ptei++) {
+ if (!(pgt->pte[ptei] & NVKM_VMM_PTE_SPTES))
+ break;
+ }
+ continue;
+ }
+
+ /* As there's no more non-UNMAPPED SPTEs left in the range
+ * covered by a number of LPTEs, the LPTEs once again take
+ * control over their address range.
+ *
+ * Determine how many LPTEs need to transition state.
+ */
+ pgt->pte[ptei] &= ~NVKM_VMM_PTE_VALID;
+ for (ptes = 1, ptei++; ptei < lpti; ptes++, ptei++) {
+ if (pgt->pte[ptei] & NVKM_VMM_PTE_SPTES)
+ break;
+ pgt->pte[ptei] &= ~NVKM_VMM_PTE_VALID;
+ }
+
+ if (pgt->pte[pteb] & NVKM_VMM_PTE_SPARSE) {
+ TRA(it, "LPTE %05x: U -> S %d PTEs", pteb, ptes);
+ pair->func->sparse(vmm, pgt->pt[0], pteb, ptes);
+ } else
+ if (pair->func->invalid) {
+ /* If the MMU supports it, restore the LPTE to the
+ * INVALID state to tell the MMU there is no point
+ * trying to fetch the corresponding SPTEs.
+ */
+ TRA(it, "LPTE %05x: U -> I %d PTEs", pteb, ptes);
+ pair->func->invalid(vmm, pgt->pt[0], pteb, ptes);
+ }
+ }
+}
+
+static bool
+nvkm_vmm_unref_ptes(struct nvkm_vmm_iter *it, u32 ptei, u32 ptes)
+{
+ const struct nvkm_vmm_desc *desc = it->desc;
+ const int type = desc->type == SPT;
+ struct nvkm_vmm_pt *pgt = it->pt[0];
+
+ /* Drop PTE references. */
+ pgt->refs[type] -= ptes;
+
+ /* Dual-PTs need special handling, unless PDE becoming invalid. */
+ if (desc->type == SPT && (pgt->refs[0] || pgt->refs[1]))
+ nvkm_vmm_unref_sptes(it, pgt, desc, ptei, ptes);
+
+ /* PT no longer neeed? Destroy it. */
+ if (!pgt->refs[type]) {
+ it->lvl++;
+ TRA(it, "%s empty", nvkm_vmm_desc_type(desc));
+ it->lvl--;
+ nvkm_vmm_unref_pdes(it);
+ return false; /* PTE writes for unmap() not necessary. */
+ }
+
+ return true;
+}
+
+static void
+nvkm_vmm_ref_sptes(struct nvkm_vmm_iter *it, struct nvkm_vmm_pt *pgt,
+ const struct nvkm_vmm_desc *desc, u32 ptei, u32 ptes)
+{
+ const struct nvkm_vmm_desc *pair = it->page[-1].desc;
+ const u32 sptb = desc->bits - pair->bits;
+ const u32 sptn = 1 << sptb;
+ struct nvkm_vmm *vmm = it->vmm;
+ u32 spti = ptei & (sptn - 1), lpti, pteb;
+
+ /* Determine how many SPTEs are being touched under each LPTE,
+ * and increase reference counts.
+ */
+ for (lpti = ptei >> sptb; ptes; spti = 0, lpti++) {
+ const u32 pten = min(sptn - spti, ptes);
+ pgt->pte[lpti] += pten;
+ ptes -= pten;
+ }
+
+ /* We're done here if there's no corresponding LPT. */
+ if (!pgt->refs[0])
+ return;
+
+ for (ptei = pteb = ptei >> sptb; ptei < lpti; pteb = ptei) {
+ /* Skip over any LPTEs that already have valid SPTEs. */
+ if (pgt->pte[pteb] & NVKM_VMM_PTE_VALID) {
+ for (ptes = 1, ptei++; ptei < lpti; ptes++, ptei++) {
+ if (!(pgt->pte[ptei] & NVKM_VMM_PTE_VALID))
+ break;
+ }
+ continue;
+ }
+
+ /* As there are now non-UNMAPPED SPTEs in the range covered
+ * by a number of LPTEs, we need to transfer control of the
+ * address range to the SPTEs.
+ *
+ * Determine how many LPTEs need to transition state.
+ */
+ pgt->pte[ptei] |= NVKM_VMM_PTE_VALID;
+ for (ptes = 1, ptei++; ptei < lpti; ptes++, ptei++) {
+ if (pgt->pte[ptei] & NVKM_VMM_PTE_VALID)
+ break;
+ pgt->pte[ptei] |= NVKM_VMM_PTE_VALID;
+ }
+
+ if (pgt->pte[pteb] & NVKM_VMM_PTE_SPARSE) {
+ const u32 spti = pteb * sptn;
+ const u32 sptc = ptes * sptn;
+ /* The entire LPTE is marked as sparse, we need
+ * to make sure that the SPTEs are too.
+ */
+ TRA(it, "SPTE %05x: U -> S %d PTEs", spti, sptc);
+ desc->func->sparse(vmm, pgt->pt[1], spti, sptc);
+ /* Sparse LPTEs prevent SPTEs from being accessed. */
+ TRA(it, "LPTE %05x: S -> U %d PTEs", pteb, ptes);
+ pair->func->unmap(vmm, pgt->pt[0], pteb, ptes);
+ } else
+ if (pair->func->invalid) {
+ /* MMU supports blocking SPTEs by marking an LPTE
+ * as INVALID. We need to reverse that here.
+ */
+ TRA(it, "LPTE %05x: I -> U %d PTEs", pteb, ptes);
+ pair->func->unmap(vmm, pgt->pt[0], pteb, ptes);
+ }
+ }
+}
+
+static bool
+nvkm_vmm_ref_ptes(struct nvkm_vmm_iter *it, u32 ptei, u32 ptes)
+{
+ const struct nvkm_vmm_desc *desc = it->desc;
+ const int type = desc->type == SPT;
+ struct nvkm_vmm_pt *pgt = it->pt[0];
+
+ /* Take PTE references. */
+ pgt->refs[type] += ptes;
+
+ /* Dual-PTs need special handling. */
+ if (desc->type == SPT)
+ nvkm_vmm_ref_sptes(it, pgt, desc, ptei, ptes);
+
+ return true;
+}
+
+static void
+nvkm_vmm_sparse_ptes(const struct nvkm_vmm_desc *desc,
+ struct nvkm_vmm_pt *pgt, u32 ptei, u32 ptes)
+{
+ if (desc->type == PGD) {
+ while (ptes--)
+ pgt->pde[ptei++] = NVKM_VMM_PDE_SPARSE;
+ } else
+ if (desc->type == LPT) {
+ memset(&pgt->pte[ptei], NVKM_VMM_PTE_SPARSE, ptes);
+ }
+}
+
+static bool
+nvkm_vmm_sparse_unref_ptes(struct nvkm_vmm_iter *it, u32 ptei, u32 ptes)
+{
+ struct nvkm_vmm_pt *pt = it->pt[0];
+ if (it->desc->type == PGD)
+ memset(&pt->pde[ptei], 0x00, sizeof(pt->pde[0]) * ptes);
+ else
+ if (it->desc->type == LPT)
+ memset(&pt->pte[ptei], 0x00, sizeof(pt->pte[0]) * ptes);
+ return nvkm_vmm_unref_ptes(it, ptei, ptes);
+}
+
+static bool
+nvkm_vmm_sparse_ref_ptes(struct nvkm_vmm_iter *it, u32 ptei, u32 ptes)
+{
+ nvkm_vmm_sparse_ptes(it->desc, it->pt[0], ptei, ptes);
+ return nvkm_vmm_ref_ptes(it, ptei, ptes);
+}
+
+static bool
+nvkm_vmm_ref_hwpt(struct nvkm_vmm_iter *it, struct nvkm_vmm_pt *pgd, u32 pdei)
+{
+ const struct nvkm_vmm_desc *desc = &it->desc[it->lvl - 1];
+ const int type = desc->type == SPT;
+ struct nvkm_vmm_pt *pgt = pgd->pde[pdei];
+ const bool zero = !pgt->sparse && !desc->func->invalid;
+ struct nvkm_vmm *vmm = it->vmm;
+ struct nvkm_mmu *mmu = vmm->mmu;
+ struct nvkm_mmu_pt *pt;
+ u32 pten = 1 << desc->bits;
+ u32 pteb, ptei, ptes;
+ u32 size = desc->size * pten;
+
+ pgd->refs[0]++;
+
+ pgt->pt[type] = nvkm_mmu_ptc_get(mmu, size, desc->align, zero);
+ if (!pgt->pt[type]) {
+ it->lvl--;
+ nvkm_vmm_unref_pdes(it);
+ return false;
+ }
+
+ if (zero)
+ goto done;
+
+ pt = pgt->pt[type];
+
+ if (desc->type == LPT && pgt->refs[1]) {
+ /* SPT already exists covering the same range as this LPT,
+ * which means we need to be careful that any LPTEs which
+ * overlap valid SPTEs are unmapped as opposed to invalid
+ * or sparse, which would prevent the MMU from looking at
+ * the SPTEs on some GPUs.
+ */
+ for (ptei = pteb = 0; ptei < pten; pteb = ptei) {
+ bool spte = pgt->pte[ptei] & NVKM_VMM_PTE_SPTES;
+ for (ptes = 1, ptei++; ptei < pten; ptes++, ptei++) {
+ bool next = pgt->pte[ptei] & NVKM_VMM_PTE_SPTES;
+ if (spte != next)
+ break;
+ }
+
+ if (!spte) {
+ if (pgt->sparse)
+ desc->func->sparse(vmm, pt, pteb, ptes);
+ else
+ desc->func->invalid(vmm, pt, pteb, ptes);
+ memset(&pgt->pte[pteb], 0x00, ptes);
+ } else {
+ desc->func->unmap(vmm, pt, pteb, ptes);
+ while (ptes--)
+ pgt->pte[pteb++] |= NVKM_VMM_PTE_VALID;
+ }
+ }
+ } else {
+ if (pgt->sparse) {
+ nvkm_vmm_sparse_ptes(desc, pgt, 0, pten);
+ desc->func->sparse(vmm, pt, 0, pten);
+ } else {
+ desc->func->invalid(vmm, pt, 0, pten);
+ }
+ }
+
+done:
+ TRA(it, "PDE write %s", nvkm_vmm_desc_type(desc));
+ it->desc[it->lvl].func->pde(it->vmm, pgd, pdei);
+ nvkm_vmm_flush_mark(it);
+ return true;
+}
+
+static bool
+nvkm_vmm_ref_swpt(struct nvkm_vmm_iter *it, struct nvkm_vmm_pt *pgd, u32 pdei)
+{
+ const struct nvkm_vmm_desc *desc = &it->desc[it->lvl - 1];
+ struct nvkm_vmm_pt *pgt = pgd->pde[pdei];
+
+ pgt = nvkm_vmm_pt_new(desc, NVKM_VMM_PDE_SPARSED(pgt), it->page);
+ if (!pgt) {
+ if (!pgd->refs[0])
+ nvkm_vmm_unref_pdes(it);
+ return false;
+ }
+
+ pgd->pde[pdei] = pgt;
+ return true;
+}
+
+static inline u64
+nvkm_vmm_iter(struct nvkm_vmm *vmm, const struct nvkm_vmm_page *page,
+ u64 addr, u64 size, const char *name, bool ref,
+ bool (*REF_PTES)(struct nvkm_vmm_iter *, u32, u32),
+ nvkm_vmm_pte_func MAP_PTES, struct nvkm_vmm_map *map,
+ nvkm_vmm_pxe_func CLR_PTES)
+{
+ const struct nvkm_vmm_desc *desc = page->desc;
+ struct nvkm_vmm_iter it;
+ u64 bits = addr >> page->shift;
+
+ it.page = page;
+ it.desc = desc;
+ it.vmm = vmm;
+ it.cnt = size >> page->shift;
+ it.flush = NVKM_VMM_LEVELS_MAX;
+
+ /* Deconstruct address into PTE indices for each mapping level. */
+ for (it.lvl = 0; desc[it.lvl].bits; it.lvl++) {
+ it.pte[it.lvl] = bits & ((1 << desc[it.lvl].bits) - 1);
+ bits >>= desc[it.lvl].bits;
+ }
+ it.max = --it.lvl;
+ it.pt[it.max] = vmm->pd;
+
+ it.lvl = 0;
+ TRA(&it, "%s: %016llx %016llx %d %lld PTEs", name,
+ addr, size, page->shift, it.cnt);
+ it.lvl = it.max;
+
+ /* Depth-first traversal of page tables. */
+ while (it.cnt) {
+ struct nvkm_vmm_pt *pgt = it.pt[it.lvl];
+ const int type = desc->type == SPT;
+ const u32 pten = 1 << desc->bits;
+ const u32 ptei = it.pte[0];
+ const u32 ptes = min_t(u64, it.cnt, pten - ptei);
+
+ /* Walk down the tree, finding page tables for each level. */
+ for (; it.lvl; it.lvl--) {
+ const u32 pdei = it.pte[it.lvl];
+ struct nvkm_vmm_pt *pgd = pgt;
+
+ /* Software PT. */
+ if (ref && NVKM_VMM_PDE_INVALID(pgd->pde[pdei])) {
+ if (!nvkm_vmm_ref_swpt(&it, pgd, pdei))
+ goto fail;
+ }
+ it.pt[it.lvl - 1] = pgt = pgd->pde[pdei];
+
+ /* Hardware PT.
+ *
+ * This is a separate step from above due to GF100 and
+ * newer having dual page tables at some levels, which
+ * are refcounted independently.
+ */
+ if (ref && !pgt->refs[desc[it.lvl - 1].type == SPT]) {
+ if (!nvkm_vmm_ref_hwpt(&it, pgd, pdei))
+ goto fail;
+ }
+ }
+
+ /* Handle PTE updates. */
+ if (!REF_PTES || REF_PTES(&it, ptei, ptes)) {
+ struct nvkm_mmu_pt *pt = pgt->pt[type];
+ if (MAP_PTES || CLR_PTES) {
+ if (MAP_PTES)
+ MAP_PTES(vmm, pt, ptei, ptes, map);
+ else
+ CLR_PTES(vmm, pt, ptei, ptes);
+ nvkm_vmm_flush_mark(&it);
+ }
+ }
+
+ /* Walk back up the tree to the next position. */
+ it.pte[it.lvl] += ptes;
+ it.cnt -= ptes;
+ if (it.cnt) {
+ while (it.pte[it.lvl] == (1 << desc[it.lvl].bits)) {
+ it.pte[it.lvl++] = 0;
+ it.pte[it.lvl]++;
+ }
+ }
+ };
+
+ nvkm_vmm_flush(&it);
+ return ~0ULL;
+
+fail:
+ /* Reconstruct the failure address so the caller is able to
+ * reverse any partially completed operations.
+ */
+ addr = it.pte[it.max--];
+ do {
+ addr = addr << desc[it.max].bits;
+ addr |= it.pte[it.max];
+ } while (it.max--);
+
+ return addr << page->shift;
+}
+
+static void
+nvkm_vmm_ptes_sparse_put(struct nvkm_vmm *vmm, const struct nvkm_vmm_page *page,
+ u64 addr, u64 size)
+{
+ nvkm_vmm_iter(vmm, page, addr, size, "sparse unref", false,
+ nvkm_vmm_sparse_unref_ptes, NULL, NULL,
+ page->desc->func->invalid ?
+ page->desc->func->invalid : page->desc->func->unmap);
+}
+
+static int
+nvkm_vmm_ptes_sparse_get(struct nvkm_vmm *vmm, const struct nvkm_vmm_page *page,
+ u64 addr, u64 size)
+{
+ if ((page->type & NVKM_VMM_PAGE_SPARSE)) {
+ u64 fail = nvkm_vmm_iter(vmm, page, addr, size, "sparse ref",
+ true, nvkm_vmm_sparse_ref_ptes, NULL,
+ NULL, page->desc->func->sparse);
+ if (fail != ~0ULL) {
+ if ((size = fail - addr))
+ nvkm_vmm_ptes_sparse_put(vmm, page, addr, size);
+ return -ENOMEM;
+ }
+ return 0;
+ }
+ return -EINVAL;
+}
+
+static int
+nvkm_vmm_ptes_sparse(struct nvkm_vmm *vmm, u64 addr, u64 size, bool ref)
+{
+ const struct nvkm_vmm_page *page = vmm->func->page;
+ int m = 0, i;
+ u64 start = addr;
+ u64 block;
+
+ while (size) {
+ /* Limit maximum page size based on remaining size. */
+ while (size < (1ULL << page[m].shift))
+ m++;
+ i = m;
+
+ /* Find largest page size suitable for alignment. */
+ while (!IS_ALIGNED(addr, 1ULL << page[i].shift))
+ i++;
+
+ /* Determine number of PTEs at this page size. */
+ if (i != m) {
+ /* Limited to alignment boundary of next page size. */
+ u64 next = 1ULL << page[i - 1].shift;
+ u64 part = ALIGN(addr, next) - addr;
+ if (size - part >= next)
+ block = (part >> page[i].shift) << page[i].shift;
+ else
+ block = (size >> page[i].shift) << page[i].shift;
+ } else {
+ block = (size >> page[i].shift) << page[i].shift;;
+ }
+
+ /* Perform operation. */
+ if (ref) {
+ int ret = nvkm_vmm_ptes_sparse_get(vmm, &page[i], addr, block);
+ if (ret) {
+ if ((size = addr - start))
+ nvkm_vmm_ptes_sparse(vmm, start, size, false);
+ return ret;
+ }
+ } else {
+ nvkm_vmm_ptes_sparse_put(vmm, &page[i], addr, block);
+ }
+
+ size -= block;
+ addr += block;
+ }
+
+ return 0;
+}
+
+static void
+nvkm_vmm_ptes_unmap_put(struct nvkm_vmm *vmm, const struct nvkm_vmm_page *page,
+ u64 addr, u64 size, bool sparse)
+{
+ const struct nvkm_vmm_desc_func *func = page->desc->func;
+ nvkm_vmm_iter(vmm, page, addr, size, "unmap + unref",
+ false, nvkm_vmm_unref_ptes, NULL, NULL,
+ sparse ? func->sparse : func->invalid ? func->invalid :
+ func->unmap);
+}
+
+static int
+nvkm_vmm_ptes_get_map(struct nvkm_vmm *vmm, const struct nvkm_vmm_page *page,
+ u64 addr, u64 size, struct nvkm_vmm_map *map,
+ nvkm_vmm_pte_func func)
+{
+ u64 fail = nvkm_vmm_iter(vmm, page, addr, size, "ref + map", true,
+ nvkm_vmm_ref_ptes, func, map, NULL);
+ if (fail != ~0ULL) {
+ if ((size = fail - addr))
+ nvkm_vmm_ptes_unmap_put(vmm, page, addr, size, false);
+ return -ENOMEM;
+ }
+ return 0;
+}
+
+static void
+nvkm_vmm_ptes_unmap(struct nvkm_vmm *vmm, const struct nvkm_vmm_page *page,
+ u64 addr, u64 size, bool sparse)
+{
+ const struct nvkm_vmm_desc_func *func = page->desc->func;
+ nvkm_vmm_iter(vmm, page, addr, size, "unmap", false, NULL, NULL, NULL,
+ sparse ? func->sparse : func->invalid ? func->invalid :
+ func->unmap);
+}
+
+static void
+nvkm_vmm_ptes_map(struct nvkm_vmm *vmm, const struct nvkm_vmm_page *page,
+ u64 addr, u64 size, struct nvkm_vmm_map *map,
+ nvkm_vmm_pte_func func)
+{
+ nvkm_vmm_iter(vmm, page, addr, size, "map", false,
+ NULL, func, map, NULL);
+}
+
+static void
+nvkm_vmm_ptes_put(struct nvkm_vmm *vmm, const struct nvkm_vmm_page *page,
+ u64 addr, u64 size)
+{
+ nvkm_vmm_iter(vmm, page, addr, size, "unref", false,
+ nvkm_vmm_unref_ptes, NULL, NULL, NULL);
+}
+
+static int
+nvkm_vmm_ptes_get(struct nvkm_vmm *vmm, const struct nvkm_vmm_page *page,
+ u64 addr, u64 size)
+{
+ u64 fail = nvkm_vmm_iter(vmm, page, addr, size, "ref", true,
+ nvkm_vmm_ref_ptes, NULL, NULL, NULL);
+ if (fail != ~0ULL) {
+ if (fail != addr)
+ nvkm_vmm_ptes_put(vmm, page, addr, fail - addr);
+ return -ENOMEM;
+ }
+ return 0;
+}
+
+static inline struct nvkm_vma *
+nvkm_vma_new(u64 addr, u64 size)
+{
+ struct nvkm_vma *vma = kzalloc(sizeof(*vma), GFP_KERNEL);
+ if (vma) {
+ vma->addr = addr;
+ vma->size = size;
+ vma->page = NVKM_VMA_PAGE_NONE;
+ vma->refd = NVKM_VMA_PAGE_NONE;
+ }
+ return vma;
+}
+
+struct nvkm_vma *
+nvkm_vma_tail(struct nvkm_vma *vma, u64 tail)
+{
+ struct nvkm_vma *new;
+
+ BUG_ON(vma->size == tail);
+
+ if (!(new = nvkm_vma_new(vma->addr + (vma->size - tail), tail)))
+ return NULL;
+ vma->size -= tail;
+
+ new->mapref = vma->mapref;
+ new->sparse = vma->sparse;
+ new->page = vma->page;
+ new->refd = vma->refd;
+ new->used = vma->used;
+ new->part = vma->part;
+ new->user = vma->user;
+ new->busy = vma->busy;
+ list_add(&new->head, &vma->head);
+ return new;
+}
+
+static void
+nvkm_vmm_free_insert(struct nvkm_vmm *vmm, struct nvkm_vma *vma)
+{
+ struct rb_node **ptr = &vmm->free.rb_node;
+ struct rb_node *parent = NULL;
+
+ while (*ptr) {
+ struct nvkm_vma *this = rb_entry(*ptr, typeof(*this), tree);
+ parent = *ptr;
+ if (vma->size < this->size)
+ ptr = &parent->rb_left;
+ else
+ if (vma->size > this->size)
+ ptr = &parent->rb_right;
+ else
+ if (vma->addr < this->addr)
+ ptr = &parent->rb_left;
+ else
+ if (vma->addr > this->addr)
+ ptr = &parent->rb_right;
+ else
+ BUG();
+ }
+
+ rb_link_node(&vma->tree, parent, ptr);
+ rb_insert_color(&vma->tree, &vmm->free);
+}
+
+void
+nvkm_vmm_node_insert(struct nvkm_vmm *vmm, struct nvkm_vma *vma)
+{
+ struct rb_node **ptr = &vmm->root.rb_node;
+ struct rb_node *parent = NULL;
+
+ while (*ptr) {
+ struct nvkm_vma *this = rb_entry(*ptr, typeof(*this), tree);
+ parent = *ptr;
+ if (vma->addr < this->addr)
+ ptr = &parent->rb_left;
+ else
+ if (vma->addr > this->addr)
+ ptr = &parent->rb_right;
+ else
+ BUG();
+ }
+
+ rb_link_node(&vma->tree, parent, ptr);
+ rb_insert_color(&vma->tree, &vmm->root);
+}
+
+struct nvkm_vma *
+nvkm_vmm_node_search(struct nvkm_vmm *vmm, u64 addr)
+{
+ struct rb_node *node = vmm->root.rb_node;
+ while (node) {
+ struct nvkm_vma *vma = rb_entry(node, typeof(*vma), tree);
+ if (addr < vma->addr)
+ node = node->rb_left;
+ else
+ if (addr >= vma->addr + vma->size)
+ node = node->rb_right;
+ else
+ return vma;
+ }
+ return NULL;
+}
+
+static void
+nvkm_vmm_dtor(struct nvkm_vmm *vmm)
+{
+ struct nvkm_vma *vma;
+ struct rb_node *node;
+
+ while ((node = rb_first(&vmm->root))) {
+ struct nvkm_vma *vma = rb_entry(node, typeof(*vma), tree);
+ nvkm_vmm_put(vmm, &vma);
+ }
+
+ if (vmm->bootstrapped) {
+ const struct nvkm_vmm_page *page = vmm->func->page;
+ const u64 limit = vmm->limit - vmm->start;
+
+ while (page[1].shift)
+ page++;
+
+ nvkm_mmu_ptc_dump(vmm->mmu);
+ nvkm_vmm_ptes_put(vmm, page, vmm->start, limit);
+ }
+
+ vma = list_first_entry(&vmm->list, typeof(*vma), head);
+ list_del(&vma->head);
+ kfree(vma);
+ WARN_ON(!list_empty(&vmm->list));
+
+ if (vmm->nullp) {
+ dma_free_coherent(vmm->mmu->subdev.device->dev, 16 * 1024,
+ vmm->nullp, vmm->null);
+ }
+
+ if (vmm->pd) {
+ nvkm_mmu_ptc_put(vmm->mmu, true, &vmm->pd->pt[0]);
+ nvkm_vmm_pt_del(&vmm->pd);
+ }
+}
+
+int
+nvkm_vmm_ctor(const struct nvkm_vmm_func *func, struct nvkm_mmu *mmu,
+ u32 pd_header, u64 addr, u64 size, struct lock_class_key *key,
+ const char *name, struct nvkm_vmm *vmm)
+{
+ static struct lock_class_key _key;
+ const struct nvkm_vmm_page *page = func->page;
+ const struct nvkm_vmm_desc *desc;
+ struct nvkm_vma *vma;
+ int levels, bits = 0;
+
+ vmm->func = func;
+ vmm->mmu = mmu;
+ vmm->name = name;
+ vmm->debug = mmu->subdev.debug;
+ kref_init(&vmm->kref);
+
+ __mutex_init(&vmm->mutex, "&vmm->mutex", key ? key : &_key);
+
+ /* Locate the smallest page size supported by the backend, it will
+ * have the the deepest nesting of page tables.
+ */
+ while (page[1].shift)
+ page++;
+
+ /* Locate the structure that describes the layout of the top-level
+ * page table, and determine the number of valid bits in a virtual
+ * address.
+ */
+ for (levels = 0, desc = page->desc; desc->bits; desc++, levels++)
+ bits += desc->bits;
+ bits += page->shift;
+ desc--;
+
+ if (WARN_ON(levels > NVKM_VMM_LEVELS_MAX))
+ return -EINVAL;
+
+ vmm->start = addr;
+ vmm->limit = size ? (addr + size) : (1ULL << bits);
+ if (vmm->start > vmm->limit || vmm->limit > (1ULL << bits))
+ return -EINVAL;
+
+ /* Allocate top-level page table. */
+ vmm->pd = nvkm_vmm_pt_new(desc, false, NULL);
+ if (!vmm->pd)
+ return -ENOMEM;
+ vmm->pd->refs[0] = 1;
+ INIT_LIST_HEAD(&vmm->join);
+
+ /* ... and the GPU storage for it, except on Tesla-class GPUs that
+ * have the PD embedded in the instance structure.
+ */
+ if (desc->size) {
+ const u32 size = pd_header + desc->size * (1 << desc->bits);
+ vmm->pd->pt[0] = nvkm_mmu_ptc_get(mmu, size, desc->align, true);
+ if (!vmm->pd->pt[0])
+ return -ENOMEM;
+ }
+
+ /* Initialise address-space MM. */
+ INIT_LIST_HEAD(&vmm->list);
+ vmm->free = RB_ROOT;
+ vmm->root = RB_ROOT;
+
+ if (!(vma = nvkm_vma_new(vmm->start, vmm->limit - vmm->start)))
+ return -ENOMEM;
+
+ nvkm_vmm_free_insert(vmm, vma);
+ list_add(&vma->head, &vmm->list);
+ return 0;
+}
+
+int
+nvkm_vmm_new_(const struct nvkm_vmm_func *func, struct nvkm_mmu *mmu,
+ u32 hdr, u64 addr, u64 size, struct lock_class_key *key,
+ const char *name, struct nvkm_vmm **pvmm)
+{
+ if (!(*pvmm = kzalloc(sizeof(**pvmm), GFP_KERNEL)))
+ return -ENOMEM;
+ return nvkm_vmm_ctor(func, mmu, hdr, addr, size, key, name, *pvmm);
+}
+
+#define node(root, dir) ((root)->head.dir == &vmm->list) ? NULL : \
+ list_entry((root)->head.dir, struct nvkm_vma, head)
+
+void
+nvkm_vmm_unmap_region(struct nvkm_vmm *vmm, struct nvkm_vma *vma)
+{
+ struct nvkm_vma *next;
+
+ nvkm_memory_tags_put(vma->memory, vmm->mmu->subdev.device, &vma->tags);
+ nvkm_memory_unref(&vma->memory);
+
+ if (vma->part) {
+ struct nvkm_vma *prev = node(vma, prev);
+ if (!prev->memory) {
+ prev->size += vma->size;
+ rb_erase(&vma->tree, &vmm->root);
+ list_del(&vma->head);
+ kfree(vma);
+ vma = prev;
+ }
+ }
+
+ next = node(vma, next);
+ if (next && next->part) {
+ if (!next->memory) {
+ vma->size += next->size;
+ rb_erase(&next->tree, &vmm->root);
+ list_del(&next->head);
+ kfree(next);
+ }
+ }
+}
+
+void
+nvkm_vmm_unmap_locked(struct nvkm_vmm *vmm, struct nvkm_vma *vma)
+{
+ const struct nvkm_vmm_page *page = &vmm->func->page[vma->refd];
+
+ if (vma->mapref) {
+ nvkm_vmm_ptes_unmap_put(vmm, page, vma->addr, vma->size, vma->sparse);
+ vma->refd = NVKM_VMA_PAGE_NONE;
+ } else {
+ nvkm_vmm_ptes_unmap(vmm, page, vma->addr, vma->size, vma->sparse);
+ }
+
+ nvkm_vmm_unmap_region(vmm, vma);
+}
+
+void
+nvkm_vmm_unmap(struct nvkm_vmm *vmm, struct nvkm_vma *vma)
+{
+ if (vma->memory) {
+ mutex_lock(&vmm->mutex);
+ nvkm_vmm_unmap_locked(vmm, vma);
+ mutex_unlock(&vmm->mutex);
+ }
+}
+
+static int
+nvkm_vmm_map_valid(struct nvkm_vmm *vmm, struct nvkm_vma *vma,
+ void *argv, u32 argc, struct nvkm_vmm_map *map)
+{
+ switch (nvkm_memory_target(map->memory)) {
+ case NVKM_MEM_TARGET_VRAM:
+ if (!(map->page->type & NVKM_VMM_PAGE_VRAM)) {
+ VMM_DEBUG(vmm, "%d !VRAM", map->page->shift);
+ return -EINVAL;
+ }
+ break;
+ case NVKM_MEM_TARGET_HOST:
+ case NVKM_MEM_TARGET_NCOH:
+ if (!(map->page->type & NVKM_VMM_PAGE_HOST)) {
+ VMM_DEBUG(vmm, "%d !HOST", map->page->shift);
+ return -EINVAL;
+ }
+ break;
+ default:
+ WARN_ON(1);
+ return -ENOSYS;
+ }
+
+ if (!IS_ALIGNED( vma->addr, 1ULL << map->page->shift) ||
+ !IS_ALIGNED((u64)vma->size, 1ULL << map->page->shift) ||
+ !IS_ALIGNED( map->offset, 1ULL << map->page->shift) ||
+ nvkm_memory_page(map->memory) < map->page->shift) {
+ VMM_DEBUG(vmm, "alignment %016llx %016llx %016llx %d %d",
+ vma->addr, (u64)vma->size, map->offset, map->page->shift,
+ nvkm_memory_page(map->memory));
+ return -EINVAL;
+ }
+
+ return vmm->func->valid(vmm, argv, argc, map);
+}
+
+static int
+nvkm_vmm_map_choose(struct nvkm_vmm *vmm, struct nvkm_vma *vma,
+ void *argv, u32 argc, struct nvkm_vmm_map *map)
+{
+ for (map->page = vmm->func->page; map->page->shift; map->page++) {
+ VMM_DEBUG(vmm, "trying %d", map->page->shift);
+ if (!nvkm_vmm_map_valid(vmm, vma, argv, argc, map))
+ return 0;
+ }
+ return -EINVAL;
+}
+
+static int
+nvkm_vmm_map_locked(struct nvkm_vmm *vmm, struct nvkm_vma *vma,
+ void *argv, u32 argc, struct nvkm_vmm_map *map)
+{
+ nvkm_vmm_pte_func func;
+ int ret;
+
+ /* Make sure we won't overrun the end of the memory object. */
+ if (unlikely(nvkm_memory_size(map->memory) < map->offset + vma->size)) {
+ VMM_DEBUG(vmm, "overrun %016llx %016llx %016llx",
+ nvkm_memory_size(map->memory),
+ map->offset, (u64)vma->size);
+ return -EINVAL;
+ }
+
+ /* Check remaining arguments for validity. */
+ if (vma->page == NVKM_VMA_PAGE_NONE &&
+ vma->refd == NVKM_VMA_PAGE_NONE) {
+ /* Find the largest page size we can perform the mapping at. */
+ const u32 debug = vmm->debug;
+ vmm->debug = 0;
+ ret = nvkm_vmm_map_choose(vmm, vma, argv, argc, map);
+ vmm->debug = debug;
+ if (ret) {
+ VMM_DEBUG(vmm, "invalid at any page size");
+ nvkm_vmm_map_choose(vmm, vma, argv, argc, map);
+ return -EINVAL;
+ }
+ } else {
+ /* Page size of the VMA is already pre-determined. */
+ if (vma->refd != NVKM_VMA_PAGE_NONE)
+ map->page = &vmm->func->page[vma->refd];
+ else
+ map->page = &vmm->func->page[vma->page];
+
+ ret = nvkm_vmm_map_valid(vmm, vma, argv, argc, map);
+ if (ret) {
+ VMM_DEBUG(vmm, "invalid %d\n", ret);
+ return ret;
+ }
+ }
+
+ /* Deal with the 'offset' argument, and fetch the backend function. */
+ map->off = map->offset;
+ if (map->mem) {
+ for (; map->off; map->mem = map->mem->next) {
+ u64 size = (u64)map->mem->length << NVKM_RAM_MM_SHIFT;
+ if (size > map->off)
+ break;
+ map->off -= size;
+ }
+ func = map->page->desc->func->mem;
+ } else
+ if (map->sgl) {
+ for (; map->off; map->sgl = sg_next(map->sgl)) {
+ u64 size = sg_dma_len(map->sgl);
+ if (size > map->off)
+ break;
+ map->off -= size;
+ }
+ func = map->page->desc->func->sgl;
+ } else {
+ map->dma += map->offset >> PAGE_SHIFT;
+ map->off = map->offset & PAGE_MASK;
+ func = map->page->desc->func->dma;
+ }
+
+ /* Perform the map. */
+ if (vma->refd == NVKM_VMA_PAGE_NONE) {
+ ret = nvkm_vmm_ptes_get_map(vmm, map->page, vma->addr, vma->size, map, func);
+ if (ret)
+ return ret;
+
+ vma->refd = map->page - vmm->func->page;
+ } else {
+ nvkm_vmm_ptes_map(vmm, map->page, vma->addr, vma->size, map, func);
+ }
+
+ nvkm_memory_tags_put(vma->memory, vmm->mmu->subdev.device, &vma->tags);
+ nvkm_memory_unref(&vma->memory);
+ vma->memory = nvkm_memory_ref(map->memory);
+ vma->tags = map->tags;
+ return 0;
+}
+
+int
+nvkm_vmm_map(struct nvkm_vmm *vmm, struct nvkm_vma *vma, void *argv, u32 argc,
+ struct nvkm_vmm_map *map)
+{
+ int ret;
+ mutex_lock(&vmm->mutex);
+ ret = nvkm_vmm_map_locked(vmm, vma, argv, argc, map);
+ vma->busy = false;
+ mutex_unlock(&vmm->mutex);
+ return ret;
+}
+
+static void
+nvkm_vmm_put_region(struct nvkm_vmm *vmm, struct nvkm_vma *vma)
+{
+ struct nvkm_vma *prev, *next;
+
+ if ((prev = node(vma, prev)) && !prev->used) {
+ rb_erase(&prev->tree, &vmm->free);
+ list_del(&prev->head);
+ vma->addr = prev->addr;
+ vma->size += prev->size;
+ kfree(prev);
+ }
+
+ if ((next = node(vma, next)) && !next->used) {
+ rb_erase(&next->tree, &vmm->free);
+ list_del(&next->head);
+ vma->size += next->size;
+ kfree(next);
+ }
+
+ nvkm_vmm_free_insert(vmm, vma);
+}
+
+void
+nvkm_vmm_put_locked(struct nvkm_vmm *vmm, struct nvkm_vma *vma)
+{
+ const struct nvkm_vmm_page *page = vmm->func->page;
+ struct nvkm_vma *next = vma;
+
+ BUG_ON(vma->part);
+
+ if (vma->mapref || !vma->sparse) {
+ do {
+ const bool map = next->memory != NULL;
+ const u8 refd = next->refd;
+ const u64 addr = next->addr;
+ u64 size = next->size;
+
+ /* Merge regions that are in the same state. */
+ while ((next = node(next, next)) && next->part &&
+ (next->memory != NULL) == map &&
+ (next->refd == refd))
+ size += next->size;
+
+ if (map) {
+ /* Region(s) are mapped, merge the unmap
+ * and dereference into a single walk of
+ * the page tree.
+ */
+ nvkm_vmm_ptes_unmap_put(vmm, &page[refd], addr,
+ size, vma->sparse);
+ } else
+ if (refd != NVKM_VMA_PAGE_NONE) {
+ /* Drop allocation-time PTE references. */
+ nvkm_vmm_ptes_put(vmm, &page[refd], addr, size);
+ }
+ } while (next && next->part);
+ }
+
+ /* Merge any mapped regions that were split from the initial
+ * address-space allocation back into the allocated VMA, and
+ * release memory/compression resources.
+ */
+ next = vma;
+ do {
+ if (next->memory)
+ nvkm_vmm_unmap_region(vmm, next);
+ } while ((next = node(vma, next)) && next->part);
+
+ if (vma->sparse && !vma->mapref) {
+ /* Sparse region that was allocated with a fixed page size,
+ * meaning all relevant PTEs were referenced once when the
+ * region was allocated, and remained that way, regardless
+ * of whether memory was mapped into it afterwards.
+ *
+ * The process of unmapping, unsparsing, and dereferencing
+ * PTEs can be done in a single page tree walk.
+ */
+ nvkm_vmm_ptes_sparse_put(vmm, &page[vma->refd], vma->addr, vma->size);
+ } else
+ if (vma->sparse) {
+ /* Sparse region that wasn't allocated with a fixed page size,
+ * PTE references were taken both at allocation time (to make
+ * the GPU see the region as sparse), and when mapping memory
+ * into the region.
+ *
+ * The latter was handled above, and the remaining references
+ * are dealt with here.
+ */
+ nvkm_vmm_ptes_sparse(vmm, vma->addr, vma->size, false);
+ }
+
+ /* Remove VMA from the list of allocated nodes. */
+ rb_erase(&vma->tree, &vmm->root);
+
+ /* Merge VMA back into the free list. */
+ vma->page = NVKM_VMA_PAGE_NONE;
+ vma->refd = NVKM_VMA_PAGE_NONE;
+ vma->used = false;
+ vma->user = false;
+ nvkm_vmm_put_region(vmm, vma);
+}
+
+void
+nvkm_vmm_put(struct nvkm_vmm *vmm, struct nvkm_vma **pvma)
+{
+ struct nvkm_vma *vma = *pvma;
+ if (vma) {
+ mutex_lock(&vmm->mutex);
+ nvkm_vmm_put_locked(vmm, vma);
+ mutex_unlock(&vmm->mutex);
+ *pvma = NULL;
+ }
+}
+
+int
+nvkm_vmm_get_locked(struct nvkm_vmm *vmm, bool getref, bool mapref, bool sparse,
+ u8 shift, u8 align, u64 size, struct nvkm_vma **pvma)
+{
+ const struct nvkm_vmm_page *page = &vmm->func->page[NVKM_VMA_PAGE_NONE];
+ struct rb_node *node = NULL, *temp;
+ struct nvkm_vma *vma = NULL, *tmp;
+ u64 addr, tail;
+ int ret;
+
+ VMM_TRACE(vmm, "getref %d mapref %d sparse %d "
+ "shift: %d align: %d size: %016llx",
+ getref, mapref, sparse, shift, align, size);
+
+ /* Zero-sized, or lazily-allocated sparse VMAs, make no sense. */
+ if (unlikely(!size || (!getref && !mapref && sparse))) {
+ VMM_DEBUG(vmm, "args %016llx %d %d %d",
+ size, getref, mapref, sparse);
+ return -EINVAL;
+ }
+
+ /* Tesla-class GPUs can only select page size per-PDE, which means
+ * we're required to know the mapping granularity up-front to find
+ * a suitable region of address-space.
+ *
+ * The same goes if we're requesting up-front allocation of PTES.
+ */
+ if (unlikely((getref || vmm->func->page_block) && !shift)) {
+ VMM_DEBUG(vmm, "page size required: %d %016llx",
+ getref, vmm->func->page_block);
+ return -EINVAL;
+ }
+
+ /* If a specific page size was requested, determine its index and
+ * make sure the requested size is a multiple of the page size.
+ */
+ if (shift) {
+ for (page = vmm->func->page; page->shift; page++) {
+ if (shift == page->shift)
+ break;
+ }
+
+ if (!page->shift || !IS_ALIGNED(size, 1ULL << page->shift)) {
+ VMM_DEBUG(vmm, "page %d %016llx", shift, size);
+ return -EINVAL;
+ }
+ align = max_t(u8, align, shift);
+ } else {
+ align = max_t(u8, align, 12);
+ }
+
+ /* Locate smallest block that can possibly satisfy the allocation. */
+ temp = vmm->free.rb_node;
+ while (temp) {
+ struct nvkm_vma *this = rb_entry(temp, typeof(*this), tree);
+ if (this->size < size) {
+ temp = temp->rb_right;
+ } else {
+ node = temp;
+ temp = temp->rb_left;
+ }
+ }
+
+ if (unlikely(!node))
+ return -ENOSPC;
+
+ /* Take into account alignment restrictions, trying larger blocks
+ * in turn until we find a suitable free block.
+ */
+ do {
+ struct nvkm_vma *this = rb_entry(node, typeof(*this), tree);
+ struct nvkm_vma *prev = node(this, prev);
+ struct nvkm_vma *next = node(this, next);
+ const int p = page - vmm->func->page;
+
+ addr = this->addr;
+ if (vmm->func->page_block && prev && prev->page != p)
+ addr = ALIGN(addr, vmm->func->page_block);
+ addr = ALIGN(addr, 1ULL << align);
+
+ tail = this->addr + this->size;
+ if (vmm->func->page_block && next && next->page != p)
+ tail = ALIGN_DOWN(addr, vmm->func->page_block);
+
+ if (addr <= tail && tail - addr >= size) {
+ rb_erase(&this->tree, &vmm->free);
+ vma = this;
+ break;
+ }
+ } while ((node = rb_next(node)));
+
+ if (unlikely(!vma))
+ return -ENOSPC;
+
+ /* If the VMA we found isn't already exactly the requested size,
+ * it needs to be split, and the remaining free blocks returned.
+ */
+ if (addr != vma->addr) {
+ if (!(tmp = nvkm_vma_tail(vma, vma->size + vma->addr - addr))) {
+ nvkm_vmm_put_region(vmm, vma);
+ return -ENOMEM;
+ }
+ nvkm_vmm_free_insert(vmm, vma);
+ vma = tmp;
+ }
+
+ if (size != vma->size) {
+ if (!(tmp = nvkm_vma_tail(vma, vma->size - size))) {
+ nvkm_vmm_put_region(vmm, vma);
+ return -ENOMEM;
+ }
+ nvkm_vmm_free_insert(vmm, tmp);
+ }
+
+ /* Pre-allocate page tables and/or setup sparse mappings. */
+ if (sparse && getref)
+ ret = nvkm_vmm_ptes_sparse_get(vmm, page, vma->addr, vma->size);
+ else if (sparse)
+ ret = nvkm_vmm_ptes_sparse(vmm, vma->addr, vma->size, true);
+ else if (getref)
+ ret = nvkm_vmm_ptes_get(vmm, page, vma->addr, vma->size);
+ else
+ ret = 0;
+ if (ret) {
+ nvkm_vmm_put_region(vmm, vma);
+ return ret;
+ }
+
+ vma->mapref = mapref && !getref;
+ vma->sparse = sparse;
+ vma->page = page - vmm->func->page;
+ vma->refd = getref ? vma->page : NVKM_VMA_PAGE_NONE;
+ vma->used = true;
+ nvkm_vmm_node_insert(vmm, vma);
+ *pvma = vma;
+ return 0;
+}
+
+int
+nvkm_vmm_get(struct nvkm_vmm *vmm, u8 page, u64 size, struct nvkm_vma **pvma)
+{
+ int ret;
+ mutex_lock(&vmm->mutex);
+ ret = nvkm_vmm_get_locked(vmm, false, true, false, page, 0, size, pvma);
+ mutex_unlock(&vmm->mutex);
+ return ret;
+}
+
+void
+nvkm_vmm_part(struct nvkm_vmm *vmm, struct nvkm_memory *inst)
+{
+ if (vmm->func->part && inst) {
+ mutex_lock(&vmm->mutex);
+ vmm->func->part(vmm, inst);
+ mutex_unlock(&vmm->mutex);
+ }
+}
+
+int
+nvkm_vmm_join(struct nvkm_vmm *vmm, struct nvkm_memory *inst)
+{
+ int ret = 0;
+ if (vmm->func->join) {
+ mutex_lock(&vmm->mutex);
+ ret = vmm->func->join(vmm, inst);
+ mutex_unlock(&vmm->mutex);
+ }
+ return ret;
+}
+
+static bool
+nvkm_vmm_boot_ptes(struct nvkm_vmm_iter *it, u32 ptei, u32 ptes)
+{
+ const struct nvkm_vmm_desc *desc = it->desc;
+ const int type = desc->type == SPT;
+ nvkm_memory_boot(it->pt[0]->pt[type]->memory, it->vmm);
+ return false;
+}
+
+int
+nvkm_vmm_boot(struct nvkm_vmm *vmm)
+{
+ const struct nvkm_vmm_page *page = vmm->func->page;
+ const u64 limit = vmm->limit - vmm->start;
+ int ret;
+
+ while (page[1].shift)
+ page++;
+
+ ret = nvkm_vmm_ptes_get(vmm, page, vmm->start, limit);
+ if (ret)
+ return ret;
+
+ nvkm_vmm_iter(vmm, page, vmm->start, limit, "bootstrap", false,
+ nvkm_vmm_boot_ptes, NULL, NULL, NULL);
+ vmm->bootstrapped = true;
+ return 0;
+}
+
+static void
+nvkm_vmm_del(struct kref *kref)
+{
+ struct nvkm_vmm *vmm = container_of(kref, typeof(*vmm), kref);
+ nvkm_vmm_dtor(vmm);
+ kfree(vmm);
+}
+
+void
+nvkm_vmm_unref(struct nvkm_vmm **pvmm)
+{
+ struct nvkm_vmm *vmm = *pvmm;
+ if (vmm) {
+ kref_put(&vmm->kref, nvkm_vmm_del);
+ *pvmm = NULL;
+ }
+}
+
+struct nvkm_vmm *
+nvkm_vmm_ref(struct nvkm_vmm *vmm)
+{
+ if (vmm)
+ kref_get(&vmm->kref);
+ return vmm;
+}
+
+int
+nvkm_vmm_new(struct nvkm_device *device, u64 addr, u64 size, void *argv,
+ u32 argc, struct lock_class_key *key, const char *name,
+ struct nvkm_vmm **pvmm)
+{
+ struct nvkm_mmu *mmu = device->mmu;
+ struct nvkm_vmm *vmm = NULL;
+ int ret;
+ ret = mmu->func->vmm.ctor(mmu, addr, size, argv, argc, key, name, &vmm);
+ if (ret)
+ nvkm_vmm_unref(&vmm);
+ *pvmm = vmm;
+ return ret;
+}
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.h b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.h
new file mode 100644
index 000000000000..6d8f61ea467a
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.h
@@ -0,0 +1,310 @@
+#ifndef __NVKM_VMM_H__
+#define __NVKM_VMM_H__
+#include "priv.h"
+#include <core/memory.h>
+enum nvkm_memory_target;
+
+struct nvkm_vmm_pt {
+ /* Some GPUs have a mapping level with a dual page tables to
+ * support large and small pages in the same address-range.
+ *
+ * We track the state of both page tables in one place, which
+ * is why there's multiple PT pointers/refcounts here.
+ */
+ struct nvkm_mmu_pt *pt[2];
+ u32 refs[2];
+
+ /* Page size handled by this PT.
+ *
+ * Tesla backend needs to know this when writinge PDEs,
+ * otherwise unnecessary.
+ */
+ u8 page;
+
+ /* Entire page table sparse.
+ *
+ * Used to propagate sparseness to child page tables.
+ */
+ bool sparse:1;
+
+ /* Tracking for page directories.
+ *
+ * The array is indexed by PDE, and will either point to the
+ * child page table, or indicate the PDE is marked as sparse.
+ **/
+#define NVKM_VMM_PDE_INVALID(pde) IS_ERR_OR_NULL(pde)
+#define NVKM_VMM_PDE_SPARSED(pde) IS_ERR(pde)
+#define NVKM_VMM_PDE_SPARSE ERR_PTR(-EBUSY)
+ struct nvkm_vmm_pt **pde;
+
+ /* Tracking for dual page tables.
+ *
+ * There's one entry for each LPTE, keeping track of whether
+ * there are valid SPTEs in the same address-range.
+ *
+ * This information is used to manage LPTE state transitions.
+ */
+#define NVKM_VMM_PTE_SPARSE 0x80
+#define NVKM_VMM_PTE_VALID 0x40
+#define NVKM_VMM_PTE_SPTES 0x3f
+ u8 pte[];
+};
+
+typedef void (*nvkm_vmm_pxe_func)(struct nvkm_vmm *,
+ struct nvkm_mmu_pt *, u32 ptei, u32 ptes);
+typedef void (*nvkm_vmm_pde_func)(struct nvkm_vmm *,
+ struct nvkm_vmm_pt *, u32 pdei);
+typedef void (*nvkm_vmm_pte_func)(struct nvkm_vmm *, struct nvkm_mmu_pt *,
+ u32 ptei, u32 ptes, struct nvkm_vmm_map *);
+
+struct nvkm_vmm_desc_func {
+ nvkm_vmm_pxe_func invalid;
+ nvkm_vmm_pxe_func unmap;
+ nvkm_vmm_pxe_func sparse;
+
+ nvkm_vmm_pde_func pde;
+
+ nvkm_vmm_pte_func mem;
+ nvkm_vmm_pte_func dma;
+ nvkm_vmm_pte_func sgl;
+};
+
+extern const struct nvkm_vmm_desc_func gf100_vmm_pgd;
+void gf100_vmm_pgd_pde(struct nvkm_vmm *, struct nvkm_vmm_pt *, u32);
+extern const struct nvkm_vmm_desc_func gf100_vmm_pgt;
+void gf100_vmm_pgt_unmap(struct nvkm_vmm *, struct nvkm_mmu_pt *, u32, u32);
+void gf100_vmm_pgt_mem(struct nvkm_vmm *, struct nvkm_mmu_pt *, u32, u32,
+ struct nvkm_vmm_map *);
+void gf100_vmm_pgt_dma(struct nvkm_vmm *, struct nvkm_mmu_pt *, u32, u32,
+ struct nvkm_vmm_map *);
+void gf100_vmm_pgt_sgl(struct nvkm_vmm *, struct nvkm_mmu_pt *, u32, u32,
+ struct nvkm_vmm_map *);
+
+void gk104_vmm_lpt_invalid(struct nvkm_vmm *, struct nvkm_mmu_pt *, u32, u32);
+
+struct nvkm_vmm_desc {
+ enum {
+ PGD,
+ PGT,
+ SPT,
+ LPT,
+ } type;
+ u8 bits; /* VMA bits covered by PT. */
+ u8 size; /* Bytes-per-PTE. */
+ u32 align; /* PT address alignment. */
+ const struct nvkm_vmm_desc_func *func;
+};
+
+extern const struct nvkm_vmm_desc gk104_vmm_desc_16_12[];
+extern const struct nvkm_vmm_desc gk104_vmm_desc_16_16[];
+extern const struct nvkm_vmm_desc gk104_vmm_desc_17_12[];
+extern const struct nvkm_vmm_desc gk104_vmm_desc_17_17[];
+
+extern const struct nvkm_vmm_desc gm200_vmm_desc_16_12[];
+extern const struct nvkm_vmm_desc gm200_vmm_desc_16_16[];
+extern const struct nvkm_vmm_desc gm200_vmm_desc_17_12[];
+extern const struct nvkm_vmm_desc gm200_vmm_desc_17_17[];
+
+extern const struct nvkm_vmm_desc gp100_vmm_desc_12[];
+extern const struct nvkm_vmm_desc gp100_vmm_desc_16[];
+
+struct nvkm_vmm_page {
+ u8 shift;
+ const struct nvkm_vmm_desc *desc;
+#define NVKM_VMM_PAGE_SPARSE 0x01
+#define NVKM_VMM_PAGE_VRAM 0x02
+#define NVKM_VMM_PAGE_HOST 0x04
+#define NVKM_VMM_PAGE_COMP 0x08
+#define NVKM_VMM_PAGE_Sxxx (NVKM_VMM_PAGE_SPARSE)
+#define NVKM_VMM_PAGE_xVxx (NVKM_VMM_PAGE_VRAM)
+#define NVKM_VMM_PAGE_SVxx (NVKM_VMM_PAGE_Sxxx | NVKM_VMM_PAGE_VRAM)
+#define NVKM_VMM_PAGE_xxHx (NVKM_VMM_PAGE_HOST)
+#define NVKM_VMM_PAGE_SxHx (NVKM_VMM_PAGE_Sxxx | NVKM_VMM_PAGE_HOST)
+#define NVKM_VMM_PAGE_xVHx (NVKM_VMM_PAGE_xVxx | NVKM_VMM_PAGE_HOST)
+#define NVKM_VMM_PAGE_SVHx (NVKM_VMM_PAGE_SVxx | NVKM_VMM_PAGE_HOST)
+#define NVKM_VMM_PAGE_xVxC (NVKM_VMM_PAGE_xVxx | NVKM_VMM_PAGE_COMP)
+#define NVKM_VMM_PAGE_SVxC (NVKM_VMM_PAGE_SVxx | NVKM_VMM_PAGE_COMP)
+#define NVKM_VMM_PAGE_xxHC (NVKM_VMM_PAGE_xxHx | NVKM_VMM_PAGE_COMP)
+#define NVKM_VMM_PAGE_SxHC (NVKM_VMM_PAGE_SxHx | NVKM_VMM_PAGE_COMP)
+ u8 type;
+};
+
+struct nvkm_vmm_func {
+ int (*join)(struct nvkm_vmm *, struct nvkm_memory *inst);
+ void (*part)(struct nvkm_vmm *, struct nvkm_memory *inst);
+
+ int (*aper)(enum nvkm_memory_target);
+ int (*valid)(struct nvkm_vmm *, void *argv, u32 argc,
+ struct nvkm_vmm_map *);
+ void (*flush)(struct nvkm_vmm *, int depth);
+
+ u64 page_block;
+ const struct nvkm_vmm_page page[];
+};
+
+struct nvkm_vmm_join {
+ struct nvkm_memory *inst;
+ struct list_head head;
+};
+
+int nvkm_vmm_new_(const struct nvkm_vmm_func *, struct nvkm_mmu *,
+ u32 pd_header, u64 addr, u64 size, struct lock_class_key *,
+ const char *name, struct nvkm_vmm **);
+int nvkm_vmm_ctor(const struct nvkm_vmm_func *, struct nvkm_mmu *,
+ u32 pd_header, u64 addr, u64 size, struct lock_class_key *,
+ const char *name, struct nvkm_vmm *);
+struct nvkm_vma *nvkm_vmm_node_search(struct nvkm_vmm *, u64 addr);
+int nvkm_vmm_get_locked(struct nvkm_vmm *, bool getref, bool mapref,
+ bool sparse, u8 page, u8 align, u64 size,
+ struct nvkm_vma **pvma);
+void nvkm_vmm_put_locked(struct nvkm_vmm *, struct nvkm_vma *);
+void nvkm_vmm_unmap_locked(struct nvkm_vmm *, struct nvkm_vma *);
+void nvkm_vmm_unmap_region(struct nvkm_vmm *vmm, struct nvkm_vma *vma);
+
+struct nvkm_vma *nvkm_vma_tail(struct nvkm_vma *, u64 tail);
+void nvkm_vmm_node_insert(struct nvkm_vmm *, struct nvkm_vma *);
+
+int nv04_vmm_new_(const struct nvkm_vmm_func *, struct nvkm_mmu *, u32,
+ u64, u64, void *, u32, struct lock_class_key *,
+ const char *, struct nvkm_vmm **);
+int nv04_vmm_valid(struct nvkm_vmm *, void *, u32, struct nvkm_vmm_map *);
+
+int gf100_vmm_new_(const struct nvkm_vmm_func *, const struct nvkm_vmm_func *,
+ struct nvkm_mmu *, u64, u64, void *, u32,
+ struct lock_class_key *, const char *, struct nvkm_vmm **);
+int gf100_vmm_join_(struct nvkm_vmm *, struct nvkm_memory *, u64 base);
+int gf100_vmm_join(struct nvkm_vmm *, struct nvkm_memory *);
+void gf100_vmm_part(struct nvkm_vmm *, struct nvkm_memory *);
+int gf100_vmm_aper(enum nvkm_memory_target);
+int gf100_vmm_valid(struct nvkm_vmm *, void *, u32, struct nvkm_vmm_map *);
+void gf100_vmm_flush_(struct nvkm_vmm *, int);
+void gf100_vmm_flush(struct nvkm_vmm *, int);
+
+int gk20a_vmm_aper(enum nvkm_memory_target);
+
+int gm200_vmm_new_(const struct nvkm_vmm_func *, const struct nvkm_vmm_func *,
+ struct nvkm_mmu *, u64, u64, void *, u32,
+ struct lock_class_key *, const char *, struct nvkm_vmm **);
+int gm200_vmm_join_(struct nvkm_vmm *, struct nvkm_memory *, u64 base);
+int gm200_vmm_join(struct nvkm_vmm *, struct nvkm_memory *);
+
+int gp100_vmm_join(struct nvkm_vmm *, struct nvkm_memory *);
+int gp100_vmm_valid(struct nvkm_vmm *, void *, u32, struct nvkm_vmm_map *);
+void gp100_vmm_flush(struct nvkm_vmm *, int);
+
+int nv04_vmm_new(struct nvkm_mmu *, u64, u64, void *, u32,
+ struct lock_class_key *, const char *, struct nvkm_vmm **);
+int nv41_vmm_new(struct nvkm_mmu *, u64, u64, void *, u32,
+ struct lock_class_key *, const char *, struct nvkm_vmm **);
+int nv44_vmm_new(struct nvkm_mmu *, u64, u64, void *, u32,
+ struct lock_class_key *, const char *, struct nvkm_vmm **);
+int nv50_vmm_new(struct nvkm_mmu *, u64, u64, void *, u32,
+ struct lock_class_key *, const char *, struct nvkm_vmm **);
+int g84_vmm_new(struct nvkm_mmu *, u64, u64, void *, u32,
+ struct lock_class_key *, const char *, struct nvkm_vmm **);
+int gf100_vmm_new(struct nvkm_mmu *, u64, u64, void *, u32,
+ struct lock_class_key *, const char *, struct nvkm_vmm **);
+int gk104_vmm_new(struct nvkm_mmu *, u64, u64, void *, u32,
+ struct lock_class_key *, const char *, struct nvkm_vmm **);
+int gk20a_vmm_new(struct nvkm_mmu *, u64, u64, void *, u32,
+ struct lock_class_key *, const char *, struct nvkm_vmm **);
+int gm200_vmm_new_fixed(struct nvkm_mmu *, u64, u64, void *, u32,
+ struct lock_class_key *, const char *,
+ struct nvkm_vmm **);
+int gm200_vmm_new(struct nvkm_mmu *, u64, u64, void *, u32,
+ struct lock_class_key *, const char *,
+ struct nvkm_vmm **);
+int gm20b_vmm_new_fixed(struct nvkm_mmu *, u64, u64, void *, u32,
+ struct lock_class_key *, const char *,
+ struct nvkm_vmm **);
+int gm20b_vmm_new(struct nvkm_mmu *, u64, u64, void *, u32,
+ struct lock_class_key *, const char *,
+ struct nvkm_vmm **);
+int gp100_vmm_new(struct nvkm_mmu *, u64, u64, void *, u32,
+ struct lock_class_key *, const char *,
+ struct nvkm_vmm **);
+int gp10b_vmm_new(struct nvkm_mmu *, u64, u64, void *, u32,
+ struct lock_class_key *, const char *,
+ struct nvkm_vmm **);
+
+#define VMM_PRINT(l,v,p,f,a...) do { \
+ struct nvkm_vmm *_vmm = (v); \
+ if (CONFIG_NOUVEAU_DEBUG >= (l) && _vmm->debug >= (l)) { \
+ nvkm_printk_(&_vmm->mmu->subdev, 0, p, "%s: "f"\n", \
+ _vmm->name, ##a); \
+ } \
+} while(0)
+#define VMM_DEBUG(v,f,a...) VMM_PRINT(NV_DBG_DEBUG, (v), info, f, ##a)
+#define VMM_TRACE(v,f,a...) VMM_PRINT(NV_DBG_TRACE, (v), info, f, ##a)
+#define VMM_SPAM(v,f,a...) VMM_PRINT(NV_DBG_SPAM , (v), dbg, f, ##a)
+
+#define VMM_MAP_ITER(VMM,PT,PTEI,PTEN,MAP,FILL,BASE,SIZE,NEXT) do { \
+ nvkm_kmap((PT)->memory); \
+ while (PTEN) { \
+ u64 _ptes = ((SIZE) - MAP->off) >> MAP->page->shift; \
+ u64 _addr = ((BASE) + MAP->off); \
+ \
+ if (_ptes > PTEN) { \
+ MAP->off += PTEN << MAP->page->shift; \
+ _ptes = PTEN; \
+ } else { \
+ MAP->off = 0; \
+ NEXT; \
+ } \
+ \
+ VMM_SPAM(VMM, "ITER %08x %08x PTE(s)", PTEI, (u32)_ptes); \
+ \
+ FILL(VMM, PT, PTEI, _ptes, MAP, _addr); \
+ PTEI += _ptes; \
+ PTEN -= _ptes; \
+ }; \
+ nvkm_done((PT)->memory); \
+} while(0)
+
+#define VMM_MAP_ITER_MEM(VMM,PT,PTEI,PTEN,MAP,FILL) \
+ VMM_MAP_ITER(VMM,PT,PTEI,PTEN,MAP,FILL, \
+ ((u64)MAP->mem->offset << NVKM_RAM_MM_SHIFT), \
+ ((u64)MAP->mem->length << NVKM_RAM_MM_SHIFT), \
+ (MAP->mem = MAP->mem->next))
+#define VMM_MAP_ITER_DMA(VMM,PT,PTEI,PTEN,MAP,FILL) \
+ VMM_MAP_ITER(VMM,PT,PTEI,PTEN,MAP,FILL, \
+ *MAP->dma, PAGE_SIZE, MAP->dma++)
+#define VMM_MAP_ITER_SGL(VMM,PT,PTEI,PTEN,MAP,FILL) \
+ VMM_MAP_ITER(VMM,PT,PTEI,PTEN,MAP,FILL, \
+ sg_dma_address(MAP->sgl), sg_dma_len(MAP->sgl), \
+ (MAP->sgl = sg_next(MAP->sgl)))
+
+#define VMM_FO(m,o,d,c,b) nvkm_fo##b((m)->memory, (o), (d), (c))
+#define VMM_WO(m,o,d,c,b) nvkm_wo##b((m)->memory, (o), (d))
+#define VMM_XO(m,v,o,d,c,b,fn,f,a...) do { \
+ const u32 _pteo = (o); u##b _data = (d); \
+ VMM_SPAM((v), " %010llx "f, (m)->addr + _pteo, _data, ##a); \
+ VMM_##fn((m), (m)->base + _pteo, _data, (c), b); \
+} while(0)
+
+#define VMM_WO032(m,v,o,d) VMM_XO((m),(v),(o),(d), 1, 32, WO, "%08x")
+#define VMM_FO032(m,v,o,d,c) \
+ VMM_XO((m),(v),(o),(d),(c), 32, FO, "%08x %08x", (c))
+
+#define VMM_WO064(m,v,o,d) VMM_XO((m),(v),(o),(d), 1, 64, WO, "%016llx")
+#define VMM_FO064(m,v,o,d,c) \
+ VMM_XO((m),(v),(o),(d),(c), 64, FO, "%016llx %08x", (c))
+
+#define VMM_XO128(m,v,o,lo,hi,c,f,a...) do { \
+ u32 _pteo = (o), _ptes = (c); \
+ const u64 _addr = (m)->addr + _pteo; \
+ VMM_SPAM((v), " %010llx %016llx%016llx"f, _addr, (hi), (lo), ##a); \
+ while (_ptes--) { \
+ nvkm_wo64((m)->memory, (m)->base + _pteo + 0, (lo)); \
+ nvkm_wo64((m)->memory, (m)->base + _pteo + 8, (hi)); \
+ _pteo += 0x10; \
+ } \
+} while(0)
+
+#define VMM_WO128(m,v,o,lo,hi) VMM_XO128((m),(v),(o),(lo),(hi), 1, "")
+#define VMM_FO128(m,v,o,lo,hi,c) do { \
+ nvkm_kmap((m)->memory); \
+ VMM_XO128((m),(v),(o),(lo),(hi),(c), " %08x", (c)); \
+ nvkm_done((m)->memory); \
+} while(0)
+#endif
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgf100.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgf100.c
new file mode 100644
index 000000000000..faf5a7e9265e
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgf100.c
@@ -0,0 +1,403 @@
+/*
+ * Copyright 2017 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+#include "vmm.h"
+
+#include <subdev/fb.h>
+#include <subdev/ltc.h>
+#include <subdev/timer.h>
+
+#include <nvif/if900d.h>
+#include <nvif/unpack.h>
+
+static inline void
+gf100_vmm_pgt_pte(struct nvkm_vmm *vmm, struct nvkm_mmu_pt *pt,
+ u32 ptei, u32 ptes, struct nvkm_vmm_map *map, u64 addr)
+{
+ u64 base = (addr >> 8) | map->type;
+ u64 data = base;
+
+ if (map->ctag && !(map->next & (1ULL << 44))) {
+ while (ptes--) {
+ data = base | ((map->ctag >> 1) << 44);
+ if (!(map->ctag++ & 1))
+ data |= BIT_ULL(60);
+
+ VMM_WO064(pt, vmm, ptei++ * 8, data);
+ base += map->next;
+ }
+ } else {
+ map->type += ptes * map->ctag;
+
+ while (ptes--) {
+ VMM_WO064(pt, vmm, ptei++ * 8, data);
+ data += map->next;
+ }
+ }
+}
+
+void
+gf100_vmm_pgt_sgl(struct nvkm_vmm *vmm, struct nvkm_mmu_pt *pt,
+ u32 ptei, u32 ptes, struct nvkm_vmm_map *map)
+{
+ VMM_MAP_ITER_SGL(vmm, pt, ptei, ptes, map, gf100_vmm_pgt_pte);
+}
+
+void
+gf100_vmm_pgt_dma(struct nvkm_vmm *vmm, struct nvkm_mmu_pt *pt,
+ u32 ptei, u32 ptes, struct nvkm_vmm_map *map)
+{
+ if (map->page->shift == PAGE_SHIFT) {
+ VMM_SPAM(vmm, "DMAA %08x %08x PTE(s)", ptei, ptes);
+ nvkm_kmap(pt->memory);
+ while (ptes--) {
+ const u64 data = (*map->dma++ >> 8) | map->type;
+ VMM_WO064(pt, vmm, ptei++ * 8, data);
+ map->type += map->ctag;
+ }
+ nvkm_done(pt->memory);
+ return;
+ }
+
+ VMM_MAP_ITER_DMA(vmm, pt, ptei, ptes, map, gf100_vmm_pgt_pte);
+}
+
+void
+gf100_vmm_pgt_mem(struct nvkm_vmm *vmm, struct nvkm_mmu_pt *pt,
+ u32 ptei, u32 ptes, struct nvkm_vmm_map *map)
+{
+ VMM_MAP_ITER_MEM(vmm, pt, ptei, ptes, map, gf100_vmm_pgt_pte);
+}
+
+void
+gf100_vmm_pgt_unmap(struct nvkm_vmm *vmm,
+ struct nvkm_mmu_pt *pt, u32 ptei, u32 ptes)
+{
+ VMM_FO064(pt, vmm, ptei * 8, 0ULL, ptes);
+}
+
+const struct nvkm_vmm_desc_func
+gf100_vmm_pgt = {
+ .unmap = gf100_vmm_pgt_unmap,
+ .mem = gf100_vmm_pgt_mem,
+ .dma = gf100_vmm_pgt_dma,
+ .sgl = gf100_vmm_pgt_sgl,
+};
+
+void
+gf100_vmm_pgd_pde(struct nvkm_vmm *vmm, struct nvkm_vmm_pt *pgd, u32 pdei)
+{
+ struct nvkm_vmm_pt *pgt = pgd->pde[pdei];
+ struct nvkm_mmu_pt *pd = pgd->pt[0];
+ struct nvkm_mmu_pt *pt;
+ u64 data = 0;
+
+ if ((pt = pgt->pt[0])) {
+ switch (nvkm_memory_target(pt->memory)) {
+ case NVKM_MEM_TARGET_VRAM: data |= 1ULL << 0; break;
+ case NVKM_MEM_TARGET_HOST: data |= 2ULL << 0;
+ data |= BIT_ULL(35); /* VOL */
+ break;
+ case NVKM_MEM_TARGET_NCOH: data |= 3ULL << 0; break;
+ default:
+ WARN_ON(1);
+ return;
+ }
+ data |= pt->addr >> 8;
+ }
+
+ if ((pt = pgt->pt[1])) {
+ switch (nvkm_memory_target(pt->memory)) {
+ case NVKM_MEM_TARGET_VRAM: data |= 1ULL << 32; break;
+ case NVKM_MEM_TARGET_HOST: data |= 2ULL << 32;
+ data |= BIT_ULL(34); /* VOL */
+ break;
+ case NVKM_MEM_TARGET_NCOH: data |= 3ULL << 32; break;
+ default:
+ WARN_ON(1);
+ return;
+ }
+ data |= pt->addr << 24;
+ }
+
+ nvkm_kmap(pd->memory);
+ VMM_WO064(pd, vmm, pdei * 8, data);
+ nvkm_done(pd->memory);
+}
+
+const struct nvkm_vmm_desc_func
+gf100_vmm_pgd = {
+ .unmap = gf100_vmm_pgt_unmap,
+ .pde = gf100_vmm_pgd_pde,
+};
+
+static const struct nvkm_vmm_desc
+gf100_vmm_desc_17_12[] = {
+ { SPT, 15, 8, 0x1000, &gf100_vmm_pgt },
+ { PGD, 13, 8, 0x1000, &gf100_vmm_pgd },
+ {}
+};
+
+static const struct nvkm_vmm_desc
+gf100_vmm_desc_17_17[] = {
+ { LPT, 10, 8, 0x1000, &gf100_vmm_pgt },
+ { PGD, 13, 8, 0x1000, &gf100_vmm_pgd },
+ {}
+};
+
+static const struct nvkm_vmm_desc
+gf100_vmm_desc_16_12[] = {
+ { SPT, 14, 8, 0x1000, &gf100_vmm_pgt },
+ { PGD, 14, 8, 0x1000, &gf100_vmm_pgd },
+ {}
+};
+
+static const struct nvkm_vmm_desc
+gf100_vmm_desc_16_16[] = {
+ { LPT, 10, 8, 0x1000, &gf100_vmm_pgt },
+ { PGD, 14, 8, 0x1000, &gf100_vmm_pgd },
+ {}
+};
+
+void
+gf100_vmm_flush_(struct nvkm_vmm *vmm, int depth)
+{
+ struct nvkm_subdev *subdev = &vmm->mmu->subdev;
+ struct nvkm_device *device = subdev->device;
+ u32 type = depth << 24;
+
+ type = 0x00000001; /* PAGE_ALL */
+ if (atomic_read(&vmm->engref[NVKM_SUBDEV_BAR]))
+ type |= 0x00000004; /* HUB_ONLY */
+
+ mutex_lock(&subdev->mutex);
+ /* Looks like maybe a "free flush slots" counter, the
+ * faster you write to 0x100cbc to more it decreases.
+ */
+ nvkm_msec(device, 2000,
+ if (nvkm_rd32(device, 0x100c80) & 0x00ff0000)
+ break;
+ );
+
+ nvkm_wr32(device, 0x100cb8, vmm->pd->pt[0]->addr >> 8);
+ nvkm_wr32(device, 0x100cbc, 0x80000000 | type);
+
+ /* Wait for flush to be queued? */
+ nvkm_msec(device, 2000,
+ if (nvkm_rd32(device, 0x100c80) & 0x00008000)
+ break;
+ );
+ mutex_unlock(&subdev->mutex);
+}
+
+void
+gf100_vmm_flush(struct nvkm_vmm *vmm, int depth)
+{
+ gf100_vmm_flush_(vmm, 0);
+}
+
+int
+gf100_vmm_valid(struct nvkm_vmm *vmm, void *argv, u32 argc,
+ struct nvkm_vmm_map *map)
+{
+ const enum nvkm_memory_target target = nvkm_memory_target(map->memory);
+ const struct nvkm_vmm_page *page = map->page;
+ const bool gm20x = page->desc->func->sparse != NULL;
+ union {
+ struct gf100_vmm_map_vn vn;
+ struct gf100_vmm_map_v0 v0;
+ } *args = argv;
+ struct nvkm_device *device = vmm->mmu->subdev.device;
+ struct nvkm_memory *memory = map->memory;
+ u8 kind, priv, ro, vol;
+ int kindn, aper, ret = -ENOSYS;
+ const u8 *kindm;
+
+ map->next = (1 << page->shift) >> 8;
+ map->type = map->ctag = 0;
+
+ if (!(ret = nvif_unpack(ret, &argv, &argc, args->v0, 0, 0, false))) {
+ vol = !!args->v0.vol;
+ ro = !!args->v0.ro;
+ priv = !!args->v0.priv;
+ kind = args->v0.kind;
+ } else
+ if (!(ret = nvif_unvers(ret, &argv, &argc, args->vn))) {
+ vol = target == NVKM_MEM_TARGET_HOST;
+ ro = 0;
+ priv = 0;
+ kind = 0x00;
+ } else {
+ VMM_DEBUG(vmm, "args");
+ return ret;
+ }
+
+ aper = vmm->func->aper(target);
+ if (WARN_ON(aper < 0))
+ return aper;
+
+ kindm = vmm->mmu->func->kind(vmm->mmu, &kindn);
+ if (kind >= kindn || kindm[kind] == 0xff) {
+ VMM_DEBUG(vmm, "kind %02x", kind);
+ return -EINVAL;
+ }
+
+ if (kindm[kind] != kind) {
+ u32 comp = (page->shift == 16 && !gm20x) ? 16 : 17;
+ u32 tags = ALIGN(nvkm_memory_size(memory), 1 << 17) >> comp;
+ if (aper != 0 || !(page->type & NVKM_VMM_PAGE_COMP)) {
+ VMM_DEBUG(vmm, "comp %d %02x", aper, page->type);
+ return -EINVAL;
+ }
+
+ ret = nvkm_memory_tags_get(memory, device, tags,
+ nvkm_ltc_tags_clear,
+ &map->tags);
+ if (ret) {
+ VMM_DEBUG(vmm, "comp %d", ret);
+ return ret;
+ }
+
+ if (map->tags->mn) {
+ u64 tags = map->tags->mn->offset + (map->offset >> 17);
+ if (page->shift == 17 || !gm20x) {
+ map->type |= tags << 44;
+ map->ctag |= 1ULL << 44;
+ map->next |= 1ULL << 44;
+ } else {
+ map->ctag |= tags << 1 | 1;
+ }
+ } else {
+ kind = kindm[kind];
+ }
+ }
+
+ map->type |= BIT(0);
+ map->type |= (u64)priv << 1;
+ map->type |= (u64) ro << 2;
+ map->type |= (u64) vol << 32;
+ map->type |= (u64)aper << 33;
+ map->type |= (u64)kind << 36;
+ return 0;
+}
+
+int
+gf100_vmm_aper(enum nvkm_memory_target target)
+{
+ switch (target) {
+ case NVKM_MEM_TARGET_VRAM: return 0;
+ case NVKM_MEM_TARGET_HOST: return 2;
+ case NVKM_MEM_TARGET_NCOH: return 3;
+ default:
+ return -EINVAL;
+ }
+}
+
+void
+gf100_vmm_part(struct nvkm_vmm *vmm, struct nvkm_memory *inst)
+{
+ nvkm_fo64(inst, 0x0200, 0x00000000, 2);
+}
+
+int
+gf100_vmm_join_(struct nvkm_vmm *vmm, struct nvkm_memory *inst, u64 base)
+{
+ struct nvkm_mmu_pt *pd = vmm->pd->pt[0];
+
+ switch (nvkm_memory_target(pd->memory)) {
+ case NVKM_MEM_TARGET_VRAM: base |= 0ULL << 0; break;
+ case NVKM_MEM_TARGET_HOST: base |= 2ULL << 0;
+ base |= BIT_ULL(2) /* VOL. */;
+ break;
+ case NVKM_MEM_TARGET_NCOH: base |= 3ULL << 0; break;
+ default:
+ WARN_ON(1);
+ return -EINVAL;
+ }
+ base |= pd->addr;
+
+ nvkm_kmap(inst);
+ nvkm_wo64(inst, 0x0200, base);
+ nvkm_wo64(inst, 0x0208, vmm->limit - 1);
+ nvkm_done(inst);
+ return 0;
+}
+
+int
+gf100_vmm_join(struct nvkm_vmm *vmm, struct nvkm_memory *inst)
+{
+ return gf100_vmm_join_(vmm, inst, 0);
+}
+
+static const struct nvkm_vmm_func
+gf100_vmm_17 = {
+ .join = gf100_vmm_join,
+ .part = gf100_vmm_part,
+ .aper = gf100_vmm_aper,
+ .valid = gf100_vmm_valid,
+ .flush = gf100_vmm_flush,
+ .page = {
+ { 17, &gf100_vmm_desc_17_17[0], NVKM_VMM_PAGE_xVxC },
+ { 12, &gf100_vmm_desc_17_12[0], NVKM_VMM_PAGE_xVHx },
+ {}
+ }
+};
+
+static const struct nvkm_vmm_func
+gf100_vmm_16 = {
+ .join = gf100_vmm_join,
+ .part = gf100_vmm_part,
+ .aper = gf100_vmm_aper,
+ .valid = gf100_vmm_valid,
+ .flush = gf100_vmm_flush,
+ .page = {
+ { 16, &gf100_vmm_desc_16_16[0], NVKM_VMM_PAGE_xVxC },
+ { 12, &gf100_vmm_desc_16_12[0], NVKM_VMM_PAGE_xVHx },
+ {}
+ }
+};
+
+int
+gf100_vmm_new_(const struct nvkm_vmm_func *func_16,
+ const struct nvkm_vmm_func *func_17,
+ struct nvkm_mmu *mmu, u64 addr, u64 size, void *argv, u32 argc,
+ struct lock_class_key *key, const char *name,
+ struct nvkm_vmm **pvmm)
+{
+ switch (mmu->subdev.device->fb->page) {
+ case 16: return nv04_vmm_new_(func_16, mmu, 0, addr, size,
+ argv, argc, key, name, pvmm);
+ case 17: return nv04_vmm_new_(func_17, mmu, 0, addr, size,
+ argv, argc, key, name, pvmm);
+ default:
+ WARN_ON(1);
+ return -EINVAL;
+ }
+}
+
+int
+gf100_vmm_new(struct nvkm_mmu *mmu, u64 addr, u64 size, void *argv, u32 argc,
+ struct lock_class_key *key, const char *name,
+ struct nvkm_vmm **pvmm)
+{
+ return gf100_vmm_new_(&gf100_vmm_16, &gf100_vmm_17, mmu, addr,
+ size, argv, argc, key, name, pvmm);
+}
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgk104.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgk104.c
new file mode 100644
index 000000000000..0ebb7bccfcd2
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgk104.c
@@ -0,0 +1,102 @@
+/*
+ * Copyright 2017 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+#include "vmm.h"
+
+void
+gk104_vmm_lpt_invalid(struct nvkm_vmm *vmm,
+ struct nvkm_mmu_pt *pt, u32 ptei, u32 ptes)
+{
+ /* VALID_FALSE + PRIV tells the MMU to ignore corresponding SPTEs. */
+ VMM_FO064(pt, vmm, ptei * 8, BIT_ULL(1) /* PRIV. */, ptes);
+}
+
+static const struct nvkm_vmm_desc_func
+gk104_vmm_lpt = {
+ .invalid = gk104_vmm_lpt_invalid,
+ .unmap = gf100_vmm_pgt_unmap,
+ .mem = gf100_vmm_pgt_mem,
+};
+
+const struct nvkm_vmm_desc
+gk104_vmm_desc_17_12[] = {
+ { SPT, 15, 8, 0x1000, &gf100_vmm_pgt },
+ { PGD, 13, 8, 0x1000, &gf100_vmm_pgd },
+ {}
+};
+
+const struct nvkm_vmm_desc
+gk104_vmm_desc_17_17[] = {
+ { LPT, 10, 8, 0x1000, &gk104_vmm_lpt },
+ { PGD, 13, 8, 0x1000, &gf100_vmm_pgd },
+ {}
+};
+
+const struct nvkm_vmm_desc
+gk104_vmm_desc_16_12[] = {
+ { SPT, 14, 8, 0x1000, &gf100_vmm_pgt },
+ { PGD, 14, 8, 0x1000, &gf100_vmm_pgd },
+ {}
+};
+
+const struct nvkm_vmm_desc
+gk104_vmm_desc_16_16[] = {
+ { LPT, 10, 8, 0x1000, &gk104_vmm_lpt },
+ { PGD, 14, 8, 0x1000, &gf100_vmm_pgd },
+ {}
+};
+
+static const struct nvkm_vmm_func
+gk104_vmm_17 = {
+ .join = gf100_vmm_join,
+ .part = gf100_vmm_part,
+ .aper = gf100_vmm_aper,
+ .valid = gf100_vmm_valid,
+ .flush = gf100_vmm_flush,
+ .page = {
+ { 17, &gk104_vmm_desc_17_17[0], NVKM_VMM_PAGE_xVxC },
+ { 12, &gk104_vmm_desc_17_12[0], NVKM_VMM_PAGE_xVHx },
+ {}
+ }
+};
+
+static const struct nvkm_vmm_func
+gk104_vmm_16 = {
+ .join = gf100_vmm_join,
+ .part = gf100_vmm_part,
+ .aper = gf100_vmm_aper,
+ .valid = gf100_vmm_valid,
+ .flush = gf100_vmm_flush,
+ .page = {
+ { 16, &gk104_vmm_desc_16_16[0], NVKM_VMM_PAGE_xVxC },
+ { 12, &gk104_vmm_desc_16_12[0], NVKM_VMM_PAGE_xVHx },
+ {}
+ }
+};
+
+int
+gk104_vmm_new(struct nvkm_mmu *mmu, u64 addr, u64 size, void *argv, u32 argc,
+ struct lock_class_key *key, const char *name,
+ struct nvkm_vmm **pvmm)
+{
+ return gf100_vmm_new_(&gk104_vmm_16, &gk104_vmm_17, mmu, addr,
+ size, argv, argc, key, name, pvmm);
+}
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgk20a.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgk20a.c
new file mode 100644
index 000000000000..8086994a0446
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgk20a.c
@@ -0,0 +1,71 @@
+/*
+ * Copyright 2017 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+#include "vmm.h"
+
+#include <core/memory.h>
+
+int
+gk20a_vmm_aper(enum nvkm_memory_target target)
+{
+ switch (target) {
+ case NVKM_MEM_TARGET_NCOH: return 0;
+ default:
+ return -EINVAL;
+ }
+}
+
+static const struct nvkm_vmm_func
+gk20a_vmm_17 = {
+ .join = gf100_vmm_join,
+ .part = gf100_vmm_part,
+ .aper = gf100_vmm_aper,
+ .valid = gf100_vmm_valid,
+ .flush = gf100_vmm_flush,
+ .page = {
+ { 17, &gk104_vmm_desc_17_17[0], NVKM_VMM_PAGE_xxHC },
+ { 12, &gk104_vmm_desc_17_12[0], NVKM_VMM_PAGE_xxHx },
+ {}
+ }
+};
+
+static const struct nvkm_vmm_func
+gk20a_vmm_16 = {
+ .join = gf100_vmm_join,
+ .part = gf100_vmm_part,
+ .aper = gf100_vmm_aper,
+ .valid = gf100_vmm_valid,
+ .flush = gf100_vmm_flush,
+ .page = {
+ { 16, &gk104_vmm_desc_16_16[0], NVKM_VMM_PAGE_xxHC },
+ { 12, &gk104_vmm_desc_16_12[0], NVKM_VMM_PAGE_xxHx },
+ {}
+ }
+};
+
+int
+gk20a_vmm_new(struct nvkm_mmu *mmu, u64 addr, u64 size, void *argv, u32 argc,
+ struct lock_class_key *key, const char *name,
+ struct nvkm_vmm **pvmm)
+{
+ return gf100_vmm_new_(&gk20a_vmm_16, &gk20a_vmm_17, mmu, addr,
+ size, argv, argc, key, name, pvmm);
+}
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgm200.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgm200.c
new file mode 100644
index 000000000000..a1676a4644fe
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgm200.c
@@ -0,0 +1,185 @@
+/*
+ * Copyright 2017 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+#include "vmm.h"
+
+#include <nvif/ifb00d.h>
+#include <nvif/unpack.h>
+
+static void
+gm200_vmm_pgt_sparse(struct nvkm_vmm *vmm,
+ struct nvkm_mmu_pt *pt, u32 ptei, u32 ptes)
+{
+ /* VALID_FALSE + VOL tells the MMU to treat the PTE as sparse. */
+ VMM_FO064(pt, vmm, ptei * 8, BIT_ULL(32) /* VOL. */, ptes);
+}
+
+static const struct nvkm_vmm_desc_func
+gm200_vmm_spt = {
+ .unmap = gf100_vmm_pgt_unmap,
+ .sparse = gm200_vmm_pgt_sparse,
+ .mem = gf100_vmm_pgt_mem,
+ .dma = gf100_vmm_pgt_dma,
+ .sgl = gf100_vmm_pgt_sgl,
+};
+
+static const struct nvkm_vmm_desc_func
+gm200_vmm_lpt = {
+ .invalid = gk104_vmm_lpt_invalid,
+ .unmap = gf100_vmm_pgt_unmap,
+ .sparse = gm200_vmm_pgt_sparse,
+ .mem = gf100_vmm_pgt_mem,
+};
+
+static void
+gm200_vmm_pgd_sparse(struct nvkm_vmm *vmm,
+ struct nvkm_mmu_pt *pt, u32 pdei, u32 pdes)
+{
+ /* VALID_FALSE + VOL_BIG tells the MMU to treat the PDE as sparse. */
+ VMM_FO064(pt, vmm, pdei * 8, BIT_ULL(35) /* VOL_BIG. */, pdes);
+}
+
+static const struct nvkm_vmm_desc_func
+gm200_vmm_pgd = {
+ .unmap = gf100_vmm_pgt_unmap,
+ .sparse = gm200_vmm_pgd_sparse,
+ .pde = gf100_vmm_pgd_pde,
+};
+
+const struct nvkm_vmm_desc
+gm200_vmm_desc_17_12[] = {
+ { SPT, 15, 8, 0x1000, &gm200_vmm_spt },
+ { PGD, 13, 8, 0x1000, &gm200_vmm_pgd },
+ {}
+};
+
+const struct nvkm_vmm_desc
+gm200_vmm_desc_17_17[] = {
+ { LPT, 10, 8, 0x1000, &gm200_vmm_lpt },
+ { PGD, 13, 8, 0x1000, &gm200_vmm_pgd },
+ {}
+};
+
+const struct nvkm_vmm_desc
+gm200_vmm_desc_16_12[] = {
+ { SPT, 14, 8, 0x1000, &gm200_vmm_spt },
+ { PGD, 14, 8, 0x1000, &gm200_vmm_pgd },
+ {}
+};
+
+const struct nvkm_vmm_desc
+gm200_vmm_desc_16_16[] = {
+ { LPT, 10, 8, 0x1000, &gm200_vmm_lpt },
+ { PGD, 14, 8, 0x1000, &gm200_vmm_pgd },
+ {}
+};
+
+int
+gm200_vmm_join_(struct nvkm_vmm *vmm, struct nvkm_memory *inst, u64 base)
+{
+ if (vmm->func->page[1].shift == 16)
+ base |= BIT_ULL(11);
+ return gf100_vmm_join_(vmm, inst, base);
+}
+
+int
+gm200_vmm_join(struct nvkm_vmm *vmm, struct nvkm_memory *inst)
+{
+ return gm200_vmm_join_(vmm, inst, 0);
+}
+
+static const struct nvkm_vmm_func
+gm200_vmm_17 = {
+ .join = gm200_vmm_join,
+ .part = gf100_vmm_part,
+ .aper = gf100_vmm_aper,
+ .valid = gf100_vmm_valid,
+ .flush = gf100_vmm_flush,
+ .page = {
+ { 27, &gm200_vmm_desc_17_17[1], NVKM_VMM_PAGE_Sxxx },
+ { 17, &gm200_vmm_desc_17_17[0], NVKM_VMM_PAGE_SVxC },
+ { 12, &gm200_vmm_desc_17_12[0], NVKM_VMM_PAGE_SVHx },
+ {}
+ }
+};
+
+static const struct nvkm_vmm_func
+gm200_vmm_16 = {
+ .join = gm200_vmm_join,
+ .part = gf100_vmm_part,
+ .aper = gf100_vmm_aper,
+ .valid = gf100_vmm_valid,
+ .flush = gf100_vmm_flush,
+ .page = {
+ { 27, &gm200_vmm_desc_16_16[1], NVKM_VMM_PAGE_Sxxx },
+ { 16, &gm200_vmm_desc_16_16[0], NVKM_VMM_PAGE_SVxC },
+ { 12, &gm200_vmm_desc_16_12[0], NVKM_VMM_PAGE_SVHx },
+ {}
+ }
+};
+
+int
+gm200_vmm_new_(const struct nvkm_vmm_func *func_16,
+ const struct nvkm_vmm_func *func_17,
+ struct nvkm_mmu *mmu, u64 addr, u64 size, void *argv, u32 argc,
+ struct lock_class_key *key, const char *name,
+ struct nvkm_vmm **pvmm)
+{
+ const struct nvkm_vmm_func *func;
+ union {
+ struct gm200_vmm_vn vn;
+ struct gm200_vmm_v0 v0;
+ } *args = argv;
+ int ret = -ENOSYS;
+
+ if (!(ret = nvif_unpack(ret, &argv, &argc, args->v0, 0, 0, false))) {
+ switch (args->v0.bigpage) {
+ case 16: func = func_16; break;
+ case 17: func = func_17; break;
+ default:
+ return -EINVAL;
+ }
+ } else
+ if (!(ret = nvif_unvers(ret, &argv, &argc, args->vn))) {
+ func = func_17;
+ } else
+ return ret;
+
+ return nvkm_vmm_new_(func, mmu, 0, addr, size, key, name, pvmm);
+}
+
+int
+gm200_vmm_new(struct nvkm_mmu *mmu, u64 addr, u64 size, void *argv, u32 argc,
+ struct lock_class_key *key, const char *name,
+ struct nvkm_vmm **pvmm)
+{
+ return gm200_vmm_new_(&gm200_vmm_16, &gm200_vmm_17, mmu, addr,
+ size, argv, argc, key, name, pvmm);
+}
+
+int
+gm200_vmm_new_fixed(struct nvkm_mmu *mmu, u64 addr, u64 size,
+ void *argv, u32 argc, struct lock_class_key *key,
+ const char *name, struct nvkm_vmm **pvmm)
+{
+ return gf100_vmm_new_(&gm200_vmm_16, &gm200_vmm_17, mmu, addr,
+ size, argv, argc, key, name, pvmm);
+}
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgm20b.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgm20b.c
new file mode 100644
index 000000000000..64d4b6cff8dd
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgm20b.c
@@ -0,0 +1,70 @@
+/*
+ * Copyright 2017 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+#include "vmm.h"
+
+static const struct nvkm_vmm_func
+gm20b_vmm_17 = {
+ .join = gm200_vmm_join,
+ .part = gf100_vmm_part,
+ .aper = gk20a_vmm_aper,
+ .valid = gf100_vmm_valid,
+ .flush = gf100_vmm_flush,
+ .page = {
+ { 27, &gm200_vmm_desc_17_17[1], NVKM_VMM_PAGE_Sxxx },
+ { 17, &gm200_vmm_desc_17_17[0], NVKM_VMM_PAGE_SxHC },
+ { 12, &gm200_vmm_desc_17_12[0], NVKM_VMM_PAGE_SxHx },
+ {}
+ }
+};
+
+static const struct nvkm_vmm_func
+gm20b_vmm_16 = {
+ .join = gm200_vmm_join,
+ .part = gf100_vmm_part,
+ .aper = gk20a_vmm_aper,
+ .valid = gf100_vmm_valid,
+ .flush = gf100_vmm_flush,
+ .page = {
+ { 27, &gm200_vmm_desc_16_16[1], NVKM_VMM_PAGE_Sxxx },
+ { 16, &gm200_vmm_desc_16_16[0], NVKM_VMM_PAGE_SxHC },
+ { 12, &gm200_vmm_desc_16_12[0], NVKM_VMM_PAGE_SxHx },
+ {}
+ }
+};
+
+int
+gm20b_vmm_new(struct nvkm_mmu *mmu, u64 addr, u64 size, void *argv, u32 argc,
+ struct lock_class_key *key, const char *name,
+ struct nvkm_vmm **pvmm)
+{
+ return gm200_vmm_new_(&gm20b_vmm_16, &gm20b_vmm_17, mmu, addr,
+ size, argv, argc, key, name, pvmm);
+}
+
+int
+gm20b_vmm_new_fixed(struct nvkm_mmu *mmu, u64 addr, u64 size,
+ void *argv, u32 argc, struct lock_class_key *key,
+ const char *name, struct nvkm_vmm **pvmm)
+{
+ return gf100_vmm_new_(&gm20b_vmm_16, &gm20b_vmm_17, mmu, addr,
+ size, argv, argc, key, name, pvmm);
+}
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgp100.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgp100.c
new file mode 100644
index 000000000000..059fafe0e771
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgp100.c
@@ -0,0 +1,347 @@
+/*
+ * Copyright 2017 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+#include "vmm.h"
+
+#include <subdev/fb.h>
+#include <subdev/ltc.h>
+
+#include <nvif/ifc00d.h>
+#include <nvif/unpack.h>
+
+static inline void
+gp100_vmm_pgt_pte(struct nvkm_vmm *vmm, struct nvkm_mmu_pt *pt,
+ u32 ptei, u32 ptes, struct nvkm_vmm_map *map, u64 addr)
+{
+ u64 data = (addr >> 4) | map->type;
+
+ map->type += ptes * map->ctag;
+
+ while (ptes--) {
+ VMM_WO064(pt, vmm, ptei++ * 8, data);
+ data += map->next;
+ }
+}
+
+static void
+gp100_vmm_pgt_sgl(struct nvkm_vmm *vmm, struct nvkm_mmu_pt *pt,
+ u32 ptei, u32 ptes, struct nvkm_vmm_map *map)
+{
+ VMM_MAP_ITER_SGL(vmm, pt, ptei, ptes, map, gp100_vmm_pgt_pte);
+}
+
+static void
+gp100_vmm_pgt_dma(struct nvkm_vmm *vmm, struct nvkm_mmu_pt *pt,
+ u32 ptei, u32 ptes, struct nvkm_vmm_map *map)
+{
+ if (map->page->shift == PAGE_SHIFT) {
+ VMM_SPAM(vmm, "DMAA %08x %08x PTE(s)", ptei, ptes);
+ nvkm_kmap(pt->memory);
+ while (ptes--) {
+ const u64 data = (*map->dma++ >> 4) | map->type;
+ VMM_WO064(pt, vmm, ptei++ * 8, data);
+ map->type += map->ctag;
+ }
+ nvkm_done(pt->memory);
+ return;
+ }
+
+ VMM_MAP_ITER_DMA(vmm, pt, ptei, ptes, map, gp100_vmm_pgt_pte);
+}
+
+static void
+gp100_vmm_pgt_mem(struct nvkm_vmm *vmm, struct nvkm_mmu_pt *pt,
+ u32 ptei, u32 ptes, struct nvkm_vmm_map *map)
+{
+ VMM_MAP_ITER_MEM(vmm, pt, ptei, ptes, map, gp100_vmm_pgt_pte);
+}
+
+static void
+gp100_vmm_pgt_sparse(struct nvkm_vmm *vmm,
+ struct nvkm_mmu_pt *pt, u32 ptei, u32 ptes)
+{
+ /* VALID_FALSE + VOL tells the MMU to treat the PTE as sparse. */
+ VMM_FO064(pt, vmm, ptei * 8, BIT_ULL(3) /* VOL. */, ptes);
+}
+
+static const struct nvkm_vmm_desc_func
+gp100_vmm_desc_spt = {
+ .unmap = gf100_vmm_pgt_unmap,
+ .sparse = gp100_vmm_pgt_sparse,
+ .mem = gp100_vmm_pgt_mem,
+ .dma = gp100_vmm_pgt_dma,
+ .sgl = gp100_vmm_pgt_sgl,
+};
+
+static void
+gp100_vmm_lpt_invalid(struct nvkm_vmm *vmm,
+ struct nvkm_mmu_pt *pt, u32 ptei, u32 ptes)
+{
+ /* VALID_FALSE + PRIV tells the MMU to ignore corresponding SPTEs. */
+ VMM_FO064(pt, vmm, ptei * 8, BIT_ULL(5) /* PRIV. */, ptes);
+}
+
+static const struct nvkm_vmm_desc_func
+gp100_vmm_desc_lpt = {
+ .invalid = gp100_vmm_lpt_invalid,
+ .unmap = gf100_vmm_pgt_unmap,
+ .sparse = gp100_vmm_pgt_sparse,
+ .mem = gp100_vmm_pgt_mem,
+};
+
+static inline void
+gp100_vmm_pd0_pte(struct nvkm_vmm *vmm, struct nvkm_mmu_pt *pt,
+ u32 ptei, u32 ptes, struct nvkm_vmm_map *map, u64 addr)
+{
+ u64 data = (addr >> 4) | map->type;
+
+ map->type += ptes * map->ctag;
+
+ while (ptes--) {
+ VMM_WO128(pt, vmm, ptei++ * 0x10, data, 0ULL);
+ data += map->next;
+ }
+}
+
+static void
+gp100_vmm_pd0_mem(struct nvkm_vmm *vmm, struct nvkm_mmu_pt *pt,
+ u32 ptei, u32 ptes, struct nvkm_vmm_map *map)
+{
+ VMM_MAP_ITER_MEM(vmm, pt, ptei, ptes, map, gp100_vmm_pd0_pte);
+}
+
+static inline bool
+gp100_vmm_pde(struct nvkm_mmu_pt *pt, u64 *data)
+{
+ switch (nvkm_memory_target(pt->memory)) {
+ case NVKM_MEM_TARGET_VRAM: *data |= 1ULL << 1; break;
+ case NVKM_MEM_TARGET_HOST: *data |= 2ULL << 1;
+ *data |= BIT_ULL(3); /* VOL. */
+ break;
+ case NVKM_MEM_TARGET_NCOH: *data |= 3ULL << 1; break;
+ default:
+ WARN_ON(1);
+ return false;
+ }
+ *data |= pt->addr >> 4;
+ return true;
+}
+
+static void
+gp100_vmm_pd0_pde(struct nvkm_vmm *vmm, struct nvkm_vmm_pt *pgd, u32 pdei)
+{
+ struct nvkm_vmm_pt *pgt = pgd->pde[pdei];
+ struct nvkm_mmu_pt *pd = pgd->pt[0];
+ u64 data[2] = {};
+
+ if (pgt->pt[0] && !gp100_vmm_pde(pgt->pt[0], &data[0]))
+ return;
+ if (pgt->pt[1] && !gp100_vmm_pde(pgt->pt[1], &data[1]))
+ return;
+
+ nvkm_kmap(pd->memory);
+ VMM_WO128(pd, vmm, pdei * 0x10, data[0], data[1]);
+ nvkm_done(pd->memory);
+}
+
+static void
+gp100_vmm_pd0_sparse(struct nvkm_vmm *vmm,
+ struct nvkm_mmu_pt *pt, u32 pdei, u32 pdes)
+{
+ /* VALID_FALSE + VOL_BIG tells the MMU to treat the PDE as sparse. */
+ VMM_FO128(pt, vmm, pdei * 0x10, BIT_ULL(3) /* VOL_BIG. */, 0ULL, pdes);
+}
+
+static void
+gp100_vmm_pd0_unmap(struct nvkm_vmm *vmm,
+ struct nvkm_mmu_pt *pt, u32 pdei, u32 pdes)
+{
+ VMM_FO128(pt, vmm, pdei * 0x10, 0ULL, 0ULL, pdes);
+}
+
+static const struct nvkm_vmm_desc_func
+gp100_vmm_desc_pd0 = {
+ .unmap = gp100_vmm_pd0_unmap,
+ .sparse = gp100_vmm_pd0_sparse,
+ .pde = gp100_vmm_pd0_pde,
+ .mem = gp100_vmm_pd0_mem,
+};
+
+static void
+gp100_vmm_pd1_pde(struct nvkm_vmm *vmm, struct nvkm_vmm_pt *pgd, u32 pdei)
+{
+ struct nvkm_vmm_pt *pgt = pgd->pde[pdei];
+ struct nvkm_mmu_pt *pd = pgd->pt[0];
+ u64 data = 0;
+
+ if (!gp100_vmm_pde(pgt->pt[0], &data))
+ return;
+
+ nvkm_kmap(pd->memory);
+ VMM_WO064(pd, vmm, pdei * 8, data);
+ nvkm_done(pd->memory);
+}
+
+static const struct nvkm_vmm_desc_func
+gp100_vmm_desc_pd1 = {
+ .unmap = gf100_vmm_pgt_unmap,
+ .sparse = gp100_vmm_pgt_sparse,
+ .pde = gp100_vmm_pd1_pde,
+};
+
+const struct nvkm_vmm_desc
+gp100_vmm_desc_16[] = {
+ { LPT, 5, 8, 0x0100, &gp100_vmm_desc_lpt },
+ { PGD, 8, 16, 0x1000, &gp100_vmm_desc_pd0 },
+ { PGD, 9, 8, 0x1000, &gp100_vmm_desc_pd1 },
+ { PGD, 9, 8, 0x1000, &gp100_vmm_desc_pd1 },
+ { PGD, 2, 8, 0x1000, &gp100_vmm_desc_pd1 },
+ {}
+};
+
+const struct nvkm_vmm_desc
+gp100_vmm_desc_12[] = {
+ { SPT, 9, 8, 0x1000, &gp100_vmm_desc_spt },
+ { PGD, 8, 16, 0x1000, &gp100_vmm_desc_pd0 },
+ { PGD, 9, 8, 0x1000, &gp100_vmm_desc_pd1 },
+ { PGD, 9, 8, 0x1000, &gp100_vmm_desc_pd1 },
+ { PGD, 2, 8, 0x1000, &gp100_vmm_desc_pd1 },
+ {}
+};
+
+int
+gp100_vmm_valid(struct nvkm_vmm *vmm, void *argv, u32 argc,
+ struct nvkm_vmm_map *map)
+{
+ const enum nvkm_memory_target target = nvkm_memory_target(map->memory);
+ const struct nvkm_vmm_page *page = map->page;
+ union {
+ struct gp100_vmm_map_vn vn;
+ struct gp100_vmm_map_v0 v0;
+ } *args = argv;
+ struct nvkm_device *device = vmm->mmu->subdev.device;
+ struct nvkm_memory *memory = map->memory;
+ u8 kind, priv, ro, vol;
+ int kindn, aper, ret = -ENOSYS;
+ const u8 *kindm;
+
+ map->next = (1ULL << page->shift) >> 4;
+ map->type = 0;
+
+ if (!(ret = nvif_unpack(ret, &argv, &argc, args->v0, 0, 0, false))) {
+ vol = !!args->v0.vol;
+ ro = !!args->v0.ro;
+ priv = !!args->v0.priv;
+ kind = args->v0.kind;
+ } else
+ if (!(ret = nvif_unvers(ret, &argv, &argc, args->vn))) {
+ vol = target == NVKM_MEM_TARGET_HOST;
+ ro = 0;
+ priv = 0;
+ kind = 0x00;
+ } else {
+ VMM_DEBUG(vmm, "args");
+ return ret;
+ }
+
+ aper = vmm->func->aper(target);
+ if (WARN_ON(aper < 0))
+ return aper;
+
+ kindm = vmm->mmu->func->kind(vmm->mmu, &kindn);
+ if (kind >= kindn || kindm[kind] == 0xff) {
+ VMM_DEBUG(vmm, "kind %02x", kind);
+ return -EINVAL;
+ }
+
+ if (kindm[kind] != kind) {
+ u64 tags = nvkm_memory_size(memory) >> 16;
+ if (aper != 0 || !(page->type & NVKM_VMM_PAGE_COMP)) {
+ VMM_DEBUG(vmm, "comp %d %02x", aper, page->type);
+ return -EINVAL;
+ }
+
+ ret = nvkm_memory_tags_get(memory, device, tags,
+ nvkm_ltc_tags_clear,
+ &map->tags);
+ if (ret) {
+ VMM_DEBUG(vmm, "comp %d", ret);
+ return ret;
+ }
+
+ if (map->tags->mn) {
+ tags = map->tags->mn->offset + (map->offset >> 16);
+ map->ctag |= ((1ULL << page->shift) >> 16) << 36;
+ map->type |= tags << 36;
+ map->next |= map->ctag;
+ } else {
+ kind = kindm[kind];
+ }
+ }
+
+ map->type |= BIT(0);
+ map->type |= (u64)aper << 1;
+ map->type |= (u64) vol << 3;
+ map->type |= (u64)priv << 5;
+ map->type |= (u64) ro << 6;
+ map->type |= (u64)kind << 56;
+ return 0;
+}
+
+void
+gp100_vmm_flush(struct nvkm_vmm *vmm, int depth)
+{
+ gf100_vmm_flush_(vmm, 5 /* CACHE_LEVEL_UP_TO_PDE3 */ - depth);
+}
+
+int
+gp100_vmm_join(struct nvkm_vmm *vmm, struct nvkm_memory *inst)
+{
+ const u64 base = BIT_ULL(10) /* VER2 */ | BIT_ULL(11); /* 64KiB */
+ return gf100_vmm_join_(vmm, inst, base);
+}
+
+static const struct nvkm_vmm_func
+gp100_vmm = {
+ .join = gp100_vmm_join,
+ .part = gf100_vmm_part,
+ .aper = gf100_vmm_aper,
+ .valid = gp100_vmm_valid,
+ .flush = gp100_vmm_flush,
+ .page = {
+ { 47, &gp100_vmm_desc_16[4], NVKM_VMM_PAGE_Sxxx },
+ { 38, &gp100_vmm_desc_16[3], NVKM_VMM_PAGE_Sxxx },
+ { 29, &gp100_vmm_desc_16[2], NVKM_VMM_PAGE_Sxxx },
+ { 21, &gp100_vmm_desc_16[1], NVKM_VMM_PAGE_SVxC },
+ { 16, &gp100_vmm_desc_16[0], NVKM_VMM_PAGE_SVxC },
+ { 12, &gp100_vmm_desc_12[0], NVKM_VMM_PAGE_SVHx },
+ {}
+ }
+};
+
+int
+gp100_vmm_new(struct nvkm_mmu *mmu, u64 addr, u64 size, void *argv, u32 argc,
+ struct lock_class_key *key, const char *name,
+ struct nvkm_vmm **pvmm)
+{
+ return nv04_vmm_new_(&gp100_vmm, mmu, 0, addr, size,
+ argv, argc, key, name, pvmm);
+}
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgp10b.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgp10b.c
new file mode 100644
index 000000000000..3dcc6bddb32f
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgp10b.c
@@ -0,0 +1,49 @@
+/*
+ * Copyright 2017 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+#include "vmm.h"
+
+static const struct nvkm_vmm_func
+gp10b_vmm = {
+ .join = gp100_vmm_join,
+ .part = gf100_vmm_part,
+ .aper = gk20a_vmm_aper,
+ .valid = gp100_vmm_valid,
+ .flush = gp100_vmm_flush,
+ .page = {
+ { 47, &gp100_vmm_desc_16[4], NVKM_VMM_PAGE_Sxxx },
+ { 38, &gp100_vmm_desc_16[3], NVKM_VMM_PAGE_Sxxx },
+ { 29, &gp100_vmm_desc_16[2], NVKM_VMM_PAGE_Sxxx },
+ { 21, &gp100_vmm_desc_16[1], NVKM_VMM_PAGE_SxHC },
+ { 16, &gp100_vmm_desc_16[0], NVKM_VMM_PAGE_SxHC },
+ { 12, &gp100_vmm_desc_12[0], NVKM_VMM_PAGE_SxHx },
+ {}
+ }
+};
+
+int
+gp10b_vmm_new(struct nvkm_mmu *mmu, u64 addr, u64 size, void *argv, u32 argc,
+ struct lock_class_key *key, const char *name,
+ struct nvkm_vmm **pvmm)
+{
+ return nv04_vmm_new_(&gp10b_vmm, mmu, 0, addr, size,
+ argv, argc, key, name, pvmm);
+}
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmnv04.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmnv04.c
new file mode 100644
index 000000000000..0cab1ffc9f64
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmnv04.c
@@ -0,0 +1,140 @@
+/*
+ * Copyright 2017 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+#include "vmm.h"
+
+#include <nvif/if000d.h>
+#include <nvif/unpack.h>
+
+static inline void
+nv04_vmm_pgt_pte(struct nvkm_vmm *vmm, struct nvkm_mmu_pt *pt,
+ u32 ptei, u32 ptes, struct nvkm_vmm_map *map, u64 addr)
+{
+ u32 data = addr | 0x00000003; /* PRESENT, RW. */
+ while (ptes--) {
+ VMM_WO032(pt, vmm, 8 + ptei++ * 4, data);
+ data += 0x00001000;
+ }
+}
+
+static void
+nv04_vmm_pgt_sgl(struct nvkm_vmm *vmm, struct nvkm_mmu_pt *pt,
+ u32 ptei, u32 ptes, struct nvkm_vmm_map *map)
+{
+ VMM_MAP_ITER_SGL(vmm, pt, ptei, ptes, map, nv04_vmm_pgt_pte);
+}
+
+static void
+nv04_vmm_pgt_dma(struct nvkm_vmm *vmm, struct nvkm_mmu_pt *pt,
+ u32 ptei, u32 ptes, struct nvkm_vmm_map *map)
+{
+#if PAGE_SHIFT == 12
+ nvkm_kmap(pt->memory);
+ while (ptes--)
+ VMM_WO032(pt, vmm, 8 + (ptei++ * 4), *map->dma++ | 0x00000003);
+ nvkm_done(pt->memory);
+#else
+ VMM_MAP_ITER_DMA(vmm, pt, ptei, ptes, map, nv04_vmm_pgt_pte);
+#endif
+}
+
+static void
+nv04_vmm_pgt_unmap(struct nvkm_vmm *vmm,
+ struct nvkm_mmu_pt *pt, u32 ptei, u32 ptes)
+{
+ VMM_FO032(pt, vmm, 8 + (ptei * 4), 0, ptes);
+}
+
+static const struct nvkm_vmm_desc_func
+nv04_vmm_desc_pgt = {
+ .unmap = nv04_vmm_pgt_unmap,
+ .dma = nv04_vmm_pgt_dma,
+ .sgl = nv04_vmm_pgt_sgl,
+};
+
+static const struct nvkm_vmm_desc
+nv04_vmm_desc_12[] = {
+ { PGT, 15, 4, 0x1000, &nv04_vmm_desc_pgt },
+ {}
+};
+
+int
+nv04_vmm_valid(struct nvkm_vmm *vmm, void *argv, u32 argc,
+ struct nvkm_vmm_map *map)
+{
+ union {
+ struct nv04_vmm_map_vn vn;
+ } *args = argv;
+ int ret = -ENOSYS;
+ if ((ret = nvif_unvers(ret, &argv, &argc, args->vn)))
+ VMM_DEBUG(vmm, "args");
+ return ret;
+}
+
+static const struct nvkm_vmm_func
+nv04_vmm = {
+ .valid = nv04_vmm_valid,
+ .page = {
+ { 12, &nv04_vmm_desc_12[0], NVKM_VMM_PAGE_HOST },
+ {}
+ }
+};
+
+int
+nv04_vmm_new_(const struct nvkm_vmm_func *func, struct nvkm_mmu *mmu,
+ u32 pd_header, u64 addr, u64 size, void *argv, u32 argc,
+ struct lock_class_key *key, const char *name,
+ struct nvkm_vmm **pvmm)
+{
+ union {
+ struct nv04_vmm_vn vn;
+ } *args = argv;
+ int ret;
+
+ ret = nvkm_vmm_new_(func, mmu, pd_header, addr, size, key, name, pvmm);
+ if (ret)
+ return ret;
+
+ return nvif_unvers(-ENOSYS, &argv, &argc, args->vn);
+}
+
+int
+nv04_vmm_new(struct nvkm_mmu *mmu, u64 addr, u64 size, void *argv, u32 argc,
+ struct lock_class_key *key, const char *name,
+ struct nvkm_vmm **pvmm)
+{
+ struct nvkm_memory *mem;
+ struct nvkm_vmm *vmm;
+ int ret;
+
+ ret = nv04_vmm_new_(&nv04_vmm, mmu, 8, addr, size,
+ argv, argc, key, name, &vmm);
+ *pvmm = vmm;
+ if (ret)
+ return ret;
+
+ mem = vmm->pd->pt[0]->memory;
+ nvkm_kmap(mem);
+ nvkm_wo32(mem, 0x00000, 0x0002103d); /* PCI, RW, PT, !LN */
+ nvkm_wo32(mem, 0x00004, vmm->limit - 1);
+ nvkm_done(mem);
+ return 0;
+}
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmnv41.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmnv41.c
new file mode 100644
index 000000000000..b595f130e573
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmnv41.c
@@ -0,0 +1,113 @@
+/*
+ * Copyright 2017 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+#include "vmm.h"
+
+#include <subdev/timer.h>
+
+static void
+nv41_vmm_pgt_pte(struct nvkm_vmm *vmm, struct nvkm_mmu_pt *pt,
+ u32 ptei, u32 ptes, struct nvkm_vmm_map *map, u64 addr)
+{
+ u32 data = (addr >> 7) | 0x00000001; /* VALID. */
+ while (ptes--) {
+ VMM_WO032(pt, vmm, ptei++ * 4, data);
+ data += 0x00000020;
+ }
+}
+
+static void
+nv41_vmm_pgt_sgl(struct nvkm_vmm *vmm, struct nvkm_mmu_pt *pt,
+ u32 ptei, u32 ptes, struct nvkm_vmm_map *map)
+{
+ VMM_MAP_ITER_SGL(vmm, pt, ptei, ptes, map, nv41_vmm_pgt_pte);
+}
+
+static void
+nv41_vmm_pgt_dma(struct nvkm_vmm *vmm, struct nvkm_mmu_pt *pt,
+ u32 ptei, u32 ptes, struct nvkm_vmm_map *map)
+{
+#if PAGE_SHIFT == 12
+ nvkm_kmap(pt->memory);
+ while (ptes--) {
+ const u32 data = (*map->dma++ >> 7) | 0x00000001;
+ VMM_WO032(pt, vmm, ptei++ * 4, data);
+ }
+ nvkm_done(pt->memory);
+#else
+ VMM_MAP_ITER_DMA(vmm, pt, ptei, ptes, map, nv41_vmm_pgt_pte);
+#endif
+}
+
+static void
+nv41_vmm_pgt_unmap(struct nvkm_vmm *vmm,
+ struct nvkm_mmu_pt *pt, u32 ptei, u32 ptes)
+{
+ VMM_FO032(pt, vmm, ptei * 4, 0, ptes);
+}
+
+static const struct nvkm_vmm_desc_func
+nv41_vmm_desc_pgt = {
+ .unmap = nv41_vmm_pgt_unmap,
+ .dma = nv41_vmm_pgt_dma,
+ .sgl = nv41_vmm_pgt_sgl,
+};
+
+static const struct nvkm_vmm_desc
+nv41_vmm_desc_12[] = {
+ { PGT, 17, 4, 0x1000, &nv41_vmm_desc_pgt },
+ {}
+};
+
+static void
+nv41_vmm_flush(struct nvkm_vmm *vmm, int level)
+{
+ struct nvkm_subdev *subdev = &vmm->mmu->subdev;
+ struct nvkm_device *device = subdev->device;
+
+ mutex_lock(&subdev->mutex);
+ nvkm_wr32(device, 0x100810, 0x00000022);
+ nvkm_msec(device, 2000,
+ if (nvkm_rd32(device, 0x100810) & 0x00000020)
+ break;
+ );
+ nvkm_wr32(device, 0x100810, 0x00000000);
+ mutex_unlock(&subdev->mutex);
+}
+
+static const struct nvkm_vmm_func
+nv41_vmm = {
+ .valid = nv04_vmm_valid,
+ .flush = nv41_vmm_flush,
+ .page = {
+ { 12, &nv41_vmm_desc_12[0], NVKM_VMM_PAGE_HOST },
+ {}
+ }
+};
+
+int
+nv41_vmm_new(struct nvkm_mmu *mmu, u64 addr, u64 size, void *argv, u32 argc,
+ struct lock_class_key *key, const char *name,
+ struct nvkm_vmm **pvmm)
+{
+ return nv04_vmm_new_(&nv41_vmm, mmu, 0, addr, size,
+ argv, argc, key, name, pvmm);
+}
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmnv44.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmnv44.c
new file mode 100644
index 000000000000..b834e4352334
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmnv44.c
@@ -0,0 +1,230 @@
+/*
+ * Copyright 2017 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+#include "vmm.h"
+
+#include <subdev/timer.h>
+
+static void
+nv44_vmm_pgt_fill(struct nvkm_vmm *vmm, struct nvkm_mmu_pt *pt,
+ dma_addr_t *list, u32 ptei, u32 ptes)
+{
+ u32 pteo = (ptei << 2) & ~0x0000000f;
+ u32 tmp[4];
+
+ tmp[0] = nvkm_ro32(pt->memory, pteo + 0x0);
+ tmp[1] = nvkm_ro32(pt->memory, pteo + 0x4);
+ tmp[2] = nvkm_ro32(pt->memory, pteo + 0x8);
+ tmp[3] = nvkm_ro32(pt->memory, pteo + 0xc);
+
+ while (ptes--) {
+ u32 addr = (list ? *list++ : vmm->null) >> 12;
+ switch (ptei++ & 0x3) {
+ case 0:
+ tmp[0] &= ~0x07ffffff;
+ tmp[0] |= addr;
+ break;
+ case 1:
+ tmp[0] &= ~0xf8000000;
+ tmp[0] |= addr << 27;
+ tmp[1] &= ~0x003fffff;
+ tmp[1] |= addr >> 5;
+ break;
+ case 2:
+ tmp[1] &= ~0xffc00000;
+ tmp[1] |= addr << 22;
+ tmp[2] &= ~0x0001ffff;
+ tmp[2] |= addr >> 10;
+ break;
+ case 3:
+ tmp[2] &= ~0xfffe0000;
+ tmp[2] |= addr << 17;
+ tmp[3] &= ~0x00000fff;
+ tmp[3] |= addr >> 15;
+ break;
+ }
+ }
+
+ VMM_WO032(pt, vmm, pteo + 0x0, tmp[0]);
+ VMM_WO032(pt, vmm, pteo + 0x4, tmp[1]);
+ VMM_WO032(pt, vmm, pteo + 0x8, tmp[2]);
+ VMM_WO032(pt, vmm, pteo + 0xc, tmp[3] | 0x40000000);
+}
+
+static void
+nv44_vmm_pgt_pte(struct nvkm_vmm *vmm, struct nvkm_mmu_pt *pt,
+ u32 ptei, u32 ptes, struct nvkm_vmm_map *map, u64 addr)
+{
+ dma_addr_t tmp[4], i;
+
+ if (ptei & 3) {
+ const u32 pten = min(ptes, 4 - (ptei & 3));
+ for (i = 0; i < pten; i++, addr += 0x1000)
+ tmp[i] = addr;
+ nv44_vmm_pgt_fill(vmm, pt, tmp, ptei, pten);
+ ptei += pten;
+ ptes -= pten;
+ }
+
+ while (ptes >= 4) {
+ for (i = 0; i < 4; i++, addr += 0x1000)
+ tmp[i] = addr >> 12;
+ VMM_WO032(pt, vmm, ptei++ * 4, tmp[0] >> 0 | tmp[1] << 27);
+ VMM_WO032(pt, vmm, ptei++ * 4, tmp[1] >> 5 | tmp[2] << 22);
+ VMM_WO032(pt, vmm, ptei++ * 4, tmp[2] >> 10 | tmp[3] << 17);
+ VMM_WO032(pt, vmm, ptei++ * 4, tmp[3] >> 15 | 0x40000000);
+ ptes -= 4;
+ }
+
+ if (ptes) {
+ for (i = 0; i < ptes; i++, addr += 0x1000)
+ tmp[i] = addr;
+ nv44_vmm_pgt_fill(vmm, pt, tmp, ptei, ptes);
+ }
+}
+
+static void
+nv44_vmm_pgt_sgl(struct nvkm_vmm *vmm, struct nvkm_mmu_pt *pt,
+ u32 ptei, u32 ptes, struct nvkm_vmm_map *map)
+{
+ VMM_MAP_ITER_SGL(vmm, pt, ptei, ptes, map, nv44_vmm_pgt_pte);
+}
+
+static void
+nv44_vmm_pgt_dma(struct nvkm_vmm *vmm, struct nvkm_mmu_pt *pt,
+ u32 ptei, u32 ptes, struct nvkm_vmm_map *map)
+{
+#if PAGE_SHIFT == 12
+ nvkm_kmap(pt->memory);
+ if (ptei & 3) {
+ const u32 pten = min(ptes, 4 - (ptei & 3));
+ nv44_vmm_pgt_fill(vmm, pt, map->dma, ptei, pten);
+ ptei += pten;
+ ptes -= pten;
+ map->dma += pten;
+ }
+
+ while (ptes >= 4) {
+ u32 tmp[4], i;
+ for (i = 0; i < 4; i++)
+ tmp[i] = *map->dma++ >> 12;
+ VMM_WO032(pt, vmm, ptei++ * 4, tmp[0] >> 0 | tmp[1] << 27);
+ VMM_WO032(pt, vmm, ptei++ * 4, tmp[1] >> 5 | tmp[2] << 22);
+ VMM_WO032(pt, vmm, ptei++ * 4, tmp[2] >> 10 | tmp[3] << 17);
+ VMM_WO032(pt, vmm, ptei++ * 4, tmp[3] >> 15 | 0x40000000);
+ ptes -= 4;
+ }
+
+ if (ptes) {
+ nv44_vmm_pgt_fill(vmm, pt, map->dma, ptei, ptes);
+ map->dma += ptes;
+ }
+ nvkm_done(pt->memory);
+#else
+ VMM_MAP_ITER_DMA(vmm, pt, ptei, ptes, map, nv44_vmm_pgt_pte);
+#endif
+}
+
+static void
+nv44_vmm_pgt_unmap(struct nvkm_vmm *vmm,
+ struct nvkm_mmu_pt *pt, u32 ptei, u32 ptes)
+{
+ nvkm_kmap(pt->memory);
+ if (ptei & 3) {
+ const u32 pten = min(ptes, 4 - (ptei & 3));
+ nv44_vmm_pgt_fill(vmm, pt, NULL, ptei, pten);
+ ptei += pten;
+ ptes -= pten;
+ }
+
+ while (ptes > 4) {
+ VMM_WO032(pt, vmm, ptei++ * 4, 0x00000000);
+ VMM_WO032(pt, vmm, ptei++ * 4, 0x00000000);
+ VMM_WO032(pt, vmm, ptei++ * 4, 0x00000000);
+ VMM_WO032(pt, vmm, ptei++ * 4, 0x00000000);
+ ptes -= 4;
+ }
+
+ if (ptes)
+ nv44_vmm_pgt_fill(vmm, pt, NULL, ptei, ptes);
+ nvkm_done(pt->memory);
+}
+
+static const struct nvkm_vmm_desc_func
+nv44_vmm_desc_pgt = {
+ .unmap = nv44_vmm_pgt_unmap,
+ .dma = nv44_vmm_pgt_dma,
+ .sgl = nv44_vmm_pgt_sgl,
+};
+
+static const struct nvkm_vmm_desc
+nv44_vmm_desc_12[] = {
+ { PGT, 17, 4, 0x80000, &nv44_vmm_desc_pgt },
+ {}
+};
+
+static void
+nv44_vmm_flush(struct nvkm_vmm *vmm, int level)
+{
+ struct nvkm_device *device = vmm->mmu->subdev.device;
+ nvkm_wr32(device, 0x100814, vmm->limit - 4096);
+ nvkm_wr32(device, 0x100808, 0x000000020);
+ nvkm_msec(device, 2000,
+ if (nvkm_rd32(device, 0x100808) & 0x00000001)
+ break;
+ );
+ nvkm_wr32(device, 0x100808, 0x00000000);
+}
+
+static const struct nvkm_vmm_func
+nv44_vmm = {
+ .valid = nv04_vmm_valid,
+ .flush = nv44_vmm_flush,
+ .page = {
+ { 12, &nv44_vmm_desc_12[0], NVKM_VMM_PAGE_HOST },
+ {}
+ }
+};
+
+int
+nv44_vmm_new(struct nvkm_mmu *mmu, u64 addr, u64 size, void *argv, u32 argc,
+ struct lock_class_key *key, const char *name,
+ struct nvkm_vmm **pvmm)
+{
+ struct nvkm_subdev *subdev = &mmu->subdev;
+ struct nvkm_vmm *vmm;
+ int ret;
+
+ ret = nv04_vmm_new_(&nv44_vmm, mmu, 0, addr, size,
+ argv, argc, key, name, &vmm);
+ *pvmm = vmm;
+ if (ret)
+ return ret;
+
+ vmm->nullp = dma_alloc_coherent(subdev->device->dev, 16 * 1024,
+ &vmm->null, GFP_KERNEL);
+ if (!vmm->nullp) {
+ nvkm_warn(subdev, "unable to allocate dummy pages\n");
+ vmm->null = 0;
+ }
+
+ return 0;
+}
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmnv50.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmnv50.c
new file mode 100644
index 000000000000..863a2edd9861
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmnv50.c
@@ -0,0 +1,385 @@
+/*
+ * Copyright 2017 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+#include "vmm.h"
+
+#include <subdev/fb.h>
+#include <subdev/timer.h>
+#include <engine/gr.h>
+
+#include <nvif/if500d.h>
+#include <nvif/unpack.h>
+
+static inline void
+nv50_vmm_pgt_pte(struct nvkm_vmm *vmm, struct nvkm_mmu_pt *pt,
+ u32 ptei, u32 ptes, struct nvkm_vmm_map *map, u64 addr)
+{
+ u64 next = addr | map->type, data;
+ u32 pten;
+ int log2blk;
+
+ map->type += ptes * map->ctag;
+
+ while (ptes) {
+ for (log2blk = 7; log2blk >= 0; log2blk--) {
+ pten = 1 << log2blk;
+ if (ptes >= pten && IS_ALIGNED(ptei, pten))
+ break;
+ }
+
+ data = next | (log2blk << 7);
+ next += pten * map->next;
+ ptes -= pten;
+
+ while (pten--)
+ VMM_WO064(pt, vmm, ptei++ * 8, data);
+ }
+}
+
+static void
+nv50_vmm_pgt_sgl(struct nvkm_vmm *vmm, struct nvkm_mmu_pt *pt,
+ u32 ptei, u32 ptes, struct nvkm_vmm_map *map)
+{
+ VMM_MAP_ITER_SGL(vmm, pt, ptei, ptes, map, nv50_vmm_pgt_pte);
+}
+
+static void
+nv50_vmm_pgt_dma(struct nvkm_vmm *vmm, struct nvkm_mmu_pt *pt,
+ u32 ptei, u32 ptes, struct nvkm_vmm_map *map)
+{
+ if (map->page->shift == PAGE_SHIFT) {
+ VMM_SPAM(vmm, "DMAA %08x %08x PTE(s)", ptei, ptes);
+ nvkm_kmap(pt->memory);
+ while (ptes--) {
+ const u64 data = *map->dma++ | map->type;
+ VMM_WO064(pt, vmm, ptei++ * 8, data);
+ map->type += map->ctag;
+ }
+ nvkm_done(pt->memory);
+ return;
+ }
+
+ VMM_MAP_ITER_DMA(vmm, pt, ptei, ptes, map, nv50_vmm_pgt_pte);
+}
+
+static void
+nv50_vmm_pgt_mem(struct nvkm_vmm *vmm, struct nvkm_mmu_pt *pt,
+ u32 ptei, u32 ptes, struct nvkm_vmm_map *map)
+{
+ VMM_MAP_ITER_MEM(vmm, pt, ptei, ptes, map, nv50_vmm_pgt_pte);
+}
+
+static void
+nv50_vmm_pgt_unmap(struct nvkm_vmm *vmm,
+ struct nvkm_mmu_pt *pt, u32 ptei, u32 ptes)
+{
+ VMM_FO064(pt, vmm, ptei * 8, 0ULL, ptes);
+}
+
+static const struct nvkm_vmm_desc_func
+nv50_vmm_pgt = {
+ .unmap = nv50_vmm_pgt_unmap,
+ .mem = nv50_vmm_pgt_mem,
+ .dma = nv50_vmm_pgt_dma,
+ .sgl = nv50_vmm_pgt_sgl,
+};
+
+static bool
+nv50_vmm_pde(struct nvkm_vmm *vmm, struct nvkm_vmm_pt *pgt, u64 *pdata)
+{
+ struct nvkm_mmu_pt *pt;
+ u64 data = 0xdeadcafe00000000ULL;
+ if (pgt && (pt = pgt->pt[0])) {
+ switch (pgt->page) {
+ case 16: data = 0x00000001; break;
+ case 12: data = 0x00000003;
+ switch (nvkm_memory_size(pt->memory)) {
+ case 0x100000: data |= 0x00000000; break;
+ case 0x040000: data |= 0x00000020; break;
+ case 0x020000: data |= 0x00000040; break;
+ case 0x010000: data |= 0x00000060; break;
+ default:
+ WARN_ON(1);
+ return false;
+ }
+ break;
+ default:
+ WARN_ON(1);
+ return false;
+ }
+
+ switch (nvkm_memory_target(pt->memory)) {
+ case NVKM_MEM_TARGET_VRAM: data |= 0x00000000; break;
+ case NVKM_MEM_TARGET_HOST: data |= 0x00000008; break;
+ case NVKM_MEM_TARGET_NCOH: data |= 0x0000000c; break;
+ default:
+ WARN_ON(1);
+ return false;
+ }
+
+ data |= pt->addr;
+ }
+ *pdata = data;
+ return true;
+}
+
+static void
+nv50_vmm_pgd_pde(struct nvkm_vmm *vmm, struct nvkm_vmm_pt *pgd, u32 pdei)
+{
+ struct nvkm_vmm_join *join;
+ u32 pdeo = vmm->mmu->func->vmm.pd_offset + (pdei * 8);
+ u64 data;
+
+ if (!nv50_vmm_pde(vmm, pgd->pde[pdei], &data))
+ return;
+
+ list_for_each_entry(join, &vmm->join, head) {
+ nvkm_kmap(join->inst);
+ nvkm_wo64(join->inst, pdeo, data);
+ nvkm_done(join->inst);
+ }
+}
+
+static const struct nvkm_vmm_desc_func
+nv50_vmm_pgd = {
+ .pde = nv50_vmm_pgd_pde,
+};
+
+static const struct nvkm_vmm_desc
+nv50_vmm_desc_12[] = {
+ { PGT, 17, 8, 0x1000, &nv50_vmm_pgt },
+ { PGD, 11, 0, 0x0000, &nv50_vmm_pgd },
+ {}
+};
+
+static const struct nvkm_vmm_desc
+nv50_vmm_desc_16[] = {
+ { PGT, 13, 8, 0x1000, &nv50_vmm_pgt },
+ { PGD, 11, 0, 0x0000, &nv50_vmm_pgd },
+ {}
+};
+
+static void
+nv50_vmm_flush(struct nvkm_vmm *vmm, int level)
+{
+ struct nvkm_subdev *subdev = &vmm->mmu->subdev;
+ struct nvkm_device *device = subdev->device;
+ int i, id;
+
+ mutex_lock(&subdev->mutex);
+ for (i = 0; i < NVKM_SUBDEV_NR; i++) {
+ if (!atomic_read(&vmm->engref[i]))
+ continue;
+
+ /* unfortunate hw bug workaround... */
+ if (i == NVKM_ENGINE_GR && device->gr) {
+ int ret = nvkm_gr_tlb_flush(device->gr);
+ if (ret != -ENODEV)
+ continue;
+ }
+
+ switch (i) {
+ case NVKM_ENGINE_GR : id = 0x00; break;
+ case NVKM_ENGINE_VP :
+ case NVKM_ENGINE_MSPDEC: id = 0x01; break;
+ case NVKM_SUBDEV_BAR : id = 0x06; break;
+ case NVKM_ENGINE_MSPPP :
+ case NVKM_ENGINE_MPEG : id = 0x08; break;
+ case NVKM_ENGINE_BSP :
+ case NVKM_ENGINE_MSVLD : id = 0x09; break;
+ case NVKM_ENGINE_CIPHER:
+ case NVKM_ENGINE_SEC : id = 0x0a; break;
+ case NVKM_ENGINE_CE0 : id = 0x0d; break;
+ default:
+ continue;
+ }
+
+ nvkm_wr32(device, 0x100c80, (id << 16) | 1);
+ if (nvkm_msec(device, 2000,
+ if (!(nvkm_rd32(device, 0x100c80) & 0x00000001))
+ break;
+ ) < 0)
+ nvkm_error(subdev, "%s mmu invalidate timeout\n",
+ nvkm_subdev_name[i]);
+ }
+ mutex_unlock(&subdev->mutex);
+}
+
+static int
+nv50_vmm_valid(struct nvkm_vmm *vmm, void *argv, u32 argc,
+ struct nvkm_vmm_map *map)
+{
+ const struct nvkm_vmm_page *page = map->page;
+ union {
+ struct nv50_vmm_map_vn vn;
+ struct nv50_vmm_map_v0 v0;
+ } *args = argv;
+ struct nvkm_device *device = vmm->mmu->subdev.device;
+ struct nvkm_ram *ram = device->fb->ram;
+ struct nvkm_memory *memory = map->memory;
+ u8 aper, kind, comp, priv, ro;
+ int kindn, ret = -ENOSYS;
+ const u8 *kindm;
+
+ map->type = map->ctag = 0;
+ map->next = 1 << page->shift;
+
+ if (!(ret = nvif_unpack(ret, &argv, &argc, args->v0, 0, 0, false))) {
+ ro = !!args->v0.ro;
+ priv = !!args->v0.priv;
+ kind = args->v0.kind & 0x7f;
+ comp = args->v0.comp & 0x03;
+ } else
+ if (!(ret = nvif_unvers(ret, &argv, &argc, args->vn))) {
+ ro = 0;
+ priv = 0;
+ kind = 0x00;
+ comp = 0;
+ } else {
+ VMM_DEBUG(vmm, "args");
+ return ret;
+ }
+
+ switch (nvkm_memory_target(memory)) {
+ case NVKM_MEM_TARGET_VRAM:
+ if (ram->stolen) {
+ map->type |= ram->stolen;
+ aper = 3;
+ } else {
+ aper = 0;
+ }
+ break;
+ case NVKM_MEM_TARGET_HOST:
+ aper = 2;
+ break;
+ case NVKM_MEM_TARGET_NCOH:
+ aper = 3;
+ break;
+ default:
+ WARN_ON(1);
+ return -EINVAL;
+ }
+
+ kindm = vmm->mmu->func->kind(vmm->mmu, &kindn);
+ if (kind >= kindn || kindm[kind] == 0x7f) {
+ VMM_DEBUG(vmm, "kind %02x", kind);
+ return -EINVAL;
+ }
+
+ if (map->mem && map->mem->type != kindm[kind]) {
+ VMM_DEBUG(vmm, "kind %02x bankswz: %d %d", kind,
+ kindm[kind], map->mem->type);
+ return -EINVAL;
+ }
+
+ if (comp) {
+ u32 tags = (nvkm_memory_size(memory) >> 16) * comp;
+ if (aper != 0 || !(page->type & NVKM_VMM_PAGE_COMP)) {
+ VMM_DEBUG(vmm, "comp %d %02x", aper, page->type);
+ return -EINVAL;
+ }
+
+ ret = nvkm_memory_tags_get(memory, device, tags, NULL,
+ &map->tags);
+ if (ret) {
+ VMM_DEBUG(vmm, "comp %d", ret);
+ return ret;
+ }
+
+ if (map->tags->mn) {
+ u32 tags = map->tags->mn->offset + (map->offset >> 16);
+ map->ctag |= (u64)comp << 49;
+ map->type |= (u64)comp << 47;
+ map->type |= (u64)tags << 49;
+ map->next |= map->ctag;
+ }
+ }
+
+ map->type |= BIT(0); /* Valid. */
+ map->type |= (u64)ro << 3;
+ map->type |= (u64)aper << 4;
+ map->type |= (u64)priv << 6;
+ map->type |= (u64)kind << 40;
+ return 0;
+}
+
+static void
+nv50_vmm_part(struct nvkm_vmm *vmm, struct nvkm_memory *inst)
+{
+ struct nvkm_vmm_join *join;
+
+ list_for_each_entry(join, &vmm->join, head) {
+ if (join->inst == inst) {
+ list_del(&join->head);
+ kfree(join);
+ break;
+ }
+ }
+}
+
+static int
+nv50_vmm_join(struct nvkm_vmm *vmm, struct nvkm_memory *inst)
+{
+ const u32 pd_offset = vmm->mmu->func->vmm.pd_offset;
+ struct nvkm_vmm_join *join;
+ int ret = 0;
+ u64 data;
+ u32 pdei;
+
+ if (!(join = kmalloc(sizeof(*join), GFP_KERNEL)))
+ return -ENOMEM;
+ join->inst = inst;
+ list_add_tail(&join->head, &vmm->join);
+
+ nvkm_kmap(join->inst);
+ for (pdei = vmm->start >> 29; pdei <= (vmm->limit - 1) >> 29; pdei++) {
+ if (!nv50_vmm_pde(vmm, vmm->pd->pde[pdei], &data)) {
+ ret = -EINVAL;
+ break;
+ }
+ nvkm_wo64(join->inst, pd_offset + (pdei * 8), data);
+ }
+ nvkm_done(join->inst);
+ return ret;
+}
+
+static const struct nvkm_vmm_func
+nv50_vmm = {
+ .join = nv50_vmm_join,
+ .part = nv50_vmm_part,
+ .valid = nv50_vmm_valid,
+ .flush = nv50_vmm_flush,
+ .page_block = 1 << 29,
+ .page = {
+ { 16, &nv50_vmm_desc_16[0], NVKM_VMM_PAGE_xVxC },
+ { 12, &nv50_vmm_desc_12[0], NVKM_VMM_PAGE_xVHx },
+ {}
+ }
+};
+
+int
+nv50_vmm_new(struct nvkm_mmu *mmu, u64 addr, u64 size, void *argv, u32 argc,
+ struct lock_class_key *key, const char *name,
+ struct nvkm_vmm **pvmm)
+{
+ return nv04_vmm_new_(&nv50_vmm, mmu, 0, addr, size,
+ argv, argc, key, name, pvmm);
+}
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/pci/base.c b/drivers/gpu/drm/nouveau/nvkm/subdev/pci/base.c
index a4cb82495cee..b1b1f3626b96 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/pci/base.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/pci/base.c
@@ -87,7 +87,7 @@ nvkm_pci_fini(struct nvkm_subdev *subdev, bool suspend)
if (pci->irq >= 0) {
free_irq(pci->irq, pci);
pci->irq = -1;
- };
+ }
if (pci->agp.bridge)
nvkm_agp_fini(pci);
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/secboot/gm200.c b/drivers/gpu/drm/nouveau/nvkm/subdev/secboot/gm200.c
index 73ca1203281d..5e91b3f90065 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/secboot/gm200.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/secboot/gm200.c
@@ -39,7 +39,7 @@ gm200_secboot_run_blob(struct nvkm_secboot *sb, struct nvkm_gpuobj *blob,
{
struct gm200_secboot *gsb = gm200_secboot(sb);
struct nvkm_subdev *subdev = &gsb->base.subdev;
- struct nvkm_vma vma;
+ struct nvkm_vma *vma = NULL;
u32 start_address;
int ret;
@@ -48,12 +48,16 @@ gm200_secboot_run_blob(struct nvkm_secboot *sb, struct nvkm_gpuobj *blob,
return ret;
/* Map the HS firmware so the HS bootloader can see it */
- ret = nvkm_gpuobj_map(blob, gsb->vm, NV_MEM_ACCESS_RW, &vma);
+ ret = nvkm_vmm_get(gsb->vmm, 12, blob->size, &vma);
if (ret) {
nvkm_falcon_put(falcon, subdev);
return ret;
}
+ ret = nvkm_memory_map(blob, 0, gsb->vmm, vma, NULL, 0);
+ if (ret)
+ goto end;
+
/* Reset and set the falcon up */
ret = nvkm_falcon_reset(falcon);
if (ret)
@@ -61,7 +65,7 @@ gm200_secboot_run_blob(struct nvkm_secboot *sb, struct nvkm_gpuobj *blob,
nvkm_falcon_bind_context(falcon, gsb->inst);
/* Load the HS bootloader into the falcon's IMEM/DMEM */
- ret = sb->acr->func->load(sb->acr, falcon, blob, vma.offset);
+ ret = sb->acr->func->load(sb->acr, falcon, blob, vma->addr);
if (ret < 0)
goto end;
@@ -91,7 +95,7 @@ end:
nvkm_mc_intr_mask(sb->subdev.device, falcon->owner->index, true);
/* We don't need the ACR firmware anymore */
- nvkm_gpuobj_unmap(&vma);
+ nvkm_vmm_put(gsb->vmm, &vma);
nvkm_falcon_put(falcon, subdev);
return ret;
@@ -102,37 +106,26 @@ gm200_secboot_oneinit(struct nvkm_secboot *sb)
{
struct gm200_secboot *gsb = gm200_secboot(sb);
struct nvkm_device *device = sb->subdev.device;
- struct nvkm_vm *vm;
- const u64 vm_area_len = 600 * 1024;
int ret;
/* Allocate instance block and VM */
- ret = nvkm_gpuobj_new(device, 0x1000, 0, true, NULL, &gsb->inst);
+ ret = nvkm_memory_new(device, NVKM_MEM_TARGET_INST, 0x1000, 0, true,
+ &gsb->inst);
if (ret)
return ret;
- ret = nvkm_gpuobj_new(device, 0x8000, 0, true, NULL, &gsb->pgd);
+ ret = nvkm_vmm_new(device, 0, 600 * 1024, NULL, 0, NULL, "acr",
+ &gsb->vmm);
if (ret)
return ret;
- ret = nvkm_vm_new(device, 0, vm_area_len, 0, NULL, &vm);
- if (ret)
- return ret;
-
- atomic_inc(&vm->engref[NVKM_SUBDEV_PMU]);
+ atomic_inc(&gsb->vmm->engref[NVKM_SUBDEV_PMU]);
+ gsb->vmm->debug = gsb->base.subdev.debug;
- ret = nvkm_vm_ref(vm, &gsb->vm, gsb->pgd);
- nvkm_vm_ref(NULL, &vm, NULL);
+ ret = nvkm_vmm_join(gsb->vmm, gsb->inst);
if (ret)
return ret;
- nvkm_kmap(gsb->inst);
- nvkm_wo32(gsb->inst, 0x200, lower_32_bits(gsb->pgd->addr));
- nvkm_wo32(gsb->inst, 0x204, upper_32_bits(gsb->pgd->addr));
- nvkm_wo32(gsb->inst, 0x208, lower_32_bits(vm_area_len - 1));
- nvkm_wo32(gsb->inst, 0x20c, upper_32_bits(vm_area_len - 1));
- nvkm_done(gsb->inst);
-
if (sb->acr->func->oneinit) {
ret = sb->acr->func->oneinit(sb->acr, sb);
if (ret)
@@ -160,9 +153,9 @@ gm200_secboot_dtor(struct nvkm_secboot *sb)
sb->acr->func->dtor(sb->acr);
- nvkm_vm_ref(NULL, &gsb->vm, gsb->pgd);
- nvkm_gpuobj_del(&gsb->pgd);
- nvkm_gpuobj_del(&gsb->inst);
+ nvkm_vmm_part(gsb->vmm, gsb->inst);
+ nvkm_vmm_unref(&gsb->vmm);
+ nvkm_memory_unref(&gsb->inst);
return gsb;
}
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/secboot/gm200.h b/drivers/gpu/drm/nouveau/nvkm/subdev/secboot/gm200.h
index c8ab3d76bdef..62c5e162099a 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/secboot/gm200.h
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/secboot/gm200.h
@@ -29,9 +29,8 @@ struct gm200_secboot {
struct nvkm_secboot base;
/* Instance block & address space used for HS FW execution */
- struct nvkm_gpuobj *inst;
- struct nvkm_gpuobj *pgd;
- struct nvkm_vm *vm;
+ struct nvkm_memory *inst;
+ struct nvkm_vmm *vmm;
};
#define gm200_secboot(sb) container_of(sb, struct gm200_secboot, base)
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/secboot/ls_ucode_msgqueue.c b/drivers/gpu/drm/nouveau/nvkm/subdev/secboot/ls_ucode_msgqueue.c
index ee989210725e..6f10b098676c 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/secboot/ls_ucode_msgqueue.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/secboot/ls_ucode_msgqueue.c
@@ -183,7 +183,7 @@ acr_ls_sec2_post_run(const struct nvkm_acr *acr, const struct nvkm_secboot *sb)
break;
);
if (reg & BIT(4)) {
- nvkm_debug(subdev, "applying workaround for start bug...");
+ nvkm_debug(subdev, "applying workaround for start bug...\n");
nvkm_falcon_start(sb->boot_falcon);
nvkm_msec(subdev->device, 1,
if ((reg = nvkm_rd32(subdev->device,
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/secboot/priv.h b/drivers/gpu/drm/nouveau/nvkm/subdev/secboot/priv.h
index 885e919a8720..d9091f029506 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/secboot/priv.h
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/secboot/priv.h
@@ -25,6 +25,7 @@
#include <subdev/secboot.h>
#include <subdev/mmu.h>
+struct nvkm_gpuobj;
struct nvkm_secboot_func {
int (*oneinit)(struct nvkm_secboot *);
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/therm/Kbuild b/drivers/gpu/drm/nouveau/nvkm/subdev/therm/Kbuild
index 2bafcc1d1818..7ba56b12badd 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/therm/Kbuild
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/therm/Kbuild
@@ -12,3 +12,4 @@ nvkm-y += nvkm/subdev/therm/gt215.o
nvkm-y += nvkm/subdev/therm/gf119.o
nvkm-y += nvkm/subdev/therm/gm107.o
nvkm-y += nvkm/subdev/therm/gm200.o
+nvkm-y += nvkm/subdev/therm/gp100.o
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/therm/base.c b/drivers/gpu/drm/nouveau/nvkm/subdev/therm/base.c
index 952a7cb0a59a..f27fc6d0d4c6 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/therm/base.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/therm/base.c
@@ -341,7 +341,8 @@ nvkm_therm_init(struct nvkm_subdev *subdev)
{
struct nvkm_therm *therm = nvkm_therm(subdev);
- therm->func->init(therm);
+ if (therm->func->init)
+ therm->func->init(therm);
if (therm->suspend >= 0) {
/* restore the pwm value only when on manual or auto mode */
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/therm/gp100.c b/drivers/gpu/drm/nouveau/nvkm/subdev/therm/gp100.c
new file mode 100644
index 000000000000..9f0dea3f61dc
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/therm/gp100.c
@@ -0,0 +1,56 @@
+/*
+ * Copyright 2017 Rhys Kidd
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Rhys Kidd
+ */
+#include "priv.h"
+
+static int
+gp100_temp_get(struct nvkm_therm *therm)
+{
+ struct nvkm_device *device = therm->subdev.device;
+ struct nvkm_subdev *subdev = &therm->subdev;
+ u32 tsensor = nvkm_rd32(device, 0x020460);
+ u32 inttemp = (tsensor & 0x0001fff8);
+
+ /* device SHADOWed */
+ if (tsensor & 0x40000000)
+ nvkm_trace(subdev, "reading temperature from SHADOWed sensor\n");
+
+ /* device valid */
+ if (tsensor & 0x20000000)
+ return (inttemp >> 8);
+ else
+ return -ENODEV;
+}
+
+static const struct nvkm_therm_func
+gp100_therm = {
+ .temp_get = gp100_temp_get,
+ .program_alarms = nvkm_therm_program_alarms_polling,
+};
+
+int
+gp100_therm_new(struct nvkm_device *device, int index,
+ struct nvkm_therm **ptherm)
+{
+ return nvkm_therm_new_(&gp100_therm, device, index, ptherm);
+}