WIPmmu: handle instance block setup

We previously required each VMM user to allocate their own page directory
and fill in the instance block themselves.

It makes more sense to handle this in a common location.

WIP: gf100 chicken-and-egg

Signed-off-by: Ben Skeggs <bskeggs@redhat.com>

23 files changed, 167 insertions, 180 deletions

diff --git a/drivers/gpu/drm/nouveau/include/nvkm/subdev/mmu.h b/drivers/gpu/drm/nouveau/include/nvkm/subdev/mmu.h
index 53ffc93d20d6..126e2da3fe22 100644
--- a/drivers/gpu/drm/nouveau/include/nvkm/subdev/mmu.h
+++ b/drivers/gpu/drm/nouveau/include/nvkm/subdev/mmu.h
@@ -10,11 +10,6 @@ struct nvkm_vm_pgt {
 	u32 refcount[2];
 };
 
-struct nvkm_vm_pgd {
-	struct list_head head;
-	struct nvkm_gpuobj *obj;
-};
-
 struct nvkm_vma {
 	struct list_head head;
 	int refcount;
@@ -32,7 +27,6 @@ struct nvkm_vm {
 	struct nvkm_mm mm;
 	struct kref refcount;
 
-	struct list_head pgd_list;
 	atomic_t engref[NVKM_SUBDEV_NR];
 
 	struct nvkm_vm_pgt *pgt;
@@ -48,7 +42,7 @@ struct nvkm_vm {
 
 int  nvkm_vm_new(struct nvkm_device *, u64 offset, u64 length, u64 mm_offset,
 		 struct lock_class_key *, struct nvkm_vm **);
-int  nvkm_vm_ref(struct nvkm_vm *, struct nvkm_vm **, struct nvkm_gpuobj *pgd);
+int  nvkm_vm_ref(struct nvkm_vm *, struct nvkm_vm **, struct nvkm_gpuobj *inst);
 int  nvkm_vm_boot(struct nvkm_vm *, u64 size);
 int  nvkm_vm_get(struct nvkm_vm *, u64 size, u32 page_shift, u32 access,
 		 struct nvkm_vma *);
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chang84.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chang84.c
index 61797c4dd07a..61b0ff6dee89 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chang84.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chang84.c
@@ -281,5 +281,5 @@ g84_fifo_chan_ctor(struct nv50_fifo *fifo, u64 vm, u64 push,
 	if (ret)
 		return ret;
 
-	return nvkm_vm_ref(chan->base.vm, &chan->vm, chan->pgd);
+	return nvkm_vm_ref(chan->base.vm, &chan->vm, chan->base.inst);
 }
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/changf100.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/changf100.h
index 7d697e2dce1a..a902f848a878 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/changf100.h
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/changf100.h
@@ -11,7 +11,6 @@ struct gf100_fifo_chan {
 	struct list_head head;
 	bool killed;
 
-	struct nvkm_gpuobj *pgd;
 	struct nvkm_vm *vm;
 
 	struct {
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/changk104.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/changk104.h
index 230f64e5f731..1cd5b8d3fb2e 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/changk104.h
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/changk104.h
@@ -12,7 +12,6 @@ struct gk104_fifo_chan {
 	struct list_head head;
 	bool killed;
 
-	struct nvkm_gpuobj *pgd;
 	struct nvkm_vm *vm;
 
 	struct {
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/channv50.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/channv50.c
index 25b60aff40e4..cfc5132ca97e 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/channv50.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/channv50.c
@@ -206,7 +206,7 @@ void *
 nv50_fifo_chan_dtor(struct nvkm_fifo_chan *base)
 {
 	struct nv50_fifo_chan *chan = nv50_fifo_chan(base);
-	nvkm_vm_ref(NULL, &chan->vm, chan->pgd);
+	nvkm_vm_ref(NULL, &chan->vm, chan->base.inst);
 	nvkm_ramht_del(&chan->ramht);
 	nvkm_gpuobj_del(&chan->pgd);
 	nvkm_gpuobj_del(&chan->eng);
@@ -266,5 +266,5 @@ nv50_fifo_chan_ctor(struct nv50_fifo *fifo, u64 vm, u64 push,
 	if (ret)
 		return ret;
 
-	return nvkm_vm_ref(chan->base.vm, &chan->vm, chan->pgd);
+	return nvkm_vm_ref(chan->base.vm, &chan->vm, chan->base.inst);
 }
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogf100.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogf100.c
index f9e0377d3d24..3475e075a0ca 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogf100.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogf100.c
@@ -191,8 +191,7 @@ static void *
 gf100_fifo_gpfifo_dtor(struct nvkm_fifo_chan *base)
 {
 	struct gf100_fifo_chan *chan = gf100_fifo_chan(base);
-	nvkm_vm_ref(NULL, &chan->vm, chan->pgd);
-	nvkm_gpuobj_del(&chan->pgd);
+	nvkm_vm_ref(NULL, &chan->vm, chan->base.inst);
 	return chan;
 }
 
@@ -216,7 +215,6 @@ gf100_fifo_gpfifo_new(struct nvkm_fifo *base, const struct nvkm_oclass *oclass,
 		struct fermi_channel_gpfifo_v0 v0;
 	} *args = data;
 	struct gf100_fifo *fifo = gf100_fifo(base);
-	struct nvkm_device *device = fifo->base.engine.subdev.device;
 	struct nvkm_object *parent = oclass->parent;
 	struct gf100_fifo_chan *chan;
 	u64 usermem, ioffset, ilength;
@@ -254,19 +252,7 @@ gf100_fifo_gpfifo_new(struct nvkm_fifo *base, const struct nvkm_oclass *oclass,
 
 	args->v0.chid = chan->base.chid;
 
-	/* page directory */
-	ret = nvkm_gpuobj_new(device, 0x10000, 0x1000, false, NULL, &chan->pgd);
-	if (ret)
-		return ret;
-
-	nvkm_kmap(chan->base.inst);
-	nvkm_wo32(chan->base.inst, 0x0200, lower_32_bits(chan->pgd->addr));
-	nvkm_wo32(chan->base.inst, 0x0204, upper_32_bits(chan->pgd->addr));
-	nvkm_wo32(chan->base.inst, 0x0208, 0xffffffff);
-	nvkm_wo32(chan->base.inst, 0x020c, 0x000000ff);
-	nvkm_done(chan->base.inst);
-
-	ret = nvkm_vm_ref(chan->base.vm, &chan->vm, chan->pgd);
+	ret = nvkm_vm_ref(chan->base.vm, &chan->vm, chan->base.inst);
 	if (ret)
 		return ret;
 
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogk104.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogk104.c
index 8abf6f8ef445..67fb07742400 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogk104.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogk104.c
@@ -204,8 +204,7 @@ static void *
 gk104_fifo_gpfifo_dtor(struct nvkm_fifo_chan *base)
 {
 	struct gk104_fifo_chan *chan = gk104_fifo_chan(base);
-	nvkm_vm_ref(NULL, &chan->vm, chan->pgd);
-	nvkm_gpuobj_del(&chan->pgd);
+	nvkm_vm_ref(NULL, &chan->vm, chan->base.inst);
 	return chan;
 }
 
@@ -233,7 +232,6 @@ gk104_fifo_gpfifo_new_(const struct gk104_fifo_chan_func *func,
 		       const struct nvkm_oclass *oclass,
 		       struct nvkm_object **pobject)
 {
-	struct nvkm_device *device = fifo->base.engine.subdev.device;
 	struct gk104_fifo_chan *chan;
 	int runlist = -1, ret = -ENOSYS, i, j;
 	u32 engines = 0, present = 0;
@@ -293,19 +291,7 @@ gk104_fifo_gpfifo_new_(const struct gk104_fifo_chan_func *func,
 
 	*chid = chan->base.chid;
 
-	/* Page directory. */
-	ret = nvkm_gpuobj_new(device, 0x10000, 0x1000, false, NULL, &chan->pgd);
-	if (ret)
-		return ret;
-
-	nvkm_kmap(chan->base.inst);
-	nvkm_wo32(chan->base.inst, 0x0200, lower_32_bits(chan->pgd->addr));
-	nvkm_wo32(chan->base.inst, 0x0204, upper_32_bits(chan->pgd->addr));
-	nvkm_wo32(chan->base.inst, 0x0208, 0xffffffff);
-	nvkm_wo32(chan->base.inst, 0x020c, 0x000000ff);
-	nvkm_done(chan->base.inst);
-
-	ret = nvkm_vm_ref(chan->base.vm, &chan->vm, chan->pgd);
+	ret = nvkm_vm_ref(chan->base.vm, &chan->vm, chan->base.inst);
 	if (ret)
 		return ret;
 
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/gf100.c b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/gf100.c
index c8bb73b5370b..1b821e81c7f6 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/gf100.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/gf100.c
@@ -49,12 +49,7 @@ gf100_bar_ctor_vm(struct gf100_bar *bar, struct gf100_bar_vm *bar_vm,
 	resource_size_t bar_len;
 	int ret;
 
-	ret = nvkm_memory_new(device, NVKM_MEM_TARGET_INST, 0x1000, 0, false,
-			      &bar_vm->mem);
-	if (ret)
-		return ret;
-
-	ret = nvkm_gpuobj_new(device, 0x8000, 0, false, NULL, &bar_vm->pgd);
+	ret = nvkm_gpuobj_new(device, 0x1000, 0, false, NULL, &bar_vm->inst);
 	if (ret)
 		return ret;
 
@@ -77,17 +72,11 @@ gf100_bar_ctor_vm(struct gf100_bar *bar, struct gf100_bar_vm *bar_vm,
 		}
 	}
 
-	ret = nvkm_vm_ref(vm, &bar_vm->vm, bar_vm->pgd);
+	ret = nvkm_vm_ref(vm, &bar_vm->vm, bar_vm->inst);
 	nvkm_vm_ref(NULL, &vm, NULL);
 	if (ret)
 		return ret;
 
-	nvkm_kmap(bar_vm->mem);
-	nvkm_wo32(bar_vm->mem, 0x0200, lower_32_bits(bar_vm->pgd->addr));
-	nvkm_wo32(bar_vm->mem, 0x0204, upper_32_bits(bar_vm->pgd->addr));
-	nvkm_wo32(bar_vm->mem, 0x0208, lower_32_bits(bar_len - 1));
-	nvkm_wo32(bar_vm->mem, 0x020c, upper_32_bits(bar_len - 1));
-	nvkm_done(bar_vm->mem);
 	return 0;
 }
 
@@ -124,11 +113,11 @@ gf100_bar_init(struct nvkm_bar *base)
 	nvkm_mask(device, 0x000200, 0x00000100, 0x00000000);
 	nvkm_mask(device, 0x000200, 0x00000100, 0x00000100);
 
-	addr = nvkm_memory_addr(bar->bar[1].mem) >> 12;
+	addr = bar->bar[1].inst->addr >> 12;
 	nvkm_wr32(device, 0x001704, 0x80000000 | addr);
 
-	if (bar->bar[0].mem) {
-		addr = nvkm_memory_addr(bar->bar[0].mem) >> 12;
+	if (bar->bar[0].inst) {
+		addr = bar->bar[0].inst->addr >> 12;
 		nvkm_wr32(device, 0x001714, 0x80000000 | addr);
 	}
 
@@ -140,13 +129,11 @@ gf100_bar_dtor(struct nvkm_bar *base)
 {
 	struct gf100_bar *bar = gf100_bar(base);
 
-	nvkm_vm_ref(NULL, &bar->bar[1].vm, bar->bar[1].pgd);
-	nvkm_gpuobj_del(&bar->bar[1].pgd);
-	nvkm_memory_del(&bar->bar[1].mem);
+	nvkm_vm_ref(NULL, &bar->bar[1].vm, bar->bar[1].inst);
+	nvkm_gpuobj_del(&bar->bar[1].inst);
 
-	nvkm_vm_ref(NULL, &bar->bar[0].vm, bar->bar[0].pgd);
-	nvkm_gpuobj_del(&bar->bar[0].pgd);
-	nvkm_memory_del(&bar->bar[0].mem);
+	nvkm_vm_ref(NULL, &bar->bar[0].vm, bar->bar[0].inst);
+	nvkm_gpuobj_del(&bar->bar[0].inst);
 	return bar;
 }
 
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/gf100.h b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/gf100.h
index f7dea69640d8..051d1faed290 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/gf100.h
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/gf100.h
@@ -5,7 +5,7 @@
 
 struct gf100_bar_vm {
 	struct nvkm_memory *mem;
-	struct nvkm_gpuobj *pgd;
+	struct nvkm_gpuobj *inst;
 	struct nvkm_vm *vm;
 };
 
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/nv50.c b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/nv50.c
index acdbcf6bcb12..bff2aeb0601b 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/nv50.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/nv50.c
@@ -94,7 +94,7 @@ nv50_bar_oneinit(struct nvkm_bar *base)
 	if (ret)
 		return ret;
 
-	ret = nvkm_vm_ref(vm, &bar->bar3_vm, bar->pgd);
+	ret = nvkm_vm_ref(vm, &bar->bar3_vm, bar->mem);
 	nvkm_vm_ref(NULL, &vm, NULL);
 	if (ret)
 		return ret;
@@ -123,7 +123,7 @@ nv50_bar_oneinit(struct nvkm_bar *base)
 
 	atomic_inc(&vm->engref[NVKM_SUBDEV_BAR]);
 
-	ret = nvkm_vm_ref(vm, &bar->bar1_vm, bar->pgd);
+	ret = nvkm_vm_ref(vm, &bar->bar1_vm, bar->mem);
 	nvkm_vm_ref(NULL, &vm, NULL);
 	if (ret)
 		return ret;
@@ -174,9 +174,9 @@ nv50_bar_dtor(struct nvkm_bar *base)
 {
 	struct nv50_bar *bar = nv50_bar(base);
 	nvkm_gpuobj_del(&bar->bar1);
-	nvkm_vm_ref(NULL, &bar->bar1_vm, bar->pgd);
+	nvkm_vm_ref(NULL, &bar->bar1_vm, bar->mem);
 	nvkm_gpuobj_del(&bar->bar3);
-	nvkm_vm_ref(NULL, &bar->bar3_vm, bar->pgd);
+	nvkm_vm_ref(NULL, &bar->bar3_vm, bar->mem);
 	nvkm_gpuobj_del(&bar->pgd);
 	nvkm_gpuobj_del(&bar->pad);
 	nvkm_gpuobj_del(&bar->mem);
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/base.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/base.c
index 4b5634b16b9a..4a94f58e7f23 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/base.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/base.c
@@ -225,7 +225,6 @@ static void
 nvkm_vm_unmap_pgt(struct nvkm_vm *vm, int big, u32 fpde, u32 lpde)
 {
 	struct nvkm_mmu *mmu = vm->mmu;
-	struct nvkm_vm_pgd *vpgd;
 	struct nvkm_vm_pgt *vpgt;
 	struct nvkm_memory *pgt;
 	u32 pde;
@@ -238,9 +237,8 @@ nvkm_vm_unmap_pgt(struct nvkm_vm *vm, int big, u32 fpde, u32 lpde)
 		pgt = vpgt->mem[big];
 		vpgt->mem[big] = NULL;
 
-		list_for_each_entry(vpgd, &vm->pgd_list, head) {
-			mmu->func->map_pgt(vpgd->obj, pde, vpgt->mem);
-		}
+		if (mmu->func->map_pgt)
+			mmu->func->map_pgt(vm, pde, vpgt->mem);
 
 		nvkm_memory_del(&pgt);
 	}
@@ -251,7 +249,6 @@ nvkm_vm_map_pgt(struct nvkm_vm *vm, u32 pde, u32 type)
 {
 	struct nvkm_mmu *mmu = vm->mmu;
 	struct nvkm_vm_pgt *vpgt = &vm->pgt[pde - vm->fpde];
-	struct nvkm_vm_pgd *vpgd;
 	int big = (type != mmu->func->spg_shift);
 	u32 pgt_size;
 	int ret;
@@ -264,9 +261,8 @@ nvkm_vm_map_pgt(struct nvkm_vm *vm, u32 pde, u32 type)
 	if (unlikely(ret))
 		return ret;
 
-	list_for_each_entry(vpgd, &vm->pgd_list, head) {
-		mmu->func->map_pgt(vpgd->obj, pde, vpgt->mem);
-	}
+	if (mmu->func->map_pgt)
+		mmu->func->map_pgt(vm, pde, vpgt->mem);
 
 	vpgt->refcount[big]++;
 	return 0;
@@ -380,78 +376,38 @@ nvkm_vm_new(struct nvkm_device *device, u64 offset, u64 length, u64 mm_offset,
 	return ret;
 }
 
-static int
-nvkm_vm_link(struct nvkm_vm *vm, struct nvkm_gpuobj *pgd)
-{
-	struct nvkm_mmu *mmu = vm->mmu;
-	struct nvkm_vm_pgd *vpgd;
-	int i;
-
-	if (!pgd)
-		return 0;
-
-	vpgd = kzalloc(sizeof(*vpgd), GFP_KERNEL);
-	if (!vpgd)
-		return -ENOMEM;
-
-	vpgd->obj = pgd;
-
-	mutex_lock(&vm->mutex);
-	for (i = vm->fpde; i <= vm->lpde; i++)
-		mmu->func->map_pgt(pgd, i, vm->pgt[i - vm->fpde].mem);
-	list_add(&vpgd->head, &vm->pgd_list);
-	mutex_unlock(&vm->mutex);
-	return 0;
-}
-
-static void
-nvkm_vm_unlink(struct nvkm_vm *vm, struct nvkm_gpuobj *mpgd)
-{
-	struct nvkm_vm_pgd *vpgd, *tmp;
-
-	if (!mpgd)
-		return;
-
-	mutex_lock(&vm->mutex);
-	list_for_each_entry_safe(vpgd, tmp, &vm->pgd_list, head) {
-		if (vpgd->obj == mpgd) {
-			list_del(&vpgd->head);
-			kfree(vpgd);
-			break;
-		}
-	}
-	mutex_unlock(&vm->mutex);
-}
-
 static void
 nvkm_vm_del(struct kref *kref)
 {
 	struct nvkm_vm *vm = container_of(kref, typeof(*vm), refcount);
-	struct nvkm_vm_pgd *vpgd, *tmp;
-
-	list_for_each_entry_safe(vpgd, tmp, &vm->pgd_list, head) {
-		nvkm_vm_unlink(vm, vpgd->obj);
-	}
 
 	vm = nvkm_vmm_dtor(vm);
 	kfree(vm);
 }
 
 int
-nvkm_vm_ref(struct nvkm_vm *ref, struct nvkm_vm **ptr, struct nvkm_gpuobj *pgd)
+nvkm_vm_ref(struct nvkm_vm *ref, struct nvkm_vm **ptr, struct nvkm_gpuobj *inst)
 {
 	if (ref) {
-		int ret = nvkm_vm_link(ref, pgd);
-		if (ret)
-			return ret;
+		if (ref->func->join && inst) {
+			int ret = ref->func->join(ref, inst), i;
+			if (ret)
+				return ret;
+
+			if (ref->mmu->func->map_pgt) {
+				for (i = ref->fpde; i <= ref->lpde; i++)
+					ref->mmu->func->map_pgt(ref, i, ref->pgt[i - ref->fpde].mem);
+			}
+		}
 
 		kref_get(&ref->refcount);
 	}
 
 	if (*ptr) {
-		if ((*ptr)->bootstrapped && pgd)
+		if ((*ptr)->func->part && inst)
+			(*ptr)->func->part(*ptr, inst);
+		if ((*ptr)->bootstrapped && inst)
 			nvkm_memory_del(&(*ptr)->pgt[0].mem[0]);
-		nvkm_vm_unlink(*ptr, pgd);
 		kref_put(&(*ptr)->refcount, nvkm_vm_del);
 	}
 
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gf100.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gf100.c
index da44472ae1d7..f2a43a2a298f 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gf100.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gf100.c
@@ -71,8 +71,9 @@ const u8 gf100_pte_storage_type_map[256] =
 
 
 static void
-gf100_vm_map_pgt(struct nvkm_gpuobj *pgd, u32 index, struct nvkm_memory *pgt[2])
+gf100_vm_map_pgt(struct nvkm_vmm *base, u32 index, struct nvkm_memory *pgt[2])
 {
+	struct gf100_vmm *vmm = gf100_vmm(base);
 	u32 pde[2] = { 0, 0 };
 
 	if (pgt[0])
@@ -80,10 +81,10 @@ gf100_vm_map_pgt(struct nvkm_gpuobj *pgd, u32 index, struct nvkm_memory *pgt[2])
 	if (pgt[1])
 		pde[0] = 0x00000001 | (nvkm_memory_addr(pgt[1]) >> 8);
 
-	nvkm_kmap(pgd);
-	nvkm_wo32(pgd, (index * 8) + 0, pde[0]);
-	nvkm_wo32(pgd, (index * 8) + 4, pde[1]);
-	nvkm_done(pgd);
+	nvkm_kmap(vmm->pd);
+	nvkm_wo32(vmm->pd, (index * 8) + 0, pde[0]);
+	nvkm_wo32(vmm->pd, (index * 8) + 4, pde[1]);
+	nvkm_done(vmm->pd);
 }
 
 static inline u64
@@ -162,9 +163,9 @@ gf100_vm_unmap(struct nvkm_vma *vma, struct nvkm_memory *pgt, u32 pte, u32 cnt)
 static void
 gf100_vm_flush(struct nvkm_vm *vm)
 {
+	struct gf100_vmm *vmm = gf100_vmm(vm);
 	struct nvkm_mmu *mmu = vm->mmu;
 	struct nvkm_device *device = mmu->subdev.device;
-	struct nvkm_vm_pgd *vpgd;
 	u32 type;
 
 	type = 0x00000001; /* PAGE_ALL */
@@ -172,24 +173,22 @@ gf100_vm_flush(struct nvkm_vm *vm)
 		type |= 0x00000004; /* HUB_ONLY */
 
 	mutex_lock(&mmu->subdev.mutex);
-	list_for_each_entry(vpgd, &vm->pgd_list, head) {
-		/* looks like maybe a "free flush slots" counter, the
-		 * faster you write to 0x100cbc to more it decreases
-		 */
-		nvkm_msec(device, 2000,
-			if (nvkm_rd32(device, 0x100c80) & 0x00ff0000)
-				break;
-		);
-
-		nvkm_wr32(device, 0x100cb8, vpgd->obj->addr >> 8);
-		nvkm_wr32(device, 0x100cbc, 0x80000000 | type);
-
-		/* wait for flush to be queued? */
-		nvkm_msec(device, 2000,
-			if (nvkm_rd32(device, 0x100c80) & 0x00008000)
-				break;
-		);
-	}
+	/* looks like maybe a "free flush slots" counter, the
+	 * faster you write to 0x100cbc to more it decreases
+	 */
+	nvkm_msec(device, 2000,
+		if (nvkm_rd32(device, 0x100c80) & 0x00ff0000)
+			break;
+	);
+
+	nvkm_wr32(device, 0x100cb8, nvkm_memory_addr(vmm->pd) >> 8);
+	nvkm_wr32(device, 0x100cbc, 0x80000000 | type);
+
+	/* wait for flush to be queued? */
+	nvkm_msec(device, 2000,
+		if (nvkm_rd32(device, 0x100c80) & 0x00008000)
+			break;
+	);
 	mutex_unlock(&mmu->subdev.mutex);
 }
 
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/nv50.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/nv50.c
index dffcc3a2a54f..053b92503ec8 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/nv50.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/nv50.c
@@ -30,8 +30,11 @@
 #include <engine/gr.h>
 
 static void
-nv50_vm_map_pgt(struct nvkm_gpuobj *pgd, u32 pde, struct nvkm_memory *pgt[2])
+nv50_vm_map_pgt(struct nvkm_vmm *base, u32 pde, struct nvkm_memory *pgt[2])
 {
+	struct nv50_vmm *vmm = nv50_vmm(base);
+	struct nv50_vmm_join *join;
+	u32 pdeo = vmm->func->pd_offset + (pde * 8);
 	u64 phys = 0xdeadcafe00000000ULL;
 	u32 coverage = 0;
 
@@ -55,10 +58,12 @@ nv50_vm_map_pgt(struct nvkm_gpuobj *pgd, u32 pde, struct nvkm_memory *pgt[2])
 			phys |= 0x20;
 	}
 
-	nvkm_kmap(pgd);
-	nvkm_wo32(pgd, (pde * 8) + 0, lower_32_bits(phys));
-	nvkm_wo32(pgd, (pde * 8) + 4, upper_32_bits(phys));
-	nvkm_done(pgd);
+	list_for_each_entry(join, &vmm->join, head) {
+		nvkm_kmap(join->inst);
+		nvkm_wo32(join->inst, pdeo + 0, lower_32_bits(phys));
+		nvkm_wo32(join->inst, pdeo + 4, upper_32_bits(phys));
+		nvkm_done(join->inst);
+	}
 }
 
 static inline u64
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/priv.h b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/priv.h
index 404a8849e181..0e2f2ccf9535 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/priv.h
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/priv.h
@@ -20,7 +20,7 @@ struct nvkm_mmu_func {
 	u8  spg_shift;
 	u8  lpg_shift;
 
-	void (*map_pgt)(struct nvkm_gpuobj *pgd, u32 pde,
+	void (*map_pgt)(struct nvkm_vmm *, u32 pde,
 			struct nvkm_memory *pgt[2]);
 	void (*map)(struct nvkm_vma *, struct nvkm_memory *,
 		    struct nvkm_mem *, u32 pte, u32 cnt,
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c
index 8fc30e7303b8..cde7f00a6be5 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c
@@ -50,7 +50,6 @@ nvkm_vmm_ctor(const struct nvkm_vmm_func *func, struct nvkm_mmu *mmu,
 		return -EINVAL;
 
 	__mutex_init(&vmm->mutex, "&vmm->mutex", key ? key : &_key);
-	INIT_LIST_HEAD(&vmm->pgd_list);
 	kref_init(&vmm->refcount);
 	vmm->fpde = vmm->start >> (mmu->func->pgt_bits + 12);
 	vmm->lpde = vmm->limit >> (mmu->func->pgt_bits + 12);
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.h b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.h
index 3222acacb4b3..5f7d35905123 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.h
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.h
@@ -13,6 +13,9 @@ struct nvkm_vmm_func {
 
 	const struct nvkm_vmm_page *(*page)(struct nvkm_vmm *);
 	u64 page_block;
+
+	int (*join)(struct nvkm_vmm *, struct nvkm_gpuobj *inst);
+	void (*part)(struct nvkm_vmm *, struct nvkm_gpuobj *inst);
 };
 
 int nvkm_vmm_ctor(const struct nvkm_vmm_func *, struct nvkm_mmu *,
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmg84.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmg84.c
index 7e63aa8698bc..70b25421f7d3 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmg84.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmg84.c
@@ -23,6 +23,7 @@
 
 static const struct nv50_vmm_func
 g84_vmm = {
+	.pd_offset = 0x200,
 };
 
 static int
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgf100.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgf100.c
index bb9e28e6d9f2..94f29af37024 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgf100.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgf100.c
@@ -21,9 +21,41 @@
  */
 #include "vmmgf100.h"
 
-#include <core/memory.h>
+#include <core/gpuobj.h>
 #include <subdev/fb.h>
 
+static void
+gf100_vmm_part(struct nvkm_vmm *base, struct nvkm_gpuobj *inst)
+{
+	/*XXX: ideally we'd do this, but there's some chicken-and-egg
+	 *     problems with the bootstrapped instmem vmm.
+	 */
+#if 0
+	nvkm_kmap(inst);
+	nvkm_wo32(inst, 0x0200, 0x00000000);
+	nvkm_wo32(inst, 0x0204, 0x00000000);
+	nvkm_wo32(inst, 0x0208, 0x00000000);
+	nvkm_wo32(inst, 0x020c, 0x00000000);
+	nvkm_done(inst);
+#endif
+}
+
+static int
+gf100_vmm_join(struct nvkm_vmm *base, struct nvkm_gpuobj *inst)
+{
+	struct gf100_vmm *vmm = gf100_vmm(base);
+	u64  addr = nvkm_memory_addr(vmm->pd);
+	u64 limit = vmm->base.limit;
+
+	nvkm_kmap(inst);
+	nvkm_wo32(inst, 0x0200, lower_32_bits(addr));
+	nvkm_wo32(inst, 0x0204, upper_32_bits(addr));
+	nvkm_wo32(inst, 0x0208, lower_32_bits(limit));
+	nvkm_wo32(inst, 0x020c, upper_32_bits(limit));
+	nvkm_done(inst);
+	return 0;
+}
+
 static const struct nvkm_vmm_page *
 gf100_vmm_page(struct nvkm_vmm *base)
 {
@@ -47,6 +79,7 @@ static void *
 gf100_vmm_dtor(struct nvkm_vmm *base)
 {
 	struct gf100_vmm *vmm = gf100_vmm(base);
+	nvkm_memory_del(&vmm->pd);
 	return vmm;
 }
 
@@ -54,6 +87,8 @@ static const struct nvkm_vmm_func
 gf100_vmm_ = {
 	.dtor = gf100_vmm_dtor,
 	.page = gf100_vmm_page,
+	.join = gf100_vmm_join,
+	.part = gf100_vmm_part,
 };
 
 int
@@ -76,6 +111,12 @@ gf100_vmm_new_(const struct gf100_vmm_func *func, struct nvkm_mmu *mmu,
 	if (ret)
 		return ret;
 
+	ret = nvkm_memory_new(mmu->subdev.device, NVKM_MEM_TARGET_INST,
+			      (vmm->base.lpde + 1) * 8, 0x1000, true,
+			      &vmm->pd);
+	if (ret)
+		return ret;
+
 	return 0;
 }
 
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgf100.h b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgf100.h
index df80c8a2a5b3..8a4b921eb5af 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgf100.h
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgf100.h
@@ -6,6 +6,8 @@
 struct gf100_vmm {
 	const struct gf100_vmm_func *func;
 	struct nvkm_vmm base;
+
+	struct nvkm_memory *pd;
 };
 
 struct gf100_vmm_func {
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmnv50.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmnv50.c
index 5a290832f45e..979878fa9ebf 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmnv50.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmnv50.c
@@ -21,6 +21,37 @@
  */
 #include "vmmnv50.h"
 
+static void
+nv50_vmm_part(struct nvkm_vmm *base, struct nvkm_gpuobj *inst)
+{
+	struct nv50_vmm *vmm = nv50_vmm(base);
+	struct nv50_vmm_join *join;
+
+	list_for_each_entry(join, &vmm->join, head) {
+		if (join->inst == inst) {
+			list_del(&join->head);
+			kfree(join);
+			return;
+		}
+	}
+
+	WARN_ON(1);
+}
+
+static int
+nv50_vmm_join(struct nvkm_vmm *base, struct nvkm_gpuobj *inst)
+{
+	struct nv50_vmm *vmm = nv50_vmm(base);
+	struct nv50_vmm_join *join;
+
+	if (!(join = kzalloc(sizeof(*join), GFP_KERNEL)))
+		return -ENOMEM;
+
+	join->inst = inst;
+	list_add_tail(&join->head, &vmm->join);
+	return 0;
+}
+
 static const struct nvkm_vmm_page *
 nv50_vmm_page(struct nvkm_vmm *base)
 {
@@ -45,6 +76,8 @@ nv50_vmm_ = {
 	.dtor = nv50_vmm_dtor,
 	.page = nv50_vmm_page,
 	.page_block = 1 << 29,
+	.join = nv50_vmm_join,
+	.part = nv50_vmm_part,
 };
 
 int
@@ -61,6 +94,7 @@ nv50_vmm_new_(const struct nv50_vmm_func *func, struct nvkm_mmu *mmu,
 	if (!(vmm = kzalloc(sizeof(*vmm), GFP_KERNEL)))
 		return -ENOMEM;
 	vmm->func = func;
+	INIT_LIST_HEAD(&vmm->join);
 	*pvmm = &vmm->base;
 
 	ret = nvkm_vmm_ctor(&nv50_vmm_, mmu, 40, addr, size, key, &vmm->base);
@@ -72,6 +106,7 @@ nv50_vmm_new_(const struct nv50_vmm_func *func, struct nvkm_mmu *mmu,
 
 static const struct nv50_vmm_func
 nv50_vmm = {
+	.pd_offset = 0x1400,
 };
 
 static int
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmnv50.h b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmnv50.h
index 33112c89d270..fe5bb0ac6330 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmnv50.h
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmnv50.h
@@ -6,9 +6,17 @@
 struct nv50_vmm {
 	const struct nv50_vmm_func *func;
 	struct nvkm_vmm base;
+
+	struct list_head join;
 };
 
 struct nv50_vmm_func {
+	u16 pd_offset;
+};
+
+struct nv50_vmm_join {
+	struct list_head head;
+	struct nvkm_gpuobj *inst;
 };
 
 int nv50_vmm_new_(const struct nv50_vmm_func *, struct nvkm_mmu *,
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/secboot/gm200.c b/drivers/gpu/drm/nouveau/nvkm/subdev/secboot/gm200.c
index 73ca1203281d..c205f2cdc915 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/secboot/gm200.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/secboot/gm200.c
@@ -111,28 +111,17 @@ gm200_secboot_oneinit(struct nvkm_secboot *sb)
 	if (ret)
 		return ret;
 
-	ret = nvkm_gpuobj_new(device, 0x8000, 0, true, NULL, &gsb->pgd);
-	if (ret)
-		return ret;
-
 	ret = nvkm_vm_new(device, 0, vm_area_len, 0, NULL, &vm);
 	if (ret)
 		return ret;
 
 	atomic_inc(&vm->engref[NVKM_SUBDEV_PMU]);
 
-	ret = nvkm_vm_ref(vm, &gsb->vm, gsb->pgd);
+	ret = nvkm_vm_ref(vm, &gsb->vm, gsb->inst);
 	nvkm_vm_ref(NULL, &vm, NULL);
 	if (ret)
 		return ret;
 
-	nvkm_kmap(gsb->inst);
-	nvkm_wo32(gsb->inst, 0x200, lower_32_bits(gsb->pgd->addr));
-	nvkm_wo32(gsb->inst, 0x204, upper_32_bits(gsb->pgd->addr));
-	nvkm_wo32(gsb->inst, 0x208, lower_32_bits(vm_area_len - 1));
-	nvkm_wo32(gsb->inst, 0x20c, upper_32_bits(vm_area_len - 1));
-	nvkm_done(gsb->inst);
-
 	if (sb->acr->func->oneinit) {
 		ret = sb->acr->func->oneinit(sb->acr, sb);
 		if (ret)
@@ -160,8 +149,7 @@ gm200_secboot_dtor(struct nvkm_secboot *sb)
 
 	sb->acr->func->dtor(sb->acr);
 
-	nvkm_vm_ref(NULL, &gsb->vm, gsb->pgd);
-	nvkm_gpuobj_del(&gsb->pgd);
+	nvkm_vm_ref(NULL, &gsb->vm, gsb->inst);
 	nvkm_gpuobj_del(&gsb->inst);
 
 	return gsb;
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/secboot/gm200.h b/drivers/gpu/drm/nouveau/nvkm/subdev/secboot/gm200.h
index c8ab3d76bdef..e91767b1b05a 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/secboot/gm200.h
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/secboot/gm200.h
@@ -30,7 +30,6 @@ struct gm200_secboot {
 
 	/* Instance block & address space used for HS FW execution */
 	struct nvkm_gpuobj *inst;
-	struct nvkm_gpuobj *pgd;
 	struct nvkm_vm *vm;
 };
 #define gm200_secboot(sb) container_of(sb, struct gm200_secboot, base)