gpu/nouveau/mmu: add a privileged method to directly manage PTEs

Privileged method to allow direct manipulation of the GPU page
table, this is for share virtual memory (SVM) or share virtual
address (SVA).

They are still things to improve on top of this:
    - avoid PT readback, generally *way* too slow
    - mutliple page sizes (huge pages)
    - systems without a 4KiB page size

Signed-off-by: Ben Skeggs <bskeggs@redhat.com>
Tested-by: Jérôme Glisse <jglisse@redhat.com>

6 files changed, 445 insertions, 34 deletions

diff --git a/drivers/gpu/drm/nouveau/include/nvif/if000c.h b/drivers/gpu/drm/nouveau/include/nvif/if000c.h
index f8e29cfee7f8..20374882ac8f 100644
--- a/drivers/gpu/drm/nouveau/include/nvif/if000c.h
+++ b/drivers/gpu/drm/nouveau/include/nvif/if000c.h
@@ -15,6 +15,8 @@ struct nvif_vmm_v0 {
 #define NVIF_VMM_V0_PUT                                                    0x02
 #define NVIF_VMM_V0_MAP                                                    0x03
 #define NVIF_VMM_V0_UNMAP                                                  0x04
+#define NVIF_VMM_V0_PFNMAP                                                 0x05
+#define NVIF_VMM_V0_PFNCLR                                                 0x06
 
 struct nvif_vmm_page_v0 {
 	__u8  version;
@@ -62,4 +64,28 @@ struct nvif_vmm_unmap_v0 {
 	__u8  pad01[7];
 	__u64 addr;
 };
+
+struct nvif_vmm_pfnmap_v0 {
+	__u8  version;
+	__u8  page;
+	__u8  pad02[6];
+	__u64 addr;
+	__u64 size;
+#define NVIF_VMM_PFNMAP_V0_ADDR                           0xfffffffffffff000ULL
+#define NVIF_VMM_PFNMAP_V0_ADDR_SHIFT                                        12
+#define NVIF_VMM_PFNMAP_V0_APER                           0x00000000000000f0ULL
+#define NVIF_VMM_PFNMAP_V0_HOST                           0x0000000000000000ULL
+#define NVIF_VMM_PFNMAP_V0_VRAM                           0x0000000000000010ULL
+#define NVIF_VMM_PFNMAP_V0_W                              0x0000000000000002ULL
+#define NVIF_VMM_PFNMAP_V0_V                              0x0000000000000001ULL
+#define NVIF_VMM_PFNMAP_V0_NONE                           0x0000000000000000ULL
+	__u64 phys[];
+};
+
+struct nvif_vmm_pfnclr_v0 {
+	__u8  version;
+	__u8  pad01[7];
+	__u64 addr;
+	__u64 size;
+};
 #endif
diff --git a/drivers/gpu/drm/nouveau/include/nvkm/subdev/mmu.h b/drivers/gpu/drm/nouveau/include/nvkm/subdev/mmu.h
index 752aa4730dab..89a38286140a 100644
--- a/drivers/gpu/drm/nouveau/include/nvkm/subdev/mmu.h
+++ b/drivers/gpu/drm/nouveau/include/nvkm/subdev/mmu.h
@@ -64,6 +64,7 @@ struct nvkm_vmm_map {
 	struct nvkm_mm_node *mem;
 	struct scatterlist *sgl;
 	dma_addr_t *dma;
+	u64 *pfn;
 	u64 off;
 
 	const struct nvkm_vmm_page *page;
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/uvmm.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/uvmm.c
index 4b9f07a31219..b483cda151bf 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/uvmm.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/uvmm.c
@@ -43,6 +43,69 @@ nvkm_uvmm_search(struct nvkm_client *client, u64 handle)
 }
 
 static int
+nvkm_uvmm_mthd_pfnclr(struct nvkm_uvmm *uvmm, void *argv, u32 argc)
+{
+	struct nvkm_client *client = uvmm->object.client;
+	union {
+		struct nvif_vmm_pfnclr_v0 v0;
+	} *args = argv;
+	struct nvkm_vmm *vmm = uvmm->vmm;
+	int ret = -ENOSYS;
+	u64 addr, size;
+
+	if (!(ret = nvif_unpack(ret, &argv, &argc, args->v0, 0, 0, false))) {
+		addr = args->v0.addr;
+		size = args->v0.size;
+	} else
+		return ret;
+
+	if (!client->super)
+		return -ENOENT;
+
+	if (size) {
+		mutex_lock(&vmm->mutex);
+		ret = nvkm_vmm_pfn_unmap(vmm, addr, size);
+		mutex_unlock(&vmm->mutex);
+	}
+
+	return ret;
+}
+
+static int
+nvkm_uvmm_mthd_pfnmap(struct nvkm_uvmm *uvmm, void *argv, u32 argc)
+{
+	struct nvkm_client *client = uvmm->object.client;
+	union {
+		struct nvif_vmm_pfnmap_v0 v0;
+	} *args = argv;
+	struct nvkm_vmm *vmm = uvmm->vmm;
+	int ret = -ENOSYS;
+	u64 addr, size, *phys;
+	u8  page;
+
+	if (!(ret = nvif_unpack(ret, &argv, &argc, args->v0, 0, 0, true))) {
+		page = args->v0.page;
+		addr = args->v0.addr;
+		size = args->v0.size;
+		phys = args->v0.phys;
+		if (argc != (size >> page) * sizeof(args->v0.phys[0]))
+			return -EINVAL;
+	} else
+		return ret;
+
+	if (!client->super)
+		return -ENOENT;
+
+	if (size) {
+		mutex_lock(&vmm->mutex);
+		ret = nvkm_vmm_pfn_map(vmm, page, addr, size, phys);
+		mutex_unlock(&vmm->mutex);
+	}
+
+	return ret;
+}
+
+static int
 nvkm_uvmm_mthd_unmap(struct nvkm_uvmm *uvmm, void *argv, u32 argc)
 {
 	struct nvkm_client *client = uvmm->object.client;
@@ -78,7 +141,7 @@ nvkm_uvmm_mthd_unmap(struct nvkm_uvmm *uvmm, void *argv, u32 argc)
 		goto done;
 	}
 
-	nvkm_vmm_unmap_locked(vmm, vma);
+	nvkm_vmm_unmap_locked(vmm, vma, false);
 	ret = 0;
 done:
 	mutex_unlock(&vmm->mutex);
@@ -124,6 +187,11 @@ nvkm_uvmm_mthd_map(struct nvkm_uvmm *uvmm, void *argv, u32 argc)
 		goto fail;
 	}
 
+	if (ret = -EINVAL, vma->mapped && !vma->memory) {
+		VMM_DEBUG(vmm, "pfnmap %016llx", addr);
+		goto fail;
+	}
+
 	if (ret = -EINVAL, vma->addr != addr || vma->size != size) {
 		if (addr + size > vma->addr + vma->size || vma->memory ||
 		    (vma->refd == NVKM_VMA_PAGE_NONE && !vma->mapref)) {
@@ -271,6 +339,8 @@ nvkm_uvmm_mthd(struct nvkm_object *object, u32 mthd, void *argv, u32 argc)
 	case NVIF_VMM_V0_PUT   : return nvkm_uvmm_mthd_put   (uvmm, argv, argc);
 	case NVIF_VMM_V0_MAP   : return nvkm_uvmm_mthd_map   (uvmm, argv, argc);
 	case NVIF_VMM_V0_UNMAP : return nvkm_uvmm_mthd_unmap (uvmm, argv, argc);
+	case NVIF_VMM_V0_PFNMAP: return nvkm_uvmm_mthd_pfnmap(uvmm, argv, argc);
+	case NVIF_VMM_V0_PFNCLR: return nvkm_uvmm_mthd_pfnclr(uvmm, argv, argc);
 	default:
 		break;
 	}
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c
index 69b61e799fd4..0481da5681eb 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c
@@ -255,11 +255,23 @@ nvkm_vmm_unref_sptes(struct nvkm_vmm_iter *it, struct nvkm_vmm_pt *pgt,
 }
 
 static bool
-nvkm_vmm_unref_ptes(struct nvkm_vmm_iter *it, u32 ptei, u32 ptes)
+nvkm_vmm_unref_ptes(struct nvkm_vmm_iter *it, bool pfn, u32 ptei, u32 ptes)
 {
 	const struct nvkm_vmm_desc *desc = it->desc;
 	const int type = desc->type == SPT;
 	struct nvkm_vmm_pt *pgt = it->pt[0];
+	bool dma;
+
+	if (pfn) {
+		/* Need to clear PTE valid bits before we dma_unmap_page(). */
+		dma = desc->func->pfn_clear(it->vmm, pgt->pt[type], ptei, ptes);
+		if (dma) {
+			/* GPU may have cached the PT, flush before unmap. */
+			nvkm_vmm_flush_mark(it);
+			nvkm_vmm_flush(it);
+			desc->func->pfn_unmap(it->vmm, pgt->pt[type], ptei, ptes);
+		}
+	}
 
 	/* Drop PTE references. */
 	pgt->refs[type] -= ptes;
@@ -349,7 +361,7 @@ nvkm_vmm_ref_sptes(struct nvkm_vmm_iter *it, struct nvkm_vmm_pt *pgt,
 }
 
 static bool
-nvkm_vmm_ref_ptes(struct nvkm_vmm_iter *it, u32 ptei, u32 ptes)
+nvkm_vmm_ref_ptes(struct nvkm_vmm_iter *it, bool pfn, u32 ptei, u32 ptes)
 {
 	const struct nvkm_vmm_desc *desc = it->desc;
 	const int type = desc->type == SPT;
@@ -379,7 +391,7 @@ nvkm_vmm_sparse_ptes(const struct nvkm_vmm_desc *desc,
 }
 
 static bool
-nvkm_vmm_sparse_unref_ptes(struct nvkm_vmm_iter *it, u32 ptei, u32 ptes)
+nvkm_vmm_sparse_unref_ptes(struct nvkm_vmm_iter *it, bool pfn, u32 ptei, u32 ptes)
 {
 	struct nvkm_vmm_pt *pt = it->pt[0];
 	if (it->desc->type == PGD)
@@ -387,14 +399,14 @@ nvkm_vmm_sparse_unref_ptes(struct nvkm_vmm_iter *it, u32 ptei, u32 ptes)
 	else
 	if (it->desc->type == LPT)
 		memset(&pt->pte[ptei], 0x00, sizeof(pt->pte[0]) * ptes);
-	return nvkm_vmm_unref_ptes(it, ptei, ptes);
+	return nvkm_vmm_unref_ptes(it, pfn, ptei, ptes);
 }
 
 static bool
-nvkm_vmm_sparse_ref_ptes(struct nvkm_vmm_iter *it, u32 ptei, u32 ptes)
+nvkm_vmm_sparse_ref_ptes(struct nvkm_vmm_iter *it, bool pfn, u32 ptei, u32 ptes)
 {
 	nvkm_vmm_sparse_ptes(it->desc, it->pt[0], ptei, ptes);
-	return nvkm_vmm_ref_ptes(it, ptei, ptes);
+	return nvkm_vmm_ref_ptes(it, pfn, ptei, ptes);
 }
 
 static bool
@@ -487,8 +499,8 @@ nvkm_vmm_ref_swpt(struct nvkm_vmm_iter *it, struct nvkm_vmm_pt *pgd, u32 pdei)
 
 static inline u64
 nvkm_vmm_iter(struct nvkm_vmm *vmm, const struct nvkm_vmm_page *page,
-	      u64 addr, u64 size, const char *name, bool ref,
-	      bool (*REF_PTES)(struct nvkm_vmm_iter *, u32, u32),
+	      u64 addr, u64 size, const char *name, bool ref, bool pfn,
+	      bool (*REF_PTES)(struct nvkm_vmm_iter *, bool pfn, u32, u32),
 	      nvkm_vmm_pte_func MAP_PTES, struct nvkm_vmm_map *map,
 	      nvkm_vmm_pxe_func CLR_PTES)
 {
@@ -548,7 +560,7 @@ nvkm_vmm_iter(struct nvkm_vmm *vmm, const struct nvkm_vmm_page *page,
 		}
 
 		/* Handle PTE updates. */
-		if (!REF_PTES || REF_PTES(&it, ptei, ptes)) {
+		if (!REF_PTES || REF_PTES(&it, pfn, ptei, ptes)) {
 			struct nvkm_mmu_pt *pt = pgt->pt[type];
 			if (MAP_PTES || CLR_PTES) {
 				if (MAP_PTES)
@@ -590,7 +602,7 @@ static void
 nvkm_vmm_ptes_sparse_put(struct nvkm_vmm *vmm, const struct nvkm_vmm_page *page,
 			 u64 addr, u64 size)
 {
-	nvkm_vmm_iter(vmm, page, addr, size, "sparse unref", false,
+	nvkm_vmm_iter(vmm, page, addr, size, "sparse unref", false, false,
 		      nvkm_vmm_sparse_unref_ptes, NULL, NULL,
 		      page->desc->func->invalid ?
 		      page->desc->func->invalid : page->desc->func->unmap);
@@ -602,8 +614,8 @@ nvkm_vmm_ptes_sparse_get(struct nvkm_vmm *vmm, const struct nvkm_vmm_page *page,
 {
 	if ((page->type & NVKM_VMM_PAGE_SPARSE)) {
 		u64 fail = nvkm_vmm_iter(vmm, page, addr, size, "sparse ref",
-					 true, nvkm_vmm_sparse_ref_ptes, NULL,
-					 NULL, page->desc->func->sparse);
+					 true, false, nvkm_vmm_sparse_ref_ptes,
+					 NULL, NULL, page->desc->func->sparse);
 		if (fail != ~0ULL) {
 			if ((size = fail - addr))
 				nvkm_vmm_ptes_sparse_put(vmm, page, addr, size);
@@ -666,11 +678,11 @@ nvkm_vmm_ptes_sparse(struct nvkm_vmm *vmm, u64 addr, u64 size, bool ref)
 
 static void
 nvkm_vmm_ptes_unmap_put(struct nvkm_vmm *vmm, const struct nvkm_vmm_page *page,
-			u64 addr, u64 size, bool sparse)
+			u64 addr, u64 size, bool sparse, bool pfn)
 {
 	const struct nvkm_vmm_desc_func *func = page->desc->func;
 	nvkm_vmm_iter(vmm, page, addr, size, "unmap + unref",
-		      false, nvkm_vmm_unref_ptes, NULL, NULL,
+		      false, pfn, nvkm_vmm_unref_ptes, NULL, NULL,
 		      sparse ? func->sparse : func->invalid ? func->invalid :
 							      func->unmap);
 }
@@ -681,10 +693,10 @@ nvkm_vmm_ptes_get_map(struct nvkm_vmm *vmm, const struct nvkm_vmm_page *page,
 		      nvkm_vmm_pte_func func)
 {
 	u64 fail = nvkm_vmm_iter(vmm, page, addr, size, "ref + map", true,
-				 nvkm_vmm_ref_ptes, func, map, NULL);
+				 false, nvkm_vmm_ref_ptes, func, map, NULL);
 	if (fail != ~0ULL) {
 		if ((size = fail - addr))
-			nvkm_vmm_ptes_unmap_put(vmm, page, addr, size, false);
+			nvkm_vmm_ptes_unmap_put(vmm, page, addr, size, false, false);
 		return -ENOMEM;
 	}
 	return 0;
@@ -692,10 +704,11 @@ nvkm_vmm_ptes_get_map(struct nvkm_vmm *vmm, const struct nvkm_vmm_page *page,
 
 static void
 nvkm_vmm_ptes_unmap(struct nvkm_vmm *vmm, const struct nvkm_vmm_page *page,
-		    u64 addr, u64 size, bool sparse)
+		    u64 addr, u64 size, bool sparse, bool pfn)
 {
 	const struct nvkm_vmm_desc_func *func = page->desc->func;
-	nvkm_vmm_iter(vmm, page, addr, size, "unmap", false, NULL, NULL, NULL,
+	nvkm_vmm_iter(vmm, page, addr, size, "unmap", false, pfn,
+		      NULL, NULL, NULL,
 		      sparse ? func->sparse : func->invalid ? func->invalid :
 							      func->unmap);
 }
@@ -705,7 +718,7 @@ nvkm_vmm_ptes_map(struct nvkm_vmm *vmm, const struct nvkm_vmm_page *page,
 		  u64 addr, u64 size, struct nvkm_vmm_map *map,
 		  nvkm_vmm_pte_func func)
 {
-	nvkm_vmm_iter(vmm, page, addr, size, "map", false,
+	nvkm_vmm_iter(vmm, page, addr, size, "map", false, false,
 		      NULL, func, map, NULL);
 }
 
@@ -713,7 +726,7 @@ static void
 nvkm_vmm_ptes_put(struct nvkm_vmm *vmm, const struct nvkm_vmm_page *page,
 		  u64 addr, u64 size)
 {
-	nvkm_vmm_iter(vmm, page, addr, size, "unref", false,
+	nvkm_vmm_iter(vmm, page, addr, size, "unref", false, false,
 		      nvkm_vmm_unref_ptes, NULL, NULL, NULL);
 }
 
@@ -721,7 +734,7 @@ static int
 nvkm_vmm_ptes_get(struct nvkm_vmm *vmm, const struct nvkm_vmm_page *page,
 		  u64 addr, u64 size)
 {
-	u64 fail = nvkm_vmm_iter(vmm, page, addr, size, "ref", true,
+	u64 fail = nvkm_vmm_iter(vmm, page, addr, size, "ref", true, false,
 				 nvkm_vmm_ref_ptes, NULL, NULL, NULL);
 	if (fail != ~0ULL) {
 		if (fail != addr)
@@ -936,11 +949,40 @@ nvkm_vmm_node_split(struct nvkm_vmm *vmm,
 }
 
 static void
+nvkm_vma_dump(struct nvkm_vma *vma)
+{
+	printk(KERN_ERR "%016llx %016llx %c%c%c%c%c%c%c%c%c %p\n",
+	       vma->addr, (u64)vma->size,
+	       vma->used ? '-' : 'F',
+	       vma->mapref ? 'R' : '-',
+	       vma->sparse ? 'S' : '-',
+	       vma->page != NVKM_VMA_PAGE_NONE ? '0' + vma->page : '-',
+	       vma->refd != NVKM_VMA_PAGE_NONE ? '0' + vma->refd : '-',
+	       vma->part ? 'P' : '-',
+	       vma->user ? 'U' : '-',
+	       vma->busy ? 'B' : '-',
+	       vma->mapped ? 'M' : '-',
+	       vma->memory);
+}
+
+static void
+nvkm_vmm_dump(struct nvkm_vmm *vmm)
+{
+	struct nvkm_vma *vma;
+	list_for_each_entry(vma, &vmm->list, head) {
+		nvkm_vma_dump(vma);
+	}
+}
+
+static void
 nvkm_vmm_dtor(struct nvkm_vmm *vmm)
 {
 	struct nvkm_vma *vma;
 	struct rb_node *node;
 
+	if (0)
+		nvkm_vmm_dump(vmm);
+
 	while ((node = rb_first(&vmm->root))) {
 		struct nvkm_vma *vma = rb_entry(node, typeof(*vma), tree);
 		nvkm_vmm_put(vmm, &vma);
@@ -1105,33 +1147,210 @@ nvkm_vmm_new_(const struct nvkm_vmm_func *func, struct nvkm_mmu *mmu,
 	return nvkm_vmm_ctor(func, mmu, hdr, managed, addr, size, key, name, *pvmm);
 }
 
+static struct nvkm_vma *
+nvkm_vmm_pfn_split_merge(struct nvkm_vmm *vmm, struct nvkm_vma *vma,
+			 u64 addr, u64 size, u8 page, bool map)
+{
+	struct nvkm_vma *prev = NULL;
+	struct nvkm_vma *next = NULL;
+
+	if (vma->addr == addr && vma->part && (prev = node(vma, prev))) {
+		if (prev->memory || prev->mapped != map)
+			prev = NULL;
+	}
+
+	if (vma->addr + vma->size == addr + size && (next = node(vma, next))) {
+		if (!next->part ||
+		    next->memory || next->mapped != map)
+			next = NULL;
+	}
+
+	if (prev || next)
+		return nvkm_vmm_node_merge(vmm, prev, vma, next, size);
+	return nvkm_vmm_node_split(vmm, vma, addr, size);
+}
+
+int
+nvkm_vmm_pfn_unmap(struct nvkm_vmm *vmm, u64 addr, u64 size)
+{
+	struct nvkm_vma *vma = nvkm_vmm_node_search(vmm, addr);
+	struct nvkm_vma *next;
+	u64 limit = addr + size;
+	u64 start = addr;
+
+	if (!vma)
+		return -EINVAL;
+
+	do {
+		if (!vma->mapped || vma->memory)
+			continue;
+
+		size = min(limit - start, vma->size - (start - vma->addr));
+
+		nvkm_vmm_ptes_unmap_put(vmm, &vmm->func->page[vma->refd],
+					start, size, false, true);
+
+		next = nvkm_vmm_pfn_split_merge(vmm, vma, start, size, 0, false);
+		if (!WARN_ON(!next)) {
+			vma = next;
+			vma->refd = NVKM_VMA_PAGE_NONE;
+			vma->mapped = false;
+		}
+	} while ((vma = node(vma, next)) && (start = vma->addr) < limit);
+
+	return 0;
+}
+
+int
+nvkm_vmm_pfn_map(struct nvkm_vmm *vmm, u8 shift, u64 addr, u64 size, u64 *pfn)
+{
+	const struct nvkm_vmm_page *page = vmm->func->page;
+	struct nvkm_vma *vma, *tmp;
+	u64 limit = addr + size;
+	u64 start = addr;
+	int pm = size >> shift;
+	int pi = 0;
+
+	/* Only support mapping where the page size of the incoming page
+	 * array matches a page size available for direct mapping.
+	 */
+	while (page->shift && page->shift != shift &&
+	       page->desc->func->pfn == NULL)
+		page++;
+
+	if (!page->shift || !IS_ALIGNED(addr, 1ULL << shift) ||
+			    !IS_ALIGNED(size, 1ULL << shift) ||
+	    addr + size < addr || addr + size > vmm->limit) {
+		VMM_DEBUG(vmm, "paged map %d %d %016llx %016llx\n",
+			  shift, page->shift, addr, size);
+		return -EINVAL;
+	}
+
+	if (!(vma = nvkm_vmm_node_search(vmm, addr)))
+		return -ENOENT;
+
+	do {
+		bool map = !!(pfn[pi] & NVKM_VMM_PFN_V);
+		bool mapped = vma->mapped;
+		u64 size = limit - start;
+		u64 addr = start;
+		int pn, ret = 0;
+
+		/* Narrow the operation window to cover a single action (page
+		 * should be mapped or not) within a single VMA.
+		 */
+		for (pn = 0; pi + pn < pm; pn++) {
+			if (map != !!(pfn[pi + pn] & NVKM_VMM_PFN_V))
+				break;
+		}
+		size = min_t(u64, size, pn << page->shift);
+		size = min_t(u64, size, vma->size + vma->addr - addr);
+
+		/* Reject any operation to unmanaged regions, and areas that
+		 * have nvkm_memory objects mapped in them already.
+		 */
+		if (!vma->mapref || vma->memory) {
+			ret = -EINVAL;
+			goto next;
+		}
+
+		/* In order to both properly refcount GPU page tables, and
+		 * prevent "normal" mappings and these direct mappings from
+		 * interfering with each other, we need to track contiguous
+		 * ranges that have been mapped with this interface.
+		 *
+		 * Here we attempt to either split an existing VMA so we're
+		 * able to flag the region as either unmapped/mapped, or to
+		 * merge with adjacent VMAs that are already compatible.
+		 *
+		 * If the region is already compatible, nothing is required.
+		 */
+		if (map != mapped) {
+			tmp = nvkm_vmm_pfn_split_merge(vmm, vma, addr, size,
+						       page -
+						       vmm->func->page, map);
+			if (WARN_ON(!tmp)) {
+				ret = -ENOMEM;
+				goto next;
+			}
+
+			if ((tmp->mapped = map))
+				tmp->refd = page - vmm->func->page;
+			else
+				tmp->refd = NVKM_VMA_PAGE_NONE;
+			vma = tmp;
+		}
+
+		/* Update HW page tables. */
+		if (map) {
+			struct nvkm_vmm_map args;
+			args.page = page;
+			args.pfn = &pfn[pi];
+
+			if (!mapped) {
+				ret = nvkm_vmm_ptes_get_map(vmm, page, addr,
+							    size, &args, page->
+							    desc->func->pfn);
+			} else {
+				nvkm_vmm_ptes_map(vmm, page, addr, size, &args,
+						  page->desc->func->pfn);
+			}
+		} else {
+			if (mapped) {
+				nvkm_vmm_ptes_unmap_put(vmm, page, addr, size,
+							false, true);
+			}
+		}
+
+next:
+		/* Iterate to next operation. */
+		if (vma->addr + vma->size == addr + size)
+			vma = node(vma, next);
+		start += size;
+
+		if (ret) {
+			/* Failure is signalled by clearing the valid bit on
+			 * any PFN that couldn't be modified as requested.
+			 */
+			while (size) {
+				pfn[pi++] = NVKM_VMM_PFN_NONE;
+				size -= 1 << page->shift;
+			}
+		} else {
+			pi += size >> page->shift;
+		}
+	} while (vma && start < limit);
+
+	return 0;
+}
+
 void
 nvkm_vmm_unmap_region(struct nvkm_vmm *vmm, struct nvkm_vma *vma)
 {
-	struct nvkm_vma *next = node(vma, next);
 	struct nvkm_vma *prev = NULL;
+	struct nvkm_vma *next;
 
 	nvkm_memory_tags_put(vma->memory, vmm->mmu->subdev.device, &vma->tags);
 	nvkm_memory_unref(&vma->memory);
 	vma->mapped = false;
 
-	if (!vma->part || ((prev = node(vma, prev)), prev->mapped))
+	if (vma->part && (prev = node(vma, prev)) && prev->mapped)
 		prev = NULL;
-	if (!next->part || next->mapped)
+	if ((next = node(vma, next)) && (!next->part || next->mapped))
 		next = NULL;
 	nvkm_vmm_node_merge(vmm, prev, vma, next, vma->size);
 }
 
 void
-nvkm_vmm_unmap_locked(struct nvkm_vmm *vmm, struct nvkm_vma *vma)
+nvkm_vmm_unmap_locked(struct nvkm_vmm *vmm, struct nvkm_vma *vma, bool pfn)
 {
 	const struct nvkm_vmm_page *page = &vmm->func->page[vma->refd];
 
 	if (vma->mapref) {
-		nvkm_vmm_ptes_unmap_put(vmm, page, vma->addr, vma->size, vma->sparse);
+		nvkm_vmm_ptes_unmap_put(vmm, page, vma->addr, vma->size, vma->sparse, pfn);
 		vma->refd = NVKM_VMA_PAGE_NONE;
 	} else {
-		nvkm_vmm_ptes_unmap(vmm, page, vma->addr, vma->size, vma->sparse);
+		nvkm_vmm_ptes_unmap(vmm, page, vma->addr, vma->size, vma->sparse, pfn);
 	}
 
 	nvkm_vmm_unmap_region(vmm, vma);
@@ -1142,7 +1361,7 @@ nvkm_vmm_unmap(struct nvkm_vmm *vmm, struct nvkm_vma *vma)
 {
 	if (vma->memory) {
 		mutex_lock(&vmm->mutex);
-		nvkm_vmm_unmap_locked(vmm, vma);
+		nvkm_vmm_unmap_locked(vmm, vma, false);
 		mutex_unlock(&vmm->mutex);
 	}
 }
@@ -1341,7 +1560,8 @@ nvkm_vmm_put_locked(struct nvkm_vmm *vmm, struct nvkm_vma *vma)
 				 * the page tree.
 				 */
 				nvkm_vmm_ptes_unmap_put(vmm, &page[refd], addr,
-							size, vma->sparse);
+							size, vma->sparse,
+							!mem);
 			} else
 			if (refd != NVKM_VMA_PAGE_NONE) {
 				/* Drop allocation-time PTE references. */
@@ -1577,7 +1797,7 @@ nvkm_vmm_join(struct nvkm_vmm *vmm, struct nvkm_memory *inst)
 }
 
 static bool
-nvkm_vmm_boot_ptes(struct nvkm_vmm_iter *it, u32 ptei, u32 ptes)
+nvkm_vmm_boot_ptes(struct nvkm_vmm_iter *it, bool pfn, u32 ptei, u32 ptes)
 {
 	const struct nvkm_vmm_desc *desc = it->desc;
 	const int type = desc->type == SPT;
@@ -1599,7 +1819,7 @@ nvkm_vmm_boot(struct nvkm_vmm *vmm)
 	if (ret)
 		return ret;
 
-	nvkm_vmm_iter(vmm, page, vmm->start, limit, "bootstrap", false,
+	nvkm_vmm_iter(vmm, page, vmm->start, limit, "bootstrap", false, false,
 		      nvkm_vmm_boot_ptes, NULL, NULL, NULL);
 	vmm->bootstrapped = true;
 	return 0;
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.h b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.h
index 937e054655b5..a227050a4115 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.h
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.h
@@ -67,6 +67,10 @@ struct nvkm_vmm_desc_func {
 	nvkm_vmm_pte_func mem;
 	nvkm_vmm_pte_func dma;
 	nvkm_vmm_pte_func sgl;
+
+	nvkm_vmm_pte_func pfn;
+	bool (*pfn_clear)(struct nvkm_vmm *, struct nvkm_mmu_pt *, u32 ptei, u32 ptes);
+	nvkm_vmm_pxe_func pfn_unmap;
 };
 
 extern const struct nvkm_vmm_desc_func gf100_vmm_pgd;
@@ -164,8 +168,20 @@ int nvkm_vmm_get_locked(struct nvkm_vmm *, bool getref, bool mapref,
 			bool sparse, u8 page, u8 align, u64 size,
 			struct nvkm_vma **pvma);
 void nvkm_vmm_put_locked(struct nvkm_vmm *, struct nvkm_vma *);
-void nvkm_vmm_unmap_locked(struct nvkm_vmm *, struct nvkm_vma *);
-void nvkm_vmm_unmap_region(struct nvkm_vmm *vmm, struct nvkm_vma *vma);
+void nvkm_vmm_unmap_locked(struct nvkm_vmm *, struct nvkm_vma *, bool pfn);
+void nvkm_vmm_unmap_region(struct nvkm_vmm *, struct nvkm_vma *);
+
+#define NVKM_VMM_PFN_ADDR                                 0xfffffffffffff000ULL
+#define NVKM_VMM_PFN_ADDR_SHIFT                                              12
+#define NVKM_VMM_PFN_APER                                 0x00000000000000f0ULL
+#define NVKM_VMM_PFN_HOST                                 0x0000000000000000ULL
+#define NVKM_VMM_PFN_VRAM                                 0x0000000000000010ULL
+#define NVKM_VMM_PFN_W                                    0x0000000000000002ULL
+#define NVKM_VMM_PFN_V                                    0x0000000000000001ULL
+#define NVKM_VMM_PFN_NONE                                 0x0000000000000000ULL
+
+int nvkm_vmm_pfn_map(struct nvkm_vmm *, u8 page, u64 addr, u64 size, u64 *pfn);
+int nvkm_vmm_pfn_unmap(struct nvkm_vmm *, u64 addr, u64 size);
 
 struct nvkm_vma *nvkm_vma_tail(struct nvkm_vma *, u64 tail);
 
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgp100.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgp100.c
index 4d186d843c6c..9a767c457039 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgp100.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgp100.c
@@ -27,6 +27,81 @@
 #include <nvif/ifc00d.h>
 #include <nvif/unpack.h>
 
+static void
+gp100_vmm_pfn_unmap(struct nvkm_vmm *vmm,
+		    struct nvkm_mmu_pt *pt, u32 ptei, u32 ptes)
+{
+	struct device *dev = vmm->mmu->subdev.device->dev;
+	dma_addr_t addr;
+
+	nvkm_kmap(pt->memory);
+	while (ptes--) {
+		u32 datalo = nvkm_ro32(pt->memory, pt->base + ptei * 8 + 0);
+		u32 datahi = nvkm_ro32(pt->memory, pt->base + ptei * 8 + 4);
+		u64 data   = (u64)datahi << 32 | datalo;
+		if ((data & (3ULL << 1)) != 0) {
+			addr = (data >> 8) << 12;
+			dma_unmap_page(dev, addr, PAGE_SIZE, DMA_BIDIRECTIONAL);
+		}
+		ptei++;
+	}
+	nvkm_done(pt->memory);
+}
+
+static bool
+gp100_vmm_pfn_clear(struct nvkm_vmm *vmm,
+		    struct nvkm_mmu_pt *pt, u32 ptei, u32 ptes)
+{
+	bool dma = false;
+	nvkm_kmap(pt->memory);
+	while (ptes--) {
+		u32 datalo = nvkm_ro32(pt->memory, pt->base + ptei * 8 + 0);
+		u32 datahi = nvkm_ro32(pt->memory, pt->base + ptei * 8 + 4);
+		u64 data   = (u64)datahi << 32 | datalo;
+		if ((data & BIT_ULL(0)) && (data & (3ULL << 1)) != 0) {
+			VMM_WO064(pt, vmm, ptei * 8, data & ~BIT_ULL(0));
+			dma = true;
+		}
+		ptei++;
+	}
+	nvkm_done(pt->memory);
+	return dma;
+}
+
+static void
+gp100_vmm_pgt_pfn(struct nvkm_vmm *vmm, struct nvkm_mmu_pt *pt,
+		  u32 ptei, u32 ptes, struct nvkm_vmm_map *map)
+{
+	struct device *dev = vmm->mmu->subdev.device->dev;
+	dma_addr_t addr;
+
+	nvkm_kmap(pt->memory);
+	while (ptes--) {
+		u64 data = 0;
+		if (!(*map->pfn & NVKM_VMM_PFN_W))
+			data |= BIT_ULL(6); /* RO. */
+
+		if (!(*map->pfn & NVKM_VMM_PFN_VRAM)) {
+			addr = *map->pfn >> NVKM_VMM_PFN_ADDR_SHIFT;
+			addr = dma_map_page(dev, pfn_to_page(addr), 0,
+					    PAGE_SIZE, DMA_BIDIRECTIONAL);
+			if (!WARN_ON(dma_mapping_error(dev, addr))) {
+				data |= addr >> 4;
+				data |= 2ULL << 1; /* SYSTEM_COHERENT_MEMORY. */
+				data |= BIT_ULL(3); /* VOL. */
+				data |= BIT_ULL(0); /* VALID. */
+			}
+		} else {
+			data |= (*map->pfn & NVKM_VMM_PFN_ADDR) >> 4;
+			data |= BIT_ULL(0); /* VALID. */
+		}
+
+		VMM_WO064(pt, vmm, ptei++ * 8, data);
+		map->pfn++;
+	}
+	nvkm_done(pt->memory);
+}
+
 static inline void
 gp100_vmm_pgt_pte(struct nvkm_vmm *vmm, struct nvkm_mmu_pt *pt,
 		  u32 ptei, u32 ptes, struct nvkm_vmm_map *map, u64 addr)
@@ -89,6 +164,9 @@ gp100_vmm_desc_spt = {
 	.mem = gp100_vmm_pgt_mem,
 	.dma = gp100_vmm_pgt_dma,
 	.sgl = gp100_vmm_pgt_sgl,
+	.pfn = gp100_vmm_pgt_pfn,
+	.pfn_clear = gp100_vmm_pfn_clear,
+	.pfn_unmap = gp100_vmm_pfn_unmap,
 };
 
 static void