drm/i915: Actually create big pages structsbig_pages

With this patch there is no change to the small chance of actually
getting big pages. This is because of the requisite contiguous pages
(and that they're read out of the page cache in order). Quoting an
anonymous someone who has tested this: "after a certain uptime, anything
beyond order 0 isn't possible"

This code simply lays in more of the required infrastructure to support
it.

This patch makes the i915_vma member big_page_promoted redundant as
there is now a pointer to the big page struct that was created for the
VMA.

Signed-off-by: Ben Widawsky <ben@bwidawsk.net>

4 files changed, 63 insertions, 13 deletions

diff --git a/drivers/gpu/drm/i915/i915_drv.h b/drivers/gpu/drm/i915/i915_drv.h
index 3790fefa27a0..28373986cdf6 100644
--- a/drivers/gpu/drm/i915/i915_drv.h
+++ b/drivers/gpu/drm/i915/i915_drv.h
@@ -556,11 +556,17 @@ enum i915_cache_level {
 
 typedef uint32_t gen6_gtt_pte_t;
 
+#define GEN7_BIG_PAGE_SIZE (32<<10)
 struct i915_vma;
 struct i915_big_page {
 	struct kref ref;
 	struct drm_mm_node node;
 	struct i915_vma *vma;
+
+#define BIG_PTES_PER_PT \
+	(PAGE_SIZE / (GEN7_BIG_PAGE_SIZE/PAGE_SIZE * sizeof(gen6_gtt_pte_t)))
+	DECLARE_BITMAP(used_slots, BIG_PTES_PER_PT);
+#undef BIG_PTES_PER_PT
 };
 
 /**
@@ -575,6 +581,8 @@ struct i915_vma {
 	struct drm_mm_node node;
 	struct drm_i915_gem_object *obj;
 	struct i915_big_page *big_page;
+	uint16_t first_big_pfn;
+	uint16_t num_big_pfns;
 	struct i915_address_space *vm;
 
 	/** This object's place on the active/inactive lists */
@@ -592,8 +600,6 @@ struct i915_vma {
 	unsigned long exec_handle;
 	struct drm_i915_gem_exec_object2 *exec_entry;
 
-	unsigned int big_page_promoted:1;
-
 	/**
 	 * How many users have pinned this object in GTT space. The following
 	 * users can each hold at most one reference: pwrite/pread, pin_ioctl
@@ -1956,8 +1962,6 @@ struct drm_i915_file_private {
 #define HAS_L3_DPF(dev) (IS_IVYBRIDGE(dev) || IS_HASWELL(dev))
 #define NUM_L3_SLICES(dev) (IS_HSW_GT3(dev) ? 2 : HAS_L3_DPF(dev))
 
-#define GEN7_BIG_PAGE_SIZE (32<<10)
-
 #define GT_FREQUENCY_MULTIPLIER 50
 
 #include "i915_trace.h"
@@ -2395,6 +2399,7 @@ void i915_gem_suspend_gtt_mappings(struct drm_device *dev);
 void i915_gem_restore_gtt_mappings(struct drm_device *dev);
 int __must_check i915_gem_vm_prepare_vma(struct i915_vma *vma,
 					 struct drm_mm_node *node);
+void big_page_release(struct kref *ref);
 void i915_gem_vm_finish_vma(struct i915_vma *vma);
 void i915_gem_init_global_gtt(struct drm_device *dev);
 void i915_gem_setup_global_gtt(struct drm_device *dev, unsigned long start,
diff --git a/drivers/gpu/drm/i915/i915_gem.c b/drivers/gpu/drm/i915/i915_gem.c
index ed982b6220a2..fb4f80689ca4 100644
--- a/drivers/gpu/drm/i915/i915_gem.c
+++ b/drivers/gpu/drm/i915/i915_gem.c
@@ -3340,6 +3340,8 @@ skip_alloc:
 err_remove_node:
 	drm_mm_remove_node(node);
 err_free_vma:
+	if (vma->big_page)
+		kref_put(&vma->big_page->ref, big_page_release);
 	i915_gem_vma_destroy(vma);
 err_unpin:
 	i915_gem_object_unpin_pages(obj);
diff --git a/drivers/gpu/drm/i915/i915_gem_gtt.c b/drivers/gpu/drm/i915/i915_gem_gtt.c
index ea809b803885..5049cd0cf23e 100644
--- a/drivers/gpu/drm/i915/i915_gem_gtt.c
+++ b/drivers/gpu/drm/i915/i915_gem_gtt.c
@@ -716,6 +716,26 @@ static bool gen7_promote(struct i915_hw_ppgtt *ppgtt, struct i915_vma *vma)
 	return true;
 }
 
+static struct i915_big_page *create_big_page(struct i915_vma *vma)
+{
+	struct i915_big_page *big = kzalloc(sizeof(*big), GFP_KERNEL);
+	if (!big)
+		return ERR_PTR(-ENOMEM);
+
+	kref_init(&big->ref);
+	vma->first_big_pfn = 0;
+	vma->num_big_pfns = vma->obj->base.size / GEN7_BIG_PAGE_SIZE;
+	bitmap_set(big->used_slots, 0, vma->num_big_pfns);
+	big->vma = vma;
+	return big;
+}
+
+static int find_first_slot(struct i915_hw_ppgtt *ppgtt, size_t size,
+			   unsigned alignment)
+{
+	return -ENOENT;
+}
+
 /*
  * Try to merge a VMA into an existing big page allocation. There are
  * essentially 3 return values:
@@ -727,7 +747,14 @@ static struct i915_big_page *gen7_big_page_merge(struct i915_hw_ppgtt *ppgtt,
 						 struct i915_vma *vma,
 						 unsigned alignment)
 {
+	int which_pde;
+
 	BUG_ON(vma->big_page);
+
+	which_pde = find_first_slot(ppgtt, vma->obj->base.size, alignment);
+	if (which_pde < 0)
+		return create_big_page(vma);
+
 	return ERR_PTR(-ENOSPC);
 }
 
@@ -762,10 +789,12 @@ struct i915_vma *gen7_merge_or_promote(struct i915_address_space *vm,
 		return NULL;
 
 	big_page = gen7_big_page_merge(ppgtt, vma, alignment);
-	if (!IS_ERR(big_page))
-		BUG(); /*Not support yet */
+	if (IS_ERR(big_page)) {
+		DRM_DEBUG_DRIVER("Failed to create big page\n");
+		return NULL;
+	}
 
-	return NULL;
+	return big_page->vma;
 }
 
 static int gen8_ppgtt_enable(struct i915_hw_ppgtt *ppgtt)
@@ -1536,9 +1565,6 @@ static void gen7_demote(struct i915_hw_ppgtt *ppgtt, struct i915_vma *vma)
 		(gen6_gtt_pte_t __iomem *)dev_priv->gtt.gsm + ppgtt->pd_offset / sizeof(gen6_gtt_pte_t);
 	int i;
 
-	if (!vma->big_page_promoted)
-		return;
-
 	for (i = 0; i < num_pde; i++) {
 		u32 pd_entry = readl(pd_addr + first_pde + i);
 		writel(pd_entry & ~GEN7_PDE_BIG, pd_addr + first_pde + i);
@@ -1548,6 +1574,19 @@ static void gen7_demote(struct i915_hw_ppgtt *ppgtt, struct i915_vma *vma)
 	trace_i915_big_page_demote(vma);
 }
 
+void big_page_release(struct kref *ref)
+{
+	struct i915_big_page *big_page =
+		container_of(ref, struct i915_big_page, ref);
+	struct i915_vma *vma = big_page->vma;
+	struct i915_hw_ppgtt *ppgtt =
+		container_of(vma->vm, struct i915_hw_ppgtt, base);
+
+	WARN_ON(bitmap_empty(big_page->used_slots, GEN7_BIG_PAGES_PER_PT) != 1);
+	ppgtt->demote(ppgtt, big_page->vma);
+	kfree(big_page);
+}
+
 void i915_gem_vm_finish_vma(struct i915_vma *vma)
 {
 	struct drm_i915_gem_object *obj = vma->obj;
@@ -1559,8 +1598,12 @@ void i915_gem_vm_finish_vma(struct i915_vma *vma)
 	if (!i915_is_ggtt(vma->vm))
 		ppgtt = vm_to_ppgtt_safe(vma->vm);
 
-	if (ppgtt && ppgtt->demote)
-		ppgtt->demote(ppgtt, vma);
+	if (vma->big_page) {
+		bitmap_clear(vma->big_page->used_slots,
+			     vma->first_big_pfn,
+			     vma->num_big_pfns);
+		kref_put(&vma->big_page->ref, big_page_release);
+	}
 
 	interruptible = do_idling(dev_priv);
 
diff --git a/drivers/gpu/drm/i915/i915_trace.h b/drivers/gpu/drm/i915/i915_trace.h
index 2b5573fe3778..2561cc913e79 100644
--- a/drivers/gpu/drm/i915/i915_trace.h
+++ b/drivers/gpu/drm/i915/i915_trace.h
@@ -76,7 +76,7 @@ TRACE_EVENT(i915_vma_bind,
 			   __entry->offset = vma->node.start;
 			   __entry->size = vma->node.size;
 			   __entry->mappable = mappable;
-			   __entry->big = !!vma->big_page_promoted;
+			   __entry->big = vma->big_page != NULL;
 			   ),
 
 	    TP_printk("obj=%p, offset=%08x size=%x%s%s vm=%p",