summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorBen Widawsky <benjamin.widawsky@intel.com>2014-04-09 20:42:57 -0700
committerBen Widawsky <benjamin.widawsky@intel.com>2014-06-20 17:05:03 -0700
commit9e3a67141393c0ff35fbc84ada19154d3f969cd1 (patch)
tree3ec4a068580f3f7c421752f3dae742c308f133fa
parente3a24b308bc63f60c8c6e878df1bb6b0d7f7d607 (diff)
drm/i915/bdw: implement alloc/teardown for 4lvl
The code for 4lvl works just as one would expect, and nicely it is able to call into the existing 3lvl page table code to handle all of the lower levels. PML4 has no special attributes. Signed-off-by: Ben Widawsky <ben@bwidawsk.net>
-rw-r--r--drivers/gpu/drm/i915/i915_gem_gtt.c170
-rw-r--r--drivers/gpu/drm/i915/i915_gem_gtt.h12
2 files changed, 163 insertions, 19 deletions
diff --git a/drivers/gpu/drm/i915/i915_gem_gtt.c b/drivers/gpu/drm/i915/i915_gem_gtt.c
index b46c0334060f..ed2b1ffb6df1 100644
--- a/drivers/gpu/drm/i915/i915_gem_gtt.c
+++ b/drivers/gpu/drm/i915/i915_gem_gtt.c
@@ -420,9 +420,12 @@ static void __pdp_fini(struct i915_pagedirpo *pdp)
static void free_pdp_single(struct i915_pagedirpo *pdp,
struct drm_device *dev)
{
- __pdp_fini(pdp);
- if (HAS_48B_PPGTT(dev))
+ if (HAS_48B_PPGTT(dev)) {
+ __pdp_fini(pdp);
+ i915_dma_unmap_single(pdp, dev);
+ __free_page(pdp->page);
kfree(pdp);
+ }
}
static int __pdp_init(struct i915_pagedirpo *pdp,
@@ -448,6 +451,58 @@ static int __pdp_init(struct i915_pagedirpo *pdp,
return 0;
}
+static struct i915_pagedirpo *alloc_pdp_single(struct i915_hw_ppgtt *ppgtt,
+ struct i915_pml4 *pml4)
+{
+ struct drm_device *dev = ppgtt->base.dev;
+ struct i915_pagedirpo *pdp;
+ int ret;
+
+ BUG_ON(!HAS_48B_PPGTT(dev));
+
+ pdp = kmalloc(sizeof(*pdp), GFP_KERNEL);
+ if (!pdp)
+ return ERR_PTR(-ENOMEM);
+
+ pdp->page = alloc_page(GFP_KERNEL | GFP_DMA32 | __GFP_ZERO);
+ if (!pdp->page) {
+ kfree(pdp);
+ return ERR_PTR(-ENOMEM);
+ }
+
+ ret = __pdp_init(pdp, dev);
+ if (ret) {
+ __free_page(pdp->page);
+ kfree(pdp);
+ return ERR_PTR(ret);
+ }
+
+ i915_dma_map_px_single(pdp, dev);
+
+ return pdp;
+}
+
+static void pml4_fini(struct i915_pml4 *pml4)
+{
+ struct i915_hw_ppgtt *ppgtt =
+ container_of(pml4, struct i915_hw_ppgtt, pml4);
+ i915_dma_unmap_single(pml4, ppgtt->base.dev);
+ __free_page(pml4->page);
+}
+
+static int pml4_init(struct i915_hw_ppgtt *ppgtt)
+{
+ struct i915_pml4 *pml4 = &ppgtt->pml4;
+
+ pml4->page = alloc_page(GFP_KERNEL | __GFP_ZERO);
+ if (!pml4->page)
+ return -ENOMEM;
+
+ i915_dma_map_px_single(pml4, ppgtt->base.dev);
+
+ return 0;
+}
+
/* Broadwell Page Directory Pointer Descriptors */
static int gen8_write_pdp(struct intel_engine_cs *ring,
unsigned entry,
@@ -736,7 +791,14 @@ static void gen8_teardown_va_range_4lvl(struct i915_address_space *vm,
struct i915_pml4 *pml4,
uint64_t start, uint64_t length)
{
- BUG();
+ struct i915_pagedirpo *pdp;
+ uint64_t temp, pml4e;
+
+ gen8_for_each_pml4e(pdp, pml4, start, length, temp, pml4e) {
+ gen8_teardown_va_range_3lvl(vm, pdp, start, length);
+ if (bitmap_empty(pdp->used_pdpes, I915_PDPES_PER_PDP(vm->dev)))
+ clear_bit(pml4e, pml4->used_pml4es);
+ }
}
static void gen8_teardown_va_range(struct i915_address_space *vm,
@@ -745,10 +807,10 @@ static void gen8_teardown_va_range(struct i915_address_space *vm,
struct i915_hw_ppgtt *ppgtt =
container_of(vm, struct i915_hw_ppgtt, base);
- if (!HAS_48B_PPGTT(vm->dev))
- gen8_teardown_va_range_3lvl(vm, &ppgtt->pdp, start, length);
- else
+ if (HAS_48B_PPGTT(vm->dev))
gen8_teardown_va_range_4lvl(vm, &ppgtt->pml4, start, length);
+ else
+ gen8_teardown_va_range_3lvl(vm, &ppgtt->pdp, start, length);
}
static void gen8_ppgtt_free(struct i915_hw_ppgtt *ppgtt)
@@ -1028,12 +1090,76 @@ err_out:
return ret;
}
-static int __noreturn gen8_alloc_va_range_4lvl(struct i915_address_space *vm,
- struct i915_pml4 *pml4,
- uint64_t start,
- uint64_t length)
+static int gen8_alloc_va_range_4lvl(struct i915_address_space *vm,
+ struct i915_pml4 *pml4,
+ uint64_t start,
+ uint64_t length)
{
- BUG();
+ DECLARE_BITMAP(new_pdps, GEN8_PML4ES_PER_PML4);
+ struct i915_hw_ppgtt *ppgtt =
+ container_of(vm, struct i915_hw_ppgtt, base);
+ struct i915_pagedirpo *pdp;
+ const uint64_t orig_start = start;
+ const uint64_t orig_length = length;
+ uint64_t temp, pml4e;
+
+ /* Do the pml4 allocations first, so we don't need to track the newly
+ * allocated tables below the pdp */
+ bitmap_zero(new_pdps, GEN8_PML4ES_PER_PML4);
+
+ /* The pagedirectory and pagetable allocations are done in the shared 3
+ * and 4 level code. Just allocate the pdps.
+ */
+ gen8_for_each_pml4e(pdp, pml4, start, length, temp, pml4e) {
+ if (!pdp) {
+ WARN_ON(test_bit(pml4e, pml4->used_pml4es));
+ pdp = alloc_pdp_single(ppgtt, pml4);
+ if (IS_ERR(pdp))
+ goto err_alloc;
+
+ pml4->pdps[pml4e] = pdp;
+ set_bit(pml4e, new_pdps);
+ trace_i915_pagedirpo_alloc(&ppgtt->base, pml4e,
+ pml4e << GEN8_PML4E_SHIFT,
+ GEN8_PML4E_SHIFT);
+
+ } else
+ WARN(!test_bit(pml4e, pml4->used_pml4es),
+ "%lld %p", pml4e, vm);
+ }
+
+ start = orig_start;
+ length = orig_length;
+
+ gen8_for_each_pml4e(pdp, pml4, start, length, temp, pml4e) {
+ int ret;
+
+ BUG_ON(!pdp);
+
+ ret = gen8_alloc_va_range_3lvl(vm, pdp, start, length);
+ if (ret)
+ goto err_out;
+ }
+
+ WARN(bitmap_weight(pml4->used_pml4es, GEN8_PML4ES_PER_PML4) > 2,
+ "The allocation has spanned more than 512GB. It is highly likely this is incorrect.");
+
+ bitmap_or(pml4->used_pml4es, new_pdps, pml4->used_pml4es,
+ GEN8_PML4ES_PER_PML4);
+
+ return 0;
+
+err_out:
+ /* This will teardown more than we allocated. It should be fine, and
+ * makes code simpler. */
+ start = orig_start;
+ length = orig_length;
+ gen8_for_each_pml4e(pdp, pml4, start, length, temp, pml4e)
+ gen8_teardown_va_range_3lvl(vm, pdp, start, length);
+
+err_alloc:
+ for_each_set_bit(pml4e, new_pdps, GEN8_PML4ES_PER_PML4)
+ free_pdp_single(pdp, vm->dev);
}
static int gen8_alloc_va_range(struct i915_address_space *vm,
@@ -1042,16 +1168,19 @@ static int gen8_alloc_va_range(struct i915_address_space *vm,
struct i915_hw_ppgtt *ppgtt =
container_of(vm, struct i915_hw_ppgtt, base);
- if (!HAS_48B_PPGTT(vm->dev))
- return gen8_alloc_va_range_3lvl(vm, &ppgtt->pdp, start, length);
- else
+ if (HAS_48B_PPGTT(vm->dev))
return gen8_alloc_va_range_4lvl(vm, &ppgtt->pml4, start, length);
+ else
+ return gen8_alloc_va_range_3lvl(vm, &ppgtt->pdp, start, length);
}
static void gen8_ppgtt_fini_common(struct i915_hw_ppgtt *ppgtt)
{
free_pt_scratch(ppgtt->scratch_pd, ppgtt->base.dev);
- free_pdp_single(&ppgtt->pdp, ppgtt->base.dev);
+ if (HAS_48B_PPGTT(ppgtt->base.dev))
+ pml4_fini(&ppgtt->pml4);
+ else
+ free_pdp_single(&ppgtt->pdp, ppgtt->base.dev);
}
/**
@@ -1073,7 +1202,13 @@ static int gen8_ppgtt_init_common(struct i915_hw_ppgtt *ppgtt, uint64_t size)
if (IS_ERR(ppgtt->scratch_pd))
return PTR_ERR(ppgtt->scratch_pd);
- if (!HAS_48B_PPGTT(ppgtt->base.dev)) {
+ if (HAS_48B_PPGTT(ppgtt->base.dev)) {
+ int ret = pml4_init(ppgtt);
+ if (ret) {
+ free_pt_scratch(ppgtt->scratch_pd, ppgtt->base.dev);
+ return ret;
+ }
+ } else {
int ret = __pdp_init(&ppgtt->pdp, false);
if (ret) {
free_pt_scratch(ppgtt->scratch_pd, ppgtt->base.dev);
@@ -1082,8 +1217,7 @@ static int gen8_ppgtt_init_common(struct i915_hw_ppgtt *ppgtt, uint64_t size)
ppgtt->switch_mm = gen8_mm_switch;
trace_i915_pagedirpo_alloc(&ppgtt->base, 0, 0, GEN8_PML4E_SHIFT);
- } else
- BUG(); /* Not yet implemented */
+ }
return 0;
}
diff --git a/drivers/gpu/drm/i915/i915_gem_gtt.h b/drivers/gpu/drm/i915/i915_gem_gtt.h
index 7dddf4355a5c..6978d7a8a76c 100644
--- a/drivers/gpu/drm/i915/i915_gem_gtt.h
+++ b/drivers/gpu/drm/i915/i915_gem_gtt.h
@@ -103,6 +103,7 @@ typedef gen8_gtt_pte_t gen8_ppgtt_pde_t;
#endif
#define GEN8_PML4ES_PER_PML4 512
#define GEN8_PML4E_SHIFT 39
+#define GEN8_PML4E_MASK (GEN8_PML4ES_PER_PML4 - 1)
#define GEN8_PDPE_SHIFT 30
/* NB: GEN8_PDPE_MASK is untrue for 32b platforms, but it has no impact on 32b page
* tables */
@@ -437,9 +438,18 @@ static inline size_t gen6_pde_count(uint32_t addr, uint32_t length)
temp = min(temp, length), \
start += temp, length -= temp)
+#define gen8_for_each_pml4e(pdp, pml4, start, length, temp, iter) \
+ for (iter = gen8_pml4e_index(start), pdp = (pml4)->pdps[iter]; \
+ length > 0 && iter < GEN8_PML4ES_PER_PML4; \
+ pdp = (pml4)->pdps[++iter], \
+ temp = ALIGN(start+1, 1ULL << GEN8_PML4E_SHIFT) - start, \
+ temp = min(temp, length), \
+ start += temp, length -= temp)
+
#define gen8_for_each_pdpe(pd, pdp, start, length, temp, iter) \
gen8_for_each_pdpe_e(pd, pdp, start, length, temp, iter, I915_PDPES_PER_PDP(dev))
+
/* Clamp length to the next pagetab boundary */
static inline uint64_t gen8_clamp_pt(uint64_t start, uint64_t length)
{
@@ -477,7 +487,7 @@ static inline uint32_t gen8_pdpe_index(uint64_t address)
static inline uint32_t gen8_pml4e_index(uint64_t address)
{
- BUG();
+ return (address >> GEN8_PML4E_SHIFT) & GEN8_PML4E_MASK;
}
static inline size_t gen8_pte_count(uint64_t addr, uint64_t length)