summaryrefslogtreecommitdiff
path: root/drivers/gpu/drm/i915/i915_gem_gtt.c
diff options
context:
space:
mode:
Diffstat (limited to 'drivers/gpu/drm/i915/i915_gem_gtt.c')
-rw-r--r--drivers/gpu/drm/i915/i915_gem_gtt.c202
1 files changed, 157 insertions, 45 deletions
diff --git a/drivers/gpu/drm/i915/i915_gem_gtt.c b/drivers/gpu/drm/i915/i915_gem_gtt.c
index 01b4b0c61eeb..6d384f68b6f3 100644
--- a/drivers/gpu/drm/i915/i915_gem_gtt.c
+++ b/drivers/gpu/drm/i915/i915_gem_gtt.c
@@ -72,10 +72,9 @@ static int sanitize_enable_ppgtt(struct drm_device *dev, int enable_ppgtt)
return HAS_ALIASING_PPGTT(dev) ? 1 : 0;
}
-
-static void ppgtt_bind_vma(struct i915_vma *vma,
- enum i915_cache_level cache_level,
- u32 flags);
+static int ppgtt_bind_vma(struct i915_vma *vma,
+ enum i915_cache_level cache_level,
+ u32 flags);
static void ppgtt_unbind_vma(struct i915_vma *vma);
static int gen8_ppgtt_enable(struct i915_hw_ppgtt *ppgtt);
@@ -239,37 +238,78 @@ static gen6_gtt_pte_t iris_pte_encode(dma_addr_t addr,
(px)->page, 0, 4096, \
PCI_DMA_BIDIRECTIONAL))
-static void free_pt_single(struct i915_pagetab *pt, struct drm_device *dev)
+static void __free_pt_single(struct i915_pagetab *pt, struct drm_device *dev,
+ int scratch)
{
+ if (WARN(scratch ^ pt->scratch,
+ "Tried to free scratch = %d. Is scratch = %d\n",
+ scratch, pt->scratch))
+ return;
+
if (WARN_ON(!pt->page))
return;
+ if (!scratch) {
+ const size_t count = INTEL_INFO(dev)->gen >= 8 ?
+ GEN8_PTES_PER_PT : GEN6_PTES_PER_PT;
+ WARN(!bitmap_empty(pt->used_ptes, count),
+ "Free page table with %d used pages\n",
+ bitmap_weight(pt->used_ptes, count));
+ }
+
i915_dma_unmap_single(pt, dev);
__free_page(pt->page);
+ kfree(pt->used_ptes);
kfree(pt);
}
+#define free_pt_single(pt, dev) \
+ __free_pt_single(pt, dev, false)
+#define free_pt_scratch(pt, dev) \
+ __free_pt_single(pt, dev, true)
+
static struct i915_pagetab *alloc_pt_single(struct drm_device *dev)
{
struct i915_pagetab *pt;
- int ret;
+ const size_t count = INTEL_INFO(dev)->gen >= 8 ?
+ GEN8_PTES_PER_PT : GEN6_PTES_PER_PT;
+ int ret = -ENOMEM;
pt = kzalloc(sizeof(*pt), GFP_KERNEL);
if (!pt)
return ERR_PTR(-ENOMEM);
+ pt->used_ptes = kcalloc(BITS_TO_LONGS(count), sizeof(*pt->used_ptes),
+ GFP_KERNEL);
+
+ if (!pt->used_ptes)
+ goto fail_bitmap;
+
pt->page = alloc_page(GFP_KERNEL | __GFP_ZERO);
- if (!pt->page) {
- kfree(pt);
- return ERR_PTR(-ENOMEM);
- }
+ if (!pt->page)
+ goto fail_page;
ret = i915_dma_map_px_single(pt, dev);
- if (ret) {
- __free_page(pt->page);
- kfree(pt);
- return ERR_PTR(ret);
- }
+ if (ret)
+ goto fail_dma;
+
+ return pt;
+
+fail_dma:
+ __free_page(pt->page);
+fail_page:
+ kfree(pt->used_ptes);
+fail_bitmap:
+ kfree(pt);
+
+ return ERR_PTR(ret);
+}
+
+static inline struct i915_pagetab *alloc_pt_scratch(struct drm_device *dev)
+{
+ struct i915_pagetab *pt = alloc_pt_single(dev);
+ if (!IS_ERR(pt))
+ pt->scratch = 1;
return pt;
}
@@ -396,7 +436,7 @@ static int gen8_mm_switch(struct i915_hw_ppgtt *ppgtt,
int used_pd = ppgtt->num_pd_entries / I915_PDES_PER_PD;
for (i = used_pd - 1; i >= 0; i--) {
- dma_addr_t addr = ppgtt->pdp.pagedir[i]->daddr;
+ dma_addr_t addr = ppgtt->pdp.pagedirs[i]->daddr;
ret = gen8_write_pdp(ring, i, addr, synchronous);
if (ret)
return ret;
@@ -423,7 +463,7 @@ static void gen8_ppgtt_clear_range(struct i915_address_space *vm,
I915_CACHE_LLC, use_scratch);
while (num_entries) {
- struct i915_pagedir *pd = ppgtt->pdp.pagedir[pdpe];
+ struct i915_pagedir *pd = ppgtt->pdp.pagedirs[pdpe];
struct i915_pagetab *pt = pd->page_tables[pde];
struct page *page_table = pt->page;
@@ -470,7 +510,7 @@ static void gen8_ppgtt_insert_entries(struct i915_address_space *vm,
break;
if (pt_vaddr == NULL) {
- struct i915_pagedir *pd = ppgtt->pdp.pagedir[pdpe];
+ struct i915_pagedir *pd = ppgtt->pdp.pagedirs[pdpe];
struct i915_pagetab *pt = pd->page_tables[pde];
struct page *page_table = pt->page;
pt_vaddr = kmap_atomic(page_table);
@@ -516,8 +556,8 @@ static void gen8_ppgtt_free(struct i915_hw_ppgtt *ppgtt)
int i;
for (i = 0; i < ppgtt->num_pd_pages; i++) {
- gen8_free_page_tables(ppgtt->pdp.pagedir[i], ppgtt->base.dev);
- free_pd_single(ppgtt->pdp.pagedir[i], ppgtt->base.dev);
+ gen8_free_page_tables(ppgtt->pdp.pagedirs[i], ppgtt->base.dev);
+ free_pd_single(ppgtt->pdp.pagedirs[i], ppgtt->base.dev);
}
}
@@ -537,7 +577,7 @@ static int gen8_ppgtt_allocate_page_tables(struct i915_hw_ppgtt *ppgtt)
int i, ret;
for (i = 0; i < ppgtt->num_pd_pages; i++) {
- ret = alloc_pt_range(ppgtt->pdp.pagedir[i],
+ ret = alloc_pt_range(ppgtt->pdp.pagedirs[i],
0, I915_PDES_PER_PD, ppgtt->base.dev);
if (ret)
goto unwind_out;
@@ -547,7 +587,7 @@ static int gen8_ppgtt_allocate_page_tables(struct i915_hw_ppgtt *ppgtt)
unwind_out:
while (i--)
- gen8_free_page_tables(ppgtt->pdp.pagedir[i], ppgtt->base.dev);
+ gen8_free_page_tables(ppgtt->pdp.pagedirs[i], ppgtt->base.dev);
return -ENOMEM;
}
@@ -558,8 +598,8 @@ static int gen8_ppgtt_allocate_page_directories(struct i915_hw_ppgtt *ppgtt,
int i;
for (i = 0; i < max_pdp; i++) {
- ppgtt->pdp.pagedir[i] = alloc_pd_single(ppgtt->base.dev);
- if (IS_ERR(ppgtt->pdp.pagedir[i]))
+ ppgtt->pdp.pagedirs[i] = alloc_pd_single(ppgtt->base.dev);
+ if (IS_ERR(ppgtt->pdp.pagedirs[i]))
goto unwind_out;
}
@@ -570,7 +610,7 @@ static int gen8_ppgtt_allocate_page_directories(struct i915_hw_ppgtt *ppgtt,
unwind_out:
while (i--)
- free_pd_single(ppgtt->pdp.pagedir[i],
+ free_pd_single(ppgtt->pdp.pagedirs[i],
ppgtt->base.dev);
return -ENOMEM;
@@ -632,9 +672,9 @@ static int gen8_ppgtt_init(struct i915_hw_ppgtt *ppgtt, uint64_t size)
* will never need to touch the PDEs again.
*/
for (i = 0; i < max_pdp; i++) {
- struct i915_pagedir *pd = ppgtt->pdp.pagedir[i];
+ struct i915_pagedir *pd = ppgtt->pdp.pagedirs[i];
gen8_ppgtt_pde_t *pd_vaddr;
- pd_vaddr = kmap_atomic(ppgtt->pdp.pagedir[i]->page);
+ pd_vaddr = kmap_atomic(ppgtt->pdp.pagedirs[i]->page);
for (j = 0; j < I915_PDES_PER_PD; j++) {
struct i915_pagetab *pt = pd->page_tables[j];
dma_addr_t addr = pt->daddr;
@@ -733,15 +773,13 @@ static void gen6_map_single(struct i915_pagedir *pd,
/* Map all the page tables found in the ppgtt structure to incrementing page
* directories. */
static void gen6_map_page_range(struct drm_i915_private *dev_priv,
- struct i915_pagedir *pd, unsigned pde, size_t n)
+ struct i915_pagedir *pd, uint32_t start, uint32_t length)
{
- if (WARN_ON(pde + n > I915_PDES_PER_PD))
- n = I915_PDES_PER_PD - pde;
-
- n += pde;
+ struct i915_pagetab *pt;
+ uint32_t pde, temp;
- for (; pde < n; pde++)
- gen6_map_single(pd, pde, pd->page_tables[pde]);
+ gen6_for_each_pde(pt, pd, start, length, temp, pde)
+ gen6_map_single(pd, pde, pt);
/* Make sure write is complete before other code can use this page
* table. Also require for WC mapped PTEs */
@@ -1030,6 +1068,51 @@ static void gen6_ppgtt_insert_entries(struct i915_address_space *vm,
kunmap_atomic(pt_vaddr);
}
+static int gen6_alloc_va_range(struct i915_address_space *vm,
+ uint64_t start, uint64_t length)
+{
+ struct i915_hw_ppgtt *ppgtt =
+ container_of(vm, struct i915_hw_ppgtt, base);
+ struct i915_pagetab *pt;
+ uint32_t pde, temp;
+
+ gen6_for_each_pde(pt, &ppgtt->pd, start, length, temp, pde) {
+ int j;
+
+ DECLARE_BITMAP(tmp_bitmap, GEN6_PTES_PER_PT);
+ bitmap_zero(tmp_bitmap, GEN6_PTES_PER_PT);
+ bitmap_set(tmp_bitmap, gen6_pte_index(start),
+ gen6_pte_count(start, length));
+
+ /* TODO: To be done in the next patch. Map the page/insert
+ * entries here */
+ for_each_set_bit(j, tmp_bitmap, GEN6_PTES_PER_PT) {
+ if (test_bit(j, pt->used_ptes)) {
+ /* Check that we're changing cache levels */
+ }
+ }
+
+ bitmap_or(pt->used_ptes, pt->used_ptes, tmp_bitmap,
+ GEN6_PTES_PER_PT);
+ }
+
+ return 0;
+}
+
+static void gen6_teardown_va_range(struct i915_address_space *vm,
+ uint64_t start, uint64_t length)
+{
+ struct i915_hw_ppgtt *ppgtt =
+ container_of(vm, struct i915_hw_ppgtt, base);
+ struct i915_pagetab *pt;
+ uint32_t pde, temp;
+
+ gen6_for_each_pde(pt, &ppgtt->pd, start, length, temp, pde) {
+ bitmap_clear(pt->used_ptes, gen6_pte_index(start),
+ gen6_pte_count(start, length));
+ }
+}
+
static void gen6_ppgtt_free(struct i915_hw_ppgtt *ppgtt)
{
int i;
@@ -1037,6 +1120,7 @@ static void gen6_ppgtt_free(struct i915_hw_ppgtt *ppgtt)
for (i = 0; i < ppgtt->num_pd_entries; i++)
free_pt_single(ppgtt->pd.page_tables[i], ppgtt->base.dev);
+ free_pt_scratch(ppgtt->scratch_pt, ppgtt->base.dev);
free_pd_single(&ppgtt->pd, ppgtt->base.dev);
}
@@ -1064,6 +1148,9 @@ static int gen6_ppgtt_allocate_page_directories(struct i915_hw_ppgtt *ppgtt)
* size. We allocate at the top of the GTT to avoid fragmentation.
*/
BUG_ON(!drm_mm_initialized(&dev_priv->gtt.base.mm));
+ ppgtt->scratch_pt = alloc_pt_scratch(ppgtt->base.dev);
+ if (IS_ERR(ppgtt->scratch_pt))
+ return PTR_ERR(ppgtt->scratch_pt);
alloc:
ret = drm_mm_insert_node_in_range_generic(&dev_priv->gtt.base.mm,
&ppgtt->node, GEN6_PD_SIZE,
@@ -1077,20 +1164,25 @@ alloc:
0, dev_priv->gtt.base.total,
0);
if (ret)
- return ret;
+ goto err_out;
retried = true;
goto alloc;
}
if (ret)
- return ret;
+ goto err_out;
+
if (ppgtt->node.start < dev_priv->gtt.mappable_end)
DRM_DEBUG("Forced to use aperture for PDEs\n");
ppgtt->num_pd_entries = I915_PDES_PER_PD;
return 0;
+
+err_out:
+ free_pt_scratch(ppgtt->scratch_pt, ppgtt->base.dev);
+ return ret;
}
static int gen6_ppgtt_alloc(struct i915_hw_ppgtt *ppgtt)
@@ -1135,6 +1227,8 @@ static int gen6_ppgtt_init(struct i915_hw_ppgtt *ppgtt)
if (ret)
return ret;
+ ppgtt->base.allocate_va_range = gen6_alloc_va_range;
+ ppgtt->base.teardown_va_range = gen6_teardown_va_range;
ppgtt->base.clear_range = gen6_ppgtt_clear_range;
ppgtt->base.insert_entries = gen6_ppgtt_insert_entries;
ppgtt->base.cleanup = gen6_ppgtt_cleanup;
@@ -1148,7 +1242,7 @@ static int gen6_ppgtt_init(struct i915_hw_ppgtt *ppgtt)
ppgtt->pd_addr = (gen6_gtt_pte_t __iomem*)dev_priv->gtt.gsm +
ppgtt->pd.pd_offset / sizeof(gen6_gtt_pte_t);
- gen6_map_page_range(dev_priv, &ppgtt->pd, 0, ppgtt->num_pd_entries);
+ gen6_map_page_range(dev_priv, &ppgtt->pd, 0, ppgtt->base.total);
DRM_DEBUG_DRIVER("Allocated pde space (%ldM) at GTT entry: %lx\n",
ppgtt->node.size >> 20,
@@ -1183,17 +1277,28 @@ int i915_gem_init_ppgtt(struct drm_device *dev, struct i915_hw_ppgtt *ppgtt)
return 0;
}
-static void
+static int
ppgtt_bind_vma(struct i915_vma *vma,
enum i915_cache_level cache_level,
u32 flags)
{
+ int ret;
+
/* Currently applicable only to VLV */
if (vma->obj->gt_ro)
flags |= PTE_READ_ONLY;
+ if (vma->vm->allocate_va_range) {
+ ret = vma->vm->allocate_va_range(vma->vm,
+ vma->node.start,
+ vma->node.size);
+ if (ret)
+ return ret;
+ }
+
vma->vm->insert_entries(vma->vm, vma->obj->pages, vma->node.start,
cache_level, flags);
+ return 0;
}
static void ppgtt_unbind_vma(struct i915_vma *vma)
@@ -1202,6 +1307,9 @@ static void ppgtt_unbind_vma(struct i915_vma *vma)
vma->node.start,
vma->obj->base.size,
true);
+ if (vma->vm->teardown_va_range)
+ vma->vm->teardown_va_range(vma->vm,
+ vma->node.start, vma->node.size);
}
extern int intel_iommu_gfx_mapped;
@@ -1509,9 +1617,9 @@ static void gen6_ggtt_clear_range(struct i915_address_space *vm,
}
-static void i915_ggtt_bind_vma(struct i915_vma *vma,
- enum i915_cache_level cache_level,
- u32 unused)
+static int i915_ggtt_bind_vma(struct i915_vma *vma,
+ enum i915_cache_level cache_level,
+ u32 unused)
{
const unsigned long entry = vma->node.start >> PAGE_SHIFT;
unsigned int flags = (cache_level == I915_CACHE_NONE) ?
@@ -1520,6 +1628,8 @@ static void i915_ggtt_bind_vma(struct i915_vma *vma,
BUG_ON(!i915_is_ggtt(vma->vm));
intel_gtt_insert_sg_entries(vma->obj->pages, entry, flags);
vma->obj->has_global_gtt_mapping = 1;
+
+ return 0;
}
static void i915_ggtt_clear_range(struct i915_address_space *vm,
@@ -1542,9 +1652,9 @@ static void i915_ggtt_unbind_vma(struct i915_vma *vma)
intel_gtt_clear_range(first, size);
}
-static void ggtt_bind_vma(struct i915_vma *vma,
- enum i915_cache_level cache_level,
- u32 flags)
+static int ggtt_bind_vma(struct i915_vma *vma,
+ enum i915_cache_level cache_level,
+ u32 flags)
{
struct drm_device *dev = vma->vm->dev;
struct drm_i915_private *dev_priv = dev->dev_private;
@@ -1576,7 +1686,7 @@ static void ggtt_bind_vma(struct i915_vma *vma,
}
if (!(flags & ALIASING_BIND))
- return;
+ return 0;
if (dev_priv->mm.aliasing_ppgtt &&
(!obj->has_aliasing_ppgtt_mapping ||
@@ -1588,6 +1698,8 @@ static void ggtt_bind_vma(struct i915_vma *vma,
cache_level, flags);
vma->obj->has_aliasing_ppgtt_mapping = 1;
}
+
+ return 0;
}
static void ggtt_unbind_vma(struct i915_vma *vma)