pipelined-fencingfor-jiri

6 files changed, 277 insertions, 243 deletions

diff --git a/drivers/gpu/drm/i915/i915_drv.h b/drivers/gpu/drm/i915/i915_drv.h
index f1acfef..077f38a 100644
--- a/drivers/gpu/drm/i915/i915_drv.h
+++ b/drivers/gpu/drm/i915/i915_drv.h
@@ -135,7 +135,10 @@ struct drm_i915_master_private {
 struct drm_i915_fence_reg {
 	struct list_head lru_list;
 	struct drm_i915_gem_object *obj;
+
 	uint32_t setup_seqno;
+	struct intel_ring_buffer *setup_ring;
+
 	int pin_count;
 };
 
@@ -909,11 +912,8 @@ struct drm_i915_gem_object {
 
 	/** Breadcrumb of last rendering to the buffer. */
 	uint32_t last_rendering_seqno;
-	struct intel_ring_buffer *ring;
-
-	/** Breadcrumb of last fenced GPU access to the buffer. */
 	uint32_t last_fenced_seqno;
-	struct intel_ring_buffer *last_fenced_ring;
+	struct intel_ring_buffer *ring;
 
 	/** Current tiling stride for the object, if it's tiled. */
 	uint32_t stride;
@@ -1230,7 +1230,8 @@ u32 i915_gem_next_request_seqno(struct intel_ring_buffer *ring);
 
 int __must_check i915_gem_object_get_fence(struct drm_i915_gem_object *obj,
 					   struct intel_ring_buffer *pipelined);
-int __must_check i915_gem_object_put_fence(struct drm_i915_gem_object *obj);
+int __must_check i915_gem_object_put_fence(struct drm_i915_gem_object *obj,
+					   struct intel_ring_buffer *pipelined);
 
 static inline void
 i915_gem_object_pin_fence(struct drm_i915_gem_object *obj)
diff --git a/drivers/gpu/drm/i915/i915_gem.c b/drivers/gpu/drm/i915/i915_gem.c
index d6700e1..00a6e0f 100644
--- a/drivers/gpu/drm/i915/i915_gem.c
+++ b/drivers/gpu/drm/i915/i915_gem.c
@@ -698,7 +698,7 @@ i915_gem_gtt_pwrite_slow(struct drm_device *dev,
 	if (ret)
 		goto out_unpin_pages;
 
-	ret = i915_gem_object_put_fence(obj);
+	ret = i915_gem_object_put_fence(obj, NULL);
 	if (ret)
 		goto out_unpin_pages;
 
@@ -969,7 +969,7 @@ i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
 		if (ret)
 			goto out_unpin;
 
-		ret = i915_gem_object_put_fence(obj);
+		ret = i915_gem_object_put_fence(obj, NULL);
 		if (ret)
 			goto out_unpin;
 
@@ -1186,7 +1186,7 @@ int i915_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
 	}
 
 	if (obj->tiling_mode == I915_TILING_NONE)
-		ret = i915_gem_object_put_fence(obj);
+		ret = i915_gem_object_put_fence(obj, NULL);
 	else
 		ret = i915_gem_object_get_fence(obj, NULL);
 	if (ret)
@@ -1652,17 +1652,8 @@ i915_gem_object_move_to_active(struct drm_i915_gem_object *obj,
 	list_move_tail(&obj->ring_list, &ring->active_list);
 
 	obj->last_rendering_seqno = seqno;
-	if (obj->fenced_gpu_access) {
-		struct drm_i915_fence_reg *reg;
-
-		BUG_ON(obj->fence_reg == I915_FENCE_REG_NONE);
-
+	if (obj->fenced_gpu_access)
 		obj->last_fenced_seqno = seqno;
-		obj->last_fenced_ring = ring;
-
-		reg = &dev_priv->fence_regs[obj->fence_reg];
-		list_move_tail(&reg->lru_list, &dev_priv->mm.fence_list);
-	}
 }
 
 static void
@@ -1698,6 +1689,7 @@ i915_gem_object_move_to_inactive(struct drm_i915_gem_object *obj)
 
 	i915_gem_object_move_off_active(obj);
 	obj->fenced_gpu_access = false;
+	obj->last_fenced_seqno = 0;
 
 	obj->active = 0;
 	obj->pending_gpu_write = false;
@@ -1872,8 +1864,6 @@ static void i915_gem_reset_fences(struct drm_device *dev)
 
 		reg->obj->fence_reg = I915_FENCE_REG_NONE;
 		reg->obj->fenced_gpu_access = false;
-		reg->obj->last_fenced_seqno = 0;
-		reg->obj->last_fenced_ring = NULL;
 		i915_gem_clear_fence_reg(dev, reg);
 	}
 }
@@ -2078,6 +2068,12 @@ i915_wait_request(struct intel_ring_buffer *ring,
 	if (seqno == ring->outstanding_lazy_request) {
 		struct drm_i915_gem_request *request;
 
+		if (ring->need_fence_flush) {
+			ret = i915_gem_flush_ring(ring, 0, I915_GEM_GPU_DOMAINS);
+			if (ret)
+				return ret;
+		}
+
 		request = kzalloc(sizeof(*request), GFP_KERNEL);
 		if (request == NULL)
 			return -ENOMEM;
@@ -2216,7 +2212,7 @@ i915_gem_object_unbind(struct drm_i915_gem_object *obj)
 	i915_gem_object_finish_gtt(obj);
 
 	/* release the fence reg _after_ flushing */
-	ret = i915_gem_object_put_fence(obj);
+	ret = i915_gem_object_put_fence(obj, NULL);
 	if (ret && ret != -EIO)
 		return ret;
 
@@ -2263,6 +2259,8 @@ i915_gem_flush_ring(struct intel_ring_buffer *ring,
 	if (flush_domains & I915_GEM_GPU_DOMAINS)
 		i915_gem_process_flushing_list(ring, flush_domains);
 
+	ring->need_fence_flush = false;
+
 	return 0;
 }
 
@@ -2299,24 +2297,28 @@ int i915_gpu_idle(struct drm_device *dev, bool do_retire)
 	return 0;
 }
 
-static int sandybridge_write_fence_reg(struct drm_i915_gem_object *obj,
-				       struct intel_ring_buffer *pipelined)
+static int sandybridge_write_fence_reg(struct drm_i915_gem_object *obj, int reg,
+				       struct intel_ring_buffer *pipelined,
+				       bool enable)
 {
 	struct drm_device *dev = obj->base.dev;
 	drm_i915_private_t *dev_priv = dev->dev_private;
-	u32 size = obj->gtt_space->size;
-	int regnum = obj->fence_reg;
 	uint64_t val;
 
-	val = (uint64_t)((obj->gtt_offset + size - 4096) &
-			 0xfffff000) << 32;
-	val |= obj->gtt_offset & 0xfffff000;
-	val |= (uint64_t)((obj->stride / 128) - 1) <<
-		SANDYBRIDGE_FENCE_PITCH_SHIFT;
+	if (enable) {
+		u32 size = obj->gtt_space->size;
 
-	if (obj->tiling_mode == I915_TILING_Y)
-		val |= 1 << I965_FENCE_TILING_Y_SHIFT;
-	val |= I965_FENCE_REG_VALID;
+		val = (uint64_t)((obj->gtt_offset + size - 4096) &
+				 0xfffff000) << 32;
+		val |= obj->gtt_offset & 0xfffff000;
+		val |= (uint64_t)((obj->stride / 128) - 1) <<
+			SANDYBRIDGE_FENCE_PITCH_SHIFT;
+
+		if (obj->tiling_mode == I915_TILING_Y)
+			val |= 1 << I965_FENCE_TILING_Y_SHIFT;
+		val |= I965_FENCE_REG_VALID;
+	} else
+		val = 0;
 
 	if (pipelined) {
 		int ret = intel_ring_begin(pipelined, 6);
@@ -2325,33 +2327,39 @@ static int sandybridge_write_fence_reg(struct drm_i915_gem_object *obj,
 
 		intel_ring_emit(pipelined, MI_NOOP);
 		intel_ring_emit(pipelined, MI_LOAD_REGISTER_IMM(2));
-		intel_ring_emit(pipelined, FENCE_REG_SANDYBRIDGE_0 + regnum*8);
+		intel_ring_emit(pipelined, FENCE_REG_SANDYBRIDGE_0 + reg*8);
 		intel_ring_emit(pipelined, (u32)val);
-		intel_ring_emit(pipelined, FENCE_REG_SANDYBRIDGE_0 + regnum*8 + 4);
+		intel_ring_emit(pipelined, FENCE_REG_SANDYBRIDGE_0 + reg*8 + 4);
 		intel_ring_emit(pipelined, (u32)(val >> 32));
 		intel_ring_advance(pipelined);
-	} else
-		I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + regnum * 8, val);
+	} else {
+		I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + reg * 8, val);
+		POSTING_READ(FENCE_REG_SANDYBRIDGE_0 + reg * 8);
+	}
 
 	return 0;
 }
 
-static int i965_write_fence_reg(struct drm_i915_gem_object *obj,
-				struct intel_ring_buffer *pipelined)
+static int i965_write_fence_reg(struct drm_i915_gem_object *obj, int reg,
+				struct intel_ring_buffer *pipelined,
+				bool enable)
 {
 	struct drm_device *dev = obj->base.dev;
 	drm_i915_private_t *dev_priv = dev->dev_private;
-	u32 size = obj->gtt_space->size;
-	int regnum = obj->fence_reg;
 	uint64_t val;
 
-	val = (uint64_t)((obj->gtt_offset + size - 4096) &
-		    0xfffff000) << 32;
-	val |= obj->gtt_offset & 0xfffff000;
-	val |= ((obj->stride / 128) - 1) << I965_FENCE_PITCH_SHIFT;
-	if (obj->tiling_mode == I915_TILING_Y)
-		val |= 1 << I965_FENCE_TILING_Y_SHIFT;
-	val |= I965_FENCE_REG_VALID;
+	if (enable) {
+		u32 size = obj->gtt_space->size;
+
+		val = (uint64_t)((obj->gtt_offset + size - 4096) &
+				 0xfffff000) << 32;
+		val |= obj->gtt_offset & 0xfffff000;
+		val |= ((obj->stride / 128) - 1) << I965_FENCE_PITCH_SHIFT;
+		if (obj->tiling_mode == I915_TILING_Y)
+			val |= 1 << I965_FENCE_TILING_Y_SHIFT;
+		val |= I965_FENCE_REG_VALID;
+	} else
+		val = 0;
 
 	if (pipelined) {
 		int ret = intel_ring_begin(pipelined, 6);
@@ -2360,50 +2368,58 @@ static int i965_write_fence_reg(struct drm_i915_gem_object *obj,
 
 		intel_ring_emit(pipelined, MI_NOOP);
 		intel_ring_emit(pipelined, MI_LOAD_REGISTER_IMM(2));
-		intel_ring_emit(pipelined, FENCE_REG_965_0 + regnum*8);
+		intel_ring_emit(pipelined, FENCE_REG_965_0 + reg*8);
 		intel_ring_emit(pipelined, (u32)val);
-		intel_ring_emit(pipelined, FENCE_REG_965_0 + regnum*8 + 4);
+		intel_ring_emit(pipelined, FENCE_REG_965_0 + reg*8 + 4);
 		intel_ring_emit(pipelined, (u32)(val >> 32));
 		intel_ring_advance(pipelined);
-	} else
-		I915_WRITE64(FENCE_REG_965_0 + regnum * 8, val);
+	} else {
+		I915_WRITE64(FENCE_REG_965_0 + reg * 8, val);
+		POSTING_READ(FENCE_REG_965_0 + reg * 8);
+	}
 
 	return 0;
 }
 
-static int i915_write_fence_reg(struct drm_i915_gem_object *obj,
-				struct intel_ring_buffer *pipelined)
+static int i915_write_fence_reg(struct drm_i915_gem_object *obj, int reg,
+				struct intel_ring_buffer *pipelined,
+				bool enable)
 {
 	struct drm_device *dev = obj->base.dev;
 	drm_i915_private_t *dev_priv = dev->dev_private;
-	u32 size = obj->gtt_space->size;
-	u32 fence_reg, val, pitch_val;
-	int tile_width;
-
-	if (WARN((obj->gtt_offset & ~I915_FENCE_START_MASK) ||
-		 (size & -size) != size ||
-		 (obj->gtt_offset & (size - 1)),
-		 "object 0x%08x [fenceable? %d] not 1M or pot-size (0x%08x) aligned\n",
-		 obj->gtt_offset, obj->map_and_fenceable, size))
-		return -EINVAL;
-
-	if (obj->tiling_mode == I915_TILING_Y && HAS_128_BYTE_Y_TILING(dev))
-		tile_width = 128;
-	else
-		tile_width = 512;
-
-	/* Note: pitch better be a power of two tile widths */
-	pitch_val = obj->stride / tile_width;
-	pitch_val = ffs(pitch_val) - 1;
-
-	val = obj->gtt_offset;
-	if (obj->tiling_mode == I915_TILING_Y)
-		val |= 1 << I830_FENCE_TILING_Y_SHIFT;
-	val |= I915_FENCE_SIZE_BITS(size);
-	val |= pitch_val << I830_FENCE_PITCH_SHIFT;
-	val |= I830_FENCE_REG_VALID;
+	u32 fence_reg, val;
+
+	if (enable) {
+		u32 size = obj->gtt_space->size;
+		int pitch_val;
+		int tile_width;
+
+		if (WARN((obj->gtt_offset & ~I915_FENCE_START_MASK) ||
+			 (size & -size) != size ||
+			 (obj->gtt_offset & (size - 1)),
+			 "object 0x%08x [fenceable? %d] not 1M or pot-size (0x%08x) aligned\n",
+			 obj->gtt_offset, obj->map_and_fenceable, size))
+			return -EINVAL;
+
+		if (obj->tiling_mode == I915_TILING_Y && HAS_128_BYTE_Y_TILING(dev))
+			tile_width = 128;
+		else
+			tile_width = 512;
+
+		/* Note: pitch better be a power of two tile widths */
+		pitch_val = obj->stride / tile_width;
+		pitch_val = ffs(pitch_val) - 1;
+
+		val = obj->gtt_offset;
+		if (obj->tiling_mode == I915_TILING_Y)
+			val |= 1 << I830_FENCE_TILING_Y_SHIFT;
+		val |= I915_FENCE_SIZE_BITS(size);
+		val |= pitch_val << I830_FENCE_PITCH_SHIFT;
+		val |= I830_FENCE_REG_VALID;
+	} else
+		val = 0;
 
-	fence_reg = obj->fence_reg;
+	fence_reg = reg;
 	if (fence_reg < 8)
 		fence_reg = FENCE_REG_830_0 + fence_reg * 4;
 	else
@@ -2419,38 +2435,44 @@ static int i915_write_fence_reg(struct drm_i915_gem_object *obj,
 		intel_ring_emit(pipelined, fence_reg);
 		intel_ring_emit(pipelined, val);
 		intel_ring_advance(pipelined);
-	} else
+	} else {
 		I915_WRITE(fence_reg, val);
+		POSTING_READ(fence_reg);
+	}
 
 	return 0;
 }
 
-static int i830_write_fence_reg(struct drm_i915_gem_object *obj,
-				struct intel_ring_buffer *pipelined)
+static int i830_write_fence_reg(struct drm_i915_gem_object *obj, int reg,
+				struct intel_ring_buffer *pipelined,
+				bool enable)
 {
 	struct drm_device *dev = obj->base.dev;
 	drm_i915_private_t *dev_priv = dev->dev_private;
-	u32 size = obj->gtt_space->size;
-	int regnum = obj->fence_reg;
 	uint32_t val;
-	uint32_t pitch_val;
 
-	if (WARN((obj->gtt_offset & ~I830_FENCE_START_MASK) ||
-		 (size & -size) != size ||
-		 (obj->gtt_offset & (size - 1)),
-		 "object 0x%08x not 512K or pot-size 0x%08x aligned\n",
-		 obj->gtt_offset, size))
-		return -EINVAL;
-
-	pitch_val = obj->stride / 128;
-	pitch_val = ffs(pitch_val) - 1;
-
-	val = obj->gtt_offset;
-	if (obj->tiling_mode == I915_TILING_Y)
-		val |= 1 << I830_FENCE_TILING_Y_SHIFT;
-	val |= I830_FENCE_SIZE_BITS(size);
-	val |= pitch_val << I830_FENCE_PITCH_SHIFT;
-	val |= I830_FENCE_REG_VALID;
+	if (enable) {
+		u32 size = obj->gtt_space->size;
+		uint32_t pitch_val;
+
+		if (WARN((obj->gtt_offset & ~I830_FENCE_START_MASK) ||
+			 (size & -size) != size ||
+			 (obj->gtt_offset & (size - 1)),
+			 "object 0x%08x not 512K or pot-size 0x%08x aligned\n",
+			 obj->gtt_offset, size))
+			return -EINVAL;
+
+		pitch_val = obj->stride / 128;
+		pitch_val = ffs(pitch_val) - 1;
+
+		val = obj->gtt_offset;
+		if (obj->tiling_mode == I915_TILING_Y)
+			val |= 1 << I830_FENCE_TILING_Y_SHIFT;
+		val |= I830_FENCE_SIZE_BITS(size);
+		val |= pitch_val << I830_FENCE_PITCH_SHIFT;
+		val |= I830_FENCE_REG_VALID;
+	} else
+		val = 0;
 
 	if (pipelined) {
 		int ret = intel_ring_begin(pipelined, 4);
@@ -2459,18 +2481,62 @@ static int i830_write_fence_reg(struct drm_i915_gem_object *obj,
 
 		intel_ring_emit(pipelined, MI_NOOP);
 		intel_ring_emit(pipelined, MI_LOAD_REGISTER_IMM(1));
-		intel_ring_emit(pipelined, FENCE_REG_830_0 + regnum*4);
+		intel_ring_emit(pipelined, FENCE_REG_830_0 + reg*4);
 		intel_ring_emit(pipelined, val);
 		intel_ring_advance(pipelined);
-	} else
-		I915_WRITE(FENCE_REG_830_0 + regnum * 4, val);
+	} else {
+		I915_WRITE(FENCE_REG_830_0 + reg * 4, val);
+		POSTING_READ(FENCE_REG_830_0 + reg * 4);
+	}
 
 	return 0;
 }
 
-static bool ring_passed_seqno(struct intel_ring_buffer *ring, u32 seqno)
+static int intel_write_fence_reg(struct drm_i915_gem_object *obj,
+				 int fence_reg,
+				 struct intel_ring_buffer *pipelined,
+				 bool enable)
 {
-	return i915_seqno_passed(ring->get_seqno(ring), seqno);
+	int ret;
+
+	mb();
+
+	switch (INTEL_INFO(obj->base.dev)->gen) {
+	case 7:
+	case 6:
+		ret = sandybridge_write_fence_reg(obj, fence_reg,
+						  pipelined, enable);
+		break;
+
+	case 5:
+	case 4:
+		ret = i965_write_fence_reg(obj, fence_reg, pipelined, enable);
+		break;
+
+	case 3:
+		ret = i915_write_fence_reg(obj, fence_reg, pipelined, enable);
+		break;
+
+	case 2:
+		ret = i830_write_fence_reg(obj, fence_reg, pipelined, enable);
+		break;
+
+	default:
+		ret= -ENODEV;
+		break;
+	}
+	if (ret)
+		return ret;
+
+	if (pipelined) {
+		struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
+		struct drm_i915_fence_reg *reg = &dev_priv->fence_regs[fence_reg];
+		reg->setup_ring = pipelined;
+		reg->setup_seqno = i915_gem_next_request_seqno(pipelined);
+		pipelined->need_fence_flush = true;
+	}
+
+	return 0;
 }
 
 static int
@@ -2481,7 +2547,7 @@ i915_gem_object_flush_fence(struct drm_i915_gem_object *obj,
 
 	if (obj->fenced_gpu_access) {
 		if (obj->base.write_domain & I915_GEM_GPU_DOMAINS) {
-			ret = i915_gem_flush_ring(obj->last_fenced_ring,
+			ret = i915_gem_flush_ring(obj->ring,
 						  0, obj->base.write_domain);
 			if (ret)
 				return ret;
@@ -2490,50 +2556,65 @@ i915_gem_object_flush_fence(struct drm_i915_gem_object *obj,
 		obj->fenced_gpu_access = false;
 	}
 
-	if (obj->last_fenced_seqno && pipelined != obj->last_fenced_ring) {
-		if (!ring_passed_seqno(obj->last_fenced_ring,
-				       obj->last_fenced_seqno)) {
-			ret = i915_wait_request(obj->last_fenced_ring,
-						obj->last_fenced_seqno,
+	if (obj->ring != pipelined && obj->last_fenced_seqno) {
+		ret = i915_wait_request(obj->ring,
+					obj->last_fenced_seqno,
+					true);
+		if (ret)
+			return ret;
+
+		obj->last_fenced_seqno = 0;
+	}
+
+	if (obj->fence_reg != I915_FENCE_REG_NONE) {
+		struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
+		struct drm_i915_fence_reg *reg;
+
+		reg = &dev_priv->fence_regs[obj->fence_reg];
+		if (reg->setup_ring && pipelined != reg->setup_ring) {
+			ret = i915_wait_request(reg->setup_ring,
+						reg->setup_seqno,
 						true);
 			if (ret)
 				return ret;
-		}
 
-		obj->last_fenced_seqno = 0;
-		obj->last_fenced_ring = NULL;
+			reg->setup_seqno = 0;
+			reg->setup_ring = NULL;
+		}
 	}
 
-	/* Ensure that all CPU reads are completed before installing a fence
-	 * and all writes before removing the fence.
-	 */
-	if (obj->base.read_domains & I915_GEM_DOMAIN_GTT)
-		mb();
-
 	return 0;
 }
 
 int
-i915_gem_object_put_fence(struct drm_i915_gem_object *obj)
+i915_gem_object_put_fence(struct drm_i915_gem_object *obj,
+			  struct intel_ring_buffer *pipelined)
 {
+	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
+	struct drm_i915_fence_reg *reg;
 	int ret;
 
-	if (obj->tiling_mode)
-		i915_gem_release_mmap(obj);
-
-	ret = i915_gem_object_flush_fence(obj, NULL);
+	ret = i915_gem_object_flush_fence(obj, pipelined);
 	if (ret)
 		return ret;
 
-	if (obj->fence_reg != I915_FENCE_REG_NONE) {
-		struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
+	if (obj->fence_reg == I915_FENCE_REG_NONE)
+		return 0;
 
-		WARN_ON(dev_priv->fence_regs[obj->fence_reg].pin_count);
-		i915_gem_clear_fence_reg(obj->base.dev,
-					 &dev_priv->fence_regs[obj->fence_reg]);
+	reg = &dev_priv->fence_regs[obj->fence_reg];
+	if (!reg->setup_seqno && !obj->last_fenced_seqno)
+		pipelined = NULL;
 
-		obj->fence_reg = I915_FENCE_REG_NONE;
-	}
+	ret = intel_write_fence_reg(obj, obj->fence_reg, pipelined, false);
+	if (ret)
+		return ret;
+
+	reg->obj = NULL;
+	list_del_init(&reg->lru_list);
+
+	if (obj->tiling_mode)
+		i915_gem_release_mmap(obj);
+	obj->fence_reg = I915_FENCE_REG_NONE;
 
 	return 0;
 }
@@ -2547,16 +2628,22 @@ i915_find_fence_reg(struct drm_device *dev,
 	int i;
 
 	/* First try to find a free reg */
-	avail = NULL;
+	avail = first = NULL;
 	for (i = dev_priv->fence_reg_start; i < dev_priv->num_fence_regs; i++) {
 		reg = &dev_priv->fence_regs[i];
-		if (!reg->obj)
+		if (!reg->obj && reg->setup_ring == pipelined)
 			return reg;
 
+		if (!reg->obj && first == NULL)
+			first = reg;
+
 		if (!reg->pin_count)
 			avail = reg;
 	}
 
+	if (first != NULL)
+		return first;
+
 	if (avail == NULL)
 		return NULL;
 
@@ -2569,9 +2656,7 @@ i915_find_fence_reg(struct drm_device *dev,
 		if (first == NULL)
 			first = reg;
 
-		if (!pipelined ||
-		    !reg->obj->last_fenced_ring ||
-		    reg->obj->last_fenced_ring == pipelined) {
+		if (!reg->obj->ring || reg->obj->ring == pipelined) {
 			avail = reg;
 			break;
 		}
@@ -2605,119 +2690,62 @@ i915_gem_object_get_fence(struct drm_i915_gem_object *obj,
 	struct drm_device *dev = obj->base.dev;
 	struct drm_i915_private *dev_priv = dev->dev_private;
 	struct drm_i915_fence_reg *reg;
+	struct drm_i915_gem_object *old = NULL;
+	bool update = true;
+	int fence_reg;
 	int ret;
 
-	/* XXX disable pipelining. There are bugs. Shocking. */
-	pipelined = NULL;
+	pipelined = NULL; /* XXX the CS appears incoherent with LOAD_REG */
 
-	/* Just update our place in the LRU if our fence is getting reused. */
 	if (obj->fence_reg != I915_FENCE_REG_NONE) {
 		reg = &dev_priv->fence_regs[obj->fence_reg];
 		list_move_tail(&reg->lru_list, &dev_priv->mm.fence_list);
 
-		if (obj->tiling_changed) {
-			ret = i915_gem_object_flush_fence(obj, pipelined);
-			if (ret)
-				return ret;
-
-			if (!obj->fenced_gpu_access && !obj->last_fenced_seqno)
-				pipelined = NULL;
-
-			if (pipelined) {
-				reg->setup_seqno =
-					i915_gem_next_request_seqno(pipelined);
-				obj->last_fenced_seqno = reg->setup_seqno;
-				obj->last_fenced_ring = pipelined;
-			}
+		update = obj->tiling_changed;
+	} else {
+		reg = i915_find_fence_reg(dev, pipelined);
+		if (reg == NULL)
+			return -EDEADLK;
 
-			goto update;
-		}
+		if (reg->obj) {
+			old = reg->obj;
+			drm_gem_object_reference(&old->base);
 
-		if (!pipelined) {
-			if (reg->setup_seqno) {
-				if (!ring_passed_seqno(obj->last_fenced_ring,
-						       reg->setup_seqno)) {
-					ret = i915_wait_request(obj->last_fenced_ring,
-								reg->setup_seqno,
-								true);
-					if (ret)
-						return ret;
-				}
-
-				reg->setup_seqno = 0;
-			}
-		} else if (obj->last_fenced_ring &&
-			   obj->last_fenced_ring != pipelined) {
-			ret = i915_gem_object_flush_fence(obj, pipelined);
+			ret = i915_gem_object_flush_fence(old, pipelined);
 			if (ret)
-				return ret;
+				goto err;
 		}
-
-		return 0;
 	}
 
-	reg = i915_find_fence_reg(dev, pipelined);
-	if (reg == NULL)
-		return -EDEADLK;
-
 	ret = i915_gem_object_flush_fence(obj, pipelined);
 	if (ret)
-		return ret;
-
-	if (reg->obj) {
-		struct drm_i915_gem_object *old = reg->obj;
-
-		drm_gem_object_reference(&old->base);
+		goto err;
 
-		if (old->tiling_mode)
-			i915_gem_release_mmap(old);
-
-		ret = i915_gem_object_flush_fence(old, pipelined);
-		if (ret) {
-			drm_gem_object_unreference(&old->base);
-			return ret;
-		}
-
-		if (old->last_fenced_seqno == 0 && obj->last_fenced_seqno == 0)
+	if (update) {
+		if (!reg->setup_seqno &&
+		    !obj->last_fenced_seqno &&
+		    !(old && old->last_fenced_seqno))
 			pipelined = NULL;
 
-		old->fence_reg = I915_FENCE_REG_NONE;
-		old->last_fenced_ring = pipelined;
-		old->last_fenced_seqno =
-			pipelined ? i915_gem_next_request_seqno(pipelined) : 0;
-
-		drm_gem_object_unreference(&old->base);
-	} else if (obj->last_fenced_seqno == 0)
-		pipelined = NULL;
-
-	reg->obj = obj;
-	list_move_tail(&reg->lru_list, &dev_priv->mm.fence_list);
-	obj->fence_reg = reg - dev_priv->fence_regs;
-	obj->last_fenced_ring = pipelined;
-
-	reg->setup_seqno =
-		pipelined ? i915_gem_next_request_seqno(pipelined) : 0;
-	obj->last_fenced_seqno = reg->setup_seqno;
-
-update:
-	obj->tiling_changed = false;
-	switch (INTEL_INFO(dev)->gen) {
-	case 7:
-	case 6:
-		ret = sandybridge_write_fence_reg(obj, pipelined);
-		break;
-	case 5:
-	case 4:
-		ret = i965_write_fence_reg(obj, pipelined);
-		break;
-	case 3:
-		ret = i915_write_fence_reg(obj, pipelined);
-		break;
-	case 2:
-		ret = i830_write_fence_reg(obj, pipelined);
-		break;
+		fence_reg = reg - dev_priv->fence_regs;
+		ret = intel_write_fence_reg(obj, fence_reg, pipelined, true);
+		if (ret == 0) {
+			reg->obj = obj;
+			list_move_tail(&reg->lru_list, &dev_priv->mm.fence_list);
+
+			obj->fence_reg = fence_reg;
+			obj->tiling_changed = false;
+			if (old) {
+				if (old->tiling_mode)
+					i915_gem_release_mmap(old);
+				old->fence_reg = I915_FENCE_REG_NONE;
+			}
+		}
 	}
 
+err:
+	if (old)
+		drm_gem_object_unreference(&old->base);
 	return ret;
 }
 
@@ -2757,7 +2785,7 @@ i915_gem_clear_fence_reg(struct drm_device *dev,
 
 	list_del_init(&reg->lru_list);
 	reg->obj = NULL;
-	reg->setup_seqno = 0;
+	reg->setup_ring = NULL;
 	reg->pin_count = 0;
 }
 
@@ -3195,7 +3223,7 @@ int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj,
 		 * currently pointing to our region in the aperture.
 		 */
 		if (INTEL_INFO(dev)->gen < 6) {
-			ret = i915_gem_object_put_fence(obj);
+			ret = i915_gem_object_put_fence(obj, NULL);
 			if (ret)
 				return ret;
 		}
diff --git a/drivers/gpu/drm/i915/i915_gem_execbuffer.c b/drivers/gpu/drm/i915/i915_gem_execbuffer.c
index 21fd5a4..eb6cfcc 100644
--- a/drivers/gpu/drm/i915/i915_gem_execbuffer.c
+++ b/drivers/gpu/drm/i915/i915_gem_execbuffer.c
@@ -173,9 +173,8 @@ i915_gem_object_set_to_gpu_domain(struct drm_i915_gem_object *obj,
 	 * write domain
 	 */
 	if (obj->base.write_domain &&
-	    (((obj->base.write_domain != obj->base.pending_read_domains ||
-	       obj->ring != ring)) ||
-	     (obj->fenced_gpu_access && !obj->pending_fenced_gpu_access))) {
+	    (obj->base.write_domain != obj->base.pending_read_domains ||
+	     obj->ring != ring)) {
 		flush_domains |= obj->base.write_domain;
 		invalidate_domains |=
 			obj->base.pending_read_domains & ~obj->base.write_domain;
@@ -494,12 +493,12 @@ pin_and_fence_object(struct drm_i915_gem_object *obj,
 				entry->flags |= __EXEC_OBJECT_HAS_FENCE;
 				i915_gem_object_pin_fence(obj);
 			} else {
-				ret = i915_gem_object_put_fence(obj);
+				ret = i915_gem_object_put_fence(obj, ring);
 				if (ret)
 					goto err_unpin;
 			}
+			obj->pending_fenced_gpu_access = true;
 		}
-		obj->pending_fenced_gpu_access = need_fence;
 	}
 
 	entry->offset = obj->gtt_offset;
@@ -893,6 +892,11 @@ i915_gem_execbuffer_move_to_gpu(struct intel_ring_buffer *ring,
 	int ret;
 
 	memset(&cd, 0, sizeof(cd));
+	if (ring->need_fence_flush) {
+		cd.invalidate_domains |= I915_GEM_GPU_DOMAINS;
+		ring->need_fence_flush = false;
+	}
+
 	list_for_each_entry(obj, objects, exec_list)
 		i915_gem_object_set_to_gpu_domain(obj, ring, &cd);
 
@@ -971,7 +975,7 @@ i915_gem_execbuffer_move_to_active(struct list_head *objects,
 
 		obj->base.read_domains = obj->base.pending_read_domains;
 		obj->base.write_domain = obj->base.pending_write_domain;
-		obj->fenced_gpu_access = obj->pending_fenced_gpu_access;
+		obj->fenced_gpu_access |= obj->pending_fenced_gpu_access;
 
 		i915_gem_object_move_to_active(obj, ring, seqno);
 		if (obj->base.write_domain) {
diff --git a/drivers/gpu/drm/i915/intel_display.c b/drivers/gpu/drm/i915/intel_display.c
index 2942e02..15a8200 100644
--- a/drivers/gpu/drm/i915/intel_display.c
+++ b/drivers/gpu/drm/i915/intel_display.c
@@ -6569,7 +6569,7 @@ static int intel_crtc_cursor_set(struct drm_crtc *crtc,
 			goto fail_locked;
 		}
 
-		ret = i915_gem_object_put_fence(obj);
+		ret = i915_gem_object_put_fence(obj, NULL);
 		if (ret) {
 			DRM_ERROR("failed to release fence for cursor");
 			goto fail_unpin;
diff --git a/drivers/gpu/drm/i915/intel_overlay.c b/drivers/gpu/drm/i915/intel_overlay.c
index 77d0fcc..23b264a 100644
--- a/drivers/gpu/drm/i915/intel_overlay.c
+++ b/drivers/gpu/drm/i915/intel_overlay.c
@@ -777,7 +777,7 @@ static int intel_overlay_do_put_image(struct intel_overlay *overlay,
 	if (ret != 0)
 		return ret;
 
-	ret = i915_gem_object_put_fence(new_bo);
+	ret = i915_gem_object_put_fence(new_bo, NULL);
 	if (ret)
 		goto out_unpin;
 
diff --git a/drivers/gpu/drm/i915/intel_ringbuffer.h b/drivers/gpu/drm/i915/intel_ringbuffer.h
index bc0365b..8004855 100644
--- a/drivers/gpu/drm/i915/intel_ringbuffer.h
+++ b/drivers/gpu/drm/i915/intel_ringbuffer.h
@@ -55,6 +55,7 @@ struct  intel_ring_buffer {
 	 * we can detect new retirements.
 	 */
 	u32		last_retired_head;
+	bool		need_fence_flush;
 
 	spinlock_t	irq_lock;
 	u32		irq_refcount;