drm/i915: Switch gen6/7 BLT add_request to MI_FLUSH_DW.flush-dw-notify

The MI_STORE_DW/MI_USER_INTERRUPT mechanism is explicitly stated to
not be reliable.  The MI_FLUSH_DW is the blessed hardware mechanism
for reliably associating flushes and seqno writes with interrupts.

3 files changed, 56 insertions, 9 deletions

diff --git a/drivers/gpu/drm/i915/i915_irq.c b/drivers/gpu/drm/i915/i915_irq.c
index b40004b5597..bdeafce7830 100644
--- a/drivers/gpu/drm/i915/i915_irq.c
+++ b/drivers/gpu/drm/i915/i915_irq.c
@@ -503,7 +503,7 @@ static irqreturn_t ivybridge_irq_handler(DRM_IRQ_ARGS)
 		notify_ring(dev, &dev_priv->ring[RCS]);
 	if (gt_iir & GT_GEN6_BSD_USER_INTERRUPT)
 		notify_ring(dev, &dev_priv->ring[VCS]);
-	if (gt_iir & GT_BLT_USER_INTERRUPT)
+	if (gt_iir & GT_GEN6_BLT_MI_FLUSH_DW_NOTIFY_INTERRUPT)
 		notify_ring(dev, &dev_priv->ring[BCS]);
 
 	if (de_iir & DE_GSE_IVB)
@@ -603,7 +603,7 @@ static irqreturn_t ironlake_irq_handler(DRM_IRQ_ARGS)
 		notify_ring(dev, &dev_priv->ring[RCS]);
 	if (gt_iir & bsd_usr_interrupt)
 		notify_ring(dev, &dev_priv->ring[VCS]);
-	if (gt_iir & GT_BLT_USER_INTERRUPT)
+	if (gt_iir & GT_GEN6_BLT_MI_FLUSH_DW_NOTIFY_INTERRUPT)
 		notify_ring(dev, &dev_priv->ring[BCS]);
 
 	if (de_iir & DE_GSE)
@@ -1833,7 +1833,7 @@ static int ironlake_irq_postinstall(struct drm_device *dev)
 		render_irqs =
 			GT_USER_INTERRUPT |
 			GT_GEN6_BSD_USER_INTERRUPT |
-			GT_BLT_USER_INTERRUPT;
+			GT_GEN6_BLT_MI_FLUSH_DW_NOTIFY_INTERRUPT;
 	else
 		render_irqs =
 			GT_USER_INTERRUPT |
@@ -1906,7 +1906,7 @@ static int ivybridge_irq_postinstall(struct drm_device *dev)
 	I915_WRITE(GTIMR, dev_priv->gt_irq_mask);
 
 	render_irqs = GT_USER_INTERRUPT | GT_GEN6_BSD_USER_INTERRUPT |
-		GT_BLT_USER_INTERRUPT;
+		GT_GEN6_BLT_MI_FLUSH_DW_NOTIFY_INTERRUPT;
 	I915_WRITE(GTIER, render_irqs);
 	POSTING_READ(GTIER);
 
diff --git a/drivers/gpu/drm/i915/i915_reg.h b/drivers/gpu/drm/i915/i915_reg.h
index 226e7ea7cf6..d740bb66e0a 100644
--- a/drivers/gpu/drm/i915/i915_reg.h
+++ b/drivers/gpu/drm/i915/i915_reg.h
@@ -182,9 +182,19 @@
  *   address/value pairs. Don't overdue it, though, x <= 2^4 must hold!
  */
 #define MI_LOAD_REGISTER_IMM(x)	MI_INSTR(0x22, 2*x-1)
+
 #define MI_FLUSH_DW		MI_INSTR(0x26, 2) /* for GEN6 */
+/* DW0 */
+#define   MI_FLUSH_DW_STORE_DATA_INDEX		(1 << 21)
+#define   MI_FLUSH_DW_POST_SYNC_NONE		(0 << 14)
+#define   MI_FLUSH_DW_POST_SYNC_WRITE_QW	(1 << 14)
+#define   MI_FLUSH_DW_POST_SYNC_WRITE_TIMESTAMP	(3 << 14)
+#define   MI_FLUSH_DW_NOTIFY_ENABLE		(1 << 8)
 #define   MI_INVALIDATE_TLB	(1<<18)
 #define   MI_INVALIDATE_BSD	(1<<7)
+/* DW1 */
+#define   MI_FLUSH_DW_GLOBAL_GTT		(1 << 2)
+
 #define MI_BATCH_BUFFER		MI_INSTR(0x30, 1)
 #define   MI_BATCH_NON_SECURE	(1)
 #define   MI_BATCH_NON_SECURE_I965 (1<<8)
@@ -2855,6 +2865,7 @@
 #define GT_BSD_USER_INTERRUPT   (1 << 5)
 #define GT_GEN6_BSD_USER_INTERRUPT	(1 << 12)
 #define GT_BLT_USER_INTERRUPT	(1 << 22)
+#define GT_GEN6_BLT_MI_FLUSH_DW_NOTIFY_INTERRUPT (1 << 26)
 
 #define GTISR   0x44010
 #define GTIMR   0x44014
diff --git a/drivers/gpu/drm/i915/intel_ringbuffer.c b/drivers/gpu/drm/i915/intel_ringbuffer.c
index b6357b11507..53ea1e4ea5a 100644
--- a/drivers/gpu/drm/i915/intel_ringbuffer.c
+++ b/drivers/gpu/drm/i915/intel_ringbuffer.c
@@ -478,6 +478,42 @@ gen6_add_request(struct intel_ring_buffer *ring,
 	return 0;
 }
 
+static int
+mi_flush_dw_add_request(struct intel_ring_buffer *ring, u32 *seqno)
+{
+	int ret;
+
+	ret = intel_ring_begin(ring, 10);
+	if (ret)
+		return ret;
+
+	*seqno = i915_gem_get_seqno(ring->dev);
+
+	update_mboxes(ring, *seqno); /* 6 dwords */
+
+	/* Note that this command flushes and invalidates the render
+	 * cache.  There is text in the spec saying "After this
+	 * command is completed with a Store DWord enabled, CPU access
+	 * to graphics memory will be coherent (assuming the Render
+	 * Cache flush is not inhibited)", except that there is no
+	 * actual bit for render cache flush inhibit, and the flush
+	 * always occurs in the hardware.
+	 */
+	intel_ring_emit(ring,
+			MI_FLUSH_DW |
+			MI_FLUSH_DW_NOTIFY_ENABLE |
+			MI_FLUSH_DW_STORE_DATA_INDEX |
+			MI_FLUSH_DW_POST_SYNC_WRITE_QW);
+	intel_ring_emit(ring,
+			I915_GEM_HWS_INDEX * 4 |
+			MI_FLUSH_DW_GLOBAL_GTT); /* HWS byte offset */
+	intel_ring_emit(ring, *seqno); /* QW low */
+	intel_ring_emit(ring, 0); /* QW high */
+	intel_ring_advance(ring);
+
+	return 0;
+}
+
 /**
  * intel_ring_sync - sync the waiter to the signaller on seqno
  *
@@ -1336,16 +1372,16 @@ static bool
 blt_ring_get_irq(struct intel_ring_buffer *ring)
 {
 	return gen6_ring_get_irq(ring,
-				 GT_BLT_USER_INTERRUPT,
-				 GEN6_BLITTER_USER_INTERRUPT);
+				 GT_GEN6_BLT_MI_FLUSH_DW_NOTIFY_INTERRUPT,
+				 GEN6_BLITTER_MI_FLUSH_DW_NOTIFY_INTERRUPT);
 }
 
 static void
 blt_ring_put_irq(struct intel_ring_buffer *ring)
 {
 	gen6_ring_put_irq(ring,
-			  GT_BLT_USER_INTERRUPT,
-			  GEN6_BLITTER_USER_INTERRUPT);
+			  GT_GEN6_BLT_MI_FLUSH_DW_NOTIFY_INTERRUPT,
+			  GEN6_BLITTER_MI_FLUSH_DW_NOTIFY_INTERRUPT);
 }
 
 
@@ -1453,7 +1489,7 @@ static const struct intel_ring_buffer gen6_blt_ring = {
 	.init			= blt_ring_init,
 	.write_tail		= ring_write_tail,
 	.flush			= blt_ring_flush,
-	.add_request		= gen6_add_request,
+	.add_request		= mi_flush_dw_add_request,
 	.get_seqno		= ring_get_seqno,
 	.irq_get		= blt_ring_get_irq,
 	.irq_put		= blt_ring_put_irq,