drm/i915: Replace bool 'enable_preemption' with 'preemption_level'

and implement several preemption level options.

New range is 0-6(+)
0: none
1: between-batch
2: mid-batch cooperative
3: mid-batch but no mid-thread or mid-object
4: mid-thread-group but not mid-thread or mid-object
5: mid-thread-group and mid-object (but still no mid-thread)
6: same as 5, but allow user override
7+: unrestricted (h/w defaults)

Signed-off-by: Dave Gordon <david.s.gordon@intel.com>

5 files changed, 157 insertions, 6 deletions

diff --git a/drivers/gpu/drm/i915/i915_params.c b/drivers/gpu/drm/i915/i915_params.c
index 7af28e29071e..50c936876919 100644
--- a/drivers/gpu/drm/i915/i915_params.c
+++ b/drivers/gpu/drm/i915/i915_params.c
@@ -59,7 +59,7 @@ struct i915_params i915 __read_mostly = {
 	.enable_dp_mst = true,
 	.inject_load_failure = 0,
 	.enable_scheduler = 1,
-	.enable_preemption = 1,
+	.preemption_level = -1,
 };
 
 module_param_named(modeset, i915.modeset, int, 0400);
@@ -216,5 +216,5 @@ MODULE_PARM_DESC(inject_load_failure,
 module_param_named_unsafe(enable_scheduler, i915.enable_scheduler, int, 0600);
 MODULE_PARM_DESC(enable_scheduler, "Enable scheduler (0 = disable, 1 = enable [default])");
 
-module_param_named_unsafe(enable_preemption, i915.enable_preemption, int, 0600);
-MODULE_PARM_DESC(enable_preemption, "Enable pre-emption within scheduler  (0 = disable, 1 = enable [default])");
+module_param_named_unsafe(preemption_level, i915.preemption_level, int, 0600);
+MODULE_PARM_DESC(preemption_level, "Pre-emption level control (-1 = platform default [default], 0 = disable, 1 = between batch only, 2 = co-operative mid-batch, 3 = more)");
diff --git a/drivers/gpu/drm/i915/i915_params.h b/drivers/gpu/drm/i915/i915_params.h
index 6031b69493ad..aba850075c20 100644
--- a/drivers/gpu/drm/i915/i915_params.h
+++ b/drivers/gpu/drm/i915/i915_params.h
@@ -51,7 +51,7 @@ struct i915_params {
 	int edp_vswing;
 	unsigned int inject_load_failure;
 	int enable_scheduler;
-	int enable_preemption;
+	int preemption_level;
 	/* leave bools at the end to not create holes */
 	bool enable_hangcheck;
 	bool fastboot;
diff --git a/drivers/gpu/drm/i915/i915_reg.h b/drivers/gpu/drm/i915/i915_reg.h
index b1a9865cbf6d..0bcf88287974 100644
--- a/drivers/gpu/drm/i915/i915_reg.h
+++ b/drivers/gpu/drm/i915/i915_reg.h
@@ -1631,6 +1631,8 @@ enum skl_disp_power_wells {
 #define BB_PREEMPT_ADDR		_MMIO(0x02148)
 #define SBB_PREEMPT_ADDR	_MMIO(0x0213c)
 #define RS_PREEMPT_STATUS	_MMIO(0x0215c)
+#define PREEMPT_DEBUG(ring)	_MMIO((ring)->mmio_base+0x248)
+#define   PREEMPT_ON_ARB_CHK_ONLY	(1<<8)
 
 #define HSW_GTT_CACHE_EN	_MMIO(0x4024)
 #define   GTT_CACHE_EN_ALL	0xF0007FFF
diff --git a/drivers/gpu/drm/i915/i915_scheduler.c b/drivers/gpu/drm/i915/i915_scheduler.c
index 5a463852adc0..723560d65361 100644
--- a/drivers/gpu/drm/i915/i915_scheduler.c
+++ b/drivers/gpu/drm/i915/i915_scheduler.c
@@ -1069,7 +1069,8 @@ int i915_scheduler_queue_execbuffer(struct i915_scheduler_queue_entry *qe)
 
 	want_preempt = node->priority >= scheduler->priority_level_preempt;
 
-	if (!i915.enable_preemption)
+	/* Pre-emption is disabled if the level control is set to 0 */
+	if (i915.preemption_level == 0)
 		want_preempt = false;
 
 	/* Pre-emption is not yet implemented in non-execlist mode */
diff --git a/drivers/gpu/drm/i915/intel_lrc.c b/drivers/gpu/drm/i915/intel_lrc.c
index 6e0ca5bef7a7..8f5d1051ac48 100644
--- a/drivers/gpu/drm/i915/intel_lrc.c
+++ b/drivers/gpu/drm/i915/intel_lrc.c
@@ -1182,6 +1182,145 @@ gen8_emit_pipe_control_qw_store_index(struct drm_i915_gem_request *request,
 	return 0;
 }
 
+#define GEN7_FF_SLICE_CS_CHICKEN1		_MMIO(0x20e0)
+#define   GEN9_FFSC_PERCTX_PREEMPT_CTRL		(1<<14)
+
+#define GEN9_CS_DEBUG_MODE			_MMIO(0x20ec)
+
+#define FF_SLICE_CS_CHICKEN2			_MMIO(0x20e4)
+/*
+#define   GPGPU_MID_THREAD_PREEMPT_ENABLE	(0<<1)
+#define   GPGPU_THREAD_GROUP_PREEMPT_ENABLE	(1<<1)
+#define   GPGPU_THREAD_PREEMPT_DISABLE		(2<<1)
+#define   GPGPU_THREAD_PREEMPT_MASK		(3<<1)
+*/
+/* FF_SLICE_CS_CHICKEN2 and GEN8_CS_CHICKEN1 */
+#define	  MID_THREAD_PREEMPTION			(0<<1)
+#define	  THREAD_GROUP_PREEMPTION		(1<<1)
+#define	  COMMAND_LEVEL_PREEMPTION		(2<<1)
+#define	  PREEMPTION_LEVEL_MASK			(3<<1)
+
+#define GEN8_CS_CHICKEN1			_MMIO(0x2580)
+/* GEN9_CS_DEBUG_MODE and GEN8_CS_CHICKEN1 */
+#define	  GEN9_GFX_REPLAY_MODE			(1<<0)
+
+static void
+emit_preemption_control(struct drm_i915_gem_request *req)
+{
+	struct intel_ringbuffer *ringbuf = req->ringbuf;
+	int preemption_level = i915.preemption_level;
+	u32 allow_user_control = false;
+	u32 allow_mid_object = false;
+	u32 mid_thread_mode = 0;
+	u32 data;
+
+	/* Preemption is always disabled while preempting */
+	if (req->scheduler_flags & I915_REQ_SF_PREEMPT) {
+		data = MI_ARB_ON_OFF | MI_ARB_DISABLE;
+		intel_logical_ring_emit(ringbuf, data);
+		intel_logical_ring_emit(ringbuf, MI_NOOP);
+		return;
+	}
+
+	/* At level 0, no preemption will be attempted */
+	if (preemption_level == 0)
+		return;
+
+	/* All other levels check for preemption before starting the batch */
+	intel_logical_ring_emit(ringbuf, MI_ARB_ON_OFF | MI_ARB_ENABLE);
+	intel_logical_ring_emit(ringbuf, MI_ARB_CHECK);
+
+	/* Non-render rings support between-batch only, for now */
+	if (req->engine->id != RCS)
+		preemption_level = 1;
+	else if (preemption_level < 0)
+		preemption_level = 3;	/* Current safe default	*/
+
+	switch (preemption_level) {
+	case 1:
+		/*
+		 * Between-batch preemption only - no mid-batch allowed.
+		 *
+		 * XXX: use preemption disable bit in debug register instead
+		 * to avoid collision with ARB_OFF/ARB_ON in workaround batch?
+		 */
+		data = MI_ARB_ON_OFF | MI_ARB_DISABLE;
+		intel_logical_ring_emit(ringbuf, data);
+		intel_logical_ring_emit(ringbuf, MI_NOOP);
+		return;
+
+	case 2:
+		/*
+		 * Co-operative mid-batch only. This register is not saved
+		 * with the context and so must be reprogrammed each time.
+		 *
+		 * XXX: need to fix up in workaround batch?
+		 */
+		data = _MASKED_BIT_ENABLE(PREEMPT_ON_ARB_CHK_ONLY);
+		intel_logical_ring_emit(ringbuf, MI_NOOP);
+		intel_logical_ring_emit(ringbuf, MI_LOAD_REGISTER_IMM(1));
+		intel_logical_ring_emit_reg(ringbuf, PREEMPT_DEBUG(req->engine));
+		intel_logical_ring_emit(ringbuf, data);
+		return;
+
+	case 3:
+		/* Mid-batch preemption allowed, but not thread-level */
+		mid_thread_mode = _MASKED_FIELD(PREEMPTION_LEVEL_MASK, COMMAND_LEVEL_PREEMPTION);
+		break;
+	case 4:
+		/* Thread-group preemption allowed, but not mid-thread */
+		mid_thread_mode = _MASKED_FIELD(PREEMPTION_LEVEL_MASK, THREAD_GROUP_PREEMPTION);
+		break;
+	case 5:
+		/* Thread-group preemption allowed, also mid-object */
+		mid_thread_mode = _MASKED_FIELD(PREEMPTION_LEVEL_MASK, THREAD_GROUP_PREEMPTION);
+		allow_mid_object = true;
+		break;
+	case 6:
+		/* Thread-group and mid-object preemption allowed; user can override */
+		mid_thread_mode = _MASKED_FIELD(PREEMPTION_LEVEL_MASK, THREAD_GROUP_PREEMPTION);
+		allow_mid_object = true;
+		allow_user_control = true;
+		break;
+
+	case 7:
+		/* XXX: any further levels of preemption? */
+	default:
+		/* Default is the maximum level permitted by the hardware */
+		return;
+	}
+
+	/*
+	 * If GEN9_FFSC_PERCTX_PREEMPT_CTRL is disabled, the values in
+	 * GEN9_CS_DEBUG_MODE and FF_SLICE_CS_CHICKEN2 will be used in
+	 * preference to those in GEN8_CS_CHICKEN1, so the user batch
+	 * cannot override them. Enabling GEN9_FFSC_PERCTX_PREEMPT_CTRL
+	 * causes the values in GEN8_CS_CHICKEN1 to be used instead;
+	 * this register has been whitelisted, so the user batch can
+	 * override the levels we set here; in particular, it can ENABLE
+	 * mid-thread preemption. We don't offer that option here as it
+	 * must only be set for GPGPU workloads, not MEDIA
+	 */
+	if (allow_user_control)
+		allow_user_control = _MASKED_BIT_ENABLE(GEN9_FFSC_PERCTX_PREEMPT_CTRL);
+	else
+		allow_user_control = _MASKED_BIT_DISABLE(GEN9_FFSC_PERCTX_PREEMPT_CTRL);
+	if (allow_mid_object)
+		allow_mid_object = _MASKED_BIT_ENABLE(GEN9_GFX_REPLAY_MODE);
+	else
+		allow_mid_object = _MASKED_BIT_DISABLE(GEN9_GFX_REPLAY_MODE);
+
+	intel_logical_ring_emit(ringbuf, MI_LOAD_REGISTER_IMM(4));
+	intel_logical_ring_emit_reg(ringbuf, GEN7_FF_SLICE_CS_CHICKEN1);
+	intel_logical_ring_emit(ringbuf, allow_user_control);
+	intel_logical_ring_emit_reg(ringbuf, GEN9_CS_DEBUG_MODE);
+	intel_logical_ring_emit(ringbuf, allow_mid_object);
+	intel_logical_ring_emit_reg(ringbuf, FF_SLICE_CS_CHICKEN2);
+	intel_logical_ring_emit(ringbuf, mid_thread_mode);
+	intel_logical_ring_emit_reg(ringbuf, GEN8_CS_CHICKEN1);
+	intel_logical_ring_emit(ringbuf, mid_thread_mode | allow_mid_object);
+}
+
 /*
  * Emit the commands to execute when preparing to start a batch
  *
@@ -1308,8 +1447,9 @@ int intel_execlists_submission_final(struct i915_execbuffer_params *params)
 	req->head = intel_ring_get_tail(ringbuf);
 
 	/*
-	 * Log the seqno of the batch we're starting
+	 * Emit the preemption control and preamble for the batch
 	 */
+	emit_preemption_control(req);
 	emit_preamble(req);
 
 	/*
@@ -2297,6 +2437,10 @@ static int gen8_emit_request(struct drm_i915_gem_request *request)
 	intel_logical_ring_emit(ringbuf, MI_USER_INTERRUPT);
 	intel_logical_ring_emit(ringbuf, MI_NOOP);
 
+	/* Always check for preemption after finishing the request */
+	intel_logical_ring_emit(ringbuf, MI_ARB_ON_OFF | MI_ARB_ENABLE);
+	intel_logical_ring_emit(ringbuf, MI_ARB_CHECK);
+
 	return intel_logical_ring_advance_and_submit(request);
 }
 
@@ -2357,6 +2501,10 @@ static int gen8_emit_request_render(struct drm_i915_gem_request *request)
 	intel_logical_ring_emit(ringbuf, MI_USER_INTERRUPT);
 	intel_logical_ring_emit(ringbuf, MI_NOOP);
 
+	/* Always check for preemption after finishing the request */
+	intel_logical_ring_emit(ringbuf, MI_ARB_ON_OFF | MI_ARB_ENABLE);
+	intel_logical_ring_emit(ringbuf, MI_ARB_CHECK);
+
 	return intel_logical_ring_advance_and_submit(request);
 }