drm/i915/preempt: scheduler logic for postprocessing preemptive requests

This patch adds the scheduler logic for postprocessing of completed
preemption requests. It cleans out both the fence_signal list (dropping
references as it goes) and the primary request_list. Requests that
didn't complete are put into the 'preempted' state for resubmission by
the scheduler, and their ringbuffers are emptied by setting head==tail
so thers is no pending work in any preempted context. The (dummy)
preemption request itself is also recycled in the same way, and should
then be (re)selected by the scheduler to be submitted next (unless
anything with even hogher priority as been queued in the meantime); but
because there are now no requests flying, the next-submitted batch will
not need to preempt, and so will be launched 'for real' as a regular
non-preemptive batch.

Actually-preemptive requests are still disabled via a module parameter
at this stage, as we don't yet have the code to emit preemption requests
into the ringbuffer.

For: VIZ-2021
Signed-off-by: Dave Gordon <david.s.gordon@intel.com>

3 files changed, 202 insertions, 1 deletions

diff --git a/drivers/gpu/drm/i915/i915_drv.h b/drivers/gpu/drm/i915/i915_drv.h
index 238747ae5bec..c862245c769d 100644
--- a/drivers/gpu/drm/i915/i915_drv.h
+++ b/drivers/gpu/drm/i915/i915_drv.h
@@ -2385,6 +2385,7 @@ struct drm_i915_gem_request * __must_check
 i915_gem_request_alloc(struct intel_engine_cs *engine,
 		       struct intel_context *ctx);
 void i915_gem_request_cancel(struct drm_i915_gem_request *req);
+void i915_gem_request_dequeue(struct drm_i915_gem_request *req);
 void i915_gem_request_enable_interrupt(struct drm_i915_gem_request *req,
 				       bool fence_locked);
 void i915_gem_request_notify(struct intel_engine_cs *ring, bool fence_locked);
diff --git a/drivers/gpu/drm/i915/i915_gem.c b/drivers/gpu/drm/i915/i915_gem.c
index 2d3c956e382b..909808fc2c74 100644
--- a/drivers/gpu/drm/i915/i915_gem.c
+++ b/drivers/gpu/drm/i915/i915_gem.c
@@ -1267,6 +1267,7 @@ int __i915_wait_request(struct drm_i915_gem_request *req,
 	might_sleep();
 	WARN(!intel_irqs_enabled(dev_priv), "IRQs disabled");
 
+	/* Lightweight check first of all */
 	if (i915_gem_request_completed(req))
 		return 0;
 
@@ -1475,6 +1476,11 @@ static void i915_gem_request_retire(struct drm_i915_gem_request *request)
 	 */
 	request->ringbuf->last_retired_head = request->postfix;
 
+	/*
+	 * Must use list_del_init() not list_del() because some other
+	 * code tests (list_empty(&request->list)) to see whether the
+	 * request is (still) on the engine->request_list!
+	 */
 	list_del_init(&request->list);
 	i915_gem_request_remove_from_client(request);
 
@@ -1508,10 +1514,18 @@ __i915_gem_request_retire__upto(struct drm_i915_gem_request *req)
 
 	lockdep_assert_held(&engine->dev->struct_mutex);
 
+	/*
+	 * If the request is not on any list, then presumably
+	 * it's already been retired?
+	 */
 	if (list_empty(&req->list))
 		return;
 
 	do {
+		/* Don't blindly assume that the request will be found! */
+		if (WARN_ON(list_empty(&engine->request_list)))
+			break;
+
 		tmp = list_first_entry(&engine->request_list,
 				       typeof(*tmp), list);
 
@@ -2745,6 +2759,24 @@ void __i915_add_request(struct drm_i915_gem_request *request,
 	intel_ring_reserved_space_end(ringbuf);
 }
 
+void i915_gem_request_dequeue(struct drm_i915_gem_request *request)
+{
+	/*
+	 * The request has been de-queued from the hardware in some manner
+	 * (e.g. through pre-emption). So it needs to be removed from the
+	 * active request list (the request list doesn't contribute to
+	 * refcounting, so we don't also have to unreference it here).
+	 *
+	 * It also needs to have its seqno cleared as that will not be
+	 * valid any longer. However, the expectation is that the request
+	 * will be resubmitted later. At that time it will be assigned a
+	 * shiny new seqno.
+	 */
+	WARN_ON(i915_gem_request_completed(request));
+	list_del_init(&request->list);
+	request->seqno = 0;
+}
+
 static bool i915_context_is_banned(struct drm_i915_private *dev_priv,
 				   const struct intel_context *ctx)
 {
diff --git a/drivers/gpu/drm/i915/i915_scheduler.c b/drivers/gpu/drm/i915/i915_scheduler.c
index c7fb5ac5cee9..571cad18e3ec 100644
--- a/drivers/gpu/drm/i915/i915_scheduler.c
+++ b/drivers/gpu/drm/i915/i915_scheduler.c
@@ -1306,6 +1306,169 @@ void i915_scheduler_reset_cleanup(struct intel_engine_cs *engine)
 	}
 }
 
+/*
+ * At this point, preemption has occurred.
+ *
+ * All the requests that had already completed before preemption will
+ * have been taken off the fence_signal_list, signalled, and put onto
+ * the fence_unsignal_list for cleanup. The preempting request itself
+ * should however still be on the fence_signal_list (and has not been
+ * signalled). There may also be additional requests on this list; they
+ * have been preempted.
+ *
+ * The request_list has not yet been processed, so it may still contain
+ * requests that have already completed. It should also contain the
+ * preempting request (not completed), and maybe additional requests;
+ * again, these have been preempted and need to be recycled through the
+ * scheduler.
+ */
+noinline
+static void
+i915_scheduler_preemption_postprocess(struct intel_engine_cs *engine)
+{
+	struct drm_i915_private *dev_priv = to_i915(engine->dev);
+	struct i915_scheduler *scheduler = dev_priv->scheduler;
+	struct i915_scheduler_queue_entry *pnode = NULL;
+	struct drm_i915_gem_request *preq = NULL;
+	struct i915_scheduler_stats *stats;
+	unsigned long flags;
+	int preempted = 0, preemptive = 0;
+
+	mutex_lock(&engine->dev->struct_mutex);
+
+	/*
+	 * FIXME: grab & empty fence_signal_list with spinlock,
+	 * then iterate after?
+	 */
+	spin_lock_irqsave(&engine->fence_lock, flags);
+	while (!list_empty(&engine->fence_signal_list)) {
+		struct i915_scheduler_queue_entry *node;
+		struct drm_i915_gem_request *req;
+
+		req = list_first_entry(&engine->fence_signal_list,
+				       struct drm_i915_gem_request,
+				       signal_link);
+		list_del_init(&req->signal_link);
+		spin_unlock_irqrestore(&engine->fence_lock, flags);
+
+		/* We should find only tracked unsignalled requests */
+		node = req->scheduler_qe;
+		WARN(!node || i915_gem_request_completed(req) ||
+		     (node->status == I915_SQS_PREEMPTED),
+		     "Invalid node state: %s [req = %d:%d]\n",
+		     node ? i915_scheduler_queue_status_str(node->status) : "NULL",
+		     req->uniq, req->seqno);
+
+		i915_gem_request_unreference(req);
+
+		spin_lock_irqsave(&engine->fence_lock, flags);
+	}
+	spin_unlock_irqrestore(&engine->fence_lock, flags);
+	/* Fence signal list must now be empty */
+
+	/*
+	 * The preemptive request and all other requests remaining on the
+	 * engine's work-in-progress list must be marked as preempted, so
+	 * the scheduler will reselect and resubmit them ...
+	 */
+	spin_lock_irqsave(&scheduler->lock, flags);
+
+	{
+		struct drm_i915_gem_request *req, *next;
+
+		list_for_each_entry_safe(req, next, &engine->request_list, list) {
+			struct i915_scheduler_queue_entry *node;
+
+			node = req->scheduler_qe;
+			if (WARN_ON(req->engine != engine))
+				continue;
+			if (i915_gem_request_completed(req))
+				continue;
+			/* Let's hope there aren't any untracked nodes here! */
+			if (WARN_ON(!node))
+				continue;
+
+			if (node->status == I915_SQS_PREEMPTED) {
+				/* Already processed in _notify() above */
+				preemptive += 1;
+				preq = req;
+				pnode = req->scheduler_qe;
+			} else if (!WARN_ON(!I915_SQS_IS_FLYING(node))) {
+				preempted += 1;
+				node->status = I915_SQS_PREEMPTED;
+				trace_i915_scheduler_unfly(engine, node);
+				trace_i915_scheduler_node_state_change(engine, node);
+				/* Empty the preempted ringbuffer */
+				intel_lr_context_resync(req->ctx, engine, false);
+			}
+
+			i915_gem_request_dequeue(req);
+		}
+	}
+
+	/* We should have found exactly ONE preemptive request */
+	WARN(preemptive != 1, "Got unexpected preemptive count II: %d!\n",
+	     preemptive);
+	stats = &scheduler->stats[engine->id];
+	stats->preempted += preempted;
+	if (stats->max_preempted < preempted)
+		stats->max_preempted = preempted;
+
+	{
+		/* XXX: Sky should be empty now */
+		struct i915_scheduler_queue_entry *node;
+
+		for_each_scheduler_node(node, engine->id)
+			WARN_ON(I915_SQS_IS_FLYING(node));
+	}
+
+	/* Anything else to do here ... ? */
+
+	/*
+	 * Postprocessing complete; the scheduler is now back in
+	 * normal non-preemptive state and can submit more requests
+	 */
+	scheduler->flags[engine->id] &= ~(I915_SF_PREEMPTING|I915_SF_PREEMPTED);
+
+	spin_unlock_irqrestore(&scheduler->lock, flags);
+
+	/* XXX: Should be nothing outstanding on request list */
+	{
+		struct drm_i915_gem_request *req;
+
+		list_for_each_entry(req, &engine->request_list, list)
+			WARN_ON(!i915_gem_request_completed(req));
+	}
+
+	/* Anything else to do here ... ? */
+	if (!WARN_ON(pnode == NULL || preq == NULL)) {
+		WARN_ON(pnode->params.request != preq);
+		WARN_ON(preq->scheduler_qe != pnode);
+		WARN_ON(preq->seqno);
+
+		/*
+		 * FIXME: assign a new reserved seqno here to ensure
+		 * we don't relaunch this request with the same seqno
+		 * FIXME: can we just clear it here instead?
+		 */
+		if (dev_priv->next_seqno == 0)
+			dev_priv->next_seqno = 1;
+		dev_priv->last_seqno = dev_priv->next_seqno++;
+		DRM_DEBUG_DRIVER("reassigning reserved seqno %08x->%08x, (seqno %08x, uniq %d)\n",
+			preq->reserved_seqno, dev_priv->last_seqno,
+			preq->seqno, preq->uniq);
+		preq->reserved_seqno = dev_priv->last_seqno;
+
+		/* FIXME: don't sleep, don't empty context? */
+		msleep(1);
+		/* Empty the preempted ringbuffer */
+		intel_lr_context_resync(preq->ctx, engine, false);
+	}
+
+	mutex_unlock(&engine->dev->struct_mutex);
+}
+
+noinline
 static bool i915_scheduler_remove(struct i915_scheduler *scheduler,
 				  struct intel_engine_cs *engine,
 				  struct list_head *remove)
@@ -1418,12 +1581,17 @@ void i915_scheduler_work_handler(struct work_struct *work)
 {
 	struct intel_engine_cs *engine;
 	struct drm_i915_private *dev_priv;
+	struct i915_scheduler *scheduler;
 	int i;
 
 	dev_priv = container_of(work, struct drm_i915_private, mm.scheduler_work);
+	scheduler = dev_priv->scheduler;
 
-	for_each_engine(engine, dev_priv, i)
+	for_each_engine(engine, dev_priv, i) {
+		if (scheduler->flags[engine->id] & I915_SF_PREEMPTED)
+			i915_scheduler_preemption_postprocess(engine);
 		i915_scheduler_process_work(engine);
+	}
 }
 
 /**