drm/i915: Make checking gt awakeness less racy

We always change the gt awakeness status while holding
mutex. But we do early bailouts by just reading the status.
Also we inspect gt awakeness in irq context.

As our current sampling of gt.awake is a mixture of
direct reads and READ_ONCEs, make it less error prone by
wrapping it into a inline and alwaus use READ ONCE.

Further, push the value through smp_store_mb so that
our interrupt handler can inspect gt awakeness correctly.

Cc: Chris Wilson <chris@chris-wilson.co.uk>
Signed-off-by: Mika Kuoppala <mika.kuoppala@linux.intel.com>

8 files changed, 25 insertions, 20 deletions

diff --git a/drivers/gpu/drm/i915/i915_debugfs.c b/drivers/gpu/drm/i915/i915_debugfs.c
index 0bb6e01121fc..d6e0acda52bc 100644
--- a/drivers/gpu/drm/i915/i915_debugfs.c
+++ b/drivers/gpu/drm/i915/i915_debugfs.c
@@ -1362,7 +1362,7 @@ static int i915_hangcheck_info(struct seq_file *m, void *unused)
 	else
 		seq_puts(m, "Hangcheck inactive\n");
 
-	seq_printf(m, "GT active? %s\n", yesno(dev_priv->gt.awake));
+	seq_printf(m, "GT active? %s\n", yesno(intel_gt_awake(dev_priv)));
 
 	for_each_engine(engine, dev_priv, id) {
 		struct intel_breadcrumbs *b = &engine->breadcrumbs;
@@ -2310,7 +2310,7 @@ static int i915_rps_boost_info(struct seq_file *m, void *data)
 
 	seq_printf(m, "RPS enabled? %d\n", rps->enabled);
 	seq_printf(m, "GPU busy? %s [%d requests]\n",
-		   yesno(dev_priv->gt.awake), dev_priv->gt.active_requests);
+		   yesno(intel_gt_awake(dev_priv)), dev_priv->gt.active_requests);
 	seq_printf(m, "CPU waiting? %d\n", count_irq_waiters(dev_priv));
 	seq_printf(m, "Boosts outstanding? %d\n",
 		   atomic_read(&rps->num_waiters));
@@ -2861,7 +2861,7 @@ static int i915_runtime_pm_status(struct seq_file *m, void *unused)
 	if (!HAS_RUNTIME_PM(dev_priv))
 		seq_puts(m, "Runtime power management not supported\n");
 
-	seq_printf(m, "GPU idle: %s\n", yesno(!dev_priv->gt.awake));
+	seq_printf(m, "GPU idle: %s\n", yesno(!intel_gt_awake(dev_priv)));
 	seq_printf(m, "IRQs disabled: %s\n",
 		   yesno(!intel_irqs_enabled(dev_priv)));
 #ifdef CONFIG_PM
@@ -3297,7 +3297,7 @@ static int i915_engine_info(struct seq_file *m, void *unused)
 	intel_runtime_pm_get(dev_priv);
 
 	seq_printf(m, "GT awake? %s\n",
-		   yesno(dev_priv->gt.awake));
+		   yesno(intel_gt_awake(dev_priv)));
 	seq_printf(m, "Global active requests: %d\n",
 		   dev_priv->gt.active_requests);
 
diff --git a/drivers/gpu/drm/i915/i915_drv.h b/drivers/gpu/drm/i915/i915_drv.h
index c7b2ca6aff05..27fb86fc0413 100644
--- a/drivers/gpu/drm/i915/i915_drv.h
+++ b/drivers/gpu/drm/i915/i915_drv.h
@@ -3357,6 +3357,11 @@ extern void intel_irq_fini(struct drm_i915_private *dev_priv);
 int intel_irq_install(struct drm_i915_private *dev_priv);
 void intel_irq_uninstall(struct drm_i915_private *dev_priv);
 
+static inline bool intel_gt_awake(const struct drm_i915_private *i915)
+{
+	return READ_ONCE(i915->gt.awake);
+}
+
 static inline bool intel_gvt_active(struct drm_i915_private *dev_priv)
 {
 	return dev_priv->gvt;
diff --git a/drivers/gpu/drm/i915/i915_gem.c b/drivers/gpu/drm/i915/i915_gem.c
index 20fcac37c85a..f53d2af71cdb 100644
--- a/drivers/gpu/drm/i915/i915_gem.c
+++ b/drivers/gpu/drm/i915/i915_gem.c
@@ -3056,7 +3056,7 @@ void i915_gem_reset(struct drm_i915_private *dev_priv)
 
 	i915_gem_restore_fences(dev_priv);
 
-	if (dev_priv->gt.awake) {
+	if (intel_gt_awake(dev_priv)) {
 		intel_sanitize_gt_powersave(dev_priv);
 		intel_enable_gt_powersave(dev_priv);
 		if (INTEL_GEN(dev_priv) >= 6)
@@ -3238,7 +3238,7 @@ i915_gem_retire_work_handler(struct work_struct *work)
 	 * We do not need to do this test under locking as in the worst-case
 	 * we queue the retire worker once too often.
 	 */
-	if (READ_ONCE(dev_priv->gt.awake)) {
+	if (intel_gt_awake(dev_priv)) {
 		i915_queue_hangcheck(dev_priv);
 		queue_delayed_work(dev_priv->wq,
 				   &dev_priv->gt.retire_work,
@@ -3254,7 +3254,7 @@ i915_gem_idle_work_handler(struct work_struct *work)
 	struct drm_device *dev = &dev_priv->drm;
 	bool rearm_hangcheck;
 
-	if (!READ_ONCE(dev_priv->gt.awake))
+	if (!intel_gt_awake(dev_priv))
 		return;
 
 	/*
@@ -3292,8 +3292,8 @@ i915_gem_idle_work_handler(struct work_struct *work)
 	intel_engines_mark_idle(dev_priv);
 	i915_gem_timelines_mark_idle(dev_priv);
 
-	GEM_BUG_ON(!dev_priv->gt.awake);
-	dev_priv->gt.awake = false;
+	GEM_BUG_ON(!intel_gt_awake(dev_priv));
+	smp_store_mb(dev_priv->gt.awake, false);
 	rearm_hangcheck = false;
 
 	if (INTEL_GEN(dev_priv) >= 6)
@@ -3304,7 +3304,7 @@ out_unlock:
 
 out_rearm:
 	if (rearm_hangcheck) {
-		GEM_BUG_ON(!dev_priv->gt.awake);
+		GEM_BUG_ON(!intel_gt_awake(dev_priv));
 		i915_queue_hangcheck(dev_priv);
 	}
 }
@@ -3456,7 +3456,7 @@ int i915_gem_wait_for_idle(struct drm_i915_private *i915, unsigned int flags)
 	int ret;
 
 	/* If the device is asleep, we have no requests outstanding */
-	if (!READ_ONCE(i915->gt.awake))
+	if (!intel_gt_awake(i915))
 		return 0;
 
 	if (flags & I915_WAIT_LOCKED) {
@@ -4681,7 +4681,7 @@ int i915_gem_suspend(struct drm_i915_private *dev_priv)
 	/* Assert that we sucessfully flushed all the work and
 	 * reset the GPU back to its idle, low power state.
 	 */
-	WARN_ON(dev_priv->gt.awake);
+	WARN_ON(intel_gt_awake(dev_priv));
 	if (WARN_ON(!intel_engines_are_idle(dev_priv)))
 		i915_gem_set_wedged(dev_priv); /* no hope, discard everything */
 
@@ -4719,7 +4719,7 @@ void i915_gem_resume(struct drm_i915_private *dev_priv)
 {
 	struct drm_device *dev = &dev_priv->drm;
 
-	WARN_ON(dev_priv->gt.awake);
+	WARN_ON(intel_gt_awake(dev_priv));
 
 	mutex_lock(&dev->struct_mutex);
 	i915_gem_restore_gtt_mappings(dev_priv);
diff --git a/drivers/gpu/drm/i915/i915_gem_request.c b/drivers/gpu/drm/i915/i915_gem_request.c
index d140fcf5c6a3..65daf9983b91 100644
--- a/drivers/gpu/drm/i915/i915_gem_request.c
+++ b/drivers/gpu/drm/i915/i915_gem_request.c
@@ -246,13 +246,13 @@ int i915_gem_set_global_seqno(struct drm_device *dev, u32 seqno)
 
 static void mark_busy(struct drm_i915_private *i915)
 {
-	if (i915->gt.awake)
+	if (intel_gt_awake(i915))
 		return;
 
 	GEM_BUG_ON(!i915->gt.active_requests);
 
 	intel_runtime_pm_get_noresume(i915);
-	i915->gt.awake = true;
+	smp_store_mb(i915->gt.awake, true);
 
 	intel_enable_gt_powersave(i915);
 	i915_update_gfx_val(i915);
@@ -292,7 +292,7 @@ static void unreserve_engine(struct intel_engine_cs *engine)
 
 	if (!--i915->gt.active_requests) {
 		/* Cancel the mark_busy() from our reserve_engine() */
-		GEM_BUG_ON(!i915->gt.awake);
+		GEM_BUG_ON(!intel_gt_awake(i915));
 		mod_delayed_work(i915->wq,
 				 &i915->gt.idle_work,
 				 msecs_to_jiffies(100));
diff --git a/drivers/gpu/drm/i915/i915_gpu_error.c b/drivers/gpu/drm/i915/i915_gpu_error.c
index 653fb69e7ecb..79c255df8d9a 100644
--- a/drivers/gpu/drm/i915/i915_gpu_error.c
+++ b/drivers/gpu/drm/i915/i915_gpu_error.c
@@ -1673,7 +1673,7 @@ static void i915_error_capture_msg(struct drm_i915_private *dev_priv,
 static void i915_capture_gen_state(struct drm_i915_private *dev_priv,
 				   struct i915_gpu_state *error)
 {
-	error->awake = dev_priv->gt.awake;
+	error->awake = intel_gt_awake(dev_priv);
 	error->wakelock = atomic_read(&dev_priv->runtime_pm.wakeref_count);
 	error->suspended = dev_priv->runtime_pm.suspended;
 
diff --git a/drivers/gpu/drm/i915/intel_hangcheck.c b/drivers/gpu/drm/i915/intel_hangcheck.c
index 12ac270a5f93..2418b82e85aa 100644
--- a/drivers/gpu/drm/i915/intel_hangcheck.c
+++ b/drivers/gpu/drm/i915/intel_hangcheck.c
@@ -431,7 +431,7 @@ static void i915_hangcheck_elapsed(struct work_struct *work)
 	if (!i915_modparams.enable_hangcheck)
 		return;
 
-	if (!READ_ONCE(dev_priv->gt.awake))
+	if (!intel_gt_awake(dev_priv))
 		return;
 
 	if (i915_terminally_wedged(&dev_priv->gpu_error))
diff --git a/drivers/gpu/drm/i915/intel_lrc.c b/drivers/gpu/drm/i915/intel_lrc.c
index fbfcf88d7fe3..264ba148db06 100644
--- a/drivers/gpu/drm/i915/intel_lrc.c
+++ b/drivers/gpu/drm/i915/intel_lrc.c
@@ -779,7 +779,7 @@ static void intel_lrc_irq_handler(unsigned long data)
 	 * that all ELSP are drained i.e. we have processed the CSB,
 	 * before allowing ourselves to idle and calling intel_runtime_pm_put().
 	 */
-	GEM_BUG_ON(!dev_priv->gt.awake);
+	GEM_BUG_ON(!intel_gt_awake(dev_priv));
 
 	intel_uncore_forcewake_get(dev_priv, execlists->fw_domains);
 
diff --git a/drivers/gpu/drm/i915/intel_pm.c b/drivers/gpu/drm/i915/intel_pm.c
index 2fcff9788b6f..ecb807b3fcc2 100644
--- a/drivers/gpu/drm/i915/intel_pm.c
+++ b/drivers/gpu/drm/i915/intel_pm.c
@@ -7729,7 +7729,7 @@ bool i915_gpu_busy(void)
 
 	spin_lock_irq(&mchdev_lock);
 	if (i915_mch_dev)
-		ret = i915_mch_dev->gt.awake;
+		ret = intel_gt_awake(i915_mch_dev);
 	spin_unlock_irq(&mchdev_lock);
 
 	return ret;