diff options
| -rw-r--r-- | drivers/gpu/drm/i915/i915_gem.c | 22 | ||||
| -rw-r--r-- | drivers/gpu/drm/i915/i915_scheduler.c | 178 | ||||
| -rw-r--r-- | drivers/gpu/drm/i915/i915_scheduler.h | 3 |
3 files changed, 203 insertions, 0 deletions
diff --git a/drivers/gpu/drm/i915/i915_gem.c b/drivers/gpu/drm/i915/i915_gem.c index 724a6a2b9084..9271b927cf5e 100644 --- a/drivers/gpu/drm/i915/i915_gem.c +++ b/drivers/gpu/drm/i915/i915_gem.c @@ -3874,6 +3874,10 @@ int i915_gpu_idle(struct drm_device *dev) /* Flush everything onto the inactive list. */ for_each_engine(engine, dev_priv) { + ret = i915_scheduler_flush(engine, true); + if (ret < 0) + return ret; + if (!i915.enable_execlists) { struct drm_i915_gem_request *req; @@ -4612,6 +4616,7 @@ i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file) struct drm_i915_gem_request *request, *target = NULL; unsigned reset_counter; int ret; + struct intel_engine_cs *engine; ret = i915_gem_wait_for_error(&dev_priv->gpu_error); if (ret) @@ -4621,6 +4626,23 @@ i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file) if (ret) return ret; + for_each_engine(engine, dev_priv) { + /* + * Flush out scheduler entries that are getting 'stale'. Note + * that the following recent_enough test will only check + * against the time at which the request was submitted to the + * hardware (i.e. when it left the scheduler) not the time it + * was submitted to the driver. + * + * Also, there is not much point worring about busy return + * codes from the scheduler flush call. Even if more work + * cannot be submitted right now for whatever reason, we + * still want to throttle against stale work that has already + * been submitted. + */ + i915_scheduler_flush_stamp(engine, recent_enough, false); + } + spin_lock(&file_priv->mm.lock); list_for_each_entry(request, &file_priv->mm.request_list, client_list) { if (time_after_eq(request->emitted_jiffies, recent_enough)) diff --git a/drivers/gpu/drm/i915/i915_scheduler.c b/drivers/gpu/drm/i915/i915_scheduler.c index cf8b50aba9bd..580b9d24031d 100644 --- a/drivers/gpu/drm/i915/i915_scheduler.c +++ b/drivers/gpu/drm/i915/i915_scheduler.c @@ -342,6 +342,10 @@ static int i915_scheduler_pop_from_queue_locked(struct intel_engine_cs *engine, * attempting to acquire a mutex while holding a spin lock is a Bad Idea. * And releasing the one before acquiring the other leads to other code * being run and interfering. + * + * Hence any caller that does not already have the mutex lock for other + * reasons should call i915_scheduler_submit_unlocked() instead in order to + * obtain the lock first. */ static int i915_scheduler_submit(struct intel_engine_cs *engine) { @@ -464,6 +468,22 @@ error: return ret; } +static int i915_scheduler_submit_unlocked(struct intel_engine_cs *engine) +{ + struct drm_device *dev = engine->dev; + int ret; + + ret = i915_mutex_lock_interruptible(dev); + if (ret) + return ret; + + ret = i915_scheduler_submit(engine); + + mutex_unlock(&dev->struct_mutex); + + return ret; +} + static void i915_generate_dependencies(struct i915_scheduler *scheduler, struct i915_scheduler_queue_entry *node, uint32_t engine) @@ -909,6 +929,164 @@ void i915_scheduler_work_handler(struct work_struct *work) i915_scheduler_process_work(engine); } +static int i915_scheduler_submit_max_priority(struct intel_engine_cs *engine, + bool is_locked) +{ + struct i915_scheduler_queue_entry *node; + struct drm_i915_private *dev_priv = to_i915(engine->dev); + struct i915_scheduler *scheduler = dev_priv->scheduler; + int ret, count = 0; + bool found; + + do { + found = false; + spin_lock_irq(&scheduler->lock); + for_each_scheduler_node(node, engine->id) { + if (!I915_SQS_IS_QUEUED(node)) + continue; + + if (node->priority < scheduler->priority_level_max) + continue; + + found = true; + break; + } + spin_unlock_irq(&scheduler->lock); + + if (!found) + break; + + if (is_locked) + ret = i915_scheduler_submit(engine); + else + ret = i915_scheduler_submit_unlocked(engine); + if (ret < 0) + return ret; + + count += ret; + } while (found); + + return count; +} + +/** + * i915_scheduler_flush_stamp - force requests of a given age through the + * scheduler. + * @engine: Engine to be flushed + * @target: Jiffy based time stamp to flush up to + * @is_locked: Is the driver mutex lock held? + * DRM has a throttle by age of request facility. This requires waiting for + * outstanding work over a given age. This function helps that by forcing + * queued batch buffers over said age through the system. + * Returns zero on success or -EAGAIN if the scheduler is busy (e.g. waiting + * for a pre-emption event to complete) but the mutex lock is held which + * would prevent the scheduler's asynchronous processing from completing. + */ +int i915_scheduler_flush_stamp(struct intel_engine_cs *engine, + unsigned long target, + bool is_locked) +{ + struct i915_scheduler_queue_entry *node; + struct drm_i915_private *dev_priv; + struct i915_scheduler *scheduler; + int flush_count = 0; + + if (!engine) + return -EINVAL; + + dev_priv = to_i915(engine->dev); + scheduler = dev_priv->scheduler; + + if (!scheduler) + return 0; + + if (is_locked && (scheduler->flags[engine->id] & I915_SF_SUBMITTING)) { + /* + * Scheduler is busy already submitting another batch, + * come back later rather than going recursive... + */ + return -EAGAIN; + } + + spin_lock_irq(&scheduler->lock); + i915_scheduler_priority_bump_clear(scheduler); + for_each_scheduler_node(node, engine->id) { + if (!I915_SQS_IS_QUEUED(node)) + continue; + + if (node->stamp > target) + continue; + + flush_count = i915_scheduler_priority_bump(scheduler, + node, scheduler->priority_level_max); + } + spin_unlock_irq(&scheduler->lock); + + if (flush_count) { + DRM_DEBUG_DRIVER("<%s> Bumped %d entries\n", engine->name, flush_count); + flush_count = i915_scheduler_submit_max_priority(engine, is_locked); + } + + return flush_count; +} + +/** + * i915_scheduler_flush - force all requests through the scheduler. + * @engine: Engine to be flushed + * @is_locked: Is the driver mutex lock held? + * For various reasons it is sometimes necessary to the scheduler out, e.g. + * due to engine reset. + * Returns zero on success or -EAGAIN if the scheduler is busy (e.g. waiting + * for a pre-emption event to complete) but the mutex lock is held which + * would prevent the scheduler's asynchronous processing from completing. + */ +int i915_scheduler_flush(struct intel_engine_cs *engine, bool is_locked) +{ + struct i915_scheduler_queue_entry *node; + struct drm_i915_private *dev_priv; + struct i915_scheduler *scheduler; + bool found; + int ret; + uint32_t count = 0; + + if (!engine) + return -EINVAL; + + dev_priv = to_i915(engine->dev); + scheduler = dev_priv->scheduler; + + if (!scheduler) + return 0; + + WARN_ON(is_locked && (scheduler->flags[engine->id] & I915_SF_SUBMITTING)); + + do { + found = false; + spin_lock_irq(&scheduler->lock); + for_each_scheduler_node(node, engine->id) { + if (!I915_SQS_IS_QUEUED(node)) + continue; + + found = true; + break; + } + spin_unlock_irq(&scheduler->lock); + + if (found) { + if (is_locked) + ret = i915_scheduler_submit(engine); + else + ret = i915_scheduler_submit_unlocked(engine); + if (ret < 0) + return ret; + + count += ret; + } + } while (found); + + return count; +} + /** * i915_scheduler_is_mutex_required - query if it is safe to hold the mutex * lock while waiting for the given request. diff --git a/drivers/gpu/drm/i915/i915_scheduler.h b/drivers/gpu/drm/i915/i915_scheduler.h index 38e860dfde77..92dd5df1fcf6 100644 --- a/drivers/gpu/drm/i915/i915_scheduler.h +++ b/drivers/gpu/drm/i915/i915_scheduler.h @@ -111,6 +111,9 @@ int i915_scheduler_queue_execbuffer(struct i915_scheduler_queue_entry *qe); bool i915_scheduler_notify_request(struct drm_i915_gem_request *req); void i915_scheduler_wakeup(struct drm_device *dev); void i915_scheduler_work_handler(struct work_struct *work); +int i915_scheduler_flush(struct intel_engine_cs *engine, bool is_locked); +int i915_scheduler_flush_stamp(struct intel_engine_cs *engine, + unsigned long stamp, bool is_locked); bool i915_scheduler_is_mutex_required(struct drm_i915_gem_request *req); #endif /* _I915_SCHEDULER_H_ */ |