summaryrefslogtreecommitdiff
path: root/drivers/iommu
diff options
context:
space:
mode:
authorJoerg Roedel <jroedel@suse.de>2017-06-02 18:13:37 +0200
committerJoerg Roedel <jroedel@suse.de>2017-06-08 14:39:51 +0200
commitfca6af6a5976dfa00182232f666b4f789c98bd0c (patch)
tree06412062076ed3cee36734665fa776cf6e0adbd7 /drivers/iommu
parenta6e3f6f030396c0576c729fd8ca4bfb654d35bfe (diff)
iommu/amd: Add per-domain timer to flush per-cpu queues
Add a timer to each dma_ops domain so that we flush unused IOTLB entries regularily, even if the queues don't get full all the time. Signed-off-by: Joerg Roedel <jroedel@suse.de>
Diffstat (limited to 'drivers/iommu')
-rw-r--r--drivers/iommu/amd_iommu.c84
1 files changed, 67 insertions, 17 deletions
diff --git a/drivers/iommu/amd_iommu.c b/drivers/iommu/amd_iommu.c
index 795208bd39bd..00c1796e07bf 100644
--- a/drivers/iommu/amd_iommu.c
+++ b/drivers/iommu/amd_iommu.c
@@ -182,6 +182,13 @@ struct dma_ops_domain {
* flushed out of the IOTLB of the domain.
*/
atomic64_t flush_finish_cnt;
+
+ /*
+ * Timer to make sure we don't keep IOVAs around unflushed
+ * for too long
+ */
+ struct timer_list flush_timer;
+ atomic_t flush_timer_on;
};
static struct iova_domain reserved_iova_ranges;
@@ -1834,6 +1841,14 @@ static int dma_ops_domain_alloc_flush_queue(struct dma_ops_domain *dom)
return 0;
}
+static void dma_ops_domain_flush_tlb(struct dma_ops_domain *dom)
+{
+ atomic64_inc(&dom->flush_start_cnt);
+ domain_flush_tlb(&dom->domain);
+ domain_flush_complete(&dom->domain);
+ atomic64_inc(&dom->flush_finish_cnt);
+}
+
static inline bool queue_ring_full(struct flush_queue *queue)
{
assert_spin_locked(&queue->lock);
@@ -1875,22 +1890,12 @@ static inline void queue_ring_remove_head(struct flush_queue *queue)
queue->head = (queue->head + 1) % FLUSH_QUEUE_SIZE;
}
-static void queue_add(struct dma_ops_domain *dom,
- unsigned long address, unsigned long pages)
+static void queue_ring_free_flushed(struct dma_ops_domain *dom,
+ struct flush_queue *queue)
{
- struct flush_queue *queue;
- unsigned long flags;
- u64 counter;
+ u64 counter = atomic64_read(&dom->flush_finish_cnt);
int idx;
- pages = __roundup_pow_of_two(pages);
- address >>= PAGE_SHIFT;
-
- counter = atomic64_read(&dom->flush_finish_cnt);
-
- queue = get_cpu_ptr(dom->flush_queue);
- spin_lock_irqsave(&queue->lock, flags);
-
queue_ring_for_each(idx, queue) {
/*
* This assumes that counter values in the ring-buffer are
@@ -1905,12 +1910,25 @@ static void queue_add(struct dma_ops_domain *dom,
queue_ring_remove_head(queue);
}
+}
+
+static void queue_add(struct dma_ops_domain *dom,
+ unsigned long address, unsigned long pages)
+{
+ struct flush_queue *queue;
+ unsigned long flags;
+ int idx;
+
+ pages = __roundup_pow_of_two(pages);
+ address >>= PAGE_SHIFT;
+
+ queue = get_cpu_ptr(dom->flush_queue);
+ spin_lock_irqsave(&queue->lock, flags);
+
+ queue_ring_free_flushed(dom, queue);
if (queue_ring_full(queue)) {
- atomic64_inc(&dom->flush_start_cnt);
- domain_flush_tlb(&dom->domain);
- domain_flush_complete(&dom->domain);
- atomic64_inc(&dom->flush_finish_cnt);
+ dma_ops_domain_flush_tlb(dom);
queue_release(dom, queue);
}
@@ -1921,9 +1939,33 @@ static void queue_add(struct dma_ops_domain *dom,
queue->entries[idx].counter = atomic64_read(&dom->flush_start_cnt);
spin_unlock_irqrestore(&queue->lock, flags);
+
+ if (atomic_cmpxchg(&dom->flush_timer_on, 0, 1) == 0)
+ mod_timer(&dom->flush_timer, jiffies + msecs_to_jiffies(10));
+
put_cpu_ptr(dom->flush_queue);
}
+static void queue_flush_timeout(unsigned long data)
+{
+ struct dma_ops_domain *dom = (struct dma_ops_domain *)data;
+ int cpu;
+
+ atomic_set(&dom->flush_timer_on, 0);
+
+ dma_ops_domain_flush_tlb(dom);
+
+ for_each_possible_cpu(cpu) {
+ struct flush_queue *queue;
+ unsigned long flags;
+
+ queue = per_cpu_ptr(dom->flush_queue, cpu);
+ spin_lock_irqsave(&queue->lock, flags);
+ queue_ring_free_flushed(dom, queue);
+ spin_unlock_irqrestore(&queue->lock, flags);
+ }
+}
+
/*
* Free a domain, only used if something went wrong in the
* allocation path and we need to free an already allocated page table
@@ -1935,6 +1977,9 @@ static void dma_ops_domain_free(struct dma_ops_domain *dom)
del_domain_from_list(&dom->domain);
+ if (timer_pending(&dom->flush_timer))
+ del_timer(&dom->flush_timer);
+
dma_ops_domain_free_flush_queue(dom);
put_iova_domain(&dom->iovad);
@@ -1978,6 +2023,11 @@ static struct dma_ops_domain *dma_ops_domain_alloc(void)
if (dma_ops_domain_alloc_flush_queue(dma_dom))
goto free_dma_dom;
+ setup_timer(&dma_dom->flush_timer, queue_flush_timeout,
+ (unsigned long)dma_dom);
+
+ atomic_set(&dma_dom->flush_timer_on, 0);
+
add_domain_to_list(&dma_dom->domain);
return dma_dom;