1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
|
// SPDX-License-Identifier: GPL-2.0
/*
* KVM coalesced MMIO
*
* Copyright (c) 2008 Bull S.A.S.
* Copyright 2009 Red Hat, Inc. and/or its affiliates.
*
* Author: Laurent Vivier <Laurent.Vivier@bull.net>
*
*/
#include <kvm/iodev.h>
#include <linux/kvm_host.h>
#include <linux/slab.h>
#include <linux/kvm.h>
#include "coalesced_mmio.h"
static inline struct kvm_coalesced_mmio_dev *to_mmio(struct kvm_io_device *dev)
{
return container_of(dev, struct kvm_coalesced_mmio_dev, dev);
}
static int coalesced_mmio_in_range(struct kvm_coalesced_mmio_dev *dev,
gpa_t addr, int len)
{
/* is it in a batchable area ?
* (addr,len) is fully included in
* (zone->addr, zone->size)
*/
if (len < 0)
return 0;
if (addr + len < addr)
return 0;
if (addr < dev->zone.addr)
return 0;
if (addr + len > dev->zone.addr + dev->zone.size)
return 0;
return 1;
}
static int coalesced_mmio_write(struct kvm_vcpu *vcpu,
struct kvm_io_device *this, gpa_t addr,
int len, const void *val)
{
struct kvm_coalesced_mmio_dev *dev = to_mmio(this);
struct kvm_coalesced_mmio_ring *ring = dev->kvm->coalesced_mmio_ring;
__u32 insert;
if (!coalesced_mmio_in_range(dev, addr, len))
return -EOPNOTSUPP;
spin_lock(&dev->kvm->ring_lock);
/*
* last is the index of the entry to fill. Verify userspace hasn't
* set last to be out of range, and that there is room in the ring.
* Leave one entry free in the ring so that userspace can differentiate
* between an empty ring and a full ring.
*/
insert = READ_ONCE(ring->last);
if (insert >= KVM_COALESCED_MMIO_MAX ||
(insert + 1) % KVM_COALESCED_MMIO_MAX == READ_ONCE(ring->first)) {
spin_unlock(&dev->kvm->ring_lock);
return -EOPNOTSUPP;
}
/* copy data in first free entry of the ring */
ring->coalesced_mmio[insert].phys_addr = addr;
ring->coalesced_mmio[insert].len = len;
memcpy(ring->coalesced_mmio[insert].data, val, len);
ring->coalesced_mmio[insert].pio = dev->zone.pio;
smp_wmb();
ring->last = (insert + 1) % KVM_COALESCED_MMIO_MAX;
spin_unlock(&dev->kvm->ring_lock);
return 0;
}
static void coalesced_mmio_destructor(struct kvm_io_device *this)
{
struct kvm_coalesced_mmio_dev *dev = to_mmio(this);
list_del(&dev->list);
kfree(dev);
}
static const struct kvm_io_device_ops coalesced_mmio_ops = {
.write = coalesced_mmio_write,
.destructor = coalesced_mmio_destructor,
};
int kvm_coalesced_mmio_init(struct kvm *kvm)
{
struct page *page;
page = alloc_page(GFP_KERNEL_ACCOUNT | __GFP_ZERO);
if (!page)
return -ENOMEM;
kvm->coalesced_mmio_ring = page_address(page);
/*
* We're using this spinlock to sync access to the coalesced ring.
* The list doesn't need its own lock since device registration and
* unregistration should only happen when kvm->slots_lock is held.
*/
spin_lock_init(&kvm->ring_lock);
INIT_LIST_HEAD(&kvm->coalesced_zones);
return 0;
}
void kvm_coalesced_mmio_free(struct kvm *kvm)
{
if (kvm->coalesced_mmio_ring)
free_page((unsigned long)kvm->coalesced_mmio_ring);
}
int kvm_vm_ioctl_register_coalesced_mmio(struct kvm *kvm,
struct kvm_coalesced_mmio_zone *zone)
{
int ret;
struct kvm_coalesced_mmio_dev *dev;
if (zone->pio != 1 && zone->pio != 0)
return -EINVAL;
dev = kzalloc(sizeof(struct kvm_coalesced_mmio_dev),
GFP_KERNEL_ACCOUNT);
if (!dev)
return -ENOMEM;
kvm_iodevice_init(&dev->dev, &coalesced_mmio_ops);
dev->kvm = kvm;
dev->zone = *zone;
mutex_lock(&kvm->slots_lock);
ret = kvm_io_bus_register_dev(kvm,
zone->pio ? KVM_PIO_BUS : KVM_MMIO_BUS,
zone->addr, zone->size, &dev->dev);
if (ret < 0)
goto out_free_dev;
list_add_tail(&dev->list, &kvm->coalesced_zones);
mutex_unlock(&kvm->slots_lock);
return 0;
out_free_dev:
mutex_unlock(&kvm->slots_lock);
kfree(dev);
return ret;
}
int kvm_vm_ioctl_unregister_coalesced_mmio(struct kvm *kvm,
struct kvm_coalesced_mmio_zone *zone)
{
struct kvm_coalesced_mmio_dev *dev, *tmp;
int r;
if (zone->pio != 1 && zone->pio != 0)
return -EINVAL;
mutex_lock(&kvm->slots_lock);
list_for_each_entry_safe(dev, tmp, &kvm->coalesced_zones, list) {
if (zone->pio == dev->zone.pio &&
coalesced_mmio_in_range(dev, zone->addr, zone->size)) {
r = kvm_io_bus_unregister_dev(kvm,
zone->pio ? KVM_PIO_BUS : KVM_MMIO_BUS, &dev->dev);
/*
* On failure, unregister destroys all devices on the
* bus, including the target device. There's no need
* to restart the walk as there aren't any zones left.
*/
if (r)
break;
}
}
mutex_unlock(&kvm->slots_lock);
/*
* Ignore the result of kvm_io_bus_unregister_dev(), from userspace's
* perspective, the coalesced MMIO is most definitely unregistered.
*/
return 0;
}
|
