blob: 07f4e60b30992a2ab93e2759764d9dad182f8644 [file] [log] [blame]
aliguori967f97f2008-12-04 19:38:57 +00001/*
2 * Virtio Support
3 *
4 * Copyright IBM, Corp. 2007
5 *
6 * Authors:
7 * Anthony Liguori <aliguori@us.ibm.com>
8 *
9 * This work is licensed under the terms of the GNU GPL, version 2. See
10 * the COPYING file in the top-level directory.
11 *
12 */
13
Peter Maydell9b8bfe22016-01-26 18:17:07 +000014#include "qemu/osdep.h"
Markus Armbrusterda34e652016-03-14 09:01:28 +010015#include "qapi/error.h"
Paolo Bonzini4771d752016-01-19 21:51:44 +010016#include "cpu.h"
Stefan Hajnoczi64979a42010-05-24 13:19:21 +010017#include "trace.h"
Edgar E. Iglesiasfdfba1a2013-11-15 14:46:38 +010018#include "exec/address-spaces.h"
Paolo Bonzini1de7afc2012-12-17 18:20:00 +010019#include "qemu/error-report.h"
Felipe Franciosid68cdae2020-10-28 13:47:03 +000020#include "qemu/log.h"
Markus Armbrusterdb725812019-08-12 07:23:50 +020021#include "qemu/main-loop.h"
Markus Armbruster0b8fa322019-05-23 16:35:07 +020022#include "qemu/module.h"
Paolo Bonzini0d09e412013-02-05 17:06:20 +010023#include "hw/virtio/virtio.h"
Markus Armbrusterca77ee22019-08-12 07:23:39 +020024#include "migration/qemu-file-types.h"
Paolo Bonzini1de7afc2012-12-17 18:20:00 +010025#include "qemu/atomic.h"
Paolo Bonzini0d09e412013-02-05 17:06:20 +010026#include "hw/virtio/virtio-bus.h"
Markus Armbrustera27bd6c2019-08-12 07:23:51 +020027#include "hw/qdev-properties.h"
Rusty Russellcee3ca02014-06-24 19:40:16 +020028#include "hw/virtio/virtio-access.h"
Jason Wang8607f5c2016-12-30 18:09:10 +080029#include "sysemu/dma.h"
Markus Armbruster54d31232019-08-12 07:23:59 +020030#include "sysemu/runstate.h"
Cornelia Huck7c78bdd2020-07-07 12:54:45 +020031#include "standard-headers/linux/virtio_ids.h"
aliguori967f97f2008-12-04 19:38:57 +000032
Peter Maydell6ce69d12013-07-16 13:25:08 +010033/*
34 * The alignment to use between consumer and producer parts of vring.
35 * x86 pagesize again. This is the default, used by transports like PCI
36 * which don't provide a means for the guest to tell the host the alignment.
37 */
aliguorif46f15b2008-12-04 19:58:45 +000038#define VIRTIO_PCI_VRING_ALIGN 4096
39
aliguori967f97f2008-12-04 19:38:57 +000040typedef struct VRingDesc
41{
42 uint64_t addr;
43 uint32_t len;
44 uint16_t flags;
45 uint16_t next;
46} VRingDesc;
47
Wei Xua40dcec2019-10-25 10:35:20 +020048typedef struct VRingPackedDesc {
49 uint64_t addr;
50 uint32_t len;
51 uint16_t id;
52 uint16_t flags;
53} VRingPackedDesc;
54
aliguori967f97f2008-12-04 19:38:57 +000055typedef struct VRingAvail
56{
57 uint16_t flags;
58 uint16_t idx;
Philippe Mathieu-Daudéf7795e42020-03-04 16:38:15 +010059 uint16_t ring[];
aliguori967f97f2008-12-04 19:38:57 +000060} VRingAvail;
61
62typedef struct VRingUsedElem
63{
64 uint32_t id;
65 uint32_t len;
66} VRingUsedElem;
67
68typedef struct VRingUsed
69{
70 uint16_t flags;
71 uint16_t idx;
Philippe Mathieu-Daudéf7795e42020-03-04 16:38:15 +010072 VRingUsedElem ring[];
aliguori967f97f2008-12-04 19:38:57 +000073} VRingUsed;
74
Paolo Bonzinic611c762017-01-27 16:40:17 +010075typedef struct VRingMemoryRegionCaches {
76 struct rcu_head rcu;
77 MemoryRegionCache desc;
78 MemoryRegionCache avail;
79 MemoryRegionCache used;
80} VRingMemoryRegionCaches;
81
aliguori967f97f2008-12-04 19:38:57 +000082typedef struct VRing
83{
84 unsigned int num;
Cornelia Huck46c5d082015-09-11 15:16:41 +020085 unsigned int num_default;
Peter Maydell6ce69d12013-07-16 13:25:08 +010086 unsigned int align;
Avi Kivitya8170e52012-10-23 12:30:10 +020087 hwaddr desc;
88 hwaddr avail;
89 hwaddr used;
Paolo Bonzinic611c762017-01-27 16:40:17 +010090 VRingMemoryRegionCaches *caches;
aliguori967f97f2008-12-04 19:38:57 +000091} VRing;
92
Wei Xua40dcec2019-10-25 10:35:20 +020093typedef struct VRingPackedDescEvent {
94 uint16_t off_wrap;
95 uint16_t flags;
96} VRingPackedDescEvent ;
97
aliguori967f97f2008-12-04 19:38:57 +000098struct VirtQueue
99{
100 VRing vring;
Jason Wang86044b22019-10-25 10:35:24 +0200101 VirtQueueElement *used_elems;
Vincenzo Maffionebe1fea92016-01-31 11:29:05 +0100102
103 /* Next head to pop */
aliguori967f97f2008-12-04 19:38:57 +0000104 uint16_t last_avail_idx;
Wei Xua40dcec2019-10-25 10:35:20 +0200105 bool last_avail_wrap_counter;
Vincenzo Maffioneb796fcd2016-01-31 11:29:04 +0100106
Vincenzo Maffionebe1fea92016-01-31 11:29:05 +0100107 /* Last avail_idx read from VQ. */
108 uint16_t shadow_avail_idx;
Wei Xua40dcec2019-10-25 10:35:20 +0200109 bool shadow_avail_wrap_counter;
Vincenzo Maffionebe1fea92016-01-31 11:29:05 +0100110
Vincenzo Maffioneb796fcd2016-01-31 11:29:04 +0100111 uint16_t used_idx;
Wei Xua40dcec2019-10-25 10:35:20 +0200112 bool used_wrap_counter;
Vincenzo Maffioneb796fcd2016-01-31 11:29:04 +0100113
Michael S. Tsirkinbcbabae2011-06-12 16:21:57 +0300114 /* Last used index value we have signalled on */
115 uint16_t signalled_used;
116
117 /* Last used index value we have signalled on */
118 bool signalled_used_valid;
119
Stefan Hajnoczi332fa822017-01-12 11:46:10 +0000120 /* Notification enabled? */
121 bool notification;
Michael S. Tsirkinbcbabae2011-06-12 16:21:57 +0300122
Jason Wange78a2b42013-01-30 19:12:37 +0800123 uint16_t queue_index;
124
Halil Pasice66bcc42016-12-19 16:44:44 +0100125 unsigned int inuse;
Michael S. Tsirkinbcbabae2011-06-12 16:21:57 +0300126
Michael S. Tsirkin7055e682009-06-21 19:50:13 +0300127 uint16_t vector;
Fam Zhengbf1780b2016-07-13 13:09:43 +0800128 VirtIOHandleOutput handle_output;
Fam Zheng07931692017-02-09 16:40:47 +0800129 VirtIOHandleAIOOutput handle_aio_output;
Michael S. Tsirkin1cbdabe2010-03-17 13:08:02 +0200130 VirtIODevice *vdev;
131 EventNotifier guest_notifier;
132 EventNotifier host_notifier;
Stefan Hajnoczifcccb272019-11-05 15:09:46 +0100133 bool host_notifier_enabled;
Jason Wange0d686b2015-04-23 14:21:46 +0800134 QLIST_ENTRY(VirtQueue) node;
aliguori967f97f2008-12-04 19:38:57 +0000135};
136
Paolo Bonzinic611c762017-01-27 16:40:17 +0100137static void virtio_free_region_cache(VRingMemoryRegionCaches *caches)
138{
139 if (!caches) {
140 return;
141 }
142
143 address_space_cache_destroy(&caches->desc);
144 address_space_cache_destroy(&caches->avail);
145 address_space_cache_destroy(&caches->used);
146 g_free(caches);
147}
148
Paolo Bonzini45641db2018-05-15 15:25:31 +0200149static void virtio_virtqueue_reset_region_cache(struct VirtQueue *vq)
150{
151 VRingMemoryRegionCaches *caches;
152
Stefan Hajnoczid73415a2020-09-23 11:56:46 +0100153 caches = qatomic_read(&vq->vring.caches);
154 qatomic_rcu_set(&vq->vring.caches, NULL);
Paolo Bonzini45641db2018-05-15 15:25:31 +0200155 if (caches) {
156 call_rcu(caches, virtio_free_region_cache, rcu);
157 }
158}
159
Paolo Bonzinic611c762017-01-27 16:40:17 +0100160static void virtio_init_region_cache(VirtIODevice *vdev, int n)
161{
162 VirtQueue *vq = &vdev->vq[n];
163 VRingMemoryRegionCaches *old = vq->vring.caches;
Paolo Bonzini45641db2018-05-15 15:25:31 +0200164 VRingMemoryRegionCaches *new = NULL;
Paolo Bonzinic611c762017-01-27 16:40:17 +0100165 hwaddr addr, size;
Jason Wange45da652017-03-15 19:48:32 +0800166 int64_t len;
Jason Wang86044b22019-10-25 10:35:24 +0200167 bool packed;
Paolo Bonzinic611c762017-01-27 16:40:17 +0100168
Paolo Bonzinic611c762017-01-27 16:40:17 +0100169
170 addr = vq->vring.desc;
171 if (!addr) {
Paolo Bonzini45641db2018-05-15 15:25:31 +0200172 goto out_no_cache;
Paolo Bonzinic611c762017-01-27 16:40:17 +0100173 }
174 new = g_new0(VRingMemoryRegionCaches, 1);
175 size = virtio_queue_get_desc_size(vdev, n);
Jason Wang86044b22019-10-25 10:35:24 +0200176 packed = virtio_vdev_has_feature(vq->vdev, VIRTIO_F_RING_PACKED) ?
177 true : false;
Jason Wange45da652017-03-15 19:48:32 +0800178 len = address_space_cache_init(&new->desc, vdev->dma_as,
Jason Wang86044b22019-10-25 10:35:24 +0200179 addr, size, packed);
Jason Wange45da652017-03-15 19:48:32 +0800180 if (len < size) {
181 virtio_error(vdev, "Cannot map desc");
182 goto err_desc;
183 }
Paolo Bonzinic611c762017-01-27 16:40:17 +0100184
Wei Xuf90cda62019-10-25 10:35:21 +0200185 size = virtio_queue_get_used_size(vdev, n);
Jason Wange45da652017-03-15 19:48:32 +0800186 len = address_space_cache_init(&new->used, vdev->dma_as,
187 vq->vring.used, size, true);
188 if (len < size) {
189 virtio_error(vdev, "Cannot map used");
190 goto err_used;
191 }
Paolo Bonzinic611c762017-01-27 16:40:17 +0100192
Wei Xuf90cda62019-10-25 10:35:21 +0200193 size = virtio_queue_get_avail_size(vdev, n);
Jason Wange45da652017-03-15 19:48:32 +0800194 len = address_space_cache_init(&new->avail, vdev->dma_as,
195 vq->vring.avail, size, false);
196 if (len < size) {
197 virtio_error(vdev, "Cannot map avail");
198 goto err_avail;
199 }
Paolo Bonzinic611c762017-01-27 16:40:17 +0100200
Stefan Hajnoczid73415a2020-09-23 11:56:46 +0100201 qatomic_rcu_set(&vq->vring.caches, new);
Paolo Bonzinic611c762017-01-27 16:40:17 +0100202 if (old) {
203 call_rcu(old, virtio_free_region_cache, rcu);
204 }
Jason Wange45da652017-03-15 19:48:32 +0800205 return;
206
207err_avail:
Paolo Bonzini45641db2018-05-15 15:25:31 +0200208 address_space_cache_destroy(&new->avail);
Jason Wange45da652017-03-15 19:48:32 +0800209err_used:
Paolo Bonzini45641db2018-05-15 15:25:31 +0200210 address_space_cache_destroy(&new->used);
Jason Wange45da652017-03-15 19:48:32 +0800211err_desc:
Paolo Bonzini45641db2018-05-15 15:25:31 +0200212 address_space_cache_destroy(&new->desc);
213out_no_cache:
Jason Wange45da652017-03-15 19:48:32 +0800214 g_free(new);
Paolo Bonzini45641db2018-05-15 15:25:31 +0200215 virtio_virtqueue_reset_region_cache(vq);
Paolo Bonzinic611c762017-01-27 16:40:17 +0100216}
217
aliguori967f97f2008-12-04 19:38:57 +0000218/* virt queue functions */
Cornelia Huckab223c92015-06-04 12:34:12 +0200219void virtio_queue_update_rings(VirtIODevice *vdev, int n)
aliguori967f97f2008-12-04 19:38:57 +0000220{
Cornelia Huckab223c92015-06-04 12:34:12 +0200221 VRing *vring = &vdev->vq[n].vring;
Paul Brook53c25ce2009-05-18 14:51:59 +0100222
Prasad J Pandit758ead32017-11-29 23:14:27 +0530223 if (!vring->num || !vring->desc || !vring->align) {
Cornelia Huckab223c92015-06-04 12:34:12 +0200224 /* not yet setup -> nothing to do */
225 return;
226 }
227 vring->avail = vring->desc + vring->num * sizeof(VRingDesc);
228 vring->used = vring_align(vring->avail +
229 offsetof(VRingAvail, ring[vring->num]),
230 vring->align);
Paolo Bonzinic611c762017-01-27 16:40:17 +0100231 virtio_init_region_cache(vdev, n);
aliguori967f97f2008-12-04 19:38:57 +0000232}
233
Paolo Bonzini97cd9652017-01-27 16:40:20 +0100234/* Called within rcu_read_lock(). */
Jason Wang86044b22019-10-25 10:35:24 +0200235static void vring_split_desc_read(VirtIODevice *vdev, VRingDesc *desc,
236 MemoryRegionCache *cache, int i)
aliguori967f97f2008-12-04 19:38:57 +0000237{
Paolo Bonzini5eba0402017-01-27 16:40:16 +0100238 address_space_read_cached(cache, i * sizeof(VRingDesc),
239 desc, sizeof(VRingDesc));
Paolo Bonziniaa570d62016-01-31 11:29:03 +0100240 virtio_tswap64s(vdev, &desc->addr);
241 virtio_tswap32s(vdev, &desc->len);
242 virtio_tswap16s(vdev, &desc->flags);
243 virtio_tswap16s(vdev, &desc->next);
aliguori967f97f2008-12-04 19:38:57 +0000244}
245
Jason Wang683f7662019-10-25 10:35:25 +0200246static void vring_packed_event_read(VirtIODevice *vdev,
247 MemoryRegionCache *cache,
248 VRingPackedDescEvent *e)
249{
250 hwaddr off_off = offsetof(VRingPackedDescEvent, off_wrap);
251 hwaddr off_flags = offsetof(VRingPackedDescEvent, flags);
252
253 address_space_read_cached(cache, off_flags, &e->flags,
254 sizeof(e->flags));
255 /* Make sure flags is seen before off_wrap */
256 smp_rmb();
257 address_space_read_cached(cache, off_off, &e->off_wrap,
258 sizeof(e->off_wrap));
259 virtio_tswap16s(vdev, &e->off_wrap);
260 virtio_tswap16s(vdev, &e->flags);
261}
262
263static void vring_packed_off_wrap_write(VirtIODevice *vdev,
264 MemoryRegionCache *cache,
265 uint16_t off_wrap)
266{
267 hwaddr off = offsetof(VRingPackedDescEvent, off_wrap);
268
269 virtio_tswap16s(vdev, &off_wrap);
270 address_space_write_cached(cache, off, &off_wrap, sizeof(off_wrap));
271 address_space_cache_invalidate(cache, off, sizeof(off_wrap));
272}
273
274static void vring_packed_flags_write(VirtIODevice *vdev,
275 MemoryRegionCache *cache, uint16_t flags)
276{
277 hwaddr off = offsetof(VRingPackedDescEvent, flags);
278
279 virtio_tswap16s(vdev, &flags);
280 address_space_write_cached(cache, off, &flags, sizeof(flags));
281 address_space_cache_invalidate(cache, off, sizeof(flags));
282}
283
Jason Wang86044b22019-10-25 10:35:24 +0200284/* Called within rcu_read_lock(). */
Jason Wange0e2d642017-03-15 19:48:31 +0800285static VRingMemoryRegionCaches *vring_get_region_caches(struct VirtQueue *vq)
286{
Stefan Hajnoczid73415a2020-09-23 11:56:46 +0100287 return qatomic_rcu_read(&vq->vring.caches);
Jason Wange0e2d642017-03-15 19:48:31 +0800288}
Stefan Hajnocziabdd16f2020-02-07 10:46:19 +0000289
Paolo Bonzini97cd9652017-01-27 16:40:20 +0100290/* Called within rcu_read_lock(). */
aliguori967f97f2008-12-04 19:38:57 +0000291static inline uint16_t vring_avail_flags(VirtQueue *vq)
292{
Jason Wange0e2d642017-03-15 19:48:31 +0800293 VRingMemoryRegionCaches *caches = vring_get_region_caches(vq);
Paolo Bonzini97cd9652017-01-27 16:40:20 +0100294 hwaddr pa = offsetof(VRingAvail, flags);
Stefan Hajnocziabdd16f2020-02-07 10:46:19 +0000295
296 if (!caches) {
297 return 0;
298 }
299
Paolo Bonzini97cd9652017-01-27 16:40:20 +0100300 return virtio_lduw_phys_cached(vq->vdev, &caches->avail, pa);
aliguori967f97f2008-12-04 19:38:57 +0000301}
302
Paolo Bonzini97cd9652017-01-27 16:40:20 +0100303/* Called within rcu_read_lock(). */
aliguori967f97f2008-12-04 19:38:57 +0000304static inline uint16_t vring_avail_idx(VirtQueue *vq)
305{
Jason Wange0e2d642017-03-15 19:48:31 +0800306 VRingMemoryRegionCaches *caches = vring_get_region_caches(vq);
Paolo Bonzini97cd9652017-01-27 16:40:20 +0100307 hwaddr pa = offsetof(VRingAvail, idx);
Stefan Hajnocziabdd16f2020-02-07 10:46:19 +0000308
309 if (!caches) {
310 return 0;
311 }
312
Paolo Bonzini97cd9652017-01-27 16:40:20 +0100313 vq->shadow_avail_idx = virtio_lduw_phys_cached(vq->vdev, &caches->avail, pa);
Vincenzo Maffionebe1fea92016-01-31 11:29:05 +0100314 return vq->shadow_avail_idx;
aliguori967f97f2008-12-04 19:38:57 +0000315}
316
Paolo Bonzini97cd9652017-01-27 16:40:20 +0100317/* Called within rcu_read_lock(). */
aliguori967f97f2008-12-04 19:38:57 +0000318static inline uint16_t vring_avail_ring(VirtQueue *vq, int i)
319{
Jason Wange0e2d642017-03-15 19:48:31 +0800320 VRingMemoryRegionCaches *caches = vring_get_region_caches(vq);
Paolo Bonzini97cd9652017-01-27 16:40:20 +0100321 hwaddr pa = offsetof(VRingAvail, ring[i]);
Stefan Hajnocziabdd16f2020-02-07 10:46:19 +0000322
323 if (!caches) {
324 return 0;
325 }
326
Paolo Bonzini97cd9652017-01-27 16:40:20 +0100327 return virtio_lduw_phys_cached(vq->vdev, &caches->avail, pa);
aliguori967f97f2008-12-04 19:38:57 +0000328}
329
Paolo Bonzini97cd9652017-01-27 16:40:20 +0100330/* Called within rcu_read_lock(). */
Michael S. Tsirkine9600c62015-02-16 22:35:46 +0100331static inline uint16_t vring_get_used_event(VirtQueue *vq)
Michael S. Tsirkinbcbabae2011-06-12 16:21:57 +0300332{
333 return vring_avail_ring(vq, vq->vring.num);
334}
335
Paolo Bonzini97cd9652017-01-27 16:40:20 +0100336/* Called within rcu_read_lock(). */
Vincenzo Maffione1cdd2ee2016-01-31 11:29:06 +0100337static inline void vring_used_write(VirtQueue *vq, VRingUsedElem *uelem,
338 int i)
aliguori967f97f2008-12-04 19:38:57 +0000339{
Jason Wange0e2d642017-03-15 19:48:31 +0800340 VRingMemoryRegionCaches *caches = vring_get_region_caches(vq);
Paolo Bonzini97cd9652017-01-27 16:40:20 +0100341 hwaddr pa = offsetof(VRingUsed, ring[i]);
Stefan Hajnocziabdd16f2020-02-07 10:46:19 +0000342
343 if (!caches) {
344 return;
345 }
346
Vincenzo Maffione1cdd2ee2016-01-31 11:29:06 +0100347 virtio_tswap32s(vq->vdev, &uelem->id);
348 virtio_tswap32s(vq->vdev, &uelem->len);
Paolo Bonzini97cd9652017-01-27 16:40:20 +0100349 address_space_write_cached(&caches->used, pa, uelem, sizeof(VRingUsedElem));
350 address_space_cache_invalidate(&caches->used, pa, sizeof(VRingUsedElem));
aliguori967f97f2008-12-04 19:38:57 +0000351}
352
Paolo Bonzini97cd9652017-01-27 16:40:20 +0100353/* Called within rcu_read_lock(). */
aliguori967f97f2008-12-04 19:38:57 +0000354static uint16_t vring_used_idx(VirtQueue *vq)
355{
Jason Wange0e2d642017-03-15 19:48:31 +0800356 VRingMemoryRegionCaches *caches = vring_get_region_caches(vq);
Paolo Bonzini97cd9652017-01-27 16:40:20 +0100357 hwaddr pa = offsetof(VRingUsed, idx);
Stefan Hajnocziabdd16f2020-02-07 10:46:19 +0000358
359 if (!caches) {
360 return 0;
361 }
362
Paolo Bonzini97cd9652017-01-27 16:40:20 +0100363 return virtio_lduw_phys_cached(vq->vdev, &caches->used, pa);
aliguori967f97f2008-12-04 19:38:57 +0000364}
365
Paolo Bonzini97cd9652017-01-27 16:40:20 +0100366/* Called within rcu_read_lock(). */
Michael S. Tsirkinbcbabae2011-06-12 16:21:57 +0300367static inline void vring_used_idx_set(VirtQueue *vq, uint16_t val)
aliguori967f97f2008-12-04 19:38:57 +0000368{
Jason Wange0e2d642017-03-15 19:48:31 +0800369 VRingMemoryRegionCaches *caches = vring_get_region_caches(vq);
Paolo Bonzini97cd9652017-01-27 16:40:20 +0100370 hwaddr pa = offsetof(VRingUsed, idx);
Stefan Hajnocziabdd16f2020-02-07 10:46:19 +0000371
372 if (caches) {
373 virtio_stw_phys_cached(vq->vdev, &caches->used, pa, val);
374 address_space_cache_invalidate(&caches->used, pa, sizeof(val));
375 }
376
Vincenzo Maffioneb796fcd2016-01-31 11:29:04 +0100377 vq->used_idx = val;
aliguori967f97f2008-12-04 19:38:57 +0000378}
379
Paolo Bonzini97cd9652017-01-27 16:40:20 +0100380/* Called within rcu_read_lock(). */
aliguori967f97f2008-12-04 19:38:57 +0000381static inline void vring_used_flags_set_bit(VirtQueue *vq, int mask)
382{
Jason Wange0e2d642017-03-15 19:48:31 +0800383 VRingMemoryRegionCaches *caches = vring_get_region_caches(vq);
Rusty Russellcee3ca02014-06-24 19:40:16 +0200384 VirtIODevice *vdev = vq->vdev;
Paolo Bonzini97cd9652017-01-27 16:40:20 +0100385 hwaddr pa = offsetof(VRingUsed, flags);
Stefan Hajnocziabdd16f2020-02-07 10:46:19 +0000386 uint16_t flags;
Paolo Bonzini97cd9652017-01-27 16:40:20 +0100387
Stefan Hajnocziabdd16f2020-02-07 10:46:19 +0000388 if (!caches) {
389 return;
390 }
391
392 flags = virtio_lduw_phys_cached(vq->vdev, &caches->used, pa);
Paolo Bonzini97cd9652017-01-27 16:40:20 +0100393 virtio_stw_phys_cached(vdev, &caches->used, pa, flags | mask);
394 address_space_cache_invalidate(&caches->used, pa, sizeof(flags));
aliguori967f97f2008-12-04 19:38:57 +0000395}
396
Paolo Bonzini97cd9652017-01-27 16:40:20 +0100397/* Called within rcu_read_lock(). */
aliguori967f97f2008-12-04 19:38:57 +0000398static inline void vring_used_flags_unset_bit(VirtQueue *vq, int mask)
399{
Jason Wange0e2d642017-03-15 19:48:31 +0800400 VRingMemoryRegionCaches *caches = vring_get_region_caches(vq);
Rusty Russellcee3ca02014-06-24 19:40:16 +0200401 VirtIODevice *vdev = vq->vdev;
Paolo Bonzini97cd9652017-01-27 16:40:20 +0100402 hwaddr pa = offsetof(VRingUsed, flags);
Stefan Hajnocziabdd16f2020-02-07 10:46:19 +0000403 uint16_t flags;
Paolo Bonzini97cd9652017-01-27 16:40:20 +0100404
Stefan Hajnocziabdd16f2020-02-07 10:46:19 +0000405 if (!caches) {
406 return;
407 }
408
409 flags = virtio_lduw_phys_cached(vq->vdev, &caches->used, pa);
Paolo Bonzini97cd9652017-01-27 16:40:20 +0100410 virtio_stw_phys_cached(vdev, &caches->used, pa, flags & ~mask);
411 address_space_cache_invalidate(&caches->used, pa, sizeof(flags));
aliguori967f97f2008-12-04 19:38:57 +0000412}
413
Paolo Bonzini97cd9652017-01-27 16:40:20 +0100414/* Called within rcu_read_lock(). */
Michael S. Tsirkine9600c62015-02-16 22:35:46 +0100415static inline void vring_set_avail_event(VirtQueue *vq, uint16_t val)
Michael S. Tsirkinbcbabae2011-06-12 16:21:57 +0300416{
Paolo Bonzini97cd9652017-01-27 16:40:20 +0100417 VRingMemoryRegionCaches *caches;
Avi Kivitya8170e52012-10-23 12:30:10 +0200418 hwaddr pa;
Stefan Hajnoczi332fa822017-01-12 11:46:10 +0000419 if (!vq->notification) {
Michael S. Tsirkinbcbabae2011-06-12 16:21:57 +0300420 return;
421 }
Paolo Bonzini97cd9652017-01-27 16:40:20 +0100422
Jason Wange0e2d642017-03-15 19:48:31 +0800423 caches = vring_get_region_caches(vq);
Stefan Hajnocziabdd16f2020-02-07 10:46:19 +0000424 if (!caches) {
425 return;
426 }
427
Paolo Bonzini97cd9652017-01-27 16:40:20 +0100428 pa = offsetof(VRingUsed, ring[vq->vring.num]);
429 virtio_stw_phys_cached(vq->vdev, &caches->used, pa, val);
Stefan Hajnoczi3cdf8472017-02-22 16:37:33 +0000430 address_space_cache_invalidate(&caches->used, pa, sizeof(val));
Michael S. Tsirkinbcbabae2011-06-12 16:21:57 +0300431}
432
Jason Wang683f7662019-10-25 10:35:25 +0200433static void virtio_queue_split_set_notification(VirtQueue *vq, int enable)
aliguori967f97f2008-12-04 19:38:57 +0000434{
Dr. David Alan Gilbertb5f53d02019-10-28 16:11:09 +0000435 RCU_READ_LOCK_GUARD();
436
Cornelia Huck95129d62015-08-17 11:48:29 +0200437 if (virtio_vdev_has_feature(vq->vdev, VIRTIO_RING_F_EVENT_IDX)) {
Michael S. Tsirkine9600c62015-02-16 22:35:46 +0100438 vring_set_avail_event(vq, vring_avail_idx(vq));
Michael S. Tsirkinbcbabae2011-06-12 16:21:57 +0300439 } else if (enable) {
aliguori967f97f2008-12-04 19:38:57 +0000440 vring_used_flags_unset_bit(vq, VRING_USED_F_NO_NOTIFY);
Michael S. Tsirkinbcbabae2011-06-12 16:21:57 +0300441 } else {
aliguori967f97f2008-12-04 19:38:57 +0000442 vring_used_flags_set_bit(vq, VRING_USED_F_NO_NOTIFY);
Michael S. Tsirkinbcbabae2011-06-12 16:21:57 +0300443 }
Michael S. Tsirkin92045d82012-04-23 14:11:14 +0300444 if (enable) {
445 /* Expose avail event/used flags before caller checks the avail idx. */
446 smp_mb();
447 }
aliguori967f97f2008-12-04 19:38:57 +0000448}
449
Jason Wang683f7662019-10-25 10:35:25 +0200450static void virtio_queue_packed_set_notification(VirtQueue *vq, int enable)
451{
452 uint16_t off_wrap;
453 VRingPackedDescEvent e;
454 VRingMemoryRegionCaches *caches;
455
Dr. David Alan Gilbertb5f53d02019-10-28 16:11:09 +0000456 RCU_READ_LOCK_GUARD();
Stefan Hajnocziabdd16f2020-02-07 10:46:19 +0000457 caches = vring_get_region_caches(vq);
458 if (!caches) {
459 return;
460 }
461
Jason Wang683f7662019-10-25 10:35:25 +0200462 vring_packed_event_read(vq->vdev, &caches->used, &e);
463
464 if (!enable) {
465 e.flags = VRING_PACKED_EVENT_FLAG_DISABLE;
466 } else if (virtio_vdev_has_feature(vq->vdev, VIRTIO_RING_F_EVENT_IDX)) {
467 off_wrap = vq->shadow_avail_idx | vq->shadow_avail_wrap_counter << 15;
468 vring_packed_off_wrap_write(vq->vdev, &caches->used, off_wrap);
469 /* Make sure off_wrap is wrote before flags */
470 smp_wmb();
471 e.flags = VRING_PACKED_EVENT_FLAG_DESC;
472 } else {
473 e.flags = VRING_PACKED_EVENT_FLAG_ENABLE;
474 }
475
476 vring_packed_flags_write(vq->vdev, &caches->used, e.flags);
477 if (enable) {
478 /* Expose avail event/used flags before caller checks the avail idx. */
479 smp_mb();
480 }
Jason Wang683f7662019-10-25 10:35:25 +0200481}
482
Stefan Hajnoczid0435bc2019-12-09 21:09:57 +0000483bool virtio_queue_get_notification(VirtQueue *vq)
484{
485 return vq->notification;
486}
487
Jason Wang683f7662019-10-25 10:35:25 +0200488void virtio_queue_set_notification(VirtQueue *vq, int enable)
489{
490 vq->notification = enable;
491
492 if (!vq->vring.desc) {
493 return;
494 }
495
496 if (virtio_vdev_has_feature(vq->vdev, VIRTIO_F_RING_PACKED)) {
497 virtio_queue_packed_set_notification(vq, enable);
498 } else {
499 virtio_queue_split_set_notification(vq, enable);
500 }
501}
502
aliguori967f97f2008-12-04 19:38:57 +0000503int virtio_queue_ready(VirtQueue *vq)
504{
505 return vq->vring.avail != 0;
506}
507
Jason Wang86044b22019-10-25 10:35:24 +0200508static void vring_packed_desc_read_flags(VirtIODevice *vdev,
509 uint16_t *flags,
510 MemoryRegionCache *cache,
511 int i)
512{
513 address_space_read_cached(cache,
514 i * sizeof(VRingPackedDesc) +
515 offsetof(VRingPackedDesc, flags),
516 flags, sizeof(*flags));
517 virtio_tswap16s(vdev, flags);
518}
519
520static void vring_packed_desc_read(VirtIODevice *vdev,
521 VRingPackedDesc *desc,
522 MemoryRegionCache *cache,
523 int i, bool strict_order)
524{
525 hwaddr off = i * sizeof(VRingPackedDesc);
526
527 vring_packed_desc_read_flags(vdev, &desc->flags, cache, i);
528
529 if (strict_order) {
530 /* Make sure flags is read before the rest fields. */
531 smp_rmb();
532 }
533
534 address_space_read_cached(cache, off + offsetof(VRingPackedDesc, addr),
535 &desc->addr, sizeof(desc->addr));
536 address_space_read_cached(cache, off + offsetof(VRingPackedDesc, id),
537 &desc->id, sizeof(desc->id));
538 address_space_read_cached(cache, off + offsetof(VRingPackedDesc, len),
539 &desc->len, sizeof(desc->len));
540 virtio_tswap64s(vdev, &desc->addr);
541 virtio_tswap16s(vdev, &desc->id);
542 virtio_tswap32s(vdev, &desc->len);
543}
544
545static void vring_packed_desc_write_data(VirtIODevice *vdev,
546 VRingPackedDesc *desc,
547 MemoryRegionCache *cache,
548 int i)
549{
550 hwaddr off_id = i * sizeof(VRingPackedDesc) +
551 offsetof(VRingPackedDesc, id);
552 hwaddr off_len = i * sizeof(VRingPackedDesc) +
553 offsetof(VRingPackedDesc, len);
554
555 virtio_tswap32s(vdev, &desc->len);
556 virtio_tswap16s(vdev, &desc->id);
557 address_space_write_cached(cache, off_id, &desc->id, sizeof(desc->id));
558 address_space_cache_invalidate(cache, off_id, sizeof(desc->id));
559 address_space_write_cached(cache, off_len, &desc->len, sizeof(desc->len));
560 address_space_cache_invalidate(cache, off_len, sizeof(desc->len));
561}
562
563static void vring_packed_desc_write_flags(VirtIODevice *vdev,
564 VRingPackedDesc *desc,
565 MemoryRegionCache *cache,
566 int i)
567{
568 hwaddr off = i * sizeof(VRingPackedDesc) + offsetof(VRingPackedDesc, flags);
569
570 virtio_tswap16s(vdev, &desc->flags);
571 address_space_write_cached(cache, off, &desc->flags, sizeof(desc->flags));
572 address_space_cache_invalidate(cache, off, sizeof(desc->flags));
573}
574
575static void vring_packed_desc_write(VirtIODevice *vdev,
576 VRingPackedDesc *desc,
577 MemoryRegionCache *cache,
578 int i, bool strict_order)
579{
580 vring_packed_desc_write_data(vdev, desc, cache, i);
581 if (strict_order) {
582 /* Make sure data is wrote before flags. */
583 smp_wmb();
584 }
585 vring_packed_desc_write_flags(vdev, desc, cache, i);
586}
587
588static inline bool is_desc_avail(uint16_t flags, bool wrap_counter)
589{
590 bool avail, used;
591
592 avail = !!(flags & (1 << VRING_PACKED_DESC_F_AVAIL));
593 used = !!(flags & (1 << VRING_PACKED_DESC_F_USED));
594 return (avail != used) && (avail == wrap_counter);
595}
596
Vincenzo Maffionebe1fea92016-01-31 11:29:05 +0100597/* Fetch avail_idx from VQ memory only when we really need to know if
Paolo Bonzini97cd9652017-01-27 16:40:20 +0100598 * guest has added some buffers.
599 * Called within rcu_read_lock(). */
600static int virtio_queue_empty_rcu(VirtQueue *vq)
aliguori967f97f2008-12-04 19:38:57 +0000601{
Michael Roth9d7bd082019-11-19 18:50:03 -0600602 if (virtio_device_disabled(vq->vdev)) {
Fam Zheng2d1df852018-09-10 22:56:15 +0800603 return 1;
604 }
605
Jason Wang168e4af2017-03-15 19:48:30 +0800606 if (unlikely(!vq->vring.avail)) {
607 return 1;
608 }
609
Vincenzo Maffionebe1fea92016-01-31 11:29:05 +0100610 if (vq->shadow_avail_idx != vq->last_avail_idx) {
611 return 0;
612 }
613
aliguori967f97f2008-12-04 19:38:57 +0000614 return vring_avail_idx(vq) == vq->last_avail_idx;
615}
616
Jason Wang86044b22019-10-25 10:35:24 +0200617static int virtio_queue_split_empty(VirtQueue *vq)
Paolo Bonzini97cd9652017-01-27 16:40:20 +0100618{
619 bool empty;
620
Michael Roth9d7bd082019-11-19 18:50:03 -0600621 if (virtio_device_disabled(vq->vdev)) {
Fam Zheng2d1df852018-09-10 22:56:15 +0800622 return 1;
623 }
624
Jason Wang168e4af2017-03-15 19:48:30 +0800625 if (unlikely(!vq->vring.avail)) {
626 return 1;
627 }
628
Paolo Bonzini97cd9652017-01-27 16:40:20 +0100629 if (vq->shadow_avail_idx != vq->last_avail_idx) {
630 return 0;
631 }
632
Dr. David Alan Gilbertb5f53d02019-10-28 16:11:09 +0000633 RCU_READ_LOCK_GUARD();
Paolo Bonzini97cd9652017-01-27 16:40:20 +0100634 empty = vring_avail_idx(vq) == vq->last_avail_idx;
Paolo Bonzini97cd9652017-01-27 16:40:20 +0100635 return empty;
636}
637
Jason Wang86044b22019-10-25 10:35:24 +0200638static int virtio_queue_packed_empty_rcu(VirtQueue *vq)
639{
640 struct VRingPackedDesc desc;
641 VRingMemoryRegionCaches *cache;
642
643 if (unlikely(!vq->vring.desc)) {
644 return 1;
645 }
646
647 cache = vring_get_region_caches(vq);
Stefan Hajnocziabdd16f2020-02-07 10:46:19 +0000648 if (!cache) {
649 return 1;
650 }
651
Jason Wang86044b22019-10-25 10:35:24 +0200652 vring_packed_desc_read_flags(vq->vdev, &desc.flags, &cache->desc,
653 vq->last_avail_idx);
654
655 return !is_desc_avail(desc.flags, vq->last_avail_wrap_counter);
656}
657
658static int virtio_queue_packed_empty(VirtQueue *vq)
659{
Dr. David Alan Gilbertb5f53d02019-10-28 16:11:09 +0000660 RCU_READ_LOCK_GUARD();
661 return virtio_queue_packed_empty_rcu(vq);
Jason Wang86044b22019-10-25 10:35:24 +0200662}
663
664int virtio_queue_empty(VirtQueue *vq)
665{
666 if (virtio_vdev_has_feature(vq->vdev, VIRTIO_F_RING_PACKED)) {
667 return virtio_queue_packed_empty(vq);
668 } else {
669 return virtio_queue_split_empty(vq);
670 }
671}
672
Jason Wangce317462015-09-25 13:21:28 +0800673static void virtqueue_unmap_sg(VirtQueue *vq, const VirtQueueElement *elem,
674 unsigned int len)
aliguori967f97f2008-12-04 19:38:57 +0000675{
Jason Wang8607f5c2016-12-30 18:09:10 +0800676 AddressSpace *dma_as = vq->vdev->dma_as;
aliguori967f97f2008-12-04 19:38:57 +0000677 unsigned int offset;
678 int i;
679
aliguori967f97f2008-12-04 19:38:57 +0000680 offset = 0;
681 for (i = 0; i < elem->in_num; i++) {
682 size_t size = MIN(len - offset, elem->in_sg[i].iov_len);
683
Jason Wang8607f5c2016-12-30 18:09:10 +0800684 dma_memory_unmap(dma_as, elem->in_sg[i].iov_base,
685 elem->in_sg[i].iov_len,
686 DMA_DIRECTION_FROM_DEVICE, size);
aliguori967f97f2008-12-04 19:38:57 +0000687
Michael S. Tsirkin0cea71a2012-09-24 15:09:30 +0200688 offset += size;
aliguori967f97f2008-12-04 19:38:57 +0000689 }
690
aliguori26b258e2009-03-28 17:46:18 +0000691 for (i = 0; i < elem->out_num; i++)
Jason Wang8607f5c2016-12-30 18:09:10 +0800692 dma_memory_unmap(dma_as, elem->out_sg[i].iov_base,
693 elem->out_sg[i].iov_len,
694 DMA_DIRECTION_TO_DEVICE,
695 elem->out_sg[i].iov_len);
Jason Wangce317462015-09-25 13:21:28 +0800696}
697
Stefan Hajnoczi2640d2a2016-09-19 14:28:03 +0100698/* virtqueue_detach_element:
699 * @vq: The #VirtQueue
700 * @elem: The #VirtQueueElement
701 * @len: number of bytes written
702 *
703 * Detach the element from the virtqueue. This function is suitable for device
704 * reset or other situations where a #VirtQueueElement is simply freed and will
705 * not be pushed or discarded.
706 */
707void virtqueue_detach_element(VirtQueue *vq, const VirtQueueElement *elem,
708 unsigned int len)
709{
Jason Wang86044b22019-10-25 10:35:24 +0200710 vq->inuse -= elem->ndescs;
Stefan Hajnoczi2640d2a2016-09-19 14:28:03 +0100711 virtqueue_unmap_sg(vq, elem, len);
712}
713
Jason Wang86044b22019-10-25 10:35:24 +0200714static void virtqueue_split_rewind(VirtQueue *vq, unsigned int num)
715{
716 vq->last_avail_idx -= num;
717}
718
719static void virtqueue_packed_rewind(VirtQueue *vq, unsigned int num)
720{
721 if (vq->last_avail_idx < num) {
722 vq->last_avail_idx = vq->vring.num + vq->last_avail_idx - num;
723 vq->last_avail_wrap_counter ^= 1;
724 } else {
725 vq->last_avail_idx -= num;
726 }
727}
728
Ladi Prosek27e57ef2016-11-03 09:55:49 +0100729/* virtqueue_unpop:
Stefan Hajnoczi2640d2a2016-09-19 14:28:03 +0100730 * @vq: The #VirtQueue
731 * @elem: The #VirtQueueElement
732 * @len: number of bytes written
733 *
734 * Pretend the most recent element wasn't popped from the virtqueue. The next
735 * call to virtqueue_pop() will refetch the element.
736 */
Ladi Prosek27e57ef2016-11-03 09:55:49 +0100737void virtqueue_unpop(VirtQueue *vq, const VirtQueueElement *elem,
738 unsigned int len)
Jason Wang29b9f5e2015-09-25 13:21:29 +0800739{
Jason Wang86044b22019-10-25 10:35:24 +0200740
741 if (virtio_vdev_has_feature(vq->vdev, VIRTIO_F_RING_PACKED)) {
742 virtqueue_packed_rewind(vq, 1);
743 } else {
744 virtqueue_split_rewind(vq, 1);
745 }
746
Stefan Hajnoczi2640d2a2016-09-19 14:28:03 +0100747 virtqueue_detach_element(vq, elem, len);
Jason Wang29b9f5e2015-09-25 13:21:29 +0800748}
749
Stefan Hajnoczi297a75e2016-09-07 17:20:48 +0200750/* virtqueue_rewind:
751 * @vq: The #VirtQueue
752 * @num: Number of elements to push back
753 *
754 * Pretend that elements weren't popped from the virtqueue. The next
755 * virtqueue_pop() will refetch the oldest element.
756 *
Ladi Prosek27e57ef2016-11-03 09:55:49 +0100757 * Use virtqueue_unpop() instead if you have a VirtQueueElement.
Stefan Hajnoczi297a75e2016-09-07 17:20:48 +0200758 *
759 * Returns: true on success, false if @num is greater than the number of in use
760 * elements.
761 */
762bool virtqueue_rewind(VirtQueue *vq, unsigned int num)
763{
764 if (num > vq->inuse) {
765 return false;
766 }
Jason Wang86044b22019-10-25 10:35:24 +0200767
Stefan Hajnoczi297a75e2016-09-07 17:20:48 +0200768 vq->inuse -= num;
Jason Wang86044b22019-10-25 10:35:24 +0200769 if (virtio_vdev_has_feature(vq->vdev, VIRTIO_F_RING_PACKED)) {
770 virtqueue_packed_rewind(vq, num);
771 } else {
772 virtqueue_split_rewind(vq, num);
773 }
Stefan Hajnoczi297a75e2016-09-07 17:20:48 +0200774 return true;
775}
776
Jason Wang86044b22019-10-25 10:35:24 +0200777static void virtqueue_split_fill(VirtQueue *vq, const VirtQueueElement *elem,
Jason Wangce317462015-09-25 13:21:28 +0800778 unsigned int len, unsigned int idx)
779{
Vincenzo Maffione1cdd2ee2016-01-31 11:29:06 +0100780 VRingUsedElem uelem;
781
Jason Wang168e4af2017-03-15 19:48:30 +0800782 if (unlikely(!vq->vring.used)) {
783 return;
784 }
785
Vincenzo Maffioneb796fcd2016-01-31 11:29:04 +0100786 idx = (idx + vq->used_idx) % vq->vring.num;
aliguori967f97f2008-12-04 19:38:57 +0000787
Vincenzo Maffione1cdd2ee2016-01-31 11:29:06 +0100788 uelem.id = elem->index;
789 uelem.len = len;
790 vring_used_write(vq, &uelem, idx);
aliguori967f97f2008-12-04 19:38:57 +0000791}
792
Jason Wang86044b22019-10-25 10:35:24 +0200793static void virtqueue_packed_fill(VirtQueue *vq, const VirtQueueElement *elem,
794 unsigned int len, unsigned int idx)
aliguori967f97f2008-12-04 19:38:57 +0000795{
Jason Wang86044b22019-10-25 10:35:24 +0200796 vq->used_elems[idx].index = elem->index;
797 vq->used_elems[idx].len = len;
798 vq->used_elems[idx].ndescs = elem->ndescs;
799}
Stefan Hajnoczif5ed3662016-09-21 16:52:19 +0100800
Jason Wang86044b22019-10-25 10:35:24 +0200801static void virtqueue_packed_fill_desc(VirtQueue *vq,
802 const VirtQueueElement *elem,
803 unsigned int idx,
804 bool strict_order)
805{
806 uint16_t head;
807 VRingMemoryRegionCaches *caches;
808 VRingPackedDesc desc = {
809 .id = elem->index,
810 .len = elem->len,
811 };
812 bool wrap_counter = vq->used_wrap_counter;
813
814 if (unlikely(!vq->vring.desc)) {
Stefan Hajnoczif5ed3662016-09-21 16:52:19 +0100815 return;
816 }
817
Jason Wang86044b22019-10-25 10:35:24 +0200818 head = vq->used_idx + idx;
819 if (head >= vq->vring.num) {
820 head -= vq->vring.num;
821 wrap_counter ^= 1;
822 }
823 if (wrap_counter) {
824 desc.flags |= (1 << VRING_PACKED_DESC_F_AVAIL);
825 desc.flags |= (1 << VRING_PACKED_DESC_F_USED);
826 } else {
827 desc.flags &= ~(1 << VRING_PACKED_DESC_F_AVAIL);
828 desc.flags &= ~(1 << VRING_PACKED_DESC_F_USED);
829 }
830
831 caches = vring_get_region_caches(vq);
Stefan Hajnocziabdd16f2020-02-07 10:46:19 +0000832 if (!caches) {
833 return;
834 }
835
Jason Wang86044b22019-10-25 10:35:24 +0200836 vring_packed_desc_write(vq->vdev, &desc, &caches->desc, head, strict_order);
837}
838
839/* Called within rcu_read_lock(). */
840void virtqueue_fill(VirtQueue *vq, const VirtQueueElement *elem,
841 unsigned int len, unsigned int idx)
842{
843 trace_virtqueue_fill(vq, elem, len, idx);
844
845 virtqueue_unmap_sg(vq, elem, len);
846
Michael Roth9d7bd082019-11-19 18:50:03 -0600847 if (virtio_device_disabled(vq->vdev)) {
Jason Wang86044b22019-10-25 10:35:24 +0200848 return;
849 }
850
851 if (virtio_vdev_has_feature(vq->vdev, VIRTIO_F_RING_PACKED)) {
852 virtqueue_packed_fill(vq, elem, len, idx);
853 } else {
854 virtqueue_split_fill(vq, elem, len, idx);
855 }
856}
857
858/* Called within rcu_read_lock(). */
859static void virtqueue_split_flush(VirtQueue *vq, unsigned int count)
860{
861 uint16_t old, new;
862
Jason Wang168e4af2017-03-15 19:48:30 +0800863 if (unlikely(!vq->vring.used)) {
864 return;
865 }
866
aliguori967f97f2008-12-04 19:38:57 +0000867 /* Make sure buffer is written before we update index. */
David Gibsonb90d2f32011-09-20 12:05:20 +1000868 smp_wmb();
Stefan Hajnoczi64979a42010-05-24 13:19:21 +0100869 trace_virtqueue_flush(vq, count);
Vincenzo Maffioneb796fcd2016-01-31 11:29:04 +0100870 old = vq->used_idx;
Michael S. Tsirkinbcbabae2011-06-12 16:21:57 +0300871 new = old + count;
872 vring_used_idx_set(vq, new);
aliguori967f97f2008-12-04 19:38:57 +0000873 vq->inuse -= count;
Michael S. Tsirkinbcbabae2011-06-12 16:21:57 +0300874 if (unlikely((int16_t)(new - vq->signalled_used) < (uint16_t)(new - old)))
875 vq->signalled_used_valid = false;
aliguori967f97f2008-12-04 19:38:57 +0000876}
877
Jason Wang86044b22019-10-25 10:35:24 +0200878static void virtqueue_packed_flush(VirtQueue *vq, unsigned int count)
879{
880 unsigned int i, ndescs = 0;
881
882 if (unlikely(!vq->vring.desc)) {
883 return;
884 }
885
886 for (i = 1; i < count; i++) {
887 virtqueue_packed_fill_desc(vq, &vq->used_elems[i], i, false);
888 ndescs += vq->used_elems[i].ndescs;
889 }
890 virtqueue_packed_fill_desc(vq, &vq->used_elems[0], 0, true);
891 ndescs += vq->used_elems[0].ndescs;
892
893 vq->inuse -= ndescs;
894 vq->used_idx += ndescs;
895 if (vq->used_idx >= vq->vring.num) {
896 vq->used_idx -= vq->vring.num;
897 vq->used_wrap_counter ^= 1;
898 }
899}
900
901void virtqueue_flush(VirtQueue *vq, unsigned int count)
902{
Michael Roth9d7bd082019-11-19 18:50:03 -0600903 if (virtio_device_disabled(vq->vdev)) {
Jason Wang86044b22019-10-25 10:35:24 +0200904 vq->inuse -= count;
905 return;
906 }
907
908 if (virtio_vdev_has_feature(vq->vdev, VIRTIO_F_RING_PACKED)) {
909 virtqueue_packed_flush(vq, count);
910 } else {
911 virtqueue_split_flush(vq, count);
912 }
913}
914
aliguori967f97f2008-12-04 19:38:57 +0000915void virtqueue_push(VirtQueue *vq, const VirtQueueElement *elem,
916 unsigned int len)
917{
Dr. David Alan Gilbertb5f53d02019-10-28 16:11:09 +0000918 RCU_READ_LOCK_GUARD();
aliguori967f97f2008-12-04 19:38:57 +0000919 virtqueue_fill(vq, elem, len, 0);
920 virtqueue_flush(vq, 1);
921}
922
Paolo Bonzini97cd9652017-01-27 16:40:20 +0100923/* Called within rcu_read_lock(). */
aliguori967f97f2008-12-04 19:38:57 +0000924static int virtqueue_num_heads(VirtQueue *vq, unsigned int idx)
925{
926 uint16_t num_heads = vring_avail_idx(vq) - idx;
927
928 /* Check it isn't doing very strange things with descriptor numbers. */
aliguoribb6834c2008-12-04 21:28:28 +0000929 if (num_heads > vq->vring.num) {
Stefan Hajnoczi4355c1a2016-09-21 16:52:25 +0100930 virtio_error(vq->vdev, "Guest moved used index from %u to %u",
Vincenzo Maffionebe1fea92016-01-31 11:29:05 +0100931 idx, vq->shadow_avail_idx);
Stefan Hajnoczi4355c1a2016-09-21 16:52:25 +0100932 return -EINVAL;
aliguoribb6834c2008-12-04 21:28:28 +0000933 }
Michael S. Tsirkina821ce52012-04-23 15:46:22 +0300934 /* On success, callers read a descriptor at vq->last_avail_idx.
935 * Make sure descriptor read does not bypass avail index read. */
936 if (num_heads) {
937 smp_rmb();
938 }
aliguori967f97f2008-12-04 19:38:57 +0000939
940 return num_heads;
941}
942
Paolo Bonzini97cd9652017-01-27 16:40:20 +0100943/* Called within rcu_read_lock(). */
Stefan Hajnoczifb1131b2016-09-21 16:52:26 +0100944static bool virtqueue_get_head(VirtQueue *vq, unsigned int idx,
945 unsigned int *head)
aliguori967f97f2008-12-04 19:38:57 +0000946{
aliguori967f97f2008-12-04 19:38:57 +0000947 /* Grab the next descriptor number they're advertising, and increment
948 * the index we've seen. */
Stefan Hajnoczifb1131b2016-09-21 16:52:26 +0100949 *head = vring_avail_ring(vq, idx % vq->vring.num);
aliguori967f97f2008-12-04 19:38:57 +0000950
951 /* If their number is silly, that's a fatal mistake. */
Stefan Hajnoczifb1131b2016-09-21 16:52:26 +0100952 if (*head >= vq->vring.num) {
953 virtio_error(vq->vdev, "Guest says index %u is available", *head);
954 return false;
aliguoribb6834c2008-12-04 21:28:28 +0000955 }
aliguori967f97f2008-12-04 19:38:57 +0000956
Stefan Hajnoczifb1131b2016-09-21 16:52:26 +0100957 return true;
aliguori967f97f2008-12-04 19:38:57 +0000958}
959
Stefan Hajnoczi412e0e82016-09-21 16:52:24 +0100960enum {
961 VIRTQUEUE_READ_DESC_ERROR = -1,
962 VIRTQUEUE_READ_DESC_DONE = 0, /* end of chain */
963 VIRTQUEUE_READ_DESC_MORE = 1, /* more buffers in chain */
964};
aliguori967f97f2008-12-04 19:38:57 +0000965
Jason Wang86044b22019-10-25 10:35:24 +0200966static int virtqueue_split_read_next_desc(VirtIODevice *vdev, VRingDesc *desc,
967 MemoryRegionCache *desc_cache,
968 unsigned int max, unsigned int *next)
Stefan Hajnoczi412e0e82016-09-21 16:52:24 +0100969{
aliguori967f97f2008-12-04 19:38:57 +0000970 /* If this descriptor says it doesn't chain, we're done. */
Paolo Bonziniaa570d62016-01-31 11:29:03 +0100971 if (!(desc->flags & VRING_DESC_F_NEXT)) {
Stefan Hajnoczi412e0e82016-09-21 16:52:24 +0100972 return VIRTQUEUE_READ_DESC_DONE;
Rusty Russellcee3ca02014-06-24 19:40:16 +0200973 }
aliguori967f97f2008-12-04 19:38:57 +0000974
975 /* Check they're not leading us off end of descriptors. */
Stefan Hajnoczi412e0e82016-09-21 16:52:24 +0100976 *next = desc->next;
aliguori967f97f2008-12-04 19:38:57 +0000977 /* Make sure compiler knows to grab that: we don't want it changing! */
David Gibsonb90d2f32011-09-20 12:05:20 +1000978 smp_wmb();
aliguori967f97f2008-12-04 19:38:57 +0000979
Stefan Hajnoczi412e0e82016-09-21 16:52:24 +0100980 if (*next >= max) {
981 virtio_error(vdev, "Desc next is %u", *next);
982 return VIRTQUEUE_READ_DESC_ERROR;
aliguoribb6834c2008-12-04 21:28:28 +0000983 }
aliguori967f97f2008-12-04 19:38:57 +0000984
Jason Wang86044b22019-10-25 10:35:24 +0200985 vring_split_desc_read(vdev, desc, desc_cache, *next);
Stefan Hajnoczi412e0e82016-09-21 16:52:24 +0100986 return VIRTQUEUE_READ_DESC_MORE;
aliguori967f97f2008-12-04 19:38:57 +0000987}
988
Jason Wang86044b22019-10-25 10:35:24 +0200989static void virtqueue_split_get_avail_bytes(VirtQueue *vq,
990 unsigned int *in_bytes, unsigned int *out_bytes,
991 unsigned max_in_bytes, unsigned max_out_bytes)
aliguori967f97f2008-12-04 19:38:57 +0000992{
Paolo Bonzini9796d0a2017-01-27 16:40:14 +0100993 VirtIODevice *vdev = vq->vdev;
994 unsigned int max, idx;
Amit Shah385ce952012-09-25 00:05:14 +0530995 unsigned int total_bufs, in_total, out_total;
Paolo Bonzini991976f2017-01-27 16:40:18 +0100996 VRingMemoryRegionCaches *caches;
Paolo Bonzini5eba0402017-01-27 16:40:16 +0100997 MemoryRegionCache indirect_desc_cache = MEMORY_REGION_CACHE_INVALID;
998 int64_t len = 0;
Stefan Hajnoczi412e0e82016-09-21 16:52:24 +0100999 int rc;
aliguori967f97f2008-12-04 19:38:57 +00001000
Dr. David Alan Gilbertb5f53d02019-10-28 16:11:09 +00001001 RCU_READ_LOCK_GUARD();
1002
aliguori967f97f2008-12-04 19:38:57 +00001003 idx = vq->last_avail_idx;
Mark McLoughlinefeea6d2009-06-17 11:38:28 +01001004 total_bufs = in_total = out_total = 0;
Paolo Bonzini9796d0a2017-01-27 16:40:14 +01001005
1006 max = vq->vring.num;
Jason Wange0e2d642017-03-15 19:48:31 +08001007 caches = vring_get_region_caches(vq);
Stefan Hajnocziabdd16f2020-02-07 10:46:19 +00001008 if (!caches) {
1009 goto err;
1010 }
1011
Stefan Hajnoczi4355c1a2016-09-21 16:52:25 +01001012 while ((rc = virtqueue_num_heads(vq, idx)) > 0) {
Paolo Bonzini991976f2017-01-27 16:40:18 +01001013 MemoryRegionCache *desc_cache = &caches->desc;
Paolo Bonzini9796d0a2017-01-27 16:40:14 +01001014 unsigned int num_bufs;
Paolo Bonziniaa570d62016-01-31 11:29:03 +01001015 VRingDesc desc;
Stefan Hajnoczib1c7c072016-09-21 16:52:23 +01001016 unsigned int i;
aliguori967f97f2008-12-04 19:38:57 +00001017
Mark McLoughlinefeea6d2009-06-17 11:38:28 +01001018 num_bufs = total_bufs;
Stefan Hajnoczifb1131b2016-09-21 16:52:26 +01001019
1020 if (!virtqueue_get_head(vq, idx++, &i)) {
1021 goto err;
1022 }
1023
Jason Wang86044b22019-10-25 10:35:24 +02001024 vring_split_desc_read(vdev, &desc, desc_cache, i);
Mark McLoughlinefeea6d2009-06-17 11:38:28 +01001025
Paolo Bonziniaa570d62016-01-31 11:29:03 +01001026 if (desc.flags & VRING_DESC_F_INDIRECT) {
Dima Stepanov74231922019-01-15 13:08:47 +03001027 if (!desc.len || (desc.len % sizeof(VRingDesc))) {
Stefan Hajnoczid65abf82016-09-21 16:52:22 +01001028 virtio_error(vdev, "Invalid size for indirect buffer table");
1029 goto err;
Mark McLoughlinefeea6d2009-06-17 11:38:28 +01001030 }
1031
1032 /* If we've got too many, that implies a descriptor loop. */
1033 if (num_bufs >= max) {
Stefan Hajnoczid65abf82016-09-21 16:52:22 +01001034 virtio_error(vdev, "Looped descriptor");
1035 goto err;
Mark McLoughlinefeea6d2009-06-17 11:38:28 +01001036 }
1037
1038 /* loop over the indirect descriptor table */
Paolo Bonzini5eba0402017-01-27 16:40:16 +01001039 len = address_space_cache_init(&indirect_desc_cache,
1040 vdev->dma_as,
1041 desc.addr, desc.len, false);
1042 desc_cache = &indirect_desc_cache;
Paolo Bonzini9796d0a2017-01-27 16:40:14 +01001043 if (len < desc.len) {
1044 virtio_error(vdev, "Cannot map indirect buffer");
1045 goto err;
1046 }
1047
Paolo Bonziniaa570d62016-01-31 11:29:03 +01001048 max = desc.len / sizeof(VRingDesc);
yinyin1ae27572013-08-22 14:47:16 +08001049 num_bufs = i = 0;
Jason Wang86044b22019-10-25 10:35:24 +02001050 vring_split_desc_read(vdev, &desc, desc_cache, i);
Mark McLoughlinefeea6d2009-06-17 11:38:28 +01001051 }
1052
aliguori967f97f2008-12-04 19:38:57 +00001053 do {
1054 /* If we've got too many, that implies a descriptor loop. */
Mark McLoughlin5774cf92009-06-17 11:37:32 +01001055 if (++num_bufs > max) {
Stefan Hajnoczid65abf82016-09-21 16:52:22 +01001056 virtio_error(vdev, "Looped descriptor");
1057 goto err;
aliguoribb6834c2008-12-04 21:28:28 +00001058 }
aliguori967f97f2008-12-04 19:38:57 +00001059
Paolo Bonziniaa570d62016-01-31 11:29:03 +01001060 if (desc.flags & VRING_DESC_F_WRITE) {
1061 in_total += desc.len;
aliguori967f97f2008-12-04 19:38:57 +00001062 } else {
Paolo Bonziniaa570d62016-01-31 11:29:03 +01001063 out_total += desc.len;
aliguori967f97f2008-12-04 19:38:57 +00001064 }
Michael S. Tsirkine1f7b482012-11-30 00:02:56 +02001065 if (in_total >= max_in_bytes && out_total >= max_out_bytes) {
1066 goto done;
1067 }
Stefan Hajnoczi412e0e82016-09-21 16:52:24 +01001068
Jason Wang86044b22019-10-25 10:35:24 +02001069 rc = virtqueue_split_read_next_desc(vdev, &desc, desc_cache, max, &i);
Stefan Hajnoczi412e0e82016-09-21 16:52:24 +01001070 } while (rc == VIRTQUEUE_READ_DESC_MORE);
1071
1072 if (rc == VIRTQUEUE_READ_DESC_ERROR) {
1073 goto err;
1074 }
Mark McLoughlinefeea6d2009-06-17 11:38:28 +01001075
Paolo Bonzini5eba0402017-01-27 16:40:16 +01001076 if (desc_cache == &indirect_desc_cache) {
1077 address_space_cache_destroy(&indirect_desc_cache);
Mark McLoughlinefeea6d2009-06-17 11:38:28 +01001078 total_bufs++;
Paolo Bonzini9796d0a2017-01-27 16:40:14 +01001079 } else {
1080 total_bufs = num_bufs;
1081 }
aliguori967f97f2008-12-04 19:38:57 +00001082 }
Stefan Hajnoczi4355c1a2016-09-21 16:52:25 +01001083
1084 if (rc < 0) {
1085 goto err;
1086 }
1087
Michael S. Tsirkine1f7b482012-11-30 00:02:56 +02001088done:
Paolo Bonzini5eba0402017-01-27 16:40:16 +01001089 address_space_cache_destroy(&indirect_desc_cache);
Amit Shah0d8d7692012-09-25 00:05:15 +05301090 if (in_bytes) {
1091 *in_bytes = in_total;
1092 }
1093 if (out_bytes) {
1094 *out_bytes = out_total;
1095 }
Stefan Hajnoczid65abf82016-09-21 16:52:22 +01001096 return;
1097
1098err:
1099 in_total = out_total = 0;
1100 goto done;
Amit Shah0d8d7692012-09-25 00:05:15 +05301101}
aliguori967f97f2008-12-04 19:38:57 +00001102
Jason Wang86044b22019-10-25 10:35:24 +02001103static int virtqueue_packed_read_next_desc(VirtQueue *vq,
1104 VRingPackedDesc *desc,
1105 MemoryRegionCache
1106 *desc_cache,
1107 unsigned int max,
1108 unsigned int *next,
1109 bool indirect)
1110{
1111 /* If this descriptor says it doesn't chain, we're done. */
1112 if (!indirect && !(desc->flags & VRING_DESC_F_NEXT)) {
1113 return VIRTQUEUE_READ_DESC_DONE;
1114 }
1115
1116 ++*next;
1117 if (*next == max) {
1118 if (indirect) {
1119 return VIRTQUEUE_READ_DESC_DONE;
1120 } else {
1121 (*next) -= vq->vring.num;
1122 }
1123 }
1124
1125 vring_packed_desc_read(vq->vdev, desc, desc_cache, *next, false);
1126 return VIRTQUEUE_READ_DESC_MORE;
1127}
1128
1129static void virtqueue_packed_get_avail_bytes(VirtQueue *vq,
1130 unsigned int *in_bytes,
1131 unsigned int *out_bytes,
1132 unsigned max_in_bytes,
1133 unsigned max_out_bytes)
1134{
1135 VirtIODevice *vdev = vq->vdev;
1136 unsigned int max, idx;
1137 unsigned int total_bufs, in_total, out_total;
1138 MemoryRegionCache *desc_cache;
1139 VRingMemoryRegionCaches *caches;
1140 MemoryRegionCache indirect_desc_cache = MEMORY_REGION_CACHE_INVALID;
1141 int64_t len = 0;
1142 VRingPackedDesc desc;
1143 bool wrap_counter;
1144
Dr. David Alan Gilbertb5f53d02019-10-28 16:11:09 +00001145 RCU_READ_LOCK_GUARD();
Jason Wang86044b22019-10-25 10:35:24 +02001146 idx = vq->last_avail_idx;
1147 wrap_counter = vq->last_avail_wrap_counter;
1148 total_bufs = in_total = out_total = 0;
1149
1150 max = vq->vring.num;
1151 caches = vring_get_region_caches(vq);
Stefan Hajnocziabdd16f2020-02-07 10:46:19 +00001152 if (!caches) {
1153 goto err;
1154 }
Jason Wang86044b22019-10-25 10:35:24 +02001155
1156 for (;;) {
1157 unsigned int num_bufs = total_bufs;
1158 unsigned int i = idx;
1159 int rc;
1160
1161 desc_cache = &caches->desc;
1162 vring_packed_desc_read(vdev, &desc, desc_cache, idx, true);
1163 if (!is_desc_avail(desc.flags, wrap_counter)) {
1164 break;
1165 }
1166
1167 if (desc.flags & VRING_DESC_F_INDIRECT) {
1168 if (desc.len % sizeof(VRingPackedDesc)) {
1169 virtio_error(vdev, "Invalid size for indirect buffer table");
1170 goto err;
1171 }
1172
1173 /* If we've got too many, that implies a descriptor loop. */
1174 if (num_bufs >= max) {
1175 virtio_error(vdev, "Looped descriptor");
1176 goto err;
1177 }
1178
1179 /* loop over the indirect descriptor table */
1180 len = address_space_cache_init(&indirect_desc_cache,
1181 vdev->dma_as,
1182 desc.addr, desc.len, false);
1183 desc_cache = &indirect_desc_cache;
1184 if (len < desc.len) {
1185 virtio_error(vdev, "Cannot map indirect buffer");
1186 goto err;
1187 }
1188
1189 max = desc.len / sizeof(VRingPackedDesc);
1190 num_bufs = i = 0;
1191 vring_packed_desc_read(vdev, &desc, desc_cache, i, false);
1192 }
1193
1194 do {
1195 /* If we've got too many, that implies a descriptor loop. */
1196 if (++num_bufs > max) {
1197 virtio_error(vdev, "Looped descriptor");
1198 goto err;
1199 }
1200
1201 if (desc.flags & VRING_DESC_F_WRITE) {
1202 in_total += desc.len;
1203 } else {
1204 out_total += desc.len;
1205 }
1206 if (in_total >= max_in_bytes && out_total >= max_out_bytes) {
1207 goto done;
1208 }
1209
1210 rc = virtqueue_packed_read_next_desc(vq, &desc, desc_cache, max,
1211 &i, desc_cache ==
1212 &indirect_desc_cache);
1213 } while (rc == VIRTQUEUE_READ_DESC_MORE);
1214
1215 if (desc_cache == &indirect_desc_cache) {
1216 address_space_cache_destroy(&indirect_desc_cache);
1217 total_bufs++;
1218 idx++;
1219 } else {
1220 idx += num_bufs - total_bufs;
1221 total_bufs = num_bufs;
1222 }
1223
1224 if (idx >= vq->vring.num) {
1225 idx -= vq->vring.num;
1226 wrap_counter ^= 1;
1227 }
1228 }
1229
1230 /* Record the index and wrap counter for a kick we want */
1231 vq->shadow_avail_idx = idx;
1232 vq->shadow_avail_wrap_counter = wrap_counter;
1233done:
1234 address_space_cache_destroy(&indirect_desc_cache);
1235 if (in_bytes) {
1236 *in_bytes = in_total;
1237 }
1238 if (out_bytes) {
1239 *out_bytes = out_total;
1240 }
Jason Wang86044b22019-10-25 10:35:24 +02001241 return;
1242
1243err:
1244 in_total = out_total = 0;
1245 goto done;
1246}
1247
1248void virtqueue_get_avail_bytes(VirtQueue *vq, unsigned int *in_bytes,
1249 unsigned int *out_bytes,
1250 unsigned max_in_bytes, unsigned max_out_bytes)
1251{
1252 uint16_t desc_size;
1253 VRingMemoryRegionCaches *caches;
1254
1255 if (unlikely(!vq->vring.desc)) {
1256 goto err;
1257 }
1258
1259 caches = vring_get_region_caches(vq);
Stefan Hajnocziabdd16f2020-02-07 10:46:19 +00001260 if (!caches) {
1261 goto err;
1262 }
1263
Jason Wang86044b22019-10-25 10:35:24 +02001264 desc_size = virtio_vdev_has_feature(vq->vdev, VIRTIO_F_RING_PACKED) ?
1265 sizeof(VRingPackedDesc) : sizeof(VRingDesc);
1266 if (caches->desc.len < vq->vring.num * desc_size) {
1267 virtio_error(vq->vdev, "Cannot map descriptor ring");
1268 goto err;
1269 }
1270
1271 if (virtio_vdev_has_feature(vq->vdev, VIRTIO_F_RING_PACKED)) {
1272 virtqueue_packed_get_avail_bytes(vq, in_bytes, out_bytes,
1273 max_in_bytes, max_out_bytes);
1274 } else {
1275 virtqueue_split_get_avail_bytes(vq, in_bytes, out_bytes,
1276 max_in_bytes, max_out_bytes);
1277 }
1278
1279 return;
1280err:
1281 if (in_bytes) {
1282 *in_bytes = 0;
1283 }
1284 if (out_bytes) {
1285 *out_bytes = 0;
1286 }
1287}
1288
Amit Shah0d8d7692012-09-25 00:05:15 +05301289int virtqueue_avail_bytes(VirtQueue *vq, unsigned int in_bytes,
1290 unsigned int out_bytes)
1291{
1292 unsigned int in_total, out_total;
1293
Michael S. Tsirkine1f7b482012-11-30 00:02:56 +02001294 virtqueue_get_avail_bytes(vq, &in_total, &out_total, in_bytes, out_bytes);
1295 return in_bytes <= in_total && out_bytes <= out_total;
aliguori967f97f2008-12-04 19:38:57 +00001296}
1297
Stefan Hajnocziec55da12016-09-21 16:52:21 +01001298static bool virtqueue_map_desc(VirtIODevice *vdev, unsigned int *p_num_sg,
1299 hwaddr *addr, struct iovec *iov,
Paolo Bonzini3b3b0622016-01-31 11:29:01 +01001300 unsigned int max_num_sg, bool is_write,
1301 hwaddr pa, size_t sz)
1302{
Stefan Hajnocziec55da12016-09-21 16:52:21 +01001303 bool ok = false;
Paolo Bonzini3b3b0622016-01-31 11:29:01 +01001304 unsigned num_sg = *p_num_sg;
1305 assert(num_sg <= max_num_sg);
1306
Prasad J Pandit1e7aed72016-07-27 21:07:56 +05301307 if (!sz) {
Stefan Hajnocziec55da12016-09-21 16:52:21 +01001308 virtio_error(vdev, "virtio: zero sized buffers are not allowed");
1309 goto out;
Prasad J Pandit1e7aed72016-07-27 21:07:56 +05301310 }
1311
Paolo Bonzini3b3b0622016-01-31 11:29:01 +01001312 while (sz) {
1313 hwaddr len = sz;
1314
1315 if (num_sg == max_num_sg) {
Stefan Hajnocziec55da12016-09-21 16:52:21 +01001316 virtio_error(vdev, "virtio: too many write descriptors in "
1317 "indirect table");
1318 goto out;
Paolo Bonzini3b3b0622016-01-31 11:29:01 +01001319 }
1320
Jason Wang8607f5c2016-12-30 18:09:10 +08001321 iov[num_sg].iov_base = dma_memory_map(vdev->dma_as, pa, &len,
1322 is_write ?
1323 DMA_DIRECTION_FROM_DEVICE :
1324 DMA_DIRECTION_TO_DEVICE);
Prasad J Pandit973e7172016-09-19 23:55:45 +05301325 if (!iov[num_sg].iov_base) {
Stefan Hajnocziec55da12016-09-21 16:52:21 +01001326 virtio_error(vdev, "virtio: bogus descriptor or out of resources");
1327 goto out;
Prasad J Pandit973e7172016-09-19 23:55:45 +05301328 }
1329
Paolo Bonzini3b3b0622016-01-31 11:29:01 +01001330 iov[num_sg].iov_len = len;
1331 addr[num_sg] = pa;
1332
1333 sz -= len;
1334 pa += len;
1335 num_sg++;
1336 }
Stefan Hajnocziec55da12016-09-21 16:52:21 +01001337 ok = true;
1338
1339out:
Paolo Bonzini3b3b0622016-01-31 11:29:01 +01001340 *p_num_sg = num_sg;
Stefan Hajnocziec55da12016-09-21 16:52:21 +01001341 return ok;
1342}
1343
1344/* Only used by error code paths before we have a VirtQueueElement (therefore
1345 * virtqueue_unmap_sg() can't be used). Assumes buffers weren't written to
1346 * yet.
1347 */
1348static void virtqueue_undo_map_desc(unsigned int out_num, unsigned int in_num,
1349 struct iovec *iov)
1350{
1351 unsigned int i;
1352
1353 for (i = 0; i < out_num + in_num; i++) {
1354 int is_write = i >= out_num;
1355
1356 cpu_physical_memory_unmap(iov->iov_base, iov->iov_len, is_write, 0);
1357 iov++;
1358 }
Paolo Bonzini3b3b0622016-01-31 11:29:01 +01001359}
1360
Jason Wang8607f5c2016-12-30 18:09:10 +08001361static void virtqueue_map_iovec(VirtIODevice *vdev, struct iovec *sg,
Dongli Zhange4fbf5b2018-11-02 14:16:36 +08001362 hwaddr *addr, unsigned int num_sg,
Philippe Mathieu-Daudé22953362020-02-19 19:19:53 +01001363 bool is_write)
Kevin Wolf42fb2e02010-08-03 16:54:38 +02001364{
1365 unsigned int i;
Avi Kivitya8170e52012-10-23 12:30:10 +02001366 hwaddr len;
Kevin Wolf42fb2e02010-08-03 16:54:38 +02001367
Dongli Zhange4fbf5b2018-11-02 14:16:36 +08001368 for (i = 0; i < num_sg; i++) {
Kevin Wolf42fb2e02010-08-03 16:54:38 +02001369 len = sg[i].iov_len;
Jason Wang8607f5c2016-12-30 18:09:10 +08001370 sg[i].iov_base = dma_memory_map(vdev->dma_as,
1371 addr[i], &len, is_write ?
1372 DMA_DIRECTION_FROM_DEVICE :
1373 DMA_DIRECTION_TO_DEVICE);
Michael S. Tsirkin8059fee2015-10-27 10:01:44 +02001374 if (!sg[i].iov_base) {
Michael Tokarev1a285892014-06-10 19:56:27 +04001375 error_report("virtio: error trying to map MMIO memory");
Kevin Wolf42fb2e02010-08-03 16:54:38 +02001376 exit(1);
1377 }
Paolo Bonzini3b3b0622016-01-31 11:29:01 +01001378 if (len != sg[i].iov_len) {
1379 error_report("virtio: unexpected memory split");
Michael S. Tsirkin8059fee2015-10-27 10:01:44 +02001380 exit(1);
1381 }
Kevin Wolf42fb2e02010-08-03 16:54:38 +02001382 }
1383}
1384
Jason Wang8607f5c2016-12-30 18:09:10 +08001385void virtqueue_map(VirtIODevice *vdev, VirtQueueElement *elem)
Michael S. Tsirkin8059fee2015-10-27 10:01:44 +02001386{
Philippe Mathieu-Daudé22953362020-02-19 19:19:53 +01001387 virtqueue_map_iovec(vdev, elem->in_sg, elem->in_addr, elem->in_num, true);
1388 virtqueue_map_iovec(vdev, elem->out_sg, elem->out_addr, elem->out_num,
1389 false);
Paolo Bonzini37246502016-01-31 11:29:00 +01001390}
1391
Ladi Prosekbf91bd22016-11-03 09:55:50 +01001392static void *virtqueue_alloc_element(size_t sz, unsigned out_num, unsigned in_num)
Paolo Bonzini37246502016-01-31 11:29:00 +01001393{
1394 VirtQueueElement *elem;
1395 size_t in_addr_ofs = QEMU_ALIGN_UP(sz, __alignof__(elem->in_addr[0]));
1396 size_t out_addr_ofs = in_addr_ofs + in_num * sizeof(elem->in_addr[0]);
1397 size_t out_addr_end = out_addr_ofs + out_num * sizeof(elem->out_addr[0]);
1398 size_t in_sg_ofs = QEMU_ALIGN_UP(out_addr_end, __alignof__(elem->in_sg[0]));
1399 size_t out_sg_ofs = in_sg_ofs + in_num * sizeof(elem->in_sg[0]);
1400 size_t out_sg_end = out_sg_ofs + out_num * sizeof(elem->out_sg[0]);
1401
1402 assert(sz >= sizeof(VirtQueueElement));
1403 elem = g_malloc(out_sg_end);
Paolo Bonzinib0ac4292017-06-01 15:54:47 +02001404 trace_virtqueue_alloc_element(elem, sz, in_num, out_num);
Paolo Bonzini37246502016-01-31 11:29:00 +01001405 elem->out_num = out_num;
1406 elem->in_num = in_num;
1407 elem->in_addr = (void *)elem + in_addr_ofs;
1408 elem->out_addr = (void *)elem + out_addr_ofs;
1409 elem->in_sg = (void *)elem + in_sg_ofs;
1410 elem->out_sg = (void *)elem + out_sg_ofs;
1411 return elem;
Michael S. Tsirkin8059fee2015-10-27 10:01:44 +02001412}
1413
Jason Wang86044b22019-10-25 10:35:24 +02001414static void *virtqueue_split_pop(VirtQueue *vq, size_t sz)
aliguori967f97f2008-12-04 19:38:57 +00001415{
Mark McLoughlin5774cf92009-06-17 11:37:32 +01001416 unsigned int i, head, max;
Paolo Bonzini991976f2017-01-27 16:40:18 +01001417 VRingMemoryRegionCaches *caches;
Paolo Bonzini5eba0402017-01-27 16:40:16 +01001418 MemoryRegionCache indirect_desc_cache = MEMORY_REGION_CACHE_INVALID;
1419 MemoryRegionCache *desc_cache;
1420 int64_t len;
Rusty Russellcee3ca02014-06-24 19:40:16 +02001421 VirtIODevice *vdev = vq->vdev;
Paolo Bonzini9796d0a2017-01-27 16:40:14 +01001422 VirtQueueElement *elem = NULL;
Wolfgang Bumiller37ef70b2017-09-20 08:09:33 +02001423 unsigned out_num, in_num, elem_entries;
Paolo Bonzini3b3b0622016-01-31 11:29:01 +01001424 hwaddr addr[VIRTQUEUE_MAX_SIZE];
1425 struct iovec iov[VIRTQUEUE_MAX_SIZE];
Paolo Bonziniaa570d62016-01-31 11:29:03 +01001426 VRingDesc desc;
Stefan Hajnoczi412e0e82016-09-21 16:52:24 +01001427 int rc;
aliguori967f97f2008-12-04 19:38:57 +00001428
Dr. David Alan Gilbertb5f53d02019-10-28 16:11:09 +00001429 RCU_READ_LOCK_GUARD();
Paolo Bonzini97cd9652017-01-27 16:40:20 +01001430 if (virtio_queue_empty_rcu(vq)) {
1431 goto done;
Paolo Bonzini51b19eb2016-02-04 16:26:51 +02001432 }
Vincenzo Maffionebe1fea92016-01-31 11:29:05 +01001433 /* Needed after virtio_queue_empty(), see comment in
1434 * virtqueue_num_heads(). */
1435 smp_rmb();
aliguori967f97f2008-12-04 19:38:57 +00001436
1437 /* When we start there are none of either input nor output. */
Wolfgang Bumiller37ef70b2017-09-20 08:09:33 +02001438 out_num = in_num = elem_entries = 0;
aliguori967f97f2008-12-04 19:38:57 +00001439
Mark McLoughlin5774cf92009-06-17 11:37:32 +01001440 max = vq->vring.num;
1441
Stefan Hajnocziafd90962016-07-19 13:07:13 +01001442 if (vq->inuse >= vq->vring.num) {
Stefan Hajnocziec55da12016-09-21 16:52:21 +01001443 virtio_error(vdev, "Virtqueue size exceeded");
Paolo Bonzini97cd9652017-01-27 16:40:20 +01001444 goto done;
Stefan Hajnocziafd90962016-07-19 13:07:13 +01001445 }
1446
Stefan Hajnoczifb1131b2016-09-21 16:52:26 +01001447 if (!virtqueue_get_head(vq, vq->last_avail_idx++, &head)) {
Paolo Bonzini97cd9652017-01-27 16:40:20 +01001448 goto done;
Stefan Hajnoczifb1131b2016-09-21 16:52:26 +01001449 }
1450
Cornelia Huck95129d62015-08-17 11:48:29 +02001451 if (virtio_vdev_has_feature(vdev, VIRTIO_RING_F_EVENT_IDX)) {
Michael S. Tsirkine9600c62015-02-16 22:35:46 +01001452 vring_set_avail_event(vq, vq->last_avail_idx);
Michael S. Tsirkinbcbabae2011-06-12 16:21:57 +03001453 }
Mark McLoughlinefeea6d2009-06-17 11:38:28 +01001454
Stefan Hajnoczifb1131b2016-09-21 16:52:26 +01001455 i = head;
Paolo Bonzini9796d0a2017-01-27 16:40:14 +01001456
Jason Wange0e2d642017-03-15 19:48:31 +08001457 caches = vring_get_region_caches(vq);
Stefan Hajnocziabdd16f2020-02-07 10:46:19 +00001458 if (!caches) {
1459 virtio_error(vdev, "Region caches not initialized");
1460 goto done;
1461 }
1462
Paolo Bonzini991976f2017-01-27 16:40:18 +01001463 if (caches->desc.len < max * sizeof(VRingDesc)) {
Paolo Bonzini9796d0a2017-01-27 16:40:14 +01001464 virtio_error(vdev, "Cannot map descriptor ring");
1465 goto done;
1466 }
1467
Paolo Bonzini991976f2017-01-27 16:40:18 +01001468 desc_cache = &caches->desc;
Jason Wang86044b22019-10-25 10:35:24 +02001469 vring_split_desc_read(vdev, &desc, desc_cache, i);
Paolo Bonziniaa570d62016-01-31 11:29:03 +01001470 if (desc.flags & VRING_DESC_F_INDIRECT) {
Dima Stepanov74231922019-01-15 13:08:47 +03001471 if (!desc.len || (desc.len % sizeof(VRingDesc))) {
Stefan Hajnocziec55da12016-09-21 16:52:21 +01001472 virtio_error(vdev, "Invalid size for indirect buffer table");
Paolo Bonzini9796d0a2017-01-27 16:40:14 +01001473 goto done;
Mark McLoughlinefeea6d2009-06-17 11:38:28 +01001474 }
1475
1476 /* loop over the indirect descriptor table */
Paolo Bonzini5eba0402017-01-27 16:40:16 +01001477 len = address_space_cache_init(&indirect_desc_cache, vdev->dma_as,
1478 desc.addr, desc.len, false);
1479 desc_cache = &indirect_desc_cache;
Paolo Bonzini9796d0a2017-01-27 16:40:14 +01001480 if (len < desc.len) {
1481 virtio_error(vdev, "Cannot map indirect buffer");
1482 goto done;
1483 }
1484
Paolo Bonziniaa570d62016-01-31 11:29:03 +01001485 max = desc.len / sizeof(VRingDesc);
Mark McLoughlinefeea6d2009-06-17 11:38:28 +01001486 i = 0;
Jason Wang86044b22019-10-25 10:35:24 +02001487 vring_split_desc_read(vdev, &desc, desc_cache, i);
Mark McLoughlinefeea6d2009-06-17 11:38:28 +01001488 }
1489
Kevin Wolf42fb2e02010-08-03 16:54:38 +02001490 /* Collect all the descriptors */
aliguori967f97f2008-12-04 19:38:57 +00001491 do {
Stefan Hajnocziec55da12016-09-21 16:52:21 +01001492 bool map_ok;
1493
Paolo Bonziniaa570d62016-01-31 11:29:03 +01001494 if (desc.flags & VRING_DESC_F_WRITE) {
Stefan Hajnocziec55da12016-09-21 16:52:21 +01001495 map_ok = virtqueue_map_desc(vdev, &in_num, addr + out_num,
1496 iov + out_num,
1497 VIRTQUEUE_MAX_SIZE - out_num, true,
1498 desc.addr, desc.len);
Kevin Wolf42fb2e02010-08-03 16:54:38 +02001499 } else {
Paolo Bonzini3b3b0622016-01-31 11:29:01 +01001500 if (in_num) {
Stefan Hajnocziec55da12016-09-21 16:52:21 +01001501 virtio_error(vdev, "Incorrect order for descriptors");
1502 goto err_undo_map;
Michael S. Tsirkinc8eac1c2011-06-20 13:42:27 +03001503 }
Stefan Hajnocziec55da12016-09-21 16:52:21 +01001504 map_ok = virtqueue_map_desc(vdev, &out_num, addr, iov,
1505 VIRTQUEUE_MAX_SIZE, false,
1506 desc.addr, desc.len);
1507 }
1508 if (!map_ok) {
1509 goto err_undo_map;
aliguoribb6834c2008-12-04 21:28:28 +00001510 }
aliguori967f97f2008-12-04 19:38:57 +00001511
1512 /* If we've got too many, that implies a descriptor loop. */
Wolfgang Bumiller37ef70b2017-09-20 08:09:33 +02001513 if (++elem_entries > max) {
Stefan Hajnocziec55da12016-09-21 16:52:21 +01001514 virtio_error(vdev, "Looped descriptor");
1515 goto err_undo_map;
aliguoribb6834c2008-12-04 21:28:28 +00001516 }
Stefan Hajnoczi412e0e82016-09-21 16:52:24 +01001517
Jason Wang86044b22019-10-25 10:35:24 +02001518 rc = virtqueue_split_read_next_desc(vdev, &desc, desc_cache, max, &i);
Stefan Hajnoczi412e0e82016-09-21 16:52:24 +01001519 } while (rc == VIRTQUEUE_READ_DESC_MORE);
1520
1521 if (rc == VIRTQUEUE_READ_DESC_ERROR) {
1522 goto err_undo_map;
1523 }
aliguori967f97f2008-12-04 19:38:57 +00001524
Paolo Bonzini3b3b0622016-01-31 11:29:01 +01001525 /* Now copy what we have collected and mapped */
1526 elem = virtqueue_alloc_element(sz, out_num, in_num);
aliguori967f97f2008-12-04 19:38:57 +00001527 elem->index = head;
Jason Wang86044b22019-10-25 10:35:24 +02001528 elem->ndescs = 1;
Paolo Bonzini3b3b0622016-01-31 11:29:01 +01001529 for (i = 0; i < out_num; i++) {
1530 elem->out_addr[i] = addr[i];
1531 elem->out_sg[i] = iov[i];
1532 }
1533 for (i = 0; i < in_num; i++) {
1534 elem->in_addr[i] = addr[out_num + i];
1535 elem->in_sg[i] = iov[out_num + i];
1536 }
aliguori967f97f2008-12-04 19:38:57 +00001537
1538 vq->inuse++;
1539
Stefan Hajnoczi64979a42010-05-24 13:19:21 +01001540 trace_virtqueue_pop(vq, elem, elem->in_num, elem->out_num);
Paolo Bonzini9796d0a2017-01-27 16:40:14 +01001541done:
Paolo Bonzini5eba0402017-01-27 16:40:16 +01001542 address_space_cache_destroy(&indirect_desc_cache);
Paolo Bonzini9796d0a2017-01-27 16:40:14 +01001543
Paolo Bonzini51b19eb2016-02-04 16:26:51 +02001544 return elem;
Stefan Hajnocziec55da12016-09-21 16:52:21 +01001545
1546err_undo_map:
1547 virtqueue_undo_map_desc(out_num, in_num, iov);
Paolo Bonzini9796d0a2017-01-27 16:40:14 +01001548 goto done;
aliguori967f97f2008-12-04 19:38:57 +00001549}
1550
Jason Wang86044b22019-10-25 10:35:24 +02001551static void *virtqueue_packed_pop(VirtQueue *vq, size_t sz)
1552{
1553 unsigned int i, max;
1554 VRingMemoryRegionCaches *caches;
1555 MemoryRegionCache indirect_desc_cache = MEMORY_REGION_CACHE_INVALID;
1556 MemoryRegionCache *desc_cache;
1557 int64_t len;
1558 VirtIODevice *vdev = vq->vdev;
1559 VirtQueueElement *elem = NULL;
1560 unsigned out_num, in_num, elem_entries;
1561 hwaddr addr[VIRTQUEUE_MAX_SIZE];
1562 struct iovec iov[VIRTQUEUE_MAX_SIZE];
1563 VRingPackedDesc desc;
1564 uint16_t id;
1565 int rc;
1566
Dr. David Alan Gilbertb5f53d02019-10-28 16:11:09 +00001567 RCU_READ_LOCK_GUARD();
Jason Wang86044b22019-10-25 10:35:24 +02001568 if (virtio_queue_packed_empty_rcu(vq)) {
1569 goto done;
1570 }
1571
1572 /* When we start there are none of either input nor output. */
1573 out_num = in_num = elem_entries = 0;
1574
1575 max = vq->vring.num;
1576
1577 if (vq->inuse >= vq->vring.num) {
1578 virtio_error(vdev, "Virtqueue size exceeded");
1579 goto done;
1580 }
1581
1582 i = vq->last_avail_idx;
1583
1584 caches = vring_get_region_caches(vq);
Stefan Hajnocziabdd16f2020-02-07 10:46:19 +00001585 if (!caches) {
1586 virtio_error(vdev, "Region caches not initialized");
1587 goto done;
1588 }
1589
Jason Wang86044b22019-10-25 10:35:24 +02001590 if (caches->desc.len < max * sizeof(VRingDesc)) {
1591 virtio_error(vdev, "Cannot map descriptor ring");
1592 goto done;
1593 }
1594
1595 desc_cache = &caches->desc;
1596 vring_packed_desc_read(vdev, &desc, desc_cache, i, true);
1597 id = desc.id;
1598 if (desc.flags & VRING_DESC_F_INDIRECT) {
1599 if (desc.len % sizeof(VRingPackedDesc)) {
1600 virtio_error(vdev, "Invalid size for indirect buffer table");
1601 goto done;
1602 }
1603
1604 /* loop over the indirect descriptor table */
1605 len = address_space_cache_init(&indirect_desc_cache, vdev->dma_as,
1606 desc.addr, desc.len, false);
1607 desc_cache = &indirect_desc_cache;
1608 if (len < desc.len) {
1609 virtio_error(vdev, "Cannot map indirect buffer");
1610 goto done;
1611 }
1612
1613 max = desc.len / sizeof(VRingPackedDesc);
1614 i = 0;
1615 vring_packed_desc_read(vdev, &desc, desc_cache, i, false);
1616 }
1617
1618 /* Collect all the descriptors */
1619 do {
1620 bool map_ok;
1621
1622 if (desc.flags & VRING_DESC_F_WRITE) {
1623 map_ok = virtqueue_map_desc(vdev, &in_num, addr + out_num,
1624 iov + out_num,
1625 VIRTQUEUE_MAX_SIZE - out_num, true,
1626 desc.addr, desc.len);
1627 } else {
1628 if (in_num) {
1629 virtio_error(vdev, "Incorrect order for descriptors");
1630 goto err_undo_map;
1631 }
1632 map_ok = virtqueue_map_desc(vdev, &out_num, addr, iov,
1633 VIRTQUEUE_MAX_SIZE, false,
1634 desc.addr, desc.len);
1635 }
1636 if (!map_ok) {
1637 goto err_undo_map;
1638 }
1639
1640 /* If we've got too many, that implies a descriptor loop. */
1641 if (++elem_entries > max) {
1642 virtio_error(vdev, "Looped descriptor");
1643 goto err_undo_map;
1644 }
1645
1646 rc = virtqueue_packed_read_next_desc(vq, &desc, desc_cache, max, &i,
1647 desc_cache ==
1648 &indirect_desc_cache);
1649 } while (rc == VIRTQUEUE_READ_DESC_MORE);
1650
1651 /* Now copy what we have collected and mapped */
1652 elem = virtqueue_alloc_element(sz, out_num, in_num);
1653 for (i = 0; i < out_num; i++) {
1654 elem->out_addr[i] = addr[i];
1655 elem->out_sg[i] = iov[i];
1656 }
1657 for (i = 0; i < in_num; i++) {
1658 elem->in_addr[i] = addr[out_num + i];
1659 elem->in_sg[i] = iov[out_num + i];
1660 }
1661
1662 elem->index = id;
1663 elem->ndescs = (desc_cache == &indirect_desc_cache) ? 1 : elem_entries;
1664 vq->last_avail_idx += elem->ndescs;
1665 vq->inuse += elem->ndescs;
1666
1667 if (vq->last_avail_idx >= vq->vring.num) {
1668 vq->last_avail_idx -= vq->vring.num;
1669 vq->last_avail_wrap_counter ^= 1;
1670 }
1671
1672 vq->shadow_avail_idx = vq->last_avail_idx;
1673 vq->shadow_avail_wrap_counter = vq->last_avail_wrap_counter;
1674
1675 trace_virtqueue_pop(vq, elem, elem->in_num, elem->out_num);
1676done:
1677 address_space_cache_destroy(&indirect_desc_cache);
Jason Wang86044b22019-10-25 10:35:24 +02001678
1679 return elem;
1680
1681err_undo_map:
1682 virtqueue_undo_map_desc(out_num, in_num, iov);
1683 goto done;
1684}
1685
1686void *virtqueue_pop(VirtQueue *vq, size_t sz)
1687{
Michael Roth9d7bd082019-11-19 18:50:03 -06001688 if (virtio_device_disabled(vq->vdev)) {
Jason Wang86044b22019-10-25 10:35:24 +02001689 return NULL;
1690 }
1691
1692 if (virtio_vdev_has_feature(vq->vdev, VIRTIO_F_RING_PACKED)) {
1693 return virtqueue_packed_pop(vq, sz);
1694 } else {
1695 return virtqueue_split_pop(vq, sz);
1696 }
1697}
1698
1699static unsigned int virtqueue_packed_drop_all(VirtQueue *vq)
1700{
1701 VRingMemoryRegionCaches *caches;
1702 MemoryRegionCache *desc_cache;
1703 unsigned int dropped = 0;
1704 VirtQueueElement elem = {};
1705 VirtIODevice *vdev = vq->vdev;
1706 VRingPackedDesc desc;
1707
1708 caches = vring_get_region_caches(vq);
Stefan Hajnocziabdd16f2020-02-07 10:46:19 +00001709 if (!caches) {
1710 return 0;
1711 }
1712
Jason Wang86044b22019-10-25 10:35:24 +02001713 desc_cache = &caches->desc;
1714
1715 virtio_queue_set_notification(vq, 0);
1716
1717 while (vq->inuse < vq->vring.num) {
1718 unsigned int idx = vq->last_avail_idx;
1719 /*
1720 * works similar to virtqueue_pop but does not map buffers
1721 * and does not allocate any memory.
1722 */
1723 vring_packed_desc_read(vdev, &desc, desc_cache,
1724 vq->last_avail_idx , true);
1725 if (!is_desc_avail(desc.flags, vq->last_avail_wrap_counter)) {
1726 break;
1727 }
1728 elem.index = desc.id;
1729 elem.ndescs = 1;
1730 while (virtqueue_packed_read_next_desc(vq, &desc, desc_cache,
1731 vq->vring.num, &idx, false)) {
1732 ++elem.ndescs;
1733 }
1734 /*
1735 * immediately push the element, nothing to unmap
1736 * as both in_num and out_num are set to 0.
1737 */
1738 virtqueue_push(vq, &elem, 0);
1739 dropped++;
1740 vq->last_avail_idx += elem.ndescs;
1741 if (vq->last_avail_idx >= vq->vring.num) {
1742 vq->last_avail_idx -= vq->vring.num;
1743 vq->last_avail_wrap_counter ^= 1;
1744 }
1745 }
1746
1747 return dropped;
1748}
1749
1750static unsigned int virtqueue_split_drop_all(VirtQueue *vq)
Yuri Benditovich54e17702016-12-13 10:12:07 +02001751{
1752 unsigned int dropped = 0;
1753 VirtQueueElement elem = {};
1754 VirtIODevice *vdev = vq->vdev;
1755 bool fEventIdx = virtio_vdev_has_feature(vdev, VIRTIO_RING_F_EVENT_IDX);
1756
Yuri Benditovich54e17702016-12-13 10:12:07 +02001757 while (!virtio_queue_empty(vq) && vq->inuse < vq->vring.num) {
1758 /* works similar to virtqueue_pop but does not map buffers
1759 * and does not allocate any memory */
1760 smp_rmb();
1761 if (!virtqueue_get_head(vq, vq->last_avail_idx, &elem.index)) {
1762 break;
1763 }
1764 vq->inuse++;
1765 vq->last_avail_idx++;
1766 if (fEventIdx) {
1767 vring_set_avail_event(vq, vq->last_avail_idx);
1768 }
1769 /* immediately push the element, nothing to unmap
1770 * as both in_num and out_num are set to 0 */
1771 virtqueue_push(vq, &elem, 0);
1772 dropped++;
1773 }
1774
1775 return dropped;
1776}
1777
Jason Wang86044b22019-10-25 10:35:24 +02001778/* virtqueue_drop_all:
1779 * @vq: The #VirtQueue
1780 * Drops all queued buffers and indicates them to the guest
1781 * as if they are done. Useful when buffers can not be
1782 * processed but must be returned to the guest.
1783 */
1784unsigned int virtqueue_drop_all(VirtQueue *vq)
1785{
1786 struct VirtIODevice *vdev = vq->vdev;
1787
Michael Roth9d7bd082019-11-19 18:50:03 -06001788 if (virtio_device_disabled(vq->vdev)) {
Jason Wang86044b22019-10-25 10:35:24 +02001789 return 0;
1790 }
1791
1792 if (virtio_vdev_has_feature(vdev, VIRTIO_F_RING_PACKED)) {
1793 return virtqueue_packed_drop_all(vq);
1794 } else {
1795 return virtqueue_split_drop_all(vq);
1796 }
1797}
1798
Paolo Bonzini37246502016-01-31 11:29:00 +01001799/* Reading and writing a structure directly to QEMUFile is *awful*, but
1800 * it is what QEMU has always done by mistake. We can change it sooner
1801 * or later by bumping the version number of the affected vm states.
1802 * In the meanwhile, since the in-memory layout of VirtQueueElement
1803 * has changed, we need to marshal to and from the layout that was
1804 * used before the change.
1805 */
1806typedef struct VirtQueueElementOld {
1807 unsigned int index;
1808 unsigned int out_num;
1809 unsigned int in_num;
1810 hwaddr in_addr[VIRTQUEUE_MAX_SIZE];
1811 hwaddr out_addr[VIRTQUEUE_MAX_SIZE];
1812 struct iovec in_sg[VIRTQUEUE_MAX_SIZE];
1813 struct iovec out_sg[VIRTQUEUE_MAX_SIZE];
1814} VirtQueueElementOld;
1815
Jason Wang8607f5c2016-12-30 18:09:10 +08001816void *qemu_get_virtqueue_element(VirtIODevice *vdev, QEMUFile *f, size_t sz)
Paolo Bonziniab281c12016-01-31 11:28:59 +01001817{
Paolo Bonzini37246502016-01-31 11:29:00 +01001818 VirtQueueElement *elem;
1819 VirtQueueElementOld data;
1820 int i;
1821
1822 qemu_get_buffer(f, (uint8_t *)&data, sizeof(VirtQueueElementOld));
1823
Michael S. Tsirkin6bdc21c2017-01-18 21:32:22 +02001824 /* TODO: teach all callers that this can fail, and return failure instead
1825 * of asserting here.
Eric Blake262a69f2017-09-11 16:13:20 -05001826 * This is just one thing (there are probably more) that must be
1827 * fixed before we can allow NDEBUG compilation.
Michael S. Tsirkin6bdc21c2017-01-18 21:32:22 +02001828 */
Michael S. Tsirkin6bdc21c2017-01-18 21:32:22 +02001829 assert(ARRAY_SIZE(data.in_addr) >= data.in_num);
1830 assert(ARRAY_SIZE(data.out_addr) >= data.out_num);
1831
Paolo Bonzini37246502016-01-31 11:29:00 +01001832 elem = virtqueue_alloc_element(sz, data.out_num, data.in_num);
1833 elem->index = data.index;
1834
1835 for (i = 0; i < elem->in_num; i++) {
1836 elem->in_addr[i] = data.in_addr[i];
1837 }
1838
1839 for (i = 0; i < elem->out_num; i++) {
1840 elem->out_addr[i] = data.out_addr[i];
1841 }
1842
1843 for (i = 0; i < elem->in_num; i++) {
1844 /* Base is overwritten by virtqueue_map. */
1845 elem->in_sg[i].iov_base = 0;
1846 elem->in_sg[i].iov_len = data.in_sg[i].iov_len;
1847 }
1848
1849 for (i = 0; i < elem->out_num; i++) {
1850 /* Base is overwritten by virtqueue_map. */
1851 elem->out_sg[i].iov_base = 0;
1852 elem->out_sg[i].iov_len = data.out_sg[i].iov_len;
1853 }
1854
Jason Wang86044b22019-10-25 10:35:24 +02001855 if (virtio_host_has_feature(vdev, VIRTIO_F_RING_PACKED)) {
1856 qemu_get_be32s(f, &elem->ndescs);
1857 }
1858
Jason Wang8607f5c2016-12-30 18:09:10 +08001859 virtqueue_map(vdev, elem);
Paolo Bonziniab281c12016-01-31 11:28:59 +01001860 return elem;
1861}
1862
Jason Wang86044b22019-10-25 10:35:24 +02001863void qemu_put_virtqueue_element(VirtIODevice *vdev, QEMUFile *f,
1864 VirtQueueElement *elem)
Paolo Bonziniab281c12016-01-31 11:28:59 +01001865{
Paolo Bonzini37246502016-01-31 11:29:00 +01001866 VirtQueueElementOld data;
1867 int i;
1868
1869 memset(&data, 0, sizeof(data));
1870 data.index = elem->index;
1871 data.in_num = elem->in_num;
1872 data.out_num = elem->out_num;
1873
1874 for (i = 0; i < elem->in_num; i++) {
1875 data.in_addr[i] = elem->in_addr[i];
1876 }
1877
1878 for (i = 0; i < elem->out_num; i++) {
1879 data.out_addr[i] = elem->out_addr[i];
1880 }
1881
1882 for (i = 0; i < elem->in_num; i++) {
1883 /* Base is overwritten by virtqueue_map when loading. Do not
1884 * save it, as it would leak the QEMU address space layout. */
1885 data.in_sg[i].iov_len = elem->in_sg[i].iov_len;
1886 }
1887
1888 for (i = 0; i < elem->out_num; i++) {
1889 /* Do not save iov_base as above. */
1890 data.out_sg[i].iov_len = elem->out_sg[i].iov_len;
1891 }
Jason Wang86044b22019-10-25 10:35:24 +02001892
1893 if (virtio_host_has_feature(vdev, VIRTIO_F_RING_PACKED)) {
1894 qemu_put_be32s(f, &elem->ndescs);
1895 }
1896
Paolo Bonzini37246502016-01-31 11:29:00 +01001897 qemu_put_buffer(f, (uint8_t *)&data, sizeof(VirtQueueElementOld));
Paolo Bonziniab281c12016-01-31 11:28:59 +01001898}
1899
aliguori967f97f2008-12-04 19:38:57 +00001900/* virtio device */
Michael S. Tsirkin7055e682009-06-21 19:50:13 +03001901static void virtio_notify_vector(VirtIODevice *vdev, uint16_t vector)
1902{
KONRAD Frederic1c819442013-04-24 10:21:21 +02001903 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
1904 VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
1905
Michael Roth9d7bd082019-11-19 18:50:03 -06001906 if (virtio_device_disabled(vdev)) {
Stefan Hajnoczif5ed3662016-09-21 16:52:19 +01001907 return;
1908 }
1909
KONRAD Frederic1c819442013-04-24 10:21:21 +02001910 if (k->notify) {
1911 k->notify(qbus->parent, vector);
Michael S. Tsirkin7055e682009-06-21 19:50:13 +03001912 }
1913}
aliguori967f97f2008-12-04 19:38:57 +00001914
Paul Brook53c25ce2009-05-18 14:51:59 +01001915void virtio_update_irq(VirtIODevice *vdev)
aliguori967f97f2008-12-04 19:38:57 +00001916{
Michael S. Tsirkin7055e682009-06-21 19:50:13 +03001917 virtio_notify_vector(vdev, VIRTIO_NO_VECTOR);
aliguori967f97f2008-12-04 19:38:57 +00001918}
1919
Cornelia Huck0b352fd2015-06-04 12:34:15 +02001920static int virtio_validate_features(VirtIODevice *vdev)
1921{
1922 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
1923
Jason Wang8607f5c2016-12-30 18:09:10 +08001924 if (virtio_host_has_feature(vdev, VIRTIO_F_IOMMU_PLATFORM) &&
1925 !virtio_vdev_has_feature(vdev, VIRTIO_F_IOMMU_PLATFORM)) {
1926 return -EFAULT;
1927 }
1928
Cornelia Huck0b352fd2015-06-04 12:34:15 +02001929 if (k->validate_features) {
1930 return k->validate_features(vdev);
1931 } else {
1932 return 0;
1933 }
1934}
1935
1936int virtio_set_status(VirtIODevice *vdev, uint8_t val)
Stefan Hajnoczi4e1837f2011-09-13 13:34:37 +01001937{
KONRAD Frederic181103c2013-04-24 10:21:20 +02001938 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
Stefan Hajnoczi4e1837f2011-09-13 13:34:37 +01001939 trace_virtio_set_status(vdev, val);
1940
Cornelia Huck95129d62015-08-17 11:48:29 +02001941 if (virtio_vdev_has_feature(vdev, VIRTIO_F_VERSION_1)) {
Cornelia Huck0b352fd2015-06-04 12:34:15 +02001942 if (!(vdev->status & VIRTIO_CONFIG_S_FEATURES_OK) &&
1943 val & VIRTIO_CONFIG_S_FEATURES_OK) {
1944 int ret = virtio_validate_features(vdev);
1945
1946 if (ret) {
1947 return ret;
1948 }
1949 }
1950 }
Xie Yongjie57f2c32019-06-26 10:31:26 +08001951
Xie Yongji4c5cf372019-06-26 10:31:30 +08001952 if ((vdev->status & VIRTIO_CONFIG_S_DRIVER_OK) !=
1953 (val & VIRTIO_CONFIG_S_DRIVER_OK)) {
1954 virtio_set_started(vdev, val & VIRTIO_CONFIG_S_DRIVER_OK);
1955 }
Xie Yongjibadaf792019-03-20 19:26:40 +08001956
KONRAD Frederic181103c2013-04-24 10:21:20 +02001957 if (k->set_status) {
1958 k->set_status(vdev, val);
Stefan Hajnoczi4e1837f2011-09-13 13:34:37 +01001959 }
1960 vdev->status = val;
Xie Yongjibadaf792019-03-20 19:26:40 +08001961
Cornelia Huck0b352fd2015-06-04 12:34:15 +02001962 return 0;
Stefan Hajnoczi4e1837f2011-09-13 13:34:37 +01001963}
1964
Greg Kurz616a6552014-06-24 19:38:54 +02001965static enum virtio_device_endian virtio_default_endian(void)
1966{
1967 if (target_words_bigendian()) {
1968 return VIRTIO_DEVICE_ENDIAN_BIG;
1969 } else {
1970 return VIRTIO_DEVICE_ENDIAN_LITTLE;
1971 }
1972}
1973
1974static enum virtio_device_endian virtio_current_cpu_endian(void)
1975{
1976 CPUClass *cc = CPU_GET_CLASS(current_cpu);
1977
1978 if (cc->virtio_is_big_endian(current_cpu)) {
1979 return VIRTIO_DEVICE_ENDIAN_BIG;
1980 } else {
1981 return VIRTIO_DEVICE_ENDIAN_LITTLE;
1982 }
1983}
1984
Paul Brook53c25ce2009-05-18 14:51:59 +01001985void virtio_reset(void *opaque)
aliguori967f97f2008-12-04 19:38:57 +00001986{
1987 VirtIODevice *vdev = opaque;
KONRAD Frederic181103c2013-04-24 10:21:20 +02001988 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
aliguori967f97f2008-12-04 19:38:57 +00001989 int i;
1990
Michael S. Tsirkine0c472d2010-09-27 18:32:52 +02001991 virtio_set_status(vdev, 0);
Greg Kurz616a6552014-06-24 19:38:54 +02001992 if (current_cpu) {
1993 /* Guest initiated reset */
1994 vdev->device_endian = virtio_current_cpu_endian();
1995 } else {
1996 /* System reset */
1997 vdev->device_endian = virtio_default_endian();
1998 }
Michael S. Tsirkine0c472d2010-09-27 18:32:52 +02001999
KONRAD Frederic181103c2013-04-24 10:21:20 +02002000 if (k->reset) {
2001 k->reset(vdev);
2002 }
aliguori967f97f2008-12-04 19:38:57 +00002003
Xie Yongji868a8f42019-06-26 10:31:28 +08002004 vdev->start_on_kick = false;
Xie Yongjibadaf792019-03-20 19:26:40 +08002005 vdev->started = false;
Stefan Hajnoczif5ed3662016-09-21 16:52:19 +01002006 vdev->broken = false;
Michael S. Tsirkin704a76f2010-01-10 13:52:47 +02002007 vdev->guest_features = 0;
aliguori967f97f2008-12-04 19:38:57 +00002008 vdev->queue_sel = 0;
2009 vdev->status = 0;
Michael Roth9d7bd082019-11-19 18:50:03 -06002010 vdev->disabled = false;
Stefan Hajnoczid73415a2020-09-23 11:56:46 +01002011 qatomic_set(&vdev->isr, 0);
Michael S. Tsirkin7055e682009-06-21 19:50:13 +03002012 vdev->config_vector = VIRTIO_NO_VECTOR;
2013 virtio_notify_vector(vdev, vdev->config_vector);
aliguori967f97f2008-12-04 19:38:57 +00002014
Jason Wang87b3bd12015-05-29 14:15:31 +08002015 for(i = 0; i < VIRTIO_QUEUE_MAX; i++) {
aliguori967f97f2008-12-04 19:38:57 +00002016 vdev->vq[i].vring.desc = 0;
2017 vdev->vq[i].vring.avail = 0;
2018 vdev->vq[i].vring.used = 0;
2019 vdev->vq[i].last_avail_idx = 0;
Vincenzo Maffionebe1fea92016-01-31 11:29:05 +01002020 vdev->vq[i].shadow_avail_idx = 0;
Vincenzo Maffioneb796fcd2016-01-31 11:29:04 +01002021 vdev->vq[i].used_idx = 0;
Jason Wang86044b22019-10-25 10:35:24 +02002022 vdev->vq[i].last_avail_wrap_counter = true;
2023 vdev->vq[i].shadow_avail_wrap_counter = true;
2024 vdev->vq[i].used_wrap_counter = true;
Jason Wange0d686b2015-04-23 14:21:46 +08002025 virtio_queue_set_vector(vdev, i, VIRTIO_NO_VECTOR);
Michael S. Tsirkinbcbabae2011-06-12 16:21:57 +03002026 vdev->vq[i].signalled_used = 0;
2027 vdev->vq[i].signalled_used_valid = false;
Stefan Hajnoczi332fa822017-01-12 11:46:10 +00002028 vdev->vq[i].notification = true;
Cornelia Huck46c5d082015-09-11 15:16:41 +02002029 vdev->vq[i].vring.num = vdev->vq[i].vring.num_default;
Stefan Hajnoczi4b7f91e2016-09-07 11:51:25 -04002030 vdev->vq[i].inuse = 0;
Jason Wange0e2d642017-03-15 19:48:31 +08002031 virtio_virtqueue_reset_region_cache(&vdev->vq[i]);
aliguori967f97f2008-12-04 19:38:57 +00002032 }
2033}
2034
Paul Brook53c25ce2009-05-18 14:51:59 +01002035uint32_t virtio_config_readb(VirtIODevice *vdev, uint32_t addr)
aliguori967f97f2008-12-04 19:38:57 +00002036{
KONRAD Frederic181103c2013-04-24 10:21:20 +02002037 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
aliguori967f97f2008-12-04 19:38:57 +00002038 uint8_t val;
2039
Jason Wang5f5a1312013-05-07 13:42:49 +08002040 if (addr + sizeof(val) > vdev->config_len) {
aliguori967f97f2008-12-04 19:38:57 +00002041 return (uint32_t)-1;
Jason Wang5f5a1312013-05-07 13:42:49 +08002042 }
2043
2044 k->get_config(vdev, vdev->config);
aliguori967f97f2008-12-04 19:38:57 +00002045
Alexander Graf06dbfc62012-01-10 23:33:10 +01002046 val = ldub_p(vdev->config + addr);
aliguori967f97f2008-12-04 19:38:57 +00002047 return val;
2048}
2049
Paul Brook53c25ce2009-05-18 14:51:59 +01002050uint32_t virtio_config_readw(VirtIODevice *vdev, uint32_t addr)
aliguori967f97f2008-12-04 19:38:57 +00002051{
KONRAD Frederic181103c2013-04-24 10:21:20 +02002052 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
aliguori967f97f2008-12-04 19:38:57 +00002053 uint16_t val;
2054
Jason Wang5f5a1312013-05-07 13:42:49 +08002055 if (addr + sizeof(val) > vdev->config_len) {
aliguori967f97f2008-12-04 19:38:57 +00002056 return (uint32_t)-1;
Jason Wang5f5a1312013-05-07 13:42:49 +08002057 }
2058
2059 k->get_config(vdev, vdev->config);
aliguori967f97f2008-12-04 19:38:57 +00002060
Alexander Graf06dbfc62012-01-10 23:33:10 +01002061 val = lduw_p(vdev->config + addr);
aliguori967f97f2008-12-04 19:38:57 +00002062 return val;
2063}
2064
Paul Brook53c25ce2009-05-18 14:51:59 +01002065uint32_t virtio_config_readl(VirtIODevice *vdev, uint32_t addr)
aliguori967f97f2008-12-04 19:38:57 +00002066{
KONRAD Frederic181103c2013-04-24 10:21:20 +02002067 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
aliguori967f97f2008-12-04 19:38:57 +00002068 uint32_t val;
2069
Jason Wang5f5a1312013-05-07 13:42:49 +08002070 if (addr + sizeof(val) > vdev->config_len) {
aliguori967f97f2008-12-04 19:38:57 +00002071 return (uint32_t)-1;
Jason Wang5f5a1312013-05-07 13:42:49 +08002072 }
2073
2074 k->get_config(vdev, vdev->config);
aliguori967f97f2008-12-04 19:38:57 +00002075
Alexander Graf06dbfc62012-01-10 23:33:10 +01002076 val = ldl_p(vdev->config + addr);
aliguori967f97f2008-12-04 19:38:57 +00002077 return val;
2078}
2079
Paul Brook53c25ce2009-05-18 14:51:59 +01002080void virtio_config_writeb(VirtIODevice *vdev, uint32_t addr, uint32_t data)
aliguori967f97f2008-12-04 19:38:57 +00002081{
KONRAD Frederic181103c2013-04-24 10:21:20 +02002082 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
aliguori967f97f2008-12-04 19:38:57 +00002083 uint8_t val = data;
2084
Jason Wang5f5a1312013-05-07 13:42:49 +08002085 if (addr + sizeof(val) > vdev->config_len) {
aliguori967f97f2008-12-04 19:38:57 +00002086 return;
Jason Wang5f5a1312013-05-07 13:42:49 +08002087 }
aliguori967f97f2008-12-04 19:38:57 +00002088
Alexander Graf06dbfc62012-01-10 23:33:10 +01002089 stb_p(vdev->config + addr, val);
aliguori967f97f2008-12-04 19:38:57 +00002090
KONRAD Frederic181103c2013-04-24 10:21:20 +02002091 if (k->set_config) {
2092 k->set_config(vdev, vdev->config);
2093 }
aliguori967f97f2008-12-04 19:38:57 +00002094}
2095
Paul Brook53c25ce2009-05-18 14:51:59 +01002096void virtio_config_writew(VirtIODevice *vdev, uint32_t addr, uint32_t data)
aliguori967f97f2008-12-04 19:38:57 +00002097{
KONRAD Frederic181103c2013-04-24 10:21:20 +02002098 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
aliguori967f97f2008-12-04 19:38:57 +00002099 uint16_t val = data;
2100
Jason Wang5f5a1312013-05-07 13:42:49 +08002101 if (addr + sizeof(val) > vdev->config_len) {
aliguori967f97f2008-12-04 19:38:57 +00002102 return;
Jason Wang5f5a1312013-05-07 13:42:49 +08002103 }
aliguori967f97f2008-12-04 19:38:57 +00002104
Alexander Graf06dbfc62012-01-10 23:33:10 +01002105 stw_p(vdev->config + addr, val);
aliguori967f97f2008-12-04 19:38:57 +00002106
KONRAD Frederic181103c2013-04-24 10:21:20 +02002107 if (k->set_config) {
2108 k->set_config(vdev, vdev->config);
2109 }
aliguori967f97f2008-12-04 19:38:57 +00002110}
2111
Paul Brook53c25ce2009-05-18 14:51:59 +01002112void virtio_config_writel(VirtIODevice *vdev, uint32_t addr, uint32_t data)
aliguori967f97f2008-12-04 19:38:57 +00002113{
KONRAD Frederic181103c2013-04-24 10:21:20 +02002114 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
aliguori967f97f2008-12-04 19:38:57 +00002115 uint32_t val = data;
2116
Jason Wang5f5a1312013-05-07 13:42:49 +08002117 if (addr + sizeof(val) > vdev->config_len) {
aliguori967f97f2008-12-04 19:38:57 +00002118 return;
Jason Wang5f5a1312013-05-07 13:42:49 +08002119 }
aliguori967f97f2008-12-04 19:38:57 +00002120
Alexander Graf06dbfc62012-01-10 23:33:10 +01002121 stl_p(vdev->config + addr, val);
aliguori967f97f2008-12-04 19:38:57 +00002122
KONRAD Frederic181103c2013-04-24 10:21:20 +02002123 if (k->set_config) {
2124 k->set_config(vdev, vdev->config);
2125 }
aliguori967f97f2008-12-04 19:38:57 +00002126}
2127
Michael S. Tsirkinadfb7432015-06-04 12:34:24 +02002128uint32_t virtio_config_modern_readb(VirtIODevice *vdev, uint32_t addr)
2129{
2130 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
2131 uint8_t val;
2132
2133 if (addr + sizeof(val) > vdev->config_len) {
2134 return (uint32_t)-1;
2135 }
2136
2137 k->get_config(vdev, vdev->config);
2138
2139 val = ldub_p(vdev->config + addr);
2140 return val;
2141}
2142
2143uint32_t virtio_config_modern_readw(VirtIODevice *vdev, uint32_t addr)
2144{
2145 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
2146 uint16_t val;
2147
2148 if (addr + sizeof(val) > vdev->config_len) {
2149 return (uint32_t)-1;
2150 }
2151
2152 k->get_config(vdev, vdev->config);
2153
2154 val = lduw_le_p(vdev->config + addr);
2155 return val;
2156}
2157
2158uint32_t virtio_config_modern_readl(VirtIODevice *vdev, uint32_t addr)
2159{
2160 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
2161 uint32_t val;
2162
2163 if (addr + sizeof(val) > vdev->config_len) {
2164 return (uint32_t)-1;
2165 }
2166
2167 k->get_config(vdev, vdev->config);
2168
2169 val = ldl_le_p(vdev->config + addr);
2170 return val;
2171}
2172
2173void virtio_config_modern_writeb(VirtIODevice *vdev,
2174 uint32_t addr, uint32_t data)
2175{
2176 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
2177 uint8_t val = data;
2178
2179 if (addr + sizeof(val) > vdev->config_len) {
2180 return;
2181 }
2182
2183 stb_p(vdev->config + addr, val);
2184
2185 if (k->set_config) {
2186 k->set_config(vdev, vdev->config);
2187 }
2188}
2189
2190void virtio_config_modern_writew(VirtIODevice *vdev,
2191 uint32_t addr, uint32_t data)
2192{
2193 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
2194 uint16_t val = data;
2195
2196 if (addr + sizeof(val) > vdev->config_len) {
2197 return;
2198 }
2199
2200 stw_le_p(vdev->config + addr, val);
2201
2202 if (k->set_config) {
2203 k->set_config(vdev, vdev->config);
2204 }
2205}
2206
2207void virtio_config_modern_writel(VirtIODevice *vdev,
2208 uint32_t addr, uint32_t data)
2209{
2210 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
2211 uint32_t val = data;
2212
2213 if (addr + sizeof(val) > vdev->config_len) {
2214 return;
2215 }
2216
2217 stl_le_p(vdev->config + addr, val);
2218
2219 if (k->set_config) {
2220 k->set_config(vdev, vdev->config);
2221 }
2222}
2223
Avi Kivitya8170e52012-10-23 12:30:10 +02002224void virtio_queue_set_addr(VirtIODevice *vdev, int n, hwaddr addr)
aliguori967f97f2008-12-04 19:38:57 +00002225{
Prasad J Pandit758ead32017-11-29 23:14:27 +05302226 if (!vdev->vq[n].vring.num) {
2227 return;
2228 }
Cornelia Huckab223c92015-06-04 12:34:12 +02002229 vdev->vq[n].vring.desc = addr;
2230 virtio_queue_update_rings(vdev, n);
Paul Brook53c25ce2009-05-18 14:51:59 +01002231}
aliguori967f97f2008-12-04 19:38:57 +00002232
Avi Kivitya8170e52012-10-23 12:30:10 +02002233hwaddr virtio_queue_get_addr(VirtIODevice *vdev, int n)
Paul Brook53c25ce2009-05-18 14:51:59 +01002234{
Cornelia Huckab223c92015-06-04 12:34:12 +02002235 return vdev->vq[n].vring.desc;
2236}
2237
2238void virtio_queue_set_rings(VirtIODevice *vdev, int n, hwaddr desc,
2239 hwaddr avail, hwaddr used)
2240{
Prasad J Pandit758ead32017-11-29 23:14:27 +05302241 if (!vdev->vq[n].vring.num) {
2242 return;
2243 }
Cornelia Huckab223c92015-06-04 12:34:12 +02002244 vdev->vq[n].vring.desc = desc;
2245 vdev->vq[n].vring.avail = avail;
2246 vdev->vq[n].vring.used = used;
Paolo Bonzinic611c762017-01-27 16:40:17 +01002247 virtio_init_region_cache(vdev, n);
Paul Brook53c25ce2009-05-18 14:51:59 +01002248}
aliguori967f97f2008-12-04 19:38:57 +00002249
Peter Maydelle63c0ba2013-07-16 13:25:07 +01002250void virtio_queue_set_num(VirtIODevice *vdev, int n, int num)
2251{
Peter Maydellf6049f42013-07-26 16:41:27 +01002252 /* Don't allow guest to flip queue between existent and
2253 * nonexistent states, or to set it to an invalid size.
2254 */
2255 if (!!num != !!vdev->vq[n].vring.num ||
2256 num > VIRTQUEUE_MAX_SIZE ||
2257 num < 0) {
2258 return;
Peter Maydelle63c0ba2013-07-16 13:25:07 +01002259 }
Peter Maydellf6049f42013-07-26 16:41:27 +01002260 vdev->vq[n].vring.num = num;
Peter Maydelle63c0ba2013-07-16 13:25:07 +01002261}
2262
Jason Wange0d686b2015-04-23 14:21:46 +08002263VirtQueue *virtio_vector_first_queue(VirtIODevice *vdev, uint16_t vector)
2264{
2265 return QLIST_FIRST(&vdev->vector_queues[vector]);
2266}
2267
2268VirtQueue *virtio_vector_next_queue(VirtQueue *vq)
2269{
2270 return QLIST_NEXT(vq, node);
2271}
2272
Paul Brook53c25ce2009-05-18 14:51:59 +01002273int virtio_queue_get_num(VirtIODevice *vdev, int n)
2274{
2275 return vdev->vq[n].vring.num;
2276}
2277
Michael S. Tsirkin8c797e72017-01-12 23:26:22 +02002278int virtio_queue_get_max_num(VirtIODevice *vdev, int n)
2279{
2280 return vdev->vq[n].vring.num_default;
2281}
2282
Jason Wang8ad176a2015-05-29 14:15:26 +08002283int virtio_get_num_queues(VirtIODevice *vdev)
2284{
2285 int i;
2286
Jason Wang87b3bd12015-05-29 14:15:31 +08002287 for (i = 0; i < VIRTIO_QUEUE_MAX; i++) {
Jason Wang8ad176a2015-05-29 14:15:26 +08002288 if (!virtio_queue_get_num(vdev, i)) {
2289 break;
2290 }
2291 }
2292
2293 return i;
2294}
2295
Peter Maydell6ce69d12013-07-16 13:25:08 +01002296void virtio_queue_set_align(VirtIODevice *vdev, int n, int align)
2297{
2298 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
2299 VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
2300
Cornelia Huckab223c92015-06-04 12:34:12 +02002301 /* virtio-1 compliant devices cannot change the alignment */
Cornelia Huck95129d62015-08-17 11:48:29 +02002302 if (virtio_vdev_has_feature(vdev, VIRTIO_F_VERSION_1)) {
Cornelia Huckab223c92015-06-04 12:34:12 +02002303 error_report("tried to modify queue alignment for virtio-1 device");
2304 return;
2305 }
Peter Maydell6ce69d12013-07-16 13:25:08 +01002306 /* Check that the transport told us it was going to do this
2307 * (so a buggy transport will immediately assert rather than
2308 * silently failing to migrate this state)
2309 */
2310 assert(k->has_variable_vring_alignment);
2311
Prasad J Pandit758ead32017-11-29 23:14:27 +05302312 if (align) {
2313 vdev->vq[n].vring.align = align;
2314 virtio_queue_update_rings(vdev, n);
2315 }
Peter Maydell6ce69d12013-07-16 13:25:08 +01002316}
2317
Fam Zheng07931692017-02-09 16:40:47 +08002318static bool virtio_queue_notify_aio_vq(VirtQueue *vq)
Michael S. Tsirkin344dc162016-04-06 12:16:25 +02002319{
Xie Yongjibadaf792019-03-20 19:26:40 +08002320 bool ret = false;
2321
Michael S. Tsirkin344dc162016-04-06 12:16:25 +02002322 if (vq->vring.desc && vq->handle_aio_output) {
2323 VirtIODevice *vdev = vq->vdev;
2324
2325 trace_virtio_queue_notify(vdev, vq - vdev->vq, vq);
Xie Yongjibadaf792019-03-20 19:26:40 +08002326 ret = vq->handle_aio_output(vdev, vq);
2327
2328 if (unlikely(vdev->start_on_kick)) {
Xie Yongjie57f2c32019-06-26 10:31:26 +08002329 virtio_set_started(vdev, true);
Xie Yongjibadaf792019-03-20 19:26:40 +08002330 }
Michael S. Tsirkin344dc162016-04-06 12:16:25 +02002331 }
Fam Zheng07931692017-02-09 16:40:47 +08002332
Xie Yongjibadaf792019-03-20 19:26:40 +08002333 return ret;
Michael S. Tsirkin344dc162016-04-06 12:16:25 +02002334}
2335
Paolo Bonzini2b2cbca2016-04-06 12:16:22 +02002336static void virtio_queue_notify_vq(VirtQueue *vq)
Stefan Hajnoczi25db9eb2010-12-17 12:01:50 +00002337{
Jason Wang9e0f5b82015-03-12 17:50:18 +08002338 if (vq->vring.desc && vq->handle_output) {
Stefan Hajnoczi25db9eb2010-12-17 12:01:50 +00002339 VirtIODevice *vdev = vq->vdev;
Jason Wang9e0f5b82015-03-12 17:50:18 +08002340
Stefan Hajnoczif5ed3662016-09-21 16:52:19 +01002341 if (unlikely(vdev->broken)) {
2342 return;
2343 }
2344
Stefan Hajnoczi25db9eb2010-12-17 12:01:50 +00002345 trace_virtio_queue_notify(vdev, vq - vdev->vq, vq);
2346 vq->handle_output(vdev, vq);
Xie Yongjibadaf792019-03-20 19:26:40 +08002347
2348 if (unlikely(vdev->start_on_kick)) {
Xie Yongjie57f2c32019-06-26 10:31:26 +08002349 virtio_set_started(vdev, true);
Xie Yongjibadaf792019-03-20 19:26:40 +08002350 }
Stefan Hajnoczi25db9eb2010-12-17 12:01:50 +00002351 }
2352}
2353
Paul Brook53c25ce2009-05-18 14:51:59 +01002354void virtio_queue_notify(VirtIODevice *vdev, int n)
2355{
Paolo Bonzinie49a6612017-02-28 14:21:32 +01002356 VirtQueue *vq = &vdev->vq[n];
2357
2358 if (unlikely(!vq->vring.desc || vdev->broken)) {
2359 return;
2360 }
2361
2362 trace_virtio_queue_notify(vdev, vq - vdev->vq, vq);
Stefan Hajnoczifcccb272019-11-05 15:09:46 +01002363 if (vq->host_notifier_enabled) {
Paolo Bonzinie49a6612017-02-28 14:21:32 +01002364 event_notifier_set(&vq->host_notifier);
2365 } else if (vq->handle_output) {
2366 vq->handle_output(vdev, vq);
Xie Yongjibadaf792019-03-20 19:26:40 +08002367
Xie Yongji8b04e2c2019-06-26 10:31:29 +08002368 if (unlikely(vdev->start_on_kick)) {
2369 virtio_set_started(vdev, true);
2370 }
Xie Yongjibadaf792019-03-20 19:26:40 +08002371 }
aliguori967f97f2008-12-04 19:38:57 +00002372}
2373
Michael S. Tsirkin7055e682009-06-21 19:50:13 +03002374uint16_t virtio_queue_vector(VirtIODevice *vdev, int n)
2375{
Jason Wang87b3bd12015-05-29 14:15:31 +08002376 return n < VIRTIO_QUEUE_MAX ? vdev->vq[n].vector :
Michael S. Tsirkin7055e682009-06-21 19:50:13 +03002377 VIRTIO_NO_VECTOR;
2378}
2379
2380void virtio_queue_set_vector(VirtIODevice *vdev, int n, uint16_t vector)
2381{
Jason Wange0d686b2015-04-23 14:21:46 +08002382 VirtQueue *vq = &vdev->vq[n];
2383
Jason Wang87b3bd12015-05-29 14:15:31 +08002384 if (n < VIRTIO_QUEUE_MAX) {
Jason Wange0d686b2015-04-23 14:21:46 +08002385 if (vdev->vector_queues &&
2386 vdev->vq[n].vector != VIRTIO_NO_VECTOR) {
2387 QLIST_REMOVE(vq, node);
2388 }
Michael S. Tsirkin7055e682009-06-21 19:50:13 +03002389 vdev->vq[n].vector = vector;
Jason Wange0d686b2015-04-23 14:21:46 +08002390 if (vdev->vector_queues &&
2391 vector != VIRTIO_NO_VECTOR) {
2392 QLIST_INSERT_HEAD(&vdev->vector_queues[vector], vq, node);
2393 }
2394 }
Michael S. Tsirkin7055e682009-06-21 19:50:13 +03002395}
2396
Paolo Bonzinif1ac6a52016-10-21 22:48:11 +02002397VirtQueue *virtio_add_queue(VirtIODevice *vdev, int queue_size,
2398 VirtIOHandleOutput handle_output)
aliguori967f97f2008-12-04 19:38:57 +00002399{
2400 int i;
2401
Jason Wang87b3bd12015-05-29 14:15:31 +08002402 for (i = 0; i < VIRTIO_QUEUE_MAX; i++) {
aliguori967f97f2008-12-04 19:38:57 +00002403 if (vdev->vq[i].vring.num == 0)
2404 break;
2405 }
2406
Jason Wang87b3bd12015-05-29 14:15:31 +08002407 if (i == VIRTIO_QUEUE_MAX || queue_size > VIRTQUEUE_MAX_SIZE)
aliguori967f97f2008-12-04 19:38:57 +00002408 abort();
2409
2410 vdev->vq[i].vring.num = queue_size;
Cornelia Huck46c5d082015-09-11 15:16:41 +02002411 vdev->vq[i].vring.num_default = queue_size;
Peter Maydell6ce69d12013-07-16 13:25:08 +01002412 vdev->vq[i].vring.align = VIRTIO_PCI_VRING_ALIGN;
aliguori967f97f2008-12-04 19:38:57 +00002413 vdev->vq[i].handle_output = handle_output;
Michael S. Tsirkin344dc162016-04-06 12:16:25 +02002414 vdev->vq[i].handle_aio_output = NULL;
Jason Wang86044b22019-10-25 10:35:24 +02002415 vdev->vq[i].used_elems = g_malloc0(sizeof(VirtQueueElement) *
2416 queue_size);
aliguori967f97f2008-12-04 19:38:57 +00002417
2418 return &vdev->vq[i];
2419}
2420
Michael S. Tsirkin722f8c52019-12-09 11:46:13 -05002421void virtio_delete_queue(VirtQueue *vq)
2422{
2423 vq->vring.num = 0;
2424 vq->vring.num_default = 0;
2425 vq->handle_output = NULL;
2426 vq->handle_aio_output = NULL;
2427 g_free(vq->used_elems);
Michael S. Tsirkin8cd353e2019-12-09 11:47:24 -05002428 vq->used_elems = NULL;
Yuri Benditovich421afd22019-12-26 06:36:48 +02002429 virtio_virtqueue_reset_region_cache(vq);
Michael S. Tsirkin722f8c52019-12-09 11:46:13 -05002430}
2431
Jason Wangf23fd812013-01-30 19:12:36 +08002432void virtio_del_queue(VirtIODevice *vdev, int n)
2433{
Jason Wang87b3bd12015-05-29 14:15:31 +08002434 if (n < 0 || n >= VIRTIO_QUEUE_MAX) {
Jason Wangf23fd812013-01-30 19:12:36 +08002435 abort();
2436 }
2437
Michael S. Tsirkin722f8c52019-12-09 11:46:13 -05002438 virtio_delete_queue(&vdev->vq[n]);
Jason Wangf23fd812013-01-30 19:12:36 +08002439}
2440
Paolo Bonzini0687c372016-11-18 16:07:01 +01002441static void virtio_set_isr(VirtIODevice *vdev, int value)
2442{
Stefan Hajnoczid73415a2020-09-23 11:56:46 +01002443 uint8_t old = qatomic_read(&vdev->isr);
Paolo Bonzini0687c372016-11-18 16:07:01 +01002444
2445 /* Do not write ISR if it does not change, so that its cacheline remains
2446 * shared in the common case where the guest does not read it.
2447 */
2448 if ((old & value) != value) {
Stefan Hajnoczid73415a2020-09-23 11:56:46 +01002449 qatomic_or(&vdev->isr, value);
Paolo Bonzini0687c372016-11-18 16:07:01 +01002450 }
2451}
2452
Jason Wang683f7662019-10-25 10:35:25 +02002453static bool virtio_split_should_notify(VirtIODevice *vdev, VirtQueue *vq)
Michael S. Tsirkinbcbabae2011-06-12 16:21:57 +03002454{
2455 uint16_t old, new;
2456 bool v;
Michael S. Tsirkina281ebc2012-04-22 16:45:53 +03002457 /* We need to expose used array entries before checking used event. */
2458 smp_mb();
Michael S. Tsirkinbcbabae2011-06-12 16:21:57 +03002459 /* Always notify when queue is empty (when feature acknowledge) */
Cornelia Huck95129d62015-08-17 11:48:29 +02002460 if (virtio_vdev_has_feature(vdev, VIRTIO_F_NOTIFY_ON_EMPTY) &&
Vincenzo Maffionebe1fea92016-01-31 11:29:05 +01002461 !vq->inuse && virtio_queue_empty(vq)) {
Michael S. Tsirkinbcbabae2011-06-12 16:21:57 +03002462 return true;
2463 }
2464
Cornelia Huck95129d62015-08-17 11:48:29 +02002465 if (!virtio_vdev_has_feature(vdev, VIRTIO_RING_F_EVENT_IDX)) {
Michael S. Tsirkinbcbabae2011-06-12 16:21:57 +03002466 return !(vring_avail_flags(vq) & VRING_AVAIL_F_NO_INTERRUPT);
2467 }
2468
2469 v = vq->signalled_used_valid;
2470 vq->signalled_used_valid = true;
2471 old = vq->signalled_used;
Vincenzo Maffioneb796fcd2016-01-31 11:29:04 +01002472 new = vq->signalled_used = vq->used_idx;
Michael S. Tsirkine9600c62015-02-16 22:35:46 +01002473 return !v || vring_need_event(vring_get_used_event(vq), new, old);
Michael S. Tsirkinbcbabae2011-06-12 16:21:57 +03002474}
2475
Jason Wang683f7662019-10-25 10:35:25 +02002476static bool vring_packed_need_event(VirtQueue *vq, bool wrap,
2477 uint16_t off_wrap, uint16_t new,
2478 uint16_t old)
2479{
2480 int off = off_wrap & ~(1 << 15);
2481
2482 if (wrap != off_wrap >> 15) {
2483 off -= vq->vring.num;
2484 }
2485
2486 return vring_need_event(off, new, old);
2487}
2488
2489static bool virtio_packed_should_notify(VirtIODevice *vdev, VirtQueue *vq)
2490{
2491 VRingPackedDescEvent e;
2492 uint16_t old, new;
2493 bool v;
2494 VRingMemoryRegionCaches *caches;
2495
2496 caches = vring_get_region_caches(vq);
Stefan Hajnocziabdd16f2020-02-07 10:46:19 +00002497 if (!caches) {
2498 return false;
2499 }
2500
Jason Wang683f7662019-10-25 10:35:25 +02002501 vring_packed_event_read(vdev, &caches->avail, &e);
2502
2503 old = vq->signalled_used;
2504 new = vq->signalled_used = vq->used_idx;
2505 v = vq->signalled_used_valid;
2506 vq->signalled_used_valid = true;
2507
2508 if (e.flags == VRING_PACKED_EVENT_FLAG_DISABLE) {
2509 return false;
2510 } else if (e.flags == VRING_PACKED_EVENT_FLAG_ENABLE) {
2511 return true;
2512 }
2513
2514 return !v || vring_packed_need_event(vq, vq->used_wrap_counter,
2515 e.off_wrap, new, old);
2516}
2517
2518/* Called within rcu_read_lock(). */
2519static bool virtio_should_notify(VirtIODevice *vdev, VirtQueue *vq)
2520{
2521 if (virtio_vdev_has_feature(vdev, VIRTIO_F_RING_PACKED)) {
2522 return virtio_packed_should_notify(vdev, vq);
2523 } else {
2524 return virtio_split_should_notify(vdev, vq);
2525 }
2526}
2527
Paolo Bonzini83d768b2016-11-18 16:07:02 +01002528void virtio_notify_irqfd(VirtIODevice *vdev, VirtQueue *vq)
2529{
Dr. David Alan Gilbertb5f53d02019-10-28 16:11:09 +00002530 WITH_RCU_READ_LOCK_GUARD() {
2531 if (!virtio_should_notify(vdev, vq)) {
2532 return;
2533 }
Paolo Bonzini83d768b2016-11-18 16:07:02 +01002534 }
2535
2536 trace_virtio_notify_irqfd(vdev, vq);
2537
2538 /*
2539 * virtio spec 1.0 says ISR bit 0 should be ignored with MSI, but
2540 * windows drivers included in virtio-win 1.8.0 (circa 2015) are
2541 * incorrectly polling this bit during crashdump and hibernation
2542 * in MSI mode, causing a hang if this bit is never updated.
2543 * Recent releases of Windows do not really shut down, but rather
2544 * log out and hibernate to make the next startup faster. Hence,
2545 * this manifested as a more serious hang during shutdown with
2546 *
2547 * Next driver release from 2016 fixed this problem, so working around it
2548 * is not a must, but it's easy to do so let's do it here.
2549 *
2550 * Note: it's safe to update ISR from any thread as it was switched
2551 * to an atomic operation.
2552 */
2553 virtio_set_isr(vq->vdev, 0x1);
2554 event_notifier_set(&vq->guest_notifier);
2555}
2556
Michael S. Tsirkinb4b98622017-02-17 04:52:16 +02002557static void virtio_irq(VirtQueue *vq)
2558{
2559 virtio_set_isr(vq->vdev, 0x1);
2560 virtio_notify_vector(vq->vdev, vq->vector);
2561}
2562
aliguori967f97f2008-12-04 19:38:57 +00002563void virtio_notify(VirtIODevice *vdev, VirtQueue *vq)
2564{
Dr. David Alan Gilbertb5f53d02019-10-28 16:11:09 +00002565 WITH_RCU_READ_LOCK_GUARD() {
2566 if (!virtio_should_notify(vdev, vq)) {
2567 return;
2568 }
Michael S. Tsirkinbcbabae2011-06-12 16:21:57 +03002569 }
aliguori967f97f2008-12-04 19:38:57 +00002570
Stefan Hajnoczi64979a42010-05-24 13:19:21 +01002571 trace_virtio_notify(vdev, vq);
Michael S. Tsirkinb4b98622017-02-17 04:52:16 +02002572 virtio_irq(vq);
aliguori967f97f2008-12-04 19:38:57 +00002573}
2574
2575void virtio_notify_config(VirtIODevice *vdev)
2576{
aliguori76251622009-01-29 17:02:13 +00002577 if (!(vdev->status & VIRTIO_CONFIG_S_DRIVER_OK))
2578 return;
2579
Paolo Bonzini0687c372016-11-18 16:07:01 +01002580 virtio_set_isr(vdev, 0x3);
Michael S. Tsirkinb8f05902015-06-04 12:34:23 +02002581 vdev->generation++;
Michael S. Tsirkin7055e682009-06-21 19:50:13 +03002582 virtio_notify_vector(vdev, vdev->config_vector);
aliguori967f97f2008-12-04 19:38:57 +00002583}
2584
Greg Kurz616a6552014-06-24 19:38:54 +02002585static bool virtio_device_endian_needed(void *opaque)
2586{
2587 VirtIODevice *vdev = opaque;
2588
2589 assert(vdev->device_endian != VIRTIO_DEVICE_ENDIAN_UNKNOWN);
Cornelia Huck95129d62015-08-17 11:48:29 +02002590 if (!virtio_vdev_has_feature(vdev, VIRTIO_F_VERSION_1)) {
Cornelia Huck3c185592015-06-04 12:34:11 +02002591 return vdev->device_endian != virtio_default_endian();
2592 }
2593 /* Devices conforming to VIRTIO 1.0 or later are always LE. */
2594 return vdev->device_endian != VIRTIO_DEVICE_ENDIAN_LITTLE;
Greg Kurz616a6552014-06-24 19:38:54 +02002595}
2596
Gerd Hoffmann019a3ed2015-06-01 10:45:40 +02002597static bool virtio_64bit_features_needed(void *opaque)
2598{
2599 VirtIODevice *vdev = opaque;
2600
2601 return (vdev->host_features >> 32) != 0;
2602}
2603
Jason Wang74aae7b2015-08-05 17:50:07 +08002604static bool virtio_virtqueue_needed(void *opaque)
2605{
2606 VirtIODevice *vdev = opaque;
2607
2608 return virtio_host_has_feature(vdev, VIRTIO_F_VERSION_1);
2609}
2610
Jason Wang86044b22019-10-25 10:35:24 +02002611static bool virtio_packed_virtqueue_needed(void *opaque)
2612{
2613 VirtIODevice *vdev = opaque;
2614
2615 return virtio_host_has_feature(vdev, VIRTIO_F_RING_PACKED);
2616}
2617
Cornelia Huck46c5d082015-09-11 15:16:41 +02002618static bool virtio_ringsize_needed(void *opaque)
2619{
2620 VirtIODevice *vdev = opaque;
2621 int i;
2622
2623 for (i = 0; i < VIRTIO_QUEUE_MAX; i++) {
2624 if (vdev->vq[i].vring.num != vdev->vq[i].vring.num_default) {
2625 return true;
2626 }
2627 }
2628 return false;
2629}
2630
Jason Wanga6df8ad2015-11-06 16:02:44 +08002631static bool virtio_extra_state_needed(void *opaque)
2632{
2633 VirtIODevice *vdev = opaque;
2634 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
2635 VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
2636
2637 return k->has_extra_state &&
2638 k->has_extra_state(qbus->parent);
2639}
2640
Stefan Hajnoczi791b1da2016-09-21 16:52:20 +01002641static bool virtio_broken_needed(void *opaque)
2642{
2643 VirtIODevice *vdev = opaque;
2644
2645 return vdev->broken;
2646}
2647
Xie Yongjibadaf792019-03-20 19:26:40 +08002648static bool virtio_started_needed(void *opaque)
2649{
2650 VirtIODevice *vdev = opaque;
2651
2652 return vdev->started;
2653}
2654
Michael Roth9d7bd082019-11-19 18:50:03 -06002655static bool virtio_disabled_needed(void *opaque)
2656{
2657 VirtIODevice *vdev = opaque;
2658
2659 return vdev->disabled;
2660}
2661
Dr. David Alan Gilbert50e5ae42016-01-06 12:23:39 +00002662static const VMStateDescription vmstate_virtqueue = {
Jason Wang74aae7b2015-08-05 17:50:07 +08002663 .name = "virtqueue_state",
Dr. David Alan Gilbert50e5ae42016-01-06 12:23:39 +00002664 .version_id = 1,
2665 .minimum_version_id = 1,
2666 .fields = (VMStateField[]) {
2667 VMSTATE_UINT64(vring.avail, struct VirtQueue),
2668 VMSTATE_UINT64(vring.used, struct VirtQueue),
2669 VMSTATE_END_OF_LIST()
2670 }
Jason Wang74aae7b2015-08-05 17:50:07 +08002671};
2672
Jason Wang86044b22019-10-25 10:35:24 +02002673static const VMStateDescription vmstate_packed_virtqueue = {
2674 .name = "packed_virtqueue_state",
2675 .version_id = 1,
2676 .minimum_version_id = 1,
2677 .fields = (VMStateField[]) {
2678 VMSTATE_UINT16(last_avail_idx, struct VirtQueue),
2679 VMSTATE_BOOL(last_avail_wrap_counter, struct VirtQueue),
2680 VMSTATE_UINT16(used_idx, struct VirtQueue),
2681 VMSTATE_BOOL(used_wrap_counter, struct VirtQueue),
2682 VMSTATE_UINT32(inuse, struct VirtQueue),
2683 VMSTATE_END_OF_LIST()
2684 }
2685};
2686
Jason Wang74aae7b2015-08-05 17:50:07 +08002687static const VMStateDescription vmstate_virtio_virtqueues = {
2688 .name = "virtio/virtqueues",
2689 .version_id = 1,
2690 .minimum_version_id = 1,
2691 .needed = &virtio_virtqueue_needed,
2692 .fields = (VMStateField[]) {
Dr. David Alan Gilbert3e996cc2016-01-29 13:18:56 +00002693 VMSTATE_STRUCT_VARRAY_POINTER_KNOWN(vq, struct VirtIODevice,
2694 VIRTIO_QUEUE_MAX, 0, vmstate_virtqueue, VirtQueue),
Jason Wang74aae7b2015-08-05 17:50:07 +08002695 VMSTATE_END_OF_LIST()
2696 }
2697};
2698
Jason Wang86044b22019-10-25 10:35:24 +02002699static const VMStateDescription vmstate_virtio_packed_virtqueues = {
2700 .name = "virtio/packed_virtqueues",
2701 .version_id = 1,
2702 .minimum_version_id = 1,
2703 .needed = &virtio_packed_virtqueue_needed,
2704 .fields = (VMStateField[]) {
2705 VMSTATE_STRUCT_VARRAY_POINTER_KNOWN(vq, struct VirtIODevice,
2706 VIRTIO_QUEUE_MAX, 0, vmstate_packed_virtqueue, VirtQueue),
2707 VMSTATE_END_OF_LIST()
2708 }
2709};
2710
Dr. David Alan Gilbert50e5ae42016-01-06 12:23:39 +00002711static const VMStateDescription vmstate_ringsize = {
Cornelia Huck46c5d082015-09-11 15:16:41 +02002712 .name = "ringsize_state",
Dr. David Alan Gilbert50e5ae42016-01-06 12:23:39 +00002713 .version_id = 1,
2714 .minimum_version_id = 1,
2715 .fields = (VMStateField[]) {
2716 VMSTATE_UINT32(vring.num_default, struct VirtQueue),
2717 VMSTATE_END_OF_LIST()
2718 }
Cornelia Huck46c5d082015-09-11 15:16:41 +02002719};
2720
2721static const VMStateDescription vmstate_virtio_ringsize = {
2722 .name = "virtio/ringsize",
2723 .version_id = 1,
2724 .minimum_version_id = 1,
2725 .needed = &virtio_ringsize_needed,
2726 .fields = (VMStateField[]) {
Dr. David Alan Gilbert3e996cc2016-01-29 13:18:56 +00002727 VMSTATE_STRUCT_VARRAY_POINTER_KNOWN(vq, struct VirtIODevice,
2728 VIRTIO_QUEUE_MAX, 0, vmstate_ringsize, VirtQueue),
Cornelia Huck46c5d082015-09-11 15:16:41 +02002729 VMSTATE_END_OF_LIST()
2730 }
2731};
2732
Jianjun Duan2c21ee72017-01-19 11:00:50 -08002733static int get_extra_state(QEMUFile *f, void *pv, size_t size,
Marc-André Lureau03fee662018-11-14 17:29:30 +04002734 const VMStateField *field)
Jason Wanga6df8ad2015-11-06 16:02:44 +08002735{
2736 VirtIODevice *vdev = pv;
2737 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
2738 VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
2739
2740 if (!k->load_extra_state) {
2741 return -1;
2742 } else {
2743 return k->load_extra_state(qbus->parent, f);
2744 }
2745}
2746
Jianjun Duan2c21ee72017-01-19 11:00:50 -08002747static int put_extra_state(QEMUFile *f, void *pv, size_t size,
Markus Armbruster3ddba9a2020-12-11 18:11:48 +01002748 const VMStateField *field, JSONWriter *vmdesc)
Jason Wanga6df8ad2015-11-06 16:02:44 +08002749{
2750 VirtIODevice *vdev = pv;
2751 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
2752 VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
2753
2754 k->save_extra_state(qbus->parent, f);
Jianjun Duan2c21ee72017-01-19 11:00:50 -08002755 return 0;
Jason Wanga6df8ad2015-11-06 16:02:44 +08002756}
2757
2758static const VMStateInfo vmstate_info_extra_state = {
2759 .name = "virtqueue_extra_state",
2760 .get = get_extra_state,
2761 .put = put_extra_state,
2762};
2763
2764static const VMStateDescription vmstate_virtio_extra_state = {
2765 .name = "virtio/extra_state",
2766 .version_id = 1,
2767 .minimum_version_id = 1,
2768 .needed = &virtio_extra_state_needed,
2769 .fields = (VMStateField[]) {
2770 {
2771 .name = "extra_state",
2772 .version_id = 0,
2773 .field_exists = NULL,
2774 .size = 0,
2775 .info = &vmstate_info_extra_state,
2776 .flags = VMS_SINGLE,
2777 .offset = 0,
2778 },
2779 VMSTATE_END_OF_LIST()
2780 }
2781};
2782
Greg Kurz616a6552014-06-24 19:38:54 +02002783static const VMStateDescription vmstate_virtio_device_endian = {
2784 .name = "virtio/device_endian",
2785 .version_id = 1,
2786 .minimum_version_id = 1,
Juan Quintela5cd8cad2014-09-23 14:09:54 +02002787 .needed = &virtio_device_endian_needed,
Greg Kurz616a6552014-06-24 19:38:54 +02002788 .fields = (VMStateField[]) {
2789 VMSTATE_UINT8(device_endian, VirtIODevice),
2790 VMSTATE_END_OF_LIST()
2791 }
2792};
2793
Gerd Hoffmann019a3ed2015-06-01 10:45:40 +02002794static const VMStateDescription vmstate_virtio_64bit_features = {
2795 .name = "virtio/64bit_features",
2796 .version_id = 1,
2797 .minimum_version_id = 1,
Juan Quintela5cd8cad2014-09-23 14:09:54 +02002798 .needed = &virtio_64bit_features_needed,
Gerd Hoffmann019a3ed2015-06-01 10:45:40 +02002799 .fields = (VMStateField[]) {
2800 VMSTATE_UINT64(guest_features, VirtIODevice),
2801 VMSTATE_END_OF_LIST()
2802 }
2803};
2804
Stefan Hajnoczi791b1da2016-09-21 16:52:20 +01002805static const VMStateDescription vmstate_virtio_broken = {
2806 .name = "virtio/broken",
2807 .version_id = 1,
2808 .minimum_version_id = 1,
2809 .needed = &virtio_broken_needed,
2810 .fields = (VMStateField[]) {
2811 VMSTATE_BOOL(broken, VirtIODevice),
2812 VMSTATE_END_OF_LIST()
2813 }
2814};
2815
Xie Yongjibadaf792019-03-20 19:26:40 +08002816static const VMStateDescription vmstate_virtio_started = {
2817 .name = "virtio/started",
2818 .version_id = 1,
2819 .minimum_version_id = 1,
2820 .needed = &virtio_started_needed,
2821 .fields = (VMStateField[]) {
2822 VMSTATE_BOOL(started, VirtIODevice),
2823 VMSTATE_END_OF_LIST()
2824 }
2825};
2826
Michael Roth9d7bd082019-11-19 18:50:03 -06002827static const VMStateDescription vmstate_virtio_disabled = {
2828 .name = "virtio/disabled",
2829 .version_id = 1,
2830 .minimum_version_id = 1,
2831 .needed = &virtio_disabled_needed,
2832 .fields = (VMStateField[]) {
2833 VMSTATE_BOOL(disabled, VirtIODevice),
2834 VMSTATE_END_OF_LIST()
2835 }
2836};
2837
Greg Kurz6b321a32014-06-24 19:22:30 +02002838static const VMStateDescription vmstate_virtio = {
2839 .name = "virtio",
2840 .version_id = 1,
2841 .minimum_version_id = 1,
2842 .minimum_version_id_old = 1,
2843 .fields = (VMStateField[]) {
2844 VMSTATE_END_OF_LIST()
Greg Kurz616a6552014-06-24 19:38:54 +02002845 },
Juan Quintela5cd8cad2014-09-23 14:09:54 +02002846 .subsections = (const VMStateDescription*[]) {
2847 &vmstate_virtio_device_endian,
2848 &vmstate_virtio_64bit_features,
Jason Wang74aae7b2015-08-05 17:50:07 +08002849 &vmstate_virtio_virtqueues,
Cornelia Huck46c5d082015-09-11 15:16:41 +02002850 &vmstate_virtio_ringsize,
Stefan Hajnoczi791b1da2016-09-21 16:52:20 +01002851 &vmstate_virtio_broken,
Jason Wanga6df8ad2015-11-06 16:02:44 +08002852 &vmstate_virtio_extra_state,
Xie Yongjibadaf792019-03-20 19:26:40 +08002853 &vmstate_virtio_started,
Jason Wang86044b22019-10-25 10:35:24 +02002854 &vmstate_virtio_packed_virtqueues,
Michael Roth9d7bd082019-11-19 18:50:03 -06002855 &vmstate_virtio_disabled,
Juan Quintela5cd8cad2014-09-23 14:09:54 +02002856 NULL
Greg Kurz6b321a32014-06-24 19:22:30 +02002857 }
2858};
2859
Dr. David Alan Gilbert2f168d02017-09-25 12:29:17 +01002860int virtio_save(VirtIODevice *vdev, QEMUFile *f)
aliguori967f97f2008-12-04 19:38:57 +00002861{
KONRAD Frederic1c819442013-04-24 10:21:21 +02002862 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
2863 VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
Greg Kurz1b5fc0d2014-06-24 19:15:31 +02002864 VirtioDeviceClass *vdc = VIRTIO_DEVICE_GET_CLASS(vdev);
Gerd Hoffmann019a3ed2015-06-01 10:45:40 +02002865 uint32_t guest_features_lo = (vdev->guest_features & 0xffffffff);
aliguori967f97f2008-12-04 19:38:57 +00002866 int i;
2867
KONRAD Frederic1c819442013-04-24 10:21:21 +02002868 if (k->save_config) {
2869 k->save_config(qbus->parent, f);
2870 }
aliguori967f97f2008-12-04 19:38:57 +00002871
aliguori967f97f2008-12-04 19:38:57 +00002872 qemu_put_8s(f, &vdev->status);
2873 qemu_put_8s(f, &vdev->isr);
2874 qemu_put_be16s(f, &vdev->queue_sel);
Gerd Hoffmann019a3ed2015-06-01 10:45:40 +02002875 qemu_put_be32s(f, &guest_features_lo);
aliguori967f97f2008-12-04 19:38:57 +00002876 qemu_put_be32(f, vdev->config_len);
2877 qemu_put_buffer(f, vdev->config, vdev->config_len);
2878
Jason Wang87b3bd12015-05-29 14:15:31 +08002879 for (i = 0; i < VIRTIO_QUEUE_MAX; i++) {
aliguori967f97f2008-12-04 19:38:57 +00002880 if (vdev->vq[i].vring.num == 0)
2881 break;
2882 }
2883
2884 qemu_put_be32(f, i);
2885
Jason Wang87b3bd12015-05-29 14:15:31 +08002886 for (i = 0; i < VIRTIO_QUEUE_MAX; i++) {
aliguori967f97f2008-12-04 19:38:57 +00002887 if (vdev->vq[i].vring.num == 0)
2888 break;
2889
2890 qemu_put_be32(f, vdev->vq[i].vring.num);
Peter Maydell6ce69d12013-07-16 13:25:08 +01002891 if (k->has_variable_vring_alignment) {
2892 qemu_put_be32(f, vdev->vq[i].vring.align);
2893 }
Stefan Hajnoczi874adf42017-02-22 16:37:34 +00002894 /*
2895 * Save desc now, the rest of the ring addresses are saved in
2896 * subsections for VIRTIO-1 devices.
2897 */
Cornelia Huckab223c92015-06-04 12:34:12 +02002898 qemu_put_be64(f, vdev->vq[i].vring.desc);
aliguori967f97f2008-12-04 19:38:57 +00002899 qemu_put_be16s(f, &vdev->vq[i].last_avail_idx);
KONRAD Frederic1c819442013-04-24 10:21:21 +02002900 if (k->save_queue) {
2901 k->save_queue(qbus->parent, i, f);
2902 }
aliguori967f97f2008-12-04 19:38:57 +00002903 }
Greg Kurz1b5fc0d2014-06-24 19:15:31 +02002904
2905 if (vdc->save != NULL) {
2906 vdc->save(vdev, f);
2907 }
Greg Kurz6b321a32014-06-24 19:22:30 +02002908
Dr. David Alan Gilbertea43e252016-10-27 18:36:36 +01002909 if (vdc->vmsd) {
Dr. David Alan Gilbert2f168d02017-09-25 12:29:17 +01002910 int ret = vmstate_save_state(f, vdc->vmsd, vdev, NULL);
2911 if (ret) {
2912 return ret;
2913 }
Dr. David Alan Gilbertea43e252016-10-27 18:36:36 +01002914 }
2915
Greg Kurz6b321a32014-06-24 19:22:30 +02002916 /* Subsections */
Dr. David Alan Gilbert2f168d02017-09-25 12:29:17 +01002917 return vmstate_save_state(f, &vmstate_virtio, vdev, NULL);
aliguori967f97f2008-12-04 19:38:57 +00002918}
2919
Dr. David Alan Gilbert59431242016-07-14 18:22:45 +01002920/* A wrapper for use as a VMState .put function */
Jianjun Duan2c21ee72017-01-19 11:00:50 -08002921static int virtio_device_put(QEMUFile *f, void *opaque, size_t size,
Markus Armbruster3ddba9a2020-12-11 18:11:48 +01002922 const VMStateField *field, JSONWriter *vmdesc)
Halil Pasic1a665852016-10-06 14:55:39 +02002923{
Dr. David Alan Gilbert2f168d02017-09-25 12:29:17 +01002924 return virtio_save(VIRTIO_DEVICE(opaque), f);
Halil Pasic1a665852016-10-06 14:55:39 +02002925}
2926
2927/* A wrapper for use as a VMState .get function */
Jianjun Duan2c21ee72017-01-19 11:00:50 -08002928static int virtio_device_get(QEMUFile *f, void *opaque, size_t size,
Marc-André Lureau03fee662018-11-14 17:29:30 +04002929 const VMStateField *field)
Halil Pasic1a665852016-10-06 14:55:39 +02002930{
2931 VirtIODevice *vdev = VIRTIO_DEVICE(opaque);
2932 DeviceClass *dc = DEVICE_CLASS(VIRTIO_DEVICE_GET_CLASS(vdev));
2933
2934 return virtio_load(vdev, f, dc->vmsd->version_id);
2935}
2936
2937const VMStateInfo virtio_vmstate_info = {
2938 .name = "virtio",
2939 .get = virtio_device_get,
2940 .put = virtio_device_put,
2941};
2942
Cornelia Huck6c0196d2015-06-04 12:34:14 +02002943static int virtio_set_features_nocheck(VirtIODevice *vdev, uint64_t val)
Paolo Bonziniad0c9332011-11-24 13:28:52 +01002944{
KONRAD Frederic181103c2013-04-24 10:21:20 +02002945 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
Cornelia Huck6b8f1022015-05-26 16:34:47 +02002946 bool bad = (val & ~(vdev->host_features)) != 0;
Paolo Bonziniad0c9332011-11-24 13:28:52 +01002947
Cornelia Huck6b8f1022015-05-26 16:34:47 +02002948 val &= vdev->host_features;
KONRAD Frederic181103c2013-04-24 10:21:20 +02002949 if (k->set_features) {
2950 k->set_features(vdev, val);
Paolo Bonziniad0c9332011-11-24 13:28:52 +01002951 }
2952 vdev->guest_features = val;
2953 return bad ? -1 : 0;
2954}
2955
Cornelia Huck6c0196d2015-06-04 12:34:14 +02002956int virtio_set_features(VirtIODevice *vdev, uint64_t val)
2957{
Paolo Bonzinidb812c42018-08-23 14:21:23 +02002958 int ret;
2959 /*
Cornelia Huck6c0196d2015-06-04 12:34:14 +02002960 * The driver must not attempt to set features after feature negotiation
2961 * has finished.
2962 */
2963 if (vdev->status & VIRTIO_CONFIG_S_FEATURES_OK) {
2964 return -EINVAL;
2965 }
Paolo Bonzinidb812c42018-08-23 14:21:23 +02002966 ret = virtio_set_features_nocheck(vdev, val);
Li Qiang2d69eba2020-09-19 01:27:06 -07002967 if (virtio_vdev_has_feature(vdev, VIRTIO_RING_F_EVENT_IDX)) {
2968 /* VIRTIO_RING_F_EVENT_IDX changes the size of the caches. */
2969 int i;
2970 for (i = 0; i < VIRTIO_QUEUE_MAX; i++) {
2971 if (vdev->vq[i].vring.num != 0) {
2972 virtio_init_region_cache(vdev, i);
Paolo Bonzinidb812c42018-08-23 14:21:23 +02002973 }
2974 }
Li Qiang2d69eba2020-09-19 01:27:06 -07002975 }
2976 if (!ret) {
Xie Yongji868a8f42019-06-26 10:31:28 +08002977 if (!virtio_device_started(vdev, vdev->status) &&
2978 !virtio_vdev_has_feature(vdev, VIRTIO_F_VERSION_1)) {
2979 vdev->start_on_kick = true;
2980 }
Paolo Bonzinidb812c42018-08-23 14:21:23 +02002981 }
2982 return ret;
Cornelia Huck6c0196d2015-06-04 12:34:14 +02002983}
2984
Stefano Garzarellaba550852019-02-21 11:33:08 +01002985size_t virtio_feature_get_config_size(VirtIOFeature *feature_sizes,
2986 uint64_t host_features)
2987{
2988 size_t config_size = 0;
2989 int i;
2990
2991 for (i = 0; feature_sizes[i].flags != 0; i++) {
2992 if (host_features & feature_sizes[i].flags) {
2993 config_size = MAX(feature_sizes[i].end, config_size);
2994 }
2995 }
2996
2997 return config_size;
2998}
2999
Greg Kurz1b5fc0d2014-06-24 19:15:31 +02003000int virtio_load(VirtIODevice *vdev, QEMUFile *f, int version_id)
aliguori967f97f2008-12-04 19:38:57 +00003001{
Michael S. Tsirkincc459952014-04-03 19:51:14 +03003002 int i, ret;
Michael S. Tsirkina890a2f2014-04-28 16:08:23 +03003003 int32_t config_len;
Michael S. Tsirkincc459952014-04-03 19:51:14 +03003004 uint32_t num;
Michael S. Tsirkin6d74ca52009-12-08 20:07:48 +02003005 uint32_t features;
KONRAD Frederic1c819442013-04-24 10:21:21 +02003006 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
3007 VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
Greg Kurz1b5fc0d2014-06-24 19:15:31 +02003008 VirtioDeviceClass *vdc = VIRTIO_DEVICE_GET_CLASS(vdev);
aliguori967f97f2008-12-04 19:38:57 +00003009
Greg Kurz616a6552014-06-24 19:38:54 +02003010 /*
3011 * We poison the endianness to ensure it does not get used before
3012 * subsections have been loaded.
3013 */
3014 vdev->device_endian = VIRTIO_DEVICE_ENDIAN_UNKNOWN;
3015
KONRAD Frederic1c819442013-04-24 10:21:21 +02003016 if (k->load_config) {
3017 ret = k->load_config(qbus->parent, f);
Michael S. Tsirkinff24bd52009-06-21 19:50:40 +03003018 if (ret)
3019 return ret;
3020 }
aliguori967f97f2008-12-04 19:38:57 +00003021
aliguori967f97f2008-12-04 19:38:57 +00003022 qemu_get_8s(f, &vdev->status);
3023 qemu_get_8s(f, &vdev->isr);
3024 qemu_get_be16s(f, &vdev->queue_sel);
Jason Wang87b3bd12015-05-29 14:15:31 +08003025 if (vdev->queue_sel >= VIRTIO_QUEUE_MAX) {
Michael Roth4b53c2c2014-04-03 19:51:46 +03003026 return -1;
3027 }
Michael S. Tsirkin6d74ca52009-12-08 20:07:48 +02003028 qemu_get_be32s(f, &features);
Paolo Bonziniad0c9332011-11-24 13:28:52 +01003029
Michael S. Tsirkin62cee1a2016-07-04 14:39:10 +03003030 /*
3031 * Temporarily set guest_features low bits - needed by
3032 * virtio net load code testing for VIRTIO_NET_F_CTRL_GUEST_OFFLOADS
3033 * VIRTIO_NET_F_GUEST_ANNOUNCE and VIRTIO_NET_F_CTRL_VQ.
3034 *
3035 * Note: devices should always test host features in future - don't create
3036 * new dependencies like this.
3037 */
3038 vdev->guest_features = features;
3039
Michael S. Tsirkina890a2f2014-04-28 16:08:23 +03003040 config_len = qemu_get_be32(f);
Dr. David Alan Gilbert2f5732e2014-06-27 20:02:48 +01003041
3042 /*
3043 * There are cases where the incoming config can be bigger or smaller
3044 * than what we have; so load what we have space for, and skip
3045 * any excess that's in the stream.
3046 */
3047 qemu_get_buffer(f, vdev->config, MIN(config_len, vdev->config_len));
3048
3049 while (config_len > vdev->config_len) {
3050 qemu_get_byte(f);
3051 config_len--;
Michael S. Tsirkina890a2f2014-04-28 16:08:23 +03003052 }
aliguori967f97f2008-12-04 19:38:57 +00003053
3054 num = qemu_get_be32(f);
3055
Jason Wang87b3bd12015-05-29 14:15:31 +08003056 if (num > VIRTIO_QUEUE_MAX) {
Cornelia Huck8a1be662016-01-08 11:00:00 +01003057 error_report("Invalid number of virtqueues: 0x%x", num);
Michael S. Tsirkincc459952014-04-03 19:51:14 +03003058 return -1;
3059 }
3060
aliguori967f97f2008-12-04 19:38:57 +00003061 for (i = 0; i < num; i++) {
3062 vdev->vq[i].vring.num = qemu_get_be32(f);
Peter Maydell6ce69d12013-07-16 13:25:08 +01003063 if (k->has_variable_vring_alignment) {
3064 vdev->vq[i].vring.align = qemu_get_be32(f);
3065 }
Cornelia Huckab223c92015-06-04 12:34:12 +02003066 vdev->vq[i].vring.desc = qemu_get_be64(f);
aliguori967f97f2008-12-04 19:38:57 +00003067 qemu_get_be16s(f, &vdev->vq[i].last_avail_idx);
Michael S. Tsirkinbcbabae2011-06-12 16:21:57 +03003068 vdev->vq[i].signalled_used_valid = false;
Stefan Hajnoczi332fa822017-01-12 11:46:10 +00003069 vdev->vq[i].notification = true;
aliguori967f97f2008-12-04 19:38:57 +00003070
Stefan Hajnoczi874adf42017-02-22 16:37:34 +00003071 if (!vdev->vq[i].vring.desc && vdev->vq[i].last_avail_idx) {
Michael S. Tsirkin1abeb5a2010-11-23 21:55:39 +02003072 error_report("VQ %d address 0x0 "
Markus Armbruster6daf1942011-06-22 14:03:54 +02003073 "inconsistent with Host index 0x%x",
Michael S. Tsirkin1abeb5a2010-11-23 21:55:39 +02003074 i, vdev->vq[i].last_avail_idx);
Stefan Hajnoczi874adf42017-02-22 16:37:34 +00003075 return -1;
Stefan Hajnoczi8275e2f2016-09-21 16:52:18 +01003076 }
KONRAD Frederic1c819442013-04-24 10:21:21 +02003077 if (k->load_queue) {
3078 ret = k->load_queue(qbus->parent, i, f);
Michael S. Tsirkinff24bd52009-06-21 19:50:40 +03003079 if (ret)
3080 return ret;
Michael S. Tsirkin7055e682009-06-21 19:50:13 +03003081 }
aliguori967f97f2008-12-04 19:38:57 +00003082 }
3083
Michael S. Tsirkin7055e682009-06-21 19:50:13 +03003084 virtio_notify_vector(vdev, VIRTIO_NO_VECTOR);
Greg Kurz1b5fc0d2014-06-24 19:15:31 +02003085
3086 if (vdc->load != NULL) {
Greg Kurz6b321a32014-06-24 19:22:30 +02003087 ret = vdc->load(vdev, f, version_id);
3088 if (ret) {
3089 return ret;
3090 }
Greg Kurz1b5fc0d2014-06-24 19:15:31 +02003091 }
3092
Dr. David Alan Gilbertea43e252016-10-27 18:36:36 +01003093 if (vdc->vmsd) {
3094 ret = vmstate_load_state(f, vdc->vmsd, vdev, version_id);
3095 if (ret) {
3096 return ret;
3097 }
3098 }
3099
Greg Kurz616a6552014-06-24 19:38:54 +02003100 /* Subsections */
3101 ret = vmstate_load_state(f, &vmstate_virtio, vdev, 1);
3102 if (ret) {
3103 return ret;
3104 }
3105
3106 if (vdev->device_endian == VIRTIO_DEVICE_ENDIAN_UNKNOWN) {
3107 vdev->device_endian = virtio_default_endian();
3108 }
3109
Gerd Hoffmann019a3ed2015-06-01 10:45:40 +02003110 if (virtio_64bit_features_needed(vdev)) {
3111 /*
3112 * Subsection load filled vdev->guest_features. Run them
3113 * through virtio_set_features to sanity-check them against
3114 * host_features.
3115 */
3116 uint64_t features64 = vdev->guest_features;
Cornelia Huck6c0196d2015-06-04 12:34:14 +02003117 if (virtio_set_features_nocheck(vdev, features64) < 0) {
Gerd Hoffmann019a3ed2015-06-01 10:45:40 +02003118 error_report("Features 0x%" PRIx64 " unsupported. "
3119 "Allowed features: 0x%" PRIx64,
3120 features64, vdev->host_features);
3121 return -1;
3122 }
3123 } else {
Cornelia Huck6c0196d2015-06-04 12:34:14 +02003124 if (virtio_set_features_nocheck(vdev, features) < 0) {
Gerd Hoffmann019a3ed2015-06-01 10:45:40 +02003125 error_report("Features 0x%x unsupported. "
3126 "Allowed features: 0x%" PRIx64,
3127 features, vdev->host_features);
3128 return -1;
3129 }
3130 }
3131
Xie Yongji868a8f42019-06-26 10:31:28 +08003132 if (!virtio_device_started(vdev, vdev->status) &&
3133 !virtio_vdev_has_feature(vdev, VIRTIO_F_VERSION_1)) {
3134 vdev->start_on_kick = true;
3135 }
3136
Dr. David Alan Gilbertb5f53d02019-10-28 16:11:09 +00003137 RCU_READ_LOCK_GUARD();
Greg Kurz616a6552014-06-24 19:38:54 +02003138 for (i = 0; i < num; i++) {
Cornelia Huckab223c92015-06-04 12:34:12 +02003139 if (vdev->vq[i].vring.desc) {
Greg Kurz616a6552014-06-24 19:38:54 +02003140 uint16_t nheads;
Stefan Hajnoczi874adf42017-02-22 16:37:34 +00003141
3142 /*
3143 * VIRTIO-1 devices migrate desc, used, and avail ring addresses so
3144 * only the region cache needs to be set up. Legacy devices need
3145 * to calculate used and avail ring addresses based on the desc
3146 * address.
3147 */
3148 if (virtio_vdev_has_feature(vdev, VIRTIO_F_VERSION_1)) {
3149 virtio_init_region_cache(vdev, i);
3150 } else {
3151 virtio_queue_update_rings(vdev, i);
3152 }
3153
Jason Wang86044b22019-10-25 10:35:24 +02003154 if (virtio_vdev_has_feature(vdev, VIRTIO_F_RING_PACKED)) {
3155 vdev->vq[i].shadow_avail_idx = vdev->vq[i].last_avail_idx;
3156 vdev->vq[i].shadow_avail_wrap_counter =
3157 vdev->vq[i].last_avail_wrap_counter;
3158 continue;
3159 }
3160
Greg Kurz616a6552014-06-24 19:38:54 +02003161 nheads = vring_avail_idx(&vdev->vq[i]) - vdev->vq[i].last_avail_idx;
3162 /* Check it isn't doing strange things with descriptor numbers. */
3163 if (nheads > vdev->vq[i].vring.num) {
John Levon4aedda22020-11-20 18:51:07 +00003164 virtio_error(vdev, "VQ %d size 0x%x Guest index 0x%x "
3165 "inconsistent with Host index 0x%x: delta 0x%x",
3166 i, vdev->vq[i].vring.num,
3167 vring_avail_idx(&vdev->vq[i]),
3168 vdev->vq[i].last_avail_idx, nheads);
3169 vdev->vq[i].used_idx = 0;
3170 vdev->vq[i].shadow_avail_idx = 0;
3171 vdev->vq[i].inuse = 0;
3172 continue;
Greg Kurz616a6552014-06-24 19:38:54 +02003173 }
Vincenzo Maffioneb796fcd2016-01-31 11:29:04 +01003174 vdev->vq[i].used_idx = vring_used_idx(&vdev->vq[i]);
Vincenzo Maffionebe1fea92016-01-31 11:29:05 +01003175 vdev->vq[i].shadow_avail_idx = vring_avail_idx(&vdev->vq[i]);
Stefan Hajnoczibccdef62016-08-15 13:54:15 +01003176
3177 /*
3178 * Some devices migrate VirtQueueElements that have been popped
3179 * from the avail ring but not yet returned to the used ring.
Halil Pasice66bcc42016-12-19 16:44:44 +01003180 * Since max ring size < UINT16_MAX it's safe to use modulo
3181 * UINT16_MAX + 1 subtraction.
Stefan Hajnoczibccdef62016-08-15 13:54:15 +01003182 */
Halil Pasice66bcc42016-12-19 16:44:44 +01003183 vdev->vq[i].inuse = (uint16_t)(vdev->vq[i].last_avail_idx -
3184 vdev->vq[i].used_idx);
Stefan Hajnoczibccdef62016-08-15 13:54:15 +01003185 if (vdev->vq[i].inuse > vdev->vq[i].vring.num) {
3186 error_report("VQ %d size 0x%x < last_avail_idx 0x%x - "
3187 "used_idx 0x%x",
3188 i, vdev->vq[i].vring.num,
3189 vdev->vq[i].last_avail_idx,
3190 vdev->vq[i].used_idx);
3191 return -1;
3192 }
Greg Kurz616a6552014-06-24 19:38:54 +02003193 }
3194 }
3195
Michael S. Tsirkin1dd71382019-10-11 15:58:03 +02003196 if (vdc->post_load) {
3197 ret = vdc->post_load(vdev);
3198 if (ret) {
3199 return ret;
3200 }
3201 }
3202
Greg Kurz616a6552014-06-24 19:38:54 +02003203 return 0;
aliguori967f97f2008-12-04 19:38:57 +00003204}
3205
KONRAD Frederic6a1a8cc2013-04-24 10:21:22 +02003206void virtio_cleanup(VirtIODevice *vdev)
aliguorib946a152009-04-17 17:11:08 +00003207{
Michael S. Tsirkin85cf2a82011-01-10 14:28:40 +02003208 qemu_del_vm_change_state_handler(vdev->vmstate);
KONRAD Frederic8e05db92013-01-15 00:08:02 +01003209}
3210
Philippe Mathieu-Daudé538f0492021-01-11 16:20:20 +01003211static void virtio_vmstate_change(void *opaque, bool running, RunState state)
Michael S. Tsirkin85cf2a82011-01-10 14:28:40 +02003212{
3213 VirtIODevice *vdev = opaque;
KONRAD Frederic1c819442013-04-24 10:21:21 +02003214 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
3215 VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
Xie Yongjie57f2c32019-06-26 10:31:26 +08003216 bool backend_run = running && virtio_device_started(vdev, vdev->status);
Michael S. Tsirkin9e8e8c42014-09-11 18:42:02 +03003217 vdev->vm_running = running;
Michael S. Tsirkin85cf2a82011-01-10 14:28:40 +02003218
3219 if (backend_run) {
3220 virtio_set_status(vdev, vdev->status);
3221 }
3222
KONRAD Frederic1c819442013-04-24 10:21:21 +02003223 if (k->vmstate_change) {
3224 k->vmstate_change(qbus->parent, backend_run);
Michael S. Tsirkin85cf2a82011-01-10 14:28:40 +02003225 }
3226
3227 if (!backend_run) {
3228 virtio_set_status(vdev, vdev->status);
3229 }
3230}
3231
Gongleic8075ca2014-09-30 14:10:38 +08003232void virtio_instance_init_common(Object *proxy_obj, void *data,
3233 size_t vdev_size, const char *vdev_name)
3234{
3235 DeviceState *vdev = data;
3236
Markus Armbruster9fc7fc42020-06-10 07:32:25 +02003237 object_initialize_child_with_props(proxy_obj, "virtio-backend", vdev,
3238 vdev_size, vdev_name, &error_abort,
3239 NULL);
Gongleic8075ca2014-09-30 14:10:38 +08003240 qdev_alias_all_properties(vdev, proxy_obj);
3241}
3242
KONRAD Frederic8e05db92013-01-15 00:08:02 +01003243void virtio_init(VirtIODevice *vdev, const char *name,
3244 uint16_t device_id, size_t config_size)
aliguori967f97f2008-12-04 19:38:57 +00003245{
Jason Wange0d686b2015-04-23 14:21:46 +08003246 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
3247 VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
Michael S. Tsirkinb8193ad2009-09-07 21:20:15 +03003248 int i;
Jason Wange0d686b2015-04-23 14:21:46 +08003249 int nvectors = k->query_nvectors ? k->query_nvectors(qbus->parent) : 0;
3250
3251 if (nvectors) {
3252 vdev->vector_queues =
3253 g_malloc0(sizeof(*vdev->vector_queues) * nvectors);
3254 }
3255
Xie Yongji868a8f42019-06-26 10:31:28 +08003256 vdev->start_on_kick = false;
Xie Yongjibadaf792019-03-20 19:26:40 +08003257 vdev->started = false;
Paul Brook53c25ce2009-05-18 14:51:59 +01003258 vdev->device_id = device_id;
aliguori967f97f2008-12-04 19:38:57 +00003259 vdev->status = 0;
Stefan Hajnoczid73415a2020-09-23 11:56:46 +01003260 qatomic_set(&vdev->isr, 0);
aliguori967f97f2008-12-04 19:38:57 +00003261 vdev->queue_sel = 0;
Michael S. Tsirkin7055e682009-06-21 19:50:13 +03003262 vdev->config_vector = VIRTIO_NO_VECTOR;
Jason Wang87b3bd12015-05-29 14:15:31 +08003263 vdev->vq = g_malloc0(sizeof(VirtQueue) * VIRTIO_QUEUE_MAX);
Luiz Capitulino13548692011-07-29 15:36:43 -03003264 vdev->vm_running = runstate_is_running();
Stefan Hajnoczif5ed3662016-09-21 16:52:19 +01003265 vdev->broken = false;
Jason Wang87b3bd12015-05-29 14:15:31 +08003266 for (i = 0; i < VIRTIO_QUEUE_MAX; i++) {
Michael S. Tsirkinb8193ad2009-09-07 21:20:15 +03003267 vdev->vq[i].vector = VIRTIO_NO_VECTOR;
Michael S. Tsirkin1cbdabe2010-03-17 13:08:02 +02003268 vdev->vq[i].vdev = vdev;
Jason Wange78a2b42013-01-30 19:12:37 +08003269 vdev->vq[i].queue_index = i;
Stefan Hajnoczifcccb272019-11-05 15:09:46 +01003270 vdev->vq[i].host_notifier_enabled = false;
Michael S. Tsirkin1cbdabe2010-03-17 13:08:02 +02003271 }
aliguori967f97f2008-12-04 19:38:57 +00003272
aliguori967f97f2008-12-04 19:38:57 +00003273 vdev->name = name;
3274 vdev->config_len = config_size;
KONRAD Frederic8e05db92013-01-15 00:08:02 +01003275 if (vdev->config_len) {
Anthony Liguori7267c092011-08-20 22:09:37 -05003276 vdev->config = g_malloc0(config_size);
KONRAD Frederic8e05db92013-01-15 00:08:02 +01003277 } else {
aliguori967f97f2008-12-04 19:38:57 +00003278 vdev->config = NULL;
KONRAD Frederic8e05db92013-01-15 00:08:02 +01003279 }
Stefan Hajnoczi1a8c0912019-06-20 18:37:09 +01003280 vdev->vmstate = qdev_add_vm_change_state_handler(DEVICE(vdev),
3281 virtio_vmstate_change, vdev);
Greg Kurz616a6552014-06-24 19:38:54 +02003282 vdev->device_endian = virtio_default_endian();
Victor Kaplansky56696552016-02-18 16:12:23 +02003283 vdev->use_guest_notifier_mask = true;
KONRAD Frederic8e05db92013-01-15 00:08:02 +01003284}
aliguori967f97f2008-12-04 19:38:57 +00003285
Cornelia Huck7c78bdd2020-07-07 12:54:45 +02003286/*
3287 * Only devices that have already been around prior to defining the virtio
3288 * standard support legacy mode; this includes devices not specified in the
3289 * standard. All newer devices conform to the virtio standard only.
3290 */
3291bool virtio_legacy_allowed(VirtIODevice *vdev)
3292{
3293 switch (vdev->device_id) {
3294 case VIRTIO_ID_NET:
3295 case VIRTIO_ID_BLOCK:
3296 case VIRTIO_ID_CONSOLE:
3297 case VIRTIO_ID_RNG:
3298 case VIRTIO_ID_BALLOON:
3299 case VIRTIO_ID_RPMSG:
3300 case VIRTIO_ID_SCSI:
3301 case VIRTIO_ID_9P:
3302 case VIRTIO_ID_RPROC_SERIAL:
3303 case VIRTIO_ID_CAIF:
3304 return true;
3305 default:
3306 return false;
3307 }
3308}
3309
Stefano Garzarellad55f5182020-09-21 14:25:03 +02003310bool virtio_legacy_check_disabled(VirtIODevice *vdev)
3311{
3312 return vdev->disable_legacy_check;
3313}
3314
Avi Kivitya8170e52012-10-23 12:30:10 +02003315hwaddr virtio_queue_get_desc_addr(VirtIODevice *vdev, int n)
Michael S. Tsirkin1cbdabe2010-03-17 13:08:02 +02003316{
3317 return vdev->vq[n].vring.desc;
3318}
3319
Laurent Vivier0c9753e2020-07-27 17:33:19 +02003320bool virtio_queue_enabled_legacy(VirtIODevice *vdev, int n)
3321{
3322 return virtio_queue_get_desc_addr(vdev, n) != 0;
3323}
3324
Jason Wang23bfaf72019-03-25 11:40:36 +08003325bool virtio_queue_enabled(VirtIODevice *vdev, int n)
3326{
Jason Wangb2a5f622020-07-01 22:55:27 +08003327 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
3328 VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
3329
3330 if (k->queue_enabled) {
3331 return k->queue_enabled(qbus->parent, n);
3332 }
Laurent Vivier0c9753e2020-07-27 17:33:19 +02003333 return virtio_queue_enabled_legacy(vdev, n);
Jason Wang23bfaf72019-03-25 11:40:36 +08003334}
3335
Avi Kivitya8170e52012-10-23 12:30:10 +02003336hwaddr virtio_queue_get_avail_addr(VirtIODevice *vdev, int n)
Michael S. Tsirkin1cbdabe2010-03-17 13:08:02 +02003337{
3338 return vdev->vq[n].vring.avail;
3339}
3340
Avi Kivitya8170e52012-10-23 12:30:10 +02003341hwaddr virtio_queue_get_used_addr(VirtIODevice *vdev, int n)
Michael S. Tsirkin1cbdabe2010-03-17 13:08:02 +02003342{
3343 return vdev->vq[n].vring.used;
3344}
3345
Avi Kivitya8170e52012-10-23 12:30:10 +02003346hwaddr virtio_queue_get_desc_size(VirtIODevice *vdev, int n)
Michael S. Tsirkin1cbdabe2010-03-17 13:08:02 +02003347{
3348 return sizeof(VRingDesc) * vdev->vq[n].vring.num;
3349}
3350
Avi Kivitya8170e52012-10-23 12:30:10 +02003351hwaddr virtio_queue_get_avail_size(VirtIODevice *vdev, int n)
Michael S. Tsirkin1cbdabe2010-03-17 13:08:02 +02003352{
Wei Xuf90cda62019-10-25 10:35:21 +02003353 int s;
3354
Jason Wang86044b22019-10-25 10:35:24 +02003355 if (virtio_vdev_has_feature(vdev, VIRTIO_F_RING_PACKED)) {
3356 return sizeof(struct VRingPackedDescEvent);
3357 }
3358
Wei Xuf90cda62019-10-25 10:35:21 +02003359 s = virtio_vdev_has_feature(vdev, VIRTIO_RING_F_EVENT_IDX) ? 2 : 0;
Michael S. Tsirkin1cbdabe2010-03-17 13:08:02 +02003360 return offsetof(VRingAvail, ring) +
Wei Xuf90cda62019-10-25 10:35:21 +02003361 sizeof(uint16_t) * vdev->vq[n].vring.num + s;
Michael S. Tsirkin1cbdabe2010-03-17 13:08:02 +02003362}
3363
Avi Kivitya8170e52012-10-23 12:30:10 +02003364hwaddr virtio_queue_get_used_size(VirtIODevice *vdev, int n)
Michael S. Tsirkin1cbdabe2010-03-17 13:08:02 +02003365{
Wei Xuf90cda62019-10-25 10:35:21 +02003366 int s;
3367
Jason Wang86044b22019-10-25 10:35:24 +02003368 if (virtio_vdev_has_feature(vdev, VIRTIO_F_RING_PACKED)) {
3369 return sizeof(struct VRingPackedDescEvent);
3370 }
3371
Wei Xuf90cda62019-10-25 10:35:21 +02003372 s = virtio_vdev_has_feature(vdev, VIRTIO_RING_F_EVENT_IDX) ? 2 : 0;
Michael S. Tsirkin1cbdabe2010-03-17 13:08:02 +02003373 return offsetof(VRingUsed, ring) +
Wei Xuf90cda62019-10-25 10:35:21 +02003374 sizeof(VRingUsedElem) * vdev->vq[n].vring.num + s;
Michael S. Tsirkin1cbdabe2010-03-17 13:08:02 +02003375}
3376
Jason Wang86044b22019-10-25 10:35:24 +02003377static unsigned int virtio_queue_packed_get_last_avail_idx(VirtIODevice *vdev,
3378 int n)
3379{
3380 unsigned int avail, used;
3381
3382 avail = vdev->vq[n].last_avail_idx;
3383 avail |= ((uint16_t)vdev->vq[n].last_avail_wrap_counter) << 15;
3384
3385 used = vdev->vq[n].used_idx;
3386 used |= ((uint16_t)vdev->vq[n].used_wrap_counter) << 15;
3387
3388 return avail | used << 16;
3389}
3390
3391static uint16_t virtio_queue_split_get_last_avail_idx(VirtIODevice *vdev,
3392 int n)
Michael S. Tsirkin1cbdabe2010-03-17 13:08:02 +02003393{
3394 return vdev->vq[n].last_avail_idx;
3395}
3396
Jason Wang86044b22019-10-25 10:35:24 +02003397unsigned int virtio_queue_get_last_avail_idx(VirtIODevice *vdev, int n)
Michael S. Tsirkin1cbdabe2010-03-17 13:08:02 +02003398{
Jason Wang86044b22019-10-25 10:35:24 +02003399 if (virtio_vdev_has_feature(vdev, VIRTIO_F_RING_PACKED)) {
3400 return virtio_queue_packed_get_last_avail_idx(vdev, n);
3401 } else {
3402 return virtio_queue_split_get_last_avail_idx(vdev, n);
3403 }
Michael S. Tsirkin1cbdabe2010-03-17 13:08:02 +02003404}
3405
Jason Wang86044b22019-10-25 10:35:24 +02003406static void virtio_queue_packed_set_last_avail_idx(VirtIODevice *vdev,
3407 int n, unsigned int idx)
3408{
3409 struct VirtQueue *vq = &vdev->vq[n];
3410
3411 vq->last_avail_idx = vq->shadow_avail_idx = idx & 0x7fff;
3412 vq->last_avail_wrap_counter =
3413 vq->shadow_avail_wrap_counter = !!(idx & 0x8000);
3414 idx >>= 16;
3415 vq->used_idx = idx & 0x7ffff;
3416 vq->used_wrap_counter = !!(idx & 0x8000);
3417}
3418
3419static void virtio_queue_split_set_last_avail_idx(VirtIODevice *vdev,
3420 int n, unsigned int idx)
3421{
3422 vdev->vq[n].last_avail_idx = idx;
3423 vdev->vq[n].shadow_avail_idx = idx;
3424}
3425
3426void virtio_queue_set_last_avail_idx(VirtIODevice *vdev, int n,
3427 unsigned int idx)
3428{
3429 if (virtio_vdev_has_feature(vdev, VIRTIO_F_RING_PACKED)) {
3430 virtio_queue_packed_set_last_avail_idx(vdev, n, idx);
3431 } else {
3432 virtio_queue_split_set_last_avail_idx(vdev, n, idx);
3433 }
3434}
3435
3436static void virtio_queue_packed_restore_last_avail_idx(VirtIODevice *vdev,
3437 int n)
3438{
3439 /* We don't have a reference like avail idx in shared memory */
3440 return;
3441}
3442
3443static void virtio_queue_split_restore_last_avail_idx(VirtIODevice *vdev,
3444 int n)
Maxime Coquelin2d4ba6c2017-11-16 19:48:34 +01003445{
Dr. David Alan Gilbertb5f53d02019-10-28 16:11:09 +00003446 RCU_READ_LOCK_GUARD();
Maxime Coquelin2d4ba6c2017-11-16 19:48:34 +01003447 if (vdev->vq[n].vring.desc) {
3448 vdev->vq[n].last_avail_idx = vring_used_idx(&vdev->vq[n]);
3449 vdev->vq[n].shadow_avail_idx = vdev->vq[n].last_avail_idx;
3450 }
Maxime Coquelin2d4ba6c2017-11-16 19:48:34 +01003451}
3452
Jason Wang86044b22019-10-25 10:35:24 +02003453void virtio_queue_restore_last_avail_idx(VirtIODevice *vdev, int n)
3454{
3455 if (virtio_vdev_has_feature(vdev, VIRTIO_F_RING_PACKED)) {
3456 virtio_queue_packed_restore_last_avail_idx(vdev, n);
3457 } else {
3458 virtio_queue_split_restore_last_avail_idx(vdev, n);
3459 }
3460}
3461
3462static void virtio_queue_packed_update_used_idx(VirtIODevice *vdev, int n)
3463{
3464 /* used idx was updated through set_last_avail_idx() */
3465 return;
3466}
3467
3468static void virtio_split_packed_update_used_idx(VirtIODevice *vdev, int n)
Yuri Benditovich312d3b32016-12-13 10:12:05 +02003469{
Dr. David Alan Gilbertb5f53d02019-10-28 16:11:09 +00003470 RCU_READ_LOCK_GUARD();
Paolo Bonzinica0176a2017-01-27 16:40:19 +01003471 if (vdev->vq[n].vring.desc) {
3472 vdev->vq[n].used_idx = vring_used_idx(&vdev->vq[n]);
3473 }
Yuri Benditovich312d3b32016-12-13 10:12:05 +02003474}
3475
Jason Wang86044b22019-10-25 10:35:24 +02003476void virtio_queue_update_used_idx(VirtIODevice *vdev, int n)
3477{
3478 if (virtio_vdev_has_feature(vdev, VIRTIO_F_RING_PACKED)) {
3479 return virtio_queue_packed_update_used_idx(vdev, n);
3480 } else {
3481 return virtio_split_packed_update_used_idx(vdev, n);
3482 }
3483}
3484
Stefan Hajnoczi6793dfd2013-08-12 11:08:09 +02003485void virtio_queue_invalidate_signalled_used(VirtIODevice *vdev, int n)
3486{
3487 vdev->vq[n].signalled_used_valid = false;
3488}
3489
Michael S. Tsirkin1cbdabe2010-03-17 13:08:02 +02003490VirtQueue *virtio_get_queue(VirtIODevice *vdev, int n)
3491{
3492 return vdev->vq + n;
3493}
3494
Jason Wange78a2b42013-01-30 19:12:37 +08003495uint16_t virtio_get_queue_index(VirtQueue *vq)
3496{
3497 return vq->queue_index;
3498}
3499
Paolo Bonzini15b2bd12012-07-05 17:16:30 +02003500static void virtio_queue_guest_notifier_read(EventNotifier *n)
3501{
3502 VirtQueue *vq = container_of(n, VirtQueue, guest_notifier);
3503 if (event_notifier_test_and_clear(n)) {
Michael S. Tsirkinb4b98622017-02-17 04:52:16 +02003504 virtio_irq(vq);
Paolo Bonzini15b2bd12012-07-05 17:16:30 +02003505 }
3506}
3507
3508void virtio_queue_set_guest_notifier_fd_handler(VirtQueue *vq, bool assign,
3509 bool with_irqfd)
3510{
3511 if (assign && !with_irqfd) {
Paolo Bonzinid6da1e92017-01-10 11:54:52 +01003512 event_notifier_set_handler(&vq->guest_notifier,
Paolo Bonzini15b2bd12012-07-05 17:16:30 +02003513 virtio_queue_guest_notifier_read);
3514 } else {
Paolo Bonzinid6da1e92017-01-10 11:54:52 +01003515 event_notifier_set_handler(&vq->guest_notifier, NULL);
Paolo Bonzini15b2bd12012-07-05 17:16:30 +02003516 }
3517 if (!assign) {
3518 /* Test and clear notifier before closing it,
3519 * in case poll callback didn't have time to run. */
3520 virtio_queue_guest_notifier_read(&vq->guest_notifier);
3521 }
3522}
3523
Michael S. Tsirkin1cbdabe2010-03-17 13:08:02 +02003524EventNotifier *virtio_queue_get_guest_notifier(VirtQueue *vq)
3525{
3526 return &vq->guest_notifier;
3527}
Paolo Bonzinib1f416a2012-07-05 17:16:29 +02003528
Michael S. Tsirkin344dc162016-04-06 12:16:25 +02003529static void virtio_queue_host_notifier_aio_read(EventNotifier *n)
Paolo Bonzinib1f416a2012-07-05 17:16:29 +02003530{
3531 VirtQueue *vq = container_of(n, VirtQueue, host_notifier);
3532 if (event_notifier_test_and_clear(n)) {
Michael S. Tsirkin344dc162016-04-06 12:16:25 +02003533 virtio_queue_notify_aio_vq(vq);
Paolo Bonzinib1f416a2012-07-05 17:16:29 +02003534 }
3535}
3536
Stefan Hajnoczia7c82152016-12-01 19:26:50 +00003537static void virtio_queue_host_notifier_aio_poll_begin(EventNotifier *n)
3538{
3539 VirtQueue *vq = container_of(n, VirtQueue, host_notifier);
3540
3541 virtio_queue_set_notification(vq, 0);
3542}
3543
Stefan Hajnoczi0062ea02016-12-01 19:26:43 +00003544static bool virtio_queue_host_notifier_aio_poll(void *opaque)
3545{
3546 EventNotifier *n = opaque;
3547 VirtQueue *vq = container_of(n, VirtQueue, host_notifier);
3548
Paolo Bonzinidd3dd4b2017-02-23 09:51:30 +01003549 if (!vq->vring.desc || virtio_queue_empty(vq)) {
Stefan Hajnoczi0062ea02016-12-01 19:26:43 +00003550 return false;
3551 }
3552
Stefan Hajnoczid0435bc2019-12-09 21:09:57 +00003553 return virtio_queue_notify_aio_vq(vq);
Stefan Hajnoczi0062ea02016-12-01 19:26:43 +00003554}
3555
Stefan Hajnoczia7c82152016-12-01 19:26:50 +00003556static void virtio_queue_host_notifier_aio_poll_end(EventNotifier *n)
3557{
3558 VirtQueue *vq = container_of(n, VirtQueue, host_notifier);
3559
3560 /* Caller polls once more after this to catch requests that race with us */
3561 virtio_queue_set_notification(vq, 1);
3562}
3563
Paolo Bonzinia1afb602016-02-14 18:17:06 +01003564void virtio_queue_aio_set_host_notifier_handler(VirtQueue *vq, AioContext *ctx,
Fam Zheng07931692017-02-09 16:40:47 +08003565 VirtIOHandleAIOOutput handle_output)
Paolo Bonzinia1afb602016-02-14 18:17:06 +01003566{
Paolo Bonzinia378b492016-04-06 12:16:28 +02003567 if (handle_output) {
3568 vq->handle_aio_output = handle_output;
Paolo Bonzinia1afb602016-02-14 18:17:06 +01003569 aio_set_event_notifier(ctx, &vq->host_notifier, true,
Stefan Hajnoczi0062ea02016-12-01 19:26:43 +00003570 virtio_queue_host_notifier_aio_read,
3571 virtio_queue_host_notifier_aio_poll);
Stefan Hajnoczia7c82152016-12-01 19:26:50 +00003572 aio_set_event_notifier_poll(ctx, &vq->host_notifier,
3573 virtio_queue_host_notifier_aio_poll_begin,
3574 virtio_queue_host_notifier_aio_poll_end);
Paolo Bonzinia1afb602016-02-14 18:17:06 +01003575 } else {
Stefan Hajnoczif6a51c82016-12-01 19:26:41 +00003576 aio_set_event_notifier(ctx, &vq->host_notifier, true, NULL, NULL);
Paolo Bonzinia1afb602016-02-14 18:17:06 +01003577 /* Test and clear notifier before after disabling event,
3578 * in case poll callback didn't have time to run. */
Michael S. Tsirkin344dc162016-04-06 12:16:25 +02003579 virtio_queue_host_notifier_aio_read(&vq->host_notifier);
Paolo Bonzinia378b492016-04-06 12:16:28 +02003580 vq->handle_aio_output = NULL;
Michael S. Tsirkin344dc162016-04-06 12:16:25 +02003581 }
3582}
3583
Paolo Bonzinifa283a42016-10-21 22:48:15 +02003584void virtio_queue_host_notifier_read(EventNotifier *n)
Michael S. Tsirkin344dc162016-04-06 12:16:25 +02003585{
3586 VirtQueue *vq = container_of(n, VirtQueue, host_notifier);
3587 if (event_notifier_test_and_clear(n)) {
3588 virtio_queue_notify_vq(vq);
Paolo Bonzinia1afb602016-02-14 18:17:06 +01003589 }
3590}
3591
Michael S. Tsirkin1cbdabe2010-03-17 13:08:02 +02003592EventNotifier *virtio_queue_get_host_notifier(VirtQueue *vq)
3593{
3594 return &vq->host_notifier;
3595}
KONRAD Frederic8e05db92013-01-15 00:08:02 +01003596
Stefan Hajnoczifcccb272019-11-05 15:09:46 +01003597void virtio_queue_set_host_notifier_enabled(VirtQueue *vq, bool enabled)
3598{
3599 vq->host_notifier_enabled = enabled;
3600}
3601
Tiwei Bie6f80e612018-04-12 23:12:30 +08003602int virtio_queue_set_host_notifier_mr(VirtIODevice *vdev, int n,
3603 MemoryRegion *mr, bool assign)
3604{
3605 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
3606 VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
3607
3608 if (k->set_host_notifier_mr) {
3609 return k->set_host_notifier_mr(qbus->parent, n, mr, assign);
3610 }
3611
3612 return -1;
3613}
3614
KONRAD Frederic1034e9c2013-04-30 16:08:48 +02003615void virtio_device_set_child_bus_name(VirtIODevice *vdev, char *bus_name)
3616{
Markus Armbruster9e288402014-06-06 18:43:29 +02003617 g_free(vdev->bus_name);
Markus Armbruster80e00902014-06-06 18:43:30 +02003618 vdev->bus_name = g_strdup(bus_name);
KONRAD Frederic1034e9c2013-04-30 16:08:48 +02003619}
3620
Stefan Hajnoczif5ed3662016-09-21 16:52:19 +01003621void GCC_FMT_ATTR(2, 3) virtio_error(VirtIODevice *vdev, const char *fmt, ...)
3622{
3623 va_list ap;
3624
3625 va_start(ap, fmt);
3626 error_vreport(fmt, ap);
3627 va_end(ap);
3628
Stefan Hajnoczif5ed3662016-09-21 16:52:19 +01003629 if (virtio_vdev_has_feature(vdev, VIRTIO_F_VERSION_1)) {
Michael S. Tsirkin8fc47c82017-12-13 21:59:54 +02003630 vdev->status = vdev->status | VIRTIO_CONFIG_S_NEEDS_RESET;
Stefan Hajnoczif5ed3662016-09-21 16:52:19 +01003631 virtio_notify_config(vdev);
3632 }
Greg Kurz66453cf2017-05-17 10:17:51 +02003633
3634 vdev->broken = true;
Stefan Hajnoczif5ed3662016-09-21 16:52:19 +01003635}
3636
Paolo Bonzinic611c762017-01-27 16:40:17 +01003637static void virtio_memory_listener_commit(MemoryListener *listener)
3638{
3639 VirtIODevice *vdev = container_of(listener, VirtIODevice, listener);
3640 int i;
3641
3642 for (i = 0; i < VIRTIO_QUEUE_MAX; i++) {
3643 if (vdev->vq[i].vring.num == 0) {
3644 break;
3645 }
3646 virtio_init_region_cache(vdev, i);
3647 }
3648}
3649
Andreas Färber1d244b42013-07-30 00:50:27 +02003650static void virtio_device_realize(DeviceState *dev, Error **errp)
KONRAD Frederic8e05db92013-01-15 00:08:02 +01003651{
Andreas Färber1d244b42013-07-30 00:50:27 +02003652 VirtIODevice *vdev = VIRTIO_DEVICE(dev);
3653 VirtioDeviceClass *vdc = VIRTIO_DEVICE_GET_CLASS(dev);
3654 Error *err = NULL;
3655
Dr. David Alan Gilbertea43e252016-10-27 18:36:36 +01003656 /* Devices should either use vmsd or the load/save methods */
3657 assert(!vdc->vmsd || !vdc->load);
3658
Andreas Färber1d244b42013-07-30 00:50:27 +02003659 if (vdc->realize != NULL) {
3660 vdc->realize(dev, &err);
3661 if (err != NULL) {
3662 error_propagate(errp, err);
3663 return;
3664 }
KONRAD Frederic8e05db92013-01-15 00:08:02 +01003665 }
Jason Wange8398042015-05-29 14:15:25 +08003666
3667 virtio_bus_device_plugged(vdev, &err);
3668 if (err != NULL) {
3669 error_propagate(errp, err);
Markus Armbrusterb69c3c22020-05-05 17:29:24 +02003670 vdc->unrealize(dev);
Jason Wange8398042015-05-29 14:15:25 +08003671 return;
3672 }
Paolo Bonzinic611c762017-01-27 16:40:17 +01003673
3674 vdev->listener.commit = virtio_memory_listener_commit;
3675 memory_listener_register(&vdev->listener, vdev->dma_as);
KONRAD Frederic8e05db92013-01-15 00:08:02 +01003676}
3677
Markus Armbrusterb69c3c22020-05-05 17:29:24 +02003678static void virtio_device_unrealize(DeviceState *dev)
KONRAD Frederic1034e9c2013-04-30 16:08:48 +02003679{
Andreas Färber1d244b42013-07-30 00:50:27 +02003680 VirtIODevice *vdev = VIRTIO_DEVICE(dev);
Andreas Färber306ec6c2013-07-30 03:50:44 +02003681 VirtioDeviceClass *vdc = VIRTIO_DEVICE_GET_CLASS(dev);
KONRAD Frederic1034e9c2013-04-30 16:08:48 +02003682
Eugenio Pérezf6ab64c2021-01-25 20:25:05 +01003683 memory_listener_unregister(&vdev->listener);
Paolo Bonzini83d07042013-12-20 19:48:51 +01003684 virtio_bus_device_unplugged(vdev);
3685
Andreas Färber306ec6c2013-07-30 03:50:44 +02003686 if (vdc->unrealize != NULL) {
Markus Armbrusterb69c3c22020-05-05 17:29:24 +02003687 vdc->unrealize(dev);
Paolo Bonzini5e96f5d2013-09-20 13:59:08 +02003688 }
Andreas Färber1d244b42013-07-30 00:50:27 +02003689
Markus Armbruster9e288402014-06-06 18:43:29 +02003690 g_free(vdev->bus_name);
3691 vdev->bus_name = NULL;
KONRAD Frederic1034e9c2013-04-30 16:08:48 +02003692}
3693
Paolo Bonzinic611c762017-01-27 16:40:17 +01003694static void virtio_device_free_virtqueues(VirtIODevice *vdev)
3695{
3696 int i;
3697 if (!vdev->vq) {
3698 return;
3699 }
3700
3701 for (i = 0; i < VIRTIO_QUEUE_MAX; i++) {
Paolo Bonzinic611c762017-01-27 16:40:17 +01003702 if (vdev->vq[i].vring.num == 0) {
3703 break;
3704 }
Jason Wange0e2d642017-03-15 19:48:31 +08003705 virtio_virtqueue_reset_region_cache(&vdev->vq[i]);
Paolo Bonzinic611c762017-01-27 16:40:17 +01003706 }
3707 g_free(vdev->vq);
3708}
3709
3710static void virtio_device_instance_finalize(Object *obj)
3711{
3712 VirtIODevice *vdev = VIRTIO_DEVICE(obj);
3713
Paolo Bonzinic611c762017-01-27 16:40:17 +01003714 virtio_device_free_virtqueues(vdev);
3715
3716 g_free(vdev->config);
3717 g_free(vdev->vector_queues);
3718}
3719
Cornelia Huck6b8f1022015-05-26 16:34:47 +02003720static Property virtio_properties[] = {
3721 DEFINE_VIRTIO_COMMON_FEATURES(VirtIODevice, host_features),
Xie Yongjie57f2c32019-06-26 10:31:26 +08003722 DEFINE_PROP_BOOL("use-started", VirtIODevice, use_started, true),
Michael Roth9d7bd082019-11-19 18:50:03 -06003723 DEFINE_PROP_BOOL("use-disabled-flag", VirtIODevice, use_disabled_flag, true),
Stefano Garzarellad55f5182020-09-21 14:25:03 +02003724 DEFINE_PROP_BOOL("x-disable-legacy-check", VirtIODevice,
3725 disable_legacy_check, false),
Cornelia Huck6b8f1022015-05-26 16:34:47 +02003726 DEFINE_PROP_END_OF_LIST(),
3727};
3728
Paolo Bonziniff4c07d2016-10-21 22:48:07 +02003729static int virtio_device_start_ioeventfd_impl(VirtIODevice *vdev)
3730{
3731 VirtioBusState *qbus = VIRTIO_BUS(qdev_get_parent_bus(DEVICE(vdev)));
Gal Hammer710fccf2018-01-29 16:20:57 +02003732 int i, n, r, err;
Paolo Bonziniff4c07d2016-10-21 22:48:07 +02003733
Gal Hammer710fccf2018-01-29 16:20:57 +02003734 memory_region_transaction_begin();
Paolo Bonziniff4c07d2016-10-21 22:48:07 +02003735 for (n = 0; n < VIRTIO_QUEUE_MAX; n++) {
Paolo Bonzinifa283a42016-10-21 22:48:15 +02003736 VirtQueue *vq = &vdev->vq[n];
Paolo Bonziniff4c07d2016-10-21 22:48:07 +02003737 if (!virtio_queue_get_num(vdev, n)) {
3738 continue;
3739 }
Paolo Bonzinied08a2a2016-10-21 22:48:14 +02003740 r = virtio_bus_set_host_notifier(qbus, n, true);
Paolo Bonziniff4c07d2016-10-21 22:48:07 +02003741 if (r < 0) {
3742 err = r;
3743 goto assign_error;
3744 }
Paolo Bonzinid6da1e92017-01-10 11:54:52 +01003745 event_notifier_set_handler(&vq->host_notifier,
Paolo Bonzinifa283a42016-10-21 22:48:15 +02003746 virtio_queue_host_notifier_read);
Paolo Bonzini6019f3b2016-10-21 22:48:12 +02003747 }
3748
3749 for (n = 0; n < VIRTIO_QUEUE_MAX; n++) {
3750 /* Kick right away to begin processing requests already in vring */
3751 VirtQueue *vq = &vdev->vq[n];
3752 if (!vq->vring.num) {
3753 continue;
3754 }
3755 event_notifier_set(&vq->host_notifier);
Paolo Bonziniff4c07d2016-10-21 22:48:07 +02003756 }
Gal Hammer710fccf2018-01-29 16:20:57 +02003757 memory_region_transaction_commit();
Paolo Bonziniff4c07d2016-10-21 22:48:07 +02003758 return 0;
3759
3760assign_error:
Gal Hammer710fccf2018-01-29 16:20:57 +02003761 i = n; /* save n for a second iteration after transaction is committed. */
Paolo Bonziniff4c07d2016-10-21 22:48:07 +02003762 while (--n >= 0) {
Paolo Bonzinifa283a42016-10-21 22:48:15 +02003763 VirtQueue *vq = &vdev->vq[n];
Paolo Bonziniff4c07d2016-10-21 22:48:07 +02003764 if (!virtio_queue_get_num(vdev, n)) {
3765 continue;
3766 }
3767
Paolo Bonzinid6da1e92017-01-10 11:54:52 +01003768 event_notifier_set_handler(&vq->host_notifier, NULL);
Paolo Bonzinied08a2a2016-10-21 22:48:14 +02003769 r = virtio_bus_set_host_notifier(qbus, n, false);
Paolo Bonziniff4c07d2016-10-21 22:48:07 +02003770 assert(r >= 0);
Gal Hammer710fccf2018-01-29 16:20:57 +02003771 }
3772 memory_region_transaction_commit();
3773
3774 while (--i >= 0) {
3775 if (!virtio_queue_get_num(vdev, i)) {
3776 continue;
3777 }
3778 virtio_bus_cleanup_host_notifier(qbus, i);
Paolo Bonziniff4c07d2016-10-21 22:48:07 +02003779 }
3780 return err;
3781}
3782
3783int virtio_device_start_ioeventfd(VirtIODevice *vdev)
3784{
3785 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
3786 VirtioBusState *vbus = VIRTIO_BUS(qbus);
3787
3788 return virtio_bus_start_ioeventfd(vbus);
3789}
3790
3791static void virtio_device_stop_ioeventfd_impl(VirtIODevice *vdev)
3792{
3793 VirtioBusState *qbus = VIRTIO_BUS(qdev_get_parent_bus(DEVICE(vdev)));
Paolo Bonziniff4c07d2016-10-21 22:48:07 +02003794 int n, r;
3795
Gal Hammer710fccf2018-01-29 16:20:57 +02003796 memory_region_transaction_begin();
Paolo Bonziniff4c07d2016-10-21 22:48:07 +02003797 for (n = 0; n < VIRTIO_QUEUE_MAX; n++) {
Paolo Bonzinifa283a42016-10-21 22:48:15 +02003798 VirtQueue *vq = &vdev->vq[n];
3799
Paolo Bonziniff4c07d2016-10-21 22:48:07 +02003800 if (!virtio_queue_get_num(vdev, n)) {
3801 continue;
3802 }
Paolo Bonzinid6da1e92017-01-10 11:54:52 +01003803 event_notifier_set_handler(&vq->host_notifier, NULL);
Paolo Bonzinied08a2a2016-10-21 22:48:14 +02003804 r = virtio_bus_set_host_notifier(qbus, n, false);
Paolo Bonziniff4c07d2016-10-21 22:48:07 +02003805 assert(r >= 0);
Gal Hammer710fccf2018-01-29 16:20:57 +02003806 }
3807 memory_region_transaction_commit();
3808
3809 for (n = 0; n < VIRTIO_QUEUE_MAX; n++) {
3810 if (!virtio_queue_get_num(vdev, n)) {
3811 continue;
3812 }
Gal Hammer76143612018-01-29 16:20:56 +02003813 virtio_bus_cleanup_host_notifier(qbus, n);
Paolo Bonziniff4c07d2016-10-21 22:48:07 +02003814 }
3815}
3816
Paolo Bonzini310837d2016-11-18 16:07:00 +01003817int virtio_device_grab_ioeventfd(VirtIODevice *vdev)
3818{
3819 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
3820 VirtioBusState *vbus = VIRTIO_BUS(qbus);
3821
3822 return virtio_bus_grab_ioeventfd(vbus);
3823}
3824
3825void virtio_device_release_ioeventfd(VirtIODevice *vdev)
3826{
3827 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
3828 VirtioBusState *vbus = VIRTIO_BUS(qbus);
3829
3830 virtio_bus_release_ioeventfd(vbus);
3831}
3832
KONRAD Frederic8e05db92013-01-15 00:08:02 +01003833static void virtio_device_class_init(ObjectClass *klass, void *data)
3834{
3835 /* Set the default value here. */
Paolo Bonziniff4c07d2016-10-21 22:48:07 +02003836 VirtioDeviceClass *vdc = VIRTIO_DEVICE_CLASS(klass);
KONRAD Frederic8e05db92013-01-15 00:08:02 +01003837 DeviceClass *dc = DEVICE_CLASS(klass);
Andreas Färber1d244b42013-07-30 00:50:27 +02003838
3839 dc->realize = virtio_device_realize;
3840 dc->unrealize = virtio_device_unrealize;
KONRAD Frederic8e05db92013-01-15 00:08:02 +01003841 dc->bus_type = TYPE_VIRTIO_BUS;
Marc-André Lureau4f67d302020-01-10 19:30:32 +04003842 device_class_set_props(dc, virtio_properties);
Paolo Bonziniff4c07d2016-10-21 22:48:07 +02003843 vdc->start_ioeventfd = virtio_device_start_ioeventfd_impl;
3844 vdc->stop_ioeventfd = virtio_device_stop_ioeventfd_impl;
Michael S. Tsirkin9b706db2016-11-04 12:04:23 +02003845
3846 vdc->legacy_features |= VIRTIO_LEGACY_FEATURES;
KONRAD Frederic8e05db92013-01-15 00:08:02 +01003847}
3848
Paolo Bonzini8e93cef2016-10-21 22:48:08 +02003849bool virtio_device_ioeventfd_enabled(VirtIODevice *vdev)
3850{
3851 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
3852 VirtioBusState *vbus = VIRTIO_BUS(qbus);
3853
3854 return virtio_bus_ioeventfd_enabled(vbus);
3855}
3856
KONRAD Frederic8e05db92013-01-15 00:08:02 +01003857static const TypeInfo virtio_device_info = {
3858 .name = TYPE_VIRTIO_DEVICE,
3859 .parent = TYPE_DEVICE,
3860 .instance_size = sizeof(VirtIODevice),
3861 .class_init = virtio_device_class_init,
Paolo Bonzinic611c762017-01-27 16:40:17 +01003862 .instance_finalize = virtio_device_instance_finalize,
KONRAD Frederic8e05db92013-01-15 00:08:02 +01003863 .abstract = true,
3864 .class_size = sizeof(VirtioDeviceClass),
3865};
3866
3867static void virtio_register_types(void)
3868{
3869 type_register_static(&virtio_device_info);
3870}
3871
3872type_init(virtio_register_types)