blob: 6aaa32a147c1ab8df3d7634db38f36b3b808ce81 [file] [log] [blame]
Paolo Bonzinif42b2202012-06-09 04:01:51 +02001/*
2 * QEMU aio implementation
3 *
4 * Copyright IBM Corp., 2008
5 * Copyright Red Hat Inc., 2012
6 *
7 * Authors:
8 * Anthony Liguori <aliguori@us.ibm.com>
9 * Paolo Bonzini <pbonzini@redhat.com>
10 *
11 * This work is licensed under the terms of the GNU GPL, version 2. See
12 * the COPYING file in the top-level directory.
13 *
14 * Contributions after 2012-01-13 are licensed under the terms of the
15 * GNU GPL, version 2 or (at your option) any later version.
16 */
17
Peter Maydelld38ea872016-01-29 17:50:05 +000018#include "qemu/osdep.h"
Paolo Bonzinif42b2202012-06-09 04:01:51 +020019#include "qemu-common.h"
Paolo Bonzini737e1502012-12-17 18:19:44 +010020#include "block/block.h"
Paolo Bonzini1de7afc2012-12-17 18:20:00 +010021#include "qemu/queue.h"
22#include "qemu/sockets.h"
Paolo Bonzinif42b2202012-06-09 04:01:51 +020023
24struct AioHandler {
25 EventNotifier *e;
Paolo Bonzinib4933172014-07-09 11:53:10 +020026 IOHandler *io_read;
27 IOHandler *io_write;
Paolo Bonzinif42b2202012-06-09 04:01:51 +020028 EventNotifierHandler *io_notify;
Paolo Bonzinif42b2202012-06-09 04:01:51 +020029 GPollFD pfd;
30 int deleted;
Paolo Bonzinib4933172014-07-09 11:53:10 +020031 void *opaque;
Fam Zhengdca21ef2015-10-23 11:08:05 +080032 bool is_external;
Paolo Bonzinif42b2202012-06-09 04:01:51 +020033 QLIST_ENTRY(AioHandler) node;
34};
35
Paolo Bonzinib4933172014-07-09 11:53:10 +020036void aio_set_fd_handler(AioContext *ctx,
37 int fd,
Fam Zhengdca21ef2015-10-23 11:08:05 +080038 bool is_external,
Paolo Bonzinib4933172014-07-09 11:53:10 +020039 IOHandler *io_read,
40 IOHandler *io_write,
41 void *opaque)
42{
43 /* fd is a SOCKET in our case */
44 AioHandler *node;
45
46 QLIST_FOREACH(node, &ctx->aio_handlers, node) {
47 if (node->pfd.fd == fd && !node->deleted) {
48 break;
49 }
50 }
51
52 /* Are we deleting the fd handler? */
53 if (!io_read && !io_write) {
54 if (node) {
55 /* If the lock is held, just mark the node as deleted */
56 if (ctx->walking_handlers) {
57 node->deleted = 1;
58 node->pfd.revents = 0;
59 } else {
60 /* Otherwise, delete it for real. We can't just mark it as
61 * deleted because deleted nodes are only cleaned up after
62 * releasing the walking_handlers lock.
63 */
64 QLIST_REMOVE(node, node);
65 g_free(node);
66 }
67 }
68 } else {
69 HANDLE event;
70
71 if (node == NULL) {
72 /* Alloc and insert if it's not already there */
Markus Armbruster3ba235a2014-12-04 13:55:09 +010073 node = g_new0(AioHandler, 1);
Paolo Bonzinib4933172014-07-09 11:53:10 +020074 node->pfd.fd = fd;
75 QLIST_INSERT_HEAD(&ctx->aio_handlers, node, node);
76 }
77
78 node->pfd.events = 0;
79 if (node->io_read) {
80 node->pfd.events |= G_IO_IN;
81 }
82 if (node->io_write) {
83 node->pfd.events |= G_IO_OUT;
84 }
85
86 node->e = &ctx->notifier;
87
88 /* Update handler with latest information */
89 node->opaque = opaque;
90 node->io_read = io_read;
91 node->io_write = io_write;
Fam Zhengdca21ef2015-10-23 11:08:05 +080092 node->is_external = is_external;
Paolo Bonzinib4933172014-07-09 11:53:10 +020093
94 event = event_notifier_get_handle(&ctx->notifier);
95 WSAEventSelect(node->pfd.fd, event,
96 FD_READ | FD_ACCEPT | FD_CLOSE |
97 FD_CONNECT | FD_WRITE | FD_OOB);
98 }
99
100 aio_notify(ctx);
101}
102
Paolo Bonzinif42b2202012-06-09 04:01:51 +0200103void aio_set_event_notifier(AioContext *ctx,
104 EventNotifier *e,
Fam Zhengdca21ef2015-10-23 11:08:05 +0800105 bool is_external,
Stefan Hajnoczif2e5dca2013-04-11 17:26:25 +0200106 EventNotifierHandler *io_notify)
Paolo Bonzinif42b2202012-06-09 04:01:51 +0200107{
108 AioHandler *node;
109
110 QLIST_FOREACH(node, &ctx->aio_handlers, node) {
111 if (node->e == e && !node->deleted) {
112 break;
113 }
114 }
115
116 /* Are we deleting the fd handler? */
117 if (!io_notify) {
118 if (node) {
Paolo Bonzinie3713e02012-09-24 14:57:41 +0200119 g_source_remove_poll(&ctx->source, &node->pfd);
120
Paolo Bonzinif42b2202012-06-09 04:01:51 +0200121 /* If the lock is held, just mark the node as deleted */
122 if (ctx->walking_handlers) {
123 node->deleted = 1;
124 node->pfd.revents = 0;
125 } else {
126 /* Otherwise, delete it for real. We can't just mark it as
127 * deleted because deleted nodes are only cleaned up after
128 * releasing the walking_handlers lock.
129 */
130 QLIST_REMOVE(node, node);
131 g_free(node);
132 }
133 }
134 } else {
135 if (node == NULL) {
136 /* Alloc and insert if it's not already there */
Markus Armbruster3ba235a2014-12-04 13:55:09 +0100137 node = g_new0(AioHandler, 1);
Paolo Bonzinif42b2202012-06-09 04:01:51 +0200138 node->e = e;
139 node->pfd.fd = (uintptr_t)event_notifier_get_handle(e);
140 node->pfd.events = G_IO_IN;
Fam Zhengdca21ef2015-10-23 11:08:05 +0800141 node->is_external = is_external;
Paolo Bonzinif42b2202012-06-09 04:01:51 +0200142 QLIST_INSERT_HEAD(&ctx->aio_handlers, node, node);
Paolo Bonzinie3713e02012-09-24 14:57:41 +0200143
144 g_source_add_poll(&ctx->source, &node->pfd);
Paolo Bonzinif42b2202012-06-09 04:01:51 +0200145 }
146 /* Update handler with latest information */
147 node->io_notify = io_notify;
Paolo Bonzinif42b2202012-06-09 04:01:51 +0200148 }
Paolo Bonzini7ed2b242012-09-25 10:22:39 +0200149
150 aio_notify(ctx);
Paolo Bonzinif42b2202012-06-09 04:01:51 +0200151}
152
Paolo Bonzinia3462c62014-07-09 11:53:08 +0200153bool aio_prepare(AioContext *ctx)
154{
Paolo Bonzinib4933172014-07-09 11:53:10 +0200155 static struct timeval tv0;
156 AioHandler *node;
157 bool have_select_revents = false;
158 fd_set rfds, wfds;
159
160 /* fill fd sets */
161 FD_ZERO(&rfds);
162 FD_ZERO(&wfds);
163 QLIST_FOREACH(node, &ctx->aio_handlers, node) {
164 if (node->io_read) {
165 FD_SET ((SOCKET)node->pfd.fd, &rfds);
166 }
167 if (node->io_write) {
168 FD_SET ((SOCKET)node->pfd.fd, &wfds);
169 }
170 }
171
172 if (select(0, &rfds, &wfds, NULL, &tv0) > 0) {
173 QLIST_FOREACH(node, &ctx->aio_handlers, node) {
174 node->pfd.revents = 0;
175 if (FD_ISSET(node->pfd.fd, &rfds)) {
176 node->pfd.revents |= G_IO_IN;
177 have_select_revents = true;
178 }
179
180 if (FD_ISSET(node->pfd.fd, &wfds)) {
181 node->pfd.revents |= G_IO_OUT;
182 have_select_revents = true;
183 }
184 }
185 }
186
187 return have_select_revents;
Paolo Bonzinia3462c62014-07-09 11:53:08 +0200188}
189
Paolo Bonzinif42b2202012-06-09 04:01:51 +0200190bool aio_pending(AioContext *ctx)
191{
192 AioHandler *node;
193
194 QLIST_FOREACH(node, &ctx->aio_handlers, node) {
195 if (node->pfd.revents && node->io_notify) {
196 return true;
197 }
Paolo Bonzinib4933172014-07-09 11:53:10 +0200198
199 if ((node->pfd.revents & G_IO_IN) && node->io_read) {
200 return true;
201 }
202 if ((node->pfd.revents & G_IO_OUT) && node->io_write) {
203 return true;
204 }
Paolo Bonzinif42b2202012-06-09 04:01:51 +0200205 }
206
207 return false;
208}
209
Paolo Bonzinia398dea2014-07-09 11:53:03 +0200210static bool aio_dispatch_handlers(AioContext *ctx, HANDLE event)
Paolo Bonzinif42b2202012-06-09 04:01:51 +0200211{
212 AioHandler *node;
Paolo Bonzinia398dea2014-07-09 11:53:03 +0200213 bool progress = false;
Paolo Bonzinif42b2202012-06-09 04:01:51 +0200214
215 /*
Paolo Bonzini87f68d32014-07-07 15:18:02 +0200216 * We have to walk very carefully in case aio_set_fd_handler is
Paolo Bonzinif42b2202012-06-09 04:01:51 +0200217 * called while we're walking.
218 */
219 node = QLIST_FIRST(&ctx->aio_handlers);
220 while (node) {
221 AioHandler *tmp;
Paolo Bonzinib4933172014-07-09 11:53:10 +0200222 int revents = node->pfd.revents;
Paolo Bonzinif42b2202012-06-09 04:01:51 +0200223
224 ctx->walking_handlers++;
225
Paolo Bonzinia398dea2014-07-09 11:53:03 +0200226 if (!node->deleted &&
Paolo Bonzinib4933172014-07-09 11:53:10 +0200227 (revents || event_notifier_get_handle(node->e) == event) &&
Paolo Bonzinia398dea2014-07-09 11:53:03 +0200228 node->io_notify) {
Paolo Bonzinif42b2202012-06-09 04:01:51 +0200229 node->pfd.revents = 0;
230 node->io_notify(node->e);
Stefan Hajnoczi164a1012013-04-11 16:56:50 +0200231
232 /* aio_notify() does not count as progress */
Stefan Hajnoczi8b2d42d2013-08-22 15:28:35 +0200233 if (node->e != &ctx->notifier) {
Stefan Hajnoczi164a1012013-04-11 16:56:50 +0200234 progress = true;
235 }
Paolo Bonzinif42b2202012-06-09 04:01:51 +0200236 }
237
Paolo Bonzinib4933172014-07-09 11:53:10 +0200238 if (!node->deleted &&
239 (node->io_read || node->io_write)) {
240 node->pfd.revents = 0;
241 if ((revents & G_IO_IN) && node->io_read) {
242 node->io_read(node->opaque);
243 progress = true;
244 }
245 if ((revents & G_IO_OUT) && node->io_write) {
246 node->io_write(node->opaque);
247 progress = true;
248 }
249
250 /* if the next select() will return an event, we have progressed */
251 if (event == event_notifier_get_handle(&ctx->notifier)) {
252 WSANETWORKEVENTS ev;
253 WSAEnumNetworkEvents(node->pfd.fd, event, &ev);
254 if (ev.lNetworkEvents) {
255 progress = true;
256 }
257 }
258 }
259
Paolo Bonzinif42b2202012-06-09 04:01:51 +0200260 tmp = node;
261 node = QLIST_NEXT(node, node);
262
263 ctx->walking_handlers--;
264
265 if (!ctx->walking_handlers && tmp->deleted) {
266 QLIST_REMOVE(tmp, node);
267 g_free(tmp);
268 }
269 }
270
Paolo Bonzinia398dea2014-07-09 11:53:03 +0200271 return progress;
272}
273
Paolo Bonzinie4c7e2d2014-07-09 11:53:05 +0200274bool aio_dispatch(AioContext *ctx)
Paolo Bonzinia398dea2014-07-09 11:53:03 +0200275{
276 bool progress;
277
Paolo Bonzinie4c7e2d2014-07-09 11:53:05 +0200278 progress = aio_bh_poll(ctx);
279 progress |= aio_dispatch_handlers(ctx, INVALID_HANDLE_VALUE);
Paolo Bonzinid397ec992014-07-09 11:53:02 +0200280 progress |= timerlistgroup_run_timers(&ctx->tlg);
Paolo Bonzinia398dea2014-07-09 11:53:03 +0200281 return progress;
282}
283
284bool aio_poll(AioContext *ctx, bool blocking)
285{
286 AioHandler *node;
287 HANDLE events[MAXIMUM_WAIT_OBJECTS + 1];
Paolo Bonzinieabc9772015-07-21 16:07:51 +0200288 bool progress, have_select_revents, first;
Paolo Bonzinia398dea2014-07-09 11:53:03 +0200289 int count;
290 int timeout;
291
Paolo Bonzini49110172015-02-20 17:26:51 +0100292 aio_context_acquire(ctx);
Paolo Bonzinia398dea2014-07-09 11:53:03 +0200293 progress = false;
294
Paolo Bonzini0a9dd162014-07-09 11:53:07 +0200295 /* aio_notify can avoid the expensive event_notifier_set if
296 * everything (file descriptors, bottom halves, timers) will
297 * be re-evaluated before the next blocking poll(). This is
298 * already true when aio_poll is called with blocking == false;
Paolo Bonzinieabc9772015-07-21 16:07:51 +0200299 * if blocking == true, it is only true after poll() returns,
300 * so disable the optimization now.
Paolo Bonzini0a9dd162014-07-09 11:53:07 +0200301 */
Paolo Bonzinieabc9772015-07-21 16:07:51 +0200302 if (blocking) {
303 atomic_add(&ctx->notify_me, 2);
304 }
Paolo Bonzini0a9dd162014-07-09 11:53:07 +0200305
Paolo Bonzini6493c972015-07-21 16:07:50 +0200306 have_select_revents = aio_prepare(ctx);
307
Paolo Bonzinif42b2202012-06-09 04:01:51 +0200308 ctx->walking_handlers++;
309
310 /* fill fd sets */
Paolo Bonzinif42b2202012-06-09 04:01:51 +0200311 count = 0;
312 QLIST_FOREACH(node, &ctx->aio_handlers, node) {
Fam Zhengc1e1e5f2015-10-23 11:08:08 +0800313 if (!node->deleted && node->io_notify
314 && aio_node_check(ctx, node->is_external)) {
Paolo Bonzinif42b2202012-06-09 04:01:51 +0200315 events[count++] = event_notifier_get_handle(node->e);
316 }
317 }
318
319 ctx->walking_handlers--;
Paolo Bonzini3672fa52014-07-09 11:53:04 +0200320 first = true;
Paolo Bonzinif42b2202012-06-09 04:01:51 +0200321
Paolo Bonzini6493c972015-07-21 16:07:50 +0200322 /* ctx->notifier is always registered. */
323 assert(count > 0);
324
325 /* Multiple iterations, all of them non-blocking except the first,
326 * may be necessary to process all pending events. After the first
327 * WaitForMultipleObjects call ctx->notify_me will be decremented.
328 */
329 do {
Paolo Bonzinib4933172014-07-09 11:53:10 +0200330 HANDLE event;
Alex Bligh438e1f42013-08-21 16:02:53 +0100331 int ret;
332
Paolo Bonzini6493c972015-07-21 16:07:50 +0200333 timeout = blocking && !have_select_revents
Paolo Bonzini845ca102014-07-09 11:53:01 +0200334 ? qemu_timeout_ns_to_ms(aio_compute_timeout(ctx)) : 0;
Paolo Bonzini49110172015-02-20 17:26:51 +0100335 if (timeout) {
336 aio_context_release(ctx);
337 }
Alex Bligh438e1f42013-08-21 16:02:53 +0100338 ret = WaitForMultipleObjects(count, events, FALSE, timeout);
Paolo Bonzinieabc9772015-07-21 16:07:51 +0200339 if (blocking) {
340 assert(first);
341 atomic_sub(&ctx->notify_me, 2);
342 }
Paolo Bonzini49110172015-02-20 17:26:51 +0100343 if (timeout) {
344 aio_context_acquire(ctx);
345 }
Paolo Bonzinif42b2202012-06-09 04:01:51 +0200346
Paolo Bonzini21a03d12015-07-21 16:07:52 +0200347 if (first) {
Paolo Bonzini05e514b2015-07-21 16:07:53 +0200348 aio_notify_accept(ctx);
Paolo Bonzini21a03d12015-07-21 16:07:52 +0200349 progress |= aio_bh_poll(ctx);
350 first = false;
Paolo Bonzini3672fa52014-07-09 11:53:04 +0200351 }
Paolo Bonzini3672fa52014-07-09 11:53:04 +0200352
Paolo Bonzinif42b2202012-06-09 04:01:51 +0200353 /* if we have any signaled events, dispatch event */
Paolo Bonzinib4933172014-07-09 11:53:10 +0200354 event = NULL;
355 if ((DWORD) (ret - WAIT_OBJECT_0) < count) {
356 event = events[ret - WAIT_OBJECT_0];
Paolo Bonzinia90d4112014-09-15 14:52:58 +0200357 events[ret - WAIT_OBJECT_0] = events[--count];
Paolo Bonzinib4933172014-07-09 11:53:10 +0200358 } else if (!have_select_revents) {
Paolo Bonzinif42b2202012-06-09 04:01:51 +0200359 break;
360 }
361
Paolo Bonzinib4933172014-07-09 11:53:10 +0200362 have_select_revents = false;
Paolo Bonzinif42b2202012-06-09 04:01:51 +0200363 blocking = false;
364
Paolo Bonzinib4933172014-07-09 11:53:10 +0200365 progress |= aio_dispatch_handlers(ctx, event);
Paolo Bonzini6493c972015-07-21 16:07:50 +0200366 } while (count > 0);
Paolo Bonzinif42b2202012-06-09 04:01:51 +0200367
Paolo Bonzinie4c7e2d2014-07-09 11:53:05 +0200368 progress |= timerlistgroup_run_timers(&ctx->tlg);
Alex Bligh438e1f42013-08-21 16:02:53 +0100369
Paolo Bonzini49110172015-02-20 17:26:51 +0100370 aio_context_release(ctx);
Stefan Hajnoczi164a1012013-04-11 16:56:50 +0200371 return progress;
Paolo Bonzinif42b2202012-06-09 04:01:51 +0200372}
Fam Zheng37fcee52015-10-30 12:06:28 +0800373
374void aio_context_setup(AioContext *ctx, Error **errp)
375{
376}