blob: 731f3826c062df3df4caac5f45d26d203489ee04 [file] [log] [blame]
aliguoria76bab42008-09-22 19:17:18 +00001/*
2 * QEMU aio implementation
3 *
4 * Copyright IBM, Corp. 2008
5 *
6 * Authors:
7 * Anthony Liguori <aliguori@us.ibm.com>
8 *
9 * This work is licensed under the terms of the GNU GPL, version 2. See
10 * the COPYING file in the top-level directory.
11 *
Paolo Bonzini6b620ca2012-01-13 17:44:23 +010012 * Contributions after 2012-01-13 are licensed under the terms of the
13 * GNU GPL, version 2 or (at your option) any later version.
aliguoria76bab42008-09-22 19:17:18 +000014 */
15
Peter Maydelld38ea872016-01-29 17:50:05 +000016#include "qemu/osdep.h"
Paolo Bonzini737e1502012-12-17 18:19:44 +010017#include "block/block.h"
Nicolas Saenz Julienne71ad4712022-04-25 09:57:23 +020018#include "block/thread-pool.h"
Kevin Wolf9ce44e22020-10-05 17:58:50 +020019#include "qemu/main-loop.h"
Stefan Hajnoczif25c0b52020-02-18 18:27:08 +000020#include "qemu/rcu.h"
Paolo Bonzini2bbf11d2017-01-12 19:07:57 +010021#include "qemu/rcu_queue.h"
Paolo Bonzini1de7afc2012-12-17 18:20:00 +010022#include "qemu/sockets.h"
Stefan Hajnoczi4a1cba32016-12-01 19:26:42 +000023#include "qemu/cutils.h"
Paolo Bonzinic2b38b22017-02-13 14:52:18 +010024#include "trace.h"
Stefan Hajnoczi1f050a42020-03-05 17:08:02 +000025#include "aio-posix.h"
aliguoria76bab42008-09-22 19:17:18 +000026
Stefan Hajnoczid37d0e32020-03-05 17:08:06 +000027/* Stop userspace polling on a handler if it isn't active for some time */
28#define POLL_IDLE_INTERVAL_NS (7 * NANOSECONDS_PER_SECOND)
29
Stefan Hajnocziaa38e192020-03-05 17:08:05 +000030bool aio_poll_disabled(AioContext *ctx)
31{
Stefan Hajnoczid73415a2020-09-23 11:56:46 +010032 return qatomic_read(&ctx->poll_disable_cnt);
Stefan Hajnocziaa38e192020-03-05 17:08:05 +000033}
34
Stefan Hajnoczi1f050a42020-03-05 17:08:02 +000035void aio_add_ready_handler(AioHandlerList *ready_list,
36 AioHandler *node,
37 int revents)
Stefan Hajnoczi7391d342020-02-14 17:17:12 +000038{
39 QLIST_SAFE_REMOVE(node, node_ready); /* remove from nested parent's list */
40 node->pfd.revents = revents;
41 QLIST_INSERT_HEAD(ready_list, node, node_ready);
42}
43
Stefan Hajnoczifc879642022-02-23 15:57:03 +000044static void aio_add_poll_ready_handler(AioHandlerList *ready_list,
45 AioHandler *node)
46{
47 QLIST_SAFE_REMOVE(node, node_ready); /* remove from nested parent's list */
48 node->poll_ready = true;
49 QLIST_INSERT_HEAD(ready_list, node, node_ready);
50}
51
Paolo Bonzinia915f4b2012-09-13 12:28:51 +020052static AioHandler *find_aio_handler(AioContext *ctx, int fd)
aliguoria76bab42008-09-22 19:17:18 +000053{
54 AioHandler *node;
55
Paolo Bonzinia915f4b2012-09-13 12:28:51 +020056 QLIST_FOREACH(node, &ctx->aio_handlers, node) {
Stefan Hajnoczi47490792020-02-14 17:17:11 +000057 if (node->pfd.fd == fd) {
58 if (!QLIST_IS_INSERTED(node, node_deleted)) {
Alexander Graf79d5ca52009-05-06 02:58:48 +020059 return node;
Stefan Hajnoczi47490792020-02-14 17:17:11 +000060 }
61 }
aliguoria76bab42008-09-22 19:17:18 +000062 }
63
64 return NULL;
65}
66
Remy Noelfef16602018-12-20 16:20:30 +010067static bool aio_remove_fd_handler(AioContext *ctx, AioHandler *node)
68{
69 /* If the GSource is in the process of being destroyed then
70 * g_source_remove_poll() causes an assertion failure. Skip
71 * removal in that case, because glib cleans up its state during
72 * destruction anyway.
73 */
74 if (!g_source_is_destroyed(&ctx->source)) {
75 g_source_remove_poll(&ctx->source, &node->pfd);
76 }
77
Stefan Hajnoczi73fd2822020-03-05 17:08:04 +000078 node->pfd.revents = 0;
Stefan Hajnoczifc879642022-02-23 15:57:03 +000079 node->poll_ready = false;
Stefan Hajnoczi73fd2822020-03-05 17:08:04 +000080
81 /* If the fd monitor has already marked it deleted, leave it alone */
82 if (QLIST_IS_INSERTED(node, node_deleted)) {
83 return false;
84 }
85
Remy Noelfef16602018-12-20 16:20:30 +010086 /* If a read is in progress, just mark the node as deleted */
87 if (qemu_lockcnt_count(&ctx->list_lock)) {
Stefan Hajnoczi47490792020-02-14 17:17:11 +000088 QLIST_INSERT_HEAD_RCU(&ctx->deleted_aio_handlers, node, node_deleted);
Remy Noelfef16602018-12-20 16:20:30 +010089 return false;
90 }
91 /* Otherwise, delete it for real. We can't just mark it as
92 * deleted because deleted nodes are only cleaned up while
93 * no one is walking the handlers list.
94 */
Stefan Hajnoczid37d0e32020-03-05 17:08:06 +000095 QLIST_SAFE_REMOVE(node, node_poll);
Remy Noelfef16602018-12-20 16:20:30 +010096 QLIST_REMOVE(node, node);
97 return true;
98}
99
Paolo Bonzinia915f4b2012-09-13 12:28:51 +0200100void aio_set_fd_handler(AioContext *ctx,
101 int fd,
Fam Zhengdca21ef2015-10-23 11:08:05 +0800102 bool is_external,
Paolo Bonzinia915f4b2012-09-13 12:28:51 +0200103 IOHandler *io_read,
104 IOHandler *io_write,
Stefan Hajnoczif6a51c82016-12-01 19:26:41 +0000105 AioPollFn *io_poll,
Stefan Hajnoczi826cc322021-12-07 13:23:31 +0000106 IOHandler *io_poll_ready,
Paolo Bonzinia915f4b2012-09-13 12:28:51 +0200107 void *opaque)
aliguoria76bab42008-09-22 19:17:18 +0000108{
109 AioHandler *node;
Remy Noelfef16602018-12-20 16:20:30 +0100110 AioHandler *new_node = NULL;
Fam Zhengfbe3fc52015-10-30 12:06:29 +0800111 bool is_new = false;
Fam Zheng0ed39f3d2015-11-16 14:32:14 +0800112 bool deleted = false;
Paolo Bonzinid7be5dd2018-09-12 19:10:38 +0200113 int poll_disable_change;
aliguoria76bab42008-09-22 19:17:18 +0000114
Stefan Hajnoczi826cc322021-12-07 13:23:31 +0000115 if (io_poll && !io_poll_ready) {
116 io_poll = NULL; /* polling only makes sense if there is a handler */
117 }
118
Paolo Bonzini2bbf11d2017-01-12 19:07:57 +0100119 qemu_lockcnt_lock(&ctx->list_lock);
120
Paolo Bonzinia915f4b2012-09-13 12:28:51 +0200121 node = find_aio_handler(ctx, fd);
aliguoria76bab42008-09-22 19:17:18 +0000122
123 /* Are we deleting the fd handler? */
Stefan Hajnoczi4a1cba32016-12-01 19:26:42 +0000124 if (!io_read && !io_write && !io_poll) {
Paolo Bonzini36173ec2016-11-08 14:55:23 +0100125 if (node == NULL) {
Paolo Bonzini2bbf11d2017-01-12 19:07:57 +0100126 qemu_lockcnt_unlock(&ctx->list_lock);
Paolo Bonzini36173ec2016-11-08 14:55:23 +0100127 return;
128 }
Remy Noel8821b342018-12-20 16:20:29 +0100129 /* Clean events in order to unregister fd from the ctx epoll. */
130 node->pfd.events = 0;
131
Paolo Bonzinid7be5dd2018-09-12 19:10:38 +0200132 poll_disable_change = -!node->io_poll;
aliguoria76bab42008-09-22 19:17:18 +0000133 } else {
Paolo Bonzinid7be5dd2018-09-12 19:10:38 +0200134 poll_disable_change = !io_poll - (node && !node->io_poll);
aliguoria76bab42008-09-22 19:17:18 +0000135 if (node == NULL) {
Fam Zhengfbe3fc52015-10-30 12:06:29 +0800136 is_new = true;
aliguoria76bab42008-09-22 19:17:18 +0000137 }
Remy Noelfef16602018-12-20 16:20:30 +0100138 /* Alloc and insert if it's not already there */
139 new_node = g_new0(AioHandler, 1);
Stefan Hajnoczi4a1cba32016-12-01 19:26:42 +0000140
aliguoria76bab42008-09-22 19:17:18 +0000141 /* Update handler with latest information */
Remy Noelfef16602018-12-20 16:20:30 +0100142 new_node->io_read = io_read;
143 new_node->io_write = io_write;
144 new_node->io_poll = io_poll;
Stefan Hajnoczi826cc322021-12-07 13:23:31 +0000145 new_node->io_poll_ready = io_poll_ready;
Remy Noelfef16602018-12-20 16:20:30 +0100146 new_node->opaque = opaque;
147 new_node->is_external = is_external;
Paolo Bonzinicd9ba1e2012-09-24 14:57:22 +0200148
Remy Noelfef16602018-12-20 16:20:30 +0100149 if (is_new) {
150 new_node->pfd.fd = fd;
151 } else {
152 new_node->pfd = node->pfd;
153 }
154 g_source_add_poll(&ctx->source, &new_node->pfd);
155
156 new_node->pfd.events = (io_read ? G_IO_IN | G_IO_HUP | G_IO_ERR : 0);
157 new_node->pfd.events |= (io_write ? G_IO_OUT | G_IO_ERR : 0);
158
159 QLIST_INSERT_HEAD_RCU(&ctx->aio_handlers, new_node, node);
160 }
Paolo Bonzini7ed2b242012-09-25 10:22:39 +0200161
Paolo Bonzinid7be5dd2018-09-12 19:10:38 +0200162 /* No need to order poll_disable_cnt writes against other updates;
163 * the counter is only used to avoid wasting time and latency on
164 * iterated polling when the system call will be ultimately necessary.
165 * Changing handlers is a rare event, and a little wasted polling until
166 * the aio_notify below is not an issue.
167 */
Stefan Hajnoczid73415a2020-09-23 11:56:46 +0100168 qatomic_set(&ctx->poll_disable_cnt,
169 qatomic_read(&ctx->poll_disable_cnt) + poll_disable_change);
Paolo Bonzinid7be5dd2018-09-12 19:10:38 +0200170
Stefan Hajnoczib3210512020-03-05 17:08:03 +0000171 ctx->fdmon_ops->update(ctx, node, new_node);
Stefan Hajnoczi73fd2822020-03-05 17:08:04 +0000172 if (node) {
173 deleted = aio_remove_fd_handler(ctx, node);
174 }
Paolo Bonzini2bbf11d2017-01-12 19:07:57 +0100175 qemu_lockcnt_unlock(&ctx->list_lock);
Paolo Bonzini7ed2b242012-09-25 10:22:39 +0200176 aio_notify(ctx);
Stefan Hajnoczi4a1cba32016-12-01 19:26:42 +0000177
Fam Zheng0ed39f3d2015-11-16 14:32:14 +0800178 if (deleted) {
179 g_free(node);
180 }
aliguoria76bab42008-09-22 19:17:18 +0000181}
182
Stefan Hajnoczi684e5082016-12-01 19:26:49 +0000183void aio_set_fd_poll(AioContext *ctx, int fd,
184 IOHandler *io_poll_begin,
185 IOHandler *io_poll_end)
186{
187 AioHandler *node = find_aio_handler(ctx, fd);
188
189 if (!node) {
190 return;
191 }
192
193 node->io_poll_begin = io_poll_begin;
194 node->io_poll_end = io_poll_end;
195}
196
Paolo Bonzinia915f4b2012-09-13 12:28:51 +0200197void aio_set_event_notifier(AioContext *ctx,
198 EventNotifier *notifier,
Fam Zhengdca21ef2015-10-23 11:08:05 +0800199 bool is_external,
Stefan Hajnoczif6a51c82016-12-01 19:26:41 +0000200 EventNotifierHandler *io_read,
Stefan Hajnoczi826cc322021-12-07 13:23:31 +0000201 AioPollFn *io_poll,
202 EventNotifierHandler *io_poll_ready)
Paolo Bonzini9958c352012-06-09 03:44:00 +0200203{
Stefan Hajnoczif6a51c82016-12-01 19:26:41 +0000204 aio_set_fd_handler(ctx, event_notifier_get_fd(notifier), is_external,
Stefan Hajnoczi826cc322021-12-07 13:23:31 +0000205 (IOHandler *)io_read, NULL, io_poll,
206 (IOHandler *)io_poll_ready, notifier);
Paolo Bonzini9958c352012-06-09 03:44:00 +0200207}
208
Stefan Hajnoczi684e5082016-12-01 19:26:49 +0000209void aio_set_event_notifier_poll(AioContext *ctx,
210 EventNotifier *notifier,
211 EventNotifierHandler *io_poll_begin,
212 EventNotifierHandler *io_poll_end)
213{
214 aio_set_fd_poll(ctx, event_notifier_get_fd(notifier),
215 (IOHandler *)io_poll_begin,
216 (IOHandler *)io_poll_end);
217}
218
Stefan Hajnoczi826cc322021-12-07 13:23:31 +0000219static bool poll_set_started(AioContext *ctx, AioHandlerList *ready_list,
220 bool started)
Stefan Hajnoczi684e5082016-12-01 19:26:49 +0000221{
222 AioHandler *node;
Stefan Hajnoczie4346192020-03-05 17:08:00 +0000223 bool progress = false;
Stefan Hajnoczi684e5082016-12-01 19:26:49 +0000224
225 if (started == ctx->poll_started) {
Stefan Hajnoczie4346192020-03-05 17:08:00 +0000226 return false;
Stefan Hajnoczi684e5082016-12-01 19:26:49 +0000227 }
228
229 ctx->poll_started = started;
230
Paolo Bonzini2bbf11d2017-01-12 19:07:57 +0100231 qemu_lockcnt_inc(&ctx->list_lock);
Stefan Hajnoczid37d0e32020-03-05 17:08:06 +0000232 QLIST_FOREACH(node, &ctx->poll_aio_handlers, node_poll) {
Stefan Hajnoczi684e5082016-12-01 19:26:49 +0000233 IOHandler *fn;
234
Stefan Hajnoczi47490792020-02-14 17:17:11 +0000235 if (QLIST_IS_INSERTED(node, node_deleted)) {
Stefan Hajnoczi684e5082016-12-01 19:26:49 +0000236 continue;
237 }
238
239 if (started) {
240 fn = node->io_poll_begin;
241 } else {
242 fn = node->io_poll_end;
243 }
244
245 if (fn) {
246 fn(node->opaque);
247 }
Stefan Hajnoczie4346192020-03-05 17:08:00 +0000248
249 /* Poll one last time in case ->io_poll_end() raced with the event */
Stefan Hajnoczi826cc322021-12-07 13:23:31 +0000250 if (!started && node->io_poll(node->opaque)) {
Stefan Hajnoczifc879642022-02-23 15:57:03 +0000251 aio_add_poll_ready_handler(ready_list, node);
Stefan Hajnoczi826cc322021-12-07 13:23:31 +0000252 progress = true;
Stefan Hajnoczie4346192020-03-05 17:08:00 +0000253 }
Stefan Hajnoczi684e5082016-12-01 19:26:49 +0000254 }
Paolo Bonzini2bbf11d2017-01-12 19:07:57 +0100255 qemu_lockcnt_dec(&ctx->list_lock);
Stefan Hajnoczie4346192020-03-05 17:08:00 +0000256
257 return progress;
Stefan Hajnoczi684e5082016-12-01 19:26:49 +0000258}
259
260
Paolo Bonzinia3462c62014-07-09 11:53:08 +0200261bool aio_prepare(AioContext *ctx)
262{
Stefan Hajnoczi826cc322021-12-07 13:23:31 +0000263 AioHandlerList ready_list = QLIST_HEAD_INITIALIZER(ready_list);
264
Stefan Hajnoczi684e5082016-12-01 19:26:49 +0000265 /* Poll mode cannot be used with glib's event loop, disable it. */
Stefan Hajnoczi826cc322021-12-07 13:23:31 +0000266 poll_set_started(ctx, &ready_list, false);
267 /* TODO what to do with this list? */
Stefan Hajnoczi684e5082016-12-01 19:26:49 +0000268
Paolo Bonzinia3462c62014-07-09 11:53:08 +0200269 return false;
270}
271
Paolo Bonzinicd9ba1e2012-09-24 14:57:22 +0200272bool aio_pending(AioContext *ctx)
273{
274 AioHandler *node;
Paolo Bonzini2bbf11d2017-01-12 19:07:57 +0100275 bool result = false;
Paolo Bonzinicd9ba1e2012-09-24 14:57:22 +0200276
Paolo Bonzini2bbf11d2017-01-12 19:07:57 +0100277 /*
278 * We have to walk very carefully in case aio_set_fd_handler is
279 * called while we're walking.
280 */
281 qemu_lockcnt_inc(&ctx->list_lock);
282
283 QLIST_FOREACH_RCU(node, &ctx->aio_handlers, node) {
Paolo Bonzinicd9ba1e2012-09-24 14:57:22 +0200284 int revents;
285
Stefan Hajnoczifc879642022-02-23 15:57:03 +0000286 /* TODO should this check poll ready? */
Paolo Bonzinicd9ba1e2012-09-24 14:57:22 +0200287 revents = node->pfd.revents & node->pfd.events;
Fam Zheng37989ce2016-04-22 21:53:55 +0800288 if (revents & (G_IO_IN | G_IO_HUP | G_IO_ERR) && node->io_read &&
289 aio_node_check(ctx, node->is_external)) {
Paolo Bonzini2bbf11d2017-01-12 19:07:57 +0100290 result = true;
291 break;
Paolo Bonzinicd9ba1e2012-09-24 14:57:22 +0200292 }
Fam Zheng37989ce2016-04-22 21:53:55 +0800293 if (revents & (G_IO_OUT | G_IO_ERR) && node->io_write &&
294 aio_node_check(ctx, node->is_external)) {
Paolo Bonzini2bbf11d2017-01-12 19:07:57 +0100295 result = true;
296 break;
Paolo Bonzinicd9ba1e2012-09-24 14:57:22 +0200297 }
298 }
Paolo Bonzini2bbf11d2017-01-12 19:07:57 +0100299 qemu_lockcnt_dec(&ctx->list_lock);
Paolo Bonzinicd9ba1e2012-09-24 14:57:22 +0200300
Paolo Bonzini2bbf11d2017-01-12 19:07:57 +0100301 return result;
Paolo Bonzinicd9ba1e2012-09-24 14:57:22 +0200302}
303
Stefan Hajnoczi47490792020-02-14 17:17:11 +0000304static void aio_free_deleted_handlers(AioContext *ctx)
305{
306 AioHandler *node;
307
308 if (QLIST_EMPTY_RCU(&ctx->deleted_aio_handlers)) {
309 return;
310 }
311 if (!qemu_lockcnt_dec_if_lock(&ctx->list_lock)) {
312 return; /* we are nested, let the parent do the freeing */
313 }
314
315 while ((node = QLIST_FIRST_RCU(&ctx->deleted_aio_handlers))) {
316 QLIST_REMOVE(node, node);
317 QLIST_REMOVE(node, node_deleted);
Stefan Hajnoczid37d0e32020-03-05 17:08:06 +0000318 QLIST_SAFE_REMOVE(node, node_poll);
Stefan Hajnoczi47490792020-02-14 17:17:11 +0000319 g_free(node);
320 }
321
322 qemu_lockcnt_inc_and_unlock(&ctx->list_lock);
323}
324
Stefan Hajnoczi7391d342020-02-14 17:17:12 +0000325static bool aio_dispatch_handler(AioContext *ctx, AioHandler *node)
326{
327 bool progress = false;
Stefan Hajnoczifc879642022-02-23 15:57:03 +0000328 bool poll_ready;
Stefan Hajnoczi7391d342020-02-14 17:17:12 +0000329 int revents;
330
331 revents = node->pfd.revents & node->pfd.events;
332 node->pfd.revents = 0;
333
Stefan Hajnoczifc879642022-02-23 15:57:03 +0000334 poll_ready = node->poll_ready;
335 node->poll_ready = false;
336
Stefan Hajnoczid37d0e32020-03-05 17:08:06 +0000337 /*
338 * Start polling AioHandlers when they become ready because activity is
339 * likely to continue. Note that starvation is theoretically possible when
340 * fdmon_supports_polling(), but only until the fd fires for the first
341 * time.
342 */
343 if (!QLIST_IS_INSERTED(node, node_deleted) &&
344 !QLIST_IS_INSERTED(node, node_poll) &&
345 node->io_poll) {
346 trace_poll_add(ctx, node, node->pfd.fd, revents);
347 if (ctx->poll_started && node->io_poll_begin) {
348 node->io_poll_begin(node->opaque);
349 }
350 QLIST_INSERT_HEAD(&ctx->poll_aio_handlers, node, node_poll);
351 }
Stefan Hajnoczi826cc322021-12-07 13:23:31 +0000352 if (!QLIST_IS_INSERTED(node, node_deleted) &&
Stefan Hajnoczifc879642022-02-23 15:57:03 +0000353 poll_ready && revents == 0 &&
Stefan Hajnoczi826cc322021-12-07 13:23:31 +0000354 aio_node_check(ctx, node->is_external) &&
355 node->io_poll_ready) {
356 node->io_poll_ready(node->opaque);
357
358 /*
359 * Return early since revents was zero. aio_notify() does not count as
360 * progress.
361 */
362 return node->opaque != &ctx->notifier;
363 }
Stefan Hajnoczid37d0e32020-03-05 17:08:06 +0000364
Stefan Hajnoczi7391d342020-02-14 17:17:12 +0000365 if (!QLIST_IS_INSERTED(node, node_deleted) &&
366 (revents & (G_IO_IN | G_IO_HUP | G_IO_ERR)) &&
367 aio_node_check(ctx, node->is_external) &&
368 node->io_read) {
369 node->io_read(node->opaque);
370
371 /* aio_notify() does not count as progress */
372 if (node->opaque != &ctx->notifier) {
373 progress = true;
374 }
375 }
376 if (!QLIST_IS_INSERTED(node, node_deleted) &&
377 (revents & (G_IO_OUT | G_IO_ERR)) &&
378 aio_node_check(ctx, node->is_external) &&
379 node->io_write) {
380 node->io_write(node->opaque);
381 progress = true;
382 }
383
384 return progress;
385}
386
387/*
388 * If we have a list of ready handlers then this is more efficient than
389 * scanning all handlers with aio_dispatch_handlers().
390 */
391static bool aio_dispatch_ready_handlers(AioContext *ctx,
392 AioHandlerList *ready_list)
393{
394 bool progress = false;
395 AioHandler *node;
396
397 while ((node = QLIST_FIRST(ready_list))) {
Stefan Hajnoczic39cbed2020-02-24 10:34:06 +0000398 QLIST_REMOVE(node, node_ready);
Stefan Hajnoczi7391d342020-02-14 17:17:12 +0000399 progress = aio_dispatch_handler(ctx, node) || progress;
400 }
401
402 return progress;
403}
404
405/* Slower than aio_dispatch_ready_handlers() but only used via glib */
Paolo Bonzini56d2c3c2017-01-12 19:07:55 +0100406static bool aio_dispatch_handlers(AioContext *ctx)
aliguoria76bab42008-09-22 19:17:18 +0000407{
Paolo Bonziniabf90d32017-01-12 19:07:56 +0100408 AioHandler *node, *tmp;
Stefan Hajnoczid0c8d2c2013-02-20 11:28:31 +0100409 bool progress = false;
aliguoria76bab42008-09-22 19:17:18 +0000410
Paolo Bonzini2bbf11d2017-01-12 19:07:57 +0100411 QLIST_FOREACH_SAFE_RCU(node, &ctx->aio_handlers, node, tmp) {
Stefan Hajnoczi7391d342020-02-14 17:17:12 +0000412 progress = aio_dispatch_handler(ctx, node) || progress;
Paolo Bonzinicd9ba1e2012-09-24 14:57:22 +0200413 }
Alex Bligh438e1f42013-08-21 16:02:53 +0100414
Paolo Bonzini56d2c3c2017-01-12 19:07:55 +0100415 return progress;
416}
417
Paolo Bonzinia153bf52017-02-13 14:52:33 +0100418void aio_dispatch(AioContext *ctx)
Paolo Bonzini56d2c3c2017-01-12 19:07:55 +0100419{
Paolo Bonzinia153bf52017-02-13 14:52:33 +0100420 qemu_lockcnt_inc(&ctx->list_lock);
Paolo Bonzinibd451432017-02-13 14:52:34 +0100421 aio_bh_poll(ctx);
Paolo Bonzinia153bf52017-02-13 14:52:33 +0100422 aio_dispatch_handlers(ctx);
Stefan Hajnoczi47490792020-02-14 17:17:11 +0000423 aio_free_deleted_handlers(ctx);
Paolo Bonzinia153bf52017-02-13 14:52:33 +0100424 qemu_lockcnt_dec(&ctx->list_lock);
Paolo Bonzini56d2c3c2017-01-12 19:07:55 +0100425
Paolo Bonzinia153bf52017-02-13 14:52:33 +0100426 timerlistgroup_run_timers(&ctx->tlg);
Stefan Hajnoczid0c8d2c2013-02-20 11:28:31 +0100427}
428
Stefan Hajnoczid37d0e32020-03-05 17:08:06 +0000429static bool run_poll_handlers_once(AioContext *ctx,
Stefan Hajnoczi826cc322021-12-07 13:23:31 +0000430 AioHandlerList *ready_list,
Stefan Hajnoczid37d0e32020-03-05 17:08:06 +0000431 int64_t now,
432 int64_t *timeout)
Stefan Hajnoczi684e5082016-12-01 19:26:49 +0000433{
434 bool progress = false;
435 AioHandler *node;
Stefan Hajnoczid37d0e32020-03-05 17:08:06 +0000436 AioHandler *tmp;
Stefan Hajnoczi684e5082016-12-01 19:26:49 +0000437
Stefan Hajnoczid37d0e32020-03-05 17:08:06 +0000438 QLIST_FOREACH_SAFE(node, &ctx->poll_aio_handlers, node_poll, tmp) {
439 if (aio_node_check(ctx, node->is_external) &&
Paolo Bonzinicfeb35d2018-09-12 19:10:40 +0200440 node->io_poll(node->opaque)) {
Stefan Hajnoczifc879642022-02-23 15:57:03 +0000441 aio_add_poll_ready_handler(ready_list, node);
Stefan Hajnoczi826cc322021-12-07 13:23:31 +0000442
Stefan Hajnoczid37d0e32020-03-05 17:08:06 +0000443 node->poll_idle_timeout = now + POLL_IDLE_INTERVAL_NS;
444
Paolo Bonzini993ed892019-04-09 14:28:23 +0200445 /*
446 * Polling was successful, exit try_poll_mode immediately
447 * to adjust the next polling time.
448 */
Paolo Bonzinie30cffa2018-09-12 19:10:39 +0200449 *timeout = 0;
Paolo Bonzinicfeb35d2018-09-12 19:10:40 +0200450 if (node->opaque != &ctx->notifier) {
451 progress = true;
452 }
Stefan Hajnoczi684e5082016-12-01 19:26:49 +0000453 }
454
455 /* Caller handles freeing deleted nodes. Don't do it here. */
456 }
457
458 return progress;
459}
460
Stefan Hajnoczid37d0e32020-03-05 17:08:06 +0000461static bool fdmon_supports_polling(AioContext *ctx)
462{
463 return ctx->fdmon_ops->need_wait != aio_poll_disabled;
464}
465
Stefan Hajnoczi826cc322021-12-07 13:23:31 +0000466static bool remove_idle_poll_handlers(AioContext *ctx,
467 AioHandlerList *ready_list,
468 int64_t now)
Stefan Hajnoczid37d0e32020-03-05 17:08:06 +0000469{
470 AioHandler *node;
471 AioHandler *tmp;
472 bool progress = false;
473
474 /*
475 * File descriptor monitoring implementations without userspace polling
476 * support suffer from starvation when a subset of handlers is polled
477 * because fds will not be processed in a timely fashion. Don't remove
478 * idle poll handlers.
479 */
480 if (!fdmon_supports_polling(ctx)) {
481 return false;
482 }
483
484 QLIST_FOREACH_SAFE(node, &ctx->poll_aio_handlers, node_poll, tmp) {
485 if (node->poll_idle_timeout == 0LL) {
486 node->poll_idle_timeout = now + POLL_IDLE_INTERVAL_NS;
487 } else if (now >= node->poll_idle_timeout) {
488 trace_poll_remove(ctx, node, node->pfd.fd);
489 node->poll_idle_timeout = 0LL;
490 QLIST_SAFE_REMOVE(node, node_poll);
491 if (ctx->poll_started && node->io_poll_end) {
492 node->io_poll_end(node->opaque);
493
494 /*
495 * Final poll in case ->io_poll_end() races with an event.
496 * Nevermind about re-adding the handler in the rare case where
497 * this causes progress.
498 */
Stefan Hajnoczi826cc322021-12-07 13:23:31 +0000499 if (node->io_poll(node->opaque)) {
Stefan Hajnoczifc879642022-02-23 15:57:03 +0000500 aio_add_poll_ready_handler(ready_list, node);
Stefan Hajnoczi826cc322021-12-07 13:23:31 +0000501 progress = true;
502 }
Stefan Hajnoczid37d0e32020-03-05 17:08:06 +0000503 }
504 }
505 }
506
507 return progress;
508}
509
Stefan Hajnoczi4a1cba32016-12-01 19:26:42 +0000510/* run_poll_handlers:
511 * @ctx: the AioContext
Stefan Hajnoczi826cc322021-12-07 13:23:31 +0000512 * @ready_list: the list to place ready handlers on
Stefan Hajnoczi4a1cba32016-12-01 19:26:42 +0000513 * @max_ns: maximum time to poll for, in nanoseconds
514 *
515 * Polls for a given time.
516 *
Paolo Bonzini2bbf11d2017-01-12 19:07:57 +0100517 * Note that the caller must have incremented ctx->list_lock.
Stefan Hajnoczi4a1cba32016-12-01 19:26:42 +0000518 *
519 * Returns: true if progress was made, false otherwise
520 */
Stefan Hajnoczi826cc322021-12-07 13:23:31 +0000521static bool run_poll_handlers(AioContext *ctx, AioHandlerList *ready_list,
522 int64_t max_ns, int64_t *timeout)
Stefan Hajnoczi4a1cba32016-12-01 19:26:42 +0000523{
Stefan Hajnoczi684e5082016-12-01 19:26:49 +0000524 bool progress;
Paolo Bonzinie30cffa2018-09-12 19:10:39 +0200525 int64_t start_time, elapsed_time;
Stefan Hajnoczi4a1cba32016-12-01 19:26:42 +0000526
Paolo Bonzini2bbf11d2017-01-12 19:07:57 +0100527 assert(qemu_lockcnt_count(&ctx->list_lock) > 0);
Stefan Hajnoczi4a1cba32016-12-01 19:26:42 +0000528
Paolo Bonzinie30cffa2018-09-12 19:10:39 +0200529 trace_run_poll_handlers_begin(ctx, max_ns, *timeout);
Stefan Hajnoczi4a1cba32016-12-01 19:26:42 +0000530
Stefan Hajnoczi3aa221b2020-03-05 17:08:01 +0000531 /*
532 * Optimization: ->io_poll() handlers often contain RCU read critical
533 * sections and we therefore see many rcu_read_lock() -> rcu_read_unlock()
534 * -> rcu_read_lock() -> ... sequences with expensive memory
535 * synchronization primitives. Make the entire polling loop an RCU
536 * critical section because nested rcu_read_lock()/rcu_read_unlock() calls
537 * are cheap.
538 */
539 RCU_READ_LOCK_GUARD();
540
Paolo Bonzinie30cffa2018-09-12 19:10:39 +0200541 start_time = qemu_clock_get_ns(QEMU_CLOCK_REALTIME);
Stefan Hajnoczi4a1cba32016-12-01 19:26:42 +0000542 do {
Stefan Hajnoczi826cc322021-12-07 13:23:31 +0000543 progress = run_poll_handlers_once(ctx, ready_list,
544 start_time, timeout);
Paolo Bonzinie30cffa2018-09-12 19:10:39 +0200545 elapsed_time = qemu_clock_get_ns(QEMU_CLOCK_REALTIME) - start_time;
Paolo Bonzini993ed892019-04-09 14:28:23 +0200546 max_ns = qemu_soonest_timeout(*timeout, max_ns);
547 assert(!(max_ns && progress));
Stefan Hajnocziaa38e192020-03-05 17:08:05 +0000548 } while (elapsed_time < max_ns && !ctx->fdmon_ops->need_wait(ctx));
Stefan Hajnoczi4a1cba32016-12-01 19:26:42 +0000549
Stefan Hajnoczi826cc322021-12-07 13:23:31 +0000550 if (remove_idle_poll_handlers(ctx, ready_list,
551 start_time + elapsed_time)) {
Stefan Hajnoczid37d0e32020-03-05 17:08:06 +0000552 *timeout = 0;
553 progress = true;
554 }
555
Paolo Bonzinie30cffa2018-09-12 19:10:39 +0200556 /* If time has passed with no successful polling, adjust *timeout to
557 * keep the same ending time.
558 */
559 if (*timeout != -1) {
560 *timeout -= MIN(*timeout, elapsed_time);
561 }
Stefan Hajnoczi4a1cba32016-12-01 19:26:42 +0000562
Paolo Bonzinie30cffa2018-09-12 19:10:39 +0200563 trace_run_poll_handlers_end(ctx, progress, *timeout);
Stefan Hajnoczi4a1cba32016-12-01 19:26:42 +0000564 return progress;
565}
566
567/* try_poll_mode:
568 * @ctx: the AioContext
Stefan Hajnoczi826cc322021-12-07 13:23:31 +0000569 * @ready_list: list to add handlers that need to be run
Paolo Bonzinie30cffa2018-09-12 19:10:39 +0200570 * @timeout: timeout for blocking wait, computed by the caller and updated if
571 * polling succeeds.
Stefan Hajnoczi4a1cba32016-12-01 19:26:42 +0000572 *
Paolo Bonzini2bbf11d2017-01-12 19:07:57 +0100573 * Note that the caller must have incremented ctx->list_lock.
Stefan Hajnoczi4a1cba32016-12-01 19:26:42 +0000574 *
575 * Returns: true if progress was made, false otherwise
576 */
Stefan Hajnoczi826cc322021-12-07 13:23:31 +0000577static bool try_poll_mode(AioContext *ctx, AioHandlerList *ready_list,
578 int64_t *timeout)
Stefan Hajnoczi4a1cba32016-12-01 19:26:42 +0000579{
Stefan Hajnoczid37d0e32020-03-05 17:08:06 +0000580 int64_t max_ns;
Stefan Hajnoczi4a1cba32016-12-01 19:26:42 +0000581
Stefan Hajnoczid37d0e32020-03-05 17:08:06 +0000582 if (QLIST_EMPTY_RCU(&ctx->poll_aio_handlers)) {
583 return false;
584 }
585
586 max_ns = qemu_soonest_timeout(*timeout, ctx->poll_ns);
Stefan Hajnocziaa38e192020-03-05 17:08:05 +0000587 if (max_ns && !ctx->fdmon_ops->need_wait(ctx)) {
Stefan Hajnoczi826cc322021-12-07 13:23:31 +0000588 poll_set_started(ctx, ready_list, true);
Stefan Hajnoczi684e5082016-12-01 19:26:49 +0000589
Stefan Hajnoczi826cc322021-12-07 13:23:31 +0000590 if (run_poll_handlers(ctx, ready_list, max_ns, timeout)) {
Paolo Bonzinie30cffa2018-09-12 19:10:39 +0200591 return true;
Stefan Hajnoczi4a1cba32016-12-01 19:26:42 +0000592 }
593 }
594
Stefan Hajnoczi826cc322021-12-07 13:23:31 +0000595 if (poll_set_started(ctx, ready_list, false)) {
Stefan Hajnoczie4346192020-03-05 17:08:00 +0000596 *timeout = 0;
597 return true;
598 }
Stefan Hajnoczi684e5082016-12-01 19:26:49 +0000599
Stefan Hajnoczie4346192020-03-05 17:08:00 +0000600 return false;
Stefan Hajnoczi4a1cba32016-12-01 19:26:42 +0000601}
602
Stefan Hajnoczid0c8d2c2013-02-20 11:28:31 +0100603bool aio_poll(AioContext *ctx, bool blocking)
604{
Stefan Hajnoczi7391d342020-02-14 17:17:12 +0000605 AioHandlerList ready_list = QLIST_HEAD_INITIALIZER(ready_list);
Stefan Hajnoczi164a1012013-04-11 16:56:50 +0200606 bool progress;
Stefan Hajnoczi44277bf2020-08-06 14:18:02 +0100607 bool use_notify_me;
Paolo Bonzinie98ab092015-02-20 17:26:50 +0100608 int64_t timeout;
Stefan Hajnoczi82a41182016-12-01 19:26:51 +0000609 int64_t start = 0;
Stefan Hajnoczid0c8d2c2013-02-20 11:28:31 +0100610
Paolo Bonzini5710a3e2020-04-07 10:07:46 -0400611 /*
612 * There cannot be two concurrent aio_poll calls for the same AioContext (or
613 * an aio_poll concurrent with a GSource prepare/check/dispatch callback).
614 * We rely on this below to avoid slow locked accesses to ctx->notify_me.
Kevin Wolf9ce44e22020-10-05 17:58:50 +0200615 *
616 * aio_poll() may only be called in the AioContext's thread. iohandler_ctx
617 * is special in that it runs in the main thread, but that thread's context
618 * is qemu_aio_context.
Paolo Bonzini5710a3e2020-04-07 10:07:46 -0400619 */
Kevin Wolf9ce44e22020-10-05 17:58:50 +0200620 assert(in_aio_context_home_thread(ctx == iohandler_get_aio_context() ?
621 qemu_get_aio_context() : ctx));
Kevin Wolf0dc165c2019-02-14 13:13:36 +0100622
Paolo Bonzini2bbf11d2017-01-12 19:07:57 +0100623 qemu_lockcnt_inc(&ctx->list_lock);
aliguoria76bab42008-09-22 19:17:18 +0000624
Stefan Hajnoczi82a41182016-12-01 19:26:51 +0000625 if (ctx->poll_max_ns) {
626 start = qemu_clock_get_ns(QEMU_CLOCK_REALTIME);
627 }
628
Paolo Bonzinie30cffa2018-09-12 19:10:39 +0200629 timeout = blocking ? aio_compute_timeout(ctx) : 0;
Stefan Hajnoczi826cc322021-12-07 13:23:31 +0000630 progress = try_poll_mode(ctx, &ready_list, &timeout);
Paolo Bonzinie30cffa2018-09-12 19:10:39 +0200631 assert(!(timeout && progress));
632
Stefan Hajnoczi44277bf2020-08-06 14:18:02 +0100633 /*
634 * aio_notify can avoid the expensive event_notifier_set if
635 * everything (file descriptors, bottom halves, timers) will
636 * be re-evaluated before the next blocking poll(). This is
637 * already true when aio_poll is called with blocking == false;
638 * if blocking == true, it is only true after poll() returns,
639 * so disable the optimization now.
640 */
641 use_notify_me = timeout != 0;
642 if (use_notify_me) {
Stefan Hajnoczid73415a2020-09-23 11:56:46 +0100643 qatomic_set(&ctx->notify_me, qatomic_read(&ctx->notify_me) + 2);
Stefan Hajnoczi44277bf2020-08-06 14:18:02 +0100644 /*
645 * Write ctx->notify_me before reading ctx->notified. Pairs with
646 * smp_mb in aio_notify().
647 */
648 smp_mb();
649
650 /* Don't block if aio_notify() was called */
Stefan Hajnoczid73415a2020-09-23 11:56:46 +0100651 if (qatomic_read(&ctx->notified)) {
Stefan Hajnoczi44277bf2020-08-06 14:18:02 +0100652 timeout = 0;
653 }
654 }
655
Paolo Bonzinie30cffa2018-09-12 19:10:39 +0200656 /* If polling is allowed, non-blocking aio_poll does not need the
657 * system call---a single round of run_poll_handlers_once suffices.
658 */
Stefan Hajnocziaa38e192020-03-05 17:08:05 +0000659 if (timeout || ctx->fdmon_ops->need_wait(ctx)) {
Stefan Hajnoczi826cc322021-12-07 13:23:31 +0000660 ctx->fdmon_ops->wait(ctx, &ready_list, timeout);
Paolo Bonzini9eb0bfc2012-04-12 14:00:56 +0200661 }
662
Stefan Hajnoczi44277bf2020-08-06 14:18:02 +0100663 if (use_notify_me) {
Paolo Bonzini5710a3e2020-04-07 10:07:46 -0400664 /* Finish the poll before clearing the flag. */
Stefan Hajnoczid73415a2020-09-23 11:56:46 +0100665 qatomic_store_release(&ctx->notify_me,
666 qatomic_read(&ctx->notify_me) - 2);
Paolo Bonzinieabc9772015-07-21 16:07:51 +0200667 }
Paolo Bonzini9eb0bfc2012-04-12 14:00:56 +0200668
Stefan Hajnoczi44277bf2020-08-06 14:18:02 +0100669 aio_notify_accept(ctx);
670
Stefan Hajnoczi82a41182016-12-01 19:26:51 +0000671 /* Adjust polling time */
672 if (ctx->poll_max_ns) {
673 int64_t block_ns = qemu_clock_get_ns(QEMU_CLOCK_REALTIME) - start;
674
675 if (block_ns <= ctx->poll_ns) {
676 /* This is the sweet spot, no adjustment needed */
677 } else if (block_ns > ctx->poll_max_ns) {
678 /* We'd have to poll for too long, poll less */
679 int64_t old = ctx->poll_ns;
680
681 if (ctx->poll_shrink) {
682 ctx->poll_ns /= ctx->poll_shrink;
683 } else {
684 ctx->poll_ns = 0;
685 }
686
687 trace_poll_shrink(ctx, old, ctx->poll_ns);
688 } else if (ctx->poll_ns < ctx->poll_max_ns &&
689 block_ns < ctx->poll_max_ns) {
690 /* There is room to grow, poll longer */
691 int64_t old = ctx->poll_ns;
692 int64_t grow = ctx->poll_grow;
693
694 if (grow == 0) {
695 grow = 2;
696 }
697
698 if (ctx->poll_ns) {
699 ctx->poll_ns *= grow;
700 } else {
701 ctx->poll_ns = 4000; /* start polling at 4 microseconds */
702 }
703
704 if (ctx->poll_ns > ctx->poll_max_ns) {
705 ctx->poll_ns = ctx->poll_max_ns;
706 }
707
708 trace_poll_grow(ctx, old, ctx->poll_ns);
709 }
710 }
711
Paolo Bonzinia153bf52017-02-13 14:52:33 +0100712 progress |= aio_bh_poll(ctx);
Stefan Hajnoczi826cc322021-12-07 13:23:31 +0000713 progress |= aio_dispatch_ready_handlers(ctx, &ready_list);
Paolo Bonzinibcdc1852012-04-12 14:00:55 +0200714
Stefan Hajnoczi47490792020-02-14 17:17:11 +0000715 aio_free_deleted_handlers(ctx);
716
Paolo Bonzinibd451432017-02-13 14:52:34 +0100717 qemu_lockcnt_dec(&ctx->list_lock);
718
Paolo Bonzinia153bf52017-02-13 14:52:33 +0100719 progress |= timerlistgroup_run_timers(&ctx->tlg);
720
Stefan Hajnoczi164a1012013-04-11 16:56:50 +0200721 return progress;
aliguoria76bab42008-09-22 19:17:18 +0000722}
Fam Zheng37fcee52015-10-30 12:06:28 +0800723
Cao jin7e003462016-07-15 18:28:44 +0800724void aio_context_setup(AioContext *ctx)
Fam Zheng37fcee52015-10-30 12:06:28 +0800725{
Stefan Hajnoczi1f050a42020-03-05 17:08:02 +0000726 ctx->fdmon_ops = &fdmon_poll_ops;
727 ctx->epollfd = -1;
728
Stefan Hajnoczi73fd2822020-03-05 17:08:04 +0000729 /* Use the fastest fd monitoring implementation if available */
730 if (fdmon_io_uring_setup(ctx)) {
731 return;
732 }
733
Stefan Hajnoczi1f050a42020-03-05 17:08:02 +0000734 fdmon_epoll_setup(ctx);
Fam Zheng37fcee52015-10-30 12:06:28 +0800735}
Stefan Hajnoczi4a1cba32016-12-01 19:26:42 +0000736
Jie Wangcd0a6d22018-05-17 08:42:43 +0800737void aio_context_destroy(AioContext *ctx)
738{
Stefan Hajnoczi73fd2822020-03-05 17:08:04 +0000739 fdmon_io_uring_destroy(ctx);
Stefan Hajnoczi1f050a42020-03-05 17:08:02 +0000740 fdmon_epoll_disable(ctx);
Stefan Hajnoczide137e42020-05-11 19:36:29 +0100741 aio_free_deleted_handlers(ctx);
Jie Wangcd0a6d22018-05-17 08:42:43 +0800742}
743
Stefan Hajnocziba607ca2020-05-11 19:36:30 +0100744void aio_context_use_g_source(AioContext *ctx)
745{
746 /*
747 * Disable io_uring when the glib main loop is used because it doesn't
748 * support mixed glib/aio_poll() usage. It relies on aio_poll() being
749 * called regularly so that changes to the monitored file descriptors are
750 * submitted, otherwise a list of pending fd handlers builds up.
751 */
752 fdmon_io_uring_destroy(ctx);
753 aio_free_deleted_handlers(ctx);
754}
755
Stefan Hajnoczi82a41182016-12-01 19:26:51 +0000756void aio_context_set_poll_params(AioContext *ctx, int64_t max_ns,
757 int64_t grow, int64_t shrink, Error **errp)
Stefan Hajnoczi4a1cba32016-12-01 19:26:42 +0000758{
Stefan Hajnoczi82a41182016-12-01 19:26:51 +0000759 /* No thread synchronization here, it doesn't matter if an incorrect value
760 * is used once.
Stefan Hajnoczi4a1cba32016-12-01 19:26:42 +0000761 */
762 ctx->poll_max_ns = max_ns;
Stefan Hajnoczi82a41182016-12-01 19:26:51 +0000763 ctx->poll_ns = 0;
764 ctx->poll_grow = grow;
765 ctx->poll_shrink = shrink;
Stefan Hajnoczi4a1cba32016-12-01 19:26:42 +0000766
767 aio_notify(ctx);
768}
Stefano Garzarella1793ad02021-07-21 11:42:10 +0200769
770void aio_context_set_aio_params(AioContext *ctx, int64_t max_batch,
771 Error **errp)
772{
773 /*
774 * No thread synchronization here, it doesn't matter if an incorrect value
775 * is used once.
776 */
777 ctx->aio_max_batch = max_batch;
778
779 aio_notify(ctx);
780}