blob: 280f27bb99f72923a200d1784b59f931babac082 [file] [log] [blame]
aliguoria76bab42008-09-22 19:17:18 +00001/*
2 * QEMU aio implementation
3 *
4 * Copyright IBM, Corp. 2008
5 *
6 * Authors:
7 * Anthony Liguori <aliguori@us.ibm.com>
8 *
9 * This work is licensed under the terms of the GNU GPL, version 2. See
10 * the COPYING file in the top-level directory.
11 *
Paolo Bonzini6b620ca2012-01-13 17:44:23 +010012 * Contributions after 2012-01-13 are licensed under the terms of the
13 * GNU GPL, version 2 or (at your option) any later version.
aliguoria76bab42008-09-22 19:17:18 +000014 */
15
Peter Maydelld38ea872016-01-29 17:50:05 +000016#include "qemu/osdep.h"
Paolo Bonzini737e1502012-12-17 18:19:44 +010017#include "block/block.h"
Stefan Hajnoczif25c0b52020-02-18 18:27:08 +000018#include "qemu/rcu.h"
Paolo Bonzini2bbf11d2017-01-12 19:07:57 +010019#include "qemu/rcu_queue.h"
Paolo Bonzini1de7afc2012-12-17 18:20:00 +010020#include "qemu/sockets.h"
Stefan Hajnoczi4a1cba32016-12-01 19:26:42 +000021#include "qemu/cutils.h"
Paolo Bonzinic2b38b22017-02-13 14:52:18 +010022#include "trace.h"
Stefan Hajnoczi1f050a42020-03-05 17:08:02 +000023#include "aio-posix.h"
aliguoria76bab42008-09-22 19:17:18 +000024
Stefan Hajnoczid37d0e32020-03-05 17:08:06 +000025/* Stop userspace polling on a handler if it isn't active for some time */
26#define POLL_IDLE_INTERVAL_NS (7 * NANOSECONDS_PER_SECOND)
27
Stefan Hajnocziaa38e192020-03-05 17:08:05 +000028bool aio_poll_disabled(AioContext *ctx)
29{
Stefan Hajnoczid73415a2020-09-23 11:56:46 +010030 return qatomic_read(&ctx->poll_disable_cnt);
Stefan Hajnocziaa38e192020-03-05 17:08:05 +000031}
32
Stefan Hajnoczi1f050a42020-03-05 17:08:02 +000033void aio_add_ready_handler(AioHandlerList *ready_list,
34 AioHandler *node,
35 int revents)
Stefan Hajnoczi7391d342020-02-14 17:17:12 +000036{
37 QLIST_SAFE_REMOVE(node, node_ready); /* remove from nested parent's list */
38 node->pfd.revents = revents;
39 QLIST_INSERT_HEAD(ready_list, node, node_ready);
40}
41
Paolo Bonzinia915f4b2012-09-13 12:28:51 +020042static AioHandler *find_aio_handler(AioContext *ctx, int fd)
aliguoria76bab42008-09-22 19:17:18 +000043{
44 AioHandler *node;
45
Paolo Bonzinia915f4b2012-09-13 12:28:51 +020046 QLIST_FOREACH(node, &ctx->aio_handlers, node) {
Stefan Hajnoczi47490792020-02-14 17:17:11 +000047 if (node->pfd.fd == fd) {
48 if (!QLIST_IS_INSERTED(node, node_deleted)) {
Alexander Graf79d5ca52009-05-06 02:58:48 +020049 return node;
Stefan Hajnoczi47490792020-02-14 17:17:11 +000050 }
51 }
aliguoria76bab42008-09-22 19:17:18 +000052 }
53
54 return NULL;
55}
56
Remy Noelfef16602018-12-20 16:20:30 +010057static bool aio_remove_fd_handler(AioContext *ctx, AioHandler *node)
58{
59 /* If the GSource is in the process of being destroyed then
60 * g_source_remove_poll() causes an assertion failure. Skip
61 * removal in that case, because glib cleans up its state during
62 * destruction anyway.
63 */
64 if (!g_source_is_destroyed(&ctx->source)) {
65 g_source_remove_poll(&ctx->source, &node->pfd);
66 }
67
Stefan Hajnoczi73fd2822020-03-05 17:08:04 +000068 node->pfd.revents = 0;
69
70 /* If the fd monitor has already marked it deleted, leave it alone */
71 if (QLIST_IS_INSERTED(node, node_deleted)) {
72 return false;
73 }
74
Remy Noelfef16602018-12-20 16:20:30 +010075 /* If a read is in progress, just mark the node as deleted */
76 if (qemu_lockcnt_count(&ctx->list_lock)) {
Stefan Hajnoczi47490792020-02-14 17:17:11 +000077 QLIST_INSERT_HEAD_RCU(&ctx->deleted_aio_handlers, node, node_deleted);
Remy Noelfef16602018-12-20 16:20:30 +010078 return false;
79 }
80 /* Otherwise, delete it for real. We can't just mark it as
81 * deleted because deleted nodes are only cleaned up while
82 * no one is walking the handlers list.
83 */
Stefan Hajnoczid37d0e32020-03-05 17:08:06 +000084 QLIST_SAFE_REMOVE(node, node_poll);
Remy Noelfef16602018-12-20 16:20:30 +010085 QLIST_REMOVE(node, node);
86 return true;
87}
88
Paolo Bonzinia915f4b2012-09-13 12:28:51 +020089void aio_set_fd_handler(AioContext *ctx,
90 int fd,
Fam Zhengdca21ef2015-10-23 11:08:05 +080091 bool is_external,
Paolo Bonzinia915f4b2012-09-13 12:28:51 +020092 IOHandler *io_read,
93 IOHandler *io_write,
Stefan Hajnoczif6a51c82016-12-01 19:26:41 +000094 AioPollFn *io_poll,
Paolo Bonzinia915f4b2012-09-13 12:28:51 +020095 void *opaque)
aliguoria76bab42008-09-22 19:17:18 +000096{
97 AioHandler *node;
Remy Noelfef16602018-12-20 16:20:30 +010098 AioHandler *new_node = NULL;
Fam Zhengfbe3fc52015-10-30 12:06:29 +080099 bool is_new = false;
Fam Zheng0ed39f3d2015-11-16 14:32:14 +0800100 bool deleted = false;
Paolo Bonzinid7be5dd2018-09-12 19:10:38 +0200101 int poll_disable_change;
aliguoria76bab42008-09-22 19:17:18 +0000102
Paolo Bonzini2bbf11d2017-01-12 19:07:57 +0100103 qemu_lockcnt_lock(&ctx->list_lock);
104
Paolo Bonzinia915f4b2012-09-13 12:28:51 +0200105 node = find_aio_handler(ctx, fd);
aliguoria76bab42008-09-22 19:17:18 +0000106
107 /* Are we deleting the fd handler? */
Stefan Hajnoczi4a1cba32016-12-01 19:26:42 +0000108 if (!io_read && !io_write && !io_poll) {
Paolo Bonzini36173ec2016-11-08 14:55:23 +0100109 if (node == NULL) {
Paolo Bonzini2bbf11d2017-01-12 19:07:57 +0100110 qemu_lockcnt_unlock(&ctx->list_lock);
Paolo Bonzini36173ec2016-11-08 14:55:23 +0100111 return;
112 }
Remy Noel8821b342018-12-20 16:20:29 +0100113 /* Clean events in order to unregister fd from the ctx epoll. */
114 node->pfd.events = 0;
115
Paolo Bonzinid7be5dd2018-09-12 19:10:38 +0200116 poll_disable_change = -!node->io_poll;
aliguoria76bab42008-09-22 19:17:18 +0000117 } else {
Paolo Bonzinid7be5dd2018-09-12 19:10:38 +0200118 poll_disable_change = !io_poll - (node && !node->io_poll);
aliguoria76bab42008-09-22 19:17:18 +0000119 if (node == NULL) {
Fam Zhengfbe3fc52015-10-30 12:06:29 +0800120 is_new = true;
aliguoria76bab42008-09-22 19:17:18 +0000121 }
Remy Noelfef16602018-12-20 16:20:30 +0100122 /* Alloc and insert if it's not already there */
123 new_node = g_new0(AioHandler, 1);
Stefan Hajnoczi4a1cba32016-12-01 19:26:42 +0000124
aliguoria76bab42008-09-22 19:17:18 +0000125 /* Update handler with latest information */
Remy Noelfef16602018-12-20 16:20:30 +0100126 new_node->io_read = io_read;
127 new_node->io_write = io_write;
128 new_node->io_poll = io_poll;
129 new_node->opaque = opaque;
130 new_node->is_external = is_external;
Paolo Bonzinicd9ba1e2012-09-24 14:57:22 +0200131
Remy Noelfef16602018-12-20 16:20:30 +0100132 if (is_new) {
133 new_node->pfd.fd = fd;
134 } else {
135 new_node->pfd = node->pfd;
136 }
137 g_source_add_poll(&ctx->source, &new_node->pfd);
138
139 new_node->pfd.events = (io_read ? G_IO_IN | G_IO_HUP | G_IO_ERR : 0);
140 new_node->pfd.events |= (io_write ? G_IO_OUT | G_IO_ERR : 0);
141
142 QLIST_INSERT_HEAD_RCU(&ctx->aio_handlers, new_node, node);
143 }
Paolo Bonzini7ed2b242012-09-25 10:22:39 +0200144
Paolo Bonzinid7be5dd2018-09-12 19:10:38 +0200145 /* No need to order poll_disable_cnt writes against other updates;
146 * the counter is only used to avoid wasting time and latency on
147 * iterated polling when the system call will be ultimately necessary.
148 * Changing handlers is a rare event, and a little wasted polling until
149 * the aio_notify below is not an issue.
150 */
Stefan Hajnoczid73415a2020-09-23 11:56:46 +0100151 qatomic_set(&ctx->poll_disable_cnt,
152 qatomic_read(&ctx->poll_disable_cnt) + poll_disable_change);
Paolo Bonzinid7be5dd2018-09-12 19:10:38 +0200153
Stefan Hajnoczib3210512020-03-05 17:08:03 +0000154 ctx->fdmon_ops->update(ctx, node, new_node);
Stefan Hajnoczi73fd2822020-03-05 17:08:04 +0000155 if (node) {
156 deleted = aio_remove_fd_handler(ctx, node);
157 }
Paolo Bonzini2bbf11d2017-01-12 19:07:57 +0100158 qemu_lockcnt_unlock(&ctx->list_lock);
Paolo Bonzini7ed2b242012-09-25 10:22:39 +0200159 aio_notify(ctx);
Stefan Hajnoczi4a1cba32016-12-01 19:26:42 +0000160
Fam Zheng0ed39f3d2015-11-16 14:32:14 +0800161 if (deleted) {
162 g_free(node);
163 }
aliguoria76bab42008-09-22 19:17:18 +0000164}
165
Stefan Hajnoczi684e5082016-12-01 19:26:49 +0000166void aio_set_fd_poll(AioContext *ctx, int fd,
167 IOHandler *io_poll_begin,
168 IOHandler *io_poll_end)
169{
170 AioHandler *node = find_aio_handler(ctx, fd);
171
172 if (!node) {
173 return;
174 }
175
176 node->io_poll_begin = io_poll_begin;
177 node->io_poll_end = io_poll_end;
178}
179
Paolo Bonzinia915f4b2012-09-13 12:28:51 +0200180void aio_set_event_notifier(AioContext *ctx,
181 EventNotifier *notifier,
Fam Zhengdca21ef2015-10-23 11:08:05 +0800182 bool is_external,
Stefan Hajnoczif6a51c82016-12-01 19:26:41 +0000183 EventNotifierHandler *io_read,
184 AioPollFn *io_poll)
Paolo Bonzini9958c352012-06-09 03:44:00 +0200185{
Stefan Hajnoczif6a51c82016-12-01 19:26:41 +0000186 aio_set_fd_handler(ctx, event_notifier_get_fd(notifier), is_external,
187 (IOHandler *)io_read, NULL, io_poll, notifier);
Paolo Bonzini9958c352012-06-09 03:44:00 +0200188}
189
Stefan Hajnoczi684e5082016-12-01 19:26:49 +0000190void aio_set_event_notifier_poll(AioContext *ctx,
191 EventNotifier *notifier,
192 EventNotifierHandler *io_poll_begin,
193 EventNotifierHandler *io_poll_end)
194{
195 aio_set_fd_poll(ctx, event_notifier_get_fd(notifier),
196 (IOHandler *)io_poll_begin,
197 (IOHandler *)io_poll_end);
198}
199
Stefan Hajnoczie4346192020-03-05 17:08:00 +0000200static bool poll_set_started(AioContext *ctx, bool started)
Stefan Hajnoczi684e5082016-12-01 19:26:49 +0000201{
202 AioHandler *node;
Stefan Hajnoczie4346192020-03-05 17:08:00 +0000203 bool progress = false;
Stefan Hajnoczi684e5082016-12-01 19:26:49 +0000204
205 if (started == ctx->poll_started) {
Stefan Hajnoczie4346192020-03-05 17:08:00 +0000206 return false;
Stefan Hajnoczi684e5082016-12-01 19:26:49 +0000207 }
208
209 ctx->poll_started = started;
210
Paolo Bonzini2bbf11d2017-01-12 19:07:57 +0100211 qemu_lockcnt_inc(&ctx->list_lock);
Stefan Hajnoczid37d0e32020-03-05 17:08:06 +0000212 QLIST_FOREACH(node, &ctx->poll_aio_handlers, node_poll) {
Stefan Hajnoczi684e5082016-12-01 19:26:49 +0000213 IOHandler *fn;
214
Stefan Hajnoczi47490792020-02-14 17:17:11 +0000215 if (QLIST_IS_INSERTED(node, node_deleted)) {
Stefan Hajnoczi684e5082016-12-01 19:26:49 +0000216 continue;
217 }
218
219 if (started) {
220 fn = node->io_poll_begin;
221 } else {
222 fn = node->io_poll_end;
223 }
224
225 if (fn) {
226 fn(node->opaque);
227 }
Stefan Hajnoczie4346192020-03-05 17:08:00 +0000228
229 /* Poll one last time in case ->io_poll_end() raced with the event */
230 if (!started) {
231 progress = node->io_poll(node->opaque) || progress;
232 }
Stefan Hajnoczi684e5082016-12-01 19:26:49 +0000233 }
Paolo Bonzini2bbf11d2017-01-12 19:07:57 +0100234 qemu_lockcnt_dec(&ctx->list_lock);
Stefan Hajnoczie4346192020-03-05 17:08:00 +0000235
236 return progress;
Stefan Hajnoczi684e5082016-12-01 19:26:49 +0000237}
238
239
Paolo Bonzinia3462c62014-07-09 11:53:08 +0200240bool aio_prepare(AioContext *ctx)
241{
Stefan Hajnoczi684e5082016-12-01 19:26:49 +0000242 /* Poll mode cannot be used with glib's event loop, disable it. */
243 poll_set_started(ctx, false);
244
Paolo Bonzinia3462c62014-07-09 11:53:08 +0200245 return false;
246}
247
Paolo Bonzinicd9ba1e2012-09-24 14:57:22 +0200248bool aio_pending(AioContext *ctx)
249{
250 AioHandler *node;
Paolo Bonzini2bbf11d2017-01-12 19:07:57 +0100251 bool result = false;
Paolo Bonzinicd9ba1e2012-09-24 14:57:22 +0200252
Paolo Bonzini2bbf11d2017-01-12 19:07:57 +0100253 /*
254 * We have to walk very carefully in case aio_set_fd_handler is
255 * called while we're walking.
256 */
257 qemu_lockcnt_inc(&ctx->list_lock);
258
259 QLIST_FOREACH_RCU(node, &ctx->aio_handlers, node) {
Paolo Bonzinicd9ba1e2012-09-24 14:57:22 +0200260 int revents;
261
Paolo Bonzinicd9ba1e2012-09-24 14:57:22 +0200262 revents = node->pfd.revents & node->pfd.events;
Fam Zheng37989ce2016-04-22 21:53:55 +0800263 if (revents & (G_IO_IN | G_IO_HUP | G_IO_ERR) && node->io_read &&
264 aio_node_check(ctx, node->is_external)) {
Paolo Bonzini2bbf11d2017-01-12 19:07:57 +0100265 result = true;
266 break;
Paolo Bonzinicd9ba1e2012-09-24 14:57:22 +0200267 }
Fam Zheng37989ce2016-04-22 21:53:55 +0800268 if (revents & (G_IO_OUT | G_IO_ERR) && node->io_write &&
269 aio_node_check(ctx, node->is_external)) {
Paolo Bonzini2bbf11d2017-01-12 19:07:57 +0100270 result = true;
271 break;
Paolo Bonzinicd9ba1e2012-09-24 14:57:22 +0200272 }
273 }
Paolo Bonzini2bbf11d2017-01-12 19:07:57 +0100274 qemu_lockcnt_dec(&ctx->list_lock);
Paolo Bonzinicd9ba1e2012-09-24 14:57:22 +0200275
Paolo Bonzini2bbf11d2017-01-12 19:07:57 +0100276 return result;
Paolo Bonzinicd9ba1e2012-09-24 14:57:22 +0200277}
278
Stefan Hajnoczi47490792020-02-14 17:17:11 +0000279static void aio_free_deleted_handlers(AioContext *ctx)
280{
281 AioHandler *node;
282
283 if (QLIST_EMPTY_RCU(&ctx->deleted_aio_handlers)) {
284 return;
285 }
286 if (!qemu_lockcnt_dec_if_lock(&ctx->list_lock)) {
287 return; /* we are nested, let the parent do the freeing */
288 }
289
290 while ((node = QLIST_FIRST_RCU(&ctx->deleted_aio_handlers))) {
291 QLIST_REMOVE(node, node);
292 QLIST_REMOVE(node, node_deleted);
Stefan Hajnoczid37d0e32020-03-05 17:08:06 +0000293 QLIST_SAFE_REMOVE(node, node_poll);
Stefan Hajnoczi47490792020-02-14 17:17:11 +0000294 g_free(node);
295 }
296
297 qemu_lockcnt_inc_and_unlock(&ctx->list_lock);
298}
299
Stefan Hajnoczi7391d342020-02-14 17:17:12 +0000300static bool aio_dispatch_handler(AioContext *ctx, AioHandler *node)
301{
302 bool progress = false;
303 int revents;
304
305 revents = node->pfd.revents & node->pfd.events;
306 node->pfd.revents = 0;
307
Stefan Hajnoczid37d0e32020-03-05 17:08:06 +0000308 /*
309 * Start polling AioHandlers when they become ready because activity is
310 * likely to continue. Note that starvation is theoretically possible when
311 * fdmon_supports_polling(), but only until the fd fires for the first
312 * time.
313 */
314 if (!QLIST_IS_INSERTED(node, node_deleted) &&
315 !QLIST_IS_INSERTED(node, node_poll) &&
316 node->io_poll) {
317 trace_poll_add(ctx, node, node->pfd.fd, revents);
318 if (ctx->poll_started && node->io_poll_begin) {
319 node->io_poll_begin(node->opaque);
320 }
321 QLIST_INSERT_HEAD(&ctx->poll_aio_handlers, node, node_poll);
322 }
323
Stefan Hajnoczi7391d342020-02-14 17:17:12 +0000324 if (!QLIST_IS_INSERTED(node, node_deleted) &&
325 (revents & (G_IO_IN | G_IO_HUP | G_IO_ERR)) &&
326 aio_node_check(ctx, node->is_external) &&
327 node->io_read) {
328 node->io_read(node->opaque);
329
330 /* aio_notify() does not count as progress */
331 if (node->opaque != &ctx->notifier) {
332 progress = true;
333 }
334 }
335 if (!QLIST_IS_INSERTED(node, node_deleted) &&
336 (revents & (G_IO_OUT | G_IO_ERR)) &&
337 aio_node_check(ctx, node->is_external) &&
338 node->io_write) {
339 node->io_write(node->opaque);
340 progress = true;
341 }
342
343 return progress;
344}
345
346/*
347 * If we have a list of ready handlers then this is more efficient than
348 * scanning all handlers with aio_dispatch_handlers().
349 */
350static bool aio_dispatch_ready_handlers(AioContext *ctx,
351 AioHandlerList *ready_list)
352{
353 bool progress = false;
354 AioHandler *node;
355
356 while ((node = QLIST_FIRST(ready_list))) {
Stefan Hajnoczic39cbed2020-02-24 10:34:06 +0000357 QLIST_REMOVE(node, node_ready);
Stefan Hajnoczi7391d342020-02-14 17:17:12 +0000358 progress = aio_dispatch_handler(ctx, node) || progress;
359 }
360
361 return progress;
362}
363
364/* Slower than aio_dispatch_ready_handlers() but only used via glib */
Paolo Bonzini56d2c3c2017-01-12 19:07:55 +0100365static bool aio_dispatch_handlers(AioContext *ctx)
aliguoria76bab42008-09-22 19:17:18 +0000366{
Paolo Bonziniabf90d32017-01-12 19:07:56 +0100367 AioHandler *node, *tmp;
Stefan Hajnoczid0c8d2c2013-02-20 11:28:31 +0100368 bool progress = false;
aliguoria76bab42008-09-22 19:17:18 +0000369
Paolo Bonzini2bbf11d2017-01-12 19:07:57 +0100370 QLIST_FOREACH_SAFE_RCU(node, &ctx->aio_handlers, node, tmp) {
Stefan Hajnoczi7391d342020-02-14 17:17:12 +0000371 progress = aio_dispatch_handler(ctx, node) || progress;
Paolo Bonzinicd9ba1e2012-09-24 14:57:22 +0200372 }
Alex Bligh438e1f42013-08-21 16:02:53 +0100373
Paolo Bonzini56d2c3c2017-01-12 19:07:55 +0100374 return progress;
375}
376
Paolo Bonzinia153bf52017-02-13 14:52:33 +0100377void aio_dispatch(AioContext *ctx)
Paolo Bonzini56d2c3c2017-01-12 19:07:55 +0100378{
Paolo Bonzinia153bf52017-02-13 14:52:33 +0100379 qemu_lockcnt_inc(&ctx->list_lock);
Paolo Bonzinibd451432017-02-13 14:52:34 +0100380 aio_bh_poll(ctx);
Paolo Bonzinia153bf52017-02-13 14:52:33 +0100381 aio_dispatch_handlers(ctx);
Stefan Hajnoczi47490792020-02-14 17:17:11 +0000382 aio_free_deleted_handlers(ctx);
Paolo Bonzinia153bf52017-02-13 14:52:33 +0100383 qemu_lockcnt_dec(&ctx->list_lock);
Paolo Bonzini56d2c3c2017-01-12 19:07:55 +0100384
Paolo Bonzinia153bf52017-02-13 14:52:33 +0100385 timerlistgroup_run_timers(&ctx->tlg);
Stefan Hajnoczid0c8d2c2013-02-20 11:28:31 +0100386}
387
Stefan Hajnoczid37d0e32020-03-05 17:08:06 +0000388static bool run_poll_handlers_once(AioContext *ctx,
389 int64_t now,
390 int64_t *timeout)
Stefan Hajnoczi684e5082016-12-01 19:26:49 +0000391{
392 bool progress = false;
393 AioHandler *node;
Stefan Hajnoczid37d0e32020-03-05 17:08:06 +0000394 AioHandler *tmp;
Stefan Hajnoczi684e5082016-12-01 19:26:49 +0000395
Stefan Hajnoczid37d0e32020-03-05 17:08:06 +0000396 QLIST_FOREACH_SAFE(node, &ctx->poll_aio_handlers, node_poll, tmp) {
397 if (aio_node_check(ctx, node->is_external) &&
Paolo Bonzinicfeb35d2018-09-12 19:10:40 +0200398 node->io_poll(node->opaque)) {
Stefan Hajnoczid37d0e32020-03-05 17:08:06 +0000399 node->poll_idle_timeout = now + POLL_IDLE_INTERVAL_NS;
400
Paolo Bonzini993ed892019-04-09 14:28:23 +0200401 /*
402 * Polling was successful, exit try_poll_mode immediately
403 * to adjust the next polling time.
404 */
Paolo Bonzinie30cffa2018-09-12 19:10:39 +0200405 *timeout = 0;
Paolo Bonzinicfeb35d2018-09-12 19:10:40 +0200406 if (node->opaque != &ctx->notifier) {
407 progress = true;
408 }
Stefan Hajnoczi684e5082016-12-01 19:26:49 +0000409 }
410
411 /* Caller handles freeing deleted nodes. Don't do it here. */
412 }
413
414 return progress;
415}
416
Stefan Hajnoczid37d0e32020-03-05 17:08:06 +0000417static bool fdmon_supports_polling(AioContext *ctx)
418{
419 return ctx->fdmon_ops->need_wait != aio_poll_disabled;
420}
421
422static bool remove_idle_poll_handlers(AioContext *ctx, int64_t now)
423{
424 AioHandler *node;
425 AioHandler *tmp;
426 bool progress = false;
427
428 /*
429 * File descriptor monitoring implementations without userspace polling
430 * support suffer from starvation when a subset of handlers is polled
431 * because fds will not be processed in a timely fashion. Don't remove
432 * idle poll handlers.
433 */
434 if (!fdmon_supports_polling(ctx)) {
435 return false;
436 }
437
438 QLIST_FOREACH_SAFE(node, &ctx->poll_aio_handlers, node_poll, tmp) {
439 if (node->poll_idle_timeout == 0LL) {
440 node->poll_idle_timeout = now + POLL_IDLE_INTERVAL_NS;
441 } else if (now >= node->poll_idle_timeout) {
442 trace_poll_remove(ctx, node, node->pfd.fd);
443 node->poll_idle_timeout = 0LL;
444 QLIST_SAFE_REMOVE(node, node_poll);
445 if (ctx->poll_started && node->io_poll_end) {
446 node->io_poll_end(node->opaque);
447
448 /*
449 * Final poll in case ->io_poll_end() races with an event.
450 * Nevermind about re-adding the handler in the rare case where
451 * this causes progress.
452 */
453 progress = node->io_poll(node->opaque) || progress;
454 }
455 }
456 }
457
458 return progress;
459}
460
Stefan Hajnoczi4a1cba32016-12-01 19:26:42 +0000461/* run_poll_handlers:
462 * @ctx: the AioContext
463 * @max_ns: maximum time to poll for, in nanoseconds
464 *
465 * Polls for a given time.
466 *
Paolo Bonzini2bbf11d2017-01-12 19:07:57 +0100467 * Note that the caller must have incremented ctx->list_lock.
Stefan Hajnoczi4a1cba32016-12-01 19:26:42 +0000468 *
469 * Returns: true if progress was made, false otherwise
470 */
Paolo Bonzinie30cffa2018-09-12 19:10:39 +0200471static bool run_poll_handlers(AioContext *ctx, int64_t max_ns, int64_t *timeout)
Stefan Hajnoczi4a1cba32016-12-01 19:26:42 +0000472{
Stefan Hajnoczi684e5082016-12-01 19:26:49 +0000473 bool progress;
Paolo Bonzinie30cffa2018-09-12 19:10:39 +0200474 int64_t start_time, elapsed_time;
Stefan Hajnoczi4a1cba32016-12-01 19:26:42 +0000475
Paolo Bonzini2bbf11d2017-01-12 19:07:57 +0100476 assert(qemu_lockcnt_count(&ctx->list_lock) > 0);
Stefan Hajnoczi4a1cba32016-12-01 19:26:42 +0000477
Paolo Bonzinie30cffa2018-09-12 19:10:39 +0200478 trace_run_poll_handlers_begin(ctx, max_ns, *timeout);
Stefan Hajnoczi4a1cba32016-12-01 19:26:42 +0000479
Stefan Hajnoczi3aa221b2020-03-05 17:08:01 +0000480 /*
481 * Optimization: ->io_poll() handlers often contain RCU read critical
482 * sections and we therefore see many rcu_read_lock() -> rcu_read_unlock()
483 * -> rcu_read_lock() -> ... sequences with expensive memory
484 * synchronization primitives. Make the entire polling loop an RCU
485 * critical section because nested rcu_read_lock()/rcu_read_unlock() calls
486 * are cheap.
487 */
488 RCU_READ_LOCK_GUARD();
489
Paolo Bonzinie30cffa2018-09-12 19:10:39 +0200490 start_time = qemu_clock_get_ns(QEMU_CLOCK_REALTIME);
Stefan Hajnoczi4a1cba32016-12-01 19:26:42 +0000491 do {
Stefan Hajnoczid37d0e32020-03-05 17:08:06 +0000492 progress = run_poll_handlers_once(ctx, start_time, timeout);
Paolo Bonzinie30cffa2018-09-12 19:10:39 +0200493 elapsed_time = qemu_clock_get_ns(QEMU_CLOCK_REALTIME) - start_time;
Paolo Bonzini993ed892019-04-09 14:28:23 +0200494 max_ns = qemu_soonest_timeout(*timeout, max_ns);
495 assert(!(max_ns && progress));
Stefan Hajnocziaa38e192020-03-05 17:08:05 +0000496 } while (elapsed_time < max_ns && !ctx->fdmon_ops->need_wait(ctx));
Stefan Hajnoczi4a1cba32016-12-01 19:26:42 +0000497
Stefan Hajnoczid37d0e32020-03-05 17:08:06 +0000498 if (remove_idle_poll_handlers(ctx, start_time + elapsed_time)) {
499 *timeout = 0;
500 progress = true;
501 }
502
Paolo Bonzinie30cffa2018-09-12 19:10:39 +0200503 /* If time has passed with no successful polling, adjust *timeout to
504 * keep the same ending time.
505 */
506 if (*timeout != -1) {
507 *timeout -= MIN(*timeout, elapsed_time);
508 }
Stefan Hajnoczi4a1cba32016-12-01 19:26:42 +0000509
Paolo Bonzinie30cffa2018-09-12 19:10:39 +0200510 trace_run_poll_handlers_end(ctx, progress, *timeout);
Stefan Hajnoczi4a1cba32016-12-01 19:26:42 +0000511 return progress;
512}
513
514/* try_poll_mode:
515 * @ctx: the AioContext
Paolo Bonzinie30cffa2018-09-12 19:10:39 +0200516 * @timeout: timeout for blocking wait, computed by the caller and updated if
517 * polling succeeds.
Stefan Hajnoczi4a1cba32016-12-01 19:26:42 +0000518 *
Paolo Bonzini2bbf11d2017-01-12 19:07:57 +0100519 * Note that the caller must have incremented ctx->list_lock.
Stefan Hajnoczi4a1cba32016-12-01 19:26:42 +0000520 *
521 * Returns: true if progress was made, false otherwise
522 */
Paolo Bonzinie30cffa2018-09-12 19:10:39 +0200523static bool try_poll_mode(AioContext *ctx, int64_t *timeout)
Stefan Hajnoczi4a1cba32016-12-01 19:26:42 +0000524{
Stefan Hajnoczid37d0e32020-03-05 17:08:06 +0000525 int64_t max_ns;
Stefan Hajnoczi4a1cba32016-12-01 19:26:42 +0000526
Stefan Hajnoczid37d0e32020-03-05 17:08:06 +0000527 if (QLIST_EMPTY_RCU(&ctx->poll_aio_handlers)) {
528 return false;
529 }
530
531 max_ns = qemu_soonest_timeout(*timeout, ctx->poll_ns);
Stefan Hajnocziaa38e192020-03-05 17:08:05 +0000532 if (max_ns && !ctx->fdmon_ops->need_wait(ctx)) {
Paolo Bonzinie30cffa2018-09-12 19:10:39 +0200533 poll_set_started(ctx, true);
Stefan Hajnoczi684e5082016-12-01 19:26:49 +0000534
Paolo Bonzinie30cffa2018-09-12 19:10:39 +0200535 if (run_poll_handlers(ctx, max_ns, timeout)) {
536 return true;
Stefan Hajnoczi4a1cba32016-12-01 19:26:42 +0000537 }
538 }
539
Stefan Hajnoczie4346192020-03-05 17:08:00 +0000540 if (poll_set_started(ctx, false)) {
541 *timeout = 0;
542 return true;
543 }
Stefan Hajnoczi684e5082016-12-01 19:26:49 +0000544
Stefan Hajnoczie4346192020-03-05 17:08:00 +0000545 return false;
Stefan Hajnoczi4a1cba32016-12-01 19:26:42 +0000546}
547
Stefan Hajnoczid0c8d2c2013-02-20 11:28:31 +0100548bool aio_poll(AioContext *ctx, bool blocking)
549{
Stefan Hajnoczi7391d342020-02-14 17:17:12 +0000550 AioHandlerList ready_list = QLIST_HEAD_INITIALIZER(ready_list);
Stefan Hajnoczi4a1cba32016-12-01 19:26:42 +0000551 int ret = 0;
Stefan Hajnoczi164a1012013-04-11 16:56:50 +0200552 bool progress;
Stefan Hajnoczi44277bf2020-08-06 14:18:02 +0100553 bool use_notify_me;
Paolo Bonzinie98ab092015-02-20 17:26:50 +0100554 int64_t timeout;
Stefan Hajnoczi82a41182016-12-01 19:26:51 +0000555 int64_t start = 0;
Stefan Hajnoczid0c8d2c2013-02-20 11:28:31 +0100556
Paolo Bonzini5710a3e2020-04-07 10:07:46 -0400557 /*
558 * There cannot be two concurrent aio_poll calls for the same AioContext (or
559 * an aio_poll concurrent with a GSource prepare/check/dispatch callback).
560 * We rely on this below to avoid slow locked accesses to ctx->notify_me.
561 */
Kevin Wolf0dc165c2019-02-14 13:13:36 +0100562 assert(in_aio_context_home_thread(ctx));
563
Paolo Bonzini2bbf11d2017-01-12 19:07:57 +0100564 qemu_lockcnt_inc(&ctx->list_lock);
aliguoria76bab42008-09-22 19:17:18 +0000565
Stefan Hajnoczi82a41182016-12-01 19:26:51 +0000566 if (ctx->poll_max_ns) {
567 start = qemu_clock_get_ns(QEMU_CLOCK_REALTIME);
568 }
569
Paolo Bonzinie30cffa2018-09-12 19:10:39 +0200570 timeout = blocking ? aio_compute_timeout(ctx) : 0;
571 progress = try_poll_mode(ctx, &timeout);
572 assert(!(timeout && progress));
573
Stefan Hajnoczi44277bf2020-08-06 14:18:02 +0100574 /*
575 * aio_notify can avoid the expensive event_notifier_set if
576 * everything (file descriptors, bottom halves, timers) will
577 * be re-evaluated before the next blocking poll(). This is
578 * already true when aio_poll is called with blocking == false;
579 * if blocking == true, it is only true after poll() returns,
580 * so disable the optimization now.
581 */
582 use_notify_me = timeout != 0;
583 if (use_notify_me) {
Stefan Hajnoczid73415a2020-09-23 11:56:46 +0100584 qatomic_set(&ctx->notify_me, qatomic_read(&ctx->notify_me) + 2);
Stefan Hajnoczi44277bf2020-08-06 14:18:02 +0100585 /*
586 * Write ctx->notify_me before reading ctx->notified. Pairs with
587 * smp_mb in aio_notify().
588 */
589 smp_mb();
590
591 /* Don't block if aio_notify() was called */
Stefan Hajnoczid73415a2020-09-23 11:56:46 +0100592 if (qatomic_read(&ctx->notified)) {
Stefan Hajnoczi44277bf2020-08-06 14:18:02 +0100593 timeout = 0;
594 }
595 }
596
Paolo Bonzinie30cffa2018-09-12 19:10:39 +0200597 /* If polling is allowed, non-blocking aio_poll does not need the
598 * system call---a single round of run_poll_handlers_once suffices.
599 */
Stefan Hajnocziaa38e192020-03-05 17:08:05 +0000600 if (timeout || ctx->fdmon_ops->need_wait(ctx)) {
Stefan Hajnoczi1f050a42020-03-05 17:08:02 +0000601 ret = ctx->fdmon_ops->wait(ctx, &ready_list, timeout);
Paolo Bonzini9eb0bfc2012-04-12 14:00:56 +0200602 }
603
Stefan Hajnoczi44277bf2020-08-06 14:18:02 +0100604 if (use_notify_me) {
Paolo Bonzini5710a3e2020-04-07 10:07:46 -0400605 /* Finish the poll before clearing the flag. */
Stefan Hajnoczid73415a2020-09-23 11:56:46 +0100606 qatomic_store_release(&ctx->notify_me,
607 qatomic_read(&ctx->notify_me) - 2);
Paolo Bonzinieabc9772015-07-21 16:07:51 +0200608 }
Paolo Bonzini9eb0bfc2012-04-12 14:00:56 +0200609
Stefan Hajnoczi44277bf2020-08-06 14:18:02 +0100610 aio_notify_accept(ctx);
611
Stefan Hajnoczi82a41182016-12-01 19:26:51 +0000612 /* Adjust polling time */
613 if (ctx->poll_max_ns) {
614 int64_t block_ns = qemu_clock_get_ns(QEMU_CLOCK_REALTIME) - start;
615
616 if (block_ns <= ctx->poll_ns) {
617 /* This is the sweet spot, no adjustment needed */
618 } else if (block_ns > ctx->poll_max_ns) {
619 /* We'd have to poll for too long, poll less */
620 int64_t old = ctx->poll_ns;
621
622 if (ctx->poll_shrink) {
623 ctx->poll_ns /= ctx->poll_shrink;
624 } else {
625 ctx->poll_ns = 0;
626 }
627
628 trace_poll_shrink(ctx, old, ctx->poll_ns);
629 } else if (ctx->poll_ns < ctx->poll_max_ns &&
630 block_ns < ctx->poll_max_ns) {
631 /* There is room to grow, poll longer */
632 int64_t old = ctx->poll_ns;
633 int64_t grow = ctx->poll_grow;
634
635 if (grow == 0) {
636 grow = 2;
637 }
638
639 if (ctx->poll_ns) {
640 ctx->poll_ns *= grow;
641 } else {
642 ctx->poll_ns = 4000; /* start polling at 4 microseconds */
643 }
644
645 if (ctx->poll_ns > ctx->poll_max_ns) {
646 ctx->poll_ns = ctx->poll_max_ns;
647 }
648
649 trace_poll_grow(ctx, old, ctx->poll_ns);
650 }
651 }
652
Paolo Bonzinia153bf52017-02-13 14:52:33 +0100653 progress |= aio_bh_poll(ctx);
654
655 if (ret > 0) {
Stefan Hajnoczi7391d342020-02-14 17:17:12 +0000656 progress |= aio_dispatch_ready_handlers(ctx, &ready_list);
Paolo Bonzini9eb0bfc2012-04-12 14:00:56 +0200657 }
Paolo Bonzinibcdc1852012-04-12 14:00:55 +0200658
Stefan Hajnoczi47490792020-02-14 17:17:11 +0000659 aio_free_deleted_handlers(ctx);
660
Paolo Bonzinibd451432017-02-13 14:52:34 +0100661 qemu_lockcnt_dec(&ctx->list_lock);
662
Paolo Bonzinia153bf52017-02-13 14:52:33 +0100663 progress |= timerlistgroup_run_timers(&ctx->tlg);
664
Stefan Hajnoczi164a1012013-04-11 16:56:50 +0200665 return progress;
aliguoria76bab42008-09-22 19:17:18 +0000666}
Fam Zheng37fcee52015-10-30 12:06:28 +0800667
Cao jin7e003462016-07-15 18:28:44 +0800668void aio_context_setup(AioContext *ctx)
Fam Zheng37fcee52015-10-30 12:06:28 +0800669{
Stefan Hajnoczi1f050a42020-03-05 17:08:02 +0000670 ctx->fdmon_ops = &fdmon_poll_ops;
671 ctx->epollfd = -1;
672
Stefan Hajnoczi73fd2822020-03-05 17:08:04 +0000673 /* Use the fastest fd monitoring implementation if available */
674 if (fdmon_io_uring_setup(ctx)) {
675 return;
676 }
677
Stefan Hajnoczi1f050a42020-03-05 17:08:02 +0000678 fdmon_epoll_setup(ctx);
Fam Zheng37fcee52015-10-30 12:06:28 +0800679}
Stefan Hajnoczi4a1cba32016-12-01 19:26:42 +0000680
Jie Wangcd0a6d22018-05-17 08:42:43 +0800681void aio_context_destroy(AioContext *ctx)
682{
Stefan Hajnoczi73fd2822020-03-05 17:08:04 +0000683 fdmon_io_uring_destroy(ctx);
Stefan Hajnoczi1f050a42020-03-05 17:08:02 +0000684 fdmon_epoll_disable(ctx);
Stefan Hajnoczide137e42020-05-11 19:36:29 +0100685 aio_free_deleted_handlers(ctx);
Jie Wangcd0a6d22018-05-17 08:42:43 +0800686}
687
Stefan Hajnocziba607ca2020-05-11 19:36:30 +0100688void aio_context_use_g_source(AioContext *ctx)
689{
690 /*
691 * Disable io_uring when the glib main loop is used because it doesn't
692 * support mixed glib/aio_poll() usage. It relies on aio_poll() being
693 * called regularly so that changes to the monitored file descriptors are
694 * submitted, otherwise a list of pending fd handlers builds up.
695 */
696 fdmon_io_uring_destroy(ctx);
697 aio_free_deleted_handlers(ctx);
698}
699
Stefan Hajnoczi82a41182016-12-01 19:26:51 +0000700void aio_context_set_poll_params(AioContext *ctx, int64_t max_ns,
701 int64_t grow, int64_t shrink, Error **errp)
Stefan Hajnoczi4a1cba32016-12-01 19:26:42 +0000702{
Stefan Hajnoczi82a41182016-12-01 19:26:51 +0000703 /* No thread synchronization here, it doesn't matter if an incorrect value
704 * is used once.
Stefan Hajnoczi4a1cba32016-12-01 19:26:42 +0000705 */
706 ctx->poll_max_ns = max_ns;
Stefan Hajnoczi82a41182016-12-01 19:26:51 +0000707 ctx->poll_ns = 0;
708 ctx->poll_grow = grow;
709 ctx->poll_shrink = shrink;
Stefan Hajnoczi4a1cba32016-12-01 19:26:42 +0000710
711 aio_notify(ctx);
712}