Christoph Hellwig | 5c6c3a6 | 2009-08-20 16:58:35 +0200 | [diff] [blame] | 1 | /* |
| 2 | * Linux native AIO support. |
| 3 | * |
| 4 | * Copyright (C) 2009 IBM, Corp. |
| 5 | * Copyright (C) 2009 Red Hat, Inc. |
| 6 | * |
| 7 | * This work is licensed under the terms of the GNU GPL, version 2 or later. |
| 8 | * See the COPYING file in the top-level directory. |
| 9 | */ |
Peter Maydell | 80c71a2 | 2016-01-18 18:01:42 +0000 | [diff] [blame] | 10 | #include "qemu/osdep.h" |
Christoph Hellwig | 5c6c3a6 | 2009-08-20 16:58:35 +0200 | [diff] [blame] | 11 | #include "qemu-common.h" |
Paolo Bonzini | 737e150 | 2012-12-17 18:19:44 +0100 | [diff] [blame] | 12 | #include "block/aio.h" |
Paolo Bonzini | 1de7afc | 2012-12-17 18:20:00 +0100 | [diff] [blame] | 13 | #include "qemu/queue.h" |
Paolo Bonzini | 9f8540e | 2012-06-09 10:57:37 +0200 | [diff] [blame] | 14 | #include "block/raw-aio.h" |
Paolo Bonzini | 1de7afc | 2012-12-17 18:20:00 +0100 | [diff] [blame] | 15 | #include "qemu/event_notifier.h" |
Christoph Hellwig | 5c6c3a6 | 2009-08-20 16:58:35 +0200 | [diff] [blame] | 16 | |
Christoph Hellwig | 5c6c3a6 | 2009-08-20 16:58:35 +0200 | [diff] [blame] | 17 | #include <libaio.h> |
| 18 | |
| 19 | /* |
| 20 | * Queue size (per-device). |
| 21 | * |
| 22 | * XXX: eventually we need to communicate this to the guest and/or make it |
| 23 | * tunable by the guest. If we get more outstanding requests at a time |
| 24 | * than this we will get EAGAIN from io_submit which is communicated to |
| 25 | * the guest as an I/O error. |
| 26 | */ |
| 27 | #define MAX_EVENTS 128 |
| 28 | |
Ming Lei | 1b3abdc | 2014-07-04 18:04:34 +0800 | [diff] [blame] | 29 | #define MAX_QUEUED_IO 128 |
| 30 | |
Christoph Hellwig | 5c6c3a6 | 2009-08-20 16:58:35 +0200 | [diff] [blame] | 31 | struct qemu_laiocb { |
Markus Armbruster | 7c84b1b | 2014-10-07 13:59:14 +0200 | [diff] [blame] | 32 | BlockAIOCB common; |
Paolo Bonzini | dd7f7ed | 2016-04-07 18:33:35 +0200 | [diff] [blame] | 33 | LinuxAioState *ctx; |
Christoph Hellwig | 5c6c3a6 | 2009-08-20 16:58:35 +0200 | [diff] [blame] | 34 | struct iocb iocb; |
| 35 | ssize_t ret; |
| 36 | size_t nbytes; |
Kevin Wolf | b161e2e | 2011-10-13 15:42:52 +0200 | [diff] [blame] | 37 | QEMUIOVector *qiov; |
| 38 | bool is_read; |
Paolo Bonzini | 28b2408 | 2014-12-11 14:52:26 +0100 | [diff] [blame] | 39 | QSIMPLEQ_ENTRY(qemu_laiocb) next; |
Christoph Hellwig | 5c6c3a6 | 2009-08-20 16:58:35 +0200 | [diff] [blame] | 40 | }; |
| 41 | |
Ming Lei | 1b3abdc | 2014-07-04 18:04:34 +0800 | [diff] [blame] | 42 | typedef struct { |
Ming Lei | 1b3abdc | 2014-07-04 18:04:34 +0800 | [diff] [blame] | 43 | int plugged; |
Paolo Bonzini | 8455ce0 | 2014-12-11 14:52:28 +0100 | [diff] [blame] | 44 | unsigned int n; |
Paolo Bonzini | 43f2376 | 2014-12-11 14:52:27 +0100 | [diff] [blame] | 45 | bool blocked; |
Paolo Bonzini | 28b2408 | 2014-12-11 14:52:26 +0100 | [diff] [blame] | 46 | QSIMPLEQ_HEAD(, qemu_laiocb) pending; |
Ming Lei | 1b3abdc | 2014-07-04 18:04:34 +0800 | [diff] [blame] | 47 | } LaioQueue; |
| 48 | |
Paolo Bonzini | dd7f7ed | 2016-04-07 18:33:35 +0200 | [diff] [blame] | 49 | struct LinuxAioState { |
Christoph Hellwig | 5c6c3a6 | 2009-08-20 16:58:35 +0200 | [diff] [blame] | 50 | io_context_t ctx; |
Paolo Bonzini | c90caf2 | 2012-02-24 08:39:02 +0100 | [diff] [blame] | 51 | EventNotifier e; |
Ming Lei | 1b3abdc | 2014-07-04 18:04:34 +0800 | [diff] [blame] | 52 | |
| 53 | /* io queue for submit at batch */ |
| 54 | LaioQueue io_q; |
Stefan Hajnoczi | 2cdff7f | 2014-08-04 16:56:33 +0100 | [diff] [blame] | 55 | |
| 56 | /* I/O completion processing */ |
| 57 | QEMUBH *completion_bh; |
| 58 | struct io_event events[MAX_EVENTS]; |
| 59 | int event_idx; |
| 60 | int event_max; |
Christoph Hellwig | 5c6c3a6 | 2009-08-20 16:58:35 +0200 | [diff] [blame] | 61 | }; |
| 62 | |
Paolo Bonzini | dd7f7ed | 2016-04-07 18:33:35 +0200 | [diff] [blame] | 63 | static void ioq_submit(LinuxAioState *s); |
Paolo Bonzini | 28b2408 | 2014-12-11 14:52:26 +0100 | [diff] [blame] | 64 | |
Christoph Hellwig | 5c6c3a6 | 2009-08-20 16:58:35 +0200 | [diff] [blame] | 65 | static inline ssize_t io_event_ret(struct io_event *ev) |
| 66 | { |
| 67 | return (ssize_t)(((uint64_t)ev->res2 << 32) | ev->res); |
| 68 | } |
| 69 | |
Kevin Wolf | db0ffc2 | 2009-10-22 17:54:41 +0200 | [diff] [blame] | 70 | /* |
| 71 | * Completes an AIO request (calls the callback and frees the ACB). |
Kevin Wolf | db0ffc2 | 2009-10-22 17:54:41 +0200 | [diff] [blame] | 72 | */ |
Paolo Bonzini | dd7f7ed | 2016-04-07 18:33:35 +0200 | [diff] [blame] | 73 | static void qemu_laio_process_completion(struct qemu_laiocb *laiocb) |
Kevin Wolf | db0ffc2 | 2009-10-22 17:54:41 +0200 | [diff] [blame] | 74 | { |
| 75 | int ret; |
| 76 | |
Kevin Wolf | db0ffc2 | 2009-10-22 17:54:41 +0200 | [diff] [blame] | 77 | ret = laiocb->ret; |
| 78 | if (ret != -ECANCELED) { |
Kevin Wolf | b161e2e | 2011-10-13 15:42:52 +0200 | [diff] [blame] | 79 | if (ret == laiocb->nbytes) { |
Kevin Wolf | db0ffc2 | 2009-10-22 17:54:41 +0200 | [diff] [blame] | 80 | ret = 0; |
Kevin Wolf | b161e2e | 2011-10-13 15:42:52 +0200 | [diff] [blame] | 81 | } else if (ret >= 0) { |
| 82 | /* Short reads mean EOF, pad with zeros. */ |
| 83 | if (laiocb->is_read) { |
Michael Tokarev | 3d9b492 | 2012-03-10 16:54:23 +0400 | [diff] [blame] | 84 | qemu_iovec_memset(laiocb->qiov, ret, 0, |
| 85 | laiocb->qiov->size - ret); |
Kevin Wolf | b161e2e | 2011-10-13 15:42:52 +0200 | [diff] [blame] | 86 | } else { |
| 87 | ret = -EINVAL; |
| 88 | } |
| 89 | } |
Kevin Wolf | db0ffc2 | 2009-10-22 17:54:41 +0200 | [diff] [blame] | 90 | } |
Fam Zheng | 771b64d | 2014-09-11 13:41:13 +0800 | [diff] [blame] | 91 | laiocb->common.cb(laiocb->common.opaque, ret); |
Kevin Wolf | db0ffc2 | 2009-10-22 17:54:41 +0200 | [diff] [blame] | 92 | |
Fam Zheng | 8007429 | 2014-09-11 13:41:28 +0800 | [diff] [blame] | 93 | qemu_aio_unref(laiocb); |
Kevin Wolf | db0ffc2 | 2009-10-22 17:54:41 +0200 | [diff] [blame] | 94 | } |
| 95 | |
Stefan Hajnoczi | 2cdff7f | 2014-08-04 16:56:33 +0100 | [diff] [blame] | 96 | /* The completion BH fetches completed I/O requests and invokes their |
| 97 | * callbacks. |
| 98 | * |
| 99 | * The function is somewhat tricky because it supports nested event loops, for |
| 100 | * example when a request callback invokes aio_poll(). In order to do this, |
Paolo Bonzini | dd7f7ed | 2016-04-07 18:33:35 +0200 | [diff] [blame] | 101 | * the completion events array and index are kept in LinuxAioState. The BH |
Stefan Hajnoczi | 2cdff7f | 2014-08-04 16:56:33 +0100 | [diff] [blame] | 102 | * reschedules itself as long as there are completions pending so it will |
| 103 | * either be called again in a nested event loop or will be called after all |
| 104 | * events have been completed. When there are no events left to complete, the |
| 105 | * BH returns without rescheduling. |
| 106 | */ |
| 107 | static void qemu_laio_completion_bh(void *opaque) |
| 108 | { |
Paolo Bonzini | dd7f7ed | 2016-04-07 18:33:35 +0200 | [diff] [blame] | 109 | LinuxAioState *s = opaque; |
Stefan Hajnoczi | 2cdff7f | 2014-08-04 16:56:33 +0100 | [diff] [blame] | 110 | |
| 111 | /* Fetch more completion events when empty */ |
| 112 | if (s->event_idx == s->event_max) { |
| 113 | do { |
| 114 | struct timespec ts = { 0 }; |
| 115 | s->event_max = io_getevents(s->ctx, MAX_EVENTS, MAX_EVENTS, |
| 116 | s->events, &ts); |
| 117 | } while (s->event_max == -EINTR); |
| 118 | |
| 119 | s->event_idx = 0; |
| 120 | if (s->event_max <= 0) { |
| 121 | s->event_max = 0; |
| 122 | return; /* no more events */ |
| 123 | } |
| 124 | } |
| 125 | |
| 126 | /* Reschedule so nested event loops see currently pending completions */ |
| 127 | qemu_bh_schedule(s->completion_bh); |
| 128 | |
| 129 | /* Process completion events */ |
| 130 | while (s->event_idx < s->event_max) { |
| 131 | struct iocb *iocb = s->events[s->event_idx].obj; |
| 132 | struct qemu_laiocb *laiocb = |
| 133 | container_of(iocb, struct qemu_laiocb, iocb); |
| 134 | |
| 135 | laiocb->ret = io_event_ret(&s->events[s->event_idx]); |
| 136 | s->event_idx++; |
| 137 | |
Paolo Bonzini | dd7f7ed | 2016-04-07 18:33:35 +0200 | [diff] [blame] | 138 | qemu_laio_process_completion(laiocb); |
Stefan Hajnoczi | 2cdff7f | 2014-08-04 16:56:33 +0100 | [diff] [blame] | 139 | } |
Paolo Bonzini | 28b2408 | 2014-12-11 14:52:26 +0100 | [diff] [blame] | 140 | |
| 141 | if (!s->io_q.plugged && !QSIMPLEQ_EMPTY(&s->io_q.pending)) { |
| 142 | ioq_submit(s); |
| 143 | } |
Stefan Hajnoczi | 2cdff7f | 2014-08-04 16:56:33 +0100 | [diff] [blame] | 144 | } |
| 145 | |
Paolo Bonzini | c90caf2 | 2012-02-24 08:39:02 +0100 | [diff] [blame] | 146 | static void qemu_laio_completion_cb(EventNotifier *e) |
Christoph Hellwig | 5c6c3a6 | 2009-08-20 16:58:35 +0200 | [diff] [blame] | 147 | { |
Paolo Bonzini | dd7f7ed | 2016-04-07 18:33:35 +0200 | [diff] [blame] | 148 | LinuxAioState *s = container_of(e, LinuxAioState, e); |
Christoph Hellwig | 5c6c3a6 | 2009-08-20 16:58:35 +0200 | [diff] [blame] | 149 | |
Stefan Hajnoczi | 2cdff7f | 2014-08-04 16:56:33 +0100 | [diff] [blame] | 150 | if (event_notifier_test_and_clear(&s->e)) { |
| 151 | qemu_bh_schedule(s->completion_bh); |
Christoph Hellwig | 5c6c3a6 | 2009-08-20 16:58:35 +0200 | [diff] [blame] | 152 | } |
| 153 | } |
| 154 | |
Markus Armbruster | 7c84b1b | 2014-10-07 13:59:14 +0200 | [diff] [blame] | 155 | static void laio_cancel(BlockAIOCB *blockacb) |
Christoph Hellwig | 5c6c3a6 | 2009-08-20 16:58:35 +0200 | [diff] [blame] | 156 | { |
| 157 | struct qemu_laiocb *laiocb = (struct qemu_laiocb *)blockacb; |
| 158 | struct io_event event; |
| 159 | int ret; |
| 160 | |
Fam Zheng | 771b64d | 2014-09-11 13:41:13 +0800 | [diff] [blame] | 161 | if (laiocb->ret != -EINPROGRESS) { |
Christoph Hellwig | 5c6c3a6 | 2009-08-20 16:58:35 +0200 | [diff] [blame] | 162 | return; |
Fam Zheng | 771b64d | 2014-09-11 13:41:13 +0800 | [diff] [blame] | 163 | } |
Christoph Hellwig | 5c6c3a6 | 2009-08-20 16:58:35 +0200 | [diff] [blame] | 164 | ret = io_cancel(laiocb->ctx->ctx, &laiocb->iocb, &event); |
Fam Zheng | 771b64d | 2014-09-11 13:41:13 +0800 | [diff] [blame] | 165 | laiocb->ret = -ECANCELED; |
| 166 | if (ret != 0) { |
| 167 | /* iocb is not cancelled, cb will be called by the event loop later */ |
Christoph Hellwig | 5c6c3a6 | 2009-08-20 16:58:35 +0200 | [diff] [blame] | 168 | return; |
| 169 | } |
| 170 | |
Fam Zheng | 771b64d | 2014-09-11 13:41:13 +0800 | [diff] [blame] | 171 | laiocb->common.cb(laiocb->common.opaque, laiocb->ret); |
Christoph Hellwig | 5c6c3a6 | 2009-08-20 16:58:35 +0200 | [diff] [blame] | 172 | } |
| 173 | |
Stefan Hajnoczi | d7331be | 2012-10-31 16:34:37 +0100 | [diff] [blame] | 174 | static const AIOCBInfo laio_aiocb_info = { |
Christoph Hellwig | 5c6c3a6 | 2009-08-20 16:58:35 +0200 | [diff] [blame] | 175 | .aiocb_size = sizeof(struct qemu_laiocb), |
Fam Zheng | 771b64d | 2014-09-11 13:41:13 +0800 | [diff] [blame] | 176 | .cancel_async = laio_cancel, |
Christoph Hellwig | 5c6c3a6 | 2009-08-20 16:58:35 +0200 | [diff] [blame] | 177 | }; |
| 178 | |
Ming Lei | 1b3abdc | 2014-07-04 18:04:34 +0800 | [diff] [blame] | 179 | static void ioq_init(LaioQueue *io_q) |
| 180 | { |
Paolo Bonzini | 28b2408 | 2014-12-11 14:52:26 +0100 | [diff] [blame] | 181 | QSIMPLEQ_INIT(&io_q->pending); |
Ming Lei | 1b3abdc | 2014-07-04 18:04:34 +0800 | [diff] [blame] | 182 | io_q->plugged = 0; |
Paolo Bonzini | 8455ce0 | 2014-12-11 14:52:28 +0100 | [diff] [blame] | 183 | io_q->n = 0; |
Paolo Bonzini | 43f2376 | 2014-12-11 14:52:27 +0100 | [diff] [blame] | 184 | io_q->blocked = false; |
Ming Lei | 1b3abdc | 2014-07-04 18:04:34 +0800 | [diff] [blame] | 185 | } |
| 186 | |
Paolo Bonzini | dd7f7ed | 2016-04-07 18:33:35 +0200 | [diff] [blame] | 187 | static void ioq_submit(LinuxAioState *s) |
Ming Lei | 1b3abdc | 2014-07-04 18:04:34 +0800 | [diff] [blame] | 188 | { |
Paolo Bonzini | 82595da | 2014-12-11 14:52:30 +0100 | [diff] [blame] | 189 | int ret, len; |
Paolo Bonzini | 28b2408 | 2014-12-11 14:52:26 +0100 | [diff] [blame] | 190 | struct qemu_laiocb *aiocb; |
| 191 | struct iocb *iocbs[MAX_QUEUED_IO]; |
Paolo Bonzini | 82595da | 2014-12-11 14:52:30 +0100 | [diff] [blame] | 192 | QSIMPLEQ_HEAD(, qemu_laiocb) completed; |
Ming Lei | 1b3abdc | 2014-07-04 18:04:34 +0800 | [diff] [blame] | 193 | |
Paolo Bonzini | 43f2376 | 2014-12-11 14:52:27 +0100 | [diff] [blame] | 194 | do { |
| 195 | len = 0; |
| 196 | QSIMPLEQ_FOREACH(aiocb, &s->io_q.pending, next) { |
| 197 | iocbs[len++] = &aiocb->iocb; |
| 198 | if (len == MAX_QUEUED_IO) { |
| 199 | break; |
| 200 | } |
Paolo Bonzini | 28b2408 | 2014-12-11 14:52:26 +0100 | [diff] [blame] | 201 | } |
Ming Lei | 1b3abdc | 2014-07-04 18:04:34 +0800 | [diff] [blame] | 202 | |
Paolo Bonzini | 43f2376 | 2014-12-11 14:52:27 +0100 | [diff] [blame] | 203 | ret = io_submit(s->ctx, len, iocbs); |
| 204 | if (ret == -EAGAIN) { |
Paolo Bonzini | 82595da | 2014-12-11 14:52:30 +0100 | [diff] [blame] | 205 | break; |
Paolo Bonzini | 43f2376 | 2014-12-11 14:52:27 +0100 | [diff] [blame] | 206 | } |
| 207 | if (ret < 0) { |
| 208 | abort(); |
| 209 | } |
Ming Lei | 1b3abdc | 2014-07-04 18:04:34 +0800 | [diff] [blame] | 210 | |
Paolo Bonzini | 82595da | 2014-12-11 14:52:30 +0100 | [diff] [blame] | 211 | s->io_q.n -= ret; |
| 212 | aiocb = container_of(iocbs[ret - 1], struct qemu_laiocb, iocb); |
| 213 | QSIMPLEQ_SPLIT_AFTER(&s->io_q.pending, aiocb, next, &completed); |
Paolo Bonzini | 43f2376 | 2014-12-11 14:52:27 +0100 | [diff] [blame] | 214 | } while (ret == len && !QSIMPLEQ_EMPTY(&s->io_q.pending)); |
Paolo Bonzini | 8455ce0 | 2014-12-11 14:52:28 +0100 | [diff] [blame] | 215 | s->io_q.blocked = (s->io_q.n > 0); |
Ming Lei | 1b3abdc | 2014-07-04 18:04:34 +0800 | [diff] [blame] | 216 | } |
| 217 | |
Paolo Bonzini | dd7f7ed | 2016-04-07 18:33:35 +0200 | [diff] [blame] | 218 | void laio_io_plug(BlockDriverState *bs, LinuxAioState *s) |
Ming Lei | 1b3abdc | 2014-07-04 18:04:34 +0800 | [diff] [blame] | 219 | { |
Paolo Bonzini | 6b98bd6 | 2016-04-07 18:33:34 +0200 | [diff] [blame] | 220 | assert(!s->io_q.plugged); |
| 221 | s->io_q.plugged = 1; |
Ming Lei | 1b3abdc | 2014-07-04 18:04:34 +0800 | [diff] [blame] | 222 | } |
| 223 | |
Paolo Bonzini | dd7f7ed | 2016-04-07 18:33:35 +0200 | [diff] [blame] | 224 | void laio_io_unplug(BlockDriverState *bs, LinuxAioState *s) |
Ming Lei | 1b3abdc | 2014-07-04 18:04:34 +0800 | [diff] [blame] | 225 | { |
Paolo Bonzini | 6b98bd6 | 2016-04-07 18:33:34 +0200 | [diff] [blame] | 226 | assert(s->io_q.plugged); |
| 227 | s->io_q.plugged = 0; |
Paolo Bonzini | 43f2376 | 2014-12-11 14:52:27 +0100 | [diff] [blame] | 228 | if (!s->io_q.blocked && !QSIMPLEQ_EMPTY(&s->io_q.pending)) { |
Paolo Bonzini | de35464 | 2014-12-11 14:52:29 +0100 | [diff] [blame] | 229 | ioq_submit(s); |
Ming Lei | 1b3abdc | 2014-07-04 18:04:34 +0800 | [diff] [blame] | 230 | } |
Ming Lei | 1b3abdc | 2014-07-04 18:04:34 +0800 | [diff] [blame] | 231 | } |
| 232 | |
Paolo Bonzini | dd7f7ed | 2016-04-07 18:33:35 +0200 | [diff] [blame] | 233 | BlockAIOCB *laio_submit(BlockDriverState *bs, LinuxAioState *s, int fd, |
Christoph Hellwig | 5c6c3a6 | 2009-08-20 16:58:35 +0200 | [diff] [blame] | 234 | int64_t sector_num, QEMUIOVector *qiov, int nb_sectors, |
Markus Armbruster | 097310b | 2014-10-07 13:59:15 +0200 | [diff] [blame] | 235 | BlockCompletionFunc *cb, void *opaque, int type) |
Christoph Hellwig | 5c6c3a6 | 2009-08-20 16:58:35 +0200 | [diff] [blame] | 236 | { |
Christoph Hellwig | 5c6c3a6 | 2009-08-20 16:58:35 +0200 | [diff] [blame] | 237 | struct qemu_laiocb *laiocb; |
| 238 | struct iocb *iocbs; |
| 239 | off_t offset = sector_num * 512; |
| 240 | |
Stefan Hajnoczi | d7331be | 2012-10-31 16:34:37 +0100 | [diff] [blame] | 241 | laiocb = qemu_aio_get(&laio_aiocb_info, bs, cb, opaque); |
Christoph Hellwig | 5c6c3a6 | 2009-08-20 16:58:35 +0200 | [diff] [blame] | 242 | laiocb->nbytes = nb_sectors * 512; |
| 243 | laiocb->ctx = s; |
| 244 | laiocb->ret = -EINPROGRESS; |
Kevin Wolf | b161e2e | 2011-10-13 15:42:52 +0200 | [diff] [blame] | 245 | laiocb->is_read = (type == QEMU_AIO_READ); |
| 246 | laiocb->qiov = qiov; |
Christoph Hellwig | 5c6c3a6 | 2009-08-20 16:58:35 +0200 | [diff] [blame] | 247 | |
| 248 | iocbs = &laiocb->iocb; |
| 249 | |
| 250 | switch (type) { |
| 251 | case QEMU_AIO_WRITE: |
| 252 | io_prep_pwritev(iocbs, fd, qiov->iov, qiov->niov, offset); |
| 253 | break; |
| 254 | case QEMU_AIO_READ: |
| 255 | io_prep_preadv(iocbs, fd, qiov->iov, qiov->niov, offset); |
| 256 | break; |
Frediano Ziglio | c30e624 | 2011-08-30 09:46:11 +0200 | [diff] [blame] | 257 | /* Currently Linux kernel does not support other operations */ |
Christoph Hellwig | 5c6c3a6 | 2009-08-20 16:58:35 +0200 | [diff] [blame] | 258 | default: |
| 259 | fprintf(stderr, "%s: invalid AIO request type 0x%x.\n", |
| 260 | __func__, type); |
| 261 | goto out_free_aiocb; |
| 262 | } |
Paolo Bonzini | c90caf2 | 2012-02-24 08:39:02 +0100 | [diff] [blame] | 263 | io_set_eventfd(&laiocb->iocb, event_notifier_get_fd(&s->e)); |
Christoph Hellwig | 5c6c3a6 | 2009-08-20 16:58:35 +0200 | [diff] [blame] | 264 | |
Paolo Bonzini | 28b2408 | 2014-12-11 14:52:26 +0100 | [diff] [blame] | 265 | QSIMPLEQ_INSERT_TAIL(&s->io_q.pending, laiocb, next); |
Paolo Bonzini | 8455ce0 | 2014-12-11 14:52:28 +0100 | [diff] [blame] | 266 | s->io_q.n++; |
Paolo Bonzini | 43f2376 | 2014-12-11 14:52:27 +0100 | [diff] [blame] | 267 | if (!s->io_q.blocked && |
Paolo Bonzini | 8455ce0 | 2014-12-11 14:52:28 +0100 | [diff] [blame] | 268 | (!s->io_q.plugged || s->io_q.n >= MAX_QUEUED_IO)) { |
Paolo Bonzini | 28b2408 | 2014-12-11 14:52:26 +0100 | [diff] [blame] | 269 | ioq_submit(s); |
Ming Lei | 1b3abdc | 2014-07-04 18:04:34 +0800 | [diff] [blame] | 270 | } |
Christoph Hellwig | 5c6c3a6 | 2009-08-20 16:58:35 +0200 | [diff] [blame] | 271 | return &laiocb->common; |
| 272 | |
Kevin Wolf | 449c184 | 2011-09-22 14:21:30 +0200 | [diff] [blame] | 273 | out_free_aiocb: |
Fam Zheng | 8007429 | 2014-09-11 13:41:28 +0800 | [diff] [blame] | 274 | qemu_aio_unref(laiocb); |
Christoph Hellwig | 5c6c3a6 | 2009-08-20 16:58:35 +0200 | [diff] [blame] | 275 | return NULL; |
| 276 | } |
| 277 | |
Paolo Bonzini | dd7f7ed | 2016-04-07 18:33:35 +0200 | [diff] [blame] | 278 | void laio_detach_aio_context(LinuxAioState *s, AioContext *old_context) |
Stefan Hajnoczi | c2f3426 | 2014-05-08 16:34:47 +0200 | [diff] [blame] | 279 | { |
Fam Zheng | dca21ef | 2015-10-23 11:08:05 +0800 | [diff] [blame] | 280 | aio_set_event_notifier(old_context, &s->e, false, NULL); |
Stefan Hajnoczi | 2cdff7f | 2014-08-04 16:56:33 +0100 | [diff] [blame] | 281 | qemu_bh_delete(s->completion_bh); |
Stefan Hajnoczi | c2f3426 | 2014-05-08 16:34:47 +0200 | [diff] [blame] | 282 | } |
| 283 | |
Paolo Bonzini | dd7f7ed | 2016-04-07 18:33:35 +0200 | [diff] [blame] | 284 | void laio_attach_aio_context(LinuxAioState *s, AioContext *new_context) |
Stefan Hajnoczi | c2f3426 | 2014-05-08 16:34:47 +0200 | [diff] [blame] | 285 | { |
Stefan Hajnoczi | 2cdff7f | 2014-08-04 16:56:33 +0100 | [diff] [blame] | 286 | s->completion_bh = aio_bh_new(new_context, qemu_laio_completion_bh, s); |
Fam Zheng | dca21ef | 2015-10-23 11:08:05 +0800 | [diff] [blame] | 287 | aio_set_event_notifier(new_context, &s->e, false, |
| 288 | qemu_laio_completion_cb); |
Stefan Hajnoczi | c2f3426 | 2014-05-08 16:34:47 +0200 | [diff] [blame] | 289 | } |
| 290 | |
Paolo Bonzini | dd7f7ed | 2016-04-07 18:33:35 +0200 | [diff] [blame] | 291 | LinuxAioState *laio_init(void) |
Christoph Hellwig | 5c6c3a6 | 2009-08-20 16:58:35 +0200 | [diff] [blame] | 292 | { |
Paolo Bonzini | dd7f7ed | 2016-04-07 18:33:35 +0200 | [diff] [blame] | 293 | LinuxAioState *s; |
Christoph Hellwig | 5c6c3a6 | 2009-08-20 16:58:35 +0200 | [diff] [blame] | 294 | |
Anthony Liguori | 7267c09 | 2011-08-20 22:09:37 -0500 | [diff] [blame] | 295 | s = g_malloc0(sizeof(*s)); |
Paolo Bonzini | c90caf2 | 2012-02-24 08:39:02 +0100 | [diff] [blame] | 296 | if (event_notifier_init(&s->e, false) < 0) { |
Christoph Hellwig | 5c6c3a6 | 2009-08-20 16:58:35 +0200 | [diff] [blame] | 297 | goto out_free_state; |
Paolo Bonzini | c90caf2 | 2012-02-24 08:39:02 +0100 | [diff] [blame] | 298 | } |
Christoph Hellwig | 5c6c3a6 | 2009-08-20 16:58:35 +0200 | [diff] [blame] | 299 | |
Paolo Bonzini | c90caf2 | 2012-02-24 08:39:02 +0100 | [diff] [blame] | 300 | if (io_setup(MAX_EVENTS, &s->ctx) != 0) { |
Christoph Hellwig | 5c6c3a6 | 2009-08-20 16:58:35 +0200 | [diff] [blame] | 301 | goto out_close_efd; |
Paolo Bonzini | c90caf2 | 2012-02-24 08:39:02 +0100 | [diff] [blame] | 302 | } |
Christoph Hellwig | 5c6c3a6 | 2009-08-20 16:58:35 +0200 | [diff] [blame] | 303 | |
Ming Lei | 1b3abdc | 2014-07-04 18:04:34 +0800 | [diff] [blame] | 304 | ioq_init(&s->io_q); |
| 305 | |
Christoph Hellwig | 5c6c3a6 | 2009-08-20 16:58:35 +0200 | [diff] [blame] | 306 | return s; |
| 307 | |
| 308 | out_close_efd: |
Paolo Bonzini | c90caf2 | 2012-02-24 08:39:02 +0100 | [diff] [blame] | 309 | event_notifier_cleanup(&s->e); |
Christoph Hellwig | 5c6c3a6 | 2009-08-20 16:58:35 +0200 | [diff] [blame] | 310 | out_free_state: |
Anthony Liguori | 7267c09 | 2011-08-20 22:09:37 -0500 | [diff] [blame] | 311 | g_free(s); |
Christoph Hellwig | 5c6c3a6 | 2009-08-20 16:58:35 +0200 | [diff] [blame] | 312 | return NULL; |
| 313 | } |
Stefan Hajnoczi | abd269b | 2014-05-08 16:34:48 +0200 | [diff] [blame] | 314 | |
Paolo Bonzini | dd7f7ed | 2016-04-07 18:33:35 +0200 | [diff] [blame] | 315 | void laio_cleanup(LinuxAioState *s) |
Stefan Hajnoczi | abd269b | 2014-05-08 16:34:48 +0200 | [diff] [blame] | 316 | { |
Stefan Hajnoczi | abd269b | 2014-05-08 16:34:48 +0200 | [diff] [blame] | 317 | event_notifier_cleanup(&s->e); |
Gonglei | a1abf40 | 2014-07-12 11:43:37 +0800 | [diff] [blame] | 318 | |
| 319 | if (io_destroy(s->ctx) != 0) { |
| 320 | fprintf(stderr, "%s: destroy AIO context %p failed\n", |
| 321 | __func__, &s->ctx); |
| 322 | } |
Stefan Hajnoczi | abd269b | 2014-05-08 16:34:48 +0200 | [diff] [blame] | 323 | g_free(s); |
| 324 | } |