blob: 90ec98ee235cf48825468ecb171ebe2cef5d91ad [file] [log] [blame]
Christoph Hellwig5c6c3a62009-08-20 16:58:35 +02001/*
2 * Linux native AIO support.
3 *
4 * Copyright (C) 2009 IBM, Corp.
5 * Copyright (C) 2009 Red Hat, Inc.
6 *
7 * This work is licensed under the terms of the GNU GPL, version 2 or later.
8 * See the COPYING file in the top-level directory.
9 */
Peter Maydell80c71a22016-01-18 18:01:42 +000010#include "qemu/osdep.h"
Christoph Hellwig5c6c3a62009-08-20 16:58:35 +020011#include "qemu-common.h"
Paolo Bonzini737e1502012-12-17 18:19:44 +010012#include "block/aio.h"
Paolo Bonzini1de7afc2012-12-17 18:20:00 +010013#include "qemu/queue.h"
Paolo Bonzini9f8540e2012-06-09 10:57:37 +020014#include "block/raw-aio.h"
Paolo Bonzini1de7afc2012-12-17 18:20:00 +010015#include "qemu/event_notifier.h"
Christoph Hellwig5c6c3a62009-08-20 16:58:35 +020016
Christoph Hellwig5c6c3a62009-08-20 16:58:35 +020017#include <libaio.h>
18
19/*
20 * Queue size (per-device).
21 *
22 * XXX: eventually we need to communicate this to the guest and/or make it
23 * tunable by the guest. If we get more outstanding requests at a time
24 * than this we will get EAGAIN from io_submit which is communicated to
25 * the guest as an I/O error.
26 */
27#define MAX_EVENTS 128
28
Ming Lei1b3abdc2014-07-04 18:04:34 +080029#define MAX_QUEUED_IO 128
30
Christoph Hellwig5c6c3a62009-08-20 16:58:35 +020031struct qemu_laiocb {
Markus Armbruster7c84b1b2014-10-07 13:59:14 +020032 BlockAIOCB common;
Paolo Bonzinidd7f7ed2016-04-07 18:33:35 +020033 LinuxAioState *ctx;
Christoph Hellwig5c6c3a62009-08-20 16:58:35 +020034 struct iocb iocb;
35 ssize_t ret;
36 size_t nbytes;
Kevin Wolfb161e2e2011-10-13 15:42:52 +020037 QEMUIOVector *qiov;
38 bool is_read;
Paolo Bonzini28b24082014-12-11 14:52:26 +010039 QSIMPLEQ_ENTRY(qemu_laiocb) next;
Christoph Hellwig5c6c3a62009-08-20 16:58:35 +020040};
41
Ming Lei1b3abdc2014-07-04 18:04:34 +080042typedef struct {
Ming Lei1b3abdc2014-07-04 18:04:34 +080043 int plugged;
Paolo Bonzini8455ce02014-12-11 14:52:28 +010044 unsigned int n;
Paolo Bonzini43f23762014-12-11 14:52:27 +010045 bool blocked;
Paolo Bonzini28b24082014-12-11 14:52:26 +010046 QSIMPLEQ_HEAD(, qemu_laiocb) pending;
Ming Lei1b3abdc2014-07-04 18:04:34 +080047} LaioQueue;
48
Paolo Bonzinidd7f7ed2016-04-07 18:33:35 +020049struct LinuxAioState {
Christoph Hellwig5c6c3a62009-08-20 16:58:35 +020050 io_context_t ctx;
Paolo Bonzinic90caf22012-02-24 08:39:02 +010051 EventNotifier e;
Ming Lei1b3abdc2014-07-04 18:04:34 +080052
53 /* io queue for submit at batch */
54 LaioQueue io_q;
Stefan Hajnoczi2cdff7f2014-08-04 16:56:33 +010055
56 /* I/O completion processing */
57 QEMUBH *completion_bh;
58 struct io_event events[MAX_EVENTS];
59 int event_idx;
60 int event_max;
Christoph Hellwig5c6c3a62009-08-20 16:58:35 +020061};
62
Paolo Bonzinidd7f7ed2016-04-07 18:33:35 +020063static void ioq_submit(LinuxAioState *s);
Paolo Bonzini28b24082014-12-11 14:52:26 +010064
Christoph Hellwig5c6c3a62009-08-20 16:58:35 +020065static inline ssize_t io_event_ret(struct io_event *ev)
66{
67 return (ssize_t)(((uint64_t)ev->res2 << 32) | ev->res);
68}
69
Kevin Wolfdb0ffc22009-10-22 17:54:41 +020070/*
71 * Completes an AIO request (calls the callback and frees the ACB).
Kevin Wolfdb0ffc22009-10-22 17:54:41 +020072 */
Paolo Bonzinidd7f7ed2016-04-07 18:33:35 +020073static void qemu_laio_process_completion(struct qemu_laiocb *laiocb)
Kevin Wolfdb0ffc22009-10-22 17:54:41 +020074{
75 int ret;
76
Kevin Wolfdb0ffc22009-10-22 17:54:41 +020077 ret = laiocb->ret;
78 if (ret != -ECANCELED) {
Kevin Wolfb161e2e2011-10-13 15:42:52 +020079 if (ret == laiocb->nbytes) {
Kevin Wolfdb0ffc22009-10-22 17:54:41 +020080 ret = 0;
Kevin Wolfb161e2e2011-10-13 15:42:52 +020081 } else if (ret >= 0) {
82 /* Short reads mean EOF, pad with zeros. */
83 if (laiocb->is_read) {
Michael Tokarev3d9b4922012-03-10 16:54:23 +040084 qemu_iovec_memset(laiocb->qiov, ret, 0,
85 laiocb->qiov->size - ret);
Kevin Wolfb161e2e2011-10-13 15:42:52 +020086 } else {
87 ret = -EINVAL;
88 }
89 }
Kevin Wolfdb0ffc22009-10-22 17:54:41 +020090 }
Fam Zheng771b64d2014-09-11 13:41:13 +080091 laiocb->common.cb(laiocb->common.opaque, ret);
Kevin Wolfdb0ffc22009-10-22 17:54:41 +020092
Fam Zheng80074292014-09-11 13:41:28 +080093 qemu_aio_unref(laiocb);
Kevin Wolfdb0ffc22009-10-22 17:54:41 +020094}
95
Stefan Hajnoczi2cdff7f2014-08-04 16:56:33 +010096/* The completion BH fetches completed I/O requests and invokes their
97 * callbacks.
98 *
99 * The function is somewhat tricky because it supports nested event loops, for
100 * example when a request callback invokes aio_poll(). In order to do this,
Paolo Bonzinidd7f7ed2016-04-07 18:33:35 +0200101 * the completion events array and index are kept in LinuxAioState. The BH
Stefan Hajnoczi2cdff7f2014-08-04 16:56:33 +0100102 * reschedules itself as long as there are completions pending so it will
103 * either be called again in a nested event loop or will be called after all
104 * events have been completed. When there are no events left to complete, the
105 * BH returns without rescheduling.
106 */
107static void qemu_laio_completion_bh(void *opaque)
108{
Paolo Bonzinidd7f7ed2016-04-07 18:33:35 +0200109 LinuxAioState *s = opaque;
Stefan Hajnoczi2cdff7f2014-08-04 16:56:33 +0100110
111 /* Fetch more completion events when empty */
112 if (s->event_idx == s->event_max) {
113 do {
114 struct timespec ts = { 0 };
115 s->event_max = io_getevents(s->ctx, MAX_EVENTS, MAX_EVENTS,
116 s->events, &ts);
117 } while (s->event_max == -EINTR);
118
119 s->event_idx = 0;
120 if (s->event_max <= 0) {
121 s->event_max = 0;
122 return; /* no more events */
123 }
124 }
125
126 /* Reschedule so nested event loops see currently pending completions */
127 qemu_bh_schedule(s->completion_bh);
128
129 /* Process completion events */
130 while (s->event_idx < s->event_max) {
131 struct iocb *iocb = s->events[s->event_idx].obj;
132 struct qemu_laiocb *laiocb =
133 container_of(iocb, struct qemu_laiocb, iocb);
134
135 laiocb->ret = io_event_ret(&s->events[s->event_idx]);
136 s->event_idx++;
137
Paolo Bonzinidd7f7ed2016-04-07 18:33:35 +0200138 qemu_laio_process_completion(laiocb);
Stefan Hajnoczi2cdff7f2014-08-04 16:56:33 +0100139 }
Paolo Bonzini28b24082014-12-11 14:52:26 +0100140
141 if (!s->io_q.plugged && !QSIMPLEQ_EMPTY(&s->io_q.pending)) {
142 ioq_submit(s);
143 }
Stefan Hajnoczi2cdff7f2014-08-04 16:56:33 +0100144}
145
Paolo Bonzinic90caf22012-02-24 08:39:02 +0100146static void qemu_laio_completion_cb(EventNotifier *e)
Christoph Hellwig5c6c3a62009-08-20 16:58:35 +0200147{
Paolo Bonzinidd7f7ed2016-04-07 18:33:35 +0200148 LinuxAioState *s = container_of(e, LinuxAioState, e);
Christoph Hellwig5c6c3a62009-08-20 16:58:35 +0200149
Stefan Hajnoczi2cdff7f2014-08-04 16:56:33 +0100150 if (event_notifier_test_and_clear(&s->e)) {
151 qemu_bh_schedule(s->completion_bh);
Christoph Hellwig5c6c3a62009-08-20 16:58:35 +0200152 }
153}
154
Markus Armbruster7c84b1b2014-10-07 13:59:14 +0200155static void laio_cancel(BlockAIOCB *blockacb)
Christoph Hellwig5c6c3a62009-08-20 16:58:35 +0200156{
157 struct qemu_laiocb *laiocb = (struct qemu_laiocb *)blockacb;
158 struct io_event event;
159 int ret;
160
Fam Zheng771b64d2014-09-11 13:41:13 +0800161 if (laiocb->ret != -EINPROGRESS) {
Christoph Hellwig5c6c3a62009-08-20 16:58:35 +0200162 return;
Fam Zheng771b64d2014-09-11 13:41:13 +0800163 }
Christoph Hellwig5c6c3a62009-08-20 16:58:35 +0200164 ret = io_cancel(laiocb->ctx->ctx, &laiocb->iocb, &event);
Fam Zheng771b64d2014-09-11 13:41:13 +0800165 laiocb->ret = -ECANCELED;
166 if (ret != 0) {
167 /* iocb is not cancelled, cb will be called by the event loop later */
Christoph Hellwig5c6c3a62009-08-20 16:58:35 +0200168 return;
169 }
170
Fam Zheng771b64d2014-09-11 13:41:13 +0800171 laiocb->common.cb(laiocb->common.opaque, laiocb->ret);
Christoph Hellwig5c6c3a62009-08-20 16:58:35 +0200172}
173
Stefan Hajnoczid7331be2012-10-31 16:34:37 +0100174static const AIOCBInfo laio_aiocb_info = {
Christoph Hellwig5c6c3a62009-08-20 16:58:35 +0200175 .aiocb_size = sizeof(struct qemu_laiocb),
Fam Zheng771b64d2014-09-11 13:41:13 +0800176 .cancel_async = laio_cancel,
Christoph Hellwig5c6c3a62009-08-20 16:58:35 +0200177};
178
Ming Lei1b3abdc2014-07-04 18:04:34 +0800179static void ioq_init(LaioQueue *io_q)
180{
Paolo Bonzini28b24082014-12-11 14:52:26 +0100181 QSIMPLEQ_INIT(&io_q->pending);
Ming Lei1b3abdc2014-07-04 18:04:34 +0800182 io_q->plugged = 0;
Paolo Bonzini8455ce02014-12-11 14:52:28 +0100183 io_q->n = 0;
Paolo Bonzini43f23762014-12-11 14:52:27 +0100184 io_q->blocked = false;
Ming Lei1b3abdc2014-07-04 18:04:34 +0800185}
186
Paolo Bonzinidd7f7ed2016-04-07 18:33:35 +0200187static void ioq_submit(LinuxAioState *s)
Ming Lei1b3abdc2014-07-04 18:04:34 +0800188{
Paolo Bonzini82595da2014-12-11 14:52:30 +0100189 int ret, len;
Paolo Bonzini28b24082014-12-11 14:52:26 +0100190 struct qemu_laiocb *aiocb;
191 struct iocb *iocbs[MAX_QUEUED_IO];
Paolo Bonzini82595da2014-12-11 14:52:30 +0100192 QSIMPLEQ_HEAD(, qemu_laiocb) completed;
Ming Lei1b3abdc2014-07-04 18:04:34 +0800193
Paolo Bonzini43f23762014-12-11 14:52:27 +0100194 do {
195 len = 0;
196 QSIMPLEQ_FOREACH(aiocb, &s->io_q.pending, next) {
197 iocbs[len++] = &aiocb->iocb;
198 if (len == MAX_QUEUED_IO) {
199 break;
200 }
Paolo Bonzini28b24082014-12-11 14:52:26 +0100201 }
Ming Lei1b3abdc2014-07-04 18:04:34 +0800202
Paolo Bonzini43f23762014-12-11 14:52:27 +0100203 ret = io_submit(s->ctx, len, iocbs);
204 if (ret == -EAGAIN) {
Paolo Bonzini82595da2014-12-11 14:52:30 +0100205 break;
Paolo Bonzini43f23762014-12-11 14:52:27 +0100206 }
207 if (ret < 0) {
208 abort();
209 }
Ming Lei1b3abdc2014-07-04 18:04:34 +0800210
Paolo Bonzini82595da2014-12-11 14:52:30 +0100211 s->io_q.n -= ret;
212 aiocb = container_of(iocbs[ret - 1], struct qemu_laiocb, iocb);
213 QSIMPLEQ_SPLIT_AFTER(&s->io_q.pending, aiocb, next, &completed);
Paolo Bonzini43f23762014-12-11 14:52:27 +0100214 } while (ret == len && !QSIMPLEQ_EMPTY(&s->io_q.pending));
Paolo Bonzini8455ce02014-12-11 14:52:28 +0100215 s->io_q.blocked = (s->io_q.n > 0);
Ming Lei1b3abdc2014-07-04 18:04:34 +0800216}
217
Paolo Bonzinidd7f7ed2016-04-07 18:33:35 +0200218void laio_io_plug(BlockDriverState *bs, LinuxAioState *s)
Ming Lei1b3abdc2014-07-04 18:04:34 +0800219{
Paolo Bonzini6b98bd62016-04-07 18:33:34 +0200220 assert(!s->io_q.plugged);
221 s->io_q.plugged = 1;
Ming Lei1b3abdc2014-07-04 18:04:34 +0800222}
223
Paolo Bonzinidd7f7ed2016-04-07 18:33:35 +0200224void laio_io_unplug(BlockDriverState *bs, LinuxAioState *s)
Ming Lei1b3abdc2014-07-04 18:04:34 +0800225{
Paolo Bonzini6b98bd62016-04-07 18:33:34 +0200226 assert(s->io_q.plugged);
227 s->io_q.plugged = 0;
Paolo Bonzini43f23762014-12-11 14:52:27 +0100228 if (!s->io_q.blocked && !QSIMPLEQ_EMPTY(&s->io_q.pending)) {
Paolo Bonzinide354642014-12-11 14:52:29 +0100229 ioq_submit(s);
Ming Lei1b3abdc2014-07-04 18:04:34 +0800230 }
Ming Lei1b3abdc2014-07-04 18:04:34 +0800231}
232
Paolo Bonzinidd7f7ed2016-04-07 18:33:35 +0200233BlockAIOCB *laio_submit(BlockDriverState *bs, LinuxAioState *s, int fd,
Christoph Hellwig5c6c3a62009-08-20 16:58:35 +0200234 int64_t sector_num, QEMUIOVector *qiov, int nb_sectors,
Markus Armbruster097310b2014-10-07 13:59:15 +0200235 BlockCompletionFunc *cb, void *opaque, int type)
Christoph Hellwig5c6c3a62009-08-20 16:58:35 +0200236{
Christoph Hellwig5c6c3a62009-08-20 16:58:35 +0200237 struct qemu_laiocb *laiocb;
238 struct iocb *iocbs;
239 off_t offset = sector_num * 512;
240
Stefan Hajnoczid7331be2012-10-31 16:34:37 +0100241 laiocb = qemu_aio_get(&laio_aiocb_info, bs, cb, opaque);
Christoph Hellwig5c6c3a62009-08-20 16:58:35 +0200242 laiocb->nbytes = nb_sectors * 512;
243 laiocb->ctx = s;
244 laiocb->ret = -EINPROGRESS;
Kevin Wolfb161e2e2011-10-13 15:42:52 +0200245 laiocb->is_read = (type == QEMU_AIO_READ);
246 laiocb->qiov = qiov;
Christoph Hellwig5c6c3a62009-08-20 16:58:35 +0200247
248 iocbs = &laiocb->iocb;
249
250 switch (type) {
251 case QEMU_AIO_WRITE:
252 io_prep_pwritev(iocbs, fd, qiov->iov, qiov->niov, offset);
253 break;
254 case QEMU_AIO_READ:
255 io_prep_preadv(iocbs, fd, qiov->iov, qiov->niov, offset);
256 break;
Frediano Ziglioc30e6242011-08-30 09:46:11 +0200257 /* Currently Linux kernel does not support other operations */
Christoph Hellwig5c6c3a62009-08-20 16:58:35 +0200258 default:
259 fprintf(stderr, "%s: invalid AIO request type 0x%x.\n",
260 __func__, type);
261 goto out_free_aiocb;
262 }
Paolo Bonzinic90caf22012-02-24 08:39:02 +0100263 io_set_eventfd(&laiocb->iocb, event_notifier_get_fd(&s->e));
Christoph Hellwig5c6c3a62009-08-20 16:58:35 +0200264
Paolo Bonzini28b24082014-12-11 14:52:26 +0100265 QSIMPLEQ_INSERT_TAIL(&s->io_q.pending, laiocb, next);
Paolo Bonzini8455ce02014-12-11 14:52:28 +0100266 s->io_q.n++;
Paolo Bonzini43f23762014-12-11 14:52:27 +0100267 if (!s->io_q.blocked &&
Paolo Bonzini8455ce02014-12-11 14:52:28 +0100268 (!s->io_q.plugged || s->io_q.n >= MAX_QUEUED_IO)) {
Paolo Bonzini28b24082014-12-11 14:52:26 +0100269 ioq_submit(s);
Ming Lei1b3abdc2014-07-04 18:04:34 +0800270 }
Christoph Hellwig5c6c3a62009-08-20 16:58:35 +0200271 return &laiocb->common;
272
Kevin Wolf449c1842011-09-22 14:21:30 +0200273out_free_aiocb:
Fam Zheng80074292014-09-11 13:41:28 +0800274 qemu_aio_unref(laiocb);
Christoph Hellwig5c6c3a62009-08-20 16:58:35 +0200275 return NULL;
276}
277
Paolo Bonzinidd7f7ed2016-04-07 18:33:35 +0200278void laio_detach_aio_context(LinuxAioState *s, AioContext *old_context)
Stefan Hajnoczic2f34262014-05-08 16:34:47 +0200279{
Fam Zhengdca21ef2015-10-23 11:08:05 +0800280 aio_set_event_notifier(old_context, &s->e, false, NULL);
Stefan Hajnoczi2cdff7f2014-08-04 16:56:33 +0100281 qemu_bh_delete(s->completion_bh);
Stefan Hajnoczic2f34262014-05-08 16:34:47 +0200282}
283
Paolo Bonzinidd7f7ed2016-04-07 18:33:35 +0200284void laio_attach_aio_context(LinuxAioState *s, AioContext *new_context)
Stefan Hajnoczic2f34262014-05-08 16:34:47 +0200285{
Stefan Hajnoczi2cdff7f2014-08-04 16:56:33 +0100286 s->completion_bh = aio_bh_new(new_context, qemu_laio_completion_bh, s);
Fam Zhengdca21ef2015-10-23 11:08:05 +0800287 aio_set_event_notifier(new_context, &s->e, false,
288 qemu_laio_completion_cb);
Stefan Hajnoczic2f34262014-05-08 16:34:47 +0200289}
290
Paolo Bonzinidd7f7ed2016-04-07 18:33:35 +0200291LinuxAioState *laio_init(void)
Christoph Hellwig5c6c3a62009-08-20 16:58:35 +0200292{
Paolo Bonzinidd7f7ed2016-04-07 18:33:35 +0200293 LinuxAioState *s;
Christoph Hellwig5c6c3a62009-08-20 16:58:35 +0200294
Anthony Liguori7267c092011-08-20 22:09:37 -0500295 s = g_malloc0(sizeof(*s));
Paolo Bonzinic90caf22012-02-24 08:39:02 +0100296 if (event_notifier_init(&s->e, false) < 0) {
Christoph Hellwig5c6c3a62009-08-20 16:58:35 +0200297 goto out_free_state;
Paolo Bonzinic90caf22012-02-24 08:39:02 +0100298 }
Christoph Hellwig5c6c3a62009-08-20 16:58:35 +0200299
Paolo Bonzinic90caf22012-02-24 08:39:02 +0100300 if (io_setup(MAX_EVENTS, &s->ctx) != 0) {
Christoph Hellwig5c6c3a62009-08-20 16:58:35 +0200301 goto out_close_efd;
Paolo Bonzinic90caf22012-02-24 08:39:02 +0100302 }
Christoph Hellwig5c6c3a62009-08-20 16:58:35 +0200303
Ming Lei1b3abdc2014-07-04 18:04:34 +0800304 ioq_init(&s->io_q);
305
Christoph Hellwig5c6c3a62009-08-20 16:58:35 +0200306 return s;
307
308out_close_efd:
Paolo Bonzinic90caf22012-02-24 08:39:02 +0100309 event_notifier_cleanup(&s->e);
Christoph Hellwig5c6c3a62009-08-20 16:58:35 +0200310out_free_state:
Anthony Liguori7267c092011-08-20 22:09:37 -0500311 g_free(s);
Christoph Hellwig5c6c3a62009-08-20 16:58:35 +0200312 return NULL;
313}
Stefan Hajnocziabd269b2014-05-08 16:34:48 +0200314
Paolo Bonzinidd7f7ed2016-04-07 18:33:35 +0200315void laio_cleanup(LinuxAioState *s)
Stefan Hajnocziabd269b2014-05-08 16:34:48 +0200316{
Stefan Hajnocziabd269b2014-05-08 16:34:48 +0200317 event_notifier_cleanup(&s->e);
Gongleia1abf402014-07-12 11:43:37 +0800318
319 if (io_destroy(s->ctx) != 0) {
320 fprintf(stderr, "%s: destroy AIO context %p failed\n",
321 __func__, &s->ctx);
322 }
Stefan Hajnocziabd269b2014-05-08 16:34:48 +0200323 g_free(s);
324}