blob: c9620a5bbd52ee8211b5f0faa8ab35690a60910d [file] [log] [blame]
aliguori244ab902009-02-05 21:23:50 +00001/*
2 * DMA helper functions
3 *
4 * Copyright (c) 2009 Red Hat
5 *
6 * This work is licensed under the terms of the GNU General Public License
7 * (GNU GPL), version 2 or later.
8 */
9
Paolo Bonzini9c17d612012-12-17 18:20:04 +010010#include "sysemu/dma.h"
Kevin Wolfc57c4652011-11-24 06:15:28 -050011#include "trace.h"
Paolo Bonzini1de7afc2012-12-17 18:20:00 +010012#include "qemu/range.h"
13#include "qemu/thread.h"
Alex Bligh6a1751b2013-08-21 16:02:47 +010014#include "qemu/main-loop.h"
aliguori244ab902009-02-05 21:23:50 +000015
David Gibsone5332e62012-06-27 14:50:43 +100016/* #define DEBUG_IOMMU */
17
Paolo Bonzinidf32fd12013-04-10 18:15:49 +020018int dma_memory_set(AddressSpace *as, dma_addr_t addr, uint8_t c, dma_addr_t len)
David Gibsond86a77f2012-06-27 14:50:38 +100019{
Paolo Bonzinidf32fd12013-04-10 18:15:49 +020020 dma_barrier(as, DMA_DIRECTION_FROM_DEVICE);
Paolo Bonzini24addbc2013-04-10 17:49:04 +020021
David Gibsond86a77f2012-06-27 14:50:38 +100022#define FILLBUF_SIZE 512
23 uint8_t fillbuf[FILLBUF_SIZE];
24 int l;
Paolo Bonzini24addbc2013-04-10 17:49:04 +020025 bool error = false;
David Gibsond86a77f2012-06-27 14:50:38 +100026
27 memset(fillbuf, c, FILLBUF_SIZE);
28 while (len > 0) {
29 l = len < FILLBUF_SIZE ? len : FILLBUF_SIZE;
Paolo Bonzini24addbc2013-04-10 17:49:04 +020030 error |= address_space_rw(as, addr, fillbuf, l, true);
Benjamin Herrenschmidtbc9b78d2012-08-14 17:41:47 +100031 len -= l;
32 addr += l;
David Gibsond86a77f2012-06-27 14:50:38 +100033 }
David Gibsone5332e62012-06-27 14:50:43 +100034
Paolo Bonzini24addbc2013-04-10 17:49:04 +020035 return error;
David Gibsond86a77f2012-06-27 14:50:38 +100036}
37
Paolo Bonzinif487b672013-06-03 14:17:19 +020038void qemu_sglist_init(QEMUSGList *qsg, DeviceState *dev, int alloc_hint,
39 AddressSpace *as)
aliguori244ab902009-02-05 21:23:50 +000040{
Anthony Liguori7267c092011-08-20 22:09:37 -050041 qsg->sg = g_malloc(alloc_hint * sizeof(ScatterGatherEntry));
aliguori244ab902009-02-05 21:23:50 +000042 qsg->nsg = 0;
43 qsg->nalloc = alloc_hint;
44 qsg->size = 0;
Paolo Bonzinidf32fd12013-04-10 18:15:49 +020045 qsg->as = as;
Paolo Bonzinif487b672013-06-03 14:17:19 +020046 qsg->dev = dev;
47 object_ref(OBJECT(dev));
aliguori244ab902009-02-05 21:23:50 +000048}
49
David Gibsond3231182011-10-31 17:06:46 +110050void qemu_sglist_add(QEMUSGList *qsg, dma_addr_t base, dma_addr_t len)
aliguori244ab902009-02-05 21:23:50 +000051{
52 if (qsg->nsg == qsg->nalloc) {
53 qsg->nalloc = 2 * qsg->nalloc + 1;
Anthony Liguori7267c092011-08-20 22:09:37 -050054 qsg->sg = g_realloc(qsg->sg, qsg->nalloc * sizeof(ScatterGatherEntry));
aliguori244ab902009-02-05 21:23:50 +000055 }
56 qsg->sg[qsg->nsg].base = base;
57 qsg->sg[qsg->nsg].len = len;
58 qsg->size += len;
59 ++qsg->nsg;
60}
61
62void qemu_sglist_destroy(QEMUSGList *qsg)
63{
Paolo Bonzinif487b672013-06-03 14:17:19 +020064 object_unref(OBJECT(qsg->dev));
Anthony Liguori7267c092011-08-20 22:09:37 -050065 g_free(qsg->sg);
Jason Baronea8d82a2012-08-03 15:57:10 -040066 memset(qsg, 0, sizeof(*qsg));
aliguori244ab902009-02-05 21:23:50 +000067}
68
aliguori59a703e2009-02-05 21:23:58 +000069typedef struct {
aliguori37b78422009-03-20 18:26:16 +000070 BlockDriverAIOCB common;
aliguori59a703e2009-02-05 21:23:58 +000071 BlockDriverState *bs;
72 BlockDriverAIOCB *acb;
73 QEMUSGList *sg;
74 uint64_t sector_num;
David Gibson43cf8ae2012-03-27 13:42:23 +110075 DMADirection dir;
Paolo Bonzinic3adb5b2011-09-16 16:40:02 +020076 bool in_cancel;
aliguori59a703e2009-02-05 21:23:58 +000077 int sg_cur_index;
David Gibsond3231182011-10-31 17:06:46 +110078 dma_addr_t sg_cur_byte;
aliguori59a703e2009-02-05 21:23:58 +000079 QEMUIOVector iov;
80 QEMUBH *bh;
Christoph Hellwigcb144cc2011-05-19 10:57:59 +020081 DMAIOFunc *io_func;
aliguori37b78422009-03-20 18:26:16 +000082} DMAAIOCB;
aliguori59a703e2009-02-05 21:23:58 +000083
84static void dma_bdrv_cb(void *opaque, int ret);
85
86static void reschedule_dma(void *opaque)
87{
aliguori37b78422009-03-20 18:26:16 +000088 DMAAIOCB *dbs = (DMAAIOCB *)opaque;
aliguori59a703e2009-02-05 21:23:58 +000089
90 qemu_bh_delete(dbs->bh);
91 dbs->bh = NULL;
Paolo Bonzinic3adb5b2011-09-16 16:40:02 +020092 dma_bdrv_cb(dbs, 0);
aliguori59a703e2009-02-05 21:23:58 +000093}
94
95static void continue_after_map_failure(void *opaque)
96{
aliguori37b78422009-03-20 18:26:16 +000097 DMAAIOCB *dbs = (DMAAIOCB *)opaque;
aliguori59a703e2009-02-05 21:23:58 +000098
99 dbs->bh = qemu_bh_new(reschedule_dma, dbs);
100 qemu_bh_schedule(dbs->bh);
101}
102
aliguori7403b142009-03-28 16:11:25 +0000103static void dma_bdrv_unmap(DMAAIOCB *dbs)
aliguori59a703e2009-02-05 21:23:58 +0000104{
aliguori59a703e2009-02-05 21:23:58 +0000105 int i;
106
aliguori59a703e2009-02-05 21:23:58 +0000107 for (i = 0; i < dbs->iov.niov; ++i) {
Paolo Bonzinidf32fd12013-04-10 18:15:49 +0200108 dma_memory_unmap(dbs->sg->as, dbs->iov.iov[i].iov_base,
David Gibsonc65bcef2012-06-27 14:50:40 +1000109 dbs->iov.iov[i].iov_len, dbs->dir,
110 dbs->iov.iov[i].iov_len);
aliguori59a703e2009-02-05 21:23:58 +0000111 }
Paolo Bonzinic3adb5b2011-09-16 16:40:02 +0200112 qemu_iovec_reset(&dbs->iov);
113}
114
115static void dma_complete(DMAAIOCB *dbs, int ret)
116{
Kevin Wolfc57c4652011-11-24 06:15:28 -0500117 trace_dma_complete(dbs, ret, dbs->common.cb);
118
Paolo Bonzinic3adb5b2011-09-16 16:40:02 +0200119 dma_bdrv_unmap(dbs);
120 if (dbs->common.cb) {
121 dbs->common.cb(dbs->common.opaque, ret);
122 }
123 qemu_iovec_destroy(&dbs->iov);
124 if (dbs->bh) {
125 qemu_bh_delete(dbs->bh);
126 dbs->bh = NULL;
127 }
128 if (!dbs->in_cancel) {
129 /* Requests may complete while dma_aio_cancel is in progress. In
130 * this case, the AIOCB should not be released because it is still
131 * referenced by dma_aio_cancel. */
132 qemu_aio_release(dbs);
133 }
aliguori7403b142009-03-28 16:11:25 +0000134}
135
blueswir1856ae5c2009-04-07 17:57:09 +0000136static void dma_bdrv_cb(void *opaque, int ret)
aliguori7403b142009-03-28 16:11:25 +0000137{
138 DMAAIOCB *dbs = (DMAAIOCB *)opaque;
David Gibsonc65bcef2012-06-27 14:50:40 +1000139 dma_addr_t cur_addr, cur_len;
aliguori7403b142009-03-28 16:11:25 +0000140 void *mem;
141
Kevin Wolfc57c4652011-11-24 06:15:28 -0500142 trace_dma_bdrv_cb(dbs, ret);
143
aliguori7403b142009-03-28 16:11:25 +0000144 dbs->acb = NULL;
145 dbs->sector_num += dbs->iov.size / 512;
146 dma_bdrv_unmap(dbs);
aliguori59a703e2009-02-05 21:23:58 +0000147
148 if (dbs->sg_cur_index == dbs->sg->nsg || ret < 0) {
Paolo Bonzinic3adb5b2011-09-16 16:40:02 +0200149 dma_complete(dbs, ret);
aliguori59a703e2009-02-05 21:23:58 +0000150 return;
151 }
152
153 while (dbs->sg_cur_index < dbs->sg->nsg) {
154 cur_addr = dbs->sg->sg[dbs->sg_cur_index].base + dbs->sg_cur_byte;
155 cur_len = dbs->sg->sg[dbs->sg_cur_index].len - dbs->sg_cur_byte;
Paolo Bonzinidf32fd12013-04-10 18:15:49 +0200156 mem = dma_memory_map(dbs->sg->as, cur_addr, &cur_len, dbs->dir);
aliguori59a703e2009-02-05 21:23:58 +0000157 if (!mem)
158 break;
159 qemu_iovec_add(&dbs->iov, mem, cur_len);
160 dbs->sg_cur_byte += cur_len;
161 if (dbs->sg_cur_byte == dbs->sg->sg[dbs->sg_cur_index].len) {
162 dbs->sg_cur_byte = 0;
163 ++dbs->sg_cur_index;
164 }
165 }
166
167 if (dbs->iov.size == 0) {
Kevin Wolfc57c4652011-11-24 06:15:28 -0500168 trace_dma_map_wait(dbs);
aliguori59a703e2009-02-05 21:23:58 +0000169 cpu_register_map_client(dbs, continue_after_map_failure);
170 return;
171 }
172
Christoph Hellwigcb144cc2011-05-19 10:57:59 +0200173 dbs->acb = dbs->io_func(dbs->bs, dbs->sector_num, &dbs->iov,
174 dbs->iov.size / 512, dma_bdrv_cb, dbs);
Paolo Bonzini6bee44e2011-11-14 17:50:52 +0100175 assert(dbs->acb);
aliguori59a703e2009-02-05 21:23:58 +0000176}
177
Christoph Hellwigc16b5a22009-05-25 12:37:32 +0200178static void dma_aio_cancel(BlockDriverAIOCB *acb)
179{
180 DMAAIOCB *dbs = container_of(acb, DMAAIOCB, common);
181
Kevin Wolfc57c4652011-11-24 06:15:28 -0500182 trace_dma_aio_cancel(dbs);
183
Christoph Hellwigc16b5a22009-05-25 12:37:32 +0200184 if (dbs->acb) {
Paolo Bonzinic3adb5b2011-09-16 16:40:02 +0200185 BlockDriverAIOCB *acb = dbs->acb;
186 dbs->acb = NULL;
187 dbs->in_cancel = true;
188 bdrv_aio_cancel(acb);
189 dbs->in_cancel = false;
Christoph Hellwigc16b5a22009-05-25 12:37:32 +0200190 }
Paolo Bonzinic3adb5b2011-09-16 16:40:02 +0200191 dbs->common.cb = NULL;
192 dma_complete(dbs, 0);
Christoph Hellwigc16b5a22009-05-25 12:37:32 +0200193}
194
Stefan Hajnoczid7331be2012-10-31 16:34:37 +0100195static const AIOCBInfo dma_aiocb_info = {
Christoph Hellwigc16b5a22009-05-25 12:37:32 +0200196 .aiocb_size = sizeof(DMAAIOCB),
197 .cancel = dma_aio_cancel,
198};
199
Christoph Hellwigcb144cc2011-05-19 10:57:59 +0200200BlockDriverAIOCB *dma_bdrv_io(
aliguori59a703e2009-02-05 21:23:58 +0000201 BlockDriverState *bs, QEMUSGList *sg, uint64_t sector_num,
Christoph Hellwigcb144cc2011-05-19 10:57:59 +0200202 DMAIOFunc *io_func, BlockDriverCompletionFunc *cb,
David Gibson43cf8ae2012-03-27 13:42:23 +1100203 void *opaque, DMADirection dir)
aliguori59a703e2009-02-05 21:23:58 +0000204{
Stefan Hajnoczid7331be2012-10-31 16:34:37 +0100205 DMAAIOCB *dbs = qemu_aio_get(&dma_aiocb_info, bs, cb, opaque);
aliguori59a703e2009-02-05 21:23:58 +0000206
David Gibson43cf8ae2012-03-27 13:42:23 +1100207 trace_dma_bdrv_io(dbs, bs, sector_num, (dir == DMA_DIRECTION_TO_DEVICE));
Kevin Wolfc57c4652011-11-24 06:15:28 -0500208
aliguori37b78422009-03-20 18:26:16 +0000209 dbs->acb = NULL;
aliguori59a703e2009-02-05 21:23:58 +0000210 dbs->bs = bs;
aliguori59a703e2009-02-05 21:23:58 +0000211 dbs->sg = sg;
212 dbs->sector_num = sector_num;
213 dbs->sg_cur_index = 0;
214 dbs->sg_cur_byte = 0;
David Gibson43cf8ae2012-03-27 13:42:23 +1100215 dbs->dir = dir;
Christoph Hellwigcb144cc2011-05-19 10:57:59 +0200216 dbs->io_func = io_func;
aliguori59a703e2009-02-05 21:23:58 +0000217 dbs->bh = NULL;
218 qemu_iovec_init(&dbs->iov, sg->nsg);
219 dma_bdrv_cb(dbs, 0);
aliguori37b78422009-03-20 18:26:16 +0000220 return &dbs->common;
aliguori59a703e2009-02-05 21:23:58 +0000221}
222
223
224BlockDriverAIOCB *dma_bdrv_read(BlockDriverState *bs,
225 QEMUSGList *sg, uint64_t sector,
226 void (*cb)(void *opaque, int ret), void *opaque)
227{
David Gibson43cf8ae2012-03-27 13:42:23 +1100228 return dma_bdrv_io(bs, sg, sector, bdrv_aio_readv, cb, opaque,
229 DMA_DIRECTION_FROM_DEVICE);
aliguori59a703e2009-02-05 21:23:58 +0000230}
231
232BlockDriverAIOCB *dma_bdrv_write(BlockDriverState *bs,
233 QEMUSGList *sg, uint64_t sector,
234 void (*cb)(void *opaque, int ret), void *opaque)
235{
David Gibson43cf8ae2012-03-27 13:42:23 +1100236 return dma_bdrv_io(bs, sg, sector, bdrv_aio_writev, cb, opaque,
237 DMA_DIRECTION_TO_DEVICE);
aliguori59a703e2009-02-05 21:23:58 +0000238}
Paolo Bonzini8171ee32011-07-06 08:02:14 +0200239
240
David Gibsonc65bcef2012-06-27 14:50:40 +1000241static uint64_t dma_buf_rw(uint8_t *ptr, int32_t len, QEMUSGList *sg,
242 DMADirection dir)
Paolo Bonzini8171ee32011-07-06 08:02:14 +0200243{
244 uint64_t resid;
245 int sg_cur_index;
246
247 resid = sg->size;
248 sg_cur_index = 0;
249 len = MIN(len, resid);
250 while (len > 0) {
251 ScatterGatherEntry entry = sg->sg[sg_cur_index++];
252 int32_t xfer = MIN(len, entry.len);
Paolo Bonzinidf32fd12013-04-10 18:15:49 +0200253 dma_memory_rw(sg->as, entry.base, ptr, xfer, dir);
Paolo Bonzini8171ee32011-07-06 08:02:14 +0200254 ptr += xfer;
255 len -= xfer;
256 resid -= xfer;
257 }
258
259 return resid;
260}
261
262uint64_t dma_buf_read(uint8_t *ptr, int32_t len, QEMUSGList *sg)
263{
David Gibsonc65bcef2012-06-27 14:50:40 +1000264 return dma_buf_rw(ptr, len, sg, DMA_DIRECTION_FROM_DEVICE);
Paolo Bonzini8171ee32011-07-06 08:02:14 +0200265}
266
267uint64_t dma_buf_write(uint8_t *ptr, int32_t len, QEMUSGList *sg)
268{
David Gibsonc65bcef2012-06-27 14:50:40 +1000269 return dma_buf_rw(ptr, len, sg, DMA_DIRECTION_TO_DEVICE);
Paolo Bonzini8171ee32011-07-06 08:02:14 +0200270}
Paolo Bonzini84a69352011-09-05 14:20:29 +0200271
272void dma_acct_start(BlockDriverState *bs, BlockAcctCookie *cookie,
273 QEMUSGList *sg, enum BlockAcctType type)
274{
275 bdrv_acct_start(bs, cookie, sg->size, type);
276}