blob: 13593d1b428e8273e2d9291f96983db8081eeb75 [file] [log] [blame]
aliguori244ab902009-02-05 21:23:50 +00001/*
2 * DMA helper functions
3 *
4 * Copyright (c) 2009 Red Hat
5 *
6 * This work is licensed under the terms of the GNU General Public License
7 * (GNU GPL), version 2 or later.
8 */
9
10#include "dma.h"
Kevin Wolfc57c4652011-11-24 06:15:28 -050011#include "trace.h"
David Gibsone5332e62012-06-27 14:50:43 +100012#include "range.h"
13#include "qemu-thread.h"
aliguori244ab902009-02-05 21:23:50 +000014
David Gibsone5332e62012-06-27 14:50:43 +100015/* #define DEBUG_IOMMU */
16
17static void do_dma_memory_set(dma_addr_t addr, uint8_t c, dma_addr_t len)
David Gibsond86a77f2012-06-27 14:50:38 +100018{
19#define FILLBUF_SIZE 512
20 uint8_t fillbuf[FILLBUF_SIZE];
21 int l;
22
23 memset(fillbuf, c, FILLBUF_SIZE);
24 while (len > 0) {
25 l = len < FILLBUF_SIZE ? len : FILLBUF_SIZE;
26 cpu_physical_memory_rw(addr, fillbuf, l, true);
27 len -= len;
28 addr += len;
29 }
David Gibsone5332e62012-06-27 14:50:43 +100030}
31
32int dma_memory_set(DMAContext *dma, dma_addr_t addr, uint8_t c, dma_addr_t len)
33{
Benjamin Herrenschmidt7a0bac42012-06-27 14:50:47 +100034 dma_barrier(dma, DMA_DIRECTION_FROM_DEVICE);
35
David Gibsone5332e62012-06-27 14:50:43 +100036 if (dma_has_iommu(dma)) {
37 return iommu_dma_memory_set(dma, addr, c, len);
38 }
39 do_dma_memory_set(addr, c, len);
40
David Gibsond86a77f2012-06-27 14:50:38 +100041 return 0;
42}
43
David Gibsonc65bcef2012-06-27 14:50:40 +100044void qemu_sglist_init(QEMUSGList *qsg, int alloc_hint, DMAContext *dma)
aliguori244ab902009-02-05 21:23:50 +000045{
Anthony Liguori7267c092011-08-20 22:09:37 -050046 qsg->sg = g_malloc(alloc_hint * sizeof(ScatterGatherEntry));
aliguori244ab902009-02-05 21:23:50 +000047 qsg->nsg = 0;
48 qsg->nalloc = alloc_hint;
49 qsg->size = 0;
David Gibsonc65bcef2012-06-27 14:50:40 +100050 qsg->dma = dma;
aliguori244ab902009-02-05 21:23:50 +000051}
52
David Gibsond3231182011-10-31 17:06:46 +110053void qemu_sglist_add(QEMUSGList *qsg, dma_addr_t base, dma_addr_t len)
aliguori244ab902009-02-05 21:23:50 +000054{
55 if (qsg->nsg == qsg->nalloc) {
56 qsg->nalloc = 2 * qsg->nalloc + 1;
Anthony Liguori7267c092011-08-20 22:09:37 -050057 qsg->sg = g_realloc(qsg->sg, qsg->nalloc * sizeof(ScatterGatherEntry));
aliguori244ab902009-02-05 21:23:50 +000058 }
59 qsg->sg[qsg->nsg].base = base;
60 qsg->sg[qsg->nsg].len = len;
61 qsg->size += len;
62 ++qsg->nsg;
63}
64
65void qemu_sglist_destroy(QEMUSGList *qsg)
66{
Anthony Liguori7267c092011-08-20 22:09:37 -050067 g_free(qsg->sg);
Jason Baronea8d82a2012-08-03 15:57:10 -040068 memset(qsg, 0, sizeof(*qsg));
aliguori244ab902009-02-05 21:23:50 +000069}
70
aliguori59a703e2009-02-05 21:23:58 +000071typedef struct {
aliguori37b78422009-03-20 18:26:16 +000072 BlockDriverAIOCB common;
aliguori59a703e2009-02-05 21:23:58 +000073 BlockDriverState *bs;
74 BlockDriverAIOCB *acb;
75 QEMUSGList *sg;
76 uint64_t sector_num;
David Gibson43cf8ae2012-03-27 13:42:23 +110077 DMADirection dir;
Paolo Bonzinic3adb5b2011-09-16 16:40:02 +020078 bool in_cancel;
aliguori59a703e2009-02-05 21:23:58 +000079 int sg_cur_index;
David Gibsond3231182011-10-31 17:06:46 +110080 dma_addr_t sg_cur_byte;
aliguori59a703e2009-02-05 21:23:58 +000081 QEMUIOVector iov;
82 QEMUBH *bh;
Christoph Hellwigcb144cc2011-05-19 10:57:59 +020083 DMAIOFunc *io_func;
aliguori37b78422009-03-20 18:26:16 +000084} DMAAIOCB;
aliguori59a703e2009-02-05 21:23:58 +000085
86static void dma_bdrv_cb(void *opaque, int ret);
87
88static void reschedule_dma(void *opaque)
89{
aliguori37b78422009-03-20 18:26:16 +000090 DMAAIOCB *dbs = (DMAAIOCB *)opaque;
aliguori59a703e2009-02-05 21:23:58 +000091
92 qemu_bh_delete(dbs->bh);
93 dbs->bh = NULL;
Paolo Bonzinic3adb5b2011-09-16 16:40:02 +020094 dma_bdrv_cb(dbs, 0);
aliguori59a703e2009-02-05 21:23:58 +000095}
96
97static void continue_after_map_failure(void *opaque)
98{
aliguori37b78422009-03-20 18:26:16 +000099 DMAAIOCB *dbs = (DMAAIOCB *)opaque;
aliguori59a703e2009-02-05 21:23:58 +0000100
101 dbs->bh = qemu_bh_new(reschedule_dma, dbs);
102 qemu_bh_schedule(dbs->bh);
103}
104
aliguori7403b142009-03-28 16:11:25 +0000105static void dma_bdrv_unmap(DMAAIOCB *dbs)
aliguori59a703e2009-02-05 21:23:58 +0000106{
aliguori59a703e2009-02-05 21:23:58 +0000107 int i;
108
aliguori59a703e2009-02-05 21:23:58 +0000109 for (i = 0; i < dbs->iov.niov; ++i) {
David Gibsonc65bcef2012-06-27 14:50:40 +1000110 dma_memory_unmap(dbs->sg->dma, dbs->iov.iov[i].iov_base,
111 dbs->iov.iov[i].iov_len, dbs->dir,
112 dbs->iov.iov[i].iov_len);
aliguori59a703e2009-02-05 21:23:58 +0000113 }
Paolo Bonzinic3adb5b2011-09-16 16:40:02 +0200114 qemu_iovec_reset(&dbs->iov);
115}
116
117static void dma_complete(DMAAIOCB *dbs, int ret)
118{
Kevin Wolfc57c4652011-11-24 06:15:28 -0500119 trace_dma_complete(dbs, ret, dbs->common.cb);
120
Paolo Bonzinic3adb5b2011-09-16 16:40:02 +0200121 dma_bdrv_unmap(dbs);
122 if (dbs->common.cb) {
123 dbs->common.cb(dbs->common.opaque, ret);
124 }
125 qemu_iovec_destroy(&dbs->iov);
126 if (dbs->bh) {
127 qemu_bh_delete(dbs->bh);
128 dbs->bh = NULL;
129 }
130 if (!dbs->in_cancel) {
131 /* Requests may complete while dma_aio_cancel is in progress. In
132 * this case, the AIOCB should not be released because it is still
133 * referenced by dma_aio_cancel. */
134 qemu_aio_release(dbs);
135 }
aliguori7403b142009-03-28 16:11:25 +0000136}
137
blueswir1856ae5c2009-04-07 17:57:09 +0000138static void dma_bdrv_cb(void *opaque, int ret)
aliguori7403b142009-03-28 16:11:25 +0000139{
140 DMAAIOCB *dbs = (DMAAIOCB *)opaque;
David Gibsonc65bcef2012-06-27 14:50:40 +1000141 dma_addr_t cur_addr, cur_len;
aliguori7403b142009-03-28 16:11:25 +0000142 void *mem;
143
Kevin Wolfc57c4652011-11-24 06:15:28 -0500144 trace_dma_bdrv_cb(dbs, ret);
145
aliguori7403b142009-03-28 16:11:25 +0000146 dbs->acb = NULL;
147 dbs->sector_num += dbs->iov.size / 512;
148 dma_bdrv_unmap(dbs);
aliguori59a703e2009-02-05 21:23:58 +0000149
150 if (dbs->sg_cur_index == dbs->sg->nsg || ret < 0) {
Paolo Bonzinic3adb5b2011-09-16 16:40:02 +0200151 dma_complete(dbs, ret);
aliguori59a703e2009-02-05 21:23:58 +0000152 return;
153 }
154
155 while (dbs->sg_cur_index < dbs->sg->nsg) {
156 cur_addr = dbs->sg->sg[dbs->sg_cur_index].base + dbs->sg_cur_byte;
157 cur_len = dbs->sg->sg[dbs->sg_cur_index].len - dbs->sg_cur_byte;
David Gibsonc65bcef2012-06-27 14:50:40 +1000158 mem = dma_memory_map(dbs->sg->dma, cur_addr, &cur_len, dbs->dir);
aliguori59a703e2009-02-05 21:23:58 +0000159 if (!mem)
160 break;
161 qemu_iovec_add(&dbs->iov, mem, cur_len);
162 dbs->sg_cur_byte += cur_len;
163 if (dbs->sg_cur_byte == dbs->sg->sg[dbs->sg_cur_index].len) {
164 dbs->sg_cur_byte = 0;
165 ++dbs->sg_cur_index;
166 }
167 }
168
169 if (dbs->iov.size == 0) {
Kevin Wolfc57c4652011-11-24 06:15:28 -0500170 trace_dma_map_wait(dbs);
aliguori59a703e2009-02-05 21:23:58 +0000171 cpu_register_map_client(dbs, continue_after_map_failure);
172 return;
173 }
174
Christoph Hellwigcb144cc2011-05-19 10:57:59 +0200175 dbs->acb = dbs->io_func(dbs->bs, dbs->sector_num, &dbs->iov,
176 dbs->iov.size / 512, dma_bdrv_cb, dbs);
Paolo Bonzini6bee44e2011-11-14 17:50:52 +0100177 assert(dbs->acb);
aliguori59a703e2009-02-05 21:23:58 +0000178}
179
Christoph Hellwigc16b5a22009-05-25 12:37:32 +0200180static void dma_aio_cancel(BlockDriverAIOCB *acb)
181{
182 DMAAIOCB *dbs = container_of(acb, DMAAIOCB, common);
183
Kevin Wolfc57c4652011-11-24 06:15:28 -0500184 trace_dma_aio_cancel(dbs);
185
Christoph Hellwigc16b5a22009-05-25 12:37:32 +0200186 if (dbs->acb) {
Paolo Bonzinic3adb5b2011-09-16 16:40:02 +0200187 BlockDriverAIOCB *acb = dbs->acb;
188 dbs->acb = NULL;
189 dbs->in_cancel = true;
190 bdrv_aio_cancel(acb);
191 dbs->in_cancel = false;
Christoph Hellwigc16b5a22009-05-25 12:37:32 +0200192 }
Paolo Bonzinic3adb5b2011-09-16 16:40:02 +0200193 dbs->common.cb = NULL;
194 dma_complete(dbs, 0);
Christoph Hellwigc16b5a22009-05-25 12:37:32 +0200195}
196
197static AIOPool dma_aio_pool = {
198 .aiocb_size = sizeof(DMAAIOCB),
199 .cancel = dma_aio_cancel,
200};
201
Christoph Hellwigcb144cc2011-05-19 10:57:59 +0200202BlockDriverAIOCB *dma_bdrv_io(
aliguori59a703e2009-02-05 21:23:58 +0000203 BlockDriverState *bs, QEMUSGList *sg, uint64_t sector_num,
Christoph Hellwigcb144cc2011-05-19 10:57:59 +0200204 DMAIOFunc *io_func, BlockDriverCompletionFunc *cb,
David Gibson43cf8ae2012-03-27 13:42:23 +1100205 void *opaque, DMADirection dir)
aliguori59a703e2009-02-05 21:23:58 +0000206{
Christoph Hellwigcb144cc2011-05-19 10:57:59 +0200207 DMAAIOCB *dbs = qemu_aio_get(&dma_aio_pool, bs, cb, opaque);
aliguori59a703e2009-02-05 21:23:58 +0000208
David Gibson43cf8ae2012-03-27 13:42:23 +1100209 trace_dma_bdrv_io(dbs, bs, sector_num, (dir == DMA_DIRECTION_TO_DEVICE));
Kevin Wolfc57c4652011-11-24 06:15:28 -0500210
aliguori37b78422009-03-20 18:26:16 +0000211 dbs->acb = NULL;
aliguori59a703e2009-02-05 21:23:58 +0000212 dbs->bs = bs;
aliguori59a703e2009-02-05 21:23:58 +0000213 dbs->sg = sg;
214 dbs->sector_num = sector_num;
215 dbs->sg_cur_index = 0;
216 dbs->sg_cur_byte = 0;
David Gibson43cf8ae2012-03-27 13:42:23 +1100217 dbs->dir = dir;
Christoph Hellwigcb144cc2011-05-19 10:57:59 +0200218 dbs->io_func = io_func;
aliguori59a703e2009-02-05 21:23:58 +0000219 dbs->bh = NULL;
220 qemu_iovec_init(&dbs->iov, sg->nsg);
221 dma_bdrv_cb(dbs, 0);
aliguori37b78422009-03-20 18:26:16 +0000222 return &dbs->common;
aliguori59a703e2009-02-05 21:23:58 +0000223}
224
225
226BlockDriverAIOCB *dma_bdrv_read(BlockDriverState *bs,
227 QEMUSGList *sg, uint64_t sector,
228 void (*cb)(void *opaque, int ret), void *opaque)
229{
David Gibson43cf8ae2012-03-27 13:42:23 +1100230 return dma_bdrv_io(bs, sg, sector, bdrv_aio_readv, cb, opaque,
231 DMA_DIRECTION_FROM_DEVICE);
aliguori59a703e2009-02-05 21:23:58 +0000232}
233
234BlockDriverAIOCB *dma_bdrv_write(BlockDriverState *bs,
235 QEMUSGList *sg, uint64_t sector,
236 void (*cb)(void *opaque, int ret), void *opaque)
237{
David Gibson43cf8ae2012-03-27 13:42:23 +1100238 return dma_bdrv_io(bs, sg, sector, bdrv_aio_writev, cb, opaque,
239 DMA_DIRECTION_TO_DEVICE);
aliguori59a703e2009-02-05 21:23:58 +0000240}
Paolo Bonzini8171ee32011-07-06 08:02:14 +0200241
242
David Gibsonc65bcef2012-06-27 14:50:40 +1000243static uint64_t dma_buf_rw(uint8_t *ptr, int32_t len, QEMUSGList *sg,
244 DMADirection dir)
Paolo Bonzini8171ee32011-07-06 08:02:14 +0200245{
246 uint64_t resid;
247 int sg_cur_index;
248
249 resid = sg->size;
250 sg_cur_index = 0;
251 len = MIN(len, resid);
252 while (len > 0) {
253 ScatterGatherEntry entry = sg->sg[sg_cur_index++];
254 int32_t xfer = MIN(len, entry.len);
David Gibsonc65bcef2012-06-27 14:50:40 +1000255 dma_memory_rw(sg->dma, entry.base, ptr, xfer, dir);
Paolo Bonzini8171ee32011-07-06 08:02:14 +0200256 ptr += xfer;
257 len -= xfer;
258 resid -= xfer;
259 }
260
261 return resid;
262}
263
264uint64_t dma_buf_read(uint8_t *ptr, int32_t len, QEMUSGList *sg)
265{
David Gibsonc65bcef2012-06-27 14:50:40 +1000266 return dma_buf_rw(ptr, len, sg, DMA_DIRECTION_FROM_DEVICE);
Paolo Bonzini8171ee32011-07-06 08:02:14 +0200267}
268
269uint64_t dma_buf_write(uint8_t *ptr, int32_t len, QEMUSGList *sg)
270{
David Gibsonc65bcef2012-06-27 14:50:40 +1000271 return dma_buf_rw(ptr, len, sg, DMA_DIRECTION_TO_DEVICE);
Paolo Bonzini8171ee32011-07-06 08:02:14 +0200272}
Paolo Bonzini84a69352011-09-05 14:20:29 +0200273
274void dma_acct_start(BlockDriverState *bs, BlockAcctCookie *cookie,
275 QEMUSGList *sg, enum BlockAcctType type)
276{
277 bdrv_acct_start(bs, cookie, sg->size, type);
278}
David Gibsone5332e62012-06-27 14:50:43 +1000279
280bool iommu_dma_memory_valid(DMAContext *dma, dma_addr_t addr, dma_addr_t len,
281 DMADirection dir)
282{
283 target_phys_addr_t paddr, plen;
284
285#ifdef DEBUG_IOMMU
286 fprintf(stderr, "dma_memory_check context=%p addr=0x" DMA_ADDR_FMT
287 " len=0x" DMA_ADDR_FMT " dir=%d\n", dma, addr, len, dir);
288#endif
289
290 while (len) {
291 if (dma->translate(dma, addr, &paddr, &plen, dir) != 0) {
292 return false;
293 }
294
295 /* The translation might be valid for larger regions. */
296 if (plen > len) {
297 plen = len;
298 }
299
300 len -= plen;
301 addr += plen;
302 }
303
304 return true;
305}
306
307int iommu_dma_memory_rw(DMAContext *dma, dma_addr_t addr,
308 void *buf, dma_addr_t len, DMADirection dir)
309{
310 target_phys_addr_t paddr, plen;
311 int err;
312
313#ifdef DEBUG_IOMMU
314 fprintf(stderr, "dma_memory_rw context=%p addr=0x" DMA_ADDR_FMT " len=0x"
315 DMA_ADDR_FMT " dir=%d\n", dma, addr, len, dir);
316#endif
317
318 while (len) {
319 err = dma->translate(dma, addr, &paddr, &plen, dir);
320 if (err) {
321 /*
322 * In case of failure on reads from the guest, we clean the
323 * destination buffer so that a device that doesn't test
324 * for errors will not expose qemu internal memory.
325 */
326 memset(buf, 0, len);
327 return -1;
328 }
329
330 /* The translation might be valid for larger regions. */
331 if (plen > len) {
332 plen = len;
333 }
334
335 cpu_physical_memory_rw(paddr, buf, plen,
336 dir == DMA_DIRECTION_FROM_DEVICE);
337
338 len -= plen;
339 addr += plen;
340 buf += plen;
341 }
342
343 return 0;
344}
345
346int iommu_dma_memory_set(DMAContext *dma, dma_addr_t addr, uint8_t c,
347 dma_addr_t len)
348{
349 target_phys_addr_t paddr, plen;
350 int err;
351
352#ifdef DEBUG_IOMMU
353 fprintf(stderr, "dma_memory_set context=%p addr=0x" DMA_ADDR_FMT
354 " len=0x" DMA_ADDR_FMT "\n", dma, addr, len);
355#endif
356
357 while (len) {
358 err = dma->translate(dma, addr, &paddr, &plen,
359 DMA_DIRECTION_FROM_DEVICE);
360 if (err) {
361 return err;
362 }
363
364 /* The translation might be valid for larger regions. */
365 if (plen > len) {
366 plen = len;
367 }
368
369 do_dma_memory_set(paddr, c, plen);
370
371 len -= plen;
372 addr += plen;
373 }
374
375 return 0;
376}
377
378void dma_context_init(DMAContext *dma, DMATranslateFunc translate,
379 DMAMapFunc map, DMAUnmapFunc unmap)
380{
381#ifdef DEBUG_IOMMU
382 fprintf(stderr, "dma_context_init(%p, %p, %p, %p)\n",
383 dma, translate, map, unmap);
384#endif
385 dma->translate = translate;
386 dma->map = map;
387 dma->unmap = unmap;
388}
389
390void *iommu_dma_memory_map(DMAContext *dma, dma_addr_t addr, dma_addr_t *len,
391 DMADirection dir)
392{
393 int err;
394 target_phys_addr_t paddr, plen;
395 void *buf;
396
397 if (dma->map) {
398 return dma->map(dma, addr, len, dir);
399 }
400
401 plen = *len;
402 err = dma->translate(dma, addr, &paddr, &plen, dir);
403 if (err) {
404 return NULL;
405 }
406
407 /*
408 * If this is true, the virtual region is contiguous,
409 * but the translated physical region isn't. We just
410 * clamp *len, much like cpu_physical_memory_map() does.
411 */
412 if (plen < *len) {
413 *len = plen;
414 }
415
416 buf = cpu_physical_memory_map(paddr, &plen,
417 dir == DMA_DIRECTION_FROM_DEVICE);
418 *len = plen;
419
420 return buf;
421}
422
423void iommu_dma_memory_unmap(DMAContext *dma, void *buffer, dma_addr_t len,
424 DMADirection dir, dma_addr_t access_len)
425{
426 if (dma->unmap) {
427 dma->unmap(dma, buffer, len, dir, access_len);
428 return;
429 }
430
431 cpu_physical_memory_unmap(buffer, len,
432 dir == DMA_DIRECTION_FROM_DEVICE,
433 access_len);
434
435}