blob: 2295928d3371c599d0fa1d9633245f466da89817 [file] [log] [blame]
Kevin Wolf00dccaf2011-01-17 16:08:14 +00001/*
2 * QEMU coroutines
3 *
4 * Copyright IBM, Corp. 2011
5 *
6 * Authors:
7 * Stefan Hajnoczi <stefanha@linux.vnet.ibm.com>
8 * Kevin Wolf <kwolf@redhat.com>
9 *
10 * This work is licensed under the terms of the GNU LGPL, version 2 or later.
11 * See the COPYING.LIB file in the top-level directory.
12 *
13 */
14
Peter Maydellaafd7582016-01-29 17:49:55 +000015#include "qemu/osdep.h"
Kevin Wolf00dccaf2011-01-17 16:08:14 +000016#include "trace.h"
17#include "qemu-common.h"
Stefan Hajnoczib84c4582013-05-17 15:51:25 +020018#include "qemu/thread.h"
Paolo Bonzini4d68e862014-12-02 12:05:48 +010019#include "qemu/atomic.h"
Daniel P. Berrange10817bf2015-09-01 14:48:02 +010020#include "qemu/coroutine.h"
21#include "qemu/coroutine_int.h"
Paolo Bonzini0c330a72017-02-13 14:52:19 +010022#include "block/aio.h"
Kevin Wolf00dccaf2011-01-17 16:08:14 +000023
Paolo Bonzini40239782013-02-19 11:59:09 +010024enum {
Paolo Bonzini4d68e862014-12-02 12:05:48 +010025 POOL_BATCH_SIZE = 64,
Paolo Bonzini40239782013-02-19 11:59:09 +010026};
27
28/** Free list to speed up creation */
Paolo Bonzini4d68e862014-12-02 12:05:48 +010029static QSLIST_HEAD(, Coroutine) release_pool = QSLIST_HEAD_INITIALIZER(pool);
30static unsigned int release_pool_size;
31static __thread QSLIST_HEAD(, Coroutine) alloc_pool = QSLIST_HEAD_INITIALIZER(pool);
Peter Lieven51a22192014-12-02 12:05:50 +010032static __thread unsigned int alloc_pool_size;
Paolo Bonzini4d68e862014-12-02 12:05:48 +010033static __thread Notifier coroutine_pool_cleanup_notifier;
34
35static void coroutine_pool_cleanup(Notifier *n, void *value)
36{
37 Coroutine *co;
38 Coroutine *tmp;
39
40 QSLIST_FOREACH_SAFE(co, &alloc_pool, pool_next, tmp) {
41 QSLIST_REMOVE_HEAD(&alloc_pool, pool_next);
42 qemu_coroutine_delete(co);
43 }
44}
Paolo Bonzini40239782013-02-19 11:59:09 +010045
Paolo Bonzini0b8b8752016-07-04 19:10:01 +020046Coroutine *qemu_coroutine_create(CoroutineEntry *entry, void *opaque)
Kevin Wolf00dccaf2011-01-17 16:08:14 +000047{
Stefan Hajnoczi70c60c02013-09-11 16:42:35 +020048 Coroutine *co = NULL;
Paolo Bonzini40239782013-02-19 11:59:09 +010049
Stefan Hajnoczi70c60c02013-09-11 16:42:35 +020050 if (CONFIG_COROUTINE_POOL) {
Paolo Bonzini4d68e862014-12-02 12:05:48 +010051 co = QSLIST_FIRST(&alloc_pool);
52 if (!co) {
53 if (release_pool_size > POOL_BATCH_SIZE) {
54 /* Slow path; a good place to register the destructor, too. */
55 if (!coroutine_pool_cleanup_notifier.notify) {
56 coroutine_pool_cleanup_notifier.notify = coroutine_pool_cleanup;
57 qemu_thread_atexit_add(&coroutine_pool_cleanup_notifier);
58 }
59
60 /* This is not exact; there could be a little skew between
61 * release_pool_size and the actual size of release_pool. But
62 * it is just a heuristic, it does not need to be perfect.
63 */
Peter Lieven51a22192014-12-02 12:05:50 +010064 alloc_pool_size = atomic_xchg(&release_pool_size, 0);
Paolo Bonzini4d68e862014-12-02 12:05:48 +010065 QSLIST_MOVE_ATOMIC(&alloc_pool, &release_pool);
66 co = QSLIST_FIRST(&alloc_pool);
67 }
Stefan Hajnoczi70c60c02013-09-11 16:42:35 +020068 }
Paolo Bonzini4d68e862014-12-02 12:05:48 +010069 if (co) {
70 QSLIST_REMOVE_HEAD(&alloc_pool, pool_next);
Peter Lieven51a22192014-12-02 12:05:50 +010071 alloc_pool_size--;
Paolo Bonzini4d68e862014-12-02 12:05:48 +010072 }
Stefan Hajnoczib84c4582013-05-17 15:51:25 +020073 }
Stefan Hajnoczib84c4582013-05-17 15:51:25 +020074
75 if (!co) {
Paolo Bonzini40239782013-02-19 11:59:09 +010076 co = qemu_coroutine_new();
77 }
78
Kevin Wolf00dccaf2011-01-17 16:08:14 +000079 co->entry = entry;
Paolo Bonzini0b8b8752016-07-04 19:10:01 +020080 co->entry_arg = opaque;
Paolo Bonzini7d9c8582016-07-04 19:09:59 +020081 QSIMPLEQ_INIT(&co->co_queue_wakeup);
Kevin Wolf00dccaf2011-01-17 16:08:14 +000082 return co;
83}
84
Paolo Bonzini40239782013-02-19 11:59:09 +010085static void coroutine_delete(Coroutine *co)
86{
Paolo Bonzini4d68e862014-12-02 12:05:48 +010087 co->caller = NULL;
88
Stefan Hajnoczi70c60c02013-09-11 16:42:35 +020089 if (CONFIG_COROUTINE_POOL) {
Paolo Bonzini4d68e862014-12-02 12:05:48 +010090 if (release_pool_size < POOL_BATCH_SIZE * 2) {
91 QSLIST_INSERT_HEAD_ATOMIC(&release_pool, co, pool_next);
92 atomic_inc(&release_pool_size);
Stefan Hajnoczi70c60c02013-09-11 16:42:35 +020093 return;
94 }
Peter Lieven51a22192014-12-02 12:05:50 +010095 if (alloc_pool_size < POOL_BATCH_SIZE) {
96 QSLIST_INSERT_HEAD(&alloc_pool, co, pool_next);
97 alloc_pool_size++;
98 return;
99 }
Paolo Bonzini40239782013-02-19 11:59:09 +0100100 }
101
102 qemu_coroutine_delete(co);
103}
104
Fam Zhengba9e75c2017-04-10 20:06:12 +0800105void qemu_aio_coroutine_enter(AioContext *ctx, Coroutine *co)
Kevin Wolf00dccaf2011-01-17 16:08:14 +0000106{
Stefan Hajnoczic40a2542018-03-22 15:28:33 +0000107 QSIMPLEQ_HEAD(, Coroutine) pending = QSIMPLEQ_HEAD_INITIALIZER(pending);
108 Coroutine *from = qemu_coroutine_self();
Kevin Wolf00dccaf2011-01-17 16:08:14 +0000109
Stefan Hajnoczic40a2542018-03-22 15:28:33 +0000110 QSIMPLEQ_INSERT_TAIL(&pending, co, co_queue_next);
Jeff Cody6133b392017-11-17 22:27:09 -0500111
Stefan Hajnoczic40a2542018-03-22 15:28:33 +0000112 /* Run co and any queued coroutines */
113 while (!QSIMPLEQ_EMPTY(&pending)) {
114 Coroutine *to = QSIMPLEQ_FIRST(&pending);
115 CoroutineAction ret;
Kevin Wolf00dccaf2011-01-17 16:08:14 +0000116
Stefan Hajnoczic40a2542018-03-22 15:28:33 +0000117 /* Cannot rely on the read barrier for to in aio_co_wake(), as there are
118 * callers outside of aio_co_wake() */
119 const char *scheduled = atomic_mb_read(&to->scheduled);
Jeff Cody6133b392017-11-17 22:27:09 -0500120
Stefan Hajnoczic40a2542018-03-22 15:28:33 +0000121 QSIMPLEQ_REMOVE_HEAD(&pending, co_queue_next);
Kevin Wolf00dccaf2011-01-17 16:08:14 +0000122
Stefan Hajnoczic40a2542018-03-22 15:28:33 +0000123 trace_qemu_aio_coroutine_enter(ctx, from, to, to->entry_arg);
Paolo Bonzini0c330a72017-02-13 14:52:19 +0100124
Stefan Hajnoczic40a2542018-03-22 15:28:33 +0000125 /* if the Coroutine has already been scheduled, entering it again will
126 * cause us to enter it twice, potentially even after the coroutine has
127 * been deleted */
128 if (scheduled) {
129 fprintf(stderr,
130 "%s: Co-routine was already scheduled in '%s'\n",
131 __func__, scheduled);
132 abort();
133 }
Paolo Bonzini0c330a72017-02-13 14:52:19 +0100134
Stefan Hajnoczic40a2542018-03-22 15:28:33 +0000135 if (to->caller) {
136 fprintf(stderr, "Co-routine re-entered recursively\n");
137 abort();
138 }
Kevin Wolfcd12bb52015-02-10 11:31:52 +0100139
Stefan Hajnoczic40a2542018-03-22 15:28:33 +0000140 to->caller = from;
141 to->ctx = ctx;
Kevin Wolfcd12bb52015-02-10 11:31:52 +0100142
Stefan Hajnoczic40a2542018-03-22 15:28:33 +0000143 /* Store to->ctx before anything that stores to. Matches
144 * barrier in aio_co_wake and qemu_co_mutex_wake.
145 */
146 smp_wmb();
Roman Pen528f4492017-06-01 18:08:47 +0200147
Stefan Hajnoczic40a2542018-03-22 15:28:33 +0000148 ret = qemu_coroutine_switch(from, to, COROUTINE_ENTER);
149
150 /* Queued coroutines are run depth-first; previously pending coroutines
151 * run after those queued more recently.
152 */
153 QSIMPLEQ_PREPEND(&pending, &to->co_queue_wakeup);
154
155 switch (ret) {
156 case COROUTINE_YIELD:
157 break;
158 case COROUTINE_TERMINATE:
159 assert(!to->locks_held);
160 trace_qemu_coroutine_terminate(to);
161 coroutine_delete(to);
162 break;
163 default:
164 abort();
165 }
Kevin Wolfcd12bb52015-02-10 11:31:52 +0100166 }
Kevin Wolf00dccaf2011-01-17 16:08:14 +0000167}
168
Fam Zhengba9e75c2017-04-10 20:06:12 +0800169void qemu_coroutine_enter(Coroutine *co)
170{
171 qemu_aio_coroutine_enter(qemu_get_current_aio_context(), co);
172}
173
Kevin Wolf536fca72016-11-07 16:34:35 +0100174void qemu_coroutine_enter_if_inactive(Coroutine *co)
175{
176 if (!qemu_coroutine_entered(co)) {
177 qemu_coroutine_enter(co);
178 }
179}
180
Kevin Wolf00dccaf2011-01-17 16:08:14 +0000181void coroutine_fn qemu_coroutine_yield(void)
182{
183 Coroutine *self = qemu_coroutine_self();
184 Coroutine *to = self->caller;
185
186 trace_qemu_coroutine_yield(self, to);
187
188 if (!to) {
189 fprintf(stderr, "Co-routine is yielding to no one\n");
190 abort();
191 }
192
193 self->caller = NULL;
Kevin Wolf315a1302015-02-10 11:17:53 +0100194 qemu_coroutine_switch(self, to, COROUTINE_YIELD);
Kevin Wolf00dccaf2011-01-17 16:08:14 +0000195}
Stefan Hajnoczif643e462016-09-27 16:18:34 +0100196
197bool qemu_coroutine_entered(Coroutine *co)
198{
199 return co->caller;
200}
Kevin Wolfaa1361d2018-08-17 18:54:18 +0200201
202AioContext *coroutine_fn qemu_coroutine_get_aio_context(Coroutine *co)
203{
204 return co->ctx;
205}