blob: 38fb6d3084dad2dd2145134d9e5da8f03d7169fe [file] [log] [blame]
Kevin Wolf00dccaf2011-01-17 16:08:14 +00001/*
2 * QEMU coroutines
3 *
4 * Copyright IBM, Corp. 2011
5 *
6 * Authors:
7 * Stefan Hajnoczi <stefanha@linux.vnet.ibm.com>
8 * Kevin Wolf <kwolf@redhat.com>
9 *
10 * This work is licensed under the terms of the GNU LGPL, version 2 or later.
11 * See the COPYING.LIB file in the top-level directory.
12 *
13 */
14
Peter Maydellaafd7582016-01-29 17:49:55 +000015#include "qemu/osdep.h"
Kevin Wolf00dccaf2011-01-17 16:08:14 +000016#include "trace.h"
Stefan Hajnoczib84c4582013-05-17 15:51:25 +020017#include "qemu/thread.h"
Paolo Bonzini4d68e862014-12-02 12:05:48 +010018#include "qemu/atomic.h"
Daniel P. Berrange10817bf2015-09-01 14:48:02 +010019#include "qemu/coroutine.h"
20#include "qemu/coroutine_int.h"
Paolo Bonzini0c330a72017-02-13 14:52:19 +010021#include "block/aio.h"
Kevin Wolf00dccaf2011-01-17 16:08:14 +000022
Paolo Bonzini40239782013-02-19 11:59:09 +010023enum {
Paolo Bonzini4d68e862014-12-02 12:05:48 +010024 POOL_BATCH_SIZE = 64,
Paolo Bonzini40239782013-02-19 11:59:09 +010025};
26
27/** Free list to speed up creation */
Paolo Bonzini4d68e862014-12-02 12:05:48 +010028static QSLIST_HEAD(, Coroutine) release_pool = QSLIST_HEAD_INITIALIZER(pool);
29static unsigned int release_pool_size;
30static __thread QSLIST_HEAD(, Coroutine) alloc_pool = QSLIST_HEAD_INITIALIZER(pool);
Peter Lieven51a22192014-12-02 12:05:50 +010031static __thread unsigned int alloc_pool_size;
Paolo Bonzini4d68e862014-12-02 12:05:48 +010032static __thread Notifier coroutine_pool_cleanup_notifier;
33
34static void coroutine_pool_cleanup(Notifier *n, void *value)
35{
36 Coroutine *co;
37 Coroutine *tmp;
38
39 QSLIST_FOREACH_SAFE(co, &alloc_pool, pool_next, tmp) {
40 QSLIST_REMOVE_HEAD(&alloc_pool, pool_next);
41 qemu_coroutine_delete(co);
42 }
43}
Paolo Bonzini40239782013-02-19 11:59:09 +010044
Paolo Bonzini0b8b8752016-07-04 19:10:01 +020045Coroutine *qemu_coroutine_create(CoroutineEntry *entry, void *opaque)
Kevin Wolf00dccaf2011-01-17 16:08:14 +000046{
Stefan Hajnoczi70c60c02013-09-11 16:42:35 +020047 Coroutine *co = NULL;
Paolo Bonzini40239782013-02-19 11:59:09 +010048
Stefan Hajnoczi70c60c02013-09-11 16:42:35 +020049 if (CONFIG_COROUTINE_POOL) {
Paolo Bonzini4d68e862014-12-02 12:05:48 +010050 co = QSLIST_FIRST(&alloc_pool);
51 if (!co) {
52 if (release_pool_size > POOL_BATCH_SIZE) {
53 /* Slow path; a good place to register the destructor, too. */
54 if (!coroutine_pool_cleanup_notifier.notify) {
55 coroutine_pool_cleanup_notifier.notify = coroutine_pool_cleanup;
56 qemu_thread_atexit_add(&coroutine_pool_cleanup_notifier);
57 }
58
59 /* This is not exact; there could be a little skew between
60 * release_pool_size and the actual size of release_pool. But
61 * it is just a heuristic, it does not need to be perfect.
62 */
Stefan Hajnoczid73415a2020-09-23 11:56:46 +010063 alloc_pool_size = qatomic_xchg(&release_pool_size, 0);
Paolo Bonzini4d68e862014-12-02 12:05:48 +010064 QSLIST_MOVE_ATOMIC(&alloc_pool, &release_pool);
65 co = QSLIST_FIRST(&alloc_pool);
66 }
Stefan Hajnoczi70c60c02013-09-11 16:42:35 +020067 }
Paolo Bonzini4d68e862014-12-02 12:05:48 +010068 if (co) {
69 QSLIST_REMOVE_HEAD(&alloc_pool, pool_next);
Peter Lieven51a22192014-12-02 12:05:50 +010070 alloc_pool_size--;
Paolo Bonzini4d68e862014-12-02 12:05:48 +010071 }
Stefan Hajnoczib84c4582013-05-17 15:51:25 +020072 }
Stefan Hajnoczib84c4582013-05-17 15:51:25 +020073
74 if (!co) {
Paolo Bonzini40239782013-02-19 11:59:09 +010075 co = qemu_coroutine_new();
76 }
77
Kevin Wolf00dccaf2011-01-17 16:08:14 +000078 co->entry = entry;
Paolo Bonzini0b8b8752016-07-04 19:10:01 +020079 co->entry_arg = opaque;
Paolo Bonzini7d9c8582016-07-04 19:09:59 +020080 QSIMPLEQ_INIT(&co->co_queue_wakeup);
Kevin Wolf00dccaf2011-01-17 16:08:14 +000081 return co;
82}
83
Paolo Bonzini40239782013-02-19 11:59:09 +010084static void coroutine_delete(Coroutine *co)
85{
Paolo Bonzini4d68e862014-12-02 12:05:48 +010086 co->caller = NULL;
87
Stefan Hajnoczi70c60c02013-09-11 16:42:35 +020088 if (CONFIG_COROUTINE_POOL) {
Paolo Bonzini4d68e862014-12-02 12:05:48 +010089 if (release_pool_size < POOL_BATCH_SIZE * 2) {
90 QSLIST_INSERT_HEAD_ATOMIC(&release_pool, co, pool_next);
Stefan Hajnoczid73415a2020-09-23 11:56:46 +010091 qatomic_inc(&release_pool_size);
Stefan Hajnoczi70c60c02013-09-11 16:42:35 +020092 return;
93 }
Peter Lieven51a22192014-12-02 12:05:50 +010094 if (alloc_pool_size < POOL_BATCH_SIZE) {
95 QSLIST_INSERT_HEAD(&alloc_pool, co, pool_next);
96 alloc_pool_size++;
97 return;
98 }
Paolo Bonzini40239782013-02-19 11:59:09 +010099 }
100
101 qemu_coroutine_delete(co);
102}
103
Fam Zhengba9e75c2017-04-10 20:06:12 +0800104void qemu_aio_coroutine_enter(AioContext *ctx, Coroutine *co)
Kevin Wolf00dccaf2011-01-17 16:08:14 +0000105{
Stefan Hajnoczic40a2542018-03-22 15:28:33 +0000106 QSIMPLEQ_HEAD(, Coroutine) pending = QSIMPLEQ_HEAD_INITIALIZER(pending);
107 Coroutine *from = qemu_coroutine_self();
Kevin Wolf00dccaf2011-01-17 16:08:14 +0000108
Stefan Hajnoczic40a2542018-03-22 15:28:33 +0000109 QSIMPLEQ_INSERT_TAIL(&pending, co, co_queue_next);
Jeff Cody6133b392017-11-17 22:27:09 -0500110
Stefan Hajnoczic40a2542018-03-22 15:28:33 +0000111 /* Run co and any queued coroutines */
112 while (!QSIMPLEQ_EMPTY(&pending)) {
113 Coroutine *to = QSIMPLEQ_FIRST(&pending);
114 CoroutineAction ret;
Kevin Wolf00dccaf2011-01-17 16:08:14 +0000115
Stefan Hajnoczic40a2542018-03-22 15:28:33 +0000116 /* Cannot rely on the read barrier for to in aio_co_wake(), as there are
117 * callers outside of aio_co_wake() */
Stefan Hajnoczid73415a2020-09-23 11:56:46 +0100118 const char *scheduled = qatomic_mb_read(&to->scheduled);
Jeff Cody6133b392017-11-17 22:27:09 -0500119
Stefan Hajnoczic40a2542018-03-22 15:28:33 +0000120 QSIMPLEQ_REMOVE_HEAD(&pending, co_queue_next);
Kevin Wolf00dccaf2011-01-17 16:08:14 +0000121
Stefan Hajnoczic40a2542018-03-22 15:28:33 +0000122 trace_qemu_aio_coroutine_enter(ctx, from, to, to->entry_arg);
Paolo Bonzini0c330a72017-02-13 14:52:19 +0100123
Stefan Hajnoczic40a2542018-03-22 15:28:33 +0000124 /* if the Coroutine has already been scheduled, entering it again will
125 * cause us to enter it twice, potentially even after the coroutine has
126 * been deleted */
127 if (scheduled) {
128 fprintf(stderr,
129 "%s: Co-routine was already scheduled in '%s'\n",
130 __func__, scheduled);
131 abort();
132 }
Paolo Bonzini0c330a72017-02-13 14:52:19 +0100133
Stefan Hajnoczic40a2542018-03-22 15:28:33 +0000134 if (to->caller) {
135 fprintf(stderr, "Co-routine re-entered recursively\n");
136 abort();
137 }
Kevin Wolfcd12bb52015-02-10 11:31:52 +0100138
Stefan Hajnoczic40a2542018-03-22 15:28:33 +0000139 to->caller = from;
140 to->ctx = ctx;
Kevin Wolfcd12bb52015-02-10 11:31:52 +0100141
Stefan Hajnoczic40a2542018-03-22 15:28:33 +0000142 /* Store to->ctx before anything that stores to. Matches
143 * barrier in aio_co_wake and qemu_co_mutex_wake.
144 */
145 smp_wmb();
Roman Pen528f4492017-06-01 18:08:47 +0200146
Stefan Hajnoczic40a2542018-03-22 15:28:33 +0000147 ret = qemu_coroutine_switch(from, to, COROUTINE_ENTER);
148
149 /* Queued coroutines are run depth-first; previously pending coroutines
150 * run after those queued more recently.
151 */
152 QSIMPLEQ_PREPEND(&pending, &to->co_queue_wakeup);
153
154 switch (ret) {
155 case COROUTINE_YIELD:
156 break;
157 case COROUTINE_TERMINATE:
158 assert(!to->locks_held);
159 trace_qemu_coroutine_terminate(to);
160 coroutine_delete(to);
161 break;
162 default:
163 abort();
164 }
Kevin Wolfcd12bb52015-02-10 11:31:52 +0100165 }
Kevin Wolf00dccaf2011-01-17 16:08:14 +0000166}
167
Fam Zhengba9e75c2017-04-10 20:06:12 +0800168void qemu_coroutine_enter(Coroutine *co)
169{
170 qemu_aio_coroutine_enter(qemu_get_current_aio_context(), co);
171}
172
Kevin Wolf536fca72016-11-07 16:34:35 +0100173void qemu_coroutine_enter_if_inactive(Coroutine *co)
174{
175 if (!qemu_coroutine_entered(co)) {
176 qemu_coroutine_enter(co);
177 }
178}
179
Kevin Wolf00dccaf2011-01-17 16:08:14 +0000180void coroutine_fn qemu_coroutine_yield(void)
181{
182 Coroutine *self = qemu_coroutine_self();
183 Coroutine *to = self->caller;
184
185 trace_qemu_coroutine_yield(self, to);
186
187 if (!to) {
188 fprintf(stderr, "Co-routine is yielding to no one\n");
189 abort();
190 }
191
192 self->caller = NULL;
Kevin Wolf315a1302015-02-10 11:17:53 +0100193 qemu_coroutine_switch(self, to, COROUTINE_YIELD);
Kevin Wolf00dccaf2011-01-17 16:08:14 +0000194}
Stefan Hajnoczif643e462016-09-27 16:18:34 +0100195
196bool qemu_coroutine_entered(Coroutine *co)
197{
198 return co->caller;
199}
Kevin Wolfaa1361d2018-08-17 18:54:18 +0200200
201AioContext *coroutine_fn qemu_coroutine_get_aio_context(Coroutine *co)
202{
203 return co->ctx;
204}