blob: 2534435388f3dce83de34d5f8a7f0594294975d3 [file] [log] [blame]
Kevin Wolfb96e9242011-06-30 17:56:46 +02001/*
2 * coroutine queues and locks
3 *
4 * Copyright (c) 2011 Kevin Wolf <kwolf@redhat.com>
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a copy
7 * of this software and associated documentation files (the "Software"), to deal
8 * in the Software without restriction, including without limitation the rights
9 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
10 * copies of the Software, and to permit persons to whom the Software is
11 * furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
22 * THE SOFTWARE.
Paolo Bonzinifed20a72017-02-13 19:12:39 +010023 *
24 * The lock-free mutex implementation is based on OSv
25 * (core/lfmutex.cc, include/lockfree/mutex.hh).
26 * Copyright (C) 2013 Cloudius Systems, Ltd.
Kevin Wolfb96e9242011-06-30 17:56:46 +020027 */
28
Peter Maydellaafd7582016-01-29 17:49:55 +000029#include "qemu/osdep.h"
Daniel P. Berrange10817bf2015-09-01 14:48:02 +010030#include "qemu/coroutine_int.h"
Paolo Bonzini480cff62017-02-13 19:12:40 +010031#include "qemu/processor.h"
Paolo Bonzini1de7afc2012-12-17 18:20:00 +010032#include "qemu/queue.h"
Paolo Bonzinia9d92352017-02-13 14:52:25 +010033#include "block/aio.h"
Kevin Wolfb96e9242011-06-30 17:56:46 +020034#include "trace.h"
35
Kevin Wolfb96e9242011-06-30 17:56:46 +020036void qemu_co_queue_init(CoQueue *queue)
37{
Paolo Bonzini7d9c8582016-07-04 19:09:59 +020038 QSIMPLEQ_INIT(&queue->entries);
Kevin Wolfb96e9242011-06-30 17:56:46 +020039}
40
Stefan Hajnoczi0421b562022-10-13 14:58:56 -040041void coroutine_fn qemu_co_queue_wait_impl(CoQueue *queue, QemuLockable *lock,
42 CoQueueWaitFlags flags)
Kevin Wolfb96e9242011-06-30 17:56:46 +020043{
44 Coroutine *self = qemu_coroutine_self();
Stefan Hajnoczi0421b562022-10-13 14:58:56 -040045 if (flags & CO_QUEUE_WAIT_FRONT) {
46 QSIMPLEQ_INSERT_HEAD(&queue->entries, self, co_queue_next);
47 } else {
48 QSIMPLEQ_INSERT_TAIL(&queue->entries, self, co_queue_next);
49 }
Paolo Bonzini1ace7ce2017-02-13 19:12:43 +010050
Paolo Bonzini1a957cf2018-02-03 10:39:33 -050051 if (lock) {
52 qemu_lockable_unlock(lock);
Paolo Bonzini1ace7ce2017-02-13 19:12:43 +010053 }
54
55 /* There is no race condition here. Other threads will call
56 * aio_co_schedule on our AioContext, which can reenter this
57 * coroutine but only after this yield and after the main loop
58 * has gone through the next iteration.
59 */
Kevin Wolfb96e9242011-06-30 17:56:46 +020060 qemu_coroutine_yield();
61 assert(qemu_in_coroutine());
Paolo Bonzini1ace7ce2017-02-13 19:12:43 +010062
63 /* TODO: OSv implements wait morphing here, where the wakeup
64 * primitive automatically places the woken coroutine on the
65 * mutex's queue. This avoids the thundering herd effect.
Paolo Bonzini1a957cf2018-02-03 10:39:33 -050066 * This could be implemented for CoMutexes, but not really for
67 * other cases of QemuLockable.
Paolo Bonzini1ace7ce2017-02-13 19:12:43 +010068 */
Paolo Bonzini1a957cf2018-02-03 10:39:33 -050069 if (lock) {
70 qemu_lockable_lock(lock);
Paolo Bonzini1ace7ce2017-02-13 19:12:43 +010071 }
Kevin Wolfb96e9242011-06-30 17:56:46 +020072}
73
Paolo Bonzini5261dd72018-02-03 10:39:34 -050074bool qemu_co_enter_next_impl(CoQueue *queue, QemuLockable *lock)
Benoît Canetb681a1c2013-07-26 22:39:22 +020075{
76 Coroutine *next;
77
Paolo Bonzini7d9c8582016-07-04 19:09:59 +020078 next = QSIMPLEQ_FIRST(&queue->entries);
Benoît Canetb681a1c2013-07-26 22:39:22 +020079 if (!next) {
80 return false;
81 }
82
Paolo Bonzini7d9c8582016-07-04 19:09:59 +020083 QSIMPLEQ_REMOVE_HEAD(&queue->entries, co_queue_next);
Paolo Bonzini5261dd72018-02-03 10:39:34 -050084 if (lock) {
85 qemu_lockable_unlock(lock);
86 }
87 aio_co_wake(next);
88 if (lock) {
89 qemu_lockable_lock(lock);
90 }
Benoît Canetb681a1c2013-07-26 22:39:22 +020091 return true;
92}
93
Paolo Bonzini248af9e2022-04-27 15:08:28 +020094bool coroutine_fn qemu_co_queue_next(CoQueue *queue)
95{
96 /* No unlock/lock needed in coroutine context. */
97 return qemu_co_enter_next_impl(queue, NULL);
98}
99
Paolo Bonzinid6ee15a2022-04-27 15:08:29 +0200100void qemu_co_enter_all_impl(CoQueue *queue, QemuLockable *lock)
101{
102 while (qemu_co_enter_next_impl(queue, lock)) {
103 /* just loop */
104 }
105}
106
Paolo Bonzinif0d43b12022-04-27 15:08:30 +0200107void coroutine_fn qemu_co_queue_restart_all(CoQueue *queue)
108{
109 /* No unlock/lock needed in coroutine context. */
110 qemu_co_enter_all_impl(queue, NULL);
111}
112
Kevin Wolfb96e9242011-06-30 17:56:46 +0200113bool qemu_co_queue_empty(CoQueue *queue)
114{
Paolo Bonzini7d9c8582016-07-04 19:09:59 +0200115 return QSIMPLEQ_FIRST(&queue->entries) == NULL;
Kevin Wolfb96e9242011-06-30 17:56:46 +0200116}
117
Paolo Bonzinifed20a72017-02-13 19:12:39 +0100118/* The wait records are handled with a multiple-producer, single-consumer
119 * lock-free queue. There cannot be two concurrent pop_waiter() calls
120 * because pop_waiter() can only be called while mutex->handoff is zero.
121 * This can happen in three cases:
122 * - in qemu_co_mutex_unlock, before the hand-off protocol has started.
123 * In this case, qemu_co_mutex_lock will see mutex->handoff == 0 and
124 * not take part in the handoff.
125 * - in qemu_co_mutex_lock, if it steals the hand-off responsibility from
126 * qemu_co_mutex_unlock. In this case, qemu_co_mutex_unlock will fail
127 * the cmpxchg (it will see either 0 or the next sequence value) and
128 * exit. The next hand-off cannot begin until qemu_co_mutex_lock has
129 * woken up someone.
130 * - in qemu_co_mutex_unlock, if it takes the hand-off token itself.
131 * In this case another iteration starts with mutex->handoff == 0;
132 * a concurrent qemu_co_mutex_lock will fail the cmpxchg, and
133 * qemu_co_mutex_unlock will go back to case (1).
134 *
135 * The following functions manage this queue.
136 */
137typedef struct CoWaitRecord {
138 Coroutine *co;
139 QSLIST_ENTRY(CoWaitRecord) next;
140} CoWaitRecord;
141
Paolo Bonzini46cd09d2022-09-22 10:49:20 +0200142static void coroutine_fn push_waiter(CoMutex *mutex, CoWaitRecord *w)
Paolo Bonzinifed20a72017-02-13 19:12:39 +0100143{
144 w->co = qemu_coroutine_self();
145 QSLIST_INSERT_HEAD_ATOMIC(&mutex->from_push, w, next);
146}
147
148static void move_waiters(CoMutex *mutex)
149{
150 QSLIST_HEAD(, CoWaitRecord) reversed;
151 QSLIST_MOVE_ATOMIC(&reversed, &mutex->from_push);
152 while (!QSLIST_EMPTY(&reversed)) {
153 CoWaitRecord *w = QSLIST_FIRST(&reversed);
154 QSLIST_REMOVE_HEAD(&reversed, next);
155 QSLIST_INSERT_HEAD(&mutex->to_pop, w, next);
156 }
157}
158
159static CoWaitRecord *pop_waiter(CoMutex *mutex)
160{
161 CoWaitRecord *w;
162
163 if (QSLIST_EMPTY(&mutex->to_pop)) {
164 move_waiters(mutex);
165 if (QSLIST_EMPTY(&mutex->to_pop)) {
166 return NULL;
167 }
168 }
169 w = QSLIST_FIRST(&mutex->to_pop);
170 QSLIST_REMOVE_HEAD(&mutex->to_pop, next);
171 return w;
172}
173
174static bool has_waiters(CoMutex *mutex)
175{
176 return QSLIST_EMPTY(&mutex->to_pop) || QSLIST_EMPTY(&mutex->from_push);
177}
178
Kevin Wolfb96e9242011-06-30 17:56:46 +0200179void qemu_co_mutex_init(CoMutex *mutex)
180{
181 memset(mutex, 0, sizeof(*mutex));
Paolo Bonzinifed20a72017-02-13 19:12:39 +0100182}
183
Paolo Bonzini480cff62017-02-13 19:12:40 +0100184static void coroutine_fn qemu_co_mutex_wake(CoMutex *mutex, Coroutine *co)
185{
186 /* Read co before co->ctx; pairs with smp_wmb() in
187 * qemu_coroutine_enter().
188 */
189 smp_read_barrier_depends();
190 mutex->ctx = co->ctx;
191 aio_co_wake(co);
192}
193
194static void coroutine_fn qemu_co_mutex_lock_slowpath(AioContext *ctx,
195 CoMutex *mutex)
Paolo Bonzinifed20a72017-02-13 19:12:39 +0100196{
197 Coroutine *self = qemu_coroutine_self();
198 CoWaitRecord w;
199 unsigned old_handoff;
200
201 trace_qemu_co_mutex_lock_entry(mutex, self);
Paolo Bonzinifed20a72017-02-13 19:12:39 +0100202 push_waiter(mutex, &w);
203
Paolo Bonzinie3a3b6e2023-03-03 10:52:59 +0100204 /*
Paolo Bonzini06831002023-03-03 14:37:51 +0100205 * Add waiter before reading mutex->handoff. Pairs with qatomic_set_mb
Paolo Bonzinie3a3b6e2023-03-03 10:52:59 +0100206 * in qemu_co_mutex_unlock.
207 */
208 smp_mb__after_rmw();
209
Paolo Bonzinifed20a72017-02-13 19:12:39 +0100210 /* This is the "Responsibility Hand-Off" protocol; a lock() picks from
211 * a concurrent unlock() the responsibility of waking somebody up.
212 */
Paolo Bonzinie3a3b6e2023-03-03 10:52:59 +0100213 old_handoff = qatomic_read(&mutex->handoff);
Paolo Bonzinifed20a72017-02-13 19:12:39 +0100214 if (old_handoff &&
215 has_waiters(mutex) &&
Stefan Hajnoczid73415a2020-09-23 11:56:46 +0100216 qatomic_cmpxchg(&mutex->handoff, old_handoff, 0) == old_handoff) {
Paolo Bonzinifed20a72017-02-13 19:12:39 +0100217 /* There can be no concurrent pops, because there can be only
218 * one active handoff at a time.
219 */
220 CoWaitRecord *to_wake = pop_waiter(mutex);
221 Coroutine *co = to_wake->co;
222 if (co == self) {
223 /* We got the lock ourselves! */
224 assert(to_wake == &w);
Paolo Bonzini480cff62017-02-13 19:12:40 +0100225 mutex->ctx = ctx;
Paolo Bonzinifed20a72017-02-13 19:12:39 +0100226 return;
227 }
228
Paolo Bonzini480cff62017-02-13 19:12:40 +0100229 qemu_co_mutex_wake(mutex, co);
Paolo Bonzinifed20a72017-02-13 19:12:39 +0100230 }
231
232 qemu_coroutine_yield();
233 trace_qemu_co_mutex_lock_return(mutex, self);
Kevin Wolfb96e9242011-06-30 17:56:46 +0200234}
235
236void coroutine_fn qemu_co_mutex_lock(CoMutex *mutex)
237{
Paolo Bonzini480cff62017-02-13 19:12:40 +0100238 AioContext *ctx = qemu_get_current_aio_context();
Kevin Wolfb96e9242011-06-30 17:56:46 +0200239 Coroutine *self = qemu_coroutine_self();
Paolo Bonzini480cff62017-02-13 19:12:40 +0100240 int waiters, i;
Kevin Wolfb96e9242011-06-30 17:56:46 +0200241
Paolo Bonzini480cff62017-02-13 19:12:40 +0100242 /* Running a very small critical section on pthread_mutex_t and CoMutex
243 * shows that pthread_mutex_t is much faster because it doesn't actually
244 * go to sleep. What happens is that the critical section is shorter
245 * than the latency of entering the kernel and thus FUTEX_WAIT always
246 * fails. With CoMutex there is no such latency but you still want to
247 * avoid wait and wakeup. So introduce it artificially.
248 */
249 i = 0;
250retry_fast_path:
Stefan Hajnoczid73415a2020-09-23 11:56:46 +0100251 waiters = qatomic_cmpxchg(&mutex->locked, 0, 1);
Paolo Bonzini480cff62017-02-13 19:12:40 +0100252 if (waiters != 0) {
253 while (waiters == 1 && ++i < 1000) {
Stefan Hajnoczid73415a2020-09-23 11:56:46 +0100254 if (qatomic_read(&mutex->ctx) == ctx) {
Paolo Bonzini480cff62017-02-13 19:12:40 +0100255 break;
256 }
Stefan Hajnoczid73415a2020-09-23 11:56:46 +0100257 if (qatomic_read(&mutex->locked) == 0) {
Paolo Bonzini480cff62017-02-13 19:12:40 +0100258 goto retry_fast_path;
259 }
260 cpu_relax();
261 }
Stefan Hajnoczid73415a2020-09-23 11:56:46 +0100262 waiters = qatomic_fetch_inc(&mutex->locked);
Paolo Bonzini480cff62017-02-13 19:12:40 +0100263 }
264
265 if (waiters == 0) {
Paolo Bonzinifed20a72017-02-13 19:12:39 +0100266 /* Uncontended. */
267 trace_qemu_co_mutex_lock_uncontended(mutex, self);
Paolo Bonzini480cff62017-02-13 19:12:40 +0100268 mutex->ctx = ctx;
Paolo Bonzinifed20a72017-02-13 19:12:39 +0100269 } else {
Paolo Bonzini480cff62017-02-13 19:12:40 +0100270 qemu_co_mutex_lock_slowpath(ctx, mutex);
Kevin Wolfb96e9242011-06-30 17:56:46 +0200271 }
Kevin Wolf0e438cd2016-08-11 17:45:06 +0200272 mutex->holder = self;
Kevin Wolf1b7f01d2016-08-11 17:51:59 +0200273 self->locks_held++;
Kevin Wolfb96e9242011-06-30 17:56:46 +0200274}
275
276void coroutine_fn qemu_co_mutex_unlock(CoMutex *mutex)
277{
278 Coroutine *self = qemu_coroutine_self();
279
280 trace_qemu_co_mutex_unlock_entry(mutex, self);
281
Paolo Bonzinifed20a72017-02-13 19:12:39 +0100282 assert(mutex->locked);
Kevin Wolf0e438cd2016-08-11 17:45:06 +0200283 assert(mutex->holder == self);
Kevin Wolfb96e9242011-06-30 17:56:46 +0200284 assert(qemu_in_coroutine());
285
Paolo Bonzini480cff62017-02-13 19:12:40 +0100286 mutex->ctx = NULL;
Kevin Wolf0e438cd2016-08-11 17:45:06 +0200287 mutex->holder = NULL;
Kevin Wolf1b7f01d2016-08-11 17:51:59 +0200288 self->locks_held--;
Stefan Hajnoczid73415a2020-09-23 11:56:46 +0100289 if (qatomic_fetch_dec(&mutex->locked) == 1) {
Paolo Bonzinifed20a72017-02-13 19:12:39 +0100290 /* No waiting qemu_co_mutex_lock(). Pfew, that was easy! */
291 return;
292 }
293
294 for (;;) {
295 CoWaitRecord *to_wake = pop_waiter(mutex);
296 unsigned our_handoff;
297
298 if (to_wake) {
Paolo Bonzini480cff62017-02-13 19:12:40 +0100299 qemu_co_mutex_wake(mutex, to_wake->co);
Paolo Bonzinifed20a72017-02-13 19:12:39 +0100300 break;
301 }
302
303 /* Some concurrent lock() is in progress (we know this because
304 * mutex->locked was >1) but it hasn't yet put itself on the wait
305 * queue. Pick a sequence number for the handoff protocol (not 0).
306 */
307 if (++mutex->sequence == 0) {
308 mutex->sequence = 1;
309 }
310
311 our_handoff = mutex->sequence;
Paolo Bonzinie3a3b6e2023-03-03 10:52:59 +0100312 /* Set handoff before checking for waiters. */
Paolo Bonzini06831002023-03-03 14:37:51 +0100313 qatomic_set_mb(&mutex->handoff, our_handoff);
Paolo Bonzinifed20a72017-02-13 19:12:39 +0100314 if (!has_waiters(mutex)) {
315 /* The concurrent lock has not added itself yet, so it
316 * will be able to pick our handoff.
317 */
318 break;
319 }
320
321 /* Try to do the handoff protocol ourselves; if somebody else has
322 * already taken it, however, we're done and they're responsible.
323 */
Stefan Hajnoczid73415a2020-09-23 11:56:46 +0100324 if (qatomic_cmpxchg(&mutex->handoff, our_handoff, 0) != our_handoff) {
Paolo Bonzinifed20a72017-02-13 19:12:39 +0100325 break;
326 }
327 }
Kevin Wolfb96e9242011-06-30 17:56:46 +0200328
329 trace_qemu_co_mutex_unlock_return(mutex, self);
330}
Aneesh Kumar K.V12888902011-08-02 12:02:51 +0530331
Paolo Bonzini050de362021-03-25 12:29:39 +0100332struct CoRwTicket {
333 bool read;
334 Coroutine *co;
335 QSIMPLEQ_ENTRY(CoRwTicket) next;
336};
337
Aneesh Kumar K.V12888902011-08-02 12:02:51 +0530338void qemu_co_rwlock_init(CoRwlock *lock)
339{
Paolo Bonzinia7b91d32017-02-13 19:12:44 +0100340 qemu_co_mutex_init(&lock->mutex);
Paolo Bonzini050de362021-03-25 12:29:39 +0100341 lock->owners = 0;
342 QSIMPLEQ_INIT(&lock->tickets);
343}
344
345/* Releases the internal CoMutex. */
Paolo Bonzini46cd09d2022-09-22 10:49:20 +0200346static void coroutine_fn qemu_co_rwlock_maybe_wake_one(CoRwlock *lock)
Paolo Bonzini050de362021-03-25 12:29:39 +0100347{
348 CoRwTicket *tkt = QSIMPLEQ_FIRST(&lock->tickets);
349 Coroutine *co = NULL;
350
351 /*
352 * Setting lock->owners here prevents rdlock and wrlock from
353 * sneaking in between unlock and wake.
354 */
355
356 if (tkt) {
357 if (tkt->read) {
358 if (lock->owners >= 0) {
359 lock->owners++;
360 co = tkt->co;
361 }
362 } else {
363 if (lock->owners == 0) {
364 lock->owners = -1;
365 co = tkt->co;
366 }
367 }
368 }
369
370 if (co) {
371 QSIMPLEQ_REMOVE_HEAD(&lock->tickets, next);
372 qemu_co_mutex_unlock(&lock->mutex);
373 aio_co_wake(co);
374 } else {
375 qemu_co_mutex_unlock(&lock->mutex);
376 }
Aneesh Kumar K.V12888902011-08-02 12:02:51 +0530377}
378
Paolo Bonzini46cd09d2022-09-22 10:49:20 +0200379void coroutine_fn qemu_co_rwlock_rdlock(CoRwlock *lock)
Aneesh Kumar K.V12888902011-08-02 12:02:51 +0530380{
Kevin Wolf1b7f01d2016-08-11 17:51:59 +0200381 Coroutine *self = qemu_coroutine_self();
382
Paolo Bonzinia7b91d32017-02-13 19:12:44 +0100383 qemu_co_mutex_lock(&lock->mutex);
384 /* For fairness, wait if a writer is in line. */
Paolo Bonzini050de362021-03-25 12:29:39 +0100385 if (lock->owners == 0 || (lock->owners > 0 && QSIMPLEQ_EMPTY(&lock->tickets))) {
386 lock->owners++;
387 qemu_co_mutex_unlock(&lock->mutex);
388 } else {
389 CoRwTicket my_ticket = { true, self };
Paolo Bonzinia7b91d32017-02-13 19:12:44 +0100390
Paolo Bonzini050de362021-03-25 12:29:39 +0100391 QSIMPLEQ_INSERT_TAIL(&lock->tickets, &my_ticket, next);
392 qemu_co_mutex_unlock(&lock->mutex);
393 qemu_coroutine_yield();
394 assert(lock->owners >= 1);
395
396 /* Possibly wake another reader, which will wake the next in line. */
397 qemu_co_mutex_lock(&lock->mutex);
398 qemu_co_rwlock_maybe_wake_one(lock);
399 }
400
Kevin Wolf1b7f01d2016-08-11 17:51:59 +0200401 self->locks_held++;
Aneesh Kumar K.V12888902011-08-02 12:02:51 +0530402}
403
Paolo Bonzini46cd09d2022-09-22 10:49:20 +0200404void coroutine_fn qemu_co_rwlock_unlock(CoRwlock *lock)
Aneesh Kumar K.V12888902011-08-02 12:02:51 +0530405{
Kevin Wolf1b7f01d2016-08-11 17:51:59 +0200406 Coroutine *self = qemu_coroutine_self();
407
Aneesh Kumar K.V12888902011-08-02 12:02:51 +0530408 assert(qemu_in_coroutine());
Paolo Bonzini050de362021-03-25 12:29:39 +0100409 self->locks_held--;
Paolo Bonzinia7b91d32017-02-13 19:12:44 +0100410
Paolo Bonzini050de362021-03-25 12:29:39 +0100411 qemu_co_mutex_lock(&lock->mutex);
412 if (lock->owners > 0) {
413 lock->owners--;
414 } else {
415 assert(lock->owners == -1);
416 lock->owners = 0;
Aneesh Kumar K.V12888902011-08-02 12:02:51 +0530417 }
Paolo Bonzini050de362021-03-25 12:29:39 +0100418
419 qemu_co_rwlock_maybe_wake_one(lock);
Aneesh Kumar K.V12888902011-08-02 12:02:51 +0530420}
421
Paolo Bonzini46cd09d2022-09-22 10:49:20 +0200422void coroutine_fn qemu_co_rwlock_downgrade(CoRwlock *lock)
Paolo Bonzini667221c2017-06-29 15:27:40 +0200423{
Paolo Bonzini050de362021-03-25 12:29:39 +0100424 qemu_co_mutex_lock(&lock->mutex);
425 assert(lock->owners == -1);
426 lock->owners = 1;
Paolo Bonzini667221c2017-06-29 15:27:40 +0200427
Paolo Bonzini050de362021-03-25 12:29:39 +0100428 /* Possibly wake another reader, which will wake the next in line. */
429 qemu_co_rwlock_maybe_wake_one(lock);
Paolo Bonzini667221c2017-06-29 15:27:40 +0200430}
431
Paolo Bonzini46cd09d2022-09-22 10:49:20 +0200432void coroutine_fn qemu_co_rwlock_wrlock(CoRwlock *lock)
Aneesh Kumar K.V12888902011-08-02 12:02:51 +0530433{
Paolo Bonzini050de362021-03-25 12:29:39 +0100434 Coroutine *self = qemu_coroutine_self();
Paolo Bonzinia7b91d32017-02-13 19:12:44 +0100435
Paolo Bonzini050de362021-03-25 12:29:39 +0100436 qemu_co_mutex_lock(&lock->mutex);
437 if (lock->owners == 0) {
438 lock->owners = -1;
439 qemu_co_mutex_unlock(&lock->mutex);
440 } else {
441 CoRwTicket my_ticket = { false, qemu_coroutine_self() };
442
443 QSIMPLEQ_INSERT_TAIL(&lock->tickets, &my_ticket, next);
444 qemu_co_mutex_unlock(&lock->mutex);
445 qemu_coroutine_yield();
446 assert(lock->owners == -1);
447 }
448
449 self->locks_held++;
Aneesh Kumar K.V12888902011-08-02 12:02:51 +0530450}
Paolo Bonzini667221c2017-06-29 15:27:40 +0200451
Paolo Bonzini46cd09d2022-09-22 10:49:20 +0200452void coroutine_fn qemu_co_rwlock_upgrade(CoRwlock *lock)
Paolo Bonzini667221c2017-06-29 15:27:40 +0200453{
Paolo Bonzini667221c2017-06-29 15:27:40 +0200454 qemu_co_mutex_lock(&lock->mutex);
Paolo Bonzini050de362021-03-25 12:29:39 +0100455 assert(lock->owners > 0);
456 /* For fairness, wait if a writer is in line. */
457 if (lock->owners == 1 && QSIMPLEQ_EMPTY(&lock->tickets)) {
458 lock->owners = -1;
459 qemu_co_mutex_unlock(&lock->mutex);
460 } else {
461 CoRwTicket my_ticket = { false, qemu_coroutine_self() };
Paolo Bonzini667221c2017-06-29 15:27:40 +0200462
Paolo Bonzini050de362021-03-25 12:29:39 +0100463 lock->owners--;
464 QSIMPLEQ_INSERT_TAIL(&lock->tickets, &my_ticket, next);
465 qemu_co_rwlock_maybe_wake_one(lock);
466 qemu_coroutine_yield();
467 assert(lock->owners == -1);
468 }
Paolo Bonzini667221c2017-06-29 15:27:40 +0200469}