Merge tag 'migration-20230509-pull-request' of https://gitlab.com/juan.quintela/qemu into staging
Migration Pull request (20230509 vintage) take 2
Hi
In this take 2:
- Change uint -> uint32_t to fix mingw32 compilation.
Please apply.
[take 1]
In this PULL request:
- 1st part of colo support for multifd (lukas)
- 1st part of disabling colo option (vladimir)
Please, apply.
# -----BEGIN PGP SIGNATURE-----
#
# iQIzBAABCAAdFiEEGJn/jt6/WMzuA0uC9IfvGFhy1yMFAmRb3dgACgkQ9IfvGFhy
# 1yNLBxAAwHiAOdSPS7TqJXH2/PkBKsd42XMtWzC9UowZ6SUdQi0Q2bQUBnygJ8BA
# 59yLOTPdwUhaPWk4KsyKM2znOCJ+f9MF5V4QXbyILf1WCAq6d+mtPwArnYF1TRwi
# XIewVDeRopdOO5lnWGcfAKZZ5WIDzA/bn6NiGLi+pQa5HGyk84Bk+tFa8kJI6xBL
# 5CWfhNTcxDNYRFg/z/9YVirkuxIXEEL6VEeRFV+pmFuj05q9bysWJkLFoEcFNawO
# gp1foHDkU7wHmHDJ3D4AVTm3TW641ft1wdlHIHZRoOiIIu3EUOoDEVVsaCfdxrY8
# pPJZ5m37wb52GIaCJmigG8rkHxIJ8xKLk4HKu4umDqFq5jZQ2krnnj7AkQhpp7p2
# aEIOXJQQq7XCsKpuvSUIexPv4gbN5SEYKi7XKoOPe3sZ03Rkn0I5xY3KSyMQMamP
# jtk8tNlRA+9Wug82eb/FtIKDj3//4SbuQOJEdRXjKJBldd3mtWTT/FRj/8oo96/p
# hmTu/cGDrP5qgtWpz0kKI/xaBf8at1nwpDgdEzOjRw4zf6xQHFjbXgJ7tQBH/JUI
# T3A9pdiXN6QdRupcWUSV0iJsfS/5i3mOUTA/C529qGXabSnZzfMK+unL/I8N02yt
# 83o7jSg22etMjaS1c+VuDmzKCAfuZloDZv2Bms/+yM/8k8Xe5S4=
# =vbqf
# -----END PGP SIGNATURE-----
# gpg: Signature made Wed 10 May 2023 07:09:28 PM BST
# gpg: using RSA key 1899FF8EDEBF58CCEE034B82F487EF185872D723
# gpg: Good signature from "Juan Quintela <quintela@redhat.com>" [undefined]
# gpg: aka "Juan Quintela <quintela@trasno.org>" [undefined]
# gpg: WARNING: This key is not certified with a trusted signature!
# gpg: There is no indication that the signature belongs to the owner.
# Primary key fingerprint: 1899 FF8E DEBF 58CC EE03 4B82 F487 EF18 5872 D723
* tag 'migration-20230509-pull-request' of https://gitlab.com/juan.quintela/qemu:
migration: block incoming colo when capability is disabled
migration: disallow change capabilities in COLO state
migration: process_incoming_migration_co: simplify code flow around ret
migration: drop colo_incoming_thread from MigrationIncomingState
build: move COLO under CONFIG_REPLICATION
colo: make colo_checkpoint_notify static and provide simpler API
block/meson.build: prefer positive condition for replication
multifd: Add the ramblock to MultiFDRecvParams
ram: Let colo_flush_ram_cache take the bitmap_mutex
ram: Add public helper to set colo bitmap
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
diff --git a/block/meson.build b/block/meson.build
index 13337bd..486dda8 100644
--- a/block/meson.build
+++ b/block/meson.build
@@ -92,7 +92,7 @@
block_ss.add(when: 'CONFIG_POSIX', if_true: [files('file-posix.c'), coref, iokit])
block_ss.add(when: libiscsi, if_true: files('iscsi-opts.c'))
block_ss.add(when: 'CONFIG_LINUX', if_true: files('nvme.c'))
-if not get_option('replication').disabled()
+if get_option('replication').allowed()
block_ss.add(files('replication.c'))
endif
block_ss.add(when: libaio, if_true: files('linux-aio.c'))
diff --git a/docs/COLO-FT.txt b/docs/COLO-FT.txt
index 8ec653f..2e760a4 100644
--- a/docs/COLO-FT.txt
+++ b/docs/COLO-FT.txt
@@ -210,6 +210,7 @@
3. On Secondary VM's QEMU monitor, issue command
{"execute":"qmp_capabilities"}
+{"execute": "migrate-set-capabilities", "arguments": {"capabilities": [ {"capability": "x-colo", "state": true } ] } }
{"execute": "nbd-server-start", "arguments": {"addr": {"type": "inet", "data": {"host": "0.0.0.0", "port": "9999"} } } }
{"execute": "nbd-server-add", "arguments": {"device": "parent0", "writable": true } }
diff --git a/hmp-commands.hx b/hmp-commands.hx
index 9afbb54..2cbd0f7 100644
--- a/hmp-commands.hx
+++ b/hmp-commands.hx
@@ -1052,6 +1052,7 @@
migration (or once already in postcopy).
ERST
+#ifdef CONFIG_REPLICATION
{
.name = "x_colo_lost_heartbeat",
.args_type = "",
@@ -1060,6 +1061,7 @@
"a failover or takeover is needed.",
.cmd = hmp_x_colo_lost_heartbeat,
},
+#endif
SRST
``x_colo_lost_heartbeat``
diff --git a/include/migration/colo.h b/include/migration/colo.h
index 5fbe1a6..7ef3154 100644
--- a/include/migration/colo.h
+++ b/include/migration/colo.h
@@ -36,6 +36,13 @@
/* failover */
void colo_do_failover(void);
-void colo_checkpoint_notify(void *opaque);
+/*
+ * colo_checkpoint_delay_set
+ *
+ * Handles change of x-checkpoint-delay migration parameter, called from
+ * migrate_params_apply() to notify COLO module about the change.
+ */
+void colo_checkpoint_delay_set(void);
+
void colo_shutdown(void);
#endif
diff --git a/migration/colo.c b/migration/colo.c
index 07bfa21..6c7c313 100644
--- a/migration/colo.c
+++ b/migration/colo.c
@@ -26,9 +26,7 @@
#include "qemu/rcu.h"
#include "migration/failover.h"
#include "migration/ram.h"
-#ifdef CONFIG_REPLICATION
#include "block/replication.h"
-#endif
#include "net/colo-compare.h"
#include "net/colo.h"
#include "block/block.h"
@@ -65,10 +63,27 @@
return runstate_check(RUN_STATE_COLO) || !runstate_is_running();
}
+static void colo_checkpoint_notify(void *opaque)
+{
+ MigrationState *s = opaque;
+ int64_t next_notify_time;
+
+ qemu_event_set(&s->colo_checkpoint_event);
+ s->colo_checkpoint_time = qemu_clock_get_ms(QEMU_CLOCK_HOST);
+ next_notify_time = s->colo_checkpoint_time + migrate_checkpoint_delay();
+ timer_mod(s->colo_delay_timer, next_notify_time);
+}
+
+void colo_checkpoint_delay_set(void)
+{
+ if (migration_in_colo_state()) {
+ colo_checkpoint_notify(migrate_get_current());
+ }
+}
+
static void secondary_vm_do_failover(void)
{
/* COLO needs enable block-replication */
-#ifdef CONFIG_REPLICATION
int old_state;
MigrationIncomingState *mis = migration_incoming_get_current();
Error *local_err = NULL;
@@ -133,14 +148,10 @@
if (mis->migration_incoming_co) {
qemu_coroutine_enter(mis->migration_incoming_co);
}
-#else
- abort();
-#endif
}
static void primary_vm_do_failover(void)
{
-#ifdef CONFIG_REPLICATION
MigrationState *s = migrate_get_current();
int old_state;
Error *local_err = NULL;
@@ -181,9 +192,6 @@
/* Notify COLO thread that failover work is finished */
qemu_sem_post(&s->colo_exit_sem);
-#else
- abort();
-#endif
}
COLOMode get_colo_mode(void)
@@ -217,7 +225,6 @@
}
}
-#ifdef CONFIG_REPLICATION
void qmp_xen_set_replication(bool enable, bool primary,
bool has_failover, bool failover,
Error **errp)
@@ -271,7 +278,6 @@
/* Notify all filters of all NIC to do checkpoint */
colo_notify_filters_event(COLO_EVENT_CHECKPOINT, errp);
}
-#endif
COLOStatus *qmp_query_colo_status(Error **errp)
{
@@ -435,15 +441,11 @@
}
qemu_mutex_lock_iothread();
-#ifdef CONFIG_REPLICATION
replication_do_checkpoint_all(&local_err);
if (local_err) {
qemu_mutex_unlock_iothread();
goto out;
}
-#else
- abort();
-#endif
colo_send_message(s->to_dst_file, COLO_MESSAGE_VMSTATE_SEND, &local_err);
if (local_err) {
@@ -561,15 +563,11 @@
object_unref(OBJECT(bioc));
qemu_mutex_lock_iothread();
-#ifdef CONFIG_REPLICATION
replication_start_all(REPLICATION_MODE_PRIMARY, &local_err);
if (local_err) {
qemu_mutex_unlock_iothread();
goto out;
}
-#else
- abort();
-#endif
vm_start();
qemu_mutex_unlock_iothread();
@@ -644,17 +642,6 @@
}
}
-void colo_checkpoint_notify(void *opaque)
-{
- MigrationState *s = opaque;
- int64_t next_notify_time;
-
- qemu_event_set(&s->colo_checkpoint_event);
- s->colo_checkpoint_time = qemu_clock_get_ms(QEMU_CLOCK_HOST);
- next_notify_time = s->colo_checkpoint_time + migrate_checkpoint_delay();
- timer_mod(s->colo_delay_timer, next_notify_time);
-}
-
void migrate_start_colo_process(MigrationState *s)
{
qemu_mutex_unlock_iothread();
@@ -748,7 +735,6 @@
return;
}
-#ifdef CONFIG_REPLICATION
replication_get_error_all(&local_err);
if (local_err) {
error_propagate(errp, local_err);
@@ -765,9 +751,6 @@
qemu_mutex_unlock_iothread();
return;
}
-#else
- abort();
-#endif
/* Notify all filters of all NIC to do checkpoint */
colo_notify_filters_event(COLO_EVENT_CHECKPOINT, &local_err);
@@ -874,15 +857,11 @@
object_unref(OBJECT(bioc));
qemu_mutex_lock_iothread();
-#ifdef CONFIG_REPLICATION
replication_start_all(REPLICATION_MODE_SECONDARY, &local_err);
if (local_err) {
qemu_mutex_unlock_iothread();
goto out;
}
-#else
- abort();
-#endif
vm_start();
qemu_mutex_unlock_iothread();
trace_colo_vm_state_change("stop", "run");
diff --git a/migration/meson.build b/migration/meson.build
index 75de868..eb41b77 100644
--- a/migration/meson.build
+++ b/migration/meson.build
@@ -13,8 +13,6 @@
'block-dirty-bitmap.c',
'channel.c',
'channel-block.c',
- 'colo-failover.c',
- 'colo.c',
'exec.c',
'fd.c',
'global_state.c',
@@ -33,6 +31,10 @@
'threadinfo.c',
), gnutls)
+if get_option('replication').allowed()
+ softmmu_ss.add(files('colo-failover.c', 'colo.c'))
+endif
+
softmmu_ss.add(when: rdma, if_true: files('rdma.c'))
if get_option('live_block_migration').allowed()
softmmu_ss.add(files('block.c'))
diff --git a/migration/migration-hmp-cmds.c b/migration/migration-hmp-cmds.c
index 4e9f00e..9885d7c 100644
--- a/migration/migration-hmp-cmds.c
+++ b/migration/migration-hmp-cmds.c
@@ -643,6 +643,7 @@
hmp_handle_error(mon, err);
}
+#ifdef CONFIG_REPLICATION
void hmp_x_colo_lost_heartbeat(Monitor *mon, const QDict *qdict)
{
Error *err = NULL;
@@ -650,6 +651,7 @@
qmp_x_colo_lost_heartbeat(&err);
hmp_handle_error(mon, err);
}
+#endif
typedef struct HMPMigrationStatus {
QEMUTimer *timer;
diff --git a/migration/migration.c b/migration/migration.c
index f9f12a1..439e865 100644
--- a/migration/migration.c
+++ b/migration/migration.c
@@ -392,6 +392,18 @@
int migration_incoming_enable_colo(void)
{
+#ifndef CONFIG_REPLICATION
+ error_report("ENABLE_COLO command come in migration stream, but COLO "
+ "module is not built in");
+ return -ENOTSUP;
+#endif
+
+ if (!migrate_colo()) {
+ error_report("ENABLE_COLO command come in migration stream, but c-colo "
+ "capability is not set");
+ return -EINVAL;
+ }
+
if (ram_block_discard_disable(true)) {
error_report("COLO: cannot disable RAM discard");
return -EBUSY;
@@ -536,8 +548,15 @@
/* Else if something went wrong then just fall out of the normal exit */
}
+ if (ret < 0) {
+ error_report("load of migration failed: %s", strerror(-ret));
+ goto fail;
+ }
+
/* we get COLO info, and know if we are in COLO mode */
- if (!ret && migration_incoming_colo_enabled()) {
+ if (migration_incoming_colo_enabled()) {
+ QemuThread colo_incoming_thread;
+
/* Make sure all file formats throw away their mutable metadata */
bdrv_activate_all(&local_err);
if (local_err) {
@@ -545,23 +564,18 @@
goto fail;
}
- qemu_thread_create(&mis->colo_incoming_thread, "COLO incoming",
+ qemu_thread_create(&colo_incoming_thread, "COLO incoming",
colo_process_incoming_thread, mis, QEMU_THREAD_JOINABLE);
- mis->have_colo_incoming_thread = true;
qemu_coroutine_yield();
qemu_mutex_unlock_iothread();
/* Wait checkpoint incoming thread exit before free resource */
- qemu_thread_join(&mis->colo_incoming_thread);
+ qemu_thread_join(&colo_incoming_thread);
qemu_mutex_lock_iothread();
/* We hold the global iothread lock, so it is safe here */
colo_release_ram_cache();
}
- if (ret < 0) {
- error_report("load of migration failed: %s", strerror(-ret));
- goto fail;
- }
mis->bh = qemu_bh_new(process_incoming_migration_bh, mis);
qemu_bh_schedule(mis->bh);
mis->migration_incoming_co = NULL;
@@ -2777,10 +2791,7 @@
runstate_set(RUN_STATE_POSTMIGRATE);
break;
case MIGRATION_STATUS_COLO:
- if (!migrate_colo()) {
- error_report("%s: critical error: calling COLO code without "
- "COLO enabled", __func__);
- }
+ assert(migrate_colo());
migrate_start_colo_process(s);
s->vm_was_running = true;
/* Fallthrough */
diff --git a/migration/migration.h b/migration/migration.h
index 3a91851..7721c76 100644
--- a/migration/migration.h
+++ b/migration/migration.h
@@ -162,8 +162,6 @@
int state;
- bool have_colo_incoming_thread;
- QemuThread colo_incoming_thread;
/* The coroutine we should enter (back) after failover */
Coroutine *migration_incoming_co;
QemuSemaphore colo_incoming_sem;
diff --git a/migration/multifd.c b/migration/multifd.c
index 4e71c19..5c4298e 100644
--- a/migration/multifd.c
+++ b/migration/multifd.c
@@ -281,7 +281,6 @@
static int multifd_recv_unfill_packet(MultiFDRecvParams *p, Error **errp)
{
MultiFDPacket_t *packet = p->packet;
- RAMBlock *block;
int i;
packet->magic = be32_to_cpu(packet->magic);
@@ -331,21 +330,21 @@
/* make sure that ramblock is 0 terminated */
packet->ramblock[255] = 0;
- block = qemu_ram_block_by_name(packet->ramblock);
- if (!block) {
+ p->block = qemu_ram_block_by_name(packet->ramblock);
+ if (!p->block) {
error_setg(errp, "multifd: unknown ram block %s",
packet->ramblock);
return -1;
}
- p->host = block->host;
+ p->host = p->block->host;
for (i = 0; i < p->normal_num; i++) {
uint64_t offset = be64_to_cpu(packet->offset[i]);
- if (offset > (block->used_length - p->page_size)) {
+ if (offset > (p->block->used_length - p->page_size)) {
error_setg(errp, "multifd: offset too long %" PRIu64
" (max " RAM_ADDR_FMT ")",
- offset, block->used_length);
+ offset, p->block->used_length);
return -1;
}
p->normal[i] = offset;
diff --git a/migration/multifd.h b/migration/multifd.h
index 7cfc265..a835643 100644
--- a/migration/multifd.h
+++ b/migration/multifd.h
@@ -175,6 +175,8 @@
uint32_t next_packet_size;
/* packets sent through this channel */
uint64_t num_packets;
+ /* ramblock */
+ RAMBlock *block;
/* ramblock host address */
uint8_t *host;
/* non zero pages recv through this channel */
diff --git a/migration/options.c b/migration/options.c
index 2e759cc..7ed88b7 100644
--- a/migration/options.c
+++ b/migration/options.c
@@ -598,7 +598,7 @@
MigrationCapabilityStatusList *cap;
bool new_caps[MIGRATION_CAPABILITY__MAX];
- if (migration_is_running(s->state)) {
+ if (migration_is_running(s->state) || migration_in_colo_state()) {
error_setg(errp, QERR_MIGRATION_ACTIVE);
return;
}
@@ -1253,9 +1253,7 @@
if (params->has_x_checkpoint_delay) {
s->parameters.x_checkpoint_delay = params->x_checkpoint_delay;
- if (migration_in_colo_state()) {
- colo_checkpoint_notify(s);
- }
+ colo_checkpoint_delay_set();
}
if (params->has_block_incremental) {
diff --git a/migration/ram.c b/migration/ram.c
index f78e991..f69d8d4 100644
--- a/migration/ram.c
+++ b/migration/ram.c
@@ -3408,6 +3408,18 @@
return ((uintptr_t)block->host + offset) & (block->page_size - 1);
}
+void colo_record_bitmap(RAMBlock *block, ram_addr_t *normal, uint32_t pages)
+{
+ qemu_mutex_lock(&ram_state->bitmap_mutex);
+ for (int i = 0; i < pages; i++) {
+ ram_addr_t offset = normal[i];
+ ram_state->migration_dirty_pages += !test_and_set_bit(
+ offset >> TARGET_PAGE_BITS,
+ block->bmap);
+ }
+ qemu_mutex_unlock(&ram_state->bitmap_mutex);
+}
+
static inline void *colo_cache_from_block_offset(RAMBlock *block,
ram_addr_t offset, bool record_bitmap)
{
@@ -3425,9 +3437,8 @@
* It help us to decide which pages in ram cache should be flushed
* into VM's RAM later.
*/
- if (record_bitmap &&
- !test_and_set_bit(offset >> TARGET_PAGE_BITS, block->bmap)) {
- ram_state->migration_dirty_pages++;
+ if (record_bitmap) {
+ colo_record_bitmap(block, &offset, 1);
}
return block->colo_cache + offset;
}
@@ -3803,6 +3814,7 @@
unsigned long offset = 0;
memory_global_dirty_log_sync();
+ qemu_mutex_lock(&ram_state->bitmap_mutex);
WITH_RCU_READ_LOCK_GUARD() {
RAMBLOCK_FOREACH_NOT_IGNORED(block) {
ramblock_sync_dirty_bitmap(ram_state, block);
@@ -3837,6 +3849,7 @@
}
}
}
+ qemu_mutex_unlock(&ram_state->bitmap_mutex);
trace_colo_flush_ram_cache_end();
}
diff --git a/migration/ram.h b/migration/ram.h
index 6fffbeb..ea1f3c2 100644
--- a/migration/ram.h
+++ b/migration/ram.h
@@ -82,6 +82,7 @@
void colo_flush_ram_cache(void);
void colo_release_ram_cache(void);
void colo_incoming_start_dirty_log(void);
+void colo_record_bitmap(RAMBlock *block, ram_addr_t *normal, uint32_t pages);
/* Background snapshot */
bool ram_write_tracking_available(void);
diff --git a/qapi/migration.json b/qapi/migration.json
index 952d3e2..179af0c 100644
--- a/qapi/migration.json
+++ b/qapi/migration.json
@@ -1366,7 +1366,8 @@
# <- { "return": {} }
##
{ 'command': 'x-colo-lost-heartbeat',
- 'features': [ 'unstable' ] }
+ 'features': [ 'unstable' ],
+ 'if': 'CONFIG_REPLICATION' }
##
# @migrate_cancel:
@@ -1638,7 +1639,8 @@
##
{ 'struct': 'COLOStatus',
'data': { 'mode': 'COLOMode', 'last-mode': 'COLOMode',
- 'reason': 'COLOExitReason' } }
+ 'reason': 'COLOExitReason' },
+ 'if': 'CONFIG_REPLICATION' }
##
# @query-colo-status:
@@ -1655,7 +1657,8 @@
# Since: 3.1
##
{ 'command': 'query-colo-status',
- 'returns': 'COLOStatus' }
+ 'returns': 'COLOStatus',
+ 'if': 'CONFIG_REPLICATION' }
##
# @migrate-recover:
diff --git a/stubs/colo.c b/stubs/colo.c
new file mode 100644
index 0000000..cf9816d
--- /dev/null
+++ b/stubs/colo.c
@@ -0,0 +1,39 @@
+#include "qemu/osdep.h"
+#include "qemu/notify.h"
+#include "net/colo-compare.h"
+#include "migration/colo.h"
+#include "migration/migration.h"
+#include "qemu/error-report.h"
+#include "qapi/qapi-commands-migration.h"
+
+void colo_shutdown(void)
+{
+}
+
+void *colo_process_incoming_thread(void *opaque)
+{
+ error_report("Impossible happend: trying to start COLO thread when COLO "
+ "module is not built in");
+ abort();
+}
+
+void colo_checkpoint_delay_set(void)
+{
+}
+
+void migrate_start_colo_process(MigrationState *s)
+{
+ error_report("Impossible happend: trying to start COLO when COLO "
+ "module is not built in");
+ abort();
+}
+
+bool migration_in_colo_state(void)
+{
+ return false;
+}
+
+bool migration_incoming_in_colo_state(void)
+{
+ return false;
+}
diff --git a/stubs/meson.build b/stubs/meson.build
index b2b5956..8412cad 100644
--- a/stubs/meson.build
+++ b/stubs/meson.build
@@ -45,6 +45,7 @@
stub_ss.add(files('target-monitor-defs.c'))
stub_ss.add(files('trace-control.c'))
stub_ss.add(files('uuid.c'))
+stub_ss.add(files('colo.c'))
stub_ss.add(files('vmstate.c'))
stub_ss.add(files('vm-stop.c'))
stub_ss.add(files('win32-kbd-hook.c'))
