blob: 16c30a9d7aa9d004e2b9a112e88f3cbaf37979c3 [file] [log] [blame]
Juan Quintela56e93d22015-05-07 19:33:31 +02001/*
2 * QEMU System Emulator
3 *
4 * Copyright (c) 2003-2008 Fabrice Bellard
Juan Quintela76cc7b52015-05-08 13:20:21 +02005 * Copyright (c) 2011-2015 Red Hat Inc
6 *
7 * Authors:
8 * Juan Quintela <quintela@redhat.com>
Juan Quintela56e93d22015-05-07 19:33:31 +02009 *
10 * Permission is hereby granted, free of charge, to any person obtaining a copy
11 * of this software and associated documentation files (the "Software"), to deal
12 * in the Software without restriction, including without limitation the rights
13 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
14 * copies of the Software, and to permit persons to whom the Software is
15 * furnished to do so, subject to the following conditions:
16 *
17 * The above copyright notice and this permission notice shall be included in
18 * all copies or substantial portions of the Software.
19 *
20 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
21 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
23 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
24 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
25 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
26 * THE SOFTWARE.
27 */
Markus Armbrustere688df62018-02-01 12:18:31 +010028
Peter Maydell1393a482016-01-26 18:16:54 +000029#include "qemu/osdep.h"
Veronia Bahaaf348b6d2016-03-20 19:16:19 +020030#include "qemu/cutils.h"
Juan Quintela56e93d22015-05-07 19:33:31 +020031#include "qemu/bitops.h"
32#include "qemu/bitmap.h"
Peter Maydellb85ea5f2022-02-08 20:08:52 +000033#include "qemu/madvise.h"
Juan Quintela7205c9e2015-05-08 13:54:36 +020034#include "qemu/main-loop.h"
Juan Quintela709e3fe2017-04-05 21:47:50 +020035#include "xbzrle.h"
Lukas Straubb5ca3362023-04-20 11:48:20 +020036#include "ram-compress.h"
Juan Quintela7b1e1a22017-04-17 20:26:27 +020037#include "ram.h"
Juan Quintela6666c962017-04-24 20:07:27 +020038#include "migration.h"
Juan Quintela947701c2023-04-26 19:04:06 +020039#include "migration-stats.h"
Juan Quintelaf2a8f0a2017-04-24 13:42:55 +020040#include "migration/register.h"
Juan Quintela7b1e1a22017-04-17 20:26:27 +020041#include "migration/misc.h"
Juan Quintela08a0aee2017-04-20 18:52:18 +020042#include "qemu-file.h"
Juan Quintelabe07b0a2017-04-20 13:12:24 +020043#include "postcopy-ram.h"
Michael S. Tsirkin53d37d32018-05-03 22:50:51 +030044#include "page_cache.h"
Juan Quintela56e93d22015-05-07 19:33:31 +020045#include "qemu/error-report.h"
Markus Armbrustere688df62018-02-01 12:18:31 +010046#include "qapi/error.h"
Juan Quintelaab7cbb02019-05-15 13:37:46 +020047#include "qapi/qapi-types-migration.h"
Markus Armbruster9af23982018-02-11 10:36:01 +010048#include "qapi/qapi-events-migration.h"
Hyman Huang(黄勇)acac51b2023-06-08 00:12:40 +080049#include "qapi/qapi-commands-migration.h"
Juan Quintela8acabf62017-10-05 22:00:31 +020050#include "qapi/qmp/qerror.h"
Juan Quintela56e93d22015-05-07 19:33:31 +020051#include "trace.h"
Juan Quintela56e93d22015-05-07 19:33:31 +020052#include "exec/ram_addr.h"
Alexey Perevalovf9494612017-10-05 14:13:20 +030053#include "exec/target_page.h"
Juan Quintela56e93d22015-05-07 19:33:31 +020054#include "qemu/rcu_queue.h"
zhanghailianga91246c2016-10-27 14:42:59 +080055#include "migration/colo.h"
Michael S. Tsirkin53d37d32018-05-03 22:50:51 +030056#include "block.h"
Claudio Fontanab0c3cf92020-06-29 11:35:03 +020057#include "sysemu/cpu-throttle.h"
Peter Xuedd090c2018-05-02 18:47:32 +080058#include "savevm.h"
Juan Quintelab9ee2f72016-01-15 11:40:13 +010059#include "qemu/iov.h"
Juan Quintelad32ca5a2020-01-22 16:16:07 +010060#include "multifd.h"
Andrey Gruzdev278e2f52021-01-29 13:14:05 +030061#include "sysemu/runstate.h"
Juan Quintela48408172023-10-11 22:35:16 +020062#include "rdma.h"
Juan Quintela1f0776f2023-03-01 21:18:45 +010063#include "options.h"
Hyman Huang(黄勇)acac51b2023-06-08 00:12:40 +080064#include "sysemu/dirtylimit.h"
65#include "sysemu/kvm.h"
Andrey Gruzdev278e2f52021-01-29 13:14:05 +030066
Lukas Straube5fdf922021-07-04 18:14:44 +020067#include "hw/boards.h" /* for machine_dump_guest_core() */
68
Andrey Gruzdev278e2f52021-01-29 13:14:05 +030069#if defined(__linux__)
70#include "qemu/userfaultfd.h"
71#endif /* defined(__linux__) */
Juan Quintela56e93d22015-05-07 19:33:31 +020072
Juan Quintela56e93d22015-05-07 19:33:31 +020073/***********************************************************/
74/* ram save/restore */
75
Juan Quintela7b548762022-07-28 10:14:42 +020076/*
77 * RAM_SAVE_FLAG_ZERO used to be named RAM_SAVE_FLAG_COMPRESS, it
78 * worked for pages that were filled with the same char. We switched
Juan Quintelabb890ed2017-04-28 09:39:55 +020079 * it to only search for the zero value. And to avoid confusion with
Juan Quintela7b548762022-07-28 10:14:42 +020080 * RAM_SAVE_FLAG_COMPRESS_PAGE just rename it.
Juan Quintelabb890ed2017-04-28 09:39:55 +020081 */
Juan Quintela7b548762022-07-28 10:14:42 +020082/*
83 * RAM_SAVE_FLAG_FULL was obsoleted in 2009, it can be reused now
84 */
85#define RAM_SAVE_FLAG_FULL 0x01
Juan Quintelabb890ed2017-04-28 09:39:55 +020086#define RAM_SAVE_FLAG_ZERO 0x02
Juan Quintela56e93d22015-05-07 19:33:31 +020087#define RAM_SAVE_FLAG_MEM_SIZE 0x04
88#define RAM_SAVE_FLAG_PAGE 0x08
89#define RAM_SAVE_FLAG_EOS 0x10
90#define RAM_SAVE_FLAG_CONTINUE 0x20
91#define RAM_SAVE_FLAG_XBZRLE 0x40
Juan Quintela10cb3332023-10-11 22:35:22 +020092/* 0x80 is reserved in rdma.h for RAM_SAVE_FLAG_HOOK */
Juan Quintela56e93d22015-05-07 19:33:31 +020093#define RAM_SAVE_FLAG_COMPRESS_PAGE 0x100
Juan Quintela294e5a42022-06-21 13:36:11 +020094#define RAM_SAVE_FLAG_MULTIFD_FLUSH 0x200
Juan Quintela7b548762022-07-28 10:14:42 +020095/* We can't use any flag that is bigger than 0x200 */
Juan Quintela56e93d22015-05-07 19:33:31 +020096
Juan Quintela93604472017-06-06 19:49:03 +020097XBZRLECacheStats xbzrle_counters;
98
Peter Xuf1668762022-10-11 17:55:55 -040099/* used by the search for pages to send */
100struct PageSearchStatus {
101 /* The migration channel used for a specific host page */
102 QEMUFile *pss_channel;
Peter Xuec6f3ab2022-10-11 17:55:56 -0400103 /* Last block from where we have sent data */
104 RAMBlock *last_sent_block;
Peter Xuf1668762022-10-11 17:55:55 -0400105 /* Current block being searched */
106 RAMBlock *block;
107 /* Current page to search from */
108 unsigned long page;
109 /* Set once we wrap around */
110 bool complete_round;
Peter Xuf1668762022-10-11 17:55:55 -0400111 /* Whether we're sending a host page */
112 bool host_page_sending;
113 /* The start/end of current host page. Invalid if host_page_sending==false */
114 unsigned long host_page_start;
115 unsigned long host_page_end;
116};
117typedef struct PageSearchStatus PageSearchStatus;
118
Juan Quintela56e93d22015-05-07 19:33:31 +0200119/* struct contains XBZRLE cache and a static page
120 used by the compression */
121static struct {
122 /* buffer used for XBZRLE encoding */
123 uint8_t *encoded_buf;
124 /* buffer for storing page content */
125 uint8_t *current_buf;
126 /* Cache for XBZRLE, Protected by lock. */
127 PageCache *cache;
128 QemuMutex lock;
Juan Quintelac00e0922017-05-09 16:22:01 +0200129 /* it will store a page full of zeros */
130 uint8_t *zero_target_page;
Juan Quintelaf265e0e2017-06-28 11:52:27 +0200131 /* buffer used for XBZRLE decoding */
132 uint8_t *decoded_buf;
Juan Quintela56e93d22015-05-07 19:33:31 +0200133} XBZRLE;
134
Juan Quintela56e93d22015-05-07 19:33:31 +0200135static void XBZRLE_cache_lock(void)
136{
Juan Quintela87dca0c2023-03-01 22:20:13 +0100137 if (migrate_xbzrle()) {
Juan Quintela56e93d22015-05-07 19:33:31 +0200138 qemu_mutex_lock(&XBZRLE.lock);
Bihong Yuf4c51a62020-10-20 11:10:45 +0800139 }
Juan Quintela56e93d22015-05-07 19:33:31 +0200140}
141
142static void XBZRLE_cache_unlock(void)
143{
Juan Quintela87dca0c2023-03-01 22:20:13 +0100144 if (migrate_xbzrle()) {
Juan Quintela56e93d22015-05-07 19:33:31 +0200145 qemu_mutex_unlock(&XBZRLE.lock);
Bihong Yuf4c51a62020-10-20 11:10:45 +0800146 }
Juan Quintela56e93d22015-05-07 19:33:31 +0200147}
148
Juan Quintela3d0684b2017-03-23 15:06:39 +0100149/**
150 * xbzrle_cache_resize: resize the xbzrle cache
151 *
Daniel P. Berrangécbde7be2021-02-19 18:40:12 +0000152 * This function is called from migrate_params_apply in main
Juan Quintela3d0684b2017-03-23 15:06:39 +0100153 * thread, possibly while a migration is in progress. A running
154 * migration may be using the cache and might finish during this call,
155 * hence changes to the cache are protected by XBZRLE.lock().
156 *
Juan Quintelac9dede22017-10-06 23:03:55 +0200157 * Returns 0 for success or -1 for error
Juan Quintela3d0684b2017-03-23 15:06:39 +0100158 *
159 * @new_size: new cache size
Juan Quintela8acabf62017-10-05 22:00:31 +0200160 * @errp: set *errp if the check failed, with reason
Juan Quintela56e93d22015-05-07 19:33:31 +0200161 */
Markus Armbruster8b9407a2021-02-02 15:17:32 +0100162int xbzrle_cache_resize(uint64_t new_size, Error **errp)
Juan Quintela56e93d22015-05-07 19:33:31 +0200163{
164 PageCache *new_cache;
Juan Quintelac9dede22017-10-06 23:03:55 +0200165 int64_t ret = 0;
Juan Quintela56e93d22015-05-07 19:33:31 +0200166
Juan Quintela8acabf62017-10-05 22:00:31 +0200167 /* Check for truncation */
168 if (new_size != (size_t)new_size) {
169 error_setg(errp, QERR_INVALID_PARAMETER_VALUE, "cache size",
170 "exceeding address space");
171 return -1;
172 }
173
Juan Quintela2a313e52017-10-06 23:00:12 +0200174 if (new_size == migrate_xbzrle_cache_size()) {
175 /* nothing to do */
Juan Quintelac9dede22017-10-06 23:03:55 +0200176 return 0;
Juan Quintela2a313e52017-10-06 23:00:12 +0200177 }
178
Juan Quintela56e93d22015-05-07 19:33:31 +0200179 XBZRLE_cache_lock();
180
181 if (XBZRLE.cache != NULL) {
Juan Quintela80f8dfd2017-10-06 22:30:45 +0200182 new_cache = cache_init(new_size, TARGET_PAGE_SIZE, errp);
Juan Quintela56e93d22015-05-07 19:33:31 +0200183 if (!new_cache) {
Juan Quintela56e93d22015-05-07 19:33:31 +0200184 ret = -1;
185 goto out;
186 }
187
188 cache_fini(XBZRLE.cache);
189 XBZRLE.cache = new_cache;
190 }
Juan Quintela56e93d22015-05-07 19:33:31 +0200191out:
192 XBZRLE_cache_unlock();
193 return ret;
194}
195
Peter Xu20123ee2022-10-11 17:55:46 -0400196static bool postcopy_preempt_active(void)
197{
198 return migrate_postcopy_preempt() && migration_in_postcopy();
199}
200
David Hildenbrandf161c882023-07-06 09:56:08 +0200201bool migrate_ram_is_ignored(RAMBlock *block)
Yury Kotovfbd162e2019-02-15 20:45:46 +0300202{
203 return !qemu_ram_is_migratable(block) ||
Steve Sistareb0182e52023-06-07 08:18:36 -0700204 (migrate_ignore_shared() && qemu_ram_is_shared(block)
205 && qemu_ram_is_named_file(block));
Yury Kotovfbd162e2019-02-15 20:45:46 +0300206}
207
Dr. David Alan Gilbert343f6322018-06-05 17:25:45 +0100208#undef RAMBLOCK_FOREACH
209
Yury Kotovfbd162e2019-02-15 20:45:46 +0300210int foreach_not_ignored_block(RAMBlockIterFunc func, void *opaque)
211{
212 RAMBlock *block;
213 int ret = 0;
214
Dr. David Alan Gilbert89ac5a12019-10-07 15:36:39 +0100215 RCU_READ_LOCK_GUARD();
216
Yury Kotovfbd162e2019-02-15 20:45:46 +0300217 RAMBLOCK_FOREACH_NOT_IGNORED(block) {
218 ret = func(block, opaque);
219 if (ret) {
220 break;
221 }
222 }
Yury Kotovfbd162e2019-02-15 20:45:46 +0300223 return ret;
224}
225
Alexey Perevalovf9494612017-10-05 14:13:20 +0300226static void ramblock_recv_map_init(void)
227{
228 RAMBlock *rb;
229
Yury Kotovfbd162e2019-02-15 20:45:46 +0300230 RAMBLOCK_FOREACH_NOT_IGNORED(rb) {
Alexey Perevalovf9494612017-10-05 14:13:20 +0300231 assert(!rb->receivedmap);
232 rb->receivedmap = bitmap_new(rb->max_length >> qemu_target_page_bits());
233 }
234}
235
236int ramblock_recv_bitmap_test(RAMBlock *rb, void *host_addr)
237{
238 return test_bit(ramblock_recv_bitmap_offset(host_addr, rb),
239 rb->receivedmap);
240}
241
Dr. David Alan Gilbert1cba9f62018-03-12 17:21:08 +0000242bool ramblock_recv_bitmap_test_byte_offset(RAMBlock *rb, uint64_t byte_offset)
243{
244 return test_bit(byte_offset >> TARGET_PAGE_BITS, rb->receivedmap);
245}
246
Alexey Perevalovf9494612017-10-05 14:13:20 +0300247void ramblock_recv_bitmap_set(RAMBlock *rb, void *host_addr)
248{
249 set_bit_atomic(ramblock_recv_bitmap_offset(host_addr, rb), rb->receivedmap);
250}
251
252void ramblock_recv_bitmap_set_range(RAMBlock *rb, void *host_addr,
253 size_t nr)
254{
255 bitmap_set_atomic(rb->receivedmap,
256 ramblock_recv_bitmap_offset(host_addr, rb),
257 nr);
258}
259
Peter Xua335deb2018-05-02 18:47:28 +0800260#define RAMBLOCK_RECV_BITMAP_ENDING (0x0123456789abcdefULL)
261
262/*
263 * Format: bitmap_size (8 bytes) + whole_bitmap (N bytes).
264 *
265 * Returns >0 if success with sent bytes, or <0 if error.
266 */
267int64_t ramblock_recv_bitmap_send(QEMUFile *file,
268 const char *block_name)
269{
270 RAMBlock *block = qemu_ram_block_by_name(block_name);
271 unsigned long *le_bitmap, nbits;
272 uint64_t size;
273
274 if (!block) {
275 error_report("%s: invalid block name: %s", __func__, block_name);
276 return -1;
277 }
278
David Hildenbrand898ba902021-04-29 13:27:06 +0200279 nbits = block->postcopy_length >> TARGET_PAGE_BITS;
Peter Xua335deb2018-05-02 18:47:28 +0800280
281 /*
282 * Make sure the tmp bitmap buffer is big enough, e.g., on 32bit
283 * machines we may need 4 more bytes for padding (see below
284 * comment). So extend it a bit before hand.
285 */
286 le_bitmap = bitmap_new(nbits + BITS_PER_LONG);
287
288 /*
289 * Always use little endian when sending the bitmap. This is
290 * required that when source and destination VMs are not using the
zhaolichang3a4452d2020-09-17 15:50:21 +0800291 * same endianness. (Note: big endian won't work.)
Peter Xua335deb2018-05-02 18:47:28 +0800292 */
293 bitmap_to_le(le_bitmap, block->receivedmap, nbits);
294
295 /* Size of the bitmap, in bytes */
Peter Xua725ef92018-07-10 17:18:55 +0800296 size = DIV_ROUND_UP(nbits, 8);
Peter Xua335deb2018-05-02 18:47:28 +0800297
298 /*
299 * size is always aligned to 8 bytes for 64bit machines, but it
300 * may not be true for 32bit machines. We need this padding to
301 * make sure the migration can survive even between 32bit and
302 * 64bit machines.
303 */
304 size = ROUND_UP(size, 8);
305
306 qemu_put_be64(file, size);
307 qemu_put_buffer(file, (const uint8_t *)le_bitmap, size);
308 /*
309 * Mark as an end, in case the middle part is screwed up due to
zhaolichang3a4452d2020-09-17 15:50:21 +0800310 * some "mysterious" reason.
Peter Xua335deb2018-05-02 18:47:28 +0800311 */
312 qemu_put_be64(file, RAMBLOCK_RECV_BITMAP_ENDING);
313 qemu_fflush(file);
314
Peter Xubf269902018-05-25 09:50:42 +0800315 g_free(le_bitmap);
Peter Xua335deb2018-05-02 18:47:28 +0800316
317 if (qemu_file_get_error(file)) {
318 return qemu_file_get_error(file);
319 }
320
321 return size + sizeof(size);
322}
323
Juan Quintelaec481c62017-03-20 22:12:40 +0100324/*
325 * An outstanding page request, on the source, having been received
326 * and queued
327 */
328struct RAMSrcPageRequest {
329 RAMBlock *rb;
330 hwaddr offset;
331 hwaddr len;
332
333 QSIMPLEQ_ENTRY(RAMSrcPageRequest) next_req;
334};
335
Juan Quintela6f37bb82017-03-13 19:26:29 +0100336/* State of RAM for migration */
337struct RAMState {
Peter Xuf1668762022-10-11 17:55:55 -0400338 /*
339 * PageSearchStatus structures for the channels when send pages.
340 * Protected by the bitmap_mutex.
341 */
342 PageSearchStatus pss[RAM_CHANNEL_MAX];
Andrey Gruzdev278e2f52021-01-29 13:14:05 +0300343 /* UFFD file descriptor, used in 'write-tracking' migration */
344 int uffdio_fd;
Juan Quintela8d80e192022-05-10 19:37:36 +0200345 /* total ram size in bytes */
346 uint64_t ram_bytes_total;
Juan Quintela6f37bb82017-03-13 19:26:29 +0100347 /* Last block that we have visited searching for dirty pages */
348 RAMBlock *last_seen_block;
Juan Quintela269ace22017-03-21 15:23:31 +0100349 /* Last dirty target page we have sent */
350 ram_addr_t last_page;
Juan Quintela6f37bb82017-03-13 19:26:29 +0100351 /* last ram version we have seen */
352 uint32_t last_version;
Juan Quintela8d820d62017-03-13 19:35:50 +0100353 /* How many times we have dirty too many pages */
354 int dirty_rate_high_cnt;
Juan Quintelaf664da82017-03-13 19:44:57 +0100355 /* these variables are used for bitmap sync */
356 /* last time we did a full bitmap_sync */
357 int64_t time_last_bitmap_sync;
Juan Quintelaeac74152017-03-28 14:59:01 +0200358 /* bytes transferred at start_time */
Juan Quintelac4bdf0c2017-03-28 14:59:54 +0200359 uint64_t bytes_xfer_prev;
Juan Quintelaa66cd902017-03-28 15:02:43 +0200360 /* number of dirty pages since start_time */
Juan Quintela68908ed2017-03-28 15:05:53 +0200361 uint64_t num_dirty_pages_period;
Juan Quintelab5833fd2017-03-13 19:49:19 +0100362 /* xbzrle misses since the beginning of the period */
363 uint64_t xbzrle_cache_miss_prev;
Wei Wange460a4b2020-04-30 08:59:35 +0800364 /* Amount of xbzrle pages since the beginning of the period */
365 uint64_t xbzrle_pages_prev;
366 /* Amount of xbzrle encoded bytes since the beginning of the period */
367 uint64_t xbzrle_bytes_prev;
Juan Quintelaf3095cc2023-05-04 13:53:23 +0200368 /* Are we really using XBZRLE (e.g., after the first round). */
369 bool xbzrle_started;
Juan Quintela05931ec2021-12-15 19:01:21 +0100370 /* Are we on the last stage of migration */
371 bool last_stage;
Xiao Guangrong76e03002018-09-06 15:01:00 +0800372 /* compression statistics since the beginning of the period */
373 /* amount of count that no free thread to compress data */
374 uint64_t compress_thread_busy_prev;
375 /* amount bytes after compression */
376 uint64_t compressed_size_prev;
377 /* amount of compressed pages */
378 uint64_t compress_pages_prev;
379
Xiao Guangrongbe8b02e2018-09-03 17:26:42 +0800380 /* total handled target pages at the beginning of period */
381 uint64_t target_page_count_prev;
382 /* total handled target pages since start */
383 uint64_t target_page_count;
Juan Quintela93604472017-06-06 19:49:03 +0200384 /* number of dirty bits in the bitmap */
Peter Xu2dfaf122017-08-02 17:41:19 +0800385 uint64_t migration_dirty_pages;
Peter Xuf1668762022-10-11 17:55:55 -0400386 /*
387 * Protects:
388 * - dirty/clear bitmap
389 * - migration_dirty_pages
390 * - pss structures
391 */
Juan Quintela108cfae2017-03-13 21:38:09 +0100392 QemuMutex bitmap_mutex;
Juan Quintela68a098f2017-03-14 13:48:42 +0100393 /* The RAMBlock used in the last src_page_requests */
394 RAMBlock *last_req_rb;
Juan Quintelaec481c62017-03-20 22:12:40 +0100395 /* Queue of outstanding page requests from the destination */
396 QemuMutex src_page_req_mutex;
Paolo Bonzinib58deb32018-12-06 11:58:10 +0100397 QSIMPLEQ_HEAD(, RAMSrcPageRequest) src_page_requests;
Peter Xu1015ff52023-10-04 18:02:36 -0400398
399 /*
400 * This is only used when postcopy is in recovery phase, to communicate
401 * between the migration thread and the return path thread on dirty
402 * bitmap synchronizations. This field is unused in other stages of
403 * RAM migration.
404 */
405 unsigned int postcopy_bmap_sync_requested;
Juan Quintela6f37bb82017-03-13 19:26:29 +0100406};
407typedef struct RAMState RAMState;
408
Juan Quintela53518d92017-05-04 11:46:24 +0200409static RAMState *ram_state;
Juan Quintela6f37bb82017-03-13 19:26:29 +0100410
Wei Wangbd227062018-12-11 16:24:51 +0800411static NotifierWithReturnList precopy_notifier_list;
412
Peter Xua1fe28d2022-01-19 16:09:18 +0800413/* Whether postcopy has queued requests? */
414static bool postcopy_has_request(RAMState *rs)
415{
416 return !QSIMPLEQ_EMPTY_ATOMIC(&rs->src_page_requests);
417}
418
Wei Wangbd227062018-12-11 16:24:51 +0800419void precopy_infrastructure_init(void)
420{
421 notifier_with_return_list_init(&precopy_notifier_list);
422}
423
424void precopy_add_notifier(NotifierWithReturn *n)
425{
426 notifier_with_return_list_add(&precopy_notifier_list, n);
427}
428
429void precopy_remove_notifier(NotifierWithReturn *n)
430{
431 notifier_with_return_remove(n);
432}
433
434int precopy_notify(PrecopyNotifyReason reason, Error **errp)
435{
436 PrecopyNotifyData pnd;
437 pnd.reason = reason;
438 pnd.errp = errp;
439
440 return notifier_with_return_list_notify(&precopy_notifier_list, &pnd);
441}
442
Juan Quintela9edabd42017-03-14 12:02:16 +0100443uint64_t ram_bytes_remaining(void)
444{
Dr. David Alan Gilbertbae416e2017-12-15 11:51:23 +0000445 return ram_state ? (ram_state->migration_dirty_pages * TARGET_PAGE_SIZE) :
446 0;
Juan Quintela9edabd42017-03-14 12:02:16 +0100447}
448
Juan Quintela26a26062022-02-22 21:02:03 +0100449void ram_transferred_add(uint64_t bytes)
David Edmondson4c2d0f62021-12-21 09:34:40 +0000450{
David Edmondsonae680662021-12-21 09:34:41 +0000451 if (runstate_is_running()) {
Juan Quintelaaff3f662023-04-26 19:37:19 +0200452 stat64_add(&mig_stats.precopy_bytes, bytes);
David Edmondsonae680662021-12-21 09:34:41 +0000453 } else if (migration_in_postcopy()) {
Juan Quintelaaff3f662023-04-26 19:37:19 +0200454 stat64_add(&mig_stats.postcopy_bytes, bytes);
David Edmondsonae680662021-12-21 09:34:41 +0000455 } else {
Juan Quintelaaff3f662023-04-26 19:37:19 +0200456 stat64_add(&mig_stats.downtime_bytes, bytes);
David Edmondsonae680662021-12-21 09:34:41 +0000457 }
Juan Quintelaaff3f662023-04-26 19:37:19 +0200458 stat64_add(&mig_stats.transferred, bytes);
David Edmondson4c2d0f62021-12-21 09:34:40 +0000459}
460
Juan Quintela4010ba32021-12-15 20:10:39 +0100461struct MigrationOps {
462 int (*ram_save_target_page)(RAMState *rs, PageSearchStatus *pss);
463};
464typedef struct MigrationOps MigrationOps;
465
466MigrationOps *migration_ops;
467
Peter Xu93589822022-10-11 17:55:57 -0400468static int ram_save_host_page_urgent(PageSearchStatus *pss);
469
Peter Xuebd88a42022-10-11 17:55:54 -0400470/* NOTE: page is the PFN not real ram_addr_t. */
471static void pss_init(PageSearchStatus *pss, RAMBlock *rb, ram_addr_t page)
472{
473 pss->block = rb;
474 pss->page = page;
475 pss->complete_round = false;
476}
477
Peter Xu93589822022-10-11 17:55:57 -0400478/*
479 * Check whether two PSSs are actively sending the same page. Return true
480 * if it is, false otherwise.
481 */
482static bool pss_overlap(PageSearchStatus *pss1, PageSearchStatus *pss2)
483{
484 return pss1->host_page_sending && pss2->host_page_sending &&
485 (pss1->host_page_start == pss2->host_page_start);
486}
487
Juan Quintela56e93d22015-05-07 19:33:31 +0200488/**
Juan Quintela3d0684b2017-03-23 15:06:39 +0100489 * save_page_header: write page header to wire
Juan Quintela56e93d22015-05-07 19:33:31 +0200490 *
491 * If this is the 1st block, it also writes the block identification
492 *
Juan Quintela3d0684b2017-03-23 15:06:39 +0100493 * Returns the number of bytes written
Juan Quintela56e93d22015-05-07 19:33:31 +0200494 *
Peter Xuec6f3ab2022-10-11 17:55:56 -0400495 * @pss: current PSS channel status
Juan Quintela56e93d22015-05-07 19:33:31 +0200496 * @block: block that contains the page we want to send
497 * @offset: offset inside the block for the page
498 * in the lower bits, it contains flags
499 */
Lukas Straub37502df2023-04-02 17:06:32 +0000500static size_t save_page_header(PageSearchStatus *pss, QEMUFile *f,
501 RAMBlock *block, ram_addr_t offset)
Juan Quintela56e93d22015-05-07 19:33:31 +0200502{
Liang Li9f5f3802015-07-13 17:34:10 +0800503 size_t size, len;
Peter Xuec6f3ab2022-10-11 17:55:56 -0400504 bool same_block = (block == pss->last_sent_block);
Juan Quintela56e93d22015-05-07 19:33:31 +0200505
Peter Xu10661f12022-10-11 17:55:48 -0400506 if (same_block) {
Juan Quintela24795692017-03-21 11:45:01 +0100507 offset |= RAM_SAVE_FLAG_CONTINUE;
508 }
Juan Quintela2bf3aa82017-05-10 13:28:13 +0200509 qemu_put_be64(f, offset);
Juan Quintela56e93d22015-05-07 19:33:31 +0200510 size = 8;
511
Peter Xu10661f12022-10-11 17:55:48 -0400512 if (!same_block) {
Liang Li9f5f3802015-07-13 17:34:10 +0800513 len = strlen(block->idstr);
Juan Quintela2bf3aa82017-05-10 13:28:13 +0200514 qemu_put_byte(f, len);
515 qemu_put_buffer(f, (uint8_t *)block->idstr, len);
Liang Li9f5f3802015-07-13 17:34:10 +0800516 size += 1 + len;
Peter Xuec6f3ab2022-10-11 17:55:56 -0400517 pss->last_sent_block = block;
Juan Quintela56e93d22015-05-07 19:33:31 +0200518 }
519 return size;
520}
521
Juan Quintela3d0684b2017-03-23 15:06:39 +0100522/**
Olaf Hering179a8082021-07-08 18:21:59 +0200523 * mig_throttle_guest_down: throttle down the guest
Juan Quintela3d0684b2017-03-23 15:06:39 +0100524 *
525 * Reduce amount of guest cpu execution to hopefully slow down memory
526 * writes. If guest dirty memory rate is reduced below the rate at
527 * which we can transfer pages to the destination then we should be
528 * able to complete migration. Some workloads dirty memory way too
529 * fast and will not effectively converge, even with auto-converge.
Jason J. Herne070afca2015-09-08 13:12:35 -0400530 */
Keqian Zhucbbf8182020-04-13 18:15:08 +0800531static void mig_throttle_guest_down(uint64_t bytes_dirty_period,
532 uint64_t bytes_dirty_threshold)
Jason J. Herne070afca2015-09-08 13:12:35 -0400533{
Juan Quintela2a8ec382023-03-02 01:22:44 +0100534 uint64_t pct_initial = migrate_cpu_throttle_initial();
Juan Quintela9605c2a2023-03-02 10:20:49 +0100535 uint64_t pct_increment = migrate_cpu_throttle_increment();
Juan Quintela873f6742023-03-02 10:29:51 +0100536 bool pct_tailslow = migrate_cpu_throttle_tailslow();
Juan Quintela24155bd2023-03-02 01:13:01 +0100537 int pct_max = migrate_max_cpu_throttle();
Jason J. Herne070afca2015-09-08 13:12:35 -0400538
Keqian Zhucbbf8182020-04-13 18:15:08 +0800539 uint64_t throttle_now = cpu_throttle_get_percentage();
540 uint64_t cpu_now, cpu_ideal, throttle_inc;
541
Jason J. Herne070afca2015-09-08 13:12:35 -0400542 /* We have not started throttling yet. Let's start it. */
543 if (!cpu_throttle_active()) {
544 cpu_throttle_set(pct_initial);
545 } else {
546 /* Throttling already on, just increase the rate */
Keqian Zhucbbf8182020-04-13 18:15:08 +0800547 if (!pct_tailslow) {
548 throttle_inc = pct_increment;
549 } else {
550 /* Compute the ideal CPU percentage used by Guest, which may
551 * make the dirty rate match the dirty rate threshold. */
552 cpu_now = 100 - throttle_now;
553 cpu_ideal = cpu_now * (bytes_dirty_threshold * 1.0 /
554 bytes_dirty_period);
555 throttle_inc = MIN(cpu_now - cpu_ideal, pct_increment);
556 }
557 cpu_throttle_set(MIN(throttle_now + throttle_inc, pct_max));
Jason J. Herne070afca2015-09-08 13:12:35 -0400558 }
559}
560
Rao, Lei91fe9a82021-11-09 11:04:54 +0800561void mig_throttle_counter_reset(void)
562{
563 RAMState *rs = ram_state;
564
565 rs->time_last_bitmap_sync = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
566 rs->num_dirty_pages_period = 0;
Juan Quintelaaff3f662023-04-26 19:37:19 +0200567 rs->bytes_xfer_prev = stat64_get(&mig_stats.transferred);
Rao, Lei91fe9a82021-11-09 11:04:54 +0800568}
569
Juan Quintela3d0684b2017-03-23 15:06:39 +0100570/**
571 * xbzrle_cache_zero_page: insert a zero page in the XBZRLE cache
572 *
573 * @current_addr: address for the zero page
574 *
575 * Update the xbzrle cache to reflect a page that's been sent as all 0.
Juan Quintela56e93d22015-05-07 19:33:31 +0200576 * The important thing is that a stale (not-yet-0'd) page be replaced
577 * by the new data.
578 * As a bonus, if the page wasn't in the cache it gets added so that
Juan Quintela3d0684b2017-03-23 15:06:39 +0100579 * when a small write is made into the 0'd page it gets XBZRLE sent.
Juan Quintela56e93d22015-05-07 19:33:31 +0200580 */
Fabiano Rosas8f47d4e2023-10-11 15:46:00 -0300581static void xbzrle_cache_zero_page(ram_addr_t current_addr)
Juan Quintela56e93d22015-05-07 19:33:31 +0200582{
Juan Quintela56e93d22015-05-07 19:33:31 +0200583 /* We don't care if this fails to allocate a new cache page
584 * as long as it updated an old one */
Juan Quintelac00e0922017-05-09 16:22:01 +0200585 cache_insert(XBZRLE.cache, current_addr, XBZRLE.zero_target_page,
Juan Quintelaaff3f662023-04-26 19:37:19 +0200586 stat64_get(&mig_stats.dirty_sync_count));
Juan Quintela56e93d22015-05-07 19:33:31 +0200587}
588
589#define ENCODING_FLAG_XBZRLE 0x1
590
591/**
592 * save_xbzrle_page: compress and send current page
593 *
594 * Returns: 1 means that we wrote the page
595 * 0 means that page is identical to the one already sent
596 * -1 means that xbzrle would be longer than normal
597 *
Juan Quintela5a987732017-03-13 19:39:02 +0100598 * @rs: current RAM state
Peter Xuec6f3ab2022-10-11 17:55:56 -0400599 * @pss: current PSS channel
Juan Quintela3d0684b2017-03-23 15:06:39 +0100600 * @current_data: pointer to the address of the page contents
601 * @current_addr: addr of the page
Juan Quintela56e93d22015-05-07 19:33:31 +0200602 * @block: block that contains the page we want to send
603 * @offset: offset inside the block for the page
Juan Quintela56e93d22015-05-07 19:33:31 +0200604 */
Peter Xuec6f3ab2022-10-11 17:55:56 -0400605static int save_xbzrle_page(RAMState *rs, PageSearchStatus *pss,
Peter Xu61717ea2022-10-11 17:55:53 -0400606 uint8_t **current_data, ram_addr_t current_addr,
607 RAMBlock *block, ram_addr_t offset)
Juan Quintela56e93d22015-05-07 19:33:31 +0200608{
609 int encoded_len = 0, bytes_xbzrle;
610 uint8_t *prev_cached_page;
Peter Xuec6f3ab2022-10-11 17:55:56 -0400611 QEMUFile *file = pss->pss_channel;
Juan Quintelaaff3f662023-04-26 19:37:19 +0200612 uint64_t generation = stat64_get(&mig_stats.dirty_sync_count);
Juan Quintela56e93d22015-05-07 19:33:31 +0200613
Juan Quintela536b5a42023-04-11 18:02:34 +0200614 if (!cache_is_cached(XBZRLE.cache, current_addr, generation)) {
Juan Quintela93604472017-06-06 19:49:03 +0200615 xbzrle_counters.cache_miss++;
Juan Quintela05931ec2021-12-15 19:01:21 +0100616 if (!rs->last_stage) {
Juan Quintela56e93d22015-05-07 19:33:31 +0200617 if (cache_insert(XBZRLE.cache, current_addr, *current_data,
Juan Quintela536b5a42023-04-11 18:02:34 +0200618 generation) == -1) {
Juan Quintela56e93d22015-05-07 19:33:31 +0200619 return -1;
620 } else {
621 /* update *current_data when the page has been
622 inserted into cache */
623 *current_data = get_cached_data(XBZRLE.cache, current_addr);
624 }
625 }
626 return -1;
627 }
628
Wei Wange460a4b2020-04-30 08:59:35 +0800629 /*
630 * Reaching here means the page has hit the xbzrle cache, no matter what
631 * encoding result it is (normal encoding, overflow or skipping the page),
zhaolichang3a4452d2020-09-17 15:50:21 +0800632 * count the page as encoded. This is used to calculate the encoding rate.
Wei Wange460a4b2020-04-30 08:59:35 +0800633 *
634 * Example: 2 pages (8KB) being encoded, first page encoding generates 2KB,
635 * 2nd page turns out to be skipped (i.e. no new bytes written to the
636 * page), the overall encoding rate will be 8KB / 2KB = 4, which has the
637 * skipped page included. In this way, the encoding rate can tell if the
638 * guest page is good for xbzrle encoding.
639 */
640 xbzrle_counters.pages++;
Juan Quintela56e93d22015-05-07 19:33:31 +0200641 prev_cached_page = get_cached_data(XBZRLE.cache, current_addr);
642
643 /* save current buffer into memory */
644 memcpy(XBZRLE.current_buf, *current_data, TARGET_PAGE_SIZE);
645
646 /* XBZRLE encoding (if there is no overflow) */
Richard Henderson7ba7db92023-05-17 20:00:30 -0700647 encoded_len = xbzrle_encode_buffer(prev_cached_page, XBZRLE.current_buf,
648 TARGET_PAGE_SIZE, XBZRLE.encoded_buf,
649 TARGET_PAGE_SIZE);
Wei Yangca353802019-06-10 08:41:59 +0800650
651 /*
652 * Update the cache contents, so that it corresponds to the data
653 * sent, in all cases except where we skip the page.
654 */
Juan Quintela05931ec2021-12-15 19:01:21 +0100655 if (!rs->last_stage && encoded_len != 0) {
Wei Yangca353802019-06-10 08:41:59 +0800656 memcpy(prev_cached_page, XBZRLE.current_buf, TARGET_PAGE_SIZE);
657 /*
658 * In the case where we couldn't compress, ensure that the caller
659 * sends the data from the cache, since the guest might have
660 * changed the RAM since we copied it.
661 */
662 *current_data = prev_cached_page;
663 }
664
Juan Quintela56e93d22015-05-07 19:33:31 +0200665 if (encoded_len == 0) {
Juan Quintela55c44462017-01-23 22:32:05 +0100666 trace_save_xbzrle_page_skipping();
Juan Quintela56e93d22015-05-07 19:33:31 +0200667 return 0;
668 } else if (encoded_len == -1) {
Juan Quintela55c44462017-01-23 22:32:05 +0100669 trace_save_xbzrle_page_overflow();
Juan Quintela93604472017-06-06 19:49:03 +0200670 xbzrle_counters.overflow++;
Wei Wange460a4b2020-04-30 08:59:35 +0800671 xbzrle_counters.bytes += TARGET_PAGE_SIZE;
Juan Quintela56e93d22015-05-07 19:33:31 +0200672 return -1;
673 }
674
Juan Quintela56e93d22015-05-07 19:33:31 +0200675 /* Send XBZRLE based compressed page */
Lukas Straub37502df2023-04-02 17:06:32 +0000676 bytes_xbzrle = save_page_header(pss, pss->pss_channel, block,
Juan Quintela204b88b2017-03-15 09:16:57 +0100677 offset | RAM_SAVE_FLAG_XBZRLE);
Peter Xu61717ea2022-10-11 17:55:53 -0400678 qemu_put_byte(file, ENCODING_FLAG_XBZRLE);
679 qemu_put_be16(file, encoded_len);
680 qemu_put_buffer(file, XBZRLE.encoded_buf, encoded_len);
Juan Quintela56e93d22015-05-07 19:33:31 +0200681 bytes_xbzrle += encoded_len + 1 + 2;
Wei Wange460a4b2020-04-30 08:59:35 +0800682 /*
683 * Like compressed_size (please see update_compress_thread_counts),
684 * the xbzrle encoded bytes don't count the 8 byte header with
685 * RAM_SAVE_FLAG_CONTINUE.
686 */
687 xbzrle_counters.bytes += bytes_xbzrle - 8;
David Edmondson4c2d0f62021-12-21 09:34:40 +0000688 ram_transferred_add(bytes_xbzrle);
Juan Quintela56e93d22015-05-07 19:33:31 +0200689
690 return 1;
691}
692
Juan Quintela3d0684b2017-03-23 15:06:39 +0100693/**
Peter Xud9e474e2022-10-11 17:55:52 -0400694 * pss_find_next_dirty: find the next dirty page of current ramblock
Dr. David Alan Gilbertf3f491f2015-11-05 18:11:01 +0000695 *
Peter Xud9e474e2022-10-11 17:55:52 -0400696 * This function updates pss->page to point to the next dirty page index
697 * within the ramblock to migrate, or the end of ramblock when nothing
698 * found. Note that when pss->host_page_sending==true it means we're
699 * during sending a host page, so we won't look for dirty page that is
700 * outside the host page boundary.
Juan Quintela3d0684b2017-03-23 15:06:39 +0100701 *
Peter Xud9e474e2022-10-11 17:55:52 -0400702 * @pss: the current page search status
Dr. David Alan Gilbertf3f491f2015-11-05 18:11:01 +0000703 */
Peter Xud9e474e2022-10-11 17:55:52 -0400704static void pss_find_next_dirty(PageSearchStatus *pss)
Juan Quintela56e93d22015-05-07 19:33:31 +0200705{
Peter Xud9e474e2022-10-11 17:55:52 -0400706 RAMBlock *rb = pss->block;
Juan Quintela6b6712e2017-03-22 15:18:04 +0100707 unsigned long size = rb->used_length >> TARGET_PAGE_BITS;
708 unsigned long *bitmap = rb->bmap;
Juan Quintela56e93d22015-05-07 19:33:31 +0200709
David Hildenbrandf161c882023-07-06 09:56:08 +0200710 if (migrate_ram_is_ignored(rb)) {
Peter Xud9e474e2022-10-11 17:55:52 -0400711 /* Points directly to the end, so we know no dirty page */
712 pss->page = size;
713 return;
Cédric Le Goaterb895de52018-05-14 08:57:00 +0200714 }
715
Peter Xud9e474e2022-10-11 17:55:52 -0400716 /*
717 * If during sending a host page, only look for dirty pages within the
718 * current host page being send.
719 */
720 if (pss->host_page_sending) {
721 assert(pss->host_page_end);
722 size = MIN(size, pss->host_page_end);
723 }
724
725 pss->page = find_next_bit(bitmap, size, pss->page);
Juan Quintela56e93d22015-05-07 19:33:31 +0200726}
727
David Hildenbrand1230a252021-09-04 18:09:07 +0200728static void migration_clear_memory_region_dirty_bitmap(RAMBlock *rb,
Wei Wang3143577d2021-07-22 04:30:55 -0400729 unsigned long page)
730{
731 uint8_t shift;
732 hwaddr size, start;
733
734 if (!rb->clear_bmap || !clear_bmap_test_and_clear(rb, page)) {
735 return;
736 }
737
738 shift = rb->clear_bmap_shift;
739 /*
740 * CLEAR_BITMAP_SHIFT_MIN should always guarantee this... this
741 * can make things easier sometimes since then start address
742 * of the small chunk will always be 64 pages aligned so the
743 * bitmap will always be aligned to unsigned long. We should
744 * even be able to remove this restriction but I'm simply
745 * keeping it.
746 */
747 assert(shift >= 6);
748
749 size = 1ULL << (TARGET_PAGE_BITS + shift);
David Hildenbrand76482972021-10-11 19:53:44 +0200750 start = QEMU_ALIGN_DOWN((ram_addr_t)page << TARGET_PAGE_BITS, size);
Wei Wang3143577d2021-07-22 04:30:55 -0400751 trace_migration_bitmap_clear_dirty(rb->idstr, start, size, page);
752 memory_region_clear_dirty_bitmap(rb->mr, start, size);
753}
754
755static void
David Hildenbrand1230a252021-09-04 18:09:07 +0200756migration_clear_memory_region_dirty_bitmap_range(RAMBlock *rb,
Wei Wang3143577d2021-07-22 04:30:55 -0400757 unsigned long start,
758 unsigned long npages)
759{
760 unsigned long i, chunk_pages = 1UL << rb->clear_bmap_shift;
761 unsigned long chunk_start = QEMU_ALIGN_DOWN(start, chunk_pages);
762 unsigned long chunk_end = QEMU_ALIGN_UP(start + npages, chunk_pages);
763
764 /*
765 * Clear pages from start to start + npages - 1, so the end boundary is
766 * exclusive.
767 */
768 for (i = chunk_start; i < chunk_end; i += chunk_pages) {
David Hildenbrand1230a252021-09-04 18:09:07 +0200769 migration_clear_memory_region_dirty_bitmap(rb, i);
Wei Wang3143577d2021-07-22 04:30:55 -0400770 }
771}
772
Rao, Leia6a83ce2021-11-09 11:04:55 +0800773/*
774 * colo_bitmap_find_diry:find contiguous dirty pages from start
775 *
776 * Returns the page offset within memory region of the start of the contiguout
777 * dirty page
778 *
779 * @rs: current RAM state
780 * @rb: RAMBlock where to search for dirty pages
781 * @start: page where we start the search
782 * @num: the number of contiguous dirty pages
783 */
784static inline
785unsigned long colo_bitmap_find_dirty(RAMState *rs, RAMBlock *rb,
786 unsigned long start, unsigned long *num)
787{
788 unsigned long size = rb->used_length >> TARGET_PAGE_BITS;
789 unsigned long *bitmap = rb->bmap;
790 unsigned long first, next;
791
792 *num = 0;
793
David Hildenbrandf161c882023-07-06 09:56:08 +0200794 if (migrate_ram_is_ignored(rb)) {
Rao, Leia6a83ce2021-11-09 11:04:55 +0800795 return size;
796 }
797
798 first = find_next_bit(bitmap, size, start);
799 if (first >= size) {
800 return first;
801 }
802 next = find_next_zero_bit(bitmap, size, first + 1);
803 assert(next >= first);
804 *num = next - first;
805 return first;
806}
807
Juan Quintela06b10682017-03-21 15:18:05 +0100808static inline bool migration_bitmap_clear_dirty(RAMState *rs,
Juan Quintelaf20e2862017-03-21 16:19:05 +0100809 RAMBlock *rb,
810 unsigned long page)
Dr. David Alan Gilberta82d5932015-11-05 18:11:09 +0000811{
812 bool ret;
Dr. David Alan Gilberta82d5932015-11-05 18:11:09 +0000813
Peter Xu002cad62019-06-03 14:50:56 +0800814 /*
815 * Clear dirty bitmap if needed. This _must_ be called before we
816 * send any of the page in the chunk because we need to make sure
817 * we can capture further page content changes when we sync dirty
818 * log the next time. So as long as we are going to send any of
819 * the page in the chunk we clear the remote dirty bitmap for all.
820 * Clearing it earlier won't be a problem, but too late will.
821 */
David Hildenbrand1230a252021-09-04 18:09:07 +0200822 migration_clear_memory_region_dirty_bitmap(rb, page);
Peter Xu002cad62019-06-03 14:50:56 +0800823
Juan Quintela6b6712e2017-03-22 15:18:04 +0100824 ret = test_and_clear_bit(page, rb->bmap);
Dr. David Alan Gilberta82d5932015-11-05 18:11:09 +0000825 if (ret) {
Juan Quintela0d8ec882017-03-13 21:21:41 +0100826 rs->migration_dirty_pages--;
Dr. David Alan Gilberta82d5932015-11-05 18:11:09 +0000827 }
Wei Wang386a9072018-12-11 16:24:49 +0800828
Dr. David Alan Gilberta82d5932015-11-05 18:11:09 +0000829 return ret;
830}
831
David Hildenbrandbe39b4c2021-10-11 19:53:41 +0200832static void dirty_bitmap_clear_section(MemoryRegionSection *section,
833 void *opaque)
834{
835 const hwaddr offset = section->offset_within_region;
836 const hwaddr size = int128_get64(section->size);
837 const unsigned long start = offset >> TARGET_PAGE_BITS;
838 const unsigned long npages = size >> TARGET_PAGE_BITS;
839 RAMBlock *rb = section->mr->ram_block;
840 uint64_t *cleared_bits = opaque;
841
842 /*
843 * We don't grab ram_state->bitmap_mutex because we expect to run
844 * only when starting migration or during postcopy recovery where
845 * we don't have concurrent access.
846 */
847 if (!migration_in_postcopy() && !migrate_background_snapshot()) {
848 migration_clear_memory_region_dirty_bitmap_range(rb, start, npages);
849 }
850 *cleared_bits += bitmap_count_one_with_offset(rb->bmap, start, npages);
851 bitmap_clear(rb->bmap, start, npages);
852}
853
854/*
855 * Exclude all dirty pages from migration that fall into a discarded range as
856 * managed by a RamDiscardManager responsible for the mapped memory region of
857 * the RAMBlock. Clear the corresponding bits in the dirty bitmaps.
858 *
859 * Discarded pages ("logically unplugged") have undefined content and must
860 * not get migrated, because even reading these pages for migration might
861 * result in undesired behavior.
862 *
863 * Returns the number of cleared bits in the RAMBlock dirty bitmap.
864 *
865 * Note: The result is only stable while migrating (precopy/postcopy).
866 */
867static uint64_t ramblock_dirty_bitmap_clear_discarded_pages(RAMBlock *rb)
868{
869 uint64_t cleared_bits = 0;
870
871 if (rb->mr && rb->bmap && memory_region_has_ram_discard_manager(rb->mr)) {
872 RamDiscardManager *rdm = memory_region_get_ram_discard_manager(rb->mr);
873 MemoryRegionSection section = {
874 .mr = rb->mr,
875 .offset_within_region = 0,
876 .size = int128_make64(qemu_ram_get_used_length(rb)),
877 };
878
879 ram_discard_manager_replay_discarded(rdm, &section,
880 dirty_bitmap_clear_section,
881 &cleared_bits);
882 }
883 return cleared_bits;
884}
885
David Hildenbrand9470c5e2021-10-11 19:53:43 +0200886/*
887 * Check if a host-page aligned page falls into a discarded range as managed by
888 * a RamDiscardManager responsible for the mapped memory region of the RAMBlock.
889 *
890 * Note: The result is only stable while migrating (precopy/postcopy).
891 */
892bool ramblock_page_is_discarded(RAMBlock *rb, ram_addr_t start)
893{
894 if (rb->mr && memory_region_has_ram_discard_manager(rb->mr)) {
895 RamDiscardManager *rdm = memory_region_get_ram_discard_manager(rb->mr);
896 MemoryRegionSection section = {
897 .mr = rb->mr,
898 .offset_within_region = start,
899 .size = int128_make64(qemu_ram_pagesize(rb)),
900 };
901
902 return !ram_discard_manager_is_populated(rdm, &section);
903 }
904 return false;
905}
906
Peter Xu267691b2019-06-03 14:50:46 +0800907/* Called with RCU critical section */
Wei Yang7a3e9572019-08-08 11:31:55 +0800908static void ramblock_sync_dirty_bitmap(RAMState *rs, RAMBlock *rb)
Juan Quintela56e93d22015-05-07 19:33:31 +0200909{
Keqian Zhufb613582020-06-22 11:20:37 +0800910 uint64_t new_dirty_pages =
911 cpu_physical_memory_sync_dirty_bitmap(rb, 0, rb->used_length);
912
913 rs->migration_dirty_pages += new_dirty_pages;
914 rs->num_dirty_pages_period += new_dirty_pages;
Juan Quintela56e93d22015-05-07 19:33:31 +0200915}
916
Juan Quintela3d0684b2017-03-23 15:06:39 +0100917/**
918 * ram_pagesize_summary: calculate all the pagesizes of a VM
919 *
920 * Returns a summary bitmap of the page sizes of all RAMBlocks
921 *
922 * For VMs with just normal pages this is equivalent to the host page
923 * size. If it's got some huge pages then it's the OR of all the
924 * different page sizes.
Dr. David Alan Gilberte8ca1db2017-02-24 18:28:29 +0000925 */
926uint64_t ram_pagesize_summary(void)
927{
928 RAMBlock *block;
929 uint64_t summary = 0;
930
Yury Kotovfbd162e2019-02-15 20:45:46 +0300931 RAMBLOCK_FOREACH_NOT_IGNORED(block) {
Dr. David Alan Gilberte8ca1db2017-02-24 18:28:29 +0000932 summary |= block->page_size;
933 }
934
935 return summary;
936}
937
Xiao Guangrongaecbfe92019-01-11 14:37:30 +0800938uint64_t ram_get_total_transferred_pages(void)
939{
Juan Quintelaaff3f662023-04-26 19:37:19 +0200940 return stat64_get(&mig_stats.normal_pages) +
941 stat64_get(&mig_stats.zero_pages) +
Juan Quintelaf5047892023-06-13 16:57:46 +0200942 ram_compressed_pages() + xbzrle_counters.pages;
Xiao Guangrongaecbfe92019-01-11 14:37:30 +0800943}
944
Xiao Guangrongb7340352018-06-04 17:55:12 +0800945static void migration_update_rates(RAMState *rs, int64_t end_time)
946{
Xiao Guangrongbe8b02e2018-09-03 17:26:42 +0800947 uint64_t page_count = rs->target_page_count - rs->target_page_count_prev;
Xiao Guangrong76e03002018-09-06 15:01:00 +0800948 double compressed_size;
Xiao Guangrongb7340352018-06-04 17:55:12 +0800949
950 /* calculate period counters */
Juan Quintelaaff3f662023-04-26 19:37:19 +0200951 stat64_set(&mig_stats.dirty_pages_rate,
Juan Quintela72f8e582023-04-11 18:19:05 +0200952 rs->num_dirty_pages_period * 1000 /
953 (end_time - rs->time_last_bitmap_sync));
Xiao Guangrongb7340352018-06-04 17:55:12 +0800954
Xiao Guangrongbe8b02e2018-09-03 17:26:42 +0800955 if (!page_count) {
Xiao Guangrongb7340352018-06-04 17:55:12 +0800956 return;
957 }
958
Juan Quintela87dca0c2023-03-01 22:20:13 +0100959 if (migrate_xbzrle()) {
Wei Wange460a4b2020-04-30 08:59:35 +0800960 double encoded_size, unencoded_size;
961
Xiao Guangrongb7340352018-06-04 17:55:12 +0800962 xbzrle_counters.cache_miss_rate = (double)(xbzrle_counters.cache_miss -
Xiao Guangrongbe8b02e2018-09-03 17:26:42 +0800963 rs->xbzrle_cache_miss_prev) / page_count;
Xiao Guangrongb7340352018-06-04 17:55:12 +0800964 rs->xbzrle_cache_miss_prev = xbzrle_counters.cache_miss;
Wei Wange460a4b2020-04-30 08:59:35 +0800965 unencoded_size = (xbzrle_counters.pages - rs->xbzrle_pages_prev) *
966 TARGET_PAGE_SIZE;
967 encoded_size = xbzrle_counters.bytes - rs->xbzrle_bytes_prev;
Wei Wang92271402020-06-17 13:13:05 -0700968 if (xbzrle_counters.pages == rs->xbzrle_pages_prev || !encoded_size) {
Wei Wange460a4b2020-04-30 08:59:35 +0800969 xbzrle_counters.encoding_rate = 0;
Wei Wange460a4b2020-04-30 08:59:35 +0800970 } else {
971 xbzrle_counters.encoding_rate = unencoded_size / encoded_size;
972 }
973 rs->xbzrle_pages_prev = xbzrle_counters.pages;
974 rs->xbzrle_bytes_prev = xbzrle_counters.bytes;
Xiao Guangrongb7340352018-06-04 17:55:12 +0800975 }
Xiao Guangrong76e03002018-09-06 15:01:00 +0800976
Juan Quintelaa7a94d12023-03-01 22:03:48 +0100977 if (migrate_compress()) {
Xiao Guangrong76e03002018-09-06 15:01:00 +0800978 compression_counters.busy_rate = (double)(compression_counters.busy -
979 rs->compress_thread_busy_prev) / page_count;
980 rs->compress_thread_busy_prev = compression_counters.busy;
981
982 compressed_size = compression_counters.compressed_size -
983 rs->compressed_size_prev;
984 if (compressed_size) {
985 double uncompressed_size = (compression_counters.pages -
986 rs->compress_pages_prev) * TARGET_PAGE_SIZE;
987
988 /* Compression-Ratio = Uncompressed-size / Compressed-size */
989 compression_counters.compression_rate =
990 uncompressed_size / compressed_size;
991
992 rs->compress_pages_prev = compression_counters.pages;
993 rs->compressed_size_prev = compression_counters.compressed_size;
994 }
995 }
Xiao Guangrongb7340352018-06-04 17:55:12 +0800996}
997
Hyman Huang(黄勇)acac51b2023-06-08 00:12:40 +0800998/*
999 * Enable dirty-limit to throttle down the guest
1000 */
1001static void migration_dirty_limit_guest(void)
1002{
1003 /*
1004 * dirty page rate quota for all vCPUs fetched from
1005 * migration parameter 'vcpu_dirty_limit'
1006 */
1007 static int64_t quota_dirtyrate;
1008 MigrationState *s = migrate_get_current();
1009
1010 /*
1011 * If dirty limit already enabled and migration parameter
1012 * vcpu-dirty-limit untouched.
1013 */
1014 if (dirtylimit_in_service() &&
1015 quota_dirtyrate == s->parameters.vcpu_dirty_limit) {
1016 return;
1017 }
1018
1019 quota_dirtyrate = s->parameters.vcpu_dirty_limit;
1020
1021 /*
1022 * Set all vCPU a quota dirtyrate, note that the second
1023 * parameter will be ignored if setting all vCPU for the vm
1024 */
1025 qmp_set_vcpu_dirty_limit(false, -1, quota_dirtyrate, NULL);
1026 trace_migration_dirty_limit_guest(quota_dirtyrate);
1027}
1028
Keqian Zhudc14a472020-02-24 10:31:42 +08001029static void migration_trigger_throttle(RAMState *rs)
1030{
Juan Quintela6499efd2023-03-02 00:59:13 +01001031 uint64_t threshold = migrate_throttle_trigger_threshold();
Peter Xu23b75762022-10-11 17:55:51 -04001032 uint64_t bytes_xfer_period =
Juan Quintelaaff3f662023-04-26 19:37:19 +02001033 stat64_get(&mig_stats.transferred) - rs->bytes_xfer_prev;
Keqian Zhudc14a472020-02-24 10:31:42 +08001034 uint64_t bytes_dirty_period = rs->num_dirty_pages_period * TARGET_PAGE_SIZE;
1035 uint64_t bytes_dirty_threshold = bytes_xfer_period * threshold / 100;
1036
1037 /* During block migration the auto-converge logic incorrectly detects
1038 * that ram migration makes no progress. Avoid this by disabling the
1039 * throttling logic during the bulk phase of block migration. */
Hyman Huang(黄勇)bb9993c2023-06-07 23:32:51 +08001040 if (blk_mig_bulk_active()) {
1041 return;
1042 }
1043
Hyman Huang(黄勇)310ad562023-06-15 21:29:44 +08001044 /*
1045 * The following detection logic can be refined later. For now:
1046 * Check to see if the ratio between dirtied bytes and the approx.
1047 * amount of bytes that just got transferred since the last time
1048 * we were in this routine reaches the threshold. If that happens
1049 * twice, start or increase throttling.
1050 */
1051 if ((bytes_dirty_period > bytes_dirty_threshold) &&
1052 (++rs->dirty_rate_high_cnt >= 2)) {
1053 rs->dirty_rate_high_cnt = 0;
1054 if (migrate_auto_converge()) {
Keqian Zhudc14a472020-02-24 10:31:42 +08001055 trace_migration_throttle();
Keqian Zhucbbf8182020-04-13 18:15:08 +08001056 mig_throttle_guest_down(bytes_dirty_period,
1057 bytes_dirty_threshold);
Hyman Huang(黄勇)acac51b2023-06-08 00:12:40 +08001058 } else if (migrate_dirty_limit()) {
1059 migration_dirty_limit_guest();
Keqian Zhudc14a472020-02-24 10:31:42 +08001060 }
1061 }
1062}
1063
Gavin Shan1e493be2023-05-09 12:21:19 +10001064static void migration_bitmap_sync(RAMState *rs, bool last_stage)
Juan Quintela56e93d22015-05-07 19:33:31 +02001065{
1066 RAMBlock *block;
Juan Quintela56e93d22015-05-07 19:33:31 +02001067 int64_t end_time;
Juan Quintela56e93d22015-05-07 19:33:31 +02001068
Juan Quintelaaff3f662023-04-26 19:37:19 +02001069 stat64_add(&mig_stats.dirty_sync_count, 1);
Juan Quintela56e93d22015-05-07 19:33:31 +02001070
Juan Quintelaf664da82017-03-13 19:44:57 +01001071 if (!rs->time_last_bitmap_sync) {
1072 rs->time_last_bitmap_sync = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
Juan Quintela56e93d22015-05-07 19:33:31 +02001073 }
1074
1075 trace_migration_bitmap_sync_start();
Gavin Shan1e493be2023-05-09 12:21:19 +10001076 memory_global_dirty_log_sync(last_stage);
Juan Quintela56e93d22015-05-07 19:33:31 +02001077
Juan Quintela108cfae2017-03-13 21:38:09 +01001078 qemu_mutex_lock(&rs->bitmap_mutex);
Dr. David Alan Gilbert89ac5a12019-10-07 15:36:39 +01001079 WITH_RCU_READ_LOCK_GUARD() {
1080 RAMBLOCK_FOREACH_NOT_IGNORED(block) {
1081 ramblock_sync_dirty_bitmap(rs, block);
1082 }
Juan Quintelaaff3f662023-04-26 19:37:19 +02001083 stat64_set(&mig_stats.dirty_bytes_last_sync, ram_bytes_remaining());
Juan Quintela56e93d22015-05-07 19:33:31 +02001084 }
Juan Quintela108cfae2017-03-13 21:38:09 +01001085 qemu_mutex_unlock(&rs->bitmap_mutex);
Juan Quintela56e93d22015-05-07 19:33:31 +02001086
Paolo Bonzini9458a9a2018-02-06 18:37:39 +01001087 memory_global_after_dirty_log_sync();
Juan Quintelaa66cd902017-03-28 15:02:43 +02001088 trace_migration_bitmap_sync_end(rs->num_dirty_pages_period);
Chao Fan1ffb5df2017-03-14 09:55:07 +08001089
Juan Quintela56e93d22015-05-07 19:33:31 +02001090 end_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
1091
1092 /* more than 1 second = 1000 millisecons */
Juan Quintelaf664da82017-03-13 19:44:57 +01001093 if (end_time > rs->time_last_bitmap_sync + 1000) {
Keqian Zhudc14a472020-02-24 10:31:42 +08001094 migration_trigger_throttle(rs);
Jason J. Herne070afca2015-09-08 13:12:35 -04001095
Xiao Guangrongb7340352018-06-04 17:55:12 +08001096 migration_update_rates(rs, end_time);
1097
Xiao Guangrongbe8b02e2018-09-03 17:26:42 +08001098 rs->target_page_count_prev = rs->target_page_count;
Felipe Franciosid693c6f2017-05-24 17:10:01 +01001099
1100 /* reset period counters */
Juan Quintelaf664da82017-03-13 19:44:57 +01001101 rs->time_last_bitmap_sync = end_time;
Juan Quintelaa66cd902017-03-28 15:02:43 +02001102 rs->num_dirty_pages_period = 0;
Juan Quintelaaff3f662023-04-26 19:37:19 +02001103 rs->bytes_xfer_prev = stat64_get(&mig_stats.transferred);
Juan Quintela56e93d22015-05-07 19:33:31 +02001104 }
Juan Quintelab8909022023-03-01 22:08:09 +01001105 if (migrate_events()) {
Juan Quintelaaff3f662023-04-26 19:37:19 +02001106 uint64_t generation = stat64_get(&mig_stats.dirty_sync_count);
Juan Quintela536b5a42023-04-11 18:02:34 +02001107 qapi_event_send_migration_pass(generation);
Dr. David Alan Gilbert4addcd42015-12-16 11:47:36 +00001108 }
Juan Quintela56e93d22015-05-07 19:33:31 +02001109}
1110
Gavin Shan1e493be2023-05-09 12:21:19 +10001111static void migration_bitmap_sync_precopy(RAMState *rs, bool last_stage)
Wei Wangbd227062018-12-11 16:24:51 +08001112{
1113 Error *local_err = NULL;
1114
1115 /*
1116 * The current notifier usage is just an optimization to migration, so we
1117 * don't stop the normal migration process in the error case.
1118 */
1119 if (precopy_notify(PRECOPY_NOTIFY_BEFORE_BITMAP_SYNC, &local_err)) {
1120 error_report_err(local_err);
Vladimir Sementsov-Ogievskiyb4a17332020-03-24 18:36:29 +03001121 local_err = NULL;
Wei Wangbd227062018-12-11 16:24:51 +08001122 }
1123
Gavin Shan1e493be2023-05-09 12:21:19 +10001124 migration_bitmap_sync(rs, last_stage);
Wei Wangbd227062018-12-11 16:24:51 +08001125
1126 if (precopy_notify(PRECOPY_NOTIFY_AFTER_BITMAP_SYNC, &local_err)) {
1127 error_report_err(local_err);
1128 }
1129}
1130
Juan Quintelaa4dbaf82021-12-16 10:19:38 +01001131void ram_release_page(const char *rbname, uint64_t offset)
Juan Quintela47fe16f2021-12-16 09:58:49 +01001132{
1133 if (!migrate_release_ram() || !migration_in_postcopy()) {
1134 return;
1135 }
1136
1137 ram_discard_range(rbname, offset, TARGET_PAGE_SIZE);
1138}
1139
Juan Quintela56e93d22015-05-07 19:33:31 +02001140/**
Juan Quintela3d0684b2017-03-23 15:06:39 +01001141 * save_zero_page: send the zero page to the stream
Juan Quintela56e93d22015-05-07 19:33:31 +02001142 *
Juan Quintela3d0684b2017-03-23 15:06:39 +01001143 * Returns the number of pages written.
Juan Quintela56e93d22015-05-07 19:33:31 +02001144 *
Fabiano Rosasccc09db2023-10-11 15:46:02 -03001145 * @rs: current RAM state
Peter Xuec6f3ab2022-10-11 17:55:56 -04001146 * @pss: current PSS channel
Juan Quintela56e93d22015-05-07 19:33:31 +02001147 * @offset: offset inside the block for the page
Juan Quintela56e93d22015-05-07 19:33:31 +02001148 */
Juan Quintelae8e4e7a2023-06-13 13:38:30 +02001149static int save_zero_page(RAMState *rs, PageSearchStatus *pss,
Peter Xu61717ea2022-10-11 17:55:53 -04001150 ram_addr_t offset)
Juan Quintela56e93d22015-05-07 19:33:31 +02001151{
Juan Quintelae8e4e7a2023-06-13 13:38:30 +02001152 uint8_t *p = pss->block->host + offset;
Fabiano Rosas8697eb82023-10-11 15:46:03 -03001153 QEMUFile *file = pss->pss_channel;
1154 int len = 0;
Juan Quintela56e93d22015-05-07 19:33:31 +02001155
Fabiano Rosas8697eb82023-10-11 15:46:03 -03001156 if (!buffer_is_zero(p, TARGET_PAGE_SIZE)) {
1157 return 0;
Juan Quintela56e93d22015-05-07 19:33:31 +02001158 }
Fabiano Rosasccc09db2023-10-11 15:46:02 -03001159
Juan Quintelae8e4e7a2023-06-13 13:38:30 +02001160 len += save_page_header(pss, file, pss->block, offset | RAM_SAVE_FLAG_ZERO);
Fabiano Rosas8697eb82023-10-11 15:46:03 -03001161 qemu_put_byte(file, 0);
1162 len += 1;
Juan Quintelae8e4e7a2023-06-13 13:38:30 +02001163 ram_release_page(pss->block->idstr, offset);
Fabiano Rosas8697eb82023-10-11 15:46:03 -03001164
Fabiano Rosasccc09db2023-10-11 15:46:02 -03001165 stat64_add(&mig_stats.zero_pages, 1);
1166 ram_transferred_add(len);
1167
1168 /*
1169 * Must let xbzrle know, otherwise a previous (now 0'd) cached
1170 * page would be stale.
1171 */
1172 if (rs->xbzrle_started) {
1173 XBZRLE_cache_lock();
Juan Quintelae8e4e7a2023-06-13 13:38:30 +02001174 xbzrle_cache_zero_page(pss->block->offset + offset);
Fabiano Rosasccc09db2023-10-11 15:46:02 -03001175 XBZRLE_cache_unlock();
1176 }
1177
Fabiano Rosas8697eb82023-10-11 15:46:03 -03001178 return len;
Juan Quintela56e93d22015-05-07 19:33:31 +02001179}
1180
Xiao Guangrong059ff0f2018-03-30 15:51:23 +08001181/*
1182 * @pages: the number of pages written by the control path,
1183 * < 0 - error
1184 * > 0 - number of pages written
1185 *
1186 * Return true if the pages has been saved, otherwise false is returned.
1187 */
Juan Quintela944853c2023-06-13 13:36:24 +02001188static bool control_save_page(PageSearchStatus *pss,
Peter Xu61717ea2022-10-11 17:55:53 -04001189 ram_addr_t offset, int *pages)
Xiao Guangrong059ff0f2018-03-30 15:51:23 +08001190{
Xiao Guangrong059ff0f2018-03-30 15:51:23 +08001191 int ret;
1192
Juan Quintela944853c2023-06-13 13:36:24 +02001193 ret = rdma_control_save_page(pss->pss_channel, pss->block->offset, offset,
Juan Quintelae4930082023-10-11 22:35:20 +02001194 TARGET_PAGE_SIZE);
Xiao Guangrong059ff0f2018-03-30 15:51:23 +08001195 if (ret == RAM_SAVE_CONTROL_NOT_SUPP) {
1196 return false;
1197 }
1198
Xiao Guangrong059ff0f2018-03-30 15:51:23 +08001199 if (ret == RAM_SAVE_CONTROL_DELAYED) {
Juan Quintela9c53d362023-05-15 21:57:08 +02001200 *pages = 1;
Xiao Guangrong059ff0f2018-03-30 15:51:23 +08001201 return true;
1202 }
Juan Quintela9c53d362023-05-15 21:57:08 +02001203 *pages = ret;
Xiao Guangrong059ff0f2018-03-30 15:51:23 +08001204 return true;
1205}
1206
Xiao Guangrong65dacaa2018-03-30 15:51:27 +08001207/*
1208 * directly send the page to the stream
1209 *
1210 * Returns the number of pages written.
1211 *
Peter Xuec6f3ab2022-10-11 17:55:56 -04001212 * @pss: current PSS channel
Xiao Guangrong65dacaa2018-03-30 15:51:27 +08001213 * @block: block that contains the page we want to send
1214 * @offset: offset inside the block for the page
1215 * @buf: the page to be sent
1216 * @async: send to page asyncly
1217 */
Peter Xuec6f3ab2022-10-11 17:55:56 -04001218static int save_normal_page(PageSearchStatus *pss, RAMBlock *block,
Peter Xu61717ea2022-10-11 17:55:53 -04001219 ram_addr_t offset, uint8_t *buf, bool async)
Xiao Guangrong65dacaa2018-03-30 15:51:27 +08001220{
Peter Xuec6f3ab2022-10-11 17:55:56 -04001221 QEMUFile *file = pss->pss_channel;
1222
Lukas Straub37502df2023-04-02 17:06:32 +00001223 ram_transferred_add(save_page_header(pss, pss->pss_channel, block,
David Edmondson4c2d0f62021-12-21 09:34:40 +00001224 offset | RAM_SAVE_FLAG_PAGE));
Xiao Guangrong65dacaa2018-03-30 15:51:27 +08001225 if (async) {
Peter Xu61717ea2022-10-11 17:55:53 -04001226 qemu_put_buffer_async(file, buf, TARGET_PAGE_SIZE,
Dr. David Alan Gilbertf912ec52022-04-06 11:25:15 +01001227 migrate_release_ram() &&
Xiao Guangrong65dacaa2018-03-30 15:51:27 +08001228 migration_in_postcopy());
1229 } else {
Peter Xu61717ea2022-10-11 17:55:53 -04001230 qemu_put_buffer(file, buf, TARGET_PAGE_SIZE);
Xiao Guangrong65dacaa2018-03-30 15:51:27 +08001231 }
David Edmondson4c2d0f62021-12-21 09:34:40 +00001232 ram_transferred_add(TARGET_PAGE_SIZE);
Juan Quintelaaff3f662023-04-26 19:37:19 +02001233 stat64_add(&mig_stats.normal_pages, 1);
Xiao Guangrong65dacaa2018-03-30 15:51:27 +08001234 return 1;
1235}
1236
Juan Quintela56e93d22015-05-07 19:33:31 +02001237/**
Juan Quintela3d0684b2017-03-23 15:06:39 +01001238 * ram_save_page: send the given page to the stream
Juan Quintela56e93d22015-05-07 19:33:31 +02001239 *
Juan Quintela3d0684b2017-03-23 15:06:39 +01001240 * Returns the number of pages written.
Dr. David Alan Gilbert3fd3c4b2015-12-10 16:31:46 +00001241 * < 0 - error
1242 * >=0 - Number of pages written - this might legally be 0
1243 * if xbzrle noticed the page was the same.
Juan Quintela56e93d22015-05-07 19:33:31 +02001244 *
Juan Quintela6f37bb82017-03-13 19:26:29 +01001245 * @rs: current RAM state
Juan Quintela56e93d22015-05-07 19:33:31 +02001246 * @block: block that contains the page we want to send
1247 * @offset: offset inside the block for the page
Juan Quintela56e93d22015-05-07 19:33:31 +02001248 */
Juan Quintela05931ec2021-12-15 19:01:21 +01001249static int ram_save_page(RAMState *rs, PageSearchStatus *pss)
Juan Quintela56e93d22015-05-07 19:33:31 +02001250{
1251 int pages = -1;
Juan Quintela56e93d22015-05-07 19:33:31 +02001252 uint8_t *p;
Juan Quintela56e93d22015-05-07 19:33:31 +02001253 bool send_async = true;
zhanghailianga08f6892016-01-15 11:37:44 +08001254 RAMBlock *block = pss->block;
Alexey Romko8bba0042020-01-10 14:51:34 +01001255 ram_addr_t offset = ((ram_addr_t)pss->page) << TARGET_PAGE_BITS;
Xiao Guangrong059ff0f2018-03-30 15:51:23 +08001256 ram_addr_t current_addr = block->offset + offset;
Juan Quintela56e93d22015-05-07 19:33:31 +02001257
Dr. David Alan Gilbert2f68e392015-08-13 11:51:30 +01001258 p = block->host + offset;
Dr. David Alan Gilbert1db9d8e2017-04-26 19:37:21 +01001259 trace_ram_save_page(block->idstr, (uint64_t)offset, p);
Juan Quintela56e93d22015-05-07 19:33:31 +02001260
Juan Quintela56e93d22015-05-07 19:33:31 +02001261 XBZRLE_cache_lock();
Juan Quintelaf3095cc2023-05-04 13:53:23 +02001262 if (rs->xbzrle_started && !migration_in_postcopy()) {
Peter Xuec6f3ab2022-10-11 17:55:56 -04001263 pages = save_xbzrle_page(rs, pss, &p, current_addr,
Peter Xu61717ea2022-10-11 17:55:53 -04001264 block, offset);
Juan Quintela05931ec2021-12-15 19:01:21 +01001265 if (!rs->last_stage) {
Xiao Guangrong059ff0f2018-03-30 15:51:23 +08001266 /* Can't send this cached data async, since the cache page
1267 * might get updated before it gets to the wire
Juan Quintela56e93d22015-05-07 19:33:31 +02001268 */
Xiao Guangrong059ff0f2018-03-30 15:51:23 +08001269 send_async = false;
Juan Quintela56e93d22015-05-07 19:33:31 +02001270 }
1271 }
1272
1273 /* XBZRLE overflow or normal page */
1274 if (pages == -1) {
Peter Xuec6f3ab2022-10-11 17:55:56 -04001275 pages = save_normal_page(pss, block, offset, p, send_async);
Juan Quintela56e93d22015-05-07 19:33:31 +02001276 }
1277
1278 XBZRLE_cache_unlock();
1279
1280 return pages;
1281}
1282
Peter Xu61717ea2022-10-11 17:55:53 -04001283static int ram_save_multifd_page(QEMUFile *file, RAMBlock *block,
Juan Quintelab9ee2f72016-01-15 11:40:13 +01001284 ram_addr_t offset)
1285{
Peter Xu61717ea2022-10-11 17:55:53 -04001286 if (multifd_queue_page(file, block, offset) < 0) {
Ivan Ren713f7622019-06-25 21:18:17 +08001287 return -1;
1288 }
Juan Quintelaaff3f662023-04-26 19:37:19 +02001289 stat64_add(&mig_stats.normal_pages, 1);
Juan Quintelab9ee2f72016-01-15 11:40:13 +01001290
1291 return 1;
1292}
1293
Xiao Guangrong32b05492018-09-06 15:01:01 +08001294static bool save_page_use_compression(RAMState *rs);
1295
Lukas Straub3e817632023-04-20 11:48:06 +02001296static int send_queued_data(CompressParam *param)
1297{
1298 PageSearchStatus *pss = &ram_state->pss[RAM_CHANNEL_PRECOPY];
1299 MigrationState *ms = migrate_get_current();
1300 QEMUFile *file = ms->to_dst_file;
1301 int len = 0;
1302
1303 RAMBlock *block = param->block;
1304 ram_addr_t offset = param->offset;
1305
1306 if (param->result == RES_NONE) {
1307 return 0;
1308 }
1309
1310 assert(block == pss->last_sent_block);
1311
1312 if (param->result == RES_ZEROPAGE) {
Lukas Straub4024cc82023-04-20 11:48:31 +02001313 assert(qemu_file_buffer_empty(param->file));
Lukas Straub3e817632023-04-20 11:48:06 +02001314 len += save_page_header(pss, file, block, offset | RAM_SAVE_FLAG_ZERO);
1315 qemu_put_byte(file, 0);
1316 len += 1;
1317 ram_release_page(block->idstr, offset);
1318 } else if (param->result == RES_COMPRESS) {
Lukas Straub4024cc82023-04-20 11:48:31 +02001319 assert(!qemu_file_buffer_empty(param->file));
Lukas Straub3e817632023-04-20 11:48:06 +02001320 len += save_page_header(pss, file, block,
1321 offset | RAM_SAVE_FLAG_COMPRESS_PAGE);
1322 len += qemu_put_qemu_file(file, param->file);
1323 } else {
1324 abort();
1325 }
1326
Lukas Straub680628d2023-04-20 11:48:10 +02001327 update_compress_thread_counts(param, len);
1328
Lukas Straub3e817632023-04-20 11:48:06 +02001329 return len;
1330}
1331
Lukas Straubef4f5f52023-04-20 11:48:13 +02001332static void ram_flush_compressed_data(RAMState *rs)
1333{
1334 if (!save_page_use_compression(rs)) {
1335 return;
1336 }
1337
1338 flush_compressed_data(send_queued_data);
1339}
1340
Juan Quintela31e2ac72022-06-21 13:29:36 +02001341#define PAGE_ALL_CLEAN 0
1342#define PAGE_TRY_AGAIN 1
1343#define PAGE_DIRTY_FOUND 2
Juan Quintela56e93d22015-05-07 19:33:31 +02001344/**
Juan Quintela3d0684b2017-03-23 15:06:39 +01001345 * find_dirty_block: find the next dirty page and update any state
1346 * associated with the search process.
Dr. David Alan Gilbertb9e60922015-09-23 15:27:11 +01001347 *
Juan Quintela31e2ac72022-06-21 13:29:36 +02001348 * Returns:
Juan Quintela294e5a42022-06-21 13:36:11 +02001349 * <0: An error happened
Juan Quintela31e2ac72022-06-21 13:29:36 +02001350 * PAGE_ALL_CLEAN: no dirty page found, give up
1351 * PAGE_TRY_AGAIN: no dirty page found, retry for next block
1352 * PAGE_DIRTY_FOUND: dirty page found
Dr. David Alan Gilbertb9e60922015-09-23 15:27:11 +01001353 *
Juan Quintela6f37bb82017-03-13 19:26:29 +01001354 * @rs: current RAM state
Juan Quintela3d0684b2017-03-23 15:06:39 +01001355 * @pss: data about the state of the current dirty page scan
1356 * @again: set to false if the search has scanned the whole of RAM
Dr. David Alan Gilbertb9e60922015-09-23 15:27:11 +01001357 */
Juan Quintela31e2ac72022-06-21 13:29:36 +02001358static int find_dirty_block(RAMState *rs, PageSearchStatus *pss)
Dr. David Alan Gilbertb9e60922015-09-23 15:27:11 +01001359{
Peter Xud9e474e2022-10-11 17:55:52 -04001360 /* Update pss->page for the next dirty bit in ramblock */
1361 pss_find_next_dirty(pss);
1362
Juan Quintela6f37bb82017-03-13 19:26:29 +01001363 if (pss->complete_round && pss->block == rs->last_seen_block &&
Juan Quintelaa935e302017-03-21 15:36:51 +01001364 pss->page >= rs->last_page) {
Dr. David Alan Gilbertb9e60922015-09-23 15:27:11 +01001365 /*
1366 * We've been once around the RAM and haven't found anything.
1367 * Give up.
1368 */
Juan Quintela31e2ac72022-06-21 13:29:36 +02001369 return PAGE_ALL_CLEAN;
Dr. David Alan Gilbertb9e60922015-09-23 15:27:11 +01001370 }
David Hildenbrand542147f2021-04-29 13:27:08 +02001371 if (!offset_in_ramblock(pss->block,
1372 ((ram_addr_t)pss->page) << TARGET_PAGE_BITS)) {
Dr. David Alan Gilbertb9e60922015-09-23 15:27:11 +01001373 /* Didn't find anything in this RAM Block */
Juan Quintelaa935e302017-03-21 15:36:51 +01001374 pss->page = 0;
Dr. David Alan Gilbertb9e60922015-09-23 15:27:11 +01001375 pss->block = QLIST_NEXT_RCU(pss->block, next);
1376 if (!pss->block) {
Juan Quintelad4f34482023-10-11 22:55:48 +02001377 if (migrate_multifd() &&
1378 !migrate_multifd_flush_after_each_section()) {
Juan Quintela294e5a42022-06-21 13:36:11 +02001379 QEMUFile *f = rs->pss[RAM_CHANNEL_PRECOPY].pss_channel;
1380 int ret = multifd_send_sync_main(f);
1381 if (ret < 0) {
1382 return ret;
1383 }
1384 qemu_put_be64(f, RAM_SAVE_FLAG_MULTIFD_FLUSH);
1385 qemu_fflush(f);
1386 }
Xiao Guangrong48df9d82018-09-06 15:00:59 +08001387 /*
1388 * If memory migration starts over, we will meet a dirtied page
1389 * which may still exists in compression threads's ring, so we
1390 * should flush the compressed data to make sure the new page
1391 * is not overwritten by the old one in the destination.
1392 *
1393 * Also If xbzrle is on, stop using the data compression at this
1394 * point. In theory, xbzrle can do better than compression.
1395 */
Lukas Straubef4f5f52023-04-20 11:48:13 +02001396 ram_flush_compressed_data(rs);
Xiao Guangrong48df9d82018-09-06 15:00:59 +08001397
Dr. David Alan Gilbertb9e60922015-09-23 15:27:11 +01001398 /* Hit the end of the list */
1399 pss->block = QLIST_FIRST_RCU(&ram_list.blocks);
1400 /* Flag that we've looped */
1401 pss->complete_round = true;
David Hildenbrand1a373522021-02-16 11:50:39 +01001402 /* After the first round, enable XBZRLE. */
Juan Quintela87dca0c2023-03-01 22:20:13 +01001403 if (migrate_xbzrle()) {
Juan Quintelaf3095cc2023-05-04 13:53:23 +02001404 rs->xbzrle_started = true;
David Hildenbrand1a373522021-02-16 11:50:39 +01001405 }
Dr. David Alan Gilbertb9e60922015-09-23 15:27:11 +01001406 }
1407 /* Didn't find anything this time, but try again on the new block */
Juan Quintela31e2ac72022-06-21 13:29:36 +02001408 return PAGE_TRY_AGAIN;
Dr. David Alan Gilbertb9e60922015-09-23 15:27:11 +01001409 } else {
Juan Quintela31e2ac72022-06-21 13:29:36 +02001410 /* We've found something */
1411 return PAGE_DIRTY_FOUND;
Dr. David Alan Gilbertb9e60922015-09-23 15:27:11 +01001412 }
1413}
1414
Juan Quintela3d0684b2017-03-23 15:06:39 +01001415/**
1416 * unqueue_page: gets a page of the queue
Dr. David Alan Gilberta82d5932015-11-05 18:11:09 +00001417 *
Juan Quintela3d0684b2017-03-23 15:06:39 +01001418 * Helper for 'get_queued_page' - gets a page off the queue
1419 *
1420 * Returns the block of the page (or NULL if none available)
1421 *
Juan Quintelaec481c62017-03-20 22:12:40 +01001422 * @rs: current RAM state
Juan Quintela3d0684b2017-03-23 15:06:39 +01001423 * @offset: used to return the offset within the RAMBlock
Dr. David Alan Gilberta82d5932015-11-05 18:11:09 +00001424 */
Juan Quintelaf20e2862017-03-21 16:19:05 +01001425static RAMBlock *unqueue_page(RAMState *rs, ram_addr_t *offset)
Dr. David Alan Gilberta82d5932015-11-05 18:11:09 +00001426{
Peter Xua1fe28d2022-01-19 16:09:18 +08001427 struct RAMSrcPageRequest *entry;
Dr. David Alan Gilberta82d5932015-11-05 18:11:09 +00001428 RAMBlock *block = NULL;
1429
Peter Xua1fe28d2022-01-19 16:09:18 +08001430 if (!postcopy_has_request(rs)) {
Xiao Guangrongae526e32018-08-21 16:10:25 +08001431 return NULL;
1432 }
1433
Daniel Brodsky6e8a3552020-04-03 21:21:08 -07001434 QEMU_LOCK_GUARD(&rs->src_page_req_mutex);
Dr. David Alan Gilberta82d5932015-11-05 18:11:09 +00001435
Peter Xua1fe28d2022-01-19 16:09:18 +08001436 /*
1437 * This should _never_ change even after we take the lock, because no one
1438 * should be taking anything off the request list other than us.
1439 */
1440 assert(postcopy_has_request(rs));
1441
1442 entry = QSIMPLEQ_FIRST(&rs->src_page_requests);
1443 block = entry->rb;
1444 *offset = entry->offset;
1445
Thomas Huth777f53c2022-08-02 08:19:49 +02001446 if (entry->len > TARGET_PAGE_SIZE) {
1447 entry->len -= TARGET_PAGE_SIZE;
1448 entry->offset += TARGET_PAGE_SIZE;
Peter Xua1fe28d2022-01-19 16:09:18 +08001449 } else {
1450 memory_region_unref(block->mr);
1451 QSIMPLEQ_REMOVE_HEAD(&rs->src_page_requests, next_req);
1452 g_free(entry);
1453 migration_consume_urgent_request();
Dr. David Alan Gilberta82d5932015-11-05 18:11:09 +00001454 }
Dr. David Alan Gilberta82d5932015-11-05 18:11:09 +00001455
1456 return block;
1457}
1458
Andrey Gruzdev278e2f52021-01-29 13:14:05 +03001459#if defined(__linux__)
1460/**
1461 * poll_fault_page: try to get next UFFD write fault page and, if pending fault
1462 * is found, return RAM block pointer and page offset
1463 *
1464 * Returns pointer to the RAMBlock containing faulting page,
1465 * NULL if no write faults are pending
1466 *
1467 * @rs: current RAM state
1468 * @offset: page offset from the beginning of the block
1469 */
1470static RAMBlock *poll_fault_page(RAMState *rs, ram_addr_t *offset)
1471{
1472 struct uffd_msg uffd_msg;
1473 void *page_address;
Andrey Gruzdev82ea3e32021-04-01 12:22:26 +03001474 RAMBlock *block;
Andrey Gruzdev278e2f52021-01-29 13:14:05 +03001475 int res;
1476
1477 if (!migrate_background_snapshot()) {
1478 return NULL;
1479 }
1480
1481 res = uffd_read_events(rs->uffdio_fd, &uffd_msg, 1);
1482 if (res <= 0) {
1483 return NULL;
1484 }
1485
1486 page_address = (void *)(uintptr_t) uffd_msg.arg.pagefault.address;
Andrey Gruzdev82ea3e32021-04-01 12:22:26 +03001487 block = qemu_ram_block_from_host(page_address, false, offset);
1488 assert(block && (block->flags & RAM_UF_WRITEPROTECT) != 0);
1489 return block;
Andrey Gruzdev278e2f52021-01-29 13:14:05 +03001490}
1491
1492/**
1493 * ram_save_release_protection: release UFFD write protection after
1494 * a range of pages has been saved
1495 *
1496 * @rs: current RAM state
1497 * @pss: page-search-status structure
1498 * @start_page: index of the first page in the range relative to pss->block
1499 *
1500 * Returns 0 on success, negative value in case of an error
1501*/
1502static int ram_save_release_protection(RAMState *rs, PageSearchStatus *pss,
1503 unsigned long start_page)
1504{
1505 int res = 0;
1506
1507 /* Check if page is from UFFD-managed region. */
1508 if (pss->block->flags & RAM_UF_WRITEPROTECT) {
1509 void *page_address = pss->block->host + (start_page << TARGET_PAGE_BITS);
Peter Xu258f5c982022-01-19 16:09:15 +08001510 uint64_t run_length = (pss->page - start_page) << TARGET_PAGE_BITS;
Andrey Gruzdev278e2f52021-01-29 13:14:05 +03001511
1512 /* Flush async buffers before un-protect. */
Peter Xu61717ea2022-10-11 17:55:53 -04001513 qemu_fflush(pss->pss_channel);
Andrey Gruzdev278e2f52021-01-29 13:14:05 +03001514 /* Un-protect memory range. */
1515 res = uffd_change_protection(rs->uffdio_fd, page_address, run_length,
1516 false, false);
1517 }
1518
1519 return res;
1520}
1521
1522/* ram_write_tracking_available: check if kernel supports required UFFD features
1523 *
1524 * Returns true if supports, false otherwise
1525 */
1526bool ram_write_tracking_available(void)
1527{
1528 uint64_t uffd_features;
1529 int res;
1530
1531 res = uffd_query_features(&uffd_features);
1532 return (res == 0 &&
1533 (uffd_features & UFFD_FEATURE_PAGEFAULT_FLAG_WP) != 0);
1534}
1535
1536/* ram_write_tracking_compatible: check if guest configuration is
1537 * compatible with 'write-tracking'
1538 *
1539 * Returns true if compatible, false otherwise
1540 */
1541bool ram_write_tracking_compatible(void)
1542{
1543 const uint64_t uffd_ioctls_mask = BIT(_UFFDIO_WRITEPROTECT);
1544 int uffd_fd;
Andrey Gruzdev82ea3e32021-04-01 12:22:26 +03001545 RAMBlock *block;
Andrey Gruzdev278e2f52021-01-29 13:14:05 +03001546 bool ret = false;
1547
1548 /* Open UFFD file descriptor */
1549 uffd_fd = uffd_create_fd(UFFD_FEATURE_PAGEFAULT_FLAG_WP, false);
1550 if (uffd_fd < 0) {
1551 return false;
1552 }
1553
1554 RCU_READ_LOCK_GUARD();
1555
Andrey Gruzdev82ea3e32021-04-01 12:22:26 +03001556 RAMBLOCK_FOREACH_NOT_IGNORED(block) {
Andrey Gruzdev278e2f52021-01-29 13:14:05 +03001557 uint64_t uffd_ioctls;
1558
1559 /* Nothing to do with read-only and MMIO-writable regions */
Andrey Gruzdev82ea3e32021-04-01 12:22:26 +03001560 if (block->mr->readonly || block->mr->rom_device) {
Andrey Gruzdev278e2f52021-01-29 13:14:05 +03001561 continue;
1562 }
1563 /* Try to register block memory via UFFD-IO to track writes */
Andrey Gruzdev82ea3e32021-04-01 12:22:26 +03001564 if (uffd_register_memory(uffd_fd, block->host, block->max_length,
Andrey Gruzdev278e2f52021-01-29 13:14:05 +03001565 UFFDIO_REGISTER_MODE_WP, &uffd_ioctls)) {
1566 goto out;
1567 }
1568 if ((uffd_ioctls & uffd_ioctls_mask) != uffd_ioctls_mask) {
1569 goto out;
1570 }
1571 }
1572 ret = true;
1573
1574out:
1575 uffd_close_fd(uffd_fd);
1576 return ret;
1577}
1578
David Hildenbrandf7b9dcf2021-10-11 19:53:45 +02001579static inline void populate_read_range(RAMBlock *block, ram_addr_t offset,
1580 ram_addr_t size)
1581{
David Hildenbrand5f19a442023-01-05 13:45:24 +01001582 const ram_addr_t end = offset + size;
1583
David Hildenbrandf7b9dcf2021-10-11 19:53:45 +02001584 /*
1585 * We read one byte of each page; this will preallocate page tables if
1586 * required and populate the shared zeropage on MAP_PRIVATE anonymous memory
1587 * where no page was populated yet. This might require adaption when
1588 * supporting other mappings, like shmem.
1589 */
David Hildenbrand5f19a442023-01-05 13:45:24 +01001590 for (; offset < end; offset += block->page_size) {
David Hildenbrandf7b9dcf2021-10-11 19:53:45 +02001591 char tmp = *((char *)block->host + offset);
1592
1593 /* Don't optimize the read out */
1594 asm volatile("" : "+r" (tmp));
1595 }
1596}
1597
David Hildenbrand6fee3a12021-10-11 19:53:46 +02001598static inline int populate_read_section(MemoryRegionSection *section,
1599 void *opaque)
1600{
1601 const hwaddr size = int128_get64(section->size);
1602 hwaddr offset = section->offset_within_region;
1603 RAMBlock *block = section->mr->ram_block;
1604
1605 populate_read_range(block, offset, size);
1606 return 0;
1607}
1608
Andrey Gruzdev278e2f52021-01-29 13:14:05 +03001609/*
David Hildenbrandf7b9dcf2021-10-11 19:53:45 +02001610 * ram_block_populate_read: preallocate page tables and populate pages in the
1611 * RAM block by reading a byte of each page.
Andrey Gruzdeveeccb992021-04-01 12:22:25 +03001612 *
1613 * Since it's solely used for userfault_fd WP feature, here we just
1614 * hardcode page size to qemu_real_host_page_size.
1615 *
Andrey Gruzdev82ea3e32021-04-01 12:22:26 +03001616 * @block: RAM block to populate
Andrey Gruzdeveeccb992021-04-01 12:22:25 +03001617 */
David Hildenbrand6fee3a12021-10-11 19:53:46 +02001618static void ram_block_populate_read(RAMBlock *rb)
Andrey Gruzdeveeccb992021-04-01 12:22:25 +03001619{
David Hildenbrand6fee3a12021-10-11 19:53:46 +02001620 /*
1621 * Skip populating all pages that fall into a discarded range as managed by
1622 * a RamDiscardManager responsible for the mapped memory region of the
1623 * RAMBlock. Such discarded ("logically unplugged") parts of a RAMBlock
1624 * must not get populated automatically. We don't have to track
1625 * modifications via userfaultfd WP reliably, because these pages will
1626 * not be part of the migration stream either way -- see
1627 * ramblock_dirty_bitmap_exclude_discarded_pages().
1628 *
1629 * Note: The result is only stable while migrating (precopy/postcopy).
1630 */
1631 if (rb->mr && memory_region_has_ram_discard_manager(rb->mr)) {
1632 RamDiscardManager *rdm = memory_region_get_ram_discard_manager(rb->mr);
1633 MemoryRegionSection section = {
1634 .mr = rb->mr,
1635 .offset_within_region = 0,
1636 .size = rb->mr->size,
1637 };
1638
1639 ram_discard_manager_replay_populated(rdm, &section,
1640 populate_read_section, NULL);
1641 } else {
1642 populate_read_range(rb, 0, rb->used_length);
1643 }
Andrey Gruzdeveeccb992021-04-01 12:22:25 +03001644}
1645
1646/*
1647 * ram_write_tracking_prepare: prepare for UFFD-WP memory tracking
1648 */
1649void ram_write_tracking_prepare(void)
1650{
Andrey Gruzdev82ea3e32021-04-01 12:22:26 +03001651 RAMBlock *block;
Andrey Gruzdeveeccb992021-04-01 12:22:25 +03001652
1653 RCU_READ_LOCK_GUARD();
1654
Andrey Gruzdev82ea3e32021-04-01 12:22:26 +03001655 RAMBLOCK_FOREACH_NOT_IGNORED(block) {
Andrey Gruzdeveeccb992021-04-01 12:22:25 +03001656 /* Nothing to do with read-only and MMIO-writable regions */
Andrey Gruzdev82ea3e32021-04-01 12:22:26 +03001657 if (block->mr->readonly || block->mr->rom_device) {
Andrey Gruzdeveeccb992021-04-01 12:22:25 +03001658 continue;
1659 }
1660
1661 /*
1662 * Populate pages of the RAM block before enabling userfault_fd
1663 * write protection.
1664 *
1665 * This stage is required since ioctl(UFFDIO_WRITEPROTECT) with
1666 * UFFDIO_WRITEPROTECT_MODE_WP mode setting would silently skip
1667 * pages with pte_none() entries in page table.
1668 */
David Hildenbrandf7b9dcf2021-10-11 19:53:45 +02001669 ram_block_populate_read(block);
Andrey Gruzdeveeccb992021-04-01 12:22:25 +03001670 }
1671}
1672
David Hildenbrande41c5772023-01-05 13:45:28 +01001673static inline int uffd_protect_section(MemoryRegionSection *section,
1674 void *opaque)
1675{
1676 const hwaddr size = int128_get64(section->size);
1677 const hwaddr offset = section->offset_within_region;
1678 RAMBlock *rb = section->mr->ram_block;
1679 int uffd_fd = (uintptr_t)opaque;
1680
1681 return uffd_change_protection(uffd_fd, rb->host + offset, size, true,
1682 false);
1683}
1684
1685static int ram_block_uffd_protect(RAMBlock *rb, int uffd_fd)
1686{
1687 assert(rb->flags & RAM_UF_WRITEPROTECT);
1688
1689 /* See ram_block_populate_read() */
1690 if (rb->mr && memory_region_has_ram_discard_manager(rb->mr)) {
1691 RamDiscardManager *rdm = memory_region_get_ram_discard_manager(rb->mr);
1692 MemoryRegionSection section = {
1693 .mr = rb->mr,
1694 .offset_within_region = 0,
1695 .size = rb->mr->size,
1696 };
1697
1698 return ram_discard_manager_replay_populated(rdm, &section,
1699 uffd_protect_section,
1700 (void *)(uintptr_t)uffd_fd);
1701 }
1702 return uffd_change_protection(uffd_fd, rb->host,
1703 rb->used_length, true, false);
1704}
1705
Andrey Gruzdeveeccb992021-04-01 12:22:25 +03001706/*
Andrey Gruzdev278e2f52021-01-29 13:14:05 +03001707 * ram_write_tracking_start: start UFFD-WP memory tracking
1708 *
1709 * Returns 0 for success or negative value in case of error
1710 */
1711int ram_write_tracking_start(void)
1712{
1713 int uffd_fd;
1714 RAMState *rs = ram_state;
Andrey Gruzdev82ea3e32021-04-01 12:22:26 +03001715 RAMBlock *block;
Andrey Gruzdev278e2f52021-01-29 13:14:05 +03001716
1717 /* Open UFFD file descriptor */
1718 uffd_fd = uffd_create_fd(UFFD_FEATURE_PAGEFAULT_FLAG_WP, true);
1719 if (uffd_fd < 0) {
1720 return uffd_fd;
1721 }
1722 rs->uffdio_fd = uffd_fd;
1723
1724 RCU_READ_LOCK_GUARD();
1725
Andrey Gruzdev82ea3e32021-04-01 12:22:26 +03001726 RAMBLOCK_FOREACH_NOT_IGNORED(block) {
Andrey Gruzdev278e2f52021-01-29 13:14:05 +03001727 /* Nothing to do with read-only and MMIO-writable regions */
Andrey Gruzdev82ea3e32021-04-01 12:22:26 +03001728 if (block->mr->readonly || block->mr->rom_device) {
Andrey Gruzdev278e2f52021-01-29 13:14:05 +03001729 continue;
1730 }
1731
1732 /* Register block memory with UFFD to track writes */
Andrey Gruzdev82ea3e32021-04-01 12:22:26 +03001733 if (uffd_register_memory(rs->uffdio_fd, block->host,
1734 block->max_length, UFFDIO_REGISTER_MODE_WP, NULL)) {
Andrey Gruzdev278e2f52021-01-29 13:14:05 +03001735 goto fail;
1736 }
David Hildenbrand72ef3a32023-01-05 13:45:25 +01001737 block->flags |= RAM_UF_WRITEPROTECT;
1738 memory_region_ref(block->mr);
1739
Andrey Gruzdev278e2f52021-01-29 13:14:05 +03001740 /* Apply UFFD write protection to the block memory range */
David Hildenbrande41c5772023-01-05 13:45:28 +01001741 if (ram_block_uffd_protect(block, uffd_fd)) {
Andrey Gruzdev278e2f52021-01-29 13:14:05 +03001742 goto fail;
1743 }
Andrey Gruzdev278e2f52021-01-29 13:14:05 +03001744
Andrey Gruzdev82ea3e32021-04-01 12:22:26 +03001745 trace_ram_write_tracking_ramblock_start(block->idstr, block->page_size,
1746 block->host, block->max_length);
Andrey Gruzdev278e2f52021-01-29 13:14:05 +03001747 }
1748
1749 return 0;
1750
1751fail:
1752 error_report("ram_write_tracking_start() failed: restoring initial memory state");
1753
Andrey Gruzdev82ea3e32021-04-01 12:22:26 +03001754 RAMBLOCK_FOREACH_NOT_IGNORED(block) {
1755 if ((block->flags & RAM_UF_WRITEPROTECT) == 0) {
Andrey Gruzdev278e2f52021-01-29 13:14:05 +03001756 continue;
1757 }
Andrey Gruzdev82ea3e32021-04-01 12:22:26 +03001758 uffd_unregister_memory(rs->uffdio_fd, block->host, block->max_length);
Andrey Gruzdev278e2f52021-01-29 13:14:05 +03001759 /* Cleanup flags and remove reference */
Andrey Gruzdev82ea3e32021-04-01 12:22:26 +03001760 block->flags &= ~RAM_UF_WRITEPROTECT;
1761 memory_region_unref(block->mr);
Andrey Gruzdev278e2f52021-01-29 13:14:05 +03001762 }
1763
1764 uffd_close_fd(uffd_fd);
1765 rs->uffdio_fd = -1;
1766 return -1;
1767}
1768
1769/**
1770 * ram_write_tracking_stop: stop UFFD-WP memory tracking and remove protection
1771 */
1772void ram_write_tracking_stop(void)
1773{
1774 RAMState *rs = ram_state;
Andrey Gruzdev82ea3e32021-04-01 12:22:26 +03001775 RAMBlock *block;
Andrey Gruzdev278e2f52021-01-29 13:14:05 +03001776
1777 RCU_READ_LOCK_GUARD();
1778
Andrey Gruzdev82ea3e32021-04-01 12:22:26 +03001779 RAMBLOCK_FOREACH_NOT_IGNORED(block) {
1780 if ((block->flags & RAM_UF_WRITEPROTECT) == 0) {
Andrey Gruzdev278e2f52021-01-29 13:14:05 +03001781 continue;
1782 }
Andrey Gruzdev82ea3e32021-04-01 12:22:26 +03001783 uffd_unregister_memory(rs->uffdio_fd, block->host, block->max_length);
Andrey Gruzdev278e2f52021-01-29 13:14:05 +03001784
Andrey Gruzdev82ea3e32021-04-01 12:22:26 +03001785 trace_ram_write_tracking_ramblock_stop(block->idstr, block->page_size,
1786 block->host, block->max_length);
Andrey Gruzdev278e2f52021-01-29 13:14:05 +03001787
1788 /* Cleanup flags and remove reference */
Andrey Gruzdev82ea3e32021-04-01 12:22:26 +03001789 block->flags &= ~RAM_UF_WRITEPROTECT;
1790 memory_region_unref(block->mr);
Andrey Gruzdev278e2f52021-01-29 13:14:05 +03001791 }
1792
1793 /* Finally close UFFD file descriptor */
1794 uffd_close_fd(rs->uffdio_fd);
1795 rs->uffdio_fd = -1;
1796}
1797
1798#else
1799/* No target OS support, stubs just fail or ignore */
1800
1801static RAMBlock *poll_fault_page(RAMState *rs, ram_addr_t *offset)
1802{
1803 (void) rs;
1804 (void) offset;
1805
1806 return NULL;
1807}
1808
1809static int ram_save_release_protection(RAMState *rs, PageSearchStatus *pss,
1810 unsigned long start_page)
1811{
1812 (void) rs;
1813 (void) pss;
1814 (void) start_page;
1815
1816 return 0;
1817}
1818
1819bool ram_write_tracking_available(void)
1820{
1821 return false;
1822}
1823
1824bool ram_write_tracking_compatible(void)
1825{
1826 assert(0);
1827 return false;
1828}
1829
1830int ram_write_tracking_start(void)
1831{
1832 assert(0);
1833 return -1;
1834}
1835
1836void ram_write_tracking_stop(void)
1837{
1838 assert(0);
1839}
1840#endif /* defined(__linux__) */
1841
Juan Quintela3d0684b2017-03-23 15:06:39 +01001842/**
Li Qiangff1543a2019-05-24 23:28:32 -07001843 * get_queued_page: unqueue a page from the postcopy requests
Dr. David Alan Gilberta82d5932015-11-05 18:11:09 +00001844 *
Juan Quintela3d0684b2017-03-23 15:06:39 +01001845 * Skips pages that are already sent (!dirty)
Dr. David Alan Gilberta82d5932015-11-05 18:11:09 +00001846 *
Wei Yanga5f7b1a2019-05-11 07:37:29 +08001847 * Returns true if a queued page is found
Juan Quintela3d0684b2017-03-23 15:06:39 +01001848 *
Juan Quintela6f37bb82017-03-13 19:26:29 +01001849 * @rs: current RAM state
Juan Quintela3d0684b2017-03-23 15:06:39 +01001850 * @pss: data about the state of the current dirty page scan
Dr. David Alan Gilberta82d5932015-11-05 18:11:09 +00001851 */
Juan Quintelaf20e2862017-03-21 16:19:05 +01001852static bool get_queued_page(RAMState *rs, PageSearchStatus *pss)
Dr. David Alan Gilberta82d5932015-11-05 18:11:09 +00001853{
1854 RAMBlock *block;
1855 ram_addr_t offset;
Thomas Huth777f53c2022-08-02 08:19:49 +02001856 bool dirty;
Dr. David Alan Gilberta82d5932015-11-05 18:11:09 +00001857
Thomas Huth777f53c2022-08-02 08:19:49 +02001858 do {
1859 block = unqueue_page(rs, &offset);
1860 /*
1861 * We're sending this page, and since it's postcopy nothing else
1862 * will dirty it, and we must make sure it doesn't get sent again
1863 * even if this queue request was received after the background
1864 * search already sent it.
1865 */
1866 if (block) {
1867 unsigned long page;
1868
1869 page = offset >> TARGET_PAGE_BITS;
1870 dirty = test_bit(page, block->bmap);
1871 if (!dirty) {
1872 trace_get_queued_page_not_dirty(block->idstr, (uint64_t)offset,
1873 page);
1874 } else {
1875 trace_get_queued_page(block->idstr, (uint64_t)offset, page);
1876 }
1877 }
1878
1879 } while (block && !dirty);
Dr. David Alan Gilberta82d5932015-11-05 18:11:09 +00001880
Peter Xub0621062022-10-11 17:55:58 -04001881 if (!block) {
Andrey Gruzdev278e2f52021-01-29 13:14:05 +03001882 /*
1883 * Poll write faults too if background snapshot is enabled; that's
1884 * when we have vcpus got blocked by the write protected pages.
1885 */
1886 block = poll_fault_page(rs, &offset);
1887 }
1888
Dr. David Alan Gilberta82d5932015-11-05 18:11:09 +00001889 if (block) {
1890 /*
Dr. David Alan Gilberta82d5932015-11-05 18:11:09 +00001891 * We want the background search to continue from the queued page
1892 * since the guest is likely to want other pages near to the page
1893 * it just requested.
1894 */
1895 pss->block = block;
Juan Quintelaa935e302017-03-21 15:36:51 +01001896 pss->page = offset >> TARGET_PAGE_BITS;
Wei Yang422314e2019-06-05 09:08:28 +08001897
1898 /*
1899 * This unqueued page would break the "one round" check, even is
1900 * really rare.
1901 */
1902 pss->complete_round = false;
Dr. David Alan Gilberta82d5932015-11-05 18:11:09 +00001903 }
1904
1905 return !!block;
1906}
1907
Juan Quintela56e93d22015-05-07 19:33:31 +02001908/**
Juan Quintela5e58f962017-04-03 22:06:54 +02001909 * migration_page_queue_free: drop any remaining pages in the ram
1910 * request queue
Dr. David Alan Gilbert6c595cd2015-11-05 18:11:08 +00001911 *
Juan Quintela3d0684b2017-03-23 15:06:39 +01001912 * It should be empty at the end anyway, but in error cases there may
1913 * be some left. in case that there is any page left, we drop it.
1914 *
Dr. David Alan Gilbert6c595cd2015-11-05 18:11:08 +00001915 */
Juan Quintela83c13382017-05-04 11:45:01 +02001916static void migration_page_queue_free(RAMState *rs)
Dr. David Alan Gilbert6c595cd2015-11-05 18:11:08 +00001917{
Juan Quintelaec481c62017-03-20 22:12:40 +01001918 struct RAMSrcPageRequest *mspr, *next_mspr;
Dr. David Alan Gilbert6c595cd2015-11-05 18:11:08 +00001919 /* This queue generally should be empty - but in the case of a failed
1920 * migration might have some droppings in.
1921 */
Dr. David Alan Gilbert89ac5a12019-10-07 15:36:39 +01001922 RCU_READ_LOCK_GUARD();
Juan Quintelaec481c62017-03-20 22:12:40 +01001923 QSIMPLEQ_FOREACH_SAFE(mspr, &rs->src_page_requests, next_req, next_mspr) {
Dr. David Alan Gilbert6c595cd2015-11-05 18:11:08 +00001924 memory_region_unref(mspr->rb->mr);
Juan Quintelaec481c62017-03-20 22:12:40 +01001925 QSIMPLEQ_REMOVE_HEAD(&rs->src_page_requests, next_req);
Dr. David Alan Gilbert6c595cd2015-11-05 18:11:08 +00001926 g_free(mspr);
1927 }
Dr. David Alan Gilbert6c595cd2015-11-05 18:11:08 +00001928}
1929
1930/**
Juan Quintela3d0684b2017-03-23 15:06:39 +01001931 * ram_save_queue_pages: queue the page for transmission
1932 *
1933 * A request from postcopy destination for example.
1934 *
1935 * Returns zero on success or negative on error
1936 *
Juan Quintela3d0684b2017-03-23 15:06:39 +01001937 * @rbname: Name of the RAMBLock of the request. NULL means the
1938 * same that last one.
1939 * @start: starting address from the start of the RAMBlock
1940 * @len: length (in bytes) to send
Dr. David Alan Gilbert6c595cd2015-11-05 18:11:08 +00001941 */
Juan Quintela96506892017-03-14 18:41:03 +01001942int ram_save_queue_pages(const char *rbname, ram_addr_t start, ram_addr_t len)
Dr. David Alan Gilbert6c595cd2015-11-05 18:11:08 +00001943{
1944 RAMBlock *ramblock;
Juan Quintela53518d92017-05-04 11:46:24 +02001945 RAMState *rs = ram_state;
Dr. David Alan Gilbert6c595cd2015-11-05 18:11:08 +00001946
Juan Quintelaaff3f662023-04-26 19:37:19 +02001947 stat64_add(&mig_stats.postcopy_requests, 1);
Dr. David Alan Gilbert89ac5a12019-10-07 15:36:39 +01001948 RCU_READ_LOCK_GUARD();
1949
Dr. David Alan Gilbert6c595cd2015-11-05 18:11:08 +00001950 if (!rbname) {
1951 /* Reuse last RAMBlock */
Juan Quintela68a098f2017-03-14 13:48:42 +01001952 ramblock = rs->last_req_rb;
Dr. David Alan Gilbert6c595cd2015-11-05 18:11:08 +00001953
1954 if (!ramblock) {
1955 /*
1956 * Shouldn't happen, we can't reuse the last RAMBlock if
1957 * it's the 1st request.
1958 */
1959 error_report("ram_save_queue_pages no previous block");
Daniel Henrique Barboza03acb4e2020-01-06 15:23:31 -03001960 return -1;
Dr. David Alan Gilbert6c595cd2015-11-05 18:11:08 +00001961 }
1962 } else {
1963 ramblock = qemu_ram_block_by_name(rbname);
1964
1965 if (!ramblock) {
1966 /* We shouldn't be asked for a non-existent RAMBlock */
1967 error_report("ram_save_queue_pages no block '%s'", rbname);
Daniel Henrique Barboza03acb4e2020-01-06 15:23:31 -03001968 return -1;
Dr. David Alan Gilbert6c595cd2015-11-05 18:11:08 +00001969 }
Juan Quintela68a098f2017-03-14 13:48:42 +01001970 rs->last_req_rb = ramblock;
Dr. David Alan Gilbert6c595cd2015-11-05 18:11:08 +00001971 }
1972 trace_ram_save_queue_pages(ramblock->idstr, start, len);
David Hildenbrand542147f2021-04-29 13:27:08 +02001973 if (!offset_in_ramblock(ramblock, start + len - 1)) {
Juan Quintela9458ad62015-11-10 17:42:05 +01001974 error_report("%s request overrun start=" RAM_ADDR_FMT " len="
1975 RAM_ADDR_FMT " blocklen=" RAM_ADDR_FMT,
Dr. David Alan Gilbert6c595cd2015-11-05 18:11:08 +00001976 __func__, start, len, ramblock->used_length);
Daniel Henrique Barboza03acb4e2020-01-06 15:23:31 -03001977 return -1;
Dr. David Alan Gilbert6c595cd2015-11-05 18:11:08 +00001978 }
1979
Peter Xu93589822022-10-11 17:55:57 -04001980 /*
1981 * When with postcopy preempt, we send back the page directly in the
1982 * rp-return thread.
1983 */
1984 if (postcopy_preempt_active()) {
1985 ram_addr_t page_start = start >> TARGET_PAGE_BITS;
1986 size_t page_size = qemu_ram_pagesize(ramblock);
1987 PageSearchStatus *pss = &ram_state->pss[RAM_CHANNEL_POSTCOPY];
1988 int ret = 0;
1989
1990 qemu_mutex_lock(&rs->bitmap_mutex);
1991
1992 pss_init(pss, ramblock, page_start);
1993 /*
1994 * Always use the preempt channel, and make sure it's there. It's
1995 * safe to access without lock, because when rp-thread is running
1996 * we should be the only one who operates on the qemufile
1997 */
1998 pss->pss_channel = migrate_get_current()->postcopy_qemufile_src;
Peter Xu93589822022-10-11 17:55:57 -04001999 assert(pss->pss_channel);
2000
2001 /*
2002 * It must be either one or multiple of host page size. Just
2003 * assert; if something wrong we're mostly split brain anyway.
2004 */
2005 assert(len % page_size == 0);
2006 while (len) {
2007 if (ram_save_host_page_urgent(pss)) {
2008 error_report("%s: ram_save_host_page_urgent() failed: "
2009 "ramblock=%s, start_addr=0x"RAM_ADDR_FMT,
2010 __func__, ramblock->idstr, start);
2011 ret = -1;
2012 break;
2013 }
2014 /*
2015 * NOTE: after ram_save_host_page_urgent() succeeded, pss->page
2016 * will automatically be moved and point to the next host page
2017 * we're going to send, so no need to update here.
2018 *
2019 * Normally QEMU never sends >1 host page in requests, so
2020 * logically we don't even need that as the loop should only
2021 * run once, but just to be consistent.
2022 */
2023 len -= page_size;
2024 };
2025 qemu_mutex_unlock(&rs->bitmap_mutex);
2026
2027 return ret;
2028 }
2029
Juan Quintelaec481c62017-03-20 22:12:40 +01002030 struct RAMSrcPageRequest *new_entry =
Markus Armbrusterb21e2382022-03-15 15:41:56 +01002031 g_new0(struct RAMSrcPageRequest, 1);
Dr. David Alan Gilbert6c595cd2015-11-05 18:11:08 +00002032 new_entry->rb = ramblock;
2033 new_entry->offset = start;
2034 new_entry->len = len;
2035
2036 memory_region_ref(ramblock->mr);
Juan Quintelaec481c62017-03-20 22:12:40 +01002037 qemu_mutex_lock(&rs->src_page_req_mutex);
2038 QSIMPLEQ_INSERT_TAIL(&rs->src_page_requests, new_entry, next_req);
Dr. David Alan Gilberte03a34f2018-06-13 11:26:42 +01002039 migration_make_urgent_request();
Juan Quintelaec481c62017-03-20 22:12:40 +01002040 qemu_mutex_unlock(&rs->src_page_req_mutex);
Dr. David Alan Gilbert6c595cd2015-11-05 18:11:08 +00002041
2042 return 0;
Dr. David Alan Gilbert6c595cd2015-11-05 18:11:08 +00002043}
2044
Xiao Guangrongd7400a32018-03-30 15:51:26 +08002045static bool save_page_use_compression(RAMState *rs)
2046{
Juan Quintelaa7a94d12023-03-01 22:03:48 +01002047 if (!migrate_compress()) {
Xiao Guangrongd7400a32018-03-30 15:51:26 +08002048 return false;
2049 }
2050
2051 /*
David Hildenbrand1a373522021-02-16 11:50:39 +01002052 * If xbzrle is enabled (e.g., after first round of migration), stop
2053 * using the data compression. In theory, xbzrle can do better than
2054 * compression.
Xiao Guangrongd7400a32018-03-30 15:51:26 +08002055 */
Juan Quintelaf3095cc2023-05-04 13:53:23 +02002056 if (rs->xbzrle_started) {
David Hildenbrand1a373522021-02-16 11:50:39 +01002057 return false;
Xiao Guangrongd7400a32018-03-30 15:51:26 +08002058 }
2059
David Hildenbrand1a373522021-02-16 11:50:39 +01002060 return true;
Xiao Guangrongd7400a32018-03-30 15:51:26 +08002061}
2062
Xiao Guangrong5e5fdcf2018-08-21 16:10:24 +08002063/*
2064 * try to compress the page before posting it out, return true if the page
2065 * has been properly handled by compression, otherwise needs other
2066 * paths to handle it
2067 */
Peter Xuec6f3ab2022-10-11 17:55:56 -04002068static bool save_compress_page(RAMState *rs, PageSearchStatus *pss,
Juan Quintelad637a182023-06-13 13:34:12 +02002069 ram_addr_t offset)
Xiao Guangrong5e5fdcf2018-08-21 16:10:24 +08002070{
2071 if (!save_page_use_compression(rs)) {
2072 return false;
2073 }
2074
2075 /*
2076 * When starting the process of a new block, the first page of
2077 * the block should be sent out before other pages in the same
2078 * block, and all the pages in last block should have been sent
2079 * out, keeping this order is important, because the 'cont' flag
2080 * is used to avoid resending the block name.
2081 *
2082 * We post the fist page as normal page as compression will take
2083 * much CPU resource.
2084 */
Juan Quintelad637a182023-06-13 13:34:12 +02002085 if (pss->block != pss->last_sent_block) {
Lukas Straubef4f5f52023-04-20 11:48:13 +02002086 ram_flush_compressed_data(rs);
Xiao Guangrong5e5fdcf2018-08-21 16:10:24 +08002087 return false;
2088 }
2089
Juan Quintelad637a182023-06-13 13:34:12 +02002090 if (compress_page_with_multi_thread(pss->block, offset,
2091 send_queued_data) > 0) {
Xiao Guangrong5e5fdcf2018-08-21 16:10:24 +08002092 return true;
2093 }
2094
Xiao Guangrong76e03002018-09-06 15:01:00 +08002095 compression_counters.busy++;
Xiao Guangrong5e5fdcf2018-08-21 16:10:24 +08002096 return false;
2097}
2098
Dr. David Alan Gilberta82d5932015-11-05 18:11:09 +00002099/**
Juan Quintela4010ba32021-12-15 20:10:39 +01002100 * ram_save_target_page_legacy: save one target page
Dr. David Alan Gilberta82d5932015-11-05 18:11:09 +00002101 *
Juan Quintela3d0684b2017-03-23 15:06:39 +01002102 * Returns the number of pages written
Dr. David Alan Gilberta82d5932015-11-05 18:11:09 +00002103 *
Juan Quintela6f37bb82017-03-13 19:26:29 +01002104 * @rs: current RAM state
Juan Quintela3d0684b2017-03-23 15:06:39 +01002105 * @pss: data about the page we want to send
Dr. David Alan Gilberta82d5932015-11-05 18:11:09 +00002106 */
Juan Quintela4010ba32021-12-15 20:10:39 +01002107static int ram_save_target_page_legacy(RAMState *rs, PageSearchStatus *pss)
Dr. David Alan Gilberta82d5932015-11-05 18:11:09 +00002108{
Xiao Guangronga8ec91f2018-03-30 15:51:25 +08002109 RAMBlock *block = pss->block;
Alexey Romko8bba0042020-01-10 14:51:34 +01002110 ram_addr_t offset = ((ram_addr_t)pss->page) << TARGET_PAGE_BITS;
Xiao Guangronga8ec91f2018-03-30 15:51:25 +08002111 int res;
2112
Juan Quintela944853c2023-06-13 13:36:24 +02002113 if (control_save_page(pss, offset, &res)) {
Xiao Guangronga8ec91f2018-03-30 15:51:25 +08002114 return res;
2115 }
2116
Juan Quintelad637a182023-06-13 13:34:12 +02002117 if (save_compress_page(rs, pss, offset)) {
Xiao Guangrong5e5fdcf2018-08-21 16:10:24 +08002118 return 1;
Xiao Guangrongd7400a32018-03-30 15:51:26 +08002119 }
2120
Juan Quintelae8e4e7a2023-06-13 13:38:30 +02002121 if (save_zero_page(rs, pss, offset)) {
Fabiano Rosas8697eb82023-10-11 15:46:03 -03002122 return 1;
Xiao Guangrongd7400a32018-03-30 15:51:26 +08002123 }
2124
Xiao Guangrongda3f56c2018-03-30 15:51:28 +08002125 /*
Peter Xu6f39c902022-10-04 14:24:30 -04002126 * Do not use multifd in postcopy as one whole host page should be
2127 * placed. Meanwhile postcopy requires atomic update of pages, so even
2128 * if host page size == guest page size the dest guest during run may
2129 * still see partially copied pages which is data corruption.
Xiao Guangrongda3f56c2018-03-30 15:51:28 +08002130 */
Juan Quintela51b07542023-03-01 22:10:29 +01002131 if (migrate_multifd() && !migration_in_postcopy()) {
Peter Xu61717ea2022-10-11 17:55:53 -04002132 return ram_save_multifd_page(pss->pss_channel, block, offset);
Dr. David Alan Gilberta82d5932015-11-05 18:11:09 +00002133 }
2134
Juan Quintela05931ec2021-12-15 19:01:21 +01002135 return ram_save_page(rs, pss);
Dr. David Alan Gilberta82d5932015-11-05 18:11:09 +00002136}
2137
Peter Xud9e474e2022-10-11 17:55:52 -04002138/* Should be called before sending a host page */
2139static void pss_host_page_prepare(PageSearchStatus *pss)
2140{
2141 /* How many guest pages are there in one host page? */
2142 size_t guest_pfns = qemu_ram_pagesize(pss->block) >> TARGET_PAGE_BITS;
2143
2144 pss->host_page_sending = true;
Peter Xu301d7ff2023-01-20 11:31:47 -05002145 if (guest_pfns <= 1) {
2146 /*
2147 * This covers both when guest psize == host psize, or when guest
2148 * has larger psize than the host (guest_pfns==0).
2149 *
2150 * For the latter, we always send one whole guest page per
2151 * iteration of the host page (example: an Alpha VM on x86 host
2152 * will have guest psize 8K while host psize 4K).
2153 */
2154 pss->host_page_start = pss->page;
2155 pss->host_page_end = pss->page + 1;
2156 } else {
2157 /*
2158 * The host page spans over multiple guest pages, we send them
2159 * within the same host page iteration.
2160 */
2161 pss->host_page_start = ROUND_DOWN(pss->page, guest_pfns);
2162 pss->host_page_end = ROUND_UP(pss->page + 1, guest_pfns);
2163 }
Peter Xud9e474e2022-10-11 17:55:52 -04002164}
2165
2166/*
2167 * Whether the page pointed by PSS is within the host page being sent.
2168 * Must be called after a previous pss_host_page_prepare().
2169 */
2170static bool pss_within_range(PageSearchStatus *pss)
2171{
2172 ram_addr_t ram_addr;
2173
2174 assert(pss->host_page_sending);
2175
2176 /* Over host-page boundary? */
2177 if (pss->page >= pss->host_page_end) {
2178 return false;
2179 }
2180
2181 ram_addr = ((ram_addr_t)pss->page) << TARGET_PAGE_BITS;
2182
2183 return offset_in_ramblock(pss->block, ram_addr);
2184}
2185
2186static void pss_host_page_finish(PageSearchStatus *pss)
2187{
2188 pss->host_page_sending = false;
2189 /* This is not needed, but just to reset it */
2190 pss->host_page_start = pss->host_page_end = 0;
2191}
2192
Peter Xu93589822022-10-11 17:55:57 -04002193/*
2194 * Send an urgent host page specified by `pss'. Need to be called with
2195 * bitmap_mutex held.
2196 *
2197 * Returns 0 if save host page succeeded, false otherwise.
2198 */
2199static int ram_save_host_page_urgent(PageSearchStatus *pss)
2200{
2201 bool page_dirty, sent = false;
2202 RAMState *rs = ram_state;
2203 int ret = 0;
2204
2205 trace_postcopy_preempt_send_host_page(pss->block->idstr, pss->page);
2206 pss_host_page_prepare(pss);
2207
2208 /*
2209 * If precopy is sending the same page, let it be done in precopy, or
2210 * we could send the same page in two channels and none of them will
2211 * receive the whole page.
2212 */
2213 if (pss_overlap(pss, &ram_state->pss[RAM_CHANNEL_PRECOPY])) {
2214 trace_postcopy_preempt_hit(pss->block->idstr,
2215 pss->page << TARGET_PAGE_BITS);
2216 return 0;
2217 }
2218
2219 do {
2220 page_dirty = migration_bitmap_clear_dirty(rs, pss->block, pss->page);
2221
2222 if (page_dirty) {
2223 /* Be strict to return code; it must be 1, or what else? */
Juan Quintela4010ba32021-12-15 20:10:39 +01002224 if (migration_ops->ram_save_target_page(rs, pss) != 1) {
Peter Xu93589822022-10-11 17:55:57 -04002225 error_report_once("%s: ram_save_target_page failed", __func__);
2226 ret = -1;
2227 goto out;
2228 }
2229 sent = true;
2230 }
2231 pss_find_next_dirty(pss);
2232 } while (pss_within_range(pss));
2233out:
2234 pss_host_page_finish(pss);
2235 /* For urgent requests, flush immediately if sent */
2236 if (sent) {
2237 qemu_fflush(pss->pss_channel);
2238 }
2239 return ret;
2240}
2241
Dr. David Alan Gilberta82d5932015-11-05 18:11:09 +00002242/**
Juan Quintela3d0684b2017-03-23 15:06:39 +01002243 * ram_save_host_page: save a whole host page
Dr. David Alan Gilberta82d5932015-11-05 18:11:09 +00002244 *
Juan Quintela3d0684b2017-03-23 15:06:39 +01002245 * Starting at *offset send pages up to the end of the current host
2246 * page. It's valid for the initial offset to point into the middle of
2247 * a host page in which case the remainder of the hostpage is sent.
2248 * Only dirty target pages are sent. Note that the host page size may
2249 * be a huge page for this block.
Peter Xuf3321552022-10-11 17:55:50 -04002250 *
Dr. David Alan Gilbert1eb3fc02017-05-17 17:58:09 +01002251 * The saving stops at the boundary of the used_length of the block
2252 * if the RAMBlock isn't a multiple of the host page size.
Dr. David Alan Gilberta82d5932015-11-05 18:11:09 +00002253 *
Peter Xuf3321552022-10-11 17:55:50 -04002254 * The caller must be with ram_state.bitmap_mutex held to call this
2255 * function. Note that this function can temporarily release the lock, but
2256 * when the function is returned it'll make sure the lock is still held.
2257 *
Juan Quintela3d0684b2017-03-23 15:06:39 +01002258 * Returns the number of pages written or negative on error
2259 *
Juan Quintela6f37bb82017-03-13 19:26:29 +01002260 * @rs: current RAM state
Juan Quintela3d0684b2017-03-23 15:06:39 +01002261 * @pss: data about the page we want to send
Dr. David Alan Gilberta82d5932015-11-05 18:11:09 +00002262 */
Juan Quintela05931ec2021-12-15 19:01:21 +01002263static int ram_save_host_page(RAMState *rs, PageSearchStatus *pss)
Dr. David Alan Gilberta82d5932015-11-05 18:11:09 +00002264{
Peter Xuf3321552022-10-11 17:55:50 -04002265 bool page_dirty, preempt_active = postcopy_preempt_active();
Dr. David Alan Gilberta82d5932015-11-05 18:11:09 +00002266 int tmppages, pages = 0;
Juan Quintelaa935e302017-03-21 15:36:51 +01002267 size_t pagesize_bits =
2268 qemu_ram_pagesize(pss->block) >> TARGET_PAGE_BITS;
Andrey Gruzdev278e2f52021-01-29 13:14:05 +03002269 unsigned long start_page = pss->page;
2270 int res;
Dr. David Alan Gilbert4c011c32017-02-24 18:28:39 +00002271
David Hildenbrandf161c882023-07-06 09:56:08 +02002272 if (migrate_ram_is_ignored(pss->block)) {
Cédric Le Goaterb895de52018-05-14 08:57:00 +02002273 error_report("block %s should not be migrated !", pss->block->idstr);
2274 return 0;
2275 }
2276
Peter Xud9e474e2022-10-11 17:55:52 -04002277 /* Update host page boundary information */
2278 pss_host_page_prepare(pss);
2279
Dr. David Alan Gilberta82d5932015-11-05 18:11:09 +00002280 do {
Peter Xuf3321552022-10-11 17:55:50 -04002281 page_dirty = migration_bitmap_clear_dirty(rs, pss->block, pss->page);
Xiao Guangrong1faa5662018-03-30 15:51:24 +08002282
Peter Xuf3321552022-10-11 17:55:50 -04002283 /* Check the pages is dirty and if it is send it */
2284 if (page_dirty) {
Kunkun Jiangba1b7c82021-03-16 20:57:16 +08002285 /*
Peter Xuf3321552022-10-11 17:55:50 -04002286 * Properly yield the lock only in postcopy preempt mode
2287 * because both migration thread and rp-return thread can
2288 * operate on the bitmaps.
Kunkun Jiangba1b7c82021-03-16 20:57:16 +08002289 */
Peter Xuf3321552022-10-11 17:55:50 -04002290 if (preempt_active) {
2291 qemu_mutex_unlock(&rs->bitmap_mutex);
Kunkun Jiangba1b7c82021-03-16 20:57:16 +08002292 }
Juan Quintela4010ba32021-12-15 20:10:39 +01002293 tmppages = migration_ops->ram_save_target_page(rs, pss);
Peter Xuf3321552022-10-11 17:55:50 -04002294 if (tmppages >= 0) {
2295 pages += tmppages;
2296 /*
2297 * Allow rate limiting to happen in the middle of huge pages if
2298 * something is sent in the current iteration.
2299 */
2300 if (pagesize_bits > 1 && tmppages > 0) {
2301 migration_rate_limit();
2302 }
2303 }
2304 if (preempt_active) {
2305 qemu_mutex_lock(&rs->bitmap_mutex);
2306 }
2307 } else {
2308 tmppages = 0;
Dr. David Alan Gilberta82d5932015-11-05 18:11:09 +00002309 }
Peter Xuf3321552022-10-11 17:55:50 -04002310
2311 if (tmppages < 0) {
Peter Xud9e474e2022-10-11 17:55:52 -04002312 pss_host_page_finish(pss);
Peter Xuf3321552022-10-11 17:55:50 -04002313 return tmppages;
2314 }
2315
Peter Xud9e474e2022-10-11 17:55:52 -04002316 pss_find_next_dirty(pss);
2317 } while (pss_within_range(pss));
2318
2319 pss_host_page_finish(pss);
Andrey Gruzdev278e2f52021-01-29 13:14:05 +03002320
2321 res = ram_save_release_protection(rs, pss, start_page);
2322 return (res < 0 ? res : pages);
Dr. David Alan Gilberta82d5932015-11-05 18:11:09 +00002323}
Dr. David Alan Gilbert6c595cd2015-11-05 18:11:08 +00002324
2325/**
Juan Quintela3d0684b2017-03-23 15:06:39 +01002326 * ram_find_and_save_block: finds a dirty page and sends it to f
Juan Quintela56e93d22015-05-07 19:33:31 +02002327 *
2328 * Called within an RCU critical section.
2329 *
Xiao Guangronge8f37352018-09-03 17:26:44 +08002330 * Returns the number of pages written where zero means no dirty pages,
2331 * or negative on error
Juan Quintela56e93d22015-05-07 19:33:31 +02002332 *
Juan Quintela6f37bb82017-03-13 19:26:29 +01002333 * @rs: current RAM state
Dr. David Alan Gilberta82d5932015-11-05 18:11:09 +00002334 *
2335 * On systems where host-page-size > target-page-size it will send all the
2336 * pages in a host page that are dirty.
Juan Quintela56e93d22015-05-07 19:33:31 +02002337 */
Juan Quintela05931ec2021-12-15 19:01:21 +01002338static int ram_find_and_save_block(RAMState *rs)
Juan Quintela56e93d22015-05-07 19:33:31 +02002339{
Peter Xuf1668762022-10-11 17:55:55 -04002340 PageSearchStatus *pss = &rs->pss[RAM_CHANNEL_PRECOPY];
Juan Quintela56e93d22015-05-07 19:33:31 +02002341 int pages = 0;
Juan Quintela56e93d22015-05-07 19:33:31 +02002342
Ashijeet Acharya0827b9e2017-02-08 19:58:45 +05302343 /* No dirty page as there is zero RAM */
Juan Quintela8d80e192022-05-10 19:37:36 +02002344 if (!rs->ram_bytes_total) {
Ashijeet Acharya0827b9e2017-02-08 19:58:45 +05302345 return pages;
2346 }
2347
Peter Xu4934a5d2022-10-04 14:24:26 -04002348 /*
2349 * Always keep last_seen_block/last_page valid during this procedure,
2350 * because find_dirty_block() relies on these values (e.g., we compare
2351 * last_seen_block with pss.block to see whether we searched all the
2352 * ramblocks) to detect the completion of migration. Having NULL value
2353 * of last_seen_block can conditionally cause below loop to run forever.
2354 */
2355 if (!rs->last_seen_block) {
2356 rs->last_seen_block = QLIST_FIRST_RCU(&ram_list.blocks);
2357 rs->last_page = 0;
2358 }
2359
Peter Xuf1668762022-10-11 17:55:55 -04002360 pss_init(pss, rs->last_seen_block, rs->last_page);
Dr. David Alan Gilbertb8fb8cb2015-09-23 15:27:10 +01002361
Juan Quintela31e2ac72022-06-21 13:29:36 +02002362 while (true){
Juan Quintela51efd362022-06-21 13:20:35 +02002363 if (!get_queued_page(rs, pss)) {
Peter Xub0621062022-10-11 17:55:58 -04002364 /* priority queue empty, so just search for something dirty */
Juan Quintela31e2ac72022-06-21 13:29:36 +02002365 int res = find_dirty_block(rs, pss);
2366 if (res != PAGE_DIRTY_FOUND) {
2367 if (res == PAGE_ALL_CLEAN) {
Juan Quintela51efd362022-06-21 13:20:35 +02002368 break;
Juan Quintela31e2ac72022-06-21 13:29:36 +02002369 } else if (res == PAGE_TRY_AGAIN) {
2370 continue;
Juan Quintela294e5a42022-06-21 13:36:11 +02002371 } else if (res < 0) {
2372 pages = res;
2373 break;
Juan Quintela51efd362022-06-21 13:20:35 +02002374 }
2375 }
Dr. David Alan Gilberta82d5932015-11-05 18:11:09 +00002376 }
Juan Quintela51efd362022-06-21 13:20:35 +02002377 pages = ram_save_host_page(rs, pss);
Juan Quintela31e2ac72022-06-21 13:29:36 +02002378 if (pages) {
2379 break;
2380 }
2381 }
Juan Quintela56e93d22015-05-07 19:33:31 +02002382
Peter Xuf1668762022-10-11 17:55:55 -04002383 rs->last_seen_block = pss->block;
2384 rs->last_page = pss->page;
Juan Quintela56e93d22015-05-07 19:33:31 +02002385
2386 return pages;
2387}
2388
Juan Quintela8008a272022-05-10 19:18:19 +02002389static uint64_t ram_bytes_total_with_ignored(void)
Juan Quintela56e93d22015-05-07 19:33:31 +02002390{
2391 RAMBlock *block;
2392 uint64_t total = 0;
2393
Dr. David Alan Gilbert89ac5a12019-10-07 15:36:39 +01002394 RCU_READ_LOCK_GUARD();
2395
Juan Quintela8008a272022-05-10 19:18:19 +02002396 RAMBLOCK_FOREACH_MIGRATABLE(block) {
2397 total += block->used_length;
Peter Xu99e15582017-05-12 12:17:39 +08002398 }
Juan Quintela56e93d22015-05-07 19:33:31 +02002399 return total;
2400}
2401
Yury Kotovfbd162e2019-02-15 20:45:46 +03002402uint64_t ram_bytes_total(void)
2403{
Juan Quintela8008a272022-05-10 19:18:19 +02002404 RAMBlock *block;
2405 uint64_t total = 0;
2406
2407 RCU_READ_LOCK_GUARD();
2408
2409 RAMBLOCK_FOREACH_NOT_IGNORED(block) {
2410 total += block->used_length;
2411 }
2412 return total;
Yury Kotovfbd162e2019-02-15 20:45:46 +03002413}
2414
Juan Quintelaf265e0e2017-06-28 11:52:27 +02002415static void xbzrle_load_setup(void)
Juan Quintela56e93d22015-05-07 19:33:31 +02002416{
Juan Quintelaf265e0e2017-06-28 11:52:27 +02002417 XBZRLE.decoded_buf = g_malloc(TARGET_PAGE_SIZE);
Juan Quintela56e93d22015-05-07 19:33:31 +02002418}
2419
Juan Quintelaf265e0e2017-06-28 11:52:27 +02002420static void xbzrle_load_cleanup(void)
2421{
2422 g_free(XBZRLE.decoded_buf);
2423 XBZRLE.decoded_buf = NULL;
2424}
2425
Peter Xu7d7c96b2017-10-19 14:31:58 +08002426static void ram_state_cleanup(RAMState **rsp)
2427{
Dr. David Alan Gilbertb9ccaf62018-02-12 16:03:39 +00002428 if (*rsp) {
2429 migration_page_queue_free(*rsp);
2430 qemu_mutex_destroy(&(*rsp)->bitmap_mutex);
2431 qemu_mutex_destroy(&(*rsp)->src_page_req_mutex);
2432 g_free(*rsp);
2433 *rsp = NULL;
2434 }
Peter Xu7d7c96b2017-10-19 14:31:58 +08002435}
2436
Peter Xu84593a02017-10-19 14:31:59 +08002437static void xbzrle_cleanup(void)
2438{
2439 XBZRLE_cache_lock();
2440 if (XBZRLE.cache) {
2441 cache_fini(XBZRLE.cache);
2442 g_free(XBZRLE.encoded_buf);
2443 g_free(XBZRLE.current_buf);
2444 g_free(XBZRLE.zero_target_page);
2445 XBZRLE.cache = NULL;
2446 XBZRLE.encoded_buf = NULL;
2447 XBZRLE.current_buf = NULL;
2448 XBZRLE.zero_target_page = NULL;
2449 }
2450 XBZRLE_cache_unlock();
2451}
2452
Juan Quintelaf265e0e2017-06-28 11:52:27 +02002453static void ram_save_cleanup(void *opaque)
Juan Quintela56e93d22015-05-07 19:33:31 +02002454{
Juan Quintela53518d92017-05-04 11:46:24 +02002455 RAMState **rsp = opaque;
Juan Quintela6b6712e2017-03-22 15:18:04 +01002456 RAMBlock *block;
Juan Quintelaeb859c52017-03-13 21:51:55 +01002457
Andrey Gruzdev278e2f52021-01-29 13:14:05 +03002458 /* We don't use dirty log with background snapshots */
2459 if (!migrate_background_snapshot()) {
2460 /* caller have hold iothread lock or is in a bh, so there is
2461 * no writing race against the migration bitmap
2462 */
Hyman Huang(黄勇)63b41db2021-06-29 16:01:19 +00002463 if (global_dirty_tracking & GLOBAL_DIRTY_MIGRATION) {
2464 /*
2465 * do not stop dirty log without starting it, since
2466 * memory_global_dirty_log_stop will assert that
2467 * memory_global_dirty_log_start/stop used in pairs
2468 */
2469 memory_global_dirty_log_stop(GLOBAL_DIRTY_MIGRATION);
2470 }
Andrey Gruzdev278e2f52021-01-29 13:14:05 +03002471 }
Juan Quintela6b6712e2017-03-22 15:18:04 +01002472
Yury Kotovfbd162e2019-02-15 20:45:46 +03002473 RAMBLOCK_FOREACH_NOT_IGNORED(block) {
Peter Xu002cad62019-06-03 14:50:56 +08002474 g_free(block->clear_bmap);
2475 block->clear_bmap = NULL;
Juan Quintela6b6712e2017-03-22 15:18:04 +01002476 g_free(block->bmap);
2477 block->bmap = NULL;
Juan Quintela56e93d22015-05-07 19:33:31 +02002478 }
2479
Peter Xu84593a02017-10-19 14:31:59 +08002480 xbzrle_cleanup();
Juan Quintelaf0afa332017-06-28 11:52:28 +02002481 compress_threads_save_cleanup();
Peter Xu7d7c96b2017-10-19 14:31:58 +08002482 ram_state_cleanup(rsp);
Juan Quintela4010ba32021-12-15 20:10:39 +01002483 g_free(migration_ops);
2484 migration_ops = NULL;
Juan Quintela56e93d22015-05-07 19:33:31 +02002485}
2486
Juan Quintela6f37bb82017-03-13 19:26:29 +01002487static void ram_state_reset(RAMState *rs)
Juan Quintela56e93d22015-05-07 19:33:31 +02002488{
Peter Xuec6f3ab2022-10-11 17:55:56 -04002489 int i;
2490
2491 for (i = 0; i < RAM_CHANNEL_MAX; i++) {
2492 rs->pss[i].last_sent_block = NULL;
2493 }
2494
Juan Quintela6f37bb82017-03-13 19:26:29 +01002495 rs->last_seen_block = NULL;
Juan Quintela269ace22017-03-21 15:23:31 +01002496 rs->last_page = 0;
Juan Quintela6f37bb82017-03-13 19:26:29 +01002497 rs->last_version = ram_list.version;
Juan Quintelaf3095cc2023-05-04 13:53:23 +02002498 rs->xbzrle_started = false;
Juan Quintela56e93d22015-05-07 19:33:31 +02002499}
2500
2501#define MAX_WAIT 50 /* ms, half buffered_file limit */
2502
Dr. David Alan Gilberte0b266f2015-11-05 18:11:02 +00002503/* **** functions for postcopy ***** */
2504
Pavel Butsykinced1c612017-02-03 18:23:21 +03002505void ram_postcopy_migrated_memory_release(MigrationState *ms)
2506{
2507 struct RAMBlock *block;
Pavel Butsykinced1c612017-02-03 18:23:21 +03002508
Yury Kotovfbd162e2019-02-15 20:45:46 +03002509 RAMBLOCK_FOREACH_NOT_IGNORED(block) {
Juan Quintela6b6712e2017-03-22 15:18:04 +01002510 unsigned long *bitmap = block->bmap;
2511 unsigned long range = block->used_length >> TARGET_PAGE_BITS;
2512 unsigned long run_start = find_next_zero_bit(bitmap, range, 0);
Pavel Butsykinced1c612017-02-03 18:23:21 +03002513
2514 while (run_start < range) {
2515 unsigned long run_end = find_next_bit(bitmap, range, run_start + 1);
Alexey Romko8bba0042020-01-10 14:51:34 +01002516 ram_discard_range(block->idstr,
2517 ((ram_addr_t)run_start) << TARGET_PAGE_BITS,
2518 ((ram_addr_t)(run_end - run_start))
2519 << TARGET_PAGE_BITS);
Pavel Butsykinced1c612017-02-03 18:23:21 +03002520 run_start = find_next_zero_bit(bitmap, range, run_end + 1);
2521 }
2522 }
2523}
2524
Juan Quintela3d0684b2017-03-23 15:06:39 +01002525/**
2526 * postcopy_send_discard_bm_ram: discard a RAMBlock
2527 *
Dr. David Alan Gilberte0b266f2015-11-05 18:11:02 +00002528 * Callback from postcopy_each_ram_send_discard for each RAMBlock
Juan Quintela3d0684b2017-03-23 15:06:39 +01002529 *
2530 * @ms: current migration state
Wei Yang89dab312019-07-15 10:05:49 +08002531 * @block: RAMBlock to discard
Dr. David Alan Gilberte0b266f2015-11-05 18:11:02 +00002532 */
Philippe Mathieu-Daudé9e7d1222021-12-30 17:05:25 +01002533static void postcopy_send_discard_bm_ram(MigrationState *ms, RAMBlock *block)
Dr. David Alan Gilberte0b266f2015-11-05 18:11:02 +00002534{
Juan Quintela6b6712e2017-03-22 15:18:04 +01002535 unsigned long end = block->used_length >> TARGET_PAGE_BITS;
Dr. David Alan Gilberte0b266f2015-11-05 18:11:02 +00002536 unsigned long current;
Wei Yang1e7cf8c2019-08-19 14:18:42 +08002537 unsigned long *bitmap = block->bmap;
Dr. David Alan Gilberte0b266f2015-11-05 18:11:02 +00002538
Juan Quintela6b6712e2017-03-22 15:18:04 +01002539 for (current = 0; current < end; ) {
Wei Yang1e7cf8c2019-08-19 14:18:42 +08002540 unsigned long one = find_next_bit(bitmap, end, current);
Wei Yang33a5cb622019-06-27 10:08:21 +08002541 unsigned long zero, discard_length;
Dr. David Alan Gilberte0b266f2015-11-05 18:11:02 +00002542
Wei Yang33a5cb622019-06-27 10:08:21 +08002543 if (one >= end) {
2544 break;
Dr. David Alan Gilberte0b266f2015-11-05 18:11:02 +00002545 }
Wei Yang33a5cb622019-06-27 10:08:21 +08002546
Wei Yang1e7cf8c2019-08-19 14:18:42 +08002547 zero = find_next_zero_bit(bitmap, end, one + 1);
Wei Yang33a5cb622019-06-27 10:08:21 +08002548
2549 if (zero >= end) {
2550 discard_length = end - one;
2551 } else {
2552 discard_length = zero - one;
2553 }
Wei Yang810cf2b2019-07-24 09:07:21 +08002554 postcopy_discard_send_range(ms, one, discard_length);
Wei Yang33a5cb622019-06-27 10:08:21 +08002555 current = one + discard_length;
Dr. David Alan Gilberte0b266f2015-11-05 18:11:02 +00002556 }
Dr. David Alan Gilberte0b266f2015-11-05 18:11:02 +00002557}
2558
Peter Xuf30c2e52021-12-07 19:50:13 +08002559static void postcopy_chunk_hostpages_pass(MigrationState *ms, RAMBlock *block);
2560
Juan Quintela3d0684b2017-03-23 15:06:39 +01002561/**
2562 * postcopy_each_ram_send_discard: discard all RAMBlocks
2563 *
Dr. David Alan Gilberte0b266f2015-11-05 18:11:02 +00002564 * Utility for the outgoing postcopy code.
2565 * Calls postcopy_send_discard_bm_ram for each RAMBlock
2566 * passing it bitmap indexes and name.
Dr. David Alan Gilberte0b266f2015-11-05 18:11:02 +00002567 * (qemu_ram_foreach_block ends up passing unscaled lengths
2568 * which would mean postcopy code would have to deal with target page)
Juan Quintela3d0684b2017-03-23 15:06:39 +01002569 *
2570 * @ms: current migration state
Dr. David Alan Gilberte0b266f2015-11-05 18:11:02 +00002571 */
Peter Xu739fcc12021-12-07 19:50:14 +08002572static void postcopy_each_ram_send_discard(MigrationState *ms)
Dr. David Alan Gilberte0b266f2015-11-05 18:11:02 +00002573{
2574 struct RAMBlock *block;
Dr. David Alan Gilberte0b266f2015-11-05 18:11:02 +00002575
Yury Kotovfbd162e2019-02-15 20:45:46 +03002576 RAMBLOCK_FOREACH_NOT_IGNORED(block) {
Wei Yang810cf2b2019-07-24 09:07:21 +08002577 postcopy_discard_send_init(ms, block->idstr);
Dr. David Alan Gilberte0b266f2015-11-05 18:11:02 +00002578
2579 /*
Peter Xuf30c2e52021-12-07 19:50:13 +08002580 * Deal with TPS != HPS and huge pages. It discard any partially sent
2581 * host-page size chunks, mark any partially dirty host-page size
2582 * chunks as all dirty. In this case the host-page is the host-page
2583 * for the particular RAMBlock, i.e. it might be a huge page.
2584 */
2585 postcopy_chunk_hostpages_pass(ms, block);
2586
2587 /*
Dr. David Alan Gilberte0b266f2015-11-05 18:11:02 +00002588 * Postcopy sends chunks of bitmap over the wire, but it
2589 * just needs indexes at this point, avoids it having
2590 * target page specific code.
2591 */
Peter Xu739fcc12021-12-07 19:50:14 +08002592 postcopy_send_discard_bm_ram(ms, block);
Wei Yang810cf2b2019-07-24 09:07:21 +08002593 postcopy_discard_send_finish(ms);
Dr. David Alan Gilberte0b266f2015-11-05 18:11:02 +00002594 }
Dr. David Alan Gilberte0b266f2015-11-05 18:11:02 +00002595}
2596
Juan Quintela3d0684b2017-03-23 15:06:39 +01002597/**
Wei Yang8324ef82019-08-19 14:18:41 +08002598 * postcopy_chunk_hostpages_pass: canonicalize bitmap in hostpages
Dr. David Alan Gilbert99e314e2015-11-05 18:11:15 +00002599 *
Juan Quintela3d0684b2017-03-23 15:06:39 +01002600 * Helper for postcopy_chunk_hostpages; it's called twice to
2601 * canonicalize the two bitmaps, that are similar, but one is
2602 * inverted.
Dr. David Alan Gilbert99e314e2015-11-05 18:11:15 +00002603 *
Juan Quintela3d0684b2017-03-23 15:06:39 +01002604 * Postcopy requires that all target pages in a hostpage are dirty or
2605 * clean, not a mix. This function canonicalizes the bitmaps.
2606 *
2607 * @ms: current migration state
Juan Quintela3d0684b2017-03-23 15:06:39 +01002608 * @block: block that contains the page we want to canonicalize
Dr. David Alan Gilbert99e314e2015-11-05 18:11:15 +00002609 */
Wei Yang1e7cf8c2019-08-19 14:18:42 +08002610static void postcopy_chunk_hostpages_pass(MigrationState *ms, RAMBlock *block)
Dr. David Alan Gilbert99e314e2015-11-05 18:11:15 +00002611{
Juan Quintela53518d92017-05-04 11:46:24 +02002612 RAMState *rs = ram_state;
Juan Quintela6b6712e2017-03-22 15:18:04 +01002613 unsigned long *bitmap = block->bmap;
Dr. David Alan Gilbert29c59172017-02-24 18:28:31 +00002614 unsigned int host_ratio = block->page_size / TARGET_PAGE_SIZE;
Juan Quintela6b6712e2017-03-22 15:18:04 +01002615 unsigned long pages = block->used_length >> TARGET_PAGE_BITS;
Dr. David Alan Gilbert99e314e2015-11-05 18:11:15 +00002616 unsigned long run_start;
2617
Dr. David Alan Gilbert29c59172017-02-24 18:28:31 +00002618 if (block->page_size == TARGET_PAGE_SIZE) {
2619 /* Easy case - TPS==HPS for a non-huge page RAMBlock */
2620 return;
2621 }
2622
Wei Yang1e7cf8c2019-08-19 14:18:42 +08002623 /* Find a dirty page */
2624 run_start = find_next_bit(bitmap, pages, 0);
Dr. David Alan Gilbert99e314e2015-11-05 18:11:15 +00002625
Juan Quintela6b6712e2017-03-22 15:18:04 +01002626 while (run_start < pages) {
Dr. David Alan Gilbert99e314e2015-11-05 18:11:15 +00002627
2628 /*
2629 * If the start of this run of pages is in the middle of a host
2630 * page, then we need to fixup this host page.
2631 */
Wei Yang9dec3cc2019-08-06 08:46:48 +08002632 if (QEMU_IS_ALIGNED(run_start, host_ratio)) {
Dr. David Alan Gilbert99e314e2015-11-05 18:11:15 +00002633 /* Find the end of this run */
Wei Yang1e7cf8c2019-08-19 14:18:42 +08002634 run_start = find_next_zero_bit(bitmap, pages, run_start + 1);
Dr. David Alan Gilbert99e314e2015-11-05 18:11:15 +00002635 /*
2636 * If the end isn't at the start of a host page, then the
2637 * run doesn't finish at the end of a host page
2638 * and we need to discard.
2639 */
Dr. David Alan Gilbert99e314e2015-11-05 18:11:15 +00002640 }
2641
Wei Yang9dec3cc2019-08-06 08:46:48 +08002642 if (!QEMU_IS_ALIGNED(run_start, host_ratio)) {
Dr. David Alan Gilbert99e314e2015-11-05 18:11:15 +00002643 unsigned long page;
Wei Yangdad45ab2019-08-06 08:46:47 +08002644 unsigned long fixup_start_addr = QEMU_ALIGN_DOWN(run_start,
2645 host_ratio);
2646 run_start = QEMU_ALIGN_UP(run_start, host_ratio);
Dr. David Alan Gilbert99e314e2015-11-05 18:11:15 +00002647
Dr. David Alan Gilbert99e314e2015-11-05 18:11:15 +00002648 /* Clean up the bitmap */
2649 for (page = fixup_start_addr;
2650 page < fixup_start_addr + host_ratio; page++) {
Dr. David Alan Gilbert99e314e2015-11-05 18:11:15 +00002651 /*
2652 * Remark them as dirty, updating the count for any pages
2653 * that weren't previously dirty.
2654 */
Juan Quintela0d8ec882017-03-13 21:21:41 +01002655 rs->migration_dirty_pages += !test_and_set_bit(page, bitmap);
Dr. David Alan Gilbert99e314e2015-11-05 18:11:15 +00002656 }
2657 }
2658
Wei Yang1e7cf8c2019-08-19 14:18:42 +08002659 /* Find the next dirty page for the next iteration */
2660 run_start = find_next_bit(bitmap, pages, run_start);
Dr. David Alan Gilbert99e314e2015-11-05 18:11:15 +00002661 }
2662}
2663
Juan Quintela3d0684b2017-03-23 15:06:39 +01002664/**
Juan Quintela3d0684b2017-03-23 15:06:39 +01002665 * ram_postcopy_send_discard_bitmap: transmit the discard bitmap
2666 *
Dr. David Alan Gilberte0b266f2015-11-05 18:11:02 +00002667 * Transmit the set of pages to be discarded after precopy to the target
2668 * these are pages that:
2669 * a) Have been previously transmitted but are now dirty again
2670 * b) Pages that have never been transmitted, this ensures that
2671 * any pages on the destination that have been mapped by background
2672 * tasks get discarded (transparent huge pages is the specific concern)
2673 * Hopefully this is pretty sparse
Juan Quintela3d0684b2017-03-23 15:06:39 +01002674 *
2675 * @ms: current migration state
Dr. David Alan Gilberte0b266f2015-11-05 18:11:02 +00002676 */
Peter Xu739fcc12021-12-07 19:50:14 +08002677void ram_postcopy_send_discard_bitmap(MigrationState *ms)
Dr. David Alan Gilberte0b266f2015-11-05 18:11:02 +00002678{
Juan Quintela53518d92017-05-04 11:46:24 +02002679 RAMState *rs = ram_state;
Dr. David Alan Gilberte0b266f2015-11-05 18:11:02 +00002680
Dr. David Alan Gilbert89ac5a12019-10-07 15:36:39 +01002681 RCU_READ_LOCK_GUARD();
Dr. David Alan Gilberte0b266f2015-11-05 18:11:02 +00002682
2683 /* This should be our last sync, the src is now paused */
Gavin Shan1e493be2023-05-09 12:21:19 +10002684 migration_bitmap_sync(rs, false);
Dr. David Alan Gilberte0b266f2015-11-05 18:11:02 +00002685
Juan Quintela6b6712e2017-03-22 15:18:04 +01002686 /* Easiest way to make sure we don't resume in the middle of a host-page */
Peter Xuec6f3ab2022-10-11 17:55:56 -04002687 rs->pss[RAM_CHANNEL_PRECOPY].last_sent_block = NULL;
Juan Quintela6b6712e2017-03-22 15:18:04 +01002688 rs->last_seen_block = NULL;
Juan Quintela6b6712e2017-03-22 15:18:04 +01002689 rs->last_page = 0;
2690
Peter Xu739fcc12021-12-07 19:50:14 +08002691 postcopy_each_ram_send_discard(ms);
Dr. David Alan Gilberte0b266f2015-11-05 18:11:02 +00002692
Peter Xu739fcc12021-12-07 19:50:14 +08002693 trace_ram_postcopy_send_discard_bitmap();
Dr. David Alan Gilberte0b266f2015-11-05 18:11:02 +00002694}
2695
Juan Quintela3d0684b2017-03-23 15:06:39 +01002696/**
2697 * ram_discard_range: discard dirtied pages at the beginning of postcopy
Dr. David Alan Gilberte0b266f2015-11-05 18:11:02 +00002698 *
Juan Quintela3d0684b2017-03-23 15:06:39 +01002699 * Returns zero on success
Dr. David Alan Gilberte0b266f2015-11-05 18:11:02 +00002700 *
Juan Quintela36449152017-03-23 15:11:59 +01002701 * @rbname: name of the RAMBlock of the request. NULL means the
2702 * same that last one.
Juan Quintela3d0684b2017-03-23 15:06:39 +01002703 * @start: RAMBlock starting page
2704 * @length: RAMBlock size
Dr. David Alan Gilberte0b266f2015-11-05 18:11:02 +00002705 */
Juan Quintelaaaa20642017-03-21 11:35:24 +01002706int ram_discard_range(const char *rbname, uint64_t start, size_t length)
Dr. David Alan Gilberte0b266f2015-11-05 18:11:02 +00002707{
Juan Quintela36449152017-03-23 15:11:59 +01002708 trace_ram_discard_range(rbname, start, length);
Dr. David Alan Gilbertd3a50382017-02-24 18:28:32 +00002709
Dr. David Alan Gilbert89ac5a12019-10-07 15:36:39 +01002710 RCU_READ_LOCK_GUARD();
Juan Quintela36449152017-03-23 15:11:59 +01002711 RAMBlock *rb = qemu_ram_block_by_name(rbname);
Dr. David Alan Gilberte0b266f2015-11-05 18:11:02 +00002712
2713 if (!rb) {
Juan Quintela36449152017-03-23 15:11:59 +01002714 error_report("ram_discard_range: Failed to find block '%s'", rbname);
Daniel Henrique Barboza03acb4e2020-01-06 15:23:31 -03002715 return -1;
Dr. David Alan Gilberte0b266f2015-11-05 18:11:02 +00002716 }
2717
Peter Xu814bb082018-07-23 20:33:02 +08002718 /*
2719 * On source VM, we don't need to update the received bitmap since
2720 * we don't even have one.
2721 */
2722 if (rb->receivedmap) {
2723 bitmap_clear(rb->receivedmap, start >> qemu_target_page_bits(),
2724 length >> qemu_target_page_bits());
2725 }
2726
Daniel Henrique Barboza03acb4e2020-01-06 15:23:31 -03002727 return ram_block_discard_range(rb, start, length);
Dr. David Alan Gilberte0b266f2015-11-05 18:11:02 +00002728}
2729
Peter Xu84593a02017-10-19 14:31:59 +08002730/*
2731 * For every allocation, we will try not to crash the VM if the
2732 * allocation failed.
2733 */
2734static int xbzrle_init(void)
2735{
2736 Error *local_err = NULL;
2737
Juan Quintela87dca0c2023-03-01 22:20:13 +01002738 if (!migrate_xbzrle()) {
Peter Xu84593a02017-10-19 14:31:59 +08002739 return 0;
2740 }
2741
2742 XBZRLE_cache_lock();
2743
2744 XBZRLE.zero_target_page = g_try_malloc0(TARGET_PAGE_SIZE);
2745 if (!XBZRLE.zero_target_page) {
2746 error_report("%s: Error allocating zero page", __func__);
2747 goto err_out;
2748 }
2749
2750 XBZRLE.cache = cache_init(migrate_xbzrle_cache_size(),
2751 TARGET_PAGE_SIZE, &local_err);
2752 if (!XBZRLE.cache) {
2753 error_report_err(local_err);
2754 goto free_zero_page;
2755 }
2756
2757 XBZRLE.encoded_buf = g_try_malloc0(TARGET_PAGE_SIZE);
2758 if (!XBZRLE.encoded_buf) {
2759 error_report("%s: Error allocating encoded_buf", __func__);
2760 goto free_cache;
2761 }
2762
2763 XBZRLE.current_buf = g_try_malloc(TARGET_PAGE_SIZE);
2764 if (!XBZRLE.current_buf) {
2765 error_report("%s: Error allocating current_buf", __func__);
2766 goto free_encoded_buf;
2767 }
2768
2769 /* We are all good */
2770 XBZRLE_cache_unlock();
2771 return 0;
2772
2773free_encoded_buf:
2774 g_free(XBZRLE.encoded_buf);
2775 XBZRLE.encoded_buf = NULL;
2776free_cache:
2777 cache_fini(XBZRLE.cache);
2778 XBZRLE.cache = NULL;
2779free_zero_page:
2780 g_free(XBZRLE.zero_target_page);
2781 XBZRLE.zero_target_page = NULL;
2782err_out:
2783 XBZRLE_cache_unlock();
2784 return -ENOMEM;
2785}
2786
Juan Quintela53518d92017-05-04 11:46:24 +02002787static int ram_state_init(RAMState **rsp)
Juan Quintela56e93d22015-05-07 19:33:31 +02002788{
Peter Xu7d00ee62017-10-19 14:31:57 +08002789 *rsp = g_try_new0(RAMState, 1);
2790
2791 if (!*rsp) {
2792 error_report("%s: Init ramstate fail", __func__);
2793 return -1;
2794 }
Juan Quintela53518d92017-05-04 11:46:24 +02002795
2796 qemu_mutex_init(&(*rsp)->bitmap_mutex);
2797 qemu_mutex_init(&(*rsp)->src_page_req_mutex);
2798 QSIMPLEQ_INIT(&(*rsp)->src_page_requests);
Juan Quintela8d80e192022-05-10 19:37:36 +02002799 (*rsp)->ram_bytes_total = ram_bytes_total();
Juan Quintela56e93d22015-05-07 19:33:31 +02002800
Peter Xu7d00ee62017-10-19 14:31:57 +08002801 /*
Ivan Ren40c4d4a2019-07-14 22:51:19 +08002802 * Count the total number of pages used by ram blocks not including any
2803 * gaps due to alignment or unplugs.
Wei Yang03158512019-06-04 14:17:27 +08002804 * This must match with the initial values of dirty bitmap.
Peter Xu7d00ee62017-10-19 14:31:57 +08002805 */
Juan Quintela8d80e192022-05-10 19:37:36 +02002806 (*rsp)->migration_dirty_pages = (*rsp)->ram_bytes_total >> TARGET_PAGE_BITS;
Peter Xu7d00ee62017-10-19 14:31:57 +08002807 ram_state_reset(*rsp);
2808
2809 return 0;
2810}
2811
Peter Xud6eff5d2017-10-19 14:32:00 +08002812static void ram_list_init_bitmaps(void)
2813{
Peter Xu002cad62019-06-03 14:50:56 +08002814 MigrationState *ms = migrate_get_current();
Peter Xud6eff5d2017-10-19 14:32:00 +08002815 RAMBlock *block;
2816 unsigned long pages;
Peter Xu002cad62019-06-03 14:50:56 +08002817 uint8_t shift;
Peter Xud6eff5d2017-10-19 14:32:00 +08002818
2819 /* Skip setting bitmap if there is no RAM */
2820 if (ram_bytes_total()) {
Peter Xu002cad62019-06-03 14:50:56 +08002821 shift = ms->clear_bitmap_shift;
2822 if (shift > CLEAR_BITMAP_SHIFT_MAX) {
2823 error_report("clear_bitmap_shift (%u) too big, using "
2824 "max value (%u)", shift, CLEAR_BITMAP_SHIFT_MAX);
2825 shift = CLEAR_BITMAP_SHIFT_MAX;
2826 } else if (shift < CLEAR_BITMAP_SHIFT_MIN) {
2827 error_report("clear_bitmap_shift (%u) too small, using "
2828 "min value (%u)", shift, CLEAR_BITMAP_SHIFT_MIN);
2829 shift = CLEAR_BITMAP_SHIFT_MIN;
2830 }
2831
Yury Kotovfbd162e2019-02-15 20:45:46 +03002832 RAMBLOCK_FOREACH_NOT_IGNORED(block) {
Peter Xud6eff5d2017-10-19 14:32:00 +08002833 pages = block->max_length >> TARGET_PAGE_BITS;
Wei Yang03158512019-06-04 14:17:27 +08002834 /*
2835 * The initial dirty bitmap for migration must be set with all
2836 * ones to make sure we'll migrate every guest RAM page to
2837 * destination.
Ivan Ren40c4d4a2019-07-14 22:51:19 +08002838 * Here we set RAMBlock.bmap all to 1 because when rebegin a
2839 * new migration after a failed migration, ram_list.
2840 * dirty_memory[DIRTY_MEMORY_MIGRATION] don't include the whole
2841 * guest memory.
Wei Yang03158512019-06-04 14:17:27 +08002842 */
Peter Xud6eff5d2017-10-19 14:32:00 +08002843 block->bmap = bitmap_new(pages);
Ivan Ren40c4d4a2019-07-14 22:51:19 +08002844 bitmap_set(block->bmap, 0, pages);
Peter Xu002cad62019-06-03 14:50:56 +08002845 block->clear_bmap_shift = shift;
2846 block->clear_bmap = bitmap_new(clear_bmap_size(pages, shift));
Peter Xud6eff5d2017-10-19 14:32:00 +08002847 }
2848 }
2849}
2850
David Hildenbrandbe39b4c2021-10-11 19:53:41 +02002851static void migration_bitmap_clear_discarded_pages(RAMState *rs)
2852{
2853 unsigned long pages;
2854 RAMBlock *rb;
2855
2856 RCU_READ_LOCK_GUARD();
2857
2858 RAMBLOCK_FOREACH_NOT_IGNORED(rb) {
2859 pages = ramblock_dirty_bitmap_clear_discarded_pages(rb);
2860 rs->migration_dirty_pages -= pages;
2861 }
2862}
2863
Peter Xud6eff5d2017-10-19 14:32:00 +08002864static void ram_init_bitmaps(RAMState *rs)
2865{
Peter Xud6eff5d2017-10-19 14:32:00 +08002866 qemu_mutex_lock_ramlist();
Peter Xud6eff5d2017-10-19 14:32:00 +08002867
Dr. David Alan Gilbert89ac5a12019-10-07 15:36:39 +01002868 WITH_RCU_READ_LOCK_GUARD() {
2869 ram_list_init_bitmaps();
Andrey Gruzdev278e2f52021-01-29 13:14:05 +03002870 /* We don't use dirty log with background snapshots */
2871 if (!migrate_background_snapshot()) {
Hyman Huang(黄勇)63b41db2021-06-29 16:01:19 +00002872 memory_global_dirty_log_start(GLOBAL_DIRTY_MIGRATION);
Gavin Shan1e493be2023-05-09 12:21:19 +10002873 migration_bitmap_sync_precopy(rs, false);
Andrey Gruzdev278e2f52021-01-29 13:14:05 +03002874 }
Dr. David Alan Gilbert89ac5a12019-10-07 15:36:39 +01002875 }
Peter Xud6eff5d2017-10-19 14:32:00 +08002876 qemu_mutex_unlock_ramlist();
David Hildenbrandbe39b4c2021-10-11 19:53:41 +02002877
2878 /*
2879 * After an eventual first bitmap sync, fixup the initial bitmap
2880 * containing all 1s to exclude any discarded pages from migration.
2881 */
2882 migration_bitmap_clear_discarded_pages(rs);
Peter Xud6eff5d2017-10-19 14:32:00 +08002883}
2884
Peter Xu7d00ee62017-10-19 14:31:57 +08002885static int ram_init_all(RAMState **rsp)
2886{
Peter Xu7d00ee62017-10-19 14:31:57 +08002887 if (ram_state_init(rsp)) {
2888 return -1;
2889 }
2890
Peter Xu84593a02017-10-19 14:31:59 +08002891 if (xbzrle_init()) {
2892 ram_state_cleanup(rsp);
2893 return -1;
Juan Quintela56e93d22015-05-07 19:33:31 +02002894 }
2895
Peter Xud6eff5d2017-10-19 14:32:00 +08002896 ram_init_bitmaps(*rsp);
zhanghailianga91246c2016-10-27 14:42:59 +08002897
2898 return 0;
2899}
2900
Peter Xu08614f32018-05-02 18:47:33 +08002901static void ram_state_resume_prepare(RAMState *rs, QEMUFile *out)
2902{
2903 RAMBlock *block;
2904 uint64_t pages = 0;
2905
2906 /*
2907 * Postcopy is not using xbzrle/compression, so no need for that.
2908 * Also, since source are already halted, we don't need to care
2909 * about dirty page logging as well.
2910 */
2911
Yury Kotovfbd162e2019-02-15 20:45:46 +03002912 RAMBLOCK_FOREACH_NOT_IGNORED(block) {
Peter Xu08614f32018-05-02 18:47:33 +08002913 pages += bitmap_count_one(block->bmap,
2914 block->used_length >> TARGET_PAGE_BITS);
2915 }
2916
2917 /* This may not be aligned with current bitmaps. Recalculate. */
2918 rs->migration_dirty_pages = pages;
2919
David Hildenbrand1a373522021-02-16 11:50:39 +01002920 ram_state_reset(rs);
Peter Xu08614f32018-05-02 18:47:33 +08002921
2922 /* Update RAMState cache of output QEMUFile */
Peter Xu7f401b82022-10-11 17:55:59 -04002923 rs->pss[RAM_CHANNEL_PRECOPY].pss_channel = out;
Peter Xu08614f32018-05-02 18:47:33 +08002924
2925 trace_ram_state_resume_prepare(pages);
2926}
2927
Juan Quintela3d0684b2017-03-23 15:06:39 +01002928/*
Wei Wang6bcb05f2018-12-11 16:24:50 +08002929 * This function clears bits of the free pages reported by the caller from the
2930 * migration dirty bitmap. @addr is the host address corresponding to the
2931 * start of the continuous guest free pages, and @len is the total bytes of
2932 * those pages.
2933 */
2934void qemu_guest_free_page_hint(void *addr, size_t len)
2935{
2936 RAMBlock *block;
2937 ram_addr_t offset;
2938 size_t used_len, start, npages;
2939 MigrationState *s = migrate_get_current();
2940
2941 /* This function is currently expected to be used during live migration */
2942 if (!migration_is_setup_or_active(s->state)) {
2943 return;
2944 }
2945
2946 for (; len > 0; len -= used_len, addr += used_len) {
2947 block = qemu_ram_block_from_host(addr, false, &offset);
2948 if (unlikely(!block || offset >= block->used_length)) {
2949 /*
2950 * The implementation might not support RAMBlock resize during
2951 * live migration, but it could happen in theory with future
2952 * updates. So we add a check here to capture that case.
2953 */
2954 error_report_once("%s unexpected error", __func__);
2955 return;
2956 }
2957
2958 if (len <= block->used_length - offset) {
2959 used_len = len;
2960 } else {
2961 used_len = block->used_length - offset;
2962 }
2963
2964 start = offset >> TARGET_PAGE_BITS;
2965 npages = used_len >> TARGET_PAGE_BITS;
2966
2967 qemu_mutex_lock(&ram_state->bitmap_mutex);
Wei Wang3143577d2021-07-22 04:30:55 -04002968 /*
2969 * The skipped free pages are equavalent to be sent from clear_bmap's
2970 * perspective, so clear the bits from the memory region bitmap which
2971 * are initially set. Otherwise those skipped pages will be sent in
2972 * the next round after syncing from the memory region bitmap.
2973 */
David Hildenbrand1230a252021-09-04 18:09:07 +02002974 migration_clear_memory_region_dirty_bitmap_range(block, start, npages);
Wei Wang6bcb05f2018-12-11 16:24:50 +08002975 ram_state->migration_dirty_pages -=
2976 bitmap_count_one_with_offset(block->bmap, start, npages);
2977 bitmap_clear(block->bmap, start, npages);
2978 qemu_mutex_unlock(&ram_state->bitmap_mutex);
2979 }
2980}
2981
2982/*
Juan Quintela3d0684b2017-03-23 15:06:39 +01002983 * Each of ram_save_setup, ram_save_iterate and ram_save_complete has
zhanghailianga91246c2016-10-27 14:42:59 +08002984 * long-running RCU critical section. When rcu-reclaims in the code
2985 * start to become numerous it will be necessary to reduce the
2986 * granularity of these critical sections.
2987 */
2988
Juan Quintela3d0684b2017-03-23 15:06:39 +01002989/**
2990 * ram_save_setup: Setup RAM for migration
2991 *
2992 * Returns zero to indicate success and negative for error
2993 *
2994 * @f: QEMUFile where to send the data
2995 * @opaque: RAMState pointer
2996 */
zhanghailianga91246c2016-10-27 14:42:59 +08002997static int ram_save_setup(QEMUFile *f, void *opaque)
2998{
Juan Quintela53518d92017-05-04 11:46:24 +02002999 RAMState **rsp = opaque;
zhanghailianga91246c2016-10-27 14:42:59 +08003000 RAMBlock *block;
Leonardo Bras33d70972022-05-13 03:28:35 -03003001 int ret;
zhanghailianga91246c2016-10-27 14:42:59 +08003002
Xiao Guangrongdcaf4462018-03-30 15:51:20 +08003003 if (compress_threads_save_setup()) {
3004 return -1;
3005 }
3006
zhanghailianga91246c2016-10-27 14:42:59 +08003007 /* migration has already setup the bitmap, reuse it. */
3008 if (!migration_in_colo_state()) {
Peter Xu7d00ee62017-10-19 14:31:57 +08003009 if (ram_init_all(rsp) != 0) {
Xiao Guangrongdcaf4462018-03-30 15:51:20 +08003010 compress_threads_save_cleanup();
zhanghailianga91246c2016-10-27 14:42:59 +08003011 return -1;
Juan Quintela53518d92017-05-04 11:46:24 +02003012 }
zhanghailianga91246c2016-10-27 14:42:59 +08003013 }
Peter Xu7f401b82022-10-11 17:55:59 -04003014 (*rsp)->pss[RAM_CHANNEL_PRECOPY].pss_channel = f;
zhanghailianga91246c2016-10-27 14:42:59 +08003015
Dr. David Alan Gilbert0e6ebd42019-10-07 15:36:38 +01003016 WITH_RCU_READ_LOCK_GUARD() {
Juan Quintela8008a272022-05-10 19:18:19 +02003017 qemu_put_be64(f, ram_bytes_total_with_ignored()
3018 | RAM_SAVE_FLAG_MEM_SIZE);
Juan Quintela56e93d22015-05-07 19:33:31 +02003019
Dr. David Alan Gilbert0e6ebd42019-10-07 15:36:38 +01003020 RAMBLOCK_FOREACH_MIGRATABLE(block) {
3021 qemu_put_byte(f, strlen(block->idstr));
3022 qemu_put_buffer(f, (uint8_t *)block->idstr, strlen(block->idstr));
3023 qemu_put_be64(f, block->used_length);
3024 if (migrate_postcopy_ram() && block->page_size !=
3025 qemu_host_page_size) {
3026 qemu_put_be64(f, block->page_size);
3027 }
3028 if (migrate_ignore_shared()) {
3029 qemu_put_be64(f, block->mr->addr);
3030 }
Yury Kotovfbd162e2019-02-15 20:45:46 +03003031 }
Juan Quintela56e93d22015-05-07 19:33:31 +02003032 }
3033
Juan Quintelab1b38382023-10-11 22:35:23 +02003034 ret = rdma_registration_start(f, RAM_CONTROL_SETUP);
Juan Quintela48408172023-10-11 22:35:16 +02003035 if (ret < 0) {
3036 qemu_file_set_error(f, ret);
3037 }
Juan Quintela5f5b8852023-10-11 22:35:17 +02003038
Juan Quintelab1b38382023-10-11 22:35:23 +02003039 ret = rdma_registration_stop(f, RAM_CONTROL_SETUP);
Juan Quintela5f5b8852023-10-11 22:35:17 +02003040 if (ret < 0) {
3041 qemu_file_set_error(f, ret);
3042 }
Juan Quintela56e93d22015-05-07 19:33:31 +02003043
Juan Quintela4010ba32021-12-15 20:10:39 +01003044 migration_ops = g_malloc0(sizeof(MigrationOps));
3045 migration_ops->ram_save_target_page = ram_save_target_page_legacy;
Fiona Ebner930e2392023-10-13 12:58:39 +02003046
3047 qemu_mutex_unlock_iothread();
李皆俊8ebb6ec2023-03-17 09:57:13 +00003048 ret = multifd_send_sync_main(f);
Fiona Ebner930e2392023-10-13 12:58:39 +02003049 qemu_mutex_lock_iothread();
Leonardo Bras33d70972022-05-13 03:28:35 -03003050 if (ret < 0) {
3051 return ret;
3052 }
3053
Juan Quintelad4f34482023-10-11 22:55:48 +02003054 if (migrate_multifd() && !migrate_multifd_flush_after_each_section()) {
Juan Quintela294e5a42022-06-21 13:36:11 +02003055 qemu_put_be64(f, RAM_SAVE_FLAG_MULTIFD_FLUSH);
3056 }
3057
Juan Quintela56e93d22015-05-07 19:33:31 +02003058 qemu_put_be64(f, RAM_SAVE_FLAG_EOS);
Juan Quintela35374cb2018-04-18 10:13:21 +02003059 qemu_fflush(f);
Juan Quintela56e93d22015-05-07 19:33:31 +02003060
3061 return 0;
3062}
3063
Juan Quintela3d0684b2017-03-23 15:06:39 +01003064/**
3065 * ram_save_iterate: iterative stage for migration
3066 *
3067 * Returns zero to indicate success and negative for error
3068 *
3069 * @f: QEMUFile where to send the data
3070 * @opaque: RAMState pointer
3071 */
Juan Quintela56e93d22015-05-07 19:33:31 +02003072static int ram_save_iterate(QEMUFile *f, void *opaque)
3073{
Juan Quintela53518d92017-05-04 11:46:24 +02003074 RAMState **temp = opaque;
3075 RAMState *rs = *temp;
Juan Quintela3d4095b2019-12-18 05:12:36 +01003076 int ret = 0;
Juan Quintela56e93d22015-05-07 19:33:31 +02003077 int i;
3078 int64_t t0;
Thomas Huth5c903082016-11-04 14:10:17 +01003079 int done = 0;
Juan Quintela56e93d22015-05-07 19:33:31 +02003080
Peter Lievenb2557342018-03-08 12:18:24 +01003081 if (blk_mig_bulk_active()) {
3082 /* Avoid transferring ram during bulk phase of block migration as
3083 * the bulk phase will usually take a long time and transferring
3084 * ram updates during that time is pointless. */
3085 goto out;
3086 }
3087
Peter Xu63268c42021-06-30 16:08:05 -04003088 /*
3089 * We'll take this lock a little bit long, but it's okay for two reasons.
3090 * Firstly, the only possible other thread to take it is who calls
3091 * qemu_guest_free_page_hint(), which should be rare; secondly, see
3092 * MAX_WAIT (if curious, further see commit 4508bd9ed8053ce) below, which
3093 * guarantees that we'll at least released it in a regular basis.
3094 */
3095 qemu_mutex_lock(&rs->bitmap_mutex);
Dr. David Alan Gilbert89ac5a12019-10-07 15:36:39 +01003096 WITH_RCU_READ_LOCK_GUARD() {
3097 if (ram_list.version != rs->last_version) {
3098 ram_state_reset(rs);
Dr. David Alan Gilberte03a34f2018-06-13 11:26:42 +01003099 }
3100
Dr. David Alan Gilbert89ac5a12019-10-07 15:36:39 +01003101 /* Read version before ram_list.blocks */
3102 smp_rmb();
Xiao Guangronge8f37352018-09-03 17:26:44 +08003103
Juan Quintelab1b38382023-10-11 22:35:23 +02003104 ret = rdma_registration_start(f, RAM_CONTROL_ROUND);
Juan Quintela48408172023-10-11 22:35:16 +02003105 if (ret < 0) {
3106 qemu_file_set_error(f, ret);
3107 }
Xiao Guangronge8f37352018-09-03 17:26:44 +08003108
Dr. David Alan Gilbert89ac5a12019-10-07 15:36:39 +01003109 t0 = qemu_clock_get_ns(QEMU_CLOCK_REALTIME);
3110 i = 0;
Juan Quintelae1fde0e2023-05-15 21:56:58 +02003111 while ((ret = migration_rate_exceeded(f)) == 0 ||
Peter Xua1fe28d2022-01-19 16:09:18 +08003112 postcopy_has_request(rs)) {
Dr. David Alan Gilbert89ac5a12019-10-07 15:36:39 +01003113 int pages;
Jason J. Herne070afca2015-09-08 13:12:35 -04003114
Dr. David Alan Gilbert89ac5a12019-10-07 15:36:39 +01003115 if (qemu_file_get_error(f)) {
Juan Quintela56e93d22015-05-07 19:33:31 +02003116 break;
3117 }
Dr. David Alan Gilbert89ac5a12019-10-07 15:36:39 +01003118
Juan Quintela05931ec2021-12-15 19:01:21 +01003119 pages = ram_find_and_save_block(rs);
Dr. David Alan Gilbert89ac5a12019-10-07 15:36:39 +01003120 /* no more pages to sent */
3121 if (pages == 0) {
3122 done = 1;
3123 break;
3124 }
3125
3126 if (pages < 0) {
3127 qemu_file_set_error(f, pages);
3128 break;
3129 }
3130
3131 rs->target_page_count += pages;
3132
3133 /*
Wei Yang644acf92019-11-07 20:39:07 +08003134 * During postcopy, it is necessary to make sure one whole host
3135 * page is sent in one chunk.
3136 */
3137 if (migrate_postcopy_ram()) {
Lukas Straubef4f5f52023-04-20 11:48:13 +02003138 ram_flush_compressed_data(rs);
Wei Yang644acf92019-11-07 20:39:07 +08003139 }
3140
3141 /*
Dr. David Alan Gilbert89ac5a12019-10-07 15:36:39 +01003142 * we want to check in the 1st loop, just in case it was the 1st
3143 * time and we had to sync the dirty bitmap.
3144 * qemu_clock_get_ns() is a bit expensive, so we only check each
3145 * some iterations
3146 */
3147 if ((i & 63) == 0) {
3148 uint64_t t1 = (qemu_clock_get_ns(QEMU_CLOCK_REALTIME) - t0) /
3149 1000000;
3150 if (t1 > MAX_WAIT) {
3151 trace_ram_save_iterate_big_wait(t1, i);
3152 break;
3153 }
3154 }
3155 i++;
Juan Quintela56e93d22015-05-07 19:33:31 +02003156 }
Juan Quintela56e93d22015-05-07 19:33:31 +02003157 }
Peter Xu63268c42021-06-30 16:08:05 -04003158 qemu_mutex_unlock(&rs->bitmap_mutex);
Juan Quintela56e93d22015-05-07 19:33:31 +02003159
3160 /*
3161 * Must occur before EOS (or any QEMUFile operation)
3162 * because of RDMA protocol.
3163 */
Juan Quintelab1b38382023-10-11 22:35:23 +02003164 ret = rdma_registration_stop(f, RAM_CONTROL_ROUND);
Juan Quintela5f5b8852023-10-11 22:35:17 +02003165 if (ret < 0) {
3166 qemu_file_set_error(f, ret);
3167 }
Juan Quintela56e93d22015-05-07 19:33:31 +02003168
Peter Lievenb2557342018-03-08 12:18:24 +01003169out:
Juan Quintelab69a0222020-01-22 11:36:12 +01003170 if (ret >= 0
3171 && migration_is_setup_or_active(migrate_get_current()->state)) {
Juan Quintelad4f34482023-10-11 22:55:48 +02003172 if (migrate_multifd() && migrate_multifd_flush_after_each_section()) {
Juan Quintelab05292c2022-06-21 12:21:32 +02003173 ret = multifd_send_sync_main(rs->pss[RAM_CHANNEL_PRECOPY].pss_channel);
3174 if (ret < 0) {
3175 return ret;
3176 }
Leonardo Bras33d70972022-05-13 03:28:35 -03003177 }
3178
Juan Quintela3d4095b2019-12-18 05:12:36 +01003179 qemu_put_be64(f, RAM_SAVE_FLAG_EOS);
3180 qemu_fflush(f);
David Edmondson4c2d0f62021-12-21 09:34:40 +00003181 ram_transferred_add(8);
Juan Quintela56e93d22015-05-07 19:33:31 +02003182
Juan Quintela3d4095b2019-12-18 05:12:36 +01003183 ret = qemu_file_get_error(f);
3184 }
Juan Quintela56e93d22015-05-07 19:33:31 +02003185 if (ret < 0) {
3186 return ret;
3187 }
3188
Thomas Huth5c903082016-11-04 14:10:17 +01003189 return done;
Juan Quintela56e93d22015-05-07 19:33:31 +02003190}
3191
Juan Quintela3d0684b2017-03-23 15:06:39 +01003192/**
3193 * ram_save_complete: function called to send the remaining amount of ram
3194 *
Xiao Guangronge8f37352018-09-03 17:26:44 +08003195 * Returns zero to indicate success or negative on error
Juan Quintela3d0684b2017-03-23 15:06:39 +01003196 *
3197 * Called with iothread lock
3198 *
3199 * @f: QEMUFile where to send the data
3200 * @opaque: RAMState pointer
3201 */
Juan Quintela56e93d22015-05-07 19:33:31 +02003202static int ram_save_complete(QEMUFile *f, void *opaque)
3203{
Juan Quintela53518d92017-05-04 11:46:24 +02003204 RAMState **temp = opaque;
3205 RAMState *rs = *temp;
Xiao Guangronge8f37352018-09-03 17:26:44 +08003206 int ret = 0;
Juan Quintela6f37bb82017-03-13 19:26:29 +01003207
Juan Quintela05931ec2021-12-15 19:01:21 +01003208 rs->last_stage = !migration_in_colo_state();
3209
Dr. David Alan Gilbert89ac5a12019-10-07 15:36:39 +01003210 WITH_RCU_READ_LOCK_GUARD() {
3211 if (!migration_in_postcopy()) {
Gavin Shan1e493be2023-05-09 12:21:19 +10003212 migration_bitmap_sync_precopy(rs, true);
Juan Quintela56e93d22015-05-07 19:33:31 +02003213 }
Dr. David Alan Gilbert89ac5a12019-10-07 15:36:39 +01003214
Juan Quintelab1b38382023-10-11 22:35:23 +02003215 ret = rdma_registration_start(f, RAM_CONTROL_FINISH);
Juan Quintela48408172023-10-11 22:35:16 +02003216 if (ret < 0) {
3217 qemu_file_set_error(f, ret);
3218 }
Dr. David Alan Gilbert89ac5a12019-10-07 15:36:39 +01003219
3220 /* try transferring iterative blocks of memory */
3221
3222 /* flush all remaining blocks regardless of rate limiting */
Peter Xuc13221b2022-10-11 17:55:45 -04003223 qemu_mutex_lock(&rs->bitmap_mutex);
Dr. David Alan Gilbert89ac5a12019-10-07 15:36:39 +01003224 while (true) {
3225 int pages;
3226
Juan Quintela05931ec2021-12-15 19:01:21 +01003227 pages = ram_find_and_save_block(rs);
Dr. David Alan Gilbert89ac5a12019-10-07 15:36:39 +01003228 /* no more blocks to sent */
3229 if (pages == 0) {
3230 break;
3231 }
3232 if (pages < 0) {
3233 ret = pages;
3234 break;
3235 }
Xiao Guangronge8f37352018-09-03 17:26:44 +08003236 }
Peter Xuc13221b2022-10-11 17:55:45 -04003237 qemu_mutex_unlock(&rs->bitmap_mutex);
Dr. David Alan Gilbert89ac5a12019-10-07 15:36:39 +01003238
Lukas Straubef4f5f52023-04-20 11:48:13 +02003239 ram_flush_compressed_data(rs);
Juan Quintela5f5b8852023-10-11 22:35:17 +02003240
Juan Quintelab1b38382023-10-11 22:35:23 +02003241 int ret = rdma_registration_stop(f, RAM_CONTROL_FINISH);
Juan Quintela5f5b8852023-10-11 22:35:17 +02003242 if (ret < 0) {
3243 qemu_file_set_error(f, ret);
3244 }
Juan Quintela56e93d22015-05-07 19:33:31 +02003245 }
3246
Leonardo Bras33d70972022-05-13 03:28:35 -03003247 if (ret < 0) {
3248 return ret;
Juan Quintela3d4095b2019-12-18 05:12:36 +01003249 }
Juan Quintela56e93d22015-05-07 19:33:31 +02003250
Peter Xu7f401b82022-10-11 17:55:59 -04003251 ret = multifd_send_sync_main(rs->pss[RAM_CHANNEL_PRECOPY].pss_channel);
Leonardo Bras33d70972022-05-13 03:28:35 -03003252 if (ret < 0) {
3253 return ret;
3254 }
3255
Juan Quintelad4f34482023-10-11 22:55:48 +02003256 if (migrate_multifd() && !migrate_multifd_flush_after_each_section()) {
Juan Quintela294e5a42022-06-21 13:36:11 +02003257 qemu_put_be64(f, RAM_SAVE_FLAG_MULTIFD_FLUSH);
3258 }
Leonardo Bras33d70972022-05-13 03:28:35 -03003259 qemu_put_be64(f, RAM_SAVE_FLAG_EOS);
3260 qemu_fflush(f);
3261
3262 return 0;
Juan Quintela56e93d22015-05-07 19:33:31 +02003263}
3264
Juan Quintela24beea42023-02-08 14:48:02 +01003265static void ram_state_pending_estimate(void *opaque, uint64_t *must_precopy,
3266 uint64_t *can_postcopy)
Juan Quintela56e93d22015-05-07 19:33:31 +02003267{
Juan Quintela53518d92017-05-04 11:46:24 +02003268 RAMState **temp = opaque;
3269 RAMState *rs = *temp;
Juan Quintela56e93d22015-05-07 19:33:31 +02003270
Juan Quintelac8df4a72022-10-03 02:00:03 +02003271 uint64_t remaining_size = rs->migration_dirty_pages * TARGET_PAGE_SIZE;
Juan Quintela56e93d22015-05-07 19:33:31 +02003272
Juan Quintelac8df4a72022-10-03 02:00:03 +02003273 if (migrate_postcopy_ram()) {
3274 /* We can do postcopy, and all the data is postcopiable */
Juan Quintela24beea42023-02-08 14:48:02 +01003275 *can_postcopy += remaining_size;
Juan Quintelac8df4a72022-10-03 02:00:03 +02003276 } else {
Juan Quintela24beea42023-02-08 14:48:02 +01003277 *must_precopy += remaining_size;
Juan Quintelac8df4a72022-10-03 02:00:03 +02003278 }
3279}
3280
Juan Quintela24beea42023-02-08 14:48:02 +01003281static void ram_state_pending_exact(void *opaque, uint64_t *must_precopy,
3282 uint64_t *can_postcopy)
Juan Quintelac8df4a72022-10-03 02:00:03 +02003283{
Juan Quintela28ef5332023-04-12 22:30:20 +02003284 MigrationState *s = migrate_get_current();
Juan Quintelac8df4a72022-10-03 02:00:03 +02003285 RAMState **temp = opaque;
3286 RAMState *rs = *temp;
3287
3288 uint64_t remaining_size = rs->migration_dirty_pages * TARGET_PAGE_SIZE;
3289
Juan Quintela28ef5332023-04-12 22:30:20 +02003290 if (!migration_in_postcopy() && remaining_size < s->threshold_size) {
Juan Quintela56e93d22015-05-07 19:33:31 +02003291 qemu_mutex_lock_iothread();
Dr. David Alan Gilbert89ac5a12019-10-07 15:36:39 +01003292 WITH_RCU_READ_LOCK_GUARD() {
Gavin Shan1e493be2023-05-09 12:21:19 +10003293 migration_bitmap_sync_precopy(rs, false);
Dr. David Alan Gilbert89ac5a12019-10-07 15:36:39 +01003294 }
Juan Quintela56e93d22015-05-07 19:33:31 +02003295 qemu_mutex_unlock_iothread();
Juan Quintela9edabd42017-03-14 12:02:16 +01003296 remaining_size = rs->migration_dirty_pages * TARGET_PAGE_SIZE;
Juan Quintela56e93d22015-05-07 19:33:31 +02003297 }
Dr. David Alan Gilbertc31b0982015-11-05 18:10:54 +00003298
Vladimir Sementsov-Ogievskiy86e11672017-07-10 19:30:15 +03003299 if (migrate_postcopy_ram()) {
3300 /* We can do postcopy, and all the data is postcopiable */
Juan Quintela24beea42023-02-08 14:48:02 +01003301 *can_postcopy += remaining_size;
Vladimir Sementsov-Ogievskiy86e11672017-07-10 19:30:15 +03003302 } else {
Juan Quintela24beea42023-02-08 14:48:02 +01003303 *must_precopy += remaining_size;
Vladimir Sementsov-Ogievskiy86e11672017-07-10 19:30:15 +03003304 }
Juan Quintela56e93d22015-05-07 19:33:31 +02003305}
3306
3307static int load_xbzrle(QEMUFile *f, ram_addr_t addr, void *host)
3308{
3309 unsigned int xh_len;
3310 int xh_flags;
Dr. David Alan Gilbert063e7602015-12-16 11:47:37 +00003311 uint8_t *loaded_data;
Juan Quintela56e93d22015-05-07 19:33:31 +02003312
Juan Quintela56e93d22015-05-07 19:33:31 +02003313 /* extract RLE header */
3314 xh_flags = qemu_get_byte(f);
3315 xh_len = qemu_get_be16(f);
3316
3317 if (xh_flags != ENCODING_FLAG_XBZRLE) {
3318 error_report("Failed to load XBZRLE page - wrong compression!");
3319 return -1;
3320 }
3321
3322 if (xh_len > TARGET_PAGE_SIZE) {
3323 error_report("Failed to load XBZRLE page - len overflow!");
3324 return -1;
3325 }
Juan Quintelaf265e0e2017-06-28 11:52:27 +02003326 loaded_data = XBZRLE.decoded_buf;
Juan Quintela56e93d22015-05-07 19:33:31 +02003327 /* load data and decode */
Juan Quintelaf265e0e2017-06-28 11:52:27 +02003328 /* it can change loaded_data to point to an internal buffer */
Dr. David Alan Gilbert063e7602015-12-16 11:47:37 +00003329 qemu_get_buffer_in_place(f, &loaded_data, xh_len);
Juan Quintela56e93d22015-05-07 19:33:31 +02003330
3331 /* decode RLE */
Dr. David Alan Gilbert063e7602015-12-16 11:47:37 +00003332 if (xbzrle_decode_buffer(loaded_data, xh_len, host,
Juan Quintela56e93d22015-05-07 19:33:31 +02003333 TARGET_PAGE_SIZE) == -1) {
3334 error_report("Failed to load XBZRLE page - decode error!");
3335 return -1;
3336 }
3337
3338 return 0;
3339}
3340
Juan Quintela3d0684b2017-03-23 15:06:39 +01003341/**
3342 * ram_block_from_stream: read a RAMBlock id from the migration stream
Dr. David Alan Gilberta7180872015-11-05 18:11:11 +00003343 *
Juan Quintela3d0684b2017-03-23 15:06:39 +01003344 * Must be called from within a rcu critical section.
3345 *
3346 * Returns a pointer from within the RCU-protected ram_list.
3347 *
Peter Xu755e8d72022-03-01 16:39:07 +08003348 * @mis: the migration incoming state pointer
Juan Quintela3d0684b2017-03-23 15:06:39 +01003349 * @f: QEMUFile where to read the data from
3350 * @flags: Page flags (mostly to see if it's a continuation of previous block)
Peter Xuc01b16e2022-07-07 14:55:04 -04003351 * @channel: the channel we're using
Dr. David Alan Gilberta7180872015-11-05 18:11:11 +00003352 */
Peter Xu755e8d72022-03-01 16:39:07 +08003353static inline RAMBlock *ram_block_from_stream(MigrationIncomingState *mis,
Peter Xuc01b16e2022-07-07 14:55:04 -04003354 QEMUFile *f, int flags,
3355 int channel)
Juan Quintela56e93d22015-05-07 19:33:31 +02003356{
Peter Xuc01b16e2022-07-07 14:55:04 -04003357 RAMBlock *block = mis->last_recv_block[channel];
Juan Quintela56e93d22015-05-07 19:33:31 +02003358 char id[256];
3359 uint8_t len;
3360
3361 if (flags & RAM_SAVE_FLAG_CONTINUE) {
zhanghailiang4c4bad42016-01-15 11:37:41 +08003362 if (!block) {
Juan Quintela56e93d22015-05-07 19:33:31 +02003363 error_report("Ack, bad migration stream!");
3364 return NULL;
3365 }
zhanghailiang4c4bad42016-01-15 11:37:41 +08003366 return block;
Juan Quintela56e93d22015-05-07 19:33:31 +02003367 }
3368
3369 len = qemu_get_byte(f);
3370 qemu_get_buffer(f, (uint8_t *)id, len);
3371 id[len] = 0;
3372
Dr. David Alan Gilberte3dd7492015-11-05 18:10:33 +00003373 block = qemu_ram_block_by_name(id);
zhanghailiang4c4bad42016-01-15 11:37:41 +08003374 if (!block) {
3375 error_report("Can't find block %s", id);
3376 return NULL;
Juan Quintela56e93d22015-05-07 19:33:31 +02003377 }
3378
David Hildenbrandf161c882023-07-06 09:56:08 +02003379 if (migrate_ram_is_ignored(block)) {
Cédric Le Goaterb895de52018-05-14 08:57:00 +02003380 error_report("block %s should not be migrated !", id);
3381 return NULL;
3382 }
3383
Peter Xuc01b16e2022-07-07 14:55:04 -04003384 mis->last_recv_block[channel] = block;
Peter Xu755e8d72022-03-01 16:39:07 +08003385
zhanghailiang4c4bad42016-01-15 11:37:41 +08003386 return block;
3387}
3388
3389static inline void *host_from_ram_block_offset(RAMBlock *block,
3390 ram_addr_t offset)
3391{
3392 if (!offset_in_ramblock(block, offset)) {
3393 return NULL;
3394 }
3395
3396 return block->host + offset;
Juan Quintela56e93d22015-05-07 19:33:31 +02003397}
3398
David Hildenbrand6a23f632021-04-29 13:27:05 +02003399static void *host_page_from_ram_block_offset(RAMBlock *block,
3400 ram_addr_t offset)
3401{
3402 /* Note: Explicitly no check against offset_in_ramblock(). */
3403 return (void *)QEMU_ALIGN_DOWN((uintptr_t)(block->host + offset),
3404 block->page_size);
3405}
3406
3407static ram_addr_t host_page_offset_from_ram_block_offset(RAMBlock *block,
3408 ram_addr_t offset)
3409{
3410 return ((uintptr_t)block->host + offset) & (block->page_size - 1);
3411}
3412
Lukas Straub871cfc52023-05-08 21:10:52 +02003413void colo_record_bitmap(RAMBlock *block, ram_addr_t *normal, uint32_t pages)
3414{
3415 qemu_mutex_lock(&ram_state->bitmap_mutex);
3416 for (int i = 0; i < pages; i++) {
3417 ram_addr_t offset = normal[i];
3418 ram_state->migration_dirty_pages += !test_and_set_bit(
3419 offset >> TARGET_PAGE_BITS,
3420 block->bmap);
3421 }
3422 qemu_mutex_unlock(&ram_state->bitmap_mutex);
3423}
3424
Zhang Chen13af18f2018-09-03 12:38:48 +08003425static inline void *colo_cache_from_block_offset(RAMBlock *block,
zhanghailiang8af66372020-02-24 14:54:11 +08003426 ram_addr_t offset, bool record_bitmap)
Zhang Chen13af18f2018-09-03 12:38:48 +08003427{
3428 if (!offset_in_ramblock(block, offset)) {
3429 return NULL;
3430 }
3431 if (!block->colo_cache) {
3432 error_report("%s: colo_cache is NULL in block :%s",
3433 __func__, block->idstr);
3434 return NULL;
3435 }
Zhang Chen7d9acaf2018-09-03 12:38:49 +08003436
3437 /*
3438 * During colo checkpoint, we need bitmap of these migrated pages.
3439 * It help us to decide which pages in ram cache should be flushed
3440 * into VM's RAM later.
3441 */
Lukas Straub871cfc52023-05-08 21:10:52 +02003442 if (record_bitmap) {
3443 colo_record_bitmap(block, &offset, 1);
Zhang Chen7d9acaf2018-09-03 12:38:49 +08003444 }
Zhang Chen13af18f2018-09-03 12:38:48 +08003445 return block->colo_cache + offset;
3446}
3447
Juan Quintela3d0684b2017-03-23 15:06:39 +01003448/**
3449 * ram_handle_compressed: handle the zero page case
3450 *
Juan Quintela56e93d22015-05-07 19:33:31 +02003451 * If a page (or a whole RDMA chunk) has been
3452 * determined to be zero, then zap it.
Juan Quintela3d0684b2017-03-23 15:06:39 +01003453 *
3454 * @host: host address for the zero page
3455 * @ch: what the page is filled from. We only support zero
3456 * @size: size of the zero page
Juan Quintela56e93d22015-05-07 19:33:31 +02003457 */
3458void ram_handle_compressed(void *host, uint8_t ch, uint64_t size)
3459{
Juan Quintelabad452a2021-11-18 15:56:38 +01003460 if (ch != 0 || !buffer_is_zero(host, size)) {
Juan Quintela56e93d22015-05-07 19:33:31 +02003461 memset(host, ch, size);
3462 }
3463}
3464
Rao, Leib70cb3b2020-10-16 13:52:01 +08003465static void colo_init_ram_state(void)
3466{
3467 ram_state_init(&ram_state);
Rao, Leib70cb3b2020-10-16 13:52:01 +08003468}
3469
Zhang Chen13af18f2018-09-03 12:38:48 +08003470/*
3471 * colo cache: this is for secondary VM, we cache the whole
3472 * memory of the secondary VM, it is need to hold the global lock
3473 * to call this helper.
3474 */
3475int colo_init_ram_cache(void)
3476{
3477 RAMBlock *block;
3478
Paolo Bonzini44901b52019-12-13 15:07:22 +01003479 WITH_RCU_READ_LOCK_GUARD() {
3480 RAMBLOCK_FOREACH_NOT_IGNORED(block) {
3481 block->colo_cache = qemu_anon_ram_alloc(block->used_length,
David Hildenbrand8dbe22c2021-05-10 13:43:21 +02003482 NULL, false, false);
Paolo Bonzini44901b52019-12-13 15:07:22 +01003483 if (!block->colo_cache) {
3484 error_report("%s: Can't alloc memory for COLO cache of block %s,"
3485 "size 0x" RAM_ADDR_FMT, __func__, block->idstr,
3486 block->used_length);
3487 RAMBLOCK_FOREACH_NOT_IGNORED(block) {
3488 if (block->colo_cache) {
3489 qemu_anon_ram_free(block->colo_cache, block->used_length);
3490 block->colo_cache = NULL;
3491 }
Dr. David Alan Gilbert89ac5a12019-10-07 15:36:39 +01003492 }
Paolo Bonzini44901b52019-12-13 15:07:22 +01003493 return -errno;
Dr. David Alan Gilbert89ac5a12019-10-07 15:36:39 +01003494 }
Lukas Straube5fdf922021-07-04 18:14:44 +02003495 if (!machine_dump_guest_core(current_machine)) {
3496 qemu_madvise(block->colo_cache, block->used_length,
3497 QEMU_MADV_DONTDUMP);
3498 }
Zhang Chen13af18f2018-09-03 12:38:48 +08003499 }
Zhang Chen13af18f2018-09-03 12:38:48 +08003500 }
Paolo Bonzini44901b52019-12-13 15:07:22 +01003501
Zhang Chen7d9acaf2018-09-03 12:38:49 +08003502 /*
3503 * Record the dirty pages that sent by PVM, we use this dirty bitmap together
3504 * with to decide which page in cache should be flushed into SVM's RAM. Here
3505 * we use the same name 'ram_bitmap' as for migration.
3506 */
3507 if (ram_bytes_total()) {
Yury Kotovfbd162e2019-02-15 20:45:46 +03003508 RAMBLOCK_FOREACH_NOT_IGNORED(block) {
Zhang Chen7d9acaf2018-09-03 12:38:49 +08003509 unsigned long pages = block->max_length >> TARGET_PAGE_BITS;
Zhang Chen7d9acaf2018-09-03 12:38:49 +08003510 block->bmap = bitmap_new(pages);
Zhang Chen7d9acaf2018-09-03 12:38:49 +08003511 }
3512 }
Zhang Chen7d9acaf2018-09-03 12:38:49 +08003513
Rao, Leib70cb3b2020-10-16 13:52:01 +08003514 colo_init_ram_state();
Zhang Chen13af18f2018-09-03 12:38:48 +08003515 return 0;
Zhang Chen13af18f2018-09-03 12:38:48 +08003516}
3517
zhanghailiang03930312020-02-24 14:54:10 +08003518/* TODO: duplicated with ram_init_bitmaps */
3519void colo_incoming_start_dirty_log(void)
3520{
3521 RAMBlock *block = NULL;
3522 /* For memory_global_dirty_log_start below. */
3523 qemu_mutex_lock_iothread();
3524 qemu_mutex_lock_ramlist();
3525
Gavin Shan1e493be2023-05-09 12:21:19 +10003526 memory_global_dirty_log_sync(false);
zhanghailiang03930312020-02-24 14:54:10 +08003527 WITH_RCU_READ_LOCK_GUARD() {
3528 RAMBLOCK_FOREACH_NOT_IGNORED(block) {
3529 ramblock_sync_dirty_bitmap(ram_state, block);
3530 /* Discard this dirty bitmap record */
3531 bitmap_zero(block->bmap, block->max_length >> TARGET_PAGE_BITS);
3532 }
Hyman Huang(黄勇)63b41db2021-06-29 16:01:19 +00003533 memory_global_dirty_log_start(GLOBAL_DIRTY_MIGRATION);
zhanghailiang03930312020-02-24 14:54:10 +08003534 }
3535 ram_state->migration_dirty_pages = 0;
3536 qemu_mutex_unlock_ramlist();
3537 qemu_mutex_unlock_iothread();
3538}
3539
Zhang Chen13af18f2018-09-03 12:38:48 +08003540/* It is need to hold the global lock to call this helper */
3541void colo_release_ram_cache(void)
3542{
3543 RAMBlock *block;
3544
Hyman Huang(黄勇)63b41db2021-06-29 16:01:19 +00003545 memory_global_dirty_log_stop(GLOBAL_DIRTY_MIGRATION);
Yury Kotovfbd162e2019-02-15 20:45:46 +03003546 RAMBLOCK_FOREACH_NOT_IGNORED(block) {
Zhang Chen7d9acaf2018-09-03 12:38:49 +08003547 g_free(block->bmap);
3548 block->bmap = NULL;
3549 }
3550
Dr. David Alan Gilbert89ac5a12019-10-07 15:36:39 +01003551 WITH_RCU_READ_LOCK_GUARD() {
3552 RAMBLOCK_FOREACH_NOT_IGNORED(block) {
3553 if (block->colo_cache) {
3554 qemu_anon_ram_free(block->colo_cache, block->used_length);
3555 block->colo_cache = NULL;
3556 }
Zhang Chen13af18f2018-09-03 12:38:48 +08003557 }
3558 }
zhanghailiang03930312020-02-24 14:54:10 +08003559 ram_state_cleanup(&ram_state);
Zhang Chen13af18f2018-09-03 12:38:48 +08003560}
3561
Juan Quintela3d0684b2017-03-23 15:06:39 +01003562/**
Juan Quintelaf265e0e2017-06-28 11:52:27 +02003563 * ram_load_setup: Setup RAM for migration incoming side
3564 *
3565 * Returns zero to indicate success and negative for error
3566 *
3567 * @f: QEMUFile where to receive the data
3568 * @opaque: RAMState pointer
3569 */
3570static int ram_load_setup(QEMUFile *f, void *opaque)
3571{
3572 xbzrle_load_setup();
Alexey Perevalovf9494612017-10-05 14:13:20 +03003573 ramblock_recv_map_init();
Zhang Chen13af18f2018-09-03 12:38:48 +08003574
Juan Quintelaf265e0e2017-06-28 11:52:27 +02003575 return 0;
3576}
3577
3578static int ram_load_cleanup(void *opaque)
3579{
Alexey Perevalovf9494612017-10-05 14:13:20 +03003580 RAMBlock *rb;
Junyan He56eb90a2018-07-18 15:48:03 +08003581
Yury Kotovfbd162e2019-02-15 20:45:46 +03003582 RAMBLOCK_FOREACH_NOT_IGNORED(rb) {
Beata Michalskabd108a42019-11-21 00:08:42 +00003583 qemu_ram_block_writeback(rb);
Junyan He56eb90a2018-07-18 15:48:03 +08003584 }
3585
Juan Quintelaf265e0e2017-06-28 11:52:27 +02003586 xbzrle_load_cleanup();
Alexey Perevalovf9494612017-10-05 14:13:20 +03003587
Yury Kotovfbd162e2019-02-15 20:45:46 +03003588 RAMBLOCK_FOREACH_NOT_IGNORED(rb) {
Alexey Perevalovf9494612017-10-05 14:13:20 +03003589 g_free(rb->receivedmap);
3590 rb->receivedmap = NULL;
3591 }
Zhang Chen13af18f2018-09-03 12:38:48 +08003592
Juan Quintelaf265e0e2017-06-28 11:52:27 +02003593 return 0;
3594}
3595
3596/**
Juan Quintela3d0684b2017-03-23 15:06:39 +01003597 * ram_postcopy_incoming_init: allocate postcopy data structures
3598 *
3599 * Returns 0 for success and negative if there was one error
3600 *
3601 * @mis: current migration incoming state
3602 *
3603 * Allocate data structures etc needed by incoming migration with
3604 * postcopy-ram. postcopy-ram's similarly names
3605 * postcopy_ram_incoming_init does the work.
Dr. David Alan Gilbert1caddf82015-11-05 18:11:03 +00003606 */
3607int ram_postcopy_incoming_init(MigrationIncomingState *mis)
3608{
David Hildenbrandc1361802018-06-20 22:27:36 +02003609 return postcopy_ram_incoming_init(mis);
Dr. David Alan Gilbert1caddf82015-11-05 18:11:03 +00003610}
3611
Juan Quintela3d0684b2017-03-23 15:06:39 +01003612/**
3613 * ram_load_postcopy: load a page in postcopy case
3614 *
3615 * Returns 0 for success or -errno in case of error
3616 *
Dr. David Alan Gilberta7180872015-11-05 18:11:11 +00003617 * Called in postcopy mode by ram_load().
3618 * rcu_read_lock is taken prior to this being called.
Juan Quintela3d0684b2017-03-23 15:06:39 +01003619 *
3620 * @f: QEMUFile where to send the data
Peter Xu36f62f12022-07-07 14:55:02 -04003621 * @channel: the channel to use for loading
Dr. David Alan Gilberta7180872015-11-05 18:11:11 +00003622 */
Peter Xu36f62f12022-07-07 14:55:02 -04003623int ram_load_postcopy(QEMUFile *f, int channel)
Dr. David Alan Gilberta7180872015-11-05 18:11:11 +00003624{
3625 int flags = 0, ret = 0;
3626 bool place_needed = false;
Peter Xu1aa83672018-07-10 17:18:53 +08003627 bool matches_target_page_size = false;
Dr. David Alan Gilberta7180872015-11-05 18:11:11 +00003628 MigrationIncomingState *mis = migration_incoming_get_current();
Peter Xu36f62f12022-07-07 14:55:02 -04003629 PostcopyTmpPage *tmp_page = &mis->postcopy_tmp_pages[channel];
Dr. David Alan Gilberta7180872015-11-05 18:11:11 +00003630
3631 while (!ret && !(flags & RAM_SAVE_FLAG_EOS)) {
3632 ram_addr_t addr;
Dr. David Alan Gilberta7180872015-11-05 18:11:11 +00003633 void *page_buffer = NULL;
3634 void *place_source = NULL;
Dr. David Alan Gilbertdf9ff5e2017-02-24 18:28:35 +00003635 RAMBlock *block = NULL;
Dr. David Alan Gilberta7180872015-11-05 18:11:11 +00003636 uint8_t ch;
Wei Yang644acf92019-11-07 20:39:07 +08003637 int len;
Dr. David Alan Gilberta7180872015-11-05 18:11:11 +00003638
3639 addr = qemu_get_be64(f);
Peter Xu7a9ddfb2018-02-08 18:31:05 +08003640
3641 /*
3642 * If qemu file error, we should stop here, and then "addr"
3643 * may be invalid
3644 */
3645 ret = qemu_file_get_error(f);
3646 if (ret) {
3647 break;
3648 }
3649
Dr. David Alan Gilberta7180872015-11-05 18:11:11 +00003650 flags = addr & ~TARGET_PAGE_MASK;
3651 addr &= TARGET_PAGE_MASK;
3652
Peter Xu36f62f12022-07-07 14:55:02 -04003653 trace_ram_load_postcopy_loop(channel, (uint64_t)addr, flags);
Wei Yang644acf92019-11-07 20:39:07 +08003654 if (flags & (RAM_SAVE_FLAG_ZERO | RAM_SAVE_FLAG_PAGE |
3655 RAM_SAVE_FLAG_COMPRESS_PAGE)) {
Peter Xuc01b16e2022-07-07 14:55:04 -04003656 block = ram_block_from_stream(mis, f, flags, channel);
David Hildenbrand6a23f632021-04-29 13:27:05 +02003657 if (!block) {
3658 ret = -EINVAL;
3659 break;
3660 }
zhanghailiang4c4bad42016-01-15 11:37:41 +08003661
David Hildenbrand898ba902021-04-29 13:27:06 +02003662 /*
3663 * Relying on used_length is racy and can result in false positives.
3664 * We might place pages beyond used_length in case RAM was shrunk
3665 * while in postcopy, which is fine - trying to place via
3666 * UFFDIO_COPY/UFFDIO_ZEROPAGE will never segfault.
3667 */
3668 if (!block->host || addr >= block->postcopy_length) {
Dr. David Alan Gilberta7180872015-11-05 18:11:11 +00003669 error_report("Illegal RAM offset " RAM_ADDR_FMT, addr);
3670 ret = -EINVAL;
3671 break;
3672 }
Peter Xu77dadc32022-03-01 16:39:04 +08003673 tmp_page->target_pages++;
Peter Xu1aa83672018-07-10 17:18:53 +08003674 matches_target_page_size = block->page_size == TARGET_PAGE_SIZE;
Dr. David Alan Gilberta7180872015-11-05 18:11:11 +00003675 /*
Dr. David Alan Gilbert28abd202017-02-24 18:28:37 +00003676 * Postcopy requires that we place whole host pages atomically;
3677 * these may be huge pages for RAMBlocks that are backed by
3678 * hugetlbfs.
Dr. David Alan Gilberta7180872015-11-05 18:11:11 +00003679 * To make it atomic, the data is read into a temporary page
3680 * that's moved into place later.
3681 * The migration protocol uses, possibly smaller, target-pages
3682 * however the source ensures it always sends all the components
Wei Yang91ba4422019-11-07 20:39:06 +08003683 * of a host page in one chunk.
Dr. David Alan Gilberta7180872015-11-05 18:11:11 +00003684 */
Peter Xu77dadc32022-03-01 16:39:04 +08003685 page_buffer = tmp_page->tmp_huge_page +
David Hildenbrand6a23f632021-04-29 13:27:05 +02003686 host_page_offset_from_ram_block_offset(block, addr);
3687 /* If all TP are zero then we can optimise the place */
Peter Xu77dadc32022-03-01 16:39:04 +08003688 if (tmp_page->target_pages == 1) {
3689 tmp_page->host_addr =
3690 host_page_from_ram_block_offset(block, addr);
3691 } else if (tmp_page->host_addr !=
3692 host_page_from_ram_block_offset(block, addr)) {
Dr. David Alan Gilbertc53b7dd2015-11-05 18:11:12 +00003693 /* not the 1st TP within the HP */
Peter Xu36f62f12022-07-07 14:55:02 -04003694 error_report("Non-same host page detected on channel %d: "
Peter Xucfc7dc82022-03-01 16:39:05 +08003695 "Target host page %p, received host page %p "
3696 "(rb %s offset 0x"RAM_ADDR_FMT" target_pages %d)",
Peter Xu36f62f12022-07-07 14:55:02 -04003697 channel, tmp_page->host_addr,
Peter Xucfc7dc82022-03-01 16:39:05 +08003698 host_page_from_ram_block_offset(block, addr),
3699 block->idstr, addr, tmp_page->target_pages);
David Hildenbrand6a23f632021-04-29 13:27:05 +02003700 ret = -EINVAL;
3701 break;
Dr. David Alan Gilberta7180872015-11-05 18:11:11 +00003702 }
3703
3704 /*
3705 * If it's the last part of a host page then we place the host
3706 * page
3707 */
Peter Xu77dadc32022-03-01 16:39:04 +08003708 if (tmp_page->target_pages ==
3709 (block->page_size / TARGET_PAGE_SIZE)) {
Wei Yang4cbb3c62019-11-07 20:39:04 +08003710 place_needed = true;
Wei Yang4cbb3c62019-11-07 20:39:04 +08003711 }
Peter Xu77dadc32022-03-01 16:39:04 +08003712 place_source = tmp_page->tmp_huge_page;
Dr. David Alan Gilberta7180872015-11-05 18:11:11 +00003713 }
3714
3715 switch (flags & ~RAM_SAVE_FLAG_CONTINUE) {
Juan Quintelabb890ed2017-04-28 09:39:55 +02003716 case RAM_SAVE_FLAG_ZERO:
Dr. David Alan Gilberta7180872015-11-05 18:11:11 +00003717 ch = qemu_get_byte(f);
Wei Yang2e36bc12019-11-07 20:39:02 +08003718 /*
3719 * Can skip to set page_buffer when
3720 * this is a zero page and (block->page_size == TARGET_PAGE_SIZE).
3721 */
3722 if (ch || !matches_target_page_size) {
3723 memset(page_buffer, ch, TARGET_PAGE_SIZE);
3724 }
Dr. David Alan Gilberta7180872015-11-05 18:11:11 +00003725 if (ch) {
Peter Xu77dadc32022-03-01 16:39:04 +08003726 tmp_page->all_zero = false;
Dr. David Alan Gilberta7180872015-11-05 18:11:11 +00003727 }
3728 break;
3729
3730 case RAM_SAVE_FLAG_PAGE:
Peter Xu77dadc32022-03-01 16:39:04 +08003731 tmp_page->all_zero = false;
Peter Xu1aa83672018-07-10 17:18:53 +08003732 if (!matches_target_page_size) {
3733 /* For huge pages, we always use temporary buffer */
Dr. David Alan Gilberta7180872015-11-05 18:11:11 +00003734 qemu_get_buffer(f, page_buffer, TARGET_PAGE_SIZE);
3735 } else {
Peter Xu1aa83672018-07-10 17:18:53 +08003736 /*
3737 * For small pages that matches target page size, we
3738 * avoid the qemu_file copy. Instead we directly use
3739 * the buffer of QEMUFile to place the page. Note: we
3740 * cannot do any QEMUFile operation before using that
3741 * buffer to make sure the buffer is valid when
3742 * placing the page.
Dr. David Alan Gilberta7180872015-11-05 18:11:11 +00003743 */
3744 qemu_get_buffer_in_place(f, (uint8_t **)&place_source,
3745 TARGET_PAGE_SIZE);
3746 }
3747 break;
Wei Yang644acf92019-11-07 20:39:07 +08003748 case RAM_SAVE_FLAG_COMPRESS_PAGE:
Peter Xu77dadc32022-03-01 16:39:04 +08003749 tmp_page->all_zero = false;
Wei Yang644acf92019-11-07 20:39:07 +08003750 len = qemu_get_be32(f);
3751 if (len < 0 || len > compressBound(TARGET_PAGE_SIZE)) {
3752 error_report("Invalid compressed data length: %d", len);
3753 ret = -EINVAL;
3754 break;
3755 }
3756 decompress_data_with_multi_threads(f, page_buffer, len);
3757 break;
Juan Quintela294e5a42022-06-21 13:36:11 +02003758 case RAM_SAVE_FLAG_MULTIFD_FLUSH:
3759 multifd_recv_sync_main();
3760 break;
Dr. David Alan Gilberta7180872015-11-05 18:11:11 +00003761 case RAM_SAVE_FLAG_EOS:
3762 /* normal exit */
Juan Quintelad4f34482023-10-11 22:55:48 +02003763 if (migrate_multifd() &&
3764 migrate_multifd_flush_after_each_section()) {
Juan Quintelab05292c2022-06-21 12:21:32 +02003765 multifd_recv_sync_main();
3766 }
Dr. David Alan Gilberta7180872015-11-05 18:11:11 +00003767 break;
3768 default:
Bihong Yu29fccad2020-10-20 11:10:42 +08003769 error_report("Unknown combination of migration flags: 0x%x"
Dr. David Alan Gilberta7180872015-11-05 18:11:11 +00003770 " (postcopy mode)", flags);
3771 ret = -EINVAL;
Peter Xu7a9ddfb2018-02-08 18:31:05 +08003772 break;
Dr. David Alan Gilberta7180872015-11-05 18:11:11 +00003773 }
3774
Wei Yang644acf92019-11-07 20:39:07 +08003775 /* Got the whole host page, wait for decompress before placing. */
3776 if (place_needed) {
3777 ret |= wait_for_decompress_done();
3778 }
3779
Peter Xu7a9ddfb2018-02-08 18:31:05 +08003780 /* Detect for any possible file errors */
3781 if (!ret && qemu_file_get_error(f)) {
3782 ret = qemu_file_get_error(f);
3783 }
3784
3785 if (!ret && place_needed) {
Peter Xu77dadc32022-03-01 16:39:04 +08003786 if (tmp_page->all_zero) {
3787 ret = postcopy_place_page_zero(mis, tmp_page->host_addr, block);
Dr. David Alan Gilberta7180872015-11-05 18:11:11 +00003788 } else {
Peter Xu77dadc32022-03-01 16:39:04 +08003789 ret = postcopy_place_page(mis, tmp_page->host_addr,
3790 place_source, block);
Dr. David Alan Gilberta7180872015-11-05 18:11:11 +00003791 }
David Hildenbrandddf35bd2020-04-21 10:52:56 +02003792 place_needed = false;
Peter Xu77dadc32022-03-01 16:39:04 +08003793 postcopy_temp_page_reset(tmp_page);
Dr. David Alan Gilberta7180872015-11-05 18:11:11 +00003794 }
Dr. David Alan Gilberta7180872015-11-05 18:11:11 +00003795 }
3796
3797 return ret;
3798}
3799
Daniel Henrique Barbozaacab30b2017-11-16 20:35:26 -02003800static bool postcopy_is_running(void)
3801{
3802 PostcopyState ps = postcopy_state_get();
3803 return ps >= POSTCOPY_INCOMING_LISTENING && ps < POSTCOPY_INCOMING_END;
3804}
3805
Zhang Chene6f4aa12018-09-03 12:38:50 +08003806/*
3807 * Flush content of RAM cache into SVM's memory.
3808 * Only flush the pages that be dirtied by PVM or SVM or both.
3809 */
Lukas Straub24fa16f2020-05-11 13:10:51 +02003810void colo_flush_ram_cache(void)
Zhang Chene6f4aa12018-09-03 12:38:50 +08003811{
3812 RAMBlock *block = NULL;
3813 void *dst_host;
3814 void *src_host;
3815 unsigned long offset = 0;
3816
Gavin Shan1e493be2023-05-09 12:21:19 +10003817 memory_global_dirty_log_sync(false);
Lukas Straub9d638402023-05-08 21:10:55 +02003818 qemu_mutex_lock(&ram_state->bitmap_mutex);
Dr. David Alan Gilbert89ac5a12019-10-07 15:36:39 +01003819 WITH_RCU_READ_LOCK_GUARD() {
3820 RAMBLOCK_FOREACH_NOT_IGNORED(block) {
3821 ramblock_sync_dirty_bitmap(ram_state, block);
Zhang Chene6f4aa12018-09-03 12:38:50 +08003822 }
3823 }
3824
Dr. David Alan Gilbert89ac5a12019-10-07 15:36:39 +01003825 trace_colo_flush_ram_cache_begin(ram_state->migration_dirty_pages);
3826 WITH_RCU_READ_LOCK_GUARD() {
3827 block = QLIST_FIRST_RCU(&ram_list.blocks);
3828
3829 while (block) {
Rao, Leia6a83ce2021-11-09 11:04:55 +08003830 unsigned long num = 0;
Dr. David Alan Gilbert89ac5a12019-10-07 15:36:39 +01003831
Rao, Leia6a83ce2021-11-09 11:04:55 +08003832 offset = colo_bitmap_find_dirty(ram_state, block, offset, &num);
David Hildenbrand542147f2021-04-29 13:27:08 +02003833 if (!offset_in_ramblock(block,
3834 ((ram_addr_t)offset) << TARGET_PAGE_BITS)) {
Dr. David Alan Gilbert89ac5a12019-10-07 15:36:39 +01003835 offset = 0;
Rao, Leia6a83ce2021-11-09 11:04:55 +08003836 num = 0;
Dr. David Alan Gilbert89ac5a12019-10-07 15:36:39 +01003837 block = QLIST_NEXT_RCU(block, next);
3838 } else {
Rao, Leia6a83ce2021-11-09 11:04:55 +08003839 unsigned long i = 0;
3840
3841 for (i = 0; i < num; i++) {
3842 migration_bitmap_clear_dirty(ram_state, block, offset + i);
3843 }
Alexey Romko8bba0042020-01-10 14:51:34 +01003844 dst_host = block->host
3845 + (((ram_addr_t)offset) << TARGET_PAGE_BITS);
3846 src_host = block->colo_cache
3847 + (((ram_addr_t)offset) << TARGET_PAGE_BITS);
Rao, Leia6a83ce2021-11-09 11:04:55 +08003848 memcpy(dst_host, src_host, TARGET_PAGE_SIZE * num);
3849 offset += num;
Dr. David Alan Gilbert89ac5a12019-10-07 15:36:39 +01003850 }
3851 }
3852 }
Lukas Straub9d638402023-05-08 21:10:55 +02003853 qemu_mutex_unlock(&ram_state->bitmap_mutex);
Zhang Chene6f4aa12018-09-03 12:38:50 +08003854 trace_colo_flush_ram_cache_end();
3855}
3856
Nikolay Borisov2f5ced52023-10-11 15:45:59 -03003857static int parse_ramblock(QEMUFile *f, RAMBlock *block, ram_addr_t length)
3858{
3859 int ret = 0;
3860 /* ADVISE is earlier, it shows the source has the postcopy capability on */
3861 bool postcopy_advised = migration_incoming_postcopy_advised();
3862
3863 assert(block);
3864
3865 if (!qemu_ram_is_migratable(block)) {
3866 error_report("block %s should not be migrated !", block->idstr);
3867 return -EINVAL;
3868 }
3869
3870 if (length != block->used_length) {
3871 Error *local_err = NULL;
3872
3873 ret = qemu_ram_resize(block, length, &local_err);
3874 if (local_err) {
3875 error_report_err(local_err);
3876 }
3877 }
3878 /* For postcopy we need to check hugepage sizes match */
3879 if (postcopy_advised && migrate_postcopy_ram() &&
3880 block->page_size != qemu_host_page_size) {
3881 uint64_t remote_page_size = qemu_get_be64(f);
3882 if (remote_page_size != block->page_size) {
3883 error_report("Mismatched RAM page size %s "
3884 "(local) %zd != %" PRId64, block->idstr,
3885 block->page_size, remote_page_size);
3886 ret = -EINVAL;
3887 }
3888 }
3889 if (migrate_ignore_shared()) {
3890 hwaddr addr = qemu_get_be64(f);
3891 if (migrate_ram_is_ignored(block) &&
3892 block->mr->addr != addr) {
3893 error_report("Mismatched GPAs for block %s "
3894 "%" PRId64 "!= %" PRId64, block->idstr,
3895 (uint64_t)addr, (uint64_t)block->mr->addr);
3896 ret = -EINVAL;
3897 }
3898 }
3899 ret = rdma_block_notification_handle(f, block->idstr);
3900 if (ret < 0) {
3901 qemu_file_set_error(f, ret);
3902 }
3903
3904 return ret;
3905}
3906
3907static int parse_ramblocks(QEMUFile *f, ram_addr_t total_ram_bytes)
3908{
3909 int ret = 0;
3910
3911 /* Synchronize RAM block list */
3912 while (!ret && total_ram_bytes) {
3913 RAMBlock *block;
3914 char id[256];
3915 ram_addr_t length;
3916 int len = qemu_get_byte(f);
3917
3918 qemu_get_buffer(f, (uint8_t *)id, len);
3919 id[len] = 0;
3920 length = qemu_get_be64(f);
3921
3922 block = qemu_ram_block_by_name(id);
3923 if (block) {
3924 ret = parse_ramblock(f, block, length);
3925 } else {
3926 error_report("Unknown ramblock \"%s\", cannot accept "
3927 "migration", id);
3928 ret = -EINVAL;
3929 }
3930 total_ram_bytes -= length;
3931 }
3932
3933 return ret;
3934}
3935
Wei Yang10da4a32019-07-25 08:20:23 +08003936/**
3937 * ram_load_precopy: load pages in precopy case
3938 *
3939 * Returns 0 for success or -errno in case of error
3940 *
3941 * Called in precopy mode by ram_load().
3942 * rcu_read_lock is taken prior to this being called.
3943 *
3944 * @f: QEMUFile where to send the data
3945 */
3946static int ram_load_precopy(QEMUFile *f)
Juan Quintela56e93d22015-05-07 19:33:31 +02003947{
Peter Xu755e8d72022-03-01 16:39:07 +08003948 MigrationIncomingState *mis = migration_incoming_get_current();
Yury Kotove65cec52019-11-25 16:36:32 +03003949 int flags = 0, ret = 0, invalid_flags = 0, len = 0, i = 0;
Nikolay Borisov2f5ced52023-10-11 15:45:59 -03003950
Juan Quintelaa7a94d12023-03-01 22:03:48 +01003951 if (!migrate_compress()) {
Juan Quintelaedc60122016-11-02 12:40:46 +01003952 invalid_flags |= RAM_SAVE_FLAG_COMPRESS_PAGE;
3953 }
Dr. David Alan Gilberta7180872015-11-05 18:11:11 +00003954
Wei Yang10da4a32019-07-25 08:20:23 +08003955 while (!ret && !(flags & RAM_SAVE_FLAG_EOS)) {
Nikolay Borisov2f5ced52023-10-11 15:45:59 -03003956 ram_addr_t addr;
zhanghailiang03930312020-02-24 14:54:10 +08003957 void *host = NULL, *host_bak = NULL;
Juan Quintela56e93d22015-05-07 19:33:31 +02003958 uint8_t ch;
3959
Yury Kotove65cec52019-11-25 16:36:32 +03003960 /*
3961 * Yield periodically to let main loop run, but an iteration of
3962 * the main loop is expensive, so do it each some iterations
3963 */
3964 if ((i & 32767) == 0 && qemu_in_coroutine()) {
3965 aio_co_schedule(qemu_get_current_aio_context(),
3966 qemu_coroutine_self());
3967 qemu_coroutine_yield();
3968 }
3969 i++;
3970
Juan Quintela56e93d22015-05-07 19:33:31 +02003971 addr = qemu_get_be64(f);
3972 flags = addr & ~TARGET_PAGE_MASK;
3973 addr &= TARGET_PAGE_MASK;
3974
Juan Quintelaedc60122016-11-02 12:40:46 +01003975 if (flags & invalid_flags) {
3976 if (flags & invalid_flags & RAM_SAVE_FLAG_COMPRESS_PAGE) {
3977 error_report("Received an unexpected compressed page");
3978 }
3979
3980 ret = -EINVAL;
3981 break;
3982 }
3983
Juan Quintelabb890ed2017-04-28 09:39:55 +02003984 if (flags & (RAM_SAVE_FLAG_ZERO | RAM_SAVE_FLAG_PAGE |
Dr. David Alan Gilberta776aa12015-11-05 18:10:39 +00003985 RAM_SAVE_FLAG_COMPRESS_PAGE | RAM_SAVE_FLAG_XBZRLE)) {
Peter Xuc01b16e2022-07-07 14:55:04 -04003986 RAMBlock *block = ram_block_from_stream(mis, f, flags,
3987 RAM_CHANNEL_PRECOPY);
zhanghailiang4c4bad42016-01-15 11:37:41 +08003988
zhanghailiang03930312020-02-24 14:54:10 +08003989 host = host_from_ram_block_offset(block, addr);
Zhang Chen13af18f2018-09-03 12:38:48 +08003990 /*
zhanghailiang03930312020-02-24 14:54:10 +08003991 * After going into COLO stage, we should not load the page
3992 * into SVM's memory directly, we put them into colo_cache firstly.
3993 * NOTE: We need to keep a copy of SVM's ram in colo_cache.
3994 * Previously, we copied all these memory in preparing stage of COLO
3995 * while we need to stop VM, which is a time-consuming process.
3996 * Here we optimize it by a trick, back-up every page while in
3997 * migration process while COLO is enabled, though it affects the
3998 * speed of the migration, but it obviously reduce the downtime of
3999 * back-up all SVM'S memory in COLO preparing stage.
Zhang Chen13af18f2018-09-03 12:38:48 +08004000 */
zhanghailiang03930312020-02-24 14:54:10 +08004001 if (migration_incoming_colo_enabled()) {
4002 if (migration_incoming_in_colo_state()) {
4003 /* In COLO stage, put all pages into cache temporarily */
zhanghailiang8af66372020-02-24 14:54:11 +08004004 host = colo_cache_from_block_offset(block, addr, true);
zhanghailiang03930312020-02-24 14:54:10 +08004005 } else {
4006 /*
4007 * In migration stage but before COLO stage,
4008 * Put all pages into both cache and SVM's memory.
4009 */
zhanghailiang8af66372020-02-24 14:54:11 +08004010 host_bak = colo_cache_from_block_offset(block, addr, false);
zhanghailiang03930312020-02-24 14:54:10 +08004011 }
Zhang Chen13af18f2018-09-03 12:38:48 +08004012 }
Dr. David Alan Gilberta776aa12015-11-05 18:10:39 +00004013 if (!host) {
4014 error_report("Illegal RAM offset " RAM_ADDR_FMT, addr);
4015 ret = -EINVAL;
4016 break;
4017 }
Zhang Chen13af18f2018-09-03 12:38:48 +08004018 if (!migration_incoming_in_colo_state()) {
4019 ramblock_recv_bitmap_set(block, host);
4020 }
4021
Dr. David Alan Gilbert1db9d8e2017-04-26 19:37:21 +01004022 trace_ram_load_loop(block->idstr, (uint64_t)addr, flags, host);
Dr. David Alan Gilberta776aa12015-11-05 18:10:39 +00004023 }
4024
Juan Quintela56e93d22015-05-07 19:33:31 +02004025 switch (flags & ~RAM_SAVE_FLAG_CONTINUE) {
4026 case RAM_SAVE_FLAG_MEM_SIZE:
Nikolay Borisov2f5ced52023-10-11 15:45:59 -03004027 ret = parse_ramblocks(f, addr);
Juan Quintela56e93d22015-05-07 19:33:31 +02004028 break;
Dr. David Alan Gilberta776aa12015-11-05 18:10:39 +00004029
Juan Quintelabb890ed2017-04-28 09:39:55 +02004030 case RAM_SAVE_FLAG_ZERO:
Juan Quintela56e93d22015-05-07 19:33:31 +02004031 ch = qemu_get_byte(f);
4032 ram_handle_compressed(host, ch, TARGET_PAGE_SIZE);
4033 break;
Dr. David Alan Gilberta776aa12015-11-05 18:10:39 +00004034
Juan Quintela56e93d22015-05-07 19:33:31 +02004035 case RAM_SAVE_FLAG_PAGE:
Juan Quintela56e93d22015-05-07 19:33:31 +02004036 qemu_get_buffer(f, host, TARGET_PAGE_SIZE);
4037 break;
Juan Quintela56e93d22015-05-07 19:33:31 +02004038
Dr. David Alan Gilberta776aa12015-11-05 18:10:39 +00004039 case RAM_SAVE_FLAG_COMPRESS_PAGE:
Juan Quintela56e93d22015-05-07 19:33:31 +02004040 len = qemu_get_be32(f);
4041 if (len < 0 || len > compressBound(TARGET_PAGE_SIZE)) {
4042 error_report("Invalid compressed data length: %d", len);
4043 ret = -EINVAL;
4044 break;
4045 }
Dr. David Alan Gilbertc1bc6622015-12-16 11:47:38 +00004046 decompress_data_with_multi_threads(f, host, len);
Juan Quintela56e93d22015-05-07 19:33:31 +02004047 break;
Dr. David Alan Gilberta776aa12015-11-05 18:10:39 +00004048
Juan Quintela56e93d22015-05-07 19:33:31 +02004049 case RAM_SAVE_FLAG_XBZRLE:
Juan Quintela56e93d22015-05-07 19:33:31 +02004050 if (load_xbzrle(f, addr, host) < 0) {
4051 error_report("Failed to decompress XBZRLE page at "
4052 RAM_ADDR_FMT, addr);
4053 ret = -EINVAL;
4054 break;
4055 }
4056 break;
Juan Quintela294e5a42022-06-21 13:36:11 +02004057 case RAM_SAVE_FLAG_MULTIFD_FLUSH:
4058 multifd_recv_sync_main();
4059 break;
Juan Quintela56e93d22015-05-07 19:33:31 +02004060 case RAM_SAVE_FLAG_EOS:
4061 /* normal exit */
Juan Quintelad4f34482023-10-11 22:55:48 +02004062 if (migrate_multifd() &&
4063 migrate_multifd_flush_after_each_section()) {
Juan Quintelab05292c2022-06-21 12:21:32 +02004064 multifd_recv_sync_main();
4065 }
Juan Quintela56e93d22015-05-07 19:33:31 +02004066 break;
Juan Quintela5f1e7542023-05-04 13:44:39 +02004067 case RAM_SAVE_FLAG_HOOK:
Juan Quintelab1b38382023-10-11 22:35:23 +02004068 ret = rdma_registration_handle(f);
Juan Quintelaf6d6c082023-10-11 22:35:18 +02004069 if (ret < 0) {
4070 qemu_file_set_error(f, ret);
4071 }
Juan Quintela5f1e7542023-05-04 13:44:39 +02004072 break;
Juan Quintela56e93d22015-05-07 19:33:31 +02004073 default:
Juan Quintela5f1e7542023-05-04 13:44:39 +02004074 error_report("Unknown combination of migration flags: 0x%x", flags);
4075 ret = -EINVAL;
Juan Quintela56e93d22015-05-07 19:33:31 +02004076 }
4077 if (!ret) {
4078 ret = qemu_file_get_error(f);
4079 }
zhanghailiang03930312020-02-24 14:54:10 +08004080 if (!ret && host_bak) {
4081 memcpy(host_bak, host, TARGET_PAGE_SIZE);
4082 }
Juan Quintela56e93d22015-05-07 19:33:31 +02004083 }
4084
Wei Yangca1a6b72019-11-07 20:39:03 +08004085 ret |= wait_for_decompress_done();
Wei Yang10da4a32019-07-25 08:20:23 +08004086 return ret;
4087}
4088
4089static int ram_load(QEMUFile *f, void *opaque, int version_id)
4090{
4091 int ret = 0;
4092 static uint64_t seq_iter;
4093 /*
4094 * If system is running in postcopy mode, page inserts to host memory must
4095 * be atomic
4096 */
4097 bool postcopy_running = postcopy_is_running();
4098
4099 seq_iter++;
4100
4101 if (version_id != 4) {
4102 return -EINVAL;
4103 }
4104
4105 /*
4106 * This RCU critical section can be very long running.
4107 * When RCU reclaims in the code start to become numerous,
4108 * it will be necessary to reduce the granularity of this
4109 * critical section.
4110 */
Dr. David Alan Gilbert89ac5a12019-10-07 15:36:39 +01004111 WITH_RCU_READ_LOCK_GUARD() {
4112 if (postcopy_running) {
Peter Xu36f62f12022-07-07 14:55:02 -04004113 /*
4114 * Note! Here RAM_CHANNEL_PRECOPY is the precopy channel of
4115 * postcopy migration, we have another RAM_CHANNEL_POSTCOPY to
4116 * service fast page faults.
4117 */
4118 ret = ram_load_postcopy(f, RAM_CHANNEL_PRECOPY);
Dr. David Alan Gilbert89ac5a12019-10-07 15:36:39 +01004119 } else {
4120 ret = ram_load_precopy(f);
4121 }
Wei Yang10da4a32019-07-25 08:20:23 +08004122 }
Juan Quintela55c44462017-01-23 22:32:05 +01004123 trace_ram_load_complete(ret, seq_iter);
Zhang Chene6f4aa12018-09-03 12:38:50 +08004124
Juan Quintela56e93d22015-05-07 19:33:31 +02004125 return ret;
4126}
4127
Vladimir Sementsov-Ogievskiyc6467622017-07-10 19:30:14 +03004128static bool ram_has_postcopy(void *opaque)
4129{
Junyan He469dd512018-07-18 15:48:02 +08004130 RAMBlock *rb;
Yury Kotovfbd162e2019-02-15 20:45:46 +03004131 RAMBLOCK_FOREACH_NOT_IGNORED(rb) {
Junyan He469dd512018-07-18 15:48:02 +08004132 if (ramblock_is_pmem(rb)) {
4133 info_report("Block: %s, host: %p is a nvdimm memory, postcopy"
4134 "is not supported now!", rb->idstr, rb->host);
4135 return false;
4136 }
4137 }
4138
Vladimir Sementsov-Ogievskiyc6467622017-07-10 19:30:14 +03004139 return migrate_postcopy_ram();
4140}
4141
Peter Xuedd090c2018-05-02 18:47:32 +08004142/* Sync all the dirty bitmap with destination VM. */
4143static int ram_dirty_bitmap_sync_all(MigrationState *s, RAMState *rs)
4144{
4145 RAMBlock *block;
4146 QEMUFile *file = s->to_dst_file;
Peter Xuedd090c2018-05-02 18:47:32 +08004147
4148 trace_ram_dirty_bitmap_sync_start();
4149
Peter Xu1015ff52023-10-04 18:02:36 -04004150 qatomic_set(&rs->postcopy_bmap_sync_requested, 0);
Yury Kotovfbd162e2019-02-15 20:45:46 +03004151 RAMBLOCK_FOREACH_NOT_IGNORED(block) {
Peter Xuedd090c2018-05-02 18:47:32 +08004152 qemu_savevm_send_recv_bitmap(file, block->idstr);
4153 trace_ram_dirty_bitmap_request(block->idstr);
Peter Xu1015ff52023-10-04 18:02:36 -04004154 qatomic_inc(&rs->postcopy_bmap_sync_requested);
Peter Xuedd090c2018-05-02 18:47:32 +08004155 }
4156
4157 trace_ram_dirty_bitmap_sync_wait();
4158
4159 /* Wait until all the ramblocks' dirty bitmap synced */
Peter Xu1015ff52023-10-04 18:02:36 -04004160 while (qatomic_read(&rs->postcopy_bmap_sync_requested)) {
Peter Xu5e79a4b2023-10-04 18:02:37 -04004161 migration_rp_wait(s);
Peter Xuedd090c2018-05-02 18:47:32 +08004162 }
4163
4164 trace_ram_dirty_bitmap_sync_complete();
4165
4166 return 0;
4167}
4168
Peter Xua335deb2018-05-02 18:47:28 +08004169/*
4170 * Read the received bitmap, revert it as the initial dirty bitmap.
4171 * This is only used when the postcopy migration is paused but wants
4172 * to resume from a middle point.
4173 */
4174int ram_dirty_bitmap_reload(MigrationState *s, RAMBlock *block)
4175{
4176 int ret = -EINVAL;
Peter Xu43044ac2021-07-22 13:58:38 -04004177 /* from_dst_file is always valid because we're within rp_thread */
Peter Xua335deb2018-05-02 18:47:28 +08004178 QEMUFile *file = s->rp_state.from_dst_file;
Philippe Mathieu-Daudé1a36e4c2023-10-11 04:36:27 +02004179 g_autofree unsigned long *le_bitmap = NULL;
4180 unsigned long nbits = block->used_length >> TARGET_PAGE_BITS;
Peter Xua725ef92018-07-10 17:18:55 +08004181 uint64_t local_size = DIV_ROUND_UP(nbits, 8);
Peter Xua335deb2018-05-02 18:47:28 +08004182 uint64_t size, end_mark;
Peter Xu1015ff52023-10-04 18:02:36 -04004183 RAMState *rs = ram_state;
Peter Xua335deb2018-05-02 18:47:28 +08004184
4185 trace_ram_dirty_bitmap_reload_begin(block->idstr);
4186
4187 if (s->state != MIGRATION_STATUS_POSTCOPY_RECOVER) {
4188 error_report("%s: incorrect state %s", __func__,
4189 MigrationStatus_str(s->state));
4190 return -EINVAL;
4191 }
4192
4193 /*
4194 * Note: see comments in ramblock_recv_bitmap_send() on why we
zhaolichang3a4452d2020-09-17 15:50:21 +08004195 * need the endianness conversion, and the paddings.
Peter Xua335deb2018-05-02 18:47:28 +08004196 */
4197 local_size = ROUND_UP(local_size, 8);
4198
4199 /* Add paddings */
4200 le_bitmap = bitmap_new(nbits + BITS_PER_LONG);
4201
4202 size = qemu_get_be64(file);
4203
4204 /* The size of the bitmap should match with our ramblock */
4205 if (size != local_size) {
4206 error_report("%s: ramblock '%s' bitmap size mismatch "
4207 "(0x%"PRIx64" != 0x%"PRIx64")", __func__,
4208 block->idstr, size, local_size);
Philippe Mathieu-Daudé1a36e4c2023-10-11 04:36:27 +02004209 return -EINVAL;
Peter Xua335deb2018-05-02 18:47:28 +08004210 }
4211
4212 size = qemu_get_buffer(file, (uint8_t *)le_bitmap, local_size);
4213 end_mark = qemu_get_be64(file);
4214
4215 ret = qemu_file_get_error(file);
4216 if (ret || size != local_size) {
4217 error_report("%s: read bitmap failed for ramblock '%s': %d"
4218 " (size 0x%"PRIx64", got: 0x%"PRIx64")",
4219 __func__, block->idstr, ret, local_size, size);
Philippe Mathieu-Daudé1a36e4c2023-10-11 04:36:27 +02004220 return -EIO;
Peter Xua335deb2018-05-02 18:47:28 +08004221 }
4222
4223 if (end_mark != RAMBLOCK_RECV_BITMAP_ENDING) {
Philippe Mathieu-Daudéaf3bbbe2020-11-03 12:25:58 +01004224 error_report("%s: ramblock '%s' end mark incorrect: 0x%"PRIx64,
Peter Xua335deb2018-05-02 18:47:28 +08004225 __func__, block->idstr, end_mark);
Philippe Mathieu-Daudé1a36e4c2023-10-11 04:36:27 +02004226 return -EINVAL;
Peter Xua335deb2018-05-02 18:47:28 +08004227 }
4228
4229 /*
zhaolichang3a4452d2020-09-17 15:50:21 +08004230 * Endianness conversion. We are during postcopy (though paused).
Peter Xua335deb2018-05-02 18:47:28 +08004231 * The dirty bitmap won't change. We can directly modify it.
4232 */
4233 bitmap_from_le(block->bmap, le_bitmap, nbits);
4234
4235 /*
4236 * What we received is "received bitmap". Revert it as the initial
4237 * dirty bitmap for this ramblock.
4238 */
4239 bitmap_complement(block->bmap, block->bmap, nbits);
4240
David Hildenbrandbe39b4c2021-10-11 19:53:41 +02004241 /* Clear dirty bits of discarded ranges that we don't want to migrate. */
4242 ramblock_dirty_bitmap_clear_discarded_pages(block);
4243
4244 /* We'll recalculate migration_dirty_pages in ram_state_resume_prepare(). */
Peter Xua335deb2018-05-02 18:47:28 +08004245 trace_ram_dirty_bitmap_reload_complete(block->idstr);
4246
Peter Xu1015ff52023-10-04 18:02:36 -04004247 qatomic_dec(&rs->postcopy_bmap_sync_requested);
4248
Peter Xuedd090c2018-05-02 18:47:32 +08004249 /*
Peter Xu5e79a4b2023-10-04 18:02:37 -04004250 * We succeeded to sync bitmap for current ramblock. Always kick the
4251 * migration thread to check whether all requested bitmaps are
4252 * reloaded. NOTE: it's racy to only kick when requested==0, because
4253 * we don't know whether the migration thread may still be increasing
4254 * it.
Peter Xuedd090c2018-05-02 18:47:32 +08004255 */
Peter Xu5e79a4b2023-10-04 18:02:37 -04004256 migration_rp_kick(s);
Peter Xuedd090c2018-05-02 18:47:32 +08004257
Philippe Mathieu-Daudé1a36e4c2023-10-11 04:36:27 +02004258 return 0;
Peter Xua335deb2018-05-02 18:47:28 +08004259}
4260
Peter Xuedd090c2018-05-02 18:47:32 +08004261static int ram_resume_prepare(MigrationState *s, void *opaque)
4262{
4263 RAMState *rs = *(RAMState **)opaque;
Peter Xu08614f32018-05-02 18:47:33 +08004264 int ret;
Peter Xuedd090c2018-05-02 18:47:32 +08004265
Peter Xu08614f32018-05-02 18:47:33 +08004266 ret = ram_dirty_bitmap_sync_all(s, rs);
4267 if (ret) {
4268 return ret;
4269 }
4270
4271 ram_state_resume_prepare(rs, s->to_dst_file);
4272
4273 return 0;
Peter Xuedd090c2018-05-02 18:47:32 +08004274}
4275
Peter Xu36f62f12022-07-07 14:55:02 -04004276void postcopy_preempt_shutdown_file(MigrationState *s)
4277{
4278 qemu_put_be64(s->postcopy_qemufile_src, RAM_SAVE_FLAG_EOS);
4279 qemu_fflush(s->postcopy_qemufile_src);
4280}
4281
Juan Quintela56e93d22015-05-07 19:33:31 +02004282static SaveVMHandlers savevm_ram_handlers = {
Juan Quintela9907e842017-06-28 11:52:24 +02004283 .save_setup = ram_save_setup,
Juan Quintela56e93d22015-05-07 19:33:31 +02004284 .save_live_iterate = ram_save_iterate,
Dr. David Alan Gilbert763c9062015-11-05 18:11:00 +00004285 .save_live_complete_postcopy = ram_save_complete,
Dr. David Alan Gilberta3e06c32015-11-05 18:10:41 +00004286 .save_live_complete_precopy = ram_save_complete,
Vladimir Sementsov-Ogievskiyc6467622017-07-10 19:30:14 +03004287 .has_postcopy = ram_has_postcopy,
Juan Quintelac8df4a72022-10-03 02:00:03 +02004288 .state_pending_exact = ram_state_pending_exact,
4289 .state_pending_estimate = ram_state_pending_estimate,
Juan Quintela56e93d22015-05-07 19:33:31 +02004290 .load_state = ram_load,
Juan Quintelaf265e0e2017-06-28 11:52:27 +02004291 .save_cleanup = ram_save_cleanup,
4292 .load_setup = ram_load_setup,
4293 .load_cleanup = ram_load_cleanup,
Peter Xuedd090c2018-05-02 18:47:32 +08004294 .resume_prepare = ram_resume_prepare,
Juan Quintela56e93d22015-05-07 19:33:31 +02004295};
4296
David Hildenbrandc7c0e722021-04-29 13:27:02 +02004297static void ram_mig_ram_block_resized(RAMBlockNotifier *n, void *host,
4298 size_t old_size, size_t new_size)
4299{
David Hildenbrandcc61c702021-04-29 13:27:04 +02004300 PostcopyState ps = postcopy_state_get();
David Hildenbrandc7c0e722021-04-29 13:27:02 +02004301 ram_addr_t offset;
4302 RAMBlock *rb = qemu_ram_block_from_host(host, false, &offset);
4303 Error *err = NULL;
4304
Dmitry Frolovf75ed592023-10-10 13:48:53 +03004305 if (!rb) {
4306 error_report("RAM block not found");
4307 return;
4308 }
4309
David Hildenbrandf161c882023-07-06 09:56:08 +02004310 if (migrate_ram_is_ignored(rb)) {
David Hildenbrandc7c0e722021-04-29 13:27:02 +02004311 return;
4312 }
4313
4314 if (!migration_is_idle()) {
4315 /*
4316 * Precopy code on the source cannot deal with the size of RAM blocks
4317 * changing at random points in time - especially after sending the
4318 * RAM block sizes in the migration stream, they must no longer change.
4319 * Abort and indicate a proper reason.
4320 */
4321 error_setg(&err, "RAM block '%s' resized during precopy.", rb->idstr);
Laurent Vivier458fecc2021-09-29 16:43:10 +02004322 migration_cancel(err);
David Hildenbrandc7c0e722021-04-29 13:27:02 +02004323 error_free(err);
David Hildenbrandc7c0e722021-04-29 13:27:02 +02004324 }
David Hildenbrandcc61c702021-04-29 13:27:04 +02004325
4326 switch (ps) {
4327 case POSTCOPY_INCOMING_ADVISE:
4328 /*
4329 * Update what ram_postcopy_incoming_init()->init_range() does at the
4330 * time postcopy was advised. Syncing RAM blocks with the source will
4331 * result in RAM resizes.
4332 */
4333 if (old_size < new_size) {
4334 if (ram_discard_range(rb->idstr, old_size, new_size - old_size)) {
4335 error_report("RAM block '%s' discard of resized RAM failed",
4336 rb->idstr);
4337 }
4338 }
David Hildenbrand898ba902021-04-29 13:27:06 +02004339 rb->postcopy_length = new_size;
David Hildenbrandcc61c702021-04-29 13:27:04 +02004340 break;
4341 case POSTCOPY_INCOMING_NONE:
4342 case POSTCOPY_INCOMING_RUNNING:
4343 case POSTCOPY_INCOMING_END:
4344 /*
4345 * Once our guest is running, postcopy does no longer care about
4346 * resizes. When growing, the new memory was not available on the
4347 * source, no handler needed.
4348 */
4349 break;
4350 default:
4351 error_report("RAM block '%s' resized during postcopy state: %d",
4352 rb->idstr, ps);
4353 exit(-1);
4354 }
David Hildenbrandc7c0e722021-04-29 13:27:02 +02004355}
4356
4357static RAMBlockNotifier ram_mig_ram_notifier = {
4358 .ram_block_resized = ram_mig_ram_block_resized,
4359};
4360
Juan Quintela56e93d22015-05-07 19:33:31 +02004361void ram_mig_init(void)
4362{
4363 qemu_mutex_init(&XBZRLE.lock);
Dr. David Alan Gilbertce62df52019-08-22 12:54:33 +01004364 register_savevm_live("ram", 0, 4, &savevm_ram_handlers, &ram_state);
David Hildenbrandc7c0e722021-04-29 13:27:02 +02004365 ram_block_notifier_add(&ram_mig_ram_notifier);
Juan Quintela56e93d22015-05-07 19:33:31 +02004366}