blob: 5a109efedabdb59e4e86c2c2714f002e324dc98c [file] [log] [blame]
Juan Quintela56e93d22015-05-07 19:33:31 +02001/*
2 * QEMU System Emulator
3 *
4 * Copyright (c) 2003-2008 Fabrice Bellard
Juan Quintela76cc7b52015-05-08 13:20:21 +02005 * Copyright (c) 2011-2015 Red Hat Inc
6 *
7 * Authors:
8 * Juan Quintela <quintela@redhat.com>
Juan Quintela56e93d22015-05-07 19:33:31 +02009 *
10 * Permission is hereby granted, free of charge, to any person obtaining a copy
11 * of this software and associated documentation files (the "Software"), to deal
12 * in the Software without restriction, including without limitation the rights
13 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
14 * copies of the Software, and to permit persons to whom the Software is
15 * furnished to do so, subject to the following conditions:
16 *
17 * The above copyright notice and this permission notice shall be included in
18 * all copies or substantial portions of the Software.
19 *
20 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
21 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
23 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
24 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
25 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
26 * THE SOFTWARE.
27 */
Peter Maydell1393a482016-01-26 18:16:54 +000028#include "qemu/osdep.h"
Paolo Bonzini33c11872016-03-15 16:58:45 +010029#include "cpu.h"
Juan Quintela56e93d22015-05-07 19:33:31 +020030#include <zlib.h>
Dr. David Alan Gilbert4addcd42015-12-16 11:47:36 +000031#include "qapi-event.h"
Veronia Bahaaf348b6d2016-03-20 19:16:19 +020032#include "qemu/cutils.h"
Juan Quintela56e93d22015-05-07 19:33:31 +020033#include "qemu/bitops.h"
34#include "qemu/bitmap.h"
Juan Quintela7205c9e2015-05-08 13:54:36 +020035#include "qemu/main-loop.h"
Juan Quintela709e3fe2017-04-05 21:47:50 +020036#include "xbzrle.h"
Juan Quintela7b1e1a22017-04-17 20:26:27 +020037#include "ram.h"
Juan Quintela6666c962017-04-24 20:07:27 +020038#include "migration.h"
Juan Quintelaf2a8f0a2017-04-24 13:42:55 +020039#include "migration/register.h"
Juan Quintela7b1e1a22017-04-17 20:26:27 +020040#include "migration/misc.h"
Juan Quintela08a0aee2017-04-20 18:52:18 +020041#include "qemu-file.h"
Juan Quintelabe07b0a2017-04-20 13:12:24 +020042#include "postcopy-ram.h"
Juan Quintela56e93d22015-05-07 19:33:31 +020043#include "migration/page_cache.h"
Juan Quintela56e93d22015-05-07 19:33:31 +020044#include "qemu/error-report.h"
Juan Quintela8acabf62017-10-05 22:00:31 +020045#include "qapi/qmp/qerror.h"
Juan Quintela56e93d22015-05-07 19:33:31 +020046#include "trace.h"
Juan Quintela56e93d22015-05-07 19:33:31 +020047#include "exec/ram_addr.h"
Alexey Perevalovf9494612017-10-05 14:13:20 +030048#include "exec/target_page.h"
Juan Quintela56e93d22015-05-07 19:33:31 +020049#include "qemu/rcu_queue.h"
zhanghailianga91246c2016-10-27 14:42:59 +080050#include "migration/colo.h"
Peter Lieven9ac78b62017-09-26 12:33:16 +020051#include "migration/block.h"
Juan Quintela56e93d22015-05-07 19:33:31 +020052
Juan Quintela56e93d22015-05-07 19:33:31 +020053/***********************************************************/
54/* ram save/restore */
55
Juan Quintelabb890ed2017-04-28 09:39:55 +020056/* RAM_SAVE_FLAG_ZERO used to be named RAM_SAVE_FLAG_COMPRESS, it
57 * worked for pages that where filled with the same char. We switched
58 * it to only search for the zero value. And to avoid confusion with
59 * RAM_SSAVE_FLAG_COMPRESS_PAGE just rename it.
60 */
61
Juan Quintela56e93d22015-05-07 19:33:31 +020062#define RAM_SAVE_FLAG_FULL 0x01 /* Obsolete, not used anymore */
Juan Quintelabb890ed2017-04-28 09:39:55 +020063#define RAM_SAVE_FLAG_ZERO 0x02
Juan Quintela56e93d22015-05-07 19:33:31 +020064#define RAM_SAVE_FLAG_MEM_SIZE 0x04
65#define RAM_SAVE_FLAG_PAGE 0x08
66#define RAM_SAVE_FLAG_EOS 0x10
67#define RAM_SAVE_FLAG_CONTINUE 0x20
68#define RAM_SAVE_FLAG_XBZRLE 0x40
69/* 0x80 is reserved in migration.h start with 0x100 next */
70#define RAM_SAVE_FLAG_COMPRESS_PAGE 0x100
71
Juan Quintela56e93d22015-05-07 19:33:31 +020072static inline bool is_zero_range(uint8_t *p, uint64_t size)
73{
Richard Hendersona1febc42016-08-29 11:46:14 -070074 return buffer_is_zero(p, size);
Juan Quintela56e93d22015-05-07 19:33:31 +020075}
76
Juan Quintela93604472017-06-06 19:49:03 +020077XBZRLECacheStats xbzrle_counters;
78
Juan Quintela56e93d22015-05-07 19:33:31 +020079/* struct contains XBZRLE cache and a static page
80 used by the compression */
81static struct {
82 /* buffer used for XBZRLE encoding */
83 uint8_t *encoded_buf;
84 /* buffer for storing page content */
85 uint8_t *current_buf;
86 /* Cache for XBZRLE, Protected by lock. */
87 PageCache *cache;
88 QemuMutex lock;
Juan Quintelac00e0922017-05-09 16:22:01 +020089 /* it will store a page full of zeros */
90 uint8_t *zero_target_page;
Juan Quintelaf265e0e2017-06-28 11:52:27 +020091 /* buffer used for XBZRLE decoding */
92 uint8_t *decoded_buf;
Juan Quintela56e93d22015-05-07 19:33:31 +020093} XBZRLE;
94
Juan Quintela56e93d22015-05-07 19:33:31 +020095static void XBZRLE_cache_lock(void)
96{
97 if (migrate_use_xbzrle())
98 qemu_mutex_lock(&XBZRLE.lock);
99}
100
101static void XBZRLE_cache_unlock(void)
102{
103 if (migrate_use_xbzrle())
104 qemu_mutex_unlock(&XBZRLE.lock);
105}
106
Juan Quintela3d0684b2017-03-23 15:06:39 +0100107/**
108 * xbzrle_cache_resize: resize the xbzrle cache
109 *
110 * This function is called from qmp_migrate_set_cache_size in main
111 * thread, possibly while a migration is in progress. A running
112 * migration may be using the cache and might finish during this call,
113 * hence changes to the cache are protected by XBZRLE.lock().
114 *
Juan Quintelac9dede22017-10-06 23:03:55 +0200115 * Returns 0 for success or -1 for error
Juan Quintela3d0684b2017-03-23 15:06:39 +0100116 *
117 * @new_size: new cache size
Juan Quintela8acabf62017-10-05 22:00:31 +0200118 * @errp: set *errp if the check failed, with reason
Juan Quintela56e93d22015-05-07 19:33:31 +0200119 */
Juan Quintelac9dede22017-10-06 23:03:55 +0200120int xbzrle_cache_resize(int64_t new_size, Error **errp)
Juan Quintela56e93d22015-05-07 19:33:31 +0200121{
122 PageCache *new_cache;
Juan Quintelac9dede22017-10-06 23:03:55 +0200123 int64_t ret = 0;
Juan Quintela56e93d22015-05-07 19:33:31 +0200124
Juan Quintela8acabf62017-10-05 22:00:31 +0200125 /* Check for truncation */
126 if (new_size != (size_t)new_size) {
127 error_setg(errp, QERR_INVALID_PARAMETER_VALUE, "cache size",
128 "exceeding address space");
129 return -1;
130 }
131
Juan Quintela2a313e52017-10-06 23:00:12 +0200132 if (new_size == migrate_xbzrle_cache_size()) {
133 /* nothing to do */
Juan Quintelac9dede22017-10-06 23:03:55 +0200134 return 0;
Juan Quintela2a313e52017-10-06 23:00:12 +0200135 }
136
Juan Quintela56e93d22015-05-07 19:33:31 +0200137 XBZRLE_cache_lock();
138
139 if (XBZRLE.cache != NULL) {
Juan Quintela80f8dfd2017-10-06 22:30:45 +0200140 new_cache = cache_init(new_size, TARGET_PAGE_SIZE, errp);
Juan Quintela56e93d22015-05-07 19:33:31 +0200141 if (!new_cache) {
Juan Quintela56e93d22015-05-07 19:33:31 +0200142 ret = -1;
143 goto out;
144 }
145
146 cache_fini(XBZRLE.cache);
147 XBZRLE.cache = new_cache;
148 }
Juan Quintela56e93d22015-05-07 19:33:31 +0200149out:
150 XBZRLE_cache_unlock();
151 return ret;
152}
153
Alexey Perevalovf9494612017-10-05 14:13:20 +0300154static void ramblock_recv_map_init(void)
155{
156 RAMBlock *rb;
157
158 RAMBLOCK_FOREACH(rb) {
159 assert(!rb->receivedmap);
160 rb->receivedmap = bitmap_new(rb->max_length >> qemu_target_page_bits());
161 }
162}
163
164int ramblock_recv_bitmap_test(RAMBlock *rb, void *host_addr)
165{
166 return test_bit(ramblock_recv_bitmap_offset(host_addr, rb),
167 rb->receivedmap);
168}
169
170void ramblock_recv_bitmap_set(RAMBlock *rb, void *host_addr)
171{
172 set_bit_atomic(ramblock_recv_bitmap_offset(host_addr, rb), rb->receivedmap);
173}
174
175void ramblock_recv_bitmap_set_range(RAMBlock *rb, void *host_addr,
176 size_t nr)
177{
178 bitmap_set_atomic(rb->receivedmap,
179 ramblock_recv_bitmap_offset(host_addr, rb),
180 nr);
181}
182
Juan Quintelaec481c62017-03-20 22:12:40 +0100183/*
184 * An outstanding page request, on the source, having been received
185 * and queued
186 */
187struct RAMSrcPageRequest {
188 RAMBlock *rb;
189 hwaddr offset;
190 hwaddr len;
191
192 QSIMPLEQ_ENTRY(RAMSrcPageRequest) next_req;
193};
194
Juan Quintela6f37bb82017-03-13 19:26:29 +0100195/* State of RAM for migration */
196struct RAMState {
Juan Quintela204b88b2017-03-15 09:16:57 +0100197 /* QEMUFile used for this migration */
198 QEMUFile *f;
Juan Quintela6f37bb82017-03-13 19:26:29 +0100199 /* Last block that we have visited searching for dirty pages */
200 RAMBlock *last_seen_block;
201 /* Last block from where we have sent data */
202 RAMBlock *last_sent_block;
Juan Quintela269ace22017-03-21 15:23:31 +0100203 /* Last dirty target page we have sent */
204 ram_addr_t last_page;
Juan Quintela6f37bb82017-03-13 19:26:29 +0100205 /* last ram version we have seen */
206 uint32_t last_version;
207 /* We are in the first round */
208 bool ram_bulk_stage;
Juan Quintela8d820d62017-03-13 19:35:50 +0100209 /* How many times we have dirty too many pages */
210 int dirty_rate_high_cnt;
Juan Quintelaf664da82017-03-13 19:44:57 +0100211 /* these variables are used for bitmap sync */
212 /* last time we did a full bitmap_sync */
213 int64_t time_last_bitmap_sync;
Juan Quintelaeac74152017-03-28 14:59:01 +0200214 /* bytes transferred at start_time */
Juan Quintelac4bdf0c2017-03-28 14:59:54 +0200215 uint64_t bytes_xfer_prev;
Juan Quintelaa66cd902017-03-28 15:02:43 +0200216 /* number of dirty pages since start_time */
Juan Quintela68908ed2017-03-28 15:05:53 +0200217 uint64_t num_dirty_pages_period;
Juan Quintelab5833fd2017-03-13 19:49:19 +0100218 /* xbzrle misses since the beginning of the period */
219 uint64_t xbzrle_cache_miss_prev;
Juan Quintela36040d92017-03-13 19:51:13 +0100220 /* number of iterations at the beginning of period */
221 uint64_t iterations_prev;
Juan Quintela23b28c32017-03-13 20:51:34 +0100222 /* Iterations since start */
223 uint64_t iterations;
Juan Quintela93604472017-06-06 19:49:03 +0200224 /* number of dirty bits in the bitmap */
Peter Xu2dfaf122017-08-02 17:41:19 +0800225 uint64_t migration_dirty_pages;
226 /* protects modification of the bitmap */
Juan Quintela108cfae2017-03-13 21:38:09 +0100227 QemuMutex bitmap_mutex;
Juan Quintela68a098f2017-03-14 13:48:42 +0100228 /* The RAMBlock used in the last src_page_requests */
229 RAMBlock *last_req_rb;
Juan Quintelaec481c62017-03-20 22:12:40 +0100230 /* Queue of outstanding page requests from the destination */
231 QemuMutex src_page_req_mutex;
232 QSIMPLEQ_HEAD(src_page_requests, RAMSrcPageRequest) src_page_requests;
Juan Quintela6f37bb82017-03-13 19:26:29 +0100233};
234typedef struct RAMState RAMState;
235
Juan Quintela53518d92017-05-04 11:46:24 +0200236static RAMState *ram_state;
Juan Quintela6f37bb82017-03-13 19:26:29 +0100237
Juan Quintela9edabd42017-03-14 12:02:16 +0100238uint64_t ram_bytes_remaining(void)
239{
Dr. David Alan Gilbertbae416e2017-12-15 11:51:23 +0000240 return ram_state ? (ram_state->migration_dirty_pages * TARGET_PAGE_SIZE) :
241 0;
Juan Quintela9edabd42017-03-14 12:02:16 +0100242}
243
Juan Quintela93604472017-06-06 19:49:03 +0200244MigrationStats ram_counters;
Juan Quintela96506892017-03-14 18:41:03 +0100245
Dr. David Alan Gilbertb8fb8cb2015-09-23 15:27:10 +0100246/* used by the search for pages to send */
247struct PageSearchStatus {
248 /* Current block being searched */
249 RAMBlock *block;
Juan Quintelaa935e302017-03-21 15:36:51 +0100250 /* Current page to search from */
251 unsigned long page;
Dr. David Alan Gilbertb8fb8cb2015-09-23 15:27:10 +0100252 /* Set once we wrap around */
253 bool complete_round;
254};
255typedef struct PageSearchStatus PageSearchStatus;
256
Juan Quintela56e93d22015-05-07 19:33:31 +0200257struct CompressParam {
Juan Quintela56e93d22015-05-07 19:33:31 +0200258 bool done;
Liang Li90e56fb2016-05-05 15:32:56 +0800259 bool quit;
Juan Quintela56e93d22015-05-07 19:33:31 +0200260 QEMUFile *file;
261 QemuMutex mutex;
262 QemuCond cond;
263 RAMBlock *block;
264 ram_addr_t offset;
265};
266typedef struct CompressParam CompressParam;
267
268struct DecompressParam {
Liang Li73a89122016-05-05 15:32:51 +0800269 bool done;
Liang Li90e56fb2016-05-05 15:32:56 +0800270 bool quit;
Juan Quintela56e93d22015-05-07 19:33:31 +0200271 QemuMutex mutex;
272 QemuCond cond;
273 void *des;
Peter Maydelld341d9f2016-01-22 15:09:21 +0000274 uint8_t *compbuf;
Juan Quintela56e93d22015-05-07 19:33:31 +0200275 int len;
276};
277typedef struct DecompressParam DecompressParam;
278
279static CompressParam *comp_param;
280static QemuThread *compress_threads;
281/* comp_done_cond is used to wake up the migration thread when
282 * one of the compression threads has finished the compression.
283 * comp_done_lock is used to co-work with comp_done_cond.
284 */
Liang Li0d9f9a52016-05-05 15:32:59 +0800285static QemuMutex comp_done_lock;
286static QemuCond comp_done_cond;
Juan Quintela56e93d22015-05-07 19:33:31 +0200287/* The empty QEMUFileOps will be used by file in CompressParam */
288static const QEMUFileOps empty_ops = { };
289
Juan Quintela56e93d22015-05-07 19:33:31 +0200290static DecompressParam *decomp_param;
291static QemuThread *decompress_threads;
Liang Li73a89122016-05-05 15:32:51 +0800292static QemuMutex decomp_done_lock;
293static QemuCond decomp_done_cond;
Juan Quintela56e93d22015-05-07 19:33:31 +0200294
Liang Lia7a9a882016-05-05 15:32:57 +0800295static int do_compress_ram_page(QEMUFile *f, RAMBlock *block,
296 ram_addr_t offset);
Juan Quintela56e93d22015-05-07 19:33:31 +0200297
298static void *do_data_compress(void *opaque)
299{
300 CompressParam *param = opaque;
Liang Lia7a9a882016-05-05 15:32:57 +0800301 RAMBlock *block;
302 ram_addr_t offset;
Juan Quintela56e93d22015-05-07 19:33:31 +0200303
Liang Lia7a9a882016-05-05 15:32:57 +0800304 qemu_mutex_lock(&param->mutex);
Liang Li90e56fb2016-05-05 15:32:56 +0800305 while (!param->quit) {
Liang Lia7a9a882016-05-05 15:32:57 +0800306 if (param->block) {
307 block = param->block;
308 offset = param->offset;
309 param->block = NULL;
310 qemu_mutex_unlock(&param->mutex);
311
312 do_compress_ram_page(param->file, block, offset);
313
Liang Li0d9f9a52016-05-05 15:32:59 +0800314 qemu_mutex_lock(&comp_done_lock);
Liang Lia7a9a882016-05-05 15:32:57 +0800315 param->done = true;
Liang Li0d9f9a52016-05-05 15:32:59 +0800316 qemu_cond_signal(&comp_done_cond);
317 qemu_mutex_unlock(&comp_done_lock);
Liang Lia7a9a882016-05-05 15:32:57 +0800318
319 qemu_mutex_lock(&param->mutex);
320 } else {
Juan Quintela56e93d22015-05-07 19:33:31 +0200321 qemu_cond_wait(&param->cond, &param->mutex);
322 }
Juan Quintela56e93d22015-05-07 19:33:31 +0200323 }
Liang Lia7a9a882016-05-05 15:32:57 +0800324 qemu_mutex_unlock(&param->mutex);
Juan Quintela56e93d22015-05-07 19:33:31 +0200325
326 return NULL;
327}
328
329static inline void terminate_compression_threads(void)
330{
331 int idx, thread_count;
332
333 thread_count = migrate_compress_threads();
Juan Quintela3d0684b2017-03-23 15:06:39 +0100334
Juan Quintela56e93d22015-05-07 19:33:31 +0200335 for (idx = 0; idx < thread_count; idx++) {
336 qemu_mutex_lock(&comp_param[idx].mutex);
Liang Li90e56fb2016-05-05 15:32:56 +0800337 comp_param[idx].quit = true;
Juan Quintela56e93d22015-05-07 19:33:31 +0200338 qemu_cond_signal(&comp_param[idx].cond);
339 qemu_mutex_unlock(&comp_param[idx].mutex);
340 }
341}
342
Juan Quintelaf0afa332017-06-28 11:52:28 +0200343static void compress_threads_save_cleanup(void)
Juan Quintela56e93d22015-05-07 19:33:31 +0200344{
345 int i, thread_count;
346
347 if (!migrate_use_compression()) {
348 return;
349 }
350 terminate_compression_threads();
351 thread_count = migrate_compress_threads();
352 for (i = 0; i < thread_count; i++) {
353 qemu_thread_join(compress_threads + i);
354 qemu_fclose(comp_param[i].file);
355 qemu_mutex_destroy(&comp_param[i].mutex);
356 qemu_cond_destroy(&comp_param[i].cond);
357 }
Liang Li0d9f9a52016-05-05 15:32:59 +0800358 qemu_mutex_destroy(&comp_done_lock);
359 qemu_cond_destroy(&comp_done_cond);
Juan Quintela56e93d22015-05-07 19:33:31 +0200360 g_free(compress_threads);
361 g_free(comp_param);
Juan Quintela56e93d22015-05-07 19:33:31 +0200362 compress_threads = NULL;
363 comp_param = NULL;
Juan Quintela56e93d22015-05-07 19:33:31 +0200364}
365
Juan Quintelaf0afa332017-06-28 11:52:28 +0200366static void compress_threads_save_setup(void)
Juan Quintela56e93d22015-05-07 19:33:31 +0200367{
368 int i, thread_count;
369
370 if (!migrate_use_compression()) {
371 return;
372 }
Juan Quintela56e93d22015-05-07 19:33:31 +0200373 thread_count = migrate_compress_threads();
374 compress_threads = g_new0(QemuThread, thread_count);
375 comp_param = g_new0(CompressParam, thread_count);
Liang Li0d9f9a52016-05-05 15:32:59 +0800376 qemu_cond_init(&comp_done_cond);
377 qemu_mutex_init(&comp_done_lock);
Juan Quintela56e93d22015-05-07 19:33:31 +0200378 for (i = 0; i < thread_count; i++) {
Cao jine110aa92016-07-29 15:10:31 +0800379 /* comp_param[i].file is just used as a dummy buffer to save data,
380 * set its ops to empty.
Juan Quintela56e93d22015-05-07 19:33:31 +0200381 */
382 comp_param[i].file = qemu_fopen_ops(NULL, &empty_ops);
383 comp_param[i].done = true;
Liang Li90e56fb2016-05-05 15:32:56 +0800384 comp_param[i].quit = false;
Juan Quintela56e93d22015-05-07 19:33:31 +0200385 qemu_mutex_init(&comp_param[i].mutex);
386 qemu_cond_init(&comp_param[i].cond);
387 qemu_thread_create(compress_threads + i, "compress",
388 do_data_compress, comp_param + i,
389 QEMU_THREAD_JOINABLE);
390 }
391}
392
Juan Quintelaf986c3d2016-01-14 16:52:55 +0100393/* Multiple fd's */
394
395struct MultiFDSendParams {
396 uint8_t id;
397 char *name;
398 QemuThread thread;
399 QemuSemaphore sem;
400 QemuMutex mutex;
401 bool quit;
402};
403typedef struct MultiFDSendParams MultiFDSendParams;
404
405struct {
406 MultiFDSendParams *params;
407 /* number of created threads */
408 int count;
409} *multifd_send_state;
410
411static void terminate_multifd_send_threads(Error *errp)
412{
413 int i;
414
415 for (i = 0; i < multifd_send_state->count; i++) {
416 MultiFDSendParams *p = &multifd_send_state->params[i];
417
418 qemu_mutex_lock(&p->mutex);
419 p->quit = true;
420 qemu_sem_post(&p->sem);
421 qemu_mutex_unlock(&p->mutex);
422 }
423}
424
425int multifd_save_cleanup(Error **errp)
426{
427 int i;
428 int ret = 0;
429
430 if (!migrate_use_multifd()) {
431 return 0;
432 }
433 terminate_multifd_send_threads(NULL);
434 for (i = 0; i < multifd_send_state->count; i++) {
435 MultiFDSendParams *p = &multifd_send_state->params[i];
436
437 qemu_thread_join(&p->thread);
438 qemu_mutex_destroy(&p->mutex);
439 qemu_sem_destroy(&p->sem);
440 g_free(p->name);
441 p->name = NULL;
442 }
443 g_free(multifd_send_state->params);
444 multifd_send_state->params = NULL;
445 g_free(multifd_send_state);
446 multifd_send_state = NULL;
447 return ret;
448}
449
450static void *multifd_send_thread(void *opaque)
451{
452 MultiFDSendParams *p = opaque;
453
454 while (true) {
455 qemu_mutex_lock(&p->mutex);
456 if (p->quit) {
457 qemu_mutex_unlock(&p->mutex);
458 break;
459 }
460 qemu_mutex_unlock(&p->mutex);
461 qemu_sem_wait(&p->sem);
462 }
463
464 return NULL;
465}
466
467int multifd_save_setup(void)
468{
469 int thread_count;
470 uint8_t i;
471
472 if (!migrate_use_multifd()) {
473 return 0;
474 }
475 thread_count = migrate_multifd_channels();
476 multifd_send_state = g_malloc0(sizeof(*multifd_send_state));
477 multifd_send_state->params = g_new0(MultiFDSendParams, thread_count);
478 multifd_send_state->count = 0;
479 for (i = 0; i < thread_count; i++) {
480 MultiFDSendParams *p = &multifd_send_state->params[i];
481
482 qemu_mutex_init(&p->mutex);
483 qemu_sem_init(&p->sem, 0);
484 p->quit = false;
485 p->id = i;
486 p->name = g_strdup_printf("multifdsend_%d", i);
487 qemu_thread_create(&p->thread, p->name, multifd_send_thread, p,
488 QEMU_THREAD_JOINABLE);
489
490 multifd_send_state->count++;
491 }
492 return 0;
493}
494
495struct MultiFDRecvParams {
496 uint8_t id;
497 char *name;
498 QemuThread thread;
499 QemuSemaphore sem;
500 QemuMutex mutex;
501 bool quit;
502};
503typedef struct MultiFDRecvParams MultiFDRecvParams;
504
505struct {
506 MultiFDRecvParams *params;
507 /* number of created threads */
508 int count;
509} *multifd_recv_state;
510
511static void terminate_multifd_recv_threads(Error *errp)
512{
513 int i;
514
515 for (i = 0; i < multifd_recv_state->count; i++) {
516 MultiFDRecvParams *p = &multifd_recv_state->params[i];
517
518 qemu_mutex_lock(&p->mutex);
519 p->quit = true;
520 qemu_sem_post(&p->sem);
521 qemu_mutex_unlock(&p->mutex);
522 }
523}
524
525int multifd_load_cleanup(Error **errp)
526{
527 int i;
528 int ret = 0;
529
530 if (!migrate_use_multifd()) {
531 return 0;
532 }
533 terminate_multifd_recv_threads(NULL);
534 for (i = 0; i < multifd_recv_state->count; i++) {
535 MultiFDRecvParams *p = &multifd_recv_state->params[i];
536
537 qemu_thread_join(&p->thread);
538 qemu_mutex_destroy(&p->mutex);
539 qemu_sem_destroy(&p->sem);
540 g_free(p->name);
541 p->name = NULL;
542 }
543 g_free(multifd_recv_state->params);
544 multifd_recv_state->params = NULL;
545 g_free(multifd_recv_state);
546 multifd_recv_state = NULL;
547
548 return ret;
549}
550
551static void *multifd_recv_thread(void *opaque)
552{
553 MultiFDRecvParams *p = opaque;
554
555 while (true) {
556 qemu_mutex_lock(&p->mutex);
557 if (p->quit) {
558 qemu_mutex_unlock(&p->mutex);
559 break;
560 }
561 qemu_mutex_unlock(&p->mutex);
562 qemu_sem_wait(&p->sem);
563 }
564
565 return NULL;
566}
567
568int multifd_load_setup(void)
569{
570 int thread_count;
571 uint8_t i;
572
573 if (!migrate_use_multifd()) {
574 return 0;
575 }
576 thread_count = migrate_multifd_channels();
577 multifd_recv_state = g_malloc0(sizeof(*multifd_recv_state));
578 multifd_recv_state->params = g_new0(MultiFDRecvParams, thread_count);
579 multifd_recv_state->count = 0;
580 for (i = 0; i < thread_count; i++) {
581 MultiFDRecvParams *p = &multifd_recv_state->params[i];
582
583 qemu_mutex_init(&p->mutex);
584 qemu_sem_init(&p->sem, 0);
585 p->quit = false;
586 p->id = i;
587 p->name = g_strdup_printf("multifdrecv_%d", i);
588 qemu_thread_create(&p->thread, p->name, multifd_recv_thread, p,
589 QEMU_THREAD_JOINABLE);
590 multifd_recv_state->count++;
591 }
592 return 0;
593}
594
Juan Quintela56e93d22015-05-07 19:33:31 +0200595/**
Juan Quintela3d0684b2017-03-23 15:06:39 +0100596 * save_page_header: write page header to wire
Juan Quintela56e93d22015-05-07 19:33:31 +0200597 *
598 * If this is the 1st block, it also writes the block identification
599 *
Juan Quintela3d0684b2017-03-23 15:06:39 +0100600 * Returns the number of bytes written
Juan Quintela56e93d22015-05-07 19:33:31 +0200601 *
602 * @f: QEMUFile where to send the data
603 * @block: block that contains the page we want to send
604 * @offset: offset inside the block for the page
605 * in the lower bits, it contains flags
606 */
Juan Quintela2bf3aa82017-05-10 13:28:13 +0200607static size_t save_page_header(RAMState *rs, QEMUFile *f, RAMBlock *block,
608 ram_addr_t offset)
Juan Quintela56e93d22015-05-07 19:33:31 +0200609{
Liang Li9f5f3802015-07-13 17:34:10 +0800610 size_t size, len;
Juan Quintela56e93d22015-05-07 19:33:31 +0200611
Juan Quintela24795692017-03-21 11:45:01 +0100612 if (block == rs->last_sent_block) {
613 offset |= RAM_SAVE_FLAG_CONTINUE;
614 }
Juan Quintela2bf3aa82017-05-10 13:28:13 +0200615 qemu_put_be64(f, offset);
Juan Quintela56e93d22015-05-07 19:33:31 +0200616 size = 8;
617
618 if (!(offset & RAM_SAVE_FLAG_CONTINUE)) {
Liang Li9f5f3802015-07-13 17:34:10 +0800619 len = strlen(block->idstr);
Juan Quintela2bf3aa82017-05-10 13:28:13 +0200620 qemu_put_byte(f, len);
621 qemu_put_buffer(f, (uint8_t *)block->idstr, len);
Liang Li9f5f3802015-07-13 17:34:10 +0800622 size += 1 + len;
Juan Quintela24795692017-03-21 11:45:01 +0100623 rs->last_sent_block = block;
Juan Quintela56e93d22015-05-07 19:33:31 +0200624 }
625 return size;
626}
627
Juan Quintela3d0684b2017-03-23 15:06:39 +0100628/**
629 * mig_throttle_guest_down: throotle down the guest
630 *
631 * Reduce amount of guest cpu execution to hopefully slow down memory
632 * writes. If guest dirty memory rate is reduced below the rate at
633 * which we can transfer pages to the destination then we should be
634 * able to complete migration. Some workloads dirty memory way too
635 * fast and will not effectively converge, even with auto-converge.
Jason J. Herne070afca2015-09-08 13:12:35 -0400636 */
637static void mig_throttle_guest_down(void)
638{
639 MigrationState *s = migrate_get_current();
Daniel P. Berrange2594f562016-04-27 11:05:14 +0100640 uint64_t pct_initial = s->parameters.cpu_throttle_initial;
641 uint64_t pct_icrement = s->parameters.cpu_throttle_increment;
Jason J. Herne070afca2015-09-08 13:12:35 -0400642
643 /* We have not started throttling yet. Let's start it. */
644 if (!cpu_throttle_active()) {
645 cpu_throttle_set(pct_initial);
646 } else {
647 /* Throttling already on, just increase the rate */
648 cpu_throttle_set(cpu_throttle_get_percentage() + pct_icrement);
649 }
650}
651
Juan Quintela3d0684b2017-03-23 15:06:39 +0100652/**
653 * xbzrle_cache_zero_page: insert a zero page in the XBZRLE cache
654 *
Juan Quintela6f37bb82017-03-13 19:26:29 +0100655 * @rs: current RAM state
Juan Quintela3d0684b2017-03-23 15:06:39 +0100656 * @current_addr: address for the zero page
657 *
658 * Update the xbzrle cache to reflect a page that's been sent as all 0.
Juan Quintela56e93d22015-05-07 19:33:31 +0200659 * The important thing is that a stale (not-yet-0'd) page be replaced
660 * by the new data.
661 * As a bonus, if the page wasn't in the cache it gets added so that
Juan Quintela3d0684b2017-03-23 15:06:39 +0100662 * when a small write is made into the 0'd page it gets XBZRLE sent.
Juan Quintela56e93d22015-05-07 19:33:31 +0200663 */
Juan Quintela6f37bb82017-03-13 19:26:29 +0100664static void xbzrle_cache_zero_page(RAMState *rs, ram_addr_t current_addr)
Juan Quintela56e93d22015-05-07 19:33:31 +0200665{
Juan Quintela6f37bb82017-03-13 19:26:29 +0100666 if (rs->ram_bulk_stage || !migrate_use_xbzrle()) {
Juan Quintela56e93d22015-05-07 19:33:31 +0200667 return;
668 }
669
670 /* We don't care if this fails to allocate a new cache page
671 * as long as it updated an old one */
Juan Quintelac00e0922017-05-09 16:22:01 +0200672 cache_insert(XBZRLE.cache, current_addr, XBZRLE.zero_target_page,
Juan Quintela93604472017-06-06 19:49:03 +0200673 ram_counters.dirty_sync_count);
Juan Quintela56e93d22015-05-07 19:33:31 +0200674}
675
676#define ENCODING_FLAG_XBZRLE 0x1
677
678/**
679 * save_xbzrle_page: compress and send current page
680 *
681 * Returns: 1 means that we wrote the page
682 * 0 means that page is identical to the one already sent
683 * -1 means that xbzrle would be longer than normal
684 *
Juan Quintela5a987732017-03-13 19:39:02 +0100685 * @rs: current RAM state
Juan Quintela3d0684b2017-03-23 15:06:39 +0100686 * @current_data: pointer to the address of the page contents
687 * @current_addr: addr of the page
Juan Quintela56e93d22015-05-07 19:33:31 +0200688 * @block: block that contains the page we want to send
689 * @offset: offset inside the block for the page
690 * @last_stage: if we are at the completion stage
Juan Quintela56e93d22015-05-07 19:33:31 +0200691 */
Juan Quintela204b88b2017-03-15 09:16:57 +0100692static int save_xbzrle_page(RAMState *rs, uint8_t **current_data,
Juan Quintela56e93d22015-05-07 19:33:31 +0200693 ram_addr_t current_addr, RAMBlock *block,
Juan Quintela072c2512017-03-14 10:27:31 +0100694 ram_addr_t offset, bool last_stage)
Juan Quintela56e93d22015-05-07 19:33:31 +0200695{
696 int encoded_len = 0, bytes_xbzrle;
697 uint8_t *prev_cached_page;
698
Juan Quintela93604472017-06-06 19:49:03 +0200699 if (!cache_is_cached(XBZRLE.cache, current_addr,
700 ram_counters.dirty_sync_count)) {
701 xbzrle_counters.cache_miss++;
Juan Quintela56e93d22015-05-07 19:33:31 +0200702 if (!last_stage) {
703 if (cache_insert(XBZRLE.cache, current_addr, *current_data,
Juan Quintela93604472017-06-06 19:49:03 +0200704 ram_counters.dirty_sync_count) == -1) {
Juan Quintela56e93d22015-05-07 19:33:31 +0200705 return -1;
706 } else {
707 /* update *current_data when the page has been
708 inserted into cache */
709 *current_data = get_cached_data(XBZRLE.cache, current_addr);
710 }
711 }
712 return -1;
713 }
714
715 prev_cached_page = get_cached_data(XBZRLE.cache, current_addr);
716
717 /* save current buffer into memory */
718 memcpy(XBZRLE.current_buf, *current_data, TARGET_PAGE_SIZE);
719
720 /* XBZRLE encoding (if there is no overflow) */
721 encoded_len = xbzrle_encode_buffer(prev_cached_page, XBZRLE.current_buf,
722 TARGET_PAGE_SIZE, XBZRLE.encoded_buf,
723 TARGET_PAGE_SIZE);
724 if (encoded_len == 0) {
Juan Quintela55c44462017-01-23 22:32:05 +0100725 trace_save_xbzrle_page_skipping();
Juan Quintela56e93d22015-05-07 19:33:31 +0200726 return 0;
727 } else if (encoded_len == -1) {
Juan Quintela55c44462017-01-23 22:32:05 +0100728 trace_save_xbzrle_page_overflow();
Juan Quintela93604472017-06-06 19:49:03 +0200729 xbzrle_counters.overflow++;
Juan Quintela56e93d22015-05-07 19:33:31 +0200730 /* update data in the cache */
731 if (!last_stage) {
732 memcpy(prev_cached_page, *current_data, TARGET_PAGE_SIZE);
733 *current_data = prev_cached_page;
734 }
735 return -1;
736 }
737
738 /* we need to update the data in the cache, in order to get the same data */
739 if (!last_stage) {
740 memcpy(prev_cached_page, XBZRLE.current_buf, TARGET_PAGE_SIZE);
741 }
742
743 /* Send XBZRLE based compressed page */
Juan Quintela2bf3aa82017-05-10 13:28:13 +0200744 bytes_xbzrle = save_page_header(rs, rs->f, block,
Juan Quintela204b88b2017-03-15 09:16:57 +0100745 offset | RAM_SAVE_FLAG_XBZRLE);
746 qemu_put_byte(rs->f, ENCODING_FLAG_XBZRLE);
747 qemu_put_be16(rs->f, encoded_len);
748 qemu_put_buffer(rs->f, XBZRLE.encoded_buf, encoded_len);
Juan Quintela56e93d22015-05-07 19:33:31 +0200749 bytes_xbzrle += encoded_len + 1 + 2;
Juan Quintela93604472017-06-06 19:49:03 +0200750 xbzrle_counters.pages++;
751 xbzrle_counters.bytes += bytes_xbzrle;
752 ram_counters.transferred += bytes_xbzrle;
Juan Quintela56e93d22015-05-07 19:33:31 +0200753
754 return 1;
755}
756
Juan Quintela3d0684b2017-03-23 15:06:39 +0100757/**
758 * migration_bitmap_find_dirty: find the next dirty page from start
Dr. David Alan Gilbertf3f491f2015-11-05 18:11:01 +0000759 *
Juan Quintela3d0684b2017-03-23 15:06:39 +0100760 * Called with rcu_read_lock() to protect migration_bitmap
761 *
762 * Returns the byte offset within memory region of the start of a dirty page
763 *
Juan Quintela6f37bb82017-03-13 19:26:29 +0100764 * @rs: current RAM state
Juan Quintela3d0684b2017-03-23 15:06:39 +0100765 * @rb: RAMBlock where to search for dirty pages
Juan Quintelaa935e302017-03-21 15:36:51 +0100766 * @start: page where we start the search
Dr. David Alan Gilbertf3f491f2015-11-05 18:11:01 +0000767 */
Juan Quintela56e93d22015-05-07 19:33:31 +0200768static inline
Juan Quintelaa935e302017-03-21 15:36:51 +0100769unsigned long migration_bitmap_find_dirty(RAMState *rs, RAMBlock *rb,
Juan Quintelaf20e2862017-03-21 16:19:05 +0100770 unsigned long start)
Juan Quintela56e93d22015-05-07 19:33:31 +0200771{
Juan Quintela6b6712e2017-03-22 15:18:04 +0100772 unsigned long size = rb->used_length >> TARGET_PAGE_BITS;
773 unsigned long *bitmap = rb->bmap;
Juan Quintela56e93d22015-05-07 19:33:31 +0200774 unsigned long next;
775
Juan Quintela6b6712e2017-03-22 15:18:04 +0100776 if (rs->ram_bulk_stage && start > 0) {
777 next = start + 1;
Juan Quintela56e93d22015-05-07 19:33:31 +0200778 } else {
Juan Quintela6b6712e2017-03-22 15:18:04 +0100779 next = find_next_bit(bitmap, size, start);
Juan Quintela56e93d22015-05-07 19:33:31 +0200780 }
781
Juan Quintela6b6712e2017-03-22 15:18:04 +0100782 return next;
Juan Quintela56e93d22015-05-07 19:33:31 +0200783}
784
Juan Quintela06b10682017-03-21 15:18:05 +0100785static inline bool migration_bitmap_clear_dirty(RAMState *rs,
Juan Quintelaf20e2862017-03-21 16:19:05 +0100786 RAMBlock *rb,
787 unsigned long page)
Dr. David Alan Gilberta82d5932015-11-05 18:11:09 +0000788{
789 bool ret;
Dr. David Alan Gilberta82d5932015-11-05 18:11:09 +0000790
Juan Quintela6b6712e2017-03-22 15:18:04 +0100791 ret = test_and_clear_bit(page, rb->bmap);
Dr. David Alan Gilberta82d5932015-11-05 18:11:09 +0000792
793 if (ret) {
Juan Quintela0d8ec882017-03-13 21:21:41 +0100794 rs->migration_dirty_pages--;
Dr. David Alan Gilberta82d5932015-11-05 18:11:09 +0000795 }
796 return ret;
797}
798
Juan Quintela15440dd2017-03-21 09:35:04 +0100799static void migration_bitmap_sync_range(RAMState *rs, RAMBlock *rb,
800 ram_addr_t start, ram_addr_t length)
Juan Quintela56e93d22015-05-07 19:33:31 +0200801{
Juan Quintela0d8ec882017-03-13 21:21:41 +0100802 rs->migration_dirty_pages +=
Juan Quintela6b6712e2017-03-22 15:18:04 +0100803 cpu_physical_memory_sync_dirty_bitmap(rb, start, length,
Juan Quintela0d8ec882017-03-13 21:21:41 +0100804 &rs->num_dirty_pages_period);
Juan Quintela56e93d22015-05-07 19:33:31 +0200805}
806
Juan Quintela3d0684b2017-03-23 15:06:39 +0100807/**
808 * ram_pagesize_summary: calculate all the pagesizes of a VM
809 *
810 * Returns a summary bitmap of the page sizes of all RAMBlocks
811 *
812 * For VMs with just normal pages this is equivalent to the host page
813 * size. If it's got some huge pages then it's the OR of all the
814 * different page sizes.
Dr. David Alan Gilberte8ca1db2017-02-24 18:28:29 +0000815 */
816uint64_t ram_pagesize_summary(void)
817{
818 RAMBlock *block;
819 uint64_t summary = 0;
820
Peter Xu99e15582017-05-12 12:17:39 +0800821 RAMBLOCK_FOREACH(block) {
Dr. David Alan Gilberte8ca1db2017-02-24 18:28:29 +0000822 summary |= block->page_size;
823 }
824
825 return summary;
826}
827
Juan Quintela8d820d62017-03-13 19:35:50 +0100828static void migration_bitmap_sync(RAMState *rs)
Juan Quintela56e93d22015-05-07 19:33:31 +0200829{
830 RAMBlock *block;
Juan Quintela56e93d22015-05-07 19:33:31 +0200831 int64_t end_time;
Juan Quintelac4bdf0c2017-03-28 14:59:54 +0200832 uint64_t bytes_xfer_now;
Juan Quintela56e93d22015-05-07 19:33:31 +0200833
Juan Quintela93604472017-06-06 19:49:03 +0200834 ram_counters.dirty_sync_count++;
Juan Quintela56e93d22015-05-07 19:33:31 +0200835
Juan Quintelaf664da82017-03-13 19:44:57 +0100836 if (!rs->time_last_bitmap_sync) {
837 rs->time_last_bitmap_sync = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
Juan Quintela56e93d22015-05-07 19:33:31 +0200838 }
839
840 trace_migration_bitmap_sync_start();
Paolo Bonzini9c1f8f42016-09-22 16:08:31 +0200841 memory_global_dirty_log_sync();
Juan Quintela56e93d22015-05-07 19:33:31 +0200842
Juan Quintela108cfae2017-03-13 21:38:09 +0100843 qemu_mutex_lock(&rs->bitmap_mutex);
Juan Quintela56e93d22015-05-07 19:33:31 +0200844 rcu_read_lock();
Peter Xu99e15582017-05-12 12:17:39 +0800845 RAMBLOCK_FOREACH(block) {
Juan Quintela15440dd2017-03-21 09:35:04 +0100846 migration_bitmap_sync_range(rs, block, 0, block->used_length);
Juan Quintela56e93d22015-05-07 19:33:31 +0200847 }
848 rcu_read_unlock();
Juan Quintela108cfae2017-03-13 21:38:09 +0100849 qemu_mutex_unlock(&rs->bitmap_mutex);
Juan Quintela56e93d22015-05-07 19:33:31 +0200850
Juan Quintelaa66cd902017-03-28 15:02:43 +0200851 trace_migration_bitmap_sync_end(rs->num_dirty_pages_period);
Chao Fan1ffb5df2017-03-14 09:55:07 +0800852
Juan Quintela56e93d22015-05-07 19:33:31 +0200853 end_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
854
855 /* more than 1 second = 1000 millisecons */
Juan Quintelaf664da82017-03-13 19:44:57 +0100856 if (end_time > rs->time_last_bitmap_sync + 1000) {
Felipe Franciosid693c6f2017-05-24 17:10:01 +0100857 /* calculate period counters */
Juan Quintela93604472017-06-06 19:49:03 +0200858 ram_counters.dirty_pages_rate = rs->num_dirty_pages_period * 1000
Felipe Franciosid693c6f2017-05-24 17:10:01 +0100859 / (end_time - rs->time_last_bitmap_sync);
Juan Quintela93604472017-06-06 19:49:03 +0200860 bytes_xfer_now = ram_counters.transferred;
Felipe Franciosid693c6f2017-05-24 17:10:01 +0100861
Peter Lieven9ac78b62017-09-26 12:33:16 +0200862 /* During block migration the auto-converge logic incorrectly detects
863 * that ram migration makes no progress. Avoid this by disabling the
864 * throttling logic during the bulk phase of block migration. */
865 if (migrate_auto_converge() && !blk_mig_bulk_active()) {
Juan Quintela56e93d22015-05-07 19:33:31 +0200866 /* The following detection logic can be refined later. For now:
867 Check to see if the dirtied bytes is 50% more than the approx.
868 amount of bytes that just got transferred since the last time we
Jason J. Herne070afca2015-09-08 13:12:35 -0400869 were in this routine. If that happens twice, start or increase
870 throttling */
Jason J. Herne070afca2015-09-08 13:12:35 -0400871
Felipe Franciosid693c6f2017-05-24 17:10:01 +0100872 if ((rs->num_dirty_pages_period * TARGET_PAGE_SIZE >
Juan Quintelaeac74152017-03-28 14:59:01 +0200873 (bytes_xfer_now - rs->bytes_xfer_prev) / 2) &&
Felipe Franciosib4a3c642017-05-24 17:10:03 +0100874 (++rs->dirty_rate_high_cnt >= 2)) {
Juan Quintela56e93d22015-05-07 19:33:31 +0200875 trace_migration_throttle();
Juan Quintela8d820d62017-03-13 19:35:50 +0100876 rs->dirty_rate_high_cnt = 0;
Jason J. Herne070afca2015-09-08 13:12:35 -0400877 mig_throttle_guest_down();
Felipe Franciosid693c6f2017-05-24 17:10:01 +0100878 }
Juan Quintela56e93d22015-05-07 19:33:31 +0200879 }
Jason J. Herne070afca2015-09-08 13:12:35 -0400880
Juan Quintela56e93d22015-05-07 19:33:31 +0200881 if (migrate_use_xbzrle()) {
Juan Quintela23b28c32017-03-13 20:51:34 +0100882 if (rs->iterations_prev != rs->iterations) {
Juan Quintela93604472017-06-06 19:49:03 +0200883 xbzrle_counters.cache_miss_rate =
884 (double)(xbzrle_counters.cache_miss -
Juan Quintelab5833fd2017-03-13 19:49:19 +0100885 rs->xbzrle_cache_miss_prev) /
Juan Quintela23b28c32017-03-13 20:51:34 +0100886 (rs->iterations - rs->iterations_prev);
Juan Quintela56e93d22015-05-07 19:33:31 +0200887 }
Juan Quintela23b28c32017-03-13 20:51:34 +0100888 rs->iterations_prev = rs->iterations;
Juan Quintela93604472017-06-06 19:49:03 +0200889 rs->xbzrle_cache_miss_prev = xbzrle_counters.cache_miss;
Juan Quintela56e93d22015-05-07 19:33:31 +0200890 }
Felipe Franciosid693c6f2017-05-24 17:10:01 +0100891
892 /* reset period counters */
Juan Quintelaf664da82017-03-13 19:44:57 +0100893 rs->time_last_bitmap_sync = end_time;
Juan Quintelaa66cd902017-03-28 15:02:43 +0200894 rs->num_dirty_pages_period = 0;
Felipe Franciosid2a4d852017-05-24 17:10:02 +0100895 rs->bytes_xfer_prev = bytes_xfer_now;
Juan Quintela56e93d22015-05-07 19:33:31 +0200896 }
Dr. David Alan Gilbert4addcd42015-12-16 11:47:36 +0000897 if (migrate_use_events()) {
Juan Quintela93604472017-06-06 19:49:03 +0200898 qapi_event_send_migration_pass(ram_counters.dirty_sync_count, NULL);
Dr. David Alan Gilbert4addcd42015-12-16 11:47:36 +0000899 }
Juan Quintela56e93d22015-05-07 19:33:31 +0200900}
901
902/**
Juan Quintela3d0684b2017-03-23 15:06:39 +0100903 * save_zero_page: send the zero page to the stream
Juan Quintela56e93d22015-05-07 19:33:31 +0200904 *
Juan Quintela3d0684b2017-03-23 15:06:39 +0100905 * Returns the number of pages written.
Juan Quintela56e93d22015-05-07 19:33:31 +0200906 *
Juan Quintelaf7ccd612017-03-13 20:30:21 +0100907 * @rs: current RAM state
Juan Quintela56e93d22015-05-07 19:33:31 +0200908 * @block: block that contains the page we want to send
909 * @offset: offset inside the block for the page
Juan Quintela56e93d22015-05-07 19:33:31 +0200910 */
Juan Quintela7faccdc2018-01-08 18:58:17 +0100911static int save_zero_page(RAMState *rs, RAMBlock *block, ram_addr_t offset)
Juan Quintela56e93d22015-05-07 19:33:31 +0200912{
Juan Quintela7faccdc2018-01-08 18:58:17 +0100913 uint8_t *p = block->host + offset;
Juan Quintela56e93d22015-05-07 19:33:31 +0200914 int pages = -1;
915
916 if (is_zero_range(p, TARGET_PAGE_SIZE)) {
Juan Quintela93604472017-06-06 19:49:03 +0200917 ram_counters.duplicate++;
918 ram_counters.transferred +=
Juan Quintelabb890ed2017-04-28 09:39:55 +0200919 save_page_header(rs, rs->f, block, offset | RAM_SAVE_FLAG_ZERO);
Juan Quintelace25d332017-03-15 11:00:51 +0100920 qemu_put_byte(rs->f, 0);
Juan Quintela93604472017-06-06 19:49:03 +0200921 ram_counters.transferred += 1;
Juan Quintela56e93d22015-05-07 19:33:31 +0200922 pages = 1;
923 }
924
925 return pages;
926}
927
Juan Quintela57273092017-03-20 22:25:28 +0100928static void ram_release_pages(const char *rbname, uint64_t offset, int pages)
Pavel Butsykin53f09a12017-02-03 18:23:20 +0300929{
Juan Quintela57273092017-03-20 22:25:28 +0100930 if (!migrate_release_ram() || !migration_in_postcopy()) {
Pavel Butsykin53f09a12017-02-03 18:23:20 +0300931 return;
932 }
933
Juan Quintelaaaa20642017-03-21 11:35:24 +0100934 ram_discard_range(rbname, offset, pages << TARGET_PAGE_BITS);
Pavel Butsykin53f09a12017-02-03 18:23:20 +0300935}
936
Juan Quintela56e93d22015-05-07 19:33:31 +0200937/**
Juan Quintela3d0684b2017-03-23 15:06:39 +0100938 * ram_save_page: send the given page to the stream
Juan Quintela56e93d22015-05-07 19:33:31 +0200939 *
Juan Quintela3d0684b2017-03-23 15:06:39 +0100940 * Returns the number of pages written.
Dr. David Alan Gilbert3fd3c4b2015-12-10 16:31:46 +0000941 * < 0 - error
942 * >=0 - Number of pages written - this might legally be 0
943 * if xbzrle noticed the page was the same.
Juan Quintela56e93d22015-05-07 19:33:31 +0200944 *
Juan Quintela6f37bb82017-03-13 19:26:29 +0100945 * @rs: current RAM state
Juan Quintela56e93d22015-05-07 19:33:31 +0200946 * @block: block that contains the page we want to send
947 * @offset: offset inside the block for the page
948 * @last_stage: if we are at the completion stage
Juan Quintela56e93d22015-05-07 19:33:31 +0200949 */
Juan Quintelaa0a8aa12017-03-20 22:29:07 +0100950static int ram_save_page(RAMState *rs, PageSearchStatus *pss, bool last_stage)
Juan Quintela56e93d22015-05-07 19:33:31 +0200951{
952 int pages = -1;
953 uint64_t bytes_xmit;
954 ram_addr_t current_addr;
Juan Quintela56e93d22015-05-07 19:33:31 +0200955 uint8_t *p;
956 int ret;
957 bool send_async = true;
zhanghailianga08f6892016-01-15 11:37:44 +0800958 RAMBlock *block = pss->block;
Juan Quintelaa935e302017-03-21 15:36:51 +0100959 ram_addr_t offset = pss->page << TARGET_PAGE_BITS;
Juan Quintela56e93d22015-05-07 19:33:31 +0200960
Dr. David Alan Gilbert2f68e392015-08-13 11:51:30 +0100961 p = block->host + offset;
Dr. David Alan Gilbert1db9d8e2017-04-26 19:37:21 +0100962 trace_ram_save_page(block->idstr, (uint64_t)offset, p);
Juan Quintela56e93d22015-05-07 19:33:31 +0200963
964 /* In doubt sent page as normal */
965 bytes_xmit = 0;
Juan Quintelace25d332017-03-15 11:00:51 +0100966 ret = ram_control_save_page(rs->f, block->offset,
Juan Quintela56e93d22015-05-07 19:33:31 +0200967 offset, TARGET_PAGE_SIZE, &bytes_xmit);
968 if (bytes_xmit) {
Juan Quintela93604472017-06-06 19:49:03 +0200969 ram_counters.transferred += bytes_xmit;
Juan Quintela56e93d22015-05-07 19:33:31 +0200970 pages = 1;
971 }
972
973 XBZRLE_cache_lock();
974
975 current_addr = block->offset + offset;
976
Juan Quintela56e93d22015-05-07 19:33:31 +0200977 if (ret != RAM_SAVE_CONTROL_NOT_SUPP) {
978 if (ret != RAM_SAVE_CONTROL_DELAYED) {
979 if (bytes_xmit > 0) {
Juan Quintela93604472017-06-06 19:49:03 +0200980 ram_counters.normal++;
Juan Quintela56e93d22015-05-07 19:33:31 +0200981 } else if (bytes_xmit == 0) {
Juan Quintela93604472017-06-06 19:49:03 +0200982 ram_counters.duplicate++;
Juan Quintela56e93d22015-05-07 19:33:31 +0200983 }
984 }
985 } else {
Juan Quintela7faccdc2018-01-08 18:58:17 +0100986 pages = save_zero_page(rs, block, offset);
Juan Quintela56e93d22015-05-07 19:33:31 +0200987 if (pages > 0) {
988 /* Must let xbzrle know, otherwise a previous (now 0'd) cached
989 * page would be stale
990 */
Juan Quintela6f37bb82017-03-13 19:26:29 +0100991 xbzrle_cache_zero_page(rs, current_addr);
Juan Quintelaa935e302017-03-21 15:36:51 +0100992 ram_release_pages(block->idstr, offset, pages);
Juan Quintela6f37bb82017-03-13 19:26:29 +0100993 } else if (!rs->ram_bulk_stage &&
Juan Quintela57273092017-03-20 22:25:28 +0100994 !migration_in_postcopy() && migrate_use_xbzrle()) {
Juan Quintela204b88b2017-03-15 09:16:57 +0100995 pages = save_xbzrle_page(rs, &p, current_addr, block,
Juan Quintela072c2512017-03-14 10:27:31 +0100996 offset, last_stage);
Juan Quintela56e93d22015-05-07 19:33:31 +0200997 if (!last_stage) {
998 /* Can't send this cached data async, since the cache page
999 * might get updated before it gets to the wire
1000 */
1001 send_async = false;
1002 }
1003 }
1004 }
1005
1006 /* XBZRLE overflow or normal page */
1007 if (pages == -1) {
Juan Quintela93604472017-06-06 19:49:03 +02001008 ram_counters.transferred +=
1009 save_page_header(rs, rs->f, block, offset | RAM_SAVE_FLAG_PAGE);
Juan Quintela56e93d22015-05-07 19:33:31 +02001010 if (send_async) {
Juan Quintelace25d332017-03-15 11:00:51 +01001011 qemu_put_buffer_async(rs->f, p, TARGET_PAGE_SIZE,
Pavel Butsykin53f09a12017-02-03 18:23:20 +03001012 migrate_release_ram() &
Juan Quintela57273092017-03-20 22:25:28 +01001013 migration_in_postcopy());
Juan Quintela56e93d22015-05-07 19:33:31 +02001014 } else {
Juan Quintelace25d332017-03-15 11:00:51 +01001015 qemu_put_buffer(rs->f, p, TARGET_PAGE_SIZE);
Juan Quintela56e93d22015-05-07 19:33:31 +02001016 }
Juan Quintela93604472017-06-06 19:49:03 +02001017 ram_counters.transferred += TARGET_PAGE_SIZE;
Juan Quintela56e93d22015-05-07 19:33:31 +02001018 pages = 1;
Juan Quintela93604472017-06-06 19:49:03 +02001019 ram_counters.normal++;
Juan Quintela56e93d22015-05-07 19:33:31 +02001020 }
1021
1022 XBZRLE_cache_unlock();
1023
1024 return pages;
1025}
1026
Liang Lia7a9a882016-05-05 15:32:57 +08001027static int do_compress_ram_page(QEMUFile *f, RAMBlock *block,
1028 ram_addr_t offset)
Juan Quintela56e93d22015-05-07 19:33:31 +02001029{
Juan Quintela53518d92017-05-04 11:46:24 +02001030 RAMState *rs = ram_state;
Juan Quintela56e93d22015-05-07 19:33:31 +02001031 int bytes_sent, blen;
Liang Lia7a9a882016-05-05 15:32:57 +08001032 uint8_t *p = block->host + (offset & TARGET_PAGE_MASK);
Juan Quintela56e93d22015-05-07 19:33:31 +02001033
Juan Quintela2bf3aa82017-05-10 13:28:13 +02001034 bytes_sent = save_page_header(rs, f, block, offset |
Juan Quintela56e93d22015-05-07 19:33:31 +02001035 RAM_SAVE_FLAG_COMPRESS_PAGE);
Liang Lia7a9a882016-05-05 15:32:57 +08001036 blen = qemu_put_compression_data(f, p, TARGET_PAGE_SIZE,
Juan Quintela56e93d22015-05-07 19:33:31 +02001037 migrate_compress_level());
Liang Lib3be2892016-05-05 15:32:54 +08001038 if (blen < 0) {
1039 bytes_sent = 0;
1040 qemu_file_set_error(migrate_get_current()->to_dst_file, blen);
1041 error_report("compressed data failed!");
1042 } else {
1043 bytes_sent += blen;
Juan Quintela57273092017-03-20 22:25:28 +01001044 ram_release_pages(block->idstr, offset & TARGET_PAGE_MASK, 1);
Liang Lib3be2892016-05-05 15:32:54 +08001045 }
Juan Quintela56e93d22015-05-07 19:33:31 +02001046
1047 return bytes_sent;
1048}
1049
Juan Quintelace25d332017-03-15 11:00:51 +01001050static void flush_compressed_data(RAMState *rs)
Juan Quintela56e93d22015-05-07 19:33:31 +02001051{
1052 int idx, len, thread_count;
1053
1054 if (!migrate_use_compression()) {
1055 return;
1056 }
1057 thread_count = migrate_compress_threads();
Liang Lia7a9a882016-05-05 15:32:57 +08001058
Liang Li0d9f9a52016-05-05 15:32:59 +08001059 qemu_mutex_lock(&comp_done_lock);
Juan Quintela56e93d22015-05-07 19:33:31 +02001060 for (idx = 0; idx < thread_count; idx++) {
Liang Lia7a9a882016-05-05 15:32:57 +08001061 while (!comp_param[idx].done) {
Liang Li0d9f9a52016-05-05 15:32:59 +08001062 qemu_cond_wait(&comp_done_cond, &comp_done_lock);
Juan Quintela56e93d22015-05-07 19:33:31 +02001063 }
Liang Lia7a9a882016-05-05 15:32:57 +08001064 }
Liang Li0d9f9a52016-05-05 15:32:59 +08001065 qemu_mutex_unlock(&comp_done_lock);
Liang Lia7a9a882016-05-05 15:32:57 +08001066
1067 for (idx = 0; idx < thread_count; idx++) {
1068 qemu_mutex_lock(&comp_param[idx].mutex);
Liang Li90e56fb2016-05-05 15:32:56 +08001069 if (!comp_param[idx].quit) {
Juan Quintelace25d332017-03-15 11:00:51 +01001070 len = qemu_put_qemu_file(rs->f, comp_param[idx].file);
Juan Quintela93604472017-06-06 19:49:03 +02001071 ram_counters.transferred += len;
Juan Quintela56e93d22015-05-07 19:33:31 +02001072 }
Liang Lia7a9a882016-05-05 15:32:57 +08001073 qemu_mutex_unlock(&comp_param[idx].mutex);
Juan Quintela56e93d22015-05-07 19:33:31 +02001074 }
1075}
1076
1077static inline void set_compress_params(CompressParam *param, RAMBlock *block,
1078 ram_addr_t offset)
1079{
1080 param->block = block;
1081 param->offset = offset;
1082}
1083
Juan Quintelace25d332017-03-15 11:00:51 +01001084static int compress_page_with_multi_thread(RAMState *rs, RAMBlock *block,
1085 ram_addr_t offset)
Juan Quintela56e93d22015-05-07 19:33:31 +02001086{
1087 int idx, thread_count, bytes_xmit = -1, pages = -1;
1088
1089 thread_count = migrate_compress_threads();
Liang Li0d9f9a52016-05-05 15:32:59 +08001090 qemu_mutex_lock(&comp_done_lock);
Juan Quintela56e93d22015-05-07 19:33:31 +02001091 while (true) {
1092 for (idx = 0; idx < thread_count; idx++) {
1093 if (comp_param[idx].done) {
Liang Lia7a9a882016-05-05 15:32:57 +08001094 comp_param[idx].done = false;
Juan Quintelace25d332017-03-15 11:00:51 +01001095 bytes_xmit = qemu_put_qemu_file(rs->f, comp_param[idx].file);
Liang Lia7a9a882016-05-05 15:32:57 +08001096 qemu_mutex_lock(&comp_param[idx].mutex);
Juan Quintela56e93d22015-05-07 19:33:31 +02001097 set_compress_params(&comp_param[idx], block, offset);
Liang Lia7a9a882016-05-05 15:32:57 +08001098 qemu_cond_signal(&comp_param[idx].cond);
1099 qemu_mutex_unlock(&comp_param[idx].mutex);
Juan Quintela56e93d22015-05-07 19:33:31 +02001100 pages = 1;
Juan Quintela93604472017-06-06 19:49:03 +02001101 ram_counters.normal++;
1102 ram_counters.transferred += bytes_xmit;
Juan Quintela56e93d22015-05-07 19:33:31 +02001103 break;
1104 }
1105 }
1106 if (pages > 0) {
1107 break;
1108 } else {
Liang Li0d9f9a52016-05-05 15:32:59 +08001109 qemu_cond_wait(&comp_done_cond, &comp_done_lock);
Juan Quintela56e93d22015-05-07 19:33:31 +02001110 }
1111 }
Liang Li0d9f9a52016-05-05 15:32:59 +08001112 qemu_mutex_unlock(&comp_done_lock);
Juan Quintela56e93d22015-05-07 19:33:31 +02001113
1114 return pages;
1115}
1116
1117/**
1118 * ram_save_compressed_page: compress the given page and send it to the stream
1119 *
Juan Quintela3d0684b2017-03-23 15:06:39 +01001120 * Returns the number of pages written.
Juan Quintela56e93d22015-05-07 19:33:31 +02001121 *
Juan Quintela6f37bb82017-03-13 19:26:29 +01001122 * @rs: current RAM state
Juan Quintela56e93d22015-05-07 19:33:31 +02001123 * @block: block that contains the page we want to send
1124 * @offset: offset inside the block for the page
1125 * @last_stage: if we are at the completion stage
Juan Quintela56e93d22015-05-07 19:33:31 +02001126 */
Juan Quintelaa0a8aa12017-03-20 22:29:07 +01001127static int ram_save_compressed_page(RAMState *rs, PageSearchStatus *pss,
1128 bool last_stage)
Juan Quintela56e93d22015-05-07 19:33:31 +02001129{
1130 int pages = -1;
Liang Lifc504382016-05-05 15:32:55 +08001131 uint64_t bytes_xmit = 0;
Juan Quintela56e93d22015-05-07 19:33:31 +02001132 uint8_t *p;
Liang Lifc504382016-05-05 15:32:55 +08001133 int ret, blen;
zhanghailianga08f6892016-01-15 11:37:44 +08001134 RAMBlock *block = pss->block;
Juan Quintelaa935e302017-03-21 15:36:51 +01001135 ram_addr_t offset = pss->page << TARGET_PAGE_BITS;
Juan Quintela56e93d22015-05-07 19:33:31 +02001136
Dr. David Alan Gilbert2f68e392015-08-13 11:51:30 +01001137 p = block->host + offset;
Juan Quintela56e93d22015-05-07 19:33:31 +02001138
Juan Quintelace25d332017-03-15 11:00:51 +01001139 ret = ram_control_save_page(rs->f, block->offset,
Juan Quintela56e93d22015-05-07 19:33:31 +02001140 offset, TARGET_PAGE_SIZE, &bytes_xmit);
1141 if (bytes_xmit) {
Juan Quintela93604472017-06-06 19:49:03 +02001142 ram_counters.transferred += bytes_xmit;
Juan Quintela56e93d22015-05-07 19:33:31 +02001143 pages = 1;
1144 }
Juan Quintela56e93d22015-05-07 19:33:31 +02001145 if (ret != RAM_SAVE_CONTROL_NOT_SUPP) {
1146 if (ret != RAM_SAVE_CONTROL_DELAYED) {
1147 if (bytes_xmit > 0) {
Juan Quintela93604472017-06-06 19:49:03 +02001148 ram_counters.normal++;
Juan Quintela56e93d22015-05-07 19:33:31 +02001149 } else if (bytes_xmit == 0) {
Juan Quintela93604472017-06-06 19:49:03 +02001150 ram_counters.duplicate++;
Juan Quintela56e93d22015-05-07 19:33:31 +02001151 }
1152 }
1153 } else {
1154 /* When starting the process of a new block, the first page of
1155 * the block should be sent out before other pages in the same
1156 * block, and all the pages in last block should have been sent
1157 * out, keeping this order is important, because the 'cont' flag
1158 * is used to avoid resending the block name.
1159 */
Juan Quintela6f37bb82017-03-13 19:26:29 +01001160 if (block != rs->last_sent_block) {
Juan Quintelace25d332017-03-15 11:00:51 +01001161 flush_compressed_data(rs);
Juan Quintela7faccdc2018-01-08 18:58:17 +01001162 pages = save_zero_page(rs, block, offset);
Juan Quintela56e93d22015-05-07 19:33:31 +02001163 if (pages == -1) {
Liang Lifc504382016-05-05 15:32:55 +08001164 /* Make sure the first page is sent out before other pages */
Juan Quintela2bf3aa82017-05-10 13:28:13 +02001165 bytes_xmit = save_page_header(rs, rs->f, block, offset |
Liang Lifc504382016-05-05 15:32:55 +08001166 RAM_SAVE_FLAG_COMPRESS_PAGE);
Juan Quintelace25d332017-03-15 11:00:51 +01001167 blen = qemu_put_compression_data(rs->f, p, TARGET_PAGE_SIZE,
Liang Lifc504382016-05-05 15:32:55 +08001168 migrate_compress_level());
1169 if (blen > 0) {
Juan Quintela93604472017-06-06 19:49:03 +02001170 ram_counters.transferred += bytes_xmit + blen;
1171 ram_counters.normal++;
Liang Lib3be2892016-05-05 15:32:54 +08001172 pages = 1;
Liang Lifc504382016-05-05 15:32:55 +08001173 } else {
Juan Quintelace25d332017-03-15 11:00:51 +01001174 qemu_file_set_error(rs->f, blen);
Liang Lifc504382016-05-05 15:32:55 +08001175 error_report("compressed data failed!");
Liang Lib3be2892016-05-05 15:32:54 +08001176 }
Juan Quintela56e93d22015-05-07 19:33:31 +02001177 }
Pavel Butsykin53f09a12017-02-03 18:23:20 +03001178 if (pages > 0) {
Juan Quintelaa935e302017-03-21 15:36:51 +01001179 ram_release_pages(block->idstr, offset, pages);
Pavel Butsykin53f09a12017-02-03 18:23:20 +03001180 }
Juan Quintela56e93d22015-05-07 19:33:31 +02001181 } else {
Juan Quintela7faccdc2018-01-08 18:58:17 +01001182 pages = save_zero_page(rs, block, offset);
Juan Quintela56e93d22015-05-07 19:33:31 +02001183 if (pages == -1) {
Juan Quintelace25d332017-03-15 11:00:51 +01001184 pages = compress_page_with_multi_thread(rs, block, offset);
Pavel Butsykin53f09a12017-02-03 18:23:20 +03001185 } else {
Juan Quintelaa935e302017-03-21 15:36:51 +01001186 ram_release_pages(block->idstr, offset, pages);
Juan Quintela56e93d22015-05-07 19:33:31 +02001187 }
1188 }
1189 }
1190
1191 return pages;
1192}
1193
Juan Quintela3d0684b2017-03-23 15:06:39 +01001194/**
1195 * find_dirty_block: find the next dirty page and update any state
1196 * associated with the search process.
Dr. David Alan Gilbertb9e60922015-09-23 15:27:11 +01001197 *
Juan Quintela3d0684b2017-03-23 15:06:39 +01001198 * Returns if a page is found
Dr. David Alan Gilbertb9e60922015-09-23 15:27:11 +01001199 *
Juan Quintela6f37bb82017-03-13 19:26:29 +01001200 * @rs: current RAM state
Juan Quintela3d0684b2017-03-23 15:06:39 +01001201 * @pss: data about the state of the current dirty page scan
1202 * @again: set to false if the search has scanned the whole of RAM
Dr. David Alan Gilbertb9e60922015-09-23 15:27:11 +01001203 */
Juan Quintelaf20e2862017-03-21 16:19:05 +01001204static bool find_dirty_block(RAMState *rs, PageSearchStatus *pss, bool *again)
Dr. David Alan Gilbertb9e60922015-09-23 15:27:11 +01001205{
Juan Quintelaf20e2862017-03-21 16:19:05 +01001206 pss->page = migration_bitmap_find_dirty(rs, pss->block, pss->page);
Juan Quintela6f37bb82017-03-13 19:26:29 +01001207 if (pss->complete_round && pss->block == rs->last_seen_block &&
Juan Quintelaa935e302017-03-21 15:36:51 +01001208 pss->page >= rs->last_page) {
Dr. David Alan Gilbertb9e60922015-09-23 15:27:11 +01001209 /*
1210 * We've been once around the RAM and haven't found anything.
1211 * Give up.
1212 */
1213 *again = false;
1214 return false;
1215 }
Juan Quintelaa935e302017-03-21 15:36:51 +01001216 if ((pss->page << TARGET_PAGE_BITS) >= pss->block->used_length) {
Dr. David Alan Gilbertb9e60922015-09-23 15:27:11 +01001217 /* Didn't find anything in this RAM Block */
Juan Quintelaa935e302017-03-21 15:36:51 +01001218 pss->page = 0;
Dr. David Alan Gilbertb9e60922015-09-23 15:27:11 +01001219 pss->block = QLIST_NEXT_RCU(pss->block, next);
1220 if (!pss->block) {
1221 /* Hit the end of the list */
1222 pss->block = QLIST_FIRST_RCU(&ram_list.blocks);
1223 /* Flag that we've looped */
1224 pss->complete_round = true;
Juan Quintela6f37bb82017-03-13 19:26:29 +01001225 rs->ram_bulk_stage = false;
Dr. David Alan Gilbertb9e60922015-09-23 15:27:11 +01001226 if (migrate_use_xbzrle()) {
1227 /* If xbzrle is on, stop using the data compression at this
1228 * point. In theory, xbzrle can do better than compression.
1229 */
Juan Quintelace25d332017-03-15 11:00:51 +01001230 flush_compressed_data(rs);
Dr. David Alan Gilbertb9e60922015-09-23 15:27:11 +01001231 }
1232 }
1233 /* Didn't find anything this time, but try again on the new block */
1234 *again = true;
1235 return false;
1236 } else {
1237 /* Can go around again, but... */
1238 *again = true;
1239 /* We've found something so probably don't need to */
1240 return true;
1241 }
1242}
1243
Juan Quintela3d0684b2017-03-23 15:06:39 +01001244/**
1245 * unqueue_page: gets a page of the queue
Dr. David Alan Gilberta82d5932015-11-05 18:11:09 +00001246 *
Juan Quintela3d0684b2017-03-23 15:06:39 +01001247 * Helper for 'get_queued_page' - gets a page off the queue
1248 *
1249 * Returns the block of the page (or NULL if none available)
1250 *
Juan Quintelaec481c62017-03-20 22:12:40 +01001251 * @rs: current RAM state
Juan Quintela3d0684b2017-03-23 15:06:39 +01001252 * @offset: used to return the offset within the RAMBlock
Dr. David Alan Gilberta82d5932015-11-05 18:11:09 +00001253 */
Juan Quintelaf20e2862017-03-21 16:19:05 +01001254static RAMBlock *unqueue_page(RAMState *rs, ram_addr_t *offset)
Dr. David Alan Gilberta82d5932015-11-05 18:11:09 +00001255{
1256 RAMBlock *block = NULL;
1257
Juan Quintelaec481c62017-03-20 22:12:40 +01001258 qemu_mutex_lock(&rs->src_page_req_mutex);
1259 if (!QSIMPLEQ_EMPTY(&rs->src_page_requests)) {
1260 struct RAMSrcPageRequest *entry =
1261 QSIMPLEQ_FIRST(&rs->src_page_requests);
Dr. David Alan Gilberta82d5932015-11-05 18:11:09 +00001262 block = entry->rb;
1263 *offset = entry->offset;
Dr. David Alan Gilberta82d5932015-11-05 18:11:09 +00001264
1265 if (entry->len > TARGET_PAGE_SIZE) {
1266 entry->len -= TARGET_PAGE_SIZE;
1267 entry->offset += TARGET_PAGE_SIZE;
1268 } else {
1269 memory_region_unref(block->mr);
Juan Quintelaec481c62017-03-20 22:12:40 +01001270 QSIMPLEQ_REMOVE_HEAD(&rs->src_page_requests, next_req);
Dr. David Alan Gilberta82d5932015-11-05 18:11:09 +00001271 g_free(entry);
1272 }
1273 }
Juan Quintelaec481c62017-03-20 22:12:40 +01001274 qemu_mutex_unlock(&rs->src_page_req_mutex);
Dr. David Alan Gilberta82d5932015-11-05 18:11:09 +00001275
1276 return block;
1277}
1278
Juan Quintela3d0684b2017-03-23 15:06:39 +01001279/**
1280 * get_queued_page: unqueue a page from the postocpy requests
Dr. David Alan Gilberta82d5932015-11-05 18:11:09 +00001281 *
Juan Quintela3d0684b2017-03-23 15:06:39 +01001282 * Skips pages that are already sent (!dirty)
Dr. David Alan Gilberta82d5932015-11-05 18:11:09 +00001283 *
Juan Quintela3d0684b2017-03-23 15:06:39 +01001284 * Returns if a queued page is found
1285 *
Juan Quintela6f37bb82017-03-13 19:26:29 +01001286 * @rs: current RAM state
Juan Quintela3d0684b2017-03-23 15:06:39 +01001287 * @pss: data about the state of the current dirty page scan
Dr. David Alan Gilberta82d5932015-11-05 18:11:09 +00001288 */
Juan Quintelaf20e2862017-03-21 16:19:05 +01001289static bool get_queued_page(RAMState *rs, PageSearchStatus *pss)
Dr. David Alan Gilberta82d5932015-11-05 18:11:09 +00001290{
1291 RAMBlock *block;
1292 ram_addr_t offset;
1293 bool dirty;
1294
1295 do {
Juan Quintelaf20e2862017-03-21 16:19:05 +01001296 block = unqueue_page(rs, &offset);
Dr. David Alan Gilberta82d5932015-11-05 18:11:09 +00001297 /*
1298 * We're sending this page, and since it's postcopy nothing else
1299 * will dirty it, and we must make sure it doesn't get sent again
1300 * even if this queue request was received after the background
1301 * search already sent it.
1302 */
1303 if (block) {
Juan Quintelaf20e2862017-03-21 16:19:05 +01001304 unsigned long page;
1305
Juan Quintela6b6712e2017-03-22 15:18:04 +01001306 page = offset >> TARGET_PAGE_BITS;
1307 dirty = test_bit(page, block->bmap);
Dr. David Alan Gilberta82d5932015-11-05 18:11:09 +00001308 if (!dirty) {
Juan Quintela06b10682017-03-21 15:18:05 +01001309 trace_get_queued_page_not_dirty(block->idstr, (uint64_t)offset,
Juan Quintela6b6712e2017-03-22 15:18:04 +01001310 page, test_bit(page, block->unsentmap));
Dr. David Alan Gilberta82d5932015-11-05 18:11:09 +00001311 } else {
Juan Quintelaf20e2862017-03-21 16:19:05 +01001312 trace_get_queued_page(block->idstr, (uint64_t)offset, page);
Dr. David Alan Gilberta82d5932015-11-05 18:11:09 +00001313 }
1314 }
1315
1316 } while (block && !dirty);
1317
1318 if (block) {
1319 /*
1320 * As soon as we start servicing pages out of order, then we have
1321 * to kill the bulk stage, since the bulk stage assumes
1322 * in (migration_bitmap_find_and_reset_dirty) that every page is
1323 * dirty, that's no longer true.
1324 */
Juan Quintela6f37bb82017-03-13 19:26:29 +01001325 rs->ram_bulk_stage = false;
Dr. David Alan Gilberta82d5932015-11-05 18:11:09 +00001326
1327 /*
1328 * We want the background search to continue from the queued page
1329 * since the guest is likely to want other pages near to the page
1330 * it just requested.
1331 */
1332 pss->block = block;
Juan Quintelaa935e302017-03-21 15:36:51 +01001333 pss->page = offset >> TARGET_PAGE_BITS;
Dr. David Alan Gilberta82d5932015-11-05 18:11:09 +00001334 }
1335
1336 return !!block;
1337}
1338
Juan Quintela56e93d22015-05-07 19:33:31 +02001339/**
Juan Quintela5e58f962017-04-03 22:06:54 +02001340 * migration_page_queue_free: drop any remaining pages in the ram
1341 * request queue
Dr. David Alan Gilbert6c595cd2015-11-05 18:11:08 +00001342 *
Juan Quintela3d0684b2017-03-23 15:06:39 +01001343 * It should be empty at the end anyway, but in error cases there may
1344 * be some left. in case that there is any page left, we drop it.
1345 *
Dr. David Alan Gilbert6c595cd2015-11-05 18:11:08 +00001346 */
Juan Quintela83c13382017-05-04 11:45:01 +02001347static void migration_page_queue_free(RAMState *rs)
Dr. David Alan Gilbert6c595cd2015-11-05 18:11:08 +00001348{
Juan Quintelaec481c62017-03-20 22:12:40 +01001349 struct RAMSrcPageRequest *mspr, *next_mspr;
Dr. David Alan Gilbert6c595cd2015-11-05 18:11:08 +00001350 /* This queue generally should be empty - but in the case of a failed
1351 * migration might have some droppings in.
1352 */
1353 rcu_read_lock();
Juan Quintelaec481c62017-03-20 22:12:40 +01001354 QSIMPLEQ_FOREACH_SAFE(mspr, &rs->src_page_requests, next_req, next_mspr) {
Dr. David Alan Gilbert6c595cd2015-11-05 18:11:08 +00001355 memory_region_unref(mspr->rb->mr);
Juan Quintelaec481c62017-03-20 22:12:40 +01001356 QSIMPLEQ_REMOVE_HEAD(&rs->src_page_requests, next_req);
Dr. David Alan Gilbert6c595cd2015-11-05 18:11:08 +00001357 g_free(mspr);
1358 }
1359 rcu_read_unlock();
1360}
1361
1362/**
Juan Quintela3d0684b2017-03-23 15:06:39 +01001363 * ram_save_queue_pages: queue the page for transmission
1364 *
1365 * A request from postcopy destination for example.
1366 *
1367 * Returns zero on success or negative on error
1368 *
Juan Quintela3d0684b2017-03-23 15:06:39 +01001369 * @rbname: Name of the RAMBLock of the request. NULL means the
1370 * same that last one.
1371 * @start: starting address from the start of the RAMBlock
1372 * @len: length (in bytes) to send
Dr. David Alan Gilbert6c595cd2015-11-05 18:11:08 +00001373 */
Juan Quintela96506892017-03-14 18:41:03 +01001374int ram_save_queue_pages(const char *rbname, ram_addr_t start, ram_addr_t len)
Dr. David Alan Gilbert6c595cd2015-11-05 18:11:08 +00001375{
1376 RAMBlock *ramblock;
Juan Quintela53518d92017-05-04 11:46:24 +02001377 RAMState *rs = ram_state;
Dr. David Alan Gilbert6c595cd2015-11-05 18:11:08 +00001378
Juan Quintela93604472017-06-06 19:49:03 +02001379 ram_counters.postcopy_requests++;
Dr. David Alan Gilbert6c595cd2015-11-05 18:11:08 +00001380 rcu_read_lock();
1381 if (!rbname) {
1382 /* Reuse last RAMBlock */
Juan Quintela68a098f2017-03-14 13:48:42 +01001383 ramblock = rs->last_req_rb;
Dr. David Alan Gilbert6c595cd2015-11-05 18:11:08 +00001384
1385 if (!ramblock) {
1386 /*
1387 * Shouldn't happen, we can't reuse the last RAMBlock if
1388 * it's the 1st request.
1389 */
1390 error_report("ram_save_queue_pages no previous block");
1391 goto err;
1392 }
1393 } else {
1394 ramblock = qemu_ram_block_by_name(rbname);
1395
1396 if (!ramblock) {
1397 /* We shouldn't be asked for a non-existent RAMBlock */
1398 error_report("ram_save_queue_pages no block '%s'", rbname);
1399 goto err;
1400 }
Juan Quintela68a098f2017-03-14 13:48:42 +01001401 rs->last_req_rb = ramblock;
Dr. David Alan Gilbert6c595cd2015-11-05 18:11:08 +00001402 }
1403 trace_ram_save_queue_pages(ramblock->idstr, start, len);
1404 if (start+len > ramblock->used_length) {
Juan Quintela9458ad62015-11-10 17:42:05 +01001405 error_report("%s request overrun start=" RAM_ADDR_FMT " len="
1406 RAM_ADDR_FMT " blocklen=" RAM_ADDR_FMT,
Dr. David Alan Gilbert6c595cd2015-11-05 18:11:08 +00001407 __func__, start, len, ramblock->used_length);
1408 goto err;
1409 }
1410
Juan Quintelaec481c62017-03-20 22:12:40 +01001411 struct RAMSrcPageRequest *new_entry =
1412 g_malloc0(sizeof(struct RAMSrcPageRequest));
Dr. David Alan Gilbert6c595cd2015-11-05 18:11:08 +00001413 new_entry->rb = ramblock;
1414 new_entry->offset = start;
1415 new_entry->len = len;
1416
1417 memory_region_ref(ramblock->mr);
Juan Quintelaec481c62017-03-20 22:12:40 +01001418 qemu_mutex_lock(&rs->src_page_req_mutex);
1419 QSIMPLEQ_INSERT_TAIL(&rs->src_page_requests, new_entry, next_req);
1420 qemu_mutex_unlock(&rs->src_page_req_mutex);
Dr. David Alan Gilbert6c595cd2015-11-05 18:11:08 +00001421 rcu_read_unlock();
1422
1423 return 0;
1424
1425err:
1426 rcu_read_unlock();
1427 return -1;
1428}
1429
Dr. David Alan Gilberta82d5932015-11-05 18:11:09 +00001430/**
Juan Quintela3d0684b2017-03-23 15:06:39 +01001431 * ram_save_target_page: save one target page
Dr. David Alan Gilberta82d5932015-11-05 18:11:09 +00001432 *
Juan Quintela3d0684b2017-03-23 15:06:39 +01001433 * Returns the number of pages written
Dr. David Alan Gilberta82d5932015-11-05 18:11:09 +00001434 *
Juan Quintela6f37bb82017-03-13 19:26:29 +01001435 * @rs: current RAM state
Juan Quintela3d0684b2017-03-23 15:06:39 +01001436 * @ms: current migration state
Juan Quintela3d0684b2017-03-23 15:06:39 +01001437 * @pss: data about the page we want to send
Dr. David Alan Gilberta82d5932015-11-05 18:11:09 +00001438 * @last_stage: if we are at the completion stage
Dr. David Alan Gilberta82d5932015-11-05 18:11:09 +00001439 */
Juan Quintelaa0a8aa12017-03-20 22:29:07 +01001440static int ram_save_target_page(RAMState *rs, PageSearchStatus *pss,
Juan Quintelaf20e2862017-03-21 16:19:05 +01001441 bool last_stage)
Dr. David Alan Gilberta82d5932015-11-05 18:11:09 +00001442{
1443 int res = 0;
1444
1445 /* Check the pages is dirty and if it is send it */
Juan Quintelaf20e2862017-03-21 16:19:05 +01001446 if (migration_bitmap_clear_dirty(rs, pss->block, pss->page)) {
Juan Quintela6d358d92017-03-16 21:29:34 +01001447 /*
1448 * If xbzrle is on, stop using the data compression after first
1449 * round of migration even if compression is enabled. In theory,
1450 * xbzrle can do better than compression.
1451 */
Juan Quintela6b6712e2017-03-22 15:18:04 +01001452 if (migrate_use_compression() &&
1453 (rs->ram_bulk_stage || !migrate_use_xbzrle())) {
Juan Quintelaa0a8aa12017-03-20 22:29:07 +01001454 res = ram_save_compressed_page(rs, pss, last_stage);
Dr. David Alan Gilberta82d5932015-11-05 18:11:09 +00001455 } else {
Juan Quintelaa0a8aa12017-03-20 22:29:07 +01001456 res = ram_save_page(rs, pss, last_stage);
Dr. David Alan Gilberta82d5932015-11-05 18:11:09 +00001457 }
1458
1459 if (res < 0) {
1460 return res;
1461 }
Juan Quintela6b6712e2017-03-22 15:18:04 +01001462 if (pss->block->unsentmap) {
1463 clear_bit(pss->page, pss->block->unsentmap);
Dr. David Alan Gilberta82d5932015-11-05 18:11:09 +00001464 }
1465 }
1466
1467 return res;
1468}
1469
1470/**
Juan Quintela3d0684b2017-03-23 15:06:39 +01001471 * ram_save_host_page: save a whole host page
Dr. David Alan Gilberta82d5932015-11-05 18:11:09 +00001472 *
Juan Quintela3d0684b2017-03-23 15:06:39 +01001473 * Starting at *offset send pages up to the end of the current host
1474 * page. It's valid for the initial offset to point into the middle of
1475 * a host page in which case the remainder of the hostpage is sent.
1476 * Only dirty target pages are sent. Note that the host page size may
1477 * be a huge page for this block.
Dr. David Alan Gilbert1eb3fc02017-05-17 17:58:09 +01001478 * The saving stops at the boundary of the used_length of the block
1479 * if the RAMBlock isn't a multiple of the host page size.
Dr. David Alan Gilberta82d5932015-11-05 18:11:09 +00001480 *
Juan Quintela3d0684b2017-03-23 15:06:39 +01001481 * Returns the number of pages written or negative on error
1482 *
Juan Quintela6f37bb82017-03-13 19:26:29 +01001483 * @rs: current RAM state
Juan Quintela3d0684b2017-03-23 15:06:39 +01001484 * @ms: current migration state
Juan Quintela3d0684b2017-03-23 15:06:39 +01001485 * @pss: data about the page we want to send
Dr. David Alan Gilberta82d5932015-11-05 18:11:09 +00001486 * @last_stage: if we are at the completion stage
Dr. David Alan Gilberta82d5932015-11-05 18:11:09 +00001487 */
Juan Quintelaa0a8aa12017-03-20 22:29:07 +01001488static int ram_save_host_page(RAMState *rs, PageSearchStatus *pss,
Juan Quintelaf20e2862017-03-21 16:19:05 +01001489 bool last_stage)
Dr. David Alan Gilberta82d5932015-11-05 18:11:09 +00001490{
1491 int tmppages, pages = 0;
Juan Quintelaa935e302017-03-21 15:36:51 +01001492 size_t pagesize_bits =
1493 qemu_ram_pagesize(pss->block) >> TARGET_PAGE_BITS;
Dr. David Alan Gilbert4c011c32017-02-24 18:28:39 +00001494
Dr. David Alan Gilberta82d5932015-11-05 18:11:09 +00001495 do {
Juan Quintelaf20e2862017-03-21 16:19:05 +01001496 tmppages = ram_save_target_page(rs, pss, last_stage);
Dr. David Alan Gilberta82d5932015-11-05 18:11:09 +00001497 if (tmppages < 0) {
1498 return tmppages;
1499 }
1500
1501 pages += tmppages;
Juan Quintelaa935e302017-03-21 15:36:51 +01001502 pss->page++;
Dr. David Alan Gilbert1eb3fc02017-05-17 17:58:09 +01001503 } while ((pss->page & (pagesize_bits - 1)) &&
1504 offset_in_ramblock(pss->block, pss->page << TARGET_PAGE_BITS));
Dr. David Alan Gilberta82d5932015-11-05 18:11:09 +00001505
1506 /* The offset we leave with is the last one we looked at */
Juan Quintelaa935e302017-03-21 15:36:51 +01001507 pss->page--;
Dr. David Alan Gilberta82d5932015-11-05 18:11:09 +00001508 return pages;
1509}
Dr. David Alan Gilbert6c595cd2015-11-05 18:11:08 +00001510
1511/**
Juan Quintela3d0684b2017-03-23 15:06:39 +01001512 * ram_find_and_save_block: finds a dirty page and sends it to f
Juan Quintela56e93d22015-05-07 19:33:31 +02001513 *
1514 * Called within an RCU critical section.
1515 *
Juan Quintela3d0684b2017-03-23 15:06:39 +01001516 * Returns the number of pages written where zero means no dirty pages
Juan Quintela56e93d22015-05-07 19:33:31 +02001517 *
Juan Quintela6f37bb82017-03-13 19:26:29 +01001518 * @rs: current RAM state
Juan Quintela56e93d22015-05-07 19:33:31 +02001519 * @last_stage: if we are at the completion stage
Dr. David Alan Gilberta82d5932015-11-05 18:11:09 +00001520 *
1521 * On systems where host-page-size > target-page-size it will send all the
1522 * pages in a host page that are dirty.
Juan Quintela56e93d22015-05-07 19:33:31 +02001523 */
1524
Juan Quintelace25d332017-03-15 11:00:51 +01001525static int ram_find_and_save_block(RAMState *rs, bool last_stage)
Juan Quintela56e93d22015-05-07 19:33:31 +02001526{
Dr. David Alan Gilbertb8fb8cb2015-09-23 15:27:10 +01001527 PageSearchStatus pss;
Juan Quintela56e93d22015-05-07 19:33:31 +02001528 int pages = 0;
Dr. David Alan Gilbertb9e60922015-09-23 15:27:11 +01001529 bool again, found;
Juan Quintela56e93d22015-05-07 19:33:31 +02001530
Ashijeet Acharya0827b9e2017-02-08 19:58:45 +05301531 /* No dirty page as there is zero RAM */
1532 if (!ram_bytes_total()) {
1533 return pages;
1534 }
1535
Juan Quintela6f37bb82017-03-13 19:26:29 +01001536 pss.block = rs->last_seen_block;
Juan Quintelaa935e302017-03-21 15:36:51 +01001537 pss.page = rs->last_page;
Dr. David Alan Gilbertb8fb8cb2015-09-23 15:27:10 +01001538 pss.complete_round = false;
1539
1540 if (!pss.block) {
1541 pss.block = QLIST_FIRST_RCU(&ram_list.blocks);
1542 }
Juan Quintela56e93d22015-05-07 19:33:31 +02001543
Dr. David Alan Gilbertb9e60922015-09-23 15:27:11 +01001544 do {
Dr. David Alan Gilberta82d5932015-11-05 18:11:09 +00001545 again = true;
Juan Quintelaf20e2862017-03-21 16:19:05 +01001546 found = get_queued_page(rs, &pss);
Dr. David Alan Gilberta82d5932015-11-05 18:11:09 +00001547
1548 if (!found) {
1549 /* priority queue empty, so just search for something dirty */
Juan Quintelaf20e2862017-03-21 16:19:05 +01001550 found = find_dirty_block(rs, &pss, &again);
Dr. David Alan Gilberta82d5932015-11-05 18:11:09 +00001551 }
Dr. David Alan Gilbertb9e60922015-09-23 15:27:11 +01001552
1553 if (found) {
Juan Quintelaf20e2862017-03-21 16:19:05 +01001554 pages = ram_save_host_page(rs, &pss, last_stage);
Juan Quintela56e93d22015-05-07 19:33:31 +02001555 }
Dr. David Alan Gilbertb9e60922015-09-23 15:27:11 +01001556 } while (!pages && again);
Juan Quintela56e93d22015-05-07 19:33:31 +02001557
Juan Quintela6f37bb82017-03-13 19:26:29 +01001558 rs->last_seen_block = pss.block;
Juan Quintelaa935e302017-03-21 15:36:51 +01001559 rs->last_page = pss.page;
Juan Quintela56e93d22015-05-07 19:33:31 +02001560
1561 return pages;
1562}
1563
1564void acct_update_position(QEMUFile *f, size_t size, bool zero)
1565{
1566 uint64_t pages = size / TARGET_PAGE_SIZE;
Juan Quintelaf7ccd612017-03-13 20:30:21 +01001567
Juan Quintela56e93d22015-05-07 19:33:31 +02001568 if (zero) {
Juan Quintela93604472017-06-06 19:49:03 +02001569 ram_counters.duplicate += pages;
Juan Quintela56e93d22015-05-07 19:33:31 +02001570 } else {
Juan Quintela93604472017-06-06 19:49:03 +02001571 ram_counters.normal += pages;
1572 ram_counters.transferred += size;
Juan Quintela56e93d22015-05-07 19:33:31 +02001573 qemu_update_position(f, size);
1574 }
1575}
1576
Juan Quintela56e93d22015-05-07 19:33:31 +02001577uint64_t ram_bytes_total(void)
1578{
1579 RAMBlock *block;
1580 uint64_t total = 0;
1581
1582 rcu_read_lock();
Peter Xu99e15582017-05-12 12:17:39 +08001583 RAMBLOCK_FOREACH(block) {
Juan Quintela56e93d22015-05-07 19:33:31 +02001584 total += block->used_length;
Peter Xu99e15582017-05-12 12:17:39 +08001585 }
Juan Quintela56e93d22015-05-07 19:33:31 +02001586 rcu_read_unlock();
1587 return total;
1588}
1589
Juan Quintelaf265e0e2017-06-28 11:52:27 +02001590static void xbzrle_load_setup(void)
Juan Quintela56e93d22015-05-07 19:33:31 +02001591{
Juan Quintelaf265e0e2017-06-28 11:52:27 +02001592 XBZRLE.decoded_buf = g_malloc(TARGET_PAGE_SIZE);
Juan Quintela56e93d22015-05-07 19:33:31 +02001593}
1594
Juan Quintelaf265e0e2017-06-28 11:52:27 +02001595static void xbzrle_load_cleanup(void)
1596{
1597 g_free(XBZRLE.decoded_buf);
1598 XBZRLE.decoded_buf = NULL;
1599}
1600
Peter Xu7d7c96b2017-10-19 14:31:58 +08001601static void ram_state_cleanup(RAMState **rsp)
1602{
1603 migration_page_queue_free(*rsp);
1604 qemu_mutex_destroy(&(*rsp)->bitmap_mutex);
1605 qemu_mutex_destroy(&(*rsp)->src_page_req_mutex);
1606 g_free(*rsp);
1607 *rsp = NULL;
1608}
1609
Peter Xu84593a02017-10-19 14:31:59 +08001610static void xbzrle_cleanup(void)
1611{
1612 XBZRLE_cache_lock();
1613 if (XBZRLE.cache) {
1614 cache_fini(XBZRLE.cache);
1615 g_free(XBZRLE.encoded_buf);
1616 g_free(XBZRLE.current_buf);
1617 g_free(XBZRLE.zero_target_page);
1618 XBZRLE.cache = NULL;
1619 XBZRLE.encoded_buf = NULL;
1620 XBZRLE.current_buf = NULL;
1621 XBZRLE.zero_target_page = NULL;
1622 }
1623 XBZRLE_cache_unlock();
1624}
1625
Juan Quintelaf265e0e2017-06-28 11:52:27 +02001626static void ram_save_cleanup(void *opaque)
Juan Quintela56e93d22015-05-07 19:33:31 +02001627{
Juan Quintela53518d92017-05-04 11:46:24 +02001628 RAMState **rsp = opaque;
Juan Quintela6b6712e2017-03-22 15:18:04 +01001629 RAMBlock *block;
Juan Quintelaeb859c52017-03-13 21:51:55 +01001630
Li Zhijian2ff64032015-07-02 20:18:05 +08001631 /* caller have hold iothread lock or is in a bh, so there is
1632 * no writing race against this migration_bitmap
1633 */
Juan Quintela6b6712e2017-03-22 15:18:04 +01001634 memory_global_dirty_log_stop();
1635
1636 QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
1637 g_free(block->bmap);
1638 block->bmap = NULL;
1639 g_free(block->unsentmap);
1640 block->unsentmap = NULL;
Juan Quintela56e93d22015-05-07 19:33:31 +02001641 }
1642
Peter Xu84593a02017-10-19 14:31:59 +08001643 xbzrle_cleanup();
Juan Quintelaf0afa332017-06-28 11:52:28 +02001644 compress_threads_save_cleanup();
Peter Xu7d7c96b2017-10-19 14:31:58 +08001645 ram_state_cleanup(rsp);
Juan Quintela56e93d22015-05-07 19:33:31 +02001646}
1647
Juan Quintela6f37bb82017-03-13 19:26:29 +01001648static void ram_state_reset(RAMState *rs)
Juan Quintela56e93d22015-05-07 19:33:31 +02001649{
Juan Quintela6f37bb82017-03-13 19:26:29 +01001650 rs->last_seen_block = NULL;
1651 rs->last_sent_block = NULL;
Juan Quintela269ace22017-03-21 15:23:31 +01001652 rs->last_page = 0;
Juan Quintela6f37bb82017-03-13 19:26:29 +01001653 rs->last_version = ram_list.version;
1654 rs->ram_bulk_stage = true;
Juan Quintela56e93d22015-05-07 19:33:31 +02001655}
1656
1657#define MAX_WAIT 50 /* ms, half buffered_file limit */
1658
Dr. David Alan Gilbert4f2e4252015-11-05 18:10:38 +00001659/*
1660 * 'expected' is the value you expect the bitmap mostly to be full
1661 * of; it won't bother printing lines that are all this value.
1662 * If 'todump' is null the migration bitmap is dumped.
1663 */
Juan Quintela6b6712e2017-03-22 15:18:04 +01001664void ram_debug_dump_bitmap(unsigned long *todump, bool expected,
1665 unsigned long pages)
Dr. David Alan Gilbert4f2e4252015-11-05 18:10:38 +00001666{
Dr. David Alan Gilbert4f2e4252015-11-05 18:10:38 +00001667 int64_t cur;
1668 int64_t linelen = 128;
1669 char linebuf[129];
1670
Juan Quintela6b6712e2017-03-22 15:18:04 +01001671 for (cur = 0; cur < pages; cur += linelen) {
Dr. David Alan Gilbert4f2e4252015-11-05 18:10:38 +00001672 int64_t curb;
1673 bool found = false;
1674 /*
1675 * Last line; catch the case where the line length
1676 * is longer than remaining ram
1677 */
Juan Quintela6b6712e2017-03-22 15:18:04 +01001678 if (cur + linelen > pages) {
1679 linelen = pages - cur;
Dr. David Alan Gilbert4f2e4252015-11-05 18:10:38 +00001680 }
1681 for (curb = 0; curb < linelen; curb++) {
1682 bool thisbit = test_bit(cur + curb, todump);
1683 linebuf[curb] = thisbit ? '1' : '.';
1684 found = found || (thisbit != expected);
1685 }
1686 if (found) {
1687 linebuf[curb] = '\0';
1688 fprintf(stderr, "0x%08" PRIx64 " : %s\n", cur, linebuf);
1689 }
1690 }
1691}
1692
Dr. David Alan Gilberte0b266f2015-11-05 18:11:02 +00001693/* **** functions for postcopy ***** */
1694
Pavel Butsykinced1c612017-02-03 18:23:21 +03001695void ram_postcopy_migrated_memory_release(MigrationState *ms)
1696{
1697 struct RAMBlock *block;
Pavel Butsykinced1c612017-02-03 18:23:21 +03001698
Peter Xu99e15582017-05-12 12:17:39 +08001699 RAMBLOCK_FOREACH(block) {
Juan Quintela6b6712e2017-03-22 15:18:04 +01001700 unsigned long *bitmap = block->bmap;
1701 unsigned long range = block->used_length >> TARGET_PAGE_BITS;
1702 unsigned long run_start = find_next_zero_bit(bitmap, range, 0);
Pavel Butsykinced1c612017-02-03 18:23:21 +03001703
1704 while (run_start < range) {
1705 unsigned long run_end = find_next_bit(bitmap, range, run_start + 1);
Juan Quintelaaaa20642017-03-21 11:35:24 +01001706 ram_discard_range(block->idstr, run_start << TARGET_PAGE_BITS,
Pavel Butsykinced1c612017-02-03 18:23:21 +03001707 (run_end - run_start) << TARGET_PAGE_BITS);
1708 run_start = find_next_zero_bit(bitmap, range, run_end + 1);
1709 }
1710 }
1711}
1712
Juan Quintela3d0684b2017-03-23 15:06:39 +01001713/**
1714 * postcopy_send_discard_bm_ram: discard a RAMBlock
1715 *
1716 * Returns zero on success
1717 *
Dr. David Alan Gilberte0b266f2015-11-05 18:11:02 +00001718 * Callback from postcopy_each_ram_send_discard for each RAMBlock
1719 * Note: At this point the 'unsentmap' is the processed bitmap combined
1720 * with the dirtymap; so a '1' means it's either dirty or unsent.
Juan Quintela3d0684b2017-03-23 15:06:39 +01001721 *
1722 * @ms: current migration state
1723 * @pds: state for postcopy
1724 * @start: RAMBlock starting page
1725 * @length: RAMBlock size
Dr. David Alan Gilberte0b266f2015-11-05 18:11:02 +00001726 */
1727static int postcopy_send_discard_bm_ram(MigrationState *ms,
1728 PostcopyDiscardState *pds,
Juan Quintela6b6712e2017-03-22 15:18:04 +01001729 RAMBlock *block)
Dr. David Alan Gilberte0b266f2015-11-05 18:11:02 +00001730{
Juan Quintela6b6712e2017-03-22 15:18:04 +01001731 unsigned long end = block->used_length >> TARGET_PAGE_BITS;
Dr. David Alan Gilberte0b266f2015-11-05 18:11:02 +00001732 unsigned long current;
Juan Quintela6b6712e2017-03-22 15:18:04 +01001733 unsigned long *unsentmap = block->unsentmap;
Dr. David Alan Gilberte0b266f2015-11-05 18:11:02 +00001734
Juan Quintela6b6712e2017-03-22 15:18:04 +01001735 for (current = 0; current < end; ) {
Dr. David Alan Gilberte0b266f2015-11-05 18:11:02 +00001736 unsigned long one = find_next_bit(unsentmap, end, current);
1737
1738 if (one <= end) {
1739 unsigned long zero = find_next_zero_bit(unsentmap, end, one + 1);
1740 unsigned long discard_length;
1741
1742 if (zero >= end) {
1743 discard_length = end - one;
1744 } else {
1745 discard_length = zero - one;
1746 }
Dr. David Alan Gilbertd688c622016-06-13 12:16:40 +01001747 if (discard_length) {
1748 postcopy_discard_send_range(ms, pds, one, discard_length);
1749 }
Dr. David Alan Gilberte0b266f2015-11-05 18:11:02 +00001750 current = one + discard_length;
1751 } else {
1752 current = one;
1753 }
1754 }
1755
1756 return 0;
1757}
1758
Juan Quintela3d0684b2017-03-23 15:06:39 +01001759/**
1760 * postcopy_each_ram_send_discard: discard all RAMBlocks
1761 *
1762 * Returns 0 for success or negative for error
1763 *
Dr. David Alan Gilberte0b266f2015-11-05 18:11:02 +00001764 * Utility for the outgoing postcopy code.
1765 * Calls postcopy_send_discard_bm_ram for each RAMBlock
1766 * passing it bitmap indexes and name.
Dr. David Alan Gilberte0b266f2015-11-05 18:11:02 +00001767 * (qemu_ram_foreach_block ends up passing unscaled lengths
1768 * which would mean postcopy code would have to deal with target page)
Juan Quintela3d0684b2017-03-23 15:06:39 +01001769 *
1770 * @ms: current migration state
Dr. David Alan Gilberte0b266f2015-11-05 18:11:02 +00001771 */
1772static int postcopy_each_ram_send_discard(MigrationState *ms)
1773{
1774 struct RAMBlock *block;
1775 int ret;
1776
Peter Xu99e15582017-05-12 12:17:39 +08001777 RAMBLOCK_FOREACH(block) {
Juan Quintela6b6712e2017-03-22 15:18:04 +01001778 PostcopyDiscardState *pds =
1779 postcopy_discard_send_init(ms, block->idstr);
Dr. David Alan Gilberte0b266f2015-11-05 18:11:02 +00001780
1781 /*
1782 * Postcopy sends chunks of bitmap over the wire, but it
1783 * just needs indexes at this point, avoids it having
1784 * target page specific code.
1785 */
Juan Quintela6b6712e2017-03-22 15:18:04 +01001786 ret = postcopy_send_discard_bm_ram(ms, pds, block);
Dr. David Alan Gilberte0b266f2015-11-05 18:11:02 +00001787 postcopy_discard_send_finish(ms, pds);
1788 if (ret) {
1789 return ret;
1790 }
1791 }
1792
1793 return 0;
1794}
1795
Juan Quintela3d0684b2017-03-23 15:06:39 +01001796/**
1797 * postcopy_chunk_hostpages_pass: canocalize bitmap in hostpages
Dr. David Alan Gilbert99e314e2015-11-05 18:11:15 +00001798 *
Juan Quintela3d0684b2017-03-23 15:06:39 +01001799 * Helper for postcopy_chunk_hostpages; it's called twice to
1800 * canonicalize the two bitmaps, that are similar, but one is
1801 * inverted.
Dr. David Alan Gilbert99e314e2015-11-05 18:11:15 +00001802 *
Juan Quintela3d0684b2017-03-23 15:06:39 +01001803 * Postcopy requires that all target pages in a hostpage are dirty or
1804 * clean, not a mix. This function canonicalizes the bitmaps.
1805 *
1806 * @ms: current migration state
1807 * @unsent_pass: if true we need to canonicalize partially unsent host pages
1808 * otherwise we need to canonicalize partially dirty host pages
1809 * @block: block that contains the page we want to canonicalize
1810 * @pds: state for postcopy
Dr. David Alan Gilbert99e314e2015-11-05 18:11:15 +00001811 */
1812static void postcopy_chunk_hostpages_pass(MigrationState *ms, bool unsent_pass,
1813 RAMBlock *block,
1814 PostcopyDiscardState *pds)
1815{
Juan Quintela53518d92017-05-04 11:46:24 +02001816 RAMState *rs = ram_state;
Juan Quintela6b6712e2017-03-22 15:18:04 +01001817 unsigned long *bitmap = block->bmap;
1818 unsigned long *unsentmap = block->unsentmap;
Dr. David Alan Gilbert29c59172017-02-24 18:28:31 +00001819 unsigned int host_ratio = block->page_size / TARGET_PAGE_SIZE;
Juan Quintela6b6712e2017-03-22 15:18:04 +01001820 unsigned long pages = block->used_length >> TARGET_PAGE_BITS;
Dr. David Alan Gilbert99e314e2015-11-05 18:11:15 +00001821 unsigned long run_start;
1822
Dr. David Alan Gilbert29c59172017-02-24 18:28:31 +00001823 if (block->page_size == TARGET_PAGE_SIZE) {
1824 /* Easy case - TPS==HPS for a non-huge page RAMBlock */
1825 return;
1826 }
1827
Dr. David Alan Gilbert99e314e2015-11-05 18:11:15 +00001828 if (unsent_pass) {
1829 /* Find a sent page */
Juan Quintela6b6712e2017-03-22 15:18:04 +01001830 run_start = find_next_zero_bit(unsentmap, pages, 0);
Dr. David Alan Gilbert99e314e2015-11-05 18:11:15 +00001831 } else {
1832 /* Find a dirty page */
Juan Quintela6b6712e2017-03-22 15:18:04 +01001833 run_start = find_next_bit(bitmap, pages, 0);
Dr. David Alan Gilbert99e314e2015-11-05 18:11:15 +00001834 }
1835
Juan Quintela6b6712e2017-03-22 15:18:04 +01001836 while (run_start < pages) {
Dr. David Alan Gilbert99e314e2015-11-05 18:11:15 +00001837 bool do_fixup = false;
1838 unsigned long fixup_start_addr;
1839 unsigned long host_offset;
1840
1841 /*
1842 * If the start of this run of pages is in the middle of a host
1843 * page, then we need to fixup this host page.
1844 */
1845 host_offset = run_start % host_ratio;
1846 if (host_offset) {
1847 do_fixup = true;
1848 run_start -= host_offset;
1849 fixup_start_addr = run_start;
1850 /* For the next pass */
1851 run_start = run_start + host_ratio;
1852 } else {
1853 /* Find the end of this run */
1854 unsigned long run_end;
1855 if (unsent_pass) {
Juan Quintela6b6712e2017-03-22 15:18:04 +01001856 run_end = find_next_bit(unsentmap, pages, run_start + 1);
Dr. David Alan Gilbert99e314e2015-11-05 18:11:15 +00001857 } else {
Juan Quintela6b6712e2017-03-22 15:18:04 +01001858 run_end = find_next_zero_bit(bitmap, pages, run_start + 1);
Dr. David Alan Gilbert99e314e2015-11-05 18:11:15 +00001859 }
1860 /*
1861 * If the end isn't at the start of a host page, then the
1862 * run doesn't finish at the end of a host page
1863 * and we need to discard.
1864 */
1865 host_offset = run_end % host_ratio;
1866 if (host_offset) {
1867 do_fixup = true;
1868 fixup_start_addr = run_end - host_offset;
1869 /*
1870 * This host page has gone, the next loop iteration starts
1871 * from after the fixup
1872 */
1873 run_start = fixup_start_addr + host_ratio;
1874 } else {
1875 /*
1876 * No discards on this iteration, next loop starts from
1877 * next sent/dirty page
1878 */
1879 run_start = run_end + 1;
1880 }
1881 }
1882
1883 if (do_fixup) {
1884 unsigned long page;
1885
1886 /* Tell the destination to discard this page */
1887 if (unsent_pass || !test_bit(fixup_start_addr, unsentmap)) {
1888 /* For the unsent_pass we:
1889 * discard partially sent pages
1890 * For the !unsent_pass (dirty) we:
1891 * discard partially dirty pages that were sent
1892 * (any partially sent pages were already discarded
1893 * by the previous unsent_pass)
1894 */
1895 postcopy_discard_send_range(ms, pds, fixup_start_addr,
1896 host_ratio);
1897 }
1898
1899 /* Clean up the bitmap */
1900 for (page = fixup_start_addr;
1901 page < fixup_start_addr + host_ratio; page++) {
1902 /* All pages in this host page are now not sent */
1903 set_bit(page, unsentmap);
1904
1905 /*
1906 * Remark them as dirty, updating the count for any pages
1907 * that weren't previously dirty.
1908 */
Juan Quintela0d8ec882017-03-13 21:21:41 +01001909 rs->migration_dirty_pages += !test_and_set_bit(page, bitmap);
Dr. David Alan Gilbert99e314e2015-11-05 18:11:15 +00001910 }
1911 }
1912
1913 if (unsent_pass) {
1914 /* Find the next sent page for the next iteration */
Juan Quintela6b6712e2017-03-22 15:18:04 +01001915 run_start = find_next_zero_bit(unsentmap, pages, run_start);
Dr. David Alan Gilbert99e314e2015-11-05 18:11:15 +00001916 } else {
1917 /* Find the next dirty page for the next iteration */
Juan Quintela6b6712e2017-03-22 15:18:04 +01001918 run_start = find_next_bit(bitmap, pages, run_start);
Dr. David Alan Gilbert99e314e2015-11-05 18:11:15 +00001919 }
1920 }
1921}
1922
Juan Quintela3d0684b2017-03-23 15:06:39 +01001923/**
1924 * postcopy_chuck_hostpages: discrad any partially sent host page
1925 *
Dr. David Alan Gilbert99e314e2015-11-05 18:11:15 +00001926 * Utility for the outgoing postcopy code.
1927 *
1928 * Discard any partially sent host-page size chunks, mark any partially
Dr. David Alan Gilbert29c59172017-02-24 18:28:31 +00001929 * dirty host-page size chunks as all dirty. In this case the host-page
1930 * is the host-page for the particular RAMBlock, i.e. it might be a huge page
Dr. David Alan Gilbert99e314e2015-11-05 18:11:15 +00001931 *
Juan Quintela3d0684b2017-03-23 15:06:39 +01001932 * Returns zero on success
1933 *
1934 * @ms: current migration state
Juan Quintela6b6712e2017-03-22 15:18:04 +01001935 * @block: block we want to work with
Dr. David Alan Gilbert99e314e2015-11-05 18:11:15 +00001936 */
Juan Quintela6b6712e2017-03-22 15:18:04 +01001937static int postcopy_chunk_hostpages(MigrationState *ms, RAMBlock *block)
Dr. David Alan Gilbert99e314e2015-11-05 18:11:15 +00001938{
Juan Quintela6b6712e2017-03-22 15:18:04 +01001939 PostcopyDiscardState *pds =
1940 postcopy_discard_send_init(ms, block->idstr);
Dr. David Alan Gilbert99e314e2015-11-05 18:11:15 +00001941
Juan Quintela6b6712e2017-03-22 15:18:04 +01001942 /* First pass: Discard all partially sent host pages */
1943 postcopy_chunk_hostpages_pass(ms, true, block, pds);
1944 /*
1945 * Second pass: Ensure that all partially dirty host pages are made
1946 * fully dirty.
1947 */
1948 postcopy_chunk_hostpages_pass(ms, false, block, pds);
Dr. David Alan Gilbert99e314e2015-11-05 18:11:15 +00001949
Juan Quintela6b6712e2017-03-22 15:18:04 +01001950 postcopy_discard_send_finish(ms, pds);
Dr. David Alan Gilbert99e314e2015-11-05 18:11:15 +00001951 return 0;
1952}
1953
Juan Quintela3d0684b2017-03-23 15:06:39 +01001954/**
1955 * ram_postcopy_send_discard_bitmap: transmit the discard bitmap
1956 *
1957 * Returns zero on success
1958 *
Dr. David Alan Gilberte0b266f2015-11-05 18:11:02 +00001959 * Transmit the set of pages to be discarded after precopy to the target
1960 * these are pages that:
1961 * a) Have been previously transmitted but are now dirty again
1962 * b) Pages that have never been transmitted, this ensures that
1963 * any pages on the destination that have been mapped by background
1964 * tasks get discarded (transparent huge pages is the specific concern)
1965 * Hopefully this is pretty sparse
Juan Quintela3d0684b2017-03-23 15:06:39 +01001966 *
1967 * @ms: current migration state
Dr. David Alan Gilberte0b266f2015-11-05 18:11:02 +00001968 */
1969int ram_postcopy_send_discard_bitmap(MigrationState *ms)
1970{
Juan Quintela53518d92017-05-04 11:46:24 +02001971 RAMState *rs = ram_state;
Juan Quintela6b6712e2017-03-22 15:18:04 +01001972 RAMBlock *block;
Dr. David Alan Gilberte0b266f2015-11-05 18:11:02 +00001973 int ret;
Dr. David Alan Gilberte0b266f2015-11-05 18:11:02 +00001974
1975 rcu_read_lock();
1976
1977 /* This should be our last sync, the src is now paused */
Juan Quintelaeb859c52017-03-13 21:51:55 +01001978 migration_bitmap_sync(rs);
Dr. David Alan Gilberte0b266f2015-11-05 18:11:02 +00001979
Juan Quintela6b6712e2017-03-22 15:18:04 +01001980 /* Easiest way to make sure we don't resume in the middle of a host-page */
1981 rs->last_seen_block = NULL;
1982 rs->last_sent_block = NULL;
1983 rs->last_page = 0;
1984
1985 QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
1986 unsigned long pages = block->used_length >> TARGET_PAGE_BITS;
1987 unsigned long *bitmap = block->bmap;
1988 unsigned long *unsentmap = block->unsentmap;
1989
1990 if (!unsentmap) {
1991 /* We don't have a safe way to resize the sentmap, so
1992 * if the bitmap was resized it will be NULL at this
1993 * point.
1994 */
1995 error_report("migration ram resized during precopy phase");
1996 rcu_read_unlock();
1997 return -EINVAL;
1998 }
1999 /* Deal with TPS != HPS and huge pages */
2000 ret = postcopy_chunk_hostpages(ms, block);
2001 if (ret) {
2002 rcu_read_unlock();
2003 return ret;
2004 }
2005
2006 /*
2007 * Update the unsentmap to be unsentmap = unsentmap | dirty
Dr. David Alan Gilberte0b266f2015-11-05 18:11:02 +00002008 */
Juan Quintela6b6712e2017-03-22 15:18:04 +01002009 bitmap_or(unsentmap, unsentmap, bitmap, pages);
Dr. David Alan Gilberte0b266f2015-11-05 18:11:02 +00002010#ifdef DEBUG_POSTCOPY
Juan Quintela6b6712e2017-03-22 15:18:04 +01002011 ram_debug_dump_bitmap(unsentmap, true, pages);
Dr. David Alan Gilberte0b266f2015-11-05 18:11:02 +00002012#endif
Juan Quintela6b6712e2017-03-22 15:18:04 +01002013 }
2014 trace_ram_postcopy_send_discard_bitmap();
Dr. David Alan Gilberte0b266f2015-11-05 18:11:02 +00002015
2016 ret = postcopy_each_ram_send_discard(ms);
2017 rcu_read_unlock();
2018
2019 return ret;
2020}
2021
Juan Quintela3d0684b2017-03-23 15:06:39 +01002022/**
2023 * ram_discard_range: discard dirtied pages at the beginning of postcopy
Dr. David Alan Gilberte0b266f2015-11-05 18:11:02 +00002024 *
Juan Quintela3d0684b2017-03-23 15:06:39 +01002025 * Returns zero on success
Dr. David Alan Gilberte0b266f2015-11-05 18:11:02 +00002026 *
Juan Quintela36449152017-03-23 15:11:59 +01002027 * @rbname: name of the RAMBlock of the request. NULL means the
2028 * same that last one.
Juan Quintela3d0684b2017-03-23 15:06:39 +01002029 * @start: RAMBlock starting page
2030 * @length: RAMBlock size
Dr. David Alan Gilberte0b266f2015-11-05 18:11:02 +00002031 */
Juan Quintelaaaa20642017-03-21 11:35:24 +01002032int ram_discard_range(const char *rbname, uint64_t start, size_t length)
Dr. David Alan Gilberte0b266f2015-11-05 18:11:02 +00002033{
2034 int ret = -1;
2035
Juan Quintela36449152017-03-23 15:11:59 +01002036 trace_ram_discard_range(rbname, start, length);
Dr. David Alan Gilbertd3a50382017-02-24 18:28:32 +00002037
Dr. David Alan Gilberte0b266f2015-11-05 18:11:02 +00002038 rcu_read_lock();
Juan Quintela36449152017-03-23 15:11:59 +01002039 RAMBlock *rb = qemu_ram_block_by_name(rbname);
Dr. David Alan Gilberte0b266f2015-11-05 18:11:02 +00002040
2041 if (!rb) {
Juan Quintela36449152017-03-23 15:11:59 +01002042 error_report("ram_discard_range: Failed to find block '%s'", rbname);
Dr. David Alan Gilberte0b266f2015-11-05 18:11:02 +00002043 goto err;
2044 }
2045
Alexey Perevalovf9494612017-10-05 14:13:20 +03002046 bitmap_clear(rb->receivedmap, start >> qemu_target_page_bits(),
2047 length >> qemu_target_page_bits());
Dr. David Alan Gilbertd3a50382017-02-24 18:28:32 +00002048 ret = ram_block_discard_range(rb, start, length);
Dr. David Alan Gilberte0b266f2015-11-05 18:11:02 +00002049
2050err:
2051 rcu_read_unlock();
2052
2053 return ret;
2054}
2055
Peter Xu84593a02017-10-19 14:31:59 +08002056/*
2057 * For every allocation, we will try not to crash the VM if the
2058 * allocation failed.
2059 */
2060static int xbzrle_init(void)
2061{
2062 Error *local_err = NULL;
2063
2064 if (!migrate_use_xbzrle()) {
2065 return 0;
2066 }
2067
2068 XBZRLE_cache_lock();
2069
2070 XBZRLE.zero_target_page = g_try_malloc0(TARGET_PAGE_SIZE);
2071 if (!XBZRLE.zero_target_page) {
2072 error_report("%s: Error allocating zero page", __func__);
2073 goto err_out;
2074 }
2075
2076 XBZRLE.cache = cache_init(migrate_xbzrle_cache_size(),
2077 TARGET_PAGE_SIZE, &local_err);
2078 if (!XBZRLE.cache) {
2079 error_report_err(local_err);
2080 goto free_zero_page;
2081 }
2082
2083 XBZRLE.encoded_buf = g_try_malloc0(TARGET_PAGE_SIZE);
2084 if (!XBZRLE.encoded_buf) {
2085 error_report("%s: Error allocating encoded_buf", __func__);
2086 goto free_cache;
2087 }
2088
2089 XBZRLE.current_buf = g_try_malloc(TARGET_PAGE_SIZE);
2090 if (!XBZRLE.current_buf) {
2091 error_report("%s: Error allocating current_buf", __func__);
2092 goto free_encoded_buf;
2093 }
2094
2095 /* We are all good */
2096 XBZRLE_cache_unlock();
2097 return 0;
2098
2099free_encoded_buf:
2100 g_free(XBZRLE.encoded_buf);
2101 XBZRLE.encoded_buf = NULL;
2102free_cache:
2103 cache_fini(XBZRLE.cache);
2104 XBZRLE.cache = NULL;
2105free_zero_page:
2106 g_free(XBZRLE.zero_target_page);
2107 XBZRLE.zero_target_page = NULL;
2108err_out:
2109 XBZRLE_cache_unlock();
2110 return -ENOMEM;
2111}
2112
Juan Quintela53518d92017-05-04 11:46:24 +02002113static int ram_state_init(RAMState **rsp)
Juan Quintela56e93d22015-05-07 19:33:31 +02002114{
Peter Xu7d00ee62017-10-19 14:31:57 +08002115 *rsp = g_try_new0(RAMState, 1);
2116
2117 if (!*rsp) {
2118 error_report("%s: Init ramstate fail", __func__);
2119 return -1;
2120 }
Juan Quintela53518d92017-05-04 11:46:24 +02002121
2122 qemu_mutex_init(&(*rsp)->bitmap_mutex);
2123 qemu_mutex_init(&(*rsp)->src_page_req_mutex);
2124 QSIMPLEQ_INIT(&(*rsp)->src_page_requests);
Juan Quintela56e93d22015-05-07 19:33:31 +02002125
Peter Xu7d00ee62017-10-19 14:31:57 +08002126 /*
2127 * Count the total number of pages used by ram blocks not including any
2128 * gaps due to alignment or unplugs.
2129 */
2130 (*rsp)->migration_dirty_pages = ram_bytes_total() >> TARGET_PAGE_BITS;
2131
2132 ram_state_reset(*rsp);
2133
2134 return 0;
2135}
2136
Peter Xud6eff5d2017-10-19 14:32:00 +08002137static void ram_list_init_bitmaps(void)
2138{
2139 RAMBlock *block;
2140 unsigned long pages;
2141
2142 /* Skip setting bitmap if there is no RAM */
2143 if (ram_bytes_total()) {
2144 QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
2145 pages = block->max_length >> TARGET_PAGE_BITS;
2146 block->bmap = bitmap_new(pages);
2147 bitmap_set(block->bmap, 0, pages);
2148 if (migrate_postcopy_ram()) {
2149 block->unsentmap = bitmap_new(pages);
2150 bitmap_set(block->unsentmap, 0, pages);
2151 }
2152 }
2153 }
2154}
2155
2156static void ram_init_bitmaps(RAMState *rs)
2157{
2158 /* For memory_global_dirty_log_start below. */
2159 qemu_mutex_lock_iothread();
2160 qemu_mutex_lock_ramlist();
2161 rcu_read_lock();
2162
2163 ram_list_init_bitmaps();
2164 memory_global_dirty_log_start();
2165 migration_bitmap_sync(rs);
2166
2167 rcu_read_unlock();
2168 qemu_mutex_unlock_ramlist();
2169 qemu_mutex_unlock_iothread();
2170}
2171
Peter Xu7d00ee62017-10-19 14:31:57 +08002172static int ram_init_all(RAMState **rsp)
2173{
Peter Xu7d00ee62017-10-19 14:31:57 +08002174 if (ram_state_init(rsp)) {
2175 return -1;
2176 }
2177
Peter Xu84593a02017-10-19 14:31:59 +08002178 if (xbzrle_init()) {
2179 ram_state_cleanup(rsp);
2180 return -1;
Juan Quintela56e93d22015-05-07 19:33:31 +02002181 }
2182
Peter Xud6eff5d2017-10-19 14:32:00 +08002183 ram_init_bitmaps(*rsp);
zhanghailianga91246c2016-10-27 14:42:59 +08002184
2185 return 0;
2186}
2187
Juan Quintela3d0684b2017-03-23 15:06:39 +01002188/*
2189 * Each of ram_save_setup, ram_save_iterate and ram_save_complete has
zhanghailianga91246c2016-10-27 14:42:59 +08002190 * long-running RCU critical section. When rcu-reclaims in the code
2191 * start to become numerous it will be necessary to reduce the
2192 * granularity of these critical sections.
2193 */
2194
Juan Quintela3d0684b2017-03-23 15:06:39 +01002195/**
2196 * ram_save_setup: Setup RAM for migration
2197 *
2198 * Returns zero to indicate success and negative for error
2199 *
2200 * @f: QEMUFile where to send the data
2201 * @opaque: RAMState pointer
2202 */
zhanghailianga91246c2016-10-27 14:42:59 +08002203static int ram_save_setup(QEMUFile *f, void *opaque)
2204{
Juan Quintela53518d92017-05-04 11:46:24 +02002205 RAMState **rsp = opaque;
zhanghailianga91246c2016-10-27 14:42:59 +08002206 RAMBlock *block;
2207
2208 /* migration has already setup the bitmap, reuse it. */
2209 if (!migration_in_colo_state()) {
Peter Xu7d00ee62017-10-19 14:31:57 +08002210 if (ram_init_all(rsp) != 0) {
zhanghailianga91246c2016-10-27 14:42:59 +08002211 return -1;
Juan Quintela53518d92017-05-04 11:46:24 +02002212 }
zhanghailianga91246c2016-10-27 14:42:59 +08002213 }
Juan Quintela53518d92017-05-04 11:46:24 +02002214 (*rsp)->f = f;
zhanghailianga91246c2016-10-27 14:42:59 +08002215
2216 rcu_read_lock();
Juan Quintela56e93d22015-05-07 19:33:31 +02002217
2218 qemu_put_be64(f, ram_bytes_total() | RAM_SAVE_FLAG_MEM_SIZE);
2219
Peter Xu99e15582017-05-12 12:17:39 +08002220 RAMBLOCK_FOREACH(block) {
Juan Quintela56e93d22015-05-07 19:33:31 +02002221 qemu_put_byte(f, strlen(block->idstr));
2222 qemu_put_buffer(f, (uint8_t *)block->idstr, strlen(block->idstr));
2223 qemu_put_be64(f, block->used_length);
Dr. David Alan Gilbertef08fb32017-02-24 18:28:30 +00002224 if (migrate_postcopy_ram() && block->page_size != qemu_host_page_size) {
2225 qemu_put_be64(f, block->page_size);
2226 }
Juan Quintela56e93d22015-05-07 19:33:31 +02002227 }
2228
2229 rcu_read_unlock();
Juan Quintelaf0afa332017-06-28 11:52:28 +02002230 compress_threads_save_setup();
Juan Quintela56e93d22015-05-07 19:33:31 +02002231
2232 ram_control_before_iterate(f, RAM_CONTROL_SETUP);
2233 ram_control_after_iterate(f, RAM_CONTROL_SETUP);
2234
2235 qemu_put_be64(f, RAM_SAVE_FLAG_EOS);
2236
2237 return 0;
2238}
2239
Juan Quintela3d0684b2017-03-23 15:06:39 +01002240/**
2241 * ram_save_iterate: iterative stage for migration
2242 *
2243 * Returns zero to indicate success and negative for error
2244 *
2245 * @f: QEMUFile where to send the data
2246 * @opaque: RAMState pointer
2247 */
Juan Quintela56e93d22015-05-07 19:33:31 +02002248static int ram_save_iterate(QEMUFile *f, void *opaque)
2249{
Juan Quintela53518d92017-05-04 11:46:24 +02002250 RAMState **temp = opaque;
2251 RAMState *rs = *temp;
Juan Quintela56e93d22015-05-07 19:33:31 +02002252 int ret;
2253 int i;
2254 int64_t t0;
Thomas Huth5c903082016-11-04 14:10:17 +01002255 int done = 0;
Juan Quintela56e93d22015-05-07 19:33:31 +02002256
2257 rcu_read_lock();
Juan Quintela6f37bb82017-03-13 19:26:29 +01002258 if (ram_list.version != rs->last_version) {
2259 ram_state_reset(rs);
Juan Quintela56e93d22015-05-07 19:33:31 +02002260 }
2261
2262 /* Read version before ram_list.blocks */
2263 smp_rmb();
2264
2265 ram_control_before_iterate(f, RAM_CONTROL_ROUND);
2266
2267 t0 = qemu_clock_get_ns(QEMU_CLOCK_REALTIME);
2268 i = 0;
2269 while ((ret = qemu_file_rate_limit(f)) == 0) {
2270 int pages;
2271
Juan Quintelace25d332017-03-15 11:00:51 +01002272 pages = ram_find_and_save_block(rs, false);
Juan Quintela56e93d22015-05-07 19:33:31 +02002273 /* no more pages to sent */
2274 if (pages == 0) {
Thomas Huth5c903082016-11-04 14:10:17 +01002275 done = 1;
Juan Quintela56e93d22015-05-07 19:33:31 +02002276 break;
2277 }
Juan Quintela23b28c32017-03-13 20:51:34 +01002278 rs->iterations++;
Jason J. Herne070afca2015-09-08 13:12:35 -04002279
Juan Quintela56e93d22015-05-07 19:33:31 +02002280 /* we want to check in the 1st loop, just in case it was the 1st time
2281 and we had to sync the dirty bitmap.
2282 qemu_get_clock_ns() is a bit expensive, so we only check each some
2283 iterations
2284 */
2285 if ((i & 63) == 0) {
2286 uint64_t t1 = (qemu_clock_get_ns(QEMU_CLOCK_REALTIME) - t0) / 1000000;
2287 if (t1 > MAX_WAIT) {
Juan Quintela55c44462017-01-23 22:32:05 +01002288 trace_ram_save_iterate_big_wait(t1, i);
Juan Quintela56e93d22015-05-07 19:33:31 +02002289 break;
2290 }
2291 }
2292 i++;
2293 }
Juan Quintelace25d332017-03-15 11:00:51 +01002294 flush_compressed_data(rs);
Juan Quintela56e93d22015-05-07 19:33:31 +02002295 rcu_read_unlock();
2296
2297 /*
2298 * Must occur before EOS (or any QEMUFile operation)
2299 * because of RDMA protocol.
2300 */
2301 ram_control_after_iterate(f, RAM_CONTROL_ROUND);
2302
2303 qemu_put_be64(f, RAM_SAVE_FLAG_EOS);
Juan Quintela93604472017-06-06 19:49:03 +02002304 ram_counters.transferred += 8;
Juan Quintela56e93d22015-05-07 19:33:31 +02002305
2306 ret = qemu_file_get_error(f);
2307 if (ret < 0) {
2308 return ret;
2309 }
2310
Thomas Huth5c903082016-11-04 14:10:17 +01002311 return done;
Juan Quintela56e93d22015-05-07 19:33:31 +02002312}
2313
Juan Quintela3d0684b2017-03-23 15:06:39 +01002314/**
2315 * ram_save_complete: function called to send the remaining amount of ram
2316 *
2317 * Returns zero to indicate success
2318 *
2319 * Called with iothread lock
2320 *
2321 * @f: QEMUFile where to send the data
2322 * @opaque: RAMState pointer
2323 */
Juan Quintela56e93d22015-05-07 19:33:31 +02002324static int ram_save_complete(QEMUFile *f, void *opaque)
2325{
Juan Quintela53518d92017-05-04 11:46:24 +02002326 RAMState **temp = opaque;
2327 RAMState *rs = *temp;
Juan Quintela6f37bb82017-03-13 19:26:29 +01002328
Juan Quintela56e93d22015-05-07 19:33:31 +02002329 rcu_read_lock();
2330
Juan Quintela57273092017-03-20 22:25:28 +01002331 if (!migration_in_postcopy()) {
Juan Quintela8d820d62017-03-13 19:35:50 +01002332 migration_bitmap_sync(rs);
Dr. David Alan Gilbert663e6c12015-11-05 18:11:13 +00002333 }
Juan Quintela56e93d22015-05-07 19:33:31 +02002334
2335 ram_control_before_iterate(f, RAM_CONTROL_FINISH);
2336
2337 /* try transferring iterative blocks of memory */
2338
2339 /* flush all remaining blocks regardless of rate limiting */
2340 while (true) {
2341 int pages;
2342
Juan Quintelace25d332017-03-15 11:00:51 +01002343 pages = ram_find_and_save_block(rs, !migration_in_colo_state());
Juan Quintela56e93d22015-05-07 19:33:31 +02002344 /* no more blocks to sent */
2345 if (pages == 0) {
2346 break;
2347 }
2348 }
2349
Juan Quintelace25d332017-03-15 11:00:51 +01002350 flush_compressed_data(rs);
Juan Quintela56e93d22015-05-07 19:33:31 +02002351 ram_control_after_iterate(f, RAM_CONTROL_FINISH);
Juan Quintela56e93d22015-05-07 19:33:31 +02002352
2353 rcu_read_unlock();
Paolo Bonzinid09a6fd2015-07-09 08:47:58 +02002354
Juan Quintela56e93d22015-05-07 19:33:31 +02002355 qemu_put_be64(f, RAM_SAVE_FLAG_EOS);
2356
2357 return 0;
2358}
2359
Dr. David Alan Gilbertc31b0982015-11-05 18:10:54 +00002360static void ram_save_pending(QEMUFile *f, void *opaque, uint64_t max_size,
2361 uint64_t *non_postcopiable_pending,
2362 uint64_t *postcopiable_pending)
Juan Quintela56e93d22015-05-07 19:33:31 +02002363{
Juan Quintela53518d92017-05-04 11:46:24 +02002364 RAMState **temp = opaque;
2365 RAMState *rs = *temp;
Juan Quintela56e93d22015-05-07 19:33:31 +02002366 uint64_t remaining_size;
2367
Juan Quintela9edabd42017-03-14 12:02:16 +01002368 remaining_size = rs->migration_dirty_pages * TARGET_PAGE_SIZE;
Juan Quintela56e93d22015-05-07 19:33:31 +02002369
Juan Quintela57273092017-03-20 22:25:28 +01002370 if (!migration_in_postcopy() &&
Dr. David Alan Gilbert663e6c12015-11-05 18:11:13 +00002371 remaining_size < max_size) {
Juan Quintela56e93d22015-05-07 19:33:31 +02002372 qemu_mutex_lock_iothread();
2373 rcu_read_lock();
Juan Quintela8d820d62017-03-13 19:35:50 +01002374 migration_bitmap_sync(rs);
Juan Quintela56e93d22015-05-07 19:33:31 +02002375 rcu_read_unlock();
2376 qemu_mutex_unlock_iothread();
Juan Quintela9edabd42017-03-14 12:02:16 +01002377 remaining_size = rs->migration_dirty_pages * TARGET_PAGE_SIZE;
Juan Quintela56e93d22015-05-07 19:33:31 +02002378 }
Dr. David Alan Gilbertc31b0982015-11-05 18:10:54 +00002379
Vladimir Sementsov-Ogievskiy86e11672017-07-10 19:30:15 +03002380 if (migrate_postcopy_ram()) {
2381 /* We can do postcopy, and all the data is postcopiable */
2382 *postcopiable_pending += remaining_size;
2383 } else {
2384 *non_postcopiable_pending += remaining_size;
2385 }
Juan Quintela56e93d22015-05-07 19:33:31 +02002386}
2387
2388static int load_xbzrle(QEMUFile *f, ram_addr_t addr, void *host)
2389{
2390 unsigned int xh_len;
2391 int xh_flags;
Dr. David Alan Gilbert063e7602015-12-16 11:47:37 +00002392 uint8_t *loaded_data;
Juan Quintela56e93d22015-05-07 19:33:31 +02002393
Juan Quintela56e93d22015-05-07 19:33:31 +02002394 /* extract RLE header */
2395 xh_flags = qemu_get_byte(f);
2396 xh_len = qemu_get_be16(f);
2397
2398 if (xh_flags != ENCODING_FLAG_XBZRLE) {
2399 error_report("Failed to load XBZRLE page - wrong compression!");
2400 return -1;
2401 }
2402
2403 if (xh_len > TARGET_PAGE_SIZE) {
2404 error_report("Failed to load XBZRLE page - len overflow!");
2405 return -1;
2406 }
Juan Quintelaf265e0e2017-06-28 11:52:27 +02002407 loaded_data = XBZRLE.decoded_buf;
Juan Quintela56e93d22015-05-07 19:33:31 +02002408 /* load data and decode */
Juan Quintelaf265e0e2017-06-28 11:52:27 +02002409 /* it can change loaded_data to point to an internal buffer */
Dr. David Alan Gilbert063e7602015-12-16 11:47:37 +00002410 qemu_get_buffer_in_place(f, &loaded_data, xh_len);
Juan Quintela56e93d22015-05-07 19:33:31 +02002411
2412 /* decode RLE */
Dr. David Alan Gilbert063e7602015-12-16 11:47:37 +00002413 if (xbzrle_decode_buffer(loaded_data, xh_len, host,
Juan Quintela56e93d22015-05-07 19:33:31 +02002414 TARGET_PAGE_SIZE) == -1) {
2415 error_report("Failed to load XBZRLE page - decode error!");
2416 return -1;
2417 }
2418
2419 return 0;
2420}
2421
Juan Quintela3d0684b2017-03-23 15:06:39 +01002422/**
2423 * ram_block_from_stream: read a RAMBlock id from the migration stream
Dr. David Alan Gilberta7180872015-11-05 18:11:11 +00002424 *
Juan Quintela3d0684b2017-03-23 15:06:39 +01002425 * Must be called from within a rcu critical section.
2426 *
2427 * Returns a pointer from within the RCU-protected ram_list.
2428 *
2429 * @f: QEMUFile where to read the data from
2430 * @flags: Page flags (mostly to see if it's a continuation of previous block)
Dr. David Alan Gilberta7180872015-11-05 18:11:11 +00002431 */
Juan Quintela3d0684b2017-03-23 15:06:39 +01002432static inline RAMBlock *ram_block_from_stream(QEMUFile *f, int flags)
Juan Quintela56e93d22015-05-07 19:33:31 +02002433{
2434 static RAMBlock *block = NULL;
2435 char id[256];
2436 uint8_t len;
2437
2438 if (flags & RAM_SAVE_FLAG_CONTINUE) {
zhanghailiang4c4bad42016-01-15 11:37:41 +08002439 if (!block) {
Juan Quintela56e93d22015-05-07 19:33:31 +02002440 error_report("Ack, bad migration stream!");
2441 return NULL;
2442 }
zhanghailiang4c4bad42016-01-15 11:37:41 +08002443 return block;
Juan Quintela56e93d22015-05-07 19:33:31 +02002444 }
2445
2446 len = qemu_get_byte(f);
2447 qemu_get_buffer(f, (uint8_t *)id, len);
2448 id[len] = 0;
2449
Dr. David Alan Gilberte3dd7492015-11-05 18:10:33 +00002450 block = qemu_ram_block_by_name(id);
zhanghailiang4c4bad42016-01-15 11:37:41 +08002451 if (!block) {
2452 error_report("Can't find block %s", id);
2453 return NULL;
Juan Quintela56e93d22015-05-07 19:33:31 +02002454 }
2455
zhanghailiang4c4bad42016-01-15 11:37:41 +08002456 return block;
2457}
2458
2459static inline void *host_from_ram_block_offset(RAMBlock *block,
2460 ram_addr_t offset)
2461{
2462 if (!offset_in_ramblock(block, offset)) {
2463 return NULL;
2464 }
2465
2466 return block->host + offset;
Juan Quintela56e93d22015-05-07 19:33:31 +02002467}
2468
Juan Quintela3d0684b2017-03-23 15:06:39 +01002469/**
2470 * ram_handle_compressed: handle the zero page case
2471 *
Juan Quintela56e93d22015-05-07 19:33:31 +02002472 * If a page (or a whole RDMA chunk) has been
2473 * determined to be zero, then zap it.
Juan Quintela3d0684b2017-03-23 15:06:39 +01002474 *
2475 * @host: host address for the zero page
2476 * @ch: what the page is filled from. We only support zero
2477 * @size: size of the zero page
Juan Quintela56e93d22015-05-07 19:33:31 +02002478 */
2479void ram_handle_compressed(void *host, uint8_t ch, uint64_t size)
2480{
2481 if (ch != 0 || !is_zero_range(host, size)) {
2482 memset(host, ch, size);
2483 }
2484}
2485
2486static void *do_data_decompress(void *opaque)
2487{
2488 DecompressParam *param = opaque;
2489 unsigned long pagesize;
Liang Li33d151f2016-05-05 15:32:58 +08002490 uint8_t *des;
2491 int len;
Juan Quintela56e93d22015-05-07 19:33:31 +02002492
Liang Li33d151f2016-05-05 15:32:58 +08002493 qemu_mutex_lock(&param->mutex);
Liang Li90e56fb2016-05-05 15:32:56 +08002494 while (!param->quit) {
Liang Li33d151f2016-05-05 15:32:58 +08002495 if (param->des) {
2496 des = param->des;
2497 len = param->len;
2498 param->des = 0;
2499 qemu_mutex_unlock(&param->mutex);
2500
Liang Li73a89122016-05-05 15:32:51 +08002501 pagesize = TARGET_PAGE_SIZE;
2502 /* uncompress() will return failed in some case, especially
2503 * when the page is dirted when doing the compression, it's
2504 * not a problem because the dirty page will be retransferred
2505 * and uncompress() won't break the data in other pages.
2506 */
Liang Li33d151f2016-05-05 15:32:58 +08002507 uncompress((Bytef *)des, &pagesize,
2508 (const Bytef *)param->compbuf, len);
Liang Li73a89122016-05-05 15:32:51 +08002509
Liang Li33d151f2016-05-05 15:32:58 +08002510 qemu_mutex_lock(&decomp_done_lock);
2511 param->done = true;
2512 qemu_cond_signal(&decomp_done_cond);
2513 qemu_mutex_unlock(&decomp_done_lock);
2514
2515 qemu_mutex_lock(&param->mutex);
2516 } else {
2517 qemu_cond_wait(&param->cond, &param->mutex);
2518 }
Juan Quintela56e93d22015-05-07 19:33:31 +02002519 }
Liang Li33d151f2016-05-05 15:32:58 +08002520 qemu_mutex_unlock(&param->mutex);
Juan Quintela56e93d22015-05-07 19:33:31 +02002521
2522 return NULL;
2523}
2524
Liang Li5533b2e2016-05-05 15:32:52 +08002525static void wait_for_decompress_done(void)
2526{
2527 int idx, thread_count;
2528
2529 if (!migrate_use_compression()) {
2530 return;
2531 }
2532
2533 thread_count = migrate_decompress_threads();
2534 qemu_mutex_lock(&decomp_done_lock);
2535 for (idx = 0; idx < thread_count; idx++) {
2536 while (!decomp_param[idx].done) {
2537 qemu_cond_wait(&decomp_done_cond, &decomp_done_lock);
2538 }
2539 }
2540 qemu_mutex_unlock(&decomp_done_lock);
2541}
2542
Juan Quintelaf0afa332017-06-28 11:52:28 +02002543static void compress_threads_load_setup(void)
Juan Quintela56e93d22015-05-07 19:33:31 +02002544{
2545 int i, thread_count;
2546
Juan Quintela3416ab52016-04-20 11:56:01 +02002547 if (!migrate_use_compression()) {
2548 return;
2549 }
Juan Quintela56e93d22015-05-07 19:33:31 +02002550 thread_count = migrate_decompress_threads();
2551 decompress_threads = g_new0(QemuThread, thread_count);
2552 decomp_param = g_new0(DecompressParam, thread_count);
Liang Li73a89122016-05-05 15:32:51 +08002553 qemu_mutex_init(&decomp_done_lock);
2554 qemu_cond_init(&decomp_done_cond);
Juan Quintela56e93d22015-05-07 19:33:31 +02002555 for (i = 0; i < thread_count; i++) {
2556 qemu_mutex_init(&decomp_param[i].mutex);
2557 qemu_cond_init(&decomp_param[i].cond);
2558 decomp_param[i].compbuf = g_malloc0(compressBound(TARGET_PAGE_SIZE));
Liang Li73a89122016-05-05 15:32:51 +08002559 decomp_param[i].done = true;
Liang Li90e56fb2016-05-05 15:32:56 +08002560 decomp_param[i].quit = false;
Juan Quintela56e93d22015-05-07 19:33:31 +02002561 qemu_thread_create(decompress_threads + i, "decompress",
2562 do_data_decompress, decomp_param + i,
2563 QEMU_THREAD_JOINABLE);
2564 }
2565}
2566
Juan Quintelaf0afa332017-06-28 11:52:28 +02002567static void compress_threads_load_cleanup(void)
Juan Quintela56e93d22015-05-07 19:33:31 +02002568{
2569 int i, thread_count;
2570
Juan Quintela3416ab52016-04-20 11:56:01 +02002571 if (!migrate_use_compression()) {
2572 return;
2573 }
Juan Quintela56e93d22015-05-07 19:33:31 +02002574 thread_count = migrate_decompress_threads();
2575 for (i = 0; i < thread_count; i++) {
2576 qemu_mutex_lock(&decomp_param[i].mutex);
Liang Li90e56fb2016-05-05 15:32:56 +08002577 decomp_param[i].quit = true;
Juan Quintela56e93d22015-05-07 19:33:31 +02002578 qemu_cond_signal(&decomp_param[i].cond);
2579 qemu_mutex_unlock(&decomp_param[i].mutex);
2580 }
2581 for (i = 0; i < thread_count; i++) {
2582 qemu_thread_join(decompress_threads + i);
2583 qemu_mutex_destroy(&decomp_param[i].mutex);
2584 qemu_cond_destroy(&decomp_param[i].cond);
2585 g_free(decomp_param[i].compbuf);
2586 }
2587 g_free(decompress_threads);
2588 g_free(decomp_param);
Juan Quintela56e93d22015-05-07 19:33:31 +02002589 decompress_threads = NULL;
2590 decomp_param = NULL;
Juan Quintela56e93d22015-05-07 19:33:31 +02002591}
2592
Dr. David Alan Gilbertc1bc6622015-12-16 11:47:38 +00002593static void decompress_data_with_multi_threads(QEMUFile *f,
Juan Quintela56e93d22015-05-07 19:33:31 +02002594 void *host, int len)
2595{
2596 int idx, thread_count;
2597
2598 thread_count = migrate_decompress_threads();
Liang Li73a89122016-05-05 15:32:51 +08002599 qemu_mutex_lock(&decomp_done_lock);
Juan Quintela56e93d22015-05-07 19:33:31 +02002600 while (true) {
2601 for (idx = 0; idx < thread_count; idx++) {
Liang Li73a89122016-05-05 15:32:51 +08002602 if (decomp_param[idx].done) {
Liang Li33d151f2016-05-05 15:32:58 +08002603 decomp_param[idx].done = false;
2604 qemu_mutex_lock(&decomp_param[idx].mutex);
Dr. David Alan Gilbertc1bc6622015-12-16 11:47:38 +00002605 qemu_get_buffer(f, decomp_param[idx].compbuf, len);
Juan Quintela56e93d22015-05-07 19:33:31 +02002606 decomp_param[idx].des = host;
2607 decomp_param[idx].len = len;
Liang Li33d151f2016-05-05 15:32:58 +08002608 qemu_cond_signal(&decomp_param[idx].cond);
2609 qemu_mutex_unlock(&decomp_param[idx].mutex);
Juan Quintela56e93d22015-05-07 19:33:31 +02002610 break;
2611 }
2612 }
2613 if (idx < thread_count) {
2614 break;
Liang Li73a89122016-05-05 15:32:51 +08002615 } else {
2616 qemu_cond_wait(&decomp_done_cond, &decomp_done_lock);
Juan Quintela56e93d22015-05-07 19:33:31 +02002617 }
2618 }
Liang Li73a89122016-05-05 15:32:51 +08002619 qemu_mutex_unlock(&decomp_done_lock);
Juan Quintela56e93d22015-05-07 19:33:31 +02002620}
2621
Juan Quintela3d0684b2017-03-23 15:06:39 +01002622/**
Juan Quintelaf265e0e2017-06-28 11:52:27 +02002623 * ram_load_setup: Setup RAM for migration incoming side
2624 *
2625 * Returns zero to indicate success and negative for error
2626 *
2627 * @f: QEMUFile where to receive the data
2628 * @opaque: RAMState pointer
2629 */
2630static int ram_load_setup(QEMUFile *f, void *opaque)
2631{
2632 xbzrle_load_setup();
Juan Quintelaf0afa332017-06-28 11:52:28 +02002633 compress_threads_load_setup();
Alexey Perevalovf9494612017-10-05 14:13:20 +03002634 ramblock_recv_map_init();
Juan Quintelaf265e0e2017-06-28 11:52:27 +02002635 return 0;
2636}
2637
2638static int ram_load_cleanup(void *opaque)
2639{
Alexey Perevalovf9494612017-10-05 14:13:20 +03002640 RAMBlock *rb;
Juan Quintelaf265e0e2017-06-28 11:52:27 +02002641 xbzrle_load_cleanup();
Juan Quintelaf0afa332017-06-28 11:52:28 +02002642 compress_threads_load_cleanup();
Alexey Perevalovf9494612017-10-05 14:13:20 +03002643
2644 RAMBLOCK_FOREACH(rb) {
2645 g_free(rb->receivedmap);
2646 rb->receivedmap = NULL;
2647 }
Juan Quintelaf265e0e2017-06-28 11:52:27 +02002648 return 0;
2649}
2650
2651/**
Juan Quintela3d0684b2017-03-23 15:06:39 +01002652 * ram_postcopy_incoming_init: allocate postcopy data structures
2653 *
2654 * Returns 0 for success and negative if there was one error
2655 *
2656 * @mis: current migration incoming state
2657 *
2658 * Allocate data structures etc needed by incoming migration with
2659 * postcopy-ram. postcopy-ram's similarly names
2660 * postcopy_ram_incoming_init does the work.
Dr. David Alan Gilbert1caddf82015-11-05 18:11:03 +00002661 */
2662int ram_postcopy_incoming_init(MigrationIncomingState *mis)
2663{
Juan Quintelab8c48992017-03-21 17:44:30 +01002664 unsigned long ram_pages = last_ram_page();
Dr. David Alan Gilbert1caddf82015-11-05 18:11:03 +00002665
2666 return postcopy_ram_incoming_init(mis, ram_pages);
2667}
2668
Juan Quintela3d0684b2017-03-23 15:06:39 +01002669/**
2670 * ram_load_postcopy: load a page in postcopy case
2671 *
2672 * Returns 0 for success or -errno in case of error
2673 *
Dr. David Alan Gilberta7180872015-11-05 18:11:11 +00002674 * Called in postcopy mode by ram_load().
2675 * rcu_read_lock is taken prior to this being called.
Juan Quintela3d0684b2017-03-23 15:06:39 +01002676 *
2677 * @f: QEMUFile where to send the data
Dr. David Alan Gilberta7180872015-11-05 18:11:11 +00002678 */
2679static int ram_load_postcopy(QEMUFile *f)
2680{
2681 int flags = 0, ret = 0;
2682 bool place_needed = false;
Dr. David Alan Gilbert28abd202017-02-24 18:28:37 +00002683 bool matching_page_sizes = false;
Dr. David Alan Gilberta7180872015-11-05 18:11:11 +00002684 MigrationIncomingState *mis = migration_incoming_get_current();
2685 /* Temporary page that is later 'placed' */
2686 void *postcopy_host_page = postcopy_get_tmp_page(mis);
Dr. David Alan Gilbertc53b7dd2015-11-05 18:11:12 +00002687 void *last_host = NULL;
Dr. David Alan Gilberta3b6ff62015-11-11 14:02:28 +00002688 bool all_zero = false;
Dr. David Alan Gilberta7180872015-11-05 18:11:11 +00002689
2690 while (!ret && !(flags & RAM_SAVE_FLAG_EOS)) {
2691 ram_addr_t addr;
2692 void *host = NULL;
2693 void *page_buffer = NULL;
2694 void *place_source = NULL;
Dr. David Alan Gilbertdf9ff5e2017-02-24 18:28:35 +00002695 RAMBlock *block = NULL;
Dr. David Alan Gilberta7180872015-11-05 18:11:11 +00002696 uint8_t ch;
Dr. David Alan Gilberta7180872015-11-05 18:11:11 +00002697
2698 addr = qemu_get_be64(f);
2699 flags = addr & ~TARGET_PAGE_MASK;
2700 addr &= TARGET_PAGE_MASK;
2701
2702 trace_ram_load_postcopy_loop((uint64_t)addr, flags);
2703 place_needed = false;
Juan Quintelabb890ed2017-04-28 09:39:55 +02002704 if (flags & (RAM_SAVE_FLAG_ZERO | RAM_SAVE_FLAG_PAGE)) {
Dr. David Alan Gilbertdf9ff5e2017-02-24 18:28:35 +00002705 block = ram_block_from_stream(f, flags);
zhanghailiang4c4bad42016-01-15 11:37:41 +08002706
2707 host = host_from_ram_block_offset(block, addr);
Dr. David Alan Gilberta7180872015-11-05 18:11:11 +00002708 if (!host) {
2709 error_report("Illegal RAM offset " RAM_ADDR_FMT, addr);
2710 ret = -EINVAL;
2711 break;
2712 }
Dr. David Alan Gilbert28abd202017-02-24 18:28:37 +00002713 matching_page_sizes = block->page_size == TARGET_PAGE_SIZE;
Dr. David Alan Gilberta7180872015-11-05 18:11:11 +00002714 /*
Dr. David Alan Gilbert28abd202017-02-24 18:28:37 +00002715 * Postcopy requires that we place whole host pages atomically;
2716 * these may be huge pages for RAMBlocks that are backed by
2717 * hugetlbfs.
Dr. David Alan Gilberta7180872015-11-05 18:11:11 +00002718 * To make it atomic, the data is read into a temporary page
2719 * that's moved into place later.
2720 * The migration protocol uses, possibly smaller, target-pages
2721 * however the source ensures it always sends all the components
2722 * of a host page in order.
2723 */
2724 page_buffer = postcopy_host_page +
Dr. David Alan Gilbert28abd202017-02-24 18:28:37 +00002725 ((uintptr_t)host & (block->page_size - 1));
Dr. David Alan Gilberta7180872015-11-05 18:11:11 +00002726 /* If all TP are zero then we can optimise the place */
Dr. David Alan Gilbert28abd202017-02-24 18:28:37 +00002727 if (!((uintptr_t)host & (block->page_size - 1))) {
Dr. David Alan Gilberta7180872015-11-05 18:11:11 +00002728 all_zero = true;
Dr. David Alan Gilbertc53b7dd2015-11-05 18:11:12 +00002729 } else {
2730 /* not the 1st TP within the HP */
2731 if (host != (last_host + TARGET_PAGE_SIZE)) {
Markus Armbruster9af9e0f2015-12-18 16:35:19 +01002732 error_report("Non-sequential target page %p/%p",
Dr. David Alan Gilbertc53b7dd2015-11-05 18:11:12 +00002733 host, last_host);
2734 ret = -EINVAL;
2735 break;
2736 }
Dr. David Alan Gilberta7180872015-11-05 18:11:11 +00002737 }
2738
Dr. David Alan Gilbertc53b7dd2015-11-05 18:11:12 +00002739
Dr. David Alan Gilberta7180872015-11-05 18:11:11 +00002740 /*
2741 * If it's the last part of a host page then we place the host
2742 * page
2743 */
2744 place_needed = (((uintptr_t)host + TARGET_PAGE_SIZE) &
Dr. David Alan Gilbert28abd202017-02-24 18:28:37 +00002745 (block->page_size - 1)) == 0;
Dr. David Alan Gilberta7180872015-11-05 18:11:11 +00002746 place_source = postcopy_host_page;
2747 }
Dr. David Alan Gilbertc53b7dd2015-11-05 18:11:12 +00002748 last_host = host;
Dr. David Alan Gilberta7180872015-11-05 18:11:11 +00002749
2750 switch (flags & ~RAM_SAVE_FLAG_CONTINUE) {
Juan Quintelabb890ed2017-04-28 09:39:55 +02002751 case RAM_SAVE_FLAG_ZERO:
Dr. David Alan Gilberta7180872015-11-05 18:11:11 +00002752 ch = qemu_get_byte(f);
2753 memset(page_buffer, ch, TARGET_PAGE_SIZE);
2754 if (ch) {
2755 all_zero = false;
2756 }
2757 break;
2758
2759 case RAM_SAVE_FLAG_PAGE:
2760 all_zero = false;
2761 if (!place_needed || !matching_page_sizes) {
2762 qemu_get_buffer(f, page_buffer, TARGET_PAGE_SIZE);
2763 } else {
2764 /* Avoids the qemu_file copy during postcopy, which is
2765 * going to do a copy later; can only do it when we
2766 * do this read in one go (matching page sizes)
2767 */
2768 qemu_get_buffer_in_place(f, (uint8_t **)&place_source,
2769 TARGET_PAGE_SIZE);
2770 }
2771 break;
2772 case RAM_SAVE_FLAG_EOS:
2773 /* normal exit */
2774 break;
2775 default:
2776 error_report("Unknown combination of migration flags: %#x"
2777 " (postcopy mode)", flags);
2778 ret = -EINVAL;
2779 }
2780
2781 if (place_needed) {
2782 /* This gets called at the last target page in the host page */
Dr. David Alan Gilbertdf9ff5e2017-02-24 18:28:35 +00002783 void *place_dest = host + TARGET_PAGE_SIZE - block->page_size;
2784
Dr. David Alan Gilberta7180872015-11-05 18:11:11 +00002785 if (all_zero) {
Dr. David Alan Gilbertdf9ff5e2017-02-24 18:28:35 +00002786 ret = postcopy_place_page_zero(mis, place_dest,
Alexey Perevalov8be46202017-10-05 14:13:18 +03002787 block);
Dr. David Alan Gilberta7180872015-11-05 18:11:11 +00002788 } else {
Dr. David Alan Gilbertdf9ff5e2017-02-24 18:28:35 +00002789 ret = postcopy_place_page(mis, place_dest,
Alexey Perevalov8be46202017-10-05 14:13:18 +03002790 place_source, block);
Dr. David Alan Gilberta7180872015-11-05 18:11:11 +00002791 }
2792 }
2793 if (!ret) {
2794 ret = qemu_file_get_error(f);
2795 }
2796 }
2797
2798 return ret;
2799}
2800
Daniel Henrique Barbozaacab30b2017-11-16 20:35:26 -02002801static bool postcopy_is_advised(void)
2802{
2803 PostcopyState ps = postcopy_state_get();
2804 return ps >= POSTCOPY_INCOMING_ADVISE && ps < POSTCOPY_INCOMING_END;
2805}
2806
2807static bool postcopy_is_running(void)
2808{
2809 PostcopyState ps = postcopy_state_get();
2810 return ps >= POSTCOPY_INCOMING_LISTENING && ps < POSTCOPY_INCOMING_END;
2811}
2812
Juan Quintela56e93d22015-05-07 19:33:31 +02002813static int ram_load(QEMUFile *f, void *opaque, int version_id)
2814{
Juan Quintelaedc60122016-11-02 12:40:46 +01002815 int flags = 0, ret = 0, invalid_flags = 0;
Juan Quintela56e93d22015-05-07 19:33:31 +02002816 static uint64_t seq_iter;
2817 int len = 0;
Dr. David Alan Gilberta7180872015-11-05 18:11:11 +00002818 /*
2819 * If system is running in postcopy mode, page inserts to host memory must
2820 * be atomic
2821 */
Daniel Henrique Barbozaacab30b2017-11-16 20:35:26 -02002822 bool postcopy_running = postcopy_is_running();
Dr. David Alan Gilbertef08fb32017-02-24 18:28:30 +00002823 /* ADVISE is earlier, it shows the source has the postcopy capability on */
Daniel Henrique Barbozaacab30b2017-11-16 20:35:26 -02002824 bool postcopy_advised = postcopy_is_advised();
Juan Quintela56e93d22015-05-07 19:33:31 +02002825
2826 seq_iter++;
2827
2828 if (version_id != 4) {
2829 ret = -EINVAL;
2830 }
2831
Juan Quintelaedc60122016-11-02 12:40:46 +01002832 if (!migrate_use_compression()) {
2833 invalid_flags |= RAM_SAVE_FLAG_COMPRESS_PAGE;
2834 }
Juan Quintela56e93d22015-05-07 19:33:31 +02002835 /* This RCU critical section can be very long running.
2836 * When RCU reclaims in the code start to become numerous,
2837 * it will be necessary to reduce the granularity of this
2838 * critical section.
2839 */
2840 rcu_read_lock();
Dr. David Alan Gilberta7180872015-11-05 18:11:11 +00002841
2842 if (postcopy_running) {
2843 ret = ram_load_postcopy(f);
2844 }
2845
2846 while (!postcopy_running && !ret && !(flags & RAM_SAVE_FLAG_EOS)) {
Juan Quintela56e93d22015-05-07 19:33:31 +02002847 ram_addr_t addr, total_ram_bytes;
Dr. David Alan Gilberta776aa12015-11-05 18:10:39 +00002848 void *host = NULL;
Juan Quintela56e93d22015-05-07 19:33:31 +02002849 uint8_t ch;
2850
2851 addr = qemu_get_be64(f);
2852 flags = addr & ~TARGET_PAGE_MASK;
2853 addr &= TARGET_PAGE_MASK;
2854
Juan Quintelaedc60122016-11-02 12:40:46 +01002855 if (flags & invalid_flags) {
2856 if (flags & invalid_flags & RAM_SAVE_FLAG_COMPRESS_PAGE) {
2857 error_report("Received an unexpected compressed page");
2858 }
2859
2860 ret = -EINVAL;
2861 break;
2862 }
2863
Juan Quintelabb890ed2017-04-28 09:39:55 +02002864 if (flags & (RAM_SAVE_FLAG_ZERO | RAM_SAVE_FLAG_PAGE |
Dr. David Alan Gilberta776aa12015-11-05 18:10:39 +00002865 RAM_SAVE_FLAG_COMPRESS_PAGE | RAM_SAVE_FLAG_XBZRLE)) {
zhanghailiang4c4bad42016-01-15 11:37:41 +08002866 RAMBlock *block = ram_block_from_stream(f, flags);
2867
2868 host = host_from_ram_block_offset(block, addr);
Dr. David Alan Gilberta776aa12015-11-05 18:10:39 +00002869 if (!host) {
2870 error_report("Illegal RAM offset " RAM_ADDR_FMT, addr);
2871 ret = -EINVAL;
2872 break;
2873 }
Alexey Perevalovf9494612017-10-05 14:13:20 +03002874 ramblock_recv_bitmap_set(block, host);
Dr. David Alan Gilbert1db9d8e2017-04-26 19:37:21 +01002875 trace_ram_load_loop(block->idstr, (uint64_t)addr, flags, host);
Dr. David Alan Gilberta776aa12015-11-05 18:10:39 +00002876 }
2877
Juan Quintela56e93d22015-05-07 19:33:31 +02002878 switch (flags & ~RAM_SAVE_FLAG_CONTINUE) {
2879 case RAM_SAVE_FLAG_MEM_SIZE:
2880 /* Synchronize RAM block list */
2881 total_ram_bytes = addr;
2882 while (!ret && total_ram_bytes) {
2883 RAMBlock *block;
Juan Quintela56e93d22015-05-07 19:33:31 +02002884 char id[256];
2885 ram_addr_t length;
2886
2887 len = qemu_get_byte(f);
2888 qemu_get_buffer(f, (uint8_t *)id, len);
2889 id[len] = 0;
2890 length = qemu_get_be64(f);
2891
Dr. David Alan Gilberte3dd7492015-11-05 18:10:33 +00002892 block = qemu_ram_block_by_name(id);
2893 if (block) {
2894 if (length != block->used_length) {
2895 Error *local_err = NULL;
Juan Quintela56e93d22015-05-07 19:33:31 +02002896
Gongleifa53a0e2016-05-10 10:04:59 +08002897 ret = qemu_ram_resize(block, length,
Dr. David Alan Gilberte3dd7492015-11-05 18:10:33 +00002898 &local_err);
2899 if (local_err) {
2900 error_report_err(local_err);
Juan Quintela56e93d22015-05-07 19:33:31 +02002901 }
Juan Quintela56e93d22015-05-07 19:33:31 +02002902 }
Dr. David Alan Gilbertef08fb32017-02-24 18:28:30 +00002903 /* For postcopy we need to check hugepage sizes match */
2904 if (postcopy_advised &&
2905 block->page_size != qemu_host_page_size) {
2906 uint64_t remote_page_size = qemu_get_be64(f);
2907 if (remote_page_size != block->page_size) {
2908 error_report("Mismatched RAM page size %s "
2909 "(local) %zd != %" PRId64,
2910 id, block->page_size,
2911 remote_page_size);
2912 ret = -EINVAL;
2913 }
2914 }
Dr. David Alan Gilberte3dd7492015-11-05 18:10:33 +00002915 ram_control_load_hook(f, RAM_CONTROL_BLOCK_REG,
2916 block->idstr);
2917 } else {
Juan Quintela56e93d22015-05-07 19:33:31 +02002918 error_report("Unknown ramblock \"%s\", cannot "
2919 "accept migration", id);
2920 ret = -EINVAL;
2921 }
2922
2923 total_ram_bytes -= length;
2924 }
2925 break;
Dr. David Alan Gilberta776aa12015-11-05 18:10:39 +00002926
Juan Quintelabb890ed2017-04-28 09:39:55 +02002927 case RAM_SAVE_FLAG_ZERO:
Juan Quintela56e93d22015-05-07 19:33:31 +02002928 ch = qemu_get_byte(f);
2929 ram_handle_compressed(host, ch, TARGET_PAGE_SIZE);
2930 break;
Dr. David Alan Gilberta776aa12015-11-05 18:10:39 +00002931
Juan Quintela56e93d22015-05-07 19:33:31 +02002932 case RAM_SAVE_FLAG_PAGE:
Juan Quintela56e93d22015-05-07 19:33:31 +02002933 qemu_get_buffer(f, host, TARGET_PAGE_SIZE);
2934 break;
Juan Quintela56e93d22015-05-07 19:33:31 +02002935
Dr. David Alan Gilberta776aa12015-11-05 18:10:39 +00002936 case RAM_SAVE_FLAG_COMPRESS_PAGE:
Juan Quintela56e93d22015-05-07 19:33:31 +02002937 len = qemu_get_be32(f);
2938 if (len < 0 || len > compressBound(TARGET_PAGE_SIZE)) {
2939 error_report("Invalid compressed data length: %d", len);
2940 ret = -EINVAL;
2941 break;
2942 }
Dr. David Alan Gilbertc1bc6622015-12-16 11:47:38 +00002943 decompress_data_with_multi_threads(f, host, len);
Juan Quintela56e93d22015-05-07 19:33:31 +02002944 break;
Dr. David Alan Gilberta776aa12015-11-05 18:10:39 +00002945
Juan Quintela56e93d22015-05-07 19:33:31 +02002946 case RAM_SAVE_FLAG_XBZRLE:
Juan Quintela56e93d22015-05-07 19:33:31 +02002947 if (load_xbzrle(f, addr, host) < 0) {
2948 error_report("Failed to decompress XBZRLE page at "
2949 RAM_ADDR_FMT, addr);
2950 ret = -EINVAL;
2951 break;
2952 }
2953 break;
2954 case RAM_SAVE_FLAG_EOS:
2955 /* normal exit */
2956 break;
2957 default:
2958 if (flags & RAM_SAVE_FLAG_HOOK) {
Dr. David Alan Gilbert632e3a52015-06-11 18:17:23 +01002959 ram_control_load_hook(f, RAM_CONTROL_HOOK, NULL);
Juan Quintela56e93d22015-05-07 19:33:31 +02002960 } else {
2961 error_report("Unknown combination of migration flags: %#x",
2962 flags);
2963 ret = -EINVAL;
2964 }
2965 }
2966 if (!ret) {
2967 ret = qemu_file_get_error(f);
2968 }
2969 }
2970
Liang Li5533b2e2016-05-05 15:32:52 +08002971 wait_for_decompress_done();
Juan Quintela56e93d22015-05-07 19:33:31 +02002972 rcu_read_unlock();
Juan Quintela55c44462017-01-23 22:32:05 +01002973 trace_ram_load_complete(ret, seq_iter);
Juan Quintela56e93d22015-05-07 19:33:31 +02002974 return ret;
2975}
2976
Vladimir Sementsov-Ogievskiyc6467622017-07-10 19:30:14 +03002977static bool ram_has_postcopy(void *opaque)
2978{
2979 return migrate_postcopy_ram();
2980}
2981
Juan Quintela56e93d22015-05-07 19:33:31 +02002982static SaveVMHandlers savevm_ram_handlers = {
Juan Quintela9907e842017-06-28 11:52:24 +02002983 .save_setup = ram_save_setup,
Juan Quintela56e93d22015-05-07 19:33:31 +02002984 .save_live_iterate = ram_save_iterate,
Dr. David Alan Gilbert763c9062015-11-05 18:11:00 +00002985 .save_live_complete_postcopy = ram_save_complete,
Dr. David Alan Gilberta3e06c32015-11-05 18:10:41 +00002986 .save_live_complete_precopy = ram_save_complete,
Vladimir Sementsov-Ogievskiyc6467622017-07-10 19:30:14 +03002987 .has_postcopy = ram_has_postcopy,
Juan Quintela56e93d22015-05-07 19:33:31 +02002988 .save_live_pending = ram_save_pending,
2989 .load_state = ram_load,
Juan Quintelaf265e0e2017-06-28 11:52:27 +02002990 .save_cleanup = ram_save_cleanup,
2991 .load_setup = ram_load_setup,
2992 .load_cleanup = ram_load_cleanup,
Juan Quintela56e93d22015-05-07 19:33:31 +02002993};
2994
2995void ram_mig_init(void)
2996{
2997 qemu_mutex_init(&XBZRLE.lock);
Juan Quintela6f37bb82017-03-13 19:26:29 +01002998 register_savevm_live(NULL, "ram", 0, 4, &savevm_ram_handlers, &ram_state);
Juan Quintela56e93d22015-05-07 19:33:31 +02002999}