blob: 84f1b0fb20378763fa3219fb77fd0b283d292b75 [file] [log] [blame]
Chuan Zheng4240dce2020-09-16 14:21:56 +08001/*
2 * Dirtyrate implement code
3 *
4 * Copyright (c) 2020 HUAWEI TECHNOLOGIES CO.,LTD.
5 *
6 * Authors:
7 * Chuan Zheng <zhengchuan@huawei.com>
8 *
9 * This work is licensed under the terms of the GNU GPL, version 2 or later.
10 * See the COPYING file in the top-level directory.
11 */
12
13#include "qemu/osdep.h"
Richard Hendersoncc37d982023-03-15 17:43:13 +000014#include "qemu/error-report.h"
Marc-André Lureau662770a2020-10-08 20:59:53 +040015#include <zlib.h>
Philippe Mathieu-Daudé1e058882023-04-05 18:04:45 +020016#include "hw/core/cpu.h"
Chuan Zheng4240dce2020-09-16 14:21:56 +080017#include "qapi/error.h"
Chuan Zheng4240dce2020-09-16 14:21:56 +080018#include "exec/ramblock.h"
Juan Quintelabeeda9b2023-05-11 16:12:05 +020019#include "exec/target_page.h"
Chuan Zheng4240dce2020-09-16 14:21:56 +080020#include "qemu/rcu_queue.h"
Hyman Huang(黄勇)0e21bf22021-06-29 16:01:23 +000021#include "qemu/main-loop.h"
Chuan Zheng4240dce2020-09-16 14:21:56 +080022#include "qapi/qapi-commands-migration.h"
Chuan Zheng3ded54b2020-09-16 14:22:00 +080023#include "ram.h"
Chuan Zheng3c0b5df2020-09-16 14:22:07 +080024#include "trace.h"
Chuan Zheng4240dce2020-09-16 14:21:56 +080025#include "dirtyrate.h"
Peter Xua4a571d2021-06-07 09:11:57 +080026#include "monitor/hmp.h"
27#include "monitor/monitor.h"
28#include "qapi/qmp/qdict.h"
Hyman Huang(黄勇)0e21bf22021-06-29 16:01:23 +000029#include "sysemu/kvm.h"
30#include "sysemu/runstate.h"
31#include "exec/memory.h"
Andrei Gudkov00a3f9c2023-04-27 15:42:57 +030032#include "qemu/xxhash.h"
Hyman Huang(黄勇)0e21bf22021-06-29 16:01:23 +000033
Hyman Huang(黄勇)4998a372021-07-20 23:19:16 +080034/*
35 * total_dirty_pages is procted by BQL and is used
36 * to stat dirty pages during the period of two
37 * memory_global_dirty_log_sync
38 */
39uint64_t total_dirty_pages;
40
Hyman Huang(黄勇)0e21bf22021-06-29 16:01:23 +000041typedef struct DirtyPageRecord {
42 uint64_t start_pages;
43 uint64_t end_pages;
44} DirtyPageRecord;
Chuan Zheng4240dce2020-09-16 14:21:56 +080045
Chuan Zheng7df3aa32020-09-16 14:21:57 +080046static int CalculatingState = DIRTY_RATE_STATUS_UNSTARTED;
Chuan Zhengc9a58d72020-09-16 14:21:59 +080047static struct DirtyRateStat DirtyStat;
Hyman Huang(黄勇)0e21bf22021-06-29 16:01:23 +000048static DirtyRateMeasureMode dirtyrate_mode =
49 DIRTY_RATE_MEASURE_MODE_PAGE_SAMPLING;
Chuan Zheng7df3aa32020-09-16 14:21:57 +080050
Hyman Huang(黄勇)82441662022-06-26 01:38:32 +080051static int64_t dirty_stat_wait(int64_t msec, int64_t initial_time)
Chuan Zhengeca58222020-09-16 14:22:04 +080052{
53 int64_t current_time;
54
55 current_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
56 if ((current_time - initial_time) >= msec) {
57 msec = current_time - initial_time;
58 } else {
59 g_usleep((msec + initial_time - current_time) * 1000);
60 }
61
62 return msec;
63}
64
Hyman Huang(黄勇)82441662022-06-26 01:38:32 +080065static inline void record_dirtypages(DirtyPageRecord *dirty_pages,
66 CPUState *cpu, bool start)
67{
68 if (start) {
69 dirty_pages[cpu->cpu_index].start_pages = cpu->dirty_pages;
70 } else {
71 dirty_pages[cpu->cpu_index].end_pages = cpu->dirty_pages;
72 }
73}
74
75static int64_t do_calculate_dirtyrate(DirtyPageRecord dirty_pages,
76 int64_t calc_time_ms)
77{
Hyman Huang(黄勇)82441662022-06-26 01:38:32 +080078 uint64_t increased_dirty_pages =
79 dirty_pages.end_pages - dirty_pages.start_pages;
Juan Quintelabeeda9b2023-05-11 16:12:05 +020080 uint64_t memory_size_MiB = qemu_target_pages_to_MiB(increased_dirty_pages);
Hyman Huang(黄勇)82441662022-06-26 01:38:32 +080081
Juan Quintelabeeda9b2023-05-11 16:12:05 +020082 return memory_size_MiB * 1000 / calc_time_ms;
Hyman Huang(黄勇)82441662022-06-26 01:38:32 +080083}
84
85void global_dirty_log_change(unsigned int flag, bool start)
86{
87 qemu_mutex_lock_iothread();
88 if (start) {
89 memory_global_dirty_log_start(flag);
90 } else {
91 memory_global_dirty_log_stop(flag);
92 }
93 qemu_mutex_unlock_iothread();
94}
95
96/*
97 * global_dirty_log_sync
98 * 1. sync dirty log from kvm
99 * 2. stop dirty tracking if needed.
100 */
101static void global_dirty_log_sync(unsigned int flag, bool one_shot)
102{
103 qemu_mutex_lock_iothread();
Gavin Shan1e493be2023-05-09 12:21:19 +1000104 memory_global_dirty_log_sync(false);
Hyman Huang(黄勇)82441662022-06-26 01:38:32 +0800105 if (one_shot) {
106 memory_global_dirty_log_stop(flag);
107 }
108 qemu_mutex_unlock_iothread();
109}
110
111static DirtyPageRecord *vcpu_dirty_stat_alloc(VcpuStat *stat)
112{
113 CPUState *cpu;
Hyman Huang(黄勇)82441662022-06-26 01:38:32 +0800114 int nvcpu = 0;
115
116 CPU_FOREACH(cpu) {
117 nvcpu++;
118 }
119
120 stat->nvcpu = nvcpu;
Markus Armbrusterc5e8d512022-09-23 10:42:54 +0200121 stat->rates = g_new0(DirtyRateVcpu, nvcpu);
Hyman Huang(黄勇)82441662022-06-26 01:38:32 +0800122
Markus Armbruster66997c42022-11-22 14:49:16 +0100123 return g_new0(DirtyPageRecord, nvcpu);
Hyman Huang(黄勇)82441662022-06-26 01:38:32 +0800124}
125
126static void vcpu_dirty_stat_collect(VcpuStat *stat,
127 DirtyPageRecord *records,
128 bool start)
129{
130 CPUState *cpu;
131
132 CPU_FOREACH(cpu) {
133 record_dirtypages(records, cpu, start);
134 }
135}
136
137int64_t vcpu_calculate_dirtyrate(int64_t calc_time_ms,
138 VcpuStat *stat,
139 unsigned int flag,
140 bool one_shot)
141{
142 DirtyPageRecord *records;
143 int64_t init_time_ms;
144 int64_t duration;
145 int64_t dirtyrate;
146 int i = 0;
147 unsigned int gen_id;
148
149retry:
150 init_time_ms = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
151
Jamie Iles370ed602023-04-27 03:09:24 +0100152 WITH_QEMU_LOCK_GUARD(&qemu_cpu_list_lock) {
153 gen_id = cpu_list_generation_id_get();
154 records = vcpu_dirty_stat_alloc(stat);
155 vcpu_dirty_stat_collect(stat, records, true);
156 }
Hyman Huang(黄勇)82441662022-06-26 01:38:32 +0800157
158 duration = dirty_stat_wait(calc_time_ms, init_time_ms);
159
160 global_dirty_log_sync(flag, one_shot);
161
Jamie Iles370ed602023-04-27 03:09:24 +0100162 WITH_QEMU_LOCK_GUARD(&qemu_cpu_list_lock) {
163 if (gen_id != cpu_list_generation_id_get()) {
164 g_free(records);
165 g_free(stat->rates);
166 cpu_list_unlock();
167 goto retry;
168 }
169 vcpu_dirty_stat_collect(stat, records, false);
Hyman Huang(黄勇)82441662022-06-26 01:38:32 +0800170 }
Hyman Huang(黄勇)82441662022-06-26 01:38:32 +0800171
172 for (i = 0; i < stat->nvcpu; i++) {
173 dirtyrate = do_calculate_dirtyrate(records[i], duration);
174
175 stat->rates[i].id = i;
176 stat->rates[i].dirty_rate = dirtyrate;
177
178 trace_dirtyrate_do_calculate_vcpu(i, dirtyrate);
179 }
180
181 g_free(records);
182
183 return duration;
184}
185
Chuan Zhengeca58222020-09-16 14:22:04 +0800186static bool is_sample_period_valid(int64_t sec)
187{
188 if (sec < MIN_FETCH_DIRTYRATE_TIME_SEC ||
189 sec > MAX_FETCH_DIRTYRATE_TIME_SEC) {
190 return false;
191 }
192
193 return true;
194}
195
Hyman Huang(黄勇)7afa08c2021-06-07 09:11:34 +0800196static bool is_sample_pages_valid(int64_t pages)
197{
198 return pages >= MIN_SAMPLE_PAGE_COUNT &&
199 pages <= MAX_SAMPLE_PAGE_COUNT;
200}
201
Chuan Zheng7df3aa32020-09-16 14:21:57 +0800202static int dirtyrate_set_state(int *state, int old_state, int new_state)
203{
204 assert(new_state < DIRTY_RATE_STATUS__MAX);
Chuan Zheng3c0b5df2020-09-16 14:22:07 +0800205 trace_dirtyrate_set_state(DirtyRateStatus_str(new_state));
Chuan Zheng7df3aa32020-09-16 14:21:57 +0800206 if (qatomic_cmpxchg(state, old_state, new_state) == old_state) {
207 return 0;
208 } else {
209 return -1;
210 }
211}
212
Chuan Zheng4c437252020-09-16 14:22:06 +0800213static struct DirtyRateInfo *query_dirty_rate_info(void)
214{
Hyman Huang(黄勇)0e21bf22021-06-29 16:01:23 +0000215 int i;
Chuan Zheng4c437252020-09-16 14:22:06 +0800216 int64_t dirty_rate = DirtyStat.dirty_rate;
Markus Armbrusterb21e2382022-03-15 15:41:56 +0100217 struct DirtyRateInfo *info = g_new0(DirtyRateInfo, 1);
Hyman Huang(黄勇)0e21bf22021-06-29 16:01:23 +0000218 DirtyRateVcpuList *head = NULL, **tail = &head;
Chuan Zheng4c437252020-09-16 14:22:06 +0800219
220 info->status = CalculatingState;
221 info->start_time = DirtyStat.start_time;
222 info->calc_time = DirtyStat.calc_time;
Hyman Huang(黄勇)7afa08c2021-06-07 09:11:34 +0800223 info->sample_pages = DirtyStat.sample_pages;
Hyman Huang(黄勇)0e21bf22021-06-29 16:01:23 +0000224 info->mode = dirtyrate_mode;
225
226 if (qatomic_read(&CalculatingState) == DIRTY_RATE_STATUS_MEASURED) {
227 info->has_dirty_rate = true;
228 info->dirty_rate = dirty_rate;
229
230 if (dirtyrate_mode == DIRTY_RATE_MEASURE_MODE_DIRTY_RING) {
231 /*
232 * set sample_pages with 0 to indicate page sampling
233 * isn't enabled
234 **/
235 info->sample_pages = 0;
236 info->has_vcpu_dirty_rate = true;
237 for (i = 0; i < DirtyStat.dirty_ring.nvcpu; i++) {
Markus Armbrusterb21e2382022-03-15 15:41:56 +0100238 DirtyRateVcpu *rate = g_new0(DirtyRateVcpu, 1);
Hyman Huang(黄勇)0e21bf22021-06-29 16:01:23 +0000239 rate->id = DirtyStat.dirty_ring.rates[i].id;
240 rate->dirty_rate = DirtyStat.dirty_ring.rates[i].dirty_rate;
241 QAPI_LIST_APPEND(tail, rate);
242 }
243 info->vcpu_dirty_rate = head;
244 }
Hyman Huang(黄勇)826b8bc2021-07-20 23:19:17 +0800245
246 if (dirtyrate_mode == DIRTY_RATE_MEASURE_MODE_DIRTY_BITMAP) {
247 info->sample_pages = 0;
248 }
Hyman Huang(黄勇)0e21bf22021-06-29 16:01:23 +0000249 }
Chuan Zheng4c437252020-09-16 14:22:06 +0800250
Chuan Zheng3c0b5df2020-09-16 14:22:07 +0800251 trace_query_dirty_rate_info(DirtyRateStatus_str(CalculatingState));
252
Chuan Zheng4c437252020-09-16 14:22:06 +0800253 return info;
254}
255
Hyman Huang(黄勇)71864ea2021-06-29 16:01:20 +0000256static void init_dirtyrate_stat(int64_t start_time,
257 struct DirtyRateConfig config)
Chuan Zhengc9a58d72020-09-16 14:21:59 +0800258{
Chuan Zhengc9a58d72020-09-16 14:21:59 +0800259 DirtyStat.dirty_rate = -1;
Chuan Zhengaa84b502020-09-29 11:42:17 +0800260 DirtyStat.start_time = start_time;
Hyman Huang(黄勇)71864ea2021-06-29 16:01:20 +0000261 DirtyStat.calc_time = config.sample_period_seconds;
262 DirtyStat.sample_pages = config.sample_pages_per_gigabytes;
263
264 switch (config.mode) {
265 case DIRTY_RATE_MEASURE_MODE_PAGE_SAMPLING:
266 DirtyStat.page_sampling.total_dirty_samples = 0;
267 DirtyStat.page_sampling.total_sample_count = 0;
268 DirtyStat.page_sampling.total_block_mem_MB = 0;
269 break;
270 case DIRTY_RATE_MEASURE_MODE_DIRTY_RING:
271 DirtyStat.dirty_ring.nvcpu = -1;
272 DirtyStat.dirty_ring.rates = NULL;
273 break;
274 default:
275 break;
276 }
Chuan Zhengc9a58d72020-09-16 14:21:59 +0800277}
278
Hyman Huang(黄勇)0e21bf22021-06-29 16:01:23 +0000279static void cleanup_dirtyrate_stat(struct DirtyRateConfig config)
280{
281 /* last calc-dirty-rate qmp use dirty ring mode */
282 if (dirtyrate_mode == DIRTY_RATE_MEASURE_MODE_DIRTY_RING) {
283 free(DirtyStat.dirty_ring.rates);
284 DirtyStat.dirty_ring.rates = NULL;
285 }
286}
287
Chuan Zhengc9a58d72020-09-16 14:21:59 +0800288static void update_dirtyrate_stat(struct RamblockDirtyInfo *info)
289{
Hyman Huang(黄勇)71864ea2021-06-29 16:01:20 +0000290 DirtyStat.page_sampling.total_dirty_samples += info->sample_dirty_count;
291 DirtyStat.page_sampling.total_sample_count += info->sample_pages_count;
Chuan Zhengc9a58d72020-09-16 14:21:59 +0800292 /* size of total pages in MB */
Juan Quintelabeeda9b2023-05-11 16:12:05 +0200293 DirtyStat.page_sampling.total_block_mem_MB +=
294 qemu_target_pages_to_MiB(info->ramblock_pages);
Chuan Zhengc9a58d72020-09-16 14:21:59 +0800295}
296
297static void update_dirtyrate(uint64_t msec)
298{
299 uint64_t dirtyrate;
Hyman Huang(黄勇)71864ea2021-06-29 16:01:20 +0000300 uint64_t total_dirty_samples = DirtyStat.page_sampling.total_dirty_samples;
301 uint64_t total_sample_count = DirtyStat.page_sampling.total_sample_count;
302 uint64_t total_block_mem_MB = DirtyStat.page_sampling.total_block_mem_MB;
Chuan Zhengc9a58d72020-09-16 14:21:59 +0800303
304 dirtyrate = total_dirty_samples * total_block_mem_MB *
305 1000 / (total_sample_count * msec);
306
307 DirtyStat.dirty_rate = dirtyrate;
308}
Chuan Zheng7df3aa32020-09-16 14:21:57 +0800309
Chuan Zhengba0e5192020-09-16 14:22:01 +0800310/*
Andrei Gudkov00a3f9c2023-04-27 15:42:57 +0300311 * Compute hash of a single page of size TARGET_PAGE_SIZE.
312 */
313static uint32_t compute_page_hash(void *ptr)
314{
Juan Quintelaedd83a72023-05-11 16:12:06 +0200315 size_t page_size = qemu_target_page_size();
Andrei Gudkov00a3f9c2023-04-27 15:42:57 +0300316 uint32_t i;
317 uint64_t v1, v2, v3, v4;
318 uint64_t res;
319 const uint64_t *p = ptr;
320
321 v1 = QEMU_XXHASH_SEED + XXH_PRIME64_1 + XXH_PRIME64_2;
322 v2 = QEMU_XXHASH_SEED + XXH_PRIME64_2;
323 v3 = QEMU_XXHASH_SEED + 0;
324 v4 = QEMU_XXHASH_SEED - XXH_PRIME64_1;
Juan Quintelaedd83a72023-05-11 16:12:06 +0200325 for (i = 0; i < page_size / 8; i += 4) {
Andrei Gudkov00a3f9c2023-04-27 15:42:57 +0300326 v1 = XXH64_round(v1, p[i + 0]);
327 v2 = XXH64_round(v2, p[i + 1]);
328 v3 = XXH64_round(v3, p[i + 2]);
329 v4 = XXH64_round(v4, p[i + 3]);
330 }
331 res = XXH64_mergerounds(v1, v2, v3, v4);
Juan Quintelaedd83a72023-05-11 16:12:06 +0200332 res += page_size;
Andrei Gudkov00a3f9c2023-04-27 15:42:57 +0300333 res = XXH64_avalanche(res);
334 return (uint32_t)(res & UINT32_MAX);
335}
336
337
338/*
Chuan Zhengba0e5192020-09-16 14:22:01 +0800339 * get hash result for the sampled memory with length of TARGET_PAGE_SIZE
340 * in ramblock, which starts from ramblock base address.
341 */
342static uint32_t get_ramblock_vfn_hash(struct RamblockDirtyInfo *info,
343 uint64_t vfn)
344{
Andrei Gudkov00a3f9c2023-04-27 15:42:57 +0300345 uint32_t hash;
Chuan Zhengba0e5192020-09-16 14:22:01 +0800346
Juan Quintelaedd83a72023-05-11 16:12:06 +0200347 hash = compute_page_hash(info->ramblock_addr +
348 vfn * qemu_target_page_size());
Chuan Zhengba0e5192020-09-16 14:22:01 +0800349
Andrei Gudkov00a3f9c2023-04-27 15:42:57 +0300350 trace_get_ramblock_vfn_hash(info->idstr, vfn, hash);
351 return hash;
Chuan Zhengba0e5192020-09-16 14:22:01 +0800352}
353
354static bool save_ramblock_hash(struct RamblockDirtyInfo *info)
355{
356 unsigned int sample_pages_count;
357 int i;
358 GRand *rand;
359
360 sample_pages_count = info->sample_pages_count;
361
362 /* ramblock size less than one page, return success to skip this ramblock */
363 if (unlikely(info->ramblock_pages == 0 || sample_pages_count == 0)) {
364 return true;
365 }
366
367 info->hash_result = g_try_malloc0_n(sample_pages_count,
368 sizeof(uint32_t));
369 if (!info->hash_result) {
370 return false;
371 }
372
373 info->sample_page_vfn = g_try_malloc0_n(sample_pages_count,
374 sizeof(uint64_t));
375 if (!info->sample_page_vfn) {
376 g_free(info->hash_result);
377 return false;
378 }
379
380 rand = g_rand_new();
381 for (i = 0; i < sample_pages_count; i++) {
382 info->sample_page_vfn[i] = g_rand_int_range(rand, 0,
383 info->ramblock_pages - 1);
384 info->hash_result[i] = get_ramblock_vfn_hash(info,
385 info->sample_page_vfn[i]);
386 }
387 g_rand_free(rand);
388
389 return true;
390}
391
392static void get_ramblock_dirty_info(RAMBlock *block,
393 struct RamblockDirtyInfo *info,
394 struct DirtyRateConfig *config)
395{
396 uint64_t sample_pages_per_gigabytes = config->sample_pages_per_gigabytes;
397
398 /* Right shift 30 bits to calc ramblock size in GB */
399 info->sample_pages_count = (qemu_ram_get_used_length(block) *
400 sample_pages_per_gigabytes) >> 30;
401 /* Right shift TARGET_PAGE_BITS to calc page count */
402 info->ramblock_pages = qemu_ram_get_used_length(block) >>
Juan Quintela148b1ad2023-05-11 16:12:07 +0200403 qemu_target_page_bits();
Chuan Zhengba0e5192020-09-16 14:22:01 +0800404 info->ramblock_addr = qemu_ram_get_host_addr(block);
405 strcpy(info->idstr, qemu_ram_get_idstr(block));
406}
407
Chuan Zhengcf0bbb42020-09-16 14:22:05 +0800408static void free_ramblock_dirty_info(struct RamblockDirtyInfo *infos, int count)
409{
410 int i;
411
412 if (!infos) {
413 return;
414 }
415
416 for (i = 0; i < count; i++) {
417 g_free(infos[i].sample_page_vfn);
418 g_free(infos[i].hash_result);
419 }
420 g_free(infos);
421}
422
Chuan Zhengf82583c2020-09-16 14:22:03 +0800423static bool skip_sample_ramblock(RAMBlock *block)
424{
425 /*
426 * Sample only blocks larger than MIN_RAMBLOCK_SIZE.
427 */
428 if (qemu_ram_get_used_length(block) < (MIN_RAMBLOCK_SIZE << 10)) {
Chuan Zheng3c0b5df2020-09-16 14:22:07 +0800429 trace_skip_sample_ramblock(block->idstr,
430 qemu_ram_get_used_length(block));
Chuan Zhengf82583c2020-09-16 14:22:03 +0800431 return true;
432 }
433
434 return false;
435}
436
Chuan Zhengba0e5192020-09-16 14:22:01 +0800437static bool record_ramblock_hash_info(struct RamblockDirtyInfo **block_dinfo,
438 struct DirtyRateConfig config,
439 int *block_count)
440{
441 struct RamblockDirtyInfo *info = NULL;
442 struct RamblockDirtyInfo *dinfo = NULL;
443 RAMBlock *block = NULL;
444 int total_count = 0;
445 int index = 0;
446 bool ret = false;
447
448 RAMBLOCK_FOREACH_MIGRATABLE(block) {
Chuan Zhengf82583c2020-09-16 14:22:03 +0800449 if (skip_sample_ramblock(block)) {
450 continue;
451 }
Chuan Zhengba0e5192020-09-16 14:22:01 +0800452 total_count++;
453 }
454
455 dinfo = g_try_malloc0_n(total_count, sizeof(struct RamblockDirtyInfo));
456 if (dinfo == NULL) {
457 goto out;
458 }
459
460 RAMBLOCK_FOREACH_MIGRATABLE(block) {
Chuan Zhengf82583c2020-09-16 14:22:03 +0800461 if (skip_sample_ramblock(block)) {
462 continue;
463 }
Chuan Zhengba0e5192020-09-16 14:22:01 +0800464 if (index >= total_count) {
465 break;
466 }
467 info = &dinfo[index];
468 get_ramblock_dirty_info(block, info, &config);
469 if (!save_ramblock_hash(info)) {
470 goto out;
471 }
472 index++;
473 }
474 ret = true;
475
476out:
477 *block_count = index;
478 *block_dinfo = dinfo;
479 return ret;
480}
481
Chuan Zheng9c043872020-09-16 14:22:02 +0800482static void calc_page_dirty_rate(struct RamblockDirtyInfo *info)
483{
Andrei Gudkov00a3f9c2023-04-27 15:42:57 +0300484 uint32_t hash;
Chuan Zheng9c043872020-09-16 14:22:02 +0800485 int i;
486
487 for (i = 0; i < info->sample_pages_count; i++) {
Andrei Gudkov00a3f9c2023-04-27 15:42:57 +0300488 hash = get_ramblock_vfn_hash(info, info->sample_page_vfn[i]);
489 if (hash != info->hash_result[i]) {
490 trace_calc_page_dirty_rate(info->idstr, hash, info->hash_result[i]);
Chuan Zheng9c043872020-09-16 14:22:02 +0800491 info->sample_dirty_count++;
492 }
493 }
494}
495
496static struct RamblockDirtyInfo *
497find_block_matched(RAMBlock *block, int count,
498 struct RamblockDirtyInfo *infos)
499{
500 int i;
Chuan Zheng9c043872020-09-16 14:22:02 +0800501
502 for (i = 0; i < count; i++) {
503 if (!strcmp(infos[i].idstr, qemu_ram_get_idstr(block))) {
504 break;
505 }
506 }
507
508 if (i == count) {
509 return NULL;
510 }
511
512 if (infos[i].ramblock_addr != qemu_ram_get_host_addr(block) ||
513 infos[i].ramblock_pages !=
Juan Quintela148b1ad2023-05-11 16:12:07 +0200514 (qemu_ram_get_used_length(block) >> qemu_target_page_bits())) {
Chuan Zheng3c0b5df2020-09-16 14:22:07 +0800515 trace_find_page_matched(block->idstr);
Chuan Zheng9c043872020-09-16 14:22:02 +0800516 return NULL;
517 }
518
Markus Armbruster66997c42022-11-22 14:49:16 +0100519 return &infos[i];
Chuan Zheng9c043872020-09-16 14:22:02 +0800520}
521
522static bool compare_page_hash_info(struct RamblockDirtyInfo *info,
523 int block_count)
524{
525 struct RamblockDirtyInfo *block_dinfo = NULL;
526 RAMBlock *block = NULL;
527
528 RAMBLOCK_FOREACH_MIGRATABLE(block) {
Chuan Zhengf82583c2020-09-16 14:22:03 +0800529 if (skip_sample_ramblock(block)) {
530 continue;
531 }
Chuan Zheng9c043872020-09-16 14:22:02 +0800532 block_dinfo = find_block_matched(block, block_count, info);
533 if (block_dinfo == NULL) {
534 continue;
535 }
536 calc_page_dirty_rate(block_dinfo);
537 update_dirtyrate_stat(block_dinfo);
538 }
539
Hyman Huang(黄勇)71864ea2021-06-29 16:01:20 +0000540 if (DirtyStat.page_sampling.total_sample_count == 0) {
Chuan Zheng9c043872020-09-16 14:22:02 +0800541 return false;
542 }
543
544 return true;
545}
546
Hyman Huang(黄勇)826b8bc2021-07-20 23:19:17 +0800547static inline void record_dirtypages_bitmap(DirtyPageRecord *dirty_pages,
548 bool start)
549{
550 if (start) {
551 dirty_pages->start_pages = total_dirty_pages;
552 } else {
553 dirty_pages->end_pages = total_dirty_pages;
554 }
555}
556
Hyman Huang(黄勇)826b8bc2021-07-20 23:19:17 +0800557static inline void dirtyrate_manual_reset_protect(void)
558{
559 RAMBlock *block = NULL;
560
561 WITH_RCU_READ_LOCK_GUARD() {
562 RAMBLOCK_FOREACH_MIGRATABLE(block) {
563 memory_region_clear_dirty_bitmap(block->mr, 0,
564 block->used_length);
565 }
566 }
567}
568
569static void calculate_dirtyrate_dirty_bitmap(struct DirtyRateConfig config)
570{
571 int64_t msec = 0;
572 int64_t start_time;
573 DirtyPageRecord dirty_pages;
574
575 qemu_mutex_lock_iothread();
576 memory_global_dirty_log_start(GLOBAL_DIRTY_DIRTY_RATE);
577
578 /*
579 * 1'round of log sync may return all 1 bits with
580 * KVM_DIRTY_LOG_INITIALLY_SET enable
581 * skip it unconditionally and start dirty tracking
582 * from 2'round of log sync
583 */
Gavin Shan1e493be2023-05-09 12:21:19 +1000584 memory_global_dirty_log_sync(false);
Hyman Huang(黄勇)826b8bc2021-07-20 23:19:17 +0800585
586 /*
587 * reset page protect manually and unconditionally.
588 * this make sure kvm dirty log be cleared if
589 * KVM_DIRTY_LOG_MANUAL_PROTECT_ENABLE cap is enabled.
590 */
591 dirtyrate_manual_reset_protect();
592 qemu_mutex_unlock_iothread();
593
594 record_dirtypages_bitmap(&dirty_pages, true);
595
596 start_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
597 DirtyStat.start_time = start_time / 1000;
598
599 msec = config.sample_period_seconds * 1000;
Hyman Huang(黄勇)82441662022-06-26 01:38:32 +0800600 msec = dirty_stat_wait(msec, start_time);
Hyman Huang(黄勇)826b8bc2021-07-20 23:19:17 +0800601 DirtyStat.calc_time = msec / 1000;
602
603 /*
Hyman Huang(黄勇)82441662022-06-26 01:38:32 +0800604 * do two things.
Hyman Huang(黄勇)826b8bc2021-07-20 23:19:17 +0800605 * 1. fetch dirty bitmap from kvm
606 * 2. stop dirty tracking
607 */
Hyman Huang(黄勇)82441662022-06-26 01:38:32 +0800608 global_dirty_log_sync(GLOBAL_DIRTY_DIRTY_RATE, true);
Hyman Huang(黄勇)826b8bc2021-07-20 23:19:17 +0800609
610 record_dirtypages_bitmap(&dirty_pages, false);
611
Hyman Huang(黄勇)82441662022-06-26 01:38:32 +0800612 DirtyStat.dirty_rate = do_calculate_dirtyrate(dirty_pages, msec);
Hyman Huang(黄勇)826b8bc2021-07-20 23:19:17 +0800613}
614
Hyman Huang(黄勇)0e21bf22021-06-29 16:01:23 +0000615static void calculate_dirtyrate_dirty_ring(struct DirtyRateConfig config)
616{
Hyman Huang(黄勇)82441662022-06-26 01:38:32 +0800617 int64_t duration;
Hyman Huang(黄勇)0e21bf22021-06-29 16:01:23 +0000618 uint64_t dirtyrate = 0;
619 uint64_t dirtyrate_sum = 0;
Hyman Huang(黄勇)0e21bf22021-06-29 16:01:23 +0000620 int i = 0;
621
Hyman Huang(黄勇)82441662022-06-26 01:38:32 +0800622 /* start log sync */
623 global_dirty_log_change(GLOBAL_DIRTY_DIRTY_RATE, true);
Hyman Huang(黄勇)0e21bf22021-06-29 16:01:23 +0000624
Hyman Huang(黄勇)82441662022-06-26 01:38:32 +0800625 DirtyStat.start_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME) / 1000;
Hyman Huang(黄勇)0e21bf22021-06-29 16:01:23 +0000626
Hyman Huang(黄勇)82441662022-06-26 01:38:32 +0800627 /* calculate vcpu dirtyrate */
628 duration = vcpu_calculate_dirtyrate(config.sample_period_seconds * 1000,
629 &DirtyStat.dirty_ring,
630 GLOBAL_DIRTY_DIRTY_RATE,
631 true);
Hyman Huang(黄勇)0e21bf22021-06-29 16:01:23 +0000632
Hyman Huang(黄勇)82441662022-06-26 01:38:32 +0800633 DirtyStat.calc_time = duration / 1000;
Hyman Huang(黄勇)0e21bf22021-06-29 16:01:23 +0000634
Hyman Huang(黄勇)82441662022-06-26 01:38:32 +0800635 /* calculate vm dirtyrate */
Hyman Huang(黄勇)0e21bf22021-06-29 16:01:23 +0000636 for (i = 0; i < DirtyStat.dirty_ring.nvcpu; i++) {
Hyman Huang(黄勇)82441662022-06-26 01:38:32 +0800637 dirtyrate = DirtyStat.dirty_ring.rates[i].dirty_rate;
Hyman Huang(黄勇)0e21bf22021-06-29 16:01:23 +0000638 DirtyStat.dirty_ring.rates[i].dirty_rate = dirtyrate;
639 dirtyrate_sum += dirtyrate;
640 }
641
642 DirtyStat.dirty_rate = dirtyrate_sum;
Hyman Huang(黄勇)0e21bf22021-06-29 16:01:23 +0000643}
644
645static void calculate_dirtyrate_sample_vm(struct DirtyRateConfig config)
Chuan Zheng4240dce2020-09-16 14:21:56 +0800646{
Chuan Zhengcf0bbb42020-09-16 14:22:05 +0800647 struct RamblockDirtyInfo *block_dinfo = NULL;
648 int block_count = 0;
649 int64_t msec = 0;
650 int64_t initial_time;
651
Chuan Zhengcf0bbb42020-09-16 14:22:05 +0800652 rcu_read_lock();
653 initial_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
654 if (!record_ramblock_hash_info(&block_dinfo, config, &block_count)) {
655 goto out;
656 }
657 rcu_read_unlock();
658
659 msec = config.sample_period_seconds * 1000;
Hyman Huang(黄勇)82441662022-06-26 01:38:32 +0800660 msec = dirty_stat_wait(msec, initial_time);
Chuan Zheng4c437252020-09-16 14:22:06 +0800661 DirtyStat.start_time = initial_time / 1000;
662 DirtyStat.calc_time = msec / 1000;
Chuan Zhengcf0bbb42020-09-16 14:22:05 +0800663
664 rcu_read_lock();
665 if (!compare_page_hash_info(block_dinfo, block_count)) {
666 goto out;
667 }
668
669 update_dirtyrate(msec);
670
671out:
672 rcu_read_unlock();
673 free_ramblock_dirty_info(block_dinfo, block_count);
Chuan Zheng4240dce2020-09-16 14:21:56 +0800674}
675
Hyman Huang(黄勇)0e21bf22021-06-29 16:01:23 +0000676static void calculate_dirtyrate(struct DirtyRateConfig config)
677{
Hyman Huang(黄勇)826b8bc2021-07-20 23:19:17 +0800678 if (config.mode == DIRTY_RATE_MEASURE_MODE_DIRTY_BITMAP) {
679 calculate_dirtyrate_dirty_bitmap(config);
680 } else if (config.mode == DIRTY_RATE_MEASURE_MODE_DIRTY_RING) {
Hyman Huang(黄勇)0e21bf22021-06-29 16:01:23 +0000681 calculate_dirtyrate_dirty_ring(config);
682 } else {
683 calculate_dirtyrate_sample_vm(config);
684 }
685
686 trace_dirtyrate_calculate(DirtyStat.dirty_rate);
687}
688
Chuan Zheng4240dce2020-09-16 14:21:56 +0800689void *get_dirtyrate_thread(void *arg)
690{
691 struct DirtyRateConfig config = *(struct DirtyRateConfig *)arg;
Chuan Zheng7df3aa32020-09-16 14:21:57 +0800692 int ret;
Hyman Huang(黄勇)15eb2d62021-06-29 16:01:21 +0000693 rcu_register_thread();
Chuan Zheng7df3aa32020-09-16 14:21:57 +0800694
695 ret = dirtyrate_set_state(&CalculatingState, DIRTY_RATE_STATUS_UNSTARTED,
696 DIRTY_RATE_STATUS_MEASURING);
697 if (ret == -1) {
698 error_report("change dirtyrate state failed.");
699 return NULL;
700 }
Chuan Zheng4240dce2020-09-16 14:21:56 +0800701
702 calculate_dirtyrate(config);
703
Chuan Zheng7df3aa32020-09-16 14:21:57 +0800704 ret = dirtyrate_set_state(&CalculatingState, DIRTY_RATE_STATUS_MEASURING,
705 DIRTY_RATE_STATUS_MEASURED);
706 if (ret == -1) {
707 error_report("change dirtyrate state failed.");
708 }
Hyman Huang(黄勇)15eb2d62021-06-29 16:01:21 +0000709
710 rcu_unregister_thread();
Chuan Zheng4240dce2020-09-16 14:21:56 +0800711 return NULL;
712}
Chuan Zheng4c437252020-09-16 14:22:06 +0800713
Hyman Huang(黄勇)0e21bf22021-06-29 16:01:23 +0000714void qmp_calc_dirty_rate(int64_t calc_time,
715 bool has_sample_pages,
716 int64_t sample_pages,
717 bool has_mode,
718 DirtyRateMeasureMode mode,
719 Error **errp)
Chuan Zheng4c437252020-09-16 14:22:06 +0800720{
721 static struct DirtyRateConfig config;
722 QemuThread thread;
723 int ret;
Hyman Huang(黄勇)9865d0f2021-06-29 16:01:22 +0000724 int64_t start_time;
Chuan Zheng4c437252020-09-16 14:22:06 +0800725
726 /*
727 * If the dirty rate is already being measured, don't attempt to start.
728 */
729 if (qatomic_read(&CalculatingState) == DIRTY_RATE_STATUS_MEASURING) {
730 error_setg(errp, "the dirty rate is already being measured.");
731 return;
732 }
733
734 if (!is_sample_period_valid(calc_time)) {
735 error_setg(errp, "calc-time is out of range[%d, %d].",
736 MIN_FETCH_DIRTYRATE_TIME_SEC,
737 MAX_FETCH_DIRTYRATE_TIME_SEC);
738 return;
739 }
740
Hyman Huang(黄勇)0e21bf22021-06-29 16:01:23 +0000741 if (!has_mode) {
742 mode = DIRTY_RATE_MEASURE_MODE_PAGE_SAMPLING;
743 }
744
Zhenzhong Duanbd9510d2022-11-29 12:04:04 +0800745 if (has_sample_pages && mode != DIRTY_RATE_MEASURE_MODE_PAGE_SAMPLING) {
746 error_setg(errp, "sample-pages is used only in page-sampling mode");
Hyman Huang(黄勇)0e21bf22021-06-29 16:01:23 +0000747 return;
748 }
749
Hyman Huang(黄勇)7afa08c2021-06-07 09:11:34 +0800750 if (has_sample_pages) {
751 if (!is_sample_pages_valid(sample_pages)) {
752 error_setg(errp, "sample-pages is out of range[%d, %d].",
753 MIN_SAMPLE_PAGE_COUNT,
754 MAX_SAMPLE_PAGE_COUNT);
755 return;
756 }
757 } else {
758 sample_pages = DIRTYRATE_DEFAULT_SAMPLE_PAGES;
759 }
760
Chuan Zheng4c437252020-09-16 14:22:06 +0800761 /*
Hyman Huang(黄勇)0e21bf22021-06-29 16:01:23 +0000762 * dirty ring mode only works when kvm dirty ring is enabled.
Hyman Huang(黄勇)826b8bc2021-07-20 23:19:17 +0800763 * on the contrary, dirty bitmap mode is not.
Hyman Huang(黄勇)0e21bf22021-06-29 16:01:23 +0000764 */
Hyman Huang(黄勇)826b8bc2021-07-20 23:19:17 +0800765 if (((mode == DIRTY_RATE_MEASURE_MODE_DIRTY_RING) &&
766 !kvm_dirty_ring_enabled()) ||
767 ((mode == DIRTY_RATE_MEASURE_MODE_DIRTY_BITMAP) &&
768 kvm_dirty_ring_enabled())) {
769 error_setg(errp, "mode %s is not enabled, use other method instead.",
770 DirtyRateMeasureMode_str(mode));
771 return;
Hyman Huang(黄勇)0e21bf22021-06-29 16:01:23 +0000772 }
773
774 /*
Chuan Zheng4c437252020-09-16 14:22:06 +0800775 * Init calculation state as unstarted.
776 */
777 ret = dirtyrate_set_state(&CalculatingState, CalculatingState,
778 DIRTY_RATE_STATUS_UNSTARTED);
779 if (ret == -1) {
780 error_setg(errp, "init dirty rate calculation state failed.");
781 return;
782 }
783
784 config.sample_period_seconds = calc_time;
Hyman Huang(黄勇)7afa08c2021-06-07 09:11:34 +0800785 config.sample_pages_per_gigabytes = sample_pages;
Hyman Huang(黄勇)0e21bf22021-06-29 16:01:23 +0000786 config.mode = mode;
787
788 cleanup_dirtyrate_stat(config);
789
790 /*
791 * update dirty rate mode so that we can figure out what mode has
792 * been used in last calculation
793 **/
794 dirtyrate_mode = mode;
Hyman Huang(黄勇)9865d0f2021-06-29 16:01:22 +0000795
796 start_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME) / 1000;
797 init_dirtyrate_stat(start_time, config);
798
Chuan Zheng4c437252020-09-16 14:22:06 +0800799 qemu_thread_create(&thread, "get_dirtyrate", get_dirtyrate_thread,
800 (void *)&config, QEMU_THREAD_DETACHED);
801}
802
803struct DirtyRateInfo *qmp_query_dirty_rate(Error **errp)
804{
805 return query_dirty_rate_info();
806}
Peter Xua4a571d2021-06-07 09:11:57 +0800807
808void hmp_info_dirty_rate(Monitor *mon, const QDict *qdict)
809{
810 DirtyRateInfo *info = query_dirty_rate_info();
811
812 monitor_printf(mon, "Status: %s\n",
813 DirtyRateStatus_str(info->status));
814 monitor_printf(mon, "Start Time: %"PRIi64" (ms)\n",
815 info->start_time);
Zhenzhong Duanbd9510d2022-11-29 12:04:04 +0800816 if (info->mode == DIRTY_RATE_MEASURE_MODE_PAGE_SAMPLING) {
817 monitor_printf(mon, "Sample Pages: %"PRIu64" (per GB)\n",
818 info->sample_pages);
819 }
Peter Xua4a571d2021-06-07 09:11:57 +0800820 monitor_printf(mon, "Period: %"PRIi64" (sec)\n",
821 info->calc_time);
Hyman Huang(黄勇)0e21bf22021-06-29 16:01:23 +0000822 monitor_printf(mon, "Mode: %s\n",
823 DirtyRateMeasureMode_str(info->mode));
Peter Xua4a571d2021-06-07 09:11:57 +0800824 monitor_printf(mon, "Dirty rate: ");
825 if (info->has_dirty_rate) {
826 monitor_printf(mon, "%"PRIi64" (MB/s)\n", info->dirty_rate);
Hyman Huang(黄勇)0e21bf22021-06-29 16:01:23 +0000827 if (info->has_vcpu_dirty_rate) {
828 DirtyRateVcpuList *rate, *head = info->vcpu_dirty_rate;
829 for (rate = head; rate != NULL; rate = rate->next) {
830 monitor_printf(mon, "vcpu[%"PRIi64"], Dirty rate: %"PRIi64
831 " (MB/s)\n", rate->value->id,
832 rate->value->dirty_rate);
833 }
834 }
Peter Xua4a571d2021-06-07 09:11:57 +0800835 } else {
836 monitor_printf(mon, "(not ready)\n");
837 }
Hyman Huang(黄勇)0e21bf22021-06-29 16:01:23 +0000838
839 qapi_free_DirtyRateVcpuList(info->vcpu_dirty_rate);
Peter Xua4a571d2021-06-07 09:11:57 +0800840 g_free(info);
841}
842
843void hmp_calc_dirty_rate(Monitor *mon, const QDict *qdict)
844{
845 int64_t sec = qdict_get_try_int(qdict, "second", 0);
846 int64_t sample_pages = qdict_get_try_int(qdict, "sample_pages_per_GB", -1);
847 bool has_sample_pages = (sample_pages != -1);
Hyman Huang(黄勇)0e21bf22021-06-29 16:01:23 +0000848 bool dirty_ring = qdict_get_try_bool(qdict, "dirty_ring", false);
Hyman Huang(黄勇)826b8bc2021-07-20 23:19:17 +0800849 bool dirty_bitmap = qdict_get_try_bool(qdict, "dirty_bitmap", false);
850 DirtyRateMeasureMode mode = DIRTY_RATE_MEASURE_MODE_PAGE_SAMPLING;
Peter Xua4a571d2021-06-07 09:11:57 +0800851 Error *err = NULL;
852
853 if (!sec) {
854 monitor_printf(mon, "Incorrect period length specified!\n");
855 return;
856 }
857
Hyman Huang(黄勇)826b8bc2021-07-20 23:19:17 +0800858 if (dirty_ring && dirty_bitmap) {
859 monitor_printf(mon, "Either dirty ring or dirty bitmap "
860 "can be specified!\n");
861 return;
862 }
863
864 if (dirty_bitmap) {
865 mode = DIRTY_RATE_MEASURE_MODE_DIRTY_BITMAP;
866 } else if (dirty_ring) {
867 mode = DIRTY_RATE_MEASURE_MODE_DIRTY_RING;
868 }
869
Hyman Huang(黄勇)0e21bf22021-06-29 16:01:23 +0000870 qmp_calc_dirty_rate(sec, has_sample_pages, sample_pages, true,
871 mode, &err);
Peter Xua4a571d2021-06-07 09:11:57 +0800872 if (err) {
873 hmp_handle_error(mon, err);
874 return;
875 }
876
877 monitor_printf(mon, "Starting dirty rate measurement with period %"PRIi64
878 " seconds\n", sec);
879 monitor_printf(mon, "[Please use 'info dirty_rate' to check results]\n");
880}