blob: c76503aea8271f5be7df8d3154e9d927c182329c [file] [log] [blame]
bellard54936002003-05-13 00:25:15 +00001/*
Paolo Bonzinid9f24bf2020-10-06 09:05:29 +02002 * RAM allocation and memory access
ths5fafdf22007-09-16 21:08:06 +00003 *
bellard54936002003-05-13 00:25:15 +00004 * Copyright (c) 2003 Fabrice Bellard
5 *
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
Chetan Pant61f3c912020-10-23 12:44:24 +00009 * version 2.1 of the License, or (at your option) any later version.
bellard54936002003-05-13 00:25:15 +000010 *
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
15 *
16 * You should have received a copy of the GNU Lesser General Public
Blue Swirl8167ee82009-07-16 20:47:01 +000017 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
bellard54936002003-05-13 00:25:15 +000018 */
Markus Armbruster14a48c12019-05-23 16:35:05 +020019
Peter Maydell7b31bbc2016-01-26 18:16:56 +000020#include "qemu/osdep.h"
Marc-André Lureauec5f7ca2022-03-23 19:57:34 +040021#include "exec/page-vary.h"
Markus Armbrusterda34e652016-03-14 09:01:28 +010022#include "qapi/error.h"
bellard54936002003-05-13 00:25:15 +000023
Veronia Bahaaf348b6d2016-03-20 19:16:19 +020024#include "qemu/cutils.h"
Richard Henderson084cfca2020-12-14 08:02:33 -060025#include "qemu/cacheflush.h"
Markus Armbrustere2c1c342022-12-21 14:35:49 +010026#include "qemu/hbitmap.h"
Peter Maydellb85ea5f2022-02-08 20:08:52 +000027#include "qemu/madvise.h"
Philippe Mathieu-Daudéd5e26812024-05-07 14:12:46 +020028#include "qemu/lockable.h"
Claudio Fontana78271682021-02-04 17:39:23 +010029
30#ifdef CONFIG_TCG
31#include "hw/core/tcg-cpu-ops.h"
32#endif /* CONFIG_TCG */
33
Paolo Bonzini63c91552016-03-15 13:18:37 +010034#include "exec/exec-all.h"
Philippe Mathieu-Daudé74781c02023-12-06 20:27:32 +010035#include "exec/page-protection.h"
Juan Quintela51180422017-04-24 20:50:19 +020036#include "exec/target_page.h"
Philippe Mathieu-Daudé68df8c82024-11-13 22:46:13 +010037#include "exec/translation-block.h"
Paolo Bonzini741da0d2014-06-27 08:40:04 +020038#include "hw/qdev-core.h"
Fam Zhengc7e002c2017-07-14 10:15:08 +080039#include "hw/qdev-properties.h"
Marcel Apfelbaum47c8ca52015-02-04 17:43:54 +020040#include "hw/boards.h"
Philippe Mathieu-Daudé32cad1f2024-12-03 15:20:13 +010041#include "system/xen.h"
42#include "system/kvm.h"
43#include "system/tcg.h"
44#include "system/qtest.h"
Paolo Bonzini1de7afc2012-12-17 18:20:00 +010045#include "qemu/timer.h"
46#include "qemu/config-file.h"
Andreas Färber75a34032013-09-02 16:57:02 +020047#include "qemu/error-report.h"
Markus Armbrusterb6b71cb2019-04-17 21:17:56 +020048#include "qemu/qemu-print.h"
Philippe Mathieu-Daudé3ab6fdc2021-12-15 19:24:21 +010049#include "qemu/log.h"
Peter Maydell5df022c2022-02-26 18:07:23 +000050#include "qemu/memalign.h"
Paolo Bonzini741da0d2014-06-27 08:40:04 +020051#include "exec/memory.h"
Paolo Bonzinidf43d492016-03-16 10:24:54 +010052#include "exec/ioport.h"
Philippe Mathieu-Daudé32cad1f2024-12-03 15:20:13 +010053#include "system/dma.h"
54#include "system/hostmem.h"
55#include "system/hw_accel.h"
56#include "system/xen-mapcache.h"
Alex Bennéed44fe132024-07-05 09:40:10 +010057#include "trace.h"
Dr. David Alan Gilbertd3a50382017-02-24 18:28:32 +000058
Dr. David Alan Gilberte2fa71f2017-02-24 18:28:33 +000059#ifdef CONFIG_FALLOCATE_PUNCH_HOLE
Dr. David Alan Gilberte2fa71f2017-02-24 18:28:33 +000060#include <linux/falloc.h>
61#endif
62
Mike Day0dc3f442013-09-05 14:41:35 -040063#include "qemu/rcu_queue.h"
Jan Kiszka4840f102015-06-18 18:47:22 +020064#include "qemu/main-loop.h"
Philippe Mathieu-Daudé32cad1f2024-12-03 15:20:13 +010065#include "system/replay.h"
Blue Swirl0cac1b62012-04-09 16:50:52 +000066
Paolo Bonzini022c62c2012-12-17 18:19:49 +010067#include "exec/memory-internal.h"
Juan Quintela220c3eb2013-10-14 17:13:59 +020068#include "exec/ram_addr.h"
Avi Kivity67d95c12011-12-15 15:25:22 +020069
Beata Michalska61c490e2019-11-21 00:08:41 +000070#include "qemu/pmem.h"
71
Bharata B Rao9dfeca72016-05-12 09:18:12 +053072#include "migration/vmstate.h"
73
Michael S. Tsirkinb35ba302013-11-11 17:52:07 +020074#include "qemu/range.h"
Michael S. Tsirkin794e8f32015-09-24 14:41:17 +030075#ifndef _WIN32
76#include "qemu/mmap-alloc.h"
77#endif
Michael S. Tsirkinb35ba302013-11-11 17:52:07 +020078
Peter Xube9b23c2017-05-12 12:17:41 +080079#include "monitor/monitor.h"
80
Jingqi Liuce317be2020-04-29 16:50:09 +080081#ifdef CONFIG_LIBDAXCTL
82#include <daxctl/libdaxctl.h>
83#endif
84
blueswir1db7b5422007-05-26 17:36:03 +000085//#define DEBUG_SUBPAGE
ths1196be32007-03-17 15:17:58 +000086
Mike Day0dc3f442013-09-05 14:41:35 -040087/* ram_list is read under rcu_read_lock()/rcu_read_unlock(). Writes
88 * are protected by the ramlist lock.
89 */
Mike Day0d53d9f2015-01-21 13:45:24 +010090RAMList ram_list = { .blocks = QLIST_HEAD_INITIALIZER(ram_list.blocks) };
Avi Kivity62152b82011-07-26 14:26:14 +030091
92static MemoryRegion *system_memory;
Avi Kivity309cb472011-08-08 16:09:03 +030093static MemoryRegion *system_io;
Avi Kivity62152b82011-07-26 14:26:14 +030094
Avi Kivityf6790af2012-10-02 20:13:51 +020095AddressSpace address_space_io;
96AddressSpace address_space_memory;
Avi Kivity2673a5d2012-10-02 18:49:28 +020097
Jan Kiszkaacc9d802013-05-26 21:55:37 +020098static MemoryRegion io_mem_unassigned;
Avi Kivity4346ae32012-02-10 17:00:01 +020099
Paolo Bonzini1db8abb2013-05-21 12:07:21 +0200100typedef struct PhysPageEntry PhysPageEntry;
101
102struct PhysPageEntry {
Michael S. Tsirkin9736e552013-11-11 14:42:43 +0200103 /* How many bits skip to next level (in units of L2_SIZE). 0 for a leaf. */
Michael S. Tsirkin8b795762013-11-11 14:51:56 +0200104 uint32_t skip : 6;
Michael S. Tsirkin9736e552013-11-11 14:42:43 +0200105 /* index into phys_sections (!skip) or phys_map_nodes (skip) */
Michael S. Tsirkin8b795762013-11-11 14:51:56 +0200106 uint32_t ptr : 26;
Paolo Bonzini1db8abb2013-05-21 12:07:21 +0200107};
108
Michael S. Tsirkin8b795762013-11-11 14:51:56 +0200109#define PHYS_MAP_NODE_NIL (((uint32_t)~0) >> 6)
110
Paolo Bonzini03f49952013-11-07 17:14:36 +0100111/* Size of the L2 (and L3, etc) page tables. */
Paolo Bonzini57271d62013-11-07 17:14:37 +0100112#define ADDR_SPACE_BITS 64
Paolo Bonzini03f49952013-11-07 17:14:36 +0100113
Michael S. Tsirkin026736c2013-11-13 20:13:03 +0200114#define P_L2_BITS 9
Paolo Bonzini03f49952013-11-07 17:14:36 +0100115#define P_L2_SIZE (1 << P_L2_BITS)
116
117#define P_L2_LEVELS (((ADDR_SPACE_BITS - TARGET_PAGE_BITS - 1) / P_L2_BITS) + 1)
118
119typedef PhysPageEntry Node[P_L2_SIZE];
Paolo Bonzini0475d942013-05-29 12:28:21 +0200120
Marcel Apfelbaum53cb28c2013-12-01 14:02:23 +0200121typedef struct PhysPageMap {
Paolo Bonzini79e2b9a2015-01-21 12:09:14 +0100122 struct rcu_head rcu;
123
Marcel Apfelbaum53cb28c2013-12-01 14:02:23 +0200124 unsigned sections_nb;
125 unsigned sections_nb_alloc;
126 unsigned nodes_nb;
127 unsigned nodes_nb_alloc;
128 Node *nodes;
129 MemoryRegionSection *sections;
130} PhysPageMap;
131
Paolo Bonzini1db8abb2013-05-21 12:07:21 +0200132struct AddressSpaceDispatch {
Fam Zheng729633c2016-03-01 14:18:24 +0800133 MemoryRegionSection *mru_section;
Paolo Bonzini1db8abb2013-05-21 12:07:21 +0200134 /* This is a multi-level map on the physical address space.
135 * The bottom level has pointers to MemoryRegionSections.
136 */
137 PhysPageEntry phys_map;
Marcel Apfelbaum53cb28c2013-12-01 14:02:23 +0200138 PhysPageMap map;
Paolo Bonzini1db8abb2013-05-21 12:07:21 +0200139};
140
Jan Kiszka90260c62013-05-26 21:46:51 +0200141#define SUBPAGE_IDX(addr) ((addr) & ~TARGET_PAGE_MASK)
142typedef struct subpage_t {
143 MemoryRegion iomem;
Alexey Kardashevskiy16620682017-09-21 18:50:58 +1000144 FlatView *fv;
Jan Kiszka90260c62013-05-26 21:46:51 +0200145 hwaddr base;
Vijaya Kumar K2615fab2016-10-24 16:26:49 +0100146 uint16_t sub_section[];
Jan Kiszka90260c62013-05-26 21:46:51 +0200147} subpage_t;
148
Liu Ping Fanb41aac42013-05-29 11:09:17 +0200149#define PHYS_SECTION_UNASSIGNED 0
Avi Kivity5312bd82012-02-12 18:32:55 +0200150
pbrooke2eef172008-06-08 01:09:01 +0000151static void io_mem_init(void);
Avi Kivity62152b82011-07-26 14:26:14 +0300152static void memory_map_init(void);
Paolo Bonzini9458a9a2018-02-06 18:37:39 +0100153static void tcg_log_global_after_sync(MemoryListener *listener);
Edgar E. Iglesias09daed82013-12-17 13:06:51 +1000154static void tcg_commit(MemoryListener *listener);
pbrooke2eef172008-06-08 01:09:01 +0000155
Peter Maydell32857f42015-10-01 15:29:50 +0100156/**
157 * CPUAddressSpace: all the information a CPU needs about an AddressSpace
158 * @cpu: the CPU whose AddressSpace this is
159 * @as: the AddressSpace itself
160 * @memory_dispatch: its dispatch pointer (cached, RCU protected)
161 * @tcg_as_listener: listener for tracking changes to the AddressSpace
162 */
Paolo Bonzini15d62532024-05-02 17:14:42 +0200163typedef struct CPUAddressSpace {
Peter Maydell32857f42015-10-01 15:29:50 +0100164 CPUState *cpu;
165 AddressSpace *as;
166 struct AddressSpaceDispatch *memory_dispatch;
167 MemoryListener tcg_as_listener;
Paolo Bonzini15d62532024-05-02 17:14:42 +0200168} CPUAddressSpace;
Peter Maydell32857f42015-10-01 15:29:50 +0100169
Gerd Hoffmann8deaf122017-04-21 11:16:25 +0200170struct DirtyBitmapSnapshot {
171 ram_addr_t start;
172 ram_addr_t end;
173 unsigned long dirty[];
174};
175
Marcel Apfelbaum53cb28c2013-12-01 14:02:23 +0200176static void phys_map_node_reserve(PhysPageMap *map, unsigned nodes)
Avi Kivityf7bf5462012-02-13 20:12:05 +0200177{
Peter Lieven101420b2016-07-15 12:03:50 +0200178 static unsigned alloc_hint = 16;
Marcel Apfelbaum53cb28c2013-12-01 14:02:23 +0200179 if (map->nodes_nb + nodes > map->nodes_nb_alloc) {
Wei Yangc95cfd02019-03-21 16:25:52 +0800180 map->nodes_nb_alloc = MAX(alloc_hint, map->nodes_nb + nodes);
Marcel Apfelbaum53cb28c2013-12-01 14:02:23 +0200181 map->nodes = g_renew(Node, map->nodes, map->nodes_nb_alloc);
Peter Lieven101420b2016-07-15 12:03:50 +0200182 alloc_hint = map->nodes_nb_alloc;
Avi Kivityf7bf5462012-02-13 20:12:05 +0200183 }
184}
185
Paolo Bonzinidb946042015-05-21 15:12:29 +0200186static uint32_t phys_map_node_alloc(PhysPageMap *map, bool leaf)
Avi Kivityd6f2ea22012-02-12 20:12:49 +0200187{
188 unsigned i;
Michael S. Tsirkin8b795762013-11-11 14:51:56 +0200189 uint32_t ret;
Paolo Bonzinidb946042015-05-21 15:12:29 +0200190 PhysPageEntry e;
191 PhysPageEntry *p;
Avi Kivityd6f2ea22012-02-12 20:12:49 +0200192
Marcel Apfelbaum53cb28c2013-12-01 14:02:23 +0200193 ret = map->nodes_nb++;
Paolo Bonzinidb946042015-05-21 15:12:29 +0200194 p = map->nodes[ret];
Avi Kivityd6f2ea22012-02-12 20:12:49 +0200195 assert(ret != PHYS_MAP_NODE_NIL);
Marcel Apfelbaum53cb28c2013-12-01 14:02:23 +0200196 assert(ret != map->nodes_nb_alloc);
Paolo Bonzinidb946042015-05-21 15:12:29 +0200197
198 e.skip = leaf ? 0 : 1;
199 e.ptr = leaf ? PHYS_SECTION_UNASSIGNED : PHYS_MAP_NODE_NIL;
Paolo Bonzini03f49952013-11-07 17:14:36 +0100200 for (i = 0; i < P_L2_SIZE; ++i) {
Paolo Bonzinidb946042015-05-21 15:12:29 +0200201 memcpy(&p[i], &e, sizeof(e));
Avi Kivityd6f2ea22012-02-12 20:12:49 +0200202 }
Avi Kivityf7bf5462012-02-13 20:12:05 +0200203 return ret;
Avi Kivityd6f2ea22012-02-12 20:12:49 +0200204}
205
Marcel Apfelbaum53cb28c2013-12-01 14:02:23 +0200206static void phys_page_set_level(PhysPageMap *map, PhysPageEntry *lp,
Wei Yang56b15072019-03-21 16:25:50 +0800207 hwaddr *index, uint64_t *nb, uint16_t leaf,
Avi Kivity29990972012-02-13 20:21:20 +0200208 int level)
Avi Kivityf7bf5462012-02-13 20:12:05 +0200209{
210 PhysPageEntry *p;
Paolo Bonzini03f49952013-11-07 17:14:36 +0100211 hwaddr step = (hwaddr)1 << (level * P_L2_BITS);
Avi Kivityf7bf5462012-02-13 20:12:05 +0200212
Michael S. Tsirkin9736e552013-11-11 14:42:43 +0200213 if (lp->skip && lp->ptr == PHYS_MAP_NODE_NIL) {
Paolo Bonzinidb946042015-05-21 15:12:29 +0200214 lp->ptr = phys_map_node_alloc(map, level == 0);
Avi Kivityf7bf5462012-02-13 20:12:05 +0200215 }
Paolo Bonzinidb946042015-05-21 15:12:29 +0200216 p = map->nodes[lp->ptr];
Paolo Bonzini03f49952013-11-07 17:14:36 +0100217 lp = &p[(*index >> (level * P_L2_BITS)) & (P_L2_SIZE - 1)];
Avi Kivityf7bf5462012-02-13 20:12:05 +0200218
Paolo Bonzini03f49952013-11-07 17:14:36 +0100219 while (*nb && lp < &p[P_L2_SIZE]) {
Avi Kivity07f07b32012-02-13 20:45:32 +0200220 if ((*index & (step - 1)) == 0 && *nb >= step) {
Michael S. Tsirkin9736e552013-11-11 14:42:43 +0200221 lp->skip = 0;
Avi Kivityc19e8802012-02-13 20:25:31 +0200222 lp->ptr = leaf;
Avi Kivity07f07b32012-02-13 20:45:32 +0200223 *index += step;
224 *nb -= step;
Avi Kivity29990972012-02-13 20:21:20 +0200225 } else {
Marcel Apfelbaum53cb28c2013-12-01 14:02:23 +0200226 phys_page_set_level(map, lp, index, nb, leaf, level - 1);
Avi Kivity29990972012-02-13 20:21:20 +0200227 }
228 ++lp;
Avi Kivityf7bf5462012-02-13 20:12:05 +0200229 }
230}
231
Avi Kivityac1970f2012-10-03 16:22:53 +0200232static void phys_page_set(AddressSpaceDispatch *d,
Wei Yang56b15072019-03-21 16:25:50 +0800233 hwaddr index, uint64_t nb,
Avi Kivity29990972012-02-13 20:21:20 +0200234 uint16_t leaf)
bellard92e873b2004-05-21 14:52:29 +0000235{
Avi Kivity29990972012-02-13 20:21:20 +0200236 /* Wildly overreserve - it doesn't matter much. */
Marcel Apfelbaum53cb28c2013-12-01 14:02:23 +0200237 phys_map_node_reserve(&d->map, 3 * P_L2_LEVELS);
bellard92e873b2004-05-21 14:52:29 +0000238
Marcel Apfelbaum53cb28c2013-12-01 14:02:23 +0200239 phys_page_set_level(&d->map, &d->phys_map, &index, &nb, leaf, P_L2_LEVELS - 1);
bellard92e873b2004-05-21 14:52:29 +0000240}
241
Michael S. Tsirkinb35ba302013-11-11 17:52:07 +0200242/* Compact a non leaf page entry. Simply detect that the entry has a single child,
243 * and update our entry so we can skip it and go directly to the destination.
244 */
Marc-André Lureauefee6782016-09-28 16:37:20 +0400245static void phys_page_compact(PhysPageEntry *lp, Node *nodes)
Michael S. Tsirkinb35ba302013-11-11 17:52:07 +0200246{
247 unsigned valid_ptr = P_L2_SIZE;
248 int valid = 0;
249 PhysPageEntry *p;
250 int i;
251
252 if (lp->ptr == PHYS_MAP_NODE_NIL) {
253 return;
254 }
255
256 p = nodes[lp->ptr];
257 for (i = 0; i < P_L2_SIZE; i++) {
258 if (p[i].ptr == PHYS_MAP_NODE_NIL) {
259 continue;
260 }
261
262 valid_ptr = i;
263 valid++;
264 if (p[i].skip) {
Marc-André Lureauefee6782016-09-28 16:37:20 +0400265 phys_page_compact(&p[i], nodes);
Michael S. Tsirkinb35ba302013-11-11 17:52:07 +0200266 }
267 }
268
269 /* We can only compress if there's only one child. */
270 if (valid != 1) {
271 return;
272 }
273
274 assert(valid_ptr < P_L2_SIZE);
275
276 /* Don't compress if it won't fit in the # of bits we have. */
Wei Yang526ca232019-03-21 16:25:55 +0800277 if (P_L2_LEVELS >= (1 << 6) &&
278 lp->skip + p[valid_ptr].skip >= (1 << 6)) {
Michael S. Tsirkinb35ba302013-11-11 17:52:07 +0200279 return;
280 }
281
282 lp->ptr = p[valid_ptr].ptr;
283 if (!p[valid_ptr].skip) {
284 /* If our only child is a leaf, make this a leaf. */
285 /* By design, we should have made this node a leaf to begin with so we
286 * should never reach here.
287 * But since it's so simple to handle this, let's do it just in case we
288 * change this rule.
289 */
290 lp->skip = 0;
291 } else {
292 lp->skip += p[valid_ptr].skip;
293 }
294}
295
Alexey Kardashevskiy8629d3f2017-09-21 18:51:00 +1000296void address_space_dispatch_compact(AddressSpaceDispatch *d)
Michael S. Tsirkinb35ba302013-11-11 17:52:07 +0200297{
Michael S. Tsirkinb35ba302013-11-11 17:52:07 +0200298 if (d->phys_map.skip) {
Marc-André Lureauefee6782016-09-28 16:37:20 +0400299 phys_page_compact(&d->phys_map, d->map.nodes);
Michael S. Tsirkinb35ba302013-11-11 17:52:07 +0200300 }
301}
302
Fam Zheng29cb5332016-03-01 14:18:23 +0800303static inline bool section_covers_addr(const MemoryRegionSection *section,
304 hwaddr addr)
305{
306 /* Memory topology clips a memory region to [0, 2^64); size.hi > 0 means
307 * the section must cover the entire address space.
308 */
Richard Henderson258dfaa2016-06-29 15:48:03 -0700309 return int128_gethi(section->size) ||
Fam Zheng29cb5332016-03-01 14:18:23 +0800310 range_covers_byte(section->offset_within_address_space,
Richard Henderson258dfaa2016-06-29 15:48:03 -0700311 int128_getlo(section->size), addr);
Fam Zheng29cb5332016-03-01 14:18:23 +0800312}
313
Peter Xu003a0cf2017-05-15 16:50:57 +0800314static MemoryRegionSection *phys_page_find(AddressSpaceDispatch *d, hwaddr addr)
bellard92e873b2004-05-21 14:52:29 +0000315{
Peter Xu003a0cf2017-05-15 16:50:57 +0800316 PhysPageEntry lp = d->phys_map, *p;
317 Node *nodes = d->map.nodes;
318 MemoryRegionSection *sections = d->map.sections;
Michael S. Tsirkin97115a82013-11-13 20:08:19 +0200319 hwaddr index = addr >> TARGET_PAGE_BITS;
Avi Kivity31ab2b42012-02-13 16:44:19 +0200320 int i;
Avi Kivityf1f6e3b2011-11-20 17:52:22 +0200321
Michael S. Tsirkin9736e552013-11-11 14:42:43 +0200322 for (i = P_L2_LEVELS; lp.skip && (i -= lp.skip) >= 0;) {
Avi Kivityc19e8802012-02-13 20:25:31 +0200323 if (lp.ptr == PHYS_MAP_NODE_NIL) {
Paolo Bonzini9affd6f2013-05-29 12:09:47 +0200324 return &sections[PHYS_SECTION_UNASSIGNED];
Avi Kivity31ab2b42012-02-13 16:44:19 +0200325 }
Paolo Bonzini9affd6f2013-05-29 12:09:47 +0200326 p = nodes[lp.ptr];
Paolo Bonzini03f49952013-11-07 17:14:36 +0100327 lp = p[(index >> (i * P_L2_BITS)) & (P_L2_SIZE - 1)];
Avi Kivityf1f6e3b2011-11-20 17:52:22 +0200328 }
Michael S. Tsirkinb35ba302013-11-11 17:52:07 +0200329
Fam Zheng29cb5332016-03-01 14:18:23 +0800330 if (section_covers_addr(&sections[lp.ptr], addr)) {
Michael S. Tsirkinb35ba302013-11-11 17:52:07 +0200331 return &sections[lp.ptr];
332 } else {
333 return &sections[PHYS_SECTION_UNASSIGNED];
334 }
Avi Kivityf3705d52012-03-08 16:16:34 +0200335}
336
Paolo Bonzini79e2b9a2015-01-21 12:09:14 +0100337/* Called from RCU critical section */
Paolo Bonzinic7086b42013-06-02 15:27:39 +0200338static MemoryRegionSection *address_space_lookup_region(AddressSpaceDispatch *d,
Jan Kiszka90260c62013-05-26 21:46:51 +0200339 hwaddr addr,
340 bool resolve_subpage)
Jan Kiszka9f029602013-05-06 16:48:02 +0200341{
Stefan Hajnoczid73415a2020-09-23 11:56:46 +0100342 MemoryRegionSection *section = qatomic_read(&d->mru_section);
Jan Kiszka90260c62013-05-26 21:46:51 +0200343 subpage_t *subpage;
344
Paolo Bonzini07c114b2017-11-15 15:11:03 +0100345 if (!section || section == &d->map.sections[PHYS_SECTION_UNASSIGNED] ||
346 !section_covers_addr(section, addr)) {
Peter Xu003a0cf2017-05-15 16:50:57 +0800347 section = phys_page_find(d, addr);
Stefan Hajnoczid73415a2020-09-23 11:56:46 +0100348 qatomic_set(&d->mru_section, section);
Fam Zheng729633c2016-03-01 14:18:24 +0800349 }
Jan Kiszka90260c62013-05-26 21:46:51 +0200350 if (resolve_subpage && section->mr->subpage) {
351 subpage = container_of(section->mr, subpage_t, iomem);
Marcel Apfelbaum53cb28c2013-12-01 14:02:23 +0200352 section = &d->map.sections[subpage->sub_section[SUBPAGE_IDX(addr)]];
Jan Kiszka90260c62013-05-26 21:46:51 +0200353 }
354 return section;
Jan Kiszka9f029602013-05-06 16:48:02 +0200355}
356
Paolo Bonzini79e2b9a2015-01-21 12:09:14 +0100357/* Called from RCU critical section */
Jan Kiszka90260c62013-05-26 21:46:51 +0200358static MemoryRegionSection *
Paolo Bonzinic7086b42013-06-02 15:27:39 +0200359address_space_translate_internal(AddressSpaceDispatch *d, hwaddr addr, hwaddr *xlat,
Jan Kiszka90260c62013-05-26 21:46:51 +0200360 hwaddr *plen, bool resolve_subpage)
Paolo Bonzini149f54b2013-05-24 12:59:37 +0200361{
362 MemoryRegionSection *section;
Paolo Bonzini965eb2f2015-06-17 10:40:27 +0200363 MemoryRegion *mr;
Paolo Bonzinia87f3952014-02-07 15:47:46 +0100364 Int128 diff;
Paolo Bonzini149f54b2013-05-24 12:59:37 +0200365
Paolo Bonzinic7086b42013-06-02 15:27:39 +0200366 section = address_space_lookup_region(d, addr, resolve_subpage);
Paolo Bonzini149f54b2013-05-24 12:59:37 +0200367 /* Compute offset within MemoryRegionSection */
368 addr -= section->offset_within_address_space;
369
370 /* Compute offset within MemoryRegion */
371 *xlat = addr + section->offset_within_region;
372
Paolo Bonzini965eb2f2015-06-17 10:40:27 +0200373 mr = section->mr;
Paolo Bonzinib242e0e2015-07-04 00:24:51 +0200374
375 /* MMIO registers can be expected to perform full-width accesses based only
376 * on their address, without considering adjacent registers that could
377 * decode to completely different MemoryRegions. When such registers
378 * exist (e.g. I/O ports 0xcf8 and 0xcf9 on most PC chipsets), MMIO
379 * regions overlap wildly. For this reason we cannot clamp the accesses
380 * here.
381 *
382 * If the length is small (as is the case for address_space_ldl/stl),
383 * everything works fine. If the incoming length is large, however,
384 * the caller really has to do the clamping through memory_access_size.
385 */
Paolo Bonzini965eb2f2015-06-17 10:40:27 +0200386 if (memory_region_is_ram(mr)) {
Paolo Bonzinie4a511f2015-06-17 10:36:54 +0200387 diff = int128_sub(section->size, int128_make64(addr));
Paolo Bonzini965eb2f2015-06-17 10:40:27 +0200388 *plen = int128_get64(int128_min(diff, int128_make64(*plen)));
389 }
Paolo Bonzini149f54b2013-05-24 12:59:37 +0200390 return section;
391}
Jan Kiszka90260c62013-05-26 21:46:51 +0200392
Peter Xud5e5faf2017-10-10 11:42:45 +0200393/**
Paolo Bonzinia411c842018-03-03 17:24:04 +0100394 * address_space_translate_iommu - translate an address through an IOMMU
395 * memory region and then through the target address space.
396 *
397 * @iommu_mr: the IOMMU memory region that we start the translation from
398 * @addr: the address to be translated through the MMU
399 * @xlat: the translated address offset within the destination memory region.
400 * It cannot be %NULL.
401 * @plen_out: valid read/write length of the translated address. It
402 * cannot be %NULL.
403 * @page_mask_out: page mask for the translated address. This
404 * should only be meaningful for IOMMU translated
405 * addresses, since there may be huge pages that this bit
406 * would tell. It can be %NULL if we don't care about it.
407 * @is_write: whether the translation operation is for write
408 * @is_mmio: whether this can be MMIO, set true if it can
409 * @target_as: the address space targeted by the IOMMU
Peter Maydell2f7b0092018-05-31 14:50:53 +0100410 * @attrs: transaction attributes
Paolo Bonzinia411c842018-03-03 17:24:04 +0100411 *
412 * This function is called from RCU critical section. It is the common
413 * part of flatview_do_translate and address_space_translate_cached.
414 */
415static MemoryRegionSection address_space_translate_iommu(IOMMUMemoryRegion *iommu_mr,
416 hwaddr *xlat,
417 hwaddr *plen_out,
418 hwaddr *page_mask_out,
419 bool is_write,
420 bool is_mmio,
Peter Maydell2f7b0092018-05-31 14:50:53 +0100421 AddressSpace **target_as,
422 MemTxAttrs attrs)
Paolo Bonzinia411c842018-03-03 17:24:04 +0100423{
424 MemoryRegionSection *section;
425 hwaddr page_mask = (hwaddr)-1;
426
427 do {
428 hwaddr addr = *xlat;
429 IOMMUMemoryRegionClass *imrc = memory_region_get_iommu_class_nocheck(iommu_mr);
Peter Maydell2c91bcf2018-06-15 14:57:16 +0100430 int iommu_idx = 0;
431 IOMMUTLBEntry iotlb;
432
433 if (imrc->attrs_to_index) {
434 iommu_idx = imrc->attrs_to_index(iommu_mr, attrs);
435 }
436
437 iotlb = imrc->translate(iommu_mr, addr, is_write ?
438 IOMMU_WO : IOMMU_RO, iommu_idx);
Paolo Bonzinia411c842018-03-03 17:24:04 +0100439
440 if (!(iotlb.perm & (1 << is_write))) {
441 goto unassigned;
442 }
443
444 addr = ((iotlb.translated_addr & ~iotlb.addr_mask)
445 | (addr & iotlb.addr_mask));
446 page_mask &= iotlb.addr_mask;
447 *plen_out = MIN(*plen_out, (addr | iotlb.addr_mask) - addr + 1);
448 *target_as = iotlb.target_as;
449
450 section = address_space_translate_internal(
451 address_space_to_dispatch(iotlb.target_as), addr, xlat,
452 plen_out, is_mmio);
453
454 iommu_mr = memory_region_get_iommu(section->mr);
455 } while (unlikely(iommu_mr));
456
457 if (page_mask_out) {
458 *page_mask_out = page_mask;
459 }
460 return *section;
461
462unassigned:
463 return (MemoryRegionSection) { .mr = &io_mem_unassigned };
464}
465
466/**
Peter Xud5e5faf2017-10-10 11:42:45 +0200467 * flatview_do_translate - translate an address in FlatView
468 *
469 * @fv: the flat view that we want to translate on
470 * @addr: the address to be translated in above address space
471 * @xlat: the translated address offset within memory region. It
472 * cannot be @NULL.
473 * @plen_out: valid read/write length of the translated address. It
474 * can be @NULL when we don't care about it.
475 * @page_mask_out: page mask for the translated address. This
476 * should only be meaningful for IOMMU translated
477 * addresses, since there may be huge pages that this bit
478 * would tell. It can be @NULL if we don't care about it.
479 * @is_write: whether the translation operation is for write
480 * @is_mmio: whether this can be MMIO, set true if it can
Paolo Bonziniad2804d2018-04-17 11:39:35 +0200481 * @target_as: the address space targeted by the IOMMU
Peter Maydell49e14aa2018-05-31 14:50:53 +0100482 * @attrs: memory transaction attributes
Peter Xud5e5faf2017-10-10 11:42:45 +0200483 *
484 * This function is called from RCU critical section
485 */
Alexey Kardashevskiy16620682017-09-21 18:50:58 +1000486static MemoryRegionSection flatview_do_translate(FlatView *fv,
487 hwaddr addr,
488 hwaddr *xlat,
Peter Xud5e5faf2017-10-10 11:42:45 +0200489 hwaddr *plen_out,
490 hwaddr *page_mask_out,
Alexey Kardashevskiy16620682017-09-21 18:50:58 +1000491 bool is_write,
492 bool is_mmio,
Peter Maydell49e14aa2018-05-31 14:50:53 +0100493 AddressSpace **target_as,
494 MemTxAttrs attrs)
Jan Kiszka90260c62013-05-26 21:46:51 +0200495{
Avi Kivity30951152012-10-30 13:47:46 +0200496 MemoryRegionSection *section;
Alexey Kardashevskiy3df9d742017-07-11 13:56:19 +1000497 IOMMUMemoryRegion *iommu_mr;
Peter Xud5e5faf2017-10-10 11:42:45 +0200498 hwaddr plen = (hwaddr)(-1);
499
Paolo Bonziniad2804d2018-04-17 11:39:35 +0200500 if (!plen_out) {
501 plen_out = &plen;
Peter Xud5e5faf2017-10-10 11:42:45 +0200502 }
Avi Kivity30951152012-10-30 13:47:46 +0200503
Paolo Bonzinia411c842018-03-03 17:24:04 +0100504 section = address_space_translate_internal(
505 flatview_to_dispatch(fv), addr, xlat,
506 plen_out, is_mmio);
Avi Kivity30951152012-10-30 13:47:46 +0200507
Paolo Bonzinia411c842018-03-03 17:24:04 +0100508 iommu_mr = memory_region_get_iommu(section->mr);
509 if (unlikely(iommu_mr)) {
510 return address_space_translate_iommu(iommu_mr, xlat,
511 plen_out, page_mask_out,
512 is_write, is_mmio,
Peter Maydell2f7b0092018-05-31 14:50:53 +0100513 target_as, attrs);
Avi Kivity30951152012-10-30 13:47:46 +0200514 }
Peter Xud5e5faf2017-10-10 11:42:45 +0200515 if (page_mask_out) {
Paolo Bonzinia411c842018-03-03 17:24:04 +0100516 /* Not behind an IOMMU, use default page size. */
517 *page_mask_out = ~TARGET_PAGE_MASK;
Peter Xud5e5faf2017-10-10 11:42:45 +0200518 }
519
Peter Xua7640402017-05-17 16:57:42 +0800520 return *section;
Peter Xua7640402017-05-17 16:57:42 +0800521}
522
523/* Called from RCU critical section */
524IOMMUTLBEntry address_space_get_iotlb_entry(AddressSpace *as, hwaddr addr,
Peter Maydell7446eb02018-05-31 14:50:53 +0100525 bool is_write, MemTxAttrs attrs)
Peter Xua7640402017-05-17 16:57:42 +0800526{
527 MemoryRegionSection section;
Peter Xu076a93d2017-10-10 11:42:46 +0200528 hwaddr xlat, page_mask;
Peter Xua7640402017-05-17 16:57:42 +0800529
Peter Xu076a93d2017-10-10 11:42:46 +0200530 /*
531 * This can never be MMIO, and we don't really care about plen,
532 * but page mask.
533 */
534 section = flatview_do_translate(address_space_to_flatview(as), addr, &xlat,
Peter Maydell49e14aa2018-05-31 14:50:53 +0100535 NULL, &page_mask, is_write, false, &as,
536 attrs);
Peter Xua7640402017-05-17 16:57:42 +0800537
538 /* Illegal translation */
539 if (section.mr == &io_mem_unassigned) {
540 goto iotlb_fail;
541 }
542
543 /* Convert memory region offset into address space offset */
544 xlat += section.offset_within_address_space -
545 section.offset_within_region;
546
Peter Xua7640402017-05-17 16:57:42 +0800547 return (IOMMUTLBEntry) {
Alexey Kardashevskiye76bb182017-09-21 18:50:53 +1000548 .target_as = as,
Peter Xu076a93d2017-10-10 11:42:46 +0200549 .iova = addr & ~page_mask,
550 .translated_addr = xlat & ~page_mask,
551 .addr_mask = page_mask,
Peter Xua7640402017-05-17 16:57:42 +0800552 /* IOTLBs are for DMAs, and DMA only allows on RAMs. */
553 .perm = IOMMU_RW,
554 };
555
556iotlb_fail:
557 return (IOMMUTLBEntry) {0};
558}
559
560/* Called from RCU critical section */
Alexey Kardashevskiy16620682017-09-21 18:50:58 +1000561MemoryRegion *flatview_translate(FlatView *fv, hwaddr addr, hwaddr *xlat,
Peter Maydellefa99a22018-05-31 14:50:52 +0100562 hwaddr *plen, bool is_write,
563 MemTxAttrs attrs)
Peter Xua7640402017-05-17 16:57:42 +0800564{
565 MemoryRegion *mr;
566 MemoryRegionSection section;
Alexey Kardashevskiy16620682017-09-21 18:50:58 +1000567 AddressSpace *as = NULL;
Peter Xua7640402017-05-17 16:57:42 +0800568
569 /* This can be MMIO, so setup MMIO bit. */
Peter Xud5e5faf2017-10-10 11:42:45 +0200570 section = flatview_do_translate(fv, addr, xlat, plen, NULL,
Peter Maydell49e14aa2018-05-31 14:50:53 +0100571 is_write, true, &as, attrs);
Peter Xua7640402017-05-17 16:57:42 +0800572 mr = section.mr;
573
Alexey Kardashevskiyfe680d02014-05-07 13:40:39 +0000574 if (xen_enabled() && memory_access_is_direct(mr, is_write)) {
Paolo Bonzinia87f3952014-02-07 15:47:46 +0100575 hwaddr page = ((addr & TARGET_PAGE_MASK) + TARGET_PAGE_SIZE) - addr;
Peter Crosthwaite23820db2015-03-16 22:35:54 -0700576 *plen = MIN(page, *plen);
Paolo Bonzinia87f3952014-02-07 15:47:46 +0100577 }
578
Avi Kivity30951152012-10-30 13:47:46 +0200579 return mr;
Jan Kiszka90260c62013-05-26 21:46:51 +0200580}
581
Peter Maydell1f871c52018-06-15 14:57:16 +0100582typedef struct TCGIOMMUNotifier {
583 IOMMUNotifier n;
584 MemoryRegion *mr;
585 CPUState *cpu;
586 int iommu_idx;
587 bool active;
588} TCGIOMMUNotifier;
589
590static void tcg_iommu_unmap_notify(IOMMUNotifier *n, IOMMUTLBEntry *iotlb)
591{
592 TCGIOMMUNotifier *notifier = container_of(n, TCGIOMMUNotifier, n);
593
594 if (!notifier->active) {
595 return;
596 }
597 tlb_flush(notifier->cpu);
598 notifier->active = false;
599 /* We leave the notifier struct on the list to avoid reallocating it later.
600 * Generally the number of IOMMUs a CPU deals with will be small.
601 * In any case we can't unregister the iommu notifier from a notify
602 * callback.
603 */
604}
605
606static void tcg_register_iommu_notifier(CPUState *cpu,
607 IOMMUMemoryRegion *iommu_mr,
608 int iommu_idx)
609{
610 /* Make sure this CPU has an IOMMU notifier registered for this
611 * IOMMU/IOMMU index combination, so that we can flush its TLB
612 * when the IOMMU tells us the mappings we've cached have changed.
613 */
614 MemoryRegion *mr = MEMORY_REGION(iommu_mr);
Philippe Mathieu-Daudébbf90192021-01-17 18:04:11 +0100615 TCGIOMMUNotifier *notifier = NULL;
Markus Armbruster805d4492020-07-22 10:40:48 +0200616 int i;
Peter Maydell1f871c52018-06-15 14:57:16 +0100617
618 for (i = 0; i < cpu->iommu_notifiers->len; i++) {
Peter Maydell5601be32019-02-01 14:55:45 +0000619 notifier = g_array_index(cpu->iommu_notifiers, TCGIOMMUNotifier *, i);
Peter Maydell1f871c52018-06-15 14:57:16 +0100620 if (notifier->mr == mr && notifier->iommu_idx == iommu_idx) {
621 break;
622 }
623 }
624 if (i == cpu->iommu_notifiers->len) {
625 /* Not found, add a new entry at the end of the array */
626 cpu->iommu_notifiers = g_array_set_size(cpu->iommu_notifiers, i + 1);
Peter Maydell5601be32019-02-01 14:55:45 +0000627 notifier = g_new0(TCGIOMMUNotifier, 1);
628 g_array_index(cpu->iommu_notifiers, TCGIOMMUNotifier *, i) = notifier;
Peter Maydell1f871c52018-06-15 14:57:16 +0100629
630 notifier->mr = mr;
631 notifier->iommu_idx = iommu_idx;
632 notifier->cpu = cpu;
633 /* Rather than trying to register interest in the specific part
634 * of the iommu's address space that we've accessed and then
635 * expand it later as subsequent accesses touch more of it, we
636 * just register interest in the whole thing, on the assumption
637 * that iommu reconfiguration will be rare.
638 */
639 iommu_notifier_init(&notifier->n,
640 tcg_iommu_unmap_notify,
641 IOMMU_NOTIFIER_UNMAP,
642 0,
643 HWADDR_MAX,
644 iommu_idx);
Markus Armbruster805d4492020-07-22 10:40:48 +0200645 memory_region_register_iommu_notifier(notifier->mr, &notifier->n,
646 &error_fatal);
Peter Maydell1f871c52018-06-15 14:57:16 +0100647 }
648
649 if (!notifier->active) {
650 notifier->active = true;
651 }
652}
653
Paolo Bonzinid9f24bf2020-10-06 09:05:29 +0200654void tcg_iommu_free_notifier_list(CPUState *cpu)
Peter Maydell1f871c52018-06-15 14:57:16 +0100655{
656 /* Destroy the CPU's notifier list */
657 int i;
658 TCGIOMMUNotifier *notifier;
659
660 for (i = 0; i < cpu->iommu_notifiers->len; i++) {
Peter Maydell5601be32019-02-01 14:55:45 +0000661 notifier = g_array_index(cpu->iommu_notifiers, TCGIOMMUNotifier *, i);
Peter Maydell1f871c52018-06-15 14:57:16 +0100662 memory_region_unregister_iommu_notifier(notifier->mr, &notifier->n);
Peter Maydell5601be32019-02-01 14:55:45 +0000663 g_free(notifier);
Peter Maydell1f871c52018-06-15 14:57:16 +0100664 }
665 g_array_free(cpu->iommu_notifiers, true);
666}
667
Paolo Bonzinid9f24bf2020-10-06 09:05:29 +0200668void tcg_iommu_init_notifier_list(CPUState *cpu)
669{
670 cpu->iommu_notifiers = g_array_new(false, true, sizeof(TCGIOMMUNotifier *));
671}
672
Paolo Bonzini79e2b9a2015-01-21 12:09:14 +0100673/* Called from RCU critical section */
Jan Kiszka90260c62013-05-26 21:46:51 +0200674MemoryRegionSection *
Richard Henderson418ade72022-06-21 08:38:29 -0700675address_space_translate_for_iotlb(CPUState *cpu, int asidx, hwaddr orig_addr,
Peter Maydell1f871c52018-06-15 14:57:16 +0100676 hwaddr *xlat, hwaddr *plen,
677 MemTxAttrs attrs, int *prot)
Jan Kiszka90260c62013-05-26 21:46:51 +0200678{
Avi Kivity30951152012-10-30 13:47:46 +0200679 MemoryRegionSection *section;
Peter Maydell1f871c52018-06-15 14:57:16 +0100680 IOMMUMemoryRegion *iommu_mr;
681 IOMMUMemoryRegionClass *imrc;
682 IOMMUTLBEntry iotlb;
683 int iommu_idx;
Richard Henderson418ade72022-06-21 08:38:29 -0700684 hwaddr addr = orig_addr;
Richard Henderson0d58c662023-08-25 16:13:17 -0700685 AddressSpaceDispatch *d = cpu->cpu_ases[asidx].memory_dispatch;
Peter Maydelld7898cd2016-01-21 14:15:05 +0000686
Peter Maydell1f871c52018-06-15 14:57:16 +0100687 for (;;) {
688 section = address_space_translate_internal(d, addr, &addr, plen, false);
689
690 iommu_mr = memory_region_get_iommu(section->mr);
691 if (!iommu_mr) {
692 break;
693 }
694
695 imrc = memory_region_get_iommu_class_nocheck(iommu_mr);
696
697 iommu_idx = imrc->attrs_to_index(iommu_mr, attrs);
698 tcg_register_iommu_notifier(cpu, iommu_mr, iommu_idx);
699 /* We need all the permissions, so pass IOMMU_NONE so the IOMMU
700 * doesn't short-cut its translation table walk.
701 */
702 iotlb = imrc->translate(iommu_mr, addr, IOMMU_NONE, iommu_idx);
703 addr = ((iotlb.translated_addr & ~iotlb.addr_mask)
704 | (addr & iotlb.addr_mask));
705 /* Update the caller's prot bits to remove permissions the IOMMU
706 * is giving us a failure response for. If we get down to no
707 * permissions left at all we can give up now.
708 */
709 if (!(iotlb.perm & IOMMU_RO)) {
710 *prot &= ~(PAGE_READ | PAGE_EXEC);
711 }
712 if (!(iotlb.perm & IOMMU_WO)) {
713 *prot &= ~PAGE_WRITE;
714 }
715
716 if (!*prot) {
717 goto translate_fail;
718 }
719
720 d = flatview_to_dispatch(address_space_to_flatview(iotlb.target_as));
721 }
Avi Kivity30951152012-10-30 13:47:46 +0200722
Alexey Kardashevskiy3df9d742017-07-11 13:56:19 +1000723 assert(!memory_region_is_iommu(section->mr));
Peter Maydell1f871c52018-06-15 14:57:16 +0100724 *xlat = addr;
Avi Kivity30951152012-10-30 13:47:46 +0200725 return section;
Peter Maydell1f871c52018-06-15 14:57:16 +0100726
727translate_fail:
Richard Henderson418ade72022-06-21 08:38:29 -0700728 /*
729 * We should be given a page-aligned address -- certainly
730 * tlb_set_page_with_attrs() does so. The page offset of xlat
731 * is used to index sections[], and PHYS_SECTION_UNASSIGNED = 0.
732 * The page portion of xlat will be logged by memory_region_access_valid()
733 * when this memory access is rejected, so use the original untranslated
734 * physical address.
735 */
736 assert((orig_addr & ~TARGET_PAGE_MASK) == 0);
737 *xlat = orig_addr;
Peter Maydell1f871c52018-06-15 14:57:16 +0100738 return &d->map.sections[PHYS_SECTION_UNASSIGNED];
Jan Kiszka90260c62013-05-26 21:46:51 +0200739}
Andreas Färber1a1562f2013-06-17 04:09:11 +0200740
Peter Xu80ceb072017-11-23 17:23:32 +0800741void cpu_address_space_init(CPUState *cpu, int asidx,
742 const char *prefix, MemoryRegion *mr)
Edgar E. Iglesias09daed82013-12-17 13:06:51 +1000743{
Peter Maydell12ebc9a2016-01-21 14:15:04 +0000744 CPUAddressSpace *newas;
Peter Xu80ceb072017-11-23 17:23:32 +0800745 AddressSpace *as = g_new0(AddressSpace, 1);
Peter Xu87a621d2017-11-23 17:23:33 +0800746 char *as_name;
Peter Xu80ceb072017-11-23 17:23:32 +0800747
748 assert(mr);
Peter Xu87a621d2017-11-23 17:23:33 +0800749 as_name = g_strdup_printf("%s-%d", prefix, cpu->cpu_index);
750 address_space_init(as, mr, as_name);
751 g_free(as_name);
Peter Maydell12ebc9a2016-01-21 14:15:04 +0000752
753 /* Target code should have set num_ases before calling us */
754 assert(asidx < cpu->num_ases);
755
Peter Maydell56943e82016-01-21 14:15:04 +0000756 if (asidx == 0) {
757 /* address space 0 gets the convenience alias */
758 cpu->as = as;
759 }
760
Peter Maydell12ebc9a2016-01-21 14:15:04 +0000761 /* KVM cannot currently support multiple address spaces. */
762 assert(asidx == 0 || !kvm_enabled());
Edgar E. Iglesias09daed82013-12-17 13:06:51 +1000763
Peter Maydell12ebc9a2016-01-21 14:15:04 +0000764 if (!cpu->cpu_ases) {
765 cpu->cpu_ases = g_new0(CPUAddressSpace, cpu->num_ases);
Salil Mehta24bec422024-07-16 12:15:01 +0100766 cpu->cpu_ases_count = cpu->num_ases;
Edgar E. Iglesias09daed82013-12-17 13:06:51 +1000767 }
Peter Maydell32857f42015-10-01 15:29:50 +0100768
Peter Maydell12ebc9a2016-01-21 14:15:04 +0000769 newas = &cpu->cpu_ases[asidx];
770 newas->cpu = cpu;
771 newas->as = as;
Peter Maydell56943e82016-01-21 14:15:04 +0000772 if (tcg_enabled()) {
Paolo Bonzini9458a9a2018-02-06 18:37:39 +0100773 newas->tcg_as_listener.log_global_after_sync = tcg_log_global_after_sync;
Peter Maydell12ebc9a2016-01-21 14:15:04 +0000774 newas->tcg_as_listener.commit = tcg_commit;
Peter Xu142518b2021-08-16 21:35:52 -0400775 newas->tcg_as_listener.name = "tcg";
Peter Maydell12ebc9a2016-01-21 14:15:04 +0000776 memory_listener_register(&newas->tcg_as_listener, as);
Peter Maydell56943e82016-01-21 14:15:04 +0000777 }
Edgar E. Iglesias09daed82013-12-17 13:06:51 +1000778}
Peter Maydell651a5bc2016-01-21 14:15:05 +0000779
Salil Mehta24bec422024-07-16 12:15:01 +0100780void cpu_address_space_destroy(CPUState *cpu, int asidx)
781{
782 CPUAddressSpace *cpuas;
783
784 assert(cpu->cpu_ases);
785 assert(asidx >= 0 && asidx < cpu->num_ases);
786 /* KVM cannot currently support multiple address spaces. */
787 assert(asidx == 0 || !kvm_enabled());
788
789 cpuas = &cpu->cpu_ases[asidx];
790 if (tcg_enabled()) {
791 memory_listener_unregister(&cpuas->tcg_as_listener);
792 }
793
794 address_space_destroy(cpuas->as);
795 g_free_rcu(cpuas->as, rcu);
796
797 if (asidx == 0) {
798 /* reset the convenience alias for address space 0 */
799 cpu->as = NULL;
800 }
801
802 if (--cpu->cpu_ases_count == 0) {
803 g_free(cpu->cpu_ases);
804 cpu->cpu_ases = NULL;
805 }
806}
807
Peter Maydell651a5bc2016-01-21 14:15:05 +0000808AddressSpace *cpu_get_address_space(CPUState *cpu, int asidx)
809{
810 /* Return the AddressSpace corresponding to the specified index */
811 return cpu->cpu_ases[asidx].as;
812}
Edgar E. Iglesias09daed82013-12-17 13:06:51 +1000813
Mike Day0dc3f442013-09-05 14:41:35 -0400814/* Called from RCU critical section */
Paolo Bonzini041603f2013-09-09 17:49:45 +0200815static RAMBlock *qemu_get_ram_block(ram_addr_t addr)
816{
817 RAMBlock *block;
818
Stefan Hajnoczid73415a2020-09-23 11:56:46 +0100819 block = qatomic_rcu_read(&ram_list.mru_block);
Michael S. Tsirkin9b8424d2014-12-15 22:55:32 +0200820 if (block && addr - block->offset < block->max_length) {
Paolo Bonzini68851b92015-10-22 13:51:30 +0200821 return block;
Paolo Bonzini041603f2013-09-09 17:49:45 +0200822 }
Peter Xu99e15582017-05-12 12:17:39 +0800823 RAMBLOCK_FOREACH(block) {
Michael S. Tsirkin9b8424d2014-12-15 22:55:32 +0200824 if (addr - block->offset < block->max_length) {
Paolo Bonzini041603f2013-09-09 17:49:45 +0200825 goto found;
826 }
827 }
828
829 fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
830 abort();
831
832found:
Stefan Hajnoczia4a411f2024-01-02 10:35:28 -0500833 /* It is safe to write mru_block outside the BQL. This
Paolo Bonzini43771532013-09-09 17:58:40 +0200834 * is what happens:
835 *
836 * mru_block = xxx
837 * rcu_read_unlock()
838 * xxx removed from list
839 * rcu_read_lock()
840 * read mru_block
841 * mru_block = NULL;
842 * call_rcu(reclaim_ramblock, xxx);
843 * rcu_read_unlock()
844 *
Stefan Hajnoczid73415a2020-09-23 11:56:46 +0100845 * qatomic_rcu_set is not needed here. The block was already published
Paolo Bonzini43771532013-09-09 17:58:40 +0200846 * when it was placed into the list. Here we're just making an extra
847 * copy of the pointer.
848 */
Paolo Bonzini041603f2013-09-09 17:49:45 +0200849 ram_list.mru_block = block;
850 return block;
851}
852
Philippe Mathieu-Daudé7e8ccf92024-03-12 21:14:56 +0100853void tlb_reset_dirty_range_all(ram_addr_t start, ram_addr_t length)
bellard1ccde1c2004-02-06 19:46:14 +0000854{
Peter Crosthwaite9a135652015-09-10 22:39:41 -0700855 CPUState *cpu;
Paolo Bonzini041603f2013-09-09 17:49:45 +0200856 ram_addr_t start1;
Juan Quintelaa2f4d5b2013-10-10 11:49:53 +0200857 RAMBlock *block;
858 ram_addr_t end;
859
Emilio G. Cotaf28d0df2018-06-22 13:45:31 -0400860 assert(tcg_enabled());
Juan Quintelaa2f4d5b2013-10-10 11:49:53 +0200861 end = TARGET_PAGE_ALIGN(start + length);
862 start &= TARGET_PAGE_MASK;
bellardf23db162005-08-21 19:12:28 +0000863
Dr. David Alan Gilbert694ea272019-10-07 15:36:41 +0100864 RCU_READ_LOCK_GUARD();
Paolo Bonzini041603f2013-09-09 17:49:45 +0200865 block = qemu_get_ram_block(start);
866 assert(block == qemu_get_ram_block(end - 1));
Michael S. Tsirkin1240be22014-11-12 11:44:41 +0200867 start1 = (uintptr_t)ramblock_ptr(block, start - block->offset);
Peter Crosthwaite9a135652015-09-10 22:39:41 -0700868 CPU_FOREACH(cpu) {
869 tlb_reset_dirty(cpu, start1, length);
870 }
Juan Quintelad24981d2012-05-22 00:42:40 +0200871}
872
873/* Note: start and end must be within the same ram block. */
Stefan Hajnoczi03eebc92014-12-02 11:23:18 +0000874bool cpu_physical_memory_test_and_clear_dirty(ram_addr_t start,
875 ram_addr_t length,
876 unsigned client)
Juan Quintelad24981d2012-05-22 00:42:40 +0200877{
Stefan Hajnoczi5b82b702016-01-25 13:33:20 +0000878 DirtyMemoryBlocks *blocks;
Matt Borgerson25aa6b32020-02-18 03:19:10 -0700879 unsigned long end, page, start_page;
Stefan Hajnoczi5b82b702016-01-25 13:33:20 +0000880 bool dirty = false;
Peter Xu077874e2019-06-03 14:50:51 +0800881 RAMBlock *ramblock;
882 uint64_t mr_offset, mr_size;
Juan Quintelad24981d2012-05-22 00:42:40 +0200883
Stefan Hajnoczi03eebc92014-12-02 11:23:18 +0000884 if (length == 0) {
885 return false;
886 }
887
888 end = TARGET_PAGE_ALIGN(start + length) >> TARGET_PAGE_BITS;
Matt Borgerson25aa6b32020-02-18 03:19:10 -0700889 start_page = start >> TARGET_PAGE_BITS;
890 page = start_page;
Stefan Hajnoczi5b82b702016-01-25 13:33:20 +0000891
Dr. David Alan Gilbert694ea272019-10-07 15:36:41 +0100892 WITH_RCU_READ_LOCK_GUARD() {
Stefan Hajnoczid73415a2020-09-23 11:56:46 +0100893 blocks = qatomic_rcu_read(&ram_list.dirty_memory[client]);
Dr. David Alan Gilbert694ea272019-10-07 15:36:41 +0100894 ramblock = qemu_get_ram_block(start);
895 /* Range sanity check on the ramblock */
896 assert(start >= ramblock->offset &&
897 start + length <= ramblock->offset + ramblock->used_length);
Stefan Hajnoczi5b82b702016-01-25 13:33:20 +0000898
Dr. David Alan Gilbert694ea272019-10-07 15:36:41 +0100899 while (page < end) {
900 unsigned long idx = page / DIRTY_MEMORY_BLOCK_SIZE;
901 unsigned long offset = page % DIRTY_MEMORY_BLOCK_SIZE;
902 unsigned long num = MIN(end - page,
903 DIRTY_MEMORY_BLOCK_SIZE - offset);
Stefan Hajnoczi5b82b702016-01-25 13:33:20 +0000904
Dr. David Alan Gilbert694ea272019-10-07 15:36:41 +0100905 dirty |= bitmap_test_and_clear_atomic(blocks->blocks[idx],
906 offset, num);
907 page += num;
908 }
Stefan Hajnoczi5b82b702016-01-25 13:33:20 +0000909
Matt Borgerson25aa6b32020-02-18 03:19:10 -0700910 mr_offset = (ram_addr_t)(start_page << TARGET_PAGE_BITS) - ramblock->offset;
911 mr_size = (end - start_page) << TARGET_PAGE_BITS;
Dr. David Alan Gilbert694ea272019-10-07 15:36:41 +0100912 memory_region_clear_dirty_bitmap(ramblock->mr, mr_offset, mr_size);
Stefan Hajnoczi5b82b702016-01-25 13:33:20 +0000913 }
914
Nicholas Piggin86a9ae82024-03-12 21:14:57 +0100915 if (dirty) {
916 cpu_physical_memory_dirty_bits_cleared(start, length);
Juan Quintelad24981d2012-05-22 00:42:40 +0200917 }
Stefan Hajnoczi03eebc92014-12-02 11:23:18 +0000918
919 return dirty;
bellard1ccde1c2004-02-06 19:46:14 +0000920}
921
Gerd Hoffmann8deaf122017-04-21 11:16:25 +0200922DirtyBitmapSnapshot *cpu_physical_memory_snapshot_and_clear_dirty
Peter Xu5dea4072019-06-03 14:50:50 +0800923 (MemoryRegion *mr, hwaddr offset, hwaddr length, unsigned client)
Gerd Hoffmann8deaf122017-04-21 11:16:25 +0200924{
925 DirtyMemoryBlocks *blocks;
Peter Maydell73188062024-07-23 18:05:13 +0100926 ram_addr_t start, first, last;
Gerd Hoffmann8deaf122017-04-21 11:16:25 +0200927 unsigned long align = 1UL << (TARGET_PAGE_BITS + BITS_PER_LEVEL);
Gerd Hoffmann8deaf122017-04-21 11:16:25 +0200928 DirtyBitmapSnapshot *snap;
929 unsigned long page, end, dest;
930
Peter Maydell73188062024-07-23 18:05:13 +0100931 start = memory_region_get_ram_addr(mr);
932 /* We know we're only called for RAM MemoryRegions */
933 assert(start != RAM_ADDR_INVALID);
934 start += offset;
935
936 first = QEMU_ALIGN_DOWN(start, align);
937 last = QEMU_ALIGN_UP(start + length, align);
938
Gerd Hoffmann8deaf122017-04-21 11:16:25 +0200939 snap = g_malloc0(sizeof(*snap) +
940 ((last - first) >> (TARGET_PAGE_BITS + 3)));
941 snap->start = first;
942 snap->end = last;
943
944 page = first >> TARGET_PAGE_BITS;
945 end = last >> TARGET_PAGE_BITS;
946 dest = 0;
947
Dr. David Alan Gilbert694ea272019-10-07 15:36:41 +0100948 WITH_RCU_READ_LOCK_GUARD() {
Stefan Hajnoczid73415a2020-09-23 11:56:46 +0100949 blocks = qatomic_rcu_read(&ram_list.dirty_memory[client]);
Gerd Hoffmann8deaf122017-04-21 11:16:25 +0200950
Dr. David Alan Gilbert694ea272019-10-07 15:36:41 +0100951 while (page < end) {
952 unsigned long idx = page / DIRTY_MEMORY_BLOCK_SIZE;
Philippe Mathieu-Daudé6ba9b602023-09-04 18:12:34 +0200953 unsigned long ofs = page % DIRTY_MEMORY_BLOCK_SIZE;
Dr. David Alan Gilbert694ea272019-10-07 15:36:41 +0100954 unsigned long num = MIN(end - page,
Philippe Mathieu-Daudé6ba9b602023-09-04 18:12:34 +0200955 DIRTY_MEMORY_BLOCK_SIZE - ofs);
Gerd Hoffmann8deaf122017-04-21 11:16:25 +0200956
Philippe Mathieu-Daudé6ba9b602023-09-04 18:12:34 +0200957 assert(QEMU_IS_ALIGNED(ofs, (1 << BITS_PER_LEVEL)));
Dr. David Alan Gilbert694ea272019-10-07 15:36:41 +0100958 assert(QEMU_IS_ALIGNED(num, (1 << BITS_PER_LEVEL)));
Philippe Mathieu-Daudé6ba9b602023-09-04 18:12:34 +0200959 ofs >>= BITS_PER_LEVEL;
Gerd Hoffmann8deaf122017-04-21 11:16:25 +0200960
Dr. David Alan Gilbert694ea272019-10-07 15:36:41 +0100961 bitmap_copy_and_clear_atomic(snap->dirty + dest,
Philippe Mathieu-Daudé6ba9b602023-09-04 18:12:34 +0200962 blocks->blocks[idx] + ofs,
Dr. David Alan Gilbert694ea272019-10-07 15:36:41 +0100963 num);
964 page += num;
965 dest += num >> BITS_PER_LEVEL;
966 }
Gerd Hoffmann8deaf122017-04-21 11:16:25 +0200967 }
968
Nicholas Piggin86a9ae82024-03-12 21:14:57 +0100969 cpu_physical_memory_dirty_bits_cleared(start, length);
Gerd Hoffmann8deaf122017-04-21 11:16:25 +0200970
Peter Xu077874e2019-06-03 14:50:51 +0800971 memory_region_clear_dirty_bitmap(mr, offset, length);
972
Gerd Hoffmann8deaf122017-04-21 11:16:25 +0200973 return snap;
974}
975
976bool cpu_physical_memory_snapshot_get_dirty(DirtyBitmapSnapshot *snap,
977 ram_addr_t start,
978 ram_addr_t length)
979{
980 unsigned long page, end;
981
982 assert(start >= snap->start);
983 assert(start + length <= snap->end);
984
985 end = TARGET_PAGE_ALIGN(start + length - snap->start) >> TARGET_PAGE_BITS;
986 page = (start - snap->start) >> TARGET_PAGE_BITS;
987
988 while (page < end) {
989 if (test_bit(page, snap->dirty)) {
990 return true;
991 }
992 page++;
993 }
994 return false;
995}
996
Paolo Bonzini79e2b9a2015-01-21 12:09:14 +0100997/* Called from RCU critical section */
Andreas Färberbb0e6272013-09-03 13:32:01 +0200998hwaddr memory_region_section_get_iotlb(CPUState *cpu,
Richard Henderson8f5db642019-09-19 21:09:58 -0700999 MemoryRegionSection *section)
Blue Swirle5548612012-04-21 13:08:33 +00001000{
Richard Henderson8f5db642019-09-19 21:09:58 -07001001 AddressSpaceDispatch *d = flatview_to_dispatch(section->fv);
1002 return section - d->map.sections;
Blue Swirle5548612012-04-21 13:08:33 +00001003}
pbrook8da3ff12008-12-01 18:59:50 +00001004
Wei Yangb797ab12019-03-21 16:25:53 +08001005static int subpage_register(subpage_t *mmio, uint32_t start, uint32_t end,
1006 uint16_t section);
Alexey Kardashevskiy16620682017-09-21 18:50:58 +10001007static subpage_t *subpage_init(FlatView *fv, hwaddr base);
Avi Kivity54688b12012-02-09 17:34:32 +02001008
Marcel Apfelbaum53cb28c2013-12-01 14:02:23 +02001009static uint16_t phys_section_add(PhysPageMap *map,
1010 MemoryRegionSection *section)
Avi Kivity5312bd82012-02-12 18:32:55 +02001011{
Paolo Bonzini68f3f652013-05-07 11:30:23 +02001012 /* The physical section number is ORed with a page-aligned
1013 * pointer to produce the iotlb entries. Thus it should
1014 * never overflow into the page-aligned value.
1015 */
Marcel Apfelbaum53cb28c2013-12-01 14:02:23 +02001016 assert(map->sections_nb < TARGET_PAGE_SIZE);
Paolo Bonzini68f3f652013-05-07 11:30:23 +02001017
Marcel Apfelbaum53cb28c2013-12-01 14:02:23 +02001018 if (map->sections_nb == map->sections_nb_alloc) {
1019 map->sections_nb_alloc = MAX(map->sections_nb_alloc * 2, 16);
1020 map->sections = g_renew(MemoryRegionSection, map->sections,
1021 map->sections_nb_alloc);
Avi Kivity5312bd82012-02-12 18:32:55 +02001022 }
Marcel Apfelbaum53cb28c2013-12-01 14:02:23 +02001023 map->sections[map->sections_nb] = *section;
Paolo Bonzinidfde4e62013-05-06 10:46:11 +02001024 memory_region_ref(section->mr);
Marcel Apfelbaum53cb28c2013-12-01 14:02:23 +02001025 return map->sections_nb++;
Avi Kivity5312bd82012-02-12 18:32:55 +02001026}
1027
Paolo Bonzini058bc4b2013-06-25 09:30:48 +02001028static void phys_section_destroy(MemoryRegion *mr)
1029{
Don Slutz55b4e802015-11-30 17:11:04 -05001030 bool have_sub_page = mr->subpage;
1031
Paolo Bonzinidfde4e62013-05-06 10:46:11 +02001032 memory_region_unref(mr);
1033
Don Slutz55b4e802015-11-30 17:11:04 -05001034 if (have_sub_page) {
Paolo Bonzini058bc4b2013-06-25 09:30:48 +02001035 subpage_t *subpage = container_of(mr, subpage_t, iomem);
Peter Crosthwaiteb4fefef2014-06-05 23:15:52 -07001036 object_unref(OBJECT(&subpage->iomem));
Paolo Bonzini058bc4b2013-06-25 09:30:48 +02001037 g_free(subpage);
1038 }
1039}
1040
Paolo Bonzini60926662013-05-29 12:30:26 +02001041static void phys_sections_free(PhysPageMap *map)
Avi Kivity5312bd82012-02-12 18:32:55 +02001042{
Paolo Bonzini9affd6f2013-05-29 12:09:47 +02001043 while (map->sections_nb > 0) {
1044 MemoryRegionSection *section = &map->sections[--map->sections_nb];
Paolo Bonzini058bc4b2013-06-25 09:30:48 +02001045 phys_section_destroy(section->mr);
1046 }
Paolo Bonzini9affd6f2013-05-29 12:09:47 +02001047 g_free(map->sections);
1048 g_free(map->nodes);
Avi Kivity5312bd82012-02-12 18:32:55 +02001049}
1050
Alexey Kardashevskiy99503222017-09-21 18:50:59 +10001051static void register_subpage(FlatView *fv, MemoryRegionSection *section)
Avi Kivity0f0cb162012-02-13 17:14:32 +02001052{
Alexey Kardashevskiy99503222017-09-21 18:50:59 +10001053 AddressSpaceDispatch *d = flatview_to_dispatch(fv);
Avi Kivity0f0cb162012-02-13 17:14:32 +02001054 subpage_t *subpage;
Avi Kivitya8170e52012-10-23 12:30:10 +02001055 hwaddr base = section->offset_within_address_space
Avi Kivity0f0cb162012-02-13 17:14:32 +02001056 & TARGET_PAGE_MASK;
Peter Xu003a0cf2017-05-15 16:50:57 +08001057 MemoryRegionSection *existing = phys_page_find(d, base);
Avi Kivity0f0cb162012-02-13 17:14:32 +02001058 MemoryRegionSection subsection = {
1059 .offset_within_address_space = base,
Paolo Bonzini052e87b2013-05-27 10:08:27 +02001060 .size = int128_make64(TARGET_PAGE_SIZE),
Avi Kivity0f0cb162012-02-13 17:14:32 +02001061 };
Avi Kivitya8170e52012-10-23 12:30:10 +02001062 hwaddr start, end;
Avi Kivity0f0cb162012-02-13 17:14:32 +02001063
Avi Kivityf3705d52012-03-08 16:16:34 +02001064 assert(existing->mr->subpage || existing->mr == &io_mem_unassigned);
Avi Kivity0f0cb162012-02-13 17:14:32 +02001065
Avi Kivityf3705d52012-03-08 16:16:34 +02001066 if (!(existing->mr->subpage)) {
Alexey Kardashevskiy16620682017-09-21 18:50:58 +10001067 subpage = subpage_init(fv, base);
1068 subsection.fv = fv;
Avi Kivity0f0cb162012-02-13 17:14:32 +02001069 subsection.mr = &subpage->iomem;
Avi Kivityac1970f2012-10-03 16:22:53 +02001070 phys_page_set(d, base >> TARGET_PAGE_BITS, 1,
Marcel Apfelbaum53cb28c2013-12-01 14:02:23 +02001071 phys_section_add(&d->map, &subsection));
Avi Kivity0f0cb162012-02-13 17:14:32 +02001072 } else {
Avi Kivityf3705d52012-03-08 16:16:34 +02001073 subpage = container_of(existing->mr, subpage_t, iomem);
Avi Kivity0f0cb162012-02-13 17:14:32 +02001074 }
1075 start = section->offset_within_address_space & ~TARGET_PAGE_MASK;
Paolo Bonzini052e87b2013-05-27 10:08:27 +02001076 end = start + int128_get64(section->size) - 1;
Marcel Apfelbaum53cb28c2013-12-01 14:02:23 +02001077 subpage_register(subpage, start, end,
1078 phys_section_add(&d->map, section));
Avi Kivity0f0cb162012-02-13 17:14:32 +02001079}
1080
1081
Alexey Kardashevskiy99503222017-09-21 18:50:59 +10001082static void register_multipage(FlatView *fv,
Paolo Bonzini052e87b2013-05-27 10:08:27 +02001083 MemoryRegionSection *section)
bellard33417e72003-08-10 21:47:01 +00001084{
Alexey Kardashevskiy99503222017-09-21 18:50:59 +10001085 AddressSpaceDispatch *d = flatview_to_dispatch(fv);
Avi Kivitya8170e52012-10-23 12:30:10 +02001086 hwaddr start_addr = section->offset_within_address_space;
Marcel Apfelbaum53cb28c2013-12-01 14:02:23 +02001087 uint16_t section_index = phys_section_add(&d->map, section);
Paolo Bonzini052e87b2013-05-27 10:08:27 +02001088 uint64_t num_pages = int128_get64(int128_rshift(section->size,
1089 TARGET_PAGE_BITS));
Avi Kivitydd811242012-01-02 12:17:03 +02001090
Paolo Bonzini733d5ef2013-05-27 10:47:10 +02001091 assert(num_pages);
1092 phys_page_set(d, start_addr >> TARGET_PAGE_BITS, num_pages, section_index);
bellard33417e72003-08-10 21:47:01 +00001093}
1094
Wei Yang494d1992019-03-11 13:42:52 +08001095/*
1096 * The range in *section* may look like this:
1097 *
1098 * |s|PPPPPPP|s|
1099 *
1100 * where s stands for subpage and P for page.
1101 */
Alexey Kardashevskiy8629d3f2017-09-21 18:51:00 +10001102void flatview_add_to_dispatch(FlatView *fv, MemoryRegionSection *section)
Avi Kivity0f0cb162012-02-13 17:14:32 +02001103{
Wei Yang494d1992019-03-11 13:42:52 +08001104 MemoryRegionSection remain = *section;
Paolo Bonzini052e87b2013-05-27 10:08:27 +02001105 Int128 page_size = int128_make64(TARGET_PAGE_SIZE);
Avi Kivity0f0cb162012-02-13 17:14:32 +02001106
Wei Yang494d1992019-03-11 13:42:52 +08001107 /* register first subpage */
1108 if (remain.offset_within_address_space & ~TARGET_PAGE_MASK) {
1109 uint64_t left = TARGET_PAGE_ALIGN(remain.offset_within_address_space)
1110 - remain.offset_within_address_space;
Paolo Bonzini733d5ef2013-05-27 10:47:10 +02001111
Wei Yang494d1992019-03-11 13:42:52 +08001112 MemoryRegionSection now = remain;
Paolo Bonzini052e87b2013-05-27 10:08:27 +02001113 now.size = int128_min(int128_make64(left), now.size);
Alexey Kardashevskiy99503222017-09-21 18:50:59 +10001114 register_subpage(fv, &now);
Wei Yang494d1992019-03-11 13:42:52 +08001115 if (int128_eq(remain.size, now.size)) {
1116 return;
1117 }
Paolo Bonzini052e87b2013-05-27 10:08:27 +02001118 remain.size = int128_sub(remain.size, now.size);
1119 remain.offset_within_address_space += int128_get64(now.size);
1120 remain.offset_within_region += int128_get64(now.size);
Avi Kivity0f0cb162012-02-13 17:14:32 +02001121 }
Wei Yang494d1992019-03-11 13:42:52 +08001122
1123 /* register whole pages */
1124 if (int128_ge(remain.size, page_size)) {
1125 MemoryRegionSection now = remain;
1126 now.size = int128_and(now.size, int128_neg(page_size));
1127 register_multipage(fv, &now);
1128 if (int128_eq(remain.size, now.size)) {
1129 return;
1130 }
1131 remain.size = int128_sub(remain.size, now.size);
1132 remain.offset_within_address_space += int128_get64(now.size);
1133 remain.offset_within_region += int128_get64(now.size);
1134 }
1135
1136 /* register last subpage */
1137 register_subpage(fv, &remain);
Avi Kivity0f0cb162012-02-13 17:14:32 +02001138}
1139
Sheng Yang62a27442010-01-26 19:21:16 +08001140void qemu_flush_coalesced_mmio_buffer(void)
1141{
1142 if (kvm_enabled())
1143 kvm_flush_coalesced_mmio_buffer();
1144}
1145
Umesh Deshpandeb2a86582011-08-17 00:01:33 -07001146void qemu_mutex_lock_ramlist(void)
1147{
1148 qemu_mutex_lock(&ram_list.mutex);
1149}
1150
1151void qemu_mutex_unlock_ramlist(void)
1152{
1153 qemu_mutex_unlock(&ram_list.mutex);
1154}
1155
Daniel P. Berrangéca411b72021-09-08 10:35:43 +01001156GString *ram_block_format(void)
Peter Xube9b23c2017-05-12 12:17:41 +08001157{
1158 RAMBlock *block;
1159 char *psize;
Daniel P. Berrangéca411b72021-09-08 10:35:43 +01001160 GString *buf = g_string_new("");
Peter Xube9b23c2017-05-12 12:17:41 +08001161
Dr. David Alan Gilbert694ea272019-10-07 15:36:41 +01001162 RCU_READ_LOCK_GUARD();
Ted Chendbc6ae92022-12-05 20:07:12 +08001163 g_string_append_printf(buf, "%24s %8s %18s %18s %18s %18s %3s\n",
1164 "Block Name", "PSize", "Offset", "Used", "Total",
1165 "HVA", "RO");
1166
Peter Xube9b23c2017-05-12 12:17:41 +08001167 RAMBLOCK_FOREACH(block) {
1168 psize = size_to_str(block->page_size);
Daniel P. Berrangéca411b72021-09-08 10:35:43 +01001169 g_string_append_printf(buf, "%24s %8s 0x%016" PRIx64 " 0x%016" PRIx64
Ted Chendbc6ae92022-12-05 20:07:12 +08001170 " 0x%016" PRIx64 " 0x%016" PRIx64 " %3s\n",
1171 block->idstr, psize,
Daniel P. Berrangéca411b72021-09-08 10:35:43 +01001172 (uint64_t)block->offset,
1173 (uint64_t)block->used_length,
Ted Chendbc6ae92022-12-05 20:07:12 +08001174 (uint64_t)block->max_length,
1175 (uint64_t)(uintptr_t)block->host,
1176 block->mr->readonly ? "ro" : "rw");
1177
Peter Xube9b23c2017-05-12 12:17:41 +08001178 g_free(psize);
1179 }
Daniel P. Berrangéca411b72021-09-08 10:35:43 +01001180
1181 return buf;
Peter Xube9b23c2017-05-12 12:17:41 +08001182}
1183
David Hildenbrand905b7ee2019-04-17 13:31:43 +02001184static int find_min_backend_pagesize(Object *obj, void *opaque)
Alexey Kardashevskiy9c607662017-03-02 13:36:11 +11001185{
Alexey Kardashevskiy9c607662017-03-02 13:36:11 +11001186 long *hpsize_min = opaque;
1187
1188 if (object_dynamic_cast(obj, TYPE_MEMORY_BACKEND)) {
David Gibson7d5489e2019-03-26 14:33:33 +11001189 HostMemoryBackend *backend = MEMORY_BACKEND(obj);
1190 long hpsize = host_memory_backend_pagesize(backend);
David Gibson2b108082018-04-03 15:05:45 +10001191
David Gibson7d5489e2019-03-26 14:33:33 +11001192 if (host_memory_backend_is_mapped(backend) && (hpsize < *hpsize_min)) {
David Gibson0de6e2a2018-04-03 14:55:11 +10001193 *hpsize_min = hpsize;
Alexey Kardashevskiy9c607662017-03-02 13:36:11 +11001194 }
1195 }
1196
1197 return 0;
1198}
1199
David Hildenbrand905b7ee2019-04-17 13:31:43 +02001200static int find_max_backend_pagesize(Object *obj, void *opaque)
1201{
1202 long *hpsize_max = opaque;
1203
1204 if (object_dynamic_cast(obj, TYPE_MEMORY_BACKEND)) {
1205 HostMemoryBackend *backend = MEMORY_BACKEND(obj);
1206 long hpsize = host_memory_backend_pagesize(backend);
1207
1208 if (host_memory_backend_is_mapped(backend) && (hpsize > *hpsize_max)) {
1209 *hpsize_max = hpsize;
1210 }
1211 }
1212
1213 return 0;
1214}
1215
1216/*
1217 * TODO: We assume right now that all mapped host memory backends are
1218 * used as RAM, however some might be used for different purposes.
1219 */
1220long qemu_minrampagesize(void)
Alexey Kardashevskiy9c607662017-03-02 13:36:11 +11001221{
1222 long hpsize = LONG_MAX;
Igor Mammedovad1172d2020-02-19 11:09:47 -05001223 Object *memdev_root = object_resolve_path("/objects", NULL);
Alexey Kardashevskiy9c607662017-03-02 13:36:11 +11001224
Igor Mammedovad1172d2020-02-19 11:09:47 -05001225 object_child_foreach(memdev_root, find_min_backend_pagesize, &hpsize);
Alexey Kardashevskiy9c607662017-03-02 13:36:11 +11001226 return hpsize;
1227}
David Hildenbrand905b7ee2019-04-17 13:31:43 +02001228
1229long qemu_maxrampagesize(void)
1230{
Igor Mammedovad1172d2020-02-19 11:09:47 -05001231 long pagesize = 0;
David Hildenbrand905b7ee2019-04-17 13:31:43 +02001232 Object *memdev_root = object_resolve_path("/objects", NULL);
1233
Igor Mammedovad1172d2020-02-19 11:09:47 -05001234 object_child_foreach(memdev_root, find_max_backend_pagesize, &pagesize);
David Hildenbrand905b7ee2019-04-17 13:31:43 +02001235 return pagesize;
1236}
Alexey Kardashevskiy9c607662017-03-02 13:36:11 +11001237
Hikaru Nishidad5dbde42018-09-24 21:32:05 +09001238#ifdef CONFIG_POSIX
Haozhong Zhangd6af99c2016-10-27 12:22:58 +08001239static int64_t get_file_size(int fd)
1240{
Stefan Hajnoczi72d41eb2019-08-30 10:30:56 +01001241 int64_t size;
1242#if defined(__linux__)
1243 struct stat st;
1244
1245 if (fstat(fd, &st) < 0) {
1246 return -errno;
1247 }
1248
1249 /* Special handling for devdax character devices */
1250 if (S_ISCHR(st.st_mode)) {
1251 g_autofree char *subsystem_path = NULL;
1252 g_autofree char *subsystem = NULL;
1253
1254 subsystem_path = g_strdup_printf("/sys/dev/char/%d:%d/subsystem",
1255 major(st.st_rdev), minor(st.st_rdev));
1256 subsystem = g_file_read_link(subsystem_path, NULL);
1257
1258 if (subsystem && g_str_has_suffix(subsystem, "/dax")) {
1259 g_autofree char *size_path = NULL;
1260 g_autofree char *size_str = NULL;
1261
1262 size_path = g_strdup_printf("/sys/dev/char/%d:%d/size",
1263 major(st.st_rdev), minor(st.st_rdev));
1264
1265 if (g_file_get_contents(size_path, &size_str, NULL, NULL)) {
1266 return g_ascii_strtoll(size_str, NULL, 0);
1267 }
1268 }
1269 }
1270#endif /* defined(__linux__) */
1271
1272 /* st.st_size may be zero for special files yet lseek(2) works */
1273 size = lseek(fd, 0, SEEK_END);
Haozhong Zhangd6af99c2016-10-27 12:22:58 +08001274 if (size < 0) {
1275 return -errno;
1276 }
1277 return size;
1278}
1279
Jingqi Liuce317be2020-04-29 16:50:09 +08001280static int64_t get_file_align(int fd)
1281{
1282 int64_t align = -1;
1283#if defined(__linux__) && defined(CONFIG_LIBDAXCTL)
1284 struct stat st;
1285
1286 if (fstat(fd, &st) < 0) {
1287 return -errno;
1288 }
1289
1290 /* Special handling for devdax character devices */
1291 if (S_ISCHR(st.st_mode)) {
1292 g_autofree char *path = NULL;
1293 g_autofree char *rpath = NULL;
1294 struct daxctl_ctx *ctx;
1295 struct daxctl_region *region;
1296 int rc = 0;
1297
1298 path = g_strdup_printf("/sys/dev/char/%d:%d",
1299 major(st.st_rdev), minor(st.st_rdev));
1300 rpath = realpath(path, NULL);
Peter Maydell8efdb7b2021-08-12 16:15:25 +01001301 if (!rpath) {
1302 return -errno;
1303 }
Jingqi Liuce317be2020-04-29 16:50:09 +08001304
1305 rc = daxctl_new(&ctx);
1306 if (rc) {
1307 return -1;
1308 }
1309
1310 daxctl_region_foreach(ctx, region) {
1311 if (strstr(rpath, daxctl_region_get_path(region))) {
1312 align = daxctl_region_get_align(region);
1313 break;
1314 }
1315 }
1316 daxctl_unref(ctx);
1317 }
1318#endif /* defined(__linux__) && defined(CONFIG_LIBDAXCTL) */
1319
1320 return align;
1321}
1322
Marc-André Lureau8d37b032017-06-02 18:12:22 +04001323static int file_ram_open(const char *path,
1324 const char *region_name,
Stefan Hajnoczi369d6dc2021-01-04 17:13:18 +00001325 bool readonly,
David Hildenbrand4d6b23f2023-09-06 14:04:58 +02001326 bool *created)
Marcelo Tosattic9027602010-03-01 20:25:08 -03001327{
1328 char *filename;
Peter Feiner8ca761f2013-03-04 13:54:25 -05001329 char *sanitized_name;
1330 char *c;
Paolo Bonzini5c3ece72016-03-17 15:53:13 +01001331 int fd = -1;
Marcelo Tosattic9027602010-03-01 20:25:08 -03001332
Marc-André Lureau8d37b032017-06-02 18:12:22 +04001333 *created = false;
Markus Armbrusterfd97fd42016-03-07 20:25:13 +01001334 for (;;) {
Stefan Hajnoczi369d6dc2021-01-04 17:13:18 +00001335 fd = open(path, readonly ? O_RDONLY : O_RDWR);
Markus Armbrusterfd97fd42016-03-07 20:25:13 +01001336 if (fd >= 0) {
David Hildenbrandca01f1b2023-09-06 14:04:59 +02001337 /*
1338 * open(O_RDONLY) won't fail with EISDIR. Check manually if we
1339 * opened a directory and fail similarly to how we fail ENOENT
1340 * in readonly mode. Note that mkstemp() would imply O_RDWR.
1341 */
1342 if (readonly) {
1343 struct stat file_stat;
1344
1345 if (fstat(fd, &file_stat)) {
1346 close(fd);
1347 if (errno == EINTR) {
1348 continue;
1349 }
1350 return -errno;
1351 } else if (S_ISDIR(file_stat.st_mode)) {
1352 close(fd);
1353 return -EISDIR;
1354 }
1355 }
Markus Armbrusterfd97fd42016-03-07 20:25:13 +01001356 /* @path names an existing file, use it */
1357 break;
1358 }
1359 if (errno == ENOENT) {
David Hildenbrand4d6b23f2023-09-06 14:04:58 +02001360 if (readonly) {
1361 /* Refuse to create new, readonly files. */
1362 return -ENOENT;
1363 }
Markus Armbrusterfd97fd42016-03-07 20:25:13 +01001364 /* @path names a file that doesn't exist, create it */
1365 fd = open(path, O_RDWR | O_CREAT | O_EXCL, 0644);
1366 if (fd >= 0) {
Marc-André Lureau8d37b032017-06-02 18:12:22 +04001367 *created = true;
Markus Armbrusterfd97fd42016-03-07 20:25:13 +01001368 break;
1369 }
1370 } else if (errno == EISDIR) {
1371 /* @path names a directory, create a file there */
1372 /* Make name safe to use with mkstemp by replacing '/' with '_'. */
Marc-André Lureau8d37b032017-06-02 18:12:22 +04001373 sanitized_name = g_strdup(region_name);
Markus Armbrusterfd97fd42016-03-07 20:25:13 +01001374 for (c = sanitized_name; *c != '\0'; c++) {
1375 if (*c == '/') {
1376 *c = '_';
1377 }
1378 }
1379
1380 filename = g_strdup_printf("%s/qemu_back_mem.%s.XXXXXX", path,
1381 sanitized_name);
1382 g_free(sanitized_name);
1383
1384 fd = mkstemp(filename);
1385 if (fd >= 0) {
1386 unlink(filename);
1387 g_free(filename);
1388 break;
1389 }
1390 g_free(filename);
1391 }
1392 if (errno != EEXIST && errno != EINTR) {
David Hildenbrand4d6b23f2023-09-06 14:04:58 +02001393 return -errno;
Markus Armbrusterfd97fd42016-03-07 20:25:13 +01001394 }
1395 /*
1396 * Try again on EINTR and EEXIST. The latter happens when
1397 * something else creates the file between our two open().
1398 */
1399 }
1400
Marc-André Lureau8d37b032017-06-02 18:12:22 +04001401 return fd;
1402}
1403
1404static void *file_ram_alloc(RAMBlock *block,
1405 ram_addr_t memory,
1406 int fd,
1407 bool truncate,
Jagannathan Raman44a4ff32021-01-29 11:46:04 -05001408 off_t offset,
Marc-André Lureau8d37b032017-06-02 18:12:22 +04001409 Error **errp)
1410{
David Hildenbrandb444f5c2021-05-10 13:43:20 +02001411 uint32_t qemu_map_flags;
Marc-André Lureau8d37b032017-06-02 18:12:22 +04001412 void *area;
1413
Dr. David Alan Gilbert863e9622016-09-29 20:09:37 +01001414 block->page_size = qemu_fd_getpagesize(fd);
Haozhong Zhang98376842017-12-11 15:28:04 +08001415 if (block->mr->align % block->page_size) {
1416 error_setg(errp, "alignment 0x%" PRIx64
1417 " must be multiples of page size 0x%zx",
1418 block->mr->align, block->page_size);
1419 return NULL;
David Hildenbrand61362b72018-06-07 17:47:05 +02001420 } else if (block->mr->align && !is_power_of_2(block->mr->align)) {
1421 error_setg(errp, "alignment 0x%" PRIx64
1422 " must be a power of two", block->mr->align);
1423 return NULL;
Alexander Graf4b870dc2023-04-03 22:14:21 +00001424 } else if (offset % block->page_size) {
1425 error_setg(errp, "offset 0x%" PRIx64
1426 " must be multiples of page size 0x%zx",
1427 offset, block->page_size);
1428 return NULL;
Haozhong Zhang98376842017-12-11 15:28:04 +08001429 }
1430 block->mr->align = MAX(block->page_size, block->mr->align);
Haozhong Zhang83606682016-10-24 20:49:37 +08001431#if defined(__s390x__)
1432 if (kvm_enabled()) {
1433 block->mr->align = MAX(block->mr->align, QEMU_VMALLOC_ALIGN);
1434 }
1435#endif
Marcelo Tosattic9027602010-03-01 20:25:08 -03001436
Dr. David Alan Gilbert863e9622016-09-29 20:09:37 +01001437 if (memory < block->page_size) {
Hu Tao557529d2014-09-09 13:28:00 +08001438 error_setg(errp, "memory size 0x" RAM_ADDR_FMT " must be equal to "
Dr. David Alan Gilbert863e9622016-09-29 20:09:37 +01001439 "or larger than page size 0x%zx",
1440 memory, block->page_size);
Marc-André Lureau8d37b032017-06-02 18:12:22 +04001441 return NULL;
Haozhong Zhang1775f112016-11-02 09:05:51 +08001442 }
1443
Dr. David Alan Gilbert863e9622016-09-29 20:09:37 +01001444 memory = ROUND_UP(memory, block->page_size);
Marcelo Tosattic9027602010-03-01 20:25:08 -03001445
1446 /*
1447 * ftruncate is not supported by hugetlbfs in older
1448 * hosts, so don't bother bailing out on errors.
1449 * If anything goes wrong with it under other filesystems,
1450 * mmap will fail.
Haozhong Zhangd6af99c2016-10-27 12:22:58 +08001451 *
1452 * Do not truncate the non-empty backend file to avoid corrupting
1453 * the existing data in the file. Disabling shrinking is not
1454 * enough. For example, the current vNVDIMM implementation stores
1455 * the guest NVDIMM labels at the end of the backend file. If the
1456 * backend file is later extended, QEMU will not be able to find
1457 * those labels. Therefore, extending the non-empty backend file
1458 * is disabled as well.
Marcelo Tosattic9027602010-03-01 20:25:08 -03001459 */
Alexander Graf4b870dc2023-04-03 22:14:21 +00001460 if (truncate && ftruncate(fd, offset + memory)) {
Yoshiaki Tamura9742bf22010-08-18 13:30:13 +09001461 perror("ftruncate");
Paolo Bonzini7f56e742014-05-14 17:43:20 +08001462 }
Marcelo Tosattic9027602010-03-01 20:25:08 -03001463
David Hildenbrand5c52a212023-09-06 14:04:54 +02001464 qemu_map_flags = (block->flags & RAM_READONLY) ? QEMU_MAP_READONLY : 0;
David Hildenbrandb444f5c2021-05-10 13:43:20 +02001465 qemu_map_flags |= (block->flags & RAM_SHARED) ? QEMU_MAP_SHARED : 0;
1466 qemu_map_flags |= (block->flags & RAM_PMEM) ? QEMU_MAP_SYNC : 0;
David Hildenbrand8dbe22c2021-05-10 13:43:21 +02001467 qemu_map_flags |= (block->flags & RAM_NORESERVE) ? QEMU_MAP_NORESERVE : 0;
David Hildenbrandb444f5c2021-05-10 13:43:20 +02001468 area = qemu_ram_mmap(fd, memory, block->mr->align, qemu_map_flags, offset);
Marcelo Tosattic9027602010-03-01 20:25:08 -03001469 if (area == MAP_FAILED) {
Paolo Bonzini7f56e742014-05-14 17:43:20 +08001470 error_setg_errno(errp, errno,
Markus Armbrusterfd97fd42016-03-07 20:25:13 +01001471 "unable to map backing store for guest RAM");
Marc-André Lureau8d37b032017-06-02 18:12:22 +04001472 return NULL;
Marcelo Tosattic9027602010-03-01 20:25:08 -03001473 }
Marcelo Tosattief36fa12013-10-28 18:51:46 -02001474
Alex Williamson04b16652010-07-02 11:13:17 -06001475 block->fd = fd;
Alexander Graf4b870dc2023-04-03 22:14:21 +00001476 block->fd_offset = offset;
Marcelo Tosattic9027602010-03-01 20:25:08 -03001477 return area;
1478}
1479#endif
1480
Dr. David Alan Gilbert154cc9e2018-01-05 17:01:37 +00001481/* Allocate space within the ram_addr_t space that governs the
1482 * dirty bitmaps.
1483 * Called with the ramlist lock held.
1484 */
Alex Williamsond17b5282010-06-25 11:08:38 -06001485static ram_addr_t find_ram_offset(ram_addr_t size)
1486{
Alex Williamson04b16652010-07-02 11:13:17 -06001487 RAMBlock *block, *next_block;
Alex Williamson3e837b22011-10-31 08:54:09 -06001488 ram_addr_t offset = RAM_ADDR_MAX, mingap = RAM_ADDR_MAX;
Alex Williamson04b16652010-07-02 11:13:17 -06001489
Stefan Hajnoczi49cd9ac2013-03-11 10:20:21 +01001490 assert(size != 0); /* it would hand out same offset multiple times */
1491
Mike Day0dc3f442013-09-05 14:41:35 -04001492 if (QLIST_EMPTY_RCU(&ram_list.blocks)) {
Alex Williamson04b16652010-07-02 11:13:17 -06001493 return 0;
Mike Day0d53d9f2015-01-21 13:45:24 +01001494 }
Alex Williamson04b16652010-07-02 11:13:17 -06001495
Peter Xu99e15582017-05-12 12:17:39 +08001496 RAMBLOCK_FOREACH(block) {
Dr. David Alan Gilbert154cc9e2018-01-05 17:01:37 +00001497 ram_addr_t candidate, next = RAM_ADDR_MAX;
Alex Williamson04b16652010-07-02 11:13:17 -06001498
Dr. David Alan Gilbert801110a2018-01-05 17:01:38 +00001499 /* Align blocks to start on a 'long' in the bitmap
1500 * which makes the bitmap sync'ing take the fast path.
1501 */
Dr. David Alan Gilbert154cc9e2018-01-05 17:01:37 +00001502 candidate = block->offset + block->max_length;
Dr. David Alan Gilbert801110a2018-01-05 17:01:38 +00001503 candidate = ROUND_UP(candidate, BITS_PER_LONG << TARGET_PAGE_BITS);
Alex Williamson04b16652010-07-02 11:13:17 -06001504
Dr. David Alan Gilbert154cc9e2018-01-05 17:01:37 +00001505 /* Search for the closest following block
1506 * and find the gap.
1507 */
Peter Xu99e15582017-05-12 12:17:39 +08001508 RAMBLOCK_FOREACH(next_block) {
Dr. David Alan Gilbert154cc9e2018-01-05 17:01:37 +00001509 if (next_block->offset >= candidate) {
Alex Williamson04b16652010-07-02 11:13:17 -06001510 next = MIN(next, next_block->offset);
1511 }
1512 }
Dr. David Alan Gilbert154cc9e2018-01-05 17:01:37 +00001513
1514 /* If it fits remember our place and remember the size
1515 * of gap, but keep going so that we might find a smaller
1516 * gap to fill so avoiding fragmentation.
1517 */
1518 if (next - candidate >= size && next - candidate < mingap) {
1519 offset = candidate;
1520 mingap = next - candidate;
Alex Williamson04b16652010-07-02 11:13:17 -06001521 }
Dr. David Alan Gilbert154cc9e2018-01-05 17:01:37 +00001522
1523 trace_find_ram_offset_loop(size, candidate, offset, next, mingap);
Alex Williamson04b16652010-07-02 11:13:17 -06001524 }
Alex Williamson3e837b22011-10-31 08:54:09 -06001525
1526 if (offset == RAM_ADDR_MAX) {
1527 fprintf(stderr, "Failed to find gap of requested size: %" PRIu64 "\n",
1528 (uint64_t)size);
1529 abort();
1530 }
1531
Dr. David Alan Gilbert154cc9e2018-01-05 17:01:37 +00001532 trace_find_ram_offset(size, offset);
1533
Alex Williamson04b16652010-07-02 11:13:17 -06001534 return offset;
1535}
1536
Jason Baronddb97f12012-08-02 15:44:16 -04001537static void qemu_ram_setup_dump(void *addr, ram_addr_t size)
1538{
1539 int ret;
Jason Baronddb97f12012-08-02 15:44:16 -04001540
1541 /* Use MADV_DONTDUMP, if user doesn't want the guest memory in the core */
Marcel Apfelbaum47c8ca52015-02-04 17:43:54 +02001542 if (!machine_dump_guest_core(current_machine)) {
Jason Baronddb97f12012-08-02 15:44:16 -04001543 ret = qemu_madvise(addr, size, QEMU_MADV_DONTDUMP);
1544 if (ret) {
1545 perror("qemu_madvise");
1546 fprintf(stderr, "madvise doesn't support MADV_DONTDUMP, "
Akihiko Odaki0ff32432024-06-28 13:05:52 +09001547 "but dump-guest-core=off specified\n");
Jason Baronddb97f12012-08-02 15:44:16 -04001548 }
1549 }
1550}
1551
Dr. David Alan Gilbert422148d2015-11-05 18:10:32 +00001552const char *qemu_ram_get_idstr(RAMBlock *rb)
1553{
1554 return rb->idstr;
1555}
1556
Yury Kotov754cb9c2019-02-15 20:45:44 +03001557void *qemu_ram_get_host_addr(RAMBlock *rb)
1558{
1559 return rb->host;
1560}
1561
1562ram_addr_t qemu_ram_get_offset(RAMBlock *rb)
1563{
1564 return rb->offset;
1565}
1566
1567ram_addr_t qemu_ram_get_used_length(RAMBlock *rb)
1568{
1569 return rb->used_length;
1570}
1571
David Hildenbrand082851a2021-04-29 13:26:59 +02001572ram_addr_t qemu_ram_get_max_length(RAMBlock *rb)
1573{
1574 return rb->max_length;
1575}
1576
Dr. David Alan Gilbert463a4ac2017-03-07 18:36:36 +00001577bool qemu_ram_is_shared(RAMBlock *rb)
1578{
1579 return rb->flags & RAM_SHARED;
1580}
1581
David Hildenbrand8dbe22c2021-05-10 13:43:21 +02001582bool qemu_ram_is_noreserve(RAMBlock *rb)
1583{
1584 return rb->flags & RAM_NORESERVE;
1585}
1586
Dr. David Alan Gilbert2ce16642018-03-12 17:20:58 +00001587/* Note: Only set at the start of postcopy */
1588bool qemu_ram_is_uf_zeroable(RAMBlock *rb)
1589{
1590 return rb->flags & RAM_UF_ZEROPAGE;
1591}
1592
1593void qemu_ram_set_uf_zeroable(RAMBlock *rb)
1594{
1595 rb->flags |= RAM_UF_ZEROPAGE;
1596}
1597
Cédric Le Goaterb895de52018-05-14 08:57:00 +02001598bool qemu_ram_is_migratable(RAMBlock *rb)
1599{
1600 return rb->flags & RAM_MIGRATABLE;
1601}
1602
1603void qemu_ram_set_migratable(RAMBlock *rb)
1604{
1605 rb->flags |= RAM_MIGRATABLE;
1606}
1607
1608void qemu_ram_unset_migratable(RAMBlock *rb)
1609{
1610 rb->flags &= ~RAM_MIGRATABLE;
1611}
1612
Steve Sistareb0182e52023-06-07 08:18:36 -07001613bool qemu_ram_is_named_file(RAMBlock *rb)
1614{
1615 return rb->flags & RAM_NAMED_FILE;
1616}
1617
Stefan Hajnoczi6d998f32022-10-13 14:59:05 -04001618int qemu_ram_get_fd(RAMBlock *rb)
1619{
1620 return rb->fd;
1621}
1622
Stefan Hajnoczia4a411f2024-01-02 10:35:28 -05001623/* Called with the BQL held. */
Gongleifa53a0e2016-05-10 10:04:59 +08001624void qemu_ram_set_idstr(RAMBlock *new_block, const char *name, DeviceState *dev)
Hu Tao20cfe882014-04-02 15:13:26 +08001625{
Gongleifa53a0e2016-05-10 10:04:59 +08001626 RAMBlock *block;
Hu Tao20cfe882014-04-02 15:13:26 +08001627
Avi Kivityc5705a72011-12-20 15:59:12 +02001628 assert(new_block);
1629 assert(!new_block->idstr[0]);
Cam Macdonell84b89d72010-07-26 18:10:57 -06001630
Anthony Liguori09e5ab62012-02-03 12:28:43 -06001631 if (dev) {
1632 char *id = qdev_get_dev_path(dev);
Cam Macdonell84b89d72010-07-26 18:10:57 -06001633 if (id) {
1634 snprintf(new_block->idstr, sizeof(new_block->idstr), "%s/", id);
Anthony Liguori7267c092011-08-20 22:09:37 -05001635 g_free(id);
Cam Macdonell84b89d72010-07-26 18:10:57 -06001636 }
1637 }
1638 pstrcat(new_block->idstr, sizeof(new_block->idstr), name);
1639
Dr. David Alan Gilbert694ea272019-10-07 15:36:41 +01001640 RCU_READ_LOCK_GUARD();
Peter Xu99e15582017-05-12 12:17:39 +08001641 RAMBLOCK_FOREACH(block) {
Gongleifa53a0e2016-05-10 10:04:59 +08001642 if (block != new_block &&
1643 !strcmp(block->idstr, new_block->idstr)) {
Cam Macdonell84b89d72010-07-26 18:10:57 -06001644 fprintf(stderr, "RAMBlock \"%s\" already registered, abort!\n",
1645 new_block->idstr);
1646 abort();
1647 }
1648 }
Avi Kivityc5705a72011-12-20 15:59:12 +02001649}
1650
Stefan Hajnoczia4a411f2024-01-02 10:35:28 -05001651/* Called with the BQL held. */
Gongleifa53a0e2016-05-10 10:04:59 +08001652void qemu_ram_unset_idstr(RAMBlock *block)
Hu Tao20cfe882014-04-02 15:13:26 +08001653{
Mike Dayae3a7042013-09-05 14:41:35 -04001654 /* FIXME: arch_init.c assumes that this is not called throughout
1655 * migration. Ignore the problem since hot-unplug during migration
1656 * does not work anyway.
1657 */
Hu Tao20cfe882014-04-02 15:13:26 +08001658 if (block) {
1659 memset(block->idstr, 0, sizeof(block->idstr));
1660 }
1661}
1662
Dr. David Alan Gilbert863e9622016-09-29 20:09:37 +01001663size_t qemu_ram_pagesize(RAMBlock *rb)
1664{
1665 return rb->page_size;
1666}
1667
Dr. David Alan Gilbert67f11b52017-02-24 18:28:34 +00001668/* Returns the largest size of page in use */
1669size_t qemu_ram_pagesize_largest(void)
1670{
1671 RAMBlock *block;
1672 size_t largest = 0;
1673
Peter Xu99e15582017-05-12 12:17:39 +08001674 RAMBLOCK_FOREACH(block) {
Dr. David Alan Gilbert67f11b52017-02-24 18:28:34 +00001675 largest = MAX(largest, qemu_ram_pagesize(block));
1676 }
1677
1678 return largest;
1679}
1680
Luiz Capitulino8490fc72012-09-05 16:50:16 -03001681static int memory_try_enable_merging(void *addr, size_t len)
1682{
Marcel Apfelbaum75cc7f02015-02-04 17:43:55 +02001683 if (!machine_mem_merge(current_machine)) {
Luiz Capitulino8490fc72012-09-05 16:50:16 -03001684 /* disabled by the user */
1685 return 0;
1686 }
1687
1688 return qemu_madvise(addr, len, QEMU_MADV_MERGEABLE);
1689}
1690
David Hildenbrandc7c0e722021-04-29 13:27:02 +02001691/*
1692 * Resizing RAM while migrating can result in the migration being canceled.
1693 * Care has to be taken if the guest might have already detected the memory.
Michael S. Tsirkin62be4e32014-11-12 14:27:41 +02001694 *
1695 * As memory core doesn't know how is memory accessed, it is up to
1696 * resize callback to update device state and/or add assertions to detect
1697 * misuse, if necessary.
1698 */
Gongleifa53a0e2016-05-10 10:04:59 +08001699int qemu_ram_resize(RAMBlock *block, ram_addr_t newsize, Error **errp)
Michael S. Tsirkin62be4e32014-11-12 14:27:41 +02001700{
David Hildenbrand8f443042021-04-29 13:27:00 +02001701 const ram_addr_t oldsize = block->used_length;
David Hildenbrandce4adc02020-04-03 11:18:27 +01001702 const ram_addr_t unaligned_size = newsize;
1703
Michael S. Tsirkin62be4e32014-11-12 14:27:41 +02001704 assert(block);
1705
Richard Henderson9260bd42024-01-02 12:57:49 +11001706 newsize = TARGET_PAGE_ALIGN(newsize);
1707 newsize = REAL_HOST_PAGE_ALIGN(newsize);
Michael S. Tsirkin129ddaf2015-02-17 10:15:30 +01001708
Michael S. Tsirkin62be4e32014-11-12 14:27:41 +02001709 if (block->used_length == newsize) {
David Hildenbrandce4adc02020-04-03 11:18:27 +01001710 /*
1711 * We don't have to resize the ram block (which only knows aligned
1712 * sizes), however, we have to notify if the unaligned size changed.
1713 */
1714 if (unaligned_size != memory_region_size(block->mr)) {
1715 memory_region_set_size(block->mr, unaligned_size);
1716 if (block->resized) {
1717 block->resized(block->idstr, unaligned_size, block->host);
1718 }
1719 }
Michael S. Tsirkin62be4e32014-11-12 14:27:41 +02001720 return 0;
1721 }
1722
1723 if (!(block->flags & RAM_RESIZEABLE)) {
1724 error_setg_errno(errp, EINVAL,
Pankaj Guptaa3a92902020-10-22 13:13:02 +02001725 "Size mismatch: %s: 0x" RAM_ADDR_FMT
1726 " != 0x" RAM_ADDR_FMT, block->idstr,
Michael S. Tsirkin62be4e32014-11-12 14:27:41 +02001727 newsize, block->used_length);
1728 return -EINVAL;
1729 }
1730
1731 if (block->max_length < newsize) {
1732 error_setg_errno(errp, EINVAL,
Pankaj Guptaa3a92902020-10-22 13:13:02 +02001733 "Size too large: %s: 0x" RAM_ADDR_FMT
Michael S. Tsirkin62be4e32014-11-12 14:27:41 +02001734 " > 0x" RAM_ADDR_FMT, block->idstr,
1735 newsize, block->max_length);
1736 return -EINVAL;
1737 }
1738
David Hildenbrand8f443042021-04-29 13:27:00 +02001739 /* Notify before modifying the ram block and touching the bitmaps. */
1740 if (block->host) {
1741 ram_block_notify_resize(block->host, oldsize, newsize);
1742 }
1743
Michael S. Tsirkin62be4e32014-11-12 14:27:41 +02001744 cpu_physical_memory_clear_dirty_range(block->offset, block->used_length);
1745 block->used_length = newsize;
Paolo Bonzini58d27072015-03-23 11:56:01 +01001746 cpu_physical_memory_set_dirty_range(block->offset, block->used_length,
1747 DIRTY_CLIENTS_ALL);
David Hildenbrandce4adc02020-04-03 11:18:27 +01001748 memory_region_set_size(block->mr, unaligned_size);
Michael S. Tsirkin62be4e32014-11-12 14:27:41 +02001749 if (block->resized) {
David Hildenbrandce4adc02020-04-03 11:18:27 +01001750 block->resized(block->idstr, unaligned_size, block->host);
Michael S. Tsirkin62be4e32014-11-12 14:27:41 +02001751 }
1752 return 0;
1753}
1754
Beata Michalska61c490e2019-11-21 00:08:41 +00001755/*
1756 * Trigger sync on the given ram block for range [start, start + length]
1757 * with the backing store if one is available.
1758 * Otherwise no-op.
1759 * @Note: this is supposed to be a synchronous op.
1760 */
Philippe Mathieu-Daudéab7e41e2020-05-08 08:24:56 +02001761void qemu_ram_msync(RAMBlock *block, ram_addr_t start, ram_addr_t length)
Beata Michalska61c490e2019-11-21 00:08:41 +00001762{
Beata Michalska61c490e2019-11-21 00:08:41 +00001763 /* The requested range should fit in within the block range */
1764 g_assert((start + length) <= block->used_length);
1765
1766#ifdef CONFIG_LIBPMEM
1767 /* The lack of support for pmem should not block the sync */
1768 if (ramblock_is_pmem(block)) {
Anthony PERARD5d4c9542019-12-19 15:43:22 +00001769 void *addr = ramblock_ptr(block, start);
Beata Michalska61c490e2019-11-21 00:08:41 +00001770 pmem_persist(addr, length);
1771 return;
1772 }
1773#endif
1774 if (block->fd >= 0) {
1775 /**
1776 * Case there is no support for PMEM or the memory has not been
1777 * specified as persistent (or is not one) - use the msync.
1778 * Less optimal but still achieves the same goal
1779 */
Anthony PERARD5d4c9542019-12-19 15:43:22 +00001780 void *addr = ramblock_ptr(block, start);
Beata Michalska61c490e2019-11-21 00:08:41 +00001781 if (qemu_msync(addr, length, block->fd)) {
1782 warn_report("%s: failed to sync memory range: start: "
1783 RAM_ADDR_FMT " length: " RAM_ADDR_FMT,
1784 __func__, start, length);
1785 }
1786 }
1787}
1788
Stefan Hajnoczi5b82b702016-01-25 13:33:20 +00001789/* Called with ram_list.mutex held */
David Hildenbrandb84f06c2024-08-28 11:07:43 +02001790static void dirty_memory_extend(ram_addr_t new_ram_size)
Stefan Hajnoczi5b82b702016-01-25 13:33:20 +00001791{
David Hildenbrandb84f06c2024-08-28 11:07:43 +02001792 unsigned int old_num_blocks = ram_list.num_dirty_blocks;
1793 unsigned int new_num_blocks = DIV_ROUND_UP(new_ram_size,
1794 DIRTY_MEMORY_BLOCK_SIZE);
Stefan Hajnoczi5b82b702016-01-25 13:33:20 +00001795 int i;
1796
1797 /* Only need to extend if block count increased */
1798 if (new_num_blocks <= old_num_blocks) {
1799 return;
1800 }
1801
1802 for (i = 0; i < DIRTY_MEMORY_NUM; i++) {
1803 DirtyMemoryBlocks *old_blocks;
1804 DirtyMemoryBlocks *new_blocks;
1805 int j;
1806
Stefan Hajnoczid73415a2020-09-23 11:56:46 +01001807 old_blocks = qatomic_rcu_read(&ram_list.dirty_memory[i]);
Stefan Hajnoczi5b82b702016-01-25 13:33:20 +00001808 new_blocks = g_malloc(sizeof(*new_blocks) +
1809 sizeof(new_blocks->blocks[0]) * new_num_blocks);
1810
1811 if (old_num_blocks) {
1812 memcpy(new_blocks->blocks, old_blocks->blocks,
1813 old_num_blocks * sizeof(old_blocks->blocks[0]));
1814 }
1815
1816 for (j = old_num_blocks; j < new_num_blocks; j++) {
1817 new_blocks->blocks[j] = bitmap_new(DIRTY_MEMORY_BLOCK_SIZE);
1818 }
1819
Stefan Hajnoczid73415a2020-09-23 11:56:46 +01001820 qatomic_rcu_set(&ram_list.dirty_memory[i], new_blocks);
Stefan Hajnoczi5b82b702016-01-25 13:33:20 +00001821
1822 if (old_blocks) {
1823 g_free_rcu(old_blocks, rcu);
1824 }
1825 }
David Hildenbrandb84f06c2024-08-28 11:07:43 +02001826
1827 ram_list.num_dirty_blocks = new_num_blocks;
Stefan Hajnoczi5b82b702016-01-25 13:33:20 +00001828}
1829
David Hildenbrand7ce18ca2021-04-06 10:01:24 +02001830static void ram_block_add(RAMBlock *new_block, Error **errp)
Avi Kivityc5705a72011-12-20 15:59:12 +02001831{
David Hildenbrand8dbe22c2021-05-10 13:43:21 +02001832 const bool noreserve = qemu_ram_is_noreserve(new_block);
David Hildenbrand7ce18ca2021-04-06 10:01:24 +02001833 const bool shared = qemu_ram_is_shared(new_block);
Paolo Bonzinie1c57ab2014-05-14 17:43:18 +08001834 RAMBlock *block;
Mike Day0d53d9f2015-01-21 13:45:24 +01001835 RAMBlock *last_block = NULL;
Xiaoyao Li15f7a802024-03-20 03:39:02 -05001836 bool free_on_error = false;
David Hildenbrandb84f06c2024-08-28 11:07:43 +02001837 ram_addr_t ram_size;
Markus Armbruster37aa7a02016-01-14 16:09:39 +01001838 Error *err = NULL;
Juan Quintela2152f5c2013-10-08 13:52:02 +02001839
Umesh Deshpandeb2a86582011-08-17 00:01:33 -07001840 qemu_mutex_lock_ramlist();
Michael S. Tsirkin9b8424d2014-12-15 22:55:32 +02001841 new_block->offset = find_ram_offset(new_block->max_length);
Paolo Bonzinie1c57ab2014-05-14 17:43:18 +08001842
1843 if (!new_block->host) {
1844 if (xen_enabled()) {
Michael S. Tsirkin9b8424d2014-12-15 22:55:32 +02001845 xen_ram_alloc(new_block->offset, new_block->max_length,
Markus Armbruster37aa7a02016-01-14 16:09:39 +01001846 new_block->mr, &err);
1847 if (err) {
1848 error_propagate(errp, err);
1849 qemu_mutex_unlock_ramlist();
Paolo Bonzini39c350e2016-03-09 18:14:01 +01001850 return;
Markus Armbruster37aa7a02016-01-14 16:09:39 +01001851 }
Paolo Bonzinie1c57ab2014-05-14 17:43:18 +08001852 } else {
David Hildenbrand25459eb2021-03-03 14:09:16 +01001853 new_block->host = qemu_anon_ram_alloc(new_block->max_length,
1854 &new_block->mr->align,
David Hildenbrand8dbe22c2021-05-10 13:43:21 +02001855 shared, noreserve);
Markus Armbruster39228252013-07-31 15:11:11 +02001856 if (!new_block->host) {
Hu Taoef701d72014-09-09 13:27:54 +08001857 error_setg_errno(errp, errno,
1858 "cannot set up guest memory '%s'",
1859 memory_region_name(new_block->mr));
1860 qemu_mutex_unlock_ramlist();
Paolo Bonzini39c350e2016-03-09 18:14:01 +01001861 return;
Markus Armbruster39228252013-07-31 15:11:11 +02001862 }
Michael S. Tsirkin9b8424d2014-12-15 22:55:32 +02001863 memory_try_enable_merging(new_block->host, new_block->max_length);
Xiaoyao Li15f7a802024-03-20 03:39:02 -05001864 free_on_error = true;
1865 }
1866 }
1867
1868 if (new_block->flags & RAM_GUEST_MEMFD) {
Zhenzhong Duan644a5272024-07-16 14:42:12 +08001869 int ret;
1870
Xiaoyao Li15f7a802024-03-20 03:39:02 -05001871 assert(kvm_enabled());
1872 assert(new_block->guest_memfd < 0);
1873
Zhenzhong Duan644a5272024-07-16 14:42:12 +08001874 ret = ram_block_discard_require(true);
1875 if (ret < 0) {
1876 error_setg_errno(errp, -ret,
Paolo Bonzini852f0042024-03-20 17:45:29 +01001877 "cannot set up private guest memory: discard currently blocked");
1878 error_append_hint(errp, "Are you using assigned devices?\n");
1879 goto out_free;
1880 }
1881
Xiaoyao Li15f7a802024-03-20 03:39:02 -05001882 new_block->guest_memfd = kvm_create_guest_memfd(new_block->max_length,
1883 0, errp);
1884 if (new_block->guest_memfd < 0) {
1885 qemu_mutex_unlock_ramlist();
1886 goto out_free;
Yoshiaki Tamura6977dfe2010-08-18 15:41:49 +09001887 }
1888 }
Cam Macdonell84b89d72010-07-26 18:10:57 -06001889
David Hildenbrandb84f06c2024-08-28 11:07:43 +02001890 ram_size = (new_block->offset + new_block->max_length) >> TARGET_PAGE_BITS;
1891 dirty_memory_extend(ram_size);
Mike Day0d53d9f2015-01-21 13:45:24 +01001892 /* Keep the list sorted from biggest to smallest block. Unlike QTAILQ,
1893 * QLIST (which has an RCU-friendly variant) does not have insertion at
1894 * tail, so save the last element in last_block.
1895 */
Peter Xu99e15582017-05-12 12:17:39 +08001896 RAMBLOCK_FOREACH(block) {
Mike Day0d53d9f2015-01-21 13:45:24 +01001897 last_block = block;
Michael S. Tsirkin9b8424d2014-12-15 22:55:32 +02001898 if (block->max_length < new_block->max_length) {
Paolo Bonziniabb26d62012-11-14 16:00:51 +01001899 break;
1900 }
1901 }
1902 if (block) {
Mike Day0dc3f442013-09-05 14:41:35 -04001903 QLIST_INSERT_BEFORE_RCU(block, new_block, next);
Mike Day0d53d9f2015-01-21 13:45:24 +01001904 } else if (last_block) {
Mike Day0dc3f442013-09-05 14:41:35 -04001905 QLIST_INSERT_AFTER_RCU(last_block, new_block, next);
Mike Day0d53d9f2015-01-21 13:45:24 +01001906 } else { /* list is empty */
Mike Day0dc3f442013-09-05 14:41:35 -04001907 QLIST_INSERT_HEAD_RCU(&ram_list.blocks, new_block, next);
Paolo Bonziniabb26d62012-11-14 16:00:51 +01001908 }
Paolo Bonzini0d6d3c82012-11-14 15:45:02 +01001909 ram_list.mru_block = NULL;
Cam Macdonell84b89d72010-07-26 18:10:57 -06001910
Mike Day0dc3f442013-09-05 14:41:35 -04001911 /* Write list before version */
1912 smp_wmb();
Umesh Deshpandef798b072011-08-18 11:41:17 -07001913 ram_list.version++;
Umesh Deshpandeb2a86582011-08-17 00:01:33 -07001914 qemu_mutex_unlock_ramlist();
Umesh Deshpandef798b072011-08-18 11:41:17 -07001915
Michael S. Tsirkin9b8424d2014-12-15 22:55:32 +02001916 cpu_physical_memory_set_dirty_range(new_block->offset,
Paolo Bonzini58d27072015-03-23 11:56:01 +01001917 new_block->used_length,
1918 DIRTY_CLIENTS_ALL);
Cam Macdonell84b89d72010-07-26 18:10:57 -06001919
Paolo Bonzinia904c912015-01-21 16:18:35 +01001920 if (new_block->host) {
1921 qemu_ram_setup_dump(new_block->host, new_block->max_length);
1922 qemu_madvise(new_block->host, new_block->max_length, QEMU_MADV_HUGEPAGE);
Alexander Bulekova028ede2020-02-19 23:11:09 -05001923 /*
1924 * MADV_DONTFORK is also needed by KVM in absence of synchronous MMU
1925 * Configure it unless the machine is a qtest server, in which case
1926 * KVM is not used and it may be forked (eg for fuzzing purposes).
1927 */
1928 if (!qtest_enabled()) {
1929 qemu_madvise(new_block->host, new_block->max_length,
1930 QEMU_MADV_DONTFORK);
1931 }
David Hildenbrand8f443042021-04-29 13:27:00 +02001932 ram_block_notify_add(new_block->host, new_block->used_length,
1933 new_block->max_length);
Paolo Bonzinie1c57ab2014-05-14 17:43:18 +08001934 }
Xiaoyao Li15f7a802024-03-20 03:39:02 -05001935 return;
1936
1937out_free:
1938 if (free_on_error) {
1939 qemu_anon_ram_free(new_block->host, new_block->max_length);
1940 new_block->host = NULL;
1941 }
Cam Macdonell84b89d72010-07-26 18:10:57 -06001942}
1943
Hikaru Nishidad5dbde42018-09-24 21:32:05 +09001944#ifdef CONFIG_POSIX
Marc-André Lureau38b33622017-06-02 18:12:23 +04001945RAMBlock *qemu_ram_alloc_from_fd(ram_addr_t size, MemoryRegion *mr,
Jagannathan Raman44a4ff32021-01-29 11:46:04 -05001946 uint32_t ram_flags, int fd, off_t offset,
David Hildenbrand5c52a212023-09-06 14:04:54 +02001947 Error **errp)
Paolo Bonzinie1c57ab2014-05-14 17:43:18 +08001948{
1949 RAMBlock *new_block;
Hu Taoef701d72014-09-09 13:27:54 +08001950 Error *local_err = NULL;
Jingqi Liuce317be2020-04-29 16:50:09 +08001951 int64_t file_size, file_align;
Paolo Bonzinie1c57ab2014-05-14 17:43:18 +08001952
Junyan Hea4de8552018-07-18 15:48:00 +08001953 /* Just support these ram flags by now. */
Sean Christopherson56918a12021-07-19 19:21:04 +08001954 assert((ram_flags & ~(RAM_SHARED | RAM_PMEM | RAM_NORESERVE |
David Hildenbrand5c52a212023-09-06 14:04:54 +02001955 RAM_PROTECTED | RAM_NAMED_FILE | RAM_READONLY |
Xiaoyao Li15f7a802024-03-20 03:39:02 -05001956 RAM_READONLY_FD | RAM_GUEST_MEMFD)) == 0);
Junyan Hea4de8552018-07-18 15:48:00 +08001957
Paolo Bonzinie1c57ab2014-05-14 17:43:18 +08001958 if (xen_enabled()) {
Paolo Bonzini7f56e742014-05-14 17:43:20 +08001959 error_setg(errp, "-mem-path not supported with Xen");
Fam Zheng528f46a2016-03-01 14:18:18 +08001960 return NULL;
Paolo Bonzinie1c57ab2014-05-14 17:43:18 +08001961 }
1962
Marc-André Lureaue45e7ae2017-06-02 18:12:21 +04001963 if (kvm_enabled() && !kvm_has_sync_mmu()) {
1964 error_setg(errp,
1965 "host lacks kvm mmu notifiers, -mem-path unsupported");
1966 return NULL;
1967 }
1968
Richard Henderson9260bd42024-01-02 12:57:49 +11001969 size = TARGET_PAGE_ALIGN(size);
1970 size = REAL_HOST_PAGE_ALIGN(size);
1971
Marc-André Lureau8d37b032017-06-02 18:12:22 +04001972 file_size = get_file_size(fd);
Alexander Graf4b870dc2023-04-03 22:14:21 +00001973 if (file_size > offset && file_size < (offset + size)) {
Igor Mammedovc001c3b2020-02-19 11:09:48 -05001974 error_setg(errp, "backing store size 0x%" PRIx64
Marc-André Lureau8d37b032017-06-02 18:12:22 +04001975 " does not match 'size' option 0x" RAM_ADDR_FMT,
Igor Mammedovc001c3b2020-02-19 11:09:48 -05001976 file_size, size);
Marc-André Lureau8d37b032017-06-02 18:12:22 +04001977 return NULL;
1978 }
1979
Jingqi Liuce317be2020-04-29 16:50:09 +08001980 file_align = get_file_align(fd);
Peter Maydell8f1bdb02021-08-12 16:06:24 +01001981 if (file_align > 0 && file_align > mr->align) {
Jingqi Liuce317be2020-04-29 16:50:09 +08001982 error_setg(errp, "backing store align 0x%" PRIx64
Jingqi Liu5f509752020-04-29 16:50:10 +08001983 " is larger than 'align' option 0x%" PRIx64,
Jingqi Liuce317be2020-04-29 16:50:09 +08001984 file_align, mr->align);
1985 return NULL;
1986 }
1987
Paolo Bonzinie1c57ab2014-05-14 17:43:18 +08001988 new_block = g_malloc0(sizeof(*new_block));
1989 new_block->mr = mr;
Michael S. Tsirkin9b8424d2014-12-15 22:55:32 +02001990 new_block->used_length = size;
1991 new_block->max_length = size;
Junyan Hecbfc0172018-07-18 15:47:58 +08001992 new_block->flags = ram_flags;
Xiaoyao Li15f7a802024-03-20 03:39:02 -05001993 new_block->guest_memfd = -1;
David Hildenbrand5c52a212023-09-06 14:04:54 +02001994 new_block->host = file_ram_alloc(new_block, size, fd, !file_size, offset,
1995 errp);
Paolo Bonzini7f56e742014-05-14 17:43:20 +08001996 if (!new_block->host) {
1997 g_free(new_block);
Fam Zheng528f46a2016-03-01 14:18:18 +08001998 return NULL;
Paolo Bonzini7f56e742014-05-14 17:43:20 +08001999 }
2000
David Hildenbrand7ce18ca2021-04-06 10:01:24 +02002001 ram_block_add(new_block, &local_err);
Hu Taoef701d72014-09-09 13:27:54 +08002002 if (local_err) {
2003 g_free(new_block);
2004 error_propagate(errp, local_err);
Fam Zheng528f46a2016-03-01 14:18:18 +08002005 return NULL;
Hu Taoef701d72014-09-09 13:27:54 +08002006 }
Fam Zheng528f46a2016-03-01 14:18:18 +08002007 return new_block;
Marc-André Lureau38b33622017-06-02 18:12:23 +04002008
2009}
2010
2011
2012RAMBlock *qemu_ram_alloc_from_file(ram_addr_t size, MemoryRegion *mr,
Junyan Hecbfc0172018-07-18 15:47:58 +08002013 uint32_t ram_flags, const char *mem_path,
David Hildenbrand5c52a212023-09-06 14:04:54 +02002014 off_t offset, Error **errp)
Marc-André Lureau38b33622017-06-02 18:12:23 +04002015{
2016 int fd;
2017 bool created;
2018 RAMBlock *block;
2019
David Hildenbrand5c52a212023-09-06 14:04:54 +02002020 fd = file_ram_open(mem_path, memory_region_name(mr),
David Hildenbrand4d6b23f2023-09-06 14:04:58 +02002021 !!(ram_flags & RAM_READONLY_FD), &created);
Marc-André Lureau38b33622017-06-02 18:12:23 +04002022 if (fd < 0) {
David Hildenbrand4d6b23f2023-09-06 14:04:58 +02002023 error_setg_errno(errp, -fd, "can't open backing store %s for guest RAM",
2024 mem_path);
David Hildenbrand6da4b1c2023-09-06 14:05:02 +02002025 if (!(ram_flags & RAM_READONLY_FD) && !(ram_flags & RAM_SHARED) &&
2026 fd == -EACCES) {
2027 /*
2028 * If we can open the file R/O (note: will never create a new file)
2029 * and we are dealing with a private mapping, there are still ways
2030 * to consume such files and get RAM instead of ROM.
2031 */
2032 fd = file_ram_open(mem_path, memory_region_name(mr), true,
2033 &created);
2034 if (fd < 0) {
2035 return NULL;
2036 }
2037 assert(!created);
2038 close(fd);
2039 error_append_hint(errp, "Consider opening the backing store"
2040 " read-only but still creating writable RAM using"
2041 " '-object memory-backend-file,readonly=on,rom=off...'"
2042 " (see \"VM templating\" documentation)\n");
2043 }
Marc-André Lureau38b33622017-06-02 18:12:23 +04002044 return NULL;
2045 }
2046
David Hildenbrand5c52a212023-09-06 14:04:54 +02002047 block = qemu_ram_alloc_from_fd(size, mr, ram_flags, fd, offset, errp);
Marc-André Lureau38b33622017-06-02 18:12:23 +04002048 if (!block) {
2049 if (created) {
2050 unlink(mem_path);
2051 }
2052 close(fd);
2053 return NULL;
2054 }
2055
2056 return block;
Paolo Bonzinie1c57ab2014-05-14 17:43:18 +08002057}
Paolo Bonzini0b183fc2014-05-14 17:43:19 +08002058#endif
Paolo Bonzinie1c57ab2014-05-14 17:43:18 +08002059
Michael S. Tsirkin62be4e32014-11-12 14:27:41 +02002060static
Fam Zheng528f46a2016-03-01 14:18:18 +08002061RAMBlock *qemu_ram_alloc_internal(ram_addr_t size, ram_addr_t max_size,
2062 void (*resized)(const char*,
2063 uint64_t length,
2064 void *host),
David Hildenbrandebef62d2021-05-10 13:43:19 +02002065 void *host, uint32_t ram_flags,
Fam Zheng528f46a2016-03-01 14:18:18 +08002066 MemoryRegion *mr, Error **errp)
Paolo Bonzinie1c57ab2014-05-14 17:43:18 +08002067{
2068 RAMBlock *new_block;
Hu Taoef701d72014-09-09 13:27:54 +08002069 Error *local_err = NULL;
Richard Henderson9260bd42024-01-02 12:57:49 +11002070 int align;
Paolo Bonzinie1c57ab2014-05-14 17:43:18 +08002071
David Hildenbrand8dbe22c2021-05-10 13:43:21 +02002072 assert((ram_flags & ~(RAM_SHARED | RAM_RESIZEABLE | RAM_PREALLOC |
Xiaoyao Li15f7a802024-03-20 03:39:02 -05002073 RAM_NORESERVE | RAM_GUEST_MEMFD)) == 0);
David Hildenbrandebef62d2021-05-10 13:43:19 +02002074 assert(!host ^ (ram_flags & RAM_PREALLOC));
2075
Richard Henderson9260bd42024-01-02 12:57:49 +11002076 align = qemu_real_host_page_size();
2077 align = MAX(align, TARGET_PAGE_SIZE);
2078 size = ROUND_UP(size, align);
2079 max_size = ROUND_UP(max_size, align);
2080
Paolo Bonzinie1c57ab2014-05-14 17:43:18 +08002081 new_block = g_malloc0(sizeof(*new_block));
2082 new_block->mr = mr;
Michael S. Tsirkin62be4e32014-11-12 14:27:41 +02002083 new_block->resized = resized;
Michael S. Tsirkin9b8424d2014-12-15 22:55:32 +02002084 new_block->used_length = size;
2085 new_block->max_length = max_size;
Michael S. Tsirkin62be4e32014-11-12 14:27:41 +02002086 assert(max_size >= size);
Paolo Bonzinie1c57ab2014-05-14 17:43:18 +08002087 new_block->fd = -1;
Xiaoyao Li15f7a802024-03-20 03:39:02 -05002088 new_block->guest_memfd = -1;
Marc-André Lureau8e3b0cb2022-03-23 19:57:22 +04002089 new_block->page_size = qemu_real_host_page_size();
Paolo Bonzinie1c57ab2014-05-14 17:43:18 +08002090 new_block->host = host;
David Hildenbrandebef62d2021-05-10 13:43:19 +02002091 new_block->flags = ram_flags;
David Hildenbrand7ce18ca2021-04-06 10:01:24 +02002092 ram_block_add(new_block, &local_err);
Hu Taoef701d72014-09-09 13:27:54 +08002093 if (local_err) {
2094 g_free(new_block);
2095 error_propagate(errp, local_err);
Fam Zheng528f46a2016-03-01 14:18:18 +08002096 return NULL;
Hu Taoef701d72014-09-09 13:27:54 +08002097 }
Fam Zheng528f46a2016-03-01 14:18:18 +08002098 return new_block;
Paolo Bonzinie1c57ab2014-05-14 17:43:18 +08002099}
2100
Fam Zheng528f46a2016-03-01 14:18:18 +08002101RAMBlock *qemu_ram_alloc_from_ptr(ram_addr_t size, void *host,
Michael S. Tsirkin62be4e32014-11-12 14:27:41 +02002102 MemoryRegion *mr, Error **errp)
2103{
David Hildenbrandebef62d2021-05-10 13:43:19 +02002104 return qemu_ram_alloc_internal(size, size, NULL, host, RAM_PREALLOC, mr,
2105 errp);
Michael S. Tsirkin62be4e32014-11-12 14:27:41 +02002106}
2107
David Hildenbrandebef62d2021-05-10 13:43:19 +02002108RAMBlock *qemu_ram_alloc(ram_addr_t size, uint32_t ram_flags,
Marcel Apfelbaum06329cc2017-12-13 16:37:37 +02002109 MemoryRegion *mr, Error **errp)
pbrook94a6b542009-04-11 17:15:54 +00002110{
Xiaoyao Li15f7a802024-03-20 03:39:02 -05002111 assert((ram_flags & ~(RAM_SHARED | RAM_NORESERVE | RAM_GUEST_MEMFD)) == 0);
David Hildenbrandebef62d2021-05-10 13:43:19 +02002112 return qemu_ram_alloc_internal(size, size, NULL, NULL, ram_flags, mr, errp);
Michael S. Tsirkin62be4e32014-11-12 14:27:41 +02002113}
2114
Fam Zheng528f46a2016-03-01 14:18:18 +08002115RAMBlock *qemu_ram_alloc_resizeable(ram_addr_t size, ram_addr_t maxsz,
Michael S. Tsirkin62be4e32014-11-12 14:27:41 +02002116 void (*resized)(const char*,
2117 uint64_t length,
2118 void *host),
2119 MemoryRegion *mr, Error **errp)
2120{
David Hildenbrandebef62d2021-05-10 13:43:19 +02002121 return qemu_ram_alloc_internal(size, maxsz, resized, NULL,
2122 RAM_RESIZEABLE, mr, errp);
pbrook94a6b542009-04-11 17:15:54 +00002123}
bellarde9a1ab12007-02-08 23:08:38 +00002124
Paolo Bonzini43771532013-09-09 17:58:40 +02002125static void reclaim_ramblock(RAMBlock *block)
2126{
2127 if (block->flags & RAM_PREALLOC) {
2128 ;
2129 } else if (xen_enabled()) {
2130 xen_invalidate_map_cache_entry(block->host);
2131#ifndef _WIN32
2132 } else if (block->fd >= 0) {
Murilo Opsfelder Araujo53adb9d2019-01-30 21:36:05 -02002133 qemu_ram_munmap(block->fd, block->host, block->max_length);
Paolo Bonzini43771532013-09-09 17:58:40 +02002134 close(block->fd);
2135#endif
2136 } else {
2137 qemu_anon_ram_free(block->host, block->max_length);
2138 }
Xiaoyao Li15f7a802024-03-20 03:39:02 -05002139
2140 if (block->guest_memfd >= 0) {
2141 close(block->guest_memfd);
Paolo Bonzini852f0042024-03-20 17:45:29 +01002142 ram_block_discard_require(false);
Xiaoyao Li15f7a802024-03-20 03:39:02 -05002143 }
2144
Paolo Bonzini43771532013-09-09 17:58:40 +02002145 g_free(block);
2146}
2147
Fam Zhengf1060c52016-03-01 14:18:22 +08002148void qemu_ram_free(RAMBlock *block)
bellarde9a1ab12007-02-08 23:08:38 +00002149{
Marc-André Lureau85bc2a12016-03-29 13:20:51 +02002150 if (!block) {
2151 return;
2152 }
2153
Paolo Bonzini0987d732016-12-21 00:31:36 +08002154 if (block->host) {
David Hildenbrand8f443042021-04-29 13:27:00 +02002155 ram_block_notify_remove(block->host, block->used_length,
2156 block->max_length);
Paolo Bonzini0987d732016-12-21 00:31:36 +08002157 }
2158
Umesh Deshpandeb2a86582011-08-17 00:01:33 -07002159 qemu_mutex_lock_ramlist();
Fam Zhengf1060c52016-03-01 14:18:22 +08002160 QLIST_REMOVE_RCU(block, next);
2161 ram_list.mru_block = NULL;
2162 /* Write list before version */
2163 smp_wmb();
2164 ram_list.version++;
2165 call_rcu(block, reclaim_ramblock, rcu);
Umesh Deshpandeb2a86582011-08-17 00:01:33 -07002166 qemu_mutex_unlock_ramlist();
bellarde9a1ab12007-02-08 23:08:38 +00002167}
2168
Huang Yingcd19cfa2011-03-02 08:56:19 +01002169#ifndef _WIN32
2170void qemu_ram_remap(ram_addr_t addr, ram_addr_t length)
2171{
2172 RAMBlock *block;
2173 ram_addr_t offset;
2174 int flags;
2175 void *area, *vaddr;
David Hildenbrand9e6b9f32023-09-06 14:04:56 +02002176 int prot;
Huang Yingcd19cfa2011-03-02 08:56:19 +01002177
Peter Xu99e15582017-05-12 12:17:39 +08002178 RAMBLOCK_FOREACH(block) {
Huang Yingcd19cfa2011-03-02 08:56:19 +01002179 offset = addr - block->offset;
Michael S. Tsirkin9b8424d2014-12-15 22:55:32 +02002180 if (offset < block->max_length) {
Michael S. Tsirkin1240be22014-11-12 11:44:41 +02002181 vaddr = ramblock_ptr(block, offset);
Paolo Bonzini7bd4f432014-05-14 17:43:22 +08002182 if (block->flags & RAM_PREALLOC) {
Huang Yingcd19cfa2011-03-02 08:56:19 +01002183 ;
Markus Armbrusterdfeaf2a2013-07-31 15:11:05 +02002184 } else if (xen_enabled()) {
2185 abort();
Huang Yingcd19cfa2011-03-02 08:56:19 +01002186 } else {
2187 flags = MAP_FIXED;
David Hildenbranddbb92ee2021-04-06 10:01:26 +02002188 flags |= block->flags & RAM_SHARED ?
2189 MAP_SHARED : MAP_PRIVATE;
David Hildenbrandd94e0bc2021-05-10 13:43:22 +02002190 flags |= block->flags & RAM_NORESERVE ? MAP_NORESERVE : 0;
David Hildenbrand9e6b9f32023-09-06 14:04:56 +02002191 prot = PROT_READ;
2192 prot |= block->flags & RAM_READONLY ? 0 : PROT_WRITE;
Markus Armbruster3435f392013-07-31 15:11:07 +02002193 if (block->fd >= 0) {
David Hildenbrand9e6b9f32023-09-06 14:04:56 +02002194 area = mmap(vaddr, length, prot, flags, block->fd,
2195 offset + block->fd_offset);
Huang Yingcd19cfa2011-03-02 08:56:19 +01002196 } else {
David Hildenbranddbb92ee2021-04-06 10:01:26 +02002197 flags |= MAP_ANONYMOUS;
David Hildenbrand9e6b9f32023-09-06 14:04:56 +02002198 area = mmap(vaddr, length, prot, flags, -1, 0);
Huang Yingcd19cfa2011-03-02 08:56:19 +01002199 }
2200 if (area != vaddr) {
Alistair Francis493d89b2018-02-03 09:43:14 +01002201 error_report("Could not remap addr: "
2202 RAM_ADDR_FMT "@" RAM_ADDR_FMT "",
2203 length, addr);
Huang Yingcd19cfa2011-03-02 08:56:19 +01002204 exit(1);
2205 }
Luiz Capitulino8490fc72012-09-05 16:50:16 -03002206 memory_try_enable_merging(vaddr, length);
Jason Baronddb97f12012-08-02 15:44:16 -04002207 qemu_ram_setup_dump(vaddr, length);
Huang Yingcd19cfa2011-03-02 08:56:19 +01002208 }
Huang Yingcd19cfa2011-03-02 08:56:19 +01002209 }
2210 }
2211}
2212#endif /* !_WIN32 */
2213
Juergen Grossa99dd332023-10-05 11:17:13 -07002214/*
2215 * Return a host pointer to guest's ram.
Edgar E. Iglesias5a5585f2024-04-30 18:49:35 +02002216 * For Xen, foreign mappings get created if they don't already exist.
Mike Day0dc3f442013-09-05 14:41:35 -04002217 *
Edgar E. Iglesias5a5585f2024-04-30 18:49:35 +02002218 * @block: block for the RAM to lookup (optional and may be NULL).
2219 * @addr: address within the memory region.
2220 * @size: pointer to requested size (optional and may be NULL).
2221 * size may get modified and return a value smaller than
2222 * what was requested.
2223 * @lock: wether to lock the mapping in xen-mapcache until invalidated.
2224 * @is_write: hint wether to map RW or RO in the xen-mapcache.
2225 * (optional and may always be set to true).
Mike Day0dc3f442013-09-05 14:41:35 -04002226 *
Paolo Bonzinie81bcda2015-12-16 10:31:26 +01002227 * Called within RCU critical section.
Mike Dayae3a7042013-09-05 14:41:35 -04002228 */
Manos Pitsidianakisaab46312024-02-15 11:15:06 +02002229static void *qemu_ram_ptr_length(RAMBlock *block, ram_addr_t addr,
Edgar E. Iglesias5a5585f2024-04-30 18:49:35 +02002230 hwaddr *size, bool lock,
2231 bool is_write)
Stefano Stabellini38bee5d2011-05-19 18:35:45 +01002232{
Juergen Grossa99dd332023-10-05 11:17:13 -07002233 hwaddr len = 0;
2234
2235 if (size && *size == 0) {
Stefano Stabellini8ab934f2011-06-27 18:26:06 +01002236 return NULL;
2237 }
Paolo Bonzinie81bcda2015-12-16 10:31:26 +01002238
Gonglei3655cb92016-02-20 10:35:20 +08002239 if (block == NULL) {
2240 block = qemu_get_ram_block(addr);
Paolo Bonzini0878d0e2016-02-22 11:02:12 +01002241 addr -= block->offset;
Gonglei3655cb92016-02-20 10:35:20 +08002242 }
Juergen Grossa99dd332023-10-05 11:17:13 -07002243 if (size) {
2244 *size = MIN(*size, block->max_length - addr);
2245 len = *size;
2246 }
Paolo Bonzinie81bcda2015-12-16 10:31:26 +01002247
2248 if (xen_enabled() && block->host == NULL) {
2249 /* We need to check if the requested address is in the RAM
2250 * because we don't want to map the entire memory in QEMU.
2251 * In that case just map the requested area.
2252 */
Edgar E. Iglesiasa5bdc452024-05-03 03:44:46 +02002253 if (xen_mr_is_memory(block->mr)) {
Edgar E. Iglesias5d1c2602024-05-03 03:44:45 +02002254 return xen_map_cache(block->mr, block->offset + addr,
Edgar E. Iglesias49a72022024-04-30 10:14:01 +02002255 len, block->offset,
2256 lock, lock, is_write);
Stefano Stabellini38bee5d2011-05-19 18:35:45 +01002257 }
2258
Edgar E. Iglesias5a5585f2024-04-30 18:49:35 +02002259 block->host = xen_map_cache(block->mr, block->offset,
Edgar E. Iglesias49a72022024-04-30 10:14:01 +02002260 block->max_length,
2261 block->offset,
2262 1, lock, is_write);
Stefano Stabellini38bee5d2011-05-19 18:35:45 +01002263 }
Paolo Bonzinie81bcda2015-12-16 10:31:26 +01002264
Paolo Bonzini0878d0e2016-02-22 11:02:12 +01002265 return ramblock_ptr(block, addr);
Stefano Stabellini38bee5d2011-05-19 18:35:45 +01002266}
2267
Juergen Grossa99dd332023-10-05 11:17:13 -07002268/*
2269 * Return a host pointer to ram allocated with qemu_ram_alloc.
2270 * This should not be used for general purpose DMA. Use address_space_map
2271 * or address_space_rw instead. For local memory (e.g. video ram) that the
2272 * device owns, use memory_region_get_ram_ptr.
2273 *
2274 * Called within RCU critical section.
2275 */
2276void *qemu_map_ram_ptr(RAMBlock *ram_block, ram_addr_t addr)
2277{
Edgar E. Iglesias5a5585f2024-04-30 18:49:35 +02002278 return qemu_ram_ptr_length(ram_block, addr, NULL, false, true);
Juergen Grossa99dd332023-10-05 11:17:13 -07002279}
2280
Dr. David Alan Gilbertf90bb712018-03-12 17:20:57 +00002281/* Return the offset of a hostpointer within a ramblock */
2282ram_addr_t qemu_ram_block_host_offset(RAMBlock *rb, void *host)
2283{
2284 ram_addr_t res = (uint8_t *)host - (uint8_t *)rb->host;
2285 assert((uintptr_t)host >= (uintptr_t)rb->host);
2286 assert(res < rb->max_length);
2287
2288 return res;
2289}
2290
Dr. David Alan Gilbert422148d2015-11-05 18:10:32 +00002291RAMBlock *qemu_ram_block_from_host(void *ptr, bool round_offset,
Dr. David Alan Gilbert422148d2015-11-05 18:10:32 +00002292 ram_addr_t *offset)
pbrook5579c7f2009-04-11 14:47:08 +00002293{
pbrook94a6b542009-04-11 17:15:54 +00002294 RAMBlock *block;
2295 uint8_t *host = ptr;
2296
Jan Kiszka868bb332011-06-21 22:59:09 +02002297 if (xen_enabled()) {
Paolo Bonzinif615f392016-05-26 10:07:50 +02002298 ram_addr_t ram_addr;
Dr. David Alan Gilbert694ea272019-10-07 15:36:41 +01002299 RCU_READ_LOCK_GUARD();
Paolo Bonzinif615f392016-05-26 10:07:50 +02002300 ram_addr = xen_ram_addr_from_mapcache(ptr);
Edgar E. Iglesias596cccc2024-07-02 00:44:20 +02002301 if (ram_addr == RAM_ADDR_INVALID) {
2302 return NULL;
2303 }
2304
Paolo Bonzinif615f392016-05-26 10:07:50 +02002305 block = qemu_get_ram_block(ram_addr);
Dr. David Alan Gilbert422148d2015-11-05 18:10:32 +00002306 if (block) {
Anthony PERARDd6b6aec2016-06-09 16:56:17 +01002307 *offset = ram_addr - block->offset;
Dr. David Alan Gilbert422148d2015-11-05 18:10:32 +00002308 }
Dr. David Alan Gilbert422148d2015-11-05 18:10:32 +00002309 return block;
Stefano Stabellini712c2b42011-05-19 18:35:46 +01002310 }
2311
Dr. David Alan Gilbert694ea272019-10-07 15:36:41 +01002312 RCU_READ_LOCK_GUARD();
Stefan Hajnoczid73415a2020-09-23 11:56:46 +01002313 block = qatomic_rcu_read(&ram_list.mru_block);
Michael S. Tsirkin9b8424d2014-12-15 22:55:32 +02002314 if (block && block->host && host - block->host < block->max_length) {
Paolo Bonzini23887b72013-05-06 14:28:39 +02002315 goto found;
2316 }
2317
Peter Xu99e15582017-05-12 12:17:39 +08002318 RAMBLOCK_FOREACH(block) {
Jun Nakajima432d2682010-08-31 16:41:25 +01002319 /* This case append when the block is not mapped. */
2320 if (block->host == NULL) {
2321 continue;
2322 }
Michael S. Tsirkin9b8424d2014-12-15 22:55:32 +02002323 if (host - block->host < block->max_length) {
Paolo Bonzini23887b72013-05-06 14:28:39 +02002324 goto found;
Alex Williamsonf471a172010-06-11 11:11:42 -06002325 }
pbrook94a6b542009-04-11 17:15:54 +00002326 }
Jun Nakajima432d2682010-08-31 16:41:25 +01002327
Paolo Bonzini1b5ec232013-05-06 14:36:15 +02002328 return NULL;
Paolo Bonzini23887b72013-05-06 14:28:39 +02002329
2330found:
Dr. David Alan Gilbert422148d2015-11-05 18:10:32 +00002331 *offset = (host - block->host);
2332 if (round_offset) {
2333 *offset &= TARGET_PAGE_MASK;
2334 }
Dr. David Alan Gilbert422148d2015-11-05 18:10:32 +00002335 return block;
2336}
2337
Dr. David Alan Gilberte3dd7492015-11-05 18:10:33 +00002338/*
2339 * Finds the named RAMBlock
2340 *
2341 * name: The name of RAMBlock to find
2342 *
2343 * Returns: RAMBlock (or NULL if not found)
2344 */
2345RAMBlock *qemu_ram_block_by_name(const char *name)
2346{
2347 RAMBlock *block;
2348
Peter Xu99e15582017-05-12 12:17:39 +08002349 RAMBLOCK_FOREACH(block) {
Dr. David Alan Gilberte3dd7492015-11-05 18:10:33 +00002350 if (!strcmp(name, block->idstr)) {
2351 return block;
2352 }
2353 }
2354
2355 return NULL;
2356}
2357
Philippe Mathieu-Daudé8d7f2e72023-10-04 11:06:28 +02002358/*
2359 * Some of the system routines need to translate from a host pointer
2360 * (typically a TLB entry) back to a ram offset.
2361 */
Paolo Bonzini07bdaa42016-03-25 12:55:08 +01002362ram_addr_t qemu_ram_addr_from_host(void *ptr)
Dr. David Alan Gilbert422148d2015-11-05 18:10:32 +00002363{
2364 RAMBlock *block;
Paolo Bonzinif615f392016-05-26 10:07:50 +02002365 ram_addr_t offset;
Dr. David Alan Gilbert422148d2015-11-05 18:10:32 +00002366
Paolo Bonzinif615f392016-05-26 10:07:50 +02002367 block = qemu_ram_block_from_host(ptr, false, &offset);
Dr. David Alan Gilbert422148d2015-11-05 18:10:32 +00002368 if (!block) {
Paolo Bonzini07bdaa42016-03-25 12:55:08 +01002369 return RAM_ADDR_INVALID;
Dr. David Alan Gilbert422148d2015-11-05 18:10:32 +00002370 }
2371
Paolo Bonzini07bdaa42016-03-25 12:55:08 +01002372 return block->offset + offset;
Marcelo Tosattie8902612010-10-11 15:31:19 -03002373}
Alex Williamsonf471a172010-06-11 11:11:42 -06002374
Richard Henderson97e03462022-08-10 12:04:15 -07002375ram_addr_t qemu_ram_addr_from_host_nofail(void *ptr)
2376{
2377 ram_addr_t ram_addr;
2378
2379 ram_addr = qemu_ram_addr_from_host(ptr);
2380 if (ram_addr == RAM_ADDR_INVALID) {
2381 error_report("Bad ram pointer %p", ptr);
2382 abort();
2383 }
2384 return ram_addr;
2385}
2386
Paolo Bonzinib2a44fc2018-03-05 00:19:49 +01002387static MemTxResult flatview_read(FlatView *fv, hwaddr addr,
Philippe Mathieu-Daudéa152be42020-02-19 19:52:44 +01002388 MemTxAttrs attrs, void *buf, hwaddr len);
Alexey Kardashevskiy16620682017-09-21 18:50:58 +10002389static MemTxResult flatview_write(FlatView *fv, hwaddr addr, MemTxAttrs attrs,
Philippe Mathieu-Daudéa152be42020-02-19 19:52:44 +01002390 const void *buf, hwaddr len);
Li Zhijian0c249ff2019-01-17 20:49:01 +08002391static bool flatview_access_valid(FlatView *fv, hwaddr addr, hwaddr len,
Peter Maydelleace72b2018-05-31 14:50:52 +01002392 bool is_write, MemTxAttrs attrs);
Alexey Kardashevskiy16620682017-09-21 18:50:58 +10002393
Peter Maydellf25a49e2015-04-26 16:49:24 +01002394static MemTxResult subpage_read(void *opaque, hwaddr addr, uint64_t *data,
2395 unsigned len, MemTxAttrs attrs)
blueswir1db7b5422007-05-26 17:36:03 +00002396{
Jan Kiszkaacc9d802013-05-26 21:55:37 +02002397 subpage_t *subpage = opaque;
Paolo Bonziniff6cff72014-12-22 13:11:39 +01002398 uint8_t buf[8];
Peter Maydell5c9eb022015-04-26 16:49:24 +01002399 MemTxResult res;
Paolo Bonzini791af8c2013-05-24 16:10:39 +02002400
blueswir1db7b5422007-05-26 17:36:03 +00002401#if defined(DEBUG_SUBPAGE)
Philippe Mathieu-Daudé883f2c52023-01-10 22:29:47 +01002402 printf("%s: subpage %p len %u addr " HWADDR_FMT_plx "\n", __func__,
Jan Kiszkaacc9d802013-05-26 21:55:37 +02002403 subpage, len, addr);
blueswir1db7b5422007-05-26 17:36:03 +00002404#endif
Alexey Kardashevskiy16620682017-09-21 18:50:58 +10002405 res = flatview_read(subpage->fv, addr + subpage->base, attrs, buf, len);
Peter Maydell5c9eb022015-04-26 16:49:24 +01002406 if (res) {
2407 return res;
Peter Maydellf25a49e2015-04-26 16:49:24 +01002408 }
Peter Maydell6d3ede52018-06-15 14:57:14 +01002409 *data = ldn_p(buf, len);
2410 return MEMTX_OK;
blueswir1db7b5422007-05-26 17:36:03 +00002411}
2412
Peter Maydellf25a49e2015-04-26 16:49:24 +01002413static MemTxResult subpage_write(void *opaque, hwaddr addr,
2414 uint64_t value, unsigned len, MemTxAttrs attrs)
blueswir1db7b5422007-05-26 17:36:03 +00002415{
Jan Kiszkaacc9d802013-05-26 21:55:37 +02002416 subpage_t *subpage = opaque;
Paolo Bonziniff6cff72014-12-22 13:11:39 +01002417 uint8_t buf[8];
Jan Kiszkaacc9d802013-05-26 21:55:37 +02002418
blueswir1db7b5422007-05-26 17:36:03 +00002419#if defined(DEBUG_SUBPAGE)
Philippe Mathieu-Daudé883f2c52023-01-10 22:29:47 +01002420 printf("%s: subpage %p len %u addr " HWADDR_FMT_plx
Jan Kiszkaacc9d802013-05-26 21:55:37 +02002421 " value %"PRIx64"\n",
2422 __func__, subpage, len, addr, value);
blueswir1db7b5422007-05-26 17:36:03 +00002423#endif
Peter Maydell6d3ede52018-06-15 14:57:14 +01002424 stn_p(buf, len, value);
Alexey Kardashevskiy16620682017-09-21 18:50:58 +10002425 return flatview_write(subpage->fv, addr + subpage->base, attrs, buf, len);
blueswir1db7b5422007-05-26 17:36:03 +00002426}
2427
Paolo Bonzinic353e4c2013-05-24 14:02:39 +02002428static bool subpage_accepts(void *opaque, hwaddr addr,
Peter Maydell8372d382018-05-31 14:50:52 +01002429 unsigned len, bool is_write,
2430 MemTxAttrs attrs)
Paolo Bonzinic353e4c2013-05-24 14:02:39 +02002431{
Jan Kiszkaacc9d802013-05-26 21:55:37 +02002432 subpage_t *subpage = opaque;
Paolo Bonzinic353e4c2013-05-24 14:02:39 +02002433#if defined(DEBUG_SUBPAGE)
Philippe Mathieu-Daudé883f2c52023-01-10 22:29:47 +01002434 printf("%s: subpage %p %c len %u addr " HWADDR_FMT_plx "\n",
Jan Kiszkaacc9d802013-05-26 21:55:37 +02002435 __func__, subpage, is_write ? 'w' : 'r', len, addr);
Paolo Bonzinic353e4c2013-05-24 14:02:39 +02002436#endif
2437
Alexey Kardashevskiy16620682017-09-21 18:50:58 +10002438 return flatview_access_valid(subpage->fv, addr + subpage->base,
Peter Maydelleace72b2018-05-31 14:50:52 +01002439 len, is_write, attrs);
Paolo Bonzinic353e4c2013-05-24 14:02:39 +02002440}
2441
Avi Kivity70c68e42012-01-02 12:32:48 +02002442static const MemoryRegionOps subpage_ops = {
Peter Maydellf25a49e2015-04-26 16:49:24 +01002443 .read_with_attrs = subpage_read,
2444 .write_with_attrs = subpage_write,
Paolo Bonziniff6cff72014-12-22 13:11:39 +01002445 .impl.min_access_size = 1,
2446 .impl.max_access_size = 8,
2447 .valid.min_access_size = 1,
2448 .valid.max_access_size = 8,
Paolo Bonzinic353e4c2013-05-24 14:02:39 +02002449 .valid.accepts = subpage_accepts,
Avi Kivity70c68e42012-01-02 12:32:48 +02002450 .endianness = DEVICE_NATIVE_ENDIAN,
blueswir1db7b5422007-05-26 17:36:03 +00002451};
2452
Wei Yangb797ab12019-03-21 16:25:53 +08002453static int subpage_register(subpage_t *mmio, uint32_t start, uint32_t end,
2454 uint16_t section)
blueswir1db7b5422007-05-26 17:36:03 +00002455{
2456 int idx, eidx;
2457
2458 if (start >= TARGET_PAGE_SIZE || end >= TARGET_PAGE_SIZE)
2459 return -1;
2460 idx = SUBPAGE_IDX(start);
2461 eidx = SUBPAGE_IDX(end);
2462#if defined(DEBUG_SUBPAGE)
Amos Kong016e9d62013-09-27 09:25:38 +08002463 printf("%s: %p start %08x end %08x idx %08x eidx %08x section %d\n",
2464 __func__, mmio, start, end, idx, eidx, section);
blueswir1db7b5422007-05-26 17:36:03 +00002465#endif
blueswir1db7b5422007-05-26 17:36:03 +00002466 for (; idx <= eidx; idx++) {
Avi Kivity5312bd82012-02-12 18:32:55 +02002467 mmio->sub_section[idx] = section;
blueswir1db7b5422007-05-26 17:36:03 +00002468 }
2469
2470 return 0;
2471}
2472
Alexey Kardashevskiy16620682017-09-21 18:50:58 +10002473static subpage_t *subpage_init(FlatView *fv, hwaddr base)
blueswir1db7b5422007-05-26 17:36:03 +00002474{
Anthony Liguoric227f092009-10-01 16:12:16 -05002475 subpage_t *mmio;
blueswir1db7b5422007-05-26 17:36:03 +00002476
Wei Yangb797ab12019-03-21 16:25:53 +08002477 /* mmio->sub_section is set to PHYS_SECTION_UNASSIGNED with g_malloc0 */
Vijaya Kumar K2615fab2016-10-24 16:26:49 +01002478 mmio = g_malloc0(sizeof(subpage_t) + TARGET_PAGE_SIZE * sizeof(uint16_t));
Alexey Kardashevskiy16620682017-09-21 18:50:58 +10002479 mmio->fv = fv;
aliguori1eec6142009-02-05 22:06:18 +00002480 mmio->base = base;
Paolo Bonzini2c9b15c2013-06-06 05:41:28 -04002481 memory_region_init_io(&mmio->iomem, NULL, &subpage_ops, mmio,
Peter Crosthwaiteb4fefef2014-06-05 23:15:52 -07002482 NULL, TARGET_PAGE_SIZE);
Avi Kivityb3b00c72012-01-02 13:20:11 +02002483 mmio->iomem.subpage = true;
blueswir1db7b5422007-05-26 17:36:03 +00002484#if defined(DEBUG_SUBPAGE)
Philippe Mathieu-Daudé883f2c52023-01-10 22:29:47 +01002485 printf("%s: %p base " HWADDR_FMT_plx " len %08x\n", __func__,
Amos Kong016e9d62013-09-27 09:25:38 +08002486 mmio, base, TARGET_PAGE_SIZE);
blueswir1db7b5422007-05-26 17:36:03 +00002487#endif
blueswir1db7b5422007-05-26 17:36:03 +00002488
2489 return mmio;
2490}
2491
Alexey Kardashevskiy16620682017-09-21 18:50:58 +10002492static uint16_t dummy_section(PhysPageMap *map, FlatView *fv, MemoryRegion *mr)
Avi Kivity5312bd82012-02-12 18:32:55 +02002493{
Alexey Kardashevskiy16620682017-09-21 18:50:58 +10002494 assert(fv);
Avi Kivity5312bd82012-02-12 18:32:55 +02002495 MemoryRegionSection section = {
Alexey Kardashevskiy16620682017-09-21 18:50:58 +10002496 .fv = fv,
Avi Kivity5312bd82012-02-12 18:32:55 +02002497 .mr = mr,
2498 .offset_within_address_space = 0,
2499 .offset_within_region = 0,
Paolo Bonzini052e87b2013-05-27 10:08:27 +02002500 .size = int128_2_64(),
Avi Kivity5312bd82012-02-12 18:32:55 +02002501 };
2502
Marcel Apfelbaum53cb28c2013-12-01 14:02:23 +02002503 return phys_section_add(map, &section);
Avi Kivity5312bd82012-02-12 18:32:55 +02002504}
2505
Peter Maydell2d54f192018-06-15 14:57:14 +01002506MemoryRegionSection *iotlb_to_section(CPUState *cpu,
2507 hwaddr index, MemTxAttrs attrs)
Avi Kivityaa102232012-03-08 17:06:55 +02002508{
Peter Maydella54c87b2016-01-21 14:15:05 +00002509 int asidx = cpu_asidx_from_attrs(cpu, attrs);
2510 CPUAddressSpace *cpuas = &cpu->cpu_ases[asidx];
Richard Henderson0d58c662023-08-25 16:13:17 -07002511 AddressSpaceDispatch *d = cpuas->memory_dispatch;
Richard Henderson86e4f932023-08-25 14:06:58 -07002512 int section_index = index & ~TARGET_PAGE_MASK;
2513 MemoryRegionSection *ret;
Paolo Bonzini9d82b5a2013-08-16 08:26:30 +02002514
Richard Henderson86e4f932023-08-25 14:06:58 -07002515 assert(section_index < d->map.sections_nb);
2516 ret = d->map.sections + section_index;
2517 assert(ret->mr);
2518 assert(ret->mr->ops);
2519
2520 return ret;
Avi Kivityaa102232012-03-08 17:06:55 +02002521}
2522
Avi Kivitye9179ce2009-06-14 11:38:52 +03002523static void io_mem_init(void)
2524{
Paolo Bonzini2c9b15c2013-06-06 05:41:28 -04002525 memory_region_init_io(&io_mem_unassigned, NULL, &unassigned_mem_ops, NULL,
Paolo Bonzini1f6245e2014-06-13 10:48:06 +02002526 NULL, UINT64_MAX);
Avi Kivitye9179ce2009-06-14 11:38:52 +03002527}
2528
Alexey Kardashevskiy8629d3f2017-09-21 18:51:00 +10002529AddressSpaceDispatch *address_space_dispatch_new(FlatView *fv)
Avi Kivityac1970f2012-10-03 16:22:53 +02002530{
Marcel Apfelbaum53cb28c2013-12-01 14:02:23 +02002531 AddressSpaceDispatch *d = g_new0(AddressSpaceDispatch, 1);
2532 uint16_t n;
2533
Alexey Kardashevskiy16620682017-09-21 18:50:58 +10002534 n = dummy_section(&d->map, fv, &io_mem_unassigned);
Marcel Apfelbaum53cb28c2013-12-01 14:02:23 +02002535 assert(n == PHYS_SECTION_UNASSIGNED);
Paolo Bonzini00752702013-05-29 12:13:54 +02002536
Michael S. Tsirkin9736e552013-11-11 14:42:43 +02002537 d->phys_map = (PhysPageEntry) { .ptr = PHYS_MAP_NODE_NIL, .skip = 1 };
Alexey Kardashevskiy66a6df12017-09-21 18:50:56 +10002538
2539 return d;
Paolo Bonzini00752702013-05-29 12:13:54 +02002540}
2541
Alexey Kardashevskiy66a6df12017-09-21 18:50:56 +10002542void address_space_dispatch_free(AddressSpaceDispatch *d)
Paolo Bonzini79e2b9a2015-01-21 12:09:14 +01002543{
2544 phys_sections_free(&d->map);
2545 g_free(d);
2546}
2547
Paolo Bonzini9458a9a2018-02-06 18:37:39 +01002548static void do_nothing(CPUState *cpu, run_on_cpu_data d)
2549{
2550}
2551
2552static void tcg_log_global_after_sync(MemoryListener *listener)
2553{
2554 CPUAddressSpace *cpuas;
2555
2556 /* Wait for the CPU to end the current TB. This avoids the following
2557 * incorrect race:
2558 *
2559 * vCPU migration
2560 * ---------------------- -------------------------
2561 * TLB check -> slow path
2562 * notdirty_mem_write
2563 * write to RAM
2564 * mark dirty
2565 * clear dirty flag
2566 * TLB check -> fast path
2567 * read memory
2568 * write to RAM
2569 *
2570 * by pushing the migration thread's memory read after the vCPU thread has
2571 * written the memory.
2572 */
Pavel Dovgalyuk86cf9e12019-09-17 12:54:06 +03002573 if (replay_mode == REPLAY_MODE_NONE) {
2574 /*
2575 * VGA can make calls to this function while updating the screen.
2576 * In record/replay mode this causes a deadlock, because
2577 * run_on_cpu waits for rr mutex. Therefore no races are possible
2578 * in this case and no need for making run_on_cpu when
Greg Kurzf18d4032021-10-15 11:29:44 +02002579 * record/replay is enabled.
Pavel Dovgalyuk86cf9e12019-09-17 12:54:06 +03002580 */
2581 cpuas = container_of(listener, CPUAddressSpace, tcg_as_listener);
2582 run_on_cpu(cpuas->cpu, do_nothing, RUN_ON_CPU_NULL);
2583 }
Paolo Bonzini9458a9a2018-02-06 18:37:39 +01002584}
2585
Richard Henderson0d58c662023-08-25 16:13:17 -07002586static void tcg_commit_cpu(CPUState *cpu, run_on_cpu_data data)
2587{
2588 CPUAddressSpace *cpuas = data.host_ptr;
2589
2590 cpuas->memory_dispatch = address_space_to_dispatch(cpuas->as);
2591 tlb_flush(cpu);
2592}
2593
Avi Kivity1d711482012-10-02 18:54:45 +02002594static void tcg_commit(MemoryListener *listener)
Avi Kivity50c1e142012-02-08 21:36:02 +02002595{
Peter Maydell32857f42015-10-01 15:29:50 +01002596 CPUAddressSpace *cpuas;
Richard Henderson0d58c662023-08-25 16:13:17 -07002597 CPUState *cpu;
Avi Kivity117712c2012-02-12 21:23:17 +02002598
Emilio G. Cotaf28d0df2018-06-22 13:45:31 -04002599 assert(tcg_enabled());
Avi Kivity117712c2012-02-12 21:23:17 +02002600 /* since each CPU stores ram addresses in its TLB cache, we must
2601 reset the modified entries */
Peter Maydell32857f42015-10-01 15:29:50 +01002602 cpuas = container_of(listener, CPUAddressSpace, tcg_as_listener);
Richard Henderson0d58c662023-08-25 16:13:17 -07002603 cpu = cpuas->cpu;
2604
2605 /*
2606 * Defer changes to as->memory_dispatch until the cpu is quiescent.
2607 * Otherwise we race between (1) other cpu threads and (2) ongoing
2608 * i/o for the current cpu thread, with data cached by mmu_lookup().
2609 *
2610 * In addition, queueing the work function will kick the cpu back to
2611 * the main loop, which will end the RCU critical section and reclaim
2612 * the memory data structures.
2613 *
2614 * That said, the listener is also called during realize, before
2615 * all of the tcg machinery for run-on is initialized: thus halt_cond.
Peter Maydell32857f42015-10-01 15:29:50 +01002616 */
Richard Henderson0d58c662023-08-25 16:13:17 -07002617 if (cpu->halt_cond) {
2618 async_run_on_cpu(cpu, tcg_commit_cpu, RUN_ON_CPU_HOST_PTR(cpuas));
2619 } else {
2620 tcg_commit_cpu(cpu, RUN_ON_CPU_HOST_PTR(cpuas));
2621 }
Avi Kivity50c1e142012-02-08 21:36:02 +02002622}
2623
Avi Kivity62152b82011-07-26 14:26:14 +03002624static void memory_map_init(void)
2625{
Anthony Liguori7267c092011-08-20 22:09:37 -05002626 system_memory = g_malloc(sizeof(*system_memory));
Paolo Bonzini03f49952013-11-07 17:14:36 +01002627
Paolo Bonzini57271d62013-11-07 17:14:37 +01002628 memory_region_init(system_memory, NULL, "system", UINT64_MAX);
Alexey Kardashevskiy7dca8042013-04-29 16:25:51 +00002629 address_space_init(&address_space_memory, system_memory, "memory");
Avi Kivity309cb472011-08-08 16:09:03 +03002630
Anthony Liguori7267c092011-08-20 22:09:37 -05002631 system_io = g_malloc(sizeof(*system_io));
Jan Kiszka3bb28b72013-09-02 18:43:30 +02002632 memory_region_init_io(system_io, NULL, &unassigned_io_ops, NULL, "io",
2633 65536);
Alexey Kardashevskiy7dca8042013-04-29 16:25:51 +00002634 address_space_init(&address_space_io, system_io, "I/O");
Avi Kivity62152b82011-07-26 14:26:14 +03002635}
2636
2637MemoryRegion *get_system_memory(void)
2638{
2639 return system_memory;
2640}
2641
Avi Kivity309cb472011-08-08 16:09:03 +03002642MemoryRegion *get_system_io(void)
2643{
2644 return system_io;
2645}
2646
Paolo Bonzini845b6212015-03-23 11:45:53 +01002647static void invalidate_and_set_dirty(MemoryRegion *mr, hwaddr addr,
Avi Kivitya8170e52012-10-23 12:30:10 +02002648 hwaddr length)
Anthony PERARD51d7a9e2012-10-03 13:49:05 +00002649{
Paolo Bonzinie87f7772015-03-25 15:21:39 +01002650 uint8_t dirty_log_mask = memory_region_get_dirty_log_mask(mr);
Peter Maydell73188062024-07-23 18:05:13 +01002651 ram_addr_t ramaddr = memory_region_get_ram_addr(mr);
2652
2653 /* We know we're only called for RAM MemoryRegions */
2654 assert(ramaddr != RAM_ADDR_INVALID);
2655 addr += ramaddr;
Paolo Bonzini0878d0e2016-02-22 11:02:12 +01002656
Paolo Bonzinie87f7772015-03-25 15:21:39 +01002657 /* No early return if dirty_log_mask is or becomes 0, because
2658 * cpu_physical_memory_set_dirty_range will still call
2659 * xen_modified_memory.
2660 */
2661 if (dirty_log_mask) {
2662 dirty_log_mask =
2663 cpu_physical_memory_range_includes_clean(addr, length, dirty_log_mask);
Anthony PERARD51d7a9e2012-10-03 13:49:05 +00002664 }
Paolo Bonzinie87f7772015-03-25 15:21:39 +01002665 if (dirty_log_mask & (1 << DIRTY_MEMORY_CODE)) {
Paolo Bonzini5aa1ef72017-07-03 17:50:40 +02002666 assert(tcg_enabled());
Richard Hendersone506ad62023-03-06 04:30:11 +03002667 tb_invalidate_phys_range(addr, addr + length - 1);
Paolo Bonzinie87f7772015-03-25 15:21:39 +01002668 dirty_log_mask &= ~(1 << DIRTY_MEMORY_CODE);
2669 }
2670 cpu_physical_memory_set_dirty_range(addr, length, dirty_log_mask);
Anthony PERARD51d7a9e2012-10-03 13:49:05 +00002671}
2672
Stefan Hajnoczi047be4e2019-01-29 11:46:04 +00002673void memory_region_flush_rom_device(MemoryRegion *mr, hwaddr addr, hwaddr size)
2674{
2675 /*
2676 * In principle this function would work on other memory region types too,
2677 * but the ROM device use case is the only one where this operation is
2678 * necessary. Other memory regions should use the
2679 * address_space_read/write() APIs.
2680 */
2681 assert(memory_region_is_romd(mr));
2682
2683 invalidate_and_set_dirty(mr, addr, size);
2684}
2685
Jagannathan Raman3123f932022-06-13 16:26:32 -04002686int memory_access_size(MemoryRegion *mr, unsigned l, hwaddr addr)
Paolo Bonzini82f25632013-05-24 11:59:43 +02002687{
Paolo Bonzinie1622f42013-07-17 13:17:41 +02002688 unsigned access_size_max = mr->ops->valid.max_access_size;
Richard Henderson23326162013-07-08 14:55:59 -07002689
2690 /* Regions are assumed to support 1-4 byte accesses unless
2691 otherwise specified. */
Richard Henderson23326162013-07-08 14:55:59 -07002692 if (access_size_max == 0) {
2693 access_size_max = 4;
Paolo Bonzini82f25632013-05-24 11:59:43 +02002694 }
Richard Henderson23326162013-07-08 14:55:59 -07002695
2696 /* Bound the maximum access by the alignment of the address. */
2697 if (!mr->ops->impl.unaligned) {
2698 unsigned align_size_max = addr & -addr;
2699 if (align_size_max != 0 && align_size_max < access_size_max) {
2700 access_size_max = align_size_max;
2701 }
2702 }
2703
2704 /* Don't attempt accesses larger than the maximum. */
2705 if (l > access_size_max) {
2706 l = access_size_max;
2707 }
Peter Maydell6554f5c2015-07-24 13:33:10 +01002708 l = pow2floor(l);
Richard Henderson23326162013-07-08 14:55:59 -07002709
2710 return l;
Paolo Bonzini82f25632013-05-24 11:59:43 +02002711}
2712
Jagannathan Raman3123f932022-06-13 16:26:32 -04002713bool prepare_mmio_access(MemoryRegion *mr)
Paolo Bonzini125b3802015-06-18 18:47:21 +02002714{
Jan Kiszka4840f102015-06-18 18:47:22 +02002715 bool release_lock = false;
2716
Stefan Hajnoczi195801d2024-01-02 10:35:25 -05002717 if (!bql_locked()) {
2718 bql_lock();
Jan Kiszka4840f102015-06-18 18:47:22 +02002719 release_lock = true;
Paolo Bonzini125b3802015-06-18 18:47:21 +02002720 }
Jan Kiszka4840f102015-06-18 18:47:22 +02002721 if (mr->flush_coalesced_mmio) {
Jan Kiszka4840f102015-06-18 18:47:22 +02002722 qemu_flush_coalesced_mmio_buffer();
Jan Kiszka4840f102015-06-18 18:47:22 +02002723 }
2724
2725 return release_lock;
Paolo Bonzini125b3802015-06-18 18:47:21 +02002726}
2727
Philippe Mathieu-Daudé3ab6fdc2021-12-15 19:24:21 +01002728/**
2729 * flatview_access_allowed
2730 * @mr: #MemoryRegion to be accessed
2731 * @attrs: memory transaction attributes
2732 * @addr: address within that memory region
2733 * @len: the number of bytes to access
2734 *
2735 * Check if a memory transaction is allowed.
2736 *
2737 * Returns: true if transaction is allowed, false if denied.
2738 */
2739static bool flatview_access_allowed(MemoryRegion *mr, MemTxAttrs attrs,
2740 hwaddr addr, hwaddr len)
2741{
2742 if (likely(!attrs.memory)) {
2743 return true;
2744 }
2745 if (memory_region_is_ram(mr)) {
2746 return true;
2747 }
BALATON Zoltan678bf8f2024-11-02 13:17:35 +01002748 qemu_log_mask(LOG_INVALID_MEM,
Philippe Mathieu-Daudé3ab6fdc2021-12-15 19:24:21 +01002749 "Invalid access to non-RAM device at "
2750 "addr 0x%" HWADDR_PRIX ", size %" HWADDR_PRIu ", "
2751 "region '%s'\n", addr, len, memory_region_name(mr));
2752 return false;
2753}
2754
Jonathan Camerone7927d32024-03-07 15:37:09 +00002755static MemTxResult flatview_write_continue_step(MemTxAttrs attrs,
2756 const uint8_t *buf,
2757 hwaddr len, hwaddr mr_addr,
2758 hwaddr *l, MemoryRegion *mr)
2759{
2760 if (!flatview_access_allowed(mr, attrs, mr_addr, *l)) {
2761 return MEMTX_ACCESS_ERROR;
2762 }
2763
2764 if (!memory_access_is_direct(mr, true)) {
2765 uint64_t val;
2766 MemTxResult result;
2767 bool release_lock = prepare_mmio_access(mr);
2768
2769 *l = memory_access_size(mr, *l, mr_addr);
2770 /*
2771 * XXX: could force current_cpu to NULL to avoid
2772 * potential bugs
2773 */
2774
2775 /*
2776 * Assure Coverity (and ourselves) that we are not going to OVERRUN
2777 * the buffer by following ldn_he_p().
2778 */
2779#ifdef QEMU_STATIC_ANALYSIS
2780 assert((*l == 1 && len >= 1) ||
2781 (*l == 2 && len >= 2) ||
2782 (*l == 4 && len >= 4) ||
2783 (*l == 8 && len >= 8));
2784#endif
2785 val = ldn_he_p(buf, *l);
2786 result = memory_region_dispatch_write(mr, mr_addr, val,
2787 size_memop(*l), attrs);
2788 if (release_lock) {
2789 bql_unlock();
2790 }
2791
2792 return result;
2793 } else {
2794 /* RAM case */
2795 uint8_t *ram_ptr = qemu_ram_ptr_length(mr->ram_block, mr_addr, l,
Edgar E. Iglesias5a5585f2024-04-30 18:49:35 +02002796 false, true);
Jonathan Camerone7927d32024-03-07 15:37:09 +00002797
2798 memmove(ram_ptr, buf, *l);
2799 invalidate_and_set_dirty(mr, mr_addr, *l);
2800
2801 return MEMTX_OK;
2802 }
2803}
2804
Paolo Bonzinia203ac72015-12-09 10:18:57 +01002805/* Called within RCU critical section. */
Alexey Kardashevskiy16620682017-09-21 18:50:58 +10002806static MemTxResult flatview_write_continue(FlatView *fv, hwaddr addr,
2807 MemTxAttrs attrs,
Philippe Mathieu-Daudéa152be42020-02-19 19:52:44 +01002808 const void *ptr,
Jonathan Cameron4c7c8562024-03-07 15:37:07 +00002809 hwaddr len, hwaddr mr_addr,
Alexey Kardashevskiy16620682017-09-21 18:50:58 +10002810 hwaddr l, MemoryRegion *mr)
bellard13eb76e2004-01-24 15:23:36 +00002811{
Peter Maydell3b643492015-04-26 16:49:23 +01002812 MemTxResult result = MEMTX_OK;
Philippe Mathieu-Daudéa152be42020-02-19 19:52:44 +01002813 const uint8_t *buf = ptr;
ths3b46e622007-09-17 08:09:54 +00002814
Paolo Bonzinia203ac72015-12-09 10:18:57 +01002815 for (;;) {
Jonathan Camerone7927d32024-03-07 15:37:09 +00002816 result |= flatview_write_continue_step(attrs, buf, len, mr_addr, &l,
2817 mr);
Jan Kiszka4840f102015-06-18 18:47:22 +02002818
bellard13eb76e2004-01-24 15:23:36 +00002819 len -= l;
2820 buf += l;
2821 addr += l;
Paolo Bonzinia203ac72015-12-09 10:18:57 +01002822
2823 if (!len) {
2824 break;
2825 }
2826
2827 l = len;
Jonathan Cameron4c7c8562024-03-07 15:37:07 +00002828 mr = flatview_translate(fv, addr, &mr_addr, &l, true, attrs);
bellard13eb76e2004-01-24 15:23:36 +00002829 }
Paolo Bonzinifd8aaa72013-05-21 09:56:55 +02002830
Peter Maydell3b643492015-04-26 16:49:23 +01002831 return result;
bellard13eb76e2004-01-24 15:23:36 +00002832}
bellard8df1cd02005-01-28 22:37:22 +00002833
Paolo Bonzini4c6ebbb2018-03-05 09:23:56 +01002834/* Called from RCU critical section. */
Alexey Kardashevskiy16620682017-09-21 18:50:58 +10002835static MemTxResult flatview_write(FlatView *fv, hwaddr addr, MemTxAttrs attrs,
Philippe Mathieu-Daudéa152be42020-02-19 19:52:44 +01002836 const void *buf, hwaddr len)
Avi Kivityac1970f2012-10-03 16:22:53 +02002837{
Paolo Bonzinieb7eeb82015-12-09 10:06:31 +01002838 hwaddr l;
Jonathan Cameron4c7c8562024-03-07 15:37:07 +00002839 hwaddr mr_addr;
Paolo Bonzinieb7eeb82015-12-09 10:06:31 +01002840 MemoryRegion *mr;
Paolo Bonzinia203ac72015-12-09 10:18:57 +01002841
Paolo Bonzini4c6ebbb2018-03-05 09:23:56 +01002842 l = len;
Jonathan Cameron4c7c8562024-03-07 15:37:07 +00002843 mr = flatview_translate(fv, addr, &mr_addr, &l, true, attrs);
Philippe Mathieu-Daudé3ab6fdc2021-12-15 19:24:21 +01002844 if (!flatview_access_allowed(mr, attrs, addr, len)) {
2845 return MEMTX_ACCESS_ERROR;
2846 }
Philippe Mathieu-Daudé58e74682021-12-15 19:24:20 +01002847 return flatview_write_continue(fv, addr, attrs, buf, len,
Jonathan Cameron4c7c8562024-03-07 15:37:07 +00002848 mr_addr, l, mr);
Paolo Bonzinia203ac72015-12-09 10:18:57 +01002849}
2850
Jonathan Camerone7927d32024-03-07 15:37:09 +00002851static MemTxResult flatview_read_continue_step(MemTxAttrs attrs, uint8_t *buf,
2852 hwaddr len, hwaddr mr_addr,
2853 hwaddr *l,
2854 MemoryRegion *mr)
2855{
2856 if (!flatview_access_allowed(mr, attrs, mr_addr, *l)) {
2857 return MEMTX_ACCESS_ERROR;
2858 }
2859
2860 if (!memory_access_is_direct(mr, false)) {
2861 /* I/O case */
2862 uint64_t val;
2863 MemTxResult result;
2864 bool release_lock = prepare_mmio_access(mr);
2865
2866 *l = memory_access_size(mr, *l, mr_addr);
2867 result = memory_region_dispatch_read(mr, mr_addr, &val, size_memop(*l),
2868 attrs);
2869
2870 /*
2871 * Assure Coverity (and ourselves) that we are not going to OVERRUN
2872 * the buffer by following stn_he_p().
2873 */
2874#ifdef QEMU_STATIC_ANALYSIS
2875 assert((*l == 1 && len >= 1) ||
2876 (*l == 2 && len >= 2) ||
2877 (*l == 4 && len >= 4) ||
2878 (*l == 8 && len >= 8));
2879#endif
2880 stn_he_p(buf, *l, val);
2881
2882 if (release_lock) {
2883 bql_unlock();
2884 }
2885 return result;
2886 } else {
2887 /* RAM case */
2888 uint8_t *ram_ptr = qemu_ram_ptr_length(mr->ram_block, mr_addr, l,
Edgar E. Iglesias5a5585f2024-04-30 18:49:35 +02002889 false, false);
Jonathan Camerone7927d32024-03-07 15:37:09 +00002890
2891 memcpy(buf, ram_ptr, *l);
2892
2893 return MEMTX_OK;
2894 }
2895}
2896
Paolo Bonzinia203ac72015-12-09 10:18:57 +01002897/* Called within RCU critical section. */
Alexey Kardashevskiy16620682017-09-21 18:50:58 +10002898MemTxResult flatview_read_continue(FlatView *fv, hwaddr addr,
Philippe Mathieu-Daudéa152be42020-02-19 19:52:44 +01002899 MemTxAttrs attrs, void *ptr,
Jonathan Cameron4c7c8562024-03-07 15:37:07 +00002900 hwaddr len, hwaddr mr_addr, hwaddr l,
Alexey Kardashevskiy16620682017-09-21 18:50:58 +10002901 MemoryRegion *mr)
Paolo Bonzinia203ac72015-12-09 10:18:57 +01002902{
Paolo Bonzinia203ac72015-12-09 10:18:57 +01002903 MemTxResult result = MEMTX_OK;
Philippe Mathieu-Daudéa152be42020-02-19 19:52:44 +01002904 uint8_t *buf = ptr;
Paolo Bonzinieb7eeb82015-12-09 10:06:31 +01002905
Alexander Bulekov7cac7fe2021-03-15 10:05:12 -04002906 fuzz_dma_read_cb(addr, len, mr);
Paolo Bonzinia203ac72015-12-09 10:18:57 +01002907 for (;;) {
Jonathan Camerone7927d32024-03-07 15:37:09 +00002908 result |= flatview_read_continue_step(attrs, buf, len, mr_addr, &l, mr);
Paolo Bonzinieb7eeb82015-12-09 10:06:31 +01002909
Paolo Bonzinieb7eeb82015-12-09 10:06:31 +01002910 len -= l;
2911 buf += l;
2912 addr += l;
Paolo Bonzinia203ac72015-12-09 10:18:57 +01002913
2914 if (!len) {
2915 break;
2916 }
2917
2918 l = len;
Jonathan Cameron4c7c8562024-03-07 15:37:07 +00002919 mr = flatview_translate(fv, addr, &mr_addr, &l, false, attrs);
Paolo Bonzinieb7eeb82015-12-09 10:06:31 +01002920 }
Paolo Bonzinia203ac72015-12-09 10:18:57 +01002921
2922 return result;
2923}
2924
Paolo Bonzinib2a44fc2018-03-05 00:19:49 +01002925/* Called from RCU critical section. */
2926static MemTxResult flatview_read(FlatView *fv, hwaddr addr,
Philippe Mathieu-Daudéa152be42020-02-19 19:52:44 +01002927 MemTxAttrs attrs, void *buf, hwaddr len)
Paolo Bonzinia203ac72015-12-09 10:18:57 +01002928{
2929 hwaddr l;
Jonathan Cameron4c7c8562024-03-07 15:37:07 +00002930 hwaddr mr_addr;
Paolo Bonzinia203ac72015-12-09 10:18:57 +01002931 MemoryRegion *mr;
Paolo Bonzinia203ac72015-12-09 10:18:57 +01002932
Paolo Bonzinib2a44fc2018-03-05 00:19:49 +01002933 l = len;
Jonathan Cameron4c7c8562024-03-07 15:37:07 +00002934 mr = flatview_translate(fv, addr, &mr_addr, &l, false, attrs);
Philippe Mathieu-Daudé3ab6fdc2021-12-15 19:24:21 +01002935 if (!flatview_access_allowed(mr, attrs, addr, len)) {
2936 return MEMTX_ACCESS_ERROR;
2937 }
Paolo Bonzinib2a44fc2018-03-05 00:19:49 +01002938 return flatview_read_continue(fv, addr, attrs, buf, len,
Jonathan Cameron4c7c8562024-03-07 15:37:07 +00002939 mr_addr, l, mr);
Avi Kivityac1970f2012-10-03 16:22:53 +02002940}
2941
Paolo Bonzinib2a44fc2018-03-05 00:19:49 +01002942MemTxResult address_space_read_full(AddressSpace *as, hwaddr addr,
Philippe Mathieu-Daudédaa3dda2020-02-19 19:54:35 +01002943 MemTxAttrs attrs, void *buf, hwaddr len)
Paolo Bonzinib2a44fc2018-03-05 00:19:49 +01002944{
2945 MemTxResult result = MEMTX_OK;
2946 FlatView *fv;
2947
2948 if (len > 0) {
Dr. David Alan Gilbert694ea272019-10-07 15:36:41 +01002949 RCU_READ_LOCK_GUARD();
Paolo Bonzinib2a44fc2018-03-05 00:19:49 +01002950 fv = address_space_to_flatview(as);
2951 result = flatview_read(fv, addr, attrs, buf, len);
Paolo Bonzinib2a44fc2018-03-05 00:19:49 +01002952 }
2953
2954 return result;
2955}
2956
Paolo Bonzini4c6ebbb2018-03-05 09:23:56 +01002957MemTxResult address_space_write(AddressSpace *as, hwaddr addr,
2958 MemTxAttrs attrs,
Philippe Mathieu-Daudédaa3dda2020-02-19 19:54:35 +01002959 const void *buf, hwaddr len)
Paolo Bonzini4c6ebbb2018-03-05 09:23:56 +01002960{
2961 MemTxResult result = MEMTX_OK;
2962 FlatView *fv;
2963
2964 if (len > 0) {
Dr. David Alan Gilbert694ea272019-10-07 15:36:41 +01002965 RCU_READ_LOCK_GUARD();
Paolo Bonzini4c6ebbb2018-03-05 09:23:56 +01002966 fv = address_space_to_flatview(as);
2967 result = flatview_write(fv, addr, attrs, buf, len);
Paolo Bonzini4c6ebbb2018-03-05 09:23:56 +01002968 }
2969
2970 return result;
2971}
2972
Paolo Bonzinidb84fd92018-03-05 09:29:04 +01002973MemTxResult address_space_rw(AddressSpace *as, hwaddr addr, MemTxAttrs attrs,
Philippe Mathieu-Daudédaa3dda2020-02-19 19:54:35 +01002974 void *buf, hwaddr len, bool is_write)
Paolo Bonzinidb84fd92018-03-05 09:29:04 +01002975{
2976 if (is_write) {
2977 return address_space_write(as, addr, attrs, buf, len);
2978 } else {
2979 return address_space_read_full(as, addr, attrs, buf, len);
2980 }
2981}
2982
Philippe Mathieu-Daudé75f01c62022-01-15 21:37:23 +01002983MemTxResult address_space_set(AddressSpace *as, hwaddr addr,
2984 uint8_t c, hwaddr len, MemTxAttrs attrs)
2985{
2986#define FILLBUF_SIZE 512
2987 uint8_t fillbuf[FILLBUF_SIZE];
2988 int l;
2989 MemTxResult error = MEMTX_OK;
2990
2991 memset(fillbuf, c, FILLBUF_SIZE);
2992 while (len > 0) {
2993 l = len < FILLBUF_SIZE ? len : FILLBUF_SIZE;
2994 error |= address_space_write(as, addr, attrs, fillbuf, l);
2995 len -= l;
2996 addr += l;
2997 }
2998
2999 return error;
3000}
3001
Philippe Mathieu-Daudéd7ef71e2020-02-19 20:02:11 +01003002void cpu_physical_memory_rw(hwaddr addr, void *buf,
Philippe Mathieu-Daudé28c80bf2020-02-19 20:32:30 +01003003 hwaddr len, bool is_write)
Avi Kivityac1970f2012-10-03 16:22:53 +02003004{
Peter Maydell5c9eb022015-04-26 16:49:24 +01003005 address_space_rw(&address_space_memory, addr, MEMTXATTRS_UNSPECIFIED,
3006 buf, len, is_write);
Avi Kivityac1970f2012-10-03 16:22:53 +02003007}
3008
Alexander Graf582b55a2013-12-11 14:17:44 +01003009enum write_rom_type {
3010 WRITE_DATA,
3011 FLUSH_CACHE,
3012};
3013
Peter Maydell75693e12018-12-14 13:30:48 +00003014static inline MemTxResult address_space_write_rom_internal(AddressSpace *as,
3015 hwaddr addr,
3016 MemTxAttrs attrs,
Philippe Mathieu-Daudédaa3dda2020-02-19 19:54:35 +01003017 const void *ptr,
Li Zhijian0c249ff2019-01-17 20:49:01 +08003018 hwaddr len,
Peter Maydell75693e12018-12-14 13:30:48 +00003019 enum write_rom_type type)
bellardd0ecd2a2006-04-23 17:14:48 +00003020{
Paolo Bonzini149f54b2013-05-24 12:59:37 +02003021 hwaddr l;
Philippe Mathieu-Daudé20804672020-02-19 18:01:32 +01003022 uint8_t *ram_ptr;
Paolo Bonzini149f54b2013-05-24 12:59:37 +02003023 hwaddr addr1;
Paolo Bonzini5c8a00c2013-05-29 12:42:00 +02003024 MemoryRegion *mr;
Philippe Mathieu-Daudédaa3dda2020-02-19 19:54:35 +01003025 const uint8_t *buf = ptr;
ths3b46e622007-09-17 08:09:54 +00003026
Dr. David Alan Gilbert694ea272019-10-07 15:36:41 +01003027 RCU_READ_LOCK_GUARD();
bellardd0ecd2a2006-04-23 17:14:48 +00003028 while (len > 0) {
Paolo Bonzini149f54b2013-05-24 12:59:37 +02003029 l = len;
Peter Maydell75693e12018-12-14 13:30:48 +00003030 mr = address_space_translate(as, addr, &addr1, &l, true, attrs);
ths3b46e622007-09-17 08:09:54 +00003031
Paolo Bonzini5c8a00c2013-05-29 12:42:00 +02003032 if (!(memory_region_is_ram(mr) ||
3033 memory_region_is_romd(mr))) {
Paolo Bonzinib242e0e2015-07-04 00:24:51 +02003034 l = memory_access_size(mr, l, addr1);
bellardd0ecd2a2006-04-23 17:14:48 +00003035 } else {
bellardd0ecd2a2006-04-23 17:14:48 +00003036 /* ROM/RAM case */
Philippe Mathieu-Daudé20804672020-02-19 18:01:32 +01003037 ram_ptr = qemu_map_ram_ptr(mr->ram_block, addr1);
Alexander Graf582b55a2013-12-11 14:17:44 +01003038 switch (type) {
3039 case WRITE_DATA:
Philippe Mathieu-Daudé20804672020-02-19 18:01:32 +01003040 memcpy(ram_ptr, buf, l);
Paolo Bonzini845b6212015-03-23 11:45:53 +01003041 invalidate_and_set_dirty(mr, addr1, l);
Alexander Graf582b55a2013-12-11 14:17:44 +01003042 break;
3043 case FLUSH_CACHE:
Richard Henderson1da8de32020-12-12 10:38:21 -06003044 flush_idcache_range((uintptr_t)ram_ptr, (uintptr_t)ram_ptr, l);
Alexander Graf582b55a2013-12-11 14:17:44 +01003045 break;
3046 }
bellardd0ecd2a2006-04-23 17:14:48 +00003047 }
3048 len -= l;
3049 buf += l;
3050 addr += l;
3051 }
Peter Maydell75693e12018-12-14 13:30:48 +00003052 return MEMTX_OK;
bellardd0ecd2a2006-04-23 17:14:48 +00003053}
3054
Alexander Graf582b55a2013-12-11 14:17:44 +01003055/* used for ROM loading : can write in RAM and ROM */
Peter Maydell3c8133f2018-12-14 13:30:48 +00003056MemTxResult address_space_write_rom(AddressSpace *as, hwaddr addr,
3057 MemTxAttrs attrs,
Philippe Mathieu-Daudédaa3dda2020-02-19 19:54:35 +01003058 const void *buf, hwaddr len)
Alexander Graf582b55a2013-12-11 14:17:44 +01003059{
Peter Maydell3c8133f2018-12-14 13:30:48 +00003060 return address_space_write_rom_internal(as, addr, attrs,
3061 buf, len, WRITE_DATA);
Alexander Graf582b55a2013-12-11 14:17:44 +01003062}
3063
Li Zhijian0c249ff2019-01-17 20:49:01 +08003064void cpu_flush_icache_range(hwaddr start, hwaddr len)
Alexander Graf582b55a2013-12-11 14:17:44 +01003065{
3066 /*
3067 * This function should do the same thing as an icache flush that was
3068 * triggered from within the guest. For TCG we are always cache coherent,
3069 * so there is no need to flush anything. For KVM / Xen we need to flush
3070 * the host's instruction cache at least.
3071 */
3072 if (tcg_enabled()) {
3073 return;
3074 }
3075
Peter Maydell75693e12018-12-14 13:30:48 +00003076 address_space_write_rom_internal(&address_space_memory,
3077 start, MEMTXATTRS_UNSPECIFIED,
3078 NULL, len, FLUSH_CACHE);
Alexander Graf582b55a2013-12-11 14:17:44 +01003079}
3080
Mattias Nissler637b0aa2024-08-19 06:54:54 -07003081/*
3082 * A magic value stored in the first 8 bytes of the bounce buffer struct. Used
3083 * to detect illegal pointers passed to address_space_unmap.
3084 */
3085#define BOUNCE_BUFFER_MAGIC 0xb4017ceb4ffe12ed
3086
3087typedef struct {
3088 uint64_t magic;
3089 MemoryRegion *mr;
3090 hwaddr addr;
3091 size_t len;
3092 uint8_t buffer[];
3093} BounceBuffer;
3094
Mattias Nissler69e78f12023-09-07 06:04:23 -07003095static void
3096address_space_unregister_map_client_do(AddressSpaceMapClient *client)
aliguoriba223c22009-01-22 16:59:16 +00003097{
Blue Swirl72cf2d42009-09-12 07:36:22 +00003098 QLIST_REMOVE(client, link);
Anthony Liguori7267c092011-08-20 22:09:37 -05003099 g_free(client);
aliguoriba223c22009-01-22 16:59:16 +00003100}
3101
Mattias Nissler5c627192023-09-07 06:04:23 -07003102static void address_space_notify_map_clients_locked(AddressSpace *as)
aliguoriba223c22009-01-22 16:59:16 +00003103{
Mattias Nissler69e78f12023-09-07 06:04:23 -07003104 AddressSpaceMapClient *client;
aliguoriba223c22009-01-22 16:59:16 +00003105
Mattias Nissler69e78f12023-09-07 06:04:23 -07003106 while (!QLIST_EMPTY(&as->map_client_list)) {
3107 client = QLIST_FIRST(&as->map_client_list);
Fam Zhenge95205e2015-03-16 17:03:37 +08003108 qemu_bh_schedule(client->bh);
Mattias Nissler5c627192023-09-07 06:04:23 -07003109 address_space_unregister_map_client_do(client);
aliguoriba223c22009-01-22 16:59:16 +00003110 }
3111}
3112
Mattias Nissler5c627192023-09-07 06:04:23 -07003113void address_space_register_map_client(AddressSpace *as, QEMUBH *bh)
bellardd0ecd2a2006-04-23 17:14:48 +00003114{
Mattias Nissler69e78f12023-09-07 06:04:23 -07003115 AddressSpaceMapClient *client = g_malloc(sizeof(*client));
bellardd0ecd2a2006-04-23 17:14:48 +00003116
Mattias Nissler69e78f12023-09-07 06:04:23 -07003117 QEMU_LOCK_GUARD(&as->map_client_list_lock);
Fam Zhenge95205e2015-03-16 17:03:37 +08003118 client->bh = bh;
Mattias Nissler69e78f12023-09-07 06:04:23 -07003119 QLIST_INSERT_HEAD(&as->map_client_list, client, link);
Mattias Nissler637b0aa2024-08-19 06:54:54 -07003120 /* Write map_client_list before reading bounce_buffer_size. */
Paolo Bonzini33828ca2023-03-03 14:36:32 +01003121 smp_mb();
Mattias Nissler637b0aa2024-08-19 06:54:54 -07003122 if (qatomic_read(&as->bounce_buffer_size) < as->max_bounce_buffer_size) {
Mattias Nissler5c627192023-09-07 06:04:23 -07003123 address_space_notify_map_clients_locked(as);
Fam Zheng33b6c2e2015-03-16 17:03:36 +08003124 }
bellardd0ecd2a2006-04-23 17:14:48 +00003125}
3126
Fam Zheng38e047b2015-03-16 17:03:35 +08003127void cpu_exec_init_all(void)
3128{
3129 qemu_mutex_init(&ram_list.mutex);
Peter Maydell20bccb82016-10-24 16:26:49 +01003130 /* The data structures we set up here depend on knowing the page size,
3131 * so no more changes can be made after this point.
3132 * In an ideal world, nothing we did before we had finished the
3133 * machine setup would care about the target page size, and we could
3134 * do this much later, rather than requiring board models to state
3135 * up front what their requirements are.
3136 */
3137 finalize_target_page_bits();
Fam Zheng38e047b2015-03-16 17:03:35 +08003138 io_mem_init();
Paolo Bonzini680a4782015-11-02 09:23:52 +01003139 memory_map_init();
Fam Zheng38e047b2015-03-16 17:03:35 +08003140}
3141
Mattias Nissler5c627192023-09-07 06:04:23 -07003142void address_space_unregister_map_client(AddressSpace *as, QEMUBH *bh)
bellardd0ecd2a2006-04-23 17:14:48 +00003143{
Mattias Nissler69e78f12023-09-07 06:04:23 -07003144 AddressSpaceMapClient *client;
bellardd0ecd2a2006-04-23 17:14:48 +00003145
Mattias Nissler69e78f12023-09-07 06:04:23 -07003146 QEMU_LOCK_GUARD(&as->map_client_list_lock);
3147 QLIST_FOREACH(client, &as->map_client_list, link) {
Fam Zhenge95205e2015-03-16 17:03:37 +08003148 if (client->bh == bh) {
Mattias Nissler5c627192023-09-07 06:04:23 -07003149 address_space_unregister_map_client_do(client);
Fam Zhenge95205e2015-03-16 17:03:37 +08003150 break;
3151 }
3152 }
bellardd0ecd2a2006-04-23 17:14:48 +00003153}
3154
Mattias Nissler5c627192023-09-07 06:04:23 -07003155static void address_space_notify_map_clients(AddressSpace *as)
bellardd0ecd2a2006-04-23 17:14:48 +00003156{
Mattias Nissler69e78f12023-09-07 06:04:23 -07003157 QEMU_LOCK_GUARD(&as->map_client_list_lock);
Mattias Nissler5c627192023-09-07 06:04:23 -07003158 address_space_notify_map_clients_locked(as);
aliguori6d16c2f2009-01-22 16:59:11 +00003159}
3160
Li Zhijian0c249ff2019-01-17 20:49:01 +08003161static bool flatview_access_valid(FlatView *fv, hwaddr addr, hwaddr len,
Peter Maydelleace72b2018-05-31 14:50:52 +01003162 bool is_write, MemTxAttrs attrs)
Paolo Bonzini51644ab2013-04-11 15:40:59 +02003163{
Paolo Bonzini5c8a00c2013-05-29 12:42:00 +02003164 MemoryRegion *mr;
Paolo Bonzini51644ab2013-04-11 15:40:59 +02003165 hwaddr l, xlat;
3166
3167 while (len > 0) {
3168 l = len;
Peter Maydellefa99a22018-05-31 14:50:52 +01003169 mr = flatview_translate(fv, addr, &xlat, &l, is_write, attrs);
Paolo Bonzini5c8a00c2013-05-29 12:42:00 +02003170 if (!memory_access_is_direct(mr, is_write)) {
3171 l = memory_access_size(mr, l, addr);
Peter Maydelleace72b2018-05-31 14:50:52 +01003172 if (!memory_region_access_valid(mr, xlat, l, is_write, attrs)) {
Paolo Bonzini51644ab2013-04-11 15:40:59 +02003173 return false;
3174 }
3175 }
3176
3177 len -= l;
3178 addr += l;
3179 }
3180 return true;
3181}
3182
Alexey Kardashevskiy16620682017-09-21 18:50:58 +10003183bool address_space_access_valid(AddressSpace *as, hwaddr addr,
Li Zhijian0c249ff2019-01-17 20:49:01 +08003184 hwaddr len, bool is_write,
Peter Maydellfddffa42018-05-31 14:50:52 +01003185 MemTxAttrs attrs)
Alexey Kardashevskiy16620682017-09-21 18:50:58 +10003186{
Paolo Bonzini11e732a2018-03-05 00:23:26 +01003187 FlatView *fv;
Paolo Bonzini11e732a2018-03-05 00:23:26 +01003188
Dr. David Alan Gilbert694ea272019-10-07 15:36:41 +01003189 RCU_READ_LOCK_GUARD();
Paolo Bonzini11e732a2018-03-05 00:23:26 +01003190 fv = address_space_to_flatview(as);
Philippe Mathieu-Daudé58e74682021-12-15 19:24:20 +01003191 return flatview_access_valid(fv, addr, len, is_write, attrs);
Alexey Kardashevskiy16620682017-09-21 18:50:58 +10003192}
3193
Paolo Bonzini715c31e2016-11-22 12:04:31 +01003194static hwaddr
Alexey Kardashevskiy16620682017-09-21 18:50:58 +10003195flatview_extend_translation(FlatView *fv, hwaddr addr,
Peter Maydell53d07902018-05-31 14:50:52 +01003196 hwaddr target_len,
3197 MemoryRegion *mr, hwaddr base, hwaddr len,
3198 bool is_write, MemTxAttrs attrs)
Paolo Bonzini715c31e2016-11-22 12:04:31 +01003199{
3200 hwaddr done = 0;
3201 hwaddr xlat;
3202 MemoryRegion *this_mr;
3203
3204 for (;;) {
3205 target_len -= len;
3206 addr += len;
3207 done += len;
3208 if (target_len == 0) {
3209 return done;
3210 }
3211
3212 len = target_len;
Alexey Kardashevskiy16620682017-09-21 18:50:58 +10003213 this_mr = flatview_translate(fv, addr, &xlat,
Peter Maydellefa99a22018-05-31 14:50:52 +01003214 &len, is_write, attrs);
Paolo Bonzini715c31e2016-11-22 12:04:31 +01003215 if (this_mr != mr || xlat != base + done) {
3216 return done;
3217 }
3218 }
3219}
3220
aliguori6d16c2f2009-01-22 16:59:11 +00003221/* Map a physical memory region into a host virtual address.
3222 * May map a subset of the requested range, given by and returned in *plen.
3223 * May return NULL if resources needed to perform the mapping are exhausted.
3224 * Use only for reads OR writes - not for read-modify-write operations.
Mattias Nissler5c627192023-09-07 06:04:23 -07003225 * Use address_space_register_map_client() to know when retrying the map
3226 * operation is likely to succeed.
aliguori6d16c2f2009-01-22 16:59:11 +00003227 */
Avi Kivityac1970f2012-10-03 16:22:53 +02003228void *address_space_map(AddressSpace *as,
Avi Kivitya8170e52012-10-23 12:30:10 +02003229 hwaddr addr,
3230 hwaddr *plen,
Peter Maydellf26404f2018-05-31 14:50:52 +01003231 bool is_write,
3232 MemTxAttrs attrs)
aliguori6d16c2f2009-01-22 16:59:11 +00003233{
Avi Kivitya8170e52012-10-23 12:30:10 +02003234 hwaddr len = *plen;
Paolo Bonzini715c31e2016-11-22 12:04:31 +01003235 hwaddr l, xlat;
3236 MemoryRegion *mr;
Paolo Bonziniad0c60f2018-03-05 00:23:26 +01003237 FlatView *fv;
aliguori6d16c2f2009-01-22 16:59:11 +00003238
Alex Bennéed44fe132024-07-05 09:40:10 +01003239 trace_address_space_map(as, addr, len, is_write, *(uint32_t *) &attrs);
3240
Paolo Bonzinie3127ae2013-06-28 17:29:27 +02003241 if (len == 0) {
3242 return NULL;
3243 }
aliguori6d16c2f2009-01-22 16:59:11 +00003244
Paolo Bonzinie3127ae2013-06-28 17:29:27 +02003245 l = len;
Dr. David Alan Gilbert694ea272019-10-07 15:36:41 +01003246 RCU_READ_LOCK_GUARD();
Paolo Bonziniad0c60f2018-03-05 00:23:26 +01003247 fv = address_space_to_flatview(as);
Peter Maydellefa99a22018-05-31 14:50:52 +01003248 mr = flatview_translate(fv, addr, &xlat, &l, is_write, attrs);
Paolo Bonzini41063e12015-03-18 14:21:43 +01003249
Paolo Bonzinie3127ae2013-06-28 17:29:27 +02003250 if (!memory_access_is_direct(mr, is_write)) {
Mattias Nissler637b0aa2024-08-19 06:54:54 -07003251 size_t used = qatomic_read(&as->bounce_buffer_size);
3252 for (;;) {
3253 hwaddr alloc = MIN(as->max_bounce_buffer_size - used, l);
3254 size_t new_size = used + alloc;
3255 size_t actual =
3256 qatomic_cmpxchg(&as->bounce_buffer_size, used, new_size);
3257 if (actual == used) {
3258 l = alloc;
3259 break;
3260 }
3261 used = actual;
3262 }
3263
3264 if (l == 0) {
Prasad J Pandit77f55ea2020-05-26 16:47:43 +05303265 *plen = 0;
Paolo Bonzinie3127ae2013-06-28 17:29:27 +02003266 return NULL;
aliguori6d16c2f2009-01-22 16:59:11 +00003267 }
Paolo Bonzinid3e71552013-06-28 17:33:29 +02003268
Mattias Nissler637b0aa2024-08-19 06:54:54 -07003269 BounceBuffer *bounce = g_malloc0(l + sizeof(BounceBuffer));
3270 bounce->magic = BOUNCE_BUFFER_MAGIC;
Paolo Bonzinid3e71552013-06-28 17:33:29 +02003271 memory_region_ref(mr);
Mattias Nissler637b0aa2024-08-19 06:54:54 -07003272 bounce->mr = mr;
3273 bounce->addr = addr;
3274 bounce->len = l;
3275
Paolo Bonzinie3127ae2013-06-28 17:29:27 +02003276 if (!is_write) {
Fea.Wangd8d5ca42024-09-12 15:04:04 +08003277 flatview_read(fv, addr, attrs,
Mattias Nissler637b0aa2024-08-19 06:54:54 -07003278 bounce->buffer, l);
Stefano Stabellini8ab934f2011-06-27 18:26:06 +01003279 }
aliguori6d16c2f2009-01-22 16:59:11 +00003280
Paolo Bonzinie3127ae2013-06-28 17:29:27 +02003281 *plen = l;
Mattias Nissler637b0aa2024-08-19 06:54:54 -07003282 return bounce->buffer;
Paolo Bonzinie3127ae2013-06-28 17:29:27 +02003283 }
3284
Paolo Bonzinid3e71552013-06-28 17:33:29 +02003285 memory_region_ref(mr);
Alexey Kardashevskiy16620682017-09-21 18:50:58 +10003286 *plen = flatview_extend_translation(fv, addr, len, mr, xlat,
Peter Maydell53d07902018-05-31 14:50:52 +01003287 l, is_write, attrs);
Alexander Bulekovfc1c8342021-01-20 01:02:55 -05003288 fuzz_dma_read_cb(addr, *plen, mr);
Edgar E. Iglesias5a5585f2024-04-30 18:49:35 +02003289 return qemu_ram_ptr_length(mr->ram_block, xlat, plen, true, is_write);
aliguori6d16c2f2009-01-22 16:59:11 +00003290}
3291
Avi Kivityac1970f2012-10-03 16:22:53 +02003292/* Unmaps a memory region previously mapped by address_space_map().
Philippe Mathieu-Daudéae5883a2020-02-19 20:12:01 +01003293 * Will also mark the memory as dirty if is_write is true. access_len gives
aliguori6d16c2f2009-01-22 16:59:11 +00003294 * the amount of memory that was actually read or written by the caller.
3295 */
Avi Kivitya8170e52012-10-23 12:30:10 +02003296void address_space_unmap(AddressSpace *as, void *buffer, hwaddr len,
Philippe Mathieu-Daudéae5883a2020-02-19 20:12:01 +01003297 bool is_write, hwaddr access_len)
aliguori6d16c2f2009-01-22 16:59:11 +00003298{
Mattias Nissler637b0aa2024-08-19 06:54:54 -07003299 MemoryRegion *mr;
3300 ram_addr_t addr1;
Paolo Bonzinid3e71552013-06-28 17:33:29 +02003301
Mattias Nissler637b0aa2024-08-19 06:54:54 -07003302 mr = memory_region_from_host(buffer, &addr1);
3303 if (mr != NULL) {
aliguori6d16c2f2009-01-22 16:59:11 +00003304 if (is_write) {
Paolo Bonzini845b6212015-03-23 11:45:53 +01003305 invalidate_and_set_dirty(mr, addr1, access_len);
aliguori6d16c2f2009-01-22 16:59:11 +00003306 }
Jan Kiszka868bb332011-06-21 22:59:09 +02003307 if (xen_enabled()) {
Jan Kiszkae41d7c62011-06-21 22:59:08 +02003308 xen_invalidate_map_cache_entry(buffer);
Anthony PERARD050a0dd2010-09-16 13:57:49 +01003309 }
Paolo Bonzinid3e71552013-06-28 17:33:29 +02003310 memory_region_unref(mr);
aliguori6d16c2f2009-01-22 16:59:11 +00003311 return;
3312 }
Mattias Nissler637b0aa2024-08-19 06:54:54 -07003313
3314
3315 BounceBuffer *bounce = container_of(buffer, BounceBuffer, buffer);
3316 assert(bounce->magic == BOUNCE_BUFFER_MAGIC);
3317
aliguori6d16c2f2009-01-22 16:59:11 +00003318 if (is_write) {
Mattias Nissler637b0aa2024-08-19 06:54:54 -07003319 address_space_write(as, bounce->addr, MEMTXATTRS_UNSPECIFIED,
3320 bounce->buffer, access_len);
aliguori6d16c2f2009-01-22 16:59:11 +00003321 }
Mattias Nissler637b0aa2024-08-19 06:54:54 -07003322
3323 qatomic_sub(&as->bounce_buffer_size, bounce->len);
3324 bounce->magic = ~BOUNCE_BUFFER_MAGIC;
3325 memory_region_unref(bounce->mr);
3326 g_free(bounce);
3327 /* Write bounce_buffer_size before reading map_client_list. */
3328 smp_mb();
Mattias Nissler5c627192023-09-07 06:04:23 -07003329 address_space_notify_map_clients(as);
aliguori6d16c2f2009-01-22 16:59:11 +00003330}
bellardd0ecd2a2006-04-23 17:14:48 +00003331
Avi Kivitya8170e52012-10-23 12:30:10 +02003332void *cpu_physical_memory_map(hwaddr addr,
3333 hwaddr *plen,
Philippe Mathieu-Daudé28c80bf2020-02-19 20:32:30 +01003334 bool is_write)
Avi Kivityac1970f2012-10-03 16:22:53 +02003335{
Peter Maydellf26404f2018-05-31 14:50:52 +01003336 return address_space_map(&address_space_memory, addr, plen, is_write,
3337 MEMTXATTRS_UNSPECIFIED);
Avi Kivityac1970f2012-10-03 16:22:53 +02003338}
3339
Avi Kivitya8170e52012-10-23 12:30:10 +02003340void cpu_physical_memory_unmap(void *buffer, hwaddr len,
Philippe Mathieu-Daudé28c80bf2020-02-19 20:32:30 +01003341 bool is_write, hwaddr access_len)
Avi Kivityac1970f2012-10-03 16:22:53 +02003342{
3343 return address_space_unmap(&address_space_memory, buffer, len, is_write, access_len);
3344}
3345
Paolo Bonzini0ce265f2016-11-22 11:34:02 +01003346#define ARG1_DECL AddressSpace *as
3347#define ARG1 as
3348#define SUFFIX
3349#define TRANSLATE(...) address_space_translate(as, __VA_ARGS__)
Paolo Bonzini0ce265f2016-11-22 11:34:02 +01003350#define RCU_READ_LOCK(...) rcu_read_lock()
3351#define RCU_READ_UNLOCK(...) rcu_read_unlock()
Paolo Bonzini139c1832020-02-04 12:41:01 +01003352#include "memory_ldst.c.inc"
Alexander Graf1e78bcc2011-07-06 09:09:23 +02003353
Paolo Bonzini1f4e4962016-11-22 12:04:52 +01003354int64_t address_space_cache_init(MemoryRegionCache *cache,
3355 AddressSpace *as,
3356 hwaddr addr,
3357 hwaddr len,
3358 bool is_write)
3359{
Paolo Bonzini48564042018-03-18 18:26:36 +01003360 AddressSpaceDispatch *d;
3361 hwaddr l;
3362 MemoryRegion *mr;
Paolo Bonzini4bfb0242020-12-01 09:29:56 -05003363 Int128 diff;
Paolo Bonzini48564042018-03-18 18:26:36 +01003364
3365 assert(len > 0);
3366
3367 l = len;
3368 cache->fv = address_space_get_flatview(as);
3369 d = flatview_to_dispatch(cache->fv);
3370 cache->mrs = *address_space_translate_internal(d, addr, &cache->xlat, &l, true);
3371
Paolo Bonzini4bfb0242020-12-01 09:29:56 -05003372 /*
3373 * cache->xlat is now relative to cache->mrs.mr, not to the section itself.
3374 * Take that into account to compute how many bytes are there between
3375 * cache->xlat and the end of the section.
3376 */
3377 diff = int128_sub(cache->mrs.size,
Yeqi Fu48805df2023-03-15 11:26:49 +08003378 int128_make64(cache->xlat - cache->mrs.offset_within_region));
Paolo Bonzini4bfb0242020-12-01 09:29:56 -05003379 l = int128_get64(int128_min(diff, int128_make64(l)));
3380
Paolo Bonzini48564042018-03-18 18:26:36 +01003381 mr = cache->mrs.mr;
3382 memory_region_ref(mr);
3383 if (memory_access_is_direct(mr, is_write)) {
Peter Maydell53d07902018-05-31 14:50:52 +01003384 /* We don't care about the memory attributes here as we're only
3385 * doing this if we found actual RAM, which behaves the same
3386 * regardless of attributes; so UNSPECIFIED is fine.
3387 */
Paolo Bonzini48564042018-03-18 18:26:36 +01003388 l = flatview_extend_translation(cache->fv, addr, len, mr,
Peter Maydell53d07902018-05-31 14:50:52 +01003389 cache->xlat, l, is_write,
3390 MEMTXATTRS_UNSPECIFIED);
Edgar E. Iglesias5a5585f2024-04-30 18:49:35 +02003391 cache->ptr = qemu_ram_ptr_length(mr->ram_block, cache->xlat, &l, true,
3392 is_write);
Paolo Bonzini48564042018-03-18 18:26:36 +01003393 } else {
3394 cache->ptr = NULL;
3395 }
3396
3397 cache->len = l;
3398 cache->is_write = is_write;
3399 return l;
Paolo Bonzini1f4e4962016-11-22 12:04:52 +01003400}
3401
3402void address_space_cache_invalidate(MemoryRegionCache *cache,
3403 hwaddr addr,
3404 hwaddr access_len)
3405{
Paolo Bonzini48564042018-03-18 18:26:36 +01003406 assert(cache->is_write);
3407 if (likely(cache->ptr)) {
3408 invalidate_and_set_dirty(cache->mrs.mr, addr + cache->xlat, access_len);
3409 }
Paolo Bonzini1f4e4962016-11-22 12:04:52 +01003410}
3411
3412void address_space_cache_destroy(MemoryRegionCache *cache)
3413{
Paolo Bonzini48564042018-03-18 18:26:36 +01003414 if (!cache->mrs.mr) {
3415 return;
3416 }
3417
3418 if (xen_enabled()) {
3419 xen_invalidate_map_cache_entry(cache->ptr);
3420 }
3421 memory_region_unref(cache->mrs.mr);
3422 flatview_unref(cache->fv);
3423 cache->mrs.mr = NULL;
3424 cache->fv = NULL;
3425}
3426
3427/* Called from RCU critical section. This function has the same
3428 * semantics as address_space_translate, but it only works on a
3429 * predefined range of a MemoryRegion that was mapped with
3430 * address_space_cache_init.
3431 */
3432static inline MemoryRegion *address_space_translate_cached(
3433 MemoryRegionCache *cache, hwaddr addr, hwaddr *xlat,
Peter Maydellbc6b1ce2018-05-31 14:50:52 +01003434 hwaddr *plen, bool is_write, MemTxAttrs attrs)
Paolo Bonzini48564042018-03-18 18:26:36 +01003435{
3436 MemoryRegionSection section;
3437 MemoryRegion *mr;
3438 IOMMUMemoryRegion *iommu_mr;
3439 AddressSpace *target_as;
3440
3441 assert(!cache->ptr);
3442 *xlat = addr + cache->xlat;
3443
3444 mr = cache->mrs.mr;
3445 iommu_mr = memory_region_get_iommu(mr);
3446 if (!iommu_mr) {
3447 /* MMIO region. */
3448 return mr;
3449 }
3450
3451 section = address_space_translate_iommu(iommu_mr, xlat, plen,
3452 NULL, is_write, true,
Peter Maydell2f7b0092018-05-31 14:50:53 +01003453 &target_as, attrs);
Paolo Bonzini48564042018-03-18 18:26:36 +01003454 return section.mr;
3455}
3456
Jonathan Cameron47293c92024-03-07 15:37:10 +00003457/* Called within RCU critical section. */
3458static MemTxResult address_space_write_continue_cached(MemTxAttrs attrs,
3459 const void *ptr,
3460 hwaddr len,
3461 hwaddr mr_addr,
3462 hwaddr l,
3463 MemoryRegion *mr)
3464{
3465 MemTxResult result = MEMTX_OK;
3466 const uint8_t *buf = ptr;
3467
3468 for (;;) {
3469 result |= flatview_write_continue_step(attrs, buf, len, mr_addr, &l,
3470 mr);
3471
3472 len -= l;
3473 buf += l;
3474 mr_addr += l;
3475
3476 if (!len) {
3477 break;
3478 }
3479
3480 l = len;
3481 }
3482
3483 return result;
3484}
3485
3486/* Called within RCU critical section. */
3487static MemTxResult address_space_read_continue_cached(MemTxAttrs attrs,
3488 void *ptr, hwaddr len,
3489 hwaddr mr_addr, hwaddr l,
3490 MemoryRegion *mr)
3491{
3492 MemTxResult result = MEMTX_OK;
3493 uint8_t *buf = ptr;
3494
3495 for (;;) {
3496 result |= flatview_read_continue_step(attrs, buf, len, mr_addr, &l, mr);
3497 len -= l;
3498 buf += l;
3499 mr_addr += l;
3500
3501 if (!len) {
3502 break;
3503 }
3504 l = len;
3505 }
3506
3507 return result;
3508}
3509
Paolo Bonzini48564042018-03-18 18:26:36 +01003510/* Called from RCU critical section. address_space_read_cached uses this
3511 * out of line function when the target is an MMIO or IOMMU region.
3512 */
Philippe Mathieu-Daudé38df19f2020-05-18 17:53:02 +02003513MemTxResult
Paolo Bonzini48564042018-03-18 18:26:36 +01003514address_space_read_cached_slow(MemoryRegionCache *cache, hwaddr addr,
Li Zhijian0c249ff2019-01-17 20:49:01 +08003515 void *buf, hwaddr len)
Paolo Bonzini48564042018-03-18 18:26:36 +01003516{
Jonathan Cameron4c7c8562024-03-07 15:37:07 +00003517 hwaddr mr_addr, l;
Paolo Bonzini48564042018-03-18 18:26:36 +01003518 MemoryRegion *mr;
3519
3520 l = len;
Jonathan Cameron4c7c8562024-03-07 15:37:07 +00003521 mr = address_space_translate_cached(cache, addr, &mr_addr, &l, false,
Peter Maydellbc6b1ce2018-05-31 14:50:52 +01003522 MEMTXATTRS_UNSPECIFIED);
Jonathan Cameron47293c92024-03-07 15:37:10 +00003523 return address_space_read_continue_cached(MEMTXATTRS_UNSPECIFIED,
3524 buf, len, mr_addr, l, mr);
Paolo Bonzini48564042018-03-18 18:26:36 +01003525}
3526
3527/* Called from RCU critical section. address_space_write_cached uses this
3528 * out of line function when the target is an MMIO or IOMMU region.
3529 */
Philippe Mathieu-Daudé38df19f2020-05-18 17:53:02 +02003530MemTxResult
Paolo Bonzini48564042018-03-18 18:26:36 +01003531address_space_write_cached_slow(MemoryRegionCache *cache, hwaddr addr,
Li Zhijian0c249ff2019-01-17 20:49:01 +08003532 const void *buf, hwaddr len)
Paolo Bonzini48564042018-03-18 18:26:36 +01003533{
Jonathan Cameron4c7c8562024-03-07 15:37:07 +00003534 hwaddr mr_addr, l;
Paolo Bonzini48564042018-03-18 18:26:36 +01003535 MemoryRegion *mr;
3536
3537 l = len;
Jonathan Cameron4c7c8562024-03-07 15:37:07 +00003538 mr = address_space_translate_cached(cache, addr, &mr_addr, &l, true,
Peter Maydellbc6b1ce2018-05-31 14:50:52 +01003539 MEMTXATTRS_UNSPECIFIED);
Jonathan Cameron47293c92024-03-07 15:37:10 +00003540 return address_space_write_continue_cached(MEMTXATTRS_UNSPECIFIED,
3541 buf, len, mr_addr, l, mr);
Paolo Bonzini1f4e4962016-11-22 12:04:52 +01003542}
3543
3544#define ARG1_DECL MemoryRegionCache *cache
3545#define ARG1 cache
Paolo Bonzini48564042018-03-18 18:26:36 +01003546#define SUFFIX _cached_slow
3547#define TRANSLATE(...) address_space_translate_cached(cache, __VA_ARGS__)
Paolo Bonzini48564042018-03-18 18:26:36 +01003548#define RCU_READ_LOCK() ((void)0)
3549#define RCU_READ_UNLOCK() ((void)0)
Paolo Bonzini139c1832020-02-04 12:41:01 +01003550#include "memory_ldst.c.inc"
Paolo Bonzini1f4e4962016-11-22 12:04:52 +01003551
aliguori5e2972f2009-03-28 17:51:36 +00003552/* virtual memory access for debug (includes writing to ROM) */
Philippe Mathieu-Daudé73842ef2022-02-03 02:13:28 +01003553int cpu_memory_rw_debug(CPUState *cpu, vaddr addr,
3554 void *ptr, size_t len, bool is_write)
bellard13eb76e2004-01-24 15:23:36 +00003555{
Avi Kivitya8170e52012-10-23 12:30:10 +02003556 hwaddr phys_addr;
Philippe Mathieu-Daudé73842ef2022-02-03 02:13:28 +01003557 vaddr l, page;
Philippe Mathieu-Daudéd7ef71e2020-02-19 20:02:11 +01003558 uint8_t *buf = ptr;
bellard13eb76e2004-01-24 15:23:36 +00003559
Christian Borntraeger79ca7a12017-03-07 15:19:08 +01003560 cpu_synchronize_state(cpu);
bellard13eb76e2004-01-24 15:23:36 +00003561 while (len > 0) {
Peter Maydell5232e4c2016-01-21 14:15:06 +00003562 int asidx;
3563 MemTxAttrs attrs;
Philippe Mathieu-Daudéddfc8b92020-05-18 17:53:03 +02003564 MemTxResult res;
Peter Maydell5232e4c2016-01-21 14:15:06 +00003565
bellard13eb76e2004-01-24 15:23:36 +00003566 page = addr & TARGET_PAGE_MASK;
Peter Maydell5232e4c2016-01-21 14:15:06 +00003567 phys_addr = cpu_get_phys_page_attrs_debug(cpu, page, &attrs);
3568 asidx = cpu_asidx_from_attrs(cpu, attrs);
bellard13eb76e2004-01-24 15:23:36 +00003569 /* if no physical page mapped, return an error */
3570 if (phys_addr == -1)
3571 return -1;
3572 l = (page + TARGET_PAGE_SIZE) - addr;
3573 if (l > len)
3574 l = len;
aliguori5e2972f2009-03-28 17:51:36 +00003575 phys_addr += (addr & ~TARGET_PAGE_MASK);
Edgar E. Iglesias2e388472013-12-13 16:31:02 +10003576 if (is_write) {
Philippe Mathieu-Daudéddfc8b92020-05-18 17:53:03 +02003577 res = address_space_write_rom(cpu->cpu_ases[asidx].as, phys_addr,
3578 attrs, buf, l);
Edgar E. Iglesias2e388472013-12-13 16:31:02 +10003579 } else {
Philippe Mathieu-Daudéddfc8b92020-05-18 17:53:03 +02003580 res = address_space_read(cpu->cpu_ases[asidx].as, phys_addr,
3581 attrs, buf, l);
3582 }
3583 if (res != MEMTX_OK) {
3584 return -1;
Edgar E. Iglesias2e388472013-12-13 16:31:02 +10003585 }
bellard13eb76e2004-01-24 15:23:36 +00003586 len -= l;
3587 buf += l;
3588 addr += l;
3589 }
3590 return 0;
3591}
Dr. David Alan Gilbert038629a2015-11-05 18:10:29 +00003592
Avi Kivitya8170e52012-10-23 12:30:10 +02003593bool cpu_physical_memory_is_io(hwaddr phys_addr)
Wen Congyang76f35532012-05-07 12:04:18 +08003594{
Paolo Bonzini5c8a00c2013-05-29 12:42:00 +02003595 MemoryRegion*mr;
Paolo Bonzini149f54b2013-05-24 12:59:37 +02003596 hwaddr l = 1;
Wen Congyang76f35532012-05-07 12:04:18 +08003597
Dr. David Alan Gilbert694ea272019-10-07 15:36:41 +01003598 RCU_READ_LOCK_GUARD();
Paolo Bonzini5c8a00c2013-05-29 12:42:00 +02003599 mr = address_space_translate(&address_space_memory,
Peter Maydellbc6b1ce2018-05-31 14:50:52 +01003600 phys_addr, &phys_addr, &l, false,
3601 MEMTXATTRS_UNSPECIFIED);
Wen Congyang76f35532012-05-07 12:04:18 +08003602
Markus Armbruster66997c42022-11-22 14:49:16 +01003603 return !(memory_region_is_ram(mr) || memory_region_is_romd(mr));
Wen Congyang76f35532012-05-07 12:04:18 +08003604}
Michael R. Hinesbd2fa512013-06-25 21:35:34 -04003605
Dr. David Alan Gilberte3807052015-05-21 13:24:13 +01003606int qemu_ram_foreach_block(RAMBlockIterFunc func, void *opaque)
Michael R. Hinesbd2fa512013-06-25 21:35:34 -04003607{
3608 RAMBlock *block;
Dr. David Alan Gilberte3807052015-05-21 13:24:13 +01003609 int ret = 0;
Michael R. Hinesbd2fa512013-06-25 21:35:34 -04003610
Dr. David Alan Gilbert694ea272019-10-07 15:36:41 +01003611 RCU_READ_LOCK_GUARD();
Peter Xu99e15582017-05-12 12:17:39 +08003612 RAMBLOCK_FOREACH(block) {
Yury Kotov754cb9c2019-02-15 20:45:44 +03003613 ret = func(block, opaque);
Dr. David Alan Gilberte3807052015-05-21 13:24:13 +01003614 if (ret) {
3615 break;
3616 }
Michael R. Hinesbd2fa512013-06-25 21:35:34 -04003617 }
Dr. David Alan Gilberte3807052015-05-21 13:24:13 +01003618 return ret;
Michael R. Hinesbd2fa512013-06-25 21:35:34 -04003619}
Dr. David Alan Gilbertd3a50382017-02-24 18:28:32 +00003620
3621/*
3622 * Unmap pages of memory from start to start+length such that
3623 * they a) read as 0, b) Trigger whatever fault mechanism
3624 * the OS provides for postcopy.
3625 * The pages must be unmapped by the end of the function.
3626 * Returns: 0 on success, none-0 on failure
3627 *
3628 */
3629int ram_block_discard_range(RAMBlock *rb, uint64_t start, size_t length)
3630{
3631 int ret = -1;
3632
3633 uint8_t *host_startaddr = rb->host + start;
3634
Marc-André Lureau619bd312020-01-03 11:39:58 +04003635 if (!QEMU_PTR_IS_ALIGNED(host_startaddr, rb->page_size)) {
Xiaoyao Liea18be72024-01-24 21:33:28 -05003636 error_report("%s: Unaligned start address: %p",
3637 __func__, host_startaddr);
Dr. David Alan Gilbertd3a50382017-02-24 18:28:32 +00003638 goto err;
3639 }
3640
David Hildenbranddcdc4602021-04-29 13:27:03 +02003641 if ((start + length) <= rb->max_length) {
Dr. David Alan Gilbertdb144f72018-03-12 17:20:56 +00003642 bool need_madvise, need_fallocate;
Marc-André Lureau619bd312020-01-03 11:39:58 +04003643 if (!QEMU_IS_ALIGNED(length, rb->page_size)) {
Xiaoyao Liea18be72024-01-24 21:33:28 -05003644 error_report("%s: Unaligned length: %zx", __func__, length);
Dr. David Alan Gilbertd3a50382017-02-24 18:28:32 +00003645 goto err;
3646 }
3647
3648 errno = ENOTSUP; /* If we are missing MADVISE etc */
3649
Dr. David Alan Gilbertdb144f72018-03-12 17:20:56 +00003650 /* The logic here is messy;
3651 * madvise DONTNEED fails for hugepages
3652 * fallocate works on hugepages and shmem
David Hildenbrandcdfa56c2021-04-06 10:01:25 +02003653 * shared anonymous memory requires madvise REMOVE
Dr. David Alan Gilbertdb144f72018-03-12 17:20:56 +00003654 */
Richard Henderson80c3aee2024-01-02 12:57:48 +11003655 need_madvise = (rb->page_size == qemu_real_host_page_size());
Dr. David Alan Gilbertdb144f72018-03-12 17:20:56 +00003656 need_fallocate = rb->fd != -1;
3657 if (need_fallocate) {
3658 /* For a file, this causes the area of the file to be zero'd
3659 * if read, and for hugetlbfs also causes it to be unmapped
3660 * so a userfault will trigger.
Dr. David Alan Gilberte2fa71f2017-02-24 18:28:33 +00003661 */
3662#ifdef CONFIG_FALLOCATE_PUNCH_HOLE
David Hildenbrand1d44ff52023-07-06 09:56:06 +02003663 /*
David Hildenbrandb2cccb52023-09-06 14:04:57 +02003664 * fallocate() will fail with readonly files. Let's print a
3665 * proper error message.
3666 */
3667 if (rb->flags & RAM_READONLY_FD) {
Xiaoyao Liea18be72024-01-24 21:33:28 -05003668 error_report("%s: Discarding RAM with readonly files is not"
3669 " supported", __func__);
David Hildenbrandb2cccb52023-09-06 14:04:57 +02003670 goto err;
3671
3672 }
3673 /*
David Hildenbrand1d44ff52023-07-06 09:56:06 +02003674 * We'll discard data from the actual file, even though we only
3675 * have a MAP_PRIVATE mapping, possibly messing with other
3676 * MAP_PRIVATE/MAP_SHARED mappings. There is no easy way to
3677 * change that behavior whithout violating the promised
3678 * semantics of ram_block_discard_range().
3679 *
3680 * Only warn, because it works as long as nobody else uses that
3681 * file.
3682 */
3683 if (!qemu_ram_is_shared(rb)) {
Xiaoyao Liea18be72024-01-24 21:33:28 -05003684 warn_report_once("%s: Discarding RAM"
David Hildenbrand1d44ff52023-07-06 09:56:06 +02003685 " in private file mappings is possibly"
3686 " dangerous, because it will modify the"
3687 " underlying file and will affect other"
Xiaoyao Liea18be72024-01-24 21:33:28 -05003688 " users of the file", __func__);
David Hildenbrand1d44ff52023-07-06 09:56:06 +02003689 }
3690
Dr. David Alan Gilberte2fa71f2017-02-24 18:28:33 +00003691 ret = fallocate(rb->fd, FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE,
3692 start, length);
Dr. David Alan Gilbertdb144f72018-03-12 17:20:56 +00003693 if (ret) {
3694 ret = -errno;
Xiaoyao Liea18be72024-01-24 21:33:28 -05003695 error_report("%s: Failed to fallocate %s:%" PRIx64 " +%zx (%d)",
3696 __func__, rb->idstr, start, length, ret);
Dr. David Alan Gilbertdb144f72018-03-12 17:20:56 +00003697 goto err;
3698 }
3699#else
3700 ret = -ENOSYS;
Xiaoyao Liea18be72024-01-24 21:33:28 -05003701 error_report("%s: fallocate not available/file"
Dr. David Alan Gilbertd3a50382017-02-24 18:28:32 +00003702 "%s:%" PRIx64 " +%zx (%d)",
Xiaoyao Liea18be72024-01-24 21:33:28 -05003703 __func__, rb->idstr, start, length, ret);
Dr. David Alan Gilbertdb144f72018-03-12 17:20:56 +00003704 goto err;
3705#endif
Dr. David Alan Gilbertd3a50382017-02-24 18:28:32 +00003706 }
Dr. David Alan Gilbertdb144f72018-03-12 17:20:56 +00003707 if (need_madvise) {
3708 /* For normal RAM this causes it to be unmapped,
3709 * for shared memory it causes the local mapping to disappear
3710 * and to fall back on the file contents (which we just
3711 * fallocate'd away).
3712 */
3713#if defined(CONFIG_MADVISE)
David Hildenbrandcdfa56c2021-04-06 10:01:25 +02003714 if (qemu_ram_is_shared(rb) && rb->fd < 0) {
3715 ret = madvise(host_startaddr, length, QEMU_MADV_REMOVE);
3716 } else {
3717 ret = madvise(host_startaddr, length, QEMU_MADV_DONTNEED);
3718 }
Dr. David Alan Gilbertdb144f72018-03-12 17:20:56 +00003719 if (ret) {
3720 ret = -errno;
Xiaoyao Liea18be72024-01-24 21:33:28 -05003721 error_report("%s: Failed to discard range "
Dr. David Alan Gilbertdb144f72018-03-12 17:20:56 +00003722 "%s:%" PRIx64 " +%zx (%d)",
Xiaoyao Liea18be72024-01-24 21:33:28 -05003723 __func__, rb->idstr, start, length, ret);
Dr. David Alan Gilbertdb144f72018-03-12 17:20:56 +00003724 goto err;
3725 }
3726#else
3727 ret = -ENOSYS;
Xiaoyao Liea18be72024-01-24 21:33:28 -05003728 error_report("%s: MADVISE not available %s:%" PRIx64 " +%zx (%d)",
3729 __func__, rb->idstr, start, length, ret);
Dr. David Alan Gilbertdb144f72018-03-12 17:20:56 +00003730 goto err;
3731#endif
3732 }
3733 trace_ram_block_discard_range(rb->idstr, host_startaddr, length,
3734 need_madvise, need_fallocate, ret);
Dr. David Alan Gilbertd3a50382017-02-24 18:28:32 +00003735 } else {
Xiaoyao Liea18be72024-01-24 21:33:28 -05003736 error_report("%s: Overrun block '%s' (%" PRIu64 "/%zx/" RAM_ADDR_FMT")",
3737 __func__, rb->idstr, start, length, rb->max_length);
Dr. David Alan Gilbertd3a50382017-02-24 18:28:32 +00003738 }
3739
3740err:
3741 return ret;
3742}
3743
Xiaoyao Lib2e94262024-03-20 03:39:07 -05003744int ram_block_discard_guest_memfd_range(RAMBlock *rb, uint64_t start,
3745 size_t length)
3746{
3747 int ret = -1;
3748
3749#ifdef CONFIG_FALLOCATE_PUNCH_HOLE
3750 ret = fallocate(rb->guest_memfd, FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE,
3751 start, length);
3752
3753 if (ret) {
3754 ret = -errno;
3755 error_report("%s: Failed to fallocate %s:%" PRIx64 " +%zx (%d)",
3756 __func__, rb->idstr, start, length, ret);
3757 }
3758#else
3759 ret = -ENOSYS;
3760 error_report("%s: fallocate not available %s:%" PRIx64 " +%zx (%d)",
3761 __func__, rb->idstr, start, length, ret);
3762#endif
3763
3764 return ret;
3765}
3766
Junyan Hea4de8552018-07-18 15:48:00 +08003767bool ramblock_is_pmem(RAMBlock *rb)
3768{
3769 return rb->flags & RAM_PMEM;
3770}
3771
Markus Armbrusterb6b71cb2019-04-17 21:17:56 +02003772static void mtree_print_phys_entries(int start, int end, int skip, int ptr)
Alexey Kardashevskiy5e8fd942017-09-21 18:51:06 +10003773{
3774 if (start == end - 1) {
Markus Armbrusterb6b71cb2019-04-17 21:17:56 +02003775 qemu_printf("\t%3d ", start);
Alexey Kardashevskiy5e8fd942017-09-21 18:51:06 +10003776 } else {
Markus Armbrusterb6b71cb2019-04-17 21:17:56 +02003777 qemu_printf("\t%3d..%-3d ", start, end - 1);
Alexey Kardashevskiy5e8fd942017-09-21 18:51:06 +10003778 }
Markus Armbrusterb6b71cb2019-04-17 21:17:56 +02003779 qemu_printf(" skip=%d ", skip);
Alexey Kardashevskiy5e8fd942017-09-21 18:51:06 +10003780 if (ptr == PHYS_MAP_NODE_NIL) {
Markus Armbrusterb6b71cb2019-04-17 21:17:56 +02003781 qemu_printf(" ptr=NIL");
Alexey Kardashevskiy5e8fd942017-09-21 18:51:06 +10003782 } else if (!skip) {
Markus Armbrusterb6b71cb2019-04-17 21:17:56 +02003783 qemu_printf(" ptr=#%d", ptr);
Alexey Kardashevskiy5e8fd942017-09-21 18:51:06 +10003784 } else {
Markus Armbrusterb6b71cb2019-04-17 21:17:56 +02003785 qemu_printf(" ptr=[%d]", ptr);
Alexey Kardashevskiy5e8fd942017-09-21 18:51:06 +10003786 }
Markus Armbrusterb6b71cb2019-04-17 21:17:56 +02003787 qemu_printf("\n");
Alexey Kardashevskiy5e8fd942017-09-21 18:51:06 +10003788}
3789
3790#define MR_SIZE(size) (int128_nz(size) ? (hwaddr)int128_get64( \
3791 int128_sub((size), int128_one())) : 0)
3792
Markus Armbrusterb6b71cb2019-04-17 21:17:56 +02003793void mtree_print_dispatch(AddressSpaceDispatch *d, MemoryRegion *root)
Alexey Kardashevskiy5e8fd942017-09-21 18:51:06 +10003794{
3795 int i;
3796
Markus Armbrusterb6b71cb2019-04-17 21:17:56 +02003797 qemu_printf(" Dispatch\n");
3798 qemu_printf(" Physical sections\n");
Alexey Kardashevskiy5e8fd942017-09-21 18:51:06 +10003799
3800 for (i = 0; i < d->map.sections_nb; ++i) {
3801 MemoryRegionSection *s = d->map.sections + i;
3802 const char *names[] = { " [unassigned]", " [not dirty]",
3803 " [ROM]", " [watch]" };
3804
Philippe Mathieu-Daudé883f2c52023-01-10 22:29:47 +01003805 qemu_printf(" #%d @" HWADDR_FMT_plx ".." HWADDR_FMT_plx
Markus Armbrusterb6b71cb2019-04-17 21:17:56 +02003806 " %s%s%s%s%s",
Alexey Kardashevskiy5e8fd942017-09-21 18:51:06 +10003807 i,
3808 s->offset_within_address_space,
Zhenzhong Duanf9c307c2022-06-22 17:59:12 +08003809 s->offset_within_address_space + MR_SIZE(s->size),
Alexey Kardashevskiy5e8fd942017-09-21 18:51:06 +10003810 s->mr->name ? s->mr->name : "(noname)",
3811 i < ARRAY_SIZE(names) ? names[i] : "",
3812 s->mr == root ? " [ROOT]" : "",
3813 s == d->mru_section ? " [MRU]" : "",
3814 s->mr->is_iommu ? " [iommu]" : "");
3815
3816 if (s->mr->alias) {
Markus Armbrusterb6b71cb2019-04-17 21:17:56 +02003817 qemu_printf(" alias=%s", s->mr->alias->name ?
Alexey Kardashevskiy5e8fd942017-09-21 18:51:06 +10003818 s->mr->alias->name : "noname");
3819 }
Markus Armbrusterb6b71cb2019-04-17 21:17:56 +02003820 qemu_printf("\n");
Alexey Kardashevskiy5e8fd942017-09-21 18:51:06 +10003821 }
3822
Markus Armbrusterb6b71cb2019-04-17 21:17:56 +02003823 qemu_printf(" Nodes (%d bits per level, %d levels) ptr=[%d] skip=%d\n",
Alexey Kardashevskiy5e8fd942017-09-21 18:51:06 +10003824 P_L2_BITS, P_L2_LEVELS, d->phys_map.ptr, d->phys_map.skip);
3825 for (i = 0; i < d->map.nodes_nb; ++i) {
3826 int j, jprev;
3827 PhysPageEntry prev;
3828 Node *n = d->map.nodes + i;
3829
Markus Armbrusterb6b71cb2019-04-17 21:17:56 +02003830 qemu_printf(" [%d]\n", i);
Alexey Kardashevskiy5e8fd942017-09-21 18:51:06 +10003831
3832 for (j = 0, jprev = 0, prev = *n[0]; j < ARRAY_SIZE(*n); ++j) {
3833 PhysPageEntry *pe = *n + j;
3834
3835 if (pe->ptr == prev.ptr && pe->skip == prev.skip) {
3836 continue;
3837 }
3838
Markus Armbrusterb6b71cb2019-04-17 21:17:56 +02003839 mtree_print_phys_entries(jprev, j, prev.skip, prev.ptr);
Alexey Kardashevskiy5e8fd942017-09-21 18:51:06 +10003840
3841 jprev = j;
3842 prev = *pe;
3843 }
3844
3845 if (jprev != ARRAY_SIZE(*n)) {
Markus Armbrusterb6b71cb2019-04-17 21:17:56 +02003846 mtree_print_phys_entries(jprev, j, prev.skip, prev.ptr);
Alexey Kardashevskiy5e8fd942017-09-21 18:51:06 +10003847 }
3848 }
3849}
3850
David Hildenbrand7e6d32e2021-04-13 11:55:29 +02003851/* Require any discards to work. */
David Hildenbrand98da4912021-04-13 11:55:28 +02003852static unsigned int ram_block_discard_required_cnt;
David Hildenbrand7e6d32e2021-04-13 11:55:29 +02003853/* Require only coordinated discards to work. */
3854static unsigned int ram_block_coordinated_discard_required_cnt;
3855/* Disable any discards. */
David Hildenbrand98da4912021-04-13 11:55:28 +02003856static unsigned int ram_block_discard_disabled_cnt;
David Hildenbrand7e6d32e2021-04-13 11:55:29 +02003857/* Disable only uncoordinated discards. */
3858static unsigned int ram_block_uncoordinated_discard_disabled_cnt;
David Hildenbrand98da4912021-04-13 11:55:28 +02003859static QemuMutex ram_block_discard_disable_mutex;
3860
3861static void ram_block_discard_disable_mutex_lock(void)
3862{
3863 static gsize initialized;
3864
3865 if (g_once_init_enter(&initialized)) {
3866 qemu_mutex_init(&ram_block_discard_disable_mutex);
3867 g_once_init_leave(&initialized, 1);
3868 }
3869 qemu_mutex_lock(&ram_block_discard_disable_mutex);
3870}
3871
3872static void ram_block_discard_disable_mutex_unlock(void)
3873{
3874 qemu_mutex_unlock(&ram_block_discard_disable_mutex);
3875}
David Hildenbrandd24f31d2020-06-26 09:22:29 +02003876
3877int ram_block_discard_disable(bool state)
3878{
David Hildenbrand98da4912021-04-13 11:55:28 +02003879 int ret = 0;
David Hildenbrandd24f31d2020-06-26 09:22:29 +02003880
David Hildenbrand98da4912021-04-13 11:55:28 +02003881 ram_block_discard_disable_mutex_lock();
David Hildenbrandd24f31d2020-06-26 09:22:29 +02003882 if (!state) {
David Hildenbrand98da4912021-04-13 11:55:28 +02003883 ram_block_discard_disabled_cnt--;
David Hildenbrand7e6d32e2021-04-13 11:55:29 +02003884 } else if (ram_block_discard_required_cnt ||
3885 ram_block_coordinated_discard_required_cnt) {
David Hildenbrand98da4912021-04-13 11:55:28 +02003886 ret = -EBUSY;
David Hildenbrand7e6d32e2021-04-13 11:55:29 +02003887 } else {
3888 ram_block_discard_disabled_cnt++;
3889 }
3890 ram_block_discard_disable_mutex_unlock();
3891 return ret;
3892}
3893
3894int ram_block_uncoordinated_discard_disable(bool state)
3895{
3896 int ret = 0;
3897
3898 ram_block_discard_disable_mutex_lock();
3899 if (!state) {
3900 ram_block_uncoordinated_discard_disabled_cnt--;
3901 } else if (ram_block_discard_required_cnt) {
3902 ret = -EBUSY;
3903 } else {
3904 ram_block_uncoordinated_discard_disabled_cnt++;
David Hildenbrandd24f31d2020-06-26 09:22:29 +02003905 }
David Hildenbrand98da4912021-04-13 11:55:28 +02003906 ram_block_discard_disable_mutex_unlock();
3907 return ret;
David Hildenbrandd24f31d2020-06-26 09:22:29 +02003908}
3909
3910int ram_block_discard_require(bool state)
3911{
David Hildenbrand98da4912021-04-13 11:55:28 +02003912 int ret = 0;
David Hildenbrandd24f31d2020-06-26 09:22:29 +02003913
David Hildenbrand98da4912021-04-13 11:55:28 +02003914 ram_block_discard_disable_mutex_lock();
David Hildenbrandd24f31d2020-06-26 09:22:29 +02003915 if (!state) {
David Hildenbrand98da4912021-04-13 11:55:28 +02003916 ram_block_discard_required_cnt--;
David Hildenbrand7e6d32e2021-04-13 11:55:29 +02003917 } else if (ram_block_discard_disabled_cnt ||
3918 ram_block_uncoordinated_discard_disabled_cnt) {
David Hildenbrand98da4912021-04-13 11:55:28 +02003919 ret = -EBUSY;
David Hildenbrand7e6d32e2021-04-13 11:55:29 +02003920 } else {
3921 ram_block_discard_required_cnt++;
3922 }
3923 ram_block_discard_disable_mutex_unlock();
3924 return ret;
3925}
3926
3927int ram_block_coordinated_discard_require(bool state)
3928{
3929 int ret = 0;
3930
3931 ram_block_discard_disable_mutex_lock();
3932 if (!state) {
3933 ram_block_coordinated_discard_required_cnt--;
3934 } else if (ram_block_discard_disabled_cnt) {
3935 ret = -EBUSY;
3936 } else {
3937 ram_block_coordinated_discard_required_cnt++;
David Hildenbrandd24f31d2020-06-26 09:22:29 +02003938 }
David Hildenbrand98da4912021-04-13 11:55:28 +02003939 ram_block_discard_disable_mutex_unlock();
3940 return ret;
David Hildenbrandd24f31d2020-06-26 09:22:29 +02003941}
3942
3943bool ram_block_discard_is_disabled(void)
3944{
David Hildenbrand7e6d32e2021-04-13 11:55:29 +02003945 return qatomic_read(&ram_block_discard_disabled_cnt) ||
3946 qatomic_read(&ram_block_uncoordinated_discard_disabled_cnt);
David Hildenbrandd24f31d2020-06-26 09:22:29 +02003947}
3948
3949bool ram_block_discard_is_required(void)
3950{
David Hildenbrand7e6d32e2021-04-13 11:55:29 +02003951 return qatomic_read(&ram_block_discard_required_cnt) ||
3952 qatomic_read(&ram_block_coordinated_discard_required_cnt);
David Hildenbrandd24f31d2020-06-26 09:22:29 +02003953}