blob: fc2b0fee01883903f696c80d8dfac99647a22c77 [file] [log] [blame]
bellard54936002003-05-13 00:25:15 +00001/*
Paolo Bonzinid9f24bf2020-10-06 09:05:29 +02002 * RAM allocation and memory access
ths5fafdf22007-09-16 21:08:06 +00003 *
bellard54936002003-05-13 00:25:15 +00004 * Copyright (c) 2003 Fabrice Bellard
5 *
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
Chetan Pant61f3c912020-10-23 12:44:24 +00009 * version 2.1 of the License, or (at your option) any later version.
bellard54936002003-05-13 00:25:15 +000010 *
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
15 *
16 * You should have received a copy of the GNU Lesser General Public
Blue Swirl8167ee82009-07-16 20:47:01 +000017 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
bellard54936002003-05-13 00:25:15 +000018 */
Markus Armbruster14a48c12019-05-23 16:35:05 +020019
Peter Maydell7b31bbc2016-01-26 18:16:56 +000020#include "qemu/osdep.h"
Marc-André Lureauec5f7ca2022-03-23 19:57:34 +040021#include "exec/page-vary.h"
Markus Armbrusterda34e652016-03-14 09:01:28 +010022#include "qapi/error.h"
bellard54936002003-05-13 00:25:15 +000023
Veronia Bahaaf348b6d2016-03-20 19:16:19 +020024#include "qemu/cutils.h"
Richard Henderson084cfca2020-12-14 08:02:33 -060025#include "qemu/cacheflush.h"
Markus Armbrustere2c1c342022-12-21 14:35:49 +010026#include "qemu/hbitmap.h"
Peter Maydellb85ea5f2022-02-08 20:08:52 +000027#include "qemu/madvise.h"
Claudio Fontana78271682021-02-04 17:39:23 +010028
29#ifdef CONFIG_TCG
30#include "hw/core/tcg-cpu-ops.h"
31#endif /* CONFIG_TCG */
32
Paolo Bonzini63c91552016-03-15 13:18:37 +010033#include "exec/exec-all.h"
Juan Quintela51180422017-04-24 20:50:19 +020034#include "exec/target_page.h"
Paolo Bonzini741da0d2014-06-27 08:40:04 +020035#include "hw/qdev-core.h"
Fam Zhengc7e002c2017-07-14 10:15:08 +080036#include "hw/qdev-properties.h"
Marcel Apfelbaum47c8ca52015-02-04 17:43:54 +020037#include "hw/boards.h"
Paolo Bonzini33c11872016-03-15 16:58:45 +010038#include "hw/xen/xen.h"
Paolo Bonzini9c17d612012-12-17 18:20:04 +010039#include "sysemu/kvm.h"
Markus Armbruster14a48c12019-05-23 16:35:05 +020040#include "sysemu/tcg.h"
Alexander Bulekova028ede2020-02-19 23:11:09 -050041#include "sysemu/qtest.h"
Paolo Bonzini1de7afc2012-12-17 18:20:00 +010042#include "qemu/timer.h"
43#include "qemu/config-file.h"
Andreas Färber75a34032013-09-02 16:57:02 +020044#include "qemu/error-report.h"
Markus Armbrusterb6b71cb2019-04-17 21:17:56 +020045#include "qemu/qemu-print.h"
Philippe Mathieu-Daudé3ab6fdc2021-12-15 19:24:21 +010046#include "qemu/log.h"
Peter Maydell5df022c2022-02-26 18:07:23 +000047#include "qemu/memalign.h"
Paolo Bonzini741da0d2014-06-27 08:40:04 +020048#include "exec/memory.h"
Paolo Bonzinidf43d492016-03-16 10:24:54 +010049#include "exec/ioport.h"
Paolo Bonzini741da0d2014-06-27 08:40:04 +020050#include "sysemu/dma.h"
Markus Armbrusterb58c5c22019-08-12 07:23:55 +020051#include "sysemu/hostmem.h"
Christian Borntraeger79ca7a12017-03-07 15:19:08 +010052#include "sysemu/hw_accel.h"
Paolo Bonzini9c17d612012-12-17 18:20:04 +010053#include "sysemu/xen-mapcache.h"
Paolo Bonzini243af022020-02-04 12:20:10 +010054#include "trace/trace-root.h"
Dr. David Alan Gilbertd3a50382017-02-24 18:28:32 +000055
Dr. David Alan Gilberte2fa71f2017-02-24 18:28:33 +000056#ifdef CONFIG_FALLOCATE_PUNCH_HOLE
Dr. David Alan Gilberte2fa71f2017-02-24 18:28:33 +000057#include <linux/falloc.h>
58#endif
59
Mike Day0dc3f442013-09-05 14:41:35 -040060#include "qemu/rcu_queue.h"
Jan Kiszka4840f102015-06-18 18:47:22 +020061#include "qemu/main-loop.h"
Paolo Bonzini3b9bd3f2020-12-16 13:27:58 +010062#include "exec/translate-all.h"
Pavel Dovgalyuk76159362015-09-17 19:25:07 +030063#include "sysemu/replay.h"
Blue Swirl0cac1b62012-04-09 16:50:52 +000064
Paolo Bonzini022c62c2012-12-17 18:19:49 +010065#include "exec/memory-internal.h"
Juan Quintela220c3eb2013-10-14 17:13:59 +020066#include "exec/ram_addr.h"
Avi Kivity67d95c12011-12-15 15:25:22 +020067
Beata Michalska61c490e2019-11-21 00:08:41 +000068#include "qemu/pmem.h"
69
Bharata B Rao9dfeca72016-05-12 09:18:12 +053070#include "migration/vmstate.h"
71
Michael S. Tsirkinb35ba302013-11-11 17:52:07 +020072#include "qemu/range.h"
Michael S. Tsirkin794e8f32015-09-24 14:41:17 +030073#ifndef _WIN32
74#include "qemu/mmap-alloc.h"
75#endif
Michael S. Tsirkinb35ba302013-11-11 17:52:07 +020076
Peter Xube9b23c2017-05-12 12:17:41 +080077#include "monitor/monitor.h"
78
Jingqi Liuce317be2020-04-29 16:50:09 +080079#ifdef CONFIG_LIBDAXCTL
80#include <daxctl/libdaxctl.h>
81#endif
82
blueswir1db7b5422007-05-26 17:36:03 +000083//#define DEBUG_SUBPAGE
ths1196be32007-03-17 15:17:58 +000084
Mike Day0dc3f442013-09-05 14:41:35 -040085/* ram_list is read under rcu_read_lock()/rcu_read_unlock(). Writes
86 * are protected by the ramlist lock.
87 */
Mike Day0d53d9f2015-01-21 13:45:24 +010088RAMList ram_list = { .blocks = QLIST_HEAD_INITIALIZER(ram_list.blocks) };
Avi Kivity62152b82011-07-26 14:26:14 +030089
90static MemoryRegion *system_memory;
Avi Kivity309cb472011-08-08 16:09:03 +030091static MemoryRegion *system_io;
Avi Kivity62152b82011-07-26 14:26:14 +030092
Avi Kivityf6790af2012-10-02 20:13:51 +020093AddressSpace address_space_io;
94AddressSpace address_space_memory;
Avi Kivity2673a5d2012-10-02 18:49:28 +020095
Jan Kiszkaacc9d802013-05-26 21:55:37 +020096static MemoryRegion io_mem_unassigned;
Avi Kivity4346ae32012-02-10 17:00:01 +020097
Paolo Bonzini1db8abb2013-05-21 12:07:21 +020098typedef struct PhysPageEntry PhysPageEntry;
99
100struct PhysPageEntry {
Michael S. Tsirkin9736e552013-11-11 14:42:43 +0200101 /* How many bits skip to next level (in units of L2_SIZE). 0 for a leaf. */
Michael S. Tsirkin8b795762013-11-11 14:51:56 +0200102 uint32_t skip : 6;
Michael S. Tsirkin9736e552013-11-11 14:42:43 +0200103 /* index into phys_sections (!skip) or phys_map_nodes (skip) */
Michael S. Tsirkin8b795762013-11-11 14:51:56 +0200104 uint32_t ptr : 26;
Paolo Bonzini1db8abb2013-05-21 12:07:21 +0200105};
106
Michael S. Tsirkin8b795762013-11-11 14:51:56 +0200107#define PHYS_MAP_NODE_NIL (((uint32_t)~0) >> 6)
108
Paolo Bonzini03f49952013-11-07 17:14:36 +0100109/* Size of the L2 (and L3, etc) page tables. */
Paolo Bonzini57271d62013-11-07 17:14:37 +0100110#define ADDR_SPACE_BITS 64
Paolo Bonzini03f49952013-11-07 17:14:36 +0100111
Michael S. Tsirkin026736c2013-11-13 20:13:03 +0200112#define P_L2_BITS 9
Paolo Bonzini03f49952013-11-07 17:14:36 +0100113#define P_L2_SIZE (1 << P_L2_BITS)
114
115#define P_L2_LEVELS (((ADDR_SPACE_BITS - TARGET_PAGE_BITS - 1) / P_L2_BITS) + 1)
116
117typedef PhysPageEntry Node[P_L2_SIZE];
Paolo Bonzini0475d942013-05-29 12:28:21 +0200118
Marcel Apfelbaum53cb28c2013-12-01 14:02:23 +0200119typedef struct PhysPageMap {
Paolo Bonzini79e2b9a2015-01-21 12:09:14 +0100120 struct rcu_head rcu;
121
Marcel Apfelbaum53cb28c2013-12-01 14:02:23 +0200122 unsigned sections_nb;
123 unsigned sections_nb_alloc;
124 unsigned nodes_nb;
125 unsigned nodes_nb_alloc;
126 Node *nodes;
127 MemoryRegionSection *sections;
128} PhysPageMap;
129
Paolo Bonzini1db8abb2013-05-21 12:07:21 +0200130struct AddressSpaceDispatch {
Fam Zheng729633c2016-03-01 14:18:24 +0800131 MemoryRegionSection *mru_section;
Paolo Bonzini1db8abb2013-05-21 12:07:21 +0200132 /* This is a multi-level map on the physical address space.
133 * The bottom level has pointers to MemoryRegionSections.
134 */
135 PhysPageEntry phys_map;
Marcel Apfelbaum53cb28c2013-12-01 14:02:23 +0200136 PhysPageMap map;
Paolo Bonzini1db8abb2013-05-21 12:07:21 +0200137};
138
Jan Kiszka90260c62013-05-26 21:46:51 +0200139#define SUBPAGE_IDX(addr) ((addr) & ~TARGET_PAGE_MASK)
140typedef struct subpage_t {
141 MemoryRegion iomem;
Alexey Kardashevskiy16620682017-09-21 18:50:58 +1000142 FlatView *fv;
Jan Kiszka90260c62013-05-26 21:46:51 +0200143 hwaddr base;
Vijaya Kumar K2615fab2016-10-24 16:26:49 +0100144 uint16_t sub_section[];
Jan Kiszka90260c62013-05-26 21:46:51 +0200145} subpage_t;
146
Liu Ping Fanb41aac42013-05-29 11:09:17 +0200147#define PHYS_SECTION_UNASSIGNED 0
Avi Kivity5312bd82012-02-12 18:32:55 +0200148
pbrooke2eef172008-06-08 01:09:01 +0000149static void io_mem_init(void);
Avi Kivity62152b82011-07-26 14:26:14 +0300150static void memory_map_init(void);
Paolo Bonzini9458a9a2018-02-06 18:37:39 +0100151static void tcg_log_global_after_sync(MemoryListener *listener);
Edgar E. Iglesias09daed82013-12-17 13:06:51 +1000152static void tcg_commit(MemoryListener *listener);
pbrooke2eef172008-06-08 01:09:01 +0000153
Peter Maydell32857f42015-10-01 15:29:50 +0100154/**
155 * CPUAddressSpace: all the information a CPU needs about an AddressSpace
156 * @cpu: the CPU whose AddressSpace this is
157 * @as: the AddressSpace itself
158 * @memory_dispatch: its dispatch pointer (cached, RCU protected)
159 * @tcg_as_listener: listener for tracking changes to the AddressSpace
160 */
161struct CPUAddressSpace {
162 CPUState *cpu;
163 AddressSpace *as;
164 struct AddressSpaceDispatch *memory_dispatch;
165 MemoryListener tcg_as_listener;
166};
167
Gerd Hoffmann8deaf122017-04-21 11:16:25 +0200168struct DirtyBitmapSnapshot {
169 ram_addr_t start;
170 ram_addr_t end;
171 unsigned long dirty[];
172};
173
Marcel Apfelbaum53cb28c2013-12-01 14:02:23 +0200174static void phys_map_node_reserve(PhysPageMap *map, unsigned nodes)
Avi Kivityf7bf5462012-02-13 20:12:05 +0200175{
Peter Lieven101420b2016-07-15 12:03:50 +0200176 static unsigned alloc_hint = 16;
Marcel Apfelbaum53cb28c2013-12-01 14:02:23 +0200177 if (map->nodes_nb + nodes > map->nodes_nb_alloc) {
Wei Yangc95cfd02019-03-21 16:25:52 +0800178 map->nodes_nb_alloc = MAX(alloc_hint, map->nodes_nb + nodes);
Marcel Apfelbaum53cb28c2013-12-01 14:02:23 +0200179 map->nodes = g_renew(Node, map->nodes, map->nodes_nb_alloc);
Peter Lieven101420b2016-07-15 12:03:50 +0200180 alloc_hint = map->nodes_nb_alloc;
Avi Kivityf7bf5462012-02-13 20:12:05 +0200181 }
182}
183
Paolo Bonzinidb946042015-05-21 15:12:29 +0200184static uint32_t phys_map_node_alloc(PhysPageMap *map, bool leaf)
Avi Kivityd6f2ea22012-02-12 20:12:49 +0200185{
186 unsigned i;
Michael S. Tsirkin8b795762013-11-11 14:51:56 +0200187 uint32_t ret;
Paolo Bonzinidb946042015-05-21 15:12:29 +0200188 PhysPageEntry e;
189 PhysPageEntry *p;
Avi Kivityd6f2ea22012-02-12 20:12:49 +0200190
Marcel Apfelbaum53cb28c2013-12-01 14:02:23 +0200191 ret = map->nodes_nb++;
Paolo Bonzinidb946042015-05-21 15:12:29 +0200192 p = map->nodes[ret];
Avi Kivityd6f2ea22012-02-12 20:12:49 +0200193 assert(ret != PHYS_MAP_NODE_NIL);
Marcel Apfelbaum53cb28c2013-12-01 14:02:23 +0200194 assert(ret != map->nodes_nb_alloc);
Paolo Bonzinidb946042015-05-21 15:12:29 +0200195
196 e.skip = leaf ? 0 : 1;
197 e.ptr = leaf ? PHYS_SECTION_UNASSIGNED : PHYS_MAP_NODE_NIL;
Paolo Bonzini03f49952013-11-07 17:14:36 +0100198 for (i = 0; i < P_L2_SIZE; ++i) {
Paolo Bonzinidb946042015-05-21 15:12:29 +0200199 memcpy(&p[i], &e, sizeof(e));
Avi Kivityd6f2ea22012-02-12 20:12:49 +0200200 }
Avi Kivityf7bf5462012-02-13 20:12:05 +0200201 return ret;
Avi Kivityd6f2ea22012-02-12 20:12:49 +0200202}
203
Marcel Apfelbaum53cb28c2013-12-01 14:02:23 +0200204static void phys_page_set_level(PhysPageMap *map, PhysPageEntry *lp,
Wei Yang56b15072019-03-21 16:25:50 +0800205 hwaddr *index, uint64_t *nb, uint16_t leaf,
Avi Kivity29990972012-02-13 20:21:20 +0200206 int level)
Avi Kivityf7bf5462012-02-13 20:12:05 +0200207{
208 PhysPageEntry *p;
Paolo Bonzini03f49952013-11-07 17:14:36 +0100209 hwaddr step = (hwaddr)1 << (level * P_L2_BITS);
Avi Kivityf7bf5462012-02-13 20:12:05 +0200210
Michael S. Tsirkin9736e552013-11-11 14:42:43 +0200211 if (lp->skip && lp->ptr == PHYS_MAP_NODE_NIL) {
Paolo Bonzinidb946042015-05-21 15:12:29 +0200212 lp->ptr = phys_map_node_alloc(map, level == 0);
Avi Kivityf7bf5462012-02-13 20:12:05 +0200213 }
Paolo Bonzinidb946042015-05-21 15:12:29 +0200214 p = map->nodes[lp->ptr];
Paolo Bonzini03f49952013-11-07 17:14:36 +0100215 lp = &p[(*index >> (level * P_L2_BITS)) & (P_L2_SIZE - 1)];
Avi Kivityf7bf5462012-02-13 20:12:05 +0200216
Paolo Bonzini03f49952013-11-07 17:14:36 +0100217 while (*nb && lp < &p[P_L2_SIZE]) {
Avi Kivity07f07b32012-02-13 20:45:32 +0200218 if ((*index & (step - 1)) == 0 && *nb >= step) {
Michael S. Tsirkin9736e552013-11-11 14:42:43 +0200219 lp->skip = 0;
Avi Kivityc19e8802012-02-13 20:25:31 +0200220 lp->ptr = leaf;
Avi Kivity07f07b32012-02-13 20:45:32 +0200221 *index += step;
222 *nb -= step;
Avi Kivity29990972012-02-13 20:21:20 +0200223 } else {
Marcel Apfelbaum53cb28c2013-12-01 14:02:23 +0200224 phys_page_set_level(map, lp, index, nb, leaf, level - 1);
Avi Kivity29990972012-02-13 20:21:20 +0200225 }
226 ++lp;
Avi Kivityf7bf5462012-02-13 20:12:05 +0200227 }
228}
229
Avi Kivityac1970f2012-10-03 16:22:53 +0200230static void phys_page_set(AddressSpaceDispatch *d,
Wei Yang56b15072019-03-21 16:25:50 +0800231 hwaddr index, uint64_t nb,
Avi Kivity29990972012-02-13 20:21:20 +0200232 uint16_t leaf)
bellard92e873b2004-05-21 14:52:29 +0000233{
Avi Kivity29990972012-02-13 20:21:20 +0200234 /* Wildly overreserve - it doesn't matter much. */
Marcel Apfelbaum53cb28c2013-12-01 14:02:23 +0200235 phys_map_node_reserve(&d->map, 3 * P_L2_LEVELS);
bellard92e873b2004-05-21 14:52:29 +0000236
Marcel Apfelbaum53cb28c2013-12-01 14:02:23 +0200237 phys_page_set_level(&d->map, &d->phys_map, &index, &nb, leaf, P_L2_LEVELS - 1);
bellard92e873b2004-05-21 14:52:29 +0000238}
239
Michael S. Tsirkinb35ba302013-11-11 17:52:07 +0200240/* Compact a non leaf page entry. Simply detect that the entry has a single child,
241 * and update our entry so we can skip it and go directly to the destination.
242 */
Marc-André Lureauefee6782016-09-28 16:37:20 +0400243static void phys_page_compact(PhysPageEntry *lp, Node *nodes)
Michael S. Tsirkinb35ba302013-11-11 17:52:07 +0200244{
245 unsigned valid_ptr = P_L2_SIZE;
246 int valid = 0;
247 PhysPageEntry *p;
248 int i;
249
250 if (lp->ptr == PHYS_MAP_NODE_NIL) {
251 return;
252 }
253
254 p = nodes[lp->ptr];
255 for (i = 0; i < P_L2_SIZE; i++) {
256 if (p[i].ptr == PHYS_MAP_NODE_NIL) {
257 continue;
258 }
259
260 valid_ptr = i;
261 valid++;
262 if (p[i].skip) {
Marc-André Lureauefee6782016-09-28 16:37:20 +0400263 phys_page_compact(&p[i], nodes);
Michael S. Tsirkinb35ba302013-11-11 17:52:07 +0200264 }
265 }
266
267 /* We can only compress if there's only one child. */
268 if (valid != 1) {
269 return;
270 }
271
272 assert(valid_ptr < P_L2_SIZE);
273
274 /* Don't compress if it won't fit in the # of bits we have. */
Wei Yang526ca232019-03-21 16:25:55 +0800275 if (P_L2_LEVELS >= (1 << 6) &&
276 lp->skip + p[valid_ptr].skip >= (1 << 6)) {
Michael S. Tsirkinb35ba302013-11-11 17:52:07 +0200277 return;
278 }
279
280 lp->ptr = p[valid_ptr].ptr;
281 if (!p[valid_ptr].skip) {
282 /* If our only child is a leaf, make this a leaf. */
283 /* By design, we should have made this node a leaf to begin with so we
284 * should never reach here.
285 * But since it's so simple to handle this, let's do it just in case we
286 * change this rule.
287 */
288 lp->skip = 0;
289 } else {
290 lp->skip += p[valid_ptr].skip;
291 }
292}
293
Alexey Kardashevskiy8629d3f2017-09-21 18:51:00 +1000294void address_space_dispatch_compact(AddressSpaceDispatch *d)
Michael S. Tsirkinb35ba302013-11-11 17:52:07 +0200295{
Michael S. Tsirkinb35ba302013-11-11 17:52:07 +0200296 if (d->phys_map.skip) {
Marc-André Lureauefee6782016-09-28 16:37:20 +0400297 phys_page_compact(&d->phys_map, d->map.nodes);
Michael S. Tsirkinb35ba302013-11-11 17:52:07 +0200298 }
299}
300
Fam Zheng29cb5332016-03-01 14:18:23 +0800301static inline bool section_covers_addr(const MemoryRegionSection *section,
302 hwaddr addr)
303{
304 /* Memory topology clips a memory region to [0, 2^64); size.hi > 0 means
305 * the section must cover the entire address space.
306 */
Richard Henderson258dfaa2016-06-29 15:48:03 -0700307 return int128_gethi(section->size) ||
Fam Zheng29cb5332016-03-01 14:18:23 +0800308 range_covers_byte(section->offset_within_address_space,
Richard Henderson258dfaa2016-06-29 15:48:03 -0700309 int128_getlo(section->size), addr);
Fam Zheng29cb5332016-03-01 14:18:23 +0800310}
311
Peter Xu003a0cf2017-05-15 16:50:57 +0800312static MemoryRegionSection *phys_page_find(AddressSpaceDispatch *d, hwaddr addr)
bellard92e873b2004-05-21 14:52:29 +0000313{
Peter Xu003a0cf2017-05-15 16:50:57 +0800314 PhysPageEntry lp = d->phys_map, *p;
315 Node *nodes = d->map.nodes;
316 MemoryRegionSection *sections = d->map.sections;
Michael S. Tsirkin97115a82013-11-13 20:08:19 +0200317 hwaddr index = addr >> TARGET_PAGE_BITS;
Avi Kivity31ab2b42012-02-13 16:44:19 +0200318 int i;
Avi Kivityf1f6e3b2011-11-20 17:52:22 +0200319
Michael S. Tsirkin9736e552013-11-11 14:42:43 +0200320 for (i = P_L2_LEVELS; lp.skip && (i -= lp.skip) >= 0;) {
Avi Kivityc19e8802012-02-13 20:25:31 +0200321 if (lp.ptr == PHYS_MAP_NODE_NIL) {
Paolo Bonzini9affd6f2013-05-29 12:09:47 +0200322 return &sections[PHYS_SECTION_UNASSIGNED];
Avi Kivity31ab2b42012-02-13 16:44:19 +0200323 }
Paolo Bonzini9affd6f2013-05-29 12:09:47 +0200324 p = nodes[lp.ptr];
Paolo Bonzini03f49952013-11-07 17:14:36 +0100325 lp = p[(index >> (i * P_L2_BITS)) & (P_L2_SIZE - 1)];
Avi Kivityf1f6e3b2011-11-20 17:52:22 +0200326 }
Michael S. Tsirkinb35ba302013-11-11 17:52:07 +0200327
Fam Zheng29cb5332016-03-01 14:18:23 +0800328 if (section_covers_addr(&sections[lp.ptr], addr)) {
Michael S. Tsirkinb35ba302013-11-11 17:52:07 +0200329 return &sections[lp.ptr];
330 } else {
331 return &sections[PHYS_SECTION_UNASSIGNED];
332 }
Avi Kivityf3705d52012-03-08 16:16:34 +0200333}
334
Paolo Bonzini79e2b9a2015-01-21 12:09:14 +0100335/* Called from RCU critical section */
Paolo Bonzinic7086b42013-06-02 15:27:39 +0200336static MemoryRegionSection *address_space_lookup_region(AddressSpaceDispatch *d,
Jan Kiszka90260c62013-05-26 21:46:51 +0200337 hwaddr addr,
338 bool resolve_subpage)
Jan Kiszka9f029602013-05-06 16:48:02 +0200339{
Stefan Hajnoczid73415a2020-09-23 11:56:46 +0100340 MemoryRegionSection *section = qatomic_read(&d->mru_section);
Jan Kiszka90260c62013-05-26 21:46:51 +0200341 subpage_t *subpage;
342
Paolo Bonzini07c114b2017-11-15 15:11:03 +0100343 if (!section || section == &d->map.sections[PHYS_SECTION_UNASSIGNED] ||
344 !section_covers_addr(section, addr)) {
Peter Xu003a0cf2017-05-15 16:50:57 +0800345 section = phys_page_find(d, addr);
Stefan Hajnoczid73415a2020-09-23 11:56:46 +0100346 qatomic_set(&d->mru_section, section);
Fam Zheng729633c2016-03-01 14:18:24 +0800347 }
Jan Kiszka90260c62013-05-26 21:46:51 +0200348 if (resolve_subpage && section->mr->subpage) {
349 subpage = container_of(section->mr, subpage_t, iomem);
Marcel Apfelbaum53cb28c2013-12-01 14:02:23 +0200350 section = &d->map.sections[subpage->sub_section[SUBPAGE_IDX(addr)]];
Jan Kiszka90260c62013-05-26 21:46:51 +0200351 }
352 return section;
Jan Kiszka9f029602013-05-06 16:48:02 +0200353}
354
Paolo Bonzini79e2b9a2015-01-21 12:09:14 +0100355/* Called from RCU critical section */
Jan Kiszka90260c62013-05-26 21:46:51 +0200356static MemoryRegionSection *
Paolo Bonzinic7086b42013-06-02 15:27:39 +0200357address_space_translate_internal(AddressSpaceDispatch *d, hwaddr addr, hwaddr *xlat,
Jan Kiszka90260c62013-05-26 21:46:51 +0200358 hwaddr *plen, bool resolve_subpage)
Paolo Bonzini149f54b2013-05-24 12:59:37 +0200359{
360 MemoryRegionSection *section;
Paolo Bonzini965eb2f2015-06-17 10:40:27 +0200361 MemoryRegion *mr;
Paolo Bonzinia87f3952014-02-07 15:47:46 +0100362 Int128 diff;
Paolo Bonzini149f54b2013-05-24 12:59:37 +0200363
Paolo Bonzinic7086b42013-06-02 15:27:39 +0200364 section = address_space_lookup_region(d, addr, resolve_subpage);
Paolo Bonzini149f54b2013-05-24 12:59:37 +0200365 /* Compute offset within MemoryRegionSection */
366 addr -= section->offset_within_address_space;
367
368 /* Compute offset within MemoryRegion */
369 *xlat = addr + section->offset_within_region;
370
Paolo Bonzini965eb2f2015-06-17 10:40:27 +0200371 mr = section->mr;
Paolo Bonzinib242e0e2015-07-04 00:24:51 +0200372
373 /* MMIO registers can be expected to perform full-width accesses based only
374 * on their address, without considering adjacent registers that could
375 * decode to completely different MemoryRegions. When such registers
376 * exist (e.g. I/O ports 0xcf8 and 0xcf9 on most PC chipsets), MMIO
377 * regions overlap wildly. For this reason we cannot clamp the accesses
378 * here.
379 *
380 * If the length is small (as is the case for address_space_ldl/stl),
381 * everything works fine. If the incoming length is large, however,
382 * the caller really has to do the clamping through memory_access_size.
383 */
Paolo Bonzini965eb2f2015-06-17 10:40:27 +0200384 if (memory_region_is_ram(mr)) {
Paolo Bonzinie4a511f2015-06-17 10:36:54 +0200385 diff = int128_sub(section->size, int128_make64(addr));
Paolo Bonzini965eb2f2015-06-17 10:40:27 +0200386 *plen = int128_get64(int128_min(diff, int128_make64(*plen)));
387 }
Paolo Bonzini149f54b2013-05-24 12:59:37 +0200388 return section;
389}
Jan Kiszka90260c62013-05-26 21:46:51 +0200390
Peter Xud5e5faf2017-10-10 11:42:45 +0200391/**
Paolo Bonzinia411c842018-03-03 17:24:04 +0100392 * address_space_translate_iommu - translate an address through an IOMMU
393 * memory region and then through the target address space.
394 *
395 * @iommu_mr: the IOMMU memory region that we start the translation from
396 * @addr: the address to be translated through the MMU
397 * @xlat: the translated address offset within the destination memory region.
398 * It cannot be %NULL.
399 * @plen_out: valid read/write length of the translated address. It
400 * cannot be %NULL.
401 * @page_mask_out: page mask for the translated address. This
402 * should only be meaningful for IOMMU translated
403 * addresses, since there may be huge pages that this bit
404 * would tell. It can be %NULL if we don't care about it.
405 * @is_write: whether the translation operation is for write
406 * @is_mmio: whether this can be MMIO, set true if it can
407 * @target_as: the address space targeted by the IOMMU
Peter Maydell2f7b0092018-05-31 14:50:53 +0100408 * @attrs: transaction attributes
Paolo Bonzinia411c842018-03-03 17:24:04 +0100409 *
410 * This function is called from RCU critical section. It is the common
411 * part of flatview_do_translate and address_space_translate_cached.
412 */
413static MemoryRegionSection address_space_translate_iommu(IOMMUMemoryRegion *iommu_mr,
414 hwaddr *xlat,
415 hwaddr *plen_out,
416 hwaddr *page_mask_out,
417 bool is_write,
418 bool is_mmio,
Peter Maydell2f7b0092018-05-31 14:50:53 +0100419 AddressSpace **target_as,
420 MemTxAttrs attrs)
Paolo Bonzinia411c842018-03-03 17:24:04 +0100421{
422 MemoryRegionSection *section;
423 hwaddr page_mask = (hwaddr)-1;
424
425 do {
426 hwaddr addr = *xlat;
427 IOMMUMemoryRegionClass *imrc = memory_region_get_iommu_class_nocheck(iommu_mr);
Peter Maydell2c91bcf2018-06-15 14:57:16 +0100428 int iommu_idx = 0;
429 IOMMUTLBEntry iotlb;
430
431 if (imrc->attrs_to_index) {
432 iommu_idx = imrc->attrs_to_index(iommu_mr, attrs);
433 }
434
435 iotlb = imrc->translate(iommu_mr, addr, is_write ?
436 IOMMU_WO : IOMMU_RO, iommu_idx);
Paolo Bonzinia411c842018-03-03 17:24:04 +0100437
438 if (!(iotlb.perm & (1 << is_write))) {
439 goto unassigned;
440 }
441
442 addr = ((iotlb.translated_addr & ~iotlb.addr_mask)
443 | (addr & iotlb.addr_mask));
444 page_mask &= iotlb.addr_mask;
445 *plen_out = MIN(*plen_out, (addr | iotlb.addr_mask) - addr + 1);
446 *target_as = iotlb.target_as;
447
448 section = address_space_translate_internal(
449 address_space_to_dispatch(iotlb.target_as), addr, xlat,
450 plen_out, is_mmio);
451
452 iommu_mr = memory_region_get_iommu(section->mr);
453 } while (unlikely(iommu_mr));
454
455 if (page_mask_out) {
456 *page_mask_out = page_mask;
457 }
458 return *section;
459
460unassigned:
461 return (MemoryRegionSection) { .mr = &io_mem_unassigned };
462}
463
464/**
Peter Xud5e5faf2017-10-10 11:42:45 +0200465 * flatview_do_translate - translate an address in FlatView
466 *
467 * @fv: the flat view that we want to translate on
468 * @addr: the address to be translated in above address space
469 * @xlat: the translated address offset within memory region. It
470 * cannot be @NULL.
471 * @plen_out: valid read/write length of the translated address. It
472 * can be @NULL when we don't care about it.
473 * @page_mask_out: page mask for the translated address. This
474 * should only be meaningful for IOMMU translated
475 * addresses, since there may be huge pages that this bit
476 * would tell. It can be @NULL if we don't care about it.
477 * @is_write: whether the translation operation is for write
478 * @is_mmio: whether this can be MMIO, set true if it can
Paolo Bonziniad2804d2018-04-17 11:39:35 +0200479 * @target_as: the address space targeted by the IOMMU
Peter Maydell49e14aa2018-05-31 14:50:53 +0100480 * @attrs: memory transaction attributes
Peter Xud5e5faf2017-10-10 11:42:45 +0200481 *
482 * This function is called from RCU critical section
483 */
Alexey Kardashevskiy16620682017-09-21 18:50:58 +1000484static MemoryRegionSection flatview_do_translate(FlatView *fv,
485 hwaddr addr,
486 hwaddr *xlat,
Peter Xud5e5faf2017-10-10 11:42:45 +0200487 hwaddr *plen_out,
488 hwaddr *page_mask_out,
Alexey Kardashevskiy16620682017-09-21 18:50:58 +1000489 bool is_write,
490 bool is_mmio,
Peter Maydell49e14aa2018-05-31 14:50:53 +0100491 AddressSpace **target_as,
492 MemTxAttrs attrs)
Jan Kiszka90260c62013-05-26 21:46:51 +0200493{
Avi Kivity30951152012-10-30 13:47:46 +0200494 MemoryRegionSection *section;
Alexey Kardashevskiy3df9d742017-07-11 13:56:19 +1000495 IOMMUMemoryRegion *iommu_mr;
Peter Xud5e5faf2017-10-10 11:42:45 +0200496 hwaddr plen = (hwaddr)(-1);
497
Paolo Bonziniad2804d2018-04-17 11:39:35 +0200498 if (!plen_out) {
499 plen_out = &plen;
Peter Xud5e5faf2017-10-10 11:42:45 +0200500 }
Avi Kivity30951152012-10-30 13:47:46 +0200501
Paolo Bonzinia411c842018-03-03 17:24:04 +0100502 section = address_space_translate_internal(
503 flatview_to_dispatch(fv), addr, xlat,
504 plen_out, is_mmio);
Avi Kivity30951152012-10-30 13:47:46 +0200505
Paolo Bonzinia411c842018-03-03 17:24:04 +0100506 iommu_mr = memory_region_get_iommu(section->mr);
507 if (unlikely(iommu_mr)) {
508 return address_space_translate_iommu(iommu_mr, xlat,
509 plen_out, page_mask_out,
510 is_write, is_mmio,
Peter Maydell2f7b0092018-05-31 14:50:53 +0100511 target_as, attrs);
Avi Kivity30951152012-10-30 13:47:46 +0200512 }
Peter Xud5e5faf2017-10-10 11:42:45 +0200513 if (page_mask_out) {
Paolo Bonzinia411c842018-03-03 17:24:04 +0100514 /* Not behind an IOMMU, use default page size. */
515 *page_mask_out = ~TARGET_PAGE_MASK;
Peter Xud5e5faf2017-10-10 11:42:45 +0200516 }
517
Peter Xua7640402017-05-17 16:57:42 +0800518 return *section;
Peter Xua7640402017-05-17 16:57:42 +0800519}
520
521/* Called from RCU critical section */
522IOMMUTLBEntry address_space_get_iotlb_entry(AddressSpace *as, hwaddr addr,
Peter Maydell7446eb02018-05-31 14:50:53 +0100523 bool is_write, MemTxAttrs attrs)
Peter Xua7640402017-05-17 16:57:42 +0800524{
525 MemoryRegionSection section;
Peter Xu076a93d2017-10-10 11:42:46 +0200526 hwaddr xlat, page_mask;
Peter Xua7640402017-05-17 16:57:42 +0800527
Peter Xu076a93d2017-10-10 11:42:46 +0200528 /*
529 * This can never be MMIO, and we don't really care about plen,
530 * but page mask.
531 */
532 section = flatview_do_translate(address_space_to_flatview(as), addr, &xlat,
Peter Maydell49e14aa2018-05-31 14:50:53 +0100533 NULL, &page_mask, is_write, false, &as,
534 attrs);
Peter Xua7640402017-05-17 16:57:42 +0800535
536 /* Illegal translation */
537 if (section.mr == &io_mem_unassigned) {
538 goto iotlb_fail;
539 }
540
541 /* Convert memory region offset into address space offset */
542 xlat += section.offset_within_address_space -
543 section.offset_within_region;
544
Peter Xua7640402017-05-17 16:57:42 +0800545 return (IOMMUTLBEntry) {
Alexey Kardashevskiye76bb182017-09-21 18:50:53 +1000546 .target_as = as,
Peter Xu076a93d2017-10-10 11:42:46 +0200547 .iova = addr & ~page_mask,
548 .translated_addr = xlat & ~page_mask,
549 .addr_mask = page_mask,
Peter Xua7640402017-05-17 16:57:42 +0800550 /* IOTLBs are for DMAs, and DMA only allows on RAMs. */
551 .perm = IOMMU_RW,
552 };
553
554iotlb_fail:
555 return (IOMMUTLBEntry) {0};
556}
557
558/* Called from RCU critical section */
Alexey Kardashevskiy16620682017-09-21 18:50:58 +1000559MemoryRegion *flatview_translate(FlatView *fv, hwaddr addr, hwaddr *xlat,
Peter Maydellefa99a22018-05-31 14:50:52 +0100560 hwaddr *plen, bool is_write,
561 MemTxAttrs attrs)
Peter Xua7640402017-05-17 16:57:42 +0800562{
563 MemoryRegion *mr;
564 MemoryRegionSection section;
Alexey Kardashevskiy16620682017-09-21 18:50:58 +1000565 AddressSpace *as = NULL;
Peter Xua7640402017-05-17 16:57:42 +0800566
567 /* This can be MMIO, so setup MMIO bit. */
Peter Xud5e5faf2017-10-10 11:42:45 +0200568 section = flatview_do_translate(fv, addr, xlat, plen, NULL,
Peter Maydell49e14aa2018-05-31 14:50:53 +0100569 is_write, true, &as, attrs);
Peter Xua7640402017-05-17 16:57:42 +0800570 mr = section.mr;
571
Alexey Kardashevskiyfe680d02014-05-07 13:40:39 +0000572 if (xen_enabled() && memory_access_is_direct(mr, is_write)) {
Paolo Bonzinia87f3952014-02-07 15:47:46 +0100573 hwaddr page = ((addr & TARGET_PAGE_MASK) + TARGET_PAGE_SIZE) - addr;
Peter Crosthwaite23820db2015-03-16 22:35:54 -0700574 *plen = MIN(page, *plen);
Paolo Bonzinia87f3952014-02-07 15:47:46 +0100575 }
576
Avi Kivity30951152012-10-30 13:47:46 +0200577 return mr;
Jan Kiszka90260c62013-05-26 21:46:51 +0200578}
579
Peter Maydell1f871c52018-06-15 14:57:16 +0100580typedef struct TCGIOMMUNotifier {
581 IOMMUNotifier n;
582 MemoryRegion *mr;
583 CPUState *cpu;
584 int iommu_idx;
585 bool active;
586} TCGIOMMUNotifier;
587
588static void tcg_iommu_unmap_notify(IOMMUNotifier *n, IOMMUTLBEntry *iotlb)
589{
590 TCGIOMMUNotifier *notifier = container_of(n, TCGIOMMUNotifier, n);
591
592 if (!notifier->active) {
593 return;
594 }
595 tlb_flush(notifier->cpu);
596 notifier->active = false;
597 /* We leave the notifier struct on the list to avoid reallocating it later.
598 * Generally the number of IOMMUs a CPU deals with will be small.
599 * In any case we can't unregister the iommu notifier from a notify
600 * callback.
601 */
602}
603
604static void tcg_register_iommu_notifier(CPUState *cpu,
605 IOMMUMemoryRegion *iommu_mr,
606 int iommu_idx)
607{
608 /* Make sure this CPU has an IOMMU notifier registered for this
609 * IOMMU/IOMMU index combination, so that we can flush its TLB
610 * when the IOMMU tells us the mappings we've cached have changed.
611 */
612 MemoryRegion *mr = MEMORY_REGION(iommu_mr);
Philippe Mathieu-Daudébbf90192021-01-17 18:04:11 +0100613 TCGIOMMUNotifier *notifier = NULL;
Markus Armbruster805d4492020-07-22 10:40:48 +0200614 int i;
Peter Maydell1f871c52018-06-15 14:57:16 +0100615
616 for (i = 0; i < cpu->iommu_notifiers->len; i++) {
Peter Maydell5601be32019-02-01 14:55:45 +0000617 notifier = g_array_index(cpu->iommu_notifiers, TCGIOMMUNotifier *, i);
Peter Maydell1f871c52018-06-15 14:57:16 +0100618 if (notifier->mr == mr && notifier->iommu_idx == iommu_idx) {
619 break;
620 }
621 }
622 if (i == cpu->iommu_notifiers->len) {
623 /* Not found, add a new entry at the end of the array */
624 cpu->iommu_notifiers = g_array_set_size(cpu->iommu_notifiers, i + 1);
Peter Maydell5601be32019-02-01 14:55:45 +0000625 notifier = g_new0(TCGIOMMUNotifier, 1);
626 g_array_index(cpu->iommu_notifiers, TCGIOMMUNotifier *, i) = notifier;
Peter Maydell1f871c52018-06-15 14:57:16 +0100627
628 notifier->mr = mr;
629 notifier->iommu_idx = iommu_idx;
630 notifier->cpu = cpu;
631 /* Rather than trying to register interest in the specific part
632 * of the iommu's address space that we've accessed and then
633 * expand it later as subsequent accesses touch more of it, we
634 * just register interest in the whole thing, on the assumption
635 * that iommu reconfiguration will be rare.
636 */
637 iommu_notifier_init(&notifier->n,
638 tcg_iommu_unmap_notify,
639 IOMMU_NOTIFIER_UNMAP,
640 0,
641 HWADDR_MAX,
642 iommu_idx);
Markus Armbruster805d4492020-07-22 10:40:48 +0200643 memory_region_register_iommu_notifier(notifier->mr, &notifier->n,
644 &error_fatal);
Peter Maydell1f871c52018-06-15 14:57:16 +0100645 }
646
647 if (!notifier->active) {
648 notifier->active = true;
649 }
650}
651
Paolo Bonzinid9f24bf2020-10-06 09:05:29 +0200652void tcg_iommu_free_notifier_list(CPUState *cpu)
Peter Maydell1f871c52018-06-15 14:57:16 +0100653{
654 /* Destroy the CPU's notifier list */
655 int i;
656 TCGIOMMUNotifier *notifier;
657
658 for (i = 0; i < cpu->iommu_notifiers->len; i++) {
Peter Maydell5601be32019-02-01 14:55:45 +0000659 notifier = g_array_index(cpu->iommu_notifiers, TCGIOMMUNotifier *, i);
Peter Maydell1f871c52018-06-15 14:57:16 +0100660 memory_region_unregister_iommu_notifier(notifier->mr, &notifier->n);
Peter Maydell5601be32019-02-01 14:55:45 +0000661 g_free(notifier);
Peter Maydell1f871c52018-06-15 14:57:16 +0100662 }
663 g_array_free(cpu->iommu_notifiers, true);
664}
665
Paolo Bonzinid9f24bf2020-10-06 09:05:29 +0200666void tcg_iommu_init_notifier_list(CPUState *cpu)
667{
668 cpu->iommu_notifiers = g_array_new(false, true, sizeof(TCGIOMMUNotifier *));
669}
670
Paolo Bonzini79e2b9a2015-01-21 12:09:14 +0100671/* Called from RCU critical section */
Jan Kiszka90260c62013-05-26 21:46:51 +0200672MemoryRegionSection *
Richard Henderson418ade72022-06-21 08:38:29 -0700673address_space_translate_for_iotlb(CPUState *cpu, int asidx, hwaddr orig_addr,
Peter Maydell1f871c52018-06-15 14:57:16 +0100674 hwaddr *xlat, hwaddr *plen,
675 MemTxAttrs attrs, int *prot)
Jan Kiszka90260c62013-05-26 21:46:51 +0200676{
Avi Kivity30951152012-10-30 13:47:46 +0200677 MemoryRegionSection *section;
Peter Maydell1f871c52018-06-15 14:57:16 +0100678 IOMMUMemoryRegion *iommu_mr;
679 IOMMUMemoryRegionClass *imrc;
680 IOMMUTLBEntry iotlb;
681 int iommu_idx;
Richard Henderson418ade72022-06-21 08:38:29 -0700682 hwaddr addr = orig_addr;
Richard Henderson0d58c662023-08-25 16:13:17 -0700683 AddressSpaceDispatch *d = cpu->cpu_ases[asidx].memory_dispatch;
Peter Maydelld7898cd2016-01-21 14:15:05 +0000684
Peter Maydell1f871c52018-06-15 14:57:16 +0100685 for (;;) {
686 section = address_space_translate_internal(d, addr, &addr, plen, false);
687
688 iommu_mr = memory_region_get_iommu(section->mr);
689 if (!iommu_mr) {
690 break;
691 }
692
693 imrc = memory_region_get_iommu_class_nocheck(iommu_mr);
694
695 iommu_idx = imrc->attrs_to_index(iommu_mr, attrs);
696 tcg_register_iommu_notifier(cpu, iommu_mr, iommu_idx);
697 /* We need all the permissions, so pass IOMMU_NONE so the IOMMU
698 * doesn't short-cut its translation table walk.
699 */
700 iotlb = imrc->translate(iommu_mr, addr, IOMMU_NONE, iommu_idx);
701 addr = ((iotlb.translated_addr & ~iotlb.addr_mask)
702 | (addr & iotlb.addr_mask));
703 /* Update the caller's prot bits to remove permissions the IOMMU
704 * is giving us a failure response for. If we get down to no
705 * permissions left at all we can give up now.
706 */
707 if (!(iotlb.perm & IOMMU_RO)) {
708 *prot &= ~(PAGE_READ | PAGE_EXEC);
709 }
710 if (!(iotlb.perm & IOMMU_WO)) {
711 *prot &= ~PAGE_WRITE;
712 }
713
714 if (!*prot) {
715 goto translate_fail;
716 }
717
718 d = flatview_to_dispatch(address_space_to_flatview(iotlb.target_as));
719 }
Avi Kivity30951152012-10-30 13:47:46 +0200720
Alexey Kardashevskiy3df9d742017-07-11 13:56:19 +1000721 assert(!memory_region_is_iommu(section->mr));
Peter Maydell1f871c52018-06-15 14:57:16 +0100722 *xlat = addr;
Avi Kivity30951152012-10-30 13:47:46 +0200723 return section;
Peter Maydell1f871c52018-06-15 14:57:16 +0100724
725translate_fail:
Richard Henderson418ade72022-06-21 08:38:29 -0700726 /*
727 * We should be given a page-aligned address -- certainly
728 * tlb_set_page_with_attrs() does so. The page offset of xlat
729 * is used to index sections[], and PHYS_SECTION_UNASSIGNED = 0.
730 * The page portion of xlat will be logged by memory_region_access_valid()
731 * when this memory access is rejected, so use the original untranslated
732 * physical address.
733 */
734 assert((orig_addr & ~TARGET_PAGE_MASK) == 0);
735 *xlat = orig_addr;
Peter Maydell1f871c52018-06-15 14:57:16 +0100736 return &d->map.sections[PHYS_SECTION_UNASSIGNED];
Jan Kiszka90260c62013-05-26 21:46:51 +0200737}
Andreas Färber1a1562f2013-06-17 04:09:11 +0200738
Peter Xu80ceb072017-11-23 17:23:32 +0800739void cpu_address_space_init(CPUState *cpu, int asidx,
740 const char *prefix, MemoryRegion *mr)
Edgar E. Iglesias09daed82013-12-17 13:06:51 +1000741{
Peter Maydell12ebc9a2016-01-21 14:15:04 +0000742 CPUAddressSpace *newas;
Peter Xu80ceb072017-11-23 17:23:32 +0800743 AddressSpace *as = g_new0(AddressSpace, 1);
Peter Xu87a621d2017-11-23 17:23:33 +0800744 char *as_name;
Peter Xu80ceb072017-11-23 17:23:32 +0800745
746 assert(mr);
Peter Xu87a621d2017-11-23 17:23:33 +0800747 as_name = g_strdup_printf("%s-%d", prefix, cpu->cpu_index);
748 address_space_init(as, mr, as_name);
749 g_free(as_name);
Peter Maydell12ebc9a2016-01-21 14:15:04 +0000750
751 /* Target code should have set num_ases before calling us */
752 assert(asidx < cpu->num_ases);
753
Peter Maydell56943e82016-01-21 14:15:04 +0000754 if (asidx == 0) {
755 /* address space 0 gets the convenience alias */
756 cpu->as = as;
757 }
758
Peter Maydell12ebc9a2016-01-21 14:15:04 +0000759 /* KVM cannot currently support multiple address spaces. */
760 assert(asidx == 0 || !kvm_enabled());
Edgar E. Iglesias09daed82013-12-17 13:06:51 +1000761
Peter Maydell12ebc9a2016-01-21 14:15:04 +0000762 if (!cpu->cpu_ases) {
763 cpu->cpu_ases = g_new0(CPUAddressSpace, cpu->num_ases);
Edgar E. Iglesias09daed82013-12-17 13:06:51 +1000764 }
Peter Maydell32857f42015-10-01 15:29:50 +0100765
Peter Maydell12ebc9a2016-01-21 14:15:04 +0000766 newas = &cpu->cpu_ases[asidx];
767 newas->cpu = cpu;
768 newas->as = as;
Peter Maydell56943e82016-01-21 14:15:04 +0000769 if (tcg_enabled()) {
Paolo Bonzini9458a9a2018-02-06 18:37:39 +0100770 newas->tcg_as_listener.log_global_after_sync = tcg_log_global_after_sync;
Peter Maydell12ebc9a2016-01-21 14:15:04 +0000771 newas->tcg_as_listener.commit = tcg_commit;
Peter Xu142518b2021-08-16 21:35:52 -0400772 newas->tcg_as_listener.name = "tcg";
Peter Maydell12ebc9a2016-01-21 14:15:04 +0000773 memory_listener_register(&newas->tcg_as_listener, as);
Peter Maydell56943e82016-01-21 14:15:04 +0000774 }
Edgar E. Iglesias09daed82013-12-17 13:06:51 +1000775}
Peter Maydell651a5bc2016-01-21 14:15:05 +0000776
777AddressSpace *cpu_get_address_space(CPUState *cpu, int asidx)
778{
779 /* Return the AddressSpace corresponding to the specified index */
780 return cpu->cpu_ases[asidx].as;
781}
Edgar E. Iglesias09daed82013-12-17 13:06:51 +1000782
Mike Day0dc3f442013-09-05 14:41:35 -0400783/* Called from RCU critical section */
Paolo Bonzini041603f2013-09-09 17:49:45 +0200784static RAMBlock *qemu_get_ram_block(ram_addr_t addr)
785{
786 RAMBlock *block;
787
Stefan Hajnoczid73415a2020-09-23 11:56:46 +0100788 block = qatomic_rcu_read(&ram_list.mru_block);
Michael S. Tsirkin9b8424d2014-12-15 22:55:32 +0200789 if (block && addr - block->offset < block->max_length) {
Paolo Bonzini68851b92015-10-22 13:51:30 +0200790 return block;
Paolo Bonzini041603f2013-09-09 17:49:45 +0200791 }
Peter Xu99e15582017-05-12 12:17:39 +0800792 RAMBLOCK_FOREACH(block) {
Michael S. Tsirkin9b8424d2014-12-15 22:55:32 +0200793 if (addr - block->offset < block->max_length) {
Paolo Bonzini041603f2013-09-09 17:49:45 +0200794 goto found;
795 }
796 }
797
798 fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
799 abort();
800
801found:
Paolo Bonzini43771532013-09-09 17:58:40 +0200802 /* It is safe to write mru_block outside the iothread lock. This
803 * is what happens:
804 *
805 * mru_block = xxx
806 * rcu_read_unlock()
807 * xxx removed from list
808 * rcu_read_lock()
809 * read mru_block
810 * mru_block = NULL;
811 * call_rcu(reclaim_ramblock, xxx);
812 * rcu_read_unlock()
813 *
Stefan Hajnoczid73415a2020-09-23 11:56:46 +0100814 * qatomic_rcu_set is not needed here. The block was already published
Paolo Bonzini43771532013-09-09 17:58:40 +0200815 * when it was placed into the list. Here we're just making an extra
816 * copy of the pointer.
817 */
Paolo Bonzini041603f2013-09-09 17:49:45 +0200818 ram_list.mru_block = block;
819 return block;
820}
821
Juan Quintelaa2f4d5b2013-10-10 11:49:53 +0200822static void tlb_reset_dirty_range_all(ram_addr_t start, ram_addr_t length)
bellard1ccde1c2004-02-06 19:46:14 +0000823{
Peter Crosthwaite9a135652015-09-10 22:39:41 -0700824 CPUState *cpu;
Paolo Bonzini041603f2013-09-09 17:49:45 +0200825 ram_addr_t start1;
Juan Quintelaa2f4d5b2013-10-10 11:49:53 +0200826 RAMBlock *block;
827 ram_addr_t end;
828
Emilio G. Cotaf28d0df2018-06-22 13:45:31 -0400829 assert(tcg_enabled());
Juan Quintelaa2f4d5b2013-10-10 11:49:53 +0200830 end = TARGET_PAGE_ALIGN(start + length);
831 start &= TARGET_PAGE_MASK;
bellardf23db162005-08-21 19:12:28 +0000832
Dr. David Alan Gilbert694ea272019-10-07 15:36:41 +0100833 RCU_READ_LOCK_GUARD();
Paolo Bonzini041603f2013-09-09 17:49:45 +0200834 block = qemu_get_ram_block(start);
835 assert(block == qemu_get_ram_block(end - 1));
Michael S. Tsirkin1240be22014-11-12 11:44:41 +0200836 start1 = (uintptr_t)ramblock_ptr(block, start - block->offset);
Peter Crosthwaite9a135652015-09-10 22:39:41 -0700837 CPU_FOREACH(cpu) {
838 tlb_reset_dirty(cpu, start1, length);
839 }
Juan Quintelad24981d2012-05-22 00:42:40 +0200840}
841
842/* Note: start and end must be within the same ram block. */
Stefan Hajnoczi03eebc92014-12-02 11:23:18 +0000843bool cpu_physical_memory_test_and_clear_dirty(ram_addr_t start,
844 ram_addr_t length,
845 unsigned client)
Juan Quintelad24981d2012-05-22 00:42:40 +0200846{
Stefan Hajnoczi5b82b702016-01-25 13:33:20 +0000847 DirtyMemoryBlocks *blocks;
Matt Borgerson25aa6b32020-02-18 03:19:10 -0700848 unsigned long end, page, start_page;
Stefan Hajnoczi5b82b702016-01-25 13:33:20 +0000849 bool dirty = false;
Peter Xu077874e2019-06-03 14:50:51 +0800850 RAMBlock *ramblock;
851 uint64_t mr_offset, mr_size;
Juan Quintelad24981d2012-05-22 00:42:40 +0200852
Stefan Hajnoczi03eebc92014-12-02 11:23:18 +0000853 if (length == 0) {
854 return false;
855 }
856
857 end = TARGET_PAGE_ALIGN(start + length) >> TARGET_PAGE_BITS;
Matt Borgerson25aa6b32020-02-18 03:19:10 -0700858 start_page = start >> TARGET_PAGE_BITS;
859 page = start_page;
Stefan Hajnoczi5b82b702016-01-25 13:33:20 +0000860
Dr. David Alan Gilbert694ea272019-10-07 15:36:41 +0100861 WITH_RCU_READ_LOCK_GUARD() {
Stefan Hajnoczid73415a2020-09-23 11:56:46 +0100862 blocks = qatomic_rcu_read(&ram_list.dirty_memory[client]);
Dr. David Alan Gilbert694ea272019-10-07 15:36:41 +0100863 ramblock = qemu_get_ram_block(start);
864 /* Range sanity check on the ramblock */
865 assert(start >= ramblock->offset &&
866 start + length <= ramblock->offset + ramblock->used_length);
Stefan Hajnoczi5b82b702016-01-25 13:33:20 +0000867
Dr. David Alan Gilbert694ea272019-10-07 15:36:41 +0100868 while (page < end) {
869 unsigned long idx = page / DIRTY_MEMORY_BLOCK_SIZE;
870 unsigned long offset = page % DIRTY_MEMORY_BLOCK_SIZE;
871 unsigned long num = MIN(end - page,
872 DIRTY_MEMORY_BLOCK_SIZE - offset);
Stefan Hajnoczi5b82b702016-01-25 13:33:20 +0000873
Dr. David Alan Gilbert694ea272019-10-07 15:36:41 +0100874 dirty |= bitmap_test_and_clear_atomic(blocks->blocks[idx],
875 offset, num);
876 page += num;
877 }
Stefan Hajnoczi5b82b702016-01-25 13:33:20 +0000878
Matt Borgerson25aa6b32020-02-18 03:19:10 -0700879 mr_offset = (ram_addr_t)(start_page << TARGET_PAGE_BITS) - ramblock->offset;
880 mr_size = (end - start_page) << TARGET_PAGE_BITS;
Dr. David Alan Gilbert694ea272019-10-07 15:36:41 +0100881 memory_region_clear_dirty_bitmap(ramblock->mr, mr_offset, mr_size);
Stefan Hajnoczi5b82b702016-01-25 13:33:20 +0000882 }
883
Stefan Hajnoczi03eebc92014-12-02 11:23:18 +0000884 if (dirty && tcg_enabled()) {
Juan Quintelaa2f4d5b2013-10-10 11:49:53 +0200885 tlb_reset_dirty_range_all(start, length);
Juan Quintelad24981d2012-05-22 00:42:40 +0200886 }
Stefan Hajnoczi03eebc92014-12-02 11:23:18 +0000887
888 return dirty;
bellard1ccde1c2004-02-06 19:46:14 +0000889}
890
Gerd Hoffmann8deaf122017-04-21 11:16:25 +0200891DirtyBitmapSnapshot *cpu_physical_memory_snapshot_and_clear_dirty
Peter Xu5dea4072019-06-03 14:50:50 +0800892 (MemoryRegion *mr, hwaddr offset, hwaddr length, unsigned client)
Gerd Hoffmann8deaf122017-04-21 11:16:25 +0200893{
894 DirtyMemoryBlocks *blocks;
Peter Xu5dea4072019-06-03 14:50:50 +0800895 ram_addr_t start = memory_region_get_ram_addr(mr) + offset;
Gerd Hoffmann8deaf122017-04-21 11:16:25 +0200896 unsigned long align = 1UL << (TARGET_PAGE_BITS + BITS_PER_LEVEL);
897 ram_addr_t first = QEMU_ALIGN_DOWN(start, align);
898 ram_addr_t last = QEMU_ALIGN_UP(start + length, align);
899 DirtyBitmapSnapshot *snap;
900 unsigned long page, end, dest;
901
902 snap = g_malloc0(sizeof(*snap) +
903 ((last - first) >> (TARGET_PAGE_BITS + 3)));
904 snap->start = first;
905 snap->end = last;
906
907 page = first >> TARGET_PAGE_BITS;
908 end = last >> TARGET_PAGE_BITS;
909 dest = 0;
910
Dr. David Alan Gilbert694ea272019-10-07 15:36:41 +0100911 WITH_RCU_READ_LOCK_GUARD() {
Stefan Hajnoczid73415a2020-09-23 11:56:46 +0100912 blocks = qatomic_rcu_read(&ram_list.dirty_memory[client]);
Gerd Hoffmann8deaf122017-04-21 11:16:25 +0200913
Dr. David Alan Gilbert694ea272019-10-07 15:36:41 +0100914 while (page < end) {
915 unsigned long idx = page / DIRTY_MEMORY_BLOCK_SIZE;
Philippe Mathieu-Daudé6ba9b602023-09-04 18:12:34 +0200916 unsigned long ofs = page % DIRTY_MEMORY_BLOCK_SIZE;
Dr. David Alan Gilbert694ea272019-10-07 15:36:41 +0100917 unsigned long num = MIN(end - page,
Philippe Mathieu-Daudé6ba9b602023-09-04 18:12:34 +0200918 DIRTY_MEMORY_BLOCK_SIZE - ofs);
Gerd Hoffmann8deaf122017-04-21 11:16:25 +0200919
Philippe Mathieu-Daudé6ba9b602023-09-04 18:12:34 +0200920 assert(QEMU_IS_ALIGNED(ofs, (1 << BITS_PER_LEVEL)));
Dr. David Alan Gilbert694ea272019-10-07 15:36:41 +0100921 assert(QEMU_IS_ALIGNED(num, (1 << BITS_PER_LEVEL)));
Philippe Mathieu-Daudé6ba9b602023-09-04 18:12:34 +0200922 ofs >>= BITS_PER_LEVEL;
Gerd Hoffmann8deaf122017-04-21 11:16:25 +0200923
Dr. David Alan Gilbert694ea272019-10-07 15:36:41 +0100924 bitmap_copy_and_clear_atomic(snap->dirty + dest,
Philippe Mathieu-Daudé6ba9b602023-09-04 18:12:34 +0200925 blocks->blocks[idx] + ofs,
Dr. David Alan Gilbert694ea272019-10-07 15:36:41 +0100926 num);
927 page += num;
928 dest += num >> BITS_PER_LEVEL;
929 }
Gerd Hoffmann8deaf122017-04-21 11:16:25 +0200930 }
931
Gerd Hoffmann8deaf122017-04-21 11:16:25 +0200932 if (tcg_enabled()) {
933 tlb_reset_dirty_range_all(start, length);
934 }
935
Peter Xu077874e2019-06-03 14:50:51 +0800936 memory_region_clear_dirty_bitmap(mr, offset, length);
937
Gerd Hoffmann8deaf122017-04-21 11:16:25 +0200938 return snap;
939}
940
941bool cpu_physical_memory_snapshot_get_dirty(DirtyBitmapSnapshot *snap,
942 ram_addr_t start,
943 ram_addr_t length)
944{
945 unsigned long page, end;
946
947 assert(start >= snap->start);
948 assert(start + length <= snap->end);
949
950 end = TARGET_PAGE_ALIGN(start + length - snap->start) >> TARGET_PAGE_BITS;
951 page = (start - snap->start) >> TARGET_PAGE_BITS;
952
953 while (page < end) {
954 if (test_bit(page, snap->dirty)) {
955 return true;
956 }
957 page++;
958 }
959 return false;
960}
961
Paolo Bonzini79e2b9a2015-01-21 12:09:14 +0100962/* Called from RCU critical section */
Andreas Färberbb0e6272013-09-03 13:32:01 +0200963hwaddr memory_region_section_get_iotlb(CPUState *cpu,
Richard Henderson8f5db642019-09-19 21:09:58 -0700964 MemoryRegionSection *section)
Blue Swirle5548612012-04-21 13:08:33 +0000965{
Richard Henderson8f5db642019-09-19 21:09:58 -0700966 AddressSpaceDispatch *d = flatview_to_dispatch(section->fv);
967 return section - d->map.sections;
Blue Swirle5548612012-04-21 13:08:33 +0000968}
pbrook8da3ff12008-12-01 18:59:50 +0000969
Wei Yangb797ab12019-03-21 16:25:53 +0800970static int subpage_register(subpage_t *mmio, uint32_t start, uint32_t end,
971 uint16_t section);
Alexey Kardashevskiy16620682017-09-21 18:50:58 +1000972static subpage_t *subpage_init(FlatView *fv, hwaddr base);
Avi Kivity54688b12012-02-09 17:34:32 +0200973
Marcel Apfelbaum53cb28c2013-12-01 14:02:23 +0200974static uint16_t phys_section_add(PhysPageMap *map,
975 MemoryRegionSection *section)
Avi Kivity5312bd82012-02-12 18:32:55 +0200976{
Paolo Bonzini68f3f652013-05-07 11:30:23 +0200977 /* The physical section number is ORed with a page-aligned
978 * pointer to produce the iotlb entries. Thus it should
979 * never overflow into the page-aligned value.
980 */
Marcel Apfelbaum53cb28c2013-12-01 14:02:23 +0200981 assert(map->sections_nb < TARGET_PAGE_SIZE);
Paolo Bonzini68f3f652013-05-07 11:30:23 +0200982
Marcel Apfelbaum53cb28c2013-12-01 14:02:23 +0200983 if (map->sections_nb == map->sections_nb_alloc) {
984 map->sections_nb_alloc = MAX(map->sections_nb_alloc * 2, 16);
985 map->sections = g_renew(MemoryRegionSection, map->sections,
986 map->sections_nb_alloc);
Avi Kivity5312bd82012-02-12 18:32:55 +0200987 }
Marcel Apfelbaum53cb28c2013-12-01 14:02:23 +0200988 map->sections[map->sections_nb] = *section;
Paolo Bonzinidfde4e62013-05-06 10:46:11 +0200989 memory_region_ref(section->mr);
Marcel Apfelbaum53cb28c2013-12-01 14:02:23 +0200990 return map->sections_nb++;
Avi Kivity5312bd82012-02-12 18:32:55 +0200991}
992
Paolo Bonzini058bc4b2013-06-25 09:30:48 +0200993static void phys_section_destroy(MemoryRegion *mr)
994{
Don Slutz55b4e802015-11-30 17:11:04 -0500995 bool have_sub_page = mr->subpage;
996
Paolo Bonzinidfde4e62013-05-06 10:46:11 +0200997 memory_region_unref(mr);
998
Don Slutz55b4e802015-11-30 17:11:04 -0500999 if (have_sub_page) {
Paolo Bonzini058bc4b2013-06-25 09:30:48 +02001000 subpage_t *subpage = container_of(mr, subpage_t, iomem);
Peter Crosthwaiteb4fefef2014-06-05 23:15:52 -07001001 object_unref(OBJECT(&subpage->iomem));
Paolo Bonzini058bc4b2013-06-25 09:30:48 +02001002 g_free(subpage);
1003 }
1004}
1005
Paolo Bonzini60926662013-05-29 12:30:26 +02001006static void phys_sections_free(PhysPageMap *map)
Avi Kivity5312bd82012-02-12 18:32:55 +02001007{
Paolo Bonzini9affd6f2013-05-29 12:09:47 +02001008 while (map->sections_nb > 0) {
1009 MemoryRegionSection *section = &map->sections[--map->sections_nb];
Paolo Bonzini058bc4b2013-06-25 09:30:48 +02001010 phys_section_destroy(section->mr);
1011 }
Paolo Bonzini9affd6f2013-05-29 12:09:47 +02001012 g_free(map->sections);
1013 g_free(map->nodes);
Avi Kivity5312bd82012-02-12 18:32:55 +02001014}
1015
Alexey Kardashevskiy99503222017-09-21 18:50:59 +10001016static void register_subpage(FlatView *fv, MemoryRegionSection *section)
Avi Kivity0f0cb162012-02-13 17:14:32 +02001017{
Alexey Kardashevskiy99503222017-09-21 18:50:59 +10001018 AddressSpaceDispatch *d = flatview_to_dispatch(fv);
Avi Kivity0f0cb162012-02-13 17:14:32 +02001019 subpage_t *subpage;
Avi Kivitya8170e52012-10-23 12:30:10 +02001020 hwaddr base = section->offset_within_address_space
Avi Kivity0f0cb162012-02-13 17:14:32 +02001021 & TARGET_PAGE_MASK;
Peter Xu003a0cf2017-05-15 16:50:57 +08001022 MemoryRegionSection *existing = phys_page_find(d, base);
Avi Kivity0f0cb162012-02-13 17:14:32 +02001023 MemoryRegionSection subsection = {
1024 .offset_within_address_space = base,
Paolo Bonzini052e87b2013-05-27 10:08:27 +02001025 .size = int128_make64(TARGET_PAGE_SIZE),
Avi Kivity0f0cb162012-02-13 17:14:32 +02001026 };
Avi Kivitya8170e52012-10-23 12:30:10 +02001027 hwaddr start, end;
Avi Kivity0f0cb162012-02-13 17:14:32 +02001028
Avi Kivityf3705d52012-03-08 16:16:34 +02001029 assert(existing->mr->subpage || existing->mr == &io_mem_unassigned);
Avi Kivity0f0cb162012-02-13 17:14:32 +02001030
Avi Kivityf3705d52012-03-08 16:16:34 +02001031 if (!(existing->mr->subpage)) {
Alexey Kardashevskiy16620682017-09-21 18:50:58 +10001032 subpage = subpage_init(fv, base);
1033 subsection.fv = fv;
Avi Kivity0f0cb162012-02-13 17:14:32 +02001034 subsection.mr = &subpage->iomem;
Avi Kivityac1970f2012-10-03 16:22:53 +02001035 phys_page_set(d, base >> TARGET_PAGE_BITS, 1,
Marcel Apfelbaum53cb28c2013-12-01 14:02:23 +02001036 phys_section_add(&d->map, &subsection));
Avi Kivity0f0cb162012-02-13 17:14:32 +02001037 } else {
Avi Kivityf3705d52012-03-08 16:16:34 +02001038 subpage = container_of(existing->mr, subpage_t, iomem);
Avi Kivity0f0cb162012-02-13 17:14:32 +02001039 }
1040 start = section->offset_within_address_space & ~TARGET_PAGE_MASK;
Paolo Bonzini052e87b2013-05-27 10:08:27 +02001041 end = start + int128_get64(section->size) - 1;
Marcel Apfelbaum53cb28c2013-12-01 14:02:23 +02001042 subpage_register(subpage, start, end,
1043 phys_section_add(&d->map, section));
Avi Kivity0f0cb162012-02-13 17:14:32 +02001044}
1045
1046
Alexey Kardashevskiy99503222017-09-21 18:50:59 +10001047static void register_multipage(FlatView *fv,
Paolo Bonzini052e87b2013-05-27 10:08:27 +02001048 MemoryRegionSection *section)
bellard33417e72003-08-10 21:47:01 +00001049{
Alexey Kardashevskiy99503222017-09-21 18:50:59 +10001050 AddressSpaceDispatch *d = flatview_to_dispatch(fv);
Avi Kivitya8170e52012-10-23 12:30:10 +02001051 hwaddr start_addr = section->offset_within_address_space;
Marcel Apfelbaum53cb28c2013-12-01 14:02:23 +02001052 uint16_t section_index = phys_section_add(&d->map, section);
Paolo Bonzini052e87b2013-05-27 10:08:27 +02001053 uint64_t num_pages = int128_get64(int128_rshift(section->size,
1054 TARGET_PAGE_BITS));
Avi Kivitydd811242012-01-02 12:17:03 +02001055
Paolo Bonzini733d5ef2013-05-27 10:47:10 +02001056 assert(num_pages);
1057 phys_page_set(d, start_addr >> TARGET_PAGE_BITS, num_pages, section_index);
bellard33417e72003-08-10 21:47:01 +00001058}
1059
Wei Yang494d1992019-03-11 13:42:52 +08001060/*
1061 * The range in *section* may look like this:
1062 *
1063 * |s|PPPPPPP|s|
1064 *
1065 * where s stands for subpage and P for page.
1066 */
Alexey Kardashevskiy8629d3f2017-09-21 18:51:00 +10001067void flatview_add_to_dispatch(FlatView *fv, MemoryRegionSection *section)
Avi Kivity0f0cb162012-02-13 17:14:32 +02001068{
Wei Yang494d1992019-03-11 13:42:52 +08001069 MemoryRegionSection remain = *section;
Paolo Bonzini052e87b2013-05-27 10:08:27 +02001070 Int128 page_size = int128_make64(TARGET_PAGE_SIZE);
Avi Kivity0f0cb162012-02-13 17:14:32 +02001071
Wei Yang494d1992019-03-11 13:42:52 +08001072 /* register first subpage */
1073 if (remain.offset_within_address_space & ~TARGET_PAGE_MASK) {
1074 uint64_t left = TARGET_PAGE_ALIGN(remain.offset_within_address_space)
1075 - remain.offset_within_address_space;
Paolo Bonzini733d5ef2013-05-27 10:47:10 +02001076
Wei Yang494d1992019-03-11 13:42:52 +08001077 MemoryRegionSection now = remain;
Paolo Bonzini052e87b2013-05-27 10:08:27 +02001078 now.size = int128_min(int128_make64(left), now.size);
Alexey Kardashevskiy99503222017-09-21 18:50:59 +10001079 register_subpage(fv, &now);
Wei Yang494d1992019-03-11 13:42:52 +08001080 if (int128_eq(remain.size, now.size)) {
1081 return;
1082 }
Paolo Bonzini052e87b2013-05-27 10:08:27 +02001083 remain.size = int128_sub(remain.size, now.size);
1084 remain.offset_within_address_space += int128_get64(now.size);
1085 remain.offset_within_region += int128_get64(now.size);
Avi Kivity0f0cb162012-02-13 17:14:32 +02001086 }
Wei Yang494d1992019-03-11 13:42:52 +08001087
1088 /* register whole pages */
1089 if (int128_ge(remain.size, page_size)) {
1090 MemoryRegionSection now = remain;
1091 now.size = int128_and(now.size, int128_neg(page_size));
1092 register_multipage(fv, &now);
1093 if (int128_eq(remain.size, now.size)) {
1094 return;
1095 }
1096 remain.size = int128_sub(remain.size, now.size);
1097 remain.offset_within_address_space += int128_get64(now.size);
1098 remain.offset_within_region += int128_get64(now.size);
1099 }
1100
1101 /* register last subpage */
1102 register_subpage(fv, &remain);
Avi Kivity0f0cb162012-02-13 17:14:32 +02001103}
1104
Sheng Yang62a27442010-01-26 19:21:16 +08001105void qemu_flush_coalesced_mmio_buffer(void)
1106{
1107 if (kvm_enabled())
1108 kvm_flush_coalesced_mmio_buffer();
1109}
1110
Umesh Deshpandeb2a86582011-08-17 00:01:33 -07001111void qemu_mutex_lock_ramlist(void)
1112{
1113 qemu_mutex_lock(&ram_list.mutex);
1114}
1115
1116void qemu_mutex_unlock_ramlist(void)
1117{
1118 qemu_mutex_unlock(&ram_list.mutex);
1119}
1120
Daniel P. Berrangéca411b72021-09-08 10:35:43 +01001121GString *ram_block_format(void)
Peter Xube9b23c2017-05-12 12:17:41 +08001122{
1123 RAMBlock *block;
1124 char *psize;
Daniel P. Berrangéca411b72021-09-08 10:35:43 +01001125 GString *buf = g_string_new("");
Peter Xube9b23c2017-05-12 12:17:41 +08001126
Dr. David Alan Gilbert694ea272019-10-07 15:36:41 +01001127 RCU_READ_LOCK_GUARD();
Ted Chendbc6ae92022-12-05 20:07:12 +08001128 g_string_append_printf(buf, "%24s %8s %18s %18s %18s %18s %3s\n",
1129 "Block Name", "PSize", "Offset", "Used", "Total",
1130 "HVA", "RO");
1131
Peter Xube9b23c2017-05-12 12:17:41 +08001132 RAMBLOCK_FOREACH(block) {
1133 psize = size_to_str(block->page_size);
Daniel P. Berrangéca411b72021-09-08 10:35:43 +01001134 g_string_append_printf(buf, "%24s %8s 0x%016" PRIx64 " 0x%016" PRIx64
Ted Chendbc6ae92022-12-05 20:07:12 +08001135 " 0x%016" PRIx64 " 0x%016" PRIx64 " %3s\n",
1136 block->idstr, psize,
Daniel P. Berrangéca411b72021-09-08 10:35:43 +01001137 (uint64_t)block->offset,
1138 (uint64_t)block->used_length,
Ted Chendbc6ae92022-12-05 20:07:12 +08001139 (uint64_t)block->max_length,
1140 (uint64_t)(uintptr_t)block->host,
1141 block->mr->readonly ? "ro" : "rw");
1142
Peter Xube9b23c2017-05-12 12:17:41 +08001143 g_free(psize);
1144 }
Daniel P. Berrangéca411b72021-09-08 10:35:43 +01001145
1146 return buf;
Peter Xube9b23c2017-05-12 12:17:41 +08001147}
1148
David Hildenbrand905b7ee2019-04-17 13:31:43 +02001149static int find_min_backend_pagesize(Object *obj, void *opaque)
Alexey Kardashevskiy9c607662017-03-02 13:36:11 +11001150{
Alexey Kardashevskiy9c607662017-03-02 13:36:11 +11001151 long *hpsize_min = opaque;
1152
1153 if (object_dynamic_cast(obj, TYPE_MEMORY_BACKEND)) {
David Gibson7d5489e2019-03-26 14:33:33 +11001154 HostMemoryBackend *backend = MEMORY_BACKEND(obj);
1155 long hpsize = host_memory_backend_pagesize(backend);
David Gibson2b108082018-04-03 15:05:45 +10001156
David Gibson7d5489e2019-03-26 14:33:33 +11001157 if (host_memory_backend_is_mapped(backend) && (hpsize < *hpsize_min)) {
David Gibson0de6e2a2018-04-03 14:55:11 +10001158 *hpsize_min = hpsize;
Alexey Kardashevskiy9c607662017-03-02 13:36:11 +11001159 }
1160 }
1161
1162 return 0;
1163}
1164
David Hildenbrand905b7ee2019-04-17 13:31:43 +02001165static int find_max_backend_pagesize(Object *obj, void *opaque)
1166{
1167 long *hpsize_max = opaque;
1168
1169 if (object_dynamic_cast(obj, TYPE_MEMORY_BACKEND)) {
1170 HostMemoryBackend *backend = MEMORY_BACKEND(obj);
1171 long hpsize = host_memory_backend_pagesize(backend);
1172
1173 if (host_memory_backend_is_mapped(backend) && (hpsize > *hpsize_max)) {
1174 *hpsize_max = hpsize;
1175 }
1176 }
1177
1178 return 0;
1179}
1180
1181/*
1182 * TODO: We assume right now that all mapped host memory backends are
1183 * used as RAM, however some might be used for different purposes.
1184 */
1185long qemu_minrampagesize(void)
Alexey Kardashevskiy9c607662017-03-02 13:36:11 +11001186{
1187 long hpsize = LONG_MAX;
Igor Mammedovad1172d2020-02-19 11:09:47 -05001188 Object *memdev_root = object_resolve_path("/objects", NULL);
Alexey Kardashevskiy9c607662017-03-02 13:36:11 +11001189
Igor Mammedovad1172d2020-02-19 11:09:47 -05001190 object_child_foreach(memdev_root, find_min_backend_pagesize, &hpsize);
Alexey Kardashevskiy9c607662017-03-02 13:36:11 +11001191 return hpsize;
1192}
David Hildenbrand905b7ee2019-04-17 13:31:43 +02001193
1194long qemu_maxrampagesize(void)
1195{
Igor Mammedovad1172d2020-02-19 11:09:47 -05001196 long pagesize = 0;
David Hildenbrand905b7ee2019-04-17 13:31:43 +02001197 Object *memdev_root = object_resolve_path("/objects", NULL);
1198
Igor Mammedovad1172d2020-02-19 11:09:47 -05001199 object_child_foreach(memdev_root, find_max_backend_pagesize, &pagesize);
David Hildenbrand905b7ee2019-04-17 13:31:43 +02001200 return pagesize;
1201}
Alexey Kardashevskiy9c607662017-03-02 13:36:11 +11001202
Hikaru Nishidad5dbde42018-09-24 21:32:05 +09001203#ifdef CONFIG_POSIX
Haozhong Zhangd6af99c2016-10-27 12:22:58 +08001204static int64_t get_file_size(int fd)
1205{
Stefan Hajnoczi72d41eb2019-08-30 10:30:56 +01001206 int64_t size;
1207#if defined(__linux__)
1208 struct stat st;
1209
1210 if (fstat(fd, &st) < 0) {
1211 return -errno;
1212 }
1213
1214 /* Special handling for devdax character devices */
1215 if (S_ISCHR(st.st_mode)) {
1216 g_autofree char *subsystem_path = NULL;
1217 g_autofree char *subsystem = NULL;
1218
1219 subsystem_path = g_strdup_printf("/sys/dev/char/%d:%d/subsystem",
1220 major(st.st_rdev), minor(st.st_rdev));
1221 subsystem = g_file_read_link(subsystem_path, NULL);
1222
1223 if (subsystem && g_str_has_suffix(subsystem, "/dax")) {
1224 g_autofree char *size_path = NULL;
1225 g_autofree char *size_str = NULL;
1226
1227 size_path = g_strdup_printf("/sys/dev/char/%d:%d/size",
1228 major(st.st_rdev), minor(st.st_rdev));
1229
1230 if (g_file_get_contents(size_path, &size_str, NULL, NULL)) {
1231 return g_ascii_strtoll(size_str, NULL, 0);
1232 }
1233 }
1234 }
1235#endif /* defined(__linux__) */
1236
1237 /* st.st_size may be zero for special files yet lseek(2) works */
1238 size = lseek(fd, 0, SEEK_END);
Haozhong Zhangd6af99c2016-10-27 12:22:58 +08001239 if (size < 0) {
1240 return -errno;
1241 }
1242 return size;
1243}
1244
Jingqi Liuce317be2020-04-29 16:50:09 +08001245static int64_t get_file_align(int fd)
1246{
1247 int64_t align = -1;
1248#if defined(__linux__) && defined(CONFIG_LIBDAXCTL)
1249 struct stat st;
1250
1251 if (fstat(fd, &st) < 0) {
1252 return -errno;
1253 }
1254
1255 /* Special handling for devdax character devices */
1256 if (S_ISCHR(st.st_mode)) {
1257 g_autofree char *path = NULL;
1258 g_autofree char *rpath = NULL;
1259 struct daxctl_ctx *ctx;
1260 struct daxctl_region *region;
1261 int rc = 0;
1262
1263 path = g_strdup_printf("/sys/dev/char/%d:%d",
1264 major(st.st_rdev), minor(st.st_rdev));
1265 rpath = realpath(path, NULL);
Peter Maydell8efdb7b2021-08-12 16:15:25 +01001266 if (!rpath) {
1267 return -errno;
1268 }
Jingqi Liuce317be2020-04-29 16:50:09 +08001269
1270 rc = daxctl_new(&ctx);
1271 if (rc) {
1272 return -1;
1273 }
1274
1275 daxctl_region_foreach(ctx, region) {
1276 if (strstr(rpath, daxctl_region_get_path(region))) {
1277 align = daxctl_region_get_align(region);
1278 break;
1279 }
1280 }
1281 daxctl_unref(ctx);
1282 }
1283#endif /* defined(__linux__) && defined(CONFIG_LIBDAXCTL) */
1284
1285 return align;
1286}
1287
Marc-André Lureau8d37b032017-06-02 18:12:22 +04001288static int file_ram_open(const char *path,
1289 const char *region_name,
Stefan Hajnoczi369d6dc2021-01-04 17:13:18 +00001290 bool readonly,
David Hildenbrand4d6b23f2023-09-06 14:04:58 +02001291 bool *created)
Marcelo Tosattic9027602010-03-01 20:25:08 -03001292{
1293 char *filename;
Peter Feiner8ca761f2013-03-04 13:54:25 -05001294 char *sanitized_name;
1295 char *c;
Paolo Bonzini5c3ece72016-03-17 15:53:13 +01001296 int fd = -1;
Marcelo Tosattic9027602010-03-01 20:25:08 -03001297
Marc-André Lureau8d37b032017-06-02 18:12:22 +04001298 *created = false;
Markus Armbrusterfd97fd42016-03-07 20:25:13 +01001299 for (;;) {
Stefan Hajnoczi369d6dc2021-01-04 17:13:18 +00001300 fd = open(path, readonly ? O_RDONLY : O_RDWR);
Markus Armbrusterfd97fd42016-03-07 20:25:13 +01001301 if (fd >= 0) {
David Hildenbrandca01f1b2023-09-06 14:04:59 +02001302 /*
1303 * open(O_RDONLY) won't fail with EISDIR. Check manually if we
1304 * opened a directory and fail similarly to how we fail ENOENT
1305 * in readonly mode. Note that mkstemp() would imply O_RDWR.
1306 */
1307 if (readonly) {
1308 struct stat file_stat;
1309
1310 if (fstat(fd, &file_stat)) {
1311 close(fd);
1312 if (errno == EINTR) {
1313 continue;
1314 }
1315 return -errno;
1316 } else if (S_ISDIR(file_stat.st_mode)) {
1317 close(fd);
1318 return -EISDIR;
1319 }
1320 }
Markus Armbrusterfd97fd42016-03-07 20:25:13 +01001321 /* @path names an existing file, use it */
1322 break;
1323 }
1324 if (errno == ENOENT) {
David Hildenbrand4d6b23f2023-09-06 14:04:58 +02001325 if (readonly) {
1326 /* Refuse to create new, readonly files. */
1327 return -ENOENT;
1328 }
Markus Armbrusterfd97fd42016-03-07 20:25:13 +01001329 /* @path names a file that doesn't exist, create it */
1330 fd = open(path, O_RDWR | O_CREAT | O_EXCL, 0644);
1331 if (fd >= 0) {
Marc-André Lureau8d37b032017-06-02 18:12:22 +04001332 *created = true;
Markus Armbrusterfd97fd42016-03-07 20:25:13 +01001333 break;
1334 }
1335 } else if (errno == EISDIR) {
1336 /* @path names a directory, create a file there */
1337 /* Make name safe to use with mkstemp by replacing '/' with '_'. */
Marc-André Lureau8d37b032017-06-02 18:12:22 +04001338 sanitized_name = g_strdup(region_name);
Markus Armbrusterfd97fd42016-03-07 20:25:13 +01001339 for (c = sanitized_name; *c != '\0'; c++) {
1340 if (*c == '/') {
1341 *c = '_';
1342 }
1343 }
1344
1345 filename = g_strdup_printf("%s/qemu_back_mem.%s.XXXXXX", path,
1346 sanitized_name);
1347 g_free(sanitized_name);
1348
1349 fd = mkstemp(filename);
1350 if (fd >= 0) {
1351 unlink(filename);
1352 g_free(filename);
1353 break;
1354 }
1355 g_free(filename);
1356 }
1357 if (errno != EEXIST && errno != EINTR) {
David Hildenbrand4d6b23f2023-09-06 14:04:58 +02001358 return -errno;
Markus Armbrusterfd97fd42016-03-07 20:25:13 +01001359 }
1360 /*
1361 * Try again on EINTR and EEXIST. The latter happens when
1362 * something else creates the file between our two open().
1363 */
1364 }
1365
Marc-André Lureau8d37b032017-06-02 18:12:22 +04001366 return fd;
1367}
1368
1369static void *file_ram_alloc(RAMBlock *block,
1370 ram_addr_t memory,
1371 int fd,
1372 bool truncate,
Jagannathan Raman44a4ff32021-01-29 11:46:04 -05001373 off_t offset,
Marc-André Lureau8d37b032017-06-02 18:12:22 +04001374 Error **errp)
1375{
David Hildenbrandb444f5c2021-05-10 13:43:20 +02001376 uint32_t qemu_map_flags;
Marc-André Lureau8d37b032017-06-02 18:12:22 +04001377 void *area;
1378
Dr. David Alan Gilbert863e9622016-09-29 20:09:37 +01001379 block->page_size = qemu_fd_getpagesize(fd);
Haozhong Zhang98376842017-12-11 15:28:04 +08001380 if (block->mr->align % block->page_size) {
1381 error_setg(errp, "alignment 0x%" PRIx64
1382 " must be multiples of page size 0x%zx",
1383 block->mr->align, block->page_size);
1384 return NULL;
David Hildenbrand61362b72018-06-07 17:47:05 +02001385 } else if (block->mr->align && !is_power_of_2(block->mr->align)) {
1386 error_setg(errp, "alignment 0x%" PRIx64
1387 " must be a power of two", block->mr->align);
1388 return NULL;
Alexander Graf4b870dc2023-04-03 22:14:21 +00001389 } else if (offset % block->page_size) {
1390 error_setg(errp, "offset 0x%" PRIx64
1391 " must be multiples of page size 0x%zx",
1392 offset, block->page_size);
1393 return NULL;
Haozhong Zhang98376842017-12-11 15:28:04 +08001394 }
1395 block->mr->align = MAX(block->page_size, block->mr->align);
Haozhong Zhang83606682016-10-24 20:49:37 +08001396#if defined(__s390x__)
1397 if (kvm_enabled()) {
1398 block->mr->align = MAX(block->mr->align, QEMU_VMALLOC_ALIGN);
1399 }
1400#endif
Marcelo Tosattic9027602010-03-01 20:25:08 -03001401
Dr. David Alan Gilbert863e9622016-09-29 20:09:37 +01001402 if (memory < block->page_size) {
Hu Tao557529d2014-09-09 13:28:00 +08001403 error_setg(errp, "memory size 0x" RAM_ADDR_FMT " must be equal to "
Dr. David Alan Gilbert863e9622016-09-29 20:09:37 +01001404 "or larger than page size 0x%zx",
1405 memory, block->page_size);
Marc-André Lureau8d37b032017-06-02 18:12:22 +04001406 return NULL;
Haozhong Zhang1775f112016-11-02 09:05:51 +08001407 }
1408
Dr. David Alan Gilbert863e9622016-09-29 20:09:37 +01001409 memory = ROUND_UP(memory, block->page_size);
Marcelo Tosattic9027602010-03-01 20:25:08 -03001410
1411 /*
1412 * ftruncate is not supported by hugetlbfs in older
1413 * hosts, so don't bother bailing out on errors.
1414 * If anything goes wrong with it under other filesystems,
1415 * mmap will fail.
Haozhong Zhangd6af99c2016-10-27 12:22:58 +08001416 *
1417 * Do not truncate the non-empty backend file to avoid corrupting
1418 * the existing data in the file. Disabling shrinking is not
1419 * enough. For example, the current vNVDIMM implementation stores
1420 * the guest NVDIMM labels at the end of the backend file. If the
1421 * backend file is later extended, QEMU will not be able to find
1422 * those labels. Therefore, extending the non-empty backend file
1423 * is disabled as well.
Marcelo Tosattic9027602010-03-01 20:25:08 -03001424 */
Alexander Graf4b870dc2023-04-03 22:14:21 +00001425 if (truncate && ftruncate(fd, offset + memory)) {
Yoshiaki Tamura9742bf22010-08-18 13:30:13 +09001426 perror("ftruncate");
Paolo Bonzini7f56e742014-05-14 17:43:20 +08001427 }
Marcelo Tosattic9027602010-03-01 20:25:08 -03001428
David Hildenbrand5c52a212023-09-06 14:04:54 +02001429 qemu_map_flags = (block->flags & RAM_READONLY) ? QEMU_MAP_READONLY : 0;
David Hildenbrandb444f5c2021-05-10 13:43:20 +02001430 qemu_map_flags |= (block->flags & RAM_SHARED) ? QEMU_MAP_SHARED : 0;
1431 qemu_map_flags |= (block->flags & RAM_PMEM) ? QEMU_MAP_SYNC : 0;
David Hildenbrand8dbe22c2021-05-10 13:43:21 +02001432 qemu_map_flags |= (block->flags & RAM_NORESERVE) ? QEMU_MAP_NORESERVE : 0;
David Hildenbrandb444f5c2021-05-10 13:43:20 +02001433 area = qemu_ram_mmap(fd, memory, block->mr->align, qemu_map_flags, offset);
Marcelo Tosattic9027602010-03-01 20:25:08 -03001434 if (area == MAP_FAILED) {
Paolo Bonzini7f56e742014-05-14 17:43:20 +08001435 error_setg_errno(errp, errno,
Markus Armbrusterfd97fd42016-03-07 20:25:13 +01001436 "unable to map backing store for guest RAM");
Marc-André Lureau8d37b032017-06-02 18:12:22 +04001437 return NULL;
Marcelo Tosattic9027602010-03-01 20:25:08 -03001438 }
Marcelo Tosattief36fa12013-10-28 18:51:46 -02001439
Alex Williamson04b16652010-07-02 11:13:17 -06001440 block->fd = fd;
Alexander Graf4b870dc2023-04-03 22:14:21 +00001441 block->fd_offset = offset;
Marcelo Tosattic9027602010-03-01 20:25:08 -03001442 return area;
1443}
1444#endif
1445
Dr. David Alan Gilbert154cc9e2018-01-05 17:01:37 +00001446/* Allocate space within the ram_addr_t space that governs the
1447 * dirty bitmaps.
1448 * Called with the ramlist lock held.
1449 */
Alex Williamsond17b5282010-06-25 11:08:38 -06001450static ram_addr_t find_ram_offset(ram_addr_t size)
1451{
Alex Williamson04b16652010-07-02 11:13:17 -06001452 RAMBlock *block, *next_block;
Alex Williamson3e837b22011-10-31 08:54:09 -06001453 ram_addr_t offset = RAM_ADDR_MAX, mingap = RAM_ADDR_MAX;
Alex Williamson04b16652010-07-02 11:13:17 -06001454
Stefan Hajnoczi49cd9ac2013-03-11 10:20:21 +01001455 assert(size != 0); /* it would hand out same offset multiple times */
1456
Mike Day0dc3f442013-09-05 14:41:35 -04001457 if (QLIST_EMPTY_RCU(&ram_list.blocks)) {
Alex Williamson04b16652010-07-02 11:13:17 -06001458 return 0;
Mike Day0d53d9f2015-01-21 13:45:24 +01001459 }
Alex Williamson04b16652010-07-02 11:13:17 -06001460
Peter Xu99e15582017-05-12 12:17:39 +08001461 RAMBLOCK_FOREACH(block) {
Dr. David Alan Gilbert154cc9e2018-01-05 17:01:37 +00001462 ram_addr_t candidate, next = RAM_ADDR_MAX;
Alex Williamson04b16652010-07-02 11:13:17 -06001463
Dr. David Alan Gilbert801110a2018-01-05 17:01:38 +00001464 /* Align blocks to start on a 'long' in the bitmap
1465 * which makes the bitmap sync'ing take the fast path.
1466 */
Dr. David Alan Gilbert154cc9e2018-01-05 17:01:37 +00001467 candidate = block->offset + block->max_length;
Dr. David Alan Gilbert801110a2018-01-05 17:01:38 +00001468 candidate = ROUND_UP(candidate, BITS_PER_LONG << TARGET_PAGE_BITS);
Alex Williamson04b16652010-07-02 11:13:17 -06001469
Dr. David Alan Gilbert154cc9e2018-01-05 17:01:37 +00001470 /* Search for the closest following block
1471 * and find the gap.
1472 */
Peter Xu99e15582017-05-12 12:17:39 +08001473 RAMBLOCK_FOREACH(next_block) {
Dr. David Alan Gilbert154cc9e2018-01-05 17:01:37 +00001474 if (next_block->offset >= candidate) {
Alex Williamson04b16652010-07-02 11:13:17 -06001475 next = MIN(next, next_block->offset);
1476 }
1477 }
Dr. David Alan Gilbert154cc9e2018-01-05 17:01:37 +00001478
1479 /* If it fits remember our place and remember the size
1480 * of gap, but keep going so that we might find a smaller
1481 * gap to fill so avoiding fragmentation.
1482 */
1483 if (next - candidate >= size && next - candidate < mingap) {
1484 offset = candidate;
1485 mingap = next - candidate;
Alex Williamson04b16652010-07-02 11:13:17 -06001486 }
Dr. David Alan Gilbert154cc9e2018-01-05 17:01:37 +00001487
1488 trace_find_ram_offset_loop(size, candidate, offset, next, mingap);
Alex Williamson04b16652010-07-02 11:13:17 -06001489 }
Alex Williamson3e837b22011-10-31 08:54:09 -06001490
1491 if (offset == RAM_ADDR_MAX) {
1492 fprintf(stderr, "Failed to find gap of requested size: %" PRIu64 "\n",
1493 (uint64_t)size);
1494 abort();
1495 }
1496
Dr. David Alan Gilbert154cc9e2018-01-05 17:01:37 +00001497 trace_find_ram_offset(size, offset);
1498
Alex Williamson04b16652010-07-02 11:13:17 -06001499 return offset;
1500}
1501
David Hildenbrandc1361802018-06-20 22:27:36 +02001502static unsigned long last_ram_page(void)
Alex Williamson04b16652010-07-02 11:13:17 -06001503{
Alex Williamsond17b5282010-06-25 11:08:38 -06001504 RAMBlock *block;
1505 ram_addr_t last = 0;
1506
Dr. David Alan Gilbert694ea272019-10-07 15:36:41 +01001507 RCU_READ_LOCK_GUARD();
Peter Xu99e15582017-05-12 12:17:39 +08001508 RAMBLOCK_FOREACH(block) {
Michael S. Tsirkin62be4e32014-11-12 14:27:41 +02001509 last = MAX(last, block->offset + block->max_length);
Mike Day0d53d9f2015-01-21 13:45:24 +01001510 }
Juan Quintelab8c48992017-03-21 17:44:30 +01001511 return last >> TARGET_PAGE_BITS;
Alex Williamsond17b5282010-06-25 11:08:38 -06001512}
1513
Jason Baronddb97f12012-08-02 15:44:16 -04001514static void qemu_ram_setup_dump(void *addr, ram_addr_t size)
1515{
1516 int ret;
Jason Baronddb97f12012-08-02 15:44:16 -04001517
1518 /* Use MADV_DONTDUMP, if user doesn't want the guest memory in the core */
Marcel Apfelbaum47c8ca52015-02-04 17:43:54 +02001519 if (!machine_dump_guest_core(current_machine)) {
Jason Baronddb97f12012-08-02 15:44:16 -04001520 ret = qemu_madvise(addr, size, QEMU_MADV_DONTDUMP);
1521 if (ret) {
1522 perror("qemu_madvise");
1523 fprintf(stderr, "madvise doesn't support MADV_DONTDUMP, "
1524 "but dump_guest_core=off specified\n");
1525 }
1526 }
1527}
1528
Dr. David Alan Gilbert422148d2015-11-05 18:10:32 +00001529const char *qemu_ram_get_idstr(RAMBlock *rb)
1530{
1531 return rb->idstr;
1532}
1533
Yury Kotov754cb9c2019-02-15 20:45:44 +03001534void *qemu_ram_get_host_addr(RAMBlock *rb)
1535{
1536 return rb->host;
1537}
1538
1539ram_addr_t qemu_ram_get_offset(RAMBlock *rb)
1540{
1541 return rb->offset;
1542}
1543
1544ram_addr_t qemu_ram_get_used_length(RAMBlock *rb)
1545{
1546 return rb->used_length;
1547}
1548
David Hildenbrand082851a2021-04-29 13:26:59 +02001549ram_addr_t qemu_ram_get_max_length(RAMBlock *rb)
1550{
1551 return rb->max_length;
1552}
1553
Dr. David Alan Gilbert463a4ac2017-03-07 18:36:36 +00001554bool qemu_ram_is_shared(RAMBlock *rb)
1555{
1556 return rb->flags & RAM_SHARED;
1557}
1558
David Hildenbrand8dbe22c2021-05-10 13:43:21 +02001559bool qemu_ram_is_noreserve(RAMBlock *rb)
1560{
1561 return rb->flags & RAM_NORESERVE;
1562}
1563
Dr. David Alan Gilbert2ce16642018-03-12 17:20:58 +00001564/* Note: Only set at the start of postcopy */
1565bool qemu_ram_is_uf_zeroable(RAMBlock *rb)
1566{
1567 return rb->flags & RAM_UF_ZEROPAGE;
1568}
1569
1570void qemu_ram_set_uf_zeroable(RAMBlock *rb)
1571{
1572 rb->flags |= RAM_UF_ZEROPAGE;
1573}
1574
Cédric Le Goaterb895de52018-05-14 08:57:00 +02001575bool qemu_ram_is_migratable(RAMBlock *rb)
1576{
1577 return rb->flags & RAM_MIGRATABLE;
1578}
1579
1580void qemu_ram_set_migratable(RAMBlock *rb)
1581{
1582 rb->flags |= RAM_MIGRATABLE;
1583}
1584
1585void qemu_ram_unset_migratable(RAMBlock *rb)
1586{
1587 rb->flags &= ~RAM_MIGRATABLE;
1588}
1589
Steve Sistareb0182e52023-06-07 08:18:36 -07001590bool qemu_ram_is_named_file(RAMBlock *rb)
1591{
1592 return rb->flags & RAM_NAMED_FILE;
1593}
1594
Stefan Hajnoczi6d998f32022-10-13 14:59:05 -04001595int qemu_ram_get_fd(RAMBlock *rb)
1596{
1597 return rb->fd;
1598}
1599
Mike Dayae3a7042013-09-05 14:41:35 -04001600/* Called with iothread lock held. */
Gongleifa53a0e2016-05-10 10:04:59 +08001601void qemu_ram_set_idstr(RAMBlock *new_block, const char *name, DeviceState *dev)
Hu Tao20cfe882014-04-02 15:13:26 +08001602{
Gongleifa53a0e2016-05-10 10:04:59 +08001603 RAMBlock *block;
Hu Tao20cfe882014-04-02 15:13:26 +08001604
Avi Kivityc5705a72011-12-20 15:59:12 +02001605 assert(new_block);
1606 assert(!new_block->idstr[0]);
Cam Macdonell84b89d72010-07-26 18:10:57 -06001607
Anthony Liguori09e5ab62012-02-03 12:28:43 -06001608 if (dev) {
1609 char *id = qdev_get_dev_path(dev);
Cam Macdonell84b89d72010-07-26 18:10:57 -06001610 if (id) {
1611 snprintf(new_block->idstr, sizeof(new_block->idstr), "%s/", id);
Anthony Liguori7267c092011-08-20 22:09:37 -05001612 g_free(id);
Cam Macdonell84b89d72010-07-26 18:10:57 -06001613 }
1614 }
1615 pstrcat(new_block->idstr, sizeof(new_block->idstr), name);
1616
Dr. David Alan Gilbert694ea272019-10-07 15:36:41 +01001617 RCU_READ_LOCK_GUARD();
Peter Xu99e15582017-05-12 12:17:39 +08001618 RAMBLOCK_FOREACH(block) {
Gongleifa53a0e2016-05-10 10:04:59 +08001619 if (block != new_block &&
1620 !strcmp(block->idstr, new_block->idstr)) {
Cam Macdonell84b89d72010-07-26 18:10:57 -06001621 fprintf(stderr, "RAMBlock \"%s\" already registered, abort!\n",
1622 new_block->idstr);
1623 abort();
1624 }
1625 }
Avi Kivityc5705a72011-12-20 15:59:12 +02001626}
1627
Mike Dayae3a7042013-09-05 14:41:35 -04001628/* Called with iothread lock held. */
Gongleifa53a0e2016-05-10 10:04:59 +08001629void qemu_ram_unset_idstr(RAMBlock *block)
Hu Tao20cfe882014-04-02 15:13:26 +08001630{
Mike Dayae3a7042013-09-05 14:41:35 -04001631 /* FIXME: arch_init.c assumes that this is not called throughout
1632 * migration. Ignore the problem since hot-unplug during migration
1633 * does not work anyway.
1634 */
Hu Tao20cfe882014-04-02 15:13:26 +08001635 if (block) {
1636 memset(block->idstr, 0, sizeof(block->idstr));
1637 }
1638}
1639
Dr. David Alan Gilbert863e9622016-09-29 20:09:37 +01001640size_t qemu_ram_pagesize(RAMBlock *rb)
1641{
1642 return rb->page_size;
1643}
1644
Dr. David Alan Gilbert67f11b52017-02-24 18:28:34 +00001645/* Returns the largest size of page in use */
1646size_t qemu_ram_pagesize_largest(void)
1647{
1648 RAMBlock *block;
1649 size_t largest = 0;
1650
Peter Xu99e15582017-05-12 12:17:39 +08001651 RAMBLOCK_FOREACH(block) {
Dr. David Alan Gilbert67f11b52017-02-24 18:28:34 +00001652 largest = MAX(largest, qemu_ram_pagesize(block));
1653 }
1654
1655 return largest;
1656}
1657
Luiz Capitulino8490fc72012-09-05 16:50:16 -03001658static int memory_try_enable_merging(void *addr, size_t len)
1659{
Marcel Apfelbaum75cc7f02015-02-04 17:43:55 +02001660 if (!machine_mem_merge(current_machine)) {
Luiz Capitulino8490fc72012-09-05 16:50:16 -03001661 /* disabled by the user */
1662 return 0;
1663 }
1664
1665 return qemu_madvise(addr, len, QEMU_MADV_MERGEABLE);
1666}
1667
David Hildenbrandc7c0e722021-04-29 13:27:02 +02001668/*
1669 * Resizing RAM while migrating can result in the migration being canceled.
1670 * Care has to be taken if the guest might have already detected the memory.
Michael S. Tsirkin62be4e32014-11-12 14:27:41 +02001671 *
1672 * As memory core doesn't know how is memory accessed, it is up to
1673 * resize callback to update device state and/or add assertions to detect
1674 * misuse, if necessary.
1675 */
Gongleifa53a0e2016-05-10 10:04:59 +08001676int qemu_ram_resize(RAMBlock *block, ram_addr_t newsize, Error **errp)
Michael S. Tsirkin62be4e32014-11-12 14:27:41 +02001677{
David Hildenbrand8f443042021-04-29 13:27:00 +02001678 const ram_addr_t oldsize = block->used_length;
David Hildenbrandce4adc02020-04-03 11:18:27 +01001679 const ram_addr_t unaligned_size = newsize;
1680
Michael S. Tsirkin62be4e32014-11-12 14:27:41 +02001681 assert(block);
1682
Dr. David Alan Gilbert4ed023c2015-11-05 18:11:16 +00001683 newsize = HOST_PAGE_ALIGN(newsize);
Michael S. Tsirkin129ddaf2015-02-17 10:15:30 +01001684
Michael S. Tsirkin62be4e32014-11-12 14:27:41 +02001685 if (block->used_length == newsize) {
David Hildenbrandce4adc02020-04-03 11:18:27 +01001686 /*
1687 * We don't have to resize the ram block (which only knows aligned
1688 * sizes), however, we have to notify if the unaligned size changed.
1689 */
1690 if (unaligned_size != memory_region_size(block->mr)) {
1691 memory_region_set_size(block->mr, unaligned_size);
1692 if (block->resized) {
1693 block->resized(block->idstr, unaligned_size, block->host);
1694 }
1695 }
Michael S. Tsirkin62be4e32014-11-12 14:27:41 +02001696 return 0;
1697 }
1698
1699 if (!(block->flags & RAM_RESIZEABLE)) {
1700 error_setg_errno(errp, EINVAL,
Pankaj Guptaa3a92902020-10-22 13:13:02 +02001701 "Size mismatch: %s: 0x" RAM_ADDR_FMT
1702 " != 0x" RAM_ADDR_FMT, block->idstr,
Michael S. Tsirkin62be4e32014-11-12 14:27:41 +02001703 newsize, block->used_length);
1704 return -EINVAL;
1705 }
1706
1707 if (block->max_length < newsize) {
1708 error_setg_errno(errp, EINVAL,
Pankaj Guptaa3a92902020-10-22 13:13:02 +02001709 "Size too large: %s: 0x" RAM_ADDR_FMT
Michael S. Tsirkin62be4e32014-11-12 14:27:41 +02001710 " > 0x" RAM_ADDR_FMT, block->idstr,
1711 newsize, block->max_length);
1712 return -EINVAL;
1713 }
1714
David Hildenbrand8f443042021-04-29 13:27:00 +02001715 /* Notify before modifying the ram block and touching the bitmaps. */
1716 if (block->host) {
1717 ram_block_notify_resize(block->host, oldsize, newsize);
1718 }
1719
Michael S. Tsirkin62be4e32014-11-12 14:27:41 +02001720 cpu_physical_memory_clear_dirty_range(block->offset, block->used_length);
1721 block->used_length = newsize;
Paolo Bonzini58d27072015-03-23 11:56:01 +01001722 cpu_physical_memory_set_dirty_range(block->offset, block->used_length,
1723 DIRTY_CLIENTS_ALL);
David Hildenbrandce4adc02020-04-03 11:18:27 +01001724 memory_region_set_size(block->mr, unaligned_size);
Michael S. Tsirkin62be4e32014-11-12 14:27:41 +02001725 if (block->resized) {
David Hildenbrandce4adc02020-04-03 11:18:27 +01001726 block->resized(block->idstr, unaligned_size, block->host);
Michael S. Tsirkin62be4e32014-11-12 14:27:41 +02001727 }
1728 return 0;
1729}
1730
Beata Michalska61c490e2019-11-21 00:08:41 +00001731/*
1732 * Trigger sync on the given ram block for range [start, start + length]
1733 * with the backing store if one is available.
1734 * Otherwise no-op.
1735 * @Note: this is supposed to be a synchronous op.
1736 */
Philippe Mathieu-Daudéab7e41e2020-05-08 08:24:56 +02001737void qemu_ram_msync(RAMBlock *block, ram_addr_t start, ram_addr_t length)
Beata Michalska61c490e2019-11-21 00:08:41 +00001738{
Beata Michalska61c490e2019-11-21 00:08:41 +00001739 /* The requested range should fit in within the block range */
1740 g_assert((start + length) <= block->used_length);
1741
1742#ifdef CONFIG_LIBPMEM
1743 /* The lack of support for pmem should not block the sync */
1744 if (ramblock_is_pmem(block)) {
Anthony PERARD5d4c9542019-12-19 15:43:22 +00001745 void *addr = ramblock_ptr(block, start);
Beata Michalska61c490e2019-11-21 00:08:41 +00001746 pmem_persist(addr, length);
1747 return;
1748 }
1749#endif
1750 if (block->fd >= 0) {
1751 /**
1752 * Case there is no support for PMEM or the memory has not been
1753 * specified as persistent (or is not one) - use the msync.
1754 * Less optimal but still achieves the same goal
1755 */
Anthony PERARD5d4c9542019-12-19 15:43:22 +00001756 void *addr = ramblock_ptr(block, start);
Beata Michalska61c490e2019-11-21 00:08:41 +00001757 if (qemu_msync(addr, length, block->fd)) {
1758 warn_report("%s: failed to sync memory range: start: "
1759 RAM_ADDR_FMT " length: " RAM_ADDR_FMT,
1760 __func__, start, length);
1761 }
1762 }
1763}
1764
Stefan Hajnoczi5b82b702016-01-25 13:33:20 +00001765/* Called with ram_list.mutex held */
1766static void dirty_memory_extend(ram_addr_t old_ram_size,
1767 ram_addr_t new_ram_size)
1768{
1769 ram_addr_t old_num_blocks = DIV_ROUND_UP(old_ram_size,
1770 DIRTY_MEMORY_BLOCK_SIZE);
1771 ram_addr_t new_num_blocks = DIV_ROUND_UP(new_ram_size,
1772 DIRTY_MEMORY_BLOCK_SIZE);
1773 int i;
1774
1775 /* Only need to extend if block count increased */
1776 if (new_num_blocks <= old_num_blocks) {
1777 return;
1778 }
1779
1780 for (i = 0; i < DIRTY_MEMORY_NUM; i++) {
1781 DirtyMemoryBlocks *old_blocks;
1782 DirtyMemoryBlocks *new_blocks;
1783 int j;
1784
Stefan Hajnoczid73415a2020-09-23 11:56:46 +01001785 old_blocks = qatomic_rcu_read(&ram_list.dirty_memory[i]);
Stefan Hajnoczi5b82b702016-01-25 13:33:20 +00001786 new_blocks = g_malloc(sizeof(*new_blocks) +
1787 sizeof(new_blocks->blocks[0]) * new_num_blocks);
1788
1789 if (old_num_blocks) {
1790 memcpy(new_blocks->blocks, old_blocks->blocks,
1791 old_num_blocks * sizeof(old_blocks->blocks[0]));
1792 }
1793
1794 for (j = old_num_blocks; j < new_num_blocks; j++) {
1795 new_blocks->blocks[j] = bitmap_new(DIRTY_MEMORY_BLOCK_SIZE);
1796 }
1797
Stefan Hajnoczid73415a2020-09-23 11:56:46 +01001798 qatomic_rcu_set(&ram_list.dirty_memory[i], new_blocks);
Stefan Hajnoczi5b82b702016-01-25 13:33:20 +00001799
1800 if (old_blocks) {
1801 g_free_rcu(old_blocks, rcu);
1802 }
1803 }
1804}
1805
David Hildenbrand7ce18ca2021-04-06 10:01:24 +02001806static void ram_block_add(RAMBlock *new_block, Error **errp)
Avi Kivityc5705a72011-12-20 15:59:12 +02001807{
David Hildenbrand8dbe22c2021-05-10 13:43:21 +02001808 const bool noreserve = qemu_ram_is_noreserve(new_block);
David Hildenbrand7ce18ca2021-04-06 10:01:24 +02001809 const bool shared = qemu_ram_is_shared(new_block);
Paolo Bonzinie1c57ab2014-05-14 17:43:18 +08001810 RAMBlock *block;
Mike Day0d53d9f2015-01-21 13:45:24 +01001811 RAMBlock *last_block = NULL;
Juan Quintela2152f5c2013-10-08 13:52:02 +02001812 ram_addr_t old_ram_size, new_ram_size;
Markus Armbruster37aa7a02016-01-14 16:09:39 +01001813 Error *err = NULL;
Juan Quintela2152f5c2013-10-08 13:52:02 +02001814
Juan Quintelab8c48992017-03-21 17:44:30 +01001815 old_ram_size = last_ram_page();
Avi Kivityc5705a72011-12-20 15:59:12 +02001816
Umesh Deshpandeb2a86582011-08-17 00:01:33 -07001817 qemu_mutex_lock_ramlist();
Michael S. Tsirkin9b8424d2014-12-15 22:55:32 +02001818 new_block->offset = find_ram_offset(new_block->max_length);
Paolo Bonzinie1c57ab2014-05-14 17:43:18 +08001819
1820 if (!new_block->host) {
1821 if (xen_enabled()) {
Michael S. Tsirkin9b8424d2014-12-15 22:55:32 +02001822 xen_ram_alloc(new_block->offset, new_block->max_length,
Markus Armbruster37aa7a02016-01-14 16:09:39 +01001823 new_block->mr, &err);
1824 if (err) {
1825 error_propagate(errp, err);
1826 qemu_mutex_unlock_ramlist();
Paolo Bonzini39c350e2016-03-09 18:14:01 +01001827 return;
Markus Armbruster37aa7a02016-01-14 16:09:39 +01001828 }
Paolo Bonzinie1c57ab2014-05-14 17:43:18 +08001829 } else {
David Hildenbrand25459eb2021-03-03 14:09:16 +01001830 new_block->host = qemu_anon_ram_alloc(new_block->max_length,
1831 &new_block->mr->align,
David Hildenbrand8dbe22c2021-05-10 13:43:21 +02001832 shared, noreserve);
Markus Armbruster39228252013-07-31 15:11:11 +02001833 if (!new_block->host) {
Hu Taoef701d72014-09-09 13:27:54 +08001834 error_setg_errno(errp, errno,
1835 "cannot set up guest memory '%s'",
1836 memory_region_name(new_block->mr));
1837 qemu_mutex_unlock_ramlist();
Paolo Bonzini39c350e2016-03-09 18:14:01 +01001838 return;
Markus Armbruster39228252013-07-31 15:11:11 +02001839 }
Michael S. Tsirkin9b8424d2014-12-15 22:55:32 +02001840 memory_try_enable_merging(new_block->host, new_block->max_length);
Yoshiaki Tamura6977dfe2010-08-18 15:41:49 +09001841 }
1842 }
Cam Macdonell84b89d72010-07-26 18:10:57 -06001843
Li Zhijiandd631692015-07-02 20:18:06 +08001844 new_ram_size = MAX(old_ram_size,
1845 (new_block->offset + new_block->max_length) >> TARGET_PAGE_BITS);
1846 if (new_ram_size > old_ram_size) {
Stefan Hajnoczi5b82b702016-01-25 13:33:20 +00001847 dirty_memory_extend(old_ram_size, new_ram_size);
Li Zhijiandd631692015-07-02 20:18:06 +08001848 }
Mike Day0d53d9f2015-01-21 13:45:24 +01001849 /* Keep the list sorted from biggest to smallest block. Unlike QTAILQ,
1850 * QLIST (which has an RCU-friendly variant) does not have insertion at
1851 * tail, so save the last element in last_block.
1852 */
Peter Xu99e15582017-05-12 12:17:39 +08001853 RAMBLOCK_FOREACH(block) {
Mike Day0d53d9f2015-01-21 13:45:24 +01001854 last_block = block;
Michael S. Tsirkin9b8424d2014-12-15 22:55:32 +02001855 if (block->max_length < new_block->max_length) {
Paolo Bonziniabb26d62012-11-14 16:00:51 +01001856 break;
1857 }
1858 }
1859 if (block) {
Mike Day0dc3f442013-09-05 14:41:35 -04001860 QLIST_INSERT_BEFORE_RCU(block, new_block, next);
Mike Day0d53d9f2015-01-21 13:45:24 +01001861 } else if (last_block) {
Mike Day0dc3f442013-09-05 14:41:35 -04001862 QLIST_INSERT_AFTER_RCU(last_block, new_block, next);
Mike Day0d53d9f2015-01-21 13:45:24 +01001863 } else { /* list is empty */
Mike Day0dc3f442013-09-05 14:41:35 -04001864 QLIST_INSERT_HEAD_RCU(&ram_list.blocks, new_block, next);
Paolo Bonziniabb26d62012-11-14 16:00:51 +01001865 }
Paolo Bonzini0d6d3c82012-11-14 15:45:02 +01001866 ram_list.mru_block = NULL;
Cam Macdonell84b89d72010-07-26 18:10:57 -06001867
Mike Day0dc3f442013-09-05 14:41:35 -04001868 /* Write list before version */
1869 smp_wmb();
Umesh Deshpandef798b072011-08-18 11:41:17 -07001870 ram_list.version++;
Umesh Deshpandeb2a86582011-08-17 00:01:33 -07001871 qemu_mutex_unlock_ramlist();
Umesh Deshpandef798b072011-08-18 11:41:17 -07001872
Michael S. Tsirkin9b8424d2014-12-15 22:55:32 +02001873 cpu_physical_memory_set_dirty_range(new_block->offset,
Paolo Bonzini58d27072015-03-23 11:56:01 +01001874 new_block->used_length,
1875 DIRTY_CLIENTS_ALL);
Cam Macdonell84b89d72010-07-26 18:10:57 -06001876
Paolo Bonzinia904c912015-01-21 16:18:35 +01001877 if (new_block->host) {
1878 qemu_ram_setup_dump(new_block->host, new_block->max_length);
1879 qemu_madvise(new_block->host, new_block->max_length, QEMU_MADV_HUGEPAGE);
Alexander Bulekova028ede2020-02-19 23:11:09 -05001880 /*
1881 * MADV_DONTFORK is also needed by KVM in absence of synchronous MMU
1882 * Configure it unless the machine is a qtest server, in which case
1883 * KVM is not used and it may be forked (eg for fuzzing purposes).
1884 */
1885 if (!qtest_enabled()) {
1886 qemu_madvise(new_block->host, new_block->max_length,
1887 QEMU_MADV_DONTFORK);
1888 }
David Hildenbrand8f443042021-04-29 13:27:00 +02001889 ram_block_notify_add(new_block->host, new_block->used_length,
1890 new_block->max_length);
Paolo Bonzinie1c57ab2014-05-14 17:43:18 +08001891 }
Cam Macdonell84b89d72010-07-26 18:10:57 -06001892}
1893
Hikaru Nishidad5dbde42018-09-24 21:32:05 +09001894#ifdef CONFIG_POSIX
Marc-André Lureau38b33622017-06-02 18:12:23 +04001895RAMBlock *qemu_ram_alloc_from_fd(ram_addr_t size, MemoryRegion *mr,
Jagannathan Raman44a4ff32021-01-29 11:46:04 -05001896 uint32_t ram_flags, int fd, off_t offset,
David Hildenbrand5c52a212023-09-06 14:04:54 +02001897 Error **errp)
Paolo Bonzinie1c57ab2014-05-14 17:43:18 +08001898{
1899 RAMBlock *new_block;
Hu Taoef701d72014-09-09 13:27:54 +08001900 Error *local_err = NULL;
Jingqi Liuce317be2020-04-29 16:50:09 +08001901 int64_t file_size, file_align;
Paolo Bonzinie1c57ab2014-05-14 17:43:18 +08001902
Junyan Hea4de8552018-07-18 15:48:00 +08001903 /* Just support these ram flags by now. */
Sean Christopherson56918a12021-07-19 19:21:04 +08001904 assert((ram_flags & ~(RAM_SHARED | RAM_PMEM | RAM_NORESERVE |
David Hildenbrand5c52a212023-09-06 14:04:54 +02001905 RAM_PROTECTED | RAM_NAMED_FILE | RAM_READONLY |
1906 RAM_READONLY_FD)) == 0);
Junyan Hea4de8552018-07-18 15:48:00 +08001907
Paolo Bonzinie1c57ab2014-05-14 17:43:18 +08001908 if (xen_enabled()) {
Paolo Bonzini7f56e742014-05-14 17:43:20 +08001909 error_setg(errp, "-mem-path not supported with Xen");
Fam Zheng528f46a2016-03-01 14:18:18 +08001910 return NULL;
Paolo Bonzinie1c57ab2014-05-14 17:43:18 +08001911 }
1912
Marc-André Lureaue45e7ae2017-06-02 18:12:21 +04001913 if (kvm_enabled() && !kvm_has_sync_mmu()) {
1914 error_setg(errp,
1915 "host lacks kvm mmu notifiers, -mem-path unsupported");
1916 return NULL;
1917 }
1918
Dr. David Alan Gilbert4ed023c2015-11-05 18:11:16 +00001919 size = HOST_PAGE_ALIGN(size);
Marc-André Lureau8d37b032017-06-02 18:12:22 +04001920 file_size = get_file_size(fd);
Alexander Graf4b870dc2023-04-03 22:14:21 +00001921 if (file_size > offset && file_size < (offset + size)) {
Igor Mammedovc001c3b2020-02-19 11:09:48 -05001922 error_setg(errp, "backing store size 0x%" PRIx64
Marc-André Lureau8d37b032017-06-02 18:12:22 +04001923 " does not match 'size' option 0x" RAM_ADDR_FMT,
Igor Mammedovc001c3b2020-02-19 11:09:48 -05001924 file_size, size);
Marc-André Lureau8d37b032017-06-02 18:12:22 +04001925 return NULL;
1926 }
1927
Jingqi Liuce317be2020-04-29 16:50:09 +08001928 file_align = get_file_align(fd);
Peter Maydell8f1bdb02021-08-12 16:06:24 +01001929 if (file_align > 0 && file_align > mr->align) {
Jingqi Liuce317be2020-04-29 16:50:09 +08001930 error_setg(errp, "backing store align 0x%" PRIx64
Jingqi Liu5f509752020-04-29 16:50:10 +08001931 " is larger than 'align' option 0x%" PRIx64,
Jingqi Liuce317be2020-04-29 16:50:09 +08001932 file_align, mr->align);
1933 return NULL;
1934 }
1935
Paolo Bonzinie1c57ab2014-05-14 17:43:18 +08001936 new_block = g_malloc0(sizeof(*new_block));
1937 new_block->mr = mr;
Michael S. Tsirkin9b8424d2014-12-15 22:55:32 +02001938 new_block->used_length = size;
1939 new_block->max_length = size;
Junyan Hecbfc0172018-07-18 15:47:58 +08001940 new_block->flags = ram_flags;
David Hildenbrand5c52a212023-09-06 14:04:54 +02001941 new_block->host = file_ram_alloc(new_block, size, fd, !file_size, offset,
1942 errp);
Paolo Bonzini7f56e742014-05-14 17:43:20 +08001943 if (!new_block->host) {
1944 g_free(new_block);
Fam Zheng528f46a2016-03-01 14:18:18 +08001945 return NULL;
Paolo Bonzini7f56e742014-05-14 17:43:20 +08001946 }
1947
David Hildenbrand7ce18ca2021-04-06 10:01:24 +02001948 ram_block_add(new_block, &local_err);
Hu Taoef701d72014-09-09 13:27:54 +08001949 if (local_err) {
1950 g_free(new_block);
1951 error_propagate(errp, local_err);
Fam Zheng528f46a2016-03-01 14:18:18 +08001952 return NULL;
Hu Taoef701d72014-09-09 13:27:54 +08001953 }
Fam Zheng528f46a2016-03-01 14:18:18 +08001954 return new_block;
Marc-André Lureau38b33622017-06-02 18:12:23 +04001955
1956}
1957
1958
1959RAMBlock *qemu_ram_alloc_from_file(ram_addr_t size, MemoryRegion *mr,
Junyan Hecbfc0172018-07-18 15:47:58 +08001960 uint32_t ram_flags, const char *mem_path,
David Hildenbrand5c52a212023-09-06 14:04:54 +02001961 off_t offset, Error **errp)
Marc-André Lureau38b33622017-06-02 18:12:23 +04001962{
1963 int fd;
1964 bool created;
1965 RAMBlock *block;
1966
David Hildenbrand5c52a212023-09-06 14:04:54 +02001967 fd = file_ram_open(mem_path, memory_region_name(mr),
David Hildenbrand4d6b23f2023-09-06 14:04:58 +02001968 !!(ram_flags & RAM_READONLY_FD), &created);
Marc-André Lureau38b33622017-06-02 18:12:23 +04001969 if (fd < 0) {
David Hildenbrand4d6b23f2023-09-06 14:04:58 +02001970 error_setg_errno(errp, -fd, "can't open backing store %s for guest RAM",
1971 mem_path);
David Hildenbrand6da4b1c2023-09-06 14:05:02 +02001972 if (!(ram_flags & RAM_READONLY_FD) && !(ram_flags & RAM_SHARED) &&
1973 fd == -EACCES) {
1974 /*
1975 * If we can open the file R/O (note: will never create a new file)
1976 * and we are dealing with a private mapping, there are still ways
1977 * to consume such files and get RAM instead of ROM.
1978 */
1979 fd = file_ram_open(mem_path, memory_region_name(mr), true,
1980 &created);
1981 if (fd < 0) {
1982 return NULL;
1983 }
1984 assert(!created);
1985 close(fd);
1986 error_append_hint(errp, "Consider opening the backing store"
1987 " read-only but still creating writable RAM using"
1988 " '-object memory-backend-file,readonly=on,rom=off...'"
1989 " (see \"VM templating\" documentation)\n");
1990 }
Marc-André Lureau38b33622017-06-02 18:12:23 +04001991 return NULL;
1992 }
1993
David Hildenbrand5c52a212023-09-06 14:04:54 +02001994 block = qemu_ram_alloc_from_fd(size, mr, ram_flags, fd, offset, errp);
Marc-André Lureau38b33622017-06-02 18:12:23 +04001995 if (!block) {
1996 if (created) {
1997 unlink(mem_path);
1998 }
1999 close(fd);
2000 return NULL;
2001 }
2002
2003 return block;
Paolo Bonzinie1c57ab2014-05-14 17:43:18 +08002004}
Paolo Bonzini0b183fc2014-05-14 17:43:19 +08002005#endif
Paolo Bonzinie1c57ab2014-05-14 17:43:18 +08002006
Michael S. Tsirkin62be4e32014-11-12 14:27:41 +02002007static
Fam Zheng528f46a2016-03-01 14:18:18 +08002008RAMBlock *qemu_ram_alloc_internal(ram_addr_t size, ram_addr_t max_size,
2009 void (*resized)(const char*,
2010 uint64_t length,
2011 void *host),
David Hildenbrandebef62d2021-05-10 13:43:19 +02002012 void *host, uint32_t ram_flags,
Fam Zheng528f46a2016-03-01 14:18:18 +08002013 MemoryRegion *mr, Error **errp)
Paolo Bonzinie1c57ab2014-05-14 17:43:18 +08002014{
2015 RAMBlock *new_block;
Hu Taoef701d72014-09-09 13:27:54 +08002016 Error *local_err = NULL;
Paolo Bonzinie1c57ab2014-05-14 17:43:18 +08002017
David Hildenbrand8dbe22c2021-05-10 13:43:21 +02002018 assert((ram_flags & ~(RAM_SHARED | RAM_RESIZEABLE | RAM_PREALLOC |
2019 RAM_NORESERVE)) == 0);
David Hildenbrandebef62d2021-05-10 13:43:19 +02002020 assert(!host ^ (ram_flags & RAM_PREALLOC));
2021
Dr. David Alan Gilbert4ed023c2015-11-05 18:11:16 +00002022 size = HOST_PAGE_ALIGN(size);
2023 max_size = HOST_PAGE_ALIGN(max_size);
Paolo Bonzinie1c57ab2014-05-14 17:43:18 +08002024 new_block = g_malloc0(sizeof(*new_block));
2025 new_block->mr = mr;
Michael S. Tsirkin62be4e32014-11-12 14:27:41 +02002026 new_block->resized = resized;
Michael S. Tsirkin9b8424d2014-12-15 22:55:32 +02002027 new_block->used_length = size;
2028 new_block->max_length = max_size;
Michael S. Tsirkin62be4e32014-11-12 14:27:41 +02002029 assert(max_size >= size);
Paolo Bonzinie1c57ab2014-05-14 17:43:18 +08002030 new_block->fd = -1;
Marc-André Lureau8e3b0cb2022-03-23 19:57:22 +04002031 new_block->page_size = qemu_real_host_page_size();
Paolo Bonzinie1c57ab2014-05-14 17:43:18 +08002032 new_block->host = host;
David Hildenbrandebef62d2021-05-10 13:43:19 +02002033 new_block->flags = ram_flags;
David Hildenbrand7ce18ca2021-04-06 10:01:24 +02002034 ram_block_add(new_block, &local_err);
Hu Taoef701d72014-09-09 13:27:54 +08002035 if (local_err) {
2036 g_free(new_block);
2037 error_propagate(errp, local_err);
Fam Zheng528f46a2016-03-01 14:18:18 +08002038 return NULL;
Hu Taoef701d72014-09-09 13:27:54 +08002039 }
Fam Zheng528f46a2016-03-01 14:18:18 +08002040 return new_block;
Paolo Bonzinie1c57ab2014-05-14 17:43:18 +08002041}
2042
Fam Zheng528f46a2016-03-01 14:18:18 +08002043RAMBlock *qemu_ram_alloc_from_ptr(ram_addr_t size, void *host,
Michael S. Tsirkin62be4e32014-11-12 14:27:41 +02002044 MemoryRegion *mr, Error **errp)
2045{
David Hildenbrandebef62d2021-05-10 13:43:19 +02002046 return qemu_ram_alloc_internal(size, size, NULL, host, RAM_PREALLOC, mr,
2047 errp);
Michael S. Tsirkin62be4e32014-11-12 14:27:41 +02002048}
2049
David Hildenbrandebef62d2021-05-10 13:43:19 +02002050RAMBlock *qemu_ram_alloc(ram_addr_t size, uint32_t ram_flags,
Marcel Apfelbaum06329cc2017-12-13 16:37:37 +02002051 MemoryRegion *mr, Error **errp)
pbrook94a6b542009-04-11 17:15:54 +00002052{
David Hildenbrand8dbe22c2021-05-10 13:43:21 +02002053 assert((ram_flags & ~(RAM_SHARED | RAM_NORESERVE)) == 0);
David Hildenbrandebef62d2021-05-10 13:43:19 +02002054 return qemu_ram_alloc_internal(size, size, NULL, NULL, ram_flags, mr, errp);
Michael S. Tsirkin62be4e32014-11-12 14:27:41 +02002055}
2056
Fam Zheng528f46a2016-03-01 14:18:18 +08002057RAMBlock *qemu_ram_alloc_resizeable(ram_addr_t size, ram_addr_t maxsz,
Michael S. Tsirkin62be4e32014-11-12 14:27:41 +02002058 void (*resized)(const char*,
2059 uint64_t length,
2060 void *host),
2061 MemoryRegion *mr, Error **errp)
2062{
David Hildenbrandebef62d2021-05-10 13:43:19 +02002063 return qemu_ram_alloc_internal(size, maxsz, resized, NULL,
2064 RAM_RESIZEABLE, mr, errp);
pbrook94a6b542009-04-11 17:15:54 +00002065}
bellarde9a1ab12007-02-08 23:08:38 +00002066
Paolo Bonzini43771532013-09-09 17:58:40 +02002067static void reclaim_ramblock(RAMBlock *block)
2068{
2069 if (block->flags & RAM_PREALLOC) {
2070 ;
2071 } else if (xen_enabled()) {
2072 xen_invalidate_map_cache_entry(block->host);
2073#ifndef _WIN32
2074 } else if (block->fd >= 0) {
Murilo Opsfelder Araujo53adb9d2019-01-30 21:36:05 -02002075 qemu_ram_munmap(block->fd, block->host, block->max_length);
Paolo Bonzini43771532013-09-09 17:58:40 +02002076 close(block->fd);
2077#endif
2078 } else {
2079 qemu_anon_ram_free(block->host, block->max_length);
2080 }
2081 g_free(block);
2082}
2083
Fam Zhengf1060c52016-03-01 14:18:22 +08002084void qemu_ram_free(RAMBlock *block)
bellarde9a1ab12007-02-08 23:08:38 +00002085{
Marc-André Lureau85bc2a12016-03-29 13:20:51 +02002086 if (!block) {
2087 return;
2088 }
2089
Paolo Bonzini0987d732016-12-21 00:31:36 +08002090 if (block->host) {
David Hildenbrand8f443042021-04-29 13:27:00 +02002091 ram_block_notify_remove(block->host, block->used_length,
2092 block->max_length);
Paolo Bonzini0987d732016-12-21 00:31:36 +08002093 }
2094
Umesh Deshpandeb2a86582011-08-17 00:01:33 -07002095 qemu_mutex_lock_ramlist();
Fam Zhengf1060c52016-03-01 14:18:22 +08002096 QLIST_REMOVE_RCU(block, next);
2097 ram_list.mru_block = NULL;
2098 /* Write list before version */
2099 smp_wmb();
2100 ram_list.version++;
2101 call_rcu(block, reclaim_ramblock, rcu);
Umesh Deshpandeb2a86582011-08-17 00:01:33 -07002102 qemu_mutex_unlock_ramlist();
bellarde9a1ab12007-02-08 23:08:38 +00002103}
2104
Huang Yingcd19cfa2011-03-02 08:56:19 +01002105#ifndef _WIN32
2106void qemu_ram_remap(ram_addr_t addr, ram_addr_t length)
2107{
2108 RAMBlock *block;
2109 ram_addr_t offset;
2110 int flags;
2111 void *area, *vaddr;
David Hildenbrand9e6b9f32023-09-06 14:04:56 +02002112 int prot;
Huang Yingcd19cfa2011-03-02 08:56:19 +01002113
Peter Xu99e15582017-05-12 12:17:39 +08002114 RAMBLOCK_FOREACH(block) {
Huang Yingcd19cfa2011-03-02 08:56:19 +01002115 offset = addr - block->offset;
Michael S. Tsirkin9b8424d2014-12-15 22:55:32 +02002116 if (offset < block->max_length) {
Michael S. Tsirkin1240be22014-11-12 11:44:41 +02002117 vaddr = ramblock_ptr(block, offset);
Paolo Bonzini7bd4f432014-05-14 17:43:22 +08002118 if (block->flags & RAM_PREALLOC) {
Huang Yingcd19cfa2011-03-02 08:56:19 +01002119 ;
Markus Armbrusterdfeaf2a2013-07-31 15:11:05 +02002120 } else if (xen_enabled()) {
2121 abort();
Huang Yingcd19cfa2011-03-02 08:56:19 +01002122 } else {
2123 flags = MAP_FIXED;
David Hildenbranddbb92ee2021-04-06 10:01:26 +02002124 flags |= block->flags & RAM_SHARED ?
2125 MAP_SHARED : MAP_PRIVATE;
David Hildenbrandd94e0bc2021-05-10 13:43:22 +02002126 flags |= block->flags & RAM_NORESERVE ? MAP_NORESERVE : 0;
David Hildenbrand9e6b9f32023-09-06 14:04:56 +02002127 prot = PROT_READ;
2128 prot |= block->flags & RAM_READONLY ? 0 : PROT_WRITE;
Markus Armbruster3435f392013-07-31 15:11:07 +02002129 if (block->fd >= 0) {
David Hildenbrand9e6b9f32023-09-06 14:04:56 +02002130 area = mmap(vaddr, length, prot, flags, block->fd,
2131 offset + block->fd_offset);
Huang Yingcd19cfa2011-03-02 08:56:19 +01002132 } else {
David Hildenbranddbb92ee2021-04-06 10:01:26 +02002133 flags |= MAP_ANONYMOUS;
David Hildenbrand9e6b9f32023-09-06 14:04:56 +02002134 area = mmap(vaddr, length, prot, flags, -1, 0);
Huang Yingcd19cfa2011-03-02 08:56:19 +01002135 }
2136 if (area != vaddr) {
Alistair Francis493d89b2018-02-03 09:43:14 +01002137 error_report("Could not remap addr: "
2138 RAM_ADDR_FMT "@" RAM_ADDR_FMT "",
2139 length, addr);
Huang Yingcd19cfa2011-03-02 08:56:19 +01002140 exit(1);
2141 }
Luiz Capitulino8490fc72012-09-05 16:50:16 -03002142 memory_try_enable_merging(vaddr, length);
Jason Baronddb97f12012-08-02 15:44:16 -04002143 qemu_ram_setup_dump(vaddr, length);
Huang Yingcd19cfa2011-03-02 08:56:19 +01002144 }
Huang Yingcd19cfa2011-03-02 08:56:19 +01002145 }
2146 }
2147}
2148#endif /* !_WIN32 */
2149
Paolo Bonzini1b5ec232013-05-06 14:36:15 +02002150/* Return a host pointer to ram allocated with qemu_ram_alloc.
Mike Dayae3a7042013-09-05 14:41:35 -04002151 * This should not be used for general purpose DMA. Use address_space_map
2152 * or address_space_rw instead. For local memory (e.g. video ram) that the
2153 * device owns, use memory_region_get_ram_ptr.
Mike Day0dc3f442013-09-05 14:41:35 -04002154 *
Paolo Bonzini49b24af2015-12-16 10:30:47 +01002155 * Called within RCU critical section.
Paolo Bonzini1b5ec232013-05-06 14:36:15 +02002156 */
Paolo Bonzini0878d0e2016-02-22 11:02:12 +01002157void *qemu_map_ram_ptr(RAMBlock *ram_block, ram_addr_t addr)
Paolo Bonzini1b5ec232013-05-06 14:36:15 +02002158{
Gonglei3655cb92016-02-20 10:35:20 +08002159 RAMBlock *block = ram_block;
2160
2161 if (block == NULL) {
2162 block = qemu_get_ram_block(addr);
Paolo Bonzini0878d0e2016-02-22 11:02:12 +01002163 addr -= block->offset;
Gonglei3655cb92016-02-20 10:35:20 +08002164 }
Mike Dayae3a7042013-09-05 14:41:35 -04002165
2166 if (xen_enabled() && block->host == NULL) {
Paolo Bonzini0d6d3c82012-11-14 15:45:02 +01002167 /* We need to check if the requested address is in the RAM
2168 * because we don't want to map the entire memory in QEMU.
2169 * In that case just map until the end of the page.
2170 */
2171 if (block->offset == 0) {
Stefano Stabellini1ff7c592017-05-03 14:00:35 -07002172 return xen_map_cache(addr, 0, 0, false);
Paolo Bonzini0d6d3c82012-11-14 15:45:02 +01002173 }
Mike Dayae3a7042013-09-05 14:41:35 -04002174
Stefano Stabellini1ff7c592017-05-03 14:00:35 -07002175 block->host = xen_map_cache(block->offset, block->max_length, 1, false);
Paolo Bonzini0d6d3c82012-11-14 15:45:02 +01002176 }
Paolo Bonzini0878d0e2016-02-22 11:02:12 +01002177 return ramblock_ptr(block, addr);
pbrookdc828ca2009-04-09 22:21:07 +00002178}
2179
Paolo Bonzini0878d0e2016-02-22 11:02:12 +01002180/* Return a host pointer to guest's ram. Similar to qemu_map_ram_ptr
Mike Dayae3a7042013-09-05 14:41:35 -04002181 * but takes a size argument.
Mike Day0dc3f442013-09-05 14:41:35 -04002182 *
Paolo Bonzinie81bcda2015-12-16 10:31:26 +01002183 * Called within RCU critical section.
Mike Dayae3a7042013-09-05 14:41:35 -04002184 */
Gonglei3655cb92016-02-20 10:35:20 +08002185static void *qemu_ram_ptr_length(RAMBlock *ram_block, ram_addr_t addr,
Anthony PERARDf5aa69b2017-07-26 17:53:26 +01002186 hwaddr *size, bool lock)
Stefano Stabellini38bee5d2011-05-19 18:35:45 +01002187{
Gonglei3655cb92016-02-20 10:35:20 +08002188 RAMBlock *block = ram_block;
Stefano Stabellini8ab934f2011-06-27 18:26:06 +01002189 if (*size == 0) {
2190 return NULL;
2191 }
Paolo Bonzinie81bcda2015-12-16 10:31:26 +01002192
Gonglei3655cb92016-02-20 10:35:20 +08002193 if (block == NULL) {
2194 block = qemu_get_ram_block(addr);
Paolo Bonzini0878d0e2016-02-22 11:02:12 +01002195 addr -= block->offset;
Gonglei3655cb92016-02-20 10:35:20 +08002196 }
Paolo Bonzini0878d0e2016-02-22 11:02:12 +01002197 *size = MIN(*size, block->max_length - addr);
Paolo Bonzinie81bcda2015-12-16 10:31:26 +01002198
2199 if (xen_enabled() && block->host == NULL) {
2200 /* We need to check if the requested address is in the RAM
2201 * because we don't want to map the entire memory in QEMU.
2202 * In that case just map the requested area.
2203 */
2204 if (block->offset == 0) {
Anthony PERARDf5aa69b2017-07-26 17:53:26 +01002205 return xen_map_cache(addr, *size, lock, lock);
Stefano Stabellini38bee5d2011-05-19 18:35:45 +01002206 }
2207
Anthony PERARDf5aa69b2017-07-26 17:53:26 +01002208 block->host = xen_map_cache(block->offset, block->max_length, 1, lock);
Stefano Stabellini38bee5d2011-05-19 18:35:45 +01002209 }
Paolo Bonzinie81bcda2015-12-16 10:31:26 +01002210
Paolo Bonzini0878d0e2016-02-22 11:02:12 +01002211 return ramblock_ptr(block, addr);
Stefano Stabellini38bee5d2011-05-19 18:35:45 +01002212}
2213
Dr. David Alan Gilbertf90bb712018-03-12 17:20:57 +00002214/* Return the offset of a hostpointer within a ramblock */
2215ram_addr_t qemu_ram_block_host_offset(RAMBlock *rb, void *host)
2216{
2217 ram_addr_t res = (uint8_t *)host - (uint8_t *)rb->host;
2218 assert((uintptr_t)host >= (uintptr_t)rb->host);
2219 assert(res < rb->max_length);
2220
2221 return res;
2222}
2223
Dr. David Alan Gilbert422148d2015-11-05 18:10:32 +00002224RAMBlock *qemu_ram_block_from_host(void *ptr, bool round_offset,
Dr. David Alan Gilbert422148d2015-11-05 18:10:32 +00002225 ram_addr_t *offset)
pbrook5579c7f2009-04-11 14:47:08 +00002226{
pbrook94a6b542009-04-11 17:15:54 +00002227 RAMBlock *block;
2228 uint8_t *host = ptr;
2229
Jan Kiszka868bb332011-06-21 22:59:09 +02002230 if (xen_enabled()) {
Paolo Bonzinif615f392016-05-26 10:07:50 +02002231 ram_addr_t ram_addr;
Dr. David Alan Gilbert694ea272019-10-07 15:36:41 +01002232 RCU_READ_LOCK_GUARD();
Paolo Bonzinif615f392016-05-26 10:07:50 +02002233 ram_addr = xen_ram_addr_from_mapcache(ptr);
2234 block = qemu_get_ram_block(ram_addr);
Dr. David Alan Gilbert422148d2015-11-05 18:10:32 +00002235 if (block) {
Anthony PERARDd6b6aec2016-06-09 16:56:17 +01002236 *offset = ram_addr - block->offset;
Dr. David Alan Gilbert422148d2015-11-05 18:10:32 +00002237 }
Dr. David Alan Gilbert422148d2015-11-05 18:10:32 +00002238 return block;
Stefano Stabellini712c2b42011-05-19 18:35:46 +01002239 }
2240
Dr. David Alan Gilbert694ea272019-10-07 15:36:41 +01002241 RCU_READ_LOCK_GUARD();
Stefan Hajnoczid73415a2020-09-23 11:56:46 +01002242 block = qatomic_rcu_read(&ram_list.mru_block);
Michael S. Tsirkin9b8424d2014-12-15 22:55:32 +02002243 if (block && block->host && host - block->host < block->max_length) {
Paolo Bonzini23887b72013-05-06 14:28:39 +02002244 goto found;
2245 }
2246
Peter Xu99e15582017-05-12 12:17:39 +08002247 RAMBLOCK_FOREACH(block) {
Jun Nakajima432d2682010-08-31 16:41:25 +01002248 /* This case append when the block is not mapped. */
2249 if (block->host == NULL) {
2250 continue;
2251 }
Michael S. Tsirkin9b8424d2014-12-15 22:55:32 +02002252 if (host - block->host < block->max_length) {
Paolo Bonzini23887b72013-05-06 14:28:39 +02002253 goto found;
Alex Williamsonf471a172010-06-11 11:11:42 -06002254 }
pbrook94a6b542009-04-11 17:15:54 +00002255 }
Jun Nakajima432d2682010-08-31 16:41:25 +01002256
Paolo Bonzini1b5ec232013-05-06 14:36:15 +02002257 return NULL;
Paolo Bonzini23887b72013-05-06 14:28:39 +02002258
2259found:
Dr. David Alan Gilbert422148d2015-11-05 18:10:32 +00002260 *offset = (host - block->host);
2261 if (round_offset) {
2262 *offset &= TARGET_PAGE_MASK;
2263 }
Dr. David Alan Gilbert422148d2015-11-05 18:10:32 +00002264 return block;
2265}
2266
Dr. David Alan Gilberte3dd7492015-11-05 18:10:33 +00002267/*
2268 * Finds the named RAMBlock
2269 *
2270 * name: The name of RAMBlock to find
2271 *
2272 * Returns: RAMBlock (or NULL if not found)
2273 */
2274RAMBlock *qemu_ram_block_by_name(const char *name)
2275{
2276 RAMBlock *block;
2277
Peter Xu99e15582017-05-12 12:17:39 +08002278 RAMBLOCK_FOREACH(block) {
Dr. David Alan Gilberte3dd7492015-11-05 18:10:33 +00002279 if (!strcmp(name, block->idstr)) {
2280 return block;
2281 }
2282 }
2283
2284 return NULL;
2285}
2286
Philippe Mathieu-Daudé8d7f2e72023-10-04 11:06:28 +02002287/*
2288 * Some of the system routines need to translate from a host pointer
2289 * (typically a TLB entry) back to a ram offset.
2290 */
Paolo Bonzini07bdaa42016-03-25 12:55:08 +01002291ram_addr_t qemu_ram_addr_from_host(void *ptr)
Dr. David Alan Gilbert422148d2015-11-05 18:10:32 +00002292{
2293 RAMBlock *block;
Paolo Bonzinif615f392016-05-26 10:07:50 +02002294 ram_addr_t offset;
Dr. David Alan Gilbert422148d2015-11-05 18:10:32 +00002295
Paolo Bonzinif615f392016-05-26 10:07:50 +02002296 block = qemu_ram_block_from_host(ptr, false, &offset);
Dr. David Alan Gilbert422148d2015-11-05 18:10:32 +00002297 if (!block) {
Paolo Bonzini07bdaa42016-03-25 12:55:08 +01002298 return RAM_ADDR_INVALID;
Dr. David Alan Gilbert422148d2015-11-05 18:10:32 +00002299 }
2300
Paolo Bonzini07bdaa42016-03-25 12:55:08 +01002301 return block->offset + offset;
Marcelo Tosattie8902612010-10-11 15:31:19 -03002302}
Alex Williamsonf471a172010-06-11 11:11:42 -06002303
Richard Henderson97e03462022-08-10 12:04:15 -07002304ram_addr_t qemu_ram_addr_from_host_nofail(void *ptr)
2305{
2306 ram_addr_t ram_addr;
2307
2308 ram_addr = qemu_ram_addr_from_host(ptr);
2309 if (ram_addr == RAM_ADDR_INVALID) {
2310 error_report("Bad ram pointer %p", ptr);
2311 abort();
2312 }
2313 return ram_addr;
2314}
2315
Paolo Bonzinib2a44fc2018-03-05 00:19:49 +01002316static MemTxResult flatview_read(FlatView *fv, hwaddr addr,
Philippe Mathieu-Daudéa152be42020-02-19 19:52:44 +01002317 MemTxAttrs attrs, void *buf, hwaddr len);
Alexey Kardashevskiy16620682017-09-21 18:50:58 +10002318static MemTxResult flatview_write(FlatView *fv, hwaddr addr, MemTxAttrs attrs,
Philippe Mathieu-Daudéa152be42020-02-19 19:52:44 +01002319 const void *buf, hwaddr len);
Li Zhijian0c249ff2019-01-17 20:49:01 +08002320static bool flatview_access_valid(FlatView *fv, hwaddr addr, hwaddr len,
Peter Maydelleace72b2018-05-31 14:50:52 +01002321 bool is_write, MemTxAttrs attrs);
Alexey Kardashevskiy16620682017-09-21 18:50:58 +10002322
Peter Maydellf25a49e2015-04-26 16:49:24 +01002323static MemTxResult subpage_read(void *opaque, hwaddr addr, uint64_t *data,
2324 unsigned len, MemTxAttrs attrs)
blueswir1db7b5422007-05-26 17:36:03 +00002325{
Jan Kiszkaacc9d802013-05-26 21:55:37 +02002326 subpage_t *subpage = opaque;
Paolo Bonziniff6cff72014-12-22 13:11:39 +01002327 uint8_t buf[8];
Peter Maydell5c9eb022015-04-26 16:49:24 +01002328 MemTxResult res;
Paolo Bonzini791af8c2013-05-24 16:10:39 +02002329
blueswir1db7b5422007-05-26 17:36:03 +00002330#if defined(DEBUG_SUBPAGE)
Philippe Mathieu-Daudé883f2c52023-01-10 22:29:47 +01002331 printf("%s: subpage %p len %u addr " HWADDR_FMT_plx "\n", __func__,
Jan Kiszkaacc9d802013-05-26 21:55:37 +02002332 subpage, len, addr);
blueswir1db7b5422007-05-26 17:36:03 +00002333#endif
Alexey Kardashevskiy16620682017-09-21 18:50:58 +10002334 res = flatview_read(subpage->fv, addr + subpage->base, attrs, buf, len);
Peter Maydell5c9eb022015-04-26 16:49:24 +01002335 if (res) {
2336 return res;
Peter Maydellf25a49e2015-04-26 16:49:24 +01002337 }
Peter Maydell6d3ede52018-06-15 14:57:14 +01002338 *data = ldn_p(buf, len);
2339 return MEMTX_OK;
blueswir1db7b5422007-05-26 17:36:03 +00002340}
2341
Peter Maydellf25a49e2015-04-26 16:49:24 +01002342static MemTxResult subpage_write(void *opaque, hwaddr addr,
2343 uint64_t value, unsigned len, MemTxAttrs attrs)
blueswir1db7b5422007-05-26 17:36:03 +00002344{
Jan Kiszkaacc9d802013-05-26 21:55:37 +02002345 subpage_t *subpage = opaque;
Paolo Bonziniff6cff72014-12-22 13:11:39 +01002346 uint8_t buf[8];
Jan Kiszkaacc9d802013-05-26 21:55:37 +02002347
blueswir1db7b5422007-05-26 17:36:03 +00002348#if defined(DEBUG_SUBPAGE)
Philippe Mathieu-Daudé883f2c52023-01-10 22:29:47 +01002349 printf("%s: subpage %p len %u addr " HWADDR_FMT_plx
Jan Kiszkaacc9d802013-05-26 21:55:37 +02002350 " value %"PRIx64"\n",
2351 __func__, subpage, len, addr, value);
blueswir1db7b5422007-05-26 17:36:03 +00002352#endif
Peter Maydell6d3ede52018-06-15 14:57:14 +01002353 stn_p(buf, len, value);
Alexey Kardashevskiy16620682017-09-21 18:50:58 +10002354 return flatview_write(subpage->fv, addr + subpage->base, attrs, buf, len);
blueswir1db7b5422007-05-26 17:36:03 +00002355}
2356
Paolo Bonzinic353e4c2013-05-24 14:02:39 +02002357static bool subpage_accepts(void *opaque, hwaddr addr,
Peter Maydell8372d382018-05-31 14:50:52 +01002358 unsigned len, bool is_write,
2359 MemTxAttrs attrs)
Paolo Bonzinic353e4c2013-05-24 14:02:39 +02002360{
Jan Kiszkaacc9d802013-05-26 21:55:37 +02002361 subpage_t *subpage = opaque;
Paolo Bonzinic353e4c2013-05-24 14:02:39 +02002362#if defined(DEBUG_SUBPAGE)
Philippe Mathieu-Daudé883f2c52023-01-10 22:29:47 +01002363 printf("%s: subpage %p %c len %u addr " HWADDR_FMT_plx "\n",
Jan Kiszkaacc9d802013-05-26 21:55:37 +02002364 __func__, subpage, is_write ? 'w' : 'r', len, addr);
Paolo Bonzinic353e4c2013-05-24 14:02:39 +02002365#endif
2366
Alexey Kardashevskiy16620682017-09-21 18:50:58 +10002367 return flatview_access_valid(subpage->fv, addr + subpage->base,
Peter Maydelleace72b2018-05-31 14:50:52 +01002368 len, is_write, attrs);
Paolo Bonzinic353e4c2013-05-24 14:02:39 +02002369}
2370
Avi Kivity70c68e42012-01-02 12:32:48 +02002371static const MemoryRegionOps subpage_ops = {
Peter Maydellf25a49e2015-04-26 16:49:24 +01002372 .read_with_attrs = subpage_read,
2373 .write_with_attrs = subpage_write,
Paolo Bonziniff6cff72014-12-22 13:11:39 +01002374 .impl.min_access_size = 1,
2375 .impl.max_access_size = 8,
2376 .valid.min_access_size = 1,
2377 .valid.max_access_size = 8,
Paolo Bonzinic353e4c2013-05-24 14:02:39 +02002378 .valid.accepts = subpage_accepts,
Avi Kivity70c68e42012-01-02 12:32:48 +02002379 .endianness = DEVICE_NATIVE_ENDIAN,
blueswir1db7b5422007-05-26 17:36:03 +00002380};
2381
Wei Yangb797ab12019-03-21 16:25:53 +08002382static int subpage_register(subpage_t *mmio, uint32_t start, uint32_t end,
2383 uint16_t section)
blueswir1db7b5422007-05-26 17:36:03 +00002384{
2385 int idx, eidx;
2386
2387 if (start >= TARGET_PAGE_SIZE || end >= TARGET_PAGE_SIZE)
2388 return -1;
2389 idx = SUBPAGE_IDX(start);
2390 eidx = SUBPAGE_IDX(end);
2391#if defined(DEBUG_SUBPAGE)
Amos Kong016e9d62013-09-27 09:25:38 +08002392 printf("%s: %p start %08x end %08x idx %08x eidx %08x section %d\n",
2393 __func__, mmio, start, end, idx, eidx, section);
blueswir1db7b5422007-05-26 17:36:03 +00002394#endif
blueswir1db7b5422007-05-26 17:36:03 +00002395 for (; idx <= eidx; idx++) {
Avi Kivity5312bd82012-02-12 18:32:55 +02002396 mmio->sub_section[idx] = section;
blueswir1db7b5422007-05-26 17:36:03 +00002397 }
2398
2399 return 0;
2400}
2401
Alexey Kardashevskiy16620682017-09-21 18:50:58 +10002402static subpage_t *subpage_init(FlatView *fv, hwaddr base)
blueswir1db7b5422007-05-26 17:36:03 +00002403{
Anthony Liguoric227f092009-10-01 16:12:16 -05002404 subpage_t *mmio;
blueswir1db7b5422007-05-26 17:36:03 +00002405
Wei Yangb797ab12019-03-21 16:25:53 +08002406 /* mmio->sub_section is set to PHYS_SECTION_UNASSIGNED with g_malloc0 */
Vijaya Kumar K2615fab2016-10-24 16:26:49 +01002407 mmio = g_malloc0(sizeof(subpage_t) + TARGET_PAGE_SIZE * sizeof(uint16_t));
Alexey Kardashevskiy16620682017-09-21 18:50:58 +10002408 mmio->fv = fv;
aliguori1eec6142009-02-05 22:06:18 +00002409 mmio->base = base;
Paolo Bonzini2c9b15c2013-06-06 05:41:28 -04002410 memory_region_init_io(&mmio->iomem, NULL, &subpage_ops, mmio,
Peter Crosthwaiteb4fefef2014-06-05 23:15:52 -07002411 NULL, TARGET_PAGE_SIZE);
Avi Kivityb3b00c72012-01-02 13:20:11 +02002412 mmio->iomem.subpage = true;
blueswir1db7b5422007-05-26 17:36:03 +00002413#if defined(DEBUG_SUBPAGE)
Philippe Mathieu-Daudé883f2c52023-01-10 22:29:47 +01002414 printf("%s: %p base " HWADDR_FMT_plx " len %08x\n", __func__,
Amos Kong016e9d62013-09-27 09:25:38 +08002415 mmio, base, TARGET_PAGE_SIZE);
blueswir1db7b5422007-05-26 17:36:03 +00002416#endif
blueswir1db7b5422007-05-26 17:36:03 +00002417
2418 return mmio;
2419}
2420
Alexey Kardashevskiy16620682017-09-21 18:50:58 +10002421static uint16_t dummy_section(PhysPageMap *map, FlatView *fv, MemoryRegion *mr)
Avi Kivity5312bd82012-02-12 18:32:55 +02002422{
Alexey Kardashevskiy16620682017-09-21 18:50:58 +10002423 assert(fv);
Avi Kivity5312bd82012-02-12 18:32:55 +02002424 MemoryRegionSection section = {
Alexey Kardashevskiy16620682017-09-21 18:50:58 +10002425 .fv = fv,
Avi Kivity5312bd82012-02-12 18:32:55 +02002426 .mr = mr,
2427 .offset_within_address_space = 0,
2428 .offset_within_region = 0,
Paolo Bonzini052e87b2013-05-27 10:08:27 +02002429 .size = int128_2_64(),
Avi Kivity5312bd82012-02-12 18:32:55 +02002430 };
2431
Marcel Apfelbaum53cb28c2013-12-01 14:02:23 +02002432 return phys_section_add(map, &section);
Avi Kivity5312bd82012-02-12 18:32:55 +02002433}
2434
Peter Maydell2d54f192018-06-15 14:57:14 +01002435MemoryRegionSection *iotlb_to_section(CPUState *cpu,
2436 hwaddr index, MemTxAttrs attrs)
Avi Kivityaa102232012-03-08 17:06:55 +02002437{
Peter Maydella54c87b2016-01-21 14:15:05 +00002438 int asidx = cpu_asidx_from_attrs(cpu, attrs);
2439 CPUAddressSpace *cpuas = &cpu->cpu_ases[asidx];
Richard Henderson0d58c662023-08-25 16:13:17 -07002440 AddressSpaceDispatch *d = cpuas->memory_dispatch;
Richard Henderson86e4f932023-08-25 14:06:58 -07002441 int section_index = index & ~TARGET_PAGE_MASK;
2442 MemoryRegionSection *ret;
Paolo Bonzini9d82b5a2013-08-16 08:26:30 +02002443
Richard Henderson86e4f932023-08-25 14:06:58 -07002444 assert(section_index < d->map.sections_nb);
2445 ret = d->map.sections + section_index;
2446 assert(ret->mr);
2447 assert(ret->mr->ops);
2448
2449 return ret;
Avi Kivityaa102232012-03-08 17:06:55 +02002450}
2451
Avi Kivitye9179ce2009-06-14 11:38:52 +03002452static void io_mem_init(void)
2453{
Paolo Bonzini2c9b15c2013-06-06 05:41:28 -04002454 memory_region_init_io(&io_mem_unassigned, NULL, &unassigned_mem_ops, NULL,
Paolo Bonzini1f6245e2014-06-13 10:48:06 +02002455 NULL, UINT64_MAX);
Avi Kivitye9179ce2009-06-14 11:38:52 +03002456}
2457
Alexey Kardashevskiy8629d3f2017-09-21 18:51:00 +10002458AddressSpaceDispatch *address_space_dispatch_new(FlatView *fv)
Avi Kivityac1970f2012-10-03 16:22:53 +02002459{
Marcel Apfelbaum53cb28c2013-12-01 14:02:23 +02002460 AddressSpaceDispatch *d = g_new0(AddressSpaceDispatch, 1);
2461 uint16_t n;
2462
Alexey Kardashevskiy16620682017-09-21 18:50:58 +10002463 n = dummy_section(&d->map, fv, &io_mem_unassigned);
Marcel Apfelbaum53cb28c2013-12-01 14:02:23 +02002464 assert(n == PHYS_SECTION_UNASSIGNED);
Paolo Bonzini00752702013-05-29 12:13:54 +02002465
Michael S. Tsirkin9736e552013-11-11 14:42:43 +02002466 d->phys_map = (PhysPageEntry) { .ptr = PHYS_MAP_NODE_NIL, .skip = 1 };
Alexey Kardashevskiy66a6df12017-09-21 18:50:56 +10002467
2468 return d;
Paolo Bonzini00752702013-05-29 12:13:54 +02002469}
2470
Alexey Kardashevskiy66a6df12017-09-21 18:50:56 +10002471void address_space_dispatch_free(AddressSpaceDispatch *d)
Paolo Bonzini79e2b9a2015-01-21 12:09:14 +01002472{
2473 phys_sections_free(&d->map);
2474 g_free(d);
2475}
2476
Paolo Bonzini9458a9a2018-02-06 18:37:39 +01002477static void do_nothing(CPUState *cpu, run_on_cpu_data d)
2478{
2479}
2480
2481static void tcg_log_global_after_sync(MemoryListener *listener)
2482{
2483 CPUAddressSpace *cpuas;
2484
2485 /* Wait for the CPU to end the current TB. This avoids the following
2486 * incorrect race:
2487 *
2488 * vCPU migration
2489 * ---------------------- -------------------------
2490 * TLB check -> slow path
2491 * notdirty_mem_write
2492 * write to RAM
2493 * mark dirty
2494 * clear dirty flag
2495 * TLB check -> fast path
2496 * read memory
2497 * write to RAM
2498 *
2499 * by pushing the migration thread's memory read after the vCPU thread has
2500 * written the memory.
2501 */
Pavel Dovgalyuk86cf9e12019-09-17 12:54:06 +03002502 if (replay_mode == REPLAY_MODE_NONE) {
2503 /*
2504 * VGA can make calls to this function while updating the screen.
2505 * In record/replay mode this causes a deadlock, because
2506 * run_on_cpu waits for rr mutex. Therefore no races are possible
2507 * in this case and no need for making run_on_cpu when
Greg Kurzf18d4032021-10-15 11:29:44 +02002508 * record/replay is enabled.
Pavel Dovgalyuk86cf9e12019-09-17 12:54:06 +03002509 */
2510 cpuas = container_of(listener, CPUAddressSpace, tcg_as_listener);
2511 run_on_cpu(cpuas->cpu, do_nothing, RUN_ON_CPU_NULL);
2512 }
Paolo Bonzini9458a9a2018-02-06 18:37:39 +01002513}
2514
Richard Henderson0d58c662023-08-25 16:13:17 -07002515static void tcg_commit_cpu(CPUState *cpu, run_on_cpu_data data)
2516{
2517 CPUAddressSpace *cpuas = data.host_ptr;
2518
2519 cpuas->memory_dispatch = address_space_to_dispatch(cpuas->as);
2520 tlb_flush(cpu);
2521}
2522
Avi Kivity1d711482012-10-02 18:54:45 +02002523static void tcg_commit(MemoryListener *listener)
Avi Kivity50c1e142012-02-08 21:36:02 +02002524{
Peter Maydell32857f42015-10-01 15:29:50 +01002525 CPUAddressSpace *cpuas;
Richard Henderson0d58c662023-08-25 16:13:17 -07002526 CPUState *cpu;
Avi Kivity117712c2012-02-12 21:23:17 +02002527
Emilio G. Cotaf28d0df2018-06-22 13:45:31 -04002528 assert(tcg_enabled());
Avi Kivity117712c2012-02-12 21:23:17 +02002529 /* since each CPU stores ram addresses in its TLB cache, we must
2530 reset the modified entries */
Peter Maydell32857f42015-10-01 15:29:50 +01002531 cpuas = container_of(listener, CPUAddressSpace, tcg_as_listener);
Richard Henderson0d58c662023-08-25 16:13:17 -07002532 cpu = cpuas->cpu;
2533
2534 /*
2535 * Defer changes to as->memory_dispatch until the cpu is quiescent.
2536 * Otherwise we race between (1) other cpu threads and (2) ongoing
2537 * i/o for the current cpu thread, with data cached by mmu_lookup().
2538 *
2539 * In addition, queueing the work function will kick the cpu back to
2540 * the main loop, which will end the RCU critical section and reclaim
2541 * the memory data structures.
2542 *
2543 * That said, the listener is also called during realize, before
2544 * all of the tcg machinery for run-on is initialized: thus halt_cond.
Peter Maydell32857f42015-10-01 15:29:50 +01002545 */
Richard Henderson0d58c662023-08-25 16:13:17 -07002546 if (cpu->halt_cond) {
2547 async_run_on_cpu(cpu, tcg_commit_cpu, RUN_ON_CPU_HOST_PTR(cpuas));
2548 } else {
2549 tcg_commit_cpu(cpu, RUN_ON_CPU_HOST_PTR(cpuas));
2550 }
Avi Kivity50c1e142012-02-08 21:36:02 +02002551}
2552
Avi Kivity62152b82011-07-26 14:26:14 +03002553static void memory_map_init(void)
2554{
Anthony Liguori7267c092011-08-20 22:09:37 -05002555 system_memory = g_malloc(sizeof(*system_memory));
Paolo Bonzini03f49952013-11-07 17:14:36 +01002556
Paolo Bonzini57271d62013-11-07 17:14:37 +01002557 memory_region_init(system_memory, NULL, "system", UINT64_MAX);
Alexey Kardashevskiy7dca8042013-04-29 16:25:51 +00002558 address_space_init(&address_space_memory, system_memory, "memory");
Avi Kivity309cb472011-08-08 16:09:03 +03002559
Anthony Liguori7267c092011-08-20 22:09:37 -05002560 system_io = g_malloc(sizeof(*system_io));
Jan Kiszka3bb28b72013-09-02 18:43:30 +02002561 memory_region_init_io(system_io, NULL, &unassigned_io_ops, NULL, "io",
2562 65536);
Alexey Kardashevskiy7dca8042013-04-29 16:25:51 +00002563 address_space_init(&address_space_io, system_io, "I/O");
Avi Kivity62152b82011-07-26 14:26:14 +03002564}
2565
2566MemoryRegion *get_system_memory(void)
2567{
2568 return system_memory;
2569}
2570
Avi Kivity309cb472011-08-08 16:09:03 +03002571MemoryRegion *get_system_io(void)
2572{
2573 return system_io;
2574}
2575
Paolo Bonzini845b6212015-03-23 11:45:53 +01002576static void invalidate_and_set_dirty(MemoryRegion *mr, hwaddr addr,
Avi Kivitya8170e52012-10-23 12:30:10 +02002577 hwaddr length)
Anthony PERARD51d7a9e2012-10-03 13:49:05 +00002578{
Paolo Bonzinie87f7772015-03-25 15:21:39 +01002579 uint8_t dirty_log_mask = memory_region_get_dirty_log_mask(mr);
Paolo Bonzini0878d0e2016-02-22 11:02:12 +01002580 addr += memory_region_get_ram_addr(mr);
2581
Paolo Bonzinie87f7772015-03-25 15:21:39 +01002582 /* No early return if dirty_log_mask is or becomes 0, because
2583 * cpu_physical_memory_set_dirty_range will still call
2584 * xen_modified_memory.
2585 */
2586 if (dirty_log_mask) {
2587 dirty_log_mask =
2588 cpu_physical_memory_range_includes_clean(addr, length, dirty_log_mask);
Anthony PERARD51d7a9e2012-10-03 13:49:05 +00002589 }
Paolo Bonzinie87f7772015-03-25 15:21:39 +01002590 if (dirty_log_mask & (1 << DIRTY_MEMORY_CODE)) {
Paolo Bonzini5aa1ef72017-07-03 17:50:40 +02002591 assert(tcg_enabled());
Richard Hendersone506ad62023-03-06 04:30:11 +03002592 tb_invalidate_phys_range(addr, addr + length - 1);
Paolo Bonzinie87f7772015-03-25 15:21:39 +01002593 dirty_log_mask &= ~(1 << DIRTY_MEMORY_CODE);
2594 }
2595 cpu_physical_memory_set_dirty_range(addr, length, dirty_log_mask);
Anthony PERARD51d7a9e2012-10-03 13:49:05 +00002596}
2597
Stefan Hajnoczi047be4e2019-01-29 11:46:04 +00002598void memory_region_flush_rom_device(MemoryRegion *mr, hwaddr addr, hwaddr size)
2599{
2600 /*
2601 * In principle this function would work on other memory region types too,
2602 * but the ROM device use case is the only one where this operation is
2603 * necessary. Other memory regions should use the
2604 * address_space_read/write() APIs.
2605 */
2606 assert(memory_region_is_romd(mr));
2607
2608 invalidate_and_set_dirty(mr, addr, size);
2609}
2610
Jagannathan Raman3123f932022-06-13 16:26:32 -04002611int memory_access_size(MemoryRegion *mr, unsigned l, hwaddr addr)
Paolo Bonzini82f25632013-05-24 11:59:43 +02002612{
Paolo Bonzinie1622f42013-07-17 13:17:41 +02002613 unsigned access_size_max = mr->ops->valid.max_access_size;
Richard Henderson23326162013-07-08 14:55:59 -07002614
2615 /* Regions are assumed to support 1-4 byte accesses unless
2616 otherwise specified. */
Richard Henderson23326162013-07-08 14:55:59 -07002617 if (access_size_max == 0) {
2618 access_size_max = 4;
Paolo Bonzini82f25632013-05-24 11:59:43 +02002619 }
Richard Henderson23326162013-07-08 14:55:59 -07002620
2621 /* Bound the maximum access by the alignment of the address. */
2622 if (!mr->ops->impl.unaligned) {
2623 unsigned align_size_max = addr & -addr;
2624 if (align_size_max != 0 && align_size_max < access_size_max) {
2625 access_size_max = align_size_max;
2626 }
2627 }
2628
2629 /* Don't attempt accesses larger than the maximum. */
2630 if (l > access_size_max) {
2631 l = access_size_max;
2632 }
Peter Maydell6554f5c2015-07-24 13:33:10 +01002633 l = pow2floor(l);
Richard Henderson23326162013-07-08 14:55:59 -07002634
2635 return l;
Paolo Bonzini82f25632013-05-24 11:59:43 +02002636}
2637
Jagannathan Raman3123f932022-06-13 16:26:32 -04002638bool prepare_mmio_access(MemoryRegion *mr)
Paolo Bonzini125b3802015-06-18 18:47:21 +02002639{
Jan Kiszka4840f102015-06-18 18:47:22 +02002640 bool release_lock = false;
2641
Philippe Mathieu-Daudé37921852020-10-30 16:37:52 +01002642 if (!qemu_mutex_iothread_locked()) {
Jan Kiszka4840f102015-06-18 18:47:22 +02002643 qemu_mutex_lock_iothread();
Jan Kiszka4840f102015-06-18 18:47:22 +02002644 release_lock = true;
Paolo Bonzini125b3802015-06-18 18:47:21 +02002645 }
Jan Kiszka4840f102015-06-18 18:47:22 +02002646 if (mr->flush_coalesced_mmio) {
Jan Kiszka4840f102015-06-18 18:47:22 +02002647 qemu_flush_coalesced_mmio_buffer();
Jan Kiszka4840f102015-06-18 18:47:22 +02002648 }
2649
2650 return release_lock;
Paolo Bonzini125b3802015-06-18 18:47:21 +02002651}
2652
Philippe Mathieu-Daudé3ab6fdc2021-12-15 19:24:21 +01002653/**
2654 * flatview_access_allowed
2655 * @mr: #MemoryRegion to be accessed
2656 * @attrs: memory transaction attributes
2657 * @addr: address within that memory region
2658 * @len: the number of bytes to access
2659 *
2660 * Check if a memory transaction is allowed.
2661 *
2662 * Returns: true if transaction is allowed, false if denied.
2663 */
2664static bool flatview_access_allowed(MemoryRegion *mr, MemTxAttrs attrs,
2665 hwaddr addr, hwaddr len)
2666{
2667 if (likely(!attrs.memory)) {
2668 return true;
2669 }
2670 if (memory_region_is_ram(mr)) {
2671 return true;
2672 }
2673 qemu_log_mask(LOG_GUEST_ERROR,
2674 "Invalid access to non-RAM device at "
2675 "addr 0x%" HWADDR_PRIX ", size %" HWADDR_PRIu ", "
2676 "region '%s'\n", addr, len, memory_region_name(mr));
2677 return false;
2678}
2679
Paolo Bonzinia203ac72015-12-09 10:18:57 +01002680/* Called within RCU critical section. */
Alexey Kardashevskiy16620682017-09-21 18:50:58 +10002681static MemTxResult flatview_write_continue(FlatView *fv, hwaddr addr,
2682 MemTxAttrs attrs,
Philippe Mathieu-Daudéa152be42020-02-19 19:52:44 +01002683 const void *ptr,
Li Zhijian0c249ff2019-01-17 20:49:01 +08002684 hwaddr len, hwaddr addr1,
Alexey Kardashevskiy16620682017-09-21 18:50:58 +10002685 hwaddr l, MemoryRegion *mr)
bellard13eb76e2004-01-24 15:23:36 +00002686{
Philippe Mathieu-Daudé20804672020-02-19 18:01:32 +01002687 uint8_t *ram_ptr;
Paolo Bonzini791af8c2013-05-24 16:10:39 +02002688 uint64_t val;
Peter Maydell3b643492015-04-26 16:49:23 +01002689 MemTxResult result = MEMTX_OK;
Jan Kiszka4840f102015-06-18 18:47:22 +02002690 bool release_lock = false;
Philippe Mathieu-Daudéa152be42020-02-19 19:52:44 +01002691 const uint8_t *buf = ptr;
ths3b46e622007-09-17 08:09:54 +00002692
Paolo Bonzinia203ac72015-12-09 10:18:57 +01002693 for (;;) {
Philippe Mathieu-Daudé3ab6fdc2021-12-15 19:24:21 +01002694 if (!flatview_access_allowed(mr, attrs, addr1, l)) {
2695 result |= MEMTX_ACCESS_ERROR;
2696 /* Keep going. */
2697 } else if (!memory_access_is_direct(mr, true)) {
Paolo Bonzinieb7eeb82015-12-09 10:06:31 +01002698 release_lock |= prepare_mmio_access(mr);
2699 l = memory_access_size(mr, l, addr1);
2700 /* XXX: could force current_cpu to NULL to avoid
2701 potential bugs */
Tony Nguyen9bf825b2019-08-24 04:36:54 +10002702 val = ldn_he_p(buf, l);
Tony Nguyen3d9e7c32019-08-24 04:36:46 +10002703 result |= memory_region_dispatch_write(mr, addr1, val,
Tony Nguyen9bf825b2019-08-24 04:36:54 +10002704 size_memop(l), attrs);
bellard13eb76e2004-01-24 15:23:36 +00002705 } else {
Paolo Bonzinieb7eeb82015-12-09 10:06:31 +01002706 /* RAM case */
Philippe Mathieu-Daudé20804672020-02-19 18:01:32 +01002707 ram_ptr = qemu_ram_ptr_length(mr->ram_block, addr1, &l, false);
Akihiko Odaki4a73aee2023-01-31 12:01:55 +09002708 memmove(ram_ptr, buf, l);
Paolo Bonzinieb7eeb82015-12-09 10:06:31 +01002709 invalidate_and_set_dirty(mr, addr1, l);
bellard13eb76e2004-01-24 15:23:36 +00002710 }
Jan Kiszka4840f102015-06-18 18:47:22 +02002711
2712 if (release_lock) {
2713 qemu_mutex_unlock_iothread();
2714 release_lock = false;
2715 }
2716
bellard13eb76e2004-01-24 15:23:36 +00002717 len -= l;
2718 buf += l;
2719 addr += l;
Paolo Bonzinia203ac72015-12-09 10:18:57 +01002720
2721 if (!len) {
2722 break;
2723 }
2724
2725 l = len;
Peter Maydellefa99a22018-05-31 14:50:52 +01002726 mr = flatview_translate(fv, addr, &addr1, &l, true, attrs);
bellard13eb76e2004-01-24 15:23:36 +00002727 }
Paolo Bonzinifd8aaa72013-05-21 09:56:55 +02002728
Peter Maydell3b643492015-04-26 16:49:23 +01002729 return result;
bellard13eb76e2004-01-24 15:23:36 +00002730}
bellard8df1cd02005-01-28 22:37:22 +00002731
Paolo Bonzini4c6ebbb2018-03-05 09:23:56 +01002732/* Called from RCU critical section. */
Alexey Kardashevskiy16620682017-09-21 18:50:58 +10002733static MemTxResult flatview_write(FlatView *fv, hwaddr addr, MemTxAttrs attrs,
Philippe Mathieu-Daudéa152be42020-02-19 19:52:44 +01002734 const void *buf, hwaddr len)
Avi Kivityac1970f2012-10-03 16:22:53 +02002735{
Paolo Bonzinieb7eeb82015-12-09 10:06:31 +01002736 hwaddr l;
Paolo Bonzinieb7eeb82015-12-09 10:06:31 +01002737 hwaddr addr1;
2738 MemoryRegion *mr;
Paolo Bonzinia203ac72015-12-09 10:18:57 +01002739
Paolo Bonzini4c6ebbb2018-03-05 09:23:56 +01002740 l = len;
Peter Maydellefa99a22018-05-31 14:50:52 +01002741 mr = flatview_translate(fv, addr, &addr1, &l, true, attrs);
Philippe Mathieu-Daudé3ab6fdc2021-12-15 19:24:21 +01002742 if (!flatview_access_allowed(mr, attrs, addr, len)) {
2743 return MEMTX_ACCESS_ERROR;
2744 }
Philippe Mathieu-Daudé58e74682021-12-15 19:24:20 +01002745 return flatview_write_continue(fv, addr, attrs, buf, len,
2746 addr1, l, mr);
Paolo Bonzinia203ac72015-12-09 10:18:57 +01002747}
2748
2749/* Called within RCU critical section. */
Alexey Kardashevskiy16620682017-09-21 18:50:58 +10002750MemTxResult flatview_read_continue(FlatView *fv, hwaddr addr,
Philippe Mathieu-Daudéa152be42020-02-19 19:52:44 +01002751 MemTxAttrs attrs, void *ptr,
Li Zhijian0c249ff2019-01-17 20:49:01 +08002752 hwaddr len, hwaddr addr1, hwaddr l,
Alexey Kardashevskiy16620682017-09-21 18:50:58 +10002753 MemoryRegion *mr)
Paolo Bonzinia203ac72015-12-09 10:18:57 +01002754{
Philippe Mathieu-Daudé20804672020-02-19 18:01:32 +01002755 uint8_t *ram_ptr;
Paolo Bonzinia203ac72015-12-09 10:18:57 +01002756 uint64_t val;
2757 MemTxResult result = MEMTX_OK;
Paolo Bonzinieb7eeb82015-12-09 10:06:31 +01002758 bool release_lock = false;
Philippe Mathieu-Daudéa152be42020-02-19 19:52:44 +01002759 uint8_t *buf = ptr;
Paolo Bonzinieb7eeb82015-12-09 10:06:31 +01002760
Alexander Bulekov7cac7fe2021-03-15 10:05:12 -04002761 fuzz_dma_read_cb(addr, len, mr);
Paolo Bonzinia203ac72015-12-09 10:18:57 +01002762 for (;;) {
Philippe Mathieu-Daudé3ab6fdc2021-12-15 19:24:21 +01002763 if (!flatview_access_allowed(mr, attrs, addr1, l)) {
2764 result |= MEMTX_ACCESS_ERROR;
2765 /* Keep going. */
2766 } else if (!memory_access_is_direct(mr, false)) {
Paolo Bonzinieb7eeb82015-12-09 10:06:31 +01002767 /* I/O case */
2768 release_lock |= prepare_mmio_access(mr);
2769 l = memory_access_size(mr, l, addr1);
Tony Nguyen3d9e7c32019-08-24 04:36:46 +10002770 result |= memory_region_dispatch_read(mr, addr1, &val,
Tony Nguyen9bf825b2019-08-24 04:36:54 +10002771 size_memop(l), attrs);
2772 stn_he_p(buf, l, val);
Paolo Bonzinieb7eeb82015-12-09 10:06:31 +01002773 } else {
2774 /* RAM case */
Philippe Mathieu-Daudé20804672020-02-19 18:01:32 +01002775 ram_ptr = qemu_ram_ptr_length(mr->ram_block, addr1, &l, false);
2776 memcpy(buf, ram_ptr, l);
Paolo Bonzinieb7eeb82015-12-09 10:06:31 +01002777 }
2778
2779 if (release_lock) {
2780 qemu_mutex_unlock_iothread();
2781 release_lock = false;
2782 }
2783
2784 len -= l;
2785 buf += l;
2786 addr += l;
Paolo Bonzinia203ac72015-12-09 10:18:57 +01002787
2788 if (!len) {
2789 break;
2790 }
2791
2792 l = len;
Peter Maydellefa99a22018-05-31 14:50:52 +01002793 mr = flatview_translate(fv, addr, &addr1, &l, false, attrs);
Paolo Bonzinieb7eeb82015-12-09 10:06:31 +01002794 }
Paolo Bonzinia203ac72015-12-09 10:18:57 +01002795
2796 return result;
2797}
2798
Paolo Bonzinib2a44fc2018-03-05 00:19:49 +01002799/* Called from RCU critical section. */
2800static MemTxResult flatview_read(FlatView *fv, hwaddr addr,
Philippe Mathieu-Daudéa152be42020-02-19 19:52:44 +01002801 MemTxAttrs attrs, void *buf, hwaddr len)
Paolo Bonzinia203ac72015-12-09 10:18:57 +01002802{
2803 hwaddr l;
2804 hwaddr addr1;
2805 MemoryRegion *mr;
Paolo Bonzinia203ac72015-12-09 10:18:57 +01002806
Paolo Bonzinib2a44fc2018-03-05 00:19:49 +01002807 l = len;
Peter Maydellefa99a22018-05-31 14:50:52 +01002808 mr = flatview_translate(fv, addr, &addr1, &l, false, attrs);
Philippe Mathieu-Daudé3ab6fdc2021-12-15 19:24:21 +01002809 if (!flatview_access_allowed(mr, attrs, addr, len)) {
2810 return MEMTX_ACCESS_ERROR;
2811 }
Paolo Bonzinib2a44fc2018-03-05 00:19:49 +01002812 return flatview_read_continue(fv, addr, attrs, buf, len,
2813 addr1, l, mr);
Avi Kivityac1970f2012-10-03 16:22:53 +02002814}
2815
Paolo Bonzinib2a44fc2018-03-05 00:19:49 +01002816MemTxResult address_space_read_full(AddressSpace *as, hwaddr addr,
Philippe Mathieu-Daudédaa3dda2020-02-19 19:54:35 +01002817 MemTxAttrs attrs, void *buf, hwaddr len)
Paolo Bonzinib2a44fc2018-03-05 00:19:49 +01002818{
2819 MemTxResult result = MEMTX_OK;
2820 FlatView *fv;
2821
2822 if (len > 0) {
Dr. David Alan Gilbert694ea272019-10-07 15:36:41 +01002823 RCU_READ_LOCK_GUARD();
Paolo Bonzinib2a44fc2018-03-05 00:19:49 +01002824 fv = address_space_to_flatview(as);
2825 result = flatview_read(fv, addr, attrs, buf, len);
Paolo Bonzinib2a44fc2018-03-05 00:19:49 +01002826 }
2827
2828 return result;
2829}
2830
Paolo Bonzini4c6ebbb2018-03-05 09:23:56 +01002831MemTxResult address_space_write(AddressSpace *as, hwaddr addr,
2832 MemTxAttrs attrs,
Philippe Mathieu-Daudédaa3dda2020-02-19 19:54:35 +01002833 const void *buf, hwaddr len)
Paolo Bonzini4c6ebbb2018-03-05 09:23:56 +01002834{
2835 MemTxResult result = MEMTX_OK;
2836 FlatView *fv;
2837
2838 if (len > 0) {
Dr. David Alan Gilbert694ea272019-10-07 15:36:41 +01002839 RCU_READ_LOCK_GUARD();
Paolo Bonzini4c6ebbb2018-03-05 09:23:56 +01002840 fv = address_space_to_flatview(as);
2841 result = flatview_write(fv, addr, attrs, buf, len);
Paolo Bonzini4c6ebbb2018-03-05 09:23:56 +01002842 }
2843
2844 return result;
2845}
2846
Paolo Bonzinidb84fd92018-03-05 09:29:04 +01002847MemTxResult address_space_rw(AddressSpace *as, hwaddr addr, MemTxAttrs attrs,
Philippe Mathieu-Daudédaa3dda2020-02-19 19:54:35 +01002848 void *buf, hwaddr len, bool is_write)
Paolo Bonzinidb84fd92018-03-05 09:29:04 +01002849{
2850 if (is_write) {
2851 return address_space_write(as, addr, attrs, buf, len);
2852 } else {
2853 return address_space_read_full(as, addr, attrs, buf, len);
2854 }
2855}
2856
Philippe Mathieu-Daudé75f01c62022-01-15 21:37:23 +01002857MemTxResult address_space_set(AddressSpace *as, hwaddr addr,
2858 uint8_t c, hwaddr len, MemTxAttrs attrs)
2859{
2860#define FILLBUF_SIZE 512
2861 uint8_t fillbuf[FILLBUF_SIZE];
2862 int l;
2863 MemTxResult error = MEMTX_OK;
2864
2865 memset(fillbuf, c, FILLBUF_SIZE);
2866 while (len > 0) {
2867 l = len < FILLBUF_SIZE ? len : FILLBUF_SIZE;
2868 error |= address_space_write(as, addr, attrs, fillbuf, l);
2869 len -= l;
2870 addr += l;
2871 }
2872
2873 return error;
2874}
2875
Philippe Mathieu-Daudéd7ef71e2020-02-19 20:02:11 +01002876void cpu_physical_memory_rw(hwaddr addr, void *buf,
Philippe Mathieu-Daudé28c80bf2020-02-19 20:32:30 +01002877 hwaddr len, bool is_write)
Avi Kivityac1970f2012-10-03 16:22:53 +02002878{
Peter Maydell5c9eb022015-04-26 16:49:24 +01002879 address_space_rw(&address_space_memory, addr, MEMTXATTRS_UNSPECIFIED,
2880 buf, len, is_write);
Avi Kivityac1970f2012-10-03 16:22:53 +02002881}
2882
Alexander Graf582b55a2013-12-11 14:17:44 +01002883enum write_rom_type {
2884 WRITE_DATA,
2885 FLUSH_CACHE,
2886};
2887
Peter Maydell75693e12018-12-14 13:30:48 +00002888static inline MemTxResult address_space_write_rom_internal(AddressSpace *as,
2889 hwaddr addr,
2890 MemTxAttrs attrs,
Philippe Mathieu-Daudédaa3dda2020-02-19 19:54:35 +01002891 const void *ptr,
Li Zhijian0c249ff2019-01-17 20:49:01 +08002892 hwaddr len,
Peter Maydell75693e12018-12-14 13:30:48 +00002893 enum write_rom_type type)
bellardd0ecd2a2006-04-23 17:14:48 +00002894{
Paolo Bonzini149f54b2013-05-24 12:59:37 +02002895 hwaddr l;
Philippe Mathieu-Daudé20804672020-02-19 18:01:32 +01002896 uint8_t *ram_ptr;
Paolo Bonzini149f54b2013-05-24 12:59:37 +02002897 hwaddr addr1;
Paolo Bonzini5c8a00c2013-05-29 12:42:00 +02002898 MemoryRegion *mr;
Philippe Mathieu-Daudédaa3dda2020-02-19 19:54:35 +01002899 const uint8_t *buf = ptr;
ths3b46e622007-09-17 08:09:54 +00002900
Dr. David Alan Gilbert694ea272019-10-07 15:36:41 +01002901 RCU_READ_LOCK_GUARD();
bellardd0ecd2a2006-04-23 17:14:48 +00002902 while (len > 0) {
Paolo Bonzini149f54b2013-05-24 12:59:37 +02002903 l = len;
Peter Maydell75693e12018-12-14 13:30:48 +00002904 mr = address_space_translate(as, addr, &addr1, &l, true, attrs);
ths3b46e622007-09-17 08:09:54 +00002905
Paolo Bonzini5c8a00c2013-05-29 12:42:00 +02002906 if (!(memory_region_is_ram(mr) ||
2907 memory_region_is_romd(mr))) {
Paolo Bonzinib242e0e2015-07-04 00:24:51 +02002908 l = memory_access_size(mr, l, addr1);
bellardd0ecd2a2006-04-23 17:14:48 +00002909 } else {
bellardd0ecd2a2006-04-23 17:14:48 +00002910 /* ROM/RAM case */
Philippe Mathieu-Daudé20804672020-02-19 18:01:32 +01002911 ram_ptr = qemu_map_ram_ptr(mr->ram_block, addr1);
Alexander Graf582b55a2013-12-11 14:17:44 +01002912 switch (type) {
2913 case WRITE_DATA:
Philippe Mathieu-Daudé20804672020-02-19 18:01:32 +01002914 memcpy(ram_ptr, buf, l);
Paolo Bonzini845b6212015-03-23 11:45:53 +01002915 invalidate_and_set_dirty(mr, addr1, l);
Alexander Graf582b55a2013-12-11 14:17:44 +01002916 break;
2917 case FLUSH_CACHE:
Richard Henderson1da8de32020-12-12 10:38:21 -06002918 flush_idcache_range((uintptr_t)ram_ptr, (uintptr_t)ram_ptr, l);
Alexander Graf582b55a2013-12-11 14:17:44 +01002919 break;
2920 }
bellardd0ecd2a2006-04-23 17:14:48 +00002921 }
2922 len -= l;
2923 buf += l;
2924 addr += l;
2925 }
Peter Maydell75693e12018-12-14 13:30:48 +00002926 return MEMTX_OK;
bellardd0ecd2a2006-04-23 17:14:48 +00002927}
2928
Alexander Graf582b55a2013-12-11 14:17:44 +01002929/* used for ROM loading : can write in RAM and ROM */
Peter Maydell3c8133f2018-12-14 13:30:48 +00002930MemTxResult address_space_write_rom(AddressSpace *as, hwaddr addr,
2931 MemTxAttrs attrs,
Philippe Mathieu-Daudédaa3dda2020-02-19 19:54:35 +01002932 const void *buf, hwaddr len)
Alexander Graf582b55a2013-12-11 14:17:44 +01002933{
Peter Maydell3c8133f2018-12-14 13:30:48 +00002934 return address_space_write_rom_internal(as, addr, attrs,
2935 buf, len, WRITE_DATA);
Alexander Graf582b55a2013-12-11 14:17:44 +01002936}
2937
Li Zhijian0c249ff2019-01-17 20:49:01 +08002938void cpu_flush_icache_range(hwaddr start, hwaddr len)
Alexander Graf582b55a2013-12-11 14:17:44 +01002939{
2940 /*
2941 * This function should do the same thing as an icache flush that was
2942 * triggered from within the guest. For TCG we are always cache coherent,
2943 * so there is no need to flush anything. For KVM / Xen we need to flush
2944 * the host's instruction cache at least.
2945 */
2946 if (tcg_enabled()) {
2947 return;
2948 }
2949
Peter Maydell75693e12018-12-14 13:30:48 +00002950 address_space_write_rom_internal(&address_space_memory,
2951 start, MEMTXATTRS_UNSPECIFIED,
2952 NULL, len, FLUSH_CACHE);
Alexander Graf582b55a2013-12-11 14:17:44 +01002953}
2954
aliguori6d16c2f2009-01-22 16:59:11 +00002955typedef struct {
Paolo Bonzinid3e71552013-06-28 17:33:29 +02002956 MemoryRegion *mr;
aliguori6d16c2f2009-01-22 16:59:11 +00002957 void *buffer;
Avi Kivitya8170e52012-10-23 12:30:10 +02002958 hwaddr addr;
2959 hwaddr len;
Fam Zhengc2cba0f2015-03-16 17:03:33 +08002960 bool in_use;
aliguori6d16c2f2009-01-22 16:59:11 +00002961} BounceBuffer;
2962
2963static BounceBuffer bounce;
2964
aliguoriba223c22009-01-22 16:59:16 +00002965typedef struct MapClient {
Fam Zhenge95205e2015-03-16 17:03:37 +08002966 QEMUBH *bh;
Blue Swirl72cf2d42009-09-12 07:36:22 +00002967 QLIST_ENTRY(MapClient) link;
aliguoriba223c22009-01-22 16:59:16 +00002968} MapClient;
2969
Fam Zheng38e047b2015-03-16 17:03:35 +08002970QemuMutex map_client_list_lock;
Paolo Bonzinib58deb32018-12-06 11:58:10 +01002971static QLIST_HEAD(, MapClient) map_client_list
Blue Swirl72cf2d42009-09-12 07:36:22 +00002972 = QLIST_HEAD_INITIALIZER(map_client_list);
aliguoriba223c22009-01-22 16:59:16 +00002973
Fam Zhenge95205e2015-03-16 17:03:37 +08002974static void cpu_unregister_map_client_do(MapClient *client)
aliguoriba223c22009-01-22 16:59:16 +00002975{
Blue Swirl72cf2d42009-09-12 07:36:22 +00002976 QLIST_REMOVE(client, link);
Anthony Liguori7267c092011-08-20 22:09:37 -05002977 g_free(client);
aliguoriba223c22009-01-22 16:59:16 +00002978}
2979
Fam Zheng33b6c2e2015-03-16 17:03:36 +08002980static void cpu_notify_map_clients_locked(void)
aliguoriba223c22009-01-22 16:59:16 +00002981{
2982 MapClient *client;
2983
Blue Swirl72cf2d42009-09-12 07:36:22 +00002984 while (!QLIST_EMPTY(&map_client_list)) {
2985 client = QLIST_FIRST(&map_client_list);
Fam Zhenge95205e2015-03-16 17:03:37 +08002986 qemu_bh_schedule(client->bh);
2987 cpu_unregister_map_client_do(client);
aliguoriba223c22009-01-22 16:59:16 +00002988 }
2989}
2990
Fam Zhenge95205e2015-03-16 17:03:37 +08002991void cpu_register_map_client(QEMUBH *bh)
bellardd0ecd2a2006-04-23 17:14:48 +00002992{
2993 MapClient *client = g_malloc(sizeof(*client));
2994
Fam Zheng38e047b2015-03-16 17:03:35 +08002995 qemu_mutex_lock(&map_client_list_lock);
Fam Zhenge95205e2015-03-16 17:03:37 +08002996 client->bh = bh;
bellardd0ecd2a2006-04-23 17:14:48 +00002997 QLIST_INSERT_HEAD(&map_client_list, client, link);
Paolo Bonzini33828ca2023-03-03 14:36:32 +01002998 /* Write map_client_list before reading in_use. */
2999 smp_mb();
Stefan Hajnoczid73415a2020-09-23 11:56:46 +01003000 if (!qatomic_read(&bounce.in_use)) {
Fam Zheng33b6c2e2015-03-16 17:03:36 +08003001 cpu_notify_map_clients_locked();
3002 }
Fam Zheng38e047b2015-03-16 17:03:35 +08003003 qemu_mutex_unlock(&map_client_list_lock);
bellardd0ecd2a2006-04-23 17:14:48 +00003004}
3005
Fam Zheng38e047b2015-03-16 17:03:35 +08003006void cpu_exec_init_all(void)
3007{
3008 qemu_mutex_init(&ram_list.mutex);
Peter Maydell20bccb82016-10-24 16:26:49 +01003009 /* The data structures we set up here depend on knowing the page size,
3010 * so no more changes can be made after this point.
3011 * In an ideal world, nothing we did before we had finished the
3012 * machine setup would care about the target page size, and we could
3013 * do this much later, rather than requiring board models to state
3014 * up front what their requirements are.
3015 */
3016 finalize_target_page_bits();
Fam Zheng38e047b2015-03-16 17:03:35 +08003017 io_mem_init();
Paolo Bonzini680a4782015-11-02 09:23:52 +01003018 memory_map_init();
Fam Zheng38e047b2015-03-16 17:03:35 +08003019 qemu_mutex_init(&map_client_list_lock);
3020}
3021
Fam Zhenge95205e2015-03-16 17:03:37 +08003022void cpu_unregister_map_client(QEMUBH *bh)
bellardd0ecd2a2006-04-23 17:14:48 +00003023{
Fam Zhenge95205e2015-03-16 17:03:37 +08003024 MapClient *client;
bellardd0ecd2a2006-04-23 17:14:48 +00003025
Fam Zhenge95205e2015-03-16 17:03:37 +08003026 qemu_mutex_lock(&map_client_list_lock);
3027 QLIST_FOREACH(client, &map_client_list, link) {
3028 if (client->bh == bh) {
3029 cpu_unregister_map_client_do(client);
3030 break;
3031 }
3032 }
3033 qemu_mutex_unlock(&map_client_list_lock);
bellardd0ecd2a2006-04-23 17:14:48 +00003034}
3035
3036static void cpu_notify_map_clients(void)
3037{
Fam Zheng38e047b2015-03-16 17:03:35 +08003038 qemu_mutex_lock(&map_client_list_lock);
Fam Zheng33b6c2e2015-03-16 17:03:36 +08003039 cpu_notify_map_clients_locked();
Fam Zheng38e047b2015-03-16 17:03:35 +08003040 qemu_mutex_unlock(&map_client_list_lock);
aliguori6d16c2f2009-01-22 16:59:11 +00003041}
3042
Li Zhijian0c249ff2019-01-17 20:49:01 +08003043static bool flatview_access_valid(FlatView *fv, hwaddr addr, hwaddr len,
Peter Maydelleace72b2018-05-31 14:50:52 +01003044 bool is_write, MemTxAttrs attrs)
Paolo Bonzini51644ab2013-04-11 15:40:59 +02003045{
Paolo Bonzini5c8a00c2013-05-29 12:42:00 +02003046 MemoryRegion *mr;
Paolo Bonzini51644ab2013-04-11 15:40:59 +02003047 hwaddr l, xlat;
3048
3049 while (len > 0) {
3050 l = len;
Peter Maydellefa99a22018-05-31 14:50:52 +01003051 mr = flatview_translate(fv, addr, &xlat, &l, is_write, attrs);
Paolo Bonzini5c8a00c2013-05-29 12:42:00 +02003052 if (!memory_access_is_direct(mr, is_write)) {
3053 l = memory_access_size(mr, l, addr);
Peter Maydelleace72b2018-05-31 14:50:52 +01003054 if (!memory_region_access_valid(mr, xlat, l, is_write, attrs)) {
Paolo Bonzini51644ab2013-04-11 15:40:59 +02003055 return false;
3056 }
3057 }
3058
3059 len -= l;
3060 addr += l;
3061 }
3062 return true;
3063}
3064
Alexey Kardashevskiy16620682017-09-21 18:50:58 +10003065bool address_space_access_valid(AddressSpace *as, hwaddr addr,
Li Zhijian0c249ff2019-01-17 20:49:01 +08003066 hwaddr len, bool is_write,
Peter Maydellfddffa42018-05-31 14:50:52 +01003067 MemTxAttrs attrs)
Alexey Kardashevskiy16620682017-09-21 18:50:58 +10003068{
Paolo Bonzini11e732a2018-03-05 00:23:26 +01003069 FlatView *fv;
Paolo Bonzini11e732a2018-03-05 00:23:26 +01003070
Dr. David Alan Gilbert694ea272019-10-07 15:36:41 +01003071 RCU_READ_LOCK_GUARD();
Paolo Bonzini11e732a2018-03-05 00:23:26 +01003072 fv = address_space_to_flatview(as);
Philippe Mathieu-Daudé58e74682021-12-15 19:24:20 +01003073 return flatview_access_valid(fv, addr, len, is_write, attrs);
Alexey Kardashevskiy16620682017-09-21 18:50:58 +10003074}
3075
Paolo Bonzini715c31e2016-11-22 12:04:31 +01003076static hwaddr
Alexey Kardashevskiy16620682017-09-21 18:50:58 +10003077flatview_extend_translation(FlatView *fv, hwaddr addr,
Peter Maydell53d07902018-05-31 14:50:52 +01003078 hwaddr target_len,
3079 MemoryRegion *mr, hwaddr base, hwaddr len,
3080 bool is_write, MemTxAttrs attrs)
Paolo Bonzini715c31e2016-11-22 12:04:31 +01003081{
3082 hwaddr done = 0;
3083 hwaddr xlat;
3084 MemoryRegion *this_mr;
3085
3086 for (;;) {
3087 target_len -= len;
3088 addr += len;
3089 done += len;
3090 if (target_len == 0) {
3091 return done;
3092 }
3093
3094 len = target_len;
Alexey Kardashevskiy16620682017-09-21 18:50:58 +10003095 this_mr = flatview_translate(fv, addr, &xlat,
Peter Maydellefa99a22018-05-31 14:50:52 +01003096 &len, is_write, attrs);
Paolo Bonzini715c31e2016-11-22 12:04:31 +01003097 if (this_mr != mr || xlat != base + done) {
3098 return done;
3099 }
3100 }
3101}
3102
aliguori6d16c2f2009-01-22 16:59:11 +00003103/* Map a physical memory region into a host virtual address.
3104 * May map a subset of the requested range, given by and returned in *plen.
3105 * May return NULL if resources needed to perform the mapping are exhausted.
3106 * Use only for reads OR writes - not for read-modify-write operations.
aliguoriba223c22009-01-22 16:59:16 +00003107 * Use cpu_register_map_client() to know when retrying the map operation is
3108 * likely to succeed.
aliguori6d16c2f2009-01-22 16:59:11 +00003109 */
Avi Kivityac1970f2012-10-03 16:22:53 +02003110void *address_space_map(AddressSpace *as,
Avi Kivitya8170e52012-10-23 12:30:10 +02003111 hwaddr addr,
3112 hwaddr *plen,
Peter Maydellf26404f2018-05-31 14:50:52 +01003113 bool is_write,
3114 MemTxAttrs attrs)
aliguori6d16c2f2009-01-22 16:59:11 +00003115{
Avi Kivitya8170e52012-10-23 12:30:10 +02003116 hwaddr len = *plen;
Paolo Bonzini715c31e2016-11-22 12:04:31 +01003117 hwaddr l, xlat;
3118 MemoryRegion *mr;
Paolo Bonziniad0c60f2018-03-05 00:23:26 +01003119 FlatView *fv;
aliguori6d16c2f2009-01-22 16:59:11 +00003120
Paolo Bonzinie3127ae2013-06-28 17:29:27 +02003121 if (len == 0) {
3122 return NULL;
3123 }
aliguori6d16c2f2009-01-22 16:59:11 +00003124
Paolo Bonzinie3127ae2013-06-28 17:29:27 +02003125 l = len;
Dr. David Alan Gilbert694ea272019-10-07 15:36:41 +01003126 RCU_READ_LOCK_GUARD();
Paolo Bonziniad0c60f2018-03-05 00:23:26 +01003127 fv = address_space_to_flatview(as);
Peter Maydellefa99a22018-05-31 14:50:52 +01003128 mr = flatview_translate(fv, addr, &xlat, &l, is_write, attrs);
Paolo Bonzini41063e12015-03-18 14:21:43 +01003129
Paolo Bonzinie3127ae2013-06-28 17:29:27 +02003130 if (!memory_access_is_direct(mr, is_write)) {
Stefan Hajnoczid73415a2020-09-23 11:56:46 +01003131 if (qatomic_xchg(&bounce.in_use, true)) {
Prasad J Pandit77f55ea2020-05-26 16:47:43 +05303132 *plen = 0;
Paolo Bonzinie3127ae2013-06-28 17:29:27 +02003133 return NULL;
aliguori6d16c2f2009-01-22 16:59:11 +00003134 }
Kevin Wolfe85d9db2013-07-22 14:30:23 +02003135 /* Avoid unbounded allocations */
3136 l = MIN(l, TARGET_PAGE_SIZE);
3137 bounce.buffer = qemu_memalign(TARGET_PAGE_SIZE, l);
Paolo Bonzinie3127ae2013-06-28 17:29:27 +02003138 bounce.addr = addr;
3139 bounce.len = l;
Paolo Bonzinid3e71552013-06-28 17:33:29 +02003140
3141 memory_region_ref(mr);
3142 bounce.mr = mr;
Paolo Bonzinie3127ae2013-06-28 17:29:27 +02003143 if (!is_write) {
Alexey Kardashevskiy16620682017-09-21 18:50:58 +10003144 flatview_read(fv, addr, MEMTXATTRS_UNSPECIFIED,
Peter Maydell5c9eb022015-04-26 16:49:24 +01003145 bounce.buffer, l);
Stefano Stabellini8ab934f2011-06-27 18:26:06 +01003146 }
aliguori6d16c2f2009-01-22 16:59:11 +00003147
Paolo Bonzinie3127ae2013-06-28 17:29:27 +02003148 *plen = l;
3149 return bounce.buffer;
3150 }
3151
Paolo Bonzinie3127ae2013-06-28 17:29:27 +02003152
Paolo Bonzinid3e71552013-06-28 17:33:29 +02003153 memory_region_ref(mr);
Alexey Kardashevskiy16620682017-09-21 18:50:58 +10003154 *plen = flatview_extend_translation(fv, addr, len, mr, xlat,
Peter Maydell53d07902018-05-31 14:50:52 +01003155 l, is_write, attrs);
Alexander Bulekovfc1c8342021-01-20 01:02:55 -05003156 fuzz_dma_read_cb(addr, *plen, mr);
Markus Armbruster66997c42022-11-22 14:49:16 +01003157 return qemu_ram_ptr_length(mr->ram_block, xlat, plen, true);
aliguori6d16c2f2009-01-22 16:59:11 +00003158}
3159
Avi Kivityac1970f2012-10-03 16:22:53 +02003160/* Unmaps a memory region previously mapped by address_space_map().
Philippe Mathieu-Daudéae5883a2020-02-19 20:12:01 +01003161 * Will also mark the memory as dirty if is_write is true. access_len gives
aliguori6d16c2f2009-01-22 16:59:11 +00003162 * the amount of memory that was actually read or written by the caller.
3163 */
Avi Kivitya8170e52012-10-23 12:30:10 +02003164void address_space_unmap(AddressSpace *as, void *buffer, hwaddr len,
Philippe Mathieu-Daudéae5883a2020-02-19 20:12:01 +01003165 bool is_write, hwaddr access_len)
aliguori6d16c2f2009-01-22 16:59:11 +00003166{
3167 if (buffer != bounce.buffer) {
Paolo Bonzinid3e71552013-06-28 17:33:29 +02003168 MemoryRegion *mr;
3169 ram_addr_t addr1;
3170
Paolo Bonzini07bdaa42016-03-25 12:55:08 +01003171 mr = memory_region_from_host(buffer, &addr1);
Paolo Bonzinid3e71552013-06-28 17:33:29 +02003172 assert(mr != NULL);
aliguori6d16c2f2009-01-22 16:59:11 +00003173 if (is_write) {
Paolo Bonzini845b6212015-03-23 11:45:53 +01003174 invalidate_and_set_dirty(mr, addr1, access_len);
aliguori6d16c2f2009-01-22 16:59:11 +00003175 }
Jan Kiszka868bb332011-06-21 22:59:09 +02003176 if (xen_enabled()) {
Jan Kiszkae41d7c62011-06-21 22:59:08 +02003177 xen_invalidate_map_cache_entry(buffer);
Anthony PERARD050a0dd2010-09-16 13:57:49 +01003178 }
Paolo Bonzinid3e71552013-06-28 17:33:29 +02003179 memory_region_unref(mr);
aliguori6d16c2f2009-01-22 16:59:11 +00003180 return;
3181 }
3182 if (is_write) {
Peter Maydell5c9eb022015-04-26 16:49:24 +01003183 address_space_write(as, bounce.addr, MEMTXATTRS_UNSPECIFIED,
3184 bounce.buffer, access_len);
aliguori6d16c2f2009-01-22 16:59:11 +00003185 }
Herve Poussineauf8a83242010-01-24 21:23:56 +00003186 qemu_vfree(bounce.buffer);
aliguori6d16c2f2009-01-22 16:59:11 +00003187 bounce.buffer = NULL;
Paolo Bonzinid3e71552013-06-28 17:33:29 +02003188 memory_region_unref(bounce.mr);
Paolo Bonzini33828ca2023-03-03 14:36:32 +01003189 /* Clear in_use before reading map_client_list. */
Paolo Bonzini06831002023-03-03 14:37:51 +01003190 qatomic_set_mb(&bounce.in_use, false);
aliguoriba223c22009-01-22 16:59:16 +00003191 cpu_notify_map_clients();
aliguori6d16c2f2009-01-22 16:59:11 +00003192}
bellardd0ecd2a2006-04-23 17:14:48 +00003193
Avi Kivitya8170e52012-10-23 12:30:10 +02003194void *cpu_physical_memory_map(hwaddr addr,
3195 hwaddr *plen,
Philippe Mathieu-Daudé28c80bf2020-02-19 20:32:30 +01003196 bool is_write)
Avi Kivityac1970f2012-10-03 16:22:53 +02003197{
Peter Maydellf26404f2018-05-31 14:50:52 +01003198 return address_space_map(&address_space_memory, addr, plen, is_write,
3199 MEMTXATTRS_UNSPECIFIED);
Avi Kivityac1970f2012-10-03 16:22:53 +02003200}
3201
Avi Kivitya8170e52012-10-23 12:30:10 +02003202void cpu_physical_memory_unmap(void *buffer, hwaddr len,
Philippe Mathieu-Daudé28c80bf2020-02-19 20:32:30 +01003203 bool is_write, hwaddr access_len)
Avi Kivityac1970f2012-10-03 16:22:53 +02003204{
3205 return address_space_unmap(&address_space_memory, buffer, len, is_write, access_len);
3206}
3207
Paolo Bonzini0ce265f2016-11-22 11:34:02 +01003208#define ARG1_DECL AddressSpace *as
3209#define ARG1 as
3210#define SUFFIX
3211#define TRANSLATE(...) address_space_translate(as, __VA_ARGS__)
Paolo Bonzini0ce265f2016-11-22 11:34:02 +01003212#define RCU_READ_LOCK(...) rcu_read_lock()
3213#define RCU_READ_UNLOCK(...) rcu_read_unlock()
Paolo Bonzini139c1832020-02-04 12:41:01 +01003214#include "memory_ldst.c.inc"
Alexander Graf1e78bcc2011-07-06 09:09:23 +02003215
Paolo Bonzini1f4e4962016-11-22 12:04:52 +01003216int64_t address_space_cache_init(MemoryRegionCache *cache,
3217 AddressSpace *as,
3218 hwaddr addr,
3219 hwaddr len,
3220 bool is_write)
3221{
Paolo Bonzini48564042018-03-18 18:26:36 +01003222 AddressSpaceDispatch *d;
3223 hwaddr l;
3224 MemoryRegion *mr;
Paolo Bonzini4bfb0242020-12-01 09:29:56 -05003225 Int128 diff;
Paolo Bonzini48564042018-03-18 18:26:36 +01003226
3227 assert(len > 0);
3228
3229 l = len;
3230 cache->fv = address_space_get_flatview(as);
3231 d = flatview_to_dispatch(cache->fv);
3232 cache->mrs = *address_space_translate_internal(d, addr, &cache->xlat, &l, true);
3233
Paolo Bonzini4bfb0242020-12-01 09:29:56 -05003234 /*
3235 * cache->xlat is now relative to cache->mrs.mr, not to the section itself.
3236 * Take that into account to compute how many bytes are there between
3237 * cache->xlat and the end of the section.
3238 */
3239 diff = int128_sub(cache->mrs.size,
Yeqi Fu48805df2023-03-15 11:26:49 +08003240 int128_make64(cache->xlat - cache->mrs.offset_within_region));
Paolo Bonzini4bfb0242020-12-01 09:29:56 -05003241 l = int128_get64(int128_min(diff, int128_make64(l)));
3242
Paolo Bonzini48564042018-03-18 18:26:36 +01003243 mr = cache->mrs.mr;
3244 memory_region_ref(mr);
3245 if (memory_access_is_direct(mr, is_write)) {
Peter Maydell53d07902018-05-31 14:50:52 +01003246 /* We don't care about the memory attributes here as we're only
3247 * doing this if we found actual RAM, which behaves the same
3248 * regardless of attributes; so UNSPECIFIED is fine.
3249 */
Paolo Bonzini48564042018-03-18 18:26:36 +01003250 l = flatview_extend_translation(cache->fv, addr, len, mr,
Peter Maydell53d07902018-05-31 14:50:52 +01003251 cache->xlat, l, is_write,
3252 MEMTXATTRS_UNSPECIFIED);
Paolo Bonzini48564042018-03-18 18:26:36 +01003253 cache->ptr = qemu_ram_ptr_length(mr->ram_block, cache->xlat, &l, true);
3254 } else {
3255 cache->ptr = NULL;
3256 }
3257
3258 cache->len = l;
3259 cache->is_write = is_write;
3260 return l;
Paolo Bonzini1f4e4962016-11-22 12:04:52 +01003261}
3262
3263void address_space_cache_invalidate(MemoryRegionCache *cache,
3264 hwaddr addr,
3265 hwaddr access_len)
3266{
Paolo Bonzini48564042018-03-18 18:26:36 +01003267 assert(cache->is_write);
3268 if (likely(cache->ptr)) {
3269 invalidate_and_set_dirty(cache->mrs.mr, addr + cache->xlat, access_len);
3270 }
Paolo Bonzini1f4e4962016-11-22 12:04:52 +01003271}
3272
3273void address_space_cache_destroy(MemoryRegionCache *cache)
3274{
Paolo Bonzini48564042018-03-18 18:26:36 +01003275 if (!cache->mrs.mr) {
3276 return;
3277 }
3278
3279 if (xen_enabled()) {
3280 xen_invalidate_map_cache_entry(cache->ptr);
3281 }
3282 memory_region_unref(cache->mrs.mr);
3283 flatview_unref(cache->fv);
3284 cache->mrs.mr = NULL;
3285 cache->fv = NULL;
3286}
3287
3288/* Called from RCU critical section. This function has the same
3289 * semantics as address_space_translate, but it only works on a
3290 * predefined range of a MemoryRegion that was mapped with
3291 * address_space_cache_init.
3292 */
3293static inline MemoryRegion *address_space_translate_cached(
3294 MemoryRegionCache *cache, hwaddr addr, hwaddr *xlat,
Peter Maydellbc6b1ce2018-05-31 14:50:52 +01003295 hwaddr *plen, bool is_write, MemTxAttrs attrs)
Paolo Bonzini48564042018-03-18 18:26:36 +01003296{
3297 MemoryRegionSection section;
3298 MemoryRegion *mr;
3299 IOMMUMemoryRegion *iommu_mr;
3300 AddressSpace *target_as;
3301
3302 assert(!cache->ptr);
3303 *xlat = addr + cache->xlat;
3304
3305 mr = cache->mrs.mr;
3306 iommu_mr = memory_region_get_iommu(mr);
3307 if (!iommu_mr) {
3308 /* MMIO region. */
3309 return mr;
3310 }
3311
3312 section = address_space_translate_iommu(iommu_mr, xlat, plen,
3313 NULL, is_write, true,
Peter Maydell2f7b0092018-05-31 14:50:53 +01003314 &target_as, attrs);
Paolo Bonzini48564042018-03-18 18:26:36 +01003315 return section.mr;
3316}
3317
3318/* Called from RCU critical section. address_space_read_cached uses this
3319 * out of line function when the target is an MMIO or IOMMU region.
3320 */
Philippe Mathieu-Daudé38df19f2020-05-18 17:53:02 +02003321MemTxResult
Paolo Bonzini48564042018-03-18 18:26:36 +01003322address_space_read_cached_slow(MemoryRegionCache *cache, hwaddr addr,
Li Zhijian0c249ff2019-01-17 20:49:01 +08003323 void *buf, hwaddr len)
Paolo Bonzini48564042018-03-18 18:26:36 +01003324{
3325 hwaddr addr1, l;
3326 MemoryRegion *mr;
3327
3328 l = len;
Peter Maydellbc6b1ce2018-05-31 14:50:52 +01003329 mr = address_space_translate_cached(cache, addr, &addr1, &l, false,
3330 MEMTXATTRS_UNSPECIFIED);
Philippe Mathieu-Daudé38df19f2020-05-18 17:53:02 +02003331 return flatview_read_continue(cache->fv,
3332 addr, MEMTXATTRS_UNSPECIFIED, buf, len,
3333 addr1, l, mr);
Paolo Bonzini48564042018-03-18 18:26:36 +01003334}
3335
3336/* Called from RCU critical section. address_space_write_cached uses this
3337 * out of line function when the target is an MMIO or IOMMU region.
3338 */
Philippe Mathieu-Daudé38df19f2020-05-18 17:53:02 +02003339MemTxResult
Paolo Bonzini48564042018-03-18 18:26:36 +01003340address_space_write_cached_slow(MemoryRegionCache *cache, hwaddr addr,
Li Zhijian0c249ff2019-01-17 20:49:01 +08003341 const void *buf, hwaddr len)
Paolo Bonzini48564042018-03-18 18:26:36 +01003342{
3343 hwaddr addr1, l;
3344 MemoryRegion *mr;
3345
3346 l = len;
Peter Maydellbc6b1ce2018-05-31 14:50:52 +01003347 mr = address_space_translate_cached(cache, addr, &addr1, &l, true,
3348 MEMTXATTRS_UNSPECIFIED);
Philippe Mathieu-Daudé38df19f2020-05-18 17:53:02 +02003349 return flatview_write_continue(cache->fv,
3350 addr, MEMTXATTRS_UNSPECIFIED, buf, len,
3351 addr1, l, mr);
Paolo Bonzini1f4e4962016-11-22 12:04:52 +01003352}
3353
3354#define ARG1_DECL MemoryRegionCache *cache
3355#define ARG1 cache
Paolo Bonzini48564042018-03-18 18:26:36 +01003356#define SUFFIX _cached_slow
3357#define TRANSLATE(...) address_space_translate_cached(cache, __VA_ARGS__)
Paolo Bonzini48564042018-03-18 18:26:36 +01003358#define RCU_READ_LOCK() ((void)0)
3359#define RCU_READ_UNLOCK() ((void)0)
Paolo Bonzini139c1832020-02-04 12:41:01 +01003360#include "memory_ldst.c.inc"
Paolo Bonzini1f4e4962016-11-22 12:04:52 +01003361
aliguori5e2972f2009-03-28 17:51:36 +00003362/* virtual memory access for debug (includes writing to ROM) */
Philippe Mathieu-Daudé73842ef2022-02-03 02:13:28 +01003363int cpu_memory_rw_debug(CPUState *cpu, vaddr addr,
3364 void *ptr, size_t len, bool is_write)
bellard13eb76e2004-01-24 15:23:36 +00003365{
Avi Kivitya8170e52012-10-23 12:30:10 +02003366 hwaddr phys_addr;
Philippe Mathieu-Daudé73842ef2022-02-03 02:13:28 +01003367 vaddr l, page;
Philippe Mathieu-Daudéd7ef71e2020-02-19 20:02:11 +01003368 uint8_t *buf = ptr;
bellard13eb76e2004-01-24 15:23:36 +00003369
Christian Borntraeger79ca7a12017-03-07 15:19:08 +01003370 cpu_synchronize_state(cpu);
bellard13eb76e2004-01-24 15:23:36 +00003371 while (len > 0) {
Peter Maydell5232e4c2016-01-21 14:15:06 +00003372 int asidx;
3373 MemTxAttrs attrs;
Philippe Mathieu-Daudéddfc8b92020-05-18 17:53:03 +02003374 MemTxResult res;
Peter Maydell5232e4c2016-01-21 14:15:06 +00003375
bellard13eb76e2004-01-24 15:23:36 +00003376 page = addr & TARGET_PAGE_MASK;
Peter Maydell5232e4c2016-01-21 14:15:06 +00003377 phys_addr = cpu_get_phys_page_attrs_debug(cpu, page, &attrs);
3378 asidx = cpu_asidx_from_attrs(cpu, attrs);
bellard13eb76e2004-01-24 15:23:36 +00003379 /* if no physical page mapped, return an error */
3380 if (phys_addr == -1)
3381 return -1;
3382 l = (page + TARGET_PAGE_SIZE) - addr;
3383 if (l > len)
3384 l = len;
aliguori5e2972f2009-03-28 17:51:36 +00003385 phys_addr += (addr & ~TARGET_PAGE_MASK);
Edgar E. Iglesias2e388472013-12-13 16:31:02 +10003386 if (is_write) {
Philippe Mathieu-Daudéddfc8b92020-05-18 17:53:03 +02003387 res = address_space_write_rom(cpu->cpu_ases[asidx].as, phys_addr,
3388 attrs, buf, l);
Edgar E. Iglesias2e388472013-12-13 16:31:02 +10003389 } else {
Philippe Mathieu-Daudéddfc8b92020-05-18 17:53:03 +02003390 res = address_space_read(cpu->cpu_ases[asidx].as, phys_addr,
3391 attrs, buf, l);
3392 }
3393 if (res != MEMTX_OK) {
3394 return -1;
Edgar E. Iglesias2e388472013-12-13 16:31:02 +10003395 }
bellard13eb76e2004-01-24 15:23:36 +00003396 len -= l;
3397 buf += l;
3398 addr += l;
3399 }
3400 return 0;
3401}
Dr. David Alan Gilbert038629a2015-11-05 18:10:29 +00003402
3403/*
3404 * Allows code that needs to deal with migration bitmaps etc to still be built
3405 * target independent.
3406 */
Juan Quintela20afaed2017-03-21 09:09:14 +01003407size_t qemu_target_page_size(void)
Dr. David Alan Gilbert038629a2015-11-05 18:10:29 +00003408{
Juan Quintela20afaed2017-03-21 09:09:14 +01003409 return TARGET_PAGE_SIZE;
Dr. David Alan Gilbert038629a2015-11-05 18:10:29 +00003410}
3411
Philippe Mathieu-Daudéb3b408f2023-05-24 11:37:35 +02003412int qemu_target_page_mask(void)
3413{
3414 return TARGET_PAGE_MASK;
3415}
3416
Juan Quintela46d702b2017-04-24 21:03:48 +02003417int qemu_target_page_bits(void)
3418{
3419 return TARGET_PAGE_BITS;
3420}
3421
3422int qemu_target_page_bits_min(void)
3423{
3424 return TARGET_PAGE_BITS_MIN;
3425}
bellard13eb76e2004-01-24 15:23:36 +00003426
Juan Quintela62c5e182023-05-11 16:12:04 +02003427/* Convert target pages to MiB (2**20). */
3428size_t qemu_target_pages_to_MiB(size_t pages)
3429{
3430 int page_bits = TARGET_PAGE_BITS;
3431
3432 /* So far, the largest (non-huge) page size is 64k, i.e. 16 bits. */
3433 g_assert(page_bits < 20);
3434
3435 return pages >> (20 - page_bits);
3436}
3437
Avi Kivitya8170e52012-10-23 12:30:10 +02003438bool cpu_physical_memory_is_io(hwaddr phys_addr)
Wen Congyang76f35532012-05-07 12:04:18 +08003439{
Paolo Bonzini5c8a00c2013-05-29 12:42:00 +02003440 MemoryRegion*mr;
Paolo Bonzini149f54b2013-05-24 12:59:37 +02003441 hwaddr l = 1;
Wen Congyang76f35532012-05-07 12:04:18 +08003442
Dr. David Alan Gilbert694ea272019-10-07 15:36:41 +01003443 RCU_READ_LOCK_GUARD();
Paolo Bonzini5c8a00c2013-05-29 12:42:00 +02003444 mr = address_space_translate(&address_space_memory,
Peter Maydellbc6b1ce2018-05-31 14:50:52 +01003445 phys_addr, &phys_addr, &l, false,
3446 MEMTXATTRS_UNSPECIFIED);
Wen Congyang76f35532012-05-07 12:04:18 +08003447
Markus Armbruster66997c42022-11-22 14:49:16 +01003448 return !(memory_region_is_ram(mr) || memory_region_is_romd(mr));
Wen Congyang76f35532012-05-07 12:04:18 +08003449}
Michael R. Hinesbd2fa512013-06-25 21:35:34 -04003450
Dr. David Alan Gilberte3807052015-05-21 13:24:13 +01003451int qemu_ram_foreach_block(RAMBlockIterFunc func, void *opaque)
Michael R. Hinesbd2fa512013-06-25 21:35:34 -04003452{
3453 RAMBlock *block;
Dr. David Alan Gilberte3807052015-05-21 13:24:13 +01003454 int ret = 0;
Michael R. Hinesbd2fa512013-06-25 21:35:34 -04003455
Dr. David Alan Gilbert694ea272019-10-07 15:36:41 +01003456 RCU_READ_LOCK_GUARD();
Peter Xu99e15582017-05-12 12:17:39 +08003457 RAMBLOCK_FOREACH(block) {
Yury Kotov754cb9c2019-02-15 20:45:44 +03003458 ret = func(block, opaque);
Dr. David Alan Gilberte3807052015-05-21 13:24:13 +01003459 if (ret) {
3460 break;
3461 }
Michael R. Hinesbd2fa512013-06-25 21:35:34 -04003462 }
Dr. David Alan Gilberte3807052015-05-21 13:24:13 +01003463 return ret;
Michael R. Hinesbd2fa512013-06-25 21:35:34 -04003464}
Dr. David Alan Gilbertd3a50382017-02-24 18:28:32 +00003465
3466/*
3467 * Unmap pages of memory from start to start+length such that
3468 * they a) read as 0, b) Trigger whatever fault mechanism
3469 * the OS provides for postcopy.
3470 * The pages must be unmapped by the end of the function.
3471 * Returns: 0 on success, none-0 on failure
3472 *
3473 */
3474int ram_block_discard_range(RAMBlock *rb, uint64_t start, size_t length)
3475{
3476 int ret = -1;
3477
3478 uint8_t *host_startaddr = rb->host + start;
3479
Marc-André Lureau619bd312020-01-03 11:39:58 +04003480 if (!QEMU_PTR_IS_ALIGNED(host_startaddr, rb->page_size)) {
Dr. David Alan Gilbertd3a50382017-02-24 18:28:32 +00003481 error_report("ram_block_discard_range: Unaligned start address: %p",
3482 host_startaddr);
3483 goto err;
3484 }
3485
David Hildenbranddcdc4602021-04-29 13:27:03 +02003486 if ((start + length) <= rb->max_length) {
Dr. David Alan Gilbertdb144f72018-03-12 17:20:56 +00003487 bool need_madvise, need_fallocate;
Marc-André Lureau619bd312020-01-03 11:39:58 +04003488 if (!QEMU_IS_ALIGNED(length, rb->page_size)) {
Wei Yang72821d92019-07-12 11:27:04 +08003489 error_report("ram_block_discard_range: Unaligned length: %zx",
3490 length);
Dr. David Alan Gilbertd3a50382017-02-24 18:28:32 +00003491 goto err;
3492 }
3493
3494 errno = ENOTSUP; /* If we are missing MADVISE etc */
3495
Dr. David Alan Gilbertdb144f72018-03-12 17:20:56 +00003496 /* The logic here is messy;
3497 * madvise DONTNEED fails for hugepages
3498 * fallocate works on hugepages and shmem
David Hildenbrandcdfa56c2021-04-06 10:01:25 +02003499 * shared anonymous memory requires madvise REMOVE
Dr. David Alan Gilbertdb144f72018-03-12 17:20:56 +00003500 */
3501 need_madvise = (rb->page_size == qemu_host_page_size);
3502 need_fallocate = rb->fd != -1;
3503 if (need_fallocate) {
3504 /* For a file, this causes the area of the file to be zero'd
3505 * if read, and for hugetlbfs also causes it to be unmapped
3506 * so a userfault will trigger.
Dr. David Alan Gilberte2fa71f2017-02-24 18:28:33 +00003507 */
3508#ifdef CONFIG_FALLOCATE_PUNCH_HOLE
David Hildenbrand1d44ff52023-07-06 09:56:06 +02003509 /*
David Hildenbrandb2cccb52023-09-06 14:04:57 +02003510 * fallocate() will fail with readonly files. Let's print a
3511 * proper error message.
3512 */
3513 if (rb->flags & RAM_READONLY_FD) {
3514 error_report("ram_block_discard_range: Discarding RAM"
3515 " with readonly files is not supported");
3516 goto err;
3517
3518 }
3519 /*
David Hildenbrand1d44ff52023-07-06 09:56:06 +02003520 * We'll discard data from the actual file, even though we only
3521 * have a MAP_PRIVATE mapping, possibly messing with other
3522 * MAP_PRIVATE/MAP_SHARED mappings. There is no easy way to
3523 * change that behavior whithout violating the promised
3524 * semantics of ram_block_discard_range().
3525 *
3526 * Only warn, because it works as long as nobody else uses that
3527 * file.
3528 */
3529 if (!qemu_ram_is_shared(rb)) {
3530 warn_report_once("ram_block_discard_range: Discarding RAM"
3531 " in private file mappings is possibly"
3532 " dangerous, because it will modify the"
3533 " underlying file and will affect other"
3534 " users of the file");
3535 }
3536
Dr. David Alan Gilberte2fa71f2017-02-24 18:28:33 +00003537 ret = fallocate(rb->fd, FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE,
3538 start, length);
Dr. David Alan Gilbertdb144f72018-03-12 17:20:56 +00003539 if (ret) {
3540 ret = -errno;
3541 error_report("ram_block_discard_range: Failed to fallocate "
3542 "%s:%" PRIx64 " +%zx (%d)",
3543 rb->idstr, start, length, ret);
3544 goto err;
3545 }
3546#else
3547 ret = -ENOSYS;
3548 error_report("ram_block_discard_range: fallocate not available/file"
Dr. David Alan Gilbertd3a50382017-02-24 18:28:32 +00003549 "%s:%" PRIx64 " +%zx (%d)",
3550 rb->idstr, start, length, ret);
Dr. David Alan Gilbertdb144f72018-03-12 17:20:56 +00003551 goto err;
3552#endif
Dr. David Alan Gilbertd3a50382017-02-24 18:28:32 +00003553 }
Dr. David Alan Gilbertdb144f72018-03-12 17:20:56 +00003554 if (need_madvise) {
3555 /* For normal RAM this causes it to be unmapped,
3556 * for shared memory it causes the local mapping to disappear
3557 * and to fall back on the file contents (which we just
3558 * fallocate'd away).
3559 */
3560#if defined(CONFIG_MADVISE)
David Hildenbrandcdfa56c2021-04-06 10:01:25 +02003561 if (qemu_ram_is_shared(rb) && rb->fd < 0) {
3562 ret = madvise(host_startaddr, length, QEMU_MADV_REMOVE);
3563 } else {
3564 ret = madvise(host_startaddr, length, QEMU_MADV_DONTNEED);
3565 }
Dr. David Alan Gilbertdb144f72018-03-12 17:20:56 +00003566 if (ret) {
3567 ret = -errno;
3568 error_report("ram_block_discard_range: Failed to discard range "
3569 "%s:%" PRIx64 " +%zx (%d)",
3570 rb->idstr, start, length, ret);
3571 goto err;
3572 }
3573#else
3574 ret = -ENOSYS;
3575 error_report("ram_block_discard_range: MADVISE not available"
3576 "%s:%" PRIx64 " +%zx (%d)",
3577 rb->idstr, start, length, ret);
3578 goto err;
3579#endif
3580 }
3581 trace_ram_block_discard_range(rb->idstr, host_startaddr, length,
3582 need_madvise, need_fallocate, ret);
Dr. David Alan Gilbertd3a50382017-02-24 18:28:32 +00003583 } else {
3584 error_report("ram_block_discard_range: Overrun block '%s' (%" PRIu64
3585 "/%zx/" RAM_ADDR_FMT")",
David Hildenbranddcdc4602021-04-29 13:27:03 +02003586 rb->idstr, start, length, rb->max_length);
Dr. David Alan Gilbertd3a50382017-02-24 18:28:32 +00003587 }
3588
3589err:
3590 return ret;
3591}
3592
Junyan Hea4de8552018-07-18 15:48:00 +08003593bool ramblock_is_pmem(RAMBlock *rb)
3594{
3595 return rb->flags & RAM_PMEM;
3596}
3597
Markus Armbrusterb6b71cb2019-04-17 21:17:56 +02003598static void mtree_print_phys_entries(int start, int end, int skip, int ptr)
Alexey Kardashevskiy5e8fd942017-09-21 18:51:06 +10003599{
3600 if (start == end - 1) {
Markus Armbrusterb6b71cb2019-04-17 21:17:56 +02003601 qemu_printf("\t%3d ", start);
Alexey Kardashevskiy5e8fd942017-09-21 18:51:06 +10003602 } else {
Markus Armbrusterb6b71cb2019-04-17 21:17:56 +02003603 qemu_printf("\t%3d..%-3d ", start, end - 1);
Alexey Kardashevskiy5e8fd942017-09-21 18:51:06 +10003604 }
Markus Armbrusterb6b71cb2019-04-17 21:17:56 +02003605 qemu_printf(" skip=%d ", skip);
Alexey Kardashevskiy5e8fd942017-09-21 18:51:06 +10003606 if (ptr == PHYS_MAP_NODE_NIL) {
Markus Armbrusterb6b71cb2019-04-17 21:17:56 +02003607 qemu_printf(" ptr=NIL");
Alexey Kardashevskiy5e8fd942017-09-21 18:51:06 +10003608 } else if (!skip) {
Markus Armbrusterb6b71cb2019-04-17 21:17:56 +02003609 qemu_printf(" ptr=#%d", ptr);
Alexey Kardashevskiy5e8fd942017-09-21 18:51:06 +10003610 } else {
Markus Armbrusterb6b71cb2019-04-17 21:17:56 +02003611 qemu_printf(" ptr=[%d]", ptr);
Alexey Kardashevskiy5e8fd942017-09-21 18:51:06 +10003612 }
Markus Armbrusterb6b71cb2019-04-17 21:17:56 +02003613 qemu_printf("\n");
Alexey Kardashevskiy5e8fd942017-09-21 18:51:06 +10003614}
3615
3616#define MR_SIZE(size) (int128_nz(size) ? (hwaddr)int128_get64( \
3617 int128_sub((size), int128_one())) : 0)
3618
Markus Armbrusterb6b71cb2019-04-17 21:17:56 +02003619void mtree_print_dispatch(AddressSpaceDispatch *d, MemoryRegion *root)
Alexey Kardashevskiy5e8fd942017-09-21 18:51:06 +10003620{
3621 int i;
3622
Markus Armbrusterb6b71cb2019-04-17 21:17:56 +02003623 qemu_printf(" Dispatch\n");
3624 qemu_printf(" Physical sections\n");
Alexey Kardashevskiy5e8fd942017-09-21 18:51:06 +10003625
3626 for (i = 0; i < d->map.sections_nb; ++i) {
3627 MemoryRegionSection *s = d->map.sections + i;
3628 const char *names[] = { " [unassigned]", " [not dirty]",
3629 " [ROM]", " [watch]" };
3630
Philippe Mathieu-Daudé883f2c52023-01-10 22:29:47 +01003631 qemu_printf(" #%d @" HWADDR_FMT_plx ".." HWADDR_FMT_plx
Markus Armbrusterb6b71cb2019-04-17 21:17:56 +02003632 " %s%s%s%s%s",
Alexey Kardashevskiy5e8fd942017-09-21 18:51:06 +10003633 i,
3634 s->offset_within_address_space,
Zhenzhong Duanf9c307c2022-06-22 17:59:12 +08003635 s->offset_within_address_space + MR_SIZE(s->size),
Alexey Kardashevskiy5e8fd942017-09-21 18:51:06 +10003636 s->mr->name ? s->mr->name : "(noname)",
3637 i < ARRAY_SIZE(names) ? names[i] : "",
3638 s->mr == root ? " [ROOT]" : "",
3639 s == d->mru_section ? " [MRU]" : "",
3640 s->mr->is_iommu ? " [iommu]" : "");
3641
3642 if (s->mr->alias) {
Markus Armbrusterb6b71cb2019-04-17 21:17:56 +02003643 qemu_printf(" alias=%s", s->mr->alias->name ?
Alexey Kardashevskiy5e8fd942017-09-21 18:51:06 +10003644 s->mr->alias->name : "noname");
3645 }
Markus Armbrusterb6b71cb2019-04-17 21:17:56 +02003646 qemu_printf("\n");
Alexey Kardashevskiy5e8fd942017-09-21 18:51:06 +10003647 }
3648
Markus Armbrusterb6b71cb2019-04-17 21:17:56 +02003649 qemu_printf(" Nodes (%d bits per level, %d levels) ptr=[%d] skip=%d\n",
Alexey Kardashevskiy5e8fd942017-09-21 18:51:06 +10003650 P_L2_BITS, P_L2_LEVELS, d->phys_map.ptr, d->phys_map.skip);
3651 for (i = 0; i < d->map.nodes_nb; ++i) {
3652 int j, jprev;
3653 PhysPageEntry prev;
3654 Node *n = d->map.nodes + i;
3655
Markus Armbrusterb6b71cb2019-04-17 21:17:56 +02003656 qemu_printf(" [%d]\n", i);
Alexey Kardashevskiy5e8fd942017-09-21 18:51:06 +10003657
3658 for (j = 0, jprev = 0, prev = *n[0]; j < ARRAY_SIZE(*n); ++j) {
3659 PhysPageEntry *pe = *n + j;
3660
3661 if (pe->ptr == prev.ptr && pe->skip == prev.skip) {
3662 continue;
3663 }
3664
Markus Armbrusterb6b71cb2019-04-17 21:17:56 +02003665 mtree_print_phys_entries(jprev, j, prev.skip, prev.ptr);
Alexey Kardashevskiy5e8fd942017-09-21 18:51:06 +10003666
3667 jprev = j;
3668 prev = *pe;
3669 }
3670
3671 if (jprev != ARRAY_SIZE(*n)) {
Markus Armbrusterb6b71cb2019-04-17 21:17:56 +02003672 mtree_print_phys_entries(jprev, j, prev.skip, prev.ptr);
Alexey Kardashevskiy5e8fd942017-09-21 18:51:06 +10003673 }
3674 }
3675}
3676
David Hildenbrand7e6d32e2021-04-13 11:55:29 +02003677/* Require any discards to work. */
David Hildenbrand98da4912021-04-13 11:55:28 +02003678static unsigned int ram_block_discard_required_cnt;
David Hildenbrand7e6d32e2021-04-13 11:55:29 +02003679/* Require only coordinated discards to work. */
3680static unsigned int ram_block_coordinated_discard_required_cnt;
3681/* Disable any discards. */
David Hildenbrand98da4912021-04-13 11:55:28 +02003682static unsigned int ram_block_discard_disabled_cnt;
David Hildenbrand7e6d32e2021-04-13 11:55:29 +02003683/* Disable only uncoordinated discards. */
3684static unsigned int ram_block_uncoordinated_discard_disabled_cnt;
David Hildenbrand98da4912021-04-13 11:55:28 +02003685static QemuMutex ram_block_discard_disable_mutex;
3686
3687static void ram_block_discard_disable_mutex_lock(void)
3688{
3689 static gsize initialized;
3690
3691 if (g_once_init_enter(&initialized)) {
3692 qemu_mutex_init(&ram_block_discard_disable_mutex);
3693 g_once_init_leave(&initialized, 1);
3694 }
3695 qemu_mutex_lock(&ram_block_discard_disable_mutex);
3696}
3697
3698static void ram_block_discard_disable_mutex_unlock(void)
3699{
3700 qemu_mutex_unlock(&ram_block_discard_disable_mutex);
3701}
David Hildenbrandd24f31d2020-06-26 09:22:29 +02003702
3703int ram_block_discard_disable(bool state)
3704{
David Hildenbrand98da4912021-04-13 11:55:28 +02003705 int ret = 0;
David Hildenbrandd24f31d2020-06-26 09:22:29 +02003706
David Hildenbrand98da4912021-04-13 11:55:28 +02003707 ram_block_discard_disable_mutex_lock();
David Hildenbrandd24f31d2020-06-26 09:22:29 +02003708 if (!state) {
David Hildenbrand98da4912021-04-13 11:55:28 +02003709 ram_block_discard_disabled_cnt--;
David Hildenbrand7e6d32e2021-04-13 11:55:29 +02003710 } else if (ram_block_discard_required_cnt ||
3711 ram_block_coordinated_discard_required_cnt) {
David Hildenbrand98da4912021-04-13 11:55:28 +02003712 ret = -EBUSY;
David Hildenbrand7e6d32e2021-04-13 11:55:29 +02003713 } else {
3714 ram_block_discard_disabled_cnt++;
3715 }
3716 ram_block_discard_disable_mutex_unlock();
3717 return ret;
3718}
3719
3720int ram_block_uncoordinated_discard_disable(bool state)
3721{
3722 int ret = 0;
3723
3724 ram_block_discard_disable_mutex_lock();
3725 if (!state) {
3726 ram_block_uncoordinated_discard_disabled_cnt--;
3727 } else if (ram_block_discard_required_cnt) {
3728 ret = -EBUSY;
3729 } else {
3730 ram_block_uncoordinated_discard_disabled_cnt++;
David Hildenbrandd24f31d2020-06-26 09:22:29 +02003731 }
David Hildenbrand98da4912021-04-13 11:55:28 +02003732 ram_block_discard_disable_mutex_unlock();
3733 return ret;
David Hildenbrandd24f31d2020-06-26 09:22:29 +02003734}
3735
3736int ram_block_discard_require(bool state)
3737{
David Hildenbrand98da4912021-04-13 11:55:28 +02003738 int ret = 0;
David Hildenbrandd24f31d2020-06-26 09:22:29 +02003739
David Hildenbrand98da4912021-04-13 11:55:28 +02003740 ram_block_discard_disable_mutex_lock();
David Hildenbrandd24f31d2020-06-26 09:22:29 +02003741 if (!state) {
David Hildenbrand98da4912021-04-13 11:55:28 +02003742 ram_block_discard_required_cnt--;
David Hildenbrand7e6d32e2021-04-13 11:55:29 +02003743 } else if (ram_block_discard_disabled_cnt ||
3744 ram_block_uncoordinated_discard_disabled_cnt) {
David Hildenbrand98da4912021-04-13 11:55:28 +02003745 ret = -EBUSY;
David Hildenbrand7e6d32e2021-04-13 11:55:29 +02003746 } else {
3747 ram_block_discard_required_cnt++;
3748 }
3749 ram_block_discard_disable_mutex_unlock();
3750 return ret;
3751}
3752
3753int ram_block_coordinated_discard_require(bool state)
3754{
3755 int ret = 0;
3756
3757 ram_block_discard_disable_mutex_lock();
3758 if (!state) {
3759 ram_block_coordinated_discard_required_cnt--;
3760 } else if (ram_block_discard_disabled_cnt) {
3761 ret = -EBUSY;
3762 } else {
3763 ram_block_coordinated_discard_required_cnt++;
David Hildenbrandd24f31d2020-06-26 09:22:29 +02003764 }
David Hildenbrand98da4912021-04-13 11:55:28 +02003765 ram_block_discard_disable_mutex_unlock();
3766 return ret;
David Hildenbrandd24f31d2020-06-26 09:22:29 +02003767}
3768
3769bool ram_block_discard_is_disabled(void)
3770{
David Hildenbrand7e6d32e2021-04-13 11:55:29 +02003771 return qatomic_read(&ram_block_discard_disabled_cnt) ||
3772 qatomic_read(&ram_block_uncoordinated_discard_disabled_cnt);
David Hildenbrandd24f31d2020-06-26 09:22:29 +02003773}
3774
3775bool ram_block_discard_is_required(void)
3776{
David Hildenbrand7e6d32e2021-04-13 11:55:29 +02003777 return qatomic_read(&ram_block_discard_required_cnt) ||
3778 qatomic_read(&ram_block_coordinated_discard_required_cnt);
David Hildenbrandd24f31d2020-06-26 09:22:29 +02003779}