blob: f7883d22469b61572b35e6ea7bfad1f22220b4bf [file] [log] [blame]
bellard54936002003-05-13 00:25:15 +00001/*
Blue Swirl5b6dd862012-12-02 16:04:43 +00002 * Virtual page mapping
ths5fafdf22007-09-16 21:08:06 +00003 *
bellard54936002003-05-13 00:25:15 +00004 * Copyright (c) 2003 Fabrice Bellard
5 *
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2 of the License, or (at your option) any later version.
10 *
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
15 *
16 * You should have received a copy of the GNU Lesser General Public
Blue Swirl8167ee82009-07-16 20:47:01 +000017 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
bellard54936002003-05-13 00:25:15 +000018 */
bellard67b915a2004-03-31 23:37:16 +000019#include "config.h"
Stefan Weil777872e2014-02-23 18:02:08 +010020#ifndef _WIN32
bellarda98d49b2004-11-14 16:22:05 +000021#include <sys/types.h>
bellardd5a8f072004-09-29 21:15:28 +000022#include <sys/mman.h>
23#endif
bellard54936002003-05-13 00:25:15 +000024
Stefan Weil055403b2010-10-22 23:03:32 +020025#include "qemu-common.h"
bellard6180a182003-09-30 21:04:53 +000026#include "cpu.h"
bellardb67d9a52008-05-23 09:57:34 +000027#include "tcg.h"
pbrookb3c77242008-06-30 16:31:04 +000028#include "hw/hw.h"
Michael S. Tsirkin4485bd22015-03-11 07:56:34 +010029#if !defined(CONFIG_USER_ONLY)
Marcel Apfelbaum47c8ca52015-02-04 17:43:54 +020030#include "hw/boards.h"
Michael S. Tsirkin4485bd22015-03-11 07:56:34 +010031#endif
Alex Williamsoncc9e98c2010-06-25 11:09:43 -060032#include "hw/qdev.h"
Paolo Bonzini1de7afc2012-12-17 18:20:00 +010033#include "qemu/osdep.h"
Paolo Bonzini9c17d612012-12-17 18:20:04 +010034#include "sysemu/kvm.h"
Markus Armbruster2ff3de62013-07-04 15:09:22 +020035#include "sysemu/sysemu.h"
Paolo Bonzini0d09e412013-02-05 17:06:20 +010036#include "hw/xen/xen.h"
Paolo Bonzini1de7afc2012-12-17 18:20:00 +010037#include "qemu/timer.h"
38#include "qemu/config-file.h"
Andreas Färber75a34032013-09-02 16:57:02 +020039#include "qemu/error-report.h"
Paolo Bonzini022c62c2012-12-17 18:19:49 +010040#include "exec/memory.h"
Paolo Bonzini9c17d612012-12-17 18:20:04 +010041#include "sysemu/dma.h"
Paolo Bonzini022c62c2012-12-17 18:19:49 +010042#include "exec/address-spaces.h"
pbrook53a59602006-03-25 19:31:22 +000043#if defined(CONFIG_USER_ONLY)
44#include <qemu.h>
Jun Nakajima432d2682010-08-31 16:41:25 +010045#else /* !CONFIG_USER_ONLY */
Paolo Bonzini9c17d612012-12-17 18:20:04 +010046#include "sysemu/xen-mapcache.h"
Stefano Stabellini6506e4f2011-05-19 18:35:44 +010047#include "trace.h"
pbrook53a59602006-03-25 19:31:22 +000048#endif
Paolo Bonzini0d6d3c82012-11-14 15:45:02 +010049#include "exec/cpu-all.h"
Mike Day0dc3f442013-09-05 14:41:35 -040050#include "qemu/rcu_queue.h"
Paolo Bonzini022c62c2012-12-17 18:19:49 +010051#include "exec/cputlb.h"
Blue Swirl5b6dd862012-12-02 16:04:43 +000052#include "translate-all.h"
Blue Swirl0cac1b62012-04-09 16:50:52 +000053
Paolo Bonzini022c62c2012-12-17 18:19:49 +010054#include "exec/memory-internal.h"
Juan Quintela220c3eb2013-10-14 17:13:59 +020055#include "exec/ram_addr.h"
Avi Kivity67d95c12011-12-15 15:25:22 +020056
Michael S. Tsirkinb35ba302013-11-11 17:52:07 +020057#include "qemu/range.h"
58
blueswir1db7b5422007-05-26 17:36:03 +000059//#define DEBUG_SUBPAGE
ths1196be32007-03-17 15:17:58 +000060
pbrook99773bd2006-04-16 15:14:59 +000061#if !defined(CONFIG_USER_ONLY)
Mike Day0dc3f442013-09-05 14:41:35 -040062/* ram_list is read under rcu_read_lock()/rcu_read_unlock(). Writes
63 * are protected by the ramlist lock.
64 */
Mike Day0d53d9f2015-01-21 13:45:24 +010065RAMList ram_list = { .blocks = QLIST_HEAD_INITIALIZER(ram_list.blocks) };
Avi Kivity62152b82011-07-26 14:26:14 +030066
67static MemoryRegion *system_memory;
Avi Kivity309cb472011-08-08 16:09:03 +030068static MemoryRegion *system_io;
Avi Kivity62152b82011-07-26 14:26:14 +030069
Avi Kivityf6790af2012-10-02 20:13:51 +020070AddressSpace address_space_io;
71AddressSpace address_space_memory;
Avi Kivity2673a5d2012-10-02 18:49:28 +020072
Paolo Bonzini0844e002013-05-24 14:37:28 +020073MemoryRegion io_mem_rom, io_mem_notdirty;
Jan Kiszkaacc9d802013-05-26 21:55:37 +020074static MemoryRegion io_mem_unassigned;
Avi Kivity0e0df1e2012-01-02 00:32:15 +020075
Paolo Bonzini7bd4f432014-05-14 17:43:22 +080076/* RAM is pre-allocated and passed into qemu_ram_alloc_from_ptr */
77#define RAM_PREALLOC (1 << 0)
78
Paolo Bonzinidbcb8982014-06-10 19:15:24 +080079/* RAM is mmap-ed with MAP_SHARED */
80#define RAM_SHARED (1 << 1)
81
Michael S. Tsirkin62be4e32014-11-12 14:27:41 +020082/* Only a portion of RAM (used_length) is actually used, and migrated.
83 * This used_length size can change across reboots.
84 */
85#define RAM_RESIZEABLE (1 << 2)
86
pbrooke2eef172008-06-08 01:09:01 +000087#endif
bellard9fa3e852004-01-04 18:06:42 +000088
Andreas Färberbdc44642013-06-24 23:50:24 +020089struct CPUTailQ cpus = QTAILQ_HEAD_INITIALIZER(cpus);
bellard6a00d602005-11-21 23:25:50 +000090/* current CPU in the current thread. It is only valid inside
91 cpu_exec() */
Andreas Färber4917cf42013-05-27 05:17:50 +020092DEFINE_TLS(CPUState *, current_cpu);
pbrook2e70f6e2008-06-29 01:03:05 +000093/* 0 = Do not count executed instructions.
thsbf20dc02008-06-30 17:22:19 +000094 1 = Precise instruction counting.
pbrook2e70f6e2008-06-29 01:03:05 +000095 2 = Adaptive rate instruction counting. */
Paolo Bonzini5708fc62012-11-26 15:36:40 +010096int use_icount;
bellard6a00d602005-11-21 23:25:50 +000097
pbrooke2eef172008-06-08 01:09:01 +000098#if !defined(CONFIG_USER_ONLY)
Avi Kivity4346ae32012-02-10 17:00:01 +020099
Paolo Bonzini1db8abb2013-05-21 12:07:21 +0200100typedef struct PhysPageEntry PhysPageEntry;
101
102struct PhysPageEntry {
Michael S. Tsirkin9736e552013-11-11 14:42:43 +0200103 /* How many bits skip to next level (in units of L2_SIZE). 0 for a leaf. */
Michael S. Tsirkin8b795762013-11-11 14:51:56 +0200104 uint32_t skip : 6;
Michael S. Tsirkin9736e552013-11-11 14:42:43 +0200105 /* index into phys_sections (!skip) or phys_map_nodes (skip) */
Michael S. Tsirkin8b795762013-11-11 14:51:56 +0200106 uint32_t ptr : 26;
Paolo Bonzini1db8abb2013-05-21 12:07:21 +0200107};
108
Michael S. Tsirkin8b795762013-11-11 14:51:56 +0200109#define PHYS_MAP_NODE_NIL (((uint32_t)~0) >> 6)
110
Paolo Bonzini03f49952013-11-07 17:14:36 +0100111/* Size of the L2 (and L3, etc) page tables. */
Paolo Bonzini57271d62013-11-07 17:14:37 +0100112#define ADDR_SPACE_BITS 64
Paolo Bonzini03f49952013-11-07 17:14:36 +0100113
Michael S. Tsirkin026736c2013-11-13 20:13:03 +0200114#define P_L2_BITS 9
Paolo Bonzini03f49952013-11-07 17:14:36 +0100115#define P_L2_SIZE (1 << P_L2_BITS)
116
117#define P_L2_LEVELS (((ADDR_SPACE_BITS - TARGET_PAGE_BITS - 1) / P_L2_BITS) + 1)
118
119typedef PhysPageEntry Node[P_L2_SIZE];
Paolo Bonzini0475d942013-05-29 12:28:21 +0200120
Marcel Apfelbaum53cb28c2013-12-01 14:02:23 +0200121typedef struct PhysPageMap {
Paolo Bonzini79e2b9a2015-01-21 12:09:14 +0100122 struct rcu_head rcu;
123
Marcel Apfelbaum53cb28c2013-12-01 14:02:23 +0200124 unsigned sections_nb;
125 unsigned sections_nb_alloc;
126 unsigned nodes_nb;
127 unsigned nodes_nb_alloc;
128 Node *nodes;
129 MemoryRegionSection *sections;
130} PhysPageMap;
131
Paolo Bonzini1db8abb2013-05-21 12:07:21 +0200132struct AddressSpaceDispatch {
Paolo Bonzini79e2b9a2015-01-21 12:09:14 +0100133 struct rcu_head rcu;
134
Paolo Bonzini1db8abb2013-05-21 12:07:21 +0200135 /* This is a multi-level map on the physical address space.
136 * The bottom level has pointers to MemoryRegionSections.
137 */
138 PhysPageEntry phys_map;
Marcel Apfelbaum53cb28c2013-12-01 14:02:23 +0200139 PhysPageMap map;
Jan Kiszkaacc9d802013-05-26 21:55:37 +0200140 AddressSpace *as;
Paolo Bonzini1db8abb2013-05-21 12:07:21 +0200141};
142
Jan Kiszka90260c62013-05-26 21:46:51 +0200143#define SUBPAGE_IDX(addr) ((addr) & ~TARGET_PAGE_MASK)
144typedef struct subpage_t {
145 MemoryRegion iomem;
Jan Kiszkaacc9d802013-05-26 21:55:37 +0200146 AddressSpace *as;
Jan Kiszka90260c62013-05-26 21:46:51 +0200147 hwaddr base;
148 uint16_t sub_section[TARGET_PAGE_SIZE];
149} subpage_t;
150
Liu Ping Fanb41aac42013-05-29 11:09:17 +0200151#define PHYS_SECTION_UNASSIGNED 0
152#define PHYS_SECTION_NOTDIRTY 1
153#define PHYS_SECTION_ROM 2
154#define PHYS_SECTION_WATCH 3
Avi Kivity5312bd82012-02-12 18:32:55 +0200155
pbrooke2eef172008-06-08 01:09:01 +0000156static void io_mem_init(void);
Avi Kivity62152b82011-07-26 14:26:14 +0300157static void memory_map_init(void);
Edgar E. Iglesias09daed82013-12-17 13:06:51 +1000158static void tcg_commit(MemoryListener *listener);
pbrooke2eef172008-06-08 01:09:01 +0000159
Avi Kivity1ec9b902012-01-02 12:47:48 +0200160static MemoryRegion io_mem_watch;
pbrook6658ffb2007-03-16 23:58:11 +0000161#endif
bellard54936002003-05-13 00:25:15 +0000162
Paul Brook6d9a1302010-02-28 23:55:53 +0000163#if !defined(CONFIG_USER_ONLY)
Avi Kivityd6f2ea22012-02-12 20:12:49 +0200164
Marcel Apfelbaum53cb28c2013-12-01 14:02:23 +0200165static void phys_map_node_reserve(PhysPageMap *map, unsigned nodes)
Avi Kivityf7bf5462012-02-13 20:12:05 +0200166{
Marcel Apfelbaum53cb28c2013-12-01 14:02:23 +0200167 if (map->nodes_nb + nodes > map->nodes_nb_alloc) {
168 map->nodes_nb_alloc = MAX(map->nodes_nb_alloc * 2, 16);
169 map->nodes_nb_alloc = MAX(map->nodes_nb_alloc, map->nodes_nb + nodes);
170 map->nodes = g_renew(Node, map->nodes, map->nodes_nb_alloc);
Avi Kivityf7bf5462012-02-13 20:12:05 +0200171 }
172}
173
Paolo Bonzinidb946042015-05-21 15:12:29 +0200174static uint32_t phys_map_node_alloc(PhysPageMap *map, bool leaf)
Avi Kivityd6f2ea22012-02-12 20:12:49 +0200175{
176 unsigned i;
Michael S. Tsirkin8b795762013-11-11 14:51:56 +0200177 uint32_t ret;
Paolo Bonzinidb946042015-05-21 15:12:29 +0200178 PhysPageEntry e;
179 PhysPageEntry *p;
Avi Kivityd6f2ea22012-02-12 20:12:49 +0200180
Marcel Apfelbaum53cb28c2013-12-01 14:02:23 +0200181 ret = map->nodes_nb++;
Paolo Bonzinidb946042015-05-21 15:12:29 +0200182 p = map->nodes[ret];
Avi Kivityd6f2ea22012-02-12 20:12:49 +0200183 assert(ret != PHYS_MAP_NODE_NIL);
Marcel Apfelbaum53cb28c2013-12-01 14:02:23 +0200184 assert(ret != map->nodes_nb_alloc);
Paolo Bonzinidb946042015-05-21 15:12:29 +0200185
186 e.skip = leaf ? 0 : 1;
187 e.ptr = leaf ? PHYS_SECTION_UNASSIGNED : PHYS_MAP_NODE_NIL;
Paolo Bonzini03f49952013-11-07 17:14:36 +0100188 for (i = 0; i < P_L2_SIZE; ++i) {
Paolo Bonzinidb946042015-05-21 15:12:29 +0200189 memcpy(&p[i], &e, sizeof(e));
Avi Kivityd6f2ea22012-02-12 20:12:49 +0200190 }
Avi Kivityf7bf5462012-02-13 20:12:05 +0200191 return ret;
Avi Kivityd6f2ea22012-02-12 20:12:49 +0200192}
193
Marcel Apfelbaum53cb28c2013-12-01 14:02:23 +0200194static void phys_page_set_level(PhysPageMap *map, PhysPageEntry *lp,
195 hwaddr *index, hwaddr *nb, uint16_t leaf,
Avi Kivity29990972012-02-13 20:21:20 +0200196 int level)
Avi Kivityf7bf5462012-02-13 20:12:05 +0200197{
198 PhysPageEntry *p;
Paolo Bonzini03f49952013-11-07 17:14:36 +0100199 hwaddr step = (hwaddr)1 << (level * P_L2_BITS);
Avi Kivityf7bf5462012-02-13 20:12:05 +0200200
Michael S. Tsirkin9736e552013-11-11 14:42:43 +0200201 if (lp->skip && lp->ptr == PHYS_MAP_NODE_NIL) {
Paolo Bonzinidb946042015-05-21 15:12:29 +0200202 lp->ptr = phys_map_node_alloc(map, level == 0);
Avi Kivityf7bf5462012-02-13 20:12:05 +0200203 }
Paolo Bonzinidb946042015-05-21 15:12:29 +0200204 p = map->nodes[lp->ptr];
Paolo Bonzini03f49952013-11-07 17:14:36 +0100205 lp = &p[(*index >> (level * P_L2_BITS)) & (P_L2_SIZE - 1)];
Avi Kivityf7bf5462012-02-13 20:12:05 +0200206
Paolo Bonzini03f49952013-11-07 17:14:36 +0100207 while (*nb && lp < &p[P_L2_SIZE]) {
Avi Kivity07f07b32012-02-13 20:45:32 +0200208 if ((*index & (step - 1)) == 0 && *nb >= step) {
Michael S. Tsirkin9736e552013-11-11 14:42:43 +0200209 lp->skip = 0;
Avi Kivityc19e8802012-02-13 20:25:31 +0200210 lp->ptr = leaf;
Avi Kivity07f07b32012-02-13 20:45:32 +0200211 *index += step;
212 *nb -= step;
Avi Kivity29990972012-02-13 20:21:20 +0200213 } else {
Marcel Apfelbaum53cb28c2013-12-01 14:02:23 +0200214 phys_page_set_level(map, lp, index, nb, leaf, level - 1);
Avi Kivity29990972012-02-13 20:21:20 +0200215 }
216 ++lp;
Avi Kivityf7bf5462012-02-13 20:12:05 +0200217 }
218}
219
Avi Kivityac1970f2012-10-03 16:22:53 +0200220static void phys_page_set(AddressSpaceDispatch *d,
Avi Kivitya8170e52012-10-23 12:30:10 +0200221 hwaddr index, hwaddr nb,
Avi Kivity29990972012-02-13 20:21:20 +0200222 uint16_t leaf)
bellard92e873b2004-05-21 14:52:29 +0000223{
Avi Kivity29990972012-02-13 20:21:20 +0200224 /* Wildly overreserve - it doesn't matter much. */
Marcel Apfelbaum53cb28c2013-12-01 14:02:23 +0200225 phys_map_node_reserve(&d->map, 3 * P_L2_LEVELS);
bellard92e873b2004-05-21 14:52:29 +0000226
Marcel Apfelbaum53cb28c2013-12-01 14:02:23 +0200227 phys_page_set_level(&d->map, &d->phys_map, &index, &nb, leaf, P_L2_LEVELS - 1);
bellard92e873b2004-05-21 14:52:29 +0000228}
229
Michael S. Tsirkinb35ba302013-11-11 17:52:07 +0200230/* Compact a non leaf page entry. Simply detect that the entry has a single child,
231 * and update our entry so we can skip it and go directly to the destination.
232 */
233static void phys_page_compact(PhysPageEntry *lp, Node *nodes, unsigned long *compacted)
234{
235 unsigned valid_ptr = P_L2_SIZE;
236 int valid = 0;
237 PhysPageEntry *p;
238 int i;
239
240 if (lp->ptr == PHYS_MAP_NODE_NIL) {
241 return;
242 }
243
244 p = nodes[lp->ptr];
245 for (i = 0; i < P_L2_SIZE; i++) {
246 if (p[i].ptr == PHYS_MAP_NODE_NIL) {
247 continue;
248 }
249
250 valid_ptr = i;
251 valid++;
252 if (p[i].skip) {
253 phys_page_compact(&p[i], nodes, compacted);
254 }
255 }
256
257 /* We can only compress if there's only one child. */
258 if (valid != 1) {
259 return;
260 }
261
262 assert(valid_ptr < P_L2_SIZE);
263
264 /* Don't compress if it won't fit in the # of bits we have. */
265 if (lp->skip + p[valid_ptr].skip >= (1 << 3)) {
266 return;
267 }
268
269 lp->ptr = p[valid_ptr].ptr;
270 if (!p[valid_ptr].skip) {
271 /* If our only child is a leaf, make this a leaf. */
272 /* By design, we should have made this node a leaf to begin with so we
273 * should never reach here.
274 * But since it's so simple to handle this, let's do it just in case we
275 * change this rule.
276 */
277 lp->skip = 0;
278 } else {
279 lp->skip += p[valid_ptr].skip;
280 }
281}
282
283static void phys_page_compact_all(AddressSpaceDispatch *d, int nodes_nb)
284{
285 DECLARE_BITMAP(compacted, nodes_nb);
286
287 if (d->phys_map.skip) {
Marcel Apfelbaum53cb28c2013-12-01 14:02:23 +0200288 phys_page_compact(&d->phys_map, d->map.nodes, compacted);
Michael S. Tsirkinb35ba302013-11-11 17:52:07 +0200289 }
290}
291
Michael S. Tsirkin97115a82013-11-13 20:08:19 +0200292static MemoryRegionSection *phys_page_find(PhysPageEntry lp, hwaddr addr,
Paolo Bonzini9affd6f2013-05-29 12:09:47 +0200293 Node *nodes, MemoryRegionSection *sections)
bellard92e873b2004-05-21 14:52:29 +0000294{
Avi Kivity31ab2b42012-02-13 16:44:19 +0200295 PhysPageEntry *p;
Michael S. Tsirkin97115a82013-11-13 20:08:19 +0200296 hwaddr index = addr >> TARGET_PAGE_BITS;
Avi Kivity31ab2b42012-02-13 16:44:19 +0200297 int i;
Avi Kivityf1f6e3b2011-11-20 17:52:22 +0200298
Michael S. Tsirkin9736e552013-11-11 14:42:43 +0200299 for (i = P_L2_LEVELS; lp.skip && (i -= lp.skip) >= 0;) {
Avi Kivityc19e8802012-02-13 20:25:31 +0200300 if (lp.ptr == PHYS_MAP_NODE_NIL) {
Paolo Bonzini9affd6f2013-05-29 12:09:47 +0200301 return &sections[PHYS_SECTION_UNASSIGNED];
Avi Kivity31ab2b42012-02-13 16:44:19 +0200302 }
Paolo Bonzini9affd6f2013-05-29 12:09:47 +0200303 p = nodes[lp.ptr];
Paolo Bonzini03f49952013-11-07 17:14:36 +0100304 lp = p[(index >> (i * P_L2_BITS)) & (P_L2_SIZE - 1)];
Avi Kivityf1f6e3b2011-11-20 17:52:22 +0200305 }
Michael S. Tsirkinb35ba302013-11-11 17:52:07 +0200306
307 if (sections[lp.ptr].size.hi ||
308 range_covers_byte(sections[lp.ptr].offset_within_address_space,
309 sections[lp.ptr].size.lo, addr)) {
310 return &sections[lp.ptr];
311 } else {
312 return &sections[PHYS_SECTION_UNASSIGNED];
313 }
Avi Kivityf3705d52012-03-08 16:16:34 +0200314}
315
Blue Swirle5548612012-04-21 13:08:33 +0000316bool memory_region_is_unassigned(MemoryRegion *mr)
317{
Paolo Bonzini2a8e7492013-05-24 14:34:08 +0200318 return mr != &io_mem_rom && mr != &io_mem_notdirty && !mr->rom_device
Blue Swirle5548612012-04-21 13:08:33 +0000319 && mr != &io_mem_watch;
320}
Paolo Bonzini149f54b2013-05-24 12:59:37 +0200321
Paolo Bonzini79e2b9a2015-01-21 12:09:14 +0100322/* Called from RCU critical section */
Paolo Bonzinic7086b42013-06-02 15:27:39 +0200323static MemoryRegionSection *address_space_lookup_region(AddressSpaceDispatch *d,
Jan Kiszka90260c62013-05-26 21:46:51 +0200324 hwaddr addr,
325 bool resolve_subpage)
Jan Kiszka9f029602013-05-06 16:48:02 +0200326{
Jan Kiszka90260c62013-05-26 21:46:51 +0200327 MemoryRegionSection *section;
328 subpage_t *subpage;
329
Marcel Apfelbaum53cb28c2013-12-01 14:02:23 +0200330 section = phys_page_find(d->phys_map, addr, d->map.nodes, d->map.sections);
Jan Kiszka90260c62013-05-26 21:46:51 +0200331 if (resolve_subpage && section->mr->subpage) {
332 subpage = container_of(section->mr, subpage_t, iomem);
Marcel Apfelbaum53cb28c2013-12-01 14:02:23 +0200333 section = &d->map.sections[subpage->sub_section[SUBPAGE_IDX(addr)]];
Jan Kiszka90260c62013-05-26 21:46:51 +0200334 }
335 return section;
Jan Kiszka9f029602013-05-06 16:48:02 +0200336}
337
Paolo Bonzini79e2b9a2015-01-21 12:09:14 +0100338/* Called from RCU critical section */
Jan Kiszka90260c62013-05-26 21:46:51 +0200339static MemoryRegionSection *
Paolo Bonzinic7086b42013-06-02 15:27:39 +0200340address_space_translate_internal(AddressSpaceDispatch *d, hwaddr addr, hwaddr *xlat,
Jan Kiszka90260c62013-05-26 21:46:51 +0200341 hwaddr *plen, bool resolve_subpage)
Paolo Bonzini149f54b2013-05-24 12:59:37 +0200342{
343 MemoryRegionSection *section;
Paolo Bonzini965eb2f2015-06-17 10:40:27 +0200344 MemoryRegion *mr;
Paolo Bonzinia87f3952014-02-07 15:47:46 +0100345 Int128 diff;
Paolo Bonzini149f54b2013-05-24 12:59:37 +0200346
Paolo Bonzinic7086b42013-06-02 15:27:39 +0200347 section = address_space_lookup_region(d, addr, resolve_subpage);
Paolo Bonzini149f54b2013-05-24 12:59:37 +0200348 /* Compute offset within MemoryRegionSection */
349 addr -= section->offset_within_address_space;
350
351 /* Compute offset within MemoryRegion */
352 *xlat = addr + section->offset_within_region;
353
Paolo Bonzini965eb2f2015-06-17 10:40:27 +0200354 mr = section->mr;
355 if (memory_region_is_ram(mr)) {
Paolo Bonzinie4a511f2015-06-17 10:36:54 +0200356 diff = int128_sub(section->size, int128_make64(addr));
Paolo Bonzini965eb2f2015-06-17 10:40:27 +0200357 *plen = int128_get64(int128_min(diff, int128_make64(*plen)));
358 }
Paolo Bonzini149f54b2013-05-24 12:59:37 +0200359 return section;
360}
Jan Kiszka90260c62013-05-26 21:46:51 +0200361
Paolo Bonzinia87f3952014-02-07 15:47:46 +0100362static inline bool memory_access_is_direct(MemoryRegion *mr, bool is_write)
363{
364 if (memory_region_is_ram(mr)) {
365 return !(is_write && mr->readonly);
366 }
367 if (memory_region_is_romd(mr)) {
368 return !is_write;
369 }
370
371 return false;
372}
373
Paolo Bonzini41063e12015-03-18 14:21:43 +0100374/* Called from RCU critical section */
Paolo Bonzini5c8a00c2013-05-29 12:42:00 +0200375MemoryRegion *address_space_translate(AddressSpace *as, hwaddr addr,
376 hwaddr *xlat, hwaddr *plen,
377 bool is_write)
Jan Kiszka90260c62013-05-26 21:46:51 +0200378{
Avi Kivity30951152012-10-30 13:47:46 +0200379 IOMMUTLBEntry iotlb;
380 MemoryRegionSection *section;
381 MemoryRegion *mr;
Avi Kivity30951152012-10-30 13:47:46 +0200382
383 for (;;) {
Paolo Bonzini79e2b9a2015-01-21 12:09:14 +0100384 AddressSpaceDispatch *d = atomic_rcu_read(&as->dispatch);
385 section = address_space_translate_internal(d, addr, &addr, plen, true);
Avi Kivity30951152012-10-30 13:47:46 +0200386 mr = section->mr;
387
388 if (!mr->iommu_ops) {
389 break;
390 }
391
Le Tan8d7b8cb2014-08-16 13:55:37 +0800392 iotlb = mr->iommu_ops->translate(mr, addr, is_write);
Avi Kivity30951152012-10-30 13:47:46 +0200393 addr = ((iotlb.translated_addr & ~iotlb.addr_mask)
394 | (addr & iotlb.addr_mask));
Peter Crosthwaite23820db2015-03-16 22:35:54 -0700395 *plen = MIN(*plen, (addr | iotlb.addr_mask) - addr + 1);
Avi Kivity30951152012-10-30 13:47:46 +0200396 if (!(iotlb.perm & (1 << is_write))) {
397 mr = &io_mem_unassigned;
398 break;
399 }
400
401 as = iotlb.target_as;
402 }
403
Alexey Kardashevskiyfe680d02014-05-07 13:40:39 +0000404 if (xen_enabled() && memory_access_is_direct(mr, is_write)) {
Paolo Bonzinia87f3952014-02-07 15:47:46 +0100405 hwaddr page = ((addr & TARGET_PAGE_MASK) + TARGET_PAGE_SIZE) - addr;
Peter Crosthwaite23820db2015-03-16 22:35:54 -0700406 *plen = MIN(page, *plen);
Paolo Bonzinia87f3952014-02-07 15:47:46 +0100407 }
408
Avi Kivity30951152012-10-30 13:47:46 +0200409 *xlat = addr;
410 return mr;
Jan Kiszka90260c62013-05-26 21:46:51 +0200411}
412
Paolo Bonzini79e2b9a2015-01-21 12:09:14 +0100413/* Called from RCU critical section */
Jan Kiszka90260c62013-05-26 21:46:51 +0200414MemoryRegionSection *
Paolo Bonzini9d82b5a2013-08-16 08:26:30 +0200415address_space_translate_for_iotlb(CPUState *cpu, hwaddr addr,
416 hwaddr *xlat, hwaddr *plen)
Jan Kiszka90260c62013-05-26 21:46:51 +0200417{
Avi Kivity30951152012-10-30 13:47:46 +0200418 MemoryRegionSection *section;
Paolo Bonzini9d82b5a2013-08-16 08:26:30 +0200419 section = address_space_translate_internal(cpu->memory_dispatch,
420 addr, xlat, plen, false);
Avi Kivity30951152012-10-30 13:47:46 +0200421
422 assert(!section->mr->iommu_ops);
423 return section;
Jan Kiszka90260c62013-05-26 21:46:51 +0200424}
bellard9fa3e852004-01-04 18:06:42 +0000425#endif
bellardfd6ce8f2003-05-14 19:00:11 +0000426
Andreas Färberb170fce2013-01-20 20:23:22 +0100427#if !defined(CONFIG_USER_ONLY)
pbrook9656f322008-07-01 20:01:19 +0000428
Juan Quintelae59fb372009-09-29 22:48:21 +0200429static int cpu_common_post_load(void *opaque, int version_id)
Juan Quintelae7f4eff2009-09-10 03:04:33 +0200430{
Andreas Färber259186a2013-01-17 18:51:17 +0100431 CPUState *cpu = opaque;
Juan Quintelae7f4eff2009-09-10 03:04:33 +0200432
aurel323098dba2009-03-07 21:28:24 +0000433 /* 0x01 was CPU_INTERRUPT_EXIT. This line can be removed when the
434 version_id is increased. */
Andreas Färber259186a2013-01-17 18:51:17 +0100435 cpu->interrupt_request &= ~0x01;
Christian Borntraegerc01a71c2014-03-17 17:13:12 +0100436 tlb_flush(cpu, 1);
pbrook9656f322008-07-01 20:01:19 +0000437
438 return 0;
439}
Juan Quintelae7f4eff2009-09-10 03:04:33 +0200440
Pavel Dovgaluk6c3bff02014-07-31 09:41:17 +0400441static int cpu_common_pre_load(void *opaque)
442{
443 CPUState *cpu = opaque;
444
Paolo Bonziniadee6422014-12-19 12:53:14 +0100445 cpu->exception_index = -1;
Pavel Dovgaluk6c3bff02014-07-31 09:41:17 +0400446
447 return 0;
448}
449
450static bool cpu_common_exception_index_needed(void *opaque)
451{
452 CPUState *cpu = opaque;
453
Paolo Bonziniadee6422014-12-19 12:53:14 +0100454 return tcg_enabled() && cpu->exception_index != -1;
Pavel Dovgaluk6c3bff02014-07-31 09:41:17 +0400455}
456
457static const VMStateDescription vmstate_cpu_common_exception_index = {
458 .name = "cpu_common/exception_index",
459 .version_id = 1,
460 .minimum_version_id = 1,
Juan Quintela5cd8cad2014-09-23 14:09:54 +0200461 .needed = cpu_common_exception_index_needed,
Pavel Dovgaluk6c3bff02014-07-31 09:41:17 +0400462 .fields = (VMStateField[]) {
463 VMSTATE_INT32(exception_index, CPUState),
464 VMSTATE_END_OF_LIST()
465 }
466};
467
Andreas Färber1a1562f2013-06-17 04:09:11 +0200468const VMStateDescription vmstate_cpu_common = {
Juan Quintelae7f4eff2009-09-10 03:04:33 +0200469 .name = "cpu_common",
470 .version_id = 1,
471 .minimum_version_id = 1,
Pavel Dovgaluk6c3bff02014-07-31 09:41:17 +0400472 .pre_load = cpu_common_pre_load,
Juan Quintelae7f4eff2009-09-10 03:04:33 +0200473 .post_load = cpu_common_post_load,
Juan Quintela35d08452014-04-16 16:01:33 +0200474 .fields = (VMStateField[]) {
Andreas Färber259186a2013-01-17 18:51:17 +0100475 VMSTATE_UINT32(halted, CPUState),
476 VMSTATE_UINT32(interrupt_request, CPUState),
Juan Quintelae7f4eff2009-09-10 03:04:33 +0200477 VMSTATE_END_OF_LIST()
Pavel Dovgaluk6c3bff02014-07-31 09:41:17 +0400478 },
Juan Quintela5cd8cad2014-09-23 14:09:54 +0200479 .subsections = (const VMStateDescription*[]) {
480 &vmstate_cpu_common_exception_index,
481 NULL
Juan Quintelae7f4eff2009-09-10 03:04:33 +0200482 }
483};
Andreas Färber1a1562f2013-06-17 04:09:11 +0200484
pbrook9656f322008-07-01 20:01:19 +0000485#endif
486
Andreas Färber38d8f5c2012-12-17 19:47:15 +0100487CPUState *qemu_get_cpu(int index)
Glauber Costa950f1472009-06-09 12:15:18 -0400488{
Andreas Färberbdc44642013-06-24 23:50:24 +0200489 CPUState *cpu;
Glauber Costa950f1472009-06-09 12:15:18 -0400490
Andreas Färberbdc44642013-06-24 23:50:24 +0200491 CPU_FOREACH(cpu) {
Andreas Färber55e5c282012-12-17 06:18:02 +0100492 if (cpu->cpu_index == index) {
Andreas Färberbdc44642013-06-24 23:50:24 +0200493 return cpu;
Andreas Färber55e5c282012-12-17 06:18:02 +0100494 }
Glauber Costa950f1472009-06-09 12:15:18 -0400495 }
496
Andreas Färberbdc44642013-06-24 23:50:24 +0200497 return NULL;
Glauber Costa950f1472009-06-09 12:15:18 -0400498}
499
Edgar E. Iglesias09daed82013-12-17 13:06:51 +1000500#if !defined(CONFIG_USER_ONLY)
501void tcg_cpu_address_space_init(CPUState *cpu, AddressSpace *as)
502{
503 /* We only support one address space per cpu at the moment. */
504 assert(cpu->as == as);
505
506 if (cpu->tcg_as_listener) {
507 memory_listener_unregister(cpu->tcg_as_listener);
508 } else {
509 cpu->tcg_as_listener = g_new0(MemoryListener, 1);
510 }
511 cpu->tcg_as_listener->commit = tcg_commit;
512 memory_listener_register(cpu->tcg_as_listener, as);
513}
514#endif
515
Andreas Färber9349b4f2012-03-14 01:38:32 +0100516void cpu_exec_init(CPUArchState *env)
bellardfd6ce8f2003-05-14 19:00:11 +0000517{
Andreas Färber9f09e182012-05-03 06:59:07 +0200518 CPUState *cpu = ENV_GET_CPU(env);
Andreas Färberb170fce2013-01-20 20:23:22 +0100519 CPUClass *cc = CPU_GET_CLASS(cpu);
Andreas Färberbdc44642013-06-24 23:50:24 +0200520 CPUState *some_cpu;
bellard6a00d602005-11-21 23:25:50 +0000521 int cpu_index;
522
pbrookc2764712009-03-07 15:24:59 +0000523#if defined(CONFIG_USER_ONLY)
524 cpu_list_lock();
525#endif
bellard6a00d602005-11-21 23:25:50 +0000526 cpu_index = 0;
Andreas Färberbdc44642013-06-24 23:50:24 +0200527 CPU_FOREACH(some_cpu) {
bellard6a00d602005-11-21 23:25:50 +0000528 cpu_index++;
529 }
Andreas Färber55e5c282012-12-17 06:18:02 +0100530 cpu->cpu_index = cpu_index;
Andreas Färber1b1ed8d2012-12-17 04:22:03 +0100531 cpu->numa_node = 0;
Andreas Färberf0c3c502013-08-26 21:22:53 +0200532 QTAILQ_INIT(&cpu->breakpoints);
Andreas Färberff4700b2013-08-26 18:23:18 +0200533 QTAILQ_INIT(&cpu->watchpoints);
Jan Kiszkadc7a09c2011-03-15 12:26:31 +0100534#ifndef CONFIG_USER_ONLY
Edgar E. Iglesias09daed82013-12-17 13:06:51 +1000535 cpu->as = &address_space_memory;
Andreas Färber9f09e182012-05-03 06:59:07 +0200536 cpu->thread_id = qemu_get_thread_id();
Paolo Bonzinicba70542015-03-09 15:28:37 +0100537 cpu_reload_memory_map(cpu);
Jan Kiszkadc7a09c2011-03-15 12:26:31 +0100538#endif
Andreas Färberbdc44642013-06-24 23:50:24 +0200539 QTAILQ_INSERT_TAIL(&cpus, cpu, node);
pbrookc2764712009-03-07 15:24:59 +0000540#if defined(CONFIG_USER_ONLY)
541 cpu_list_unlock();
542#endif
Andreas Färbere0d47942013-07-29 04:07:50 +0200543 if (qdev_get_vmsd(DEVICE(cpu)) == NULL) {
544 vmstate_register(NULL, cpu_index, &vmstate_cpu_common, cpu);
545 }
pbrookb3c77242008-06-30 16:31:04 +0000546#if defined(CPU_SAVE_VERSION) && !defined(CONFIG_USER_ONLY)
Alex Williamson0be71e32010-06-25 11:09:07 -0600547 register_savevm(NULL, "cpu", cpu_index, CPU_SAVE_VERSION,
pbrookb3c77242008-06-30 16:31:04 +0000548 cpu_save, cpu_load, env);
Andreas Färberb170fce2013-01-20 20:23:22 +0100549 assert(cc->vmsd == NULL);
Andreas Färbere0d47942013-07-29 04:07:50 +0200550 assert(qdev_get_vmsd(DEVICE(cpu)) == NULL);
pbrookb3c77242008-06-30 16:31:04 +0000551#endif
Andreas Färberb170fce2013-01-20 20:23:22 +0100552 if (cc->vmsd != NULL) {
553 vmstate_register(NULL, cpu_index, cc->vmsd, cpu);
554 }
bellardfd6ce8f2003-05-14 19:00:11 +0000555}
556
Paul Brook94df27f2010-02-28 23:47:45 +0000557#if defined(CONFIG_USER_ONLY)
Andreas Färber00b941e2013-06-29 18:55:54 +0200558static void breakpoint_invalidate(CPUState *cpu, target_ulong pc)
Paul Brook94df27f2010-02-28 23:47:45 +0000559{
560 tb_invalidate_phys_page_range(pc, pc + 1, 0);
561}
562#else
Andreas Färber00b941e2013-06-29 18:55:54 +0200563static void breakpoint_invalidate(CPUState *cpu, target_ulong pc)
Max Filippov1e7855a2012-04-10 02:48:17 +0400564{
Max Filippove8262a12013-09-27 22:29:17 +0400565 hwaddr phys = cpu_get_phys_page_debug(cpu, pc);
566 if (phys != -1) {
Edgar E. Iglesias09daed82013-12-17 13:06:51 +1000567 tb_invalidate_phys_addr(cpu->as,
Edgar E. Iglesias29d8ec72013-11-07 19:43:10 +0100568 phys | (pc & ~TARGET_PAGE_MASK));
Max Filippove8262a12013-09-27 22:29:17 +0400569 }
Max Filippov1e7855a2012-04-10 02:48:17 +0400570}
bellardc27004e2005-01-03 23:35:10 +0000571#endif
bellardd720b932004-04-25 17:57:43 +0000572
Paul Brookc527ee82010-03-01 03:31:14 +0000573#if defined(CONFIG_USER_ONLY)
Andreas Färber75a34032013-09-02 16:57:02 +0200574void cpu_watchpoint_remove_all(CPUState *cpu, int mask)
Paul Brookc527ee82010-03-01 03:31:14 +0000575
576{
577}
578
Peter Maydell3ee887e2014-09-12 14:06:48 +0100579int cpu_watchpoint_remove(CPUState *cpu, vaddr addr, vaddr len,
580 int flags)
581{
582 return -ENOSYS;
583}
584
585void cpu_watchpoint_remove_by_ref(CPUState *cpu, CPUWatchpoint *watchpoint)
586{
587}
588
Andreas Färber75a34032013-09-02 16:57:02 +0200589int cpu_watchpoint_insert(CPUState *cpu, vaddr addr, vaddr len,
Paul Brookc527ee82010-03-01 03:31:14 +0000590 int flags, CPUWatchpoint **watchpoint)
591{
592 return -ENOSYS;
593}
594#else
pbrook6658ffb2007-03-16 23:58:11 +0000595/* Add a watchpoint. */
Andreas Färber75a34032013-09-02 16:57:02 +0200596int cpu_watchpoint_insert(CPUState *cpu, vaddr addr, vaddr len,
aliguoria1d1bb32008-11-18 20:07:32 +0000597 int flags, CPUWatchpoint **watchpoint)
pbrook6658ffb2007-03-16 23:58:11 +0000598{
aliguoric0ce9982008-11-25 22:13:57 +0000599 CPUWatchpoint *wp;
pbrook6658ffb2007-03-16 23:58:11 +0000600
Peter Maydell05068c02014-09-12 14:06:48 +0100601 /* forbid ranges which are empty or run off the end of the address space */
Max Filippov07e28632014-09-17 22:03:36 -0700602 if (len == 0 || (addr + len - 1) < addr) {
Andreas Färber75a34032013-09-02 16:57:02 +0200603 error_report("tried to set invalid watchpoint at %"
604 VADDR_PRIx ", len=%" VADDR_PRIu, addr, len);
aliguorib4051332008-11-18 20:14:20 +0000605 return -EINVAL;
606 }
Anthony Liguori7267c092011-08-20 22:09:37 -0500607 wp = g_malloc(sizeof(*wp));
pbrook6658ffb2007-03-16 23:58:11 +0000608
aliguoria1d1bb32008-11-18 20:07:32 +0000609 wp->vaddr = addr;
Peter Maydell05068c02014-09-12 14:06:48 +0100610 wp->len = len;
aliguoria1d1bb32008-11-18 20:07:32 +0000611 wp->flags = flags;
612
aliguori2dc9f412008-11-18 20:56:59 +0000613 /* keep all GDB-injected watchpoints in front */
Andreas Färberff4700b2013-08-26 18:23:18 +0200614 if (flags & BP_GDB) {
615 QTAILQ_INSERT_HEAD(&cpu->watchpoints, wp, entry);
616 } else {
617 QTAILQ_INSERT_TAIL(&cpu->watchpoints, wp, entry);
618 }
aliguoria1d1bb32008-11-18 20:07:32 +0000619
Andreas Färber31b030d2013-09-04 01:29:02 +0200620 tlb_flush_page(cpu, addr);
aliguoria1d1bb32008-11-18 20:07:32 +0000621
622 if (watchpoint)
623 *watchpoint = wp;
624 return 0;
pbrook6658ffb2007-03-16 23:58:11 +0000625}
626
aliguoria1d1bb32008-11-18 20:07:32 +0000627/* Remove a specific watchpoint. */
Andreas Färber75a34032013-09-02 16:57:02 +0200628int cpu_watchpoint_remove(CPUState *cpu, vaddr addr, vaddr len,
aliguoria1d1bb32008-11-18 20:07:32 +0000629 int flags)
pbrook6658ffb2007-03-16 23:58:11 +0000630{
aliguoria1d1bb32008-11-18 20:07:32 +0000631 CPUWatchpoint *wp;
pbrook6658ffb2007-03-16 23:58:11 +0000632
Andreas Färberff4700b2013-08-26 18:23:18 +0200633 QTAILQ_FOREACH(wp, &cpu->watchpoints, entry) {
Peter Maydell05068c02014-09-12 14:06:48 +0100634 if (addr == wp->vaddr && len == wp->len
aliguori6e140f22008-11-18 20:37:55 +0000635 && flags == (wp->flags & ~BP_WATCHPOINT_HIT)) {
Andreas Färber75a34032013-09-02 16:57:02 +0200636 cpu_watchpoint_remove_by_ref(cpu, wp);
pbrook6658ffb2007-03-16 23:58:11 +0000637 return 0;
638 }
639 }
aliguoria1d1bb32008-11-18 20:07:32 +0000640 return -ENOENT;
pbrook6658ffb2007-03-16 23:58:11 +0000641}
642
aliguoria1d1bb32008-11-18 20:07:32 +0000643/* Remove a specific watchpoint by reference. */
Andreas Färber75a34032013-09-02 16:57:02 +0200644void cpu_watchpoint_remove_by_ref(CPUState *cpu, CPUWatchpoint *watchpoint)
aliguoria1d1bb32008-11-18 20:07:32 +0000645{
Andreas Färberff4700b2013-08-26 18:23:18 +0200646 QTAILQ_REMOVE(&cpu->watchpoints, watchpoint, entry);
edgar_igl7d03f822008-05-17 18:58:29 +0000647
Andreas Färber31b030d2013-09-04 01:29:02 +0200648 tlb_flush_page(cpu, watchpoint->vaddr);
aliguoria1d1bb32008-11-18 20:07:32 +0000649
Anthony Liguori7267c092011-08-20 22:09:37 -0500650 g_free(watchpoint);
edgar_igl7d03f822008-05-17 18:58:29 +0000651}
652
aliguoria1d1bb32008-11-18 20:07:32 +0000653/* Remove all matching watchpoints. */
Andreas Färber75a34032013-09-02 16:57:02 +0200654void cpu_watchpoint_remove_all(CPUState *cpu, int mask)
aliguoria1d1bb32008-11-18 20:07:32 +0000655{
aliguoric0ce9982008-11-25 22:13:57 +0000656 CPUWatchpoint *wp, *next;
aliguoria1d1bb32008-11-18 20:07:32 +0000657
Andreas Färberff4700b2013-08-26 18:23:18 +0200658 QTAILQ_FOREACH_SAFE(wp, &cpu->watchpoints, entry, next) {
Andreas Färber75a34032013-09-02 16:57:02 +0200659 if (wp->flags & mask) {
660 cpu_watchpoint_remove_by_ref(cpu, wp);
661 }
aliguoric0ce9982008-11-25 22:13:57 +0000662 }
aliguoria1d1bb32008-11-18 20:07:32 +0000663}
Peter Maydell05068c02014-09-12 14:06:48 +0100664
665/* Return true if this watchpoint address matches the specified
666 * access (ie the address range covered by the watchpoint overlaps
667 * partially or completely with the address range covered by the
668 * access).
669 */
670static inline bool cpu_watchpoint_address_matches(CPUWatchpoint *wp,
671 vaddr addr,
672 vaddr len)
673{
674 /* We know the lengths are non-zero, but a little caution is
675 * required to avoid errors in the case where the range ends
676 * exactly at the top of the address space and so addr + len
677 * wraps round to zero.
678 */
679 vaddr wpend = wp->vaddr + wp->len - 1;
680 vaddr addrend = addr + len - 1;
681
682 return !(addr > wpend || wp->vaddr > addrend);
683}
684
Paul Brookc527ee82010-03-01 03:31:14 +0000685#endif
aliguoria1d1bb32008-11-18 20:07:32 +0000686
687/* Add a breakpoint. */
Andreas Färberb3310ab2013-09-02 17:26:20 +0200688int cpu_breakpoint_insert(CPUState *cpu, vaddr pc, int flags,
aliguoria1d1bb32008-11-18 20:07:32 +0000689 CPUBreakpoint **breakpoint)
bellard4c3a88a2003-07-26 12:06:08 +0000690{
aliguoric0ce9982008-11-25 22:13:57 +0000691 CPUBreakpoint *bp;
ths3b46e622007-09-17 08:09:54 +0000692
Anthony Liguori7267c092011-08-20 22:09:37 -0500693 bp = g_malloc(sizeof(*bp));
aliguoria1d1bb32008-11-18 20:07:32 +0000694
695 bp->pc = pc;
696 bp->flags = flags;
697
aliguori2dc9f412008-11-18 20:56:59 +0000698 /* keep all GDB-injected breakpoints in front */
Andreas Färber00b941e2013-06-29 18:55:54 +0200699 if (flags & BP_GDB) {
Andreas Färberf0c3c502013-08-26 21:22:53 +0200700 QTAILQ_INSERT_HEAD(&cpu->breakpoints, bp, entry);
Andreas Färber00b941e2013-06-29 18:55:54 +0200701 } else {
Andreas Färberf0c3c502013-08-26 21:22:53 +0200702 QTAILQ_INSERT_TAIL(&cpu->breakpoints, bp, entry);
Andreas Färber00b941e2013-06-29 18:55:54 +0200703 }
aliguoria1d1bb32008-11-18 20:07:32 +0000704
Andreas Färberf0c3c502013-08-26 21:22:53 +0200705 breakpoint_invalidate(cpu, pc);
aliguoria1d1bb32008-11-18 20:07:32 +0000706
Andreas Färber00b941e2013-06-29 18:55:54 +0200707 if (breakpoint) {
aliguoria1d1bb32008-11-18 20:07:32 +0000708 *breakpoint = bp;
Andreas Färber00b941e2013-06-29 18:55:54 +0200709 }
aliguoria1d1bb32008-11-18 20:07:32 +0000710 return 0;
aliguoria1d1bb32008-11-18 20:07:32 +0000711}
712
713/* Remove a specific breakpoint. */
Andreas Färberb3310ab2013-09-02 17:26:20 +0200714int cpu_breakpoint_remove(CPUState *cpu, vaddr pc, int flags)
aliguoria1d1bb32008-11-18 20:07:32 +0000715{
aliguoria1d1bb32008-11-18 20:07:32 +0000716 CPUBreakpoint *bp;
717
Andreas Färberf0c3c502013-08-26 21:22:53 +0200718 QTAILQ_FOREACH(bp, &cpu->breakpoints, entry) {
aliguoria1d1bb32008-11-18 20:07:32 +0000719 if (bp->pc == pc && bp->flags == flags) {
Andreas Färberb3310ab2013-09-02 17:26:20 +0200720 cpu_breakpoint_remove_by_ref(cpu, bp);
bellard4c3a88a2003-07-26 12:06:08 +0000721 return 0;
aliguoria1d1bb32008-11-18 20:07:32 +0000722 }
bellard4c3a88a2003-07-26 12:06:08 +0000723 }
aliguoria1d1bb32008-11-18 20:07:32 +0000724 return -ENOENT;
bellard4c3a88a2003-07-26 12:06:08 +0000725}
726
aliguoria1d1bb32008-11-18 20:07:32 +0000727/* Remove a specific breakpoint by reference. */
Andreas Färberb3310ab2013-09-02 17:26:20 +0200728void cpu_breakpoint_remove_by_ref(CPUState *cpu, CPUBreakpoint *breakpoint)
bellard4c3a88a2003-07-26 12:06:08 +0000729{
Andreas Färberf0c3c502013-08-26 21:22:53 +0200730 QTAILQ_REMOVE(&cpu->breakpoints, breakpoint, entry);
731
732 breakpoint_invalidate(cpu, breakpoint->pc);
aliguoria1d1bb32008-11-18 20:07:32 +0000733
Anthony Liguori7267c092011-08-20 22:09:37 -0500734 g_free(breakpoint);
aliguoria1d1bb32008-11-18 20:07:32 +0000735}
736
737/* Remove all matching breakpoints. */
Andreas Färberb3310ab2013-09-02 17:26:20 +0200738void cpu_breakpoint_remove_all(CPUState *cpu, int mask)
aliguoria1d1bb32008-11-18 20:07:32 +0000739{
aliguoric0ce9982008-11-25 22:13:57 +0000740 CPUBreakpoint *bp, *next;
aliguoria1d1bb32008-11-18 20:07:32 +0000741
Andreas Färberf0c3c502013-08-26 21:22:53 +0200742 QTAILQ_FOREACH_SAFE(bp, &cpu->breakpoints, entry, next) {
Andreas Färberb3310ab2013-09-02 17:26:20 +0200743 if (bp->flags & mask) {
744 cpu_breakpoint_remove_by_ref(cpu, bp);
745 }
aliguoric0ce9982008-11-25 22:13:57 +0000746 }
bellard4c3a88a2003-07-26 12:06:08 +0000747}
748
bellardc33a3462003-07-29 20:50:33 +0000749/* enable or disable single step mode. EXCP_DEBUG is returned by the
750 CPU loop after each instruction */
Andreas Färber3825b282013-06-24 18:41:06 +0200751void cpu_single_step(CPUState *cpu, int enabled)
bellardc33a3462003-07-29 20:50:33 +0000752{
Andreas Färbered2803d2013-06-21 20:20:45 +0200753 if (cpu->singlestep_enabled != enabled) {
754 cpu->singlestep_enabled = enabled;
755 if (kvm_enabled()) {
Stefan Weil38e478e2013-07-25 20:50:21 +0200756 kvm_update_guest_debug(cpu, 0);
Andreas Färbered2803d2013-06-21 20:20:45 +0200757 } else {
Stuart Bradyccbb4d42009-05-03 12:15:06 +0100758 /* must flush all the translated code to avoid inconsistencies */
aliguorie22a25c2009-03-12 20:12:48 +0000759 /* XXX: only flush what is necessary */
Stefan Weil38e478e2013-07-25 20:50:21 +0200760 CPUArchState *env = cpu->env_ptr;
aliguorie22a25c2009-03-12 20:12:48 +0000761 tb_flush(env);
762 }
bellardc33a3462003-07-29 20:50:33 +0000763 }
bellardc33a3462003-07-29 20:50:33 +0000764}
765
Andreas Färbera47dddd2013-09-03 17:38:47 +0200766void cpu_abort(CPUState *cpu, const char *fmt, ...)
bellard75012672003-06-21 13:11:07 +0000767{
768 va_list ap;
pbrook493ae1f2007-11-23 16:53:59 +0000769 va_list ap2;
bellard75012672003-06-21 13:11:07 +0000770
771 va_start(ap, fmt);
pbrook493ae1f2007-11-23 16:53:59 +0000772 va_copy(ap2, ap);
bellard75012672003-06-21 13:11:07 +0000773 fprintf(stderr, "qemu: fatal: ");
774 vfprintf(stderr, fmt, ap);
775 fprintf(stderr, "\n");
Andreas Färber878096e2013-05-27 01:33:50 +0200776 cpu_dump_state(cpu, stderr, fprintf, CPU_DUMP_FPU | CPU_DUMP_CCOP);
aliguori93fcfe32009-01-15 22:34:14 +0000777 if (qemu_log_enabled()) {
778 qemu_log("qemu: fatal: ");
779 qemu_log_vprintf(fmt, ap2);
780 qemu_log("\n");
Andreas Färbera0762852013-06-16 07:28:50 +0200781 log_cpu_state(cpu, CPU_DUMP_FPU | CPU_DUMP_CCOP);
aliguori31b1a7b2009-01-15 22:35:09 +0000782 qemu_log_flush();
aliguori93fcfe32009-01-15 22:34:14 +0000783 qemu_log_close();
balrog924edca2007-06-10 14:07:13 +0000784 }
pbrook493ae1f2007-11-23 16:53:59 +0000785 va_end(ap2);
j_mayerf9373292007-09-29 12:18:20 +0000786 va_end(ap);
Riku Voipiofd052bf2010-01-25 14:30:49 +0200787#if defined(CONFIG_USER_ONLY)
788 {
789 struct sigaction act;
790 sigfillset(&act.sa_mask);
791 act.sa_handler = SIG_DFL;
792 sigaction(SIGABRT, &act, NULL);
793 }
794#endif
bellard75012672003-06-21 13:11:07 +0000795 abort();
796}
797
bellard01243112004-01-04 15:48:17 +0000798#if !defined(CONFIG_USER_ONLY)
Mike Day0dc3f442013-09-05 14:41:35 -0400799/* Called from RCU critical section */
Paolo Bonzini041603f2013-09-09 17:49:45 +0200800static RAMBlock *qemu_get_ram_block(ram_addr_t addr)
801{
802 RAMBlock *block;
803
Paolo Bonzini43771532013-09-09 17:58:40 +0200804 block = atomic_rcu_read(&ram_list.mru_block);
Michael S. Tsirkin9b8424d2014-12-15 22:55:32 +0200805 if (block && addr - block->offset < block->max_length) {
Paolo Bonzini041603f2013-09-09 17:49:45 +0200806 goto found;
807 }
Mike Day0dc3f442013-09-05 14:41:35 -0400808 QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
Michael S. Tsirkin9b8424d2014-12-15 22:55:32 +0200809 if (addr - block->offset < block->max_length) {
Paolo Bonzini041603f2013-09-09 17:49:45 +0200810 goto found;
811 }
812 }
813
814 fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
815 abort();
816
817found:
Paolo Bonzini43771532013-09-09 17:58:40 +0200818 /* It is safe to write mru_block outside the iothread lock. This
819 * is what happens:
820 *
821 * mru_block = xxx
822 * rcu_read_unlock()
823 * xxx removed from list
824 * rcu_read_lock()
825 * read mru_block
826 * mru_block = NULL;
827 * call_rcu(reclaim_ramblock, xxx);
828 * rcu_read_unlock()
829 *
830 * atomic_rcu_set is not needed here. The block was already published
831 * when it was placed into the list. Here we're just making an extra
832 * copy of the pointer.
833 */
Paolo Bonzini041603f2013-09-09 17:49:45 +0200834 ram_list.mru_block = block;
835 return block;
836}
837
Juan Quintelaa2f4d5b2013-10-10 11:49:53 +0200838static void tlb_reset_dirty_range_all(ram_addr_t start, ram_addr_t length)
bellard1ccde1c2004-02-06 19:46:14 +0000839{
Paolo Bonzini041603f2013-09-09 17:49:45 +0200840 ram_addr_t start1;
Juan Quintelaa2f4d5b2013-10-10 11:49:53 +0200841 RAMBlock *block;
842 ram_addr_t end;
843
844 end = TARGET_PAGE_ALIGN(start + length);
845 start &= TARGET_PAGE_MASK;
bellardf23db162005-08-21 19:12:28 +0000846
Mike Day0dc3f442013-09-05 14:41:35 -0400847 rcu_read_lock();
Paolo Bonzini041603f2013-09-09 17:49:45 +0200848 block = qemu_get_ram_block(start);
849 assert(block == qemu_get_ram_block(end - 1));
Michael S. Tsirkin1240be22014-11-12 11:44:41 +0200850 start1 = (uintptr_t)ramblock_ptr(block, start - block->offset);
Blue Swirle5548612012-04-21 13:08:33 +0000851 cpu_tlb_reset_dirty_all(start1, length);
Mike Day0dc3f442013-09-05 14:41:35 -0400852 rcu_read_unlock();
Juan Quintelad24981d2012-05-22 00:42:40 +0200853}
854
855/* Note: start and end must be within the same ram block. */
Stefan Hajnoczi03eebc92014-12-02 11:23:18 +0000856bool cpu_physical_memory_test_and_clear_dirty(ram_addr_t start,
857 ram_addr_t length,
858 unsigned client)
Juan Quintelad24981d2012-05-22 00:42:40 +0200859{
Stefan Hajnoczi03eebc92014-12-02 11:23:18 +0000860 unsigned long end, page;
861 bool dirty;
Juan Quintelad24981d2012-05-22 00:42:40 +0200862
Stefan Hajnoczi03eebc92014-12-02 11:23:18 +0000863 if (length == 0) {
864 return false;
865 }
866
867 end = TARGET_PAGE_ALIGN(start + length) >> TARGET_PAGE_BITS;
868 page = start >> TARGET_PAGE_BITS;
869 dirty = bitmap_test_and_clear_atomic(ram_list.dirty_memory[client],
870 page, end - page);
871
872 if (dirty && tcg_enabled()) {
Juan Quintelaa2f4d5b2013-10-10 11:49:53 +0200873 tlb_reset_dirty_range_all(start, length);
Juan Quintelad24981d2012-05-22 00:42:40 +0200874 }
Stefan Hajnoczi03eebc92014-12-02 11:23:18 +0000875
876 return dirty;
bellard1ccde1c2004-02-06 19:46:14 +0000877}
878
Paolo Bonzini79e2b9a2015-01-21 12:09:14 +0100879/* Called from RCU critical section */
Andreas Färberbb0e6272013-09-03 13:32:01 +0200880hwaddr memory_region_section_get_iotlb(CPUState *cpu,
Paolo Bonzini149f54b2013-05-24 12:59:37 +0200881 MemoryRegionSection *section,
882 target_ulong vaddr,
883 hwaddr paddr, hwaddr xlat,
884 int prot,
885 target_ulong *address)
Blue Swirle5548612012-04-21 13:08:33 +0000886{
Avi Kivitya8170e52012-10-23 12:30:10 +0200887 hwaddr iotlb;
Blue Swirle5548612012-04-21 13:08:33 +0000888 CPUWatchpoint *wp;
889
Blue Swirlcc5bea62012-04-14 14:56:48 +0000890 if (memory_region_is_ram(section->mr)) {
Blue Swirle5548612012-04-21 13:08:33 +0000891 /* Normal RAM. */
892 iotlb = (memory_region_get_ram_addr(section->mr) & TARGET_PAGE_MASK)
Paolo Bonzini149f54b2013-05-24 12:59:37 +0200893 + xlat;
Blue Swirle5548612012-04-21 13:08:33 +0000894 if (!section->readonly) {
Liu Ping Fanb41aac42013-05-29 11:09:17 +0200895 iotlb |= PHYS_SECTION_NOTDIRTY;
Blue Swirle5548612012-04-21 13:08:33 +0000896 } else {
Liu Ping Fanb41aac42013-05-29 11:09:17 +0200897 iotlb |= PHYS_SECTION_ROM;
Blue Swirle5548612012-04-21 13:08:33 +0000898 }
899 } else {
Edgar E. Iglesias1b3fb982013-11-07 18:43:28 +0100900 iotlb = section - section->address_space->dispatch->map.sections;
Paolo Bonzini149f54b2013-05-24 12:59:37 +0200901 iotlb += xlat;
Blue Swirle5548612012-04-21 13:08:33 +0000902 }
903
904 /* Make accesses to pages with watchpoints go via the
905 watchpoint trap routines. */
Andreas Färberff4700b2013-08-26 18:23:18 +0200906 QTAILQ_FOREACH(wp, &cpu->watchpoints, entry) {
Peter Maydell05068c02014-09-12 14:06:48 +0100907 if (cpu_watchpoint_address_matches(wp, vaddr, TARGET_PAGE_SIZE)) {
Blue Swirle5548612012-04-21 13:08:33 +0000908 /* Avoid trapping reads of pages with a write breakpoint. */
909 if ((prot & PAGE_WRITE) || (wp->flags & BP_MEM_READ)) {
Liu Ping Fanb41aac42013-05-29 11:09:17 +0200910 iotlb = PHYS_SECTION_WATCH + paddr;
Blue Swirle5548612012-04-21 13:08:33 +0000911 *address |= TLB_MMIO;
912 break;
913 }
914 }
915 }
916
917 return iotlb;
918}
bellard9fa3e852004-01-04 18:06:42 +0000919#endif /* defined(CONFIG_USER_ONLY) */
920
pbrooke2eef172008-06-08 01:09:01 +0000921#if !defined(CONFIG_USER_ONLY)
pbrook8da3ff12008-12-01 18:59:50 +0000922
Anthony Liguoric227f092009-10-01 16:12:16 -0500923static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
Avi Kivity5312bd82012-02-12 18:32:55 +0200924 uint16_t section);
Jan Kiszkaacc9d802013-05-26 21:55:37 +0200925static subpage_t *subpage_init(AddressSpace *as, hwaddr base);
Avi Kivity54688b12012-02-09 17:34:32 +0200926
Igor Mammedova2b257d2014-10-31 16:38:37 +0000927static void *(*phys_mem_alloc)(size_t size, uint64_t *align) =
928 qemu_anon_ram_alloc;
Markus Armbruster91138032013-07-31 15:11:08 +0200929
930/*
931 * Set a custom physical guest memory alloator.
932 * Accelerators with unusual needs may need this. Hopefully, we can
933 * get rid of it eventually.
934 */
Igor Mammedova2b257d2014-10-31 16:38:37 +0000935void phys_mem_set_alloc(void *(*alloc)(size_t, uint64_t *align))
Markus Armbruster91138032013-07-31 15:11:08 +0200936{
937 phys_mem_alloc = alloc;
938}
939
Marcel Apfelbaum53cb28c2013-12-01 14:02:23 +0200940static uint16_t phys_section_add(PhysPageMap *map,
941 MemoryRegionSection *section)
Avi Kivity5312bd82012-02-12 18:32:55 +0200942{
Paolo Bonzini68f3f652013-05-07 11:30:23 +0200943 /* The physical section number is ORed with a page-aligned
944 * pointer to produce the iotlb entries. Thus it should
945 * never overflow into the page-aligned value.
946 */
Marcel Apfelbaum53cb28c2013-12-01 14:02:23 +0200947 assert(map->sections_nb < TARGET_PAGE_SIZE);
Paolo Bonzini68f3f652013-05-07 11:30:23 +0200948
Marcel Apfelbaum53cb28c2013-12-01 14:02:23 +0200949 if (map->sections_nb == map->sections_nb_alloc) {
950 map->sections_nb_alloc = MAX(map->sections_nb_alloc * 2, 16);
951 map->sections = g_renew(MemoryRegionSection, map->sections,
952 map->sections_nb_alloc);
Avi Kivity5312bd82012-02-12 18:32:55 +0200953 }
Marcel Apfelbaum53cb28c2013-12-01 14:02:23 +0200954 map->sections[map->sections_nb] = *section;
Paolo Bonzinidfde4e62013-05-06 10:46:11 +0200955 memory_region_ref(section->mr);
Marcel Apfelbaum53cb28c2013-12-01 14:02:23 +0200956 return map->sections_nb++;
Avi Kivity5312bd82012-02-12 18:32:55 +0200957}
958
Paolo Bonzini058bc4b2013-06-25 09:30:48 +0200959static void phys_section_destroy(MemoryRegion *mr)
960{
Paolo Bonzinidfde4e62013-05-06 10:46:11 +0200961 memory_region_unref(mr);
962
Paolo Bonzini058bc4b2013-06-25 09:30:48 +0200963 if (mr->subpage) {
964 subpage_t *subpage = container_of(mr, subpage_t, iomem);
Peter Crosthwaiteb4fefef2014-06-05 23:15:52 -0700965 object_unref(OBJECT(&subpage->iomem));
Paolo Bonzini058bc4b2013-06-25 09:30:48 +0200966 g_free(subpage);
967 }
968}
969
Paolo Bonzini60926662013-05-29 12:30:26 +0200970static void phys_sections_free(PhysPageMap *map)
Avi Kivity5312bd82012-02-12 18:32:55 +0200971{
Paolo Bonzini9affd6f2013-05-29 12:09:47 +0200972 while (map->sections_nb > 0) {
973 MemoryRegionSection *section = &map->sections[--map->sections_nb];
Paolo Bonzini058bc4b2013-06-25 09:30:48 +0200974 phys_section_destroy(section->mr);
975 }
Paolo Bonzini9affd6f2013-05-29 12:09:47 +0200976 g_free(map->sections);
977 g_free(map->nodes);
Avi Kivity5312bd82012-02-12 18:32:55 +0200978}
979
Avi Kivityac1970f2012-10-03 16:22:53 +0200980static void register_subpage(AddressSpaceDispatch *d, MemoryRegionSection *section)
Avi Kivity0f0cb162012-02-13 17:14:32 +0200981{
982 subpage_t *subpage;
Avi Kivitya8170e52012-10-23 12:30:10 +0200983 hwaddr base = section->offset_within_address_space
Avi Kivity0f0cb162012-02-13 17:14:32 +0200984 & TARGET_PAGE_MASK;
Michael S. Tsirkin97115a82013-11-13 20:08:19 +0200985 MemoryRegionSection *existing = phys_page_find(d->phys_map, base,
Marcel Apfelbaum53cb28c2013-12-01 14:02:23 +0200986 d->map.nodes, d->map.sections);
Avi Kivity0f0cb162012-02-13 17:14:32 +0200987 MemoryRegionSection subsection = {
988 .offset_within_address_space = base,
Paolo Bonzini052e87b2013-05-27 10:08:27 +0200989 .size = int128_make64(TARGET_PAGE_SIZE),
Avi Kivity0f0cb162012-02-13 17:14:32 +0200990 };
Avi Kivitya8170e52012-10-23 12:30:10 +0200991 hwaddr start, end;
Avi Kivity0f0cb162012-02-13 17:14:32 +0200992
Avi Kivityf3705d52012-03-08 16:16:34 +0200993 assert(existing->mr->subpage || existing->mr == &io_mem_unassigned);
Avi Kivity0f0cb162012-02-13 17:14:32 +0200994
Avi Kivityf3705d52012-03-08 16:16:34 +0200995 if (!(existing->mr->subpage)) {
Jan Kiszkaacc9d802013-05-26 21:55:37 +0200996 subpage = subpage_init(d->as, base);
Edgar E. Iglesias3be91e82013-11-07 18:42:51 +0100997 subsection.address_space = d->as;
Avi Kivity0f0cb162012-02-13 17:14:32 +0200998 subsection.mr = &subpage->iomem;
Avi Kivityac1970f2012-10-03 16:22:53 +0200999 phys_page_set(d, base >> TARGET_PAGE_BITS, 1,
Marcel Apfelbaum53cb28c2013-12-01 14:02:23 +02001000 phys_section_add(&d->map, &subsection));
Avi Kivity0f0cb162012-02-13 17:14:32 +02001001 } else {
Avi Kivityf3705d52012-03-08 16:16:34 +02001002 subpage = container_of(existing->mr, subpage_t, iomem);
Avi Kivity0f0cb162012-02-13 17:14:32 +02001003 }
1004 start = section->offset_within_address_space & ~TARGET_PAGE_MASK;
Paolo Bonzini052e87b2013-05-27 10:08:27 +02001005 end = start + int128_get64(section->size) - 1;
Marcel Apfelbaum53cb28c2013-12-01 14:02:23 +02001006 subpage_register(subpage, start, end,
1007 phys_section_add(&d->map, section));
Avi Kivity0f0cb162012-02-13 17:14:32 +02001008}
1009
1010
Paolo Bonzini052e87b2013-05-27 10:08:27 +02001011static void register_multipage(AddressSpaceDispatch *d,
1012 MemoryRegionSection *section)
bellard33417e72003-08-10 21:47:01 +00001013{
Avi Kivitya8170e52012-10-23 12:30:10 +02001014 hwaddr start_addr = section->offset_within_address_space;
Marcel Apfelbaum53cb28c2013-12-01 14:02:23 +02001015 uint16_t section_index = phys_section_add(&d->map, section);
Paolo Bonzini052e87b2013-05-27 10:08:27 +02001016 uint64_t num_pages = int128_get64(int128_rshift(section->size,
1017 TARGET_PAGE_BITS));
Avi Kivitydd811242012-01-02 12:17:03 +02001018
Paolo Bonzini733d5ef2013-05-27 10:47:10 +02001019 assert(num_pages);
1020 phys_page_set(d, start_addr >> TARGET_PAGE_BITS, num_pages, section_index);
bellard33417e72003-08-10 21:47:01 +00001021}
1022
Avi Kivityac1970f2012-10-03 16:22:53 +02001023static void mem_add(MemoryListener *listener, MemoryRegionSection *section)
Avi Kivity0f0cb162012-02-13 17:14:32 +02001024{
Paolo Bonzini89ae3372013-06-02 10:39:07 +02001025 AddressSpace *as = container_of(listener, AddressSpace, dispatch_listener);
Paolo Bonzini00752702013-05-29 12:13:54 +02001026 AddressSpaceDispatch *d = as->next_dispatch;
Paolo Bonzini99b9cc02013-05-27 13:18:01 +02001027 MemoryRegionSection now = *section, remain = *section;
Paolo Bonzini052e87b2013-05-27 10:08:27 +02001028 Int128 page_size = int128_make64(TARGET_PAGE_SIZE);
Avi Kivity0f0cb162012-02-13 17:14:32 +02001029
Paolo Bonzini733d5ef2013-05-27 10:47:10 +02001030 if (now.offset_within_address_space & ~TARGET_PAGE_MASK) {
1031 uint64_t left = TARGET_PAGE_ALIGN(now.offset_within_address_space)
1032 - now.offset_within_address_space;
1033
Paolo Bonzini052e87b2013-05-27 10:08:27 +02001034 now.size = int128_min(int128_make64(left), now.size);
Avi Kivityac1970f2012-10-03 16:22:53 +02001035 register_subpage(d, &now);
Paolo Bonzini733d5ef2013-05-27 10:47:10 +02001036 } else {
Paolo Bonzini052e87b2013-05-27 10:08:27 +02001037 now.size = int128_zero();
Paolo Bonzini733d5ef2013-05-27 10:47:10 +02001038 }
Paolo Bonzini052e87b2013-05-27 10:08:27 +02001039 while (int128_ne(remain.size, now.size)) {
1040 remain.size = int128_sub(remain.size, now.size);
1041 remain.offset_within_address_space += int128_get64(now.size);
1042 remain.offset_within_region += int128_get64(now.size);
Tyler Hall69b67642012-07-25 18:45:04 -04001043 now = remain;
Paolo Bonzini052e87b2013-05-27 10:08:27 +02001044 if (int128_lt(remain.size, page_size)) {
Paolo Bonzini733d5ef2013-05-27 10:47:10 +02001045 register_subpage(d, &now);
Hu Tao88266242013-08-29 18:21:16 +08001046 } else if (remain.offset_within_address_space & ~TARGET_PAGE_MASK) {
Paolo Bonzini052e87b2013-05-27 10:08:27 +02001047 now.size = page_size;
Avi Kivityac1970f2012-10-03 16:22:53 +02001048 register_subpage(d, &now);
Tyler Hall69b67642012-07-25 18:45:04 -04001049 } else {
Paolo Bonzini052e87b2013-05-27 10:08:27 +02001050 now.size = int128_and(now.size, int128_neg(page_size));
Avi Kivityac1970f2012-10-03 16:22:53 +02001051 register_multipage(d, &now);
Tyler Hall69b67642012-07-25 18:45:04 -04001052 }
Avi Kivity0f0cb162012-02-13 17:14:32 +02001053 }
1054}
1055
Sheng Yang62a27442010-01-26 19:21:16 +08001056void qemu_flush_coalesced_mmio_buffer(void)
1057{
1058 if (kvm_enabled())
1059 kvm_flush_coalesced_mmio_buffer();
1060}
1061
Umesh Deshpandeb2a86582011-08-17 00:01:33 -07001062void qemu_mutex_lock_ramlist(void)
1063{
1064 qemu_mutex_lock(&ram_list.mutex);
1065}
1066
1067void qemu_mutex_unlock_ramlist(void)
1068{
1069 qemu_mutex_unlock(&ram_list.mutex);
1070}
1071
Markus Armbrustere1e84ba2013-07-31 15:11:10 +02001072#ifdef __linux__
Marcelo Tosattic9027602010-03-01 20:25:08 -03001073
1074#include <sys/vfs.h>
1075
1076#define HUGETLBFS_MAGIC 0x958458f6
1077
Hu Taofc7a5802014-09-09 13:28:01 +08001078static long gethugepagesize(const char *path, Error **errp)
Marcelo Tosattic9027602010-03-01 20:25:08 -03001079{
1080 struct statfs fs;
1081 int ret;
1082
1083 do {
Yoshiaki Tamura9742bf22010-08-18 13:30:13 +09001084 ret = statfs(path, &fs);
Marcelo Tosattic9027602010-03-01 20:25:08 -03001085 } while (ret != 0 && errno == EINTR);
1086
1087 if (ret != 0) {
Hu Taofc7a5802014-09-09 13:28:01 +08001088 error_setg_errno(errp, errno, "failed to get page size of file %s",
1089 path);
Yoshiaki Tamura9742bf22010-08-18 13:30:13 +09001090 return 0;
Marcelo Tosattic9027602010-03-01 20:25:08 -03001091 }
1092
1093 if (fs.f_type != HUGETLBFS_MAGIC)
Yoshiaki Tamura9742bf22010-08-18 13:30:13 +09001094 fprintf(stderr, "Warning: path not on HugeTLBFS: %s\n", path);
Marcelo Tosattic9027602010-03-01 20:25:08 -03001095
1096 return fs.f_bsize;
1097}
1098
Alex Williamson04b16652010-07-02 11:13:17 -06001099static void *file_ram_alloc(RAMBlock *block,
1100 ram_addr_t memory,
Paolo Bonzini7f56e742014-05-14 17:43:20 +08001101 const char *path,
1102 Error **errp)
Marcelo Tosattic9027602010-03-01 20:25:08 -03001103{
1104 char *filename;
Peter Feiner8ca761f2013-03-04 13:54:25 -05001105 char *sanitized_name;
1106 char *c;
Hu Tao557529d2014-09-09 13:28:00 +08001107 void *area = NULL;
Marcelo Tosattic9027602010-03-01 20:25:08 -03001108 int fd;
Hu Tao557529d2014-09-09 13:28:00 +08001109 uint64_t hpagesize;
Hu Taofc7a5802014-09-09 13:28:01 +08001110 Error *local_err = NULL;
Marcelo Tosattic9027602010-03-01 20:25:08 -03001111
Hu Taofc7a5802014-09-09 13:28:01 +08001112 hpagesize = gethugepagesize(path, &local_err);
1113 if (local_err) {
1114 error_propagate(errp, local_err);
Marcelo Tosattif9a49df2014-02-04 13:41:53 -05001115 goto error;
Marcelo Tosattic9027602010-03-01 20:25:08 -03001116 }
Igor Mammedova2b257d2014-10-31 16:38:37 +00001117 block->mr->align = hpagesize;
Marcelo Tosattic9027602010-03-01 20:25:08 -03001118
1119 if (memory < hpagesize) {
Hu Tao557529d2014-09-09 13:28:00 +08001120 error_setg(errp, "memory size 0x" RAM_ADDR_FMT " must be equal to "
1121 "or larger than huge page size 0x%" PRIx64,
1122 memory, hpagesize);
1123 goto error;
Marcelo Tosattic9027602010-03-01 20:25:08 -03001124 }
1125
1126 if (kvm_enabled() && !kvm_has_sync_mmu()) {
Paolo Bonzini7f56e742014-05-14 17:43:20 +08001127 error_setg(errp,
1128 "host lacks kvm mmu notifiers, -mem-path unsupported");
Marcelo Tosattif9a49df2014-02-04 13:41:53 -05001129 goto error;
Marcelo Tosattic9027602010-03-01 20:25:08 -03001130 }
1131
Peter Feiner8ca761f2013-03-04 13:54:25 -05001132 /* Make name safe to use with mkstemp by replacing '/' with '_'. */
Peter Crosthwaite83234bf2014-08-14 23:54:29 -07001133 sanitized_name = g_strdup(memory_region_name(block->mr));
Peter Feiner8ca761f2013-03-04 13:54:25 -05001134 for (c = sanitized_name; *c != '\0'; c++) {
1135 if (*c == '/')
1136 *c = '_';
1137 }
1138
1139 filename = g_strdup_printf("%s/qemu_back_mem.%s.XXXXXX", path,
1140 sanitized_name);
1141 g_free(sanitized_name);
Marcelo Tosattic9027602010-03-01 20:25:08 -03001142
1143 fd = mkstemp(filename);
1144 if (fd < 0) {
Paolo Bonzini7f56e742014-05-14 17:43:20 +08001145 error_setg_errno(errp, errno,
1146 "unable to create backing store for hugepages");
Stefan Weile4ada482013-01-16 18:37:23 +01001147 g_free(filename);
Marcelo Tosattif9a49df2014-02-04 13:41:53 -05001148 goto error;
Marcelo Tosattic9027602010-03-01 20:25:08 -03001149 }
1150 unlink(filename);
Stefan Weile4ada482013-01-16 18:37:23 +01001151 g_free(filename);
Marcelo Tosattic9027602010-03-01 20:25:08 -03001152
1153 memory = (memory+hpagesize-1) & ~(hpagesize-1);
1154
1155 /*
1156 * ftruncate is not supported by hugetlbfs in older
1157 * hosts, so don't bother bailing out on errors.
1158 * If anything goes wrong with it under other filesystems,
1159 * mmap will fail.
1160 */
Paolo Bonzini7f56e742014-05-14 17:43:20 +08001161 if (ftruncate(fd, memory)) {
Yoshiaki Tamura9742bf22010-08-18 13:30:13 +09001162 perror("ftruncate");
Paolo Bonzini7f56e742014-05-14 17:43:20 +08001163 }
Marcelo Tosattic9027602010-03-01 20:25:08 -03001164
Paolo Bonzinidbcb8982014-06-10 19:15:24 +08001165 area = mmap(0, memory, PROT_READ | PROT_WRITE,
1166 (block->flags & RAM_SHARED ? MAP_SHARED : MAP_PRIVATE),
1167 fd, 0);
Marcelo Tosattic9027602010-03-01 20:25:08 -03001168 if (area == MAP_FAILED) {
Paolo Bonzini7f56e742014-05-14 17:43:20 +08001169 error_setg_errno(errp, errno,
1170 "unable to map backing store for hugepages");
Yoshiaki Tamura9742bf22010-08-18 13:30:13 +09001171 close(fd);
Marcelo Tosattif9a49df2014-02-04 13:41:53 -05001172 goto error;
Marcelo Tosattic9027602010-03-01 20:25:08 -03001173 }
Marcelo Tosattief36fa12013-10-28 18:51:46 -02001174
1175 if (mem_prealloc) {
Paolo Bonzini38183312014-05-14 17:43:21 +08001176 os_mem_prealloc(fd, area, memory);
Marcelo Tosattief36fa12013-10-28 18:51:46 -02001177 }
1178
Alex Williamson04b16652010-07-02 11:13:17 -06001179 block->fd = fd;
Marcelo Tosattic9027602010-03-01 20:25:08 -03001180 return area;
Marcelo Tosattif9a49df2014-02-04 13:41:53 -05001181
1182error:
1183 if (mem_prealloc) {
Gonglei81b07352015-02-25 12:22:31 +08001184 error_report("%s", error_get_pretty(*errp));
Marcelo Tosattif9a49df2014-02-04 13:41:53 -05001185 exit(1);
1186 }
1187 return NULL;
Marcelo Tosattic9027602010-03-01 20:25:08 -03001188}
1189#endif
1190
Mike Day0dc3f442013-09-05 14:41:35 -04001191/* Called with the ramlist lock held. */
Alex Williamsond17b5282010-06-25 11:08:38 -06001192static ram_addr_t find_ram_offset(ram_addr_t size)
1193{
Alex Williamson04b16652010-07-02 11:13:17 -06001194 RAMBlock *block, *next_block;
Alex Williamson3e837b22011-10-31 08:54:09 -06001195 ram_addr_t offset = RAM_ADDR_MAX, mingap = RAM_ADDR_MAX;
Alex Williamson04b16652010-07-02 11:13:17 -06001196
Stefan Hajnoczi49cd9ac2013-03-11 10:20:21 +01001197 assert(size != 0); /* it would hand out same offset multiple times */
1198
Mike Day0dc3f442013-09-05 14:41:35 -04001199 if (QLIST_EMPTY_RCU(&ram_list.blocks)) {
Alex Williamson04b16652010-07-02 11:13:17 -06001200 return 0;
Mike Day0d53d9f2015-01-21 13:45:24 +01001201 }
Alex Williamson04b16652010-07-02 11:13:17 -06001202
Mike Day0dc3f442013-09-05 14:41:35 -04001203 QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
Anthony PERARDf15fbc42011-07-20 08:17:42 +00001204 ram_addr_t end, next = RAM_ADDR_MAX;
Alex Williamson04b16652010-07-02 11:13:17 -06001205
Michael S. Tsirkin62be4e32014-11-12 14:27:41 +02001206 end = block->offset + block->max_length;
Alex Williamson04b16652010-07-02 11:13:17 -06001207
Mike Day0dc3f442013-09-05 14:41:35 -04001208 QLIST_FOREACH_RCU(next_block, &ram_list.blocks, next) {
Alex Williamson04b16652010-07-02 11:13:17 -06001209 if (next_block->offset >= end) {
1210 next = MIN(next, next_block->offset);
1211 }
1212 }
1213 if (next - end >= size && next - end < mingap) {
Alex Williamson3e837b22011-10-31 08:54:09 -06001214 offset = end;
Alex Williamson04b16652010-07-02 11:13:17 -06001215 mingap = next - end;
1216 }
1217 }
Alex Williamson3e837b22011-10-31 08:54:09 -06001218
1219 if (offset == RAM_ADDR_MAX) {
1220 fprintf(stderr, "Failed to find gap of requested size: %" PRIu64 "\n",
1221 (uint64_t)size);
1222 abort();
1223 }
1224
Alex Williamson04b16652010-07-02 11:13:17 -06001225 return offset;
1226}
1227
Juan Quintela652d7ec2012-07-20 10:37:54 +02001228ram_addr_t last_ram_offset(void)
Alex Williamson04b16652010-07-02 11:13:17 -06001229{
Alex Williamsond17b5282010-06-25 11:08:38 -06001230 RAMBlock *block;
1231 ram_addr_t last = 0;
1232
Mike Day0dc3f442013-09-05 14:41:35 -04001233 rcu_read_lock();
1234 QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
Michael S. Tsirkin62be4e32014-11-12 14:27:41 +02001235 last = MAX(last, block->offset + block->max_length);
Mike Day0d53d9f2015-01-21 13:45:24 +01001236 }
Mike Day0dc3f442013-09-05 14:41:35 -04001237 rcu_read_unlock();
Alex Williamsond17b5282010-06-25 11:08:38 -06001238 return last;
1239}
1240
Jason Baronddb97f12012-08-02 15:44:16 -04001241static void qemu_ram_setup_dump(void *addr, ram_addr_t size)
1242{
1243 int ret;
Jason Baronddb97f12012-08-02 15:44:16 -04001244
1245 /* Use MADV_DONTDUMP, if user doesn't want the guest memory in the core */
Marcel Apfelbaum47c8ca52015-02-04 17:43:54 +02001246 if (!machine_dump_guest_core(current_machine)) {
Jason Baronddb97f12012-08-02 15:44:16 -04001247 ret = qemu_madvise(addr, size, QEMU_MADV_DONTDUMP);
1248 if (ret) {
1249 perror("qemu_madvise");
1250 fprintf(stderr, "madvise doesn't support MADV_DONTDUMP, "
1251 "but dump_guest_core=off specified\n");
1252 }
1253 }
1254}
1255
Mike Day0dc3f442013-09-05 14:41:35 -04001256/* Called within an RCU critical section, or while the ramlist lock
1257 * is held.
1258 */
Hu Tao20cfe882014-04-02 15:13:26 +08001259static RAMBlock *find_ram_block(ram_addr_t addr)
Cam Macdonell84b89d72010-07-26 18:10:57 -06001260{
Hu Tao20cfe882014-04-02 15:13:26 +08001261 RAMBlock *block;
Cam Macdonell84b89d72010-07-26 18:10:57 -06001262
Mike Day0dc3f442013-09-05 14:41:35 -04001263 QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
Avi Kivityc5705a72011-12-20 15:59:12 +02001264 if (block->offset == addr) {
Hu Tao20cfe882014-04-02 15:13:26 +08001265 return block;
Avi Kivityc5705a72011-12-20 15:59:12 +02001266 }
1267 }
Hu Tao20cfe882014-04-02 15:13:26 +08001268
1269 return NULL;
1270}
1271
Mike Dayae3a7042013-09-05 14:41:35 -04001272/* Called with iothread lock held. */
Hu Tao20cfe882014-04-02 15:13:26 +08001273void qemu_ram_set_idstr(ram_addr_t addr, const char *name, DeviceState *dev)
1274{
Mike Dayae3a7042013-09-05 14:41:35 -04001275 RAMBlock *new_block, *block;
Hu Tao20cfe882014-04-02 15:13:26 +08001276
Mike Day0dc3f442013-09-05 14:41:35 -04001277 rcu_read_lock();
Mike Dayae3a7042013-09-05 14:41:35 -04001278 new_block = find_ram_block(addr);
Avi Kivityc5705a72011-12-20 15:59:12 +02001279 assert(new_block);
1280 assert(!new_block->idstr[0]);
Cam Macdonell84b89d72010-07-26 18:10:57 -06001281
Anthony Liguori09e5ab62012-02-03 12:28:43 -06001282 if (dev) {
1283 char *id = qdev_get_dev_path(dev);
Cam Macdonell84b89d72010-07-26 18:10:57 -06001284 if (id) {
1285 snprintf(new_block->idstr, sizeof(new_block->idstr), "%s/", id);
Anthony Liguori7267c092011-08-20 22:09:37 -05001286 g_free(id);
Cam Macdonell84b89d72010-07-26 18:10:57 -06001287 }
1288 }
1289 pstrcat(new_block->idstr, sizeof(new_block->idstr), name);
1290
Mike Day0dc3f442013-09-05 14:41:35 -04001291 QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
Avi Kivityc5705a72011-12-20 15:59:12 +02001292 if (block != new_block && !strcmp(block->idstr, new_block->idstr)) {
Cam Macdonell84b89d72010-07-26 18:10:57 -06001293 fprintf(stderr, "RAMBlock \"%s\" already registered, abort!\n",
1294 new_block->idstr);
1295 abort();
1296 }
1297 }
Mike Day0dc3f442013-09-05 14:41:35 -04001298 rcu_read_unlock();
Avi Kivityc5705a72011-12-20 15:59:12 +02001299}
1300
Mike Dayae3a7042013-09-05 14:41:35 -04001301/* Called with iothread lock held. */
Hu Tao20cfe882014-04-02 15:13:26 +08001302void qemu_ram_unset_idstr(ram_addr_t addr)
1303{
Mike Dayae3a7042013-09-05 14:41:35 -04001304 RAMBlock *block;
Hu Tao20cfe882014-04-02 15:13:26 +08001305
Mike Dayae3a7042013-09-05 14:41:35 -04001306 /* FIXME: arch_init.c assumes that this is not called throughout
1307 * migration. Ignore the problem since hot-unplug during migration
1308 * does not work anyway.
1309 */
1310
Mike Day0dc3f442013-09-05 14:41:35 -04001311 rcu_read_lock();
Mike Dayae3a7042013-09-05 14:41:35 -04001312 block = find_ram_block(addr);
Hu Tao20cfe882014-04-02 15:13:26 +08001313 if (block) {
1314 memset(block->idstr, 0, sizeof(block->idstr));
1315 }
Mike Day0dc3f442013-09-05 14:41:35 -04001316 rcu_read_unlock();
Hu Tao20cfe882014-04-02 15:13:26 +08001317}
1318
Luiz Capitulino8490fc72012-09-05 16:50:16 -03001319static int memory_try_enable_merging(void *addr, size_t len)
1320{
Marcel Apfelbaum75cc7f02015-02-04 17:43:55 +02001321 if (!machine_mem_merge(current_machine)) {
Luiz Capitulino8490fc72012-09-05 16:50:16 -03001322 /* disabled by the user */
1323 return 0;
1324 }
1325
1326 return qemu_madvise(addr, len, QEMU_MADV_MERGEABLE);
1327}
1328
Michael S. Tsirkin62be4e32014-11-12 14:27:41 +02001329/* Only legal before guest might have detected the memory size: e.g. on
1330 * incoming migration, or right after reset.
1331 *
1332 * As memory core doesn't know how is memory accessed, it is up to
1333 * resize callback to update device state and/or add assertions to detect
1334 * misuse, if necessary.
1335 */
1336int qemu_ram_resize(ram_addr_t base, ram_addr_t newsize, Error **errp)
1337{
1338 RAMBlock *block = find_ram_block(base);
1339
1340 assert(block);
1341
Michael S. Tsirkin129ddaf2015-02-17 10:15:30 +01001342 newsize = TARGET_PAGE_ALIGN(newsize);
1343
Michael S. Tsirkin62be4e32014-11-12 14:27:41 +02001344 if (block->used_length == newsize) {
1345 return 0;
1346 }
1347
1348 if (!(block->flags & RAM_RESIZEABLE)) {
1349 error_setg_errno(errp, EINVAL,
1350 "Length mismatch: %s: 0x" RAM_ADDR_FMT
1351 " in != 0x" RAM_ADDR_FMT, block->idstr,
1352 newsize, block->used_length);
1353 return -EINVAL;
1354 }
1355
1356 if (block->max_length < newsize) {
1357 error_setg_errno(errp, EINVAL,
1358 "Length too large: %s: 0x" RAM_ADDR_FMT
1359 " > 0x" RAM_ADDR_FMT, block->idstr,
1360 newsize, block->max_length);
1361 return -EINVAL;
1362 }
1363
1364 cpu_physical_memory_clear_dirty_range(block->offset, block->used_length);
1365 block->used_length = newsize;
Paolo Bonzini58d27072015-03-23 11:56:01 +01001366 cpu_physical_memory_set_dirty_range(block->offset, block->used_length,
1367 DIRTY_CLIENTS_ALL);
Michael S. Tsirkin62be4e32014-11-12 14:27:41 +02001368 memory_region_set_size(block->mr, newsize);
1369 if (block->resized) {
1370 block->resized(block->idstr, newsize, block->host);
1371 }
1372 return 0;
1373}
1374
Hu Taoef701d72014-09-09 13:27:54 +08001375static ram_addr_t ram_block_add(RAMBlock *new_block, Error **errp)
Avi Kivityc5705a72011-12-20 15:59:12 +02001376{
Paolo Bonzinie1c57ab2014-05-14 17:43:18 +08001377 RAMBlock *block;
Mike Day0d53d9f2015-01-21 13:45:24 +01001378 RAMBlock *last_block = NULL;
Juan Quintela2152f5c2013-10-08 13:52:02 +02001379 ram_addr_t old_ram_size, new_ram_size;
1380
1381 old_ram_size = last_ram_offset() >> TARGET_PAGE_BITS;
Avi Kivityc5705a72011-12-20 15:59:12 +02001382
Umesh Deshpandeb2a86582011-08-17 00:01:33 -07001383 qemu_mutex_lock_ramlist();
Michael S. Tsirkin9b8424d2014-12-15 22:55:32 +02001384 new_block->offset = find_ram_offset(new_block->max_length);
Paolo Bonzinie1c57ab2014-05-14 17:43:18 +08001385
1386 if (!new_block->host) {
1387 if (xen_enabled()) {
Michael S. Tsirkin9b8424d2014-12-15 22:55:32 +02001388 xen_ram_alloc(new_block->offset, new_block->max_length,
1389 new_block->mr);
Paolo Bonzinie1c57ab2014-05-14 17:43:18 +08001390 } else {
Michael S. Tsirkin9b8424d2014-12-15 22:55:32 +02001391 new_block->host = phys_mem_alloc(new_block->max_length,
Igor Mammedova2b257d2014-10-31 16:38:37 +00001392 &new_block->mr->align);
Markus Armbruster39228252013-07-31 15:11:11 +02001393 if (!new_block->host) {
Hu Taoef701d72014-09-09 13:27:54 +08001394 error_setg_errno(errp, errno,
1395 "cannot set up guest memory '%s'",
1396 memory_region_name(new_block->mr));
1397 qemu_mutex_unlock_ramlist();
1398 return -1;
Markus Armbruster39228252013-07-31 15:11:11 +02001399 }
Michael S. Tsirkin9b8424d2014-12-15 22:55:32 +02001400 memory_try_enable_merging(new_block->host, new_block->max_length);
Yoshiaki Tamura6977dfe2010-08-18 15:41:49 +09001401 }
1402 }
Cam Macdonell84b89d72010-07-26 18:10:57 -06001403
Mike Day0d53d9f2015-01-21 13:45:24 +01001404 /* Keep the list sorted from biggest to smallest block. Unlike QTAILQ,
1405 * QLIST (which has an RCU-friendly variant) does not have insertion at
1406 * tail, so save the last element in last_block.
1407 */
Mike Day0dc3f442013-09-05 14:41:35 -04001408 QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
Mike Day0d53d9f2015-01-21 13:45:24 +01001409 last_block = block;
Michael S. Tsirkin9b8424d2014-12-15 22:55:32 +02001410 if (block->max_length < new_block->max_length) {
Paolo Bonziniabb26d62012-11-14 16:00:51 +01001411 break;
1412 }
1413 }
1414 if (block) {
Mike Day0dc3f442013-09-05 14:41:35 -04001415 QLIST_INSERT_BEFORE_RCU(block, new_block, next);
Mike Day0d53d9f2015-01-21 13:45:24 +01001416 } else if (last_block) {
Mike Day0dc3f442013-09-05 14:41:35 -04001417 QLIST_INSERT_AFTER_RCU(last_block, new_block, next);
Mike Day0d53d9f2015-01-21 13:45:24 +01001418 } else { /* list is empty */
Mike Day0dc3f442013-09-05 14:41:35 -04001419 QLIST_INSERT_HEAD_RCU(&ram_list.blocks, new_block, next);
Paolo Bonziniabb26d62012-11-14 16:00:51 +01001420 }
Paolo Bonzini0d6d3c82012-11-14 15:45:02 +01001421 ram_list.mru_block = NULL;
Cam Macdonell84b89d72010-07-26 18:10:57 -06001422
Mike Day0dc3f442013-09-05 14:41:35 -04001423 /* Write list before version */
1424 smp_wmb();
Umesh Deshpandef798b072011-08-18 11:41:17 -07001425 ram_list.version++;
Umesh Deshpandeb2a86582011-08-17 00:01:33 -07001426 qemu_mutex_unlock_ramlist();
Umesh Deshpandef798b072011-08-18 11:41:17 -07001427
Juan Quintela2152f5c2013-10-08 13:52:02 +02001428 new_ram_size = last_ram_offset() >> TARGET_PAGE_BITS;
1429
1430 if (new_ram_size > old_ram_size) {
Juan Quintela1ab4c8c2013-10-08 16:14:39 +02001431 int i;
Mike Dayae3a7042013-09-05 14:41:35 -04001432
1433 /* ram_list.dirty_memory[] is protected by the iothread lock. */
Juan Quintela1ab4c8c2013-10-08 16:14:39 +02001434 for (i = 0; i < DIRTY_MEMORY_NUM; i++) {
1435 ram_list.dirty_memory[i] =
1436 bitmap_zero_extend(ram_list.dirty_memory[i],
1437 old_ram_size, new_ram_size);
1438 }
Juan Quintela2152f5c2013-10-08 13:52:02 +02001439 }
Michael S. Tsirkin9b8424d2014-12-15 22:55:32 +02001440 cpu_physical_memory_set_dirty_range(new_block->offset,
Paolo Bonzini58d27072015-03-23 11:56:01 +01001441 new_block->used_length,
1442 DIRTY_CLIENTS_ALL);
Cam Macdonell84b89d72010-07-26 18:10:57 -06001443
Paolo Bonzinia904c912015-01-21 16:18:35 +01001444 if (new_block->host) {
1445 qemu_ram_setup_dump(new_block->host, new_block->max_length);
1446 qemu_madvise(new_block->host, new_block->max_length, QEMU_MADV_HUGEPAGE);
1447 qemu_madvise(new_block->host, new_block->max_length, QEMU_MADV_DONTFORK);
1448 if (kvm_enabled()) {
1449 kvm_setup_guest_memory(new_block->host, new_block->max_length);
1450 }
Paolo Bonzinie1c57ab2014-05-14 17:43:18 +08001451 }
Cam Macdonell84b89d72010-07-26 18:10:57 -06001452
1453 return new_block->offset;
1454}
1455
Paolo Bonzini0b183fc2014-05-14 17:43:19 +08001456#ifdef __linux__
Paolo Bonzinie1c57ab2014-05-14 17:43:18 +08001457ram_addr_t qemu_ram_alloc_from_file(ram_addr_t size, MemoryRegion *mr,
Paolo Bonzinidbcb8982014-06-10 19:15:24 +08001458 bool share, const char *mem_path,
Paolo Bonzini7f56e742014-05-14 17:43:20 +08001459 Error **errp)
Paolo Bonzinie1c57ab2014-05-14 17:43:18 +08001460{
1461 RAMBlock *new_block;
Hu Taoef701d72014-09-09 13:27:54 +08001462 ram_addr_t addr;
1463 Error *local_err = NULL;
Paolo Bonzinie1c57ab2014-05-14 17:43:18 +08001464
1465 if (xen_enabled()) {
Paolo Bonzini7f56e742014-05-14 17:43:20 +08001466 error_setg(errp, "-mem-path not supported with Xen");
1467 return -1;
Paolo Bonzinie1c57ab2014-05-14 17:43:18 +08001468 }
1469
1470 if (phys_mem_alloc != qemu_anon_ram_alloc) {
1471 /*
1472 * file_ram_alloc() needs to allocate just like
1473 * phys_mem_alloc, but we haven't bothered to provide
1474 * a hook there.
1475 */
Paolo Bonzini7f56e742014-05-14 17:43:20 +08001476 error_setg(errp,
1477 "-mem-path not supported with this accelerator");
1478 return -1;
Paolo Bonzinie1c57ab2014-05-14 17:43:18 +08001479 }
1480
1481 size = TARGET_PAGE_ALIGN(size);
1482 new_block = g_malloc0(sizeof(*new_block));
1483 new_block->mr = mr;
Michael S. Tsirkin9b8424d2014-12-15 22:55:32 +02001484 new_block->used_length = size;
1485 new_block->max_length = size;
Paolo Bonzinidbcb8982014-06-10 19:15:24 +08001486 new_block->flags = share ? RAM_SHARED : 0;
Paolo Bonzini7f56e742014-05-14 17:43:20 +08001487 new_block->host = file_ram_alloc(new_block, size,
1488 mem_path, errp);
1489 if (!new_block->host) {
1490 g_free(new_block);
1491 return -1;
1492 }
1493
Hu Taoef701d72014-09-09 13:27:54 +08001494 addr = ram_block_add(new_block, &local_err);
1495 if (local_err) {
1496 g_free(new_block);
1497 error_propagate(errp, local_err);
1498 return -1;
1499 }
1500 return addr;
Paolo Bonzinie1c57ab2014-05-14 17:43:18 +08001501}
Paolo Bonzini0b183fc2014-05-14 17:43:19 +08001502#endif
Paolo Bonzinie1c57ab2014-05-14 17:43:18 +08001503
Michael S. Tsirkin62be4e32014-11-12 14:27:41 +02001504static
1505ram_addr_t qemu_ram_alloc_internal(ram_addr_t size, ram_addr_t max_size,
1506 void (*resized)(const char*,
1507 uint64_t length,
1508 void *host),
1509 void *host, bool resizeable,
Hu Taoef701d72014-09-09 13:27:54 +08001510 MemoryRegion *mr, Error **errp)
Paolo Bonzinie1c57ab2014-05-14 17:43:18 +08001511{
1512 RAMBlock *new_block;
Hu Taoef701d72014-09-09 13:27:54 +08001513 ram_addr_t addr;
1514 Error *local_err = NULL;
Paolo Bonzinie1c57ab2014-05-14 17:43:18 +08001515
1516 size = TARGET_PAGE_ALIGN(size);
Michael S. Tsirkin62be4e32014-11-12 14:27:41 +02001517 max_size = TARGET_PAGE_ALIGN(max_size);
Paolo Bonzinie1c57ab2014-05-14 17:43:18 +08001518 new_block = g_malloc0(sizeof(*new_block));
1519 new_block->mr = mr;
Michael S. Tsirkin62be4e32014-11-12 14:27:41 +02001520 new_block->resized = resized;
Michael S. Tsirkin9b8424d2014-12-15 22:55:32 +02001521 new_block->used_length = size;
1522 new_block->max_length = max_size;
Michael S. Tsirkin62be4e32014-11-12 14:27:41 +02001523 assert(max_size >= size);
Paolo Bonzinie1c57ab2014-05-14 17:43:18 +08001524 new_block->fd = -1;
1525 new_block->host = host;
1526 if (host) {
Paolo Bonzini7bd4f432014-05-14 17:43:22 +08001527 new_block->flags |= RAM_PREALLOC;
Paolo Bonzinie1c57ab2014-05-14 17:43:18 +08001528 }
Michael S. Tsirkin62be4e32014-11-12 14:27:41 +02001529 if (resizeable) {
1530 new_block->flags |= RAM_RESIZEABLE;
1531 }
Hu Taoef701d72014-09-09 13:27:54 +08001532 addr = ram_block_add(new_block, &local_err);
1533 if (local_err) {
1534 g_free(new_block);
1535 error_propagate(errp, local_err);
1536 return -1;
1537 }
1538 return addr;
Paolo Bonzinie1c57ab2014-05-14 17:43:18 +08001539}
1540
Michael S. Tsirkin62be4e32014-11-12 14:27:41 +02001541ram_addr_t qemu_ram_alloc_from_ptr(ram_addr_t size, void *host,
1542 MemoryRegion *mr, Error **errp)
1543{
1544 return qemu_ram_alloc_internal(size, size, NULL, host, false, mr, errp);
1545}
1546
Hu Taoef701d72014-09-09 13:27:54 +08001547ram_addr_t qemu_ram_alloc(ram_addr_t size, MemoryRegion *mr, Error **errp)
pbrook94a6b542009-04-11 17:15:54 +00001548{
Michael S. Tsirkin62be4e32014-11-12 14:27:41 +02001549 return qemu_ram_alloc_internal(size, size, NULL, NULL, false, mr, errp);
1550}
1551
1552ram_addr_t qemu_ram_alloc_resizeable(ram_addr_t size, ram_addr_t maxsz,
1553 void (*resized)(const char*,
1554 uint64_t length,
1555 void *host),
1556 MemoryRegion *mr, Error **errp)
1557{
1558 return qemu_ram_alloc_internal(size, maxsz, resized, NULL, true, mr, errp);
pbrook94a6b542009-04-11 17:15:54 +00001559}
bellarde9a1ab12007-02-08 23:08:38 +00001560
Alex Williamson1f2e98b2011-05-03 12:48:09 -06001561void qemu_ram_free_from_ptr(ram_addr_t addr)
1562{
1563 RAMBlock *block;
1564
Umesh Deshpandeb2a86582011-08-17 00:01:33 -07001565 qemu_mutex_lock_ramlist();
Mike Day0dc3f442013-09-05 14:41:35 -04001566 QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
Alex Williamson1f2e98b2011-05-03 12:48:09 -06001567 if (addr == block->offset) {
Mike Day0dc3f442013-09-05 14:41:35 -04001568 QLIST_REMOVE_RCU(block, next);
Paolo Bonzini0d6d3c82012-11-14 15:45:02 +01001569 ram_list.mru_block = NULL;
Mike Day0dc3f442013-09-05 14:41:35 -04001570 /* Write list before version */
1571 smp_wmb();
Umesh Deshpandef798b072011-08-18 11:41:17 -07001572 ram_list.version++;
Paolo Bonzini43771532013-09-09 17:58:40 +02001573 g_free_rcu(block, rcu);
Umesh Deshpandeb2a86582011-08-17 00:01:33 -07001574 break;
Alex Williamson1f2e98b2011-05-03 12:48:09 -06001575 }
1576 }
Umesh Deshpandeb2a86582011-08-17 00:01:33 -07001577 qemu_mutex_unlock_ramlist();
Alex Williamson1f2e98b2011-05-03 12:48:09 -06001578}
1579
Paolo Bonzini43771532013-09-09 17:58:40 +02001580static void reclaim_ramblock(RAMBlock *block)
1581{
1582 if (block->flags & RAM_PREALLOC) {
1583 ;
1584 } else if (xen_enabled()) {
1585 xen_invalidate_map_cache_entry(block->host);
1586#ifndef _WIN32
1587 } else if (block->fd >= 0) {
1588 munmap(block->host, block->max_length);
1589 close(block->fd);
1590#endif
1591 } else {
1592 qemu_anon_ram_free(block->host, block->max_length);
1593 }
1594 g_free(block);
1595}
1596
Anthony Liguoric227f092009-10-01 16:12:16 -05001597void qemu_ram_free(ram_addr_t addr)
bellarde9a1ab12007-02-08 23:08:38 +00001598{
Alex Williamson04b16652010-07-02 11:13:17 -06001599 RAMBlock *block;
1600
Umesh Deshpandeb2a86582011-08-17 00:01:33 -07001601 qemu_mutex_lock_ramlist();
Mike Day0dc3f442013-09-05 14:41:35 -04001602 QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
Alex Williamson04b16652010-07-02 11:13:17 -06001603 if (addr == block->offset) {
Mike Day0dc3f442013-09-05 14:41:35 -04001604 QLIST_REMOVE_RCU(block, next);
Paolo Bonzini0d6d3c82012-11-14 15:45:02 +01001605 ram_list.mru_block = NULL;
Mike Day0dc3f442013-09-05 14:41:35 -04001606 /* Write list before version */
1607 smp_wmb();
Umesh Deshpandef798b072011-08-18 11:41:17 -07001608 ram_list.version++;
Paolo Bonzini43771532013-09-09 17:58:40 +02001609 call_rcu(block, reclaim_ramblock, rcu);
Umesh Deshpandeb2a86582011-08-17 00:01:33 -07001610 break;
Alex Williamson04b16652010-07-02 11:13:17 -06001611 }
1612 }
Umesh Deshpandeb2a86582011-08-17 00:01:33 -07001613 qemu_mutex_unlock_ramlist();
bellarde9a1ab12007-02-08 23:08:38 +00001614}
1615
Huang Yingcd19cfa2011-03-02 08:56:19 +01001616#ifndef _WIN32
1617void qemu_ram_remap(ram_addr_t addr, ram_addr_t length)
1618{
1619 RAMBlock *block;
1620 ram_addr_t offset;
1621 int flags;
1622 void *area, *vaddr;
1623
Mike Day0dc3f442013-09-05 14:41:35 -04001624 QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
Huang Yingcd19cfa2011-03-02 08:56:19 +01001625 offset = addr - block->offset;
Michael S. Tsirkin9b8424d2014-12-15 22:55:32 +02001626 if (offset < block->max_length) {
Michael S. Tsirkin1240be22014-11-12 11:44:41 +02001627 vaddr = ramblock_ptr(block, offset);
Paolo Bonzini7bd4f432014-05-14 17:43:22 +08001628 if (block->flags & RAM_PREALLOC) {
Huang Yingcd19cfa2011-03-02 08:56:19 +01001629 ;
Markus Armbrusterdfeaf2a2013-07-31 15:11:05 +02001630 } else if (xen_enabled()) {
1631 abort();
Huang Yingcd19cfa2011-03-02 08:56:19 +01001632 } else {
1633 flags = MAP_FIXED;
Markus Armbruster3435f392013-07-31 15:11:07 +02001634 if (block->fd >= 0) {
Paolo Bonzinidbcb8982014-06-10 19:15:24 +08001635 flags |= (block->flags & RAM_SHARED ?
1636 MAP_SHARED : MAP_PRIVATE);
Markus Armbruster3435f392013-07-31 15:11:07 +02001637 area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
1638 flags, block->fd, offset);
Huang Yingcd19cfa2011-03-02 08:56:19 +01001639 } else {
Markus Armbruster2eb9fba2013-07-31 15:11:09 +02001640 /*
1641 * Remap needs to match alloc. Accelerators that
1642 * set phys_mem_alloc never remap. If they did,
1643 * we'd need a remap hook here.
1644 */
1645 assert(phys_mem_alloc == qemu_anon_ram_alloc);
1646
Huang Yingcd19cfa2011-03-02 08:56:19 +01001647 flags |= MAP_PRIVATE | MAP_ANONYMOUS;
1648 area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
1649 flags, -1, 0);
Huang Yingcd19cfa2011-03-02 08:56:19 +01001650 }
1651 if (area != vaddr) {
Anthony PERARDf15fbc42011-07-20 08:17:42 +00001652 fprintf(stderr, "Could not remap addr: "
1653 RAM_ADDR_FMT "@" RAM_ADDR_FMT "\n",
Huang Yingcd19cfa2011-03-02 08:56:19 +01001654 length, addr);
1655 exit(1);
1656 }
Luiz Capitulino8490fc72012-09-05 16:50:16 -03001657 memory_try_enable_merging(vaddr, length);
Jason Baronddb97f12012-08-02 15:44:16 -04001658 qemu_ram_setup_dump(vaddr, length);
Huang Yingcd19cfa2011-03-02 08:56:19 +01001659 }
Huang Yingcd19cfa2011-03-02 08:56:19 +01001660 }
1661 }
1662}
1663#endif /* !_WIN32 */
1664
Paolo Bonzinia35ba7b2014-06-10 19:15:23 +08001665int qemu_get_ram_fd(ram_addr_t addr)
1666{
Mike Dayae3a7042013-09-05 14:41:35 -04001667 RAMBlock *block;
1668 int fd;
Paolo Bonzinia35ba7b2014-06-10 19:15:23 +08001669
Mike Day0dc3f442013-09-05 14:41:35 -04001670 rcu_read_lock();
Mike Dayae3a7042013-09-05 14:41:35 -04001671 block = qemu_get_ram_block(addr);
1672 fd = block->fd;
Mike Day0dc3f442013-09-05 14:41:35 -04001673 rcu_read_unlock();
Mike Dayae3a7042013-09-05 14:41:35 -04001674 return fd;
Paolo Bonzinia35ba7b2014-06-10 19:15:23 +08001675}
1676
Damjan Marion3fd74b82014-06-26 23:01:32 +02001677void *qemu_get_ram_block_host_ptr(ram_addr_t addr)
1678{
Mike Dayae3a7042013-09-05 14:41:35 -04001679 RAMBlock *block;
1680 void *ptr;
Damjan Marion3fd74b82014-06-26 23:01:32 +02001681
Mike Day0dc3f442013-09-05 14:41:35 -04001682 rcu_read_lock();
Mike Dayae3a7042013-09-05 14:41:35 -04001683 block = qemu_get_ram_block(addr);
1684 ptr = ramblock_ptr(block, 0);
Mike Day0dc3f442013-09-05 14:41:35 -04001685 rcu_read_unlock();
Mike Dayae3a7042013-09-05 14:41:35 -04001686 return ptr;
Damjan Marion3fd74b82014-06-26 23:01:32 +02001687}
1688
Paolo Bonzini1b5ec232013-05-06 14:36:15 +02001689/* Return a host pointer to ram allocated with qemu_ram_alloc.
Mike Dayae3a7042013-09-05 14:41:35 -04001690 * This should not be used for general purpose DMA. Use address_space_map
1691 * or address_space_rw instead. For local memory (e.g. video ram) that the
1692 * device owns, use memory_region_get_ram_ptr.
Mike Day0dc3f442013-09-05 14:41:35 -04001693 *
1694 * By the time this function returns, the returned pointer is not protected
1695 * by RCU anymore. If the caller is not within an RCU critical section and
1696 * does not hold the iothread lock, it must have other means of protecting the
1697 * pointer, such as a reference to the region that includes the incoming
1698 * ram_addr_t.
Paolo Bonzini1b5ec232013-05-06 14:36:15 +02001699 */
1700void *qemu_get_ram_ptr(ram_addr_t addr)
1701{
Mike Dayae3a7042013-09-05 14:41:35 -04001702 RAMBlock *block;
1703 void *ptr;
Paolo Bonzini1b5ec232013-05-06 14:36:15 +02001704
Mike Day0dc3f442013-09-05 14:41:35 -04001705 rcu_read_lock();
Mike Dayae3a7042013-09-05 14:41:35 -04001706 block = qemu_get_ram_block(addr);
1707
1708 if (xen_enabled() && block->host == NULL) {
Paolo Bonzini0d6d3c82012-11-14 15:45:02 +01001709 /* We need to check if the requested address is in the RAM
1710 * because we don't want to map the entire memory in QEMU.
1711 * In that case just map until the end of the page.
1712 */
1713 if (block->offset == 0) {
Mike Dayae3a7042013-09-05 14:41:35 -04001714 ptr = xen_map_cache(addr, 0, 0);
Mike Day0dc3f442013-09-05 14:41:35 -04001715 goto unlock;
Paolo Bonzini0d6d3c82012-11-14 15:45:02 +01001716 }
Mike Dayae3a7042013-09-05 14:41:35 -04001717
1718 block->host = xen_map_cache(block->offset, block->max_length, 1);
Paolo Bonzini0d6d3c82012-11-14 15:45:02 +01001719 }
Mike Dayae3a7042013-09-05 14:41:35 -04001720 ptr = ramblock_ptr(block, addr - block->offset);
1721
Mike Day0dc3f442013-09-05 14:41:35 -04001722unlock:
1723 rcu_read_unlock();
Mike Dayae3a7042013-09-05 14:41:35 -04001724 return ptr;
pbrookdc828ca2009-04-09 22:21:07 +00001725}
1726
Stefano Stabellini38bee5d2011-05-19 18:35:45 +01001727/* Return a host pointer to guest's ram. Similar to qemu_get_ram_ptr
Mike Dayae3a7042013-09-05 14:41:35 -04001728 * but takes a size argument.
Mike Day0dc3f442013-09-05 14:41:35 -04001729 *
1730 * By the time this function returns, the returned pointer is not protected
1731 * by RCU anymore. If the caller is not within an RCU critical section and
1732 * does not hold the iothread lock, it must have other means of protecting the
1733 * pointer, such as a reference to the region that includes the incoming
1734 * ram_addr_t.
Mike Dayae3a7042013-09-05 14:41:35 -04001735 */
Peter Maydellcb85f7a2013-07-08 09:44:04 +01001736static void *qemu_ram_ptr_length(ram_addr_t addr, hwaddr *size)
Stefano Stabellini38bee5d2011-05-19 18:35:45 +01001737{
Mike Dayae3a7042013-09-05 14:41:35 -04001738 void *ptr;
Stefano Stabellini8ab934f2011-06-27 18:26:06 +01001739 if (*size == 0) {
1740 return NULL;
1741 }
Jan Kiszka868bb332011-06-21 22:59:09 +02001742 if (xen_enabled()) {
Jan Kiszkae41d7c62011-06-21 22:59:08 +02001743 return xen_map_cache(addr, *size, 1);
Jan Kiszka868bb332011-06-21 22:59:09 +02001744 } else {
Stefano Stabellini38bee5d2011-05-19 18:35:45 +01001745 RAMBlock *block;
Mike Day0dc3f442013-09-05 14:41:35 -04001746 rcu_read_lock();
1747 QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
Michael S. Tsirkin9b8424d2014-12-15 22:55:32 +02001748 if (addr - block->offset < block->max_length) {
1749 if (addr - block->offset + *size > block->max_length)
1750 *size = block->max_length - addr + block->offset;
Mike Dayae3a7042013-09-05 14:41:35 -04001751 ptr = ramblock_ptr(block, addr - block->offset);
Mike Day0dc3f442013-09-05 14:41:35 -04001752 rcu_read_unlock();
Mike Dayae3a7042013-09-05 14:41:35 -04001753 return ptr;
Stefano Stabellini38bee5d2011-05-19 18:35:45 +01001754 }
1755 }
1756
1757 fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
1758 abort();
Stefano Stabellini38bee5d2011-05-19 18:35:45 +01001759 }
1760}
1761
Paolo Bonzini7443b432013-06-03 12:44:02 +02001762/* Some of the softmmu routines need to translate from a host pointer
Mike Dayae3a7042013-09-05 14:41:35 -04001763 * (typically a TLB entry) back to a ram offset.
1764 *
1765 * By the time this function returns, the returned pointer is not protected
1766 * by RCU anymore. If the caller is not within an RCU critical section and
1767 * does not hold the iothread lock, it must have other means of protecting the
1768 * pointer, such as a reference to the region that includes the incoming
1769 * ram_addr_t.
1770 */
Paolo Bonzini1b5ec232013-05-06 14:36:15 +02001771MemoryRegion *qemu_ram_addr_from_host(void *ptr, ram_addr_t *ram_addr)
pbrook5579c7f2009-04-11 14:47:08 +00001772{
pbrook94a6b542009-04-11 17:15:54 +00001773 RAMBlock *block;
1774 uint8_t *host = ptr;
Mike Dayae3a7042013-09-05 14:41:35 -04001775 MemoryRegion *mr;
pbrook94a6b542009-04-11 17:15:54 +00001776
Jan Kiszka868bb332011-06-21 22:59:09 +02001777 if (xen_enabled()) {
Mike Day0dc3f442013-09-05 14:41:35 -04001778 rcu_read_lock();
Jan Kiszkae41d7c62011-06-21 22:59:08 +02001779 *ram_addr = xen_ram_addr_from_mapcache(ptr);
Mike Dayae3a7042013-09-05 14:41:35 -04001780 mr = qemu_get_ram_block(*ram_addr)->mr;
Mike Day0dc3f442013-09-05 14:41:35 -04001781 rcu_read_unlock();
Mike Dayae3a7042013-09-05 14:41:35 -04001782 return mr;
Stefano Stabellini712c2b42011-05-19 18:35:46 +01001783 }
1784
Mike Day0dc3f442013-09-05 14:41:35 -04001785 rcu_read_lock();
1786 block = atomic_rcu_read(&ram_list.mru_block);
Michael S. Tsirkin9b8424d2014-12-15 22:55:32 +02001787 if (block && block->host && host - block->host < block->max_length) {
Paolo Bonzini23887b72013-05-06 14:28:39 +02001788 goto found;
1789 }
1790
Mike Day0dc3f442013-09-05 14:41:35 -04001791 QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
Jun Nakajima432d2682010-08-31 16:41:25 +01001792 /* This case append when the block is not mapped. */
1793 if (block->host == NULL) {
1794 continue;
1795 }
Michael S. Tsirkin9b8424d2014-12-15 22:55:32 +02001796 if (host - block->host < block->max_length) {
Paolo Bonzini23887b72013-05-06 14:28:39 +02001797 goto found;
Alex Williamsonf471a172010-06-11 11:11:42 -06001798 }
pbrook94a6b542009-04-11 17:15:54 +00001799 }
Jun Nakajima432d2682010-08-31 16:41:25 +01001800
Mike Day0dc3f442013-09-05 14:41:35 -04001801 rcu_read_unlock();
Paolo Bonzini1b5ec232013-05-06 14:36:15 +02001802 return NULL;
Paolo Bonzini23887b72013-05-06 14:28:39 +02001803
1804found:
1805 *ram_addr = block->offset + (host - block->host);
Mike Dayae3a7042013-09-05 14:41:35 -04001806 mr = block->mr;
Mike Day0dc3f442013-09-05 14:41:35 -04001807 rcu_read_unlock();
Mike Dayae3a7042013-09-05 14:41:35 -04001808 return mr;
Marcelo Tosattie8902612010-10-11 15:31:19 -03001809}
Alex Williamsonf471a172010-06-11 11:11:42 -06001810
Avi Kivitya8170e52012-10-23 12:30:10 +02001811static void notdirty_mem_write(void *opaque, hwaddr ram_addr,
Avi Kivity0e0df1e2012-01-02 00:32:15 +02001812 uint64_t val, unsigned size)
bellard1ccde1c2004-02-06 19:46:14 +00001813{
Juan Quintela52159192013-10-08 12:44:04 +02001814 if (!cpu_physical_memory_get_dirty_flag(ram_addr, DIRTY_MEMORY_CODE)) {
Avi Kivity0e0df1e2012-01-02 00:32:15 +02001815 tb_invalidate_phys_page_fast(ram_addr, size);
bellard3a7d9292005-08-21 09:26:42 +00001816 }
Avi Kivity0e0df1e2012-01-02 00:32:15 +02001817 switch (size) {
1818 case 1:
1819 stb_p(qemu_get_ram_ptr(ram_addr), val);
1820 break;
1821 case 2:
1822 stw_p(qemu_get_ram_ptr(ram_addr), val);
1823 break;
1824 case 4:
1825 stl_p(qemu_get_ram_ptr(ram_addr), val);
1826 break;
1827 default:
1828 abort();
1829 }
Paolo Bonzini58d27072015-03-23 11:56:01 +01001830 /* Set both VGA and migration bits for simplicity and to remove
1831 * the notdirty callback faster.
1832 */
1833 cpu_physical_memory_set_dirty_range(ram_addr, size,
1834 DIRTY_CLIENTS_NOCODE);
bellardf23db162005-08-21 19:12:28 +00001835 /* we remove the notdirty callback only if the code has been
1836 flushed */
Juan Quintelaa2cd8c82013-10-10 11:20:22 +02001837 if (!cpu_physical_memory_is_clean(ram_addr)) {
Andreas Färber4917cf42013-05-27 05:17:50 +02001838 CPUArchState *env = current_cpu->env_ptr;
Andreas Färber93afead2013-08-26 03:41:01 +02001839 tlb_set_dirty(env, current_cpu->mem_io_vaddr);
Andreas Färber4917cf42013-05-27 05:17:50 +02001840 }
bellard1ccde1c2004-02-06 19:46:14 +00001841}
1842
Paolo Bonzinib018ddf2013-05-24 14:48:38 +02001843static bool notdirty_mem_accepts(void *opaque, hwaddr addr,
1844 unsigned size, bool is_write)
1845{
1846 return is_write;
1847}
1848
Avi Kivity0e0df1e2012-01-02 00:32:15 +02001849static const MemoryRegionOps notdirty_mem_ops = {
Avi Kivity0e0df1e2012-01-02 00:32:15 +02001850 .write = notdirty_mem_write,
Paolo Bonzinib018ddf2013-05-24 14:48:38 +02001851 .valid.accepts = notdirty_mem_accepts,
Avi Kivity0e0df1e2012-01-02 00:32:15 +02001852 .endianness = DEVICE_NATIVE_ENDIAN,
bellard1ccde1c2004-02-06 19:46:14 +00001853};
1854
pbrook0f459d12008-06-09 00:20:13 +00001855/* Generate a debug exception if a watchpoint has been hit. */
Peter Maydell66b9b432015-04-26 16:49:24 +01001856static void check_watchpoint(int offset, int len, MemTxAttrs attrs, int flags)
pbrook0f459d12008-06-09 00:20:13 +00001857{
Andreas Färber93afead2013-08-26 03:41:01 +02001858 CPUState *cpu = current_cpu;
1859 CPUArchState *env = cpu->env_ptr;
aliguori06d55cc2008-11-18 20:24:06 +00001860 target_ulong pc, cs_base;
pbrook0f459d12008-06-09 00:20:13 +00001861 target_ulong vaddr;
aliguoria1d1bb32008-11-18 20:07:32 +00001862 CPUWatchpoint *wp;
aliguori06d55cc2008-11-18 20:24:06 +00001863 int cpu_flags;
pbrook0f459d12008-06-09 00:20:13 +00001864
Andreas Färberff4700b2013-08-26 18:23:18 +02001865 if (cpu->watchpoint_hit) {
aliguori06d55cc2008-11-18 20:24:06 +00001866 /* We re-entered the check after replacing the TB. Now raise
1867 * the debug interrupt so that is will trigger after the
1868 * current instruction. */
Andreas Färber93afead2013-08-26 03:41:01 +02001869 cpu_interrupt(cpu, CPU_INTERRUPT_DEBUG);
aliguori06d55cc2008-11-18 20:24:06 +00001870 return;
1871 }
Andreas Färber93afead2013-08-26 03:41:01 +02001872 vaddr = (cpu->mem_io_vaddr & TARGET_PAGE_MASK) + offset;
Andreas Färberff4700b2013-08-26 18:23:18 +02001873 QTAILQ_FOREACH(wp, &cpu->watchpoints, entry) {
Peter Maydell05068c02014-09-12 14:06:48 +01001874 if (cpu_watchpoint_address_matches(wp, vaddr, len)
1875 && (wp->flags & flags)) {
Peter Maydell08225672014-09-12 14:06:48 +01001876 if (flags == BP_MEM_READ) {
1877 wp->flags |= BP_WATCHPOINT_HIT_READ;
1878 } else {
1879 wp->flags |= BP_WATCHPOINT_HIT_WRITE;
1880 }
1881 wp->hitaddr = vaddr;
Peter Maydell66b9b432015-04-26 16:49:24 +01001882 wp->hitattrs = attrs;
Andreas Färberff4700b2013-08-26 18:23:18 +02001883 if (!cpu->watchpoint_hit) {
1884 cpu->watchpoint_hit = wp;
Andreas Färber239c51a2013-09-01 17:12:23 +02001885 tb_check_watchpoint(cpu);
aliguori6e140f22008-11-18 20:37:55 +00001886 if (wp->flags & BP_STOP_BEFORE_ACCESS) {
Andreas Färber27103422013-08-26 08:31:06 +02001887 cpu->exception_index = EXCP_DEBUG;
Andreas Färber5638d182013-08-27 17:52:12 +02001888 cpu_loop_exit(cpu);
aliguori6e140f22008-11-18 20:37:55 +00001889 } else {
1890 cpu_get_tb_cpu_state(env, &pc, &cs_base, &cpu_flags);
Andreas Färber648f0342013-09-01 17:43:17 +02001891 tb_gen_code(cpu, pc, cs_base, cpu_flags, 1);
Andreas Färber0ea8cb82013-09-03 02:12:23 +02001892 cpu_resume_from_signal(cpu, NULL);
aliguori6e140f22008-11-18 20:37:55 +00001893 }
aliguori06d55cc2008-11-18 20:24:06 +00001894 }
aliguori6e140f22008-11-18 20:37:55 +00001895 } else {
1896 wp->flags &= ~BP_WATCHPOINT_HIT;
pbrook0f459d12008-06-09 00:20:13 +00001897 }
1898 }
1899}
1900
pbrook6658ffb2007-03-16 23:58:11 +00001901/* Watchpoint access routines. Watchpoints are inserted using TLB tricks,
1902 so these check for a hit then pass through to the normal out-of-line
1903 phys routines. */
Peter Maydell66b9b432015-04-26 16:49:24 +01001904static MemTxResult watch_mem_read(void *opaque, hwaddr addr, uint64_t *pdata,
1905 unsigned size, MemTxAttrs attrs)
pbrook6658ffb2007-03-16 23:58:11 +00001906{
Peter Maydell66b9b432015-04-26 16:49:24 +01001907 MemTxResult res;
1908 uint64_t data;
pbrook6658ffb2007-03-16 23:58:11 +00001909
Peter Maydell66b9b432015-04-26 16:49:24 +01001910 check_watchpoint(addr & ~TARGET_PAGE_MASK, size, attrs, BP_MEM_READ);
Avi Kivity1ec9b902012-01-02 12:47:48 +02001911 switch (size) {
Max Filippov67364152012-01-29 00:01:40 +04001912 case 1:
Peter Maydell66b9b432015-04-26 16:49:24 +01001913 data = address_space_ldub(&address_space_memory, addr, attrs, &res);
Max Filippov67364152012-01-29 00:01:40 +04001914 break;
1915 case 2:
Peter Maydell66b9b432015-04-26 16:49:24 +01001916 data = address_space_lduw(&address_space_memory, addr, attrs, &res);
Max Filippov67364152012-01-29 00:01:40 +04001917 break;
1918 case 4:
Peter Maydell66b9b432015-04-26 16:49:24 +01001919 data = address_space_ldl(&address_space_memory, addr, attrs, &res);
Max Filippov67364152012-01-29 00:01:40 +04001920 break;
Avi Kivity1ec9b902012-01-02 12:47:48 +02001921 default: abort();
1922 }
Peter Maydell66b9b432015-04-26 16:49:24 +01001923 *pdata = data;
1924 return res;
1925}
1926
1927static MemTxResult watch_mem_write(void *opaque, hwaddr addr,
1928 uint64_t val, unsigned size,
1929 MemTxAttrs attrs)
1930{
1931 MemTxResult res;
1932
1933 check_watchpoint(addr & ~TARGET_PAGE_MASK, size, attrs, BP_MEM_WRITE);
1934 switch (size) {
1935 case 1:
1936 address_space_stb(&address_space_memory, addr, val, attrs, &res);
1937 break;
1938 case 2:
1939 address_space_stw(&address_space_memory, addr, val, attrs, &res);
1940 break;
1941 case 4:
1942 address_space_stl(&address_space_memory, addr, val, attrs, &res);
1943 break;
1944 default: abort();
1945 }
1946 return res;
pbrook6658ffb2007-03-16 23:58:11 +00001947}
1948
Avi Kivity1ec9b902012-01-02 12:47:48 +02001949static const MemoryRegionOps watch_mem_ops = {
Peter Maydell66b9b432015-04-26 16:49:24 +01001950 .read_with_attrs = watch_mem_read,
1951 .write_with_attrs = watch_mem_write,
Avi Kivity1ec9b902012-01-02 12:47:48 +02001952 .endianness = DEVICE_NATIVE_ENDIAN,
pbrook6658ffb2007-03-16 23:58:11 +00001953};
pbrook6658ffb2007-03-16 23:58:11 +00001954
Peter Maydellf25a49e2015-04-26 16:49:24 +01001955static MemTxResult subpage_read(void *opaque, hwaddr addr, uint64_t *data,
1956 unsigned len, MemTxAttrs attrs)
blueswir1db7b5422007-05-26 17:36:03 +00001957{
Jan Kiszkaacc9d802013-05-26 21:55:37 +02001958 subpage_t *subpage = opaque;
Paolo Bonziniff6cff72014-12-22 13:11:39 +01001959 uint8_t buf[8];
Peter Maydell5c9eb022015-04-26 16:49:24 +01001960 MemTxResult res;
Paolo Bonzini791af8c2013-05-24 16:10:39 +02001961
blueswir1db7b5422007-05-26 17:36:03 +00001962#if defined(DEBUG_SUBPAGE)
Amos Kong016e9d62013-09-27 09:25:38 +08001963 printf("%s: subpage %p len %u addr " TARGET_FMT_plx "\n", __func__,
Jan Kiszkaacc9d802013-05-26 21:55:37 +02001964 subpage, len, addr);
blueswir1db7b5422007-05-26 17:36:03 +00001965#endif
Peter Maydell5c9eb022015-04-26 16:49:24 +01001966 res = address_space_read(subpage->as, addr + subpage->base,
1967 attrs, buf, len);
1968 if (res) {
1969 return res;
Peter Maydellf25a49e2015-04-26 16:49:24 +01001970 }
Jan Kiszkaacc9d802013-05-26 21:55:37 +02001971 switch (len) {
1972 case 1:
Peter Maydellf25a49e2015-04-26 16:49:24 +01001973 *data = ldub_p(buf);
1974 return MEMTX_OK;
Jan Kiszkaacc9d802013-05-26 21:55:37 +02001975 case 2:
Peter Maydellf25a49e2015-04-26 16:49:24 +01001976 *data = lduw_p(buf);
1977 return MEMTX_OK;
Jan Kiszkaacc9d802013-05-26 21:55:37 +02001978 case 4:
Peter Maydellf25a49e2015-04-26 16:49:24 +01001979 *data = ldl_p(buf);
1980 return MEMTX_OK;
Paolo Bonziniff6cff72014-12-22 13:11:39 +01001981 case 8:
Peter Maydellf25a49e2015-04-26 16:49:24 +01001982 *data = ldq_p(buf);
1983 return MEMTX_OK;
Jan Kiszkaacc9d802013-05-26 21:55:37 +02001984 default:
1985 abort();
1986 }
blueswir1db7b5422007-05-26 17:36:03 +00001987}
1988
Peter Maydellf25a49e2015-04-26 16:49:24 +01001989static MemTxResult subpage_write(void *opaque, hwaddr addr,
1990 uint64_t value, unsigned len, MemTxAttrs attrs)
blueswir1db7b5422007-05-26 17:36:03 +00001991{
Jan Kiszkaacc9d802013-05-26 21:55:37 +02001992 subpage_t *subpage = opaque;
Paolo Bonziniff6cff72014-12-22 13:11:39 +01001993 uint8_t buf[8];
Jan Kiszkaacc9d802013-05-26 21:55:37 +02001994
blueswir1db7b5422007-05-26 17:36:03 +00001995#if defined(DEBUG_SUBPAGE)
Amos Kong016e9d62013-09-27 09:25:38 +08001996 printf("%s: subpage %p len %u addr " TARGET_FMT_plx
Jan Kiszkaacc9d802013-05-26 21:55:37 +02001997 " value %"PRIx64"\n",
1998 __func__, subpage, len, addr, value);
blueswir1db7b5422007-05-26 17:36:03 +00001999#endif
Jan Kiszkaacc9d802013-05-26 21:55:37 +02002000 switch (len) {
2001 case 1:
2002 stb_p(buf, value);
2003 break;
2004 case 2:
2005 stw_p(buf, value);
2006 break;
2007 case 4:
2008 stl_p(buf, value);
2009 break;
Paolo Bonziniff6cff72014-12-22 13:11:39 +01002010 case 8:
2011 stq_p(buf, value);
2012 break;
Jan Kiszkaacc9d802013-05-26 21:55:37 +02002013 default:
2014 abort();
2015 }
Peter Maydell5c9eb022015-04-26 16:49:24 +01002016 return address_space_write(subpage->as, addr + subpage->base,
2017 attrs, buf, len);
blueswir1db7b5422007-05-26 17:36:03 +00002018}
2019
Paolo Bonzinic353e4c2013-05-24 14:02:39 +02002020static bool subpage_accepts(void *opaque, hwaddr addr,
Amos Kong016e9d62013-09-27 09:25:38 +08002021 unsigned len, bool is_write)
Paolo Bonzinic353e4c2013-05-24 14:02:39 +02002022{
Jan Kiszkaacc9d802013-05-26 21:55:37 +02002023 subpage_t *subpage = opaque;
Paolo Bonzinic353e4c2013-05-24 14:02:39 +02002024#if defined(DEBUG_SUBPAGE)
Amos Kong016e9d62013-09-27 09:25:38 +08002025 printf("%s: subpage %p %c len %u addr " TARGET_FMT_plx "\n",
Jan Kiszkaacc9d802013-05-26 21:55:37 +02002026 __func__, subpage, is_write ? 'w' : 'r', len, addr);
Paolo Bonzinic353e4c2013-05-24 14:02:39 +02002027#endif
2028
Jan Kiszkaacc9d802013-05-26 21:55:37 +02002029 return address_space_access_valid(subpage->as, addr + subpage->base,
Amos Kong016e9d62013-09-27 09:25:38 +08002030 len, is_write);
Paolo Bonzinic353e4c2013-05-24 14:02:39 +02002031}
2032
Avi Kivity70c68e42012-01-02 12:32:48 +02002033static const MemoryRegionOps subpage_ops = {
Peter Maydellf25a49e2015-04-26 16:49:24 +01002034 .read_with_attrs = subpage_read,
2035 .write_with_attrs = subpage_write,
Paolo Bonziniff6cff72014-12-22 13:11:39 +01002036 .impl.min_access_size = 1,
2037 .impl.max_access_size = 8,
2038 .valid.min_access_size = 1,
2039 .valid.max_access_size = 8,
Paolo Bonzinic353e4c2013-05-24 14:02:39 +02002040 .valid.accepts = subpage_accepts,
Avi Kivity70c68e42012-01-02 12:32:48 +02002041 .endianness = DEVICE_NATIVE_ENDIAN,
blueswir1db7b5422007-05-26 17:36:03 +00002042};
2043
Anthony Liguoric227f092009-10-01 16:12:16 -05002044static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
Avi Kivity5312bd82012-02-12 18:32:55 +02002045 uint16_t section)
blueswir1db7b5422007-05-26 17:36:03 +00002046{
2047 int idx, eidx;
2048
2049 if (start >= TARGET_PAGE_SIZE || end >= TARGET_PAGE_SIZE)
2050 return -1;
2051 idx = SUBPAGE_IDX(start);
2052 eidx = SUBPAGE_IDX(end);
2053#if defined(DEBUG_SUBPAGE)
Amos Kong016e9d62013-09-27 09:25:38 +08002054 printf("%s: %p start %08x end %08x idx %08x eidx %08x section %d\n",
2055 __func__, mmio, start, end, idx, eidx, section);
blueswir1db7b5422007-05-26 17:36:03 +00002056#endif
blueswir1db7b5422007-05-26 17:36:03 +00002057 for (; idx <= eidx; idx++) {
Avi Kivity5312bd82012-02-12 18:32:55 +02002058 mmio->sub_section[idx] = section;
blueswir1db7b5422007-05-26 17:36:03 +00002059 }
2060
2061 return 0;
2062}
2063
Jan Kiszkaacc9d802013-05-26 21:55:37 +02002064static subpage_t *subpage_init(AddressSpace *as, hwaddr base)
blueswir1db7b5422007-05-26 17:36:03 +00002065{
Anthony Liguoric227f092009-10-01 16:12:16 -05002066 subpage_t *mmio;
blueswir1db7b5422007-05-26 17:36:03 +00002067
Anthony Liguori7267c092011-08-20 22:09:37 -05002068 mmio = g_malloc0(sizeof(subpage_t));
aliguori1eec6142009-02-05 22:06:18 +00002069
Jan Kiszkaacc9d802013-05-26 21:55:37 +02002070 mmio->as = as;
aliguori1eec6142009-02-05 22:06:18 +00002071 mmio->base = base;
Paolo Bonzini2c9b15c2013-06-06 05:41:28 -04002072 memory_region_init_io(&mmio->iomem, NULL, &subpage_ops, mmio,
Peter Crosthwaiteb4fefef2014-06-05 23:15:52 -07002073 NULL, TARGET_PAGE_SIZE);
Avi Kivityb3b00c72012-01-02 13:20:11 +02002074 mmio->iomem.subpage = true;
blueswir1db7b5422007-05-26 17:36:03 +00002075#if defined(DEBUG_SUBPAGE)
Amos Kong016e9d62013-09-27 09:25:38 +08002076 printf("%s: %p base " TARGET_FMT_plx " len %08x\n", __func__,
2077 mmio, base, TARGET_PAGE_SIZE);
blueswir1db7b5422007-05-26 17:36:03 +00002078#endif
Liu Ping Fanb41aac42013-05-29 11:09:17 +02002079 subpage_register(mmio, 0, TARGET_PAGE_SIZE-1, PHYS_SECTION_UNASSIGNED);
blueswir1db7b5422007-05-26 17:36:03 +00002080
2081 return mmio;
2082}
2083
Peter Crosthwaitea656e222014-06-02 19:08:44 -07002084static uint16_t dummy_section(PhysPageMap *map, AddressSpace *as,
2085 MemoryRegion *mr)
Avi Kivity5312bd82012-02-12 18:32:55 +02002086{
Peter Crosthwaitea656e222014-06-02 19:08:44 -07002087 assert(as);
Avi Kivity5312bd82012-02-12 18:32:55 +02002088 MemoryRegionSection section = {
Peter Crosthwaitea656e222014-06-02 19:08:44 -07002089 .address_space = as,
Avi Kivity5312bd82012-02-12 18:32:55 +02002090 .mr = mr,
2091 .offset_within_address_space = 0,
2092 .offset_within_region = 0,
Paolo Bonzini052e87b2013-05-27 10:08:27 +02002093 .size = int128_2_64(),
Avi Kivity5312bd82012-02-12 18:32:55 +02002094 };
2095
Marcel Apfelbaum53cb28c2013-12-01 14:02:23 +02002096 return phys_section_add(map, &section);
Avi Kivity5312bd82012-02-12 18:32:55 +02002097}
2098
Paolo Bonzini9d82b5a2013-08-16 08:26:30 +02002099MemoryRegion *iotlb_to_region(CPUState *cpu, hwaddr index)
Avi Kivityaa102232012-03-08 17:06:55 +02002100{
Paolo Bonzini79e2b9a2015-01-21 12:09:14 +01002101 AddressSpaceDispatch *d = atomic_rcu_read(&cpu->memory_dispatch);
2102 MemoryRegionSection *sections = d->map.sections;
Paolo Bonzini9d82b5a2013-08-16 08:26:30 +02002103
2104 return sections[index & ~TARGET_PAGE_MASK].mr;
Avi Kivityaa102232012-03-08 17:06:55 +02002105}
2106
Avi Kivitye9179ce2009-06-14 11:38:52 +03002107static void io_mem_init(void)
2108{
Paolo Bonzini1f6245e2014-06-13 10:48:06 +02002109 memory_region_init_io(&io_mem_rom, NULL, &unassigned_mem_ops, NULL, NULL, UINT64_MAX);
Paolo Bonzini2c9b15c2013-06-06 05:41:28 -04002110 memory_region_init_io(&io_mem_unassigned, NULL, &unassigned_mem_ops, NULL,
Paolo Bonzini1f6245e2014-06-13 10:48:06 +02002111 NULL, UINT64_MAX);
Paolo Bonzini2c9b15c2013-06-06 05:41:28 -04002112 memory_region_init_io(&io_mem_notdirty, NULL, &notdirty_mem_ops, NULL,
Paolo Bonzini1f6245e2014-06-13 10:48:06 +02002113 NULL, UINT64_MAX);
Paolo Bonzini2c9b15c2013-06-06 05:41:28 -04002114 memory_region_init_io(&io_mem_watch, NULL, &watch_mem_ops, NULL,
Paolo Bonzini1f6245e2014-06-13 10:48:06 +02002115 NULL, UINT64_MAX);
Avi Kivitye9179ce2009-06-14 11:38:52 +03002116}
2117
Avi Kivityac1970f2012-10-03 16:22:53 +02002118static void mem_begin(MemoryListener *listener)
2119{
Paolo Bonzini89ae3372013-06-02 10:39:07 +02002120 AddressSpace *as = container_of(listener, AddressSpace, dispatch_listener);
Marcel Apfelbaum53cb28c2013-12-01 14:02:23 +02002121 AddressSpaceDispatch *d = g_new0(AddressSpaceDispatch, 1);
2122 uint16_t n;
2123
Peter Crosthwaitea656e222014-06-02 19:08:44 -07002124 n = dummy_section(&d->map, as, &io_mem_unassigned);
Marcel Apfelbaum53cb28c2013-12-01 14:02:23 +02002125 assert(n == PHYS_SECTION_UNASSIGNED);
Peter Crosthwaitea656e222014-06-02 19:08:44 -07002126 n = dummy_section(&d->map, as, &io_mem_notdirty);
Marcel Apfelbaum53cb28c2013-12-01 14:02:23 +02002127 assert(n == PHYS_SECTION_NOTDIRTY);
Peter Crosthwaitea656e222014-06-02 19:08:44 -07002128 n = dummy_section(&d->map, as, &io_mem_rom);
Marcel Apfelbaum53cb28c2013-12-01 14:02:23 +02002129 assert(n == PHYS_SECTION_ROM);
Peter Crosthwaitea656e222014-06-02 19:08:44 -07002130 n = dummy_section(&d->map, as, &io_mem_watch);
Marcel Apfelbaum53cb28c2013-12-01 14:02:23 +02002131 assert(n == PHYS_SECTION_WATCH);
Paolo Bonzini00752702013-05-29 12:13:54 +02002132
Michael S. Tsirkin9736e552013-11-11 14:42:43 +02002133 d->phys_map = (PhysPageEntry) { .ptr = PHYS_MAP_NODE_NIL, .skip = 1 };
Paolo Bonzini00752702013-05-29 12:13:54 +02002134 d->as = as;
2135 as->next_dispatch = d;
2136}
2137
Paolo Bonzini79e2b9a2015-01-21 12:09:14 +01002138static void address_space_dispatch_free(AddressSpaceDispatch *d)
2139{
2140 phys_sections_free(&d->map);
2141 g_free(d);
2142}
2143
Paolo Bonzini00752702013-05-29 12:13:54 +02002144static void mem_commit(MemoryListener *listener)
2145{
2146 AddressSpace *as = container_of(listener, AddressSpace, dispatch_listener);
Paolo Bonzini0475d942013-05-29 12:28:21 +02002147 AddressSpaceDispatch *cur = as->dispatch;
2148 AddressSpaceDispatch *next = as->next_dispatch;
Avi Kivityac1970f2012-10-03 16:22:53 +02002149
Marcel Apfelbaum53cb28c2013-12-01 14:02:23 +02002150 phys_page_compact_all(next, next->map.nodes_nb);
Michael S. Tsirkinb35ba302013-11-11 17:52:07 +02002151
Paolo Bonzini79e2b9a2015-01-21 12:09:14 +01002152 atomic_rcu_set(&as->dispatch, next);
Marcel Apfelbaum53cb28c2013-12-01 14:02:23 +02002153 if (cur) {
Paolo Bonzini79e2b9a2015-01-21 12:09:14 +01002154 call_rcu(cur, address_space_dispatch_free, rcu);
Marcel Apfelbaum53cb28c2013-12-01 14:02:23 +02002155 }
Paolo Bonzini9affd6f2013-05-29 12:09:47 +02002156}
2157
Avi Kivity1d711482012-10-02 18:54:45 +02002158static void tcg_commit(MemoryListener *listener)
Avi Kivity50c1e142012-02-08 21:36:02 +02002159{
Andreas Färber182735e2013-05-29 22:29:20 +02002160 CPUState *cpu;
Avi Kivity117712c2012-02-12 21:23:17 +02002161
2162 /* since each CPU stores ram addresses in its TLB cache, we must
2163 reset the modified entries */
2164 /* XXX: slow ! */
Andreas Färberbdc44642013-06-24 23:50:24 +02002165 CPU_FOREACH(cpu) {
Edgar E. Iglesias33bde2e2013-11-21 19:06:30 +01002166 /* FIXME: Disentangle the cpu.h circular files deps so we can
2167 directly get the right CPU from listener. */
2168 if (cpu->tcg_as_listener != listener) {
2169 continue;
2170 }
Paolo Bonzini76e5c762015-01-15 12:46:47 +01002171 cpu_reload_memory_map(cpu);
Avi Kivity117712c2012-02-12 21:23:17 +02002172 }
Avi Kivity50c1e142012-02-08 21:36:02 +02002173}
2174
Avi Kivityac1970f2012-10-03 16:22:53 +02002175void address_space_init_dispatch(AddressSpace *as)
2176{
Paolo Bonzini00752702013-05-29 12:13:54 +02002177 as->dispatch = NULL;
Paolo Bonzini89ae3372013-06-02 10:39:07 +02002178 as->dispatch_listener = (MemoryListener) {
Avi Kivityac1970f2012-10-03 16:22:53 +02002179 .begin = mem_begin,
Paolo Bonzini00752702013-05-29 12:13:54 +02002180 .commit = mem_commit,
Avi Kivityac1970f2012-10-03 16:22:53 +02002181 .region_add = mem_add,
2182 .region_nop = mem_add,
2183 .priority = 0,
2184 };
Paolo Bonzini89ae3372013-06-02 10:39:07 +02002185 memory_listener_register(&as->dispatch_listener, as);
Avi Kivityac1970f2012-10-03 16:22:53 +02002186}
2187
Paolo Bonzini6e48e8f2015-02-10 10:25:44 -07002188void address_space_unregister(AddressSpace *as)
2189{
2190 memory_listener_unregister(&as->dispatch_listener);
2191}
2192
Avi Kivity83f3c252012-10-07 12:59:55 +02002193void address_space_destroy_dispatch(AddressSpace *as)
2194{
2195 AddressSpaceDispatch *d = as->dispatch;
2196
Paolo Bonzini79e2b9a2015-01-21 12:09:14 +01002197 atomic_rcu_set(&as->dispatch, NULL);
2198 if (d) {
2199 call_rcu(d, address_space_dispatch_free, rcu);
2200 }
Avi Kivity83f3c252012-10-07 12:59:55 +02002201}
2202
Avi Kivity62152b82011-07-26 14:26:14 +03002203static void memory_map_init(void)
2204{
Anthony Liguori7267c092011-08-20 22:09:37 -05002205 system_memory = g_malloc(sizeof(*system_memory));
Paolo Bonzini03f49952013-11-07 17:14:36 +01002206
Paolo Bonzini57271d62013-11-07 17:14:37 +01002207 memory_region_init(system_memory, NULL, "system", UINT64_MAX);
Alexey Kardashevskiy7dca8042013-04-29 16:25:51 +00002208 address_space_init(&address_space_memory, system_memory, "memory");
Avi Kivity309cb472011-08-08 16:09:03 +03002209
Anthony Liguori7267c092011-08-20 22:09:37 -05002210 system_io = g_malloc(sizeof(*system_io));
Jan Kiszka3bb28b72013-09-02 18:43:30 +02002211 memory_region_init_io(system_io, NULL, &unassigned_io_ops, NULL, "io",
2212 65536);
Alexey Kardashevskiy7dca8042013-04-29 16:25:51 +00002213 address_space_init(&address_space_io, system_io, "I/O");
Avi Kivity62152b82011-07-26 14:26:14 +03002214}
2215
2216MemoryRegion *get_system_memory(void)
2217{
2218 return system_memory;
2219}
2220
Avi Kivity309cb472011-08-08 16:09:03 +03002221MemoryRegion *get_system_io(void)
2222{
2223 return system_io;
2224}
2225
pbrooke2eef172008-06-08 01:09:01 +00002226#endif /* !defined(CONFIG_USER_ONLY) */
2227
bellard13eb76e2004-01-24 15:23:36 +00002228/* physical memory access (slow version, mainly for debug) */
2229#if defined(CONFIG_USER_ONLY)
Andreas Färberf17ec442013-06-29 19:40:58 +02002230int cpu_memory_rw_debug(CPUState *cpu, target_ulong addr,
Paul Brooka68fe892010-03-01 00:08:59 +00002231 uint8_t *buf, int len, int is_write)
bellard13eb76e2004-01-24 15:23:36 +00002232{
2233 int l, flags;
2234 target_ulong page;
pbrook53a59602006-03-25 19:31:22 +00002235 void * p;
bellard13eb76e2004-01-24 15:23:36 +00002236
2237 while (len > 0) {
2238 page = addr & TARGET_PAGE_MASK;
2239 l = (page + TARGET_PAGE_SIZE) - addr;
2240 if (l > len)
2241 l = len;
2242 flags = page_get_flags(page);
2243 if (!(flags & PAGE_VALID))
Paul Brooka68fe892010-03-01 00:08:59 +00002244 return -1;
bellard13eb76e2004-01-24 15:23:36 +00002245 if (is_write) {
2246 if (!(flags & PAGE_WRITE))
Paul Brooka68fe892010-03-01 00:08:59 +00002247 return -1;
bellard579a97f2007-11-11 14:26:47 +00002248 /* XXX: this code should not depend on lock_user */
aurel3272fb7da2008-04-27 23:53:45 +00002249 if (!(p = lock_user(VERIFY_WRITE, addr, l, 0)))
Paul Brooka68fe892010-03-01 00:08:59 +00002250 return -1;
aurel3272fb7da2008-04-27 23:53:45 +00002251 memcpy(p, buf, l);
2252 unlock_user(p, addr, l);
bellard13eb76e2004-01-24 15:23:36 +00002253 } else {
2254 if (!(flags & PAGE_READ))
Paul Brooka68fe892010-03-01 00:08:59 +00002255 return -1;
bellard579a97f2007-11-11 14:26:47 +00002256 /* XXX: this code should not depend on lock_user */
aurel3272fb7da2008-04-27 23:53:45 +00002257 if (!(p = lock_user(VERIFY_READ, addr, l, 1)))
Paul Brooka68fe892010-03-01 00:08:59 +00002258 return -1;
aurel3272fb7da2008-04-27 23:53:45 +00002259 memcpy(buf, p, l);
aurel325b257572008-04-28 08:54:59 +00002260 unlock_user(p, addr, 0);
bellard13eb76e2004-01-24 15:23:36 +00002261 }
2262 len -= l;
2263 buf += l;
2264 addr += l;
2265 }
Paul Brooka68fe892010-03-01 00:08:59 +00002266 return 0;
bellard13eb76e2004-01-24 15:23:36 +00002267}
bellard8df1cd02005-01-28 22:37:22 +00002268
bellard13eb76e2004-01-24 15:23:36 +00002269#else
Anthony PERARD51d7a9e2012-10-03 13:49:05 +00002270
Paolo Bonzini845b6212015-03-23 11:45:53 +01002271static void invalidate_and_set_dirty(MemoryRegion *mr, hwaddr addr,
Avi Kivitya8170e52012-10-23 12:30:10 +02002272 hwaddr length)
Anthony PERARD51d7a9e2012-10-03 13:49:05 +00002273{
Paolo Bonzinie87f7772015-03-25 15:21:39 +01002274 uint8_t dirty_log_mask = memory_region_get_dirty_log_mask(mr);
2275 /* No early return if dirty_log_mask is or becomes 0, because
2276 * cpu_physical_memory_set_dirty_range will still call
2277 * xen_modified_memory.
2278 */
2279 if (dirty_log_mask) {
2280 dirty_log_mask =
2281 cpu_physical_memory_range_includes_clean(addr, length, dirty_log_mask);
Anthony PERARD51d7a9e2012-10-03 13:49:05 +00002282 }
Paolo Bonzinie87f7772015-03-25 15:21:39 +01002283 if (dirty_log_mask & (1 << DIRTY_MEMORY_CODE)) {
2284 tb_invalidate_phys_range(addr, addr + length);
2285 dirty_log_mask &= ~(1 << DIRTY_MEMORY_CODE);
2286 }
2287 cpu_physical_memory_set_dirty_range(addr, length, dirty_log_mask);
Anthony PERARD51d7a9e2012-10-03 13:49:05 +00002288}
2289
Richard Henderson23326162013-07-08 14:55:59 -07002290static int memory_access_size(MemoryRegion *mr, unsigned l, hwaddr addr)
Paolo Bonzini82f25632013-05-24 11:59:43 +02002291{
Paolo Bonzinie1622f42013-07-17 13:17:41 +02002292 unsigned access_size_max = mr->ops->valid.max_access_size;
Richard Henderson23326162013-07-08 14:55:59 -07002293
2294 /* Regions are assumed to support 1-4 byte accesses unless
2295 otherwise specified. */
Richard Henderson23326162013-07-08 14:55:59 -07002296 if (access_size_max == 0) {
2297 access_size_max = 4;
Paolo Bonzini82f25632013-05-24 11:59:43 +02002298 }
Richard Henderson23326162013-07-08 14:55:59 -07002299
2300 /* Bound the maximum access by the alignment of the address. */
2301 if (!mr->ops->impl.unaligned) {
2302 unsigned align_size_max = addr & -addr;
2303 if (align_size_max != 0 && align_size_max < access_size_max) {
2304 access_size_max = align_size_max;
2305 }
2306 }
2307
2308 /* Don't attempt accesses larger than the maximum. */
2309 if (l > access_size_max) {
2310 l = access_size_max;
2311 }
Paolo Bonzini098178f2013-07-29 14:27:39 +02002312 if (l & (l - 1)) {
2313 l = 1 << (qemu_fls(l) - 1);
2314 }
Richard Henderson23326162013-07-08 14:55:59 -07002315
2316 return l;
Paolo Bonzini82f25632013-05-24 11:59:43 +02002317}
2318
Peter Maydell5c9eb022015-04-26 16:49:24 +01002319MemTxResult address_space_rw(AddressSpace *as, hwaddr addr, MemTxAttrs attrs,
2320 uint8_t *buf, int len, bool is_write)
bellard13eb76e2004-01-24 15:23:36 +00002321{
Paolo Bonzini149f54b2013-05-24 12:59:37 +02002322 hwaddr l;
bellard13eb76e2004-01-24 15:23:36 +00002323 uint8_t *ptr;
Paolo Bonzini791af8c2013-05-24 16:10:39 +02002324 uint64_t val;
Paolo Bonzini149f54b2013-05-24 12:59:37 +02002325 hwaddr addr1;
Paolo Bonzini5c8a00c2013-05-29 12:42:00 +02002326 MemoryRegion *mr;
Peter Maydell3b643492015-04-26 16:49:23 +01002327 MemTxResult result = MEMTX_OK;
ths3b46e622007-09-17 08:09:54 +00002328
Paolo Bonzini41063e12015-03-18 14:21:43 +01002329 rcu_read_lock();
bellard13eb76e2004-01-24 15:23:36 +00002330 while (len > 0) {
Paolo Bonzini149f54b2013-05-24 12:59:37 +02002331 l = len;
Paolo Bonzini5c8a00c2013-05-29 12:42:00 +02002332 mr = address_space_translate(as, addr, &addr1, &l, is_write);
ths3b46e622007-09-17 08:09:54 +00002333
bellard13eb76e2004-01-24 15:23:36 +00002334 if (is_write) {
Paolo Bonzini5c8a00c2013-05-29 12:42:00 +02002335 if (!memory_access_is_direct(mr, is_write)) {
2336 l = memory_access_size(mr, l, addr1);
Andreas Färber4917cf42013-05-27 05:17:50 +02002337 /* XXX: could force current_cpu to NULL to avoid
bellard6a00d602005-11-21 23:25:50 +00002338 potential bugs */
Richard Henderson23326162013-07-08 14:55:59 -07002339 switch (l) {
2340 case 8:
2341 /* 64 bit write access */
2342 val = ldq_p(buf);
Peter Maydell3b643492015-04-26 16:49:23 +01002343 result |= memory_region_dispatch_write(mr, addr1, val, 8,
2344 attrs);
Richard Henderson23326162013-07-08 14:55:59 -07002345 break;
2346 case 4:
bellard1c213d12005-09-03 10:49:04 +00002347 /* 32 bit write access */
bellardc27004e2005-01-03 23:35:10 +00002348 val = ldl_p(buf);
Peter Maydell3b643492015-04-26 16:49:23 +01002349 result |= memory_region_dispatch_write(mr, addr1, val, 4,
2350 attrs);
Richard Henderson23326162013-07-08 14:55:59 -07002351 break;
2352 case 2:
bellard1c213d12005-09-03 10:49:04 +00002353 /* 16 bit write access */
bellardc27004e2005-01-03 23:35:10 +00002354 val = lduw_p(buf);
Peter Maydell3b643492015-04-26 16:49:23 +01002355 result |= memory_region_dispatch_write(mr, addr1, val, 2,
2356 attrs);
Richard Henderson23326162013-07-08 14:55:59 -07002357 break;
2358 case 1:
bellard1c213d12005-09-03 10:49:04 +00002359 /* 8 bit write access */
bellardc27004e2005-01-03 23:35:10 +00002360 val = ldub_p(buf);
Peter Maydell3b643492015-04-26 16:49:23 +01002361 result |= memory_region_dispatch_write(mr, addr1, val, 1,
2362 attrs);
Richard Henderson23326162013-07-08 14:55:59 -07002363 break;
2364 default:
2365 abort();
bellard13eb76e2004-01-24 15:23:36 +00002366 }
Paolo Bonzini2bbfa052013-05-24 12:29:54 +02002367 } else {
Paolo Bonzini5c8a00c2013-05-29 12:42:00 +02002368 addr1 += memory_region_get_ram_addr(mr);
bellard13eb76e2004-01-24 15:23:36 +00002369 /* RAM case */
pbrook5579c7f2009-04-11 14:47:08 +00002370 ptr = qemu_get_ram_ptr(addr1);
bellard13eb76e2004-01-24 15:23:36 +00002371 memcpy(ptr, buf, l);
Paolo Bonzini845b6212015-03-23 11:45:53 +01002372 invalidate_and_set_dirty(mr, addr1, l);
bellard13eb76e2004-01-24 15:23:36 +00002373 }
2374 } else {
Paolo Bonzini5c8a00c2013-05-29 12:42:00 +02002375 if (!memory_access_is_direct(mr, is_write)) {
bellard13eb76e2004-01-24 15:23:36 +00002376 /* I/O case */
Paolo Bonzini5c8a00c2013-05-29 12:42:00 +02002377 l = memory_access_size(mr, l, addr1);
Richard Henderson23326162013-07-08 14:55:59 -07002378 switch (l) {
2379 case 8:
2380 /* 64 bit read access */
Peter Maydell3b643492015-04-26 16:49:23 +01002381 result |= memory_region_dispatch_read(mr, addr1, &val, 8,
2382 attrs);
Richard Henderson23326162013-07-08 14:55:59 -07002383 stq_p(buf, val);
2384 break;
2385 case 4:
bellard13eb76e2004-01-24 15:23:36 +00002386 /* 32 bit read access */
Peter Maydell3b643492015-04-26 16:49:23 +01002387 result |= memory_region_dispatch_read(mr, addr1, &val, 4,
2388 attrs);
bellardc27004e2005-01-03 23:35:10 +00002389 stl_p(buf, val);
Richard Henderson23326162013-07-08 14:55:59 -07002390 break;
2391 case 2:
bellard13eb76e2004-01-24 15:23:36 +00002392 /* 16 bit read access */
Peter Maydell3b643492015-04-26 16:49:23 +01002393 result |= memory_region_dispatch_read(mr, addr1, &val, 2,
2394 attrs);
bellardc27004e2005-01-03 23:35:10 +00002395 stw_p(buf, val);
Richard Henderson23326162013-07-08 14:55:59 -07002396 break;
2397 case 1:
bellard1c213d12005-09-03 10:49:04 +00002398 /* 8 bit read access */
Peter Maydell3b643492015-04-26 16:49:23 +01002399 result |= memory_region_dispatch_read(mr, addr1, &val, 1,
2400 attrs);
bellardc27004e2005-01-03 23:35:10 +00002401 stb_p(buf, val);
Richard Henderson23326162013-07-08 14:55:59 -07002402 break;
2403 default:
2404 abort();
bellard13eb76e2004-01-24 15:23:36 +00002405 }
2406 } else {
2407 /* RAM case */
Paolo Bonzini5c8a00c2013-05-29 12:42:00 +02002408 ptr = qemu_get_ram_ptr(mr->ram_addr + addr1);
Avi Kivityf3705d52012-03-08 16:16:34 +02002409 memcpy(buf, ptr, l);
bellard13eb76e2004-01-24 15:23:36 +00002410 }
2411 }
2412 len -= l;
2413 buf += l;
2414 addr += l;
2415 }
Paolo Bonzini41063e12015-03-18 14:21:43 +01002416 rcu_read_unlock();
Paolo Bonzinifd8aaa72013-05-21 09:56:55 +02002417
Peter Maydell3b643492015-04-26 16:49:23 +01002418 return result;
bellard13eb76e2004-01-24 15:23:36 +00002419}
bellard8df1cd02005-01-28 22:37:22 +00002420
Peter Maydell5c9eb022015-04-26 16:49:24 +01002421MemTxResult address_space_write(AddressSpace *as, hwaddr addr, MemTxAttrs attrs,
2422 const uint8_t *buf, int len)
Avi Kivityac1970f2012-10-03 16:22:53 +02002423{
Peter Maydell5c9eb022015-04-26 16:49:24 +01002424 return address_space_rw(as, addr, attrs, (uint8_t *)buf, len, true);
Avi Kivityac1970f2012-10-03 16:22:53 +02002425}
2426
Peter Maydell5c9eb022015-04-26 16:49:24 +01002427MemTxResult address_space_read(AddressSpace *as, hwaddr addr, MemTxAttrs attrs,
2428 uint8_t *buf, int len)
Avi Kivityac1970f2012-10-03 16:22:53 +02002429{
Peter Maydell5c9eb022015-04-26 16:49:24 +01002430 return address_space_rw(as, addr, attrs, buf, len, false);
Avi Kivityac1970f2012-10-03 16:22:53 +02002431}
2432
2433
Avi Kivitya8170e52012-10-23 12:30:10 +02002434void cpu_physical_memory_rw(hwaddr addr, uint8_t *buf,
Avi Kivityac1970f2012-10-03 16:22:53 +02002435 int len, int is_write)
2436{
Peter Maydell5c9eb022015-04-26 16:49:24 +01002437 address_space_rw(&address_space_memory, addr, MEMTXATTRS_UNSPECIFIED,
2438 buf, len, is_write);
Avi Kivityac1970f2012-10-03 16:22:53 +02002439}
2440
Alexander Graf582b55a2013-12-11 14:17:44 +01002441enum write_rom_type {
2442 WRITE_DATA,
2443 FLUSH_CACHE,
2444};
2445
Edgar E. Iglesias2a221652013-12-13 16:28:52 +10002446static inline void cpu_physical_memory_write_rom_internal(AddressSpace *as,
Alexander Graf582b55a2013-12-11 14:17:44 +01002447 hwaddr addr, const uint8_t *buf, int len, enum write_rom_type type)
bellardd0ecd2a2006-04-23 17:14:48 +00002448{
Paolo Bonzini149f54b2013-05-24 12:59:37 +02002449 hwaddr l;
bellardd0ecd2a2006-04-23 17:14:48 +00002450 uint8_t *ptr;
Paolo Bonzini149f54b2013-05-24 12:59:37 +02002451 hwaddr addr1;
Paolo Bonzini5c8a00c2013-05-29 12:42:00 +02002452 MemoryRegion *mr;
ths3b46e622007-09-17 08:09:54 +00002453
Paolo Bonzini41063e12015-03-18 14:21:43 +01002454 rcu_read_lock();
bellardd0ecd2a2006-04-23 17:14:48 +00002455 while (len > 0) {
Paolo Bonzini149f54b2013-05-24 12:59:37 +02002456 l = len;
Edgar E. Iglesias2a221652013-12-13 16:28:52 +10002457 mr = address_space_translate(as, addr, &addr1, &l, true);
ths3b46e622007-09-17 08:09:54 +00002458
Paolo Bonzini5c8a00c2013-05-29 12:42:00 +02002459 if (!(memory_region_is_ram(mr) ||
2460 memory_region_is_romd(mr))) {
bellardd0ecd2a2006-04-23 17:14:48 +00002461 /* do nothing */
2462 } else {
Paolo Bonzini5c8a00c2013-05-29 12:42:00 +02002463 addr1 += memory_region_get_ram_addr(mr);
bellardd0ecd2a2006-04-23 17:14:48 +00002464 /* ROM/RAM case */
pbrook5579c7f2009-04-11 14:47:08 +00002465 ptr = qemu_get_ram_ptr(addr1);
Alexander Graf582b55a2013-12-11 14:17:44 +01002466 switch (type) {
2467 case WRITE_DATA:
2468 memcpy(ptr, buf, l);
Paolo Bonzini845b6212015-03-23 11:45:53 +01002469 invalidate_and_set_dirty(mr, addr1, l);
Alexander Graf582b55a2013-12-11 14:17:44 +01002470 break;
2471 case FLUSH_CACHE:
2472 flush_icache_range((uintptr_t)ptr, (uintptr_t)ptr + l);
2473 break;
2474 }
bellardd0ecd2a2006-04-23 17:14:48 +00002475 }
2476 len -= l;
2477 buf += l;
2478 addr += l;
2479 }
Paolo Bonzini41063e12015-03-18 14:21:43 +01002480 rcu_read_unlock();
bellardd0ecd2a2006-04-23 17:14:48 +00002481}
2482
Alexander Graf582b55a2013-12-11 14:17:44 +01002483/* used for ROM loading : can write in RAM and ROM */
Edgar E. Iglesias2a221652013-12-13 16:28:52 +10002484void cpu_physical_memory_write_rom(AddressSpace *as, hwaddr addr,
Alexander Graf582b55a2013-12-11 14:17:44 +01002485 const uint8_t *buf, int len)
2486{
Edgar E. Iglesias2a221652013-12-13 16:28:52 +10002487 cpu_physical_memory_write_rom_internal(as, addr, buf, len, WRITE_DATA);
Alexander Graf582b55a2013-12-11 14:17:44 +01002488}
2489
2490void cpu_flush_icache_range(hwaddr start, int len)
2491{
2492 /*
2493 * This function should do the same thing as an icache flush that was
2494 * triggered from within the guest. For TCG we are always cache coherent,
2495 * so there is no need to flush anything. For KVM / Xen we need to flush
2496 * the host's instruction cache at least.
2497 */
2498 if (tcg_enabled()) {
2499 return;
2500 }
2501
Edgar E. Iglesias2a221652013-12-13 16:28:52 +10002502 cpu_physical_memory_write_rom_internal(&address_space_memory,
2503 start, NULL, len, FLUSH_CACHE);
Alexander Graf582b55a2013-12-11 14:17:44 +01002504}
2505
aliguori6d16c2f2009-01-22 16:59:11 +00002506typedef struct {
Paolo Bonzinid3e71552013-06-28 17:33:29 +02002507 MemoryRegion *mr;
aliguori6d16c2f2009-01-22 16:59:11 +00002508 void *buffer;
Avi Kivitya8170e52012-10-23 12:30:10 +02002509 hwaddr addr;
2510 hwaddr len;
Fam Zhengc2cba0f2015-03-16 17:03:33 +08002511 bool in_use;
aliguori6d16c2f2009-01-22 16:59:11 +00002512} BounceBuffer;
2513
2514static BounceBuffer bounce;
2515
aliguoriba223c22009-01-22 16:59:16 +00002516typedef struct MapClient {
Fam Zhenge95205e2015-03-16 17:03:37 +08002517 QEMUBH *bh;
Blue Swirl72cf2d42009-09-12 07:36:22 +00002518 QLIST_ENTRY(MapClient) link;
aliguoriba223c22009-01-22 16:59:16 +00002519} MapClient;
2520
Fam Zheng38e047b2015-03-16 17:03:35 +08002521QemuMutex map_client_list_lock;
Blue Swirl72cf2d42009-09-12 07:36:22 +00002522static QLIST_HEAD(map_client_list, MapClient) map_client_list
2523 = QLIST_HEAD_INITIALIZER(map_client_list);
aliguoriba223c22009-01-22 16:59:16 +00002524
Fam Zhenge95205e2015-03-16 17:03:37 +08002525static void cpu_unregister_map_client_do(MapClient *client)
aliguoriba223c22009-01-22 16:59:16 +00002526{
Blue Swirl72cf2d42009-09-12 07:36:22 +00002527 QLIST_REMOVE(client, link);
Anthony Liguori7267c092011-08-20 22:09:37 -05002528 g_free(client);
aliguoriba223c22009-01-22 16:59:16 +00002529}
2530
Fam Zheng33b6c2e2015-03-16 17:03:36 +08002531static void cpu_notify_map_clients_locked(void)
aliguoriba223c22009-01-22 16:59:16 +00002532{
2533 MapClient *client;
2534
Blue Swirl72cf2d42009-09-12 07:36:22 +00002535 while (!QLIST_EMPTY(&map_client_list)) {
2536 client = QLIST_FIRST(&map_client_list);
Fam Zhenge95205e2015-03-16 17:03:37 +08002537 qemu_bh_schedule(client->bh);
2538 cpu_unregister_map_client_do(client);
aliguoriba223c22009-01-22 16:59:16 +00002539 }
2540}
2541
Fam Zhenge95205e2015-03-16 17:03:37 +08002542void cpu_register_map_client(QEMUBH *bh)
bellardd0ecd2a2006-04-23 17:14:48 +00002543{
2544 MapClient *client = g_malloc(sizeof(*client));
2545
Fam Zheng38e047b2015-03-16 17:03:35 +08002546 qemu_mutex_lock(&map_client_list_lock);
Fam Zhenge95205e2015-03-16 17:03:37 +08002547 client->bh = bh;
bellardd0ecd2a2006-04-23 17:14:48 +00002548 QLIST_INSERT_HEAD(&map_client_list, client, link);
Fam Zheng33b6c2e2015-03-16 17:03:36 +08002549 if (!atomic_read(&bounce.in_use)) {
2550 cpu_notify_map_clients_locked();
2551 }
Fam Zheng38e047b2015-03-16 17:03:35 +08002552 qemu_mutex_unlock(&map_client_list_lock);
bellardd0ecd2a2006-04-23 17:14:48 +00002553}
2554
Fam Zheng38e047b2015-03-16 17:03:35 +08002555void cpu_exec_init_all(void)
2556{
2557 qemu_mutex_init(&ram_list.mutex);
2558 memory_map_init();
2559 io_mem_init();
2560 qemu_mutex_init(&map_client_list_lock);
2561}
2562
Fam Zhenge95205e2015-03-16 17:03:37 +08002563void cpu_unregister_map_client(QEMUBH *bh)
bellardd0ecd2a2006-04-23 17:14:48 +00002564{
Fam Zhenge95205e2015-03-16 17:03:37 +08002565 MapClient *client;
bellardd0ecd2a2006-04-23 17:14:48 +00002566
Fam Zhenge95205e2015-03-16 17:03:37 +08002567 qemu_mutex_lock(&map_client_list_lock);
2568 QLIST_FOREACH(client, &map_client_list, link) {
2569 if (client->bh == bh) {
2570 cpu_unregister_map_client_do(client);
2571 break;
2572 }
2573 }
2574 qemu_mutex_unlock(&map_client_list_lock);
bellardd0ecd2a2006-04-23 17:14:48 +00002575}
2576
2577static void cpu_notify_map_clients(void)
2578{
Fam Zheng38e047b2015-03-16 17:03:35 +08002579 qemu_mutex_lock(&map_client_list_lock);
Fam Zheng33b6c2e2015-03-16 17:03:36 +08002580 cpu_notify_map_clients_locked();
Fam Zheng38e047b2015-03-16 17:03:35 +08002581 qemu_mutex_unlock(&map_client_list_lock);
aliguori6d16c2f2009-01-22 16:59:11 +00002582}
2583
Paolo Bonzini51644ab2013-04-11 15:40:59 +02002584bool address_space_access_valid(AddressSpace *as, hwaddr addr, int len, bool is_write)
2585{
Paolo Bonzini5c8a00c2013-05-29 12:42:00 +02002586 MemoryRegion *mr;
Paolo Bonzini51644ab2013-04-11 15:40:59 +02002587 hwaddr l, xlat;
2588
Paolo Bonzini41063e12015-03-18 14:21:43 +01002589 rcu_read_lock();
Paolo Bonzini51644ab2013-04-11 15:40:59 +02002590 while (len > 0) {
2591 l = len;
Paolo Bonzini5c8a00c2013-05-29 12:42:00 +02002592 mr = address_space_translate(as, addr, &xlat, &l, is_write);
2593 if (!memory_access_is_direct(mr, is_write)) {
2594 l = memory_access_size(mr, l, addr);
2595 if (!memory_region_access_valid(mr, xlat, l, is_write)) {
Paolo Bonzini51644ab2013-04-11 15:40:59 +02002596 return false;
2597 }
2598 }
2599
2600 len -= l;
2601 addr += l;
2602 }
Paolo Bonzini41063e12015-03-18 14:21:43 +01002603 rcu_read_unlock();
Paolo Bonzini51644ab2013-04-11 15:40:59 +02002604 return true;
2605}
2606
aliguori6d16c2f2009-01-22 16:59:11 +00002607/* Map a physical memory region into a host virtual address.
2608 * May map a subset of the requested range, given by and returned in *plen.
2609 * May return NULL if resources needed to perform the mapping are exhausted.
2610 * Use only for reads OR writes - not for read-modify-write operations.
aliguoriba223c22009-01-22 16:59:16 +00002611 * Use cpu_register_map_client() to know when retrying the map operation is
2612 * likely to succeed.
aliguori6d16c2f2009-01-22 16:59:11 +00002613 */
Avi Kivityac1970f2012-10-03 16:22:53 +02002614void *address_space_map(AddressSpace *as,
Avi Kivitya8170e52012-10-23 12:30:10 +02002615 hwaddr addr,
2616 hwaddr *plen,
Avi Kivityac1970f2012-10-03 16:22:53 +02002617 bool is_write)
aliguori6d16c2f2009-01-22 16:59:11 +00002618{
Avi Kivitya8170e52012-10-23 12:30:10 +02002619 hwaddr len = *plen;
Paolo Bonzinie3127ae2013-06-28 17:29:27 +02002620 hwaddr done = 0;
2621 hwaddr l, xlat, base;
2622 MemoryRegion *mr, *this_mr;
2623 ram_addr_t raddr;
aliguori6d16c2f2009-01-22 16:59:11 +00002624
Paolo Bonzinie3127ae2013-06-28 17:29:27 +02002625 if (len == 0) {
2626 return NULL;
2627 }
aliguori6d16c2f2009-01-22 16:59:11 +00002628
Paolo Bonzinie3127ae2013-06-28 17:29:27 +02002629 l = len;
Paolo Bonzini41063e12015-03-18 14:21:43 +01002630 rcu_read_lock();
Paolo Bonzinie3127ae2013-06-28 17:29:27 +02002631 mr = address_space_translate(as, addr, &xlat, &l, is_write);
Paolo Bonzini41063e12015-03-18 14:21:43 +01002632
Paolo Bonzinie3127ae2013-06-28 17:29:27 +02002633 if (!memory_access_is_direct(mr, is_write)) {
Fam Zhengc2cba0f2015-03-16 17:03:33 +08002634 if (atomic_xchg(&bounce.in_use, true)) {
Paolo Bonzini41063e12015-03-18 14:21:43 +01002635 rcu_read_unlock();
Paolo Bonzinie3127ae2013-06-28 17:29:27 +02002636 return NULL;
aliguori6d16c2f2009-01-22 16:59:11 +00002637 }
Kevin Wolfe85d9db2013-07-22 14:30:23 +02002638 /* Avoid unbounded allocations */
2639 l = MIN(l, TARGET_PAGE_SIZE);
2640 bounce.buffer = qemu_memalign(TARGET_PAGE_SIZE, l);
Paolo Bonzinie3127ae2013-06-28 17:29:27 +02002641 bounce.addr = addr;
2642 bounce.len = l;
Paolo Bonzinid3e71552013-06-28 17:33:29 +02002643
2644 memory_region_ref(mr);
2645 bounce.mr = mr;
Paolo Bonzinie3127ae2013-06-28 17:29:27 +02002646 if (!is_write) {
Peter Maydell5c9eb022015-04-26 16:49:24 +01002647 address_space_read(as, addr, MEMTXATTRS_UNSPECIFIED,
2648 bounce.buffer, l);
Stefano Stabellini8ab934f2011-06-27 18:26:06 +01002649 }
aliguori6d16c2f2009-01-22 16:59:11 +00002650
Paolo Bonzini41063e12015-03-18 14:21:43 +01002651 rcu_read_unlock();
Paolo Bonzinie3127ae2013-06-28 17:29:27 +02002652 *plen = l;
2653 return bounce.buffer;
2654 }
2655
2656 base = xlat;
2657 raddr = memory_region_get_ram_addr(mr);
2658
2659 for (;;) {
aliguori6d16c2f2009-01-22 16:59:11 +00002660 len -= l;
2661 addr += l;
Paolo Bonzinie3127ae2013-06-28 17:29:27 +02002662 done += l;
2663 if (len == 0) {
2664 break;
2665 }
2666
2667 l = len;
2668 this_mr = address_space_translate(as, addr, &xlat, &l, is_write);
2669 if (this_mr != mr || xlat != base + done) {
2670 break;
2671 }
aliguori6d16c2f2009-01-22 16:59:11 +00002672 }
Paolo Bonzinie3127ae2013-06-28 17:29:27 +02002673
Paolo Bonzinid3e71552013-06-28 17:33:29 +02002674 memory_region_ref(mr);
Paolo Bonzini41063e12015-03-18 14:21:43 +01002675 rcu_read_unlock();
Paolo Bonzinie3127ae2013-06-28 17:29:27 +02002676 *plen = done;
2677 return qemu_ram_ptr_length(raddr + base, plen);
aliguori6d16c2f2009-01-22 16:59:11 +00002678}
2679
Avi Kivityac1970f2012-10-03 16:22:53 +02002680/* Unmaps a memory region previously mapped by address_space_map().
aliguori6d16c2f2009-01-22 16:59:11 +00002681 * Will also mark the memory as dirty if is_write == 1. access_len gives
2682 * the amount of memory that was actually read or written by the caller.
2683 */
Avi Kivitya8170e52012-10-23 12:30:10 +02002684void address_space_unmap(AddressSpace *as, void *buffer, hwaddr len,
2685 int is_write, hwaddr access_len)
aliguori6d16c2f2009-01-22 16:59:11 +00002686{
2687 if (buffer != bounce.buffer) {
Paolo Bonzinid3e71552013-06-28 17:33:29 +02002688 MemoryRegion *mr;
2689 ram_addr_t addr1;
2690
2691 mr = qemu_ram_addr_from_host(buffer, &addr1);
2692 assert(mr != NULL);
aliguori6d16c2f2009-01-22 16:59:11 +00002693 if (is_write) {
Paolo Bonzini845b6212015-03-23 11:45:53 +01002694 invalidate_and_set_dirty(mr, addr1, access_len);
aliguori6d16c2f2009-01-22 16:59:11 +00002695 }
Jan Kiszka868bb332011-06-21 22:59:09 +02002696 if (xen_enabled()) {
Jan Kiszkae41d7c62011-06-21 22:59:08 +02002697 xen_invalidate_map_cache_entry(buffer);
Anthony PERARD050a0dd2010-09-16 13:57:49 +01002698 }
Paolo Bonzinid3e71552013-06-28 17:33:29 +02002699 memory_region_unref(mr);
aliguori6d16c2f2009-01-22 16:59:11 +00002700 return;
2701 }
2702 if (is_write) {
Peter Maydell5c9eb022015-04-26 16:49:24 +01002703 address_space_write(as, bounce.addr, MEMTXATTRS_UNSPECIFIED,
2704 bounce.buffer, access_len);
aliguori6d16c2f2009-01-22 16:59:11 +00002705 }
Herve Poussineauf8a83242010-01-24 21:23:56 +00002706 qemu_vfree(bounce.buffer);
aliguori6d16c2f2009-01-22 16:59:11 +00002707 bounce.buffer = NULL;
Paolo Bonzinid3e71552013-06-28 17:33:29 +02002708 memory_region_unref(bounce.mr);
Fam Zhengc2cba0f2015-03-16 17:03:33 +08002709 atomic_mb_set(&bounce.in_use, false);
aliguoriba223c22009-01-22 16:59:16 +00002710 cpu_notify_map_clients();
aliguori6d16c2f2009-01-22 16:59:11 +00002711}
bellardd0ecd2a2006-04-23 17:14:48 +00002712
Avi Kivitya8170e52012-10-23 12:30:10 +02002713void *cpu_physical_memory_map(hwaddr addr,
2714 hwaddr *plen,
Avi Kivityac1970f2012-10-03 16:22:53 +02002715 int is_write)
2716{
2717 return address_space_map(&address_space_memory, addr, plen, is_write);
2718}
2719
Avi Kivitya8170e52012-10-23 12:30:10 +02002720void cpu_physical_memory_unmap(void *buffer, hwaddr len,
2721 int is_write, hwaddr access_len)
Avi Kivityac1970f2012-10-03 16:22:53 +02002722{
2723 return address_space_unmap(&address_space_memory, buffer, len, is_write, access_len);
2724}
2725
bellard8df1cd02005-01-28 22:37:22 +00002726/* warning: addr must be aligned */
Peter Maydell50013112015-04-26 16:49:24 +01002727static inline uint32_t address_space_ldl_internal(AddressSpace *as, hwaddr addr,
2728 MemTxAttrs attrs,
2729 MemTxResult *result,
2730 enum device_endian endian)
bellard8df1cd02005-01-28 22:37:22 +00002731{
bellard8df1cd02005-01-28 22:37:22 +00002732 uint8_t *ptr;
Paolo Bonzini791af8c2013-05-24 16:10:39 +02002733 uint64_t val;
Paolo Bonzini5c8a00c2013-05-29 12:42:00 +02002734 MemoryRegion *mr;
Paolo Bonzini149f54b2013-05-24 12:59:37 +02002735 hwaddr l = 4;
2736 hwaddr addr1;
Peter Maydell50013112015-04-26 16:49:24 +01002737 MemTxResult r;
bellard8df1cd02005-01-28 22:37:22 +00002738
Paolo Bonzini41063e12015-03-18 14:21:43 +01002739 rcu_read_lock();
Edgar E. Iglesiasfdfba1a2013-11-15 14:46:38 +01002740 mr = address_space_translate(as, addr, &addr1, &l, false);
Paolo Bonzini5c8a00c2013-05-29 12:42:00 +02002741 if (l < 4 || !memory_access_is_direct(mr, false)) {
bellard8df1cd02005-01-28 22:37:22 +00002742 /* I/O case */
Peter Maydell50013112015-04-26 16:49:24 +01002743 r = memory_region_dispatch_read(mr, addr1, &val, 4, attrs);
Alexander Graf1e78bcc2011-07-06 09:09:23 +02002744#if defined(TARGET_WORDS_BIGENDIAN)
2745 if (endian == DEVICE_LITTLE_ENDIAN) {
2746 val = bswap32(val);
2747 }
2748#else
2749 if (endian == DEVICE_BIG_ENDIAN) {
2750 val = bswap32(val);
2751 }
2752#endif
bellard8df1cd02005-01-28 22:37:22 +00002753 } else {
2754 /* RAM case */
Paolo Bonzini5c8a00c2013-05-29 12:42:00 +02002755 ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(mr)
Avi Kivity06ef3522012-02-13 16:11:22 +02002756 & TARGET_PAGE_MASK)
Paolo Bonzini149f54b2013-05-24 12:59:37 +02002757 + addr1);
Alexander Graf1e78bcc2011-07-06 09:09:23 +02002758 switch (endian) {
2759 case DEVICE_LITTLE_ENDIAN:
2760 val = ldl_le_p(ptr);
2761 break;
2762 case DEVICE_BIG_ENDIAN:
2763 val = ldl_be_p(ptr);
2764 break;
2765 default:
2766 val = ldl_p(ptr);
2767 break;
2768 }
Peter Maydell50013112015-04-26 16:49:24 +01002769 r = MEMTX_OK;
2770 }
2771 if (result) {
2772 *result = r;
bellard8df1cd02005-01-28 22:37:22 +00002773 }
Paolo Bonzini41063e12015-03-18 14:21:43 +01002774 rcu_read_unlock();
bellard8df1cd02005-01-28 22:37:22 +00002775 return val;
2776}
2777
Peter Maydell50013112015-04-26 16:49:24 +01002778uint32_t address_space_ldl(AddressSpace *as, hwaddr addr,
2779 MemTxAttrs attrs, MemTxResult *result)
2780{
2781 return address_space_ldl_internal(as, addr, attrs, result,
2782 DEVICE_NATIVE_ENDIAN);
2783}
2784
2785uint32_t address_space_ldl_le(AddressSpace *as, hwaddr addr,
2786 MemTxAttrs attrs, MemTxResult *result)
2787{
2788 return address_space_ldl_internal(as, addr, attrs, result,
2789 DEVICE_LITTLE_ENDIAN);
2790}
2791
2792uint32_t address_space_ldl_be(AddressSpace *as, hwaddr addr,
2793 MemTxAttrs attrs, MemTxResult *result)
2794{
2795 return address_space_ldl_internal(as, addr, attrs, result,
2796 DEVICE_BIG_ENDIAN);
2797}
2798
Edgar E. Iglesiasfdfba1a2013-11-15 14:46:38 +01002799uint32_t ldl_phys(AddressSpace *as, hwaddr addr)
Alexander Graf1e78bcc2011-07-06 09:09:23 +02002800{
Peter Maydell50013112015-04-26 16:49:24 +01002801 return address_space_ldl(as, addr, MEMTXATTRS_UNSPECIFIED, NULL);
Alexander Graf1e78bcc2011-07-06 09:09:23 +02002802}
2803
Edgar E. Iglesiasfdfba1a2013-11-15 14:46:38 +01002804uint32_t ldl_le_phys(AddressSpace *as, hwaddr addr)
Alexander Graf1e78bcc2011-07-06 09:09:23 +02002805{
Peter Maydell50013112015-04-26 16:49:24 +01002806 return address_space_ldl_le(as, addr, MEMTXATTRS_UNSPECIFIED, NULL);
Alexander Graf1e78bcc2011-07-06 09:09:23 +02002807}
2808
Edgar E. Iglesiasfdfba1a2013-11-15 14:46:38 +01002809uint32_t ldl_be_phys(AddressSpace *as, hwaddr addr)
Alexander Graf1e78bcc2011-07-06 09:09:23 +02002810{
Peter Maydell50013112015-04-26 16:49:24 +01002811 return address_space_ldl_be(as, addr, MEMTXATTRS_UNSPECIFIED, NULL);
Alexander Graf1e78bcc2011-07-06 09:09:23 +02002812}
2813
bellard84b7b8e2005-11-28 21:19:04 +00002814/* warning: addr must be aligned */
Peter Maydell50013112015-04-26 16:49:24 +01002815static inline uint64_t address_space_ldq_internal(AddressSpace *as, hwaddr addr,
2816 MemTxAttrs attrs,
2817 MemTxResult *result,
2818 enum device_endian endian)
bellard84b7b8e2005-11-28 21:19:04 +00002819{
bellard84b7b8e2005-11-28 21:19:04 +00002820 uint8_t *ptr;
2821 uint64_t val;
Paolo Bonzini5c8a00c2013-05-29 12:42:00 +02002822 MemoryRegion *mr;
Paolo Bonzini149f54b2013-05-24 12:59:37 +02002823 hwaddr l = 8;
2824 hwaddr addr1;
Peter Maydell50013112015-04-26 16:49:24 +01002825 MemTxResult r;
bellard84b7b8e2005-11-28 21:19:04 +00002826
Paolo Bonzini41063e12015-03-18 14:21:43 +01002827 rcu_read_lock();
Edgar E. Iglesias2c174492013-12-17 14:05:40 +10002828 mr = address_space_translate(as, addr, &addr1, &l,
Paolo Bonzini5c8a00c2013-05-29 12:42:00 +02002829 false);
2830 if (l < 8 || !memory_access_is_direct(mr, false)) {
bellard84b7b8e2005-11-28 21:19:04 +00002831 /* I/O case */
Peter Maydell50013112015-04-26 16:49:24 +01002832 r = memory_region_dispatch_read(mr, addr1, &val, 8, attrs);
Paolo Bonzini968a5622013-05-24 17:58:37 +02002833#if defined(TARGET_WORDS_BIGENDIAN)
2834 if (endian == DEVICE_LITTLE_ENDIAN) {
2835 val = bswap64(val);
2836 }
2837#else
2838 if (endian == DEVICE_BIG_ENDIAN) {
2839 val = bswap64(val);
2840 }
2841#endif
bellard84b7b8e2005-11-28 21:19:04 +00002842 } else {
2843 /* RAM case */
Paolo Bonzini5c8a00c2013-05-29 12:42:00 +02002844 ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(mr)
Avi Kivity06ef3522012-02-13 16:11:22 +02002845 & TARGET_PAGE_MASK)
Paolo Bonzini149f54b2013-05-24 12:59:37 +02002846 + addr1);
Alexander Graf1e78bcc2011-07-06 09:09:23 +02002847 switch (endian) {
2848 case DEVICE_LITTLE_ENDIAN:
2849 val = ldq_le_p(ptr);
2850 break;
2851 case DEVICE_BIG_ENDIAN:
2852 val = ldq_be_p(ptr);
2853 break;
2854 default:
2855 val = ldq_p(ptr);
2856 break;
2857 }
Peter Maydell50013112015-04-26 16:49:24 +01002858 r = MEMTX_OK;
2859 }
2860 if (result) {
2861 *result = r;
bellard84b7b8e2005-11-28 21:19:04 +00002862 }
Paolo Bonzini41063e12015-03-18 14:21:43 +01002863 rcu_read_unlock();
bellard84b7b8e2005-11-28 21:19:04 +00002864 return val;
2865}
2866
Peter Maydell50013112015-04-26 16:49:24 +01002867uint64_t address_space_ldq(AddressSpace *as, hwaddr addr,
2868 MemTxAttrs attrs, MemTxResult *result)
2869{
2870 return address_space_ldq_internal(as, addr, attrs, result,
2871 DEVICE_NATIVE_ENDIAN);
2872}
2873
2874uint64_t address_space_ldq_le(AddressSpace *as, hwaddr addr,
2875 MemTxAttrs attrs, MemTxResult *result)
2876{
2877 return address_space_ldq_internal(as, addr, attrs, result,
2878 DEVICE_LITTLE_ENDIAN);
2879}
2880
2881uint64_t address_space_ldq_be(AddressSpace *as, hwaddr addr,
2882 MemTxAttrs attrs, MemTxResult *result)
2883{
2884 return address_space_ldq_internal(as, addr, attrs, result,
2885 DEVICE_BIG_ENDIAN);
2886}
2887
Edgar E. Iglesias2c174492013-12-17 14:05:40 +10002888uint64_t ldq_phys(AddressSpace *as, hwaddr addr)
Alexander Graf1e78bcc2011-07-06 09:09:23 +02002889{
Peter Maydell50013112015-04-26 16:49:24 +01002890 return address_space_ldq(as, addr, MEMTXATTRS_UNSPECIFIED, NULL);
Alexander Graf1e78bcc2011-07-06 09:09:23 +02002891}
2892
Edgar E. Iglesias2c174492013-12-17 14:05:40 +10002893uint64_t ldq_le_phys(AddressSpace *as, hwaddr addr)
Alexander Graf1e78bcc2011-07-06 09:09:23 +02002894{
Peter Maydell50013112015-04-26 16:49:24 +01002895 return address_space_ldq_le(as, addr, MEMTXATTRS_UNSPECIFIED, NULL);
Alexander Graf1e78bcc2011-07-06 09:09:23 +02002896}
2897
Edgar E. Iglesias2c174492013-12-17 14:05:40 +10002898uint64_t ldq_be_phys(AddressSpace *as, hwaddr addr)
Alexander Graf1e78bcc2011-07-06 09:09:23 +02002899{
Peter Maydell50013112015-04-26 16:49:24 +01002900 return address_space_ldq_be(as, addr, MEMTXATTRS_UNSPECIFIED, NULL);
Alexander Graf1e78bcc2011-07-06 09:09:23 +02002901}
2902
bellardaab33092005-10-30 20:48:42 +00002903/* XXX: optimize */
Peter Maydell50013112015-04-26 16:49:24 +01002904uint32_t address_space_ldub(AddressSpace *as, hwaddr addr,
2905 MemTxAttrs attrs, MemTxResult *result)
bellardaab33092005-10-30 20:48:42 +00002906{
2907 uint8_t val;
Peter Maydell50013112015-04-26 16:49:24 +01002908 MemTxResult r;
2909
2910 r = address_space_rw(as, addr, attrs, &val, 1, 0);
2911 if (result) {
2912 *result = r;
2913 }
bellardaab33092005-10-30 20:48:42 +00002914 return val;
2915}
2916
Peter Maydell50013112015-04-26 16:49:24 +01002917uint32_t ldub_phys(AddressSpace *as, hwaddr addr)
2918{
2919 return address_space_ldub(as, addr, MEMTXATTRS_UNSPECIFIED, NULL);
2920}
2921
Michael S. Tsirkin733f0b02010-04-06 14:18:19 +03002922/* warning: addr must be aligned */
Peter Maydell50013112015-04-26 16:49:24 +01002923static inline uint32_t address_space_lduw_internal(AddressSpace *as,
2924 hwaddr addr,
2925 MemTxAttrs attrs,
2926 MemTxResult *result,
2927 enum device_endian endian)
bellardaab33092005-10-30 20:48:42 +00002928{
Michael S. Tsirkin733f0b02010-04-06 14:18:19 +03002929 uint8_t *ptr;
2930 uint64_t val;
Paolo Bonzini5c8a00c2013-05-29 12:42:00 +02002931 MemoryRegion *mr;
Paolo Bonzini149f54b2013-05-24 12:59:37 +02002932 hwaddr l = 2;
2933 hwaddr addr1;
Peter Maydell50013112015-04-26 16:49:24 +01002934 MemTxResult r;
Michael S. Tsirkin733f0b02010-04-06 14:18:19 +03002935
Paolo Bonzini41063e12015-03-18 14:21:43 +01002936 rcu_read_lock();
Edgar E. Iglesias41701aa2013-12-17 14:33:56 +10002937 mr = address_space_translate(as, addr, &addr1, &l,
Paolo Bonzini5c8a00c2013-05-29 12:42:00 +02002938 false);
2939 if (l < 2 || !memory_access_is_direct(mr, false)) {
Michael S. Tsirkin733f0b02010-04-06 14:18:19 +03002940 /* I/O case */
Peter Maydell50013112015-04-26 16:49:24 +01002941 r = memory_region_dispatch_read(mr, addr1, &val, 2, attrs);
Alexander Graf1e78bcc2011-07-06 09:09:23 +02002942#if defined(TARGET_WORDS_BIGENDIAN)
2943 if (endian == DEVICE_LITTLE_ENDIAN) {
2944 val = bswap16(val);
2945 }
2946#else
2947 if (endian == DEVICE_BIG_ENDIAN) {
2948 val = bswap16(val);
2949 }
2950#endif
Michael S. Tsirkin733f0b02010-04-06 14:18:19 +03002951 } else {
2952 /* RAM case */
Paolo Bonzini5c8a00c2013-05-29 12:42:00 +02002953 ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(mr)
Avi Kivity06ef3522012-02-13 16:11:22 +02002954 & TARGET_PAGE_MASK)
Paolo Bonzini149f54b2013-05-24 12:59:37 +02002955 + addr1);
Alexander Graf1e78bcc2011-07-06 09:09:23 +02002956 switch (endian) {
2957 case DEVICE_LITTLE_ENDIAN:
2958 val = lduw_le_p(ptr);
2959 break;
2960 case DEVICE_BIG_ENDIAN:
2961 val = lduw_be_p(ptr);
2962 break;
2963 default:
2964 val = lduw_p(ptr);
2965 break;
2966 }
Peter Maydell50013112015-04-26 16:49:24 +01002967 r = MEMTX_OK;
2968 }
2969 if (result) {
2970 *result = r;
Michael S. Tsirkin733f0b02010-04-06 14:18:19 +03002971 }
Paolo Bonzini41063e12015-03-18 14:21:43 +01002972 rcu_read_unlock();
Michael S. Tsirkin733f0b02010-04-06 14:18:19 +03002973 return val;
bellardaab33092005-10-30 20:48:42 +00002974}
2975
Peter Maydell50013112015-04-26 16:49:24 +01002976uint32_t address_space_lduw(AddressSpace *as, hwaddr addr,
2977 MemTxAttrs attrs, MemTxResult *result)
2978{
2979 return address_space_lduw_internal(as, addr, attrs, result,
2980 DEVICE_NATIVE_ENDIAN);
2981}
2982
2983uint32_t address_space_lduw_le(AddressSpace *as, hwaddr addr,
2984 MemTxAttrs attrs, MemTxResult *result)
2985{
2986 return address_space_lduw_internal(as, addr, attrs, result,
2987 DEVICE_LITTLE_ENDIAN);
2988}
2989
2990uint32_t address_space_lduw_be(AddressSpace *as, hwaddr addr,
2991 MemTxAttrs attrs, MemTxResult *result)
2992{
2993 return address_space_lduw_internal(as, addr, attrs, result,
2994 DEVICE_BIG_ENDIAN);
2995}
2996
Edgar E. Iglesias41701aa2013-12-17 14:33:56 +10002997uint32_t lduw_phys(AddressSpace *as, hwaddr addr)
Alexander Graf1e78bcc2011-07-06 09:09:23 +02002998{
Peter Maydell50013112015-04-26 16:49:24 +01002999 return address_space_lduw(as, addr, MEMTXATTRS_UNSPECIFIED, NULL);
Alexander Graf1e78bcc2011-07-06 09:09:23 +02003000}
3001
Edgar E. Iglesias41701aa2013-12-17 14:33:56 +10003002uint32_t lduw_le_phys(AddressSpace *as, hwaddr addr)
Alexander Graf1e78bcc2011-07-06 09:09:23 +02003003{
Peter Maydell50013112015-04-26 16:49:24 +01003004 return address_space_lduw_le(as, addr, MEMTXATTRS_UNSPECIFIED, NULL);
Alexander Graf1e78bcc2011-07-06 09:09:23 +02003005}
3006
Edgar E. Iglesias41701aa2013-12-17 14:33:56 +10003007uint32_t lduw_be_phys(AddressSpace *as, hwaddr addr)
Alexander Graf1e78bcc2011-07-06 09:09:23 +02003008{
Peter Maydell50013112015-04-26 16:49:24 +01003009 return address_space_lduw_be(as, addr, MEMTXATTRS_UNSPECIFIED, NULL);
Alexander Graf1e78bcc2011-07-06 09:09:23 +02003010}
3011
bellard8df1cd02005-01-28 22:37:22 +00003012/* warning: addr must be aligned. The ram page is not masked as dirty
3013 and the code inside is not invalidated. It is useful if the dirty
3014 bits are used to track modified PTEs */
Peter Maydell50013112015-04-26 16:49:24 +01003015void address_space_stl_notdirty(AddressSpace *as, hwaddr addr, uint32_t val,
3016 MemTxAttrs attrs, MemTxResult *result)
bellard8df1cd02005-01-28 22:37:22 +00003017{
bellard8df1cd02005-01-28 22:37:22 +00003018 uint8_t *ptr;
Paolo Bonzini5c8a00c2013-05-29 12:42:00 +02003019 MemoryRegion *mr;
Paolo Bonzini149f54b2013-05-24 12:59:37 +02003020 hwaddr l = 4;
3021 hwaddr addr1;
Peter Maydell50013112015-04-26 16:49:24 +01003022 MemTxResult r;
Paolo Bonzini845b6212015-03-23 11:45:53 +01003023 uint8_t dirty_log_mask;
bellard8df1cd02005-01-28 22:37:22 +00003024
Paolo Bonzini41063e12015-03-18 14:21:43 +01003025 rcu_read_lock();
Edgar E. Iglesias2198a122013-11-28 10:13:41 +01003026 mr = address_space_translate(as, addr, &addr1, &l,
Paolo Bonzini5c8a00c2013-05-29 12:42:00 +02003027 true);
3028 if (l < 4 || !memory_access_is_direct(mr, true)) {
Peter Maydell50013112015-04-26 16:49:24 +01003029 r = memory_region_dispatch_write(mr, addr1, val, 4, attrs);
bellard8df1cd02005-01-28 22:37:22 +00003030 } else {
Paolo Bonzini5c8a00c2013-05-29 12:42:00 +02003031 addr1 += memory_region_get_ram_addr(mr) & TARGET_PAGE_MASK;
pbrook5579c7f2009-04-11 14:47:08 +00003032 ptr = qemu_get_ram_ptr(addr1);
bellard8df1cd02005-01-28 22:37:22 +00003033 stl_p(ptr, val);
aliguori74576192008-10-06 14:02:03 +00003034
Paolo Bonzini845b6212015-03-23 11:45:53 +01003035 dirty_log_mask = memory_region_get_dirty_log_mask(mr);
3036 dirty_log_mask &= ~(1 << DIRTY_MEMORY_CODE);
Paolo Bonzini58d27072015-03-23 11:56:01 +01003037 cpu_physical_memory_set_dirty_range(addr1, 4, dirty_log_mask);
Peter Maydell50013112015-04-26 16:49:24 +01003038 r = MEMTX_OK;
3039 }
3040 if (result) {
3041 *result = r;
bellard8df1cd02005-01-28 22:37:22 +00003042 }
Paolo Bonzini41063e12015-03-18 14:21:43 +01003043 rcu_read_unlock();
bellard8df1cd02005-01-28 22:37:22 +00003044}
3045
Peter Maydell50013112015-04-26 16:49:24 +01003046void stl_phys_notdirty(AddressSpace *as, hwaddr addr, uint32_t val)
3047{
3048 address_space_stl_notdirty(as, addr, val, MEMTXATTRS_UNSPECIFIED, NULL);
3049}
3050
bellard8df1cd02005-01-28 22:37:22 +00003051/* warning: addr must be aligned */
Peter Maydell50013112015-04-26 16:49:24 +01003052static inline void address_space_stl_internal(AddressSpace *as,
3053 hwaddr addr, uint32_t val,
3054 MemTxAttrs attrs,
3055 MemTxResult *result,
3056 enum device_endian endian)
bellard8df1cd02005-01-28 22:37:22 +00003057{
bellard8df1cd02005-01-28 22:37:22 +00003058 uint8_t *ptr;
Paolo Bonzini5c8a00c2013-05-29 12:42:00 +02003059 MemoryRegion *mr;
Paolo Bonzini149f54b2013-05-24 12:59:37 +02003060 hwaddr l = 4;
3061 hwaddr addr1;
Peter Maydell50013112015-04-26 16:49:24 +01003062 MemTxResult r;
bellard8df1cd02005-01-28 22:37:22 +00003063
Paolo Bonzini41063e12015-03-18 14:21:43 +01003064 rcu_read_lock();
Edgar E. Iglesiasab1da852013-12-17 15:07:29 +10003065 mr = address_space_translate(as, addr, &addr1, &l,
Paolo Bonzini5c8a00c2013-05-29 12:42:00 +02003066 true);
3067 if (l < 4 || !memory_access_is_direct(mr, true)) {
Alexander Graf1e78bcc2011-07-06 09:09:23 +02003068#if defined(TARGET_WORDS_BIGENDIAN)
3069 if (endian == DEVICE_LITTLE_ENDIAN) {
3070 val = bswap32(val);
3071 }
3072#else
3073 if (endian == DEVICE_BIG_ENDIAN) {
3074 val = bswap32(val);
3075 }
3076#endif
Peter Maydell50013112015-04-26 16:49:24 +01003077 r = memory_region_dispatch_write(mr, addr1, val, 4, attrs);
bellard8df1cd02005-01-28 22:37:22 +00003078 } else {
bellard8df1cd02005-01-28 22:37:22 +00003079 /* RAM case */
Paolo Bonzini5c8a00c2013-05-29 12:42:00 +02003080 addr1 += memory_region_get_ram_addr(mr) & TARGET_PAGE_MASK;
pbrook5579c7f2009-04-11 14:47:08 +00003081 ptr = qemu_get_ram_ptr(addr1);
Alexander Graf1e78bcc2011-07-06 09:09:23 +02003082 switch (endian) {
3083 case DEVICE_LITTLE_ENDIAN:
3084 stl_le_p(ptr, val);
3085 break;
3086 case DEVICE_BIG_ENDIAN:
3087 stl_be_p(ptr, val);
3088 break;
3089 default:
3090 stl_p(ptr, val);
3091 break;
3092 }
Paolo Bonzini845b6212015-03-23 11:45:53 +01003093 invalidate_and_set_dirty(mr, addr1, 4);
Peter Maydell50013112015-04-26 16:49:24 +01003094 r = MEMTX_OK;
bellard8df1cd02005-01-28 22:37:22 +00003095 }
Peter Maydell50013112015-04-26 16:49:24 +01003096 if (result) {
3097 *result = r;
3098 }
Paolo Bonzini41063e12015-03-18 14:21:43 +01003099 rcu_read_unlock();
Peter Maydell50013112015-04-26 16:49:24 +01003100}
3101
3102void address_space_stl(AddressSpace *as, hwaddr addr, uint32_t val,
3103 MemTxAttrs attrs, MemTxResult *result)
3104{
3105 address_space_stl_internal(as, addr, val, attrs, result,
3106 DEVICE_NATIVE_ENDIAN);
3107}
3108
3109void address_space_stl_le(AddressSpace *as, hwaddr addr, uint32_t val,
3110 MemTxAttrs attrs, MemTxResult *result)
3111{
3112 address_space_stl_internal(as, addr, val, attrs, result,
3113 DEVICE_LITTLE_ENDIAN);
3114}
3115
3116void address_space_stl_be(AddressSpace *as, hwaddr addr, uint32_t val,
3117 MemTxAttrs attrs, MemTxResult *result)
3118{
3119 address_space_stl_internal(as, addr, val, attrs, result,
3120 DEVICE_BIG_ENDIAN);
bellard8df1cd02005-01-28 22:37:22 +00003121}
3122
Edgar E. Iglesiasab1da852013-12-17 15:07:29 +10003123void stl_phys(AddressSpace *as, hwaddr addr, uint32_t val)
Alexander Graf1e78bcc2011-07-06 09:09:23 +02003124{
Peter Maydell50013112015-04-26 16:49:24 +01003125 address_space_stl(as, addr, val, MEMTXATTRS_UNSPECIFIED, NULL);
Alexander Graf1e78bcc2011-07-06 09:09:23 +02003126}
3127
Edgar E. Iglesiasab1da852013-12-17 15:07:29 +10003128void stl_le_phys(AddressSpace *as, hwaddr addr, uint32_t val)
Alexander Graf1e78bcc2011-07-06 09:09:23 +02003129{
Peter Maydell50013112015-04-26 16:49:24 +01003130 address_space_stl_le(as, addr, val, MEMTXATTRS_UNSPECIFIED, NULL);
Alexander Graf1e78bcc2011-07-06 09:09:23 +02003131}
3132
Edgar E. Iglesiasab1da852013-12-17 15:07:29 +10003133void stl_be_phys(AddressSpace *as, hwaddr addr, uint32_t val)
Alexander Graf1e78bcc2011-07-06 09:09:23 +02003134{
Peter Maydell50013112015-04-26 16:49:24 +01003135 address_space_stl_be(as, addr, val, MEMTXATTRS_UNSPECIFIED, NULL);
Alexander Graf1e78bcc2011-07-06 09:09:23 +02003136}
3137
bellardaab33092005-10-30 20:48:42 +00003138/* XXX: optimize */
Peter Maydell50013112015-04-26 16:49:24 +01003139void address_space_stb(AddressSpace *as, hwaddr addr, uint32_t val,
3140 MemTxAttrs attrs, MemTxResult *result)
bellardaab33092005-10-30 20:48:42 +00003141{
3142 uint8_t v = val;
Peter Maydell50013112015-04-26 16:49:24 +01003143 MemTxResult r;
3144
3145 r = address_space_rw(as, addr, attrs, &v, 1, 1);
3146 if (result) {
3147 *result = r;
3148 }
3149}
3150
3151void stb_phys(AddressSpace *as, hwaddr addr, uint32_t val)
3152{
3153 address_space_stb(as, addr, val, MEMTXATTRS_UNSPECIFIED, NULL);
bellardaab33092005-10-30 20:48:42 +00003154}
3155
Michael S. Tsirkin733f0b02010-04-06 14:18:19 +03003156/* warning: addr must be aligned */
Peter Maydell50013112015-04-26 16:49:24 +01003157static inline void address_space_stw_internal(AddressSpace *as,
3158 hwaddr addr, uint32_t val,
3159 MemTxAttrs attrs,
3160 MemTxResult *result,
3161 enum device_endian endian)
bellardaab33092005-10-30 20:48:42 +00003162{
Michael S. Tsirkin733f0b02010-04-06 14:18:19 +03003163 uint8_t *ptr;
Paolo Bonzini5c8a00c2013-05-29 12:42:00 +02003164 MemoryRegion *mr;
Paolo Bonzini149f54b2013-05-24 12:59:37 +02003165 hwaddr l = 2;
3166 hwaddr addr1;
Peter Maydell50013112015-04-26 16:49:24 +01003167 MemTxResult r;
Michael S. Tsirkin733f0b02010-04-06 14:18:19 +03003168
Paolo Bonzini41063e12015-03-18 14:21:43 +01003169 rcu_read_lock();
Edgar E. Iglesias5ce59442013-12-17 15:22:06 +10003170 mr = address_space_translate(as, addr, &addr1, &l, true);
Paolo Bonzini5c8a00c2013-05-29 12:42:00 +02003171 if (l < 2 || !memory_access_is_direct(mr, true)) {
Alexander Graf1e78bcc2011-07-06 09:09:23 +02003172#if defined(TARGET_WORDS_BIGENDIAN)
3173 if (endian == DEVICE_LITTLE_ENDIAN) {
3174 val = bswap16(val);
3175 }
3176#else
3177 if (endian == DEVICE_BIG_ENDIAN) {
3178 val = bswap16(val);
3179 }
3180#endif
Peter Maydell50013112015-04-26 16:49:24 +01003181 r = memory_region_dispatch_write(mr, addr1, val, 2, attrs);
Michael S. Tsirkin733f0b02010-04-06 14:18:19 +03003182 } else {
Michael S. Tsirkin733f0b02010-04-06 14:18:19 +03003183 /* RAM case */
Paolo Bonzini5c8a00c2013-05-29 12:42:00 +02003184 addr1 += memory_region_get_ram_addr(mr) & TARGET_PAGE_MASK;
Michael S. Tsirkin733f0b02010-04-06 14:18:19 +03003185 ptr = qemu_get_ram_ptr(addr1);
Alexander Graf1e78bcc2011-07-06 09:09:23 +02003186 switch (endian) {
3187 case DEVICE_LITTLE_ENDIAN:
3188 stw_le_p(ptr, val);
3189 break;
3190 case DEVICE_BIG_ENDIAN:
3191 stw_be_p(ptr, val);
3192 break;
3193 default:
3194 stw_p(ptr, val);
3195 break;
3196 }
Paolo Bonzini845b6212015-03-23 11:45:53 +01003197 invalidate_and_set_dirty(mr, addr1, 2);
Peter Maydell50013112015-04-26 16:49:24 +01003198 r = MEMTX_OK;
Michael S. Tsirkin733f0b02010-04-06 14:18:19 +03003199 }
Peter Maydell50013112015-04-26 16:49:24 +01003200 if (result) {
3201 *result = r;
3202 }
Paolo Bonzini41063e12015-03-18 14:21:43 +01003203 rcu_read_unlock();
Peter Maydell50013112015-04-26 16:49:24 +01003204}
3205
3206void address_space_stw(AddressSpace *as, hwaddr addr, uint32_t val,
3207 MemTxAttrs attrs, MemTxResult *result)
3208{
3209 address_space_stw_internal(as, addr, val, attrs, result,
3210 DEVICE_NATIVE_ENDIAN);
3211}
3212
3213void address_space_stw_le(AddressSpace *as, hwaddr addr, uint32_t val,
3214 MemTxAttrs attrs, MemTxResult *result)
3215{
3216 address_space_stw_internal(as, addr, val, attrs, result,
3217 DEVICE_LITTLE_ENDIAN);
3218}
3219
3220void address_space_stw_be(AddressSpace *as, hwaddr addr, uint32_t val,
3221 MemTxAttrs attrs, MemTxResult *result)
3222{
3223 address_space_stw_internal(as, addr, val, attrs, result,
3224 DEVICE_BIG_ENDIAN);
bellardaab33092005-10-30 20:48:42 +00003225}
3226
Edgar E. Iglesias5ce59442013-12-17 15:22:06 +10003227void stw_phys(AddressSpace *as, hwaddr addr, uint32_t val)
Alexander Graf1e78bcc2011-07-06 09:09:23 +02003228{
Peter Maydell50013112015-04-26 16:49:24 +01003229 address_space_stw(as, addr, val, MEMTXATTRS_UNSPECIFIED, NULL);
Alexander Graf1e78bcc2011-07-06 09:09:23 +02003230}
3231
Edgar E. Iglesias5ce59442013-12-17 15:22:06 +10003232void stw_le_phys(AddressSpace *as, hwaddr addr, uint32_t val)
Alexander Graf1e78bcc2011-07-06 09:09:23 +02003233{
Peter Maydell50013112015-04-26 16:49:24 +01003234 address_space_stw_le(as, addr, val, MEMTXATTRS_UNSPECIFIED, NULL);
Alexander Graf1e78bcc2011-07-06 09:09:23 +02003235}
3236
Edgar E. Iglesias5ce59442013-12-17 15:22:06 +10003237void stw_be_phys(AddressSpace *as, hwaddr addr, uint32_t val)
Alexander Graf1e78bcc2011-07-06 09:09:23 +02003238{
Peter Maydell50013112015-04-26 16:49:24 +01003239 address_space_stw_be(as, addr, val, MEMTXATTRS_UNSPECIFIED, NULL);
Alexander Graf1e78bcc2011-07-06 09:09:23 +02003240}
3241
bellardaab33092005-10-30 20:48:42 +00003242/* XXX: optimize */
Peter Maydell50013112015-04-26 16:49:24 +01003243void address_space_stq(AddressSpace *as, hwaddr addr, uint64_t val,
3244 MemTxAttrs attrs, MemTxResult *result)
3245{
3246 MemTxResult r;
3247 val = tswap64(val);
3248 r = address_space_rw(as, addr, attrs, (void *) &val, 8, 1);
3249 if (result) {
3250 *result = r;
3251 }
3252}
3253
3254void address_space_stq_le(AddressSpace *as, hwaddr addr, uint64_t val,
3255 MemTxAttrs attrs, MemTxResult *result)
3256{
3257 MemTxResult r;
3258 val = cpu_to_le64(val);
3259 r = address_space_rw(as, addr, attrs, (void *) &val, 8, 1);
3260 if (result) {
3261 *result = r;
3262 }
3263}
3264void address_space_stq_be(AddressSpace *as, hwaddr addr, uint64_t val,
3265 MemTxAttrs attrs, MemTxResult *result)
3266{
3267 MemTxResult r;
3268 val = cpu_to_be64(val);
3269 r = address_space_rw(as, addr, attrs, (void *) &val, 8, 1);
3270 if (result) {
3271 *result = r;
3272 }
3273}
3274
Edgar E. Iglesiasf6066042013-11-28 00:11:44 +01003275void stq_phys(AddressSpace *as, hwaddr addr, uint64_t val)
bellardaab33092005-10-30 20:48:42 +00003276{
Peter Maydell50013112015-04-26 16:49:24 +01003277 address_space_stq(as, addr, val, MEMTXATTRS_UNSPECIFIED, NULL);
bellardaab33092005-10-30 20:48:42 +00003278}
3279
Edgar E. Iglesiasf6066042013-11-28 00:11:44 +01003280void stq_le_phys(AddressSpace *as, hwaddr addr, uint64_t val)
Alexander Graf1e78bcc2011-07-06 09:09:23 +02003281{
Peter Maydell50013112015-04-26 16:49:24 +01003282 address_space_stq_le(as, addr, val, MEMTXATTRS_UNSPECIFIED, NULL);
Alexander Graf1e78bcc2011-07-06 09:09:23 +02003283}
3284
Edgar E. Iglesiasf6066042013-11-28 00:11:44 +01003285void stq_be_phys(AddressSpace *as, hwaddr addr, uint64_t val)
Alexander Graf1e78bcc2011-07-06 09:09:23 +02003286{
Peter Maydell50013112015-04-26 16:49:24 +01003287 address_space_stq_be(as, addr, val, MEMTXATTRS_UNSPECIFIED, NULL);
Alexander Graf1e78bcc2011-07-06 09:09:23 +02003288}
3289
aliguori5e2972f2009-03-28 17:51:36 +00003290/* virtual memory access for debug (includes writing to ROM) */
Andreas Färberf17ec442013-06-29 19:40:58 +02003291int cpu_memory_rw_debug(CPUState *cpu, target_ulong addr,
bellardb448f2f2004-02-25 23:24:04 +00003292 uint8_t *buf, int len, int is_write)
bellard13eb76e2004-01-24 15:23:36 +00003293{
3294 int l;
Avi Kivitya8170e52012-10-23 12:30:10 +02003295 hwaddr phys_addr;
j_mayer9b3c35e2007-04-07 11:21:28 +00003296 target_ulong page;
bellard13eb76e2004-01-24 15:23:36 +00003297
3298 while (len > 0) {
3299 page = addr & TARGET_PAGE_MASK;
Andreas Färberf17ec442013-06-29 19:40:58 +02003300 phys_addr = cpu_get_phys_page_debug(cpu, page);
bellard13eb76e2004-01-24 15:23:36 +00003301 /* if no physical page mapped, return an error */
3302 if (phys_addr == -1)
3303 return -1;
3304 l = (page + TARGET_PAGE_SIZE) - addr;
3305 if (l > len)
3306 l = len;
aliguori5e2972f2009-03-28 17:51:36 +00003307 phys_addr += (addr & ~TARGET_PAGE_MASK);
Edgar E. Iglesias2e388472013-12-13 16:31:02 +10003308 if (is_write) {
3309 cpu_physical_memory_write_rom(cpu->as, phys_addr, buf, l);
3310 } else {
Peter Maydell5c9eb022015-04-26 16:49:24 +01003311 address_space_rw(cpu->as, phys_addr, MEMTXATTRS_UNSPECIFIED,
3312 buf, l, 0);
Edgar E. Iglesias2e388472013-12-13 16:31:02 +10003313 }
bellard13eb76e2004-01-24 15:23:36 +00003314 len -= l;
3315 buf += l;
3316 addr += l;
3317 }
3318 return 0;
3319}
Paul Brooka68fe892010-03-01 00:08:59 +00003320#endif
bellard13eb76e2004-01-24 15:23:36 +00003321
Blue Swirl8e4a4242013-01-06 18:30:17 +00003322/*
3323 * A helper function for the _utterly broken_ virtio device model to find out if
3324 * it's running on a big endian machine. Don't do this at home kids!
3325 */
Greg Kurz98ed8ec2014-06-24 19:26:29 +02003326bool target_words_bigendian(void);
3327bool target_words_bigendian(void)
Blue Swirl8e4a4242013-01-06 18:30:17 +00003328{
3329#if defined(TARGET_WORDS_BIGENDIAN)
3330 return true;
3331#else
3332 return false;
3333#endif
3334}
3335
Wen Congyang76f35532012-05-07 12:04:18 +08003336#ifndef CONFIG_USER_ONLY
Avi Kivitya8170e52012-10-23 12:30:10 +02003337bool cpu_physical_memory_is_io(hwaddr phys_addr)
Wen Congyang76f35532012-05-07 12:04:18 +08003338{
Paolo Bonzini5c8a00c2013-05-29 12:42:00 +02003339 MemoryRegion*mr;
Paolo Bonzini149f54b2013-05-24 12:59:37 +02003340 hwaddr l = 1;
Paolo Bonzini41063e12015-03-18 14:21:43 +01003341 bool res;
Wen Congyang76f35532012-05-07 12:04:18 +08003342
Paolo Bonzini41063e12015-03-18 14:21:43 +01003343 rcu_read_lock();
Paolo Bonzini5c8a00c2013-05-29 12:42:00 +02003344 mr = address_space_translate(&address_space_memory,
3345 phys_addr, &phys_addr, &l, false);
Wen Congyang76f35532012-05-07 12:04:18 +08003346
Paolo Bonzini41063e12015-03-18 14:21:43 +01003347 res = !(memory_region_is_ram(mr) || memory_region_is_romd(mr));
3348 rcu_read_unlock();
3349 return res;
Wen Congyang76f35532012-05-07 12:04:18 +08003350}
Michael R. Hinesbd2fa512013-06-25 21:35:34 -04003351
Dr. David Alan Gilberte3807052015-05-21 13:24:13 +01003352int qemu_ram_foreach_block(RAMBlockIterFunc func, void *opaque)
Michael R. Hinesbd2fa512013-06-25 21:35:34 -04003353{
3354 RAMBlock *block;
Dr. David Alan Gilberte3807052015-05-21 13:24:13 +01003355 int ret = 0;
Michael R. Hinesbd2fa512013-06-25 21:35:34 -04003356
Mike Day0dc3f442013-09-05 14:41:35 -04003357 rcu_read_lock();
3358 QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
Dr. David Alan Gilberte3807052015-05-21 13:24:13 +01003359 ret = func(block->idstr, block->host, block->offset,
3360 block->used_length, opaque);
3361 if (ret) {
3362 break;
3363 }
Michael R. Hinesbd2fa512013-06-25 21:35:34 -04003364 }
Mike Day0dc3f442013-09-05 14:41:35 -04003365 rcu_read_unlock();
Dr. David Alan Gilberte3807052015-05-21 13:24:13 +01003366 return ret;
Michael R. Hinesbd2fa512013-06-25 21:35:34 -04003367}
Peter Maydellec3f8c92013-06-27 20:53:38 +01003368#endif