blob: e2e7422609b92b582ae06511a5602ab155c1b111 [file] [log] [blame]
bellardd19893d2003-06-15 19:58:51 +00001/*
2 * Host code generation
ths5fafdf22007-09-16 21:08:06 +00003 *
bellardd19893d2003-06-15 19:58:51 +00004 * Copyright (c) 2003 Fabrice Bellard
5 *
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2 of the License, or (at your option) any later version.
10 *
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
15 *
16 * You should have received a copy of the GNU Lesser General Public
Blue Swirl8167ee82009-07-16 20:47:01 +000017 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
bellardd19893d2003-06-15 19:58:51 +000018 */
Blue Swirl5b6dd862012-12-02 16:04:43 +000019#ifdef _WIN32
20#include <windows.h>
21#else
22#include <sys/types.h>
23#include <sys/mman.h>
24#endif
bellardd19893d2003-06-15 19:58:51 +000025#include <stdarg.h>
26#include <stdlib.h>
27#include <stdio.h>
28#include <string.h>
29#include <inttypes.h>
30
31#include "config.h"
bellard20543962003-06-15 23:28:43 +000032
Blue Swirl5b6dd862012-12-02 16:04:43 +000033#include "qemu-common.h"
bellardaf5ad102004-01-04 23:28:12 +000034#define NO_CPU_IO_DEFS
bellardd3eead22003-09-30 20:59:51 +000035#include "cpu.h"
Alex Bennée6db8b532014-08-01 17:08:57 +010036#include "trace.h"
Paolo Bonzini76cad712012-10-24 11:12:21 +020037#include "disas/disas.h"
bellard57fec1f2008-02-01 10:50:11 +000038#include "tcg.h"
Blue Swirl5b6dd862012-12-02 16:04:43 +000039#if defined(CONFIG_USER_ONLY)
40#include "qemu.h"
41#if defined(__FreeBSD__) || defined(__FreeBSD_kernel__)
42#include <sys/param.h>
43#if __FreeBSD_version >= 700104
44#define HAVE_KINFO_GETVMMAP
45#define sigqueue sigqueue_freebsd /* avoid redefinition */
46#include <sys/time.h>
47#include <sys/proc.h>
48#include <machine/profile.h>
49#define _KERNEL
50#include <sys/user.h>
51#undef _KERNEL
52#undef sigqueue
53#include <libutil.h>
54#endif
55#endif
Paolo Bonzini0bc3cd62013-04-08 17:29:59 +020056#else
57#include "exec/address-spaces.h"
Blue Swirl5b6dd862012-12-02 16:04:43 +000058#endif
59
Paolo Bonzini022c62c2012-12-17 18:19:49 +010060#include "exec/cputlb.h"
Blue Swirl5b6dd862012-12-02 16:04:43 +000061#include "translate-all.h"
Emilio G. Cota510a6472015-04-22 17:50:52 -040062#include "qemu/bitmap.h"
Alexey Kardashevskiy0aa09892013-04-22 17:42:50 +100063#include "qemu/timer.h"
Blue Swirl5b6dd862012-12-02 16:04:43 +000064
65//#define DEBUG_TB_INVALIDATE
66//#define DEBUG_FLUSH
67/* make various TB consistency checks */
68//#define DEBUG_TB_CHECK
69
70#if !defined(CONFIG_USER_ONLY)
71/* TB consistency checks only implemented for usermode emulation. */
72#undef DEBUG_TB_CHECK
73#endif
74
75#define SMC_BITMAP_USE_THRESHOLD 10
76
Blue Swirl5b6dd862012-12-02 16:04:43 +000077typedef struct PageDesc {
78 /* list of TBs intersecting this ram page */
79 TranslationBlock *first_tb;
80 /* in order to optimize self modifying code, we count the number
81 of lookups we do to a given page to use a bitmap */
82 unsigned int code_write_count;
Emilio G. Cota510a6472015-04-22 17:50:52 -040083 unsigned long *code_bitmap;
Blue Swirl5b6dd862012-12-02 16:04:43 +000084#if defined(CONFIG_USER_ONLY)
85 unsigned long flags;
86#endif
87} PageDesc;
88
89/* In system mode we want L1_MAP to be based on ram offsets,
90 while in user mode we want it to be based on virtual addresses. */
91#if !defined(CONFIG_USER_ONLY)
92#if HOST_LONG_BITS < TARGET_PHYS_ADDR_SPACE_BITS
93# define L1_MAP_ADDR_SPACE_BITS HOST_LONG_BITS
94#else
95# define L1_MAP_ADDR_SPACE_BITS TARGET_PHYS_ADDR_SPACE_BITS
96#endif
97#else
98# define L1_MAP_ADDR_SPACE_BITS TARGET_VIRT_ADDR_SPACE_BITS
99#endif
100
Paolo Bonzini03f49952013-11-07 17:14:36 +0100101/* Size of the L2 (and L3, etc) page tables. */
102#define V_L2_BITS 10
103#define V_L2_SIZE (1 << V_L2_BITS)
104
Blue Swirl5b6dd862012-12-02 16:04:43 +0000105/* The bits remaining after N lower levels of page tables. */
106#define V_L1_BITS_REM \
Paolo Bonzini03f49952013-11-07 17:14:36 +0100107 ((L1_MAP_ADDR_SPACE_BITS - TARGET_PAGE_BITS) % V_L2_BITS)
Blue Swirl5b6dd862012-12-02 16:04:43 +0000108
109#if V_L1_BITS_REM < 4
Paolo Bonzini03f49952013-11-07 17:14:36 +0100110#define V_L1_BITS (V_L1_BITS_REM + V_L2_BITS)
Blue Swirl5b6dd862012-12-02 16:04:43 +0000111#else
112#define V_L1_BITS V_L1_BITS_REM
113#endif
114
115#define V_L1_SIZE ((target_ulong)1 << V_L1_BITS)
116
117#define V_L1_SHIFT (L1_MAP_ADDR_SPACE_BITS - TARGET_PAGE_BITS - V_L1_BITS)
118
119uintptr_t qemu_real_host_page_size;
120uintptr_t qemu_host_page_size;
121uintptr_t qemu_host_page_mask;
122
123/* This is a multi-level map on the virtual address space.
124 The bottom level has pointers to PageDesc. */
125static void *l1_map[V_L1_SIZE];
126
bellard57fec1f2008-02-01 10:50:11 +0000127/* code generation context */
128TCGContext tcg_ctx;
bellardd19893d2003-06-15 19:58:51 +0000129
Blue Swirl5b6dd862012-12-02 16:04:43 +0000130static void tb_link_page(TranslationBlock *tb, tb_page_addr_t phys_pc,
131 tb_page_addr_t phys_page2);
Blue Swirla8a826a2012-12-04 20:16:07 +0000132static TranslationBlock *tb_find_pc(uintptr_t tc_ptr);
Blue Swirl5b6dd862012-12-02 16:04:43 +0000133
bellard57fec1f2008-02-01 10:50:11 +0000134void cpu_gen_init(void)
135{
136 tcg_context_init(&tcg_ctx);
bellard57fec1f2008-02-01 10:50:11 +0000137}
138
bellardd19893d2003-06-15 19:58:51 +0000139/* return non zero if the very first instruction is invalid so that
ths5fafdf22007-09-16 21:08:06 +0000140 the virtual CPU can trigger an exception.
bellardd19893d2003-06-15 19:58:51 +0000141
142 '*gen_code_size_ptr' contains the size of the generated code (host
143 code).
144*/
Andreas Färber9349b4f2012-03-14 01:38:32 +0100145int cpu_gen_code(CPUArchState *env, TranslationBlock *tb, int *gen_code_size_ptr)
bellardd19893d2003-06-15 19:58:51 +0000146{
bellard57fec1f2008-02-01 10:50:11 +0000147 TCGContext *s = &tcg_ctx;
Richard Henderson1813e172014-03-28 12:56:22 -0700148 tcg_insn_unit *gen_code_buf;
bellardd19893d2003-06-15 19:58:51 +0000149 int gen_code_size;
bellard57fec1f2008-02-01 10:50:11 +0000150#ifdef CONFIG_PROFILER
151 int64_t ti;
152#endif
153
154#ifdef CONFIG_PROFILER
bellardb67d9a52008-05-23 09:57:34 +0000155 s->tb_count1++; /* includes aborted translations because of
156 exceptions */
bellard57fec1f2008-02-01 10:50:11 +0000157 ti = profile_getclock();
158#endif
159 tcg_func_start(s);
bellardd19893d2003-06-15 19:58:51 +0000160
ths2cfc5f12008-07-18 18:01:29 +0000161 gen_intermediate_code(env, tb);
162
Alex Bennée6db8b532014-08-01 17:08:57 +0100163 trace_translate_block(tb, tb->pc, tb->tc_ptr);
164
bellardec6338b2007-11-08 14:25:03 +0000165 /* generate machine code */
bellard57fec1f2008-02-01 10:50:11 +0000166 gen_code_buf = tb->tc_ptr;
bellardec6338b2007-11-08 14:25:03 +0000167 tb->tb_next_offset[0] = 0xffff;
168 tb->tb_next_offset[1] = 0xffff;
bellard57fec1f2008-02-01 10:50:11 +0000169 s->tb_next_offset = tb->tb_next_offset;
bellard4cbb86e2003-09-17 22:53:29 +0000170#ifdef USE_DIRECT_JUMP
bellard57fec1f2008-02-01 10:50:11 +0000171 s->tb_jmp_offset = tb->tb_jmp_offset;
172 s->tb_next = NULL;
bellardd19893d2003-06-15 19:58:51 +0000173#else
bellard57fec1f2008-02-01 10:50:11 +0000174 s->tb_jmp_offset = NULL;
175 s->tb_next = tb->tb_next;
bellardd19893d2003-06-15 19:58:51 +0000176#endif
bellard57fec1f2008-02-01 10:50:11 +0000177
178#ifdef CONFIG_PROFILER
bellardb67d9a52008-05-23 09:57:34 +0000179 s->tb_count++;
180 s->interm_time += profile_getclock() - ti;
181 s->code_time -= profile_getclock();
bellard57fec1f2008-02-01 10:50:11 +0000182#endif
aurel3254604f72008-12-07 20:35:00 +0000183 gen_code_size = tcg_gen_code(s, gen_code_buf);
bellardd19893d2003-06-15 19:58:51 +0000184 *gen_code_size_ptr = gen_code_size;
bellard57fec1f2008-02-01 10:50:11 +0000185#ifdef CONFIG_PROFILER
bellardb67d9a52008-05-23 09:57:34 +0000186 s->code_time += profile_getclock();
187 s->code_in_len += tb->size;
188 s->code_out_len += gen_code_size;
bellard57fec1f2008-02-01 10:50:11 +0000189#endif
190
bellardd19893d2003-06-15 19:58:51 +0000191#ifdef DEBUG_DISAS
aliguori8fec2b82009-01-15 22:36:53 +0000192 if (qemu_loglevel_mask(CPU_LOG_TB_OUT_ASM)) {
Richard Henderson1813e172014-03-28 12:56:22 -0700193 qemu_log("OUT: [size=%d]\n", gen_code_size);
194 log_disas(tb->tc_ptr, gen_code_size);
aliguori93fcfe32009-01-15 22:34:14 +0000195 qemu_log("\n");
aliguori31b1a7b2009-01-15 22:35:09 +0000196 qemu_log_flush();
bellardd19893d2003-06-15 19:58:51 +0000197 }
198#endif
199 return 0;
200}
201
ths5fafdf22007-09-16 21:08:06 +0000202/* The cpu state corresponding to 'searched_pc' is restored.
bellardd19893d2003-06-15 19:58:51 +0000203 */
Andreas Färber74f10512013-09-01 17:02:58 +0200204static int cpu_restore_state_from_tb(CPUState *cpu, TranslationBlock *tb,
Blue Swirla8a826a2012-12-04 20:16:07 +0000205 uintptr_t searched_pc)
bellardd19893d2003-06-15 19:58:51 +0000206{
Andreas Färber74f10512013-09-01 17:02:58 +0200207 CPUArchState *env = cpu->env_ptr;
bellard57fec1f2008-02-01 10:50:11 +0000208 TCGContext *s = &tcg_ctx;
209 int j;
Stefan Weil6375e092012-04-06 22:26:15 +0200210 uintptr_t tc_ptr;
bellard57fec1f2008-02-01 10:50:11 +0000211#ifdef CONFIG_PROFILER
212 int64_t ti;
213#endif
214
215#ifdef CONFIG_PROFILER
216 ti = profile_getclock();
217#endif
218 tcg_func_start(s);
bellardd19893d2003-06-15 19:58:51 +0000219
ths2cfc5f12008-07-18 18:01:29 +0000220 gen_intermediate_code_pc(env, tb);
ths3b46e622007-09-17 08:09:54 +0000221
Paolo Bonzinibd792552014-11-26 13:39:59 +0300222 if (tb->cflags & CF_USE_ICOUNT) {
pbrook2e70f6e2008-06-29 01:03:05 +0000223 /* Reset the cycle counter to the start of the block. */
Andreas Färber28ecfd72013-08-26 05:51:49 +0200224 cpu->icount_decr.u16.low += tb->icount;
pbrook2e70f6e2008-06-29 01:03:05 +0000225 /* Clear the IO flag. */
Andreas Färber99df7dc2013-08-26 05:15:23 +0200226 cpu->can_do_io = 0;
pbrook2e70f6e2008-06-29 01:03:05 +0000227 }
228
bellardd19893d2003-06-15 19:58:51 +0000229 /* find opc index corresponding to search_pc */
Stefan Weil6375e092012-04-06 22:26:15 +0200230 tc_ptr = (uintptr_t)tb->tc_ptr;
bellardd19893d2003-06-15 19:58:51 +0000231 if (searched_pc < tc_ptr)
232 return -1;
bellard57fec1f2008-02-01 10:50:11 +0000233
234 s->tb_next_offset = tb->tb_next_offset;
235#ifdef USE_DIRECT_JUMP
236 s->tb_jmp_offset = tb->tb_jmp_offset;
237 s->tb_next = NULL;
238#else
239 s->tb_jmp_offset = NULL;
240 s->tb_next = tb->tb_next;
241#endif
Richard Henderson1813e172014-03-28 12:56:22 -0700242 j = tcg_gen_code_search_pc(s, (tcg_insn_unit *)tc_ptr,
243 searched_pc - tc_ptr);
bellard57fec1f2008-02-01 10:50:11 +0000244 if (j < 0)
245 return -1;
bellardd19893d2003-06-15 19:58:51 +0000246 /* now find start of instruction before */
Evgeny Voevodinab1103d2012-11-21 11:43:06 +0400247 while (s->gen_opc_instr_start[j] == 0) {
bellardd19893d2003-06-15 19:58:51 +0000248 j--;
Evgeny Voevodinab1103d2012-11-21 11:43:06 +0400249 }
Andreas Färber28ecfd72013-08-26 05:51:49 +0200250 cpu->icount_decr.u16.low -= s->gen_opc_icount[j];
ths3b46e622007-09-17 08:09:54 +0000251
Stefan Weile87b7cb2011-04-18 06:39:52 +0000252 restore_state_to_opc(env, tb, j);
bellard57fec1f2008-02-01 10:50:11 +0000253
254#ifdef CONFIG_PROFILER
bellardb67d9a52008-05-23 09:57:34 +0000255 s->restore_time += profile_getclock() - ti;
256 s->restore_count++;
bellard57fec1f2008-02-01 10:50:11 +0000257#endif
bellardd19893d2003-06-15 19:58:51 +0000258 return 0;
259}
Blue Swirl5b6dd862012-12-02 16:04:43 +0000260
Andreas Färber3f38f302013-09-01 16:51:34 +0200261bool cpu_restore_state(CPUState *cpu, uintptr_t retaddr)
Blue Swirla8a826a2012-12-04 20:16:07 +0000262{
263 TranslationBlock *tb;
264
265 tb = tb_find_pc(retaddr);
266 if (tb) {
Andreas Färber74f10512013-09-01 17:02:58 +0200267 cpu_restore_state_from_tb(cpu, tb, retaddr);
Pavel Dovgalyukd8a499f2014-11-26 13:40:16 +0300268 if (tb->cflags & CF_NOCACHE) {
269 /* one-shot translation, invalidate it immediately */
270 cpu->current_tb = NULL;
271 tb_phys_invalidate(tb, -1);
272 tb_free(tb);
273 }
Blue Swirla8a826a2012-12-04 20:16:07 +0000274 return true;
275 }
276 return false;
277}
278
Blue Swirl5b6dd862012-12-02 16:04:43 +0000279#ifdef _WIN32
SeokYeon Hwang2d8ac5e2014-12-23 22:26:54 +0000280static __attribute__((unused)) void map_exec(void *addr, long size)
Blue Swirl5b6dd862012-12-02 16:04:43 +0000281{
282 DWORD old_protect;
283 VirtualProtect(addr, size,
284 PAGE_EXECUTE_READWRITE, &old_protect);
285}
286#else
SeokYeon Hwang2d8ac5e2014-12-23 22:26:54 +0000287static __attribute__((unused)) void map_exec(void *addr, long size)
Blue Swirl5b6dd862012-12-02 16:04:43 +0000288{
289 unsigned long start, end, page_size;
290
291 page_size = getpagesize();
292 start = (unsigned long)addr;
293 start &= ~(page_size - 1);
294
295 end = (unsigned long)addr + size;
296 end += page_size - 1;
297 end &= ~(page_size - 1);
298
299 mprotect((void *)start, end - start,
300 PROT_READ | PROT_WRITE | PROT_EXEC);
301}
302#endif
303
Alexey Kardashevskiy47c16ed2014-01-17 11:12:07 -0700304void page_size_init(void)
Blue Swirl5b6dd862012-12-02 16:04:43 +0000305{
306 /* NOTE: we can always suppose that qemu_host_page_size >=
307 TARGET_PAGE_SIZE */
Blue Swirl5b6dd862012-12-02 16:04:43 +0000308 qemu_real_host_page_size = getpagesize();
Blue Swirl5b6dd862012-12-02 16:04:43 +0000309 if (qemu_host_page_size == 0) {
310 qemu_host_page_size = qemu_real_host_page_size;
311 }
312 if (qemu_host_page_size < TARGET_PAGE_SIZE) {
313 qemu_host_page_size = TARGET_PAGE_SIZE;
314 }
315 qemu_host_page_mask = ~(qemu_host_page_size - 1);
Alexey Kardashevskiy47c16ed2014-01-17 11:12:07 -0700316}
Blue Swirl5b6dd862012-12-02 16:04:43 +0000317
Alexey Kardashevskiy47c16ed2014-01-17 11:12:07 -0700318static void page_init(void)
319{
320 page_size_init();
Blue Swirl5b6dd862012-12-02 16:04:43 +0000321#if defined(CONFIG_BSD) && defined(CONFIG_USER_ONLY)
322 {
323#ifdef HAVE_KINFO_GETVMMAP
324 struct kinfo_vmentry *freep;
325 int i, cnt;
326
327 freep = kinfo_getvmmap(getpid(), &cnt);
328 if (freep) {
329 mmap_lock();
330 for (i = 0; i < cnt; i++) {
331 unsigned long startaddr, endaddr;
332
333 startaddr = freep[i].kve_start;
334 endaddr = freep[i].kve_end;
335 if (h2g_valid(startaddr)) {
336 startaddr = h2g(startaddr) & TARGET_PAGE_MASK;
337
338 if (h2g_valid(endaddr)) {
339 endaddr = h2g(endaddr);
340 page_set_flags(startaddr, endaddr, PAGE_RESERVED);
341 } else {
342#if TARGET_ABI_BITS <= L1_MAP_ADDR_SPACE_BITS
343 endaddr = ~0ul;
344 page_set_flags(startaddr, endaddr, PAGE_RESERVED);
345#endif
346 }
347 }
348 }
349 free(freep);
350 mmap_unlock();
351 }
352#else
353 FILE *f;
354
355 last_brk = (unsigned long)sbrk(0);
356
357 f = fopen("/compat/linux/proc/self/maps", "r");
358 if (f) {
359 mmap_lock();
360
361 do {
362 unsigned long startaddr, endaddr;
363 int n;
364
365 n = fscanf(f, "%lx-%lx %*[^\n]\n", &startaddr, &endaddr);
366
367 if (n == 2 && h2g_valid(startaddr)) {
368 startaddr = h2g(startaddr) & TARGET_PAGE_MASK;
369
370 if (h2g_valid(endaddr)) {
371 endaddr = h2g(endaddr);
372 } else {
373 endaddr = ~0ul;
374 }
375 page_set_flags(startaddr, endaddr, PAGE_RESERVED);
376 }
377 } while (!feof(f));
378
379 fclose(f);
380 mmap_unlock();
381 }
382#endif
383 }
384#endif
385}
386
387static PageDesc *page_find_alloc(tb_page_addr_t index, int alloc)
388{
389 PageDesc *pd;
390 void **lp;
391 int i;
392
Blue Swirl5b6dd862012-12-02 16:04:43 +0000393 /* Level 1. Always allocated. */
394 lp = l1_map + ((index >> V_L1_SHIFT) & (V_L1_SIZE - 1));
395
396 /* Level 2..N-1. */
Paolo Bonzini03f49952013-11-07 17:14:36 +0100397 for (i = V_L1_SHIFT / V_L2_BITS - 1; i > 0; i--) {
Blue Swirl5b6dd862012-12-02 16:04:43 +0000398 void **p = *lp;
399
400 if (p == NULL) {
401 if (!alloc) {
402 return NULL;
403 }
Emilio G. Cotae3a0abf2015-04-09 16:07:33 -0400404 p = g_new0(void *, V_L2_SIZE);
Blue Swirl5b6dd862012-12-02 16:04:43 +0000405 *lp = p;
406 }
407
Paolo Bonzini03f49952013-11-07 17:14:36 +0100408 lp = p + ((index >> (i * V_L2_BITS)) & (V_L2_SIZE - 1));
Blue Swirl5b6dd862012-12-02 16:04:43 +0000409 }
410
411 pd = *lp;
412 if (pd == NULL) {
413 if (!alloc) {
414 return NULL;
415 }
Emilio G. Cotae3a0abf2015-04-09 16:07:33 -0400416 pd = g_new0(PageDesc, V_L2_SIZE);
Blue Swirl5b6dd862012-12-02 16:04:43 +0000417 *lp = pd;
418 }
419
Paolo Bonzini03f49952013-11-07 17:14:36 +0100420 return pd + (index & (V_L2_SIZE - 1));
Blue Swirl5b6dd862012-12-02 16:04:43 +0000421}
422
423static inline PageDesc *page_find(tb_page_addr_t index)
424{
425 return page_find_alloc(index, 0);
426}
427
428#if !defined(CONFIG_USER_ONLY)
429#define mmap_lock() do { } while (0)
430#define mmap_unlock() do { } while (0)
431#endif
432
433#if defined(CONFIG_USER_ONLY)
434/* Currently it is not recommended to allocate big chunks of data in
435 user mode. It will change when a dedicated libc will be used. */
436/* ??? 64-bit hosts ought to have no problem mmaping data outside the
437 region in which the guest needs to run. Revisit this. */
438#define USE_STATIC_CODE_GEN_BUFFER
439#endif
440
441/* ??? Should configure for this, not list operating systems here. */
442#if (defined(__linux__) \
443 || defined(__FreeBSD__) || defined(__FreeBSD_kernel__) \
444 || defined(__DragonFly__) || defined(__OpenBSD__) \
445 || defined(__NetBSD__))
446# define USE_MMAP
447#endif
448
449/* Minimum size of the code gen buffer. This number is randomly chosen,
450 but not so small that we can't have a fair number of TB's live. */
451#define MIN_CODE_GEN_BUFFER_SIZE (1024u * 1024)
452
453/* Maximum size of the code gen buffer we'd like to use. Unless otherwise
454 indicated, this is constrained by the range of direct branches on the
455 host cpu, as used by the TCG implementation of goto_tb. */
456#if defined(__x86_64__)
457# define MAX_CODE_GEN_BUFFER_SIZE (2ul * 1024 * 1024 * 1024)
458#elif defined(__sparc__)
459# define MAX_CODE_GEN_BUFFER_SIZE (2ul * 1024 * 1024 * 1024)
Claudio Fontana4a136e02013-06-12 16:20:22 +0100460#elif defined(__aarch64__)
461# define MAX_CODE_GEN_BUFFER_SIZE (128ul * 1024 * 1024)
Blue Swirl5b6dd862012-12-02 16:04:43 +0000462#elif defined(__arm__)
463# define MAX_CODE_GEN_BUFFER_SIZE (16u * 1024 * 1024)
464#elif defined(__s390x__)
465 /* We have a +- 4GB range on the branches; leave some slop. */
466# define MAX_CODE_GEN_BUFFER_SIZE (3ul * 1024 * 1024 * 1024)
Richard Henderson479eb122014-04-24 08:25:03 -0700467#elif defined(__mips__)
468 /* We have a 256MB branch region, but leave room to make sure the
469 main executable is also within that region. */
470# define MAX_CODE_GEN_BUFFER_SIZE (128ul * 1024 * 1024)
Blue Swirl5b6dd862012-12-02 16:04:43 +0000471#else
472# define MAX_CODE_GEN_BUFFER_SIZE ((size_t)-1)
473#endif
474
475#define DEFAULT_CODE_GEN_BUFFER_SIZE_1 (32u * 1024 * 1024)
476
477#define DEFAULT_CODE_GEN_BUFFER_SIZE \
478 (DEFAULT_CODE_GEN_BUFFER_SIZE_1 < MAX_CODE_GEN_BUFFER_SIZE \
479 ? DEFAULT_CODE_GEN_BUFFER_SIZE_1 : MAX_CODE_GEN_BUFFER_SIZE)
480
481static inline size_t size_code_gen_buffer(size_t tb_size)
482{
483 /* Size the buffer. */
484 if (tb_size == 0) {
485#ifdef USE_STATIC_CODE_GEN_BUFFER
486 tb_size = DEFAULT_CODE_GEN_BUFFER_SIZE;
487#else
488 /* ??? Needs adjustments. */
489 /* ??? If we relax the requirement that CONFIG_USER_ONLY use the
490 static buffer, we could size this on RESERVED_VA, on the text
491 segment size of the executable, or continue to use the default. */
492 tb_size = (unsigned long)(ram_size / 4);
493#endif
494 }
495 if (tb_size < MIN_CODE_GEN_BUFFER_SIZE) {
496 tb_size = MIN_CODE_GEN_BUFFER_SIZE;
497 }
498 if (tb_size > MAX_CODE_GEN_BUFFER_SIZE) {
499 tb_size = MAX_CODE_GEN_BUFFER_SIZE;
500 }
Evgeny Voevodin0b0d3322013-02-01 01:47:22 +0700501 tcg_ctx.code_gen_buffer_size = tb_size;
Blue Swirl5b6dd862012-12-02 16:04:43 +0000502 return tb_size;
503}
504
Richard Henderson483c76e2014-04-24 09:16:07 -0700505#ifdef __mips__
506/* In order to use J and JAL within the code_gen_buffer, we require
507 that the buffer not cross a 256MB boundary. */
508static inline bool cross_256mb(void *addr, size_t size)
509{
510 return ((uintptr_t)addr ^ ((uintptr_t)addr + size)) & 0xf0000000;
511}
512
513/* We weren't able to allocate a buffer without crossing that boundary,
514 so make do with the larger portion of the buffer that doesn't cross.
515 Returns the new base of the buffer, and adjusts code_gen_buffer_size. */
516static inline void *split_cross_256mb(void *buf1, size_t size1)
517{
518 void *buf2 = (void *)(((uintptr_t)buf1 + size1) & 0xf0000000);
519 size_t size2 = buf1 + size1 - buf2;
520
521 size1 = buf2 - buf1;
522 if (size1 < size2) {
523 size1 = size2;
524 buf1 = buf2;
525 }
526
527 tcg_ctx.code_gen_buffer_size = size1;
528 return buf1;
529}
530#endif
531
Blue Swirl5b6dd862012-12-02 16:04:43 +0000532#ifdef USE_STATIC_CODE_GEN_BUFFER
533static uint8_t static_code_gen_buffer[DEFAULT_CODE_GEN_BUFFER_SIZE]
534 __attribute__((aligned(CODE_GEN_ALIGN)));
535
536static inline void *alloc_code_gen_buffer(void)
537{
Richard Henderson483c76e2014-04-24 09:16:07 -0700538 void *buf = static_code_gen_buffer;
539#ifdef __mips__
540 if (cross_256mb(buf, tcg_ctx.code_gen_buffer_size)) {
541 buf = split_cross_256mb(buf, tcg_ctx.code_gen_buffer_size);
542 }
543#endif
544 map_exec(buf, tcg_ctx.code_gen_buffer_size);
545 return buf;
Blue Swirl5b6dd862012-12-02 16:04:43 +0000546}
547#elif defined(USE_MMAP)
548static inline void *alloc_code_gen_buffer(void)
549{
550 int flags = MAP_PRIVATE | MAP_ANONYMOUS;
551 uintptr_t start = 0;
552 void *buf;
553
554 /* Constrain the position of the buffer based on the host cpu.
555 Note that these addresses are chosen in concert with the
556 addresses assigned in the relevant linker script file. */
557# if defined(__PIE__) || defined(__PIC__)
558 /* Don't bother setting a preferred location if we're building
559 a position-independent executable. We're more likely to get
560 an address near the main executable if we let the kernel
561 choose the address. */
562# elif defined(__x86_64__) && defined(MAP_32BIT)
563 /* Force the memory down into low memory with the executable.
564 Leave the choice of exact location with the kernel. */
565 flags |= MAP_32BIT;
566 /* Cannot expect to map more than 800MB in low memory. */
Evgeny Voevodin0b0d3322013-02-01 01:47:22 +0700567 if (tcg_ctx.code_gen_buffer_size > 800u * 1024 * 1024) {
568 tcg_ctx.code_gen_buffer_size = 800u * 1024 * 1024;
Blue Swirl5b6dd862012-12-02 16:04:43 +0000569 }
570# elif defined(__sparc__)
571 start = 0x40000000ul;
572# elif defined(__s390x__)
573 start = 0x90000000ul;
Richard Henderson479eb122014-04-24 08:25:03 -0700574# elif defined(__mips__)
575 /* ??? We ought to more explicitly manage layout for softmmu too. */
576# ifdef CONFIG_USER_ONLY
577 start = 0x68000000ul;
578# elif _MIPS_SIM == _ABI64
579 start = 0x128000000ul;
580# else
581 start = 0x08000000ul;
582# endif
Blue Swirl5b6dd862012-12-02 16:04:43 +0000583# endif
584
Evgeny Voevodin0b0d3322013-02-01 01:47:22 +0700585 buf = mmap((void *)start, tcg_ctx.code_gen_buffer_size,
Blue Swirl5b6dd862012-12-02 16:04:43 +0000586 PROT_WRITE | PROT_READ | PROT_EXEC, flags, -1, 0);
Richard Henderson483c76e2014-04-24 09:16:07 -0700587 if (buf == MAP_FAILED) {
588 return NULL;
589 }
590
591#ifdef __mips__
592 if (cross_256mb(buf, tcg_ctx.code_gen_buffer_size)) {
Stefan Weil5d831be2014-06-13 20:42:57 +0200593 /* Try again, with the original still mapped, to avoid re-acquiring
Richard Henderson483c76e2014-04-24 09:16:07 -0700594 that 256mb crossing. This time don't specify an address. */
595 size_t size2, size1 = tcg_ctx.code_gen_buffer_size;
596 void *buf2 = mmap(NULL, size1, PROT_WRITE | PROT_READ | PROT_EXEC,
597 flags, -1, 0);
598 if (buf2 != MAP_FAILED) {
599 if (!cross_256mb(buf2, size1)) {
600 /* Success! Use the new buffer. */
601 munmap(buf, size1);
602 return buf2;
603 }
604 /* Failure. Work with what we had. */
605 munmap(buf2, size1);
606 }
607
608 /* Split the original buffer. Free the smaller half. */
609 buf2 = split_cross_256mb(buf, size1);
610 size2 = tcg_ctx.code_gen_buffer_size;
611 munmap(buf + (buf == buf2 ? size2 : 0), size1 - size2);
612 return buf2;
613 }
614#endif
615
616 return buf;
Blue Swirl5b6dd862012-12-02 16:04:43 +0000617}
618#else
619static inline void *alloc_code_gen_buffer(void)
620{
Markus Armbruster8b98ade2015-02-04 11:26:07 +0100621 void *buf = g_try_malloc(tcg_ctx.code_gen_buffer_size);
Blue Swirl5b6dd862012-12-02 16:04:43 +0000622
Richard Henderson483c76e2014-04-24 09:16:07 -0700623 if (buf == NULL) {
624 return NULL;
Blue Swirl5b6dd862012-12-02 16:04:43 +0000625 }
Richard Henderson483c76e2014-04-24 09:16:07 -0700626
627#ifdef __mips__
628 if (cross_256mb(buf, tcg_ctx.code_gen_buffer_size)) {
629 void *buf2 = g_malloc(tcg_ctx.code_gen_buffer_size);
630 if (buf2 != NULL && !cross_256mb(buf2, size1)) {
631 /* Success! Use the new buffer. */
632 free(buf);
633 buf = buf2;
634 } else {
635 /* Failure. Work with what we had. Since this is malloc
636 and not mmap, we can't free the other half. */
637 free(buf2);
638 buf = split_cross_256mb(buf, tcg_ctx.code_gen_buffer_size);
639 }
640 }
641#endif
642
643 map_exec(buf, tcg_ctx.code_gen_buffer_size);
Blue Swirl5b6dd862012-12-02 16:04:43 +0000644 return buf;
645}
646#endif /* USE_STATIC_CODE_GEN_BUFFER, USE_MMAP */
647
648static inline void code_gen_alloc(size_t tb_size)
649{
Evgeny Voevodin0b0d3322013-02-01 01:47:22 +0700650 tcg_ctx.code_gen_buffer_size = size_code_gen_buffer(tb_size);
651 tcg_ctx.code_gen_buffer = alloc_code_gen_buffer();
652 if (tcg_ctx.code_gen_buffer == NULL) {
Blue Swirl5b6dd862012-12-02 16:04:43 +0000653 fprintf(stderr, "Could not allocate dynamic translator buffer\n");
654 exit(1);
655 }
656
Evgeny Voevodin0b0d3322013-02-01 01:47:22 +0700657 qemu_madvise(tcg_ctx.code_gen_buffer, tcg_ctx.code_gen_buffer_size,
658 QEMU_MADV_HUGEPAGE);
Blue Swirl5b6dd862012-12-02 16:04:43 +0000659
660 /* Steal room for the prologue at the end of the buffer. This ensures
661 (via the MAX_CODE_GEN_BUFFER_SIZE limits above) that direct branches
662 from TB's to the prologue are going to be in range. It also means
663 that we don't need to mark (additional) portions of the data segment
664 as executable. */
Evgeny Voevodin0b0d3322013-02-01 01:47:22 +0700665 tcg_ctx.code_gen_prologue = tcg_ctx.code_gen_buffer +
666 tcg_ctx.code_gen_buffer_size - 1024;
667 tcg_ctx.code_gen_buffer_size -= 1024;
Blue Swirl5b6dd862012-12-02 16:04:43 +0000668
Evgeny Voevodin0b0d3322013-02-01 01:47:22 +0700669 tcg_ctx.code_gen_buffer_max_size = tcg_ctx.code_gen_buffer_size -
Blue Swirl5b6dd862012-12-02 16:04:43 +0000670 (TCG_MAX_OP_SIZE * OPC_BUF_SIZE);
Evgeny Voevodin0b0d3322013-02-01 01:47:22 +0700671 tcg_ctx.code_gen_max_blocks = tcg_ctx.code_gen_buffer_size /
672 CODE_GEN_AVG_BLOCK_SIZE;
Evgeny Voevodin5e5f07e2013-02-01 01:47:23 +0700673 tcg_ctx.tb_ctx.tbs =
674 g_malloc(tcg_ctx.code_gen_max_blocks * sizeof(TranslationBlock));
Blue Swirl5b6dd862012-12-02 16:04:43 +0000675}
676
677/* Must be called before using the QEMU cpus. 'tb_size' is the size
678 (in bytes) allocated to the translation buffer. Zero means default
679 size. */
680void tcg_exec_init(unsigned long tb_size)
681{
682 cpu_gen_init();
683 code_gen_alloc(tb_size);
Evgeny Voevodin0b0d3322013-02-01 01:47:22 +0700684 tcg_ctx.code_gen_ptr = tcg_ctx.code_gen_buffer;
685 tcg_register_jit(tcg_ctx.code_gen_buffer, tcg_ctx.code_gen_buffer_size);
Blue Swirl5b6dd862012-12-02 16:04:43 +0000686 page_init();
687#if !defined(CONFIG_USER_ONLY) || !defined(CONFIG_USE_GUEST_BASE)
688 /* There's no guest base to take into account, so go ahead and
689 initialize the prologue now. */
690 tcg_prologue_init(&tcg_ctx);
691#endif
692}
693
694bool tcg_enabled(void)
695{
Evgeny Voevodin0b0d3322013-02-01 01:47:22 +0700696 return tcg_ctx.code_gen_buffer != NULL;
Blue Swirl5b6dd862012-12-02 16:04:43 +0000697}
698
699/* Allocate a new translation block. Flush the translation buffer if
700 too many translation blocks or too much generated code. */
701static TranslationBlock *tb_alloc(target_ulong pc)
702{
703 TranslationBlock *tb;
704
Evgeny Voevodin5e5f07e2013-02-01 01:47:23 +0700705 if (tcg_ctx.tb_ctx.nb_tbs >= tcg_ctx.code_gen_max_blocks ||
Evgeny Voevodin0b0d3322013-02-01 01:47:22 +0700706 (tcg_ctx.code_gen_ptr - tcg_ctx.code_gen_buffer) >=
707 tcg_ctx.code_gen_buffer_max_size) {
Blue Swirl5b6dd862012-12-02 16:04:43 +0000708 return NULL;
709 }
Evgeny Voevodin5e5f07e2013-02-01 01:47:23 +0700710 tb = &tcg_ctx.tb_ctx.tbs[tcg_ctx.tb_ctx.nb_tbs++];
Blue Swirl5b6dd862012-12-02 16:04:43 +0000711 tb->pc = pc;
712 tb->cflags = 0;
713 return tb;
714}
715
716void tb_free(TranslationBlock *tb)
717{
718 /* In practice this is mostly used for single use temporary TB
719 Ignore the hard cases and just back up if this TB happens to
720 be the last one generated. */
Evgeny Voevodin5e5f07e2013-02-01 01:47:23 +0700721 if (tcg_ctx.tb_ctx.nb_tbs > 0 &&
722 tb == &tcg_ctx.tb_ctx.tbs[tcg_ctx.tb_ctx.nb_tbs - 1]) {
Evgeny Voevodin0b0d3322013-02-01 01:47:22 +0700723 tcg_ctx.code_gen_ptr = tb->tc_ptr;
Evgeny Voevodin5e5f07e2013-02-01 01:47:23 +0700724 tcg_ctx.tb_ctx.nb_tbs--;
Blue Swirl5b6dd862012-12-02 16:04:43 +0000725 }
726}
727
728static inline void invalidate_page_bitmap(PageDesc *p)
729{
730 if (p->code_bitmap) {
731 g_free(p->code_bitmap);
732 p->code_bitmap = NULL;
733 }
734 p->code_write_count = 0;
735}
736
737/* Set to NULL all the 'first_tb' fields in all PageDescs. */
738static void page_flush_tb_1(int level, void **lp)
739{
740 int i;
741
742 if (*lp == NULL) {
743 return;
744 }
745 if (level == 0) {
746 PageDesc *pd = *lp;
747
Paolo Bonzini03f49952013-11-07 17:14:36 +0100748 for (i = 0; i < V_L2_SIZE; ++i) {
Blue Swirl5b6dd862012-12-02 16:04:43 +0000749 pd[i].first_tb = NULL;
750 invalidate_page_bitmap(pd + i);
751 }
752 } else {
753 void **pp = *lp;
754
Paolo Bonzini03f49952013-11-07 17:14:36 +0100755 for (i = 0; i < V_L2_SIZE; ++i) {
Blue Swirl5b6dd862012-12-02 16:04:43 +0000756 page_flush_tb_1(level - 1, pp + i);
757 }
758 }
759}
760
761static void page_flush_tb(void)
762{
763 int i;
764
765 for (i = 0; i < V_L1_SIZE; i++) {
Paolo Bonzini03f49952013-11-07 17:14:36 +0100766 page_flush_tb_1(V_L1_SHIFT / V_L2_BITS - 1, l1_map + i);
Blue Swirl5b6dd862012-12-02 16:04:43 +0000767 }
768}
769
770/* flush all the translation blocks */
771/* XXX: tb_flush is currently not thread safe */
772void tb_flush(CPUArchState *env1)
773{
Andreas Färbera47dddd2013-09-03 17:38:47 +0200774 CPUState *cpu = ENV_GET_CPU(env1);
Blue Swirl5b6dd862012-12-02 16:04:43 +0000775
776#if defined(DEBUG_FLUSH)
777 printf("qemu: flush code_size=%ld nb_tbs=%d avg_tb_size=%ld\n",
Evgeny Voevodin0b0d3322013-02-01 01:47:22 +0700778 (unsigned long)(tcg_ctx.code_gen_ptr - tcg_ctx.code_gen_buffer),
Evgeny Voevodin5e5f07e2013-02-01 01:47:23 +0700779 tcg_ctx.tb_ctx.nb_tbs, tcg_ctx.tb_ctx.nb_tbs > 0 ?
Evgeny Voevodin0b0d3322013-02-01 01:47:22 +0700780 ((unsigned long)(tcg_ctx.code_gen_ptr - tcg_ctx.code_gen_buffer)) /
Evgeny Voevodin5e5f07e2013-02-01 01:47:23 +0700781 tcg_ctx.tb_ctx.nb_tbs : 0);
Blue Swirl5b6dd862012-12-02 16:04:43 +0000782#endif
Evgeny Voevodin0b0d3322013-02-01 01:47:22 +0700783 if ((unsigned long)(tcg_ctx.code_gen_ptr - tcg_ctx.code_gen_buffer)
784 > tcg_ctx.code_gen_buffer_size) {
Andreas Färbera47dddd2013-09-03 17:38:47 +0200785 cpu_abort(cpu, "Internal error: code buffer overflow\n");
Blue Swirl5b6dd862012-12-02 16:04:43 +0000786 }
Evgeny Voevodin5e5f07e2013-02-01 01:47:23 +0700787 tcg_ctx.tb_ctx.nb_tbs = 0;
Blue Swirl5b6dd862012-12-02 16:04:43 +0000788
Andreas Färberbdc44642013-06-24 23:50:24 +0200789 CPU_FOREACH(cpu) {
Andreas Färber8cd70432013-08-26 06:03:38 +0200790 memset(cpu->tb_jmp_cache, 0, sizeof(cpu->tb_jmp_cache));
Blue Swirl5b6dd862012-12-02 16:04:43 +0000791 }
792
Richard Hendersoneb2535f2013-12-07 10:44:52 +1300793 memset(tcg_ctx.tb_ctx.tb_phys_hash, 0, sizeof(tcg_ctx.tb_ctx.tb_phys_hash));
Blue Swirl5b6dd862012-12-02 16:04:43 +0000794 page_flush_tb();
795
Evgeny Voevodin0b0d3322013-02-01 01:47:22 +0700796 tcg_ctx.code_gen_ptr = tcg_ctx.code_gen_buffer;
Blue Swirl5b6dd862012-12-02 16:04:43 +0000797 /* XXX: flush processor icache at this point if cache flush is
798 expensive */
Evgeny Voevodin5e5f07e2013-02-01 01:47:23 +0700799 tcg_ctx.tb_ctx.tb_flush_count++;
Blue Swirl5b6dd862012-12-02 16:04:43 +0000800}
801
802#ifdef DEBUG_TB_CHECK
803
804static void tb_invalidate_check(target_ulong address)
805{
806 TranslationBlock *tb;
807 int i;
808
809 address &= TARGET_PAGE_MASK;
810 for (i = 0; i < CODE_GEN_PHYS_HASH_SIZE; i++) {
Evgeny Voevodin5e5f07e2013-02-01 01:47:23 +0700811 for (tb = tb_ctx.tb_phys_hash[i]; tb != NULL; tb = tb->phys_hash_next) {
Blue Swirl5b6dd862012-12-02 16:04:43 +0000812 if (!(address + TARGET_PAGE_SIZE <= tb->pc ||
813 address >= tb->pc + tb->size)) {
814 printf("ERROR invalidate: address=" TARGET_FMT_lx
815 " PC=%08lx size=%04x\n",
816 address, (long)tb->pc, tb->size);
817 }
818 }
819 }
820}
821
822/* verify that all the pages have correct rights for code */
823static void tb_page_check(void)
824{
825 TranslationBlock *tb;
826 int i, flags1, flags2;
827
828 for (i = 0; i < CODE_GEN_PHYS_HASH_SIZE; i++) {
Evgeny Voevodin5e5f07e2013-02-01 01:47:23 +0700829 for (tb = tcg_ctx.tb_ctx.tb_phys_hash[i]; tb != NULL;
830 tb = tb->phys_hash_next) {
Blue Swirl5b6dd862012-12-02 16:04:43 +0000831 flags1 = page_get_flags(tb->pc);
832 flags2 = page_get_flags(tb->pc + tb->size - 1);
833 if ((flags1 & PAGE_WRITE) || (flags2 & PAGE_WRITE)) {
834 printf("ERROR page flags: PC=%08lx size=%04x f1=%x f2=%x\n",
835 (long)tb->pc, tb->size, flags1, flags2);
836 }
837 }
838 }
839}
840
841#endif
842
陳韋任 (Wei-Ren Chen)0c884d12012-12-20 09:39:16 +0800843static inline void tb_hash_remove(TranslationBlock **ptb, TranslationBlock *tb)
Blue Swirl5b6dd862012-12-02 16:04:43 +0000844{
845 TranslationBlock *tb1;
846
847 for (;;) {
848 tb1 = *ptb;
849 if (tb1 == tb) {
陳韋任 (Wei-Ren Chen)0c884d12012-12-20 09:39:16 +0800850 *ptb = tb1->phys_hash_next;
Blue Swirl5b6dd862012-12-02 16:04:43 +0000851 break;
852 }
陳韋任 (Wei-Ren Chen)0c884d12012-12-20 09:39:16 +0800853 ptb = &tb1->phys_hash_next;
Blue Swirl5b6dd862012-12-02 16:04:43 +0000854 }
855}
856
857static inline void tb_page_remove(TranslationBlock **ptb, TranslationBlock *tb)
858{
859 TranslationBlock *tb1;
860 unsigned int n1;
861
862 for (;;) {
863 tb1 = *ptb;
864 n1 = (uintptr_t)tb1 & 3;
865 tb1 = (TranslationBlock *)((uintptr_t)tb1 & ~3);
866 if (tb1 == tb) {
867 *ptb = tb1->page_next[n1];
868 break;
869 }
870 ptb = &tb1->page_next[n1];
871 }
872}
873
874static inline void tb_jmp_remove(TranslationBlock *tb, int n)
875{
876 TranslationBlock *tb1, **ptb;
877 unsigned int n1;
878
879 ptb = &tb->jmp_next[n];
880 tb1 = *ptb;
881 if (tb1) {
882 /* find tb(n) in circular list */
883 for (;;) {
884 tb1 = *ptb;
885 n1 = (uintptr_t)tb1 & 3;
886 tb1 = (TranslationBlock *)((uintptr_t)tb1 & ~3);
887 if (n1 == n && tb1 == tb) {
888 break;
889 }
890 if (n1 == 2) {
891 ptb = &tb1->jmp_first;
892 } else {
893 ptb = &tb1->jmp_next[n1];
894 }
895 }
896 /* now we can suppress tb(n) from the list */
897 *ptb = tb->jmp_next[n];
898
899 tb->jmp_next[n] = NULL;
900 }
901}
902
903/* reset the jump entry 'n' of a TB so that it is not chained to
904 another TB */
905static inline void tb_reset_jump(TranslationBlock *tb, int n)
906{
907 tb_set_jmp_target(tb, n, (uintptr_t)(tb->tc_ptr + tb->tb_next_offset[n]));
908}
909
陳韋任 (Wei-Ren Chen)0c884d12012-12-20 09:39:16 +0800910/* invalidate one TB */
Blue Swirl5b6dd862012-12-02 16:04:43 +0000911void tb_phys_invalidate(TranslationBlock *tb, tb_page_addr_t page_addr)
912{
Andreas Färber182735e2013-05-29 22:29:20 +0200913 CPUState *cpu;
Blue Swirl5b6dd862012-12-02 16:04:43 +0000914 PageDesc *p;
915 unsigned int h, n1;
916 tb_page_addr_t phys_pc;
917 TranslationBlock *tb1, *tb2;
918
919 /* remove the TB from the hash list */
920 phys_pc = tb->page_addr[0] + (tb->pc & ~TARGET_PAGE_MASK);
921 h = tb_phys_hash_func(phys_pc);
Evgeny Voevodin5e5f07e2013-02-01 01:47:23 +0700922 tb_hash_remove(&tcg_ctx.tb_ctx.tb_phys_hash[h], tb);
Blue Swirl5b6dd862012-12-02 16:04:43 +0000923
924 /* remove the TB from the page list */
925 if (tb->page_addr[0] != page_addr) {
926 p = page_find(tb->page_addr[0] >> TARGET_PAGE_BITS);
927 tb_page_remove(&p->first_tb, tb);
928 invalidate_page_bitmap(p);
929 }
930 if (tb->page_addr[1] != -1 && tb->page_addr[1] != page_addr) {
931 p = page_find(tb->page_addr[1] >> TARGET_PAGE_BITS);
932 tb_page_remove(&p->first_tb, tb);
933 invalidate_page_bitmap(p);
934 }
935
Evgeny Voevodin5e5f07e2013-02-01 01:47:23 +0700936 tcg_ctx.tb_ctx.tb_invalidated_flag = 1;
Blue Swirl5b6dd862012-12-02 16:04:43 +0000937
938 /* remove the TB from the hash list */
939 h = tb_jmp_cache_hash_func(tb->pc);
Andreas Färberbdc44642013-06-24 23:50:24 +0200940 CPU_FOREACH(cpu) {
Andreas Färber8cd70432013-08-26 06:03:38 +0200941 if (cpu->tb_jmp_cache[h] == tb) {
942 cpu->tb_jmp_cache[h] = NULL;
Blue Swirl5b6dd862012-12-02 16:04:43 +0000943 }
944 }
945
946 /* suppress this TB from the two jump lists */
947 tb_jmp_remove(tb, 0);
948 tb_jmp_remove(tb, 1);
949
950 /* suppress any remaining jumps to this TB */
951 tb1 = tb->jmp_first;
952 for (;;) {
953 n1 = (uintptr_t)tb1 & 3;
954 if (n1 == 2) {
955 break;
956 }
957 tb1 = (TranslationBlock *)((uintptr_t)tb1 & ~3);
958 tb2 = tb1->jmp_next[n1];
959 tb_reset_jump(tb1, n1);
960 tb1->jmp_next[n1] = NULL;
961 tb1 = tb2;
962 }
963 tb->jmp_first = (TranslationBlock *)((uintptr_t)tb | 2); /* fail safe */
964
Evgeny Voevodin5e5f07e2013-02-01 01:47:23 +0700965 tcg_ctx.tb_ctx.tb_phys_invalidate_count++;
Blue Swirl5b6dd862012-12-02 16:04:43 +0000966}
967
Blue Swirl5b6dd862012-12-02 16:04:43 +0000968static void build_page_bitmap(PageDesc *p)
969{
970 int n, tb_start, tb_end;
971 TranslationBlock *tb;
972
Emilio G. Cota510a6472015-04-22 17:50:52 -0400973 p->code_bitmap = bitmap_new(TARGET_PAGE_SIZE);
Blue Swirl5b6dd862012-12-02 16:04:43 +0000974
975 tb = p->first_tb;
976 while (tb != NULL) {
977 n = (uintptr_t)tb & 3;
978 tb = (TranslationBlock *)((uintptr_t)tb & ~3);
979 /* NOTE: this is subtle as a TB may span two physical pages */
980 if (n == 0) {
981 /* NOTE: tb_end may be after the end of the page, but
982 it is not a problem */
983 tb_start = tb->pc & ~TARGET_PAGE_MASK;
984 tb_end = tb_start + tb->size;
985 if (tb_end > TARGET_PAGE_SIZE) {
986 tb_end = TARGET_PAGE_SIZE;
987 }
988 } else {
989 tb_start = 0;
990 tb_end = ((tb->pc + tb->size) & ~TARGET_PAGE_MASK);
991 }
Emilio G. Cota510a6472015-04-22 17:50:52 -0400992 bitmap_set(p->code_bitmap, tb_start, tb_end - tb_start);
Blue Swirl5b6dd862012-12-02 16:04:43 +0000993 tb = tb->page_next[n];
994 }
995}
996
Andreas Färber648f0342013-09-01 17:43:17 +0200997TranslationBlock *tb_gen_code(CPUState *cpu,
Blue Swirl5b6dd862012-12-02 16:04:43 +0000998 target_ulong pc, target_ulong cs_base,
999 int flags, int cflags)
1000{
Andreas Färber648f0342013-09-01 17:43:17 +02001001 CPUArchState *env = cpu->env_ptr;
Blue Swirl5b6dd862012-12-02 16:04:43 +00001002 TranslationBlock *tb;
Blue Swirl5b6dd862012-12-02 16:04:43 +00001003 tb_page_addr_t phys_pc, phys_page2;
1004 target_ulong virt_page2;
1005 int code_gen_size;
1006
1007 phys_pc = get_page_addr_code(env, pc);
Paolo Bonzini02663592014-11-26 13:39:53 +03001008 if (use_icount) {
1009 cflags |= CF_USE_ICOUNT;
1010 }
Blue Swirl5b6dd862012-12-02 16:04:43 +00001011 tb = tb_alloc(pc);
1012 if (!tb) {
1013 /* flush must be done */
1014 tb_flush(env);
1015 /* cannot fail at this point */
1016 tb = tb_alloc(pc);
1017 /* Don't forget to invalidate previous TB info. */
Evgeny Voevodin5e5f07e2013-02-01 01:47:23 +07001018 tcg_ctx.tb_ctx.tb_invalidated_flag = 1;
Blue Swirl5b6dd862012-12-02 16:04:43 +00001019 }
Richard Henderson1813e172014-03-28 12:56:22 -07001020 tb->tc_ptr = tcg_ctx.code_gen_ptr;
Blue Swirl5b6dd862012-12-02 16:04:43 +00001021 tb->cs_base = cs_base;
1022 tb->flags = flags;
1023 tb->cflags = cflags;
1024 cpu_gen_code(env, tb, &code_gen_size);
Evgeny Voevodin0b0d3322013-02-01 01:47:22 +07001025 tcg_ctx.code_gen_ptr = (void *)(((uintptr_t)tcg_ctx.code_gen_ptr +
1026 code_gen_size + CODE_GEN_ALIGN - 1) & ~(CODE_GEN_ALIGN - 1));
Blue Swirl5b6dd862012-12-02 16:04:43 +00001027
1028 /* check next page if needed */
1029 virt_page2 = (pc + tb->size - 1) & TARGET_PAGE_MASK;
1030 phys_page2 = -1;
1031 if ((pc & TARGET_PAGE_MASK) != virt_page2) {
1032 phys_page2 = get_page_addr_code(env, virt_page2);
1033 }
1034 tb_link_page(tb, phys_pc, phys_page2);
1035 return tb;
1036}
1037
1038/*
1039 * Invalidate all TBs which intersect with the target physical address range
1040 * [start;end[. NOTE: start and end may refer to *different* physical pages.
1041 * 'is_cpu_write_access' should be true if called from a real cpu write
1042 * access: the virtual CPU will exit the current TB if code is modified inside
1043 * this TB.
1044 */
Paolo Bonzini35865332015-04-22 14:20:35 +02001045void tb_invalidate_phys_range(tb_page_addr_t start, tb_page_addr_t end)
Blue Swirl5b6dd862012-12-02 16:04:43 +00001046{
1047 while (start < end) {
Paolo Bonzini35865332015-04-22 14:20:35 +02001048 tb_invalidate_phys_page_range(start, end, 0);
Blue Swirl5b6dd862012-12-02 16:04:43 +00001049 start &= TARGET_PAGE_MASK;
1050 start += TARGET_PAGE_SIZE;
1051 }
1052}
1053
1054/*
1055 * Invalidate all TBs which intersect with the target physical address range
1056 * [start;end[. NOTE: start and end must refer to the *same* physical page.
1057 * 'is_cpu_write_access' should be true if called from a real cpu write
1058 * access: the virtual CPU will exit the current TB if code is modified inside
1059 * this TB.
1060 */
1061void tb_invalidate_phys_page_range(tb_page_addr_t start, tb_page_addr_t end,
1062 int is_cpu_write_access)
1063{
1064 TranslationBlock *tb, *tb_next, *saved_tb;
Andreas Färber4917cf42013-05-27 05:17:50 +02001065 CPUState *cpu = current_cpu;
Andreas Färberbaea4fa2013-09-03 10:51:26 +02001066#if defined(TARGET_HAS_PRECISE_SMC)
Andreas Färber4917cf42013-05-27 05:17:50 +02001067 CPUArchState *env = NULL;
1068#endif
Blue Swirl5b6dd862012-12-02 16:04:43 +00001069 tb_page_addr_t tb_start, tb_end;
1070 PageDesc *p;
1071 int n;
1072#ifdef TARGET_HAS_PRECISE_SMC
1073 int current_tb_not_found = is_cpu_write_access;
1074 TranslationBlock *current_tb = NULL;
1075 int current_tb_modified = 0;
1076 target_ulong current_pc = 0;
1077 target_ulong current_cs_base = 0;
1078 int current_flags = 0;
1079#endif /* TARGET_HAS_PRECISE_SMC */
1080
1081 p = page_find(start >> TARGET_PAGE_BITS);
1082 if (!p) {
1083 return;
1084 }
Andreas Färberbaea4fa2013-09-03 10:51:26 +02001085#if defined(TARGET_HAS_PRECISE_SMC)
Andreas Färber4917cf42013-05-27 05:17:50 +02001086 if (cpu != NULL) {
1087 env = cpu->env_ptr;
Andreas Färberd77953b2013-01-16 19:29:31 +01001088 }
Andreas Färber4917cf42013-05-27 05:17:50 +02001089#endif
Blue Swirl5b6dd862012-12-02 16:04:43 +00001090
1091 /* we remove all the TBs in the range [start, end[ */
1092 /* XXX: see if in some cases it could be faster to invalidate all
1093 the code */
1094 tb = p->first_tb;
1095 while (tb != NULL) {
1096 n = (uintptr_t)tb & 3;
1097 tb = (TranslationBlock *)((uintptr_t)tb & ~3);
1098 tb_next = tb->page_next[n];
1099 /* NOTE: this is subtle as a TB may span two physical pages */
1100 if (n == 0) {
1101 /* NOTE: tb_end may be after the end of the page, but
1102 it is not a problem */
1103 tb_start = tb->page_addr[0] + (tb->pc & ~TARGET_PAGE_MASK);
1104 tb_end = tb_start + tb->size;
1105 } else {
1106 tb_start = tb->page_addr[1];
1107 tb_end = tb_start + ((tb->pc + tb->size) & ~TARGET_PAGE_MASK);
1108 }
1109 if (!(tb_end <= start || tb_start >= end)) {
1110#ifdef TARGET_HAS_PRECISE_SMC
1111 if (current_tb_not_found) {
1112 current_tb_not_found = 0;
1113 current_tb = NULL;
Andreas Färber93afead2013-08-26 03:41:01 +02001114 if (cpu->mem_io_pc) {
Blue Swirl5b6dd862012-12-02 16:04:43 +00001115 /* now we have a real cpu fault */
Andreas Färber93afead2013-08-26 03:41:01 +02001116 current_tb = tb_find_pc(cpu->mem_io_pc);
Blue Swirl5b6dd862012-12-02 16:04:43 +00001117 }
1118 }
1119 if (current_tb == tb &&
1120 (current_tb->cflags & CF_COUNT_MASK) != 1) {
1121 /* If we are modifying the current TB, we must stop
1122 its execution. We could be more precise by checking
1123 that the modification is after the current PC, but it
1124 would require a specialized function to partially
1125 restore the CPU state */
1126
1127 current_tb_modified = 1;
Andreas Färber74f10512013-09-01 17:02:58 +02001128 cpu_restore_state_from_tb(cpu, current_tb, cpu->mem_io_pc);
Blue Swirl5b6dd862012-12-02 16:04:43 +00001129 cpu_get_tb_cpu_state(env, &current_pc, &current_cs_base,
1130 &current_flags);
1131 }
1132#endif /* TARGET_HAS_PRECISE_SMC */
1133 /* we need to do that to handle the case where a signal
1134 occurs while doing tb_phys_invalidate() */
1135 saved_tb = NULL;
Andreas Färberd77953b2013-01-16 19:29:31 +01001136 if (cpu != NULL) {
1137 saved_tb = cpu->current_tb;
1138 cpu->current_tb = NULL;
Blue Swirl5b6dd862012-12-02 16:04:43 +00001139 }
1140 tb_phys_invalidate(tb, -1);
Andreas Färberd77953b2013-01-16 19:29:31 +01001141 if (cpu != NULL) {
1142 cpu->current_tb = saved_tb;
Andreas Färberc3affe52013-01-18 15:03:43 +01001143 if (cpu->interrupt_request && cpu->current_tb) {
1144 cpu_interrupt(cpu, cpu->interrupt_request);
Blue Swirl5b6dd862012-12-02 16:04:43 +00001145 }
1146 }
1147 }
1148 tb = tb_next;
1149 }
1150#if !defined(CONFIG_USER_ONLY)
1151 /* if no code remaining, no need to continue to use slow writes */
1152 if (!p->first_tb) {
1153 invalidate_page_bitmap(p);
Paolo Bonzinifc377bc2015-04-22 14:20:35 +02001154 tlb_unprotect_code(start);
Blue Swirl5b6dd862012-12-02 16:04:43 +00001155 }
1156#endif
1157#ifdef TARGET_HAS_PRECISE_SMC
1158 if (current_tb_modified) {
1159 /* we generate a block containing just the instruction
1160 modifying the memory. It will ensure that it cannot modify
1161 itself */
Andreas Färberd77953b2013-01-16 19:29:31 +01001162 cpu->current_tb = NULL;
Andreas Färber648f0342013-09-01 17:43:17 +02001163 tb_gen_code(cpu, current_pc, current_cs_base, current_flags, 1);
Andreas Färber0ea8cb82013-09-03 02:12:23 +02001164 cpu_resume_from_signal(cpu, NULL);
Blue Swirl5b6dd862012-12-02 16:04:43 +00001165 }
1166#endif
1167}
1168
1169/* len must be <= 8 and start must be a multiple of len */
1170void tb_invalidate_phys_page_fast(tb_page_addr_t start, int len)
1171{
1172 PageDesc *p;
Blue Swirl5b6dd862012-12-02 16:04:43 +00001173
1174#if 0
1175 if (1) {
1176 qemu_log("modifying code at 0x%x size=%d EIP=%x PC=%08x\n",
1177 cpu_single_env->mem_io_vaddr, len,
1178 cpu_single_env->eip,
1179 cpu_single_env->eip +
1180 (intptr_t)cpu_single_env->segs[R_CS].base);
1181 }
1182#endif
1183 p = page_find(start >> TARGET_PAGE_BITS);
1184 if (!p) {
1185 return;
1186 }
Paolo Bonzinifc377bc2015-04-22 14:20:35 +02001187 if (!p->code_bitmap &&
1188 ++p->code_write_count >= SMC_BITMAP_USE_THRESHOLD) {
1189 /* build code bitmap */
1190 build_page_bitmap(p);
1191 }
Blue Swirl5b6dd862012-12-02 16:04:43 +00001192 if (p->code_bitmap) {
Emilio G. Cota510a6472015-04-22 17:50:52 -04001193 unsigned int nr;
1194 unsigned long b;
1195
1196 nr = start & ~TARGET_PAGE_MASK;
1197 b = p->code_bitmap[BIT_WORD(nr)] >> (nr & (BITS_PER_LONG - 1));
Blue Swirl5b6dd862012-12-02 16:04:43 +00001198 if (b & ((1 << len) - 1)) {
1199 goto do_invalidate;
1200 }
1201 } else {
1202 do_invalidate:
1203 tb_invalidate_phys_page_range(start, start + len, 1);
1204 }
1205}
1206
1207#if !defined(CONFIG_SOFTMMU)
1208static void tb_invalidate_phys_page(tb_page_addr_t addr,
Alexander Grafd02532f2013-07-06 14:17:57 +02001209 uintptr_t pc, void *puc,
1210 bool locked)
Blue Swirl5b6dd862012-12-02 16:04:43 +00001211{
1212 TranslationBlock *tb;
1213 PageDesc *p;
1214 int n;
1215#ifdef TARGET_HAS_PRECISE_SMC
1216 TranslationBlock *current_tb = NULL;
Andreas Färber4917cf42013-05-27 05:17:50 +02001217 CPUState *cpu = current_cpu;
1218 CPUArchState *env = NULL;
Blue Swirl5b6dd862012-12-02 16:04:43 +00001219 int current_tb_modified = 0;
1220 target_ulong current_pc = 0;
1221 target_ulong current_cs_base = 0;
1222 int current_flags = 0;
1223#endif
1224
1225 addr &= TARGET_PAGE_MASK;
1226 p = page_find(addr >> TARGET_PAGE_BITS);
1227 if (!p) {
1228 return;
1229 }
1230 tb = p->first_tb;
1231#ifdef TARGET_HAS_PRECISE_SMC
1232 if (tb && pc != 0) {
1233 current_tb = tb_find_pc(pc);
1234 }
Andreas Färber4917cf42013-05-27 05:17:50 +02001235 if (cpu != NULL) {
1236 env = cpu->env_ptr;
Andreas Färberd77953b2013-01-16 19:29:31 +01001237 }
Blue Swirl5b6dd862012-12-02 16:04:43 +00001238#endif
1239 while (tb != NULL) {
1240 n = (uintptr_t)tb & 3;
1241 tb = (TranslationBlock *)((uintptr_t)tb & ~3);
1242#ifdef TARGET_HAS_PRECISE_SMC
1243 if (current_tb == tb &&
1244 (current_tb->cflags & CF_COUNT_MASK) != 1) {
1245 /* If we are modifying the current TB, we must stop
1246 its execution. We could be more precise by checking
1247 that the modification is after the current PC, but it
1248 would require a specialized function to partially
1249 restore the CPU state */
1250
1251 current_tb_modified = 1;
Andreas Färber74f10512013-09-01 17:02:58 +02001252 cpu_restore_state_from_tb(cpu, current_tb, pc);
Blue Swirl5b6dd862012-12-02 16:04:43 +00001253 cpu_get_tb_cpu_state(env, &current_pc, &current_cs_base,
1254 &current_flags);
1255 }
1256#endif /* TARGET_HAS_PRECISE_SMC */
1257 tb_phys_invalidate(tb, addr);
1258 tb = tb->page_next[n];
1259 }
1260 p->first_tb = NULL;
1261#ifdef TARGET_HAS_PRECISE_SMC
1262 if (current_tb_modified) {
1263 /* we generate a block containing just the instruction
1264 modifying the memory. It will ensure that it cannot modify
1265 itself */
Andreas Färberd77953b2013-01-16 19:29:31 +01001266 cpu->current_tb = NULL;
Andreas Färber648f0342013-09-01 17:43:17 +02001267 tb_gen_code(cpu, current_pc, current_cs_base, current_flags, 1);
Alexander Grafd02532f2013-07-06 14:17:57 +02001268 if (locked) {
1269 mmap_unlock();
1270 }
Andreas Färber0ea8cb82013-09-03 02:12:23 +02001271 cpu_resume_from_signal(cpu, puc);
Blue Swirl5b6dd862012-12-02 16:04:43 +00001272 }
1273#endif
1274}
1275#endif
1276
1277/* add the tb in the target page and protect it if necessary */
1278static inline void tb_alloc_page(TranslationBlock *tb,
1279 unsigned int n, tb_page_addr_t page_addr)
1280{
1281 PageDesc *p;
1282#ifndef CONFIG_USER_ONLY
1283 bool page_already_protected;
1284#endif
1285
1286 tb->page_addr[n] = page_addr;
1287 p = page_find_alloc(page_addr >> TARGET_PAGE_BITS, 1);
1288 tb->page_next[n] = p->first_tb;
1289#ifndef CONFIG_USER_ONLY
1290 page_already_protected = p->first_tb != NULL;
1291#endif
1292 p->first_tb = (TranslationBlock *)((uintptr_t)tb | n);
1293 invalidate_page_bitmap(p);
1294
Blue Swirl5b6dd862012-12-02 16:04:43 +00001295#if defined(CONFIG_USER_ONLY)
1296 if (p->flags & PAGE_WRITE) {
1297 target_ulong addr;
1298 PageDesc *p2;
1299 int prot;
1300
1301 /* force the host page as non writable (writes will have a
1302 page fault + mprotect overhead) */
1303 page_addr &= qemu_host_page_mask;
1304 prot = 0;
1305 for (addr = page_addr; addr < page_addr + qemu_host_page_size;
1306 addr += TARGET_PAGE_SIZE) {
1307
1308 p2 = page_find(addr >> TARGET_PAGE_BITS);
1309 if (!p2) {
1310 continue;
1311 }
1312 prot |= p2->flags;
1313 p2->flags &= ~PAGE_WRITE;
1314 }
1315 mprotect(g2h(page_addr), qemu_host_page_size,
1316 (prot & PAGE_BITS) & ~PAGE_WRITE);
1317#ifdef DEBUG_TB_INVALIDATE
1318 printf("protecting code page: 0x" TARGET_FMT_lx "\n",
1319 page_addr);
1320#endif
1321 }
1322#else
1323 /* if some code is already present, then the pages are already
1324 protected. So we handle the case where only the first TB is
1325 allocated in a physical page */
1326 if (!page_already_protected) {
1327 tlb_protect_code(page_addr);
1328 }
1329#endif
Blue Swirl5b6dd862012-12-02 16:04:43 +00001330}
1331
1332/* add a new TB and link it to the physical page tables. phys_page2 is
1333 (-1) to indicate that only one page contains the TB. */
1334static void tb_link_page(TranslationBlock *tb, tb_page_addr_t phys_pc,
1335 tb_page_addr_t phys_page2)
1336{
1337 unsigned int h;
1338 TranslationBlock **ptb;
1339
1340 /* Grab the mmap lock to stop another thread invalidating this TB
1341 before we are done. */
1342 mmap_lock();
1343 /* add in the physical hash table */
1344 h = tb_phys_hash_func(phys_pc);
Evgeny Voevodin5e5f07e2013-02-01 01:47:23 +07001345 ptb = &tcg_ctx.tb_ctx.tb_phys_hash[h];
Blue Swirl5b6dd862012-12-02 16:04:43 +00001346 tb->phys_hash_next = *ptb;
1347 *ptb = tb;
1348
1349 /* add in the page list */
1350 tb_alloc_page(tb, 0, phys_pc & TARGET_PAGE_MASK);
1351 if (phys_page2 != -1) {
1352 tb_alloc_page(tb, 1, phys_page2);
1353 } else {
1354 tb->page_addr[1] = -1;
1355 }
1356
1357 tb->jmp_first = (TranslationBlock *)((uintptr_t)tb | 2);
1358 tb->jmp_next[0] = NULL;
1359 tb->jmp_next[1] = NULL;
1360
1361 /* init original jump addresses */
1362 if (tb->tb_next_offset[0] != 0xffff) {
1363 tb_reset_jump(tb, 0);
1364 }
1365 if (tb->tb_next_offset[1] != 0xffff) {
1366 tb_reset_jump(tb, 1);
1367 }
1368
1369#ifdef DEBUG_TB_CHECK
1370 tb_page_check();
1371#endif
1372 mmap_unlock();
1373}
1374
Blue Swirl5b6dd862012-12-02 16:04:43 +00001375/* find the TB 'tb' such that tb[0].tc_ptr <= tc_ptr <
1376 tb[1].tc_ptr. Return NULL if not found */
Blue Swirla8a826a2012-12-04 20:16:07 +00001377static TranslationBlock *tb_find_pc(uintptr_t tc_ptr)
Blue Swirl5b6dd862012-12-02 16:04:43 +00001378{
1379 int m_min, m_max, m;
1380 uintptr_t v;
1381 TranslationBlock *tb;
1382
Evgeny Voevodin5e5f07e2013-02-01 01:47:23 +07001383 if (tcg_ctx.tb_ctx.nb_tbs <= 0) {
Blue Swirl5b6dd862012-12-02 16:04:43 +00001384 return NULL;
1385 }
Evgeny Voevodin0b0d3322013-02-01 01:47:22 +07001386 if (tc_ptr < (uintptr_t)tcg_ctx.code_gen_buffer ||
1387 tc_ptr >= (uintptr_t)tcg_ctx.code_gen_ptr) {
Blue Swirl5b6dd862012-12-02 16:04:43 +00001388 return NULL;
1389 }
1390 /* binary search (cf Knuth) */
1391 m_min = 0;
Evgeny Voevodin5e5f07e2013-02-01 01:47:23 +07001392 m_max = tcg_ctx.tb_ctx.nb_tbs - 1;
Blue Swirl5b6dd862012-12-02 16:04:43 +00001393 while (m_min <= m_max) {
1394 m = (m_min + m_max) >> 1;
Evgeny Voevodin5e5f07e2013-02-01 01:47:23 +07001395 tb = &tcg_ctx.tb_ctx.tbs[m];
Blue Swirl5b6dd862012-12-02 16:04:43 +00001396 v = (uintptr_t)tb->tc_ptr;
1397 if (v == tc_ptr) {
1398 return tb;
1399 } else if (tc_ptr < v) {
1400 m_max = m - 1;
1401 } else {
1402 m_min = m + 1;
1403 }
1404 }
Evgeny Voevodin5e5f07e2013-02-01 01:47:23 +07001405 return &tcg_ctx.tb_ctx.tbs[m_max];
Blue Swirl5b6dd862012-12-02 16:04:43 +00001406}
1407
Peter Maydellec53b452015-01-20 15:19:32 +00001408#if !defined(CONFIG_USER_ONLY)
Edgar E. Iglesias29d8ec72013-11-07 19:43:10 +01001409void tb_invalidate_phys_addr(AddressSpace *as, hwaddr addr)
Blue Swirl5b6dd862012-12-02 16:04:43 +00001410{
1411 ram_addr_t ram_addr;
Paolo Bonzini5c8a00c2013-05-29 12:42:00 +02001412 MemoryRegion *mr;
Paolo Bonzini149f54b2013-05-24 12:59:37 +02001413 hwaddr l = 1;
Blue Swirl5b6dd862012-12-02 16:04:43 +00001414
Paolo Bonzini41063e12015-03-18 14:21:43 +01001415 rcu_read_lock();
Edgar E. Iglesias29d8ec72013-11-07 19:43:10 +01001416 mr = address_space_translate(as, addr, &addr, &l, false);
Paolo Bonzini5c8a00c2013-05-29 12:42:00 +02001417 if (!(memory_region_is_ram(mr)
1418 || memory_region_is_romd(mr))) {
Paolo Bonzini41063e12015-03-18 14:21:43 +01001419 rcu_read_unlock();
Blue Swirl5b6dd862012-12-02 16:04:43 +00001420 return;
1421 }
Paolo Bonzini5c8a00c2013-05-29 12:42:00 +02001422 ram_addr = (memory_region_get_ram_addr(mr) & TARGET_PAGE_MASK)
Paolo Bonzini149f54b2013-05-24 12:59:37 +02001423 + addr;
Blue Swirl5b6dd862012-12-02 16:04:43 +00001424 tb_invalidate_phys_page_range(ram_addr, ram_addr + 1, 0);
Paolo Bonzini41063e12015-03-18 14:21:43 +01001425 rcu_read_unlock();
Blue Swirl5b6dd862012-12-02 16:04:43 +00001426}
Peter Maydellec53b452015-01-20 15:19:32 +00001427#endif /* !defined(CONFIG_USER_ONLY) */
Blue Swirl5b6dd862012-12-02 16:04:43 +00001428
Andreas Färber239c51a2013-09-01 17:12:23 +02001429void tb_check_watchpoint(CPUState *cpu)
Blue Swirl5b6dd862012-12-02 16:04:43 +00001430{
1431 TranslationBlock *tb;
1432
Andreas Färber93afead2013-08-26 03:41:01 +02001433 tb = tb_find_pc(cpu->mem_io_pc);
Blue Swirl5b6dd862012-12-02 16:04:43 +00001434 if (!tb) {
Andreas Färbera47dddd2013-09-03 17:38:47 +02001435 cpu_abort(cpu, "check_watchpoint: could not find TB for pc=%p",
Andreas Färber93afead2013-08-26 03:41:01 +02001436 (void *)cpu->mem_io_pc);
Blue Swirl5b6dd862012-12-02 16:04:43 +00001437 }
Andreas Färber74f10512013-09-01 17:02:58 +02001438 cpu_restore_state_from_tb(cpu, tb, cpu->mem_io_pc);
Blue Swirl5b6dd862012-12-02 16:04:43 +00001439 tb_phys_invalidate(tb, -1);
1440}
1441
1442#ifndef CONFIG_USER_ONLY
1443/* mask must never be zero, except for A20 change call */
Andreas Färberc3affe52013-01-18 15:03:43 +01001444static void tcg_handle_interrupt(CPUState *cpu, int mask)
Blue Swirl5b6dd862012-12-02 16:04:43 +00001445{
Blue Swirl5b6dd862012-12-02 16:04:43 +00001446 int old_mask;
1447
Andreas Färber259186a2013-01-17 18:51:17 +01001448 old_mask = cpu->interrupt_request;
1449 cpu->interrupt_request |= mask;
Blue Swirl5b6dd862012-12-02 16:04:43 +00001450
1451 /*
1452 * If called from iothread context, wake the target cpu in
1453 * case its halted.
1454 */
1455 if (!qemu_cpu_is_self(cpu)) {
1456 qemu_cpu_kick(cpu);
1457 return;
1458 }
1459
1460 if (use_icount) {
Andreas Färber28ecfd72013-08-26 05:51:49 +02001461 cpu->icount_decr.u16.high = 0xffff;
Andreas Färber99df7dc2013-08-26 05:15:23 +02001462 if (!cpu_can_do_io(cpu)
Blue Swirl5b6dd862012-12-02 16:04:43 +00001463 && (mask & ~old_mask) != 0) {
Andreas Färbera47dddd2013-09-03 17:38:47 +02001464 cpu_abort(cpu, "Raised interrupt while not in I/O function");
Blue Swirl5b6dd862012-12-02 16:04:43 +00001465 }
1466 } else {
Peter Maydell378df4b2013-02-22 18:10:03 +00001467 cpu->tcg_exit_req = 1;
Blue Swirl5b6dd862012-12-02 16:04:43 +00001468 }
1469}
1470
1471CPUInterruptHandler cpu_interrupt_handler = tcg_handle_interrupt;
1472
1473/* in deterministic execution mode, instructions doing device I/Os
1474 must be at the end of the TB */
Andreas Färber90b40a62013-09-01 17:21:47 +02001475void cpu_io_recompile(CPUState *cpu, uintptr_t retaddr)
Blue Swirl5b6dd862012-12-02 16:04:43 +00001476{
Andreas Färbera47dddd2013-09-03 17:38:47 +02001477#if defined(TARGET_MIPS) || defined(TARGET_SH4)
Andreas Färber90b40a62013-09-01 17:21:47 +02001478 CPUArchState *env = cpu->env_ptr;
Andreas Färbera47dddd2013-09-03 17:38:47 +02001479#endif
Blue Swirl5b6dd862012-12-02 16:04:43 +00001480 TranslationBlock *tb;
1481 uint32_t n, cflags;
1482 target_ulong pc, cs_base;
1483 uint64_t flags;
1484
1485 tb = tb_find_pc(retaddr);
1486 if (!tb) {
Andreas Färbera47dddd2013-09-03 17:38:47 +02001487 cpu_abort(cpu, "cpu_io_recompile: could not find TB for pc=%p",
Blue Swirl5b6dd862012-12-02 16:04:43 +00001488 (void *)retaddr);
1489 }
Andreas Färber28ecfd72013-08-26 05:51:49 +02001490 n = cpu->icount_decr.u16.low + tb->icount;
Andreas Färber74f10512013-09-01 17:02:58 +02001491 cpu_restore_state_from_tb(cpu, tb, retaddr);
Blue Swirl5b6dd862012-12-02 16:04:43 +00001492 /* Calculate how many instructions had been executed before the fault
1493 occurred. */
Andreas Färber28ecfd72013-08-26 05:51:49 +02001494 n = n - cpu->icount_decr.u16.low;
Blue Swirl5b6dd862012-12-02 16:04:43 +00001495 /* Generate a new TB ending on the I/O insn. */
1496 n++;
1497 /* On MIPS and SH, delay slot instructions can only be restarted if
1498 they were already the first instruction in the TB. If this is not
1499 the first instruction in a TB then re-execute the preceding
1500 branch. */
1501#if defined(TARGET_MIPS)
1502 if ((env->hflags & MIPS_HFLAG_BMASK) != 0 && n > 1) {
Maciej W. Rozyckic3577472014-11-07 20:05:35 +00001503 env->active_tc.PC -= (env->hflags & MIPS_HFLAG_B16 ? 2 : 4);
Andreas Färber28ecfd72013-08-26 05:51:49 +02001504 cpu->icount_decr.u16.low++;
Blue Swirl5b6dd862012-12-02 16:04:43 +00001505 env->hflags &= ~MIPS_HFLAG_BMASK;
1506 }
1507#elif defined(TARGET_SH4)
1508 if ((env->flags & ((DELAY_SLOT | DELAY_SLOT_CONDITIONAL))) != 0
1509 && n > 1) {
1510 env->pc -= 2;
Andreas Färber28ecfd72013-08-26 05:51:49 +02001511 cpu->icount_decr.u16.low++;
Blue Swirl5b6dd862012-12-02 16:04:43 +00001512 env->flags &= ~(DELAY_SLOT | DELAY_SLOT_CONDITIONAL);
1513 }
1514#endif
1515 /* This should never happen. */
1516 if (n > CF_COUNT_MASK) {
Andreas Färbera47dddd2013-09-03 17:38:47 +02001517 cpu_abort(cpu, "TB too big during recompile");
Blue Swirl5b6dd862012-12-02 16:04:43 +00001518 }
1519
1520 cflags = n | CF_LAST_IO;
1521 pc = tb->pc;
1522 cs_base = tb->cs_base;
1523 flags = tb->flags;
1524 tb_phys_invalidate(tb, -1);
1525 /* FIXME: In theory this could raise an exception. In practice
1526 we have already translated the block once so it's probably ok. */
Andreas Färber648f0342013-09-01 17:43:17 +02001527 tb_gen_code(cpu, pc, cs_base, flags, cflags);
Blue Swirl5b6dd862012-12-02 16:04:43 +00001528 /* TODO: If env->pc != tb->pc (i.e. the faulting instruction was not
1529 the first in the TB) then we end up generating a whole new TB and
1530 repeating the fault, which is horribly inefficient.
1531 Better would be to execute just this insn uncached, or generate a
1532 second new TB. */
Andreas Färber0ea8cb82013-09-03 02:12:23 +02001533 cpu_resume_from_signal(cpu, NULL);
Blue Swirl5b6dd862012-12-02 16:04:43 +00001534}
1535
Andreas Färber611d4f92013-09-01 17:52:07 +02001536void tb_flush_jmp_cache(CPUState *cpu, target_ulong addr)
Blue Swirl5b6dd862012-12-02 16:04:43 +00001537{
1538 unsigned int i;
1539
1540 /* Discard jump cache entries for any tb which might potentially
1541 overlap the flushed page. */
1542 i = tb_jmp_cache_hash_page(addr - TARGET_PAGE_SIZE);
Andreas Färber8cd70432013-08-26 06:03:38 +02001543 memset(&cpu->tb_jmp_cache[i], 0,
Blue Swirl5b6dd862012-12-02 16:04:43 +00001544 TB_JMP_PAGE_SIZE * sizeof(TranslationBlock *));
1545
1546 i = tb_jmp_cache_hash_page(addr);
Andreas Färber8cd70432013-08-26 06:03:38 +02001547 memset(&cpu->tb_jmp_cache[i], 0,
Blue Swirl5b6dd862012-12-02 16:04:43 +00001548 TB_JMP_PAGE_SIZE * sizeof(TranslationBlock *));
1549}
1550
1551void dump_exec_info(FILE *f, fprintf_function cpu_fprintf)
1552{
1553 int i, target_code_size, max_target_code_size;
1554 int direct_jmp_count, direct_jmp2_count, cross_page;
1555 TranslationBlock *tb;
1556
1557 target_code_size = 0;
1558 max_target_code_size = 0;
1559 cross_page = 0;
1560 direct_jmp_count = 0;
1561 direct_jmp2_count = 0;
Evgeny Voevodin5e5f07e2013-02-01 01:47:23 +07001562 for (i = 0; i < tcg_ctx.tb_ctx.nb_tbs; i++) {
1563 tb = &tcg_ctx.tb_ctx.tbs[i];
Blue Swirl5b6dd862012-12-02 16:04:43 +00001564 target_code_size += tb->size;
1565 if (tb->size > max_target_code_size) {
1566 max_target_code_size = tb->size;
1567 }
1568 if (tb->page_addr[1] != -1) {
1569 cross_page++;
1570 }
1571 if (tb->tb_next_offset[0] != 0xffff) {
1572 direct_jmp_count++;
1573 if (tb->tb_next_offset[1] != 0xffff) {
1574 direct_jmp2_count++;
1575 }
1576 }
1577 }
1578 /* XXX: avoid using doubles ? */
1579 cpu_fprintf(f, "Translation buffer state:\n");
1580 cpu_fprintf(f, "gen code size %td/%zd\n",
Evgeny Voevodin0b0d3322013-02-01 01:47:22 +07001581 tcg_ctx.code_gen_ptr - tcg_ctx.code_gen_buffer,
1582 tcg_ctx.code_gen_buffer_max_size);
Blue Swirl5b6dd862012-12-02 16:04:43 +00001583 cpu_fprintf(f, "TB count %d/%d\n",
Evgeny Voevodin5e5f07e2013-02-01 01:47:23 +07001584 tcg_ctx.tb_ctx.nb_tbs, tcg_ctx.code_gen_max_blocks);
Blue Swirl5b6dd862012-12-02 16:04:43 +00001585 cpu_fprintf(f, "TB avg target size %d max=%d bytes\n",
Evgeny Voevodin5e5f07e2013-02-01 01:47:23 +07001586 tcg_ctx.tb_ctx.nb_tbs ? target_code_size /
1587 tcg_ctx.tb_ctx.nb_tbs : 0,
1588 max_target_code_size);
Blue Swirl5b6dd862012-12-02 16:04:43 +00001589 cpu_fprintf(f, "TB avg host size %td bytes (expansion ratio: %0.1f)\n",
Evgeny Voevodin5e5f07e2013-02-01 01:47:23 +07001590 tcg_ctx.tb_ctx.nb_tbs ? (tcg_ctx.code_gen_ptr -
1591 tcg_ctx.code_gen_buffer) /
1592 tcg_ctx.tb_ctx.nb_tbs : 0,
1593 target_code_size ? (double) (tcg_ctx.code_gen_ptr -
1594 tcg_ctx.code_gen_buffer) /
1595 target_code_size : 0);
1596 cpu_fprintf(f, "cross page TB count %d (%d%%)\n", cross_page,
1597 tcg_ctx.tb_ctx.nb_tbs ? (cross_page * 100) /
1598 tcg_ctx.tb_ctx.nb_tbs : 0);
Blue Swirl5b6dd862012-12-02 16:04:43 +00001599 cpu_fprintf(f, "direct jump count %d (%d%%) (2 jumps=%d %d%%)\n",
1600 direct_jmp_count,
Evgeny Voevodin5e5f07e2013-02-01 01:47:23 +07001601 tcg_ctx.tb_ctx.nb_tbs ? (direct_jmp_count * 100) /
1602 tcg_ctx.tb_ctx.nb_tbs : 0,
Blue Swirl5b6dd862012-12-02 16:04:43 +00001603 direct_jmp2_count,
Evgeny Voevodin5e5f07e2013-02-01 01:47:23 +07001604 tcg_ctx.tb_ctx.nb_tbs ? (direct_jmp2_count * 100) /
1605 tcg_ctx.tb_ctx.nb_tbs : 0);
Blue Swirl5b6dd862012-12-02 16:04:43 +00001606 cpu_fprintf(f, "\nStatistics:\n");
Evgeny Voevodin5e5f07e2013-02-01 01:47:23 +07001607 cpu_fprintf(f, "TB flush count %d\n", tcg_ctx.tb_ctx.tb_flush_count);
1608 cpu_fprintf(f, "TB invalidate count %d\n",
1609 tcg_ctx.tb_ctx.tb_phys_invalidate_count);
Blue Swirl5b6dd862012-12-02 16:04:43 +00001610 cpu_fprintf(f, "TLB flush count %d\n", tlb_flush_count);
1611 tcg_dump_info(f, cpu_fprintf);
1612}
1613
Max Filippov246ae242014-11-02 11:04:18 +03001614void dump_opcount_info(FILE *f, fprintf_function cpu_fprintf)
1615{
1616 tcg_dump_op_count(f, cpu_fprintf);
1617}
1618
Blue Swirl5b6dd862012-12-02 16:04:43 +00001619#else /* CONFIG_USER_ONLY */
1620
Andreas Färberc3affe52013-01-18 15:03:43 +01001621void cpu_interrupt(CPUState *cpu, int mask)
Blue Swirl5b6dd862012-12-02 16:04:43 +00001622{
Andreas Färber259186a2013-01-17 18:51:17 +01001623 cpu->interrupt_request |= mask;
Peter Maydell378df4b2013-02-22 18:10:03 +00001624 cpu->tcg_exit_req = 1;
Blue Swirl5b6dd862012-12-02 16:04:43 +00001625}
1626
1627/*
1628 * Walks guest process memory "regions" one by one
1629 * and calls callback function 'fn' for each region.
1630 */
1631struct walk_memory_regions_data {
1632 walk_memory_regions_fn fn;
1633 void *priv;
Mikhail Ilyin1a1c4db2014-09-08 17:28:56 +04001634 target_ulong start;
Blue Swirl5b6dd862012-12-02 16:04:43 +00001635 int prot;
1636};
1637
1638static int walk_memory_regions_end(struct walk_memory_regions_data *data,
Mikhail Ilyin1a1c4db2014-09-08 17:28:56 +04001639 target_ulong end, int new_prot)
Blue Swirl5b6dd862012-12-02 16:04:43 +00001640{
Mikhail Ilyin1a1c4db2014-09-08 17:28:56 +04001641 if (data->start != -1u) {
Blue Swirl5b6dd862012-12-02 16:04:43 +00001642 int rc = data->fn(data->priv, data->start, end, data->prot);
1643 if (rc != 0) {
1644 return rc;
1645 }
1646 }
1647
Mikhail Ilyin1a1c4db2014-09-08 17:28:56 +04001648 data->start = (new_prot ? end : -1u);
Blue Swirl5b6dd862012-12-02 16:04:43 +00001649 data->prot = new_prot;
1650
1651 return 0;
1652}
1653
1654static int walk_memory_regions_1(struct walk_memory_regions_data *data,
Mikhail Ilyin1a1c4db2014-09-08 17:28:56 +04001655 target_ulong base, int level, void **lp)
Blue Swirl5b6dd862012-12-02 16:04:43 +00001656{
Mikhail Ilyin1a1c4db2014-09-08 17:28:56 +04001657 target_ulong pa;
Blue Swirl5b6dd862012-12-02 16:04:43 +00001658 int i, rc;
1659
1660 if (*lp == NULL) {
1661 return walk_memory_regions_end(data, base, 0);
1662 }
1663
1664 if (level == 0) {
1665 PageDesc *pd = *lp;
1666
Paolo Bonzini03f49952013-11-07 17:14:36 +01001667 for (i = 0; i < V_L2_SIZE; ++i) {
Blue Swirl5b6dd862012-12-02 16:04:43 +00001668 int prot = pd[i].flags;
1669
1670 pa = base | (i << TARGET_PAGE_BITS);
1671 if (prot != data->prot) {
1672 rc = walk_memory_regions_end(data, pa, prot);
1673 if (rc != 0) {
1674 return rc;
1675 }
1676 }
1677 }
1678 } else {
1679 void **pp = *lp;
1680
Paolo Bonzini03f49952013-11-07 17:14:36 +01001681 for (i = 0; i < V_L2_SIZE; ++i) {
Mikhail Ilyin1a1c4db2014-09-08 17:28:56 +04001682 pa = base | ((target_ulong)i <<
Paolo Bonzini03f49952013-11-07 17:14:36 +01001683 (TARGET_PAGE_BITS + V_L2_BITS * level));
Blue Swirl5b6dd862012-12-02 16:04:43 +00001684 rc = walk_memory_regions_1(data, pa, level - 1, pp + i);
1685 if (rc != 0) {
1686 return rc;
1687 }
1688 }
1689 }
1690
1691 return 0;
1692}
1693
1694int walk_memory_regions(void *priv, walk_memory_regions_fn fn)
1695{
1696 struct walk_memory_regions_data data;
1697 uintptr_t i;
1698
1699 data.fn = fn;
1700 data.priv = priv;
Mikhail Ilyin1a1c4db2014-09-08 17:28:56 +04001701 data.start = -1u;
Blue Swirl5b6dd862012-12-02 16:04:43 +00001702 data.prot = 0;
1703
1704 for (i = 0; i < V_L1_SIZE; i++) {
Mikhail Ilyin1a1c4db2014-09-08 17:28:56 +04001705 int rc = walk_memory_regions_1(&data, (target_ulong)i << (V_L1_SHIFT + TARGET_PAGE_BITS),
Paolo Bonzini03f49952013-11-07 17:14:36 +01001706 V_L1_SHIFT / V_L2_BITS - 1, l1_map + i);
Blue Swirl5b6dd862012-12-02 16:04:43 +00001707 if (rc != 0) {
1708 return rc;
1709 }
1710 }
1711
1712 return walk_memory_regions_end(&data, 0, 0);
1713}
1714
Mikhail Ilyin1a1c4db2014-09-08 17:28:56 +04001715static int dump_region(void *priv, target_ulong start,
1716 target_ulong end, unsigned long prot)
Blue Swirl5b6dd862012-12-02 16:04:43 +00001717{
1718 FILE *f = (FILE *)priv;
1719
Mikhail Ilyin1a1c4db2014-09-08 17:28:56 +04001720 (void) fprintf(f, TARGET_FMT_lx"-"TARGET_FMT_lx
1721 " "TARGET_FMT_lx" %c%c%c\n",
Blue Swirl5b6dd862012-12-02 16:04:43 +00001722 start, end, end - start,
1723 ((prot & PAGE_READ) ? 'r' : '-'),
1724 ((prot & PAGE_WRITE) ? 'w' : '-'),
1725 ((prot & PAGE_EXEC) ? 'x' : '-'));
1726
1727 return 0;
1728}
1729
1730/* dump memory mappings */
1731void page_dump(FILE *f)
1732{
Mikhail Ilyin1a1c4db2014-09-08 17:28:56 +04001733 const int length = sizeof(target_ulong) * 2;
Stefan Weil227b8172013-09-12 20:09:06 +02001734 (void) fprintf(f, "%-*s %-*s %-*s %s\n",
1735 length, "start", length, "end", length, "size", "prot");
Blue Swirl5b6dd862012-12-02 16:04:43 +00001736 walk_memory_regions(f, dump_region);
1737}
1738
1739int page_get_flags(target_ulong address)
1740{
1741 PageDesc *p;
1742
1743 p = page_find(address >> TARGET_PAGE_BITS);
1744 if (!p) {
1745 return 0;
1746 }
1747 return p->flags;
1748}
1749
1750/* Modify the flags of a page and invalidate the code if necessary.
1751 The flag PAGE_WRITE_ORG is positioned automatically depending
1752 on PAGE_WRITE. The mmap_lock should already be held. */
1753void page_set_flags(target_ulong start, target_ulong end, int flags)
1754{
1755 target_ulong addr, len;
1756
1757 /* This function should never be called with addresses outside the
1758 guest address space. If this assert fires, it probably indicates
1759 a missing call to h2g_valid. */
1760#if TARGET_ABI_BITS > L1_MAP_ADDR_SPACE_BITS
Mikhail Ilyin1a1c4db2014-09-08 17:28:56 +04001761 assert(end < ((target_ulong)1 << L1_MAP_ADDR_SPACE_BITS));
Blue Swirl5b6dd862012-12-02 16:04:43 +00001762#endif
1763 assert(start < end);
1764
1765 start = start & TARGET_PAGE_MASK;
1766 end = TARGET_PAGE_ALIGN(end);
1767
1768 if (flags & PAGE_WRITE) {
1769 flags |= PAGE_WRITE_ORG;
1770 }
1771
1772 for (addr = start, len = end - start;
1773 len != 0;
1774 len -= TARGET_PAGE_SIZE, addr += TARGET_PAGE_SIZE) {
1775 PageDesc *p = page_find_alloc(addr >> TARGET_PAGE_BITS, 1);
1776
1777 /* If the write protection bit is set, then we invalidate
1778 the code inside. */
1779 if (!(p->flags & PAGE_WRITE) &&
1780 (flags & PAGE_WRITE) &&
1781 p->first_tb) {
Alexander Grafd02532f2013-07-06 14:17:57 +02001782 tb_invalidate_phys_page(addr, 0, NULL, false);
Blue Swirl5b6dd862012-12-02 16:04:43 +00001783 }
1784 p->flags = flags;
1785 }
1786}
1787
1788int page_check_range(target_ulong start, target_ulong len, int flags)
1789{
1790 PageDesc *p;
1791 target_ulong end;
1792 target_ulong addr;
1793
1794 /* This function should never be called with addresses outside the
1795 guest address space. If this assert fires, it probably indicates
1796 a missing call to h2g_valid. */
1797#if TARGET_ABI_BITS > L1_MAP_ADDR_SPACE_BITS
Mikhail Ilyin1a1c4db2014-09-08 17:28:56 +04001798 assert(start < ((target_ulong)1 << L1_MAP_ADDR_SPACE_BITS));
Blue Swirl5b6dd862012-12-02 16:04:43 +00001799#endif
1800
1801 if (len == 0) {
1802 return 0;
1803 }
1804 if (start + len - 1 < start) {
1805 /* We've wrapped around. */
1806 return -1;
1807 }
1808
1809 /* must do before we loose bits in the next step */
1810 end = TARGET_PAGE_ALIGN(start + len);
1811 start = start & TARGET_PAGE_MASK;
1812
1813 for (addr = start, len = end - start;
1814 len != 0;
1815 len -= TARGET_PAGE_SIZE, addr += TARGET_PAGE_SIZE) {
1816 p = page_find(addr >> TARGET_PAGE_BITS);
1817 if (!p) {
1818 return -1;
1819 }
1820 if (!(p->flags & PAGE_VALID)) {
1821 return -1;
1822 }
1823
1824 if ((flags & PAGE_READ) && !(p->flags & PAGE_READ)) {
1825 return -1;
1826 }
1827 if (flags & PAGE_WRITE) {
1828 if (!(p->flags & PAGE_WRITE_ORG)) {
1829 return -1;
1830 }
1831 /* unprotect the page if it was put read-only because it
1832 contains translated code */
1833 if (!(p->flags & PAGE_WRITE)) {
1834 if (!page_unprotect(addr, 0, NULL)) {
1835 return -1;
1836 }
1837 }
Blue Swirl5b6dd862012-12-02 16:04:43 +00001838 }
1839 }
1840 return 0;
1841}
1842
1843/* called from signal handler: invalidate the code and unprotect the
1844 page. Return TRUE if the fault was successfully handled. */
1845int page_unprotect(target_ulong address, uintptr_t pc, void *puc)
1846{
1847 unsigned int prot;
1848 PageDesc *p;
1849 target_ulong host_start, host_end, addr;
1850
1851 /* Technically this isn't safe inside a signal handler. However we
1852 know this only ever happens in a synchronous SEGV handler, so in
1853 practice it seems to be ok. */
1854 mmap_lock();
1855
1856 p = page_find(address >> TARGET_PAGE_BITS);
1857 if (!p) {
1858 mmap_unlock();
1859 return 0;
1860 }
1861
1862 /* if the page was really writable, then we change its
1863 protection back to writable */
1864 if ((p->flags & PAGE_WRITE_ORG) && !(p->flags & PAGE_WRITE)) {
1865 host_start = address & qemu_host_page_mask;
1866 host_end = host_start + qemu_host_page_size;
1867
1868 prot = 0;
1869 for (addr = host_start ; addr < host_end ; addr += TARGET_PAGE_SIZE) {
1870 p = page_find(addr >> TARGET_PAGE_BITS);
1871 p->flags |= PAGE_WRITE;
1872 prot |= p->flags;
1873
1874 /* and since the content will be modified, we must invalidate
1875 the corresponding translated code. */
Alexander Grafd02532f2013-07-06 14:17:57 +02001876 tb_invalidate_phys_page(addr, pc, puc, true);
Blue Swirl5b6dd862012-12-02 16:04:43 +00001877#ifdef DEBUG_TB_CHECK
1878 tb_invalidate_check(addr);
1879#endif
1880 }
1881 mprotect((void *)g2h(host_start), qemu_host_page_size,
1882 prot & PAGE_BITS);
1883
1884 mmap_unlock();
1885 return 1;
1886 }
1887 mmap_unlock();
1888 return 0;
1889}
1890#endif /* CONFIG_USER_ONLY */