blob: 9f47ce7f71453229f2b299c566b5562193996857 [file] [log] [blame]
bellardd19893d2003-06-15 19:58:51 +00001/*
2 * Host code generation
ths5fafdf22007-09-16 21:08:06 +00003 *
bellardd19893d2003-06-15 19:58:51 +00004 * Copyright (c) 2003 Fabrice Bellard
5 *
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2 of the License, or (at your option) any later version.
10 *
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
15 *
16 * You should have received a copy of the GNU Lesser General Public
Blue Swirl8167ee82009-07-16 20:47:01 +000017 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
bellardd19893d2003-06-15 19:58:51 +000018 */
Blue Swirl5b6dd862012-12-02 16:04:43 +000019#ifdef _WIN32
20#include <windows.h>
21#else
22#include <sys/types.h>
23#include <sys/mman.h>
24#endif
bellardd19893d2003-06-15 19:58:51 +000025#include <stdarg.h>
26#include <stdlib.h>
27#include <stdio.h>
28#include <string.h>
29#include <inttypes.h>
30
31#include "config.h"
bellard20543962003-06-15 23:28:43 +000032
Blue Swirl5b6dd862012-12-02 16:04:43 +000033#include "qemu-common.h"
bellardaf5ad102004-01-04 23:28:12 +000034#define NO_CPU_IO_DEFS
bellardd3eead22003-09-30 20:59:51 +000035#include "cpu.h"
Alex Bennée6db8b532014-08-01 17:08:57 +010036#include "trace.h"
Paolo Bonzini76cad712012-10-24 11:12:21 +020037#include "disas/disas.h"
bellard57fec1f2008-02-01 10:50:11 +000038#include "tcg.h"
Blue Swirl5b6dd862012-12-02 16:04:43 +000039#if defined(CONFIG_USER_ONLY)
40#include "qemu.h"
41#if defined(__FreeBSD__) || defined(__FreeBSD_kernel__)
42#include <sys/param.h>
43#if __FreeBSD_version >= 700104
44#define HAVE_KINFO_GETVMMAP
45#define sigqueue sigqueue_freebsd /* avoid redefinition */
46#include <sys/time.h>
47#include <sys/proc.h>
48#include <machine/profile.h>
49#define _KERNEL
50#include <sys/user.h>
51#undef _KERNEL
52#undef sigqueue
53#include <libutil.h>
54#endif
55#endif
Paolo Bonzini0bc3cd62013-04-08 17:29:59 +020056#else
57#include "exec/address-spaces.h"
Blue Swirl5b6dd862012-12-02 16:04:43 +000058#endif
59
Paolo Bonzini022c62c2012-12-17 18:19:49 +010060#include "exec/cputlb.h"
Blue Swirl5b6dd862012-12-02 16:04:43 +000061#include "translate-all.h"
Alexey Kardashevskiy0aa09892013-04-22 17:42:50 +100062#include "qemu/timer.h"
Blue Swirl5b6dd862012-12-02 16:04:43 +000063
64//#define DEBUG_TB_INVALIDATE
65//#define DEBUG_FLUSH
66/* make various TB consistency checks */
67//#define DEBUG_TB_CHECK
68
69#if !defined(CONFIG_USER_ONLY)
70/* TB consistency checks only implemented for usermode emulation. */
71#undef DEBUG_TB_CHECK
72#endif
73
74#define SMC_BITMAP_USE_THRESHOLD 10
75
Blue Swirl5b6dd862012-12-02 16:04:43 +000076typedef struct PageDesc {
77 /* list of TBs intersecting this ram page */
78 TranslationBlock *first_tb;
79 /* in order to optimize self modifying code, we count the number
80 of lookups we do to a given page to use a bitmap */
81 unsigned int code_write_count;
82 uint8_t *code_bitmap;
83#if defined(CONFIG_USER_ONLY)
84 unsigned long flags;
85#endif
86} PageDesc;
87
88/* In system mode we want L1_MAP to be based on ram offsets,
89 while in user mode we want it to be based on virtual addresses. */
90#if !defined(CONFIG_USER_ONLY)
91#if HOST_LONG_BITS < TARGET_PHYS_ADDR_SPACE_BITS
92# define L1_MAP_ADDR_SPACE_BITS HOST_LONG_BITS
93#else
94# define L1_MAP_ADDR_SPACE_BITS TARGET_PHYS_ADDR_SPACE_BITS
95#endif
96#else
97# define L1_MAP_ADDR_SPACE_BITS TARGET_VIRT_ADDR_SPACE_BITS
98#endif
99
Paolo Bonzini03f49952013-11-07 17:14:36 +0100100/* Size of the L2 (and L3, etc) page tables. */
101#define V_L2_BITS 10
102#define V_L2_SIZE (1 << V_L2_BITS)
103
Blue Swirl5b6dd862012-12-02 16:04:43 +0000104/* The bits remaining after N lower levels of page tables. */
105#define V_L1_BITS_REM \
Paolo Bonzini03f49952013-11-07 17:14:36 +0100106 ((L1_MAP_ADDR_SPACE_BITS - TARGET_PAGE_BITS) % V_L2_BITS)
Blue Swirl5b6dd862012-12-02 16:04:43 +0000107
108#if V_L1_BITS_REM < 4
Paolo Bonzini03f49952013-11-07 17:14:36 +0100109#define V_L1_BITS (V_L1_BITS_REM + V_L2_BITS)
Blue Swirl5b6dd862012-12-02 16:04:43 +0000110#else
111#define V_L1_BITS V_L1_BITS_REM
112#endif
113
114#define V_L1_SIZE ((target_ulong)1 << V_L1_BITS)
115
116#define V_L1_SHIFT (L1_MAP_ADDR_SPACE_BITS - TARGET_PAGE_BITS - V_L1_BITS)
117
118uintptr_t qemu_real_host_page_size;
119uintptr_t qemu_host_page_size;
120uintptr_t qemu_host_page_mask;
121
122/* This is a multi-level map on the virtual address space.
123 The bottom level has pointers to PageDesc. */
124static void *l1_map[V_L1_SIZE];
125
bellard57fec1f2008-02-01 10:50:11 +0000126/* code generation context */
127TCGContext tcg_ctx;
bellardd19893d2003-06-15 19:58:51 +0000128
Blue Swirl5b6dd862012-12-02 16:04:43 +0000129static void tb_link_page(TranslationBlock *tb, tb_page_addr_t phys_pc,
130 tb_page_addr_t phys_page2);
Blue Swirla8a826a2012-12-04 20:16:07 +0000131static TranslationBlock *tb_find_pc(uintptr_t tc_ptr);
Blue Swirl5b6dd862012-12-02 16:04:43 +0000132
bellard57fec1f2008-02-01 10:50:11 +0000133void cpu_gen_init(void)
134{
135 tcg_context_init(&tcg_ctx);
bellard57fec1f2008-02-01 10:50:11 +0000136}
137
bellardd19893d2003-06-15 19:58:51 +0000138/* return non zero if the very first instruction is invalid so that
ths5fafdf22007-09-16 21:08:06 +0000139 the virtual CPU can trigger an exception.
bellardd19893d2003-06-15 19:58:51 +0000140
141 '*gen_code_size_ptr' contains the size of the generated code (host
142 code).
143*/
Andreas Färber9349b4f2012-03-14 01:38:32 +0100144int cpu_gen_code(CPUArchState *env, TranslationBlock *tb, int *gen_code_size_ptr)
bellardd19893d2003-06-15 19:58:51 +0000145{
bellard57fec1f2008-02-01 10:50:11 +0000146 TCGContext *s = &tcg_ctx;
Richard Henderson1813e172014-03-28 12:56:22 -0700147 tcg_insn_unit *gen_code_buf;
bellardd19893d2003-06-15 19:58:51 +0000148 int gen_code_size;
bellard57fec1f2008-02-01 10:50:11 +0000149#ifdef CONFIG_PROFILER
150 int64_t ti;
151#endif
152
153#ifdef CONFIG_PROFILER
bellardb67d9a52008-05-23 09:57:34 +0000154 s->tb_count1++; /* includes aborted translations because of
155 exceptions */
bellard57fec1f2008-02-01 10:50:11 +0000156 ti = profile_getclock();
157#endif
158 tcg_func_start(s);
bellardd19893d2003-06-15 19:58:51 +0000159
ths2cfc5f12008-07-18 18:01:29 +0000160 gen_intermediate_code(env, tb);
161
Alex Bennée6db8b532014-08-01 17:08:57 +0100162 trace_translate_block(tb, tb->pc, tb->tc_ptr);
163
bellardec6338b2007-11-08 14:25:03 +0000164 /* generate machine code */
bellard57fec1f2008-02-01 10:50:11 +0000165 gen_code_buf = tb->tc_ptr;
bellardec6338b2007-11-08 14:25:03 +0000166 tb->tb_next_offset[0] = 0xffff;
167 tb->tb_next_offset[1] = 0xffff;
bellard57fec1f2008-02-01 10:50:11 +0000168 s->tb_next_offset = tb->tb_next_offset;
bellard4cbb86e2003-09-17 22:53:29 +0000169#ifdef USE_DIRECT_JUMP
bellard57fec1f2008-02-01 10:50:11 +0000170 s->tb_jmp_offset = tb->tb_jmp_offset;
171 s->tb_next = NULL;
bellardd19893d2003-06-15 19:58:51 +0000172#else
bellard57fec1f2008-02-01 10:50:11 +0000173 s->tb_jmp_offset = NULL;
174 s->tb_next = tb->tb_next;
bellardd19893d2003-06-15 19:58:51 +0000175#endif
bellard57fec1f2008-02-01 10:50:11 +0000176
177#ifdef CONFIG_PROFILER
bellardb67d9a52008-05-23 09:57:34 +0000178 s->tb_count++;
179 s->interm_time += profile_getclock() - ti;
180 s->code_time -= profile_getclock();
bellard57fec1f2008-02-01 10:50:11 +0000181#endif
aurel3254604f72008-12-07 20:35:00 +0000182 gen_code_size = tcg_gen_code(s, gen_code_buf);
bellardd19893d2003-06-15 19:58:51 +0000183 *gen_code_size_ptr = gen_code_size;
bellard57fec1f2008-02-01 10:50:11 +0000184#ifdef CONFIG_PROFILER
bellardb67d9a52008-05-23 09:57:34 +0000185 s->code_time += profile_getclock();
186 s->code_in_len += tb->size;
187 s->code_out_len += gen_code_size;
bellard57fec1f2008-02-01 10:50:11 +0000188#endif
189
bellardd19893d2003-06-15 19:58:51 +0000190#ifdef DEBUG_DISAS
aliguori8fec2b82009-01-15 22:36:53 +0000191 if (qemu_loglevel_mask(CPU_LOG_TB_OUT_ASM)) {
Richard Henderson1813e172014-03-28 12:56:22 -0700192 qemu_log("OUT: [size=%d]\n", gen_code_size);
193 log_disas(tb->tc_ptr, gen_code_size);
aliguori93fcfe32009-01-15 22:34:14 +0000194 qemu_log("\n");
aliguori31b1a7b2009-01-15 22:35:09 +0000195 qemu_log_flush();
bellardd19893d2003-06-15 19:58:51 +0000196 }
197#endif
198 return 0;
199}
200
ths5fafdf22007-09-16 21:08:06 +0000201/* The cpu state corresponding to 'searched_pc' is restored.
bellardd19893d2003-06-15 19:58:51 +0000202 */
Andreas Färber74f10512013-09-01 17:02:58 +0200203static int cpu_restore_state_from_tb(CPUState *cpu, TranslationBlock *tb,
Blue Swirla8a826a2012-12-04 20:16:07 +0000204 uintptr_t searched_pc)
bellardd19893d2003-06-15 19:58:51 +0000205{
Andreas Färber74f10512013-09-01 17:02:58 +0200206 CPUArchState *env = cpu->env_ptr;
bellard57fec1f2008-02-01 10:50:11 +0000207 TCGContext *s = &tcg_ctx;
208 int j;
Stefan Weil6375e092012-04-06 22:26:15 +0200209 uintptr_t tc_ptr;
bellard57fec1f2008-02-01 10:50:11 +0000210#ifdef CONFIG_PROFILER
211 int64_t ti;
212#endif
213
214#ifdef CONFIG_PROFILER
215 ti = profile_getclock();
216#endif
217 tcg_func_start(s);
bellardd19893d2003-06-15 19:58:51 +0000218
ths2cfc5f12008-07-18 18:01:29 +0000219 gen_intermediate_code_pc(env, tb);
ths3b46e622007-09-17 08:09:54 +0000220
Paolo Bonzinibd792552014-11-26 13:39:59 +0300221 if (tb->cflags & CF_USE_ICOUNT) {
pbrook2e70f6e2008-06-29 01:03:05 +0000222 /* Reset the cycle counter to the start of the block. */
Andreas Färber28ecfd72013-08-26 05:51:49 +0200223 cpu->icount_decr.u16.low += tb->icount;
pbrook2e70f6e2008-06-29 01:03:05 +0000224 /* Clear the IO flag. */
Andreas Färber99df7dc2013-08-26 05:15:23 +0200225 cpu->can_do_io = 0;
pbrook2e70f6e2008-06-29 01:03:05 +0000226 }
227
bellardd19893d2003-06-15 19:58:51 +0000228 /* find opc index corresponding to search_pc */
Stefan Weil6375e092012-04-06 22:26:15 +0200229 tc_ptr = (uintptr_t)tb->tc_ptr;
bellardd19893d2003-06-15 19:58:51 +0000230 if (searched_pc < tc_ptr)
231 return -1;
bellard57fec1f2008-02-01 10:50:11 +0000232
233 s->tb_next_offset = tb->tb_next_offset;
234#ifdef USE_DIRECT_JUMP
235 s->tb_jmp_offset = tb->tb_jmp_offset;
236 s->tb_next = NULL;
237#else
238 s->tb_jmp_offset = NULL;
239 s->tb_next = tb->tb_next;
240#endif
Richard Henderson1813e172014-03-28 12:56:22 -0700241 j = tcg_gen_code_search_pc(s, (tcg_insn_unit *)tc_ptr,
242 searched_pc - tc_ptr);
bellard57fec1f2008-02-01 10:50:11 +0000243 if (j < 0)
244 return -1;
bellardd19893d2003-06-15 19:58:51 +0000245 /* now find start of instruction before */
Evgeny Voevodinab1103d2012-11-21 11:43:06 +0400246 while (s->gen_opc_instr_start[j] == 0) {
bellardd19893d2003-06-15 19:58:51 +0000247 j--;
Evgeny Voevodinab1103d2012-11-21 11:43:06 +0400248 }
Andreas Färber28ecfd72013-08-26 05:51:49 +0200249 cpu->icount_decr.u16.low -= s->gen_opc_icount[j];
ths3b46e622007-09-17 08:09:54 +0000250
Stefan Weile87b7cb2011-04-18 06:39:52 +0000251 restore_state_to_opc(env, tb, j);
bellard57fec1f2008-02-01 10:50:11 +0000252
253#ifdef CONFIG_PROFILER
bellardb67d9a52008-05-23 09:57:34 +0000254 s->restore_time += profile_getclock() - ti;
255 s->restore_count++;
bellard57fec1f2008-02-01 10:50:11 +0000256#endif
bellardd19893d2003-06-15 19:58:51 +0000257 return 0;
258}
Blue Swirl5b6dd862012-12-02 16:04:43 +0000259
Andreas Färber3f38f302013-09-01 16:51:34 +0200260bool cpu_restore_state(CPUState *cpu, uintptr_t retaddr)
Blue Swirla8a826a2012-12-04 20:16:07 +0000261{
262 TranslationBlock *tb;
263
264 tb = tb_find_pc(retaddr);
265 if (tb) {
Andreas Färber74f10512013-09-01 17:02:58 +0200266 cpu_restore_state_from_tb(cpu, tb, retaddr);
Pavel Dovgalyukd8a499f2014-11-26 13:40:16 +0300267 if (tb->cflags & CF_NOCACHE) {
268 /* one-shot translation, invalidate it immediately */
269 cpu->current_tb = NULL;
270 tb_phys_invalidate(tb, -1);
271 tb_free(tb);
272 }
Blue Swirla8a826a2012-12-04 20:16:07 +0000273 return true;
274 }
275 return false;
276}
277
Blue Swirl5b6dd862012-12-02 16:04:43 +0000278#ifdef _WIN32
SeokYeon Hwang2d8ac5e2014-12-23 22:26:54 +0000279static __attribute__((unused)) void map_exec(void *addr, long size)
Blue Swirl5b6dd862012-12-02 16:04:43 +0000280{
281 DWORD old_protect;
282 VirtualProtect(addr, size,
283 PAGE_EXECUTE_READWRITE, &old_protect);
284}
285#else
SeokYeon Hwang2d8ac5e2014-12-23 22:26:54 +0000286static __attribute__((unused)) void map_exec(void *addr, long size)
Blue Swirl5b6dd862012-12-02 16:04:43 +0000287{
288 unsigned long start, end, page_size;
289
290 page_size = getpagesize();
291 start = (unsigned long)addr;
292 start &= ~(page_size - 1);
293
294 end = (unsigned long)addr + size;
295 end += page_size - 1;
296 end &= ~(page_size - 1);
297
298 mprotect((void *)start, end - start,
299 PROT_READ | PROT_WRITE | PROT_EXEC);
300}
301#endif
302
Alexey Kardashevskiy47c16ed2014-01-17 11:12:07 -0700303void page_size_init(void)
Blue Swirl5b6dd862012-12-02 16:04:43 +0000304{
305 /* NOTE: we can always suppose that qemu_host_page_size >=
306 TARGET_PAGE_SIZE */
Blue Swirl5b6dd862012-12-02 16:04:43 +0000307 qemu_real_host_page_size = getpagesize();
Blue Swirl5b6dd862012-12-02 16:04:43 +0000308 if (qemu_host_page_size == 0) {
309 qemu_host_page_size = qemu_real_host_page_size;
310 }
311 if (qemu_host_page_size < TARGET_PAGE_SIZE) {
312 qemu_host_page_size = TARGET_PAGE_SIZE;
313 }
314 qemu_host_page_mask = ~(qemu_host_page_size - 1);
Alexey Kardashevskiy47c16ed2014-01-17 11:12:07 -0700315}
Blue Swirl5b6dd862012-12-02 16:04:43 +0000316
Alexey Kardashevskiy47c16ed2014-01-17 11:12:07 -0700317static void page_init(void)
318{
319 page_size_init();
Blue Swirl5b6dd862012-12-02 16:04:43 +0000320#if defined(CONFIG_BSD) && defined(CONFIG_USER_ONLY)
321 {
322#ifdef HAVE_KINFO_GETVMMAP
323 struct kinfo_vmentry *freep;
324 int i, cnt;
325
326 freep = kinfo_getvmmap(getpid(), &cnt);
327 if (freep) {
328 mmap_lock();
329 for (i = 0; i < cnt; i++) {
330 unsigned long startaddr, endaddr;
331
332 startaddr = freep[i].kve_start;
333 endaddr = freep[i].kve_end;
334 if (h2g_valid(startaddr)) {
335 startaddr = h2g(startaddr) & TARGET_PAGE_MASK;
336
337 if (h2g_valid(endaddr)) {
338 endaddr = h2g(endaddr);
339 page_set_flags(startaddr, endaddr, PAGE_RESERVED);
340 } else {
341#if TARGET_ABI_BITS <= L1_MAP_ADDR_SPACE_BITS
342 endaddr = ~0ul;
343 page_set_flags(startaddr, endaddr, PAGE_RESERVED);
344#endif
345 }
346 }
347 }
348 free(freep);
349 mmap_unlock();
350 }
351#else
352 FILE *f;
353
354 last_brk = (unsigned long)sbrk(0);
355
356 f = fopen("/compat/linux/proc/self/maps", "r");
357 if (f) {
358 mmap_lock();
359
360 do {
361 unsigned long startaddr, endaddr;
362 int n;
363
364 n = fscanf(f, "%lx-%lx %*[^\n]\n", &startaddr, &endaddr);
365
366 if (n == 2 && h2g_valid(startaddr)) {
367 startaddr = h2g(startaddr) & TARGET_PAGE_MASK;
368
369 if (h2g_valid(endaddr)) {
370 endaddr = h2g(endaddr);
371 } else {
372 endaddr = ~0ul;
373 }
374 page_set_flags(startaddr, endaddr, PAGE_RESERVED);
375 }
376 } while (!feof(f));
377
378 fclose(f);
379 mmap_unlock();
380 }
381#endif
382 }
383#endif
384}
385
386static PageDesc *page_find_alloc(tb_page_addr_t index, int alloc)
387{
388 PageDesc *pd;
389 void **lp;
390 int i;
391
392#if defined(CONFIG_USER_ONLY)
393 /* We can't use g_malloc because it may recurse into a locked mutex. */
394# define ALLOC(P, SIZE) \
395 do { \
396 P = mmap(NULL, SIZE, PROT_READ | PROT_WRITE, \
397 MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); \
398 } while (0)
399#else
400# define ALLOC(P, SIZE) \
401 do { P = g_malloc0(SIZE); } while (0)
402#endif
403
404 /* Level 1. Always allocated. */
405 lp = l1_map + ((index >> V_L1_SHIFT) & (V_L1_SIZE - 1));
406
407 /* Level 2..N-1. */
Paolo Bonzini03f49952013-11-07 17:14:36 +0100408 for (i = V_L1_SHIFT / V_L2_BITS - 1; i > 0; i--) {
Blue Swirl5b6dd862012-12-02 16:04:43 +0000409 void **p = *lp;
410
411 if (p == NULL) {
412 if (!alloc) {
413 return NULL;
414 }
Paolo Bonzini03f49952013-11-07 17:14:36 +0100415 ALLOC(p, sizeof(void *) * V_L2_SIZE);
Blue Swirl5b6dd862012-12-02 16:04:43 +0000416 *lp = p;
417 }
418
Paolo Bonzini03f49952013-11-07 17:14:36 +0100419 lp = p + ((index >> (i * V_L2_BITS)) & (V_L2_SIZE - 1));
Blue Swirl5b6dd862012-12-02 16:04:43 +0000420 }
421
422 pd = *lp;
423 if (pd == NULL) {
424 if (!alloc) {
425 return NULL;
426 }
Paolo Bonzini03f49952013-11-07 17:14:36 +0100427 ALLOC(pd, sizeof(PageDesc) * V_L2_SIZE);
Blue Swirl5b6dd862012-12-02 16:04:43 +0000428 *lp = pd;
429 }
430
431#undef ALLOC
432
Paolo Bonzini03f49952013-11-07 17:14:36 +0100433 return pd + (index & (V_L2_SIZE - 1));
Blue Swirl5b6dd862012-12-02 16:04:43 +0000434}
435
436static inline PageDesc *page_find(tb_page_addr_t index)
437{
438 return page_find_alloc(index, 0);
439}
440
441#if !defined(CONFIG_USER_ONLY)
442#define mmap_lock() do { } while (0)
443#define mmap_unlock() do { } while (0)
444#endif
445
446#if defined(CONFIG_USER_ONLY)
447/* Currently it is not recommended to allocate big chunks of data in
448 user mode. It will change when a dedicated libc will be used. */
449/* ??? 64-bit hosts ought to have no problem mmaping data outside the
450 region in which the guest needs to run. Revisit this. */
451#define USE_STATIC_CODE_GEN_BUFFER
452#endif
453
454/* ??? Should configure for this, not list operating systems here. */
455#if (defined(__linux__) \
456 || defined(__FreeBSD__) || defined(__FreeBSD_kernel__) \
457 || defined(__DragonFly__) || defined(__OpenBSD__) \
458 || defined(__NetBSD__))
459# define USE_MMAP
460#endif
461
462/* Minimum size of the code gen buffer. This number is randomly chosen,
463 but not so small that we can't have a fair number of TB's live. */
464#define MIN_CODE_GEN_BUFFER_SIZE (1024u * 1024)
465
466/* Maximum size of the code gen buffer we'd like to use. Unless otherwise
467 indicated, this is constrained by the range of direct branches on the
468 host cpu, as used by the TCG implementation of goto_tb. */
469#if defined(__x86_64__)
470# define MAX_CODE_GEN_BUFFER_SIZE (2ul * 1024 * 1024 * 1024)
471#elif defined(__sparc__)
472# define MAX_CODE_GEN_BUFFER_SIZE (2ul * 1024 * 1024 * 1024)
Claudio Fontana4a136e02013-06-12 16:20:22 +0100473#elif defined(__aarch64__)
474# define MAX_CODE_GEN_BUFFER_SIZE (128ul * 1024 * 1024)
Blue Swirl5b6dd862012-12-02 16:04:43 +0000475#elif defined(__arm__)
476# define MAX_CODE_GEN_BUFFER_SIZE (16u * 1024 * 1024)
477#elif defined(__s390x__)
478 /* We have a +- 4GB range on the branches; leave some slop. */
479# define MAX_CODE_GEN_BUFFER_SIZE (3ul * 1024 * 1024 * 1024)
Richard Henderson479eb122014-04-24 08:25:03 -0700480#elif defined(__mips__)
481 /* We have a 256MB branch region, but leave room to make sure the
482 main executable is also within that region. */
483# define MAX_CODE_GEN_BUFFER_SIZE (128ul * 1024 * 1024)
Blue Swirl5b6dd862012-12-02 16:04:43 +0000484#else
485# define MAX_CODE_GEN_BUFFER_SIZE ((size_t)-1)
486#endif
487
488#define DEFAULT_CODE_GEN_BUFFER_SIZE_1 (32u * 1024 * 1024)
489
490#define DEFAULT_CODE_GEN_BUFFER_SIZE \
491 (DEFAULT_CODE_GEN_BUFFER_SIZE_1 < MAX_CODE_GEN_BUFFER_SIZE \
492 ? DEFAULT_CODE_GEN_BUFFER_SIZE_1 : MAX_CODE_GEN_BUFFER_SIZE)
493
494static inline size_t size_code_gen_buffer(size_t tb_size)
495{
496 /* Size the buffer. */
497 if (tb_size == 0) {
498#ifdef USE_STATIC_CODE_GEN_BUFFER
499 tb_size = DEFAULT_CODE_GEN_BUFFER_SIZE;
500#else
501 /* ??? Needs adjustments. */
502 /* ??? If we relax the requirement that CONFIG_USER_ONLY use the
503 static buffer, we could size this on RESERVED_VA, on the text
504 segment size of the executable, or continue to use the default. */
505 tb_size = (unsigned long)(ram_size / 4);
506#endif
507 }
508 if (tb_size < MIN_CODE_GEN_BUFFER_SIZE) {
509 tb_size = MIN_CODE_GEN_BUFFER_SIZE;
510 }
511 if (tb_size > MAX_CODE_GEN_BUFFER_SIZE) {
512 tb_size = MAX_CODE_GEN_BUFFER_SIZE;
513 }
Evgeny Voevodin0b0d3322013-02-01 01:47:22 +0700514 tcg_ctx.code_gen_buffer_size = tb_size;
Blue Swirl5b6dd862012-12-02 16:04:43 +0000515 return tb_size;
516}
517
Richard Henderson483c76e2014-04-24 09:16:07 -0700518#ifdef __mips__
519/* In order to use J and JAL within the code_gen_buffer, we require
520 that the buffer not cross a 256MB boundary. */
521static inline bool cross_256mb(void *addr, size_t size)
522{
523 return ((uintptr_t)addr ^ ((uintptr_t)addr + size)) & 0xf0000000;
524}
525
526/* We weren't able to allocate a buffer without crossing that boundary,
527 so make do with the larger portion of the buffer that doesn't cross.
528 Returns the new base of the buffer, and adjusts code_gen_buffer_size. */
529static inline void *split_cross_256mb(void *buf1, size_t size1)
530{
531 void *buf2 = (void *)(((uintptr_t)buf1 + size1) & 0xf0000000);
532 size_t size2 = buf1 + size1 - buf2;
533
534 size1 = buf2 - buf1;
535 if (size1 < size2) {
536 size1 = size2;
537 buf1 = buf2;
538 }
539
540 tcg_ctx.code_gen_buffer_size = size1;
541 return buf1;
542}
543#endif
544
Blue Swirl5b6dd862012-12-02 16:04:43 +0000545#ifdef USE_STATIC_CODE_GEN_BUFFER
546static uint8_t static_code_gen_buffer[DEFAULT_CODE_GEN_BUFFER_SIZE]
547 __attribute__((aligned(CODE_GEN_ALIGN)));
548
549static inline void *alloc_code_gen_buffer(void)
550{
Richard Henderson483c76e2014-04-24 09:16:07 -0700551 void *buf = static_code_gen_buffer;
552#ifdef __mips__
553 if (cross_256mb(buf, tcg_ctx.code_gen_buffer_size)) {
554 buf = split_cross_256mb(buf, tcg_ctx.code_gen_buffer_size);
555 }
556#endif
557 map_exec(buf, tcg_ctx.code_gen_buffer_size);
558 return buf;
Blue Swirl5b6dd862012-12-02 16:04:43 +0000559}
560#elif defined(USE_MMAP)
561static inline void *alloc_code_gen_buffer(void)
562{
563 int flags = MAP_PRIVATE | MAP_ANONYMOUS;
564 uintptr_t start = 0;
565 void *buf;
566
567 /* Constrain the position of the buffer based on the host cpu.
568 Note that these addresses are chosen in concert with the
569 addresses assigned in the relevant linker script file. */
570# if defined(__PIE__) || defined(__PIC__)
571 /* Don't bother setting a preferred location if we're building
572 a position-independent executable. We're more likely to get
573 an address near the main executable if we let the kernel
574 choose the address. */
575# elif defined(__x86_64__) && defined(MAP_32BIT)
576 /* Force the memory down into low memory with the executable.
577 Leave the choice of exact location with the kernel. */
578 flags |= MAP_32BIT;
579 /* Cannot expect to map more than 800MB in low memory. */
Evgeny Voevodin0b0d3322013-02-01 01:47:22 +0700580 if (tcg_ctx.code_gen_buffer_size > 800u * 1024 * 1024) {
581 tcg_ctx.code_gen_buffer_size = 800u * 1024 * 1024;
Blue Swirl5b6dd862012-12-02 16:04:43 +0000582 }
583# elif defined(__sparc__)
584 start = 0x40000000ul;
585# elif defined(__s390x__)
586 start = 0x90000000ul;
Richard Henderson479eb122014-04-24 08:25:03 -0700587# elif defined(__mips__)
588 /* ??? We ought to more explicitly manage layout for softmmu too. */
589# ifdef CONFIG_USER_ONLY
590 start = 0x68000000ul;
591# elif _MIPS_SIM == _ABI64
592 start = 0x128000000ul;
593# else
594 start = 0x08000000ul;
595# endif
Blue Swirl5b6dd862012-12-02 16:04:43 +0000596# endif
597
Evgeny Voevodin0b0d3322013-02-01 01:47:22 +0700598 buf = mmap((void *)start, tcg_ctx.code_gen_buffer_size,
Blue Swirl5b6dd862012-12-02 16:04:43 +0000599 PROT_WRITE | PROT_READ | PROT_EXEC, flags, -1, 0);
Richard Henderson483c76e2014-04-24 09:16:07 -0700600 if (buf == MAP_FAILED) {
601 return NULL;
602 }
603
604#ifdef __mips__
605 if (cross_256mb(buf, tcg_ctx.code_gen_buffer_size)) {
Stefan Weil5d831be2014-06-13 20:42:57 +0200606 /* Try again, with the original still mapped, to avoid re-acquiring
Richard Henderson483c76e2014-04-24 09:16:07 -0700607 that 256mb crossing. This time don't specify an address. */
608 size_t size2, size1 = tcg_ctx.code_gen_buffer_size;
609 void *buf2 = mmap(NULL, size1, PROT_WRITE | PROT_READ | PROT_EXEC,
610 flags, -1, 0);
611 if (buf2 != MAP_FAILED) {
612 if (!cross_256mb(buf2, size1)) {
613 /* Success! Use the new buffer. */
614 munmap(buf, size1);
615 return buf2;
616 }
617 /* Failure. Work with what we had. */
618 munmap(buf2, size1);
619 }
620
621 /* Split the original buffer. Free the smaller half. */
622 buf2 = split_cross_256mb(buf, size1);
623 size2 = tcg_ctx.code_gen_buffer_size;
624 munmap(buf + (buf == buf2 ? size2 : 0), size1 - size2);
625 return buf2;
626 }
627#endif
628
629 return buf;
Blue Swirl5b6dd862012-12-02 16:04:43 +0000630}
631#else
632static inline void *alloc_code_gen_buffer(void)
633{
Markus Armbruster8b98ade2015-02-04 11:26:07 +0100634 void *buf = g_try_malloc(tcg_ctx.code_gen_buffer_size);
Blue Swirl5b6dd862012-12-02 16:04:43 +0000635
Richard Henderson483c76e2014-04-24 09:16:07 -0700636 if (buf == NULL) {
637 return NULL;
Blue Swirl5b6dd862012-12-02 16:04:43 +0000638 }
Richard Henderson483c76e2014-04-24 09:16:07 -0700639
640#ifdef __mips__
641 if (cross_256mb(buf, tcg_ctx.code_gen_buffer_size)) {
642 void *buf2 = g_malloc(tcg_ctx.code_gen_buffer_size);
643 if (buf2 != NULL && !cross_256mb(buf2, size1)) {
644 /* Success! Use the new buffer. */
645 free(buf);
646 buf = buf2;
647 } else {
648 /* Failure. Work with what we had. Since this is malloc
649 and not mmap, we can't free the other half. */
650 free(buf2);
651 buf = split_cross_256mb(buf, tcg_ctx.code_gen_buffer_size);
652 }
653 }
654#endif
655
656 map_exec(buf, tcg_ctx.code_gen_buffer_size);
Blue Swirl5b6dd862012-12-02 16:04:43 +0000657 return buf;
658}
659#endif /* USE_STATIC_CODE_GEN_BUFFER, USE_MMAP */
660
661static inline void code_gen_alloc(size_t tb_size)
662{
Evgeny Voevodin0b0d3322013-02-01 01:47:22 +0700663 tcg_ctx.code_gen_buffer_size = size_code_gen_buffer(tb_size);
664 tcg_ctx.code_gen_buffer = alloc_code_gen_buffer();
665 if (tcg_ctx.code_gen_buffer == NULL) {
Blue Swirl5b6dd862012-12-02 16:04:43 +0000666 fprintf(stderr, "Could not allocate dynamic translator buffer\n");
667 exit(1);
668 }
669
Evgeny Voevodin0b0d3322013-02-01 01:47:22 +0700670 qemu_madvise(tcg_ctx.code_gen_buffer, tcg_ctx.code_gen_buffer_size,
671 QEMU_MADV_HUGEPAGE);
Blue Swirl5b6dd862012-12-02 16:04:43 +0000672
673 /* Steal room for the prologue at the end of the buffer. This ensures
674 (via the MAX_CODE_GEN_BUFFER_SIZE limits above) that direct branches
675 from TB's to the prologue are going to be in range. It also means
676 that we don't need to mark (additional) portions of the data segment
677 as executable. */
Evgeny Voevodin0b0d3322013-02-01 01:47:22 +0700678 tcg_ctx.code_gen_prologue = tcg_ctx.code_gen_buffer +
679 tcg_ctx.code_gen_buffer_size - 1024;
680 tcg_ctx.code_gen_buffer_size -= 1024;
Blue Swirl5b6dd862012-12-02 16:04:43 +0000681
Evgeny Voevodin0b0d3322013-02-01 01:47:22 +0700682 tcg_ctx.code_gen_buffer_max_size = tcg_ctx.code_gen_buffer_size -
Blue Swirl5b6dd862012-12-02 16:04:43 +0000683 (TCG_MAX_OP_SIZE * OPC_BUF_SIZE);
Evgeny Voevodin0b0d3322013-02-01 01:47:22 +0700684 tcg_ctx.code_gen_max_blocks = tcg_ctx.code_gen_buffer_size /
685 CODE_GEN_AVG_BLOCK_SIZE;
Evgeny Voevodin5e5f07e2013-02-01 01:47:23 +0700686 tcg_ctx.tb_ctx.tbs =
687 g_malloc(tcg_ctx.code_gen_max_blocks * sizeof(TranslationBlock));
Blue Swirl5b6dd862012-12-02 16:04:43 +0000688}
689
690/* Must be called before using the QEMU cpus. 'tb_size' is the size
691 (in bytes) allocated to the translation buffer. Zero means default
692 size. */
693void tcg_exec_init(unsigned long tb_size)
694{
695 cpu_gen_init();
696 code_gen_alloc(tb_size);
Evgeny Voevodin0b0d3322013-02-01 01:47:22 +0700697 tcg_ctx.code_gen_ptr = tcg_ctx.code_gen_buffer;
698 tcg_register_jit(tcg_ctx.code_gen_buffer, tcg_ctx.code_gen_buffer_size);
Blue Swirl5b6dd862012-12-02 16:04:43 +0000699 page_init();
700#if !defined(CONFIG_USER_ONLY) || !defined(CONFIG_USE_GUEST_BASE)
701 /* There's no guest base to take into account, so go ahead and
702 initialize the prologue now. */
703 tcg_prologue_init(&tcg_ctx);
704#endif
705}
706
707bool tcg_enabled(void)
708{
Evgeny Voevodin0b0d3322013-02-01 01:47:22 +0700709 return tcg_ctx.code_gen_buffer != NULL;
Blue Swirl5b6dd862012-12-02 16:04:43 +0000710}
711
712/* Allocate a new translation block. Flush the translation buffer if
713 too many translation blocks or too much generated code. */
714static TranslationBlock *tb_alloc(target_ulong pc)
715{
716 TranslationBlock *tb;
717
Evgeny Voevodin5e5f07e2013-02-01 01:47:23 +0700718 if (tcg_ctx.tb_ctx.nb_tbs >= tcg_ctx.code_gen_max_blocks ||
Evgeny Voevodin0b0d3322013-02-01 01:47:22 +0700719 (tcg_ctx.code_gen_ptr - tcg_ctx.code_gen_buffer) >=
720 tcg_ctx.code_gen_buffer_max_size) {
Blue Swirl5b6dd862012-12-02 16:04:43 +0000721 return NULL;
722 }
Evgeny Voevodin5e5f07e2013-02-01 01:47:23 +0700723 tb = &tcg_ctx.tb_ctx.tbs[tcg_ctx.tb_ctx.nb_tbs++];
Blue Swirl5b6dd862012-12-02 16:04:43 +0000724 tb->pc = pc;
725 tb->cflags = 0;
726 return tb;
727}
728
729void tb_free(TranslationBlock *tb)
730{
731 /* In practice this is mostly used for single use temporary TB
732 Ignore the hard cases and just back up if this TB happens to
733 be the last one generated. */
Evgeny Voevodin5e5f07e2013-02-01 01:47:23 +0700734 if (tcg_ctx.tb_ctx.nb_tbs > 0 &&
735 tb == &tcg_ctx.tb_ctx.tbs[tcg_ctx.tb_ctx.nb_tbs - 1]) {
Evgeny Voevodin0b0d3322013-02-01 01:47:22 +0700736 tcg_ctx.code_gen_ptr = tb->tc_ptr;
Evgeny Voevodin5e5f07e2013-02-01 01:47:23 +0700737 tcg_ctx.tb_ctx.nb_tbs--;
Blue Swirl5b6dd862012-12-02 16:04:43 +0000738 }
739}
740
741static inline void invalidate_page_bitmap(PageDesc *p)
742{
743 if (p->code_bitmap) {
744 g_free(p->code_bitmap);
745 p->code_bitmap = NULL;
746 }
747 p->code_write_count = 0;
748}
749
750/* Set to NULL all the 'first_tb' fields in all PageDescs. */
751static void page_flush_tb_1(int level, void **lp)
752{
753 int i;
754
755 if (*lp == NULL) {
756 return;
757 }
758 if (level == 0) {
759 PageDesc *pd = *lp;
760
Paolo Bonzini03f49952013-11-07 17:14:36 +0100761 for (i = 0; i < V_L2_SIZE; ++i) {
Blue Swirl5b6dd862012-12-02 16:04:43 +0000762 pd[i].first_tb = NULL;
763 invalidate_page_bitmap(pd + i);
764 }
765 } else {
766 void **pp = *lp;
767
Paolo Bonzini03f49952013-11-07 17:14:36 +0100768 for (i = 0; i < V_L2_SIZE; ++i) {
Blue Swirl5b6dd862012-12-02 16:04:43 +0000769 page_flush_tb_1(level - 1, pp + i);
770 }
771 }
772}
773
774static void page_flush_tb(void)
775{
776 int i;
777
778 for (i = 0; i < V_L1_SIZE; i++) {
Paolo Bonzini03f49952013-11-07 17:14:36 +0100779 page_flush_tb_1(V_L1_SHIFT / V_L2_BITS - 1, l1_map + i);
Blue Swirl5b6dd862012-12-02 16:04:43 +0000780 }
781}
782
783/* flush all the translation blocks */
784/* XXX: tb_flush is currently not thread safe */
785void tb_flush(CPUArchState *env1)
786{
Andreas Färbera47dddd2013-09-03 17:38:47 +0200787 CPUState *cpu = ENV_GET_CPU(env1);
Blue Swirl5b6dd862012-12-02 16:04:43 +0000788
789#if defined(DEBUG_FLUSH)
790 printf("qemu: flush code_size=%ld nb_tbs=%d avg_tb_size=%ld\n",
Evgeny Voevodin0b0d3322013-02-01 01:47:22 +0700791 (unsigned long)(tcg_ctx.code_gen_ptr - tcg_ctx.code_gen_buffer),
Evgeny Voevodin5e5f07e2013-02-01 01:47:23 +0700792 tcg_ctx.tb_ctx.nb_tbs, tcg_ctx.tb_ctx.nb_tbs > 0 ?
Evgeny Voevodin0b0d3322013-02-01 01:47:22 +0700793 ((unsigned long)(tcg_ctx.code_gen_ptr - tcg_ctx.code_gen_buffer)) /
Evgeny Voevodin5e5f07e2013-02-01 01:47:23 +0700794 tcg_ctx.tb_ctx.nb_tbs : 0);
Blue Swirl5b6dd862012-12-02 16:04:43 +0000795#endif
Evgeny Voevodin0b0d3322013-02-01 01:47:22 +0700796 if ((unsigned long)(tcg_ctx.code_gen_ptr - tcg_ctx.code_gen_buffer)
797 > tcg_ctx.code_gen_buffer_size) {
Andreas Färbera47dddd2013-09-03 17:38:47 +0200798 cpu_abort(cpu, "Internal error: code buffer overflow\n");
Blue Swirl5b6dd862012-12-02 16:04:43 +0000799 }
Evgeny Voevodin5e5f07e2013-02-01 01:47:23 +0700800 tcg_ctx.tb_ctx.nb_tbs = 0;
Blue Swirl5b6dd862012-12-02 16:04:43 +0000801
Andreas Färberbdc44642013-06-24 23:50:24 +0200802 CPU_FOREACH(cpu) {
Andreas Färber8cd70432013-08-26 06:03:38 +0200803 memset(cpu->tb_jmp_cache, 0, sizeof(cpu->tb_jmp_cache));
Blue Swirl5b6dd862012-12-02 16:04:43 +0000804 }
805
Richard Hendersoneb2535f2013-12-07 10:44:52 +1300806 memset(tcg_ctx.tb_ctx.tb_phys_hash, 0, sizeof(tcg_ctx.tb_ctx.tb_phys_hash));
Blue Swirl5b6dd862012-12-02 16:04:43 +0000807 page_flush_tb();
808
Evgeny Voevodin0b0d3322013-02-01 01:47:22 +0700809 tcg_ctx.code_gen_ptr = tcg_ctx.code_gen_buffer;
Blue Swirl5b6dd862012-12-02 16:04:43 +0000810 /* XXX: flush processor icache at this point if cache flush is
811 expensive */
Evgeny Voevodin5e5f07e2013-02-01 01:47:23 +0700812 tcg_ctx.tb_ctx.tb_flush_count++;
Blue Swirl5b6dd862012-12-02 16:04:43 +0000813}
814
815#ifdef DEBUG_TB_CHECK
816
817static void tb_invalidate_check(target_ulong address)
818{
819 TranslationBlock *tb;
820 int i;
821
822 address &= TARGET_PAGE_MASK;
823 for (i = 0; i < CODE_GEN_PHYS_HASH_SIZE; i++) {
Evgeny Voevodin5e5f07e2013-02-01 01:47:23 +0700824 for (tb = tb_ctx.tb_phys_hash[i]; tb != NULL; tb = tb->phys_hash_next) {
Blue Swirl5b6dd862012-12-02 16:04:43 +0000825 if (!(address + TARGET_PAGE_SIZE <= tb->pc ||
826 address >= tb->pc + tb->size)) {
827 printf("ERROR invalidate: address=" TARGET_FMT_lx
828 " PC=%08lx size=%04x\n",
829 address, (long)tb->pc, tb->size);
830 }
831 }
832 }
833}
834
835/* verify that all the pages have correct rights for code */
836static void tb_page_check(void)
837{
838 TranslationBlock *tb;
839 int i, flags1, flags2;
840
841 for (i = 0; i < CODE_GEN_PHYS_HASH_SIZE; i++) {
Evgeny Voevodin5e5f07e2013-02-01 01:47:23 +0700842 for (tb = tcg_ctx.tb_ctx.tb_phys_hash[i]; tb != NULL;
843 tb = tb->phys_hash_next) {
Blue Swirl5b6dd862012-12-02 16:04:43 +0000844 flags1 = page_get_flags(tb->pc);
845 flags2 = page_get_flags(tb->pc + tb->size - 1);
846 if ((flags1 & PAGE_WRITE) || (flags2 & PAGE_WRITE)) {
847 printf("ERROR page flags: PC=%08lx size=%04x f1=%x f2=%x\n",
848 (long)tb->pc, tb->size, flags1, flags2);
849 }
850 }
851 }
852}
853
854#endif
855
陳韋任 (Wei-Ren Chen)0c884d12012-12-20 09:39:16 +0800856static inline void tb_hash_remove(TranslationBlock **ptb, TranslationBlock *tb)
Blue Swirl5b6dd862012-12-02 16:04:43 +0000857{
858 TranslationBlock *tb1;
859
860 for (;;) {
861 tb1 = *ptb;
862 if (tb1 == tb) {
陳韋任 (Wei-Ren Chen)0c884d12012-12-20 09:39:16 +0800863 *ptb = tb1->phys_hash_next;
Blue Swirl5b6dd862012-12-02 16:04:43 +0000864 break;
865 }
陳韋任 (Wei-Ren Chen)0c884d12012-12-20 09:39:16 +0800866 ptb = &tb1->phys_hash_next;
Blue Swirl5b6dd862012-12-02 16:04:43 +0000867 }
868}
869
870static inline void tb_page_remove(TranslationBlock **ptb, TranslationBlock *tb)
871{
872 TranslationBlock *tb1;
873 unsigned int n1;
874
875 for (;;) {
876 tb1 = *ptb;
877 n1 = (uintptr_t)tb1 & 3;
878 tb1 = (TranslationBlock *)((uintptr_t)tb1 & ~3);
879 if (tb1 == tb) {
880 *ptb = tb1->page_next[n1];
881 break;
882 }
883 ptb = &tb1->page_next[n1];
884 }
885}
886
887static inline void tb_jmp_remove(TranslationBlock *tb, int n)
888{
889 TranslationBlock *tb1, **ptb;
890 unsigned int n1;
891
892 ptb = &tb->jmp_next[n];
893 tb1 = *ptb;
894 if (tb1) {
895 /* find tb(n) in circular list */
896 for (;;) {
897 tb1 = *ptb;
898 n1 = (uintptr_t)tb1 & 3;
899 tb1 = (TranslationBlock *)((uintptr_t)tb1 & ~3);
900 if (n1 == n && tb1 == tb) {
901 break;
902 }
903 if (n1 == 2) {
904 ptb = &tb1->jmp_first;
905 } else {
906 ptb = &tb1->jmp_next[n1];
907 }
908 }
909 /* now we can suppress tb(n) from the list */
910 *ptb = tb->jmp_next[n];
911
912 tb->jmp_next[n] = NULL;
913 }
914}
915
916/* reset the jump entry 'n' of a TB so that it is not chained to
917 another TB */
918static inline void tb_reset_jump(TranslationBlock *tb, int n)
919{
920 tb_set_jmp_target(tb, n, (uintptr_t)(tb->tc_ptr + tb->tb_next_offset[n]));
921}
922
陳韋任 (Wei-Ren Chen)0c884d12012-12-20 09:39:16 +0800923/* invalidate one TB */
Blue Swirl5b6dd862012-12-02 16:04:43 +0000924void tb_phys_invalidate(TranslationBlock *tb, tb_page_addr_t page_addr)
925{
Andreas Färber182735e2013-05-29 22:29:20 +0200926 CPUState *cpu;
Blue Swirl5b6dd862012-12-02 16:04:43 +0000927 PageDesc *p;
928 unsigned int h, n1;
929 tb_page_addr_t phys_pc;
930 TranslationBlock *tb1, *tb2;
931
932 /* remove the TB from the hash list */
933 phys_pc = tb->page_addr[0] + (tb->pc & ~TARGET_PAGE_MASK);
934 h = tb_phys_hash_func(phys_pc);
Evgeny Voevodin5e5f07e2013-02-01 01:47:23 +0700935 tb_hash_remove(&tcg_ctx.tb_ctx.tb_phys_hash[h], tb);
Blue Swirl5b6dd862012-12-02 16:04:43 +0000936
937 /* remove the TB from the page list */
938 if (tb->page_addr[0] != page_addr) {
939 p = page_find(tb->page_addr[0] >> TARGET_PAGE_BITS);
940 tb_page_remove(&p->first_tb, tb);
941 invalidate_page_bitmap(p);
942 }
943 if (tb->page_addr[1] != -1 && tb->page_addr[1] != page_addr) {
944 p = page_find(tb->page_addr[1] >> TARGET_PAGE_BITS);
945 tb_page_remove(&p->first_tb, tb);
946 invalidate_page_bitmap(p);
947 }
948
Evgeny Voevodin5e5f07e2013-02-01 01:47:23 +0700949 tcg_ctx.tb_ctx.tb_invalidated_flag = 1;
Blue Swirl5b6dd862012-12-02 16:04:43 +0000950
951 /* remove the TB from the hash list */
952 h = tb_jmp_cache_hash_func(tb->pc);
Andreas Färberbdc44642013-06-24 23:50:24 +0200953 CPU_FOREACH(cpu) {
Andreas Färber8cd70432013-08-26 06:03:38 +0200954 if (cpu->tb_jmp_cache[h] == tb) {
955 cpu->tb_jmp_cache[h] = NULL;
Blue Swirl5b6dd862012-12-02 16:04:43 +0000956 }
957 }
958
959 /* suppress this TB from the two jump lists */
960 tb_jmp_remove(tb, 0);
961 tb_jmp_remove(tb, 1);
962
963 /* suppress any remaining jumps to this TB */
964 tb1 = tb->jmp_first;
965 for (;;) {
966 n1 = (uintptr_t)tb1 & 3;
967 if (n1 == 2) {
968 break;
969 }
970 tb1 = (TranslationBlock *)((uintptr_t)tb1 & ~3);
971 tb2 = tb1->jmp_next[n1];
972 tb_reset_jump(tb1, n1);
973 tb1->jmp_next[n1] = NULL;
974 tb1 = tb2;
975 }
976 tb->jmp_first = (TranslationBlock *)((uintptr_t)tb | 2); /* fail safe */
977
Evgeny Voevodin5e5f07e2013-02-01 01:47:23 +0700978 tcg_ctx.tb_ctx.tb_phys_invalidate_count++;
Blue Swirl5b6dd862012-12-02 16:04:43 +0000979}
980
981static inline void set_bits(uint8_t *tab, int start, int len)
982{
983 int end, mask, end1;
984
985 end = start + len;
986 tab += start >> 3;
987 mask = 0xff << (start & 7);
988 if ((start & ~7) == (end & ~7)) {
989 if (start < end) {
990 mask &= ~(0xff << (end & 7));
991 *tab |= mask;
992 }
993 } else {
994 *tab++ |= mask;
995 start = (start + 8) & ~7;
996 end1 = end & ~7;
997 while (start < end1) {
998 *tab++ = 0xff;
999 start += 8;
1000 }
1001 if (start < end) {
1002 mask = ~(0xff << (end & 7));
1003 *tab |= mask;
1004 }
1005 }
1006}
1007
1008static void build_page_bitmap(PageDesc *p)
1009{
1010 int n, tb_start, tb_end;
1011 TranslationBlock *tb;
1012
1013 p->code_bitmap = g_malloc0(TARGET_PAGE_SIZE / 8);
1014
1015 tb = p->first_tb;
1016 while (tb != NULL) {
1017 n = (uintptr_t)tb & 3;
1018 tb = (TranslationBlock *)((uintptr_t)tb & ~3);
1019 /* NOTE: this is subtle as a TB may span two physical pages */
1020 if (n == 0) {
1021 /* NOTE: tb_end may be after the end of the page, but
1022 it is not a problem */
1023 tb_start = tb->pc & ~TARGET_PAGE_MASK;
1024 tb_end = tb_start + tb->size;
1025 if (tb_end > TARGET_PAGE_SIZE) {
1026 tb_end = TARGET_PAGE_SIZE;
1027 }
1028 } else {
1029 tb_start = 0;
1030 tb_end = ((tb->pc + tb->size) & ~TARGET_PAGE_MASK);
1031 }
1032 set_bits(p->code_bitmap, tb_start, tb_end - tb_start);
1033 tb = tb->page_next[n];
1034 }
1035}
1036
Andreas Färber648f0342013-09-01 17:43:17 +02001037TranslationBlock *tb_gen_code(CPUState *cpu,
Blue Swirl5b6dd862012-12-02 16:04:43 +00001038 target_ulong pc, target_ulong cs_base,
1039 int flags, int cflags)
1040{
Andreas Färber648f0342013-09-01 17:43:17 +02001041 CPUArchState *env = cpu->env_ptr;
Blue Swirl5b6dd862012-12-02 16:04:43 +00001042 TranslationBlock *tb;
Blue Swirl5b6dd862012-12-02 16:04:43 +00001043 tb_page_addr_t phys_pc, phys_page2;
1044 target_ulong virt_page2;
1045 int code_gen_size;
1046
1047 phys_pc = get_page_addr_code(env, pc);
Paolo Bonzini02663592014-11-26 13:39:53 +03001048 if (use_icount) {
1049 cflags |= CF_USE_ICOUNT;
1050 }
Blue Swirl5b6dd862012-12-02 16:04:43 +00001051 tb = tb_alloc(pc);
1052 if (!tb) {
1053 /* flush must be done */
1054 tb_flush(env);
1055 /* cannot fail at this point */
1056 tb = tb_alloc(pc);
1057 /* Don't forget to invalidate previous TB info. */
Evgeny Voevodin5e5f07e2013-02-01 01:47:23 +07001058 tcg_ctx.tb_ctx.tb_invalidated_flag = 1;
Blue Swirl5b6dd862012-12-02 16:04:43 +00001059 }
Richard Henderson1813e172014-03-28 12:56:22 -07001060 tb->tc_ptr = tcg_ctx.code_gen_ptr;
Blue Swirl5b6dd862012-12-02 16:04:43 +00001061 tb->cs_base = cs_base;
1062 tb->flags = flags;
1063 tb->cflags = cflags;
1064 cpu_gen_code(env, tb, &code_gen_size);
Evgeny Voevodin0b0d3322013-02-01 01:47:22 +07001065 tcg_ctx.code_gen_ptr = (void *)(((uintptr_t)tcg_ctx.code_gen_ptr +
1066 code_gen_size + CODE_GEN_ALIGN - 1) & ~(CODE_GEN_ALIGN - 1));
Blue Swirl5b6dd862012-12-02 16:04:43 +00001067
1068 /* check next page if needed */
1069 virt_page2 = (pc + tb->size - 1) & TARGET_PAGE_MASK;
1070 phys_page2 = -1;
1071 if ((pc & TARGET_PAGE_MASK) != virt_page2) {
1072 phys_page2 = get_page_addr_code(env, virt_page2);
1073 }
1074 tb_link_page(tb, phys_pc, phys_page2);
1075 return tb;
1076}
1077
1078/*
1079 * Invalidate all TBs which intersect with the target physical address range
1080 * [start;end[. NOTE: start and end may refer to *different* physical pages.
1081 * 'is_cpu_write_access' should be true if called from a real cpu write
1082 * access: the virtual CPU will exit the current TB if code is modified inside
1083 * this TB.
1084 */
1085void tb_invalidate_phys_range(tb_page_addr_t start, tb_page_addr_t end,
1086 int is_cpu_write_access)
1087{
1088 while (start < end) {
1089 tb_invalidate_phys_page_range(start, end, is_cpu_write_access);
1090 start &= TARGET_PAGE_MASK;
1091 start += TARGET_PAGE_SIZE;
1092 }
1093}
1094
1095/*
1096 * Invalidate all TBs which intersect with the target physical address range
1097 * [start;end[. NOTE: start and end must refer to the *same* physical page.
1098 * 'is_cpu_write_access' should be true if called from a real cpu write
1099 * access: the virtual CPU will exit the current TB if code is modified inside
1100 * this TB.
1101 */
1102void tb_invalidate_phys_page_range(tb_page_addr_t start, tb_page_addr_t end,
1103 int is_cpu_write_access)
1104{
1105 TranslationBlock *tb, *tb_next, *saved_tb;
Andreas Färber4917cf42013-05-27 05:17:50 +02001106 CPUState *cpu = current_cpu;
Andreas Färberbaea4fa2013-09-03 10:51:26 +02001107#if defined(TARGET_HAS_PRECISE_SMC)
Andreas Färber4917cf42013-05-27 05:17:50 +02001108 CPUArchState *env = NULL;
1109#endif
Blue Swirl5b6dd862012-12-02 16:04:43 +00001110 tb_page_addr_t tb_start, tb_end;
1111 PageDesc *p;
1112 int n;
1113#ifdef TARGET_HAS_PRECISE_SMC
1114 int current_tb_not_found = is_cpu_write_access;
1115 TranslationBlock *current_tb = NULL;
1116 int current_tb_modified = 0;
1117 target_ulong current_pc = 0;
1118 target_ulong current_cs_base = 0;
1119 int current_flags = 0;
1120#endif /* TARGET_HAS_PRECISE_SMC */
1121
1122 p = page_find(start >> TARGET_PAGE_BITS);
1123 if (!p) {
1124 return;
1125 }
1126 if (!p->code_bitmap &&
1127 ++p->code_write_count >= SMC_BITMAP_USE_THRESHOLD &&
1128 is_cpu_write_access) {
1129 /* build code bitmap */
1130 build_page_bitmap(p);
1131 }
Andreas Färberbaea4fa2013-09-03 10:51:26 +02001132#if defined(TARGET_HAS_PRECISE_SMC)
Andreas Färber4917cf42013-05-27 05:17:50 +02001133 if (cpu != NULL) {
1134 env = cpu->env_ptr;
Andreas Färberd77953b2013-01-16 19:29:31 +01001135 }
Andreas Färber4917cf42013-05-27 05:17:50 +02001136#endif
Blue Swirl5b6dd862012-12-02 16:04:43 +00001137
1138 /* we remove all the TBs in the range [start, end[ */
1139 /* XXX: see if in some cases it could be faster to invalidate all
1140 the code */
1141 tb = p->first_tb;
1142 while (tb != NULL) {
1143 n = (uintptr_t)tb & 3;
1144 tb = (TranslationBlock *)((uintptr_t)tb & ~3);
1145 tb_next = tb->page_next[n];
1146 /* NOTE: this is subtle as a TB may span two physical pages */
1147 if (n == 0) {
1148 /* NOTE: tb_end may be after the end of the page, but
1149 it is not a problem */
1150 tb_start = tb->page_addr[0] + (tb->pc & ~TARGET_PAGE_MASK);
1151 tb_end = tb_start + tb->size;
1152 } else {
1153 tb_start = tb->page_addr[1];
1154 tb_end = tb_start + ((tb->pc + tb->size) & ~TARGET_PAGE_MASK);
1155 }
1156 if (!(tb_end <= start || tb_start >= end)) {
1157#ifdef TARGET_HAS_PRECISE_SMC
1158 if (current_tb_not_found) {
1159 current_tb_not_found = 0;
1160 current_tb = NULL;
Andreas Färber93afead2013-08-26 03:41:01 +02001161 if (cpu->mem_io_pc) {
Blue Swirl5b6dd862012-12-02 16:04:43 +00001162 /* now we have a real cpu fault */
Andreas Färber93afead2013-08-26 03:41:01 +02001163 current_tb = tb_find_pc(cpu->mem_io_pc);
Blue Swirl5b6dd862012-12-02 16:04:43 +00001164 }
1165 }
1166 if (current_tb == tb &&
1167 (current_tb->cflags & CF_COUNT_MASK) != 1) {
1168 /* If we are modifying the current TB, we must stop
1169 its execution. We could be more precise by checking
1170 that the modification is after the current PC, but it
1171 would require a specialized function to partially
1172 restore the CPU state */
1173
1174 current_tb_modified = 1;
Andreas Färber74f10512013-09-01 17:02:58 +02001175 cpu_restore_state_from_tb(cpu, current_tb, cpu->mem_io_pc);
Blue Swirl5b6dd862012-12-02 16:04:43 +00001176 cpu_get_tb_cpu_state(env, &current_pc, &current_cs_base,
1177 &current_flags);
1178 }
1179#endif /* TARGET_HAS_PRECISE_SMC */
1180 /* we need to do that to handle the case where a signal
1181 occurs while doing tb_phys_invalidate() */
1182 saved_tb = NULL;
Andreas Färberd77953b2013-01-16 19:29:31 +01001183 if (cpu != NULL) {
1184 saved_tb = cpu->current_tb;
1185 cpu->current_tb = NULL;
Blue Swirl5b6dd862012-12-02 16:04:43 +00001186 }
1187 tb_phys_invalidate(tb, -1);
Andreas Färberd77953b2013-01-16 19:29:31 +01001188 if (cpu != NULL) {
1189 cpu->current_tb = saved_tb;
Andreas Färberc3affe52013-01-18 15:03:43 +01001190 if (cpu->interrupt_request && cpu->current_tb) {
1191 cpu_interrupt(cpu, cpu->interrupt_request);
Blue Swirl5b6dd862012-12-02 16:04:43 +00001192 }
1193 }
1194 }
1195 tb = tb_next;
1196 }
1197#if !defined(CONFIG_USER_ONLY)
1198 /* if no code remaining, no need to continue to use slow writes */
1199 if (!p->first_tb) {
1200 invalidate_page_bitmap(p);
1201 if (is_cpu_write_access) {
Andreas Färberbaea4fa2013-09-03 10:51:26 +02001202 tlb_unprotect_code_phys(cpu, start, cpu->mem_io_vaddr);
Blue Swirl5b6dd862012-12-02 16:04:43 +00001203 }
1204 }
1205#endif
1206#ifdef TARGET_HAS_PRECISE_SMC
1207 if (current_tb_modified) {
1208 /* we generate a block containing just the instruction
1209 modifying the memory. It will ensure that it cannot modify
1210 itself */
Andreas Färberd77953b2013-01-16 19:29:31 +01001211 cpu->current_tb = NULL;
Andreas Färber648f0342013-09-01 17:43:17 +02001212 tb_gen_code(cpu, current_pc, current_cs_base, current_flags, 1);
Andreas Färber0ea8cb82013-09-03 02:12:23 +02001213 cpu_resume_from_signal(cpu, NULL);
Blue Swirl5b6dd862012-12-02 16:04:43 +00001214 }
1215#endif
1216}
1217
1218/* len must be <= 8 and start must be a multiple of len */
1219void tb_invalidate_phys_page_fast(tb_page_addr_t start, int len)
1220{
1221 PageDesc *p;
1222 int offset, b;
1223
1224#if 0
1225 if (1) {
1226 qemu_log("modifying code at 0x%x size=%d EIP=%x PC=%08x\n",
1227 cpu_single_env->mem_io_vaddr, len,
1228 cpu_single_env->eip,
1229 cpu_single_env->eip +
1230 (intptr_t)cpu_single_env->segs[R_CS].base);
1231 }
1232#endif
1233 p = page_find(start >> TARGET_PAGE_BITS);
1234 if (!p) {
1235 return;
1236 }
1237 if (p->code_bitmap) {
1238 offset = start & ~TARGET_PAGE_MASK;
1239 b = p->code_bitmap[offset >> 3] >> (offset & 7);
1240 if (b & ((1 << len) - 1)) {
1241 goto do_invalidate;
1242 }
1243 } else {
1244 do_invalidate:
1245 tb_invalidate_phys_page_range(start, start + len, 1);
1246 }
1247}
1248
1249#if !defined(CONFIG_SOFTMMU)
1250static void tb_invalidate_phys_page(tb_page_addr_t addr,
Alexander Grafd02532f2013-07-06 14:17:57 +02001251 uintptr_t pc, void *puc,
1252 bool locked)
Blue Swirl5b6dd862012-12-02 16:04:43 +00001253{
1254 TranslationBlock *tb;
1255 PageDesc *p;
1256 int n;
1257#ifdef TARGET_HAS_PRECISE_SMC
1258 TranslationBlock *current_tb = NULL;
Andreas Färber4917cf42013-05-27 05:17:50 +02001259 CPUState *cpu = current_cpu;
1260 CPUArchState *env = NULL;
Blue Swirl5b6dd862012-12-02 16:04:43 +00001261 int current_tb_modified = 0;
1262 target_ulong current_pc = 0;
1263 target_ulong current_cs_base = 0;
1264 int current_flags = 0;
1265#endif
1266
1267 addr &= TARGET_PAGE_MASK;
1268 p = page_find(addr >> TARGET_PAGE_BITS);
1269 if (!p) {
1270 return;
1271 }
1272 tb = p->first_tb;
1273#ifdef TARGET_HAS_PRECISE_SMC
1274 if (tb && pc != 0) {
1275 current_tb = tb_find_pc(pc);
1276 }
Andreas Färber4917cf42013-05-27 05:17:50 +02001277 if (cpu != NULL) {
1278 env = cpu->env_ptr;
Andreas Färberd77953b2013-01-16 19:29:31 +01001279 }
Blue Swirl5b6dd862012-12-02 16:04:43 +00001280#endif
1281 while (tb != NULL) {
1282 n = (uintptr_t)tb & 3;
1283 tb = (TranslationBlock *)((uintptr_t)tb & ~3);
1284#ifdef TARGET_HAS_PRECISE_SMC
1285 if (current_tb == tb &&
1286 (current_tb->cflags & CF_COUNT_MASK) != 1) {
1287 /* If we are modifying the current TB, we must stop
1288 its execution. We could be more precise by checking
1289 that the modification is after the current PC, but it
1290 would require a specialized function to partially
1291 restore the CPU state */
1292
1293 current_tb_modified = 1;
Andreas Färber74f10512013-09-01 17:02:58 +02001294 cpu_restore_state_from_tb(cpu, current_tb, pc);
Blue Swirl5b6dd862012-12-02 16:04:43 +00001295 cpu_get_tb_cpu_state(env, &current_pc, &current_cs_base,
1296 &current_flags);
1297 }
1298#endif /* TARGET_HAS_PRECISE_SMC */
1299 tb_phys_invalidate(tb, addr);
1300 tb = tb->page_next[n];
1301 }
1302 p->first_tb = NULL;
1303#ifdef TARGET_HAS_PRECISE_SMC
1304 if (current_tb_modified) {
1305 /* we generate a block containing just the instruction
1306 modifying the memory. It will ensure that it cannot modify
1307 itself */
Andreas Färberd77953b2013-01-16 19:29:31 +01001308 cpu->current_tb = NULL;
Andreas Färber648f0342013-09-01 17:43:17 +02001309 tb_gen_code(cpu, current_pc, current_cs_base, current_flags, 1);
Alexander Grafd02532f2013-07-06 14:17:57 +02001310 if (locked) {
1311 mmap_unlock();
1312 }
Andreas Färber0ea8cb82013-09-03 02:12:23 +02001313 cpu_resume_from_signal(cpu, puc);
Blue Swirl5b6dd862012-12-02 16:04:43 +00001314 }
1315#endif
1316}
1317#endif
1318
1319/* add the tb in the target page and protect it if necessary */
1320static inline void tb_alloc_page(TranslationBlock *tb,
1321 unsigned int n, tb_page_addr_t page_addr)
1322{
1323 PageDesc *p;
1324#ifndef CONFIG_USER_ONLY
1325 bool page_already_protected;
1326#endif
1327
1328 tb->page_addr[n] = page_addr;
1329 p = page_find_alloc(page_addr >> TARGET_PAGE_BITS, 1);
1330 tb->page_next[n] = p->first_tb;
1331#ifndef CONFIG_USER_ONLY
1332 page_already_protected = p->first_tb != NULL;
1333#endif
1334 p->first_tb = (TranslationBlock *)((uintptr_t)tb | n);
1335 invalidate_page_bitmap(p);
1336
1337#if defined(TARGET_HAS_SMC) || 1
1338
1339#if defined(CONFIG_USER_ONLY)
1340 if (p->flags & PAGE_WRITE) {
1341 target_ulong addr;
1342 PageDesc *p2;
1343 int prot;
1344
1345 /* force the host page as non writable (writes will have a
1346 page fault + mprotect overhead) */
1347 page_addr &= qemu_host_page_mask;
1348 prot = 0;
1349 for (addr = page_addr; addr < page_addr + qemu_host_page_size;
1350 addr += TARGET_PAGE_SIZE) {
1351
1352 p2 = page_find(addr >> TARGET_PAGE_BITS);
1353 if (!p2) {
1354 continue;
1355 }
1356 prot |= p2->flags;
1357 p2->flags &= ~PAGE_WRITE;
1358 }
1359 mprotect(g2h(page_addr), qemu_host_page_size,
1360 (prot & PAGE_BITS) & ~PAGE_WRITE);
1361#ifdef DEBUG_TB_INVALIDATE
1362 printf("protecting code page: 0x" TARGET_FMT_lx "\n",
1363 page_addr);
1364#endif
1365 }
1366#else
1367 /* if some code is already present, then the pages are already
1368 protected. So we handle the case where only the first TB is
1369 allocated in a physical page */
1370 if (!page_already_protected) {
1371 tlb_protect_code(page_addr);
1372 }
1373#endif
1374
1375#endif /* TARGET_HAS_SMC */
1376}
1377
1378/* add a new TB and link it to the physical page tables. phys_page2 is
1379 (-1) to indicate that only one page contains the TB. */
1380static void tb_link_page(TranslationBlock *tb, tb_page_addr_t phys_pc,
1381 tb_page_addr_t phys_page2)
1382{
1383 unsigned int h;
1384 TranslationBlock **ptb;
1385
1386 /* Grab the mmap lock to stop another thread invalidating this TB
1387 before we are done. */
1388 mmap_lock();
1389 /* add in the physical hash table */
1390 h = tb_phys_hash_func(phys_pc);
Evgeny Voevodin5e5f07e2013-02-01 01:47:23 +07001391 ptb = &tcg_ctx.tb_ctx.tb_phys_hash[h];
Blue Swirl5b6dd862012-12-02 16:04:43 +00001392 tb->phys_hash_next = *ptb;
1393 *ptb = tb;
1394
1395 /* add in the page list */
1396 tb_alloc_page(tb, 0, phys_pc & TARGET_PAGE_MASK);
1397 if (phys_page2 != -1) {
1398 tb_alloc_page(tb, 1, phys_page2);
1399 } else {
1400 tb->page_addr[1] = -1;
1401 }
1402
1403 tb->jmp_first = (TranslationBlock *)((uintptr_t)tb | 2);
1404 tb->jmp_next[0] = NULL;
1405 tb->jmp_next[1] = NULL;
1406
1407 /* init original jump addresses */
1408 if (tb->tb_next_offset[0] != 0xffff) {
1409 tb_reset_jump(tb, 0);
1410 }
1411 if (tb->tb_next_offset[1] != 0xffff) {
1412 tb_reset_jump(tb, 1);
1413 }
1414
1415#ifdef DEBUG_TB_CHECK
1416 tb_page_check();
1417#endif
1418 mmap_unlock();
1419}
1420
Blue Swirl5b6dd862012-12-02 16:04:43 +00001421/* find the TB 'tb' such that tb[0].tc_ptr <= tc_ptr <
1422 tb[1].tc_ptr. Return NULL if not found */
Blue Swirla8a826a2012-12-04 20:16:07 +00001423static TranslationBlock *tb_find_pc(uintptr_t tc_ptr)
Blue Swirl5b6dd862012-12-02 16:04:43 +00001424{
1425 int m_min, m_max, m;
1426 uintptr_t v;
1427 TranslationBlock *tb;
1428
Evgeny Voevodin5e5f07e2013-02-01 01:47:23 +07001429 if (tcg_ctx.tb_ctx.nb_tbs <= 0) {
Blue Swirl5b6dd862012-12-02 16:04:43 +00001430 return NULL;
1431 }
Evgeny Voevodin0b0d3322013-02-01 01:47:22 +07001432 if (tc_ptr < (uintptr_t)tcg_ctx.code_gen_buffer ||
1433 tc_ptr >= (uintptr_t)tcg_ctx.code_gen_ptr) {
Blue Swirl5b6dd862012-12-02 16:04:43 +00001434 return NULL;
1435 }
1436 /* binary search (cf Knuth) */
1437 m_min = 0;
Evgeny Voevodin5e5f07e2013-02-01 01:47:23 +07001438 m_max = tcg_ctx.tb_ctx.nb_tbs - 1;
Blue Swirl5b6dd862012-12-02 16:04:43 +00001439 while (m_min <= m_max) {
1440 m = (m_min + m_max) >> 1;
Evgeny Voevodin5e5f07e2013-02-01 01:47:23 +07001441 tb = &tcg_ctx.tb_ctx.tbs[m];
Blue Swirl5b6dd862012-12-02 16:04:43 +00001442 v = (uintptr_t)tb->tc_ptr;
1443 if (v == tc_ptr) {
1444 return tb;
1445 } else if (tc_ptr < v) {
1446 m_max = m - 1;
1447 } else {
1448 m_min = m + 1;
1449 }
1450 }
Evgeny Voevodin5e5f07e2013-02-01 01:47:23 +07001451 return &tcg_ctx.tb_ctx.tbs[m_max];
Blue Swirl5b6dd862012-12-02 16:04:43 +00001452}
1453
Peter Maydellec53b452015-01-20 15:19:32 +00001454#if !defined(CONFIG_USER_ONLY)
Edgar E. Iglesias29d8ec72013-11-07 19:43:10 +01001455void tb_invalidate_phys_addr(AddressSpace *as, hwaddr addr)
Blue Swirl5b6dd862012-12-02 16:04:43 +00001456{
1457 ram_addr_t ram_addr;
Paolo Bonzini5c8a00c2013-05-29 12:42:00 +02001458 MemoryRegion *mr;
Paolo Bonzini149f54b2013-05-24 12:59:37 +02001459 hwaddr l = 1;
Blue Swirl5b6dd862012-12-02 16:04:43 +00001460
Edgar E. Iglesias29d8ec72013-11-07 19:43:10 +01001461 mr = address_space_translate(as, addr, &addr, &l, false);
Paolo Bonzini5c8a00c2013-05-29 12:42:00 +02001462 if (!(memory_region_is_ram(mr)
1463 || memory_region_is_romd(mr))) {
Blue Swirl5b6dd862012-12-02 16:04:43 +00001464 return;
1465 }
Paolo Bonzini5c8a00c2013-05-29 12:42:00 +02001466 ram_addr = (memory_region_get_ram_addr(mr) & TARGET_PAGE_MASK)
Paolo Bonzini149f54b2013-05-24 12:59:37 +02001467 + addr;
Blue Swirl5b6dd862012-12-02 16:04:43 +00001468 tb_invalidate_phys_page_range(ram_addr, ram_addr + 1, 0);
1469}
Peter Maydellec53b452015-01-20 15:19:32 +00001470#endif /* !defined(CONFIG_USER_ONLY) */
Blue Swirl5b6dd862012-12-02 16:04:43 +00001471
Andreas Färber239c51a2013-09-01 17:12:23 +02001472void tb_check_watchpoint(CPUState *cpu)
Blue Swirl5b6dd862012-12-02 16:04:43 +00001473{
1474 TranslationBlock *tb;
1475
Andreas Färber93afead2013-08-26 03:41:01 +02001476 tb = tb_find_pc(cpu->mem_io_pc);
Blue Swirl5b6dd862012-12-02 16:04:43 +00001477 if (!tb) {
Andreas Färbera47dddd2013-09-03 17:38:47 +02001478 cpu_abort(cpu, "check_watchpoint: could not find TB for pc=%p",
Andreas Färber93afead2013-08-26 03:41:01 +02001479 (void *)cpu->mem_io_pc);
Blue Swirl5b6dd862012-12-02 16:04:43 +00001480 }
Andreas Färber74f10512013-09-01 17:02:58 +02001481 cpu_restore_state_from_tb(cpu, tb, cpu->mem_io_pc);
Blue Swirl5b6dd862012-12-02 16:04:43 +00001482 tb_phys_invalidate(tb, -1);
1483}
1484
1485#ifndef CONFIG_USER_ONLY
1486/* mask must never be zero, except for A20 change call */
Andreas Färberc3affe52013-01-18 15:03:43 +01001487static void tcg_handle_interrupt(CPUState *cpu, int mask)
Blue Swirl5b6dd862012-12-02 16:04:43 +00001488{
Blue Swirl5b6dd862012-12-02 16:04:43 +00001489 int old_mask;
1490
Andreas Färber259186a2013-01-17 18:51:17 +01001491 old_mask = cpu->interrupt_request;
1492 cpu->interrupt_request |= mask;
Blue Swirl5b6dd862012-12-02 16:04:43 +00001493
1494 /*
1495 * If called from iothread context, wake the target cpu in
1496 * case its halted.
1497 */
1498 if (!qemu_cpu_is_self(cpu)) {
1499 qemu_cpu_kick(cpu);
1500 return;
1501 }
1502
1503 if (use_icount) {
Andreas Färber28ecfd72013-08-26 05:51:49 +02001504 cpu->icount_decr.u16.high = 0xffff;
Andreas Färber99df7dc2013-08-26 05:15:23 +02001505 if (!cpu_can_do_io(cpu)
Blue Swirl5b6dd862012-12-02 16:04:43 +00001506 && (mask & ~old_mask) != 0) {
Andreas Färbera47dddd2013-09-03 17:38:47 +02001507 cpu_abort(cpu, "Raised interrupt while not in I/O function");
Blue Swirl5b6dd862012-12-02 16:04:43 +00001508 }
1509 } else {
Peter Maydell378df4b2013-02-22 18:10:03 +00001510 cpu->tcg_exit_req = 1;
Blue Swirl5b6dd862012-12-02 16:04:43 +00001511 }
1512}
1513
1514CPUInterruptHandler cpu_interrupt_handler = tcg_handle_interrupt;
1515
1516/* in deterministic execution mode, instructions doing device I/Os
1517 must be at the end of the TB */
Andreas Färber90b40a62013-09-01 17:21:47 +02001518void cpu_io_recompile(CPUState *cpu, uintptr_t retaddr)
Blue Swirl5b6dd862012-12-02 16:04:43 +00001519{
Andreas Färbera47dddd2013-09-03 17:38:47 +02001520#if defined(TARGET_MIPS) || defined(TARGET_SH4)
Andreas Färber90b40a62013-09-01 17:21:47 +02001521 CPUArchState *env = cpu->env_ptr;
Andreas Färbera47dddd2013-09-03 17:38:47 +02001522#endif
Blue Swirl5b6dd862012-12-02 16:04:43 +00001523 TranslationBlock *tb;
1524 uint32_t n, cflags;
1525 target_ulong pc, cs_base;
1526 uint64_t flags;
1527
1528 tb = tb_find_pc(retaddr);
1529 if (!tb) {
Andreas Färbera47dddd2013-09-03 17:38:47 +02001530 cpu_abort(cpu, "cpu_io_recompile: could not find TB for pc=%p",
Blue Swirl5b6dd862012-12-02 16:04:43 +00001531 (void *)retaddr);
1532 }
Andreas Färber28ecfd72013-08-26 05:51:49 +02001533 n = cpu->icount_decr.u16.low + tb->icount;
Andreas Färber74f10512013-09-01 17:02:58 +02001534 cpu_restore_state_from_tb(cpu, tb, retaddr);
Blue Swirl5b6dd862012-12-02 16:04:43 +00001535 /* Calculate how many instructions had been executed before the fault
1536 occurred. */
Andreas Färber28ecfd72013-08-26 05:51:49 +02001537 n = n - cpu->icount_decr.u16.low;
Blue Swirl5b6dd862012-12-02 16:04:43 +00001538 /* Generate a new TB ending on the I/O insn. */
1539 n++;
1540 /* On MIPS and SH, delay slot instructions can only be restarted if
1541 they were already the first instruction in the TB. If this is not
1542 the first instruction in a TB then re-execute the preceding
1543 branch. */
1544#if defined(TARGET_MIPS)
1545 if ((env->hflags & MIPS_HFLAG_BMASK) != 0 && n > 1) {
Maciej W. Rozyckic3577472014-11-07 20:05:35 +00001546 env->active_tc.PC -= (env->hflags & MIPS_HFLAG_B16 ? 2 : 4);
Andreas Färber28ecfd72013-08-26 05:51:49 +02001547 cpu->icount_decr.u16.low++;
Blue Swirl5b6dd862012-12-02 16:04:43 +00001548 env->hflags &= ~MIPS_HFLAG_BMASK;
1549 }
1550#elif defined(TARGET_SH4)
1551 if ((env->flags & ((DELAY_SLOT | DELAY_SLOT_CONDITIONAL))) != 0
1552 && n > 1) {
1553 env->pc -= 2;
Andreas Färber28ecfd72013-08-26 05:51:49 +02001554 cpu->icount_decr.u16.low++;
Blue Swirl5b6dd862012-12-02 16:04:43 +00001555 env->flags &= ~(DELAY_SLOT | DELAY_SLOT_CONDITIONAL);
1556 }
1557#endif
1558 /* This should never happen. */
1559 if (n > CF_COUNT_MASK) {
Andreas Färbera47dddd2013-09-03 17:38:47 +02001560 cpu_abort(cpu, "TB too big during recompile");
Blue Swirl5b6dd862012-12-02 16:04:43 +00001561 }
1562
1563 cflags = n | CF_LAST_IO;
1564 pc = tb->pc;
1565 cs_base = tb->cs_base;
1566 flags = tb->flags;
1567 tb_phys_invalidate(tb, -1);
1568 /* FIXME: In theory this could raise an exception. In practice
1569 we have already translated the block once so it's probably ok. */
Andreas Färber648f0342013-09-01 17:43:17 +02001570 tb_gen_code(cpu, pc, cs_base, flags, cflags);
Blue Swirl5b6dd862012-12-02 16:04:43 +00001571 /* TODO: If env->pc != tb->pc (i.e. the faulting instruction was not
1572 the first in the TB) then we end up generating a whole new TB and
1573 repeating the fault, which is horribly inefficient.
1574 Better would be to execute just this insn uncached, or generate a
1575 second new TB. */
Andreas Färber0ea8cb82013-09-03 02:12:23 +02001576 cpu_resume_from_signal(cpu, NULL);
Blue Swirl5b6dd862012-12-02 16:04:43 +00001577}
1578
Andreas Färber611d4f92013-09-01 17:52:07 +02001579void tb_flush_jmp_cache(CPUState *cpu, target_ulong addr)
Blue Swirl5b6dd862012-12-02 16:04:43 +00001580{
1581 unsigned int i;
1582
1583 /* Discard jump cache entries for any tb which might potentially
1584 overlap the flushed page. */
1585 i = tb_jmp_cache_hash_page(addr - TARGET_PAGE_SIZE);
Andreas Färber8cd70432013-08-26 06:03:38 +02001586 memset(&cpu->tb_jmp_cache[i], 0,
Blue Swirl5b6dd862012-12-02 16:04:43 +00001587 TB_JMP_PAGE_SIZE * sizeof(TranslationBlock *));
1588
1589 i = tb_jmp_cache_hash_page(addr);
Andreas Färber8cd70432013-08-26 06:03:38 +02001590 memset(&cpu->tb_jmp_cache[i], 0,
Blue Swirl5b6dd862012-12-02 16:04:43 +00001591 TB_JMP_PAGE_SIZE * sizeof(TranslationBlock *));
1592}
1593
1594void dump_exec_info(FILE *f, fprintf_function cpu_fprintf)
1595{
1596 int i, target_code_size, max_target_code_size;
1597 int direct_jmp_count, direct_jmp2_count, cross_page;
1598 TranslationBlock *tb;
1599
1600 target_code_size = 0;
1601 max_target_code_size = 0;
1602 cross_page = 0;
1603 direct_jmp_count = 0;
1604 direct_jmp2_count = 0;
Evgeny Voevodin5e5f07e2013-02-01 01:47:23 +07001605 for (i = 0; i < tcg_ctx.tb_ctx.nb_tbs; i++) {
1606 tb = &tcg_ctx.tb_ctx.tbs[i];
Blue Swirl5b6dd862012-12-02 16:04:43 +00001607 target_code_size += tb->size;
1608 if (tb->size > max_target_code_size) {
1609 max_target_code_size = tb->size;
1610 }
1611 if (tb->page_addr[1] != -1) {
1612 cross_page++;
1613 }
1614 if (tb->tb_next_offset[0] != 0xffff) {
1615 direct_jmp_count++;
1616 if (tb->tb_next_offset[1] != 0xffff) {
1617 direct_jmp2_count++;
1618 }
1619 }
1620 }
1621 /* XXX: avoid using doubles ? */
1622 cpu_fprintf(f, "Translation buffer state:\n");
1623 cpu_fprintf(f, "gen code size %td/%zd\n",
Evgeny Voevodin0b0d3322013-02-01 01:47:22 +07001624 tcg_ctx.code_gen_ptr - tcg_ctx.code_gen_buffer,
1625 tcg_ctx.code_gen_buffer_max_size);
Blue Swirl5b6dd862012-12-02 16:04:43 +00001626 cpu_fprintf(f, "TB count %d/%d\n",
Evgeny Voevodin5e5f07e2013-02-01 01:47:23 +07001627 tcg_ctx.tb_ctx.nb_tbs, tcg_ctx.code_gen_max_blocks);
Blue Swirl5b6dd862012-12-02 16:04:43 +00001628 cpu_fprintf(f, "TB avg target size %d max=%d bytes\n",
Evgeny Voevodin5e5f07e2013-02-01 01:47:23 +07001629 tcg_ctx.tb_ctx.nb_tbs ? target_code_size /
1630 tcg_ctx.tb_ctx.nb_tbs : 0,
1631 max_target_code_size);
Blue Swirl5b6dd862012-12-02 16:04:43 +00001632 cpu_fprintf(f, "TB avg host size %td bytes (expansion ratio: %0.1f)\n",
Evgeny Voevodin5e5f07e2013-02-01 01:47:23 +07001633 tcg_ctx.tb_ctx.nb_tbs ? (tcg_ctx.code_gen_ptr -
1634 tcg_ctx.code_gen_buffer) /
1635 tcg_ctx.tb_ctx.nb_tbs : 0,
1636 target_code_size ? (double) (tcg_ctx.code_gen_ptr -
1637 tcg_ctx.code_gen_buffer) /
1638 target_code_size : 0);
1639 cpu_fprintf(f, "cross page TB count %d (%d%%)\n", cross_page,
1640 tcg_ctx.tb_ctx.nb_tbs ? (cross_page * 100) /
1641 tcg_ctx.tb_ctx.nb_tbs : 0);
Blue Swirl5b6dd862012-12-02 16:04:43 +00001642 cpu_fprintf(f, "direct jump count %d (%d%%) (2 jumps=%d %d%%)\n",
1643 direct_jmp_count,
Evgeny Voevodin5e5f07e2013-02-01 01:47:23 +07001644 tcg_ctx.tb_ctx.nb_tbs ? (direct_jmp_count * 100) /
1645 tcg_ctx.tb_ctx.nb_tbs : 0,
Blue Swirl5b6dd862012-12-02 16:04:43 +00001646 direct_jmp2_count,
Evgeny Voevodin5e5f07e2013-02-01 01:47:23 +07001647 tcg_ctx.tb_ctx.nb_tbs ? (direct_jmp2_count * 100) /
1648 tcg_ctx.tb_ctx.nb_tbs : 0);
Blue Swirl5b6dd862012-12-02 16:04:43 +00001649 cpu_fprintf(f, "\nStatistics:\n");
Evgeny Voevodin5e5f07e2013-02-01 01:47:23 +07001650 cpu_fprintf(f, "TB flush count %d\n", tcg_ctx.tb_ctx.tb_flush_count);
1651 cpu_fprintf(f, "TB invalidate count %d\n",
1652 tcg_ctx.tb_ctx.tb_phys_invalidate_count);
Blue Swirl5b6dd862012-12-02 16:04:43 +00001653 cpu_fprintf(f, "TLB flush count %d\n", tlb_flush_count);
1654 tcg_dump_info(f, cpu_fprintf);
1655}
1656
Max Filippov246ae242014-11-02 11:04:18 +03001657void dump_opcount_info(FILE *f, fprintf_function cpu_fprintf)
1658{
1659 tcg_dump_op_count(f, cpu_fprintf);
1660}
1661
Blue Swirl5b6dd862012-12-02 16:04:43 +00001662#else /* CONFIG_USER_ONLY */
1663
Andreas Färberc3affe52013-01-18 15:03:43 +01001664void cpu_interrupt(CPUState *cpu, int mask)
Blue Swirl5b6dd862012-12-02 16:04:43 +00001665{
Andreas Färber259186a2013-01-17 18:51:17 +01001666 cpu->interrupt_request |= mask;
Peter Maydell378df4b2013-02-22 18:10:03 +00001667 cpu->tcg_exit_req = 1;
Blue Swirl5b6dd862012-12-02 16:04:43 +00001668}
1669
1670/*
1671 * Walks guest process memory "regions" one by one
1672 * and calls callback function 'fn' for each region.
1673 */
1674struct walk_memory_regions_data {
1675 walk_memory_regions_fn fn;
1676 void *priv;
Mikhail Ilyin1a1c4db2014-09-08 17:28:56 +04001677 target_ulong start;
Blue Swirl5b6dd862012-12-02 16:04:43 +00001678 int prot;
1679};
1680
1681static int walk_memory_regions_end(struct walk_memory_regions_data *data,
Mikhail Ilyin1a1c4db2014-09-08 17:28:56 +04001682 target_ulong end, int new_prot)
Blue Swirl5b6dd862012-12-02 16:04:43 +00001683{
Mikhail Ilyin1a1c4db2014-09-08 17:28:56 +04001684 if (data->start != -1u) {
Blue Swirl5b6dd862012-12-02 16:04:43 +00001685 int rc = data->fn(data->priv, data->start, end, data->prot);
1686 if (rc != 0) {
1687 return rc;
1688 }
1689 }
1690
Mikhail Ilyin1a1c4db2014-09-08 17:28:56 +04001691 data->start = (new_prot ? end : -1u);
Blue Swirl5b6dd862012-12-02 16:04:43 +00001692 data->prot = new_prot;
1693
1694 return 0;
1695}
1696
1697static int walk_memory_regions_1(struct walk_memory_regions_data *data,
Mikhail Ilyin1a1c4db2014-09-08 17:28:56 +04001698 target_ulong base, int level, void **lp)
Blue Swirl5b6dd862012-12-02 16:04:43 +00001699{
Mikhail Ilyin1a1c4db2014-09-08 17:28:56 +04001700 target_ulong pa;
Blue Swirl5b6dd862012-12-02 16:04:43 +00001701 int i, rc;
1702
1703 if (*lp == NULL) {
1704 return walk_memory_regions_end(data, base, 0);
1705 }
1706
1707 if (level == 0) {
1708 PageDesc *pd = *lp;
1709
Paolo Bonzini03f49952013-11-07 17:14:36 +01001710 for (i = 0; i < V_L2_SIZE; ++i) {
Blue Swirl5b6dd862012-12-02 16:04:43 +00001711 int prot = pd[i].flags;
1712
1713 pa = base | (i << TARGET_PAGE_BITS);
1714 if (prot != data->prot) {
1715 rc = walk_memory_regions_end(data, pa, prot);
1716 if (rc != 0) {
1717 return rc;
1718 }
1719 }
1720 }
1721 } else {
1722 void **pp = *lp;
1723
Paolo Bonzini03f49952013-11-07 17:14:36 +01001724 for (i = 0; i < V_L2_SIZE; ++i) {
Mikhail Ilyin1a1c4db2014-09-08 17:28:56 +04001725 pa = base | ((target_ulong)i <<
Paolo Bonzini03f49952013-11-07 17:14:36 +01001726 (TARGET_PAGE_BITS + V_L2_BITS * level));
Blue Swirl5b6dd862012-12-02 16:04:43 +00001727 rc = walk_memory_regions_1(data, pa, level - 1, pp + i);
1728 if (rc != 0) {
1729 return rc;
1730 }
1731 }
1732 }
1733
1734 return 0;
1735}
1736
1737int walk_memory_regions(void *priv, walk_memory_regions_fn fn)
1738{
1739 struct walk_memory_regions_data data;
1740 uintptr_t i;
1741
1742 data.fn = fn;
1743 data.priv = priv;
Mikhail Ilyin1a1c4db2014-09-08 17:28:56 +04001744 data.start = -1u;
Blue Swirl5b6dd862012-12-02 16:04:43 +00001745 data.prot = 0;
1746
1747 for (i = 0; i < V_L1_SIZE; i++) {
Mikhail Ilyin1a1c4db2014-09-08 17:28:56 +04001748 int rc = walk_memory_regions_1(&data, (target_ulong)i << (V_L1_SHIFT + TARGET_PAGE_BITS),
Paolo Bonzini03f49952013-11-07 17:14:36 +01001749 V_L1_SHIFT / V_L2_BITS - 1, l1_map + i);
Blue Swirl5b6dd862012-12-02 16:04:43 +00001750 if (rc != 0) {
1751 return rc;
1752 }
1753 }
1754
1755 return walk_memory_regions_end(&data, 0, 0);
1756}
1757
Mikhail Ilyin1a1c4db2014-09-08 17:28:56 +04001758static int dump_region(void *priv, target_ulong start,
1759 target_ulong end, unsigned long prot)
Blue Swirl5b6dd862012-12-02 16:04:43 +00001760{
1761 FILE *f = (FILE *)priv;
1762
Mikhail Ilyin1a1c4db2014-09-08 17:28:56 +04001763 (void) fprintf(f, TARGET_FMT_lx"-"TARGET_FMT_lx
1764 " "TARGET_FMT_lx" %c%c%c\n",
Blue Swirl5b6dd862012-12-02 16:04:43 +00001765 start, end, end - start,
1766 ((prot & PAGE_READ) ? 'r' : '-'),
1767 ((prot & PAGE_WRITE) ? 'w' : '-'),
1768 ((prot & PAGE_EXEC) ? 'x' : '-'));
1769
1770 return 0;
1771}
1772
1773/* dump memory mappings */
1774void page_dump(FILE *f)
1775{
Mikhail Ilyin1a1c4db2014-09-08 17:28:56 +04001776 const int length = sizeof(target_ulong) * 2;
Stefan Weil227b8172013-09-12 20:09:06 +02001777 (void) fprintf(f, "%-*s %-*s %-*s %s\n",
1778 length, "start", length, "end", length, "size", "prot");
Blue Swirl5b6dd862012-12-02 16:04:43 +00001779 walk_memory_regions(f, dump_region);
1780}
1781
1782int page_get_flags(target_ulong address)
1783{
1784 PageDesc *p;
1785
1786 p = page_find(address >> TARGET_PAGE_BITS);
1787 if (!p) {
1788 return 0;
1789 }
1790 return p->flags;
1791}
1792
1793/* Modify the flags of a page and invalidate the code if necessary.
1794 The flag PAGE_WRITE_ORG is positioned automatically depending
1795 on PAGE_WRITE. The mmap_lock should already be held. */
1796void page_set_flags(target_ulong start, target_ulong end, int flags)
1797{
1798 target_ulong addr, len;
1799
1800 /* This function should never be called with addresses outside the
1801 guest address space. If this assert fires, it probably indicates
1802 a missing call to h2g_valid. */
1803#if TARGET_ABI_BITS > L1_MAP_ADDR_SPACE_BITS
Mikhail Ilyin1a1c4db2014-09-08 17:28:56 +04001804 assert(end < ((target_ulong)1 << L1_MAP_ADDR_SPACE_BITS));
Blue Swirl5b6dd862012-12-02 16:04:43 +00001805#endif
1806 assert(start < end);
1807
1808 start = start & TARGET_PAGE_MASK;
1809 end = TARGET_PAGE_ALIGN(end);
1810
1811 if (flags & PAGE_WRITE) {
1812 flags |= PAGE_WRITE_ORG;
1813 }
1814
1815 for (addr = start, len = end - start;
1816 len != 0;
1817 len -= TARGET_PAGE_SIZE, addr += TARGET_PAGE_SIZE) {
1818 PageDesc *p = page_find_alloc(addr >> TARGET_PAGE_BITS, 1);
1819
1820 /* If the write protection bit is set, then we invalidate
1821 the code inside. */
1822 if (!(p->flags & PAGE_WRITE) &&
1823 (flags & PAGE_WRITE) &&
1824 p->first_tb) {
Alexander Grafd02532f2013-07-06 14:17:57 +02001825 tb_invalidate_phys_page(addr, 0, NULL, false);
Blue Swirl5b6dd862012-12-02 16:04:43 +00001826 }
1827 p->flags = flags;
1828 }
1829}
1830
1831int page_check_range(target_ulong start, target_ulong len, int flags)
1832{
1833 PageDesc *p;
1834 target_ulong end;
1835 target_ulong addr;
1836
1837 /* This function should never be called with addresses outside the
1838 guest address space. If this assert fires, it probably indicates
1839 a missing call to h2g_valid. */
1840#if TARGET_ABI_BITS > L1_MAP_ADDR_SPACE_BITS
Mikhail Ilyin1a1c4db2014-09-08 17:28:56 +04001841 assert(start < ((target_ulong)1 << L1_MAP_ADDR_SPACE_BITS));
Blue Swirl5b6dd862012-12-02 16:04:43 +00001842#endif
1843
1844 if (len == 0) {
1845 return 0;
1846 }
1847 if (start + len - 1 < start) {
1848 /* We've wrapped around. */
1849 return -1;
1850 }
1851
1852 /* must do before we loose bits in the next step */
1853 end = TARGET_PAGE_ALIGN(start + len);
1854 start = start & TARGET_PAGE_MASK;
1855
1856 for (addr = start, len = end - start;
1857 len != 0;
1858 len -= TARGET_PAGE_SIZE, addr += TARGET_PAGE_SIZE) {
1859 p = page_find(addr >> TARGET_PAGE_BITS);
1860 if (!p) {
1861 return -1;
1862 }
1863 if (!(p->flags & PAGE_VALID)) {
1864 return -1;
1865 }
1866
1867 if ((flags & PAGE_READ) && !(p->flags & PAGE_READ)) {
1868 return -1;
1869 }
1870 if (flags & PAGE_WRITE) {
1871 if (!(p->flags & PAGE_WRITE_ORG)) {
1872 return -1;
1873 }
1874 /* unprotect the page if it was put read-only because it
1875 contains translated code */
1876 if (!(p->flags & PAGE_WRITE)) {
1877 if (!page_unprotect(addr, 0, NULL)) {
1878 return -1;
1879 }
1880 }
Blue Swirl5b6dd862012-12-02 16:04:43 +00001881 }
1882 }
1883 return 0;
1884}
1885
1886/* called from signal handler: invalidate the code and unprotect the
1887 page. Return TRUE if the fault was successfully handled. */
1888int page_unprotect(target_ulong address, uintptr_t pc, void *puc)
1889{
1890 unsigned int prot;
1891 PageDesc *p;
1892 target_ulong host_start, host_end, addr;
1893
1894 /* Technically this isn't safe inside a signal handler. However we
1895 know this only ever happens in a synchronous SEGV handler, so in
1896 practice it seems to be ok. */
1897 mmap_lock();
1898
1899 p = page_find(address >> TARGET_PAGE_BITS);
1900 if (!p) {
1901 mmap_unlock();
1902 return 0;
1903 }
1904
1905 /* if the page was really writable, then we change its
1906 protection back to writable */
1907 if ((p->flags & PAGE_WRITE_ORG) && !(p->flags & PAGE_WRITE)) {
1908 host_start = address & qemu_host_page_mask;
1909 host_end = host_start + qemu_host_page_size;
1910
1911 prot = 0;
1912 for (addr = host_start ; addr < host_end ; addr += TARGET_PAGE_SIZE) {
1913 p = page_find(addr >> TARGET_PAGE_BITS);
1914 p->flags |= PAGE_WRITE;
1915 prot |= p->flags;
1916
1917 /* and since the content will be modified, we must invalidate
1918 the corresponding translated code. */
Alexander Grafd02532f2013-07-06 14:17:57 +02001919 tb_invalidate_phys_page(addr, pc, puc, true);
Blue Swirl5b6dd862012-12-02 16:04:43 +00001920#ifdef DEBUG_TB_CHECK
1921 tb_invalidate_check(addr);
1922#endif
1923 }
1924 mprotect((void *)g2h(host_start), qemu_host_page_size,
1925 prot & PAGE_BITS);
1926
1927 mmap_unlock();
1928 return 1;
1929 }
1930 mmap_unlock();
1931 return 0;
1932}
1933#endif /* CONFIG_USER_ONLY */