blob: 4888f531395576cfcae864c3866edf55cd0173f3 [file] [log] [blame]
bellard54936002003-05-13 00:25:15 +00001/*
2 * mmap support for qemu
ths5fafdf22007-09-16 21:08:06 +00003 *
bellard54936002003-05-13 00:25:15 +00004 * Copyright (c) 2003 Fabrice Bellard
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License as published by
8 * the Free Software Foundation; either version 2 of the License, or
9 * (at your option) any later version.
10 *
11 * This program is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU General Public License for more details.
15 *
16 * You should have received a copy of the GNU General Public License
Blue Swirl8167ee82009-07-16 20:47:01 +000017 * along with this program; if not, see <http://www.gnu.org/licenses/>.
bellard54936002003-05-13 00:25:15 +000018 */
Peter Maydelld39594e2016-01-26 18:17:02 +000019#include "qemu/osdep.h"
bellard54936002003-05-13 00:25:15 +000020
21#include "qemu.h"
blueswir178f5bf12008-10-02 19:55:50 +000022#include "qemu-common.h"
Paolo Bonzini1652b972015-04-22 14:15:48 +020023#include "translate-all.h"
bellard54936002003-05-13 00:25:15 +000024
25//#define DEBUG_MMAP
26
Blue Swirl1e6eec82009-09-05 10:14:07 +000027static pthread_mutex_t mmap_mutex = PTHREAD_MUTEX_INITIALIZER;
Juan Quinteladfd3f852009-09-23 01:19:03 +020028static __thread int mmap_lock_count;
pbrookc8a706f2008-06-02 16:16:42 +000029
30void mmap_lock(void)
31{
32 if (mmap_lock_count++ == 0) {
33 pthread_mutex_lock(&mmap_mutex);
34 }
35}
36
37void mmap_unlock(void)
38{
39 if (--mmap_lock_count == 0) {
40 pthread_mutex_unlock(&mmap_mutex);
41 }
42}
pbrookd5975362008-06-07 20:50:51 +000043
Alex Bennée301e40e2016-10-27 16:10:00 +010044bool have_mmap_lock(void)
45{
46 return mmap_lock_count > 0 ? true : false;
47}
48
pbrookd5975362008-06-07 20:50:51 +000049/* Grab lock to make sure things are in a consistent state after fork(). */
50void mmap_fork_start(void)
51{
52 if (mmap_lock_count)
53 abort();
54 pthread_mutex_lock(&mmap_mutex);
55}
56
57void mmap_fork_end(int child)
58{
59 if (child)
60 pthread_mutex_init(&mmap_mutex, NULL);
61 else
62 pthread_mutex_unlock(&mmap_mutex);
63}
pbrookc8a706f2008-06-02 16:16:42 +000064
pbrook53a59602006-03-25 19:31:22 +000065/* NOTE: all the constants are the HOST ones, but addresses are target. */
blueswir1992f48a2007-10-14 16:27:31 +000066int target_mprotect(abi_ulong start, abi_ulong len, int prot)
bellard54936002003-05-13 00:25:15 +000067{
blueswir1992f48a2007-10-14 16:27:31 +000068 abi_ulong end, host_start, host_end, addr;
bellard54936002003-05-13 00:25:15 +000069 int prot1, ret;
70
71#ifdef DEBUG_MMAP
Blue Swirl0bf9e312009-07-20 17:19:25 +000072 printf("mprotect: start=0x" TARGET_ABI_FMT_lx
73 "len=0x" TARGET_ABI_FMT_lx " prot=%c%c%c\n", start, len,
bellard54936002003-05-13 00:25:15 +000074 prot & PROT_READ ? 'r' : '-',
75 prot & PROT_WRITE ? 'w' : '-',
76 prot & PROT_EXEC ? 'x' : '-');
77#endif
78
79 if ((start & ~TARGET_PAGE_MASK) != 0)
80 return -EINVAL;
81 len = TARGET_PAGE_ALIGN(len);
82 end = start + len;
83 if (end < start)
84 return -EINVAL;
balrog171cd1c2008-04-24 21:11:41 +000085 prot &= PROT_READ | PROT_WRITE | PROT_EXEC;
bellard54936002003-05-13 00:25:15 +000086 if (len == 0)
87 return 0;
ths3b46e622007-09-17 08:09:54 +000088
pbrookc8a706f2008-06-02 16:16:42 +000089 mmap_lock();
bellard83fb7ad2004-07-05 21:25:26 +000090 host_start = start & qemu_host_page_mask;
bellard54936002003-05-13 00:25:15 +000091 host_end = HOST_PAGE_ALIGN(end);
92 if (start > host_start) {
93 /* handle host page containing start */
94 prot1 = prot;
95 for(addr = host_start; addr < start; addr += TARGET_PAGE_SIZE) {
96 prot1 |= page_get_flags(addr);
97 }
bellard83fb7ad2004-07-05 21:25:26 +000098 if (host_end == host_start + qemu_host_page_size) {
bellardd418c812003-05-13 00:57:50 +000099 for(addr = end; addr < host_end; addr += TARGET_PAGE_SIZE) {
100 prot1 |= page_get_flags(addr);
101 }
102 end = host_end;
103 }
pbrook53a59602006-03-25 19:31:22 +0000104 ret = mprotect(g2h(host_start), qemu_host_page_size, prot1 & PAGE_BITS);
bellard54936002003-05-13 00:25:15 +0000105 if (ret != 0)
pbrookc8a706f2008-06-02 16:16:42 +0000106 goto error;
bellard83fb7ad2004-07-05 21:25:26 +0000107 host_start += qemu_host_page_size;
bellard54936002003-05-13 00:25:15 +0000108 }
109 if (end < host_end) {
bellard54936002003-05-13 00:25:15 +0000110 prot1 = prot;
111 for(addr = end; addr < host_end; addr += TARGET_PAGE_SIZE) {
112 prot1 |= page_get_flags(addr);
113 }
ths5fafdf22007-09-16 21:08:06 +0000114 ret = mprotect(g2h(host_end - qemu_host_page_size), qemu_host_page_size,
bellard54936002003-05-13 00:25:15 +0000115 prot1 & PAGE_BITS);
116 if (ret != 0)
pbrookc8a706f2008-06-02 16:16:42 +0000117 goto error;
bellard83fb7ad2004-07-05 21:25:26 +0000118 host_end -= qemu_host_page_size;
bellard54936002003-05-13 00:25:15 +0000119 }
ths3b46e622007-09-17 08:09:54 +0000120
bellard54936002003-05-13 00:25:15 +0000121 /* handle the pages in the middle */
122 if (host_start < host_end) {
pbrook53a59602006-03-25 19:31:22 +0000123 ret = mprotect(g2h(host_start), host_end - host_start, prot);
bellard54936002003-05-13 00:25:15 +0000124 if (ret != 0)
pbrookc8a706f2008-06-02 16:16:42 +0000125 goto error;
bellard54936002003-05-13 00:25:15 +0000126 }
bellard54936002003-05-13 00:25:15 +0000127 page_set_flags(start, start + len, prot | PAGE_VALID);
pbrookc8a706f2008-06-02 16:16:42 +0000128 mmap_unlock();
bellard54936002003-05-13 00:25:15 +0000129 return 0;
pbrookc8a706f2008-06-02 16:16:42 +0000130error:
131 mmap_unlock();
132 return ret;
bellard54936002003-05-13 00:25:15 +0000133}
134
135/* map an incomplete host page */
blueswir1992f48a2007-10-14 16:27:31 +0000136static int mmap_frag(abi_ulong real_start,
137 abi_ulong start, abi_ulong end,
138 int prot, int flags, int fd, abi_ulong offset)
bellard54936002003-05-13 00:25:15 +0000139{
ths80210bc2007-11-02 19:08:57 +0000140 abi_ulong real_end, addr;
pbrook53a59602006-03-25 19:31:22 +0000141 void *host_start;
bellard54936002003-05-13 00:25:15 +0000142 int prot1, prot_new;
143
pbrook53a59602006-03-25 19:31:22 +0000144 real_end = real_start + qemu_host_page_size;
145 host_start = g2h(real_start);
bellard54936002003-05-13 00:25:15 +0000146
147 /* get the protection of the target pages outside the mapping */
148 prot1 = 0;
pbrook53a59602006-03-25 19:31:22 +0000149 for(addr = real_start; addr < real_end; addr++) {
bellard54936002003-05-13 00:25:15 +0000150 if (addr < start || addr >= end)
151 prot1 |= page_get_flags(addr);
152 }
ths3b46e622007-09-17 08:09:54 +0000153
bellard54936002003-05-13 00:25:15 +0000154 if (prot1 == 0) {
155 /* no page was there, so we allocate one */
ths80210bc2007-11-02 19:08:57 +0000156 void *p = mmap(host_start, qemu_host_page_size, prot,
157 flags | MAP_ANONYMOUS, -1, 0);
158 if (p == MAP_FAILED)
159 return -1;
pbrook53a59602006-03-25 19:31:22 +0000160 prot1 = prot;
bellard54936002003-05-13 00:25:15 +0000161 }
162 prot1 &= PAGE_BITS;
163
164 prot_new = prot | prot1;
165 if (!(flags & MAP_ANONYMOUS)) {
166 /* msync() won't work here, so we return an error if write is
167 possible while it is a shared mapping */
168 if ((flags & MAP_TYPE) == MAP_SHARED &&
169 (prot & PROT_WRITE))
Juan Quintelaee636502010-01-20 00:56:24 +0100170 return -1;
bellard54936002003-05-13 00:25:15 +0000171
172 /* adjust protection to be able to read */
173 if (!(prot1 & PROT_WRITE))
pbrook53a59602006-03-25 19:31:22 +0000174 mprotect(host_start, qemu_host_page_size, prot1 | PROT_WRITE);
ths3b46e622007-09-17 08:09:54 +0000175
bellard54936002003-05-13 00:25:15 +0000176 /* read the corresponding file data */
Kirill A. Shutemovfb7e3782010-01-20 00:56:20 +0100177 if (pread(fd, g2h(start), end - start, offset) == -1)
178 return -1;
ths3b46e622007-09-17 08:09:54 +0000179
bellard54936002003-05-13 00:25:15 +0000180 /* put final protection */
181 if (prot_new != (prot1 | PROT_WRITE))
pbrook53a59602006-03-25 19:31:22 +0000182 mprotect(host_start, qemu_host_page_size, prot_new);
bellard54936002003-05-13 00:25:15 +0000183 } else {
bellard54936002003-05-13 00:25:15 +0000184 if (prot_new != prot1) {
pbrook53a59602006-03-25 19:31:22 +0000185 mprotect(host_start, qemu_host_page_size, prot_new);
bellard54936002003-05-13 00:25:15 +0000186 }
Chen Gange6deac92015-12-30 09:10:54 +0800187 if (prot_new & PROT_WRITE) {
188 memset(g2h(start), 0, end - start);
189 }
bellard54936002003-05-13 00:25:15 +0000190 }
191 return 0;
192}
193
Richard Henderson14f24e12010-03-10 15:39:07 -0800194#if HOST_LONG_BITS == 64 && TARGET_ABI_BITS == 64
195# define TASK_UNMAPPED_BASE (1ul << 38)
bellarda03e2d42007-11-14 11:29:07 +0000196#else
Richard Henderson14f24e12010-03-10 15:39:07 -0800197# define TASK_UNMAPPED_BASE 0x40000000
bellarda03e2d42007-11-14 11:29:07 +0000198#endif
Peter Maydell59e9d912012-03-08 14:40:33 +0000199abi_ulong mmap_next_start = TASK_UNMAPPED_BASE;
bellarda03e2d42007-11-14 11:29:07 +0000200
pbrook07765902008-05-31 16:33:53 +0000201unsigned long last_brk;
202
Paul Brook68a1c812010-05-29 02:27:35 +0100203/* Subroutine of mmap_find_vma, used when we have pre-allocated a chunk
204 of guest address space. */
205static abi_ulong mmap_find_vma_reserved(abi_ulong start, abi_ulong size)
206{
207 abi_ulong addr;
Peter Maydell59e9d912012-03-08 14:40:33 +0000208 abi_ulong end_addr;
Paul Brook68a1c812010-05-29 02:27:35 +0100209 int prot;
210 int looped = 0;
211
Laurent Vivierb76f21a2015-08-24 14:53:54 +0200212 if (size > reserved_va) {
Paul Brook68a1c812010-05-29 02:27:35 +0100213 return (abi_ulong)-1;
214 }
215
Peter Maydell59e9d912012-03-08 14:40:33 +0000216 size = HOST_PAGE_ALIGN(size);
217 end_addr = start + size;
Laurent Vivierb76f21a2015-08-24 14:53:54 +0200218 if (end_addr > reserved_va) {
219 end_addr = reserved_va;
Peter Maydell59e9d912012-03-08 14:40:33 +0000220 }
221 addr = end_addr - qemu_host_page_size;
222
223 while (1) {
224 if (addr > end_addr) {
Paul Brook68a1c812010-05-29 02:27:35 +0100225 if (looped) {
226 return (abi_ulong)-1;
227 }
Laurent Vivierb76f21a2015-08-24 14:53:54 +0200228 end_addr = reserved_va;
Peter Maydell59e9d912012-03-08 14:40:33 +0000229 addr = end_addr - qemu_host_page_size;
Paul Brook68a1c812010-05-29 02:27:35 +0100230 looped = 1;
231 continue;
232 }
233 prot = page_get_flags(addr);
234 if (prot) {
Peter Maydell59e9d912012-03-08 14:40:33 +0000235 end_addr = addr;
Paul Brook68a1c812010-05-29 02:27:35 +0100236 }
Peter Maydell59e9d912012-03-08 14:40:33 +0000237 if (addr + size == end_addr) {
238 break;
239 }
240 addr -= qemu_host_page_size;
Paul Brook68a1c812010-05-29 02:27:35 +0100241 }
Peter Maydell59e9d912012-03-08 14:40:33 +0000242
243 if (start == mmap_next_start) {
244 mmap_next_start = addr;
245 }
246
247 return addr;
Paul Brook68a1c812010-05-29 02:27:35 +0100248}
249
Kirill A. Shutemovfe3b4152009-08-13 21:03:58 +0300250/*
251 * Find and reserve a free memory area of size 'size'. The search
252 * starts at 'start'.
253 * It must be called with mmap_lock() held.
254 * Return -1 if error.
255 */
Riku Voipio9ad197d2009-04-21 17:23:23 +0300256abi_ulong mmap_find_vma(abi_ulong start, abi_ulong size)
bellarda03e2d42007-11-14 11:29:07 +0000257{
Richard Henderson14f24e12010-03-10 15:39:07 -0800258 void *ptr, *prev;
Kirill A. Shutemovfe3b4152009-08-13 21:03:58 +0300259 abi_ulong addr;
Richard Henderson14f24e12010-03-10 15:39:07 -0800260 int wrapped, repeat;
Kirill A. Shutemovfe3b4152009-08-13 21:03:58 +0300261
262 /* If 'start' == 0, then a default start address is used. */
Richard Henderson14f24e12010-03-10 15:39:07 -0800263 if (start == 0) {
Kirill A. Shutemovfe3b4152009-08-13 21:03:58 +0300264 start = mmap_next_start;
Richard Henderson14f24e12010-03-10 15:39:07 -0800265 } else {
266 start &= qemu_host_page_mask;
267 }
268
269 size = HOST_PAGE_ALIGN(size);
Kirill A. Shutemovfe3b4152009-08-13 21:03:58 +0300270
Laurent Vivierb76f21a2015-08-24 14:53:54 +0200271 if (reserved_va) {
Paul Brook68a1c812010-05-29 02:27:35 +0100272 return mmap_find_vma_reserved(start, size);
273 }
274
bellarda03e2d42007-11-14 11:29:07 +0000275 addr = start;
Richard Henderson14f24e12010-03-10 15:39:07 -0800276 wrapped = repeat = 0;
277 prev = 0;
Kirill A. Shutemovfe3b4152009-08-13 21:03:58 +0300278
Richard Henderson14f24e12010-03-10 15:39:07 -0800279 for (;; prev = ptr) {
Kirill A. Shutemovfe3b4152009-08-13 21:03:58 +0300280 /*
281 * Reserve needed memory area to avoid a race.
282 * It should be discarded using:
283 * - mmap() with MAP_FIXED flag
284 * - mremap() with MREMAP_FIXED flag
285 * - shmat() with SHM_REMAP flag
286 */
Richard Henderson14f24e12010-03-10 15:39:07 -0800287 ptr = mmap(g2h(addr), size, PROT_NONE,
Kirill A. Shutemovfe3b4152009-08-13 21:03:58 +0300288 MAP_ANONYMOUS|MAP_PRIVATE|MAP_NORESERVE, -1, 0);
289
290 /* ENOMEM, if host address space has no memory */
Richard Henderson14f24e12010-03-10 15:39:07 -0800291 if (ptr == MAP_FAILED) {
Kirill A. Shutemovfe3b4152009-08-13 21:03:58 +0300292 return (abi_ulong)-1;
Richard Henderson14f24e12010-03-10 15:39:07 -0800293 }
Kirill A. Shutemovfe3b4152009-08-13 21:03:58 +0300294
Richard Henderson14f24e12010-03-10 15:39:07 -0800295 /* Count the number of sequential returns of the same address.
296 This is used to modify the search algorithm below. */
297 repeat = (ptr == prev ? repeat + 1 : 0);
Kirill A. Shutemovfe3b4152009-08-13 21:03:58 +0300298
Richard Henderson14f24e12010-03-10 15:39:07 -0800299 if (h2g_valid(ptr + size - 1)) {
300 addr = h2g(ptr);
301
302 if ((addr & ~TARGET_PAGE_MASK) == 0) {
303 /* Success. */
304 if (start == mmap_next_start && addr >= TASK_UNMAPPED_BASE) {
305 mmap_next_start = addr + size;
306 }
307 return addr;
308 }
309
310 /* The address is not properly aligned for the target. */
311 switch (repeat) {
312 case 0:
313 /* Assume the result that the kernel gave us is the
314 first with enough free space, so start again at the
315 next higher target page. */
316 addr = TARGET_PAGE_ALIGN(addr);
317 break;
318 case 1:
319 /* Sometimes the kernel decides to perform the allocation
320 at the top end of memory instead. */
321 addr &= TARGET_PAGE_MASK;
322 break;
323 case 2:
324 /* Start over at low memory. */
325 addr = 0;
326 break;
327 default:
328 /* Fail. This unaligned block must the last. */
329 addr = -1;
330 break;
331 }
332 } else {
333 /* Since the result the kernel gave didn't fit, start
334 again at low memory. If any repetition, fail. */
335 addr = (repeat ? -1 : 0);
336 }
337
338 /* Unmap and try again. */
Kirill A. Shutemovfe3b4152009-08-13 21:03:58 +0300339 munmap(ptr, size);
Kirill A. Shutemovfe3b4152009-08-13 21:03:58 +0300340
Richard Henderson14f24e12010-03-10 15:39:07 -0800341 /* ENOMEM if we checked the whole of the target address space. */
Blue Swirld0b3e4f2010-09-18 05:53:14 +0000342 if (addr == (abi_ulong)-1) {
bellarda03e2d42007-11-14 11:29:07 +0000343 return (abi_ulong)-1;
Richard Henderson14f24e12010-03-10 15:39:07 -0800344 } else if (addr == 0) {
345 if (wrapped) {
346 return (abi_ulong)-1;
347 }
348 wrapped = 1;
349 /* Don't actually use 0 when wrapping, instead indicate
Stefan Weil8186e782011-04-28 17:20:41 +0200350 that we'd truly like an allocation in low memory. */
Richard Henderson14f24e12010-03-10 15:39:07 -0800351 addr = (mmap_min_addr > TARGET_PAGE_SIZE
352 ? TARGET_PAGE_ALIGN(mmap_min_addr)
353 : TARGET_PAGE_SIZE);
354 } else if (wrapped && addr >= start) {
355 return (abi_ulong)-1;
356 }
bellarda03e2d42007-11-14 11:29:07 +0000357 }
bellarda03e2d42007-11-14 11:29:07 +0000358}
359
bellard54936002003-05-13 00:25:15 +0000360/* NOTE: all the constants are the HOST ones */
blueswir1992f48a2007-10-14 16:27:31 +0000361abi_long target_mmap(abi_ulong start, abi_ulong len, int prot,
362 int flags, int fd, abi_ulong offset)
bellard54936002003-05-13 00:25:15 +0000363{
blueswir1992f48a2007-10-14 16:27:31 +0000364 abi_ulong ret, end, real_start, real_end, retaddr, host_offset, host_len;
bellard54936002003-05-13 00:25:15 +0000365
pbrookc8a706f2008-06-02 16:16:42 +0000366 mmap_lock();
bellard54936002003-05-13 00:25:15 +0000367#ifdef DEBUG_MMAP
368 {
Blue Swirl0bf9e312009-07-20 17:19:25 +0000369 printf("mmap: start=0x" TARGET_ABI_FMT_lx
370 " len=0x" TARGET_ABI_FMT_lx " prot=%c%c%c flags=",
ths5fafdf22007-09-16 21:08:06 +0000371 start, len,
bellard54936002003-05-13 00:25:15 +0000372 prot & PROT_READ ? 'r' : '-',
373 prot & PROT_WRITE ? 'w' : '-',
374 prot & PROT_EXEC ? 'x' : '-');
375 if (flags & MAP_FIXED)
376 printf("MAP_FIXED ");
377 if (flags & MAP_ANONYMOUS)
378 printf("MAP_ANON ");
379 switch(flags & MAP_TYPE) {
380 case MAP_PRIVATE:
381 printf("MAP_PRIVATE ");
382 break;
383 case MAP_SHARED:
384 printf("MAP_SHARED ");
385 break;
386 default:
387 printf("[MAP_TYPE=0x%x] ", flags & MAP_TYPE);
388 break;
389 }
Blue Swirl0bf9e312009-07-20 17:19:25 +0000390 printf("fd=%d offset=" TARGET_ABI_FMT_lx "\n", fd, offset);
bellard54936002003-05-13 00:25:15 +0000391 }
392#endif
393
pbrooke89f07d2006-02-04 20:46:24 +0000394 if (offset & ~TARGET_PAGE_MASK) {
395 errno = EINVAL;
pbrookc8a706f2008-06-02 16:16:42 +0000396 goto fail;
pbrooke89f07d2006-02-04 20:46:24 +0000397 }
bellard54936002003-05-13 00:25:15 +0000398
399 len = TARGET_PAGE_ALIGN(len);
400 if (len == 0)
pbrookc8a706f2008-06-02 16:16:42 +0000401 goto the_end;
pbrook53a59602006-03-25 19:31:22 +0000402 real_start = start & qemu_host_page_mask;
Richard Hendersona5e7ee42012-06-01 16:07:52 -0700403 host_offset = offset & qemu_host_page_mask;
404
405 /* If the user is asking for the kernel to find a location, do that
406 before we truncate the length for mapping files below. */
407 if (!(flags & MAP_FIXED)) {
408 host_len = len + offset - host_offset;
409 host_len = HOST_PAGE_ALIGN(host_len);
410 start = mmap_find_vma(real_start, host_len);
411 if (start == (abi_ulong)-1) {
412 errno = ENOMEM;
413 goto fail;
414 }
415 }
bellard54936002003-05-13 00:25:15 +0000416
edgar_igl54c5a2a2009-02-03 23:06:34 +0000417 /* When mapping files into a memory area larger than the file, accesses
418 to pages beyond the file size will cause a SIGBUS.
419
420 For example, if mmaping a file of 100 bytes on a host with 4K pages
421 emulating a target with 8K pages, the target expects to be able to
422 access the first 8K. But the host will trap us on any access beyond
423 4K.
424
425 When emulating a target with a larger page-size than the hosts, we
426 may need to truncate file maps at EOF and add extra anonymous pages
427 up to the targets page boundary. */
428
Marc-André Lureau35f2fd02017-01-19 10:15:33 -0500429 if ((qemu_real_host_page_size < qemu_host_page_size) &&
430 !(flags & MAP_ANONYMOUS)) {
431 struct stat sb;
edgar_igl54c5a2a2009-02-03 23:06:34 +0000432
433 if (fstat (fd, &sb) == -1)
434 goto fail;
435
436 /* Are we trying to create a map beyond EOF?. */
437 if (offset + len > sb.st_size) {
438 /* If so, truncate the file map at eof aligned with
439 the hosts real pagesize. Additional anonymous maps
440 will be created beyond EOF. */
Paolo Bonzini0c2d70c2015-12-02 13:00:54 +0100441 len = REAL_HOST_PAGE_ALIGN(sb.st_size - offset);
edgar_igl54c5a2a2009-02-03 23:06:34 +0000442 }
443 }
444
bellard54936002003-05-13 00:25:15 +0000445 if (!(flags & MAP_FIXED)) {
Richard Hendersona5e7ee42012-06-01 16:07:52 -0700446 unsigned long host_start;
bellarda03e2d42007-11-14 11:29:07 +0000447 void *p;
Richard Hendersona5e7ee42012-06-01 16:07:52 -0700448
bellarda03e2d42007-11-14 11:29:07 +0000449 host_len = len + offset - host_offset;
450 host_len = HOST_PAGE_ALIGN(host_len);
Richard Hendersona5e7ee42012-06-01 16:07:52 -0700451
bellarda03e2d42007-11-14 11:29:07 +0000452 /* Note: we prefer to control the mapping address. It is
453 especially important if qemu_host_page_size >
454 qemu_real_host_page_size */
Richard Hendersona5e7ee42012-06-01 16:07:52 -0700455 p = mmap(g2h(start), host_len, prot,
456 flags | MAP_FIXED | MAP_ANONYMOUS, -1, 0);
bellarda03e2d42007-11-14 11:29:07 +0000457 if (p == MAP_FAILED)
pbrookc8a706f2008-06-02 16:16:42 +0000458 goto fail;
bellarda03e2d42007-11-14 11:29:07 +0000459 /* update start so that it points to the file position at 'offset' */
460 host_start = (unsigned long)p;
edgar_igl54c5a2a2009-02-03 23:06:34 +0000461 if (!(flags & MAP_ANONYMOUS)) {
Richard Hendersona5e7ee42012-06-01 16:07:52 -0700462 p = mmap(g2h(start), len, prot,
edgar_igl54c5a2a2009-02-03 23:06:34 +0000463 flags | MAP_FIXED, fd, host_offset);
JĂ¼rg Billeter83842742013-06-29 11:41:32 +0200464 if (p == MAP_FAILED) {
465 munmap(g2h(start), host_len);
466 goto fail;
467 }
bellarda03e2d42007-11-14 11:29:07 +0000468 host_start += offset - host_offset;
edgar_igl54c5a2a2009-02-03 23:06:34 +0000469 }
bellarda03e2d42007-11-14 11:29:07 +0000470 start = h2g(host_start);
471 } else {
472 if (start & ~TARGET_PAGE_MASK) {
pbrooke89f07d2006-02-04 20:46:24 +0000473 errno = EINVAL;
pbrookc8a706f2008-06-02 16:16:42 +0000474 goto fail;
pbrooke89f07d2006-02-04 20:46:24 +0000475 }
bellarda03e2d42007-11-14 11:29:07 +0000476 end = start + len;
477 real_end = HOST_PAGE_ALIGN(end);
balrog7ab240a2008-04-26 12:17:34 +0000478
aurel3245bc1f52008-12-08 18:12:33 +0000479 /*
480 * Test if requested memory area fits target address space
481 * It can fail only on 64-bit host with 32-bit target.
482 * On any other target/host host mmap() handles this error correctly.
483 */
484 if ((unsigned long)start + len - 1 > (abi_ulong) -1) {
485 errno = EINVAL;
486 goto fail;
487 }
488
bellarda03e2d42007-11-14 11:29:07 +0000489 /* worst case: we cannot map the file because the offset is not
490 aligned, so we read it */
491 if (!(flags & MAP_ANONYMOUS) &&
492 (offset & ~qemu_host_page_mask) != (start & ~qemu_host_page_mask)) {
493 /* msync() won't work here, so we return an error if write is
494 possible while it is a shared mapping */
495 if ((flags & MAP_TYPE) == MAP_SHARED &&
496 (prot & PROT_WRITE)) {
497 errno = EINVAL;
pbrookc8a706f2008-06-02 16:16:42 +0000498 goto fail;
bellarda03e2d42007-11-14 11:29:07 +0000499 }
500 retaddr = target_mmap(start, len, prot | PROT_WRITE,
501 MAP_FIXED | MAP_PRIVATE | MAP_ANONYMOUS,
502 -1, 0);
503 if (retaddr == -1)
pbrookc8a706f2008-06-02 16:16:42 +0000504 goto fail;
Kirill A. Shutemovfb7e3782010-01-20 00:56:20 +0100505 if (pread(fd, g2h(start), len, offset) == -1)
506 goto fail;
bellarda03e2d42007-11-14 11:29:07 +0000507 if (!(prot & PROT_WRITE)) {
508 ret = target_mprotect(start, len, prot);
Paolo Bonzini86abac02015-09-14 12:31:44 +0200509 assert(ret == 0);
bellarda03e2d42007-11-14 11:29:07 +0000510 }
511 goto the_end;
bellard54936002003-05-13 00:25:15 +0000512 }
bellarda03e2d42007-11-14 11:29:07 +0000513
514 /* handle the start of the mapping */
515 if (start > real_start) {
516 if (real_end == real_start + qemu_host_page_size) {
517 /* one single host page */
518 ret = mmap_frag(real_start, start, end,
519 prot, flags, fd, offset);
520 if (ret == -1)
pbrookc8a706f2008-06-02 16:16:42 +0000521 goto fail;
bellarda03e2d42007-11-14 11:29:07 +0000522 goto the_end1;
523 }
524 ret = mmap_frag(real_start, start, real_start + qemu_host_page_size,
bellard54936002003-05-13 00:25:15 +0000525 prot, flags, fd, offset);
526 if (ret == -1)
pbrookc8a706f2008-06-02 16:16:42 +0000527 goto fail;
bellarda03e2d42007-11-14 11:29:07 +0000528 real_start += qemu_host_page_size;
bellard54936002003-05-13 00:25:15 +0000529 }
bellarda03e2d42007-11-14 11:29:07 +0000530 /* handle the end of the mapping */
531 if (end < real_end) {
532 ret = mmap_frag(real_end - qemu_host_page_size,
Chen Gang530c0032015-12-24 09:07:33 +0800533 real_end - qemu_host_page_size, end,
bellarda03e2d42007-11-14 11:29:07 +0000534 prot, flags, fd,
535 offset + real_end - qemu_host_page_size - start);
536 if (ret == -1)
pbrookc8a706f2008-06-02 16:16:42 +0000537 goto fail;
bellarda03e2d42007-11-14 11:29:07 +0000538 real_end -= qemu_host_page_size;
539 }
ths3b46e622007-09-17 08:09:54 +0000540
bellarda03e2d42007-11-14 11:29:07 +0000541 /* map the middle (easier) */
542 if (real_start < real_end) {
543 void *p;
544 unsigned long offset1;
545 if (flags & MAP_ANONYMOUS)
546 offset1 = 0;
547 else
548 offset1 = offset + real_start - start;
549 p = mmap(g2h(real_start), real_end - real_start,
550 prot, flags, fd, offset1);
551 if (p == MAP_FAILED)
pbrookc8a706f2008-06-02 16:16:42 +0000552 goto fail;
bellarda03e2d42007-11-14 11:29:07 +0000553 }
bellard54936002003-05-13 00:25:15 +0000554 }
555 the_end1:
556 page_set_flags(start, start + len, prot | PAGE_VALID);
557 the_end:
558#ifdef DEBUG_MMAP
Blue Swirl0bf9e312009-07-20 17:19:25 +0000559 printf("ret=0x" TARGET_ABI_FMT_lx "\n", start);
bellard54936002003-05-13 00:25:15 +0000560 page_dump(stdout);
561 printf("\n");
562#endif
Paolo Bonzini35865332015-04-22 14:20:35 +0200563 tb_invalidate_phys_range(start, start + len);
pbrookc8a706f2008-06-02 16:16:42 +0000564 mmap_unlock();
bellard54936002003-05-13 00:25:15 +0000565 return start;
pbrookc8a706f2008-06-02 16:16:42 +0000566fail:
567 mmap_unlock();
568 return -1;
bellard54936002003-05-13 00:25:15 +0000569}
570
Paul Brook68a1c812010-05-29 02:27:35 +0100571static void mmap_reserve(abi_ulong start, abi_ulong size)
572{
573 abi_ulong real_start;
574 abi_ulong real_end;
575 abi_ulong addr;
576 abi_ulong end;
577 int prot;
578
579 real_start = start & qemu_host_page_mask;
580 real_end = HOST_PAGE_ALIGN(start + size);
581 end = start + size;
582 if (start > real_start) {
583 /* handle host page containing start */
584 prot = 0;
585 for (addr = real_start; addr < start; addr += TARGET_PAGE_SIZE) {
586 prot |= page_get_flags(addr);
587 }
588 if (real_end == real_start + qemu_host_page_size) {
589 for (addr = end; addr < real_end; addr += TARGET_PAGE_SIZE) {
590 prot |= page_get_flags(addr);
591 }
592 end = real_end;
593 }
594 if (prot != 0)
595 real_start += qemu_host_page_size;
596 }
597 if (end < real_end) {
598 prot = 0;
599 for (addr = end; addr < real_end; addr += TARGET_PAGE_SIZE) {
600 prot |= page_get_flags(addr);
601 }
602 if (prot != 0)
603 real_end -= qemu_host_page_size;
604 }
605 if (real_start != real_end) {
606 mmap(g2h(real_start), real_end - real_start, PROT_NONE,
607 MAP_FIXED | MAP_ANONYMOUS | MAP_PRIVATE | MAP_NORESERVE,
608 -1, 0);
609 }
610}
611
blueswir1992f48a2007-10-14 16:27:31 +0000612int target_munmap(abi_ulong start, abi_ulong len)
bellard54936002003-05-13 00:25:15 +0000613{
blueswir1992f48a2007-10-14 16:27:31 +0000614 abi_ulong end, real_start, real_end, addr;
bellard54936002003-05-13 00:25:15 +0000615 int prot, ret;
616
617#ifdef DEBUG_MMAP
Blue Swirl0bf9e312009-07-20 17:19:25 +0000618 printf("munmap: start=0x" TARGET_ABI_FMT_lx " len=0x"
619 TARGET_ABI_FMT_lx "\n",
620 start, len);
bellard54936002003-05-13 00:25:15 +0000621#endif
622 if (start & ~TARGET_PAGE_MASK)
623 return -EINVAL;
624 len = TARGET_PAGE_ALIGN(len);
625 if (len == 0)
626 return -EINVAL;
pbrookc8a706f2008-06-02 16:16:42 +0000627 mmap_lock();
bellard54936002003-05-13 00:25:15 +0000628 end = start + len;
pbrook53a59602006-03-25 19:31:22 +0000629 real_start = start & qemu_host_page_mask;
630 real_end = HOST_PAGE_ALIGN(end);
bellard54936002003-05-13 00:25:15 +0000631
pbrook53a59602006-03-25 19:31:22 +0000632 if (start > real_start) {
bellard54936002003-05-13 00:25:15 +0000633 /* handle host page containing start */
634 prot = 0;
pbrook53a59602006-03-25 19:31:22 +0000635 for(addr = real_start; addr < start; addr += TARGET_PAGE_SIZE) {
bellard54936002003-05-13 00:25:15 +0000636 prot |= page_get_flags(addr);
637 }
pbrook53a59602006-03-25 19:31:22 +0000638 if (real_end == real_start + qemu_host_page_size) {
639 for(addr = end; addr < real_end; addr += TARGET_PAGE_SIZE) {
bellardd418c812003-05-13 00:57:50 +0000640 prot |= page_get_flags(addr);
641 }
pbrook53a59602006-03-25 19:31:22 +0000642 end = real_end;
bellardd418c812003-05-13 00:57:50 +0000643 }
bellard54936002003-05-13 00:25:15 +0000644 if (prot != 0)
pbrook53a59602006-03-25 19:31:22 +0000645 real_start += qemu_host_page_size;
bellard54936002003-05-13 00:25:15 +0000646 }
pbrook53a59602006-03-25 19:31:22 +0000647 if (end < real_end) {
bellard54936002003-05-13 00:25:15 +0000648 prot = 0;
pbrook53a59602006-03-25 19:31:22 +0000649 for(addr = end; addr < real_end; addr += TARGET_PAGE_SIZE) {
bellard54936002003-05-13 00:25:15 +0000650 prot |= page_get_flags(addr);
651 }
652 if (prot != 0)
pbrook53a59602006-03-25 19:31:22 +0000653 real_end -= qemu_host_page_size;
bellard54936002003-05-13 00:25:15 +0000654 }
ths3b46e622007-09-17 08:09:54 +0000655
pbrookc8a706f2008-06-02 16:16:42 +0000656 ret = 0;
bellard54936002003-05-13 00:25:15 +0000657 /* unmap what we can */
pbrook53a59602006-03-25 19:31:22 +0000658 if (real_start < real_end) {
Laurent Vivierb76f21a2015-08-24 14:53:54 +0200659 if (reserved_va) {
Paul Brook68a1c812010-05-29 02:27:35 +0100660 mmap_reserve(real_start, real_end - real_start);
661 } else {
662 ret = munmap(g2h(real_start), real_end - real_start);
663 }
bellard54936002003-05-13 00:25:15 +0000664 }
665
Alexander Graf77a8f1a2012-05-10 22:40:10 +0000666 if (ret == 0) {
pbrookc8a706f2008-06-02 16:16:42 +0000667 page_set_flags(start, start + len, 0);
Paolo Bonzini35865332015-04-22 14:20:35 +0200668 tb_invalidate_phys_range(start, start + len);
Alexander Graf77a8f1a2012-05-10 22:40:10 +0000669 }
pbrookc8a706f2008-06-02 16:16:42 +0000670 mmap_unlock();
671 return ret;
bellard54936002003-05-13 00:25:15 +0000672}
673
blueswir1992f48a2007-10-14 16:27:31 +0000674abi_long target_mremap(abi_ulong old_addr, abi_ulong old_size,
675 abi_ulong new_size, unsigned long flags,
676 abi_ulong new_addr)
bellard54936002003-05-13 00:25:15 +0000677{
678 int prot;
aurel32f19412a2008-12-08 18:12:40 +0000679 void *host_addr;
bellard54936002003-05-13 00:25:15 +0000680
pbrookc8a706f2008-06-02 16:16:42 +0000681 mmap_lock();
aurel32f19412a2008-12-08 18:12:40 +0000682
Paul Brook68a1c812010-05-29 02:27:35 +0100683 if (flags & MREMAP_FIXED) {
Felix Janda52956a92016-09-30 19:39:27 -0400684 host_addr = mremap(g2h(old_addr), old_size, new_size,
685 flags, g2h(new_addr));
Paul Brook68a1c812010-05-29 02:27:35 +0100686
Laurent Vivierb76f21a2015-08-24 14:53:54 +0200687 if (reserved_va && host_addr != MAP_FAILED) {
Paul Brook68a1c812010-05-29 02:27:35 +0100688 /* If new and old addresses overlap then the above mremap will
689 already have failed with EINVAL. */
690 mmap_reserve(old_addr, old_size);
691 }
692 } else if (flags & MREMAP_MAYMOVE) {
aurel32f19412a2008-12-08 18:12:40 +0000693 abi_ulong mmap_start;
694
695 mmap_start = mmap_find_vma(0, new_size);
696
697 if (mmap_start == -1) {
698 errno = ENOMEM;
699 host_addr = MAP_FAILED;
Paul Brook68a1c812010-05-29 02:27:35 +0100700 } else {
Felix Janda52956a92016-09-30 19:39:27 -0400701 host_addr = mremap(g2h(old_addr), old_size, new_size,
702 flags | MREMAP_FIXED, g2h(mmap_start));
Laurent Vivierb76f21a2015-08-24 14:53:54 +0200703 if (reserved_va) {
amateurc65ffe62010-09-14 13:22:34 +0800704 mmap_reserve(old_addr, old_size);
705 }
Paul Brook68a1c812010-05-29 02:27:35 +0100706 }
blueswir13af72a42008-12-15 17:58:49 +0000707 } else {
Paul Brook68a1c812010-05-29 02:27:35 +0100708 int prot = 0;
Laurent Vivierb76f21a2015-08-24 14:53:54 +0200709 if (reserved_va && old_size < new_size) {
Paul Brook68a1c812010-05-29 02:27:35 +0100710 abi_ulong addr;
711 for (addr = old_addr + old_size;
712 addr < old_addr + new_size;
713 addr++) {
714 prot |= page_get_flags(addr);
715 }
716 }
717 if (prot == 0) {
718 host_addr = mremap(g2h(old_addr), old_size, new_size, flags);
Laurent Vivierb76f21a2015-08-24 14:53:54 +0200719 if (host_addr != MAP_FAILED && reserved_va && old_size > new_size) {
Paul Brook68a1c812010-05-29 02:27:35 +0100720 mmap_reserve(old_addr + old_size, new_size - old_size);
721 }
722 } else {
723 errno = ENOMEM;
724 host_addr = MAP_FAILED;
725 }
aurel32f19412a2008-12-08 18:12:40 +0000726 /* Check if address fits target address space */
727 if ((unsigned long)host_addr + new_size > (abi_ulong)-1) {
728 /* Revert mremap() changes */
729 host_addr = mremap(g2h(old_addr), new_size, old_size, flags);
730 errno = ENOMEM;
731 host_addr = MAP_FAILED;
732 }
733 }
734
735 if (host_addr == MAP_FAILED) {
pbrookc8a706f2008-06-02 16:16:42 +0000736 new_addr = -1;
737 } else {
738 new_addr = h2g(host_addr);
739 prot = page_get_flags(old_addr);
740 page_set_flags(old_addr, old_addr + old_size, 0);
741 page_set_flags(new_addr, new_addr + new_size, prot | PAGE_VALID);
742 }
Paolo Bonzini35865332015-04-22 14:20:35 +0200743 tb_invalidate_phys_range(new_addr, new_addr + new_size);
pbrookc8a706f2008-06-02 16:16:42 +0000744 mmap_unlock();
bellard54936002003-05-13 00:25:15 +0000745 return new_addr;
746}
747
blueswir1992f48a2007-10-14 16:27:31 +0000748int target_msync(abi_ulong start, abi_ulong len, int flags)
bellard54936002003-05-13 00:25:15 +0000749{
blueswir1992f48a2007-10-14 16:27:31 +0000750 abi_ulong end;
bellard54936002003-05-13 00:25:15 +0000751
752 if (start & ~TARGET_PAGE_MASK)
753 return -EINVAL;
754 len = TARGET_PAGE_ALIGN(len);
bellard54936002003-05-13 00:25:15 +0000755 end = start + len;
bellardd418c812003-05-13 00:57:50 +0000756 if (end < start)
757 return -EINVAL;
758 if (end == start)
759 return 0;
ths3b46e622007-09-17 08:09:54 +0000760
bellard83fb7ad2004-07-05 21:25:26 +0000761 start &= qemu_host_page_mask;
pbrook53a59602006-03-25 19:31:22 +0000762 return msync(g2h(start), end - start, flags);
bellard54936002003-05-13 00:25:15 +0000763}