blob: 5ed7d29183ec8c8cc1eb8f6a664a34c4b04b1476 [file] [log] [blame]
Michael S. Tsirkin794e8f32015-09-24 14:41:17 +03001/*
2 * Support for RAM backed by mmaped host memory.
3 *
4 * Copyright (c) 2015 Red Hat, Inc.
5 *
6 * Authors:
7 * Michael S. Tsirkin <mst@redhat.com>
8 *
9 * This work is licensed under the terms of the GNU GPL, version 2 or
10 * later. See the COPYING file in the top-level directory.
11 */
Markus Armbrustera9c94272016-06-22 19:11:19 +020012
Zhang Yi119906af2019-04-22 08:48:48 +080013#ifdef CONFIG_LINUX
14#include <linux/mman.h>
15#else /* !CONFIG_LINUX */
16#define MAP_SYNC 0x0
17#define MAP_SHARED_VALIDATE 0x0
18#endif /* CONFIG_LINUX */
19
Peter Maydellaafd7582016-01-29 17:49:55 +000020#include "qemu/osdep.h"
Markus Armbrustera9c94272016-06-22 19:11:19 +020021#include "qemu/mmap-alloc.h"
Cao jin4a3ecf22016-11-02 21:44:46 +080022#include "qemu/host-utils.h"
David Hildenbrandd94e0bc2021-05-10 13:43:22 +020023#include "qemu/cutils.h"
David Hildenbrand8dbe22c2021-05-10 13:43:21 +020024#include "qemu/error-report.h"
Michael S. Tsirkin794e8f32015-09-24 14:41:17 +030025
Michael S. Tsirkin7197fb42015-12-02 21:14:12 +020026#define HUGETLBFS_MAGIC 0x958458f6
27
28#ifdef CONFIG_LINUX
29#include <sys/vfs.h>
30#endif
31
32size_t qemu_fd_getpagesize(int fd)
33{
34#ifdef CONFIG_LINUX
35 struct statfs fs;
36 int ret;
37
38 if (fd != -1) {
39 do {
40 ret = fstatfs(fd, &fs);
41 } while (ret != 0 && errno == EINTR);
42
43 if (ret == 0 && fs.f_type == HUGETLBFS_MAGIC) {
44 return fs.f_bsize;
45 }
46 }
Peter Maydell57d1f6d2017-12-08 16:57:28 +000047#ifdef __sparc__
48 /* SPARC Linux needs greater alignment than the pagesize */
49 return QEMU_VMALLOC_ALIGN;
50#endif
Michael S. Tsirkin7197fb42015-12-02 21:14:12 +020051#endif
52
Marc-André Lureau8e3b0cb2022-03-23 19:57:22 +040053 return qemu_real_host_page_size();
Michael S. Tsirkin7197fb42015-12-02 21:14:12 +020054}
55
David Hildenbrandd94e0bc2021-05-10 13:43:22 +020056#define OVERCOMMIT_MEMORY_PATH "/proc/sys/vm/overcommit_memory"
57static bool map_noreserve_effective(int fd, uint32_t qemu_map_flags)
58{
59#if defined(__linux__)
60 const bool readonly = qemu_map_flags & QEMU_MAP_READONLY;
61 const bool shared = qemu_map_flags & QEMU_MAP_SHARED;
62 gchar *content = NULL;
63 const char *endptr;
64 unsigned int tmp;
65
66 /*
67 * hugeltb accounting is different than ordinary swap reservation:
68 * a) Hugetlb pages from the pool are reserved for both private and
69 * shared mappings. For shared mappings, all mappers have to specify
70 * MAP_NORESERVE.
71 * b) MAP_NORESERVE is not affected by /proc/sys/vm/overcommit_memory.
72 */
Marc-André Lureau8e3b0cb2022-03-23 19:57:22 +040073 if (qemu_fd_getpagesize(fd) != qemu_real_host_page_size()) {
David Hildenbrandd94e0bc2021-05-10 13:43:22 +020074 return true;
75 }
76
77 /*
78 * Accountable mappings in the kernel that can be affected by MAP_NORESEVE
79 * are private writable mappings (see mm/mmap.c:accountable_mapping() in
80 * Linux). For all shared or readonly mappings, MAP_NORESERVE is always
81 * implicitly active -- no reservation; this includes shmem. The only
82 * exception is shared anonymous memory, it is accounted like private
83 * anonymous memory.
84 */
85 if (readonly || (shared && fd >= 0)) {
86 return true;
87 }
88
89 /*
90 * MAP_NORESERVE is globally ignored for applicable !hugetlb mappings when
91 * memory overcommit is set to "never". Sparse memory regions aren't really
92 * possible in this system configuration.
93 *
94 * Bail out now instead of silently committing way more memory than
95 * currently desired by the user.
96 */
97 if (g_file_get_contents(OVERCOMMIT_MEMORY_PATH, &content, NULL, NULL) &&
98 !qemu_strtoui(content, &endptr, 0, &tmp) &&
99 (!endptr || *endptr == '\n')) {
100 if (tmp == 2) {
101 error_report("Skipping reservation of swap space is not supported:"
102 " \"" OVERCOMMIT_MEMORY_PATH "\" is \"2\"");
103 return false;
104 }
105 return true;
106 }
107 /* this interface has been around since Linux 2.6 */
108 error_report("Skipping reservation of swap space is not supported:"
109 " Could not read: \"" OVERCOMMIT_MEMORY_PATH "\"");
110 return false;
111#endif
112 /*
113 * E.g., FreeBSD used to define MAP_NORESERVE, never implemented it,
114 * and removed it a while ago.
115 */
116 error_report("Skipping reservation of swap space is not supported");
117 return false;
118}
119
David Hildenbrand01c26ad2021-05-10 13:43:15 +0200120/*
121 * Reserve a new memory region of the requested size to be used for mapping
122 * from the given fd (if any).
123 */
124static void *mmap_reserve(size_t size, int fd)
125{
126 int flags = MAP_PRIVATE;
127
128#if defined(__powerpc64__) && defined(__linux__)
129 /*
130 * On ppc64 mappings in the same segment (aka slice) must share the same
131 * page size. Since we will be re-allocating part of this segment
132 * from the supplied fd, we should make sure to use the same page size, to
133 * this end we mmap the supplied fd. In this case, set MAP_NORESERVE to
134 * avoid allocating backing store memory.
135 * We do this unless we are using the system page size, in which case
136 * anonymous memory is OK.
137 */
Marc-André Lureau8e3b0cb2022-03-23 19:57:22 +0400138 if (fd == -1 || qemu_fd_getpagesize(fd) == qemu_real_host_page_size()) {
David Hildenbrand01c26ad2021-05-10 13:43:15 +0200139 fd = -1;
140 flags |= MAP_ANONYMOUS;
141 } else {
142 flags |= MAP_NORESERVE;
143 }
144#else
145 fd = -1;
146 flags |= MAP_ANONYMOUS;
147#endif
148
149 return mmap(0, size, PROT_NONE, flags, fd, 0);
150}
151
David Hildenbrandd01cbf82021-05-10 13:43:16 +0200152/*
153 * Activate memory in a reserved region from the given fd (if any), to make
154 * it accessible.
155 */
David Hildenbrandb444f5c2021-05-10 13:43:20 +0200156static void *mmap_activate(void *ptr, size_t size, int fd,
157 uint32_t qemu_map_flags, off_t map_offset)
David Hildenbrandd01cbf82021-05-10 13:43:16 +0200158{
David Hildenbrand8dbe22c2021-05-10 13:43:21 +0200159 const bool noreserve = qemu_map_flags & QEMU_MAP_NORESERVE;
David Hildenbrandb444f5c2021-05-10 13:43:20 +0200160 const bool readonly = qemu_map_flags & QEMU_MAP_READONLY;
161 const bool shared = qemu_map_flags & QEMU_MAP_SHARED;
162 const bool sync = qemu_map_flags & QEMU_MAP_SYNC;
David Hildenbrandd01cbf82021-05-10 13:43:16 +0200163 const int prot = PROT_READ | (readonly ? 0 : PROT_WRITE);
164 int map_sync_flags = 0;
165 int flags = MAP_FIXED;
166 void *activated_ptr;
167
David Hildenbrandd94e0bc2021-05-10 13:43:22 +0200168 if (noreserve && !map_noreserve_effective(fd, qemu_map_flags)) {
David Hildenbrand8dbe22c2021-05-10 13:43:21 +0200169 return MAP_FAILED;
170 }
171
David Hildenbrandd01cbf82021-05-10 13:43:16 +0200172 flags |= fd == -1 ? MAP_ANONYMOUS : 0;
173 flags |= shared ? MAP_SHARED : MAP_PRIVATE;
David Hildenbrandd94e0bc2021-05-10 13:43:22 +0200174 flags |= noreserve ? MAP_NORESERVE : 0;
David Hildenbrandb444f5c2021-05-10 13:43:20 +0200175 if (shared && sync) {
David Hildenbrandd01cbf82021-05-10 13:43:16 +0200176 map_sync_flags = MAP_SYNC | MAP_SHARED_VALIDATE;
177 }
178
179 activated_ptr = mmap(ptr, size, prot, flags | map_sync_flags, fd,
180 map_offset);
181 if (activated_ptr == MAP_FAILED && map_sync_flags) {
182 if (errno == ENOTSUP) {
183 char *proc_link = g_strdup_printf("/proc/self/fd/%d", fd);
184 char *file_name = g_malloc0(PATH_MAX);
185 int len = readlink(proc_link, file_name, PATH_MAX - 1);
186
187 if (len < 0) {
188 len = 0;
189 }
190 file_name[len] = '\0';
191 fprintf(stderr, "Warning: requesting persistence across crashes "
192 "for backend file %s failed. Proceeding without "
193 "persistence, data might become corrupted in case of host "
194 "crash.\n", file_name);
195 g_free(proc_link);
196 g_free(file_name);
Igor Mammedovcdcf7662021-01-11 15:33:32 -0500197 warn_report("Using non DAX backing file with 'pmem=on' option"
198 " is deprecated");
David Hildenbrandd01cbf82021-05-10 13:43:16 +0200199 }
200 /*
201 * If mmap failed with MAP_SHARED_VALIDATE | MAP_SYNC, we will try
202 * again without these flags to handle backwards compatibility.
203 */
204 activated_ptr = mmap(ptr, size, prot, flags, fd, map_offset);
205 }
206 return activated_ptr;
207}
208
David Hildenbrandadad0b32021-05-10 13:43:14 +0200209static inline size_t mmap_guard_pagesize(int fd)
210{
211#if defined(__powerpc64__) && defined(__linux__)
212 /* Mappings in the same segment must share the same page size */
213 return qemu_fd_getpagesize(fd);
214#else
Marc-André Lureau8e3b0cb2022-03-23 19:57:22 +0400215 return qemu_real_host_page_size();
David Hildenbrandadad0b32021-05-10 13:43:14 +0200216#endif
217}
218
Zhang Yi2ac0f162019-02-08 18:10:37 +0800219void *qemu_ram_mmap(int fd,
220 size_t size,
221 size_t align,
David Hildenbrandb444f5c2021-05-10 13:43:20 +0200222 uint32_t qemu_map_flags,
Jagannathan Raman44a4ff32021-01-29 11:46:04 -0500223 off_t map_offset)
Michael S. Tsirkin794e8f32015-09-24 14:41:17 +0300224{
David Hildenbrandadad0b32021-05-10 13:43:14 +0200225 const size_t guard_pagesize = mmap_guard_pagesize(fd);
David Hildenbrandd01cbf82021-05-10 13:43:16 +0200226 size_t offset, total;
227 void *ptr, *guardptr;
Murilo Opsfelder Araujo2044c3e2019-01-30 21:36:04 -0200228
Michael S. Tsirkin794e8f32015-09-24 14:41:17 +0300229 /*
230 * Note: this always allocates at least one extra page of virtual address
231 * space, even if size is already aligned.
232 */
Murilo Opsfelder Araujo2044c3e2019-01-30 21:36:04 -0200233 total = size + align;
234
David Hildenbrand01c26ad2021-05-10 13:43:15 +0200235 guardptr = mmap_reserve(total, fd);
Murilo Opsfelder Araujo2044c3e2019-01-30 21:36:04 -0200236 if (guardptr == MAP_FAILED) {
Michael S. Tsirkin9d4ec932015-10-25 17:07:45 +0200237 return MAP_FAILED;
Michael S. Tsirkin794e8f32015-09-24 14:41:17 +0300238 }
239
Cao jin4a3ecf22016-11-02 21:44:46 +0800240 assert(is_power_of_2(align));
Michael S. Tsirkin794e8f32015-09-24 14:41:17 +0300241 /* Always align to host page size */
David Hildenbrandadad0b32021-05-10 13:43:14 +0200242 assert(align >= guard_pagesize);
Michael S. Tsirkin794e8f32015-09-24 14:41:17 +0300243
Murilo Opsfelder Araujo2044c3e2019-01-30 21:36:04 -0200244 offset = QEMU_ALIGN_UP((uintptr_t)guardptr, align) - (uintptr_t)guardptr;
245
David Hildenbrandb444f5c2021-05-10 13:43:20 +0200246 ptr = mmap_activate(guardptr + offset, size, fd, qemu_map_flags,
David Hildenbrandd01cbf82021-05-10 13:43:16 +0200247 map_offset);
Murilo Opsfelder Araujo2044c3e2019-01-30 21:36:04 -0200248 if (ptr == MAP_FAILED) {
249 munmap(guardptr, total);
Michael S. Tsirkin9d4ec932015-10-25 17:07:45 +0200250 return MAP_FAILED;
Michael S. Tsirkin794e8f32015-09-24 14:41:17 +0300251 }
252
Michael S. Tsirkin794e8f32015-09-24 14:41:17 +0300253 if (offset > 0) {
Murilo Opsfelder Araujo2044c3e2019-01-30 21:36:04 -0200254 munmap(guardptr, offset);
Michael S. Tsirkin794e8f32015-09-24 14:41:17 +0300255 }
256
257 /*
258 * Leave a single PROT_NONE page allocated after the RAM block, to serve as
259 * a guard page guarding against potential buffer overflows.
260 */
Cao jin6e4c8902016-11-02 21:44:47 +0800261 total -= offset;
David Hildenbrandadad0b32021-05-10 13:43:14 +0200262 if (total > size + guard_pagesize) {
263 munmap(ptr + size + guard_pagesize, total - size - guard_pagesize);
Michael S. Tsirkin794e8f32015-09-24 14:41:17 +0300264 }
265
Murilo Opsfelder Araujo2044c3e2019-01-30 21:36:04 -0200266 return ptr;
Michael S. Tsirkin794e8f32015-09-24 14:41:17 +0300267}
268
Murilo Opsfelder Araujo53adb9d2019-01-30 21:36:05 -0200269void qemu_ram_munmap(int fd, void *ptr, size_t size)
Michael S. Tsirkin794e8f32015-09-24 14:41:17 +0300270{
271 if (ptr) {
272 /* Unmap both the RAM block and the guard page */
David Hildenbrandadad0b32021-05-10 13:43:14 +0200273 munmap(ptr, size + mmap_guard_pagesize(fd));
Michael S. Tsirkin794e8f32015-09-24 14:41:17 +0300274 }
275}