blob: 55025cc366992b2a1d4a21b426edffe3e37f0ff6 [file] [log] [blame]
aliguori05330442008-11-05 16:29:27 +00001/*
2 * QEMU KVM support
3 *
4 * Copyright IBM, Corp. 2008
aliguori5832d1f2008-11-24 19:36:26 +00005 * Red Hat, Inc. 2008
aliguori05330442008-11-05 16:29:27 +00006 *
7 * Authors:
8 * Anthony Liguori <aliguori@us.ibm.com>
aliguori5832d1f2008-11-24 19:36:26 +00009 * Glauber Costa <gcosta@redhat.com>
aliguori05330442008-11-05 16:29:27 +000010 *
11 * This work is licensed under the terms of the GNU GPL, version 2 or later.
12 * See the COPYING file in the top-level directory.
13 *
14 */
15
16#include <sys/types.h>
17#include <sys/ioctl.h>
18#include <sys/mman.h>
aliguori984b5182008-11-13 19:21:00 +000019#include <stdarg.h>
aliguori05330442008-11-05 16:29:27 +000020
21#include <linux/kvm.h>
22
23#include "qemu-common.h"
Paolo Bonzini1de7afc2012-12-17 18:20:00 +010024#include "qemu/atomic.h"
25#include "qemu/option.h"
26#include "qemu/config-file.h"
Paolo Bonzini9c17d612012-12-17 18:20:04 +010027#include "sysemu/sysemu.h"
Eduardo Habkost782c3f22014-09-26 17:45:24 -030028#include "sysemu/accel.h"
Jan Kiszkad33a1812009-05-02 00:29:37 +020029#include "hw/hw.h"
Michael S. Tsirkina2cb15b2012-12-12 14:24:50 +020030#include "hw/pci/msi.h"
Cornelia Huckd426d9f2013-07-15 17:45:03 +020031#include "hw/s390x/adapter.h"
Paolo Bonzini022c62c2012-12-17 18:19:49 +010032#include "exec/gdbstub.h"
Paolo Bonzini9c17d612012-12-17 18:20:04 +010033#include "sysemu/kvm.h"
Paolo Bonzini1de7afc2012-12-17 18:20:00 +010034#include "qemu/bswap.h"
Paolo Bonzini022c62c2012-12-17 18:19:49 +010035#include "exec/memory.h"
Juan Quintela747afd52013-11-04 12:59:02 +010036#include "exec/ram_addr.h"
Paolo Bonzini022c62c2012-12-17 18:19:49 +010037#include "exec/address-spaces.h"
Paolo Bonzini1de7afc2012-12-17 18:20:00 +010038#include "qemu/event_notifier.h"
Kazuya Saito9c775722013-03-29 13:27:05 +090039#include "trace.h"
aliguori05330442008-11-05 16:29:27 +000040
Aneesh Kumar K.V135a1292013-12-23 21:10:40 +053041#include "hw/boards.h"
42
Stefan Hajnoczid2f2b8a2011-01-10 13:50:05 +020043/* This check must be after config-host.h is included */
44#ifdef CONFIG_EVENTFD
45#include <sys/eventfd.h>
46#endif
47
Stefan Weil93148aa2012-02-26 18:46:12 +010048/* KVM uses PAGE_SIZE in its definition of COALESCED_MMIO_MAX */
aliguorif65ed4c2008-12-09 20:09:57 +000049#define PAGE_SIZE TARGET_PAGE_SIZE
50
aliguori05330442008-11-05 16:29:27 +000051//#define DEBUG_KVM
52
53#ifdef DEBUG_KVM
Blue Swirl8c0d5772010-04-18 14:22:14 +000054#define DPRINTF(fmt, ...) \
aliguori05330442008-11-05 16:29:27 +000055 do { fprintf(stderr, fmt, ## __VA_ARGS__); } while (0)
56#else
Blue Swirl8c0d5772010-04-18 14:22:14 +000057#define DPRINTF(fmt, ...) \
aliguori05330442008-11-05 16:29:27 +000058 do { } while (0)
59#endif
60
Jan Kiszka04fa27f2012-05-16 15:41:10 -030061#define KVM_MSI_HASHTAB_SIZE 256
62
aliguori34fc6432008-11-19 17:41:58 +000063typedef struct KVMSlot
64{
Avi Kivitya8170e52012-10-23 12:30:10 +020065 hwaddr start_addr;
Anthony Liguoric227f092009-10-01 16:12:16 -050066 ram_addr_t memory_size;
Avi Kivity9f213ed2011-12-15 19:55:26 +020067 void *ram;
aliguori34fc6432008-11-19 17:41:58 +000068 int slot;
69 int flags;
70} KVMSlot;
aliguori05330442008-11-05 16:29:27 +000071
aliguori5832d1f2008-11-24 19:36:26 +000072typedef struct kvm_dirty_log KVMDirtyLog;
73
Paolo Bonzini9d1c35d2014-10-10 12:23:35 +020074struct KVMState
aliguori05330442008-11-05 16:29:27 +000075{
Eduardo Habkostfc020862014-09-26 17:45:32 -030076 AccelState parent_obj;
77
Alex Williamsonfb541ca2013-11-22 12:12:44 -070078 KVMSlot *slots;
79 int nr_slots;
aliguori05330442008-11-05 16:29:27 +000080 int fd;
81 int vmfd;
aliguorif65ed4c2008-12-09 20:09:57 +000082 int coalesced_mmio;
Sheng Yang62a27442010-01-26 19:21:16 +080083 struct kvm_coalesced_mmio_ring *coalesced_mmio_ring;
Avi Kivity1cae88b2011-10-18 19:43:12 +020084 bool coalesced_flush_in_progress;
Jan Kiszkae69917e2009-05-01 20:42:15 +020085 int broken_set_mem_region;
Jan Kiszka4495d6a2009-05-01 20:52:46 +020086 int migration_log;
Jan Kiszkaa0fb0022009-11-25 00:33:03 +010087 int vcpu_events;
Jan Kiszkab0b1d692010-03-01 19:10:29 +010088 int robust_singlestep;
Jan Kiszkaff44f1a2010-03-12 15:20:49 +010089 int debugregs;
aliguorie22a25c2009-03-12 20:12:48 +000090#ifdef KVM_CAP_SET_GUEST_DEBUG
91 struct kvm_sw_breakpoint_head kvm_sw_breakpoints;
92#endif
Jan Kiszka8a7c7392012-03-02 20:28:48 +010093 int pit_state2;
Sheng Yangf1665b22010-06-17 17:53:07 +080094 int xsave, xcrs;
Stefan Hajnoczid2f2b8a2011-01-10 13:50:05 +020095 int many_ioeventfds;
Jan Kiszka3ab73842012-08-27 08:28:39 +020096 int intx_set_mask;
David Gibson92e4b512012-03-07 14:41:09 +000097 /* The man page (and posix) say ioctl numbers are signed int, but
98 * they're not. Linux, glibc and *BSD all treat ioctl numbers as
99 * unsigned, and treating them as signed here can break things */
Jan Kiszkae333cd62012-08-24 13:34:47 +0200100 unsigned irq_set_ioctl;
James Hoganaed6efb2014-06-17 23:10:31 +0100101 unsigned int sigmask_len;
Jan Kiszka84b058d2011-10-15 11:49:47 +0200102#ifdef KVM_CAP_IRQ_ROUTING
103 struct kvm_irq_routing *irq_routes;
104 int nr_allocated_irq_routes;
105 uint32_t *used_gsi_bitmap;
Jan Kiszka4e2e4e62012-05-16 15:41:08 -0300106 unsigned int gsi_count;
Jan Kiszka04fa27f2012-05-16 15:41:10 -0300107 QTAILQ_HEAD(msi_hashtab, KVMMSIRoute) msi_hashtab[KVM_MSI_HASHTAB_SIZE];
Jan Kiszka4a3adeb2012-05-16 15:41:14 -0300108 bool direct_msi;
Jan Kiszka84b058d2011-10-15 11:49:47 +0200109#endif
Paolo Bonzini9d1c35d2014-10-10 12:23:35 +0200110};
aliguori05330442008-11-05 16:29:27 +0000111
Eduardo Habkost782c3f22014-09-26 17:45:24 -0300112#define TYPE_KVM_ACCEL ACCEL_CLASS_NAME("kvm")
113
Eduardo Habkostfc020862014-09-26 17:45:32 -0300114#define KVM_STATE(obj) \
115 OBJECT_CHECK(KVMState, (obj), TYPE_KVM_ACCEL)
116
Jan Kiszka6a7af8c2011-02-07 12:19:25 +0100117KVMState *kvm_state;
Jan Kiszka3d4b2642012-01-31 19:17:52 +0100118bool kvm_kernel_irqchip;
Peter Maydell7ae26bd2012-07-26 15:35:11 +0100119bool kvm_async_interrupts_allowed;
Alexander Graf215e79c2013-04-24 22:24:12 +0200120bool kvm_halt_in_kernel_allowed;
Nikolay Nikolaev69e03ae2014-05-27 15:03:35 +0300121bool kvm_eventfds_allowed;
Peter Maydellcc7e0dd2012-07-26 15:35:14 +0100122bool kvm_irqfds_allowed;
Eric Augerf41389a2014-10-31 13:38:18 +0000123bool kvm_resamplefds_allowed;
Peter Maydell614e41b2012-07-26 15:35:15 +0100124bool kvm_msi_via_irqfd_allowed;
Peter Maydellf3e1bed2012-07-26 15:35:16 +0100125bool kvm_gsi_routing_allowed;
Alexey Kardashevskiy76fe21d2013-09-03 18:08:25 +1000126bool kvm_gsi_direct_mapping;
Igor Mammedov13eed942013-04-23 10:29:36 +0200127bool kvm_allowed;
Jordan Justendf9c8b72013-05-29 01:27:25 -0700128bool kvm_readonly_mem_allowed;
Dominik Dingeld0a073a2015-03-12 13:53:49 +0100129bool kvm_vm_attributes_allowed;
aliguori05330442008-11-05 16:29:27 +0000130
Jan Kiszka94a8d392011-01-21 21:48:17 +0100131static const KVMCapabilityInfo kvm_required_capabilites[] = {
132 KVM_CAP_INFO(USER_MEMORY),
133 KVM_CAP_INFO(DESTROY_MEMORY_REGION_WORKS),
134 KVM_CAP_LAST_INFO
135};
136
Igor Mammedovb8865592014-10-31 16:38:32 +0000137static KVMSlot *kvm_get_free_slot(KVMState *s)
aliguori05330442008-11-05 16:29:27 +0000138{
139 int i;
140
Alex Williamsonfb541ca2013-11-22 12:12:44 -0700141 for (i = 0; i < s->nr_slots; i++) {
Jan Kiszkaa426e122011-01-04 09:32:13 +0100142 if (s->slots[i].memory_size == 0) {
aliguori05330442008-11-05 16:29:27 +0000143 return &s->slots[i];
Jan Kiszkaa426e122011-01-04 09:32:13 +0100144 }
aliguori05330442008-11-05 16:29:27 +0000145 }
146
Igor Mammedovb8865592014-10-31 16:38:32 +0000147 return NULL;
148}
149
150bool kvm_has_free_slot(MachineState *ms)
151{
152 return kvm_get_free_slot(KVM_STATE(ms->accelerator));
153}
154
155static KVMSlot *kvm_alloc_slot(KVMState *s)
156{
157 KVMSlot *slot = kvm_get_free_slot(s);
158
159 if (slot) {
160 return slot;
161 }
162
aliguorid3f8d372009-04-17 14:26:29 +0000163 fprintf(stderr, "%s: no free slot available\n", __func__);
164 abort();
165}
166
167static KVMSlot *kvm_lookup_matching_slot(KVMState *s,
Avi Kivitya8170e52012-10-23 12:30:10 +0200168 hwaddr start_addr,
169 hwaddr end_addr)
aliguorid3f8d372009-04-17 14:26:29 +0000170{
171 int i;
172
Alex Williamsonfb541ca2013-11-22 12:12:44 -0700173 for (i = 0; i < s->nr_slots; i++) {
aliguorid3f8d372009-04-17 14:26:29 +0000174 KVMSlot *mem = &s->slots[i];
175
176 if (start_addr == mem->start_addr &&
177 end_addr == mem->start_addr + mem->memory_size) {
178 return mem;
179 }
180 }
181
aliguori05330442008-11-05 16:29:27 +0000182 return NULL;
183}
184
aliguori6152e2a2009-04-17 14:26:33 +0000185/*
186 * Find overlapping slot with lowest start address
187 */
188static KVMSlot *kvm_lookup_overlapping_slot(KVMState *s,
Avi Kivitya8170e52012-10-23 12:30:10 +0200189 hwaddr start_addr,
190 hwaddr end_addr)
aliguori05330442008-11-05 16:29:27 +0000191{
aliguori6152e2a2009-04-17 14:26:33 +0000192 KVMSlot *found = NULL;
aliguori05330442008-11-05 16:29:27 +0000193 int i;
194
Alex Williamsonfb541ca2013-11-22 12:12:44 -0700195 for (i = 0; i < s->nr_slots; i++) {
aliguori05330442008-11-05 16:29:27 +0000196 KVMSlot *mem = &s->slots[i];
197
aliguori6152e2a2009-04-17 14:26:33 +0000198 if (mem->memory_size == 0 ||
199 (found && found->start_addr < mem->start_addr)) {
200 continue;
201 }
202
203 if (end_addr > mem->start_addr &&
204 start_addr < mem->start_addr + mem->memory_size) {
205 found = mem;
206 }
aliguori05330442008-11-05 16:29:27 +0000207 }
208
aliguori6152e2a2009-04-17 14:26:33 +0000209 return found;
aliguori05330442008-11-05 16:29:27 +0000210}
211
Avi Kivity9f213ed2011-12-15 19:55:26 +0200212int kvm_physical_memory_addr_from_host(KVMState *s, void *ram,
Avi Kivitya8170e52012-10-23 12:30:10 +0200213 hwaddr *phys_addr)
Huang Ying983dfc32010-10-11 15:31:20 -0300214{
215 int i;
216
Alex Williamsonfb541ca2013-11-22 12:12:44 -0700217 for (i = 0; i < s->nr_slots; i++) {
Huang Ying983dfc32010-10-11 15:31:20 -0300218 KVMSlot *mem = &s->slots[i];
219
Avi Kivity9f213ed2011-12-15 19:55:26 +0200220 if (ram >= mem->ram && ram < mem->ram + mem->memory_size) {
221 *phys_addr = mem->start_addr + (ram - mem->ram);
Huang Ying983dfc32010-10-11 15:31:20 -0300222 return 1;
223 }
224 }
225
226 return 0;
227}
228
aliguori5832d1f2008-11-24 19:36:26 +0000229static int kvm_set_user_memory_region(KVMState *s, KVMSlot *slot)
230{
231 struct kvm_userspace_memory_region mem;
232
233 mem.slot = slot->slot;
234 mem.guest_phys_addr = slot->start_addr;
Avi Kivity9f213ed2011-12-15 19:55:26 +0200235 mem.userspace_addr = (unsigned long)slot->ram;
aliguori5832d1f2008-11-24 19:36:26 +0000236 mem.flags = slot->flags;
Jan Kiszka4495d6a2009-05-01 20:52:46 +0200237 if (s->migration_log) {
238 mem.flags |= KVM_MEM_LOG_DIRTY_PAGES;
239 }
Xiao Guangrong651eb0f2013-05-31 16:52:18 +0800240
241 if (slot->memory_size && mem.flags & KVM_MEM_READONLY) {
Jordan Justen235e8982013-05-29 01:27:26 -0700242 /* Set the slot size to 0 before setting the slot to the desired
243 * value. This is needed based on KVM commit 75d61fbc. */
244 mem.memory_size = 0;
245 kvm_vm_ioctl(s, KVM_SET_USER_MEMORY_REGION, &mem);
246 }
247 mem.memory_size = slot->memory_size;
aliguori5832d1f2008-11-24 19:36:26 +0000248 return kvm_vm_ioctl(s, KVM_SET_USER_MEMORY_REGION, &mem);
249}
250
Andreas Färber504134d2012-12-17 06:38:45 +0100251int kvm_init_vcpu(CPUState *cpu)
aliguori05330442008-11-05 16:29:27 +0000252{
253 KVMState *s = kvm_state;
254 long mmap_size;
255 int ret;
256
Blue Swirl8c0d5772010-04-18 14:22:14 +0000257 DPRINTF("kvm_init_vcpu\n");
aliguori05330442008-11-05 16:29:27 +0000258
Eduardo Habkostb164e482013-01-22 18:25:01 -0200259 ret = kvm_vm_ioctl(s, KVM_CREATE_VCPU, (void *)kvm_arch_vcpu_id(cpu));
aliguori05330442008-11-05 16:29:27 +0000260 if (ret < 0) {
Blue Swirl8c0d5772010-04-18 14:22:14 +0000261 DPRINTF("kvm_create_vcpu failed\n");
aliguori05330442008-11-05 16:29:27 +0000262 goto err;
263 }
264
Andreas Färber8737c512012-10-31 05:29:00 +0100265 cpu->kvm_fd = ret;
Andreas Färbera60f24b2012-12-01 05:35:08 +0100266 cpu->kvm_state = s;
Andreas Färber20d695a2012-10-31 06:57:49 +0100267 cpu->kvm_vcpu_dirty = true;
aliguori05330442008-11-05 16:29:27 +0000268
269 mmap_size = kvm_ioctl(s, KVM_GET_VCPU_MMAP_SIZE, 0);
270 if (mmap_size < 0) {
Jan Kiszka748a6802011-02-01 22:15:48 +0100271 ret = mmap_size;
Blue Swirl8c0d5772010-04-18 14:22:14 +0000272 DPRINTF("KVM_GET_VCPU_MMAP_SIZE failed\n");
aliguori05330442008-11-05 16:29:27 +0000273 goto err;
274 }
275
Andreas Färberf7575c962012-12-01 06:18:14 +0100276 cpu->kvm_run = mmap(NULL, mmap_size, PROT_READ | PROT_WRITE, MAP_SHARED,
Andreas Färber8737c512012-10-31 05:29:00 +0100277 cpu->kvm_fd, 0);
Andreas Färberf7575c962012-12-01 06:18:14 +0100278 if (cpu->kvm_run == MAP_FAILED) {
aliguori05330442008-11-05 16:29:27 +0000279 ret = -errno;
Blue Swirl8c0d5772010-04-18 14:22:14 +0000280 DPRINTF("mmap'ing vcpu state failed\n");
aliguori05330442008-11-05 16:29:27 +0000281 goto err;
282 }
283
Jan Kiszkaa426e122011-01-04 09:32:13 +0100284 if (s->coalesced_mmio && !s->coalesced_mmio_ring) {
285 s->coalesced_mmio_ring =
Andreas Färberf7575c962012-12-01 06:18:14 +0100286 (void *)cpu->kvm_run + s->coalesced_mmio * PAGE_SIZE;
Jan Kiszkaa426e122011-01-04 09:32:13 +0100287 }
Sheng Yang62a27442010-01-26 19:21:16 +0800288
Andreas Färber20d695a2012-10-31 06:57:49 +0100289 ret = kvm_arch_init_vcpu(cpu);
aliguori05330442008-11-05 16:29:27 +0000290err:
291 return ret;
292}
293
aliguori5832d1f2008-11-24 19:36:26 +0000294/*
295 * dirty pages logging control
296 */
Michael S. Tsirkin25254bb2011-04-06 22:09:54 +0300297
Jordan Justen235e8982013-05-29 01:27:26 -0700298static int kvm_mem_flags(KVMState *s, bool log_dirty, bool readonly)
Michael S. Tsirkin25254bb2011-04-06 22:09:54 +0300299{
Jordan Justen235e8982013-05-29 01:27:26 -0700300 int flags = 0;
301 flags = log_dirty ? KVM_MEM_LOG_DIRTY_PAGES : 0;
302 if (readonly && kvm_readonly_mem_allowed) {
303 flags |= KVM_MEM_READONLY;
304 }
305 return flags;
Michael S. Tsirkin25254bb2011-04-06 22:09:54 +0300306}
307
308static int kvm_slot_dirty_pages_log_change(KVMSlot *mem, bool log_dirty)
aliguori5832d1f2008-11-24 19:36:26 +0000309{
310 KVMState *s = kvm_state;
Michael S. Tsirkin25254bb2011-04-06 22:09:54 +0300311 int flags, mask = KVM_MEM_LOG_DIRTY_PAGES;
Jan Kiszka4495d6a2009-05-01 20:52:46 +0200312 int old_flags;
313
Jan Kiszka4495d6a2009-05-01 20:52:46 +0200314 old_flags = mem->flags;
aliguori5832d1f2008-11-24 19:36:26 +0000315
Jordan Justen235e8982013-05-29 01:27:26 -0700316 flags = (mem->flags & ~mask) | kvm_mem_flags(s, log_dirty, false);
aliguori5832d1f2008-11-24 19:36:26 +0000317 mem->flags = flags;
318
Jan Kiszka4495d6a2009-05-01 20:52:46 +0200319 /* If nothing changed effectively, no need to issue ioctl */
320 if (s->migration_log) {
321 flags |= KVM_MEM_LOG_DIRTY_PAGES;
322 }
Michael S. Tsirkin25254bb2011-04-06 22:09:54 +0300323
Jan Kiszka4495d6a2009-05-01 20:52:46 +0200324 if (flags == old_flags) {
Michael S. Tsirkin25254bb2011-04-06 22:09:54 +0300325 return 0;
Jan Kiszka4495d6a2009-05-01 20:52:46 +0200326 }
327
aliguori5832d1f2008-11-24 19:36:26 +0000328 return kvm_set_user_memory_region(s, mem);
329}
330
Avi Kivitya8170e52012-10-23 12:30:10 +0200331static int kvm_dirty_pages_log_change(hwaddr phys_addr,
Michael S. Tsirkin25254bb2011-04-06 22:09:54 +0300332 ram_addr_t size, bool log_dirty)
333{
334 KVMState *s = kvm_state;
335 KVMSlot *mem = kvm_lookup_matching_slot(s, phys_addr, phys_addr + size);
336
337 if (mem == NULL) {
338 fprintf(stderr, "BUG: %s: invalid parameters " TARGET_FMT_plx "-"
339 TARGET_FMT_plx "\n", __func__, phys_addr,
Avi Kivitya8170e52012-10-23 12:30:10 +0200340 (hwaddr)(phys_addr + size - 1));
Michael S. Tsirkin25254bb2011-04-06 22:09:54 +0300341 return -EINVAL;
342 }
343 return kvm_slot_dirty_pages_log_change(mem, log_dirty);
344}
345
Avi Kivitya01672d2011-12-18 14:06:05 +0200346static void kvm_log_start(MemoryListener *listener,
347 MemoryRegionSection *section)
aliguori5832d1f2008-11-24 19:36:26 +0000348{
Avi Kivitya01672d2011-12-18 14:06:05 +0200349 int r;
350
351 r = kvm_dirty_pages_log_change(section->offset_within_address_space,
Paolo Bonzini052e87b2013-05-27 10:08:27 +0200352 int128_get64(section->size), true);
Avi Kivitya01672d2011-12-18 14:06:05 +0200353 if (r < 0) {
354 abort();
355 }
aliguori5832d1f2008-11-24 19:36:26 +0000356}
357
Avi Kivitya01672d2011-12-18 14:06:05 +0200358static void kvm_log_stop(MemoryListener *listener,
359 MemoryRegionSection *section)
aliguori5832d1f2008-11-24 19:36:26 +0000360{
Avi Kivitya01672d2011-12-18 14:06:05 +0200361 int r;
362
363 r = kvm_dirty_pages_log_change(section->offset_within_address_space,
Paolo Bonzini052e87b2013-05-27 10:08:27 +0200364 int128_get64(section->size), false);
Avi Kivitya01672d2011-12-18 14:06:05 +0200365 if (r < 0) {
366 abort();
367 }
aliguori5832d1f2008-11-24 19:36:26 +0000368}
369
Radim Krčmář8c1ac472015-02-20 17:06:15 +0100370static int kvm_set_migration_log(bool enable)
Jan Kiszka4495d6a2009-05-01 20:52:46 +0200371{
372 KVMState *s = kvm_state;
373 KVMSlot *mem;
374 int i, err;
375
376 s->migration_log = enable;
377
Alex Williamsonfb541ca2013-11-22 12:12:44 -0700378 for (i = 0; i < s->nr_slots; i++) {
Jan Kiszka4495d6a2009-05-01 20:52:46 +0200379 mem = &s->slots[i];
380
Alex Williamson70fedd72010-07-14 13:36:49 -0600381 if (!mem->memory_size) {
382 continue;
383 }
Jan Kiszka4495d6a2009-05-01 20:52:46 +0200384 if (!!(mem->flags & KVM_MEM_LOG_DIRTY_PAGES) == enable) {
385 continue;
386 }
387 err = kvm_set_user_memory_region(s, mem);
388 if (err) {
389 return err;
390 }
391 }
392 return 0;
393}
394
Marcelo Tosatti8369e012010-04-23 14:04:14 -0300395/* get kvm's dirty pages bitmap and update qemu's */
Avi Kivityffcde122011-12-19 13:18:13 +0200396static int kvm_get_dirty_pages_log_range(MemoryRegionSection *section,
397 unsigned long *bitmap)
Alexander Graf96c16062009-07-27 12:49:56 +0200398{
Juan Quintelac9dd46f2013-11-05 15:45:46 +0100399 ram_addr_t start = section->offset_within_region + section->mr->ram_addr;
Juan Quintela5ff7fb72013-11-05 15:52:54 +0100400 ram_addr_t pages = int128_get64(section->size) / getpagesize();
Marcelo Tosatti8369e012010-04-23 14:04:14 -0300401
Juan Quintela5ff7fb72013-11-05 15:52:54 +0100402 cpu_physical_memory_set_dirty_lebitmap(bitmap, start, pages);
Marcelo Tosatti8369e012010-04-23 14:04:14 -0300403 return 0;
Alexander Graf96c16062009-07-27 12:49:56 +0200404}
405
Marcelo Tosatti8369e012010-04-23 14:04:14 -0300406#define ALIGN(x, y) (((x)+(y)-1) & ~((y)-1))
407
aliguori5832d1f2008-11-24 19:36:26 +0000408/**
409 * kvm_physical_sync_dirty_bitmap - Grab dirty bitmap from kernel space
Blue Swirlfd4aa972011-10-16 16:04:59 +0000410 * This function updates qemu's dirty bitmap using
411 * memory_region_set_dirty(). This means all bits are set
412 * to dirty.
aliguori5832d1f2008-11-24 19:36:26 +0000413 *
aliguorid3f8d372009-04-17 14:26:29 +0000414 * @start_add: start of logged region.
aliguori5832d1f2008-11-24 19:36:26 +0000415 * @end_addr: end of logged region.
416 */
Avi Kivityffcde122011-12-19 13:18:13 +0200417static int kvm_physical_sync_dirty_bitmap(MemoryRegionSection *section)
aliguori5832d1f2008-11-24 19:36:26 +0000418{
419 KVMState *s = kvm_state;
Jan Kiszka151f7742009-05-01 20:52:47 +0200420 unsigned long size, allocated_size = 0;
Christian Borntraegerd229b982014-10-14 11:50:27 +0200421 KVMDirtyLog d = {};
Jan Kiszka151f7742009-05-01 20:52:47 +0200422 KVMSlot *mem;
423 int ret = 0;
Avi Kivitya8170e52012-10-23 12:30:10 +0200424 hwaddr start_addr = section->offset_within_address_space;
Paolo Bonzini052e87b2013-05-27 10:08:27 +0200425 hwaddr end_addr = start_addr + int128_get64(section->size);
aliguori5832d1f2008-11-24 19:36:26 +0000426
Jan Kiszka151f7742009-05-01 20:52:47 +0200427 d.dirty_bitmap = NULL;
428 while (start_addr < end_addr) {
429 mem = kvm_lookup_overlapping_slot(s, start_addr, end_addr);
430 if (mem == NULL) {
431 break;
432 }
433
Michael Tokarev51b0c602011-04-26 20:13:49 +0400434 /* XXX bad kernel interface alert
435 * For dirty bitmap, kernel allocates array of size aligned to
436 * bits-per-long. But for case when the kernel is 64bits and
437 * the userspace is 32bits, userspace can't align to the same
438 * bits-per-long, since sizeof(long) is different between kernel
439 * and user space. This way, userspace will provide buffer which
440 * may be 4 bytes less than the kernel will use, resulting in
441 * userspace memory corruption (which is not detectable by valgrind
442 * too, in most cases).
443 * So for now, let's align to 64 instead of HOST_LONG_BITS here, in
444 * a hope that sizeof(long) wont become >8 any time soon.
445 */
446 size = ALIGN(((mem->memory_size) >> TARGET_PAGE_BITS),
447 /*HOST_LONG_BITS*/ 64) / 8;
Jan Kiszka151f7742009-05-01 20:52:47 +0200448 if (!d.dirty_bitmap) {
Anthony Liguori7267c092011-08-20 22:09:37 -0500449 d.dirty_bitmap = g_malloc(size);
Jan Kiszka151f7742009-05-01 20:52:47 +0200450 } else if (size > allocated_size) {
Anthony Liguori7267c092011-08-20 22:09:37 -0500451 d.dirty_bitmap = g_realloc(d.dirty_bitmap, size);
Jan Kiszka151f7742009-05-01 20:52:47 +0200452 }
453 allocated_size = size;
454 memset(d.dirty_bitmap, 0, allocated_size);
455
456 d.slot = mem->slot;
457
Michael Tokarev50212d62014-04-14 16:14:04 +0400458 if (kvm_vm_ioctl(s, KVM_GET_DIRTY_LOG, &d) == -1) {
Blue Swirl8c0d5772010-04-18 14:22:14 +0000459 DPRINTF("ioctl failed %d\n", errno);
Jan Kiszka151f7742009-05-01 20:52:47 +0200460 ret = -1;
461 break;
462 }
463
Avi Kivityffcde122011-12-19 13:18:13 +0200464 kvm_get_dirty_pages_log_range(section, d.dirty_bitmap);
Marcelo Tosatti8369e012010-04-23 14:04:14 -0300465 start_addr = mem->start_addr + mem->memory_size;
aliguori5832d1f2008-11-24 19:36:26 +0000466 }
Anthony Liguori7267c092011-08-20 22:09:37 -0500467 g_free(d.dirty_bitmap);
Jan Kiszka151f7742009-05-01 20:52:47 +0200468
469 return ret;
aliguori5832d1f2008-11-24 19:36:26 +0000470}
471
Avi Kivity95d29942012-10-02 18:21:54 +0200472static void kvm_coalesce_mmio_region(MemoryListener *listener,
473 MemoryRegionSection *secion,
Avi Kivitya8170e52012-10-23 12:30:10 +0200474 hwaddr start, hwaddr size)
aliguorif65ed4c2008-12-09 20:09:57 +0000475{
aliguorif65ed4c2008-12-09 20:09:57 +0000476 KVMState *s = kvm_state;
477
478 if (s->coalesced_mmio) {
479 struct kvm_coalesced_mmio_zone zone;
480
481 zone.addr = start;
482 zone.size = size;
Michael S. Tsirkin7e680752012-02-29 17:54:29 +0200483 zone.pad = 0;
aliguorif65ed4c2008-12-09 20:09:57 +0000484
Avi Kivity95d29942012-10-02 18:21:54 +0200485 (void)kvm_vm_ioctl(s, KVM_REGISTER_COALESCED_MMIO, &zone);
aliguorif65ed4c2008-12-09 20:09:57 +0000486 }
aliguorif65ed4c2008-12-09 20:09:57 +0000487}
488
Avi Kivity95d29942012-10-02 18:21:54 +0200489static void kvm_uncoalesce_mmio_region(MemoryListener *listener,
490 MemoryRegionSection *secion,
Avi Kivitya8170e52012-10-23 12:30:10 +0200491 hwaddr start, hwaddr size)
aliguorif65ed4c2008-12-09 20:09:57 +0000492{
aliguorif65ed4c2008-12-09 20:09:57 +0000493 KVMState *s = kvm_state;
494
495 if (s->coalesced_mmio) {
496 struct kvm_coalesced_mmio_zone zone;
497
498 zone.addr = start;
499 zone.size = size;
Michael S. Tsirkin7e680752012-02-29 17:54:29 +0200500 zone.pad = 0;
aliguorif65ed4c2008-12-09 20:09:57 +0000501
Avi Kivity95d29942012-10-02 18:21:54 +0200502 (void)kvm_vm_ioctl(s, KVM_UNREGISTER_COALESCED_MMIO, &zone);
aliguorif65ed4c2008-12-09 20:09:57 +0000503 }
aliguorif65ed4c2008-12-09 20:09:57 +0000504}
505
Anthony Liguoriad7b8b32009-05-08 15:33:24 -0500506int kvm_check_extension(KVMState *s, unsigned int extension)
507{
508 int ret;
509
510 ret = kvm_ioctl(s, KVM_CHECK_EXTENSION, extension);
511 if (ret < 0) {
512 ret = 0;
513 }
514
515 return ret;
516}
517
Alexander Graf7d0a07f2014-07-14 19:15:15 +0200518int kvm_vm_check_extension(KVMState *s, unsigned int extension)
519{
520 int ret;
521
522 ret = kvm_vm_ioctl(s, KVM_CHECK_EXTENSION, extension);
523 if (ret < 0) {
524 /* VM wide version not implemented, use global one instead */
525 ret = kvm_check_extension(s, extension);
526 }
527
528 return ret;
529}
530
Alexey Kardashevskiy584f2be2014-01-10 18:20:18 +1100531static int kvm_set_ioeventfd_mmio(int fd, hwaddr addr, uint32_t val,
Michael S. Tsirkin41cb62c2013-04-02 16:52:25 +0300532 bool assign, uint32_t size, bool datamatch)
Michael S. Tsirkin500ffd42013-04-02 00:05:21 +0300533{
534 int ret;
535 struct kvm_ioeventfd iofd;
536
Michael S. Tsirkin41cb62c2013-04-02 16:52:25 +0300537 iofd.datamatch = datamatch ? val : 0;
Michael S. Tsirkin500ffd42013-04-02 00:05:21 +0300538 iofd.addr = addr;
539 iofd.len = size;
Michael S. Tsirkin41cb62c2013-04-02 16:52:25 +0300540 iofd.flags = 0;
Michael S. Tsirkin500ffd42013-04-02 00:05:21 +0300541 iofd.fd = fd;
542
543 if (!kvm_enabled()) {
544 return -ENOSYS;
545 }
546
Michael S. Tsirkin41cb62c2013-04-02 16:52:25 +0300547 if (datamatch) {
548 iofd.flags |= KVM_IOEVENTFD_FLAG_DATAMATCH;
549 }
Michael S. Tsirkin500ffd42013-04-02 00:05:21 +0300550 if (!assign) {
551 iofd.flags |= KVM_IOEVENTFD_FLAG_DEASSIGN;
552 }
553
554 ret = kvm_vm_ioctl(kvm_state, KVM_IOEVENTFD, &iofd);
555
556 if (ret < 0) {
557 return -errno;
558 }
559
560 return 0;
561}
562
Michael S. Tsirkin44c3f8f2013-04-02 00:54:45 +0300563static int kvm_set_ioeventfd_pio(int fd, uint16_t addr, uint16_t val,
Michael S. Tsirkin41cb62c2013-04-02 16:52:25 +0300564 bool assign, uint32_t size, bool datamatch)
Michael S. Tsirkin500ffd42013-04-02 00:05:21 +0300565{
566 struct kvm_ioeventfd kick = {
Michael S. Tsirkin41cb62c2013-04-02 16:52:25 +0300567 .datamatch = datamatch ? val : 0,
Michael S. Tsirkin500ffd42013-04-02 00:05:21 +0300568 .addr = addr,
Michael S. Tsirkin41cb62c2013-04-02 16:52:25 +0300569 .flags = KVM_IOEVENTFD_FLAG_PIO,
Michael S. Tsirkin44c3f8f2013-04-02 00:54:45 +0300570 .len = size,
Michael S. Tsirkin500ffd42013-04-02 00:05:21 +0300571 .fd = fd,
572 };
573 int r;
574 if (!kvm_enabled()) {
575 return -ENOSYS;
576 }
Michael S. Tsirkin41cb62c2013-04-02 16:52:25 +0300577 if (datamatch) {
578 kick.flags |= KVM_IOEVENTFD_FLAG_DATAMATCH;
579 }
Michael S. Tsirkin500ffd42013-04-02 00:05:21 +0300580 if (!assign) {
581 kick.flags |= KVM_IOEVENTFD_FLAG_DEASSIGN;
582 }
583 r = kvm_vm_ioctl(kvm_state, KVM_IOEVENTFD, &kick);
584 if (r < 0) {
585 return r;
586 }
587 return 0;
588}
589
590
Stefan Hajnoczid2f2b8a2011-01-10 13:50:05 +0200591static int kvm_check_many_ioeventfds(void)
592{
Stefan Hajnoczid0dcac82011-01-25 16:17:14 +0000593 /* Userspace can use ioeventfd for io notification. This requires a host
594 * that supports eventfd(2) and an I/O thread; since eventfd does not
595 * support SIGIO it cannot interrupt the vcpu.
596 *
597 * Older kernels have a 6 device limit on the KVM io bus. Find out so we
Stefan Hajnoczid2f2b8a2011-01-10 13:50:05 +0200598 * can avoid creating too many ioeventfds.
599 */
Anthony Liguori12d45362011-08-22 08:24:58 -0500600#if defined(CONFIG_EVENTFD)
Stefan Hajnoczid2f2b8a2011-01-10 13:50:05 +0200601 int ioeventfds[7];
602 int i, ret = 0;
603 for (i = 0; i < ARRAY_SIZE(ioeventfds); i++) {
604 ioeventfds[i] = eventfd(0, EFD_CLOEXEC);
605 if (ioeventfds[i] < 0) {
606 break;
607 }
Michael S. Tsirkin41cb62c2013-04-02 16:52:25 +0300608 ret = kvm_set_ioeventfd_pio(ioeventfds[i], 0, i, true, 2, true);
Stefan Hajnoczid2f2b8a2011-01-10 13:50:05 +0200609 if (ret < 0) {
610 close(ioeventfds[i]);
611 break;
612 }
613 }
614
615 /* Decide whether many devices are supported or not */
616 ret = i == ARRAY_SIZE(ioeventfds);
617
618 while (i-- > 0) {
Michael S. Tsirkin41cb62c2013-04-02 16:52:25 +0300619 kvm_set_ioeventfd_pio(ioeventfds[i], 0, i, false, 2, true);
Stefan Hajnoczid2f2b8a2011-01-10 13:50:05 +0200620 close(ioeventfds[i]);
621 }
622 return ret;
623#else
624 return 0;
625#endif
626}
627
Jan Kiszka94a8d392011-01-21 21:48:17 +0100628static const KVMCapabilityInfo *
629kvm_check_extension_list(KVMState *s, const KVMCapabilityInfo *list)
630{
631 while (list->name) {
632 if (!kvm_check_extension(s, list->value)) {
633 return list;
634 }
635 list++;
636 }
637 return NULL;
638}
639
Avi Kivitya01672d2011-12-18 14:06:05 +0200640static void kvm_set_phys_mem(MemoryRegionSection *section, bool add)
Michael S. Tsirkin46dbef62010-01-27 22:07:08 +0200641{
642 KVMState *s = kvm_state;
Michael S. Tsirkin46dbef62010-01-27 22:07:08 +0200643 KVMSlot *mem, old;
644 int err;
Avi Kivitya01672d2011-12-18 14:06:05 +0200645 MemoryRegion *mr = section->mr;
646 bool log_dirty = memory_region_is_logging(mr);
Jordan Justen235e8982013-05-29 01:27:26 -0700647 bool writeable = !mr->readonly && !mr->rom_device;
648 bool readonly_flag = mr->readonly || memory_region_is_romd(mr);
Avi Kivitya8170e52012-10-23 12:30:10 +0200649 hwaddr start_addr = section->offset_within_address_space;
Paolo Bonzini052e87b2013-05-27 10:08:27 +0200650 ram_addr_t size = int128_get64(section->size);
Avi Kivity9f213ed2011-12-15 19:55:26 +0200651 void *ram = NULL;
Avi Kivity8f6f9622012-02-29 13:22:12 +0200652 unsigned delta;
Michael S. Tsirkin46dbef62010-01-27 22:07:08 +0200653
Gleb Natapov14542fe2010-07-28 18:13:23 +0300654 /* kvm works in page size chunks, but the function may be called
Alexander Graff2a64032014-11-07 22:12:48 +0100655 with sub-page size and unaligned start address. Pad the start
656 address to next and truncate size to previous page boundary. */
657 delta = (TARGET_PAGE_SIZE - (start_addr & ~TARGET_PAGE_MASK));
658 delta &= ~TARGET_PAGE_MASK;
Avi Kivity8f6f9622012-02-29 13:22:12 +0200659 if (delta > size) {
660 return;
661 }
662 start_addr += delta;
663 size -= delta;
664 size &= TARGET_PAGE_MASK;
665 if (!size || (start_addr & ~TARGET_PAGE_MASK)) {
666 return;
667 }
Michael S. Tsirkin46dbef62010-01-27 22:07:08 +0200668
Avi Kivitya01672d2011-12-18 14:06:05 +0200669 if (!memory_region_is_ram(mr)) {
Jordan Justen235e8982013-05-29 01:27:26 -0700670 if (writeable || !kvm_readonly_mem_allowed) {
671 return;
672 } else if (!mr->romd_mode) {
673 /* If the memory device is not in romd_mode, then we actually want
674 * to remove the kvm memory slot so all accesses will trap. */
675 add = false;
676 }
Avi Kivity9f213ed2011-12-15 19:55:26 +0200677 }
678
Avi Kivity8f6f9622012-02-29 13:22:12 +0200679 ram = memory_region_get_ram_ptr(mr) + section->offset_within_region + delta;
Avi Kivitya01672d2011-12-18 14:06:05 +0200680
Michael S. Tsirkin46dbef62010-01-27 22:07:08 +0200681 while (1) {
682 mem = kvm_lookup_overlapping_slot(s, start_addr, start_addr + size);
683 if (!mem) {
684 break;
685 }
686
Avi Kivitya01672d2011-12-18 14:06:05 +0200687 if (add && start_addr >= mem->start_addr &&
Michael S. Tsirkin46dbef62010-01-27 22:07:08 +0200688 (start_addr + size <= mem->start_addr + mem->memory_size) &&
Avi Kivity9f213ed2011-12-15 19:55:26 +0200689 (ram - start_addr == mem->ram - mem->start_addr)) {
Michael S. Tsirkin46dbef62010-01-27 22:07:08 +0200690 /* The new slot fits into the existing one and comes with
Michael S. Tsirkin25254bb2011-04-06 22:09:54 +0300691 * identical parameters - update flags and done. */
692 kvm_slot_dirty_pages_log_change(mem, log_dirty);
Michael S. Tsirkin46dbef62010-01-27 22:07:08 +0200693 return;
694 }
695
696 old = *mem;
697
Avi Kivity3fbffb62012-01-15 16:13:59 +0200698 if (mem->flags & KVM_MEM_LOG_DIRTY_PAGES) {
699 kvm_physical_sync_dirty_bitmap(section);
700 }
701
Michael S. Tsirkin46dbef62010-01-27 22:07:08 +0200702 /* unregister the overlapping slot */
703 mem->memory_size = 0;
704 err = kvm_set_user_memory_region(s, mem);
705 if (err) {
706 fprintf(stderr, "%s: error unregistering overlapping slot: %s\n",
707 __func__, strerror(-err));
708 abort();
709 }
710
711 /* Workaround for older KVM versions: we can't join slots, even not by
712 * unregistering the previous ones and then registering the larger
713 * slot. We have to maintain the existing fragmentation. Sigh.
714 *
715 * This workaround assumes that the new slot starts at the same
716 * address as the first existing one. If not or if some overlapping
717 * slot comes around later, we will fail (not seen in practice so far)
718 * - and actually require a recent KVM version. */
719 if (s->broken_set_mem_region &&
Avi Kivitya01672d2011-12-18 14:06:05 +0200720 old.start_addr == start_addr && old.memory_size < size && add) {
Michael S. Tsirkin46dbef62010-01-27 22:07:08 +0200721 mem = kvm_alloc_slot(s);
722 mem->memory_size = old.memory_size;
723 mem->start_addr = old.start_addr;
Avi Kivity9f213ed2011-12-15 19:55:26 +0200724 mem->ram = old.ram;
Jordan Justen235e8982013-05-29 01:27:26 -0700725 mem->flags = kvm_mem_flags(s, log_dirty, readonly_flag);
Michael S. Tsirkin46dbef62010-01-27 22:07:08 +0200726
727 err = kvm_set_user_memory_region(s, mem);
728 if (err) {
729 fprintf(stderr, "%s: error updating slot: %s\n", __func__,
730 strerror(-err));
731 abort();
732 }
733
734 start_addr += old.memory_size;
Avi Kivity9f213ed2011-12-15 19:55:26 +0200735 ram += old.memory_size;
Michael S. Tsirkin46dbef62010-01-27 22:07:08 +0200736 size -= old.memory_size;
737 continue;
738 }
739
740 /* register prefix slot */
741 if (old.start_addr < start_addr) {
742 mem = kvm_alloc_slot(s);
743 mem->memory_size = start_addr - old.start_addr;
744 mem->start_addr = old.start_addr;
Avi Kivity9f213ed2011-12-15 19:55:26 +0200745 mem->ram = old.ram;
Jordan Justen235e8982013-05-29 01:27:26 -0700746 mem->flags = kvm_mem_flags(s, log_dirty, readonly_flag);
Michael S. Tsirkin46dbef62010-01-27 22:07:08 +0200747
748 err = kvm_set_user_memory_region(s, mem);
749 if (err) {
750 fprintf(stderr, "%s: error registering prefix slot: %s\n",
751 __func__, strerror(-err));
Alexander Grafd4d68682011-04-16 10:15:11 +0200752#ifdef TARGET_PPC
753 fprintf(stderr, "%s: This is probably because your kernel's " \
754 "PAGE_SIZE is too big. Please try to use 4k " \
755 "PAGE_SIZE!\n", __func__);
756#endif
Michael S. Tsirkin46dbef62010-01-27 22:07:08 +0200757 abort();
758 }
759 }
760
761 /* register suffix slot */
762 if (old.start_addr + old.memory_size > start_addr + size) {
763 ram_addr_t size_delta;
764
765 mem = kvm_alloc_slot(s);
766 mem->start_addr = start_addr + size;
767 size_delta = mem->start_addr - old.start_addr;
768 mem->memory_size = old.memory_size - size_delta;
Avi Kivity9f213ed2011-12-15 19:55:26 +0200769 mem->ram = old.ram + size_delta;
Jordan Justen235e8982013-05-29 01:27:26 -0700770 mem->flags = kvm_mem_flags(s, log_dirty, readonly_flag);
Michael S. Tsirkin46dbef62010-01-27 22:07:08 +0200771
772 err = kvm_set_user_memory_region(s, mem);
773 if (err) {
774 fprintf(stderr, "%s: error registering suffix slot: %s\n",
775 __func__, strerror(-err));
776 abort();
777 }
778 }
779 }
780
781 /* in case the KVM bug workaround already "consumed" the new slot */
Jan Kiszkaa426e122011-01-04 09:32:13 +0100782 if (!size) {
Michael S. Tsirkin46dbef62010-01-27 22:07:08 +0200783 return;
Jan Kiszkaa426e122011-01-04 09:32:13 +0100784 }
Avi Kivitya01672d2011-12-18 14:06:05 +0200785 if (!add) {
Michael S. Tsirkin46dbef62010-01-27 22:07:08 +0200786 return;
Jan Kiszkaa426e122011-01-04 09:32:13 +0100787 }
Michael S. Tsirkin46dbef62010-01-27 22:07:08 +0200788 mem = kvm_alloc_slot(s);
789 mem->memory_size = size;
790 mem->start_addr = start_addr;
Avi Kivity9f213ed2011-12-15 19:55:26 +0200791 mem->ram = ram;
Jordan Justen235e8982013-05-29 01:27:26 -0700792 mem->flags = kvm_mem_flags(s, log_dirty, readonly_flag);
Michael S. Tsirkin46dbef62010-01-27 22:07:08 +0200793
794 err = kvm_set_user_memory_region(s, mem);
795 if (err) {
796 fprintf(stderr, "%s: error registering slot: %s\n", __func__,
797 strerror(-err));
798 abort();
799 }
800}
801
Avi Kivitya01672d2011-12-18 14:06:05 +0200802static void kvm_region_add(MemoryListener *listener,
803 MemoryRegionSection *section)
Michael S. Tsirkin7b8f3b72010-01-27 22:07:21 +0200804{
Paolo Bonzinidfde4e62013-05-06 10:46:11 +0200805 memory_region_ref(section->mr);
Avi Kivitya01672d2011-12-18 14:06:05 +0200806 kvm_set_phys_mem(section, true);
Michael S. Tsirkin7b8f3b72010-01-27 22:07:21 +0200807}
808
Avi Kivitya01672d2011-12-18 14:06:05 +0200809static void kvm_region_del(MemoryListener *listener,
810 MemoryRegionSection *section)
Michael S. Tsirkin7b8f3b72010-01-27 22:07:21 +0200811{
Avi Kivitya01672d2011-12-18 14:06:05 +0200812 kvm_set_phys_mem(section, false);
Paolo Bonzinidfde4e62013-05-06 10:46:11 +0200813 memory_region_unref(section->mr);
Michael S. Tsirkin7b8f3b72010-01-27 22:07:21 +0200814}
815
Avi Kivitya01672d2011-12-18 14:06:05 +0200816static void kvm_log_sync(MemoryListener *listener,
817 MemoryRegionSection *section)
Michael S. Tsirkin7b8f3b72010-01-27 22:07:21 +0200818{
Avi Kivitya01672d2011-12-18 14:06:05 +0200819 int r;
820
Avi Kivityffcde122011-12-19 13:18:13 +0200821 r = kvm_physical_sync_dirty_bitmap(section);
Avi Kivitya01672d2011-12-18 14:06:05 +0200822 if (r < 0) {
823 abort();
824 }
Michael S. Tsirkin7b8f3b72010-01-27 22:07:21 +0200825}
826
Avi Kivitya01672d2011-12-18 14:06:05 +0200827static void kvm_log_global_start(struct MemoryListener *listener)
828{
829 int r;
830
831 r = kvm_set_migration_log(1);
832 assert(r >= 0);
833}
834
835static void kvm_log_global_stop(struct MemoryListener *listener)
836{
837 int r;
838
839 r = kvm_set_migration_log(0);
840 assert(r >= 0);
841}
842
Avi Kivityd22b0962012-09-30 22:21:11 +0200843static void kvm_mem_ioeventfd_add(MemoryListener *listener,
844 MemoryRegionSection *section,
845 bool match_data, uint64_t data,
846 EventNotifier *e)
847{
848 int fd = event_notifier_get_fd(e);
Avi Kivity80a1ea32012-02-08 16:39:06 +0200849 int r;
850
Michael S. Tsirkin4b8f1c82012-03-20 14:31:38 +0200851 r = kvm_set_ioeventfd_mmio(fd, section->offset_within_address_space,
Paolo Bonzini052e87b2013-05-27 10:08:27 +0200852 data, true, int128_get64(section->size),
853 match_data);
Avi Kivity80a1ea32012-02-08 16:39:06 +0200854 if (r < 0) {
Amos Kongfa4ba922013-05-22 12:57:35 +0800855 fprintf(stderr, "%s: error adding ioeventfd: %s\n",
856 __func__, strerror(-r));
Avi Kivity80a1ea32012-02-08 16:39:06 +0200857 abort();
858 }
859}
860
Avi Kivityd22b0962012-09-30 22:21:11 +0200861static void kvm_mem_ioeventfd_del(MemoryListener *listener,
862 MemoryRegionSection *section,
863 bool match_data, uint64_t data,
864 EventNotifier *e)
Avi Kivity80a1ea32012-02-08 16:39:06 +0200865{
Avi Kivityd22b0962012-09-30 22:21:11 +0200866 int fd = event_notifier_get_fd(e);
Avi Kivity80a1ea32012-02-08 16:39:06 +0200867 int r;
868
Michael S. Tsirkin4b8f1c82012-03-20 14:31:38 +0200869 r = kvm_set_ioeventfd_mmio(fd, section->offset_within_address_space,
Paolo Bonzini052e87b2013-05-27 10:08:27 +0200870 data, false, int128_get64(section->size),
871 match_data);
Avi Kivity80a1ea32012-02-08 16:39:06 +0200872 if (r < 0) {
873 abort();
874 }
875}
876
Avi Kivityd22b0962012-09-30 22:21:11 +0200877static void kvm_io_ioeventfd_add(MemoryListener *listener,
878 MemoryRegionSection *section,
879 bool match_data, uint64_t data,
880 EventNotifier *e)
Avi Kivity80a1ea32012-02-08 16:39:06 +0200881{
Avi Kivityd22b0962012-09-30 22:21:11 +0200882 int fd = event_notifier_get_fd(e);
Avi Kivity80a1ea32012-02-08 16:39:06 +0200883 int r;
884
Michael S. Tsirkin44c3f8f2013-04-02 00:54:45 +0300885 r = kvm_set_ioeventfd_pio(fd, section->offset_within_address_space,
Paolo Bonzini052e87b2013-05-27 10:08:27 +0200886 data, true, int128_get64(section->size),
887 match_data);
Avi Kivity80a1ea32012-02-08 16:39:06 +0200888 if (r < 0) {
Amos Kongfa4ba922013-05-22 12:57:35 +0800889 fprintf(stderr, "%s: error adding ioeventfd: %s\n",
890 __func__, strerror(-r));
Avi Kivity80a1ea32012-02-08 16:39:06 +0200891 abort();
892 }
893}
894
Avi Kivityd22b0962012-09-30 22:21:11 +0200895static void kvm_io_ioeventfd_del(MemoryListener *listener,
896 MemoryRegionSection *section,
897 bool match_data, uint64_t data,
898 EventNotifier *e)
Avi Kivity80a1ea32012-02-08 16:39:06 +0200899
900{
Avi Kivityd22b0962012-09-30 22:21:11 +0200901 int fd = event_notifier_get_fd(e);
Avi Kivity80a1ea32012-02-08 16:39:06 +0200902 int r;
903
Michael S. Tsirkin44c3f8f2013-04-02 00:54:45 +0300904 r = kvm_set_ioeventfd_pio(fd, section->offset_within_address_space,
Paolo Bonzini052e87b2013-05-27 10:08:27 +0200905 data, false, int128_get64(section->size),
906 match_data);
Avi Kivity80a1ea32012-02-08 16:39:06 +0200907 if (r < 0) {
908 abort();
909 }
910}
911
Avi Kivitya01672d2011-12-18 14:06:05 +0200912static MemoryListener kvm_memory_listener = {
913 .region_add = kvm_region_add,
914 .region_del = kvm_region_del,
Anthony PERARDe5896b12011-02-07 12:19:23 +0100915 .log_start = kvm_log_start,
916 .log_stop = kvm_log_stop,
Avi Kivitya01672d2011-12-18 14:06:05 +0200917 .log_sync = kvm_log_sync,
918 .log_global_start = kvm_log_global_start,
919 .log_global_stop = kvm_log_global_stop,
Avi Kivityd22b0962012-09-30 22:21:11 +0200920 .eventfd_add = kvm_mem_ioeventfd_add,
921 .eventfd_del = kvm_mem_ioeventfd_del,
Avi Kivity95d29942012-10-02 18:21:54 +0200922 .coalesced_mmio_add = kvm_coalesce_mmio_region,
923 .coalesced_mmio_del = kvm_uncoalesce_mmio_region,
Avi Kivityd22b0962012-09-30 22:21:11 +0200924 .priority = 10,
925};
926
927static MemoryListener kvm_io_listener = {
Avi Kivityd22b0962012-09-30 22:21:11 +0200928 .eventfd_add = kvm_io_ioeventfd_add,
929 .eventfd_del = kvm_io_ioeventfd_del,
Avi Kivity72e22d22012-02-08 15:05:50 +0200930 .priority = 10,
Michael S. Tsirkin7b8f3b72010-01-27 22:07:21 +0200931};
932
Andreas Färberc3affe52013-01-18 15:03:43 +0100933static void kvm_handle_interrupt(CPUState *cpu, int mask)
Jan Kiszkaaa7f74d2011-04-13 01:32:56 +0200934{
Andreas Färber259186a2013-01-17 18:51:17 +0100935 cpu->interrupt_request |= mask;
Jan Kiszkaaa7f74d2011-04-13 01:32:56 +0200936
Andreas Färber60e82572012-05-02 22:23:49 +0200937 if (!qemu_cpu_is_self(cpu)) {
Andreas Färberc08d7422012-05-03 04:34:15 +0200938 qemu_cpu_kick(cpu);
Jan Kiszkaaa7f74d2011-04-13 01:32:56 +0200939 }
940}
941
Peter Maydell3889c3f2012-07-26 15:35:12 +0100942int kvm_set_irq(KVMState *s, int irq, int level)
Jan Kiszka84b058d2011-10-15 11:49:47 +0200943{
944 struct kvm_irq_level event;
945 int ret;
946
Peter Maydell7ae26bd2012-07-26 15:35:11 +0100947 assert(kvm_async_interrupts_enabled());
Jan Kiszka84b058d2011-10-15 11:49:47 +0200948
949 event.level = level;
950 event.irq = irq;
Jan Kiszkae333cd62012-08-24 13:34:47 +0200951 ret = kvm_vm_ioctl(s, s->irq_set_ioctl, &event);
Jan Kiszka84b058d2011-10-15 11:49:47 +0200952 if (ret < 0) {
Peter Maydell3889c3f2012-07-26 15:35:12 +0100953 perror("kvm_set_irq");
Jan Kiszka84b058d2011-10-15 11:49:47 +0200954 abort();
955 }
956
Jan Kiszkae333cd62012-08-24 13:34:47 +0200957 return (s->irq_set_ioctl == KVM_IRQ_LINE) ? 1 : event.status;
Jan Kiszka84b058d2011-10-15 11:49:47 +0200958}
959
960#ifdef KVM_CAP_IRQ_ROUTING
Jan Kiszkad3d3bef2012-06-05 21:03:57 +0200961typedef struct KVMMSIRoute {
962 struct kvm_irq_routing_entry kroute;
963 QTAILQ_ENTRY(KVMMSIRoute) entry;
964} KVMMSIRoute;
965
Jan Kiszka84b058d2011-10-15 11:49:47 +0200966static void set_gsi(KVMState *s, unsigned int gsi)
967{
Jan Kiszka84b058d2011-10-15 11:49:47 +0200968 s->used_gsi_bitmap[gsi / 32] |= 1U << (gsi % 32);
969}
970
Jan Kiszka04fa27f2012-05-16 15:41:10 -0300971static void clear_gsi(KVMState *s, unsigned int gsi)
972{
973 s->used_gsi_bitmap[gsi / 32] &= ~(1U << (gsi % 32));
974}
975
Alexander Graf7b774592013-04-16 15:58:13 +0200976void kvm_init_irq_routing(KVMState *s)
Jan Kiszka84b058d2011-10-15 11:49:47 +0200977{
Jan Kiszka04fa27f2012-05-16 15:41:10 -0300978 int gsi_count, i;
Jan Kiszka84b058d2011-10-15 11:49:47 +0200979
Alexander Graf00008412014-06-06 14:46:05 +0200980 gsi_count = kvm_check_extension(s, KVM_CAP_IRQ_ROUTING) - 1;
Jan Kiszka84b058d2011-10-15 11:49:47 +0200981 if (gsi_count > 0) {
982 unsigned int gsi_bits, i;
983
984 /* Round up so we can search ints using ffs */
Jason Baronbc8c6782012-03-28 14:18:05 -0400985 gsi_bits = ALIGN(gsi_count, 32);
Jan Kiszka84b058d2011-10-15 11:49:47 +0200986 s->used_gsi_bitmap = g_malloc0(gsi_bits / 8);
Jan Kiszka4e2e4e62012-05-16 15:41:08 -0300987 s->gsi_count = gsi_count;
Jan Kiszka84b058d2011-10-15 11:49:47 +0200988
989 /* Mark any over-allocated bits as already in use */
990 for (i = gsi_count; i < gsi_bits; i++) {
991 set_gsi(s, i);
992 }
993 }
994
995 s->irq_routes = g_malloc0(sizeof(*s->irq_routes));
996 s->nr_allocated_irq_routes = 0;
997
Jan Kiszka4a3adeb2012-05-16 15:41:14 -0300998 if (!s->direct_msi) {
999 for (i = 0; i < KVM_MSI_HASHTAB_SIZE; i++) {
1000 QTAILQ_INIT(&s->msi_hashtab[i]);
1001 }
Jan Kiszka04fa27f2012-05-16 15:41:10 -03001002 }
1003
Jan Kiszka84b058d2011-10-15 11:49:47 +02001004 kvm_arch_init_irq_routing(s);
1005}
1006
Alexander Grafcb925cf2013-04-17 01:11:55 +02001007void kvm_irqchip_commit_routes(KVMState *s)
Jan Kiszkae7b20302012-05-17 10:32:35 -03001008{
1009 int ret;
1010
1011 s->irq_routes->flags = 0;
1012 ret = kvm_vm_ioctl(s, KVM_SET_GSI_ROUTING, s->irq_routes);
1013 assert(ret == 0);
1014}
1015
Jan Kiszka84b058d2011-10-15 11:49:47 +02001016static void kvm_add_routing_entry(KVMState *s,
1017 struct kvm_irq_routing_entry *entry)
1018{
1019 struct kvm_irq_routing_entry *new;
1020 int n, size;
1021
1022 if (s->irq_routes->nr == s->nr_allocated_irq_routes) {
1023 n = s->nr_allocated_irq_routes * 2;
1024 if (n < 64) {
1025 n = 64;
1026 }
1027 size = sizeof(struct kvm_irq_routing);
1028 size += n * sizeof(*new);
1029 s->irq_routes = g_realloc(s->irq_routes, size);
1030 s->nr_allocated_irq_routes = n;
1031 }
1032 n = s->irq_routes->nr++;
1033 new = &s->irq_routes->entries[n];
Michael S. Tsirkin0fbc2072013-06-04 14:52:32 +03001034
1035 *new = *entry;
Jan Kiszka84b058d2011-10-15 11:49:47 +02001036
1037 set_gsi(s, entry->gsi);
1038}
1039
Jan Kiszkacc574072012-08-27 08:28:38 +02001040static int kvm_update_routing_entry(KVMState *s,
1041 struct kvm_irq_routing_entry *new_entry)
1042{
1043 struct kvm_irq_routing_entry *entry;
1044 int n;
1045
1046 for (n = 0; n < s->irq_routes->nr; n++) {
1047 entry = &s->irq_routes->entries[n];
1048 if (entry->gsi != new_entry->gsi) {
1049 continue;
1050 }
1051
Michael S. Tsirkin40509f72013-06-04 14:52:35 +03001052 if(!memcmp(entry, new_entry, sizeof *entry)) {
1053 return 0;
1054 }
1055
Michael S. Tsirkin0fbc2072013-06-04 14:52:32 +03001056 *entry = *new_entry;
Jan Kiszkacc574072012-08-27 08:28:38 +02001057
1058 kvm_irqchip_commit_routes(s);
1059
1060 return 0;
1061 }
1062
1063 return -ESRCH;
1064}
1065
Jan Kiszka1df186d2012-05-17 10:32:32 -03001066void kvm_irqchip_add_irq_route(KVMState *s, int irq, int irqchip, int pin)
Jan Kiszka84b058d2011-10-15 11:49:47 +02001067{
Michael S. Tsirkin0fbc2072013-06-04 14:52:32 +03001068 struct kvm_irq_routing_entry e = {};
Jan Kiszka84b058d2011-10-15 11:49:47 +02001069
Jan Kiszka4e2e4e62012-05-16 15:41:08 -03001070 assert(pin < s->gsi_count);
1071
Jan Kiszka84b058d2011-10-15 11:49:47 +02001072 e.gsi = irq;
1073 e.type = KVM_IRQ_ROUTING_IRQCHIP;
1074 e.flags = 0;
1075 e.u.irqchip.irqchip = irqchip;
1076 e.u.irqchip.pin = pin;
1077 kvm_add_routing_entry(s, &e);
1078}
1079
Jan Kiszka1e2aa8b2012-05-17 10:32:34 -03001080void kvm_irqchip_release_virq(KVMState *s, int virq)
Jan Kiszka04fa27f2012-05-16 15:41:10 -03001081{
1082 struct kvm_irq_routing_entry *e;
1083 int i;
1084
Alexey Kardashevskiy76fe21d2013-09-03 18:08:25 +10001085 if (kvm_gsi_direct_mapping()) {
1086 return;
1087 }
1088
Jan Kiszka04fa27f2012-05-16 15:41:10 -03001089 for (i = 0; i < s->irq_routes->nr; i++) {
1090 e = &s->irq_routes->entries[i];
1091 if (e->gsi == virq) {
1092 s->irq_routes->nr--;
1093 *e = s->irq_routes->entries[s->irq_routes->nr];
1094 }
1095 }
1096 clear_gsi(s, virq);
1097}
1098
1099static unsigned int kvm_hash_msi(uint32_t data)
1100{
1101 /* This is optimized for IA32 MSI layout. However, no other arch shall
1102 * repeat the mistake of not providing a direct MSI injection API. */
1103 return data & 0xff;
1104}
1105
1106static void kvm_flush_dynamic_msi_routes(KVMState *s)
1107{
1108 KVMMSIRoute *route, *next;
1109 unsigned int hash;
1110
1111 for (hash = 0; hash < KVM_MSI_HASHTAB_SIZE; hash++) {
1112 QTAILQ_FOREACH_SAFE(route, &s->msi_hashtab[hash], entry, next) {
1113 kvm_irqchip_release_virq(s, route->kroute.gsi);
1114 QTAILQ_REMOVE(&s->msi_hashtab[hash], route, entry);
1115 g_free(route);
1116 }
1117 }
1118}
1119
1120static int kvm_irqchip_get_virq(KVMState *s)
1121{
1122 uint32_t *word = s->used_gsi_bitmap;
1123 int max_words = ALIGN(s->gsi_count, 32) / 32;
1124 int i, bit;
1125 bool retry = true;
1126
1127again:
1128 /* Return the lowest unused GSI in the bitmap */
1129 for (i = 0; i < max_words; i++) {
1130 bit = ffs(~word[i]);
1131 if (!bit) {
1132 continue;
1133 }
1134
1135 return bit - 1 + i * 32;
1136 }
Jan Kiszka4a3adeb2012-05-16 15:41:14 -03001137 if (!s->direct_msi && retry) {
Jan Kiszka04fa27f2012-05-16 15:41:10 -03001138 retry = false;
1139 kvm_flush_dynamic_msi_routes(s);
1140 goto again;
1141 }
1142 return -ENOSPC;
1143
1144}
1145
1146static KVMMSIRoute *kvm_lookup_msi_route(KVMState *s, MSIMessage msg)
1147{
1148 unsigned int hash = kvm_hash_msi(msg.data);
1149 KVMMSIRoute *route;
1150
1151 QTAILQ_FOREACH(route, &s->msi_hashtab[hash], entry) {
1152 if (route->kroute.u.msi.address_lo == (uint32_t)msg.address &&
1153 route->kroute.u.msi.address_hi == (msg.address >> 32) &&
Alexander Grafd07cc1f2013-04-16 15:05:22 +02001154 route->kroute.u.msi.data == le32_to_cpu(msg.data)) {
Jan Kiszka04fa27f2012-05-16 15:41:10 -03001155 return route;
1156 }
1157 }
1158 return NULL;
1159}
1160
1161int kvm_irqchip_send_msi(KVMState *s, MSIMessage msg)
1162{
Jan Kiszka4a3adeb2012-05-16 15:41:14 -03001163 struct kvm_msi msi;
Jan Kiszka04fa27f2012-05-16 15:41:10 -03001164 KVMMSIRoute *route;
1165
Jan Kiszka4a3adeb2012-05-16 15:41:14 -03001166 if (s->direct_msi) {
1167 msi.address_lo = (uint32_t)msg.address;
1168 msi.address_hi = msg.address >> 32;
Alexander Grafd07cc1f2013-04-16 15:05:22 +02001169 msi.data = le32_to_cpu(msg.data);
Jan Kiszka4a3adeb2012-05-16 15:41:14 -03001170 msi.flags = 0;
1171 memset(msi.pad, 0, sizeof(msi.pad));
1172
1173 return kvm_vm_ioctl(s, KVM_SIGNAL_MSI, &msi);
1174 }
1175
Jan Kiszka04fa27f2012-05-16 15:41:10 -03001176 route = kvm_lookup_msi_route(s, msg);
1177 if (!route) {
Jan Kiszkae7b20302012-05-17 10:32:35 -03001178 int virq;
Jan Kiszka04fa27f2012-05-16 15:41:10 -03001179
1180 virq = kvm_irqchip_get_virq(s);
1181 if (virq < 0) {
1182 return virq;
1183 }
1184
Michael S. Tsirkin0fbc2072013-06-04 14:52:32 +03001185 route = g_malloc0(sizeof(KVMMSIRoute));
Jan Kiszka04fa27f2012-05-16 15:41:10 -03001186 route->kroute.gsi = virq;
1187 route->kroute.type = KVM_IRQ_ROUTING_MSI;
1188 route->kroute.flags = 0;
1189 route->kroute.u.msi.address_lo = (uint32_t)msg.address;
1190 route->kroute.u.msi.address_hi = msg.address >> 32;
Alexander Grafd07cc1f2013-04-16 15:05:22 +02001191 route->kroute.u.msi.data = le32_to_cpu(msg.data);
Jan Kiszka04fa27f2012-05-16 15:41:10 -03001192
1193 kvm_add_routing_entry(s, &route->kroute);
Alexander Grafcb925cf2013-04-17 01:11:55 +02001194 kvm_irqchip_commit_routes(s);
Jan Kiszka04fa27f2012-05-16 15:41:10 -03001195
1196 QTAILQ_INSERT_TAIL(&s->msi_hashtab[kvm_hash_msi(msg.data)], route,
1197 entry);
Jan Kiszka04fa27f2012-05-16 15:41:10 -03001198 }
1199
1200 assert(route->kroute.type == KVM_IRQ_ROUTING_MSI);
1201
Peter Maydell3889c3f2012-07-26 15:35:12 +01001202 return kvm_set_irq(s, route->kroute.gsi, 1);
Jan Kiszka04fa27f2012-05-16 15:41:10 -03001203}
1204
Jan Kiszka92b4e482012-05-17 10:32:33 -03001205int kvm_irqchip_add_msi_route(KVMState *s, MSIMessage msg)
1206{
Michael S. Tsirkin0fbc2072013-06-04 14:52:32 +03001207 struct kvm_irq_routing_entry kroute = {};
Jan Kiszka92b4e482012-05-17 10:32:33 -03001208 int virq;
1209
Alexey Kardashevskiy76fe21d2013-09-03 18:08:25 +10001210 if (kvm_gsi_direct_mapping()) {
1211 return msg.data & 0xffff;
1212 }
1213
Peter Maydellf3e1bed2012-07-26 15:35:16 +01001214 if (!kvm_gsi_routing_enabled()) {
Jan Kiszka92b4e482012-05-17 10:32:33 -03001215 return -ENOSYS;
1216 }
1217
1218 virq = kvm_irqchip_get_virq(s);
1219 if (virq < 0) {
1220 return virq;
1221 }
1222
1223 kroute.gsi = virq;
1224 kroute.type = KVM_IRQ_ROUTING_MSI;
1225 kroute.flags = 0;
1226 kroute.u.msi.address_lo = (uint32_t)msg.address;
1227 kroute.u.msi.address_hi = msg.address >> 32;
Alexander Grafd07cc1f2013-04-16 15:05:22 +02001228 kroute.u.msi.data = le32_to_cpu(msg.data);
Frank Blaschka9e03a042015-01-09 09:04:40 +01001229 if (kvm_arch_fixup_msi_route(&kroute, msg.address, msg.data)) {
1230 kvm_irqchip_release_virq(s, virq);
1231 return -EINVAL;
1232 }
Jan Kiszka92b4e482012-05-17 10:32:33 -03001233
1234 kvm_add_routing_entry(s, &kroute);
Alexander Grafcb925cf2013-04-17 01:11:55 +02001235 kvm_irqchip_commit_routes(s);
Jan Kiszka92b4e482012-05-17 10:32:33 -03001236
1237 return virq;
1238}
1239
Jan Kiszkacc574072012-08-27 08:28:38 +02001240int kvm_irqchip_update_msi_route(KVMState *s, int virq, MSIMessage msg)
1241{
Michael S. Tsirkin0fbc2072013-06-04 14:52:32 +03001242 struct kvm_irq_routing_entry kroute = {};
Jan Kiszkacc574072012-08-27 08:28:38 +02001243
Alexey Kardashevskiy76fe21d2013-09-03 18:08:25 +10001244 if (kvm_gsi_direct_mapping()) {
1245 return 0;
1246 }
1247
Jan Kiszkacc574072012-08-27 08:28:38 +02001248 if (!kvm_irqchip_in_kernel()) {
1249 return -ENOSYS;
1250 }
1251
1252 kroute.gsi = virq;
1253 kroute.type = KVM_IRQ_ROUTING_MSI;
1254 kroute.flags = 0;
1255 kroute.u.msi.address_lo = (uint32_t)msg.address;
1256 kroute.u.msi.address_hi = msg.address >> 32;
Alexander Grafd07cc1f2013-04-16 15:05:22 +02001257 kroute.u.msi.data = le32_to_cpu(msg.data);
Frank Blaschka9e03a042015-01-09 09:04:40 +01001258 if (kvm_arch_fixup_msi_route(&kroute, msg.address, msg.data)) {
1259 return -EINVAL;
1260 }
Jan Kiszkacc574072012-08-27 08:28:38 +02001261
1262 return kvm_update_routing_entry(s, &kroute);
1263}
1264
Vincenzo Maffioneca916d32013-07-22 11:51:33 +02001265static int kvm_irqchip_assign_irqfd(KVMState *s, int fd, int rfd, int virq,
1266 bool assign)
Jan Kiszka39853bb2012-05-17 10:32:36 -03001267{
1268 struct kvm_irqfd irqfd = {
1269 .fd = fd,
1270 .gsi = virq,
1271 .flags = assign ? 0 : KVM_IRQFD_FLAG_DEASSIGN,
1272 };
1273
Vincenzo Maffioneca916d32013-07-22 11:51:33 +02001274 if (rfd != -1) {
1275 irqfd.flags |= KVM_IRQFD_FLAG_RESAMPLE;
1276 irqfd.resamplefd = rfd;
1277 }
1278
Peter Maydellcc7e0dd2012-07-26 15:35:14 +01001279 if (!kvm_irqfds_enabled()) {
Jan Kiszka39853bb2012-05-17 10:32:36 -03001280 return -ENOSYS;
1281 }
1282
1283 return kvm_vm_ioctl(s, KVM_IRQFD, &irqfd);
1284}
1285
Cornelia Huckd426d9f2013-07-15 17:45:03 +02001286int kvm_irqchip_add_adapter_route(KVMState *s, AdapterInfo *adapter)
1287{
Christian Borntraegere9af2fe2014-11-20 22:10:58 +01001288 struct kvm_irq_routing_entry kroute = {};
Cornelia Huckd426d9f2013-07-15 17:45:03 +02001289 int virq;
1290
1291 if (!kvm_gsi_routing_enabled()) {
1292 return -ENOSYS;
1293 }
1294
1295 virq = kvm_irqchip_get_virq(s);
1296 if (virq < 0) {
1297 return virq;
1298 }
1299
1300 kroute.gsi = virq;
1301 kroute.type = KVM_IRQ_ROUTING_S390_ADAPTER;
1302 kroute.flags = 0;
1303 kroute.u.adapter.summary_addr = adapter->summary_addr;
1304 kroute.u.adapter.ind_addr = adapter->ind_addr;
1305 kroute.u.adapter.summary_offset = adapter->summary_offset;
1306 kroute.u.adapter.ind_offset = adapter->ind_offset;
1307 kroute.u.adapter.adapter_id = adapter->adapter_id;
1308
1309 kvm_add_routing_entry(s, &kroute);
1310 kvm_irqchip_commit_routes(s);
1311
1312 return virq;
1313}
1314
Jan Kiszka84b058d2011-10-15 11:49:47 +02001315#else /* !KVM_CAP_IRQ_ROUTING */
1316
Alexander Graf7b774592013-04-16 15:58:13 +02001317void kvm_init_irq_routing(KVMState *s)
Jan Kiszka84b058d2011-10-15 11:49:47 +02001318{
1319}
Jan Kiszka04fa27f2012-05-16 15:41:10 -03001320
Jan Kiszkad3d3bef2012-06-05 21:03:57 +02001321void kvm_irqchip_release_virq(KVMState *s, int virq)
1322{
1323}
1324
Jan Kiszka04fa27f2012-05-16 15:41:10 -03001325int kvm_irqchip_send_msi(KVMState *s, MSIMessage msg)
1326{
1327 abort();
1328}
Jan Kiszka92b4e482012-05-17 10:32:33 -03001329
1330int kvm_irqchip_add_msi_route(KVMState *s, MSIMessage msg)
1331{
Alex Williamsondf410672012-06-25 09:40:39 -06001332 return -ENOSYS;
Jan Kiszka92b4e482012-05-17 10:32:33 -03001333}
Jan Kiszka39853bb2012-05-17 10:32:36 -03001334
Cornelia Huckd426d9f2013-07-15 17:45:03 +02001335int kvm_irqchip_add_adapter_route(KVMState *s, AdapterInfo *adapter)
1336{
1337 return -ENOSYS;
1338}
1339
Jan Kiszka39853bb2012-05-17 10:32:36 -03001340static int kvm_irqchip_assign_irqfd(KVMState *s, int fd, int virq, bool assign)
1341{
1342 abort();
1343}
Michael S. Tsirkindabe3142013-01-15 19:50:13 +02001344
1345int kvm_irqchip_update_msi_route(KVMState *s, int virq, MSIMessage msg)
1346{
1347 return -ENOSYS;
1348}
Jan Kiszka84b058d2011-10-15 11:49:47 +02001349#endif /* !KVM_CAP_IRQ_ROUTING */
1350
Vincenzo Maffioneca916d32013-07-22 11:51:33 +02001351int kvm_irqchip_add_irqfd_notifier(KVMState *s, EventNotifier *n,
1352 EventNotifier *rn, int virq)
Jan Kiszka39853bb2012-05-17 10:32:36 -03001353{
Vincenzo Maffioneca916d32013-07-22 11:51:33 +02001354 return kvm_irqchip_assign_irqfd(s, event_notifier_get_fd(n),
1355 rn ? event_notifier_get_fd(rn) : -1, virq, true);
Jan Kiszka39853bb2012-05-17 10:32:36 -03001356}
1357
Jan Kiszkab131c742012-08-20 10:55:56 +02001358int kvm_irqchip_remove_irqfd_notifier(KVMState *s, EventNotifier *n, int virq)
Paolo Bonzini15b2bd12012-07-05 17:16:30 +02001359{
Vincenzo Maffioneca916d32013-07-22 11:51:33 +02001360 return kvm_irqchip_assign_irqfd(s, event_notifier_get_fd(n), -1, virq,
1361 false);
Paolo Bonzini15b2bd12012-07-05 17:16:30 +02001362}
1363
Marcel Apfelbaum446f16a2015-03-10 18:59:54 +02001364static int kvm_irqchip_create(MachineState *machine, KVMState *s)
Jan Kiszka84b058d2011-10-15 11:49:47 +02001365{
Jan Kiszka84b058d2011-10-15 11:49:47 +02001366 int ret;
1367
Marcel Apfelbaum446f16a2015-03-10 18:59:54 +02001368 if (!machine_kernel_irqchip_allowed(machine) ||
Cornelia Huckd426d9f2013-07-15 17:45:03 +02001369 (!kvm_check_extension(s, KVM_CAP_IRQCHIP) &&
1370 (kvm_vm_enable_cap(s, KVM_CAP_S390_IRQCHIP, 0) < 0))) {
Jan Kiszka84b058d2011-10-15 11:49:47 +02001371 return 0;
1372 }
1373
Christoffer Dalld6032e02014-02-26 17:20:00 +00001374 /* First probe and see if there's a arch-specific hook to create the
1375 * in-kernel irqchip for us */
1376 ret = kvm_arch_irqchip_create(s);
Jan Kiszka84b058d2011-10-15 11:49:47 +02001377 if (ret < 0) {
Jan Kiszka84b058d2011-10-15 11:49:47 +02001378 return ret;
Christoffer Dalld6032e02014-02-26 17:20:00 +00001379 } else if (ret == 0) {
1380 ret = kvm_vm_ioctl(s, KVM_CREATE_IRQCHIP);
1381 if (ret < 0) {
1382 fprintf(stderr, "Create kernel irqchip failed\n");
1383 return ret;
1384 }
Jan Kiszka84b058d2011-10-15 11:49:47 +02001385 }
1386
Jan Kiszka3d4b2642012-01-31 19:17:52 +01001387 kvm_kernel_irqchip = true;
Peter Maydell7ae26bd2012-07-26 15:35:11 +01001388 /* If we have an in-kernel IRQ chip then we must have asynchronous
1389 * interrupt delivery (though the reverse is not necessarily true)
1390 */
1391 kvm_async_interrupts_allowed = true;
Alexander Graf215e79c2013-04-24 22:24:12 +02001392 kvm_halt_in_kernel_allowed = true;
Jan Kiszka84b058d2011-10-15 11:49:47 +02001393
1394 kvm_init_irq_routing(s);
1395
1396 return 0;
1397}
1398
Andrew Jones670436c2013-08-23 15:24:37 +02001399/* Find number of supported CPUs using the recommended
1400 * procedure from the kernel API documentation to cope with
1401 * older kernels that may be missing capabilities.
1402 */
1403static int kvm_recommended_vcpus(KVMState *s)
1404{
1405 int ret = kvm_check_extension(s, KVM_CAP_NR_VCPUS);
1406 return (ret) ? ret : 4;
1407}
1408
Dunrong Huang3ed444e2012-07-31 19:18:17 +08001409static int kvm_max_vcpus(KVMState *s)
1410{
Andrew Jones670436c2013-08-23 15:24:37 +02001411 int ret = kvm_check_extension(s, KVM_CAP_MAX_VCPUS);
1412 return (ret) ? ret : kvm_recommended_vcpus(s);
Dunrong Huang3ed444e2012-07-31 19:18:17 +08001413}
1414
Eduardo Habkostf6a1ef62014-09-26 17:45:30 -03001415static int kvm_init(MachineState *ms)
aliguori05330442008-11-05 16:29:27 +00001416{
Eduardo Habkostf6a1ef62014-09-26 17:45:30 -03001417 MachineClass *mc = MACHINE_GET_CLASS(ms);
Jan Kiszka168ccc12009-06-07 11:30:25 +02001418 static const char upgrade_note[] =
1419 "Please upgrade to at least kernel 2.6.29 or recent kvm-kmod\n"
1420 "(see http://sourceforge.net/projects/kvm).\n";
Andrew Jones670436c2013-08-23 15:24:37 +02001421 struct {
1422 const char *name;
1423 int num;
1424 } num_cpus[] = {
1425 { "SMP", smp_cpus },
1426 { "hotpluggable", max_cpus },
1427 { NULL, }
1428 }, *nc = num_cpus;
1429 int soft_vcpus_limit, hard_vcpus_limit;
aliguori05330442008-11-05 16:29:27 +00001430 KVMState *s;
Jan Kiszka94a8d392011-01-21 21:48:17 +01001431 const KVMCapabilityInfo *missing_cap;
aliguori05330442008-11-05 16:29:27 +00001432 int ret;
Aneesh Kumar K.V135a1292013-12-23 21:10:40 +05301433 int i, type = 0;
1434 const char *kvm_type;
aliguori05330442008-11-05 16:29:27 +00001435
Eduardo Habkostfc020862014-09-26 17:45:32 -03001436 s = KVM_STATE(ms->accelerator);
aliguori05330442008-11-05 16:29:27 +00001437
David Gibson3145fcb2012-04-04 11:15:54 +10001438 /*
1439 * On systems where the kernel can support different base page
1440 * sizes, host page size may be different from TARGET_PAGE_SIZE,
1441 * even with KVM. TARGET_PAGE_SIZE is assumed to be the minimum
1442 * page size for the system though.
1443 */
1444 assert(TARGET_PAGE_SIZE <= getpagesize());
Alexey Kardashevskiy47c16ed2014-01-17 11:12:07 -07001445 page_size_init();
David Gibson3145fcb2012-04-04 11:15:54 +10001446
James Hoganaed6efb2014-06-17 23:10:31 +01001447 s->sigmask_len = 8;
1448
aliguorie22a25c2009-03-12 20:12:48 +00001449#ifdef KVM_CAP_SET_GUEST_DEBUG
Blue Swirl72cf2d42009-09-12 07:36:22 +00001450 QTAILQ_INIT(&s->kvm_sw_breakpoints);
aliguorie22a25c2009-03-12 20:12:48 +00001451#endif
aliguori05330442008-11-05 16:29:27 +00001452 s->vmfd = -1;
Kevin Wolf40ff6d72009-12-02 12:24:42 +01001453 s->fd = qemu_open("/dev/kvm", O_RDWR);
aliguori05330442008-11-05 16:29:27 +00001454 if (s->fd == -1) {
1455 fprintf(stderr, "Could not access KVM kernel module: %m\n");
1456 ret = -errno;
1457 goto err;
1458 }
1459
1460 ret = kvm_ioctl(s, KVM_GET_API_VERSION, 0);
1461 if (ret < KVM_API_VERSION) {
Eduardo Habkost0e1dac62014-05-30 17:26:22 -03001462 if (ret >= 0) {
aliguori05330442008-11-05 16:29:27 +00001463 ret = -EINVAL;
Jan Kiszkaa426e122011-01-04 09:32:13 +01001464 }
aliguori05330442008-11-05 16:29:27 +00001465 fprintf(stderr, "kvm version too old\n");
1466 goto err;
1467 }
1468
1469 if (ret > KVM_API_VERSION) {
1470 ret = -EINVAL;
1471 fprintf(stderr, "kvm version not supported\n");
1472 goto err;
1473 }
1474
Alex Williamsonfb541ca2013-11-22 12:12:44 -07001475 s->nr_slots = kvm_check_extension(s, KVM_CAP_NR_MEMSLOTS);
1476
1477 /* If unspecified, use the default value */
1478 if (!s->nr_slots) {
1479 s->nr_slots = 32;
1480 }
1481
1482 s->slots = g_malloc0(s->nr_slots * sizeof(KVMSlot));
1483
1484 for (i = 0; i < s->nr_slots; i++) {
1485 s->slots[i].slot = i;
1486 }
1487
Andrew Jones670436c2013-08-23 15:24:37 +02001488 /* check the vcpu limits */
1489 soft_vcpus_limit = kvm_recommended_vcpus(s);
1490 hard_vcpus_limit = kvm_max_vcpus(s);
Dunrong Huang3ed444e2012-07-31 19:18:17 +08001491
Andrew Jones670436c2013-08-23 15:24:37 +02001492 while (nc->name) {
1493 if (nc->num > soft_vcpus_limit) {
1494 fprintf(stderr,
1495 "Warning: Number of %s cpus requested (%d) exceeds "
1496 "the recommended cpus supported by KVM (%d)\n",
1497 nc->name, nc->num, soft_vcpus_limit);
1498
1499 if (nc->num > hard_vcpus_limit) {
Andrew Jones670436c2013-08-23 15:24:37 +02001500 fprintf(stderr, "Number of %s cpus requested (%d) exceeds "
1501 "the maximum cpus supported by KVM (%d)\n",
1502 nc->name, nc->num, hard_vcpus_limit);
Marcelo Tosatti9ba3cf52014-02-25 23:22:07 -03001503 exit(1);
Andrew Jones670436c2013-08-23 15:24:37 +02001504 }
1505 }
1506 nc++;
Marcelo Tosatti7dc52522013-08-12 16:56:31 -03001507 }
1508
Aneesh Kumar K.V135a1292013-12-23 21:10:40 +05301509 kvm_type = qemu_opt_get(qemu_get_machine_opts(), "kvm-type");
Marcel Apfelbaumf1e29872014-04-09 20:34:52 +03001510 if (mc->kvm_type) {
1511 type = mc->kvm_type(kvm_type);
Aneesh Kumar K.V135a1292013-12-23 21:10:40 +05301512 } else if (kvm_type) {
Eduardo Habkost0e1dac62014-05-30 17:26:22 -03001513 ret = -EINVAL;
Aneesh Kumar K.V135a1292013-12-23 21:10:40 +05301514 fprintf(stderr, "Invalid argument kvm-type=%s\n", kvm_type);
1515 goto err;
1516 }
1517
thomas knych94ccff12014-01-09 13:14:23 -08001518 do {
Aneesh Kumar K.V135a1292013-12-23 21:10:40 +05301519 ret = kvm_ioctl(s, KVM_CREATE_VM, type);
thomas knych94ccff12014-01-09 13:14:23 -08001520 } while (ret == -EINTR);
1521
1522 if (ret < 0) {
Alexander Graf521f4382014-01-27 15:18:09 +01001523 fprintf(stderr, "ioctl(KVM_CREATE_VM) failed: %d %s\n", -ret,
thomas knych94ccff12014-01-09 13:14:23 -08001524 strerror(-ret));
1525
Alexander Graf0104dca2010-04-01 18:42:37 +02001526#ifdef TARGET_S390X
1527 fprintf(stderr, "Please add the 'switch_amode' kernel parameter to "
1528 "your host kernel command line\n");
1529#endif
aliguori05330442008-11-05 16:29:27 +00001530 goto err;
Alexander Graf0104dca2010-04-01 18:42:37 +02001531 }
aliguori05330442008-11-05 16:29:27 +00001532
thomas knych94ccff12014-01-09 13:14:23 -08001533 s->vmfd = ret;
Jan Kiszka94a8d392011-01-21 21:48:17 +01001534 missing_cap = kvm_check_extension_list(s, kvm_required_capabilites);
1535 if (!missing_cap) {
1536 missing_cap =
1537 kvm_check_extension_list(s, kvm_arch_required_capabilities);
1538 }
1539 if (missing_cap) {
Anthony Liguoriad7b8b32009-05-08 15:33:24 -05001540 ret = -EINVAL;
Jan Kiszka94a8d392011-01-21 21:48:17 +01001541 fprintf(stderr, "kvm does not support %s\n%s",
1542 missing_cap->name, upgrade_note);
aliguori05330442008-11-05 16:29:27 +00001543 goto err;
1544 }
1545
Anthony Liguoriad7b8b32009-05-08 15:33:24 -05001546 s->coalesced_mmio = kvm_check_extension(s, KVM_CAP_COALESCED_MMIO);
aliguorif65ed4c2008-12-09 20:09:57 +00001547
Jan Kiszkae69917e2009-05-01 20:42:15 +02001548 s->broken_set_mem_region = 1;
Lai Jiangshan14a09512010-12-10 15:52:36 +08001549 ret = kvm_check_extension(s, KVM_CAP_JOIN_MEMORY_REGIONS_WORKS);
Jan Kiszkae69917e2009-05-01 20:42:15 +02001550 if (ret > 0) {
1551 s->broken_set_mem_region = 0;
1552 }
Jan Kiszkae69917e2009-05-01 20:42:15 +02001553
Jan Kiszkaa0fb0022009-11-25 00:33:03 +01001554#ifdef KVM_CAP_VCPU_EVENTS
1555 s->vcpu_events = kvm_check_extension(s, KVM_CAP_VCPU_EVENTS);
1556#endif
1557
Jan Kiszkab0b1d692010-03-01 19:10:29 +01001558 s->robust_singlestep =
1559 kvm_check_extension(s, KVM_CAP_X86_ROBUST_SINGLESTEP);
Jan Kiszkab0b1d692010-03-01 19:10:29 +01001560
Jan Kiszkaff44f1a2010-03-12 15:20:49 +01001561#ifdef KVM_CAP_DEBUGREGS
1562 s->debugregs = kvm_check_extension(s, KVM_CAP_DEBUGREGS);
1563#endif
1564
Sheng Yangf1665b22010-06-17 17:53:07 +08001565#ifdef KVM_CAP_XSAVE
1566 s->xsave = kvm_check_extension(s, KVM_CAP_XSAVE);
1567#endif
1568
Sheng Yangf1665b22010-06-17 17:53:07 +08001569#ifdef KVM_CAP_XCRS
1570 s->xcrs = kvm_check_extension(s, KVM_CAP_XCRS);
1571#endif
1572
Jan Kiszka8a7c7392012-03-02 20:28:48 +01001573#ifdef KVM_CAP_PIT_STATE2
1574 s->pit_state2 = kvm_check_extension(s, KVM_CAP_PIT_STATE2);
1575#endif
1576
Jan Kiszkad3d3bef2012-06-05 21:03:57 +02001577#ifdef KVM_CAP_IRQ_ROUTING
Jan Kiszka4a3adeb2012-05-16 15:41:14 -03001578 s->direct_msi = (kvm_check_extension(s, KVM_CAP_SIGNAL_MSI) > 0);
Jan Kiszkad3d3bef2012-06-05 21:03:57 +02001579#endif
Jan Kiszka4a3adeb2012-05-16 15:41:14 -03001580
Jan Kiszka3ab73842012-08-27 08:28:39 +02001581 s->intx_set_mask = kvm_check_extension(s, KVM_CAP_PCI_2_3);
1582
Jan Kiszkae333cd62012-08-24 13:34:47 +02001583 s->irq_set_ioctl = KVM_IRQ_LINE;
Peter Maydell8732fbd2012-08-15 12:08:13 +01001584 if (kvm_check_extension(s, KVM_CAP_IRQ_INJECT_STATUS)) {
Jan Kiszkae333cd62012-08-24 13:34:47 +02001585 s->irq_set_ioctl = KVM_IRQ_LINE_STATUS;
Peter Maydell8732fbd2012-08-15 12:08:13 +01001586 }
1587
Jordan Justendf9c8b72013-05-29 01:27:25 -07001588#ifdef KVM_CAP_READONLY_MEM
1589 kvm_readonly_mem_allowed =
1590 (kvm_check_extension(s, KVM_CAP_READONLY_MEM) > 0);
1591#endif
1592
Nikolay Nikolaev69e03ae2014-05-27 15:03:35 +03001593 kvm_eventfds_allowed =
1594 (kvm_check_extension(s, KVM_CAP_IOEVENTFD) > 0);
1595
Eric Augerf41389a2014-10-31 13:38:18 +00001596 kvm_irqfds_allowed =
1597 (kvm_check_extension(s, KVM_CAP_IRQFD) > 0);
1598
1599 kvm_resamplefds_allowed =
1600 (kvm_check_extension(s, KVM_CAP_IRQFD_RESAMPLE) > 0);
1601
Dominik Dingeld0a073a2015-03-12 13:53:49 +01001602 kvm_vm_attributes_allowed =
1603 (kvm_check_extension(s, KVM_CAP_VM_ATTRIBUTES) > 0);
1604
Marcel Apfelbaumb16565b2015-02-04 17:43:51 +02001605 ret = kvm_arch_init(ms, s);
Jan Kiszkaa426e122011-01-04 09:32:13 +01001606 if (ret < 0) {
aliguori05330442008-11-05 16:29:27 +00001607 goto err;
Jan Kiszkaa426e122011-01-04 09:32:13 +01001608 }
aliguori05330442008-11-05 16:29:27 +00001609
Marcel Apfelbaum446f16a2015-03-10 18:59:54 +02001610 ret = kvm_irqchip_create(ms, s);
Jan Kiszka84b058d2011-10-15 11:49:47 +02001611 if (ret < 0) {
1612 goto err;
1613 }
1614
aliguori05330442008-11-05 16:29:27 +00001615 kvm_state = s;
Avi Kivityf6790af2012-10-02 20:13:51 +02001616 memory_listener_register(&kvm_memory_listener, &address_space_memory);
1617 memory_listener_register(&kvm_io_listener, &address_space_io);
aliguori05330442008-11-05 16:29:27 +00001618
Stefan Hajnoczid2f2b8a2011-01-10 13:50:05 +02001619 s->many_ioeventfds = kvm_check_many_ioeventfds();
1620
Jan Kiszkaaa7f74d2011-04-13 01:32:56 +02001621 cpu_interrupt_handler = kvm_handle_interrupt;
1622
aliguori05330442008-11-05 16:29:27 +00001623 return 0;
1624
1625err:
Eduardo Habkost0e1dac62014-05-30 17:26:22 -03001626 assert(ret < 0);
Stefan Weil6d1cc322012-09-03 22:40:40 +02001627 if (s->vmfd >= 0) {
1628 close(s->vmfd);
1629 }
1630 if (s->fd != -1) {
1631 close(s->fd);
aliguori05330442008-11-05 16:29:27 +00001632 }
Alex Williamsonfb541ca2013-11-22 12:12:44 -07001633 g_free(s->slots);
aliguori05330442008-11-05 16:29:27 +00001634
1635 return ret;
1636}
1637
James Hoganaed6efb2014-06-17 23:10:31 +01001638void kvm_set_sigmask_len(KVMState *s, unsigned int sigmask_len)
1639{
1640 s->sigmask_len = sigmask_len;
1641}
1642
Jan Kiszkab30e93e2011-02-01 22:16:01 +01001643static void kvm_handle_io(uint16_t port, void *data, int direction, int size,
1644 uint32_t count)
aliguori05330442008-11-05 16:29:27 +00001645{
1646 int i;
1647 uint8_t *ptr = data;
1648
1649 for (i = 0; i < count; i++) {
Jan Kiszka354678c2013-08-13 14:43:57 +02001650 address_space_rw(&address_space_io, port, ptr, size,
1651 direction == KVM_EXIT_IO_OUT);
aliguori05330442008-11-05 16:29:27 +00001652 ptr += size;
1653 }
aliguori05330442008-11-05 16:29:27 +00001654}
1655
Andreas Färber5326ab52013-05-27 01:55:29 +02001656static int kvm_handle_internal_error(CPUState *cpu, struct kvm_run *run)
Marcelo Tosatti7c80eef2010-03-23 13:37:11 -03001657{
Radim Krčmář977c7b62014-01-21 18:11:31 +01001658 fprintf(stderr, "KVM internal error. Suberror: %d\n",
1659 run->internal.suberror);
1660
Marcelo Tosatti7c80eef2010-03-23 13:37:11 -03001661 if (kvm_check_extension(kvm_state, KVM_CAP_INTERNAL_ERROR_DATA)) {
1662 int i;
1663
Marcelo Tosatti7c80eef2010-03-23 13:37:11 -03001664 for (i = 0; i < run->internal.ndata; ++i) {
1665 fprintf(stderr, "extra data[%d]: %"PRIx64"\n",
1666 i, (uint64_t)run->internal.data[i]);
1667 }
1668 }
Marcelo Tosatti7c80eef2010-03-23 13:37:11 -03001669 if (run->internal.suberror == KVM_INTERNAL_ERROR_EMULATION) {
1670 fprintf(stderr, "emulation failure\n");
Andreas Färber20d695a2012-10-31 06:57:49 +01001671 if (!kvm_arch_stop_on_emulation_error(cpu)) {
Andreas Färber878096e2013-05-27 01:33:50 +02001672 cpu_dump_state(cpu, stderr, fprintf, CPU_DUMP_CODE);
Jan Kiszkad73cd8f2011-03-15 12:26:27 +01001673 return EXCP_INTERRUPT;
Jan Kiszkaa426e122011-01-04 09:32:13 +01001674 }
Marcelo Tosatti7c80eef2010-03-23 13:37:11 -03001675 }
1676 /* FIXME: Should trigger a qmp message to let management know
1677 * something went wrong.
1678 */
Jan Kiszka73aaec42011-01-21 21:48:06 +01001679 return -1;
Marcelo Tosatti7c80eef2010-03-23 13:37:11 -03001680}
Marcelo Tosatti7c80eef2010-03-23 13:37:11 -03001681
Sheng Yang62a27442010-01-26 19:21:16 +08001682void kvm_flush_coalesced_mmio_buffer(void)
aliguorif65ed4c2008-12-09 20:09:57 +00001683{
aliguorif65ed4c2008-12-09 20:09:57 +00001684 KVMState *s = kvm_state;
Avi Kivity1cae88b2011-10-18 19:43:12 +02001685
1686 if (s->coalesced_flush_in_progress) {
1687 return;
1688 }
1689
1690 s->coalesced_flush_in_progress = true;
1691
Sheng Yang62a27442010-01-26 19:21:16 +08001692 if (s->coalesced_mmio_ring) {
1693 struct kvm_coalesced_mmio_ring *ring = s->coalesced_mmio_ring;
aliguorif65ed4c2008-12-09 20:09:57 +00001694 while (ring->first != ring->last) {
1695 struct kvm_coalesced_mmio *ent;
1696
1697 ent = &ring->coalesced_mmio[ring->first];
1698
1699 cpu_physical_memory_write(ent->phys_addr, ent->data, ent->len);
Marcelo Tosatti85199472010-02-22 13:57:54 -03001700 smp_wmb();
aliguorif65ed4c2008-12-09 20:09:57 +00001701 ring->first = (ring->first + 1) % KVM_COALESCED_MMIO_MAX;
1702 }
1703 }
Avi Kivity1cae88b2011-10-18 19:43:12 +02001704
1705 s->coalesced_flush_in_progress = false;
aliguorif65ed4c2008-12-09 20:09:57 +00001706}
1707
Andreas Färber20d695a2012-10-31 06:57:49 +01001708static void do_kvm_cpu_synchronize_state(void *arg)
Avi Kivity4c0960c2009-08-17 23:19:53 +03001709{
Andreas Färber20d695a2012-10-31 06:57:49 +01001710 CPUState *cpu = arg;
Jan Kiszka2705d562010-05-04 09:45:23 -03001711
Andreas Färber20d695a2012-10-31 06:57:49 +01001712 if (!cpu->kvm_vcpu_dirty) {
1713 kvm_arch_get_registers(cpu);
1714 cpu->kvm_vcpu_dirty = true;
Avi Kivity4c0960c2009-08-17 23:19:53 +03001715 }
1716}
1717
Andreas Färberdd1750d2013-05-01 13:45:44 +02001718void kvm_cpu_synchronize_state(CPUState *cpu)
Jan Kiszka2705d562010-05-04 09:45:23 -03001719{
Andreas Färber20d695a2012-10-31 06:57:49 +01001720 if (!cpu->kvm_vcpu_dirty) {
1721 run_on_cpu(cpu, do_kvm_cpu_synchronize_state, cpu);
Jan Kiszkaa426e122011-01-04 09:32:13 +01001722 }
Jan Kiszka2705d562010-05-04 09:45:23 -03001723}
1724
David Hildenbrandc8e20852014-08-20 14:55:25 +02001725static void do_kvm_cpu_synchronize_post_reset(void *arg)
1726{
1727 CPUState *cpu = arg;
1728
1729 kvm_arch_put_registers(cpu, KVM_PUT_RESET_STATE);
1730 cpu->kvm_vcpu_dirty = false;
1731}
1732
Igor Mammedov3f24a582013-04-11 16:51:41 +02001733void kvm_cpu_synchronize_post_reset(CPUState *cpu)
Jan Kiszkaea375f92010-03-01 19:10:30 +01001734{
David Hildenbrandc8e20852014-08-20 14:55:25 +02001735 run_on_cpu(cpu, do_kvm_cpu_synchronize_post_reset, cpu);
1736}
1737
1738static void do_kvm_cpu_synchronize_post_init(void *arg)
1739{
1740 CPUState *cpu = arg;
1741
1742 kvm_arch_put_registers(cpu, KVM_PUT_FULL_STATE);
Andreas Färber20d695a2012-10-31 06:57:49 +01001743 cpu->kvm_vcpu_dirty = false;
Jan Kiszkaea375f92010-03-01 19:10:30 +01001744}
1745
Igor Mammedov3f24a582013-04-11 16:51:41 +02001746void kvm_cpu_synchronize_post_init(CPUState *cpu)
Jan Kiszkaea375f92010-03-01 19:10:30 +01001747{
David Hildenbrandc8e20852014-08-20 14:55:25 +02001748 run_on_cpu(cpu, do_kvm_cpu_synchronize_post_init, cpu);
Jan Kiszkaea375f92010-03-01 19:10:30 +01001749}
1750
Marcelo Tosattide9d61e2014-09-05 10:52:46 -03001751void kvm_cpu_clean_state(CPUState *cpu)
1752{
1753 cpu->kvm_vcpu_dirty = false;
1754}
1755
Andreas Färber1458c362013-05-26 23:46:55 +02001756int kvm_cpu_exec(CPUState *cpu)
aliguori05330442008-11-05 16:29:27 +00001757{
Andreas Färberf7575c962012-12-01 06:18:14 +01001758 struct kvm_run *run = cpu->kvm_run;
Jan Kiszka7cbb5332011-03-15 12:26:25 +01001759 int ret, run_ret;
aliguori05330442008-11-05 16:29:27 +00001760
Blue Swirl8c0d5772010-04-18 14:22:14 +00001761 DPRINTF("kvm_cpu_exec()\n");
aliguori05330442008-11-05 16:29:27 +00001762
Andreas Färber20d695a2012-10-31 06:57:49 +01001763 if (kvm_arch_process_async_events(cpu)) {
Andreas Färberfcd7d002012-12-17 08:02:44 +01001764 cpu->exit_request = 0;
Jan Kiszka6792a572011-02-07 12:19:18 +01001765 return EXCP_HLT;
Jan Kiszka9ccfac92011-02-01 22:16:00 +01001766 }
1767
aliguori05330442008-11-05 16:29:27 +00001768 do {
Andreas Färber20d695a2012-10-31 06:57:49 +01001769 if (cpu->kvm_vcpu_dirty) {
1770 kvm_arch_put_registers(cpu, KVM_PUT_RUNTIME_STATE);
1771 cpu->kvm_vcpu_dirty = false;
Avi Kivity4c0960c2009-08-17 23:19:53 +03001772 }
1773
Andreas Färber20d695a2012-10-31 06:57:49 +01001774 kvm_arch_pre_run(cpu, run);
Andreas Färberfcd7d002012-12-17 08:02:44 +01001775 if (cpu->exit_request) {
Jan Kiszka9ccfac92011-02-01 22:16:00 +01001776 DPRINTF("interrupt exit requested\n");
1777 /*
1778 * KVM requires us to reenter the kernel after IO exits to complete
1779 * instruction emulation. This self-signal will ensure that we
1780 * leave ASAP again.
1781 */
1782 qemu_cpu_kick_self();
1783 }
Glauber Costad549db52009-10-07 16:38:03 -03001784 qemu_mutex_unlock_iothread();
Jan Kiszka9ccfac92011-02-01 22:16:00 +01001785
Andreas Färber1bc22652012-10-31 06:06:49 +01001786 run_ret = kvm_vcpu_ioctl(cpu, KVM_RUN, 0);
Jan Kiszka9ccfac92011-02-01 22:16:00 +01001787
Glauber Costad549db52009-10-07 16:38:03 -03001788 qemu_mutex_lock_iothread();
Andreas Färber20d695a2012-10-31 06:57:49 +01001789 kvm_arch_post_run(cpu, run);
aliguori05330442008-11-05 16:29:27 +00001790
Jan Kiszka7cbb5332011-03-15 12:26:25 +01001791 if (run_ret < 0) {
Jan Kiszkadc77d342011-03-15 12:26:26 +01001792 if (run_ret == -EINTR || run_ret == -EAGAIN) {
1793 DPRINTF("io window exit\n");
Jan Kiszkad73cd8f2011-03-15 12:26:27 +01001794 ret = EXCP_INTERRUPT;
Jan Kiszkadc77d342011-03-15 12:26:26 +01001795 break;
1796 }
Michael Ellerman7b011fb2011-12-16 11:20:20 +11001797 fprintf(stderr, "error: kvm run failed %s\n",
1798 strerror(-run_ret));
Paolo Bonzinia85e1302014-08-29 15:58:20 +02001799 ret = -1;
1800 break;
aliguori05330442008-11-05 16:29:27 +00001801 }
1802
Kazuya Saitob76ac802013-03-29 13:27:52 +09001803 trace_kvm_run_exit(cpu->cpu_index, run->exit_reason);
aliguori05330442008-11-05 16:29:27 +00001804 switch (run->exit_reason) {
1805 case KVM_EXIT_IO:
Blue Swirl8c0d5772010-04-18 14:22:14 +00001806 DPRINTF("handle_io\n");
Jan Kiszkab30e93e2011-02-01 22:16:01 +01001807 kvm_handle_io(run->io.port,
1808 (uint8_t *)run + run->io.data_offset,
1809 run->io.direction,
1810 run->io.size,
1811 run->io.count);
Jan Kiszkad73cd8f2011-03-15 12:26:27 +01001812 ret = 0;
aliguori05330442008-11-05 16:29:27 +00001813 break;
1814 case KVM_EXIT_MMIO:
Blue Swirl8c0d5772010-04-18 14:22:14 +00001815 DPRINTF("handle_mmio\n");
aliguori05330442008-11-05 16:29:27 +00001816 cpu_physical_memory_rw(run->mmio.phys_addr,
1817 run->mmio.data,
1818 run->mmio.len,
1819 run->mmio.is_write);
Jan Kiszkad73cd8f2011-03-15 12:26:27 +01001820 ret = 0;
aliguori05330442008-11-05 16:29:27 +00001821 break;
1822 case KVM_EXIT_IRQ_WINDOW_OPEN:
Blue Swirl8c0d5772010-04-18 14:22:14 +00001823 DPRINTF("irq_window_open\n");
Jan Kiszkad73cd8f2011-03-15 12:26:27 +01001824 ret = EXCP_INTERRUPT;
aliguori05330442008-11-05 16:29:27 +00001825 break;
1826 case KVM_EXIT_SHUTDOWN:
Blue Swirl8c0d5772010-04-18 14:22:14 +00001827 DPRINTF("shutdown\n");
aliguori05330442008-11-05 16:29:27 +00001828 qemu_system_reset_request();
Jan Kiszkad73cd8f2011-03-15 12:26:27 +01001829 ret = EXCP_INTERRUPT;
aliguori05330442008-11-05 16:29:27 +00001830 break;
1831 case KVM_EXIT_UNKNOWN:
Jan Kiszkabb44e0d2011-01-21 21:48:07 +01001832 fprintf(stderr, "KVM: unknown exit, hardware reason %" PRIx64 "\n",
1833 (uint64_t)run->hw.hardware_exit_reason);
Jan Kiszka73aaec42011-01-21 21:48:06 +01001834 ret = -1;
aliguori05330442008-11-05 16:29:27 +00001835 break;
Marcelo Tosatti7c80eef2010-03-23 13:37:11 -03001836 case KVM_EXIT_INTERNAL_ERROR:
Andreas Färber5326ab52013-05-27 01:55:29 +02001837 ret = kvm_handle_internal_error(cpu, run);
Marcelo Tosatti7c80eef2010-03-23 13:37:11 -03001838 break;
Pranavkumar Sawargaonkar99040442014-06-19 18:06:25 +01001839 case KVM_EXIT_SYSTEM_EVENT:
1840 switch (run->system_event.type) {
1841 case KVM_SYSTEM_EVENT_SHUTDOWN:
1842 qemu_system_shutdown_request();
1843 ret = EXCP_INTERRUPT;
1844 break;
1845 case KVM_SYSTEM_EVENT_RESET:
1846 qemu_system_reset_request();
1847 ret = EXCP_INTERRUPT;
1848 break;
1849 default:
1850 DPRINTF("kvm_arch_handle_exit\n");
1851 ret = kvm_arch_handle_exit(cpu, run);
1852 break;
1853 }
1854 break;
aliguori05330442008-11-05 16:29:27 +00001855 default:
Blue Swirl8c0d5772010-04-18 14:22:14 +00001856 DPRINTF("kvm_arch_handle_exit\n");
Andreas Färber20d695a2012-10-31 06:57:49 +01001857 ret = kvm_arch_handle_exit(cpu, run);
aliguori05330442008-11-05 16:29:27 +00001858 break;
1859 }
Jan Kiszkad73cd8f2011-03-15 12:26:27 +01001860 } while (ret == 0);
aliguori05330442008-11-05 16:29:27 +00001861
Jan Kiszka73aaec42011-01-21 21:48:06 +01001862 if (ret < 0) {
Andreas Färber878096e2013-05-27 01:33:50 +02001863 cpu_dump_state(cpu, stderr, fprintf, CPU_DUMP_CODE);
Luiz Capitulino0461d5a2011-09-30 14:45:27 -03001864 vm_stop(RUN_STATE_INTERNAL_ERROR);
Jan Kiszka73aaec42011-01-21 21:48:06 +01001865 }
aliguoribecfc392008-11-10 15:55:14 +00001866
Andreas Färberfcd7d002012-12-17 08:02:44 +01001867 cpu->exit_request = 0;
aliguori05330442008-11-05 16:29:27 +00001868 return ret;
1869}
1870
aliguori984b5182008-11-13 19:21:00 +00001871int kvm_ioctl(KVMState *s, int type, ...)
aliguori05330442008-11-05 16:29:27 +00001872{
1873 int ret;
aliguori984b5182008-11-13 19:21:00 +00001874 void *arg;
1875 va_list ap;
aliguori05330442008-11-05 16:29:27 +00001876
aliguori984b5182008-11-13 19:21:00 +00001877 va_start(ap, type);
1878 arg = va_arg(ap, void *);
1879 va_end(ap);
1880
Kazuya Saito9c775722013-03-29 13:27:05 +09001881 trace_kvm_ioctl(type, arg);
aliguori984b5182008-11-13 19:21:00 +00001882 ret = ioctl(s->fd, type, arg);
Jan Kiszkaa426e122011-01-04 09:32:13 +01001883 if (ret == -1) {
aliguori05330442008-11-05 16:29:27 +00001884 ret = -errno;
Jan Kiszkaa426e122011-01-04 09:32:13 +01001885 }
aliguori05330442008-11-05 16:29:27 +00001886 return ret;
1887}
1888
aliguori984b5182008-11-13 19:21:00 +00001889int kvm_vm_ioctl(KVMState *s, int type, ...)
aliguori05330442008-11-05 16:29:27 +00001890{
1891 int ret;
aliguori984b5182008-11-13 19:21:00 +00001892 void *arg;
1893 va_list ap;
aliguori05330442008-11-05 16:29:27 +00001894
aliguori984b5182008-11-13 19:21:00 +00001895 va_start(ap, type);
1896 arg = va_arg(ap, void *);
1897 va_end(ap);
1898
Kazuya Saito9c775722013-03-29 13:27:05 +09001899 trace_kvm_vm_ioctl(type, arg);
aliguori984b5182008-11-13 19:21:00 +00001900 ret = ioctl(s->vmfd, type, arg);
Jan Kiszkaa426e122011-01-04 09:32:13 +01001901 if (ret == -1) {
aliguori05330442008-11-05 16:29:27 +00001902 ret = -errno;
Jan Kiszkaa426e122011-01-04 09:32:13 +01001903 }
aliguori05330442008-11-05 16:29:27 +00001904 return ret;
1905}
1906
Andreas Färber1bc22652012-10-31 06:06:49 +01001907int kvm_vcpu_ioctl(CPUState *cpu, int type, ...)
aliguori05330442008-11-05 16:29:27 +00001908{
1909 int ret;
aliguori984b5182008-11-13 19:21:00 +00001910 void *arg;
1911 va_list ap;
aliguori05330442008-11-05 16:29:27 +00001912
aliguori984b5182008-11-13 19:21:00 +00001913 va_start(ap, type);
1914 arg = va_arg(ap, void *);
1915 va_end(ap);
1916
Kazuya Saito9c775722013-03-29 13:27:05 +09001917 trace_kvm_vcpu_ioctl(cpu->cpu_index, type, arg);
Andreas Färber8737c512012-10-31 05:29:00 +01001918 ret = ioctl(cpu->kvm_fd, type, arg);
Jan Kiszkaa426e122011-01-04 09:32:13 +01001919 if (ret == -1) {
aliguori05330442008-11-05 16:29:27 +00001920 ret = -errno;
Jan Kiszkaa426e122011-01-04 09:32:13 +01001921 }
aliguori05330442008-11-05 16:29:27 +00001922 return ret;
1923}
aliguoribd322082008-12-04 20:33:06 +00001924
Christoffer Dall0a6a7cc2014-02-26 17:20:00 +00001925int kvm_device_ioctl(int fd, int type, ...)
1926{
1927 int ret;
1928 void *arg;
1929 va_list ap;
1930
1931 va_start(ap, type);
1932 arg = va_arg(ap, void *);
1933 va_end(ap);
1934
1935 trace_kvm_device_ioctl(fd, type, arg);
1936 ret = ioctl(fd, type, arg);
1937 if (ret == -1) {
1938 ret = -errno;
1939 }
1940 return ret;
1941}
1942
Dominik Dingeld0a073a2015-03-12 13:53:49 +01001943int kvm_vm_check_attr(KVMState *s, uint32_t group, uint64_t attr)
1944{
1945 int ret;
1946 struct kvm_device_attr attribute = {
1947 .group = group,
1948 .attr = attr,
1949 };
1950
1951 if (!kvm_vm_attributes_allowed) {
1952 return 0;
1953 }
1954
1955 ret = kvm_vm_ioctl(s, KVM_HAS_DEVICE_ATTR, &attribute);
1956 /* kvm returns 0 on success for HAS_DEVICE_ATTR */
1957 return ret ? 0 : 1;
1958}
1959
aliguoribd322082008-12-04 20:33:06 +00001960int kvm_has_sync_mmu(void)
1961{
Jan Kiszka94a8d392011-01-21 21:48:17 +01001962 return kvm_check_extension(kvm_state, KVM_CAP_SYNC_MMU);
aliguoribd322082008-12-04 20:33:06 +00001963}
aliguorie22a25c2009-03-12 20:12:48 +00001964
Jan Kiszkaa0fb0022009-11-25 00:33:03 +01001965int kvm_has_vcpu_events(void)
1966{
1967 return kvm_state->vcpu_events;
1968}
1969
Jan Kiszkab0b1d692010-03-01 19:10:29 +01001970int kvm_has_robust_singlestep(void)
1971{
1972 return kvm_state->robust_singlestep;
1973}
1974
Jan Kiszkaff44f1a2010-03-12 15:20:49 +01001975int kvm_has_debugregs(void)
1976{
1977 return kvm_state->debugregs;
1978}
1979
Sheng Yangf1665b22010-06-17 17:53:07 +08001980int kvm_has_xsave(void)
1981{
1982 return kvm_state->xsave;
1983}
1984
1985int kvm_has_xcrs(void)
1986{
1987 return kvm_state->xcrs;
1988}
1989
Jan Kiszka8a7c7392012-03-02 20:28:48 +01001990int kvm_has_pit_state2(void)
1991{
1992 return kvm_state->pit_state2;
1993}
1994
Stefan Hajnoczid2f2b8a2011-01-10 13:50:05 +02001995int kvm_has_many_ioeventfds(void)
1996{
1997 if (!kvm_enabled()) {
1998 return 0;
1999 }
2000 return kvm_state->many_ioeventfds;
2001}
2002
Jan Kiszka84b058d2011-10-15 11:49:47 +02002003int kvm_has_gsi_routing(void)
2004{
Alexander Grafa9c5eb02012-01-25 18:28:05 +01002005#ifdef KVM_CAP_IRQ_ROUTING
Jan Kiszka84b058d2011-10-15 11:49:47 +02002006 return kvm_check_extension(kvm_state, KVM_CAP_IRQ_ROUTING);
Alexander Grafa9c5eb02012-01-25 18:28:05 +01002007#else
2008 return false;
2009#endif
Jan Kiszka84b058d2011-10-15 11:49:47 +02002010}
2011
Jan Kiszka3ab73842012-08-27 08:28:39 +02002012int kvm_has_intx_set_mask(void)
2013{
2014 return kvm_state->intx_set_mask;
2015}
2016
Jan Kiszka6f0437e2009-04-26 18:03:40 +02002017void kvm_setup_guest_memory(void *start, size_t size)
2018{
2019 if (!kvm_has_sync_mmu()) {
Andreas Färbere78815a2010-09-25 11:26:05 +00002020 int ret = qemu_madvise(start, size, QEMU_MADV_DONTFORK);
Jan Kiszka6f0437e2009-04-26 18:03:40 +02002021
2022 if (ret) {
Andreas Färbere78815a2010-09-25 11:26:05 +00002023 perror("qemu_madvise");
2024 fprintf(stderr,
2025 "Need MADV_DONTFORK in absence of synchronous KVM MMU\n");
Jan Kiszka6f0437e2009-04-26 18:03:40 +02002026 exit(1);
2027 }
Jan Kiszka6f0437e2009-04-26 18:03:40 +02002028 }
2029}
2030
aliguorie22a25c2009-03-12 20:12:48 +00002031#ifdef KVM_CAP_SET_GUEST_DEBUG
Andreas Färbera60f24b2012-12-01 05:35:08 +01002032struct kvm_sw_breakpoint *kvm_find_sw_breakpoint(CPUState *cpu,
aliguorie22a25c2009-03-12 20:12:48 +00002033 target_ulong pc)
2034{
2035 struct kvm_sw_breakpoint *bp;
2036
Andreas Färbera60f24b2012-12-01 05:35:08 +01002037 QTAILQ_FOREACH(bp, &cpu->kvm_state->kvm_sw_breakpoints, entry) {
Jan Kiszkaa426e122011-01-04 09:32:13 +01002038 if (bp->pc == pc) {
aliguorie22a25c2009-03-12 20:12:48 +00002039 return bp;
Jan Kiszkaa426e122011-01-04 09:32:13 +01002040 }
aliguorie22a25c2009-03-12 20:12:48 +00002041 }
2042 return NULL;
2043}
2044
Andreas Färbera60f24b2012-12-01 05:35:08 +01002045int kvm_sw_breakpoints_active(CPUState *cpu)
aliguorie22a25c2009-03-12 20:12:48 +00002046{
Andreas Färbera60f24b2012-12-01 05:35:08 +01002047 return !QTAILQ_EMPTY(&cpu->kvm_state->kvm_sw_breakpoints);
aliguorie22a25c2009-03-12 20:12:48 +00002048}
2049
Glauber Costa452e4752009-07-16 17:55:28 -04002050struct kvm_set_guest_debug_data {
2051 struct kvm_guest_debug dbg;
Andreas Färbera60f24b2012-12-01 05:35:08 +01002052 CPUState *cpu;
Glauber Costa452e4752009-07-16 17:55:28 -04002053 int err;
2054};
2055
2056static void kvm_invoke_set_guest_debug(void *data)
2057{
2058 struct kvm_set_guest_debug_data *dbg_data = data;
Jan Kiszkab3807722009-09-17 20:05:58 +02002059
Andreas Färbera60f24b2012-12-01 05:35:08 +01002060 dbg_data->err = kvm_vcpu_ioctl(dbg_data->cpu, KVM_SET_GUEST_DEBUG,
2061 &dbg_data->dbg);
Glauber Costa452e4752009-07-16 17:55:28 -04002062}
2063
Stefan Weil38e478e2013-07-25 20:50:21 +02002064int kvm_update_guest_debug(CPUState *cpu, unsigned long reinject_trap)
aliguorie22a25c2009-03-12 20:12:48 +00002065{
Glauber Costa452e4752009-07-16 17:55:28 -04002066 struct kvm_set_guest_debug_data data;
aliguorie22a25c2009-03-12 20:12:48 +00002067
Jan Kiszkab0b1d692010-03-01 19:10:29 +01002068 data.dbg.control = reinject_trap;
aliguorie22a25c2009-03-12 20:12:48 +00002069
Andreas Färbered2803d2013-06-21 20:20:45 +02002070 if (cpu->singlestep_enabled) {
Jan Kiszkab0b1d692010-03-01 19:10:29 +01002071 data.dbg.control |= KVM_GUESTDBG_ENABLE | KVM_GUESTDBG_SINGLESTEP;
2072 }
Andreas Färber20d695a2012-10-31 06:57:49 +01002073 kvm_arch_update_guest_debug(cpu, &data.dbg);
Andreas Färbera60f24b2012-12-01 05:35:08 +01002074 data.cpu = cpu;
aliguorie22a25c2009-03-12 20:12:48 +00002075
Andreas Färberf100f0b2012-05-03 14:58:47 +02002076 run_on_cpu(cpu, kvm_invoke_set_guest_debug, &data);
Glauber Costa452e4752009-07-16 17:55:28 -04002077 return data.err;
aliguorie22a25c2009-03-12 20:12:48 +00002078}
2079
Andreas Färber62278812013-06-27 17:12:06 +02002080int kvm_insert_breakpoint(CPUState *cpu, target_ulong addr,
aliguorie22a25c2009-03-12 20:12:48 +00002081 target_ulong len, int type)
2082{
2083 struct kvm_sw_breakpoint *bp;
aliguorie22a25c2009-03-12 20:12:48 +00002084 int err;
2085
2086 if (type == GDB_BREAKPOINT_SW) {
Andreas Färber80b7cd72013-06-19 17:37:31 +02002087 bp = kvm_find_sw_breakpoint(cpu, addr);
aliguorie22a25c2009-03-12 20:12:48 +00002088 if (bp) {
2089 bp->use_count++;
2090 return 0;
2091 }
2092
Anthony Liguori7267c092011-08-20 22:09:37 -05002093 bp = g_malloc(sizeof(struct kvm_sw_breakpoint));
aliguorie22a25c2009-03-12 20:12:48 +00002094 bp->pc = addr;
2095 bp->use_count = 1;
Andreas Färber80b7cd72013-06-19 17:37:31 +02002096 err = kvm_arch_insert_sw_breakpoint(cpu, bp);
aliguorie22a25c2009-03-12 20:12:48 +00002097 if (err) {
Anthony Liguori7267c092011-08-20 22:09:37 -05002098 g_free(bp);
aliguorie22a25c2009-03-12 20:12:48 +00002099 return err;
2100 }
2101
Andreas Färber80b7cd72013-06-19 17:37:31 +02002102 QTAILQ_INSERT_HEAD(&cpu->kvm_state->kvm_sw_breakpoints, bp, entry);
aliguorie22a25c2009-03-12 20:12:48 +00002103 } else {
2104 err = kvm_arch_insert_hw_breakpoint(addr, len, type);
Jan Kiszkaa426e122011-01-04 09:32:13 +01002105 if (err) {
aliguorie22a25c2009-03-12 20:12:48 +00002106 return err;
Jan Kiszkaa426e122011-01-04 09:32:13 +01002107 }
aliguorie22a25c2009-03-12 20:12:48 +00002108 }
2109
Andreas Färberbdc44642013-06-24 23:50:24 +02002110 CPU_FOREACH(cpu) {
Stefan Weil38e478e2013-07-25 20:50:21 +02002111 err = kvm_update_guest_debug(cpu, 0);
Jan Kiszkaa426e122011-01-04 09:32:13 +01002112 if (err) {
aliguorie22a25c2009-03-12 20:12:48 +00002113 return err;
Jan Kiszkaa426e122011-01-04 09:32:13 +01002114 }
aliguorie22a25c2009-03-12 20:12:48 +00002115 }
2116 return 0;
2117}
2118
Andreas Färber62278812013-06-27 17:12:06 +02002119int kvm_remove_breakpoint(CPUState *cpu, target_ulong addr,
aliguorie22a25c2009-03-12 20:12:48 +00002120 target_ulong len, int type)
2121{
2122 struct kvm_sw_breakpoint *bp;
aliguorie22a25c2009-03-12 20:12:48 +00002123 int err;
2124
2125 if (type == GDB_BREAKPOINT_SW) {
Andreas Färber80b7cd72013-06-19 17:37:31 +02002126 bp = kvm_find_sw_breakpoint(cpu, addr);
Jan Kiszkaa426e122011-01-04 09:32:13 +01002127 if (!bp) {
aliguorie22a25c2009-03-12 20:12:48 +00002128 return -ENOENT;
Jan Kiszkaa426e122011-01-04 09:32:13 +01002129 }
aliguorie22a25c2009-03-12 20:12:48 +00002130
2131 if (bp->use_count > 1) {
2132 bp->use_count--;
2133 return 0;
2134 }
2135
Andreas Färber80b7cd72013-06-19 17:37:31 +02002136 err = kvm_arch_remove_sw_breakpoint(cpu, bp);
Jan Kiszkaa426e122011-01-04 09:32:13 +01002137 if (err) {
aliguorie22a25c2009-03-12 20:12:48 +00002138 return err;
Jan Kiszkaa426e122011-01-04 09:32:13 +01002139 }
aliguorie22a25c2009-03-12 20:12:48 +00002140
Andreas Färber80b7cd72013-06-19 17:37:31 +02002141 QTAILQ_REMOVE(&cpu->kvm_state->kvm_sw_breakpoints, bp, entry);
Anthony Liguori7267c092011-08-20 22:09:37 -05002142 g_free(bp);
aliguorie22a25c2009-03-12 20:12:48 +00002143 } else {
2144 err = kvm_arch_remove_hw_breakpoint(addr, len, type);
Jan Kiszkaa426e122011-01-04 09:32:13 +01002145 if (err) {
aliguorie22a25c2009-03-12 20:12:48 +00002146 return err;
Jan Kiszkaa426e122011-01-04 09:32:13 +01002147 }
aliguorie22a25c2009-03-12 20:12:48 +00002148 }
2149
Andreas Färberbdc44642013-06-24 23:50:24 +02002150 CPU_FOREACH(cpu) {
Stefan Weil38e478e2013-07-25 20:50:21 +02002151 err = kvm_update_guest_debug(cpu, 0);
Jan Kiszkaa426e122011-01-04 09:32:13 +01002152 if (err) {
aliguorie22a25c2009-03-12 20:12:48 +00002153 return err;
Jan Kiszkaa426e122011-01-04 09:32:13 +01002154 }
aliguorie22a25c2009-03-12 20:12:48 +00002155 }
2156 return 0;
2157}
2158
Andreas Färber1d5791f2013-05-27 14:40:48 +02002159void kvm_remove_all_breakpoints(CPUState *cpu)
aliguorie22a25c2009-03-12 20:12:48 +00002160{
2161 struct kvm_sw_breakpoint *bp, *next;
Andreas Färber80b7cd72013-06-19 17:37:31 +02002162 KVMState *s = cpu->kvm_state;
Chen Gangdc54e252014-07-19 09:21:46 +08002163 CPUState *tmpcpu;
aliguorie22a25c2009-03-12 20:12:48 +00002164
Blue Swirl72cf2d42009-09-12 07:36:22 +00002165 QTAILQ_FOREACH_SAFE(bp, &s->kvm_sw_breakpoints, entry, next) {
Andreas Färber80b7cd72013-06-19 17:37:31 +02002166 if (kvm_arch_remove_sw_breakpoint(cpu, bp) != 0) {
aliguorie22a25c2009-03-12 20:12:48 +00002167 /* Try harder to find a CPU that currently sees the breakpoint. */
Chen Gangdc54e252014-07-19 09:21:46 +08002168 CPU_FOREACH(tmpcpu) {
2169 if (kvm_arch_remove_sw_breakpoint(tmpcpu, bp) == 0) {
aliguorie22a25c2009-03-12 20:12:48 +00002170 break;
Jan Kiszkaa426e122011-01-04 09:32:13 +01002171 }
aliguorie22a25c2009-03-12 20:12:48 +00002172 }
2173 }
Jan Kiszka78021d62012-11-12 15:04:35 +01002174 QTAILQ_REMOVE(&s->kvm_sw_breakpoints, bp, entry);
2175 g_free(bp);
aliguorie22a25c2009-03-12 20:12:48 +00002176 }
2177 kvm_arch_remove_all_hw_breakpoints();
2178
Andreas Färberbdc44642013-06-24 23:50:24 +02002179 CPU_FOREACH(cpu) {
Stefan Weil38e478e2013-07-25 20:50:21 +02002180 kvm_update_guest_debug(cpu, 0);
Jan Kiszkaa426e122011-01-04 09:32:13 +01002181 }
aliguorie22a25c2009-03-12 20:12:48 +00002182}
2183
2184#else /* !KVM_CAP_SET_GUEST_DEBUG */
2185
Stefan Weil38e478e2013-07-25 20:50:21 +02002186int kvm_update_guest_debug(CPUState *cpu, unsigned long reinject_trap)
aliguorie22a25c2009-03-12 20:12:48 +00002187{
2188 return -EINVAL;
2189}
2190
Andreas Färber62278812013-06-27 17:12:06 +02002191int kvm_insert_breakpoint(CPUState *cpu, target_ulong addr,
aliguorie22a25c2009-03-12 20:12:48 +00002192 target_ulong len, int type)
2193{
2194 return -EINVAL;
2195}
2196
Andreas Färber62278812013-06-27 17:12:06 +02002197int kvm_remove_breakpoint(CPUState *cpu, target_ulong addr,
aliguorie22a25c2009-03-12 20:12:48 +00002198 target_ulong len, int type)
2199{
2200 return -EINVAL;
2201}
2202
Andreas Färber1d5791f2013-05-27 14:40:48 +02002203void kvm_remove_all_breakpoints(CPUState *cpu)
aliguorie22a25c2009-03-12 20:12:48 +00002204{
2205}
2206#endif /* !KVM_CAP_SET_GUEST_DEBUG */
Marcelo Tosatticc84de92010-02-17 20:14:42 -02002207
Andreas Färber491d6e82013-05-26 23:38:10 +02002208int kvm_set_signal_mask(CPUState *cpu, const sigset_t *sigset)
Marcelo Tosatticc84de92010-02-17 20:14:42 -02002209{
James Hoganaed6efb2014-06-17 23:10:31 +01002210 KVMState *s = kvm_state;
Marcelo Tosatticc84de92010-02-17 20:14:42 -02002211 struct kvm_signal_mask *sigmask;
2212 int r;
2213
Jan Kiszkaa426e122011-01-04 09:32:13 +01002214 if (!sigset) {
Andreas Färber1bc22652012-10-31 06:06:49 +01002215 return kvm_vcpu_ioctl(cpu, KVM_SET_SIGNAL_MASK, NULL);
Jan Kiszkaa426e122011-01-04 09:32:13 +01002216 }
Marcelo Tosatticc84de92010-02-17 20:14:42 -02002217
Anthony Liguori7267c092011-08-20 22:09:37 -05002218 sigmask = g_malloc(sizeof(*sigmask) + sizeof(*sigset));
Marcelo Tosatticc84de92010-02-17 20:14:42 -02002219
James Hoganaed6efb2014-06-17 23:10:31 +01002220 sigmask->len = s->sigmask_len;
Marcelo Tosatticc84de92010-02-17 20:14:42 -02002221 memcpy(sigmask->sigset, sigset, sizeof(*sigset));
Andreas Färber1bc22652012-10-31 06:06:49 +01002222 r = kvm_vcpu_ioctl(cpu, KVM_SET_SIGNAL_MASK, sigmask);
Anthony Liguori7267c092011-08-20 22:09:37 -05002223 g_free(sigmask);
Marcelo Tosatticc84de92010-02-17 20:14:42 -02002224
2225 return r;
2226}
Andreas Färber290adf32013-01-17 09:30:27 +01002227int kvm_on_sigbus_vcpu(CPUState *cpu, int code, void *addr)
Jan Kiszkaa1b87fe2011-02-01 22:15:51 +01002228{
Andreas Färber20d695a2012-10-31 06:57:49 +01002229 return kvm_arch_on_sigbus_vcpu(cpu, code, addr);
Jan Kiszkaa1b87fe2011-02-01 22:15:51 +01002230}
2231
2232int kvm_on_sigbus(int code, void *addr)
2233{
2234 return kvm_arch_on_sigbus(code, addr);
2235}
Christoffer Dall0a6a7cc2014-02-26 17:20:00 +00002236
2237int kvm_create_device(KVMState *s, uint64_t type, bool test)
2238{
2239 int ret;
2240 struct kvm_create_device create_dev;
2241
2242 create_dev.type = type;
2243 create_dev.fd = -1;
2244 create_dev.flags = test ? KVM_CREATE_DEVICE_TEST : 0;
2245
2246 if (!kvm_check_extension(s, KVM_CAP_DEVICE_CTRL)) {
2247 return -ENOTSUP;
2248 }
2249
2250 ret = kvm_vm_ioctl(s, KVM_CREATE_DEVICE, &create_dev);
2251 if (ret) {
2252 return ret;
2253 }
2254
2255 return test ? 0 : create_dev.fd;
2256}
Cornelia Huckada41352014-05-09 10:06:46 +02002257
2258int kvm_set_one_reg(CPUState *cs, uint64_t id, void *source)
2259{
2260 struct kvm_one_reg reg;
2261 int r;
2262
2263 reg.id = id;
2264 reg.addr = (uintptr_t) source;
2265 r = kvm_vcpu_ioctl(cs, KVM_SET_ONE_REG, &reg);
2266 if (r) {
2267 trace_kvm_failed_reg_set(id, strerror(r));
2268 }
2269 return r;
2270}
2271
2272int kvm_get_one_reg(CPUState *cs, uint64_t id, void *target)
2273{
2274 struct kvm_one_reg reg;
2275 int r;
2276
2277 reg.id = id;
2278 reg.addr = (uintptr_t) target;
2279 r = kvm_vcpu_ioctl(cs, KVM_GET_ONE_REG, &reg);
2280 if (r) {
2281 trace_kvm_failed_reg_get(id, strerror(r));
2282 }
2283 return r;
2284}
Eduardo Habkost782c3f22014-09-26 17:45:24 -03002285
2286static void kvm_accel_class_init(ObjectClass *oc, void *data)
2287{
2288 AccelClass *ac = ACCEL_CLASS(oc);
2289 ac->name = "KVM";
Eduardo Habkost0d15da82014-09-26 17:45:29 -03002290 ac->init_machine = kvm_init;
Eduardo Habkost782c3f22014-09-26 17:45:24 -03002291 ac->allowed = &kvm_allowed;
2292}
2293
2294static const TypeInfo kvm_accel_type = {
2295 .name = TYPE_KVM_ACCEL,
2296 .parent = TYPE_ACCEL,
2297 .class_init = kvm_accel_class_init,
Eduardo Habkostfc020862014-09-26 17:45:32 -03002298 .instance_size = sizeof(KVMState),
Eduardo Habkost782c3f22014-09-26 17:45:24 -03002299};
2300
2301static void kvm_type_init(void)
2302{
2303 type_register_static(&kvm_accel_type);
2304}
2305
2306type_init(kvm_type_init);