blob: c4428386689cd837fc3f73b28086622f56b61688 [file] [log] [blame]
aliguori05330442008-11-05 16:29:27 +00001/*
2 * QEMU KVM support
3 *
4 * Copyright IBM, Corp. 2008
aliguori5832d1f2008-11-24 19:36:26 +00005 * Red Hat, Inc. 2008
aliguori05330442008-11-05 16:29:27 +00006 *
7 * Authors:
8 * Anthony Liguori <aliguori@us.ibm.com>
aliguori5832d1f2008-11-24 19:36:26 +00009 * Glauber Costa <gcosta@redhat.com>
aliguori05330442008-11-05 16:29:27 +000010 *
11 * This work is licensed under the terms of the GNU GPL, version 2 or later.
12 * See the COPYING file in the top-level directory.
13 *
14 */
15
16#include <sys/types.h>
17#include <sys/ioctl.h>
18#include <sys/mman.h>
aliguori984b5182008-11-13 19:21:00 +000019#include <stdarg.h>
aliguori05330442008-11-05 16:29:27 +000020
21#include <linux/kvm.h>
22
23#include "qemu-common.h"
Paolo Bonzini1de7afc2012-12-17 18:20:00 +010024#include "qemu/atomic.h"
25#include "qemu/option.h"
26#include "qemu/config-file.h"
Pavel Fedin4b3cfe72015-09-24 01:29:36 +010027#include "qemu/error-report.h"
Jan Kiszkad33a1812009-05-02 00:29:37 +020028#include "hw/hw.h"
Michael S. Tsirkina2cb15b2012-12-12 14:24:50 +020029#include "hw/pci/msi.h"
Cornelia Huckd426d9f2013-07-15 17:45:03 +020030#include "hw/s390x/adapter.h"
Paolo Bonzini022c62c2012-12-17 18:19:49 +010031#include "exec/gdbstub.h"
Paolo Bonzini8571ed32015-06-18 18:28:45 +020032#include "sysemu/kvm_int.h"
Paolo Bonzini1de7afc2012-12-17 18:20:00 +010033#include "qemu/bswap.h"
Paolo Bonzini022c62c2012-12-17 18:19:49 +010034#include "exec/memory.h"
Juan Quintela747afd52013-11-04 12:59:02 +010035#include "exec/ram_addr.h"
Paolo Bonzini022c62c2012-12-17 18:19:49 +010036#include "exec/address-spaces.h"
Paolo Bonzini1de7afc2012-12-17 18:20:00 +010037#include "qemu/event_notifier.h"
Kazuya Saito9c775722013-03-29 13:27:05 +090038#include "trace.h"
Eric Auger197e3522015-07-06 12:15:13 -060039#include "hw/irq.h"
aliguori05330442008-11-05 16:29:27 +000040
Aneesh Kumar K.V135a1292013-12-23 21:10:40 +053041#include "hw/boards.h"
42
Stefan Hajnoczid2f2b8a2011-01-10 13:50:05 +020043/* This check must be after config-host.h is included */
44#ifdef CONFIG_EVENTFD
45#include <sys/eventfd.h>
46#endif
47
Stefan Weil93148aa2012-02-26 18:46:12 +010048/* KVM uses PAGE_SIZE in its definition of COALESCED_MMIO_MAX */
aliguorif65ed4c2008-12-09 20:09:57 +000049#define PAGE_SIZE TARGET_PAGE_SIZE
50
aliguori05330442008-11-05 16:29:27 +000051//#define DEBUG_KVM
52
53#ifdef DEBUG_KVM
Blue Swirl8c0d5772010-04-18 14:22:14 +000054#define DPRINTF(fmt, ...) \
aliguori05330442008-11-05 16:29:27 +000055 do { fprintf(stderr, fmt, ## __VA_ARGS__); } while (0)
56#else
Blue Swirl8c0d5772010-04-18 14:22:14 +000057#define DPRINTF(fmt, ...) \
aliguori05330442008-11-05 16:29:27 +000058 do { } while (0)
59#endif
60
Jan Kiszka04fa27f2012-05-16 15:41:10 -030061#define KVM_MSI_HASHTAB_SIZE 256
62
Paolo Bonzini9d1c35d2014-10-10 12:23:35 +020063struct KVMState
aliguori05330442008-11-05 16:29:27 +000064{
Eduardo Habkostfc020862014-09-26 17:45:32 -030065 AccelState parent_obj;
66
Alex Williamsonfb541ca2013-11-22 12:12:44 -070067 int nr_slots;
aliguori05330442008-11-05 16:29:27 +000068 int fd;
69 int vmfd;
aliguorif65ed4c2008-12-09 20:09:57 +000070 int coalesced_mmio;
Sheng Yang62a27442010-01-26 19:21:16 +080071 struct kvm_coalesced_mmio_ring *coalesced_mmio_ring;
Avi Kivity1cae88b2011-10-18 19:43:12 +020072 bool coalesced_flush_in_progress;
Jan Kiszkae69917e2009-05-01 20:42:15 +020073 int broken_set_mem_region;
Jan Kiszkaa0fb0022009-11-25 00:33:03 +010074 int vcpu_events;
Jan Kiszkab0b1d692010-03-01 19:10:29 +010075 int robust_singlestep;
Jan Kiszkaff44f1a2010-03-12 15:20:49 +010076 int debugregs;
aliguorie22a25c2009-03-12 20:12:48 +000077#ifdef KVM_CAP_SET_GUEST_DEBUG
78 struct kvm_sw_breakpoint_head kvm_sw_breakpoints;
79#endif
Stefan Hajnoczid2f2b8a2011-01-10 13:50:05 +020080 int many_ioeventfds;
Jan Kiszka3ab73842012-08-27 08:28:39 +020081 int intx_set_mask;
David Gibson92e4b512012-03-07 14:41:09 +000082 /* The man page (and posix) say ioctl numbers are signed int, but
83 * they're not. Linux, glibc and *BSD all treat ioctl numbers as
84 * unsigned, and treating them as signed here can break things */
Jan Kiszkae333cd62012-08-24 13:34:47 +020085 unsigned irq_set_ioctl;
James Hoganaed6efb2014-06-17 23:10:31 +010086 unsigned int sigmask_len;
Eric Auger197e3522015-07-06 12:15:13 -060087 GHashTable *gsimap;
Jan Kiszka84b058d2011-10-15 11:49:47 +020088#ifdef KVM_CAP_IRQ_ROUTING
89 struct kvm_irq_routing *irq_routes;
90 int nr_allocated_irq_routes;
91 uint32_t *used_gsi_bitmap;
Jan Kiszka4e2e4e62012-05-16 15:41:08 -030092 unsigned int gsi_count;
Jan Kiszka04fa27f2012-05-16 15:41:10 -030093 QTAILQ_HEAD(msi_hashtab, KVMMSIRoute) msi_hashtab[KVM_MSI_HASHTAB_SIZE];
Jan Kiszka84b058d2011-10-15 11:49:47 +020094#endif
Paolo Bonzini7bbda042015-06-18 18:30:13 +020095 KVMMemoryListener memory_listener;
Paolo Bonzini9d1c35d2014-10-10 12:23:35 +020096};
aliguori05330442008-11-05 16:29:27 +000097
Jan Kiszka6a7af8c2011-02-07 12:19:25 +010098KVMState *kvm_state;
Jan Kiszka3d4b2642012-01-31 19:17:52 +010099bool kvm_kernel_irqchip;
Peter Maydell7ae26bd2012-07-26 15:35:11 +0100100bool kvm_async_interrupts_allowed;
Alexander Graf215e79c2013-04-24 22:24:12 +0200101bool kvm_halt_in_kernel_allowed;
Nikolay Nikolaev69e03ae2014-05-27 15:03:35 +0300102bool kvm_eventfds_allowed;
Peter Maydellcc7e0dd2012-07-26 15:35:14 +0100103bool kvm_irqfds_allowed;
Eric Augerf41389a2014-10-31 13:38:18 +0000104bool kvm_resamplefds_allowed;
Peter Maydell614e41b2012-07-26 15:35:15 +0100105bool kvm_msi_via_irqfd_allowed;
Peter Maydellf3e1bed2012-07-26 15:35:16 +0100106bool kvm_gsi_routing_allowed;
Alexey Kardashevskiy76fe21d2013-09-03 18:08:25 +1000107bool kvm_gsi_direct_mapping;
Igor Mammedov13eed942013-04-23 10:29:36 +0200108bool kvm_allowed;
Jordan Justendf9c8b72013-05-29 01:27:25 -0700109bool kvm_readonly_mem_allowed;
Dominik Dingeld0a073a2015-03-12 13:53:49 +0100110bool kvm_vm_attributes_allowed;
Pavel Fedin50bf31b2015-10-15 16:44:50 +0300111bool kvm_direct_msi_allowed;
aliguori05330442008-11-05 16:29:27 +0000112
Jan Kiszka94a8d392011-01-21 21:48:17 +0100113static const KVMCapabilityInfo kvm_required_capabilites[] = {
114 KVM_CAP_INFO(USER_MEMORY),
115 KVM_CAP_INFO(DESTROY_MEMORY_REGION_WORKS),
116 KVM_CAP_LAST_INFO
117};
118
Paolo Bonzini7bbda042015-06-18 18:30:13 +0200119static KVMSlot *kvm_get_free_slot(KVMMemoryListener *kml)
aliguori05330442008-11-05 16:29:27 +0000120{
Paolo Bonzini7bbda042015-06-18 18:30:13 +0200121 KVMState *s = kvm_state;
aliguori05330442008-11-05 16:29:27 +0000122 int i;
123
Alex Williamsonfb541ca2013-11-22 12:12:44 -0700124 for (i = 0; i < s->nr_slots; i++) {
Paolo Bonzini7bbda042015-06-18 18:30:13 +0200125 if (kml->slots[i].memory_size == 0) {
126 return &kml->slots[i];
Jan Kiszkaa426e122011-01-04 09:32:13 +0100127 }
aliguori05330442008-11-05 16:29:27 +0000128 }
129
Igor Mammedovb8865592014-10-31 16:38:32 +0000130 return NULL;
131}
132
133bool kvm_has_free_slot(MachineState *ms)
134{
Paolo Bonzini7bbda042015-06-18 18:30:13 +0200135 KVMState *s = KVM_STATE(ms->accelerator);
136
137 return kvm_get_free_slot(&s->memory_listener);
Igor Mammedovb8865592014-10-31 16:38:32 +0000138}
139
Paolo Bonzini7bbda042015-06-18 18:30:13 +0200140static KVMSlot *kvm_alloc_slot(KVMMemoryListener *kml)
Igor Mammedovb8865592014-10-31 16:38:32 +0000141{
Paolo Bonzini7bbda042015-06-18 18:30:13 +0200142 KVMSlot *slot = kvm_get_free_slot(kml);
Igor Mammedovb8865592014-10-31 16:38:32 +0000143
144 if (slot) {
145 return slot;
146 }
147
aliguorid3f8d372009-04-17 14:26:29 +0000148 fprintf(stderr, "%s: no free slot available\n", __func__);
149 abort();
150}
151
Paolo Bonzini7bbda042015-06-18 18:30:13 +0200152static KVMSlot *kvm_lookup_matching_slot(KVMMemoryListener *kml,
Avi Kivitya8170e52012-10-23 12:30:10 +0200153 hwaddr start_addr,
154 hwaddr end_addr)
aliguorid3f8d372009-04-17 14:26:29 +0000155{
Paolo Bonzini7bbda042015-06-18 18:30:13 +0200156 KVMState *s = kvm_state;
aliguorid3f8d372009-04-17 14:26:29 +0000157 int i;
158
Alex Williamsonfb541ca2013-11-22 12:12:44 -0700159 for (i = 0; i < s->nr_slots; i++) {
Paolo Bonzini7bbda042015-06-18 18:30:13 +0200160 KVMSlot *mem = &kml->slots[i];
aliguorid3f8d372009-04-17 14:26:29 +0000161
162 if (start_addr == mem->start_addr &&
163 end_addr == mem->start_addr + mem->memory_size) {
164 return mem;
165 }
166 }
167
aliguori05330442008-11-05 16:29:27 +0000168 return NULL;
169}
170
aliguori6152e2a2009-04-17 14:26:33 +0000171/*
172 * Find overlapping slot with lowest start address
173 */
Paolo Bonzini7bbda042015-06-18 18:30:13 +0200174static KVMSlot *kvm_lookup_overlapping_slot(KVMMemoryListener *kml,
Avi Kivitya8170e52012-10-23 12:30:10 +0200175 hwaddr start_addr,
176 hwaddr end_addr)
aliguori05330442008-11-05 16:29:27 +0000177{
Paolo Bonzini7bbda042015-06-18 18:30:13 +0200178 KVMState *s = kvm_state;
aliguori6152e2a2009-04-17 14:26:33 +0000179 KVMSlot *found = NULL;
aliguori05330442008-11-05 16:29:27 +0000180 int i;
181
Alex Williamsonfb541ca2013-11-22 12:12:44 -0700182 for (i = 0; i < s->nr_slots; i++) {
Paolo Bonzini7bbda042015-06-18 18:30:13 +0200183 KVMSlot *mem = &kml->slots[i];
aliguori05330442008-11-05 16:29:27 +0000184
aliguori6152e2a2009-04-17 14:26:33 +0000185 if (mem->memory_size == 0 ||
186 (found && found->start_addr < mem->start_addr)) {
187 continue;
188 }
189
190 if (end_addr > mem->start_addr &&
191 start_addr < mem->start_addr + mem->memory_size) {
192 found = mem;
193 }
aliguori05330442008-11-05 16:29:27 +0000194 }
195
aliguori6152e2a2009-04-17 14:26:33 +0000196 return found;
aliguori05330442008-11-05 16:29:27 +0000197}
198
Avi Kivity9f213ed2011-12-15 19:55:26 +0200199int kvm_physical_memory_addr_from_host(KVMState *s, void *ram,
Avi Kivitya8170e52012-10-23 12:30:10 +0200200 hwaddr *phys_addr)
Huang Ying983dfc32010-10-11 15:31:20 -0300201{
Paolo Bonzini7bbda042015-06-18 18:30:13 +0200202 KVMMemoryListener *kml = &s->memory_listener;
Huang Ying983dfc32010-10-11 15:31:20 -0300203 int i;
204
Alex Williamsonfb541ca2013-11-22 12:12:44 -0700205 for (i = 0; i < s->nr_slots; i++) {
Paolo Bonzini7bbda042015-06-18 18:30:13 +0200206 KVMSlot *mem = &kml->slots[i];
Huang Ying983dfc32010-10-11 15:31:20 -0300207
Avi Kivity9f213ed2011-12-15 19:55:26 +0200208 if (ram >= mem->ram && ram < mem->ram + mem->memory_size) {
209 *phys_addr = mem->start_addr + (ram - mem->ram);
Huang Ying983dfc32010-10-11 15:31:20 -0300210 return 1;
211 }
212 }
213
214 return 0;
215}
216
Paolo Bonzini7bbda042015-06-18 18:30:13 +0200217static int kvm_set_user_memory_region(KVMMemoryListener *kml, KVMSlot *slot)
aliguori5832d1f2008-11-24 19:36:26 +0000218{
Paolo Bonzini7bbda042015-06-18 18:30:13 +0200219 KVMState *s = kvm_state;
aliguori5832d1f2008-11-24 19:36:26 +0000220 struct kvm_userspace_memory_region mem;
221
Paolo Bonzini38bfe692015-06-18 18:30:14 +0200222 mem.slot = slot->slot | (kml->as_id << 16);
aliguori5832d1f2008-11-24 19:36:26 +0000223 mem.guest_phys_addr = slot->start_addr;
Avi Kivity9f213ed2011-12-15 19:55:26 +0200224 mem.userspace_addr = (unsigned long)slot->ram;
aliguori5832d1f2008-11-24 19:36:26 +0000225 mem.flags = slot->flags;
Xiao Guangrong651eb0f2013-05-31 16:52:18 +0800226
227 if (slot->memory_size && mem.flags & KVM_MEM_READONLY) {
Jordan Justen235e8982013-05-29 01:27:26 -0700228 /* Set the slot size to 0 before setting the slot to the desired
229 * value. This is needed based on KVM commit 75d61fbc. */
230 mem.memory_size = 0;
231 kvm_vm_ioctl(s, KVM_SET_USER_MEMORY_REGION, &mem);
232 }
233 mem.memory_size = slot->memory_size;
aliguori5832d1f2008-11-24 19:36:26 +0000234 return kvm_vm_ioctl(s, KVM_SET_USER_MEMORY_REGION, &mem);
235}
236
Andreas Färber504134d2012-12-17 06:38:45 +0100237int kvm_init_vcpu(CPUState *cpu)
aliguori05330442008-11-05 16:29:27 +0000238{
239 KVMState *s = kvm_state;
240 long mmap_size;
241 int ret;
242
Blue Swirl8c0d5772010-04-18 14:22:14 +0000243 DPRINTF("kvm_init_vcpu\n");
aliguori05330442008-11-05 16:29:27 +0000244
Eduardo Habkostb164e482013-01-22 18:25:01 -0200245 ret = kvm_vm_ioctl(s, KVM_CREATE_VCPU, (void *)kvm_arch_vcpu_id(cpu));
aliguori05330442008-11-05 16:29:27 +0000246 if (ret < 0) {
Blue Swirl8c0d5772010-04-18 14:22:14 +0000247 DPRINTF("kvm_create_vcpu failed\n");
aliguori05330442008-11-05 16:29:27 +0000248 goto err;
249 }
250
Andreas Färber8737c512012-10-31 05:29:00 +0100251 cpu->kvm_fd = ret;
Andreas Färbera60f24b2012-12-01 05:35:08 +0100252 cpu->kvm_state = s;
Andreas Färber20d695a2012-10-31 06:57:49 +0100253 cpu->kvm_vcpu_dirty = true;
aliguori05330442008-11-05 16:29:27 +0000254
255 mmap_size = kvm_ioctl(s, KVM_GET_VCPU_MMAP_SIZE, 0);
256 if (mmap_size < 0) {
Jan Kiszka748a6802011-02-01 22:15:48 +0100257 ret = mmap_size;
Blue Swirl8c0d5772010-04-18 14:22:14 +0000258 DPRINTF("KVM_GET_VCPU_MMAP_SIZE failed\n");
aliguori05330442008-11-05 16:29:27 +0000259 goto err;
260 }
261
Andreas Färberf7575c962012-12-01 06:18:14 +0100262 cpu->kvm_run = mmap(NULL, mmap_size, PROT_READ | PROT_WRITE, MAP_SHARED,
Andreas Färber8737c512012-10-31 05:29:00 +0100263 cpu->kvm_fd, 0);
Andreas Färberf7575c962012-12-01 06:18:14 +0100264 if (cpu->kvm_run == MAP_FAILED) {
aliguori05330442008-11-05 16:29:27 +0000265 ret = -errno;
Blue Swirl8c0d5772010-04-18 14:22:14 +0000266 DPRINTF("mmap'ing vcpu state failed\n");
aliguori05330442008-11-05 16:29:27 +0000267 goto err;
268 }
269
Jan Kiszkaa426e122011-01-04 09:32:13 +0100270 if (s->coalesced_mmio && !s->coalesced_mmio_ring) {
271 s->coalesced_mmio_ring =
Andreas Färberf7575c962012-12-01 06:18:14 +0100272 (void *)cpu->kvm_run + s->coalesced_mmio * PAGE_SIZE;
Jan Kiszkaa426e122011-01-04 09:32:13 +0100273 }
Sheng Yang62a27442010-01-26 19:21:16 +0800274
Andreas Färber20d695a2012-10-31 06:57:49 +0100275 ret = kvm_arch_init_vcpu(cpu);
aliguori05330442008-11-05 16:29:27 +0000276err:
277 return ret;
278}
279
aliguori5832d1f2008-11-24 19:36:26 +0000280/*
281 * dirty pages logging control
282 */
Michael S. Tsirkin25254bb2011-04-06 22:09:54 +0300283
Andrew Jonesd6ff5cb2015-06-18 18:28:43 +0200284static int kvm_mem_flags(MemoryRegion *mr)
Michael S. Tsirkin25254bb2011-04-06 22:09:54 +0300285{
Andrew Jonesd6ff5cb2015-06-18 18:28:43 +0200286 bool readonly = mr->readonly || memory_region_is_romd(mr);
Jordan Justen235e8982013-05-29 01:27:26 -0700287 int flags = 0;
Andrew Jonesd6ff5cb2015-06-18 18:28:43 +0200288
289 if (memory_region_get_dirty_log_mask(mr) != 0) {
290 flags |= KVM_MEM_LOG_DIRTY_PAGES;
291 }
Jordan Justen235e8982013-05-29 01:27:26 -0700292 if (readonly && kvm_readonly_mem_allowed) {
293 flags |= KVM_MEM_READONLY;
294 }
295 return flags;
Michael S. Tsirkin25254bb2011-04-06 22:09:54 +0300296}
297
Paolo Bonzini7bbda042015-06-18 18:30:13 +0200298static int kvm_slot_update_flags(KVMMemoryListener *kml, KVMSlot *mem,
299 MemoryRegion *mr)
aliguori5832d1f2008-11-24 19:36:26 +0000300{
Jan Kiszka4495d6a2009-05-01 20:52:46 +0200301 int old_flags;
302
Jan Kiszka4495d6a2009-05-01 20:52:46 +0200303 old_flags = mem->flags;
Andrew Jonesd6ff5cb2015-06-18 18:28:43 +0200304 mem->flags = kvm_mem_flags(mr);
aliguori5832d1f2008-11-24 19:36:26 +0000305
Jan Kiszka4495d6a2009-05-01 20:52:46 +0200306 /* If nothing changed effectively, no need to issue ioctl */
Andrew Jonesd6ff5cb2015-06-18 18:28:43 +0200307 if (mem->flags == old_flags) {
Michael S. Tsirkin25254bb2011-04-06 22:09:54 +0300308 return 0;
Jan Kiszka4495d6a2009-05-01 20:52:46 +0200309 }
310
Paolo Bonzini7bbda042015-06-18 18:30:13 +0200311 return kvm_set_user_memory_region(kml, mem);
aliguori5832d1f2008-11-24 19:36:26 +0000312}
313
Paolo Bonzini7bbda042015-06-18 18:30:13 +0200314static int kvm_section_update_flags(KVMMemoryListener *kml,
315 MemoryRegionSection *section)
Michael S. Tsirkin25254bb2011-04-06 22:09:54 +0300316{
Andrew Jonesd6ff5cb2015-06-18 18:28:43 +0200317 hwaddr phys_addr = section->offset_within_address_space;
318 ram_addr_t size = int128_get64(section->size);
Paolo Bonzini7bbda042015-06-18 18:30:13 +0200319 KVMSlot *mem = kvm_lookup_matching_slot(kml, phys_addr, phys_addr + size);
Michael S. Tsirkin25254bb2011-04-06 22:09:54 +0300320
321 if (mem == NULL) {
Paolo Bonziniea8cb1a2015-04-27 14:51:31 +0200322 return 0;
323 } else {
Paolo Bonzini7bbda042015-06-18 18:30:13 +0200324 return kvm_slot_update_flags(kml, mem, section->mr);
Michael S. Tsirkin25254bb2011-04-06 22:09:54 +0300325 }
Michael S. Tsirkin25254bb2011-04-06 22:09:54 +0300326}
327
Avi Kivitya01672d2011-12-18 14:06:05 +0200328static void kvm_log_start(MemoryListener *listener,
Paolo Bonzinib2dfd712015-04-25 14:38:30 +0200329 MemoryRegionSection *section,
330 int old, int new)
aliguori5832d1f2008-11-24 19:36:26 +0000331{
Paolo Bonzini7bbda042015-06-18 18:30:13 +0200332 KVMMemoryListener *kml = container_of(listener, KVMMemoryListener, listener);
Avi Kivitya01672d2011-12-18 14:06:05 +0200333 int r;
334
Paolo Bonzinib2dfd712015-04-25 14:38:30 +0200335 if (old != 0) {
336 return;
337 }
338
Paolo Bonzini7bbda042015-06-18 18:30:13 +0200339 r = kvm_section_update_flags(kml, section);
Avi Kivitya01672d2011-12-18 14:06:05 +0200340 if (r < 0) {
341 abort();
342 }
aliguori5832d1f2008-11-24 19:36:26 +0000343}
344
Avi Kivitya01672d2011-12-18 14:06:05 +0200345static void kvm_log_stop(MemoryListener *listener,
Paolo Bonzinib2dfd712015-04-25 14:38:30 +0200346 MemoryRegionSection *section,
347 int old, int new)
aliguori5832d1f2008-11-24 19:36:26 +0000348{
Paolo Bonzini7bbda042015-06-18 18:30:13 +0200349 KVMMemoryListener *kml = container_of(listener, KVMMemoryListener, listener);
Avi Kivitya01672d2011-12-18 14:06:05 +0200350 int r;
351
Paolo Bonzinib2dfd712015-04-25 14:38:30 +0200352 if (new != 0) {
353 return;
354 }
355
Paolo Bonzini7bbda042015-06-18 18:30:13 +0200356 r = kvm_section_update_flags(kml, section);
Avi Kivitya01672d2011-12-18 14:06:05 +0200357 if (r < 0) {
358 abort();
359 }
aliguori5832d1f2008-11-24 19:36:26 +0000360}
361
Marcelo Tosatti8369e012010-04-23 14:04:14 -0300362/* get kvm's dirty pages bitmap and update qemu's */
Avi Kivityffcde122011-12-19 13:18:13 +0200363static int kvm_get_dirty_pages_log_range(MemoryRegionSection *section,
364 unsigned long *bitmap)
Alexander Graf96c16062009-07-27 12:49:56 +0200365{
Juan Quintelac9dd46f2013-11-05 15:45:46 +0100366 ram_addr_t start = section->offset_within_region + section->mr->ram_addr;
Juan Quintela5ff7fb72013-11-05 15:52:54 +0100367 ram_addr_t pages = int128_get64(section->size) / getpagesize();
Marcelo Tosatti8369e012010-04-23 14:04:14 -0300368
Juan Quintela5ff7fb72013-11-05 15:52:54 +0100369 cpu_physical_memory_set_dirty_lebitmap(bitmap, start, pages);
Marcelo Tosatti8369e012010-04-23 14:04:14 -0300370 return 0;
Alexander Graf96c16062009-07-27 12:49:56 +0200371}
372
Marcelo Tosatti8369e012010-04-23 14:04:14 -0300373#define ALIGN(x, y) (((x)+(y)-1) & ~((y)-1))
374
aliguori5832d1f2008-11-24 19:36:26 +0000375/**
376 * kvm_physical_sync_dirty_bitmap - Grab dirty bitmap from kernel space
Blue Swirlfd4aa972011-10-16 16:04:59 +0000377 * This function updates qemu's dirty bitmap using
378 * memory_region_set_dirty(). This means all bits are set
379 * to dirty.
aliguori5832d1f2008-11-24 19:36:26 +0000380 *
aliguorid3f8d372009-04-17 14:26:29 +0000381 * @start_add: start of logged region.
aliguori5832d1f2008-11-24 19:36:26 +0000382 * @end_addr: end of logged region.
383 */
Paolo Bonzini7bbda042015-06-18 18:30:13 +0200384static int kvm_physical_sync_dirty_bitmap(KVMMemoryListener *kml,
385 MemoryRegionSection *section)
aliguori5832d1f2008-11-24 19:36:26 +0000386{
387 KVMState *s = kvm_state;
Jan Kiszka151f7742009-05-01 20:52:47 +0200388 unsigned long size, allocated_size = 0;
Paolo Bonzini714f78c2015-06-18 18:28:44 +0200389 struct kvm_dirty_log d = {};
Jan Kiszka151f7742009-05-01 20:52:47 +0200390 KVMSlot *mem;
391 int ret = 0;
Avi Kivitya8170e52012-10-23 12:30:10 +0200392 hwaddr start_addr = section->offset_within_address_space;
Paolo Bonzini052e87b2013-05-27 10:08:27 +0200393 hwaddr end_addr = start_addr + int128_get64(section->size);
aliguori5832d1f2008-11-24 19:36:26 +0000394
Jan Kiszka151f7742009-05-01 20:52:47 +0200395 d.dirty_bitmap = NULL;
396 while (start_addr < end_addr) {
Paolo Bonzini7bbda042015-06-18 18:30:13 +0200397 mem = kvm_lookup_overlapping_slot(kml, start_addr, end_addr);
Jan Kiszka151f7742009-05-01 20:52:47 +0200398 if (mem == NULL) {
399 break;
400 }
401
Michael Tokarev51b0c602011-04-26 20:13:49 +0400402 /* XXX bad kernel interface alert
403 * For dirty bitmap, kernel allocates array of size aligned to
404 * bits-per-long. But for case when the kernel is 64bits and
405 * the userspace is 32bits, userspace can't align to the same
406 * bits-per-long, since sizeof(long) is different between kernel
407 * and user space. This way, userspace will provide buffer which
408 * may be 4 bytes less than the kernel will use, resulting in
409 * userspace memory corruption (which is not detectable by valgrind
410 * too, in most cases).
411 * So for now, let's align to 64 instead of HOST_LONG_BITS here, in
412 * a hope that sizeof(long) wont become >8 any time soon.
413 */
414 size = ALIGN(((mem->memory_size) >> TARGET_PAGE_BITS),
415 /*HOST_LONG_BITS*/ 64) / 8;
Jan Kiszka151f7742009-05-01 20:52:47 +0200416 if (!d.dirty_bitmap) {
Anthony Liguori7267c092011-08-20 22:09:37 -0500417 d.dirty_bitmap = g_malloc(size);
Jan Kiszka151f7742009-05-01 20:52:47 +0200418 } else if (size > allocated_size) {
Anthony Liguori7267c092011-08-20 22:09:37 -0500419 d.dirty_bitmap = g_realloc(d.dirty_bitmap, size);
Jan Kiszka151f7742009-05-01 20:52:47 +0200420 }
421 allocated_size = size;
422 memset(d.dirty_bitmap, 0, allocated_size);
423
Paolo Bonzini38bfe692015-06-18 18:30:14 +0200424 d.slot = mem->slot | (kml->as_id << 16);
Michael Tokarev50212d62014-04-14 16:14:04 +0400425 if (kvm_vm_ioctl(s, KVM_GET_DIRTY_LOG, &d) == -1) {
Blue Swirl8c0d5772010-04-18 14:22:14 +0000426 DPRINTF("ioctl failed %d\n", errno);
Jan Kiszka151f7742009-05-01 20:52:47 +0200427 ret = -1;
428 break;
429 }
430
Avi Kivityffcde122011-12-19 13:18:13 +0200431 kvm_get_dirty_pages_log_range(section, d.dirty_bitmap);
Marcelo Tosatti8369e012010-04-23 14:04:14 -0300432 start_addr = mem->start_addr + mem->memory_size;
aliguori5832d1f2008-11-24 19:36:26 +0000433 }
Anthony Liguori7267c092011-08-20 22:09:37 -0500434 g_free(d.dirty_bitmap);
Jan Kiszka151f7742009-05-01 20:52:47 +0200435
436 return ret;
aliguori5832d1f2008-11-24 19:36:26 +0000437}
438
Avi Kivity95d29942012-10-02 18:21:54 +0200439static void kvm_coalesce_mmio_region(MemoryListener *listener,
440 MemoryRegionSection *secion,
Avi Kivitya8170e52012-10-23 12:30:10 +0200441 hwaddr start, hwaddr size)
aliguorif65ed4c2008-12-09 20:09:57 +0000442{
aliguorif65ed4c2008-12-09 20:09:57 +0000443 KVMState *s = kvm_state;
444
445 if (s->coalesced_mmio) {
446 struct kvm_coalesced_mmio_zone zone;
447
448 zone.addr = start;
449 zone.size = size;
Michael S. Tsirkin7e680752012-02-29 17:54:29 +0200450 zone.pad = 0;
aliguorif65ed4c2008-12-09 20:09:57 +0000451
Avi Kivity95d29942012-10-02 18:21:54 +0200452 (void)kvm_vm_ioctl(s, KVM_REGISTER_COALESCED_MMIO, &zone);
aliguorif65ed4c2008-12-09 20:09:57 +0000453 }
aliguorif65ed4c2008-12-09 20:09:57 +0000454}
455
Avi Kivity95d29942012-10-02 18:21:54 +0200456static void kvm_uncoalesce_mmio_region(MemoryListener *listener,
457 MemoryRegionSection *secion,
Avi Kivitya8170e52012-10-23 12:30:10 +0200458 hwaddr start, hwaddr size)
aliguorif65ed4c2008-12-09 20:09:57 +0000459{
aliguorif65ed4c2008-12-09 20:09:57 +0000460 KVMState *s = kvm_state;
461
462 if (s->coalesced_mmio) {
463 struct kvm_coalesced_mmio_zone zone;
464
465 zone.addr = start;
466 zone.size = size;
Michael S. Tsirkin7e680752012-02-29 17:54:29 +0200467 zone.pad = 0;
aliguorif65ed4c2008-12-09 20:09:57 +0000468
Avi Kivity95d29942012-10-02 18:21:54 +0200469 (void)kvm_vm_ioctl(s, KVM_UNREGISTER_COALESCED_MMIO, &zone);
aliguorif65ed4c2008-12-09 20:09:57 +0000470 }
aliguorif65ed4c2008-12-09 20:09:57 +0000471}
472
Anthony Liguoriad7b8b32009-05-08 15:33:24 -0500473int kvm_check_extension(KVMState *s, unsigned int extension)
474{
475 int ret;
476
477 ret = kvm_ioctl(s, KVM_CHECK_EXTENSION, extension);
478 if (ret < 0) {
479 ret = 0;
480 }
481
482 return ret;
483}
484
Alexander Graf7d0a07f2014-07-14 19:15:15 +0200485int kvm_vm_check_extension(KVMState *s, unsigned int extension)
486{
487 int ret;
488
489 ret = kvm_vm_ioctl(s, KVM_CHECK_EXTENSION, extension);
490 if (ret < 0) {
491 /* VM wide version not implemented, use global one instead */
492 ret = kvm_check_extension(s, extension);
493 }
494
495 return ret;
496}
497
Greg Kurzb680c5b2015-03-13 22:23:37 +0100498static uint32_t adjust_ioeventfd_endianness(uint32_t val, uint32_t size)
499{
500#if defined(HOST_WORDS_BIGENDIAN) != defined(TARGET_WORDS_BIGENDIAN)
501 /* The kernel expects ioeventfd values in HOST_WORDS_BIGENDIAN
502 * endianness, but the memory core hands them in target endianness.
503 * For example, PPC is always treated as big-endian even if running
504 * on KVM and on PPC64LE. Correct here.
505 */
506 switch (size) {
507 case 2:
508 val = bswap16(val);
509 break;
510 case 4:
511 val = bswap32(val);
512 break;
513 }
514#endif
515 return val;
516}
517
Alexey Kardashevskiy584f2be2014-01-10 18:20:18 +1100518static int kvm_set_ioeventfd_mmio(int fd, hwaddr addr, uint32_t val,
Michael S. Tsirkin41cb62c2013-04-02 16:52:25 +0300519 bool assign, uint32_t size, bool datamatch)
Michael S. Tsirkin500ffd42013-04-02 00:05:21 +0300520{
521 int ret;
Thomas Huth03a96b82015-04-27 18:59:04 +0200522 struct kvm_ioeventfd iofd = {
523 .datamatch = datamatch ? adjust_ioeventfd_endianness(val, size) : 0,
524 .addr = addr,
525 .len = size,
526 .flags = 0,
527 .fd = fd,
528 };
Michael S. Tsirkin500ffd42013-04-02 00:05:21 +0300529
530 if (!kvm_enabled()) {
531 return -ENOSYS;
532 }
533
Michael S. Tsirkin41cb62c2013-04-02 16:52:25 +0300534 if (datamatch) {
535 iofd.flags |= KVM_IOEVENTFD_FLAG_DATAMATCH;
536 }
Michael S. Tsirkin500ffd42013-04-02 00:05:21 +0300537 if (!assign) {
538 iofd.flags |= KVM_IOEVENTFD_FLAG_DEASSIGN;
539 }
540
541 ret = kvm_vm_ioctl(kvm_state, KVM_IOEVENTFD, &iofd);
542
543 if (ret < 0) {
544 return -errno;
545 }
546
547 return 0;
548}
549
Michael S. Tsirkin44c3f8f2013-04-02 00:54:45 +0300550static int kvm_set_ioeventfd_pio(int fd, uint16_t addr, uint16_t val,
Michael S. Tsirkin41cb62c2013-04-02 16:52:25 +0300551 bool assign, uint32_t size, bool datamatch)
Michael S. Tsirkin500ffd42013-04-02 00:05:21 +0300552{
553 struct kvm_ioeventfd kick = {
Greg Kurzb680c5b2015-03-13 22:23:37 +0100554 .datamatch = datamatch ? adjust_ioeventfd_endianness(val, size) : 0,
Michael S. Tsirkin500ffd42013-04-02 00:05:21 +0300555 .addr = addr,
Michael S. Tsirkin41cb62c2013-04-02 16:52:25 +0300556 .flags = KVM_IOEVENTFD_FLAG_PIO,
Michael S. Tsirkin44c3f8f2013-04-02 00:54:45 +0300557 .len = size,
Michael S. Tsirkin500ffd42013-04-02 00:05:21 +0300558 .fd = fd,
559 };
560 int r;
561 if (!kvm_enabled()) {
562 return -ENOSYS;
563 }
Michael S. Tsirkin41cb62c2013-04-02 16:52:25 +0300564 if (datamatch) {
565 kick.flags |= KVM_IOEVENTFD_FLAG_DATAMATCH;
566 }
Michael S. Tsirkin500ffd42013-04-02 00:05:21 +0300567 if (!assign) {
568 kick.flags |= KVM_IOEVENTFD_FLAG_DEASSIGN;
569 }
570 r = kvm_vm_ioctl(kvm_state, KVM_IOEVENTFD, &kick);
571 if (r < 0) {
572 return r;
573 }
574 return 0;
575}
576
577
Stefan Hajnoczid2f2b8a2011-01-10 13:50:05 +0200578static int kvm_check_many_ioeventfds(void)
579{
Stefan Hajnoczid0dcac82011-01-25 16:17:14 +0000580 /* Userspace can use ioeventfd for io notification. This requires a host
581 * that supports eventfd(2) and an I/O thread; since eventfd does not
582 * support SIGIO it cannot interrupt the vcpu.
583 *
584 * Older kernels have a 6 device limit on the KVM io bus. Find out so we
Stefan Hajnoczid2f2b8a2011-01-10 13:50:05 +0200585 * can avoid creating too many ioeventfds.
586 */
Anthony Liguori12d45362011-08-22 08:24:58 -0500587#if defined(CONFIG_EVENTFD)
Stefan Hajnoczid2f2b8a2011-01-10 13:50:05 +0200588 int ioeventfds[7];
589 int i, ret = 0;
590 for (i = 0; i < ARRAY_SIZE(ioeventfds); i++) {
591 ioeventfds[i] = eventfd(0, EFD_CLOEXEC);
592 if (ioeventfds[i] < 0) {
593 break;
594 }
Michael S. Tsirkin41cb62c2013-04-02 16:52:25 +0300595 ret = kvm_set_ioeventfd_pio(ioeventfds[i], 0, i, true, 2, true);
Stefan Hajnoczid2f2b8a2011-01-10 13:50:05 +0200596 if (ret < 0) {
597 close(ioeventfds[i]);
598 break;
599 }
600 }
601
602 /* Decide whether many devices are supported or not */
603 ret = i == ARRAY_SIZE(ioeventfds);
604
605 while (i-- > 0) {
Michael S. Tsirkin41cb62c2013-04-02 16:52:25 +0300606 kvm_set_ioeventfd_pio(ioeventfds[i], 0, i, false, 2, true);
Stefan Hajnoczid2f2b8a2011-01-10 13:50:05 +0200607 close(ioeventfds[i]);
608 }
609 return ret;
610#else
611 return 0;
612#endif
613}
614
Jan Kiszka94a8d392011-01-21 21:48:17 +0100615static const KVMCapabilityInfo *
616kvm_check_extension_list(KVMState *s, const KVMCapabilityInfo *list)
617{
618 while (list->name) {
619 if (!kvm_check_extension(s, list->value)) {
620 return list;
621 }
622 list++;
623 }
624 return NULL;
625}
626
Paolo Bonzini7bbda042015-06-18 18:30:13 +0200627static void kvm_set_phys_mem(KVMMemoryListener *kml,
628 MemoryRegionSection *section, bool add)
Michael S. Tsirkin46dbef62010-01-27 22:07:08 +0200629{
630 KVMState *s = kvm_state;
Michael S. Tsirkin46dbef62010-01-27 22:07:08 +0200631 KVMSlot *mem, old;
632 int err;
Avi Kivitya01672d2011-12-18 14:06:05 +0200633 MemoryRegion *mr = section->mr;
Jordan Justen235e8982013-05-29 01:27:26 -0700634 bool writeable = !mr->readonly && !mr->rom_device;
Avi Kivitya8170e52012-10-23 12:30:10 +0200635 hwaddr start_addr = section->offset_within_address_space;
Paolo Bonzini052e87b2013-05-27 10:08:27 +0200636 ram_addr_t size = int128_get64(section->size);
Avi Kivity9f213ed2011-12-15 19:55:26 +0200637 void *ram = NULL;
Avi Kivity8f6f9622012-02-29 13:22:12 +0200638 unsigned delta;
Michael S. Tsirkin46dbef62010-01-27 22:07:08 +0200639
Gleb Natapov14542fe2010-07-28 18:13:23 +0300640 /* kvm works in page size chunks, but the function may be called
Alexander Graff2a64032014-11-07 22:12:48 +0100641 with sub-page size and unaligned start address. Pad the start
642 address to next and truncate size to previous page boundary. */
Alexey Kardashevskiyb232c782015-10-06 14:30:57 +1100643 delta = qemu_real_host_page_size - (start_addr & ~qemu_real_host_page_mask);
644 delta &= ~qemu_real_host_page_mask;
Avi Kivity8f6f9622012-02-29 13:22:12 +0200645 if (delta > size) {
646 return;
647 }
648 start_addr += delta;
649 size -= delta;
Alexey Kardashevskiyb232c782015-10-06 14:30:57 +1100650 size &= qemu_real_host_page_mask;
651 if (!size || (start_addr & ~qemu_real_host_page_mask)) {
Avi Kivity8f6f9622012-02-29 13:22:12 +0200652 return;
653 }
Michael S. Tsirkin46dbef62010-01-27 22:07:08 +0200654
Avi Kivitya01672d2011-12-18 14:06:05 +0200655 if (!memory_region_is_ram(mr)) {
Jordan Justen235e8982013-05-29 01:27:26 -0700656 if (writeable || !kvm_readonly_mem_allowed) {
657 return;
658 } else if (!mr->romd_mode) {
659 /* If the memory device is not in romd_mode, then we actually want
660 * to remove the kvm memory slot so all accesses will trap. */
661 add = false;
662 }
Avi Kivity9f213ed2011-12-15 19:55:26 +0200663 }
664
Avi Kivity8f6f9622012-02-29 13:22:12 +0200665 ram = memory_region_get_ram_ptr(mr) + section->offset_within_region + delta;
Avi Kivitya01672d2011-12-18 14:06:05 +0200666
Michael S. Tsirkin46dbef62010-01-27 22:07:08 +0200667 while (1) {
Paolo Bonzini7bbda042015-06-18 18:30:13 +0200668 mem = kvm_lookup_overlapping_slot(kml, start_addr, start_addr + size);
Michael S. Tsirkin46dbef62010-01-27 22:07:08 +0200669 if (!mem) {
670 break;
671 }
672
Avi Kivitya01672d2011-12-18 14:06:05 +0200673 if (add && start_addr >= mem->start_addr &&
Michael S. Tsirkin46dbef62010-01-27 22:07:08 +0200674 (start_addr + size <= mem->start_addr + mem->memory_size) &&
Avi Kivity9f213ed2011-12-15 19:55:26 +0200675 (ram - start_addr == mem->ram - mem->start_addr)) {
Michael S. Tsirkin46dbef62010-01-27 22:07:08 +0200676 /* The new slot fits into the existing one and comes with
Michael S. Tsirkin25254bb2011-04-06 22:09:54 +0300677 * identical parameters - update flags and done. */
Paolo Bonzini7bbda042015-06-18 18:30:13 +0200678 kvm_slot_update_flags(kml, mem, mr);
Michael S. Tsirkin46dbef62010-01-27 22:07:08 +0200679 return;
680 }
681
682 old = *mem;
683
Paolo Bonzini1bfbac42015-03-23 10:57:21 +0100684 if (mem->flags & KVM_MEM_LOG_DIRTY_PAGES) {
Paolo Bonzini7bbda042015-06-18 18:30:13 +0200685 kvm_physical_sync_dirty_bitmap(kml, section);
Avi Kivity3fbffb62012-01-15 16:13:59 +0200686 }
687
Michael S. Tsirkin46dbef62010-01-27 22:07:08 +0200688 /* unregister the overlapping slot */
689 mem->memory_size = 0;
Paolo Bonzini7bbda042015-06-18 18:30:13 +0200690 err = kvm_set_user_memory_region(kml, mem);
Michael S. Tsirkin46dbef62010-01-27 22:07:08 +0200691 if (err) {
692 fprintf(stderr, "%s: error unregistering overlapping slot: %s\n",
693 __func__, strerror(-err));
694 abort();
695 }
696
697 /* Workaround for older KVM versions: we can't join slots, even not by
698 * unregistering the previous ones and then registering the larger
699 * slot. We have to maintain the existing fragmentation. Sigh.
700 *
701 * This workaround assumes that the new slot starts at the same
702 * address as the first existing one. If not or if some overlapping
703 * slot comes around later, we will fail (not seen in practice so far)
704 * - and actually require a recent KVM version. */
705 if (s->broken_set_mem_region &&
Avi Kivitya01672d2011-12-18 14:06:05 +0200706 old.start_addr == start_addr && old.memory_size < size && add) {
Paolo Bonzini7bbda042015-06-18 18:30:13 +0200707 mem = kvm_alloc_slot(kml);
Michael S. Tsirkin46dbef62010-01-27 22:07:08 +0200708 mem->memory_size = old.memory_size;
709 mem->start_addr = old.start_addr;
Avi Kivity9f213ed2011-12-15 19:55:26 +0200710 mem->ram = old.ram;
Andrew Jonesd6ff5cb2015-06-18 18:28:43 +0200711 mem->flags = kvm_mem_flags(mr);
Michael S. Tsirkin46dbef62010-01-27 22:07:08 +0200712
Paolo Bonzini7bbda042015-06-18 18:30:13 +0200713 err = kvm_set_user_memory_region(kml, mem);
Michael S. Tsirkin46dbef62010-01-27 22:07:08 +0200714 if (err) {
715 fprintf(stderr, "%s: error updating slot: %s\n", __func__,
716 strerror(-err));
717 abort();
718 }
719
720 start_addr += old.memory_size;
Avi Kivity9f213ed2011-12-15 19:55:26 +0200721 ram += old.memory_size;
Michael S. Tsirkin46dbef62010-01-27 22:07:08 +0200722 size -= old.memory_size;
723 continue;
724 }
725
726 /* register prefix slot */
727 if (old.start_addr < start_addr) {
Paolo Bonzini7bbda042015-06-18 18:30:13 +0200728 mem = kvm_alloc_slot(kml);
Michael S. Tsirkin46dbef62010-01-27 22:07:08 +0200729 mem->memory_size = start_addr - old.start_addr;
730 mem->start_addr = old.start_addr;
Avi Kivity9f213ed2011-12-15 19:55:26 +0200731 mem->ram = old.ram;
Andrew Jonesd6ff5cb2015-06-18 18:28:43 +0200732 mem->flags = kvm_mem_flags(mr);
Michael S. Tsirkin46dbef62010-01-27 22:07:08 +0200733
Paolo Bonzini7bbda042015-06-18 18:30:13 +0200734 err = kvm_set_user_memory_region(kml, mem);
Michael S. Tsirkin46dbef62010-01-27 22:07:08 +0200735 if (err) {
736 fprintf(stderr, "%s: error registering prefix slot: %s\n",
737 __func__, strerror(-err));
Alexander Grafd4d68682011-04-16 10:15:11 +0200738#ifdef TARGET_PPC
739 fprintf(stderr, "%s: This is probably because your kernel's " \
740 "PAGE_SIZE is too big. Please try to use 4k " \
741 "PAGE_SIZE!\n", __func__);
742#endif
Michael S. Tsirkin46dbef62010-01-27 22:07:08 +0200743 abort();
744 }
745 }
746
747 /* register suffix slot */
748 if (old.start_addr + old.memory_size > start_addr + size) {
749 ram_addr_t size_delta;
750
Paolo Bonzini7bbda042015-06-18 18:30:13 +0200751 mem = kvm_alloc_slot(kml);
Michael S. Tsirkin46dbef62010-01-27 22:07:08 +0200752 mem->start_addr = start_addr + size;
753 size_delta = mem->start_addr - old.start_addr;
754 mem->memory_size = old.memory_size - size_delta;
Avi Kivity9f213ed2011-12-15 19:55:26 +0200755 mem->ram = old.ram + size_delta;
Andrew Jonesd6ff5cb2015-06-18 18:28:43 +0200756 mem->flags = kvm_mem_flags(mr);
Michael S. Tsirkin46dbef62010-01-27 22:07:08 +0200757
Paolo Bonzini7bbda042015-06-18 18:30:13 +0200758 err = kvm_set_user_memory_region(kml, mem);
Michael S. Tsirkin46dbef62010-01-27 22:07:08 +0200759 if (err) {
760 fprintf(stderr, "%s: error registering suffix slot: %s\n",
761 __func__, strerror(-err));
762 abort();
763 }
764 }
765 }
766
767 /* in case the KVM bug workaround already "consumed" the new slot */
Jan Kiszkaa426e122011-01-04 09:32:13 +0100768 if (!size) {
Michael S. Tsirkin46dbef62010-01-27 22:07:08 +0200769 return;
Jan Kiszkaa426e122011-01-04 09:32:13 +0100770 }
Avi Kivitya01672d2011-12-18 14:06:05 +0200771 if (!add) {
Michael S. Tsirkin46dbef62010-01-27 22:07:08 +0200772 return;
Jan Kiszkaa426e122011-01-04 09:32:13 +0100773 }
Paolo Bonzini7bbda042015-06-18 18:30:13 +0200774 mem = kvm_alloc_slot(kml);
Michael S. Tsirkin46dbef62010-01-27 22:07:08 +0200775 mem->memory_size = size;
776 mem->start_addr = start_addr;
Avi Kivity9f213ed2011-12-15 19:55:26 +0200777 mem->ram = ram;
Andrew Jonesd6ff5cb2015-06-18 18:28:43 +0200778 mem->flags = kvm_mem_flags(mr);
Michael S. Tsirkin46dbef62010-01-27 22:07:08 +0200779
Paolo Bonzini7bbda042015-06-18 18:30:13 +0200780 err = kvm_set_user_memory_region(kml, mem);
Michael S. Tsirkin46dbef62010-01-27 22:07:08 +0200781 if (err) {
782 fprintf(stderr, "%s: error registering slot: %s\n", __func__,
783 strerror(-err));
784 abort();
785 }
786}
787
Avi Kivitya01672d2011-12-18 14:06:05 +0200788static void kvm_region_add(MemoryListener *listener,
789 MemoryRegionSection *section)
Michael S. Tsirkin7b8f3b72010-01-27 22:07:21 +0200790{
Paolo Bonzini7bbda042015-06-18 18:30:13 +0200791 KVMMemoryListener *kml = container_of(listener, KVMMemoryListener, listener);
792
Paolo Bonzinidfde4e62013-05-06 10:46:11 +0200793 memory_region_ref(section->mr);
Paolo Bonzini7bbda042015-06-18 18:30:13 +0200794 kvm_set_phys_mem(kml, section, true);
Michael S. Tsirkin7b8f3b72010-01-27 22:07:21 +0200795}
796
Avi Kivitya01672d2011-12-18 14:06:05 +0200797static void kvm_region_del(MemoryListener *listener,
798 MemoryRegionSection *section)
Michael S. Tsirkin7b8f3b72010-01-27 22:07:21 +0200799{
Paolo Bonzini7bbda042015-06-18 18:30:13 +0200800 KVMMemoryListener *kml = container_of(listener, KVMMemoryListener, listener);
801
802 kvm_set_phys_mem(kml, section, false);
Paolo Bonzinidfde4e62013-05-06 10:46:11 +0200803 memory_region_unref(section->mr);
Michael S. Tsirkin7b8f3b72010-01-27 22:07:21 +0200804}
805
Avi Kivitya01672d2011-12-18 14:06:05 +0200806static void kvm_log_sync(MemoryListener *listener,
807 MemoryRegionSection *section)
Michael S. Tsirkin7b8f3b72010-01-27 22:07:21 +0200808{
Paolo Bonzini7bbda042015-06-18 18:30:13 +0200809 KVMMemoryListener *kml = container_of(listener, KVMMemoryListener, listener);
Avi Kivitya01672d2011-12-18 14:06:05 +0200810 int r;
811
Paolo Bonzini7bbda042015-06-18 18:30:13 +0200812 r = kvm_physical_sync_dirty_bitmap(kml, section);
Avi Kivitya01672d2011-12-18 14:06:05 +0200813 if (r < 0) {
814 abort();
815 }
Michael S. Tsirkin7b8f3b72010-01-27 22:07:21 +0200816}
817
Avi Kivityd22b0962012-09-30 22:21:11 +0200818static void kvm_mem_ioeventfd_add(MemoryListener *listener,
819 MemoryRegionSection *section,
820 bool match_data, uint64_t data,
821 EventNotifier *e)
822{
823 int fd = event_notifier_get_fd(e);
Avi Kivity80a1ea32012-02-08 16:39:06 +0200824 int r;
825
Michael S. Tsirkin4b8f1c82012-03-20 14:31:38 +0200826 r = kvm_set_ioeventfd_mmio(fd, section->offset_within_address_space,
Paolo Bonzini052e87b2013-05-27 10:08:27 +0200827 data, true, int128_get64(section->size),
828 match_data);
Avi Kivity80a1ea32012-02-08 16:39:06 +0200829 if (r < 0) {
Amos Kongfa4ba922013-05-22 12:57:35 +0800830 fprintf(stderr, "%s: error adding ioeventfd: %s\n",
831 __func__, strerror(-r));
Avi Kivity80a1ea32012-02-08 16:39:06 +0200832 abort();
833 }
834}
835
Avi Kivityd22b0962012-09-30 22:21:11 +0200836static void kvm_mem_ioeventfd_del(MemoryListener *listener,
837 MemoryRegionSection *section,
838 bool match_data, uint64_t data,
839 EventNotifier *e)
Avi Kivity80a1ea32012-02-08 16:39:06 +0200840{
Avi Kivityd22b0962012-09-30 22:21:11 +0200841 int fd = event_notifier_get_fd(e);
Avi Kivity80a1ea32012-02-08 16:39:06 +0200842 int r;
843
Michael S. Tsirkin4b8f1c82012-03-20 14:31:38 +0200844 r = kvm_set_ioeventfd_mmio(fd, section->offset_within_address_space,
Paolo Bonzini052e87b2013-05-27 10:08:27 +0200845 data, false, int128_get64(section->size),
846 match_data);
Avi Kivity80a1ea32012-02-08 16:39:06 +0200847 if (r < 0) {
848 abort();
849 }
850}
851
Avi Kivityd22b0962012-09-30 22:21:11 +0200852static void kvm_io_ioeventfd_add(MemoryListener *listener,
853 MemoryRegionSection *section,
854 bool match_data, uint64_t data,
855 EventNotifier *e)
Avi Kivity80a1ea32012-02-08 16:39:06 +0200856{
Avi Kivityd22b0962012-09-30 22:21:11 +0200857 int fd = event_notifier_get_fd(e);
Avi Kivity80a1ea32012-02-08 16:39:06 +0200858 int r;
859
Michael S. Tsirkin44c3f8f2013-04-02 00:54:45 +0300860 r = kvm_set_ioeventfd_pio(fd, section->offset_within_address_space,
Paolo Bonzini052e87b2013-05-27 10:08:27 +0200861 data, true, int128_get64(section->size),
862 match_data);
Avi Kivity80a1ea32012-02-08 16:39:06 +0200863 if (r < 0) {
Amos Kongfa4ba922013-05-22 12:57:35 +0800864 fprintf(stderr, "%s: error adding ioeventfd: %s\n",
865 __func__, strerror(-r));
Avi Kivity80a1ea32012-02-08 16:39:06 +0200866 abort();
867 }
868}
869
Avi Kivityd22b0962012-09-30 22:21:11 +0200870static void kvm_io_ioeventfd_del(MemoryListener *listener,
871 MemoryRegionSection *section,
872 bool match_data, uint64_t data,
873 EventNotifier *e)
Avi Kivity80a1ea32012-02-08 16:39:06 +0200874
875{
Avi Kivityd22b0962012-09-30 22:21:11 +0200876 int fd = event_notifier_get_fd(e);
Avi Kivity80a1ea32012-02-08 16:39:06 +0200877 int r;
878
Michael S. Tsirkin44c3f8f2013-04-02 00:54:45 +0300879 r = kvm_set_ioeventfd_pio(fd, section->offset_within_address_space,
Paolo Bonzini052e87b2013-05-27 10:08:27 +0200880 data, false, int128_get64(section->size),
881 match_data);
Avi Kivity80a1ea32012-02-08 16:39:06 +0200882 if (r < 0) {
883 abort();
884 }
885}
886
Paolo Bonzini38bfe692015-06-18 18:30:14 +0200887void kvm_memory_listener_register(KVMState *s, KVMMemoryListener *kml,
888 AddressSpace *as, int as_id)
Paolo Bonzini7bbda042015-06-18 18:30:13 +0200889{
890 int i;
891
892 kml->slots = g_malloc0(s->nr_slots * sizeof(KVMSlot));
Paolo Bonzini38bfe692015-06-18 18:30:14 +0200893 kml->as_id = as_id;
Paolo Bonzini7bbda042015-06-18 18:30:13 +0200894
895 for (i = 0; i < s->nr_slots; i++) {
896 kml->slots[i].slot = i;
897 }
898
899 kml->listener.region_add = kvm_region_add;
900 kml->listener.region_del = kvm_region_del;
901 kml->listener.log_start = kvm_log_start;
902 kml->listener.log_stop = kvm_log_stop;
903 kml->listener.log_sync = kvm_log_sync;
904 kml->listener.priority = 10;
905
906 memory_listener_register(&kml->listener, as);
907}
Avi Kivityd22b0962012-09-30 22:21:11 +0200908
909static MemoryListener kvm_io_listener = {
Avi Kivityd22b0962012-09-30 22:21:11 +0200910 .eventfd_add = kvm_io_ioeventfd_add,
911 .eventfd_del = kvm_io_ioeventfd_del,
Avi Kivity72e22d22012-02-08 15:05:50 +0200912 .priority = 10,
Michael S. Tsirkin7b8f3b72010-01-27 22:07:21 +0200913};
914
Andreas Färberc3affe52013-01-18 15:03:43 +0100915static void kvm_handle_interrupt(CPUState *cpu, int mask)
Jan Kiszkaaa7f74d2011-04-13 01:32:56 +0200916{
Andreas Färber259186a2013-01-17 18:51:17 +0100917 cpu->interrupt_request |= mask;
Jan Kiszkaaa7f74d2011-04-13 01:32:56 +0200918
Andreas Färber60e82572012-05-02 22:23:49 +0200919 if (!qemu_cpu_is_self(cpu)) {
Andreas Färberc08d7422012-05-03 04:34:15 +0200920 qemu_cpu_kick(cpu);
Jan Kiszkaaa7f74d2011-04-13 01:32:56 +0200921 }
922}
923
Peter Maydell3889c3f2012-07-26 15:35:12 +0100924int kvm_set_irq(KVMState *s, int irq, int level)
Jan Kiszka84b058d2011-10-15 11:49:47 +0200925{
926 struct kvm_irq_level event;
927 int ret;
928
Peter Maydell7ae26bd2012-07-26 15:35:11 +0100929 assert(kvm_async_interrupts_enabled());
Jan Kiszka84b058d2011-10-15 11:49:47 +0200930
931 event.level = level;
932 event.irq = irq;
Jan Kiszkae333cd62012-08-24 13:34:47 +0200933 ret = kvm_vm_ioctl(s, s->irq_set_ioctl, &event);
Jan Kiszka84b058d2011-10-15 11:49:47 +0200934 if (ret < 0) {
Peter Maydell3889c3f2012-07-26 15:35:12 +0100935 perror("kvm_set_irq");
Jan Kiszka84b058d2011-10-15 11:49:47 +0200936 abort();
937 }
938
Jan Kiszkae333cd62012-08-24 13:34:47 +0200939 return (s->irq_set_ioctl == KVM_IRQ_LINE) ? 1 : event.status;
Jan Kiszka84b058d2011-10-15 11:49:47 +0200940}
941
942#ifdef KVM_CAP_IRQ_ROUTING
Jan Kiszkad3d3bef2012-06-05 21:03:57 +0200943typedef struct KVMMSIRoute {
944 struct kvm_irq_routing_entry kroute;
945 QTAILQ_ENTRY(KVMMSIRoute) entry;
946} KVMMSIRoute;
947
Jan Kiszka84b058d2011-10-15 11:49:47 +0200948static void set_gsi(KVMState *s, unsigned int gsi)
949{
Jan Kiszka84b058d2011-10-15 11:49:47 +0200950 s->used_gsi_bitmap[gsi / 32] |= 1U << (gsi % 32);
951}
952
Jan Kiszka04fa27f2012-05-16 15:41:10 -0300953static void clear_gsi(KVMState *s, unsigned int gsi)
954{
955 s->used_gsi_bitmap[gsi / 32] &= ~(1U << (gsi % 32));
956}
957
Alexander Graf7b774592013-04-16 15:58:13 +0200958void kvm_init_irq_routing(KVMState *s)
Jan Kiszka84b058d2011-10-15 11:49:47 +0200959{
Jan Kiszka04fa27f2012-05-16 15:41:10 -0300960 int gsi_count, i;
Jan Kiszka84b058d2011-10-15 11:49:47 +0200961
Alexander Graf00008412014-06-06 14:46:05 +0200962 gsi_count = kvm_check_extension(s, KVM_CAP_IRQ_ROUTING) - 1;
Jan Kiszka84b058d2011-10-15 11:49:47 +0200963 if (gsi_count > 0) {
964 unsigned int gsi_bits, i;
965
966 /* Round up so we can search ints using ffs */
Jason Baronbc8c6782012-03-28 14:18:05 -0400967 gsi_bits = ALIGN(gsi_count, 32);
Jan Kiszka84b058d2011-10-15 11:49:47 +0200968 s->used_gsi_bitmap = g_malloc0(gsi_bits / 8);
Jan Kiszka4e2e4e62012-05-16 15:41:08 -0300969 s->gsi_count = gsi_count;
Jan Kiszka84b058d2011-10-15 11:49:47 +0200970
971 /* Mark any over-allocated bits as already in use */
972 for (i = gsi_count; i < gsi_bits; i++) {
973 set_gsi(s, i);
974 }
975 }
976
977 s->irq_routes = g_malloc0(sizeof(*s->irq_routes));
978 s->nr_allocated_irq_routes = 0;
979
Pavel Fedin50bf31b2015-10-15 16:44:50 +0300980 if (!kvm_direct_msi_allowed) {
Jan Kiszka4a3adeb2012-05-16 15:41:14 -0300981 for (i = 0; i < KVM_MSI_HASHTAB_SIZE; i++) {
982 QTAILQ_INIT(&s->msi_hashtab[i]);
983 }
Jan Kiszka04fa27f2012-05-16 15:41:10 -0300984 }
985
Jan Kiszka84b058d2011-10-15 11:49:47 +0200986 kvm_arch_init_irq_routing(s);
987}
988
Alexander Grafcb925cf2013-04-17 01:11:55 +0200989void kvm_irqchip_commit_routes(KVMState *s)
Jan Kiszkae7b20302012-05-17 10:32:35 -0300990{
991 int ret;
992
993 s->irq_routes->flags = 0;
994 ret = kvm_vm_ioctl(s, KVM_SET_GSI_ROUTING, s->irq_routes);
995 assert(ret == 0);
996}
997
Jan Kiszka84b058d2011-10-15 11:49:47 +0200998static void kvm_add_routing_entry(KVMState *s,
999 struct kvm_irq_routing_entry *entry)
1000{
1001 struct kvm_irq_routing_entry *new;
1002 int n, size;
1003
1004 if (s->irq_routes->nr == s->nr_allocated_irq_routes) {
1005 n = s->nr_allocated_irq_routes * 2;
1006 if (n < 64) {
1007 n = 64;
1008 }
1009 size = sizeof(struct kvm_irq_routing);
1010 size += n * sizeof(*new);
1011 s->irq_routes = g_realloc(s->irq_routes, size);
1012 s->nr_allocated_irq_routes = n;
1013 }
1014 n = s->irq_routes->nr++;
1015 new = &s->irq_routes->entries[n];
Michael S. Tsirkin0fbc2072013-06-04 14:52:32 +03001016
1017 *new = *entry;
Jan Kiszka84b058d2011-10-15 11:49:47 +02001018
1019 set_gsi(s, entry->gsi);
1020}
1021
Jan Kiszkacc574072012-08-27 08:28:38 +02001022static int kvm_update_routing_entry(KVMState *s,
1023 struct kvm_irq_routing_entry *new_entry)
1024{
1025 struct kvm_irq_routing_entry *entry;
1026 int n;
1027
1028 for (n = 0; n < s->irq_routes->nr; n++) {
1029 entry = &s->irq_routes->entries[n];
1030 if (entry->gsi != new_entry->gsi) {
1031 continue;
1032 }
1033
Michael S. Tsirkin40509f72013-06-04 14:52:35 +03001034 if(!memcmp(entry, new_entry, sizeof *entry)) {
1035 return 0;
1036 }
1037
Michael S. Tsirkin0fbc2072013-06-04 14:52:32 +03001038 *entry = *new_entry;
Jan Kiszkacc574072012-08-27 08:28:38 +02001039
1040 kvm_irqchip_commit_routes(s);
1041
1042 return 0;
1043 }
1044
1045 return -ESRCH;
1046}
1047
Jan Kiszka1df186d2012-05-17 10:32:32 -03001048void kvm_irqchip_add_irq_route(KVMState *s, int irq, int irqchip, int pin)
Jan Kiszka84b058d2011-10-15 11:49:47 +02001049{
Michael S. Tsirkin0fbc2072013-06-04 14:52:32 +03001050 struct kvm_irq_routing_entry e = {};
Jan Kiszka84b058d2011-10-15 11:49:47 +02001051
Jan Kiszka4e2e4e62012-05-16 15:41:08 -03001052 assert(pin < s->gsi_count);
1053
Jan Kiszka84b058d2011-10-15 11:49:47 +02001054 e.gsi = irq;
1055 e.type = KVM_IRQ_ROUTING_IRQCHIP;
1056 e.flags = 0;
1057 e.u.irqchip.irqchip = irqchip;
1058 e.u.irqchip.pin = pin;
1059 kvm_add_routing_entry(s, &e);
1060}
1061
Jan Kiszka1e2aa8b2012-05-17 10:32:34 -03001062void kvm_irqchip_release_virq(KVMState *s, int virq)
Jan Kiszka04fa27f2012-05-16 15:41:10 -03001063{
1064 struct kvm_irq_routing_entry *e;
1065 int i;
1066
Alexey Kardashevskiy76fe21d2013-09-03 18:08:25 +10001067 if (kvm_gsi_direct_mapping()) {
1068 return;
1069 }
1070
Jan Kiszka04fa27f2012-05-16 15:41:10 -03001071 for (i = 0; i < s->irq_routes->nr; i++) {
1072 e = &s->irq_routes->entries[i];
1073 if (e->gsi == virq) {
1074 s->irq_routes->nr--;
1075 *e = s->irq_routes->entries[s->irq_routes->nr];
1076 }
1077 }
1078 clear_gsi(s, virq);
1079}
1080
1081static unsigned int kvm_hash_msi(uint32_t data)
1082{
1083 /* This is optimized for IA32 MSI layout. However, no other arch shall
1084 * repeat the mistake of not providing a direct MSI injection API. */
1085 return data & 0xff;
1086}
1087
1088static void kvm_flush_dynamic_msi_routes(KVMState *s)
1089{
1090 KVMMSIRoute *route, *next;
1091 unsigned int hash;
1092
1093 for (hash = 0; hash < KVM_MSI_HASHTAB_SIZE; hash++) {
1094 QTAILQ_FOREACH_SAFE(route, &s->msi_hashtab[hash], entry, next) {
1095 kvm_irqchip_release_virq(s, route->kroute.gsi);
1096 QTAILQ_REMOVE(&s->msi_hashtab[hash], route, entry);
1097 g_free(route);
1098 }
1099 }
1100}
1101
1102static int kvm_irqchip_get_virq(KVMState *s)
1103{
1104 uint32_t *word = s->used_gsi_bitmap;
1105 int max_words = ALIGN(s->gsi_count, 32) / 32;
Stefan Hajnoczibd2a8882015-03-23 15:29:27 +00001106 int i, zeroes;
Jan Kiszka04fa27f2012-05-16 15:41:10 -03001107
马文霜bdf02632015-07-01 15:41:41 +02001108 /*
1109 * PIC and IOAPIC share the first 16 GSI numbers, thus the available
1110 * GSI numbers are more than the number of IRQ route. Allocating a GSI
1111 * number can succeed even though a new route entry cannot be added.
1112 * When this happens, flush dynamic MSI entries to free IRQ route entries.
1113 */
Pavel Fedin50bf31b2015-10-15 16:44:50 +03001114 if (!kvm_direct_msi_allowed && s->irq_routes->nr == s->gsi_count) {
马文霜bdf02632015-07-01 15:41:41 +02001115 kvm_flush_dynamic_msi_routes(s);
1116 }
1117
Jan Kiszka04fa27f2012-05-16 15:41:10 -03001118 /* Return the lowest unused GSI in the bitmap */
1119 for (i = 0; i < max_words; i++) {
Stefan Hajnoczibd2a8882015-03-23 15:29:27 +00001120 zeroes = ctz32(~word[i]);
1121 if (zeroes == 32) {
Jan Kiszka04fa27f2012-05-16 15:41:10 -03001122 continue;
1123 }
1124
Stefan Hajnoczibd2a8882015-03-23 15:29:27 +00001125 return zeroes + i * 32;
Jan Kiszka04fa27f2012-05-16 15:41:10 -03001126 }
Jan Kiszka04fa27f2012-05-16 15:41:10 -03001127 return -ENOSPC;
1128
1129}
1130
1131static KVMMSIRoute *kvm_lookup_msi_route(KVMState *s, MSIMessage msg)
1132{
1133 unsigned int hash = kvm_hash_msi(msg.data);
1134 KVMMSIRoute *route;
1135
1136 QTAILQ_FOREACH(route, &s->msi_hashtab[hash], entry) {
1137 if (route->kroute.u.msi.address_lo == (uint32_t)msg.address &&
1138 route->kroute.u.msi.address_hi == (msg.address >> 32) &&
Alexander Grafd07cc1f2013-04-16 15:05:22 +02001139 route->kroute.u.msi.data == le32_to_cpu(msg.data)) {
Jan Kiszka04fa27f2012-05-16 15:41:10 -03001140 return route;
1141 }
1142 }
1143 return NULL;
1144}
1145
1146int kvm_irqchip_send_msi(KVMState *s, MSIMessage msg)
1147{
Jan Kiszka4a3adeb2012-05-16 15:41:14 -03001148 struct kvm_msi msi;
Jan Kiszka04fa27f2012-05-16 15:41:10 -03001149 KVMMSIRoute *route;
1150
Pavel Fedin50bf31b2015-10-15 16:44:50 +03001151 if (kvm_direct_msi_allowed) {
Jan Kiszka4a3adeb2012-05-16 15:41:14 -03001152 msi.address_lo = (uint32_t)msg.address;
1153 msi.address_hi = msg.address >> 32;
Alexander Grafd07cc1f2013-04-16 15:05:22 +02001154 msi.data = le32_to_cpu(msg.data);
Jan Kiszka4a3adeb2012-05-16 15:41:14 -03001155 msi.flags = 0;
1156 memset(msi.pad, 0, sizeof(msi.pad));
1157
1158 return kvm_vm_ioctl(s, KVM_SIGNAL_MSI, &msi);
1159 }
1160
Jan Kiszka04fa27f2012-05-16 15:41:10 -03001161 route = kvm_lookup_msi_route(s, msg);
1162 if (!route) {
Jan Kiszkae7b20302012-05-17 10:32:35 -03001163 int virq;
Jan Kiszka04fa27f2012-05-16 15:41:10 -03001164
1165 virq = kvm_irqchip_get_virq(s);
1166 if (virq < 0) {
1167 return virq;
1168 }
1169
Michael S. Tsirkin0fbc2072013-06-04 14:52:32 +03001170 route = g_malloc0(sizeof(KVMMSIRoute));
Jan Kiszka04fa27f2012-05-16 15:41:10 -03001171 route->kroute.gsi = virq;
1172 route->kroute.type = KVM_IRQ_ROUTING_MSI;
1173 route->kroute.flags = 0;
1174 route->kroute.u.msi.address_lo = (uint32_t)msg.address;
1175 route->kroute.u.msi.address_hi = msg.address >> 32;
Alexander Grafd07cc1f2013-04-16 15:05:22 +02001176 route->kroute.u.msi.data = le32_to_cpu(msg.data);
Jan Kiszka04fa27f2012-05-16 15:41:10 -03001177
1178 kvm_add_routing_entry(s, &route->kroute);
Alexander Grafcb925cf2013-04-17 01:11:55 +02001179 kvm_irqchip_commit_routes(s);
Jan Kiszka04fa27f2012-05-16 15:41:10 -03001180
1181 QTAILQ_INSERT_TAIL(&s->msi_hashtab[kvm_hash_msi(msg.data)], route,
1182 entry);
Jan Kiszka04fa27f2012-05-16 15:41:10 -03001183 }
1184
1185 assert(route->kroute.type == KVM_IRQ_ROUTING_MSI);
1186
Peter Maydell3889c3f2012-07-26 15:35:12 +01001187 return kvm_set_irq(s, route->kroute.gsi, 1);
Jan Kiszka04fa27f2012-05-16 15:41:10 -03001188}
1189
Pavel Fedindc9f06c2015-10-15 16:44:52 +03001190int kvm_irqchip_add_msi_route(KVMState *s, MSIMessage msg, PCIDevice *dev)
Jan Kiszka92b4e482012-05-17 10:32:33 -03001191{
Michael S. Tsirkin0fbc2072013-06-04 14:52:32 +03001192 struct kvm_irq_routing_entry kroute = {};
Jan Kiszka92b4e482012-05-17 10:32:33 -03001193 int virq;
1194
Alexey Kardashevskiy76fe21d2013-09-03 18:08:25 +10001195 if (kvm_gsi_direct_mapping()) {
Eric Auger1850b6b2015-06-02 14:56:23 +01001196 return kvm_arch_msi_data_to_gsi(msg.data);
Alexey Kardashevskiy76fe21d2013-09-03 18:08:25 +10001197 }
1198
Peter Maydellf3e1bed2012-07-26 15:35:16 +01001199 if (!kvm_gsi_routing_enabled()) {
Jan Kiszka92b4e482012-05-17 10:32:33 -03001200 return -ENOSYS;
1201 }
1202
1203 virq = kvm_irqchip_get_virq(s);
1204 if (virq < 0) {
1205 return virq;
1206 }
1207
1208 kroute.gsi = virq;
1209 kroute.type = KVM_IRQ_ROUTING_MSI;
1210 kroute.flags = 0;
1211 kroute.u.msi.address_lo = (uint32_t)msg.address;
1212 kroute.u.msi.address_hi = msg.address >> 32;
Alexander Grafd07cc1f2013-04-16 15:05:22 +02001213 kroute.u.msi.data = le32_to_cpu(msg.data);
Pavel Fedindc9f06c2015-10-15 16:44:52 +03001214 if (kvm_arch_fixup_msi_route(&kroute, msg.address, msg.data, dev)) {
Frank Blaschka9e03a042015-01-09 09:04:40 +01001215 kvm_irqchip_release_virq(s, virq);
1216 return -EINVAL;
1217 }
Jan Kiszka92b4e482012-05-17 10:32:33 -03001218
1219 kvm_add_routing_entry(s, &kroute);
Alexander Grafcb925cf2013-04-17 01:11:55 +02001220 kvm_irqchip_commit_routes(s);
Jan Kiszka92b4e482012-05-17 10:32:33 -03001221
1222 return virq;
1223}
1224
Pavel Fedindc9f06c2015-10-15 16:44:52 +03001225int kvm_irqchip_update_msi_route(KVMState *s, int virq, MSIMessage msg,
1226 PCIDevice *dev)
Jan Kiszkacc574072012-08-27 08:28:38 +02001227{
Michael S. Tsirkin0fbc2072013-06-04 14:52:32 +03001228 struct kvm_irq_routing_entry kroute = {};
Jan Kiszkacc574072012-08-27 08:28:38 +02001229
Alexey Kardashevskiy76fe21d2013-09-03 18:08:25 +10001230 if (kvm_gsi_direct_mapping()) {
1231 return 0;
1232 }
1233
Jan Kiszkacc574072012-08-27 08:28:38 +02001234 if (!kvm_irqchip_in_kernel()) {
1235 return -ENOSYS;
1236 }
1237
1238 kroute.gsi = virq;
1239 kroute.type = KVM_IRQ_ROUTING_MSI;
1240 kroute.flags = 0;
1241 kroute.u.msi.address_lo = (uint32_t)msg.address;
1242 kroute.u.msi.address_hi = msg.address >> 32;
Alexander Grafd07cc1f2013-04-16 15:05:22 +02001243 kroute.u.msi.data = le32_to_cpu(msg.data);
Pavel Fedindc9f06c2015-10-15 16:44:52 +03001244 if (kvm_arch_fixup_msi_route(&kroute, msg.address, msg.data, dev)) {
Frank Blaschka9e03a042015-01-09 09:04:40 +01001245 return -EINVAL;
1246 }
Jan Kiszkacc574072012-08-27 08:28:38 +02001247
1248 return kvm_update_routing_entry(s, &kroute);
1249}
1250
Vincenzo Maffioneca916d32013-07-22 11:51:33 +02001251static int kvm_irqchip_assign_irqfd(KVMState *s, int fd, int rfd, int virq,
1252 bool assign)
Jan Kiszka39853bb2012-05-17 10:32:36 -03001253{
1254 struct kvm_irqfd irqfd = {
1255 .fd = fd,
1256 .gsi = virq,
1257 .flags = assign ? 0 : KVM_IRQFD_FLAG_DEASSIGN,
1258 };
1259
Vincenzo Maffioneca916d32013-07-22 11:51:33 +02001260 if (rfd != -1) {
1261 irqfd.flags |= KVM_IRQFD_FLAG_RESAMPLE;
1262 irqfd.resamplefd = rfd;
1263 }
1264
Peter Maydellcc7e0dd2012-07-26 15:35:14 +01001265 if (!kvm_irqfds_enabled()) {
Jan Kiszka39853bb2012-05-17 10:32:36 -03001266 return -ENOSYS;
1267 }
1268
1269 return kvm_vm_ioctl(s, KVM_IRQFD, &irqfd);
1270}
1271
Cornelia Huckd426d9f2013-07-15 17:45:03 +02001272int kvm_irqchip_add_adapter_route(KVMState *s, AdapterInfo *adapter)
1273{
Christian Borntraegere9af2fe2014-11-20 22:10:58 +01001274 struct kvm_irq_routing_entry kroute = {};
Cornelia Huckd426d9f2013-07-15 17:45:03 +02001275 int virq;
1276
1277 if (!kvm_gsi_routing_enabled()) {
1278 return -ENOSYS;
1279 }
1280
1281 virq = kvm_irqchip_get_virq(s);
1282 if (virq < 0) {
1283 return virq;
1284 }
1285
1286 kroute.gsi = virq;
1287 kroute.type = KVM_IRQ_ROUTING_S390_ADAPTER;
1288 kroute.flags = 0;
1289 kroute.u.adapter.summary_addr = adapter->summary_addr;
1290 kroute.u.adapter.ind_addr = adapter->ind_addr;
1291 kroute.u.adapter.summary_offset = adapter->summary_offset;
1292 kroute.u.adapter.ind_offset = adapter->ind_offset;
1293 kroute.u.adapter.adapter_id = adapter->adapter_id;
1294
1295 kvm_add_routing_entry(s, &kroute);
Cornelia Huckd426d9f2013-07-15 17:45:03 +02001296
1297 return virq;
1298}
1299
Jan Kiszka84b058d2011-10-15 11:49:47 +02001300#else /* !KVM_CAP_IRQ_ROUTING */
1301
Alexander Graf7b774592013-04-16 15:58:13 +02001302void kvm_init_irq_routing(KVMState *s)
Jan Kiszka84b058d2011-10-15 11:49:47 +02001303{
1304}
Jan Kiszka04fa27f2012-05-16 15:41:10 -03001305
Jan Kiszkad3d3bef2012-06-05 21:03:57 +02001306void kvm_irqchip_release_virq(KVMState *s, int virq)
1307{
1308}
1309
Jan Kiszka04fa27f2012-05-16 15:41:10 -03001310int kvm_irqchip_send_msi(KVMState *s, MSIMessage msg)
1311{
1312 abort();
1313}
Jan Kiszka92b4e482012-05-17 10:32:33 -03001314
1315int kvm_irqchip_add_msi_route(KVMState *s, MSIMessage msg)
1316{
Alex Williamsondf410672012-06-25 09:40:39 -06001317 return -ENOSYS;
Jan Kiszka92b4e482012-05-17 10:32:33 -03001318}
Jan Kiszka39853bb2012-05-17 10:32:36 -03001319
Cornelia Huckd426d9f2013-07-15 17:45:03 +02001320int kvm_irqchip_add_adapter_route(KVMState *s, AdapterInfo *adapter)
1321{
1322 return -ENOSYS;
1323}
1324
Jan Kiszka39853bb2012-05-17 10:32:36 -03001325static int kvm_irqchip_assign_irqfd(KVMState *s, int fd, int virq, bool assign)
1326{
1327 abort();
1328}
Michael S. Tsirkindabe3142013-01-15 19:50:13 +02001329
1330int kvm_irqchip_update_msi_route(KVMState *s, int virq, MSIMessage msg)
1331{
1332 return -ENOSYS;
1333}
Jan Kiszka84b058d2011-10-15 11:49:47 +02001334#endif /* !KVM_CAP_IRQ_ROUTING */
1335
Eric Auger1c9b71a2015-07-06 12:15:13 -06001336int kvm_irqchip_add_irqfd_notifier_gsi(KVMState *s, EventNotifier *n,
1337 EventNotifier *rn, int virq)
Jan Kiszka39853bb2012-05-17 10:32:36 -03001338{
Vincenzo Maffioneca916d32013-07-22 11:51:33 +02001339 return kvm_irqchip_assign_irqfd(s, event_notifier_get_fd(n),
1340 rn ? event_notifier_get_fd(rn) : -1, virq, true);
Jan Kiszka39853bb2012-05-17 10:32:36 -03001341}
1342
Eric Auger1c9b71a2015-07-06 12:15:13 -06001343int kvm_irqchip_remove_irqfd_notifier_gsi(KVMState *s, EventNotifier *n,
1344 int virq)
Paolo Bonzini15b2bd12012-07-05 17:16:30 +02001345{
Vincenzo Maffioneca916d32013-07-22 11:51:33 +02001346 return kvm_irqchip_assign_irqfd(s, event_notifier_get_fd(n), -1, virq,
1347 false);
Paolo Bonzini15b2bd12012-07-05 17:16:30 +02001348}
1349
Eric Auger197e3522015-07-06 12:15:13 -06001350int kvm_irqchip_add_irqfd_notifier(KVMState *s, EventNotifier *n,
1351 EventNotifier *rn, qemu_irq irq)
1352{
1353 gpointer key, gsi;
1354 gboolean found = g_hash_table_lookup_extended(s->gsimap, irq, &key, &gsi);
1355
1356 if (!found) {
1357 return -ENXIO;
1358 }
1359 return kvm_irqchip_add_irqfd_notifier_gsi(s, n, rn, GPOINTER_TO_INT(gsi));
1360}
1361
1362int kvm_irqchip_remove_irqfd_notifier(KVMState *s, EventNotifier *n,
1363 qemu_irq irq)
1364{
1365 gpointer key, gsi;
1366 gboolean found = g_hash_table_lookup_extended(s->gsimap, irq, &key, &gsi);
1367
1368 if (!found) {
1369 return -ENXIO;
1370 }
1371 return kvm_irqchip_remove_irqfd_notifier_gsi(s, n, GPOINTER_TO_INT(gsi));
1372}
1373
1374void kvm_irqchip_set_qemuirq_gsi(KVMState *s, qemu_irq irq, int gsi)
1375{
1376 g_hash_table_insert(s->gsimap, irq, GINT_TO_POINTER(gsi));
1377}
1378
Paolo Bonzini8db49362015-06-18 18:30:15 +02001379static void kvm_irqchip_create(MachineState *machine, KVMState *s)
Jan Kiszka84b058d2011-10-15 11:49:47 +02001380{
Jan Kiszka84b058d2011-10-15 11:49:47 +02001381 int ret;
1382
Paolo Bonzini8db49362015-06-18 18:30:15 +02001383 if (kvm_check_extension(s, KVM_CAP_IRQCHIP)) {
1384 ;
1385 } else if (kvm_check_extension(s, KVM_CAP_S390_IRQCHIP)) {
1386 ret = kvm_vm_enable_cap(s, KVM_CAP_S390_IRQCHIP, 0);
1387 if (ret < 0) {
1388 fprintf(stderr, "Enable kernel irqchip failed: %s\n", strerror(-ret));
1389 exit(1);
1390 }
1391 } else {
1392 return;
Jan Kiszka84b058d2011-10-15 11:49:47 +02001393 }
1394
Christoffer Dalld6032e02014-02-26 17:20:00 +00001395 /* First probe and see if there's a arch-specific hook to create the
1396 * in-kernel irqchip for us */
1397 ret = kvm_arch_irqchip_create(s);
Paolo Bonzini8db49362015-06-18 18:30:15 +02001398 if (ret == 0) {
Christoffer Dalld6032e02014-02-26 17:20:00 +00001399 ret = kvm_vm_ioctl(s, KVM_CREATE_IRQCHIP);
Paolo Bonzini8db49362015-06-18 18:30:15 +02001400 }
1401 if (ret < 0) {
1402 fprintf(stderr, "Create kernel irqchip failed: %s\n", strerror(-ret));
1403 exit(1);
Jan Kiszka84b058d2011-10-15 11:49:47 +02001404 }
1405
Jan Kiszka3d4b2642012-01-31 19:17:52 +01001406 kvm_kernel_irqchip = true;
Peter Maydell7ae26bd2012-07-26 15:35:11 +01001407 /* If we have an in-kernel IRQ chip then we must have asynchronous
1408 * interrupt delivery (though the reverse is not necessarily true)
1409 */
1410 kvm_async_interrupts_allowed = true;
Alexander Graf215e79c2013-04-24 22:24:12 +02001411 kvm_halt_in_kernel_allowed = true;
Jan Kiszka84b058d2011-10-15 11:49:47 +02001412
1413 kvm_init_irq_routing(s);
1414
Eric Auger197e3522015-07-06 12:15:13 -06001415 s->gsimap = g_hash_table_new(g_direct_hash, g_direct_equal);
Jan Kiszka84b058d2011-10-15 11:49:47 +02001416}
1417
Andrew Jones670436c2013-08-23 15:24:37 +02001418/* Find number of supported CPUs using the recommended
1419 * procedure from the kernel API documentation to cope with
1420 * older kernels that may be missing capabilities.
1421 */
1422static int kvm_recommended_vcpus(KVMState *s)
1423{
1424 int ret = kvm_check_extension(s, KVM_CAP_NR_VCPUS);
1425 return (ret) ? ret : 4;
1426}
1427
Dunrong Huang3ed444e2012-07-31 19:18:17 +08001428static int kvm_max_vcpus(KVMState *s)
1429{
Andrew Jones670436c2013-08-23 15:24:37 +02001430 int ret = kvm_check_extension(s, KVM_CAP_MAX_VCPUS);
1431 return (ret) ? ret : kvm_recommended_vcpus(s);
Dunrong Huang3ed444e2012-07-31 19:18:17 +08001432}
1433
Eduardo Habkostf6a1ef62014-09-26 17:45:30 -03001434static int kvm_init(MachineState *ms)
aliguori05330442008-11-05 16:29:27 +00001435{
Eduardo Habkostf6a1ef62014-09-26 17:45:30 -03001436 MachineClass *mc = MACHINE_GET_CLASS(ms);
Jan Kiszka168ccc12009-06-07 11:30:25 +02001437 static const char upgrade_note[] =
1438 "Please upgrade to at least kernel 2.6.29 or recent kvm-kmod\n"
1439 "(see http://sourceforge.net/projects/kvm).\n";
Andrew Jones670436c2013-08-23 15:24:37 +02001440 struct {
1441 const char *name;
1442 int num;
1443 } num_cpus[] = {
1444 { "SMP", smp_cpus },
1445 { "hotpluggable", max_cpus },
1446 { NULL, }
1447 }, *nc = num_cpus;
1448 int soft_vcpus_limit, hard_vcpus_limit;
aliguori05330442008-11-05 16:29:27 +00001449 KVMState *s;
Jan Kiszka94a8d392011-01-21 21:48:17 +01001450 const KVMCapabilityInfo *missing_cap;
aliguori05330442008-11-05 16:29:27 +00001451 int ret;
Paolo Bonzini7bbda042015-06-18 18:30:13 +02001452 int type = 0;
Aneesh Kumar K.V135a1292013-12-23 21:10:40 +05301453 const char *kvm_type;
aliguori05330442008-11-05 16:29:27 +00001454
Eduardo Habkostfc020862014-09-26 17:45:32 -03001455 s = KVM_STATE(ms->accelerator);
aliguori05330442008-11-05 16:29:27 +00001456
David Gibson3145fcb2012-04-04 11:15:54 +10001457 /*
1458 * On systems where the kernel can support different base page
1459 * sizes, host page size may be different from TARGET_PAGE_SIZE,
1460 * even with KVM. TARGET_PAGE_SIZE is assumed to be the minimum
1461 * page size for the system though.
1462 */
1463 assert(TARGET_PAGE_SIZE <= getpagesize());
Alexey Kardashevskiy47c16ed2014-01-17 11:12:07 -07001464 page_size_init();
David Gibson3145fcb2012-04-04 11:15:54 +10001465
James Hoganaed6efb2014-06-17 23:10:31 +01001466 s->sigmask_len = 8;
1467
aliguorie22a25c2009-03-12 20:12:48 +00001468#ifdef KVM_CAP_SET_GUEST_DEBUG
Blue Swirl72cf2d42009-09-12 07:36:22 +00001469 QTAILQ_INIT(&s->kvm_sw_breakpoints);
aliguorie22a25c2009-03-12 20:12:48 +00001470#endif
aliguori05330442008-11-05 16:29:27 +00001471 s->vmfd = -1;
Kevin Wolf40ff6d72009-12-02 12:24:42 +01001472 s->fd = qemu_open("/dev/kvm", O_RDWR);
aliguori05330442008-11-05 16:29:27 +00001473 if (s->fd == -1) {
1474 fprintf(stderr, "Could not access KVM kernel module: %m\n");
1475 ret = -errno;
1476 goto err;
1477 }
1478
1479 ret = kvm_ioctl(s, KVM_GET_API_VERSION, 0);
1480 if (ret < KVM_API_VERSION) {
Eduardo Habkost0e1dac62014-05-30 17:26:22 -03001481 if (ret >= 0) {
aliguori05330442008-11-05 16:29:27 +00001482 ret = -EINVAL;
Jan Kiszkaa426e122011-01-04 09:32:13 +01001483 }
aliguori05330442008-11-05 16:29:27 +00001484 fprintf(stderr, "kvm version too old\n");
1485 goto err;
1486 }
1487
1488 if (ret > KVM_API_VERSION) {
1489 ret = -EINVAL;
1490 fprintf(stderr, "kvm version not supported\n");
1491 goto err;
1492 }
1493
Alex Williamsonfb541ca2013-11-22 12:12:44 -07001494 s->nr_slots = kvm_check_extension(s, KVM_CAP_NR_MEMSLOTS);
1495
1496 /* If unspecified, use the default value */
1497 if (!s->nr_slots) {
1498 s->nr_slots = 32;
1499 }
1500
Andrew Jones670436c2013-08-23 15:24:37 +02001501 /* check the vcpu limits */
1502 soft_vcpus_limit = kvm_recommended_vcpus(s);
1503 hard_vcpus_limit = kvm_max_vcpus(s);
Dunrong Huang3ed444e2012-07-31 19:18:17 +08001504
Andrew Jones670436c2013-08-23 15:24:37 +02001505 while (nc->name) {
1506 if (nc->num > soft_vcpus_limit) {
1507 fprintf(stderr,
1508 "Warning: Number of %s cpus requested (%d) exceeds "
1509 "the recommended cpus supported by KVM (%d)\n",
1510 nc->name, nc->num, soft_vcpus_limit);
1511
1512 if (nc->num > hard_vcpus_limit) {
Andrew Jones670436c2013-08-23 15:24:37 +02001513 fprintf(stderr, "Number of %s cpus requested (%d) exceeds "
1514 "the maximum cpus supported by KVM (%d)\n",
1515 nc->name, nc->num, hard_vcpus_limit);
Marcelo Tosatti9ba3cf52014-02-25 23:22:07 -03001516 exit(1);
Andrew Jones670436c2013-08-23 15:24:37 +02001517 }
1518 }
1519 nc++;
Marcelo Tosatti7dc52522013-08-12 16:56:31 -03001520 }
1521
Aneesh Kumar K.V135a1292013-12-23 21:10:40 +05301522 kvm_type = qemu_opt_get(qemu_get_machine_opts(), "kvm-type");
Marcel Apfelbaumf1e29872014-04-09 20:34:52 +03001523 if (mc->kvm_type) {
1524 type = mc->kvm_type(kvm_type);
Aneesh Kumar K.V135a1292013-12-23 21:10:40 +05301525 } else if (kvm_type) {
Eduardo Habkost0e1dac62014-05-30 17:26:22 -03001526 ret = -EINVAL;
Aneesh Kumar K.V135a1292013-12-23 21:10:40 +05301527 fprintf(stderr, "Invalid argument kvm-type=%s\n", kvm_type);
1528 goto err;
1529 }
1530
thomas knych94ccff12014-01-09 13:14:23 -08001531 do {
Aneesh Kumar K.V135a1292013-12-23 21:10:40 +05301532 ret = kvm_ioctl(s, KVM_CREATE_VM, type);
thomas knych94ccff12014-01-09 13:14:23 -08001533 } while (ret == -EINTR);
1534
1535 if (ret < 0) {
Alexander Graf521f4382014-01-27 15:18:09 +01001536 fprintf(stderr, "ioctl(KVM_CREATE_VM) failed: %d %s\n", -ret,
thomas knych94ccff12014-01-09 13:14:23 -08001537 strerror(-ret));
1538
Alexander Graf0104dca2010-04-01 18:42:37 +02001539#ifdef TARGET_S390X
Cornelia Huck2c80e992015-04-23 17:03:46 +02001540 if (ret == -EINVAL) {
1541 fprintf(stderr,
1542 "Host kernel setup problem detected. Please verify:\n");
1543 fprintf(stderr, "- for kernels supporting the switch_amode or"
1544 " user_mode parameters, whether\n");
1545 fprintf(stderr,
1546 " user space is running in primary address space\n");
1547 fprintf(stderr,
1548 "- for kernels supporting the vm.allocate_pgste sysctl, "
1549 "whether it is enabled\n");
1550 }
Alexander Graf0104dca2010-04-01 18:42:37 +02001551#endif
aliguori05330442008-11-05 16:29:27 +00001552 goto err;
Alexander Graf0104dca2010-04-01 18:42:37 +02001553 }
aliguori05330442008-11-05 16:29:27 +00001554
thomas knych94ccff12014-01-09 13:14:23 -08001555 s->vmfd = ret;
Jan Kiszka94a8d392011-01-21 21:48:17 +01001556 missing_cap = kvm_check_extension_list(s, kvm_required_capabilites);
1557 if (!missing_cap) {
1558 missing_cap =
1559 kvm_check_extension_list(s, kvm_arch_required_capabilities);
1560 }
1561 if (missing_cap) {
Anthony Liguoriad7b8b32009-05-08 15:33:24 -05001562 ret = -EINVAL;
Jan Kiszka94a8d392011-01-21 21:48:17 +01001563 fprintf(stderr, "kvm does not support %s\n%s",
1564 missing_cap->name, upgrade_note);
aliguori05330442008-11-05 16:29:27 +00001565 goto err;
1566 }
1567
Anthony Liguoriad7b8b32009-05-08 15:33:24 -05001568 s->coalesced_mmio = kvm_check_extension(s, KVM_CAP_COALESCED_MMIO);
aliguorif65ed4c2008-12-09 20:09:57 +00001569
Jan Kiszkae69917e2009-05-01 20:42:15 +02001570 s->broken_set_mem_region = 1;
Lai Jiangshan14a09512010-12-10 15:52:36 +08001571 ret = kvm_check_extension(s, KVM_CAP_JOIN_MEMORY_REGIONS_WORKS);
Jan Kiszkae69917e2009-05-01 20:42:15 +02001572 if (ret > 0) {
1573 s->broken_set_mem_region = 0;
1574 }
Jan Kiszkae69917e2009-05-01 20:42:15 +02001575
Jan Kiszkaa0fb0022009-11-25 00:33:03 +01001576#ifdef KVM_CAP_VCPU_EVENTS
1577 s->vcpu_events = kvm_check_extension(s, KVM_CAP_VCPU_EVENTS);
1578#endif
1579
Jan Kiszkab0b1d692010-03-01 19:10:29 +01001580 s->robust_singlestep =
1581 kvm_check_extension(s, KVM_CAP_X86_ROBUST_SINGLESTEP);
Jan Kiszkab0b1d692010-03-01 19:10:29 +01001582
Jan Kiszkaff44f1a2010-03-12 15:20:49 +01001583#ifdef KVM_CAP_DEBUGREGS
1584 s->debugregs = kvm_check_extension(s, KVM_CAP_DEBUGREGS);
1585#endif
1586
Jan Kiszkad3d3bef2012-06-05 21:03:57 +02001587#ifdef KVM_CAP_IRQ_ROUTING
Pavel Fedin50bf31b2015-10-15 16:44:50 +03001588 kvm_direct_msi_allowed = (kvm_check_extension(s, KVM_CAP_SIGNAL_MSI) > 0);
Jan Kiszkad3d3bef2012-06-05 21:03:57 +02001589#endif
Jan Kiszka4a3adeb2012-05-16 15:41:14 -03001590
Jan Kiszka3ab73842012-08-27 08:28:39 +02001591 s->intx_set_mask = kvm_check_extension(s, KVM_CAP_PCI_2_3);
1592
Jan Kiszkae333cd62012-08-24 13:34:47 +02001593 s->irq_set_ioctl = KVM_IRQ_LINE;
Peter Maydell8732fbd2012-08-15 12:08:13 +01001594 if (kvm_check_extension(s, KVM_CAP_IRQ_INJECT_STATUS)) {
Jan Kiszkae333cd62012-08-24 13:34:47 +02001595 s->irq_set_ioctl = KVM_IRQ_LINE_STATUS;
Peter Maydell8732fbd2012-08-15 12:08:13 +01001596 }
1597
Jordan Justendf9c8b72013-05-29 01:27:25 -07001598#ifdef KVM_CAP_READONLY_MEM
1599 kvm_readonly_mem_allowed =
1600 (kvm_check_extension(s, KVM_CAP_READONLY_MEM) > 0);
1601#endif
1602
Nikolay Nikolaev69e03ae2014-05-27 15:03:35 +03001603 kvm_eventfds_allowed =
1604 (kvm_check_extension(s, KVM_CAP_IOEVENTFD) > 0);
1605
Eric Augerf41389a2014-10-31 13:38:18 +00001606 kvm_irqfds_allowed =
1607 (kvm_check_extension(s, KVM_CAP_IRQFD) > 0);
1608
1609 kvm_resamplefds_allowed =
1610 (kvm_check_extension(s, KVM_CAP_IRQFD_RESAMPLE) > 0);
1611
Dominik Dingeld0a073a2015-03-12 13:53:49 +01001612 kvm_vm_attributes_allowed =
1613 (kvm_check_extension(s, KVM_CAP_VM_ATTRIBUTES) > 0);
1614
Marcel Apfelbaumb16565b2015-02-04 17:43:51 +02001615 ret = kvm_arch_init(ms, s);
Jan Kiszkaa426e122011-01-04 09:32:13 +01001616 if (ret < 0) {
aliguori05330442008-11-05 16:29:27 +00001617 goto err;
Jan Kiszkaa426e122011-01-04 09:32:13 +01001618 }
aliguori05330442008-11-05 16:29:27 +00001619
Paolo Bonzini8db49362015-06-18 18:30:15 +02001620 if (machine_kernel_irqchip_allowed(ms)) {
1621 kvm_irqchip_create(ms, s);
Jan Kiszka84b058d2011-10-15 11:49:47 +02001622 }
1623
aliguori05330442008-11-05 16:29:27 +00001624 kvm_state = s;
Paolo Bonzini7bbda042015-06-18 18:30:13 +02001625
1626 s->memory_listener.listener.eventfd_add = kvm_mem_ioeventfd_add;
1627 s->memory_listener.listener.eventfd_del = kvm_mem_ioeventfd_del;
1628 s->memory_listener.listener.coalesced_mmio_add = kvm_coalesce_mmio_region;
1629 s->memory_listener.listener.coalesced_mmio_del = kvm_uncoalesce_mmio_region;
1630
1631 kvm_memory_listener_register(s, &s->memory_listener,
Paolo Bonzini38bfe692015-06-18 18:30:14 +02001632 &address_space_memory, 0);
Paolo Bonzini7bbda042015-06-18 18:30:13 +02001633 memory_listener_register(&kvm_io_listener,
1634 &address_space_io);
aliguori05330442008-11-05 16:29:27 +00001635
Stefan Hajnoczid2f2b8a2011-01-10 13:50:05 +02001636 s->many_ioeventfds = kvm_check_many_ioeventfds();
1637
Jan Kiszkaaa7f74d2011-04-13 01:32:56 +02001638 cpu_interrupt_handler = kvm_handle_interrupt;
1639
aliguori05330442008-11-05 16:29:27 +00001640 return 0;
1641
1642err:
Eduardo Habkost0e1dac62014-05-30 17:26:22 -03001643 assert(ret < 0);
Stefan Weil6d1cc322012-09-03 22:40:40 +02001644 if (s->vmfd >= 0) {
1645 close(s->vmfd);
1646 }
1647 if (s->fd != -1) {
1648 close(s->fd);
aliguori05330442008-11-05 16:29:27 +00001649 }
Paolo Bonzini7bbda042015-06-18 18:30:13 +02001650 g_free(s->memory_listener.slots);
aliguori05330442008-11-05 16:29:27 +00001651
1652 return ret;
1653}
1654
James Hoganaed6efb2014-06-17 23:10:31 +01001655void kvm_set_sigmask_len(KVMState *s, unsigned int sigmask_len)
1656{
1657 s->sigmask_len = sigmask_len;
1658}
1659
Paolo Bonzini4c663752015-04-08 13:30:58 +02001660static void kvm_handle_io(uint16_t port, MemTxAttrs attrs, void *data, int direction,
1661 int size, uint32_t count)
aliguori05330442008-11-05 16:29:27 +00001662{
1663 int i;
1664 uint8_t *ptr = data;
1665
1666 for (i = 0; i < count; i++) {
Paolo Bonzini4c663752015-04-08 13:30:58 +02001667 address_space_rw(&address_space_io, port, attrs,
Peter Maydell5c9eb022015-04-26 16:49:24 +01001668 ptr, size,
Jan Kiszka354678c2013-08-13 14:43:57 +02001669 direction == KVM_EXIT_IO_OUT);
aliguori05330442008-11-05 16:29:27 +00001670 ptr += size;
1671 }
aliguori05330442008-11-05 16:29:27 +00001672}
1673
Andreas Färber5326ab52013-05-27 01:55:29 +02001674static int kvm_handle_internal_error(CPUState *cpu, struct kvm_run *run)
Marcelo Tosatti7c80eef2010-03-23 13:37:11 -03001675{
Radim Krčmář977c7b62014-01-21 18:11:31 +01001676 fprintf(stderr, "KVM internal error. Suberror: %d\n",
1677 run->internal.suberror);
1678
Marcelo Tosatti7c80eef2010-03-23 13:37:11 -03001679 if (kvm_check_extension(kvm_state, KVM_CAP_INTERNAL_ERROR_DATA)) {
1680 int i;
1681
Marcelo Tosatti7c80eef2010-03-23 13:37:11 -03001682 for (i = 0; i < run->internal.ndata; ++i) {
1683 fprintf(stderr, "extra data[%d]: %"PRIx64"\n",
1684 i, (uint64_t)run->internal.data[i]);
1685 }
1686 }
Marcelo Tosatti7c80eef2010-03-23 13:37:11 -03001687 if (run->internal.suberror == KVM_INTERNAL_ERROR_EMULATION) {
1688 fprintf(stderr, "emulation failure\n");
Andreas Färber20d695a2012-10-31 06:57:49 +01001689 if (!kvm_arch_stop_on_emulation_error(cpu)) {
Andreas Färber878096e2013-05-27 01:33:50 +02001690 cpu_dump_state(cpu, stderr, fprintf, CPU_DUMP_CODE);
Jan Kiszkad73cd8f2011-03-15 12:26:27 +01001691 return EXCP_INTERRUPT;
Jan Kiszkaa426e122011-01-04 09:32:13 +01001692 }
Marcelo Tosatti7c80eef2010-03-23 13:37:11 -03001693 }
1694 /* FIXME: Should trigger a qmp message to let management know
1695 * something went wrong.
1696 */
Jan Kiszka73aaec42011-01-21 21:48:06 +01001697 return -1;
Marcelo Tosatti7c80eef2010-03-23 13:37:11 -03001698}
Marcelo Tosatti7c80eef2010-03-23 13:37:11 -03001699
Sheng Yang62a27442010-01-26 19:21:16 +08001700void kvm_flush_coalesced_mmio_buffer(void)
aliguorif65ed4c2008-12-09 20:09:57 +00001701{
aliguorif65ed4c2008-12-09 20:09:57 +00001702 KVMState *s = kvm_state;
Avi Kivity1cae88b2011-10-18 19:43:12 +02001703
1704 if (s->coalesced_flush_in_progress) {
1705 return;
1706 }
1707
1708 s->coalesced_flush_in_progress = true;
1709
Sheng Yang62a27442010-01-26 19:21:16 +08001710 if (s->coalesced_mmio_ring) {
1711 struct kvm_coalesced_mmio_ring *ring = s->coalesced_mmio_ring;
aliguorif65ed4c2008-12-09 20:09:57 +00001712 while (ring->first != ring->last) {
1713 struct kvm_coalesced_mmio *ent;
1714
1715 ent = &ring->coalesced_mmio[ring->first];
1716
1717 cpu_physical_memory_write(ent->phys_addr, ent->data, ent->len);
Marcelo Tosatti85199472010-02-22 13:57:54 -03001718 smp_wmb();
aliguorif65ed4c2008-12-09 20:09:57 +00001719 ring->first = (ring->first + 1) % KVM_COALESCED_MMIO_MAX;
1720 }
1721 }
Avi Kivity1cae88b2011-10-18 19:43:12 +02001722
1723 s->coalesced_flush_in_progress = false;
aliguorif65ed4c2008-12-09 20:09:57 +00001724}
1725
Andreas Färber20d695a2012-10-31 06:57:49 +01001726static void do_kvm_cpu_synchronize_state(void *arg)
Avi Kivity4c0960c2009-08-17 23:19:53 +03001727{
Andreas Färber20d695a2012-10-31 06:57:49 +01001728 CPUState *cpu = arg;
Jan Kiszka2705d562010-05-04 09:45:23 -03001729
Andreas Färber20d695a2012-10-31 06:57:49 +01001730 if (!cpu->kvm_vcpu_dirty) {
1731 kvm_arch_get_registers(cpu);
1732 cpu->kvm_vcpu_dirty = true;
Avi Kivity4c0960c2009-08-17 23:19:53 +03001733 }
1734}
1735
Andreas Färberdd1750d2013-05-01 13:45:44 +02001736void kvm_cpu_synchronize_state(CPUState *cpu)
Jan Kiszka2705d562010-05-04 09:45:23 -03001737{
Andreas Färber20d695a2012-10-31 06:57:49 +01001738 if (!cpu->kvm_vcpu_dirty) {
1739 run_on_cpu(cpu, do_kvm_cpu_synchronize_state, cpu);
Jan Kiszkaa426e122011-01-04 09:32:13 +01001740 }
Jan Kiszka2705d562010-05-04 09:45:23 -03001741}
1742
David Hildenbrandc8e20852014-08-20 14:55:25 +02001743static void do_kvm_cpu_synchronize_post_reset(void *arg)
1744{
1745 CPUState *cpu = arg;
1746
1747 kvm_arch_put_registers(cpu, KVM_PUT_RESET_STATE);
1748 cpu->kvm_vcpu_dirty = false;
1749}
1750
Igor Mammedov3f24a582013-04-11 16:51:41 +02001751void kvm_cpu_synchronize_post_reset(CPUState *cpu)
Jan Kiszkaea375f92010-03-01 19:10:30 +01001752{
David Hildenbrandc8e20852014-08-20 14:55:25 +02001753 run_on_cpu(cpu, do_kvm_cpu_synchronize_post_reset, cpu);
1754}
1755
1756static void do_kvm_cpu_synchronize_post_init(void *arg)
1757{
1758 CPUState *cpu = arg;
1759
1760 kvm_arch_put_registers(cpu, KVM_PUT_FULL_STATE);
Andreas Färber20d695a2012-10-31 06:57:49 +01001761 cpu->kvm_vcpu_dirty = false;
Jan Kiszkaea375f92010-03-01 19:10:30 +01001762}
1763
Igor Mammedov3f24a582013-04-11 16:51:41 +02001764void kvm_cpu_synchronize_post_init(CPUState *cpu)
Jan Kiszkaea375f92010-03-01 19:10:30 +01001765{
David Hildenbrandc8e20852014-08-20 14:55:25 +02001766 run_on_cpu(cpu, do_kvm_cpu_synchronize_post_init, cpu);
Jan Kiszkaea375f92010-03-01 19:10:30 +01001767}
1768
Marcelo Tosattide9d61e2014-09-05 10:52:46 -03001769void kvm_cpu_clean_state(CPUState *cpu)
1770{
1771 cpu->kvm_vcpu_dirty = false;
1772}
1773
Andreas Färber1458c362013-05-26 23:46:55 +02001774int kvm_cpu_exec(CPUState *cpu)
aliguori05330442008-11-05 16:29:27 +00001775{
Andreas Färberf7575c962012-12-01 06:18:14 +01001776 struct kvm_run *run = cpu->kvm_run;
Jan Kiszka7cbb5332011-03-15 12:26:25 +01001777 int ret, run_ret;
aliguori05330442008-11-05 16:29:27 +00001778
Blue Swirl8c0d5772010-04-18 14:22:14 +00001779 DPRINTF("kvm_cpu_exec()\n");
aliguori05330442008-11-05 16:29:27 +00001780
Andreas Färber20d695a2012-10-31 06:57:49 +01001781 if (kvm_arch_process_async_events(cpu)) {
Andreas Färberfcd7d002012-12-17 08:02:44 +01001782 cpu->exit_request = 0;
Jan Kiszka6792a572011-02-07 12:19:18 +01001783 return EXCP_HLT;
Jan Kiszka9ccfac92011-02-01 22:16:00 +01001784 }
1785
Jan Kiszka4b8523e2015-06-18 18:47:23 +02001786 qemu_mutex_unlock_iothread();
1787
aliguori05330442008-11-05 16:29:27 +00001788 do {
Paolo Bonzini4c663752015-04-08 13:30:58 +02001789 MemTxAttrs attrs;
1790
Andreas Färber20d695a2012-10-31 06:57:49 +01001791 if (cpu->kvm_vcpu_dirty) {
1792 kvm_arch_put_registers(cpu, KVM_PUT_RUNTIME_STATE);
1793 cpu->kvm_vcpu_dirty = false;
Avi Kivity4c0960c2009-08-17 23:19:53 +03001794 }
1795
Andreas Färber20d695a2012-10-31 06:57:49 +01001796 kvm_arch_pre_run(cpu, run);
Andreas Färberfcd7d002012-12-17 08:02:44 +01001797 if (cpu->exit_request) {
Jan Kiszka9ccfac92011-02-01 22:16:00 +01001798 DPRINTF("interrupt exit requested\n");
1799 /*
1800 * KVM requires us to reenter the kernel after IO exits to complete
1801 * instruction emulation. This self-signal will ensure that we
1802 * leave ASAP again.
1803 */
1804 qemu_cpu_kick_self();
1805 }
Jan Kiszka9ccfac92011-02-01 22:16:00 +01001806
Andreas Färber1bc22652012-10-31 06:06:49 +01001807 run_ret = kvm_vcpu_ioctl(cpu, KVM_RUN, 0);
Jan Kiszka9ccfac92011-02-01 22:16:00 +01001808
Paolo Bonzini4c663752015-04-08 13:30:58 +02001809 attrs = kvm_arch_post_run(cpu, run);
aliguori05330442008-11-05 16:29:27 +00001810
Jan Kiszka7cbb5332011-03-15 12:26:25 +01001811 if (run_ret < 0) {
Jan Kiszkadc77d342011-03-15 12:26:26 +01001812 if (run_ret == -EINTR || run_ret == -EAGAIN) {
1813 DPRINTF("io window exit\n");
Jan Kiszkad73cd8f2011-03-15 12:26:27 +01001814 ret = EXCP_INTERRUPT;
Jan Kiszkadc77d342011-03-15 12:26:26 +01001815 break;
1816 }
Michael Ellerman7b011fb2011-12-16 11:20:20 +11001817 fprintf(stderr, "error: kvm run failed %s\n",
1818 strerror(-run_ret));
Laurent Vivierdae02ba2015-05-18 21:06:47 +02001819#ifdef TARGET_PPC
1820 if (run_ret == -EBUSY) {
1821 fprintf(stderr,
1822 "This is probably because your SMT is enabled.\n"
1823 "VCPU can only run on primary threads with all "
1824 "secondary threads offline.\n");
1825 }
1826#endif
Paolo Bonzinia85e1302014-08-29 15:58:20 +02001827 ret = -1;
1828 break;
aliguori05330442008-11-05 16:29:27 +00001829 }
1830
Kazuya Saitob76ac802013-03-29 13:27:52 +09001831 trace_kvm_run_exit(cpu->cpu_index, run->exit_reason);
aliguori05330442008-11-05 16:29:27 +00001832 switch (run->exit_reason) {
1833 case KVM_EXIT_IO:
Blue Swirl8c0d5772010-04-18 14:22:14 +00001834 DPRINTF("handle_io\n");
Jan Kiszka80b7d2e2015-06-18 18:47:24 +02001835 /* Called outside BQL */
Paolo Bonzini4c663752015-04-08 13:30:58 +02001836 kvm_handle_io(run->io.port, attrs,
Jan Kiszkab30e93e2011-02-01 22:16:01 +01001837 (uint8_t *)run + run->io.data_offset,
1838 run->io.direction,
1839 run->io.size,
1840 run->io.count);
Jan Kiszkad73cd8f2011-03-15 12:26:27 +01001841 ret = 0;
aliguori05330442008-11-05 16:29:27 +00001842 break;
1843 case KVM_EXIT_MMIO:
Blue Swirl8c0d5772010-04-18 14:22:14 +00001844 DPRINTF("handle_mmio\n");
Paolo Bonzinide7ea882015-06-18 18:47:26 +02001845 /* Called outside BQL */
Paolo Bonzini4c663752015-04-08 13:30:58 +02001846 address_space_rw(&address_space_memory,
1847 run->mmio.phys_addr, attrs,
1848 run->mmio.data,
1849 run->mmio.len,
1850 run->mmio.is_write);
Jan Kiszkad73cd8f2011-03-15 12:26:27 +01001851 ret = 0;
aliguori05330442008-11-05 16:29:27 +00001852 break;
1853 case KVM_EXIT_IRQ_WINDOW_OPEN:
Blue Swirl8c0d5772010-04-18 14:22:14 +00001854 DPRINTF("irq_window_open\n");
Jan Kiszkad73cd8f2011-03-15 12:26:27 +01001855 ret = EXCP_INTERRUPT;
aliguori05330442008-11-05 16:29:27 +00001856 break;
1857 case KVM_EXIT_SHUTDOWN:
Blue Swirl8c0d5772010-04-18 14:22:14 +00001858 DPRINTF("shutdown\n");
aliguori05330442008-11-05 16:29:27 +00001859 qemu_system_reset_request();
Jan Kiszkad73cd8f2011-03-15 12:26:27 +01001860 ret = EXCP_INTERRUPT;
aliguori05330442008-11-05 16:29:27 +00001861 break;
1862 case KVM_EXIT_UNKNOWN:
Jan Kiszkabb44e0d2011-01-21 21:48:07 +01001863 fprintf(stderr, "KVM: unknown exit, hardware reason %" PRIx64 "\n",
1864 (uint64_t)run->hw.hardware_exit_reason);
Jan Kiszka73aaec42011-01-21 21:48:06 +01001865 ret = -1;
aliguori05330442008-11-05 16:29:27 +00001866 break;
Marcelo Tosatti7c80eef2010-03-23 13:37:11 -03001867 case KVM_EXIT_INTERNAL_ERROR:
Andreas Färber5326ab52013-05-27 01:55:29 +02001868 ret = kvm_handle_internal_error(cpu, run);
Marcelo Tosatti7c80eef2010-03-23 13:37:11 -03001869 break;
Pranavkumar Sawargaonkar99040442014-06-19 18:06:25 +01001870 case KVM_EXIT_SYSTEM_EVENT:
1871 switch (run->system_event.type) {
1872 case KVM_SYSTEM_EVENT_SHUTDOWN:
1873 qemu_system_shutdown_request();
1874 ret = EXCP_INTERRUPT;
1875 break;
1876 case KVM_SYSTEM_EVENT_RESET:
1877 qemu_system_reset_request();
1878 ret = EXCP_INTERRUPT;
1879 break;
Andrey Smetanin7c207b92015-07-03 15:01:43 +03001880 case KVM_SYSTEM_EVENT_CRASH:
1881 qemu_mutex_lock_iothread();
1882 qemu_system_guest_panicked();
1883 qemu_mutex_unlock_iothread();
1884 ret = 0;
1885 break;
Pranavkumar Sawargaonkar99040442014-06-19 18:06:25 +01001886 default:
1887 DPRINTF("kvm_arch_handle_exit\n");
1888 ret = kvm_arch_handle_exit(cpu, run);
1889 break;
1890 }
1891 break;
aliguori05330442008-11-05 16:29:27 +00001892 default:
Blue Swirl8c0d5772010-04-18 14:22:14 +00001893 DPRINTF("kvm_arch_handle_exit\n");
Andreas Färber20d695a2012-10-31 06:57:49 +01001894 ret = kvm_arch_handle_exit(cpu, run);
aliguori05330442008-11-05 16:29:27 +00001895 break;
1896 }
Jan Kiszkad73cd8f2011-03-15 12:26:27 +01001897 } while (ret == 0);
aliguori05330442008-11-05 16:29:27 +00001898
Jan Kiszka4b8523e2015-06-18 18:47:23 +02001899 qemu_mutex_lock_iothread();
1900
Jan Kiszka73aaec42011-01-21 21:48:06 +01001901 if (ret < 0) {
Andreas Färber878096e2013-05-27 01:33:50 +02001902 cpu_dump_state(cpu, stderr, fprintf, CPU_DUMP_CODE);
Luiz Capitulino0461d5a2011-09-30 14:45:27 -03001903 vm_stop(RUN_STATE_INTERNAL_ERROR);
Jan Kiszka73aaec42011-01-21 21:48:06 +01001904 }
aliguoribecfc392008-11-10 15:55:14 +00001905
Andreas Färberfcd7d002012-12-17 08:02:44 +01001906 cpu->exit_request = 0;
aliguori05330442008-11-05 16:29:27 +00001907 return ret;
1908}
1909
aliguori984b5182008-11-13 19:21:00 +00001910int kvm_ioctl(KVMState *s, int type, ...)
aliguori05330442008-11-05 16:29:27 +00001911{
1912 int ret;
aliguori984b5182008-11-13 19:21:00 +00001913 void *arg;
1914 va_list ap;
aliguori05330442008-11-05 16:29:27 +00001915
aliguori984b5182008-11-13 19:21:00 +00001916 va_start(ap, type);
1917 arg = va_arg(ap, void *);
1918 va_end(ap);
1919
Kazuya Saito9c775722013-03-29 13:27:05 +09001920 trace_kvm_ioctl(type, arg);
aliguori984b5182008-11-13 19:21:00 +00001921 ret = ioctl(s->fd, type, arg);
Jan Kiszkaa426e122011-01-04 09:32:13 +01001922 if (ret == -1) {
aliguori05330442008-11-05 16:29:27 +00001923 ret = -errno;
Jan Kiszkaa426e122011-01-04 09:32:13 +01001924 }
aliguori05330442008-11-05 16:29:27 +00001925 return ret;
1926}
1927
aliguori984b5182008-11-13 19:21:00 +00001928int kvm_vm_ioctl(KVMState *s, int type, ...)
aliguori05330442008-11-05 16:29:27 +00001929{
1930 int ret;
aliguori984b5182008-11-13 19:21:00 +00001931 void *arg;
1932 va_list ap;
aliguori05330442008-11-05 16:29:27 +00001933
aliguori984b5182008-11-13 19:21:00 +00001934 va_start(ap, type);
1935 arg = va_arg(ap, void *);
1936 va_end(ap);
1937
Kazuya Saito9c775722013-03-29 13:27:05 +09001938 trace_kvm_vm_ioctl(type, arg);
aliguori984b5182008-11-13 19:21:00 +00001939 ret = ioctl(s->vmfd, type, arg);
Jan Kiszkaa426e122011-01-04 09:32:13 +01001940 if (ret == -1) {
aliguori05330442008-11-05 16:29:27 +00001941 ret = -errno;
Jan Kiszkaa426e122011-01-04 09:32:13 +01001942 }
aliguori05330442008-11-05 16:29:27 +00001943 return ret;
1944}
1945
Andreas Färber1bc22652012-10-31 06:06:49 +01001946int kvm_vcpu_ioctl(CPUState *cpu, int type, ...)
aliguori05330442008-11-05 16:29:27 +00001947{
1948 int ret;
aliguori984b5182008-11-13 19:21:00 +00001949 void *arg;
1950 va_list ap;
aliguori05330442008-11-05 16:29:27 +00001951
aliguori984b5182008-11-13 19:21:00 +00001952 va_start(ap, type);
1953 arg = va_arg(ap, void *);
1954 va_end(ap);
1955
Kazuya Saito9c775722013-03-29 13:27:05 +09001956 trace_kvm_vcpu_ioctl(cpu->cpu_index, type, arg);
Andreas Färber8737c512012-10-31 05:29:00 +01001957 ret = ioctl(cpu->kvm_fd, type, arg);
Jan Kiszkaa426e122011-01-04 09:32:13 +01001958 if (ret == -1) {
aliguori05330442008-11-05 16:29:27 +00001959 ret = -errno;
Jan Kiszkaa426e122011-01-04 09:32:13 +01001960 }
aliguori05330442008-11-05 16:29:27 +00001961 return ret;
1962}
aliguoribd322082008-12-04 20:33:06 +00001963
Christoffer Dall0a6a7cc2014-02-26 17:20:00 +00001964int kvm_device_ioctl(int fd, int type, ...)
1965{
1966 int ret;
1967 void *arg;
1968 va_list ap;
1969
1970 va_start(ap, type);
1971 arg = va_arg(ap, void *);
1972 va_end(ap);
1973
1974 trace_kvm_device_ioctl(fd, type, arg);
1975 ret = ioctl(fd, type, arg);
1976 if (ret == -1) {
1977 ret = -errno;
1978 }
1979 return ret;
1980}
1981
Dominik Dingeld0a073a2015-03-12 13:53:49 +01001982int kvm_vm_check_attr(KVMState *s, uint32_t group, uint64_t attr)
1983{
1984 int ret;
1985 struct kvm_device_attr attribute = {
1986 .group = group,
1987 .attr = attr,
1988 };
1989
1990 if (!kvm_vm_attributes_allowed) {
1991 return 0;
1992 }
1993
1994 ret = kvm_vm_ioctl(s, KVM_HAS_DEVICE_ATTR, &attribute);
1995 /* kvm returns 0 on success for HAS_DEVICE_ATTR */
1996 return ret ? 0 : 1;
1997}
1998
Pavel Fedin4b3cfe72015-09-24 01:29:36 +01001999int kvm_device_check_attr(int dev_fd, uint32_t group, uint64_t attr)
2000{
2001 struct kvm_device_attr attribute = {
2002 .group = group,
2003 .attr = attr,
2004 .flags = 0,
2005 };
2006
2007 return kvm_device_ioctl(dev_fd, KVM_HAS_DEVICE_ATTR, &attribute) ? 0 : 1;
2008}
2009
2010void kvm_device_access(int fd, int group, uint64_t attr,
2011 void *val, bool write)
2012{
2013 struct kvm_device_attr kvmattr;
2014 int err;
2015
2016 kvmattr.flags = 0;
2017 kvmattr.group = group;
2018 kvmattr.attr = attr;
2019 kvmattr.addr = (uintptr_t)val;
2020
2021 err = kvm_device_ioctl(fd,
2022 write ? KVM_SET_DEVICE_ATTR : KVM_GET_DEVICE_ATTR,
2023 &kvmattr);
2024 if (err < 0) {
2025 error_report("KVM_%s_DEVICE_ATTR failed: %s\n"
2026 "Group %d attr 0x%016" PRIx64, write ? "SET" : "GET",
2027 strerror(-err), group, attr);
2028 abort();
2029 }
2030}
2031
aliguoribd322082008-12-04 20:33:06 +00002032int kvm_has_sync_mmu(void)
2033{
Jan Kiszka94a8d392011-01-21 21:48:17 +01002034 return kvm_check_extension(kvm_state, KVM_CAP_SYNC_MMU);
aliguoribd322082008-12-04 20:33:06 +00002035}
aliguorie22a25c2009-03-12 20:12:48 +00002036
Jan Kiszkaa0fb0022009-11-25 00:33:03 +01002037int kvm_has_vcpu_events(void)
2038{
2039 return kvm_state->vcpu_events;
2040}
2041
Jan Kiszkab0b1d692010-03-01 19:10:29 +01002042int kvm_has_robust_singlestep(void)
2043{
2044 return kvm_state->robust_singlestep;
2045}
2046
Jan Kiszkaff44f1a2010-03-12 15:20:49 +01002047int kvm_has_debugregs(void)
2048{
2049 return kvm_state->debugregs;
2050}
2051
Stefan Hajnoczid2f2b8a2011-01-10 13:50:05 +02002052int kvm_has_many_ioeventfds(void)
2053{
2054 if (!kvm_enabled()) {
2055 return 0;
2056 }
2057 return kvm_state->many_ioeventfds;
2058}
2059
Jan Kiszka84b058d2011-10-15 11:49:47 +02002060int kvm_has_gsi_routing(void)
2061{
Alexander Grafa9c5eb02012-01-25 18:28:05 +01002062#ifdef KVM_CAP_IRQ_ROUTING
Jan Kiszka84b058d2011-10-15 11:49:47 +02002063 return kvm_check_extension(kvm_state, KVM_CAP_IRQ_ROUTING);
Alexander Grafa9c5eb02012-01-25 18:28:05 +01002064#else
2065 return false;
2066#endif
Jan Kiszka84b058d2011-10-15 11:49:47 +02002067}
2068
Jan Kiszka3ab73842012-08-27 08:28:39 +02002069int kvm_has_intx_set_mask(void)
2070{
2071 return kvm_state->intx_set_mask;
2072}
2073
Jan Kiszka6f0437e2009-04-26 18:03:40 +02002074void kvm_setup_guest_memory(void *start, size_t size)
2075{
2076 if (!kvm_has_sync_mmu()) {
Andreas Färbere78815a2010-09-25 11:26:05 +00002077 int ret = qemu_madvise(start, size, QEMU_MADV_DONTFORK);
Jan Kiszka6f0437e2009-04-26 18:03:40 +02002078
2079 if (ret) {
Andreas Färbere78815a2010-09-25 11:26:05 +00002080 perror("qemu_madvise");
2081 fprintf(stderr,
2082 "Need MADV_DONTFORK in absence of synchronous KVM MMU\n");
Jan Kiszka6f0437e2009-04-26 18:03:40 +02002083 exit(1);
2084 }
Jan Kiszka6f0437e2009-04-26 18:03:40 +02002085 }
2086}
2087
aliguorie22a25c2009-03-12 20:12:48 +00002088#ifdef KVM_CAP_SET_GUEST_DEBUG
Andreas Färbera60f24b2012-12-01 05:35:08 +01002089struct kvm_sw_breakpoint *kvm_find_sw_breakpoint(CPUState *cpu,
aliguorie22a25c2009-03-12 20:12:48 +00002090 target_ulong pc)
2091{
2092 struct kvm_sw_breakpoint *bp;
2093
Andreas Färbera60f24b2012-12-01 05:35:08 +01002094 QTAILQ_FOREACH(bp, &cpu->kvm_state->kvm_sw_breakpoints, entry) {
Jan Kiszkaa426e122011-01-04 09:32:13 +01002095 if (bp->pc == pc) {
aliguorie22a25c2009-03-12 20:12:48 +00002096 return bp;
Jan Kiszkaa426e122011-01-04 09:32:13 +01002097 }
aliguorie22a25c2009-03-12 20:12:48 +00002098 }
2099 return NULL;
2100}
2101
Andreas Färbera60f24b2012-12-01 05:35:08 +01002102int kvm_sw_breakpoints_active(CPUState *cpu)
aliguorie22a25c2009-03-12 20:12:48 +00002103{
Andreas Färbera60f24b2012-12-01 05:35:08 +01002104 return !QTAILQ_EMPTY(&cpu->kvm_state->kvm_sw_breakpoints);
aliguorie22a25c2009-03-12 20:12:48 +00002105}
2106
Glauber Costa452e4752009-07-16 17:55:28 -04002107struct kvm_set_guest_debug_data {
2108 struct kvm_guest_debug dbg;
Andreas Färbera60f24b2012-12-01 05:35:08 +01002109 CPUState *cpu;
Glauber Costa452e4752009-07-16 17:55:28 -04002110 int err;
2111};
2112
2113static void kvm_invoke_set_guest_debug(void *data)
2114{
2115 struct kvm_set_guest_debug_data *dbg_data = data;
Jan Kiszkab3807722009-09-17 20:05:58 +02002116
Andreas Färbera60f24b2012-12-01 05:35:08 +01002117 dbg_data->err = kvm_vcpu_ioctl(dbg_data->cpu, KVM_SET_GUEST_DEBUG,
2118 &dbg_data->dbg);
Glauber Costa452e4752009-07-16 17:55:28 -04002119}
2120
Stefan Weil38e478e2013-07-25 20:50:21 +02002121int kvm_update_guest_debug(CPUState *cpu, unsigned long reinject_trap)
aliguorie22a25c2009-03-12 20:12:48 +00002122{
Glauber Costa452e4752009-07-16 17:55:28 -04002123 struct kvm_set_guest_debug_data data;
aliguorie22a25c2009-03-12 20:12:48 +00002124
Jan Kiszkab0b1d692010-03-01 19:10:29 +01002125 data.dbg.control = reinject_trap;
aliguorie22a25c2009-03-12 20:12:48 +00002126
Andreas Färbered2803d2013-06-21 20:20:45 +02002127 if (cpu->singlestep_enabled) {
Jan Kiszkab0b1d692010-03-01 19:10:29 +01002128 data.dbg.control |= KVM_GUESTDBG_ENABLE | KVM_GUESTDBG_SINGLESTEP;
2129 }
Andreas Färber20d695a2012-10-31 06:57:49 +01002130 kvm_arch_update_guest_debug(cpu, &data.dbg);
Andreas Färbera60f24b2012-12-01 05:35:08 +01002131 data.cpu = cpu;
aliguorie22a25c2009-03-12 20:12:48 +00002132
Andreas Färberf100f0b2012-05-03 14:58:47 +02002133 run_on_cpu(cpu, kvm_invoke_set_guest_debug, &data);
Glauber Costa452e4752009-07-16 17:55:28 -04002134 return data.err;
aliguorie22a25c2009-03-12 20:12:48 +00002135}
2136
Andreas Färber62278812013-06-27 17:12:06 +02002137int kvm_insert_breakpoint(CPUState *cpu, target_ulong addr,
aliguorie22a25c2009-03-12 20:12:48 +00002138 target_ulong len, int type)
2139{
2140 struct kvm_sw_breakpoint *bp;
aliguorie22a25c2009-03-12 20:12:48 +00002141 int err;
2142
2143 if (type == GDB_BREAKPOINT_SW) {
Andreas Färber80b7cd72013-06-19 17:37:31 +02002144 bp = kvm_find_sw_breakpoint(cpu, addr);
aliguorie22a25c2009-03-12 20:12:48 +00002145 if (bp) {
2146 bp->use_count++;
2147 return 0;
2148 }
2149
Anthony Liguori7267c092011-08-20 22:09:37 -05002150 bp = g_malloc(sizeof(struct kvm_sw_breakpoint));
aliguorie22a25c2009-03-12 20:12:48 +00002151 bp->pc = addr;
2152 bp->use_count = 1;
Andreas Färber80b7cd72013-06-19 17:37:31 +02002153 err = kvm_arch_insert_sw_breakpoint(cpu, bp);
aliguorie22a25c2009-03-12 20:12:48 +00002154 if (err) {
Anthony Liguori7267c092011-08-20 22:09:37 -05002155 g_free(bp);
aliguorie22a25c2009-03-12 20:12:48 +00002156 return err;
2157 }
2158
Andreas Färber80b7cd72013-06-19 17:37:31 +02002159 QTAILQ_INSERT_HEAD(&cpu->kvm_state->kvm_sw_breakpoints, bp, entry);
aliguorie22a25c2009-03-12 20:12:48 +00002160 } else {
2161 err = kvm_arch_insert_hw_breakpoint(addr, len, type);
Jan Kiszkaa426e122011-01-04 09:32:13 +01002162 if (err) {
aliguorie22a25c2009-03-12 20:12:48 +00002163 return err;
Jan Kiszkaa426e122011-01-04 09:32:13 +01002164 }
aliguorie22a25c2009-03-12 20:12:48 +00002165 }
2166
Andreas Färberbdc44642013-06-24 23:50:24 +02002167 CPU_FOREACH(cpu) {
Stefan Weil38e478e2013-07-25 20:50:21 +02002168 err = kvm_update_guest_debug(cpu, 0);
Jan Kiszkaa426e122011-01-04 09:32:13 +01002169 if (err) {
aliguorie22a25c2009-03-12 20:12:48 +00002170 return err;
Jan Kiszkaa426e122011-01-04 09:32:13 +01002171 }
aliguorie22a25c2009-03-12 20:12:48 +00002172 }
2173 return 0;
2174}
2175
Andreas Färber62278812013-06-27 17:12:06 +02002176int kvm_remove_breakpoint(CPUState *cpu, target_ulong addr,
aliguorie22a25c2009-03-12 20:12:48 +00002177 target_ulong len, int type)
2178{
2179 struct kvm_sw_breakpoint *bp;
aliguorie22a25c2009-03-12 20:12:48 +00002180 int err;
2181
2182 if (type == GDB_BREAKPOINT_SW) {
Andreas Färber80b7cd72013-06-19 17:37:31 +02002183 bp = kvm_find_sw_breakpoint(cpu, addr);
Jan Kiszkaa426e122011-01-04 09:32:13 +01002184 if (!bp) {
aliguorie22a25c2009-03-12 20:12:48 +00002185 return -ENOENT;
Jan Kiszkaa426e122011-01-04 09:32:13 +01002186 }
aliguorie22a25c2009-03-12 20:12:48 +00002187
2188 if (bp->use_count > 1) {
2189 bp->use_count--;
2190 return 0;
2191 }
2192
Andreas Färber80b7cd72013-06-19 17:37:31 +02002193 err = kvm_arch_remove_sw_breakpoint(cpu, bp);
Jan Kiszkaa426e122011-01-04 09:32:13 +01002194 if (err) {
aliguorie22a25c2009-03-12 20:12:48 +00002195 return err;
Jan Kiszkaa426e122011-01-04 09:32:13 +01002196 }
aliguorie22a25c2009-03-12 20:12:48 +00002197
Andreas Färber80b7cd72013-06-19 17:37:31 +02002198 QTAILQ_REMOVE(&cpu->kvm_state->kvm_sw_breakpoints, bp, entry);
Anthony Liguori7267c092011-08-20 22:09:37 -05002199 g_free(bp);
aliguorie22a25c2009-03-12 20:12:48 +00002200 } else {
2201 err = kvm_arch_remove_hw_breakpoint(addr, len, type);
Jan Kiszkaa426e122011-01-04 09:32:13 +01002202 if (err) {
aliguorie22a25c2009-03-12 20:12:48 +00002203 return err;
Jan Kiszkaa426e122011-01-04 09:32:13 +01002204 }
aliguorie22a25c2009-03-12 20:12:48 +00002205 }
2206
Andreas Färberbdc44642013-06-24 23:50:24 +02002207 CPU_FOREACH(cpu) {
Stefan Weil38e478e2013-07-25 20:50:21 +02002208 err = kvm_update_guest_debug(cpu, 0);
Jan Kiszkaa426e122011-01-04 09:32:13 +01002209 if (err) {
aliguorie22a25c2009-03-12 20:12:48 +00002210 return err;
Jan Kiszkaa426e122011-01-04 09:32:13 +01002211 }
aliguorie22a25c2009-03-12 20:12:48 +00002212 }
2213 return 0;
2214}
2215
Andreas Färber1d5791f2013-05-27 14:40:48 +02002216void kvm_remove_all_breakpoints(CPUState *cpu)
aliguorie22a25c2009-03-12 20:12:48 +00002217{
2218 struct kvm_sw_breakpoint *bp, *next;
Andreas Färber80b7cd72013-06-19 17:37:31 +02002219 KVMState *s = cpu->kvm_state;
Chen Gangdc54e252014-07-19 09:21:46 +08002220 CPUState *tmpcpu;
aliguorie22a25c2009-03-12 20:12:48 +00002221
Blue Swirl72cf2d42009-09-12 07:36:22 +00002222 QTAILQ_FOREACH_SAFE(bp, &s->kvm_sw_breakpoints, entry, next) {
Andreas Färber80b7cd72013-06-19 17:37:31 +02002223 if (kvm_arch_remove_sw_breakpoint(cpu, bp) != 0) {
aliguorie22a25c2009-03-12 20:12:48 +00002224 /* Try harder to find a CPU that currently sees the breakpoint. */
Chen Gangdc54e252014-07-19 09:21:46 +08002225 CPU_FOREACH(tmpcpu) {
2226 if (kvm_arch_remove_sw_breakpoint(tmpcpu, bp) == 0) {
aliguorie22a25c2009-03-12 20:12:48 +00002227 break;
Jan Kiszkaa426e122011-01-04 09:32:13 +01002228 }
aliguorie22a25c2009-03-12 20:12:48 +00002229 }
2230 }
Jan Kiszka78021d62012-11-12 15:04:35 +01002231 QTAILQ_REMOVE(&s->kvm_sw_breakpoints, bp, entry);
2232 g_free(bp);
aliguorie22a25c2009-03-12 20:12:48 +00002233 }
2234 kvm_arch_remove_all_hw_breakpoints();
2235
Andreas Färberbdc44642013-06-24 23:50:24 +02002236 CPU_FOREACH(cpu) {
Stefan Weil38e478e2013-07-25 20:50:21 +02002237 kvm_update_guest_debug(cpu, 0);
Jan Kiszkaa426e122011-01-04 09:32:13 +01002238 }
aliguorie22a25c2009-03-12 20:12:48 +00002239}
2240
2241#else /* !KVM_CAP_SET_GUEST_DEBUG */
2242
Stefan Weil38e478e2013-07-25 20:50:21 +02002243int kvm_update_guest_debug(CPUState *cpu, unsigned long reinject_trap)
aliguorie22a25c2009-03-12 20:12:48 +00002244{
2245 return -EINVAL;
2246}
2247
Andreas Färber62278812013-06-27 17:12:06 +02002248int kvm_insert_breakpoint(CPUState *cpu, target_ulong addr,
aliguorie22a25c2009-03-12 20:12:48 +00002249 target_ulong len, int type)
2250{
2251 return -EINVAL;
2252}
2253
Andreas Färber62278812013-06-27 17:12:06 +02002254int kvm_remove_breakpoint(CPUState *cpu, target_ulong addr,
aliguorie22a25c2009-03-12 20:12:48 +00002255 target_ulong len, int type)
2256{
2257 return -EINVAL;
2258}
2259
Andreas Färber1d5791f2013-05-27 14:40:48 +02002260void kvm_remove_all_breakpoints(CPUState *cpu)
aliguorie22a25c2009-03-12 20:12:48 +00002261{
2262}
2263#endif /* !KVM_CAP_SET_GUEST_DEBUG */
Marcelo Tosatticc84de92010-02-17 20:14:42 -02002264
Andreas Färber491d6e82013-05-26 23:38:10 +02002265int kvm_set_signal_mask(CPUState *cpu, const sigset_t *sigset)
Marcelo Tosatticc84de92010-02-17 20:14:42 -02002266{
James Hoganaed6efb2014-06-17 23:10:31 +01002267 KVMState *s = kvm_state;
Marcelo Tosatticc84de92010-02-17 20:14:42 -02002268 struct kvm_signal_mask *sigmask;
2269 int r;
2270
Jan Kiszkaa426e122011-01-04 09:32:13 +01002271 if (!sigset) {
Andreas Färber1bc22652012-10-31 06:06:49 +01002272 return kvm_vcpu_ioctl(cpu, KVM_SET_SIGNAL_MASK, NULL);
Jan Kiszkaa426e122011-01-04 09:32:13 +01002273 }
Marcelo Tosatticc84de92010-02-17 20:14:42 -02002274
Anthony Liguori7267c092011-08-20 22:09:37 -05002275 sigmask = g_malloc(sizeof(*sigmask) + sizeof(*sigset));
Marcelo Tosatticc84de92010-02-17 20:14:42 -02002276
James Hoganaed6efb2014-06-17 23:10:31 +01002277 sigmask->len = s->sigmask_len;
Marcelo Tosatticc84de92010-02-17 20:14:42 -02002278 memcpy(sigmask->sigset, sigset, sizeof(*sigset));
Andreas Färber1bc22652012-10-31 06:06:49 +01002279 r = kvm_vcpu_ioctl(cpu, KVM_SET_SIGNAL_MASK, sigmask);
Anthony Liguori7267c092011-08-20 22:09:37 -05002280 g_free(sigmask);
Marcelo Tosatticc84de92010-02-17 20:14:42 -02002281
2282 return r;
2283}
Andreas Färber290adf32013-01-17 09:30:27 +01002284int kvm_on_sigbus_vcpu(CPUState *cpu, int code, void *addr)
Jan Kiszkaa1b87fe2011-02-01 22:15:51 +01002285{
Andreas Färber20d695a2012-10-31 06:57:49 +01002286 return kvm_arch_on_sigbus_vcpu(cpu, code, addr);
Jan Kiszkaa1b87fe2011-02-01 22:15:51 +01002287}
2288
2289int kvm_on_sigbus(int code, void *addr)
2290{
2291 return kvm_arch_on_sigbus(code, addr);
2292}
Christoffer Dall0a6a7cc2014-02-26 17:20:00 +00002293
2294int kvm_create_device(KVMState *s, uint64_t type, bool test)
2295{
2296 int ret;
2297 struct kvm_create_device create_dev;
2298
2299 create_dev.type = type;
2300 create_dev.fd = -1;
2301 create_dev.flags = test ? KVM_CREATE_DEVICE_TEST : 0;
2302
2303 if (!kvm_check_extension(s, KVM_CAP_DEVICE_CTRL)) {
2304 return -ENOTSUP;
2305 }
2306
2307 ret = kvm_vm_ioctl(s, KVM_CREATE_DEVICE, &create_dev);
2308 if (ret) {
2309 return ret;
2310 }
2311
2312 return test ? 0 : create_dev.fd;
2313}
Cornelia Huckada41352014-05-09 10:06:46 +02002314
2315int kvm_set_one_reg(CPUState *cs, uint64_t id, void *source)
2316{
2317 struct kvm_one_reg reg;
2318 int r;
2319
2320 reg.id = id;
2321 reg.addr = (uintptr_t) source;
2322 r = kvm_vcpu_ioctl(cs, KVM_SET_ONE_REG, &reg);
2323 if (r) {
2324 trace_kvm_failed_reg_set(id, strerror(r));
2325 }
2326 return r;
2327}
2328
2329int kvm_get_one_reg(CPUState *cs, uint64_t id, void *target)
2330{
2331 struct kvm_one_reg reg;
2332 int r;
2333
2334 reg.id = id;
2335 reg.addr = (uintptr_t) target;
2336 r = kvm_vcpu_ioctl(cs, KVM_GET_ONE_REG, &reg);
2337 if (r) {
2338 trace_kvm_failed_reg_get(id, strerror(r));
2339 }
2340 return r;
2341}
Eduardo Habkost782c3f22014-09-26 17:45:24 -03002342
2343static void kvm_accel_class_init(ObjectClass *oc, void *data)
2344{
2345 AccelClass *ac = ACCEL_CLASS(oc);
2346 ac->name = "KVM";
Eduardo Habkost0d15da82014-09-26 17:45:29 -03002347 ac->init_machine = kvm_init;
Eduardo Habkost782c3f22014-09-26 17:45:24 -03002348 ac->allowed = &kvm_allowed;
2349}
2350
2351static const TypeInfo kvm_accel_type = {
2352 .name = TYPE_KVM_ACCEL,
2353 .parent = TYPE_ACCEL,
2354 .class_init = kvm_accel_class_init,
Eduardo Habkostfc020862014-09-26 17:45:32 -03002355 .instance_size = sizeof(KVMState),
Eduardo Habkost782c3f22014-09-26 17:45:24 -03002356};
2357
2358static void kvm_type_init(void)
2359{
2360 type_register_static(&kvm_accel_type);
2361}
2362
2363type_init(kvm_type_init);