Merge remote-tracking branch 'remotes/stefanha/tags/tracing-pull-request' into staging
Pull request
* Python 3 support in simpletrace.py
* Convert DPRINTF() to trace events
# gpg: Signature made Fri 29 Jun 2018 18:53:05 BST
# gpg: using RSA key 9CA4ABB381AB73C8
# gpg: Good signature from "Stefan Hajnoczi <stefanha@redhat.com>"
# gpg: aka "Stefan Hajnoczi <stefanha@gmail.com>"
# Primary key fingerprint: 8695 A8BF D3F9 7CDA AC35 775A 9CA4 ABB3 81AB 73C8
* remotes/stefanha/tags/tracing-pull-request:
hw/block/pflash_cfi: Convert from DPRINTF() macro to trace events
hw/block/fdc: Convert from FLOPPY_DPRINTF() macro to trace events
hw/net/etraxfs_eth: Convert printf() calls to trace events
hw/net/ne2000: Convert printf() calls to trace events
hw/net/ne2000: Add trace events
hw/input/tsc2005: Convert a fprintf() call to trace events
hw/char/parallel: Convert from pdebug() macro to trace events
hw/char/serial: Convert from DPRINTF macro to trace events
sdcard: Reduce sdcard_set_blocklen() trace digits
trace: Fix format string for the struct timeval members casted to size_t
simpletrace: Convert name from mapping record to str
Signed-off-by: Peter Maydell <peter.maydell@linaro.org>
diff --git a/Makefile.target b/Makefile.target
index a9d8928..4d56298 100644
--- a/Makefile.target
+++ b/Makefile.target
@@ -143,6 +143,7 @@
obj-y += memory.o
obj-y += memory_mapping.o
obj-y += dump.o
+obj-$(TARGET_X86_64) += win_dump.o
obj-y += migration/ram.o
LIBS := $(libs_softmmu) $(LIBS)
diff --git a/accel/kvm/kvm-all.c b/accel/kvm/kvm-all.c
index ffee68e..eb7db92 100644
--- a/accel/kvm/kvm-all.c
+++ b/accel/kvm/kvm-all.c
@@ -256,7 +256,7 @@
return 0;
}
-static int kvm_set_user_memory_region(KVMMemoryListener *kml, KVMSlot *slot)
+static int kvm_set_user_memory_region(KVMMemoryListener *kml, KVMSlot *slot, bool new)
{
KVMState *s = kvm_state;
struct kvm_userspace_memory_region mem;
@@ -267,7 +267,7 @@
mem.userspace_addr = (unsigned long)slot->ram;
mem.flags = slot->flags;
- if (slot->memory_size && mem.flags & KVM_MEM_READONLY) {
+ if (slot->memory_size && !new && (mem.flags ^ slot->old_flags) & KVM_MEM_READONLY) {
/* Set the slot size to 0 before setting the slot to the desired
* value. This is needed based on KVM commit 75d61fbc. */
mem.memory_size = 0;
@@ -275,6 +275,7 @@
}
mem.memory_size = slot->memory_size;
ret = kvm_vm_ioctl(s, KVM_SET_USER_MEMORY_REGION, &mem);
+ slot->old_flags = mem.flags;
trace_kvm_set_user_memory(mem.slot, mem.flags, mem.guest_phys_addr,
mem.memory_size, mem.userspace_addr, ret);
return ret;
@@ -391,17 +392,14 @@
static int kvm_slot_update_flags(KVMMemoryListener *kml, KVMSlot *mem,
MemoryRegion *mr)
{
- int old_flags;
-
- old_flags = mem->flags;
mem->flags = kvm_mem_flags(mr);
/* If nothing changed effectively, no need to issue ioctl */
- if (mem->flags == old_flags) {
+ if (mem->flags == mem->old_flags) {
return 0;
}
- return kvm_set_user_memory_region(kml, mem);
+ return kvm_set_user_memory_region(kml, mem, false);
}
static int kvm_section_update_flags(KVMMemoryListener *kml,
@@ -755,7 +753,8 @@
/* unregister the slot */
mem->memory_size = 0;
- err = kvm_set_user_memory_region(kml, mem);
+ mem->flags = 0;
+ err = kvm_set_user_memory_region(kml, mem, false);
if (err) {
fprintf(stderr, "%s: error unregistering slot: %s\n",
__func__, strerror(-err));
@@ -771,7 +770,7 @@
mem->ram = ram;
mem->flags = kvm_mem_flags(mr);
- err = kvm_set_user_memory_region(kml, mem);
+ err = kvm_set_user_memory_region(kml, mem, true);
if (err) {
fprintf(stderr, "%s: error registering slot: %s\n", __func__,
strerror(-err));
diff --git a/accel/tcg/translate-all.c b/accel/tcg/translate-all.c
index ad997cb..e8228bf 100644
--- a/accel/tcg/translate-all.c
+++ b/accel/tcg/translate-all.c
@@ -46,7 +46,7 @@
#endif
#endif
#else
-#include "exec/address-spaces.h"
+#include "exec/ram_addr.h"
#endif
#include "exec/cputlb.h"
@@ -1934,7 +1934,11 @@
*
* Called with mmap_lock held for user-mode emulation.
*/
-void tb_invalidate_phys_range(tb_page_addr_t start, tb_page_addr_t end)
+#ifdef CONFIG_SOFTMMU
+void tb_invalidate_phys_range(ram_addr_t start, ram_addr_t end)
+#else
+void tb_invalidate_phys_range(target_ulong start, target_ulong end)
+#endif
{
struct page_collection *pages;
tb_page_addr_t next;
@@ -2073,26 +2077,6 @@
}
#endif
-#if !defined(CONFIG_USER_ONLY)
-void tb_invalidate_phys_addr(AddressSpace *as, hwaddr addr, MemTxAttrs attrs)
-{
- ram_addr_t ram_addr;
- MemoryRegion *mr;
- hwaddr l = 1;
-
- rcu_read_lock();
- mr = address_space_translate(as, addr, &addr, &l, false, attrs);
- if (!(memory_region_is_ram(mr)
- || memory_region_is_romd(mr))) {
- rcu_read_unlock();
- return;
- }
- ram_addr = memory_region_get_ram_addr(mr) + addr;
- tb_invalidate_phys_page_range(ram_addr, ram_addr + 1, 0);
- rcu_read_unlock();
-}
-#endif /* !defined(CONFIG_USER_ONLY) */
-
/* user-mode: call with mmap_lock held */
void tb_check_watchpoint(CPUState *cpu)
{
diff --git a/accel/tcg/translate-all.h b/accel/tcg/translate-all.h
index e6cb963..08e2f23 100644
--- a/accel/tcg/translate-all.h
+++ b/accel/tcg/translate-all.h
@@ -30,7 +30,6 @@
tb_page_addr_t start, int len);
void tb_invalidate_phys_page_range(tb_page_addr_t start, tb_page_addr_t end,
int is_cpu_write_access);
-void tb_invalidate_phys_range(tb_page_addr_t start, tb_page_addr_t end);
void tb_check_watchpoint(CPUState *cpu);
#ifdef CONFIG_USER_ONLY
diff --git a/accel/tcg/user-exec-stub.c b/accel/tcg/user-exec-stub.c
index dbcf1ad..a32b449 100644
--- a/accel/tcg/user-exec-stub.c
+++ b/accel/tcg/user-exec-stub.c
@@ -2,6 +2,9 @@
#include "qemu-common.h"
#include "qom/cpu.h"
#include "sysemu/replay.h"
+#include "sysemu/sysemu.h"
+
+bool enable_cpu_pm = false;
void cpu_resume(CPUState *cpu)
{
diff --git a/backends/hostmem.c b/backends/hostmem.c
index 3627e61..4908946 100644
--- a/backends/hostmem.c
+++ b/backends/hostmem.c
@@ -247,8 +247,7 @@
return memory_region_size(&backend->mr) != 0;
}
-MemoryRegion *
-host_memory_backend_get_memory(HostMemoryBackend *backend, Error **errp)
+MemoryRegion *host_memory_backend_get_memory(HostMemoryBackend *backend)
{
return host_memory_backend_mr_inited(backend) ? &backend->mr : NULL;
}
diff --git a/block.c b/block.c
index 1b8147c..70a46fd 100644
--- a/block.c
+++ b/block.c
@@ -725,7 +725,7 @@
* Set the current 'total_sectors' value
* Return 0 on success, -errno on error.
*/
-static int refresh_total_sectors(BlockDriverState *bs, int64_t hint)
+int refresh_total_sectors(BlockDriverState *bs, int64_t hint)
{
BlockDriver *drv = bs->drv;
@@ -2226,16 +2226,6 @@
}
}
-static void bdrv_parent_cb_resize(BlockDriverState *bs)
-{
- BdrvChild *c;
- QLIST_FOREACH(c, &bs->parents, next_parent) {
- if (c->role->resize) {
- c->role->resize(c);
- }
- }
-}
-
/*
* Sets the backing file link of a BDS. A new reference is created; callers
* which don't need their own reference any more must call bdrv_unref().
@@ -3786,58 +3776,6 @@
}
/**
- * Truncate file to 'offset' bytes (needed only for file protocols)
- */
-int bdrv_truncate(BdrvChild *child, int64_t offset, PreallocMode prealloc,
- Error **errp)
-{
- BlockDriverState *bs = child->bs;
- BlockDriver *drv = bs->drv;
- int ret;
-
- assert(child->perm & BLK_PERM_RESIZE);
-
- /* if bs->drv == NULL, bs is closed, so there's nothing to do here */
- if (!drv) {
- error_setg(errp, "No medium inserted");
- return -ENOMEDIUM;
- }
- if (offset < 0) {
- error_setg(errp, "Image size cannot be negative");
- return -EINVAL;
- }
-
- if (!drv->bdrv_truncate) {
- if (bs->file && drv->is_filter) {
- return bdrv_truncate(bs->file, offset, prealloc, errp);
- }
- error_setg(errp, "Image format driver does not support resize");
- return -ENOTSUP;
- }
- if (bs->read_only) {
- error_setg(errp, "Image is read-only");
- return -EACCES;
- }
-
- assert(!(bs->open_flags & BDRV_O_INACTIVE));
-
- ret = drv->bdrv_truncate(bs, offset, prealloc, errp);
- if (ret < 0) {
- return ret;
- }
- ret = refresh_total_sectors(bs, offset >> BDRV_SECTOR_BITS);
- if (ret < 0) {
- error_setg_errno(errp, -ret, "Could not refresh total sector count");
- } else {
- offset = bs->total_sectors * BDRV_SECTOR_SIZE;
- }
- bdrv_dirty_bitmap_truncate(bs, offset);
- bdrv_parent_cb_resize(bs);
- atomic_inc(&bs->write_gen);
- return ret;
-}
-
-/**
* Length of a allocated file in bytes. Sparse files are counted by actual
* allocated space. Return < 0 if error or unknown.
*/
diff --git a/block/copy-on-read.c b/block/copy-on-read.c
index 6a97208..1dcdaee 100644
--- a/block/copy-on-read.c
+++ b/block/copy-on-read.c
@@ -80,10 +80,10 @@
}
-static int cor_truncate(BlockDriverState *bs, int64_t offset,
- PreallocMode prealloc, Error **errp)
+static int coroutine_fn cor_co_truncate(BlockDriverState *bs, int64_t offset,
+ PreallocMode prealloc, Error **errp)
{
- return bdrv_truncate(bs->file, offset, prealloc, errp);
+ return bdrv_co_truncate(bs->file, offset, prealloc, errp);
}
@@ -147,7 +147,7 @@
.bdrv_child_perm = cor_child_perm,
.bdrv_getlength = cor_getlength,
- .bdrv_truncate = cor_truncate,
+ .bdrv_co_truncate = cor_co_truncate,
.bdrv_co_preadv = cor_co_preadv,
.bdrv_co_pwritev = cor_co_pwritev,
diff --git a/block/crypto.c b/block/crypto.c
index 82091c5..994172a 100644
--- a/block/crypto.c
+++ b/block/crypto.c
@@ -148,108 +148,36 @@
QCryptoBlockOpenOptions *
-block_crypto_open_opts_init(QCryptoBlockFormat format,
- QDict *opts,
- Error **errp)
+block_crypto_open_opts_init(QDict *opts, Error **errp)
{
Visitor *v;
- QCryptoBlockOpenOptions *ret = NULL;
- Error *local_err = NULL;
+ QCryptoBlockOpenOptions *ret;
- ret = g_new0(QCryptoBlockOpenOptions, 1);
- ret->format = format;
-
- v = qobject_input_visitor_new_flat_confused(opts, &local_err);
- if (local_err) {
- goto out;
+ v = qobject_input_visitor_new_flat_confused(opts, errp);
+ if (!v) {
+ return NULL;
}
- visit_start_struct(v, NULL, NULL, 0, &local_err);
- if (local_err) {
- goto out;
- }
+ visit_type_QCryptoBlockOpenOptions(v, NULL, &ret, errp);
- switch (format) {
- case Q_CRYPTO_BLOCK_FORMAT_LUKS:
- visit_type_QCryptoBlockOptionsLUKS_members(
- v, &ret->u.luks, &local_err);
- break;
-
- case Q_CRYPTO_BLOCK_FORMAT_QCOW:
- visit_type_QCryptoBlockOptionsQCow_members(
- v, &ret->u.qcow, &local_err);
- break;
-
- default:
- error_setg(&local_err, "Unsupported block format %d", format);
- break;
- }
- if (!local_err) {
- visit_check_struct(v, &local_err);
- }
-
- visit_end_struct(v, NULL);
-
- out:
- if (local_err) {
- error_propagate(errp, local_err);
- qapi_free_QCryptoBlockOpenOptions(ret);
- ret = NULL;
- }
visit_free(v);
return ret;
}
QCryptoBlockCreateOptions *
-block_crypto_create_opts_init(QCryptoBlockFormat format,
- QDict *opts,
- Error **errp)
+block_crypto_create_opts_init(QDict *opts, Error **errp)
{
Visitor *v;
- QCryptoBlockCreateOptions *ret = NULL;
- Error *local_err = NULL;
+ QCryptoBlockCreateOptions *ret;
- ret = g_new0(QCryptoBlockCreateOptions, 1);
- ret->format = format;
-
- v = qobject_input_visitor_new_flat_confused(opts, &local_err);
- if (local_err) {
- goto out;
+ v = qobject_input_visitor_new_flat_confused(opts, errp);
+ if (!v) {
+ return NULL;
}
- visit_start_struct(v, NULL, NULL, 0, &local_err);
- if (local_err) {
- goto out;
- }
+ visit_type_QCryptoBlockCreateOptions(v, NULL, &ret, errp);
- switch (format) {
- case Q_CRYPTO_BLOCK_FORMAT_LUKS:
- visit_type_QCryptoBlockCreateOptionsLUKS_members(
- v, &ret->u.luks, &local_err);
- break;
-
- case Q_CRYPTO_BLOCK_FORMAT_QCOW:
- visit_type_QCryptoBlockOptionsQCow_members(
- v, &ret->u.qcow, &local_err);
- break;
-
- default:
- error_setg(&local_err, "Unsupported block format %d", format);
- break;
- }
- if (!local_err) {
- visit_check_struct(v, &local_err);
- }
-
- visit_end_struct(v, NULL);
-
- out:
- if (local_err) {
- error_propagate(errp, local_err);
- qapi_free_QCryptoBlockCreateOptions(ret);
- ret = NULL;
- }
visit_free(v);
return ret;
}
@@ -287,8 +215,9 @@
}
cryptoopts = qemu_opts_to_qdict(opts, NULL);
+ qdict_put_str(cryptoopts, "format", QCryptoBlockFormat_str(format));
- open_opts = block_crypto_open_opts_init(format, cryptoopts, errp);
+ open_opts = block_crypto_open_opts_init(cryptoopts, errp);
if (!open_opts) {
goto cleanup;
}
@@ -357,8 +286,9 @@
return ret;
}
-static int block_crypto_truncate(BlockDriverState *bs, int64_t offset,
- PreallocMode prealloc, Error **errp)
+static int coroutine_fn
+block_crypto_co_truncate(BlockDriverState *bs, int64_t offset,
+ PreallocMode prealloc, Error **errp)
{
BlockCrypto *crypto = bs->opaque;
uint64_t payload_offset =
@@ -371,7 +301,7 @@
offset += payload_offset;
- return bdrv_truncate(bs->file, offset, prealloc, errp);
+ return bdrv_co_truncate(bs->file, offset, prealloc, errp);
}
static void block_crypto_close(BlockDriverState *bs)
@@ -611,8 +541,8 @@
&block_crypto_create_opts_luks,
true);
- create_opts = block_crypto_create_opts_init(Q_CRYPTO_BLOCK_FORMAT_LUKS,
- cryptoopts, errp);
+ qdict_put_str(cryptoopts, "format", "luks");
+ create_opts = block_crypto_create_opts_init(cryptoopts, errp);
if (!create_opts) {
ret = -EINVAL;
goto fail;
@@ -700,7 +630,7 @@
.bdrv_child_perm = bdrv_format_default_perms,
.bdrv_co_create = block_crypto_co_create_luks,
.bdrv_co_create_opts = block_crypto_co_create_opts_luks,
- .bdrv_truncate = block_crypto_truncate,
+ .bdrv_co_truncate = block_crypto_co_truncate,
.create_opts = &block_crypto_create_opts_luks,
.bdrv_reopen_prepare = block_crypto_reopen_prepare,
diff --git a/block/crypto.h b/block/crypto.h
index 0f985ea..dd7d479 100644
--- a/block/crypto.h
+++ b/block/crypto.h
@@ -89,13 +89,9 @@
}
QCryptoBlockCreateOptions *
-block_crypto_create_opts_init(QCryptoBlockFormat format,
- QDict *opts,
- Error **errp);
+block_crypto_create_opts_init(QDict *opts, Error **errp);
QCryptoBlockOpenOptions *
-block_crypto_open_opts_init(QCryptoBlockFormat format,
- QDict *opts,
- Error **errp);
+block_crypto_open_opts_init(QDict *opts, Error **errp);
#endif /* BLOCK_CRYPTO_H__ */
diff --git a/block/file-posix.c b/block/file-posix.c
index 43b963b..829ee53 100644
--- a/block/file-posix.c
+++ b/block/file-posix.c
@@ -188,8 +188,16 @@
#define aio_ioctl_cmd aio_nbytes /* for QEMU_AIO_IOCTL */
off_t aio_offset;
int aio_type;
- int aio_fd2;
- off_t aio_offset2;
+ union {
+ struct {
+ int aio_fd2;
+ off_t aio_offset2;
+ };
+ struct {
+ PreallocMode prealloc;
+ Error **errp;
+ };
+ };
} RawPosixAIOData;
#if defined(__FreeBSD__) || defined(__FreeBSD_kernel__)
@@ -1480,20 +1488,21 @@
ssize_t ret = copy_file_range(aiocb->aio_fildes, &in_off,
aiocb->aio_fd2, &out_off,
bytes, 0);
- if (ret == -EINTR) {
- continue;
+ if (ret == 0) {
+ /* No progress (e.g. when beyond EOF), let the caller fall back to
+ * buffer I/O. */
+ return -ENOSPC;
}
if (ret < 0) {
- if (errno == ENOSYS) {
+ switch (errno) {
+ case ENOSYS:
return -ENOTSUP;
- } else {
+ case EINTR:
+ continue;
+ default:
return -errno;
}
}
- if (!ret) {
- /* No progress (e.g. when beyond EOF), fall back to buffer I/O. */
- return -ENOTSUP;
- }
bytes -= ret;
}
return 0;
@@ -1539,6 +1548,122 @@
return ret;
}
+static int handle_aiocb_truncate(RawPosixAIOData *aiocb)
+{
+ int result = 0;
+ int64_t current_length = 0;
+ char *buf = NULL;
+ struct stat st;
+ int fd = aiocb->aio_fildes;
+ int64_t offset = aiocb->aio_offset;
+ Error **errp = aiocb->errp;
+
+ if (fstat(fd, &st) < 0) {
+ result = -errno;
+ error_setg_errno(errp, -result, "Could not stat file");
+ return result;
+ }
+
+ current_length = st.st_size;
+ if (current_length > offset && aiocb->prealloc != PREALLOC_MODE_OFF) {
+ error_setg(errp, "Cannot use preallocation for shrinking files");
+ return -ENOTSUP;
+ }
+
+ switch (aiocb->prealloc) {
+#ifdef CONFIG_POSIX_FALLOCATE
+ case PREALLOC_MODE_FALLOC:
+ /*
+ * Truncating before posix_fallocate() makes it about twice slower on
+ * file systems that do not support fallocate(), trying to check if a
+ * block is allocated before allocating it, so don't do that here.
+ */
+ if (offset != current_length) {
+ result = -posix_fallocate(fd, current_length,
+ offset - current_length);
+ if (result != 0) {
+ /* posix_fallocate() doesn't set errno. */
+ error_setg_errno(errp, -result,
+ "Could not preallocate new data");
+ }
+ } else {
+ result = 0;
+ }
+ goto out;
+#endif
+ case PREALLOC_MODE_FULL:
+ {
+ int64_t num = 0, left = offset - current_length;
+ off_t seek_result;
+
+ /*
+ * Knowing the final size from the beginning could allow the file
+ * system driver to do less allocations and possibly avoid
+ * fragmentation of the file.
+ */
+ if (ftruncate(fd, offset) != 0) {
+ result = -errno;
+ error_setg_errno(errp, -result, "Could not resize file");
+ goto out;
+ }
+
+ buf = g_malloc0(65536);
+
+ seek_result = lseek(fd, current_length, SEEK_SET);
+ if (seek_result < 0) {
+ result = -errno;
+ error_setg_errno(errp, -result,
+ "Failed to seek to the old end of file");
+ goto out;
+ }
+
+ while (left > 0) {
+ num = MIN(left, 65536);
+ result = write(fd, buf, num);
+ if (result < 0) {
+ result = -errno;
+ error_setg_errno(errp, -result,
+ "Could not write zeros for preallocation");
+ goto out;
+ }
+ left -= result;
+ }
+ if (result >= 0) {
+ result = fsync(fd);
+ if (result < 0) {
+ result = -errno;
+ error_setg_errno(errp, -result,
+ "Could not flush file to disk");
+ goto out;
+ }
+ }
+ goto out;
+ }
+ case PREALLOC_MODE_OFF:
+ if (ftruncate(fd, offset) != 0) {
+ result = -errno;
+ error_setg_errno(errp, -result, "Could not resize file");
+ }
+ return result;
+ default:
+ result = -ENOTSUP;
+ error_setg(errp, "Unsupported preallocation mode: %s",
+ PreallocMode_str(aiocb->prealloc));
+ return result;
+ }
+
+out:
+ if (result < 0) {
+ if (ftruncate(fd, current_length) < 0) {
+ error_report("Failed to restore old file length: %s",
+ strerror(errno));
+ }
+ }
+
+ g_free(buf);
+ return result;
+}
+
static int aio_worker(void *arg)
{
RawPosixAIOData *aiocb = arg;
@@ -1582,6 +1707,9 @@
case QEMU_AIO_COPY_RANGE:
ret = handle_aiocb_copy_range(aiocb);
break;
+ case QEMU_AIO_TRUNCATE:
+ ret = handle_aiocb_truncate(aiocb);
+ break;
default:
fprintf(stderr, "invalid aio request (0x%x)\n", aiocb->aio_type);
ret = -EINVAL;
@@ -1627,31 +1755,6 @@
return paio_submit_co_full(bs, fd, offset, -1, 0, qiov, bytes, type);
}
-static BlockAIOCB *paio_submit(BlockDriverState *bs, int fd,
- int64_t offset, QEMUIOVector *qiov, int bytes,
- BlockCompletionFunc *cb, void *opaque, int type)
-{
- RawPosixAIOData *acb = g_new(RawPosixAIOData, 1);
- ThreadPool *pool;
-
- acb->bs = bs;
- acb->aio_type = type;
- acb->aio_fildes = fd;
-
- acb->aio_nbytes = bytes;
- acb->aio_offset = offset;
-
- if (qiov) {
- acb->aio_iov = qiov->iov;
- acb->aio_niov = qiov->niov;
- assert(qiov->size == acb->aio_nbytes);
- }
-
- trace_paio_submit(acb, opaque, offset, bytes, type);
- pool = aio_get_thread_pool(bdrv_get_aio_context(bs));
- return thread_pool_submit_aio(pool, aio_worker, acb, cb, opaque);
-}
-
static int coroutine_fn raw_co_prw(BlockDriverState *bs, uint64_t offset,
uint64_t bytes, QEMUIOVector *qiov, int type)
{
@@ -1718,15 +1821,17 @@
#endif
}
-static BlockAIOCB *raw_aio_flush(BlockDriverState *bs,
- BlockCompletionFunc *cb, void *opaque)
+static int raw_co_flush_to_disk(BlockDriverState *bs)
{
BDRVRawState *s = bs->opaque;
+ int ret;
- if (fd_open(bs) < 0)
- return NULL;
+ ret = fd_open(bs);
+ if (ret < 0) {
+ return ret;
+ }
- return paio_submit(bs, s->fd, 0, NULL, 0, cb, opaque, QEMU_AIO_FLUSH);
+ return paio_submit_co(bs, s->fd, 0, NULL, 0, QEMU_AIO_FLUSH);
}
static void raw_aio_attach_aio_context(BlockDriverState *bs,
@@ -1765,121 +1870,29 @@
*
* Returns: 0 on success, -errno on failure.
*/
-static int raw_regular_truncate(int fd, int64_t offset, PreallocMode prealloc,
- Error **errp)
+static int coroutine_fn
+raw_regular_truncate(BlockDriverState *bs, int fd, int64_t offset,
+ PreallocMode prealloc, Error **errp)
{
- int result = 0;
- int64_t current_length = 0;
- char *buf = NULL;
- struct stat st;
+ RawPosixAIOData *acb = g_new(RawPosixAIOData, 1);
+ ThreadPool *pool;
- if (fstat(fd, &st) < 0) {
- result = -errno;
- error_setg_errno(errp, -result, "Could not stat file");
- return result;
- }
+ *acb = (RawPosixAIOData) {
+ .bs = bs,
+ .aio_fildes = fd,
+ .aio_type = QEMU_AIO_TRUNCATE,
+ .aio_offset = offset,
+ .prealloc = prealloc,
+ .errp = errp,
+ };
- current_length = st.st_size;
- if (current_length > offset && prealloc != PREALLOC_MODE_OFF) {
- error_setg(errp, "Cannot use preallocation for shrinking files");
- return -ENOTSUP;
- }
-
- switch (prealloc) {
-#ifdef CONFIG_POSIX_FALLOCATE
- case PREALLOC_MODE_FALLOC:
- /*
- * Truncating before posix_fallocate() makes it about twice slower on
- * file systems that do not support fallocate(), trying to check if a
- * block is allocated before allocating it, so don't do that here.
- */
- if (offset != current_length) {
- result = -posix_fallocate(fd, current_length, offset - current_length);
- if (result != 0) {
- /* posix_fallocate() doesn't set errno. */
- error_setg_errno(errp, -result,
- "Could not preallocate new data");
- }
- } else {
- result = 0;
- }
- goto out;
-#endif
- case PREALLOC_MODE_FULL:
- {
- int64_t num = 0, left = offset - current_length;
- off_t seek_result;
-
- /*
- * Knowing the final size from the beginning could allow the file
- * system driver to do less allocations and possibly avoid
- * fragmentation of the file.
- */
- if (ftruncate(fd, offset) != 0) {
- result = -errno;
- error_setg_errno(errp, -result, "Could not resize file");
- goto out;
- }
-
- buf = g_malloc0(65536);
-
- seek_result = lseek(fd, current_length, SEEK_SET);
- if (seek_result < 0) {
- result = -errno;
- error_setg_errno(errp, -result,
- "Failed to seek to the old end of file");
- goto out;
- }
-
- while (left > 0) {
- num = MIN(left, 65536);
- result = write(fd, buf, num);
- if (result < 0) {
- result = -errno;
- error_setg_errno(errp, -result,
- "Could not write zeros for preallocation");
- goto out;
- }
- left -= result;
- }
- if (result >= 0) {
- result = fsync(fd);
- if (result < 0) {
- result = -errno;
- error_setg_errno(errp, -result,
- "Could not flush file to disk");
- goto out;
- }
- }
- goto out;
- }
- case PREALLOC_MODE_OFF:
- if (ftruncate(fd, offset) != 0) {
- result = -errno;
- error_setg_errno(errp, -result, "Could not resize file");
- }
- return result;
- default:
- result = -ENOTSUP;
- error_setg(errp, "Unsupported preallocation mode: %s",
- PreallocMode_str(prealloc));
- return result;
- }
-
-out:
- if (result < 0) {
- if (ftruncate(fd, current_length) < 0) {
- error_report("Failed to restore old file length: %s",
- strerror(errno));
- }
- }
-
- g_free(buf);
- return result;
+ /* @bs can be NULL, bdrv_get_aio_context() returns the main context then */
+ pool = aio_get_thread_pool(bdrv_get_aio_context(bs));
+ return thread_pool_submit_co(pool, aio_worker, acb);
}
-static int raw_truncate(BlockDriverState *bs, int64_t offset,
- PreallocMode prealloc, Error **errp)
+static int coroutine_fn raw_co_truncate(BlockDriverState *bs, int64_t offset,
+ PreallocMode prealloc, Error **errp)
{
BDRVRawState *s = bs->opaque;
struct stat st;
@@ -1892,7 +1905,7 @@
}
if (S_ISREG(st.st_mode)) {
- return raw_regular_truncate(s->fd, offset, prealloc, errp);
+ return raw_regular_truncate(bs, s->fd, offset, prealloc, errp);
}
if (prealloc != PREALLOC_MODE_OFF) {
@@ -2094,7 +2107,8 @@
return (int64_t)st.st_blocks * 512;
}
-static int raw_co_create(BlockdevCreateOptions *options, Error **errp)
+static int coroutine_fn
+raw_co_create(BlockdevCreateOptions *options, Error **errp)
{
BlockdevCreateOptionsFile *file_opts;
int fd;
@@ -2146,7 +2160,7 @@
}
/* Clear the file by truncating it to 0 */
- result = raw_regular_truncate(fd, 0, PREALLOC_MODE_OFF, errp);
+ result = raw_regular_truncate(NULL, fd, 0, PREALLOC_MODE_OFF, errp);
if (result < 0) {
goto out_close;
}
@@ -2168,8 +2182,8 @@
/* Resize and potentially preallocate the file to the desired
* final size */
- result = raw_regular_truncate(fd, file_opts->size, file_opts->preallocation,
- errp);
+ result = raw_regular_truncate(NULL, fd, file_opts->size,
+ file_opts->preallocation, errp);
if (result < 0) {
goto out_close;
}
@@ -2490,14 +2504,12 @@
#endif /* !__linux__ */
}
-static coroutine_fn BlockAIOCB *raw_aio_pdiscard(BlockDriverState *bs,
- int64_t offset, int bytes,
- BlockCompletionFunc *cb, void *opaque)
+static coroutine_fn int
+raw_co_pdiscard(BlockDriverState *bs, int64_t offset, int bytes)
{
BDRVRawState *s = bs->opaque;
- return paio_submit(bs, s->fd, offset, NULL, bytes,
- cb, opaque, QEMU_AIO_DISCARD);
+ return paio_submit_co(bs, s->fd, offset, NULL, bytes, QEMU_AIO_DISCARD);
}
static int coroutine_fn raw_co_pwrite_zeroes(
@@ -2616,8 +2628,8 @@
.bdrv_co_preadv = raw_co_preadv,
.bdrv_co_pwritev = raw_co_pwritev,
- .bdrv_aio_flush = raw_aio_flush,
- .bdrv_aio_pdiscard = raw_aio_pdiscard,
+ .bdrv_co_flush_to_disk = raw_co_flush_to_disk,
+ .bdrv_co_pdiscard = raw_co_pdiscard,
.bdrv_co_copy_range_from = raw_co_copy_range_from,
.bdrv_co_copy_range_to = raw_co_copy_range_to,
.bdrv_refresh_limits = raw_refresh_limits,
@@ -2625,7 +2637,7 @@
.bdrv_io_unplug = raw_aio_unplug,
.bdrv_attach_aio_context = raw_aio_attach_aio_context,
- .bdrv_truncate = raw_truncate,
+ .bdrv_co_truncate = raw_co_truncate,
.bdrv_getlength = raw_getlength,
.bdrv_get_info = raw_get_info,
.bdrv_get_allocated_file_size
@@ -2983,17 +2995,18 @@
return -EIO;
}
-static coroutine_fn BlockAIOCB *hdev_aio_pdiscard(BlockDriverState *bs,
- int64_t offset, int bytes,
- BlockCompletionFunc *cb, void *opaque)
+static coroutine_fn int
+hdev_co_pdiscard(BlockDriverState *bs, int64_t offset, int bytes)
{
BDRVRawState *s = bs->opaque;
+ int ret;
- if (fd_open(bs) < 0) {
- return NULL;
+ ret = fd_open(bs);
+ if (ret < 0) {
+ return ret;
}
- return paio_submit(bs, s->fd, offset, NULL, bytes,
- cb, opaque, QEMU_AIO_DISCARD|QEMU_AIO_BLKDEV);
+ return paio_submit_co(bs, s->fd, offset, NULL, bytes,
+ QEMU_AIO_DISCARD | QEMU_AIO_BLKDEV);
}
static coroutine_fn int hdev_co_pwrite_zeroes(BlockDriverState *bs,
@@ -3097,15 +3110,15 @@
.bdrv_co_preadv = raw_co_preadv,
.bdrv_co_pwritev = raw_co_pwritev,
- .bdrv_aio_flush = raw_aio_flush,
- .bdrv_aio_pdiscard = hdev_aio_pdiscard,
+ .bdrv_co_flush_to_disk = raw_co_flush_to_disk,
+ .bdrv_co_pdiscard = hdev_co_pdiscard,
.bdrv_co_copy_range_from = raw_co_copy_range_from,
.bdrv_co_copy_range_to = raw_co_copy_range_to,
.bdrv_refresh_limits = raw_refresh_limits,
.bdrv_io_plug = raw_aio_plug,
.bdrv_io_unplug = raw_aio_unplug,
- .bdrv_truncate = raw_truncate,
+ .bdrv_co_truncate = raw_co_truncate,
.bdrv_getlength = raw_getlength,
.bdrv_get_info = raw_get_info,
.bdrv_get_allocated_file_size
@@ -3222,12 +3235,12 @@
.bdrv_co_preadv = raw_co_preadv,
.bdrv_co_pwritev = raw_co_pwritev,
- .bdrv_aio_flush = raw_aio_flush,
+ .bdrv_co_flush_to_disk = raw_co_flush_to_disk,
.bdrv_refresh_limits = raw_refresh_limits,
.bdrv_io_plug = raw_aio_plug,
.bdrv_io_unplug = raw_aio_unplug,
- .bdrv_truncate = raw_truncate,
+ .bdrv_co_truncate = raw_co_truncate,
.bdrv_getlength = raw_getlength,
.has_variable_length = true,
.bdrv_get_allocated_file_size
@@ -3352,12 +3365,12 @@
.bdrv_co_preadv = raw_co_preadv,
.bdrv_co_pwritev = raw_co_pwritev,
- .bdrv_aio_flush = raw_aio_flush,
+ .bdrv_co_flush_to_disk = raw_co_flush_to_disk,
.bdrv_refresh_limits = raw_refresh_limits,
.bdrv_io_plug = raw_aio_plug,
.bdrv_io_unplug = raw_aio_unplug,
- .bdrv_truncate = raw_truncate,
+ .bdrv_co_truncate = raw_co_truncate,
.bdrv_getlength = raw_getlength,
.has_variable_length = true,
.bdrv_get_allocated_file_size
diff --git a/block/file-win32.c b/block/file-win32.c
index 3c67db4..0411fe8 100644
--- a/block/file-win32.c
+++ b/block/file-win32.c
@@ -467,8 +467,8 @@
}
}
-static int raw_truncate(BlockDriverState *bs, int64_t offset,
- PreallocMode prealloc, Error **errp)
+static int coroutine_fn raw_co_truncate(BlockDriverState *bs, int64_t offset,
+ PreallocMode prealloc, Error **errp)
{
BDRVRawState *s = bs->opaque;
LONG low, high;
@@ -640,7 +640,7 @@
.bdrv_aio_pwritev = raw_aio_pwritev,
.bdrv_aio_flush = raw_aio_flush,
- .bdrv_truncate = raw_truncate,
+ .bdrv_co_truncate = raw_co_truncate,
.bdrv_getlength = raw_getlength,
.bdrv_get_allocated_file_size
= raw_get_allocated_file_size,
diff --git a/block/gluster.c b/block/gluster.c
index b5fe7f3..a4e1c8e 100644
--- a/block/gluster.c
+++ b/block/gluster.c
@@ -1177,8 +1177,10 @@
return acb.ret;
}
-static int qemu_gluster_truncate(BlockDriverState *bs, int64_t offset,
- PreallocMode prealloc, Error **errp)
+static coroutine_fn int qemu_gluster_co_truncate(BlockDriverState *bs,
+ int64_t offset,
+ PreallocMode prealloc,
+ Error **errp)
{
BDRVGlusterState *s = bs->opaque;
return qemu_gluster_do_truncate(s->fd, offset, prealloc, errp);
@@ -1499,7 +1501,7 @@
.bdrv_co_create_opts = qemu_gluster_co_create_opts,
.bdrv_getlength = qemu_gluster_getlength,
.bdrv_get_allocated_file_size = qemu_gluster_allocated_file_size,
- .bdrv_truncate = qemu_gluster_truncate,
+ .bdrv_co_truncate = qemu_gluster_co_truncate,
.bdrv_co_readv = qemu_gluster_co_readv,
.bdrv_co_writev = qemu_gluster_co_writev,
.bdrv_co_flush_to_disk = qemu_gluster_co_flush_to_disk,
@@ -1528,7 +1530,7 @@
.bdrv_co_create_opts = qemu_gluster_co_create_opts,
.bdrv_getlength = qemu_gluster_getlength,
.bdrv_get_allocated_file_size = qemu_gluster_allocated_file_size,
- .bdrv_truncate = qemu_gluster_truncate,
+ .bdrv_co_truncate = qemu_gluster_co_truncate,
.bdrv_co_readv = qemu_gluster_co_readv,
.bdrv_co_writev = qemu_gluster_co_writev,
.bdrv_co_flush_to_disk = qemu_gluster_co_flush_to_disk,
@@ -1557,7 +1559,7 @@
.bdrv_co_create_opts = qemu_gluster_co_create_opts,
.bdrv_getlength = qemu_gluster_getlength,
.bdrv_get_allocated_file_size = qemu_gluster_allocated_file_size,
- .bdrv_truncate = qemu_gluster_truncate,
+ .bdrv_co_truncate = qemu_gluster_co_truncate,
.bdrv_co_readv = qemu_gluster_co_readv,
.bdrv_co_writev = qemu_gluster_co_writev,
.bdrv_co_flush_to_disk = qemu_gluster_co_flush_to_disk,
@@ -1592,7 +1594,7 @@
.bdrv_co_create_opts = qemu_gluster_co_create_opts,
.bdrv_getlength = qemu_gluster_getlength,
.bdrv_get_allocated_file_size = qemu_gluster_allocated_file_size,
- .bdrv_truncate = qemu_gluster_truncate,
+ .bdrv_co_truncate = qemu_gluster_co_truncate,
.bdrv_co_readv = qemu_gluster_co_readv,
.bdrv_co_writev = qemu_gluster_co_writev,
.bdrv_co_flush_to_disk = qemu_gluster_co_flush_to_disk,
diff --git a/block/io.c b/block/io.c
index ef4fedd..7035b78 100644
--- a/block/io.c
+++ b/block/io.c
@@ -1429,24 +1429,6 @@
return ret;
}
-static int coroutine_fn bdrv_co_do_readv(BdrvChild *child,
- int64_t sector_num, int nb_sectors, QEMUIOVector *qiov,
- BdrvRequestFlags flags)
-{
- if (nb_sectors < 0 || nb_sectors > BDRV_REQUEST_MAX_SECTORS) {
- return -EINVAL;
- }
-
- return bdrv_co_preadv(child, sector_num << BDRV_SECTOR_BITS,
- nb_sectors << BDRV_SECTOR_BITS, qiov, flags);
-}
-
-int coroutine_fn bdrv_co_readv(BdrvChild *child, int64_t sector_num,
- int nb_sectors, QEMUIOVector *qiov)
-{
- return bdrv_co_do_readv(child, sector_num, nb_sectors, qiov, 0);
-}
-
static int coroutine_fn bdrv_co_do_pwrite_zeroes(BlockDriverState *bs,
int64_t offset, int bytes, BdrvRequestFlags flags)
{
@@ -1889,24 +1871,6 @@
return ret;
}
-static int coroutine_fn bdrv_co_do_writev(BdrvChild *child,
- int64_t sector_num, int nb_sectors, QEMUIOVector *qiov,
- BdrvRequestFlags flags)
-{
- if (nb_sectors < 0 || nb_sectors > BDRV_REQUEST_MAX_SECTORS) {
- return -EINVAL;
- }
-
- return bdrv_co_pwritev(child, sector_num << BDRV_SECTOR_BITS,
- nb_sectors << BDRV_SECTOR_BITS, qiov, flags);
-}
-
-int coroutine_fn bdrv_co_writev(BdrvChild *child, int64_t sector_num,
- int nb_sectors, QEMUIOVector *qiov)
-{
- return bdrv_co_do_writev(child, sector_num, nb_sectors, qiov, 0);
-}
-
int coroutine_fn bdrv_co_pwrite_zeroes(BdrvChild *child, int64_t offset,
int bytes, BdrvRequestFlags flags)
{
@@ -2932,6 +2896,9 @@
BdrvRequestFlags flags,
bool recurse_src)
{
+ BdrvTrackedRequest src_req, dst_req;
+ BlockDriverState *src_bs = src->bs;
+ BlockDriverState *dst_bs = dst->bs;
int ret;
if (!src || !dst || !src->bs || !dst->bs) {
@@ -2955,17 +2922,31 @@
|| src->bs->encrypted || dst->bs->encrypted) {
return -ENOTSUP;
}
+ bdrv_inc_in_flight(src_bs);
+ bdrv_inc_in_flight(dst_bs);
+ tracked_request_begin(&src_req, src_bs, src_offset,
+ bytes, BDRV_TRACKED_READ);
+ tracked_request_begin(&dst_req, dst_bs, dst_offset,
+ bytes, BDRV_TRACKED_WRITE);
+
+ wait_serialising_requests(&src_req);
+ wait_serialising_requests(&dst_req);
if (recurse_src) {
- return src->bs->drv->bdrv_co_copy_range_from(src->bs,
- src, src_offset,
- dst, dst_offset,
- bytes, flags);
+ ret = src->bs->drv->bdrv_co_copy_range_from(src->bs,
+ src, src_offset,
+ dst, dst_offset,
+ bytes, flags);
} else {
- return dst->bs->drv->bdrv_co_copy_range_to(dst->bs,
- src, src_offset,
- dst, dst_offset,
- bytes, flags);
+ ret = dst->bs->drv->bdrv_co_copy_range_to(dst->bs,
+ src, src_offset,
+ dst, dst_offset,
+ bytes, flags);
}
+ tracked_request_end(&src_req);
+ tracked_request_end(&dst_req);
+ bdrv_dec_in_flight(src_bs);
+ bdrv_dec_in_flight(dst_bs);
+ return ret;
}
/* Copy range from @src to @dst.
@@ -2996,27 +2977,141 @@
BdrvChild *dst, uint64_t dst_offset,
uint64_t bytes, BdrvRequestFlags flags)
{
- BdrvTrackedRequest src_req, dst_req;
- BlockDriverState *src_bs = src->bs;
- BlockDriverState *dst_bs = dst->bs;
+ return bdrv_co_copy_range_from(src, src_offset,
+ dst, dst_offset,
+ bytes, flags);
+}
+
+static void bdrv_parent_cb_resize(BlockDriverState *bs)
+{
+ BdrvChild *c;
+ QLIST_FOREACH(c, &bs->parents, next_parent) {
+ if (c->role->resize) {
+ c->role->resize(c);
+ }
+ }
+}
+
+/**
+ * Truncate file to 'offset' bytes (needed only for file protocols)
+ */
+int coroutine_fn bdrv_co_truncate(BdrvChild *child, int64_t offset,
+ PreallocMode prealloc, Error **errp)
+{
+ BlockDriverState *bs = child->bs;
+ BlockDriver *drv = bs->drv;
+ BdrvTrackedRequest req;
+ int64_t old_size, new_bytes;
int ret;
- bdrv_inc_in_flight(src_bs);
- bdrv_inc_in_flight(dst_bs);
- tracked_request_begin(&src_req, src_bs, src_offset,
- bytes, BDRV_TRACKED_READ);
- tracked_request_begin(&dst_req, dst_bs, dst_offset,
- bytes, BDRV_TRACKED_WRITE);
+ assert(child->perm & BLK_PERM_RESIZE);
- wait_serialising_requests(&src_req);
- wait_serialising_requests(&dst_req);
- ret = bdrv_co_copy_range_from(src, src_offset,
- dst, dst_offset,
- bytes, flags);
+ /* if bs->drv == NULL, bs is closed, so there's nothing to do here */
+ if (!drv) {
+ error_setg(errp, "No medium inserted");
+ return -ENOMEDIUM;
+ }
+ if (offset < 0) {
+ error_setg(errp, "Image size cannot be negative");
+ return -EINVAL;
+ }
- tracked_request_end(&src_req);
- tracked_request_end(&dst_req);
- bdrv_dec_in_flight(src_bs);
- bdrv_dec_in_flight(dst_bs);
+ old_size = bdrv_getlength(bs);
+ if (old_size < 0) {
+ error_setg_errno(errp, -old_size, "Failed to get old image size");
+ return old_size;
+ }
+
+ if (offset > old_size) {
+ new_bytes = offset - old_size;
+ } else {
+ new_bytes = 0;
+ }
+
+ bdrv_inc_in_flight(bs);
+ tracked_request_begin(&req, bs, offset, new_bytes, BDRV_TRACKED_TRUNCATE);
+
+ /* If we are growing the image and potentially using preallocation for the
+ * new area, we need to make sure that no write requests are made to it
+ * concurrently or they might be overwritten by preallocation. */
+ if (new_bytes) {
+ mark_request_serialising(&req, 1);
+ wait_serialising_requests(&req);
+ }
+
+ if (!drv->bdrv_co_truncate) {
+ if (bs->file && drv->is_filter) {
+ ret = bdrv_co_truncate(bs->file, offset, prealloc, errp);
+ goto out;
+ }
+ error_setg(errp, "Image format driver does not support resize");
+ ret = -ENOTSUP;
+ goto out;
+ }
+ if (bs->read_only) {
+ error_setg(errp, "Image is read-only");
+ ret = -EACCES;
+ goto out;
+ }
+
+ assert(!(bs->open_flags & BDRV_O_INACTIVE));
+
+ ret = drv->bdrv_co_truncate(bs, offset, prealloc, errp);
+ if (ret < 0) {
+ goto out;
+ }
+ ret = refresh_total_sectors(bs, offset >> BDRV_SECTOR_BITS);
+ if (ret < 0) {
+ error_setg_errno(errp, -ret, "Could not refresh total sector count");
+ } else {
+ offset = bs->total_sectors * BDRV_SECTOR_SIZE;
+ }
+ bdrv_dirty_bitmap_truncate(bs, offset);
+ bdrv_parent_cb_resize(bs);
+ atomic_inc(&bs->write_gen);
+
+out:
+ tracked_request_end(&req);
+ bdrv_dec_in_flight(bs);
+
return ret;
}
+
+typedef struct TruncateCo {
+ BdrvChild *child;
+ int64_t offset;
+ PreallocMode prealloc;
+ Error **errp;
+ int ret;
+} TruncateCo;
+
+static void coroutine_fn bdrv_truncate_co_entry(void *opaque)
+{
+ TruncateCo *tco = opaque;
+ tco->ret = bdrv_co_truncate(tco->child, tco->offset, tco->prealloc,
+ tco->errp);
+}
+
+int bdrv_truncate(BdrvChild *child, int64_t offset, PreallocMode prealloc,
+ Error **errp)
+{
+ Coroutine *co;
+ TruncateCo tco = {
+ .child = child,
+ .offset = offset,
+ .prealloc = prealloc,
+ .errp = errp,
+ .ret = NOT_DONE,
+ };
+
+ if (qemu_in_coroutine()) {
+ /* Fast-path if already in coroutine context */
+ bdrv_truncate_co_entry(&tco);
+ } else {
+ co = qemu_coroutine_create(bdrv_truncate_co_entry, &tco);
+ qemu_coroutine_enter(co);
+ BDRV_POLL_WHILE(child->bs, tco.ret == NOT_DONE);
+ }
+
+ return tco.ret;
+}
diff --git a/block/iscsi.c b/block/iscsi.c
index 9f00fb4..9beb06d 100644
--- a/block/iscsi.c
+++ b/block/iscsi.c
@@ -2085,8 +2085,8 @@
}
}
-static int iscsi_truncate(BlockDriverState *bs, int64_t offset,
- PreallocMode prealloc, Error **errp)
+static int coroutine_fn iscsi_co_truncate(BlockDriverState *bs, int64_t offset,
+ PreallocMode prealloc, Error **errp)
{
IscsiLun *iscsilun = bs->opaque;
Error *local_err = NULL;
@@ -2226,7 +2226,7 @@
desc[5] = (dd->designator_type & 0xF)
| ((dd->association & 3) << 4);
desc[7] = dd->designator_length;
- memcpy(desc + 8, dd->designator, dd->designator_length);
+ memcpy(desc + 8, dd->designator, MIN(dd->designator_length, 20));
desc[28] = 0;
desc[29] = (lun->block_size >> 16) & 0xFF;
@@ -2431,7 +2431,7 @@
.bdrv_getlength = iscsi_getlength,
.bdrv_get_info = iscsi_get_info,
- .bdrv_truncate = iscsi_truncate,
+ .bdrv_co_truncate = iscsi_co_truncate,
.bdrv_refresh_limits = iscsi_refresh_limits,
.bdrv_co_block_status = iscsi_co_block_status,
@@ -2468,7 +2468,7 @@
.bdrv_getlength = iscsi_getlength,
.bdrv_get_info = iscsi_get_info,
- .bdrv_truncate = iscsi_truncate,
+ .bdrv_co_truncate = iscsi_co_truncate,
.bdrv_refresh_limits = iscsi_refresh_limits,
.bdrv_co_block_status = iscsi_co_block_status,
diff --git a/block/nfs.c b/block/nfs.c
index 743ca04..eab1a2c 100644
--- a/block/nfs.c
+++ b/block/nfs.c
@@ -743,8 +743,9 @@
return (task.ret < 0 ? task.ret : st.st_blocks * 512);
}
-static int nfs_file_truncate(BlockDriverState *bs, int64_t offset,
- PreallocMode prealloc, Error **errp)
+static int coroutine_fn
+nfs_file_co_truncate(BlockDriverState *bs, int64_t offset,
+ PreallocMode prealloc, Error **errp)
{
NFSClient *client = bs->opaque;
int ret;
@@ -873,7 +874,7 @@
.bdrv_has_zero_init = nfs_has_zero_init,
.bdrv_get_allocated_file_size = nfs_get_allocated_file_size,
- .bdrv_truncate = nfs_file_truncate,
+ .bdrv_co_truncate = nfs_file_co_truncate,
.bdrv_file_open = nfs_file_open,
.bdrv_close = nfs_file_close,
diff --git a/block/parallels.c b/block/parallels.c
index fd215e2..cc94458 100644
--- a/block/parallels.c
+++ b/block/parallels.c
@@ -227,14 +227,15 @@
};
qemu_iovec_init_external(&qiov, &iov, 1);
- ret = bdrv_co_readv(bs->backing, idx * s->tracks, nb_cow_sectors,
- &qiov);
+ ret = bdrv_co_preadv(bs->backing, idx * s->tracks * BDRV_SECTOR_SIZE,
+ nb_cow_bytes, &qiov, 0);
if (ret < 0) {
qemu_vfree(iov.iov_base);
return ret;
}
- ret = bdrv_co_writev(bs->file, s->data_end, nb_cow_sectors, &qiov);
+ ret = bdrv_co_pwritev(bs->file, s->data_end * BDRV_SECTOR_SIZE,
+ nb_cow_bytes, &qiov, 0);
qemu_vfree(iov.iov_base);
if (ret < 0) {
return ret;
@@ -340,7 +341,8 @@
qemu_iovec_reset(&hd_qiov);
qemu_iovec_concat(&hd_qiov, qiov, bytes_done, nbytes);
- ret = bdrv_co_writev(bs->file, position, n, &hd_qiov);
+ ret = bdrv_co_pwritev(bs->file, position * BDRV_SECTOR_SIZE, nbytes,
+ &hd_qiov, 0);
if (ret < 0) {
break;
}
@@ -379,7 +381,8 @@
if (position < 0) {
if (bs->backing) {
- ret = bdrv_co_readv(bs->backing, sector_num, n, &hd_qiov);
+ ret = bdrv_co_preadv(bs->backing, sector_num * BDRV_SECTOR_SIZE,
+ nbytes, &hd_qiov, 0);
if (ret < 0) {
break;
}
@@ -387,7 +390,8 @@
qemu_iovec_memset(&hd_qiov, 0, 0, nbytes);
}
} else {
- ret = bdrv_co_readv(bs->file, position, n, &hd_qiov);
+ ret = bdrv_co_preadv(bs->file, position * BDRV_SECTOR_SIZE, nbytes,
+ &hd_qiov, 0);
if (ret < 0) {
break;
}
diff --git a/block/qcow.c b/block/qcow.c
index 5532731..102d058 100644
--- a/block/qcow.c
+++ b/block/qcow.c
@@ -70,7 +70,6 @@
typedef struct BDRVQcowState {
int cluster_bits;
int cluster_size;
- int cluster_sectors;
int l2_bits;
int l2_size;
unsigned int l1_size;
@@ -203,9 +202,8 @@
ret = -EINVAL;
goto fail;
}
- qdict_del(encryptopts, "format");
- crypto_opts = block_crypto_open_opts_init(
- Q_CRYPTO_BLOCK_FORMAT_QCOW, encryptopts, errp);
+ qdict_put_str(encryptopts, "format", "qcow");
+ crypto_opts = block_crypto_open_opts_init(encryptopts, errp);
if (!crypto_opts) {
ret = -EINVAL;
goto fail;
@@ -236,7 +234,6 @@
}
s->cluster_bits = header.cluster_bits;
s->cluster_size = 1 << s->cluster_bits;
- s->cluster_sectors = 1 << (s->cluster_bits - 9);
s->l2_bits = header.l2_bits;
s->l2_size = 1 << s->l2_bits;
bs->total_sectors = header.size / 512;
@@ -346,8 +343,8 @@
*
* 0 to not allocate.
*
- * 1 to allocate a normal cluster (for sector indexes 'n_start' to
- * 'n_end')
+ * 1 to allocate a normal cluster (for sector-aligned byte offsets 'n_start'
+ * to 'n_end' within the cluster)
*
* 2 to allocate a compressed cluster of size
* 'compressed_size'. 'compressed_size' must be > 0 and <
@@ -441,9 +438,10 @@
if (!allocate)
return 0;
BLKDBG_EVENT(bs->file, BLKDBG_CLUSTER_ALLOC);
+ assert(QEMU_IS_ALIGNED(n_start | n_end, BDRV_SECTOR_SIZE));
/* allocate a new cluster */
if ((cluster_offset & QCOW_OFLAG_COMPRESSED) &&
- (n_end - n_start) < s->cluster_sectors) {
+ (n_end - n_start) < s->cluster_size) {
/* if the cluster is already compressed, we must
decompress it in the case it is not completely
overwritten */
@@ -481,16 +479,15 @@
/* if encrypted, we must initialize the cluster
content which won't be written */
if (bs->encrypted &&
- (n_end - n_start) < s->cluster_sectors) {
- uint64_t start_sect;
+ (n_end - n_start) < s->cluster_size) {
+ uint64_t start_offset;
assert(s->crypto);
- start_sect = (offset & ~(s->cluster_size - 1)) >> 9;
- for(i = 0; i < s->cluster_sectors; i++) {
+ start_offset = offset & ~(s->cluster_size - 1);
+ for (i = 0; i < s->cluster_size; i += BDRV_SECTOR_SIZE) {
if (i < n_start || i >= n_end) {
- memset(s->cluster_data, 0x00, 512);
+ memset(s->cluster_data, 0x00, BDRV_SECTOR_SIZE);
if (qcrypto_block_encrypt(s->crypto,
- (start_sect + i) *
- BDRV_SECTOR_SIZE,
+ start_offset + i,
s->cluster_data,
BDRV_SECTOR_SIZE,
NULL) < 0) {
@@ -498,8 +495,9 @@
}
BLKDBG_EVENT(bs->file, BLKDBG_WRITE_AIO);
ret = bdrv_pwrite(bs->file,
- cluster_offset + i * 512,
- s->cluster_data, 512);
+ cluster_offset + i,
+ s->cluster_data,
+ BDRV_SECTOR_SIZE);
if (ret < 0) {
return ret;
}
@@ -613,11 +611,21 @@
return 0;
}
-static coroutine_fn int qcow_co_readv(BlockDriverState *bs, int64_t sector_num,
- int nb_sectors, QEMUIOVector *qiov)
+static void qcow_refresh_limits(BlockDriverState *bs, Error **errp)
+{
+ /* At least encrypted images require 512-byte alignment. Apply the
+ * limit universally, rather than just on encrypted images, as
+ * it's easier to let the block layer handle rounding than to
+ * audit this code further. */
+ bs->bl.request_alignment = BDRV_SECTOR_SIZE;
+}
+
+static coroutine_fn int qcow_co_preadv(BlockDriverState *bs, uint64_t offset,
+ uint64_t bytes, QEMUIOVector *qiov,
+ int flags)
{
BDRVQcowState *s = bs->opaque;
- int index_in_cluster;
+ int offset_in_cluster;
int ret = 0, n;
uint64_t cluster_offset;
struct iovec hd_iov;
@@ -625,6 +633,7 @@
uint8_t *buf;
void *orig_buf;
+ assert(!flags);
if (qiov->niov > 1) {
buf = orig_buf = qemu_try_blockalign(bs, qiov->size);
if (buf == NULL) {
@@ -637,36 +646,35 @@
qemu_co_mutex_lock(&s->lock);
- while (nb_sectors != 0) {
+ while (bytes != 0) {
/* prepare next request */
- ret = get_cluster_offset(bs, sector_num << 9,
- 0, 0, 0, 0, &cluster_offset);
+ ret = get_cluster_offset(bs, offset, 0, 0, 0, 0, &cluster_offset);
if (ret < 0) {
break;
}
- index_in_cluster = sector_num & (s->cluster_sectors - 1);
- n = s->cluster_sectors - index_in_cluster;
- if (n > nb_sectors) {
- n = nb_sectors;
+ offset_in_cluster = offset & (s->cluster_size - 1);
+ n = s->cluster_size - offset_in_cluster;
+ if (n > bytes) {
+ n = bytes;
}
if (!cluster_offset) {
if (bs->backing) {
/* read from the base image */
hd_iov.iov_base = (void *)buf;
- hd_iov.iov_len = n * 512;
+ hd_iov.iov_len = n;
qemu_iovec_init_external(&hd_qiov, &hd_iov, 1);
qemu_co_mutex_unlock(&s->lock);
/* qcow2 emits this on bs->file instead of bs->backing */
BLKDBG_EVENT(bs->file, BLKDBG_READ_BACKING_AIO);
- ret = bdrv_co_readv(bs->backing, sector_num, n, &hd_qiov);
+ ret = bdrv_co_preadv(bs->backing, offset, n, &hd_qiov, 0);
qemu_co_mutex_lock(&s->lock);
if (ret < 0) {
break;
}
} else {
/* Note: in this case, no need to wait */
- memset(buf, 0, 512 * n);
+ memset(buf, 0, n);
}
} else if (cluster_offset & QCOW_OFLAG_COMPRESSED) {
/* add AIO support for compressed blocks ? */
@@ -674,21 +682,19 @@
ret = -EIO;
break;
}
- memcpy(buf,
- s->cluster_cache + index_in_cluster * 512, 512 * n);
+ memcpy(buf, s->cluster_cache + offset_in_cluster, n);
} else {
if ((cluster_offset & 511) != 0) {
ret = -EIO;
break;
}
hd_iov.iov_base = (void *)buf;
- hd_iov.iov_len = n * 512;
+ hd_iov.iov_len = n;
qemu_iovec_init_external(&hd_qiov, &hd_iov, 1);
qemu_co_mutex_unlock(&s->lock);
BLKDBG_EVENT(bs->file, BLKDBG_READ_AIO);
- ret = bdrv_co_readv(bs->file,
- (cluster_offset >> 9) + index_in_cluster,
- n, &hd_qiov);
+ ret = bdrv_co_preadv(bs->file, cluster_offset + offset_in_cluster,
+ n, &hd_qiov, 0);
qemu_co_mutex_lock(&s->lock);
if (ret < 0) {
break;
@@ -696,8 +702,7 @@
if (bs->encrypted) {
assert(s->crypto);
if (qcrypto_block_decrypt(s->crypto,
- sector_num * BDRV_SECTOR_SIZE, buf,
- n * BDRV_SECTOR_SIZE, NULL) < 0) {
+ offset, buf, n, NULL) < 0) {
ret = -EIO;
break;
}
@@ -705,9 +710,9 @@
}
ret = 0;
- nb_sectors -= n;
- sector_num += n;
- buf += n * 512;
+ bytes -= n;
+ offset += n;
+ buf += n;
}
qemu_co_mutex_unlock(&s->lock);
@@ -720,12 +725,12 @@
return ret;
}
-static coroutine_fn int qcow_co_writev(BlockDriverState *bs, int64_t sector_num,
- int nb_sectors, QEMUIOVector *qiov,
- int flags)
+static coroutine_fn int qcow_co_pwritev(BlockDriverState *bs, uint64_t offset,
+ uint64_t bytes, QEMUIOVector *qiov,
+ int flags)
{
BDRVQcowState *s = bs->opaque;
- int index_in_cluster;
+ int offset_in_cluster;
uint64_t cluster_offset;
int ret = 0, n;
struct iovec hd_iov;
@@ -751,16 +756,14 @@
qemu_co_mutex_lock(&s->lock);
- while (nb_sectors != 0) {
-
- index_in_cluster = sector_num & (s->cluster_sectors - 1);
- n = s->cluster_sectors - index_in_cluster;
- if (n > nb_sectors) {
- n = nb_sectors;
+ while (bytes != 0) {
+ offset_in_cluster = offset & (s->cluster_size - 1);
+ n = s->cluster_size - offset_in_cluster;
+ if (n > bytes) {
+ n = bytes;
}
- ret = get_cluster_offset(bs, sector_num << 9, 1, 0,
- index_in_cluster,
- index_in_cluster + n, &cluster_offset);
+ ret = get_cluster_offset(bs, offset, 1, 0, offset_in_cluster,
+ offset_in_cluster + n, &cluster_offset);
if (ret < 0) {
break;
}
@@ -770,30 +773,28 @@
}
if (bs->encrypted) {
assert(s->crypto);
- if (qcrypto_block_encrypt(s->crypto, sector_num * BDRV_SECTOR_SIZE,
- buf, n * BDRV_SECTOR_SIZE, NULL) < 0) {
+ if (qcrypto_block_encrypt(s->crypto, offset, buf, n, NULL) < 0) {
ret = -EIO;
break;
}
}
hd_iov.iov_base = (void *)buf;
- hd_iov.iov_len = n * 512;
+ hd_iov.iov_len = n;
qemu_iovec_init_external(&hd_qiov, &hd_iov, 1);
qemu_co_mutex_unlock(&s->lock);
BLKDBG_EVENT(bs->file, BLKDBG_WRITE_AIO);
- ret = bdrv_co_writev(bs->file,
- (cluster_offset >> 9) + index_in_cluster,
- n, &hd_qiov);
+ ret = bdrv_co_pwritev(bs->file, cluster_offset + offset_in_cluster,
+ n, &hd_qiov, 0);
qemu_co_mutex_lock(&s->lock);
if (ret < 0) {
break;
}
ret = 0;
- nb_sectors -= n;
- sector_num += n;
- buf += n * 512;
+ bytes -= n;
+ offset += n;
+ buf += n;
}
qemu_co_mutex_unlock(&s->lock);
@@ -1108,8 +1109,7 @@
if (ret != Z_STREAM_END || out_len >= s->cluster_size) {
/* could not compress: write normal cluster */
- ret = qcow_co_writev(bs, offset >> BDRV_SECTOR_BITS,
- bytes >> BDRV_SECTOR_BITS, qiov, 0);
+ ret = qcow_co_pwritev(bs, offset, bytes, qiov, 0);
if (ret < 0) {
goto fail;
}
@@ -1194,9 +1194,10 @@
.bdrv_co_create_opts = qcow_co_create_opts,
.bdrv_has_zero_init = bdrv_has_zero_init_1,
.supports_backing = true,
+ .bdrv_refresh_limits = qcow_refresh_limits,
- .bdrv_co_readv = qcow_co_readv,
- .bdrv_co_writev = qcow_co_writev,
+ .bdrv_co_preadv = qcow_co_preadv,
+ .bdrv_co_pwritev = qcow_co_pwritev,
.bdrv_co_block_status = qcow_co_block_status,
.bdrv_make_empty = qcow_make_empty,
diff --git a/block/qcow2-cluster.c b/block/qcow2-cluster.c
index 0d74584..d37fe08 100644
--- a/block/qcow2-cluster.c
+++ b/block/qcow2-cluster.c
@@ -994,6 +994,17 @@
return ret;
}
+/**
+ * Frees the allocated clusters because the request failed and they won't
+ * actually be linked.
+ */
+void qcow2_alloc_cluster_abort(BlockDriverState *bs, QCowL2Meta *m)
+{
+ BDRVQcow2State *s = bs->opaque;
+ qcow2_free_clusters(bs, m->alloc_offset, m->nb_clusters << s->cluster_bits,
+ QCOW2_DISCARD_NEVER);
+}
+
/*
* Returns the number of contiguous clusters that can be used for an allocating
* write, but require COW to be performed (this includes yet unallocated space,
diff --git a/block/qcow2.c b/block/qcow2.c
index a3a3aa2..2f9e58e 100644
--- a/block/qcow2.c
+++ b/block/qcow2.c
@@ -1040,9 +1040,8 @@
ret = -EINVAL;
goto fail;
}
- qdict_del(encryptopts, "format");
- r->crypto_opts = block_crypto_open_opts_init(
- Q_CRYPTO_BLOCK_FORMAT_QCOW, encryptopts, errp);
+ qdict_put_str(encryptopts, "format", "qcow");
+ r->crypto_opts = block_crypto_open_opts_init(encryptopts, errp);
break;
case QCOW_CRYPT_LUKS:
@@ -1053,9 +1052,8 @@
ret = -EINVAL;
goto fail;
}
- qdict_del(encryptopts, "format");
- r->crypto_opts = block_crypto_open_opts_init(
- Q_CRYPTO_BLOCK_FORMAT_LUKS, encryptopts, errp);
+ qdict_put_str(encryptopts, "format", "luks");
+ r->crypto_opts = block_crypto_open_opts_init(encryptopts, errp);
break;
default:
@@ -1772,11 +1770,13 @@
while (l2meta != NULL) {
QCowL2Meta *next;
- if (!ret && link_l2) {
+ if (link_l2) {
ret = qcow2_alloc_cluster_link_l2(bs, l2meta);
if (ret) {
goto out;
}
+ } else {
+ qcow2_alloc_cluster_abort(bs, l2meta);
}
/* Take the request off the list of running requests */
@@ -2521,15 +2521,6 @@
return ret;
}
-
-typedef struct PreallocCo {
- BlockDriverState *bs;
- uint64_t offset;
- uint64_t new_length;
-
- int ret;
-} PreallocCo;
-
/**
* Preallocates metadata structures for data clusters between @offset (in the
* guest disk) and @new_length (which is thus generally the new guest disk
@@ -2537,21 +2528,15 @@
*
* Returns: 0 on success, -errno on failure.
*/
-static void coroutine_fn preallocate_co(void *opaque)
+static int coroutine_fn preallocate_co(BlockDriverState *bs, uint64_t offset,
+ uint64_t new_length)
{
- PreallocCo *params = opaque;
- BlockDriverState *bs = params->bs;
- uint64_t offset = params->offset;
- uint64_t new_length = params->new_length;
- BDRVQcow2State *s = bs->opaque;
uint64_t bytes;
uint64_t host_offset = 0;
unsigned int cur_bytes;
int ret;
QCowL2Meta *meta;
- qemu_co_mutex_lock(&s->lock);
-
assert(offset <= new_length);
bytes = new_length - offset;
@@ -2560,7 +2545,7 @@
ret = qcow2_alloc_cluster_offset(bs, offset, &cur_bytes,
&host_offset, &meta);
if (ret < 0) {
- goto done;
+ return ret;
}
while (meta) {
@@ -2570,7 +2555,7 @@
if (ret < 0) {
qcow2_free_any_clusters(bs, meta->alloc_offset,
meta->nb_clusters, QCOW2_DISCARD_NEVER);
- goto done;
+ return ret;
}
/* There are no dependent requests, but we need to remove our
@@ -2597,35 +2582,11 @@
ret = bdrv_pwrite(bs->file, (host_offset + cur_bytes) - 1,
&data, 1);
if (ret < 0) {
- goto done;
+ return ret;
}
}
- ret = 0;
-
-done:
- qemu_co_mutex_unlock(&s->lock);
- params->ret = ret;
-}
-
-static int preallocate(BlockDriverState *bs,
- uint64_t offset, uint64_t new_length)
-{
- PreallocCo params = {
- .bs = bs,
- .offset = offset,
- .new_length = new_length,
- .ret = -EINPROGRESS,
- };
-
- if (qemu_in_coroutine()) {
- preallocate_co(¶ms);
- } else {
- Coroutine *co = qemu_coroutine_create(preallocate_co, ¶ms);
- bdrv_coroutine_enter(bs, co);
- BDRV_POLL_WHILE(bs, params.ret == -EINPROGRESS);
- }
- return params.ret;
+ return 0;
}
/* qcow2_refcount_metadata_size:
@@ -3041,7 +3002,11 @@
/* And if we're supposed to preallocate metadata, do that now */
if (qcow2_opts->preallocation != PREALLOC_MODE_OFF) {
- ret = preallocate(blk_bs(blk), 0, qcow2_opts->size);
+ BDRVQcow2State *s = blk_bs(blk)->opaque;
+ qemu_co_mutex_lock(&s->lock);
+ ret = preallocate_co(blk_bs(blk), 0, qcow2_opts->size);
+ qemu_co_mutex_unlock(&s->lock);
+
if (ret < 0) {
error_setg_errno(errp, -ret, "Could not preallocate metadata");
goto out;
@@ -3422,6 +3387,7 @@
}
bytes -= cur_bytes;
+ src_offset += cur_bytes;
dst_offset += cur_bytes;
}
ret = 0;
@@ -3437,8 +3403,8 @@
return ret;
}
-static int qcow2_truncate(BlockDriverState *bs, int64_t offset,
- PreallocMode prealloc, Error **errp)
+static int coroutine_fn qcow2_co_truncate(BlockDriverState *bs, int64_t offset,
+ PreallocMode prealloc, Error **errp)
{
BDRVQcow2State *s = bs->opaque;
uint64_t old_length;
@@ -3458,17 +3424,21 @@
return -EINVAL;
}
+ qemu_co_mutex_lock(&s->lock);
+
/* cannot proceed if image has snapshots */
if (s->nb_snapshots) {
error_setg(errp, "Can't resize an image which has snapshots");
- return -ENOTSUP;
+ ret = -ENOTSUP;
+ goto fail;
}
/* cannot proceed if image has bitmaps */
if (s->nb_bitmaps) {
/* TODO: resize bitmaps in the image */
error_setg(errp, "Can't resize an image which has bitmaps");
- return -ENOTSUP;
+ ret = -ENOTSUP;
+ goto fail;
}
old_length = bs->total_sectors * 512;
@@ -3479,7 +3449,8 @@
if (prealloc != PREALLOC_MODE_OFF) {
error_setg(errp,
"Preallocation can't be used for shrinking an image");
- return -EINVAL;
+ ret = -EINVAL;
+ goto fail;
}
ret = qcow2_cluster_discard(bs, ROUND_UP(offset, s->cluster_size),
@@ -3488,40 +3459,42 @@
QCOW2_DISCARD_ALWAYS, true);
if (ret < 0) {
error_setg_errno(errp, -ret, "Failed to discard cropped clusters");
- return ret;
+ goto fail;
}
ret = qcow2_shrink_l1_table(bs, new_l1_size);
if (ret < 0) {
error_setg_errno(errp, -ret,
"Failed to reduce the number of L2 tables");
- return ret;
+ goto fail;
}
ret = qcow2_shrink_reftable(bs);
if (ret < 0) {
error_setg_errno(errp, -ret,
"Failed to discard unused refblocks");
- return ret;
+ goto fail;
}
old_file_size = bdrv_getlength(bs->file->bs);
if (old_file_size < 0) {
error_setg_errno(errp, -old_file_size,
"Failed to inquire current file length");
- return old_file_size;
+ ret = old_file_size;
+ goto fail;
}
last_cluster = qcow2_get_last_cluster(bs, old_file_size);
if (last_cluster < 0) {
error_setg_errno(errp, -last_cluster,
"Failed to find the last cluster");
- return last_cluster;
+ ret = last_cluster;
+ goto fail;
}
if ((last_cluster + 1) * s->cluster_size < old_file_size) {
Error *local_err = NULL;
- bdrv_truncate(bs->file, (last_cluster + 1) * s->cluster_size,
- PREALLOC_MODE_OFF, &local_err);
+ bdrv_co_truncate(bs->file, (last_cluster + 1) * s->cluster_size,
+ PREALLOC_MODE_OFF, &local_err);
if (local_err) {
warn_reportf_err(local_err,
"Failed to truncate the tail of the image: ");
@@ -3531,7 +3504,7 @@
ret = qcow2_grow_l1_table(bs, new_l1_size, true);
if (ret < 0) {
error_setg_errno(errp, -ret, "Failed to grow the L1 table");
- return ret;
+ goto fail;
}
}
@@ -3540,10 +3513,10 @@
break;
case PREALLOC_MODE_METADATA:
- ret = preallocate(bs, old_length, offset);
+ ret = preallocate_co(bs, old_length, offset);
if (ret < 0) {
error_setg_errno(errp, -ret, "Preallocation failed");
- return ret;
+ goto fail;
}
break;
@@ -3559,7 +3532,8 @@
if (old_file_size < 0) {
error_setg_errno(errp, -old_file_size,
"Failed to inquire current file length");
- return old_file_size;
+ ret = old_file_size;
+ goto fail;
}
old_file_size = ROUND_UP(old_file_size, s->cluster_size);
@@ -3589,7 +3563,8 @@
if (allocation_start < 0) {
error_setg_errno(errp, -allocation_start,
"Failed to resize refcount structures");
- return allocation_start;
+ ret = allocation_start;
+ goto fail;
}
clusters_allocated = qcow2_alloc_clusters_at(bs, allocation_start,
@@ -3597,7 +3572,8 @@
if (clusters_allocated < 0) {
error_setg_errno(errp, -clusters_allocated,
"Failed to allocate data clusters");
- return -clusters_allocated;
+ ret = clusters_allocated;
+ goto fail;
}
assert(clusters_allocated == nb_new_data_clusters);
@@ -3605,13 +3581,13 @@
/* Allocate the data area */
new_file_size = allocation_start +
nb_new_data_clusters * s->cluster_size;
- ret = bdrv_truncate(bs->file, new_file_size, prealloc, errp);
+ ret = bdrv_co_truncate(bs->file, new_file_size, prealloc, errp);
if (ret < 0) {
error_prepend(errp, "Failed to resize underlying file: ");
qcow2_free_clusters(bs, allocation_start,
nb_new_data_clusters * s->cluster_size,
QCOW2_DISCARD_OTHER);
- return ret;
+ goto fail;
}
/* Create the necessary L2 entries */
@@ -3634,7 +3610,7 @@
qcow2_free_clusters(bs, host_offset,
nb_new_data_clusters * s->cluster_size,
QCOW2_DISCARD_OTHER);
- return ret;
+ goto fail;
}
guest_offset += nb_clusters * s->cluster_size;
@@ -3650,11 +3626,11 @@
if (prealloc != PREALLOC_MODE_OFF) {
/* Flush metadata before actually changing the image size */
- ret = bdrv_flush(bs);
+ ret = qcow2_write_caches(bs);
if (ret < 0) {
error_setg_errno(errp, -ret,
"Failed to flush the preallocated area to disk");
- return ret;
+ goto fail;
}
}
@@ -3664,11 +3640,14 @@
&offset, sizeof(uint64_t));
if (ret < 0) {
error_setg_errno(errp, -ret, "Failed to update the image size");
- return ret;
+ goto fail;
}
s->l1_vm_state_index = new_l1_size;
- return 0;
+ ret = 0;
+fail:
+ qemu_co_mutex_unlock(&s->lock);
+ return ret;
}
/* XXX: put compressed sectors first, then all the cluster aligned
@@ -3692,7 +3671,8 @@
if (cluster_offset < 0) {
return cluster_offset;
}
- return bdrv_truncate(bs->file, cluster_offset, PREALLOC_MODE_OFF, NULL);
+ return bdrv_co_truncate(bs->file, cluster_offset, PREALLOC_MODE_OFF,
+ NULL);
}
if (offset_into_cluster(s, offset)) {
@@ -4696,7 +4676,7 @@
.bdrv_co_pdiscard = qcow2_co_pdiscard,
.bdrv_co_copy_range_from = qcow2_co_copy_range_from,
.bdrv_co_copy_range_to = qcow2_co_copy_range_to,
- .bdrv_truncate = qcow2_truncate,
+ .bdrv_co_truncate = qcow2_co_truncate,
.bdrv_co_pwritev_compressed = qcow2_co_pwritev_compressed,
.bdrv_make_empty = qcow2_make_empty,
diff --git a/block/qcow2.h b/block/qcow2.h
index 01b5250..1c9c0d3 100644
--- a/block/qcow2.h
+++ b/block/qcow2.h
@@ -614,6 +614,7 @@
int compressed_size);
int qcow2_alloc_cluster_link_l2(BlockDriverState *bs, QCowL2Meta *m);
+void qcow2_alloc_cluster_abort(BlockDriverState *bs, QCowL2Meta *m);
int qcow2_cluster_discard(BlockDriverState *bs, uint64_t offset,
uint64_t bytes, enum qcow2_discard_type type,
bool full_discard);
diff --git a/block/qed.c b/block/qed.c
index 2363814..689ea9d 100644
--- a/block/qed.c
+++ b/block/qed.c
@@ -1467,8 +1467,10 @@
QED_AIOCB_WRITE | QED_AIOCB_ZERO);
}
-static int bdrv_qed_truncate(BlockDriverState *bs, int64_t offset,
- PreallocMode prealloc, Error **errp)
+static int coroutine_fn bdrv_qed_co_truncate(BlockDriverState *bs,
+ int64_t offset,
+ PreallocMode prealloc,
+ Error **errp)
{
BDRVQEDState *s = bs->opaque;
uint64_t old_image_size;
@@ -1678,7 +1680,7 @@
.bdrv_co_readv = bdrv_qed_co_readv,
.bdrv_co_writev = bdrv_qed_co_writev,
.bdrv_co_pwrite_zeroes = bdrv_qed_co_pwrite_zeroes,
- .bdrv_truncate = bdrv_qed_truncate,
+ .bdrv_co_truncate = bdrv_qed_co_truncate,
.bdrv_getlength = bdrv_qed_getlength,
.bdrv_get_info = bdrv_qed_get_info,
.bdrv_refresh_limits = bdrv_qed_refresh_limits,
diff --git a/block/raw-format.c b/block/raw-format.c
index f2e468d..b78da56 100644
--- a/block/raw-format.c
+++ b/block/raw-format.c
@@ -366,8 +366,8 @@
}
}
-static int raw_truncate(BlockDriverState *bs, int64_t offset,
- PreallocMode prealloc, Error **errp)
+static int coroutine_fn raw_co_truncate(BlockDriverState *bs, int64_t offset,
+ PreallocMode prealloc, Error **errp)
{
BDRVRawState *s = bs->opaque;
@@ -383,7 +383,7 @@
s->size = offset;
offset += s->offset;
- return bdrv_truncate(bs->file, offset, prealloc, errp);
+ return bdrv_co_truncate(bs->file, offset, prealloc, errp);
}
static void raw_eject(BlockDriverState *bs, bool eject_flag)
@@ -545,7 +545,7 @@
.bdrv_co_block_status = &raw_co_block_status,
.bdrv_co_copy_range_from = &raw_co_copy_range_from,
.bdrv_co_copy_range_to = &raw_co_copy_range_to,
- .bdrv_truncate = &raw_truncate,
+ .bdrv_co_truncate = &raw_co_truncate,
.bdrv_getlength = &raw_getlength,
.has_variable_length = true,
.bdrv_measure = &raw_measure,
diff --git a/block/rbd.c b/block/rbd.c
index f2c6965..ca8e5bb 100644
--- a/block/rbd.c
+++ b/block/rbd.c
@@ -990,8 +990,10 @@
return info.size;
}
-static int qemu_rbd_truncate(BlockDriverState *bs, int64_t offset,
- PreallocMode prealloc, Error **errp)
+static int coroutine_fn qemu_rbd_co_truncate(BlockDriverState *bs,
+ int64_t offset,
+ PreallocMode prealloc,
+ Error **errp)
{
BDRVRBDState *s = bs->opaque;
int r;
@@ -1184,7 +1186,7 @@
.bdrv_get_info = qemu_rbd_getinfo,
.create_opts = &qemu_rbd_create_opts,
.bdrv_getlength = qemu_rbd_getlength,
- .bdrv_truncate = qemu_rbd_truncate,
+ .bdrv_co_truncate = qemu_rbd_co_truncate,
.protocol_name = "rbd",
.bdrv_aio_preadv = qemu_rbd_aio_preadv,
diff --git a/block/replication.c b/block/replication.c
index 826db7b..6349d69 100644
--- a/block/replication.c
+++ b/block/replication.c
@@ -246,13 +246,14 @@
backup_cow_request_begin(&req, child->bs->job,
sector_num * BDRV_SECTOR_SIZE,
remaining_bytes);
- ret = bdrv_co_readv(bs->file, sector_num, remaining_sectors,
- qiov);
+ ret = bdrv_co_preadv(bs->file, sector_num * BDRV_SECTOR_SIZE,
+ remaining_bytes, qiov, 0);
backup_cow_request_end(&req);
goto out;
}
- ret = bdrv_co_readv(bs->file, sector_num, remaining_sectors, qiov);
+ ret = bdrv_co_preadv(bs->file, sector_num * BDRV_SECTOR_SIZE,
+ remaining_sectors * BDRV_SECTOR_SIZE, qiov, 0);
out:
return replication_return_value(s, ret);
}
@@ -279,8 +280,8 @@
}
if (ret == 0) {
- ret = bdrv_co_writev(top, sector_num,
- remaining_sectors, qiov);
+ ret = bdrv_co_pwritev(top, sector_num * BDRV_SECTOR_SIZE,
+ remaining_sectors * BDRV_SECTOR_SIZE, qiov, 0);
return replication_return_value(s, ret);
}
@@ -306,7 +307,8 @@
qemu_iovec_concat(&hd_qiov, qiov, bytes_done, count);
target = ret ? top : base;
- ret = bdrv_co_writev(target, sector_num, n, &hd_qiov);
+ ret = bdrv_co_pwritev(target, sector_num * BDRV_SECTOR_SIZE,
+ n * BDRV_SECTOR_SIZE, &hd_qiov, 0);
if (ret < 0) {
goto out1;
}
diff --git a/block/sheepdog.c b/block/sheepdog.c
index 665b176..b229a66 100644
--- a/block/sheepdog.c
+++ b/block/sheepdog.c
@@ -2292,8 +2292,8 @@
return s->inode.vdi_size;
}
-static int sd_truncate(BlockDriverState *bs, int64_t offset,
- PreallocMode prealloc, Error **errp)
+static int coroutine_fn sd_co_truncate(BlockDriverState *bs, int64_t offset,
+ PreallocMode prealloc, Error **errp)
{
BDRVSheepdogState *s = bs->opaque;
int ret, fd;
@@ -2609,7 +2609,7 @@
assert(!flags);
if (offset > s->inode.vdi_size) {
- ret = sd_truncate(bs, offset, PREALLOC_MODE_OFF, NULL);
+ ret = sd_co_truncate(bs, offset, PREALLOC_MODE_OFF, NULL);
if (ret < 0) {
return ret;
}
@@ -3231,7 +3231,7 @@
.bdrv_has_zero_init = bdrv_has_zero_init_1,
.bdrv_getlength = sd_getlength,
.bdrv_get_allocated_file_size = sd_get_allocated_file_size,
- .bdrv_truncate = sd_truncate,
+ .bdrv_co_truncate = sd_co_truncate,
.bdrv_co_readv = sd_co_readv,
.bdrv_co_writev = sd_co_writev,
@@ -3268,7 +3268,7 @@
.bdrv_has_zero_init = bdrv_has_zero_init_1,
.bdrv_getlength = sd_getlength,
.bdrv_get_allocated_file_size = sd_get_allocated_file_size,
- .bdrv_truncate = sd_truncate,
+ .bdrv_co_truncate = sd_co_truncate,
.bdrv_co_readv = sd_co_readv,
.bdrv_co_writev = sd_co_writev,
@@ -3305,7 +3305,7 @@
.bdrv_has_zero_init = bdrv_has_zero_init_1,
.bdrv_getlength = sd_getlength,
.bdrv_get_allocated_file_size = sd_get_allocated_file_size,
- .bdrv_truncate = sd_truncate,
+ .bdrv_co_truncate = sd_co_truncate,
.bdrv_co_readv = sd_co_readv,
.bdrv_co_writev = sd_co_writev,
diff --git a/block/ssh.c b/block/ssh.c
index da7bbf7..7fbc27a 100644
--- a/block/ssh.c
+++ b/block/ssh.c
@@ -1243,8 +1243,8 @@
return length;
}
-static int ssh_truncate(BlockDriverState *bs, int64_t offset,
- PreallocMode prealloc, Error **errp)
+static int coroutine_fn ssh_co_truncate(BlockDriverState *bs, int64_t offset,
+ PreallocMode prealloc, Error **errp)
{
BDRVSSHState *s = bs->opaque;
@@ -1279,7 +1279,7 @@
.bdrv_co_readv = ssh_co_readv,
.bdrv_co_writev = ssh_co_writev,
.bdrv_getlength = ssh_getlength,
- .bdrv_truncate = ssh_truncate,
+ .bdrv_co_truncate = ssh_co_truncate,
.bdrv_co_flush_to_disk = ssh_co_flush,
.create_opts = &ssh_create_opts,
};
diff --git a/block/vhdx.c b/block/vhdx.c
index a677703..4d08197 100644
--- a/block/vhdx.c
+++ b/block/vhdx.c
@@ -1127,9 +1127,9 @@
break;
case PAYLOAD_BLOCK_FULLY_PRESENT:
qemu_co_mutex_unlock(&s->lock);
- ret = bdrv_co_readv(bs->file,
- sinfo.file_offset >> BDRV_SECTOR_BITS,
- sinfo.sectors_avail, &hd_qiov);
+ ret = bdrv_co_preadv(bs->file, sinfo.file_offset,
+ sinfo.sectors_avail * BDRV_SECTOR_SIZE,
+ &hd_qiov, 0);
qemu_co_mutex_lock(&s->lock);
if (ret < 0) {
goto exit;
@@ -1349,9 +1349,9 @@
}
/* block exists, so we can just overwrite it */
qemu_co_mutex_unlock(&s->lock);
- ret = bdrv_co_writev(bs->file,
- sinfo.file_offset >> BDRV_SECTOR_BITS,
- sectors_to_write, &hd_qiov);
+ ret = bdrv_co_pwritev(bs->file, sinfo.file_offset,
+ sectors_to_write * BDRV_SECTOR_SIZE,
+ &hd_qiov, 0);
qemu_co_mutex_lock(&s->lock);
if (ret < 0) {
goto error_bat_restore;
diff --git a/chardev/char-stdio.c b/chardev/char-stdio.c
index 96375f2..9624220 100644
--- a/chardev/char-stdio.c
+++ b/chardev/char-stdio.c
@@ -46,8 +46,10 @@
static void term_exit(void)
{
- tcsetattr(0, TCSANOW, &oldtty);
- fcntl(0, F_SETFL, old_fd0_flags);
+ if (stdio_in_use) {
+ tcsetattr(0, TCSANOW, &oldtty);
+ fcntl(0, F_SETFL, old_fd0_flags);
+ }
}
static void qemu_chr_set_echo_stdio(Chardev *chr, bool echo)
diff --git a/configure b/configure
index 4d12cfb..65548df 100755
--- a/configure
+++ b/configure
@@ -456,6 +456,7 @@
vxhs=""
libxml2=""
docker="no"
+debug_mutex="no"
# cross compilers defaults, can be overridden with --cross-cc-ARCH
cross_cc_aarch64="aarch64-linux-gnu-gcc"
@@ -1041,6 +1042,7 @@
--enable-debug)
# Enable debugging options that aren't excessively noisy
debug_tcg="yes"
+ debug_mutex="yes"
debug="yes"
strip_opt="no"
fortify_source="no"
@@ -1411,6 +1413,10 @@
;;
--disable-git-update) git_update=no
;;
+ --enable-debug-mutex) debug_mutex=yes
+ ;;
+ --disable-debug-mutex) debug_mutex=no
+ ;;
*)
echo "ERROR: unknown option $opt"
echo "Try '$0 --help' for more information"
@@ -1685,6 +1691,7 @@
crypto-afalg Linux AF_ALG crypto backend driver
vhost-user vhost-user support
capstone capstone disassembler support
+ debug-mutex mutex debugging support
NOTE: The object files are built at the place where configure is launched
EOF
@@ -3433,11 +3440,7 @@
##########################################
# glib support probe
-if test "$mingw32" = yes; then
- glib_req_ver=2.30
-else
- glib_req_ver=2.22
-fi
+glib_req_ver=2.40
glib_modules=gthread-2.0
if test "$modules" = yes; then
glib_modules="$glib_modules gmodule-export-2.0"
@@ -4794,6 +4797,21 @@
fi
##########################################
+# check if we have strchrnul
+
+strchrnul=no
+cat > $TMPC << EOF
+#include <string.h>
+int main(void);
+// Use a haystack that the compiler shouldn't be able to constant fold
+char *haystack = (char*)&main;
+int main(void) { return strchrnul(haystack, 'x') != &haystack[6]; }
+EOF
+if compile_prog "" "" ; then
+ strchrnul=yes
+fi
+
+##########################################
# check if trace backend exists
$python "$source_path/scripts/tracetool.py" "--backends=$trace_backends" --check-backends > /dev/null 2> /dev/null
@@ -5951,6 +5969,7 @@
echo "coroutine backend $coroutine"
echo "coroutine pool $coroutine_pool"
echo "debug stack usage $debug_stack_usage"
+echo "mutex debugging $debug_mutex"
echo "crypto afalg $crypto_afalg"
echo "GlusterFS support $glusterfs"
echo "gcov $gcov_tool"
@@ -6276,6 +6295,9 @@
if test "$sem_timedwait" = "yes" ; then
echo "CONFIG_SEM_TIMEDWAIT=y" >> $config_host_mak
fi
+if test "$strchrnul" = "yes" ; then
+ echo "HAVE_STRCHRNUL=y" >> $config_host_mak
+fi
if test "$byteswap_h" = "yes" ; then
echo "CONFIG_BYTESWAP_H=y" >> $config_host_mak
fi
@@ -6704,6 +6726,9 @@
if test "$capstone" != "no" ; then
echo "CONFIG_CAPSTONE=y" >> $config_host_mak
fi
+if test "$debug_mutex" = "yes" ; then
+ echo "CONFIG_DEBUG_MUTEX=y" >> $config_host_mak
+fi
# Hold two types of flag:
# CONFIG_THREAD_SETNAME_BYTHREAD - we've got a way of setting the name on
diff --git a/crypto/hash-glib.c b/crypto/hash-glib.c
index a5871cc..a0096c7 100644
--- a/crypto/hash-glib.c
+++ b/crypto/hash-glib.c
@@ -30,11 +30,7 @@
[QCRYPTO_HASH_ALG_SHA224] = -1,
[QCRYPTO_HASH_ALG_SHA256] = G_CHECKSUM_SHA256,
[QCRYPTO_HASH_ALG_SHA384] = -1,
-#if GLIB_CHECK_VERSION(2, 36, 0)
[QCRYPTO_HASH_ALG_SHA512] = G_CHECKSUM_SHA512,
-#else
- [QCRYPTO_HASH_ALG_SHA512] = -1,
-#endif
[QCRYPTO_HASH_ALG_RIPEMD160] = -1,
};
diff --git a/crypto/hmac-glib.c b/crypto/hmac-glib.c
index a6c1730..7df6273 100644
--- a/crypto/hmac-glib.c
+++ b/crypto/hmac-glib.c
@@ -17,9 +17,6 @@
#include "crypto/hmac.h"
#include "hmacpriv.h"
-/* Support for HMAC Algos has been added in GLib 2.30 */
-#if GLIB_CHECK_VERSION(2, 30, 0)
-
static int qcrypto_hmac_alg_map[QCRYPTO_HASH_ALG__MAX] = {
[QCRYPTO_HASH_ALG_MD5] = G_CHECKSUM_MD5,
[QCRYPTO_HASH_ALG_SHA1] = G_CHECKSUM_SHA1,
@@ -126,39 +123,6 @@
return 0;
}
-#else
-
-bool qcrypto_hmac_supports(QCryptoHashAlgorithm alg)
-{
- return false;
-}
-
-void *qcrypto_hmac_ctx_new(QCryptoHashAlgorithm alg,
- const uint8_t *key, size_t nkey,
- Error **errp)
-{
- return NULL;
-}
-
-static void
-qcrypto_glib_hmac_ctx_free(QCryptoHmac *hmac)
-{
- return;
-}
-
-static int
-qcrypto_glib_hmac_bytesv(QCryptoHmac *hmac,
- const struct iovec *iov,
- size_t niov,
- uint8_t **result,
- size_t *resultlen,
- Error **errp)
-{
- return -1;
-}
-
-#endif
-
QCryptoHmacDriver qcrypto_hmac_lib_driver = {
.hmac_bytesv = qcrypto_glib_hmac_bytesv,
.hmac_free = qcrypto_glib_hmac_ctx_free,
diff --git a/device_tree.c b/device_tree.c
index 52c3358..6d9c972 100644
--- a/device_tree.c
+++ b/device_tree.c
@@ -140,15 +140,16 @@
const char *parent_node;
if (strstr(dirname, root_dir) != dirname) {
- error_setg(&error_fatal, "%s: %s must be searched within %s",
- __func__, dirname, root_dir);
+ error_report("%s: %s must be searched within %s",
+ __func__, dirname, root_dir);
+ exit(1);
}
parent_node = &dirname[strlen(SYSFS_DT_BASEDIR)];
d = opendir(dirname);
if (!d) {
- error_setg(&error_fatal, "%s cannot open %s", __func__, dirname);
- return;
+ error_report("%s cannot open %s", __func__, dirname);
+ exit(1);
}
while ((de = readdir(d)) != NULL) {
@@ -162,7 +163,8 @@
tmpnam = g_strdup_printf("%s/%s", dirname, de->d_name);
if (lstat(tmpnam, &st) < 0) {
- error_setg(&error_fatal, "%s cannot lstat %s", __func__, tmpnam);
+ error_report("%s cannot lstat %s", __func__, tmpnam);
+ exit(1);
}
if (S_ISREG(st.st_mode)) {
@@ -170,8 +172,9 @@
gsize len;
if (!g_file_get_contents(tmpnam, &val, &len, NULL)) {
- error_setg(&error_fatal, "%s not able to extract info from %s",
- __func__, tmpnam);
+ error_report("%s not able to extract info from %s",
+ __func__, tmpnam);
+ exit(1);
}
if (strlen(parent_node) > 0) {
@@ -206,9 +209,9 @@
host_fdt = create_device_tree(&host_fdt_size);
read_fstree(host_fdt, SYSFS_DT_BASEDIR);
if (fdt_check_header(host_fdt)) {
- error_setg(&error_fatal,
- "%s host device tree extracted into memory is invalid",
- __func__);
+ error_report("%s host device tree extracted into memory is invalid",
+ __func__);
+ exit(1);
}
return host_fdt;
}
@@ -229,6 +232,61 @@
return offset;
}
+char **qemu_fdt_node_unit_path(void *fdt, const char *name, Error **errp)
+{
+ char *prefix = g_strdup_printf("%s@", name);
+ unsigned int path_len = 16, n = 0;
+ GSList *path_list = NULL, *iter;
+ const char *iter_name;
+ int offset, len, ret;
+ char **path_array;
+
+ offset = fdt_next_node(fdt, -1, NULL);
+
+ while (offset >= 0) {
+ iter_name = fdt_get_name(fdt, offset, &len);
+ if (!iter_name) {
+ offset = len;
+ break;
+ }
+ if (!strcmp(iter_name, name) || g_str_has_prefix(iter_name, prefix)) {
+ char *path;
+
+ path = g_malloc(path_len);
+ while ((ret = fdt_get_path(fdt, offset, path, path_len))
+ == -FDT_ERR_NOSPACE) {
+ path_len += 16;
+ path = g_realloc(path, path_len);
+ }
+ path_list = g_slist_prepend(path_list, path);
+ n++;
+ }
+ offset = fdt_next_node(fdt, offset, NULL);
+ }
+ g_free(prefix);
+
+ if (offset < 0 && offset != -FDT_ERR_NOTFOUND) {
+ error_setg(errp, "%s: abort parsing dt for %s node units: %s",
+ __func__, name, fdt_strerror(offset));
+ for (iter = path_list; iter; iter = iter->next) {
+ g_free(iter->data);
+ }
+ g_slist_free(path_list);
+ return NULL;
+ }
+
+ path_array = g_new(char *, n + 1);
+ path_array[n--] = NULL;
+
+ for (iter = path_list; iter; iter = iter->next) {
+ path_array[n--] = iter->data;
+ }
+
+ g_slist_free(path_list);
+
+ return path_array;
+}
+
char **qemu_fdt_node_path(void *fdt, const char *name, char *compat,
Error **errp)
{
diff --git a/docs/COLO-FT.txt b/docs/COLO-FT.txt
index e289be2..d7c7dcd 100644
--- a/docs/COLO-FT.txt
+++ b/docs/COLO-FT.txt
@@ -113,16 +113,16 @@
== Test procedure ==
1. Startup qemu
Primary:
-# qemu-kvm -enable-kvm -m 2048 -smp 2 -qmp stdio -vnc :7 -name primary \
- -device piix3-usb-uhci \
+# qemu-system-x86_64 -accel kvm -m 2048 -smp 2 -qmp stdio -name primary \
+ -device piix3-usb-uhci -vnc :7 \
-device usb-tablet -netdev tap,id=hn0,vhost=off \
-device virtio-net-pci,id=net-pci0,netdev=hn0 \
-drive if=virtio,id=primary-disk0,driver=quorum,read-pattern=fifo,vote-threshold=1,\
children.0.file.filename=1.raw,\
children.0.driver=raw -S
Secondary:
-# qemu-kvm -enable-kvm -m 2048 -smp 2 -qmp stdio -vnc :7 -name secondary \
- -device piix3-usb-uhci \
+# qemu-system-x86_64 -accel kvm -m 2048 -smp 2 -qmp stdio -name secondary \
+ -device piix3-usb-uhci -vnc :7 \
-device usb-tablet -netdev tap,id=hn0,vhost=off \
-device virtio-net-pci,id=net-pci0,netdev=hn0 \
-drive if=none,id=secondary-disk0,file.filename=1.raw,driver=raw,node-name=node0 \
diff --git a/docs/can.txt b/docs/can.txt
index a357105..7ba23b2 100644
--- a/docs/can.txt
+++ b/docs/can.txt
@@ -52,7 +52,7 @@
The tested setup was Linux 4.9 kernel on the host and guest side.
Example for qemu-system-x86_64:
- qemu-system-x86_64 -enable-kvm -kernel /boot/vmlinuz-4.9.0-4-amd64 \
+ qemu-system-x86_64 -accel kvm -kernel /boot/vmlinuz-4.9.0-4-amd64 \
-initrd ramdisk.cpio \
-virtfs local,path=shareddir,security_model=none,mount_tag=shareddir \
-object can-bus,id=canbus0 \
@@ -104,4 +104,4 @@
Slides
http://rtime.felk.cvut.cz/publications/public/rtlws2015-qemu-can-slides.pdf
(5) Linux SocketCAN utilities
- https://github.com/linux-can/can-utils/
\ No newline at end of file
+ https://github.com/linux-can/can-utils/
diff --git a/docs/multi-thread-compression.txt b/docs/multi-thread-compression.txt
index d0caaf7..bb88c6b 100644
--- a/docs/multi-thread-compression.txt
+++ b/docs/multi-thread-compression.txt
@@ -62,7 +62,7 @@
NIC: Intel I350 (10/100/1000Mbps)
Host OS: CentOS 7 64-bit
Guest OS: RHEL 6.5 64-bit
-Parameter: qemu-system-x86_64 -enable-kvm -smp 4 -m 4096
+Parameter: qemu-system-x86_64 -accel kvm -smp 4 -m 4096
/share/ia32e_rhel6u5.qcow -monitor stdio
There is no additional application is running on the guest when doing
diff --git a/docs/multiseat.txt b/docs/multiseat.txt
index 807518c..dc28cdb 100644
--- a/docs/multiseat.txt
+++ b/docs/multiseat.txt
@@ -18,7 +18,7 @@
Next put together the qemu command line (sdk/gtk):
-qemu -enable-kvm -usb $memory $disk $whatever \
+qemu -accel kvm -usb $memory $disk $whatever \
-display [ sdl | gtk ] \
-vga std \
-device usb-tablet
diff --git a/docs/specs/tpm.txt b/docs/specs/tpm.txt
index c230c4c..70ad4a0 100644
--- a/docs/specs/tpm.txt
+++ b/docs/specs/tpm.txt
@@ -98,7 +98,7 @@
Command line to start QEMU with the TPM passthrough device using the host's
hardware TPM /dev/tpm0:
-qemu-system-x86_64 -display sdl -enable-kvm \
+qemu-system-x86_64 -display sdl -accel kvm \
-m 1024 -boot d -bios bios-256k.bin -boot menu=on \
-tpmdev passthrough,id=tpm0,path=/dev/tpm0 \
-device tpm-tis,tpmdev=tpm0 test.img
@@ -164,7 +164,7 @@
Command line to start QEMU with the TPM emulator device communicating with
the swtpm:
-qemu-system-x86_64 -display sdl -enable-kvm \
+qemu-system-x86_64 -display sdl -accel kvm \
-m 1024 -boot d -bios bios-256k.bin -boot menu=on \
-chardev socket,id=chrtpm,path=/tmp/mytpm1/swtpm-sock \
-tpmdev emulator,id=tpm0,chardev=chrtpm \
@@ -222,7 +222,7 @@
In a 2nd terminal start the VM:
-qemu-system-x86_64 -display sdl -enable-kvm \
+qemu-system-x86_64 -display sdl -accel kvm \
-m 1024 -boot d -bios bios-256k.bin -boot menu=on \
-chardev socket,id=chrtpm,path=/tmp/mytpm1/swtpm-sock \
-tpmdev emulator,id=tpm0,chardev=chrtpm \
@@ -255,7 +255,7 @@
In the 2nd terminal restore the state of the VM using the additonal
'-incoming' option.
-qemu-system-x86_64 -display sdl -enable-kvm \
+qemu-system-x86_64 -display sdl -accel kvm \
-m 1024 -boot d -bios bios-256k.bin -boot menu=on \
-chardev socket,id=chrtpm,path=/tmp/mytpm1/swtpm-sock \
-tpmdev emulator,id=tpm0,chardev=chrtpm \
diff --git a/dump.c b/dump.c
index b54cd42..04467b3 100644
--- a/dump.c
+++ b/dump.c
@@ -29,6 +29,10 @@
#include "qemu/error-report.h"
#include "hw/misc/vmcoreinfo.h"
+#ifdef TARGET_X86_64
+#include "win_dump.h"
+#endif
+
#include <zlib.h>
#ifdef CONFIG_LZO
#include <lzo/lzo1x.h>
@@ -1866,7 +1870,11 @@
Error *local_err = NULL;
DumpQueryResult *result = NULL;
- if (s->has_format && s->format != DUMP_GUEST_MEMORY_FORMAT_ELF) {
+ if (s->has_format && s->format == DUMP_GUEST_MEMORY_FORMAT_WIN_DMP) {
+#ifdef TARGET_X86_64
+ create_win_dump(s, &local_err);
+#endif
+ } else if (s->has_format && s->format != DUMP_GUEST_MEMORY_FORMAT_ELF) {
create_kdump_vmcore(s, &local_err);
} else {
create_vmcore(s, &local_err);
@@ -1970,6 +1978,13 @@
}
#endif
+#ifndef TARGET_X86_64
+ if (has_format && format == DUMP_GUEST_MEMORY_FORMAT_WIN_DMP) {
+ error_setg(errp, "Windows dump is only available for x86-64");
+ return;
+ }
+#endif
+
#if !defined(WIN32)
if (strstart(file, "fd:", &p)) {
fd = monitor_get_fd(cur_mon, p, errp);
@@ -2044,5 +2059,12 @@
item->value = DUMP_GUEST_MEMORY_FORMAT_KDUMP_SNAPPY;
#endif
+ /* Windows dump is available only if target is x86_64 */
+#ifdef TARGET_X86_64
+ item->next = g_malloc0(sizeof(DumpGuestMemoryFormatList));
+ item = item->next;
+ item->value = DUMP_GUEST_MEMORY_FORMAT_WIN_DMP;
+#endif
+
return cap;
}
diff --git a/exec.c b/exec.c
index 1e37f75..cdcf769 100644
--- a/exec.c
+++ b/exec.c
@@ -1028,13 +1028,36 @@
}
#if defined(CONFIG_USER_ONLY)
-static void breakpoint_invalidate(CPUState *cpu, target_ulong pc)
+void tb_invalidate_phys_addr(target_ulong addr)
{
mmap_lock();
- tb_invalidate_phys_page_range(pc, pc + 1, 0);
+ tb_invalidate_phys_page_range(addr, addr + 1, 0);
mmap_unlock();
}
+
+static void breakpoint_invalidate(CPUState *cpu, target_ulong pc)
+{
+ tb_invalidate_phys_addr(pc);
+}
#else
+void tb_invalidate_phys_addr(AddressSpace *as, hwaddr addr, MemTxAttrs attrs)
+{
+ ram_addr_t ram_addr;
+ MemoryRegion *mr;
+ hwaddr l = 1;
+
+ rcu_read_lock();
+ mr = address_space_translate(as, addr, &addr, &l, false, attrs);
+ if (!(memory_region_is_ram(mr)
+ || memory_region_is_romd(mr))) {
+ rcu_read_unlock();
+ return;
+ }
+ ram_addr = memory_region_get_ram_addr(mr) + addr;
+ tb_invalidate_phys_page_range(ram_addr, ram_addr + 1, 0);
+ rcu_read_unlock();
+}
+
static void breakpoint_invalidate(CPUState *cpu, target_ulong pc)
{
MemTxAttrs attrs;
@@ -1818,6 +1841,10 @@
" must be multiples of page size 0x%zx",
block->mr->align, block->page_size);
return NULL;
+ } else if (block->mr->align && !is_power_of_2(block->mr->align)) {
+ error_setg(errp, "alignment 0x%" PRIx64
+ " must be a power of two", block->mr->align);
+ return NULL;
}
block->mr->align = MAX(block->page_size, block->mr->align);
#if defined(__s390x__)
@@ -3146,9 +3173,7 @@
}
if (dirty_log_mask & (1 << DIRTY_MEMORY_CODE)) {
assert(tcg_enabled());
- mmap_lock();
tb_invalidate_phys_range(addr, addr + length);
- mmap_unlock();
dirty_log_mask &= ~(1 << DIRTY_MEMORY_CODE);
}
cpu_physical_memory_set_dirty_range(addr, length, dirty_log_mask);
@@ -3702,9 +3727,6 @@
#define ARG1 as
#define SUFFIX
#define TRANSLATE(...) address_space_translate(as, __VA_ARGS__)
-#define IS_DIRECT(mr, is_write) memory_access_is_direct(mr, is_write)
-#define MAP_RAM(mr, ofs) qemu_map_ram_ptr((mr)->ram_block, ofs)
-#define INVALIDATE(mr, ofs, len) invalidate_and_set_dirty(mr, ofs, len)
#define RCU_READ_LOCK(...) rcu_read_lock()
#define RCU_READ_UNLOCK(...) rcu_read_unlock()
#include "memory_ldst.inc.c"
@@ -3841,9 +3863,6 @@
#define ARG1 cache
#define SUFFIX _cached_slow
#define TRANSLATE(...) address_space_translate_cached(cache, __VA_ARGS__)
-#define IS_DIRECT(mr, is_write) memory_access_is_direct(mr, is_write)
-#define MAP_RAM(mr, ofs) (cache->ptr + (ofs - cache->xlat))
-#define INVALIDATE(mr, ofs, len) invalidate_and_set_dirty(mr, ofs, len)
#define RCU_READ_LOCK() ((void)0)
#define RCU_READ_UNLOCK() ((void)0)
#include "memory_ldst.inc.c"
diff --git a/hmp-commands-info.hx b/hmp-commands-info.hx
index 6db3457..a482b6e 100644
--- a/hmp-commands-info.hx
+++ b/hmp-commands-info.hx
@@ -201,7 +201,7 @@
STEXI
@item info pic
@findex info pic
-Show i8259 (PIC) state.
+Show PIC state.
ETEXI
{
@@ -253,10 +253,11 @@
{
.name = "mtree",
- .args_type = "flatview:-f,dispatch_tree:-d",
- .params = "[-f][-d]",
+ .args_type = "flatview:-f,dispatch_tree:-d,owner:-o",
+ .params = "[-f][-d][-o]",
.help = "show memory tree (-f: dump flat view for address spaces;"
- "-d: dump dispatch tree, valid with -f only)",
+ "-d: dump dispatch tree, valid with -f only);"
+ "-o: dump region owners/parents",
.cmd = hmp_info_mtree,
},
diff --git a/hmp-commands.hx b/hmp-commands.hx
index ba9cdb8..c1fc747 100644
--- a/hmp-commands.hx
+++ b/hmp-commands.hx
@@ -1136,30 +1136,33 @@
{
.name = "dump-guest-memory",
- .args_type = "paging:-p,detach:-d,zlib:-z,lzo:-l,snappy:-s,filename:F,begin:l?,length:l?",
- .params = "[-p] [-d] [-z|-l|-s] filename [begin length]",
+ .args_type = "paging:-p,detach:-d,windmp:-w,zlib:-z,lzo:-l,snappy:-s,filename:F,begin:l?,length:l?",
+ .params = "[-p] [-d] [-z|-l|-s|-w] filename [begin length]",
.help = "dump guest memory into file 'filename'.\n\t\t\t"
"-p: do paging to get guest's memory mapping.\n\t\t\t"
"-d: return immediately (do not wait for completion).\n\t\t\t"
"-z: dump in kdump-compressed format, with zlib compression.\n\t\t\t"
"-l: dump in kdump-compressed format, with lzo compression.\n\t\t\t"
"-s: dump in kdump-compressed format, with snappy compression.\n\t\t\t"
+ "-w: dump in Windows crashdump format (can be used instead of ELF-dump converting),\n\t\t\t"
+ " for Windows x64 guests with vmcoreinfo driver only.\n\t\t\t"
"begin: the starting physical address.\n\t\t\t"
"length: the memory size, in bytes.",
.cmd = hmp_dump_guest_memory,
},
-
STEXI
@item dump-guest-memory [-p] @var{filename} @var{begin} @var{length}
-@item dump-guest-memory [-z|-l|-s] @var{filename}
+@item dump-guest-memory [-z|-l|-s|-w] @var{filename}
@findex dump-guest-memory
Dump guest memory to @var{protocol}. The file can be processed with crash or
-gdb. Without -z|-l|-s, the dump format is ELF.
+gdb. Without -z|-l|-s|-w, the dump format is ELF.
-p: do paging to get guest's memory mapping.
-z: dump in kdump-compressed format, with zlib compression.
-l: dump in kdump-compressed format, with lzo compression.
-s: dump in kdump-compressed format, with snappy compression.
+ -w: dump in Windows crashdump format (can be used instead of ELF-dump converting),
+ for Windows x64 guests with vmcoreinfo driver only
filename: dump file name.
begin: the starting physical address. It's optional, and should be
specified together with length.
diff --git a/hmp.c b/hmp.c
index 0da0b0a..fe4477a 100644
--- a/hmp.c
+++ b/hmp.c
@@ -2014,6 +2014,7 @@
void hmp_dump_guest_memory(Monitor *mon, const QDict *qdict)
{
Error *err = NULL;
+ bool win_dmp = qdict_get_try_bool(qdict, "windmp", false);
bool paging = qdict_get_try_bool(qdict, "paging", false);
bool zlib = qdict_get_try_bool(qdict, "zlib", false);
bool lzo = qdict_get_try_bool(qdict, "lzo", false);
@@ -2028,12 +2029,16 @@
enum DumpGuestMemoryFormat dump_format = DUMP_GUEST_MEMORY_FORMAT_ELF;
char *prot;
- if (zlib + lzo + snappy > 1) {
- error_setg(&err, "only one of '-z|-l|-s' can be set");
+ if (zlib + lzo + snappy + win_dmp > 1) {
+ error_setg(&err, "only one of '-z|-l|-s|-w' can be set");
hmp_handle_error(mon, &err);
return;
}
+ if (win_dmp) {
+ dump_format = DUMP_GUEST_MEMORY_FORMAT_WIN_DMP;
+ }
+
if (zlib) {
dump_format = DUMP_GUEST_MEMORY_FORMAT_KDUMP_ZLIB;
}
@@ -2140,12 +2145,12 @@
int has_hold_time = qdict_haskey(qdict, "hold-time");
int hold_time = qdict_get_try_int(qdict, "hold-time", -1);
Error *err = NULL;
- char *separator;
+ const char *separator;
int keyname_len;
while (1) {
- separator = strchr(keys, '-');
- keyname_len = separator ? separator - keys : strlen(keys);
+ separator = qemu_strchrnul(keys, '-');
+ keyname_len = separator - keys;
/* Be compatible with old interface, convert user inputted "<" */
if (keys[0] == '<' && keyname_len == 1) {
@@ -2182,7 +2187,7 @@
keylist->value->u.qcode.data = idx;
}
- if (!separator) {
+ if (!*separator) {
break;
}
keys = separator + 1;
diff --git a/hw/9pfs/9p-local.c b/hw/9pfs/9p-local.c
index 5721eff..c30f4f2 100644
--- a/hw/9pfs/9p-local.c
+++ b/hw/9pfs/9p-local.c
@@ -65,7 +65,7 @@
assert(*path != '/');
head = g_strdup(path);
- c = strchrnul(path, '/');
+ c = qemu_strchrnul(path, '/');
if (*c) {
/* Intermediate path element */
head[c - path] = 0;
@@ -308,7 +308,7 @@
if (credp->fc_gid != -1) {
gid = credp->fc_gid;
}
- if (credp->fc_mode != -1) {
+ if (credp->fc_mode != (mode_t)-1) {
mode = credp->fc_mode;
}
if (credp->fc_rdev != -1) {
@@ -414,7 +414,7 @@
return err;
}
}
- if (credp->fc_mode != -1) {
+ if (credp->fc_mode != (mode_t)-1) {
uint32_t tmp_mode = cpu_to_le32(credp->fc_mode);
err = fsetxattrat_nofollow(dirfd, path, "user.virtfs.mode", &tmp_mode,
sizeof(mode_t), 0);
diff --git a/hw/arm/boot.c b/hw/arm/boot.c
index 1e48166..e09201c 100644
--- a/hw/arm/boot.c
+++ b/hw/arm/boot.c
@@ -490,11 +490,13 @@
hwaddr addr_limit, AddressSpace *as)
{
void *fdt = NULL;
- int size, rc;
+ int size, rc, n = 0;
uint32_t acells, scells;
char *nodename;
unsigned int i;
hwaddr mem_base, mem_len;
+ char **node_path;
+ Error *err = NULL;
if (binfo->dtb_filename) {
char *filename;
@@ -546,12 +548,21 @@
goto fail;
}
+ /* nop all root nodes matching /memory or /memory@unit-address */
+ node_path = qemu_fdt_node_unit_path(fdt, "memory", &err);
+ if (err) {
+ error_report_err(err);
+ goto fail;
+ }
+ while (node_path[n]) {
+ if (g_str_has_prefix(node_path[n], "/memory")) {
+ qemu_fdt_nop_node(fdt, node_path[n]);
+ }
+ n++;
+ }
+ g_strfreev(node_path);
+
if (nb_numa_nodes > 0) {
- /*
- * Turn the /memory node created before into a NOP node, then create
- * /memory@addr nodes for all numa nodes respectively.
- */
- qemu_fdt_nop_node(fdt, "/memory");
mem_base = binfo->loader_start;
for (i = 0; i < nb_numa_nodes; i++) {
mem_len = numa_info[i].node_mem;
@@ -572,24 +583,18 @@
g_free(nodename);
}
} else {
- Error *err = NULL;
+ nodename = g_strdup_printf("/memory@%" PRIx64, binfo->loader_start);
+ qemu_fdt_add_subnode(fdt, nodename);
+ qemu_fdt_setprop_string(fdt, nodename, "device_type", "memory");
- rc = fdt_path_offset(fdt, "/memory");
- if (rc < 0) {
- qemu_fdt_add_subnode(fdt, "/memory");
- }
-
- if (!qemu_fdt_getprop(fdt, "/memory", "device_type", NULL, &err)) {
- qemu_fdt_setprop_string(fdt, "/memory", "device_type", "memory");
- }
-
- rc = qemu_fdt_setprop_sized_cells(fdt, "/memory", "reg",
+ rc = qemu_fdt_setprop_sized_cells(fdt, nodename, "reg",
acells, binfo->loader_start,
scells, binfo->ram_size);
if (rc < 0) {
- fprintf(stderr, "couldn't set /memory/reg\n");
+ fprintf(stderr, "couldn't set %s reg\n", nodename);
goto fail;
}
+ g_free(nodename);
}
rc = fdt_path_offset(fdt, "/chosen");
diff --git a/hw/arm/fsl-imx7.c b/hw/arm/fsl-imx7.c
index 26c1d27..44fde03 100644
--- a/hw/arm/fsl-imx7.c
+++ b/hw/arm/fsl-imx7.c
@@ -324,7 +324,7 @@
FSL_IMX7_ECSPI4_ADDR,
};
- static const hwaddr FSL_IMX7_SPIn_IRQ[FSL_IMX7_NUM_ECSPIS] = {
+ static const int FSL_IMX7_SPIn_IRQ[FSL_IMX7_NUM_ECSPIS] = {
FSL_IMX7_ECSPI1_IRQ,
FSL_IMX7_ECSPI2_IRQ,
FSL_IMX7_ECSPI3_IRQ,
@@ -349,7 +349,7 @@
FSL_IMX7_I2C4_ADDR,
};
- static const hwaddr FSL_IMX7_I2Cn_IRQ[FSL_IMX7_NUM_I2CS] = {
+ static const int FSL_IMX7_I2Cn_IRQ[FSL_IMX7_NUM_I2CS] = {
FSL_IMX7_I2C1_IRQ,
FSL_IMX7_I2C2_IRQ,
FSL_IMX7_I2C3_IRQ,
@@ -459,7 +459,7 @@
/*
* SRC
*/
- create_unimplemented_device("sdma", FSL_IMX7_SRC_ADDR, FSL_IMX7_SRC_SIZE);
+ create_unimplemented_device("src", FSL_IMX7_SRC_ADDR, FSL_IMX7_SRC_SIZE);
/*
* Watchdog
@@ -515,7 +515,7 @@
FSL_IMX7_USB3_ADDR,
};
- static const hwaddr FSL_IMX7_USBn_IRQ[FSL_IMX7_NUM_USBS] = {
+ static const int FSL_IMX7_USBn_IRQ[FSL_IMX7_NUM_USBS] = {
FSL_IMX7_USB1_IRQ,
FSL_IMX7_USB2_IRQ,
FSL_IMX7_USB3_IRQ,
diff --git a/hw/arm/mcimx7d-sabre.c b/hw/arm/mcimx7d-sabre.c
index 95fb409..9c5f0e7 100644
--- a/hw/arm/mcimx7d-sabre.c
+++ b/hw/arm/mcimx7d-sabre.c
@@ -18,10 +18,8 @@
#include "hw/arm/fsl-imx7.h"
#include "hw/boards.h"
#include "sysemu/sysemu.h"
-#include "sysemu/device_tree.h"
#include "qemu/error-report.h"
#include "sysemu/qtest.h"
-#include "net/net.h"
typedef struct {
FslIMX7State soc;
diff --git a/hw/arm/sysbus-fdt.c b/hw/arm/sysbus-fdt.c
index 277ed87..0d4c757 100644
--- a/hw/arm/sysbus-fdt.c
+++ b/hw/arm/sysbus-fdt.c
@@ -92,16 +92,20 @@
r = qemu_fdt_getprop(host_fdt, node_path,
props[i].name,
&prop_len,
- props[i].optional ? &err : &error_fatal);
+ &err);
if (r) {
qemu_fdt_setprop(guest_fdt, nodename,
props[i].name, r, prop_len);
} else {
- if (prop_len != -FDT_ERR_NOTFOUND) {
- /* optional property not returned although property exists */
- error_report_err(err);
- } else {
+ if (props[i].optional && prop_len == -FDT_ERR_NOTFOUND) {
+ /* optional property does not exist */
error_free(err);
+ } else {
+ error_report_err(err);
+ }
+ if (!props[i].optional) {
+ /* mandatory property not found: bail out */
+ exit(1);
}
}
}
@@ -138,9 +142,9 @@
node_offset = fdt_node_offset_by_phandle(host_fdt, host_phandle);
if (node_offset <= 0) {
- error_setg(&error_fatal,
- "not able to locate clock handle %d in host device tree",
- host_phandle);
+ error_report("not able to locate clock handle %d in host device tree",
+ host_phandle);
+ exit(1);
}
node_path = g_malloc(path_len);
while ((ret = fdt_get_path(host_fdt, node_offset, node_path, path_len))
@@ -149,16 +153,16 @@
node_path = g_realloc(node_path, path_len);
}
if (ret < 0) {
- error_setg(&error_fatal,
- "not able to retrieve node path for clock handle %d",
- host_phandle);
+ error_report("not able to retrieve node path for clock handle %d",
+ host_phandle);
+ exit(1);
}
r = qemu_fdt_getprop(host_fdt, node_path, "compatible", &prop_len,
&error_fatal);
if (strcmp(r, "fixed-clock")) {
- error_setg(&error_fatal,
- "clock handle %d is not a fixed clock", host_phandle);
+ error_report("clock handle %d is not a fixed clock", host_phandle);
+ exit(1);
}
nodename = strrchr(node_path, '/');
@@ -301,34 +305,37 @@
dt_name = sysfs_to_dt_name(vbasedev->name);
if (!dt_name) {
- error_setg(&error_fatal, "%s incorrect sysfs device name %s",
- __func__, vbasedev->name);
+ error_report("%s incorrect sysfs device name %s",
+ __func__, vbasedev->name);
+ exit(1);
}
node_path = qemu_fdt_node_path(host_fdt, dt_name, vdev->compat,
&error_fatal);
if (!node_path || !node_path[0]) {
- error_setg(&error_fatal, "%s unable to retrieve node path for %s/%s",
- __func__, dt_name, vdev->compat);
+ error_report("%s unable to retrieve node path for %s/%s",
+ __func__, dt_name, vdev->compat);
+ exit(1);
}
if (node_path[1]) {
- error_setg(&error_fatal, "%s more than one node matching %s/%s!",
- __func__, dt_name, vdev->compat);
+ error_report("%s more than one node matching %s/%s!",
+ __func__, dt_name, vdev->compat);
+ exit(1);
}
g_free(dt_name);
if (vbasedev->num_regions != 5) {
- error_setg(&error_fatal, "%s Does the host dt node combine XGBE/PHY?",
- __func__);
+ error_report("%s Does the host dt node combine XGBE/PHY?", __func__);
+ exit(1);
}
/* generate nodes for DMA_CLK and PTP_CLK */
r = qemu_fdt_getprop(host_fdt, node_path[0], "clocks",
&prop_len, &error_fatal);
if (prop_len != 8) {
- error_setg(&error_fatal, "%s clocks property should contain 2 handles",
- __func__);
+ error_report("%s clocks property should contain 2 handles", __func__);
+ exit(1);
}
host_clock_phandles = (uint32_t *)r;
guest_clock_phandles[0] = qemu_fdt_alloc_phandle(guest_fdt);
diff --git a/hw/arm/virt.c b/hw/arm/virt.c
index 742f68a..281ddcd 100644
--- a/hw/arm/virt.c
+++ b/hw/arm/virt.c
@@ -204,13 +204,8 @@
qemu_fdt_setprop_cell(fdt, "/", "#address-cells", 0x2);
qemu_fdt_setprop_cell(fdt, "/", "#size-cells", 0x2);
- /*
- * /chosen and /memory nodes must exist for load_dtb
- * to fill in necessary properties later
- */
+ /* /chosen must exist for load_dtb to fill in necessary properties later */
qemu_fdt_add_subnode(fdt, "/chosen");
- qemu_fdt_add_subnode(fdt, "/memory");
- qemu_fdt_setprop_string(fdt, "/memory", "device_type", "memory");
/* Clock node, for the benefit of the UART. The kernel device tree
* binding documentation claims the PL011 node clock properties are
@@ -369,58 +364,72 @@
static void fdt_add_its_gic_node(VirtMachineState *vms)
{
+ char *nodename;
+
vms->msi_phandle = qemu_fdt_alloc_phandle(vms->fdt);
- qemu_fdt_add_subnode(vms->fdt, "/intc/its");
- qemu_fdt_setprop_string(vms->fdt, "/intc/its", "compatible",
+ nodename = g_strdup_printf("/intc/its@%" PRIx64,
+ vms->memmap[VIRT_GIC_ITS].base);
+ qemu_fdt_add_subnode(vms->fdt, nodename);
+ qemu_fdt_setprop_string(vms->fdt, nodename, "compatible",
"arm,gic-v3-its");
- qemu_fdt_setprop(vms->fdt, "/intc/its", "msi-controller", NULL, 0);
- qemu_fdt_setprop_sized_cells(vms->fdt, "/intc/its", "reg",
+ qemu_fdt_setprop(vms->fdt, nodename, "msi-controller", NULL, 0);
+ qemu_fdt_setprop_sized_cells(vms->fdt, nodename, "reg",
2, vms->memmap[VIRT_GIC_ITS].base,
2, vms->memmap[VIRT_GIC_ITS].size);
- qemu_fdt_setprop_cell(vms->fdt, "/intc/its", "phandle", vms->msi_phandle);
+ qemu_fdt_setprop_cell(vms->fdt, nodename, "phandle", vms->msi_phandle);
+ g_free(nodename);
}
static void fdt_add_v2m_gic_node(VirtMachineState *vms)
{
+ char *nodename;
+
+ nodename = g_strdup_printf("/intc/v2m@%" PRIx64,
+ vms->memmap[VIRT_GIC_V2M].base);
vms->msi_phandle = qemu_fdt_alloc_phandle(vms->fdt);
- qemu_fdt_add_subnode(vms->fdt, "/intc/v2m");
- qemu_fdt_setprop_string(vms->fdt, "/intc/v2m", "compatible",
+ qemu_fdt_add_subnode(vms->fdt, nodename);
+ qemu_fdt_setprop_string(vms->fdt, nodename, "compatible",
"arm,gic-v2m-frame");
- qemu_fdt_setprop(vms->fdt, "/intc/v2m", "msi-controller", NULL, 0);
- qemu_fdt_setprop_sized_cells(vms->fdt, "/intc/v2m", "reg",
+ qemu_fdt_setprop(vms->fdt, nodename, "msi-controller", NULL, 0);
+ qemu_fdt_setprop_sized_cells(vms->fdt, nodename, "reg",
2, vms->memmap[VIRT_GIC_V2M].base,
2, vms->memmap[VIRT_GIC_V2M].size);
- qemu_fdt_setprop_cell(vms->fdt, "/intc/v2m", "phandle", vms->msi_phandle);
+ qemu_fdt_setprop_cell(vms->fdt, nodename, "phandle", vms->msi_phandle);
+ g_free(nodename);
}
static void fdt_add_gic_node(VirtMachineState *vms)
{
+ char *nodename;
+
vms->gic_phandle = qemu_fdt_alloc_phandle(vms->fdt);
qemu_fdt_setprop_cell(vms->fdt, "/", "interrupt-parent", vms->gic_phandle);
- qemu_fdt_add_subnode(vms->fdt, "/intc");
- qemu_fdt_setprop_cell(vms->fdt, "/intc", "#interrupt-cells", 3);
- qemu_fdt_setprop(vms->fdt, "/intc", "interrupt-controller", NULL, 0);
- qemu_fdt_setprop_cell(vms->fdt, "/intc", "#address-cells", 0x2);
- qemu_fdt_setprop_cell(vms->fdt, "/intc", "#size-cells", 0x2);
- qemu_fdt_setprop(vms->fdt, "/intc", "ranges", NULL, 0);
+ nodename = g_strdup_printf("/intc@%" PRIx64,
+ vms->memmap[VIRT_GIC_DIST].base);
+ qemu_fdt_add_subnode(vms->fdt, nodename);
+ qemu_fdt_setprop_cell(vms->fdt, nodename, "#interrupt-cells", 3);
+ qemu_fdt_setprop(vms->fdt, nodename, "interrupt-controller", NULL, 0);
+ qemu_fdt_setprop_cell(vms->fdt, nodename, "#address-cells", 0x2);
+ qemu_fdt_setprop_cell(vms->fdt, nodename, "#size-cells", 0x2);
+ qemu_fdt_setprop(vms->fdt, nodename, "ranges", NULL, 0);
if (vms->gic_version == 3) {
int nb_redist_regions = virt_gicv3_redist_region_count(vms);
- qemu_fdt_setprop_string(vms->fdt, "/intc", "compatible",
+ qemu_fdt_setprop_string(vms->fdt, nodename, "compatible",
"arm,gic-v3");
- qemu_fdt_setprop_cell(vms->fdt, "/intc",
+ qemu_fdt_setprop_cell(vms->fdt, nodename,
"#redistributor-regions", nb_redist_regions);
if (nb_redist_regions == 1) {
- qemu_fdt_setprop_sized_cells(vms->fdt, "/intc", "reg",
+ qemu_fdt_setprop_sized_cells(vms->fdt, nodename, "reg",
2, vms->memmap[VIRT_GIC_DIST].base,
2, vms->memmap[VIRT_GIC_DIST].size,
2, vms->memmap[VIRT_GIC_REDIST].base,
2, vms->memmap[VIRT_GIC_REDIST].size);
} else {
- qemu_fdt_setprop_sized_cells(vms->fdt, "/intc", "reg",
+ qemu_fdt_setprop_sized_cells(vms->fdt, nodename, "reg",
2, vms->memmap[VIRT_GIC_DIST].base,
2, vms->memmap[VIRT_GIC_DIST].size,
2, vms->memmap[VIRT_GIC_REDIST].base,
@@ -430,22 +439,23 @@
}
if (vms->virt) {
- qemu_fdt_setprop_cells(vms->fdt, "/intc", "interrupts",
+ qemu_fdt_setprop_cells(vms->fdt, nodename, "interrupts",
GIC_FDT_IRQ_TYPE_PPI, ARCH_GICV3_MAINT_IRQ,
GIC_FDT_IRQ_FLAGS_LEVEL_HI);
}
} else {
/* 'cortex-a15-gic' means 'GIC v2' */
- qemu_fdt_setprop_string(vms->fdt, "/intc", "compatible",
+ qemu_fdt_setprop_string(vms->fdt, nodename, "compatible",
"arm,cortex-a15-gic");
- qemu_fdt_setprop_sized_cells(vms->fdt, "/intc", "reg",
+ qemu_fdt_setprop_sized_cells(vms->fdt, nodename, "reg",
2, vms->memmap[VIRT_GIC_DIST].base,
2, vms->memmap[VIRT_GIC_DIST].size,
2, vms->memmap[VIRT_GIC_CPU].base,
2, vms->memmap[VIRT_GIC_CPU].size);
}
- qemu_fdt_setprop_cell(vms->fdt, "/intc", "phandle", vms->gic_phandle);
+ qemu_fdt_setprop_cell(vms->fdt, nodename, "phandle", vms->gic_phandle);
+ g_free(nodename);
}
static void fdt_add_pmu_nodes(const VirtMachineState *vms)
diff --git a/hw/block/dataplane/virtio-blk.c b/hw/block/dataplane/virtio-blk.c
index d648aeb..8c37bd3 100644
--- a/hw/block/dataplane/virtio-blk.c
+++ b/hw/block/dataplane/virtio-blk.c
@@ -190,8 +190,8 @@
/* Set up guest notifier (irq) */
r = k->set_guest_notifiers(qbus->parent, nvqs, true);
if (r != 0) {
- fprintf(stderr, "virtio-blk failed to set guest notifier (%d), "
- "ensure -enable-kvm is set\n", r);
+ error_report("virtio-blk failed to set guest notifier (%d), "
+ "ensure -accel kvm is set.", r);
goto fail_guest_notifiers;
}
diff --git a/hw/block/fdc.c b/hw/block/fdc.c
index c7b4fe9..2e9c1e1 100644
--- a/hw/block/fdc.c
+++ b/hw/block/fdc.c
@@ -397,16 +397,9 @@
nb_sectors,
FloppyDriveType_str(parse->drive));
}
+ assert(type_match != -1 && "misconfigured fd_format");
match = type_match;
}
-
- /* No match of any kind found -- fd_format is misconfigured, abort. */
- if (match == -1) {
- error_setg(&error_abort, "No candidate geometries present in table "
- " for floppy drive type '%s'",
- FloppyDriveType_str(drv->drive));
- }
-
parse = &(fd_formats[match]);
out:
diff --git a/hw/block/nvme.c b/hw/block/nvme.c
index d5bf95b..156ecf3 100644
--- a/hw/block/nvme.c
+++ b/hw/block/nvme.c
@@ -18,7 +18,8 @@
* Usage: add options:
* -drive file=<file>,if=none,id=<drive_id>
* -device nvme,drive=<drive_id>,serial=<serial>,id=<id[optional]>, \
- * cmb_size_mb=<cmb_size_mb[optional]>
+ * cmb_size_mb=<cmb_size_mb[optional]>, \
+ * num_queues=<N[optional]>
*
* Note cmb_size_mb denotes size of CMB in MB. CMB is assumed to be at
* offset 0 in BAR2 and supports only WDS, RDS and SQS for now.
@@ -1232,7 +1233,6 @@
pcie_endpoint_cap_init(&n->parent_obj, 0x80);
n->num_namespaces = 1;
- n->num_queues = 64;
n->reg_size = pow2ceil(0x1004 + 2 * (n->num_queues + 1) * 4);
n->ns_size = bs_size / (uint64_t)n->num_namespaces;
@@ -1341,6 +1341,7 @@
DEFINE_BLOCK_PROPERTIES(NvmeCtrl, conf),
DEFINE_PROP_STRING("serial", NvmeCtrl, serial),
DEFINE_PROP_UINT32("cmb_size_mb", NvmeCtrl, cmb_size_mb, 0),
+ DEFINE_PROP_UINT32("num_queues", NvmeCtrl, num_queues, 64),
DEFINE_PROP_END_OF_LIST(),
};
diff --git a/hw/char/serial.c b/hw/char/serial.c
index 26f53ba..cd7d747 100644
--- a/hw/char/serial.c
+++ b/hw/char/serial.c
@@ -261,7 +261,7 @@
if (s->mcr & UART_MCR_LOOP) {
/* in loopback mode, say that we just received a char */
serial_receive1(s, &s->tsr, 1);
- } else if (qemu_chr_fe_write(&s->chr, &s->tsr, 1) != 1 &&
+ } else if (qemu_chr_fe_write(&s->chr, &s->tsr, 1) == 0 &&
s->tsr_retry < MAX_XMIT_RETRY) {
assert(s->watch_tag == 0);
s->watch_tag =
diff --git a/hw/i386/kvm/ioapic.c b/hw/i386/kvm/ioapic.c
index 646f624..5b40d75 100644
--- a/hw/i386/kvm/ioapic.c
+++ b/hw/i386/kvm/ioapic.c
@@ -112,15 +112,6 @@
}
}
-void kvm_ioapic_dump_state(Monitor *mon, const QDict *qdict)
-{
- IOAPICCommonState *s = IOAPIC_COMMON(object_resolve_path("ioapic", NULL));
-
- assert(s);
- kvm_ioapic_get(s);
- ioapic_print_redtbl(mon, s);
-}
-
static void kvm_ioapic_reset(DeviceState *dev)
{
IOAPICCommonState *s = IOAPIC_COMMON(dev);
@@ -132,8 +123,10 @@
static void kvm_ioapic_set_irq(void *opaque, int irq, int level)
{
KVMIOAPICState *s = opaque;
+ IOAPICCommonState *common = IOAPIC_COMMON(s);
int delivered;
+ ioapic_stat_update_irq(common, irq, level);
delivered = kvm_set_irq(kvm_state, s->kvm_gsi_base + irq, level);
apic_report_irq_delivered(delivered);
}
diff --git a/hw/i386/pc.c b/hw/i386/pc.c
index 622e49d..f310040 100644
--- a/hw/i386/pc.c
+++ b/hw/i386/pc.c
@@ -1674,27 +1674,11 @@
}
}
-static void pc_dimm_plug(HotplugHandler *hotplug_dev,
- DeviceState *dev, Error **errp)
+static void pc_memory_pre_plug(HotplugHandler *hotplug_dev, DeviceState *dev,
+ Error **errp)
{
- HotplugHandlerClass *hhc;
- Error *local_err = NULL;
- PCMachineState *pcms = PC_MACHINE(hotplug_dev);
- PCMachineClass *pcmc = PC_MACHINE_GET_CLASS(pcms);
- PCDIMMDevice *dimm = PC_DIMM(dev);
- PCDIMMDeviceClass *ddc = PC_DIMM_GET_CLASS(dimm);
- MemoryRegion *mr;
- uint64_t align = TARGET_PAGE_SIZE;
- bool is_nvdimm = object_dynamic_cast(OBJECT(dev), TYPE_NVDIMM);
-
- mr = ddc->get_memory_region(dimm, &local_err);
- if (local_err) {
- goto out;
- }
-
- if (memory_region_get_alignment(mr) && pcmc->enforce_aligned_dimm) {
- align = memory_region_get_alignment(mr);
- }
+ const PCMachineState *pcms = PC_MACHINE(hotplug_dev);
+ const bool is_nvdimm = object_dynamic_cast(OBJECT(dev), TYPE_NVDIMM);
/*
* When -no-acpi is used with Q35 machine type, no ACPI is built,
@@ -1702,18 +1686,35 @@
* addition to cover this case.
*/
if (!pcms->acpi_dev || !acpi_enabled) {
- error_setg(&local_err,
+ error_setg(errp,
"memory hotplug is not enabled: missing acpi device or acpi disabled");
- goto out;
+ return;
}
if (is_nvdimm && !pcms->acpi_nvdimm_state.is_enabled) {
- error_setg(&local_err,
- "nvdimm is not enabled: missing 'nvdimm' in '-M'");
- goto out;
+ error_setg(errp, "nvdimm is not enabled: missing 'nvdimm' in '-M'");
+ return;
+ }
+}
+
+static void pc_memory_plug(HotplugHandler *hotplug_dev,
+ DeviceState *dev, Error **errp)
+{
+ HotplugHandlerClass *hhc;
+ Error *local_err = NULL;
+ PCMachineState *pcms = PC_MACHINE(hotplug_dev);
+ PCMachineClass *pcmc = PC_MACHINE_GET_CLASS(pcms);
+ PCDIMMDevice *dimm = PC_DIMM(dev);
+ PCDIMMDeviceClass *ddc = PC_DIMM_GET_CLASS(dimm);
+ MemoryRegion *mr = ddc->get_memory_region(dimm, &error_abort);
+ uint64_t align = TARGET_PAGE_SIZE;
+ bool is_nvdimm = object_dynamic_cast(OBJECT(dev), TYPE_NVDIMM);
+
+ if (memory_region_get_alignment(mr) && pcmc->enforce_aligned_dimm) {
+ align = memory_region_get_alignment(mr);
}
- pc_dimm_memory_plug(dev, MACHINE(pcms), align, &local_err);
+ pc_dimm_plug(dev, MACHINE(pcms), align, &local_err);
if (local_err) {
goto out;
}
@@ -1728,8 +1729,8 @@
error_propagate(errp, local_err);
}
-static void pc_dimm_unplug_request(HotplugHandler *hotplug_dev,
- DeviceState *dev, Error **errp)
+static void pc_memory_unplug_request(HotplugHandler *hotplug_dev,
+ DeviceState *dev, Error **errp)
{
HotplugHandlerClass *hhc;
Error *local_err = NULL;
@@ -1759,8 +1760,8 @@
error_propagate(errp, local_err);
}
-static void pc_dimm_unplug(HotplugHandler *hotplug_dev,
- DeviceState *dev, Error **errp)
+static void pc_memory_unplug(HotplugHandler *hotplug_dev,
+ DeviceState *dev, Error **errp)
{
PCMachineState *pcms = PC_MACHINE(hotplug_dev);
HotplugHandlerClass *hhc;
@@ -1773,7 +1774,7 @@
goto out;
}
- pc_dimm_memory_unplug(dev, MACHINE(pcms));
+ pc_dimm_unplug(dev, MACHINE(pcms));
object_unparent(OBJECT(dev));
out:
@@ -2006,7 +2007,9 @@
static void pc_machine_device_pre_plug_cb(HotplugHandler *hotplug_dev,
DeviceState *dev, Error **errp)
{
- if (object_dynamic_cast(OBJECT(dev), TYPE_CPU)) {
+ if (object_dynamic_cast(OBJECT(dev), TYPE_PC_DIMM)) {
+ pc_memory_pre_plug(hotplug_dev, dev, errp);
+ } else if (object_dynamic_cast(OBJECT(dev), TYPE_CPU)) {
pc_cpu_pre_plug(hotplug_dev, dev, errp);
}
}
@@ -2015,7 +2018,7 @@
DeviceState *dev, Error **errp)
{
if (object_dynamic_cast(OBJECT(dev), TYPE_PC_DIMM)) {
- pc_dimm_plug(hotplug_dev, dev, errp);
+ pc_memory_plug(hotplug_dev, dev, errp);
} else if (object_dynamic_cast(OBJECT(dev), TYPE_CPU)) {
pc_cpu_plug(hotplug_dev, dev, errp);
}
@@ -2025,7 +2028,7 @@
DeviceState *dev, Error **errp)
{
if (object_dynamic_cast(OBJECT(dev), TYPE_PC_DIMM)) {
- pc_dimm_unplug_request(hotplug_dev, dev, errp);
+ pc_memory_unplug_request(hotplug_dev, dev, errp);
} else if (object_dynamic_cast(OBJECT(dev), TYPE_CPU)) {
pc_cpu_unplug_request_cb(hotplug_dev, dev, errp);
} else {
@@ -2038,7 +2041,7 @@
DeviceState *dev, Error **errp)
{
if (object_dynamic_cast(OBJECT(dev), TYPE_PC_DIMM)) {
- pc_dimm_unplug(hotplug_dev, dev, errp);
+ pc_memory_unplug(hotplug_dev, dev, errp);
} else if (object_dynamic_cast(OBJECT(dev), TYPE_CPU)) {
pc_cpu_unplug_cb(hotplug_dev, dev, errp);
} else {
diff --git a/hw/intc/ioapic.c b/hw/intc/ioapic.c
index c45f073..b393780 100644
--- a/hw/intc/ioapic.c
+++ b/hw/intc/ioapic.c
@@ -148,6 +148,7 @@
* the cleanest way of doing it but it should work. */
trace_ioapic_set_irq(vector, level);
+ ioapic_stat_update_irq(s, vector, level);
if (vector == 0) {
vector = 2;
}
@@ -233,17 +234,6 @@
}
}
-void ioapic_dump_state(Monitor *mon, const QDict *qdict)
-{
- int i;
-
- for (i = 0; i < MAX_IOAPICS; i++) {
- if (ioapics[i] != 0) {
- ioapic_print_redtbl(mon, ioapics[i]);
- }
- }
-}
-
static uint64_t
ioapic_mem_read(void *opaque, hwaddr addr, unsigned int size)
{
diff --git a/hw/intc/ioapic_common.c b/hw/intc/ioapic_common.c
index 3b3d0a7..692dc37 100644
--- a/hw/intc/ioapic_common.c
+++ b/hw/intc/ioapic_common.c
@@ -24,6 +24,7 @@
#include "monitor/monitor.h"
#include "hw/i386/ioapic.h"
#include "hw/i386/ioapic_internal.h"
+#include "hw/intc/intc.h"
#include "hw/sysbus.h"
/* ioapic_no count start from 0 to MAX_IOAPICS,
@@ -34,6 +35,28 @@
*/
int ioapic_no;
+void ioapic_stat_update_irq(IOAPICCommonState *s, int irq, int level)
+{
+ if (level != s->irq_level[irq]) {
+ s->irq_level[irq] = level;
+ if (level == 1) {
+ s->irq_count[irq]++;
+ }
+ }
+}
+
+static bool ioapic_get_statistics(InterruptStatsProvider *obj,
+ uint64_t **irq_counts,
+ unsigned int *nb_irqs)
+{
+ IOAPICCommonState *s = IOAPIC_COMMON(obj);
+
+ *irq_counts = s->irq_count;
+ *nb_irqs = IOAPIC_NUM_PINS;
+
+ return true;
+}
+
static void ioapic_irr_dump(Monitor *mon, const char *name, uint32_t bitmap)
{
int i;
@@ -58,7 +81,7 @@
uint32_t remote_irr = 0;
int i;
- monitor_printf(mon, "ioapic ver=0x%x id=0x%02x sel=0x%02x",
+ monitor_printf(mon, "ioapic0: ver=0x%x id=0x%02x sel=0x%02x",
s->version, s->id, s->ioregsel);
if (s->ioregsel) {
monitor_printf(mon, " (redir[%u])\n",
@@ -70,7 +93,7 @@
uint64_t entry = s->ioredtbl[i];
uint32_t delm = (uint32_t)((entry & IOAPIC_LVT_DELIV_MODE) >>
IOAPIC_LVT_DELIV_MODE_SHIFT);
- monitor_printf(mon, "pin %-2u 0x%016"PRIx64" dest=%"PRIx64
+ monitor_printf(mon, " pin %-2u 0x%016"PRIx64" dest=%"PRIx64
" vec=%-3"PRIu64" %s %-5s %-6s %-6s %s\n",
i, entry,
(entry >> IOAPIC_LVT_DEST_SHIFT) &
@@ -85,8 +108,8 @@
remote_irr |= entry & IOAPIC_LVT_TRIGGER_MODE ?
(entry & IOAPIC_LVT_REMOTE_IRR ? (1 << i) : 0) : 0;
}
- ioapic_irr_dump(mon, "IRR", s->irr);
- ioapic_irr_dump(mon, "Remote IRR", remote_irr);
+ ioapic_irr_dump(mon, " IRR", s->irr);
+ ioapic_irr_dump(mon, " Remote IRR", remote_irr);
}
void ioapic_reset_common(DeviceState *dev)
@@ -142,6 +165,15 @@
ioapic_no++;
}
+static void ioapic_print_info(InterruptStatsProvider *obj,
+ Monitor *mon)
+{
+ IOAPICCommonState *s = IOAPIC_COMMON(obj);
+
+ ioapic_dispatch_pre_save(s);
+ ioapic_print_redtbl(mon, s);
+}
+
static const VMStateDescription vmstate_ioapic_common = {
.name = "ioapic",
.version_id = 3,
@@ -161,9 +193,12 @@
static void ioapic_common_class_init(ObjectClass *klass, void *data)
{
DeviceClass *dc = DEVICE_CLASS(klass);
+ InterruptStatsProviderClass *ic = INTERRUPT_STATS_PROVIDER_CLASS(klass);
dc->realize = ioapic_common_realize;
dc->vmsd = &vmstate_ioapic_common;
+ ic->print_info = ioapic_print_info;
+ ic->get_statistics = ioapic_get_statistics;
}
static const TypeInfo ioapic_common_type = {
@@ -173,6 +208,10 @@
.class_size = sizeof(IOAPICCommonClass),
.class_init = ioapic_common_class_init,
.abstract = true,
+ .interfaces = (InterfaceInfo[]) {
+ { TYPE_INTERRUPT_STATS_PROVIDER },
+ { }
+ },
};
static void ioapic_common_register_types(void)
diff --git a/hw/mem/memory-device.c b/hw/mem/memory-device.c
index 3e04f39..6de4f70 100644
--- a/hw/mem/memory-device.c
+++ b/hw/mem/memory-device.c
@@ -116,9 +116,15 @@
address_space_start = ms->device_memory->base;
address_space_end = address_space_start +
memory_region_size(&ms->device_memory->mr);
- g_assert(QEMU_ALIGN_UP(address_space_start, align) == address_space_start);
g_assert(address_space_end >= address_space_start);
+ /* address_space_start indicates the maximum alignment we expect */
+ if (QEMU_ALIGN_UP(address_space_start, align) != address_space_start) {
+ error_setg(errp, "the alignment (0%" PRIx64 ") is not supported",
+ align);
+ return 0;
+ }
+
memory_device_check_addable(ms, size, errp);
if (*errp) {
return 0;
diff --git a/hw/mem/nvdimm.c b/hw/mem/nvdimm.c
index 4087aca..021d1c3 100644
--- a/hw/mem/nvdimm.c
+++ b/hw/mem/nvdimm.c
@@ -43,7 +43,7 @@
Error *local_err = NULL;
uint64_t value;
- if (memory_region_size(&nvdimm->nvdimm_mr)) {
+ if (nvdimm->nvdimm_mr) {
error_setg(&local_err, "cannot change property value");
goto out;
}
@@ -64,52 +64,36 @@
error_propagate(errp, local_err);
}
-static bool nvdimm_get_unarmed(Object *obj, Error **errp)
-{
- NVDIMMDevice *nvdimm = NVDIMM(obj);
-
- return nvdimm->unarmed;
-}
-
-static void nvdimm_set_unarmed(Object *obj, bool value, Error **errp)
-{
- NVDIMMDevice *nvdimm = NVDIMM(obj);
- Error *local_err = NULL;
-
- if (memory_region_size(&nvdimm->nvdimm_mr)) {
- error_setg(&local_err, "cannot change property value");
- goto out;
- }
-
- nvdimm->unarmed = value;
-
- out:
- error_propagate(errp, local_err);
-}
-
static void nvdimm_init(Object *obj)
{
object_property_add(obj, NVDIMM_LABEL_SIZE_PROP, "int",
nvdimm_get_label_size, nvdimm_set_label_size, NULL,
NULL, NULL);
- object_property_add_bool(obj, NVDIMM_UNARMED_PROP,
- nvdimm_get_unarmed, nvdimm_set_unarmed, NULL);
}
-static MemoryRegion *nvdimm_get_memory_region(PCDIMMDevice *dimm, Error **errp)
+static void nvdimm_finalize(Object *obj)
{
- NVDIMMDevice *nvdimm = NVDIMM(dimm);
+ NVDIMMDevice *nvdimm = NVDIMM(obj);
- return &nvdimm->nvdimm_mr;
+ g_free(nvdimm->nvdimm_mr);
}
-static void nvdimm_realize(PCDIMMDevice *dimm, Error **errp)
+static void nvdimm_prepare_memory_region(NVDIMMDevice *nvdimm, Error **errp)
{
- MemoryRegion *mr = host_memory_backend_get_memory(dimm->hostmem, errp);
- NVDIMMDevice *nvdimm = NVDIMM(dimm);
- uint64_t align, pmem_size, size = memory_region_size(mr);
+ PCDIMMDevice *dimm = PC_DIMM(nvdimm);
+ uint64_t align, pmem_size, size;
+ MemoryRegion *mr;
+ g_assert(!nvdimm->nvdimm_mr);
+
+ if (!dimm->hostmem) {
+ error_setg(errp, "'" PC_DIMM_MEMDEV_PROP "' property must be set");
+ return;
+ }
+
+ mr = host_memory_backend_get_memory(dimm->hostmem);
align = memory_region_get_alignment(mr);
+ size = memory_region_size(mr);
pmem_size = size - nvdimm->label_size;
nvdimm->label_data = memory_region_get_ram_ptr(mr) + pmem_size;
@@ -127,9 +111,34 @@
return;
}
- memory_region_init_alias(&nvdimm->nvdimm_mr, OBJECT(dimm),
+ nvdimm->nvdimm_mr = g_new(MemoryRegion, 1);
+ memory_region_init_alias(nvdimm->nvdimm_mr, OBJECT(dimm),
"nvdimm-memory", mr, 0, pmem_size);
- nvdimm->nvdimm_mr.align = align;
+ nvdimm->nvdimm_mr->align = align;
+}
+
+static MemoryRegion *nvdimm_get_memory_region(PCDIMMDevice *dimm, Error **errp)
+{
+ NVDIMMDevice *nvdimm = NVDIMM(dimm);
+ Error *local_err = NULL;
+
+ if (!nvdimm->nvdimm_mr) {
+ nvdimm_prepare_memory_region(nvdimm, &local_err);
+ if (local_err) {
+ error_propagate(errp, local_err);
+ return NULL;
+ }
+ }
+ return nvdimm->nvdimm_mr;
+}
+
+static void nvdimm_realize(PCDIMMDevice *dimm, Error **errp)
+{
+ NVDIMMDevice *nvdimm = NVDIMM(dimm);
+
+ if (!nvdimm->nvdimm_mr) {
+ nvdimm_prepare_memory_region(nvdimm, errp);
+ }
}
/*
@@ -161,24 +170,25 @@
memcpy(nvdimm->label_data + offset, buf, size);
- mr = host_memory_backend_get_memory(dimm->hostmem, &error_abort);
+ mr = host_memory_backend_get_memory(dimm->hostmem);
backend_offset = memory_region_size(mr) - nvdimm->label_size + offset;
memory_region_set_dirty(mr, backend_offset, size);
}
-static MemoryRegion *nvdimm_get_vmstate_memory_region(PCDIMMDevice *dimm)
-{
- return host_memory_backend_get_memory(dimm->hostmem, &error_abort);
-}
+static Property nvdimm_properties[] = {
+ DEFINE_PROP_BOOL(NVDIMM_UNARMED_PROP, NVDIMMDevice, unarmed, false),
+ DEFINE_PROP_END_OF_LIST(),
+};
static void nvdimm_class_init(ObjectClass *oc, void *data)
{
PCDIMMDeviceClass *ddc = PC_DIMM_CLASS(oc);
NVDIMMClass *nvc = NVDIMM_CLASS(oc);
+ DeviceClass *dc = DEVICE_CLASS(oc);
ddc->realize = nvdimm_realize;
ddc->get_memory_region = nvdimm_get_memory_region;
- ddc->get_vmstate_memory_region = nvdimm_get_vmstate_memory_region;
+ dc->props = nvdimm_properties;
nvc->read_label_data = nvdimm_read_label_data;
nvc->write_label_data = nvdimm_write_label_data;
@@ -191,6 +201,7 @@
.class_init = nvdimm_class_init,
.instance_size = sizeof(NVDIMMDevice),
.instance_init = nvdimm_init,
+ .instance_finalize = nvdimm_finalize,
};
static void nvdimm_register_types(void)
diff --git a/hw/mem/pc-dimm.c b/hw/mem/pc-dimm.c
index 12da89d..65843bc 100644
--- a/hw/mem/pc-dimm.c
+++ b/hw/mem/pc-dimm.c
@@ -27,27 +27,20 @@
#include "sysemu/numa.h"
#include "trace.h"
-typedef struct pc_dimms_capacity {
- uint64_t size;
- Error **errp;
-} pc_dimms_capacity;
+static int pc_dimm_get_free_slot(const int *hint, int max_slots, Error **errp);
-void pc_dimm_memory_plug(DeviceState *dev, MachineState *machine,
- uint64_t align, Error **errp)
+void pc_dimm_plug(DeviceState *dev, MachineState *machine, uint64_t align,
+ Error **errp)
{
int slot;
PCDIMMDevice *dimm = PC_DIMM(dev);
PCDIMMDeviceClass *ddc = PC_DIMM_GET_CLASS(dimm);
- MemoryRegion *vmstate_mr = ddc->get_vmstate_memory_region(dimm);
+ MemoryRegion *vmstate_mr = ddc->get_vmstate_memory_region(dimm,
+ &error_abort);
+ MemoryRegion *mr = ddc->get_memory_region(dimm, &error_abort);
Error *local_err = NULL;
- MemoryRegion *mr;
uint64_t addr;
- mr = ddc->get_memory_region(dimm, &local_err);
- if (local_err) {
- goto out;
- }
-
addr = object_property_get_uint(OBJECT(dimm),
PC_DIMM_ADDR_PROP, &local_err);
if (local_err) {
@@ -89,11 +82,12 @@
error_propagate(errp, local_err);
}
-void pc_dimm_memory_unplug(DeviceState *dev, MachineState *machine)
+void pc_dimm_unplug(DeviceState *dev, MachineState *machine)
{
PCDIMMDevice *dimm = PC_DIMM(dev);
PCDIMMDeviceClass *ddc = PC_DIMM_GET_CLASS(dimm);
- MemoryRegion *vmstate_mr = ddc->get_vmstate_memory_region(dimm);
+ MemoryRegion *vmstate_mr = ddc->get_vmstate_memory_region(dimm,
+ &error_abort);
MemoryRegion *mr = ddc->get_memory_region(dimm, &error_abort);
memory_device_unplug_region(machine, mr);
@@ -116,7 +110,7 @@
return 0;
}
-int pc_dimm_get_free_slot(const int *hint, int max_slots, Error **errp)
+static int pc_dimm_get_free_slot(const int *hint, int max_slots, Error **errp)
{
unsigned long *bitmap;
int slot = 0;
@@ -229,12 +223,7 @@
return NULL;
}
- return host_memory_backend_get_memory(dimm->hostmem, errp);
-}
-
-static MemoryRegion *pc_dimm_get_vmstate_memory_region(PCDIMMDevice *dimm)
-{
- return host_memory_backend_get_memory(dimm->hostmem, &error_abort);
+ return host_memory_backend_get_memory(dimm->hostmem);
}
static uint64_t pc_dimm_md_get_addr(const MemoryDeviceState *md)
@@ -301,7 +290,7 @@
dc->desc = "DIMM memory module";
ddc->get_memory_region = pc_dimm_get_memory_region;
- ddc->get_vmstate_memory_region = pc_dimm_get_vmstate_memory_region;
+ ddc->get_vmstate_memory_region = pc_dimm_get_memory_region;
mdc->get_addr = pc_dimm_md_get_addr;
/* for a dimm plugged_size == region_size */
diff --git a/hw/mips/mips_jazz.c b/hw/mips/mips_jazz.c
index 90cb306..1afbe3c 100644
--- a/hw/mips/mips_jazz.c
+++ b/hw/mips/mips_jazz.c
@@ -145,10 +145,10 @@
ISABus *isa_bus;
ISADevice *pit;
DriveInfo *fds[MAX_FD];
- qemu_irq esp_reset, dma_enable;
MemoryRegion *ram = g_new(MemoryRegion, 1);
MemoryRegion *bios = g_new(MemoryRegion, 1);
MemoryRegion *bios2 = g_new(MemoryRegion, 1);
+ SysBusESPState *sysbus_esp;
ESPState *esp;
/* init CPUs */
@@ -281,8 +281,21 @@
}
/* SCSI adapter */
- esp = esp_init(0x80002000, 0, rc4030_dma_read, rc4030_dma_write, dmas[0],
- qdev_get_gpio_in(rc4030, 5), &esp_reset, &dma_enable);
+ dev = qdev_create(NULL, TYPE_ESP);
+ sysbus_esp = ESP_STATE(dev);
+ esp = &sysbus_esp->esp;
+ esp->dma_memory_read = rc4030_dma_read;
+ esp->dma_memory_write = rc4030_dma_write;
+ esp->dma_opaque = dmas[0];
+ sysbus_esp->it_shift = 0;
+ /* XXX for now until rc4030 has been changed to use DMA enable signal */
+ esp->dma_enabled = 1;
+ qdev_init_nofail(dev);
+
+ sysbus = SYS_BUS_DEVICE(dev);
+ sysbus_connect_irq(sysbus, 0, qdev_get_gpio_in(rc4030, 5));
+ sysbus_mmio_map(sysbus, 0, 0x80002000);
+
scsi_bus_legacy_handle_cmdline(&esp->bus);
/* Floppy */
diff --git a/hw/misc/ivshmem.c b/hw/misc/ivshmem.c
index 16f0370..ee01c5e 100644
--- a/hw/misc/ivshmem.c
+++ b/hw/misc/ivshmem.c
@@ -909,8 +909,7 @@
if (s->hostmem != NULL) {
IVSHMEM_DPRINTF("using hostmem\n");
- s->ivshmem_bar2 = host_memory_backend_get_memory(s->hostmem,
- &error_abort);
+ s->ivshmem_bar2 = host_memory_backend_get_memory(s->hostmem);
} else {
Chardev *chr = qemu_chr_fe_get_driver(&s->server_chr);
assert(chr);
diff --git a/hw/ppc/spapr.c b/hw/ppc/spapr.c
index 0d032a1..b32b971 100644
--- a/hw/ppc/spapr.c
+++ b/hw/ppc/spapr.c
@@ -3149,18 +3149,14 @@
sPAPRMachineState *ms = SPAPR_MACHINE(hotplug_dev);
PCDIMMDevice *dimm = PC_DIMM(dev);
PCDIMMDeviceClass *ddc = PC_DIMM_GET_CLASS(dimm);
- MemoryRegion *mr;
+ MemoryRegion *mr = ddc->get_memory_region(dimm, &error_abort);
uint64_t align, size, addr;
uint32_t node;
- mr = ddc->get_memory_region(dimm, &local_err);
- if (local_err) {
- goto out;
- }
align = memory_region_get_alignment(mr);
size = memory_region_size(mr);
- pc_dimm_memory_plug(dev, MACHINE(ms), align, &local_err);
+ pc_dimm_plug(dev, MACHINE(ms), align, &local_err);
if (local_err) {
goto out;
}
@@ -3183,7 +3179,7 @@
return;
out_unplug:
- pc_dimm_memory_unplug(dev, MACHINE(ms));
+ pc_dimm_unplug(dev, MACHINE(ms));
out:
error_propagate(errp, local_err);
}
@@ -3332,7 +3328,7 @@
sPAPRMachineState *spapr = SPAPR_MACHINE(hotplug_dev);
sPAPRDIMMState *ds = spapr_pending_dimm_unplugs_find(spapr, PC_DIMM(dev));
- pc_dimm_memory_unplug(dev, MACHINE(hotplug_dev));
+ pc_dimm_unplug(dev, MACHINE(hotplug_dev));
object_unparent(OBJECT(dev));
spapr_pending_dimm_unplugs_remove(spapr, ds);
}
@@ -3344,16 +3340,12 @@
Error *local_err = NULL;
PCDIMMDevice *dimm = PC_DIMM(dev);
PCDIMMDeviceClass *ddc = PC_DIMM_GET_CLASS(dimm);
- MemoryRegion *mr;
+ MemoryRegion *mr = ddc->get_memory_region(dimm, &error_abort);
uint32_t nr_lmbs;
uint64_t size, addr_start, addr;
int i;
sPAPRDRConnector *drc;
- mr = ddc->get_memory_region(dimm, &local_err);
- if (local_err) {
- goto out;
- }
size = memory_region_size(mr);
nr_lmbs = size / SPAPR_MEMORY_BLOCK_SIZE;
diff --git a/hw/scsi/esp.c b/hw/scsi/esp.c
index 9ed9727..630d923 100644
--- a/hw/scsi/esp.c
+++ b/hw/scsi/esp.c
@@ -619,36 +619,6 @@
.valid.accepts = esp_mem_accepts,
};
-ESPState *esp_init(hwaddr espaddr, int it_shift,
- ESPDMAMemoryReadWriteFunc dma_memory_read,
- ESPDMAMemoryReadWriteFunc dma_memory_write,
- void *dma_opaque, qemu_irq irq, qemu_irq *reset,
- qemu_irq *dma_enable)
-{
- DeviceState *dev;
- SysBusDevice *s;
- SysBusESPState *sysbus;
- ESPState *esp;
-
- dev = qdev_create(NULL, TYPE_ESP);
- sysbus = ESP_STATE(dev);
- esp = &sysbus->esp;
- esp->dma_memory_read = dma_memory_read;
- esp->dma_memory_write = dma_memory_write;
- esp->dma_opaque = dma_opaque;
- sysbus->it_shift = it_shift;
- /* XXX for now until rc4030 has been changed to use DMA enable signal */
- esp->dma_enabled = 1;
- qdev_init_nofail(dev);
- s = SYS_BUS_DEVICE(dev);
- sysbus_connect_irq(s, 0, irq);
- sysbus_mmio_map(s, 0, espaddr);
- *reset = qdev_get_gpio_in(dev, 0);
- *dma_enable = qdev_get_gpio_in(dev, 1);
-
- return esp;
-}
-
static const struct SCSIBusInfo esp_scsi_info = {
.tcq = false,
.max_target = ESP_MAX_DEVS,
diff --git a/hw/scsi/scsi-bus.c b/hw/scsi/scsi-bus.c
index 9646743..5905f6b 100644
--- a/hw/scsi/scsi-bus.c
+++ b/hw/scsi/scsi-bus.c
@@ -226,6 +226,8 @@
SCSIDevice *scsi_bus_legacy_add_drive(SCSIBus *bus, BlockBackend *blk,
int unit, bool removable, int bootindex,
bool share_rw,
+ BlockdevOnError rerror,
+ BlockdevOnError werror,
const char *serial, Error **errp)
{
const char *driver;
@@ -262,6 +264,10 @@
object_unparent(OBJECT(dev));
return NULL;
}
+
+ qdev_prop_set_enum(dev, "rerror", rerror);
+ qdev_prop_set_enum(dev, "werror", werror);
+
object_property_set_bool(OBJECT(dev), true, "realized", &err);
if (err != NULL) {
error_propagate(errp, err);
@@ -285,7 +291,10 @@
}
qemu_opts_loc_restore(dinfo->opts);
scsi_bus_legacy_add_drive(bus, blk_by_legacy_dinfo(dinfo),
- unit, false, -1, false, NULL, &error_fatal);
+ unit, false, -1, false,
+ BLOCKDEV_ON_ERROR_AUTO,
+ BLOCKDEV_ON_ERROR_AUTO,
+ NULL, &error_fatal);
}
loc_pop(&loc);
}
diff --git a/hw/scsi/scsi-disk.c b/hw/scsi/scsi-disk.c
index aeaf611..55a34b3 100644
--- a/hw/scsi/scsi-disk.c
+++ b/hw/scsi/scsi-disk.c
@@ -585,219 +585,228 @@
return (uint8_t *)r->iov.iov_base;
}
+int scsi_disk_emulate_vpd_page(SCSIRequest *req, uint8_t *outbuf)
+{
+ SCSIDiskState *s = DO_UPCAST(SCSIDiskState, qdev, req->dev);
+ uint8_t page_code = req->cmd.buf[2];
+ int start, buflen = 0;
+
+ outbuf[buflen++] = s->qdev.type & 0x1f;
+ outbuf[buflen++] = page_code;
+ outbuf[buflen++] = 0x00;
+ outbuf[buflen++] = 0x00;
+ start = buflen;
+
+ switch (page_code) {
+ case 0x00: /* Supported page codes, mandatory */
+ {
+ DPRINTF("Inquiry EVPD[Supported pages] "
+ "buffer size %zd\n", req->cmd.xfer);
+ outbuf[buflen++] = 0x00; /* list of supported pages (this page) */
+ if (s->serial) {
+ outbuf[buflen++] = 0x80; /* unit serial number */
+ }
+ outbuf[buflen++] = 0x83; /* device identification */
+ if (s->qdev.type == TYPE_DISK) {
+ outbuf[buflen++] = 0xb0; /* block limits */
+ outbuf[buflen++] = 0xb1; /* block device characteristics */
+ outbuf[buflen++] = 0xb2; /* thin provisioning */
+ }
+ break;
+ }
+ case 0x80: /* Device serial number, optional */
+ {
+ int l;
+
+ if (!s->serial) {
+ DPRINTF("Inquiry (EVPD[Serial number] not supported\n");
+ return -1;
+ }
+
+ l = strlen(s->serial);
+ if (l > 36) {
+ l = 36;
+ }
+
+ DPRINTF("Inquiry EVPD[Serial number] "
+ "buffer size %zd\n", req->cmd.xfer);
+ memcpy(outbuf + buflen, s->serial, l);
+ buflen += l;
+ break;
+ }
+
+ case 0x83: /* Device identification page, mandatory */
+ {
+ const char *str = s->serial ?: blk_name(s->qdev.conf.blk);
+ int max_len = s->serial ? 20 : 255 - 8;
+ int id_len = strlen(str);
+
+ if (id_len > max_len) {
+ id_len = max_len;
+ }
+ DPRINTF("Inquiry EVPD[Device identification] "
+ "buffer size %zd\n", req->cmd.xfer);
+
+ outbuf[buflen++] = 0x2; /* ASCII */
+ outbuf[buflen++] = 0; /* not officially assigned */
+ outbuf[buflen++] = 0; /* reserved */
+ outbuf[buflen++] = id_len; /* length of data following */
+ memcpy(outbuf + buflen, str, id_len);
+ buflen += id_len;
+
+ if (s->qdev.wwn) {
+ outbuf[buflen++] = 0x1; /* Binary */
+ outbuf[buflen++] = 0x3; /* NAA */
+ outbuf[buflen++] = 0; /* reserved */
+ outbuf[buflen++] = 8;
+ stq_be_p(&outbuf[buflen], s->qdev.wwn);
+ buflen += 8;
+ }
+
+ if (s->qdev.port_wwn) {
+ outbuf[buflen++] = 0x61; /* SAS / Binary */
+ outbuf[buflen++] = 0x93; /* PIV / Target port / NAA */
+ outbuf[buflen++] = 0; /* reserved */
+ outbuf[buflen++] = 8;
+ stq_be_p(&outbuf[buflen], s->qdev.port_wwn);
+ buflen += 8;
+ }
+
+ if (s->port_index) {
+ outbuf[buflen++] = 0x61; /* SAS / Binary */
+
+ /* PIV/Target port/relative target port */
+ outbuf[buflen++] = 0x94;
+
+ outbuf[buflen++] = 0; /* reserved */
+ outbuf[buflen++] = 4;
+ stw_be_p(&outbuf[buflen + 2], s->port_index);
+ buflen += 4;
+ }
+ break;
+ }
+ case 0xb0: /* block limits */
+ {
+ unsigned int unmap_sectors =
+ s->qdev.conf.discard_granularity / s->qdev.blocksize;
+ unsigned int min_io_size =
+ s->qdev.conf.min_io_size / s->qdev.blocksize;
+ unsigned int opt_io_size =
+ s->qdev.conf.opt_io_size / s->qdev.blocksize;
+ unsigned int max_unmap_sectors =
+ s->max_unmap_size / s->qdev.blocksize;
+ unsigned int max_io_sectors =
+ s->max_io_size / s->qdev.blocksize;
+
+ if (s->qdev.type == TYPE_ROM) {
+ DPRINTF("Inquiry (EVPD[%02X] not supported for CDROM\n",
+ page_code);
+ return -1;
+ }
+ if (s->qdev.type == TYPE_DISK) {
+ int max_transfer_blk = blk_get_max_transfer(s->qdev.conf.blk);
+ int max_io_sectors_blk =
+ max_transfer_blk / s->qdev.blocksize;
+
+ max_io_sectors =
+ MIN_NON_ZERO(max_io_sectors_blk, max_io_sectors);
+
+ /* min_io_size and opt_io_size can't be greater than
+ * max_io_sectors */
+ if (min_io_size) {
+ min_io_size = MIN(min_io_size, max_io_sectors);
+ }
+ if (opt_io_size) {
+ opt_io_size = MIN(opt_io_size, max_io_sectors);
+ }
+ }
+ /* required VPD size with unmap support */
+ buflen = 0x40;
+ memset(outbuf + 4, 0, buflen - 4);
+
+ outbuf[4] = 0x1; /* wsnz */
+
+ /* optimal transfer length granularity */
+ outbuf[6] = (min_io_size >> 8) & 0xff;
+ outbuf[7] = min_io_size & 0xff;
+
+ /* maximum transfer length */
+ outbuf[8] = (max_io_sectors >> 24) & 0xff;
+ outbuf[9] = (max_io_sectors >> 16) & 0xff;
+ outbuf[10] = (max_io_sectors >> 8) & 0xff;
+ outbuf[11] = max_io_sectors & 0xff;
+
+ /* optimal transfer length */
+ outbuf[12] = (opt_io_size >> 24) & 0xff;
+ outbuf[13] = (opt_io_size >> 16) & 0xff;
+ outbuf[14] = (opt_io_size >> 8) & 0xff;
+ outbuf[15] = opt_io_size & 0xff;
+
+ /* max unmap LBA count, default is 1GB */
+ outbuf[20] = (max_unmap_sectors >> 24) & 0xff;
+ outbuf[21] = (max_unmap_sectors >> 16) & 0xff;
+ outbuf[22] = (max_unmap_sectors >> 8) & 0xff;
+ outbuf[23] = max_unmap_sectors & 0xff;
+
+ /* max unmap descriptors, 255 fit in 4 kb with an 8-byte header */
+ outbuf[24] = 0;
+ outbuf[25] = 0;
+ outbuf[26] = 0;
+ outbuf[27] = 255;
+
+ /* optimal unmap granularity */
+ outbuf[28] = (unmap_sectors >> 24) & 0xff;
+ outbuf[29] = (unmap_sectors >> 16) & 0xff;
+ outbuf[30] = (unmap_sectors >> 8) & 0xff;
+ outbuf[31] = unmap_sectors & 0xff;
+
+ /* max write same size */
+ outbuf[36] = 0;
+ outbuf[37] = 0;
+ outbuf[38] = 0;
+ outbuf[39] = 0;
+
+ outbuf[40] = (max_io_sectors >> 24) & 0xff;
+ outbuf[41] = (max_io_sectors >> 16) & 0xff;
+ outbuf[42] = (max_io_sectors >> 8) & 0xff;
+ outbuf[43] = max_io_sectors & 0xff;
+ break;
+ }
+ case 0xb1: /* block device characteristics */
+ {
+ buflen = 8;
+ outbuf[4] = (s->rotation_rate >> 8) & 0xff;
+ outbuf[5] = s->rotation_rate & 0xff;
+ outbuf[6] = 0;
+ outbuf[7] = 0;
+ break;
+ }
+ case 0xb2: /* thin provisioning */
+ {
+ buflen = 8;
+ outbuf[4] = 0;
+ outbuf[5] = 0xe0; /* unmap & write_same 10/16 all supported */
+ outbuf[6] = s->qdev.conf.discard_granularity ? 2 : 1;
+ outbuf[7] = 0;
+ break;
+ }
+ default:
+ return -1;
+ }
+ /* done with EVPD */
+ assert(buflen - start <= 255);
+ outbuf[start - 1] = buflen - start;
+ return buflen;
+}
+
static int scsi_disk_emulate_inquiry(SCSIRequest *req, uint8_t *outbuf)
{
SCSIDiskState *s = DO_UPCAST(SCSIDiskState, qdev, req->dev);
int buflen = 0;
- int start;
if (req->cmd.buf[1] & 0x1) {
/* Vital product data */
- uint8_t page_code = req->cmd.buf[2];
-
- outbuf[buflen++] = s->qdev.type & 0x1f;
- outbuf[buflen++] = page_code ; // this page
- outbuf[buflen++] = 0x00;
- outbuf[buflen++] = 0x00;
- start = buflen;
-
- switch (page_code) {
- case 0x00: /* Supported page codes, mandatory */
- {
- DPRINTF("Inquiry EVPD[Supported pages] "
- "buffer size %zd\n", req->cmd.xfer);
- outbuf[buflen++] = 0x00; // list of supported pages (this page)
- if (s->serial) {
- outbuf[buflen++] = 0x80; // unit serial number
- }
- outbuf[buflen++] = 0x83; // device identification
- if (s->qdev.type == TYPE_DISK) {
- outbuf[buflen++] = 0xb0; // block limits
- outbuf[buflen++] = 0xb1; /* block device characteristics */
- outbuf[buflen++] = 0xb2; // thin provisioning
- }
- break;
- }
- case 0x80: /* Device serial number, optional */
- {
- int l;
-
- if (!s->serial) {
- DPRINTF("Inquiry (EVPD[Serial number] not supported\n");
- return -1;
- }
-
- l = strlen(s->serial);
- if (l > 36) {
- l = 36;
- }
-
- DPRINTF("Inquiry EVPD[Serial number] "
- "buffer size %zd\n", req->cmd.xfer);
- memcpy(outbuf+buflen, s->serial, l);
- buflen += l;
- break;
- }
-
- case 0x83: /* Device identification page, mandatory */
- {
- const char *str = s->serial ?: blk_name(s->qdev.conf.blk);
- int max_len = s->serial ? 20 : 255 - 8;
- int id_len = strlen(str);
-
- if (id_len > max_len) {
- id_len = max_len;
- }
- DPRINTF("Inquiry EVPD[Device identification] "
- "buffer size %zd\n", req->cmd.xfer);
-
- outbuf[buflen++] = 0x2; // ASCII
- outbuf[buflen++] = 0; // not officially assigned
- outbuf[buflen++] = 0; // reserved
- outbuf[buflen++] = id_len; // length of data following
- memcpy(outbuf+buflen, str, id_len);
- buflen += id_len;
-
- if (s->qdev.wwn) {
- outbuf[buflen++] = 0x1; // Binary
- outbuf[buflen++] = 0x3; // NAA
- outbuf[buflen++] = 0; // reserved
- outbuf[buflen++] = 8;
- stq_be_p(&outbuf[buflen], s->qdev.wwn);
- buflen += 8;
- }
-
- if (s->qdev.port_wwn) {
- outbuf[buflen++] = 0x61; // SAS / Binary
- outbuf[buflen++] = 0x93; // PIV / Target port / NAA
- outbuf[buflen++] = 0; // reserved
- outbuf[buflen++] = 8;
- stq_be_p(&outbuf[buflen], s->qdev.port_wwn);
- buflen += 8;
- }
-
- if (s->port_index) {
- outbuf[buflen++] = 0x61; // SAS / Binary
- outbuf[buflen++] = 0x94; // PIV / Target port / relative target port
- outbuf[buflen++] = 0; // reserved
- outbuf[buflen++] = 4;
- stw_be_p(&outbuf[buflen + 2], s->port_index);
- buflen += 4;
- }
- break;
- }
- case 0xb0: /* block limits */
- {
- unsigned int unmap_sectors =
- s->qdev.conf.discard_granularity / s->qdev.blocksize;
- unsigned int min_io_size =
- s->qdev.conf.min_io_size / s->qdev.blocksize;
- unsigned int opt_io_size =
- s->qdev.conf.opt_io_size / s->qdev.blocksize;
- unsigned int max_unmap_sectors =
- s->max_unmap_size / s->qdev.blocksize;
- unsigned int max_io_sectors =
- s->max_io_size / s->qdev.blocksize;
-
- if (s->qdev.type == TYPE_ROM) {
- DPRINTF("Inquiry (EVPD[%02X] not supported for CDROM\n",
- page_code);
- return -1;
- }
- if (s->qdev.type == TYPE_DISK) {
- int max_transfer_blk = blk_get_max_transfer(s->qdev.conf.blk);
- int max_io_sectors_blk =
- max_transfer_blk / s->qdev.blocksize;
-
- max_io_sectors =
- MIN_NON_ZERO(max_io_sectors_blk, max_io_sectors);
-
- /* min_io_size and opt_io_size can't be greater than
- * max_io_sectors */
- if (min_io_size) {
- min_io_size = MIN(min_io_size, max_io_sectors);
- }
- if (opt_io_size) {
- opt_io_size = MIN(opt_io_size, max_io_sectors);
- }
- }
- /* required VPD size with unmap support */
- buflen = 0x40;
- memset(outbuf + 4, 0, buflen - 4);
-
- outbuf[4] = 0x1; /* wsnz */
-
- /* optimal transfer length granularity */
- outbuf[6] = (min_io_size >> 8) & 0xff;
- outbuf[7] = min_io_size & 0xff;
-
- /* maximum transfer length */
- outbuf[8] = (max_io_sectors >> 24) & 0xff;
- outbuf[9] = (max_io_sectors >> 16) & 0xff;
- outbuf[10] = (max_io_sectors >> 8) & 0xff;
- outbuf[11] = max_io_sectors & 0xff;
-
- /* optimal transfer length */
- outbuf[12] = (opt_io_size >> 24) & 0xff;
- outbuf[13] = (opt_io_size >> 16) & 0xff;
- outbuf[14] = (opt_io_size >> 8) & 0xff;
- outbuf[15] = opt_io_size & 0xff;
-
- /* max unmap LBA count, default is 1GB */
- outbuf[20] = (max_unmap_sectors >> 24) & 0xff;
- outbuf[21] = (max_unmap_sectors >> 16) & 0xff;
- outbuf[22] = (max_unmap_sectors >> 8) & 0xff;
- outbuf[23] = max_unmap_sectors & 0xff;
-
- /* max unmap descriptors, 255 fit in 4 kb with an 8-byte header. */
- outbuf[24] = 0;
- outbuf[25] = 0;
- outbuf[26] = 0;
- outbuf[27] = 255;
-
- /* optimal unmap granularity */
- outbuf[28] = (unmap_sectors >> 24) & 0xff;
- outbuf[29] = (unmap_sectors >> 16) & 0xff;
- outbuf[30] = (unmap_sectors >> 8) & 0xff;
- outbuf[31] = unmap_sectors & 0xff;
-
- /* max write same size */
- outbuf[36] = 0;
- outbuf[37] = 0;
- outbuf[38] = 0;
- outbuf[39] = 0;
-
- outbuf[40] = (max_io_sectors >> 24) & 0xff;
- outbuf[41] = (max_io_sectors >> 16) & 0xff;
- outbuf[42] = (max_io_sectors >> 8) & 0xff;
- outbuf[43] = max_io_sectors & 0xff;
- break;
- }
- case 0xb1: /* block device characteristics */
- {
- buflen = 8;
- outbuf[4] = (s->rotation_rate >> 8) & 0xff;
- outbuf[5] = s->rotation_rate & 0xff;
- outbuf[6] = 0;
- outbuf[7] = 0;
- break;
- }
- case 0xb2: /* thin provisioning */
- {
- buflen = 8;
- outbuf[4] = 0;
- outbuf[5] = 0xe0; /* unmap & write_same 10/16 all supported */
- outbuf[6] = s->qdev.conf.discard_granularity ? 2 : 1;
- outbuf[7] = 0;
- break;
- }
- default:
- return -1;
- }
- /* done with EVPD */
- assert(buflen - start <= 255);
- outbuf[start - 1] = buflen - start;
- return buflen;
+ return scsi_disk_emulate_vpd_page(req, outbuf);
}
/* Standard INQUIRY data */
@@ -2569,8 +2578,6 @@
{
uint8_t cmd[16];
uint8_t buf[36];
- uint8_t sensebuf[8];
- sg_io_hdr_t io_header;
int ret;
memset(cmd, 0, sizeof(cmd));
@@ -2578,19 +2585,9 @@
cmd[0] = INQUIRY;
cmd[4] = sizeof(buf);
- memset(&io_header, 0, sizeof(io_header));
- io_header.interface_id = 'S';
- io_header.dxfer_direction = SG_DXFER_FROM_DEV;
- io_header.dxfer_len = sizeof(buf);
- io_header.dxferp = buf;
- io_header.cmdp = cmd;
- io_header.cmd_len = sizeof(cmd);
- io_header.mx_sb_len = sizeof(sensebuf);
- io_header.sbp = sensebuf;
- io_header.timeout = 6000; /* XXX */
-
- ret = blk_ioctl(s->qdev.conf.blk, SG_IO, &io_header);
- if (ret < 0 || io_header.driver_status || io_header.host_status) {
+ ret = scsi_SG_IO_FROM_DEV(s->qdev.conf.blk, cmd, sizeof(cmd),
+ buf, sizeof(buf));
+ if (ret < 0) {
return -1;
}
s->qdev.type = buf[0];
@@ -2648,7 +2645,7 @@
s->features |= (1 << SCSI_DISK_F_NO_REMOVABLE_DEVOPS);
scsi_realize(&s->qdev, errp);
- scsi_generic_read_device_identification(&s->qdev);
+ scsi_generic_read_device_inquiry(&s->qdev);
}
typedef struct SCSIBlockReq {
@@ -3039,6 +3036,10 @@
DEFINE_PROP_DRIVE("drive", SCSIDiskState, qdev.conf.blk),
DEFINE_PROP_BOOL("share-rw", SCSIDiskState, qdev.conf.share_rw, false),
DEFINE_PROP_UINT16("rotation_rate", SCSIDiskState, rotation_rate, 0),
+ DEFINE_PROP_UINT64("max_unmap_size", SCSIDiskState, max_unmap_size,
+ DEFAULT_MAX_UNMAP_SIZE),
+ DEFINE_PROP_UINT64("max_io_size", SCSIDiskState, max_io_size,
+ DEFAULT_MAX_IO_SIZE),
DEFINE_PROP_INT32("scsi_version", SCSIDiskState, qdev.default_scsi_version,
-1),
DEFINE_PROP_END_OF_LIST(),
diff --git a/hw/scsi/scsi-generic.c b/hw/scsi/scsi-generic.c
index 03bce8f..d60c4d0 100644
--- a/hw/scsi/scsi-generic.c
+++ b/hw/scsi/scsi-generic.c
@@ -142,10 +142,84 @@
return 0;
}
+static void scsi_handle_inquiry_reply(SCSIGenericReq *r, SCSIDevice *s)
+{
+ uint8_t page, page_len;
+
+ /*
+ * EVPD set to zero returns the standard INQUIRY data.
+ *
+ * Check if scsi_version is unset (-1) to avoid re-defining it
+ * each time an INQUIRY with standard data is received.
+ * scsi_version is initialized with -1 in scsi_generic_reset
+ * and scsi_disk_reset, making sure that we'll set the
+ * scsi_version after a reset. If the version field of the
+ * INQUIRY response somehow changes after a guest reboot,
+ * we'll be able to keep track of it.
+ *
+ * On SCSI-2 and older, first 3 bits of byte 2 is the
+ * ANSI-approved version, while on later versions the
+ * whole byte 2 contains the version. Check if we're dealing
+ * with a newer version and, in that case, assign the
+ * whole byte.
+ */
+ if (s->scsi_version == -1 && !(r->req.cmd.buf[1] & 0x01)) {
+ s->scsi_version = r->buf[2] & 0x07;
+ if (s->scsi_version > 2) {
+ s->scsi_version = r->buf[2];
+ }
+ }
+
+ if (s->type == TYPE_DISK && (r->req.cmd.buf[1] & 0x01)) {
+ page = r->req.cmd.buf[2];
+ if (page == 0xb0) {
+ uint32_t max_transfer =
+ blk_get_max_transfer(s->conf.blk) / s->blocksize;
+
+ assert(max_transfer);
+ stl_be_p(&r->buf[8], max_transfer);
+ /* Also take care of the opt xfer len. */
+ stl_be_p(&r->buf[12],
+ MIN_NON_ZERO(max_transfer, ldl_be_p(&r->buf[12])));
+ } else if (page == 0x00 && s->needs_vpd_bl_emulation) {
+ /*
+ * Now we're capable of supplying the VPD Block Limits
+ * response if the hardware can't. Add it in the INQUIRY
+ * Supported VPD pages response in case we are using the
+ * emulation for this device.
+ *
+ * This way, the guest kernel will be aware of the support
+ * and will use it to proper setup the SCSI device.
+ */
+ page_len = r->buf[3];
+ r->buf[page_len + 4] = 0xb0;
+ r->buf[3] = ++page_len;
+ }
+ }
+}
+
+static int scsi_emulate_block_limits(SCSIGenericReq *r)
+{
+ r->buflen = scsi_disk_emulate_vpd_page(&r->req, r->buf);
+ r->io_header.sb_len_wr = 0;
+
+ /*
+ * We have valid contents in the reply buffer but the
+ * io_header can report a sense error coming from
+ * the hardware in scsi_command_complete_noio. Clean
+ * up the io_header to avoid reporting it.
+ */
+ r->io_header.driver_status = 0;
+ r->io_header.status = 0;
+
+ return r->buflen;
+}
+
static void scsi_read_complete(void * opaque, int ret)
{
SCSIGenericReq *r = (SCSIGenericReq *)opaque;
SCSIDevice *s = r->req.dev;
+ SCSISense sense;
int len;
assert(r->req.aiocb != NULL);
@@ -162,6 +236,27 @@
DPRINTF("Data ready tag=0x%x len=%d\n", r->req.tag, len);
r->len = -1;
+
+ /*
+ * Check if this is a VPD Block Limits request that
+ * resulted in sense error but would need emulation.
+ * In this case, emulate a valid VPD response.
+ */
+ if (s->needs_vpd_bl_emulation) {
+ int is_vpd_bl = r->req.cmd.buf[0] == INQUIRY &&
+ r->req.cmd.buf[1] & 0x01 &&
+ r->req.cmd.buf[2] == 0xb0;
+
+ if (is_vpd_bl && sg_io_sense_from_errno(-ret, &r->io_header, &sense)) {
+ len = scsi_emulate_block_limits(r);
+ /*
+ * No need to let scsi_read_complete go on and handle an
+ * INQUIRY VPD BL request we created manually.
+ */
+ goto req_complete;
+ }
+ }
+
if (len == 0) {
scsi_command_complete_noio(r, 0);
goto done;
@@ -194,40 +289,10 @@
}
}
if (r->req.cmd.buf[0] == INQUIRY) {
- /*
- * EVPD set to zero returns the standard INQUIRY data.
- *
- * Check if scsi_version is unset (-1) to avoid re-defining it
- * each time an INQUIRY with standard data is received.
- * scsi_version is initialized with -1 in scsi_generic_reset
- * and scsi_disk_reset, making sure that we'll set the
- * scsi_version after a reset. If the version field of the
- * INQUIRY response somehow changes after a guest reboot,
- * we'll be able to keep track of it.
- *
- * On SCSI-2 and older, first 3 bits of byte 2 is the
- * ANSI-approved version, while on later versions the
- * whole byte 2 contains the version. Check if we're dealing
- * with a newer version and, in that case, assign the
- * whole byte.
- */
- if (s->scsi_version == -1 && !(r->req.cmd.buf[1] & 0x01)) {
- s->scsi_version = r->buf[2] & 0x07;
- if (s->scsi_version > 2) {
- s->scsi_version = r->buf[2];
- }
- }
- if (s->type == TYPE_DISK && r->req.cmd.buf[2] == 0xb0) {
- uint32_t max_transfer =
- blk_get_max_transfer(s->conf.blk) / s->blocksize;
-
- assert(max_transfer);
- stl_be_p(&r->buf[8], max_transfer);
- /* Also take care of the opt xfer len. */
- stl_be_p(&r->buf[12],
- MIN_NON_ZERO(max_transfer, ldl_be_p(&r->buf[12])));
- }
+ scsi_handle_inquiry_reply(r, s);
}
+
+req_complete:
scsi_req_data(&r->req, len);
scsi_req_unref(&r->req);
@@ -404,12 +469,77 @@
return -EINVAL;
}
-void scsi_generic_read_device_identification(SCSIDevice *s)
+int scsi_SG_IO_FROM_DEV(BlockBackend *blk, uint8_t *cmd, uint8_t cmd_size,
+ uint8_t *buf, uint8_t buf_size)
+{
+ sg_io_hdr_t io_header;
+ uint8_t sensebuf[8];
+ int ret;
+
+ memset(&io_header, 0, sizeof(io_header));
+ io_header.interface_id = 'S';
+ io_header.dxfer_direction = SG_DXFER_FROM_DEV;
+ io_header.dxfer_len = buf_size;
+ io_header.dxferp = buf;
+ io_header.cmdp = cmd;
+ io_header.cmd_len = cmd_size;
+ io_header.mx_sb_len = sizeof(sensebuf);
+ io_header.sbp = sensebuf;
+ io_header.timeout = 6000; /* XXX */
+
+ ret = blk_ioctl(blk, SG_IO, &io_header);
+ if (ret < 0 || io_header.driver_status || io_header.host_status) {
+ return -1;
+ }
+ return 0;
+}
+
+/*
+ * Executes an INQUIRY request with EVPD set to retrieve the
+ * available VPD pages of the device. If the device does
+ * not support the Block Limits page (page 0xb0), set
+ * the needs_vpd_bl_emulation flag for future use.
+ */
+static void scsi_generic_set_vpd_bl_emulation(SCSIDevice *s)
{
uint8_t cmd[6];
uint8_t buf[250];
- uint8_t sensebuf[8];
- sg_io_hdr_t io_header;
+ uint8_t page_len;
+ int ret, i;
+
+ memset(cmd, 0, sizeof(cmd));
+ memset(buf, 0, sizeof(buf));
+ cmd[0] = INQUIRY;
+ cmd[1] = 1;
+ cmd[2] = 0x00;
+ cmd[4] = sizeof(buf);
+
+ ret = scsi_SG_IO_FROM_DEV(s->conf.blk, cmd, sizeof(cmd),
+ buf, sizeof(buf));
+ if (ret < 0) {
+ /*
+ * Do not assume anything if we can't retrieve the
+ * INQUIRY response to assert the VPD Block Limits
+ * support.
+ */
+ s->needs_vpd_bl_emulation = false;
+ return;
+ }
+
+ page_len = buf[3];
+ for (i = 4; i < page_len + 4; i++) {
+ if (buf[i] == 0xb0) {
+ s->needs_vpd_bl_emulation = false;
+ return;
+ }
+ }
+ s->needs_vpd_bl_emulation = true;
+}
+
+static void scsi_generic_read_device_identification(SCSIDevice *s)
+{
+ uint8_t cmd[6];
+ uint8_t buf[250];
int ret;
int i, len;
@@ -420,19 +550,9 @@
cmd[2] = 0x83;
cmd[4] = sizeof(buf);
- memset(&io_header, 0, sizeof(io_header));
- io_header.interface_id = 'S';
- io_header.dxfer_direction = SG_DXFER_FROM_DEV;
- io_header.dxfer_len = sizeof(buf);
- io_header.dxferp = buf;
- io_header.cmdp = cmd;
- io_header.cmd_len = sizeof(cmd);
- io_header.mx_sb_len = sizeof(sensebuf);
- io_header.sbp = sensebuf;
- io_header.timeout = 6000; /* XXX */
-
- ret = blk_ioctl(s->conf.blk, SG_IO, &io_header);
- if (ret < 0 || io_header.driver_status || io_header.host_status) {
+ ret = scsi_SG_IO_FROM_DEV(s->conf.blk, cmd, sizeof(cmd),
+ buf, sizeof(buf));
+ if (ret < 0) {
return;
}
@@ -461,12 +581,20 @@
}
}
+void scsi_generic_read_device_inquiry(SCSIDevice *s)
+{
+ scsi_generic_read_device_identification(s);
+ if (s->type == TYPE_DISK) {
+ scsi_generic_set_vpd_bl_emulation(s);
+ } else {
+ s->needs_vpd_bl_emulation = false;
+ }
+}
+
static int get_stream_blocksize(BlockBackend *blk)
{
uint8_t cmd[6];
uint8_t buf[12];
- uint8_t sensebuf[8];
- sg_io_hdr_t io_header;
int ret;
memset(cmd, 0, sizeof(cmd));
@@ -474,21 +602,11 @@
cmd[0] = MODE_SENSE;
cmd[4] = sizeof(buf);
- memset(&io_header, 0, sizeof(io_header));
- io_header.interface_id = 'S';
- io_header.dxfer_direction = SG_DXFER_FROM_DEV;
- io_header.dxfer_len = sizeof(buf);
- io_header.dxferp = buf;
- io_header.cmdp = cmd;
- io_header.cmd_len = sizeof(cmd);
- io_header.mx_sb_len = sizeof(sensebuf);
- io_header.sbp = sensebuf;
- io_header.timeout = 6000; /* XXX */
-
- ret = blk_ioctl(blk, SG_IO, &io_header);
- if (ret < 0 || io_header.driver_status || io_header.host_status) {
+ ret = scsi_SG_IO_FROM_DEV(blk, cmd, sizeof(cmd), buf, sizeof(buf));
+ if (ret < 0) {
return -1;
}
+
return (buf[9] << 16) | (buf[10] << 8) | buf[11];
}
@@ -574,7 +692,7 @@
/* Only used by scsi-block, but initialize it nevertheless to be clean. */
s->default_scsi_version = -1;
- scsi_generic_read_device_identification(s);
+ scsi_generic_read_device_inquiry(s);
}
const SCSIReqOps scsi_generic_req_ops = {
diff --git a/hw/scsi/virtio-scsi-dataplane.c b/hw/scsi/virtio-scsi-dataplane.c
index 912e500..b995bab 100644
--- a/hw/scsi/virtio-scsi-dataplane.c
+++ b/hw/scsi/virtio-scsi-dataplane.c
@@ -142,8 +142,8 @@
/* Set up guest notifier (irq) */
rc = k->set_guest_notifiers(qbus->parent, vs->conf.num_queues + 2, true);
if (rc != 0) {
- fprintf(stderr, "virtio-scsi: Failed to set guest notifiers (%d), "
- "ensure -enable-kvm is set\n", rc);
+ error_report("virtio-scsi: Failed to set guest notifiers (%d), "
+ "ensure -accel kvm is set.", rc);
goto fail_guest_notifiers;
}
diff --git a/hw/sd/bcm2835_sdhost.c b/hw/sd/bcm2835_sdhost.c
index ebf3b92..4df4de7 100644
--- a/hw/sd/bcm2835_sdhost.c
+++ b/hw/sd/bcm2835_sdhost.c
@@ -118,8 +118,6 @@
goto error;
}
if (!(s->cmd & SDCMD_NO_RESPONSE)) {
-#define RWORD(n) (((uint32_t)rsp[n] << 24) | (rsp[n + 1] << 16) \
- | (rsp[n + 2] << 8) | rsp[n + 3])
if (rlen == 0 || (rlen == 4 && (s->cmd & SDCMD_LONG_RESPONSE))) {
goto error;
}
@@ -127,15 +125,14 @@
goto error;
}
if (rlen == 4) {
- s->rsp[0] = RWORD(0);
+ s->rsp[0] = ldl_be_p(&rsp[0]);
s->rsp[1] = s->rsp[2] = s->rsp[3] = 0;
} else {
- s->rsp[0] = RWORD(12);
- s->rsp[1] = RWORD(8);
- s->rsp[2] = RWORD(4);
- s->rsp[3] = RWORD(0);
+ s->rsp[0] = ldl_be_p(&rsp[12]);
+ s->rsp[1] = ldl_be_p(&rsp[8]);
+ s->rsp[2] = ldl_be_p(&rsp[4]);
+ s->rsp[3] = ldl_be_p(&rsp[0]);
}
-#undef RWORD
}
/* We never really delay commands, so if this was a 'busywait' command
* then we've completed it now and can raise the interrupt.
diff --git a/hw/sd/core.c b/hw/sd/core.c
index 820345f..107e6d7 100644
--- a/hw/sd/core.c
+++ b/hw/sd/core.c
@@ -91,7 +91,7 @@
{
SDState *card = get_card(sdbus);
- trace_sdbus_command(sdbus_name(sdbus), req->cmd, req->arg, req->crc);
+ trace_sdbus_command(sdbus_name(sdbus), req->cmd, req->arg);
if (card) {
SDCardClass *sc = SD_CARD_GET_CLASS(card);
diff --git a/hw/sd/milkymist-memcard.c b/hw/sd/milkymist-memcard.c
index fcbccf5..df42aa1 100644
--- a/hw/sd/milkymist-memcard.c
+++ b/hw/sd/milkymist-memcard.c
@@ -100,8 +100,7 @@
SDRequest req;
req.cmd = s->command[0] & 0x3f;
- req.arg = (s->command[1] << 24) | (s->command[2] << 16)
- | (s->command[3] << 8) | s->command[4];
+ req.arg = ldl_be_p(s->command + 1);
req.crc = s->command[5];
s->response[0] = req.cmd;
diff --git a/hw/sd/omap_mmc.c b/hw/sd/omap_mmc.c
index aa2a816..671264b 100644
--- a/hw/sd/omap_mmc.c
+++ b/hw/sd/omap_mmc.c
@@ -163,8 +163,7 @@
CID_CSD_OVERWRITE;
if (host->sdio & (1 << 13))
mask |= AKE_SEQ_ERROR;
- rspstatus = (response[0] << 24) | (response[1] << 16) |
- (response[2] << 8) | (response[3] << 0);
+ rspstatus = ldl_be_p(response);
break;
case sd_r2:
@@ -182,8 +181,7 @@
}
rsplen = 4;
- rspstatus = (response[0] << 24) | (response[1] << 16) |
- (response[2] << 8) | (response[3] << 0);
+ rspstatus = ldl_be_p(response);
if (rspstatus & 0x80000000)
host->status &= 0xe000;
else
diff --git a/hw/sd/pl181.c b/hw/sd/pl181.c
index 1cc94db..3ad7e92 100644
--- a/hw/sd/pl181.c
+++ b/hw/sd/pl181.c
@@ -182,23 +182,20 @@
if (rlen < 0)
goto error;
if (s->cmd & PL181_CMD_RESPONSE) {
-#define RWORD(n) (((uint32_t)response[n] << 24) | (response[n + 1] << 16) \
- | (response[n + 2] << 8) | response[n + 3])
if (rlen == 0 || (rlen == 4 && (s->cmd & PL181_CMD_LONGRESP)))
goto error;
if (rlen != 4 && rlen != 16)
goto error;
- s->response[0] = RWORD(0);
+ s->response[0] = ldl_be_p(&response[0]);
if (rlen == 4) {
s->response[1] = s->response[2] = s->response[3] = 0;
} else {
- s->response[1] = RWORD(4);
- s->response[2] = RWORD(8);
- s->response[3] = RWORD(12) & ~1;
+ s->response[1] = ldl_be_p(&response[4]);
+ s->response[2] = ldl_be_p(&response[8]);
+ s->response[3] = ldl_be_p(&response[12]) & ~1;
}
DPRINTF("Response received\n");
s->status |= PL181_STATUS_CMDRESPEND;
-#undef RWORD
} else {
DPRINTF("Command sent\n");
s->status |= PL181_STATUS_CMDSENT;
diff --git a/hw/sd/sdhci.c b/hw/sd/sdhci.c
index 3017e5a..321d02d 100644
--- a/hw/sd/sdhci.c
+++ b/hw/sd/sdhci.c
@@ -342,17 +342,13 @@
if (s->cmdreg & SDHC_CMD_RESPONSE) {
if (rlen == 4) {
- s->rspreg[0] = (response[0] << 24) | (response[1] << 16) |
- (response[2] << 8) | response[3];
+ s->rspreg[0] = ldl_be_p(response);
s->rspreg[1] = s->rspreg[2] = s->rspreg[3] = 0;
trace_sdhci_response4(s->rspreg[0]);
} else if (rlen == 16) {
- s->rspreg[0] = (response[11] << 24) | (response[12] << 16) |
- (response[13] << 8) | response[14];
- s->rspreg[1] = (response[7] << 24) | (response[8] << 16) |
- (response[9] << 8) | response[10];
- s->rspreg[2] = (response[3] << 24) | (response[4] << 16) |
- (response[5] << 8) | response[6];
+ s->rspreg[0] = ldl_be_p(&response[11]);
+ s->rspreg[1] = ldl_be_p(&response[7]);
+ s->rspreg[2] = ldl_be_p(&response[3]);
s->rspreg[3] = (response[0] << 16) | (response[1] << 8) |
response[2];
trace_sdhci_response16(s->rspreg[3], s->rspreg[2],
@@ -396,8 +392,7 @@
trace_sdhci_end_transfer(request.cmd, request.arg);
sdbus_do_command(&s->sdbus, &request, response);
/* Auto CMD12 response goes to the upper Response register */
- s->rspreg[3] = (response[0] << 24) | (response[1] << 16) |
- (response[2] << 8) | response[3];
+ s->rspreg[3] = ldl_be_p(response);
}
s->prnsts &= ~(SDHC_DOING_READ | SDHC_DOING_WRITE |
diff --git a/hw/sd/ssi-sd.c b/hw/sd/ssi-sd.c
index 96542ec..95a143b 100644
--- a/hw/sd/ssi-sd.c
+++ b/hw/sd/ssi-sd.c
@@ -96,8 +96,7 @@
uint8_t longresp[16];
/* FIXME: Check CRC. */
request.cmd = s->cmd;
- request.arg = (s->cmdarg[0] << 24) | (s->cmdarg[1] << 16)
- | (s->cmdarg[2] << 8) | s->cmdarg[3];
+ request.arg = ldl_be_p(s->cmdarg);
DPRINTF("CMD%d arg 0x%08x\n", s->cmd, request.arg);
s->arglen = sdbus_do_command(&s->sdbus, &request, longresp);
if (s->arglen <= 0) {
@@ -122,8 +121,7 @@
/* CMD13 returns a 2-byte statuse work. Other commands
only return the first byte. */
s->arglen = (s->cmd == 13) ? 2 : 1;
- cardstatus = (longresp[0] << 24) | (longresp[1] << 16)
- | (longresp[2] << 8) | longresp[3];
+ cardstatus = ldl_be_p(longresp);
status = 0;
if (((cardstatus >> 9) & 0xf) < 4)
status |= SSI_SDR_IDLE;
diff --git a/hw/sd/trace-events b/hw/sd/trace-events
index d7880bc..fb0615c 100644
--- a/hw/sd/trace-events
+++ b/hw/sd/trace-events
@@ -7,7 +7,7 @@
bcm2835_sdhost_update_irq(uint32_t irq) "IRQ bits 0x%x\n"
# hw/sd/core.c
-sdbus_command(const char *bus_name, uint8_t cmd, uint32_t arg, uint8_t crc) "@%s CMD%02d arg 0x%08x crc 0x%02x"
+sdbus_command(const char *bus_name, uint8_t cmd, uint32_t arg) "@%s CMD%02d arg 0x%08x"
sdbus_read(const char *bus_name, uint8_t value) "@%s value 0x%02x"
sdbus_write(const char *bus_name, uint8_t value) "@%s value 0x%02x"
sdbus_set_voltage(const char *bus_name, uint16_t millivolts) "@%s %u (mV)"
diff --git a/hw/usb/dev-storage.c b/hw/usb/dev-storage.c
index c99398b..cd5551d 100644
--- a/hw/usb/dev-storage.c
+++ b/hw/usb/dev-storage.c
@@ -625,6 +625,7 @@
&usb_msd_scsi_info_storage, NULL);
scsi_dev = scsi_bus_legacy_add_drive(&s->bus, blk, 0, !!s->removable,
s->conf.bootindex, s->conf.share_rw,
+ s->conf.rerror, s->conf.werror,
dev->serial,
errp);
blk_unref(blk);
@@ -671,6 +672,7 @@
static Property msd_properties[] = {
DEFINE_BLOCK_PROPERTIES(MSDState, conf),
+ DEFINE_BLOCK_ERROR_PROPERTIES(MSDState, conf),
DEFINE_PROP_BIT("removable", MSDState, removable, 0, false),
DEFINE_PROP_END_OF_LIST(),
};
diff --git a/include/block/block.h b/include/block/block.h
index b1d6fdb..2ffc1c6 100644
--- a/include/block/block.h
+++ b/include/block/block.h
@@ -285,10 +285,6 @@
int bdrv_pwritev(BdrvChild *child, int64_t offset, QEMUIOVector *qiov);
int bdrv_pwrite_sync(BdrvChild *child, int64_t offset,
const void *buf, int count);
-int coroutine_fn bdrv_co_readv(BdrvChild *child, int64_t sector_num,
- int nb_sectors, QEMUIOVector *qiov);
-int coroutine_fn bdrv_co_writev(BdrvChild *child, int64_t sector_num,
- int nb_sectors, QEMUIOVector *qiov);
/*
* Efficiently zero a region of the disk image. Note that this is a regular
* I/O request like read or write and should have a reasonable size. This
@@ -300,8 +296,12 @@
BlockDriverState *bdrv_find_backing_image(BlockDriverState *bs,
const char *backing_file);
void bdrv_refresh_filename(BlockDriverState *bs);
+
+int coroutine_fn bdrv_co_truncate(BdrvChild *child, int64_t offset,
+ PreallocMode prealloc, Error **errp);
int bdrv_truncate(BdrvChild *child, int64_t offset, PreallocMode prealloc,
Error **errp);
+
int64_t bdrv_nb_sectors(BlockDriverState *bs);
int64_t bdrv_getlength(BlockDriverState *bs);
int64_t bdrv_get_allocated_file_size(BlockDriverState *bs);
diff --git a/include/block/block_int.h b/include/block/block_int.h
index 74646ed..af71b41 100644
--- a/include/block/block_int.h
+++ b/include/block/block_int.h
@@ -63,6 +63,7 @@
BDRV_TRACKED_READ,
BDRV_TRACKED_WRITE,
BDRV_TRACKED_DISCARD,
+ BDRV_TRACKED_TRUNCATE,
};
typedef struct BdrvTrackedRequest {
@@ -289,8 +290,8 @@
* bdrv_parse_filename.
*/
const char *protocol_name;
- int (*bdrv_truncate)(BlockDriverState *bs, int64_t offset,
- PreallocMode prealloc, Error **errp);
+ int coroutine_fn (*bdrv_co_truncate)(BlockDriverState *bs, int64_t offset,
+ PreallocMode prealloc, Error **errp);
int64_t (*bdrv_getlength)(BlockDriverState *bs);
bool has_variable_length;
@@ -1157,4 +1158,6 @@
BdrvChild *dst, uint64_t dst_offset,
uint64_t bytes, BdrvRequestFlags flags);
+int refresh_total_sectors(BlockDriverState *bs, int64_t hint);
+
#endif /* BLOCK_INT_H */
diff --git a/include/block/raw-aio.h b/include/block/raw-aio.h
index 8d698cc..6799614 100644
--- a/include/block/raw-aio.h
+++ b/include/block/raw-aio.h
@@ -26,6 +26,7 @@
#define QEMU_AIO_DISCARD 0x0010
#define QEMU_AIO_WRITE_ZEROES 0x0020
#define QEMU_AIO_COPY_RANGE 0x0040
+#define QEMU_AIO_TRUNCATE 0x0080
#define QEMU_AIO_TYPE_MASK \
(QEMU_AIO_READ | \
QEMU_AIO_WRITE | \
@@ -33,7 +34,8 @@
QEMU_AIO_FLUSH | \
QEMU_AIO_DISCARD | \
QEMU_AIO_WRITE_ZEROES | \
- QEMU_AIO_COPY_RANGE)
+ QEMU_AIO_COPY_RANGE | \
+ QEMU_AIO_TRUNCATE)
/* AIO flags */
#define QEMU_AIO_MISALIGNED 0x1000
diff --git a/include/exec/exec-all.h b/include/exec/exec-all.h
index 25a6f28..6a7e7a8 100644
--- a/include/exec/exec-all.h
+++ b/include/exec/exec-all.h
@@ -299,14 +299,14 @@
static inline void tlb_flush_by_mmuidx_all_cpus(CPUState *cpu, uint16_t idxmap)
{
}
+
static inline void tlb_flush_by_mmuidx_all_cpus_synced(CPUState *cpu,
uint16_t idxmap)
{
}
-static inline void tb_invalidate_phys_addr(AddressSpace *as, hwaddr addr,
- MemTxAttrs attrs)
-{
-}
+
+void tb_invalidate_phys_addr(target_ulong addr);
+void tb_invalidate_phys_range(target_ulong start, target_ulong end);
#endif
#define CODE_GEN_ALIGN 16 /* must be >= of the size of a icache line */
diff --git a/include/exec/memory.h b/include/exec/memory.h
index 050323f..448d41a 100644
--- a/include/exec/memory.h
+++ b/include/exec/memory.h
@@ -1665,7 +1665,7 @@
void memory_global_dirty_log_stop(void);
void mtree_info(fprintf_function mon_printf, void *f, bool flatview,
- bool dispatch_tree);
+ bool dispatch_tree, bool owner);
/**
* memory_region_request_mmio_ptr: request a pointer to an mmio
diff --git a/include/exec/ram_addr.h b/include/exec/ram_addr.h
index 33c361c..cf4ce06 100644
--- a/include/exec/ram_addr.h
+++ b/include/exec/ram_addr.h
@@ -93,6 +93,8 @@
#define DIRTY_CLIENTS_ALL ((1 << DIRTY_MEMORY_NUM) - 1)
#define DIRTY_CLIENTS_NOCODE (DIRTY_CLIENTS_ALL & ~(1 << DIRTY_MEMORY_CODE))
+void tb_invalidate_phys_range(ram_addr_t start, ram_addr_t end);
+
static inline bool cpu_physical_memory_get_dirty(ram_addr_t start,
ram_addr_t length,
unsigned client)
diff --git a/include/glib-compat.h b/include/glib-compat.h
index c49cf87..fdf95a2 100644
--- a/include/glib-compat.h
+++ b/include/glib-compat.h
@@ -16,28 +16,73 @@
#ifndef QEMU_GLIB_COMPAT_H
#define QEMU_GLIB_COMPAT_H
+/* Ask for warnings for anything that was marked deprecated in
+ * the defined version, or before. It is a candidate for rewrite.
+ */
+#define GLIB_VERSION_MIN_REQUIRED GLIB_VERSION_2_40
+
+/* Ask for warnings if code tries to use function that did not
+ * exist in the defined version. These risk breaking builds
+ */
+#define GLIB_VERSION_MAX_ALLOWED GLIB_VERSION_2_40
+
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
+
#include <glib.h>
-/* GLIB version compatibility flags */
-#if !GLIB_CHECK_VERSION(2, 26, 0)
-#define G_TIME_SPAN_SECOND (G_GINT64_CONSTANT(1000000))
-#endif
+/*
+ * Note that because of the GLIB_VERSION_MAX_ALLOWED constant above, allowing
+ * use of functions from newer GLib via this compat header needs a little
+ * trickery to prevent warnings being emitted.
+ *
+ * Consider a function from newer glib-X.Y that we want to use
+ *
+ * int g_foo(const char *wibble)
+ *
+ * We must define a static inline function with the same signature that does
+ * what we need, but with a "_qemu" suffix e.g.
+ *
+ * static inline void g_foo_qemu(const char *wibble)
+ * {
+ * #if GLIB_CHECK_VERSION(X, Y, 0)
+ * g_foo(wibble)
+ * #else
+ * g_something_equivalent_in_older_glib(wibble);
+ * #endif
+ * }
+ *
+ * The #pragma at the top of this file turns off -Wdeprecated-declarations,
+ * ensuring this wrapper function impl doesn't trigger the compiler warning
+ * about using too new glib APIs. Finally we can do
+ *
+ * #define g_foo(a) g_foo_qemu(a)
+ *
+ * So now the code elsewhere in QEMU, which *does* have the
+ * -Wdeprecated-declarations warning active, can call g_foo(...) as normal,
+ * without generating warnings.
+ */
-#if !GLIB_CHECK_VERSION(2, 28, 0)
-static inline gint64 qemu_g_get_monotonic_time(void)
+static inline gboolean g_strv_contains_qemu(const gchar *const *strv,
+ const gchar *str)
{
- /* g_get_monotonic_time() is best-effort so we can use the wall clock as a
- * fallback.
- */
+#if GLIB_CHECK_VERSION(2, 44, 0)
+ return g_strv_contains(strv, str);
+#else
+ g_return_val_if_fail(strv != NULL, FALSE);
+ g_return_val_if_fail(str != NULL, FALSE);
- GTimeVal time;
- g_get_current_time(&time);
+ for (; *strv != NULL; strv++) {
+ if (g_str_equal(str, *strv)) {
+ return TRUE;
+ }
+ }
- return time.tv_sec * G_TIME_SPAN_SECOND + time.tv_usec;
-}
-/* work around distro backports of this interface */
-#define g_get_monotonic_time() qemu_g_get_monotonic_time()
+ return FALSE;
#endif
+}
+#define g_strv_contains(a, b) g_strv_contains_qemu(a, b)
+
#if defined(_WIN32) && !GLIB_CHECK_VERSION(2, 50, 0)
/*
@@ -48,228 +93,6 @@
gint g_poll_fixed(GPollFD *fds, guint nfds, gint timeout);
#endif
-#if !GLIB_CHECK_VERSION(2, 30, 0)
-/* Not a 100% compatible implementation, but good enough for most
- * cases. Placeholders are only supported at the end of the
- * template. */
-static inline gchar *qemu_g_dir_make_tmp(gchar const *tmpl, GError **error)
-{
- gchar *path = g_build_filename(g_get_tmp_dir(), tmpl ?: ".XXXXXX", NULL);
-
- if (mkdtemp(path) != NULL) {
- return path;
- }
- /* Error occurred, clean up. */
- g_set_error(error, G_FILE_ERROR, g_file_error_from_errno(errno),
- "mkdtemp() failed");
- g_free(path);
- return NULL;
-}
-#define g_dir_make_tmp(tmpl, error) qemu_g_dir_make_tmp(tmpl, error)
-#endif /* glib 2.30 */
-
-#if !GLIB_CHECK_VERSION(2, 31, 0)
-/* before glib-2.31, GMutex and GCond was dynamic-only (there was a separate
- * GStaticMutex, but it didn't work with condition variables).
- *
- * Our implementation uses GOnce to fake a static implementation that does
- * not require separate initialization.
- * We need to rename the types to avoid passing our CompatGMutex/CompatGCond
- * by mistake to a function that expects GMutex/GCond. However, for ease
- * of use we keep the GLib function names. GLib uses macros for the
- * implementation, we use inline functions instead and undefine the macros.
- */
-
-typedef struct CompatGMutex {
- GOnce once;
-} CompatGMutex;
-
-typedef struct CompatGCond {
- GOnce once;
-} CompatGCond;
-
-static inline gpointer do_g_mutex_new(gpointer unused)
-{
- return (gpointer) g_mutex_new();
-}
-
-static inline void g_mutex_init(CompatGMutex *mutex)
-{
- mutex->once = (GOnce) G_ONCE_INIT;
-}
-
-static inline void g_mutex_clear(CompatGMutex *mutex)
-{
- g_assert(mutex->once.status != G_ONCE_STATUS_PROGRESS);
- if (mutex->once.retval) {
- g_mutex_free((GMutex *) mutex->once.retval);
- }
- mutex->once = (GOnce) G_ONCE_INIT;
-}
-
-static inline void (g_mutex_lock)(CompatGMutex *mutex)
-{
- g_once(&mutex->once, do_g_mutex_new, NULL);
- g_mutex_lock((GMutex *) mutex->once.retval);
-}
-#undef g_mutex_lock
-
-static inline gboolean (g_mutex_trylock)(CompatGMutex *mutex)
-{
- g_once(&mutex->once, do_g_mutex_new, NULL);
- return g_mutex_trylock((GMutex *) mutex->once.retval);
-}
-#undef g_mutex_trylock
-
-
-static inline void (g_mutex_unlock)(CompatGMutex *mutex)
-{
- g_mutex_unlock((GMutex *) mutex->once.retval);
-}
-#undef g_mutex_unlock
-
-static inline gpointer do_g_cond_new(gpointer unused)
-{
- return (gpointer) g_cond_new();
-}
-
-static inline void g_cond_init(CompatGCond *cond)
-{
- cond->once = (GOnce) G_ONCE_INIT;
-}
-
-static inline void g_cond_clear(CompatGCond *cond)
-{
- g_assert(cond->once.status != G_ONCE_STATUS_PROGRESS);
- if (cond->once.retval) {
- g_cond_free((GCond *) cond->once.retval);
- }
- cond->once = (GOnce) G_ONCE_INIT;
-}
-
-static inline void (g_cond_wait)(CompatGCond *cond, CompatGMutex *mutex)
-{
- g_assert(mutex->once.status != G_ONCE_STATUS_PROGRESS);
- g_once(&cond->once, do_g_cond_new, NULL);
- g_cond_wait((GCond *) cond->once.retval, (GMutex *) mutex->once.retval);
-}
-#undef g_cond_wait
-
-static inline void (g_cond_broadcast)(CompatGCond *cond)
-{
- g_once(&cond->once, do_g_cond_new, NULL);
- g_cond_broadcast((GCond *) cond->once.retval);
-}
-#undef g_cond_broadcast
-
-static inline void (g_cond_signal)(CompatGCond *cond)
-{
- g_once(&cond->once, do_g_cond_new, NULL);
- g_cond_signal((GCond *) cond->once.retval);
-}
-#undef g_cond_signal
-
-static inline gboolean (g_cond_timed_wait)(CompatGCond *cond,
- CompatGMutex *mutex,
- GTimeVal *time)
-{
- g_assert(mutex->once.status != G_ONCE_STATUS_PROGRESS);
- g_once(&cond->once, do_g_cond_new, NULL);
- return g_cond_timed_wait((GCond *) cond->once.retval,
- (GMutex *) mutex->once.retval, time);
-}
-#undef g_cond_timed_wait
-
-/* This is not a macro, because it didn't exist until 2.32. */
-static inline gboolean g_cond_wait_until(CompatGCond *cond, CompatGMutex *mutex,
- gint64 end_time)
-{
- GTimeVal time;
-
- /* Convert from monotonic to CLOCK_REALTIME. */
- end_time -= g_get_monotonic_time();
- g_get_current_time(&time);
- end_time += time.tv_sec * G_TIME_SPAN_SECOND + time.tv_usec;
-
- time.tv_sec = end_time / G_TIME_SPAN_SECOND;
- time.tv_usec = end_time % G_TIME_SPAN_SECOND;
- return g_cond_timed_wait(cond, mutex, &time);
-}
-
-/* before 2.31 there was no g_thread_new() */
-static inline GThread *g_thread_new(const char *name,
- GThreadFunc func, gpointer data)
-{
- GThread *thread = g_thread_create(func, data, TRUE, NULL);
- if (!thread) {
- g_error("creating thread");
- }
- return thread;
-}
-#else
-#define CompatGMutex GMutex
-#define CompatGCond GCond
-#endif /* glib 2.31 */
-
-#if !GLIB_CHECK_VERSION(2, 32, 0)
-/* Beware, function returns gboolean since 2.39.2, see GLib commit 9101915 */
-static inline void g_hash_table_add(GHashTable *hash_table, gpointer key)
-{
- g_hash_table_replace(hash_table, key, key);
-}
-
-static inline gboolean g_hash_table_contains(GHashTable *hash_table,
- gpointer key)
-{
- return g_hash_table_lookup_extended(hash_table, key, NULL, NULL);
-}
-#define G_SOURCE_CONTINUE TRUE
-#define G_SOURCE_REMOVE FALSE
-#endif
-
-#ifndef g_assert_true
-#define g_assert_true(expr) \
- do { \
- if (G_LIKELY(expr)) { \
- } else { \
- g_assertion_message(G_LOG_DOMAIN, __FILE__, __LINE__, G_STRFUNC, \
- "'" #expr "' should be TRUE"); \
- } \
- } while (0)
-#endif
-
-#ifndef g_assert_false
-#define g_assert_false(expr) \
- do { \
- if (G_LIKELY(!(expr))) { \
- } else { \
- g_assertion_message(G_LOG_DOMAIN, __FILE__, __LINE__, G_STRFUNC, \
- "'" #expr "' should be FALSE"); \
- } \
- } while (0)
-#endif
-
-#ifndef g_assert_null
-#define g_assert_null(expr) \
- do { \
- if (G_LIKELY((expr) == NULL)) { \
- } else { \
- g_assertion_message(G_LOG_DOMAIN, __FILE__, __LINE__, G_STRFUNC, \
- "'" #expr "' should be NULL"); \
- } \
- } while (0)
-#endif
-
-#ifndef g_assert_nonnull
-#define g_assert_nonnull(expr) \
- do { \
- if (G_LIKELY((expr) != NULL)) { \
- } else { \
- g_assertion_message(G_LOG_DOMAIN, __FILE__, __LINE__, G_STRFUNC, \
- "'" #expr "' should not be NULL"); \
- } \
- } while (0)
-#endif
#ifndef g_assert_cmpmem
#define g_assert_cmpmem(m1, l1, m2, l2) \
@@ -288,80 +111,6 @@
} while (0)
#endif
-#if !GLIB_CHECK_VERSION(2, 28, 0)
-static inline void g_list_free_full(GList *list, GDestroyNotify free_func)
-{
- GList *l;
-
- for (l = list; l; l = l->next) {
- free_func(l->data);
- }
-
- g_list_free(list);
-}
-
-static inline void g_slist_free_full(GSList *list, GDestroyNotify free_func)
-{
- GSList *l;
-
- for (l = list; l; l = l->next) {
- free_func(l->data);
- }
-
- g_slist_free(list);
-}
-#endif
-
-#if !GLIB_CHECK_VERSION(2, 26, 0)
-static inline void g_source_set_name(GSource *source, const char *name)
-{
- /* This is just a debugging aid, so leaving it a no-op */
-}
-static inline void g_source_set_name_by_id(guint tag, const char *name)
-{
- /* This is just a debugging aid, so leaving it a no-op */
-}
-#endif
-
-#if !GLIB_CHECK_VERSION(2, 36, 0)
-/* Always fail. This will not include error_report output in the test log,
- * sending it instead to stderr.
- */
-#define g_test_initialized() (0)
-#endif
-#if !GLIB_CHECK_VERSION(2, 38, 0)
-#ifdef CONFIG_HAS_GLIB_SUBPROCESS_TESTS
-#error schizophrenic detection of glib subprocess testing
-#endif
-#define g_test_subprocess() (0)
-#endif
-
-
-#if !GLIB_CHECK_VERSION(2, 34, 0)
-static inline void
-g_test_add_data_func_full(const char *path,
- gpointer data,
- gpointer fn,
- gpointer data_free_func)
-{
-#if GLIB_CHECK_VERSION(2, 26, 0)
- /* back-compat casts, remove this once we can require new-enough glib */
- g_test_add_vtable(path, 0, data, NULL,
- (GTestFixtureFunc)fn, (GTestFixtureFunc) data_free_func);
-#else
- /* back-compat casts, remove this once we can require new-enough glib */
- g_test_add_vtable(path, 0, data, NULL,
- (void (*)(void)) fn, (void (*)(void)) data_free_func);
-#endif
-}
-#endif
-
-/* Small compat shim from glib 2.32 */
-#ifndef G_SOURCE_CONTINUE
-#define G_SOURCE_CONTINUE TRUE
-#endif
-#ifndef G_SOURCE_REMOVE
-#define G_SOURCE_REMOVE FALSE
-#endif
+#pragma GCC diagnostic pop
#endif
diff --git a/include/hw/i386/ioapic_internal.h b/include/hw/i386/ioapic_internal.h
index a11d86d..9848f39 100644
--- a/include/hw/i386/ioapic_internal.h
+++ b/include/hw/i386/ioapic_internal.h
@@ -109,10 +109,13 @@
uint64_t ioredtbl[IOAPIC_NUM_PINS];
Notifier machine_done;
uint8_t version;
+ uint64_t irq_count[IOAPIC_NUM_PINS];
+ int irq_level[IOAPIC_NUM_PINS];
};
void ioapic_reset_common(DeviceState *dev);
void ioapic_print_redtbl(Monitor *mon, IOAPICCommonState *s);
+void ioapic_stat_update_irq(IOAPICCommonState *s, int irq, int level);
#endif /* QEMU_IOAPIC_INTERNAL_H */
diff --git a/include/hw/i386/pc.h b/include/hw/i386/pc.h
index 316230e..4d99d69 100644
--- a/include/hw/i386/pc.h
+++ b/include/hw/i386/pc.h
@@ -154,9 +154,6 @@
/* ioapic.c */
-void kvm_ioapic_dump_state(Monitor *mon, const QDict *qdict);
-void ioapic_dump_state(Monitor *mon, const QDict *qdict);
-
/* Global System Interrupts */
#define GSI_NUM_PINS IOAPIC_NUM_PINS
diff --git a/include/hw/mem/nvdimm.h b/include/hw/mem/nvdimm.h
index 9340631..c5c9b3c 100644
--- a/include/hw/mem/nvdimm.h
+++ b/include/hw/mem/nvdimm.h
@@ -74,7 +74,7 @@
* it's the PMEM region in NVDIMM device, which is presented to
* guest via ACPI NFIT and _FIT method if NVDIMM hotplug is supported.
*/
- MemoryRegion nvdimm_mr;
+ MemoryRegion *nvdimm_mr;
/*
* The 'on' value results in the unarmed flag set in ACPI NFIT,
diff --git a/include/hw/mem/pc-dimm.h b/include/hw/mem/pc-dimm.h
index 627c860..26ebb7d 100644
--- a/include/hw/mem/pc-dimm.h
+++ b/include/hw/mem/pc-dimm.h
@@ -62,9 +62,11 @@
* @realize: called after common dimm is realized so that the dimm based
* devices get the chance to do specified operations.
* @get_memory_region: returns #MemoryRegion associated with @dimm which
- * is directly mapped into the physical address space of guest.
+ * is directly mapped into the physical address space of guest. Will not
+ * fail after the device was realized.
* @get_vmstate_memory_region: returns #MemoryRegion which indicates the
- * memory of @dimm should be kept during live migration.
+ * memory of @dimm should be kept during live migration. Will not fail
+ * after the device was realized.
*/
typedef struct PCDIMMDeviceClass {
/* private */
@@ -73,12 +75,11 @@
/* public */
void (*realize)(PCDIMMDevice *dimm, Error **errp);
MemoryRegion *(*get_memory_region)(PCDIMMDevice *dimm, Error **errp);
- MemoryRegion *(*get_vmstate_memory_region)(PCDIMMDevice *dimm);
+ MemoryRegion *(*get_vmstate_memory_region)(PCDIMMDevice *dimm,
+ Error **errp);
} PCDIMMDeviceClass;
-int pc_dimm_get_free_slot(const int *hint, int max_slots, Error **errp);
-
-void pc_dimm_memory_plug(DeviceState *dev, MachineState *machine,
- uint64_t align, Error **errp);
-void pc_dimm_memory_unplug(DeviceState *dev, MachineState *machine);
+void pc_dimm_plug(DeviceState *dev, MachineState *machine, uint64_t align,
+ Error **errp);
+void pc_dimm_unplug(DeviceState *dev, MachineState *machine);
#endif
diff --git a/include/hw/scsi/esp.h b/include/hw/scsi/esp.h
index 93fdace..682a0d2 100644
--- a/include/hw/scsi/esp.h
+++ b/include/hw/scsi/esp.h
@@ -131,11 +131,6 @@
#define TCHI_FAS100A 0x4
#define TCHI_AM53C974 0x12
-ESPState *esp_init(hwaddr espaddr, int it_shift,
- ESPDMAMemoryReadWriteFunc dma_memory_read,
- ESPDMAMemoryReadWriteFunc dma_memory_write,
- void *dma_opaque, qemu_irq irq, qemu_irq *reset,
- qemu_irq *dma_enable);
void esp_dma_enable(ESPState *s, int irq, int level);
void esp_request_cancelled(SCSIRequest *req);
void esp_command_complete(SCSIRequest *req, uint32_t status, size_t resid);
diff --git a/include/hw/scsi/scsi.h b/include/hw/scsi/scsi.h
index e35137e..ee3a411 100644
--- a/include/hw/scsi/scsi.h
+++ b/include/hw/scsi/scsi.h
@@ -87,6 +87,7 @@
uint64_t port_wwn;
int scsi_version;
int default_scsi_version;
+ bool needs_vpd_bl_emulation;
};
extern const VMStateDescription vmstate_scsi_device;
@@ -154,6 +155,8 @@
SCSIDevice *scsi_bus_legacy_add_drive(SCSIBus *bus, BlockBackend *blk,
int unit, bool removable, int bootindex,
bool share_rw,
+ BlockdevOnError rerror,
+ BlockdevOnError werror,
const char *serial, Error **errp);
void scsi_bus_legacy_handle_cmdline(SCSIBus *bus);
void scsi_legacy_handle_cmdline(void);
@@ -184,8 +187,11 @@
void scsi_device_set_ua(SCSIDevice *sdev, SCSISense sense);
void scsi_device_report_change(SCSIDevice *dev, SCSISense sense);
void scsi_device_unit_attention_reported(SCSIDevice *dev);
-void scsi_generic_read_device_identification(SCSIDevice *dev);
+void scsi_generic_read_device_inquiry(SCSIDevice *dev);
int scsi_device_get_sense(SCSIDevice *dev, uint8_t *buf, int len, bool fixed);
+int scsi_disk_emulate_vpd_page(SCSIRequest *req, uint8_t *outbuf);
+int scsi_SG_IO_FROM_DEV(BlockBackend *blk, uint8_t *cmd, uint8_t cmd_size,
+ uint8_t *buf, uint8_t buf_size);
SCSIDevice *scsi_device_find(SCSIBus *bus, int channel, int target, int lun);
/* scsi-generic.c. */
diff --git a/include/qemu/cutils.h b/include/qemu/cutils.h
index a663340..274d419 100644
--- a/include/qemu/cutils.h
+++ b/include/qemu/cutils.h
@@ -122,6 +122,14 @@
* Returns: the pointer originally in @input.
*/
char *qemu_strsep(char **input, const char *delim);
+#ifdef HAVE_STRCHRNUL
+static inline const char *qemu_strchrnul(const char *s, int c)
+{
+ return strchrnul(s, c);
+}
+#else
+const char *qemu_strchrnul(const char *s, int c);
+#endif
time_t mktimegm(struct tm *tm);
int qemu_fdatasync(int fd);
int fcntl_setfl(int fd, int flag);
diff --git a/include/qemu/iova-tree.h b/include/qemu/iova-tree.h
index b061932..b66cf93 100644
--- a/include/qemu/iova-tree.h
+++ b/include/qemu/iova-tree.h
@@ -23,7 +23,6 @@
* for the thread safety issue.
*/
-#include "qemu/osdep.h"
#include "exec/memory.h"
#include "exec/hwaddr.h"
diff --git a/include/qemu/main-loop.h b/include/qemu/main-loop.h
index 6b4b60b..721aa24 100644
--- a/include/qemu/main-loop.h
+++ b/include/qemu/main-loop.h
@@ -168,6 +168,20 @@
/* async I/O support */
typedef void IOReadHandler(void *opaque, const uint8_t *buf, int size);
+
+/**
+ * IOCanReadHandler: Return the number of bytes that #IOReadHandler can accept
+ *
+ * This function reports how many bytes #IOReadHandler is prepared to accept.
+ * #IOReadHandler may be invoked with up to this number of bytes. If this
+ * function returns 0 then #IOReadHandler is not invoked.
+ *
+ * This function is typically called from an event loop. If the number of
+ * bytes changes outside the event loop (e.g. because a vcpu thread drained the
+ * buffer), then it is necessary to kick the event loop so that this function
+ * is called again. aio_notify() or qemu_notify_event() can be used to kick
+ * the event loop.
+ */
typedef int IOCanReadHandler(void *opaque);
/**
diff --git a/include/qemu/osdep.h b/include/qemu/osdep.h
index 9ed6242..a91068d 100644
--- a/include/qemu/osdep.h
+++ b/include/qemu/osdep.h
@@ -33,6 +33,21 @@
#else
#include "exec/poison.h"
#endif
+#ifdef __COVERITY__
+/* Coverity does not like the new _Float* types that are used by
+ * recent glibc, and croaks on every single file that includes
+ * stdlib.h. These typedefs are enough to please it.
+ *
+ * Note that these fix parse errors so they cannot be placed in
+ * scripts/coverity-model.c.
+ */
+typedef float _Float32;
+typedef double _Float32x;
+typedef double _Float64;
+typedef __float80 _Float64x;
+typedef __float128 _Float128;
+#endif
+
#include "qemu/compiler.h"
/* Older versions of C++ don't get definitions of various macros from
diff --git a/include/qemu/thread-posix.h b/include/qemu/thread-posix.h
index f3f47e4..fd27b34 100644
--- a/include/qemu/thread-posix.h
+++ b/include/qemu/thread-posix.h
@@ -12,6 +12,10 @@
struct QemuMutex {
pthread_mutex_t lock;
+#ifdef CONFIG_DEBUG_MUTEX
+ const char *file;
+ int line;
+#endif
bool initialized;
};
diff --git a/include/qemu/thread-win32.h b/include/qemu/thread-win32.h
index 3a05e3b..d668d78 100644
--- a/include/qemu/thread-win32.h
+++ b/include/qemu/thread-win32.h
@@ -5,6 +5,10 @@
struct QemuMutex {
SRWLOCK lock;
+#ifdef CONFIG_DEBUG_MUTEX
+ const char *file;
+ int line;
+#endif
bool initialized;
};
diff --git a/include/scsi/pr-manager.h b/include/scsi/pr-manager.h
index 5d2f13a..50a77b0 100644
--- a/include/scsi/pr-manager.h
+++ b/include/scsi/pr-manager.h
@@ -33,23 +33,16 @@
/* <public> */
int (*run)(PRManager *pr_mgr, int fd, struct sg_io_hdr *hdr);
+ bool (*is_connected)(PRManager *pr_mgr);
} PRManagerClass;
+bool pr_manager_is_connected(PRManager *pr_mgr);
BlockAIOCB *pr_manager_execute(PRManager *pr_mgr,
AioContext *ctx, int fd,
struct sg_io_hdr *hdr,
BlockCompletionFunc *complete,
void *opaque);
-#ifdef CONFIG_LINUX
PRManager *pr_manager_lookup(const char *id, Error **errp);
-#else
-static inline PRManager *pr_manager_lookup(const char *id, Error **errp)
-{
- /* The classes do not exist at all! */
- error_setg(errp, "No persistent reservation manager with id '%s'", id);
- return NULL;
-}
-#endif
#endif
diff --git a/include/sysemu/device_tree.h b/include/sysemu/device_tree.h
index e22e5be..c16fd69 100644
--- a/include/sysemu/device_tree.h
+++ b/include/sysemu/device_tree.h
@@ -43,6 +43,22 @@
char **qemu_fdt_node_path(void *fdt, const char *name, char *compat,
Error **errp);
+/**
+ * qemu_fdt_node_unit_path: return the paths of nodes matching a given
+ * node-name, ie. node-name and node-name@unit-address
+ * @fdt: pointer to the dt blob
+ * @name: node name
+ * @errp: handle to an error object
+ *
+ * returns a newly allocated NULL-terminated array of node paths.
+ * Use g_strfreev() to free it. If one or more nodes were found, the
+ * array contains the path of each node and the last element equals to
+ * NULL. If there is no error but no matching node was found, the
+ * returned array contains a single element equal to NULL. If an error
+ * was encountered when parsing the blob, the function returns NULL
+ */
+char **qemu_fdt_node_unit_path(void *fdt, const char *name, Error **errp);
+
int qemu_fdt_setprop(void *fdt, const char *node_path,
const char *property, const void *val, int size);
int qemu_fdt_setprop_cell(void *fdt, const char *node_path,
diff --git a/include/sysemu/hostmem.h b/include/sysemu/hostmem.h
index 5beb0ef..6e6bd2c 100644
--- a/include/sysemu/hostmem.h
+++ b/include/sysemu/hostmem.h
@@ -62,8 +62,7 @@
};
bool host_memory_backend_mr_inited(HostMemoryBackend *backend);
-MemoryRegion *host_memory_backend_get_memory(HostMemoryBackend *backend,
- Error **errp);
+MemoryRegion *host_memory_backend_get_memory(HostMemoryBackend *backend);
void host_memory_backend_set_mapped(HostMemoryBackend *backend, bool mapped);
bool host_memory_backend_is_mapped(HostMemoryBackend *backend);
diff --git a/include/sysemu/kvm_int.h b/include/sysemu/kvm_int.h
index 888557a..f838412 100644
--- a/include/sysemu/kvm_int.h
+++ b/include/sysemu/kvm_int.h
@@ -20,6 +20,7 @@
void *ram;
int slot;
int flags;
+ int old_flags;
} KVMSlot;
typedef struct KVMMemoryListener {
diff --git a/include/sysemu/sysemu.h b/include/sysemu/sysemu.h
index e893f72..b921c6f 100644
--- a/include/sysemu/sysemu.h
+++ b/include/sysemu/sysemu.h
@@ -128,6 +128,7 @@
extern uint8_t *boot_splash_filedata;
extern size_t boot_splash_filedata_size;
extern bool enable_mlock;
+extern bool enable_cpu_pm;
extern uint8_t qemu_extra_params_fw[2];
extern QEMUClockType rtc_clock;
extern const char *mem_path;
diff --git a/linux-user/elfload.c b/linux-user/elfload.c
index 13bc78d..942a1b6 100644
--- a/linux-user/elfload.c
+++ b/linux-user/elfload.c
@@ -583,7 +583,9 @@
ARM_HWCAP_A64_FPHP | ARM_HWCAP_A64_ASIMDHP);
GET_FEATURE(ARM_FEATURE_V8_ATOMICS, ARM_HWCAP_A64_ATOMICS);
GET_FEATURE(ARM_FEATURE_V8_RDM, ARM_HWCAP_A64_ASIMDRDM);
+ GET_FEATURE(ARM_FEATURE_V8_DOTPROD, ARM_HWCAP_A64_ASIMDDP);
GET_FEATURE(ARM_FEATURE_V8_FCMA, ARM_HWCAP_A64_FCMA);
+ GET_FEATURE(ARM_FEATURE_SVE, ARM_HWCAP_A64_SVE);
#undef GET_FEATURE
return hwcaps;
diff --git a/linux-user/mmap.c b/linux-user/mmap.c
index 9168a20..d0c50e4 100644
--- a/linux-user/mmap.c
+++ b/linux-user/mmap.c
@@ -20,7 +20,6 @@
#include "qemu.h"
#include "qemu-common.h"
-#include "translate-all.h"
//#define DEBUG_MMAP
diff --git a/memory.c b/memory.c
index 21aa57d..e9cd446 100644
--- a/memory.c
+++ b/memory.c
@@ -2858,10 +2858,49 @@
int128_sub((size), int128_one())) : 0)
#define MTREE_INDENT " "
+static void mtree_expand_owner(fprintf_function mon_printf, void *f,
+ const char *label, Object *obj)
+{
+ DeviceState *dev = (DeviceState *) object_dynamic_cast(obj, TYPE_DEVICE);
+
+ mon_printf(f, " %s:{%s", label, dev ? "dev" : "obj");
+ if (dev && dev->id) {
+ mon_printf(f, " id=%s", dev->id);
+ } else {
+ gchar *canonical_path = object_get_canonical_path(obj);
+ if (canonical_path) {
+ mon_printf(f, " path=%s", canonical_path);
+ g_free(canonical_path);
+ } else {
+ mon_printf(f, " type=%s", object_get_typename(obj));
+ }
+ }
+ mon_printf(f, "}");
+}
+
+static void mtree_print_mr_owner(fprintf_function mon_printf, void *f,
+ const MemoryRegion *mr)
+{
+ Object *owner = mr->owner;
+ Object *parent = memory_region_owner((MemoryRegion *)mr);
+
+ if (!owner && !parent) {
+ mon_printf(f, " orphan");
+ return;
+ }
+ if (owner) {
+ mtree_expand_owner(mon_printf, f, "owner", owner);
+ }
+ if (parent && parent != owner) {
+ mtree_expand_owner(mon_printf, f, "parent", parent);
+ }
+}
+
static void mtree_print_mr(fprintf_function mon_printf, void *f,
const MemoryRegion *mr, unsigned int level,
hwaddr base,
- MemoryRegionListHead *alias_print_queue)
+ MemoryRegionListHead *alias_print_queue,
+ bool owner)
{
MemoryRegionList *new_ml, *ml, *next_ml;
MemoryRegionListHead submr_print_queue;
@@ -2907,7 +2946,7 @@
}
mon_printf(f, TARGET_FMT_plx "-" TARGET_FMT_plx
" (prio %d, %s): alias %s @%s " TARGET_FMT_plx
- "-" TARGET_FMT_plx "%s\n",
+ "-" TARGET_FMT_plx "%s",
cur_start, cur_end,
mr->priority,
memory_region_type((MemoryRegion *)mr),
@@ -2916,15 +2955,22 @@
mr->alias_offset,
mr->alias_offset + MR_SIZE(mr->size),
mr->enabled ? "" : " [disabled]");
+ if (owner) {
+ mtree_print_mr_owner(mon_printf, f, mr);
+ }
} else {
mon_printf(f,
- TARGET_FMT_plx "-" TARGET_FMT_plx " (prio %d, %s): %s%s\n",
+ TARGET_FMT_plx "-" TARGET_FMT_plx " (prio %d, %s): %s%s",
cur_start, cur_end,
mr->priority,
memory_region_type((MemoryRegion *)mr),
memory_region_name(mr),
mr->enabled ? "" : " [disabled]");
+ if (owner) {
+ mtree_print_mr_owner(mon_printf, f, mr);
+ }
}
+ mon_printf(f, "\n");
QTAILQ_INIT(&submr_print_queue);
@@ -2947,7 +2993,7 @@
QTAILQ_FOREACH(ml, &submr_print_queue, mrqueue) {
mtree_print_mr(mon_printf, f, ml->mr, level + 1, cur_start,
- alias_print_queue);
+ alias_print_queue, owner);
}
QTAILQ_FOREACH_SAFE(ml, &submr_print_queue, mrqueue, next_ml) {
@@ -2960,6 +3006,7 @@
void *f;
int counter;
bool dispatch_tree;
+ bool owner;
};
static void mtree_print_flatview(gpointer key, gpointer value,
@@ -3000,7 +3047,7 @@
mr = range->mr;
if (range->offset_in_region) {
p(f, MTREE_INDENT TARGET_FMT_plx "-"
- TARGET_FMT_plx " (prio %d, %s): %s @" TARGET_FMT_plx "\n",
+ TARGET_FMT_plx " (prio %d, %s): %s @" TARGET_FMT_plx,
int128_get64(range->addr.start),
int128_get64(range->addr.start) + MR_SIZE(range->addr.size),
mr->priority,
@@ -3009,13 +3056,17 @@
range->offset_in_region);
} else {
p(f, MTREE_INDENT TARGET_FMT_plx "-"
- TARGET_FMT_plx " (prio %d, %s): %s\n",
+ TARGET_FMT_plx " (prio %d, %s): %s",
int128_get64(range->addr.start),
int128_get64(range->addr.start) + MR_SIZE(range->addr.size),
mr->priority,
range->readonly ? "rom" : memory_region_type(mr),
memory_region_name(mr));
}
+ if (fvi->owner) {
+ mtree_print_mr_owner(p, f, mr);
+ }
+ p(f, "\n");
range++;
}
@@ -3041,7 +3092,7 @@
}
void mtree_info(fprintf_function mon_printf, void *f, bool flatview,
- bool dispatch_tree)
+ bool dispatch_tree, bool owner)
{
MemoryRegionListHead ml_head;
MemoryRegionList *ml, *ml2;
@@ -3053,7 +3104,8 @@
.mon_printf = mon_printf,
.f = f,
.counter = 0,
- .dispatch_tree = dispatch_tree
+ .dispatch_tree = dispatch_tree,
+ .owner = owner,
};
GArray *fv_address_spaces;
GHashTable *views = g_hash_table_new(g_direct_hash, g_direct_equal);
@@ -3085,14 +3137,14 @@
QTAILQ_FOREACH(as, &address_spaces, address_spaces_link) {
mon_printf(f, "address-space: %s\n", as->name);
- mtree_print_mr(mon_printf, f, as->root, 1, 0, &ml_head);
+ mtree_print_mr(mon_printf, f, as->root, 1, 0, &ml_head, owner);
mon_printf(f, "\n");
}
/* print aliased regions */
QTAILQ_FOREACH(ml, &ml_head, mrqueue) {
mon_printf(f, "memory-region: %s\n", memory_region_name(ml->mr));
- mtree_print_mr(mon_printf, f, ml->mr, 1, 0, &ml_head);
+ mtree_print_mr(mon_printf, f, ml->mr, 1, 0, &ml_head, owner);
mon_printf(f, "\n");
}
diff --git a/memory_ldst.inc.c b/memory_ldst.inc.c
index 1548398..acf865b 100644
--- a/memory_ldst.inc.c
+++ b/memory_ldst.inc.c
@@ -34,7 +34,7 @@
RCU_READ_LOCK();
mr = TRANSLATE(addr, &addr1, &l, false, attrs);
- if (l < 4 || !IS_DIRECT(mr, false)) {
+ if (l < 4 || !memory_access_is_direct(mr, false)) {
release_lock |= prepare_mmio_access(mr);
/* I/O case */
@@ -50,7 +50,7 @@
#endif
} else {
/* RAM case */
- ptr = MAP_RAM(mr, addr1);
+ ptr = qemu_map_ram_ptr(mr->ram_block, addr1);
switch (endian) {
case DEVICE_LITTLE_ENDIAN:
val = ldl_le_p(ptr);
@@ -110,7 +110,7 @@
RCU_READ_LOCK();
mr = TRANSLATE(addr, &addr1, &l, false, attrs);
- if (l < 8 || !IS_DIRECT(mr, false)) {
+ if (l < 8 || !memory_access_is_direct(mr, false)) {
release_lock |= prepare_mmio_access(mr);
/* I/O case */
@@ -126,7 +126,7 @@
#endif
} else {
/* RAM case */
- ptr = MAP_RAM(mr, addr1);
+ ptr = qemu_map_ram_ptr(mr->ram_block, addr1);
switch (endian) {
case DEVICE_LITTLE_ENDIAN:
val = ldq_le_p(ptr);
@@ -184,14 +184,14 @@
RCU_READ_LOCK();
mr = TRANSLATE(addr, &addr1, &l, false, attrs);
- if (!IS_DIRECT(mr, false)) {
+ if (!memory_access_is_direct(mr, false)) {
release_lock |= prepare_mmio_access(mr);
/* I/O case */
r = memory_region_dispatch_read(mr, addr1, &val, 1, attrs);
} else {
/* RAM case */
- ptr = MAP_RAM(mr, addr1);
+ ptr = qemu_map_ram_ptr(mr->ram_block, addr1);
val = ldub_p(ptr);
r = MEMTX_OK;
}
@@ -220,7 +220,7 @@
RCU_READ_LOCK();
mr = TRANSLATE(addr, &addr1, &l, false, attrs);
- if (l < 2 || !IS_DIRECT(mr, false)) {
+ if (l < 2 || !memory_access_is_direct(mr, false)) {
release_lock |= prepare_mmio_access(mr);
/* I/O case */
@@ -236,7 +236,7 @@
#endif
} else {
/* RAM case */
- ptr = MAP_RAM(mr, addr1);
+ ptr = qemu_map_ram_ptr(mr->ram_block, addr1);
switch (endian) {
case DEVICE_LITTLE_ENDIAN:
val = lduw_le_p(ptr);
@@ -297,12 +297,12 @@
RCU_READ_LOCK();
mr = TRANSLATE(addr, &addr1, &l, true, attrs);
- if (l < 4 || !IS_DIRECT(mr, true)) {
+ if (l < 4 || !memory_access_is_direct(mr, true)) {
release_lock |= prepare_mmio_access(mr);
r = memory_region_dispatch_write(mr, addr1, val, 4, attrs);
} else {
- ptr = MAP_RAM(mr, addr1);
+ ptr = qemu_map_ram_ptr(mr->ram_block, addr1);
stl_p(ptr, val);
dirty_log_mask = memory_region_get_dirty_log_mask(mr);
@@ -334,7 +334,7 @@
RCU_READ_LOCK();
mr = TRANSLATE(addr, &addr1, &l, true, attrs);
- if (l < 4 || !IS_DIRECT(mr, true)) {
+ if (l < 4 || !memory_access_is_direct(mr, true)) {
release_lock |= prepare_mmio_access(mr);
#if defined(TARGET_WORDS_BIGENDIAN)
@@ -349,7 +349,7 @@
r = memory_region_dispatch_write(mr, addr1, val, 4, attrs);
} else {
/* RAM case */
- ptr = MAP_RAM(mr, addr1);
+ ptr = qemu_map_ram_ptr(mr->ram_block, addr1);
switch (endian) {
case DEVICE_LITTLE_ENDIAN:
stl_le_p(ptr, val);
@@ -361,7 +361,7 @@
stl_p(ptr, val);
break;
}
- INVALIDATE(mr, addr1, 4);
+ invalidate_and_set_dirty(mr, addr1, 4);
r = MEMTX_OK;
}
if (result) {
@@ -406,14 +406,14 @@
RCU_READ_LOCK();
mr = TRANSLATE(addr, &addr1, &l, true, attrs);
- if (!IS_DIRECT(mr, true)) {
+ if (!memory_access_is_direct(mr, true)) {
release_lock |= prepare_mmio_access(mr);
r = memory_region_dispatch_write(mr, addr1, val, 1, attrs);
} else {
/* RAM case */
- ptr = MAP_RAM(mr, addr1);
+ ptr = qemu_map_ram_ptr(mr->ram_block, addr1);
stb_p(ptr, val);
- INVALIDATE(mr, addr1, 1);
+ invalidate_and_set_dirty(mr, addr1, 1);
r = MEMTX_OK;
}
if (result) {
@@ -439,7 +439,7 @@
RCU_READ_LOCK();
mr = TRANSLATE(addr, &addr1, &l, true, attrs);
- if (l < 2 || !IS_DIRECT(mr, true)) {
+ if (l < 2 || !memory_access_is_direct(mr, true)) {
release_lock |= prepare_mmio_access(mr);
#if defined(TARGET_WORDS_BIGENDIAN)
@@ -454,7 +454,7 @@
r = memory_region_dispatch_write(mr, addr1, val, 2, attrs);
} else {
/* RAM case */
- ptr = MAP_RAM(mr, addr1);
+ ptr = qemu_map_ram_ptr(mr->ram_block, addr1);
switch (endian) {
case DEVICE_LITTLE_ENDIAN:
stw_le_p(ptr, val);
@@ -466,7 +466,7 @@
stw_p(ptr, val);
break;
}
- INVALIDATE(mr, addr1, 2);
+ invalidate_and_set_dirty(mr, addr1, 2);
r = MEMTX_OK;
}
if (result) {
@@ -512,7 +512,7 @@
RCU_READ_LOCK();
mr = TRANSLATE(addr, &addr1, &l, true, attrs);
- if (l < 8 || !IS_DIRECT(mr, true)) {
+ if (l < 8 || !memory_access_is_direct(mr, true)) {
release_lock |= prepare_mmio_access(mr);
#if defined(TARGET_WORDS_BIGENDIAN)
@@ -527,7 +527,7 @@
r = memory_region_dispatch_write(mr, addr1, val, 8, attrs);
} else {
/* RAM case */
- ptr = MAP_RAM(mr, addr1);
+ ptr = qemu_map_ram_ptr(mr->ram_block, addr1);
switch (endian) {
case DEVICE_LITTLE_ENDIAN:
stq_le_p(ptr, val);
@@ -539,7 +539,7 @@
stq_p(ptr, val);
break;
}
- INVALIDATE(mr, addr1, 8);
+ invalidate_and_set_dirty(mr, addr1, 8);
r = MEMTX_OK;
}
if (result) {
@@ -576,8 +576,5 @@
#undef ARG1
#undef SUFFIX
#undef TRANSLATE
-#undef IS_DIRECT
-#undef MAP_RAM
-#undef INVALIDATE
#undef RCU_READ_LOCK
#undef RCU_READ_UNLOCK
diff --git a/monitor.c b/monitor.c
index 0730a27..7b473aa 100644
--- a/monitor.c
+++ b/monitor.c
@@ -820,9 +820,7 @@
p = list;
for(;;) {
pstart = p;
- p = strchr(p, '|');
- if (!p)
- p = pstart + strlen(pstart);
+ p = qemu_strchrnul(p, '|');
if ((p - pstart) == len && !memcmp(pstart, name, len))
return 1;
if (*p == '\0')
@@ -2007,8 +2005,10 @@
{
bool flatview = qdict_get_try_bool(qdict, "flatview", false);
bool dispatch_tree = qdict_get_try_bool(qdict, "dispatch_tree", false);
+ bool owner = qdict_get_try_bool(qdict, "owner", false);
- mtree_info((fprintf_function)monitor_printf, mon, flatview, dispatch_tree);
+ mtree_info((fprintf_function)monitor_printf, mon, flatview, dispatch_tree,
+ owner);
}
static void hmp_info_numa(Monitor *mon, const QDict *qdict)
@@ -3489,9 +3489,7 @@
p = list;
for(;;) {
pstart = p;
- p = strchr(p, '|');
- if (!p)
- p = pstart + strlen(pstart);
+ p = qemu_strchrnul(p, '|');
len = p - pstart;
if (len > sizeof(cmd) - 2)
len = sizeof(cmd) - 2;
diff --git a/numa.c b/numa.c
index 33572bf..5f6367b 100644
--- a/numa.c
+++ b/numa.c
@@ -523,8 +523,7 @@
if (!backend) {
continue;
}
- MemoryRegion *seg = host_memory_backend_get_memory(backend,
- &error_fatal);
+ MemoryRegion *seg = host_memory_backend_get_memory(backend);
if (memory_region_is_mapped(seg)) {
char *path = object_get_canonical_path_component(OBJECT(backend));
@@ -567,10 +566,8 @@
if (pcdimm_info) {
node_mem[pcdimm_info->node].node_mem += pcdimm_info->size;
- if (pcdimm_info->hotpluggable && pcdimm_info->hotplugged) {
- node_mem[pcdimm_info->node].node_plugged_mem +=
- pcdimm_info->size;
- }
+ node_mem[pcdimm_info->node].node_plugged_mem +=
+ pcdimm_info->size;
}
}
}
diff --git a/qapi/block.json b/qapi/block.json
index ca807f1..11f01f2 100644
--- a/qapi/block.json
+++ b/qapi/block.json
@@ -78,6 +78,34 @@
'data': { 'device': 'str', 'name': 'str' } }
##
+# @PRManagerInfo:
+#
+# Information about a persistent reservation manager
+#
+# @id: the identifier of the persistent reservation manager
+#
+# @connected: true if the persistent reservation manager is connected to
+# the underlying storage or helper
+#
+# Since: 3.0
+##
+{ 'struct': 'PRManagerInfo',
+ 'data': {'id': 'str', 'connected': 'bool'} }
+
+##
+# @query-pr-managers:
+#
+# Returns a list of information about each persistent reservation manager.
+#
+# Returns: a list of @PRManagerInfo for each persistent reservation manager
+#
+# Since: 3.0
+##
+{ 'command': 'query-pr-managers', 'returns': ['PRManagerInfo'],
+ 'allow-preconfig': true }
+
+
+##
# @blockdev-snapshot-internal-sync:
#
# Synchronously take an internal snapshot of a block device, when the
@@ -331,6 +359,30 @@
'data': { 'device': 'str', 'id': 'str', 'tray-open': 'bool' } }
##
+# @PR_MANAGER_STATUS_CHANGED:
+#
+# Emitted whenever the connected status of a persistent reservation
+# manager changes.
+#
+# @id: The id of the PR manager object
+#
+# @connected: true if the PR manager is connected to a backend
+#
+# Since: 3.0
+#
+# Example:
+#
+# <- { "event": "PR_MANAGER_STATUS_CHANGED",
+# "data": { "id": "pr-helper0",
+# "connected": true
+# },
+# "timestamp": { "seconds": 1519840375, "microseconds": 450486 } }
+#
+##
+{ 'event': 'PR_MANAGER_STATUS_CHANGED',
+ 'data': { 'id': 'str', 'connected': 'bool' } }
+
+##
# @QuorumOpType:
#
# An enumeration of the quorum operation types
diff --git a/qapi/job.json b/qapi/job.json
index 9d074eb..a121b61 100644
--- a/qapi/job.json
+++ b/qapi/job.json
@@ -104,7 +104,7 @@
# @id: The job identifier
# @status: The new job status
#
-# Since: 2.13
+# Since: 3.0
##
{ 'event': 'JOB_STATUS_CHANGE',
'data': { 'id': 'str',
@@ -126,7 +126,7 @@
#
# @id: The job identifier.
#
-# Since: 2.13
+# Since: 3.0
##
{ 'command': 'job-pause', 'data': { 'id': 'str' } }
@@ -140,7 +140,7 @@
#
# @id : The job identifier.
#
-# Since: 2.13
+# Since: 3.0
##
{ 'command': 'job-resume', 'data': { 'id': 'str' } }
@@ -159,7 +159,7 @@
#
# @id: The job identifier.
#
-# Since: 2.13
+# Since: 3.0
##
{ 'command': 'job-cancel', 'data': { 'id': 'str' } }
@@ -171,7 +171,7 @@
#
# @id: The job identifier.
#
-# Since: 2.13
+# Since: 3.0
##
{ 'command': 'job-complete', 'data': { 'id': 'str' } }
@@ -187,7 +187,7 @@
#
# @id: The job identifier.
#
-# Since: 2.13
+# Since: 3.0
##
{ 'command': 'job-dismiss', 'data': { 'id': 'str' } }
@@ -205,7 +205,7 @@
# @id: The identifier of any job in the transaction, or of a job that is not
# part of any transaction.
#
-# Since: 2.13
+# Since: 3.0
##
{ 'command': 'job-finalize', 'data': { 'id': 'str' } }
@@ -237,7 +237,7 @@
# the reason for the job failure. It should not be parsed
# by applications.
#
-# Since: 2.13
+# Since: 3.0
##
{ 'struct': 'JobInfo',
'data': { 'id': 'str', 'type': 'JobType', 'status': 'JobStatus',
@@ -251,6 +251,6 @@
#
# Returns: a list with a @JobInfo for each active job
#
-# Since: 2.13
+# Since: 3.0
##
{ 'command': 'query-jobs', 'returns': ['JobInfo'] }
diff --git a/qapi/misc.json b/qapi/misc.json
index c6bc18a..29da785 100644
--- a/qapi/misc.json
+++ b/qapi/misc.json
@@ -1677,10 +1677,13 @@
#
# @kdump-snappy: kdump-compressed format with snappy-compressed
#
+# @win-dmp: Windows full crashdump format,
+# can be used instead of ELF converting (since 2.13)
+#
# Since: 2.0
##
{ 'enum': 'DumpGuestMemoryFormat',
- 'data': [ 'elf', 'kdump-zlib', 'kdump-lzo', 'kdump-snappy' ] }
+ 'data': [ 'elf', 'kdump-zlib', 'kdump-lzo', 'kdump-snappy', 'win-dmp' ] }
##
# @dump-guest-memory:
diff --git a/qemu-doc.texi b/qemu-doc.texi
index 16fcb47..1cb3ba4 100644
--- a/qemu-doc.texi
+++ b/qemu-doc.texi
@@ -2912,6 +2912,11 @@
The @code{-clock} option is ignored since QEMU version 1.7.0. There is no
replacement since it is not needed anymore.
+@subsection -enable-hax (since 3.0.0)
+
+The @option{-enable-hax} option has been replaced by @option{-accel hax}.
+Both options have been introduced in QEMU version 2.9.0.
+
@section QEMU Machine Protocol (QMP) commands
@subsection block-dirty-bitmap-add "autoload" parameter (since 2.12.0)
diff --git a/qemu-options.hx b/qemu-options.hx
index d5b0c26..81b1e99 100644
--- a/qemu-options.hx
+++ b/qemu-options.hx
@@ -130,7 +130,7 @@
DEF("accel", HAS_ARG, QEMU_OPTION_accel,
"-accel [accel=]accelerator[,thread=single|multi]\n"
" select accelerator (kvm, xen, hax, hvf, whpx or tcg; use 'help' for a list)\n"
- " thread=single|multi (enable multi-threaded TCG)", QEMU_ARCH_ALL)
+ " thread=single|multi (enable multi-threaded TCG)\n", QEMU_ARCH_ALL)
STEXI
@item -accel @var{name}[,prop=@var{value}[,...]]
@findex -accel
@@ -3325,6 +3325,30 @@
(enabled by default).
ETEXI
+DEF("overcommit", HAS_ARG, QEMU_OPTION_overcommit,
+ "--overcommit [mem-lock=on|off][cpu-pm=on|off]\n"
+ " run qemu with overcommit hints\n"
+ " mem-lock=on|off controls memory lock support (default: off)\n"
+ " cpu-pm=on|off controls cpu power management (default: off)\n",
+ QEMU_ARCH_ALL)
+STEXI
+@item -overcommit mem-lock=on|off
+@item -overcommit cpu-pm=on|off
+@findex -overcommit
+Run qemu with hints about host resource overcommit. The default is
+to assume that host overcommits all resources.
+
+Locking qemu and guest memory can be enabled via @option{mem-lock=on} (disabled
+by default). This works when host memory is not overcommitted and reduces the
+worst-case latency for guest. This is equivalent to @option{realtime}.
+
+Guest ability to manage power state of host cpus (increasing latency for other
+processes on the same host cpu, but decreasing latency for guest) can be
+enabled via @option{cpu-pm=on} (disabled by default). This works best when
+host CPU is not overcommitted. When used, host estimates of CPU cycle and power
+utilization will be incorrect, not taking into account guest idle time.
+ETEXI
+
DEF("gdb", HAS_ARG, QEMU_OPTION_gdb, \
"-gdb dev wait for gdb connection on 'dev'\n", QEMU_ARCH_ALL)
STEXI
@@ -3421,7 +3445,7 @@
Enable HAX (Hardware-based Acceleration eXecution) support. This option
is only available if HAX support is enabled when compiling. HAX is only
applicable to MAC and Windows platform, and thus does not conflict with
-KVM.
+KVM. This option is deprecated, use @option{-accel hax} instead.
ETEXI
DEF("xen-domid", HAS_ARG, QEMU_OPTION_xen_domid,
diff --git a/qga/commands.c b/qga/commands.c
index cce3010..0c7d138 100644
--- a/qga/commands.c
+++ b/qga/commands.c
@@ -414,10 +414,8 @@
argv = guest_exec_get_args(&arglist, true);
envp = has_env ? guest_exec_get_args(env, false) : NULL;
- flags = G_SPAWN_SEARCH_PATH | G_SPAWN_DO_NOT_REAP_CHILD;
-#if GLIB_CHECK_VERSION(2, 33, 2)
- flags |= G_SPAWN_SEARCH_PATH_FROM_ENVP;
-#endif
+ flags = G_SPAWN_SEARCH_PATH | G_SPAWN_DO_NOT_REAP_CHILD |
+ G_SPAWN_SEARCH_PATH_FROM_ENVP;
if (!has_output) {
flags |= G_SPAWN_STDOUT_TO_DEV_NULL | G_SPAWN_STDERR_TO_DEV_NULL;
}
@@ -514,7 +512,6 @@
GuestTimezone *qmp_guest_get_timezone(Error **errp)
{
-#if GLIB_CHECK_VERSION(2, 28, 0)
GuestTimezone *info = NULL;
GTimeZone *tz = NULL;
gint64 now = 0;
@@ -544,8 +541,4 @@
error:
g_free(info);
return NULL;
-#else
- error_setg(errp, QERR_UNSUPPORTED);
- return NULL;
-#endif
}
diff --git a/qobject/block-qdict.c b/qobject/block-qdict.c
index 36129e7..80c6530 100644
--- a/qobject/block-qdict.c
+++ b/qobject/block-qdict.c
@@ -97,7 +97,7 @@
const QDictEntry *entry, *next;
QDict *dict_val;
QList *list_val;
- char *new_key;
+ char *key, *new_key;
entry = qdict_first(qdict);
@@ -106,10 +106,12 @@
value = qdict_entry_value(entry);
dict_val = qobject_to(QDict, value);
list_val = qobject_to(QList, value);
- new_key = NULL;
if (prefix) {
- new_key = g_strdup_printf("%s.%s", prefix, entry->key);
+ key = new_key = g_strdup_printf("%s.%s", prefix, entry->key);
+ } else {
+ key = entry->key;
+ new_key = NULL;
}
/*
@@ -125,19 +127,17 @@
* well advised not to modify them altogether.)
*/
if (dict_val && qdict_size(dict_val)) {
- qdict_flatten_qdict(dict_val, target,
- new_key ? new_key : entry->key);
+ qdict_flatten_qdict(dict_val, target, key);
if (target == qdict) {
qdict_del(qdict, entry->key);
}
} else if (list_val && !qlist_empty(list_val)) {
- qdict_flatten_qlist(list_val, target,
- new_key ? new_key : entry->key);
+ qdict_flatten_qlist(list_val, target, key);
if (target == qdict) {
qdict_del(qdict, entry->key);
}
} else if (target != qdict) {
- qdict_put_obj(target, new_key, qobject_ref(value));
+ qdict_put_obj(target, key, qobject_ref(value));
}
g_free(new_key);
diff --git a/scsi/Makefile.objs b/scsi/Makefile.objs
index 4d25e47..bb8789c 100644
--- a/scsi/Makefile.objs
+++ b/scsi/Makefile.objs
@@ -1,3 +1,4 @@
block-obj-y += utils.o
block-obj-$(CONFIG_LINUX) += pr-manager.o pr-manager-helper.o
+block-obj-$(call lnot,$(CONFIG_LINUX)) += pr-manager-stub.o
diff --git a/scsi/pr-manager-helper.c b/scsi/pr-manager-helper.c
index 82ff6b6..519a296 100644
--- a/scsi/pr-manager-helper.c
+++ b/scsi/pr-manager-helper.c
@@ -17,6 +17,7 @@
#include "io/channel.h"
#include "io/channel-socket.h"
#include "pr-helper.h"
+#include "qapi/qapi-events-block.h"
#include <scsi/sg.h>
@@ -38,6 +39,16 @@
QIOChannel *ioc;
} PRManagerHelper;
+static void pr_manager_send_status_changed_event(PRManagerHelper *pr_mgr)
+{
+ char *id = object_get_canonical_path_component(OBJECT(pr_mgr));
+
+ if (id) {
+ qapi_event_send_pr_manager_status_changed(id, !!pr_mgr->ioc,
+ &error_abort);
+ }
+}
+
/* Called with lock held. */
static int pr_manager_helper_read(PRManagerHelper *pr_mgr,
void *buf, int sz, Error **errp)
@@ -47,6 +58,7 @@
if (r < 0) {
object_unref(OBJECT(pr_mgr->ioc));
pr_mgr->ioc = NULL;
+ pr_manager_send_status_changed_event(pr_mgr);
return -EINVAL;
}
@@ -71,6 +83,8 @@
if (n_written <= 0) {
assert(n_written != QIO_CHANNEL_ERR_BLOCK);
object_unref(OBJECT(pr_mgr->ioc));
+ pr_mgr->ioc = NULL;
+ pr_manager_send_status_changed_event(pr_mgr);
return n_written < 0 ? -EINVAL : 0;
}
@@ -126,6 +140,7 @@
goto out_close;
}
+ pr_manager_send_status_changed_event(pr_mgr);
return 0;
out_close:
@@ -234,6 +249,18 @@
return ret;
}
+static bool pr_manager_helper_is_connected(PRManager *p)
+{
+ PRManagerHelper *pr_mgr = PR_MANAGER_HELPER(p);
+ bool result;
+
+ qemu_mutex_lock(&pr_mgr->lock);
+ result = (pr_mgr->ioc != NULL);
+ qemu_mutex_unlock(&pr_mgr->lock);
+
+ return result;
+}
+
static void pr_manager_helper_complete(UserCreatable *uc, Error **errp)
{
PRManagerHelper *pr_mgr = PR_MANAGER_HELPER(uc);
@@ -283,6 +310,7 @@
&error_abort);
uc_klass->complete = pr_manager_helper_complete;
prmgr_klass->run = pr_manager_helper_run;
+ prmgr_klass->is_connected = pr_manager_helper_is_connected;
}
static const TypeInfo pr_manager_helper_info = {
diff --git a/scsi/pr-manager-stub.c b/scsi/pr-manager-stub.c
new file mode 100644
index 0000000..738b6d7
--- /dev/null
+++ b/scsi/pr-manager-stub.c
@@ -0,0 +1,30 @@
+/*
+ * Persistent reservation manager - stub for non-Linux platforms
+ *
+ * Copyright (c) 2018 Red Hat, Inc.
+ *
+ * Author: Paolo Bonzini <pbonzini@redhat.com>
+ *
+ * This code is licensed under the LGPL.
+ *
+ */
+
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "scsi/pr-manager.h"
+#include "trace.h"
+#include "qapi/qapi-types-block.h"
+#include "qapi/qapi-commands-block.h"
+
+PRManager *pr_manager_lookup(const char *id, Error **errp)
+{
+ /* The classes do not exist at all! */
+ error_setg(errp, "No persistent reservation manager with id '%s'", id);
+ return NULL;
+}
+
+
+PRManagerInfoList *qmp_query_pr_managers(Error **errp)
+{
+ return NULL;
+}
diff --git a/scsi/pr-manager.c b/scsi/pr-manager.c
index 87c45db..2a8f300 100644
--- a/scsi/pr-manager.c
+++ b/scsi/pr-manager.c
@@ -17,6 +17,10 @@
#include "block/thread-pool.h"
#include "scsi/pr-manager.h"
#include "trace.h"
+#include "qapi/qapi-types-block.h"
+#include "qapi/qapi-commands-block.h"
+
+#define PR_MANAGER_PATH "/objects"
typedef struct PRManagerData {
PRManager *pr_mgr;
@@ -64,6 +68,14 @@
data, complete, opaque);
}
+bool pr_manager_is_connected(PRManager *pr_mgr)
+{
+ PRManagerClass *pr_mgr_class =
+ PR_MANAGER_GET_CLASS(pr_mgr);
+
+ return !pr_mgr_class->is_connected || pr_mgr_class->is_connected(pr_mgr);
+}
+
static const TypeInfo pr_manager_info = {
.parent = TYPE_OBJECT,
.name = TYPE_PR_MANAGER,
@@ -105,5 +117,38 @@
type_register_static(&pr_manager_info);
}
+static int query_one_pr_manager(Object *object, void *opaque)
+{
+ PRManagerInfoList ***prev = opaque;
+ PRManagerInfoList *elem;
+ PRManagerInfo *info;
+ PRManager *pr_mgr;
+
+ pr_mgr = (PRManager *)object_dynamic_cast(object, TYPE_PR_MANAGER);
+ if (!pr_mgr) {
+ return 0;
+ }
+
+ elem = g_new0(PRManagerInfoList, 1);
+ info = g_new0(PRManagerInfo, 1);
+ info->id = object_get_canonical_path_component(object);
+ info->connected = pr_manager_is_connected(pr_mgr);
+ elem->value = info;
+ elem->next = NULL;
+
+ **prev = elem;
+ *prev = &elem->next;
+ return 0;
+}
+
+PRManagerInfoList *qmp_query_pr_managers(Error **errp)
+{
+ PRManagerInfoList *head = NULL;
+ PRManagerInfoList **prev = &head;
+ Object *container = container_get(object_get_root(), PR_MANAGER_PATH);
+
+ object_child_foreach(container, query_one_pr_manager, &prev);
+ return head;
+}
type_init(pr_manager_register_types);
diff --git a/scsi/qemu-pr-helper.c b/scsi/qemu-pr-helper.c
index d0f8317..0218d65 100644
--- a/scsi/qemu-pr-helper.c
+++ b/scsi/qemu-pr-helper.c
@@ -74,8 +74,16 @@
static int gid = -1;
#endif
+static void compute_default_paths(void)
+{
+ if (!socket_path) {
+ socket_path = qemu_get_local_state_pathname("run/qemu-pr-helper.sock");
+ }
+}
+
static void usage(const char *name)
{
+ compute_default_paths();
(printf) (
"Usage: %s [OPTIONS] FILE\n"
"Persistent Reservation helper program for QEMU\n"
@@ -550,7 +558,11 @@
#ifdef CONFIG_MPATH
if (is_mpath(fd)) {
/* multipath_pr_in fills the whole input buffer. */
- return multipath_pr_in(fd, cdb, sense, data, *resp_sz);
+ int r = multipath_pr_in(fd, cdb, sense, data, *resp_sz);
+ if (r != GOOD) {
+ *resp_sz = 0;
+ }
+ return r;
}
#endif
@@ -845,13 +857,6 @@
return NULL;
}
-static void compute_default_paths(void)
-{
- if (!socket_path) {
- socket_path = qemu_get_local_state_pathname("run/qemu-pr-helper.sock");
- }
-}
-
static void termsig_handler(int signum)
{
atomic_cmpxchg(&state, RUNNING, TERMINATE);
diff --git a/target/arm/cpu.c b/target/arm/cpu.c
index 2ae4fff..82ff450 100644
--- a/target/arm/cpu.c
+++ b/target/arm/cpu.c
@@ -164,6 +164,13 @@
env->cp15.sctlr_el[1] |= SCTLR_UCT | SCTLR_UCI | SCTLR_DZE;
/* and to the FP/Neon instructions */
env->cp15.cpacr_el1 = deposit64(env->cp15.cpacr_el1, 20, 2, 3);
+ /* and to the SVE instructions */
+ env->cp15.cpacr_el1 = deposit64(env->cp15.cpacr_el1, 16, 2, 3);
+ env->cp15.cptr_el[3] |= CPTR_EZ;
+ /* with maximum vector length */
+ env->vfp.zcr_el[1] = ARM_MAX_VQ - 1;
+ env->vfp.zcr_el[2] = ARM_MAX_VQ - 1;
+ env->vfp.zcr_el[3] = ARM_MAX_VQ - 1;
#else
/* Reset into the highest available EL */
if (arm_feature(env, ARM_FEATURE_EL3)) {
@@ -793,9 +800,20 @@
/* Some features automatically imply others: */
if (arm_feature(env, ARM_FEATURE_V8)) {
- set_feature(env, ARM_FEATURE_V7);
+ set_feature(env, ARM_FEATURE_V7VE);
+ }
+ if (arm_feature(env, ARM_FEATURE_V7VE)) {
+ /* v7 Virtualization Extensions. In real hardware this implies
+ * EL2 and also the presence of the Security Extensions.
+ * For QEMU, for backwards-compatibility we implement some
+ * CPUs or CPU configs which have no actual EL2 or EL3 but do
+ * include the various other features that V7VE implies.
+ * Presence of EL2 itself is ARM_FEATURE_EL2, and of the
+ * Security Extensions is ARM_FEATURE_EL3.
+ */
set_feature(env, ARM_FEATURE_ARM_DIV);
set_feature(env, ARM_FEATURE_LPAE);
+ set_feature(env, ARM_FEATURE_V7);
}
if (arm_feature(env, ARM_FEATURE_V7)) {
set_feature(env, ARM_FEATURE_VAPA);
@@ -1255,6 +1273,7 @@
cpu->id_isar3 = 0x01111110;
cpu->id_isar4 = 0x01310102;
cpu->id_isar5 = 0x00000000;
+ cpu->id_isar6 = 0x00000000;
}
static void cortex_m4_initfn(Object *obj)
@@ -1281,6 +1300,7 @@
cpu->id_isar3 = 0x01111110;
cpu->id_isar4 = 0x01310102;
cpu->id_isar5 = 0x00000000;
+ cpu->id_isar6 = 0x00000000;
}
static void cortex_m33_initfn(Object *obj)
@@ -1309,6 +1329,7 @@
cpu->id_isar3 = 0x01111131;
cpu->id_isar4 = 0x01310132;
cpu->id_isar5 = 0x00000000;
+ cpu->id_isar6 = 0x00000000;
cpu->clidr = 0x00000000;
cpu->ctr = 0x8000c000;
}
@@ -1359,6 +1380,7 @@
cpu->id_isar3 = 0x01112131;
cpu->id_isar4 = 0x0010142;
cpu->id_isar5 = 0x0;
+ cpu->id_isar6 = 0x0;
cpu->mp_is_up = true;
cpu->pmsav7_dregion = 16;
define_arm_cp_regs(cpu, cortexr5_cp_reginfo);
@@ -1517,15 +1539,13 @@
ARMCPU *cpu = ARM_CPU(obj);
cpu->dtb_compatible = "arm,cortex-a7";
- set_feature(&cpu->env, ARM_FEATURE_V7);
+ set_feature(&cpu->env, ARM_FEATURE_V7VE);
set_feature(&cpu->env, ARM_FEATURE_VFP4);
set_feature(&cpu->env, ARM_FEATURE_NEON);
set_feature(&cpu->env, ARM_FEATURE_THUMB2EE);
- set_feature(&cpu->env, ARM_FEATURE_ARM_DIV);
set_feature(&cpu->env, ARM_FEATURE_GENERIC_TIMER);
set_feature(&cpu->env, ARM_FEATURE_DUMMY_C15_REGS);
set_feature(&cpu->env, ARM_FEATURE_CBAR_RO);
- set_feature(&cpu->env, ARM_FEATURE_LPAE);
set_feature(&cpu->env, ARM_FEATURE_EL3);
cpu->kvm_target = QEMU_KVM_ARM_TARGET_CORTEX_A7;
cpu->midr = 0x410fc075;
@@ -1562,15 +1582,13 @@
ARMCPU *cpu = ARM_CPU(obj);
cpu->dtb_compatible = "arm,cortex-a15";
- set_feature(&cpu->env, ARM_FEATURE_V7);
+ set_feature(&cpu->env, ARM_FEATURE_V7VE);
set_feature(&cpu->env, ARM_FEATURE_VFP4);
set_feature(&cpu->env, ARM_FEATURE_NEON);
set_feature(&cpu->env, ARM_FEATURE_THUMB2EE);
- set_feature(&cpu->env, ARM_FEATURE_ARM_DIV);
set_feature(&cpu->env, ARM_FEATURE_GENERIC_TIMER);
set_feature(&cpu->env, ARM_FEATURE_DUMMY_C15_REGS);
set_feature(&cpu->env, ARM_FEATURE_CBAR_RO);
- set_feature(&cpu->env, ARM_FEATURE_LPAE);
set_feature(&cpu->env, ARM_FEATURE_EL3);
cpu->kvm_target = QEMU_KVM_ARM_TARGET_CORTEX_A15;
cpu->midr = 0x412fc0f1;
@@ -1789,15 +1807,13 @@
* since we don't correctly set the ID registers to advertise them,
*/
set_feature(&cpu->env, ARM_FEATURE_V8);
- set_feature(&cpu->env, ARM_FEATURE_VFP4);
- set_feature(&cpu->env, ARM_FEATURE_NEON);
- set_feature(&cpu->env, ARM_FEATURE_THUMB2EE);
set_feature(&cpu->env, ARM_FEATURE_V8_AES);
set_feature(&cpu->env, ARM_FEATURE_V8_SHA1);
set_feature(&cpu->env, ARM_FEATURE_V8_SHA256);
set_feature(&cpu->env, ARM_FEATURE_V8_PMULL);
set_feature(&cpu->env, ARM_FEATURE_CRC);
set_feature(&cpu->env, ARM_FEATURE_V8_RDM);
+ set_feature(&cpu->env, ARM_FEATURE_V8_DOTPROD);
set_feature(&cpu->env, ARM_FEATURE_V8_FCMA);
#endif
}
diff --git a/target/arm/cpu.h b/target/arm/cpu.h
index a4507a2..e310ffc 100644
--- a/target/arm/cpu.h
+++ b/target/arm/cpu.h
@@ -813,6 +813,7 @@
uint32_t id_isar3;
uint32_t id_isar4;
uint32_t id_isar5;
+ uint32_t id_isar6;
uint64_t id_aa64pfr0;
uint64_t id_aa64pfr1;
uint64_t id_aa64dfr0;
@@ -1442,6 +1443,7 @@
ARM_FEATURE_OMAPCP, /* OMAP specific CP15 ops handling. */
ARM_FEATURE_THUMB2EE,
ARM_FEATURE_V7MP, /* v7 Multiprocessing Extensions */
+ ARM_FEATURE_V7VE, /* v7 Virtualization Extensions (non-EL2 parts) */
ARM_FEATURE_V4T,
ARM_FEATURE_V5,
ARM_FEATURE_STRONGARM,
@@ -1480,6 +1482,7 @@
ARM_FEATURE_V8_SM4, /* implements SM4 part of v8 Crypto Extensions */
ARM_FEATURE_V8_ATOMICS, /* ARMv8.1-Atomics feature */
ARM_FEATURE_V8_RDM, /* implements v8.1 simd round multiply */
+ ARM_FEATURE_V8_DOTPROD, /* implements v8.2 simd dot product */
ARM_FEATURE_V8_FP16, /* implements v8.2 half-precision float */
ARM_FEATURE_V8_FCMA, /* has complex number part of v8.3 extensions. */
ARM_FEATURE_M_MAIN, /* M profile Main Extension */
diff --git a/target/arm/cpu64.c b/target/arm/cpu64.c
index c50dcd4..d0581d5 100644
--- a/target/arm/cpu64.c
+++ b/target/arm/cpu64.c
@@ -139,6 +139,7 @@
cpu->id_isar3 = 0x01112131;
cpu->id_isar4 = 0x00011142;
cpu->id_isar5 = 0x00011121;
+ cpu->id_isar6 = 0;
cpu->id_aa64pfr0 = 0x00002222;
cpu->id_aa64dfr0 = 0x10305106;
cpu->pmceid0 = 0x00000000;
@@ -199,6 +200,7 @@
cpu->id_isar3 = 0x01112131;
cpu->id_isar4 = 0x00011142;
cpu->id_isar5 = 0x00011121;
+ cpu->id_isar6 = 0;
cpu->id_aa64pfr0 = 0x00002222;
cpu->id_aa64dfr0 = 0x10305106;
cpu->id_aa64isar0 = 0x00011120;
@@ -235,23 +237,16 @@
* whereas the architecture requires them to be present in both if
* present in either.
*/
- set_feature(&cpu->env, ARM_FEATURE_V8);
- set_feature(&cpu->env, ARM_FEATURE_VFP4);
- set_feature(&cpu->env, ARM_FEATURE_NEON);
- set_feature(&cpu->env, ARM_FEATURE_AARCH64);
- set_feature(&cpu->env, ARM_FEATURE_V8_AES);
- set_feature(&cpu->env, ARM_FEATURE_V8_SHA1);
- set_feature(&cpu->env, ARM_FEATURE_V8_SHA256);
set_feature(&cpu->env, ARM_FEATURE_V8_SHA512);
set_feature(&cpu->env, ARM_FEATURE_V8_SHA3);
set_feature(&cpu->env, ARM_FEATURE_V8_SM3);
set_feature(&cpu->env, ARM_FEATURE_V8_SM4);
- set_feature(&cpu->env, ARM_FEATURE_V8_PMULL);
- set_feature(&cpu->env, ARM_FEATURE_CRC);
set_feature(&cpu->env, ARM_FEATURE_V8_ATOMICS);
set_feature(&cpu->env, ARM_FEATURE_V8_RDM);
+ set_feature(&cpu->env, ARM_FEATURE_V8_DOTPROD);
set_feature(&cpu->env, ARM_FEATURE_V8_FP16);
set_feature(&cpu->env, ARM_FEATURE_V8_FCMA);
+ set_feature(&cpu->env, ARM_FEATURE_SVE);
/* For usermode -cpu max we can use a larger and more efficient DCZ
* blocksize since we don't have to follow what the hardware does.
*/
diff --git a/target/arm/helper-sve.h b/target/arm/helper-sve.h
index 2e76084..023952a 100644
--- a/target/arm/helper-sve.h
+++ b/target/arm/helper-sve.h
@@ -274,6 +274,11 @@
DEF_HELPER_FLAGS_3(sve_clr_s, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
DEF_HELPER_FLAGS_3(sve_clr_d, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve_movz_b, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve_movz_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve_movz_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve_movz_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+
DEF_HELPER_FLAGS_4(sve_asr_zpzi_b, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve_asr_zpzi_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve_asr_zpzi_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
@@ -719,3 +724,680 @@
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(gvec_rsqrts_d, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
+
+DEF_HELPER_FLAGS_4(sve_faddv_h, TCG_CALL_NO_RWG,
+ i64, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve_faddv_s, TCG_CALL_NO_RWG,
+ i64, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve_faddv_d, TCG_CALL_NO_RWG,
+ i64, ptr, ptr, ptr, i32)
+
+DEF_HELPER_FLAGS_4(sve_fmaxnmv_h, TCG_CALL_NO_RWG,
+ i64, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve_fmaxnmv_s, TCG_CALL_NO_RWG,
+ i64, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve_fmaxnmv_d, TCG_CALL_NO_RWG,
+ i64, ptr, ptr, ptr, i32)
+
+DEF_HELPER_FLAGS_4(sve_fminnmv_h, TCG_CALL_NO_RWG,
+ i64, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve_fminnmv_s, TCG_CALL_NO_RWG,
+ i64, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve_fminnmv_d, TCG_CALL_NO_RWG,
+ i64, ptr, ptr, ptr, i32)
+
+DEF_HELPER_FLAGS_4(sve_fmaxv_h, TCG_CALL_NO_RWG,
+ i64, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve_fmaxv_s, TCG_CALL_NO_RWG,
+ i64, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve_fmaxv_d, TCG_CALL_NO_RWG,
+ i64, ptr, ptr, ptr, i32)
+
+DEF_HELPER_FLAGS_4(sve_fminv_h, TCG_CALL_NO_RWG,
+ i64, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve_fminv_s, TCG_CALL_NO_RWG,
+ i64, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve_fminv_d, TCG_CALL_NO_RWG,
+ i64, ptr, ptr, ptr, i32)
+
+DEF_HELPER_FLAGS_5(sve_fadda_h, TCG_CALL_NO_RWG,
+ i64, i64, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve_fadda_s, TCG_CALL_NO_RWG,
+ i64, i64, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve_fadda_d, TCG_CALL_NO_RWG,
+ i64, i64, ptr, ptr, ptr, i32)
+
+DEF_HELPER_FLAGS_5(sve_fcmge0_h, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve_fcmge0_s, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve_fcmge0_d, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+
+DEF_HELPER_FLAGS_5(sve_fcmgt0_h, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve_fcmgt0_s, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve_fcmgt0_d, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+
+DEF_HELPER_FLAGS_5(sve_fcmlt0_h, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve_fcmlt0_s, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve_fcmlt0_d, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+
+DEF_HELPER_FLAGS_5(sve_fcmle0_h, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve_fcmle0_s, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve_fcmle0_d, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+
+DEF_HELPER_FLAGS_5(sve_fcmeq0_h, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve_fcmeq0_s, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve_fcmeq0_d, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+
+DEF_HELPER_FLAGS_5(sve_fcmne0_h, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve_fcmne0_s, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve_fcmne0_d, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+
+DEF_HELPER_FLAGS_6(sve_fadd_h, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_6(sve_fadd_s, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_6(sve_fadd_d, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, ptr, i32)
+
+DEF_HELPER_FLAGS_6(sve_fsub_h, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_6(sve_fsub_s, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_6(sve_fsub_d, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, ptr, i32)
+
+DEF_HELPER_FLAGS_6(sve_fmul_h, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_6(sve_fmul_s, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_6(sve_fmul_d, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, ptr, i32)
+
+DEF_HELPER_FLAGS_6(sve_fdiv_h, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_6(sve_fdiv_s, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_6(sve_fdiv_d, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, ptr, i32)
+
+DEF_HELPER_FLAGS_6(sve_fmin_h, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_6(sve_fmin_s, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_6(sve_fmin_d, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, ptr, i32)
+
+DEF_HELPER_FLAGS_6(sve_fmax_h, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_6(sve_fmax_s, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_6(sve_fmax_d, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, ptr, i32)
+
+DEF_HELPER_FLAGS_6(sve_fminnum_h, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_6(sve_fminnum_s, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_6(sve_fminnum_d, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, ptr, i32)
+
+DEF_HELPER_FLAGS_6(sve_fmaxnum_h, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_6(sve_fmaxnum_s, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_6(sve_fmaxnum_d, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, ptr, i32)
+
+DEF_HELPER_FLAGS_6(sve_fabd_h, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_6(sve_fabd_s, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_6(sve_fabd_d, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, ptr, i32)
+
+DEF_HELPER_FLAGS_6(sve_fscalbn_h, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_6(sve_fscalbn_s, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_6(sve_fscalbn_d, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, ptr, i32)
+
+DEF_HELPER_FLAGS_6(sve_fmulx_h, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_6(sve_fmulx_s, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_6(sve_fmulx_d, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, ptr, i32)
+
+DEF_HELPER_FLAGS_6(sve_fadds_h, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, i64, ptr, i32)
+DEF_HELPER_FLAGS_6(sve_fadds_s, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, i64, ptr, i32)
+DEF_HELPER_FLAGS_6(sve_fadds_d, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, i64, ptr, i32)
+
+DEF_HELPER_FLAGS_6(sve_fsubs_h, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, i64, ptr, i32)
+DEF_HELPER_FLAGS_6(sve_fsubs_s, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, i64, ptr, i32)
+DEF_HELPER_FLAGS_6(sve_fsubs_d, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, i64, ptr, i32)
+
+DEF_HELPER_FLAGS_6(sve_fmuls_h, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, i64, ptr, i32)
+DEF_HELPER_FLAGS_6(sve_fmuls_s, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, i64, ptr, i32)
+DEF_HELPER_FLAGS_6(sve_fmuls_d, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, i64, ptr, i32)
+
+DEF_HELPER_FLAGS_6(sve_fsubrs_h, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, i64, ptr, i32)
+DEF_HELPER_FLAGS_6(sve_fsubrs_s, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, i64, ptr, i32)
+DEF_HELPER_FLAGS_6(sve_fsubrs_d, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, i64, ptr, i32)
+
+DEF_HELPER_FLAGS_6(sve_fmaxnms_h, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, i64, ptr, i32)
+DEF_HELPER_FLAGS_6(sve_fmaxnms_s, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, i64, ptr, i32)
+DEF_HELPER_FLAGS_6(sve_fmaxnms_d, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, i64, ptr, i32)
+
+DEF_HELPER_FLAGS_6(sve_fminnms_h, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, i64, ptr, i32)
+DEF_HELPER_FLAGS_6(sve_fminnms_s, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, i64, ptr, i32)
+DEF_HELPER_FLAGS_6(sve_fminnms_d, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, i64, ptr, i32)
+
+DEF_HELPER_FLAGS_6(sve_fmaxs_h, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, i64, ptr, i32)
+DEF_HELPER_FLAGS_6(sve_fmaxs_s, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, i64, ptr, i32)
+DEF_HELPER_FLAGS_6(sve_fmaxs_d, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, i64, ptr, i32)
+
+DEF_HELPER_FLAGS_6(sve_fmins_h, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, i64, ptr, i32)
+DEF_HELPER_FLAGS_6(sve_fmins_s, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, i64, ptr, i32)
+DEF_HELPER_FLAGS_6(sve_fmins_d, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, i64, ptr, i32)
+
+DEF_HELPER_FLAGS_5(sve_fcvt_sh, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve_fcvt_dh, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve_fcvt_hs, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve_fcvt_ds, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve_fcvt_hd, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve_fcvt_sd, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+
+DEF_HELPER_FLAGS_5(sve_fcvtzs_hh, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve_fcvtzs_hs, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve_fcvtzs_ss, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve_fcvtzs_ds, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve_fcvtzs_hd, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve_fcvtzs_sd, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve_fcvtzs_dd, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+
+DEF_HELPER_FLAGS_5(sve_fcvtzu_hh, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve_fcvtzu_hs, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve_fcvtzu_ss, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve_fcvtzu_ds, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve_fcvtzu_hd, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve_fcvtzu_sd, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve_fcvtzu_dd, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+
+DEF_HELPER_FLAGS_5(sve_frint_h, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve_frint_s, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve_frint_d, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+
+DEF_HELPER_FLAGS_5(sve_frintx_h, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve_frintx_s, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve_frintx_d, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+
+DEF_HELPER_FLAGS_5(sve_frecpx_h, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve_frecpx_s, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve_frecpx_d, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+
+DEF_HELPER_FLAGS_5(sve_fsqrt_h, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve_fsqrt_s, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve_fsqrt_d, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+
+DEF_HELPER_FLAGS_5(sve_scvt_hh, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve_scvt_sh, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve_scvt_dh, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve_scvt_ss, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve_scvt_sd, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve_scvt_ds, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve_scvt_dd, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+
+DEF_HELPER_FLAGS_5(sve_ucvt_hh, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve_ucvt_sh, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve_ucvt_dh, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve_ucvt_ss, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve_ucvt_sd, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve_ucvt_ds, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve_ucvt_dd, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+
+DEF_HELPER_FLAGS_6(sve_fcmge_h, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_6(sve_fcmge_s, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_6(sve_fcmge_d, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, ptr, i32)
+
+DEF_HELPER_FLAGS_6(sve_fcmgt_h, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_6(sve_fcmgt_s, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_6(sve_fcmgt_d, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, ptr, i32)
+
+DEF_HELPER_FLAGS_6(sve_fcmeq_h, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_6(sve_fcmeq_s, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_6(sve_fcmeq_d, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, ptr, i32)
+
+DEF_HELPER_FLAGS_6(sve_fcmne_h, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_6(sve_fcmne_s, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_6(sve_fcmne_d, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, ptr, i32)
+
+DEF_HELPER_FLAGS_6(sve_fcmuo_h, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_6(sve_fcmuo_s, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_6(sve_fcmuo_d, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, ptr, i32)
+
+DEF_HELPER_FLAGS_6(sve_facge_h, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_6(sve_facge_s, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_6(sve_facge_d, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, ptr, i32)
+
+DEF_HELPER_FLAGS_6(sve_facgt_h, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_6(sve_facgt_s, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_6(sve_facgt_d, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, ptr, i32)
+
+DEF_HELPER_FLAGS_6(sve_fcadd_h, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_6(sve_fcadd_s, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_6(sve_fcadd_d, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, ptr, i32)
+
+DEF_HELPER_FLAGS_3(sve_fmla_zpzzz_h, TCG_CALL_NO_RWG, void, env, ptr, i32)
+DEF_HELPER_FLAGS_3(sve_fmla_zpzzz_s, TCG_CALL_NO_RWG, void, env, ptr, i32)
+DEF_HELPER_FLAGS_3(sve_fmla_zpzzz_d, TCG_CALL_NO_RWG, void, env, ptr, i32)
+
+DEF_HELPER_FLAGS_3(sve_fmls_zpzzz_h, TCG_CALL_NO_RWG, void, env, ptr, i32)
+DEF_HELPER_FLAGS_3(sve_fmls_zpzzz_s, TCG_CALL_NO_RWG, void, env, ptr, i32)
+DEF_HELPER_FLAGS_3(sve_fmls_zpzzz_d, TCG_CALL_NO_RWG, void, env, ptr, i32)
+
+DEF_HELPER_FLAGS_3(sve_fnmla_zpzzz_h, TCG_CALL_NO_RWG, void, env, ptr, i32)
+DEF_HELPER_FLAGS_3(sve_fnmla_zpzzz_s, TCG_CALL_NO_RWG, void, env, ptr, i32)
+DEF_HELPER_FLAGS_3(sve_fnmla_zpzzz_d, TCG_CALL_NO_RWG, void, env, ptr, i32)
+
+DEF_HELPER_FLAGS_3(sve_fnmls_zpzzz_h, TCG_CALL_NO_RWG, void, env, ptr, i32)
+DEF_HELPER_FLAGS_3(sve_fnmls_zpzzz_s, TCG_CALL_NO_RWG, void, env, ptr, i32)
+DEF_HELPER_FLAGS_3(sve_fnmls_zpzzz_d, TCG_CALL_NO_RWG, void, env, ptr, i32)
+
+DEF_HELPER_FLAGS_3(sve_fcmla_zpzzz_h, TCG_CALL_NO_RWG, void, env, ptr, i32)
+DEF_HELPER_FLAGS_3(sve_fcmla_zpzzz_s, TCG_CALL_NO_RWG, void, env, ptr, i32)
+DEF_HELPER_FLAGS_3(sve_fcmla_zpzzz_d, TCG_CALL_NO_RWG, void, env, ptr, i32)
+
+DEF_HELPER_FLAGS_5(sve_ftmad_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve_ftmad_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve_ftmad_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32)
+
+DEF_HELPER_FLAGS_4(sve_ld1bb_r, TCG_CALL_NO_WG, void, env, ptr, tl, i32)
+DEF_HELPER_FLAGS_4(sve_ld2bb_r, TCG_CALL_NO_WG, void, env, ptr, tl, i32)
+DEF_HELPER_FLAGS_4(sve_ld3bb_r, TCG_CALL_NO_WG, void, env, ptr, tl, i32)
+DEF_HELPER_FLAGS_4(sve_ld4bb_r, TCG_CALL_NO_WG, void, env, ptr, tl, i32)
+
+DEF_HELPER_FLAGS_4(sve_ld1hh_r, TCG_CALL_NO_WG, void, env, ptr, tl, i32)
+DEF_HELPER_FLAGS_4(sve_ld2hh_r, TCG_CALL_NO_WG, void, env, ptr, tl, i32)
+DEF_HELPER_FLAGS_4(sve_ld3hh_r, TCG_CALL_NO_WG, void, env, ptr, tl, i32)
+DEF_HELPER_FLAGS_4(sve_ld4hh_r, TCG_CALL_NO_WG, void, env, ptr, tl, i32)
+
+DEF_HELPER_FLAGS_4(sve_ld1ss_r, TCG_CALL_NO_WG, void, env, ptr, tl, i32)
+DEF_HELPER_FLAGS_4(sve_ld2ss_r, TCG_CALL_NO_WG, void, env, ptr, tl, i32)
+DEF_HELPER_FLAGS_4(sve_ld3ss_r, TCG_CALL_NO_WG, void, env, ptr, tl, i32)
+DEF_HELPER_FLAGS_4(sve_ld4ss_r, TCG_CALL_NO_WG, void, env, ptr, tl, i32)
+
+DEF_HELPER_FLAGS_4(sve_ld1dd_r, TCG_CALL_NO_WG, void, env, ptr, tl, i32)
+DEF_HELPER_FLAGS_4(sve_ld2dd_r, TCG_CALL_NO_WG, void, env, ptr, tl, i32)
+DEF_HELPER_FLAGS_4(sve_ld3dd_r, TCG_CALL_NO_WG, void, env, ptr, tl, i32)
+DEF_HELPER_FLAGS_4(sve_ld4dd_r, TCG_CALL_NO_WG, void, env, ptr, tl, i32)
+
+DEF_HELPER_FLAGS_4(sve_ld1bhu_r, TCG_CALL_NO_WG, void, env, ptr, tl, i32)
+DEF_HELPER_FLAGS_4(sve_ld1bsu_r, TCG_CALL_NO_WG, void, env, ptr, tl, i32)
+DEF_HELPER_FLAGS_4(sve_ld1bdu_r, TCG_CALL_NO_WG, void, env, ptr, tl, i32)
+DEF_HELPER_FLAGS_4(sve_ld1bhs_r, TCG_CALL_NO_WG, void, env, ptr, tl, i32)
+DEF_HELPER_FLAGS_4(sve_ld1bss_r, TCG_CALL_NO_WG, void, env, ptr, tl, i32)
+DEF_HELPER_FLAGS_4(sve_ld1bds_r, TCG_CALL_NO_WG, void, env, ptr, tl, i32)
+
+DEF_HELPER_FLAGS_4(sve_ld1hsu_r, TCG_CALL_NO_WG, void, env, ptr, tl, i32)
+DEF_HELPER_FLAGS_4(sve_ld1hdu_r, TCG_CALL_NO_WG, void, env, ptr, tl, i32)
+DEF_HELPER_FLAGS_4(sve_ld1hss_r, TCG_CALL_NO_WG, void, env, ptr, tl, i32)
+DEF_HELPER_FLAGS_4(sve_ld1hds_r, TCG_CALL_NO_WG, void, env, ptr, tl, i32)
+
+DEF_HELPER_FLAGS_4(sve_ld1sdu_r, TCG_CALL_NO_WG, void, env, ptr, tl, i32)
+DEF_HELPER_FLAGS_4(sve_ld1sds_r, TCG_CALL_NO_WG, void, env, ptr, tl, i32)
+
+DEF_HELPER_FLAGS_4(sve_ldff1bb_r, TCG_CALL_NO_WG, void, env, ptr, tl, i32)
+DEF_HELPER_FLAGS_4(sve_ldff1bhu_r, TCG_CALL_NO_WG, void, env, ptr, tl, i32)
+DEF_HELPER_FLAGS_4(sve_ldff1bsu_r, TCG_CALL_NO_WG, void, env, ptr, tl, i32)
+DEF_HELPER_FLAGS_4(sve_ldff1bdu_r, TCG_CALL_NO_WG, void, env, ptr, tl, i32)
+DEF_HELPER_FLAGS_4(sve_ldff1bhs_r, TCG_CALL_NO_WG, void, env, ptr, tl, i32)
+DEF_HELPER_FLAGS_4(sve_ldff1bss_r, TCG_CALL_NO_WG, void, env, ptr, tl, i32)
+DEF_HELPER_FLAGS_4(sve_ldff1bds_r, TCG_CALL_NO_WG, void, env, ptr, tl, i32)
+
+DEF_HELPER_FLAGS_4(sve_ldff1hh_r, TCG_CALL_NO_WG, void, env, ptr, tl, i32)
+DEF_HELPER_FLAGS_4(sve_ldff1hsu_r, TCG_CALL_NO_WG, void, env, ptr, tl, i32)
+DEF_HELPER_FLAGS_4(sve_ldff1hdu_r, TCG_CALL_NO_WG, void, env, ptr, tl, i32)
+DEF_HELPER_FLAGS_4(sve_ldff1hss_r, TCG_CALL_NO_WG, void, env, ptr, tl, i32)
+DEF_HELPER_FLAGS_4(sve_ldff1hds_r, TCG_CALL_NO_WG, void, env, ptr, tl, i32)
+
+DEF_HELPER_FLAGS_4(sve_ldff1ss_r, TCG_CALL_NO_WG, void, env, ptr, tl, i32)
+DEF_HELPER_FLAGS_4(sve_ldff1sdu_r, TCG_CALL_NO_WG, void, env, ptr, tl, i32)
+DEF_HELPER_FLAGS_4(sve_ldff1sds_r, TCG_CALL_NO_WG, void, env, ptr, tl, i32)
+
+DEF_HELPER_FLAGS_4(sve_ldff1dd_r, TCG_CALL_NO_WG, void, env, ptr, tl, i32)
+
+DEF_HELPER_FLAGS_4(sve_ldnf1bb_r, TCG_CALL_NO_WG, void, env, ptr, tl, i32)
+DEF_HELPER_FLAGS_4(sve_ldnf1bhu_r, TCG_CALL_NO_WG, void, env, ptr, tl, i32)
+DEF_HELPER_FLAGS_4(sve_ldnf1bsu_r, TCG_CALL_NO_WG, void, env, ptr, tl, i32)
+DEF_HELPER_FLAGS_4(sve_ldnf1bdu_r, TCG_CALL_NO_WG, void, env, ptr, tl, i32)
+DEF_HELPER_FLAGS_4(sve_ldnf1bhs_r, TCG_CALL_NO_WG, void, env, ptr, tl, i32)
+DEF_HELPER_FLAGS_4(sve_ldnf1bss_r, TCG_CALL_NO_WG, void, env, ptr, tl, i32)
+DEF_HELPER_FLAGS_4(sve_ldnf1bds_r, TCG_CALL_NO_WG, void, env, ptr, tl, i32)
+
+DEF_HELPER_FLAGS_4(sve_ldnf1hh_r, TCG_CALL_NO_WG, void, env, ptr, tl, i32)
+DEF_HELPER_FLAGS_4(sve_ldnf1hsu_r, TCG_CALL_NO_WG, void, env, ptr, tl, i32)
+DEF_HELPER_FLAGS_4(sve_ldnf1hdu_r, TCG_CALL_NO_WG, void, env, ptr, tl, i32)
+DEF_HELPER_FLAGS_4(sve_ldnf1hss_r, TCG_CALL_NO_WG, void, env, ptr, tl, i32)
+DEF_HELPER_FLAGS_4(sve_ldnf1hds_r, TCG_CALL_NO_WG, void, env, ptr, tl, i32)
+
+DEF_HELPER_FLAGS_4(sve_ldnf1ss_r, TCG_CALL_NO_WG, void, env, ptr, tl, i32)
+DEF_HELPER_FLAGS_4(sve_ldnf1sdu_r, TCG_CALL_NO_WG, void, env, ptr, tl, i32)
+DEF_HELPER_FLAGS_4(sve_ldnf1sds_r, TCG_CALL_NO_WG, void, env, ptr, tl, i32)
+
+DEF_HELPER_FLAGS_4(sve_ldnf1dd_r, TCG_CALL_NO_WG, void, env, ptr, tl, i32)
+
+DEF_HELPER_FLAGS_4(sve_st1bb_r, TCG_CALL_NO_WG, void, env, ptr, tl, i32)
+DEF_HELPER_FLAGS_4(sve_st2bb_r, TCG_CALL_NO_WG, void, env, ptr, tl, i32)
+DEF_HELPER_FLAGS_4(sve_st3bb_r, TCG_CALL_NO_WG, void, env, ptr, tl, i32)
+DEF_HELPER_FLAGS_4(sve_st4bb_r, TCG_CALL_NO_WG, void, env, ptr, tl, i32)
+
+DEF_HELPER_FLAGS_4(sve_st1hh_r, TCG_CALL_NO_WG, void, env, ptr, tl, i32)
+DEF_HELPER_FLAGS_4(sve_st2hh_r, TCG_CALL_NO_WG, void, env, ptr, tl, i32)
+DEF_HELPER_FLAGS_4(sve_st3hh_r, TCG_CALL_NO_WG, void, env, ptr, tl, i32)
+DEF_HELPER_FLAGS_4(sve_st4hh_r, TCG_CALL_NO_WG, void, env, ptr, tl, i32)
+
+DEF_HELPER_FLAGS_4(sve_st1ss_r, TCG_CALL_NO_WG, void, env, ptr, tl, i32)
+DEF_HELPER_FLAGS_4(sve_st2ss_r, TCG_CALL_NO_WG, void, env, ptr, tl, i32)
+DEF_HELPER_FLAGS_4(sve_st3ss_r, TCG_CALL_NO_WG, void, env, ptr, tl, i32)
+DEF_HELPER_FLAGS_4(sve_st4ss_r, TCG_CALL_NO_WG, void, env, ptr, tl, i32)
+
+DEF_HELPER_FLAGS_4(sve_st1dd_r, TCG_CALL_NO_WG, void, env, ptr, tl, i32)
+DEF_HELPER_FLAGS_4(sve_st2dd_r, TCG_CALL_NO_WG, void, env, ptr, tl, i32)
+DEF_HELPER_FLAGS_4(sve_st3dd_r, TCG_CALL_NO_WG, void, env, ptr, tl, i32)
+DEF_HELPER_FLAGS_4(sve_st4dd_r, TCG_CALL_NO_WG, void, env, ptr, tl, i32)
+
+DEF_HELPER_FLAGS_4(sve_st1bh_r, TCG_CALL_NO_WG, void, env, ptr, tl, i32)
+DEF_HELPER_FLAGS_4(sve_st1bs_r, TCG_CALL_NO_WG, void, env, ptr, tl, i32)
+DEF_HELPER_FLAGS_4(sve_st1bd_r, TCG_CALL_NO_WG, void, env, ptr, tl, i32)
+
+DEF_HELPER_FLAGS_4(sve_st1hs_r, TCG_CALL_NO_WG, void, env, ptr, tl, i32)
+DEF_HELPER_FLAGS_4(sve_st1hd_r, TCG_CALL_NO_WG, void, env, ptr, tl, i32)
+
+DEF_HELPER_FLAGS_4(sve_st1sd_r, TCG_CALL_NO_WG, void, env, ptr, tl, i32)
+
+DEF_HELPER_FLAGS_6(sve_ldbsu_zsu, TCG_CALL_NO_WG,
+ void, env, ptr, ptr, ptr, tl, i32)
+DEF_HELPER_FLAGS_6(sve_ldhsu_zsu, TCG_CALL_NO_WG,
+ void, env, ptr, ptr, ptr, tl, i32)
+DEF_HELPER_FLAGS_6(sve_ldssu_zsu, TCG_CALL_NO_WG,
+ void, env, ptr, ptr, ptr, tl, i32)
+DEF_HELPER_FLAGS_6(sve_ldbss_zsu, TCG_CALL_NO_WG,
+ void, env, ptr, ptr, ptr, tl, i32)
+DEF_HELPER_FLAGS_6(sve_ldhss_zsu, TCG_CALL_NO_WG,
+ void, env, ptr, ptr, ptr, tl, i32)
+
+DEF_HELPER_FLAGS_6(sve_ldbsu_zss, TCG_CALL_NO_WG,
+ void, env, ptr, ptr, ptr, tl, i32)
+DEF_HELPER_FLAGS_6(sve_ldhsu_zss, TCG_CALL_NO_WG,
+ void, env, ptr, ptr, ptr, tl, i32)
+DEF_HELPER_FLAGS_6(sve_ldssu_zss, TCG_CALL_NO_WG,
+ void, env, ptr, ptr, ptr, tl, i32)
+DEF_HELPER_FLAGS_6(sve_ldbss_zss, TCG_CALL_NO_WG,
+ void, env, ptr, ptr, ptr, tl, i32)
+DEF_HELPER_FLAGS_6(sve_ldhss_zss, TCG_CALL_NO_WG,
+ void, env, ptr, ptr, ptr, tl, i32)
+
+DEF_HELPER_FLAGS_6(sve_ldbdu_zsu, TCG_CALL_NO_WG,
+ void, env, ptr, ptr, ptr, tl, i32)
+DEF_HELPER_FLAGS_6(sve_ldhdu_zsu, TCG_CALL_NO_WG,
+ void, env, ptr, ptr, ptr, tl, i32)
+DEF_HELPER_FLAGS_6(sve_ldsdu_zsu, TCG_CALL_NO_WG,
+ void, env, ptr, ptr, ptr, tl, i32)
+DEF_HELPER_FLAGS_6(sve_ldddu_zsu, TCG_CALL_NO_WG,
+ void, env, ptr, ptr, ptr, tl, i32)
+DEF_HELPER_FLAGS_6(sve_ldbds_zsu, TCG_CALL_NO_WG,
+ void, env, ptr, ptr, ptr, tl, i32)
+DEF_HELPER_FLAGS_6(sve_ldhds_zsu, TCG_CALL_NO_WG,
+ void, env, ptr, ptr, ptr, tl, i32)
+DEF_HELPER_FLAGS_6(sve_ldsds_zsu, TCG_CALL_NO_WG,
+ void, env, ptr, ptr, ptr, tl, i32)
+
+DEF_HELPER_FLAGS_6(sve_ldbdu_zss, TCG_CALL_NO_WG,
+ void, env, ptr, ptr, ptr, tl, i32)
+DEF_HELPER_FLAGS_6(sve_ldhdu_zss, TCG_CALL_NO_WG,
+ void, env, ptr, ptr, ptr, tl, i32)
+DEF_HELPER_FLAGS_6(sve_ldsdu_zss, TCG_CALL_NO_WG,
+ void, env, ptr, ptr, ptr, tl, i32)
+DEF_HELPER_FLAGS_6(sve_ldddu_zss, TCG_CALL_NO_WG,
+ void, env, ptr, ptr, ptr, tl, i32)
+DEF_HELPER_FLAGS_6(sve_ldbds_zss, TCG_CALL_NO_WG,
+ void, env, ptr, ptr, ptr, tl, i32)
+DEF_HELPER_FLAGS_6(sve_ldhds_zss, TCG_CALL_NO_WG,
+ void, env, ptr, ptr, ptr, tl, i32)
+DEF_HELPER_FLAGS_6(sve_ldsds_zss, TCG_CALL_NO_WG,
+ void, env, ptr, ptr, ptr, tl, i32)
+
+DEF_HELPER_FLAGS_6(sve_ldbdu_zd, TCG_CALL_NO_WG,
+ void, env, ptr, ptr, ptr, tl, i32)
+DEF_HELPER_FLAGS_6(sve_ldhdu_zd, TCG_CALL_NO_WG,
+ void, env, ptr, ptr, ptr, tl, i32)
+DEF_HELPER_FLAGS_6(sve_ldsdu_zd, TCG_CALL_NO_WG,
+ void, env, ptr, ptr, ptr, tl, i32)
+DEF_HELPER_FLAGS_6(sve_ldddu_zd, TCG_CALL_NO_WG,
+ void, env, ptr, ptr, ptr, tl, i32)
+DEF_HELPER_FLAGS_6(sve_ldbds_zd, TCG_CALL_NO_WG,
+ void, env, ptr, ptr, ptr, tl, i32)
+DEF_HELPER_FLAGS_6(sve_ldhds_zd, TCG_CALL_NO_WG,
+ void, env, ptr, ptr, ptr, tl, i32)
+DEF_HELPER_FLAGS_6(sve_ldsds_zd, TCG_CALL_NO_WG,
+ void, env, ptr, ptr, ptr, tl, i32)
+
+DEF_HELPER_FLAGS_6(sve_ldffbsu_zsu, TCG_CALL_NO_WG,
+ void, env, ptr, ptr, ptr, tl, i32)
+DEF_HELPER_FLAGS_6(sve_ldffhsu_zsu, TCG_CALL_NO_WG,
+ void, env, ptr, ptr, ptr, tl, i32)
+DEF_HELPER_FLAGS_6(sve_ldffssu_zsu, TCG_CALL_NO_WG,
+ void, env, ptr, ptr, ptr, tl, i32)
+DEF_HELPER_FLAGS_6(sve_ldffbss_zsu, TCG_CALL_NO_WG,
+ void, env, ptr, ptr, ptr, tl, i32)
+DEF_HELPER_FLAGS_6(sve_ldffhss_zsu, TCG_CALL_NO_WG,
+ void, env, ptr, ptr, ptr, tl, i32)
+
+DEF_HELPER_FLAGS_6(sve_ldffbsu_zss, TCG_CALL_NO_WG,
+ void, env, ptr, ptr, ptr, tl, i32)
+DEF_HELPER_FLAGS_6(sve_ldffhsu_zss, TCG_CALL_NO_WG,
+ void, env, ptr, ptr, ptr, tl, i32)
+DEF_HELPER_FLAGS_6(sve_ldffssu_zss, TCG_CALL_NO_WG,
+ void, env, ptr, ptr, ptr, tl, i32)
+DEF_HELPER_FLAGS_6(sve_ldffbss_zss, TCG_CALL_NO_WG,
+ void, env, ptr, ptr, ptr, tl, i32)
+DEF_HELPER_FLAGS_6(sve_ldffhss_zss, TCG_CALL_NO_WG,
+ void, env, ptr, ptr, ptr, tl, i32)
+
+DEF_HELPER_FLAGS_6(sve_ldffbdu_zsu, TCG_CALL_NO_WG,
+ void, env, ptr, ptr, ptr, tl, i32)
+DEF_HELPER_FLAGS_6(sve_ldffhdu_zsu, TCG_CALL_NO_WG,
+ void, env, ptr, ptr, ptr, tl, i32)
+DEF_HELPER_FLAGS_6(sve_ldffsdu_zsu, TCG_CALL_NO_WG,
+ void, env, ptr, ptr, ptr, tl, i32)
+DEF_HELPER_FLAGS_6(sve_ldffddu_zsu, TCG_CALL_NO_WG,
+ void, env, ptr, ptr, ptr, tl, i32)
+DEF_HELPER_FLAGS_6(sve_ldffbds_zsu, TCG_CALL_NO_WG,
+ void, env, ptr, ptr, ptr, tl, i32)
+DEF_HELPER_FLAGS_6(sve_ldffhds_zsu, TCG_CALL_NO_WG,
+ void, env, ptr, ptr, ptr, tl, i32)
+DEF_HELPER_FLAGS_6(sve_ldffsds_zsu, TCG_CALL_NO_WG,
+ void, env, ptr, ptr, ptr, tl, i32)
+
+DEF_HELPER_FLAGS_6(sve_ldffbdu_zss, TCG_CALL_NO_WG,
+ void, env, ptr, ptr, ptr, tl, i32)
+DEF_HELPER_FLAGS_6(sve_ldffhdu_zss, TCG_CALL_NO_WG,
+ void, env, ptr, ptr, ptr, tl, i32)
+DEF_HELPER_FLAGS_6(sve_ldffsdu_zss, TCG_CALL_NO_WG,
+ void, env, ptr, ptr, ptr, tl, i32)
+DEF_HELPER_FLAGS_6(sve_ldffddu_zss, TCG_CALL_NO_WG,
+ void, env, ptr, ptr, ptr, tl, i32)
+DEF_HELPER_FLAGS_6(sve_ldffbds_zss, TCG_CALL_NO_WG,
+ void, env, ptr, ptr, ptr, tl, i32)
+DEF_HELPER_FLAGS_6(sve_ldffhds_zss, TCG_CALL_NO_WG,
+ void, env, ptr, ptr, ptr, tl, i32)
+DEF_HELPER_FLAGS_6(sve_ldffsds_zss, TCG_CALL_NO_WG,
+ void, env, ptr, ptr, ptr, tl, i32)
+
+DEF_HELPER_FLAGS_6(sve_ldffbdu_zd, TCG_CALL_NO_WG,
+ void, env, ptr, ptr, ptr, tl, i32)
+DEF_HELPER_FLAGS_6(sve_ldffhdu_zd, TCG_CALL_NO_WG,
+ void, env, ptr, ptr, ptr, tl, i32)
+DEF_HELPER_FLAGS_6(sve_ldffsdu_zd, TCG_CALL_NO_WG,
+ void, env, ptr, ptr, ptr, tl, i32)
+DEF_HELPER_FLAGS_6(sve_ldffddu_zd, TCG_CALL_NO_WG,
+ void, env, ptr, ptr, ptr, tl, i32)
+DEF_HELPER_FLAGS_6(sve_ldffbds_zd, TCG_CALL_NO_WG,
+ void, env, ptr, ptr, ptr, tl, i32)
+DEF_HELPER_FLAGS_6(sve_ldffhds_zd, TCG_CALL_NO_WG,
+ void, env, ptr, ptr, ptr, tl, i32)
+DEF_HELPER_FLAGS_6(sve_ldffsds_zd, TCG_CALL_NO_WG,
+ void, env, ptr, ptr, ptr, tl, i32)
+
+DEF_HELPER_FLAGS_6(sve_stbs_zsu, TCG_CALL_NO_WG,
+ void, env, ptr, ptr, ptr, tl, i32)
+DEF_HELPER_FLAGS_6(sve_sths_zsu, TCG_CALL_NO_WG,
+ void, env, ptr, ptr, ptr, tl, i32)
+DEF_HELPER_FLAGS_6(sve_stss_zsu, TCG_CALL_NO_WG,
+ void, env, ptr, ptr, ptr, tl, i32)
+
+DEF_HELPER_FLAGS_6(sve_stbs_zss, TCG_CALL_NO_WG,
+ void, env, ptr, ptr, ptr, tl, i32)
+DEF_HELPER_FLAGS_6(sve_sths_zss, TCG_CALL_NO_WG,
+ void, env, ptr, ptr, ptr, tl, i32)
+DEF_HELPER_FLAGS_6(sve_stss_zss, TCG_CALL_NO_WG,
+ void, env, ptr, ptr, ptr, tl, i32)
+
+DEF_HELPER_FLAGS_6(sve_stbd_zsu, TCG_CALL_NO_WG,
+ void, env, ptr, ptr, ptr, tl, i32)
+DEF_HELPER_FLAGS_6(sve_sthd_zsu, TCG_CALL_NO_WG,
+ void, env, ptr, ptr, ptr, tl, i32)
+DEF_HELPER_FLAGS_6(sve_stsd_zsu, TCG_CALL_NO_WG,
+ void, env, ptr, ptr, ptr, tl, i32)
+DEF_HELPER_FLAGS_6(sve_stdd_zsu, TCG_CALL_NO_WG,
+ void, env, ptr, ptr, ptr, tl, i32)
+
+DEF_HELPER_FLAGS_6(sve_stbd_zss, TCG_CALL_NO_WG,
+ void, env, ptr, ptr, ptr, tl, i32)
+DEF_HELPER_FLAGS_6(sve_sthd_zss, TCG_CALL_NO_WG,
+ void, env, ptr, ptr, ptr, tl, i32)
+DEF_HELPER_FLAGS_6(sve_stsd_zss, TCG_CALL_NO_WG,
+ void, env, ptr, ptr, ptr, tl, i32)
+DEF_HELPER_FLAGS_6(sve_stdd_zss, TCG_CALL_NO_WG,
+ void, env, ptr, ptr, ptr, tl, i32)
+
+DEF_HELPER_FLAGS_6(sve_stbd_zd, TCG_CALL_NO_WG,
+ void, env, ptr, ptr, ptr, tl, i32)
+DEF_HELPER_FLAGS_6(sve_sthd_zd, TCG_CALL_NO_WG,
+ void, env, ptr, ptr, ptr, tl, i32)
+DEF_HELPER_FLAGS_6(sve_stsd_zd, TCG_CALL_NO_WG,
+ void, env, ptr, ptr, ptr, tl, i32)
+DEF_HELPER_FLAGS_6(sve_stdd_zd, TCG_CALL_NO_WG,
+ void, env, ptr, ptr, ptr, tl, i32)
diff --git a/target/arm/helper.c b/target/arm/helper.c
index 3c6a4c5..a2ac960 100644
--- a/target/arm/helper.c
+++ b/target/arm/helper.c
@@ -1404,7 +1404,7 @@
.writefn = pmuserenr_write, .raw_writefn = raw_write },
{ .name = "PMINTENSET", .cp = 15, .crn = 9, .crm = 14, .opc1 = 0, .opc2 = 1,
.access = PL1_RW, .accessfn = access_tpm,
- .type = ARM_CP_ALIAS,
+ .type = ARM_CP_ALIAS | ARM_CP_IO,
.fieldoffset = offsetoflow32(CPUARMState, cp15.c9_pminten),
.resetvalue = 0,
.writefn = pmintenset_write, .raw_writefn = raw_write },
@@ -2167,11 +2167,32 @@
};
#else
-/* In user-mode none of the generic timer registers are accessible,
- * and their implementation depends on QEMU_CLOCK_VIRTUAL and qdev gpio outputs,
- * so instead just don't register any of them.
+
+/* In user-mode most of the generic timer registers are inaccessible
+ * however modern kernels (4.12+) allow access to cntvct_el0
*/
+
+static uint64_t gt_virt_cnt_read(CPUARMState *env, const ARMCPRegInfo *ri)
+{
+ /* Currently we have no support for QEMUTimer in linux-user so we
+ * can't call gt_get_countervalue(env), instead we directly
+ * call the lower level functions.
+ */
+ return cpu_get_clock() / GTIMER_SCALE;
+}
+
static const ARMCPRegInfo generic_timer_cp_reginfo[] = {
+ { .name = "CNTFRQ_EL0", .state = ARM_CP_STATE_AA64,
+ .opc0 = 3, .opc1 = 3, .crn = 14, .crm = 0, .opc2 = 0,
+ .type = ARM_CP_CONST, .access = PL0_R /* no PL1_RW in linux-user */,
+ .fieldoffset = offsetof(CPUARMState, cp15.c14_cntfrq),
+ .resetvalue = NANOSECONDS_PER_SECOND / GTIMER_SCALE,
+ },
+ { .name = "CNTVCT_EL0", .state = ARM_CP_STATE_AA64,
+ .opc0 = 3, .opc1 = 3, .crn = 14, .crm = 0, .opc2 = 2,
+ .access = PL0_R, .type = ARM_CP_NO_RAW | ARM_CP_IO,
+ .readfn = gt_virt_cnt_read,
+ },
REGINFO_SENTINEL
};
@@ -4393,7 +4414,7 @@
static const ARMCPRegInfo zcr_el1_reginfo = {
.name = "ZCR_EL1", .state = ARM_CP_STATE_AA64,
.opc0 = 3, .opc1 = 0, .crn = 1, .crm = 2, .opc2 = 0,
- .access = PL1_RW, .type = ARM_CP_SVE | ARM_CP_FPU,
+ .access = PL1_RW, .type = ARM_CP_SVE,
.fieldoffset = offsetof(CPUARMState, vfp.zcr_el[1]),
.writefn = zcr_write, .raw_writefn = raw_write
};
@@ -4401,7 +4422,7 @@
static const ARMCPRegInfo zcr_el2_reginfo = {
.name = "ZCR_EL2", .state = ARM_CP_STATE_AA64,
.opc0 = 3, .opc1 = 4, .crn = 1, .crm = 2, .opc2 = 0,
- .access = PL2_RW, .type = ARM_CP_SVE | ARM_CP_FPU,
+ .access = PL2_RW, .type = ARM_CP_SVE,
.fieldoffset = offsetof(CPUARMState, vfp.zcr_el[2]),
.writefn = zcr_write, .raw_writefn = raw_write
};
@@ -4409,14 +4430,14 @@
static const ARMCPRegInfo zcr_no_el2_reginfo = {
.name = "ZCR_EL2", .state = ARM_CP_STATE_AA64,
.opc0 = 3, .opc1 = 4, .crn = 1, .crm = 2, .opc2 = 0,
- .access = PL2_RW, .type = ARM_CP_SVE | ARM_CP_FPU,
+ .access = PL2_RW, .type = ARM_CP_SVE,
.readfn = arm_cp_read_zero, .writefn = arm_cp_write_ignore
};
static const ARMCPRegInfo zcr_el3_reginfo = {
.name = "ZCR_EL3", .state = ARM_CP_STATE_AA64,
.opc0 = 3, .opc1 = 6, .crn = 1, .crm = 2, .opc2 = 0,
- .access = PL3_RW, .type = ARM_CP_SVE | ARM_CP_FPU,
+ .access = PL3_RW, .type = ARM_CP_SVE,
.fieldoffset = offsetof(CPUARMState, vfp.zcr_el[3]),
.writefn = zcr_write, .raw_writefn = raw_write
};
@@ -4851,11 +4872,10 @@
.opc0 = 3, .opc1 = 0, .crn = 0, .crm = 2, .opc2 = 6,
.access = PL1_R, .type = ARM_CP_CONST,
.resetvalue = cpu->id_mmfr4 },
- /* 7 is as yet unallocated and must RAZ */
- { .name = "ID_ISAR7_RESERVED", .state = ARM_CP_STATE_BOTH,
+ { .name = "ID_ISAR6", .state = ARM_CP_STATE_BOTH,
.opc0 = 3, .opc1 = 0, .crn = 0, .crm = 2, .opc2 = 7,
.access = PL1_R, .type = ARM_CP_CONST,
- .resetvalue = 0 },
+ .resetvalue = cpu->id_isar6 },
REGINFO_SENTINEL
};
define_arm_cp_regs(cpu, v6_idregs);
@@ -11407,7 +11427,7 @@
}
#define CONV_FTOI(name, ftype, fsz, sign, round) \
-uint32_t HELPER(name)(ftype x, void *fpstp) \
+sign##int32_t HELPER(name)(ftype x, void *fpstp) \
{ \
float_status *fpst = fpstp; \
if (float##fsz##_is_any_nan(x)) { \
diff --git a/target/arm/helper.h b/target/arm/helper.h
index 879a722..59e8c3b 100644
--- a/target/arm/helper.h
+++ b/target/arm/helper.h
@@ -134,12 +134,12 @@
DEF_HELPER_2(vfp_touizh, i32, f16, ptr)
DEF_HELPER_2(vfp_touizs, i32, f32, ptr)
DEF_HELPER_2(vfp_touizd, i32, f64, ptr)
-DEF_HELPER_2(vfp_tosih, i32, f16, ptr)
-DEF_HELPER_2(vfp_tosis, i32, f32, ptr)
-DEF_HELPER_2(vfp_tosid, i32, f64, ptr)
-DEF_HELPER_2(vfp_tosizh, i32, f16, ptr)
-DEF_HELPER_2(vfp_tosizs, i32, f32, ptr)
-DEF_HELPER_2(vfp_tosizd, i32, f64, ptr)
+DEF_HELPER_2(vfp_tosih, s32, f16, ptr)
+DEF_HELPER_2(vfp_tosis, s32, f32, ptr)
+DEF_HELPER_2(vfp_tosid, s32, f64, ptr)
+DEF_HELPER_2(vfp_tosizh, s32, f16, ptr)
+DEF_HELPER_2(vfp_tosizs, s32, f32, ptr)
+DEF_HELPER_2(vfp_tosizd, s32, f64, ptr)
DEF_HELPER_3(vfp_toshs_round_to_zero, i32, f32, i32, ptr)
DEF_HELPER_3(vfp_tosls_round_to_zero, i32, f32, i32, ptr)
@@ -583,6 +583,16 @@
DEF_HELPER_FLAGS_5(gvec_qrdmlsh_s32, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(gvec_sdot_b, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(gvec_udot_b, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(gvec_sdot_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(gvec_udot_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+
+DEF_HELPER_FLAGS_4(gvec_sdot_idx_b, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(gvec_udot_idx_b, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(gvec_sdot_idx_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(gvec_udot_idx_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+
DEF_HELPER_FLAGS_5(gvec_fcaddh, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(gvec_fcadds, TCG_CALL_NO_RWG,
@@ -601,6 +611,14 @@
DEF_HELPER_FLAGS_5(gvec_fcmlad, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(gvec_frecpe_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(gvec_frecpe_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(gvec_frecpe_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+
+DEF_HELPER_FLAGS_4(gvec_frsqrte_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(gvec_frsqrte_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(gvec_frsqrte_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+
DEF_HELPER_FLAGS_5(gvec_fadd_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(gvec_fadd_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(gvec_fadd_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32)
@@ -620,6 +638,20 @@
DEF_HELPER_FLAGS_5(gvec_ftsmul_d, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(gvec_fmul_idx_h, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(gvec_fmul_idx_s, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(gvec_fmul_idx_d, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+
+DEF_HELPER_FLAGS_6(gvec_fmla_idx_h, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_6(gvec_fmla_idx_s, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_6(gvec_fmla_idx_d, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, ptr, i32)
+
#ifdef TARGET_AARCH64
#include "helper-a64.h"
#include "helper-sve.h"
diff --git a/target/arm/kvm32.c b/target/arm/kvm32.c
index 1740cda..4e91c11 100644
--- a/target/arm/kvm32.c
+++ b/target/arm/kvm32.c
@@ -36,7 +36,7 @@
* and then query that CPU for the relevant ID registers.
*/
int i, ret, fdarray[3];
- uint32_t midr, id_pfr0, id_isar0, mvfr1;
+ uint32_t midr, id_pfr0, mvfr1;
uint64_t features = 0;
/* Old kernels may not know about the PREFERRED_TARGET ioctl: however
* we know these will only support creating one kind of guest CPU,
@@ -60,11 +60,6 @@
},
{
.id = KVM_REG_ARM | KVM_REG_SIZE_U32
- | ENCODE_CP_REG(15, 0, 0, 0, 2, 0, 0),
- .addr = (uintptr_t)&id_isar0,
- },
- {
- .id = KVM_REG_ARM | KVM_REG_SIZE_U32
| KVM_REG_ARM_VFP | KVM_REG_ARM_VFP_MVFR1,
.addr = (uintptr_t)&mvfr1,
},
@@ -98,26 +93,14 @@
/* Now we've retrieved all the register information we can
* set the feature bits based on the ID register fields.
* We can assume any KVM supporting CPU is at least a v7
- * with VFPv3, LPAE and the generic timers; this in turn implies
- * most of the other feature bits, but a few must be tested.
+ * with VFPv3, virtualization extensions, and the generic
+ * timers; this in turn implies most of the other feature
+ * bits, but a few must be tested.
*/
- set_feature(&features, ARM_FEATURE_V7);
+ set_feature(&features, ARM_FEATURE_V7VE);
set_feature(&features, ARM_FEATURE_VFP3);
- set_feature(&features, ARM_FEATURE_LPAE);
set_feature(&features, ARM_FEATURE_GENERIC_TIMER);
- switch (extract32(id_isar0, 24, 4)) {
- case 1:
- set_feature(&features, ARM_FEATURE_THUMB_DIV);
- break;
- case 2:
- set_feature(&features, ARM_FEATURE_ARM_DIV);
- set_feature(&features, ARM_FEATURE_THUMB_DIV);
- break;
- default:
- break;
- }
-
if (extract32(id_pfr0, 12, 4) == 1) {
set_feature(&features, ARM_FEATURE_THUMB2EE);
}
diff --git a/target/arm/sve.decode b/target/arm/sve.decode
index 6f436f9..e10b689 100644
--- a/target/arm/sve.decode
+++ b/target/arm/sve.decode
@@ -27,6 +27,9 @@
%imm7_22_16 22:2 16:5
%imm8_16_10 16:5 10:3
%imm9_16_10 16:s6 10:3
+%size_23 23:2
+%dtype_23_13 23:2 13:2
+%index3_22_19 22:1 19:2
# A combination of tsz:imm3 -- extract esize.
%tszimm_esz 22:2 5:5 !function=tszimm_esz
@@ -45,6 +48,9 @@
# Unsigned 8-bit immediate, optionally shifted left by 8.
%sh8_i8u 5:9 !function=expand_imm_sh8u
+# Unsigned load of msz into esz=2, represented as a dtype.
+%msz_dtype 23:2 !function=msz_dtype
+
# Either a copy of rd (at bit 0), or a different source
# as propagated via the MOVPRFX instruction.
%reg_movprfx 0:5
@@ -71,6 +77,14 @@
&incdec2_cnt rd rn pat esz imm d u
&incdec_pred rd pg esz d u
&incdec2_pred rd rn pg esz d u
+&rprr_load rd pg rn rm dtype nreg
+&rpri_load rd pg rn imm dtype nreg
+&rprr_store rd pg rn rm msz esz nreg
+&rpri_store rd pg rn imm msz esz nreg
+&rprr_gather_load rd pg rn rm esz msz u ff xs scale
+&rpri_gather_load rd pg rn imm esz msz u ff
+&rprr_scatter_store rd pg rn rm esz msz xs scale
+&rpri_scatter_store rd pg rn imm esz msz
###########################################################################
# Named instruction formats. These are generally used to
@@ -120,10 +134,16 @@
&rprrr_esz ra=%reg_movprfx
@rdn_pg_ra_rm ........ esz:2 . rm:5 ... pg:3 ra:5 rd:5 \
&rprrr_esz rn=%reg_movprfx
+@rdn_pg_rm_ra ........ esz:2 . ra:5 ... pg:3 rm:5 rd:5 \
+ &rprrr_esz rn=%reg_movprfx
# One register operand, with governing predicate, vector element size
@rd_pg_rn ........ esz:2 ... ... ... pg:3 rn:5 rd:5 &rpr_esz
@rd_pg4_pn ........ esz:2 ... ... .. pg:4 . rn:4 rd:5 &rpr_esz
+@pd_pg_rn ........ esz:2 ... ... ... pg:3 rn:5 . rd:4 &rpr_esz
+
+# One register operand, with governing predicate, no vector element size
+@rd_pg_rn_e0 ........ .. ... ... ... pg:3 rn:5 rd:5 &rpr_esz esz=0
# Two register operands with a 6-bit signed immediate.
@rd_rn_i6 ........ ... rn:5 ..... imm:s6 rd:5 &rri
@@ -142,6 +162,10 @@
@rdn_pg4 ........ esz:2 .. pg:4 ... ........ rd:5 \
&rpri_esz rn=%reg_movprfx
+# Two register operand, one one-bit floating-point operand.
+@rdn_i1 ........ esz:2 ......... pg:3 .... imm:1 rd:5 \
+ &rpri_esz rn=%reg_movprfx
+
# Two register operand, one encoded bitmask.
@rdn_dbm ........ .. .... dbm:13 rd:5 \
&rr_dbm rn=%reg_movprfx
@@ -170,6 +194,41 @@
@incdec2_pred ........ esz:2 .... .. ..... .. pg:4 rd:5 \
&incdec2_pred rn=%reg_movprfx
+# Loads; user must fill in NREG.
+@rprr_load_dt ....... dtype:4 rm:5 ... pg:3 rn:5 rd:5 &rprr_load
+@rpri_load_dt ....... dtype:4 . imm:s4 ... pg:3 rn:5 rd:5 &rpri_load
+
+@rprr_load_msz ....... .... rm:5 ... pg:3 rn:5 rd:5 \
+ &rprr_load dtype=%msz_dtype
+@rpri_load_msz ....... .... . imm:s4 ... pg:3 rn:5 rd:5 \
+ &rpri_load dtype=%msz_dtype
+
+# Gather Loads.
+@rprr_g_load_u ....... .. . . rm:5 . u:1 ff:1 pg:3 rn:5 rd:5 \
+ &rprr_gather_load xs=2
+@rprr_g_load_xs_u ....... .. xs:1 . rm:5 . u:1 ff:1 pg:3 rn:5 rd:5 \
+ &rprr_gather_load
+@rprr_g_load_xs_u_sc ....... .. xs:1 scale:1 rm:5 . u:1 ff:1 pg:3 rn:5 rd:5 \
+ &rprr_gather_load
+@rprr_g_load_xs_sc ....... .. xs:1 scale:1 rm:5 . . ff:1 pg:3 rn:5 rd:5 \
+ &rprr_gather_load
+@rprr_g_load_u_sc ....... .. . scale:1 rm:5 . u:1 ff:1 pg:3 rn:5 rd:5 \
+ &rprr_gather_load xs=2
+@rprr_g_load_sc ....... .. . scale:1 rm:5 . . ff:1 pg:3 rn:5 rd:5 \
+ &rprr_gather_load xs=2
+@rpri_g_load ....... msz:2 .. imm:5 . u:1 ff:1 pg:3 rn:5 rd:5 \
+ &rpri_gather_load
+
+# Stores; user must fill in ESZ, MSZ, NREG as needed.
+@rprr_store ....... .. .. rm:5 ... pg:3 rn:5 rd:5 &rprr_store
+@rpri_store_msz ....... msz:2 .. . imm:s4 ... pg:3 rn:5 rd:5 &rpri_store
+@rprr_store_esz_n0 ....... .. esz:2 rm:5 ... pg:3 rn:5 rd:5 \
+ &rprr_store nreg=0
+@rprr_scatter_store ....... msz:2 .. rm:5 ... pg:3 rn:5 rd:5 \
+ &rprr_scatter_store
+@rpri_scatter_store ....... msz:2 .. imm:5 ... pg:3 rn:5 rd:5 \
+ &rpri_scatter_store
+
###########################################################################
# Instruction patterns. Grouped according to the SVE encodingindex.xhtml.
@@ -211,6 +270,10 @@
EORV 00000100 .. 011 001 001 ... ..... ..... @rd_pg_rn
ANDV 00000100 .. 011 010 001 ... ..... ..... @rd_pg_rn
+# SVE constructive prefix (predicated)
+MOVPRFX_z 00000100 .. 010 000 001 ... ..... ..... @rd_pg_rn
+MOVPRFX_m 00000100 .. 010 001 001 ... ..... ..... @rd_pg_rn
+
# SVE integer add reduction (predicated)
# Note that saddv requires size != 3.
UADDV 00000100 .. 000 001 001 ... ..... ..... @rd_pg_rn
@@ -271,6 +334,17 @@
SXTW 00000100 .. 010 100 101 ... ..... ..... @rd_pg_rn
UXTW 00000100 .. 010 101 101 ... ..... ..... @rd_pg_rn
+### SVE Floating Point Compare - Vectors Group
+
+# SVE floating-point compare vectors
+FCMGE_ppzz 01100101 .. 0 ..... 010 ... ..... 0 .... @pd_pg_rn_rm
+FCMGT_ppzz 01100101 .. 0 ..... 010 ... ..... 1 .... @pd_pg_rn_rm
+FCMEQ_ppzz 01100101 .. 0 ..... 011 ... ..... 0 .... @pd_pg_rn_rm
+FCMNE_ppzz 01100101 .. 0 ..... 011 ... ..... 1 .... @pd_pg_rn_rm
+FCMUO_ppzz 01100101 .. 0 ..... 110 ... ..... 0 .... @pd_pg_rn_rm
+FACGE_ppzz 01100101 .. 0 ..... 110 ... ..... 1 .... @pd_pg_rn_rm
+FACGT_ppzz 01100101 .. 0 ..... 111 ... ..... 1 .... @pd_pg_rn_rm
+
### SVE Integer Multiply-Add Group
# SVE integer multiply-add writing addend (predicated)
@@ -348,6 +422,9 @@
### SVE Integer Misc - Unpredicated Group
+# SVE constructive prefix (unpredicated)
+MOVPRFX 00000100 00 1 00000 101111 rn:5 rd:5
+
# SVE floating-point exponential accelerator
# Note esz != 0
FEXPA 00000100 .. 1 00000 101110 ..... ..... @rd_rn
@@ -648,6 +725,74 @@
# SVE integer multiply immediate (unpredicated)
MUL_zzi 00100101 .. 110 000 110 ........ ..... @rdn_i8s
+# SVE integer dot product (unpredicated)
+DOT_zzz 01000100 1 sz:1 0 rm:5 00000 u:1 rn:5 rd:5 ra=%reg_movprfx
+
+# SVE integer dot product (indexed)
+DOT_zzx 01000100 101 index:2 rm:3 00000 u:1 rn:5 rd:5 \
+ sz=0 ra=%reg_movprfx
+DOT_zzx 01000100 111 index:1 rm:4 00000 u:1 rn:5 rd:5 \
+ sz=1 ra=%reg_movprfx
+
+# SVE floating-point complex add (predicated)
+FCADD 01100100 esz:2 00000 rot:1 100 pg:3 rm:5 rd:5 \
+ rn=%reg_movprfx
+
+# SVE floating-point complex multiply-add (predicated)
+FCMLA_zpzzz 01100100 esz:2 0 rm:5 0 rot:2 pg:3 rn:5 rd:5 \
+ ra=%reg_movprfx
+
+# SVE floating-point complex multiply-add (indexed)
+FCMLA_zzxz 01100100 10 1 index:2 rm:3 0001 rot:2 rn:5 rd:5 \
+ ra=%reg_movprfx esz=1
+FCMLA_zzxz 01100100 11 1 index:1 rm:4 0001 rot:2 rn:5 rd:5 \
+ ra=%reg_movprfx esz=2
+
+### SVE FP Multiply-Add Indexed Group
+
+# SVE floating-point multiply-add (indexed)
+FMLA_zzxz 01100100 0.1 .. rm:3 00000 sub:1 rn:5 rd:5 \
+ ra=%reg_movprfx index=%index3_22_19 esz=1
+FMLA_zzxz 01100100 101 index:2 rm:3 00000 sub:1 rn:5 rd:5 \
+ ra=%reg_movprfx esz=2
+FMLA_zzxz 01100100 111 index:1 rm:4 00000 sub:1 rn:5 rd:5 \
+ ra=%reg_movprfx esz=3
+
+### SVE FP Multiply Indexed Group
+
+# SVE floating-point multiply (indexed)
+FMUL_zzx 01100100 0.1 .. rm:3 001000 rn:5 rd:5 \
+ index=%index3_22_19 esz=1
+FMUL_zzx 01100100 101 index:2 rm:3 001000 rn:5 rd:5 esz=2
+FMUL_zzx 01100100 111 index:1 rm:4 001000 rn:5 rd:5 esz=3
+
+### SVE FP Fast Reduction Group
+
+FADDV 01100101 .. 000 000 001 ... ..... ..... @rd_pg_rn
+FMAXNMV 01100101 .. 000 100 001 ... ..... ..... @rd_pg_rn
+FMINNMV 01100101 .. 000 101 001 ... ..... ..... @rd_pg_rn
+FMAXV 01100101 .. 000 110 001 ... ..... ..... @rd_pg_rn
+FMINV 01100101 .. 000 111 001 ... ..... ..... @rd_pg_rn
+
+## SVE Floating Point Unary Operations - Unpredicated Group
+
+FRECPE 01100101 .. 001 110 001100 ..... ..... @rd_rn
+FRSQRTE 01100101 .. 001 111 001100 ..... ..... @rd_rn
+
+### SVE FP Compare with Zero Group
+
+FCMGE_ppz0 01100101 .. 0100 00 001 ... ..... 0 .... @pd_pg_rn
+FCMGT_ppz0 01100101 .. 0100 00 001 ... ..... 1 .... @pd_pg_rn
+FCMLT_ppz0 01100101 .. 0100 01 001 ... ..... 0 .... @pd_pg_rn
+FCMLE_ppz0 01100101 .. 0100 01 001 ... ..... 1 .... @pd_pg_rn
+FCMEQ_ppz0 01100101 .. 0100 10 001 ... ..... 0 .... @pd_pg_rn
+FCMNE_ppz0 01100101 .. 0100 11 001 ... ..... 0 .... @pd_pg_rn
+
+### SVE FP Accumulating Reduction Group
+
+# SVE floating-point serial reduction (predicated)
+FADDA 01100101 .. 011 000 001 ... ..... ..... @rdn_pg_rm
+
### SVE Floating Point Arithmetic - Unpredicated Group
# SVE floating-point arithmetic (unpredicated)
@@ -658,6 +803,108 @@
FRECPS 01100101 .. 0 ..... 000 110 ..... ..... @rd_rn_rm
FRSQRTS 01100101 .. 0 ..... 000 111 ..... ..... @rd_rn_rm
+### SVE FP Arithmetic Predicated Group
+
+# SVE floating-point arithmetic (predicated)
+FADD_zpzz 01100101 .. 00 0000 100 ... ..... ..... @rdn_pg_rm
+FSUB_zpzz 01100101 .. 00 0001 100 ... ..... ..... @rdn_pg_rm
+FMUL_zpzz 01100101 .. 00 0010 100 ... ..... ..... @rdn_pg_rm
+FSUB_zpzz 01100101 .. 00 0011 100 ... ..... ..... @rdm_pg_rn # FSUBR
+FMAXNM_zpzz 01100101 .. 00 0100 100 ... ..... ..... @rdn_pg_rm
+FMINNM_zpzz 01100101 .. 00 0101 100 ... ..... ..... @rdn_pg_rm
+FMAX_zpzz 01100101 .. 00 0110 100 ... ..... ..... @rdn_pg_rm
+FMIN_zpzz 01100101 .. 00 0111 100 ... ..... ..... @rdn_pg_rm
+FABD 01100101 .. 00 1000 100 ... ..... ..... @rdn_pg_rm
+FSCALE 01100101 .. 00 1001 100 ... ..... ..... @rdn_pg_rm
+FMULX 01100101 .. 00 1010 100 ... ..... ..... @rdn_pg_rm
+FDIV 01100101 .. 00 1100 100 ... ..... ..... @rdm_pg_rn # FDIVR
+FDIV 01100101 .. 00 1101 100 ... ..... ..... @rdn_pg_rm
+
+# SVE floating-point arithmetic with immediate (predicated)
+FADD_zpzi 01100101 .. 011 000 100 ... 0000 . ..... @rdn_i1
+FSUB_zpzi 01100101 .. 011 001 100 ... 0000 . ..... @rdn_i1
+FMUL_zpzi 01100101 .. 011 010 100 ... 0000 . ..... @rdn_i1
+FSUBR_zpzi 01100101 .. 011 011 100 ... 0000 . ..... @rdn_i1
+FMAXNM_zpzi 01100101 .. 011 100 100 ... 0000 . ..... @rdn_i1
+FMINNM_zpzi 01100101 .. 011 101 100 ... 0000 . ..... @rdn_i1
+FMAX_zpzi 01100101 .. 011 110 100 ... 0000 . ..... @rdn_i1
+FMIN_zpzi 01100101 .. 011 111 100 ... 0000 . ..... @rdn_i1
+
+# SVE floating-point trig multiply-add coefficient
+FTMAD 01100101 esz:2 010 imm:3 100000 rm:5 rd:5 rn=%reg_movprfx
+
+### SVE FP Multiply-Add Group
+
+# SVE floating-point multiply-accumulate writing addend
+FMLA_zpzzz 01100101 .. 1 ..... 000 ... ..... ..... @rda_pg_rn_rm
+FMLS_zpzzz 01100101 .. 1 ..... 001 ... ..... ..... @rda_pg_rn_rm
+FNMLA_zpzzz 01100101 .. 1 ..... 010 ... ..... ..... @rda_pg_rn_rm
+FNMLS_zpzzz 01100101 .. 1 ..... 011 ... ..... ..... @rda_pg_rn_rm
+
+# SVE floating-point multiply-accumulate writing multiplicand
+# Alter the operand extraction order and reuse the helpers from above.
+# FMAD, FMSB, FNMAD, FNMS
+FMLA_zpzzz 01100101 .. 1 ..... 100 ... ..... ..... @rdn_pg_rm_ra
+FMLS_zpzzz 01100101 .. 1 ..... 101 ... ..... ..... @rdn_pg_rm_ra
+FNMLA_zpzzz 01100101 .. 1 ..... 110 ... ..... ..... @rdn_pg_rm_ra
+FNMLS_zpzzz 01100101 .. 1 ..... 111 ... ..... ..... @rdn_pg_rm_ra
+
+### SVE FP Unary Operations Predicated Group
+
+# SVE floating-point convert precision
+FCVT_sh 01100101 10 0010 00 101 ... ..... ..... @rd_pg_rn_e0
+FCVT_hs 01100101 10 0010 01 101 ... ..... ..... @rd_pg_rn_e0
+FCVT_dh 01100101 11 0010 00 101 ... ..... ..... @rd_pg_rn_e0
+FCVT_hd 01100101 11 0010 01 101 ... ..... ..... @rd_pg_rn_e0
+FCVT_ds 01100101 11 0010 10 101 ... ..... ..... @rd_pg_rn_e0
+FCVT_sd 01100101 11 0010 11 101 ... ..... ..... @rd_pg_rn_e0
+
+# SVE floating-point convert to integer
+FCVTZS_hh 01100101 01 011 01 0 101 ... ..... ..... @rd_pg_rn_e0
+FCVTZU_hh 01100101 01 011 01 1 101 ... ..... ..... @rd_pg_rn_e0
+FCVTZS_hs 01100101 01 011 10 0 101 ... ..... ..... @rd_pg_rn_e0
+FCVTZU_hs 01100101 01 011 10 1 101 ... ..... ..... @rd_pg_rn_e0
+FCVTZS_hd 01100101 01 011 11 0 101 ... ..... ..... @rd_pg_rn_e0
+FCVTZU_hd 01100101 01 011 11 1 101 ... ..... ..... @rd_pg_rn_e0
+FCVTZS_ss 01100101 10 011 10 0 101 ... ..... ..... @rd_pg_rn_e0
+FCVTZU_ss 01100101 10 011 10 1 101 ... ..... ..... @rd_pg_rn_e0
+FCVTZS_ds 01100101 11 011 00 0 101 ... ..... ..... @rd_pg_rn_e0
+FCVTZU_ds 01100101 11 011 00 1 101 ... ..... ..... @rd_pg_rn_e0
+FCVTZS_sd 01100101 11 011 10 0 101 ... ..... ..... @rd_pg_rn_e0
+FCVTZU_sd 01100101 11 011 10 1 101 ... ..... ..... @rd_pg_rn_e0
+FCVTZS_dd 01100101 11 011 11 0 101 ... ..... ..... @rd_pg_rn_e0
+FCVTZU_dd 01100101 11 011 11 1 101 ... ..... ..... @rd_pg_rn_e0
+
+# SVE floating-point round to integral value
+FRINTN 01100101 .. 000 000 101 ... ..... ..... @rd_pg_rn
+FRINTP 01100101 .. 000 001 101 ... ..... ..... @rd_pg_rn
+FRINTM 01100101 .. 000 010 101 ... ..... ..... @rd_pg_rn
+FRINTZ 01100101 .. 000 011 101 ... ..... ..... @rd_pg_rn
+FRINTA 01100101 .. 000 100 101 ... ..... ..... @rd_pg_rn
+FRINTX 01100101 .. 000 110 101 ... ..... ..... @rd_pg_rn
+FRINTI 01100101 .. 000 111 101 ... ..... ..... @rd_pg_rn
+
+# SVE floating-point unary operations
+FRECPX 01100101 .. 001 100 101 ... ..... ..... @rd_pg_rn
+FSQRT 01100101 .. 001 101 101 ... ..... ..... @rd_pg_rn
+
+# SVE integer convert to floating-point
+SCVTF_hh 01100101 01 010 01 0 101 ... ..... ..... @rd_pg_rn_e0
+SCVTF_sh 01100101 01 010 10 0 101 ... ..... ..... @rd_pg_rn_e0
+SCVTF_dh 01100101 01 010 11 0 101 ... ..... ..... @rd_pg_rn_e0
+SCVTF_ss 01100101 10 010 10 0 101 ... ..... ..... @rd_pg_rn_e0
+SCVTF_sd 01100101 11 010 00 0 101 ... ..... ..... @rd_pg_rn_e0
+SCVTF_ds 01100101 11 010 10 0 101 ... ..... ..... @rd_pg_rn_e0
+SCVTF_dd 01100101 11 010 11 0 101 ... ..... ..... @rd_pg_rn_e0
+
+UCVTF_hh 01100101 01 010 01 1 101 ... ..... ..... @rd_pg_rn_e0
+UCVTF_sh 01100101 01 010 10 1 101 ... ..... ..... @rd_pg_rn_e0
+UCVTF_dh 01100101 01 010 11 1 101 ... ..... ..... @rd_pg_rn_e0
+UCVTF_ss 01100101 10 010 10 1 101 ... ..... ..... @rd_pg_rn_e0
+UCVTF_sd 01100101 11 010 00 1 101 ... ..... ..... @rd_pg_rn_e0
+UCVTF_ds 01100101 11 010 10 1 101 ... ..... ..... @rd_pg_rn_e0
+UCVTF_dd 01100101 11 010 11 1 101 ... ..... ..... @rd_pg_rn_e0
+
### SVE Memory - 32-bit Gather and Unsized Contiguous Group
# SVE load predicate register
@@ -665,3 +912,183 @@
# SVE load vector register
LDR_zri 10000101 10 ...... 010 ... ..... ..... @rd_rn_i9
+
+# SVE load and broadcast element
+LD1R_zpri 1000010 .. 1 imm:6 1.. pg:3 rn:5 rd:5 \
+ &rpri_load dtype=%dtype_23_13 nreg=0
+
+# SVE 32-bit gather load (scalar plus 32-bit unscaled offsets)
+# SVE 32-bit gather load (scalar plus 32-bit scaled offsets)
+LD1_zprz 1000010 00 .0 ..... 0.. ... ..... ..... \
+ @rprr_g_load_xs_u esz=2 msz=0 scale=0
+LD1_zprz 1000010 01 .. ..... 0.. ... ..... ..... \
+ @rprr_g_load_xs_u_sc esz=2 msz=1
+LD1_zprz 1000010 10 .. ..... 01. ... ..... ..... \
+ @rprr_g_load_xs_sc esz=2 msz=2 u=1
+
+# SVE 32-bit gather load (vector plus immediate)
+LD1_zpiz 1000010 .. 01 ..... 1.. ... ..... ..... \
+ @rpri_g_load esz=2
+
+### SVE Memory Contiguous Load Group
+
+# SVE contiguous load (scalar plus scalar)
+LD_zprr 1010010 .... ..... 010 ... ..... ..... @rprr_load_dt nreg=0
+
+# SVE contiguous first-fault load (scalar plus scalar)
+LDFF1_zprr 1010010 .... ..... 011 ... ..... ..... @rprr_load_dt nreg=0
+
+# SVE contiguous load (scalar plus immediate)
+LD_zpri 1010010 .... 0.... 101 ... ..... ..... @rpri_load_dt nreg=0
+
+# SVE contiguous non-fault load (scalar plus immediate)
+LDNF1_zpri 1010010 .... 1.... 101 ... ..... ..... @rpri_load_dt nreg=0
+
+# SVE contiguous non-temporal load (scalar plus scalar)
+# LDNT1B, LDNT1H, LDNT1W, LDNT1D
+# SVE load multiple structures (scalar plus scalar)
+# LD2B, LD2H, LD2W, LD2D; etc.
+LD_zprr 1010010 .. nreg:2 ..... 110 ... ..... ..... @rprr_load_msz
+
+# SVE contiguous non-temporal load (scalar plus immediate)
+# LDNT1B, LDNT1H, LDNT1W, LDNT1D
+# SVE load multiple structures (scalar plus immediate)
+# LD2B, LD2H, LD2W, LD2D; etc.
+LD_zpri 1010010 .. nreg:2 0.... 111 ... ..... ..... @rpri_load_msz
+
+# SVE load and broadcast quadword (scalar plus scalar)
+LD1RQ_zprr 1010010 .. 00 ..... 000 ... ..... ..... \
+ @rprr_load_msz nreg=0
+
+# SVE load and broadcast quadword (scalar plus immediate)
+# LD1RQB, LD1RQH, LD1RQS, LD1RQD
+LD1RQ_zpri 1010010 .. 00 0.... 001 ... ..... ..... \
+ @rpri_load_msz nreg=0
+
+# SVE 32-bit gather prefetch (scalar plus 32-bit scaled offsets)
+PRF 1000010 00 -1 ----- 0-- --- ----- 0 ----
+
+# SVE 32-bit gather prefetch (vector plus immediate)
+PRF 1000010 -- 00 ----- 111 --- ----- 0 ----
+
+# SVE contiguous prefetch (scalar plus immediate)
+PRF 1000010 11 1- ----- 0-- --- ----- 0 ----
+
+# SVE contiguous prefetch (scalar plus scalar)
+PRF_rr 1000010 -- 00 rm:5 110 --- ----- 0 ----
+
+### SVE Memory 64-bit Gather Group
+
+# SVE 64-bit gather load (scalar plus 32-bit unpacked unscaled offsets)
+# SVE 64-bit gather load (scalar plus 32-bit unpacked scaled offsets)
+LD1_zprz 1100010 00 .0 ..... 0.. ... ..... ..... \
+ @rprr_g_load_xs_u esz=3 msz=0 scale=0
+LD1_zprz 1100010 01 .. ..... 0.. ... ..... ..... \
+ @rprr_g_load_xs_u_sc esz=3 msz=1
+LD1_zprz 1100010 10 .. ..... 0.. ... ..... ..... \
+ @rprr_g_load_xs_u_sc esz=3 msz=2
+LD1_zprz 1100010 11 .. ..... 01. ... ..... ..... \
+ @rprr_g_load_xs_sc esz=3 msz=3 u=1
+
+# SVE 64-bit gather load (scalar plus 64-bit unscaled offsets)
+# SVE 64-bit gather load (scalar plus 64-bit scaled offsets)
+LD1_zprz 1100010 00 10 ..... 1.. ... ..... ..... \
+ @rprr_g_load_u esz=3 msz=0 scale=0
+LD1_zprz 1100010 01 1. ..... 1.. ... ..... ..... \
+ @rprr_g_load_u_sc esz=3 msz=1
+LD1_zprz 1100010 10 1. ..... 1.. ... ..... ..... \
+ @rprr_g_load_u_sc esz=3 msz=2
+LD1_zprz 1100010 11 1. ..... 11. ... ..... ..... \
+ @rprr_g_load_sc esz=3 msz=3 u=1
+
+# SVE 64-bit gather load (vector plus immediate)
+LD1_zpiz 1100010 .. 01 ..... 1.. ... ..... ..... \
+ @rpri_g_load esz=3
+
+# SVE 64-bit gather prefetch (scalar plus 64-bit scaled offsets)
+PRF 1100010 00 11 ----- 1-- --- ----- 0 ----
+
+# SVE 64-bit gather prefetch (scalar plus unpacked 32-bit scaled offsets)
+PRF 1100010 00 -1 ----- 0-- --- ----- 0 ----
+
+# SVE 64-bit gather prefetch (vector plus immediate)
+PRF 1100010 -- 00 ----- 111 --- ----- 0 ----
+
+### SVE Memory Store Group
+
+# SVE store predicate register
+STR_pri 1110010 11 0. ..... 000 ... ..... 0 .... @pd_rn_i9
+
+# SVE store vector register
+STR_zri 1110010 11 0. ..... 010 ... ..... ..... @rd_rn_i9
+
+# SVE contiguous store (scalar plus immediate)
+# ST1B, ST1H, ST1W, ST1D; require msz <= esz
+ST_zpri 1110010 .. esz:2 0.... 111 ... ..... ..... \
+ @rpri_store_msz nreg=0
+
+# SVE contiguous store (scalar plus scalar)
+# ST1B, ST1H, ST1W, ST1D; require msz <= esz
+# Enumerate msz lest we conflict with STR_zri.
+ST_zprr 1110010 00 .. ..... 010 ... ..... ..... \
+ @rprr_store_esz_n0 msz=0
+ST_zprr 1110010 01 .. ..... 010 ... ..... ..... \
+ @rprr_store_esz_n0 msz=1
+ST_zprr 1110010 10 .. ..... 010 ... ..... ..... \
+ @rprr_store_esz_n0 msz=2
+ST_zprr 1110010 11 11 ..... 010 ... ..... ..... \
+ @rprr_store msz=3 esz=3 nreg=0
+
+# SVE contiguous non-temporal store (scalar plus immediate) (nreg == 0)
+# SVE store multiple structures (scalar plus immediate) (nreg != 0)
+ST_zpri 1110010 .. nreg:2 1.... 111 ... ..... ..... \
+ @rpri_store_msz esz=%size_23
+
+# SVE contiguous non-temporal store (scalar plus scalar) (nreg == 0)
+# SVE store multiple structures (scalar plus scalar) (nreg != 0)
+ST_zprr 1110010 msz:2 nreg:2 ..... 011 ... ..... ..... \
+ @rprr_store esz=%size_23
+
+# SVE 32-bit scatter store (scalar plus 32-bit scaled offsets)
+# Require msz > 0 && msz <= esz.
+ST1_zprz 1110010 .. 11 ..... 100 ... ..... ..... \
+ @rprr_scatter_store xs=0 esz=2 scale=1
+ST1_zprz 1110010 .. 11 ..... 110 ... ..... ..... \
+ @rprr_scatter_store xs=1 esz=2 scale=1
+
+# SVE 32-bit scatter store (scalar plus 32-bit unscaled offsets)
+# Require msz <= esz.
+ST1_zprz 1110010 .. 10 ..... 100 ... ..... ..... \
+ @rprr_scatter_store xs=0 esz=2 scale=0
+ST1_zprz 1110010 .. 10 ..... 110 ... ..... ..... \
+ @rprr_scatter_store xs=1 esz=2 scale=0
+
+# SVE 64-bit scatter store (scalar plus 64-bit scaled offset)
+# Require msz > 0
+ST1_zprz 1110010 .. 01 ..... 101 ... ..... ..... \
+ @rprr_scatter_store xs=2 esz=3 scale=1
+
+# SVE 64-bit scatter store (scalar plus 64-bit unscaled offset)
+ST1_zprz 1110010 .. 00 ..... 101 ... ..... ..... \
+ @rprr_scatter_store xs=2 esz=3 scale=0
+
+# SVE 64-bit scatter store (vector plus immediate)
+ST1_zpiz 1110010 .. 10 ..... 101 ... ..... ..... \
+ @rpri_scatter_store esz=3
+
+# SVE 32-bit scatter store (vector plus immediate)
+ST1_zpiz 1110010 .. 11 ..... 101 ... ..... ..... \
+ @rpri_scatter_store esz=2
+
+# SVE 64-bit scatter store (scalar plus unpacked 32-bit scaled offset)
+# Require msz > 0
+ST1_zprz 1110010 .. 01 ..... 100 ... ..... ..... \
+ @rprr_scatter_store xs=0 esz=3 scale=1
+ST1_zprz 1110010 .. 01 ..... 110 ... ..... ..... \
+ @rprr_scatter_store xs=1 esz=3 scale=1
+
+# SVE 64-bit scatter store (scalar plus unpacked 32-bit unscaled offset)
+ST1_zprz 1110010 .. 00 ..... 100 ... ..... ..... \
+ @rprr_scatter_store xs=0 esz=3 scale=0
+ST1_zprz 1110010 .. 00 ..... 110 ... ..... ..... \
+ @rprr_scatter_store xs=1 esz=3 scale=0
diff --git a/target/arm/sve_helper.c b/target/arm/sve_helper.c
index 128bbf9..a03ca77 100644
--- a/target/arm/sve_helper.c
+++ b/target/arm/sve_helper.c
@@ -369,7 +369,17 @@
#define DO_MIN(N, M) ((N) >= (M) ? (M) : (N))
#define DO_ABD(N, M) ((N) >= (M) ? (N) - (M) : (M) - (N))
#define DO_MUL(N, M) (N * M)
-#define DO_DIV(N, M) (M ? N / M : 0)
+
+
+/*
+ * We must avoid the C undefined behaviour cases: division by
+ * zero and signed division of INT_MIN by -1. Both of these
+ * have architecturally defined required results for Arm.
+ * We special case all signed divisions by -1 to avoid having
+ * to deduce the minimum integer for the type involved.
+ */
+#define DO_SDIV(N, M) (unlikely(M == 0) ? 0 : unlikely(M == -1) ? -N : N / M)
+#define DO_UDIV(N, M) (unlikely(M == 0) ? 0 : N / M)
DO_ZPZZ(sve_and_zpzz_b, uint8_t, H1, DO_AND)
DO_ZPZZ(sve_and_zpzz_h, uint16_t, H1_2, DO_AND)
@@ -477,11 +487,11 @@
DO_ZPZZ(sve_umulh_zpzz_s, uint32_t, H1_4, do_mulh_s)
DO_ZPZZ_D(sve_umulh_zpzz_d, uint64_t, do_umulh_d)
-DO_ZPZZ(sve_sdiv_zpzz_s, int32_t, H1_4, DO_DIV)
-DO_ZPZZ_D(sve_sdiv_zpzz_d, int64_t, DO_DIV)
+DO_ZPZZ(sve_sdiv_zpzz_s, int32_t, H1_4, DO_SDIV)
+DO_ZPZZ_D(sve_sdiv_zpzz_d, int64_t, DO_SDIV)
-DO_ZPZZ(sve_udiv_zpzz_s, uint32_t, H1_4, DO_DIV)
-DO_ZPZZ_D(sve_udiv_zpzz_d, uint64_t, DO_DIV)
+DO_ZPZZ(sve_udiv_zpzz_s, uint32_t, H1_4, DO_UDIV)
+DO_ZPZZ_D(sve_udiv_zpzz_d, uint64_t, DO_UDIV)
/* Note that all bits of the shift are significant
and not modulo the element size. */
@@ -995,6 +1005,47 @@
}
}
+/* Copy Zn into Zd, and store zero into inactive elements. */
+void HELPER(sve_movz_b)(void *vd, void *vn, void *vg, uint32_t desc)
+{
+ intptr_t i, opr_sz = simd_oprsz(desc) / 8;
+ uint64_t *d = vd, *n = vn;
+ uint8_t *pg = vg;
+ for (i = 0; i < opr_sz; i += 1) {
+ d[i] = n[i] & expand_pred_b(pg[H1(i)]);
+ }
+}
+
+void HELPER(sve_movz_h)(void *vd, void *vn, void *vg, uint32_t desc)
+{
+ intptr_t i, opr_sz = simd_oprsz(desc) / 8;
+ uint64_t *d = vd, *n = vn;
+ uint8_t *pg = vg;
+ for (i = 0; i < opr_sz; i += 1) {
+ d[i] = n[i] & expand_pred_h(pg[H1(i)]);
+ }
+}
+
+void HELPER(sve_movz_s)(void *vd, void *vn, void *vg, uint32_t desc)
+{
+ intptr_t i, opr_sz = simd_oprsz(desc) / 8;
+ uint64_t *d = vd, *n = vn;
+ uint8_t *pg = vg;
+ for (i = 0; i < opr_sz; i += 1) {
+ d[i] = n[i] & expand_pred_s(pg[H1(i)]);
+ }
+}
+
+void HELPER(sve_movz_d)(void *vd, void *vn, void *vg, uint32_t desc)
+{
+ intptr_t i, opr_sz = simd_oprsz(desc) / 8;
+ uint64_t *d = vd, *n = vn;
+ uint8_t *pg = vg;
+ for (i = 0; i < opr_sz; i += 1) {
+ d[i] = n[1] & -(uint64_t)(pg[H1(i)] & 1);
+ }
+}
+
/* Three-operand expander, immediate operand, controlled by a predicate.
*/
#define DO_ZPZI(NAME, TYPE, H, OP) \
@@ -2810,3 +2861,1817 @@
return predtest_ones(d, oprsz, esz_mask);
}
+
+/* Recursive reduction on a function;
+ * C.f. the ARM ARM function ReducePredicated.
+ *
+ * While it would be possible to write this without the DATA temporary,
+ * it is much simpler to process the predicate register this way.
+ * The recursion is bounded to depth 7 (128 fp16 elements), so there's
+ * little to gain with a more complex non-recursive form.
+ */
+#define DO_REDUCE(NAME, TYPE, H, FUNC, IDENT) \
+static TYPE NAME##_reduce(TYPE *data, float_status *status, uintptr_t n) \
+{ \
+ if (n == 1) { \
+ return *data; \
+ } else { \
+ uintptr_t half = n / 2; \
+ TYPE lo = NAME##_reduce(data, status, half); \
+ TYPE hi = NAME##_reduce(data + half, status, half); \
+ return TYPE##_##FUNC(lo, hi, status); \
+ } \
+} \
+uint64_t HELPER(NAME)(void *vn, void *vg, void *vs, uint32_t desc) \
+{ \
+ uintptr_t i, oprsz = simd_oprsz(desc), maxsz = simd_maxsz(desc); \
+ TYPE data[sizeof(ARMVectorReg) / sizeof(TYPE)]; \
+ for (i = 0; i < oprsz; ) { \
+ uint16_t pg = *(uint16_t *)(vg + H1_2(i >> 3)); \
+ do { \
+ TYPE nn = *(TYPE *)(vn + H(i)); \
+ *(TYPE *)((void *)data + i) = (pg & 1 ? nn : IDENT); \
+ i += sizeof(TYPE), pg >>= sizeof(TYPE); \
+ } while (i & 15); \
+ } \
+ for (; i < maxsz; i += sizeof(TYPE)) { \
+ *(TYPE *)((void *)data + i) = IDENT; \
+ } \
+ return NAME##_reduce(data, vs, maxsz / sizeof(TYPE)); \
+}
+
+DO_REDUCE(sve_faddv_h, float16, H1_2, add, float16_zero)
+DO_REDUCE(sve_faddv_s, float32, H1_4, add, float32_zero)
+DO_REDUCE(sve_faddv_d, float64, , add, float64_zero)
+
+/* Identity is floatN_default_nan, without the function call. */
+DO_REDUCE(sve_fminnmv_h, float16, H1_2, minnum, 0x7E00)
+DO_REDUCE(sve_fminnmv_s, float32, H1_4, minnum, 0x7FC00000)
+DO_REDUCE(sve_fminnmv_d, float64, , minnum, 0x7FF8000000000000ULL)
+
+DO_REDUCE(sve_fmaxnmv_h, float16, H1_2, maxnum, 0x7E00)
+DO_REDUCE(sve_fmaxnmv_s, float32, H1_4, maxnum, 0x7FC00000)
+DO_REDUCE(sve_fmaxnmv_d, float64, , maxnum, 0x7FF8000000000000ULL)
+
+DO_REDUCE(sve_fminv_h, float16, H1_2, min, float16_infinity)
+DO_REDUCE(sve_fminv_s, float32, H1_4, min, float32_infinity)
+DO_REDUCE(sve_fminv_d, float64, , min, float64_infinity)
+
+DO_REDUCE(sve_fmaxv_h, float16, H1_2, max, float16_chs(float16_infinity))
+DO_REDUCE(sve_fmaxv_s, float32, H1_4, max, float32_chs(float32_infinity))
+DO_REDUCE(sve_fmaxv_d, float64, , max, float64_chs(float64_infinity))
+
+#undef DO_REDUCE
+
+uint64_t HELPER(sve_fadda_h)(uint64_t nn, void *vm, void *vg,
+ void *status, uint32_t desc)
+{
+ intptr_t i = 0, opr_sz = simd_oprsz(desc);
+ float16 result = nn;
+
+ do {
+ uint16_t pg = *(uint16_t *)(vg + H1_2(i >> 3));
+ do {
+ if (pg & 1) {
+ float16 mm = *(float16 *)(vm + H1_2(i));
+ result = float16_add(result, mm, status);
+ }
+ i += sizeof(float16), pg >>= sizeof(float16);
+ } while (i & 15);
+ } while (i < opr_sz);
+
+ return result;
+}
+
+uint64_t HELPER(sve_fadda_s)(uint64_t nn, void *vm, void *vg,
+ void *status, uint32_t desc)
+{
+ intptr_t i = 0, opr_sz = simd_oprsz(desc);
+ float32 result = nn;
+
+ do {
+ uint16_t pg = *(uint16_t *)(vg + H1_2(i >> 3));
+ do {
+ if (pg & 1) {
+ float32 mm = *(float32 *)(vm + H1_2(i));
+ result = float32_add(result, mm, status);
+ }
+ i += sizeof(float32), pg >>= sizeof(float32);
+ } while (i & 15);
+ } while (i < opr_sz);
+
+ return result;
+}
+
+uint64_t HELPER(sve_fadda_d)(uint64_t nn, void *vm, void *vg,
+ void *status, uint32_t desc)
+{
+ intptr_t i = 0, opr_sz = simd_oprsz(desc) / 8;
+ uint64_t *m = vm;
+ uint8_t *pg = vg;
+
+ for (i = 0; i < opr_sz; i++) {
+ if (pg[H1(i)] & 1) {
+ nn = float64_add(nn, m[i], status);
+ }
+ }
+
+ return nn;
+}
+
+/* Fully general three-operand expander, controlled by a predicate,
+ * With the extra float_status parameter.
+ */
+#define DO_ZPZZ_FP(NAME, TYPE, H, OP) \
+void HELPER(NAME)(void *vd, void *vn, void *vm, void *vg, \
+ void *status, uint32_t desc) \
+{ \
+ intptr_t i = simd_oprsz(desc); \
+ uint64_t *g = vg; \
+ do { \
+ uint64_t pg = g[(i - 1) >> 6]; \
+ do { \
+ i -= sizeof(TYPE); \
+ if (likely((pg >> (i & 63)) & 1)) { \
+ TYPE nn = *(TYPE *)(vn + H(i)); \
+ TYPE mm = *(TYPE *)(vm + H(i)); \
+ *(TYPE *)(vd + H(i)) = OP(nn, mm, status); \
+ } \
+ } while (i & 63); \
+ } while (i != 0); \
+}
+
+DO_ZPZZ_FP(sve_fadd_h, uint16_t, H1_2, float16_add)
+DO_ZPZZ_FP(sve_fadd_s, uint32_t, H1_4, float32_add)
+DO_ZPZZ_FP(sve_fadd_d, uint64_t, , float64_add)
+
+DO_ZPZZ_FP(sve_fsub_h, uint16_t, H1_2, float16_sub)
+DO_ZPZZ_FP(sve_fsub_s, uint32_t, H1_4, float32_sub)
+DO_ZPZZ_FP(sve_fsub_d, uint64_t, , float64_sub)
+
+DO_ZPZZ_FP(sve_fmul_h, uint16_t, H1_2, float16_mul)
+DO_ZPZZ_FP(sve_fmul_s, uint32_t, H1_4, float32_mul)
+DO_ZPZZ_FP(sve_fmul_d, uint64_t, , float64_mul)
+
+DO_ZPZZ_FP(sve_fdiv_h, uint16_t, H1_2, float16_div)
+DO_ZPZZ_FP(sve_fdiv_s, uint32_t, H1_4, float32_div)
+DO_ZPZZ_FP(sve_fdiv_d, uint64_t, , float64_div)
+
+DO_ZPZZ_FP(sve_fmin_h, uint16_t, H1_2, float16_min)
+DO_ZPZZ_FP(sve_fmin_s, uint32_t, H1_4, float32_min)
+DO_ZPZZ_FP(sve_fmin_d, uint64_t, , float64_min)
+
+DO_ZPZZ_FP(sve_fmax_h, uint16_t, H1_2, float16_max)
+DO_ZPZZ_FP(sve_fmax_s, uint32_t, H1_4, float32_max)
+DO_ZPZZ_FP(sve_fmax_d, uint64_t, , float64_max)
+
+DO_ZPZZ_FP(sve_fminnum_h, uint16_t, H1_2, float16_minnum)
+DO_ZPZZ_FP(sve_fminnum_s, uint32_t, H1_4, float32_minnum)
+DO_ZPZZ_FP(sve_fminnum_d, uint64_t, , float64_minnum)
+
+DO_ZPZZ_FP(sve_fmaxnum_h, uint16_t, H1_2, float16_maxnum)
+DO_ZPZZ_FP(sve_fmaxnum_s, uint32_t, H1_4, float32_maxnum)
+DO_ZPZZ_FP(sve_fmaxnum_d, uint64_t, , float64_maxnum)
+
+static inline float16 abd_h(float16 a, float16 b, float_status *s)
+{
+ return float16_abs(float16_sub(a, b, s));
+}
+
+static inline float32 abd_s(float32 a, float32 b, float_status *s)
+{
+ return float32_abs(float32_sub(a, b, s));
+}
+
+static inline float64 abd_d(float64 a, float64 b, float_status *s)
+{
+ return float64_abs(float64_sub(a, b, s));
+}
+
+DO_ZPZZ_FP(sve_fabd_h, uint16_t, H1_2, abd_h)
+DO_ZPZZ_FP(sve_fabd_s, uint32_t, H1_4, abd_s)
+DO_ZPZZ_FP(sve_fabd_d, uint64_t, , abd_d)
+
+static inline float64 scalbn_d(float64 a, int64_t b, float_status *s)
+{
+ int b_int = MIN(MAX(b, INT_MIN), INT_MAX);
+ return float64_scalbn(a, b_int, s);
+}
+
+DO_ZPZZ_FP(sve_fscalbn_h, int16_t, H1_2, float16_scalbn)
+DO_ZPZZ_FP(sve_fscalbn_s, int32_t, H1_4, float32_scalbn)
+DO_ZPZZ_FP(sve_fscalbn_d, int64_t, , scalbn_d)
+
+DO_ZPZZ_FP(sve_fmulx_h, uint16_t, H1_2, helper_advsimd_mulxh)
+DO_ZPZZ_FP(sve_fmulx_s, uint32_t, H1_4, helper_vfp_mulxs)
+DO_ZPZZ_FP(sve_fmulx_d, uint64_t, , helper_vfp_mulxd)
+
+#undef DO_ZPZZ_FP
+
+/* Three-operand expander, with one scalar operand, controlled by
+ * a predicate, with the extra float_status parameter.
+ */
+#define DO_ZPZS_FP(NAME, TYPE, H, OP) \
+void HELPER(NAME)(void *vd, void *vn, void *vg, uint64_t scalar, \
+ void *status, uint32_t desc) \
+{ \
+ intptr_t i = simd_oprsz(desc); \
+ uint64_t *g = vg; \
+ TYPE mm = scalar; \
+ do { \
+ uint64_t pg = g[(i - 1) >> 6]; \
+ do { \
+ i -= sizeof(TYPE); \
+ if (likely((pg >> (i & 63)) & 1)) { \
+ TYPE nn = *(TYPE *)(vn + H(i)); \
+ *(TYPE *)(vd + H(i)) = OP(nn, mm, status); \
+ } \
+ } while (i & 63); \
+ } while (i != 0); \
+}
+
+DO_ZPZS_FP(sve_fadds_h, float16, H1_2, float16_add)
+DO_ZPZS_FP(sve_fadds_s, float32, H1_4, float32_add)
+DO_ZPZS_FP(sve_fadds_d, float64, , float64_add)
+
+DO_ZPZS_FP(sve_fsubs_h, float16, H1_2, float16_sub)
+DO_ZPZS_FP(sve_fsubs_s, float32, H1_4, float32_sub)
+DO_ZPZS_FP(sve_fsubs_d, float64, , float64_sub)
+
+DO_ZPZS_FP(sve_fmuls_h, float16, H1_2, float16_mul)
+DO_ZPZS_FP(sve_fmuls_s, float32, H1_4, float32_mul)
+DO_ZPZS_FP(sve_fmuls_d, float64, , float64_mul)
+
+static inline float16 subr_h(float16 a, float16 b, float_status *s)
+{
+ return float16_sub(b, a, s);
+}
+
+static inline float32 subr_s(float32 a, float32 b, float_status *s)
+{
+ return float32_sub(b, a, s);
+}
+
+static inline float64 subr_d(float64 a, float64 b, float_status *s)
+{
+ return float64_sub(b, a, s);
+}
+
+DO_ZPZS_FP(sve_fsubrs_h, float16, H1_2, subr_h)
+DO_ZPZS_FP(sve_fsubrs_s, float32, H1_4, subr_s)
+DO_ZPZS_FP(sve_fsubrs_d, float64, , subr_d)
+
+DO_ZPZS_FP(sve_fmaxnms_h, float16, H1_2, float16_maxnum)
+DO_ZPZS_FP(sve_fmaxnms_s, float32, H1_4, float32_maxnum)
+DO_ZPZS_FP(sve_fmaxnms_d, float64, , float64_maxnum)
+
+DO_ZPZS_FP(sve_fminnms_h, float16, H1_2, float16_minnum)
+DO_ZPZS_FP(sve_fminnms_s, float32, H1_4, float32_minnum)
+DO_ZPZS_FP(sve_fminnms_d, float64, , float64_minnum)
+
+DO_ZPZS_FP(sve_fmaxs_h, float16, H1_2, float16_max)
+DO_ZPZS_FP(sve_fmaxs_s, float32, H1_4, float32_max)
+DO_ZPZS_FP(sve_fmaxs_d, float64, , float64_max)
+
+DO_ZPZS_FP(sve_fmins_h, float16, H1_2, float16_min)
+DO_ZPZS_FP(sve_fmins_s, float32, H1_4, float32_min)
+DO_ZPZS_FP(sve_fmins_d, float64, , float64_min)
+
+/* Fully general two-operand expander, controlled by a predicate,
+ * With the extra float_status parameter.
+ */
+#define DO_ZPZ_FP(NAME, TYPE, H, OP) \
+void HELPER(NAME)(void *vd, void *vn, void *vg, void *status, uint32_t desc) \
+{ \
+ intptr_t i = simd_oprsz(desc); \
+ uint64_t *g = vg; \
+ do { \
+ uint64_t pg = g[(i - 1) >> 6]; \
+ do { \
+ i -= sizeof(TYPE); \
+ if (likely((pg >> (i & 63)) & 1)) { \
+ TYPE nn = *(TYPE *)(vn + H(i)); \
+ *(TYPE *)(vd + H(i)) = OP(nn, status); \
+ } \
+ } while (i & 63); \
+ } while (i != 0); \
+}
+
+/* SVE fp16 conversions always use IEEE mode. Like AdvSIMD, they ignore
+ * FZ16. When converting from fp16, this affects flushing input denormals;
+ * when converting to fp16, this affects flushing output denormals.
+ */
+static inline float32 sve_f16_to_f32(float16 f, float_status *fpst)
+{
+ flag save = get_flush_inputs_to_zero(fpst);
+ float32 ret;
+
+ set_flush_inputs_to_zero(false, fpst);
+ ret = float16_to_float32(f, true, fpst);
+ set_flush_inputs_to_zero(save, fpst);
+ return ret;
+}
+
+static inline float64 sve_f16_to_f64(float16 f, float_status *fpst)
+{
+ flag save = get_flush_inputs_to_zero(fpst);
+ float64 ret;
+
+ set_flush_inputs_to_zero(false, fpst);
+ ret = float16_to_float64(f, true, fpst);
+ set_flush_inputs_to_zero(save, fpst);
+ return ret;
+}
+
+static inline float16 sve_f32_to_f16(float32 f, float_status *fpst)
+{
+ flag save = get_flush_to_zero(fpst);
+ float16 ret;
+
+ set_flush_to_zero(false, fpst);
+ ret = float32_to_float16(f, true, fpst);
+ set_flush_to_zero(save, fpst);
+ return ret;
+}
+
+static inline float16 sve_f64_to_f16(float64 f, float_status *fpst)
+{
+ flag save = get_flush_to_zero(fpst);
+ float16 ret;
+
+ set_flush_to_zero(false, fpst);
+ ret = float64_to_float16(f, true, fpst);
+ set_flush_to_zero(save, fpst);
+ return ret;
+}
+
+static inline int16_t vfp_float16_to_int16_rtz(float16 f, float_status *s)
+{
+ if (float16_is_any_nan(f)) {
+ float_raise(float_flag_invalid, s);
+ return 0;
+ }
+ return float16_to_int16_round_to_zero(f, s);
+}
+
+static inline int64_t vfp_float16_to_int64_rtz(float16 f, float_status *s)
+{
+ if (float16_is_any_nan(f)) {
+ float_raise(float_flag_invalid, s);
+ return 0;
+ }
+ return float16_to_int64_round_to_zero(f, s);
+}
+
+static inline int64_t vfp_float32_to_int64_rtz(float32 f, float_status *s)
+{
+ if (float32_is_any_nan(f)) {
+ float_raise(float_flag_invalid, s);
+ return 0;
+ }
+ return float32_to_int64_round_to_zero(f, s);
+}
+
+static inline int64_t vfp_float64_to_int64_rtz(float64 f, float_status *s)
+{
+ if (float64_is_any_nan(f)) {
+ float_raise(float_flag_invalid, s);
+ return 0;
+ }
+ return float64_to_int64_round_to_zero(f, s);
+}
+
+static inline uint16_t vfp_float16_to_uint16_rtz(float16 f, float_status *s)
+{
+ if (float16_is_any_nan(f)) {
+ float_raise(float_flag_invalid, s);
+ return 0;
+ }
+ return float16_to_uint16_round_to_zero(f, s);
+}
+
+static inline uint64_t vfp_float16_to_uint64_rtz(float16 f, float_status *s)
+{
+ if (float16_is_any_nan(f)) {
+ float_raise(float_flag_invalid, s);
+ return 0;
+ }
+ return float16_to_uint64_round_to_zero(f, s);
+}
+
+static inline uint64_t vfp_float32_to_uint64_rtz(float32 f, float_status *s)
+{
+ if (float32_is_any_nan(f)) {
+ float_raise(float_flag_invalid, s);
+ return 0;
+ }
+ return float32_to_uint64_round_to_zero(f, s);
+}
+
+static inline uint64_t vfp_float64_to_uint64_rtz(float64 f, float_status *s)
+{
+ if (float64_is_any_nan(f)) {
+ float_raise(float_flag_invalid, s);
+ return 0;
+ }
+ return float64_to_uint64_round_to_zero(f, s);
+}
+
+DO_ZPZ_FP(sve_fcvt_sh, uint32_t, H1_4, sve_f32_to_f16)
+DO_ZPZ_FP(sve_fcvt_hs, uint32_t, H1_4, sve_f16_to_f32)
+DO_ZPZ_FP(sve_fcvt_dh, uint64_t, , sve_f64_to_f16)
+DO_ZPZ_FP(sve_fcvt_hd, uint64_t, , sve_f16_to_f64)
+DO_ZPZ_FP(sve_fcvt_ds, uint64_t, , float64_to_float32)
+DO_ZPZ_FP(sve_fcvt_sd, uint64_t, , float32_to_float64)
+
+DO_ZPZ_FP(sve_fcvtzs_hh, uint16_t, H1_2, vfp_float16_to_int16_rtz)
+DO_ZPZ_FP(sve_fcvtzs_hs, uint32_t, H1_4, helper_vfp_tosizh)
+DO_ZPZ_FP(sve_fcvtzs_ss, uint32_t, H1_4, helper_vfp_tosizs)
+DO_ZPZ_FP(sve_fcvtzs_hd, uint64_t, , vfp_float16_to_int64_rtz)
+DO_ZPZ_FP(sve_fcvtzs_sd, uint64_t, , vfp_float32_to_int64_rtz)
+DO_ZPZ_FP(sve_fcvtzs_ds, uint64_t, , helper_vfp_tosizd)
+DO_ZPZ_FP(sve_fcvtzs_dd, uint64_t, , vfp_float64_to_int64_rtz)
+
+DO_ZPZ_FP(sve_fcvtzu_hh, uint16_t, H1_2, vfp_float16_to_uint16_rtz)
+DO_ZPZ_FP(sve_fcvtzu_hs, uint32_t, H1_4, helper_vfp_touizh)
+DO_ZPZ_FP(sve_fcvtzu_ss, uint32_t, H1_4, helper_vfp_touizs)
+DO_ZPZ_FP(sve_fcvtzu_hd, uint64_t, , vfp_float16_to_uint64_rtz)
+DO_ZPZ_FP(sve_fcvtzu_sd, uint64_t, , vfp_float32_to_uint64_rtz)
+DO_ZPZ_FP(sve_fcvtzu_ds, uint64_t, , helper_vfp_touizd)
+DO_ZPZ_FP(sve_fcvtzu_dd, uint64_t, , vfp_float64_to_uint64_rtz)
+
+DO_ZPZ_FP(sve_frint_h, uint16_t, H1_2, helper_advsimd_rinth)
+DO_ZPZ_FP(sve_frint_s, uint32_t, H1_4, helper_rints)
+DO_ZPZ_FP(sve_frint_d, uint64_t, , helper_rintd)
+
+DO_ZPZ_FP(sve_frintx_h, uint16_t, H1_2, float16_round_to_int)
+DO_ZPZ_FP(sve_frintx_s, uint32_t, H1_4, float32_round_to_int)
+DO_ZPZ_FP(sve_frintx_d, uint64_t, , float64_round_to_int)
+
+DO_ZPZ_FP(sve_frecpx_h, uint16_t, H1_2, helper_frecpx_f16)
+DO_ZPZ_FP(sve_frecpx_s, uint32_t, H1_4, helper_frecpx_f32)
+DO_ZPZ_FP(sve_frecpx_d, uint64_t, , helper_frecpx_f64)
+
+DO_ZPZ_FP(sve_fsqrt_h, uint16_t, H1_2, float16_sqrt)
+DO_ZPZ_FP(sve_fsqrt_s, uint32_t, H1_4, float32_sqrt)
+DO_ZPZ_FP(sve_fsqrt_d, uint64_t, , float64_sqrt)
+
+DO_ZPZ_FP(sve_scvt_hh, uint16_t, H1_2, int16_to_float16)
+DO_ZPZ_FP(sve_scvt_sh, uint32_t, H1_4, int32_to_float16)
+DO_ZPZ_FP(sve_scvt_ss, uint32_t, H1_4, int32_to_float32)
+DO_ZPZ_FP(sve_scvt_sd, uint64_t, , int32_to_float64)
+DO_ZPZ_FP(sve_scvt_dh, uint64_t, , int64_to_float16)
+DO_ZPZ_FP(sve_scvt_ds, uint64_t, , int64_to_float32)
+DO_ZPZ_FP(sve_scvt_dd, uint64_t, , int64_to_float64)
+
+DO_ZPZ_FP(sve_ucvt_hh, uint16_t, H1_2, uint16_to_float16)
+DO_ZPZ_FP(sve_ucvt_sh, uint32_t, H1_4, uint32_to_float16)
+DO_ZPZ_FP(sve_ucvt_ss, uint32_t, H1_4, uint32_to_float32)
+DO_ZPZ_FP(sve_ucvt_sd, uint64_t, , uint32_to_float64)
+DO_ZPZ_FP(sve_ucvt_dh, uint64_t, , uint64_to_float16)
+DO_ZPZ_FP(sve_ucvt_ds, uint64_t, , uint64_to_float32)
+DO_ZPZ_FP(sve_ucvt_dd, uint64_t, , uint64_to_float64)
+
+#undef DO_ZPZ_FP
+
+/* 4-operand predicated multiply-add. This requires 7 operands to pass
+ * "properly", so we need to encode some of the registers into DESC.
+ */
+QEMU_BUILD_BUG_ON(SIMD_DATA_SHIFT + 20 > 32);
+
+static void do_fmla_zpzzz_h(CPUARMState *env, void *vg, uint32_t desc,
+ uint16_t neg1, uint16_t neg3)
+{
+ intptr_t i = simd_oprsz(desc);
+ unsigned rd = extract32(desc, SIMD_DATA_SHIFT, 5);
+ unsigned rn = extract32(desc, SIMD_DATA_SHIFT + 5, 5);
+ unsigned rm = extract32(desc, SIMD_DATA_SHIFT + 10, 5);
+ unsigned ra = extract32(desc, SIMD_DATA_SHIFT + 15, 5);
+ void *vd = &env->vfp.zregs[rd];
+ void *vn = &env->vfp.zregs[rn];
+ void *vm = &env->vfp.zregs[rm];
+ void *va = &env->vfp.zregs[ra];
+ uint64_t *g = vg;
+
+ do {
+ uint64_t pg = g[(i - 1) >> 6];
+ do {
+ i -= 2;
+ if (likely((pg >> (i & 63)) & 1)) {
+ float16 e1, e2, e3, r;
+
+ e1 = *(uint16_t *)(vn + H1_2(i)) ^ neg1;
+ e2 = *(uint16_t *)(vm + H1_2(i));
+ e3 = *(uint16_t *)(va + H1_2(i)) ^ neg3;
+ r = float16_muladd(e1, e2, e3, 0, &env->vfp.fp_status);
+ *(uint16_t *)(vd + H1_2(i)) = r;
+ }
+ } while (i & 63);
+ } while (i != 0);
+}
+
+void HELPER(sve_fmla_zpzzz_h)(CPUARMState *env, void *vg, uint32_t desc)
+{
+ do_fmla_zpzzz_h(env, vg, desc, 0, 0);
+}
+
+void HELPER(sve_fmls_zpzzz_h)(CPUARMState *env, void *vg, uint32_t desc)
+{
+ do_fmla_zpzzz_h(env, vg, desc, 0x8000, 0);
+}
+
+void HELPER(sve_fnmla_zpzzz_h)(CPUARMState *env, void *vg, uint32_t desc)
+{
+ do_fmla_zpzzz_h(env, vg, desc, 0x8000, 0x8000);
+}
+
+void HELPER(sve_fnmls_zpzzz_h)(CPUARMState *env, void *vg, uint32_t desc)
+{
+ do_fmla_zpzzz_h(env, vg, desc, 0, 0x8000);
+}
+
+static void do_fmla_zpzzz_s(CPUARMState *env, void *vg, uint32_t desc,
+ uint32_t neg1, uint32_t neg3)
+{
+ intptr_t i = simd_oprsz(desc);
+ unsigned rd = extract32(desc, SIMD_DATA_SHIFT, 5);
+ unsigned rn = extract32(desc, SIMD_DATA_SHIFT + 5, 5);
+ unsigned rm = extract32(desc, SIMD_DATA_SHIFT + 10, 5);
+ unsigned ra = extract32(desc, SIMD_DATA_SHIFT + 15, 5);
+ void *vd = &env->vfp.zregs[rd];
+ void *vn = &env->vfp.zregs[rn];
+ void *vm = &env->vfp.zregs[rm];
+ void *va = &env->vfp.zregs[ra];
+ uint64_t *g = vg;
+
+ do {
+ uint64_t pg = g[(i - 1) >> 6];
+ do {
+ i -= 4;
+ if (likely((pg >> (i & 63)) & 1)) {
+ float32 e1, e2, e3, r;
+
+ e1 = *(uint32_t *)(vn + H1_4(i)) ^ neg1;
+ e2 = *(uint32_t *)(vm + H1_4(i));
+ e3 = *(uint32_t *)(va + H1_4(i)) ^ neg3;
+ r = float32_muladd(e1, e2, e3, 0, &env->vfp.fp_status);
+ *(uint32_t *)(vd + H1_4(i)) = r;
+ }
+ } while (i & 63);
+ } while (i != 0);
+}
+
+void HELPER(sve_fmla_zpzzz_s)(CPUARMState *env, void *vg, uint32_t desc)
+{
+ do_fmla_zpzzz_s(env, vg, desc, 0, 0);
+}
+
+void HELPER(sve_fmls_zpzzz_s)(CPUARMState *env, void *vg, uint32_t desc)
+{
+ do_fmla_zpzzz_s(env, vg, desc, 0x80000000, 0);
+}
+
+void HELPER(sve_fnmla_zpzzz_s)(CPUARMState *env, void *vg, uint32_t desc)
+{
+ do_fmla_zpzzz_s(env, vg, desc, 0x80000000, 0x80000000);
+}
+
+void HELPER(sve_fnmls_zpzzz_s)(CPUARMState *env, void *vg, uint32_t desc)
+{
+ do_fmla_zpzzz_s(env, vg, desc, 0, 0x80000000);
+}
+
+static void do_fmla_zpzzz_d(CPUARMState *env, void *vg, uint32_t desc,
+ uint64_t neg1, uint64_t neg3)
+{
+ intptr_t i = simd_oprsz(desc);
+ unsigned rd = extract32(desc, SIMD_DATA_SHIFT, 5);
+ unsigned rn = extract32(desc, SIMD_DATA_SHIFT + 5, 5);
+ unsigned rm = extract32(desc, SIMD_DATA_SHIFT + 10, 5);
+ unsigned ra = extract32(desc, SIMD_DATA_SHIFT + 15, 5);
+ void *vd = &env->vfp.zregs[rd];
+ void *vn = &env->vfp.zregs[rn];
+ void *vm = &env->vfp.zregs[rm];
+ void *va = &env->vfp.zregs[ra];
+ uint64_t *g = vg;
+
+ do {
+ uint64_t pg = g[(i - 1) >> 6];
+ do {
+ i -= 8;
+ if (likely((pg >> (i & 63)) & 1)) {
+ float64 e1, e2, e3, r;
+
+ e1 = *(uint64_t *)(vn + i) ^ neg1;
+ e2 = *(uint64_t *)(vm + i);
+ e3 = *(uint64_t *)(va + i) ^ neg3;
+ r = float64_muladd(e1, e2, e3, 0, &env->vfp.fp_status);
+ *(uint64_t *)(vd + i) = r;
+ }
+ } while (i & 63);
+ } while (i != 0);
+}
+
+void HELPER(sve_fmla_zpzzz_d)(CPUARMState *env, void *vg, uint32_t desc)
+{
+ do_fmla_zpzzz_d(env, vg, desc, 0, 0);
+}
+
+void HELPER(sve_fmls_zpzzz_d)(CPUARMState *env, void *vg, uint32_t desc)
+{
+ do_fmla_zpzzz_d(env, vg, desc, INT64_MIN, 0);
+}
+
+void HELPER(sve_fnmla_zpzzz_d)(CPUARMState *env, void *vg, uint32_t desc)
+{
+ do_fmla_zpzzz_d(env, vg, desc, INT64_MIN, INT64_MIN);
+}
+
+void HELPER(sve_fnmls_zpzzz_d)(CPUARMState *env, void *vg, uint32_t desc)
+{
+ do_fmla_zpzzz_d(env, vg, desc, 0, INT64_MIN);
+}
+
+/* Two operand floating-point comparison controlled by a predicate.
+ * Unlike the integer version, we are not allowed to optimistically
+ * compare operands, since the comparison may have side effects wrt
+ * the FPSR.
+ */
+#define DO_FPCMP_PPZZ(NAME, TYPE, H, OP) \
+void HELPER(NAME)(void *vd, void *vn, void *vm, void *vg, \
+ void *status, uint32_t desc) \
+{ \
+ intptr_t i = simd_oprsz(desc), j = (i - 1) >> 6; \
+ uint64_t *d = vd, *g = vg; \
+ do { \
+ uint64_t out = 0, pg = g[j]; \
+ do { \
+ i -= sizeof(TYPE), out <<= sizeof(TYPE); \
+ if (likely((pg >> (i & 63)) & 1)) { \
+ TYPE nn = *(TYPE *)(vn + H(i)); \
+ TYPE mm = *(TYPE *)(vm + H(i)); \
+ out |= OP(TYPE, nn, mm, status); \
+ } \
+ } while (i & 63); \
+ d[j--] = out; \
+ } while (i > 0); \
+}
+
+#define DO_FPCMP_PPZZ_H(NAME, OP) \
+ DO_FPCMP_PPZZ(NAME##_h, float16, H1_2, OP)
+#define DO_FPCMP_PPZZ_S(NAME, OP) \
+ DO_FPCMP_PPZZ(NAME##_s, float32, H1_4, OP)
+#define DO_FPCMP_PPZZ_D(NAME, OP) \
+ DO_FPCMP_PPZZ(NAME##_d, float64, , OP)
+
+#define DO_FPCMP_PPZZ_ALL(NAME, OP) \
+ DO_FPCMP_PPZZ_H(NAME, OP) \
+ DO_FPCMP_PPZZ_S(NAME, OP) \
+ DO_FPCMP_PPZZ_D(NAME, OP)
+
+#define DO_FCMGE(TYPE, X, Y, ST) TYPE##_compare(Y, X, ST) <= 0
+#define DO_FCMGT(TYPE, X, Y, ST) TYPE##_compare(Y, X, ST) < 0
+#define DO_FCMLE(TYPE, X, Y, ST) TYPE##_compare(X, Y, ST) <= 0
+#define DO_FCMLT(TYPE, X, Y, ST) TYPE##_compare(X, Y, ST) < 0
+#define DO_FCMEQ(TYPE, X, Y, ST) TYPE##_compare_quiet(X, Y, ST) == 0
+#define DO_FCMNE(TYPE, X, Y, ST) TYPE##_compare_quiet(X, Y, ST) != 0
+#define DO_FCMUO(TYPE, X, Y, ST) \
+ TYPE##_compare_quiet(X, Y, ST) == float_relation_unordered
+#define DO_FACGE(TYPE, X, Y, ST) \
+ TYPE##_compare(TYPE##_abs(Y), TYPE##_abs(X), ST) <= 0
+#define DO_FACGT(TYPE, X, Y, ST) \
+ TYPE##_compare(TYPE##_abs(Y), TYPE##_abs(X), ST) < 0
+
+DO_FPCMP_PPZZ_ALL(sve_fcmge, DO_FCMGE)
+DO_FPCMP_PPZZ_ALL(sve_fcmgt, DO_FCMGT)
+DO_FPCMP_PPZZ_ALL(sve_fcmeq, DO_FCMEQ)
+DO_FPCMP_PPZZ_ALL(sve_fcmne, DO_FCMNE)
+DO_FPCMP_PPZZ_ALL(sve_fcmuo, DO_FCMUO)
+DO_FPCMP_PPZZ_ALL(sve_facge, DO_FACGE)
+DO_FPCMP_PPZZ_ALL(sve_facgt, DO_FACGT)
+
+#undef DO_FPCMP_PPZZ_ALL
+#undef DO_FPCMP_PPZZ_D
+#undef DO_FPCMP_PPZZ_S
+#undef DO_FPCMP_PPZZ_H
+#undef DO_FPCMP_PPZZ
+
+/* One operand floating-point comparison against zero, controlled
+ * by a predicate.
+ */
+#define DO_FPCMP_PPZ0(NAME, TYPE, H, OP) \
+void HELPER(NAME)(void *vd, void *vn, void *vg, \
+ void *status, uint32_t desc) \
+{ \
+ intptr_t i = simd_oprsz(desc), j = (i - 1) >> 6; \
+ uint64_t *d = vd, *g = vg; \
+ do { \
+ uint64_t out = 0, pg = g[j]; \
+ do { \
+ i -= sizeof(TYPE), out <<= sizeof(TYPE); \
+ if ((pg >> (i & 63)) & 1) { \
+ TYPE nn = *(TYPE *)(vn + H(i)); \
+ out |= OP(TYPE, nn, 0, status); \
+ } \
+ } while (i & 63); \
+ d[j--] = out; \
+ } while (i > 0); \
+}
+
+#define DO_FPCMP_PPZ0_H(NAME, OP) \
+ DO_FPCMP_PPZ0(NAME##_h, float16, H1_2, OP)
+#define DO_FPCMP_PPZ0_S(NAME, OP) \
+ DO_FPCMP_PPZ0(NAME##_s, float32, H1_4, OP)
+#define DO_FPCMP_PPZ0_D(NAME, OP) \
+ DO_FPCMP_PPZ0(NAME##_d, float64, , OP)
+
+#define DO_FPCMP_PPZ0_ALL(NAME, OP) \
+ DO_FPCMP_PPZ0_H(NAME, OP) \
+ DO_FPCMP_PPZ0_S(NAME, OP) \
+ DO_FPCMP_PPZ0_D(NAME, OP)
+
+DO_FPCMP_PPZ0_ALL(sve_fcmge0, DO_FCMGE)
+DO_FPCMP_PPZ0_ALL(sve_fcmgt0, DO_FCMGT)
+DO_FPCMP_PPZ0_ALL(sve_fcmle0, DO_FCMLE)
+DO_FPCMP_PPZ0_ALL(sve_fcmlt0, DO_FCMLT)
+DO_FPCMP_PPZ0_ALL(sve_fcmeq0, DO_FCMEQ)
+DO_FPCMP_PPZ0_ALL(sve_fcmne0, DO_FCMNE)
+
+/* FP Trig Multiply-Add. */
+
+void HELPER(sve_ftmad_h)(void *vd, void *vn, void *vm, void *vs, uint32_t desc)
+{
+ static const float16 coeff[16] = {
+ 0x3c00, 0xb155, 0x2030, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
+ 0x3c00, 0xb800, 0x293a, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
+ };
+ intptr_t i, opr_sz = simd_oprsz(desc) / sizeof(float16);
+ intptr_t x = simd_data(desc);
+ float16 *d = vd, *n = vn, *m = vm;
+ for (i = 0; i < opr_sz; i++) {
+ float16 mm = m[i];
+ intptr_t xx = x;
+ if (float16_is_neg(mm)) {
+ mm = float16_abs(mm);
+ xx += 8;
+ }
+ d[i] = float16_muladd(n[i], mm, coeff[xx], 0, vs);
+ }
+}
+
+void HELPER(sve_ftmad_s)(void *vd, void *vn, void *vm, void *vs, uint32_t desc)
+{
+ static const float32 coeff[16] = {
+ 0x3f800000, 0xbe2aaaab, 0x3c088886, 0xb95008b9,
+ 0x36369d6d, 0x00000000, 0x00000000, 0x00000000,
+ 0x3f800000, 0xbf000000, 0x3d2aaaa6, 0xbab60705,
+ 0x37cd37cc, 0x00000000, 0x00000000, 0x00000000,
+ };
+ intptr_t i, opr_sz = simd_oprsz(desc) / sizeof(float32);
+ intptr_t x = simd_data(desc);
+ float32 *d = vd, *n = vn, *m = vm;
+ for (i = 0; i < opr_sz; i++) {
+ float32 mm = m[i];
+ intptr_t xx = x;
+ if (float32_is_neg(mm)) {
+ mm = float32_abs(mm);
+ xx += 8;
+ }
+ d[i] = float32_muladd(n[i], mm, coeff[xx], 0, vs);
+ }
+}
+
+void HELPER(sve_ftmad_d)(void *vd, void *vn, void *vm, void *vs, uint32_t desc)
+{
+ static const float64 coeff[16] = {
+ 0x3ff0000000000000ull, 0xbfc5555555555543ull,
+ 0x3f8111111110f30cull, 0xbf2a01a019b92fc6ull,
+ 0x3ec71de351f3d22bull, 0xbe5ae5e2b60f7b91ull,
+ 0x3de5d8408868552full, 0x0000000000000000ull,
+ 0x3ff0000000000000ull, 0xbfe0000000000000ull,
+ 0x3fa5555555555536ull, 0xbf56c16c16c13a0bull,
+ 0x3efa01a019b1e8d8ull, 0xbe927e4f7282f468ull,
+ 0x3e21ee96d2641b13ull, 0xbda8f76380fbb401ull,
+ };
+ intptr_t i, opr_sz = simd_oprsz(desc) / sizeof(float64);
+ intptr_t x = simd_data(desc);
+ float64 *d = vd, *n = vn, *m = vm;
+ for (i = 0; i < opr_sz; i++) {
+ float64 mm = m[i];
+ intptr_t xx = x;
+ if (float64_is_neg(mm)) {
+ mm = float64_abs(mm);
+ xx += 8;
+ }
+ d[i] = float64_muladd(n[i], mm, coeff[xx], 0, vs);
+ }
+}
+
+/*
+ * FP Complex Add
+ */
+
+void HELPER(sve_fcadd_h)(void *vd, void *vn, void *vm, void *vg,
+ void *vs, uint32_t desc)
+{
+ intptr_t j, i = simd_oprsz(desc);
+ uint64_t *g = vg;
+ float16 neg_imag = float16_set_sign(0, simd_data(desc));
+ float16 neg_real = float16_chs(neg_imag);
+
+ do {
+ uint64_t pg = g[(i - 1) >> 6];
+ do {
+ float16 e0, e1, e2, e3;
+
+ /* I holds the real index; J holds the imag index. */
+ j = i - sizeof(float16);
+ i -= 2 * sizeof(float16);
+
+ e0 = *(float16 *)(vn + H1_2(i));
+ e1 = *(float16 *)(vm + H1_2(j)) ^ neg_real;
+ e2 = *(float16 *)(vn + H1_2(j));
+ e3 = *(float16 *)(vm + H1_2(i)) ^ neg_imag;
+
+ if (likely((pg >> (i & 63)) & 1)) {
+ *(float16 *)(vd + H1_2(i)) = float16_add(e0, e1, vs);
+ }
+ if (likely((pg >> (j & 63)) & 1)) {
+ *(float16 *)(vd + H1_2(j)) = float16_add(e2, e3, vs);
+ }
+ } while (i & 63);
+ } while (i != 0);
+}
+
+void HELPER(sve_fcadd_s)(void *vd, void *vn, void *vm, void *vg,
+ void *vs, uint32_t desc)
+{
+ intptr_t j, i = simd_oprsz(desc);
+ uint64_t *g = vg;
+ float32 neg_imag = float32_set_sign(0, simd_data(desc));
+ float32 neg_real = float32_chs(neg_imag);
+
+ do {
+ uint64_t pg = g[(i - 1) >> 6];
+ do {
+ float32 e0, e1, e2, e3;
+
+ /* I holds the real index; J holds the imag index. */
+ j = i - sizeof(float32);
+ i -= 2 * sizeof(float32);
+
+ e0 = *(float32 *)(vn + H1_2(i));
+ e1 = *(float32 *)(vm + H1_2(j)) ^ neg_real;
+ e2 = *(float32 *)(vn + H1_2(j));
+ e3 = *(float32 *)(vm + H1_2(i)) ^ neg_imag;
+
+ if (likely((pg >> (i & 63)) & 1)) {
+ *(float32 *)(vd + H1_2(i)) = float32_add(e0, e1, vs);
+ }
+ if (likely((pg >> (j & 63)) & 1)) {
+ *(float32 *)(vd + H1_2(j)) = float32_add(e2, e3, vs);
+ }
+ } while (i & 63);
+ } while (i != 0);
+}
+
+void HELPER(sve_fcadd_d)(void *vd, void *vn, void *vm, void *vg,
+ void *vs, uint32_t desc)
+{
+ intptr_t j, i = simd_oprsz(desc);
+ uint64_t *g = vg;
+ float64 neg_imag = float64_set_sign(0, simd_data(desc));
+ float64 neg_real = float64_chs(neg_imag);
+
+ do {
+ uint64_t pg = g[(i - 1) >> 6];
+ do {
+ float64 e0, e1, e2, e3;
+
+ /* I holds the real index; J holds the imag index. */
+ j = i - sizeof(float64);
+ i -= 2 * sizeof(float64);
+
+ e0 = *(float64 *)(vn + H1_2(i));
+ e1 = *(float64 *)(vm + H1_2(j)) ^ neg_real;
+ e2 = *(float64 *)(vn + H1_2(j));
+ e3 = *(float64 *)(vm + H1_2(i)) ^ neg_imag;
+
+ if (likely((pg >> (i & 63)) & 1)) {
+ *(float64 *)(vd + H1_2(i)) = float64_add(e0, e1, vs);
+ }
+ if (likely((pg >> (j & 63)) & 1)) {
+ *(float64 *)(vd + H1_2(j)) = float64_add(e2, e3, vs);
+ }
+ } while (i & 63);
+ } while (i != 0);
+}
+
+/*
+ * FP Complex Multiply
+ */
+
+QEMU_BUILD_BUG_ON(SIMD_DATA_SHIFT + 22 > 32);
+
+void HELPER(sve_fcmla_zpzzz_h)(CPUARMState *env, void *vg, uint32_t desc)
+{
+ intptr_t j, i = simd_oprsz(desc);
+ unsigned rd = extract32(desc, SIMD_DATA_SHIFT, 5);
+ unsigned rn = extract32(desc, SIMD_DATA_SHIFT + 5, 5);
+ unsigned rm = extract32(desc, SIMD_DATA_SHIFT + 10, 5);
+ unsigned ra = extract32(desc, SIMD_DATA_SHIFT + 15, 5);
+ unsigned rot = extract32(desc, SIMD_DATA_SHIFT + 20, 2);
+ bool flip = rot & 1;
+ float16 neg_imag, neg_real;
+ void *vd = &env->vfp.zregs[rd];
+ void *vn = &env->vfp.zregs[rn];
+ void *vm = &env->vfp.zregs[rm];
+ void *va = &env->vfp.zregs[ra];
+ uint64_t *g = vg;
+
+ neg_imag = float16_set_sign(0, (rot & 2) != 0);
+ neg_real = float16_set_sign(0, rot == 1 || rot == 2);
+
+ do {
+ uint64_t pg = g[(i - 1) >> 6];
+ do {
+ float16 e1, e2, e3, e4, nr, ni, mr, mi, d;
+
+ /* I holds the real index; J holds the imag index. */
+ j = i - sizeof(float16);
+ i -= 2 * sizeof(float16);
+
+ nr = *(float16 *)(vn + H1_2(i));
+ ni = *(float16 *)(vn + H1_2(j));
+ mr = *(float16 *)(vm + H1_2(i));
+ mi = *(float16 *)(vm + H1_2(j));
+
+ e2 = (flip ? ni : nr);
+ e1 = (flip ? mi : mr) ^ neg_real;
+ e4 = e2;
+ e3 = (flip ? mr : mi) ^ neg_imag;
+
+ if (likely((pg >> (i & 63)) & 1)) {
+ d = *(float16 *)(va + H1_2(i));
+ d = float16_muladd(e2, e1, d, 0, &env->vfp.fp_status_f16);
+ *(float16 *)(vd + H1_2(i)) = d;
+ }
+ if (likely((pg >> (j & 63)) & 1)) {
+ d = *(float16 *)(va + H1_2(j));
+ d = float16_muladd(e4, e3, d, 0, &env->vfp.fp_status_f16);
+ *(float16 *)(vd + H1_2(j)) = d;
+ }
+ } while (i & 63);
+ } while (i != 0);
+}
+
+void HELPER(sve_fcmla_zpzzz_s)(CPUARMState *env, void *vg, uint32_t desc)
+{
+ intptr_t j, i = simd_oprsz(desc);
+ unsigned rd = extract32(desc, SIMD_DATA_SHIFT, 5);
+ unsigned rn = extract32(desc, SIMD_DATA_SHIFT + 5, 5);
+ unsigned rm = extract32(desc, SIMD_DATA_SHIFT + 10, 5);
+ unsigned ra = extract32(desc, SIMD_DATA_SHIFT + 15, 5);
+ unsigned rot = extract32(desc, SIMD_DATA_SHIFT + 20, 2);
+ bool flip = rot & 1;
+ float32 neg_imag, neg_real;
+ void *vd = &env->vfp.zregs[rd];
+ void *vn = &env->vfp.zregs[rn];
+ void *vm = &env->vfp.zregs[rm];
+ void *va = &env->vfp.zregs[ra];
+ uint64_t *g = vg;
+
+ neg_imag = float32_set_sign(0, (rot & 2) != 0);
+ neg_real = float32_set_sign(0, rot == 1 || rot == 2);
+
+ do {
+ uint64_t pg = g[(i - 1) >> 6];
+ do {
+ float32 e1, e2, e3, e4, nr, ni, mr, mi, d;
+
+ /* I holds the real index; J holds the imag index. */
+ j = i - sizeof(float32);
+ i -= 2 * sizeof(float32);
+
+ nr = *(float32 *)(vn + H1_2(i));
+ ni = *(float32 *)(vn + H1_2(j));
+ mr = *(float32 *)(vm + H1_2(i));
+ mi = *(float32 *)(vm + H1_2(j));
+
+ e2 = (flip ? ni : nr);
+ e1 = (flip ? mi : mr) ^ neg_real;
+ e4 = e2;
+ e3 = (flip ? mr : mi) ^ neg_imag;
+
+ if (likely((pg >> (i & 63)) & 1)) {
+ d = *(float32 *)(va + H1_2(i));
+ d = float32_muladd(e2, e1, d, 0, &env->vfp.fp_status);
+ *(float32 *)(vd + H1_2(i)) = d;
+ }
+ if (likely((pg >> (j & 63)) & 1)) {
+ d = *(float32 *)(va + H1_2(j));
+ d = float32_muladd(e4, e3, d, 0, &env->vfp.fp_status);
+ *(float32 *)(vd + H1_2(j)) = d;
+ }
+ } while (i & 63);
+ } while (i != 0);
+}
+
+void HELPER(sve_fcmla_zpzzz_d)(CPUARMState *env, void *vg, uint32_t desc)
+{
+ intptr_t j, i = simd_oprsz(desc);
+ unsigned rd = extract32(desc, SIMD_DATA_SHIFT, 5);
+ unsigned rn = extract32(desc, SIMD_DATA_SHIFT + 5, 5);
+ unsigned rm = extract32(desc, SIMD_DATA_SHIFT + 10, 5);
+ unsigned ra = extract32(desc, SIMD_DATA_SHIFT + 15, 5);
+ unsigned rot = extract32(desc, SIMD_DATA_SHIFT + 20, 2);
+ bool flip = rot & 1;
+ float64 neg_imag, neg_real;
+ void *vd = &env->vfp.zregs[rd];
+ void *vn = &env->vfp.zregs[rn];
+ void *vm = &env->vfp.zregs[rm];
+ void *va = &env->vfp.zregs[ra];
+ uint64_t *g = vg;
+
+ neg_imag = float64_set_sign(0, (rot & 2) != 0);
+ neg_real = float64_set_sign(0, rot == 1 || rot == 2);
+
+ do {
+ uint64_t pg = g[(i - 1) >> 6];
+ do {
+ float64 e1, e2, e3, e4, nr, ni, mr, mi, d;
+
+ /* I holds the real index; J holds the imag index. */
+ j = i - sizeof(float64);
+ i -= 2 * sizeof(float64);
+
+ nr = *(float64 *)(vn + H1_2(i));
+ ni = *(float64 *)(vn + H1_2(j));
+ mr = *(float64 *)(vm + H1_2(i));
+ mi = *(float64 *)(vm + H1_2(j));
+
+ e2 = (flip ? ni : nr);
+ e1 = (flip ? mi : mr) ^ neg_real;
+ e4 = e2;
+ e3 = (flip ? mr : mi) ^ neg_imag;
+
+ if (likely((pg >> (i & 63)) & 1)) {
+ d = *(float64 *)(va + H1_2(i));
+ d = float64_muladd(e2, e1, d, 0, &env->vfp.fp_status);
+ *(float64 *)(vd + H1_2(i)) = d;
+ }
+ if (likely((pg >> (j & 63)) & 1)) {
+ d = *(float64 *)(va + H1_2(j));
+ d = float64_muladd(e4, e3, d, 0, &env->vfp.fp_status);
+ *(float64 *)(vd + H1_2(j)) = d;
+ }
+ } while (i & 63);
+ } while (i != 0);
+}
+
+/*
+ * Load contiguous data, protected by a governing predicate.
+ */
+#define DO_LD1(NAME, FN, TYPEE, TYPEM, H) \
+static void do_##NAME(CPUARMState *env, void *vd, void *vg, \
+ target_ulong addr, intptr_t oprsz, \
+ uintptr_t ra) \
+{ \
+ intptr_t i = 0; \
+ do { \
+ uint16_t pg = *(uint16_t *)(vg + H1_2(i >> 3)); \
+ do { \
+ TYPEM m = 0; \
+ if (pg & 1) { \
+ m = FN(env, addr, ra); \
+ } \
+ *(TYPEE *)(vd + H(i)) = m; \
+ i += sizeof(TYPEE), pg >>= sizeof(TYPEE); \
+ addr += sizeof(TYPEM); \
+ } while (i & 15); \
+ } while (i < oprsz); \
+} \
+void HELPER(NAME)(CPUARMState *env, void *vg, \
+ target_ulong addr, uint32_t desc) \
+{ \
+ do_##NAME(env, &env->vfp.zregs[simd_data(desc)], vg, \
+ addr, simd_oprsz(desc), GETPC()); \
+}
+
+#define DO_LD2(NAME, FN, TYPEE, TYPEM, H) \
+void HELPER(NAME)(CPUARMState *env, void *vg, \
+ target_ulong addr, uint32_t desc) \
+{ \
+ intptr_t i, oprsz = simd_oprsz(desc); \
+ intptr_t ra = GETPC(); \
+ unsigned rd = simd_data(desc); \
+ void *d1 = &env->vfp.zregs[rd]; \
+ void *d2 = &env->vfp.zregs[(rd + 1) & 31]; \
+ for (i = 0; i < oprsz; ) { \
+ uint16_t pg = *(uint16_t *)(vg + H1_2(i >> 3)); \
+ do { \
+ TYPEM m1 = 0, m2 = 0; \
+ if (pg & 1) { \
+ m1 = FN(env, addr, ra); \
+ m2 = FN(env, addr + sizeof(TYPEM), ra); \
+ } \
+ *(TYPEE *)(d1 + H(i)) = m1; \
+ *(TYPEE *)(d2 + H(i)) = m2; \
+ i += sizeof(TYPEE), pg >>= sizeof(TYPEE); \
+ addr += 2 * sizeof(TYPEM); \
+ } while (i & 15); \
+ } \
+}
+
+#define DO_LD3(NAME, FN, TYPEE, TYPEM, H) \
+void HELPER(NAME)(CPUARMState *env, void *vg, \
+ target_ulong addr, uint32_t desc) \
+{ \
+ intptr_t i, oprsz = simd_oprsz(desc); \
+ intptr_t ra = GETPC(); \
+ unsigned rd = simd_data(desc); \
+ void *d1 = &env->vfp.zregs[rd]; \
+ void *d2 = &env->vfp.zregs[(rd + 1) & 31]; \
+ void *d3 = &env->vfp.zregs[(rd + 2) & 31]; \
+ for (i = 0; i < oprsz; ) { \
+ uint16_t pg = *(uint16_t *)(vg + H1_2(i >> 3)); \
+ do { \
+ TYPEM m1 = 0, m2 = 0, m3 = 0; \
+ if (pg & 1) { \
+ m1 = FN(env, addr, ra); \
+ m2 = FN(env, addr + sizeof(TYPEM), ra); \
+ m3 = FN(env, addr + 2 * sizeof(TYPEM), ra); \
+ } \
+ *(TYPEE *)(d1 + H(i)) = m1; \
+ *(TYPEE *)(d2 + H(i)) = m2; \
+ *(TYPEE *)(d3 + H(i)) = m3; \
+ i += sizeof(TYPEE), pg >>= sizeof(TYPEE); \
+ addr += 3 * sizeof(TYPEM); \
+ } while (i & 15); \
+ } \
+}
+
+#define DO_LD4(NAME, FN, TYPEE, TYPEM, H) \
+void HELPER(NAME)(CPUARMState *env, void *vg, \
+ target_ulong addr, uint32_t desc) \
+{ \
+ intptr_t i, oprsz = simd_oprsz(desc); \
+ intptr_t ra = GETPC(); \
+ unsigned rd = simd_data(desc); \
+ void *d1 = &env->vfp.zregs[rd]; \
+ void *d2 = &env->vfp.zregs[(rd + 1) & 31]; \
+ void *d3 = &env->vfp.zregs[(rd + 2) & 31]; \
+ void *d4 = &env->vfp.zregs[(rd + 3) & 31]; \
+ for (i = 0; i < oprsz; ) { \
+ uint16_t pg = *(uint16_t *)(vg + H1_2(i >> 3)); \
+ do { \
+ TYPEM m1 = 0, m2 = 0, m3 = 0, m4 = 0; \
+ if (pg & 1) { \
+ m1 = FN(env, addr, ra); \
+ m2 = FN(env, addr + sizeof(TYPEM), ra); \
+ m3 = FN(env, addr + 2 * sizeof(TYPEM), ra); \
+ m4 = FN(env, addr + 3 * sizeof(TYPEM), ra); \
+ } \
+ *(TYPEE *)(d1 + H(i)) = m1; \
+ *(TYPEE *)(d2 + H(i)) = m2; \
+ *(TYPEE *)(d3 + H(i)) = m3; \
+ *(TYPEE *)(d4 + H(i)) = m4; \
+ i += sizeof(TYPEE), pg >>= sizeof(TYPEE); \
+ addr += 4 * sizeof(TYPEM); \
+ } while (i & 15); \
+ } \
+}
+
+DO_LD1(sve_ld1bhu_r, cpu_ldub_data_ra, uint16_t, uint8_t, H1_2)
+DO_LD1(sve_ld1bhs_r, cpu_ldsb_data_ra, uint16_t, int8_t, H1_2)
+DO_LD1(sve_ld1bsu_r, cpu_ldub_data_ra, uint32_t, uint8_t, H1_4)
+DO_LD1(sve_ld1bss_r, cpu_ldsb_data_ra, uint32_t, int8_t, H1_4)
+DO_LD1(sve_ld1bdu_r, cpu_ldub_data_ra, uint64_t, uint8_t, )
+DO_LD1(sve_ld1bds_r, cpu_ldsb_data_ra, uint64_t, int8_t, )
+
+DO_LD1(sve_ld1hsu_r, cpu_lduw_data_ra, uint32_t, uint16_t, H1_4)
+DO_LD1(sve_ld1hss_r, cpu_ldsw_data_ra, uint32_t, int8_t, H1_4)
+DO_LD1(sve_ld1hdu_r, cpu_lduw_data_ra, uint64_t, uint16_t, )
+DO_LD1(sve_ld1hds_r, cpu_ldsw_data_ra, uint64_t, int16_t, )
+
+DO_LD1(sve_ld1sdu_r, cpu_ldl_data_ra, uint64_t, uint32_t, )
+DO_LD1(sve_ld1sds_r, cpu_ldl_data_ra, uint64_t, int32_t, )
+
+DO_LD1(sve_ld1bb_r, cpu_ldub_data_ra, uint8_t, uint8_t, H1)
+DO_LD2(sve_ld2bb_r, cpu_ldub_data_ra, uint8_t, uint8_t, H1)
+DO_LD3(sve_ld3bb_r, cpu_ldub_data_ra, uint8_t, uint8_t, H1)
+DO_LD4(sve_ld4bb_r, cpu_ldub_data_ra, uint8_t, uint8_t, H1)
+
+DO_LD1(sve_ld1hh_r, cpu_lduw_data_ra, uint16_t, uint16_t, H1_2)
+DO_LD2(sve_ld2hh_r, cpu_lduw_data_ra, uint16_t, uint16_t, H1_2)
+DO_LD3(sve_ld3hh_r, cpu_lduw_data_ra, uint16_t, uint16_t, H1_2)
+DO_LD4(sve_ld4hh_r, cpu_lduw_data_ra, uint16_t, uint16_t, H1_2)
+
+DO_LD1(sve_ld1ss_r, cpu_ldl_data_ra, uint32_t, uint32_t, H1_4)
+DO_LD2(sve_ld2ss_r, cpu_ldl_data_ra, uint32_t, uint32_t, H1_4)
+DO_LD3(sve_ld3ss_r, cpu_ldl_data_ra, uint32_t, uint32_t, H1_4)
+DO_LD4(sve_ld4ss_r, cpu_ldl_data_ra, uint32_t, uint32_t, H1_4)
+
+DO_LD1(sve_ld1dd_r, cpu_ldq_data_ra, uint64_t, uint64_t, )
+DO_LD2(sve_ld2dd_r, cpu_ldq_data_ra, uint64_t, uint64_t, )
+DO_LD3(sve_ld3dd_r, cpu_ldq_data_ra, uint64_t, uint64_t, )
+DO_LD4(sve_ld4dd_r, cpu_ldq_data_ra, uint64_t, uint64_t, )
+
+#undef DO_LD1
+#undef DO_LD2
+#undef DO_LD3
+#undef DO_LD4
+
+/*
+ * Load contiguous data, first-fault and no-fault.
+ */
+
+#ifdef CONFIG_USER_ONLY
+
+/* Fault on byte I. All bits in FFR from I are cleared. The vector
+ * result from I is CONSTRAINED UNPREDICTABLE; we choose the MERGE
+ * option, which leaves subsequent data unchanged.
+ */
+static void record_fault(CPUARMState *env, uintptr_t i, uintptr_t oprsz)
+{
+ uint64_t *ffr = env->vfp.pregs[FFR_PRED_NUM].p;
+
+ if (i & 63) {
+ ffr[i / 64] &= MAKE_64BIT_MASK(0, i & 63);
+ i = ROUND_UP(i, 64);
+ }
+ for (; i < oprsz; i += 64) {
+ ffr[i / 64] = 0;
+ }
+}
+
+/* Hold the mmap lock during the operation so that there is no race
+ * between page_check_range and the load operation. We expect the
+ * usual case to have no faults at all, so we check the whole range
+ * first and if successful defer to the normal load operation.
+ *
+ * TODO: Change mmap_lock to a rwlock so that multiple readers
+ * can run simultaneously. This will probably help other uses
+ * within QEMU as well.
+ */
+#define DO_LDFF1(PART, FN, TYPEE, TYPEM, H) \
+static void do_sve_ldff1##PART(CPUARMState *env, void *vd, void *vg, \
+ target_ulong addr, intptr_t oprsz, \
+ bool first, uintptr_t ra) \
+{ \
+ intptr_t i = 0; \
+ do { \
+ uint16_t pg = *(uint16_t *)(vg + H1_2(i >> 3)); \
+ do { \
+ TYPEM m = 0; \
+ if (pg & 1) { \
+ if (!first && \
+ unlikely(page_check_range(addr, sizeof(TYPEM), \
+ PAGE_READ))) { \
+ record_fault(env, i, oprsz); \
+ return; \
+ } \
+ m = FN(env, addr, ra); \
+ first = false; \
+ } \
+ *(TYPEE *)(vd + H(i)) = m; \
+ i += sizeof(TYPEE), pg >>= sizeof(TYPEE); \
+ addr += sizeof(TYPEM); \
+ } while (i & 15); \
+ } while (i < oprsz); \
+} \
+void HELPER(sve_ldff1##PART)(CPUARMState *env, void *vg, \
+ target_ulong addr, uint32_t desc) \
+{ \
+ intptr_t oprsz = simd_oprsz(desc); \
+ unsigned rd = simd_data(desc); \
+ void *vd = &env->vfp.zregs[rd]; \
+ mmap_lock(); \
+ if (likely(page_check_range(addr, oprsz, PAGE_READ) == 0)) { \
+ do_sve_ld1##PART(env, vd, vg, addr, oprsz, GETPC()); \
+ } else { \
+ do_sve_ldff1##PART(env, vd, vg, addr, oprsz, true, GETPC()); \
+ } \
+ mmap_unlock(); \
+}
+
+/* No-fault loads are like first-fault loads without the
+ * first faulting special case.
+ */
+#define DO_LDNF1(PART) \
+void HELPER(sve_ldnf1##PART)(CPUARMState *env, void *vg, \
+ target_ulong addr, uint32_t desc) \
+{ \
+ intptr_t oprsz = simd_oprsz(desc); \
+ unsigned rd = simd_data(desc); \
+ void *vd = &env->vfp.zregs[rd]; \
+ mmap_lock(); \
+ if (likely(page_check_range(addr, oprsz, PAGE_READ) == 0)) { \
+ do_sve_ld1##PART(env, vd, vg, addr, oprsz, GETPC()); \
+ } else { \
+ do_sve_ldff1##PART(env, vd, vg, addr, oprsz, false, GETPC()); \
+ } \
+ mmap_unlock(); \
+}
+
+#else
+
+/* TODO: System mode is not yet supported.
+ * This would probably use tlb_vaddr_to_host.
+ */
+#define DO_LDFF1(PART, FN, TYPEE, TYPEM, H) \
+void HELPER(sve_ldff1##PART)(CPUARMState *env, void *vg, \
+ target_ulong addr, uint32_t desc) \
+{ \
+ g_assert_not_reached(); \
+}
+
+#define DO_LDNF1(PART) \
+void HELPER(sve_ldnf1##PART)(CPUARMState *env, void *vg, \
+ target_ulong addr, uint32_t desc) \
+{ \
+ g_assert_not_reached(); \
+}
+
+#endif
+
+DO_LDFF1(bb_r, cpu_ldub_data_ra, uint8_t, uint8_t, H1)
+DO_LDFF1(bhu_r, cpu_ldub_data_ra, uint16_t, uint8_t, H1_2)
+DO_LDFF1(bhs_r, cpu_ldsb_data_ra, uint16_t, int8_t, H1_2)
+DO_LDFF1(bsu_r, cpu_ldub_data_ra, uint32_t, uint8_t, H1_4)
+DO_LDFF1(bss_r, cpu_ldsb_data_ra, uint32_t, int8_t, H1_4)
+DO_LDFF1(bdu_r, cpu_ldub_data_ra, uint64_t, uint8_t, )
+DO_LDFF1(bds_r, cpu_ldsb_data_ra, uint64_t, int8_t, )
+
+DO_LDFF1(hh_r, cpu_lduw_data_ra, uint16_t, uint16_t, H1_2)
+DO_LDFF1(hsu_r, cpu_lduw_data_ra, uint32_t, uint16_t, H1_4)
+DO_LDFF1(hss_r, cpu_ldsw_data_ra, uint32_t, int8_t, H1_4)
+DO_LDFF1(hdu_r, cpu_lduw_data_ra, uint64_t, uint16_t, )
+DO_LDFF1(hds_r, cpu_ldsw_data_ra, uint64_t, int16_t, )
+
+DO_LDFF1(ss_r, cpu_ldl_data_ra, uint32_t, uint32_t, H1_4)
+DO_LDFF1(sdu_r, cpu_ldl_data_ra, uint64_t, uint32_t, )
+DO_LDFF1(sds_r, cpu_ldl_data_ra, uint64_t, int32_t, )
+
+DO_LDFF1(dd_r, cpu_ldq_data_ra, uint64_t, uint64_t, )
+
+#undef DO_LDFF1
+
+DO_LDNF1(bb_r)
+DO_LDNF1(bhu_r)
+DO_LDNF1(bhs_r)
+DO_LDNF1(bsu_r)
+DO_LDNF1(bss_r)
+DO_LDNF1(bdu_r)
+DO_LDNF1(bds_r)
+
+DO_LDNF1(hh_r)
+DO_LDNF1(hsu_r)
+DO_LDNF1(hss_r)
+DO_LDNF1(hdu_r)
+DO_LDNF1(hds_r)
+
+DO_LDNF1(ss_r)
+DO_LDNF1(sdu_r)
+DO_LDNF1(sds_r)
+
+DO_LDNF1(dd_r)
+
+#undef DO_LDNF1
+
+/*
+ * Store contiguous data, protected by a governing predicate.
+ */
+#define DO_ST1(NAME, FN, TYPEE, TYPEM, H) \
+void HELPER(NAME)(CPUARMState *env, void *vg, \
+ target_ulong addr, uint32_t desc) \
+{ \
+ intptr_t i, oprsz = simd_oprsz(desc); \
+ intptr_t ra = GETPC(); \
+ unsigned rd = simd_data(desc); \
+ void *vd = &env->vfp.zregs[rd]; \
+ for (i = 0; i < oprsz; ) { \
+ uint16_t pg = *(uint16_t *)(vg + H1_2(i >> 3)); \
+ do { \
+ if (pg & 1) { \
+ TYPEM m = *(TYPEE *)(vd + H(i)); \
+ FN(env, addr, m, ra); \
+ } \
+ i += sizeof(TYPEE), pg >>= sizeof(TYPEE); \
+ addr += sizeof(TYPEM); \
+ } while (i & 15); \
+ } \
+}
+
+#define DO_ST1_D(NAME, FN, TYPEM) \
+void HELPER(NAME)(CPUARMState *env, void *vg, \
+ target_ulong addr, uint32_t desc) \
+{ \
+ intptr_t i, oprsz = simd_oprsz(desc) / 8; \
+ intptr_t ra = GETPC(); \
+ unsigned rd = simd_data(desc); \
+ uint64_t *d = &env->vfp.zregs[rd].d[0]; \
+ uint8_t *pg = vg; \
+ for (i = 0; i < oprsz; i += 1) { \
+ if (pg[H1(i)] & 1) { \
+ FN(env, addr, d[i], ra); \
+ } \
+ addr += sizeof(TYPEM); \
+ } \
+}
+
+#define DO_ST2(NAME, FN, TYPEE, TYPEM, H) \
+void HELPER(NAME)(CPUARMState *env, void *vg, \
+ target_ulong addr, uint32_t desc) \
+{ \
+ intptr_t i, oprsz = simd_oprsz(desc); \
+ intptr_t ra = GETPC(); \
+ unsigned rd = simd_data(desc); \
+ void *d1 = &env->vfp.zregs[rd]; \
+ void *d2 = &env->vfp.zregs[(rd + 1) & 31]; \
+ for (i = 0; i < oprsz; ) { \
+ uint16_t pg = *(uint16_t *)(vg + H1_2(i >> 3)); \
+ do { \
+ if (pg & 1) { \
+ TYPEM m1 = *(TYPEE *)(d1 + H(i)); \
+ TYPEM m2 = *(TYPEE *)(d2 + H(i)); \
+ FN(env, addr, m1, ra); \
+ FN(env, addr + sizeof(TYPEM), m2, ra); \
+ } \
+ i += sizeof(TYPEE), pg >>= sizeof(TYPEE); \
+ addr += 2 * sizeof(TYPEM); \
+ } while (i & 15); \
+ } \
+}
+
+#define DO_ST3(NAME, FN, TYPEE, TYPEM, H) \
+void HELPER(NAME)(CPUARMState *env, void *vg, \
+ target_ulong addr, uint32_t desc) \
+{ \
+ intptr_t i, oprsz = simd_oprsz(desc); \
+ intptr_t ra = GETPC(); \
+ unsigned rd = simd_data(desc); \
+ void *d1 = &env->vfp.zregs[rd]; \
+ void *d2 = &env->vfp.zregs[(rd + 1) & 31]; \
+ void *d3 = &env->vfp.zregs[(rd + 2) & 31]; \
+ for (i = 0; i < oprsz; ) { \
+ uint16_t pg = *(uint16_t *)(vg + H1_2(i >> 3)); \
+ do { \
+ if (pg & 1) { \
+ TYPEM m1 = *(TYPEE *)(d1 + H(i)); \
+ TYPEM m2 = *(TYPEE *)(d2 + H(i)); \
+ TYPEM m3 = *(TYPEE *)(d3 + H(i)); \
+ FN(env, addr, m1, ra); \
+ FN(env, addr + sizeof(TYPEM), m2, ra); \
+ FN(env, addr + 2 * sizeof(TYPEM), m3, ra); \
+ } \
+ i += sizeof(TYPEE), pg >>= sizeof(TYPEE); \
+ addr += 3 * sizeof(TYPEM); \
+ } while (i & 15); \
+ } \
+}
+
+#define DO_ST4(NAME, FN, TYPEE, TYPEM, H) \
+void HELPER(NAME)(CPUARMState *env, void *vg, \
+ target_ulong addr, uint32_t desc) \
+{ \
+ intptr_t i, oprsz = simd_oprsz(desc); \
+ intptr_t ra = GETPC(); \
+ unsigned rd = simd_data(desc); \
+ void *d1 = &env->vfp.zregs[rd]; \
+ void *d2 = &env->vfp.zregs[(rd + 1) & 31]; \
+ void *d3 = &env->vfp.zregs[(rd + 2) & 31]; \
+ void *d4 = &env->vfp.zregs[(rd + 3) & 31]; \
+ for (i = 0; i < oprsz; ) { \
+ uint16_t pg = *(uint16_t *)(vg + H1_2(i >> 3)); \
+ do { \
+ if (pg & 1) { \
+ TYPEM m1 = *(TYPEE *)(d1 + H(i)); \
+ TYPEM m2 = *(TYPEE *)(d2 + H(i)); \
+ TYPEM m3 = *(TYPEE *)(d3 + H(i)); \
+ TYPEM m4 = *(TYPEE *)(d4 + H(i)); \
+ FN(env, addr, m1, ra); \
+ FN(env, addr + sizeof(TYPEM), m2, ra); \
+ FN(env, addr + 2 * sizeof(TYPEM), m3, ra); \
+ FN(env, addr + 3 * sizeof(TYPEM), m4, ra); \
+ } \
+ i += sizeof(TYPEE), pg >>= sizeof(TYPEE); \
+ addr += 4 * sizeof(TYPEM); \
+ } while (i & 15); \
+ } \
+}
+
+DO_ST1(sve_st1bh_r, cpu_stb_data_ra, uint16_t, uint8_t, H1_2)
+DO_ST1(sve_st1bs_r, cpu_stb_data_ra, uint32_t, uint8_t, H1_4)
+DO_ST1_D(sve_st1bd_r, cpu_stb_data_ra, uint8_t)
+
+DO_ST1(sve_st1hs_r, cpu_stw_data_ra, uint32_t, uint16_t, H1_4)
+DO_ST1_D(sve_st1hd_r, cpu_stw_data_ra, uint16_t)
+
+DO_ST1_D(sve_st1sd_r, cpu_stl_data_ra, uint32_t)
+
+DO_ST1(sve_st1bb_r, cpu_stb_data_ra, uint8_t, uint8_t, H1)
+DO_ST2(sve_st2bb_r, cpu_stb_data_ra, uint8_t, uint8_t, H1)
+DO_ST3(sve_st3bb_r, cpu_stb_data_ra, uint8_t, uint8_t, H1)
+DO_ST4(sve_st4bb_r, cpu_stb_data_ra, uint8_t, uint8_t, H1)
+
+DO_ST1(sve_st1hh_r, cpu_stw_data_ra, uint16_t, uint16_t, H1_2)
+DO_ST2(sve_st2hh_r, cpu_stw_data_ra, uint16_t, uint16_t, H1_2)
+DO_ST3(sve_st3hh_r, cpu_stw_data_ra, uint16_t, uint16_t, H1_2)
+DO_ST4(sve_st4hh_r, cpu_stw_data_ra, uint16_t, uint16_t, H1_2)
+
+DO_ST1(sve_st1ss_r, cpu_stl_data_ra, uint32_t, uint32_t, H1_4)
+DO_ST2(sve_st2ss_r, cpu_stl_data_ra, uint32_t, uint32_t, H1_4)
+DO_ST3(sve_st3ss_r, cpu_stl_data_ra, uint32_t, uint32_t, H1_4)
+DO_ST4(sve_st4ss_r, cpu_stl_data_ra, uint32_t, uint32_t, H1_4)
+
+DO_ST1_D(sve_st1dd_r, cpu_stq_data_ra, uint64_t)
+
+void HELPER(sve_st2dd_r)(CPUARMState *env, void *vg,
+ target_ulong addr, uint32_t desc)
+{
+ intptr_t i, oprsz = simd_oprsz(desc) / 8;
+ intptr_t ra = GETPC();
+ unsigned rd = simd_data(desc);
+ uint64_t *d1 = &env->vfp.zregs[rd].d[0];
+ uint64_t *d2 = &env->vfp.zregs[(rd + 1) & 31].d[0];
+ uint8_t *pg = vg;
+
+ for (i = 0; i < oprsz; i += 1) {
+ if (pg[H1(i)] & 1) {
+ cpu_stq_data_ra(env, addr, d1[i], ra);
+ cpu_stq_data_ra(env, addr + 8, d2[i], ra);
+ }
+ addr += 2 * 8;
+ }
+}
+
+void HELPER(sve_st3dd_r)(CPUARMState *env, void *vg,
+ target_ulong addr, uint32_t desc)
+{
+ intptr_t i, oprsz = simd_oprsz(desc) / 8;
+ intptr_t ra = GETPC();
+ unsigned rd = simd_data(desc);
+ uint64_t *d1 = &env->vfp.zregs[rd].d[0];
+ uint64_t *d2 = &env->vfp.zregs[(rd + 1) & 31].d[0];
+ uint64_t *d3 = &env->vfp.zregs[(rd + 2) & 31].d[0];
+ uint8_t *pg = vg;
+
+ for (i = 0; i < oprsz; i += 1) {
+ if (pg[H1(i)] & 1) {
+ cpu_stq_data_ra(env, addr, d1[i], ra);
+ cpu_stq_data_ra(env, addr + 8, d2[i], ra);
+ cpu_stq_data_ra(env, addr + 16, d3[i], ra);
+ }
+ addr += 3 * 8;
+ }
+}
+
+void HELPER(sve_st4dd_r)(CPUARMState *env, void *vg,
+ target_ulong addr, uint32_t desc)
+{
+ intptr_t i, oprsz = simd_oprsz(desc) / 8;
+ intptr_t ra = GETPC();
+ unsigned rd = simd_data(desc);
+ uint64_t *d1 = &env->vfp.zregs[rd].d[0];
+ uint64_t *d2 = &env->vfp.zregs[(rd + 1) & 31].d[0];
+ uint64_t *d3 = &env->vfp.zregs[(rd + 2) & 31].d[0];
+ uint64_t *d4 = &env->vfp.zregs[(rd + 3) & 31].d[0];
+ uint8_t *pg = vg;
+
+ for (i = 0; i < oprsz; i += 1) {
+ if (pg[H1(i)] & 1) {
+ cpu_stq_data_ra(env, addr, d1[i], ra);
+ cpu_stq_data_ra(env, addr + 8, d2[i], ra);
+ cpu_stq_data_ra(env, addr + 16, d3[i], ra);
+ cpu_stq_data_ra(env, addr + 24, d4[i], ra);
+ }
+ addr += 4 * 8;
+ }
+}
+
+/* Loads with a vector index. */
+
+#define DO_LD1_ZPZ_S(NAME, TYPEI, TYPEM, FN) \
+void HELPER(NAME)(CPUARMState *env, void *vd, void *vg, void *vm, \
+ target_ulong base, uint32_t desc) \
+{ \
+ intptr_t i, oprsz = simd_oprsz(desc); \
+ unsigned scale = simd_data(desc); \
+ uintptr_t ra = GETPC(); \
+ for (i = 0; i < oprsz; i++) { \
+ uint16_t pg = *(uint16_t *)(vg + H1_2(i >> 3)); \
+ do { \
+ TYPEM m = 0; \
+ if (pg & 1) { \
+ target_ulong off = *(TYPEI *)(vm + H1_4(i)); \
+ m = FN(env, base + (off << scale), ra); \
+ } \
+ *(uint32_t *)(vd + H1_4(i)) = m; \
+ i += 4, pg >>= 4; \
+ } while (i & 15); \
+ } \
+}
+
+#define DO_LD1_ZPZ_D(NAME, TYPEI, TYPEM, FN) \
+void HELPER(NAME)(CPUARMState *env, void *vd, void *vg, void *vm, \
+ target_ulong base, uint32_t desc) \
+{ \
+ intptr_t i, oprsz = simd_oprsz(desc) / 8; \
+ unsigned scale = simd_data(desc); \
+ uintptr_t ra = GETPC(); \
+ uint64_t *d = vd, *m = vm; uint8_t *pg = vg; \
+ for (i = 0; i < oprsz; i++) { \
+ TYPEM mm = 0; \
+ if (pg[H1(i)] & 1) { \
+ target_ulong off = (TYPEI)m[i]; \
+ mm = FN(env, base + (off << scale), ra); \
+ } \
+ d[i] = mm; \
+ } \
+}
+
+DO_LD1_ZPZ_S(sve_ldbsu_zsu, uint32_t, uint8_t, cpu_ldub_data_ra)
+DO_LD1_ZPZ_S(sve_ldhsu_zsu, uint32_t, uint16_t, cpu_lduw_data_ra)
+DO_LD1_ZPZ_S(sve_ldssu_zsu, uint32_t, uint32_t, cpu_ldl_data_ra)
+DO_LD1_ZPZ_S(sve_ldbss_zsu, uint32_t, int8_t, cpu_ldub_data_ra)
+DO_LD1_ZPZ_S(sve_ldhss_zsu, uint32_t, int16_t, cpu_lduw_data_ra)
+
+DO_LD1_ZPZ_S(sve_ldbsu_zss, int32_t, uint8_t, cpu_ldub_data_ra)
+DO_LD1_ZPZ_S(sve_ldhsu_zss, int32_t, uint16_t, cpu_lduw_data_ra)
+DO_LD1_ZPZ_S(sve_ldssu_zss, int32_t, uint32_t, cpu_ldl_data_ra)
+DO_LD1_ZPZ_S(sve_ldbss_zss, int32_t, int8_t, cpu_ldub_data_ra)
+DO_LD1_ZPZ_S(sve_ldhss_zss, int32_t, int16_t, cpu_lduw_data_ra)
+
+DO_LD1_ZPZ_D(sve_ldbdu_zsu, uint32_t, uint8_t, cpu_ldub_data_ra)
+DO_LD1_ZPZ_D(sve_ldhdu_zsu, uint32_t, uint16_t, cpu_lduw_data_ra)
+DO_LD1_ZPZ_D(sve_ldsdu_zsu, uint32_t, uint32_t, cpu_ldl_data_ra)
+DO_LD1_ZPZ_D(sve_ldddu_zsu, uint32_t, uint64_t, cpu_ldq_data_ra)
+DO_LD1_ZPZ_D(sve_ldbds_zsu, uint32_t, int8_t, cpu_ldub_data_ra)
+DO_LD1_ZPZ_D(sve_ldhds_zsu, uint32_t, int16_t, cpu_lduw_data_ra)
+DO_LD1_ZPZ_D(sve_ldsds_zsu, uint32_t, int32_t, cpu_ldl_data_ra)
+
+DO_LD1_ZPZ_D(sve_ldbdu_zss, int32_t, uint8_t, cpu_ldub_data_ra)
+DO_LD1_ZPZ_D(sve_ldhdu_zss, int32_t, uint16_t, cpu_lduw_data_ra)
+DO_LD1_ZPZ_D(sve_ldsdu_zss, int32_t, uint32_t, cpu_ldl_data_ra)
+DO_LD1_ZPZ_D(sve_ldddu_zss, int32_t, uint64_t, cpu_ldq_data_ra)
+DO_LD1_ZPZ_D(sve_ldbds_zss, int32_t, int8_t, cpu_ldub_data_ra)
+DO_LD1_ZPZ_D(sve_ldhds_zss, int32_t, int16_t, cpu_lduw_data_ra)
+DO_LD1_ZPZ_D(sve_ldsds_zss, int32_t, int32_t, cpu_ldl_data_ra)
+
+DO_LD1_ZPZ_D(sve_ldbdu_zd, uint64_t, uint8_t, cpu_ldub_data_ra)
+DO_LD1_ZPZ_D(sve_ldhdu_zd, uint64_t, uint16_t, cpu_lduw_data_ra)
+DO_LD1_ZPZ_D(sve_ldsdu_zd, uint64_t, uint32_t, cpu_ldl_data_ra)
+DO_LD1_ZPZ_D(sve_ldddu_zd, uint64_t, uint64_t, cpu_ldq_data_ra)
+DO_LD1_ZPZ_D(sve_ldbds_zd, uint64_t, int8_t, cpu_ldub_data_ra)
+DO_LD1_ZPZ_D(sve_ldhds_zd, uint64_t, int16_t, cpu_lduw_data_ra)
+DO_LD1_ZPZ_D(sve_ldsds_zd, uint64_t, int32_t, cpu_ldl_data_ra)
+
+/* First fault loads with a vector index. */
+
+#ifdef CONFIG_USER_ONLY
+
+#define DO_LDFF1_ZPZ(NAME, TYPEE, TYPEI, TYPEM, FN, H) \
+void HELPER(NAME)(CPUARMState *env, void *vd, void *vg, void *vm, \
+ target_ulong base, uint32_t desc) \
+{ \
+ intptr_t i, oprsz = simd_oprsz(desc); \
+ unsigned scale = simd_data(desc); \
+ uintptr_t ra = GETPC(); \
+ bool first = true; \
+ mmap_lock(); \
+ for (i = 0; i < oprsz; i++) { \
+ uint16_t pg = *(uint16_t *)(vg + H1_2(i >> 3)); \
+ do { \
+ TYPEM m = 0; \
+ if (pg & 1) { \
+ target_ulong off = *(TYPEI *)(vm + H(i)); \
+ target_ulong addr = base + (off << scale); \
+ if (!first && \
+ page_check_range(addr, sizeof(TYPEM), PAGE_READ)) { \
+ record_fault(env, i, oprsz); \
+ goto exit; \
+ } \
+ m = FN(env, addr, ra); \
+ first = false; \
+ } \
+ *(TYPEE *)(vd + H(i)) = m; \
+ i += sizeof(TYPEE), pg >>= sizeof(TYPEE); \
+ } while (i & 15); \
+ } \
+ exit: \
+ mmap_unlock(); \
+}
+
+#else
+
+#define DO_LDFF1_ZPZ(NAME, TYPEE, TYPEI, TYPEM, FN, H) \
+void HELPER(NAME)(CPUARMState *env, void *vd, void *vg, void *vm, \
+ target_ulong base, uint32_t desc) \
+{ \
+ g_assert_not_reached(); \
+}
+
+#endif
+
+#define DO_LDFF1_ZPZ_S(NAME, TYPEI, TYPEM, FN) \
+ DO_LDFF1_ZPZ(NAME, uint32_t, TYPEI, TYPEM, FN, H1_4)
+#define DO_LDFF1_ZPZ_D(NAME, TYPEI, TYPEM, FN) \
+ DO_LDFF1_ZPZ(NAME, uint64_t, TYPEI, TYPEM, FN, )
+
+DO_LDFF1_ZPZ_S(sve_ldffbsu_zsu, uint32_t, uint8_t, cpu_ldub_data_ra)
+DO_LDFF1_ZPZ_S(sve_ldffhsu_zsu, uint32_t, uint16_t, cpu_lduw_data_ra)
+DO_LDFF1_ZPZ_S(sve_ldffssu_zsu, uint32_t, uint32_t, cpu_ldl_data_ra)
+DO_LDFF1_ZPZ_S(sve_ldffbss_zsu, uint32_t, int8_t, cpu_ldub_data_ra)
+DO_LDFF1_ZPZ_S(sve_ldffhss_zsu, uint32_t, int16_t, cpu_lduw_data_ra)
+
+DO_LDFF1_ZPZ_S(sve_ldffbsu_zss, int32_t, uint8_t, cpu_ldub_data_ra)
+DO_LDFF1_ZPZ_S(sve_ldffhsu_zss, int32_t, uint16_t, cpu_lduw_data_ra)
+DO_LDFF1_ZPZ_S(sve_ldffssu_zss, int32_t, uint32_t, cpu_ldl_data_ra)
+DO_LDFF1_ZPZ_S(sve_ldffbss_zss, int32_t, int8_t, cpu_ldub_data_ra)
+DO_LDFF1_ZPZ_S(sve_ldffhss_zss, int32_t, int16_t, cpu_lduw_data_ra)
+
+DO_LDFF1_ZPZ_D(sve_ldffbdu_zsu, uint32_t, uint8_t, cpu_ldub_data_ra)
+DO_LDFF1_ZPZ_D(sve_ldffhdu_zsu, uint32_t, uint16_t, cpu_lduw_data_ra)
+DO_LDFF1_ZPZ_D(sve_ldffsdu_zsu, uint32_t, uint32_t, cpu_ldl_data_ra)
+DO_LDFF1_ZPZ_D(sve_ldffddu_zsu, uint32_t, uint64_t, cpu_ldq_data_ra)
+DO_LDFF1_ZPZ_D(sve_ldffbds_zsu, uint32_t, int8_t, cpu_ldub_data_ra)
+DO_LDFF1_ZPZ_D(sve_ldffhds_zsu, uint32_t, int16_t, cpu_lduw_data_ra)
+DO_LDFF1_ZPZ_D(sve_ldffsds_zsu, uint32_t, int32_t, cpu_ldl_data_ra)
+
+DO_LDFF1_ZPZ_D(sve_ldffbdu_zss, int32_t, uint8_t, cpu_ldub_data_ra)
+DO_LDFF1_ZPZ_D(sve_ldffhdu_zss, int32_t, uint16_t, cpu_lduw_data_ra)
+DO_LDFF1_ZPZ_D(sve_ldffsdu_zss, int32_t, uint32_t, cpu_ldl_data_ra)
+DO_LDFF1_ZPZ_D(sve_ldffddu_zss, int32_t, uint64_t, cpu_ldq_data_ra)
+DO_LDFF1_ZPZ_D(sve_ldffbds_zss, int32_t, int8_t, cpu_ldub_data_ra)
+DO_LDFF1_ZPZ_D(sve_ldffhds_zss, int32_t, int16_t, cpu_lduw_data_ra)
+DO_LDFF1_ZPZ_D(sve_ldffsds_zss, int32_t, int32_t, cpu_ldl_data_ra)
+
+DO_LDFF1_ZPZ_D(sve_ldffbdu_zd, uint64_t, uint8_t, cpu_ldub_data_ra)
+DO_LDFF1_ZPZ_D(sve_ldffhdu_zd, uint64_t, uint16_t, cpu_lduw_data_ra)
+DO_LDFF1_ZPZ_D(sve_ldffsdu_zd, uint64_t, uint32_t, cpu_ldl_data_ra)
+DO_LDFF1_ZPZ_D(sve_ldffddu_zd, uint64_t, uint64_t, cpu_ldq_data_ra)
+DO_LDFF1_ZPZ_D(sve_ldffbds_zd, uint64_t, int8_t, cpu_ldub_data_ra)
+DO_LDFF1_ZPZ_D(sve_ldffhds_zd, uint64_t, int16_t, cpu_lduw_data_ra)
+DO_LDFF1_ZPZ_D(sve_ldffsds_zd, uint64_t, int32_t, cpu_ldl_data_ra)
+
+/* Stores with a vector index. */
+
+#define DO_ST1_ZPZ_S(NAME, TYPEI, FN) \
+void HELPER(NAME)(CPUARMState *env, void *vd, void *vg, void *vm, \
+ target_ulong base, uint32_t desc) \
+{ \
+ intptr_t i, oprsz = simd_oprsz(desc); \
+ unsigned scale = simd_data(desc); \
+ uintptr_t ra = GETPC(); \
+ for (i = 0; i < oprsz; ) { \
+ uint16_t pg = *(uint16_t *)(vg + H1_2(i >> 3)); \
+ do { \
+ if (likely(pg & 1)) { \
+ target_ulong off = *(TYPEI *)(vm + H1_4(i)); \
+ uint32_t d = *(uint32_t *)(vd + H1_4(i)); \
+ FN(env, base + (off << scale), d, ra); \
+ } \
+ i += sizeof(uint32_t), pg >>= sizeof(uint32_t); \
+ } while (i & 15); \
+ } \
+}
+
+#define DO_ST1_ZPZ_D(NAME, TYPEI, FN) \
+void HELPER(NAME)(CPUARMState *env, void *vd, void *vg, void *vm, \
+ target_ulong base, uint32_t desc) \
+{ \
+ intptr_t i, oprsz = simd_oprsz(desc) / 8; \
+ unsigned scale = simd_data(desc); \
+ uintptr_t ra = GETPC(); \
+ uint64_t *d = vd, *m = vm; uint8_t *pg = vg; \
+ for (i = 0; i < oprsz; i++) { \
+ if (likely(pg[H1(i)] & 1)) { \
+ target_ulong off = (target_ulong)(TYPEI)m[i] << scale; \
+ FN(env, base + off, d[i], ra); \
+ } \
+ } \
+}
+
+DO_ST1_ZPZ_S(sve_stbs_zsu, uint32_t, cpu_stb_data_ra)
+DO_ST1_ZPZ_S(sve_sths_zsu, uint32_t, cpu_stw_data_ra)
+DO_ST1_ZPZ_S(sve_stss_zsu, uint32_t, cpu_stl_data_ra)
+
+DO_ST1_ZPZ_S(sve_stbs_zss, int32_t, cpu_stb_data_ra)
+DO_ST1_ZPZ_S(sve_sths_zss, int32_t, cpu_stw_data_ra)
+DO_ST1_ZPZ_S(sve_stss_zss, int32_t, cpu_stl_data_ra)
+
+DO_ST1_ZPZ_D(sve_stbd_zsu, uint32_t, cpu_stb_data_ra)
+DO_ST1_ZPZ_D(sve_sthd_zsu, uint32_t, cpu_stw_data_ra)
+DO_ST1_ZPZ_D(sve_stsd_zsu, uint32_t, cpu_stl_data_ra)
+DO_ST1_ZPZ_D(sve_stdd_zsu, uint32_t, cpu_stq_data_ra)
+
+DO_ST1_ZPZ_D(sve_stbd_zss, int32_t, cpu_stb_data_ra)
+DO_ST1_ZPZ_D(sve_sthd_zss, int32_t, cpu_stw_data_ra)
+DO_ST1_ZPZ_D(sve_stsd_zss, int32_t, cpu_stl_data_ra)
+DO_ST1_ZPZ_D(sve_stdd_zss, int32_t, cpu_stq_data_ra)
+
+DO_ST1_ZPZ_D(sve_stbd_zd, uint64_t, cpu_stb_data_ra)
+DO_ST1_ZPZ_D(sve_sthd_zd, uint64_t, cpu_stw_data_ra)
+DO_ST1_ZPZ_D(sve_stsd_zd, uint64_t, cpu_stl_data_ra)
+DO_ST1_ZPZ_D(sve_stdd_zd, uint64_t, cpu_stq_data_ra)
diff --git a/target/arm/translate-a64.c b/target/arm/translate-a64.c
index 8d8a4ce..45a6c2a 100644
--- a/target/arm/translate-a64.c
+++ b/target/arm/translate-a64.c
@@ -640,6 +640,16 @@
vec_full_reg_size(s), gvec_op);
}
+/* Expand a 3-operand operation using an out-of-line helper. */
+static void gen_gvec_op3_ool(DisasContext *s, bool is_q, int rd,
+ int rn, int rm, int data, gen_helper_gvec_3 *fn)
+{
+ tcg_gen_gvec_3_ool(vec_full_reg_offset(s, rd),
+ vec_full_reg_offset(s, rn),
+ vec_full_reg_offset(s, rm),
+ is_q ? 16 : 8, vec_full_reg_size(s), data, fn);
+}
+
/* Expand a 3-operand + env pointer operation using
* an out-of-line helper.
*/
@@ -1623,11 +1633,10 @@
default:
break;
}
- if ((ri->type & ARM_CP_SVE) && !sve_access_check(s)) {
- return;
- }
if ((ri->type & ARM_CP_FPU) && !fp_access_check(s)) {
return;
+ } else if ((ri->type & ARM_CP_SVE) && !sve_access_check(s)) {
+ return;
}
if ((tb_cflags(s->base.tb) & CF_USE_ICOUNT) && (ri->type & ARM_CP_IO)) {
@@ -11336,6 +11345,14 @@
}
feature = ARM_FEATURE_V8_RDM;
break;
+ case 0x02: /* SDOT (vector) */
+ case 0x12: /* UDOT (vector) */
+ if (size != MO_32) {
+ unallocated_encoding(s);
+ return;
+ }
+ feature = ARM_FEATURE_V8_DOTPROD;
+ break;
case 0x8: /* FCMLA, #0 */
case 0x9: /* FCMLA, #90 */
case 0xa: /* FCMLA, #180 */
@@ -11389,6 +11406,11 @@
}
return;
+ case 0x2: /* SDOT / UDOT */
+ gen_gvec_op3_ool(s, is_q, rd, rn, rm, 0,
+ u ? gen_helper_gvec_udot_b : gen_helper_gvec_sdot_b);
+ return;
+
case 0x8: /* FCMLA, #0 */
case 0x9: /* FCMLA, #90 */
case 0xa: /* FCMLA, #180 */
@@ -12568,6 +12590,13 @@
return;
}
break;
+ case 0x0e: /* SDOT */
+ case 0x1e: /* UDOT */
+ if (size != MO_32 || !arm_dc_feature(s, ARM_FEATURE_V8_DOTPROD)) {
+ unallocated_encoding(s);
+ return;
+ }
+ break;
case 0x11: /* FCMLA #0 */
case 0x13: /* FCMLA #90 */
case 0x15: /* FCMLA #180 */
@@ -12665,19 +12694,28 @@
}
switch (16 * u + opcode) {
+ case 0x0e: /* SDOT */
+ case 0x1e: /* UDOT */
+ gen_gvec_op3_ool(s, is_q, rd, rn, rm, index,
+ u ? gen_helper_gvec_udot_idx_b
+ : gen_helper_gvec_sdot_idx_b);
+ return;
case 0x11: /* FCMLA #0 */
case 0x13: /* FCMLA #90 */
case 0x15: /* FCMLA #180 */
case 0x17: /* FCMLA #270 */
- tcg_gen_gvec_3_ptr(vec_full_reg_offset(s, rd),
- vec_full_reg_offset(s, rn),
- vec_reg_offset(s, rm, index, size), fpst,
- is_q ? 16 : 8, vec_full_reg_size(s),
- extract32(insn, 13, 2), /* rot */
- size == MO_64
- ? gen_helper_gvec_fcmlas_idx
- : gen_helper_gvec_fcmlah_idx);
- tcg_temp_free_ptr(fpst);
+ {
+ int rot = extract32(insn, 13, 2);
+ int data = (index << 2) | rot;
+ tcg_gen_gvec_3_ptr(vec_full_reg_offset(s, rd),
+ vec_full_reg_offset(s, rn),
+ vec_full_reg_offset(s, rm), fpst,
+ is_q ? 16 : 8, vec_full_reg_size(s), data,
+ size == MO_64
+ ? gen_helper_gvec_fcmlas_idx
+ : gen_helper_gvec_fcmlah_idx);
+ tcg_temp_free_ptr(fpst);
+ }
return;
}
diff --git a/target/arm/translate-sve.c b/target/arm/translate-sve.c
index 226c975..c080345 100644
--- a/target/arm/translate-sve.c
+++ b/target/arm/translate-sve.c
@@ -32,6 +32,7 @@
#include "exec/log.h"
#include "trace-tcg.h"
#include "translate-a64.h"
+#include "fpu/softfloat.h"
typedef void GVecGen2sFn(unsigned, uint32_t, uint32_t,
@@ -42,6 +43,10 @@
typedef void gen_helper_gvec_flags_4(TCGv_i32, TCGv_ptr, TCGv_ptr,
TCGv_ptr, TCGv_ptr, TCGv_i32);
+typedef void gen_helper_gvec_mem(TCGv_env, TCGv_ptr, TCGv_i64, TCGv_i32);
+typedef void gen_helper_gvec_mem_scatter(TCGv_env, TCGv_ptr, TCGv_ptr,
+ TCGv_ptr, TCGv_i64, TCGv_i32);
+
/*
* Helpers for extracting complex instruction fields.
*/
@@ -82,6 +87,15 @@
return (uint8_t)x << (x & 0x100 ? 8 : 0);
}
+/* Convert a 2-bit memory size (msz) to a 4-bit data type (dtype)
+ * with unsigned data. C.f. SVE Memory Contiguous Load Group.
+ */
+static inline int msz_dtype(int msz)
+{
+ static const uint8_t dtype[4] = { 0, 5, 10, 15 };
+ return dtype[msz];
+}
+
/*
* Include the generated decoder.
*/
@@ -337,6 +351,23 @@
return true;
}
+/* Select active elememnts from Zn and inactive elements from Zm,
+ * storing the result in Zd.
+ */
+static void do_sel_z(DisasContext *s, int rd, int rn, int rm, int pg, int esz)
+{
+ static gen_helper_gvec_4 * const fns[4] = {
+ gen_helper_sve_sel_zpzz_b, gen_helper_sve_sel_zpzz_h,
+ gen_helper_sve_sel_zpzz_s, gen_helper_sve_sel_zpzz_d
+ };
+ unsigned vsz = vec_full_reg_size(s);
+ tcg_gen_gvec_4_ool(vec_full_reg_offset(s, rd),
+ vec_full_reg_offset(s, rn),
+ vec_full_reg_offset(s, rm),
+ pred_full_reg_offset(s, pg),
+ vsz, vsz, 0, fns[esz]);
+}
+
#define DO_ZPZZ(NAME, name) \
static bool trans_##NAME##_zpzz(DisasContext *s, arg_rprr_esz *a, \
uint32_t insn) \
@@ -387,7 +418,13 @@
return do_zpzz_ool(s, a, fns[a->esz]);
}
-DO_ZPZZ(SEL, sel)
+static bool trans_SEL_zpzz(DisasContext *s, arg_rprr_esz *a, uint32_t insn)
+{
+ if (sve_access_check(s)) {
+ do_sel_z(s, a->rd, a->rn, a->rm, a->pg, a->esz);
+ }
+ return true;
+}
#undef DO_ZPZZ
@@ -595,6 +632,20 @@
return true;
}
+/* Copy Zn into Zd, storing zeros into inactive elements. */
+static void do_movz_zpz(DisasContext *s, int rd, int rn, int pg, int esz)
+{
+ static gen_helper_gvec_3 * const fns[4] = {
+ gen_helper_sve_movz_b, gen_helper_sve_movz_h,
+ gen_helper_sve_movz_s, gen_helper_sve_movz_d,
+ };
+ unsigned vsz = vec_full_reg_size(s);
+ tcg_gen_gvec_3_ool(vec_full_reg_offset(s, rd),
+ vec_full_reg_offset(s, rn),
+ pred_full_reg_offset(s, pg),
+ vsz, vsz, 0, fns[esz]);
+}
+
static bool do_zpzi_ool(DisasContext *s, arg_rpri_esz *a,
gen_helper_gvec_3 *fn)
{
@@ -3372,6 +3423,310 @@
#undef DO_ZZI
+static bool trans_DOT_zzz(DisasContext *s, arg_DOT_zzz *a, uint32_t insn)
+{
+ static gen_helper_gvec_3 * const fns[2][2] = {
+ { gen_helper_gvec_sdot_b, gen_helper_gvec_sdot_h },
+ { gen_helper_gvec_udot_b, gen_helper_gvec_udot_h }
+ };
+
+ if (sve_access_check(s)) {
+ unsigned vsz = vec_full_reg_size(s);
+ tcg_gen_gvec_3_ool(vec_full_reg_offset(s, a->rd),
+ vec_full_reg_offset(s, a->rn),
+ vec_full_reg_offset(s, a->rm),
+ vsz, vsz, 0, fns[a->u][a->sz]);
+ }
+ return true;
+}
+
+static bool trans_DOT_zzx(DisasContext *s, arg_DOT_zzx *a, uint32_t insn)
+{
+ static gen_helper_gvec_3 * const fns[2][2] = {
+ { gen_helper_gvec_sdot_idx_b, gen_helper_gvec_sdot_idx_h },
+ { gen_helper_gvec_udot_idx_b, gen_helper_gvec_udot_idx_h }
+ };
+
+ if (sve_access_check(s)) {
+ unsigned vsz = vec_full_reg_size(s);
+ tcg_gen_gvec_3_ool(vec_full_reg_offset(s, a->rd),
+ vec_full_reg_offset(s, a->rn),
+ vec_full_reg_offset(s, a->rm),
+ vsz, vsz, a->index, fns[a->u][a->sz]);
+ }
+ return true;
+}
+
+
+/*
+ *** SVE Floating Point Multiply-Add Indexed Group
+ */
+
+static bool trans_FMLA_zzxz(DisasContext *s, arg_FMLA_zzxz *a, uint32_t insn)
+{
+ static gen_helper_gvec_4_ptr * const fns[3] = {
+ gen_helper_gvec_fmla_idx_h,
+ gen_helper_gvec_fmla_idx_s,
+ gen_helper_gvec_fmla_idx_d,
+ };
+
+ if (sve_access_check(s)) {
+ unsigned vsz = vec_full_reg_size(s);
+ TCGv_ptr status = get_fpstatus_ptr(a->esz == MO_16);
+ tcg_gen_gvec_4_ptr(vec_full_reg_offset(s, a->rd),
+ vec_full_reg_offset(s, a->rn),
+ vec_full_reg_offset(s, a->rm),
+ vec_full_reg_offset(s, a->ra),
+ status, vsz, vsz, (a->index << 1) | a->sub,
+ fns[a->esz - 1]);
+ tcg_temp_free_ptr(status);
+ }
+ return true;
+}
+
+/*
+ *** SVE Floating Point Multiply Indexed Group
+ */
+
+static bool trans_FMUL_zzx(DisasContext *s, arg_FMUL_zzx *a, uint32_t insn)
+{
+ static gen_helper_gvec_3_ptr * const fns[3] = {
+ gen_helper_gvec_fmul_idx_h,
+ gen_helper_gvec_fmul_idx_s,
+ gen_helper_gvec_fmul_idx_d,
+ };
+
+ if (sve_access_check(s)) {
+ unsigned vsz = vec_full_reg_size(s);
+ TCGv_ptr status = get_fpstatus_ptr(a->esz == MO_16);
+ tcg_gen_gvec_3_ptr(vec_full_reg_offset(s, a->rd),
+ vec_full_reg_offset(s, a->rn),
+ vec_full_reg_offset(s, a->rm),
+ status, vsz, vsz, a->index, fns[a->esz - 1]);
+ tcg_temp_free_ptr(status);
+ }
+ return true;
+}
+
+/*
+ *** SVE Floating Point Fast Reduction Group
+ */
+
+typedef void gen_helper_fp_reduce(TCGv_i64, TCGv_ptr, TCGv_ptr,
+ TCGv_ptr, TCGv_i32);
+
+static void do_reduce(DisasContext *s, arg_rpr_esz *a,
+ gen_helper_fp_reduce *fn)
+{
+ unsigned vsz = vec_full_reg_size(s);
+ unsigned p2vsz = pow2ceil(vsz);
+ TCGv_i32 t_desc = tcg_const_i32(simd_desc(vsz, p2vsz, 0));
+ TCGv_ptr t_zn, t_pg, status;
+ TCGv_i64 temp;
+
+ temp = tcg_temp_new_i64();
+ t_zn = tcg_temp_new_ptr();
+ t_pg = tcg_temp_new_ptr();
+
+ tcg_gen_addi_ptr(t_zn, cpu_env, vec_full_reg_offset(s, a->rn));
+ tcg_gen_addi_ptr(t_pg, cpu_env, pred_full_reg_offset(s, a->pg));
+ status = get_fpstatus_ptr(a->esz == MO_16);
+
+ fn(temp, t_zn, t_pg, status, t_desc);
+ tcg_temp_free_ptr(t_zn);
+ tcg_temp_free_ptr(t_pg);
+ tcg_temp_free_ptr(status);
+ tcg_temp_free_i32(t_desc);
+
+ write_fp_dreg(s, a->rd, temp);
+ tcg_temp_free_i64(temp);
+}
+
+#define DO_VPZ(NAME, name) \
+static bool trans_##NAME(DisasContext *s, arg_rpr_esz *a, uint32_t insn) \
+{ \
+ static gen_helper_fp_reduce * const fns[3] = { \
+ gen_helper_sve_##name##_h, \
+ gen_helper_sve_##name##_s, \
+ gen_helper_sve_##name##_d, \
+ }; \
+ if (a->esz == 0) { \
+ return false; \
+ } \
+ if (sve_access_check(s)) { \
+ do_reduce(s, a, fns[a->esz - 1]); \
+ } \
+ return true; \
+}
+
+DO_VPZ(FADDV, faddv)
+DO_VPZ(FMINNMV, fminnmv)
+DO_VPZ(FMAXNMV, fmaxnmv)
+DO_VPZ(FMINV, fminv)
+DO_VPZ(FMAXV, fmaxv)
+
+/*
+ *** SVE Floating Point Unary Operations - Unpredicated Group
+ */
+
+static void do_zz_fp(DisasContext *s, arg_rr_esz *a, gen_helper_gvec_2_ptr *fn)
+{
+ unsigned vsz = vec_full_reg_size(s);
+ TCGv_ptr status = get_fpstatus_ptr(a->esz == MO_16);
+
+ tcg_gen_gvec_2_ptr(vec_full_reg_offset(s, a->rd),
+ vec_full_reg_offset(s, a->rn),
+ status, vsz, vsz, 0, fn);
+ tcg_temp_free_ptr(status);
+}
+
+static bool trans_FRECPE(DisasContext *s, arg_rr_esz *a, uint32_t insn)
+{
+ static gen_helper_gvec_2_ptr * const fns[3] = {
+ gen_helper_gvec_frecpe_h,
+ gen_helper_gvec_frecpe_s,
+ gen_helper_gvec_frecpe_d,
+ };
+ if (a->esz == 0) {
+ return false;
+ }
+ if (sve_access_check(s)) {
+ do_zz_fp(s, a, fns[a->esz - 1]);
+ }
+ return true;
+}
+
+static bool trans_FRSQRTE(DisasContext *s, arg_rr_esz *a, uint32_t insn)
+{
+ static gen_helper_gvec_2_ptr * const fns[3] = {
+ gen_helper_gvec_frsqrte_h,
+ gen_helper_gvec_frsqrte_s,
+ gen_helper_gvec_frsqrte_d,
+ };
+ if (a->esz == 0) {
+ return false;
+ }
+ if (sve_access_check(s)) {
+ do_zz_fp(s, a, fns[a->esz - 1]);
+ }
+ return true;
+}
+
+/*
+ *** SVE Floating Point Compare with Zero Group
+ */
+
+static void do_ppz_fp(DisasContext *s, arg_rpr_esz *a,
+ gen_helper_gvec_3_ptr *fn)
+{
+ unsigned vsz = vec_full_reg_size(s);
+ TCGv_ptr status = get_fpstatus_ptr(a->esz == MO_16);
+
+ tcg_gen_gvec_3_ptr(pred_full_reg_offset(s, a->rd),
+ vec_full_reg_offset(s, a->rn),
+ pred_full_reg_offset(s, a->pg),
+ status, vsz, vsz, 0, fn);
+ tcg_temp_free_ptr(status);
+}
+
+#define DO_PPZ(NAME, name) \
+static bool trans_##NAME(DisasContext *s, arg_rpr_esz *a, uint32_t insn) \
+{ \
+ static gen_helper_gvec_3_ptr * const fns[3] = { \
+ gen_helper_sve_##name##_h, \
+ gen_helper_sve_##name##_s, \
+ gen_helper_sve_##name##_d, \
+ }; \
+ if (a->esz == 0) { \
+ return false; \
+ } \
+ if (sve_access_check(s)) { \
+ do_ppz_fp(s, a, fns[a->esz - 1]); \
+ } \
+ return true; \
+}
+
+DO_PPZ(FCMGE_ppz0, fcmge0)
+DO_PPZ(FCMGT_ppz0, fcmgt0)
+DO_PPZ(FCMLE_ppz0, fcmle0)
+DO_PPZ(FCMLT_ppz0, fcmlt0)
+DO_PPZ(FCMEQ_ppz0, fcmeq0)
+DO_PPZ(FCMNE_ppz0, fcmne0)
+
+#undef DO_PPZ
+
+/*
+ *** SVE floating-point trig multiply-add coefficient
+ */
+
+static bool trans_FTMAD(DisasContext *s, arg_FTMAD *a, uint32_t insn)
+{
+ static gen_helper_gvec_3_ptr * const fns[3] = {
+ gen_helper_sve_ftmad_h,
+ gen_helper_sve_ftmad_s,
+ gen_helper_sve_ftmad_d,
+ };
+
+ if (a->esz == 0) {
+ return false;
+ }
+ if (sve_access_check(s)) {
+ unsigned vsz = vec_full_reg_size(s);
+ TCGv_ptr status = get_fpstatus_ptr(a->esz == MO_16);
+ tcg_gen_gvec_3_ptr(vec_full_reg_offset(s, a->rd),
+ vec_full_reg_offset(s, a->rn),
+ vec_full_reg_offset(s, a->rm),
+ status, vsz, vsz, a->imm, fns[a->esz - 1]);
+ tcg_temp_free_ptr(status);
+ }
+ return true;
+}
+
+/*
+ *** SVE Floating Point Accumulating Reduction Group
+ */
+
+static bool trans_FADDA(DisasContext *s, arg_rprr_esz *a, uint32_t insn)
+{
+ typedef void fadda_fn(TCGv_i64, TCGv_i64, TCGv_ptr,
+ TCGv_ptr, TCGv_ptr, TCGv_i32);
+ static fadda_fn * const fns[3] = {
+ gen_helper_sve_fadda_h,
+ gen_helper_sve_fadda_s,
+ gen_helper_sve_fadda_d,
+ };
+ unsigned vsz = vec_full_reg_size(s);
+ TCGv_ptr t_rm, t_pg, t_fpst;
+ TCGv_i64 t_val;
+ TCGv_i32 t_desc;
+
+ if (a->esz == 0) {
+ return false;
+ }
+ if (!sve_access_check(s)) {
+ return true;
+ }
+
+ t_val = load_esz(cpu_env, vec_reg_offset(s, a->rn, 0, a->esz), a->esz);
+ t_rm = tcg_temp_new_ptr();
+ t_pg = tcg_temp_new_ptr();
+ tcg_gen_addi_ptr(t_rm, cpu_env, vec_full_reg_offset(s, a->rm));
+ tcg_gen_addi_ptr(t_pg, cpu_env, pred_full_reg_offset(s, a->pg));
+ t_fpst = get_fpstatus_ptr(a->esz == MO_16);
+ t_desc = tcg_const_i32(simd_desc(vsz, vsz, 0));
+
+ fns[a->esz - 1](t_val, t_val, t_rm, t_pg, t_fpst, t_desc);
+
+ tcg_temp_free_i32(t_desc);
+ tcg_temp_free_ptr(t_fpst);
+ tcg_temp_free_ptr(t_pg);
+ tcg_temp_free_ptr(t_rm);
+
+ write_fp_dreg(s, a->rd, t_val);
+ tcg_temp_free_i64(t_val);
+ return true;
+}
+
/*
*** SVE Floating Point Arithmetic - Unpredicated Group
*/
@@ -3415,6 +3770,592 @@
#undef DO_FP3
/*
+ *** SVE Floating Point Arithmetic - Predicated Group
+ */
+
+static bool do_zpzz_fp(DisasContext *s, arg_rprr_esz *a,
+ gen_helper_gvec_4_ptr *fn)
+{
+ if (fn == NULL) {
+ return false;
+ }
+ if (sve_access_check(s)) {
+ unsigned vsz = vec_full_reg_size(s);
+ TCGv_ptr status = get_fpstatus_ptr(a->esz == MO_16);
+ tcg_gen_gvec_4_ptr(vec_full_reg_offset(s, a->rd),
+ vec_full_reg_offset(s, a->rn),
+ vec_full_reg_offset(s, a->rm),
+ pred_full_reg_offset(s, a->pg),
+ status, vsz, vsz, 0, fn);
+ tcg_temp_free_ptr(status);
+ }
+ return true;
+}
+
+#define DO_FP3(NAME, name) \
+static bool trans_##NAME(DisasContext *s, arg_rprr_esz *a, uint32_t insn) \
+{ \
+ static gen_helper_gvec_4_ptr * const fns[4] = { \
+ NULL, gen_helper_sve_##name##_h, \
+ gen_helper_sve_##name##_s, gen_helper_sve_##name##_d \
+ }; \
+ return do_zpzz_fp(s, a, fns[a->esz]); \
+}
+
+DO_FP3(FADD_zpzz, fadd)
+DO_FP3(FSUB_zpzz, fsub)
+DO_FP3(FMUL_zpzz, fmul)
+DO_FP3(FMIN_zpzz, fmin)
+DO_FP3(FMAX_zpzz, fmax)
+DO_FP3(FMINNM_zpzz, fminnum)
+DO_FP3(FMAXNM_zpzz, fmaxnum)
+DO_FP3(FABD, fabd)
+DO_FP3(FSCALE, fscalbn)
+DO_FP3(FDIV, fdiv)
+DO_FP3(FMULX, fmulx)
+
+#undef DO_FP3
+
+typedef void gen_helper_sve_fp2scalar(TCGv_ptr, TCGv_ptr, TCGv_ptr,
+ TCGv_i64, TCGv_ptr, TCGv_i32);
+
+static void do_fp_scalar(DisasContext *s, int zd, int zn, int pg, bool is_fp16,
+ TCGv_i64 scalar, gen_helper_sve_fp2scalar *fn)
+{
+ unsigned vsz = vec_full_reg_size(s);
+ TCGv_ptr t_zd, t_zn, t_pg, status;
+ TCGv_i32 desc;
+
+ t_zd = tcg_temp_new_ptr();
+ t_zn = tcg_temp_new_ptr();
+ t_pg = tcg_temp_new_ptr();
+ tcg_gen_addi_ptr(t_zd, cpu_env, vec_full_reg_offset(s, zd));
+ tcg_gen_addi_ptr(t_zn, cpu_env, vec_full_reg_offset(s, zn));
+ tcg_gen_addi_ptr(t_pg, cpu_env, pred_full_reg_offset(s, pg));
+
+ status = get_fpstatus_ptr(is_fp16);
+ desc = tcg_const_i32(simd_desc(vsz, vsz, 0));
+ fn(t_zd, t_zn, t_pg, scalar, status, desc);
+
+ tcg_temp_free_i32(desc);
+ tcg_temp_free_ptr(status);
+ tcg_temp_free_ptr(t_pg);
+ tcg_temp_free_ptr(t_zn);
+ tcg_temp_free_ptr(t_zd);
+}
+
+static void do_fp_imm(DisasContext *s, arg_rpri_esz *a, uint64_t imm,
+ gen_helper_sve_fp2scalar *fn)
+{
+ TCGv_i64 temp = tcg_const_i64(imm);
+ do_fp_scalar(s, a->rd, a->rn, a->pg, a->esz == MO_16, temp, fn);
+ tcg_temp_free_i64(temp);
+}
+
+#define DO_FP_IMM(NAME, name, const0, const1) \
+static bool trans_##NAME##_zpzi(DisasContext *s, arg_rpri_esz *a, \
+ uint32_t insn) \
+{ \
+ static gen_helper_sve_fp2scalar * const fns[3] = { \
+ gen_helper_sve_##name##_h, \
+ gen_helper_sve_##name##_s, \
+ gen_helper_sve_##name##_d \
+ }; \
+ static uint64_t const val[3][2] = { \
+ { float16_##const0, float16_##const1 }, \
+ { float32_##const0, float32_##const1 }, \
+ { float64_##const0, float64_##const1 }, \
+ }; \
+ if (a->esz == 0) { \
+ return false; \
+ } \
+ if (sve_access_check(s)) { \
+ do_fp_imm(s, a, val[a->esz - 1][a->imm], fns[a->esz - 1]); \
+ } \
+ return true; \
+}
+
+#define float16_two make_float16(0x4000)
+#define float32_two make_float32(0x40000000)
+#define float64_two make_float64(0x4000000000000000ULL)
+
+DO_FP_IMM(FADD, fadds, half, one)
+DO_FP_IMM(FSUB, fsubs, half, one)
+DO_FP_IMM(FMUL, fmuls, half, two)
+DO_FP_IMM(FSUBR, fsubrs, half, one)
+DO_FP_IMM(FMAXNM, fmaxnms, zero, one)
+DO_FP_IMM(FMINNM, fminnms, zero, one)
+DO_FP_IMM(FMAX, fmaxs, zero, one)
+DO_FP_IMM(FMIN, fmins, zero, one)
+
+#undef DO_FP_IMM
+
+static bool do_fp_cmp(DisasContext *s, arg_rprr_esz *a,
+ gen_helper_gvec_4_ptr *fn)
+{
+ if (fn == NULL) {
+ return false;
+ }
+ if (sve_access_check(s)) {
+ unsigned vsz = vec_full_reg_size(s);
+ TCGv_ptr status = get_fpstatus_ptr(a->esz == MO_16);
+ tcg_gen_gvec_4_ptr(pred_full_reg_offset(s, a->rd),
+ vec_full_reg_offset(s, a->rn),
+ vec_full_reg_offset(s, a->rm),
+ pred_full_reg_offset(s, a->pg),
+ status, vsz, vsz, 0, fn);
+ tcg_temp_free_ptr(status);
+ }
+ return true;
+}
+
+#define DO_FPCMP(NAME, name) \
+static bool trans_##NAME##_ppzz(DisasContext *s, arg_rprr_esz *a, \
+ uint32_t insn) \
+{ \
+ static gen_helper_gvec_4_ptr * const fns[4] = { \
+ NULL, gen_helper_sve_##name##_h, \
+ gen_helper_sve_##name##_s, gen_helper_sve_##name##_d \
+ }; \
+ return do_fp_cmp(s, a, fns[a->esz]); \
+}
+
+DO_FPCMP(FCMGE, fcmge)
+DO_FPCMP(FCMGT, fcmgt)
+DO_FPCMP(FCMEQ, fcmeq)
+DO_FPCMP(FCMNE, fcmne)
+DO_FPCMP(FCMUO, fcmuo)
+DO_FPCMP(FACGE, facge)
+DO_FPCMP(FACGT, facgt)
+
+#undef DO_FPCMP
+
+static bool trans_FCADD(DisasContext *s, arg_FCADD *a, uint32_t insn)
+{
+ static gen_helper_gvec_4_ptr * const fns[3] = {
+ gen_helper_sve_fcadd_h,
+ gen_helper_sve_fcadd_s,
+ gen_helper_sve_fcadd_d
+ };
+
+ if (a->esz == 0) {
+ return false;
+ }
+ if (sve_access_check(s)) {
+ unsigned vsz = vec_full_reg_size(s);
+ TCGv_ptr status = get_fpstatus_ptr(a->esz == MO_16);
+ tcg_gen_gvec_4_ptr(vec_full_reg_offset(s, a->rd),
+ vec_full_reg_offset(s, a->rn),
+ vec_full_reg_offset(s, a->rm),
+ pred_full_reg_offset(s, a->pg),
+ status, vsz, vsz, a->rot, fns[a->esz - 1]);
+ tcg_temp_free_ptr(status);
+ }
+ return true;
+}
+
+typedef void gen_helper_sve_fmla(TCGv_env, TCGv_ptr, TCGv_i32);
+
+static bool do_fmla(DisasContext *s, arg_rprrr_esz *a, gen_helper_sve_fmla *fn)
+{
+ if (fn == NULL) {
+ return false;
+ }
+ if (!sve_access_check(s)) {
+ return true;
+ }
+
+ unsigned vsz = vec_full_reg_size(s);
+ unsigned desc;
+ TCGv_i32 t_desc;
+ TCGv_ptr pg = tcg_temp_new_ptr();
+
+ /* We would need 7 operands to pass these arguments "properly".
+ * So we encode all the register numbers into the descriptor.
+ */
+ desc = deposit32(a->rd, 5, 5, a->rn);
+ desc = deposit32(desc, 10, 5, a->rm);
+ desc = deposit32(desc, 15, 5, a->ra);
+ desc = simd_desc(vsz, vsz, desc);
+
+ t_desc = tcg_const_i32(desc);
+ tcg_gen_addi_ptr(pg, cpu_env, pred_full_reg_offset(s, a->pg));
+ fn(cpu_env, pg, t_desc);
+ tcg_temp_free_i32(t_desc);
+ tcg_temp_free_ptr(pg);
+ return true;
+}
+
+#define DO_FMLA(NAME, name) \
+static bool trans_##NAME(DisasContext *s, arg_rprrr_esz *a, uint32_t insn) \
+{ \
+ static gen_helper_sve_fmla * const fns[4] = { \
+ NULL, gen_helper_sve_##name##_h, \
+ gen_helper_sve_##name##_s, gen_helper_sve_##name##_d \
+ }; \
+ return do_fmla(s, a, fns[a->esz]); \
+}
+
+DO_FMLA(FMLA_zpzzz, fmla_zpzzz)
+DO_FMLA(FMLS_zpzzz, fmls_zpzzz)
+DO_FMLA(FNMLA_zpzzz, fnmla_zpzzz)
+DO_FMLA(FNMLS_zpzzz, fnmls_zpzzz)
+
+#undef DO_FMLA
+
+static bool trans_FCMLA_zpzzz(DisasContext *s,
+ arg_FCMLA_zpzzz *a, uint32_t insn)
+{
+ static gen_helper_sve_fmla * const fns[3] = {
+ gen_helper_sve_fcmla_zpzzz_h,
+ gen_helper_sve_fcmla_zpzzz_s,
+ gen_helper_sve_fcmla_zpzzz_d,
+ };
+
+ if (a->esz == 0) {
+ return false;
+ }
+ if (sve_access_check(s)) {
+ unsigned vsz = vec_full_reg_size(s);
+ unsigned desc;
+ TCGv_i32 t_desc;
+ TCGv_ptr pg = tcg_temp_new_ptr();
+
+ /* We would need 7 operands to pass these arguments "properly".
+ * So we encode all the register numbers into the descriptor.
+ */
+ desc = deposit32(a->rd, 5, 5, a->rn);
+ desc = deposit32(desc, 10, 5, a->rm);
+ desc = deposit32(desc, 15, 5, a->ra);
+ desc = deposit32(desc, 20, 2, a->rot);
+ desc = sextract32(desc, 0, 22);
+ desc = simd_desc(vsz, vsz, desc);
+
+ t_desc = tcg_const_i32(desc);
+ tcg_gen_addi_ptr(pg, cpu_env, pred_full_reg_offset(s, a->pg));
+ fns[a->esz - 1](cpu_env, pg, t_desc);
+ tcg_temp_free_i32(t_desc);
+ tcg_temp_free_ptr(pg);
+ }
+ return true;
+}
+
+static bool trans_FCMLA_zzxz(DisasContext *s, arg_FCMLA_zzxz *a, uint32_t insn)
+{
+ static gen_helper_gvec_3_ptr * const fns[2] = {
+ gen_helper_gvec_fcmlah_idx,
+ gen_helper_gvec_fcmlas_idx,
+ };
+
+ tcg_debug_assert(a->esz == 1 || a->esz == 2);
+ tcg_debug_assert(a->rd == a->ra);
+ if (sve_access_check(s)) {
+ unsigned vsz = vec_full_reg_size(s);
+ TCGv_ptr status = get_fpstatus_ptr(a->esz == MO_16);
+ tcg_gen_gvec_3_ptr(vec_full_reg_offset(s, a->rd),
+ vec_full_reg_offset(s, a->rn),
+ vec_full_reg_offset(s, a->rm),
+ status, vsz, vsz,
+ a->index * 4 + a->rot,
+ fns[a->esz - 1]);
+ tcg_temp_free_ptr(status);
+ }
+ return true;
+}
+
+/*
+ *** SVE Floating Point Unary Operations Predicated Group
+ */
+
+static bool do_zpz_ptr(DisasContext *s, int rd, int rn, int pg,
+ bool is_fp16, gen_helper_gvec_3_ptr *fn)
+{
+ if (sve_access_check(s)) {
+ unsigned vsz = vec_full_reg_size(s);
+ TCGv_ptr status = get_fpstatus_ptr(is_fp16);
+ tcg_gen_gvec_3_ptr(vec_full_reg_offset(s, rd),
+ vec_full_reg_offset(s, rn),
+ pred_full_reg_offset(s, pg),
+ status, vsz, vsz, 0, fn);
+ tcg_temp_free_ptr(status);
+ }
+ return true;
+}
+
+static bool trans_FCVT_sh(DisasContext *s, arg_rpr_esz *a, uint32_t insn)
+{
+ return do_zpz_ptr(s, a->rd, a->rn, a->pg, true, gen_helper_sve_fcvt_sh);
+}
+
+static bool trans_FCVT_hs(DisasContext *s, arg_rpr_esz *a, uint32_t insn)
+{
+ return do_zpz_ptr(s, a->rd, a->rn, a->pg, false, gen_helper_sve_fcvt_hs);
+}
+
+static bool trans_FCVT_dh(DisasContext *s, arg_rpr_esz *a, uint32_t insn)
+{
+ return do_zpz_ptr(s, a->rd, a->rn, a->pg, true, gen_helper_sve_fcvt_dh);
+}
+
+static bool trans_FCVT_hd(DisasContext *s, arg_rpr_esz *a, uint32_t insn)
+{
+ return do_zpz_ptr(s, a->rd, a->rn, a->pg, false, gen_helper_sve_fcvt_hd);
+}
+
+static bool trans_FCVT_ds(DisasContext *s, arg_rpr_esz *a, uint32_t insn)
+{
+ return do_zpz_ptr(s, a->rd, a->rn, a->pg, false, gen_helper_sve_fcvt_ds);
+}
+
+static bool trans_FCVT_sd(DisasContext *s, arg_rpr_esz *a, uint32_t insn)
+{
+ return do_zpz_ptr(s, a->rd, a->rn, a->pg, false, gen_helper_sve_fcvt_sd);
+}
+
+static bool trans_FCVTZS_hh(DisasContext *s, arg_rpr_esz *a, uint32_t insn)
+{
+ return do_zpz_ptr(s, a->rd, a->rn, a->pg, true, gen_helper_sve_fcvtzs_hh);
+}
+
+static bool trans_FCVTZU_hh(DisasContext *s, arg_rpr_esz *a, uint32_t insn)
+{
+ return do_zpz_ptr(s, a->rd, a->rn, a->pg, true, gen_helper_sve_fcvtzu_hh);
+}
+
+static bool trans_FCVTZS_hs(DisasContext *s, arg_rpr_esz *a, uint32_t insn)
+{
+ return do_zpz_ptr(s, a->rd, a->rn, a->pg, true, gen_helper_sve_fcvtzs_hs);
+}
+
+static bool trans_FCVTZU_hs(DisasContext *s, arg_rpr_esz *a, uint32_t insn)
+{
+ return do_zpz_ptr(s, a->rd, a->rn, a->pg, true, gen_helper_sve_fcvtzu_hs);
+}
+
+static bool trans_FCVTZS_hd(DisasContext *s, arg_rpr_esz *a, uint32_t insn)
+{
+ return do_zpz_ptr(s, a->rd, a->rn, a->pg, true, gen_helper_sve_fcvtzs_hd);
+}
+
+static bool trans_FCVTZU_hd(DisasContext *s, arg_rpr_esz *a, uint32_t insn)
+{
+ return do_zpz_ptr(s, a->rd, a->rn, a->pg, true, gen_helper_sve_fcvtzu_hd);
+}
+
+static bool trans_FCVTZS_ss(DisasContext *s, arg_rpr_esz *a, uint32_t insn)
+{
+ return do_zpz_ptr(s, a->rd, a->rn, a->pg, false, gen_helper_sve_fcvtzs_ss);
+}
+
+static bool trans_FCVTZU_ss(DisasContext *s, arg_rpr_esz *a, uint32_t insn)
+{
+ return do_zpz_ptr(s, a->rd, a->rn, a->pg, false, gen_helper_sve_fcvtzu_ss);
+}
+
+static bool trans_FCVTZS_sd(DisasContext *s, arg_rpr_esz *a, uint32_t insn)
+{
+ return do_zpz_ptr(s, a->rd, a->rn, a->pg, false, gen_helper_sve_fcvtzs_sd);
+}
+
+static bool trans_FCVTZU_sd(DisasContext *s, arg_rpr_esz *a, uint32_t insn)
+{
+ return do_zpz_ptr(s, a->rd, a->rn, a->pg, false, gen_helper_sve_fcvtzu_sd);
+}
+
+static bool trans_FCVTZS_ds(DisasContext *s, arg_rpr_esz *a, uint32_t insn)
+{
+ return do_zpz_ptr(s, a->rd, a->rn, a->pg, false, gen_helper_sve_fcvtzs_ds);
+}
+
+static bool trans_FCVTZU_ds(DisasContext *s, arg_rpr_esz *a, uint32_t insn)
+{
+ return do_zpz_ptr(s, a->rd, a->rn, a->pg, false, gen_helper_sve_fcvtzu_ds);
+}
+
+static bool trans_FCVTZS_dd(DisasContext *s, arg_rpr_esz *a, uint32_t insn)
+{
+ return do_zpz_ptr(s, a->rd, a->rn, a->pg, false, gen_helper_sve_fcvtzs_dd);
+}
+
+static bool trans_FCVTZU_dd(DisasContext *s, arg_rpr_esz *a, uint32_t insn)
+{
+ return do_zpz_ptr(s, a->rd, a->rn, a->pg, false, gen_helper_sve_fcvtzu_dd);
+}
+
+static gen_helper_gvec_3_ptr * const frint_fns[3] = {
+ gen_helper_sve_frint_h,
+ gen_helper_sve_frint_s,
+ gen_helper_sve_frint_d
+};
+
+static bool trans_FRINTI(DisasContext *s, arg_rpr_esz *a, uint32_t insn)
+{
+ if (a->esz == 0) {
+ return false;
+ }
+ return do_zpz_ptr(s, a->rd, a->rn, a->pg, a->esz == MO_16,
+ frint_fns[a->esz - 1]);
+}
+
+static bool trans_FRINTX(DisasContext *s, arg_rpr_esz *a, uint32_t insn)
+{
+ static gen_helper_gvec_3_ptr * const fns[3] = {
+ gen_helper_sve_frintx_h,
+ gen_helper_sve_frintx_s,
+ gen_helper_sve_frintx_d
+ };
+ if (a->esz == 0) {
+ return false;
+ }
+ return do_zpz_ptr(s, a->rd, a->rn, a->pg, a->esz == MO_16, fns[a->esz - 1]);
+}
+
+static bool do_frint_mode(DisasContext *s, arg_rpr_esz *a, int mode)
+{
+ if (a->esz == 0) {
+ return false;
+ }
+ if (sve_access_check(s)) {
+ unsigned vsz = vec_full_reg_size(s);
+ TCGv_i32 tmode = tcg_const_i32(mode);
+ TCGv_ptr status = get_fpstatus_ptr(a->esz == MO_16);
+
+ gen_helper_set_rmode(tmode, tmode, status);
+
+ tcg_gen_gvec_3_ptr(vec_full_reg_offset(s, a->rd),
+ vec_full_reg_offset(s, a->rn),
+ pred_full_reg_offset(s, a->pg),
+ status, vsz, vsz, 0, frint_fns[a->esz - 1]);
+
+ gen_helper_set_rmode(tmode, tmode, status);
+ tcg_temp_free_i32(tmode);
+ tcg_temp_free_ptr(status);
+ }
+ return true;
+}
+
+static bool trans_FRINTN(DisasContext *s, arg_rpr_esz *a, uint32_t insn)
+{
+ return do_frint_mode(s, a, float_round_nearest_even);
+}
+
+static bool trans_FRINTP(DisasContext *s, arg_rpr_esz *a, uint32_t insn)
+{
+ return do_frint_mode(s, a, float_round_up);
+}
+
+static bool trans_FRINTM(DisasContext *s, arg_rpr_esz *a, uint32_t insn)
+{
+ return do_frint_mode(s, a, float_round_down);
+}
+
+static bool trans_FRINTZ(DisasContext *s, arg_rpr_esz *a, uint32_t insn)
+{
+ return do_frint_mode(s, a, float_round_to_zero);
+}
+
+static bool trans_FRINTA(DisasContext *s, arg_rpr_esz *a, uint32_t insn)
+{
+ return do_frint_mode(s, a, float_round_ties_away);
+}
+
+static bool trans_FRECPX(DisasContext *s, arg_rpr_esz *a, uint32_t insn)
+{
+ static gen_helper_gvec_3_ptr * const fns[3] = {
+ gen_helper_sve_frecpx_h,
+ gen_helper_sve_frecpx_s,
+ gen_helper_sve_frecpx_d
+ };
+ if (a->esz == 0) {
+ return false;
+ }
+ return do_zpz_ptr(s, a->rd, a->rn, a->pg, a->esz == MO_16, fns[a->esz - 1]);
+}
+
+static bool trans_FSQRT(DisasContext *s, arg_rpr_esz *a, uint32_t insn)
+{
+ static gen_helper_gvec_3_ptr * const fns[3] = {
+ gen_helper_sve_fsqrt_h,
+ gen_helper_sve_fsqrt_s,
+ gen_helper_sve_fsqrt_d
+ };
+ if (a->esz == 0) {
+ return false;
+ }
+ return do_zpz_ptr(s, a->rd, a->rn, a->pg, a->esz == MO_16, fns[a->esz - 1]);
+}
+
+static bool trans_SCVTF_hh(DisasContext *s, arg_rpr_esz *a, uint32_t insn)
+{
+ return do_zpz_ptr(s, a->rd, a->rn, a->pg, true, gen_helper_sve_scvt_hh);
+}
+
+static bool trans_SCVTF_sh(DisasContext *s, arg_rpr_esz *a, uint32_t insn)
+{
+ return do_zpz_ptr(s, a->rd, a->rn, a->pg, true, gen_helper_sve_scvt_sh);
+}
+
+static bool trans_SCVTF_dh(DisasContext *s, arg_rpr_esz *a, uint32_t insn)
+{
+ return do_zpz_ptr(s, a->rd, a->rn, a->pg, true, gen_helper_sve_scvt_dh);
+}
+
+static bool trans_SCVTF_ss(DisasContext *s, arg_rpr_esz *a, uint32_t insn)
+{
+ return do_zpz_ptr(s, a->rd, a->rn, a->pg, false, gen_helper_sve_scvt_ss);
+}
+
+static bool trans_SCVTF_ds(DisasContext *s, arg_rpr_esz *a, uint32_t insn)
+{
+ return do_zpz_ptr(s, a->rd, a->rn, a->pg, false, gen_helper_sve_scvt_ds);
+}
+
+static bool trans_SCVTF_sd(DisasContext *s, arg_rpr_esz *a, uint32_t insn)
+{
+ return do_zpz_ptr(s, a->rd, a->rn, a->pg, false, gen_helper_sve_scvt_sd);
+}
+
+static bool trans_SCVTF_dd(DisasContext *s, arg_rpr_esz *a, uint32_t insn)
+{
+ return do_zpz_ptr(s, a->rd, a->rn, a->pg, false, gen_helper_sve_scvt_dd);
+}
+
+static bool trans_UCVTF_hh(DisasContext *s, arg_rpr_esz *a, uint32_t insn)
+{
+ return do_zpz_ptr(s, a->rd, a->rn, a->pg, true, gen_helper_sve_ucvt_hh);
+}
+
+static bool trans_UCVTF_sh(DisasContext *s, arg_rpr_esz *a, uint32_t insn)
+{
+ return do_zpz_ptr(s, a->rd, a->rn, a->pg, true, gen_helper_sve_ucvt_sh);
+}
+
+static bool trans_UCVTF_dh(DisasContext *s, arg_rpr_esz *a, uint32_t insn)
+{
+ return do_zpz_ptr(s, a->rd, a->rn, a->pg, true, gen_helper_sve_ucvt_dh);
+}
+
+static bool trans_UCVTF_ss(DisasContext *s, arg_rpr_esz *a, uint32_t insn)
+{
+ return do_zpz_ptr(s, a->rd, a->rn, a->pg, false, gen_helper_sve_ucvt_ss);
+}
+
+static bool trans_UCVTF_ds(DisasContext *s, arg_rpr_esz *a, uint32_t insn)
+{
+ return do_zpz_ptr(s, a->rd, a->rn, a->pg, false, gen_helper_sve_ucvt_ds);
+}
+
+static bool trans_UCVTF_sd(DisasContext *s, arg_rpr_esz *a, uint32_t insn)
+{
+ return do_zpz_ptr(s, a->rd, a->rn, a->pg, false, gen_helper_sve_ucvt_sd);
+}
+
+static bool trans_UCVTF_dd(DisasContext *s, arg_rpr_esz *a, uint32_t insn)
+{
+ return do_zpz_ptr(s, a->rd, a->rn, a->pg, false, gen_helper_sve_ucvt_dd);
+}
+
+/*
*** SVE Memory - 32-bit Gather and Unsized Contiguous Group
*/
@@ -3507,6 +4448,89 @@
tcg_temp_free_i64(t0);
}
+/* Similarly for stores. */
+static void do_str(DisasContext *s, uint32_t vofs, uint32_t len,
+ int rn, int imm)
+{
+ uint32_t len_align = QEMU_ALIGN_DOWN(len, 8);
+ uint32_t len_remain = len % 8;
+ uint32_t nparts = len / 8 + ctpop8(len_remain);
+ int midx = get_mem_index(s);
+ TCGv_i64 addr, t0;
+
+ addr = tcg_temp_new_i64();
+ t0 = tcg_temp_new_i64();
+
+ /* Note that unpredicated load/store of vector/predicate registers
+ * are defined as a stream of bytes, which equates to little-endian
+ * operations on larger quantities. There is no nice way to force
+ * a little-endian store for aarch64_be-linux-user out of line.
+ *
+ * Attempt to keep code expansion to a minimum by limiting the
+ * amount of unrolling done.
+ */
+ if (nparts <= 4) {
+ int i;
+
+ for (i = 0; i < len_align; i += 8) {
+ tcg_gen_ld_i64(t0, cpu_env, vofs + i);
+ tcg_gen_addi_i64(addr, cpu_reg_sp(s, rn), imm + i);
+ tcg_gen_qemu_st_i64(t0, addr, midx, MO_LEQ);
+ }
+ } else {
+ TCGLabel *loop = gen_new_label();
+ TCGv_ptr t2, i = tcg_const_local_ptr(0);
+
+ gen_set_label(loop);
+
+ t2 = tcg_temp_new_ptr();
+ tcg_gen_add_ptr(t2, cpu_env, i);
+ tcg_gen_ld_i64(t0, t2, vofs);
+
+ /* Minimize the number of local temps that must be re-read from
+ * the stack each iteration. Instead, re-compute values other
+ * than the loop counter.
+ */
+ tcg_gen_addi_ptr(t2, i, imm);
+ tcg_gen_extu_ptr_i64(addr, t2);
+ tcg_gen_add_i64(addr, addr, cpu_reg_sp(s, rn));
+ tcg_temp_free_ptr(t2);
+
+ tcg_gen_qemu_st_i64(t0, addr, midx, MO_LEQ);
+
+ tcg_gen_addi_ptr(i, i, 8);
+
+ tcg_gen_brcondi_ptr(TCG_COND_LTU, i, len_align, loop);
+ tcg_temp_free_ptr(i);
+ }
+
+ /* Predicate register stores can be any multiple of 2. */
+ if (len_remain) {
+ tcg_gen_ld_i64(t0, cpu_env, vofs + len_align);
+ tcg_gen_addi_i64(addr, cpu_reg_sp(s, rn), imm + len_align);
+
+ switch (len_remain) {
+ case 2:
+ case 4:
+ case 8:
+ tcg_gen_qemu_st_i64(t0, addr, midx, MO_LE | ctz32(len_remain));
+ break;
+
+ case 6:
+ tcg_gen_qemu_st_i64(t0, addr, midx, MO_LEUL);
+ tcg_gen_addi_i64(addr, addr, 4);
+ tcg_gen_shri_i64(t0, t0, 32);
+ tcg_gen_qemu_st_i64(t0, addr, midx, MO_LEUW);
+ break;
+
+ default:
+ g_assert_not_reached();
+ }
+ }
+ tcg_temp_free_i64(addr);
+ tcg_temp_free_i64(t0);
+}
+
static bool trans_LDR_zri(DisasContext *s, arg_rri *a, uint32_t insn)
{
if (sve_access_check(s)) {
@@ -3526,3 +4550,665 @@
}
return true;
}
+
+static bool trans_STR_zri(DisasContext *s, arg_rri *a, uint32_t insn)
+{
+ if (sve_access_check(s)) {
+ int size = vec_full_reg_size(s);
+ int off = vec_full_reg_offset(s, a->rd);
+ do_str(s, off, size, a->rn, a->imm * size);
+ }
+ return true;
+}
+
+static bool trans_STR_pri(DisasContext *s, arg_rri *a, uint32_t insn)
+{
+ if (sve_access_check(s)) {
+ int size = pred_full_reg_size(s);
+ int off = pred_full_reg_offset(s, a->rd);
+ do_str(s, off, size, a->rn, a->imm * size);
+ }
+ return true;
+}
+
+/*
+ *** SVE Memory - Contiguous Load Group
+ */
+
+/* The memory mode of the dtype. */
+static const TCGMemOp dtype_mop[16] = {
+ MO_UB, MO_UB, MO_UB, MO_UB,
+ MO_SL, MO_UW, MO_UW, MO_UW,
+ MO_SW, MO_SW, MO_UL, MO_UL,
+ MO_SB, MO_SB, MO_SB, MO_Q
+};
+
+#define dtype_msz(x) (dtype_mop[x] & MO_SIZE)
+
+/* The vector element size of dtype. */
+static const uint8_t dtype_esz[16] = {
+ 0, 1, 2, 3,
+ 3, 1, 2, 3,
+ 3, 2, 2, 3,
+ 3, 2, 1, 3
+};
+
+static void do_mem_zpa(DisasContext *s, int zt, int pg, TCGv_i64 addr,
+ gen_helper_gvec_mem *fn)
+{
+ unsigned vsz = vec_full_reg_size(s);
+ TCGv_ptr t_pg;
+ TCGv_i32 desc;
+
+ /* For e.g. LD4, there are not enough arguments to pass all 4
+ * registers as pointers, so encode the regno into the data field.
+ * For consistency, do this even for LD1.
+ */
+ desc = tcg_const_i32(simd_desc(vsz, vsz, zt));
+ t_pg = tcg_temp_new_ptr();
+
+ tcg_gen_addi_ptr(t_pg, cpu_env, pred_full_reg_offset(s, pg));
+ fn(cpu_env, t_pg, addr, desc);
+
+ tcg_temp_free_ptr(t_pg);
+ tcg_temp_free_i32(desc);
+}
+
+static void do_ld_zpa(DisasContext *s, int zt, int pg,
+ TCGv_i64 addr, int dtype, int nreg)
+{
+ static gen_helper_gvec_mem * const fns[16][4] = {
+ { gen_helper_sve_ld1bb_r, gen_helper_sve_ld2bb_r,
+ gen_helper_sve_ld3bb_r, gen_helper_sve_ld4bb_r },
+ { gen_helper_sve_ld1bhu_r, NULL, NULL, NULL },
+ { gen_helper_sve_ld1bsu_r, NULL, NULL, NULL },
+ { gen_helper_sve_ld1bdu_r, NULL, NULL, NULL },
+
+ { gen_helper_sve_ld1sds_r, NULL, NULL, NULL },
+ { gen_helper_sve_ld1hh_r, gen_helper_sve_ld2hh_r,
+ gen_helper_sve_ld3hh_r, gen_helper_sve_ld4hh_r },
+ { gen_helper_sve_ld1hsu_r, NULL, NULL, NULL },
+ { gen_helper_sve_ld1hdu_r, NULL, NULL, NULL },
+
+ { gen_helper_sve_ld1hds_r, NULL, NULL, NULL },
+ { gen_helper_sve_ld1hss_r, NULL, NULL, NULL },
+ { gen_helper_sve_ld1ss_r, gen_helper_sve_ld2ss_r,
+ gen_helper_sve_ld3ss_r, gen_helper_sve_ld4ss_r },
+ { gen_helper_sve_ld1sdu_r, NULL, NULL, NULL },
+
+ { gen_helper_sve_ld1bds_r, NULL, NULL, NULL },
+ { gen_helper_sve_ld1bss_r, NULL, NULL, NULL },
+ { gen_helper_sve_ld1bhs_r, NULL, NULL, NULL },
+ { gen_helper_sve_ld1dd_r, gen_helper_sve_ld2dd_r,
+ gen_helper_sve_ld3dd_r, gen_helper_sve_ld4dd_r },
+ };
+ gen_helper_gvec_mem *fn = fns[dtype][nreg];
+
+ /* While there are holes in the table, they are not
+ * accessible via the instruction encoding.
+ */
+ assert(fn != NULL);
+ do_mem_zpa(s, zt, pg, addr, fn);
+}
+
+static bool trans_LD_zprr(DisasContext *s, arg_rprr_load *a, uint32_t insn)
+{
+ if (a->rm == 31) {
+ return false;
+ }
+ if (sve_access_check(s)) {
+ TCGv_i64 addr = new_tmp_a64(s);
+ tcg_gen_muli_i64(addr, cpu_reg(s, a->rm),
+ (a->nreg + 1) << dtype_msz(a->dtype));
+ tcg_gen_add_i64(addr, addr, cpu_reg_sp(s, a->rn));
+ do_ld_zpa(s, a->rd, a->pg, addr, a->dtype, a->nreg);
+ }
+ return true;
+}
+
+static bool trans_LD_zpri(DisasContext *s, arg_rpri_load *a, uint32_t insn)
+{
+ if (sve_access_check(s)) {
+ int vsz = vec_full_reg_size(s);
+ int elements = vsz >> dtype_esz[a->dtype];
+ TCGv_i64 addr = new_tmp_a64(s);
+
+ tcg_gen_addi_i64(addr, cpu_reg_sp(s, a->rn),
+ (a->imm * elements * (a->nreg + 1))
+ << dtype_msz(a->dtype));
+ do_ld_zpa(s, a->rd, a->pg, addr, a->dtype, a->nreg);
+ }
+ return true;
+}
+
+static bool trans_LDFF1_zprr(DisasContext *s, arg_rprr_load *a, uint32_t insn)
+{
+ static gen_helper_gvec_mem * const fns[16] = {
+ gen_helper_sve_ldff1bb_r,
+ gen_helper_sve_ldff1bhu_r,
+ gen_helper_sve_ldff1bsu_r,
+ gen_helper_sve_ldff1bdu_r,
+
+ gen_helper_sve_ldff1sds_r,
+ gen_helper_sve_ldff1hh_r,
+ gen_helper_sve_ldff1hsu_r,
+ gen_helper_sve_ldff1hdu_r,
+
+ gen_helper_sve_ldff1hds_r,
+ gen_helper_sve_ldff1hss_r,
+ gen_helper_sve_ldff1ss_r,
+ gen_helper_sve_ldff1sdu_r,
+
+ gen_helper_sve_ldff1bds_r,
+ gen_helper_sve_ldff1bss_r,
+ gen_helper_sve_ldff1bhs_r,
+ gen_helper_sve_ldff1dd_r,
+ };
+
+ if (sve_access_check(s)) {
+ TCGv_i64 addr = new_tmp_a64(s);
+ tcg_gen_shli_i64(addr, cpu_reg(s, a->rm), dtype_msz(a->dtype));
+ tcg_gen_add_i64(addr, addr, cpu_reg_sp(s, a->rn));
+ do_mem_zpa(s, a->rd, a->pg, addr, fns[a->dtype]);
+ }
+ return true;
+}
+
+static bool trans_LDNF1_zpri(DisasContext *s, arg_rpri_load *a, uint32_t insn)
+{
+ static gen_helper_gvec_mem * const fns[16] = {
+ gen_helper_sve_ldnf1bb_r,
+ gen_helper_sve_ldnf1bhu_r,
+ gen_helper_sve_ldnf1bsu_r,
+ gen_helper_sve_ldnf1bdu_r,
+
+ gen_helper_sve_ldnf1sds_r,
+ gen_helper_sve_ldnf1hh_r,
+ gen_helper_sve_ldnf1hsu_r,
+ gen_helper_sve_ldnf1hdu_r,
+
+ gen_helper_sve_ldnf1hds_r,
+ gen_helper_sve_ldnf1hss_r,
+ gen_helper_sve_ldnf1ss_r,
+ gen_helper_sve_ldnf1sdu_r,
+
+ gen_helper_sve_ldnf1bds_r,
+ gen_helper_sve_ldnf1bss_r,
+ gen_helper_sve_ldnf1bhs_r,
+ gen_helper_sve_ldnf1dd_r,
+ };
+
+ if (sve_access_check(s)) {
+ int vsz = vec_full_reg_size(s);
+ int elements = vsz >> dtype_esz[a->dtype];
+ int off = (a->imm * elements) << dtype_msz(a->dtype);
+ TCGv_i64 addr = new_tmp_a64(s);
+
+ tcg_gen_addi_i64(addr, cpu_reg_sp(s, a->rn), off);
+ do_mem_zpa(s, a->rd, a->pg, addr, fns[a->dtype]);
+ }
+ return true;
+}
+
+static void do_ldrq(DisasContext *s, int zt, int pg, TCGv_i64 addr, int msz)
+{
+ static gen_helper_gvec_mem * const fns[4] = {
+ gen_helper_sve_ld1bb_r, gen_helper_sve_ld1hh_r,
+ gen_helper_sve_ld1ss_r, gen_helper_sve_ld1dd_r,
+ };
+ unsigned vsz = vec_full_reg_size(s);
+ TCGv_ptr t_pg;
+ TCGv_i32 desc;
+
+ /* Load the first quadword using the normal predicated load helpers. */
+ desc = tcg_const_i32(simd_desc(16, 16, zt));
+ t_pg = tcg_temp_new_ptr();
+
+ tcg_gen_addi_ptr(t_pg, cpu_env, pred_full_reg_offset(s, pg));
+ fns[msz](cpu_env, t_pg, addr, desc);
+
+ tcg_temp_free_ptr(t_pg);
+ tcg_temp_free_i32(desc);
+
+ /* Replicate that first quadword. */
+ if (vsz > 16) {
+ unsigned dofs = vec_full_reg_offset(s, zt);
+ tcg_gen_gvec_dup_mem(4, dofs + 16, dofs, vsz - 16, vsz - 16);
+ }
+}
+
+static bool trans_LD1RQ_zprr(DisasContext *s, arg_rprr_load *a, uint32_t insn)
+{
+ if (a->rm == 31) {
+ return false;
+ }
+ if (sve_access_check(s)) {
+ int msz = dtype_msz(a->dtype);
+ TCGv_i64 addr = new_tmp_a64(s);
+ tcg_gen_shli_i64(addr, cpu_reg(s, a->rm), msz);
+ tcg_gen_add_i64(addr, addr, cpu_reg_sp(s, a->rn));
+ do_ldrq(s, a->rd, a->pg, addr, msz);
+ }
+ return true;
+}
+
+static bool trans_LD1RQ_zpri(DisasContext *s, arg_rpri_load *a, uint32_t insn)
+{
+ if (sve_access_check(s)) {
+ TCGv_i64 addr = new_tmp_a64(s);
+ tcg_gen_addi_i64(addr, cpu_reg_sp(s, a->rn), a->imm * 16);
+ do_ldrq(s, a->rd, a->pg, addr, dtype_msz(a->dtype));
+ }
+ return true;
+}
+
+/* Load and broadcast element. */
+static bool trans_LD1R_zpri(DisasContext *s, arg_rpri_load *a, uint32_t insn)
+{
+ if (!sve_access_check(s)) {
+ return true;
+ }
+
+ unsigned vsz = vec_full_reg_size(s);
+ unsigned psz = pred_full_reg_size(s);
+ unsigned esz = dtype_esz[a->dtype];
+ TCGLabel *over = gen_new_label();
+ TCGv_i64 temp;
+
+ /* If the guarding predicate has no bits set, no load occurs. */
+ if (psz <= 8) {
+ /* Reduce the pred_esz_masks value simply to reduce the
+ * size of the code generated here.
+ */
+ uint64_t psz_mask = MAKE_64BIT_MASK(0, psz * 8);
+ temp = tcg_temp_new_i64();
+ tcg_gen_ld_i64(temp, cpu_env, pred_full_reg_offset(s, a->pg));
+ tcg_gen_andi_i64(temp, temp, pred_esz_masks[esz] & psz_mask);
+ tcg_gen_brcondi_i64(TCG_COND_EQ, temp, 0, over);
+ tcg_temp_free_i64(temp);
+ } else {
+ TCGv_i32 t32 = tcg_temp_new_i32();
+ find_last_active(s, t32, esz, a->pg);
+ tcg_gen_brcondi_i32(TCG_COND_LT, t32, 0, over);
+ tcg_temp_free_i32(t32);
+ }
+
+ /* Load the data. */
+ temp = tcg_temp_new_i64();
+ tcg_gen_addi_i64(temp, cpu_reg_sp(s, a->rn), a->imm << esz);
+ tcg_gen_qemu_ld_i64(temp, temp, get_mem_index(s),
+ s->be_data | dtype_mop[a->dtype]);
+
+ /* Broadcast to *all* elements. */
+ tcg_gen_gvec_dup_i64(esz, vec_full_reg_offset(s, a->rd),
+ vsz, vsz, temp);
+ tcg_temp_free_i64(temp);
+
+ /* Zero the inactive elements. */
+ gen_set_label(over);
+ do_movz_zpz(s, a->rd, a->rd, a->pg, esz);
+ return true;
+}
+
+static void do_st_zpa(DisasContext *s, int zt, int pg, TCGv_i64 addr,
+ int msz, int esz, int nreg)
+{
+ static gen_helper_gvec_mem * const fn_single[4][4] = {
+ { gen_helper_sve_st1bb_r, gen_helper_sve_st1bh_r,
+ gen_helper_sve_st1bs_r, gen_helper_sve_st1bd_r },
+ { NULL, gen_helper_sve_st1hh_r,
+ gen_helper_sve_st1hs_r, gen_helper_sve_st1hd_r },
+ { NULL, NULL,
+ gen_helper_sve_st1ss_r, gen_helper_sve_st1sd_r },
+ { NULL, NULL, NULL, gen_helper_sve_st1dd_r },
+ };
+ static gen_helper_gvec_mem * const fn_multiple[3][4] = {
+ { gen_helper_sve_st2bb_r, gen_helper_sve_st2hh_r,
+ gen_helper_sve_st2ss_r, gen_helper_sve_st2dd_r },
+ { gen_helper_sve_st3bb_r, gen_helper_sve_st3hh_r,
+ gen_helper_sve_st3ss_r, gen_helper_sve_st3dd_r },
+ { gen_helper_sve_st4bb_r, gen_helper_sve_st4hh_r,
+ gen_helper_sve_st4ss_r, gen_helper_sve_st4dd_r },
+ };
+ gen_helper_gvec_mem *fn;
+
+ if (nreg == 0) {
+ /* ST1 */
+ fn = fn_single[msz][esz];
+ } else {
+ /* ST2, ST3, ST4 -- msz == esz, enforced by encoding */
+ assert(msz == esz);
+ fn = fn_multiple[nreg - 1][msz];
+ }
+ assert(fn != NULL);
+ do_mem_zpa(s, zt, pg, addr, fn);
+}
+
+static bool trans_ST_zprr(DisasContext *s, arg_rprr_store *a, uint32_t insn)
+{
+ if (a->rm == 31 || a->msz > a->esz) {
+ return false;
+ }
+ if (sve_access_check(s)) {
+ TCGv_i64 addr = new_tmp_a64(s);
+ tcg_gen_muli_i64(addr, cpu_reg(s, a->rm), (a->nreg + 1) << a->msz);
+ tcg_gen_add_i64(addr, addr, cpu_reg_sp(s, a->rn));
+ do_st_zpa(s, a->rd, a->pg, addr, a->msz, a->esz, a->nreg);
+ }
+ return true;
+}
+
+static bool trans_ST_zpri(DisasContext *s, arg_rpri_store *a, uint32_t insn)
+{
+ if (a->msz > a->esz) {
+ return false;
+ }
+ if (sve_access_check(s)) {
+ int vsz = vec_full_reg_size(s);
+ int elements = vsz >> a->esz;
+ TCGv_i64 addr = new_tmp_a64(s);
+
+ tcg_gen_addi_i64(addr, cpu_reg_sp(s, a->rn),
+ (a->imm * elements * (a->nreg + 1)) << a->msz);
+ do_st_zpa(s, a->rd, a->pg, addr, a->msz, a->esz, a->nreg);
+ }
+ return true;
+}
+
+/*
+ *** SVE gather loads / scatter stores
+ */
+
+static void do_mem_zpz(DisasContext *s, int zt, int pg, int zm, int scale,
+ TCGv_i64 scalar, gen_helper_gvec_mem_scatter *fn)
+{
+ unsigned vsz = vec_full_reg_size(s);
+ TCGv_i32 desc = tcg_const_i32(simd_desc(vsz, vsz, scale));
+ TCGv_ptr t_zm = tcg_temp_new_ptr();
+ TCGv_ptr t_pg = tcg_temp_new_ptr();
+ TCGv_ptr t_zt = tcg_temp_new_ptr();
+
+ tcg_gen_addi_ptr(t_pg, cpu_env, pred_full_reg_offset(s, pg));
+ tcg_gen_addi_ptr(t_zm, cpu_env, vec_full_reg_offset(s, zm));
+ tcg_gen_addi_ptr(t_zt, cpu_env, vec_full_reg_offset(s, zt));
+ fn(cpu_env, t_zt, t_pg, t_zm, scalar, desc);
+
+ tcg_temp_free_ptr(t_zt);
+ tcg_temp_free_ptr(t_zm);
+ tcg_temp_free_ptr(t_pg);
+ tcg_temp_free_i32(desc);
+}
+
+/* Indexed by [ff][xs][u][msz]. */
+static gen_helper_gvec_mem_scatter * const gather_load_fn32[2][2][2][3] = {
+ { { { gen_helper_sve_ldbss_zsu,
+ gen_helper_sve_ldhss_zsu,
+ NULL, },
+ { gen_helper_sve_ldbsu_zsu,
+ gen_helper_sve_ldhsu_zsu,
+ gen_helper_sve_ldssu_zsu, } },
+ { { gen_helper_sve_ldbss_zss,
+ gen_helper_sve_ldhss_zss,
+ NULL, },
+ { gen_helper_sve_ldbsu_zss,
+ gen_helper_sve_ldhsu_zss,
+ gen_helper_sve_ldssu_zss, } } },
+
+ { { { gen_helper_sve_ldffbss_zsu,
+ gen_helper_sve_ldffhss_zsu,
+ NULL, },
+ { gen_helper_sve_ldffbsu_zsu,
+ gen_helper_sve_ldffhsu_zsu,
+ gen_helper_sve_ldffssu_zsu, } },
+ { { gen_helper_sve_ldffbss_zss,
+ gen_helper_sve_ldffhss_zss,
+ NULL, },
+ { gen_helper_sve_ldffbsu_zss,
+ gen_helper_sve_ldffhsu_zss,
+ gen_helper_sve_ldffssu_zss, } } }
+};
+
+/* Note that we overload xs=2 to indicate 64-bit offset. */
+static gen_helper_gvec_mem_scatter * const gather_load_fn64[2][3][2][4] = {
+ { { { gen_helper_sve_ldbds_zsu,
+ gen_helper_sve_ldhds_zsu,
+ gen_helper_sve_ldsds_zsu,
+ NULL, },
+ { gen_helper_sve_ldbdu_zsu,
+ gen_helper_sve_ldhdu_zsu,
+ gen_helper_sve_ldsdu_zsu,
+ gen_helper_sve_ldddu_zsu, } },
+ { { gen_helper_sve_ldbds_zss,
+ gen_helper_sve_ldhds_zss,
+ gen_helper_sve_ldsds_zss,
+ NULL, },
+ { gen_helper_sve_ldbdu_zss,
+ gen_helper_sve_ldhdu_zss,
+ gen_helper_sve_ldsdu_zss,
+ gen_helper_sve_ldddu_zss, } },
+ { { gen_helper_sve_ldbds_zd,
+ gen_helper_sve_ldhds_zd,
+ gen_helper_sve_ldsds_zd,
+ NULL, },
+ { gen_helper_sve_ldbdu_zd,
+ gen_helper_sve_ldhdu_zd,
+ gen_helper_sve_ldsdu_zd,
+ gen_helper_sve_ldddu_zd, } } },
+
+ { { { gen_helper_sve_ldffbds_zsu,
+ gen_helper_sve_ldffhds_zsu,
+ gen_helper_sve_ldffsds_zsu,
+ NULL, },
+ { gen_helper_sve_ldffbdu_zsu,
+ gen_helper_sve_ldffhdu_zsu,
+ gen_helper_sve_ldffsdu_zsu,
+ gen_helper_sve_ldffddu_zsu, } },
+ { { gen_helper_sve_ldffbds_zss,
+ gen_helper_sve_ldffhds_zss,
+ gen_helper_sve_ldffsds_zss,
+ NULL, },
+ { gen_helper_sve_ldffbdu_zss,
+ gen_helper_sve_ldffhdu_zss,
+ gen_helper_sve_ldffsdu_zss,
+ gen_helper_sve_ldffddu_zss, } },
+ { { gen_helper_sve_ldffbds_zd,
+ gen_helper_sve_ldffhds_zd,
+ gen_helper_sve_ldffsds_zd,
+ NULL, },
+ { gen_helper_sve_ldffbdu_zd,
+ gen_helper_sve_ldffhdu_zd,
+ gen_helper_sve_ldffsdu_zd,
+ gen_helper_sve_ldffddu_zd, } } }
+};
+
+static bool trans_LD1_zprz(DisasContext *s, arg_LD1_zprz *a, uint32_t insn)
+{
+ gen_helper_gvec_mem_scatter *fn = NULL;
+
+ if (!sve_access_check(s)) {
+ return true;
+ }
+
+ switch (a->esz) {
+ case MO_32:
+ fn = gather_load_fn32[a->ff][a->xs][a->u][a->msz];
+ break;
+ case MO_64:
+ fn = gather_load_fn64[a->ff][a->xs][a->u][a->msz];
+ break;
+ }
+ assert(fn != NULL);
+
+ do_mem_zpz(s, a->rd, a->pg, a->rm, a->scale * a->msz,
+ cpu_reg_sp(s, a->rn), fn);
+ return true;
+}
+
+static bool trans_LD1_zpiz(DisasContext *s, arg_LD1_zpiz *a, uint32_t insn)
+{
+ gen_helper_gvec_mem_scatter *fn = NULL;
+ TCGv_i64 imm;
+
+ if (a->esz < a->msz || (a->esz == a->msz && !a->u)) {
+ return false;
+ }
+ if (!sve_access_check(s)) {
+ return true;
+ }
+
+ switch (a->esz) {
+ case MO_32:
+ fn = gather_load_fn32[a->ff][0][a->u][a->msz];
+ break;
+ case MO_64:
+ fn = gather_load_fn64[a->ff][2][a->u][a->msz];
+ break;
+ }
+ assert(fn != NULL);
+
+ /* Treat LD1_zpiz (zn[x] + imm) the same way as LD1_zprz (rn + zm[x])
+ * by loading the immediate into the scalar parameter.
+ */
+ imm = tcg_const_i64(a->imm << a->msz);
+ do_mem_zpz(s, a->rd, a->pg, a->rn, 0, imm, fn);
+ tcg_temp_free_i64(imm);
+ return true;
+}
+
+/* Indexed by [xs][msz]. */
+static gen_helper_gvec_mem_scatter * const scatter_store_fn32[2][3] = {
+ { gen_helper_sve_stbs_zsu,
+ gen_helper_sve_sths_zsu,
+ gen_helper_sve_stss_zsu, },
+ { gen_helper_sve_stbs_zss,
+ gen_helper_sve_sths_zss,
+ gen_helper_sve_stss_zss, },
+};
+
+/* Note that we overload xs=2 to indicate 64-bit offset. */
+static gen_helper_gvec_mem_scatter * const scatter_store_fn64[3][4] = {
+ { gen_helper_sve_stbd_zsu,
+ gen_helper_sve_sthd_zsu,
+ gen_helper_sve_stsd_zsu,
+ gen_helper_sve_stdd_zsu, },
+ { gen_helper_sve_stbd_zss,
+ gen_helper_sve_sthd_zss,
+ gen_helper_sve_stsd_zss,
+ gen_helper_sve_stdd_zss, },
+ { gen_helper_sve_stbd_zd,
+ gen_helper_sve_sthd_zd,
+ gen_helper_sve_stsd_zd,
+ gen_helper_sve_stdd_zd, },
+};
+
+static bool trans_ST1_zprz(DisasContext *s, arg_ST1_zprz *a, uint32_t insn)
+{
+ gen_helper_gvec_mem_scatter *fn;
+
+ if (a->esz < a->msz || (a->msz == 0 && a->scale)) {
+ return false;
+ }
+ if (!sve_access_check(s)) {
+ return true;
+ }
+ switch (a->esz) {
+ case MO_32:
+ fn = scatter_store_fn32[a->xs][a->msz];
+ break;
+ case MO_64:
+ fn = scatter_store_fn64[a->xs][a->msz];
+ break;
+ default:
+ g_assert_not_reached();
+ }
+ do_mem_zpz(s, a->rd, a->pg, a->rm, a->scale * a->msz,
+ cpu_reg_sp(s, a->rn), fn);
+ return true;
+}
+
+static bool trans_ST1_zpiz(DisasContext *s, arg_ST1_zpiz *a, uint32_t insn)
+{
+ gen_helper_gvec_mem_scatter *fn = NULL;
+ TCGv_i64 imm;
+
+ if (a->esz < a->msz) {
+ return false;
+ }
+ if (!sve_access_check(s)) {
+ return true;
+ }
+
+ switch (a->esz) {
+ case MO_32:
+ fn = scatter_store_fn32[0][a->msz];
+ break;
+ case MO_64:
+ fn = scatter_store_fn64[2][a->msz];
+ break;
+ }
+ assert(fn != NULL);
+
+ /* Treat ST1_zpiz (zn[x] + imm) the same way as ST1_zprz (rn + zm[x])
+ * by loading the immediate into the scalar parameter.
+ */
+ imm = tcg_const_i64(a->imm << a->msz);
+ do_mem_zpz(s, a->rd, a->pg, a->rn, 0, imm, fn);
+ tcg_temp_free_i64(imm);
+ return true;
+}
+
+/*
+ * Prefetches
+ */
+
+static bool trans_PRF(DisasContext *s, arg_PRF *a, uint32_t insn)
+{
+ /* Prefetch is a nop within QEMU. */
+ sve_access_check(s);
+ return true;
+}
+
+static bool trans_PRF_rr(DisasContext *s, arg_PRF_rr *a, uint32_t insn)
+{
+ if (a->rm == 31) {
+ return false;
+ }
+ /* Prefetch is a nop within QEMU. */
+ sve_access_check(s);
+ return true;
+}
+
+/*
+ * Move Prefix
+ *
+ * TODO: The implementation so far could handle predicated merging movprfx.
+ * The helper functions as written take an extra source register to
+ * use in the operation, but the result is only written when predication
+ * succeeds. For unpredicated movprfx, we need to rearrange the helpers
+ * to allow the final write back to the destination to be unconditional.
+ * For predicated zeroing movprfx, we need to rearrange the helpers to
+ * allow the final write back to zero inactives.
+ *
+ * In the meantime, just emit the moves.
+ */
+
+static bool trans_MOVPRFX(DisasContext *s, arg_MOVPRFX *a, uint32_t insn)
+{
+ return do_mov_z(s, a->rd, a->rn);
+}
+
+static bool trans_MOVPRFX_m(DisasContext *s, arg_rpr_esz *a, uint32_t insn)
+{
+ if (sve_access_check(s)) {
+ do_sel_z(s, a->rd, a->rn, a->rd, a->pg, a->esz);
+ }
+ return true;
+}
+
+static bool trans_MOVPRFX_z(DisasContext *s, arg_rpr_esz *a, uint32_t insn)
+{
+ if (sve_access_check(s)) {
+ do_movz_zpz(s, a->rd, a->rn, a->pg, a->esz);
+ }
+ return true;
+}
diff --git a/target/arm/translate.c b/target/arm/translate.c
index 2a3e4f5..f845da7 100644
--- a/target/arm/translate.c
+++ b/target/arm/translate.c
@@ -7762,9 +7762,10 @@
*/
static int disas_neon_insn_3same_ext(DisasContext *s, uint32_t insn)
{
- gen_helper_gvec_3_ptr *fn_gvec_ptr;
- int rd, rn, rm, rot, size, opr_sz;
- TCGv_ptr fpst;
+ gen_helper_gvec_3 *fn_gvec = NULL;
+ gen_helper_gvec_3_ptr *fn_gvec_ptr = NULL;
+ int rd, rn, rm, opr_sz;
+ int data = 0;
bool q;
q = extract32(insn, 6, 1);
@@ -7777,8 +7778,8 @@
if ((insn & 0xfe200f10) == 0xfc200800) {
/* VCMLA -- 1111 110R R.1S .... .... 1000 ...0 .... */
- size = extract32(insn, 20, 1);
- rot = extract32(insn, 23, 2);
+ int size = extract32(insn, 20, 1);
+ data = extract32(insn, 23, 2); /* rot */
if (!arm_dc_feature(s, ARM_FEATURE_V8_FCMA)
|| (!size && !arm_dc_feature(s, ARM_FEATURE_V8_FP16))) {
return 1;
@@ -7786,13 +7787,20 @@
fn_gvec_ptr = size ? gen_helper_gvec_fcmlas : gen_helper_gvec_fcmlah;
} else if ((insn & 0xfea00f10) == 0xfc800800) {
/* VCADD -- 1111 110R 1.0S .... .... 1000 ...0 .... */
- size = extract32(insn, 20, 1);
- rot = extract32(insn, 24, 1);
+ int size = extract32(insn, 20, 1);
+ data = extract32(insn, 24, 1); /* rot */
if (!arm_dc_feature(s, ARM_FEATURE_V8_FCMA)
|| (!size && !arm_dc_feature(s, ARM_FEATURE_V8_FP16))) {
return 1;
}
fn_gvec_ptr = size ? gen_helper_gvec_fcadds : gen_helper_gvec_fcaddh;
+ } else if ((insn & 0xfeb00f00) == 0xfc200d00) {
+ /* V[US]DOT -- 1111 1100 0.10 .... .... 1101 .Q.U .... */
+ bool u = extract32(insn, 4, 1);
+ if (!arm_dc_feature(s, ARM_FEATURE_V8_DOTPROD)) {
+ return 1;
+ }
+ fn_gvec = u ? gen_helper_gvec_udot_b : gen_helper_gvec_sdot_b;
} else {
return 1;
}
@@ -7807,12 +7815,19 @@
}
opr_sz = (1 + q) * 8;
- fpst = get_fpstatus_ptr(1);
- tcg_gen_gvec_3_ptr(vfp_reg_offset(1, rd),
- vfp_reg_offset(1, rn),
- vfp_reg_offset(1, rm), fpst,
- opr_sz, opr_sz, rot, fn_gvec_ptr);
- tcg_temp_free_ptr(fpst);
+ if (fn_gvec_ptr) {
+ TCGv_ptr fpst = get_fpstatus_ptr(1);
+ tcg_gen_gvec_3_ptr(vfp_reg_offset(1, rd),
+ vfp_reg_offset(1, rn),
+ vfp_reg_offset(1, rm), fpst,
+ opr_sz, opr_sz, data, fn_gvec_ptr);
+ tcg_temp_free_ptr(fpst);
+ } else {
+ tcg_gen_gvec_3_ool(vfp_reg_offset(1, rd),
+ vfp_reg_offset(1, rn),
+ vfp_reg_offset(1, rm),
+ opr_sz, opr_sz, data, fn_gvec);
+ }
return 0;
}
@@ -7826,26 +7841,52 @@
static int disas_neon_insn_2reg_scalar_ext(DisasContext *s, uint32_t insn)
{
- int rd, rn, rm, rot, size, opr_sz;
- TCGv_ptr fpst;
+ gen_helper_gvec_3 *fn_gvec = NULL;
+ gen_helper_gvec_3_ptr *fn_gvec_ptr = NULL;
+ int rd, rn, rm, opr_sz, data;
bool q;
q = extract32(insn, 6, 1);
VFP_DREG_D(rd, insn);
VFP_DREG_N(rn, insn);
- VFP_DREG_M(rm, insn);
if ((rd | rn) & q) {
return 1;
}
if ((insn & 0xff000f10) == 0xfe000800) {
/* VCMLA (indexed) -- 1111 1110 S.RR .... .... 1000 ...0 .... */
- rot = extract32(insn, 20, 2);
- size = extract32(insn, 23, 1);
- if (!arm_dc_feature(s, ARM_FEATURE_V8_FCMA)
- || (!size && !arm_dc_feature(s, ARM_FEATURE_V8_FP16))) {
+ int rot = extract32(insn, 20, 2);
+ int size = extract32(insn, 23, 1);
+ int index;
+
+ if (!arm_dc_feature(s, ARM_FEATURE_V8_FCMA)) {
return 1;
}
+ if (size == 0) {
+ if (!arm_dc_feature(s, ARM_FEATURE_V8_FP16)) {
+ return 1;
+ }
+ /* For fp16, rm is just Vm, and index is M. */
+ rm = extract32(insn, 0, 4);
+ index = extract32(insn, 5, 1);
+ } else {
+ /* For fp32, rm is the usual M:Vm, and index is 0. */
+ VFP_DREG_M(rm, insn);
+ index = 0;
+ }
+ data = (index << 2) | rot;
+ fn_gvec_ptr = (size ? gen_helper_gvec_fcmlas_idx
+ : gen_helper_gvec_fcmlah_idx);
+ } else if ((insn & 0xffb00f00) == 0xfe200d00) {
+ /* V[US]DOT -- 1111 1110 0.10 .... .... 1101 .Q.U .... */
+ int u = extract32(insn, 4, 1);
+ if (!arm_dc_feature(s, ARM_FEATURE_V8_DOTPROD)) {
+ return 1;
+ }
+ fn_gvec = u ? gen_helper_gvec_udot_idx_b : gen_helper_gvec_sdot_idx_b;
+ /* rm is just Vm, and index is M. */
+ data = extract32(insn, 5, 1); /* index */
+ rm = extract32(insn, 0, 4);
} else {
return 1;
}
@@ -7860,14 +7901,19 @@
}
opr_sz = (1 + q) * 8;
- fpst = get_fpstatus_ptr(1);
- tcg_gen_gvec_3_ptr(vfp_reg_offset(1, rd),
- vfp_reg_offset(1, rn),
- vfp_reg_offset(1, rm), fpst,
- opr_sz, opr_sz, rot,
- size ? gen_helper_gvec_fcmlas_idx
- : gen_helper_gvec_fcmlah_idx);
- tcg_temp_free_ptr(fpst);
+ if (fn_gvec_ptr) {
+ TCGv_ptr fpst = get_fpstatus_ptr(1);
+ tcg_gen_gvec_3_ptr(vfp_reg_offset(1, rd),
+ vfp_reg_offset(1, rn),
+ vfp_reg_offset(1, rm), fpst,
+ opr_sz, opr_sz, data, fn_gvec_ptr);
+ tcg_temp_free_ptr(fpst);
+ } else {
+ tcg_gen_gvec_3_ool(vfp_reg_offset(1, rd),
+ vfp_reg_offset(1, rn),
+ vfp_reg_offset(1, rm),
+ opr_sz, opr_sz, data, fn_gvec);
+ }
return 0;
}
diff --git a/target/arm/vec_helper.c b/target/arm/vec_helper.c
index f504dd5..37f3387 100644
--- a/target/arm/vec_helper.c
+++ b/target/arm/vec_helper.c
@@ -194,6 +194,197 @@
clear_tail(d, opr_sz, simd_maxsz(desc));
}
+/* Integer 8 and 16-bit dot-product.
+ *
+ * Note that for the loops herein, host endianness does not matter
+ * with respect to the ordering of data within the 64-bit lanes.
+ * All elements are treated equally, no matter where they are.
+ */
+
+void HELPER(gvec_sdot_b)(void *vd, void *vn, void *vm, uint32_t desc)
+{
+ intptr_t i, opr_sz = simd_oprsz(desc);
+ uint32_t *d = vd;
+ int8_t *n = vn, *m = vm;
+
+ for (i = 0; i < opr_sz / 4; ++i) {
+ d[i] += n[i * 4 + 0] * m[i * 4 + 0]
+ + n[i * 4 + 1] * m[i * 4 + 1]
+ + n[i * 4 + 2] * m[i * 4 + 2]
+ + n[i * 4 + 3] * m[i * 4 + 3];
+ }
+ clear_tail(d, opr_sz, simd_maxsz(desc));
+}
+
+void HELPER(gvec_udot_b)(void *vd, void *vn, void *vm, uint32_t desc)
+{
+ intptr_t i, opr_sz = simd_oprsz(desc);
+ uint32_t *d = vd;
+ uint8_t *n = vn, *m = vm;
+
+ for (i = 0; i < opr_sz / 4; ++i) {
+ d[i] += n[i * 4 + 0] * m[i * 4 + 0]
+ + n[i * 4 + 1] * m[i * 4 + 1]
+ + n[i * 4 + 2] * m[i * 4 + 2]
+ + n[i * 4 + 3] * m[i * 4 + 3];
+ }
+ clear_tail(d, opr_sz, simd_maxsz(desc));
+}
+
+void HELPER(gvec_sdot_h)(void *vd, void *vn, void *vm, uint32_t desc)
+{
+ intptr_t i, opr_sz = simd_oprsz(desc);
+ uint64_t *d = vd;
+ int16_t *n = vn, *m = vm;
+
+ for (i = 0; i < opr_sz / 8; ++i) {
+ d[i] += (int64_t)n[i * 4 + 0] * m[i * 4 + 0]
+ + (int64_t)n[i * 4 + 1] * m[i * 4 + 1]
+ + (int64_t)n[i * 4 + 2] * m[i * 4 + 2]
+ + (int64_t)n[i * 4 + 3] * m[i * 4 + 3];
+ }
+ clear_tail(d, opr_sz, simd_maxsz(desc));
+}
+
+void HELPER(gvec_udot_h)(void *vd, void *vn, void *vm, uint32_t desc)
+{
+ intptr_t i, opr_sz = simd_oprsz(desc);
+ uint64_t *d = vd;
+ uint16_t *n = vn, *m = vm;
+
+ for (i = 0; i < opr_sz / 8; ++i) {
+ d[i] += (uint64_t)n[i * 4 + 0] * m[i * 4 + 0]
+ + (uint64_t)n[i * 4 + 1] * m[i * 4 + 1]
+ + (uint64_t)n[i * 4 + 2] * m[i * 4 + 2]
+ + (uint64_t)n[i * 4 + 3] * m[i * 4 + 3];
+ }
+ clear_tail(d, opr_sz, simd_maxsz(desc));
+}
+
+void HELPER(gvec_sdot_idx_b)(void *vd, void *vn, void *vm, uint32_t desc)
+{
+ intptr_t i, segend, opr_sz = simd_oprsz(desc), opr_sz_4 = opr_sz / 4;
+ intptr_t index = simd_data(desc);
+ uint32_t *d = vd;
+ int8_t *n = vn;
+ int8_t *m_indexed = (int8_t *)vm + index * 4;
+
+ /* Notice the special case of opr_sz == 8, from aa64/aa32 advsimd.
+ * Otherwise opr_sz is a multiple of 16.
+ */
+ segend = MIN(4, opr_sz_4);
+ i = 0;
+ do {
+ int8_t m0 = m_indexed[i * 4 + 0];
+ int8_t m1 = m_indexed[i * 4 + 1];
+ int8_t m2 = m_indexed[i * 4 + 2];
+ int8_t m3 = m_indexed[i * 4 + 3];
+
+ do {
+ d[i] += n[i * 4 + 0] * m0
+ + n[i * 4 + 1] * m1
+ + n[i * 4 + 2] * m2
+ + n[i * 4 + 3] * m3;
+ } while (++i < segend);
+ segend = i + 4;
+ } while (i < opr_sz_4);
+
+ clear_tail(d, opr_sz, simd_maxsz(desc));
+}
+
+void HELPER(gvec_udot_idx_b)(void *vd, void *vn, void *vm, uint32_t desc)
+{
+ intptr_t i, segend, opr_sz = simd_oprsz(desc), opr_sz_4 = opr_sz / 4;
+ intptr_t index = simd_data(desc);
+ uint32_t *d = vd;
+ uint8_t *n = vn;
+ uint8_t *m_indexed = (uint8_t *)vm + index * 4;
+
+ /* Notice the special case of opr_sz == 8, from aa64/aa32 advsimd.
+ * Otherwise opr_sz is a multiple of 16.
+ */
+ segend = MIN(4, opr_sz_4);
+ i = 0;
+ do {
+ uint8_t m0 = m_indexed[i * 4 + 0];
+ uint8_t m1 = m_indexed[i * 4 + 1];
+ uint8_t m2 = m_indexed[i * 4 + 2];
+ uint8_t m3 = m_indexed[i * 4 + 3];
+
+ do {
+ d[i] += n[i * 4 + 0] * m0
+ + n[i * 4 + 1] * m1
+ + n[i * 4 + 2] * m2
+ + n[i * 4 + 3] * m3;
+ } while (++i < segend);
+ segend = i + 4;
+ } while (i < opr_sz_4);
+
+ clear_tail(d, opr_sz, simd_maxsz(desc));
+}
+
+void HELPER(gvec_sdot_idx_h)(void *vd, void *vn, void *vm, uint32_t desc)
+{
+ intptr_t i, opr_sz = simd_oprsz(desc), opr_sz_8 = opr_sz / 8;
+ intptr_t index = simd_data(desc);
+ uint64_t *d = vd;
+ int16_t *n = vn;
+ int16_t *m_indexed = (int16_t *)vm + index * 4;
+
+ /* This is supported by SVE only, so opr_sz is always a multiple of 16.
+ * Process the entire segment all at once, writing back the results
+ * only after we've consumed all of the inputs.
+ */
+ for (i = 0; i < opr_sz_8 ; i += 2) {
+ uint64_t d0, d1;
+
+ d0 = n[i * 4 + 0] * (int64_t)m_indexed[i * 4 + 0];
+ d0 += n[i * 4 + 1] * (int64_t)m_indexed[i * 4 + 1];
+ d0 += n[i * 4 + 2] * (int64_t)m_indexed[i * 4 + 2];
+ d0 += n[i * 4 + 3] * (int64_t)m_indexed[i * 4 + 3];
+ d1 = n[i * 4 + 4] * (int64_t)m_indexed[i * 4 + 0];
+ d1 += n[i * 4 + 5] * (int64_t)m_indexed[i * 4 + 1];
+ d1 += n[i * 4 + 6] * (int64_t)m_indexed[i * 4 + 2];
+ d1 += n[i * 4 + 7] * (int64_t)m_indexed[i * 4 + 3];
+
+ d[i + 0] += d0;
+ d[i + 1] += d1;
+ }
+
+ clear_tail(d, opr_sz, simd_maxsz(desc));
+}
+
+void HELPER(gvec_udot_idx_h)(void *vd, void *vn, void *vm, uint32_t desc)
+{
+ intptr_t i, opr_sz = simd_oprsz(desc), opr_sz_8 = opr_sz / 8;
+ intptr_t index = simd_data(desc);
+ uint64_t *d = vd;
+ uint16_t *n = vn;
+ uint16_t *m_indexed = (uint16_t *)vm + index * 4;
+
+ /* This is supported by SVE only, so opr_sz is always a multiple of 16.
+ * Process the entire segment all at once, writing back the results
+ * only after we've consumed all of the inputs.
+ */
+ for (i = 0; i < opr_sz_8 ; i += 2) {
+ uint64_t d0, d1;
+
+ d0 = n[i * 4 + 0] * (uint64_t)m_indexed[i * 4 + 0];
+ d0 += n[i * 4 + 1] * (uint64_t)m_indexed[i * 4 + 1];
+ d0 += n[i * 4 + 2] * (uint64_t)m_indexed[i * 4 + 2];
+ d0 += n[i * 4 + 3] * (uint64_t)m_indexed[i * 4 + 3];
+ d1 = n[i * 4 + 4] * (uint64_t)m_indexed[i * 4 + 0];
+ d1 += n[i * 4 + 5] * (uint64_t)m_indexed[i * 4 + 1];
+ d1 += n[i * 4 + 6] * (uint64_t)m_indexed[i * 4 + 2];
+ d1 += n[i * 4 + 7] * (uint64_t)m_indexed[i * 4 + 3];
+
+ d[i + 0] += d0;
+ d[i + 1] += d1;
+ }
+
+ clear_tail(d, opr_sz, simd_maxsz(desc));
+}
+
void HELPER(gvec_fcaddh)(void *vd, void *vn, void *vm,
void *vfpst, uint32_t desc)
{
@@ -317,23 +508,29 @@
float_status *fpst = vfpst;
intptr_t flip = extract32(desc, SIMD_DATA_SHIFT, 1);
uint32_t neg_imag = extract32(desc, SIMD_DATA_SHIFT + 1, 1);
+ intptr_t index = extract32(desc, SIMD_DATA_SHIFT + 2, 2);
uint32_t neg_real = flip ^ neg_imag;
- uintptr_t i;
- float16 e1 = m[H2(flip)];
- float16 e3 = m[H2(1 - flip)];
+ intptr_t elements = opr_sz / sizeof(float16);
+ intptr_t eltspersegment = 16 / sizeof(float16);
+ intptr_t i, j;
/* Shift boolean to the sign bit so we can xor to negate. */
neg_real <<= 15;
neg_imag <<= 15;
- e1 ^= neg_real;
- e3 ^= neg_imag;
- for (i = 0; i < opr_sz / 2; i += 2) {
- float16 e2 = n[H2(i + flip)];
- float16 e4 = e2;
+ for (i = 0; i < elements; i += eltspersegment) {
+ float16 mr = m[H2(i + 2 * index + 0)];
+ float16 mi = m[H2(i + 2 * index + 1)];
+ float16 e1 = neg_real ^ (flip ? mi : mr);
+ float16 e3 = neg_imag ^ (flip ? mr : mi);
- d[H2(i)] = float16_muladd(e2, e1, d[H2(i)], 0, fpst);
- d[H2(i + 1)] = float16_muladd(e4, e3, d[H2(i + 1)], 0, fpst);
+ for (j = i; j < i + eltspersegment; j += 2) {
+ float16 e2 = n[H2(j + flip)];
+ float16 e4 = e2;
+
+ d[H2(j)] = float16_muladd(e2, e1, d[H2(j)], 0, fpst);
+ d[H2(j + 1)] = float16_muladd(e4, e3, d[H2(j + 1)], 0, fpst);
+ }
}
clear_tail(d, opr_sz, simd_maxsz(desc));
}
@@ -377,23 +574,29 @@
float_status *fpst = vfpst;
intptr_t flip = extract32(desc, SIMD_DATA_SHIFT, 1);
uint32_t neg_imag = extract32(desc, SIMD_DATA_SHIFT + 1, 1);
+ intptr_t index = extract32(desc, SIMD_DATA_SHIFT + 2, 2);
uint32_t neg_real = flip ^ neg_imag;
- uintptr_t i;
- float32 e1 = m[H4(flip)];
- float32 e3 = m[H4(1 - flip)];
+ intptr_t elements = opr_sz / sizeof(float32);
+ intptr_t eltspersegment = 16 / sizeof(float32);
+ intptr_t i, j;
/* Shift boolean to the sign bit so we can xor to negate. */
neg_real <<= 31;
neg_imag <<= 31;
- e1 ^= neg_real;
- e3 ^= neg_imag;
- for (i = 0; i < opr_sz / 4; i += 2) {
- float32 e2 = n[H4(i + flip)];
- float32 e4 = e2;
+ for (i = 0; i < elements; i += eltspersegment) {
+ float32 mr = m[H4(i + 2 * index + 0)];
+ float32 mi = m[H4(i + 2 * index + 1)];
+ float32 e1 = neg_real ^ (flip ? mi : mr);
+ float32 e3 = neg_imag ^ (flip ? mr : mi);
- d[H4(i)] = float32_muladd(e2, e1, d[H4(i)], 0, fpst);
- d[H4(i + 1)] = float32_muladd(e4, e3, d[H4(i + 1)], 0, fpst);
+ for (j = i; j < i + eltspersegment; j += 2) {
+ float32 e2 = n[H4(j + flip)];
+ float32 e4 = e2;
+
+ d[H4(j)] = float32_muladd(e2, e1, d[H4(j)], 0, fpst);
+ d[H4(j + 1)] = float32_muladd(e4, e3, d[H4(j + 1)], 0, fpst);
+ }
}
clear_tail(d, opr_sz, simd_maxsz(desc));
}
@@ -427,6 +630,26 @@
clear_tail(d, opr_sz, simd_maxsz(desc));
}
+#define DO_2OP(NAME, FUNC, TYPE) \
+void HELPER(NAME)(void *vd, void *vn, void *stat, uint32_t desc) \
+{ \
+ intptr_t i, oprsz = simd_oprsz(desc); \
+ TYPE *d = vd, *n = vn; \
+ for (i = 0; i < oprsz / sizeof(TYPE); i++) { \
+ d[i] = FUNC(n[i], stat); \
+ } \
+}
+
+DO_2OP(gvec_frecpe_h, helper_recpe_f16, float16)
+DO_2OP(gvec_frecpe_s, helper_recpe_f32, float32)
+DO_2OP(gvec_frecpe_d, helper_recpe_f64, float64)
+
+DO_2OP(gvec_frsqrte_h, helper_rsqrte_f16, float16)
+DO_2OP(gvec_frsqrte_s, helper_rsqrte_f32, float32)
+DO_2OP(gvec_frsqrte_d, helper_rsqrte_f64, float64)
+
+#undef DO_2OP
+
/* Floating-point trigonometric starting value.
* See the ARM ARM pseudocode function FPTrigSMul.
*/
@@ -495,3 +718,51 @@
#endif
#undef DO_3OP
+
+/* For the indexed ops, SVE applies the index per 128-bit vector segment.
+ * For AdvSIMD, there is of course only one such vector segment.
+ */
+
+#define DO_MUL_IDX(NAME, TYPE, H) \
+void HELPER(NAME)(void *vd, void *vn, void *vm, void *stat, uint32_t desc) \
+{ \
+ intptr_t i, j, oprsz = simd_oprsz(desc), segment = 16 / sizeof(TYPE); \
+ intptr_t idx = simd_data(desc); \
+ TYPE *d = vd, *n = vn, *m = vm; \
+ for (i = 0; i < oprsz / sizeof(TYPE); i += segment) { \
+ TYPE mm = m[H(i + idx)]; \
+ for (j = 0; j < segment; j++) { \
+ d[i + j] = TYPE##_mul(n[i + j], mm, stat); \
+ } \
+ } \
+}
+
+DO_MUL_IDX(gvec_fmul_idx_h, float16, H2)
+DO_MUL_IDX(gvec_fmul_idx_s, float32, H4)
+DO_MUL_IDX(gvec_fmul_idx_d, float64, )
+
+#undef DO_MUL_IDX
+
+#define DO_FMLA_IDX(NAME, TYPE, H) \
+void HELPER(NAME)(void *vd, void *vn, void *vm, void *va, \
+ void *stat, uint32_t desc) \
+{ \
+ intptr_t i, j, oprsz = simd_oprsz(desc), segment = 16 / sizeof(TYPE); \
+ TYPE op1_neg = extract32(desc, SIMD_DATA_SHIFT, 1); \
+ intptr_t idx = desc >> (SIMD_DATA_SHIFT + 1); \
+ TYPE *d = vd, *n = vn, *m = vm, *a = va; \
+ op1_neg <<= (8 * sizeof(TYPE) - 1); \
+ for (i = 0; i < oprsz / sizeof(TYPE); i += segment) { \
+ TYPE mm = m[H(i + idx)]; \
+ for (j = 0; j < segment; j++) { \
+ d[i + j] = TYPE##_muladd(n[i + j] ^ op1_neg, \
+ mm, a[i + j], 0, stat); \
+ } \
+ } \
+}
+
+DO_FMLA_IDX(gvec_fmla_idx_h, float16, H2)
+DO_FMLA_IDX(gvec_fmla_idx_s, float32, H4)
+DO_FMLA_IDX(gvec_fmla_idx_d, float64, )
+
+#undef DO_FMLA_IDX
diff --git a/target/i386/cpu.c b/target/i386/cpu.c
index e6c2f8a..1e6a7d0 100644
--- a/target/i386/cpu.c
+++ b/target/i386/cpu.c
@@ -3959,11 +3959,11 @@
}
break;
case 5:
- /* mwait info: needed for Core compatibility */
- *eax = 0; /* Smallest monitor-line size in bytes */
- *ebx = 0; /* Largest monitor-line size in bytes */
- *ecx = CPUID_MWAIT_EMX | CPUID_MWAIT_IBE;
- *edx = 0;
+ /* MONITOR/MWAIT Leaf */
+ *eax = cpu->mwait.eax; /* Smallest monitor-line size in bytes */
+ *ebx = cpu->mwait.ebx; /* Largest monitor-line size in bytes */
+ *ecx = cpu->mwait.ecx; /* flags */
+ *edx = cpu->mwait.edx; /* mwait substates */
break;
case 6:
/* Thermal and Power Leaf */
@@ -4804,13 +4804,25 @@
Error *local_err = NULL;
static bool ht_warned;
- if (xcc->host_cpuid_required && !accel_uses_host_cpuid()) {
- char *name = x86_cpu_class_get_model_name(xcc);
- error_setg(&local_err, "CPU model '%s' requires KVM", name);
- g_free(name);
- goto out;
+ if (xcc->host_cpuid_required) {
+ if (!accel_uses_host_cpuid()) {
+ char *name = x86_cpu_class_get_model_name(xcc);
+ error_setg(&local_err, "CPU model '%s' requires KVM", name);
+ g_free(name);
+ goto out;
+ }
+
+ if (enable_cpu_pm) {
+ host_cpuid(5, 0, &cpu->mwait.eax, &cpu->mwait.ebx,
+ &cpu->mwait.ecx, &cpu->mwait.edx);
+ env->features[FEAT_1_ECX] |= CPUID_EXT_MONITOR;
+ }
}
+ /* mwait extended info: needed for Core compatibility */
+ /* We always wake on interrupt even if host does not have the capability */
+ cpu->mwait.ecx |= CPUID_MWAIT_EMX | CPUID_MWAIT_IBE;
+
if (cpu->apic_id == UNASSIGNED_APIC_ID) {
error_setg(errp, "apic-id property was not initialized properly");
return;
diff --git a/target/i386/cpu.h b/target/i386/cpu.h
index 89c82be..8eaefee 100644
--- a/target/i386/cpu.h
+++ b/target/i386/cpu.h
@@ -1382,6 +1382,15 @@
/* if true the CPUID code directly forward host cache leaves to the guest */
bool cache_info_passthrough;
+ /* if true the CPUID code directly forwards
+ * host monitor/mwait leaves to the guest */
+ struct {
+ uint32_t eax;
+ uint32_t ebx;
+ uint32_t ecx;
+ uint32_t edx;
+ } mwait;
+
/* Features that were filtered out because of missing host capabilities */
uint32_t filtered_features[FEATURE_WORDS];
@@ -1840,8 +1849,8 @@
/* svm_helper.c */
void cpu_svm_check_intercept_param(CPUX86State *env1, uint32_t type,
uint64_t param, uintptr_t retaddr);
-void cpu_vmexit(CPUX86State *nenv, uint32_t exit_code, uint64_t exit_info_1,
- uintptr_t retaddr);
+void QEMU_NORETURN cpu_vmexit(CPUX86State *nenv, uint32_t exit_code,
+ uint64_t exit_info_1, uintptr_t retaddr);
void do_vmexit(CPUX86State *env, uint32_t exit_code, uint64_t exit_info_1);
/* seg_helper.c */
diff --git a/target/i386/kvm.c b/target/i386/kvm.c
index 5116a8a..032f0ad 100644
--- a/target/i386/kvm.c
+++ b/target/i386/kvm.c
@@ -366,6 +366,15 @@
if (!kvm_irqchip_in_kernel()) {
ret &= ~CPUID_EXT_X2APIC;
}
+
+ if (enable_cpu_pm) {
+ int disable_exits = kvm_check_extension(s,
+ KVM_CAP_X86_DISABLE_EXITS);
+
+ if (disable_exits & KVM_X86_DISABLE_EXITS_MWAIT) {
+ ret |= CPUID_EXT_MONITOR;
+ }
+ }
} else if (function == 6 && reg == R_EAX) {
ret |= CPUID_6_EAX_ARAT; /* safe to allow because of emulated APIC */
} else if (function == 7 && index == 0 && reg == R_EBX) {
@@ -1387,6 +1396,29 @@
smram_machine_done.notify = register_smram_listener;
qemu_add_machine_init_done_notifier(&smram_machine_done);
}
+
+ if (enable_cpu_pm) {
+ int disable_exits = kvm_check_extension(s, KVM_CAP_X86_DISABLE_EXITS);
+ int ret;
+
+/* Work around for kernel header with a typo. TODO: fix header and drop. */
+#if defined(KVM_X86_DISABLE_EXITS_HTL) && !defined(KVM_X86_DISABLE_EXITS_HLT)
+#define KVM_X86_DISABLE_EXITS_HLT KVM_X86_DISABLE_EXITS_HTL
+#endif
+ if (disable_exits) {
+ disable_exits &= (KVM_X86_DISABLE_EXITS_MWAIT |
+ KVM_X86_DISABLE_EXITS_HLT |
+ KVM_X86_DISABLE_EXITS_PAUSE);
+ }
+
+ ret = kvm_vm_enable_cap(s, KVM_CAP_X86_DISABLE_EXITS, 0,
+ disable_exits);
+ if (ret < 0) {
+ error_report("kvm: guest stopping CPU not supported: %s",
+ strerror(-ret));
+ }
+ }
+
return 0;
}
diff --git a/target/i386/monitor.c b/target/i386/monitor.c
index a890b3c..6bbb3a9 100644
--- a/target/i386/monitor.c
+++ b/target/i386/monitor.c
@@ -658,12 +658,8 @@
void hmp_info_io_apic(Monitor *mon, const QDict *qdict)
{
- if (kvm_irqchip_in_kernel() &&
- !kvm_irqchip_is_split()) {
- kvm_ioapic_dump_state(mon, qdict);
- } else {
- ioapic_dump_state(mon, qdict);
- }
+ monitor_printf(mon, "This command is obsolete and will be "
+ "removed soon. Please use 'info pic' instead.\n");
}
SevInfo *qmp_query_sev(Error **errp)
diff --git a/target/i386/seg_helper.c b/target/i386/seg_helper.c
index 600a4d7..00301a0 100644
--- a/target/i386/seg_helper.c
+++ b/target/i386/seg_helper.c
@@ -1337,6 +1337,7 @@
ret = true;
} else if ((interrupt_request & CPU_INTERRUPT_NMI) &&
!(env->hflags2 & HF2_NMI_MASK)) {
+ cpu_svm_check_intercept_param(env, SVM_EXIT_NMI, 0, 0);
cs->interrupt_request &= ~CPU_INTERRUPT_NMI;
env->hflags2 |= HF2_NMI_MASK;
do_interrupt_x86_hardirq(env, EXCP02_NMI, 1);
diff --git a/target/i386/svm_helper.c b/target/i386/svm_helper.c
index 3504923..f245aec 100644
--- a/target/i386/svm_helper.c
+++ b/target/i386/svm_helper.c
@@ -62,6 +62,7 @@
void cpu_vmexit(CPUX86State *nenv, uint32_t exit_code, uint64_t exit_info_1,
uintptr_t retaddr)
{
+ assert(0);
}
void helper_svm_check_intercept_param(CPUX86State *env, uint32_t type,
diff --git a/target/i386/translate.c b/target/i386/translate.c
index 697a918..07d185e 100644
--- a/target/i386/translate.c
+++ b/target/i386/translate.c
@@ -4059,34 +4059,26 @@
ot = mo_64_32(s->dflag);
gen_ldst_modrm(env, s, modrm, ot, OR_TMP0, 0);
+ tcg_gen_mov_tl(cpu_cc_src, cpu_T0);
switch (reg & 7) {
case 1: /* blsr By,Ey */
+ tcg_gen_subi_tl(cpu_T1, cpu_T0, 1);
+ tcg_gen_and_tl(cpu_T0, cpu_T0, cpu_T1);
+ break;
+ case 2: /* blsmsk By,Ey */
+ tcg_gen_subi_tl(cpu_T1, cpu_T0, 1);
+ tcg_gen_xor_tl(cpu_T0, cpu_T0, cpu_T1);
+ break;
+ case 3: /* blsi By, Ey */
tcg_gen_neg_tl(cpu_T1, cpu_T0);
tcg_gen_and_tl(cpu_T0, cpu_T0, cpu_T1);
- gen_op_mov_reg_v(ot, s->vex_v, cpu_T0);
- gen_op_update2_cc();
- set_cc_op(s, CC_OP_BMILGB + ot);
break;
-
- case 2: /* blsmsk By,Ey */
- tcg_gen_mov_tl(cpu_cc_src, cpu_T0);
- tcg_gen_subi_tl(cpu_T0, cpu_T0, 1);
- tcg_gen_xor_tl(cpu_T0, cpu_T0, cpu_cc_src);
- tcg_gen_mov_tl(cpu_cc_dst, cpu_T0);
- set_cc_op(s, CC_OP_BMILGB + ot);
- break;
-
- case 3: /* blsi By, Ey */
- tcg_gen_mov_tl(cpu_cc_src, cpu_T0);
- tcg_gen_subi_tl(cpu_T0, cpu_T0, 1);
- tcg_gen_and_tl(cpu_T0, cpu_T0, cpu_cc_src);
- tcg_gen_mov_tl(cpu_cc_dst, cpu_T0);
- set_cc_op(s, CC_OP_BMILGB + ot);
- break;
-
default:
goto unknown_op;
}
+ tcg_gen_mov_tl(cpu_cc_dst, cpu_T0);
+ gen_op_mov_reg_v(ot, s->vex_v, cpu_T0);
+ set_cc_op(s, CC_OP_BMILGB + ot);
break;
default:
@@ -7452,8 +7444,9 @@
break;
}
gen_update_cc_op(s);
- gen_jmp_im(pc_start - s->cs_base);
gen_helper_stgi(cpu_env);
+ gen_jmp_im(s->pc - s->cs_base);
+ gen_eob(s);
break;
case 0xdd: /* CLGI */
diff --git a/target/i386/whp-dispatch.h b/target/i386/whp-dispatch.h
new file mode 100644
index 0000000..d8d3485
--- /dev/null
+++ b/target/i386/whp-dispatch.h
@@ -0,0 +1,56 @@
+#include "windows.h"
+#include <stdbool.h>
+
+#include <WinHvPlatform.h>
+#include <WinHvEmulation.h>
+
+#ifndef WHP_DISPATCH_H
+#define WHP_DISPATCH_H
+
+
+#define LIST_WINHVPLATFORM_FUNCTIONS(X) \
+ X(HRESULT, WHvGetCapability, (WHV_CAPABILITY_CODE CapabilityCode, VOID* CapabilityBuffer, UINT32 CapabilityBufferSizeInBytes, UINT32* WrittenSizeInBytes)) \
+ X(HRESULT, WHvCreatePartition, (WHV_PARTITION_HANDLE* Partition)) \
+ X(HRESULT, WHvSetupPartition, (WHV_PARTITION_HANDLE Partition)) \
+ X(HRESULT, WHvDeletePartition, (WHV_PARTITION_HANDLE Partition)) \
+ X(HRESULT, WHvGetPartitionProperty, (WHV_PARTITION_HANDLE Partition, WHV_PARTITION_PROPERTY_CODE PropertyCode, VOID* PropertyBuffer, UINT32 PropertyBufferSizeInBytes, UINT32* WrittenSizeInBytes)) \
+ X(HRESULT, WHvSetPartitionProperty, (WHV_PARTITION_HANDLE Partition, WHV_PARTITION_PROPERTY_CODE PropertyCode, const VOID* PropertyBuffer, UINT32 PropertyBufferSizeInBytes)) \
+ X(HRESULT, WHvMapGpaRange, (WHV_PARTITION_HANDLE Partition, VOID* SourceAddress, WHV_GUEST_PHYSICAL_ADDRESS GuestAddress, UINT64 SizeInBytes, WHV_MAP_GPA_RANGE_FLAGS Flags)) \
+ X(HRESULT, WHvUnmapGpaRange, (WHV_PARTITION_HANDLE Partition, WHV_GUEST_PHYSICAL_ADDRESS GuestAddress, UINT64 SizeInBytes)) \
+ X(HRESULT, WHvTranslateGva, (WHV_PARTITION_HANDLE Partition, UINT32 VpIndex, WHV_GUEST_VIRTUAL_ADDRESS Gva, WHV_TRANSLATE_GVA_FLAGS TranslateFlags, WHV_TRANSLATE_GVA_RESULT* TranslationResult, WHV_GUEST_PHYSICAL_ADDRESS* Gpa)) \
+ X(HRESULT, WHvCreateVirtualProcessor, (WHV_PARTITION_HANDLE Partition, UINT32 VpIndex, UINT32 Flags)) \
+ X(HRESULT, WHvDeleteVirtualProcessor, (WHV_PARTITION_HANDLE Partition, UINT32 VpIndex)) \
+ X(HRESULT, WHvRunVirtualProcessor, (WHV_PARTITION_HANDLE Partition, UINT32 VpIndex, VOID* ExitContext, UINT32 ExitContextSizeInBytes)) \
+ X(HRESULT, WHvCancelRunVirtualProcessor, (WHV_PARTITION_HANDLE Partition, UINT32 VpIndex, UINT32 Flags)) \
+ X(HRESULT, WHvGetVirtualProcessorRegisters, (WHV_PARTITION_HANDLE Partition, UINT32 VpIndex, const WHV_REGISTER_NAME* RegisterNames, UINT32 RegisterCount, WHV_REGISTER_VALUE* RegisterValues)) \
+ X(HRESULT, WHvSetVirtualProcessorRegisters, (WHV_PARTITION_HANDLE Partition, UINT32 VpIndex, const WHV_REGISTER_NAME* RegisterNames, UINT32 RegisterCount, const WHV_REGISTER_VALUE* RegisterValues)) \
+
+
+#define LIST_WINHVEMULATION_FUNCTIONS(X) \
+ X(HRESULT, WHvEmulatorCreateEmulator, (const WHV_EMULATOR_CALLBACKS* Callbacks, WHV_EMULATOR_HANDLE* Emulator)) \
+ X(HRESULT, WHvEmulatorDestroyEmulator, (WHV_EMULATOR_HANDLE Emulator)) \
+ X(HRESULT, WHvEmulatorTryIoEmulation, (WHV_EMULATOR_HANDLE Emulator, VOID* Context, const WHV_VP_EXIT_CONTEXT* VpContext, const WHV_X64_IO_PORT_ACCESS_CONTEXT* IoInstructionContext, WHV_EMULATOR_STATUS* EmulatorReturnStatus)) \
+ X(HRESULT, WHvEmulatorTryMmioEmulation, (WHV_EMULATOR_HANDLE Emulator, VOID* Context, const WHV_VP_EXIT_CONTEXT* VpContext, const WHV_MEMORY_ACCESS_CONTEXT* MmioInstructionContext, WHV_EMULATOR_STATUS* EmulatorReturnStatus)) \
+
+
+#define WHP_DEFINE_TYPE(return_type, function_name, signature) \
+ typedef return_type (WINAPI *function_name ## _t) signature;
+
+#define WHP_DECLARE_MEMBER(return_type, function_name, signature) \
+ function_name ## _t function_name;
+
+/* Define function typedef */
+LIST_WINHVPLATFORM_FUNCTIONS(WHP_DEFINE_TYPE)
+LIST_WINHVEMULATION_FUNCTIONS(WHP_DEFINE_TYPE)
+
+struct WHPDispatch {
+ LIST_WINHVPLATFORM_FUNCTIONS(WHP_DECLARE_MEMBER)
+ LIST_WINHVEMULATION_FUNCTIONS(WHP_DECLARE_MEMBER)
+};
+
+extern struct WHPDispatch whp_dispatch;
+
+bool init_whp_dispatch(void);
+
+
+#endif /* WHP_DISPATCH_H */
diff --git a/target/i386/whpx-all.c b/target/i386/whpx-all.c
index 6b42096..57e53e1 100644
--- a/target/i386/whpx-all.c
+++ b/target/i386/whpx-all.c
@@ -932,6 +932,7 @@
case WHvRunVpExitReasonX64InterruptWindow:
vcpu->window_registered = 0;
+ ret = 0;
break;
case WHvRunVpExitReasonX64Halt:
@@ -943,6 +944,40 @@
ret = 1;
break;
+ case WHvRunVpExitReasonX64MsrAccess: {
+ WHV_REGISTER_VALUE reg_values[3] = {0};
+ WHV_REGISTER_NAME reg_names[3];
+ UINT32 reg_count;
+
+ reg_names[0] = WHvX64RegisterRip;
+ reg_names[1] = WHvX64RegisterRax;
+ reg_names[2] = WHvX64RegisterRdx;
+
+ reg_values[0].Reg64 =
+ vcpu->exit_ctx.VpContext.Rip +
+ vcpu->exit_ctx.VpContext.InstructionLength;
+
+ /*
+ * For all unsupported MSR access we:
+ * ignore writes
+ * return 0 on read.
+ */
+ reg_count = vcpu->exit_ctx.MsrAccess.AccessInfo.IsWrite ?
+ 1 : 3;
+
+ hr = whp_dispatch.WHvSetVirtualProcessorRegisters(
+ whpx->partition,
+ cpu->cpu_index,
+ reg_names, reg_count,
+ reg_values);
+
+ if (FAILED(hr)) {
+ error_report("WHPX: Failed to set MsrAccess state "
+ " registers, hr=%08lx", hr);
+ }
+ ret = 0;
+ break;
+ }
case WHvRunVpExitReasonX64Cpuid: {
WHV_REGISTER_VALUE reg_values[5];
WHV_REGISTER_NAME reg_names[5];
@@ -964,6 +999,16 @@
rdx = vcpu->exit_ctx.CpuidAccess.DefaultResultRdx;
rbx = vcpu->exit_ctx.CpuidAccess.DefaultResultRbx;
break;
+ case 0x80000001:
+ rax = vcpu->exit_ctx.CpuidAccess.DefaultResultRax;
+ /* Remove any support of OSVW */
+ rcx =
+ vcpu->exit_ctx.CpuidAccess.DefaultResultRcx &
+ ~CPUID_EXT3_OSVW;
+
+ rdx = vcpu->exit_ctx.CpuidAccess.DefaultResultRdx;
+ rbx = vcpu->exit_ctx.CpuidAccess.DefaultResultRbx;
+ break;
default:
rax = vcpu->exit_ctx.CpuidAccess.DefaultResultRax;
rcx = vcpu->exit_ctx.CpuidAccess.DefaultResultRcx;
@@ -1000,7 +1045,6 @@
case WHvRunVpExitReasonUnrecoverableException:
case WHvRunVpExitReasonInvalidVpRegisterValue:
case WHvRunVpExitReasonUnsupportedFeature:
- case WHvRunVpExitReasonX64MsrAccess:
case WHvRunVpExitReasonException:
default:
error_report("WHPX: Unexpected VP exit code %d",
@@ -1368,6 +1412,7 @@
}
memset(&prop, 0, sizeof(WHV_PARTITION_PROPERTY));
+ prop.ExtendedVmExits.X64MsrExit = 1;
prop.ExtendedVmExits.X64CpuidExit = 1;
hr = whp_dispatch.WHvSetPartitionProperty(
whpx->partition,
@@ -1376,18 +1421,19 @@
sizeof(WHV_PARTITION_PROPERTY));
if (FAILED(hr)) {
- error_report("WHPX: Failed to enable partition extended X64CpuidExit"
- " hr=%08lx", hr);
+ error_report("WHPX: Failed to enable partition extended X64MsrExit and"
+ " X64CpuidExit hr=%08lx", hr);
ret = -EINVAL;
goto error;
}
- UINT32 cpuidExitList[] = {1};
+ UINT32 cpuidExitList[] = {1, 0x80000001};
hr = whp_dispatch.WHvSetPartitionProperty(
whpx->partition,
WHvPartitionPropertyCodeCpuidExitList,
cpuidExitList,
RTL_NUMBER_OF(cpuidExitList) * sizeof(UINT32));
+
if (FAILED(hr)) {
error_report("WHPX: Failed to set partition CpuidExitList hr=%08lx",
hr);
diff --git a/target/xtensa/op_helper.c b/target/xtensa/op_helper.c
index 8a8c763..bbbbb33 100644
--- a/target/xtensa/op_helper.c
+++ b/target/xtensa/op_helper.c
@@ -36,11 +36,6 @@
#include "qemu/timer.h"
#include "fpu/softfloat.h"
-#ifdef CONFIG_USER_ONLY
-/* tb_invalidate_phys_range */
-#include "accel/tcg/translate-all.h"
-#endif
-
#ifndef CONFIG_USER_ONLY
void xtensa_cpu_do_unaligned_access(CPUState *cs,
@@ -114,9 +109,7 @@
static void tb_invalidate_virtual_addr(CPUXtensaState *env, uint32_t vaddr)
{
- mmap_lock();
- tb_invalidate_phys_range(vaddr, vaddr + 1);
- mmap_unlock();
+ tb_invalidate_phys_addr(vaddr);
}
#endif
diff --git a/tests/atomic_add-bench.c b/tests/atomic_add-bench.c
index caa1e8e..f96d448 100644
--- a/tests/atomic_add-bench.c
+++ b/tests/atomic_add-bench.c
@@ -8,6 +8,7 @@
} QEMU_ALIGNED(64);
struct count {
+ QemuMutex lock;
unsigned long val;
} QEMU_ALIGNED(64);
@@ -18,11 +19,13 @@
static struct count *counts;
static unsigned int duration = 1;
static unsigned int range = 1024;
+static bool use_mutex;
static bool test_start;
static bool test_stop;
static const char commands_string[] =
" -n = number of threads\n"
+ " -m = use mutexes instead of atomic increments\n"
" -d = duration in seconds\n"
" -r = range (will be rounded up to pow2)";
@@ -59,7 +62,13 @@
info->r = xorshift64star(info->r);
index = info->r & (range - 1);
- atomic_inc(&counts[index].val);
+ if (use_mutex) {
+ qemu_mutex_lock(&counts[index].lock);
+ counts[index].val += 1;
+ qemu_mutex_unlock(&counts[index].lock);
+ } else {
+ atomic_inc(&counts[index].val);
+ }
}
return NULL;
}
@@ -91,6 +100,9 @@
th_info = g_new(struct thread_info, n_threads);
counts = qemu_memalign(64, sizeof(*counts) * range);
memset(counts, 0, sizeof(*counts) * range);
+ for (i = 0; i < range; i++) {
+ qemu_mutex_init(&counts[i].lock);
+ }
for (i = 0; i < n_threads; i++) {
struct thread_info *info = &th_info[i];
@@ -131,7 +143,7 @@
int c;
for (;;) {
- c = getopt(argc, argv, "hd:n:r:");
+ c = getopt(argc, argv, "hd:n:mr:");
if (c < 0) {
break;
}
@@ -145,6 +157,9 @@
case 'n':
n_threads = atoi(optarg);
break;
+ case 'm':
+ use_mutex = true;
+ break;
case 'r':
range = pow2ceil(atoi(optarg));
break;
diff --git a/tests/boot-serial-test.c b/tests/boot-serial-test.c
index 4d6815c..952a2e7 100644
--- a/tests/boot-serial-test.c
+++ b/tests/boot-serial-test.c
@@ -111,9 +111,8 @@
{ NULL }
};
-static void check_guest_output(const testdef_t *test, int fd)
+static bool check_guest_output(const testdef_t *test, int fd)
{
- bool output_ok = false;
int i, nbr = 0, pos = 0, ccnt;
char ch;
@@ -125,8 +124,7 @@
pos += 1;
if (test->expect[pos] == '\0') {
/* We've reached the end of the expected string! */
- output_ok = true;
- goto done;
+ return true;
}
} else {
pos = 0;
@@ -136,8 +134,7 @@
g_usleep(10000);
}
-done:
- g_assert(output_ok);
+ return false;
}
static void test_machine(const void *data)
@@ -180,12 +177,16 @@
"-no-shutdown -serial chardev:serial0 %s",
codeparam, code ? codetmp : "",
test->machine, serialtmp, test->extra);
- unlink(serialtmp);
if (code) {
unlink(codetmp);
}
- check_guest_output(test, ser_fd);
+ if (!check_guest_output(test, ser_fd)) {
+ g_error("Failed to find expected string. Please check '%s'",
+ serialtmp);
+ }
+ unlink(serialtmp);
+
qtest_quit(global_qtest);
close(ser_fd);
diff --git a/tests/docker/dockerfiles/centos6.docker b/tests/docker/dockerfiles/centos6.docker
deleted file mode 100644
index ad24319..0000000
--- a/tests/docker/dockerfiles/centos6.docker
+++ /dev/null
@@ -1,30 +0,0 @@
-FROM centos:6
-RUN yum install -y epel-release centos-release-xen
-ENV PACKAGES \
- bison \
- bzip2-devel \
- ccache \
- csnappy-devel \
- flex \
- g++ \
- gcc \
- gettext \
- git \
- glib2-devel \
- libepoxy-devel \
- libfdt-devel \
- librdmacm-devel \
- lzo-devel \
- make \
- mesa-libEGL-devel \
- mesa-libgbm-devel \
- pixman-devel \
- SDL-devel \
- spice-glib-devel \
- spice-server-devel \
- tar \
- vte-devel \
- xen-devel \
- zlib-devel
-RUN yum install -y $PACKAGES
-RUN rpm -q $PACKAGES | sort > /packages.txt
diff --git a/tests/docker/dockerfiles/min-glib.docker b/tests/docker/dockerfiles/min-glib.docker
deleted file mode 100644
index f2eed97..0000000
--- a/tests/docker/dockerfiles/min-glib.docker
+++ /dev/null
@@ -1,8 +0,0 @@
-FROM centos:6
-RUN yum install -y \
- tar gettext git make gcc g++ \
- zlib-devel SDL-devel pixman-devel \
- epel-release
-RUN yum install -y libfdt-devel ccache
-RUN yum downgrade -y http://vault.centos.org/6.0/os/x86_64/Packages/glib2-2.22.5-5.el6.x86_64.rpm
-RUN yum install -y http://vault.centos.org/6.0/os/x86_64/Packages/glib2-devel-2.22.5-5.el6.x86_64.rpm
diff --git a/tests/ivshmem-test.c b/tests/ivshmem-test.c
index 8af16ee..9b407a3 100644
--- a/tests/ivshmem-test.c
+++ b/tests/ivshmem-test.c
@@ -504,12 +504,6 @@
const char *arch = qtest_get_arch();
gchar dir[] = "/tmp/ivshmem-test.XXXXXX";
-#if !GLIB_CHECK_VERSION(2, 31, 0)
- if (!g_thread_supported()) {
- g_thread_init(NULL);
- }
-#endif
-
g_test_init(&argc, &argv, NULL);
qtest_add_abrt_handler(abrt_handler, NULL);
diff --git a/tests/qemu-iotests/026 b/tests/qemu-iotests/026
index 7fadfba..582d254 100755
--- a/tests/qemu-iotests/026
+++ b/tests/qemu-iotests/026
@@ -200,6 +200,23 @@
done
done
+echo
+echo === Avoid cluster leaks after temporary failure ===
+echo
+
+cat > "$TEST_DIR/blkdebug.conf" <<EOF
+[inject-error]
+event = "write_aio"
+errno = "5"
+once = "on"
+EOF
+
+# After the failed first write, do a second write so that the updated refcount
+# block is actually written back
+_make_test_img 64M
+$QEMU_IO -c "write 0 1M" -c "write 0 1M" "$BLKDBG_TEST_IMG" | _filter_qemu_io
+_check_test_img
+
# success, all done
echo "*** done"
rm -f $seq.full
diff --git a/tests/qemu-iotests/026.out b/tests/qemu-iotests/026.out
index 8e89416..dd10a82 100644
--- a/tests/qemu-iotests/026.out
+++ b/tests/qemu-iotests/026.out
@@ -675,4 +675,12 @@
96 leaked clusters were found on the image.
This means waste of disk space, but no harm to data.
+
+=== Avoid cluster leaks after temporary failure ===
+
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864
+write failed: Input/output error
+wrote 1048576/1048576 bytes at offset 0
+1 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+No errors were found on the image.
*** done
diff --git a/tests/qemu-iotests/026.out.nocache b/tests/qemu-iotests/026.out.nocache
index ea2e166..1ca6cda 100644
--- a/tests/qemu-iotests/026.out.nocache
+++ b/tests/qemu-iotests/026.out.nocache
@@ -541,7 +541,7 @@
Failed to flush the refcount block cache: No space left on device
write failed: No space left on device
-11 leaked clusters were found on the image.
+10 leaked clusters were found on the image.
This means waste of disk space, but no harm to data.
Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
@@ -569,7 +569,7 @@
Failed to flush the refcount block cache: No space left on device
write failed: No space left on device
-11 leaked clusters were found on the image.
+10 leaked clusters were found on the image.
This means waste of disk space, but no harm to data.
Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
@@ -597,7 +597,7 @@
Failed to flush the refcount block cache: No space left on device
write failed: No space left on device
-11 leaked clusters were found on the image.
+10 leaked clusters were found on the image.
This means waste of disk space, but no harm to data.
Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
@@ -683,4 +683,12 @@
96 leaked clusters were found on the image.
This means waste of disk space, but no harm to data.
+
+=== Avoid cluster leaks after temporary failure ===
+
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864
+write failed: Input/output error
+wrote 1048576/1048576 bytes at offset 0
+1 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+No errors were found on the image.
*** done
diff --git a/tests/qemu-iotests/063 b/tests/qemu-iotests/063
index e4f6ea9..adc037c 100755
--- a/tests/qemu-iotests/063
+++ b/tests/qemu-iotests/063
@@ -91,6 +91,15 @@
exit 1
fi
+echo "== Regression testing for copy offloading bug =="
+
+_make_test_img 1M
+TEST_IMG="$TEST_IMG.target" _make_test_img 1M
+$QEMU_IO -c 'write -P 1 0 512k' -c 'write -P 2 512k 512k' "$TEST_IMG" | _filter_qemu_io
+$QEMU_IO -c 'write -P 4 512k 512k' -c 'write -P 3 0 512k' "$TEST_IMG.target" | _filter_qemu_io
+$QEMU_IMG convert -n -O $IMGFMT "$TEST_IMG" "$TEST_IMG.target"
+$QEMU_IMG compare "$TEST_IMG" "$TEST_IMG.target"
+
echo "*** done"
rm -f $seq.full
status=0
diff --git a/tests/qemu-iotests/063.out b/tests/qemu-iotests/063.out
index de1c99a..7b691b2 100644
--- a/tests/qemu-iotests/063.out
+++ b/tests/qemu-iotests/063.out
@@ -7,4 +7,16 @@
No errors were found on the image.
== Testing conversion to a smaller file fails ==
Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=2097152
+== Regression testing for copy offloading bug ==
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1048576
+Formatting 'TEST_DIR/t.IMGFMT.target', fmt=IMGFMT size=1048576
+wrote 524288/524288 bytes at offset 0
+512 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 524288/524288 bytes at offset 524288
+512 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 524288/524288 bytes at offset 524288
+512 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 524288/524288 bytes at offset 0
+512 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+Images are identical.
*** done
diff --git a/tests/test-qga.c b/tests/test-qga.c
index 18e63cb..30c9643 100644
--- a/tests/test-qga.c
+++ b/tests/test-qga.c
@@ -744,12 +744,10 @@
strv = g_key_file_get_string_list(kf, "general", "blacklist", &n, &error);
g_assert_cmpint(n, ==, 2);
-#if GLIB_CHECK_VERSION(2, 44, 0)
g_assert_true(g_strv_contains((const char * const *)strv,
"guest-ping"));
g_assert_true(g_strv_contains((const char * const *)strv,
"guest-get-time"));
-#endif
g_assert_no_error(error);
g_strfreev(strv);
diff --git a/tests/test-qmp-event.c b/tests/test-qmp-event.c
index 3a7c227..8677094 100644
--- a/tests/test-qmp-event.c
+++ b/tests/test-qmp-event.c
@@ -32,7 +32,7 @@
} QDictCmpData;
TestEventData *test_event_data;
-static CompatGMutex test_event_lock;
+static GMutex test_event_lock;
/* Only compares bool, int, string */
static
@@ -242,12 +242,6 @@
int main(int argc, char **argv)
{
-#if !GLIB_CHECK_VERSION(2, 31, 0)
- if (!g_thread_supported()) {
- g_thread_init(NULL);
- }
-#endif
-
qmp_event_set_func_emit(event_test_emit);
g_test_init(&argc, &argv, NULL);
diff --git a/tests/tpm-emu.h b/tests/tpm-emu.h
index ef4bfa8..08f9024 100644
--- a/tests/tpm-emu.h
+++ b/tests/tpm-emu.h
@@ -24,8 +24,8 @@
} QEMU_PACKED;
typedef struct TestState {
- CompatGMutex data_mutex;
- CompatGCond data_cond;
+ GMutex data_mutex;
+ GCond data_cond;
SocketAddress *addr;
QIOChannel *tpm_ioc;
GThread *emu_tpm_thread;
diff --git a/tests/vhost-user-test.c b/tests/vhost-user-test.c
index bbc8091..8ff2106 100644
--- a/tests/vhost-user-test.c
+++ b/tests/vhost-user-test.c
@@ -32,14 +32,6 @@
#include <linux/virtio_net.h>
#include <sys/vfs.h>
-/* GLIB version compatibility flags */
-#if !GLIB_CHECK_VERSION(2, 26, 0)
-#define G_TIME_SPAN_SECOND (G_GINT64_CONSTANT(1000000))
-#endif
-
-#if GLIB_CHECK_VERSION(2, 28, 0)
-#define HAVE_MONOTONIC_TIME
-#endif
#define QEMU_CMD_MEM " -m %d -object memory-backend-file,id=mem,size=%dM," \
"mem-path=%s,share=on -numa node,memdev=mem"
@@ -150,8 +142,8 @@
int fds_num;
int fds[VHOST_MEMORY_MAX_NREGIONS];
VhostUserMemory memory;
- CompatGMutex data_mutex;
- CompatGCond data_cond;
+ GMutex data_mutex;
+ GCond data_cond;
int log_fd;
uint64_t rings;
bool test_fail;
@@ -642,21 +634,7 @@
return FALSE;
}
-#if !GLIB_CHECK_VERSION(2,36,0)
-/* this callback is unnecessary with glib >2.36, the default
- * prepare for the source does the same */
-static gboolean
-test_migrate_source_prepare(GSource *source, gint *timeout)
-{
- *timeout = -1;
- return FALSE;
-}
-#endif
-
GSourceFuncs test_migrate_source_funcs = {
-#if !GLIB_CHECK_VERSION(2,36,0)
- .prepare = test_migrate_source_prepare,
-#endif
.check = test_migrate_source_check,
};
diff --git a/trace/control-target.c b/trace/control-target.c
index 706b2ce..ceb55c7 100644
--- a/trace/control-target.c
+++ b/trace/control-target.c
@@ -11,7 +11,6 @@
#include "cpu.h"
#include "trace-root.h"
#include "trace/control.h"
-#include "translate-all.h"
void trace_event_set_state_dynamic_init(TraceEvent *ev, bool state)
diff --git a/trace/simple.c b/trace/simple.c
index e82018d..701dec6 100644
--- a/trace/simple.c
+++ b/trace/simple.c
@@ -36,9 +36,9 @@
* Trace records are written out by a dedicated thread. The thread waits for
* records to become available, writes them out, and then waits again.
*/
-static CompatGMutex trace_lock;
-static CompatGCond trace_available_cond;
-static CompatGCond trace_empty_cond;
+static GMutex trace_lock;
+static GCond trace_available_cond;
+static GCond trace_empty_cond;
static bool trace_available;
static bool trace_writeout_enabled;
diff --git a/util/cutils.c b/util/cutils.c
index 0de69e6..9205e09 100644
--- a/util/cutils.c
+++ b/util/cutils.c
@@ -545,6 +545,21 @@
}
/**
+ * Searches for the first occurrence of 'c' in 's', and returns a pointer
+ * to the trailing null byte if none was found.
+ */
+#ifndef HAVE_STRCHRNUL
+const char *qemu_strchrnul(const char *s, int c)
+{
+ const char *e = strchr(s, c);
+ if (!e) {
+ e = s + strlen(s);
+ }
+ return e;
+}
+#endif
+
+/**
* parse_uint:
*
* @s: String to parse
diff --git a/util/iova-tree.c b/util/iova-tree.c
index 2d9cebf..7990692 100644
--- a/util/iova-tree.c
+++ b/util/iova-tree.c
@@ -9,7 +9,7 @@
* This work is licensed under the terms of the GNU GPL, version 2 or later.
*/
-#include <glib.h>
+#include "qemu/osdep.h"
#include "qemu/iova-tree.h"
struct IOVATree {
diff --git a/util/osdep.c b/util/osdep.c
index ea51d50..1c8d1e2 100644
--- a/util/osdep.c
+++ b/util/osdep.c
@@ -504,20 +504,6 @@
return 0;
}
-#if !GLIB_CHECK_VERSION(2, 31, 0)
-/* Ensure that glib is running in multi-threaded mode
- * Old versions of glib require explicit initialization. Failure to do
- * this results in the single-threaded code paths being taken inside
- * glib. For example, the g_slice allocator will not be thread-safe
- * and cause crashes.
- */
-static void __attribute__((constructor)) thread_init(void)
-{
- if (!g_thread_supported()) {
- g_thread_init(NULL);
- }
-}
-#endif
#ifndef CONFIG_IOVEC
/* helper function for iov_send_recv() */
diff --git a/util/qemu-option.c b/util/qemu-option.c
index ba44a08..19761e3 100644
--- a/util/qemu-option.c
+++ b/util/qemu-option.c
@@ -77,11 +77,7 @@
*value = NULL;
while (1) {
- offset = strchr(p, ',');
- if (!offset) {
- offset = p + strlen(p);
- }
-
+ offset = qemu_strchrnul(p, ',');
length = offset - p;
if (*offset != '\0' && *(offset + 1) == ',') {
length++;
diff --git a/util/qemu-thread-common.h b/util/qemu-thread-common.h
new file mode 100644
index 0000000..a0ea7c0
--- /dev/null
+++ b/util/qemu-thread-common.h
@@ -0,0 +1,55 @@
+/*
+ * Common qemu-thread implementation header file.
+ *
+ * Copyright Red Hat, Inc. 2018
+ *
+ * Authors:
+ * Peter Xu <peterx@redhat.com>,
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ */
+
+#ifndef QEMU_THREAD_COMMON_H
+#define QEMU_THREAD_COMMON_H
+
+#include "qemu/typedefs.h"
+#include "qemu/thread.h"
+#include "trace.h"
+
+static inline void qemu_mutex_post_init(QemuMutex *mutex)
+{
+#ifdef CONFIG_DEBUG_MUTEX
+ mutex->file = NULL;
+ mutex->line = 0;
+#endif
+ mutex->initialized = true;
+}
+
+static inline void qemu_mutex_pre_lock(QemuMutex *mutex,
+ const char *file, int line)
+{
+ trace_qemu_mutex_lock(mutex, file, line);
+}
+
+static inline void qemu_mutex_post_lock(QemuMutex *mutex,
+ const char *file, int line)
+{
+#ifdef CONFIG_DEBUG_MUTEX
+ mutex->file = file;
+ mutex->line = line;
+#endif
+ trace_qemu_mutex_locked(mutex, file, line);
+}
+
+static inline void qemu_mutex_pre_unlock(QemuMutex *mutex,
+ const char *file, int line)
+{
+#ifdef CONFIG_DEBUG_MUTEX
+ mutex->file = NULL;
+ mutex->line = 0;
+#endif
+ trace_qemu_mutex_unlock(mutex, file, line);
+}
+
+#endif
diff --git a/util/qemu-thread-posix.c b/util/qemu-thread-posix.c
index a1c34ba..dfa66ff 100644
--- a/util/qemu-thread-posix.c
+++ b/util/qemu-thread-posix.c
@@ -14,7 +14,7 @@
#include "qemu/thread.h"
#include "qemu/atomic.h"
#include "qemu/notify.h"
-#include "trace.h"
+#include "qemu-thread-common.h"
static bool name_threads;
@@ -43,7 +43,7 @@
err = pthread_mutex_init(&mutex->lock, NULL);
if (err)
error_exit(err, __func__);
- mutex->initialized = true;
+ qemu_mutex_post_init(mutex);
}
void qemu_mutex_destroy(QemuMutex *mutex)
@@ -62,13 +62,11 @@
int err;
assert(mutex->initialized);
- trace_qemu_mutex_lock(mutex, file, line);
-
+ qemu_mutex_pre_lock(mutex, file, line);
err = pthread_mutex_lock(&mutex->lock);
if (err)
error_exit(err, __func__);
-
- trace_qemu_mutex_locked(mutex, file, line);
+ qemu_mutex_post_lock(mutex, file, line);
}
int qemu_mutex_trylock_impl(QemuMutex *mutex, const char *file, const int line)
@@ -78,7 +76,7 @@
assert(mutex->initialized);
err = pthread_mutex_trylock(&mutex->lock);
if (err == 0) {
- trace_qemu_mutex_locked(mutex, file, line);
+ qemu_mutex_post_lock(mutex, file, line);
return 0;
}
if (err != EBUSY) {
@@ -92,11 +90,10 @@
int err;
assert(mutex->initialized);
+ qemu_mutex_pre_unlock(mutex, file, line);
err = pthread_mutex_unlock(&mutex->lock);
if (err)
error_exit(err, __func__);
-
- trace_qemu_mutex_unlock(mutex, file, line);
}
void qemu_rec_mutex_init(QemuRecMutex *mutex)
@@ -160,9 +157,9 @@
int err;
assert(cond->initialized);
- trace_qemu_mutex_unlock(mutex, file, line);
+ qemu_mutex_pre_unlock(mutex, file, line);
err = pthread_cond_wait(&cond->cond, &mutex->lock);
- trace_qemu_mutex_locked(mutex, file, line);
+ qemu_mutex_post_lock(mutex, file, line);
if (err)
error_exit(err, __func__);
}
diff --git a/util/qemu-thread-win32.c b/util/qemu-thread-win32.c
index ab60c0d..b303188 100644
--- a/util/qemu-thread-win32.c
+++ b/util/qemu-thread-win32.c
@@ -19,7 +19,7 @@
#include "qemu-common.h"
#include "qemu/thread.h"
#include "qemu/notify.h"
-#include "trace.h"
+#include "qemu-thread-common.h"
#include <process.h>
static bool name_threads;
@@ -46,7 +46,7 @@
void qemu_mutex_init(QemuMutex *mutex)
{
InitializeSRWLock(&mutex->lock);
- mutex->initialized = true;
+ qemu_mutex_post_init(mutex);
}
void qemu_mutex_destroy(QemuMutex *mutex)
@@ -59,10 +59,9 @@
void qemu_mutex_lock_impl(QemuMutex *mutex, const char *file, const int line)
{
assert(mutex->initialized);
- trace_qemu_mutex_lock(mutex, file, line);
-
+ qemu_mutex_pre_lock(mutex, file, line);
AcquireSRWLockExclusive(&mutex->lock);
- trace_qemu_mutex_locked(mutex, file, line);
+ qemu_mutex_post_lock(mutex, file, line);
}
int qemu_mutex_trylock_impl(QemuMutex *mutex, const char *file, const int line)
@@ -72,7 +71,7 @@
assert(mutex->initialized);
owned = TryAcquireSRWLockExclusive(&mutex->lock);
if (owned) {
- trace_qemu_mutex_locked(mutex, file, line);
+ qemu_mutex_post_lock(mutex, file, line);
return 0;
}
return -EBUSY;
@@ -81,7 +80,7 @@
void qemu_mutex_unlock_impl(QemuMutex *mutex, const char *file, const int line)
{
assert(mutex->initialized);
- trace_qemu_mutex_unlock(mutex, file, line);
+ qemu_mutex_pre_unlock(mutex, file, line);
ReleaseSRWLockExclusive(&mutex->lock);
}
@@ -145,9 +144,9 @@
void qemu_cond_wait_impl(QemuCond *cond, QemuMutex *mutex, const char *file, const int line)
{
assert(cond->initialized);
- trace_qemu_mutex_unlock(mutex, file, line);
+ qemu_mutex_pre_unlock(mutex, file, line);
SleepConditionVariableSRW(&cond->var, &mutex->lock, INFINITE, 0);
- trace_qemu_mutex_locked(mutex, file, line);
+ qemu_mutex_post_lock(mutex, file, line);
}
void qemu_sem_init(QemuSemaphore *sem, int init)
diff --git a/util/uri.c b/util/uri.c
index 8624a7a..8bdef84 100644
--- a/util/uri.c
+++ b/util/uri.c
@@ -52,6 +52,7 @@
*/
#include "qemu/osdep.h"
+#include "qemu/cutils.h"
#include "qemu/uri.h"
@@ -2266,10 +2267,7 @@
/* Find the next separator, or end of the string. */
end = strchr(query, '&');
if (!end) {
- end = strchr(query, ';');
- }
- if (!end) {
- end = query + strlen(query);
+ end = qemu_strchrnul(query, ';');
}
/* Find the first '=' character between here and end. */
diff --git a/vl.c b/vl.c
index d26f19b..ef6cfce 100644
--- a/vl.c
+++ b/vl.c
@@ -142,6 +142,7 @@
const char *mem_path = NULL;
int mem_prealloc = 0; /* force preallocation of physical target memory */
bool enable_mlock = false;
+bool enable_cpu_pm = false;
int nb_nics;
NICInfo nd_table[MAX_NICS];
int autostart;
@@ -390,6 +391,22 @@
},
};
+static QemuOptsList qemu_overcommit_opts = {
+ .name = "overcommit",
+ .head = QTAILQ_HEAD_INITIALIZER(qemu_overcommit_opts.head),
+ .desc = {
+ {
+ .name = "mem-lock",
+ .type = QEMU_OPT_BOOL,
+ },
+ {
+ .name = "cpu-pm",
+ .type = QEMU_OPT_BOOL,
+ },
+ { /* end of list */ }
+ },
+};
+
static QemuOptsList qemu_msg_opts = {
.name = "msg",
.head = QTAILQ_HEAD_INITIALIZER(qemu_msg_opts.head),
@@ -3581,6 +3598,7 @@
qemu_opts_parse_noisily(olist, "accel=kvm", false);
break;
case QEMU_OPTION_enable_hax:
+ warn_report("Option is deprecated, use '-accel hax' instead");
olist = qemu_find_opts("machine");
qemu_opts_parse_noisily(olist, "accel=hax", false);
break;
@@ -3905,7 +3923,20 @@
if (!opts) {
exit(1);
}
- enable_mlock = qemu_opt_get_bool(opts, "mlock", true);
+ /* Don't override the -overcommit option if set */
+ enable_mlock = enable_mlock ||
+ qemu_opt_get_bool(opts, "mlock", true);
+ break;
+ case QEMU_OPTION_overcommit:
+ opts = qemu_opts_parse_noisily(qemu_find_opts("overcommit"),
+ optarg, false);
+ if (!opts) {
+ exit(1);
+ }
+ /* Don't override the -realtime option if set */
+ enable_mlock = enable_mlock ||
+ qemu_opt_get_bool(opts, "mem-lock", false);
+ enable_cpu_pm = qemu_opt_get_bool(opts, "cpu-pm", false);
break;
case QEMU_OPTION_msg:
opts = qemu_opts_parse_noisily(qemu_find_opts("msg"), optarg,
diff --git a/win_dump.c b/win_dump.c
new file mode 100644
index 0000000..b15c191
--- /dev/null
+++ b/win_dump.c
@@ -0,0 +1,385 @@
+/*
+ * Windows crashdump
+ *
+ * Copyright (c) 2018 Virtuozzo International GmbH
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ *
+ */
+
+#include "qemu/osdep.h"
+#include "qemu/cutils.h"
+#include "elf.h"
+#include "cpu.h"
+#include "exec/hwaddr.h"
+#include "monitor/monitor.h"
+#include "sysemu/kvm.h"
+#include "sysemu/dump.h"
+#include "sysemu/sysemu.h"
+#include "sysemu/memory_mapping.h"
+#include "sysemu/cpus.h"
+#include "qapi/error.h"
+#include "qapi/qmp/qerror.h"
+#include "qemu/error-report.h"
+#include "hw/misc/vmcoreinfo.h"
+#include "win_dump.h"
+
+static size_t write_run(WinDumpPhyMemRun64 *run, int fd, Error **errp)
+{
+ void *buf;
+ uint64_t addr = run->BasePage << TARGET_PAGE_BITS;
+ uint64_t size = run->PageCount << TARGET_PAGE_BITS;
+ uint64_t len = size;
+
+ buf = cpu_physical_memory_map(addr, &len, false);
+ if (!buf) {
+ error_setg(errp, "win-dump: failed to map run");
+ return 0;
+ }
+ if (len != size) {
+ error_setg(errp, "win-dump: failed to map entire run");
+ len = 0;
+ goto out_unmap;
+ }
+
+ len = qemu_write_full(fd, buf, len);
+ if (len != size) {
+ error_setg(errp, QERR_IO_ERROR);
+ }
+
+out_unmap:
+ cpu_physical_memory_unmap(buf, addr, false, len);
+
+ return len;
+}
+
+static void write_runs(DumpState *s, WinDumpHeader64 *h, Error **errp)
+{
+ WinDumpPhyMemDesc64 *desc = &h->PhysicalMemoryBlock;
+ WinDumpPhyMemRun64 *run = desc->Run;
+ Error *local_err = NULL;
+ int i;
+
+ for (i = 0; i < desc->NumberOfRuns; i++) {
+ s->written_size += write_run(run + i, s->fd, &local_err);
+ if (local_err) {
+ error_propagate(errp, local_err);
+ return;
+ }
+ }
+}
+
+static void patch_mm_pfn_database(WinDumpHeader64 *h, Error **errp)
+{
+ if (cpu_memory_rw_debug(first_cpu,
+ h->KdDebuggerDataBlock + KDBG_MM_PFN_DATABASE_OFFSET64,
+ (uint8_t *)&h->PfnDatabase, sizeof(h->PfnDatabase), 0)) {
+ error_setg(errp, "win-dump: failed to read MmPfnDatabase");
+ return;
+ }
+}
+
+static void patch_bugcheck_data(WinDumpHeader64 *h, Error **errp)
+{
+ uint64_t KiBugcheckData;
+
+ if (cpu_memory_rw_debug(first_cpu,
+ h->KdDebuggerDataBlock + KDBG_KI_BUGCHECK_DATA_OFFSET64,
+ (uint8_t *)&KiBugcheckData, sizeof(KiBugcheckData), 0)) {
+ error_setg(errp, "win-dump: failed to read KiBugcheckData");
+ return;
+ }
+
+ if (cpu_memory_rw_debug(first_cpu,
+ KiBugcheckData,
+ h->BugcheckData, sizeof(h->BugcheckData), 0)) {
+ error_setg(errp, "win-dump: failed to read bugcheck data");
+ return;
+ }
+
+ /*
+ * If BugcheckCode wasn't saved, we consider guest OS as alive.
+ */
+
+ if (!h->BugcheckCode) {
+ h->BugcheckCode = LIVE_SYSTEM_DUMP;
+ }
+}
+
+/*
+ * This routine tries to correct mistakes in crashdump header.
+ */
+static void patch_header(WinDumpHeader64 *h)
+{
+ Error *local_err = NULL;
+
+ h->RequiredDumpSpace = sizeof(WinDumpHeader64) +
+ (h->PhysicalMemoryBlock.NumberOfPages << TARGET_PAGE_BITS);
+ h->PhysicalMemoryBlock.unused = 0;
+ h->unused1 = 0;
+
+ patch_mm_pfn_database(h, &local_err);
+ if (local_err) {
+ warn_report_err(local_err);
+ local_err = NULL;
+ }
+ patch_bugcheck_data(h, &local_err);
+ if (local_err) {
+ warn_report_err(local_err);
+ }
+}
+
+static void check_header(WinDumpHeader64 *h, Error **errp)
+{
+ const char Signature[] = "PAGE";
+ const char ValidDump[] = "DU64";
+
+ if (memcmp(h->Signature, Signature, sizeof(h->Signature))) {
+ error_setg(errp, "win-dump: invalid header, expected '%.4s',"
+ " got '%.4s'", Signature, h->Signature);
+ return;
+ }
+
+ if (memcmp(h->ValidDump, ValidDump, sizeof(h->ValidDump))) {
+ error_setg(errp, "win-dump: invalid header, expected '%.4s',"
+ " got '%.4s'", ValidDump, h->ValidDump);
+ return;
+ }
+}
+
+static void check_kdbg(WinDumpHeader64 *h, Error **errp)
+{
+ const char OwnerTag[] = "KDBG";
+ char read_OwnerTag[4];
+ uint64_t KdDebuggerDataBlock = h->KdDebuggerDataBlock;
+ bool try_fallback = true;
+
+try_again:
+ if (cpu_memory_rw_debug(first_cpu,
+ KdDebuggerDataBlock + KDBG_OWNER_TAG_OFFSET64,
+ (uint8_t *)&read_OwnerTag, sizeof(read_OwnerTag), 0)) {
+ error_setg(errp, "win-dump: failed to read OwnerTag");
+ return;
+ }
+
+ if (memcmp(read_OwnerTag, OwnerTag, sizeof(read_OwnerTag))) {
+ if (try_fallback) {
+ /*
+ * If attempt to use original KDBG failed
+ * (most likely because of its encryption),
+ * we try to use KDBG obtained by guest driver.
+ */
+
+ KdDebuggerDataBlock = h->BugcheckParameter1;
+ try_fallback = false;
+ goto try_again;
+ } else {
+ error_setg(errp, "win-dump: invalid KDBG OwnerTag,"
+ " expected '%.4s', got '%.4s'",
+ OwnerTag, read_OwnerTag);
+ return;
+ }
+ }
+
+ h->KdDebuggerDataBlock = KdDebuggerDataBlock;
+}
+
+struct saved_context {
+ WinContext ctx;
+ uint64_t addr;
+};
+
+static void patch_and_save_context(WinDumpHeader64 *h,
+ struct saved_context *saved_ctx,
+ Error **errp)
+{
+ uint64_t KiProcessorBlock;
+ uint16_t OffsetPrcbContext;
+ CPUState *cpu;
+ int i = 0;
+
+ if (cpu_memory_rw_debug(first_cpu,
+ h->KdDebuggerDataBlock + KDBG_KI_PROCESSOR_BLOCK_OFFSET64,
+ (uint8_t *)&KiProcessorBlock, sizeof(KiProcessorBlock), 0)) {
+ error_setg(errp, "win-dump: failed to read KiProcessorBlock");
+ return;
+ }
+
+ if (cpu_memory_rw_debug(first_cpu,
+ h->KdDebuggerDataBlock + KDBG_OFFSET_PRCB_CONTEXT_OFFSET64,
+ (uint8_t *)&OffsetPrcbContext, sizeof(OffsetPrcbContext), 0)) {
+ error_setg(errp, "win-dump: failed to read OffsetPrcbContext");
+ return;
+ }
+
+ CPU_FOREACH(cpu) {
+ X86CPU *x86_cpu = X86_CPU(cpu);
+ CPUX86State *env = &x86_cpu->env;
+ uint64_t Prcb;
+ uint64_t Context;
+ WinContext ctx;
+
+ if (cpu_memory_rw_debug(first_cpu,
+ KiProcessorBlock + i * sizeof(uint64_t),
+ (uint8_t *)&Prcb, sizeof(Prcb), 0)) {
+ error_setg(errp, "win-dump: failed to read"
+ " CPU #%d PRCB location", i);
+ return;
+ }
+
+ if (cpu_memory_rw_debug(first_cpu,
+ Prcb + OffsetPrcbContext,
+ (uint8_t *)&Context, sizeof(Context), 0)) {
+ error_setg(errp, "win-dump: failed to read"
+ " CPU #%d ContextFrame location", i);
+ return;
+ }
+
+ saved_ctx[i].addr = Context;
+
+ ctx = (WinContext){
+ .ContextFlags = WIN_CTX_ALL,
+ .MxCsr = env->mxcsr,
+
+ .SegEs = env->segs[0].selector,
+ .SegCs = env->segs[1].selector,
+ .SegSs = env->segs[2].selector,
+ .SegDs = env->segs[3].selector,
+ .SegFs = env->segs[4].selector,
+ .SegGs = env->segs[5].selector,
+ .EFlags = cpu_compute_eflags(env),
+
+ .Dr0 = env->dr[0],
+ .Dr1 = env->dr[1],
+ .Dr2 = env->dr[2],
+ .Dr3 = env->dr[3],
+ .Dr6 = env->dr[6],
+ .Dr7 = env->dr[7],
+
+ .Rax = env->regs[R_EAX],
+ .Rbx = env->regs[R_EBX],
+ .Rcx = env->regs[R_ECX],
+ .Rdx = env->regs[R_EDX],
+ .Rsp = env->regs[R_ESP],
+ .Rbp = env->regs[R_EBP],
+ .Rsi = env->regs[R_ESI],
+ .Rdi = env->regs[R_EDI],
+ .R8 = env->regs[8],
+ .R9 = env->regs[9],
+ .R10 = env->regs[10],
+ .R11 = env->regs[11],
+ .R12 = env->regs[12],
+ .R13 = env->regs[13],
+ .R14 = env->regs[14],
+ .R15 = env->regs[15],
+
+ .Rip = env->eip,
+ .FltSave = {
+ .MxCsr = env->mxcsr,
+ },
+ };
+
+ if (cpu_memory_rw_debug(first_cpu, Context,
+ (uint8_t *)&saved_ctx[i].ctx, sizeof(WinContext), 0)) {
+ error_setg(errp, "win-dump: failed to save CPU #%d context", i);
+ return;
+ }
+
+ if (cpu_memory_rw_debug(first_cpu, Context,
+ (uint8_t *)&ctx, sizeof(WinContext), 1)) {
+ error_setg(errp, "win-dump: failed to write CPU #%d context", i);
+ return;
+ }
+
+ i++;
+ }
+}
+
+static void restore_context(WinDumpHeader64 *h,
+ struct saved_context *saved_ctx)
+{
+ int i;
+ Error *err = NULL;
+
+ for (i = 0; i < h->NumberProcessors; i++) {
+ if (cpu_memory_rw_debug(first_cpu, saved_ctx[i].addr,
+ (uint8_t *)&saved_ctx[i].ctx, sizeof(WinContext), 1)) {
+ error_setg(&err, "win-dump: failed to restore CPU #%d context", i);
+ warn_report_err(err);
+ }
+ }
+}
+
+void create_win_dump(DumpState *s, Error **errp)
+{
+ WinDumpHeader64 *h = (WinDumpHeader64 *)(s->guest_note +
+ VMCOREINFO_ELF_NOTE_HDR_SIZE);
+ X86CPU *first_x86_cpu = X86_CPU(first_cpu);
+ uint64_t saved_cr3 = first_x86_cpu->env.cr[3];
+ struct saved_context *saved_ctx = NULL;
+ Error *local_err = NULL;
+
+ if (s->guest_note_size != sizeof(WinDumpHeader64) +
+ VMCOREINFO_ELF_NOTE_HDR_SIZE) {
+ error_setg(errp, "win-dump: invalid vmcoreinfo note size");
+ return;
+ }
+
+ check_header(h, &local_err);
+ if (local_err) {
+ error_propagate(errp, local_err);
+ return;
+ }
+
+ /*
+ * Further access to kernel structures by virtual addresses
+ * should be made from system context.
+ */
+
+ first_x86_cpu->env.cr[3] = h->DirectoryTableBase;
+
+ check_kdbg(h, &local_err);
+ if (local_err) {
+ error_propagate(errp, local_err);
+ goto out_cr3;
+ }
+
+ patch_header(h);
+
+ saved_ctx = g_new(struct saved_context, h->NumberProcessors);
+
+ /*
+ * Always patch context because there is no way
+ * to determine if the system-saved context is valid
+ */
+
+ patch_and_save_context(h, saved_ctx, &local_err);
+ if (local_err) {
+ error_propagate(errp, local_err);
+ goto out_free;
+ }
+
+ s->total_size = h->RequiredDumpSpace;
+
+ s->written_size = qemu_write_full(s->fd, h, sizeof(*h));
+ if (s->written_size != sizeof(*h)) {
+ error_setg(errp, QERR_IO_ERROR);
+ goto out_restore;
+ }
+
+ write_runs(s, h, &local_err);
+ if (local_err) {
+ error_propagate(errp, local_err);
+ goto out_restore;
+ }
+
+out_restore:
+ restore_context(h, saved_ctx);
+out_free:
+ g_free(saved_ctx);
+out_cr3:
+ first_x86_cpu->env.cr[3] = saved_cr3;
+
+ return;
+}
diff --git a/win_dump.h b/win_dump.h
new file mode 100644
index 0000000..f9e1faf
--- /dev/null
+++ b/win_dump.h
@@ -0,0 +1,176 @@
+/*
+ * Windows crashdump
+ *
+ * Copyright (c) 2018 Virtuozzo International GmbH
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ *
+ */
+
+typedef struct WinDumpPhyMemRun64 {
+ uint64_t BasePage;
+ uint64_t PageCount;
+} QEMU_PACKED WinDumpPhyMemRun64;
+
+typedef struct WinDumpPhyMemDesc64 {
+ uint32_t NumberOfRuns;
+ uint32_t unused;
+ uint64_t NumberOfPages;
+ WinDumpPhyMemRun64 Run[43];
+} QEMU_PACKED WinDumpPhyMemDesc64;
+
+typedef struct WinDumpExceptionRecord {
+ uint32_t ExceptionCode;
+ uint32_t ExceptionFlags;
+ uint64_t ExceptionRecord;
+ uint64_t ExceptionAddress;
+ uint32_t NumberParameters;
+ uint32_t unused;
+ uint64_t ExceptionInformation[15];
+} QEMU_PACKED WinDumpExceptionRecord;
+
+typedef struct WinDumpHeader64 {
+ char Signature[4];
+ char ValidDump[4];
+ uint32_t MajorVersion;
+ uint32_t MinorVersion;
+ uint64_t DirectoryTableBase;
+ uint64_t PfnDatabase;
+ uint64_t PsLoadedModuleList;
+ uint64_t PsActiveProcessHead;
+ uint32_t MachineImageType;
+ uint32_t NumberProcessors;
+ union {
+ struct {
+ uint32_t BugcheckCode;
+ uint32_t unused0;
+ uint64_t BugcheckParameter1;
+ uint64_t BugcheckParameter2;
+ uint64_t BugcheckParameter3;
+ uint64_t BugcheckParameter4;
+ };
+ uint8_t BugcheckData[40];
+ };
+ uint8_t VersionUser[32];
+ uint64_t KdDebuggerDataBlock;
+ union {
+ WinDumpPhyMemDesc64 PhysicalMemoryBlock;
+ uint8_t PhysicalMemoryBlockBuffer[704];
+ };
+ union {
+ uint8_t ContextBuffer[3000];
+ };
+ WinDumpExceptionRecord Exception;
+ uint32_t DumpType;
+ uint32_t unused1;
+ uint64_t RequiredDumpSpace;
+ uint64_t SystemTime;
+ char Comment[128];
+ uint64_t SystemUpTime;
+ uint32_t MiniDumpFields;
+ uint32_t SecondaryDataState;
+ uint32_t ProductType;
+ uint32_t SuiteMask;
+ uint32_t WriterStatus;
+ uint8_t unused2;
+ uint8_t KdSecondaryVersion;
+ uint8_t reserved[4018];
+} QEMU_PACKED WinDumpHeader64;
+
+void create_win_dump(DumpState *s, Error **errp);
+
+#define KDBG_OWNER_TAG_OFFSET64 0x10
+#define KDBG_MM_PFN_DATABASE_OFFSET64 0xC0
+#define KDBG_KI_BUGCHECK_DATA_OFFSET64 0x88
+#define KDBG_KI_PROCESSOR_BLOCK_OFFSET64 0x218
+#define KDBG_OFFSET_PRCB_CONTEXT_OFFSET64 0x338
+
+#define VMCOREINFO_ELF_NOTE_HDR_SIZE 24
+
+#define WIN_CTX_X64 0x00100000L
+
+#define WIN_CTX_CTL 0x00000001L
+#define WIN_CTX_INT 0x00000002L
+#define WIN_CTX_SEG 0x00000004L
+#define WIN_CTX_FP 0x00000008L
+#define WIN_CTX_DBG 0x00000010L
+
+#define WIN_CTX_FULL (WIN_CTX_X64 | WIN_CTX_CTL | WIN_CTX_INT | WIN_CTX_FP)
+#define WIN_CTX_ALL (WIN_CTX_FULL | WIN_CTX_SEG | WIN_CTX_DBG)
+
+#define LIVE_SYSTEM_DUMP 0x00000161
+
+typedef struct WinM128A {
+ uint64_t low;
+ int64_t high;
+} QEMU_ALIGNED(16) WinM128A;
+
+typedef struct WinContext {
+ uint64_t PHome[6];
+
+ uint32_t ContextFlags;
+ uint32_t MxCsr;
+
+ uint16_t SegCs;
+ uint16_t SegDs;
+ uint16_t SegEs;
+ uint16_t SegFs;
+ uint16_t SegGs;
+ uint16_t SegSs;
+ uint32_t EFlags;
+
+ uint64_t Dr0;
+ uint64_t Dr1;
+ uint64_t Dr2;
+ uint64_t Dr3;
+ uint64_t Dr6;
+ uint64_t Dr7;
+
+ uint64_t Rax;
+ uint64_t Rcx;
+ uint64_t Rdx;
+ uint64_t Rbx;
+ uint64_t Rsp;
+ uint64_t Rbp;
+ uint64_t Rsi;
+ uint64_t Rdi;
+ uint64_t R8;
+ uint64_t R9;
+ uint64_t R10;
+ uint64_t R11;
+ uint64_t R12;
+ uint64_t R13;
+ uint64_t R14;
+ uint64_t R15;
+
+ uint64_t Rip;
+
+ struct {
+ uint16_t ControlWord;
+ uint16_t StatusWord;
+ uint8_t TagWord;
+ uint8_t Reserved1;
+ uint16_t ErrorOpcode;
+ uint32_t ErrorOffset;
+ uint16_t ErrorSelector;
+ uint16_t Reserved2;
+ uint32_t DataOffset;
+ uint16_t DataSelector;
+ uint16_t Reserved3;
+ uint32_t MxCsr;
+ uint32_t MxCsr_Mask;
+ WinM128A FloatRegisters[8];
+ WinM128A XmmRegisters[16];
+ uint8_t Reserved4[96];
+ } FltSave;
+
+ WinM128A VectorRegister[26];
+ uint64_t VectorControl;
+
+ uint64_t DebugControl;
+ uint64_t LastBranchToRip;
+ uint64_t LastBranchFromRip;
+ uint64_t LastExceptionToRip;
+ uint64_t LastExceptionFromRip;
+} QEMU_ALIGNED(16) WinContext;
