exec: introduce memory_ldst.inc.c
Templatize the address_space_* and *_phys functions, so that we can add
similar functions in the next patch that work with a lightweight,
cache-like version of address_space_map/unmap.
Reviewed-by: Stefan Hajnoczi <stefanha@redhat.com>
Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>
diff --git a/exec.c b/exec.c
index 1c62d01..8568a6f 100644
--- a/exec.c
+++ b/exec.c
@@ -3058,677 +3058,16 @@
return address_space_unmap(&address_space_memory, buffer, len, is_write, access_len);
}
-/* warning: addr must be aligned */
-static inline uint32_t address_space_ldl_internal(AddressSpace *as, hwaddr addr,
- MemTxAttrs attrs,
- MemTxResult *result,
- enum device_endian endian)
-{
- uint8_t *ptr;
- uint64_t val;
- MemoryRegion *mr;
- hwaddr l = 4;
- hwaddr addr1;
- MemTxResult r;
- bool release_lock = false;
-
- rcu_read_lock();
- mr = address_space_translate(as, addr, &addr1, &l, false);
- if (l < 4 || !memory_access_is_direct(mr, false)) {
- release_lock |= prepare_mmio_access(mr);
-
- /* I/O case */
- r = memory_region_dispatch_read(mr, addr1, &val, 4, attrs);
-#if defined(TARGET_WORDS_BIGENDIAN)
- if (endian == DEVICE_LITTLE_ENDIAN) {
- val = bswap32(val);
- }
-#else
- if (endian == DEVICE_BIG_ENDIAN) {
- val = bswap32(val);
- }
-#endif
- } else {
- /* RAM case */
- ptr = qemu_map_ram_ptr(mr->ram_block, addr1);
- switch (endian) {
- case DEVICE_LITTLE_ENDIAN:
- val = ldl_le_p(ptr);
- break;
- case DEVICE_BIG_ENDIAN:
- val = ldl_be_p(ptr);
- break;
- default:
- val = ldl_p(ptr);
- break;
- }
- r = MEMTX_OK;
- }
- if (result) {
- *result = r;
- }
- if (release_lock) {
- qemu_mutex_unlock_iothread();
- }
- rcu_read_unlock();
- return val;
-}
-
-uint32_t address_space_ldl(AddressSpace *as, hwaddr addr,
- MemTxAttrs attrs, MemTxResult *result)
-{
- return address_space_ldl_internal(as, addr, attrs, result,
- DEVICE_NATIVE_ENDIAN);
-}
-
-uint32_t address_space_ldl_le(AddressSpace *as, hwaddr addr,
- MemTxAttrs attrs, MemTxResult *result)
-{
- return address_space_ldl_internal(as, addr, attrs, result,
- DEVICE_LITTLE_ENDIAN);
-}
-
-uint32_t address_space_ldl_be(AddressSpace *as, hwaddr addr,
- MemTxAttrs attrs, MemTxResult *result)
-{
- return address_space_ldl_internal(as, addr, attrs, result,
- DEVICE_BIG_ENDIAN);
-}
-
-uint32_t ldl_phys(AddressSpace *as, hwaddr addr)
-{
- return address_space_ldl(as, addr, MEMTXATTRS_UNSPECIFIED, NULL);
-}
-
-uint32_t ldl_le_phys(AddressSpace *as, hwaddr addr)
-{
- return address_space_ldl_le(as, addr, MEMTXATTRS_UNSPECIFIED, NULL);
-}
-
-uint32_t ldl_be_phys(AddressSpace *as, hwaddr addr)
-{
- return address_space_ldl_be(as, addr, MEMTXATTRS_UNSPECIFIED, NULL);
-}
-
-/* warning: addr must be aligned */
-static inline uint64_t address_space_ldq_internal(AddressSpace *as, hwaddr addr,
- MemTxAttrs attrs,
- MemTxResult *result,
- enum device_endian endian)
-{
- uint8_t *ptr;
- uint64_t val;
- MemoryRegion *mr;
- hwaddr l = 8;
- hwaddr addr1;
- MemTxResult r;
- bool release_lock = false;
-
- rcu_read_lock();
- mr = address_space_translate(as, addr, &addr1, &l,
- false);
- if (l < 8 || !memory_access_is_direct(mr, false)) {
- release_lock |= prepare_mmio_access(mr);
-
- /* I/O case */
- r = memory_region_dispatch_read(mr, addr1, &val, 8, attrs);
-#if defined(TARGET_WORDS_BIGENDIAN)
- if (endian == DEVICE_LITTLE_ENDIAN) {
- val = bswap64(val);
- }
-#else
- if (endian == DEVICE_BIG_ENDIAN) {
- val = bswap64(val);
- }
-#endif
- } else {
- /* RAM case */
- ptr = qemu_map_ram_ptr(mr->ram_block, addr1);
- switch (endian) {
- case DEVICE_LITTLE_ENDIAN:
- val = ldq_le_p(ptr);
- break;
- case DEVICE_BIG_ENDIAN:
- val = ldq_be_p(ptr);
- break;
- default:
- val = ldq_p(ptr);
- break;
- }
- r = MEMTX_OK;
- }
- if (result) {
- *result = r;
- }
- if (release_lock) {
- qemu_mutex_unlock_iothread();
- }
- rcu_read_unlock();
- return val;
-}
-
-uint64_t address_space_ldq(AddressSpace *as, hwaddr addr,
- MemTxAttrs attrs, MemTxResult *result)
-{
- return address_space_ldq_internal(as, addr, attrs, result,
- DEVICE_NATIVE_ENDIAN);
-}
-
-uint64_t address_space_ldq_le(AddressSpace *as, hwaddr addr,
- MemTxAttrs attrs, MemTxResult *result)
-{
- return address_space_ldq_internal(as, addr, attrs, result,
- DEVICE_LITTLE_ENDIAN);
-}
-
-uint64_t address_space_ldq_be(AddressSpace *as, hwaddr addr,
- MemTxAttrs attrs, MemTxResult *result)
-{
- return address_space_ldq_internal(as, addr, attrs, result,
- DEVICE_BIG_ENDIAN);
-}
-
-uint64_t ldq_phys(AddressSpace *as, hwaddr addr)
-{
- return address_space_ldq(as, addr, MEMTXATTRS_UNSPECIFIED, NULL);
-}
-
-uint64_t ldq_le_phys(AddressSpace *as, hwaddr addr)
-{
- return address_space_ldq_le(as, addr, MEMTXATTRS_UNSPECIFIED, NULL);
-}
-
-uint64_t ldq_be_phys(AddressSpace *as, hwaddr addr)
-{
- return address_space_ldq_be(as, addr, MEMTXATTRS_UNSPECIFIED, NULL);
-}
-
-uint32_t address_space_ldub(AddressSpace *as, hwaddr addr,
- MemTxAttrs attrs, MemTxResult *result)
-{
- uint8_t *ptr;
- uint64_t val;
- MemoryRegion *mr;
- hwaddr l = 1;
- hwaddr addr1;
- MemTxResult r;
- bool release_lock = false;
-
- rcu_read_lock();
- mr = address_space_translate(as, addr, &addr1, &l, false);
- if (!memory_access_is_direct(mr, false)) {
- release_lock |= prepare_mmio_access(mr);
-
- /* I/O case */
- r = memory_region_dispatch_read(mr, addr1, &val, 1, attrs);
- } else {
- /* RAM case */
- ptr = qemu_map_ram_ptr(mr->ram_block, addr1);
- val = ldub_p(ptr);
- r = MEMTX_OK;
- }
- if (result) {
- *result = r;
- }
- if (release_lock) {
- qemu_mutex_unlock_iothread();
- }
- rcu_read_unlock();
- return val;
-}
-
-uint32_t ldub_phys(AddressSpace *as, hwaddr addr)
-{
- return address_space_ldub(as, addr, MEMTXATTRS_UNSPECIFIED, NULL);
-}
-
-/* warning: addr must be aligned */
-static inline uint32_t address_space_lduw_internal(AddressSpace *as,
- hwaddr addr,
- MemTxAttrs attrs,
- MemTxResult *result,
- enum device_endian endian)
-{
- uint8_t *ptr;
- uint64_t val;
- MemoryRegion *mr;
- hwaddr l = 2;
- hwaddr addr1;
- MemTxResult r;
- bool release_lock = false;
-
- rcu_read_lock();
- mr = address_space_translate(as, addr, &addr1, &l,
- false);
- if (l < 2 || !memory_access_is_direct(mr, false)) {
- release_lock |= prepare_mmio_access(mr);
-
- /* I/O case */
- r = memory_region_dispatch_read(mr, addr1, &val, 2, attrs);
-#if defined(TARGET_WORDS_BIGENDIAN)
- if (endian == DEVICE_LITTLE_ENDIAN) {
- val = bswap16(val);
- }
-#else
- if (endian == DEVICE_BIG_ENDIAN) {
- val = bswap16(val);
- }
-#endif
- } else {
- /* RAM case */
- ptr = qemu_map_ram_ptr(mr->ram_block, addr1);
- switch (endian) {
- case DEVICE_LITTLE_ENDIAN:
- val = lduw_le_p(ptr);
- break;
- case DEVICE_BIG_ENDIAN:
- val = lduw_be_p(ptr);
- break;
- default:
- val = lduw_p(ptr);
- break;
- }
- r = MEMTX_OK;
- }
- if (result) {
- *result = r;
- }
- if (release_lock) {
- qemu_mutex_unlock_iothread();
- }
- rcu_read_unlock();
- return val;
-}
-
-uint32_t address_space_lduw(AddressSpace *as, hwaddr addr,
- MemTxAttrs attrs, MemTxResult *result)
-{
- return address_space_lduw_internal(as, addr, attrs, result,
- DEVICE_NATIVE_ENDIAN);
-}
-
-uint32_t address_space_lduw_le(AddressSpace *as, hwaddr addr,
- MemTxAttrs attrs, MemTxResult *result)
-{
- return address_space_lduw_internal(as, addr, attrs, result,
- DEVICE_LITTLE_ENDIAN);
-}
-
-uint32_t address_space_lduw_be(AddressSpace *as, hwaddr addr,
- MemTxAttrs attrs, MemTxResult *result)
-{
- return address_space_lduw_internal(as, addr, attrs, result,
- DEVICE_BIG_ENDIAN);
-}
-
-uint32_t lduw_phys(AddressSpace *as, hwaddr addr)
-{
- return address_space_lduw(as, addr, MEMTXATTRS_UNSPECIFIED, NULL);
-}
-
-uint32_t lduw_le_phys(AddressSpace *as, hwaddr addr)
-{
- return address_space_lduw_le(as, addr, MEMTXATTRS_UNSPECIFIED, NULL);
-}
-
-uint32_t lduw_be_phys(AddressSpace *as, hwaddr addr)
-{
- return address_space_lduw_be(as, addr, MEMTXATTRS_UNSPECIFIED, NULL);
-}
-
-/* warning: addr must be aligned. The ram page is not masked as dirty
- and the code inside is not invalidated. It is useful if the dirty
- bits are used to track modified PTEs */
-void address_space_stl_notdirty(AddressSpace *as, hwaddr addr, uint32_t val,
- MemTxAttrs attrs, MemTxResult *result)
-{
- uint8_t *ptr;
- MemoryRegion *mr;
- hwaddr l = 4;
- hwaddr addr1;
- MemTxResult r;
- uint8_t dirty_log_mask;
- bool release_lock = false;
-
- rcu_read_lock();
- mr = address_space_translate(as, addr, &addr1, &l,
- true);
- if (l < 4 || !memory_access_is_direct(mr, true)) {
- release_lock |= prepare_mmio_access(mr);
-
- r = memory_region_dispatch_write(mr, addr1, val, 4, attrs);
- } else {
- ptr = qemu_map_ram_ptr(mr->ram_block, addr1);
- stl_p(ptr, val);
-
- dirty_log_mask = memory_region_get_dirty_log_mask(mr);
- dirty_log_mask &= ~(1 << DIRTY_MEMORY_CODE);
- cpu_physical_memory_set_dirty_range(memory_region_get_ram_addr(mr) + addr,
- 4, dirty_log_mask);
- r = MEMTX_OK;
- }
- if (result) {
- *result = r;
- }
- if (release_lock) {
- qemu_mutex_unlock_iothread();
- }
- rcu_read_unlock();
-}
-
-void stl_phys_notdirty(AddressSpace *as, hwaddr addr, uint32_t val)
-{
- address_space_stl_notdirty(as, addr, val, MEMTXATTRS_UNSPECIFIED, NULL);
-}
-
-/* warning: addr must be aligned */
-static inline void address_space_stl_internal(AddressSpace *as,
- hwaddr addr, uint32_t val,
- MemTxAttrs attrs,
- MemTxResult *result,
- enum device_endian endian)
-{
- uint8_t *ptr;
- MemoryRegion *mr;
- hwaddr l = 4;
- hwaddr addr1;
- MemTxResult r;
- bool release_lock = false;
-
- rcu_read_lock();
- mr = address_space_translate(as, addr, &addr1, &l,
- true);
- if (l < 4 || !memory_access_is_direct(mr, true)) {
- release_lock |= prepare_mmio_access(mr);
-
-#if defined(TARGET_WORDS_BIGENDIAN)
- if (endian == DEVICE_LITTLE_ENDIAN) {
- val = bswap32(val);
- }
-#else
- if (endian == DEVICE_BIG_ENDIAN) {
- val = bswap32(val);
- }
-#endif
- r = memory_region_dispatch_write(mr, addr1, val, 4, attrs);
- } else {
- /* RAM case */
- ptr = qemu_map_ram_ptr(mr->ram_block, addr1);
- switch (endian) {
- case DEVICE_LITTLE_ENDIAN:
- stl_le_p(ptr, val);
- break;
- case DEVICE_BIG_ENDIAN:
- stl_be_p(ptr, val);
- break;
- default:
- stl_p(ptr, val);
- break;
- }
- invalidate_and_set_dirty(mr, addr1, 4);
- r = MEMTX_OK;
- }
- if (result) {
- *result = r;
- }
- if (release_lock) {
- qemu_mutex_unlock_iothread();
- }
- rcu_read_unlock();
-}
-
-void address_space_stl(AddressSpace *as, hwaddr addr, uint32_t val,
- MemTxAttrs attrs, MemTxResult *result)
-{
- address_space_stl_internal(as, addr, val, attrs, result,
- DEVICE_NATIVE_ENDIAN);
-}
-
-void address_space_stl_le(AddressSpace *as, hwaddr addr, uint32_t val,
- MemTxAttrs attrs, MemTxResult *result)
-{
- address_space_stl_internal(as, addr, val, attrs, result,
- DEVICE_LITTLE_ENDIAN);
-}
-
-void address_space_stl_be(AddressSpace *as, hwaddr addr, uint32_t val,
- MemTxAttrs attrs, MemTxResult *result)
-{
- address_space_stl_internal(as, addr, val, attrs, result,
- DEVICE_BIG_ENDIAN);
-}
-
-void stl_phys(AddressSpace *as, hwaddr addr, uint32_t val)
-{
- address_space_stl(as, addr, val, MEMTXATTRS_UNSPECIFIED, NULL);
-}
-
-void stl_le_phys(AddressSpace *as, hwaddr addr, uint32_t val)
-{
- address_space_stl_le(as, addr, val, MEMTXATTRS_UNSPECIFIED, NULL);
-}
-
-void stl_be_phys(AddressSpace *as, hwaddr addr, uint32_t val)
-{
- address_space_stl_be(as, addr, val, MEMTXATTRS_UNSPECIFIED, NULL);
-}
-
-void address_space_stb(AddressSpace *as, hwaddr addr, uint32_t val,
- MemTxAttrs attrs, MemTxResult *result)
-{
- uint8_t *ptr;
- MemoryRegion *mr;
- hwaddr l = 1;
- hwaddr addr1;
- MemTxResult r;
- bool release_lock = false;
-
- rcu_read_lock();
- mr = address_space_translate(as, addr, &addr1, &l, true);
- if (!memory_access_is_direct(mr, true)) {
- release_lock |= prepare_mmio_access(mr);
- r = memory_region_dispatch_write(mr, addr1, val, 1, attrs);
- } else {
- /* RAM case */
- ptr = qemu_map_ram_ptr(mr->ram_block, addr1);
- stb_p(ptr, val);
- invalidate_and_set_dirty(mr, addr1, 1);
- r = MEMTX_OK;
- }
- if (result) {
- *result = r;
- }
- if (release_lock) {
- qemu_mutex_unlock_iothread();
- }
- rcu_read_unlock();
-}
-
-void stb_phys(AddressSpace *as, hwaddr addr, uint32_t val)
-{
- address_space_stb(as, addr, val, MEMTXATTRS_UNSPECIFIED, NULL);
-}
-
-/* warning: addr must be aligned */
-static inline void address_space_stw_internal(AddressSpace *as,
- hwaddr addr, uint32_t val,
- MemTxAttrs attrs,
- MemTxResult *result,
- enum device_endian endian)
-{
- uint8_t *ptr;
- MemoryRegion *mr;
- hwaddr l = 2;
- hwaddr addr1;
- MemTxResult r;
- bool release_lock = false;
-
- rcu_read_lock();
- mr = address_space_translate(as, addr, &addr1, &l, true);
- if (l < 2 || !memory_access_is_direct(mr, true)) {
- release_lock |= prepare_mmio_access(mr);
-
-#if defined(TARGET_WORDS_BIGENDIAN)
- if (endian == DEVICE_LITTLE_ENDIAN) {
- val = bswap16(val);
- }
-#else
- if (endian == DEVICE_BIG_ENDIAN) {
- val = bswap16(val);
- }
-#endif
- r = memory_region_dispatch_write(mr, addr1, val, 2, attrs);
- } else {
- /* RAM case */
- ptr = qemu_map_ram_ptr(mr->ram_block, addr1);
- switch (endian) {
- case DEVICE_LITTLE_ENDIAN:
- stw_le_p(ptr, val);
- break;
- case DEVICE_BIG_ENDIAN:
- stw_be_p(ptr, val);
- break;
- default:
- stw_p(ptr, val);
- break;
- }
- invalidate_and_set_dirty(mr, addr1, 2);
- r = MEMTX_OK;
- }
- if (result) {
- *result = r;
- }
- if (release_lock) {
- qemu_mutex_unlock_iothread();
- }
- rcu_read_unlock();
-}
-
-void address_space_stw(AddressSpace *as, hwaddr addr, uint32_t val,
- MemTxAttrs attrs, MemTxResult *result)
-{
- address_space_stw_internal(as, addr, val, attrs, result,
- DEVICE_NATIVE_ENDIAN);
-}
-
-void address_space_stw_le(AddressSpace *as, hwaddr addr, uint32_t val,
- MemTxAttrs attrs, MemTxResult *result)
-{
- address_space_stw_internal(as, addr, val, attrs, result,
- DEVICE_LITTLE_ENDIAN);
-}
-
-void address_space_stw_be(AddressSpace *as, hwaddr addr, uint32_t val,
- MemTxAttrs attrs, MemTxResult *result)
-{
- address_space_stw_internal(as, addr, val, attrs, result,
- DEVICE_BIG_ENDIAN);
-}
-
-void stw_phys(AddressSpace *as, hwaddr addr, uint32_t val)
-{
- address_space_stw(as, addr, val, MEMTXATTRS_UNSPECIFIED, NULL);
-}
-
-void stw_le_phys(AddressSpace *as, hwaddr addr, uint32_t val)
-{
- address_space_stw_le(as, addr, val, MEMTXATTRS_UNSPECIFIED, NULL);
-}
-
-void stw_be_phys(AddressSpace *as, hwaddr addr, uint32_t val)
-{
- address_space_stw_be(as, addr, val, MEMTXATTRS_UNSPECIFIED, NULL);
-}
-
-static inline void address_space_stq_internal(AddressSpace *as,
- hwaddr addr, uint64_t val,
- MemTxAttrs attrs,
- MemTxResult *result,
- enum device_endian endian)
-{
- uint8_t *ptr;
- MemoryRegion *mr;
- hwaddr l = 8;
- hwaddr addr1;
- MemTxResult r;
- bool release_lock = false;
-
- rcu_read_lock();
- mr = address_space_translate(as, addr, &addr1, &l, true);
- if (l < 8 || !memory_access_is_direct(mr, true)) {
- release_lock |= prepare_mmio_access(mr);
-
-#if defined(TARGET_WORDS_BIGENDIAN)
- if (endian == DEVICE_LITTLE_ENDIAN) {
- val = bswap64(val);
- }
-#else
- if (endian == DEVICE_BIG_ENDIAN) {
- val = bswap64(val);
- }
-#endif
- r = memory_region_dispatch_write(mr, addr1, val, 8, attrs);
- } else {
- /* RAM case */
- ptr = qemu_map_ram_ptr(mr->ram_block, addr1);
- switch (endian) {
- case DEVICE_LITTLE_ENDIAN:
- stq_le_p(ptr, val);
- break;
- case DEVICE_BIG_ENDIAN:
- stq_be_p(ptr, val);
- break;
- default:
- stq_p(ptr, val);
- break;
- }
- invalidate_and_set_dirty(mr, addr1, 8);
- r = MEMTX_OK;
- }
- if (result) {
- *result = r;
- }
- if (release_lock) {
- qemu_mutex_unlock_iothread();
- }
- rcu_read_unlock();
-}
-
-void address_space_stq(AddressSpace *as, hwaddr addr, uint64_t val,
- MemTxAttrs attrs, MemTxResult *result)
-{
- address_space_stq_internal(as, addr, val, attrs, result,
- DEVICE_NATIVE_ENDIAN);
-}
-
-void address_space_stq_le(AddressSpace *as, hwaddr addr, uint64_t val,
- MemTxAttrs attrs, MemTxResult *result)
-{
- address_space_stq_internal(as, addr, val, attrs, result,
- DEVICE_LITTLE_ENDIAN);
-}
-
-void address_space_stq_be(AddressSpace *as, hwaddr addr, uint64_t val,
- MemTxAttrs attrs, MemTxResult *result)
-{
- address_space_stq_internal(as, addr, val, attrs, result,
- DEVICE_BIG_ENDIAN);
-}
-
-void stq_phys(AddressSpace *as, hwaddr addr, uint64_t val)
-{
- address_space_stq(as, addr, val, MEMTXATTRS_UNSPECIFIED, NULL);
-}
-
-void stq_le_phys(AddressSpace *as, hwaddr addr, uint64_t val)
-{
- address_space_stq_le(as, addr, val, MEMTXATTRS_UNSPECIFIED, NULL);
-}
-
-void stq_be_phys(AddressSpace *as, hwaddr addr, uint64_t val)
-{
- address_space_stq_be(as, addr, val, MEMTXATTRS_UNSPECIFIED, NULL);
-}
+#define ARG1_DECL AddressSpace *as
+#define ARG1 as
+#define SUFFIX
+#define TRANSLATE(...) address_space_translate(as, __VA_ARGS__)
+#define IS_DIRECT(mr, is_write) memory_access_is_direct(mr, is_write)
+#define MAP_RAM(mr, ofs) qemu_map_ram_ptr((mr)->ram_block, ofs)
+#define INVALIDATE(mr, ofs, len) invalidate_and_set_dirty(mr, ofs, len)
+#define RCU_READ_LOCK(...) rcu_read_lock()
+#define RCU_READ_UNLOCK(...) rcu_read_unlock()
+#include "memory_ldst.inc.c"
/* virtual memory access for debug (includes writing to ROM) */
int cpu_memory_rw_debug(CPUState *cpu, target_ulong addr,
