tcg: Widen CPUTLBEntry comparators to 64-bits
This makes CPUTLBEntry agnostic to the address size of the guest.
When 32-bit addresses are in effect, we can simply read the low
32 bits of the 64-bit field. Similarly when we need to update
the field for setting TLB_NOTDIRTY.
For TCG backends that could in theory be big-endian, but in
practice are not (arm, loongarch, riscv), use QEMU_BUILD_BUG_ON
to document and ensure this is not accidentally missed.
For s390x, which is always big-endian, use HOST_BIG_ENDIAN anyway,
to document the reason for the adjustment.
For sparc64 and ppc64, always perform a 64-bit load, and rely on
the following 32-bit comparison to ignore the high bits.
Rearrange mips and ppc if ladders for clarity.
Reviewed-by: Anton Johansson <anjo@rev.ng>
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
diff --git a/accel/tcg/cputlb.c b/accel/tcg/cputlb.c
index 90c72c9..6beaeb0 100644
--- a/accel/tcg/cputlb.c
+++ b/accel/tcg/cputlb.c
@@ -1000,11 +1000,15 @@
addr &= TARGET_PAGE_MASK;
addr += tlb_entry->addend;
if ((addr - start) < length) {
-#if TCG_OVERSIZED_GUEST
+#if TARGET_LONG_BITS == 32
+ uint32_t *ptr_write = (uint32_t *)&tlb_entry->addr_write;
+ ptr_write += HOST_BIG_ENDIAN;
+ qatomic_set(ptr_write, *ptr_write | TLB_NOTDIRTY);
+#elif TCG_OVERSIZED_GUEST
tlb_entry->addr_write |= TLB_NOTDIRTY;
#else
qatomic_set(&tlb_entry->addr_write,
- tlb_entry->addr_write | TLB_NOTDIRTY);
+ tlb_entry->addr_write | TLB_NOTDIRTY);
#endif
}
}
diff --git a/include/exec/cpu-defs.h b/include/exec/cpu-defs.h
index a6e0cf1..b757d37 100644
--- a/include/exec/cpu-defs.h
+++ b/include/exec/cpu-defs.h
@@ -65,11 +65,7 @@
/* use a fully associative victim tlb of 8 entries */
#define CPU_VTLB_SIZE 8
-#if HOST_LONG_BITS == 32 && TARGET_LONG_BITS == 32
-#define CPU_TLB_ENTRY_BITS 4
-#else
#define CPU_TLB_ENTRY_BITS 5
-#endif
#define CPU_TLB_DYN_MIN_BITS 6
#define CPU_TLB_DYN_DEFAULT_BITS 8
@@ -95,33 +91,26 @@
# endif
/* Minimalized TLB entry for use by TCG fast path. */
-typedef struct CPUTLBEntry {
- /* bit TARGET_LONG_BITS to TARGET_PAGE_BITS : virtual address
- bit TARGET_PAGE_BITS-1..4 : Nonzero for accesses that should not
- go directly to ram.
- bit 3 : indicates that the entry is invalid
- bit 2..0 : zero
- */
- union {
- struct {
- target_ulong addr_read;
- target_ulong addr_write;
- target_ulong addr_code;
- /* Addend to virtual address to get host address. IO accesses
- use the corresponding iotlb value. */
- uintptr_t addend;
- };
+typedef union CPUTLBEntry {
+ struct {
+ uint64_t addr_read;
+ uint64_t addr_write;
+ uint64_t addr_code;
/*
- * Padding to get a power of two size, as well as index
- * access to addr_{read,write,code}.
+ * Addend to virtual address to get host address. IO accesses
+ * use the corresponding iotlb value.
*/
- target_ulong addr_idx[(1 << CPU_TLB_ENTRY_BITS) / TARGET_LONG_SIZE];
+ uintptr_t addend;
};
+ /*
+ * Padding to get a power of two size, as well as index
+ * access to addr_{read,write,code}.
+ */
+ uint64_t addr_idx[(1 << CPU_TLB_ENTRY_BITS) / sizeof(uint64_t)];
} CPUTLBEntry;
QEMU_BUILD_BUG_ON(sizeof(CPUTLBEntry) != (1 << CPU_TLB_ENTRY_BITS));
-
#endif /* !CONFIG_USER_ONLY && CONFIG_TCG */
#if !defined(CONFIG_USER_ONLY)
diff --git a/include/exec/cpu_ldst.h b/include/exec/cpu_ldst.h
index 5939688..a43b34e 100644
--- a/include/exec/cpu_ldst.h
+++ b/include/exec/cpu_ldst.h
@@ -334,18 +334,25 @@
{
/* Do not rearrange the CPUTLBEntry structure members. */
QEMU_BUILD_BUG_ON(offsetof(CPUTLBEntry, addr_read) !=
- MMU_DATA_LOAD * TARGET_LONG_SIZE);
+ MMU_DATA_LOAD * sizeof(uint64_t));
QEMU_BUILD_BUG_ON(offsetof(CPUTLBEntry, addr_write) !=
- MMU_DATA_STORE * TARGET_LONG_SIZE);
+ MMU_DATA_STORE * sizeof(uint64_t));
QEMU_BUILD_BUG_ON(offsetof(CPUTLBEntry, addr_code) !=
- MMU_INST_FETCH * TARGET_LONG_SIZE);
+ MMU_INST_FETCH * sizeof(uint64_t));
- const target_ulong *ptr = &entry->addr_idx[access_type];
-#if TCG_OVERSIZED_GUEST
- return *ptr;
+#if TARGET_LONG_BITS == 32
+ /* Use qatomic_read, in case of addr_write; only care about low bits. */
+ const uint32_t *ptr = (uint32_t *)&entry->addr_idx[access_type];
+ ptr += HOST_BIG_ENDIAN;
+ return qatomic_read(ptr);
#else
+ const uint64_t *ptr = &entry->addr_idx[access_type];
+# if TCG_OVERSIZED_GUEST
+ return *ptr;
+# else
/* ofs might correspond to .addr_write, so use qatomic_read */
return qatomic_read(ptr);
+# endif
#endif
}
diff --git a/tcg/aarch64/tcg-target.c.inc b/tcg/aarch64/tcg-target.c.inc
index 261ad25..e23c57e 100644
--- a/tcg/aarch64/tcg-target.c.inc
+++ b/tcg/aarch64/tcg-target.c.inc
@@ -1690,6 +1690,7 @@
tcg_out_insn(s, 3502, ADD, 1, TCG_REG_TMP1, TCG_REG_TMP1, TCG_REG_TMP0);
/* Load the tlb comparator into TMP0, and the fast path addend into TMP1. */
+ QEMU_BUILD_BUG_ON(HOST_BIG_ENDIAN);
tcg_out_ld(s, addr_type, TCG_REG_TMP0, TCG_REG_TMP1,
is_ld ? offsetof(CPUTLBEntry, addr_read)
: offsetof(CPUTLBEntry, addr_write));
diff --git a/tcg/arm/tcg-target.c.inc b/tcg/arm/tcg-target.c.inc
index 20cc1cc..64eb0cb 100644
--- a/tcg/arm/tcg-target.c.inc
+++ b/tcg/arm/tcg-target.c.inc
@@ -1430,6 +1430,7 @@
* Add the tlb_table pointer, creating the CPUTLBEntry address in R1.
* Load the tlb comparator into R2/R3 and the fast path addend into R1.
*/
+ QEMU_BUILD_BUG_ON(HOST_BIG_ENDIAN);
if (cmp_off == 0) {
if (s->addr_type == TCG_TYPE_I32) {
tcg_out_ld32_rwb(s, COND_AL, TCG_REG_R2, TCG_REG_R1, TCG_REG_R0);
diff --git a/tcg/loongarch64/tcg-target.c.inc b/tcg/loongarch64/tcg-target.c.inc
index 0bae922..e89f3b8 100644
--- a/tcg/loongarch64/tcg-target.c.inc
+++ b/tcg/loongarch64/tcg-target.c.inc
@@ -875,6 +875,7 @@
tcg_out_opc_add_d(s, TCG_REG_TMP2, TCG_REG_TMP2, TCG_REG_TMP1);
/* Load the tlb comparator and the addend. */
+ QEMU_BUILD_BUG_ON(HOST_BIG_ENDIAN);
tcg_out_ld(s, addr_type, TCG_REG_TMP0, TCG_REG_TMP2,
is_ld ? offsetof(CPUTLBEntry, addr_read)
: offsetof(CPUTLBEntry, addr_write));
diff --git a/tcg/mips/tcg-target.c.inc b/tcg/mips/tcg-target.c.inc
index 3274d9a..4c0de0a 100644
--- a/tcg/mips/tcg-target.c.inc
+++ b/tcg/mips/tcg-target.c.inc
@@ -1311,14 +1311,17 @@
/* Add the tlb_table pointer, creating the CPUTLBEntry address in TMP3. */
tcg_out_opc_reg(s, ALIAS_PADD, TCG_TMP3, TCG_TMP3, TCG_TMP1);
+ if (TCG_TARGET_REG_BITS == 32 || addr_type == TCG_TYPE_I32) {
+ /* Load the (low half) tlb comparator. */
+ tcg_out_ld(s, TCG_TYPE_I32, TCG_TMP0, TCG_TMP3,
+ cmp_off + HOST_BIG_ENDIAN * 4);
+ } else {
+ tcg_out_ld(s, TCG_TYPE_I64, TCG_TMP0, TCG_TMP3, cmp_off);
+ }
+
if (TCG_TARGET_REG_BITS == 64 || addr_type == TCG_TYPE_I32) {
- /* Load the tlb comparator. */
- tcg_out_ld(s, addr_type, TCG_TMP0, TCG_TMP3, cmp_off);
/* Load the tlb addend for the fast path. */
tcg_out_ld(s, TCG_TYPE_PTR, TCG_TMP3, TCG_TMP3, add_off);
- } else {
- /* Load the low half of the tlb comparator. */
- tcg_out_ldst(s, OPC_LW, TCG_TMP0, TCG_TMP3, cmp_off + LO_OFF);
}
/*
diff --git a/tcg/ppc/tcg-target.c.inc b/tcg/ppc/tcg-target.c.inc
index 11955a6..73f6ea0 100644
--- a/tcg/ppc/tcg-target.c.inc
+++ b/tcg/ppc/tcg-target.c.inc
@@ -2098,20 +2098,24 @@
}
tcg_out32(s, AND | SAB(TCG_REG_TMP1, TCG_REG_TMP1, TCG_REG_R0));
- /* Load the (low part) TLB comparator into TMP2. */
- if (cmp_off == 0
- && (TCG_TARGET_REG_BITS == 64 || addr_type == TCG_TYPE_I32)) {
- uint32_t lxu = (TCG_TARGET_REG_BITS == 32 || addr_type == TCG_TYPE_I32
- ? LWZUX : LDUX);
- tcg_out32(s, lxu | TAB(TCG_REG_TMP2, TCG_REG_TMP1, TCG_REG_TMP2));
+ /*
+ * Load the (low part) TLB comparator into TMP2.
+ * For 64-bit host, always load the entire 64-bit slot for simplicity.
+ * We will ignore the high bits with tcg_out_cmp(..., addr_type).
+ */
+ if (TCG_TARGET_REG_BITS == 64) {
+ if (cmp_off == 0) {
+ tcg_out32(s, LDUX | TAB(TCG_REG_TMP2, TCG_REG_TMP1, TCG_REG_TMP2));
+ } else {
+ tcg_out32(s, ADD | TAB(TCG_REG_TMP1, TCG_REG_TMP1, TCG_REG_TMP2));
+ tcg_out_ld(s, TCG_TYPE_I64, TCG_REG_TMP2, TCG_REG_TMP1, cmp_off);
+ }
+ } else if (cmp_off == 0 && !HOST_BIG_ENDIAN) {
+ tcg_out32(s, LWZUX | TAB(TCG_REG_TMP2, TCG_REG_TMP1, TCG_REG_TMP2));
} else {
tcg_out32(s, ADD | TAB(TCG_REG_TMP1, TCG_REG_TMP1, TCG_REG_TMP2));
- if (TCG_TARGET_REG_BITS == 32 && addr_type != TCG_TYPE_I32) {
- tcg_out_ld(s, TCG_TYPE_I32, TCG_REG_TMP2,
- TCG_REG_TMP1, cmp_off + 4 * HOST_BIG_ENDIAN);
- } else {
- tcg_out_ld(s, addr_type, TCG_REG_TMP2, TCG_REG_TMP1, cmp_off);
- }
+ tcg_out_ld(s, TCG_TYPE_I32, TCG_REG_TMP2, TCG_REG_TMP1,
+ cmp_off + 4 * HOST_BIG_ENDIAN);
}
/*
diff --git a/tcg/riscv/tcg-target.c.inc b/tcg/riscv/tcg-target.c.inc
index a8f99f7..a6d56e2 100644
--- a/tcg/riscv/tcg-target.c.inc
+++ b/tcg/riscv/tcg-target.c.inc
@@ -1249,6 +1249,7 @@
}
/* Load the tlb comparator and the addend. */
+ QEMU_BUILD_BUG_ON(HOST_BIG_ENDIAN);
tcg_out_ld(s, addr_type, TCG_REG_TMP0, TCG_REG_TMP2,
is_ld ? offsetof(CPUTLBEntry, addr_read)
: offsetof(CPUTLBEntry, addr_write));
diff --git a/tcg/s390x/tcg-target.c.inc b/tcg/s390x/tcg-target.c.inc
index 2795242..03be800 100644
--- a/tcg/s390x/tcg-target.c.inc
+++ b/tcg/s390x/tcg-target.c.inc
@@ -1796,6 +1796,7 @@
ofs = offsetof(CPUTLBEntry, addr_write);
}
if (addr_type == TCG_TYPE_I32) {
+ ofs += HOST_BIG_ENDIAN * 4;
tcg_out_insn(s, RX, C, TCG_REG_R0, TCG_TMP0, TCG_REG_NONE, ofs);
} else {
tcg_out_insn(s, RXY, CG, TCG_REG_R0, TCG_TMP0, TCG_REG_NONE, ofs);
diff --git a/tcg/sparc64/tcg-target.c.inc b/tcg/sparc64/tcg-target.c.inc
index 48efd83..6c60657 100644
--- a/tcg/sparc64/tcg-target.c.inc
+++ b/tcg/sparc64/tcg-target.c.inc
@@ -1063,8 +1063,12 @@
/* Add the tlb_table pointer, creating the CPUTLBEntry address into R2. */
tcg_out_arith(s, TCG_REG_T1, TCG_REG_T1, TCG_REG_T3, ARITH_ADD);
- /* Load the tlb comparator and the addend. */
- tcg_out_ld(s, addr_type, TCG_REG_T2, TCG_REG_T1, cmp_off);
+ /*
+ * Load the tlb comparator and the addend.
+ * Always load the entire 64-bit comparator for simplicity.
+ * We will ignore the high bits via BPCC_ICC below.
+ */
+ tcg_out_ld(s, TCG_TYPE_I64, TCG_REG_T2, TCG_REG_T1, cmp_off);
tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_T1, TCG_REG_T1, add_off);
h->base = TCG_REG_T1;
