blob: bd75da8dbcfb29f88599a876b89d36543c287348 [file] [log] [blame]
Richard Henderson8ae08862022-06-08 19:38:48 +01001/*
2 * ARM page table walking.
3 *
4 * This code is licensed under the GNU GPL v2 or later.
5 *
6 * SPDX-License-Identifier: GPL-2.0-or-later
7 */
8
9#include "qemu/osdep.h"
10#include "qemu/log.h"
Richard Henderson1f2e87e2022-06-08 19:38:49 +010011#include "qemu/range.h"
Richard Henderson71943a12022-10-24 15:18:49 +100012#include "qemu/main-loop.h"
Richard Hendersonf3639a62022-10-10 20:18:57 -070013#include "exec/exec-all.h"
Richard Henderson8ae08862022-06-08 19:38:48 +010014#include "cpu.h"
15#include "internals.h"
Richard Henderson2c1f4292022-06-08 19:38:50 +010016#include "idau.h"
Richard Henderson8ae08862022-06-08 19:38:48 +010017
18
Richard Henderson6d2654f2022-10-10 20:18:54 -070019typedef struct S1Translate {
20 ARMMMUIdx in_mmu_idx;
Richard Henderson48da29e2022-10-24 15:18:39 +100021 ARMMMUIdx in_ptw_idx;
Richard Henderson6d2654f2022-10-10 20:18:54 -070022 bool in_secure;
Richard Henderson4a358552022-10-10 20:18:55 -070023 bool in_debug;
Richard Henderson6d2654f2022-10-10 20:18:54 -070024 bool out_secure;
Richard Henderson71943a12022-10-24 15:18:49 +100025 bool out_rw;
Richard Henderson4e7a2c92022-10-10 20:18:56 -070026 bool out_be;
Richard Henderson71943a12022-10-24 15:18:49 +100027 hwaddr out_virt;
Richard Henderson6d2654f2022-10-10 20:18:54 -070028 hwaddr out_phys;
Richard Hendersonf3639a62022-10-10 20:18:57 -070029 void *out_host;
Richard Henderson6d2654f2022-10-10 20:18:54 -070030} S1Translate;
31
32static bool get_phys_addr_lpae(CPUARMState *env, S1Translate *ptw,
33 uint64_t address,
34 MMUAccessType access_type, bool s1_is_el0,
Richard Hendersonc23f08a2022-10-01 09:22:38 -070035 GetPhysAddrResult *result, ARMMMUFaultInfo *fi)
Richard Henderson11552bb2022-06-08 19:38:51 +010036 __attribute__((nonnull));
37
Richard Henderson3f5a74c2022-10-10 20:18:58 -070038static bool get_phys_addr_with_struct(CPUARMState *env, S1Translate *ptw,
39 target_ulong address,
40 MMUAccessType access_type,
41 GetPhysAddrResult *result,
42 ARMMMUFaultInfo *fi)
43 __attribute__((nonnull));
44
Richard Henderson1c73d842022-06-08 19:38:52 +010045/* This mapping is common between ID_AA64MMFR0.PARANGE and TCR_ELx.{I}PS. */
46static const uint8_t pamax_map[] = {
47 [0] = 32,
48 [1] = 36,
49 [2] = 40,
50 [3] = 42,
51 [4] = 44,
52 [5] = 48,
53 [6] = 52,
54};
55
56/* The cpu-specific constant value of PAMax; also used by hw/arm/virt. */
57unsigned int arm_pamax(ARMCPU *cpu)
58{
Richard Henderson22536b12022-06-18 17:15:40 -070059 if (arm_feature(&cpu->env, ARM_FEATURE_AARCH64)) {
60 unsigned int parange =
61 FIELD_EX64(cpu->isar.id_aa64mmfr0, ID_AA64MMFR0, PARANGE);
Richard Henderson1c73d842022-06-08 19:38:52 +010062
Richard Henderson22536b12022-06-18 17:15:40 -070063 /*
64 * id_aa64mmfr0 is a read-only register so values outside of the
65 * supported mappings can be considered an implementation error.
66 */
67 assert(parange < ARRAY_SIZE(pamax_map));
68 return pamax_map[parange];
69 }
Richard Henderson59e1b8a2022-06-18 17:15:41 -070070
71 /*
72 * In machvirt_init, we call arm_pamax on a cpu that is not fully
73 * initialized, so we can't rely on the propagation done in realize.
74 */
75 if (arm_feature(&cpu->env, ARM_FEATURE_LPAE) ||
76 arm_feature(&cpu->env, ARM_FEATURE_V7VE)) {
Richard Henderson22536b12022-06-18 17:15:40 -070077 /* v7 with LPAE */
78 return 40;
79 }
80 /* Anything else */
81 return 32;
Richard Henderson1c73d842022-06-08 19:38:52 +010082}
83
Richard Henderson1d261252022-06-08 19:38:54 +010084/*
85 * Convert a possible stage1+2 MMU index into the appropriate stage 1 MMU index
86 */
87ARMMMUIdx stage_1_mmu_idx(ARMMMUIdx mmu_idx)
88{
89 switch (mmu_idx) {
Richard Henderson1d261252022-06-08 19:38:54 +010090 case ARMMMUIdx_E10_0:
91 return ARMMMUIdx_Stage1_E0;
92 case ARMMMUIdx_E10_1:
93 return ARMMMUIdx_Stage1_E1;
94 case ARMMMUIdx_E10_1_PAN:
95 return ARMMMUIdx_Stage1_E1_PAN;
96 default:
97 return mmu_idx;
98 }
99}
100
101ARMMMUIdx arm_stage1_mmu_idx(CPUARMState *env)
102{
103 return stage_1_mmu_idx(arm_mmu_idx(env));
104}
105
Richard Henderson11552bb2022-06-08 19:38:51 +0100106static bool regime_translation_big_endian(CPUARMState *env, ARMMMUIdx mmu_idx)
107{
108 return (regime_sctlr(env, mmu_idx) & SCTLR_EE) != 0;
109}
110
Richard Henderson3b318aa2022-06-08 19:38:53 +0100111/* Return the TTBR associated with this translation regime */
112static uint64_t regime_ttbr(CPUARMState *env, ARMMMUIdx mmu_idx, int ttbrn)
113{
114 if (mmu_idx == ARMMMUIdx_Stage2) {
115 return env->cp15.vttbr_el2;
116 }
117 if (mmu_idx == ARMMMUIdx_Stage2_S) {
118 return env->cp15.vsttbr_el2;
119 }
120 if (ttbrn == 0) {
121 return env->cp15.ttbr0_el[regime_el(env, mmu_idx)];
122 } else {
123 return env->cp15.ttbr1_el[regime_el(env, mmu_idx)];
124 }
125}
126
Richard Henderson8db1a3a2022-06-08 19:38:53 +0100127/* Return true if the specified stage of address translation is disabled */
Richard Henderson7e80c0a2022-10-01 09:22:40 -0700128static bool regime_translation_disabled(CPUARMState *env, ARMMMUIdx mmu_idx,
129 bool is_secure)
Richard Henderson8db1a3a2022-06-08 19:38:53 +0100130{
131 uint64_t hcr_el2;
132
133 if (arm_feature(env, ARM_FEATURE_M)) {
Richard Henderson7e80c0a2022-10-01 09:22:40 -0700134 switch (env->v7m.mpu_ctrl[is_secure] &
Richard Henderson8db1a3a2022-06-08 19:38:53 +0100135 (R_V7M_MPU_CTRL_ENABLE_MASK | R_V7M_MPU_CTRL_HFNMIENA_MASK)) {
136 case R_V7M_MPU_CTRL_ENABLE_MASK:
137 /* Enabled, but not for HardFault and NMI */
138 return mmu_idx & ARM_MMU_IDX_M_NEGPRI;
139 case R_V7M_MPU_CTRL_ENABLE_MASK | R_V7M_MPU_CTRL_HFNMIENA_MASK:
140 /* Enabled for all cases */
141 return false;
142 case 0:
143 default:
144 /*
145 * HFNMIENA set and ENABLE clear is UNPREDICTABLE, but
146 * we warned about that in armv7m_nvic.c when the guest set it.
147 */
148 return true;
149 }
150 }
151
Richard Henderson2189c792022-10-01 09:22:53 -0700152 hcr_el2 = arm_hcr_el2_eff_secstate(env, is_secure);
Richard Henderson8db1a3a2022-06-08 19:38:53 +0100153
Richard Henderson3b2af992022-10-01 09:22:47 -0700154 switch (mmu_idx) {
155 case ARMMMUIdx_Stage2:
156 case ARMMMUIdx_Stage2_S:
Richard Henderson8db1a3a2022-06-08 19:38:53 +0100157 /* HCR.DC means HCR.VM behaves as 1 */
158 return (hcr_el2 & (HCR_DC | HCR_VM)) == 0;
Richard Henderson8db1a3a2022-06-08 19:38:53 +0100159
Richard Henderson3b2af992022-10-01 09:22:47 -0700160 case ARMMMUIdx_E10_0:
161 case ARMMMUIdx_E10_1:
162 case ARMMMUIdx_E10_1_PAN:
Richard Hendersonfdf12932022-10-01 09:22:48 -0700163 /* TGE means that EL0/1 act as if SCTLR_EL1.M is zero */
164 if (hcr_el2 & HCR_TGE) {
Richard Henderson8db1a3a2022-06-08 19:38:53 +0100165 return true;
166 }
Richard Henderson3b2af992022-10-01 09:22:47 -0700167 break;
Richard Henderson8db1a3a2022-06-08 19:38:53 +0100168
Richard Henderson3b2af992022-10-01 09:22:47 -0700169 case ARMMMUIdx_Stage1_E0:
170 case ARMMMUIdx_Stage1_E1:
171 case ARMMMUIdx_Stage1_E1_PAN:
Richard Henderson8db1a3a2022-06-08 19:38:53 +0100172 /* HCR.DC means SCTLR_EL1.M behaves as 0 */
Richard Henderson3b2af992022-10-01 09:22:47 -0700173 if (hcr_el2 & HCR_DC) {
174 return true;
175 }
176 break;
177
178 case ARMMMUIdx_E20_0:
179 case ARMMMUIdx_E20_2:
180 case ARMMMUIdx_E20_2_PAN:
181 case ARMMMUIdx_E2:
182 case ARMMMUIdx_E3:
183 break;
184
Richard Hendersona1ce3082022-10-10 20:18:51 -0700185 case ARMMMUIdx_Phys_NS:
186 case ARMMMUIdx_Phys_S:
187 /* No translation for physical address spaces. */
188 return true;
189
Richard Henderson3b2af992022-10-01 09:22:47 -0700190 default:
191 g_assert_not_reached();
Richard Henderson8db1a3a2022-06-08 19:38:53 +0100192 }
193
194 return (regime_sctlr(env, mmu_idx) & SCTLR_M) == 0;
195}
196
Richard Hendersonf3639a62022-10-10 20:18:57 -0700197static bool S2_attrs_are_device(uint64_t hcr, uint8_t attrs)
Richard Henderson11552bb2022-06-08 19:38:51 +0100198{
199 /*
200 * For an S1 page table walk, the stage 1 attributes are always
201 * some form of "this is Normal memory". The combined S1+S2
202 * attributes are therefore only Device if stage 2 specifies Device.
203 * With HCR_EL2.FWB == 0 this is when descriptor bits [5:4] are 0b00,
204 * ie when cacheattrs.attrs bits [3:2] are 0b00.
205 * With HCR_EL2.FWB == 1 this is when descriptor bit [4] is 0, ie
206 * when cacheattrs.attrs bit [2] is 0.
207 */
Richard Hendersonac76c2e2022-10-01 09:22:52 -0700208 if (hcr & HCR_FWB) {
Richard Hendersonf3639a62022-10-10 20:18:57 -0700209 return (attrs & 0x4) == 0;
Richard Henderson11552bb2022-06-08 19:38:51 +0100210 } else {
Richard Hendersonf3639a62022-10-10 20:18:57 -0700211 return (attrs & 0xc) == 0;
Richard Henderson11552bb2022-06-08 19:38:51 +0100212 }
213}
214
215/* Translate a S1 pagetable walk through S2 if needed. */
Richard Henderson6d2654f2022-10-10 20:18:54 -0700216static bool S1_ptw_translate(CPUARMState *env, S1Translate *ptw,
217 hwaddr addr, ARMMMUFaultInfo *fi)
Richard Henderson11552bb2022-06-08 19:38:51 +0100218{
Richard Henderson6d2654f2022-10-10 20:18:54 -0700219 bool is_secure = ptw->in_secure;
Richard Hendersonf3639a62022-10-10 20:18:57 -0700220 ARMMMUIdx mmu_idx = ptw->in_mmu_idx;
Richard Henderson48da29e2022-10-24 15:18:39 +1000221 ARMMMUIdx s2_mmu_idx = ptw->in_ptw_idx;
Richard Hendersonf3639a62022-10-10 20:18:57 -0700222 uint8_t pte_attrs;
223 bool pte_secure;
Richard Hendersonbf25b7b2022-10-01 09:22:39 -0700224
Richard Henderson71943a12022-10-24 15:18:49 +1000225 ptw->out_virt = addr;
226
Richard Hendersonf3639a62022-10-10 20:18:57 -0700227 if (unlikely(ptw->in_debug)) {
228 /*
229 * From gdbstub, do not use softmmu so that we don't modify the
230 * state of the cpu at all, including softmmu tlb contents.
231 */
Richard Henderson48da29e2022-10-24 15:18:39 +1000232 if (regime_is_stage2(s2_mmu_idx)) {
Richard Hendersonf3639a62022-10-10 20:18:57 -0700233 S1Translate s2ptw = {
234 .in_mmu_idx = s2_mmu_idx,
Richard Henderson48da29e2022-10-24 15:18:39 +1000235 .in_ptw_idx = is_secure ? ARMMMUIdx_Phys_S : ARMMMUIdx_Phys_NS,
Richard Hendersonf3639a62022-10-10 20:18:57 -0700236 .in_secure = is_secure,
237 .in_debug = true,
238 };
239 GetPhysAddrResult s2 = { };
Richard Henderson48da29e2022-10-24 15:18:39 +1000240
Richard Henderson4a1103a2023-01-13 19:46:05 -1000241 if (get_phys_addr_lpae(env, &s2ptw, addr, MMU_DATA_LOAD,
242 false, &s2, fi)) {
Richard Hendersonf3639a62022-10-10 20:18:57 -0700243 goto fail;
244 }
245 ptw->out_phys = s2.f.phys_addr;
246 pte_attrs = s2.cacheattrs.attrs;
247 pte_secure = s2.f.attrs.secure;
Richard Henderson48da29e2022-10-24 15:18:39 +1000248 } else {
249 /* Regime is physical. */
250 ptw->out_phys = addr;
251 pte_attrs = 0;
252 pte_secure = is_secure;
Richard Henderson11552bb2022-06-08 19:38:51 +0100253 }
Richard Hendersonf3639a62022-10-10 20:18:57 -0700254 ptw->out_host = NULL;
Richard Henderson71943a12022-10-24 15:18:49 +1000255 ptw->out_rw = false;
Richard Hendersonf3639a62022-10-10 20:18:57 -0700256 } else {
Fabiano Rosas0d3de772023-02-17 17:11:35 -0300257#ifdef CONFIG_TCG
Richard Hendersonf3639a62022-10-10 20:18:57 -0700258 CPUTLBEntryFull *full;
259 int flags;
Richard Hendersonac76c2e2022-10-01 09:22:52 -0700260
Richard Hendersonf3639a62022-10-10 20:18:57 -0700261 env->tlb_fi = fi;
Richard Hendersond507e6c2023-02-23 14:44:14 -1000262 flags = probe_access_full(env, addr, 0, MMU_DATA_LOAD,
Richard Hendersonf3639a62022-10-10 20:18:57 -0700263 arm_to_core_mmu_idx(s2_mmu_idx),
264 true, &ptw->out_host, &full, 0);
265 env->tlb_fi = NULL;
266
267 if (unlikely(flags & TLB_INVALID_MASK)) {
268 goto fail;
269 }
Richard Henderson9d2617a2023-01-26 13:31:34 -1000270 ptw->out_phys = full->phys_addr | (addr & ~TARGET_PAGE_MASK);
Richard Henderson71943a12022-10-24 15:18:49 +1000271 ptw->out_rw = full->prot & PAGE_WRITE;
Richard Hendersonf3639a62022-10-10 20:18:57 -0700272 pte_attrs = full->pte_attrs;
273 pte_secure = full->attrs.secure;
Fabiano Rosas0d3de772023-02-17 17:11:35 -0300274#else
275 g_assert_not_reached();
276#endif
Richard Hendersonf3639a62022-10-10 20:18:57 -0700277 }
278
Richard Henderson48da29e2022-10-24 15:18:39 +1000279 if (regime_is_stage2(s2_mmu_idx)) {
Richard Hendersonf3639a62022-10-10 20:18:57 -0700280 uint64_t hcr = arm_hcr_el2_eff_secstate(env, is_secure);
281
282 if ((hcr & HCR_PTW) && S2_attrs_are_device(hcr, pte_attrs)) {
Richard Henderson11552bb2022-06-08 19:38:51 +0100283 /*
284 * PTW set and S1 walk touched S2 Device memory:
285 * generate Permission fault.
286 */
287 fi->type = ARMFault_Permission;
288 fi->s2addr = addr;
289 fi->stage2 = true;
290 fi->s1ptw = true;
Richard Hendersonab1f7882022-10-01 09:22:50 -0700291 fi->s1ns = !is_secure;
Richard Henderson6d2654f2022-10-10 20:18:54 -0700292 return false;
Richard Henderson11552bb2022-06-08 19:38:51 +0100293 }
Richard Henderson11552bb2022-06-08 19:38:51 +0100294 }
Richard Henderson6d2654f2022-10-10 20:18:54 -0700295
Richard Hendersonf3639a62022-10-10 20:18:57 -0700296 /* Check if page table walk is to secure or non-secure PA space. */
297 ptw->out_secure = (is_secure
298 && !(pte_secure
299 ? env->cp15.vstcr_el2 & VSTCR_SW
300 : env->cp15.vtcr_el2 & VTCR_NSW));
301 ptw->out_be = regime_translation_big_endian(env, mmu_idx);
Richard Henderson6d2654f2022-10-10 20:18:54 -0700302 return true;
Richard Hendersonf3639a62022-10-10 20:18:57 -0700303
304 fail:
305 assert(fi->type != ARMFault_None);
306 fi->s2addr = addr;
307 fi->stage2 = true;
308 fi->s1ptw = true;
309 fi->s1ns = !is_secure;
310 return false;
Richard Henderson11552bb2022-06-08 19:38:51 +0100311}
312
313/* All loads done in the course of a page table walk go through here. */
Richard Henderson93e5b3a2022-10-24 15:18:42 +1000314static uint32_t arm_ldl_ptw(CPUARMState *env, S1Translate *ptw,
Richard Henderson6d2654f2022-10-10 20:18:54 -0700315 ARMMMUFaultInfo *fi)
Richard Henderson11552bb2022-06-08 19:38:51 +0100316{
Richard Henderson5e798872022-06-08 19:38:54 +0100317 CPUState *cs = env_cpu(env);
Richard Henderson71943a12022-10-24 15:18:49 +1000318 void *host = ptw->out_host;
Richard Henderson11552bb2022-06-08 19:38:51 +0100319 uint32_t data;
320
Richard Henderson71943a12022-10-24 15:18:49 +1000321 if (likely(host)) {
Richard Hendersonf3639a62022-10-10 20:18:57 -0700322 /* Page tables are in RAM, and we have the host address. */
Richard Henderson71943a12022-10-24 15:18:49 +1000323 data = qatomic_read((uint32_t *)host);
Richard Hendersonf3639a62022-10-10 20:18:57 -0700324 if (ptw->out_be) {
Richard Henderson71943a12022-10-24 15:18:49 +1000325 data = be32_to_cpu(data);
Richard Hendersonf3639a62022-10-10 20:18:57 -0700326 } else {
Richard Henderson71943a12022-10-24 15:18:49 +1000327 data = le32_to_cpu(data);
Richard Hendersonf3639a62022-10-10 20:18:57 -0700328 }
Richard Henderson11552bb2022-06-08 19:38:51 +0100329 } else {
Richard Hendersonf3639a62022-10-10 20:18:57 -0700330 /* Page tables are in MMIO. */
331 MemTxAttrs attrs = { .secure = ptw->out_secure };
332 AddressSpace *as = arm_addressspace(cs, attrs);
333 MemTxResult result = MEMTX_OK;
334
335 if (ptw->out_be) {
336 data = address_space_ldl_be(as, ptw->out_phys, attrs, &result);
337 } else {
338 data = address_space_ldl_le(as, ptw->out_phys, attrs, &result);
339 }
340 if (unlikely(result != MEMTX_OK)) {
341 fi->type = ARMFault_SyncExternalOnWalk;
342 fi->ea = arm_extabort_type(result);
343 return 0;
344 }
Richard Henderson11552bb2022-06-08 19:38:51 +0100345 }
Richard Hendersonf3639a62022-10-10 20:18:57 -0700346 return data;
Richard Henderson11552bb2022-06-08 19:38:51 +0100347}
348
Richard Henderson93e5b3a2022-10-24 15:18:42 +1000349static uint64_t arm_ldq_ptw(CPUARMState *env, S1Translate *ptw,
Richard Henderson6d2654f2022-10-10 20:18:54 -0700350 ARMMMUFaultInfo *fi)
Richard Henderson11552bb2022-06-08 19:38:51 +0100351{
Richard Henderson5e798872022-06-08 19:38:54 +0100352 CPUState *cs = env_cpu(env);
Richard Henderson71943a12022-10-24 15:18:49 +1000353 void *host = ptw->out_host;
Richard Henderson11552bb2022-06-08 19:38:51 +0100354 uint64_t data;
355
Richard Henderson71943a12022-10-24 15:18:49 +1000356 if (likely(host)) {
Richard Hendersonf3639a62022-10-10 20:18:57 -0700357 /* Page tables are in RAM, and we have the host address. */
Richard Henderson71943a12022-10-24 15:18:49 +1000358#ifdef CONFIG_ATOMIC64
359 data = qatomic_read__nocheck((uint64_t *)host);
Richard Hendersonf3639a62022-10-10 20:18:57 -0700360 if (ptw->out_be) {
Richard Henderson71943a12022-10-24 15:18:49 +1000361 data = be64_to_cpu(data);
Richard Hendersonf3639a62022-10-10 20:18:57 -0700362 } else {
Richard Henderson71943a12022-10-24 15:18:49 +1000363 data = le64_to_cpu(data);
Richard Hendersonf3639a62022-10-10 20:18:57 -0700364 }
Richard Henderson71943a12022-10-24 15:18:49 +1000365#else
366 if (ptw->out_be) {
367 data = ldq_be_p(host);
368 } else {
369 data = ldq_le_p(host);
370 }
371#endif
Richard Henderson11552bb2022-06-08 19:38:51 +0100372 } else {
Richard Hendersonf3639a62022-10-10 20:18:57 -0700373 /* Page tables are in MMIO. */
374 MemTxAttrs attrs = { .secure = ptw->out_secure };
375 AddressSpace *as = arm_addressspace(cs, attrs);
376 MemTxResult result = MEMTX_OK;
377
378 if (ptw->out_be) {
379 data = address_space_ldq_be(as, ptw->out_phys, attrs, &result);
380 } else {
381 data = address_space_ldq_le(as, ptw->out_phys, attrs, &result);
382 }
383 if (unlikely(result != MEMTX_OK)) {
384 fi->type = ARMFault_SyncExternalOnWalk;
385 fi->ea = arm_extabort_type(result);
386 return 0;
387 }
Richard Henderson11552bb2022-06-08 19:38:51 +0100388 }
Richard Hendersonf3639a62022-10-10 20:18:57 -0700389 return data;
Richard Henderson11552bb2022-06-08 19:38:51 +0100390}
391
Richard Henderson71943a12022-10-24 15:18:49 +1000392static uint64_t arm_casq_ptw(CPUARMState *env, uint64_t old_val,
393 uint64_t new_val, S1Translate *ptw,
394 ARMMMUFaultInfo *fi)
395{
396 uint64_t cur_val;
397 void *host = ptw->out_host;
398
399 if (unlikely(!host)) {
400 fi->type = ARMFault_UnsuppAtomicUpdate;
401 fi->s1ptw = true;
402 return 0;
403 }
404
405 /*
406 * Raising a stage2 Protection fault for an atomic update to a read-only
407 * page is delayed until it is certain that there is a change to make.
408 */
409 if (unlikely(!ptw->out_rw)) {
410 int flags;
411 void *discard;
412
413 env->tlb_fi = fi;
Daniel Henrique Barboza1770b2f2023-02-23 20:44:24 -0300414 flags = probe_access_flags(env, ptw->out_virt, 0, MMU_DATA_STORE,
Richard Henderson71943a12022-10-24 15:18:49 +1000415 arm_to_core_mmu_idx(ptw->in_ptw_idx),
416 true, &discard, 0);
417 env->tlb_fi = NULL;
418
419 if (unlikely(flags & TLB_INVALID_MASK)) {
420 assert(fi->type != ARMFault_None);
421 fi->s2addr = ptw->out_virt;
422 fi->stage2 = true;
423 fi->s1ptw = true;
424 fi->s1ns = !ptw->in_secure;
425 return 0;
426 }
427
428 /* In case CAS mismatches and we loop, remember writability. */
429 ptw->out_rw = true;
430 }
431
432#ifdef CONFIG_ATOMIC64
433 if (ptw->out_be) {
434 old_val = cpu_to_be64(old_val);
435 new_val = cpu_to_be64(new_val);
436 cur_val = qatomic_cmpxchg__nocheck((uint64_t *)host, old_val, new_val);
437 cur_val = be64_to_cpu(cur_val);
438 } else {
439 old_val = cpu_to_le64(old_val);
440 new_val = cpu_to_le64(new_val);
441 cur_val = qatomic_cmpxchg__nocheck((uint64_t *)host, old_val, new_val);
442 cur_val = le64_to_cpu(cur_val);
443 }
444#else
445 /*
446 * We can't support the full 64-bit atomic cmpxchg on the host.
447 * Because this is only used for FEAT_HAFDBS, which is only for AA64,
448 * we know that TCG_OVERSIZED_GUEST is set, which means that we are
449 * running in round-robin mode and could only race with dma i/o.
450 */
451#ifndef TCG_OVERSIZED_GUEST
452# error "Unexpected configuration"
453#endif
454 bool locked = qemu_mutex_iothread_locked();
455 if (!locked) {
456 qemu_mutex_lock_iothread();
457 }
458 if (ptw->out_be) {
459 cur_val = ldq_be_p(host);
460 if (cur_val == old_val) {
461 stq_be_p(host, new_val);
462 }
463 } else {
464 cur_val = ldq_le_p(host);
465 if (cur_val == old_val) {
466 stq_le_p(host, new_val);
467 }
468 }
469 if (!locked) {
470 qemu_mutex_unlock_iothread();
471 }
472#endif
473
474 return cur_val;
475}
476
Richard Henderson4c74ab12022-06-08 19:38:51 +0100477static bool get_level1_table_address(CPUARMState *env, ARMMMUIdx mmu_idx,
478 uint32_t *table, uint32_t address)
479{
480 /* Note that we can only get here for an AArch32 PL0/PL1 lookup */
Peter Maydellc1547bb2022-07-14 14:22:59 +0100481 uint64_t tcr = regime_tcr(env, mmu_idx);
Peter Maydell9e70e262022-07-14 14:22:58 +0100482 int maskshift = extract32(tcr, 0, 3);
483 uint32_t mask = ~(((uint32_t)0xffffffffu) >> maskshift);
484 uint32_t base_mask;
Richard Henderson4c74ab12022-06-08 19:38:51 +0100485
Peter Maydell9e70e262022-07-14 14:22:58 +0100486 if (address & mask) {
487 if (tcr & TTBCR_PD1) {
Richard Henderson4c74ab12022-06-08 19:38:51 +0100488 /* Translation table walk disabled for TTBR1 */
489 return false;
490 }
491 *table = regime_ttbr(env, mmu_idx, 1) & 0xffffc000;
492 } else {
Peter Maydell9e70e262022-07-14 14:22:58 +0100493 if (tcr & TTBCR_PD0) {
Richard Henderson4c74ab12022-06-08 19:38:51 +0100494 /* Translation table walk disabled for TTBR0 */
495 return false;
496 }
Peter Maydell9e70e262022-07-14 14:22:58 +0100497 base_mask = ~((uint32_t)0x3fffu >> maskshift);
498 *table = regime_ttbr(env, mmu_idx, 0) & base_mask;
Richard Henderson4c74ab12022-06-08 19:38:51 +0100499 }
500 *table |= (address >> 18) & 0x3ffc;
501 return true;
502}
503
Richard Henderson4845d3b2022-06-08 19:38:53 +0100504/*
505 * Translate section/page access permissions to page R/W protection flags
506 * @env: CPUARMState
507 * @mmu_idx: MMU index indicating required translation regime
508 * @ap: The 3-bit access permissions (AP[2:0])
509 * @domain_prot: The 2-bit domain access permissions
Timofey Kutergin6f2d9d72022-11-03 13:10:41 +0000510 * @is_user: TRUE if accessing from PL0
Richard Henderson4845d3b2022-06-08 19:38:53 +0100511 */
Timofey Kutergin6f2d9d72022-11-03 13:10:41 +0000512static int ap_to_rw_prot_is_user(CPUARMState *env, ARMMMUIdx mmu_idx,
513 int ap, int domain_prot, bool is_user)
Richard Henderson4845d3b2022-06-08 19:38:53 +0100514{
Richard Henderson4845d3b2022-06-08 19:38:53 +0100515 if (domain_prot == 3) {
516 return PAGE_READ | PAGE_WRITE;
517 }
518
519 switch (ap) {
520 case 0:
521 if (arm_feature(env, ARM_FEATURE_V7)) {
522 return 0;
523 }
524 switch (regime_sctlr(env, mmu_idx) & (SCTLR_S | SCTLR_R)) {
525 case SCTLR_S:
526 return is_user ? 0 : PAGE_READ;
527 case SCTLR_R:
528 return PAGE_READ;
529 default:
530 return 0;
531 }
532 case 1:
533 return is_user ? 0 : PAGE_READ | PAGE_WRITE;
534 case 2:
535 if (is_user) {
536 return PAGE_READ;
537 } else {
538 return PAGE_READ | PAGE_WRITE;
539 }
540 case 3:
541 return PAGE_READ | PAGE_WRITE;
542 case 4: /* Reserved. */
543 return 0;
544 case 5:
545 return is_user ? 0 : PAGE_READ;
546 case 6:
547 return PAGE_READ;
548 case 7:
549 if (!arm_feature(env, ARM_FEATURE_V6K)) {
550 return 0;
551 }
552 return PAGE_READ;
553 default:
554 g_assert_not_reached();
555 }
556}
557
558/*
Timofey Kutergin6f2d9d72022-11-03 13:10:41 +0000559 * Translate section/page access permissions to page R/W protection flags
560 * @env: CPUARMState
561 * @mmu_idx: MMU index indicating required translation regime
562 * @ap: The 3-bit access permissions (AP[2:0])
563 * @domain_prot: The 2-bit domain access permissions
564 */
565static int ap_to_rw_prot(CPUARMState *env, ARMMMUIdx mmu_idx,
566 int ap, int domain_prot)
567{
568 return ap_to_rw_prot_is_user(env, mmu_idx, ap, domain_prot,
569 regime_is_user(env, mmu_idx));
570}
571
572/*
Richard Henderson4845d3b2022-06-08 19:38:53 +0100573 * Translate section/page access permissions to page R/W protection flags.
574 * @ap: The 2-bit simple AP (AP[2:1])
575 * @is_user: TRUE if accessing from PL0
576 */
577static int simple_ap_to_rw_prot_is_user(int ap, bool is_user)
578{
579 switch (ap) {
580 case 0:
581 return is_user ? 0 : PAGE_READ | PAGE_WRITE;
582 case 1:
583 return PAGE_READ | PAGE_WRITE;
584 case 2:
585 return is_user ? 0 : PAGE_READ;
586 case 3:
587 return PAGE_READ;
588 default:
589 g_assert_not_reached();
590 }
591}
592
593static int simple_ap_to_rw_prot(CPUARMState *env, ARMMMUIdx mmu_idx, int ap)
594{
595 return simple_ap_to_rw_prot_is_user(ap, regime_is_user(env, mmu_idx));
596}
597
Richard Henderson6d2654f2022-10-10 20:18:54 -0700598static bool get_phys_addr_v5(CPUARMState *env, S1Translate *ptw,
599 uint32_t address, MMUAccessType access_type,
600 GetPhysAddrResult *result, ARMMMUFaultInfo *fi)
Richard Hendersonf2d2f5c2022-06-08 19:38:48 +0100601{
Richard Hendersonf2d2f5c2022-06-08 19:38:48 +0100602 int level = 1;
603 uint32_t table;
604 uint32_t desc;
605 int type;
606 int ap;
607 int domain = 0;
608 int domain_prot;
609 hwaddr phys_addr;
610 uint32_t dacr;
611
612 /* Pagetable walk. */
613 /* Lookup l1 descriptor. */
Richard Henderson6d2654f2022-10-10 20:18:54 -0700614 if (!get_level1_table_address(env, ptw->in_mmu_idx, &table, address)) {
Richard Hendersonf2d2f5c2022-06-08 19:38:48 +0100615 /* Section translation fault if page walk is disabled by PD0 or PD1 */
616 fi->type = ARMFault_Translation;
617 goto do_fault;
618 }
Richard Henderson93e5b3a2022-10-24 15:18:42 +1000619 if (!S1_ptw_translate(env, ptw, table, fi)) {
620 goto do_fault;
621 }
622 desc = arm_ldl_ptw(env, ptw, fi);
Richard Hendersonf2d2f5c2022-06-08 19:38:48 +0100623 if (fi->type != ARMFault_None) {
624 goto do_fault;
625 }
626 type = (desc & 3);
627 domain = (desc >> 5) & 0x0f;
Richard Henderson6d2654f2022-10-10 20:18:54 -0700628 if (regime_el(env, ptw->in_mmu_idx) == 1) {
Richard Hendersonf2d2f5c2022-06-08 19:38:48 +0100629 dacr = env->cp15.dacr_ns;
630 } else {
631 dacr = env->cp15.dacr_s;
632 }
633 domain_prot = (dacr >> (domain * 2)) & 3;
634 if (type == 0) {
635 /* Section translation fault. */
636 fi->type = ARMFault_Translation;
637 goto do_fault;
638 }
639 if (type != 2) {
640 level = 2;
641 }
642 if (domain_prot == 0 || domain_prot == 2) {
643 fi->type = ARMFault_Domain;
644 goto do_fault;
645 }
646 if (type == 2) {
647 /* 1Mb section. */
648 phys_addr = (desc & 0xfff00000) | (address & 0x000fffff);
649 ap = (desc >> 10) & 3;
Richard Henderson7fa7ea82022-10-01 09:22:56 -0700650 result->f.lg_page_size = 20; /* 1MB */
Richard Hendersonf2d2f5c2022-06-08 19:38:48 +0100651 } else {
652 /* Lookup l2 entry. */
653 if (type == 1) {
654 /* Coarse pagetable. */
655 table = (desc & 0xfffffc00) | ((address >> 10) & 0x3fc);
656 } else {
657 /* Fine pagetable. */
658 table = (desc & 0xfffff000) | ((address >> 8) & 0xffc);
659 }
Richard Henderson93e5b3a2022-10-24 15:18:42 +1000660 if (!S1_ptw_translate(env, ptw, table, fi)) {
661 goto do_fault;
662 }
663 desc = arm_ldl_ptw(env, ptw, fi);
Richard Hendersonf2d2f5c2022-06-08 19:38:48 +0100664 if (fi->type != ARMFault_None) {
665 goto do_fault;
666 }
667 switch (desc & 3) {
668 case 0: /* Page translation fault. */
669 fi->type = ARMFault_Translation;
670 goto do_fault;
671 case 1: /* 64k page. */
672 phys_addr = (desc & 0xffff0000) | (address & 0xffff);
673 ap = (desc >> (4 + ((address >> 13) & 6))) & 3;
Richard Henderson7fa7ea82022-10-01 09:22:56 -0700674 result->f.lg_page_size = 16;
Richard Hendersonf2d2f5c2022-06-08 19:38:48 +0100675 break;
676 case 2: /* 4k page. */
677 phys_addr = (desc & 0xfffff000) | (address & 0xfff);
678 ap = (desc >> (4 + ((address >> 9) & 6))) & 3;
Richard Henderson7fa7ea82022-10-01 09:22:56 -0700679 result->f.lg_page_size = 12;
Richard Hendersonf2d2f5c2022-06-08 19:38:48 +0100680 break;
681 case 3: /* 1k page, or ARMv6/XScale "extended small (4k) page" */
682 if (type == 1) {
683 /* ARMv6/XScale extended small page format */
684 if (arm_feature(env, ARM_FEATURE_XSCALE)
685 || arm_feature(env, ARM_FEATURE_V6)) {
686 phys_addr = (desc & 0xfffff000) | (address & 0xfff);
Richard Henderson7fa7ea82022-10-01 09:22:56 -0700687 result->f.lg_page_size = 12;
Richard Hendersonf2d2f5c2022-06-08 19:38:48 +0100688 } else {
689 /*
690 * UNPREDICTABLE in ARMv5; we choose to take a
691 * page translation fault.
692 */
693 fi->type = ARMFault_Translation;
694 goto do_fault;
695 }
696 } else {
697 phys_addr = (desc & 0xfffffc00) | (address & 0x3ff);
Richard Henderson7fa7ea82022-10-01 09:22:56 -0700698 result->f.lg_page_size = 10;
Richard Hendersonf2d2f5c2022-06-08 19:38:48 +0100699 }
700 ap = (desc >> 4) & 3;
701 break;
702 default:
703 /* Never happens, but compiler isn't smart enough to tell. */
704 g_assert_not_reached();
705 }
706 }
Richard Henderson6d2654f2022-10-10 20:18:54 -0700707 result->f.prot = ap_to_rw_prot(env, ptw->in_mmu_idx, ap, domain_prot);
Richard Henderson7fa7ea82022-10-01 09:22:56 -0700708 result->f.prot |= result->f.prot ? PAGE_EXEC : 0;
709 if (!(result->f.prot & (1 << access_type))) {
Richard Hendersonf2d2f5c2022-06-08 19:38:48 +0100710 /* Access permission fault. */
711 fi->type = ARMFault_Permission;
712 goto do_fault;
713 }
Richard Henderson7fa7ea82022-10-01 09:22:56 -0700714 result->f.phys_addr = phys_addr;
Richard Hendersonf2d2f5c2022-06-08 19:38:48 +0100715 return false;
716do_fault:
717 fi->domain = domain;
718 fi->level = level;
719 return true;
720}
721
Richard Henderson6d2654f2022-10-10 20:18:54 -0700722static bool get_phys_addr_v6(CPUARMState *env, S1Translate *ptw,
723 uint32_t address, MMUAccessType access_type,
724 GetPhysAddrResult *result, ARMMMUFaultInfo *fi)
Richard Henderson53c038e2022-06-08 19:38:48 +0100725{
Richard Henderson53c038e2022-06-08 19:38:48 +0100726 ARMCPU *cpu = env_archcpu(env);
Richard Henderson6d2654f2022-10-10 20:18:54 -0700727 ARMMMUIdx mmu_idx = ptw->in_mmu_idx;
Richard Henderson53c038e2022-06-08 19:38:48 +0100728 int level = 1;
729 uint32_t table;
730 uint32_t desc;
731 uint32_t xn;
732 uint32_t pxn = 0;
733 int type;
734 int ap;
735 int domain = 0;
736 int domain_prot;
737 hwaddr phys_addr;
738 uint32_t dacr;
739 bool ns;
Timofey Kutergin6f2d9d72022-11-03 13:10:41 +0000740 int user_prot;
Richard Henderson53c038e2022-06-08 19:38:48 +0100741
742 /* Pagetable walk. */
743 /* Lookup l1 descriptor. */
744 if (!get_level1_table_address(env, mmu_idx, &table, address)) {
745 /* Section translation fault if page walk is disabled by PD0 or PD1 */
746 fi->type = ARMFault_Translation;
747 goto do_fault;
748 }
Richard Henderson93e5b3a2022-10-24 15:18:42 +1000749 if (!S1_ptw_translate(env, ptw, table, fi)) {
750 goto do_fault;
751 }
752 desc = arm_ldl_ptw(env, ptw, fi);
Richard Henderson53c038e2022-06-08 19:38:48 +0100753 if (fi->type != ARMFault_None) {
754 goto do_fault;
755 }
756 type = (desc & 3);
757 if (type == 0 || (type == 3 && !cpu_isar_feature(aa32_pxn, cpu))) {
758 /* Section translation fault, or attempt to use the encoding
759 * which is Reserved on implementations without PXN.
760 */
761 fi->type = ARMFault_Translation;
762 goto do_fault;
763 }
764 if ((type == 1) || !(desc & (1 << 18))) {
765 /* Page or Section. */
766 domain = (desc >> 5) & 0x0f;
767 }
768 if (regime_el(env, mmu_idx) == 1) {
769 dacr = env->cp15.dacr_ns;
770 } else {
771 dacr = env->cp15.dacr_s;
772 }
773 if (type == 1) {
774 level = 2;
775 }
776 domain_prot = (dacr >> (domain * 2)) & 3;
777 if (domain_prot == 0 || domain_prot == 2) {
778 /* Section or Page domain fault */
779 fi->type = ARMFault_Domain;
780 goto do_fault;
781 }
782 if (type != 1) {
783 if (desc & (1 << 18)) {
784 /* Supersection. */
785 phys_addr = (desc & 0xff000000) | (address & 0x00ffffff);
786 phys_addr |= (uint64_t)extract32(desc, 20, 4) << 32;
787 phys_addr |= (uint64_t)extract32(desc, 5, 4) << 36;
Richard Henderson7fa7ea82022-10-01 09:22:56 -0700788 result->f.lg_page_size = 24; /* 16MB */
Richard Henderson53c038e2022-06-08 19:38:48 +0100789 } else {
790 /* Section. */
791 phys_addr = (desc & 0xfff00000) | (address & 0x000fffff);
Richard Henderson7fa7ea82022-10-01 09:22:56 -0700792 result->f.lg_page_size = 20; /* 1MB */
Richard Henderson53c038e2022-06-08 19:38:48 +0100793 }
794 ap = ((desc >> 10) & 3) | ((desc >> 13) & 4);
795 xn = desc & (1 << 4);
796 pxn = desc & 1;
797 ns = extract32(desc, 19, 1);
798 } else {
799 if (cpu_isar_feature(aa32_pxn, cpu)) {
800 pxn = (desc >> 2) & 1;
801 }
802 ns = extract32(desc, 3, 1);
803 /* Lookup l2 entry. */
804 table = (desc & 0xfffffc00) | ((address >> 10) & 0x3fc);
Richard Henderson93e5b3a2022-10-24 15:18:42 +1000805 if (!S1_ptw_translate(env, ptw, table, fi)) {
806 goto do_fault;
807 }
808 desc = arm_ldl_ptw(env, ptw, fi);
Richard Henderson53c038e2022-06-08 19:38:48 +0100809 if (fi->type != ARMFault_None) {
810 goto do_fault;
811 }
812 ap = ((desc >> 4) & 3) | ((desc >> 7) & 4);
813 switch (desc & 3) {
814 case 0: /* Page translation fault. */
815 fi->type = ARMFault_Translation;
816 goto do_fault;
817 case 1: /* 64k page. */
818 phys_addr = (desc & 0xffff0000) | (address & 0xffff);
819 xn = desc & (1 << 15);
Richard Henderson7fa7ea82022-10-01 09:22:56 -0700820 result->f.lg_page_size = 16;
Richard Henderson53c038e2022-06-08 19:38:48 +0100821 break;
822 case 2: case 3: /* 4k page. */
823 phys_addr = (desc & 0xfffff000) | (address & 0xfff);
824 xn = desc & 1;
Richard Henderson7fa7ea82022-10-01 09:22:56 -0700825 result->f.lg_page_size = 12;
Richard Henderson53c038e2022-06-08 19:38:48 +0100826 break;
827 default:
828 /* Never happens, but compiler isn't smart enough to tell. */
829 g_assert_not_reached();
830 }
831 }
832 if (domain_prot == 3) {
Richard Henderson7fa7ea82022-10-01 09:22:56 -0700833 result->f.prot = PAGE_READ | PAGE_WRITE | PAGE_EXEC;
Richard Henderson53c038e2022-06-08 19:38:48 +0100834 } else {
835 if (pxn && !regime_is_user(env, mmu_idx)) {
836 xn = 1;
837 }
838 if (xn && access_type == MMU_INST_FETCH) {
839 fi->type = ARMFault_Permission;
840 goto do_fault;
841 }
842
843 if (arm_feature(env, ARM_FEATURE_V6K) &&
844 (regime_sctlr(env, mmu_idx) & SCTLR_AFE)) {
845 /* The simplified model uses AP[0] as an access control bit. */
846 if ((ap & 1) == 0) {
847 /* Access flag fault. */
848 fi->type = ARMFault_AccessFlag;
849 goto do_fault;
850 }
Richard Henderson7fa7ea82022-10-01 09:22:56 -0700851 result->f.prot = simple_ap_to_rw_prot(env, mmu_idx, ap >> 1);
Timofey Kutergin6f2d9d72022-11-03 13:10:41 +0000852 user_prot = simple_ap_to_rw_prot_is_user(ap >> 1, 1);
Richard Henderson53c038e2022-06-08 19:38:48 +0100853 } else {
Richard Henderson7fa7ea82022-10-01 09:22:56 -0700854 result->f.prot = ap_to_rw_prot(env, mmu_idx, ap, domain_prot);
Timofey Kutergin6f2d9d72022-11-03 13:10:41 +0000855 user_prot = ap_to_rw_prot_is_user(env, mmu_idx, ap, domain_prot, 1);
Richard Henderson53c038e2022-06-08 19:38:48 +0100856 }
Richard Henderson7fa7ea82022-10-01 09:22:56 -0700857 if (result->f.prot && !xn) {
858 result->f.prot |= PAGE_EXEC;
Richard Henderson53c038e2022-06-08 19:38:48 +0100859 }
Richard Henderson7fa7ea82022-10-01 09:22:56 -0700860 if (!(result->f.prot & (1 << access_type))) {
Richard Henderson53c038e2022-06-08 19:38:48 +0100861 /* Access permission fault. */
862 fi->type = ARMFault_Permission;
863 goto do_fault;
864 }
Timofey Kutergin6f2d9d72022-11-03 13:10:41 +0000865 if (regime_is_pan(env, mmu_idx) &&
866 !regime_is_user(env, mmu_idx) &&
867 user_prot &&
868 access_type != MMU_INST_FETCH) {
869 /* Privileged Access Never fault */
870 fi->type = ARMFault_Permission;
871 goto do_fault;
872 }
Richard Henderson53c038e2022-06-08 19:38:48 +0100873 }
874 if (ns) {
875 /* The NS bit will (as required by the architecture) have no effect if
876 * the CPU doesn't support TZ or this is a non-secure translation
877 * regime, because the attribute will already be non-secure.
878 */
Richard Henderson7fa7ea82022-10-01 09:22:56 -0700879 result->f.attrs.secure = false;
Richard Henderson53c038e2022-06-08 19:38:48 +0100880 }
Richard Henderson7fa7ea82022-10-01 09:22:56 -0700881 result->f.phys_addr = phys_addr;
Richard Henderson53c038e2022-06-08 19:38:48 +0100882 return false;
883do_fault:
884 fi->domain = domain;
885 fi->level = level;
886 return true;
887}
888
Richard Hendersonf8526ed2022-06-08 19:38:52 +0100889/*
890 * Translate S2 section/page access permissions to protection flags
891 * @env: CPUARMState
892 * @s2ap: The 2-bit stage2 access permissions (S2AP)
893 * @xn: XN (execute-never) bits
894 * @s1_is_el0: true if this is S2 of an S1+2 walk for EL0
895 */
896static int get_S2prot(CPUARMState *env, int s2ap, int xn, bool s1_is_el0)
897{
898 int prot = 0;
899
900 if (s2ap & 1) {
901 prot |= PAGE_READ;
902 }
903 if (s2ap & 2) {
904 prot |= PAGE_WRITE;
905 }
906
907 if (cpu_isar_feature(any_tts2uxn, env_archcpu(env))) {
908 switch (xn) {
909 case 0:
910 prot |= PAGE_EXEC;
911 break;
912 case 1:
913 if (s1_is_el0) {
914 prot |= PAGE_EXEC;
915 }
916 break;
917 case 2:
918 break;
919 case 3:
920 if (!s1_is_el0) {
921 prot |= PAGE_EXEC;
922 }
923 break;
924 default:
925 g_assert_not_reached();
926 }
927 } else {
928 if (!extract32(xn, 1, 1)) {
929 if (arm_el_is_aa64(env, 2) || prot & PAGE_READ) {
930 prot |= PAGE_EXEC;
931 }
932 }
933 }
934 return prot;
935}
936
937/*
938 * Translate section/page access permissions to protection flags
939 * @env: CPUARMState
940 * @mmu_idx: MMU index indicating required translation regime
941 * @is_aa64: TRUE if AArch64
942 * @ap: The 2-bit simple AP (AP[2:1])
943 * @ns: NS (non-secure) bit
944 * @xn: XN (execute-never) bit
945 * @pxn: PXN (privileged execute-never) bit
946 */
947static int get_S1prot(CPUARMState *env, ARMMMUIdx mmu_idx, bool is_aa64,
948 int ap, int ns, int xn, int pxn)
949{
Peter Maydelldd171432023-04-20 10:21:16 +0100950 ARMCPU *cpu = env_archcpu(env);
Richard Hendersonf8526ed2022-06-08 19:38:52 +0100951 bool is_user = regime_is_user(env, mmu_idx);
952 int prot_rw, user_rw;
953 bool have_wxn;
954 int wxn = 0;
955
Richard Hendersonedc05dd2022-10-24 15:18:38 +1000956 assert(!regime_is_stage2(mmu_idx));
Richard Hendersonf8526ed2022-06-08 19:38:52 +0100957
958 user_rw = simple_ap_to_rw_prot_is_user(ap, true);
959 if (is_user) {
960 prot_rw = user_rw;
961 } else {
Peter Maydelldd171432023-04-20 10:21:16 +0100962 /*
963 * PAN controls can forbid data accesses but don't affect insn fetch.
964 * Plain PAN forbids data accesses if EL0 has data permissions;
965 * PAN3 forbids data accesses if EL0 has either data or exec perms.
966 * Note that for AArch64 the 'user can exec' case is exactly !xn.
967 * We make the IMPDEF choices that SCR_EL3.SIF and Realm EL2&0
968 * do not affect EPAN.
969 */
Richard Hendersonf8526ed2022-06-08 19:38:52 +0100970 if (user_rw && regime_is_pan(env, mmu_idx)) {
Peter Maydelldd171432023-04-20 10:21:16 +0100971 prot_rw = 0;
972 } else if (cpu_isar_feature(aa64_pan3, cpu) && is_aa64 &&
973 regime_is_pan(env, mmu_idx) &&
974 (regime_sctlr(env, mmu_idx) & SCTLR_EPAN) && !xn) {
Richard Hendersonf8526ed2022-06-08 19:38:52 +0100975 prot_rw = 0;
976 } else {
977 prot_rw = simple_ap_to_rw_prot_is_user(ap, false);
978 }
979 }
980
981 if (ns && arm_is_secure(env) && (env->cp15.scr_el3 & SCR_SIF)) {
982 return prot_rw;
983 }
984
985 /* TODO have_wxn should be replaced with
986 * ARM_FEATURE_V8 || (ARM_FEATURE_V7 && ARM_FEATURE_EL2)
987 * when ARM_FEATURE_EL2 starts getting set. For now we assume all LPAE
988 * compatible processors have EL2, which is required for [U]WXN.
989 */
990 have_wxn = arm_feature(env, ARM_FEATURE_LPAE);
991
992 if (have_wxn) {
993 wxn = regime_sctlr(env, mmu_idx) & SCTLR_WXN;
994 }
995
996 if (is_aa64) {
997 if (regime_has_2_ranges(mmu_idx) && !is_user) {
998 xn = pxn || (user_rw & PAGE_WRITE);
999 }
1000 } else if (arm_feature(env, ARM_FEATURE_V7)) {
1001 switch (regime_el(env, mmu_idx)) {
1002 case 1:
1003 case 3:
1004 if (is_user) {
1005 xn = xn || !(user_rw & PAGE_READ);
1006 } else {
1007 int uwxn = 0;
1008 if (have_wxn) {
1009 uwxn = regime_sctlr(env, mmu_idx) & SCTLR_UWXN;
1010 }
1011 xn = xn || !(prot_rw & PAGE_READ) || pxn ||
1012 (uwxn && (user_rw & PAGE_WRITE));
1013 }
1014 break;
1015 case 2:
1016 break;
1017 }
1018 } else {
1019 xn = wxn = 0;
1020 }
1021
1022 if (xn || (wxn && (prot_rw & PAGE_WRITE))) {
1023 return prot_rw;
1024 }
1025 return prot_rw | PAGE_EXEC;
1026}
1027
Richard Henderson2f0ec922022-06-08 19:38:53 +01001028static ARMVAParameters aa32_va_parameters(CPUARMState *env, uint32_t va,
1029 ARMMMUIdx mmu_idx)
1030{
Peter Maydellc1547bb2022-07-14 14:22:59 +01001031 uint64_t tcr = regime_tcr(env, mmu_idx);
Richard Henderson2f0ec922022-06-08 19:38:53 +01001032 uint32_t el = regime_el(env, mmu_idx);
1033 int select, tsz;
1034 bool epd, hpd;
1035
1036 assert(mmu_idx != ARMMMUIdx_Stage2_S);
1037
1038 if (mmu_idx == ARMMMUIdx_Stage2) {
1039 /* VTCR */
1040 bool sext = extract32(tcr, 4, 1);
1041 bool sign = extract32(tcr, 3, 1);
1042
1043 /*
1044 * If the sign-extend bit is not the same as t0sz[3], the result
1045 * is unpredictable. Flag this as a guest error.
1046 */
1047 if (sign != sext) {
1048 qemu_log_mask(LOG_GUEST_ERROR,
1049 "AArch32: VTCR.S / VTCR.T0SZ[3] mismatch\n");
1050 }
1051 tsz = sextract32(tcr, 0, 4) + 8;
1052 select = 0;
1053 hpd = false;
1054 epd = false;
1055 } else if (el == 2) {
1056 /* HTCR */
1057 tsz = extract32(tcr, 0, 3);
1058 select = 0;
1059 hpd = extract64(tcr, 24, 1);
1060 epd = false;
1061 } else {
1062 int t0sz = extract32(tcr, 0, 3);
1063 int t1sz = extract32(tcr, 16, 3);
1064
1065 if (t1sz == 0) {
1066 select = va > (0xffffffffu >> t0sz);
1067 } else {
1068 /* Note that we will detect errors later. */
1069 select = va >= ~(0xffffffffu >> t1sz);
1070 }
1071 if (!select) {
1072 tsz = t0sz;
1073 epd = extract32(tcr, 7, 1);
1074 hpd = extract64(tcr, 41, 1);
1075 } else {
1076 tsz = t1sz;
1077 epd = extract32(tcr, 23, 1);
1078 hpd = extract64(tcr, 42, 1);
1079 }
1080 /* For aarch32, hpd0 is not enabled without t2e as well. */
1081 hpd &= extract32(tcr, 6, 1);
1082 }
1083
1084 return (ARMVAParameters) {
1085 .tsz = tsz,
1086 .select = select,
1087 .epd = epd,
1088 .hpd = hpd,
1089 };
1090}
1091
Richard Hendersonc5168782022-06-08 19:38:52 +01001092/*
1093 * check_s2_mmu_setup
1094 * @cpu: ARMCPU
1095 * @is_aa64: True if the translation regime is in AArch64 state
Richard Henderson0ffe5b72023-02-27 12:58:32 -10001096 * @tcr: VTCR_EL2 or VSTCR_EL2
1097 * @ds: Effective value of TCR.DS.
1098 * @iasize: Bitsize of IPAs
Richard Hendersonc5168782022-06-08 19:38:52 +01001099 * @stride: Page-table stride (See the ARM ARM)
1100 *
Richard Henderson0ffe5b72023-02-27 12:58:32 -10001101 * Decode the starting level of the S2 lookup, returning INT_MIN if
1102 * the configuration is invalid.
Richard Hendersonc5168782022-06-08 19:38:52 +01001103 */
Richard Henderson0ffe5b72023-02-27 12:58:32 -10001104static int check_s2_mmu_setup(ARMCPU *cpu, bool is_aa64, uint64_t tcr,
1105 bool ds, int iasize, int stride)
Richard Hendersonc5168782022-06-08 19:38:52 +01001106{
Richard Henderson0ffe5b72023-02-27 12:58:32 -10001107 int sl0, sl2, startlevel, granulebits, levels;
1108 int s1_min_iasize, s1_max_iasize;
Richard Hendersonc5168782022-06-08 19:38:52 +01001109
Richard Henderson0ffe5b72023-02-27 12:58:32 -10001110 sl0 = extract32(tcr, 6, 2);
Richard Hendersonc5168782022-06-08 19:38:52 +01001111 if (is_aa64) {
Richard Henderson0ffe5b72023-02-27 12:58:32 -10001112 /*
1113 * AArch64.S2InvalidTxSZ: While we checked tsz_oob near the top of
1114 * get_phys_addr_lpae, that used aa64_va_parameters which apply
1115 * to aarch64. If Stage1 is aarch32, the min_txsz is larger.
1116 * See AArch64.S2MinTxSZ, where min_tsz is 24, translated to
1117 * inputsize is 64 - 24 = 40.
1118 */
1119 if (iasize < 40 && !arm_el_is_aa64(&cpu->env, 1)) {
1120 goto fail;
1121 }
1122
1123 /*
1124 * AArch64.S2InvalidSL: Interpretation of SL depends on the page size,
1125 * so interleave AArch64.S2StartLevel.
1126 */
Richard Hendersonc5168782022-06-08 19:38:52 +01001127 switch (stride) {
Richard Henderson0ffe5b72023-02-27 12:58:32 -10001128 case 9: /* 4KB */
1129 /* SL2 is RES0 unless DS=1 & 4KB granule. */
1130 sl2 = extract64(tcr, 33, 1);
1131 if (ds && sl2) {
1132 if (sl0 != 0) {
1133 goto fail;
1134 }
1135 startlevel = -1;
1136 } else {
1137 startlevel = 2 - sl0;
1138 switch (sl0) {
1139 case 2:
1140 if (arm_pamax(cpu) < 44) {
1141 goto fail;
1142 }
1143 break;
1144 case 3:
1145 if (!cpu_isar_feature(aa64_st, cpu)) {
1146 goto fail;
1147 }
1148 startlevel = 3;
1149 break;
1150 }
Richard Hendersonc5168782022-06-08 19:38:52 +01001151 }
1152 break;
Richard Henderson0ffe5b72023-02-27 12:58:32 -10001153 case 11: /* 16KB */
1154 switch (sl0) {
1155 case 2:
1156 if (arm_pamax(cpu) < 42) {
1157 goto fail;
1158 }
1159 break;
1160 case 3:
1161 if (!ds) {
1162 goto fail;
1163 }
1164 break;
Richard Hendersonc5168782022-06-08 19:38:52 +01001165 }
Richard Henderson0ffe5b72023-02-27 12:58:32 -10001166 startlevel = 3 - sl0;
Richard Hendersonc5168782022-06-08 19:38:52 +01001167 break;
Richard Henderson0ffe5b72023-02-27 12:58:32 -10001168 case 13: /* 64KB */
1169 switch (sl0) {
1170 case 2:
1171 if (arm_pamax(cpu) < 44) {
1172 goto fail;
1173 }
1174 break;
1175 case 3:
1176 goto fail;
Richard Hendersonc5168782022-06-08 19:38:52 +01001177 }
Richard Henderson0ffe5b72023-02-27 12:58:32 -10001178 startlevel = 3 - sl0;
Richard Hendersonc5168782022-06-08 19:38:52 +01001179 break;
1180 default:
1181 g_assert_not_reached();
1182 }
Richard Hendersonc5168782022-06-08 19:38:52 +01001183 } else {
Richard Henderson0ffe5b72023-02-27 12:58:32 -10001184 /*
1185 * Things are simpler for AArch32 EL2, with only 4k pages.
1186 * There is no separate S2InvalidSL function, but AArch32.S2Walk
1187 * begins with walkparms.sl0 in {'1x'}.
1188 */
Richard Hendersonc5168782022-06-08 19:38:52 +01001189 assert(stride == 9);
Richard Henderson0ffe5b72023-02-27 12:58:32 -10001190 if (sl0 >= 2) {
1191 goto fail;
Richard Hendersonc5168782022-06-08 19:38:52 +01001192 }
Richard Henderson0ffe5b72023-02-27 12:58:32 -10001193 startlevel = 2 - sl0;
Richard Hendersonc5168782022-06-08 19:38:52 +01001194 }
Richard Henderson0ffe5b72023-02-27 12:58:32 -10001195
1196 /* AArch{64,32}.S2InconsistentSL are functionally equivalent. */
1197 levels = 3 - startlevel;
1198 granulebits = stride + 3;
1199
1200 s1_min_iasize = levels * stride + granulebits + 1;
1201 s1_max_iasize = s1_min_iasize + (stride - 1) + 4;
1202
1203 if (iasize >= s1_min_iasize && iasize <= s1_max_iasize) {
1204 return startlevel;
1205 }
1206
1207 fail:
1208 return INT_MIN;
Richard Hendersonc5168782022-06-08 19:38:52 +01001209}
1210
Richard Henderson32832222022-06-08 19:38:51 +01001211/**
1212 * get_phys_addr_lpae: perform one stage of page table walk, LPAE format
1213 *
1214 * Returns false if the translation was successful. Otherwise, phys_ptr,
1215 * attrs, prot and page_size may not be filled in, and the populated fsr
1216 * value provides information on why the translation aborted, in the format
1217 * of a long-format DFSR/IFSR fault register, with the following caveat:
1218 * the WnR bit is never set (the caller must do this).
1219 *
1220 * @env: CPUARMState
Richard Henderson6d2654f2022-10-10 20:18:54 -07001221 * @ptw: Current and next stage parameters for the walk.
Richard Henderson32832222022-06-08 19:38:51 +01001222 * @address: virtual address to get physical address for
1223 * @access_type: MMU_DATA_LOAD, MMU_DATA_STORE or MMU_INST_FETCH
Richard Henderson6d2654f2022-10-10 20:18:54 -07001224 * @s1_is_el0: if @ptw->in_mmu_idx is ARMMMUIdx_Stage2
1225 * (so this is a stage 2 page table walk),
1226 * must be true if this is stage 2 of a stage 1+2
Richard Henderson32832222022-06-08 19:38:51 +01001227 * walk for an EL0 access. If @mmu_idx is anything else,
1228 * @s1_is_el0 is ignored.
Richard Henderson03ee9bb2022-08-22 08:26:38 -07001229 * @result: set on translation success,
Richard Henderson32832222022-06-08 19:38:51 +01001230 * @fi: set to fault info if the translation fails
Richard Henderson32832222022-06-08 19:38:51 +01001231 */
Richard Henderson6d2654f2022-10-10 20:18:54 -07001232static bool get_phys_addr_lpae(CPUARMState *env, S1Translate *ptw,
1233 uint64_t address,
1234 MMUAccessType access_type, bool s1_is_el0,
Richard Hendersonc23f08a2022-10-01 09:22:38 -07001235 GetPhysAddrResult *result, ARMMMUFaultInfo *fi)
Richard Henderson32832222022-06-08 19:38:51 +01001236{
1237 ARMCPU *cpu = env_archcpu(env);
Richard Henderson6d2654f2022-10-10 20:18:54 -07001238 ARMMMUIdx mmu_idx = ptw->in_mmu_idx;
1239 bool is_secure = ptw->in_secure;
Ard Biesheuvel15f8f462022-11-22 16:55:15 +01001240 int32_t level;
Richard Henderson32832222022-06-08 19:38:51 +01001241 ARMVAParameters param;
1242 uint64_t ttbr;
1243 hwaddr descaddr, indexmask, indexmask_grainsize;
1244 uint32_t tableattrs;
1245 target_ulong page_size;
Richard Henderson45666092022-10-24 15:18:46 +10001246 uint64_t attrs;
Richard Henderson32832222022-06-08 19:38:51 +01001247 int32_t stride;
1248 int addrsize, inputsize, outputsize;
Peter Maydellc1547bb2022-07-14 14:22:59 +01001249 uint64_t tcr = regime_tcr(env, mmu_idx);
Richard Henderson32832222022-06-08 19:38:51 +01001250 int ap, ns, xn, pxn;
1251 uint32_t el = regime_el(env, mmu_idx);
1252 uint64_t descaddrmask;
1253 bool aarch64 = arm_el_is_aa64(env, el);
Richard Henderson71943a12022-10-24 15:18:49 +10001254 uint64_t descriptor, new_descriptor;
Richard Hendersonfe4ddc12022-10-24 15:18:44 +10001255 bool nstable;
Richard Henderson32832222022-06-08 19:38:51 +01001256
1257 /* TODO: This code does not support shareability levels. */
1258 if (aarch64) {
1259 int ps;
1260
1261 param = aa64_va_parameters(env, address, mmu_idx,
1262 access_type != MMU_INST_FETCH);
1263 level = 0;
1264
1265 /*
1266 * If TxSZ is programmed to a value larger than the maximum,
1267 * or smaller than the effective minimum, it is IMPLEMENTATION
1268 * DEFINED whether we behave as if the field were programmed
1269 * within bounds, or if a level 0 Translation fault is generated.
1270 *
1271 * With FEAT_LVA, fault on less than minimum becomes required,
1272 * so our choice is to always raise the fault.
1273 */
1274 if (param.tsz_oob) {
Richard Henderson27c1b812022-10-24 15:18:45 +10001275 goto do_translation_fault;
Richard Henderson32832222022-06-08 19:38:51 +01001276 }
1277
1278 addrsize = 64 - 8 * param.tbi;
1279 inputsize = 64 - param.tsz;
1280
1281 /*
1282 * Bound PS by PARANGE to find the effective output address size.
1283 * ID_AA64MMFR0 is a read-only register so values outside of the
1284 * supported mappings can be considered an implementation error.
1285 */
1286 ps = FIELD_EX64(cpu->isar.id_aa64mmfr0, ID_AA64MMFR0, PARANGE);
1287 ps = MIN(ps, param.ps);
1288 assert(ps < ARRAY_SIZE(pamax_map));
1289 outputsize = pamax_map[ps];
Ard Biesheuvel312b71a2022-11-21 11:45:13 +00001290
1291 /*
1292 * With LPA2, the effective output address (OA) size is at most 48 bits
1293 * unless TCR.DS == 1
1294 */
1295 if (!param.ds && param.gran != Gran64K) {
1296 outputsize = MIN(outputsize, 48);
1297 }
Richard Henderson32832222022-06-08 19:38:51 +01001298 } else {
1299 param = aa32_va_parameters(env, address, mmu_idx);
1300 level = 1;
1301 addrsize = (mmu_idx == ARMMMUIdx_Stage2 ? 40 : 32);
1302 inputsize = addrsize - param.tsz;
1303 outputsize = 40;
1304 }
1305
1306 /*
1307 * We determined the region when collecting the parameters, but we
1308 * have not yet validated that the address is valid for the region.
1309 * Extract the top bits and verify that they all match select.
1310 *
1311 * For aa32, if inputsize == addrsize, then we have selected the
1312 * region by exclusion in aa32_va_parameters and there is no more
1313 * validation to do here.
1314 */
1315 if (inputsize < addrsize) {
1316 target_ulong top_bits = sextract64(address, inputsize,
1317 addrsize - inputsize);
1318 if (-top_bits != param.select) {
1319 /* The gap between the two regions is a Translation fault */
Richard Henderson27c1b812022-10-24 15:18:45 +10001320 goto do_translation_fault;
Richard Henderson32832222022-06-08 19:38:51 +01001321 }
1322 }
1323
Peter Maydell3c003f72022-10-03 17:23:14 +01001324 stride = arm_granule_bits(param.gran) - 3;
Richard Henderson32832222022-06-08 19:38:51 +01001325
1326 /*
1327 * Note that QEMU ignores shareability and cacheability attributes,
1328 * so we don't need to do anything with the SH, ORGN, IRGN fields
1329 * in the TTBCR. Similarly, TTBCR:A1 selects whether we get the
1330 * ASID from TTBR0 or TTBR1, but QEMU's TLB doesn't currently
1331 * implement any ASID-like capability so we can ignore it (instead
1332 * we will always flush the TLB any time the ASID is changed).
1333 */
1334 ttbr = regime_ttbr(env, mmu_idx, param.select);
1335
1336 /*
1337 * Here we should have set up all the parameters for the translation:
1338 * inputsize, ttbr, epd, stride, tbi
1339 */
1340
1341 if (param.epd) {
1342 /*
1343 * Translation table walk disabled => Translation fault on TLB miss
1344 * Note: This is always 0 on 64-bit EL2 and EL3.
1345 */
Richard Henderson27c1b812022-10-24 15:18:45 +10001346 goto do_translation_fault;
Richard Henderson32832222022-06-08 19:38:51 +01001347 }
1348
Richard Hendersonedc05dd2022-10-24 15:18:38 +10001349 if (!regime_is_stage2(mmu_idx)) {
Richard Henderson32832222022-06-08 19:38:51 +01001350 /*
1351 * The starting level depends on the virtual address size (which can
1352 * be up to 48 bits) and the translation granule size. It indicates
1353 * the number of strides (stride bits at a time) needed to
1354 * consume the bits of the input address. In the pseudocode this is:
1355 * level = 4 - RoundUp((inputsize - grainsize) / stride)
1356 * where their 'inputsize' is our 'inputsize', 'grainsize' is
1357 * our 'stride + 3' and 'stride' is our 'stride'.
1358 * Applying the usual "rounded up m/n is (m+n-1)/n" and simplifying:
1359 * = 4 - (inputsize - stride - 3 + stride - 1) / stride
1360 * = 4 - (inputsize - 4) / stride;
1361 */
1362 level = 4 - (inputsize - 4) / stride;
1363 } else {
Richard Henderson0ffe5b72023-02-27 12:58:32 -10001364 int startlevel = check_s2_mmu_setup(cpu, aarch64, tcr, param.ds,
1365 inputsize, stride);
1366 if (startlevel == INT_MIN) {
1367 level = 0;
Richard Henderson27c1b812022-10-24 15:18:45 +10001368 goto do_translation_fault;
Richard Henderson32832222022-06-08 19:38:51 +01001369 }
1370 level = startlevel;
1371 }
1372
1373 indexmask_grainsize = MAKE_64BIT_MASK(0, stride + 3);
1374 indexmask = MAKE_64BIT_MASK(0, inputsize - (stride * (4 - level)));
1375
1376 /* Now we can extract the actual base address from the TTBR */
1377 descaddr = extract64(ttbr, 0, 48);
1378
1379 /*
1380 * For FEAT_LPA and PS=6, bits [51:48] of descaddr are in [5:2] of TTBR.
1381 *
1382 * Otherwise, if the base address is out of range, raise AddressSizeFault.
1383 * In the pseudocode, this is !IsZero(baseregister<47:outputsize>),
1384 * but we've just cleared the bits above 47, so simplify the test.
1385 */
1386 if (outputsize > 48) {
1387 descaddr |= extract64(ttbr, 2, 4) << 48;
1388 } else if (descaddr >> outputsize) {
1389 level = 0;
Richard Henderson27c1b812022-10-24 15:18:45 +10001390 fi->type = ARMFault_AddressSize;
Richard Henderson32832222022-06-08 19:38:51 +01001391 goto do_fault;
1392 }
1393
1394 /*
1395 * We rely on this masking to clear the RES0 bits at the bottom of the TTBR
1396 * and also to mask out CnP (bit 0) which could validly be non-zero.
1397 */
1398 descaddr &= ~indexmask;
1399
1400 /*
1401 * For AArch32, the address field in the descriptor goes up to bit 39
1402 * for both v7 and v8. However, for v8 the SBZ bits [47:40] must be 0
1403 * or an AddressSize fault is raised. So for v8 we extract those SBZ
1404 * bits as part of the address, which will be checked via outputsize.
1405 * For AArch64, the address field goes up to bit 47, or 49 with FEAT_LPA2;
1406 * the highest bits of a 52-bit output are placed elsewhere.
1407 */
1408 if (param.ds) {
1409 descaddrmask = MAKE_64BIT_MASK(0, 50);
1410 } else if (arm_feature(env, ARM_FEATURE_V8)) {
1411 descaddrmask = MAKE_64BIT_MASK(0, 48);
1412 } else {
1413 descaddrmask = MAKE_64BIT_MASK(0, 40);
1414 }
1415 descaddrmask &= ~indexmask_grainsize;
1416
1417 /*
1418 * Secure accesses start with the page table in secure memory and
1419 * can be downgraded to non-secure at any step. Non-secure accesses
1420 * remain non-secure. We implement this by just ORing in the NSTable/NS
1421 * bits at each step.
1422 */
Richard Hendersonc23f08a2022-10-01 09:22:38 -07001423 tableattrs = is_secure ? 0 : (1 << 4);
Richard Henderson32832222022-06-08 19:38:51 +01001424
Richard Hendersonfe4ddc12022-10-24 15:18:44 +10001425 next_level:
1426 descaddr |= (address >> (stride * (4 - level))) & indexmask;
1427 descaddr &= ~7ULL;
1428 nstable = extract32(tableattrs, 4, 1);
Richard Hendersoncead7fa2022-11-02 16:47:06 +11001429 if (nstable) {
Richard Henderson32832222022-06-08 19:38:51 +01001430 /*
Richard Hendersonfe4ddc12022-10-24 15:18:44 +10001431 * Stage2_S -> Stage2 or Phys_S -> Phys_NS
1432 * Assert that the non-secure idx are even, and relative order.
Richard Henderson32832222022-06-08 19:38:51 +01001433 */
Richard Hendersonfe4ddc12022-10-24 15:18:44 +10001434 QEMU_BUILD_BUG_ON((ARMMMUIdx_Phys_NS & 1) != 0);
1435 QEMU_BUILD_BUG_ON((ARMMMUIdx_Stage2 & 1) != 0);
1436 QEMU_BUILD_BUG_ON(ARMMMUIdx_Phys_NS + 1 != ARMMMUIdx_Phys_S);
1437 QEMU_BUILD_BUG_ON(ARMMMUIdx_Stage2 + 1 != ARMMMUIdx_Stage2_S);
1438 ptw->in_ptw_idx &= ~1;
1439 ptw->in_secure = false;
Richard Henderson32832222022-06-08 19:38:51 +01001440 }
Richard Hendersonfe4ddc12022-10-24 15:18:44 +10001441 if (!S1_ptw_translate(env, ptw, descaddr, fi)) {
1442 goto do_fault;
1443 }
1444 descriptor = arm_ldq_ptw(env, ptw, fi);
1445 if (fi->type != ARMFault_None) {
1446 goto do_fault;
1447 }
Richard Henderson71943a12022-10-24 15:18:49 +10001448 new_descriptor = descriptor;
Richard Hendersonfe4ddc12022-10-24 15:18:44 +10001449
Richard Henderson71943a12022-10-24 15:18:49 +10001450 restart_atomic_update:
Richard Hendersonfe4ddc12022-10-24 15:18:44 +10001451 if (!(descriptor & 1) || (!(descriptor & 2) && (level == 3))) {
1452 /* Invalid, or the Reserved level 3 encoding */
Richard Henderson27c1b812022-10-24 15:18:45 +10001453 goto do_translation_fault;
Richard Hendersonfe4ddc12022-10-24 15:18:44 +10001454 }
1455
1456 descaddr = descriptor & descaddrmask;
1457
1458 /*
1459 * For FEAT_LPA and PS=6, bits [51:48] of descaddr are in [15:12]
1460 * of descriptor. For FEAT_LPA2 and effective DS, bits [51:50] of
1461 * descaddr are in [9:8]. Otherwise, if descaddr is out of range,
1462 * raise AddressSizeFault.
1463 */
1464 if (outputsize > 48) {
1465 if (param.ds) {
1466 descaddr |= extract64(descriptor, 8, 2) << 50;
1467 } else {
1468 descaddr |= extract64(descriptor, 12, 4) << 48;
1469 }
1470 } else if (descaddr >> outputsize) {
Richard Henderson27c1b812022-10-24 15:18:45 +10001471 fi->type = ARMFault_AddressSize;
Richard Hendersonfe4ddc12022-10-24 15:18:44 +10001472 goto do_fault;
1473 }
1474
1475 if ((descriptor & 2) && (level < 3)) {
1476 /*
1477 * Table entry. The top five bits are attributes which may
1478 * propagate down through lower levels of the table (and
1479 * which are all arranged so that 0 means "no effect", so
1480 * we can gather them up by ORing in the bits at each level).
1481 */
1482 tableattrs |= extract64(descriptor, 59, 5);
1483 level++;
1484 indexmask = indexmask_grainsize;
1485 goto next_level;
1486 }
1487
1488 /*
1489 * Block entry at level 1 or 2, or page entry at level 3.
1490 * These are basically the same thing, although the number
1491 * of bits we pull in from the vaddr varies. Note that although
1492 * descaddrmask masks enough of the low bits of the descriptor
1493 * to give a correct page or table address, the address field
1494 * in a block descriptor is smaller; so we need to explicitly
1495 * clear the lower bits here before ORing in the low vaddr bits.
Richard Henderson71943a12022-10-24 15:18:49 +10001496 *
1497 * Afterward, descaddr is the final physical address.
Richard Hendersonfe4ddc12022-10-24 15:18:44 +10001498 */
1499 page_size = (1ULL << ((stride * (4 - level)) + 3));
1500 descaddr &= ~(hwaddr)(page_size - 1);
1501 descaddr |= (address & (page_size - 1));
Richard Hendersonfe4ddc12022-10-24 15:18:44 +10001502
Richard Henderson71943a12022-10-24 15:18:49 +10001503 if (likely(!ptw->in_debug)) {
1504 /*
1505 * Access flag.
1506 * If HA is enabled, prepare to update the descriptor below.
1507 * Otherwise, pass the access fault on to software.
1508 */
1509 if (!(descriptor & (1 << 10))) {
1510 if (param.ha) {
1511 new_descriptor |= 1 << 10; /* AF */
1512 } else {
1513 fi->type = ARMFault_AccessFlag;
1514 goto do_fault;
1515 }
1516 }
Richard Henderson65c123f2022-10-24 15:18:50 +10001517
1518 /*
1519 * Dirty Bit.
1520 * If HD is enabled, pre-emptively set/clear the appropriate AP/S2AP
1521 * bit for writeback. The actual write protection test may still be
1522 * overridden by tableattrs, to be merged below.
1523 */
1524 if (param.hd
1525 && extract64(descriptor, 51, 1) /* DBM */
1526 && access_type == MMU_DATA_STORE) {
1527 if (regime_is_stage2(mmu_idx)) {
1528 new_descriptor |= 1ull << 7; /* set S2AP[1] */
1529 } else {
1530 new_descriptor &= ~(1ull << 7); /* clear AP[2] */
1531 }
1532 }
Richard Henderson71943a12022-10-24 15:18:49 +10001533 }
1534
Richard Hendersonfe4ddc12022-10-24 15:18:44 +10001535 /*
Richard Henderson71943a12022-10-24 15:18:49 +10001536 * Extract attributes from the (modified) descriptor, and apply
1537 * table descriptors. Stage 2 table descriptors do not include
1538 * any attribute fields. HPD disables all the table attributes
1539 * except NSTable.
Richard Hendersonfe4ddc12022-10-24 15:18:44 +10001540 */
Richard Henderson71943a12022-10-24 15:18:49 +10001541 attrs = new_descriptor & (MAKE_64BIT_MASK(2, 10) | MAKE_64BIT_MASK(50, 14));
Richard Henderson34a57fa2022-10-24 15:18:48 +10001542 if (!regime_is_stage2(mmu_idx)) {
1543 attrs |= nstable << 5; /* NS */
1544 if (!param.hpd) {
1545 attrs |= extract64(tableattrs, 0, 2) << 53; /* XN, PXN */
1546 /*
1547 * The sense of AP[1] vs APTable[0] is reversed, as APTable[0] == 1
1548 * means "force PL1 access only", which means forcing AP[1] to 0.
1549 */
1550 attrs &= ~(extract64(tableattrs, 2, 1) << 6); /* !APT[0] => AP[1] */
1551 attrs |= extract32(tableattrs, 3, 1) << 7; /* APT[1] => AP[2] */
1552 }
1553 }
Richard Hendersonfe4ddc12022-10-24 15:18:44 +10001554
Richard Henderson45666092022-10-24 15:18:46 +10001555 ap = extract32(attrs, 6, 2);
Richard Hendersonedc05dd2022-10-24 15:18:38 +10001556 if (regime_is_stage2(mmu_idx)) {
Richard Henderson32832222022-06-08 19:38:51 +01001557 ns = mmu_idx == ARMMMUIdx_Stage2;
Richard Henderson45666092022-10-24 15:18:46 +10001558 xn = extract64(attrs, 53, 2);
Richard Henderson7fa7ea82022-10-01 09:22:56 -07001559 result->f.prot = get_S2prot(env, ap, xn, s1_is_el0);
Richard Henderson32832222022-06-08 19:38:51 +01001560 } else {
Richard Henderson45666092022-10-24 15:18:46 +10001561 ns = extract32(attrs, 5, 1);
1562 xn = extract64(attrs, 54, 1);
1563 pxn = extract64(attrs, 53, 1);
Richard Henderson7fa7ea82022-10-01 09:22:56 -07001564 result->f.prot = get_S1prot(env, mmu_idx, aarch64, ap, ns, xn, pxn);
Richard Henderson32832222022-06-08 19:38:51 +01001565 }
1566
Richard Henderson7fa7ea82022-10-01 09:22:56 -07001567 if (!(result->f.prot & (1 << access_type))) {
Richard Henderson27c1b812022-10-24 15:18:45 +10001568 fi->type = ARMFault_Permission;
Richard Henderson32832222022-06-08 19:38:51 +01001569 goto do_fault;
1570 }
1571
Richard Henderson71943a12022-10-24 15:18:49 +10001572 /* If FEAT_HAFDBS has made changes, update the PTE. */
1573 if (new_descriptor != descriptor) {
1574 new_descriptor = arm_casq_ptw(env, descriptor, new_descriptor, ptw, fi);
1575 if (fi->type != ARMFault_None) {
1576 goto do_fault;
1577 }
1578 /*
1579 * I_YZSVV says that if the in-memory descriptor has changed,
1580 * then we must use the information in that new value
1581 * (which might include a different output address, different
1582 * attributes, or generate a fault).
1583 * Restart the handling of the descriptor value from scratch.
1584 */
1585 if (new_descriptor != descriptor) {
1586 descriptor = new_descriptor;
1587 goto restart_atomic_update;
1588 }
1589 }
1590
Richard Henderson32832222022-06-08 19:38:51 +01001591 if (ns) {
1592 /*
1593 * The NS bit will (as required by the architecture) have no effect if
1594 * the CPU doesn't support TZ or this is a non-secure translation
1595 * regime, because the attribute will already be non-secure.
1596 */
Richard Henderson7fa7ea82022-10-01 09:22:56 -07001597 result->f.attrs.secure = false;
Richard Henderson32832222022-06-08 19:38:51 +01001598 }
Richard Henderson937f2242022-10-10 20:18:50 -07001599
Richard Hendersonedc05dd2022-10-24 15:18:38 +10001600 if (regime_is_stage2(mmu_idx)) {
Richard Henderson03ee9bb2022-08-22 08:26:38 -07001601 result->cacheattrs.is_s2_format = true;
Richard Henderson45666092022-10-24 15:18:46 +10001602 result->cacheattrs.attrs = extract32(attrs, 2, 4);
Richard Henderson32832222022-06-08 19:38:51 +01001603 } else {
1604 /* Index into MAIR registers for cache attributes */
Richard Henderson45666092022-10-24 15:18:46 +10001605 uint8_t attrindx = extract32(attrs, 2, 3);
Richard Henderson32832222022-06-08 19:38:51 +01001606 uint64_t mair = env->cp15.mair_el[regime_el(env, mmu_idx)];
1607 assert(attrindx <= 7);
Richard Henderson03ee9bb2022-08-22 08:26:38 -07001608 result->cacheattrs.is_s2_format = false;
1609 result->cacheattrs.attrs = extract64(mair, attrindx * 8, 8);
Richard Henderson6a3b1e42023-04-07 11:51:48 -07001610
1611 /* When in aarch64 mode, and BTI is enabled, remember GP in the TLB. */
1612 if (aarch64 && cpu_isar_feature(aa64_bti, cpu)) {
1613 result->f.guarded = extract64(attrs, 50, 1); /* GP */
1614 }
Richard Henderson32832222022-06-08 19:38:51 +01001615 }
1616
1617 /*
1618 * For FEAT_LPA2 and effective DS, the SH field in the attributes
1619 * was re-purposed for output address bits. The SH attribute in
1620 * that case comes from TCR_ELx, which we extracted earlier.
1621 */
1622 if (param.ds) {
Richard Henderson03ee9bb2022-08-22 08:26:38 -07001623 result->cacheattrs.shareability = param.sh;
Richard Henderson32832222022-06-08 19:38:51 +01001624 } else {
Richard Henderson45666092022-10-24 15:18:46 +10001625 result->cacheattrs.shareability = extract32(attrs, 8, 2);
Richard Henderson32832222022-06-08 19:38:51 +01001626 }
1627
Richard Henderson7fa7ea82022-10-01 09:22:56 -07001628 result->f.phys_addr = descaddr;
1629 result->f.lg_page_size = ctz64(page_size);
Richard Henderson32832222022-06-08 19:38:51 +01001630 return false;
1631
Richard Henderson27c1b812022-10-24 15:18:45 +10001632 do_translation_fault:
1633 fi->type = ARMFault_Translation;
1634 do_fault:
Richard Henderson32832222022-06-08 19:38:51 +01001635 fi->level = level;
1636 /* Tag the error as S2 for failed S1 PTW at S2 or ordinary S2. */
Richard Hendersonedc05dd2022-10-24 15:18:38 +10001637 fi->stage2 = fi->s1ptw || regime_is_stage2(mmu_idx);
Richard Henderson32832222022-06-08 19:38:51 +01001638 fi->s1ns = mmu_idx == ARMMMUIdx_Stage2;
1639 return true;
1640}
1641
Richard Henderson9a12fb32022-06-08 19:38:49 +01001642static bool get_phys_addr_pmsav5(CPUARMState *env, uint32_t address,
1643 MMUAccessType access_type, ARMMMUIdx mmu_idx,
Richard Hendersona5b50922022-08-22 08:26:55 -07001644 bool is_secure, GetPhysAddrResult *result,
Richard Henderson9a12fb32022-06-08 19:38:49 +01001645 ARMMMUFaultInfo *fi)
1646{
1647 int n;
1648 uint32_t mask;
1649 uint32_t base;
1650 bool is_user = regime_is_user(env, mmu_idx);
1651
Richard Henderson7e80c0a2022-10-01 09:22:40 -07001652 if (regime_translation_disabled(env, mmu_idx, is_secure)) {
Richard Henderson9a12fb32022-06-08 19:38:49 +01001653 /* MPU disabled. */
Richard Henderson7fa7ea82022-10-01 09:22:56 -07001654 result->f.phys_addr = address;
1655 result->f.prot = PAGE_READ | PAGE_WRITE | PAGE_EXEC;
Richard Henderson9a12fb32022-06-08 19:38:49 +01001656 return false;
1657 }
1658
Richard Henderson7fa7ea82022-10-01 09:22:56 -07001659 result->f.phys_addr = address;
Richard Henderson9a12fb32022-06-08 19:38:49 +01001660 for (n = 7; n >= 0; n--) {
1661 base = env->cp15.c6_region[n];
1662 if ((base & 1) == 0) {
1663 continue;
1664 }
1665 mask = 1 << ((base >> 1) & 0x1f);
1666 /* Keep this shift separate from the above to avoid an
1667 (undefined) << 32. */
1668 mask = (mask << 1) - 1;
1669 if (((base ^ address) & ~mask) == 0) {
1670 break;
1671 }
1672 }
1673 if (n < 0) {
1674 fi->type = ARMFault_Background;
1675 return true;
1676 }
1677
1678 if (access_type == MMU_INST_FETCH) {
1679 mask = env->cp15.pmsav5_insn_ap;
1680 } else {
1681 mask = env->cp15.pmsav5_data_ap;
1682 }
1683 mask = (mask >> (n * 4)) & 0xf;
1684 switch (mask) {
1685 case 0:
1686 fi->type = ARMFault_Permission;
1687 fi->level = 1;
1688 return true;
1689 case 1:
1690 if (is_user) {
1691 fi->type = ARMFault_Permission;
1692 fi->level = 1;
1693 return true;
1694 }
Richard Henderson7fa7ea82022-10-01 09:22:56 -07001695 result->f.prot = PAGE_READ | PAGE_WRITE;
Richard Henderson9a12fb32022-06-08 19:38:49 +01001696 break;
1697 case 2:
Richard Henderson7fa7ea82022-10-01 09:22:56 -07001698 result->f.prot = PAGE_READ;
Richard Henderson9a12fb32022-06-08 19:38:49 +01001699 if (!is_user) {
Richard Henderson7fa7ea82022-10-01 09:22:56 -07001700 result->f.prot |= PAGE_WRITE;
Richard Henderson9a12fb32022-06-08 19:38:49 +01001701 }
1702 break;
1703 case 3:
Richard Henderson7fa7ea82022-10-01 09:22:56 -07001704 result->f.prot = PAGE_READ | PAGE_WRITE;
Richard Henderson9a12fb32022-06-08 19:38:49 +01001705 break;
1706 case 5:
1707 if (is_user) {
1708 fi->type = ARMFault_Permission;
1709 fi->level = 1;
1710 return true;
1711 }
Richard Henderson7fa7ea82022-10-01 09:22:56 -07001712 result->f.prot = PAGE_READ;
Richard Henderson9a12fb32022-06-08 19:38:49 +01001713 break;
1714 case 6:
Richard Henderson7fa7ea82022-10-01 09:22:56 -07001715 result->f.prot = PAGE_READ;
Richard Henderson9a12fb32022-06-08 19:38:49 +01001716 break;
1717 default:
1718 /* Bad permission. */
1719 fi->type = ARMFault_Permission;
1720 fi->level = 1;
1721 return true;
1722 }
Richard Henderson7fa7ea82022-10-01 09:22:56 -07001723 result->f.prot |= PAGE_EXEC;
Richard Henderson9a12fb32022-06-08 19:38:49 +01001724 return false;
1725}
1726
Richard Hendersonfedbaa02022-06-08 19:38:50 +01001727static void get_phys_addr_pmsav7_default(CPUARMState *env, ARMMMUIdx mmu_idx,
Richard Henderson7fa7ea82022-10-01 09:22:56 -07001728 int32_t address, uint8_t *prot)
Richard Henderson7d2e08c2022-06-08 19:38:49 +01001729{
1730 if (!arm_feature(env, ARM_FEATURE_M)) {
1731 *prot = PAGE_READ | PAGE_WRITE;
1732 switch (address) {
1733 case 0xF0000000 ... 0xFFFFFFFF:
1734 if (regime_sctlr(env, mmu_idx) & SCTLR_V) {
1735 /* hivecs execing is ok */
1736 *prot |= PAGE_EXEC;
1737 }
1738 break;
1739 case 0x00000000 ... 0x7FFFFFFF:
1740 *prot |= PAGE_EXEC;
1741 break;
1742 }
1743 } else {
1744 /* Default system address map for M profile cores.
1745 * The architecture specifies which regions are execute-never;
1746 * at the MPU level no other checks are defined.
1747 */
1748 switch (address) {
1749 case 0x00000000 ... 0x1fffffff: /* ROM */
1750 case 0x20000000 ... 0x3fffffff: /* SRAM */
1751 case 0x60000000 ... 0x7fffffff: /* RAM */
1752 case 0x80000000 ... 0x9fffffff: /* RAM */
1753 *prot = PAGE_READ | PAGE_WRITE | PAGE_EXEC;
1754 break;
1755 case 0x40000000 ... 0x5fffffff: /* Peripheral */
1756 case 0xa0000000 ... 0xbfffffff: /* Device */
1757 case 0xc0000000 ... 0xdfffffff: /* Device */
1758 case 0xe0000000 ... 0xffffffff: /* System */
1759 *prot = PAGE_READ | PAGE_WRITE;
1760 break;
1761 default:
1762 g_assert_not_reached();
1763 }
1764 }
1765}
1766
Richard Henderson47ff5ba2022-06-08 19:38:50 +01001767static bool m_is_ppb_region(CPUARMState *env, uint32_t address)
1768{
1769 /* True if address is in the M profile PPB region 0xe0000000 - 0xe00fffff */
1770 return arm_feature(env, ARM_FEATURE_M) &&
1771 extract32(address, 20, 12) == 0xe00;
1772}
1773
1774static bool m_is_system_region(CPUARMState *env, uint32_t address)
1775{
1776 /*
1777 * True if address is in the M profile system region
1778 * 0xe0000000 - 0xffffffff
1779 */
1780 return arm_feature(env, ARM_FEATURE_M) && extract32(address, 29, 3) == 0x7;
1781}
1782
Richard Hendersonc8e436c2022-06-08 19:38:50 +01001783static bool pmsav7_use_background_region(ARMCPU *cpu, ARMMMUIdx mmu_idx,
Richard Henderson1a469cf2022-08-22 08:26:51 -07001784 bool is_secure, bool is_user)
Richard Hendersonc8e436c2022-06-08 19:38:50 +01001785{
1786 /*
1787 * Return true if we should use the default memory map as a
1788 * "background" region if there are no hits against any MPU regions.
1789 */
1790 CPUARMState *env = &cpu->env;
1791
1792 if (is_user) {
1793 return false;
1794 }
1795
1796 if (arm_feature(env, ARM_FEATURE_M)) {
Richard Henderson1a469cf2022-08-22 08:26:51 -07001797 return env->v7m.mpu_ctrl[is_secure] & R_V7M_MPU_CTRL_PRIVDEFENA_MASK;
Richard Hendersonc8e436c2022-06-08 19:38:50 +01001798 }
Tobias Röhmelfca45e32022-12-06 11:25:03 +01001799
1800 if (mmu_idx == ARMMMUIdx_Stage2) {
1801 return false;
1802 }
1803
1804 return regime_sctlr(env, mmu_idx) & SCTLR_BR;
Richard Hendersonc8e436c2022-06-08 19:38:50 +01001805}
1806
Richard Henderson1f2e87e2022-06-08 19:38:49 +01001807static bool get_phys_addr_pmsav7(CPUARMState *env, uint32_t address,
1808 MMUAccessType access_type, ARMMMUIdx mmu_idx,
Richard Henderson957a0bb2022-08-22 08:26:53 -07001809 bool secure, GetPhysAddrResult *result,
Richard Henderson1f2e87e2022-06-08 19:38:49 +01001810 ARMMMUFaultInfo *fi)
1811{
1812 ARMCPU *cpu = env_archcpu(env);
1813 int n;
1814 bool is_user = regime_is_user(env, mmu_idx);
1815
Richard Henderson7fa7ea82022-10-01 09:22:56 -07001816 result->f.phys_addr = address;
1817 result->f.lg_page_size = TARGET_PAGE_BITS;
1818 result->f.prot = 0;
Richard Henderson1f2e87e2022-06-08 19:38:49 +01001819
Richard Henderson7e80c0a2022-10-01 09:22:40 -07001820 if (regime_translation_disabled(env, mmu_idx, secure) ||
Richard Henderson1f2e87e2022-06-08 19:38:49 +01001821 m_is_ppb_region(env, address)) {
1822 /*
1823 * MPU disabled or M profile PPB access: use default memory map.
1824 * The other case which uses the default memory map in the
1825 * v7M ARM ARM pseudocode is exception vector reads from the vector
1826 * table. In QEMU those accesses are done in arm_v7m_load_vector(),
1827 * which always does a direct read using address_space_ldl(), rather
1828 * than going via this function, so we don't need to check that here.
1829 */
Richard Henderson7fa7ea82022-10-01 09:22:56 -07001830 get_phys_addr_pmsav7_default(env, mmu_idx, address, &result->f.prot);
Richard Henderson1f2e87e2022-06-08 19:38:49 +01001831 } else { /* MPU enabled */
1832 for (n = (int)cpu->pmsav7_dregion - 1; n >= 0; n--) {
1833 /* region search */
1834 uint32_t base = env->pmsav7.drbar[n];
1835 uint32_t rsize = extract32(env->pmsav7.drsr[n], 1, 5);
1836 uint32_t rmask;
1837 bool srdis = false;
1838
1839 if (!(env->pmsav7.drsr[n] & 0x1)) {
1840 continue;
1841 }
1842
1843 if (!rsize) {
1844 qemu_log_mask(LOG_GUEST_ERROR,
1845 "DRSR[%d]: Rsize field cannot be 0\n", n);
1846 continue;
1847 }
1848 rsize++;
1849 rmask = (1ull << rsize) - 1;
1850
1851 if (base & rmask) {
1852 qemu_log_mask(LOG_GUEST_ERROR,
1853 "DRBAR[%d]: 0x%" PRIx32 " misaligned "
1854 "to DRSR region size, mask = 0x%" PRIx32 "\n",
1855 n, base, rmask);
1856 continue;
1857 }
1858
1859 if (address < base || address > base + rmask) {
1860 /*
1861 * Address not in this region. We must check whether the
1862 * region covers addresses in the same page as our address.
1863 * In that case we must not report a size that covers the
1864 * whole page for a subsequent hit against a different MPU
1865 * region or the background region, because it would result in
1866 * incorrect TLB hits for subsequent accesses to addresses that
1867 * are in this MPU region.
1868 */
1869 if (ranges_overlap(base, rmask,
1870 address & TARGET_PAGE_MASK,
1871 TARGET_PAGE_SIZE)) {
Richard Henderson7fa7ea82022-10-01 09:22:56 -07001872 result->f.lg_page_size = 0;
Richard Henderson1f2e87e2022-06-08 19:38:49 +01001873 }
1874 continue;
1875 }
1876
1877 /* Region matched */
1878
1879 if (rsize >= 8) { /* no subregions for regions < 256 bytes */
1880 int i, snd;
1881 uint32_t srdis_mask;
1882
1883 rsize -= 3; /* sub region size (power of 2) */
1884 snd = ((address - base) >> rsize) & 0x7;
1885 srdis = extract32(env->pmsav7.drsr[n], snd + 8, 1);
1886
1887 srdis_mask = srdis ? 0x3 : 0x0;
1888 for (i = 2; i <= 8 && rsize < TARGET_PAGE_BITS; i *= 2) {
1889 /*
1890 * This will check in groups of 2, 4 and then 8, whether
1891 * the subregion bits are consistent. rsize is incremented
1892 * back up to give the region size, considering consistent
1893 * adjacent subregions as one region. Stop testing if rsize
1894 * is already big enough for an entire QEMU page.
1895 */
1896 int snd_rounded = snd & ~(i - 1);
1897 uint32_t srdis_multi = extract32(env->pmsav7.drsr[n],
1898 snd_rounded + 8, i);
1899 if (srdis_mask ^ srdis_multi) {
1900 break;
1901 }
1902 srdis_mask = (srdis_mask << i) | srdis_mask;
1903 rsize++;
1904 }
1905 }
1906 if (srdis) {
1907 continue;
1908 }
1909 if (rsize < TARGET_PAGE_BITS) {
Richard Henderson7fa7ea82022-10-01 09:22:56 -07001910 result->f.lg_page_size = rsize;
Richard Henderson1f2e87e2022-06-08 19:38:49 +01001911 }
1912 break;
1913 }
1914
1915 if (n == -1) { /* no hits */
Richard Henderson1a469cf2022-08-22 08:26:51 -07001916 if (!pmsav7_use_background_region(cpu, mmu_idx, secure, is_user)) {
Richard Henderson1f2e87e2022-06-08 19:38:49 +01001917 /* background fault */
1918 fi->type = ARMFault_Background;
1919 return true;
1920 }
Richard Henderson7fa7ea82022-10-01 09:22:56 -07001921 get_phys_addr_pmsav7_default(env, mmu_idx, address,
1922 &result->f.prot);
Richard Henderson1f2e87e2022-06-08 19:38:49 +01001923 } else { /* a MPU hit! */
1924 uint32_t ap = extract32(env->pmsav7.dracr[n], 8, 3);
1925 uint32_t xn = extract32(env->pmsav7.dracr[n], 12, 1);
1926
1927 if (m_is_system_region(env, address)) {
1928 /* System space is always execute never */
1929 xn = 1;
1930 }
1931
1932 if (is_user) { /* User mode AP bit decoding */
1933 switch (ap) {
1934 case 0:
1935 case 1:
1936 case 5:
1937 break; /* no access */
1938 case 3:
Richard Henderson7fa7ea82022-10-01 09:22:56 -07001939 result->f.prot |= PAGE_WRITE;
Richard Henderson1f2e87e2022-06-08 19:38:49 +01001940 /* fall through */
1941 case 2:
1942 case 6:
Richard Henderson7fa7ea82022-10-01 09:22:56 -07001943 result->f.prot |= PAGE_READ | PAGE_EXEC;
Richard Henderson1f2e87e2022-06-08 19:38:49 +01001944 break;
1945 case 7:
1946 /* for v7M, same as 6; for R profile a reserved value */
1947 if (arm_feature(env, ARM_FEATURE_M)) {
Richard Henderson7fa7ea82022-10-01 09:22:56 -07001948 result->f.prot |= PAGE_READ | PAGE_EXEC;
Richard Henderson1f2e87e2022-06-08 19:38:49 +01001949 break;
1950 }
1951 /* fall through */
1952 default:
1953 qemu_log_mask(LOG_GUEST_ERROR,
1954 "DRACR[%d]: Bad value for AP bits: 0x%"
1955 PRIx32 "\n", n, ap);
1956 }
1957 } else { /* Priv. mode AP bits decoding */
1958 switch (ap) {
1959 case 0:
1960 break; /* no access */
1961 case 1:
1962 case 2:
1963 case 3:
Richard Henderson7fa7ea82022-10-01 09:22:56 -07001964 result->f.prot |= PAGE_WRITE;
Richard Henderson1f2e87e2022-06-08 19:38:49 +01001965 /* fall through */
1966 case 5:
1967 case 6:
Richard Henderson7fa7ea82022-10-01 09:22:56 -07001968 result->f.prot |= PAGE_READ | PAGE_EXEC;
Richard Henderson1f2e87e2022-06-08 19:38:49 +01001969 break;
1970 case 7:
1971 /* for v7M, same as 6; for R profile a reserved value */
1972 if (arm_feature(env, ARM_FEATURE_M)) {
Richard Henderson7fa7ea82022-10-01 09:22:56 -07001973 result->f.prot |= PAGE_READ | PAGE_EXEC;
Richard Henderson1f2e87e2022-06-08 19:38:49 +01001974 break;
1975 }
1976 /* fall through */
1977 default:
1978 qemu_log_mask(LOG_GUEST_ERROR,
1979 "DRACR[%d]: Bad value for AP bits: 0x%"
1980 PRIx32 "\n", n, ap);
1981 }
1982 }
1983
1984 /* execute never */
1985 if (xn) {
Richard Henderson7fa7ea82022-10-01 09:22:56 -07001986 result->f.prot &= ~PAGE_EXEC;
Richard Henderson1f2e87e2022-06-08 19:38:49 +01001987 }
1988 }
1989 }
1990
1991 fi->type = ARMFault_Permission;
1992 fi->level = 1;
Richard Henderson7fa7ea82022-10-01 09:22:56 -07001993 return !(result->f.prot & (1 << access_type));
Richard Henderson1f2e87e2022-06-08 19:38:49 +01001994}
1995
Tobias Röhmelfca45e32022-12-06 11:25:03 +01001996static uint32_t *regime_rbar(CPUARMState *env, ARMMMUIdx mmu_idx,
1997 uint32_t secure)
1998{
1999 if (regime_el(env, mmu_idx) == 2) {
2000 return env->pmsav8.hprbar;
2001 } else {
2002 return env->pmsav8.rbar[secure];
2003 }
2004}
2005
2006static uint32_t *regime_rlar(CPUARMState *env, ARMMMUIdx mmu_idx,
2007 uint32_t secure)
2008{
2009 if (regime_el(env, mmu_idx) == 2) {
2010 return env->pmsav8.hprlar;
2011 } else {
2012 return env->pmsav8.rlar[secure];
2013 }
2014}
2015
Richard Hendersonfedbaa02022-06-08 19:38:50 +01002016bool pmsav8_mpu_lookup(CPUARMState *env, uint32_t address,
2017 MMUAccessType access_type, ARMMMUIdx mmu_idx,
Richard Hendersone9fb7092022-08-22 08:26:47 -07002018 bool secure, GetPhysAddrResult *result,
2019 ARMMMUFaultInfo *fi, uint32_t *mregion)
Richard Hendersonfedbaa02022-06-08 19:38:50 +01002020{
2021 /*
2022 * Perform a PMSAv8 MPU lookup (without also doing the SAU check
2023 * that a full phys-to-virt translation does).
2024 * mregion is (if not NULL) set to the region number which matched,
2025 * or -1 if no region number is returned (MPU off, address did not
2026 * hit a region, address hit in multiple regions).
Richard Henderson652c7502022-08-22 08:26:45 -07002027 * If the region hit doesn't cover the entire TARGET_PAGE the address
2028 * is within, then we set the result page_size to 1 to force the
2029 * memory system to use a subpage.
Richard Hendersonfedbaa02022-06-08 19:38:50 +01002030 */
2031 ARMCPU *cpu = env_archcpu(env);
2032 bool is_user = regime_is_user(env, mmu_idx);
Richard Hendersonfedbaa02022-06-08 19:38:50 +01002033 int n;
2034 int matchregion = -1;
2035 bool hit = false;
2036 uint32_t addr_page_base = address & TARGET_PAGE_MASK;
2037 uint32_t addr_page_limit = addr_page_base + (TARGET_PAGE_SIZE - 1);
Tobias Röhmelfca45e32022-12-06 11:25:03 +01002038 int region_counter;
2039
2040 if (regime_el(env, mmu_idx) == 2) {
2041 region_counter = cpu->pmsav8r_hdregion;
2042 } else {
2043 region_counter = cpu->pmsav7_dregion;
2044 }
Richard Hendersonfedbaa02022-06-08 19:38:50 +01002045
Richard Henderson7fa7ea82022-10-01 09:22:56 -07002046 result->f.lg_page_size = TARGET_PAGE_BITS;
2047 result->f.phys_addr = address;
2048 result->f.prot = 0;
Richard Hendersonfedbaa02022-06-08 19:38:50 +01002049 if (mregion) {
2050 *mregion = -1;
2051 }
2052
Tobias Röhmelfca45e32022-12-06 11:25:03 +01002053 if (mmu_idx == ARMMMUIdx_Stage2) {
2054 fi->stage2 = true;
2055 }
2056
Richard Hendersonfedbaa02022-06-08 19:38:50 +01002057 /*
2058 * Unlike the ARM ARM pseudocode, we don't need to check whether this
2059 * was an exception vector read from the vector table (which is always
2060 * done using the default system address map), because those accesses
2061 * are done in arm_v7m_load_vector(), which always does a direct
2062 * read using address_space_ldl(), rather than going via this function.
2063 */
Richard Henderson7e80c0a2022-10-01 09:22:40 -07002064 if (regime_translation_disabled(env, mmu_idx, secure)) { /* MPU disabled */
Richard Hendersonfedbaa02022-06-08 19:38:50 +01002065 hit = true;
2066 } else if (m_is_ppb_region(env, address)) {
2067 hit = true;
2068 } else {
Richard Henderson1a469cf2022-08-22 08:26:51 -07002069 if (pmsav7_use_background_region(cpu, mmu_idx, secure, is_user)) {
Richard Hendersonfedbaa02022-06-08 19:38:50 +01002070 hit = true;
2071 }
2072
Tobias Röhmelfca45e32022-12-06 11:25:03 +01002073 uint32_t bitmask;
2074 if (arm_feature(env, ARM_FEATURE_M)) {
2075 bitmask = 0x1f;
2076 } else {
2077 bitmask = 0x3f;
2078 fi->level = 0;
2079 }
2080
2081 for (n = region_counter - 1; n >= 0; n--) {
Richard Hendersonfedbaa02022-06-08 19:38:50 +01002082 /* region search */
2083 /*
Tobias Röhmelfca45e32022-12-06 11:25:03 +01002084 * Note that the base address is bits [31:x] from the register
2085 * with bits [x-1:0] all zeroes, but the limit address is bits
2086 * [31:x] from the register with bits [x:0] all ones. Where x is
2087 * 5 for Cortex-M and 6 for Cortex-R
Richard Hendersonfedbaa02022-06-08 19:38:50 +01002088 */
Tobias Röhmelfca45e32022-12-06 11:25:03 +01002089 uint32_t base = regime_rbar(env, mmu_idx, secure)[n] & ~bitmask;
2090 uint32_t limit = regime_rlar(env, mmu_idx, secure)[n] | bitmask;
Richard Hendersonfedbaa02022-06-08 19:38:50 +01002091
Tobias Röhmelfca45e32022-12-06 11:25:03 +01002092 if (!(regime_rlar(env, mmu_idx, secure)[n] & 0x1)) {
Richard Hendersonfedbaa02022-06-08 19:38:50 +01002093 /* Region disabled */
2094 continue;
2095 }
2096
2097 if (address < base || address > limit) {
2098 /*
2099 * Address not in this region. We must check whether the
2100 * region covers addresses in the same page as our address.
2101 * In that case we must not report a size that covers the
2102 * whole page for a subsequent hit against a different MPU
2103 * region or the background region, because it would result in
2104 * incorrect TLB hits for subsequent accesses to addresses that
2105 * are in this MPU region.
2106 */
2107 if (limit >= base &&
2108 ranges_overlap(base, limit - base + 1,
2109 addr_page_base,
2110 TARGET_PAGE_SIZE)) {
Richard Henderson7fa7ea82022-10-01 09:22:56 -07002111 result->f.lg_page_size = 0;
Richard Hendersonfedbaa02022-06-08 19:38:50 +01002112 }
2113 continue;
2114 }
2115
2116 if (base > addr_page_base || limit < addr_page_limit) {
Richard Henderson7fa7ea82022-10-01 09:22:56 -07002117 result->f.lg_page_size = 0;
Richard Hendersonfedbaa02022-06-08 19:38:50 +01002118 }
2119
2120 if (matchregion != -1) {
2121 /*
2122 * Multiple regions match -- always a failure (unlike
2123 * PMSAv7 where highest-numbered-region wins)
2124 */
2125 fi->type = ARMFault_Permission;
Tobias Röhmelfca45e32022-12-06 11:25:03 +01002126 if (arm_feature(env, ARM_FEATURE_M)) {
2127 fi->level = 1;
2128 }
Richard Hendersonfedbaa02022-06-08 19:38:50 +01002129 return true;
2130 }
2131
2132 matchregion = n;
2133 hit = true;
2134 }
2135 }
2136
2137 if (!hit) {
Tobias Röhmelfca45e32022-12-06 11:25:03 +01002138 if (arm_feature(env, ARM_FEATURE_M)) {
2139 fi->type = ARMFault_Background;
2140 } else {
2141 fi->type = ARMFault_Permission;
2142 }
Richard Hendersonfedbaa02022-06-08 19:38:50 +01002143 return true;
2144 }
2145
2146 if (matchregion == -1) {
2147 /* hit using the background region */
Richard Henderson7fa7ea82022-10-01 09:22:56 -07002148 get_phys_addr_pmsav7_default(env, mmu_idx, address, &result->f.prot);
Richard Hendersonfedbaa02022-06-08 19:38:50 +01002149 } else {
Tobias Röhmelfca45e32022-12-06 11:25:03 +01002150 uint32_t matched_rbar = regime_rbar(env, mmu_idx, secure)[matchregion];
2151 uint32_t matched_rlar = regime_rlar(env, mmu_idx, secure)[matchregion];
2152 uint32_t ap = extract32(matched_rbar, 1, 2);
2153 uint32_t xn = extract32(matched_rbar, 0, 1);
Richard Hendersonfedbaa02022-06-08 19:38:50 +01002154 bool pxn = false;
2155
2156 if (arm_feature(env, ARM_FEATURE_V8_1M)) {
Tobias Röhmelfca45e32022-12-06 11:25:03 +01002157 pxn = extract32(matched_rlar, 4, 1);
Richard Hendersonfedbaa02022-06-08 19:38:50 +01002158 }
2159
2160 if (m_is_system_region(env, address)) {
2161 /* System space is always execute never */
2162 xn = 1;
2163 }
2164
Tobias Röhmelfca45e32022-12-06 11:25:03 +01002165 if (regime_el(env, mmu_idx) == 2) {
2166 result->f.prot = simple_ap_to_rw_prot_is_user(ap,
2167 mmu_idx != ARMMMUIdx_E2);
2168 } else {
2169 result->f.prot = simple_ap_to_rw_prot(env, mmu_idx, ap);
2170 }
2171
2172 if (!arm_feature(env, ARM_FEATURE_M)) {
2173 uint8_t attrindx = extract32(matched_rlar, 1, 3);
2174 uint64_t mair = env->cp15.mair_el[regime_el(env, mmu_idx)];
2175 uint8_t sh = extract32(matched_rlar, 3, 2);
2176
2177 if (regime_sctlr(env, mmu_idx) & SCTLR_WXN &&
2178 result->f.prot & PAGE_WRITE && mmu_idx != ARMMMUIdx_Stage2) {
2179 xn = 0x1;
2180 }
2181
2182 if ((regime_el(env, mmu_idx) == 1) &&
2183 regime_sctlr(env, mmu_idx) & SCTLR_UWXN && ap == 0x1) {
2184 pxn = 0x1;
2185 }
2186
2187 result->cacheattrs.is_s2_format = false;
2188 result->cacheattrs.attrs = extract64(mair, attrindx * 8, 8);
2189 result->cacheattrs.shareability = sh;
2190 }
2191
Richard Henderson7fa7ea82022-10-01 09:22:56 -07002192 if (result->f.prot && !xn && !(pxn && !is_user)) {
2193 result->f.prot |= PAGE_EXEC;
Richard Hendersonfedbaa02022-06-08 19:38:50 +01002194 }
Tobias Röhmelfca45e32022-12-06 11:25:03 +01002195
Richard Hendersonfedbaa02022-06-08 19:38:50 +01002196 if (mregion) {
2197 *mregion = matchregion;
2198 }
2199 }
2200
2201 fi->type = ARMFault_Permission;
Tobias Röhmelfca45e32022-12-06 11:25:03 +01002202 if (arm_feature(env, ARM_FEATURE_M)) {
2203 fi->level = 1;
2204 }
Richard Henderson7fa7ea82022-10-01 09:22:56 -07002205 return !(result->f.prot & (1 << access_type));
Richard Hendersonfedbaa02022-06-08 19:38:50 +01002206}
2207
Richard Henderson2c1f4292022-06-08 19:38:50 +01002208static bool v8m_is_sau_exempt(CPUARMState *env,
2209 uint32_t address, MMUAccessType access_type)
2210{
2211 /*
2212 * The architecture specifies that certain address ranges are
2213 * exempt from v8M SAU/IDAU checks.
2214 */
2215 return
2216 (access_type == MMU_INST_FETCH && m_is_system_region(env, address)) ||
2217 (address >= 0xe0000000 && address <= 0xe0002fff) ||
2218 (address >= 0xe000e000 && address <= 0xe000efff) ||
2219 (address >= 0xe002e000 && address <= 0xe002efff) ||
2220 (address >= 0xe0040000 && address <= 0xe0041fff) ||
2221 (address >= 0xe00ff000 && address <= 0xe00fffff);
2222}
2223
2224void v8m_security_lookup(CPUARMState *env, uint32_t address,
Richard Hendersondbf2a712022-08-22 08:26:46 -07002225 MMUAccessType access_type, ARMMMUIdx mmu_idx,
2226 bool is_secure, V8M_SAttributes *sattrs)
Richard Henderson2c1f4292022-06-08 19:38:50 +01002227{
2228 /*
2229 * Look up the security attributes for this address. Compare the
2230 * pseudocode SecurityCheck() function.
2231 * We assume the caller has zero-initialized *sattrs.
2232 */
2233 ARMCPU *cpu = env_archcpu(env);
2234 int r;
2235 bool idau_exempt = false, idau_ns = true, idau_nsc = true;
2236 int idau_region = IREGION_NOTVALID;
2237 uint32_t addr_page_base = address & TARGET_PAGE_MASK;
2238 uint32_t addr_page_limit = addr_page_base + (TARGET_PAGE_SIZE - 1);
2239
2240 if (cpu->idau) {
2241 IDAUInterfaceClass *iic = IDAU_INTERFACE_GET_CLASS(cpu->idau);
2242 IDAUInterface *ii = IDAU_INTERFACE(cpu->idau);
2243
2244 iic->check(ii, address, &idau_region, &idau_exempt, &idau_ns,
2245 &idau_nsc);
2246 }
2247
2248 if (access_type == MMU_INST_FETCH && extract32(address, 28, 4) == 0xf) {
2249 /* 0xf0000000..0xffffffff is always S for insn fetches */
2250 return;
2251 }
2252
2253 if (idau_exempt || v8m_is_sau_exempt(env, address, access_type)) {
Richard Hendersondbf2a712022-08-22 08:26:46 -07002254 sattrs->ns = !is_secure;
Richard Henderson2c1f4292022-06-08 19:38:50 +01002255 return;
2256 }
2257
2258 if (idau_region != IREGION_NOTVALID) {
2259 sattrs->irvalid = true;
2260 sattrs->iregion = idau_region;
2261 }
2262
2263 switch (env->sau.ctrl & 3) {
2264 case 0: /* SAU.ENABLE == 0, SAU.ALLNS == 0 */
2265 break;
2266 case 2: /* SAU.ENABLE == 0, SAU.ALLNS == 1 */
2267 sattrs->ns = true;
2268 break;
2269 default: /* SAU.ENABLE == 1 */
2270 for (r = 0; r < cpu->sau_sregion; r++) {
2271 if (env->sau.rlar[r] & 1) {
2272 uint32_t base = env->sau.rbar[r] & ~0x1f;
2273 uint32_t limit = env->sau.rlar[r] | 0x1f;
2274
2275 if (base <= address && limit >= address) {
2276 if (base > addr_page_base || limit < addr_page_limit) {
2277 sattrs->subpage = true;
2278 }
2279 if (sattrs->srvalid) {
2280 /*
2281 * If we hit in more than one region then we must report
2282 * as Secure, not NS-Callable, with no valid region
2283 * number info.
2284 */
2285 sattrs->ns = false;
2286 sattrs->nsc = false;
2287 sattrs->sregion = 0;
2288 sattrs->srvalid = false;
2289 break;
2290 } else {
2291 if (env->sau.rlar[r] & 2) {
2292 sattrs->nsc = true;
2293 } else {
2294 sattrs->ns = true;
2295 }
2296 sattrs->srvalid = true;
2297 sattrs->sregion = r;
2298 }
2299 } else {
2300 /*
2301 * Address not in this region. We must check whether the
2302 * region covers addresses in the same page as our address.
2303 * In that case we must not report a size that covers the
2304 * whole page for a subsequent hit against a different MPU
2305 * region or the background region, because it would result
2306 * in incorrect TLB hits for subsequent accesses to
2307 * addresses that are in this MPU region.
2308 */
2309 if (limit >= base &&
2310 ranges_overlap(base, limit - base + 1,
2311 addr_page_base,
2312 TARGET_PAGE_SIZE)) {
2313 sattrs->subpage = true;
2314 }
2315 }
2316 }
2317 }
2318 break;
2319 }
2320
2321 /*
2322 * The IDAU will override the SAU lookup results if it specifies
2323 * higher security than the SAU does.
2324 */
2325 if (!idau_ns) {
2326 if (sattrs->ns || (!idau_nsc && sattrs->nsc)) {
2327 sattrs->ns = false;
2328 sattrs->nsc = idau_nsc;
2329 }
2330 }
2331}
2332
Richard Henderson730d5c32022-06-08 19:38:49 +01002333static bool get_phys_addr_pmsav8(CPUARMState *env, uint32_t address,
2334 MMUAccessType access_type, ARMMMUIdx mmu_idx,
Richard Hendersonbe0ca942022-08-22 08:26:50 -07002335 bool secure, GetPhysAddrResult *result,
Richard Henderson730d5c32022-06-08 19:38:49 +01002336 ARMMMUFaultInfo *fi)
2337{
Richard Henderson730d5c32022-06-08 19:38:49 +01002338 V8M_SAttributes sattrs = {};
2339 bool ret;
Richard Henderson730d5c32022-06-08 19:38:49 +01002340
2341 if (arm_feature(env, ARM_FEATURE_M_SECURITY)) {
Richard Hendersondbf2a712022-08-22 08:26:46 -07002342 v8m_security_lookup(env, address, access_type, mmu_idx,
2343 secure, &sattrs);
Richard Henderson730d5c32022-06-08 19:38:49 +01002344 if (access_type == MMU_INST_FETCH) {
2345 /*
2346 * Instruction fetches always use the MMU bank and the
2347 * transaction attribute determined by the fetch address,
2348 * regardless of CPU state. This is painful for QEMU
2349 * to handle, because it would mean we need to encode
2350 * into the mmu_idx not just the (user, negpri) information
2351 * for the current security state but also that for the
2352 * other security state, which would balloon the number
2353 * of mmu_idx values needed alarmingly.
2354 * Fortunately we can avoid this because it's not actually
2355 * possible to arbitrarily execute code from memory with
2356 * the wrong security attribute: it will always generate
2357 * an exception of some kind or another, apart from the
2358 * special case of an NS CPU executing an SG instruction
2359 * in S&NSC memory. So we always just fail the translation
2360 * here and sort things out in the exception handler
2361 * (including possibly emulating an SG instruction).
2362 */
2363 if (sattrs.ns != !secure) {
2364 if (sattrs.nsc) {
2365 fi->type = ARMFault_QEMU_NSCExec;
2366 } else {
2367 fi->type = ARMFault_QEMU_SFault;
2368 }
Richard Henderson7fa7ea82022-10-01 09:22:56 -07002369 result->f.lg_page_size = sattrs.subpage ? 0 : TARGET_PAGE_BITS;
2370 result->f.phys_addr = address;
2371 result->f.prot = 0;
Richard Henderson730d5c32022-06-08 19:38:49 +01002372 return true;
2373 }
2374 } else {
2375 /*
2376 * For data accesses we always use the MMU bank indicated
2377 * by the current CPU state, but the security attributes
2378 * might downgrade a secure access to nonsecure.
2379 */
2380 if (sattrs.ns) {
Richard Henderson7fa7ea82022-10-01 09:22:56 -07002381 result->f.attrs.secure = false;
Richard Henderson730d5c32022-06-08 19:38:49 +01002382 } else if (!secure) {
2383 /*
2384 * NS access to S memory must fault.
2385 * Architecturally we should first check whether the
2386 * MPU information for this address indicates that we
2387 * are doing an unaligned access to Device memory, which
2388 * should generate a UsageFault instead. QEMU does not
2389 * currently check for that kind of unaligned access though.
2390 * If we added it we would need to do so as a special case
2391 * for M_FAKE_FSR_SFAULT in arm_v7m_cpu_do_interrupt().
2392 */
2393 fi->type = ARMFault_QEMU_SFault;
Richard Henderson7fa7ea82022-10-01 09:22:56 -07002394 result->f.lg_page_size = sattrs.subpage ? 0 : TARGET_PAGE_BITS;
2395 result->f.phys_addr = address;
2396 result->f.prot = 0;
Richard Henderson730d5c32022-06-08 19:38:49 +01002397 return true;
2398 }
2399 }
2400 }
2401
Richard Hendersone9fb7092022-08-22 08:26:47 -07002402 ret = pmsav8_mpu_lookup(env, address, access_type, mmu_idx, secure,
Richard Henderson652c7502022-08-22 08:26:45 -07002403 result, fi, NULL);
2404 if (sattrs.subpage) {
Richard Henderson7fa7ea82022-10-01 09:22:56 -07002405 result->f.lg_page_size = 0;
Richard Henderson652c7502022-08-22 08:26:45 -07002406 }
Richard Henderson730d5c32022-06-08 19:38:49 +01002407 return ret;
2408}
2409
Richard Henderson966f4bb2022-06-08 19:38:51 +01002410/*
2411 * Translate from the 4-bit stage 2 representation of
2412 * memory attributes (without cache-allocation hints) to
2413 * the 8-bit representation of the stage 1 MAIR registers
2414 * (which includes allocation hints).
2415 *
2416 * ref: shared/translation/attrs/S2AttrDecode()
2417 * .../S2ConvertAttrsHints()
2418 */
Richard Hendersonac76c2e2022-10-01 09:22:52 -07002419static uint8_t convert_stage2_attrs(uint64_t hcr, uint8_t s2attrs)
Richard Henderson966f4bb2022-06-08 19:38:51 +01002420{
2421 uint8_t hiattr = extract32(s2attrs, 2, 2);
2422 uint8_t loattr = extract32(s2attrs, 0, 2);
2423 uint8_t hihint = 0, lohint = 0;
2424
2425 if (hiattr != 0) { /* normal memory */
Richard Hendersonac76c2e2022-10-01 09:22:52 -07002426 if (hcr & HCR_CD) { /* cache disabled */
Richard Henderson966f4bb2022-06-08 19:38:51 +01002427 hiattr = loattr = 1; /* non-cacheable */
2428 } else {
2429 if (hiattr != 1) { /* Write-through or write-back */
2430 hihint = 3; /* RW allocate */
2431 }
2432 if (loattr != 1) { /* Write-through or write-back */
2433 lohint = 3; /* RW allocate */
2434 }
2435 }
2436 }
2437
2438 return (hiattr << 6) | (hihint << 4) | (loattr << 2) | lohint;
2439}
2440
2441/*
2442 * Combine either inner or outer cacheability attributes for normal
2443 * memory, according to table D4-42 and pseudocode procedure
2444 * CombineS1S2AttrHints() of ARM DDI 0487B.b (the ARMv8 ARM).
2445 *
2446 * NB: only stage 1 includes allocation hints (RW bits), leading to
2447 * some asymmetry.
2448 */
2449static uint8_t combine_cacheattr_nibble(uint8_t s1, uint8_t s2)
2450{
2451 if (s1 == 4 || s2 == 4) {
2452 /* non-cacheable has precedence */
2453 return 4;
2454 } else if (extract32(s1, 2, 2) == 0 || extract32(s1, 2, 2) == 2) {
2455 /* stage 1 write-through takes precedence */
2456 return s1;
2457 } else if (extract32(s2, 2, 2) == 2) {
2458 /* stage 2 write-through takes precedence, but the allocation hint
2459 * is still taken from stage 1
2460 */
2461 return (2 << 2) | extract32(s1, 0, 2);
2462 } else { /* write-back */
2463 return s1;
2464 }
2465}
2466
2467/*
2468 * Combine the memory type and cacheability attributes of
2469 * s1 and s2 for the HCR_EL2.FWB == 0 case, returning the
2470 * combined attributes in MAIR_EL1 format.
2471 */
Richard Hendersonac76c2e2022-10-01 09:22:52 -07002472static uint8_t combined_attrs_nofwb(uint64_t hcr,
Richard Henderson966f4bb2022-06-08 19:38:51 +01002473 ARMCacheAttrs s1, ARMCacheAttrs s2)
2474{
2475 uint8_t s1lo, s2lo, s1hi, s2hi, s2_mair_attrs, ret_attrs;
2476
Tobias Röhmelfaa14512022-12-06 11:25:00 +01002477 if (s2.is_s2_format) {
2478 s2_mair_attrs = convert_stage2_attrs(hcr, s2.attrs);
2479 } else {
2480 s2_mair_attrs = s2.attrs;
2481 }
Richard Henderson966f4bb2022-06-08 19:38:51 +01002482
2483 s1lo = extract32(s1.attrs, 0, 4);
2484 s2lo = extract32(s2_mair_attrs, 0, 4);
2485 s1hi = extract32(s1.attrs, 4, 4);
2486 s2hi = extract32(s2_mair_attrs, 4, 4);
2487
2488 /* Combine memory type and cacheability attributes */
2489 if (s1hi == 0 || s2hi == 0) {
2490 /* Device has precedence over normal */
2491 if (s1lo == 0 || s2lo == 0) {
2492 /* nGnRnE has precedence over anything */
2493 ret_attrs = 0;
2494 } else if (s1lo == 4 || s2lo == 4) {
2495 /* non-Reordering has precedence over Reordering */
2496 ret_attrs = 4; /* nGnRE */
2497 } else if (s1lo == 8 || s2lo == 8) {
2498 /* non-Gathering has precedence over Gathering */
2499 ret_attrs = 8; /* nGRE */
2500 } else {
2501 ret_attrs = 0xc; /* GRE */
2502 }
2503 } else { /* Normal memory */
2504 /* Outer/inner cacheability combine independently */
2505 ret_attrs = combine_cacheattr_nibble(s1hi, s2hi) << 4
2506 | combine_cacheattr_nibble(s1lo, s2lo);
2507 }
2508 return ret_attrs;
2509}
2510
2511static uint8_t force_cacheattr_nibble_wb(uint8_t attr)
2512{
2513 /*
2514 * Given the 4 bits specifying the outer or inner cacheability
2515 * in MAIR format, return a value specifying Normal Write-Back,
2516 * with the allocation and transient hints taken from the input
2517 * if the input specified some kind of cacheable attribute.
2518 */
2519 if (attr == 0 || attr == 4) {
2520 /*
2521 * 0 == an UNPREDICTABLE encoding
2522 * 4 == Non-cacheable
2523 * Either way, force Write-Back RW allocate non-transient
2524 */
2525 return 0xf;
2526 }
2527 /* Change WriteThrough to WriteBack, keep allocation and transient hints */
2528 return attr | 4;
2529}
2530
2531/*
2532 * Combine the memory type and cacheability attributes of
2533 * s1 and s2 for the HCR_EL2.FWB == 1 case, returning the
2534 * combined attributes in MAIR_EL1 format.
2535 */
Richard Henderson72cef092022-10-01 09:22:51 -07002536static uint8_t combined_attrs_fwb(ARMCacheAttrs s1, ARMCacheAttrs s2)
Richard Henderson966f4bb2022-06-08 19:38:51 +01002537{
Tobias Röhmelfaa14512022-12-06 11:25:00 +01002538 assert(s2.is_s2_format && !s1.is_s2_format);
2539
Richard Henderson966f4bb2022-06-08 19:38:51 +01002540 switch (s2.attrs) {
2541 case 7:
2542 /* Use stage 1 attributes */
2543 return s1.attrs;
2544 case 6:
2545 /*
2546 * Force Normal Write-Back. Note that if S1 is Normal cacheable
2547 * then we take the allocation hints from it; otherwise it is
2548 * RW allocate, non-transient.
2549 */
2550 if ((s1.attrs & 0xf0) == 0) {
2551 /* S1 is Device */
2552 return 0xff;
2553 }
2554 /* Need to check the Inner and Outer nibbles separately */
2555 return force_cacheattr_nibble_wb(s1.attrs & 0xf) |
2556 force_cacheattr_nibble_wb(s1.attrs >> 4) << 4;
2557 case 5:
2558 /* If S1 attrs are Device, use them; otherwise Normal Non-cacheable */
2559 if ((s1.attrs & 0xf0) == 0) {
2560 return s1.attrs;
2561 }
2562 return 0x44;
2563 case 0 ... 3:
2564 /* Force Device, of subtype specified by S2 */
2565 return s2.attrs << 2;
2566 default:
2567 /*
2568 * RESERVED values (including RES0 descriptor bit [5] being nonzero);
2569 * arbitrarily force Device.
2570 */
2571 return 0;
2572 }
2573}
2574
2575/*
2576 * Combine S1 and S2 cacheability/shareability attributes, per D4.5.4
2577 * and CombineS1S2Desc()
2578 *
2579 * @env: CPUARMState
2580 * @s1: Attributes from stage 1 walk
2581 * @s2: Attributes from stage 2 walk
2582 */
Richard Hendersonac76c2e2022-10-01 09:22:52 -07002583static ARMCacheAttrs combine_cacheattrs(uint64_t hcr,
Richard Henderson966f4bb2022-06-08 19:38:51 +01002584 ARMCacheAttrs s1, ARMCacheAttrs s2)
2585{
2586 ARMCacheAttrs ret;
2587 bool tagged = false;
2588
Tobias Röhmelfaa14512022-12-06 11:25:00 +01002589 assert(!s1.is_s2_format);
Richard Henderson966f4bb2022-06-08 19:38:51 +01002590 ret.is_s2_format = false;
Richard Henderson8539dc02023-04-07 11:51:49 -07002591 ret.guarded = s1.guarded;
Richard Henderson966f4bb2022-06-08 19:38:51 +01002592
2593 if (s1.attrs == 0xf0) {
2594 tagged = true;
2595 s1.attrs = 0xff;
2596 }
2597
2598 /* Combine shareability attributes (table D4-43) */
2599 if (s1.shareability == 2 || s2.shareability == 2) {
2600 /* if either are outer-shareable, the result is outer-shareable */
2601 ret.shareability = 2;
2602 } else if (s1.shareability == 3 || s2.shareability == 3) {
2603 /* if either are inner-shareable, the result is inner-shareable */
2604 ret.shareability = 3;
2605 } else {
2606 /* both non-shareable */
2607 ret.shareability = 0;
2608 }
2609
2610 /* Combine memory type and cacheability attributes */
Richard Hendersonac76c2e2022-10-01 09:22:52 -07002611 if (hcr & HCR_FWB) {
Richard Henderson72cef092022-10-01 09:22:51 -07002612 ret.attrs = combined_attrs_fwb(s1, s2);
Richard Henderson966f4bb2022-06-08 19:38:51 +01002613 } else {
Richard Hendersonac76c2e2022-10-01 09:22:52 -07002614 ret.attrs = combined_attrs_nofwb(hcr, s1, s2);
Richard Henderson966f4bb2022-06-08 19:38:51 +01002615 }
2616
2617 /*
2618 * Any location for which the resultant memory type is any
2619 * type of Device memory is always treated as Outer Shareable.
2620 * Any location for which the resultant memory type is Normal
2621 * Inner Non-cacheable, Outer Non-cacheable is always treated
2622 * as Outer Shareable.
2623 * TODO: FEAT_XS adds another value (0x40) also meaning iNCoNC
2624 */
2625 if ((ret.attrs & 0xf0) == 0 || ret.attrs == 0x44) {
2626 ret.shareability = 2;
2627 }
2628
2629 /* TODO: CombineS1S2Desc does not consider transient, only WB, RWA. */
2630 if (tagged && ret.attrs == 0xff) {
2631 ret.attrs = 0xf0;
2632 }
2633
2634 return ret;
2635}
2636
Richard Henderson448e42f2022-10-01 09:22:54 -07002637/*
2638 * MMU disabled. S1 addresses within aa64 translation regimes are
2639 * still checked for bounds -- see AArch64.S1DisabledOutput().
2640 */
2641static bool get_phys_addr_disabled(CPUARMState *env, target_ulong address,
2642 MMUAccessType access_type,
2643 ARMMMUIdx mmu_idx, bool is_secure,
2644 GetPhysAddrResult *result,
2645 ARMMMUFaultInfo *fi)
2646{
Richard Henderson5b74f9b2022-10-01 09:22:55 -07002647 uint8_t memattr = 0x00; /* Device nGnRnE */
2648 uint8_t shareability = 0; /* non-sharable */
Richard Hendersona1ce3082022-10-10 20:18:51 -07002649 int r_el;
Richard Henderson448e42f2022-10-01 09:22:54 -07002650
Richard Hendersona1ce3082022-10-10 20:18:51 -07002651 switch (mmu_idx) {
2652 case ARMMMUIdx_Stage2:
2653 case ARMMMUIdx_Stage2_S:
2654 case ARMMMUIdx_Phys_NS:
2655 case ARMMMUIdx_Phys_S:
2656 break;
Richard Henderson5b74f9b2022-10-01 09:22:55 -07002657
Richard Hendersona1ce3082022-10-10 20:18:51 -07002658 default:
2659 r_el = regime_el(env, mmu_idx);
Richard Henderson448e42f2022-10-01 09:22:54 -07002660 if (arm_el_is_aa64(env, r_el)) {
2661 int pamax = arm_pamax(env_archcpu(env));
2662 uint64_t tcr = env->cp15.tcr_el[r_el];
2663 int addrtop, tbi;
2664
2665 tbi = aa64_va_parameter_tbi(tcr, mmu_idx);
2666 if (access_type == MMU_INST_FETCH) {
2667 tbi &= ~aa64_va_parameter_tbid(tcr, mmu_idx);
2668 }
2669 tbi = (tbi >> extract64(address, 55, 1)) & 1;
2670 addrtop = (tbi ? 55 : 63);
2671
2672 if (extract64(address, pamax, addrtop - pamax + 1) != 0) {
2673 fi->type = ARMFault_AddressSize;
2674 fi->level = 0;
2675 fi->stage2 = false;
2676 return 1;
2677 }
2678
2679 /*
2680 * When TBI is disabled, we've just validated that all of the
2681 * bits above PAMax are zero, so logically we only need to
2682 * clear the top byte for TBI. But it's clearer to follow
2683 * the pseudocode set of addrdesc.paddress.
2684 */
2685 address = extract64(address, 0, 52);
2686 }
Richard Henderson5b74f9b2022-10-01 09:22:55 -07002687
2688 /* Fill in cacheattr a-la AArch64.TranslateAddressS1Off. */
2689 if (r_el == 1) {
2690 uint64_t hcr = arm_hcr_el2_eff_secstate(env, is_secure);
2691 if (hcr & HCR_DC) {
2692 if (hcr & HCR_DCT) {
2693 memattr = 0xf0; /* Tagged, Normal, WB, RWA */
2694 } else {
2695 memattr = 0xff; /* Normal, WB, RWA */
2696 }
2697 }
2698 }
2699 if (memattr == 0 && access_type == MMU_INST_FETCH) {
2700 if (regime_sctlr(env, mmu_idx) & SCTLR_I) {
2701 memattr = 0xee; /* Normal, WT, RA, NT */
2702 } else {
2703 memattr = 0x44; /* Normal, NC, No */
2704 }
2705 shareability = 2; /* outer sharable */
2706 }
2707 result->cacheattrs.is_s2_format = false;
Richard Hendersona1ce3082022-10-10 20:18:51 -07002708 break;
Richard Henderson448e42f2022-10-01 09:22:54 -07002709 }
2710
Richard Henderson7fa7ea82022-10-01 09:22:56 -07002711 result->f.phys_addr = address;
2712 result->f.prot = PAGE_READ | PAGE_WRITE | PAGE_EXEC;
2713 result->f.lg_page_size = TARGET_PAGE_BITS;
Richard Henderson5b74f9b2022-10-01 09:22:55 -07002714 result->cacheattrs.shareability = shareability;
Richard Henderson448e42f2022-10-01 09:22:54 -07002715 result->cacheattrs.attrs = memattr;
Richard Henderson6b72c542022-10-10 20:18:59 -07002716 return false;
Richard Henderson448e42f2022-10-01 09:22:54 -07002717}
2718
Richard Henderson3f5a74c2022-10-10 20:18:58 -07002719static bool get_phys_addr_twostage(CPUARMState *env, S1Translate *ptw,
2720 target_ulong address,
2721 MMUAccessType access_type,
2722 GetPhysAddrResult *result,
2723 ARMMMUFaultInfo *fi)
2724{
2725 hwaddr ipa;
Richard Hendersonc8d6c282022-10-24 15:18:51 +10002726 int s1_prot, s1_lgpgsz;
Richard Henderson3f5a74c2022-10-10 20:18:58 -07002727 bool is_secure = ptw->in_secure;
Richard Henderson6b72c542022-10-10 20:18:59 -07002728 bool ret, ipa_secure, s2walk_secure;
Richard Henderson3f5a74c2022-10-10 20:18:58 -07002729 ARMCacheAttrs cacheattrs1;
2730 bool is_el0;
2731 uint64_t hcr;
2732
2733 ret = get_phys_addr_with_struct(env, ptw, address, access_type, result, fi);
2734
Peter Maydell26ba00c2022-11-21 21:24:04 +00002735 /* If S1 fails, return early. */
2736 if (ret) {
Richard Henderson3f5a74c2022-10-10 20:18:58 -07002737 return ret;
2738 }
2739
2740 ipa = result->f.phys_addr;
2741 ipa_secure = result->f.attrs.secure;
2742 if (is_secure) {
2743 /* Select TCR based on the NS bit from the S1 walk. */
2744 s2walk_secure = !(ipa_secure
2745 ? env->cp15.vstcr_el2 & VSTCR_SW
2746 : env->cp15.vtcr_el2 & VTCR_NSW);
2747 } else {
2748 assert(!ipa_secure);
2749 s2walk_secure = false;
2750 }
2751
2752 is_el0 = ptw->in_mmu_idx == ARMMMUIdx_Stage1_E0;
2753 ptw->in_mmu_idx = s2walk_secure ? ARMMMUIdx_Stage2_S : ARMMMUIdx_Stage2;
Richard Henderson48da29e2022-10-24 15:18:39 +10002754 ptw->in_ptw_idx = s2walk_secure ? ARMMMUIdx_Phys_S : ARMMMUIdx_Phys_NS;
Richard Henderson3f5a74c2022-10-10 20:18:58 -07002755 ptw->in_secure = s2walk_secure;
2756
2757 /*
2758 * S1 is done, now do S2 translation.
2759 * Save the stage1 results so that we may merge prot and cacheattrs later.
2760 */
2761 s1_prot = result->f.prot;
Richard Hendersonc8d6c282022-10-24 15:18:51 +10002762 s1_lgpgsz = result->f.lg_page_size;
Richard Henderson3f5a74c2022-10-10 20:18:58 -07002763 cacheattrs1 = result->cacheattrs;
2764 memset(result, 0, sizeof(*result));
2765
Tobias Röhmelfca45e32022-12-06 11:25:03 +01002766 if (arm_feature(env, ARM_FEATURE_PMSA)) {
2767 ret = get_phys_addr_pmsav8(env, ipa, access_type,
2768 ptw->in_mmu_idx, is_secure, result, fi);
2769 } else {
2770 ret = get_phys_addr_lpae(env, ptw, ipa, access_type,
2771 is_el0, result, fi);
2772 }
Richard Henderson3f5a74c2022-10-10 20:18:58 -07002773 fi->s2addr = ipa;
2774
2775 /* Combine the S1 and S2 perms. */
2776 result->f.prot &= s1_prot;
2777
2778 /* If S2 fails, return early. */
2779 if (ret) {
2780 return ret;
2781 }
2782
Richard Hendersonc8d6c282022-10-24 15:18:51 +10002783 /*
Peter Maydell9e65f4e2022-12-12 14:27:08 +00002784 * If either S1 or S2 returned a result smaller than TARGET_PAGE_SIZE,
2785 * this means "don't put this in the TLB"; in this case, return a
2786 * result with lg_page_size == 0 to achieve that. Otherwise,
2787 * use the maximum of the S1 & S2 page size, so that invalidation
2788 * of pages > TARGET_PAGE_SIZE works correctly. (This works even though
2789 * we know the combined result permissions etc only cover the minimum
2790 * of the S1 and S2 page size, because we know that the common TLB code
2791 * never actually creates TLB entries bigger than TARGET_PAGE_SIZE,
2792 * and passing a larger page size value only affects invalidations.)
Richard Hendersonc8d6c282022-10-24 15:18:51 +10002793 */
Peter Maydell9e65f4e2022-12-12 14:27:08 +00002794 if (result->f.lg_page_size < TARGET_PAGE_BITS ||
2795 s1_lgpgsz < TARGET_PAGE_BITS) {
2796 result->f.lg_page_size = 0;
2797 } else if (result->f.lg_page_size < s1_lgpgsz) {
Richard Hendersonc8d6c282022-10-24 15:18:51 +10002798 result->f.lg_page_size = s1_lgpgsz;
2799 }
2800
Richard Henderson3f5a74c2022-10-10 20:18:58 -07002801 /* Combine the S1 and S2 cache attributes. */
2802 hcr = arm_hcr_el2_eff_secstate(env, is_secure);
2803 if (hcr & HCR_DC) {
2804 /*
2805 * HCR.DC forces the first stage attributes to
2806 * Normal Non-Shareable,
2807 * Inner Write-Back Read-Allocate Write-Allocate,
2808 * Outer Write-Back Read-Allocate Write-Allocate.
2809 * Do not overwrite Tagged within attrs.
2810 */
2811 if (cacheattrs1.attrs != 0xf0) {
2812 cacheattrs1.attrs = 0xff;
2813 }
2814 cacheattrs1.shareability = 0;
2815 }
2816 result->cacheattrs = combine_cacheattrs(hcr, cacheattrs1,
2817 result->cacheattrs);
2818
2819 /*
2820 * Check if IPA translates to secure or non-secure PA space.
2821 * Note that VSTCR overrides VTCR and {N}SW overrides {N}SA.
2822 */
2823 result->f.attrs.secure =
2824 (is_secure
2825 && !(env->cp15.vstcr_el2 & (VSTCR_SA | VSTCR_SW))
2826 && (ipa_secure
2827 || !(env->cp15.vtcr_el2 & (VTCR_NSA | VTCR_NSW))));
2828
Richard Henderson6b72c542022-10-10 20:18:59 -07002829 return false;
Richard Henderson3f5a74c2022-10-10 20:18:58 -07002830}
2831
Richard Henderson4a358552022-10-10 20:18:55 -07002832static bool get_phys_addr_with_struct(CPUARMState *env, S1Translate *ptw,
2833 target_ulong address,
2834 MMUAccessType access_type,
2835 GetPhysAddrResult *result,
2836 ARMMMUFaultInfo *fi)
Richard Henderson8ae08862022-06-08 19:38:48 +01002837{
Richard Henderson4a358552022-10-10 20:18:55 -07002838 ARMMMUIdx mmu_idx = ptw->in_mmu_idx;
Richard Henderson4a358552022-10-10 20:18:55 -07002839 bool is_secure = ptw->in_secure;
Richard Henderson48da29e2022-10-24 15:18:39 +10002840 ARMMMUIdx s1_mmu_idx;
Richard Henderson8ae08862022-06-08 19:38:48 +01002841
Richard Hendersoncead7fa2022-11-02 16:47:06 +11002842 /*
2843 * The page table entries may downgrade secure to non-secure, but
2844 * cannot upgrade an non-secure translation regime's attributes
2845 * to secure.
2846 */
2847 result->f.attrs.secure = is_secure;
2848
Richard Henderson48da29e2022-10-24 15:18:39 +10002849 switch (mmu_idx) {
2850 case ARMMMUIdx_Phys_S:
2851 case ARMMMUIdx_Phys_NS:
2852 /* Checking Phys early avoids special casing later vs regime_el. */
2853 return get_phys_addr_disabled(env, address, access_type, mmu_idx,
2854 is_secure, result, fi);
2855
2856 case ARMMMUIdx_Stage1_E0:
2857 case ARMMMUIdx_Stage1_E1:
2858 case ARMMMUIdx_Stage1_E1_PAN:
2859 /* First stage lookup uses second stage for ptw. */
2860 ptw->in_ptw_idx = is_secure ? ARMMMUIdx_Stage2_S : ARMMMUIdx_Stage2;
2861 break;
2862
2863 case ARMMMUIdx_E10_0:
2864 s1_mmu_idx = ARMMMUIdx_Stage1_E0;
2865 goto do_twostage;
2866 case ARMMMUIdx_E10_1:
2867 s1_mmu_idx = ARMMMUIdx_Stage1_E1;
2868 goto do_twostage;
2869 case ARMMMUIdx_E10_1_PAN:
2870 s1_mmu_idx = ARMMMUIdx_Stage1_E1_PAN;
2871 do_twostage:
Richard Henderson8ae08862022-06-08 19:38:48 +01002872 /*
2873 * Call ourselves recursively to do the stage 1 and then stage 2
Richard Henderson3f5a74c2022-10-10 20:18:58 -07002874 * translations if mmu_idx is a two-stage regime, and EL2 present.
2875 * Otherwise, a stage1+stage2 translation is just stage 1.
Richard Henderson8ae08862022-06-08 19:38:48 +01002876 */
Richard Henderson3f5a74c2022-10-10 20:18:58 -07002877 ptw->in_mmu_idx = mmu_idx = s1_mmu_idx;
Peter Maydell26ba00c2022-11-21 21:24:04 +00002878 if (arm_feature(env, ARM_FEATURE_EL2) &&
2879 !regime_translation_disabled(env, ARMMMUIdx_Stage2, is_secure)) {
Richard Henderson3f5a74c2022-10-10 20:18:58 -07002880 return get_phys_addr_twostage(env, ptw, address, access_type,
2881 result, fi);
Richard Henderson8ae08862022-06-08 19:38:48 +01002882 }
Richard Henderson48da29e2022-10-24 15:18:39 +10002883 /* fall through */
2884
2885 default:
2886 /* Single stage and second stage uses physical for ptw. */
2887 ptw->in_ptw_idx = is_secure ? ARMMMUIdx_Phys_S : ARMMMUIdx_Phys_NS;
2888 break;
Richard Henderson8ae08862022-06-08 19:38:48 +01002889 }
2890
Richard Henderson7fa7ea82022-10-01 09:22:56 -07002891 result->f.attrs.user = regime_is_user(env, mmu_idx);
Richard Henderson8ae08862022-06-08 19:38:48 +01002892
2893 /*
2894 * Fast Context Switch Extension. This doesn't exist at all in v8.
2895 * In v7 and earlier it affects all stage 1 translations.
2896 */
2897 if (address < 0x02000000 && mmu_idx != ARMMMUIdx_Stage2
2898 && !arm_feature(env, ARM_FEATURE_V8)) {
2899 if (regime_el(env, mmu_idx) == 3) {
2900 address += env->cp15.fcseidr_s;
2901 } else {
2902 address += env->cp15.fcseidr_ns;
2903 }
2904 }
2905
2906 if (arm_feature(env, ARM_FEATURE_PMSA)) {
2907 bool ret;
Richard Henderson7fa7ea82022-10-01 09:22:56 -07002908 result->f.lg_page_size = TARGET_PAGE_BITS;
Richard Henderson8ae08862022-06-08 19:38:48 +01002909
2910 if (arm_feature(env, ARM_FEATURE_V8)) {
2911 /* PMSAv8 */
2912 ret = get_phys_addr_pmsav8(env, address, access_type, mmu_idx,
Richard Hendersonbe0ca942022-08-22 08:26:50 -07002913 is_secure, result, fi);
Richard Henderson8ae08862022-06-08 19:38:48 +01002914 } else if (arm_feature(env, ARM_FEATURE_V7)) {
2915 /* PMSAv7 */
2916 ret = get_phys_addr_pmsav7(env, address, access_type, mmu_idx,
Richard Henderson957a0bb2022-08-22 08:26:53 -07002917 is_secure, result, fi);
Richard Henderson8ae08862022-06-08 19:38:48 +01002918 } else {
2919 /* Pre-v7 MPU */
2920 ret = get_phys_addr_pmsav5(env, address, access_type, mmu_idx,
Richard Hendersona5b50922022-08-22 08:26:55 -07002921 is_secure, result, fi);
Richard Henderson8ae08862022-06-08 19:38:48 +01002922 }
2923 qemu_log_mask(CPU_LOG_MMU, "PMSA MPU lookup for %s at 0x%08" PRIx32
2924 " mmu_idx %u -> %s (prot %c%c%c)\n",
2925 access_type == MMU_DATA_LOAD ? "reading" :
2926 (access_type == MMU_DATA_STORE ? "writing" : "execute"),
2927 (uint32_t)address, mmu_idx,
2928 ret ? "Miss" : "Hit",
Richard Henderson7fa7ea82022-10-01 09:22:56 -07002929 result->f.prot & PAGE_READ ? 'r' : '-',
2930 result->f.prot & PAGE_WRITE ? 'w' : '-',
2931 result->f.prot & PAGE_EXEC ? 'x' : '-');
Richard Henderson8ae08862022-06-08 19:38:48 +01002932
2933 return ret;
2934 }
2935
2936 /* Definitely a real MMU, not an MPU */
2937
Richard Henderson7e80c0a2022-10-01 09:22:40 -07002938 if (regime_translation_disabled(env, mmu_idx, is_secure)) {
Richard Henderson448e42f2022-10-01 09:22:54 -07002939 return get_phys_addr_disabled(env, address, access_type, mmu_idx,
2940 is_secure, result, fi);
Richard Henderson8ae08862022-06-08 19:38:48 +01002941 }
Richard Henderson6d2654f2022-10-10 20:18:54 -07002942
Richard Henderson8ae08862022-06-08 19:38:48 +01002943 if (regime_using_lpae_format(env, mmu_idx)) {
Richard Henderson4a358552022-10-10 20:18:55 -07002944 return get_phys_addr_lpae(env, ptw, address, access_type, false,
Richard Henderson6d2654f2022-10-10 20:18:54 -07002945 result, fi);
Timofey Kutergin6f2d9d72022-11-03 13:10:41 +00002946 } else if (arm_feature(env, ARM_FEATURE_V7) ||
2947 regime_sctlr(env, mmu_idx) & SCTLR_XP) {
Richard Henderson4a358552022-10-10 20:18:55 -07002948 return get_phys_addr_v6(env, ptw, address, access_type, result, fi);
Richard Henderson8ae08862022-06-08 19:38:48 +01002949 } else {
Richard Henderson4a358552022-10-10 20:18:55 -07002950 return get_phys_addr_v5(env, ptw, address, access_type, result, fi);
Richard Henderson8ae08862022-06-08 19:38:48 +01002951 }
2952}
Richard Henderson23971202022-06-08 19:38:54 +01002953
Richard Henderson4a358552022-10-10 20:18:55 -07002954bool get_phys_addr_with_secure(CPUARMState *env, target_ulong address,
2955 MMUAccessType access_type, ARMMMUIdx mmu_idx,
2956 bool is_secure, GetPhysAddrResult *result,
2957 ARMMMUFaultInfo *fi)
2958{
2959 S1Translate ptw = {
2960 .in_mmu_idx = mmu_idx,
2961 .in_secure = is_secure,
2962 };
2963 return get_phys_addr_with_struct(env, &ptw, address, access_type,
2964 result, fi);
2965}
2966
Richard Hendersondef8aa52022-10-01 09:22:41 -07002967bool get_phys_addr(CPUARMState *env, target_ulong address,
2968 MMUAccessType access_type, ARMMMUIdx mmu_idx,
2969 GetPhysAddrResult *result, ARMMMUFaultInfo *fi)
2970{
Richard Henderson03bea662022-10-01 09:22:44 -07002971 bool is_secure;
2972
2973 switch (mmu_idx) {
2974 case ARMMMUIdx_E10_0:
2975 case ARMMMUIdx_E10_1:
2976 case ARMMMUIdx_E10_1_PAN:
2977 case ARMMMUIdx_E20_0:
2978 case ARMMMUIdx_E20_2:
2979 case ARMMMUIdx_E20_2_PAN:
2980 case ARMMMUIdx_Stage1_E0:
2981 case ARMMMUIdx_Stage1_E1:
2982 case ARMMMUIdx_Stage1_E1_PAN:
2983 case ARMMMUIdx_E2:
Richard Hendersond902ae72022-10-01 09:22:46 -07002984 is_secure = arm_is_secure_below_el3(env);
2985 break;
Richard Henderson03bea662022-10-01 09:22:44 -07002986 case ARMMMUIdx_Stage2:
Richard Hendersona1ce3082022-10-10 20:18:51 -07002987 case ARMMMUIdx_Phys_NS:
Richard Henderson03bea662022-10-01 09:22:44 -07002988 case ARMMMUIdx_MPrivNegPri:
2989 case ARMMMUIdx_MUserNegPri:
2990 case ARMMMUIdx_MPriv:
2991 case ARMMMUIdx_MUser:
2992 is_secure = false;
2993 break;
Richard Hendersond902ae72022-10-01 09:22:46 -07002994 case ARMMMUIdx_E3:
Richard Henderson03bea662022-10-01 09:22:44 -07002995 case ARMMMUIdx_Stage2_S:
Richard Hendersona1ce3082022-10-10 20:18:51 -07002996 case ARMMMUIdx_Phys_S:
Richard Henderson03bea662022-10-01 09:22:44 -07002997 case ARMMMUIdx_MSPrivNegPri:
2998 case ARMMMUIdx_MSUserNegPri:
2999 case ARMMMUIdx_MSPriv:
3000 case ARMMMUIdx_MSUser:
3001 is_secure = true;
3002 break;
3003 default:
3004 g_assert_not_reached();
3005 }
Richard Hendersondef8aa52022-10-01 09:22:41 -07003006 return get_phys_addr_with_secure(env, address, access_type, mmu_idx,
Richard Henderson03bea662022-10-01 09:22:44 -07003007 is_secure, result, fi);
Richard Hendersondef8aa52022-10-01 09:22:41 -07003008}
3009
Richard Henderson23971202022-06-08 19:38:54 +01003010hwaddr arm_cpu_get_phys_page_attrs_debug(CPUState *cs, vaddr addr,
3011 MemTxAttrs *attrs)
3012{
3013 ARMCPU *cpu = ARM_CPU(cs);
3014 CPUARMState *env = &cpu->env;
Richard Henderson4a358552022-10-10 20:18:55 -07003015 S1Translate ptw = {
3016 .in_mmu_idx = arm_mmu_idx(env),
3017 .in_secure = arm_is_secure(env),
3018 .in_debug = true,
3019 };
Richard Hendersonde05a702022-08-22 08:26:36 -07003020 GetPhysAddrResult res = {};
Richard Henderson23971202022-06-08 19:38:54 +01003021 ARMMMUFaultInfo fi = {};
Richard Hendersonde05a702022-08-22 08:26:36 -07003022 bool ret;
Richard Henderson23971202022-06-08 19:38:54 +01003023
Richard Henderson4a358552022-10-10 20:18:55 -07003024 ret = get_phys_addr_with_struct(env, &ptw, addr, MMU_DATA_LOAD, &res, &fi);
Richard Henderson7fa7ea82022-10-01 09:22:56 -07003025 *attrs = res.f.attrs;
Richard Henderson23971202022-06-08 19:38:54 +01003026
3027 if (ret) {
3028 return -1;
3029 }
Richard Henderson7fa7ea82022-10-01 09:22:56 -07003030 return res.f.phys_addr;
Richard Henderson23971202022-06-08 19:38:54 +01003031}