blob: 4578b185be76caafd529d0f01ba47a2e6c9972a5 [file] [log] [blame]
bellardc896fe22008-02-01 10:05:41 +00001/*
2 * Tiny Code Generator for QEMU
3 *
4 * Copyright (c) 2008 Fabrice Bellard
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a copy
7 * of this software and associated documentation files (the "Software"), to deal
8 * in the Software without restriction, including without limitation the rights
9 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
10 * copies of the Software, and to permit persons to whom the Software is
11 * furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
22 * THE SOFTWARE.
23 */
24
Peter Maydell757e7252016-01-26 18:17:08 +000025#include "qemu/osdep.h"
aurel32cca82982009-04-16 09:58:30 +000026
Richard Henderson813da622012-03-19 12:25:11 -070027/* Define to jump the ELF file used to communicate with GDB. */
28#undef DEBUG_JIT
29
Emilio G. Cota72fd2ef2018-10-10 10:48:53 -040030#include "qemu/error-report.h"
Veronia Bahaaf348b6d2016-03-20 19:16:19 +020031#include "qemu/cutils.h"
Paolo Bonzini1de7afc2012-12-17 18:20:00 +010032#include "qemu/host-utils.h"
Markus Armbrusterd4c51a02019-04-17 21:17:51 +020033#include "qemu/qemu-print.h"
Richard Henderson084cfca2020-12-14 08:02:33 -060034#include "qemu/cacheflush.h"
Peter Maydellad768e62022-02-08 20:08:55 +000035#include "qemu/cacheinfo.h"
Richard W.M. Jones533206f2023-03-03 08:49:48 +000036#include "qemu/timer.h"
Richard Hendersoncac9b0f2023-04-01 21:22:06 -070037#include "exec/translation-block.h"
Richard Hendersond0a9bb52023-03-27 16:07:15 -070038#include "exec/tlb-common.h"
Richard Hendersond7ec12f2023-09-29 19:54:54 -070039#include "tcg/startup.h"
Richard Hendersonad3d0e42023-03-28 18:17:24 -070040#include "tcg/tcg-op-common.h"
Richard Henderson813da622012-03-19 12:25:11 -070041
Richard Hendersonedee2572013-08-20 17:20:30 -070042#if UINTPTR_MAX == UINT32_MAX
Richard Henderson813da622012-03-19 12:25:11 -070043# define ELF_CLASS ELFCLASS32
Richard Hendersonedee2572013-08-20 17:20:30 -070044#else
45# define ELF_CLASS ELFCLASS64
Richard Henderson813da622012-03-19 12:25:11 -070046#endif
Marc-André Lureaue03b5682022-03-23 19:57:17 +040047#if HOST_BIG_ENDIAN
Richard Henderson813da622012-03-19 12:25:11 -070048# define ELF_DATA ELFDATA2MSB
49#else
50# define ELF_DATA ELFDATA2LSB
51#endif
52
bellardc896fe22008-02-01 10:05:41 +000053#include "elf.h"
Paolo Bonzini508127e2016-01-07 16:55:28 +030054#include "exec/log.h"
Richard Hendersond2ba8022021-07-27 11:10:22 -100055#include "tcg/tcg-ldst.h"
Richard Henderson47f73132023-02-24 22:45:43 -100056#include "tcg/tcg-temp-internal.h"
Richard Henderson5ff72582021-03-09 16:24:14 -060057#include "tcg-internal.h"
Ilya Leoshkevich327b75a2024-01-25 06:46:30 +010058#include "tcg/perf.h"
Richard Henderson7d478302023-04-30 08:24:36 +010059#ifdef CONFIG_USER_ONLY
Philippe Mathieu-Daudéd3cbde72024-03-26 12:27:29 +010060#include "user/guest-base.h"
Richard Henderson7d478302023-04-30 08:24:36 +010061#endif
bellardc896fe22008-02-01 10:05:41 +000062
Paolo Bonzini139c1832020-02-04 12:41:01 +010063/* Forward declarations for functions declared in tcg-target.c.inc and
Peter Maydellce151102016-02-23 14:49:41 +000064 used here. */
Richard Hendersone4d58b42010-06-02 17:26:56 -070065static void tcg_target_init(TCGContext *s);
66static void tcg_target_qemu_prologue(TCGContext *s);
Richard Henderson6ac17782018-11-30 11:52:48 -080067static bool patch_reloc(tcg_insn_unit *code_ptr, int type,
Richard Henderson2ba7fae22013-08-20 15:30:10 -070068 intptr_t value, intptr_t addend);
bellardc896fe22008-02-01 10:05:41 +000069
Richard Henderson497a22e2013-06-05 07:39:57 -070070/* The CIE and FDE header definitions will be common to all hosts. */
71typedef struct {
72 uint32_t len __attribute__((aligned((sizeof(void *)))));
73 uint32_t id;
74 uint8_t version;
75 char augmentation[1];
76 uint8_t code_align;
77 uint8_t data_align;
78 uint8_t return_column;
79} DebugFrameCIE;
80
81typedef struct QEMU_PACKED {
82 uint32_t len __attribute__((aligned((sizeof(void *)))));
83 uint32_t cie_offset;
Richard Hendersonedee2572013-08-20 17:20:30 -070084 uintptr_t func_start;
85 uintptr_t func_len;
Richard Henderson497a22e2013-06-05 07:39:57 -070086} DebugFrameFDEHeader;
87
Richard Henderson2c907842014-05-15 12:48:01 -070088typedef struct QEMU_PACKED {
89 DebugFrameCIE cie;
90 DebugFrameFDEHeader fde;
91} DebugFrameHeader;
92
Richard Henderson2528f772023-04-07 18:18:03 -050093typedef struct TCGLabelQemuLdst {
94 bool is_ld; /* qemu_ld: true, qemu_st: false */
95 MemOpIdx oi;
96 TCGType type; /* result type of a load */
97 TCGReg addrlo_reg; /* reg index for low word of guest virtual addr */
98 TCGReg addrhi_reg; /* reg index for high word of guest virtual addr */
99 TCGReg datalo_reg; /* reg index for low word to be loaded or stored */
100 TCGReg datahi_reg; /* reg index for high word to be loaded or stored */
101 const tcg_insn_unit *raddr; /* addr of the next IR of qemu_ld/st IR */
102 tcg_insn_unit *label_ptr[2]; /* label pointers to be updated */
103 QSIMPLEQ_ENTRY(TCGLabelQemuLdst) next;
104} TCGLabelQemuLdst;
105
Richard Henderson755bf9e2020-10-29 09:17:30 -0700106static void tcg_register_jit_int(const void *buf, size_t size,
Richard Henderson2c907842014-05-15 12:48:01 -0700107 const void *debug_frame,
108 size_t debug_frame_size)
Richard Henderson813da622012-03-19 12:25:11 -0700109 __attribute__((unused));
110
Paolo Bonzini139c1832020-02-04 12:41:01 +0100111/* Forward declarations for functions declared and used in tcg-target.c.inc. */
Richard Henderson9358fbb2023-08-15 16:34:59 +0000112static void tcg_out_tb_start(TCGContext *s);
Richard Henderson2a534af2011-11-09 08:03:34 +0000113static void tcg_out_ld(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg1,
Richard Hendersona05b5b92013-08-20 17:07:26 -0700114 intptr_t arg2);
Richard Henderson78113e82019-03-16 17:48:18 +0000115static bool tcg_out_mov(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg);
Stefan Weilc0ad3002011-09-17 22:00:29 +0200116static void tcg_out_movi(TCGContext *s, TCGType type,
Richard Henderson2a534af2011-11-09 08:03:34 +0000117 TCGReg ret, tcg_target_long arg);
Richard Henderson678155b2023-04-05 11:17:01 -0700118static void tcg_out_ext8s(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg);
Richard Henderson753e42e2023-04-05 14:49:59 -0700119static void tcg_out_ext16s(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg);
Richard Hendersond0e66c82023-04-05 13:26:51 -0700120static void tcg_out_ext8u(TCGContext *s, TCGReg ret, TCGReg arg);
Richard Henderson379afdf2023-04-05 16:25:22 -0700121static void tcg_out_ext16u(TCGContext *s, TCGReg ret, TCGReg arg);
Richard Henderson52bf3392023-04-05 17:50:09 -0700122static void tcg_out_ext32s(TCGContext *s, TCGReg ret, TCGReg arg);
Richard Henderson9ecf5f62023-04-05 18:07:05 -0700123static void tcg_out_ext32u(TCGContext *s, TCGReg ret, TCGReg arg);
Richard Henderson9c6aa272023-04-05 18:30:56 -0700124static void tcg_out_exts_i32_i64(TCGContext *s, TCGReg ret, TCGReg arg);
Richard Hendersonb9bfe002023-04-05 18:56:28 -0700125static void tcg_out_extu_i32_i64(TCGContext *s, TCGReg ret, TCGReg arg);
Richard Hendersonb8b94ac2023-04-05 19:58:35 -0700126static void tcg_out_extrl_i64_i32(TCGContext *s, TCGReg ret, TCGReg arg);
Richard Henderson313bdea2022-10-31 09:22:59 +1100127static void tcg_out_addi_ptr(TCGContext *s, TCGReg, TCGReg, tcg_target_long);
Richard Henderson129f1f92023-04-05 22:27:03 -0700128static bool tcg_out_xchg(TCGContext *s, TCGType type, TCGReg r1, TCGReg r2);
Richard Hendersonb55a8d92022-11-26 12:42:06 -0800129static void tcg_out_exit_tb(TCGContext *s, uintptr_t arg);
Richard Hendersoncf7d6b82022-11-26 17:14:05 -0800130static void tcg_out_goto_tb(TCGContext *s, int which);
Miroslav Rezanina5e8892d2021-03-12 13:14:18 +0100131static void tcg_out_op(TCGContext *s, TCGOpcode opc,
132 const TCGArg args[TCG_MAX_OP_ARGS],
133 const int const_args[TCG_MAX_OP_ARGS]);
Richard Hendersond2fd7452017-09-14 13:53:46 -0700134#if TCG_TARGET_MAYBE_vec
Richard Hendersone7632cf2019-03-18 15:32:44 +0000135static bool tcg_out_dup_vec(TCGContext *s, TCGType type, unsigned vece,
136 TCGReg dst, TCGReg src);
Richard Hendersond6ecb4a2019-03-18 12:00:39 -0700137static bool tcg_out_dupm_vec(TCGContext *s, TCGType type, unsigned vece,
138 TCGReg dst, TCGReg base, intptr_t offset);
Richard Henderson4e186172020-03-31 01:02:08 -0700139static void tcg_out_dupi_vec(TCGContext *s, TCGType type, unsigned vece,
140 TCGReg dst, int64_t arg);
Miroslav Rezanina5e8892d2021-03-12 13:14:18 +0100141static void tcg_out_vec_op(TCGContext *s, TCGOpcode opc,
142 unsigned vecl, unsigned vece,
143 const TCGArg args[TCG_MAX_OP_ARGS],
144 const int const_args[TCG_MAX_OP_ARGS]);
Richard Hendersond2fd7452017-09-14 13:53:46 -0700145#else
Richard Hendersone7632cf2019-03-18 15:32:44 +0000146static inline bool tcg_out_dup_vec(TCGContext *s, TCGType type, unsigned vece,
147 TCGReg dst, TCGReg src)
148{
149 g_assert_not_reached();
150}
Richard Hendersond6ecb4a2019-03-18 12:00:39 -0700151static inline bool tcg_out_dupm_vec(TCGContext *s, TCGType type, unsigned vece,
152 TCGReg dst, TCGReg base, intptr_t offset)
153{
154 g_assert_not_reached();
155}
Richard Henderson4e186172020-03-31 01:02:08 -0700156static inline void tcg_out_dupi_vec(TCGContext *s, TCGType type, unsigned vece,
157 TCGReg dst, int64_t arg)
Richard Hendersone7632cf2019-03-18 15:32:44 +0000158{
159 g_assert_not_reached();
160}
Miroslav Rezanina5e8892d2021-03-12 13:14:18 +0100161static inline void tcg_out_vec_op(TCGContext *s, TCGOpcode opc,
162 unsigned vecl, unsigned vece,
163 const TCGArg args[TCG_MAX_OP_ARGS],
164 const int const_args[TCG_MAX_OP_ARGS])
Richard Hendersond2fd7452017-09-14 13:53:46 -0700165{
166 g_assert_not_reached();
167}
168#endif
Richard Henderson2a534af2011-11-09 08:03:34 +0000169static void tcg_out_st(TCGContext *s, TCGType type, TCGReg arg, TCGReg arg1,
Richard Hendersona05b5b92013-08-20 17:07:26 -0700170 intptr_t arg2);
Richard Henderson59d7c142016-06-19 22:59:13 -0700171static bool tcg_out_sti(TCGContext *s, TCGType type, TCGArg val,
172 TCGReg base, intptr_t ofs);
Richard Henderson7b7d8b22021-01-30 14:24:25 -0800173static void tcg_out_call(TCGContext *s, const tcg_insn_unit *target,
Richard Hendersoncee44b02022-10-18 17:51:41 +1000174 const TCGHelperInfo *info);
Richard Henderson5e3d0c12022-10-20 00:55:36 +1000175static TCGReg tcg_target_call_oarg_reg(TCGCallReturnKind kind, int slot);
Richard Henderson21e9a8a2023-10-27 15:44:45 -0700176static bool tcg_target_const_match(int64_t val, int ct,
177 TCGType type, TCGCond cond, int vece);
Richard Henderson659ef5c2017-07-30 12:30:41 -0700178#ifdef TCG_TARGET_NEED_LDST_LABELS
Richard Hendersonaeee05f2019-04-21 14:51:00 -0700179static int tcg_out_ldst_finalize(TCGContext *s);
Richard Henderson659ef5c2017-07-30 12:30:41 -0700180#endif
bellardc896fe22008-02-01 10:05:41 +0000181
Richard Henderson23088ca2023-10-01 17:12:32 +0000182#ifndef CONFIG_USER_ONLY
183#define guest_base ({ qemu_build_not_reached(); (uintptr_t)0; })
184#endif
185
Richard Henderson8429a1c2023-04-09 22:59:09 -0700186typedef struct TCGLdstHelperParam {
187 TCGReg (*ra_gen)(TCGContext *s, const TCGLabelQemuLdst *l, int arg_reg);
188 unsigned ntmp;
189 int tmp[3];
190} TCGLdstHelperParam;
191
192static void tcg_out_ld_helper_args(TCGContext *s, const TCGLabelQemuLdst *l,
193 const TCGLdstHelperParam *p)
194 __attribute__((unused));
195static void tcg_out_ld_helper_ret(TCGContext *s, const TCGLabelQemuLdst *l,
196 bool load_sign, const TCGLdstHelperParam *p)
197 __attribute__((unused));
198static void tcg_out_st_helper_args(TCGContext *s, const TCGLabelQemuLdst *l,
199 const TCGLdstHelperParam *p)
200 __attribute__((unused));
201
Richard Hendersonde950162022-11-07 19:08:33 +1100202static void * const qemu_ld_helpers[MO_SSIZE + 1] __attribute__((unused)) = {
Richard Henderson0cadc1e2022-11-01 12:51:04 +1100203 [MO_UB] = helper_ldub_mmu,
204 [MO_SB] = helper_ldsb_mmu,
205 [MO_UW] = helper_lduw_mmu,
206 [MO_SW] = helper_ldsw_mmu,
207 [MO_UL] = helper_ldul_mmu,
208 [MO_UQ] = helper_ldq_mmu,
209#if TCG_TARGET_REG_BITS == 64
210 [MO_SL] = helper_ldsl_mmu,
Richard Hendersonebebea52023-04-17 10:20:51 +0200211 [MO_128] = helper_ld16_mmu,
Richard Henderson0cadc1e2022-11-01 12:51:04 +1100212#endif
213};
214
Richard Hendersonde950162022-11-07 19:08:33 +1100215static void * const qemu_st_helpers[MO_SIZE + 1] __attribute__((unused)) = {
Richard Henderson0cadc1e2022-11-01 12:51:04 +1100216 [MO_8] = helper_stb_mmu,
217 [MO_16] = helper_stw_mmu,
218 [MO_32] = helper_stl_mmu,
219 [MO_64] = helper_stq_mmu,
Richard Hendersonebebea52023-04-17 10:20:51 +0200220#if TCG_TARGET_REG_BITS == 64
221 [MO_128] = helper_st16_mmu,
222#endif
Richard Henderson0cadc1e2022-11-01 12:51:04 +1100223};
Richard Henderson0cadc1e2022-11-01 12:51:04 +1100224
Richard Hendersone63b8a22022-11-08 09:23:54 +1100225typedef struct {
226 MemOp atom; /* lg2 bits of atomicity required */
227 MemOp align; /* lg2 bits of alignment to use */
228} TCGAtomAlign;
229
230static TCGAtomAlign atom_and_align_for_opc(TCGContext *s, MemOp opc,
231 MemOp host_atom, bool allow_two_ops)
232 __attribute__((unused));
233
Richard Henderson397caba2023-10-01 07:53:03 -0700234#ifdef CONFIG_USER_ONLY
235bool tcg_use_softmmu;
236#endif
237
Richard Henderson42eb6df2021-03-13 13:36:51 -0600238TCGContext tcg_init_ctx;
239__thread TCGContext *tcg_ctx;
240
Richard Henderson5ff72582021-03-09 16:24:14 -0600241TCGContext **tcg_ctxs;
Richard Henderson0e2d61c2021-03-09 23:06:32 -0600242unsigned int tcg_cur_ctxs;
243unsigned int tcg_max_ctxs;
Richard Hendersonad75a512023-09-13 16:37:36 -0700244TCGv_env tcg_env;
Richard Hendersonc8bc1162020-11-05 15:41:38 -0800245const void *tcg_code_gen_epilogue;
Richard Hendersondb0c51a2020-10-28 12:05:44 -0700246uintptr_t tcg_splitwx_diff;
Emilio G. Cotadf2cce22017-07-12 18:26:40 -0400247
Richard Hendersonb91ccb32020-10-28 14:11:54 -0700248#ifndef CONFIG_TCG_INTERPRETER
249tcg_prologue_fn *tcg_qemu_tb_exec;
250#endif
251
Richard Hendersond2fd7452017-09-14 13:53:46 -0700252static TCGRegSet tcg_target_available_regs[TCG_TYPE_COUNT];
blueswir1b1d8e522008-10-26 13:43:07 +0000253static TCGRegSet tcg_target_call_clobber_regs;
bellardc896fe22008-02-01 10:05:41 +0000254
Richard Henderson1813e172014-03-28 12:56:22 -0700255#if TCG_TARGET_INSN_UNIT_SIZE == 1
Peter Maydell4196dca2014-06-07 18:08:44 +0100256static __attribute__((unused)) inline void tcg_out8(TCGContext *s, uint8_t v)
bellardc896fe22008-02-01 10:05:41 +0000257{
258 *s->code_ptr++ = v;
259}
260
Peter Maydell4196dca2014-06-07 18:08:44 +0100261static __attribute__((unused)) inline void tcg_patch8(tcg_insn_unit *p,
262 uint8_t v)
Peter Maydell5c53bb82014-03-28 15:29:48 +0000263{
Richard Henderson1813e172014-03-28 12:56:22 -0700264 *p = v;
Peter Maydell5c53bb82014-03-28 15:29:48 +0000265}
Richard Henderson1813e172014-03-28 12:56:22 -0700266#endif
Peter Maydell5c53bb82014-03-28 15:29:48 +0000267
Richard Henderson1813e172014-03-28 12:56:22 -0700268#if TCG_TARGET_INSN_UNIT_SIZE <= 2
Peter Maydell4196dca2014-06-07 18:08:44 +0100269static __attribute__((unused)) inline void tcg_out16(TCGContext *s, uint16_t v)
bellardc896fe22008-02-01 10:05:41 +0000270{
Richard Henderson1813e172014-03-28 12:56:22 -0700271 if (TCG_TARGET_INSN_UNIT_SIZE == 2) {
272 *s->code_ptr++ = v;
273 } else {
274 tcg_insn_unit *p = s->code_ptr;
275 memcpy(p, &v, sizeof(v));
276 s->code_ptr = p + (2 / TCG_TARGET_INSN_UNIT_SIZE);
277 }
bellardc896fe22008-02-01 10:05:41 +0000278}
279
Peter Maydell4196dca2014-06-07 18:08:44 +0100280static __attribute__((unused)) inline void tcg_patch16(tcg_insn_unit *p,
281 uint16_t v)
Peter Maydell5c53bb82014-03-28 15:29:48 +0000282{
Richard Henderson1813e172014-03-28 12:56:22 -0700283 if (TCG_TARGET_INSN_UNIT_SIZE == 2) {
284 *p = v;
285 } else {
286 memcpy(p, &v, sizeof(v));
287 }
Peter Maydell5c53bb82014-03-28 15:29:48 +0000288}
Richard Henderson1813e172014-03-28 12:56:22 -0700289#endif
Peter Maydell5c53bb82014-03-28 15:29:48 +0000290
Richard Henderson1813e172014-03-28 12:56:22 -0700291#if TCG_TARGET_INSN_UNIT_SIZE <= 4
Peter Maydell4196dca2014-06-07 18:08:44 +0100292static __attribute__((unused)) inline void tcg_out32(TCGContext *s, uint32_t v)
bellardc896fe22008-02-01 10:05:41 +0000293{
Richard Henderson1813e172014-03-28 12:56:22 -0700294 if (TCG_TARGET_INSN_UNIT_SIZE == 4) {
295 *s->code_ptr++ = v;
296 } else {
297 tcg_insn_unit *p = s->code_ptr;
298 memcpy(p, &v, sizeof(v));
299 s->code_ptr = p + (4 / TCG_TARGET_INSN_UNIT_SIZE);
300 }
bellardc896fe22008-02-01 10:05:41 +0000301}
302
Peter Maydell4196dca2014-06-07 18:08:44 +0100303static __attribute__((unused)) inline void tcg_patch32(tcg_insn_unit *p,
304 uint32_t v)
Peter Maydell5c53bb82014-03-28 15:29:48 +0000305{
Richard Henderson1813e172014-03-28 12:56:22 -0700306 if (TCG_TARGET_INSN_UNIT_SIZE == 4) {
307 *p = v;
308 } else {
309 memcpy(p, &v, sizeof(v));
310 }
Peter Maydell5c53bb82014-03-28 15:29:48 +0000311}
Richard Henderson1813e172014-03-28 12:56:22 -0700312#endif
Peter Maydell5c53bb82014-03-28 15:29:48 +0000313
Richard Henderson1813e172014-03-28 12:56:22 -0700314#if TCG_TARGET_INSN_UNIT_SIZE <= 8
Peter Maydell4196dca2014-06-07 18:08:44 +0100315static __attribute__((unused)) inline void tcg_out64(TCGContext *s, uint64_t v)
Richard Hendersonac26eb62013-07-25 09:42:17 -1000316{
Richard Henderson1813e172014-03-28 12:56:22 -0700317 if (TCG_TARGET_INSN_UNIT_SIZE == 8) {
318 *s->code_ptr++ = v;
319 } else {
320 tcg_insn_unit *p = s->code_ptr;
321 memcpy(p, &v, sizeof(v));
322 s->code_ptr = p + (8 / TCG_TARGET_INSN_UNIT_SIZE);
323 }
Richard Hendersonac26eb62013-07-25 09:42:17 -1000324}
325
Peter Maydell4196dca2014-06-07 18:08:44 +0100326static __attribute__((unused)) inline void tcg_patch64(tcg_insn_unit *p,
327 uint64_t v)
Peter Maydell5c53bb82014-03-28 15:29:48 +0000328{
Richard Henderson1813e172014-03-28 12:56:22 -0700329 if (TCG_TARGET_INSN_UNIT_SIZE == 8) {
330 *p = v;
331 } else {
332 memcpy(p, &v, sizeof(v));
333 }
Peter Maydell5c53bb82014-03-28 15:29:48 +0000334}
Richard Henderson1813e172014-03-28 12:56:22 -0700335#endif
Peter Maydell5c53bb82014-03-28 15:29:48 +0000336
bellardc896fe22008-02-01 10:05:41 +0000337/* label relocation processing */
338
Richard Henderson1813e172014-03-28 12:56:22 -0700339static void tcg_out_reloc(TCGContext *s, tcg_insn_unit *code_ptr, int type,
Richard Hendersonbec16312015-02-13 13:39:54 -0800340 TCGLabel *l, intptr_t addend)
bellardc896fe22008-02-01 10:05:41 +0000341{
Richard Henderson7ecd02a2019-04-21 13:34:35 -0700342 TCGRelocation *r = tcg_malloc(sizeof(TCGRelocation));
bellardc896fe22008-02-01 10:05:41 +0000343
Richard Henderson7ecd02a2019-04-21 13:34:35 -0700344 r->type = type;
345 r->ptr = code_ptr;
346 r->addend = addend;
347 QSIMPLEQ_INSERT_TAIL(&l->relocs, r, next);
bellardc896fe22008-02-01 10:05:41 +0000348}
349
Richard Henderson92ab8e72020-10-28 18:55:50 -0700350static void tcg_out_label(TCGContext *s, TCGLabel *l)
bellardc896fe22008-02-01 10:05:41 +0000351{
Aurelien Jarnoeabb7b92016-04-21 10:48:49 +0200352 tcg_debug_assert(!l->has_value);
bellardc896fe22008-02-01 10:05:41 +0000353 l->has_value = 1;
Richard Henderson92ab8e72020-10-28 18:55:50 -0700354 l->u.value_ptr = tcg_splitwx_to_rx(s->code_ptr);
bellardc896fe22008-02-01 10:05:41 +0000355}
356
Richard Henderson42a268c2015-02-13 12:51:55 -0800357TCGLabel *gen_new_label(void)
bellardc896fe22008-02-01 10:05:41 +0000358{
Emilio G. Cotab1311c42017-07-12 17:15:52 -0400359 TCGContext *s = tcg_ctx;
Richard Henderson51e39722015-02-13 18:51:05 -0800360 TCGLabel *l = tcg_malloc(sizeof(TCGLabel));
bellardc896fe22008-02-01 10:05:41 +0000361
Richard Henderson7ecd02a2019-04-21 13:34:35 -0700362 memset(l, 0, sizeof(TCGLabel));
363 l->id = s->nb_labels++;
Richard Hendersonf85b1fc2023-03-03 13:47:27 -0800364 QSIMPLEQ_INIT(&l->branches);
Richard Henderson7ecd02a2019-04-21 13:34:35 -0700365 QSIMPLEQ_INIT(&l->relocs);
366
Richard Hendersonbef16ab2019-02-07 13:26:40 +0000367 QSIMPLEQ_INSERT_TAIL(&s->labels, l, next);
Richard Henderson42a268c2015-02-13 12:51:55 -0800368
369 return l;
bellardc896fe22008-02-01 10:05:41 +0000370}
371
Richard Henderson7ecd02a2019-04-21 13:34:35 -0700372static bool tcg_resolve_relocs(TCGContext *s)
373{
374 TCGLabel *l;
375
376 QSIMPLEQ_FOREACH(l, &s->labels, next) {
377 TCGRelocation *r;
378 uintptr_t value = l->u.value;
379
380 QSIMPLEQ_FOREACH(r, &l->relocs, next) {
381 if (!patch_reloc(r->ptr, r->type, value, r->addend)) {
382 return false;
383 }
384 }
385 }
386 return true;
387}
388
Richard Henderson9f754622018-06-14 19:57:03 -1000389static void set_jmp_reset_offset(TCGContext *s, int which)
390{
Richard Hendersonf14bed32020-11-02 19:36:20 -0800391 /*
392 * We will check for overflow at the end of the opcode loop in
393 * tcg_gen_code, where we bound tcg_current_code_size to UINT16_MAX.
394 */
Richard Hendersonb7e4afb2022-11-26 18:39:55 -0800395 s->gen_tb->jmp_reset_offset[which] = tcg_current_code_size(s);
Richard Henderson9f754622018-06-14 19:57:03 -1000396}
397
Richard Hendersonb52a2c02022-11-26 15:18:44 -0800398static void G_GNUC_UNUSED set_jmp_insn_offset(TCGContext *s, int which)
399{
400 /*
401 * We will check for overflow at the end of the opcode loop in
402 * tcg_gen_code, where we bound tcg_current_code_size to UINT16_MAX.
403 */
Richard Henderson9da60792022-11-26 18:54:23 -0800404 s->gen_tb->jmp_insn_offset[which] = tcg_current_code_size(s);
Richard Hendersonb52a2c02022-11-26 15:18:44 -0800405}
406
Richard Hendersonbecc4522022-11-26 17:42:11 -0800407static uintptr_t G_GNUC_UNUSED get_jmp_target_addr(TCGContext *s, int which)
408{
409 /*
410 * Return the read-execute version of the pointer, for the benefit
411 * of any pc-relative addressing mode.
412 */
Richard Henderson9da60792022-11-26 18:54:23 -0800413 return (uintptr_t)tcg_splitwx_to_rx(&s->gen_tb->jmp_target_addr[which]);
Richard Hendersonbecc4522022-11-26 17:42:11 -0800414}
415
Richard Henderson397caba2023-10-01 07:53:03 -0700416static int __attribute__((unused))
417tlb_mask_table_ofs(TCGContext *s, int which)
Richard Hendersond0a9bb52023-03-27 16:07:15 -0700418{
Richard Henderson7857ee12023-09-13 17:56:21 -0700419 return (offsetof(CPUNegativeOffsetState, tlb.f[which]) -
420 sizeof(CPUNegativeOffsetState));
Richard Hendersond0a9bb52023-03-27 16:07:15 -0700421}
Richard Hendersond0a9bb52023-03-27 16:07:15 -0700422
Richard Hendersondb6b7d02021-01-31 23:29:26 -1000423/* Signal overflow, starting over with fewer guest insns. */
Marc-André Lureau89057702022-04-20 17:26:02 +0400424static G_NORETURN
425void tcg_raise_tb_overflow(TCGContext *s)
Richard Hendersondb6b7d02021-01-31 23:29:26 -1000426{
427 siglongjmp(s->jmp_trans, -2);
428}
429
Richard Henderson8429a1c2023-04-09 22:59:09 -0700430/*
431 * Used by tcg_out_movext{1,2} to hold the arguments for tcg_out_movext.
432 * By the time we arrive at tcg_out_movext1, @dst is always a TCGReg.
433 *
434 * However, tcg_out_helper_load_slots reuses this field to hold an
435 * argument slot number (which may designate a argument register or an
436 * argument stack slot), converting to TCGReg once all arguments that
437 * are destined for the stack are processed.
438 */
Richard Henderson129f1f92023-04-05 22:27:03 -0700439typedef struct TCGMovExtend {
Richard Henderson8429a1c2023-04-09 22:59:09 -0700440 unsigned dst;
Richard Henderson129f1f92023-04-05 22:27:03 -0700441 TCGReg src;
442 TCGType dst_type;
443 TCGType src_type;
444 MemOp src_ext;
445} TCGMovExtend;
446
Richard Hendersonb3dfd5f2023-04-05 21:16:28 -0700447/**
448 * tcg_out_movext -- move and extend
449 * @s: tcg context
450 * @dst_type: integral type for destination
451 * @dst: destination register
452 * @src_type: integral type for source
453 * @src_ext: extension to apply to source
454 * @src: source register
455 *
456 * Move or extend @src into @dst, depending on @src_ext and the types.
457 */
Richard Henderson129f1f92023-04-05 22:27:03 -0700458static void tcg_out_movext(TCGContext *s, TCGType dst_type, TCGReg dst,
459 TCGType src_type, MemOp src_ext, TCGReg src)
Richard Hendersonb3dfd5f2023-04-05 21:16:28 -0700460{
461 switch (src_ext) {
462 case MO_UB:
463 tcg_out_ext8u(s, dst, src);
464 break;
465 case MO_SB:
466 tcg_out_ext8s(s, dst_type, dst, src);
467 break;
468 case MO_UW:
469 tcg_out_ext16u(s, dst, src);
470 break;
471 case MO_SW:
472 tcg_out_ext16s(s, dst_type, dst, src);
473 break;
474 case MO_UL:
475 case MO_SL:
476 if (dst_type == TCG_TYPE_I32) {
477 if (src_type == TCG_TYPE_I32) {
478 tcg_out_mov(s, TCG_TYPE_I32, dst, src);
479 } else {
480 tcg_out_extrl_i64_i32(s, dst, src);
481 }
482 } else if (src_type == TCG_TYPE_I32) {
483 if (src_ext & MO_SIGN) {
484 tcg_out_exts_i32_i64(s, dst, src);
485 } else {
486 tcg_out_extu_i32_i64(s, dst, src);
487 }
488 } else {
489 if (src_ext & MO_SIGN) {
490 tcg_out_ext32s(s, dst, src);
491 } else {
492 tcg_out_ext32u(s, dst, src);
493 }
494 }
495 break;
496 case MO_UQ:
497 tcg_debug_assert(TCG_TARGET_REG_BITS == 64);
498 if (dst_type == TCG_TYPE_I32) {
499 tcg_out_extrl_i64_i32(s, dst, src);
500 } else {
501 tcg_out_mov(s, TCG_TYPE_I64, dst, src);
502 }
503 break;
504 default:
505 g_assert_not_reached();
506 }
507}
508
Richard Henderson129f1f92023-04-05 22:27:03 -0700509/* Minor variations on a theme, using a structure. */
510static void tcg_out_movext1_new_src(TCGContext *s, const TCGMovExtend *i,
511 TCGReg src)
512{
513 tcg_out_movext(s, i->dst_type, i->dst, i->src_type, i->src_ext, src);
514}
515
516static void tcg_out_movext1(TCGContext *s, const TCGMovExtend *i)
517{
518 tcg_out_movext1_new_src(s, i, i->src);
519}
520
521/**
522 * tcg_out_movext2 -- move and extend two pair
523 * @s: tcg context
524 * @i1: first move description
525 * @i2: second move description
526 * @scratch: temporary register, or -1 for none
527 *
528 * As tcg_out_movext, for both @i1 and @i2, caring for overlap
529 * between the sources and destinations.
530 */
531
Richard Henderson8429a1c2023-04-09 22:59:09 -0700532static void tcg_out_movext2(TCGContext *s, const TCGMovExtend *i1,
533 const TCGMovExtend *i2, int scratch)
Richard Henderson129f1f92023-04-05 22:27:03 -0700534{
535 TCGReg src1 = i1->src;
536 TCGReg src2 = i2->src;
537
538 if (i1->dst != src2) {
539 tcg_out_movext1(s, i1);
540 tcg_out_movext1(s, i2);
541 return;
542 }
543 if (i2->dst == src1) {
544 TCGType src1_type = i1->src_type;
545 TCGType src2_type = i2->src_type;
546
547 if (tcg_out_xchg(s, MAX(src1_type, src2_type), src1, src2)) {
548 /* The data is now in the correct registers, now extend. */
549 src1 = i2->src;
550 src2 = i1->src;
551 } else {
552 tcg_debug_assert(scratch >= 0);
553 tcg_out_mov(s, src1_type, scratch, src1);
554 src1 = scratch;
555 }
556 }
557 tcg_out_movext1_new_src(s, i2, src2);
558 tcg_out_movext1_new_src(s, i1, src1);
559}
560
Richard Henderson2462e302023-05-14 09:58:39 -0700561/**
562 * tcg_out_movext3 -- move and extend three pair
563 * @s: tcg context
564 * @i1: first move description
565 * @i2: second move description
566 * @i3: third move description
567 * @scratch: temporary register, or -1 for none
568 *
569 * As tcg_out_movext, for all of @i1, @i2 and @i3, caring for overlap
570 * between the sources and destinations.
571 */
572
573static void tcg_out_movext3(TCGContext *s, const TCGMovExtend *i1,
574 const TCGMovExtend *i2, const TCGMovExtend *i3,
575 int scratch)
576{
577 TCGReg src1 = i1->src;
578 TCGReg src2 = i2->src;
579 TCGReg src3 = i3->src;
580
581 if (i1->dst != src2 && i1->dst != src3) {
582 tcg_out_movext1(s, i1);
583 tcg_out_movext2(s, i2, i3, scratch);
584 return;
585 }
586 if (i2->dst != src1 && i2->dst != src3) {
587 tcg_out_movext1(s, i2);
588 tcg_out_movext2(s, i1, i3, scratch);
589 return;
590 }
591 if (i3->dst != src1 && i3->dst != src2) {
592 tcg_out_movext1(s, i3);
593 tcg_out_movext2(s, i1, i2, scratch);
594 return;
595 }
596
597 /*
598 * There is a cycle. Since there are only 3 nodes, the cycle is
599 * either "clockwise" or "anti-clockwise", and can be solved with
600 * a single scratch or two xchg.
601 */
602 if (i1->dst == src2 && i2->dst == src3 && i3->dst == src1) {
603 /* "Clockwise" */
604 if (tcg_out_xchg(s, MAX(i1->src_type, i2->src_type), src1, src2)) {
605 tcg_out_xchg(s, MAX(i2->src_type, i3->src_type), src2, src3);
606 /* The data is now in the correct registers, now extend. */
607 tcg_out_movext1_new_src(s, i1, i1->dst);
608 tcg_out_movext1_new_src(s, i2, i2->dst);
609 tcg_out_movext1_new_src(s, i3, i3->dst);
610 } else {
611 tcg_debug_assert(scratch >= 0);
612 tcg_out_mov(s, i1->src_type, scratch, src1);
613 tcg_out_movext1(s, i3);
614 tcg_out_movext1(s, i2);
615 tcg_out_movext1_new_src(s, i1, scratch);
616 }
617 } else if (i1->dst == src3 && i2->dst == src1 && i3->dst == src2) {
618 /* "Anti-clockwise" */
619 if (tcg_out_xchg(s, MAX(i2->src_type, i3->src_type), src2, src3)) {
620 tcg_out_xchg(s, MAX(i1->src_type, i2->src_type), src1, src2);
621 /* The data is now in the correct registers, now extend. */
622 tcg_out_movext1_new_src(s, i1, i1->dst);
623 tcg_out_movext1_new_src(s, i2, i2->dst);
624 tcg_out_movext1_new_src(s, i3, i3->dst);
625 } else {
626 tcg_debug_assert(scratch >= 0);
627 tcg_out_mov(s, i1->src_type, scratch, src1);
628 tcg_out_movext1(s, i2);
629 tcg_out_movext1(s, i3);
630 tcg_out_movext1_new_src(s, i1, scratch);
631 }
632 } else {
633 g_assert_not_reached();
634 }
635}
636
Richard Henderson4c22e842020-10-16 22:20:55 -0700637#define C_PFX1(P, A) P##A
638#define C_PFX2(P, A, B) P##A##_##B
639#define C_PFX3(P, A, B, C) P##A##_##B##_##C
640#define C_PFX4(P, A, B, C, D) P##A##_##B##_##C##_##D
641#define C_PFX5(P, A, B, C, D, E) P##A##_##B##_##C##_##D##_##E
642#define C_PFX6(P, A, B, C, D, E, F) P##A##_##B##_##C##_##D##_##E##_##F
643
644/* Define an enumeration for the various combinations. */
645
646#define C_O0_I1(I1) C_PFX1(c_o0_i1_, I1),
647#define C_O0_I2(I1, I2) C_PFX2(c_o0_i2_, I1, I2),
648#define C_O0_I3(I1, I2, I3) C_PFX3(c_o0_i3_, I1, I2, I3),
649#define C_O0_I4(I1, I2, I3, I4) C_PFX4(c_o0_i4_, I1, I2, I3, I4),
650
651#define C_O1_I1(O1, I1) C_PFX2(c_o1_i1_, O1, I1),
652#define C_O1_I2(O1, I1, I2) C_PFX3(c_o1_i2_, O1, I1, I2),
653#define C_O1_I3(O1, I1, I2, I3) C_PFX4(c_o1_i3_, O1, I1, I2, I3),
654#define C_O1_I4(O1, I1, I2, I3, I4) C_PFX5(c_o1_i4_, O1, I1, I2, I3, I4),
655
656#define C_N1_I2(O1, I1, I2) C_PFX3(c_n1_i2_, O1, I1, I2),
Richard Hendersonca5bed02024-01-02 01:27:18 +0000657#define C_N1O1_I1(O1, O2, I1) C_PFX3(c_n1o1_i1_, O1, O2, I1),
Richard Hendersonfa645b42023-09-16 15:01:45 -0700658#define C_N2_I1(O1, O2, I1) C_PFX3(c_n2_i1_, O1, O2, I1),
Richard Henderson4c22e842020-10-16 22:20:55 -0700659
660#define C_O2_I1(O1, O2, I1) C_PFX3(c_o2_i1_, O1, O2, I1),
661#define C_O2_I2(O1, O2, I1, I2) C_PFX4(c_o2_i2_, O1, O2, I1, I2),
662#define C_O2_I3(O1, O2, I1, I2, I3) C_PFX5(c_o2_i3_, O1, O2, I1, I2, I3),
663#define C_O2_I4(O1, O2, I1, I2, I3, I4) C_PFX6(c_o2_i4_, O1, O2, I1, I2, I3, I4),
Ilya Leoshkevich22d2e532023-07-20 00:11:18 +0200664#define C_N1_O1_I4(O1, O2, I1, I2, I3, I4) C_PFX6(c_n1_o1_i4_, O1, O2, I1, I2, I3, I4),
Richard Henderson4c22e842020-10-16 22:20:55 -0700665
666typedef enum {
667#include "tcg-target-con-set.h"
668} TCGConstraintSetIndex;
669
670static TCGConstraintSetIndex tcg_target_op_def(TCGOpcode);
671
672#undef C_O0_I1
673#undef C_O0_I2
674#undef C_O0_I3
675#undef C_O0_I4
676#undef C_O1_I1
677#undef C_O1_I2
678#undef C_O1_I3
679#undef C_O1_I4
680#undef C_N1_I2
Richard Hendersonca5bed02024-01-02 01:27:18 +0000681#undef C_N1O1_I1
Richard Hendersonfa645b42023-09-16 15:01:45 -0700682#undef C_N2_I1
Richard Henderson4c22e842020-10-16 22:20:55 -0700683#undef C_O2_I1
684#undef C_O2_I2
685#undef C_O2_I3
686#undef C_O2_I4
Ilya Leoshkevich22d2e532023-07-20 00:11:18 +0200687#undef C_N1_O1_I4
Richard Henderson4c22e842020-10-16 22:20:55 -0700688
689/* Put all of the constraint sets into an array, indexed by the enum. */
690
691#define C_O0_I1(I1) { .args_ct_str = { #I1 } },
692#define C_O0_I2(I1, I2) { .args_ct_str = { #I1, #I2 } },
693#define C_O0_I3(I1, I2, I3) { .args_ct_str = { #I1, #I2, #I3 } },
694#define C_O0_I4(I1, I2, I3, I4) { .args_ct_str = { #I1, #I2, #I3, #I4 } },
695
696#define C_O1_I1(O1, I1) { .args_ct_str = { #O1, #I1 } },
697#define C_O1_I2(O1, I1, I2) { .args_ct_str = { #O1, #I1, #I2 } },
698#define C_O1_I3(O1, I1, I2, I3) { .args_ct_str = { #O1, #I1, #I2, #I3 } },
699#define C_O1_I4(O1, I1, I2, I3, I4) { .args_ct_str = { #O1, #I1, #I2, #I3, #I4 } },
700
701#define C_N1_I2(O1, I1, I2) { .args_ct_str = { "&" #O1, #I1, #I2 } },
Richard Hendersonca5bed02024-01-02 01:27:18 +0000702#define C_N1O1_I1(O1, O2, I1) { .args_ct_str = { "&" #O1, #O2, #I1 } },
Richard Hendersonfa645b42023-09-16 15:01:45 -0700703#define C_N2_I1(O1, O2, I1) { .args_ct_str = { "&" #O1, "&" #O2, #I1 } },
Richard Henderson4c22e842020-10-16 22:20:55 -0700704
705#define C_O2_I1(O1, O2, I1) { .args_ct_str = { #O1, #O2, #I1 } },
706#define C_O2_I2(O1, O2, I1, I2) { .args_ct_str = { #O1, #O2, #I1, #I2 } },
707#define C_O2_I3(O1, O2, I1, I2, I3) { .args_ct_str = { #O1, #O2, #I1, #I2, #I3 } },
708#define C_O2_I4(O1, O2, I1, I2, I3, I4) { .args_ct_str = { #O1, #O2, #I1, #I2, #I3, #I4 } },
Ilya Leoshkevich22d2e532023-07-20 00:11:18 +0200709#define C_N1_O1_I4(O1, O2, I1, I2, I3, I4) { .args_ct_str = { "&" #O1, #O2, #I1, #I2, #I3, #I4 } },
Richard Henderson4c22e842020-10-16 22:20:55 -0700710
711static const TCGTargetOpDef constraint_sets[] = {
712#include "tcg-target-con-set.h"
713};
714
715
716#undef C_O0_I1
717#undef C_O0_I2
718#undef C_O0_I3
719#undef C_O0_I4
720#undef C_O1_I1
721#undef C_O1_I2
722#undef C_O1_I3
723#undef C_O1_I4
724#undef C_N1_I2
Richard Hendersonca5bed02024-01-02 01:27:18 +0000725#undef C_N1O1_I1
Richard Hendersonfa645b42023-09-16 15:01:45 -0700726#undef C_N2_I1
Richard Henderson4c22e842020-10-16 22:20:55 -0700727#undef C_O2_I1
728#undef C_O2_I2
729#undef C_O2_I3
730#undef C_O2_I4
Ilya Leoshkevich22d2e532023-07-20 00:11:18 +0200731#undef C_N1_O1_I4
Richard Henderson4c22e842020-10-16 22:20:55 -0700732
733/* Expand the enumerator to be returned from tcg_target_op_def(). */
734
735#define C_O0_I1(I1) C_PFX1(c_o0_i1_, I1)
736#define C_O0_I2(I1, I2) C_PFX2(c_o0_i2_, I1, I2)
737#define C_O0_I3(I1, I2, I3) C_PFX3(c_o0_i3_, I1, I2, I3)
738#define C_O0_I4(I1, I2, I3, I4) C_PFX4(c_o0_i4_, I1, I2, I3, I4)
739
740#define C_O1_I1(O1, I1) C_PFX2(c_o1_i1_, O1, I1)
741#define C_O1_I2(O1, I1, I2) C_PFX3(c_o1_i2_, O1, I1, I2)
742#define C_O1_I3(O1, I1, I2, I3) C_PFX4(c_o1_i3_, O1, I1, I2, I3)
743#define C_O1_I4(O1, I1, I2, I3, I4) C_PFX5(c_o1_i4_, O1, I1, I2, I3, I4)
744
745#define C_N1_I2(O1, I1, I2) C_PFX3(c_n1_i2_, O1, I1, I2)
Richard Hendersonca5bed02024-01-02 01:27:18 +0000746#define C_N1O1_I1(O1, O2, I1) C_PFX3(c_n1o1_i1_, O1, O2, I1)
Richard Hendersonfa645b42023-09-16 15:01:45 -0700747#define C_N2_I1(O1, O2, I1) C_PFX3(c_n2_i1_, O1, O2, I1)
Richard Henderson4c22e842020-10-16 22:20:55 -0700748
749#define C_O2_I1(O1, O2, I1) C_PFX3(c_o2_i1_, O1, O2, I1)
750#define C_O2_I2(O1, O2, I1, I2) C_PFX4(c_o2_i2_, O1, O2, I1, I2)
751#define C_O2_I3(O1, O2, I1, I2, I3) C_PFX5(c_o2_i3_, O1, O2, I1, I2, I3)
752#define C_O2_I4(O1, O2, I1, I2, I3, I4) C_PFX6(c_o2_i4_, O1, O2, I1, I2, I3, I4)
Ilya Leoshkevich22d2e532023-07-20 00:11:18 +0200753#define C_N1_O1_I4(O1, O2, I1, I2, I3, I4) C_PFX6(c_n1_o1_i4_, O1, O2, I1, I2, I3, I4)
Richard Henderson4c22e842020-10-16 22:20:55 -0700754
Paolo Bonzini139c1832020-02-04 12:41:01 +0100755#include "tcg-target.c.inc"
bellardc896fe22008-02-01 10:05:41 +0000756
Richard Henderson7857ee12023-09-13 17:56:21 -0700757#ifndef CONFIG_TCG_INTERPRETER
758/* Validate CPUTLBDescFast placement. */
759QEMU_BUILD_BUG_ON((int)(offsetof(CPUNegativeOffsetState, tlb.f[0]) -
760 sizeof(CPUNegativeOffsetState))
761 < MIN_TLB_MASK_TABLE_OFS);
762#endif
763
Emilio G. Cotae8feb962017-07-07 19:24:20 -0400764/*
Emilio G. Cota3468b592017-07-19 18:57:58 -0400765 * All TCG threads except the parent (i.e. the one that called tcg_context_init
766 * and registered the target's TCG globals) must register with this function
767 * before initiating translation.
768 *
769 * In user-mode we just point tcg_ctx to tcg_init_ctx. See the documentation
770 * of tcg_region_init() for the reasoning behind this.
771 *
Philippe Mathieu-Daudé7893e422023-10-04 11:06:20 +0200772 * In system-mode each caller registers its context in tcg_ctxs[]. Note that in
773 * system-mode tcg_ctxs[] does not track tcg_ctx_init, since the initial context
Emilio G. Cota3468b592017-07-19 18:57:58 -0400774 * is not used anymore for translation once this function is called.
775 *
Philippe Mathieu-Daudé7893e422023-10-04 11:06:20 +0200776 * Not tracking tcg_init_ctx in tcg_ctxs[] in system-mode keeps code that
777 * iterates over the array (e.g. tcg_code_size() the same for both system/user
778 * modes.
Emilio G. Cota3468b592017-07-19 18:57:58 -0400779 */
780#ifdef CONFIG_USER_ONLY
781void tcg_register_thread(void)
782{
783 tcg_ctx = &tcg_init_ctx;
784}
785#else
786void tcg_register_thread(void)
787{
788 TCGContext *s = g_malloc(sizeof(*s));
789 unsigned int i, n;
Emilio G. Cota3468b592017-07-19 18:57:58 -0400790
791 *s = tcg_init_ctx;
792
793 /* Relink mem_base. */
794 for (i = 0, n = tcg_init_ctx.nb_globals; i < n; ++i) {
795 if (tcg_init_ctx.temps[i].mem_base) {
796 ptrdiff_t b = tcg_init_ctx.temps[i].mem_base - tcg_init_ctx.temps;
797 tcg_debug_assert(b >= 0 && b < n);
798 s->temps[i].mem_base = &s->temps[b];
799 }
800 }
801
802 /* Claim an entry in tcg_ctxs */
Richard Henderson0e2d61c2021-03-09 23:06:32 -0600803 n = qatomic_fetch_inc(&tcg_cur_ctxs);
804 g_assert(n < tcg_max_ctxs);
Stefan Hajnoczid73415a2020-09-23 11:56:46 +0100805 qatomic_set(&tcg_ctxs[n], s);
Emilio G. Cota3468b592017-07-19 18:57:58 -0400806
Emilio G. Cota38b47b12018-12-07 15:33:56 -0500807 if (n > 0) {
Richard Hendersonbf042e82021-03-09 16:33:15 -0600808 tcg_region_initial_alloc(s);
Emilio G. Cota38b47b12018-12-07 15:33:56 -0500809 }
810
Emilio G. Cota3468b592017-07-19 18:57:58 -0400811 tcg_ctx = s;
Emilio G. Cota3468b592017-07-19 18:57:58 -0400812}
813#endif /* !CONFIG_USER_ONLY */
814
bellardc896fe22008-02-01 10:05:41 +0000815/* pool based memory allocation */
816void *tcg_malloc_internal(TCGContext *s, int size)
817{
818 TCGPool *p;
819 int pool_size;
Richard Hendersona813e362022-11-30 22:38:25 -0800820
bellardc896fe22008-02-01 10:05:41 +0000821 if (size > TCG_POOL_CHUNK_SIZE) {
822 /* big malloc: insert a new pool (XXX: could optimize) */
Anthony Liguori7267c092011-08-20 22:09:37 -0500823 p = g_malloc(sizeof(TCGPool) + size);
bellardc896fe22008-02-01 10:05:41 +0000824 p->size = size;
Kirill Batuzov40552992012-03-02 13:22:17 +0400825 p->next = s->pool_first_large;
826 s->pool_first_large = p;
827 return p->data;
bellardc896fe22008-02-01 10:05:41 +0000828 } else {
829 p = s->pool_current;
830 if (!p) {
831 p = s->pool_first;
832 if (!p)
833 goto new_pool;
834 } else {
835 if (!p->next) {
836 new_pool:
837 pool_size = TCG_POOL_CHUNK_SIZE;
Anthony Liguori7267c092011-08-20 22:09:37 -0500838 p = g_malloc(sizeof(TCGPool) + pool_size);
bellardc896fe22008-02-01 10:05:41 +0000839 p->size = pool_size;
840 p->next = NULL;
Richard Hendersona813e362022-11-30 22:38:25 -0800841 if (s->pool_current) {
bellardc896fe22008-02-01 10:05:41 +0000842 s->pool_current->next = p;
Richard Hendersona813e362022-11-30 22:38:25 -0800843 } else {
bellardc896fe22008-02-01 10:05:41 +0000844 s->pool_first = p;
Richard Hendersona813e362022-11-30 22:38:25 -0800845 }
bellardc896fe22008-02-01 10:05:41 +0000846 } else {
847 p = p->next;
848 }
849 }
850 }
851 s->pool_current = p;
852 s->pool_cur = p->data + size;
853 s->pool_end = p->data + p->size;
854 return p->data;
855}
856
857void tcg_pool_reset(TCGContext *s)
858{
Kirill Batuzov40552992012-03-02 13:22:17 +0400859 TCGPool *p, *t;
860 for (p = s->pool_first_large; p; p = t) {
861 t = p->next;
862 g_free(p);
863 }
864 s->pool_first_large = NULL;
bellardc896fe22008-02-01 10:05:41 +0000865 s->pool_cur = s->pool_end = NULL;
866 s->pool_current = NULL;
867}
868
Richard Henderson8429a1c2023-04-09 22:59:09 -0700869/*
870 * Create TCGHelperInfo structures for "tcg/tcg-ldst.h" functions,
871 * akin to what "exec/helper-tcg.h" does with DEF_HELPER_FLAGS_N.
872 * We only use these for layout in tcg_out_ld_helper_ret and
873 * tcg_out_st_helper_args, and share them between several of
874 * the helpers, with the end result that it's easier to build manually.
875 */
876
877#if TCG_TARGET_REG_BITS == 32
878# define dh_typecode_ttl dh_typecode_i32
879#else
880# define dh_typecode_ttl dh_typecode_i64
881#endif
882
883static TCGHelperInfo info_helper_ld32_mmu = {
884 .flags = TCG_CALL_NO_WG,
885 .typemask = dh_typemask(ttl, 0) /* return tcg_target_ulong */
886 | dh_typemask(env, 1)
Richard Henderson24e46e62023-04-26 22:09:47 +0100887 | dh_typemask(i64, 2) /* uint64_t addr */
Richard Henderson8429a1c2023-04-09 22:59:09 -0700888 | dh_typemask(i32, 3) /* unsigned oi */
889 | dh_typemask(ptr, 4) /* uintptr_t ra */
890};
891
892static TCGHelperInfo info_helper_ld64_mmu = {
893 .flags = TCG_CALL_NO_WG,
894 .typemask = dh_typemask(i64, 0) /* return uint64_t */
895 | dh_typemask(env, 1)
Richard Henderson24e46e62023-04-26 22:09:47 +0100896 | dh_typemask(i64, 2) /* uint64_t addr */
Richard Henderson8429a1c2023-04-09 22:59:09 -0700897 | dh_typemask(i32, 3) /* unsigned oi */
898 | dh_typemask(ptr, 4) /* uintptr_t ra */
899};
900
Richard Hendersonebebea52023-04-17 10:20:51 +0200901static TCGHelperInfo info_helper_ld128_mmu = {
902 .flags = TCG_CALL_NO_WG,
903 .typemask = dh_typemask(i128, 0) /* return Int128 */
904 | dh_typemask(env, 1)
Richard Henderson24e46e62023-04-26 22:09:47 +0100905 | dh_typemask(i64, 2) /* uint64_t addr */
Richard Hendersonebebea52023-04-17 10:20:51 +0200906 | dh_typemask(i32, 3) /* unsigned oi */
907 | dh_typemask(ptr, 4) /* uintptr_t ra */
908};
909
Richard Henderson8429a1c2023-04-09 22:59:09 -0700910static TCGHelperInfo info_helper_st32_mmu = {
911 .flags = TCG_CALL_NO_WG,
912 .typemask = dh_typemask(void, 0)
913 | dh_typemask(env, 1)
Richard Henderson24e46e62023-04-26 22:09:47 +0100914 | dh_typemask(i64, 2) /* uint64_t addr */
Richard Henderson8429a1c2023-04-09 22:59:09 -0700915 | dh_typemask(i32, 3) /* uint32_t data */
916 | dh_typemask(i32, 4) /* unsigned oi */
917 | dh_typemask(ptr, 5) /* uintptr_t ra */
918};
919
920static TCGHelperInfo info_helper_st64_mmu = {
921 .flags = TCG_CALL_NO_WG,
922 .typemask = dh_typemask(void, 0)
923 | dh_typemask(env, 1)
Richard Henderson24e46e62023-04-26 22:09:47 +0100924 | dh_typemask(i64, 2) /* uint64_t addr */
Richard Henderson8429a1c2023-04-09 22:59:09 -0700925 | dh_typemask(i64, 3) /* uint64_t data */
926 | dh_typemask(i32, 4) /* unsigned oi */
927 | dh_typemask(ptr, 5) /* uintptr_t ra */
928};
929
Richard Hendersonebebea52023-04-17 10:20:51 +0200930static TCGHelperInfo info_helper_st128_mmu = {
931 .flags = TCG_CALL_NO_WG,
932 .typemask = dh_typemask(void, 0)
933 | dh_typemask(env, 1)
Richard Henderson24e46e62023-04-26 22:09:47 +0100934 | dh_typemask(i64, 2) /* uint64_t addr */
Richard Hendersonebebea52023-04-17 10:20:51 +0200935 | dh_typemask(i128, 3) /* Int128 data */
936 | dh_typemask(i32, 4) /* unsigned oi */
937 | dh_typemask(ptr, 5) /* uintptr_t ra */
938};
939
Richard Henderson22f15572021-03-18 12:46:44 -0600940#ifdef CONFIG_TCG_INTERPRETER
Philippe Mathieu-Daudéc6ef8c72022-11-22 19:08:02 +0100941static ffi_type *typecode_to_ffi(int argmask)
942{
Richard Hendersone9709e12022-10-21 10:47:54 +1000943 /*
944 * libffi does not support __int128_t, so we have forced Int128
945 * to use the structure definition instead of the builtin type.
946 */
947 static ffi_type *ffi_type_i128_elements[3] = {
948 &ffi_type_uint64,
949 &ffi_type_uint64,
950 NULL
951 };
952 static ffi_type ffi_type_i128 = {
953 .size = 16,
954 .alignment = __alignof__(Int128),
955 .type = FFI_TYPE_STRUCT,
956 .elements = ffi_type_i128_elements,
957 };
958
Philippe Mathieu-Daudéc6ef8c72022-11-22 19:08:02 +0100959 switch (argmask) {
960 case dh_typecode_void:
961 return &ffi_type_void;
962 case dh_typecode_i32:
963 return &ffi_type_uint32;
964 case dh_typecode_s32:
965 return &ffi_type_sint32;
966 case dh_typecode_i64:
967 return &ffi_type_uint64;
968 case dh_typecode_s64:
969 return &ffi_type_sint64;
970 case dh_typecode_ptr:
971 return &ffi_type_pointer;
Richard Hendersone9709e12022-10-21 10:47:54 +1000972 case dh_typecode_i128:
973 return &ffi_type_i128;
Philippe Mathieu-Daudéc6ef8c72022-11-22 19:08:02 +0100974 }
975 g_assert_not_reached();
976}
Philippe Mathieu-Daudé0c22e172022-11-22 19:08:03 +0100977
Richard Hendersond53106c2023-03-31 10:37:04 -0700978static ffi_cif *init_ffi_layout(TCGHelperInfo *info)
Philippe Mathieu-Daudé0c22e172022-11-22 19:08:03 +0100979{
Richard Hendersond53106c2023-03-31 10:37:04 -0700980 unsigned typemask = info->typemask;
981 struct {
982 ffi_cif cif;
983 ffi_type *args[];
984 } *ca;
985 ffi_status status;
986 int nargs;
Richard Hendersonf9c4bb82022-11-22 19:08:04 +0100987
Richard Hendersond53106c2023-03-31 10:37:04 -0700988 /* Ignoring the return type, find the last non-zero field. */
989 nargs = 32 - clz32(typemask >> 3);
990 nargs = DIV_ROUND_UP(nargs, 3);
991 assert(nargs <= MAX_CALL_IARGS);
Philippe Mathieu-Daudé0c22e172022-11-22 19:08:03 +0100992
Richard Hendersond53106c2023-03-31 10:37:04 -0700993 ca = g_malloc0(sizeof(*ca) + nargs * sizeof(ffi_type *));
994 ca->cif.rtype = typecode_to_ffi(typemask & 7);
995 ca->cif.nargs = nargs;
996
997 if (nargs != 0) {
998 ca->cif.arg_types = ca->args;
999 for (int j = 0; j < nargs; ++j) {
1000 int typecode = extract32(typemask, (j + 1) * 3, 3);
1001 ca->args[j] = typecode_to_ffi(typecode);
Philippe Mathieu-Daudé0c22e172022-11-22 19:08:03 +01001002 }
Philippe Mathieu-Daudé0c22e172022-11-22 19:08:03 +01001003 }
Richard Hendersonf9c4bb82022-11-22 19:08:04 +01001004
Richard Hendersond53106c2023-03-31 10:37:04 -07001005 status = ffi_prep_cif(&ca->cif, FFI_DEFAULT_ABI, nargs,
1006 ca->cif.rtype, ca->cif.arg_types);
1007 assert(status == FFI_OK);
1008
1009 return &ca->cif;
Philippe Mathieu-Daudé0c22e172022-11-22 19:08:03 +01001010}
Richard Hendersond53106c2023-03-31 10:37:04 -07001011
1012#define HELPER_INFO_INIT(I) (&(I)->cif)
1013#define HELPER_INFO_INIT_VAL(I) init_ffi_layout(I)
1014#else
1015#define HELPER_INFO_INIT(I) (&(I)->init)
1016#define HELPER_INFO_INIT_VAL(I) 1
Philippe Mathieu-Daudé0c22e172022-11-22 19:08:03 +01001017#endif /* CONFIG_TCG_INTERPRETER */
Richard Henderson22f15572021-03-18 12:46:44 -06001018
Richard Henderson338b61e2023-04-08 17:28:07 -07001019static inline bool arg_slot_reg_p(unsigned arg_slot)
1020{
1021 /*
1022 * Split the sizeof away from the comparison to avoid Werror from
1023 * "unsigned < 0 is always false", when iarg_regs is empty.
1024 */
1025 unsigned nreg = ARRAY_SIZE(tcg_target_call_iarg_regs);
1026 return arg_slot < nreg;
1027}
1028
Richard Hendersond78e4a42023-04-08 19:05:10 -07001029static inline int arg_slot_stk_ofs(unsigned arg_slot)
1030{
1031 unsigned max = TCG_STATIC_CALL_ARGS_SIZE / sizeof(tcg_target_long);
1032 unsigned stk_slot = arg_slot - ARRAY_SIZE(tcg_target_call_iarg_regs);
1033
1034 tcg_debug_assert(stk_slot < max);
1035 return TCG_TARGET_CALL_STACK_OFFSET + stk_slot * sizeof(tcg_target_long);
1036}
1037
Richard Henderson39004a72022-11-11 10:09:37 +10001038typedef struct TCGCumulativeArgs {
1039 int arg_idx; /* tcg_gen_callN args[] */
1040 int info_in_idx; /* TCGHelperInfo in[] */
1041 int arg_slot; /* regs+stack slot */
1042 int ref_slot; /* stack slots for references */
1043} TCGCumulativeArgs;
1044
1045static void layout_arg_even(TCGCumulativeArgs *cum)
1046{
1047 cum->arg_slot += cum->arg_slot & 1;
1048}
1049
1050static void layout_arg_1(TCGCumulativeArgs *cum, TCGHelperInfo *info,
1051 TCGCallArgumentKind kind)
1052{
1053 TCGCallArgumentLoc *loc = &info->in[cum->info_in_idx];
1054
1055 *loc = (TCGCallArgumentLoc){
1056 .kind = kind,
1057 .arg_idx = cum->arg_idx,
1058 .arg_slot = cum->arg_slot,
1059 };
1060 cum->info_in_idx++;
1061 cum->arg_slot++;
1062}
1063
1064static void layout_arg_normal_n(TCGCumulativeArgs *cum,
1065 TCGHelperInfo *info, int n)
1066{
1067 TCGCallArgumentLoc *loc = &info->in[cum->info_in_idx];
1068
1069 for (int i = 0; i < n; ++i) {
1070 /* Layout all using the same arg_idx, adjusting the subindex. */
1071 loc[i] = (TCGCallArgumentLoc){
1072 .kind = TCG_CALL_ARG_NORMAL,
1073 .arg_idx = cum->arg_idx,
1074 .tmp_subindex = i,
1075 .arg_slot = cum->arg_slot + i,
1076 };
1077 }
1078 cum->info_in_idx += n;
1079 cum->arg_slot += n;
1080}
1081
Richard Henderson313bdea2022-10-31 09:22:59 +11001082static void layout_arg_by_ref(TCGCumulativeArgs *cum, TCGHelperInfo *info)
1083{
1084 TCGCallArgumentLoc *loc = &info->in[cum->info_in_idx];
1085 int n = 128 / TCG_TARGET_REG_BITS;
1086
1087 /* The first subindex carries the pointer. */
1088 layout_arg_1(cum, info, TCG_CALL_ARG_BY_REF);
1089
1090 /*
1091 * The callee is allowed to clobber memory associated with
1092 * structure pass by-reference. Therefore we must make copies.
1093 * Allocate space from "ref_slot", which will be adjusted to
1094 * follow the parameters on the stack.
1095 */
1096 loc[0].ref_slot = cum->ref_slot;
1097
1098 /*
1099 * Subsequent words also go into the reference slot, but
1100 * do not accumulate into the regular arguments.
1101 */
1102 for (int i = 1; i < n; ++i) {
1103 loc[i] = (TCGCallArgumentLoc){
1104 .kind = TCG_CALL_ARG_BY_REF_N,
1105 .arg_idx = cum->arg_idx,
1106 .tmp_subindex = i,
1107 .ref_slot = cum->ref_slot + i,
1108 };
1109 }
Richard Hendersone18ed262023-07-07 10:17:44 +00001110 cum->info_in_idx += n - 1; /* i=0 accounted for in layout_arg_1 */
Richard Henderson313bdea2022-10-31 09:22:59 +11001111 cum->ref_slot += n;
1112}
1113
Richard Henderson39004a72022-11-11 10:09:37 +10001114static void init_call_layout(TCGHelperInfo *info)
1115{
1116 int max_reg_slots = ARRAY_SIZE(tcg_target_call_iarg_regs);
1117 int max_stk_slots = TCG_STATIC_CALL_ARGS_SIZE / sizeof(tcg_target_long);
1118 unsigned typemask = info->typemask;
1119 unsigned typecode;
1120 TCGCumulativeArgs cum = { };
1121
1122 /*
1123 * Parse and place any function return value.
1124 */
1125 typecode = typemask & 7;
1126 switch (typecode) {
1127 case dh_typecode_void:
1128 info->nr_out = 0;
1129 break;
1130 case dh_typecode_i32:
1131 case dh_typecode_s32:
1132 case dh_typecode_ptr:
1133 info->nr_out = 1;
1134 info->out_kind = TCG_CALL_RET_NORMAL;
1135 break;
1136 case dh_typecode_i64:
1137 case dh_typecode_s64:
1138 info->nr_out = 64 / TCG_TARGET_REG_BITS;
1139 info->out_kind = TCG_CALL_RET_NORMAL;
Richard Henderson5e3d0c12022-10-20 00:55:36 +10001140 /* Query the last register now to trigger any assert early. */
1141 tcg_target_call_oarg_reg(info->out_kind, info->nr_out - 1);
Richard Henderson466d3752022-11-11 11:01:13 +10001142 break;
1143 case dh_typecode_i128:
1144 info->nr_out = 128 / TCG_TARGET_REG_BITS;
Richard Henderson5427a9a2022-10-20 07:54:48 +10001145 info->out_kind = TCG_TARGET_CALL_RET_I128;
1146 switch (TCG_TARGET_CALL_RET_I128) {
Richard Henderson466d3752022-11-11 11:01:13 +10001147 case TCG_CALL_RET_NORMAL:
Richard Henderson5e3d0c12022-10-20 00:55:36 +10001148 /* Query the last register now to trigger any assert early. */
1149 tcg_target_call_oarg_reg(info->out_kind, info->nr_out - 1);
Richard Henderson466d3752022-11-11 11:01:13 +10001150 break;
Richard Hendersonc6556aa2022-10-20 01:13:52 +10001151 case TCG_CALL_RET_BY_VEC:
1152 /* Query the single register now to trigger any assert early. */
1153 tcg_target_call_oarg_reg(TCG_CALL_RET_BY_VEC, 0);
1154 break;
Richard Henderson313bdea2022-10-31 09:22:59 +11001155 case TCG_CALL_RET_BY_REF:
1156 /*
1157 * Allocate the first argument to the output.
1158 * We don't need to store this anywhere, just make it
1159 * unavailable for use in the input loop below.
1160 */
1161 cum.arg_slot = 1;
1162 break;
Richard Henderson466d3752022-11-11 11:01:13 +10001163 default:
1164 qemu_build_not_reached();
1165 }
Richard Henderson39004a72022-11-11 10:09:37 +10001166 break;
1167 default:
1168 g_assert_not_reached();
1169 }
Richard Henderson39004a72022-11-11 10:09:37 +10001170
1171 /*
1172 * Parse and place function arguments.
1173 */
1174 for (typemask >>= 3; typemask; typemask >>= 3, cum.arg_idx++) {
1175 TCGCallArgumentKind kind;
1176 TCGType type;
1177
1178 typecode = typemask & 7;
1179 switch (typecode) {
1180 case dh_typecode_i32:
1181 case dh_typecode_s32:
1182 type = TCG_TYPE_I32;
1183 break;
1184 case dh_typecode_i64:
1185 case dh_typecode_s64:
1186 type = TCG_TYPE_I64;
1187 break;
1188 case dh_typecode_ptr:
1189 type = TCG_TYPE_PTR;
1190 break;
Richard Henderson466d3752022-11-11 11:01:13 +10001191 case dh_typecode_i128:
1192 type = TCG_TYPE_I128;
1193 break;
Richard Henderson39004a72022-11-11 10:09:37 +10001194 default:
1195 g_assert_not_reached();
1196 }
1197
1198 switch (type) {
1199 case TCG_TYPE_I32:
1200 switch (TCG_TARGET_CALL_ARG_I32) {
1201 case TCG_CALL_ARG_EVEN:
1202 layout_arg_even(&cum);
1203 /* fall through */
1204 case TCG_CALL_ARG_NORMAL:
1205 layout_arg_1(&cum, info, TCG_CALL_ARG_NORMAL);
1206 break;
1207 case TCG_CALL_ARG_EXTEND:
1208 kind = TCG_CALL_ARG_EXTEND_U + (typecode & 1);
1209 layout_arg_1(&cum, info, kind);
1210 break;
1211 default:
1212 qemu_build_not_reached();
1213 }
1214 break;
1215
1216 case TCG_TYPE_I64:
1217 switch (TCG_TARGET_CALL_ARG_I64) {
1218 case TCG_CALL_ARG_EVEN:
1219 layout_arg_even(&cum);
1220 /* fall through */
1221 case TCG_CALL_ARG_NORMAL:
1222 if (TCG_TARGET_REG_BITS == 32) {
1223 layout_arg_normal_n(&cum, info, 2);
1224 } else {
1225 layout_arg_1(&cum, info, TCG_CALL_ARG_NORMAL);
1226 }
1227 break;
1228 default:
1229 qemu_build_not_reached();
1230 }
1231 break;
1232
Richard Henderson466d3752022-11-11 11:01:13 +10001233 case TCG_TYPE_I128:
Richard Henderson5427a9a2022-10-20 07:54:48 +10001234 switch (TCG_TARGET_CALL_ARG_I128) {
Richard Henderson466d3752022-11-11 11:01:13 +10001235 case TCG_CALL_ARG_EVEN:
1236 layout_arg_even(&cum);
1237 /* fall through */
1238 case TCG_CALL_ARG_NORMAL:
1239 layout_arg_normal_n(&cum, info, 128 / TCG_TARGET_REG_BITS);
1240 break;
Richard Henderson313bdea2022-10-31 09:22:59 +11001241 case TCG_CALL_ARG_BY_REF:
1242 layout_arg_by_ref(&cum, info);
1243 break;
Richard Henderson466d3752022-11-11 11:01:13 +10001244 default:
1245 qemu_build_not_reached();
1246 }
1247 break;
1248
Richard Henderson39004a72022-11-11 10:09:37 +10001249 default:
1250 g_assert_not_reached();
1251 }
1252 }
1253 info->nr_in = cum.info_in_idx;
1254
1255 /* Validate that we didn't overrun the input array. */
1256 assert(cum.info_in_idx <= ARRAY_SIZE(info->in));
1257 /* Validate the backend has enough argument space. */
1258 assert(cum.arg_slot <= max_reg_slots + max_stk_slots);
Richard Henderson313bdea2022-10-31 09:22:59 +11001259
1260 /*
1261 * Relocate the "ref_slot" area to the end of the parameters.
1262 * Minimizing this stack offset helps code size for x86,
1263 * which has a signed 8-bit offset encoding.
1264 */
1265 if (cum.ref_slot != 0) {
1266 int ref_base = 0;
1267
1268 if (cum.arg_slot > max_reg_slots) {
1269 int align = __alignof(Int128) / sizeof(tcg_target_long);
1270
1271 ref_base = cum.arg_slot - max_reg_slots;
1272 if (align > 1) {
1273 ref_base = ROUND_UP(ref_base, align);
1274 }
1275 }
1276 assert(ref_base + cum.ref_slot <= max_stk_slots);
Richard Hendersond78e4a42023-04-08 19:05:10 -07001277 ref_base += max_reg_slots;
Richard Henderson313bdea2022-10-31 09:22:59 +11001278
1279 if (ref_base != 0) {
1280 for (int i = cum.info_in_idx - 1; i >= 0; --i) {
1281 TCGCallArgumentLoc *loc = &info->in[i];
1282 switch (loc->kind) {
1283 case TCG_CALL_ARG_BY_REF:
1284 case TCG_CALL_ARG_BY_REF_N:
1285 loc->ref_slot += ref_base;
1286 break;
1287 default:
1288 break;
1289 }
1290 }
1291 }
1292 }
Richard Henderson39004a72022-11-11 10:09:37 +10001293}
1294
Richard Henderson91478ce2015-08-18 23:23:08 -07001295static int indirect_reg_alloc_order[ARRAY_SIZE(tcg_target_reg_alloc_order)];
Richard Hendersonf69d2772016-11-18 09:31:40 +01001296static void process_op_defs(TCGContext *s);
Richard Henderson1c2adb92017-10-10 14:34:37 -07001297static TCGTemp *tcg_global_reg_new_internal(TCGContext *s, TCGType type,
1298 TCGReg reg, const char *name);
Richard Henderson91478ce2015-08-18 23:23:08 -07001299
Richard Henderson43b972b2021-03-09 22:52:45 -06001300static void tcg_context_init(unsigned max_cpus)
bellardc896fe22008-02-01 10:05:41 +00001301{
Richard Hendersona76aabd2021-03-09 17:24:33 -06001302 TCGContext *s = &tcg_init_ctx;
Richard Henderson100b5e02013-09-14 15:57:22 -07001303 int op, total_args, n, i;
bellardc896fe22008-02-01 10:05:41 +00001304 TCGOpDef *def;
1305 TCGArgConstraint *args_ct;
Richard Henderson1c2adb92017-10-10 14:34:37 -07001306 TCGTemp *ts;
bellardc896fe22008-02-01 10:05:41 +00001307
1308 memset(s, 0, sizeof(*s));
bellardc896fe22008-02-01 10:05:41 +00001309 s->nb_globals = 0;
Richard Hendersonc70fbf02016-06-23 20:34:22 -07001310
bellardc896fe22008-02-01 10:05:41 +00001311 /* Count total number of arguments and allocate the corresponding
1312 space */
1313 total_args = 0;
1314 for(op = 0; op < NB_OPS; op++) {
1315 def = &tcg_op_defs[op];
1316 n = def->nb_iargs + def->nb_oargs;
1317 total_args += n;
1318 }
1319
Richard Hendersonbc2b17e2019-04-04 19:34:19 -07001320 args_ct = g_new0(TCGArgConstraint, total_args);
bellardc896fe22008-02-01 10:05:41 +00001321
1322 for(op = 0; op < NB_OPS; op++) {
1323 def = &tcg_op_defs[op];
1324 def->args_ct = args_ct;
bellardc896fe22008-02-01 10:05:41 +00001325 n = def->nb_iargs + def->nb_oargs;
bellardc896fe22008-02-01 10:05:41 +00001326 args_ct += n;
1327 }
Richard Henderson5cd8f622013-09-14 15:09:39 -07001328
Richard Henderson8429a1c2023-04-09 22:59:09 -07001329 init_call_layout(&info_helper_ld32_mmu);
1330 init_call_layout(&info_helper_ld64_mmu);
Richard Hendersonebebea52023-04-17 10:20:51 +02001331 init_call_layout(&info_helper_ld128_mmu);
Richard Henderson8429a1c2023-04-09 22:59:09 -07001332 init_call_layout(&info_helper_st32_mmu);
1333 init_call_layout(&info_helper_st64_mmu);
Richard Hendersonebebea52023-04-17 10:20:51 +02001334 init_call_layout(&info_helper_st128_mmu);
Richard Henderson8429a1c2023-04-09 22:59:09 -07001335
bellardc896fe22008-02-01 10:05:41 +00001336 tcg_target_init(s);
Richard Hendersonf69d2772016-11-18 09:31:40 +01001337 process_op_defs(s);
Richard Henderson91478ce2015-08-18 23:23:08 -07001338
1339 /* Reverse the order of the saved registers, assuming they're all at
1340 the start of tcg_target_reg_alloc_order. */
1341 for (n = 0; n < ARRAY_SIZE(tcg_target_reg_alloc_order); ++n) {
1342 int r = tcg_target_reg_alloc_order[n];
1343 if (tcg_regset_test_reg(tcg_target_call_clobber_regs, r)) {
1344 break;
1345 }
1346 }
1347 for (i = 0; i < n; ++i) {
1348 indirect_reg_alloc_order[i] = tcg_target_reg_alloc_order[n - 1 - i];
1349 }
1350 for (; i < ARRAY_SIZE(tcg_target_reg_alloc_order); ++i) {
1351 indirect_reg_alloc_order[i] = tcg_target_reg_alloc_order[i];
1352 }
Emilio G. Cotab1311c42017-07-12 17:15:52 -04001353
1354 tcg_ctx = s;
Emilio G. Cota3468b592017-07-19 18:57:58 -04001355 /*
1356 * In user-mode we simply share the init context among threads, since we
1357 * use a single region. See the documentation tcg_region_init() for the
1358 * reasoning behind this.
Philippe Mathieu-Daudé7893e422023-10-04 11:06:20 +02001359 * In system-mode we will have at most max_cpus TCG threads.
Emilio G. Cota3468b592017-07-19 18:57:58 -04001360 */
1361#ifdef CONFIG_USER_ONLY
Emilio G. Cotadf2cce22017-07-12 18:26:40 -04001362 tcg_ctxs = &tcg_ctx;
Richard Henderson0e2d61c2021-03-09 23:06:32 -06001363 tcg_cur_ctxs = 1;
1364 tcg_max_ctxs = 1;
Emilio G. Cota3468b592017-07-19 18:57:58 -04001365#else
Richard Henderson0e2d61c2021-03-09 23:06:32 -06001366 tcg_max_ctxs = max_cpus;
1367 tcg_ctxs = g_new0(TCGContext *, max_cpus);
Emilio G. Cota3468b592017-07-19 18:57:58 -04001368#endif
Richard Henderson1c2adb92017-10-10 14:34:37 -07001369
1370 tcg_debug_assert(!tcg_regset_test_reg(s->reserved_regs, TCG_AREG0));
1371 ts = tcg_global_reg_new_internal(s, TCG_TYPE_PTR, TCG_AREG0, "env");
Richard Hendersonad75a512023-09-13 16:37:36 -07001372 tcg_env = temp_tcgv_ptr(ts);
Richard Henderson9002ec72010-05-06 08:50:41 -07001373}
bellardb03cce82008-05-10 10:52:05 +00001374
Richard Henderson43b972b2021-03-09 22:52:45 -06001375void tcg_init(size_t tb_size, int splitwx, unsigned max_cpus)
Richard Hendersona76aabd2021-03-09 17:24:33 -06001376{
Richard Henderson43b972b2021-03-09 22:52:45 -06001377 tcg_context_init(max_cpus);
1378 tcg_region_init(tb_size, splitwx, max_cpus);
Richard Hendersona76aabd2021-03-09 17:24:33 -06001379}
1380
Emilio G. Cota6e3b2bf2017-06-06 19:12:25 -04001381/*
1382 * Allocate TBs right before their corresponding translated code, making
1383 * sure that TBs and code are on different cache lines.
1384 */
1385TranslationBlock *tcg_tb_alloc(TCGContext *s)
1386{
1387 uintptr_t align = qemu_icache_linesize;
1388 TranslationBlock *tb;
1389 void *next;
1390
Emilio G. Cotae8feb962017-07-07 19:24:20 -04001391 retry:
Emilio G. Cota6e3b2bf2017-06-06 19:12:25 -04001392 tb = (void *)ROUND_UP((uintptr_t)s->code_gen_ptr, align);
1393 next = (void *)ROUND_UP((uintptr_t)(tb + 1), align);
1394
1395 if (unlikely(next > s->code_gen_highwater)) {
Emilio G. Cotae8feb962017-07-07 19:24:20 -04001396 if (tcg_region_alloc(s)) {
1397 return NULL;
1398 }
1399 goto retry;
Emilio G. Cota6e3b2bf2017-06-06 19:12:25 -04001400 }
Stefan Hajnoczid73415a2020-09-23 11:56:46 +01001401 qatomic_set(&s->code_gen_ptr, next);
Emilio G. Cota6e3b2bf2017-06-06 19:12:25 -04001402 return tb;
1403}
1404
Richard Henderson935f75a2023-09-29 19:35:26 -07001405void tcg_prologue_init(void)
Richard Henderson9002ec72010-05-06 08:50:41 -07001406{
Richard Henderson935f75a2023-09-29 19:35:26 -07001407 TCGContext *s = tcg_ctx;
Richard Hendersonb0a07942021-03-09 08:45:58 -08001408 size_t prologue_size;
Richard Henderson8163b742015-09-18 23:43:05 -07001409
Richard Hendersonb0a07942021-03-09 08:45:58 -08001410 s->code_ptr = s->code_gen_ptr;
1411 s->code_buf = s->code_gen_ptr;
Richard Henderson5b38ee32017-10-25 07:14:20 -07001412 s->data_gen_ptr = NULL;
Richard Hendersonb91ccb32020-10-28 14:11:54 -07001413
1414#ifndef CONFIG_TCG_INTERPRETER
Richard Hendersonb0a07942021-03-09 08:45:58 -08001415 tcg_qemu_tb_exec = (tcg_prologue_fn *)tcg_splitwx_to_rx(s->code_ptr);
Richard Hendersonb91ccb32020-10-28 14:11:54 -07001416#endif
Richard Henderson8163b742015-09-18 23:43:05 -07001417
Richard Henderson5b38ee32017-10-25 07:14:20 -07001418#ifdef TCG_TARGET_NEED_POOL_LABELS
1419 s->pool_labels = NULL;
1420#endif
1421
Roman Bolshakov653b87e2021-01-13 06:28:07 +03001422 qemu_thread_jit_write();
Richard Henderson8163b742015-09-18 23:43:05 -07001423 /* Generate the prologue. */
bellardb03cce82008-05-10 10:52:05 +00001424 tcg_target_qemu_prologue(s);
Richard Henderson5b38ee32017-10-25 07:14:20 -07001425
1426#ifdef TCG_TARGET_NEED_POOL_LABELS
1427 /* Allow the prologue to put e.g. guest_base into a pool entry. */
1428 {
Richard Henderson17689872019-04-21 13:51:56 -07001429 int result = tcg_out_pool_finalize(s);
1430 tcg_debug_assert(result == 0);
Richard Henderson5b38ee32017-10-25 07:14:20 -07001431 }
1432#endif
1433
Richard Hendersonb0a07942021-03-09 08:45:58 -08001434 prologue_size = tcg_current_code_size(s);
Ilya Leoshkevich5584e2d2023-01-12 16:20:13 +01001435 perf_report_prologue(s->code_gen_ptr, prologue_size);
Richard Hendersonb0a07942021-03-09 08:45:58 -08001436
Richard Hendersondf5d2b12020-12-12 09:08:02 -06001437#ifndef CONFIG_TCG_INTERPRETER
Richard Hendersonb0a07942021-03-09 08:45:58 -08001438 flush_idcache_range((uintptr_t)tcg_splitwx_to_rx(s->code_buf),
1439 (uintptr_t)s->code_buf, prologue_size);
Richard Hendersondf5d2b12020-12-12 09:08:02 -06001440#endif
Richard Henderson8163b742015-09-18 23:43:05 -07001441
Richard Hendersond6b64b22013-03-31 13:15:19 -07001442 if (qemu_loglevel_mask(CPU_LOG_TB_OUT_ASM)) {
Richard Hendersonc60f5992022-04-17 11:29:47 -07001443 FILE *logfile = qemu_log_trylock();
Richard Henderson78b54852022-04-17 11:29:49 -07001444 if (logfile) {
1445 fprintf(logfile, "PROLOGUE: [size=%zu]\n", prologue_size);
1446 if (s->data_gen_ptr) {
1447 size_t code_size = s->data_gen_ptr - s->code_gen_ptr;
1448 size_t data_size = prologue_size - code_size;
1449 size_t i;
Richard Henderson5b38ee32017-10-25 07:14:20 -07001450
Richard Henderson78b54852022-04-17 11:29:49 -07001451 disas(logfile, s->code_gen_ptr, code_size);
Richard Henderson5b38ee32017-10-25 07:14:20 -07001452
Richard Henderson78b54852022-04-17 11:29:49 -07001453 for (i = 0; i < data_size; i += sizeof(tcg_target_ulong)) {
1454 if (sizeof(tcg_target_ulong) == 8) {
1455 fprintf(logfile,
1456 "0x%08" PRIxPTR ": .quad 0x%016" PRIx64 "\n",
1457 (uintptr_t)s->data_gen_ptr + i,
1458 *(uint64_t *)(s->data_gen_ptr + i));
1459 } else {
1460 fprintf(logfile,
1461 "0x%08" PRIxPTR ": .long 0x%08x\n",
1462 (uintptr_t)s->data_gen_ptr + i,
1463 *(uint32_t *)(s->data_gen_ptr + i));
1464 }
Richard Henderson5b38ee32017-10-25 07:14:20 -07001465 }
Richard Henderson78b54852022-04-17 11:29:49 -07001466 } else {
1467 disas(logfile, s->code_gen_ptr, prologue_size);
Richard Henderson5b38ee32017-10-25 07:14:20 -07001468 }
Richard Henderson78b54852022-04-17 11:29:49 -07001469 fprintf(logfile, "\n");
Richard Henderson78b54852022-04-17 11:29:49 -07001470 qemu_log_unlock(logfile);
Richard Henderson5b38ee32017-10-25 07:14:20 -07001471 }
Richard Hendersond6b64b22013-03-31 13:15:19 -07001472 }
Emilio G. Cotacedbcb02017-04-26 23:29:14 -04001473
Richard Henderson6eea0432021-02-02 09:40:22 -10001474#ifndef CONFIG_TCG_INTERPRETER
1475 /*
1476 * Assert that goto_ptr is implemented completely, setting an epilogue.
1477 * For tci, we use NULL as the signal to return from the interpreter,
1478 * so skip this check.
1479 */
Richard Hendersonf4e01e32021-06-29 14:47:39 -07001480 tcg_debug_assert(tcg_code_gen_epilogue != NULL);
Richard Henderson6eea0432021-02-02 09:40:22 -10001481#endif
Richard Hendersond1c74ab2021-07-09 19:45:42 -07001482
1483 tcg_region_prologue_set(s);
bellardc896fe22008-02-01 10:05:41 +00001484}
1485
bellardc896fe22008-02-01 10:05:41 +00001486void tcg_func_start(TCGContext *s)
1487{
1488 tcg_pool_reset(s);
1489 s->nb_temps = s->nb_globals;
Richard Henderson0ec9eab2013-09-19 12:16:45 -07001490
1491 /* No temps have been previously allocated for size or locality. */
Richard Henderson04e006a2024-12-07 18:14:45 +00001492 tcg_temp_ebb_reset_freed(s);
Richard Henderson0ec9eab2013-09-19 12:16:45 -07001493
Richard Hendersonc0522132020-03-29 18:55:52 -07001494 /* No constant temps have been previously allocated. */
1495 for (int i = 0; i < TCG_TYPE_COUNT; ++i) {
1496 if (s->const_table[i]) {
1497 g_hash_table_remove_all(s->const_table[i]);
1498 }
1499 }
1500
Richard Hendersonabebf922018-05-08 19:18:59 +00001501 s->nb_ops = 0;
bellardc896fe22008-02-01 10:05:41 +00001502 s->nb_labels = 0;
1503 s->current_frame_offset = s->frame_start;
1504
Richard Henderson0a209d42012-09-21 17:18:16 -07001505#ifdef CONFIG_DEBUG_TCG
1506 s->goto_tb_issue_mask = 0;
1507#endif
1508
Richard Henderson15fa08f2017-11-02 15:19:14 +01001509 QTAILQ_INIT(&s->ops);
1510 QTAILQ_INIT(&s->free_ops);
Richard Henderson07843f72024-03-13 13:32:29 -10001511 s->emit_before_op = NULL;
Richard Hendersonbef16ab2019-02-07 13:26:40 +00001512 QSIMPLEQ_INIT(&s->labels);
Richard Henderson4baf3972023-03-09 17:46:16 -08001513
1514 tcg_debug_assert(s->addr_type == TCG_TYPE_I32 ||
1515 s->addr_type == TCG_TYPE_I64);
Richard Hendersond0a9bb52023-03-27 16:07:15 -07001516
Richard Henderson747bd692023-03-31 21:30:31 -07001517 tcg_debug_assert(s->insn_start_words > 0);
bellardc896fe22008-02-01 10:05:41 +00001518}
1519
Richard Hendersonae30e862021-01-23 12:11:17 -10001520static TCGTemp *tcg_temp_alloc(TCGContext *s)
Richard Henderson7ca4b752013-09-19 08:46:21 -07001521{
1522 int n = s->nb_temps++;
Richard Hendersonae30e862021-01-23 12:11:17 -10001523
1524 if (n >= TCG_MAX_TEMPS) {
Richard Hendersondb6b7d02021-01-31 23:29:26 -10001525 tcg_raise_tb_overflow(s);
Richard Hendersonae30e862021-01-23 12:11:17 -10001526 }
Richard Henderson7ca4b752013-09-19 08:46:21 -07001527 return memset(&s->temps[n], 0, sizeof(TCGTemp));
1528}
1529
Richard Hendersonae30e862021-01-23 12:11:17 -10001530static TCGTemp *tcg_global_alloc(TCGContext *s)
Richard Henderson7ca4b752013-09-19 08:46:21 -07001531{
Richard Hendersonfa477d22016-11-02 11:20:15 -06001532 TCGTemp *ts;
1533
Richard Henderson7ca4b752013-09-19 08:46:21 -07001534 tcg_debug_assert(s->nb_globals == s->nb_temps);
Richard Hendersonae30e862021-01-23 12:11:17 -10001535 tcg_debug_assert(s->nb_globals < TCG_MAX_TEMPS);
Richard Henderson7ca4b752013-09-19 08:46:21 -07001536 s->nb_globals++;
Richard Hendersonfa477d22016-11-02 11:20:15 -06001537 ts = tcg_temp_alloc(s);
Richard Hendersonee17db82020-03-29 10:11:56 -07001538 ts->kind = TEMP_GLOBAL;
Richard Hendersonfa477d22016-11-02 11:20:15 -06001539
1540 return ts;
bellardc896fe22008-02-01 10:05:41 +00001541}
1542
Richard Henderson085272b2017-10-20 00:05:45 -07001543static TCGTemp *tcg_global_reg_new_internal(TCGContext *s, TCGType type,
1544 TCGReg reg, const char *name)
bellardc896fe22008-02-01 10:05:41 +00001545{
bellardc896fe22008-02-01 10:05:41 +00001546 TCGTemp *ts;
bellardc896fe22008-02-01 10:05:41 +00001547
Richard Henderson1a057552023-04-05 12:08:46 -07001548 tcg_debug_assert(TCG_TARGET_REG_BITS == 64 || type == TCG_TYPE_I32);
Richard Henderson7ca4b752013-09-19 08:46:21 -07001549
1550 ts = tcg_global_alloc(s);
bellardc896fe22008-02-01 10:05:41 +00001551 ts->base_type = type;
1552 ts->type = type;
Richard Hendersonee17db82020-03-29 10:11:56 -07001553 ts->kind = TEMP_FIXED;
bellardc896fe22008-02-01 10:05:41 +00001554 ts->reg = reg;
bellardc896fe22008-02-01 10:05:41 +00001555 ts->name = name;
bellardc896fe22008-02-01 10:05:41 +00001556 tcg_regset_set_reg(s->reserved_regs, reg);
Richard Henderson7ca4b752013-09-19 08:46:21 -07001557
Richard Henderson085272b2017-10-20 00:05:45 -07001558 return ts;
pbrooka7812ae2008-11-17 14:43:54 +00001559}
1560
Richard Hendersonb6638662013-09-18 14:54:45 -07001561void tcg_set_frame(TCGContext *s, TCGReg reg, intptr_t start, intptr_t size)
pbrooka7812ae2008-11-17 14:43:54 +00001562{
Richard Hendersonb3a62932013-09-18 14:12:53 -07001563 s->frame_start = start;
1564 s->frame_end = start + size;
Richard Henderson085272b2017-10-20 00:05:45 -07001565 s->frame_temp
1566 = tcg_global_reg_new_internal(s, TCG_TYPE_PTR, reg, "_frame");
Richard Hendersonb3a62932013-09-18 14:12:53 -07001567}
pbrooka7812ae2008-11-17 14:43:54 +00001568
Richard Henderson4643f3e2023-10-29 14:08:47 -07001569static TCGTemp *tcg_global_mem_new_internal(TCGv_ptr base, intptr_t offset,
1570 const char *name, TCGType type)
bellardc896fe22008-02-01 10:05:41 +00001571{
Emilio G. Cotab1311c42017-07-12 17:15:52 -04001572 TCGContext *s = tcg_ctx;
Richard Hendersondc41aa72017-10-20 00:30:24 -07001573 TCGTemp *base_ts = tcgv_ptr_temp(base);
Richard Henderson7ca4b752013-09-19 08:46:21 -07001574 TCGTemp *ts = tcg_global_alloc(s);
Richard Hendersonaef85402022-10-19 11:53:27 +10001575 int indirect_reg = 0;
bellardc896fe22008-02-01 10:05:41 +00001576
Richard Hendersonc0522132020-03-29 18:55:52 -07001577 switch (base_ts->kind) {
1578 case TEMP_FIXED:
1579 break;
1580 case TEMP_GLOBAL:
Richard Henderson5a184072016-06-23 20:34:33 -07001581 /* We do not support double-indirect registers. */
1582 tcg_debug_assert(!base_ts->indirect_reg);
Richard Hendersonb3915db2013-09-19 10:36:18 -07001583 base_ts->indirect_base = 1;
Richard Henderson5a184072016-06-23 20:34:33 -07001584 s->nb_indirects += (TCG_TARGET_REG_BITS == 32 && type == TCG_TYPE_I64
1585 ? 2 : 1);
1586 indirect_reg = 1;
Richard Hendersonc0522132020-03-29 18:55:52 -07001587 break;
1588 default:
1589 g_assert_not_reached();
Richard Hendersonb3915db2013-09-19 10:36:18 -07001590 }
1591
Richard Henderson7ca4b752013-09-19 08:46:21 -07001592 if (TCG_TARGET_REG_BITS == 32 && type == TCG_TYPE_I64) {
1593 TCGTemp *ts2 = tcg_global_alloc(s);
bellardc896fe22008-02-01 10:05:41 +00001594 char buf[64];
Richard Henderson7ca4b752013-09-19 08:46:21 -07001595
1596 ts->base_type = TCG_TYPE_I64;
bellardc896fe22008-02-01 10:05:41 +00001597 ts->type = TCG_TYPE_I32;
Richard Hendersonb3915db2013-09-19 10:36:18 -07001598 ts->indirect_reg = indirect_reg;
bellardc896fe22008-02-01 10:05:41 +00001599 ts->mem_allocated = 1;
Richard Hendersonb3a62932013-09-18 14:12:53 -07001600 ts->mem_base = base_ts;
Richard Hendersonaef85402022-10-19 11:53:27 +10001601 ts->mem_offset = offset;
bellardc896fe22008-02-01 10:05:41 +00001602 pstrcpy(buf, sizeof(buf), name);
1603 pstrcat(buf, sizeof(buf), "_0");
1604 ts->name = strdup(buf);
bellardc896fe22008-02-01 10:05:41 +00001605
Richard Henderson7ca4b752013-09-19 08:46:21 -07001606 tcg_debug_assert(ts2 == ts + 1);
1607 ts2->base_type = TCG_TYPE_I64;
1608 ts2->type = TCG_TYPE_I32;
Richard Hendersonb3915db2013-09-19 10:36:18 -07001609 ts2->indirect_reg = indirect_reg;
Richard Henderson7ca4b752013-09-19 08:46:21 -07001610 ts2->mem_allocated = 1;
1611 ts2->mem_base = base_ts;
Richard Hendersonaef85402022-10-19 11:53:27 +10001612 ts2->mem_offset = offset + 4;
Richard Hendersonfac87bd2022-10-19 11:26:37 +10001613 ts2->temp_subindex = 1;
bellardc896fe22008-02-01 10:05:41 +00001614 pstrcpy(buf, sizeof(buf), name);
1615 pstrcat(buf, sizeof(buf), "_1");
Richard Henderson120c1082016-06-17 17:02:20 -07001616 ts2->name = strdup(buf);
Richard Henderson7ca4b752013-09-19 08:46:21 -07001617 } else {
bellardc896fe22008-02-01 10:05:41 +00001618 ts->base_type = type;
1619 ts->type = type;
Richard Hendersonb3915db2013-09-19 10:36:18 -07001620 ts->indirect_reg = indirect_reg;
bellardc896fe22008-02-01 10:05:41 +00001621 ts->mem_allocated = 1;
Richard Hendersonb3a62932013-09-18 14:12:53 -07001622 ts->mem_base = base_ts;
bellardc896fe22008-02-01 10:05:41 +00001623 ts->mem_offset = offset;
bellardc896fe22008-02-01 10:05:41 +00001624 ts->name = name;
bellardc896fe22008-02-01 10:05:41 +00001625 }
Richard Henderson085272b2017-10-20 00:05:45 -07001626 return ts;
bellardc896fe22008-02-01 10:05:41 +00001627}
1628
Richard Henderson4643f3e2023-10-29 14:08:47 -07001629TCGv_i32 tcg_global_mem_new_i32(TCGv_ptr reg, intptr_t off, const char *name)
1630{
1631 TCGTemp *ts = tcg_global_mem_new_internal(reg, off, name, TCG_TYPE_I32);
1632 return temp_tcgv_i32(ts);
1633}
1634
1635TCGv_i64 tcg_global_mem_new_i64(TCGv_ptr reg, intptr_t off, const char *name)
1636{
1637 TCGTemp *ts = tcg_global_mem_new_internal(reg, off, name, TCG_TYPE_I64);
1638 return temp_tcgv_i64(ts);
1639}
1640
1641TCGv_ptr tcg_global_mem_new_ptr(TCGv_ptr reg, intptr_t off, const char *name)
1642{
1643 TCGTemp *ts = tcg_global_mem_new_internal(reg, off, name, TCG_TYPE_PTR);
1644 return temp_tcgv_ptr(ts);
1645}
1646
Richard Hendersonfb04ab72024-01-10 18:21:58 +11001647TCGTemp *tcg_temp_new_internal(TCGType type, TCGTempKind kind)
bellardc896fe22008-02-01 10:05:41 +00001648{
Emilio G. Cotab1311c42017-07-12 17:15:52 -04001649 TCGContext *s = tcg_ctx;
bellardc896fe22008-02-01 10:05:41 +00001650 TCGTemp *ts;
Richard Hendersone1c08b02023-01-29 14:02:59 -10001651 int n;
bellardc896fe22008-02-01 10:05:41 +00001652
Richard Hendersone1c08b02023-01-29 14:02:59 -10001653 if (kind == TEMP_EBB) {
1654 int idx = find_first_bit(s->free_temps[type].l, TCG_MAX_TEMPS);
Richard Henderson0ec9eab2013-09-19 12:16:45 -07001655
Richard Hendersone1c08b02023-01-29 14:02:59 -10001656 if (idx < TCG_MAX_TEMPS) {
1657 /* There is already an available temp with the right type. */
1658 clear_bit(idx, s->free_temps[type].l);
Richard Henderson43eef722022-10-20 08:03:41 +10001659
Richard Hendersone1c08b02023-01-29 14:02:59 -10001660 ts = &s->temps[idx];
1661 ts->temp_allocated = 1;
1662 tcg_debug_assert(ts->base_type == type);
1663 tcg_debug_assert(ts->kind == kind);
Richard Henderson2f2e9112023-02-24 16:15:18 -10001664 return ts;
Richard Henderson43eef722022-10-20 08:03:41 +10001665 }
Richard Hendersone1c08b02023-01-29 14:02:59 -10001666 } else {
1667 tcg_debug_assert(kind == TEMP_TB);
1668 }
Richard Henderson43eef722022-10-20 08:03:41 +10001669
Richard Hendersone1c08b02023-01-29 14:02:59 -10001670 switch (type) {
1671 case TCG_TYPE_I32:
1672 case TCG_TYPE_V64:
1673 case TCG_TYPE_V128:
1674 case TCG_TYPE_V256:
1675 n = 1;
1676 break;
1677 case TCG_TYPE_I64:
1678 n = 64 / TCG_TARGET_REG_BITS;
1679 break;
1680 case TCG_TYPE_I128:
1681 n = 128 / TCG_TARGET_REG_BITS;
1682 break;
1683 default:
1684 g_assert_not_reached();
1685 }
Richard Henderson7ca4b752013-09-19 08:46:21 -07001686
Richard Hendersone1c08b02023-01-29 14:02:59 -10001687 ts = tcg_temp_alloc(s);
1688 ts->base_type = type;
1689 ts->temp_allocated = 1;
1690 ts->kind = kind;
Richard Henderson43eef722022-10-20 08:03:41 +10001691
Richard Hendersone1c08b02023-01-29 14:02:59 -10001692 if (n == 1) {
1693 ts->type = type;
1694 } else {
1695 ts->type = TCG_TYPE_REG;
Richard Henderson43eef722022-10-20 08:03:41 +10001696
Richard Hendersone1c08b02023-01-29 14:02:59 -10001697 for (int i = 1; i < n; ++i) {
1698 TCGTemp *ts2 = tcg_temp_alloc(s);
1699
1700 tcg_debug_assert(ts2 == ts + i);
1701 ts2->base_type = type;
1702 ts2->type = TCG_TYPE_REG;
1703 ts2->temp_allocated = 1;
1704 ts2->temp_subindex = i;
1705 ts2->kind = kind;
bellarde8996ee2008-05-23 17:33:39 +00001706 }
bellardc896fe22008-02-01 10:05:41 +00001707 }
Richard Henderson085272b2017-10-20 00:05:45 -07001708 return ts;
bellardc896fe22008-02-01 10:05:41 +00001709}
1710
Richard Henderson4643f3e2023-10-29 14:08:47 -07001711TCGv_i32 tcg_temp_new_i32(void)
1712{
1713 return temp_tcgv_i32(tcg_temp_new_internal(TCG_TYPE_I32, TEMP_TB));
1714}
1715
1716TCGv_i32 tcg_temp_ebb_new_i32(void)
1717{
1718 return temp_tcgv_i32(tcg_temp_new_internal(TCG_TYPE_I32, TEMP_EBB));
1719}
1720
1721TCGv_i64 tcg_temp_new_i64(void)
1722{
1723 return temp_tcgv_i64(tcg_temp_new_internal(TCG_TYPE_I64, TEMP_TB));
1724}
1725
1726TCGv_i64 tcg_temp_ebb_new_i64(void)
1727{
1728 return temp_tcgv_i64(tcg_temp_new_internal(TCG_TYPE_I64, TEMP_EBB));
1729}
1730
1731TCGv_ptr tcg_temp_new_ptr(void)
1732{
1733 return temp_tcgv_ptr(tcg_temp_new_internal(TCG_TYPE_PTR, TEMP_TB));
1734}
1735
1736TCGv_ptr tcg_temp_ebb_new_ptr(void)
1737{
1738 return temp_tcgv_ptr(tcg_temp_new_internal(TCG_TYPE_PTR, TEMP_EBB));
1739}
1740
1741TCGv_i128 tcg_temp_new_i128(void)
1742{
1743 return temp_tcgv_i128(tcg_temp_new_internal(TCG_TYPE_I128, TEMP_TB));
1744}
1745
1746TCGv_i128 tcg_temp_ebb_new_i128(void)
1747{
1748 return temp_tcgv_i128(tcg_temp_new_internal(TCG_TYPE_I128, TEMP_EBB));
1749}
1750
Richard Hendersond2fd7452017-09-14 13:53:46 -07001751TCGv_vec tcg_temp_new_vec(TCGType type)
1752{
1753 TCGTemp *t;
1754
1755#ifdef CONFIG_DEBUG_TCG
1756 switch (type) {
1757 case TCG_TYPE_V64:
1758 assert(TCG_TARGET_HAS_v64);
1759 break;
1760 case TCG_TYPE_V128:
1761 assert(TCG_TARGET_HAS_v128);
1762 break;
1763 case TCG_TYPE_V256:
1764 assert(TCG_TARGET_HAS_v256);
1765 break;
1766 default:
1767 g_assert_not_reached();
1768 }
1769#endif
1770
Richard Hendersonbbf989b2023-01-29 13:46:06 -10001771 t = tcg_temp_new_internal(type, TEMP_EBB);
Richard Hendersond2fd7452017-09-14 13:53:46 -07001772 return temp_tcgv_vec(t);
1773}
1774
1775/* Create a new temp of the same type as an existing temp. */
1776TCGv_vec tcg_temp_new_vec_matching(TCGv_vec match)
1777{
1778 TCGTemp *t = tcgv_vec_temp(match);
1779
1780 tcg_debug_assert(t->temp_allocated != 0);
1781
Richard Hendersonbbf989b2023-01-29 13:46:06 -10001782 t = tcg_temp_new_internal(t->base_type, TEMP_EBB);
Richard Hendersond2fd7452017-09-14 13:53:46 -07001783 return temp_tcgv_vec(t);
1784}
1785
Richard Henderson5bfa8032018-02-22 18:17:57 -08001786void tcg_temp_free_internal(TCGTemp *ts)
bellardc896fe22008-02-01 10:05:41 +00001787{
Emilio G. Cotab1311c42017-07-12 17:15:52 -04001788 TCGContext *s = tcg_ctx;
bellardc896fe22008-02-01 10:05:41 +00001789
Richard Hendersonc7482432022-03-16 09:34:18 -07001790 switch (ts->kind) {
1791 case TEMP_CONST:
Richard Hendersonf57c6912023-01-29 10:55:52 -10001792 case TEMP_TB:
Richard Henderson2f2e9112023-02-24 16:15:18 -10001793 /* Silently ignore free. */
1794 break;
1795 case TEMP_EBB:
1796 tcg_debug_assert(ts->temp_allocated != 0);
1797 ts->temp_allocated = 0;
1798 set_bit(temp_idx(ts), s->free_temps[ts->base_type].l);
Richard Hendersonc7482432022-03-16 09:34:18 -07001799 break;
1800 default:
Richard Henderson2f2e9112023-02-24 16:15:18 -10001801 /* It never made sense to free TEMP_FIXED or TEMP_GLOBAL. */
Richard Hendersonc7482432022-03-16 09:34:18 -07001802 g_assert_not_reached();
Richard Hendersonc0522132020-03-29 18:55:52 -07001803 }
bellarde8996ee2008-05-23 17:33:39 +00001804}
1805
Richard Henderson58b79712023-10-29 14:08:48 -07001806void tcg_temp_free_i32(TCGv_i32 arg)
1807{
1808 tcg_temp_free_internal(tcgv_i32_temp(arg));
1809}
1810
1811void tcg_temp_free_i64(TCGv_i64 arg)
1812{
1813 tcg_temp_free_internal(tcgv_i64_temp(arg));
1814}
1815
1816void tcg_temp_free_i128(TCGv_i128 arg)
1817{
1818 tcg_temp_free_internal(tcgv_i128_temp(arg));
1819}
1820
1821void tcg_temp_free_ptr(TCGv_ptr arg)
1822{
1823 tcg_temp_free_internal(tcgv_ptr_temp(arg));
1824}
1825
1826void tcg_temp_free_vec(TCGv_vec arg)
1827{
1828 tcg_temp_free_internal(tcgv_vec_temp(arg));
1829}
1830
Richard Hendersonc0522132020-03-29 18:55:52 -07001831TCGTemp *tcg_constant_internal(TCGType type, int64_t val)
1832{
1833 TCGContext *s = tcg_ctx;
1834 GHashTable *h = s->const_table[type];
1835 TCGTemp *ts;
1836
1837 if (h == NULL) {
1838 h = g_hash_table_new(g_int64_hash, g_int64_equal);
1839 s->const_table[type] = h;
1840 }
1841
1842 ts = g_hash_table_lookup(h, &val);
1843 if (ts == NULL) {
Richard Hendersonaef85402022-10-19 11:53:27 +10001844 int64_t *val_ptr;
1845
Richard Hendersonc0522132020-03-29 18:55:52 -07001846 ts = tcg_temp_alloc(s);
1847
1848 if (TCG_TARGET_REG_BITS == 32 && type == TCG_TYPE_I64) {
1849 TCGTemp *ts2 = tcg_temp_alloc(s);
1850
Richard Hendersonaef85402022-10-19 11:53:27 +10001851 tcg_debug_assert(ts2 == ts + 1);
1852
Richard Hendersonc0522132020-03-29 18:55:52 -07001853 ts->base_type = TCG_TYPE_I64;
1854 ts->type = TCG_TYPE_I32;
1855 ts->kind = TEMP_CONST;
1856 ts->temp_allocated = 1;
Richard Hendersonc0522132020-03-29 18:55:52 -07001857
Richard Hendersonc0522132020-03-29 18:55:52 -07001858 ts2->base_type = TCG_TYPE_I64;
1859 ts2->type = TCG_TYPE_I32;
1860 ts2->kind = TEMP_CONST;
1861 ts2->temp_allocated = 1;
Richard Hendersonfac87bd2022-10-19 11:26:37 +10001862 ts2->temp_subindex = 1;
Richard Hendersonaef85402022-10-19 11:53:27 +10001863
1864 /*
1865 * Retain the full value of the 64-bit constant in the low
1866 * part, so that the hash table works. Actual uses will
1867 * truncate the value to the low part.
1868 */
1869 ts[HOST_BIG_ENDIAN].val = val;
1870 ts[!HOST_BIG_ENDIAN].val = val >> 32;
1871 val_ptr = &ts[HOST_BIG_ENDIAN].val;
Richard Hendersonc0522132020-03-29 18:55:52 -07001872 } else {
1873 ts->base_type = type;
1874 ts->type = type;
1875 ts->kind = TEMP_CONST;
1876 ts->temp_allocated = 1;
1877 ts->val = val;
Richard Hendersonaef85402022-10-19 11:53:27 +10001878 val_ptr = &ts->val;
Richard Hendersonc0522132020-03-29 18:55:52 -07001879 }
Richard Hendersonaef85402022-10-19 11:53:27 +10001880 g_hash_table_insert(h, val_ptr, ts);
Richard Hendersonc0522132020-03-29 18:55:52 -07001881 }
1882
1883 return ts;
1884}
1885
Richard Henderson16edaee2023-10-29 14:08:46 -07001886TCGv_i32 tcg_constant_i32(int32_t val)
1887{
1888 return temp_tcgv_i32(tcg_constant_internal(TCG_TYPE_I32, val));
1889}
1890
1891TCGv_i64 tcg_constant_i64(int64_t val)
1892{
1893 return temp_tcgv_i64(tcg_constant_internal(TCG_TYPE_I64, val));
1894}
1895
1896TCGv_ptr tcg_constant_ptr_int(intptr_t val)
1897{
1898 return temp_tcgv_ptr(tcg_constant_internal(TCG_TYPE_PTR, val));
1899}
1900
Richard Hendersonc0522132020-03-29 18:55:52 -07001901TCGv_vec tcg_constant_vec(TCGType type, unsigned vece, int64_t val)
1902{
1903 val = dup_const(vece, val);
1904 return temp_tcgv_vec(tcg_constant_internal(type, val));
1905}
1906
Richard Henderson88d40052020-09-03 18:18:08 -07001907TCGv_vec tcg_constant_vec_matching(TCGv_vec match, unsigned vece, int64_t val)
1908{
1909 TCGTemp *t = tcgv_vec_temp(match);
1910
1911 tcg_debug_assert(t->temp_allocated != 0);
1912 return tcg_constant_vec(t->base_type, vece, val);
1913}
1914
Richard Henderson177f6482023-03-30 08:09:03 -07001915#ifdef CONFIG_DEBUG_TCG
1916size_t temp_idx(TCGTemp *ts)
1917{
1918 ptrdiff_t n = ts - tcg_ctx->temps;
1919 assert(n >= 0 && n < tcg_ctx->nb_temps);
1920 return n;
1921}
1922
1923TCGTemp *tcgv_i32_temp(TCGv_i32 v)
1924{
1925 uintptr_t o = (uintptr_t)v - offsetof(TCGContext, temps);
1926
1927 assert(o < sizeof(TCGTemp) * tcg_ctx->nb_temps);
1928 assert(o % sizeof(TCGTemp) == 0);
1929
1930 return (void *)tcg_ctx + (uintptr_t)v;
1931}
1932#endif /* CONFIG_DEBUG_TCG */
1933
Richard Hendersonbe0f34b2017-08-17 07:43:20 -07001934/* Return true if OP may appear in the opcode stream.
1935 Test the runtime variable that controls each opcode. */
1936bool tcg_op_supported(TCGOpcode op)
1937{
Richard Hendersond2fd7452017-09-14 13:53:46 -07001938 const bool have_vec
1939 = TCG_TARGET_HAS_v64 | TCG_TARGET_HAS_v128 | TCG_TARGET_HAS_v256;
1940
Richard Hendersonbe0f34b2017-08-17 07:43:20 -07001941 switch (op) {
1942 case INDEX_op_discard:
1943 case INDEX_op_set_label:
1944 case INDEX_op_call:
1945 case INDEX_op_br:
1946 case INDEX_op_mb:
1947 case INDEX_op_insn_start:
1948 case INDEX_op_exit_tb:
1949 case INDEX_op_goto_tb:
Richard Hendersonf4e01e32021-06-29 14:47:39 -07001950 case INDEX_op_goto_ptr:
Richard Hendersonfecccfc2023-05-16 20:07:20 -07001951 case INDEX_op_qemu_ld_a32_i32:
1952 case INDEX_op_qemu_ld_a64_i32:
1953 case INDEX_op_qemu_st_a32_i32:
1954 case INDEX_op_qemu_st_a64_i32:
1955 case INDEX_op_qemu_ld_a32_i64:
1956 case INDEX_op_qemu_ld_a64_i64:
1957 case INDEX_op_qemu_st_a32_i64:
1958 case INDEX_op_qemu_st_a64_i64:
Richard Hendersonbe0f34b2017-08-17 07:43:20 -07001959 return true;
1960
Richard Hendersonfecccfc2023-05-16 20:07:20 -07001961 case INDEX_op_qemu_st8_a32_i32:
1962 case INDEX_op_qemu_st8_a64_i32:
Richard Henderson07ce0b02020-12-09 13:58:39 -06001963 return TCG_TARGET_HAS_qemu_st8_i32;
1964
Richard Hendersonfecccfc2023-05-16 20:07:20 -07001965 case INDEX_op_qemu_ld_a32_i128:
1966 case INDEX_op_qemu_ld_a64_i128:
1967 case INDEX_op_qemu_st_a32_i128:
1968 case INDEX_op_qemu_st_a64_i128:
Richard Henderson12fde9b2022-11-07 10:42:56 +11001969 return TCG_TARGET_HAS_qemu_ldst_i128;
1970
Richard Hendersonbe0f34b2017-08-17 07:43:20 -07001971 case INDEX_op_mov_i32:
Richard Hendersonbe0f34b2017-08-17 07:43:20 -07001972 case INDEX_op_setcond_i32:
1973 case INDEX_op_brcond_i32:
Richard Henderson3871be72023-10-25 21:14:01 -07001974 case INDEX_op_movcond_i32:
Richard Hendersonbe0f34b2017-08-17 07:43:20 -07001975 case INDEX_op_ld8u_i32:
1976 case INDEX_op_ld8s_i32:
1977 case INDEX_op_ld16u_i32:
1978 case INDEX_op_ld16s_i32:
1979 case INDEX_op_ld_i32:
1980 case INDEX_op_st8_i32:
1981 case INDEX_op_st16_i32:
1982 case INDEX_op_st_i32:
1983 case INDEX_op_add_i32:
1984 case INDEX_op_sub_i32:
Richard Hendersonb701f192023-10-25 21:14:04 -07001985 case INDEX_op_neg_i32:
Richard Hendersonbe0f34b2017-08-17 07:43:20 -07001986 case INDEX_op_mul_i32:
1987 case INDEX_op_and_i32:
1988 case INDEX_op_or_i32:
1989 case INDEX_op_xor_i32:
1990 case INDEX_op_shl_i32:
1991 case INDEX_op_shr_i32:
1992 case INDEX_op_sar_i32:
1993 return true;
1994
Richard Henderson36355022023-08-04 23:24:04 +00001995 case INDEX_op_negsetcond_i32:
1996 return TCG_TARGET_HAS_negsetcond_i32;
Richard Hendersonbe0f34b2017-08-17 07:43:20 -07001997 case INDEX_op_div_i32:
1998 case INDEX_op_divu_i32:
1999 return TCG_TARGET_HAS_div_i32;
2000 case INDEX_op_rem_i32:
2001 case INDEX_op_remu_i32:
2002 return TCG_TARGET_HAS_rem_i32;
2003 case INDEX_op_div2_i32:
2004 case INDEX_op_divu2_i32:
2005 return TCG_TARGET_HAS_div2_i32;
2006 case INDEX_op_rotl_i32:
2007 case INDEX_op_rotr_i32:
2008 return TCG_TARGET_HAS_rot_i32;
2009 case INDEX_op_deposit_i32:
2010 return TCG_TARGET_HAS_deposit_i32;
2011 case INDEX_op_extract_i32:
2012 return TCG_TARGET_HAS_extract_i32;
2013 case INDEX_op_sextract_i32:
2014 return TCG_TARGET_HAS_sextract_i32;
Richard Hendersonfce12962019-02-25 10:29:25 -08002015 case INDEX_op_extract2_i32:
2016 return TCG_TARGET_HAS_extract2_i32;
Richard Hendersonbe0f34b2017-08-17 07:43:20 -07002017 case INDEX_op_add2_i32:
2018 return TCG_TARGET_HAS_add2_i32;
2019 case INDEX_op_sub2_i32:
2020 return TCG_TARGET_HAS_sub2_i32;
2021 case INDEX_op_mulu2_i32:
2022 return TCG_TARGET_HAS_mulu2_i32;
2023 case INDEX_op_muls2_i32:
2024 return TCG_TARGET_HAS_muls2_i32;
2025 case INDEX_op_muluh_i32:
2026 return TCG_TARGET_HAS_muluh_i32;
2027 case INDEX_op_mulsh_i32:
2028 return TCG_TARGET_HAS_mulsh_i32;
2029 case INDEX_op_ext8s_i32:
2030 return TCG_TARGET_HAS_ext8s_i32;
2031 case INDEX_op_ext16s_i32:
2032 return TCG_TARGET_HAS_ext16s_i32;
2033 case INDEX_op_ext8u_i32:
2034 return TCG_TARGET_HAS_ext8u_i32;
2035 case INDEX_op_ext16u_i32:
2036 return TCG_TARGET_HAS_ext16u_i32;
2037 case INDEX_op_bswap16_i32:
2038 return TCG_TARGET_HAS_bswap16_i32;
2039 case INDEX_op_bswap32_i32:
2040 return TCG_TARGET_HAS_bswap32_i32;
2041 case INDEX_op_not_i32:
2042 return TCG_TARGET_HAS_not_i32;
Richard Hendersonbe0f34b2017-08-17 07:43:20 -07002043 case INDEX_op_andc_i32:
2044 return TCG_TARGET_HAS_andc_i32;
2045 case INDEX_op_orc_i32:
2046 return TCG_TARGET_HAS_orc_i32;
2047 case INDEX_op_eqv_i32:
2048 return TCG_TARGET_HAS_eqv_i32;
2049 case INDEX_op_nand_i32:
2050 return TCG_TARGET_HAS_nand_i32;
2051 case INDEX_op_nor_i32:
2052 return TCG_TARGET_HAS_nor_i32;
2053 case INDEX_op_clz_i32:
2054 return TCG_TARGET_HAS_clz_i32;
2055 case INDEX_op_ctz_i32:
2056 return TCG_TARGET_HAS_ctz_i32;
2057 case INDEX_op_ctpop_i32:
2058 return TCG_TARGET_HAS_ctpop_i32;
2059
2060 case INDEX_op_brcond2_i32:
2061 case INDEX_op_setcond2_i32:
2062 return TCG_TARGET_REG_BITS == 32;
2063
2064 case INDEX_op_mov_i64:
Richard Hendersonbe0f34b2017-08-17 07:43:20 -07002065 case INDEX_op_setcond_i64:
2066 case INDEX_op_brcond_i64:
Richard Henderson3871be72023-10-25 21:14:01 -07002067 case INDEX_op_movcond_i64:
Richard Hendersonbe0f34b2017-08-17 07:43:20 -07002068 case INDEX_op_ld8u_i64:
2069 case INDEX_op_ld8s_i64:
2070 case INDEX_op_ld16u_i64:
2071 case INDEX_op_ld16s_i64:
2072 case INDEX_op_ld32u_i64:
2073 case INDEX_op_ld32s_i64:
2074 case INDEX_op_ld_i64:
2075 case INDEX_op_st8_i64:
2076 case INDEX_op_st16_i64:
2077 case INDEX_op_st32_i64:
2078 case INDEX_op_st_i64:
2079 case INDEX_op_add_i64:
2080 case INDEX_op_sub_i64:
Richard Hendersonb701f192023-10-25 21:14:04 -07002081 case INDEX_op_neg_i64:
Richard Hendersonbe0f34b2017-08-17 07:43:20 -07002082 case INDEX_op_mul_i64:
2083 case INDEX_op_and_i64:
2084 case INDEX_op_or_i64:
2085 case INDEX_op_xor_i64:
2086 case INDEX_op_shl_i64:
2087 case INDEX_op_shr_i64:
2088 case INDEX_op_sar_i64:
2089 case INDEX_op_ext_i32_i64:
2090 case INDEX_op_extu_i32_i64:
2091 return TCG_TARGET_REG_BITS == 64;
2092
Richard Henderson36355022023-08-04 23:24:04 +00002093 case INDEX_op_negsetcond_i64:
2094 return TCG_TARGET_HAS_negsetcond_i64;
Richard Hendersonbe0f34b2017-08-17 07:43:20 -07002095 case INDEX_op_div_i64:
2096 case INDEX_op_divu_i64:
2097 return TCG_TARGET_HAS_div_i64;
2098 case INDEX_op_rem_i64:
2099 case INDEX_op_remu_i64:
2100 return TCG_TARGET_HAS_rem_i64;
2101 case INDEX_op_div2_i64:
2102 case INDEX_op_divu2_i64:
2103 return TCG_TARGET_HAS_div2_i64;
2104 case INDEX_op_rotl_i64:
2105 case INDEX_op_rotr_i64:
2106 return TCG_TARGET_HAS_rot_i64;
2107 case INDEX_op_deposit_i64:
2108 return TCG_TARGET_HAS_deposit_i64;
2109 case INDEX_op_extract_i64:
2110 return TCG_TARGET_HAS_extract_i64;
2111 case INDEX_op_sextract_i64:
2112 return TCG_TARGET_HAS_sextract_i64;
Richard Hendersonfce12962019-02-25 10:29:25 -08002113 case INDEX_op_extract2_i64:
2114 return TCG_TARGET_HAS_extract2_i64;
Richard Hendersonbe0f34b2017-08-17 07:43:20 -07002115 case INDEX_op_extrl_i64_i32:
Richard Hendersonbe0f34b2017-08-17 07:43:20 -07002116 case INDEX_op_extrh_i64_i32:
Richard Henderson13d885b2023-08-22 10:51:10 -07002117 return TCG_TARGET_HAS_extr_i64_i32;
Richard Hendersonbe0f34b2017-08-17 07:43:20 -07002118 case INDEX_op_ext8s_i64:
2119 return TCG_TARGET_HAS_ext8s_i64;
2120 case INDEX_op_ext16s_i64:
2121 return TCG_TARGET_HAS_ext16s_i64;
2122 case INDEX_op_ext32s_i64:
2123 return TCG_TARGET_HAS_ext32s_i64;
2124 case INDEX_op_ext8u_i64:
2125 return TCG_TARGET_HAS_ext8u_i64;
2126 case INDEX_op_ext16u_i64:
2127 return TCG_TARGET_HAS_ext16u_i64;
2128 case INDEX_op_ext32u_i64:
2129 return TCG_TARGET_HAS_ext32u_i64;
2130 case INDEX_op_bswap16_i64:
2131 return TCG_TARGET_HAS_bswap16_i64;
2132 case INDEX_op_bswap32_i64:
2133 return TCG_TARGET_HAS_bswap32_i64;
2134 case INDEX_op_bswap64_i64:
2135 return TCG_TARGET_HAS_bswap64_i64;
2136 case INDEX_op_not_i64:
2137 return TCG_TARGET_HAS_not_i64;
Richard Hendersonbe0f34b2017-08-17 07:43:20 -07002138 case INDEX_op_andc_i64:
2139 return TCG_TARGET_HAS_andc_i64;
2140 case INDEX_op_orc_i64:
2141 return TCG_TARGET_HAS_orc_i64;
2142 case INDEX_op_eqv_i64:
2143 return TCG_TARGET_HAS_eqv_i64;
2144 case INDEX_op_nand_i64:
2145 return TCG_TARGET_HAS_nand_i64;
2146 case INDEX_op_nor_i64:
2147 return TCG_TARGET_HAS_nor_i64;
2148 case INDEX_op_clz_i64:
2149 return TCG_TARGET_HAS_clz_i64;
2150 case INDEX_op_ctz_i64:
2151 return TCG_TARGET_HAS_ctz_i64;
2152 case INDEX_op_ctpop_i64:
2153 return TCG_TARGET_HAS_ctpop_i64;
2154 case INDEX_op_add2_i64:
2155 return TCG_TARGET_HAS_add2_i64;
2156 case INDEX_op_sub2_i64:
2157 return TCG_TARGET_HAS_sub2_i64;
2158 case INDEX_op_mulu2_i64:
2159 return TCG_TARGET_HAS_mulu2_i64;
2160 case INDEX_op_muls2_i64:
2161 return TCG_TARGET_HAS_muls2_i64;
2162 case INDEX_op_muluh_i64:
2163 return TCG_TARGET_HAS_muluh_i64;
2164 case INDEX_op_mulsh_i64:
2165 return TCG_TARGET_HAS_mulsh_i64;
2166
Richard Hendersond2fd7452017-09-14 13:53:46 -07002167 case INDEX_op_mov_vec:
2168 case INDEX_op_dup_vec:
Richard Henderson37ee55a2019-03-17 01:55:22 +00002169 case INDEX_op_dupm_vec:
Richard Hendersond2fd7452017-09-14 13:53:46 -07002170 case INDEX_op_ld_vec:
2171 case INDEX_op_st_vec:
2172 case INDEX_op_add_vec:
2173 case INDEX_op_sub_vec:
2174 case INDEX_op_and_vec:
2175 case INDEX_op_or_vec:
2176 case INDEX_op_xor_vec:
Richard Henderson212be172017-11-17 20:47:42 +01002177 case INDEX_op_cmp_vec:
Richard Hendersond2fd7452017-09-14 13:53:46 -07002178 return have_vec;
2179 case INDEX_op_dup2_vec:
2180 return have_vec && TCG_TARGET_REG_BITS == 32;
2181 case INDEX_op_not_vec:
2182 return have_vec && TCG_TARGET_HAS_not_vec;
2183 case INDEX_op_neg_vec:
2184 return have_vec && TCG_TARGET_HAS_neg_vec;
Richard Hendersonbcefc902019-04-17 13:53:02 -10002185 case INDEX_op_abs_vec:
2186 return have_vec && TCG_TARGET_HAS_abs_vec;
Richard Hendersond2fd7452017-09-14 13:53:46 -07002187 case INDEX_op_andc_vec:
2188 return have_vec && TCG_TARGET_HAS_andc_vec;
2189 case INDEX_op_orc_vec:
2190 return have_vec && TCG_TARGET_HAS_orc_vec;
Richard Hendersoned523472021-12-16 11:17:46 -08002191 case INDEX_op_nand_vec:
2192 return have_vec && TCG_TARGET_HAS_nand_vec;
2193 case INDEX_op_nor_vec:
2194 return have_vec && TCG_TARGET_HAS_nor_vec;
2195 case INDEX_op_eqv_vec:
2196 return have_vec && TCG_TARGET_HAS_eqv_vec;
Richard Henderson37740302017-11-21 10:11:14 +01002197 case INDEX_op_mul_vec:
2198 return have_vec && TCG_TARGET_HAS_mul_vec;
Richard Hendersond0ec9792017-11-17 14:35:11 +01002199 case INDEX_op_shli_vec:
2200 case INDEX_op_shri_vec:
2201 case INDEX_op_sari_vec:
2202 return have_vec && TCG_TARGET_HAS_shi_vec;
2203 case INDEX_op_shls_vec:
2204 case INDEX_op_shrs_vec:
2205 case INDEX_op_sars_vec:
2206 return have_vec && TCG_TARGET_HAS_shs_vec;
2207 case INDEX_op_shlv_vec:
2208 case INDEX_op_shrv_vec:
2209 case INDEX_op_sarv_vec:
2210 return have_vec && TCG_TARGET_HAS_shv_vec;
Richard Hendersonb0f7e742020-04-19 18:01:52 -07002211 case INDEX_op_rotli_vec:
2212 return have_vec && TCG_TARGET_HAS_roti_vec;
Richard Henderson23850a72020-04-20 08:22:44 -07002213 case INDEX_op_rotls_vec:
2214 return have_vec && TCG_TARGET_HAS_rots_vec;
Richard Henderson5d0ceda2020-04-19 19:47:59 -07002215 case INDEX_op_rotlv_vec:
2216 case INDEX_op_rotrv_vec:
2217 return have_vec && TCG_TARGET_HAS_rotv_vec;
Richard Henderson8afaf052018-12-17 18:01:47 -08002218 case INDEX_op_ssadd_vec:
2219 case INDEX_op_usadd_vec:
2220 case INDEX_op_sssub_vec:
2221 case INDEX_op_ussub_vec:
2222 return have_vec && TCG_TARGET_HAS_sat_vec;
Richard Hendersondd0a0fc2018-12-17 19:35:46 -08002223 case INDEX_op_smin_vec:
2224 case INDEX_op_umin_vec:
2225 case INDEX_op_smax_vec:
2226 case INDEX_op_umax_vec:
2227 return have_vec && TCG_TARGET_HAS_minmax_vec;
Richard Henderson38dc1292019-04-30 11:02:23 -07002228 case INDEX_op_bitsel_vec:
2229 return have_vec && TCG_TARGET_HAS_bitsel_vec;
Richard Hendersonf75da292019-04-30 13:01:12 -07002230 case INDEX_op_cmpsel_vec:
2231 return have_vec && TCG_TARGET_HAS_cmpsel_vec;
Richard Hendersond2fd7452017-09-14 13:53:46 -07002232
Richard Hendersondb432672017-09-15 14:11:45 -07002233 default:
2234 tcg_debug_assert(op > INDEX_op_last_generic && op < NB_OPS);
2235 return true;
Richard Hendersonbe0f34b2017-08-17 07:43:20 -07002236 }
Richard Hendersonbe0f34b2017-08-17 07:43:20 -07002237}
2238
Richard Henderson39004a72022-11-11 10:09:37 +10002239static TCGOp *tcg_op_alloc(TCGOpcode opc, unsigned nargs);
2240
Richard Henderson83a0ad22024-03-14 13:48:41 -10002241static void tcg_gen_callN(void *func, TCGHelperInfo *info,
2242 TCGTemp *ret, TCGTemp **args)
bellardc896fe22008-02-01 10:05:41 +00002243{
Richard Henderson39004a72022-11-11 10:09:37 +10002244 TCGv_i64 extend_free[MAX_CALL_IARGS];
2245 int n_extend = 0;
Richard Henderson75e8b9b2016-12-08 10:52:57 -08002246 TCGOp *op;
Richard Henderson39004a72022-11-11 10:09:37 +10002247 int i, n, pi = 0, total_args;
Richard Hendersonafb49892014-04-07 15:10:05 -07002248
Richard Hendersond53106c2023-03-31 10:37:04 -07002249 if (unlikely(g_once_init_enter(HELPER_INFO_INIT(info)))) {
2250 init_call_layout(info);
2251 g_once_init_leave(HELPER_INFO_INIT(info), HELPER_INFO_INIT_VAL(info));
2252 }
2253
Richard Henderson39004a72022-11-11 10:09:37 +10002254 total_args = info->nr_out + info->nr_in + 2;
2255 op = tcg_op_alloc(INDEX_op_call, total_args);
Richard Henderson2bece2c2010-06-14 17:35:27 -07002256
Emilio G. Cota38b47b12018-12-07 15:33:56 -05002257#ifdef CONFIG_PLUGIN
Emilio Cota17083f62023-01-24 18:01:25 +00002258 /* Flag helpers that may affect guest state */
Richard Hendersonb0748972024-03-14 22:07:07 -10002259 if (tcg_ctx->plugin_insn && !(info->flags & TCG_CALL_NO_SIDE_EFFECTS)) {
Emilio G. Cota38b47b12018-12-07 15:33:56 -05002260 tcg_ctx->plugin_insn->calls_helpers = true;
2261 }
2262#endif
2263
Richard Henderson39004a72022-11-11 10:09:37 +10002264 TCGOP_CALLO(op) = n = info->nr_out;
2265 switch (n) {
2266 case 0:
2267 tcg_debug_assert(ret == NULL);
2268 break;
2269 case 1:
2270 tcg_debug_assert(ret != NULL);
2271 op->args[pi++] = temp_arg(ret);
2272 break;
2273 case 2:
Richard Henderson466d3752022-11-11 11:01:13 +10002274 case 4:
Richard Henderson39004a72022-11-11 10:09:37 +10002275 tcg_debug_assert(ret != NULL);
Richard Henderson466d3752022-11-11 11:01:13 +10002276 tcg_debug_assert(ret->base_type == ret->type + ctz32(n));
Richard Henderson39004a72022-11-11 10:09:37 +10002277 tcg_debug_assert(ret->temp_subindex == 0);
Richard Henderson466d3752022-11-11 11:01:13 +10002278 for (i = 0; i < n; ++i) {
2279 op->args[pi++] = temp_arg(ret + i);
2280 }
Richard Henderson39004a72022-11-11 10:09:37 +10002281 break;
2282 default:
2283 g_assert_not_reached();
2284 }
Richard Henderson7319d832021-03-18 10:01:01 -06002285
Richard Henderson39004a72022-11-11 10:09:37 +10002286 TCGOP_CALLI(op) = n = info->nr_in;
2287 for (i = 0; i < n; i++) {
2288 const TCGCallArgumentLoc *loc = &info->in[i];
2289 TCGTemp *ts = args[loc->arg_idx] + loc->tmp_subindex;
2290
2291 switch (loc->kind) {
2292 case TCG_CALL_ARG_NORMAL:
Richard Henderson313bdea2022-10-31 09:22:59 +11002293 case TCG_CALL_ARG_BY_REF:
2294 case TCG_CALL_ARG_BY_REF_N:
Richard Henderson39004a72022-11-11 10:09:37 +10002295 op->args[pi++] = temp_arg(ts);
2296 break;
2297
2298 case TCG_CALL_ARG_EXTEND_U:
2299 case TCG_CALL_ARG_EXTEND_S:
2300 {
Richard Henderson5dd48602023-01-29 13:26:49 -10002301 TCGv_i64 temp = tcg_temp_ebb_new_i64();
Richard Henderson39004a72022-11-11 10:09:37 +10002302 TCGv_i32 orig = temp_tcgv_i32(ts);
2303
2304 if (loc->kind == TCG_CALL_ARG_EXTEND_S) {
Richard Hendersoneb8b0222022-10-16 20:07:48 +10002305 tcg_gen_ext_i32_i64(temp, orig);
2306 } else {
2307 tcg_gen_extu_i32_i64(temp, orig);
2308 }
Richard Henderson39004a72022-11-11 10:09:37 +10002309 op->args[pi++] = tcgv_i64_arg(temp);
2310 extend_free[n_extend++] = temp;
Richard Henderson2bece2c2010-06-14 17:35:27 -07002311 }
Richard Hendersone2a9dd62022-10-17 15:55:56 +10002312 break;
Richard Henderson7319d832021-03-18 10:01:01 -06002313
Richard Hendersone2a9dd62022-10-17 15:55:56 +10002314 default:
2315 g_assert_not_reached();
bellardc896fe22008-02-01 10:05:41 +00002316 }
2317 }
Richard Henderson83a0ad22024-03-14 13:48:41 -10002318 op->args[pi++] = (uintptr_t)func;
Richard Henderson3e92aa32021-03-18 11:29:50 -06002319 op->args[pi++] = (uintptr_t)info;
Richard Henderson39004a72022-11-11 10:09:37 +10002320 tcg_debug_assert(pi == total_args);
pbrooka7812ae2008-11-17 14:43:54 +00002321
Richard Henderson07843f72024-03-13 13:32:29 -10002322 if (tcg_ctx->emit_before_op) {
2323 QTAILQ_INSERT_BEFORE(tcg_ctx->emit_before_op, op, link);
2324 } else {
2325 QTAILQ_INSERT_TAIL(&tcg_ctx->ops, op, link);
2326 }
Richard Henderson2bece2c2010-06-14 17:35:27 -07002327
Richard Henderson39004a72022-11-11 10:09:37 +10002328 tcg_debug_assert(n_extend < ARRAY_SIZE(extend_free));
2329 for (i = 0; i < n_extend; ++i) {
2330 tcg_temp_free_i64(extend_free[i]);
Richard Henderson2bece2c2010-06-14 17:35:27 -07002331 }
bellardc896fe22008-02-01 10:05:41 +00002332}
bellardc896fe22008-02-01 10:05:41 +00002333
Richard Henderson83a0ad22024-03-14 13:48:41 -10002334void tcg_gen_call0(void *func, TCGHelperInfo *info, TCGTemp *ret)
Richard Hendersona3a692b2023-03-29 22:14:36 -07002335{
Richard Henderson83a0ad22024-03-14 13:48:41 -10002336 tcg_gen_callN(func, info, ret, NULL);
Richard Hendersona3a692b2023-03-29 22:14:36 -07002337}
2338
Richard Henderson83a0ad22024-03-14 13:48:41 -10002339void tcg_gen_call1(void *func, TCGHelperInfo *info, TCGTemp *ret, TCGTemp *t1)
Richard Hendersona3a692b2023-03-29 22:14:36 -07002340{
Richard Henderson83a0ad22024-03-14 13:48:41 -10002341 tcg_gen_callN(func, info, ret, &t1);
Richard Hendersona3a692b2023-03-29 22:14:36 -07002342}
2343
Richard Henderson83a0ad22024-03-14 13:48:41 -10002344void tcg_gen_call2(void *func, TCGHelperInfo *info, TCGTemp *ret,
2345 TCGTemp *t1, TCGTemp *t2)
Richard Hendersona3a692b2023-03-29 22:14:36 -07002346{
2347 TCGTemp *args[2] = { t1, t2 };
Richard Henderson83a0ad22024-03-14 13:48:41 -10002348 tcg_gen_callN(func, info, ret, args);
Richard Hendersona3a692b2023-03-29 22:14:36 -07002349}
2350
Richard Henderson83a0ad22024-03-14 13:48:41 -10002351void tcg_gen_call3(void *func, TCGHelperInfo *info, TCGTemp *ret,
2352 TCGTemp *t1, TCGTemp *t2, TCGTemp *t3)
Richard Hendersona3a692b2023-03-29 22:14:36 -07002353{
2354 TCGTemp *args[3] = { t1, t2, t3 };
Richard Henderson83a0ad22024-03-14 13:48:41 -10002355 tcg_gen_callN(func, info, ret, args);
Richard Hendersona3a692b2023-03-29 22:14:36 -07002356}
2357
Richard Henderson83a0ad22024-03-14 13:48:41 -10002358void tcg_gen_call4(void *func, TCGHelperInfo *info, TCGTemp *ret,
2359 TCGTemp *t1, TCGTemp *t2, TCGTemp *t3, TCGTemp *t4)
Richard Hendersona3a692b2023-03-29 22:14:36 -07002360{
2361 TCGTemp *args[4] = { t1, t2, t3, t4 };
Richard Henderson83a0ad22024-03-14 13:48:41 -10002362 tcg_gen_callN(func, info, ret, args);
Richard Hendersona3a692b2023-03-29 22:14:36 -07002363}
2364
Richard Henderson83a0ad22024-03-14 13:48:41 -10002365void tcg_gen_call5(void *func, TCGHelperInfo *info, TCGTemp *ret, TCGTemp *t1,
Richard Hendersona3a692b2023-03-29 22:14:36 -07002366 TCGTemp *t2, TCGTemp *t3, TCGTemp *t4, TCGTemp *t5)
2367{
2368 TCGTemp *args[5] = { t1, t2, t3, t4, t5 };
Richard Henderson83a0ad22024-03-14 13:48:41 -10002369 tcg_gen_callN(func, info, ret, args);
Richard Hendersona3a692b2023-03-29 22:14:36 -07002370}
2371
Richard Henderson83a0ad22024-03-14 13:48:41 -10002372void tcg_gen_call6(void *func, TCGHelperInfo *info, TCGTemp *ret,
2373 TCGTemp *t1, TCGTemp *t2, TCGTemp *t3,
2374 TCGTemp *t4, TCGTemp *t5, TCGTemp *t6)
Richard Hendersona3a692b2023-03-29 22:14:36 -07002375{
2376 TCGTemp *args[6] = { t1, t2, t3, t4, t5, t6 };
Richard Henderson83a0ad22024-03-14 13:48:41 -10002377 tcg_gen_callN(func, info, ret, args);
Richard Hendersona3a692b2023-03-29 22:14:36 -07002378}
2379
Richard Henderson83a0ad22024-03-14 13:48:41 -10002380void tcg_gen_call7(void *func, TCGHelperInfo *info, TCGTemp *ret, TCGTemp *t1,
Richard Hendersona3a692b2023-03-29 22:14:36 -07002381 TCGTemp *t2, TCGTemp *t3, TCGTemp *t4,
2382 TCGTemp *t5, TCGTemp *t6, TCGTemp *t7)
2383{
2384 TCGTemp *args[7] = { t1, t2, t3, t4, t5, t6, t7 };
Richard Henderson83a0ad22024-03-14 13:48:41 -10002385 tcg_gen_callN(func, info, ret, args);
Richard Hendersona3a692b2023-03-29 22:14:36 -07002386}
2387
blueswir18fcd3692008-08-17 20:26:25 +00002388static void tcg_reg_alloc_start(TCGContext *s)
bellardc896fe22008-02-01 10:05:41 +00002389{
Richard Hendersonac3b8892016-11-02 11:21:44 -06002390 int i, n;
Richard Hendersonac3b8892016-11-02 11:21:44 -06002391
Richard Hendersonee17db82020-03-29 10:11:56 -07002392 for (i = 0, n = s->nb_temps; i < n; i++) {
2393 TCGTemp *ts = &s->temps[i];
2394 TCGTempVal val = TEMP_VAL_MEM;
2395
2396 switch (ts->kind) {
Richard Hendersonc0522132020-03-29 18:55:52 -07002397 case TEMP_CONST:
2398 val = TEMP_VAL_CONST;
2399 break;
Richard Hendersonee17db82020-03-29 10:11:56 -07002400 case TEMP_FIXED:
2401 val = TEMP_VAL_REG;
2402 break;
2403 case TEMP_GLOBAL:
2404 break;
Richard Hendersonc7482432022-03-16 09:34:18 -07002405 case TEMP_EBB:
Richard Hendersonee17db82020-03-29 10:11:56 -07002406 val = TEMP_VAL_DEAD;
2407 /* fall through */
Richard Hendersonf57c6912023-01-29 10:55:52 -10002408 case TEMP_TB:
Richard Hendersonee17db82020-03-29 10:11:56 -07002409 ts->mem_allocated = 0;
2410 break;
2411 default:
2412 g_assert_not_reached();
2413 }
2414 ts->val_type = val;
bellarde8996ee2008-05-23 17:33:39 +00002415 }
Richard Hendersonf8b2f202013-09-18 15:21:56 -07002416
2417 memset(s->reg_to_temp, 0, sizeof(s->reg_to_temp));
bellardc896fe22008-02-01 10:05:41 +00002418}
2419
Richard Hendersonf8b2f202013-09-18 15:21:56 -07002420static char *tcg_get_arg_str_ptr(TCGContext *s, char *buf, int buf_size,
2421 TCGTemp *ts)
bellardc896fe22008-02-01 10:05:41 +00002422{
Richard Henderson1807f4c2017-06-20 12:24:57 -07002423 int idx = temp_idx(ts);
pbrookac56dd42008-02-03 19:56:33 +00002424
Richard Hendersonee17db82020-03-29 10:11:56 -07002425 switch (ts->kind) {
2426 case TEMP_FIXED:
2427 case TEMP_GLOBAL:
pbrookac56dd42008-02-03 19:56:33 +00002428 pstrcpy(buf, buf_size, ts->name);
Richard Hendersonee17db82020-03-29 10:11:56 -07002429 break;
Richard Hendersonf57c6912023-01-29 10:55:52 -10002430 case TEMP_TB:
Richard Hendersonf8b2f202013-09-18 15:21:56 -07002431 snprintf(buf, buf_size, "loc%d", idx - s->nb_globals);
Richard Hendersonee17db82020-03-29 10:11:56 -07002432 break;
Richard Hendersonc7482432022-03-16 09:34:18 -07002433 case TEMP_EBB:
Richard Hendersonf8b2f202013-09-18 15:21:56 -07002434 snprintf(buf, buf_size, "tmp%d", idx - s->nb_globals);
Richard Hendersonee17db82020-03-29 10:11:56 -07002435 break;
Richard Hendersonc0522132020-03-29 18:55:52 -07002436 case TEMP_CONST:
2437 switch (ts->type) {
2438 case TCG_TYPE_I32:
2439 snprintf(buf, buf_size, "$0x%x", (int32_t)ts->val);
2440 break;
2441#if TCG_TARGET_REG_BITS > 32
2442 case TCG_TYPE_I64:
2443 snprintf(buf, buf_size, "$0x%" PRIx64, ts->val);
2444 break;
2445#endif
2446 case TCG_TYPE_V64:
2447 case TCG_TYPE_V128:
2448 case TCG_TYPE_V256:
2449 snprintf(buf, buf_size, "v%d$0x%" PRIx64,
2450 64 << (ts->type - TCG_TYPE_V64), ts->val);
2451 break;
2452 default:
2453 g_assert_not_reached();
2454 }
2455 break;
bellardc896fe22008-02-01 10:05:41 +00002456 }
2457 return buf;
2458}
2459
Richard Henderson43439132017-06-19 23:18:10 -07002460static char *tcg_get_arg_str(TCGContext *s, char *buf,
2461 int buf_size, TCGArg arg)
Richard Hendersonf8b2f202013-09-18 15:21:56 -07002462{
Richard Henderson43439132017-06-19 23:18:10 -07002463 return tcg_get_arg_str_ptr(s, buf, buf_size, arg_temp(arg));
Richard Hendersonf8b2f202013-09-18 15:21:56 -07002464}
2465
blueswir1f48f3ed2008-09-14 07:45:17 +00002466static const char * const cond_name[] =
2467{
Richard Henderson0aed2572012-09-24 14:21:40 -07002468 [TCG_COND_NEVER] = "never",
2469 [TCG_COND_ALWAYS] = "always",
blueswir1f48f3ed2008-09-14 07:45:17 +00002470 [TCG_COND_EQ] = "eq",
2471 [TCG_COND_NE] = "ne",
2472 [TCG_COND_LT] = "lt",
2473 [TCG_COND_GE] = "ge",
2474 [TCG_COND_LE] = "le",
2475 [TCG_COND_GT] = "gt",
2476 [TCG_COND_LTU] = "ltu",
2477 [TCG_COND_GEU] = "geu",
2478 [TCG_COND_LEU] = "leu",
Richard Hendersond48097d2023-10-23 18:53:27 -07002479 [TCG_COND_GTU] = "gtu",
2480 [TCG_COND_TSTEQ] = "tsteq",
2481 [TCG_COND_TSTNE] = "tstne",
blueswir1f48f3ed2008-09-14 07:45:17 +00002482};
2483
Richard Henderson12fde9b2022-11-07 10:42:56 +11002484static const char * const ldst_name[(MO_BSWAP | MO_SSIZE) + 1] =
Richard Hendersonf713d6a2013-09-04 08:11:05 -07002485{
2486 [MO_UB] = "ub",
2487 [MO_SB] = "sb",
2488 [MO_LEUW] = "leuw",
2489 [MO_LESW] = "lesw",
2490 [MO_LEUL] = "leul",
2491 [MO_LESL] = "lesl",
Frédéric Pétrotfc313c62022-01-06 22:00:51 +01002492 [MO_LEUQ] = "leq",
Richard Hendersonf713d6a2013-09-04 08:11:05 -07002493 [MO_BEUW] = "beuw",
2494 [MO_BESW] = "besw",
2495 [MO_BEUL] = "beul",
2496 [MO_BESL] = "besl",
Frédéric Pétrotfc313c62022-01-06 22:00:51 +01002497 [MO_BEUQ] = "beq",
Richard Henderson12fde9b2022-11-07 10:42:56 +11002498 [MO_128 + MO_BE] = "beo",
2499 [MO_128 + MO_LE] = "leo",
Richard Hendersonf713d6a2013-09-04 08:11:05 -07002500};
2501
Sergey Sorokin1f00b272016-06-23 21:16:46 +03002502static const char * const alignment_name[(MO_AMASK >> MO_ASHIFT) + 1] = {
Sergey Sorokin1f00b272016-06-23 21:16:46 +03002503 [MO_UNALN >> MO_ASHIFT] = "un+",
Sergey Sorokin1f00b272016-06-23 21:16:46 +03002504 [MO_ALIGN >> MO_ASHIFT] = "al+",
Sergey Sorokin1f00b272016-06-23 21:16:46 +03002505 [MO_ALIGN_2 >> MO_ASHIFT] = "al2+",
2506 [MO_ALIGN_4 >> MO_ASHIFT] = "al4+",
2507 [MO_ALIGN_8 >> MO_ASHIFT] = "al8+",
2508 [MO_ALIGN_16 >> MO_ASHIFT] = "al16+",
2509 [MO_ALIGN_32 >> MO_ASHIFT] = "al32+",
2510 [MO_ALIGN_64 >> MO_ASHIFT] = "al64+",
2511};
2512
Richard Henderson37031fe2022-10-21 21:24:40 +10002513static const char * const atom_name[(MO_ATOM_MASK >> MO_ATOM_SHIFT) + 1] = {
2514 [MO_ATOM_IFALIGN >> MO_ATOM_SHIFT] = "",
2515 [MO_ATOM_IFALIGN_PAIR >> MO_ATOM_SHIFT] = "pair+",
2516 [MO_ATOM_WITHIN16 >> MO_ATOM_SHIFT] = "w16+",
2517 [MO_ATOM_WITHIN16_PAIR >> MO_ATOM_SHIFT] = "w16p+",
2518 [MO_ATOM_SUBALIGN >> MO_ATOM_SHIFT] = "sub+",
2519 [MO_ATOM_NONE >> MO_ATOM_SHIFT] = "noat+",
2520};
2521
Richard Henderson587195b2021-06-12 21:32:27 -07002522static const char bswap_flag_name[][6] = {
2523 [TCG_BSWAP_IZ] = "iz",
2524 [TCG_BSWAP_OZ] = "oz",
2525 [TCG_BSWAP_OS] = "os",
2526 [TCG_BSWAP_IZ | TCG_BSWAP_OZ] = "iz,oz",
2527 [TCG_BSWAP_IZ | TCG_BSWAP_OS] = "iz,os",
2528};
2529
Richard Hendersonb384c732024-03-15 10:33:49 -10002530#ifdef CONFIG_PLUGIN
2531static const char * const plugin_from_name[] = {
2532 "from-tb",
2533 "from-insn",
2534 "after-insn",
2535 "after-tb",
2536};
2537#endif
2538
Richard Hendersonb0164862018-11-27 07:16:21 -08002539static inline bool tcg_regset_single(TCGRegSet d)
2540{
2541 return (d & (d - 1)) == 0;
2542}
2543
2544static inline TCGReg tcg_regset_first(TCGRegSet d)
2545{
2546 if (TCG_TARGET_NB_REGS <= 32) {
2547 return ctz32(d);
2548 } else {
2549 return ctz64(d);
2550 }
2551}
2552
Richard Hendersonb7a83ff2022-04-17 11:29:51 -07002553/* Return only the number of characters output -- no error return. */
2554#define ne_fprintf(...) \
2555 ({ int ret_ = fprintf(__VA_ARGS__); ret_ >= 0 ? ret_ : 0; })
2556
Richard Hendersonb384c732024-03-15 10:33:49 -10002557void tcg_dump_ops(TCGContext *s, FILE *f, bool have_prefs)
bellardc896fe22008-02-01 10:05:41 +00002558{
bellardc896fe22008-02-01 10:05:41 +00002559 char buf[128];
Richard Hendersonc45cb8b2014-09-19 13:49:15 -07002560 TCGOp *op;
bellardc896fe22008-02-01 10:05:41 +00002561
Richard Henderson15fa08f2017-11-02 15:19:14 +01002562 QTAILQ_FOREACH(op, &s->ops, link) {
Richard Hendersonc45cb8b2014-09-19 13:49:15 -07002563 int i, k, nb_oargs, nb_iargs, nb_cargs;
2564 const TCGOpDef *def;
Richard Hendersonc45cb8b2014-09-19 13:49:15 -07002565 TCGOpcode c;
Richard Hendersonbdfb4602016-06-23 19:15:55 -07002566 int col = 0;
Richard Hendersonc45cb8b2014-09-19 13:49:15 -07002567
Richard Hendersonc45cb8b2014-09-19 13:49:15 -07002568 c = op->opc;
bellardc896fe22008-02-01 10:05:41 +00002569 def = &tcg_op_defs[c];
Richard Hendersonc45cb8b2014-09-19 13:49:15 -07002570
Richard Henderson765b8422015-08-29 12:37:33 -07002571 if (c == INDEX_op_insn_start) {
Richard Hendersonb0164862018-11-27 07:16:21 -08002572 nb_oargs = 0;
Richard Hendersonb7a83ff2022-04-17 11:29:51 -07002573 col += ne_fprintf(f, "\n ----");
Richard Henderson9aef40e2015-08-30 09:21:33 -07002574
Richard Henderson747bd692023-03-31 21:30:31 -07002575 for (i = 0, k = s->insn_start_words; i < k; ++i) {
Richard Hendersonc9ad8d22023-03-08 12:24:41 -08002576 col += ne_fprintf(f, " %016" PRIx64,
2577 tcg_get_insn_start_param(op, i));
Blue Swirleeacee42012-06-03 16:35:32 +00002578 }
bellard7e4597d2008-05-22 16:56:05 +00002579 } else if (c == INDEX_op_call) {
Richard Henderson3e92aa32021-03-18 11:29:50 -06002580 const TCGHelperInfo *info = tcg_call_info(op);
Richard Hendersonfa52e662021-03-18 16:40:07 -06002581 void *func = tcg_call_func(op);
Richard Henderson3e92aa32021-03-18 11:29:50 -06002582
bellardc896fe22008-02-01 10:05:41 +00002583 /* variable number of arguments */
Richard Hendersoncd9090a2017-11-14 13:02:51 +01002584 nb_oargs = TCGOP_CALLO(op);
2585 nb_iargs = TCGOP_CALLI(op);
bellardc896fe22008-02-01 10:05:41 +00002586 nb_cargs = def->nb_cargs;
bellardc896fe22008-02-01 10:05:41 +00002587
Richard Hendersonb7a83ff2022-04-17 11:29:51 -07002588 col += ne_fprintf(f, " %s ", def->name);
Richard Henderson3e92aa32021-03-18 11:29:50 -06002589
2590 /*
2591 * Print the function name from TCGHelperInfo, if available.
2592 * Note that plugins have a template function for the info,
2593 * but the actual function pointer comes from the plugin.
2594 */
Richard Henderson3e92aa32021-03-18 11:29:50 -06002595 if (func == info->func) {
Richard Hendersonb7a83ff2022-04-17 11:29:51 -07002596 col += ne_fprintf(f, "%s", info->name);
Richard Henderson3e92aa32021-03-18 11:29:50 -06002597 } else {
Richard Hendersonb7a83ff2022-04-17 11:29:51 -07002598 col += ne_fprintf(f, "plugin(%p)", func);
Richard Henderson3e92aa32021-03-18 11:29:50 -06002599 }
2600
Richard Hendersonb7a83ff2022-04-17 11:29:51 -07002601 col += ne_fprintf(f, ",$0x%x,$%d", info->flags, nb_oargs);
Richard Hendersoncf066672014-03-22 20:06:52 -07002602 for (i = 0; i < nb_oargs; i++) {
Richard Hendersonb7a83ff2022-04-17 11:29:51 -07002603 col += ne_fprintf(f, ",%s", tcg_get_arg_str(s, buf, sizeof(buf),
2604 op->args[i]));
bellardb03cce82008-05-10 10:52:05 +00002605 }
Richard Hendersoncf066672014-03-22 20:06:52 -07002606 for (i = 0; i < nb_iargs; i++) {
Richard Hendersonefee3742016-12-08 13:12:08 -08002607 TCGArg arg = op->args[nb_oargs + i];
Richard Henderson39004a72022-11-11 10:09:37 +10002608 const char *t = tcg_get_arg_str(s, buf, sizeof(buf), arg);
Richard Hendersonb7a83ff2022-04-17 11:29:51 -07002609 col += ne_fprintf(f, ",%s", t);
bellarde8996ee2008-05-23 17:33:39 +00002610 }
bellardb03cce82008-05-10 10:52:05 +00002611 } else {
Richard Hendersonb7a83ff2022-04-17 11:29:51 -07002612 col += ne_fprintf(f, " %s ", def->name);
Richard Hendersonc45cb8b2014-09-19 13:49:15 -07002613
2614 nb_oargs = def->nb_oargs;
2615 nb_iargs = def->nb_iargs;
2616 nb_cargs = def->nb_cargs;
2617
Richard Hendersond2fd7452017-09-14 13:53:46 -07002618 if (def->flags & TCG_OPF_VECTOR) {
Richard Hendersonb7a83ff2022-04-17 11:29:51 -07002619 col += ne_fprintf(f, "v%d,e%d,", 64 << TCGOP_VECL(op),
2620 8 << TCGOP_VECE(op));
Richard Hendersond2fd7452017-09-14 13:53:46 -07002621 }
2622
bellardb03cce82008-05-10 10:52:05 +00002623 k = 0;
Richard Hendersonc45cb8b2014-09-19 13:49:15 -07002624 for (i = 0; i < nb_oargs; i++) {
Richard Hendersonb7a83ff2022-04-17 11:29:51 -07002625 const char *sep = k ? "," : "";
2626 col += ne_fprintf(f, "%s%s", sep,
2627 tcg_get_arg_str(s, buf, sizeof(buf),
2628 op->args[k++]));
bellardb03cce82008-05-10 10:52:05 +00002629 }
Richard Hendersonc45cb8b2014-09-19 13:49:15 -07002630 for (i = 0; i < nb_iargs; i++) {
Richard Hendersonb7a83ff2022-04-17 11:29:51 -07002631 const char *sep = k ? "," : "";
2632 col += ne_fprintf(f, "%s%s", sep,
2633 tcg_get_arg_str(s, buf, sizeof(buf),
2634 op->args[k++]));
bellardb03cce82008-05-10 10:52:05 +00002635 }
Richard Hendersonbe210ac2010-01-07 10:13:31 -08002636 switch (c) {
2637 case INDEX_op_brcond_i32:
Richard Hendersonffc5ea02012-09-21 10:13:34 -07002638 case INDEX_op_setcond_i32:
Richard Henderson36355022023-08-04 23:24:04 +00002639 case INDEX_op_negsetcond_i32:
Richard Hendersonffc5ea02012-09-21 10:13:34 -07002640 case INDEX_op_movcond_i32:
Richard Hendersonbe210ac2010-01-07 10:13:31 -08002641 case INDEX_op_brcond2_i32:
Richard Hendersonbe210ac2010-01-07 10:13:31 -08002642 case INDEX_op_setcond2_i32:
Richard Hendersonffc5ea02012-09-21 10:13:34 -07002643 case INDEX_op_brcond_i64:
Richard Hendersonbe210ac2010-01-07 10:13:31 -08002644 case INDEX_op_setcond_i64:
Richard Henderson36355022023-08-04 23:24:04 +00002645 case INDEX_op_negsetcond_i64:
Richard Hendersonffc5ea02012-09-21 10:13:34 -07002646 case INDEX_op_movcond_i64:
Richard Henderson212be172017-11-17 20:47:42 +01002647 case INDEX_op_cmp_vec:
Richard Hendersonf75da292019-04-30 13:01:12 -07002648 case INDEX_op_cmpsel_vec:
Richard Hendersonefee3742016-12-08 13:12:08 -08002649 if (op->args[k] < ARRAY_SIZE(cond_name)
2650 && cond_name[op->args[k]]) {
Richard Hendersonb7a83ff2022-04-17 11:29:51 -07002651 col += ne_fprintf(f, ",%s", cond_name[op->args[k++]]);
Blue Swirleeacee42012-06-03 16:35:32 +00002652 } else {
Richard Hendersonb7a83ff2022-04-17 11:29:51 -07002653 col += ne_fprintf(f, ",$0x%" TCG_PRIlx, op->args[k++]);
Blue Swirleeacee42012-06-03 16:35:32 +00002654 }
blueswir1f48f3ed2008-09-14 07:45:17 +00002655 i = 1;
Richard Hendersonbe210ac2010-01-07 10:13:31 -08002656 break;
Richard Hendersonfecccfc2023-05-16 20:07:20 -07002657 case INDEX_op_qemu_ld_a32_i32:
2658 case INDEX_op_qemu_ld_a64_i32:
2659 case INDEX_op_qemu_st_a32_i32:
2660 case INDEX_op_qemu_st_a64_i32:
2661 case INDEX_op_qemu_st8_a32_i32:
2662 case INDEX_op_qemu_st8_a64_i32:
2663 case INDEX_op_qemu_ld_a32_i64:
2664 case INDEX_op_qemu_ld_a64_i64:
2665 case INDEX_op_qemu_st_a32_i64:
2666 case INDEX_op_qemu_st_a64_i64:
2667 case INDEX_op_qemu_ld_a32_i128:
2668 case INDEX_op_qemu_ld_a64_i128:
2669 case INDEX_op_qemu_st_a32_i128:
2670 case INDEX_op_qemu_st_a64_i128:
Richard Henderson59227d52015-05-12 11:51:44 -07002671 {
Richard Henderson37031fe2022-10-21 21:24:40 +10002672 const char *s_al, *s_op, *s_at;
Richard Henderson9002ffc2021-07-25 12:06:49 -10002673 MemOpIdx oi = op->args[k++];
Philippe Mathieu-Daudé9a239c62023-09-04 18:12:13 +02002674 MemOp mop = get_memop(oi);
Richard Henderson59227d52015-05-12 11:51:44 -07002675 unsigned ix = get_mmuidx(oi);
2676
Philippe Mathieu-Daudé9a239c62023-09-04 18:12:13 +02002677 s_al = alignment_name[(mop & MO_AMASK) >> MO_ASHIFT];
2678 s_op = ldst_name[mop & (MO_BSWAP | MO_SSIZE)];
2679 s_at = atom_name[(mop & MO_ATOM_MASK) >> MO_ATOM_SHIFT];
2680 mop &= ~(MO_AMASK | MO_BSWAP | MO_SSIZE | MO_ATOM_MASK);
Richard Henderson37031fe2022-10-21 21:24:40 +10002681
2682 /* If all fields are accounted for, print symbolically. */
Philippe Mathieu-Daudé9a239c62023-09-04 18:12:13 +02002683 if (!mop && s_al && s_op && s_at) {
Richard Henderson37031fe2022-10-21 21:24:40 +10002684 col += ne_fprintf(f, ",%s%s%s,%u",
2685 s_at, s_al, s_op, ix);
Richard Henderson59c4b7e2015-06-01 14:38:56 -07002686 } else {
Philippe Mathieu-Daudé9a239c62023-09-04 18:12:13 +02002687 mop = get_memop(oi);
2688 col += ne_fprintf(f, ",$0x%x,%u", mop, ix);
Richard Henderson59227d52015-05-12 11:51:44 -07002689 }
2690 i = 1;
Richard Hendersonf713d6a2013-09-04 08:11:05 -07002691 }
Richard Hendersonf713d6a2013-09-04 08:11:05 -07002692 break;
Richard Henderson587195b2021-06-12 21:32:27 -07002693 case INDEX_op_bswap16_i32:
2694 case INDEX_op_bswap16_i64:
2695 case INDEX_op_bswap32_i32:
2696 case INDEX_op_bswap32_i64:
2697 case INDEX_op_bswap64_i64:
2698 {
2699 TCGArg flags = op->args[k];
2700 const char *name = NULL;
2701
2702 if (flags < ARRAY_SIZE(bswap_flag_name)) {
2703 name = bswap_flag_name[flags];
2704 }
2705 if (name) {
Richard Hendersonb7a83ff2022-04-17 11:29:51 -07002706 col += ne_fprintf(f, ",%s", name);
Richard Henderson587195b2021-06-12 21:32:27 -07002707 } else {
Richard Hendersonb7a83ff2022-04-17 11:29:51 -07002708 col += ne_fprintf(f, ",$0x%" TCG_PRIlx, flags);
Richard Henderson587195b2021-06-12 21:32:27 -07002709 }
2710 i = k = 1;
2711 }
2712 break;
Richard Hendersonb384c732024-03-15 10:33:49 -10002713#ifdef CONFIG_PLUGIN
2714 case INDEX_op_plugin_cb:
2715 {
2716 TCGArg from = op->args[k++];
2717 const char *name = NULL;
2718
2719 if (from < ARRAY_SIZE(plugin_from_name)) {
2720 name = plugin_from_name[from];
2721 }
2722 if (name) {
2723 col += ne_fprintf(f, "%s", name);
2724 } else {
2725 col += ne_fprintf(f, "$0x%" TCG_PRIlx, from);
2726 }
2727 i = 1;
2728 }
2729 break;
2730#endif
Richard Hendersonbe210ac2010-01-07 10:13:31 -08002731 default:
blueswir1f48f3ed2008-09-14 07:45:17 +00002732 i = 0;
Richard Hendersonbe210ac2010-01-07 10:13:31 -08002733 break;
2734 }
Richard Henderson51e39722015-02-13 18:51:05 -08002735 switch (c) {
2736 case INDEX_op_set_label:
2737 case INDEX_op_br:
2738 case INDEX_op_brcond_i32:
2739 case INDEX_op_brcond_i64:
2740 case INDEX_op_brcond2_i32:
Richard Hendersonb7a83ff2022-04-17 11:29:51 -07002741 col += ne_fprintf(f, "%s$L%d", k ? "," : "",
2742 arg_label(op->args[k])->id);
Richard Henderson51e39722015-02-13 18:51:05 -08002743 i++, k++;
2744 break;
Richard Henderson34708672021-02-18 17:05:55 -08002745 case INDEX_op_mb:
2746 {
2747 TCGBar membar = op->args[k];
2748 const char *b_op, *m_op;
2749
2750 switch (membar & TCG_BAR_SC) {
2751 case 0:
2752 b_op = "none";
2753 break;
2754 case TCG_BAR_LDAQ:
2755 b_op = "acq";
2756 break;
2757 case TCG_BAR_STRL:
2758 b_op = "rel";
2759 break;
2760 case TCG_BAR_SC:
2761 b_op = "seq";
2762 break;
2763 default:
2764 g_assert_not_reached();
2765 }
2766
2767 switch (membar & TCG_MO_ALL) {
2768 case 0:
2769 m_op = "none";
2770 break;
2771 case TCG_MO_LD_LD:
2772 m_op = "rr";
2773 break;
2774 case TCG_MO_LD_ST:
2775 m_op = "rw";
2776 break;
2777 case TCG_MO_ST_LD:
2778 m_op = "wr";
2779 break;
2780 case TCG_MO_ST_ST:
2781 m_op = "ww";
2782 break;
2783 case TCG_MO_LD_LD | TCG_MO_LD_ST:
2784 m_op = "rr+rw";
2785 break;
2786 case TCG_MO_LD_LD | TCG_MO_ST_LD:
2787 m_op = "rr+wr";
2788 break;
2789 case TCG_MO_LD_LD | TCG_MO_ST_ST:
2790 m_op = "rr+ww";
2791 break;
2792 case TCG_MO_LD_ST | TCG_MO_ST_LD:
2793 m_op = "rw+wr";
2794 break;
2795 case TCG_MO_LD_ST | TCG_MO_ST_ST:
2796 m_op = "rw+ww";
2797 break;
2798 case TCG_MO_ST_LD | TCG_MO_ST_ST:
2799 m_op = "wr+ww";
2800 break;
2801 case TCG_MO_LD_LD | TCG_MO_LD_ST | TCG_MO_ST_LD:
2802 m_op = "rr+rw+wr";
2803 break;
2804 case TCG_MO_LD_LD | TCG_MO_LD_ST | TCG_MO_ST_ST:
2805 m_op = "rr+rw+ww";
2806 break;
2807 case TCG_MO_LD_LD | TCG_MO_ST_LD | TCG_MO_ST_ST:
2808 m_op = "rr+wr+ww";
2809 break;
2810 case TCG_MO_LD_ST | TCG_MO_ST_LD | TCG_MO_ST_ST:
2811 m_op = "rw+wr+ww";
2812 break;
2813 case TCG_MO_ALL:
2814 m_op = "all";
2815 break;
2816 default:
2817 g_assert_not_reached();
2818 }
2819
2820 col += ne_fprintf(f, "%s%s:%s", (k ? "," : ""), b_op, m_op);
2821 i++, k++;
2822 }
2823 break;
Richard Henderson51e39722015-02-13 18:51:05 -08002824 default:
2825 break;
2826 }
2827 for (; i < nb_cargs; i++, k++) {
Richard Hendersonb7a83ff2022-04-17 11:29:51 -07002828 col += ne_fprintf(f, "%s$0x%" TCG_PRIlx, k ? "," : "",
2829 op->args[k]);
Richard Hendersonbdfb4602016-06-23 19:15:55 -07002830 }
2831 }
Richard Hendersonbdfb4602016-06-23 19:15:55 -07002832
Richard Henderson1894f692018-11-27 12:46:00 -08002833 if (have_prefs || op->life) {
Richard Hendersonb7a83ff2022-04-17 11:29:51 -07002834 for (; col < 40; ++col) {
2835 putc(' ', f);
Richard Hendersonbdfb4602016-06-23 19:15:55 -07002836 }
Richard Henderson1894f692018-11-27 12:46:00 -08002837 }
2838
2839 if (op->life) {
2840 unsigned life = op->life;
Richard Hendersonbdfb4602016-06-23 19:15:55 -07002841
2842 if (life & (SYNC_ARG * 3)) {
Richard Hendersonb7a83ff2022-04-17 11:29:51 -07002843 ne_fprintf(f, " sync:");
Richard Hendersonbdfb4602016-06-23 19:15:55 -07002844 for (i = 0; i < 2; ++i) {
2845 if (life & (SYNC_ARG << i)) {
Richard Hendersonb7a83ff2022-04-17 11:29:51 -07002846 ne_fprintf(f, " %d", i);
Richard Hendersonbdfb4602016-06-23 19:15:55 -07002847 }
2848 }
2849 }
2850 life /= DEAD_ARG;
2851 if (life) {
Richard Hendersonb7a83ff2022-04-17 11:29:51 -07002852 ne_fprintf(f, " dead:");
Richard Hendersonbdfb4602016-06-23 19:15:55 -07002853 for (i = 0; life; ++i, life >>= 1) {
2854 if (life & 1) {
Richard Hendersonb7a83ff2022-04-17 11:29:51 -07002855 ne_fprintf(f, " %d", i);
Richard Hendersonbdfb4602016-06-23 19:15:55 -07002856 }
2857 }
bellardb03cce82008-05-10 10:52:05 +00002858 }
bellardc896fe22008-02-01 10:05:41 +00002859 }
Richard Henderson1894f692018-11-27 12:46:00 -08002860
2861 if (have_prefs) {
2862 for (i = 0; i < nb_oargs; ++i) {
Richard Henderson31fd8842022-11-11 15:10:51 +10002863 TCGRegSet set = output_pref(op, i);
Richard Henderson1894f692018-11-27 12:46:00 -08002864
2865 if (i == 0) {
Richard Hendersonb7a83ff2022-04-17 11:29:51 -07002866 ne_fprintf(f, " pref=");
Richard Henderson1894f692018-11-27 12:46:00 -08002867 } else {
Richard Hendersonb7a83ff2022-04-17 11:29:51 -07002868 ne_fprintf(f, ",");
Richard Henderson1894f692018-11-27 12:46:00 -08002869 }
2870 if (set == 0) {
Richard Hendersonb7a83ff2022-04-17 11:29:51 -07002871 ne_fprintf(f, "none");
Richard Henderson1894f692018-11-27 12:46:00 -08002872 } else if (set == MAKE_64BIT_MASK(0, TCG_TARGET_NB_REGS)) {
Richard Hendersonb7a83ff2022-04-17 11:29:51 -07002873 ne_fprintf(f, "all");
Richard Henderson1894f692018-11-27 12:46:00 -08002874#ifdef CONFIG_DEBUG_TCG
2875 } else if (tcg_regset_single(set)) {
2876 TCGReg reg = tcg_regset_first(set);
Richard Hendersonb7a83ff2022-04-17 11:29:51 -07002877 ne_fprintf(f, "%s", tcg_target_reg_names[reg]);
Richard Henderson1894f692018-11-27 12:46:00 -08002878#endif
2879 } else if (TCG_TARGET_NB_REGS <= 32) {
Richard Hendersonb7a83ff2022-04-17 11:29:51 -07002880 ne_fprintf(f, "0x%x", (uint32_t)set);
Richard Henderson1894f692018-11-27 12:46:00 -08002881 } else {
Richard Hendersonb7a83ff2022-04-17 11:29:51 -07002882 ne_fprintf(f, "0x%" PRIx64, (uint64_t)set);
Richard Henderson1894f692018-11-27 12:46:00 -08002883 }
2884 }
2885 }
2886
Richard Hendersonb7a83ff2022-04-17 11:29:51 -07002887 putc('\n', f);
bellardc896fe22008-02-01 10:05:41 +00002888 }
2889}
2890
2891/* we give more priority to constraints with less registers */
2892static int get_constraint_priority(const TCGOpDef *def, int k)
2893{
Richard Henderson74a11792020-09-03 15:56:24 -07002894 const TCGArgConstraint *arg_ct = &def->args_ct[k];
Richard Henderson29f5e922022-10-14 07:37:38 +11002895 int n = ctpop64(arg_ct->regs);
bellardc896fe22008-02-01 10:05:41 +00002896
Richard Henderson29f5e922022-10-14 07:37:38 +11002897 /*
2898 * Sort constraints of a single register first, which includes output
2899 * aliases (which must exactly match the input already allocated).
2900 */
2901 if (n == 1 || arg_ct->oalias) {
2902 return INT_MAX;
bellardc896fe22008-02-01 10:05:41 +00002903 }
Richard Henderson29f5e922022-10-14 07:37:38 +11002904
2905 /*
2906 * Sort register pairs next, first then second immediately after.
2907 * Arbitrarily sort multiple pairs by the index of the first reg;
2908 * there shouldn't be many pairs.
2909 */
2910 switch (arg_ct->pair) {
2911 case 1:
2912 case 3:
2913 return (k + 1) * 2;
2914 case 2:
2915 return (arg_ct->pair_index + 1) * 2 - 1;
2916 }
2917
2918 /* Finally, sort by decreasing register count. */
2919 assert(n > 1);
2920 return -n;
bellardc896fe22008-02-01 10:05:41 +00002921}
2922
2923/* sort from highest priority to lowest */
2924static void sort_constraints(TCGOpDef *def, int start, int n)
2925{
Richard Henderson66792f92019-04-04 09:37:38 +07002926 int i, j;
2927 TCGArgConstraint *a = def->args_ct;
bellardc896fe22008-02-01 10:05:41 +00002928
Richard Henderson66792f92019-04-04 09:37:38 +07002929 for (i = 0; i < n; i++) {
2930 a[start + i].sort_index = start + i;
2931 }
2932 if (n <= 1) {
bellardc896fe22008-02-01 10:05:41 +00002933 return;
Richard Henderson66792f92019-04-04 09:37:38 +07002934 }
2935 for (i = 0; i < n - 1; i++) {
2936 for (j = i + 1; j < n; j++) {
2937 int p1 = get_constraint_priority(def, a[start + i].sort_index);
2938 int p2 = get_constraint_priority(def, a[start + j].sort_index);
bellardc896fe22008-02-01 10:05:41 +00002939 if (p1 < p2) {
Richard Henderson66792f92019-04-04 09:37:38 +07002940 int tmp = a[start + i].sort_index;
2941 a[start + i].sort_index = a[start + j].sort_index;
2942 a[start + j].sort_index = tmp;
bellardc896fe22008-02-01 10:05:41 +00002943 }
2944 }
2945 }
2946}
2947
Richard Hendersonf69d2772016-11-18 09:31:40 +01002948static void process_op_defs(TCGContext *s)
bellardc896fe22008-02-01 10:05:41 +00002949{
Richard Hendersona9751602010-03-19 11:12:29 -07002950 TCGOpcode op;
bellardc896fe22008-02-01 10:05:41 +00002951
Richard Hendersonf69d2772016-11-18 09:31:40 +01002952 for (op = 0; op < NB_OPS; op++) {
2953 TCGOpDef *def = &tcg_op_defs[op];
2954 const TCGTargetOpDef *tdefs;
Richard Henderson29f5e922022-10-14 07:37:38 +11002955 bool saw_alias_pair = false;
2956 int i, o, i2, o2, nb_args;
Richard Hendersonf69d2772016-11-18 09:31:40 +01002957
2958 if (def->flags & TCG_OPF_NOT_PRESENT) {
2959 continue;
2960 }
2961
bellardc896fe22008-02-01 10:05:41 +00002962 nb_args = def->nb_iargs + def->nb_oargs;
Richard Hendersonf69d2772016-11-18 09:31:40 +01002963 if (nb_args == 0) {
2964 continue;
2965 }
2966
Richard Henderson4c22e842020-10-16 22:20:55 -07002967 /*
2968 * Macro magic should make it impossible, but double-check that
2969 * the array index is in range. Since the signness of an enum
2970 * is implementation defined, force the result to unsigned.
2971 */
2972 unsigned con_set = tcg_target_op_def(op);
2973 tcg_debug_assert(con_set < ARRAY_SIZE(constraint_sets));
2974 tdefs = &constraint_sets[con_set];
Richard Hendersonf69d2772016-11-18 09:31:40 +01002975
2976 for (i = 0; i < nb_args; i++) {
2977 const char *ct_str = tdefs->args_ct_str[i];
Philippe Mathieu-Daudé8940ea02022-12-19 23:09:23 +01002978 bool input_p = i >= def->nb_oargs;
2979
Richard Hendersonf69d2772016-11-18 09:31:40 +01002980 /* Incomplete TCGTargetOpDef entry. */
Aurelien Jarnoeabb7b92016-04-21 10:48:49 +02002981 tcg_debug_assert(ct_str != NULL);
Richard Hendersonf69d2772016-11-18 09:31:40 +01002982
Philippe Mathieu-Daudé8940ea02022-12-19 23:09:23 +01002983 switch (*ct_str) {
2984 case '0' ... '9':
2985 o = *ct_str - '0';
2986 tcg_debug_assert(input_p);
2987 tcg_debug_assert(o < def->nb_oargs);
2988 tcg_debug_assert(def->args_ct[o].regs != 0);
2989 tcg_debug_assert(!def->args_ct[o].oalias);
2990 def->args_ct[i] = def->args_ct[o];
2991 /* The output sets oalias. */
2992 def->args_ct[o].oalias = 1;
2993 def->args_ct[o].alias_index = i;
2994 /* The input sets ialias. */
2995 def->args_ct[i].ialias = 1;
2996 def->args_ct[i].alias_index = o;
Richard Henderson29f5e922022-10-14 07:37:38 +11002997 if (def->args_ct[i].pair) {
2998 saw_alias_pair = true;
2999 }
Philippe Mathieu-Daudé8940ea02022-12-19 23:09:23 +01003000 tcg_debug_assert(ct_str[1] == '\0');
3001 continue;
3002
3003 case '&':
3004 tcg_debug_assert(!input_p);
3005 def->args_ct[i].newreg = true;
3006 ct_str++;
3007 break;
Richard Henderson29f5e922022-10-14 07:37:38 +11003008
3009 case 'p': /* plus */
3010 /* Allocate to the register after the previous. */
3011 tcg_debug_assert(i > (input_p ? def->nb_oargs : 0));
3012 o = i - 1;
3013 tcg_debug_assert(!def->args_ct[o].pair);
3014 tcg_debug_assert(!def->args_ct[o].ct);
3015 def->args_ct[i] = (TCGArgConstraint){
3016 .pair = 2,
3017 .pair_index = o,
3018 .regs = def->args_ct[o].regs << 1,
Richard Hendersonca5bed02024-01-02 01:27:18 +00003019 .newreg = def->args_ct[o].newreg,
Richard Henderson29f5e922022-10-14 07:37:38 +11003020 };
3021 def->args_ct[o].pair = 1;
3022 def->args_ct[o].pair_index = i;
3023 tcg_debug_assert(ct_str[1] == '\0');
3024 continue;
3025
3026 case 'm': /* minus */
3027 /* Allocate to the register before the previous. */
3028 tcg_debug_assert(i > (input_p ? def->nb_oargs : 0));
3029 o = i - 1;
3030 tcg_debug_assert(!def->args_ct[o].pair);
3031 tcg_debug_assert(!def->args_ct[o].ct);
3032 def->args_ct[i] = (TCGArgConstraint){
3033 .pair = 1,
3034 .pair_index = o,
3035 .regs = def->args_ct[o].regs >> 1,
Richard Hendersonca5bed02024-01-02 01:27:18 +00003036 .newreg = def->args_ct[o].newreg,
Richard Henderson29f5e922022-10-14 07:37:38 +11003037 };
3038 def->args_ct[o].pair = 2;
3039 def->args_ct[o].pair_index = i;
3040 tcg_debug_assert(ct_str[1] == '\0');
3041 continue;
Philippe Mathieu-Daudé8940ea02022-12-19 23:09:23 +01003042 }
3043
3044 do {
3045 switch (*ct_str) {
Richard Henderson17280ff2016-11-18 17:41:24 +01003046 case 'i':
3047 def->args_ct[i].ct |= TCG_CT_CONST;
Richard Henderson17280ff2016-11-18 17:41:24 +01003048 break;
Richard Henderson358b4922020-10-16 15:27:46 -07003049
Richard Henderson358b4922020-10-16 15:27:46 -07003050 /* Include all of the target-specific constraints. */
3051
3052#undef CONST
3053#define CONST(CASE, MASK) \
Philippe Mathieu-Daudé8940ea02022-12-19 23:09:23 +01003054 case CASE: def->args_ct[i].ct |= MASK; break;
Richard Henderson358b4922020-10-16 15:27:46 -07003055#define REGS(CASE, MASK) \
Philippe Mathieu-Daudé8940ea02022-12-19 23:09:23 +01003056 case CASE: def->args_ct[i].regs |= MASK; break;
Richard Henderson358b4922020-10-16 15:27:46 -07003057
3058#include "tcg-target-con-str.h"
3059
3060#undef REGS
3061#undef CONST
Richard Henderson17280ff2016-11-18 17:41:24 +01003062 default:
Philippe Mathieu-Daudé8940ea02022-12-19 23:09:23 +01003063 case '0' ... '9':
3064 case '&':
Richard Henderson29f5e922022-10-14 07:37:38 +11003065 case 'p':
3066 case 'm':
Richard Henderson17280ff2016-11-18 17:41:24 +01003067 /* Typo in TCGTargetOpDef constraint. */
Richard Henderson358b4922020-10-16 15:27:46 -07003068 g_assert_not_reached();
bellardc896fe22008-02-01 10:05:41 +00003069 }
Philippe Mathieu-Daudé8940ea02022-12-19 23:09:23 +01003070 } while (*++ct_str != '\0');
bellardc896fe22008-02-01 10:05:41 +00003071 }
3072
Stefan Weilc68aaa12010-02-15 17:17:21 +01003073 /* TCGTargetOpDef entry with too much information? */
Aurelien Jarnoeabb7b92016-04-21 10:48:49 +02003074 tcg_debug_assert(i == TCG_MAX_OP_ARGS || tdefs->args_ct_str[i] == NULL);
Stefan Weilc68aaa12010-02-15 17:17:21 +01003075
Richard Henderson29f5e922022-10-14 07:37:38 +11003076 /*
3077 * Fix up output pairs that are aliased with inputs.
3078 * When we created the alias, we copied pair from the output.
3079 * There are three cases:
3080 * (1a) Pairs of inputs alias pairs of outputs.
3081 * (1b) One input aliases the first of a pair of outputs.
3082 * (2) One input aliases the second of a pair of outputs.
3083 *
3084 * Case 1a is handled by making sure that the pair_index'es are
3085 * properly updated so that they appear the same as a pair of inputs.
3086 *
3087 * Case 1b is handled by setting the pair_index of the input to
3088 * itself, simply so it doesn't point to an unrelated argument.
3089 * Since we don't encounter the "second" during the input allocation
3090 * phase, nothing happens with the second half of the input pair.
3091 *
3092 * Case 2 is handled by setting the second input to pair=3, the
3093 * first output to pair=3, and the pair_index'es to match.
3094 */
3095 if (saw_alias_pair) {
3096 for (i = def->nb_oargs; i < nb_args; i++) {
3097 /*
3098 * Since [0-9pm] must be alone in the constraint string,
3099 * the only way they can both be set is if the pair comes
3100 * from the output alias.
3101 */
3102 if (!def->args_ct[i].ialias) {
3103 continue;
3104 }
3105 switch (def->args_ct[i].pair) {
3106 case 0:
3107 break;
3108 case 1:
3109 o = def->args_ct[i].alias_index;
3110 o2 = def->args_ct[o].pair_index;
3111 tcg_debug_assert(def->args_ct[o].pair == 1);
3112 tcg_debug_assert(def->args_ct[o2].pair == 2);
3113 if (def->args_ct[o2].oalias) {
3114 /* Case 1a */
3115 i2 = def->args_ct[o2].alias_index;
3116 tcg_debug_assert(def->args_ct[i2].pair == 2);
3117 def->args_ct[i2].pair_index = i;
3118 def->args_ct[i].pair_index = i2;
3119 } else {
3120 /* Case 1b */
3121 def->args_ct[i].pair_index = i;
3122 }
3123 break;
3124 case 2:
3125 o = def->args_ct[i].alias_index;
3126 o2 = def->args_ct[o].pair_index;
3127 tcg_debug_assert(def->args_ct[o].pair == 2);
3128 tcg_debug_assert(def->args_ct[o2].pair == 1);
3129 if (def->args_ct[o2].oalias) {
3130 /* Case 1a */
3131 i2 = def->args_ct[o2].alias_index;
3132 tcg_debug_assert(def->args_ct[i2].pair == 1);
3133 def->args_ct[i2].pair_index = i;
3134 def->args_ct[i].pair_index = i2;
3135 } else {
3136 /* Case 2 */
3137 def->args_ct[i].pair = 3;
3138 def->args_ct[o2].pair = 3;
3139 def->args_ct[i].pair_index = o2;
3140 def->args_ct[o2].pair_index = i;
3141 }
3142 break;
3143 default:
3144 g_assert_not_reached();
3145 }
3146 }
3147 }
3148
bellardc896fe22008-02-01 10:05:41 +00003149 /* sort the constraints (XXX: this is just an heuristic) */
3150 sort_constraints(def, 0, def->nb_oargs);
3151 sort_constraints(def, def->nb_oargs, def->nb_iargs);
bellardc896fe22008-02-01 10:05:41 +00003152 }
bellardc896fe22008-02-01 10:05:41 +00003153}
3154
Richard Hendersonf85b1fc2023-03-03 13:47:27 -08003155static void remove_label_use(TCGOp *op, int idx)
3156{
3157 TCGLabel *label = arg_label(op->args[idx]);
3158 TCGLabelUse *use;
3159
3160 QSIMPLEQ_FOREACH(use, &label->branches, next) {
3161 if (use->op == op) {
3162 QSIMPLEQ_REMOVE(&label->branches, use, TCGLabelUse, next);
3163 return;
3164 }
3165 }
3166 g_assert_not_reached();
3167}
3168
Richard Henderson0c627cd2014-03-30 16:51:54 -07003169void tcg_op_remove(TCGContext *s, TCGOp *op)
3170{
Richard Hendersond88a1172018-11-26 12:47:28 -08003171 switch (op->opc) {
3172 case INDEX_op_br:
Richard Hendersonf85b1fc2023-03-03 13:47:27 -08003173 remove_label_use(op, 0);
Richard Hendersond88a1172018-11-26 12:47:28 -08003174 break;
3175 case INDEX_op_brcond_i32:
3176 case INDEX_op_brcond_i64:
Richard Hendersonf85b1fc2023-03-03 13:47:27 -08003177 remove_label_use(op, 3);
Richard Hendersond88a1172018-11-26 12:47:28 -08003178 break;
3179 case INDEX_op_brcond2_i32:
Richard Hendersonf85b1fc2023-03-03 13:47:27 -08003180 remove_label_use(op, 5);
Richard Hendersond88a1172018-11-26 12:47:28 -08003181 break;
3182 default:
3183 break;
3184 }
3185
Richard Henderson15fa08f2017-11-02 15:19:14 +01003186 QTAILQ_REMOVE(&s->ops, op, link);
3187 QTAILQ_INSERT_TAIL(&s->free_ops, op, link);
Richard Hendersonabebf922018-05-08 19:18:59 +00003188 s->nb_ops--;
Richard Henderson0c627cd2014-03-30 16:51:54 -07003189}
3190
Richard Hendersona80cdd32021-06-04 14:26:45 -07003191void tcg_remove_ops_after(TCGOp *op)
3192{
3193 TCGContext *s = tcg_ctx;
3194
3195 while (true) {
3196 TCGOp *last = tcg_last_op();
3197 if (last == op) {
3198 return;
3199 }
3200 tcg_op_remove(s, last);
3201 }
3202}
3203
Philippe Mathieu-Daudéd4478942022-12-18 22:18:31 +01003204static TCGOp *tcg_op_alloc(TCGOpcode opc, unsigned nargs)
Richard Henderson15fa08f2017-11-02 15:19:14 +01003205{
3206 TCGContext *s = tcg_ctx;
Richard Hendersoncb10bc62022-12-18 22:18:32 +01003207 TCGOp *op = NULL;
Richard Henderson15fa08f2017-11-02 15:19:14 +01003208
Richard Hendersoncb10bc62022-12-18 22:18:32 +01003209 if (unlikely(!QTAILQ_EMPTY(&s->free_ops))) {
3210 QTAILQ_FOREACH(op, &s->free_ops, link) {
3211 if (nargs <= op->nargs) {
3212 QTAILQ_REMOVE(&s->free_ops, op, link);
3213 nargs = op->nargs;
3214 goto found;
3215 }
3216 }
Richard Henderson15fa08f2017-11-02 15:19:14 +01003217 }
Richard Hendersoncb10bc62022-12-18 22:18:32 +01003218
3219 /* Most opcodes have 3 or 4 operands: reduce fragmentation. */
3220 nargs = MAX(4, nargs);
3221 op = tcg_malloc(sizeof(TCGOp) + sizeof(TCGArg) * nargs);
3222
3223 found:
Richard Henderson15fa08f2017-11-02 15:19:14 +01003224 memset(op, 0, offsetof(TCGOp, link));
3225 op->opc = opc;
Richard Hendersoncb10bc62022-12-18 22:18:32 +01003226 op->nargs = nargs;
Richard Henderson15fa08f2017-11-02 15:19:14 +01003227
Richard Hendersoncb10bc62022-12-18 22:18:32 +01003228 /* Check for bitfield overflow. */
3229 tcg_debug_assert(op->nargs == nargs);
3230
3231 s->nb_ops++;
Richard Henderson15fa08f2017-11-02 15:19:14 +01003232 return op;
3233}
3234
Philippe Mathieu-Daudéd4478942022-12-18 22:18:31 +01003235TCGOp *tcg_emit_op(TCGOpcode opc, unsigned nargs)
Richard Henderson15fa08f2017-11-02 15:19:14 +01003236{
Philippe Mathieu-Daudéd4478942022-12-18 22:18:31 +01003237 TCGOp *op = tcg_op_alloc(opc, nargs);
Richard Henderson07843f72024-03-13 13:32:29 -10003238
3239 if (tcg_ctx->emit_before_op) {
3240 QTAILQ_INSERT_BEFORE(tcg_ctx->emit_before_op, op, link);
3241 } else {
3242 QTAILQ_INSERT_TAIL(&tcg_ctx->ops, op, link);
3243 }
Richard Henderson15fa08f2017-11-02 15:19:14 +01003244 return op;
3245}
3246
Philippe Mathieu-Daudéd4478942022-12-18 22:18:31 +01003247TCGOp *tcg_op_insert_before(TCGContext *s, TCGOp *old_op,
3248 TCGOpcode opc, unsigned nargs)
Richard Henderson5a184072016-06-23 20:34:33 -07003249{
Philippe Mathieu-Daudéd4478942022-12-18 22:18:31 +01003250 TCGOp *new_op = tcg_op_alloc(opc, nargs);
Richard Henderson15fa08f2017-11-02 15:19:14 +01003251 QTAILQ_INSERT_BEFORE(old_op, new_op, link);
Richard Henderson5a184072016-06-23 20:34:33 -07003252 return new_op;
3253}
3254
Philippe Mathieu-Daudéd4478942022-12-18 22:18:31 +01003255TCGOp *tcg_op_insert_after(TCGContext *s, TCGOp *old_op,
3256 TCGOpcode opc, unsigned nargs)
Richard Henderson5a184072016-06-23 20:34:33 -07003257{
Philippe Mathieu-Daudéd4478942022-12-18 22:18:31 +01003258 TCGOp *new_op = tcg_op_alloc(opc, nargs);
Richard Henderson15fa08f2017-11-02 15:19:14 +01003259 QTAILQ_INSERT_AFTER(&s->ops, old_op, new_op, link);
Richard Henderson5a184072016-06-23 20:34:33 -07003260 return new_op;
3261}
3262
Richard Henderson968f3052023-03-03 14:22:02 -08003263static void move_label_uses(TCGLabel *to, TCGLabel *from)
3264{
3265 TCGLabelUse *u;
3266
3267 QSIMPLEQ_FOREACH(u, &from->branches, next) {
3268 TCGOp *op = u->op;
3269 switch (op->opc) {
3270 case INDEX_op_br:
3271 op->args[0] = label_arg(to);
3272 break;
3273 case INDEX_op_brcond_i32:
3274 case INDEX_op_brcond_i64:
3275 op->args[3] = label_arg(to);
3276 break;
3277 case INDEX_op_brcond2_i32:
3278 op->args[5] = label_arg(to);
3279 break;
3280 default:
3281 g_assert_not_reached();
3282 }
3283 }
3284
3285 QSIMPLEQ_CONCAT(&to->branches, &from->branches);
3286}
3287
Richard Hendersonb4fc67c2018-11-26 14:28:28 -08003288/* Reachable analysis : remove unreachable code. */
Richard Henderson9bbee4c2023-02-24 12:07:33 -10003289static void __attribute__((noinline))
3290reachable_code_pass(TCGContext *s)
Richard Hendersonb4fc67c2018-11-26 14:28:28 -08003291{
Richard Henderson4d89d0b2023-01-29 10:37:19 -10003292 TCGOp *op, *op_next, *op_prev;
Richard Hendersonb4fc67c2018-11-26 14:28:28 -08003293 bool dead = false;
3294
3295 QTAILQ_FOREACH_SAFE(op, &s->ops, link, op_next) {
3296 bool remove = dead;
3297 TCGLabel *label;
Richard Hendersonb4fc67c2018-11-26 14:28:28 -08003298
3299 switch (op->opc) {
3300 case INDEX_op_set_label:
3301 label = arg_label(op->args[0]);
Richard Henderson4d89d0b2023-01-29 10:37:19 -10003302
3303 /*
Richard Henderson968f3052023-03-03 14:22:02 -08003304 * Note that the first op in the TB is always a load,
3305 * so there is always something before a label.
3306 */
3307 op_prev = QTAILQ_PREV(op, link);
3308
3309 /*
3310 * If we find two sequential labels, move all branches to
3311 * reference the second label and remove the first label.
3312 * Do this before branch to next optimization, so that the
3313 * middle label is out of the way.
3314 */
3315 if (op_prev->opc == INDEX_op_set_label) {
3316 move_label_uses(label, arg_label(op_prev->args[0]));
3317 tcg_op_remove(s, op_prev);
3318 op_prev = QTAILQ_PREV(op, link);
3319 }
3320
3321 /*
Richard Henderson4d89d0b2023-01-29 10:37:19 -10003322 * Optimization can fold conditional branches to unconditional.
3323 * If we find a label which is preceded by an unconditional
3324 * branch to next, remove the branch. We couldn't do this when
3325 * processing the branch because any dead code between the branch
3326 * and label had not yet been removed.
3327 */
Richard Henderson4d89d0b2023-01-29 10:37:19 -10003328 if (op_prev->opc == INDEX_op_br &&
3329 label == arg_label(op_prev->args[0])) {
3330 tcg_op_remove(s, op_prev);
3331 /* Fall through means insns become live again. */
3332 dead = false;
3333 }
3334
Richard Hendersonf85b1fc2023-03-03 13:47:27 -08003335 if (QSIMPLEQ_EMPTY(&label->branches)) {
Richard Hendersonb4fc67c2018-11-26 14:28:28 -08003336 /*
3337 * While there is an occasional backward branch, virtually
3338 * all branches generated by the translators are forward.
3339 * Which means that generally we will have already removed
3340 * all references to the label that will be, and there is
3341 * little to be gained by iterating.
3342 */
3343 remove = true;
3344 } else {
3345 /* Once we see a label, insns become live again. */
3346 dead = false;
3347 remove = false;
Richard Hendersonb4fc67c2018-11-26 14:28:28 -08003348 }
3349 break;
3350
3351 case INDEX_op_br:
3352 case INDEX_op_exit_tb:
3353 case INDEX_op_goto_ptr:
3354 /* Unconditional branches; everything following is dead. */
3355 dead = true;
3356 break;
3357
3358 case INDEX_op_call:
3359 /* Notice noreturn helper calls, raising exceptions. */
Richard Henderson90163902021-03-18 10:21:45 -06003360 if (tcg_call_flags(op) & TCG_CALL_NO_RETURN) {
Richard Hendersonb4fc67c2018-11-26 14:28:28 -08003361 dead = true;
3362 }
3363 break;
3364
3365 case INDEX_op_insn_start:
3366 /* Never remove -- we need to keep these for unwind. */
3367 remove = false;
3368 break;
3369
3370 default:
3371 break;
3372 }
3373
3374 if (remove) {
3375 tcg_op_remove(s, op);
3376 }
3377 }
3378}
3379
Richard Hendersonc70fbf02016-06-23 20:34:22 -07003380#define TS_DEAD 1
3381#define TS_MEM 2
3382
Richard Henderson5a184072016-06-23 20:34:33 -07003383#define IS_DEAD_ARG(n) (arg_life & (DEAD_ARG << (n)))
3384#define NEED_SYNC_ARG(n) (arg_life & (SYNC_ARG << (n)))
3385
Richard Henderson25f49c52018-11-27 12:45:26 -08003386/* For liveness_pass_1, the register preferences for a given temp. */
3387static inline TCGRegSet *la_temp_pref(TCGTemp *ts)
3388{
3389 return ts->state_ptr;
3390}
3391
3392/* For liveness_pass_1, reset the preferences for a given temp to the
3393 * maximal regset for its type.
3394 */
3395static inline void la_reset_pref(TCGTemp *ts)
3396{
3397 *la_temp_pref(ts)
3398 = (ts->state == TS_DEAD ? 0 : tcg_target_available_regs[ts->type]);
3399}
3400
Aurelien Jarno9c43b682012-10-09 21:53:07 +02003401/* liveness analysis: end of function: all temps are dead, and globals
3402 should be in memory. */
Richard Henderson2616c802018-11-27 13:37:24 -08003403static void la_func_end(TCGContext *s, int ng, int nt)
bellardc896fe22008-02-01 10:05:41 +00003404{
Richard Hendersonb83eabe2016-11-01 15:56:04 -06003405 int i;
3406
3407 for (i = 0; i < ng; ++i) {
3408 s->temps[i].state = TS_DEAD | TS_MEM;
Richard Henderson25f49c52018-11-27 12:45:26 -08003409 la_reset_pref(&s->temps[i]);
Richard Hendersonb83eabe2016-11-01 15:56:04 -06003410 }
3411 for (i = ng; i < nt; ++i) {
3412 s->temps[i].state = TS_DEAD;
Richard Henderson25f49c52018-11-27 12:45:26 -08003413 la_reset_pref(&s->temps[i]);
Richard Hendersonb83eabe2016-11-01 15:56:04 -06003414 }
bellardc896fe22008-02-01 10:05:41 +00003415}
3416
Aurelien Jarno9c43b682012-10-09 21:53:07 +02003417/* liveness analysis: end of basic block: all temps are dead, globals
3418 and local temps should be in memory. */
Richard Henderson2616c802018-11-27 13:37:24 -08003419static void la_bb_end(TCGContext *s, int ng, int nt)
bellard641d5fb2008-05-25 17:24:00 +00003420{
Richard Hendersonb83eabe2016-11-01 15:56:04 -06003421 int i;
bellard641d5fb2008-05-25 17:24:00 +00003422
Richard Hendersonee17db82020-03-29 10:11:56 -07003423 for (i = 0; i < nt; ++i) {
3424 TCGTemp *ts = &s->temps[i];
3425 int state;
3426
3427 switch (ts->kind) {
3428 case TEMP_FIXED:
3429 case TEMP_GLOBAL:
Richard Hendersonf57c6912023-01-29 10:55:52 -10003430 case TEMP_TB:
Richard Hendersonee17db82020-03-29 10:11:56 -07003431 state = TS_DEAD | TS_MEM;
3432 break;
Richard Hendersonc7482432022-03-16 09:34:18 -07003433 case TEMP_EBB:
Richard Hendersonc0522132020-03-29 18:55:52 -07003434 case TEMP_CONST:
Richard Hendersonee17db82020-03-29 10:11:56 -07003435 state = TS_DEAD;
3436 break;
3437 default:
3438 g_assert_not_reached();
3439 }
3440 ts->state = state;
3441 la_reset_pref(ts);
bellard641d5fb2008-05-25 17:24:00 +00003442 }
3443}
3444
Richard Hendersonf65a0612018-11-27 14:00:35 -08003445/* liveness analysis: sync globals back to memory. */
3446static void la_global_sync(TCGContext *s, int ng)
3447{
3448 int i;
3449
3450 for (i = 0; i < ng; ++i) {
Richard Henderson25f49c52018-11-27 12:45:26 -08003451 int state = s->temps[i].state;
3452 s->temps[i].state = state | TS_MEM;
3453 if (state == TS_DEAD) {
3454 /* If the global was previously dead, reset prefs. */
3455 la_reset_pref(&s->temps[i]);
3456 }
Richard Hendersonf65a0612018-11-27 14:00:35 -08003457 }
3458}
3459
Richard Hendersonb4cb76e2020-10-08 15:21:43 -05003460/*
Richard Hendersonc7482432022-03-16 09:34:18 -07003461 * liveness analysis: conditional branch: all temps are dead unless
3462 * explicitly live-across-conditional-branch, globals and local temps
3463 * should be synced.
Richard Hendersonb4cb76e2020-10-08 15:21:43 -05003464 */
3465static void la_bb_sync(TCGContext *s, int ng, int nt)
3466{
3467 la_global_sync(s, ng);
3468
3469 for (int i = ng; i < nt; ++i) {
Richard Hendersonc0522132020-03-29 18:55:52 -07003470 TCGTemp *ts = &s->temps[i];
3471 int state;
3472
3473 switch (ts->kind) {
Richard Hendersonf57c6912023-01-29 10:55:52 -10003474 case TEMP_TB:
Richard Hendersonc0522132020-03-29 18:55:52 -07003475 state = ts->state;
3476 ts->state = state | TS_MEM;
Richard Hendersonb4cb76e2020-10-08 15:21:43 -05003477 if (state != TS_DEAD) {
3478 continue;
3479 }
Richard Hendersonc0522132020-03-29 18:55:52 -07003480 break;
Richard Hendersonc7482432022-03-16 09:34:18 -07003481 case TEMP_EBB:
Richard Hendersonc0522132020-03-29 18:55:52 -07003482 case TEMP_CONST:
3483 continue;
3484 default:
3485 g_assert_not_reached();
Richard Hendersonb4cb76e2020-10-08 15:21:43 -05003486 }
3487 la_reset_pref(&s->temps[i]);
3488 }
3489}
3490
Richard Hendersonf65a0612018-11-27 14:00:35 -08003491/* liveness analysis: sync globals back to memory and kill. */
3492static void la_global_kill(TCGContext *s, int ng)
3493{
3494 int i;
3495
3496 for (i = 0; i < ng; i++) {
3497 s->temps[i].state = TS_DEAD | TS_MEM;
Richard Henderson25f49c52018-11-27 12:45:26 -08003498 la_reset_pref(&s->temps[i]);
3499 }
3500}
3501
3502/* liveness analysis: note live globals crossing calls. */
3503static void la_cross_call(TCGContext *s, int nt)
3504{
3505 TCGRegSet mask = ~tcg_target_call_clobber_regs;
3506 int i;
3507
3508 for (i = 0; i < nt; i++) {
3509 TCGTemp *ts = &s->temps[i];
3510 if (!(ts->state & TS_DEAD)) {
3511 TCGRegSet *pset = la_temp_pref(ts);
3512 TCGRegSet set = *pset;
3513
3514 set &= mask;
3515 /* If the combination is not possible, restart. */
3516 if (set == 0) {
3517 set = tcg_target_available_regs[ts->type] & mask;
3518 }
3519 *pset = set;
3520 }
Richard Hendersonf65a0612018-11-27 14:00:35 -08003521 }
3522}
3523
Richard Henderson874b8572023-01-29 11:50:20 -10003524/*
3525 * Liveness analysis: Verify the lifetime of TEMP_TB, and reduce
3526 * to TEMP_EBB, if possible.
3527 */
3528static void __attribute__((noinline))
3529liveness_pass_0(TCGContext *s)
3530{
3531 void * const multiple_ebb = (void *)(uintptr_t)-1;
3532 int nb_temps = s->nb_temps;
3533 TCGOp *op, *ebb;
3534
3535 for (int i = s->nb_globals; i < nb_temps; ++i) {
3536 s->temps[i].state_ptr = NULL;
3537 }
3538
3539 /*
3540 * Represent each EBB by the op at which it begins. In the case of
3541 * the first EBB, this is the first op, otherwise it is a label.
3542 * Collect the uses of each TEMP_TB: NULL for unused, EBB for use
3543 * within a single EBB, else MULTIPLE_EBB.
3544 */
3545 ebb = QTAILQ_FIRST(&s->ops);
3546 QTAILQ_FOREACH(op, &s->ops, link) {
3547 const TCGOpDef *def;
3548 int nb_oargs, nb_iargs;
3549
3550 switch (op->opc) {
3551 case INDEX_op_set_label:
3552 ebb = op;
3553 continue;
3554 case INDEX_op_discard:
3555 continue;
3556 case INDEX_op_call:
3557 nb_oargs = TCGOP_CALLO(op);
3558 nb_iargs = TCGOP_CALLI(op);
3559 break;
3560 default:
3561 def = &tcg_op_defs[op->opc];
3562 nb_oargs = def->nb_oargs;
3563 nb_iargs = def->nb_iargs;
3564 break;
3565 }
3566
3567 for (int i = 0; i < nb_oargs + nb_iargs; ++i) {
3568 TCGTemp *ts = arg_temp(op->args[i]);
3569
3570 if (ts->kind != TEMP_TB) {
3571 continue;
3572 }
3573 if (ts->state_ptr == NULL) {
3574 ts->state_ptr = ebb;
3575 } else if (ts->state_ptr != ebb) {
3576 ts->state_ptr = multiple_ebb;
3577 }
3578 }
3579 }
3580
3581 /*
3582 * For TEMP_TB that turned out not to be used beyond one EBB,
3583 * reduce the liveness to TEMP_EBB.
3584 */
3585 for (int i = s->nb_globals; i < nb_temps; ++i) {
3586 TCGTemp *ts = &s->temps[i];
3587 if (ts->kind == TEMP_TB && ts->state_ptr != multiple_ebb) {
3588 ts->kind = TEMP_EBB;
3589 }
3590 }
3591}
3592
Richard Hendersona1b3c482016-06-22 15:46:09 -07003593/* Liveness analysis : update the opc_arg_life array to tell if a
bellardc896fe22008-02-01 10:05:41 +00003594 given input arguments is dead. Instructions updating dead
3595 temporaries are removed. */
Richard Henderson9bbee4c2023-02-24 12:07:33 -10003596static void __attribute__((noinline))
3597liveness_pass_1(TCGContext *s)
bellardc896fe22008-02-01 10:05:41 +00003598{
Richard Hendersonc70fbf02016-06-23 20:34:22 -07003599 int nb_globals = s->nb_globals;
Richard Henderson2616c802018-11-27 13:37:24 -08003600 int nb_temps = s->nb_temps;
Richard Henderson15fa08f2017-11-02 15:19:14 +01003601 TCGOp *op, *op_prev;
Richard Henderson25f49c52018-11-27 12:45:26 -08003602 TCGRegSet *prefs;
3603 int i;
3604
3605 prefs = tcg_malloc(sizeof(TCGRegSet) * nb_temps);
3606 for (i = 0; i < nb_temps; ++i) {
3607 s->temps[i].state_ptr = prefs + i;
3608 }
Richard Hendersona1b3c482016-06-22 15:46:09 -07003609
Richard Hendersonae36a242018-11-27 13:45:08 -08003610 /* ??? Should be redundant with the exit_tb that ends the TB. */
Richard Henderson2616c802018-11-27 13:37:24 -08003611 la_func_end(s, nb_globals, nb_temps);
bellardc896fe22008-02-01 10:05:41 +00003612
Paolo Bonzinieae3eb32018-12-06 13:10:34 +01003613 QTAILQ_FOREACH_REVERSE_SAFE(op, &s->ops, link, op_prev) {
Richard Henderson25f49c52018-11-27 12:45:26 -08003614 int nb_iargs, nb_oargs;
Richard Hendersonc45cb8b2014-09-19 13:49:15 -07003615 TCGOpcode opc_new, opc_new2;
3616 bool have_opc_new2;
Richard Hendersona1b3c482016-06-22 15:46:09 -07003617 TCGLifeData arg_life = 0;
Richard Henderson25f49c52018-11-27 12:45:26 -08003618 TCGTemp *ts;
Richard Hendersonc45cb8b2014-09-19 13:49:15 -07003619 TCGOpcode opc = op->opc;
3620 const TCGOpDef *def = &tcg_op_defs[opc];
3621
Richard Hendersonc45cb8b2014-09-19 13:49:15 -07003622 switch (opc) {
bellardc896fe22008-02-01 10:05:41 +00003623 case INDEX_op_call:
bellardc6e113f2008-05-17 12:42:15 +00003624 {
Richard Henderson39004a72022-11-11 10:09:37 +10003625 const TCGHelperInfo *info = tcg_call_info(op);
3626 int call_flags = tcg_call_flags(op);
bellardc896fe22008-02-01 10:05:41 +00003627
Richard Hendersoncd9090a2017-11-14 13:02:51 +01003628 nb_oargs = TCGOP_CALLO(op);
3629 nb_iargs = TCGOP_CALLI(op);
bellardc896fe22008-02-01 10:05:41 +00003630
Richard Hendersonc45cb8b2014-09-19 13:49:15 -07003631 /* pure functions can be removed if their result is unused */
Aurelien Jarno78505272012-10-09 21:53:08 +02003632 if (call_flags & TCG_CALL_NO_SIDE_EFFECTS) {
Richard Hendersoncf066672014-03-22 20:06:52 -07003633 for (i = 0; i < nb_oargs; i++) {
Richard Henderson25f49c52018-11-27 12:45:26 -08003634 ts = arg_temp(op->args[i]);
3635 if (ts->state != TS_DEAD) {
bellardc6e113f2008-05-17 12:42:15 +00003636 goto do_not_remove_call;
Aurelien Jarno9c43b682012-10-09 21:53:07 +02003637 }
bellardc6e113f2008-05-17 12:42:15 +00003638 }
Richard Hendersonc45cb8b2014-09-19 13:49:15 -07003639 goto do_remove;
Richard Henderson152c35a2018-11-27 13:32:33 -08003640 }
3641 do_not_remove_call:
bellardc6e113f2008-05-17 12:42:15 +00003642
Richard Henderson25f49c52018-11-27 12:45:26 -08003643 /* Output args are dead. */
Richard Henderson152c35a2018-11-27 13:32:33 -08003644 for (i = 0; i < nb_oargs; i++) {
Richard Henderson25f49c52018-11-27 12:45:26 -08003645 ts = arg_temp(op->args[i]);
3646 if (ts->state & TS_DEAD) {
Richard Henderson152c35a2018-11-27 13:32:33 -08003647 arg_life |= DEAD_ARG << i;
bellardc6e113f2008-05-17 12:42:15 +00003648 }
Richard Henderson25f49c52018-11-27 12:45:26 -08003649 if (ts->state & TS_MEM) {
Richard Henderson152c35a2018-11-27 13:32:33 -08003650 arg_life |= SYNC_ARG << i;
3651 }
Richard Henderson25f49c52018-11-27 12:45:26 -08003652 ts->state = TS_DEAD;
3653 la_reset_pref(ts);
Richard Henderson152c35a2018-11-27 13:32:33 -08003654 }
Aurelien Jarno78505272012-10-09 21:53:08 +02003655
Richard Henderson31fd8842022-11-11 15:10:51 +10003656 /* Not used -- it will be tcg_target_call_oarg_reg(). */
3657 memset(op->output_pref, 0, sizeof(op->output_pref));
3658
Richard Henderson152c35a2018-11-27 13:32:33 -08003659 if (!(call_flags & (TCG_CALL_NO_WRITE_GLOBALS |
3660 TCG_CALL_NO_READ_GLOBALS))) {
Richard Hendersonf65a0612018-11-27 14:00:35 -08003661 la_global_kill(s, nb_globals);
Richard Henderson152c35a2018-11-27 13:32:33 -08003662 } else if (!(call_flags & TCG_CALL_NO_READ_GLOBALS)) {
Richard Hendersonf65a0612018-11-27 14:00:35 -08003663 la_global_sync(s, nb_globals);
Richard Henderson152c35a2018-11-27 13:32:33 -08003664 }
aurel32b9c18f52009-04-06 12:33:59 +00003665
Richard Henderson25f49c52018-11-27 12:45:26 -08003666 /* Record arguments that die in this helper. */
Richard Henderson152c35a2018-11-27 13:32:33 -08003667 for (i = nb_oargs; i < nb_iargs + nb_oargs; i++) {
Richard Henderson25f49c52018-11-27 12:45:26 -08003668 ts = arg_temp(op->args[i]);
Richard Henderson39004a72022-11-11 10:09:37 +10003669 if (ts->state & TS_DEAD) {
Richard Henderson152c35a2018-11-27 13:32:33 -08003670 arg_life |= DEAD_ARG << i;
bellardc6e113f2008-05-17 12:42:15 +00003671 }
Richard Henderson152c35a2018-11-27 13:32:33 -08003672 }
Richard Henderson25f49c52018-11-27 12:45:26 -08003673
3674 /* For all live registers, remove call-clobbered prefs. */
3675 la_cross_call(s, nb_temps);
3676
Richard Henderson39004a72022-11-11 10:09:37 +10003677 /*
3678 * Input arguments are live for preceding opcodes.
3679 *
3680 * For those arguments that die, and will be allocated in
3681 * registers, clear the register set for that arg, to be
3682 * filled in below. For args that will be on the stack,
3683 * reset to any available reg. Process arguments in reverse
3684 * order so that if a temp is used more than once, the stack
3685 * reset to max happens before the register reset to 0.
3686 */
3687 for (i = nb_iargs - 1; i >= 0; i--) {
3688 const TCGCallArgumentLoc *loc = &info->in[i];
3689 ts = arg_temp(op->args[nb_oargs + i]);
Richard Henderson25f49c52018-11-27 12:45:26 -08003690
Richard Henderson39004a72022-11-11 10:09:37 +10003691 if (ts->state & TS_DEAD) {
3692 switch (loc->kind) {
3693 case TCG_CALL_ARG_NORMAL:
3694 case TCG_CALL_ARG_EXTEND_U:
3695 case TCG_CALL_ARG_EXTEND_S:
Richard Henderson338b61e2023-04-08 17:28:07 -07003696 if (arg_slot_reg_p(loc->arg_slot)) {
Richard Henderson39004a72022-11-11 10:09:37 +10003697 *la_temp_pref(ts) = 0;
3698 break;
3699 }
3700 /* fall through */
3701 default:
3702 *la_temp_pref(ts) =
3703 tcg_target_available_regs[ts->type];
3704 break;
3705 }
Richard Henderson25f49c52018-11-27 12:45:26 -08003706 ts->state &= ~TS_DEAD;
3707 }
3708 }
3709
Richard Henderson39004a72022-11-11 10:09:37 +10003710 /*
3711 * For each input argument, add its input register to prefs.
3712 * If a temp is used once, this produces a single set bit;
3713 * if a temp is used multiple times, this produces a set.
3714 */
3715 for (i = 0; i < nb_iargs; i++) {
3716 const TCGCallArgumentLoc *loc = &info->in[i];
3717 ts = arg_temp(op->args[nb_oargs + i]);
3718
3719 switch (loc->kind) {
3720 case TCG_CALL_ARG_NORMAL:
3721 case TCG_CALL_ARG_EXTEND_U:
3722 case TCG_CALL_ARG_EXTEND_S:
Richard Henderson338b61e2023-04-08 17:28:07 -07003723 if (arg_slot_reg_p(loc->arg_slot)) {
Richard Henderson39004a72022-11-11 10:09:37 +10003724 tcg_regset_set_reg(*la_temp_pref(ts),
3725 tcg_target_call_iarg_regs[loc->arg_slot]);
3726 }
3727 break;
3728 default:
3729 break;
Aurelien Jarnoc19f47b2015-06-04 21:47:08 +02003730 }
bellardc896fe22008-02-01 10:05:41 +00003731 }
bellardc896fe22008-02-01 10:05:41 +00003732 }
bellardc896fe22008-02-01 10:05:41 +00003733 break;
Richard Henderson765b8422015-08-29 12:37:33 -07003734 case INDEX_op_insn_start:
bellardc896fe22008-02-01 10:05:41 +00003735 break;
bellard5ff9d6a2008-02-04 00:37:54 +00003736 case INDEX_op_discard:
bellard5ff9d6a2008-02-04 00:37:54 +00003737 /* mark the temporary as dead */
Richard Henderson25f49c52018-11-27 12:45:26 -08003738 ts = arg_temp(op->args[0]);
3739 ts->state = TS_DEAD;
3740 la_reset_pref(ts);
bellard5ff9d6a2008-02-04 00:37:54 +00003741 break;
Richard Henderson1305c452012-10-02 11:32:29 -07003742
3743 case INDEX_op_add2_i32:
Richard Hendersonc45cb8b2014-09-19 13:49:15 -07003744 opc_new = INDEX_op_add_i32;
Richard Hendersonf1fae402013-02-19 23:52:02 -08003745 goto do_addsub2;
Richard Henderson1305c452012-10-02 11:32:29 -07003746 case INDEX_op_sub2_i32:
Richard Hendersonc45cb8b2014-09-19 13:49:15 -07003747 opc_new = INDEX_op_sub_i32;
Richard Hendersonf1fae402013-02-19 23:52:02 -08003748 goto do_addsub2;
3749 case INDEX_op_add2_i64:
Richard Hendersonc45cb8b2014-09-19 13:49:15 -07003750 opc_new = INDEX_op_add_i64;
Richard Hendersonf1fae402013-02-19 23:52:02 -08003751 goto do_addsub2;
3752 case INDEX_op_sub2_i64:
Richard Hendersonc45cb8b2014-09-19 13:49:15 -07003753 opc_new = INDEX_op_sub_i64;
Richard Hendersonf1fae402013-02-19 23:52:02 -08003754 do_addsub2:
Richard Henderson1305c452012-10-02 11:32:29 -07003755 nb_iargs = 4;
3756 nb_oargs = 2;
3757 /* Test if the high part of the operation is dead, but not
3758 the low part. The result can be optimized to a simple
3759 add or sub. This happens often for x86_64 guest when the
3760 cpu mode is set to 32 bit. */
Richard Hendersonb83eabe2016-11-01 15:56:04 -06003761 if (arg_temp(op->args[1])->state == TS_DEAD) {
3762 if (arg_temp(op->args[0])->state == TS_DEAD) {
Richard Henderson1305c452012-10-02 11:32:29 -07003763 goto do_remove;
3764 }
Richard Hendersonc45cb8b2014-09-19 13:49:15 -07003765 /* Replace the opcode and adjust the args in place,
3766 leaving 3 unused args at the end. */
3767 op->opc = opc = opc_new;
Richard Hendersonefee3742016-12-08 13:12:08 -08003768 op->args[1] = op->args[2];
3769 op->args[2] = op->args[4];
Richard Henderson1305c452012-10-02 11:32:29 -07003770 /* Fall through and mark the single-word operation live. */
3771 nb_iargs = 2;
3772 nb_oargs = 1;
3773 }
3774 goto do_not_remove;
3775
Richard Henderson14149682012-10-02 11:32:30 -07003776 case INDEX_op_mulu2_i32:
Richard Hendersonc45cb8b2014-09-19 13:49:15 -07003777 opc_new = INDEX_op_mul_i32;
3778 opc_new2 = INDEX_op_muluh_i32;
3779 have_opc_new2 = TCG_TARGET_HAS_muluh_i32;
Richard Henderson03271522013-08-14 14:35:56 -07003780 goto do_mul2;
Richard Hendersonf1fae402013-02-19 23:52:02 -08003781 case INDEX_op_muls2_i32:
Richard Hendersonc45cb8b2014-09-19 13:49:15 -07003782 opc_new = INDEX_op_mul_i32;
3783 opc_new2 = INDEX_op_mulsh_i32;
3784 have_opc_new2 = TCG_TARGET_HAS_mulsh_i32;
Richard Hendersonf1fae402013-02-19 23:52:02 -08003785 goto do_mul2;
3786 case INDEX_op_mulu2_i64:
Richard Hendersonc45cb8b2014-09-19 13:49:15 -07003787 opc_new = INDEX_op_mul_i64;
3788 opc_new2 = INDEX_op_muluh_i64;
3789 have_opc_new2 = TCG_TARGET_HAS_muluh_i64;
Richard Henderson03271522013-08-14 14:35:56 -07003790 goto do_mul2;
Richard Hendersonf1fae402013-02-19 23:52:02 -08003791 case INDEX_op_muls2_i64:
Richard Hendersonc45cb8b2014-09-19 13:49:15 -07003792 opc_new = INDEX_op_mul_i64;
3793 opc_new2 = INDEX_op_mulsh_i64;
3794 have_opc_new2 = TCG_TARGET_HAS_mulsh_i64;
Richard Henderson03271522013-08-14 14:35:56 -07003795 goto do_mul2;
Richard Hendersonf1fae402013-02-19 23:52:02 -08003796 do_mul2:
Richard Henderson14149682012-10-02 11:32:30 -07003797 nb_iargs = 2;
3798 nb_oargs = 2;
Richard Hendersonb83eabe2016-11-01 15:56:04 -06003799 if (arg_temp(op->args[1])->state == TS_DEAD) {
3800 if (arg_temp(op->args[0])->state == TS_DEAD) {
Richard Henderson03271522013-08-14 14:35:56 -07003801 /* Both parts of the operation are dead. */
Richard Henderson14149682012-10-02 11:32:30 -07003802 goto do_remove;
3803 }
Richard Henderson03271522013-08-14 14:35:56 -07003804 /* The high part of the operation is dead; generate the low. */
Richard Hendersonc45cb8b2014-09-19 13:49:15 -07003805 op->opc = opc = opc_new;
Richard Hendersonefee3742016-12-08 13:12:08 -08003806 op->args[1] = op->args[2];
3807 op->args[2] = op->args[3];
Richard Hendersonb83eabe2016-11-01 15:56:04 -06003808 } else if (arg_temp(op->args[0])->state == TS_DEAD && have_opc_new2) {
Richard Hendersonc45cb8b2014-09-19 13:49:15 -07003809 /* The low part of the operation is dead; generate the high. */
3810 op->opc = opc = opc_new2;
Richard Hendersonefee3742016-12-08 13:12:08 -08003811 op->args[0] = op->args[1];
3812 op->args[1] = op->args[2];
3813 op->args[2] = op->args[3];
Richard Henderson03271522013-08-14 14:35:56 -07003814 } else {
3815 goto do_not_remove;
Richard Henderson14149682012-10-02 11:32:30 -07003816 }
Richard Henderson03271522013-08-14 14:35:56 -07003817 /* Mark the single-word operation live. */
3818 nb_oargs = 1;
Richard Henderson14149682012-10-02 11:32:30 -07003819 goto do_not_remove;
3820
bellardc896fe22008-02-01 10:05:41 +00003821 default:
Richard Henderson1305c452012-10-02 11:32:29 -07003822 /* XXX: optimize by hardcoding common cases (e.g. triadic ops) */
aurel3249516bc2008-12-07 18:15:45 +00003823 nb_iargs = def->nb_iargs;
3824 nb_oargs = def->nb_oargs;
bellardc896fe22008-02-01 10:05:41 +00003825
aurel3249516bc2008-12-07 18:15:45 +00003826 /* Test if the operation can be removed because all
3827 its outputs are dead. We assume that nb_oargs == 0
3828 implies side effects */
3829 if (!(def->flags & TCG_OPF_SIDE_EFFECTS) && nb_oargs != 0) {
Richard Hendersonc45cb8b2014-09-19 13:49:15 -07003830 for (i = 0; i < nb_oargs; i++) {
Richard Hendersonb83eabe2016-11-01 15:56:04 -06003831 if (arg_temp(op->args[i])->state != TS_DEAD) {
aurel3249516bc2008-12-07 18:15:45 +00003832 goto do_not_remove;
Aurelien Jarno9c43b682012-10-09 21:53:07 +02003833 }
bellardc896fe22008-02-01 10:05:41 +00003834 }
Richard Henderson152c35a2018-11-27 13:32:33 -08003835 goto do_remove;
3836 }
3837 goto do_not_remove;
aurel3249516bc2008-12-07 18:15:45 +00003838
Richard Henderson152c35a2018-11-27 13:32:33 -08003839 do_remove:
3840 tcg_op_remove(s, op);
3841 break;
aurel3249516bc2008-12-07 18:15:45 +00003842
Richard Henderson152c35a2018-11-27 13:32:33 -08003843 do_not_remove:
Richard Henderson152c35a2018-11-27 13:32:33 -08003844 for (i = 0; i < nb_oargs; i++) {
Richard Henderson25f49c52018-11-27 12:45:26 -08003845 ts = arg_temp(op->args[i]);
3846
3847 /* Remember the preference of the uses that followed. */
Richard Henderson31fd8842022-11-11 15:10:51 +10003848 if (i < ARRAY_SIZE(op->output_pref)) {
3849 op->output_pref[i] = *la_temp_pref(ts);
3850 }
Richard Henderson25f49c52018-11-27 12:45:26 -08003851
3852 /* Output args are dead. */
3853 if (ts->state & TS_DEAD) {
Richard Henderson152c35a2018-11-27 13:32:33 -08003854 arg_life |= DEAD_ARG << i;
Aurelien Jarnoc19f47b2015-06-04 21:47:08 +02003855 }
Richard Henderson25f49c52018-11-27 12:45:26 -08003856 if (ts->state & TS_MEM) {
Richard Henderson152c35a2018-11-27 13:32:33 -08003857 arg_life |= SYNC_ARG << i;
aurel3249516bc2008-12-07 18:15:45 +00003858 }
Richard Henderson25f49c52018-11-27 12:45:26 -08003859 ts->state = TS_DEAD;
3860 la_reset_pref(ts);
Richard Henderson152c35a2018-11-27 13:32:33 -08003861 }
3862
Richard Henderson25f49c52018-11-27 12:45:26 -08003863 /* If end of basic block, update. */
Richard Hendersonae36a242018-11-27 13:45:08 -08003864 if (def->flags & TCG_OPF_BB_EXIT) {
3865 la_func_end(s, nb_globals, nb_temps);
Richard Hendersonb4cb76e2020-10-08 15:21:43 -05003866 } else if (def->flags & TCG_OPF_COND_BRANCH) {
3867 la_bb_sync(s, nb_globals, nb_temps);
Richard Hendersonae36a242018-11-27 13:45:08 -08003868 } else if (def->flags & TCG_OPF_BB_END) {
Richard Henderson2616c802018-11-27 13:37:24 -08003869 la_bb_end(s, nb_globals, nb_temps);
Richard Henderson152c35a2018-11-27 13:32:33 -08003870 } else if (def->flags & TCG_OPF_SIDE_EFFECTS) {
Richard Hendersonf65a0612018-11-27 14:00:35 -08003871 la_global_sync(s, nb_globals);
Richard Henderson25f49c52018-11-27 12:45:26 -08003872 if (def->flags & TCG_OPF_CALL_CLOBBER) {
3873 la_cross_call(s, nb_temps);
3874 }
Richard Henderson152c35a2018-11-27 13:32:33 -08003875 }
3876
Richard Henderson25f49c52018-11-27 12:45:26 -08003877 /* Record arguments that die in this opcode. */
Richard Henderson152c35a2018-11-27 13:32:33 -08003878 for (i = nb_oargs; i < nb_oargs + nb_iargs; i++) {
Richard Henderson25f49c52018-11-27 12:45:26 -08003879 ts = arg_temp(op->args[i]);
3880 if (ts->state & TS_DEAD) {
Richard Henderson152c35a2018-11-27 13:32:33 -08003881 arg_life |= DEAD_ARG << i;
3882 }
3883 }
Richard Henderson25f49c52018-11-27 12:45:26 -08003884
3885 /* Input arguments are live for preceding opcodes. */
Richard Henderson152c35a2018-11-27 13:32:33 -08003886 for (i = nb_oargs; i < nb_oargs + nb_iargs; i++) {
Richard Henderson25f49c52018-11-27 12:45:26 -08003887 ts = arg_temp(op->args[i]);
3888 if (ts->state & TS_DEAD) {
3889 /* For operands that were dead, initially allow
3890 all regs for the type. */
3891 *la_temp_pref(ts) = tcg_target_available_regs[ts->type];
3892 ts->state &= ~TS_DEAD;
3893 }
3894 }
3895
3896 /* Incorporate constraints for this operand. */
3897 switch (opc) {
3898 case INDEX_op_mov_i32:
3899 case INDEX_op_mov_i64:
3900 /* Note that these are TCG_OPF_NOT_PRESENT and do not
3901 have proper constraints. That said, special case
3902 moves to propagate preferences backward. */
3903 if (IS_DEAD_ARG(1)) {
3904 *la_temp_pref(arg_temp(op->args[0]))
3905 = *la_temp_pref(arg_temp(op->args[1]));
3906 }
3907 break;
3908
3909 default:
3910 for (i = nb_oargs; i < nb_oargs + nb_iargs; i++) {
3911 const TCGArgConstraint *ct = &def->args_ct[i];
3912 TCGRegSet set, *pset;
3913
3914 ts = arg_temp(op->args[i]);
3915 pset = la_temp_pref(ts);
3916 set = *pset;
3917
Richard Henderson9be0d082020-09-03 15:19:03 -07003918 set &= ct->regs;
Richard Hendersonbc2b17e2019-04-04 19:34:19 -07003919 if (ct->ialias) {
Richard Henderson31fd8842022-11-11 15:10:51 +10003920 set &= output_pref(op, ct->alias_index);
Richard Henderson25f49c52018-11-27 12:45:26 -08003921 }
3922 /* If the combination is not possible, restart. */
3923 if (set == 0) {
Richard Henderson9be0d082020-09-03 15:19:03 -07003924 set = ct->regs;
Richard Henderson25f49c52018-11-27 12:45:26 -08003925 }
3926 *pset = set;
3927 }
3928 break;
bellardc896fe22008-02-01 10:05:41 +00003929 }
3930 break;
3931 }
Richard Hendersonbee158c2016-06-22 20:43:29 -07003932 op->life = arg_life;
Evgeny Voevodin1ff0a2c2012-11-12 13:27:48 +04003933 }
bellardc896fe22008-02-01 10:05:41 +00003934}
bellardc896fe22008-02-01 10:05:41 +00003935
Richard Henderson5a184072016-06-23 20:34:33 -07003936/* Liveness analysis: Convert indirect regs to direct temporaries. */
Richard Henderson9bbee4c2023-02-24 12:07:33 -10003937static bool __attribute__((noinline))
3938liveness_pass_2(TCGContext *s)
Richard Henderson5a184072016-06-23 20:34:33 -07003939{
3940 int nb_globals = s->nb_globals;
Richard Henderson15fa08f2017-11-02 15:19:14 +01003941 int nb_temps, i;
Richard Henderson5a184072016-06-23 20:34:33 -07003942 bool changes = false;
Richard Henderson15fa08f2017-11-02 15:19:14 +01003943 TCGOp *op, *op_next;
Richard Henderson5a184072016-06-23 20:34:33 -07003944
Richard Henderson5a184072016-06-23 20:34:33 -07003945 /* Create a temporary for each indirect global. */
3946 for (i = 0; i < nb_globals; ++i) {
3947 TCGTemp *its = &s->temps[i];
3948 if (its->indirect_reg) {
3949 TCGTemp *dts = tcg_temp_alloc(s);
3950 dts->type = its->type;
3951 dts->base_type = its->base_type;
Richard Hendersone1e64652023-02-03 12:58:12 -10003952 dts->temp_subindex = its->temp_subindex;
Richard Hendersonc7482432022-03-16 09:34:18 -07003953 dts->kind = TEMP_EBB;
Richard Hendersonb83eabe2016-11-01 15:56:04 -06003954 its->state_ptr = dts;
3955 } else {
3956 its->state_ptr = NULL;
Richard Henderson5a184072016-06-23 20:34:33 -07003957 }
Richard Hendersonb83eabe2016-11-01 15:56:04 -06003958 /* All globals begin dead. */
3959 its->state = TS_DEAD;
Richard Henderson5a184072016-06-23 20:34:33 -07003960 }
Richard Hendersonb83eabe2016-11-01 15:56:04 -06003961 for (nb_temps = s->nb_temps; i < nb_temps; ++i) {
3962 TCGTemp *its = &s->temps[i];
3963 its->state_ptr = NULL;
3964 its->state = TS_DEAD;
3965 }
Richard Henderson5a184072016-06-23 20:34:33 -07003966
Richard Henderson15fa08f2017-11-02 15:19:14 +01003967 QTAILQ_FOREACH_SAFE(op, &s->ops, link, op_next) {
Richard Henderson5a184072016-06-23 20:34:33 -07003968 TCGOpcode opc = op->opc;
3969 const TCGOpDef *def = &tcg_op_defs[opc];
3970 TCGLifeData arg_life = op->life;
3971 int nb_iargs, nb_oargs, call_flags;
Richard Hendersonb83eabe2016-11-01 15:56:04 -06003972 TCGTemp *arg_ts, *dir_ts;
Richard Henderson5a184072016-06-23 20:34:33 -07003973
Richard Henderson5a184072016-06-23 20:34:33 -07003974 if (opc == INDEX_op_call) {
Richard Hendersoncd9090a2017-11-14 13:02:51 +01003975 nb_oargs = TCGOP_CALLO(op);
3976 nb_iargs = TCGOP_CALLI(op);
Richard Henderson90163902021-03-18 10:21:45 -06003977 call_flags = tcg_call_flags(op);
Richard Henderson5a184072016-06-23 20:34:33 -07003978 } else {
3979 nb_iargs = def->nb_iargs;
3980 nb_oargs = def->nb_oargs;
3981
3982 /* Set flags similar to how calls require. */
Richard Hendersonb4cb76e2020-10-08 15:21:43 -05003983 if (def->flags & TCG_OPF_COND_BRANCH) {
3984 /* Like reading globals: sync_globals */
3985 call_flags = TCG_CALL_NO_WRITE_GLOBALS;
3986 } else if (def->flags & TCG_OPF_BB_END) {
Richard Henderson5a184072016-06-23 20:34:33 -07003987 /* Like writing globals: save_globals */
3988 call_flags = 0;
3989 } else if (def->flags & TCG_OPF_SIDE_EFFECTS) {
3990 /* Like reading globals: sync_globals */
3991 call_flags = TCG_CALL_NO_WRITE_GLOBALS;
3992 } else {
3993 /* No effect on globals. */
3994 call_flags = (TCG_CALL_NO_READ_GLOBALS |
3995 TCG_CALL_NO_WRITE_GLOBALS);
3996 }
3997 }
3998
3999 /* Make sure that input arguments are available. */
4000 for (i = nb_oargs; i < nb_iargs + nb_oargs; i++) {
Richard Hendersonb83eabe2016-11-01 15:56:04 -06004001 arg_ts = arg_temp(op->args[i]);
Richard Henderson39004a72022-11-11 10:09:37 +10004002 dir_ts = arg_ts->state_ptr;
4003 if (dir_ts && arg_ts->state == TS_DEAD) {
4004 TCGOpcode lopc = (arg_ts->type == TCG_TYPE_I32
4005 ? INDEX_op_ld_i32
4006 : INDEX_op_ld_i64);
4007 TCGOp *lop = tcg_op_insert_before(s, op, lopc, 3);
Richard Henderson5a184072016-06-23 20:34:33 -07004008
Richard Henderson39004a72022-11-11 10:09:37 +10004009 lop->args[0] = temp_arg(dir_ts);
4010 lop->args[1] = temp_arg(arg_ts->mem_base);
4011 lop->args[2] = arg_ts->mem_offset;
Richard Henderson5a184072016-06-23 20:34:33 -07004012
Richard Henderson39004a72022-11-11 10:09:37 +10004013 /* Loaded, but synced with memory. */
4014 arg_ts->state = TS_MEM;
Richard Henderson5a184072016-06-23 20:34:33 -07004015 }
4016 }
4017
4018 /* Perform input replacement, and mark inputs that became dead.
4019 No action is required except keeping temp_state up to date
4020 so that we reload when needed. */
4021 for (i = nb_oargs; i < nb_iargs + nb_oargs; i++) {
Richard Hendersonb83eabe2016-11-01 15:56:04 -06004022 arg_ts = arg_temp(op->args[i]);
Richard Henderson39004a72022-11-11 10:09:37 +10004023 dir_ts = arg_ts->state_ptr;
4024 if (dir_ts) {
4025 op->args[i] = temp_arg(dir_ts);
4026 changes = true;
4027 if (IS_DEAD_ARG(i)) {
4028 arg_ts->state = TS_DEAD;
Richard Henderson5a184072016-06-23 20:34:33 -07004029 }
4030 }
4031 }
4032
4033 /* Liveness analysis should ensure that the following are
4034 all correct, for call sites and basic block end points. */
4035 if (call_flags & TCG_CALL_NO_READ_GLOBALS) {
4036 /* Nothing to do */
4037 } else if (call_flags & TCG_CALL_NO_WRITE_GLOBALS) {
4038 for (i = 0; i < nb_globals; ++i) {
4039 /* Liveness should see that globals are synced back,
4040 that is, either TS_DEAD or TS_MEM. */
Richard Hendersonb83eabe2016-11-01 15:56:04 -06004041 arg_ts = &s->temps[i];
4042 tcg_debug_assert(arg_ts->state_ptr == 0
4043 || arg_ts->state != 0);
Richard Henderson5a184072016-06-23 20:34:33 -07004044 }
4045 } else {
4046 for (i = 0; i < nb_globals; ++i) {
4047 /* Liveness should see that globals are saved back,
4048 that is, TS_DEAD, waiting to be reloaded. */
Richard Hendersonb83eabe2016-11-01 15:56:04 -06004049 arg_ts = &s->temps[i];
4050 tcg_debug_assert(arg_ts->state_ptr == 0
4051 || arg_ts->state == TS_DEAD);
Richard Henderson5a184072016-06-23 20:34:33 -07004052 }
4053 }
4054
4055 /* Outputs become available. */
Richard Henderson61f15c42020-04-23 12:27:53 -07004056 if (opc == INDEX_op_mov_i32 || opc == INDEX_op_mov_i64) {
4057 arg_ts = arg_temp(op->args[0]);
Richard Hendersonb83eabe2016-11-01 15:56:04 -06004058 dir_ts = arg_ts->state_ptr;
Richard Henderson61f15c42020-04-23 12:27:53 -07004059 if (dir_ts) {
4060 op->args[0] = temp_arg(dir_ts);
4061 changes = true;
4062
4063 /* The output is now live and modified. */
4064 arg_ts->state = 0;
4065
4066 if (NEED_SYNC_ARG(0)) {
4067 TCGOpcode sopc = (arg_ts->type == TCG_TYPE_I32
4068 ? INDEX_op_st_i32
4069 : INDEX_op_st_i64);
Philippe Mathieu-Daudéd4478942022-12-18 22:18:31 +01004070 TCGOp *sop = tcg_op_insert_after(s, op, sopc, 3);
Richard Henderson61f15c42020-04-23 12:27:53 -07004071 TCGTemp *out_ts = dir_ts;
4072
4073 if (IS_DEAD_ARG(0)) {
4074 out_ts = arg_temp(op->args[1]);
4075 arg_ts->state = TS_DEAD;
4076 tcg_op_remove(s, op);
4077 } else {
4078 arg_ts->state = TS_MEM;
4079 }
4080
4081 sop->args[0] = temp_arg(out_ts);
4082 sop->args[1] = temp_arg(arg_ts->mem_base);
4083 sop->args[2] = arg_ts->mem_offset;
4084 } else {
4085 tcg_debug_assert(!IS_DEAD_ARG(0));
4086 }
Richard Henderson5a184072016-06-23 20:34:33 -07004087 }
Richard Henderson61f15c42020-04-23 12:27:53 -07004088 } else {
4089 for (i = 0; i < nb_oargs; i++) {
4090 arg_ts = arg_temp(op->args[i]);
4091 dir_ts = arg_ts->state_ptr;
4092 if (!dir_ts) {
4093 continue;
4094 }
4095 op->args[i] = temp_arg(dir_ts);
4096 changes = true;
Richard Henderson5a184072016-06-23 20:34:33 -07004097
Richard Henderson61f15c42020-04-23 12:27:53 -07004098 /* The output is now live and modified. */
4099 arg_ts->state = 0;
Richard Henderson5a184072016-06-23 20:34:33 -07004100
Richard Henderson61f15c42020-04-23 12:27:53 -07004101 /* Sync outputs upon their last write. */
4102 if (NEED_SYNC_ARG(i)) {
4103 TCGOpcode sopc = (arg_ts->type == TCG_TYPE_I32
4104 ? INDEX_op_st_i32
4105 : INDEX_op_st_i64);
Philippe Mathieu-Daudéd4478942022-12-18 22:18:31 +01004106 TCGOp *sop = tcg_op_insert_after(s, op, sopc, 3);
Richard Henderson5a184072016-06-23 20:34:33 -07004107
Richard Henderson61f15c42020-04-23 12:27:53 -07004108 sop->args[0] = temp_arg(dir_ts);
4109 sop->args[1] = temp_arg(arg_ts->mem_base);
4110 sop->args[2] = arg_ts->mem_offset;
Richard Henderson5a184072016-06-23 20:34:33 -07004111
Richard Henderson61f15c42020-04-23 12:27:53 -07004112 arg_ts->state = TS_MEM;
4113 }
4114 /* Drop outputs that are dead. */
4115 if (IS_DEAD_ARG(i)) {
4116 arg_ts->state = TS_DEAD;
4117 }
Richard Henderson5a184072016-06-23 20:34:33 -07004118 }
4119 }
4120 }
4121
4122 return changes;
4123}
4124
Richard Henderson2272e4a2016-11-09 15:25:09 +01004125static void temp_allocate_frame(TCGContext *s, TCGTemp *ts)
bellardc896fe22008-02-01 10:05:41 +00004126{
Richard Henderson31c96412022-10-19 12:41:15 +10004127 intptr_t off;
Richard Henderson273eb502022-10-19 12:03:40 +10004128 int size, align;
Richard Hendersonc1c09192021-06-18 21:53:27 -07004129
Richard Henderson273eb502022-10-19 12:03:40 +10004130 /* When allocating an object, look at the full type. */
4131 size = tcg_type_size(ts->base_type);
4132 switch (ts->base_type) {
Richard Hendersonc1c09192021-06-18 21:53:27 -07004133 case TCG_TYPE_I32:
Richard Henderson31c96412022-10-19 12:41:15 +10004134 align = 4;
Richard Hendersonc1c09192021-06-18 21:53:27 -07004135 break;
4136 case TCG_TYPE_I64:
4137 case TCG_TYPE_V64:
Richard Henderson31c96412022-10-19 12:41:15 +10004138 align = 8;
Richard Hendersonc1c09192021-06-18 21:53:27 -07004139 break;
Richard Henderson43eef722022-10-20 08:03:41 +10004140 case TCG_TYPE_I128:
Richard Hendersonc1c09192021-06-18 21:53:27 -07004141 case TCG_TYPE_V128:
Richard Hendersonc1c09192021-06-18 21:53:27 -07004142 case TCG_TYPE_V256:
Richard Henderson43eef722022-10-20 08:03:41 +10004143 /*
4144 * Note that we do not require aligned storage for V256,
4145 * and that we provide alignment for I128 to match V128,
4146 * even if that's above what the host ABI requires.
4147 */
Richard Henderson31c96412022-10-19 12:41:15 +10004148 align = 16;
Richard Hendersonc1c09192021-06-18 21:53:27 -07004149 break;
4150 default:
4151 g_assert_not_reached();
Blue Swirlb591dc52011-05-14 14:03:22 +00004152 }
Richard Hendersonc1c09192021-06-18 21:53:27 -07004153
Richard Hendersonb9537d52021-09-12 10:49:25 -07004154 /*
4155 * Assume the stack is sufficiently aligned.
4156 * This affects e.g. ARM NEON, where we have 8 byte stack alignment
4157 * and do not require 16 byte vector alignment. This seems slightly
4158 * easier than fully parameterizing the above switch statement.
4159 */
4160 align = MIN(TCG_TARGET_STACK_ALIGN, align);
Richard Hendersonc1c09192021-06-18 21:53:27 -07004161 off = ROUND_UP(s->current_frame_offset, align);
Richard Henderson732d5892021-06-19 06:32:03 -07004162
4163 /* If we've exhausted the stack frame, restart with a smaller TB. */
4164 if (off + size > s->frame_end) {
4165 tcg_raise_tb_overflow(s);
4166 }
Richard Hendersonc1c09192021-06-18 21:53:27 -07004167 s->current_frame_offset = off + size;
Richard Henderson9defd1b2021-06-18 16:49:26 -07004168#if defined(__sparc__)
Richard Henderson273eb502022-10-19 12:03:40 +10004169 off += TCG_TARGET_STACK_BIAS;
Richard Henderson9defd1b2021-06-18 16:49:26 -07004170#endif
Richard Henderson273eb502022-10-19 12:03:40 +10004171
4172 /* If the object was subdivided, assign memory to all the parts. */
4173 if (ts->base_type != ts->type) {
4174 int part_size = tcg_type_size(ts->type);
4175 int part_count = size / part_size;
4176
4177 /*
4178 * Each part is allocated sequentially in tcg_temp_new_internal.
4179 * Jump back to the first part by subtracting the current index.
4180 */
4181 ts -= ts->temp_subindex;
4182 for (int i = 0; i < part_count; ++i) {
4183 ts[i].mem_offset = off + i * part_size;
4184 ts[i].mem_base = s->frame_temp;
4185 ts[i].mem_allocated = 1;
4186 }
4187 } else {
4188 ts->mem_offset = off;
4189 ts->mem_base = s->frame_temp;
4190 ts->mem_allocated = 1;
4191 }
bellardc896fe22008-02-01 10:05:41 +00004192}
4193
Richard Henderson098859f2022-12-01 01:05:05 -08004194/* Assign @reg to @ts, and update reg_to_temp[]. */
4195static void set_temp_val_reg(TCGContext *s, TCGTemp *ts, TCGReg reg)
4196{
4197 if (ts->val_type == TEMP_VAL_REG) {
4198 TCGReg old = ts->reg;
4199 tcg_debug_assert(s->reg_to_temp[old] == ts);
4200 if (old == reg) {
4201 return;
4202 }
4203 s->reg_to_temp[old] = NULL;
4204 }
4205 tcg_debug_assert(s->reg_to_temp[reg] == NULL);
4206 s->reg_to_temp[reg] = ts;
4207 ts->val_type = TEMP_VAL_REG;
4208 ts->reg = reg;
4209}
4210
4211/* Assign a non-register value type to @ts, and update reg_to_temp[]. */
4212static void set_temp_val_nonreg(TCGContext *s, TCGTemp *ts, TCGTempVal type)
4213{
4214 tcg_debug_assert(type != TEMP_VAL_REG);
4215 if (ts->val_type == TEMP_VAL_REG) {
4216 TCGReg reg = ts->reg;
4217 tcg_debug_assert(s->reg_to_temp[reg] == ts);
4218 s->reg_to_temp[reg] = NULL;
4219 }
4220 ts->val_type = type;
4221}
4222
Richard Hendersonb7224522018-11-27 07:48:06 -08004223static void temp_load(TCGContext *, TCGTemp *, TCGRegSet, TCGRegSet, TCGRegSet);
Richard Hendersonb3915db2013-09-19 10:36:18 -07004224
Richard Henderson59d7c142016-06-19 22:59:13 -07004225/* Mark a temporary as free or dead. If 'free_or_dead' is negative,
4226 mark it free; otherwise mark it dead. */
4227static void temp_free_or_dead(TCGContext *s, TCGTemp *ts, int free_or_dead)
bellardc896fe22008-02-01 10:05:41 +00004228{
Richard Hendersonc0522132020-03-29 18:55:52 -07004229 TCGTempVal new_type;
4230
4231 switch (ts->kind) {
4232 case TEMP_FIXED:
Richard Henderson59d7c142016-06-19 22:59:13 -07004233 return;
Richard Hendersonc0522132020-03-29 18:55:52 -07004234 case TEMP_GLOBAL:
Richard Hendersonf57c6912023-01-29 10:55:52 -10004235 case TEMP_TB:
Richard Hendersonc0522132020-03-29 18:55:52 -07004236 new_type = TEMP_VAL_MEM;
4237 break;
Richard Hendersonc7482432022-03-16 09:34:18 -07004238 case TEMP_EBB:
Richard Hendersonc0522132020-03-29 18:55:52 -07004239 new_type = free_or_dead < 0 ? TEMP_VAL_MEM : TEMP_VAL_DEAD;
4240 break;
4241 case TEMP_CONST:
4242 new_type = TEMP_VAL_CONST;
4243 break;
4244 default:
4245 g_assert_not_reached();
Richard Henderson59d7c142016-06-19 22:59:13 -07004246 }
Richard Henderson098859f2022-12-01 01:05:05 -08004247 set_temp_val_nonreg(s, ts, new_type);
Richard Henderson59d7c142016-06-19 22:59:13 -07004248}
bellardc896fe22008-02-01 10:05:41 +00004249
Richard Henderson59d7c142016-06-19 22:59:13 -07004250/* Mark a temporary as dead. */
4251static inline void temp_dead(TCGContext *s, TCGTemp *ts)
4252{
4253 temp_free_or_dead(s, ts, 1);
4254}
4255
4256/* Sync a temporary to memory. 'allocated_regs' is used in case a temporary
4257 registers needs to be allocated to store a constant. If 'free_or_dead'
4258 is non-zero, subsequently release the temporary; if it is positive, the
4259 temp is dead; if it is negative, the temp is free. */
Richard Henderson98b4e182018-11-27 15:35:04 -08004260static void temp_sync(TCGContext *s, TCGTemp *ts, TCGRegSet allocated_regs,
4261 TCGRegSet preferred_regs, int free_or_dead)
Richard Henderson59d7c142016-06-19 22:59:13 -07004262{
Richard Hendersonc0522132020-03-29 18:55:52 -07004263 if (!temp_readonly(ts) && !ts->mem_coherent) {
Aurelien Jarno7f6ceed2012-10-09 21:53:06 +02004264 if (!ts->mem_allocated) {
Richard Henderson2272e4a2016-11-09 15:25:09 +01004265 temp_allocate_frame(s, ts);
Richard Henderson59d7c142016-06-19 22:59:13 -07004266 }
Richard Henderson59d7c142016-06-19 22:59:13 -07004267 switch (ts->val_type) {
4268 case TEMP_VAL_CONST:
4269 /* If we're going to free the temp immediately, then we won't
4270 require it later in a register, so attempt to store the
4271 constant to memory directly. */
4272 if (free_or_dead
4273 && tcg_out_sti(s, ts->type, ts->val,
4274 ts->mem_base->reg, ts->mem_offset)) {
4275 break;
4276 }
4277 temp_load(s, ts, tcg_target_available_regs[ts->type],
Richard Henderson98b4e182018-11-27 15:35:04 -08004278 allocated_regs, preferred_regs);
Richard Henderson59d7c142016-06-19 22:59:13 -07004279 /* fallthrough */
4280
4281 case TEMP_VAL_REG:
4282 tcg_out_st(s, ts->type, ts->reg,
4283 ts->mem_base->reg, ts->mem_offset);
4284 break;
4285
4286 case TEMP_VAL_MEM:
4287 break;
4288
4289 case TEMP_VAL_DEAD:
4290 default:
Richard Henderson732e89f2023-04-05 12:09:14 -07004291 g_assert_not_reached();
Richard Henderson59d7c142016-06-19 22:59:13 -07004292 }
4293 ts->mem_coherent = 1;
Aurelien Jarno7f6ceed2012-10-09 21:53:06 +02004294 }
Richard Henderson59d7c142016-06-19 22:59:13 -07004295 if (free_or_dead) {
4296 temp_free_or_dead(s, ts, free_or_dead);
4297 }
Aurelien Jarno7f6ceed2012-10-09 21:53:06 +02004298}
4299
4300/* free register 'reg' by spilling the corresponding temporary if necessary */
Richard Hendersonb3915db2013-09-19 10:36:18 -07004301static void tcg_reg_free(TCGContext *s, TCGReg reg, TCGRegSet allocated_regs)
Aurelien Jarno7f6ceed2012-10-09 21:53:06 +02004302{
Richard Hendersonf8b2f202013-09-18 15:21:56 -07004303 TCGTemp *ts = s->reg_to_temp[reg];
Richard Hendersonf8b2f202013-09-18 15:21:56 -07004304 if (ts != NULL) {
Richard Henderson98b4e182018-11-27 15:35:04 -08004305 temp_sync(s, ts, allocated_regs, 0, -1);
bellardc896fe22008-02-01 10:05:41 +00004306 }
4307}
4308
Richard Hendersonb0164862018-11-27 07:16:21 -08004309/**
4310 * tcg_reg_alloc:
4311 * @required_regs: Set of registers in which we must allocate.
4312 * @allocated_regs: Set of registers which must be avoided.
4313 * @preferred_regs: Set of registers we should prefer.
4314 * @rev: True if we search the registers in "indirect" order.
4315 *
4316 * The allocated register must be in @required_regs & ~@allocated_regs,
4317 * but if we can put it in @preferred_regs we may save a move later.
4318 */
4319static TCGReg tcg_reg_alloc(TCGContext *s, TCGRegSet required_regs,
4320 TCGRegSet allocated_regs,
4321 TCGRegSet preferred_regs, bool rev)
bellardc896fe22008-02-01 10:05:41 +00004322{
Richard Hendersonb0164862018-11-27 07:16:21 -08004323 int i, j, f, n = ARRAY_SIZE(tcg_target_reg_alloc_order);
4324 TCGRegSet reg_ct[2];
Richard Henderson91478ce2015-08-18 23:23:08 -07004325 const int *order;
bellardc896fe22008-02-01 10:05:41 +00004326
Richard Hendersonb0164862018-11-27 07:16:21 -08004327 reg_ct[1] = required_regs & ~allocated_regs;
4328 tcg_debug_assert(reg_ct[1] != 0);
4329 reg_ct[0] = reg_ct[1] & preferred_regs;
4330
4331 /* Skip the preferred_regs option if it cannot be satisfied,
4332 or if the preference made no difference. */
4333 f = reg_ct[0] == 0 || reg_ct[0] == reg_ct[1];
4334
Richard Henderson91478ce2015-08-18 23:23:08 -07004335 order = rev ? indirect_reg_alloc_order : tcg_target_reg_alloc_order;
bellardc896fe22008-02-01 10:05:41 +00004336
Richard Hendersonb0164862018-11-27 07:16:21 -08004337 /* Try free registers, preferences first. */
4338 for (j = f; j < 2; j++) {
4339 TCGRegSet set = reg_ct[j];
4340
4341 if (tcg_regset_single(set)) {
4342 /* One register in the set. */
4343 TCGReg reg = tcg_regset_first(set);
4344 if (s->reg_to_temp[reg] == NULL) {
4345 return reg;
4346 }
4347 } else {
4348 for (i = 0; i < n; i++) {
4349 TCGReg reg = order[i];
4350 if (s->reg_to_temp[reg] == NULL &&
4351 tcg_regset_test_reg(set, reg)) {
4352 return reg;
4353 }
4354 }
4355 }
bellardc896fe22008-02-01 10:05:41 +00004356 }
4357
Richard Hendersonb0164862018-11-27 07:16:21 -08004358 /* We must spill something. */
4359 for (j = f; j < 2; j++) {
4360 TCGRegSet set = reg_ct[j];
4361
4362 if (tcg_regset_single(set)) {
4363 /* One register in the set. */
4364 TCGReg reg = tcg_regset_first(set);
Richard Hendersonb3915db2013-09-19 10:36:18 -07004365 tcg_reg_free(s, reg, allocated_regs);
bellardc896fe22008-02-01 10:05:41 +00004366 return reg;
Richard Hendersonb0164862018-11-27 07:16:21 -08004367 } else {
4368 for (i = 0; i < n; i++) {
4369 TCGReg reg = order[i];
4370 if (tcg_regset_test_reg(set, reg)) {
4371 tcg_reg_free(s, reg, allocated_regs);
4372 return reg;
4373 }
4374 }
bellardc896fe22008-02-01 10:05:41 +00004375 }
4376 }
4377
Richard Henderson732e89f2023-04-05 12:09:14 -07004378 g_assert_not_reached();
bellardc896fe22008-02-01 10:05:41 +00004379}
4380
Richard Henderson29f5e922022-10-14 07:37:38 +11004381static TCGReg tcg_reg_alloc_pair(TCGContext *s, TCGRegSet required_regs,
4382 TCGRegSet allocated_regs,
4383 TCGRegSet preferred_regs, bool rev)
4384{
4385 int i, j, k, fmin, n = ARRAY_SIZE(tcg_target_reg_alloc_order);
4386 TCGRegSet reg_ct[2];
4387 const int *order;
4388
4389 /* Ensure that if I is not in allocated_regs, I+1 is not either. */
4390 reg_ct[1] = required_regs & ~(allocated_regs | (allocated_regs >> 1));
4391 tcg_debug_assert(reg_ct[1] != 0);
4392 reg_ct[0] = reg_ct[1] & preferred_regs;
4393
4394 order = rev ? indirect_reg_alloc_order : tcg_target_reg_alloc_order;
4395
4396 /*
4397 * Skip the preferred_regs option if it cannot be satisfied,
4398 * or if the preference made no difference.
4399 */
4400 k = reg_ct[0] == 0 || reg_ct[0] == reg_ct[1];
4401
4402 /*
4403 * Minimize the number of flushes by looking for 2 free registers first,
4404 * then a single flush, then two flushes.
4405 */
4406 for (fmin = 2; fmin >= 0; fmin--) {
4407 for (j = k; j < 2; j++) {
4408 TCGRegSet set = reg_ct[j];
4409
4410 for (i = 0; i < n; i++) {
4411 TCGReg reg = order[i];
4412
4413 if (tcg_regset_test_reg(set, reg)) {
4414 int f = !s->reg_to_temp[reg] + !s->reg_to_temp[reg + 1];
4415 if (f >= fmin) {
4416 tcg_reg_free(s, reg, allocated_regs);
4417 tcg_reg_free(s, reg + 1, allocated_regs);
4418 return reg;
4419 }
4420 }
4421 }
4422 }
4423 }
Richard Henderson732e89f2023-04-05 12:09:14 -07004424 g_assert_not_reached();
Richard Henderson29f5e922022-10-14 07:37:38 +11004425}
4426
Richard Henderson40ae5c62013-09-19 08:02:05 -07004427/* Make sure the temporary is in a register. If needed, allocate the register
4428 from DESIRED while avoiding ALLOCATED. */
4429static void temp_load(TCGContext *s, TCGTemp *ts, TCGRegSet desired_regs,
Richard Hendersonb7224522018-11-27 07:48:06 -08004430 TCGRegSet allocated_regs, TCGRegSet preferred_regs)
Richard Henderson40ae5c62013-09-19 08:02:05 -07004431{
4432 TCGReg reg;
4433
4434 switch (ts->val_type) {
4435 case TEMP_VAL_REG:
4436 return;
4437 case TEMP_VAL_CONST:
Richard Hendersonb0164862018-11-27 07:16:21 -08004438 reg = tcg_reg_alloc(s, desired_regs, allocated_regs,
Richard Hendersonb7224522018-11-27 07:48:06 -08004439 preferred_regs, ts->indirect_base);
Richard Henderson0a6a8bc2020-03-31 05:43:23 -07004440 if (ts->type <= TCG_TYPE_I64) {
4441 tcg_out_movi(s, ts->type, reg, ts->val);
4442 } else {
Richard Henderson4e186172020-03-31 01:02:08 -07004443 uint64_t val = ts->val;
4444 MemOp vece = MO_64;
4445
4446 /*
4447 * Find the minimal vector element that matches the constant.
4448 * The targets will, in general, have to do this search anyway,
4449 * do this generically.
4450 */
Richard Henderson4e186172020-03-31 01:02:08 -07004451 if (val == dup_const(MO_8, val)) {
4452 vece = MO_8;
4453 } else if (val == dup_const(MO_16, val)) {
4454 vece = MO_16;
Richard Henderson0b4286d2020-09-06 17:33:18 -07004455 } else if (val == dup_const(MO_32, val)) {
Richard Henderson4e186172020-03-31 01:02:08 -07004456 vece = MO_32;
4457 }
4458
4459 tcg_out_dupi_vec(s, ts->type, vece, reg, ts->val);
Richard Henderson0a6a8bc2020-03-31 05:43:23 -07004460 }
Richard Henderson40ae5c62013-09-19 08:02:05 -07004461 ts->mem_coherent = 0;
4462 break;
4463 case TEMP_VAL_MEM:
Richard Hendersonb0164862018-11-27 07:16:21 -08004464 reg = tcg_reg_alloc(s, desired_regs, allocated_regs,
Richard Hendersonb7224522018-11-27 07:48:06 -08004465 preferred_regs, ts->indirect_base);
Richard Henderson40ae5c62013-09-19 08:02:05 -07004466 tcg_out_ld(s, ts->type, reg, ts->mem_base->reg, ts->mem_offset);
4467 ts->mem_coherent = 1;
4468 break;
4469 case TEMP_VAL_DEAD:
4470 default:
Richard Henderson732e89f2023-04-05 12:09:14 -07004471 g_assert_not_reached();
Richard Henderson40ae5c62013-09-19 08:02:05 -07004472 }
Richard Henderson098859f2022-12-01 01:05:05 -08004473 set_temp_val_reg(s, ts, reg);
Richard Henderson40ae5c62013-09-19 08:02:05 -07004474}
4475
Richard Henderson59d7c142016-06-19 22:59:13 -07004476/* Save a temporary to memory. 'allocated_regs' is used in case a
4477 temporary registers needs to be allocated to store a constant. */
4478static void temp_save(TCGContext *s, TCGTemp *ts, TCGRegSet allocated_regs)
Aurelien Jarno1ad80722012-10-09 21:53:06 +02004479{
Richard Henderson5a184072016-06-23 20:34:33 -07004480 /* The liveness analysis already ensures that globals are back
4481 in memory. Keep an tcg_debug_assert for safety. */
Richard Hendersone01fa972020-03-29 10:40:49 -07004482 tcg_debug_assert(ts->val_type == TEMP_VAL_MEM || temp_readonly(ts));
Aurelien Jarno1ad80722012-10-09 21:53:06 +02004483}
4484
Dong Xu Wang9814dd22011-11-22 18:06:22 +08004485/* save globals to their canonical location and assume they can be
bellarde8996ee2008-05-23 17:33:39 +00004486 modified be the following code. 'allocated_regs' is used in case a
4487 temporary registers needs to be allocated to store a constant. */
4488static void save_globals(TCGContext *s, TCGRegSet allocated_regs)
bellardc896fe22008-02-01 10:05:41 +00004489{
Richard Hendersonac3b8892016-11-02 11:21:44 -06004490 int i, n;
bellardc896fe22008-02-01 10:05:41 +00004491
Richard Hendersonac3b8892016-11-02 11:21:44 -06004492 for (i = 0, n = s->nb_globals; i < n; i++) {
Richard Hendersonb13eb722013-09-18 15:35:32 -07004493 temp_save(s, &s->temps[i], allocated_regs);
bellardc896fe22008-02-01 10:05:41 +00004494 }
bellarde5097dc2008-05-21 16:24:20 +00004495}
4496
Aurelien Jarno3d5c5f82012-10-09 21:53:08 +02004497/* sync globals to their canonical location and assume they can be
4498 read by the following code. 'allocated_regs' is used in case a
4499 temporary registers needs to be allocated to store a constant. */
4500static void sync_globals(TCGContext *s, TCGRegSet allocated_regs)
4501{
Richard Hendersonac3b8892016-11-02 11:21:44 -06004502 int i, n;
Aurelien Jarno3d5c5f82012-10-09 21:53:08 +02004503
Richard Hendersonac3b8892016-11-02 11:21:44 -06004504 for (i = 0, n = s->nb_globals; i < n; i++) {
Richard Henderson12b9b112013-09-18 15:33:00 -07004505 TCGTemp *ts = &s->temps[i];
Richard Henderson5a184072016-06-23 20:34:33 -07004506 tcg_debug_assert(ts->val_type != TEMP_VAL_REG
Richard Hendersonee17db82020-03-29 10:11:56 -07004507 || ts->kind == TEMP_FIXED
Richard Henderson5a184072016-06-23 20:34:33 -07004508 || ts->mem_coherent);
Aurelien Jarno3d5c5f82012-10-09 21:53:08 +02004509 }
4510}
4511
bellarde5097dc2008-05-21 16:24:20 +00004512/* at the end of a basic block, we assume all temporaries are dead and
bellarde8996ee2008-05-23 17:33:39 +00004513 all globals are stored at their canonical location. */
4514static void tcg_reg_alloc_bb_end(TCGContext *s, TCGRegSet allocated_regs)
bellarde5097dc2008-05-21 16:24:20 +00004515{
bellarde5097dc2008-05-21 16:24:20 +00004516 int i;
4517
Richard Hendersonb13eb722013-09-18 15:35:32 -07004518 for (i = s->nb_globals; i < s->nb_temps; i++) {
4519 TCGTemp *ts = &s->temps[i];
Richard Hendersonc0522132020-03-29 18:55:52 -07004520
4521 switch (ts->kind) {
Richard Hendersonf57c6912023-01-29 10:55:52 -10004522 case TEMP_TB:
Richard Hendersonb13eb722013-09-18 15:35:32 -07004523 temp_save(s, ts, allocated_regs);
Richard Hendersonc0522132020-03-29 18:55:52 -07004524 break;
Richard Hendersonc7482432022-03-16 09:34:18 -07004525 case TEMP_EBB:
Richard Henderson5a184072016-06-23 20:34:33 -07004526 /* The liveness analysis already ensures that temps are dead.
4527 Keep an tcg_debug_assert for safety. */
4528 tcg_debug_assert(ts->val_type == TEMP_VAL_DEAD);
Richard Hendersonc0522132020-03-29 18:55:52 -07004529 break;
4530 case TEMP_CONST:
4531 /* Similarly, we should have freed any allocated register. */
4532 tcg_debug_assert(ts->val_type == TEMP_VAL_CONST);
4533 break;
4534 default:
4535 g_assert_not_reached();
bellardc896fe22008-02-01 10:05:41 +00004536 }
4537 }
bellarde8996ee2008-05-23 17:33:39 +00004538
4539 save_globals(s, allocated_regs);
bellardc896fe22008-02-01 10:05:41 +00004540}
4541
Richard Hendersonbab16712019-03-18 11:20:27 -07004542/*
Richard Hendersonc7482432022-03-16 09:34:18 -07004543 * At a conditional branch, we assume all temporaries are dead unless
4544 * explicitly live-across-conditional-branch; all globals and local
4545 * temps are synced to their location.
Richard Hendersonb4cb76e2020-10-08 15:21:43 -05004546 */
4547static void tcg_reg_alloc_cbranch(TCGContext *s, TCGRegSet allocated_regs)
4548{
4549 sync_globals(s, allocated_regs);
4550
4551 for (int i = s->nb_globals; i < s->nb_temps; i++) {
4552 TCGTemp *ts = &s->temps[i];
4553 /*
4554 * The liveness analysis already ensures that temps are dead.
4555 * Keep tcg_debug_asserts for safety.
4556 */
Richard Hendersonc0522132020-03-29 18:55:52 -07004557 switch (ts->kind) {
Richard Hendersonf57c6912023-01-29 10:55:52 -10004558 case TEMP_TB:
Richard Hendersonb4cb76e2020-10-08 15:21:43 -05004559 tcg_debug_assert(ts->val_type != TEMP_VAL_REG || ts->mem_coherent);
Richard Hendersonc0522132020-03-29 18:55:52 -07004560 break;
Richard Hendersonc7482432022-03-16 09:34:18 -07004561 case TEMP_EBB:
Richard Hendersonc0522132020-03-29 18:55:52 -07004562 case TEMP_CONST:
4563 break;
4564 default:
4565 g_assert_not_reached();
Richard Hendersonb4cb76e2020-10-08 15:21:43 -05004566 }
4567 }
4568}
4569
4570/*
Richard Hendersonc58f4c92020-04-17 13:22:43 -07004571 * Specialized code generation for INDEX_op_mov_* with a constant.
Richard Hendersonbab16712019-03-18 11:20:27 -07004572 */
Paolo Bonzini0fe4fca2016-09-15 15:16:00 +02004573static void tcg_reg_alloc_do_movi(TCGContext *s, TCGTemp *ots,
Richard Hendersonba877192018-11-27 15:39:21 -08004574 tcg_target_ulong val, TCGLifeData arg_life,
4575 TCGRegSet preferred_regs)
bellarde8996ee2008-05-23 17:33:39 +00004576{
Richard Hendersond63e3b62019-03-16 17:48:02 +00004577 /* ENV should not be modified. */
Richard Hendersone01fa972020-03-29 10:40:49 -07004578 tcg_debug_assert(!temp_readonly(ots));
Richard Henderson59d7c142016-06-19 22:59:13 -07004579
4580 /* The movi is not explicitly generated here. */
Richard Henderson098859f2022-12-01 01:05:05 -08004581 set_temp_val_nonreg(s, ots, TEMP_VAL_CONST);
Richard Henderson59d7c142016-06-19 22:59:13 -07004582 ots->val = val;
4583 ots->mem_coherent = 0;
Aurelien Jarnoec7a8692012-10-09 21:53:07 +02004584 if (NEED_SYNC_ARG(0)) {
Richard Hendersonba877192018-11-27 15:39:21 -08004585 temp_sync(s, ots, s->reserved_regs, preferred_regs, IS_DEAD_ARG(0));
Richard Henderson59d7c142016-06-19 22:59:13 -07004586 } else if (IS_DEAD_ARG(0)) {
Richard Hendersonf8bf00f2013-09-18 15:29:18 -07004587 temp_dead(s, ots);
Aurelien Jarno4c4e1ab2012-10-09 21:53:07 +02004588 }
bellarde8996ee2008-05-23 17:33:39 +00004589}
4590
Richard Hendersonbab16712019-03-18 11:20:27 -07004591/*
4592 * Specialized code generation for INDEX_op_mov_*.
4593 */
Richard Hendersondd186292016-12-08 13:42:08 -08004594static void tcg_reg_alloc_mov(TCGContext *s, const TCGOp *op)
bellardc896fe22008-02-01 10:05:41 +00004595{
Richard Hendersondd186292016-12-08 13:42:08 -08004596 const TCGLifeData arg_life = op->life;
Richard Henderson69e37062018-11-27 07:44:51 -08004597 TCGRegSet allocated_regs, preferred_regs;
bellardc896fe22008-02-01 10:05:41 +00004598 TCGTemp *ts, *ots;
Richard Henderson450445d2014-05-13 14:50:18 -07004599 TCGType otype, itype;
Richard Henderson098859f2022-12-01 01:05:05 -08004600 TCGReg oreg, ireg;
bellardc896fe22008-02-01 10:05:41 +00004601
Richard Hendersond21369f2017-09-11 11:58:44 -07004602 allocated_regs = s->reserved_regs;
Richard Henderson31fd8842022-11-11 15:10:51 +10004603 preferred_regs = output_pref(op, 0);
Richard Henderson43439132017-06-19 23:18:10 -07004604 ots = arg_temp(op->args[0]);
4605 ts = arg_temp(op->args[1]);
Richard Henderson450445d2014-05-13 14:50:18 -07004606
Richard Hendersond63e3b62019-03-16 17:48:02 +00004607 /* ENV should not be modified. */
Richard Hendersone01fa972020-03-29 10:40:49 -07004608 tcg_debug_assert(!temp_readonly(ots));
Richard Hendersond63e3b62019-03-16 17:48:02 +00004609
Richard Henderson450445d2014-05-13 14:50:18 -07004610 /* Note that otype != itype for no-op truncation. */
4611 otype = ots->type;
4612 itype = ts->type;
bellardc896fe22008-02-01 10:05:41 +00004613
Paolo Bonzini0fe4fca2016-09-15 15:16:00 +02004614 if (ts->val_type == TEMP_VAL_CONST) {
4615 /* propagate constant or generate sti */
4616 tcg_target_ulong val = ts->val;
4617 if (IS_DEAD_ARG(1)) {
4618 temp_dead(s, ts);
4619 }
Richard Henderson69e37062018-11-27 07:44:51 -08004620 tcg_reg_alloc_do_movi(s, ots, val, arg_life, preferred_regs);
Paolo Bonzini0fe4fca2016-09-15 15:16:00 +02004621 return;
4622 }
4623
4624 /* If the source value is in memory we're going to be forced
4625 to have it in a register in order to perform the copy. Copy
4626 the SOURCE value into its own register first, that way we
4627 don't have to reload SOURCE the next time it is used. */
4628 if (ts->val_type == TEMP_VAL_MEM) {
Richard Henderson69e37062018-11-27 07:44:51 -08004629 temp_load(s, ts, tcg_target_available_regs[itype],
4630 allocated_regs, preferred_regs);
Aurelien Jarnoc29c1d72012-10-09 21:53:07 +02004631 }
Paolo Bonzini0fe4fca2016-09-15 15:16:00 +02004632 tcg_debug_assert(ts->val_type == TEMP_VAL_REG);
Richard Henderson098859f2022-12-01 01:05:05 -08004633 ireg = ts->reg;
4634
Richard Hendersond63e3b62019-03-16 17:48:02 +00004635 if (IS_DEAD_ARG(0)) {
Aurelien Jarnoc29c1d72012-10-09 21:53:07 +02004636 /* mov to a non-saved dead register makes no sense (even with
4637 liveness analysis disabled). */
Aurelien Jarnoeabb7b92016-04-21 10:48:49 +02004638 tcg_debug_assert(NEED_SYNC_ARG(0));
Aurelien Jarnoc29c1d72012-10-09 21:53:07 +02004639 if (!ots->mem_allocated) {
Richard Henderson2272e4a2016-11-09 15:25:09 +01004640 temp_allocate_frame(s, ots);
Aurelien Jarnoc29c1d72012-10-09 21:53:07 +02004641 }
Richard Henderson098859f2022-12-01 01:05:05 -08004642 tcg_out_st(s, otype, ireg, ots->mem_base->reg, ots->mem_offset);
Aurelien Jarnoc29c1d72012-10-09 21:53:07 +02004643 if (IS_DEAD_ARG(1)) {
Richard Hendersonf8bf00f2013-09-18 15:29:18 -07004644 temp_dead(s, ts);
Aurelien Jarnoc29c1d72012-10-09 21:53:07 +02004645 }
Richard Hendersonf8bf00f2013-09-18 15:29:18 -07004646 temp_dead(s, ots);
Richard Henderson098859f2022-12-01 01:05:05 -08004647 return;
4648 }
4649
4650 if (IS_DEAD_ARG(1) && ts->kind != TEMP_FIXED) {
4651 /*
4652 * The mov can be suppressed. Kill input first, so that it
4653 * is unlinked from reg_to_temp, then set the output to the
4654 * reg that we saved from the input.
4655 */
4656 temp_dead(s, ts);
4657 oreg = ireg;
Aurelien Jarnoc29c1d72012-10-09 21:53:07 +02004658 } else {
Richard Henderson098859f2022-12-01 01:05:05 -08004659 if (ots->val_type == TEMP_VAL_REG) {
4660 oreg = ots->reg;
bellardc896fe22008-02-01 10:05:41 +00004661 } else {
Richard Henderson098859f2022-12-01 01:05:05 -08004662 /* Make sure to not spill the input register during allocation. */
4663 oreg = tcg_reg_alloc(s, tcg_target_available_regs[otype],
4664 allocated_regs | ((TCGRegSet)1 << ireg),
4665 preferred_regs, ots->indirect_base);
bellardc896fe22008-02-01 10:05:41 +00004666 }
Richard Henderson098859f2022-12-01 01:05:05 -08004667 if (!tcg_out_mov(s, otype, oreg, ireg)) {
4668 /*
4669 * Cross register class move not supported.
4670 * Store the source register into the destination slot
4671 * and leave the destination temp as TEMP_VAL_MEM.
4672 */
4673 assert(!temp_readonly(ots));
4674 if (!ts->mem_allocated) {
4675 temp_allocate_frame(s, ots);
4676 }
4677 tcg_out_st(s, ts->type, ireg, ots->mem_base->reg, ots->mem_offset);
4678 set_temp_val_nonreg(s, ts, TEMP_VAL_MEM);
4679 ots->mem_coherent = 1;
4680 return;
bellardc896fe22008-02-01 10:05:41 +00004681 }
Aurelien Jarnoec7a8692012-10-09 21:53:07 +02004682 }
Richard Henderson098859f2022-12-01 01:05:05 -08004683 set_temp_val_reg(s, ots, oreg);
4684 ots->mem_coherent = 0;
4685
4686 if (NEED_SYNC_ARG(0)) {
4687 temp_sync(s, ots, allocated_regs, 0, 0);
4688 }
bellardc896fe22008-02-01 10:05:41 +00004689}
4690
Richard Hendersonbab16712019-03-18 11:20:27 -07004691/*
4692 * Specialized code generation for INDEX_op_dup_vec.
4693 */
4694static void tcg_reg_alloc_dup(TCGContext *s, const TCGOp *op)
4695{
4696 const TCGLifeData arg_life = op->life;
4697 TCGRegSet dup_out_regs, dup_in_regs;
4698 TCGTemp *its, *ots;
4699 TCGType itype, vtype;
4700 unsigned vece;
Richard Henderson31c96412022-10-19 12:41:15 +10004701 int lowpart_ofs;
Richard Hendersonbab16712019-03-18 11:20:27 -07004702 bool ok;
4703
4704 ots = arg_temp(op->args[0]);
4705 its = arg_temp(op->args[1]);
4706
4707 /* ENV should not be modified. */
Richard Hendersone01fa972020-03-29 10:40:49 -07004708 tcg_debug_assert(!temp_readonly(ots));
Richard Hendersonbab16712019-03-18 11:20:27 -07004709
4710 itype = its->type;
4711 vece = TCGOP_VECE(op);
4712 vtype = TCGOP_VECL(op) + TCG_TYPE_V64;
4713
4714 if (its->val_type == TEMP_VAL_CONST) {
4715 /* Propagate constant via movi -> dupi. */
4716 tcg_target_ulong val = its->val;
4717 if (IS_DEAD_ARG(1)) {
4718 temp_dead(s, its);
4719 }
Richard Henderson31fd8842022-11-11 15:10:51 +10004720 tcg_reg_alloc_do_movi(s, ots, val, arg_life, output_pref(op, 0));
Richard Hendersonbab16712019-03-18 11:20:27 -07004721 return;
4722 }
4723
Richard Henderson9be0d082020-09-03 15:19:03 -07004724 dup_out_regs = tcg_op_defs[INDEX_op_dup_vec].args_ct[0].regs;
4725 dup_in_regs = tcg_op_defs[INDEX_op_dup_vec].args_ct[1].regs;
Richard Hendersonbab16712019-03-18 11:20:27 -07004726
4727 /* Allocate the output register now. */
4728 if (ots->val_type != TEMP_VAL_REG) {
4729 TCGRegSet allocated_regs = s->reserved_regs;
Richard Henderson098859f2022-12-01 01:05:05 -08004730 TCGReg oreg;
Richard Hendersonbab16712019-03-18 11:20:27 -07004731
4732 if (!IS_DEAD_ARG(1) && its->val_type == TEMP_VAL_REG) {
4733 /* Make sure to not spill the input register. */
4734 tcg_regset_set_reg(allocated_regs, its->reg);
4735 }
Richard Henderson098859f2022-12-01 01:05:05 -08004736 oreg = tcg_reg_alloc(s, dup_out_regs, allocated_regs,
Richard Henderson31fd8842022-11-11 15:10:51 +10004737 output_pref(op, 0), ots->indirect_base);
Richard Henderson098859f2022-12-01 01:05:05 -08004738 set_temp_val_reg(s, ots, oreg);
Richard Hendersonbab16712019-03-18 11:20:27 -07004739 }
4740
4741 switch (its->val_type) {
4742 case TEMP_VAL_REG:
4743 /*
4744 * The dup constriaints must be broad, covering all possible VECE.
4745 * However, tcg_op_dup_vec() gets to see the VECE and we allow it
4746 * to fail, indicating that extra moves are required for that case.
4747 */
4748 if (tcg_regset_test_reg(dup_in_regs, its->reg)) {
4749 if (tcg_out_dup_vec(s, vtype, vece, ots->reg, its->reg)) {
4750 goto done;
4751 }
4752 /* Try again from memory or a vector input register. */
4753 }
4754 if (!its->mem_coherent) {
4755 /*
4756 * The input register is not synced, and so an extra store
4757 * would be required to use memory. Attempt an integer-vector
4758 * register move first. We do not have a TCGRegSet for this.
4759 */
4760 if (tcg_out_mov(s, itype, ots->reg, its->reg)) {
4761 break;
4762 }
4763 /* Sync the temp back to its slot and load from there. */
4764 temp_sync(s, its, s->reserved_regs, 0, 0);
4765 }
4766 /* fall through */
4767
4768 case TEMP_VAL_MEM:
Richard Henderson31c96412022-10-19 12:41:15 +10004769 lowpart_ofs = 0;
4770 if (HOST_BIG_ENDIAN) {
4771 lowpart_ofs = tcg_type_size(itype) - (1 << vece);
4772 }
Richard Hendersond6ecb4a2019-03-18 12:00:39 -07004773 if (tcg_out_dupm_vec(s, vtype, vece, ots->reg, its->mem_base->reg,
Richard Henderson31c96412022-10-19 12:41:15 +10004774 its->mem_offset + lowpart_ofs)) {
Richard Hendersond6ecb4a2019-03-18 12:00:39 -07004775 goto done;
4776 }
Richard Henderson098859f2022-12-01 01:05:05 -08004777 /* Load the input into the destination vector register. */
Richard Hendersonbab16712019-03-18 11:20:27 -07004778 tcg_out_ld(s, itype, ots->reg, its->mem_base->reg, its->mem_offset);
4779 break;
4780
4781 default:
4782 g_assert_not_reached();
4783 }
4784
4785 /* We now have a vector input register, so dup must succeed. */
4786 ok = tcg_out_dup_vec(s, vtype, vece, ots->reg, ots->reg);
4787 tcg_debug_assert(ok);
4788
4789 done:
Richard Henderson36f55392022-12-01 00:44:13 -08004790 ots->mem_coherent = 0;
Richard Hendersonbab16712019-03-18 11:20:27 -07004791 if (IS_DEAD_ARG(1)) {
4792 temp_dead(s, its);
4793 }
4794 if (NEED_SYNC_ARG(0)) {
4795 temp_sync(s, ots, s->reserved_regs, 0, 0);
4796 }
4797 if (IS_DEAD_ARG(0)) {
4798 temp_dead(s, ots);
4799 }
4800}
4801
Richard Hendersondd186292016-12-08 13:42:08 -08004802static void tcg_reg_alloc_op(TCGContext *s, const TCGOp *op)
bellardc896fe22008-02-01 10:05:41 +00004803{
Richard Hendersondd186292016-12-08 13:42:08 -08004804 const TCGLifeData arg_life = op->life;
4805 const TCGOpDef * const def = &tcg_op_defs[op->opc];
Richard Henderson82790a82016-11-18 08:35:03 +01004806 TCGRegSet i_allocated_regs;
4807 TCGRegSet o_allocated_regs;
Richard Hendersonb6638662013-09-18 14:54:45 -07004808 int i, k, nb_iargs, nb_oargs;
4809 TCGReg reg;
bellardc896fe22008-02-01 10:05:41 +00004810 TCGArg arg;
4811 const TCGArgConstraint *arg_ct;
4812 TCGTemp *ts;
4813 TCGArg new_args[TCG_MAX_OP_ARGS];
4814 int const_args[TCG_MAX_OP_ARGS];
Richard Henderson21e9a8a2023-10-27 15:44:45 -07004815 TCGCond op_cond;
bellardc896fe22008-02-01 10:05:41 +00004816
4817 nb_oargs = def->nb_oargs;
4818 nb_iargs = def->nb_iargs;
4819
4820 /* copy constants */
Richard Hendersona813e362022-11-30 22:38:25 -08004821 memcpy(new_args + nb_oargs + nb_iargs,
Richard Hendersondd186292016-12-08 13:42:08 -08004822 op->args + nb_oargs + nb_iargs,
bellardc896fe22008-02-01 10:05:41 +00004823 sizeof(TCGArg) * def->nb_cargs);
4824
Richard Hendersond21369f2017-09-11 11:58:44 -07004825 i_allocated_regs = s->reserved_regs;
4826 o_allocated_regs = s->reserved_regs;
Richard Henderson82790a82016-11-18 08:35:03 +01004827
Richard Henderson21e9a8a2023-10-27 15:44:45 -07004828 switch (op->opc) {
4829 case INDEX_op_brcond_i32:
4830 case INDEX_op_brcond_i64:
4831 op_cond = op->args[2];
4832 break;
4833 case INDEX_op_setcond_i32:
4834 case INDEX_op_setcond_i64:
4835 case INDEX_op_negsetcond_i32:
4836 case INDEX_op_negsetcond_i64:
4837 case INDEX_op_cmp_vec:
4838 op_cond = op->args[3];
4839 break;
4840 case INDEX_op_brcond2_i32:
4841 op_cond = op->args[4];
4842 break;
4843 case INDEX_op_movcond_i32:
4844 case INDEX_op_movcond_i64:
4845 case INDEX_op_setcond2_i32:
4846 case INDEX_op_cmpsel_vec:
4847 op_cond = op->args[5];
4848 break;
4849 default:
4850 /* No condition within opcode. */
4851 op_cond = TCG_COND_ALWAYS;
4852 break;
4853 }
4854
Richard Hendersona813e362022-11-30 22:38:25 -08004855 /* satisfy input constraints */
Richard Hendersondd186292016-12-08 13:42:08 -08004856 for (k = 0; k < nb_iargs; k++) {
Richard Henderson29f5e922022-10-14 07:37:38 +11004857 TCGRegSet i_preferred_regs, i_required_regs;
4858 bool allocate_new_reg, copyto_new_reg;
4859 TCGTemp *ts2;
4860 int i1, i2;
Richard Hendersond62816f2018-11-27 20:21:31 -08004861
Richard Henderson66792f92019-04-04 09:37:38 +07004862 i = def->args_ct[nb_oargs + k].sort_index;
Richard Hendersondd186292016-12-08 13:42:08 -08004863 arg = op->args[i];
bellardc896fe22008-02-01 10:05:41 +00004864 arg_ct = &def->args_ct[i];
Richard Henderson43439132017-06-19 23:18:10 -07004865 ts = arg_temp(arg);
Richard Henderson40ae5c62013-09-19 08:02:05 -07004866
4867 if (ts->val_type == TEMP_VAL_CONST
Richard Henderson21e9a8a2023-10-27 15:44:45 -07004868 && tcg_target_const_match(ts->val, arg_ct->ct, ts->type,
4869 op_cond, TCGOP_VECE(op))) {
Richard Henderson40ae5c62013-09-19 08:02:05 -07004870 /* constant is OK for instruction */
4871 const_args[i] = 1;
4872 new_args[i] = ts->val;
Richard Hendersond62816f2018-11-27 20:21:31 -08004873 continue;
bellardc896fe22008-02-01 10:05:41 +00004874 }
Richard Henderson40ae5c62013-09-19 08:02:05 -07004875
Richard Henderson1c1824d2022-10-09 21:06:31 -07004876 reg = ts->reg;
4877 i_preferred_regs = 0;
Richard Henderson29f5e922022-10-14 07:37:38 +11004878 i_required_regs = arg_ct->regs;
Richard Henderson1c1824d2022-10-09 21:06:31 -07004879 allocate_new_reg = false;
Richard Henderson29f5e922022-10-14 07:37:38 +11004880 copyto_new_reg = false;
Richard Henderson1c1824d2022-10-09 21:06:31 -07004881
Richard Henderson29f5e922022-10-14 07:37:38 +11004882 switch (arg_ct->pair) {
4883 case 0: /* not paired */
4884 if (arg_ct->ialias) {
Richard Henderson31fd8842022-11-11 15:10:51 +10004885 i_preferred_regs = output_pref(op, arg_ct->alias_index);
Richard Henderson29f5e922022-10-14 07:37:38 +11004886
4887 /*
4888 * If the input is readonly, then it cannot also be an
4889 * output and aliased to itself. If the input is not
4890 * dead after the instruction, we must allocate a new
4891 * register and move it.
4892 */
Ilya Leoshkevich22d2e532023-07-20 00:11:18 +02004893 if (temp_readonly(ts) || !IS_DEAD_ARG(i)
4894 || def->args_ct[arg_ct->alias_index].newreg) {
Richard Henderson29f5e922022-10-14 07:37:38 +11004895 allocate_new_reg = true;
4896 } else if (ts->val_type == TEMP_VAL_REG) {
4897 /*
4898 * Check if the current register has already been
4899 * allocated for another input.
4900 */
4901 allocate_new_reg =
4902 tcg_regset_test_reg(i_allocated_regs, reg);
4903 }
4904 }
4905 if (!allocate_new_reg) {
4906 temp_load(s, ts, i_required_regs, i_allocated_regs,
4907 i_preferred_regs);
4908 reg = ts->reg;
4909 allocate_new_reg = !tcg_regset_test_reg(i_required_regs, reg);
4910 }
4911 if (allocate_new_reg) {
4912 /*
4913 * Allocate a new register matching the constraint
4914 * and move the temporary register into it.
4915 */
4916 temp_load(s, ts, tcg_target_available_regs[ts->type],
4917 i_allocated_regs, 0);
4918 reg = tcg_reg_alloc(s, i_required_regs, i_allocated_regs,
4919 i_preferred_regs, ts->indirect_base);
4920 copyto_new_reg = true;
4921 }
4922 break;
4923
4924 case 1:
4925 /* First of an input pair; if i1 == i2, the second is an output. */
4926 i1 = i;
4927 i2 = arg_ct->pair_index;
4928 ts2 = i1 != i2 ? arg_temp(op->args[i2]) : NULL;
4929
4930 /*
4931 * It is easier to default to allocating a new pair
4932 * and to identify a few cases where it's not required.
4933 */
4934 if (arg_ct->ialias) {
Richard Henderson31fd8842022-11-11 15:10:51 +10004935 i_preferred_regs = output_pref(op, arg_ct->alias_index);
Richard Henderson29f5e922022-10-14 07:37:38 +11004936 if (IS_DEAD_ARG(i1) &&
4937 IS_DEAD_ARG(i2) &&
4938 !temp_readonly(ts) &&
4939 ts->val_type == TEMP_VAL_REG &&
4940 ts->reg < TCG_TARGET_NB_REGS - 1 &&
4941 tcg_regset_test_reg(i_required_regs, reg) &&
4942 !tcg_regset_test_reg(i_allocated_regs, reg) &&
4943 !tcg_regset_test_reg(i_allocated_regs, reg + 1) &&
4944 (ts2
4945 ? ts2->val_type == TEMP_VAL_REG &&
4946 ts2->reg == reg + 1 &&
4947 !temp_readonly(ts2)
4948 : s->reg_to_temp[reg + 1] == NULL)) {
4949 break;
4950 }
4951 } else {
4952 /* Without aliasing, the pair must also be an input. */
4953 tcg_debug_assert(ts2);
4954 if (ts->val_type == TEMP_VAL_REG &&
4955 ts2->val_type == TEMP_VAL_REG &&
4956 ts2->reg == reg + 1 &&
4957 tcg_regset_test_reg(i_required_regs, reg)) {
4958 break;
4959 }
4960 }
4961 reg = tcg_reg_alloc_pair(s, i_required_regs, i_allocated_regs,
4962 0, ts->indirect_base);
4963 goto do_pair;
4964
4965 case 2: /* pair second */
4966 reg = new_args[arg_ct->pair_index] + 1;
4967 goto do_pair;
4968
4969 case 3: /* ialias with second output, no first input */
4970 tcg_debug_assert(arg_ct->ialias);
Richard Henderson31fd8842022-11-11 15:10:51 +10004971 i_preferred_regs = output_pref(op, arg_ct->alias_index);
Richard Hendersond62816f2018-11-27 20:21:31 -08004972
Richard Henderson29f5e922022-10-14 07:37:38 +11004973 if (IS_DEAD_ARG(i) &&
4974 !temp_readonly(ts) &&
4975 ts->val_type == TEMP_VAL_REG &&
4976 reg > 0 &&
4977 s->reg_to_temp[reg - 1] == NULL &&
4978 tcg_regset_test_reg(i_required_regs, reg) &&
4979 !tcg_regset_test_reg(i_allocated_regs, reg) &&
4980 !tcg_regset_test_reg(i_allocated_regs, reg - 1)) {
4981 tcg_regset_set_reg(i_allocated_regs, reg - 1);
4982 break;
Richard Hendersonc0522132020-03-29 18:55:52 -07004983 }
Richard Henderson29f5e922022-10-14 07:37:38 +11004984 reg = tcg_reg_alloc_pair(s, i_required_regs >> 1,
4985 i_allocated_regs, 0,
4986 ts->indirect_base);
4987 tcg_regset_set_reg(i_allocated_regs, reg);
4988 reg += 1;
4989 goto do_pair;
Richard Hendersond62816f2018-11-27 20:21:31 -08004990
Richard Henderson29f5e922022-10-14 07:37:38 +11004991 do_pair:
Richard Hendersonc0522132020-03-29 18:55:52 -07004992 /*
Richard Henderson29f5e922022-10-14 07:37:38 +11004993 * If an aliased input is not dead after the instruction,
4994 * we must allocate a new register and move it.
Richard Hendersonc0522132020-03-29 18:55:52 -07004995 */
Richard Henderson29f5e922022-10-14 07:37:38 +11004996 if (arg_ct->ialias && (!IS_DEAD_ARG(i) || temp_readonly(ts))) {
4997 TCGRegSet t_allocated_regs = i_allocated_regs;
4998
4999 /*
5000 * Because of the alias, and the continued life, make sure
5001 * that the temp is somewhere *other* than the reg pair,
5002 * and we get a copy in reg.
5003 */
5004 tcg_regset_set_reg(t_allocated_regs, reg);
5005 tcg_regset_set_reg(t_allocated_regs, reg + 1);
5006 if (ts->val_type == TEMP_VAL_REG && ts->reg == reg) {
5007 /* If ts was already in reg, copy it somewhere else. */
5008 TCGReg nr;
5009 bool ok;
5010
5011 tcg_debug_assert(ts->kind != TEMP_FIXED);
5012 nr = tcg_reg_alloc(s, tcg_target_available_regs[ts->type],
5013 t_allocated_regs, 0, ts->indirect_base);
5014 ok = tcg_out_mov(s, ts->type, nr, reg);
5015 tcg_debug_assert(ok);
5016
5017 set_temp_val_reg(s, ts, nr);
5018 } else {
5019 temp_load(s, ts, tcg_target_available_regs[ts->type],
5020 t_allocated_regs, 0);
5021 copyto_new_reg = true;
5022 }
5023 } else {
5024 /* Preferably allocate to reg, otherwise copy. */
5025 i_required_regs = (TCGRegSet)1 << reg;
5026 temp_load(s, ts, i_required_regs, i_allocated_regs,
5027 i_preferred_regs);
5028 copyto_new_reg = ts->reg != reg;
5029 }
5030 break;
5031
5032 default:
5033 g_assert_not_reached();
5034 }
5035
5036 if (copyto_new_reg) {
Richard Henderson78113e82019-03-16 17:48:18 +00005037 if (!tcg_out_mov(s, ts->type, reg, ts->reg)) {
Richard Henderson240c08d2019-03-16 17:48:32 +00005038 /*
5039 * Cross register class move not supported. Sync the
5040 * temp back to its slot and load from there.
5041 */
5042 temp_sync(s, ts, i_allocated_regs, 0, 0);
5043 tcg_out_ld(s, ts->type, reg,
5044 ts->mem_base->reg, ts->mem_offset);
Richard Henderson78113e82019-03-16 17:48:18 +00005045 }
bellardc896fe22008-02-01 10:05:41 +00005046 }
bellardc896fe22008-02-01 10:05:41 +00005047 new_args[i] = reg;
5048 const_args[i] = 0;
Richard Henderson82790a82016-11-18 08:35:03 +01005049 tcg_regset_set_reg(i_allocated_regs, reg);
bellardc896fe22008-02-01 10:05:41 +00005050 }
Richard Hendersona813e362022-11-30 22:38:25 -08005051
Aurelien Jarnoa52ad072012-10-09 21:53:07 +02005052 /* mark dead temporaries and free the associated registers */
5053 for (i = nb_oargs; i < nb_oargs + nb_iargs; i++) {
5054 if (IS_DEAD_ARG(i)) {
Richard Henderson43439132017-06-19 23:18:10 -07005055 temp_dead(s, arg_temp(op->args[i]));
Aurelien Jarnoa52ad072012-10-09 21:53:07 +02005056 }
5057 }
5058
Richard Hendersonb4cb76e2020-10-08 15:21:43 -05005059 if (def->flags & TCG_OPF_COND_BRANCH) {
5060 tcg_reg_alloc_cbranch(s, i_allocated_regs);
5061 } else if (def->flags & TCG_OPF_BB_END) {
Richard Henderson82790a82016-11-18 08:35:03 +01005062 tcg_reg_alloc_bb_end(s, i_allocated_regs);
bellarde8996ee2008-05-23 17:33:39 +00005063 } else {
bellarde8996ee2008-05-23 17:33:39 +00005064 if (def->flags & TCG_OPF_CALL_CLOBBER) {
Richard Hendersona813e362022-11-30 22:38:25 -08005065 /* XXX: permit generic clobber register list ? */
Richard Hendersonc8074022016-02-09 10:43:42 +11005066 for (i = 0; i < TCG_TARGET_NB_REGS; i++) {
5067 if (tcg_regset_test_reg(tcg_target_call_clobber_regs, i)) {
Richard Henderson82790a82016-11-18 08:35:03 +01005068 tcg_reg_free(s, i, i_allocated_regs);
bellarde8996ee2008-05-23 17:33:39 +00005069 }
5070 }
Aurelien Jarno3d5c5f82012-10-09 21:53:08 +02005071 }
5072 if (def->flags & TCG_OPF_SIDE_EFFECTS) {
5073 /* sync globals if the op has side effects and might trigger
5074 an exception. */
Richard Henderson82790a82016-11-18 08:35:03 +01005075 sync_globals(s, i_allocated_regs);
bellarde8996ee2008-05-23 17:33:39 +00005076 }
Richard Hendersona813e362022-11-30 22:38:25 -08005077
bellarde8996ee2008-05-23 17:33:39 +00005078 /* satisfy the output constraints */
bellarde8996ee2008-05-23 17:33:39 +00005079 for(k = 0; k < nb_oargs; k++) {
Richard Henderson66792f92019-04-04 09:37:38 +07005080 i = def->args_ct[k].sort_index;
Richard Hendersondd186292016-12-08 13:42:08 -08005081 arg = op->args[i];
bellarde8996ee2008-05-23 17:33:39 +00005082 arg_ct = &def->args_ct[i];
Richard Henderson43439132017-06-19 23:18:10 -07005083 ts = arg_temp(arg);
Richard Hendersond63e3b62019-03-16 17:48:02 +00005084
5085 /* ENV should not be modified. */
Richard Hendersone01fa972020-03-29 10:40:49 -07005086 tcg_debug_assert(!temp_readonly(ts));
Richard Hendersond63e3b62019-03-16 17:48:02 +00005087
Richard Henderson29f5e922022-10-14 07:37:38 +11005088 switch (arg_ct->pair) {
5089 case 0: /* not paired */
5090 if (arg_ct->oalias && !const_args[arg_ct->alias_index]) {
5091 reg = new_args[arg_ct->alias_index];
5092 } else if (arg_ct->newreg) {
5093 reg = tcg_reg_alloc(s, arg_ct->regs,
5094 i_allocated_regs | o_allocated_regs,
Richard Henderson31fd8842022-11-11 15:10:51 +10005095 output_pref(op, k), ts->indirect_base);
Richard Henderson29f5e922022-10-14 07:37:38 +11005096 } else {
5097 reg = tcg_reg_alloc(s, arg_ct->regs, o_allocated_regs,
Richard Henderson31fd8842022-11-11 15:10:51 +10005098 output_pref(op, k), ts->indirect_base);
Richard Henderson29f5e922022-10-14 07:37:38 +11005099 }
5100 break;
5101
5102 case 1: /* first of pair */
Richard Henderson29f5e922022-10-14 07:37:38 +11005103 if (arg_ct->oalias) {
5104 reg = new_args[arg_ct->alias_index];
Richard Hendersonca5bed02024-01-02 01:27:18 +00005105 } else if (arg_ct->newreg) {
5106 reg = tcg_reg_alloc_pair(s, arg_ct->regs,
5107 i_allocated_regs | o_allocated_regs,
5108 output_pref(op, k),
5109 ts->indirect_base);
5110 } else {
5111 reg = tcg_reg_alloc_pair(s, arg_ct->regs, o_allocated_regs,
5112 output_pref(op, k),
5113 ts->indirect_base);
Richard Henderson29f5e922022-10-14 07:37:38 +11005114 }
Richard Henderson29f5e922022-10-14 07:37:38 +11005115 break;
5116
5117 case 2: /* second of pair */
Richard Henderson29f5e922022-10-14 07:37:38 +11005118 if (arg_ct->oalias) {
5119 reg = new_args[arg_ct->alias_index];
5120 } else {
5121 reg = new_args[arg_ct->pair_index] + 1;
5122 }
5123 break;
5124
5125 case 3: /* first of pair, aliasing with a second input */
5126 tcg_debug_assert(!arg_ct->newreg);
5127 reg = new_args[arg_ct->pair_index] - 1;
5128 break;
5129
5130 default:
5131 g_assert_not_reached();
bellarde8996ee2008-05-23 17:33:39 +00005132 }
Richard Henderson82790a82016-11-18 08:35:03 +01005133 tcg_regset_set_reg(o_allocated_regs, reg);
Richard Henderson098859f2022-12-01 01:05:05 -08005134 set_temp_val_reg(s, ts, reg);
Richard Hendersond63e3b62019-03-16 17:48:02 +00005135 ts->mem_coherent = 0;
bellarde8996ee2008-05-23 17:33:39 +00005136 new_args[i] = reg;
bellardc896fe22008-02-01 10:05:41 +00005137 }
5138 }
5139
bellardc896fe22008-02-01 10:05:41 +00005140 /* emit instruction */
Richard Henderson678155b2023-04-05 11:17:01 -07005141 switch (op->opc) {
5142 case INDEX_op_ext8s_i32:
5143 tcg_out_ext8s(s, TCG_TYPE_I32, new_args[0], new_args[1]);
5144 break;
5145 case INDEX_op_ext8s_i64:
5146 tcg_out_ext8s(s, TCG_TYPE_I64, new_args[0], new_args[1]);
5147 break;
Richard Hendersond0e66c82023-04-05 13:26:51 -07005148 case INDEX_op_ext8u_i32:
5149 case INDEX_op_ext8u_i64:
5150 tcg_out_ext8u(s, new_args[0], new_args[1]);
5151 break;
Richard Henderson753e42e2023-04-05 14:49:59 -07005152 case INDEX_op_ext16s_i32:
5153 tcg_out_ext16s(s, TCG_TYPE_I32, new_args[0], new_args[1]);
5154 break;
5155 case INDEX_op_ext16s_i64:
5156 tcg_out_ext16s(s, TCG_TYPE_I64, new_args[0], new_args[1]);
5157 break;
Richard Henderson379afdf2023-04-05 16:25:22 -07005158 case INDEX_op_ext16u_i32:
5159 case INDEX_op_ext16u_i64:
5160 tcg_out_ext16u(s, new_args[0], new_args[1]);
5161 break;
Richard Henderson52bf3392023-04-05 17:50:09 -07005162 case INDEX_op_ext32s_i64:
5163 tcg_out_ext32s(s, new_args[0], new_args[1]);
5164 break;
Richard Henderson9ecf5f62023-04-05 18:07:05 -07005165 case INDEX_op_ext32u_i64:
5166 tcg_out_ext32u(s, new_args[0], new_args[1]);
5167 break;
Richard Henderson9c6aa272023-04-05 18:30:56 -07005168 case INDEX_op_ext_i32_i64:
5169 tcg_out_exts_i32_i64(s, new_args[0], new_args[1]);
5170 break;
Richard Hendersonb9bfe002023-04-05 18:56:28 -07005171 case INDEX_op_extu_i32_i64:
5172 tcg_out_extu_i32_i64(s, new_args[0], new_args[1]);
5173 break;
Richard Hendersonb8b94ac2023-04-05 19:58:35 -07005174 case INDEX_op_extrl_i64_i32:
5175 tcg_out_extrl_i64_i32(s, new_args[0], new_args[1]);
5176 break;
Richard Henderson678155b2023-04-05 11:17:01 -07005177 default:
5178 if (def->flags & TCG_OPF_VECTOR) {
5179 tcg_out_vec_op(s, op->opc, TCGOP_VECL(op), TCGOP_VECE(op),
5180 new_args, const_args);
5181 } else {
5182 tcg_out_op(s, op->opc, new_args, const_args);
5183 }
5184 break;
Richard Hendersond2fd7452017-09-14 13:53:46 -07005185 }
5186
bellardc896fe22008-02-01 10:05:41 +00005187 /* move the outputs in the correct register if needed */
5188 for(i = 0; i < nb_oargs; i++) {
Richard Henderson43439132017-06-19 23:18:10 -07005189 ts = arg_temp(op->args[i]);
Richard Hendersond63e3b62019-03-16 17:48:02 +00005190
5191 /* ENV should not be modified. */
Richard Hendersone01fa972020-03-29 10:40:49 -07005192 tcg_debug_assert(!temp_readonly(ts));
Richard Hendersond63e3b62019-03-16 17:48:02 +00005193
Aurelien Jarnoec7a8692012-10-09 21:53:07 +02005194 if (NEED_SYNC_ARG(i)) {
Richard Henderson98b4e182018-11-27 15:35:04 -08005195 temp_sync(s, ts, o_allocated_regs, 0, IS_DEAD_ARG(i));
Richard Henderson59d7c142016-06-19 22:59:13 -07005196 } else if (IS_DEAD_ARG(i)) {
Richard Hendersonf8bf00f2013-09-18 15:29:18 -07005197 temp_dead(s, ts);
Aurelien Jarnoec7a8692012-10-09 21:53:07 +02005198 }
bellardc896fe22008-02-01 10:05:41 +00005199 }
5200}
5201
Richard Hendersonefe86b22020-03-31 02:33:21 -07005202static bool tcg_reg_alloc_dup2(TCGContext *s, const TCGOp *op)
5203{
5204 const TCGLifeData arg_life = op->life;
5205 TCGTemp *ots, *itsl, *itsh;
5206 TCGType vtype = TCGOP_VECL(op) + TCG_TYPE_V64;
5207
5208 /* This opcode is only valid for 32-bit hosts, for 64-bit elements. */
5209 tcg_debug_assert(TCG_TARGET_REG_BITS == 32);
5210 tcg_debug_assert(TCGOP_VECE(op) == MO_64);
5211
5212 ots = arg_temp(op->args[0]);
5213 itsl = arg_temp(op->args[1]);
5214 itsh = arg_temp(op->args[2]);
5215
5216 /* ENV should not be modified. */
5217 tcg_debug_assert(!temp_readonly(ots));
5218
5219 /* Allocate the output register now. */
5220 if (ots->val_type != TEMP_VAL_REG) {
5221 TCGRegSet allocated_regs = s->reserved_regs;
5222 TCGRegSet dup_out_regs =
5223 tcg_op_defs[INDEX_op_dup_vec].args_ct[0].regs;
Richard Henderson098859f2022-12-01 01:05:05 -08005224 TCGReg oreg;
Richard Hendersonefe86b22020-03-31 02:33:21 -07005225
5226 /* Make sure to not spill the input registers. */
5227 if (!IS_DEAD_ARG(1) && itsl->val_type == TEMP_VAL_REG) {
5228 tcg_regset_set_reg(allocated_regs, itsl->reg);
5229 }
5230 if (!IS_DEAD_ARG(2) && itsh->val_type == TEMP_VAL_REG) {
5231 tcg_regset_set_reg(allocated_regs, itsh->reg);
5232 }
5233
Richard Henderson098859f2022-12-01 01:05:05 -08005234 oreg = tcg_reg_alloc(s, dup_out_regs, allocated_regs,
Richard Henderson31fd8842022-11-11 15:10:51 +10005235 output_pref(op, 0), ots->indirect_base);
Richard Henderson098859f2022-12-01 01:05:05 -08005236 set_temp_val_reg(s, ots, oreg);
Richard Hendersonefe86b22020-03-31 02:33:21 -07005237 }
5238
5239 /* Promote dup2 of immediates to dupi_vec. */
5240 if (itsl->val_type == TEMP_VAL_CONST && itsh->val_type == TEMP_VAL_CONST) {
5241 uint64_t val = deposit64(itsl->val, 32, 32, itsh->val);
5242 MemOp vece = MO_64;
5243
5244 if (val == dup_const(MO_8, val)) {
5245 vece = MO_8;
5246 } else if (val == dup_const(MO_16, val)) {
5247 vece = MO_16;
5248 } else if (val == dup_const(MO_32, val)) {
5249 vece = MO_32;
5250 }
5251
5252 tcg_out_dupi_vec(s, vtype, vece, ots->reg, val);
5253 goto done;
5254 }
5255
5256 /* If the two inputs form one 64-bit value, try dupm_vec. */
Richard Hendersonaef85402022-10-19 11:53:27 +10005257 if (itsl->temp_subindex == HOST_BIG_ENDIAN &&
5258 itsh->temp_subindex == !HOST_BIG_ENDIAN &&
5259 itsl == itsh + (HOST_BIG_ENDIAN ? 1 : -1)) {
5260 TCGTemp *its = itsl - HOST_BIG_ENDIAN;
5261
5262 temp_sync(s, its + 0, s->reserved_regs, 0, 0);
5263 temp_sync(s, its + 1, s->reserved_regs, 0, 0);
5264
Richard Hendersonefe86b22020-03-31 02:33:21 -07005265 if (tcg_out_dupm_vec(s, vtype, MO_64, ots->reg,
5266 its->mem_base->reg, its->mem_offset)) {
5267 goto done;
5268 }
5269 }
5270
5271 /* Fall back to generic expansion. */
5272 return false;
5273
5274 done:
Richard Henderson36f55392022-12-01 00:44:13 -08005275 ots->mem_coherent = 0;
Richard Hendersonefe86b22020-03-31 02:33:21 -07005276 if (IS_DEAD_ARG(1)) {
5277 temp_dead(s, itsl);
5278 }
5279 if (IS_DEAD_ARG(2)) {
5280 temp_dead(s, itsh);
5281 }
5282 if (NEED_SYNC_ARG(0)) {
5283 temp_sync(s, ots, s->reserved_regs, 0, IS_DEAD_ARG(0));
5284 } else if (IS_DEAD_ARG(0)) {
5285 temp_dead(s, ots);
5286 }
5287 return true;
5288}
5289
Richard Henderson39004a72022-11-11 10:09:37 +10005290static void load_arg_reg(TCGContext *s, TCGReg reg, TCGTemp *ts,
5291 TCGRegSet allocated_regs)
5292{
5293 if (ts->val_type == TEMP_VAL_REG) {
5294 if (ts->reg != reg) {
5295 tcg_reg_free(s, reg, allocated_regs);
5296 if (!tcg_out_mov(s, ts->type, reg, ts->reg)) {
5297 /*
5298 * Cross register class move not supported. Sync the
5299 * temp back to its slot and load from there.
5300 */
5301 temp_sync(s, ts, allocated_regs, 0, 0);
5302 tcg_out_ld(s, ts->type, reg,
5303 ts->mem_base->reg, ts->mem_offset);
5304 }
5305 }
5306 } else {
5307 TCGRegSet arg_set = 0;
5308
5309 tcg_reg_free(s, reg, allocated_regs);
5310 tcg_regset_set_reg(arg_set, reg);
5311 temp_load(s, ts, arg_set, allocated_regs, 0);
5312 }
5313}
5314
Richard Hendersond78e4a42023-04-08 19:05:10 -07005315static void load_arg_stk(TCGContext *s, unsigned arg_slot, TCGTemp *ts,
Richard Henderson39004a72022-11-11 10:09:37 +10005316 TCGRegSet allocated_regs)
5317{
5318 /*
5319 * When the destination is on the stack, load up the temp and store.
5320 * If there are many call-saved registers, the temp might live to
5321 * see another use; otherwise it'll be discarded.
5322 */
5323 temp_load(s, ts, tcg_target_available_regs[ts->type], allocated_regs, 0);
5324 tcg_out_st(s, ts->type, ts->reg, TCG_REG_CALL_STACK,
Richard Hendersond78e4a42023-04-08 19:05:10 -07005325 arg_slot_stk_ofs(arg_slot));
Richard Henderson39004a72022-11-11 10:09:37 +10005326}
5327
5328static void load_arg_normal(TCGContext *s, const TCGCallArgumentLoc *l,
5329 TCGTemp *ts, TCGRegSet *allocated_regs)
5330{
Richard Henderson338b61e2023-04-08 17:28:07 -07005331 if (arg_slot_reg_p(l->arg_slot)) {
Richard Henderson39004a72022-11-11 10:09:37 +10005332 TCGReg reg = tcg_target_call_iarg_regs[l->arg_slot];
5333 load_arg_reg(s, reg, ts, *allocated_regs);
5334 tcg_regset_set_reg(*allocated_regs, reg);
5335 } else {
Richard Hendersond78e4a42023-04-08 19:05:10 -07005336 load_arg_stk(s, l->arg_slot, ts, *allocated_regs);
Richard Henderson39004a72022-11-11 10:09:37 +10005337 }
5338}
5339
Richard Hendersond78e4a42023-04-08 19:05:10 -07005340static void load_arg_ref(TCGContext *s, unsigned arg_slot, TCGReg ref_base,
Richard Henderson313bdea2022-10-31 09:22:59 +11005341 intptr_t ref_off, TCGRegSet *allocated_regs)
5342{
5343 TCGReg reg;
Richard Henderson313bdea2022-10-31 09:22:59 +11005344
Richard Hendersond78e4a42023-04-08 19:05:10 -07005345 if (arg_slot_reg_p(arg_slot)) {
Richard Henderson313bdea2022-10-31 09:22:59 +11005346 reg = tcg_target_call_iarg_regs[arg_slot];
5347 tcg_reg_free(s, reg, *allocated_regs);
5348 tcg_out_addi_ptr(s, reg, ref_base, ref_off);
5349 tcg_regset_set_reg(*allocated_regs, reg);
5350 } else {
5351 reg = tcg_reg_alloc(s, tcg_target_available_regs[TCG_TYPE_PTR],
5352 *allocated_regs, 0, false);
5353 tcg_out_addi_ptr(s, reg, ref_base, ref_off);
5354 tcg_out_st(s, TCG_TYPE_PTR, reg, TCG_REG_CALL_STACK,
Richard Hendersond78e4a42023-04-08 19:05:10 -07005355 arg_slot_stk_ofs(arg_slot));
Richard Henderson313bdea2022-10-31 09:22:59 +11005356 }
5357}
5358
Richard Hendersondd186292016-12-08 13:42:08 -08005359static void tcg_reg_alloc_call(TCGContext *s, TCGOp *op)
bellardc896fe22008-02-01 10:05:41 +00005360{
Richard Hendersoncd9090a2017-11-14 13:02:51 +01005361 const int nb_oargs = TCGOP_CALLO(op);
5362 const int nb_iargs = TCGOP_CALLI(op);
Richard Hendersondd186292016-12-08 13:42:08 -08005363 const TCGLifeData arg_life = op->life;
Richard Henderson39004a72022-11-11 10:09:37 +10005364 const TCGHelperInfo *info = tcg_call_info(op);
5365 TCGRegSet allocated_regs = s->reserved_regs;
5366 int i;
bellardc896fe22008-02-01 10:05:41 +00005367
Richard Henderson39004a72022-11-11 10:09:37 +10005368 /*
5369 * Move inputs into place in reverse order,
5370 * so that we place stacked arguments first.
5371 */
5372 for (i = nb_iargs - 1; i >= 0; --i) {
5373 const TCGCallArgumentLoc *loc = &info->in[i];
5374 TCGTemp *ts = arg_temp(op->args[nb_oargs + i]);
bellardc896fe22008-02-01 10:05:41 +00005375
Richard Henderson39004a72022-11-11 10:09:37 +10005376 switch (loc->kind) {
5377 case TCG_CALL_ARG_NORMAL:
5378 case TCG_CALL_ARG_EXTEND_U:
5379 case TCG_CALL_ARG_EXTEND_S:
5380 load_arg_normal(s, loc, ts, &allocated_regs);
5381 break;
Richard Henderson313bdea2022-10-31 09:22:59 +11005382 case TCG_CALL_ARG_BY_REF:
5383 load_arg_stk(s, loc->ref_slot, ts, allocated_regs);
5384 load_arg_ref(s, loc->arg_slot, TCG_REG_CALL_STACK,
Richard Hendersond78e4a42023-04-08 19:05:10 -07005385 arg_slot_stk_ofs(loc->ref_slot),
Richard Henderson313bdea2022-10-31 09:22:59 +11005386 &allocated_regs);
5387 break;
5388 case TCG_CALL_ARG_BY_REF_N:
5389 load_arg_stk(s, loc->ref_slot, ts, allocated_regs);
5390 break;
Richard Henderson39004a72022-11-11 10:09:37 +10005391 default:
5392 g_assert_not_reached();
bellardc896fe22008-02-01 10:05:41 +00005393 }
bellardc896fe22008-02-01 10:05:41 +00005394 }
Richard Hendersona813e362022-11-30 22:38:25 -08005395
Richard Henderson39004a72022-11-11 10:09:37 +10005396 /* Mark dead temporaries and free the associated registers. */
Richard Hendersondd186292016-12-08 13:42:08 -08005397 for (i = nb_oargs; i < nb_iargs + nb_oargs; i++) {
Aurelien Jarno866cb6c2011-05-17 18:25:45 +02005398 if (IS_DEAD_ARG(i)) {
Richard Henderson43439132017-06-19 23:18:10 -07005399 temp_dead(s, arg_temp(op->args[i]));
bellardc896fe22008-02-01 10:05:41 +00005400 }
5401 }
Richard Hendersona813e362022-11-30 22:38:25 -08005402
Richard Henderson39004a72022-11-11 10:09:37 +10005403 /* Clobber call registers. */
Richard Hendersonc8074022016-02-09 10:43:42 +11005404 for (i = 0; i < TCG_TARGET_NB_REGS; i++) {
5405 if (tcg_regset_test_reg(tcg_target_call_clobber_regs, i)) {
Richard Hendersonb3915db2013-09-19 10:36:18 -07005406 tcg_reg_free(s, i, allocated_regs);
bellardc896fe22008-02-01 10:05:41 +00005407 }
5408 }
Aurelien Jarno78505272012-10-09 21:53:08 +02005409
Richard Henderson39004a72022-11-11 10:09:37 +10005410 /*
5411 * Save globals if they might be written by the helper,
5412 * sync them if they might be read.
5413 */
5414 if (info->flags & TCG_CALL_NO_READ_GLOBALS) {
Aurelien Jarno78505272012-10-09 21:53:08 +02005415 /* Nothing to do */
Richard Henderson39004a72022-11-11 10:09:37 +10005416 } else if (info->flags & TCG_CALL_NO_WRITE_GLOBALS) {
Aurelien Jarno78505272012-10-09 21:53:08 +02005417 sync_globals(s, allocated_regs);
5418 } else {
aurel32b9c18f52009-04-06 12:33:59 +00005419 save_globals(s, allocated_regs);
5420 }
bellardc896fe22008-02-01 10:05:41 +00005421
Richard Henderson313bdea2022-10-31 09:22:59 +11005422 /*
5423 * If the ABI passes a pointer to the returned struct as the first
5424 * argument, load that now. Pass a pointer to the output home slot.
5425 */
5426 if (info->out_kind == TCG_CALL_RET_BY_REF) {
5427 TCGTemp *ts = arg_temp(op->args[0]);
5428
5429 if (!ts->mem_allocated) {
5430 temp_allocate_frame(s, ts);
5431 }
5432 load_arg_ref(s, 0, ts->mem_base->reg, ts->mem_offset, &allocated_regs);
5433 }
5434
Richard Hendersoncee44b02022-10-18 17:51:41 +10005435 tcg_out_call(s, tcg_call_func(op), info);
bellardc896fe22008-02-01 10:05:41 +00005436
Richard Henderson39004a72022-11-11 10:09:37 +10005437 /* Assign output registers and emit moves if needed. */
5438 switch (info->out_kind) {
5439 case TCG_CALL_RET_NORMAL:
5440 for (i = 0; i < nb_oargs; i++) {
5441 TCGTemp *ts = arg_temp(op->args[i]);
Richard Henderson5e3d0c12022-10-20 00:55:36 +10005442 TCGReg reg = tcg_target_call_oarg_reg(TCG_CALL_RET_NORMAL, i);
Richard Hendersond63e3b62019-03-16 17:48:02 +00005443
Richard Henderson39004a72022-11-11 10:09:37 +10005444 /* ENV should not be modified. */
5445 tcg_debug_assert(!temp_readonly(ts));
Richard Hendersond63e3b62019-03-16 17:48:02 +00005446
Richard Henderson39004a72022-11-11 10:09:37 +10005447 set_temp_val_reg(s, ts, reg);
5448 ts->mem_coherent = 0;
5449 }
5450 break;
Richard Henderson313bdea2022-10-31 09:22:59 +11005451
Richard Hendersonc6556aa2022-10-20 01:13:52 +10005452 case TCG_CALL_RET_BY_VEC:
5453 {
5454 TCGTemp *ts = arg_temp(op->args[0]);
5455
5456 tcg_debug_assert(ts->base_type == TCG_TYPE_I128);
5457 tcg_debug_assert(ts->temp_subindex == 0);
5458 if (!ts->mem_allocated) {
5459 temp_allocate_frame(s, ts);
5460 }
5461 tcg_out_st(s, TCG_TYPE_V128,
5462 tcg_target_call_oarg_reg(TCG_CALL_RET_BY_VEC, 0),
5463 ts->mem_base->reg, ts->mem_offset);
5464 }
5465 /* fall through to mark all parts in memory */
5466
Richard Henderson313bdea2022-10-31 09:22:59 +11005467 case TCG_CALL_RET_BY_REF:
5468 /* The callee has performed a write through the reference. */
5469 for (i = 0; i < nb_oargs; i++) {
5470 TCGTemp *ts = arg_temp(op->args[i]);
5471 ts->val_type = TEMP_VAL_MEM;
5472 }
5473 break;
5474
Richard Henderson39004a72022-11-11 10:09:37 +10005475 default:
5476 g_assert_not_reached();
5477 }
5478
5479 /* Flush or discard output registers as needed. */
5480 for (i = 0; i < nb_oargs; i++) {
5481 TCGTemp *ts = arg_temp(op->args[i]);
Richard Hendersond63e3b62019-03-16 17:48:02 +00005482 if (NEED_SYNC_ARG(i)) {
Richard Henderson39004a72022-11-11 10:09:37 +10005483 temp_sync(s, ts, s->reserved_regs, 0, IS_DEAD_ARG(i));
Richard Hendersond63e3b62019-03-16 17:48:02 +00005484 } else if (IS_DEAD_ARG(i)) {
5485 temp_dead(s, ts);
bellardc896fe22008-02-01 10:05:41 +00005486 }
5487 }
bellardc896fe22008-02-01 10:05:41 +00005488}
5489
Richard Hendersone63b8a22022-11-08 09:23:54 +11005490/**
5491 * atom_and_align_for_opc:
5492 * @s: tcg context
5493 * @opc: memory operation code
5494 * @host_atom: MO_ATOM_{IFALIGN,WITHIN16,SUBALIGN} for host operations
5495 * @allow_two_ops: true if we are prepared to issue two operations
5496 *
5497 * Return the alignment and atomicity to use for the inline fast path
5498 * for the given memory operation. The alignment may be larger than
5499 * that specified in @opc, and the correct alignment will be diagnosed
5500 * by the slow path helper.
5501 *
5502 * If @allow_two_ops, the host is prepared to test for 2x alignment,
5503 * and issue two loads or stores for subalignment.
5504 */
5505static TCGAtomAlign atom_and_align_for_opc(TCGContext *s, MemOp opc,
5506 MemOp host_atom, bool allow_two_ops)
5507{
Richard Hendersonc5809ee2024-10-04 13:34:42 -07005508 MemOp align = memop_alignment_bits(opc);
Richard Hendersone63b8a22022-11-08 09:23:54 +11005509 MemOp size = opc & MO_SIZE;
5510 MemOp half = size ? size - 1 : 0;
Richard Hendersoncbb14552023-12-12 11:35:42 -08005511 MemOp atom = opc & MO_ATOM_MASK;
Richard Hendersone63b8a22022-11-08 09:23:54 +11005512 MemOp atmax;
Richard Hendersone63b8a22022-11-08 09:23:54 +11005513
5514 switch (atom) {
5515 case MO_ATOM_NONE:
5516 /* The operation requires no specific atomicity. */
5517 atmax = MO_8;
5518 break;
5519
5520 case MO_ATOM_IFALIGN:
5521 atmax = size;
5522 break;
5523
5524 case MO_ATOM_IFALIGN_PAIR:
5525 atmax = half;
5526 break;
5527
5528 case MO_ATOM_WITHIN16:
5529 atmax = size;
5530 if (size == MO_128) {
5531 /* Misalignment implies !within16, and therefore no atomicity. */
5532 } else if (host_atom != MO_ATOM_WITHIN16) {
5533 /* The host does not implement within16, so require alignment. */
5534 align = MAX(align, size);
5535 }
5536 break;
5537
5538 case MO_ATOM_WITHIN16_PAIR:
5539 atmax = size;
5540 /*
5541 * Misalignment implies !within16, and therefore half atomicity.
5542 * Any host prepared for two operations can implement this with
5543 * half alignment.
5544 */
5545 if (host_atom != MO_ATOM_WITHIN16 && allow_two_ops) {
5546 align = MAX(align, half);
5547 }
5548 break;
5549
5550 case MO_ATOM_SUBALIGN:
5551 atmax = size;
5552 if (host_atom != MO_ATOM_SUBALIGN) {
5553 /* If unaligned but not odd, there are subobjects up to half. */
5554 if (allow_two_ops) {
5555 align = MAX(align, half);
5556 } else {
5557 align = MAX(align, size);
5558 }
5559 }
5560 break;
5561
5562 default:
5563 g_assert_not_reached();
5564 }
5565
5566 return (TCGAtomAlign){ .atom = atmax, .align = align };
5567}
5568
Richard Henderson8429a1c2023-04-09 22:59:09 -07005569/*
5570 * Similarly for qemu_ld/st slow path helpers.
5571 * We must re-implement tcg_gen_callN and tcg_reg_alloc_call simultaneously,
5572 * using only the provided backend tcg_out_* functions.
5573 */
5574
5575static int tcg_out_helper_stk_ofs(TCGType type, unsigned slot)
5576{
5577 int ofs = arg_slot_stk_ofs(slot);
5578
5579 /*
5580 * Each stack slot is TCG_TARGET_LONG_BITS. If the host does not
5581 * require extension to uint64_t, adjust the address for uint32_t.
5582 */
5583 if (HOST_BIG_ENDIAN &&
5584 TCG_TARGET_REG_BITS == 64 &&
5585 type == TCG_TYPE_I32) {
5586 ofs += 4;
5587 }
5588 return ofs;
5589}
5590
Richard Henderson8d314042023-05-14 10:07:22 -07005591static void tcg_out_helper_load_slots(TCGContext *s,
5592 unsigned nmov, TCGMovExtend *mov,
5593 const TCGLdstHelperParam *parm)
Richard Henderson8429a1c2023-04-09 22:59:09 -07005594{
Richard Henderson8d314042023-05-14 10:07:22 -07005595 unsigned i;
Richard Henderson2462e302023-05-14 09:58:39 -07005596 TCGReg dst3;
5597
Richard Henderson8d314042023-05-14 10:07:22 -07005598 /*
5599 * Start from the end, storing to the stack first.
5600 * This frees those registers, so we need not consider overlap.
5601 */
5602 for (i = nmov; i-- > 0; ) {
5603 unsigned slot = mov[i].dst;
5604
5605 if (arg_slot_reg_p(slot)) {
5606 goto found_reg;
5607 }
5608
5609 TCGReg src = mov[i].src;
5610 TCGType dst_type = mov[i].dst_type;
5611 MemOp dst_mo = dst_type == TCG_TYPE_I32 ? MO_32 : MO_64;
5612
5613 /* The argument is going onto the stack; extend into scratch. */
5614 if ((mov[i].src_ext & MO_SIZE) != dst_mo) {
5615 tcg_debug_assert(parm->ntmp != 0);
5616 mov[i].dst = src = parm->tmp[0];
5617 tcg_out_movext1(s, &mov[i]);
5618 }
5619
5620 tcg_out_st(s, dst_type, src, TCG_REG_CALL_STACK,
5621 tcg_out_helper_stk_ofs(dst_type, slot));
5622 }
5623 return;
5624
5625 found_reg:
5626 /*
5627 * The remaining arguments are in registers.
5628 * Convert slot numbers to argument registers.
5629 */
5630 nmov = i + 1;
5631 for (i = 0; i < nmov; ++i) {
5632 mov[i].dst = tcg_target_call_iarg_regs[mov[i].dst];
5633 }
5634
Richard Henderson8429a1c2023-04-09 22:59:09 -07005635 switch (nmov) {
Richard Henderson2462e302023-05-14 09:58:39 -07005636 case 4:
Richard Henderson8429a1c2023-04-09 22:59:09 -07005637 /* The backend must have provided enough temps for the worst case. */
Richard Henderson2462e302023-05-14 09:58:39 -07005638 tcg_debug_assert(parm->ntmp >= 2);
Richard Henderson8429a1c2023-04-09 22:59:09 -07005639
Richard Henderson2462e302023-05-14 09:58:39 -07005640 dst3 = mov[3].dst;
5641 for (unsigned j = 0; j < 3; ++j) {
5642 if (dst3 == mov[j].src) {
5643 /*
5644 * Conflict. Copy the source to a temporary, perform the
5645 * remaining moves, then the extension from our scratch
5646 * on the way out.
5647 */
5648 TCGReg scratch = parm->tmp[1];
Richard Henderson8429a1c2023-04-09 22:59:09 -07005649
Richard Henderson2462e302023-05-14 09:58:39 -07005650 tcg_out_mov(s, mov[3].src_type, scratch, mov[3].src);
5651 tcg_out_movext3(s, mov, mov + 1, mov + 2, parm->tmp[0]);
5652 tcg_out_movext1_new_src(s, &mov[3], scratch);
5653 break;
Richard Henderson8429a1c2023-04-09 22:59:09 -07005654 }
Richard Henderson8429a1c2023-04-09 22:59:09 -07005655 }
Richard Henderson8429a1c2023-04-09 22:59:09 -07005656
Richard Henderson2462e302023-05-14 09:58:39 -07005657 /* No conflicts: perform this move and continue. */
5658 tcg_out_movext1(s, &mov[3]);
5659 /* fall through */
5660
5661 case 3:
5662 tcg_out_movext3(s, mov, mov + 1, mov + 2,
5663 parm->ntmp ? parm->tmp[0] : -1);
5664 break;
Richard Henderson8429a1c2023-04-09 22:59:09 -07005665 case 2:
Richard Henderson2462e302023-05-14 09:58:39 -07005666 tcg_out_movext2(s, mov, mov + 1,
5667 parm->ntmp ? parm->tmp[0] : -1);
5668 break;
Richard Henderson8429a1c2023-04-09 22:59:09 -07005669 case 1:
5670 tcg_out_movext1(s, mov);
Richard Henderson2462e302023-05-14 09:58:39 -07005671 break;
5672 default:
Richard Henderson8429a1c2023-04-09 22:59:09 -07005673 g_assert_not_reached();
5674 }
5675}
5676
Richard Henderson8429a1c2023-04-09 22:59:09 -07005677static void tcg_out_helper_load_imm(TCGContext *s, unsigned slot,
5678 TCGType type, tcg_target_long imm,
5679 const TCGLdstHelperParam *parm)
5680{
5681 if (arg_slot_reg_p(slot)) {
5682 tcg_out_movi(s, type, tcg_target_call_iarg_regs[slot], imm);
5683 } else {
5684 int ofs = tcg_out_helper_stk_ofs(type, slot);
5685 if (!tcg_out_sti(s, type, imm, TCG_REG_CALL_STACK, ofs)) {
5686 tcg_debug_assert(parm->ntmp != 0);
5687 tcg_out_movi(s, type, parm->tmp[0], imm);
5688 tcg_out_st(s, type, parm->tmp[0], TCG_REG_CALL_STACK, ofs);
5689 }
5690 }
5691}
5692
5693static void tcg_out_helper_load_common_args(TCGContext *s,
5694 const TCGLabelQemuLdst *ldst,
5695 const TCGLdstHelperParam *parm,
5696 const TCGHelperInfo *info,
5697 unsigned next_arg)
5698{
5699 TCGMovExtend ptr_mov = {
5700 .dst_type = TCG_TYPE_PTR,
5701 .src_type = TCG_TYPE_PTR,
5702 .src_ext = sizeof(void *) == 4 ? MO_32 : MO_64
5703 };
5704 const TCGCallArgumentLoc *loc = &info->in[0];
5705 TCGType type;
5706 unsigned slot;
5707 tcg_target_ulong imm;
5708
5709 /*
5710 * Handle env, which is always first.
5711 */
5712 ptr_mov.dst = loc->arg_slot;
5713 ptr_mov.src = TCG_AREG0;
5714 tcg_out_helper_load_slots(s, 1, &ptr_mov, parm);
5715
5716 /*
5717 * Handle oi.
5718 */
5719 imm = ldst->oi;
5720 loc = &info->in[next_arg];
5721 type = TCG_TYPE_I32;
5722 switch (loc->kind) {
5723 case TCG_CALL_ARG_NORMAL:
5724 break;
5725 case TCG_CALL_ARG_EXTEND_U:
5726 case TCG_CALL_ARG_EXTEND_S:
5727 /* No extension required for MemOpIdx. */
5728 tcg_debug_assert(imm <= INT32_MAX);
5729 type = TCG_TYPE_REG;
5730 break;
5731 default:
5732 g_assert_not_reached();
5733 }
5734 tcg_out_helper_load_imm(s, loc->arg_slot, type, imm, parm);
5735 next_arg++;
5736
5737 /*
5738 * Handle ra.
5739 */
5740 loc = &info->in[next_arg];
5741 slot = loc->arg_slot;
5742 if (parm->ra_gen) {
5743 int arg_reg = -1;
5744 TCGReg ra_reg;
5745
5746 if (arg_slot_reg_p(slot)) {
5747 arg_reg = tcg_target_call_iarg_regs[slot];
5748 }
5749 ra_reg = parm->ra_gen(s, ldst, arg_reg);
5750
5751 ptr_mov.dst = slot;
5752 ptr_mov.src = ra_reg;
5753 tcg_out_helper_load_slots(s, 1, &ptr_mov, parm);
5754 } else {
5755 imm = (uintptr_t)ldst->raddr;
5756 tcg_out_helper_load_imm(s, slot, TCG_TYPE_PTR, imm, parm);
5757 }
5758}
5759
5760static unsigned tcg_out_helper_add_mov(TCGMovExtend *mov,
5761 const TCGCallArgumentLoc *loc,
5762 TCGType dst_type, TCGType src_type,
5763 TCGReg lo, TCGReg hi)
5764{
Richard Hendersonebebea52023-04-17 10:20:51 +02005765 MemOp reg_mo;
5766
Richard Henderson8429a1c2023-04-09 22:59:09 -07005767 if (dst_type <= TCG_TYPE_REG) {
5768 MemOp src_ext;
5769
5770 switch (loc->kind) {
5771 case TCG_CALL_ARG_NORMAL:
5772 src_ext = src_type == TCG_TYPE_I32 ? MO_32 : MO_64;
5773 break;
5774 case TCG_CALL_ARG_EXTEND_U:
5775 dst_type = TCG_TYPE_REG;
5776 src_ext = MO_UL;
5777 break;
5778 case TCG_CALL_ARG_EXTEND_S:
5779 dst_type = TCG_TYPE_REG;
5780 src_ext = MO_SL;
5781 break;
5782 default:
5783 g_assert_not_reached();
5784 }
5785
5786 mov[0].dst = loc->arg_slot;
5787 mov[0].dst_type = dst_type;
5788 mov[0].src = lo;
5789 mov[0].src_type = src_type;
5790 mov[0].src_ext = src_ext;
5791 return 1;
5792 }
5793
Richard Hendersonebebea52023-04-17 10:20:51 +02005794 if (TCG_TARGET_REG_BITS == 32) {
5795 assert(dst_type == TCG_TYPE_I64);
5796 reg_mo = MO_32;
5797 } else {
5798 assert(dst_type == TCG_TYPE_I128);
5799 reg_mo = MO_64;
5800 }
Richard Henderson8429a1c2023-04-09 22:59:09 -07005801
5802 mov[0].dst = loc[HOST_BIG_ENDIAN].arg_slot;
5803 mov[0].src = lo;
Richard Hendersonebebea52023-04-17 10:20:51 +02005804 mov[0].dst_type = TCG_TYPE_REG;
5805 mov[0].src_type = TCG_TYPE_REG;
5806 mov[0].src_ext = reg_mo;
Richard Henderson8429a1c2023-04-09 22:59:09 -07005807
5808 mov[1].dst = loc[!HOST_BIG_ENDIAN].arg_slot;
5809 mov[1].src = hi;
Richard Hendersonebebea52023-04-17 10:20:51 +02005810 mov[1].dst_type = TCG_TYPE_REG;
5811 mov[1].src_type = TCG_TYPE_REG;
5812 mov[1].src_ext = reg_mo;
Richard Henderson8429a1c2023-04-09 22:59:09 -07005813
5814 return 2;
5815}
5816
5817static void tcg_out_ld_helper_args(TCGContext *s, const TCGLabelQemuLdst *ldst,
5818 const TCGLdstHelperParam *parm)
5819{
5820 const TCGHelperInfo *info;
5821 const TCGCallArgumentLoc *loc;
5822 TCGMovExtend mov[2];
5823 unsigned next_arg, nmov;
5824 MemOp mop = get_memop(ldst->oi);
5825
5826 switch (mop & MO_SIZE) {
5827 case MO_8:
5828 case MO_16:
5829 case MO_32:
5830 info = &info_helper_ld32_mmu;
5831 break;
5832 case MO_64:
5833 info = &info_helper_ld64_mmu;
5834 break;
Richard Hendersonebebea52023-04-17 10:20:51 +02005835 case MO_128:
5836 info = &info_helper_ld128_mmu;
5837 break;
Richard Henderson8429a1c2023-04-09 22:59:09 -07005838 default:
5839 g_assert_not_reached();
5840 }
5841
5842 /* Defer env argument. */
5843 next_arg = 1;
5844
5845 loc = &info->in[next_arg];
Richard Hendersonc31e5fa2023-04-28 09:14:17 +01005846 if (TCG_TARGET_REG_BITS == 32 && s->addr_type == TCG_TYPE_I32) {
Richard Henderson24e46e62023-04-26 22:09:47 +01005847 /*
5848 * 32-bit host with 32-bit guest: zero-extend the guest address
5849 * to 64-bits for the helper by storing the low part, then
5850 * load a zero for the high part.
5851 */
5852 tcg_out_helper_add_mov(mov, loc + HOST_BIG_ENDIAN,
5853 TCG_TYPE_I32, TCG_TYPE_I32,
5854 ldst->addrlo_reg, -1);
5855 tcg_out_helper_load_slots(s, 1, mov, parm);
Richard Henderson8429a1c2023-04-09 22:59:09 -07005856
Richard Henderson24e46e62023-04-26 22:09:47 +01005857 tcg_out_helper_load_imm(s, loc[!HOST_BIG_ENDIAN].arg_slot,
5858 TCG_TYPE_I32, 0, parm);
5859 next_arg += 2;
Richard Hendersonc31e5fa2023-04-28 09:14:17 +01005860 } else {
5861 nmov = tcg_out_helper_add_mov(mov, loc, TCG_TYPE_I64, s->addr_type,
5862 ldst->addrlo_reg, ldst->addrhi_reg);
5863 tcg_out_helper_load_slots(s, nmov, mov, parm);
5864 next_arg += nmov;
Richard Henderson24e46e62023-04-26 22:09:47 +01005865 }
Richard Henderson8429a1c2023-04-09 22:59:09 -07005866
Richard Hendersonebebea52023-04-17 10:20:51 +02005867 switch (info->out_kind) {
5868 case TCG_CALL_RET_NORMAL:
5869 case TCG_CALL_RET_BY_VEC:
5870 break;
5871 case TCG_CALL_RET_BY_REF:
5872 /*
5873 * The return reference is in the first argument slot.
5874 * We need memory in which to return: re-use the top of stack.
5875 */
5876 {
5877 int ofs_slot0 = TCG_TARGET_CALL_STACK_OFFSET;
5878
5879 if (arg_slot_reg_p(0)) {
5880 tcg_out_addi_ptr(s, tcg_target_call_iarg_regs[0],
5881 TCG_REG_CALL_STACK, ofs_slot0);
5882 } else {
5883 tcg_debug_assert(parm->ntmp != 0);
5884 tcg_out_addi_ptr(s, parm->tmp[0],
5885 TCG_REG_CALL_STACK, ofs_slot0);
5886 tcg_out_st(s, TCG_TYPE_PTR, parm->tmp[0],
5887 TCG_REG_CALL_STACK, ofs_slot0);
5888 }
5889 }
5890 break;
5891 default:
5892 g_assert_not_reached();
5893 }
Richard Henderson8429a1c2023-04-09 22:59:09 -07005894
5895 tcg_out_helper_load_common_args(s, ldst, parm, info, next_arg);
5896}
5897
5898static void tcg_out_ld_helper_ret(TCGContext *s, const TCGLabelQemuLdst *ldst,
5899 bool load_sign,
5900 const TCGLdstHelperParam *parm)
5901{
Richard Hendersonebebea52023-04-17 10:20:51 +02005902 MemOp mop = get_memop(ldst->oi);
Richard Henderson8429a1c2023-04-09 22:59:09 -07005903 TCGMovExtend mov[2];
Richard Hendersonebebea52023-04-17 10:20:51 +02005904 int ofs_slot0;
Richard Henderson8429a1c2023-04-09 22:59:09 -07005905
Richard Hendersonebebea52023-04-17 10:20:51 +02005906 switch (ldst->type) {
5907 case TCG_TYPE_I64:
5908 if (TCG_TARGET_REG_BITS == 32) {
5909 break;
5910 }
5911 /* fall through */
Richard Henderson8429a1c2023-04-09 22:59:09 -07005912
Richard Hendersonebebea52023-04-17 10:20:51 +02005913 case TCG_TYPE_I32:
Richard Henderson8429a1c2023-04-09 22:59:09 -07005914 mov[0].dst = ldst->datalo_reg;
5915 mov[0].src = tcg_target_call_oarg_reg(TCG_CALL_RET_NORMAL, 0);
5916 mov[0].dst_type = ldst->type;
5917 mov[0].src_type = TCG_TYPE_REG;
5918
5919 /*
5920 * If load_sign, then we allowed the helper to perform the
5921 * appropriate sign extension to tcg_target_ulong, and all
5922 * we need now is a plain move.
5923 *
5924 * If they do not, then we expect the relevant extension
5925 * instruction to be no more expensive than a move, and
5926 * we thus save the icache etc by only using one of two
5927 * helper functions.
5928 */
5929 if (load_sign || !(mop & MO_SIGN)) {
5930 if (TCG_TARGET_REG_BITS == 32 || ldst->type == TCG_TYPE_I32) {
5931 mov[0].src_ext = MO_32;
5932 } else {
5933 mov[0].src_ext = MO_64;
5934 }
5935 } else {
5936 mov[0].src_ext = mop & MO_SSIZE;
5937 }
5938 tcg_out_movext1(s, mov);
Richard Hendersonebebea52023-04-17 10:20:51 +02005939 return;
Richard Henderson8429a1c2023-04-09 22:59:09 -07005940
Richard Hendersonebebea52023-04-17 10:20:51 +02005941 case TCG_TYPE_I128:
5942 tcg_debug_assert(TCG_TARGET_REG_BITS == 64);
5943 ofs_slot0 = TCG_TARGET_CALL_STACK_OFFSET;
5944 switch (TCG_TARGET_CALL_RET_I128) {
5945 case TCG_CALL_RET_NORMAL:
5946 break;
5947 case TCG_CALL_RET_BY_VEC:
5948 tcg_out_st(s, TCG_TYPE_V128,
5949 tcg_target_call_oarg_reg(TCG_CALL_RET_BY_VEC, 0),
5950 TCG_REG_CALL_STACK, ofs_slot0);
5951 /* fall through */
5952 case TCG_CALL_RET_BY_REF:
5953 tcg_out_ld(s, TCG_TYPE_I64, ldst->datalo_reg,
5954 TCG_REG_CALL_STACK, ofs_slot0 + 8 * HOST_BIG_ENDIAN);
5955 tcg_out_ld(s, TCG_TYPE_I64, ldst->datahi_reg,
5956 TCG_REG_CALL_STACK, ofs_slot0 + 8 * !HOST_BIG_ENDIAN);
5957 return;
5958 default:
5959 g_assert_not_reached();
5960 }
5961 break;
Richard Henderson8429a1c2023-04-09 22:59:09 -07005962
Richard Hendersonebebea52023-04-17 10:20:51 +02005963 default:
5964 g_assert_not_reached();
Richard Henderson8429a1c2023-04-09 22:59:09 -07005965 }
Richard Hendersonebebea52023-04-17 10:20:51 +02005966
5967 mov[0].dst = ldst->datalo_reg;
5968 mov[0].src =
5969 tcg_target_call_oarg_reg(TCG_CALL_RET_NORMAL, HOST_BIG_ENDIAN);
Richard Henderson723d3a22023-05-24 12:59:12 -07005970 mov[0].dst_type = TCG_TYPE_REG;
5971 mov[0].src_type = TCG_TYPE_REG;
Richard Hendersonebebea52023-04-17 10:20:51 +02005972 mov[0].src_ext = TCG_TARGET_REG_BITS == 32 ? MO_32 : MO_64;
5973
5974 mov[1].dst = ldst->datahi_reg;
5975 mov[1].src =
5976 tcg_target_call_oarg_reg(TCG_CALL_RET_NORMAL, !HOST_BIG_ENDIAN);
5977 mov[1].dst_type = TCG_TYPE_REG;
5978 mov[1].src_type = TCG_TYPE_REG;
5979 mov[1].src_ext = TCG_TARGET_REG_BITS == 32 ? MO_32 : MO_64;
5980
5981 tcg_out_movext2(s, mov, mov + 1, parm->ntmp ? parm->tmp[0] : -1);
Richard Henderson8429a1c2023-04-09 22:59:09 -07005982}
5983
5984static void tcg_out_st_helper_args(TCGContext *s, const TCGLabelQemuLdst *ldst,
5985 const TCGLdstHelperParam *parm)
5986{
5987 const TCGHelperInfo *info;
5988 const TCGCallArgumentLoc *loc;
5989 TCGMovExtend mov[4];
5990 TCGType data_type;
5991 unsigned next_arg, nmov, n;
5992 MemOp mop = get_memop(ldst->oi);
5993
5994 switch (mop & MO_SIZE) {
5995 case MO_8:
5996 case MO_16:
5997 case MO_32:
5998 info = &info_helper_st32_mmu;
5999 data_type = TCG_TYPE_I32;
6000 break;
6001 case MO_64:
6002 info = &info_helper_st64_mmu;
6003 data_type = TCG_TYPE_I64;
6004 break;
Richard Hendersonebebea52023-04-17 10:20:51 +02006005 case MO_128:
6006 info = &info_helper_st128_mmu;
6007 data_type = TCG_TYPE_I128;
6008 break;
Richard Henderson8429a1c2023-04-09 22:59:09 -07006009 default:
6010 g_assert_not_reached();
6011 }
6012
6013 /* Defer env argument. */
6014 next_arg = 1;
6015 nmov = 0;
6016
6017 /* Handle addr argument. */
6018 loc = &info->in[next_arg];
Richard Hendersonc31e5fa2023-04-28 09:14:17 +01006019 if (TCG_TARGET_REG_BITS == 32 && s->addr_type == TCG_TYPE_I32) {
Richard Henderson24e46e62023-04-26 22:09:47 +01006020 /*
6021 * 32-bit host with 32-bit guest: zero-extend the guest address
6022 * to 64-bits for the helper by storing the low part. Later,
6023 * after we have processed the register inputs, we will load a
6024 * zero for the high part.
6025 */
6026 tcg_out_helper_add_mov(mov, loc + HOST_BIG_ENDIAN,
6027 TCG_TYPE_I32, TCG_TYPE_I32,
6028 ldst->addrlo_reg, -1);
6029 next_arg += 2;
6030 nmov += 1;
Richard Hendersonc31e5fa2023-04-28 09:14:17 +01006031 } else {
6032 n = tcg_out_helper_add_mov(mov, loc, TCG_TYPE_I64, s->addr_type,
6033 ldst->addrlo_reg, ldst->addrhi_reg);
6034 next_arg += n;
6035 nmov += n;
Richard Henderson24e46e62023-04-26 22:09:47 +01006036 }
Richard Henderson8429a1c2023-04-09 22:59:09 -07006037
6038 /* Handle data argument. */
6039 loc = &info->in[next_arg];
Richard Hendersonebebea52023-04-17 10:20:51 +02006040 switch (loc->kind) {
6041 case TCG_CALL_ARG_NORMAL:
6042 case TCG_CALL_ARG_EXTEND_U:
6043 case TCG_CALL_ARG_EXTEND_S:
6044 n = tcg_out_helper_add_mov(mov + nmov, loc, data_type, ldst->type,
6045 ldst->datalo_reg, ldst->datahi_reg);
6046 next_arg += n;
6047 nmov += n;
6048 tcg_out_helper_load_slots(s, nmov, mov, parm);
6049 break;
Richard Henderson8429a1c2023-04-09 22:59:09 -07006050
Richard Hendersonebebea52023-04-17 10:20:51 +02006051 case TCG_CALL_ARG_BY_REF:
6052 tcg_debug_assert(TCG_TARGET_REG_BITS == 64);
6053 tcg_debug_assert(data_type == TCG_TYPE_I128);
6054 tcg_out_st(s, TCG_TYPE_I64,
6055 HOST_BIG_ENDIAN ? ldst->datahi_reg : ldst->datalo_reg,
6056 TCG_REG_CALL_STACK, arg_slot_stk_ofs(loc[0].ref_slot));
6057 tcg_out_st(s, TCG_TYPE_I64,
6058 HOST_BIG_ENDIAN ? ldst->datalo_reg : ldst->datahi_reg,
6059 TCG_REG_CALL_STACK, arg_slot_stk_ofs(loc[1].ref_slot));
6060
6061 tcg_out_helper_load_slots(s, nmov, mov, parm);
6062
6063 if (arg_slot_reg_p(loc->arg_slot)) {
6064 tcg_out_addi_ptr(s, tcg_target_call_iarg_regs[loc->arg_slot],
6065 TCG_REG_CALL_STACK,
6066 arg_slot_stk_ofs(loc->ref_slot));
6067 } else {
6068 tcg_debug_assert(parm->ntmp != 0);
6069 tcg_out_addi_ptr(s, parm->tmp[0], TCG_REG_CALL_STACK,
6070 arg_slot_stk_ofs(loc->ref_slot));
6071 tcg_out_st(s, TCG_TYPE_PTR, parm->tmp[0],
6072 TCG_REG_CALL_STACK, arg_slot_stk_ofs(loc->arg_slot));
6073 }
6074 next_arg += 2;
6075 break;
6076
6077 default:
6078 g_assert_not_reached();
6079 }
6080
Richard Hendersonc31e5fa2023-04-28 09:14:17 +01006081 if (TCG_TARGET_REG_BITS == 32 && s->addr_type == TCG_TYPE_I32) {
6082 /* Zero extend the address by loading a zero for the high part. */
Richard Henderson24e46e62023-04-26 22:09:47 +01006083 loc = &info->in[1 + !HOST_BIG_ENDIAN];
6084 tcg_out_helper_load_imm(s, loc->arg_slot, TCG_TYPE_I32, 0, parm);
6085 }
6086
Richard Henderson8429a1c2023-04-09 22:59:09 -07006087 tcg_out_helper_load_common_args(s, ldst, parm, info, next_arg);
6088}
6089
Richard Henderson76cef4b2023-03-08 16:48:02 -08006090int tcg_gen_code(TCGContext *s, TranslationBlock *tb, uint64_t pc_start)
bellardc896fe22008-02-01 10:05:41 +00006091{
Richard Henderson747bd692023-03-31 21:30:31 -07006092 int i, start_words, num_insns;
Richard Henderson15fa08f2017-11-02 15:19:14 +01006093 TCGOp *op;
bellardc896fe22008-02-01 10:05:41 +00006094
Alex Bennéed977e1c2016-03-15 14:30:21 +00006095 if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP)
Richard Hendersonfbf59aa2022-08-15 15:16:06 -05006096 && qemu_log_in_addr_range(pc_start))) {
Richard Hendersonc60f5992022-04-17 11:29:47 -07006097 FILE *logfile = qemu_log_trylock();
Richard Henderson78b54852022-04-17 11:29:49 -07006098 if (logfile) {
6099 fprintf(logfile, "OP:\n");
Richard Hendersonb7a83ff2022-04-17 11:29:51 -07006100 tcg_dump_ops(s, logfile, false);
Richard Henderson78b54852022-04-17 11:29:49 -07006101 fprintf(logfile, "\n");
6102 qemu_log_unlock(logfile);
6103 }
bellardc896fe22008-02-01 10:05:41 +00006104 }
bellardc896fe22008-02-01 10:05:41 +00006105
Richard Hendersonbef16ab2019-02-07 13:26:40 +00006106#ifdef CONFIG_DEBUG_TCG
6107 /* Ensure all labels referenced have been emitted. */
6108 {
6109 TCGLabel *l;
6110 bool error = false;
6111
6112 QSIMPLEQ_FOREACH(l, &s->labels, next) {
Richard Hendersonf85b1fc2023-03-03 13:47:27 -08006113 if (unlikely(!l->present) && !QSIMPLEQ_EMPTY(&l->branches)) {
Richard Hendersonbef16ab2019-02-07 13:26:40 +00006114 qemu_log_mask(CPU_LOG_TB_OP,
6115 "$L%d referenced but not present.\n", l->id);
6116 error = true;
6117 }
6118 }
6119 assert(!error);
6120 }
6121#endif
6122
Richard Henderson04e006a2024-12-07 18:14:45 +00006123 /* Do not reuse any EBB that may be allocated within the TB. */
6124 tcg_temp_ebb_reset_freed(s);
6125
Richard Hendersonc45cb8b2014-09-19 13:49:15 -07006126 tcg_optimize(s);
Kirill Batuzov8f2e8c02011-07-07 16:37:12 +04006127
Richard Hendersonb4fc67c2018-11-26 14:28:28 -08006128 reachable_code_pass(s);
Richard Henderson874b8572023-01-29 11:50:20 -10006129 liveness_pass_0(s);
Richard Hendersonb83eabe2016-11-01 15:56:04 -06006130 liveness_pass_1(s);
Richard Henderson5a184072016-06-23 20:34:33 -07006131
Richard Hendersonb83eabe2016-11-01 15:56:04 -06006132 if (s->nb_indirects > 0) {
Richard Hendersonb83eabe2016-11-01 15:56:04 -06006133 if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP_IND)
Richard Hendersonfbf59aa2022-08-15 15:16:06 -05006134 && qemu_log_in_addr_range(pc_start))) {
Richard Hendersonc60f5992022-04-17 11:29:47 -07006135 FILE *logfile = qemu_log_trylock();
Richard Henderson78b54852022-04-17 11:29:49 -07006136 if (logfile) {
6137 fprintf(logfile, "OP before indirect lowering:\n");
Richard Hendersonb7a83ff2022-04-17 11:29:51 -07006138 tcg_dump_ops(s, logfile, false);
Richard Henderson78b54852022-04-17 11:29:49 -07006139 fprintf(logfile, "\n");
6140 qemu_log_unlock(logfile);
6141 }
Richard Hendersonb83eabe2016-11-01 15:56:04 -06006142 }
Richard Henderson645e3a82023-04-01 16:06:47 -07006143
Richard Hendersonb83eabe2016-11-01 15:56:04 -06006144 /* Replace indirect temps with direct temps. */
6145 if (liveness_pass_2(s)) {
6146 /* If changes were made, re-run liveness. */
6147 liveness_pass_1(s);
Richard Henderson5a184072016-06-23 20:34:33 -07006148 }
6149 }
Aurelien Jarnoc5cc28f2012-09-06 16:47:13 +02006150
Alex Bennéed977e1c2016-03-15 14:30:21 +00006151 if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP_OPT)
Richard Hendersonfbf59aa2022-08-15 15:16:06 -05006152 && qemu_log_in_addr_range(pc_start))) {
Richard Hendersonc60f5992022-04-17 11:29:47 -07006153 FILE *logfile = qemu_log_trylock();
Richard Henderson78b54852022-04-17 11:29:49 -07006154 if (logfile) {
6155 fprintf(logfile, "OP after optimization and liveness analysis:\n");
Richard Hendersonb7a83ff2022-04-17 11:29:51 -07006156 tcg_dump_ops(s, logfile, true);
Richard Henderson78b54852022-04-17 11:29:49 -07006157 fprintf(logfile, "\n");
6158 qemu_log_unlock(logfile);
6159 }
bellardc896fe22008-02-01 10:05:41 +00006160 }
bellardc896fe22008-02-01 10:05:41 +00006161
Richard Henderson35abb002022-11-06 10:55:37 +11006162 /* Initialize goto_tb jump offsets. */
Richard Henderson3a50f422022-11-26 18:20:57 -08006163 tb->jmp_reset_offset[0] = TB_JMP_OFFSET_INVALID;
6164 tb->jmp_reset_offset[1] = TB_JMP_OFFSET_INVALID;
Richard Henderson9da60792022-11-26 18:54:23 -08006165 tb->jmp_insn_offset[0] = TB_JMP_OFFSET_INVALID;
6166 tb->jmp_insn_offset[1] = TB_JMP_OFFSET_INVALID;
Richard Henderson35abb002022-11-06 10:55:37 +11006167
bellardc896fe22008-02-01 10:05:41 +00006168 tcg_reg_alloc_start(s);
6169
Richard Hendersondb0c51a2020-10-28 12:05:44 -07006170 /*
6171 * Reset the buffer pointers when restarting after overflow.
6172 * TODO: Move this into translate-all.c with the rest of the
6173 * buffer management. Having only this done here is confusing.
6174 */
6175 s->code_buf = tcg_splitwx_to_rw(tb->tc.ptr);
6176 s->code_ptr = s->code_buf;
Richard Hendersona7cfd752024-10-16 17:31:05 +00006177 s->data_gen_ptr = NULL;
bellardc896fe22008-02-01 10:05:41 +00006178
Richard Henderson659ef5c2017-07-30 12:30:41 -07006179#ifdef TCG_TARGET_NEED_LDST_LABELS
Laurent Vivier6001f772018-04-30 01:58:40 +02006180 QSIMPLEQ_INIT(&s->ldst_labels);
Richard Henderson659ef5c2017-07-30 12:30:41 -07006181#endif
Richard Henderson57a26942017-07-30 13:13:21 -07006182#ifdef TCG_TARGET_NEED_POOL_LABELS
6183 s->pool_labels = NULL;
6184#endif
Richard Henderson9ecefc82013-10-03 14:51:24 -05006185
Richard Henderson747bd692023-03-31 21:30:31 -07006186 start_words = s->insn_start_words;
6187 s->gen_insn_data =
6188 tcg_malloc(sizeof(uint64_t) * s->gen_tb->icount * start_words);
6189
Richard Henderson9358fbb2023-08-15 16:34:59 +00006190 tcg_out_tb_start(s);
6191
Richard Hendersonfca8a502015-09-01 19:11:45 -07006192 num_insns = -1;
Richard Henderson15fa08f2017-11-02 15:19:14 +01006193 QTAILQ_FOREACH(op, &s->ops, link) {
Richard Hendersonc45cb8b2014-09-19 13:49:15 -07006194 TCGOpcode opc = op->opc;
blueswir1b3db8752008-03-08 13:33:42 +00006195
Richard Hendersonc45cb8b2014-09-19 13:49:15 -07006196 switch (opc) {
bellardc896fe22008-02-01 10:05:41 +00006197 case INDEX_op_mov_i32:
bellardc896fe22008-02-01 10:05:41 +00006198 case INDEX_op_mov_i64:
Richard Hendersond2fd7452017-09-14 13:53:46 -07006199 case INDEX_op_mov_vec:
Richard Hendersondd186292016-12-08 13:42:08 -08006200 tcg_reg_alloc_mov(s, op);
bellardc896fe22008-02-01 10:05:41 +00006201 break;
Richard Hendersonbab16712019-03-18 11:20:27 -07006202 case INDEX_op_dup_vec:
6203 tcg_reg_alloc_dup(s, op);
6204 break;
Richard Henderson765b8422015-08-29 12:37:33 -07006205 case INDEX_op_insn_start:
Richard Hendersonfca8a502015-09-01 19:11:45 -07006206 if (num_insns >= 0) {
Richard Henderson9f754622018-06-14 19:57:03 -10006207 size_t off = tcg_current_code_size(s);
6208 s->gen_insn_end_off[num_insns] = off;
6209 /* Assert that we do not overflow our stored offset. */
6210 assert(s->gen_insn_end_off[num_insns] == off);
Richard Hendersonfca8a502015-09-01 19:11:45 -07006211 }
6212 num_insns++;
Richard Henderson747bd692023-03-31 21:30:31 -07006213 for (i = 0; i < start_words; ++i) {
6214 s->gen_insn_data[num_insns * start_words + i] =
Richard Hendersonc9ad8d22023-03-08 12:24:41 -08006215 tcg_get_insn_start_param(op, i);
Richard Hendersonbad729e2015-09-01 15:51:12 -07006216 }
bellardc896fe22008-02-01 10:05:41 +00006217 break;
bellard5ff9d6a2008-02-04 00:37:54 +00006218 case INDEX_op_discard:
Richard Henderson43439132017-06-19 23:18:10 -07006219 temp_dead(s, arg_temp(op->args[0]));
bellard5ff9d6a2008-02-04 00:37:54 +00006220 break;
bellardc896fe22008-02-01 10:05:41 +00006221 case INDEX_op_set_label:
bellarde8996ee2008-05-23 17:33:39 +00006222 tcg_reg_alloc_bb_end(s, s->reserved_regs);
Richard Henderson92ab8e72020-10-28 18:55:50 -07006223 tcg_out_label(s, arg_label(op->args[0]));
bellardc896fe22008-02-01 10:05:41 +00006224 break;
6225 case INDEX_op_call:
Richard Hendersondd186292016-12-08 13:42:08 -08006226 tcg_reg_alloc_call(s, op);
Richard Hendersonc45cb8b2014-09-19 13:49:15 -07006227 break;
Richard Hendersonb55a8d92022-11-26 12:42:06 -08006228 case INDEX_op_exit_tb:
6229 tcg_out_exit_tb(s, op->args[0]);
6230 break;
Richard Hendersoncf7d6b82022-11-26 17:14:05 -08006231 case INDEX_op_goto_tb:
6232 tcg_out_goto_tb(s, op->args[0]);
6233 break;
Richard Hendersonefe86b22020-03-31 02:33:21 -07006234 case INDEX_op_dup2_vec:
6235 if (tcg_reg_alloc_dup2(s, op)) {
6236 break;
6237 }
6238 /* fall through */
bellardc896fe22008-02-01 10:05:41 +00006239 default:
Richard Henderson25c4d9c2011-08-17 14:11:46 -07006240 /* Sanity check that we've not introduced any unhandled opcodes. */
Richard Hendersonbe0f34b2017-08-17 07:43:20 -07006241 tcg_debug_assert(tcg_op_supported(opc));
bellardc896fe22008-02-01 10:05:41 +00006242 /* Note: in order to speed up the code, it would be much
6243 faster to have specialized register allocator functions for
6244 some common argument patterns */
Richard Hendersondd186292016-12-08 13:42:08 -08006245 tcg_reg_alloc_op(s, op);
bellardc896fe22008-02-01 10:05:41 +00006246 break;
6247 }
Richard Hendersonb125f9d2015-09-22 13:01:15 -07006248 /* Test for (pending) buffer overflow. The assumption is that any
6249 one operation beginning below the high water mark cannot overrun
6250 the buffer completely. Thus we can test for overflow after
6251 generating code without having to check during generation. */
John Clarke644da9b2015-11-19 10:30:50 +01006252 if (unlikely((void *)s->code_ptr > s->code_gen_highwater)) {
Richard Hendersonb125f9d2015-09-22 13:01:15 -07006253 return -1;
6254 }
Richard Henderson6e6c4ef2019-04-15 22:06:39 -10006255 /* Test for TB overflow, as seen by gen_insn_end_off. */
6256 if (unlikely(tcg_current_code_size(s) > UINT16_MAX)) {
6257 return -2;
6258 }
bellardc896fe22008-02-01 10:05:41 +00006259 }
Richard Henderson747bd692023-03-31 21:30:31 -07006260 tcg_debug_assert(num_insns + 1 == s->gen_tb->icount);
Richard Hendersonfca8a502015-09-01 19:11:45 -07006261 s->gen_insn_end_off[num_insns] = tcg_current_code_size(s);
Richard Hendersonc45cb8b2014-09-19 13:49:15 -07006262
Yeongkyoon Leeb76f0d82012-10-31 16:04:25 +09006263 /* Generate TB finalization at the end of block */
Richard Henderson659ef5c2017-07-30 12:30:41 -07006264#ifdef TCG_TARGET_NEED_LDST_LABELS
Richard Hendersonaeee05f2019-04-21 14:51:00 -07006265 i = tcg_out_ldst_finalize(s);
6266 if (i < 0) {
6267 return i;
Richard Henderson23dceda2015-12-02 13:59:59 -08006268 }
Richard Henderson659ef5c2017-07-30 12:30:41 -07006269#endif
Richard Henderson57a26942017-07-30 13:13:21 -07006270#ifdef TCG_TARGET_NEED_POOL_LABELS
Richard Henderson17689872019-04-21 13:51:56 -07006271 i = tcg_out_pool_finalize(s);
6272 if (i < 0) {
6273 return i;
Richard Henderson57a26942017-07-30 13:13:21 -07006274 }
6275#endif
Richard Henderson7ecd02a2019-04-21 13:34:35 -07006276 if (!tcg_resolve_relocs(s)) {
6277 return -2;
6278 }
bellardc896fe22008-02-01 10:05:41 +00006279
Richard Hendersondf5d2b12020-12-12 09:08:02 -06006280#ifndef CONFIG_TCG_INTERPRETER
bellardc896fe22008-02-01 10:05:41 +00006281 /* flush instruction cache */
Richard Hendersondb0c51a2020-10-28 12:05:44 -07006282 flush_idcache_range((uintptr_t)tcg_splitwx_to_rx(s->code_buf),
6283 (uintptr_t)s->code_buf,
Richard Henderson1da8de32020-12-12 10:38:21 -06006284 tcg_ptr_byte_diff(s->code_ptr, s->code_buf));
Richard Hendersondf5d2b12020-12-12 09:08:02 -06006285#endif
Stefan Weil2aeabc02012-03-02 23:30:07 +01006286
Richard Henderson1813e172014-03-28 12:56:22 -07006287 return tcg_current_code_size(s);
bellardc896fe22008-02-01 10:05:41 +00006288}
6289
Richard Henderson813da622012-03-19 12:25:11 -07006290#ifdef ELF_HOST_MACHINE
Richard Henderson5872bbf2012-03-24 10:47:36 -07006291/* In order to use this feature, the backend needs to do three things:
6292
6293 (1) Define ELF_HOST_MACHINE to indicate both what value to
6294 put into the ELF image and to indicate support for the feature.
6295
6296 (2) Define tcg_register_jit. This should create a buffer containing
6297 the contents of a .debug_frame section that describes the post-
6298 prologue unwind info for the tcg machine.
6299
6300 (3) Call tcg_register_jit_int, with the constructed .debug_frame.
6301*/
Richard Henderson813da622012-03-19 12:25:11 -07006302
6303/* Begin GDB interface. THE FOLLOWING MUST MATCH GDB DOCS. */
6304typedef enum {
6305 JIT_NOACTION = 0,
6306 JIT_REGISTER_FN,
6307 JIT_UNREGISTER_FN
6308} jit_actions_t;
6309
6310struct jit_code_entry {
6311 struct jit_code_entry *next_entry;
6312 struct jit_code_entry *prev_entry;
6313 const void *symfile_addr;
6314 uint64_t symfile_size;
6315};
6316
6317struct jit_descriptor {
6318 uint32_t version;
6319 uint32_t action_flag;
6320 struct jit_code_entry *relevant_entry;
6321 struct jit_code_entry *first_entry;
6322};
6323
6324void __jit_debug_register_code(void) __attribute__((noinline));
6325void __jit_debug_register_code(void)
6326{
6327 asm("");
6328}
6329
6330/* Must statically initialize the version, because GDB may check
6331 the version before we can set it. */
6332struct jit_descriptor __jit_debug_descriptor = { 1, 0, 0, 0 };
6333
6334/* End GDB interface. */
6335
6336static int find_string(const char *strtab, const char *str)
6337{
6338 const char *p = strtab + 1;
6339
6340 while (1) {
6341 if (strcmp(p, str) == 0) {
6342 return p - strtab;
6343 }
6344 p += strlen(p) + 1;
6345 }
6346}
6347
Richard Henderson755bf9e2020-10-29 09:17:30 -07006348static void tcg_register_jit_int(const void *buf_ptr, size_t buf_size,
Richard Henderson2c907842014-05-15 12:48:01 -07006349 const void *debug_frame,
6350 size_t debug_frame_size)
Richard Henderson813da622012-03-19 12:25:11 -07006351{
Richard Henderson5872bbf2012-03-24 10:47:36 -07006352 struct __attribute__((packed)) DebugInfo {
6353 uint32_t len;
6354 uint16_t version;
6355 uint32_t abbrev;
6356 uint8_t ptr_size;
6357 uint8_t cu_die;
6358 uint16_t cu_lang;
6359 uintptr_t cu_low_pc;
6360 uintptr_t cu_high_pc;
6361 uint8_t fn_die;
6362 char fn_name[16];
6363 uintptr_t fn_low_pc;
6364 uintptr_t fn_high_pc;
6365 uint8_t cu_eoc;
6366 };
Richard Henderson813da622012-03-19 12:25:11 -07006367
6368 struct ElfImage {
6369 ElfW(Ehdr) ehdr;
6370 ElfW(Phdr) phdr;
Richard Henderson5872bbf2012-03-24 10:47:36 -07006371 ElfW(Shdr) shdr[7];
6372 ElfW(Sym) sym[2];
6373 struct DebugInfo di;
6374 uint8_t da[24];
6375 char str[80];
6376 };
6377
6378 struct ElfImage *img;
6379
6380 static const struct ElfImage img_template = {
6381 .ehdr = {
6382 .e_ident[EI_MAG0] = ELFMAG0,
6383 .e_ident[EI_MAG1] = ELFMAG1,
6384 .e_ident[EI_MAG2] = ELFMAG2,
6385 .e_ident[EI_MAG3] = ELFMAG3,
6386 .e_ident[EI_CLASS] = ELF_CLASS,
6387 .e_ident[EI_DATA] = ELF_DATA,
6388 .e_ident[EI_VERSION] = EV_CURRENT,
6389 .e_type = ET_EXEC,
6390 .e_machine = ELF_HOST_MACHINE,
6391 .e_version = EV_CURRENT,
6392 .e_phoff = offsetof(struct ElfImage, phdr),
6393 .e_shoff = offsetof(struct ElfImage, shdr),
6394 .e_ehsize = sizeof(ElfW(Shdr)),
6395 .e_phentsize = sizeof(ElfW(Phdr)),
6396 .e_phnum = 1,
6397 .e_shentsize = sizeof(ElfW(Shdr)),
6398 .e_shnum = ARRAY_SIZE(img->shdr),
6399 .e_shstrndx = ARRAY_SIZE(img->shdr) - 1,
Richard Hendersonabbb3ea2012-03-24 10:47:37 -07006400#ifdef ELF_HOST_FLAGS
6401 .e_flags = ELF_HOST_FLAGS,
6402#endif
6403#ifdef ELF_OSABI
6404 .e_ident[EI_OSABI] = ELF_OSABI,
6405#endif
Richard Henderson5872bbf2012-03-24 10:47:36 -07006406 },
6407 .phdr = {
6408 .p_type = PT_LOAD,
6409 .p_flags = PF_X,
6410 },
6411 .shdr = {
6412 [0] = { .sh_type = SHT_NULL },
6413 /* Trick: The contents of code_gen_buffer are not present in
6414 this fake ELF file; that got allocated elsewhere. Therefore
6415 we mark .text as SHT_NOBITS (similar to .bss) so that readers
6416 will not look for contents. We can record any address. */
6417 [1] = { /* .text */
6418 .sh_type = SHT_NOBITS,
6419 .sh_flags = SHF_EXECINSTR | SHF_ALLOC,
6420 },
6421 [2] = { /* .debug_info */
6422 .sh_type = SHT_PROGBITS,
6423 .sh_offset = offsetof(struct ElfImage, di),
6424 .sh_size = sizeof(struct DebugInfo),
6425 },
6426 [3] = { /* .debug_abbrev */
6427 .sh_type = SHT_PROGBITS,
6428 .sh_offset = offsetof(struct ElfImage, da),
6429 .sh_size = sizeof(img->da),
6430 },
6431 [4] = { /* .debug_frame */
6432 .sh_type = SHT_PROGBITS,
6433 .sh_offset = sizeof(struct ElfImage),
6434 },
6435 [5] = { /* .symtab */
6436 .sh_type = SHT_SYMTAB,
6437 .sh_offset = offsetof(struct ElfImage, sym),
6438 .sh_size = sizeof(img->sym),
6439 .sh_info = 1,
6440 .sh_link = ARRAY_SIZE(img->shdr) - 1,
6441 .sh_entsize = sizeof(ElfW(Sym)),
6442 },
6443 [6] = { /* .strtab */
6444 .sh_type = SHT_STRTAB,
6445 .sh_offset = offsetof(struct ElfImage, str),
6446 .sh_size = sizeof(img->str),
6447 }
6448 },
6449 .sym = {
6450 [1] = { /* code_gen_buffer */
6451 .st_info = ELF_ST_INFO(STB_GLOBAL, STT_FUNC),
6452 .st_shndx = 1,
6453 }
6454 },
6455 .di = {
6456 .len = sizeof(struct DebugInfo) - 4,
6457 .version = 2,
6458 .ptr_size = sizeof(void *),
6459 .cu_die = 1,
6460 .cu_lang = 0x8001, /* DW_LANG_Mips_Assembler */
6461 .fn_die = 2,
6462 .fn_name = "code_gen_buffer"
6463 },
6464 .da = {
6465 1, /* abbrev number (the cu) */
6466 0x11, 1, /* DW_TAG_compile_unit, has children */
6467 0x13, 0x5, /* DW_AT_language, DW_FORM_data2 */
6468 0x11, 0x1, /* DW_AT_low_pc, DW_FORM_addr */
6469 0x12, 0x1, /* DW_AT_high_pc, DW_FORM_addr */
6470 0, 0, /* end of abbrev */
6471 2, /* abbrev number (the fn) */
6472 0x2e, 0, /* DW_TAG_subprogram, no children */
6473 0x3, 0x8, /* DW_AT_name, DW_FORM_string */
6474 0x11, 0x1, /* DW_AT_low_pc, DW_FORM_addr */
6475 0x12, 0x1, /* DW_AT_high_pc, DW_FORM_addr */
6476 0, 0, /* end of abbrev */
6477 0 /* no more abbrev */
6478 },
6479 .str = "\0" ".text\0" ".debug_info\0" ".debug_abbrev\0"
6480 ".debug_frame\0" ".symtab\0" ".strtab\0" "code_gen_buffer",
Richard Henderson813da622012-03-19 12:25:11 -07006481 };
6482
6483 /* We only need a single jit entry; statically allocate it. */
6484 static struct jit_code_entry one_entry;
6485
Richard Henderson5872bbf2012-03-24 10:47:36 -07006486 uintptr_t buf = (uintptr_t)buf_ptr;
Richard Henderson813da622012-03-19 12:25:11 -07006487 size_t img_size = sizeof(struct ElfImage) + debug_frame_size;
Richard Henderson2c907842014-05-15 12:48:01 -07006488 DebugFrameHeader *dfh;
Richard Henderson813da622012-03-19 12:25:11 -07006489
Richard Henderson5872bbf2012-03-24 10:47:36 -07006490 img = g_malloc(img_size);
6491 *img = img_template;
Richard Henderson813da622012-03-19 12:25:11 -07006492
Richard Henderson5872bbf2012-03-24 10:47:36 -07006493 img->phdr.p_vaddr = buf;
6494 img->phdr.p_paddr = buf;
6495 img->phdr.p_memsz = buf_size;
Richard Henderson813da622012-03-19 12:25:11 -07006496
Richard Henderson5872bbf2012-03-24 10:47:36 -07006497 img->shdr[1].sh_name = find_string(img->str, ".text");
6498 img->shdr[1].sh_addr = buf;
6499 img->shdr[1].sh_size = buf_size;
Richard Henderson813da622012-03-19 12:25:11 -07006500
Richard Henderson5872bbf2012-03-24 10:47:36 -07006501 img->shdr[2].sh_name = find_string(img->str, ".debug_info");
6502 img->shdr[3].sh_name = find_string(img->str, ".debug_abbrev");
6503
6504 img->shdr[4].sh_name = find_string(img->str, ".debug_frame");
6505 img->shdr[4].sh_size = debug_frame_size;
6506
6507 img->shdr[5].sh_name = find_string(img->str, ".symtab");
6508 img->shdr[6].sh_name = find_string(img->str, ".strtab");
6509
6510 img->sym[1].st_name = find_string(img->str, "code_gen_buffer");
6511 img->sym[1].st_value = buf;
6512 img->sym[1].st_size = buf_size;
6513
6514 img->di.cu_low_pc = buf;
Richard Henderson45aba092013-05-24 14:16:14 -07006515 img->di.cu_high_pc = buf + buf_size;
Richard Henderson5872bbf2012-03-24 10:47:36 -07006516 img->di.fn_low_pc = buf;
Richard Henderson45aba092013-05-24 14:16:14 -07006517 img->di.fn_high_pc = buf + buf_size;
Richard Henderson813da622012-03-19 12:25:11 -07006518
Richard Henderson2c907842014-05-15 12:48:01 -07006519 dfh = (DebugFrameHeader *)(img + 1);
6520 memcpy(dfh, debug_frame, debug_frame_size);
6521 dfh->fde.func_start = buf;
6522 dfh->fde.func_len = buf_size;
6523
Richard Henderson813da622012-03-19 12:25:11 -07006524#ifdef DEBUG_JIT
6525 /* Enable this block to be able to debug the ELF image file creation.
6526 One can use readelf, objdump, or other inspection utilities. */
6527 {
Bin Mengeb6b2ed2022-10-27 19:36:17 +01006528 g_autofree char *jit = g_strdup_printf("%s/qemu.jit", g_get_tmp_dir());
6529 FILE *f = fopen(jit, "w+b");
Richard Henderson813da622012-03-19 12:25:11 -07006530 if (f) {
Richard Henderson5872bbf2012-03-24 10:47:36 -07006531 if (fwrite(img, img_size, 1, f) != img_size) {
Richard Henderson813da622012-03-19 12:25:11 -07006532 /* Avoid stupid unused return value warning for fwrite. */
6533 }
6534 fclose(f);
6535 }
6536 }
6537#endif
6538
6539 one_entry.symfile_addr = img;
6540 one_entry.symfile_size = img_size;
6541
6542 __jit_debug_descriptor.action_flag = JIT_REGISTER_FN;
6543 __jit_debug_descriptor.relevant_entry = &one_entry;
6544 __jit_debug_descriptor.first_entry = &one_entry;
6545 __jit_debug_register_code();
6546}
6547#else
Richard Henderson5872bbf2012-03-24 10:47:36 -07006548/* No support for the feature. Provide the entry point expected by exec.c,
6549 and implement the internal function we declared earlier. */
Richard Henderson813da622012-03-19 12:25:11 -07006550
Richard Henderson755bf9e2020-10-29 09:17:30 -07006551static void tcg_register_jit_int(const void *buf, size_t size,
Richard Henderson2c907842014-05-15 12:48:01 -07006552 const void *debug_frame,
6553 size_t debug_frame_size)
Richard Henderson813da622012-03-19 12:25:11 -07006554{
6555}
6556
Richard Henderson755bf9e2020-10-29 09:17:30 -07006557void tcg_register_jit(const void *buf, size_t buf_size)
Richard Henderson813da622012-03-19 12:25:11 -07006558{
6559}
6560#endif /* ELF_HOST_MACHINE */
Richard Hendersondb432672017-09-15 14:11:45 -07006561
6562#if !TCG_TARGET_MAYBE_vec
6563void tcg_expand_vec_op(TCGOpcode o, TCGType t, unsigned e, TCGArg a0, ...)
6564{
6565 g_assert_not_reached();
6566}
6567#endif