blob: 4142d42d7752223828e1e73bd7a62a8713a995e1 [file] [log] [blame]
bellardc896fe22008-02-01 10:05:41 +00001/*
2 * Tiny Code Generator for QEMU
3 *
4 * Copyright (c) 2008 Fabrice Bellard
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a copy
7 * of this software and associated documentation files (the "Software"), to deal
8 * in the Software without restriction, including without limitation the rights
9 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
10 * copies of the Software, and to permit persons to whom the Software is
11 * furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
22 * THE SOFTWARE.
23 */
24
bellardc896fe22008-02-01 10:05:41 +000025/* define it to use liveness analysis (better code) */
Kirill Batuzov8f2e8c02011-07-07 16:37:12 +040026#define USE_TCG_OPTIMIZATIONS
bellardc896fe22008-02-01 10:05:41 +000027
Peter Maydell757e7252016-01-26 18:17:08 +000028#include "qemu/osdep.h"
aurel32cca82982009-04-16 09:58:30 +000029
Richard Henderson813da622012-03-19 12:25:11 -070030/* Define to jump the ELF file used to communicate with GDB. */
31#undef DEBUG_JIT
32
Emilio G. Cota72fd2ef2018-10-10 10:48:53 -040033#include "qemu/error-report.h"
Veronia Bahaaf348b6d2016-03-20 19:16:19 +020034#include "qemu/cutils.h"
Paolo Bonzini1de7afc2012-12-17 18:20:00 +010035#include "qemu/host-utils.h"
Markus Armbrusterd4c51a02019-04-17 21:17:51 +020036#include "qemu/qemu-print.h"
Paolo Bonzini1de7afc2012-12-17 18:20:00 +010037#include "qemu/timer.h"
Richard Henderson084cfca2020-12-14 08:02:33 -060038#include "qemu/cacheflush.h"
bellardc896fe22008-02-01 10:05:41 +000039
Stefan Weilc5d3c492014-02-21 20:52:39 +010040/* Note: the long term plan is to reduce the dependencies on the QEMU
bellardc896fe22008-02-01 10:05:41 +000041 CPU definitions. Currently they are used for qemu_ld/st
42 instructions */
43#define NO_CPU_IO_DEFS
bellardc896fe22008-02-01 10:05:41 +000044
Paolo Bonzini63c91552016-03-15 13:18:37 +010045#include "exec/exec-all.h"
Philippe Mathieu-Daudédcb32f12020-01-01 12:23:00 +010046#include "tcg/tcg-op.h"
Richard Henderson813da622012-03-19 12:25:11 -070047
Richard Hendersonedee2572013-08-20 17:20:30 -070048#if UINTPTR_MAX == UINT32_MAX
Richard Henderson813da622012-03-19 12:25:11 -070049# define ELF_CLASS ELFCLASS32
Richard Hendersonedee2572013-08-20 17:20:30 -070050#else
51# define ELF_CLASS ELFCLASS64
Richard Henderson813da622012-03-19 12:25:11 -070052#endif
53#ifdef HOST_WORDS_BIGENDIAN
54# define ELF_DATA ELFDATA2MSB
55#else
56# define ELF_DATA ELFDATA2LSB
57#endif
58
bellardc896fe22008-02-01 10:05:41 +000059#include "elf.h"
Paolo Bonzini508127e2016-01-07 16:55:28 +030060#include "exec/log.h"
Richard Henderson5ff72582021-03-09 16:24:14 -060061#include "tcg-internal.h"
bellardc896fe22008-02-01 10:05:41 +000062
Richard Henderson22f15572021-03-18 12:46:44 -060063#ifdef CONFIG_TCG_INTERPRETER
64#include <ffi.h>
65#endif
66
Paolo Bonzini139c1832020-02-04 12:41:01 +010067/* Forward declarations for functions declared in tcg-target.c.inc and
Peter Maydellce151102016-02-23 14:49:41 +000068 used here. */
Richard Hendersone4d58b42010-06-02 17:26:56 -070069static void tcg_target_init(TCGContext *s);
70static void tcg_target_qemu_prologue(TCGContext *s);
Richard Henderson6ac17782018-11-30 11:52:48 -080071static bool patch_reloc(tcg_insn_unit *code_ptr, int type,
Richard Henderson2ba7fae22013-08-20 15:30:10 -070072 intptr_t value, intptr_t addend);
bellardc896fe22008-02-01 10:05:41 +000073
Richard Henderson497a22e2013-06-05 07:39:57 -070074/* The CIE and FDE header definitions will be common to all hosts. */
75typedef struct {
76 uint32_t len __attribute__((aligned((sizeof(void *)))));
77 uint32_t id;
78 uint8_t version;
79 char augmentation[1];
80 uint8_t code_align;
81 uint8_t data_align;
82 uint8_t return_column;
83} DebugFrameCIE;
84
85typedef struct QEMU_PACKED {
86 uint32_t len __attribute__((aligned((sizeof(void *)))));
87 uint32_t cie_offset;
Richard Hendersonedee2572013-08-20 17:20:30 -070088 uintptr_t func_start;
89 uintptr_t func_len;
Richard Henderson497a22e2013-06-05 07:39:57 -070090} DebugFrameFDEHeader;
91
Richard Henderson2c907842014-05-15 12:48:01 -070092typedef struct QEMU_PACKED {
93 DebugFrameCIE cie;
94 DebugFrameFDEHeader fde;
95} DebugFrameHeader;
96
Richard Henderson755bf9e2020-10-29 09:17:30 -070097static void tcg_register_jit_int(const void *buf, size_t size,
Richard Henderson2c907842014-05-15 12:48:01 -070098 const void *debug_frame,
99 size_t debug_frame_size)
Richard Henderson813da622012-03-19 12:25:11 -0700100 __attribute__((unused));
101
Paolo Bonzini139c1832020-02-04 12:41:01 +0100102/* Forward declarations for functions declared and used in tcg-target.c.inc. */
Richard Henderson2a534af2011-11-09 08:03:34 +0000103static void tcg_out_ld(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg1,
Richard Hendersona05b5b92013-08-20 17:07:26 -0700104 intptr_t arg2);
Richard Henderson78113e82019-03-16 17:48:18 +0000105static bool tcg_out_mov(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg);
Stefan Weilc0ad3002011-09-17 22:00:29 +0200106static void tcg_out_movi(TCGContext *s, TCGType type,
Richard Henderson2a534af2011-11-09 08:03:34 +0000107 TCGReg ret, tcg_target_long arg);
Miroslav Rezanina5e8892d2021-03-12 13:14:18 +0100108static void tcg_out_op(TCGContext *s, TCGOpcode opc,
109 const TCGArg args[TCG_MAX_OP_ARGS],
110 const int const_args[TCG_MAX_OP_ARGS]);
Richard Hendersond2fd7452017-09-14 13:53:46 -0700111#if TCG_TARGET_MAYBE_vec
Richard Hendersone7632cf2019-03-18 15:32:44 +0000112static bool tcg_out_dup_vec(TCGContext *s, TCGType type, unsigned vece,
113 TCGReg dst, TCGReg src);
Richard Hendersond6ecb4a2019-03-18 12:00:39 -0700114static bool tcg_out_dupm_vec(TCGContext *s, TCGType type, unsigned vece,
115 TCGReg dst, TCGReg base, intptr_t offset);
Richard Henderson4e186172020-03-31 01:02:08 -0700116static void tcg_out_dupi_vec(TCGContext *s, TCGType type, unsigned vece,
117 TCGReg dst, int64_t arg);
Miroslav Rezanina5e8892d2021-03-12 13:14:18 +0100118static void tcg_out_vec_op(TCGContext *s, TCGOpcode opc,
119 unsigned vecl, unsigned vece,
120 const TCGArg args[TCG_MAX_OP_ARGS],
121 const int const_args[TCG_MAX_OP_ARGS]);
Richard Hendersond2fd7452017-09-14 13:53:46 -0700122#else
Richard Hendersone7632cf2019-03-18 15:32:44 +0000123static inline bool tcg_out_dup_vec(TCGContext *s, TCGType type, unsigned vece,
124 TCGReg dst, TCGReg src)
125{
126 g_assert_not_reached();
127}
Richard Hendersond6ecb4a2019-03-18 12:00:39 -0700128static inline bool tcg_out_dupm_vec(TCGContext *s, TCGType type, unsigned vece,
129 TCGReg dst, TCGReg base, intptr_t offset)
130{
131 g_assert_not_reached();
132}
Richard Henderson4e186172020-03-31 01:02:08 -0700133static inline void tcg_out_dupi_vec(TCGContext *s, TCGType type, unsigned vece,
134 TCGReg dst, int64_t arg)
Richard Hendersone7632cf2019-03-18 15:32:44 +0000135{
136 g_assert_not_reached();
137}
Miroslav Rezanina5e8892d2021-03-12 13:14:18 +0100138static inline void tcg_out_vec_op(TCGContext *s, TCGOpcode opc,
139 unsigned vecl, unsigned vece,
140 const TCGArg args[TCG_MAX_OP_ARGS],
141 const int const_args[TCG_MAX_OP_ARGS])
Richard Hendersond2fd7452017-09-14 13:53:46 -0700142{
143 g_assert_not_reached();
144}
145#endif
Richard Henderson2a534af2011-11-09 08:03:34 +0000146static void tcg_out_st(TCGContext *s, TCGType type, TCGReg arg, TCGReg arg1,
Richard Hendersona05b5b92013-08-20 17:07:26 -0700147 intptr_t arg2);
Richard Henderson59d7c142016-06-19 22:59:13 -0700148static bool tcg_out_sti(TCGContext *s, TCGType type, TCGArg val,
149 TCGReg base, intptr_t ofs);
Richard Henderson7b7d8b22021-01-30 14:24:25 -0800150#ifdef CONFIG_TCG_INTERPRETER
151static void tcg_out_call(TCGContext *s, const tcg_insn_unit *target,
152 ffi_cif *cif);
153#else
Richard Henderson2be7d762020-10-28 15:29:04 -0700154static void tcg_out_call(TCGContext *s, const tcg_insn_unit *target);
Richard Henderson7b7d8b22021-01-30 14:24:25 -0800155#endif
Richard Hendersona4fbbd72021-05-03 16:47:37 -0700156static bool tcg_target_const_match(int64_t val, TCGType type, int ct);
Richard Henderson659ef5c2017-07-30 12:30:41 -0700157#ifdef TCG_TARGET_NEED_LDST_LABELS
Richard Hendersonaeee05f2019-04-21 14:51:00 -0700158static int tcg_out_ldst_finalize(TCGContext *s);
Richard Henderson659ef5c2017-07-30 12:30:41 -0700159#endif
bellardc896fe22008-02-01 10:05:41 +0000160
Richard Henderson42eb6df2021-03-13 13:36:51 -0600161TCGContext tcg_init_ctx;
162__thread TCGContext *tcg_ctx;
163
Richard Henderson5ff72582021-03-09 16:24:14 -0600164TCGContext **tcg_ctxs;
Richard Henderson0e2d61c2021-03-09 23:06:32 -0600165unsigned int tcg_cur_ctxs;
166unsigned int tcg_max_ctxs;
Richard Henderson1c2adb92017-10-10 14:34:37 -0700167TCGv_env cpu_env = 0;
Richard Hendersonc8bc1162020-11-05 15:41:38 -0800168const void *tcg_code_gen_epilogue;
Richard Hendersondb0c51a2020-10-28 12:05:44 -0700169uintptr_t tcg_splitwx_diff;
Emilio G. Cotadf2cce22017-07-12 18:26:40 -0400170
Richard Hendersonb91ccb32020-10-28 14:11:54 -0700171#ifndef CONFIG_TCG_INTERPRETER
172tcg_prologue_fn *tcg_qemu_tb_exec;
173#endif
174
Richard Hendersond2fd7452017-09-14 13:53:46 -0700175static TCGRegSet tcg_target_available_regs[TCG_TYPE_COUNT];
blueswir1b1d8e522008-10-26 13:43:07 +0000176static TCGRegSet tcg_target_call_clobber_regs;
bellardc896fe22008-02-01 10:05:41 +0000177
Richard Henderson1813e172014-03-28 12:56:22 -0700178#if TCG_TARGET_INSN_UNIT_SIZE == 1
Peter Maydell4196dca2014-06-07 18:08:44 +0100179static __attribute__((unused)) inline void tcg_out8(TCGContext *s, uint8_t v)
bellardc896fe22008-02-01 10:05:41 +0000180{
181 *s->code_ptr++ = v;
182}
183
Peter Maydell4196dca2014-06-07 18:08:44 +0100184static __attribute__((unused)) inline void tcg_patch8(tcg_insn_unit *p,
185 uint8_t v)
Peter Maydell5c53bb82014-03-28 15:29:48 +0000186{
Richard Henderson1813e172014-03-28 12:56:22 -0700187 *p = v;
Peter Maydell5c53bb82014-03-28 15:29:48 +0000188}
Richard Henderson1813e172014-03-28 12:56:22 -0700189#endif
Peter Maydell5c53bb82014-03-28 15:29:48 +0000190
Richard Henderson1813e172014-03-28 12:56:22 -0700191#if TCG_TARGET_INSN_UNIT_SIZE <= 2
Peter Maydell4196dca2014-06-07 18:08:44 +0100192static __attribute__((unused)) inline void tcg_out16(TCGContext *s, uint16_t v)
bellardc896fe22008-02-01 10:05:41 +0000193{
Richard Henderson1813e172014-03-28 12:56:22 -0700194 if (TCG_TARGET_INSN_UNIT_SIZE == 2) {
195 *s->code_ptr++ = v;
196 } else {
197 tcg_insn_unit *p = s->code_ptr;
198 memcpy(p, &v, sizeof(v));
199 s->code_ptr = p + (2 / TCG_TARGET_INSN_UNIT_SIZE);
200 }
bellardc896fe22008-02-01 10:05:41 +0000201}
202
Peter Maydell4196dca2014-06-07 18:08:44 +0100203static __attribute__((unused)) inline void tcg_patch16(tcg_insn_unit *p,
204 uint16_t v)
Peter Maydell5c53bb82014-03-28 15:29:48 +0000205{
Richard Henderson1813e172014-03-28 12:56:22 -0700206 if (TCG_TARGET_INSN_UNIT_SIZE == 2) {
207 *p = v;
208 } else {
209 memcpy(p, &v, sizeof(v));
210 }
Peter Maydell5c53bb82014-03-28 15:29:48 +0000211}
Richard Henderson1813e172014-03-28 12:56:22 -0700212#endif
Peter Maydell5c53bb82014-03-28 15:29:48 +0000213
Richard Henderson1813e172014-03-28 12:56:22 -0700214#if TCG_TARGET_INSN_UNIT_SIZE <= 4
Peter Maydell4196dca2014-06-07 18:08:44 +0100215static __attribute__((unused)) inline void tcg_out32(TCGContext *s, uint32_t v)
bellardc896fe22008-02-01 10:05:41 +0000216{
Richard Henderson1813e172014-03-28 12:56:22 -0700217 if (TCG_TARGET_INSN_UNIT_SIZE == 4) {
218 *s->code_ptr++ = v;
219 } else {
220 tcg_insn_unit *p = s->code_ptr;
221 memcpy(p, &v, sizeof(v));
222 s->code_ptr = p + (4 / TCG_TARGET_INSN_UNIT_SIZE);
223 }
bellardc896fe22008-02-01 10:05:41 +0000224}
225
Peter Maydell4196dca2014-06-07 18:08:44 +0100226static __attribute__((unused)) inline void tcg_patch32(tcg_insn_unit *p,
227 uint32_t v)
Peter Maydell5c53bb82014-03-28 15:29:48 +0000228{
Richard Henderson1813e172014-03-28 12:56:22 -0700229 if (TCG_TARGET_INSN_UNIT_SIZE == 4) {
230 *p = v;
231 } else {
232 memcpy(p, &v, sizeof(v));
233 }
Peter Maydell5c53bb82014-03-28 15:29:48 +0000234}
Richard Henderson1813e172014-03-28 12:56:22 -0700235#endif
Peter Maydell5c53bb82014-03-28 15:29:48 +0000236
Richard Henderson1813e172014-03-28 12:56:22 -0700237#if TCG_TARGET_INSN_UNIT_SIZE <= 8
Peter Maydell4196dca2014-06-07 18:08:44 +0100238static __attribute__((unused)) inline void tcg_out64(TCGContext *s, uint64_t v)
Richard Hendersonac26eb62013-07-25 09:42:17 -1000239{
Richard Henderson1813e172014-03-28 12:56:22 -0700240 if (TCG_TARGET_INSN_UNIT_SIZE == 8) {
241 *s->code_ptr++ = v;
242 } else {
243 tcg_insn_unit *p = s->code_ptr;
244 memcpy(p, &v, sizeof(v));
245 s->code_ptr = p + (8 / TCG_TARGET_INSN_UNIT_SIZE);
246 }
Richard Hendersonac26eb62013-07-25 09:42:17 -1000247}
248
Peter Maydell4196dca2014-06-07 18:08:44 +0100249static __attribute__((unused)) inline void tcg_patch64(tcg_insn_unit *p,
250 uint64_t v)
Peter Maydell5c53bb82014-03-28 15:29:48 +0000251{
Richard Henderson1813e172014-03-28 12:56:22 -0700252 if (TCG_TARGET_INSN_UNIT_SIZE == 8) {
253 *p = v;
254 } else {
255 memcpy(p, &v, sizeof(v));
256 }
Peter Maydell5c53bb82014-03-28 15:29:48 +0000257}
Richard Henderson1813e172014-03-28 12:56:22 -0700258#endif
Peter Maydell5c53bb82014-03-28 15:29:48 +0000259
bellardc896fe22008-02-01 10:05:41 +0000260/* label relocation processing */
261
Richard Henderson1813e172014-03-28 12:56:22 -0700262static void tcg_out_reloc(TCGContext *s, tcg_insn_unit *code_ptr, int type,
Richard Hendersonbec16312015-02-13 13:39:54 -0800263 TCGLabel *l, intptr_t addend)
bellardc896fe22008-02-01 10:05:41 +0000264{
Richard Henderson7ecd02a2019-04-21 13:34:35 -0700265 TCGRelocation *r = tcg_malloc(sizeof(TCGRelocation));
bellardc896fe22008-02-01 10:05:41 +0000266
Richard Henderson7ecd02a2019-04-21 13:34:35 -0700267 r->type = type;
268 r->ptr = code_ptr;
269 r->addend = addend;
270 QSIMPLEQ_INSERT_TAIL(&l->relocs, r, next);
bellardc896fe22008-02-01 10:05:41 +0000271}
272
Richard Henderson92ab8e72020-10-28 18:55:50 -0700273static void tcg_out_label(TCGContext *s, TCGLabel *l)
bellardc896fe22008-02-01 10:05:41 +0000274{
Aurelien Jarnoeabb7b92016-04-21 10:48:49 +0200275 tcg_debug_assert(!l->has_value);
bellardc896fe22008-02-01 10:05:41 +0000276 l->has_value = 1;
Richard Henderson92ab8e72020-10-28 18:55:50 -0700277 l->u.value_ptr = tcg_splitwx_to_rx(s->code_ptr);
bellardc896fe22008-02-01 10:05:41 +0000278}
279
Richard Henderson42a268c2015-02-13 12:51:55 -0800280TCGLabel *gen_new_label(void)
bellardc896fe22008-02-01 10:05:41 +0000281{
Emilio G. Cotab1311c42017-07-12 17:15:52 -0400282 TCGContext *s = tcg_ctx;
Richard Henderson51e39722015-02-13 18:51:05 -0800283 TCGLabel *l = tcg_malloc(sizeof(TCGLabel));
bellardc896fe22008-02-01 10:05:41 +0000284
Richard Henderson7ecd02a2019-04-21 13:34:35 -0700285 memset(l, 0, sizeof(TCGLabel));
286 l->id = s->nb_labels++;
287 QSIMPLEQ_INIT(&l->relocs);
288
Richard Hendersonbef16ab2019-02-07 13:26:40 +0000289 QSIMPLEQ_INSERT_TAIL(&s->labels, l, next);
Richard Henderson42a268c2015-02-13 12:51:55 -0800290
291 return l;
bellardc896fe22008-02-01 10:05:41 +0000292}
293
Richard Henderson7ecd02a2019-04-21 13:34:35 -0700294static bool tcg_resolve_relocs(TCGContext *s)
295{
296 TCGLabel *l;
297
298 QSIMPLEQ_FOREACH(l, &s->labels, next) {
299 TCGRelocation *r;
300 uintptr_t value = l->u.value;
301
302 QSIMPLEQ_FOREACH(r, &l->relocs, next) {
303 if (!patch_reloc(r->ptr, r->type, value, r->addend)) {
304 return false;
305 }
306 }
307 }
308 return true;
309}
310
Richard Henderson9f754622018-06-14 19:57:03 -1000311static void set_jmp_reset_offset(TCGContext *s, int which)
312{
Richard Hendersonf14bed32020-11-02 19:36:20 -0800313 /*
314 * We will check for overflow at the end of the opcode loop in
315 * tcg_gen_code, where we bound tcg_current_code_size to UINT16_MAX.
316 */
317 s->tb_jmp_reset_offset[which] = tcg_current_code_size(s);
Richard Henderson9f754622018-06-14 19:57:03 -1000318}
319
Richard Hendersondb6b7d02021-01-31 23:29:26 -1000320/* Signal overflow, starting over with fewer guest insns. */
321static void QEMU_NORETURN tcg_raise_tb_overflow(TCGContext *s)
322{
323 siglongjmp(s->jmp_trans, -2);
324}
325
Richard Henderson4c22e842020-10-16 22:20:55 -0700326#define C_PFX1(P, A) P##A
327#define C_PFX2(P, A, B) P##A##_##B
328#define C_PFX3(P, A, B, C) P##A##_##B##_##C
329#define C_PFX4(P, A, B, C, D) P##A##_##B##_##C##_##D
330#define C_PFX5(P, A, B, C, D, E) P##A##_##B##_##C##_##D##_##E
331#define C_PFX6(P, A, B, C, D, E, F) P##A##_##B##_##C##_##D##_##E##_##F
332
333/* Define an enumeration for the various combinations. */
334
335#define C_O0_I1(I1) C_PFX1(c_o0_i1_, I1),
336#define C_O0_I2(I1, I2) C_PFX2(c_o0_i2_, I1, I2),
337#define C_O0_I3(I1, I2, I3) C_PFX3(c_o0_i3_, I1, I2, I3),
338#define C_O0_I4(I1, I2, I3, I4) C_PFX4(c_o0_i4_, I1, I2, I3, I4),
339
340#define C_O1_I1(O1, I1) C_PFX2(c_o1_i1_, O1, I1),
341#define C_O1_I2(O1, I1, I2) C_PFX3(c_o1_i2_, O1, I1, I2),
342#define C_O1_I3(O1, I1, I2, I3) C_PFX4(c_o1_i3_, O1, I1, I2, I3),
343#define C_O1_I4(O1, I1, I2, I3, I4) C_PFX5(c_o1_i4_, O1, I1, I2, I3, I4),
344
345#define C_N1_I2(O1, I1, I2) C_PFX3(c_n1_i2_, O1, I1, I2),
346
347#define C_O2_I1(O1, O2, I1) C_PFX3(c_o2_i1_, O1, O2, I1),
348#define C_O2_I2(O1, O2, I1, I2) C_PFX4(c_o2_i2_, O1, O2, I1, I2),
349#define C_O2_I3(O1, O2, I1, I2, I3) C_PFX5(c_o2_i3_, O1, O2, I1, I2, I3),
350#define C_O2_I4(O1, O2, I1, I2, I3, I4) C_PFX6(c_o2_i4_, O1, O2, I1, I2, I3, I4),
351
352typedef enum {
353#include "tcg-target-con-set.h"
354} TCGConstraintSetIndex;
355
356static TCGConstraintSetIndex tcg_target_op_def(TCGOpcode);
357
358#undef C_O0_I1
359#undef C_O0_I2
360#undef C_O0_I3
361#undef C_O0_I4
362#undef C_O1_I1
363#undef C_O1_I2
364#undef C_O1_I3
365#undef C_O1_I4
366#undef C_N1_I2
367#undef C_O2_I1
368#undef C_O2_I2
369#undef C_O2_I3
370#undef C_O2_I4
371
372/* Put all of the constraint sets into an array, indexed by the enum. */
373
374#define C_O0_I1(I1) { .args_ct_str = { #I1 } },
375#define C_O0_I2(I1, I2) { .args_ct_str = { #I1, #I2 } },
376#define C_O0_I3(I1, I2, I3) { .args_ct_str = { #I1, #I2, #I3 } },
377#define C_O0_I4(I1, I2, I3, I4) { .args_ct_str = { #I1, #I2, #I3, #I4 } },
378
379#define C_O1_I1(O1, I1) { .args_ct_str = { #O1, #I1 } },
380#define C_O1_I2(O1, I1, I2) { .args_ct_str = { #O1, #I1, #I2 } },
381#define C_O1_I3(O1, I1, I2, I3) { .args_ct_str = { #O1, #I1, #I2, #I3 } },
382#define C_O1_I4(O1, I1, I2, I3, I4) { .args_ct_str = { #O1, #I1, #I2, #I3, #I4 } },
383
384#define C_N1_I2(O1, I1, I2) { .args_ct_str = { "&" #O1, #I1, #I2 } },
385
386#define C_O2_I1(O1, O2, I1) { .args_ct_str = { #O1, #O2, #I1 } },
387#define C_O2_I2(O1, O2, I1, I2) { .args_ct_str = { #O1, #O2, #I1, #I2 } },
388#define C_O2_I3(O1, O2, I1, I2, I3) { .args_ct_str = { #O1, #O2, #I1, #I2, #I3 } },
389#define C_O2_I4(O1, O2, I1, I2, I3, I4) { .args_ct_str = { #O1, #O2, #I1, #I2, #I3, #I4 } },
390
391static const TCGTargetOpDef constraint_sets[] = {
392#include "tcg-target-con-set.h"
393};
394
395
396#undef C_O0_I1
397#undef C_O0_I2
398#undef C_O0_I3
399#undef C_O0_I4
400#undef C_O1_I1
401#undef C_O1_I2
402#undef C_O1_I3
403#undef C_O1_I4
404#undef C_N1_I2
405#undef C_O2_I1
406#undef C_O2_I2
407#undef C_O2_I3
408#undef C_O2_I4
409
410/* Expand the enumerator to be returned from tcg_target_op_def(). */
411
412#define C_O0_I1(I1) C_PFX1(c_o0_i1_, I1)
413#define C_O0_I2(I1, I2) C_PFX2(c_o0_i2_, I1, I2)
414#define C_O0_I3(I1, I2, I3) C_PFX3(c_o0_i3_, I1, I2, I3)
415#define C_O0_I4(I1, I2, I3, I4) C_PFX4(c_o0_i4_, I1, I2, I3, I4)
416
417#define C_O1_I1(O1, I1) C_PFX2(c_o1_i1_, O1, I1)
418#define C_O1_I2(O1, I1, I2) C_PFX3(c_o1_i2_, O1, I1, I2)
419#define C_O1_I3(O1, I1, I2, I3) C_PFX4(c_o1_i3_, O1, I1, I2, I3)
420#define C_O1_I4(O1, I1, I2, I3, I4) C_PFX5(c_o1_i4_, O1, I1, I2, I3, I4)
421
422#define C_N1_I2(O1, I1, I2) C_PFX3(c_n1_i2_, O1, I1, I2)
423
424#define C_O2_I1(O1, O2, I1) C_PFX3(c_o2_i1_, O1, O2, I1)
425#define C_O2_I2(O1, O2, I1, I2) C_PFX4(c_o2_i2_, O1, O2, I1, I2)
426#define C_O2_I3(O1, O2, I1, I2, I3) C_PFX5(c_o2_i3_, O1, O2, I1, I2, I3)
427#define C_O2_I4(O1, O2, I1, I2, I3, I4) C_PFX6(c_o2_i4_, O1, O2, I1, I2, I3, I4)
428
Paolo Bonzini139c1832020-02-04 12:41:01 +0100429#include "tcg-target.c.inc"
bellardc896fe22008-02-01 10:05:41 +0000430
Emilio G. Cota38b47b12018-12-07 15:33:56 -0500431static void alloc_tcg_plugin_context(TCGContext *s)
432{
433#ifdef CONFIG_PLUGIN
434 s->plugin_tb = g_new0(struct qemu_plugin_tb, 1);
435 s->plugin_tb->insns =
436 g_ptr_array_new_with_free_func(qemu_plugin_insn_cleanup_fn);
437#endif
438}
439
Emilio G. Cotae8feb962017-07-07 19:24:20 -0400440/*
Emilio G. Cota3468b592017-07-19 18:57:58 -0400441 * All TCG threads except the parent (i.e. the one that called tcg_context_init
442 * and registered the target's TCG globals) must register with this function
443 * before initiating translation.
444 *
445 * In user-mode we just point tcg_ctx to tcg_init_ctx. See the documentation
446 * of tcg_region_init() for the reasoning behind this.
447 *
448 * In softmmu each caller registers its context in tcg_ctxs[]. Note that in
449 * softmmu tcg_ctxs[] does not track tcg_ctx_init, since the initial context
450 * is not used anymore for translation once this function is called.
451 *
452 * Not tracking tcg_init_ctx in tcg_ctxs[] in softmmu keeps code that iterates
453 * over the array (e.g. tcg_code_size() the same for both softmmu and user-mode.
454 */
455#ifdef CONFIG_USER_ONLY
456void tcg_register_thread(void)
457{
458 tcg_ctx = &tcg_init_ctx;
459}
460#else
461void tcg_register_thread(void)
462{
463 TCGContext *s = g_malloc(sizeof(*s));
464 unsigned int i, n;
Emilio G. Cota3468b592017-07-19 18:57:58 -0400465
466 *s = tcg_init_ctx;
467
468 /* Relink mem_base. */
469 for (i = 0, n = tcg_init_ctx.nb_globals; i < n; ++i) {
470 if (tcg_init_ctx.temps[i].mem_base) {
471 ptrdiff_t b = tcg_init_ctx.temps[i].mem_base - tcg_init_ctx.temps;
472 tcg_debug_assert(b >= 0 && b < n);
473 s->temps[i].mem_base = &s->temps[b];
474 }
475 }
476
477 /* Claim an entry in tcg_ctxs */
Richard Henderson0e2d61c2021-03-09 23:06:32 -0600478 n = qatomic_fetch_inc(&tcg_cur_ctxs);
479 g_assert(n < tcg_max_ctxs);
Stefan Hajnoczid73415a2020-09-23 11:56:46 +0100480 qatomic_set(&tcg_ctxs[n], s);
Emilio G. Cota3468b592017-07-19 18:57:58 -0400481
Emilio G. Cota38b47b12018-12-07 15:33:56 -0500482 if (n > 0) {
483 alloc_tcg_plugin_context(s);
Richard Hendersonbf042e82021-03-09 16:33:15 -0600484 tcg_region_initial_alloc(s);
Emilio G. Cota38b47b12018-12-07 15:33:56 -0500485 }
486
Emilio G. Cota3468b592017-07-19 18:57:58 -0400487 tcg_ctx = s;
Emilio G. Cota3468b592017-07-19 18:57:58 -0400488}
489#endif /* !CONFIG_USER_ONLY */
490
bellardc896fe22008-02-01 10:05:41 +0000491/* pool based memory allocation */
492void *tcg_malloc_internal(TCGContext *s, int size)
493{
494 TCGPool *p;
495 int pool_size;
496
497 if (size > TCG_POOL_CHUNK_SIZE) {
498 /* big malloc: insert a new pool (XXX: could optimize) */
Anthony Liguori7267c092011-08-20 22:09:37 -0500499 p = g_malloc(sizeof(TCGPool) + size);
bellardc896fe22008-02-01 10:05:41 +0000500 p->size = size;
Kirill Batuzov40552992012-03-02 13:22:17 +0400501 p->next = s->pool_first_large;
502 s->pool_first_large = p;
503 return p->data;
bellardc896fe22008-02-01 10:05:41 +0000504 } else {
505 p = s->pool_current;
506 if (!p) {
507 p = s->pool_first;
508 if (!p)
509 goto new_pool;
510 } else {
511 if (!p->next) {
512 new_pool:
513 pool_size = TCG_POOL_CHUNK_SIZE;
Anthony Liguori7267c092011-08-20 22:09:37 -0500514 p = g_malloc(sizeof(TCGPool) + pool_size);
bellardc896fe22008-02-01 10:05:41 +0000515 p->size = pool_size;
516 p->next = NULL;
517 if (s->pool_current)
518 s->pool_current->next = p;
519 else
520 s->pool_first = p;
521 } else {
522 p = p->next;
523 }
524 }
525 }
526 s->pool_current = p;
527 s->pool_cur = p->data + size;
528 s->pool_end = p->data + p->size;
529 return p->data;
530}
531
532void tcg_pool_reset(TCGContext *s)
533{
Kirill Batuzov40552992012-03-02 13:22:17 +0400534 TCGPool *p, *t;
535 for (p = s->pool_first_large; p; p = t) {
536 t = p->next;
537 g_free(p);
538 }
539 s->pool_first_large = NULL;
bellardc896fe22008-02-01 10:05:41 +0000540 s->pool_cur = s->pool_end = NULL;
541 s->pool_current = NULL;
542}
543
Richard Henderson2ef61752014-04-07 22:31:41 -0700544#include "exec/helper-proto.h"
545
Richard Henderson100b5e02013-09-14 15:57:22 -0700546static const TCGHelperInfo all_helpers[] = {
Richard Henderson2ef61752014-04-07 22:31:41 -0700547#include "exec/helper-tcg.h"
Richard Henderson100b5e02013-09-14 15:57:22 -0700548};
Emilio G. Cota619205f2017-07-05 18:41:23 -0400549static GHashTable *helper_table;
Richard Henderson100b5e02013-09-14 15:57:22 -0700550
Richard Henderson22f15572021-03-18 12:46:44 -0600551#ifdef CONFIG_TCG_INTERPRETER
552static GHashTable *ffi_table;
553
554static ffi_type * const typecode_to_ffi[8] = {
555 [dh_typecode_void] = &ffi_type_void,
556 [dh_typecode_i32] = &ffi_type_uint32,
557 [dh_typecode_s32] = &ffi_type_sint32,
558 [dh_typecode_i64] = &ffi_type_uint64,
559 [dh_typecode_s64] = &ffi_type_sint64,
560 [dh_typecode_ptr] = &ffi_type_pointer,
561};
562#endif
563
Richard Henderson91478ce2015-08-18 23:23:08 -0700564static int indirect_reg_alloc_order[ARRAY_SIZE(tcg_target_reg_alloc_order)];
Richard Hendersonf69d2772016-11-18 09:31:40 +0100565static void process_op_defs(TCGContext *s);
Richard Henderson1c2adb92017-10-10 14:34:37 -0700566static TCGTemp *tcg_global_reg_new_internal(TCGContext *s, TCGType type,
567 TCGReg reg, const char *name);
Richard Henderson91478ce2015-08-18 23:23:08 -0700568
Richard Henderson43b972b2021-03-09 22:52:45 -0600569static void tcg_context_init(unsigned max_cpus)
bellardc896fe22008-02-01 10:05:41 +0000570{
Richard Hendersona76aabd2021-03-09 17:24:33 -0600571 TCGContext *s = &tcg_init_ctx;
Richard Henderson100b5e02013-09-14 15:57:22 -0700572 int op, total_args, n, i;
bellardc896fe22008-02-01 10:05:41 +0000573 TCGOpDef *def;
574 TCGArgConstraint *args_ct;
Richard Henderson1c2adb92017-10-10 14:34:37 -0700575 TCGTemp *ts;
bellardc896fe22008-02-01 10:05:41 +0000576
577 memset(s, 0, sizeof(*s));
bellardc896fe22008-02-01 10:05:41 +0000578 s->nb_globals = 0;
Richard Hendersonc70fbf02016-06-23 20:34:22 -0700579
bellardc896fe22008-02-01 10:05:41 +0000580 /* Count total number of arguments and allocate the corresponding
581 space */
582 total_args = 0;
583 for(op = 0; op < NB_OPS; op++) {
584 def = &tcg_op_defs[op];
585 n = def->nb_iargs + def->nb_oargs;
586 total_args += n;
587 }
588
Richard Hendersonbc2b17e2019-04-04 19:34:19 -0700589 args_ct = g_new0(TCGArgConstraint, total_args);
bellardc896fe22008-02-01 10:05:41 +0000590
591 for(op = 0; op < NB_OPS; op++) {
592 def = &tcg_op_defs[op];
593 def->args_ct = args_ct;
bellardc896fe22008-02-01 10:05:41 +0000594 n = def->nb_iargs + def->nb_oargs;
bellardc896fe22008-02-01 10:05:41 +0000595 args_ct += n;
596 }
Richard Henderson5cd8f622013-09-14 15:09:39 -0700597
598 /* Register helpers. */
Richard Henderson84fd9dd2013-09-14 16:44:31 -0700599 /* Use g_direct_hash/equal for direct pointer comparisons on func. */
Emilio G. Cota619205f2017-07-05 18:41:23 -0400600 helper_table = g_hash_table_new(NULL, NULL);
Richard Henderson84fd9dd2013-09-14 16:44:31 -0700601
Richard Henderson100b5e02013-09-14 15:57:22 -0700602 for (i = 0; i < ARRAY_SIZE(all_helpers); ++i) {
Richard Henderson84fd9dd2013-09-14 16:44:31 -0700603 g_hash_table_insert(helper_table, (gpointer)all_helpers[i].func,
Richard Henderson72866e82014-04-08 00:17:53 -0700604 (gpointer)&all_helpers[i]);
Richard Henderson100b5e02013-09-14 15:57:22 -0700605 }
Richard Henderson5cd8f622013-09-14 15:09:39 -0700606
Richard Henderson22f15572021-03-18 12:46:44 -0600607#ifdef CONFIG_TCG_INTERPRETER
608 /* g_direct_hash/equal for direct comparisons on uint32_t. */
609 ffi_table = g_hash_table_new(NULL, NULL);
610 for (i = 0; i < ARRAY_SIZE(all_helpers); ++i) {
611 struct {
612 ffi_cif cif;
613 ffi_type *args[];
614 } *ca;
615 uint32_t typemask = all_helpers[i].typemask;
616 gpointer hash = (gpointer)(uintptr_t)typemask;
617 ffi_status status;
618 int nargs;
619
620 if (g_hash_table_lookup(ffi_table, hash)) {
621 continue;
622 }
623
624 /* Ignoring the return type, find the last non-zero field. */
625 nargs = 32 - clz32(typemask >> 3);
626 nargs = DIV_ROUND_UP(nargs, 3);
627
628 ca = g_malloc0(sizeof(*ca) + nargs * sizeof(ffi_type *));
629 ca->cif.rtype = typecode_to_ffi[typemask & 7];
630 ca->cif.nargs = nargs;
631
632 if (nargs != 0) {
633 ca->cif.arg_types = ca->args;
634 for (i = 0; i < nargs; ++i) {
635 int typecode = extract32(typemask, (i + 1) * 3, 3);
636 ca->args[i] = typecode_to_ffi[typecode];
637 }
638 }
639
640 status = ffi_prep_cif(&ca->cif, FFI_DEFAULT_ABI, nargs,
641 ca->cif.rtype, ca->cif.arg_types);
642 assert(status == FFI_OK);
643
644 g_hash_table_insert(ffi_table, hash, (gpointer)&ca->cif);
645 }
646#endif
647
bellardc896fe22008-02-01 10:05:41 +0000648 tcg_target_init(s);
Richard Hendersonf69d2772016-11-18 09:31:40 +0100649 process_op_defs(s);
Richard Henderson91478ce2015-08-18 23:23:08 -0700650
651 /* Reverse the order of the saved registers, assuming they're all at
652 the start of tcg_target_reg_alloc_order. */
653 for (n = 0; n < ARRAY_SIZE(tcg_target_reg_alloc_order); ++n) {
654 int r = tcg_target_reg_alloc_order[n];
655 if (tcg_regset_test_reg(tcg_target_call_clobber_regs, r)) {
656 break;
657 }
658 }
659 for (i = 0; i < n; ++i) {
660 indirect_reg_alloc_order[i] = tcg_target_reg_alloc_order[n - 1 - i];
661 }
662 for (; i < ARRAY_SIZE(tcg_target_reg_alloc_order); ++i) {
663 indirect_reg_alloc_order[i] = tcg_target_reg_alloc_order[i];
664 }
Emilio G. Cotab1311c42017-07-12 17:15:52 -0400665
Emilio G. Cota38b47b12018-12-07 15:33:56 -0500666 alloc_tcg_plugin_context(s);
667
Emilio G. Cotab1311c42017-07-12 17:15:52 -0400668 tcg_ctx = s;
Emilio G. Cota3468b592017-07-19 18:57:58 -0400669 /*
670 * In user-mode we simply share the init context among threads, since we
671 * use a single region. See the documentation tcg_region_init() for the
672 * reasoning behind this.
673 * In softmmu we will have at most max_cpus TCG threads.
674 */
675#ifdef CONFIG_USER_ONLY
Emilio G. Cotadf2cce22017-07-12 18:26:40 -0400676 tcg_ctxs = &tcg_ctx;
Richard Henderson0e2d61c2021-03-09 23:06:32 -0600677 tcg_cur_ctxs = 1;
678 tcg_max_ctxs = 1;
Emilio G. Cota3468b592017-07-19 18:57:58 -0400679#else
Richard Henderson0e2d61c2021-03-09 23:06:32 -0600680 tcg_max_ctxs = max_cpus;
681 tcg_ctxs = g_new0(TCGContext *, max_cpus);
Emilio G. Cota3468b592017-07-19 18:57:58 -0400682#endif
Richard Henderson1c2adb92017-10-10 14:34:37 -0700683
684 tcg_debug_assert(!tcg_regset_test_reg(s->reserved_regs, TCG_AREG0));
685 ts = tcg_global_reg_new_internal(s, TCG_TYPE_PTR, TCG_AREG0, "env");
686 cpu_env = temp_tcgv_ptr(ts);
Richard Henderson9002ec72010-05-06 08:50:41 -0700687}
bellardb03cce82008-05-10 10:52:05 +0000688
Richard Henderson43b972b2021-03-09 22:52:45 -0600689void tcg_init(size_t tb_size, int splitwx, unsigned max_cpus)
Richard Hendersona76aabd2021-03-09 17:24:33 -0600690{
Richard Henderson43b972b2021-03-09 22:52:45 -0600691 tcg_context_init(max_cpus);
692 tcg_region_init(tb_size, splitwx, max_cpus);
Richard Hendersona76aabd2021-03-09 17:24:33 -0600693}
694
Emilio G. Cota6e3b2bf2017-06-06 19:12:25 -0400695/*
696 * Allocate TBs right before their corresponding translated code, making
697 * sure that TBs and code are on different cache lines.
698 */
699TranslationBlock *tcg_tb_alloc(TCGContext *s)
700{
701 uintptr_t align = qemu_icache_linesize;
702 TranslationBlock *tb;
703 void *next;
704
Emilio G. Cotae8feb962017-07-07 19:24:20 -0400705 retry:
Emilio G. Cota6e3b2bf2017-06-06 19:12:25 -0400706 tb = (void *)ROUND_UP((uintptr_t)s->code_gen_ptr, align);
707 next = (void *)ROUND_UP((uintptr_t)(tb + 1), align);
708
709 if (unlikely(next > s->code_gen_highwater)) {
Emilio G. Cotae8feb962017-07-07 19:24:20 -0400710 if (tcg_region_alloc(s)) {
711 return NULL;
712 }
713 goto retry;
Emilio G. Cota6e3b2bf2017-06-06 19:12:25 -0400714 }
Stefan Hajnoczid73415a2020-09-23 11:56:46 +0100715 qatomic_set(&s->code_gen_ptr, next);
Richard Henderson57a26942017-07-30 13:13:21 -0700716 s->data_gen_ptr = NULL;
Emilio G. Cota6e3b2bf2017-06-06 19:12:25 -0400717 return tb;
718}
719
Richard Henderson9002ec72010-05-06 08:50:41 -0700720void tcg_prologue_init(TCGContext *s)
721{
Richard Hendersonb0a07942021-03-09 08:45:58 -0800722 size_t prologue_size;
Richard Henderson8163b742015-09-18 23:43:05 -0700723
Richard Hendersonb0a07942021-03-09 08:45:58 -0800724 s->code_ptr = s->code_gen_ptr;
725 s->code_buf = s->code_gen_ptr;
Richard Henderson5b38ee32017-10-25 07:14:20 -0700726 s->data_gen_ptr = NULL;
Richard Hendersonb91ccb32020-10-28 14:11:54 -0700727
728#ifndef CONFIG_TCG_INTERPRETER
Richard Hendersonb0a07942021-03-09 08:45:58 -0800729 tcg_qemu_tb_exec = (tcg_prologue_fn *)tcg_splitwx_to_rx(s->code_ptr);
Richard Hendersonb91ccb32020-10-28 14:11:54 -0700730#endif
Richard Henderson8163b742015-09-18 23:43:05 -0700731
Richard Henderson5b38ee32017-10-25 07:14:20 -0700732#ifdef TCG_TARGET_NEED_POOL_LABELS
733 s->pool_labels = NULL;
734#endif
735
Roman Bolshakov653b87e2021-01-13 06:28:07 +0300736 qemu_thread_jit_write();
Richard Henderson8163b742015-09-18 23:43:05 -0700737 /* Generate the prologue. */
bellardb03cce82008-05-10 10:52:05 +0000738 tcg_target_qemu_prologue(s);
Richard Henderson5b38ee32017-10-25 07:14:20 -0700739
740#ifdef TCG_TARGET_NEED_POOL_LABELS
741 /* Allow the prologue to put e.g. guest_base into a pool entry. */
742 {
Richard Henderson17689872019-04-21 13:51:56 -0700743 int result = tcg_out_pool_finalize(s);
744 tcg_debug_assert(result == 0);
Richard Henderson5b38ee32017-10-25 07:14:20 -0700745 }
746#endif
747
Richard Hendersonb0a07942021-03-09 08:45:58 -0800748 prologue_size = tcg_current_code_size(s);
749
Richard Hendersondf5d2b12020-12-12 09:08:02 -0600750#ifndef CONFIG_TCG_INTERPRETER
Richard Hendersonb0a07942021-03-09 08:45:58 -0800751 flush_idcache_range((uintptr_t)tcg_splitwx_to_rx(s->code_buf),
752 (uintptr_t)s->code_buf, prologue_size);
Richard Hendersondf5d2b12020-12-12 09:08:02 -0600753#endif
Richard Henderson8163b742015-09-18 23:43:05 -0700754
Richard Hendersond6b64b22013-03-31 13:15:19 -0700755#ifdef DEBUG_DISAS
756 if (qemu_loglevel_mask(CPU_LOG_TB_OUT_ASM)) {
Robert Foleyfc59d2d2019-11-18 16:15:26 -0500757 FILE *logfile = qemu_log_lock();
Richard Henderson8163b742015-09-18 23:43:05 -0700758 qemu_log("PROLOGUE: [size=%zu]\n", prologue_size);
Richard Henderson5b38ee32017-10-25 07:14:20 -0700759 if (s->data_gen_ptr) {
Richard Hendersonb0a07942021-03-09 08:45:58 -0800760 size_t code_size = s->data_gen_ptr - s->code_gen_ptr;
Richard Henderson5b38ee32017-10-25 07:14:20 -0700761 size_t data_size = prologue_size - code_size;
762 size_t i;
763
Richard Hendersonb0a07942021-03-09 08:45:58 -0800764 log_disas(s->code_gen_ptr, code_size);
Richard Henderson5b38ee32017-10-25 07:14:20 -0700765
766 for (i = 0; i < data_size; i += sizeof(tcg_target_ulong)) {
767 if (sizeof(tcg_target_ulong) == 8) {
768 qemu_log("0x%08" PRIxPTR ": .quad 0x%016" PRIx64 "\n",
769 (uintptr_t)s->data_gen_ptr + i,
770 *(uint64_t *)(s->data_gen_ptr + i));
771 } else {
772 qemu_log("0x%08" PRIxPTR ": .long 0x%08x\n",
773 (uintptr_t)s->data_gen_ptr + i,
774 *(uint32_t *)(s->data_gen_ptr + i));
775 }
776 }
777 } else {
Richard Hendersonb0a07942021-03-09 08:45:58 -0800778 log_disas(s->code_gen_ptr, prologue_size);
Richard Henderson5b38ee32017-10-25 07:14:20 -0700779 }
Richard Hendersond6b64b22013-03-31 13:15:19 -0700780 qemu_log("\n");
781 qemu_log_flush();
Robert Foleyfc59d2d2019-11-18 16:15:26 -0500782 qemu_log_unlock(logfile);
Richard Hendersond6b64b22013-03-31 13:15:19 -0700783 }
784#endif
Emilio G. Cotacedbcb02017-04-26 23:29:14 -0400785
Richard Henderson6eea0432021-02-02 09:40:22 -1000786#ifndef CONFIG_TCG_INTERPRETER
787 /*
788 * Assert that goto_ptr is implemented completely, setting an epilogue.
789 * For tci, we use NULL as the signal to return from the interpreter,
790 * so skip this check.
791 */
Richard Hendersonf4e01e32021-06-29 14:47:39 -0700792 tcg_debug_assert(tcg_code_gen_epilogue != NULL);
Richard Henderson6eea0432021-02-02 09:40:22 -1000793#endif
Richard Hendersond1c74ab2021-07-09 19:45:42 -0700794
795 tcg_region_prologue_set(s);
bellardc896fe22008-02-01 10:05:41 +0000796}
797
bellardc896fe22008-02-01 10:05:41 +0000798void tcg_func_start(TCGContext *s)
799{
800 tcg_pool_reset(s);
801 s->nb_temps = s->nb_globals;
Richard Henderson0ec9eab2013-09-19 12:16:45 -0700802
803 /* No temps have been previously allocated for size or locality. */
804 memset(s->free_temps, 0, sizeof(s->free_temps));
805
Richard Hendersonc0522132020-03-29 18:55:52 -0700806 /* No constant temps have been previously allocated. */
807 for (int i = 0; i < TCG_TYPE_COUNT; ++i) {
808 if (s->const_table[i]) {
809 g_hash_table_remove_all(s->const_table[i]);
810 }
811 }
812
Richard Hendersonabebf922018-05-08 19:18:59 +0000813 s->nb_ops = 0;
bellardc896fe22008-02-01 10:05:41 +0000814 s->nb_labels = 0;
815 s->current_frame_offset = s->frame_start;
816
Richard Henderson0a209d42012-09-21 17:18:16 -0700817#ifdef CONFIG_DEBUG_TCG
818 s->goto_tb_issue_mask = 0;
819#endif
820
Richard Henderson15fa08f2017-11-02 15:19:14 +0100821 QTAILQ_INIT(&s->ops);
822 QTAILQ_INIT(&s->free_ops);
Richard Hendersonbef16ab2019-02-07 13:26:40 +0000823 QSIMPLEQ_INIT(&s->labels);
bellardc896fe22008-02-01 10:05:41 +0000824}
825
Richard Hendersonae30e862021-01-23 12:11:17 -1000826static TCGTemp *tcg_temp_alloc(TCGContext *s)
Richard Henderson7ca4b752013-09-19 08:46:21 -0700827{
828 int n = s->nb_temps++;
Richard Hendersonae30e862021-01-23 12:11:17 -1000829
830 if (n >= TCG_MAX_TEMPS) {
Richard Hendersondb6b7d02021-01-31 23:29:26 -1000831 tcg_raise_tb_overflow(s);
Richard Hendersonae30e862021-01-23 12:11:17 -1000832 }
Richard Henderson7ca4b752013-09-19 08:46:21 -0700833 return memset(&s->temps[n], 0, sizeof(TCGTemp));
834}
835
Richard Hendersonae30e862021-01-23 12:11:17 -1000836static TCGTemp *tcg_global_alloc(TCGContext *s)
Richard Henderson7ca4b752013-09-19 08:46:21 -0700837{
Richard Hendersonfa477d22016-11-02 11:20:15 -0600838 TCGTemp *ts;
839
Richard Henderson7ca4b752013-09-19 08:46:21 -0700840 tcg_debug_assert(s->nb_globals == s->nb_temps);
Richard Hendersonae30e862021-01-23 12:11:17 -1000841 tcg_debug_assert(s->nb_globals < TCG_MAX_TEMPS);
Richard Henderson7ca4b752013-09-19 08:46:21 -0700842 s->nb_globals++;
Richard Hendersonfa477d22016-11-02 11:20:15 -0600843 ts = tcg_temp_alloc(s);
Richard Hendersonee17db82020-03-29 10:11:56 -0700844 ts->kind = TEMP_GLOBAL;
Richard Hendersonfa477d22016-11-02 11:20:15 -0600845
846 return ts;
bellardc896fe22008-02-01 10:05:41 +0000847}
848
Richard Henderson085272b2017-10-20 00:05:45 -0700849static TCGTemp *tcg_global_reg_new_internal(TCGContext *s, TCGType type,
850 TCGReg reg, const char *name)
bellardc896fe22008-02-01 10:05:41 +0000851{
bellardc896fe22008-02-01 10:05:41 +0000852 TCGTemp *ts;
bellardc896fe22008-02-01 10:05:41 +0000853
Richard Hendersonb3a62932013-09-18 14:12:53 -0700854 if (TCG_TARGET_REG_BITS == 32 && type != TCG_TYPE_I32) {
bellardc896fe22008-02-01 10:05:41 +0000855 tcg_abort();
Richard Hendersonb3a62932013-09-18 14:12:53 -0700856 }
Richard Henderson7ca4b752013-09-19 08:46:21 -0700857
858 ts = tcg_global_alloc(s);
bellardc896fe22008-02-01 10:05:41 +0000859 ts->base_type = type;
860 ts->type = type;
Richard Hendersonee17db82020-03-29 10:11:56 -0700861 ts->kind = TEMP_FIXED;
bellardc896fe22008-02-01 10:05:41 +0000862 ts->reg = reg;
bellardc896fe22008-02-01 10:05:41 +0000863 ts->name = name;
bellardc896fe22008-02-01 10:05:41 +0000864 tcg_regset_set_reg(s->reserved_regs, reg);
Richard Henderson7ca4b752013-09-19 08:46:21 -0700865
Richard Henderson085272b2017-10-20 00:05:45 -0700866 return ts;
pbrooka7812ae2008-11-17 14:43:54 +0000867}
868
Richard Hendersonb6638662013-09-18 14:54:45 -0700869void tcg_set_frame(TCGContext *s, TCGReg reg, intptr_t start, intptr_t size)
pbrooka7812ae2008-11-17 14:43:54 +0000870{
Richard Hendersonb3a62932013-09-18 14:12:53 -0700871 s->frame_start = start;
872 s->frame_end = start + size;
Richard Henderson085272b2017-10-20 00:05:45 -0700873 s->frame_temp
874 = tcg_global_reg_new_internal(s, TCG_TYPE_PTR, reg, "_frame");
Richard Hendersonb3a62932013-09-18 14:12:53 -0700875}
pbrooka7812ae2008-11-17 14:43:54 +0000876
Richard Henderson085272b2017-10-20 00:05:45 -0700877TCGTemp *tcg_global_mem_new_internal(TCGType type, TCGv_ptr base,
878 intptr_t offset, const char *name)
bellardc896fe22008-02-01 10:05:41 +0000879{
Emilio G. Cotab1311c42017-07-12 17:15:52 -0400880 TCGContext *s = tcg_ctx;
Richard Hendersondc41aa72017-10-20 00:30:24 -0700881 TCGTemp *base_ts = tcgv_ptr_temp(base);
Richard Henderson7ca4b752013-09-19 08:46:21 -0700882 TCGTemp *ts = tcg_global_alloc(s);
Richard Hendersonb3915db2013-09-19 10:36:18 -0700883 int indirect_reg = 0, bigendian = 0;
Richard Henderson7ca4b752013-09-19 08:46:21 -0700884#ifdef HOST_WORDS_BIGENDIAN
885 bigendian = 1;
886#endif
bellardc896fe22008-02-01 10:05:41 +0000887
Richard Hendersonc0522132020-03-29 18:55:52 -0700888 switch (base_ts->kind) {
889 case TEMP_FIXED:
890 break;
891 case TEMP_GLOBAL:
Richard Henderson5a184072016-06-23 20:34:33 -0700892 /* We do not support double-indirect registers. */
893 tcg_debug_assert(!base_ts->indirect_reg);
Richard Hendersonb3915db2013-09-19 10:36:18 -0700894 base_ts->indirect_base = 1;
Richard Henderson5a184072016-06-23 20:34:33 -0700895 s->nb_indirects += (TCG_TARGET_REG_BITS == 32 && type == TCG_TYPE_I64
896 ? 2 : 1);
897 indirect_reg = 1;
Richard Hendersonc0522132020-03-29 18:55:52 -0700898 break;
899 default:
900 g_assert_not_reached();
Richard Hendersonb3915db2013-09-19 10:36:18 -0700901 }
902
Richard Henderson7ca4b752013-09-19 08:46:21 -0700903 if (TCG_TARGET_REG_BITS == 32 && type == TCG_TYPE_I64) {
904 TCGTemp *ts2 = tcg_global_alloc(s);
bellardc896fe22008-02-01 10:05:41 +0000905 char buf[64];
Richard Henderson7ca4b752013-09-19 08:46:21 -0700906
907 ts->base_type = TCG_TYPE_I64;
bellardc896fe22008-02-01 10:05:41 +0000908 ts->type = TCG_TYPE_I32;
Richard Hendersonb3915db2013-09-19 10:36:18 -0700909 ts->indirect_reg = indirect_reg;
bellardc896fe22008-02-01 10:05:41 +0000910 ts->mem_allocated = 1;
Richard Hendersonb3a62932013-09-18 14:12:53 -0700911 ts->mem_base = base_ts;
Richard Henderson7ca4b752013-09-19 08:46:21 -0700912 ts->mem_offset = offset + bigendian * 4;
bellardc896fe22008-02-01 10:05:41 +0000913 pstrcpy(buf, sizeof(buf), name);
914 pstrcat(buf, sizeof(buf), "_0");
915 ts->name = strdup(buf);
bellardc896fe22008-02-01 10:05:41 +0000916
Richard Henderson7ca4b752013-09-19 08:46:21 -0700917 tcg_debug_assert(ts2 == ts + 1);
918 ts2->base_type = TCG_TYPE_I64;
919 ts2->type = TCG_TYPE_I32;
Richard Hendersonb3915db2013-09-19 10:36:18 -0700920 ts2->indirect_reg = indirect_reg;
Richard Henderson7ca4b752013-09-19 08:46:21 -0700921 ts2->mem_allocated = 1;
922 ts2->mem_base = base_ts;
923 ts2->mem_offset = offset + (1 - bigendian) * 4;
bellardc896fe22008-02-01 10:05:41 +0000924 pstrcpy(buf, sizeof(buf), name);
925 pstrcat(buf, sizeof(buf), "_1");
Richard Henderson120c1082016-06-17 17:02:20 -0700926 ts2->name = strdup(buf);
Richard Henderson7ca4b752013-09-19 08:46:21 -0700927 } else {
bellardc896fe22008-02-01 10:05:41 +0000928 ts->base_type = type;
929 ts->type = type;
Richard Hendersonb3915db2013-09-19 10:36:18 -0700930 ts->indirect_reg = indirect_reg;
bellardc896fe22008-02-01 10:05:41 +0000931 ts->mem_allocated = 1;
Richard Hendersonb3a62932013-09-18 14:12:53 -0700932 ts->mem_base = base_ts;
bellardc896fe22008-02-01 10:05:41 +0000933 ts->mem_offset = offset;
bellardc896fe22008-02-01 10:05:41 +0000934 ts->name = name;
bellardc896fe22008-02-01 10:05:41 +0000935 }
Richard Henderson085272b2017-10-20 00:05:45 -0700936 return ts;
bellardc896fe22008-02-01 10:05:41 +0000937}
938
Richard Henderson5bfa8032018-02-22 18:17:57 -0800939TCGTemp *tcg_temp_new_internal(TCGType type, bool temp_local)
bellardc896fe22008-02-01 10:05:41 +0000940{
Emilio G. Cotab1311c42017-07-12 17:15:52 -0400941 TCGContext *s = tcg_ctx;
Richard Hendersonee17db82020-03-29 10:11:56 -0700942 TCGTempKind kind = temp_local ? TEMP_LOCAL : TEMP_NORMAL;
bellardc896fe22008-02-01 10:05:41 +0000943 TCGTemp *ts;
bellard641d5fb2008-05-25 17:24:00 +0000944 int idx, k;
bellardc896fe22008-02-01 10:05:41 +0000945
Richard Henderson0ec9eab2013-09-19 12:16:45 -0700946 k = type + (temp_local ? TCG_TYPE_COUNT : 0);
947 idx = find_first_bit(s->free_temps[k].l, TCG_MAX_TEMPS);
948 if (idx < TCG_MAX_TEMPS) {
949 /* There is already an available temp with the right type. */
950 clear_bit(idx, s->free_temps[k].l);
951
bellarde8996ee2008-05-23 17:33:39 +0000952 ts = &s->temps[idx];
bellarde8996ee2008-05-23 17:33:39 +0000953 ts->temp_allocated = 1;
Richard Henderson7ca4b752013-09-19 08:46:21 -0700954 tcg_debug_assert(ts->base_type == type);
Richard Hendersonee17db82020-03-29 10:11:56 -0700955 tcg_debug_assert(ts->kind == kind);
bellarde8996ee2008-05-23 17:33:39 +0000956 } else {
Richard Henderson7ca4b752013-09-19 08:46:21 -0700957 ts = tcg_temp_alloc(s);
958 if (TCG_TARGET_REG_BITS == 32 && type == TCG_TYPE_I64) {
959 TCGTemp *ts2 = tcg_temp_alloc(s);
960
bellarde8996ee2008-05-23 17:33:39 +0000961 ts->base_type = type;
962 ts->type = TCG_TYPE_I32;
963 ts->temp_allocated = 1;
Richard Hendersonee17db82020-03-29 10:11:56 -0700964 ts->kind = kind;
Richard Henderson7ca4b752013-09-19 08:46:21 -0700965
966 tcg_debug_assert(ts2 == ts + 1);
967 ts2->base_type = TCG_TYPE_I64;
968 ts2->type = TCG_TYPE_I32;
969 ts2->temp_allocated = 1;
Richard Hendersonee17db82020-03-29 10:11:56 -0700970 ts2->kind = kind;
Richard Henderson7ca4b752013-09-19 08:46:21 -0700971 } else {
bellarde8996ee2008-05-23 17:33:39 +0000972 ts->base_type = type;
973 ts->type = type;
974 ts->temp_allocated = 1;
Richard Hendersonee17db82020-03-29 10:11:56 -0700975 ts->kind = kind;
bellarde8996ee2008-05-23 17:33:39 +0000976 }
bellardc896fe22008-02-01 10:05:41 +0000977 }
Peter Maydell27bfd832011-03-06 21:39:53 +0000978
979#if defined(CONFIG_DEBUG_TCG)
980 s->temps_in_use++;
981#endif
Richard Henderson085272b2017-10-20 00:05:45 -0700982 return ts;
bellardc896fe22008-02-01 10:05:41 +0000983}
984
Richard Hendersond2fd7452017-09-14 13:53:46 -0700985TCGv_vec tcg_temp_new_vec(TCGType type)
986{
987 TCGTemp *t;
988
989#ifdef CONFIG_DEBUG_TCG
990 switch (type) {
991 case TCG_TYPE_V64:
992 assert(TCG_TARGET_HAS_v64);
993 break;
994 case TCG_TYPE_V128:
995 assert(TCG_TARGET_HAS_v128);
996 break;
997 case TCG_TYPE_V256:
998 assert(TCG_TARGET_HAS_v256);
999 break;
1000 default:
1001 g_assert_not_reached();
1002 }
1003#endif
1004
1005 t = tcg_temp_new_internal(type, 0);
1006 return temp_tcgv_vec(t);
1007}
1008
1009/* Create a new temp of the same type as an existing temp. */
1010TCGv_vec tcg_temp_new_vec_matching(TCGv_vec match)
1011{
1012 TCGTemp *t = tcgv_vec_temp(match);
1013
1014 tcg_debug_assert(t->temp_allocated != 0);
1015
1016 t = tcg_temp_new_internal(t->base_type, 0);
1017 return temp_tcgv_vec(t);
1018}
1019
Richard Henderson5bfa8032018-02-22 18:17:57 -08001020void tcg_temp_free_internal(TCGTemp *ts)
bellardc896fe22008-02-01 10:05:41 +00001021{
Emilio G. Cotab1311c42017-07-12 17:15:52 -04001022 TCGContext *s = tcg_ctx;
Richard Henderson085272b2017-10-20 00:05:45 -07001023 int k, idx;
bellardc896fe22008-02-01 10:05:41 +00001024
Richard Hendersonc0522132020-03-29 18:55:52 -07001025 /* In order to simplify users of tcg_constant_*, silently ignore free. */
1026 if (ts->kind == TEMP_CONST) {
1027 return;
1028 }
1029
Peter Maydell27bfd832011-03-06 21:39:53 +00001030#if defined(CONFIG_DEBUG_TCG)
1031 s->temps_in_use--;
1032 if (s->temps_in_use < 0) {
1033 fprintf(stderr, "More temporaries freed than allocated!\n");
1034 }
1035#endif
1036
Richard Hendersonee17db82020-03-29 10:11:56 -07001037 tcg_debug_assert(ts->kind < TEMP_GLOBAL);
Aurelien Jarnoeabb7b92016-04-21 10:48:49 +02001038 tcg_debug_assert(ts->temp_allocated != 0);
bellarde8996ee2008-05-23 17:33:39 +00001039 ts->temp_allocated = 0;
Richard Henderson0ec9eab2013-09-19 12:16:45 -07001040
Richard Henderson085272b2017-10-20 00:05:45 -07001041 idx = temp_idx(ts);
Richard Hendersonee17db82020-03-29 10:11:56 -07001042 k = ts->base_type + (ts->kind == TEMP_NORMAL ? 0 : TCG_TYPE_COUNT);
Richard Henderson0ec9eab2013-09-19 12:16:45 -07001043 set_bit(idx, s->free_temps[k].l);
bellarde8996ee2008-05-23 17:33:39 +00001044}
1045
Richard Hendersonc0522132020-03-29 18:55:52 -07001046TCGTemp *tcg_constant_internal(TCGType type, int64_t val)
1047{
1048 TCGContext *s = tcg_ctx;
1049 GHashTable *h = s->const_table[type];
1050 TCGTemp *ts;
1051
1052 if (h == NULL) {
1053 h = g_hash_table_new(g_int64_hash, g_int64_equal);
1054 s->const_table[type] = h;
1055 }
1056
1057 ts = g_hash_table_lookup(h, &val);
1058 if (ts == NULL) {
1059 ts = tcg_temp_alloc(s);
1060
1061 if (TCG_TARGET_REG_BITS == 32 && type == TCG_TYPE_I64) {
1062 TCGTemp *ts2 = tcg_temp_alloc(s);
1063
1064 ts->base_type = TCG_TYPE_I64;
1065 ts->type = TCG_TYPE_I32;
1066 ts->kind = TEMP_CONST;
1067 ts->temp_allocated = 1;
1068 /*
1069 * Retain the full value of the 64-bit constant in the low
1070 * part, so that the hash table works. Actual uses will
1071 * truncate the value to the low part.
1072 */
1073 ts->val = val;
1074
1075 tcg_debug_assert(ts2 == ts + 1);
1076 ts2->base_type = TCG_TYPE_I64;
1077 ts2->type = TCG_TYPE_I32;
1078 ts2->kind = TEMP_CONST;
1079 ts2->temp_allocated = 1;
1080 ts2->val = val >> 32;
1081 } else {
1082 ts->base_type = type;
1083 ts->type = type;
1084 ts->kind = TEMP_CONST;
1085 ts->temp_allocated = 1;
1086 ts->val = val;
1087 }
1088 g_hash_table_insert(h, &ts->val, ts);
1089 }
1090
1091 return ts;
1092}
1093
1094TCGv_vec tcg_constant_vec(TCGType type, unsigned vece, int64_t val)
1095{
1096 val = dup_const(vece, val);
1097 return temp_tcgv_vec(tcg_constant_internal(type, val));
1098}
1099
Richard Henderson88d40052020-09-03 18:18:08 -07001100TCGv_vec tcg_constant_vec_matching(TCGv_vec match, unsigned vece, int64_t val)
1101{
1102 TCGTemp *t = tcgv_vec_temp(match);
1103
1104 tcg_debug_assert(t->temp_allocated != 0);
1105 return tcg_constant_vec(t->base_type, vece, val);
1106}
1107
pbrooka7812ae2008-11-17 14:43:54 +00001108TCGv_i32 tcg_const_i32(int32_t val)
1109{
1110 TCGv_i32 t0;
1111 t0 = tcg_temp_new_i32();
bellarde8996ee2008-05-23 17:33:39 +00001112 tcg_gen_movi_i32(t0, val);
1113 return t0;
bellardc896fe22008-02-01 10:05:41 +00001114}
1115
pbrooka7812ae2008-11-17 14:43:54 +00001116TCGv_i64 tcg_const_i64(int64_t val)
bellardc896fe22008-02-01 10:05:41 +00001117{
pbrooka7812ae2008-11-17 14:43:54 +00001118 TCGv_i64 t0;
1119 t0 = tcg_temp_new_i64();
bellarde8996ee2008-05-23 17:33:39 +00001120 tcg_gen_movi_i64(t0, val);
1121 return t0;
bellardc896fe22008-02-01 10:05:41 +00001122}
1123
pbrooka7812ae2008-11-17 14:43:54 +00001124TCGv_i32 tcg_const_local_i32(int32_t val)
aurel32bdffd4a2008-10-21 11:30:45 +00001125{
pbrooka7812ae2008-11-17 14:43:54 +00001126 TCGv_i32 t0;
1127 t0 = tcg_temp_local_new_i32();
aurel32bdffd4a2008-10-21 11:30:45 +00001128 tcg_gen_movi_i32(t0, val);
1129 return t0;
1130}
1131
pbrooka7812ae2008-11-17 14:43:54 +00001132TCGv_i64 tcg_const_local_i64(int64_t val)
aurel32bdffd4a2008-10-21 11:30:45 +00001133{
pbrooka7812ae2008-11-17 14:43:54 +00001134 TCGv_i64 t0;
1135 t0 = tcg_temp_local_new_i64();
aurel32bdffd4a2008-10-21 11:30:45 +00001136 tcg_gen_movi_i64(t0, val);
1137 return t0;
1138}
1139
Peter Maydell27bfd832011-03-06 21:39:53 +00001140#if defined(CONFIG_DEBUG_TCG)
1141void tcg_clear_temp_count(void)
1142{
Emilio G. Cotab1311c42017-07-12 17:15:52 -04001143 TCGContext *s = tcg_ctx;
Peter Maydell27bfd832011-03-06 21:39:53 +00001144 s->temps_in_use = 0;
1145}
1146
1147int tcg_check_temp_count(void)
1148{
Emilio G. Cotab1311c42017-07-12 17:15:52 -04001149 TCGContext *s = tcg_ctx;
Peter Maydell27bfd832011-03-06 21:39:53 +00001150 if (s->temps_in_use) {
1151 /* Clear the count so that we don't give another
1152 * warning immediately next time around.
1153 */
1154 s->temps_in_use = 0;
1155 return 1;
1156 }
1157 return 0;
1158}
1159#endif
1160
Richard Hendersonbe0f34b2017-08-17 07:43:20 -07001161/* Return true if OP may appear in the opcode stream.
1162 Test the runtime variable that controls each opcode. */
1163bool tcg_op_supported(TCGOpcode op)
1164{
Richard Hendersond2fd7452017-09-14 13:53:46 -07001165 const bool have_vec
1166 = TCG_TARGET_HAS_v64 | TCG_TARGET_HAS_v128 | TCG_TARGET_HAS_v256;
1167
Richard Hendersonbe0f34b2017-08-17 07:43:20 -07001168 switch (op) {
1169 case INDEX_op_discard:
1170 case INDEX_op_set_label:
1171 case INDEX_op_call:
1172 case INDEX_op_br:
1173 case INDEX_op_mb:
1174 case INDEX_op_insn_start:
1175 case INDEX_op_exit_tb:
1176 case INDEX_op_goto_tb:
Richard Hendersonf4e01e32021-06-29 14:47:39 -07001177 case INDEX_op_goto_ptr:
Richard Hendersonbe0f34b2017-08-17 07:43:20 -07001178 case INDEX_op_qemu_ld_i32:
1179 case INDEX_op_qemu_st_i32:
1180 case INDEX_op_qemu_ld_i64:
1181 case INDEX_op_qemu_st_i64:
1182 return true;
1183
Richard Henderson07ce0b02020-12-09 13:58:39 -06001184 case INDEX_op_qemu_st8_i32:
1185 return TCG_TARGET_HAS_qemu_st8_i32;
1186
Richard Hendersonbe0f34b2017-08-17 07:43:20 -07001187 case INDEX_op_mov_i32:
Richard Hendersonbe0f34b2017-08-17 07:43:20 -07001188 case INDEX_op_setcond_i32:
1189 case INDEX_op_brcond_i32:
1190 case INDEX_op_ld8u_i32:
1191 case INDEX_op_ld8s_i32:
1192 case INDEX_op_ld16u_i32:
1193 case INDEX_op_ld16s_i32:
1194 case INDEX_op_ld_i32:
1195 case INDEX_op_st8_i32:
1196 case INDEX_op_st16_i32:
1197 case INDEX_op_st_i32:
1198 case INDEX_op_add_i32:
1199 case INDEX_op_sub_i32:
1200 case INDEX_op_mul_i32:
1201 case INDEX_op_and_i32:
1202 case INDEX_op_or_i32:
1203 case INDEX_op_xor_i32:
1204 case INDEX_op_shl_i32:
1205 case INDEX_op_shr_i32:
1206 case INDEX_op_sar_i32:
1207 return true;
1208
1209 case INDEX_op_movcond_i32:
1210 return TCG_TARGET_HAS_movcond_i32;
1211 case INDEX_op_div_i32:
1212 case INDEX_op_divu_i32:
1213 return TCG_TARGET_HAS_div_i32;
1214 case INDEX_op_rem_i32:
1215 case INDEX_op_remu_i32:
1216 return TCG_TARGET_HAS_rem_i32;
1217 case INDEX_op_div2_i32:
1218 case INDEX_op_divu2_i32:
1219 return TCG_TARGET_HAS_div2_i32;
1220 case INDEX_op_rotl_i32:
1221 case INDEX_op_rotr_i32:
1222 return TCG_TARGET_HAS_rot_i32;
1223 case INDEX_op_deposit_i32:
1224 return TCG_TARGET_HAS_deposit_i32;
1225 case INDEX_op_extract_i32:
1226 return TCG_TARGET_HAS_extract_i32;
1227 case INDEX_op_sextract_i32:
1228 return TCG_TARGET_HAS_sextract_i32;
Richard Hendersonfce12962019-02-25 10:29:25 -08001229 case INDEX_op_extract2_i32:
1230 return TCG_TARGET_HAS_extract2_i32;
Richard Hendersonbe0f34b2017-08-17 07:43:20 -07001231 case INDEX_op_add2_i32:
1232 return TCG_TARGET_HAS_add2_i32;
1233 case INDEX_op_sub2_i32:
1234 return TCG_TARGET_HAS_sub2_i32;
1235 case INDEX_op_mulu2_i32:
1236 return TCG_TARGET_HAS_mulu2_i32;
1237 case INDEX_op_muls2_i32:
1238 return TCG_TARGET_HAS_muls2_i32;
1239 case INDEX_op_muluh_i32:
1240 return TCG_TARGET_HAS_muluh_i32;
1241 case INDEX_op_mulsh_i32:
1242 return TCG_TARGET_HAS_mulsh_i32;
1243 case INDEX_op_ext8s_i32:
1244 return TCG_TARGET_HAS_ext8s_i32;
1245 case INDEX_op_ext16s_i32:
1246 return TCG_TARGET_HAS_ext16s_i32;
1247 case INDEX_op_ext8u_i32:
1248 return TCG_TARGET_HAS_ext8u_i32;
1249 case INDEX_op_ext16u_i32:
1250 return TCG_TARGET_HAS_ext16u_i32;
1251 case INDEX_op_bswap16_i32:
1252 return TCG_TARGET_HAS_bswap16_i32;
1253 case INDEX_op_bswap32_i32:
1254 return TCG_TARGET_HAS_bswap32_i32;
1255 case INDEX_op_not_i32:
1256 return TCG_TARGET_HAS_not_i32;
1257 case INDEX_op_neg_i32:
1258 return TCG_TARGET_HAS_neg_i32;
1259 case INDEX_op_andc_i32:
1260 return TCG_TARGET_HAS_andc_i32;
1261 case INDEX_op_orc_i32:
1262 return TCG_TARGET_HAS_orc_i32;
1263 case INDEX_op_eqv_i32:
1264 return TCG_TARGET_HAS_eqv_i32;
1265 case INDEX_op_nand_i32:
1266 return TCG_TARGET_HAS_nand_i32;
1267 case INDEX_op_nor_i32:
1268 return TCG_TARGET_HAS_nor_i32;
1269 case INDEX_op_clz_i32:
1270 return TCG_TARGET_HAS_clz_i32;
1271 case INDEX_op_ctz_i32:
1272 return TCG_TARGET_HAS_ctz_i32;
1273 case INDEX_op_ctpop_i32:
1274 return TCG_TARGET_HAS_ctpop_i32;
1275
1276 case INDEX_op_brcond2_i32:
1277 case INDEX_op_setcond2_i32:
1278 return TCG_TARGET_REG_BITS == 32;
1279
1280 case INDEX_op_mov_i64:
Richard Hendersonbe0f34b2017-08-17 07:43:20 -07001281 case INDEX_op_setcond_i64:
1282 case INDEX_op_brcond_i64:
1283 case INDEX_op_ld8u_i64:
1284 case INDEX_op_ld8s_i64:
1285 case INDEX_op_ld16u_i64:
1286 case INDEX_op_ld16s_i64:
1287 case INDEX_op_ld32u_i64:
1288 case INDEX_op_ld32s_i64:
1289 case INDEX_op_ld_i64:
1290 case INDEX_op_st8_i64:
1291 case INDEX_op_st16_i64:
1292 case INDEX_op_st32_i64:
1293 case INDEX_op_st_i64:
1294 case INDEX_op_add_i64:
1295 case INDEX_op_sub_i64:
1296 case INDEX_op_mul_i64:
1297 case INDEX_op_and_i64:
1298 case INDEX_op_or_i64:
1299 case INDEX_op_xor_i64:
1300 case INDEX_op_shl_i64:
1301 case INDEX_op_shr_i64:
1302 case INDEX_op_sar_i64:
1303 case INDEX_op_ext_i32_i64:
1304 case INDEX_op_extu_i32_i64:
1305 return TCG_TARGET_REG_BITS == 64;
1306
1307 case INDEX_op_movcond_i64:
1308 return TCG_TARGET_HAS_movcond_i64;
1309 case INDEX_op_div_i64:
1310 case INDEX_op_divu_i64:
1311 return TCG_TARGET_HAS_div_i64;
1312 case INDEX_op_rem_i64:
1313 case INDEX_op_remu_i64:
1314 return TCG_TARGET_HAS_rem_i64;
1315 case INDEX_op_div2_i64:
1316 case INDEX_op_divu2_i64:
1317 return TCG_TARGET_HAS_div2_i64;
1318 case INDEX_op_rotl_i64:
1319 case INDEX_op_rotr_i64:
1320 return TCG_TARGET_HAS_rot_i64;
1321 case INDEX_op_deposit_i64:
1322 return TCG_TARGET_HAS_deposit_i64;
1323 case INDEX_op_extract_i64:
1324 return TCG_TARGET_HAS_extract_i64;
1325 case INDEX_op_sextract_i64:
1326 return TCG_TARGET_HAS_sextract_i64;
Richard Hendersonfce12962019-02-25 10:29:25 -08001327 case INDEX_op_extract2_i64:
1328 return TCG_TARGET_HAS_extract2_i64;
Richard Hendersonbe0f34b2017-08-17 07:43:20 -07001329 case INDEX_op_extrl_i64_i32:
1330 return TCG_TARGET_HAS_extrl_i64_i32;
1331 case INDEX_op_extrh_i64_i32:
1332 return TCG_TARGET_HAS_extrh_i64_i32;
1333 case INDEX_op_ext8s_i64:
1334 return TCG_TARGET_HAS_ext8s_i64;
1335 case INDEX_op_ext16s_i64:
1336 return TCG_TARGET_HAS_ext16s_i64;
1337 case INDEX_op_ext32s_i64:
1338 return TCG_TARGET_HAS_ext32s_i64;
1339 case INDEX_op_ext8u_i64:
1340 return TCG_TARGET_HAS_ext8u_i64;
1341 case INDEX_op_ext16u_i64:
1342 return TCG_TARGET_HAS_ext16u_i64;
1343 case INDEX_op_ext32u_i64:
1344 return TCG_TARGET_HAS_ext32u_i64;
1345 case INDEX_op_bswap16_i64:
1346 return TCG_TARGET_HAS_bswap16_i64;
1347 case INDEX_op_bswap32_i64:
1348 return TCG_TARGET_HAS_bswap32_i64;
1349 case INDEX_op_bswap64_i64:
1350 return TCG_TARGET_HAS_bswap64_i64;
1351 case INDEX_op_not_i64:
1352 return TCG_TARGET_HAS_not_i64;
1353 case INDEX_op_neg_i64:
1354 return TCG_TARGET_HAS_neg_i64;
1355 case INDEX_op_andc_i64:
1356 return TCG_TARGET_HAS_andc_i64;
1357 case INDEX_op_orc_i64:
1358 return TCG_TARGET_HAS_orc_i64;
1359 case INDEX_op_eqv_i64:
1360 return TCG_TARGET_HAS_eqv_i64;
1361 case INDEX_op_nand_i64:
1362 return TCG_TARGET_HAS_nand_i64;
1363 case INDEX_op_nor_i64:
1364 return TCG_TARGET_HAS_nor_i64;
1365 case INDEX_op_clz_i64:
1366 return TCG_TARGET_HAS_clz_i64;
1367 case INDEX_op_ctz_i64:
1368 return TCG_TARGET_HAS_ctz_i64;
1369 case INDEX_op_ctpop_i64:
1370 return TCG_TARGET_HAS_ctpop_i64;
1371 case INDEX_op_add2_i64:
1372 return TCG_TARGET_HAS_add2_i64;
1373 case INDEX_op_sub2_i64:
1374 return TCG_TARGET_HAS_sub2_i64;
1375 case INDEX_op_mulu2_i64:
1376 return TCG_TARGET_HAS_mulu2_i64;
1377 case INDEX_op_muls2_i64:
1378 return TCG_TARGET_HAS_muls2_i64;
1379 case INDEX_op_muluh_i64:
1380 return TCG_TARGET_HAS_muluh_i64;
1381 case INDEX_op_mulsh_i64:
1382 return TCG_TARGET_HAS_mulsh_i64;
1383
Richard Hendersond2fd7452017-09-14 13:53:46 -07001384 case INDEX_op_mov_vec:
1385 case INDEX_op_dup_vec:
Richard Henderson37ee55a2019-03-17 01:55:22 +00001386 case INDEX_op_dupm_vec:
Richard Hendersond2fd7452017-09-14 13:53:46 -07001387 case INDEX_op_ld_vec:
1388 case INDEX_op_st_vec:
1389 case INDEX_op_add_vec:
1390 case INDEX_op_sub_vec:
1391 case INDEX_op_and_vec:
1392 case INDEX_op_or_vec:
1393 case INDEX_op_xor_vec:
Richard Henderson212be172017-11-17 20:47:42 +01001394 case INDEX_op_cmp_vec:
Richard Hendersond2fd7452017-09-14 13:53:46 -07001395 return have_vec;
1396 case INDEX_op_dup2_vec:
1397 return have_vec && TCG_TARGET_REG_BITS == 32;
1398 case INDEX_op_not_vec:
1399 return have_vec && TCG_TARGET_HAS_not_vec;
1400 case INDEX_op_neg_vec:
1401 return have_vec && TCG_TARGET_HAS_neg_vec;
Richard Hendersonbcefc902019-04-17 13:53:02 -10001402 case INDEX_op_abs_vec:
1403 return have_vec && TCG_TARGET_HAS_abs_vec;
Richard Hendersond2fd7452017-09-14 13:53:46 -07001404 case INDEX_op_andc_vec:
1405 return have_vec && TCG_TARGET_HAS_andc_vec;
1406 case INDEX_op_orc_vec:
1407 return have_vec && TCG_TARGET_HAS_orc_vec;
Richard Henderson37740302017-11-21 10:11:14 +01001408 case INDEX_op_mul_vec:
1409 return have_vec && TCG_TARGET_HAS_mul_vec;
Richard Hendersond0ec9792017-11-17 14:35:11 +01001410 case INDEX_op_shli_vec:
1411 case INDEX_op_shri_vec:
1412 case INDEX_op_sari_vec:
1413 return have_vec && TCG_TARGET_HAS_shi_vec;
1414 case INDEX_op_shls_vec:
1415 case INDEX_op_shrs_vec:
1416 case INDEX_op_sars_vec:
1417 return have_vec && TCG_TARGET_HAS_shs_vec;
1418 case INDEX_op_shlv_vec:
1419 case INDEX_op_shrv_vec:
1420 case INDEX_op_sarv_vec:
1421 return have_vec && TCG_TARGET_HAS_shv_vec;
Richard Hendersonb0f7e742020-04-19 18:01:52 -07001422 case INDEX_op_rotli_vec:
1423 return have_vec && TCG_TARGET_HAS_roti_vec;
Richard Henderson23850a72020-04-20 08:22:44 -07001424 case INDEX_op_rotls_vec:
1425 return have_vec && TCG_TARGET_HAS_rots_vec;
Richard Henderson5d0ceda2020-04-19 19:47:59 -07001426 case INDEX_op_rotlv_vec:
1427 case INDEX_op_rotrv_vec:
1428 return have_vec && TCG_TARGET_HAS_rotv_vec;
Richard Henderson8afaf052018-12-17 18:01:47 -08001429 case INDEX_op_ssadd_vec:
1430 case INDEX_op_usadd_vec:
1431 case INDEX_op_sssub_vec:
1432 case INDEX_op_ussub_vec:
1433 return have_vec && TCG_TARGET_HAS_sat_vec;
Richard Hendersondd0a0fc2018-12-17 19:35:46 -08001434 case INDEX_op_smin_vec:
1435 case INDEX_op_umin_vec:
1436 case INDEX_op_smax_vec:
1437 case INDEX_op_umax_vec:
1438 return have_vec && TCG_TARGET_HAS_minmax_vec;
Richard Henderson38dc1292019-04-30 11:02:23 -07001439 case INDEX_op_bitsel_vec:
1440 return have_vec && TCG_TARGET_HAS_bitsel_vec;
Richard Hendersonf75da292019-04-30 13:01:12 -07001441 case INDEX_op_cmpsel_vec:
1442 return have_vec && TCG_TARGET_HAS_cmpsel_vec;
Richard Hendersond2fd7452017-09-14 13:53:46 -07001443
Richard Hendersondb432672017-09-15 14:11:45 -07001444 default:
1445 tcg_debug_assert(op > INDEX_op_last_generic && op < NB_OPS);
1446 return true;
Richard Hendersonbe0f34b2017-08-17 07:43:20 -07001447 }
Richard Hendersonbe0f34b2017-08-17 07:43:20 -07001448}
1449
bellard39cf05d2008-05-22 14:59:57 +00001450/* Note: we convert the 64 bit args to 32 bit and do some alignment
1451 and endian swap. Maybe it would be better to do the alignment
1452 and endian swap in tcg_reg_alloc_call(). */
Richard Hendersonae8b75d2017-10-15 13:27:56 -07001453void tcg_gen_callN(void *func, TCGTemp *ret, int nargs, TCGTemp **args)
bellardc896fe22008-02-01 10:05:41 +00001454{
Richard Henderson75e8b9b2016-12-08 10:52:57 -08001455 int i, real_args, nb_rets, pi;
Richard Henderson3e92aa32021-03-18 11:29:50 -06001456 unsigned typemask;
1457 const TCGHelperInfo *info;
Richard Henderson75e8b9b2016-12-08 10:52:57 -08001458 TCGOp *op;
Richard Hendersonafb49892014-04-07 15:10:05 -07001459
Emilio G. Cota619205f2017-07-05 18:41:23 -04001460 info = g_hash_table_lookup(helper_table, (gpointer)func);
Richard Henderson7319d832021-03-18 10:01:01 -06001461 typemask = info->typemask;
Richard Henderson2bece2c2010-06-14 17:35:27 -07001462
Emilio G. Cota38b47b12018-12-07 15:33:56 -05001463#ifdef CONFIG_PLUGIN
1464 /* detect non-plugin helpers */
1465 if (tcg_ctx->plugin_insn && unlikely(strncmp(info->name, "plugin_", 7))) {
1466 tcg_ctx->plugin_insn->calls_helpers = true;
1467 }
1468#endif
1469
Richard Henderson34b1a492014-03-04 13:39:48 -08001470#if defined(__sparc__) && !defined(__arch64__) \
1471 && !defined(CONFIG_TCG_INTERPRETER)
1472 /* We have 64-bit values in one register, but need to pass as two
1473 separate parameters. Split them. */
Richard Henderson7319d832021-03-18 10:01:01 -06001474 int orig_typemask = typemask;
Richard Henderson34b1a492014-03-04 13:39:48 -08001475 int orig_nargs = nargs;
1476 TCGv_i64 retl, reth;
Richard Hendersonae8b75d2017-10-15 13:27:56 -07001477 TCGTemp *split_args[MAX_OPC_PARAM];
Richard Henderson34b1a492014-03-04 13:39:48 -08001478
Richard Hendersonf7647182017-11-02 12:47:37 +01001479 retl = NULL;
1480 reth = NULL;
Richard Henderson7319d832021-03-18 10:01:01 -06001481 typemask = 0;
1482 for (i = real_args = 0; i < nargs; ++i) {
1483 int argtype = extract32(orig_typemask, (i + 1) * 3, 3);
1484 bool is_64bit = (argtype & ~1) == dh_typecode_i64;
1485
1486 if (is_64bit) {
1487 TCGv_i64 orig = temp_tcgv_i64(args[i]);
1488 TCGv_i32 h = tcg_temp_new_i32();
1489 TCGv_i32 l = tcg_temp_new_i32();
1490 tcg_gen_extr_i64_i32(l, h, orig);
1491 split_args[real_args++] = tcgv_i32_temp(h);
1492 typemask |= dh_typecode_i32 << (real_args * 3);
1493 split_args[real_args++] = tcgv_i32_temp(l);
1494 typemask |= dh_typecode_i32 << (real_args * 3);
1495 } else {
1496 split_args[real_args++] = args[i];
1497 typemask |= argtype << (real_args * 3);
Richard Henderson34b1a492014-03-04 13:39:48 -08001498 }
Richard Henderson34b1a492014-03-04 13:39:48 -08001499 }
Richard Henderson7319d832021-03-18 10:01:01 -06001500 nargs = real_args;
1501 args = split_args;
Richard Henderson34b1a492014-03-04 13:39:48 -08001502#elif defined(TCG_TARGET_EXTEND_ARGS) && TCG_TARGET_REG_BITS == 64
Richard Henderson2bece2c2010-06-14 17:35:27 -07001503 for (i = 0; i < nargs; ++i) {
Richard Henderson7319d832021-03-18 10:01:01 -06001504 int argtype = extract32(typemask, (i + 1) * 3, 3);
1505 bool is_32bit = (argtype & ~1) == dh_typecode_i32;
1506 bool is_signed = argtype & 1;
1507
1508 if (is_32bit) {
Richard Henderson2bece2c2010-06-14 17:35:27 -07001509 TCGv_i64 temp = tcg_temp_new_i64();
Richard Henderson085272b2017-10-20 00:05:45 -07001510 TCGv_i64 orig = temp_tcgv_i64(args[i]);
Richard Henderson2bece2c2010-06-14 17:35:27 -07001511 if (is_signed) {
1512 tcg_gen_ext32s_i64(temp, orig);
1513 } else {
1514 tcg_gen_ext32u_i64(temp, orig);
1515 }
Richard Hendersonae8b75d2017-10-15 13:27:56 -07001516 args[i] = tcgv_i64_temp(temp);
Richard Henderson2bece2c2010-06-14 17:35:27 -07001517 }
1518 }
1519#endif /* TCG_TARGET_EXTEND_ARGS */
1520
Richard Henderson15fa08f2017-11-02 15:19:14 +01001521 op = tcg_emit_op(INDEX_op_call);
Richard Henderson75e8b9b2016-12-08 10:52:57 -08001522
1523 pi = 0;
Richard Hendersonae8b75d2017-10-15 13:27:56 -07001524 if (ret != NULL) {
Richard Henderson34b1a492014-03-04 13:39:48 -08001525#if defined(__sparc__) && !defined(__arch64__) \
1526 && !defined(CONFIG_TCG_INTERPRETER)
Richard Henderson7319d832021-03-18 10:01:01 -06001527 if ((typemask & 6) == dh_typecode_i64) {
Richard Henderson34b1a492014-03-04 13:39:48 -08001528 /* The 32-bit ABI is going to return the 64-bit value in
1529 the %o0/%o1 register pair. Prepare for this by using
1530 two return temporaries, and reassemble below. */
1531 retl = tcg_temp_new_i64();
1532 reth = tcg_temp_new_i64();
Richard Hendersonae8b75d2017-10-15 13:27:56 -07001533 op->args[pi++] = tcgv_i64_arg(reth);
1534 op->args[pi++] = tcgv_i64_arg(retl);
Richard Henderson34b1a492014-03-04 13:39:48 -08001535 nb_rets = 2;
1536 } else {
Richard Hendersonae8b75d2017-10-15 13:27:56 -07001537 op->args[pi++] = temp_arg(ret);
Richard Henderson34b1a492014-03-04 13:39:48 -08001538 nb_rets = 1;
1539 }
1540#else
Richard Henderson7319d832021-03-18 10:01:01 -06001541 if (TCG_TARGET_REG_BITS < 64 && (typemask & 6) == dh_typecode_i64) {
Richard Henderson02eb19d2014-03-31 14:09:13 -07001542#ifdef HOST_WORDS_BIGENDIAN
Richard Hendersonae8b75d2017-10-15 13:27:56 -07001543 op->args[pi++] = temp_arg(ret + 1);
1544 op->args[pi++] = temp_arg(ret);
pbrooka7812ae2008-11-17 14:43:54 +00001545#else
Richard Hendersonae8b75d2017-10-15 13:27:56 -07001546 op->args[pi++] = temp_arg(ret);
1547 op->args[pi++] = temp_arg(ret + 1);
pbrooka7812ae2008-11-17 14:43:54 +00001548#endif
1549 nb_rets = 2;
Richard Henderson34b1a492014-03-04 13:39:48 -08001550 } else {
Richard Hendersonae8b75d2017-10-15 13:27:56 -07001551 op->args[pi++] = temp_arg(ret);
pbrooka7812ae2008-11-17 14:43:54 +00001552 nb_rets = 1;
1553 }
Richard Henderson34b1a492014-03-04 13:39:48 -08001554#endif
pbrooka7812ae2008-11-17 14:43:54 +00001555 } else {
1556 nb_rets = 0;
1557 }
Richard Hendersoncd9090a2017-11-14 13:02:51 +01001558 TCGOP_CALLO(op) = nb_rets;
Richard Henderson75e8b9b2016-12-08 10:52:57 -08001559
pbrooka7812ae2008-11-17 14:43:54 +00001560 real_args = 0;
1561 for (i = 0; i < nargs; i++) {
Richard Henderson7319d832021-03-18 10:01:01 -06001562 int argtype = extract32(typemask, (i + 1) * 3, 3);
1563 bool is_64bit = (argtype & ~1) == dh_typecode_i64;
Richard Henderson7b7d8b22021-01-30 14:24:25 -08001564 bool want_align = false;
1565
1566#if defined(CONFIG_TCG_INTERPRETER)
1567 /*
1568 * Align all arguments, so that they land in predictable places
1569 * for passing off to ffi_call.
1570 */
1571 want_align = true;
1572#elif defined(TCG_TARGET_CALL_ALIGN_ARGS)
1573 /* Some targets want aligned 64 bit args */
1574 want_align = is_64bit;
1575#endif
1576
1577 if (TCG_TARGET_REG_BITS < 64 && want_align && (real_args & 1)) {
1578 op->args[pi++] = TCG_CALL_DUMMY_ARG;
1579 real_args++;
1580 }
Richard Henderson7319d832021-03-18 10:01:01 -06001581
Richard Hendersonbbb8a1b2014-04-08 08:39:43 -07001582 if (TCG_TARGET_REG_BITS < 64 && is_64bit) {
Richard Henderson7b7d8b22021-01-30 14:24:25 -08001583 /*
1584 * If stack grows up, then we will be placing successive
1585 * arguments at lower addresses, which means we need to
1586 * reverse the order compared to how we would normally
1587 * treat either big or little-endian. For those arguments
1588 * that will wind up in registers, this still works for
1589 * HPPA (the only current STACK_GROWSUP target) since the
1590 * argument registers are *also* allocated in decreasing
1591 * order. If another such target is added, this logic may
1592 * have to get more complicated to differentiate between
1593 * stack arguments and register arguments.
1594 */
Richard Henderson02eb19d2014-03-31 14:09:13 -07001595#if defined(HOST_WORDS_BIGENDIAN) != defined(TCG_TARGET_STACK_GROWSUP)
Richard Hendersonae8b75d2017-10-15 13:27:56 -07001596 op->args[pi++] = temp_arg(args[i] + 1);
1597 op->args[pi++] = temp_arg(args[i]);
bellardc896fe22008-02-01 10:05:41 +00001598#else
Richard Hendersonae8b75d2017-10-15 13:27:56 -07001599 op->args[pi++] = temp_arg(args[i]);
1600 op->args[pi++] = temp_arg(args[i] + 1);
bellardc896fe22008-02-01 10:05:41 +00001601#endif
pbrooka7812ae2008-11-17 14:43:54 +00001602 real_args += 2;
Richard Henderson2bece2c2010-06-14 17:35:27 -07001603 continue;
bellardc896fe22008-02-01 10:05:41 +00001604 }
Richard Henderson2bece2c2010-06-14 17:35:27 -07001605
Richard Hendersonae8b75d2017-10-15 13:27:56 -07001606 op->args[pi++] = temp_arg(args[i]);
Richard Henderson2bece2c2010-06-14 17:35:27 -07001607 real_args++;
bellardc896fe22008-02-01 10:05:41 +00001608 }
Richard Henderson75e8b9b2016-12-08 10:52:57 -08001609 op->args[pi++] = (uintptr_t)func;
Richard Henderson3e92aa32021-03-18 11:29:50 -06001610 op->args[pi++] = (uintptr_t)info;
Richard Hendersoncd9090a2017-11-14 13:02:51 +01001611 TCGOP_CALLI(op) = real_args;
pbrooka7812ae2008-11-17 14:43:54 +00001612
Richard Henderson75e8b9b2016-12-08 10:52:57 -08001613 /* Make sure the fields didn't overflow. */
Richard Hendersoncd9090a2017-11-14 13:02:51 +01001614 tcg_debug_assert(TCGOP_CALLI(op) == real_args);
Richard Henderson75e8b9b2016-12-08 10:52:57 -08001615 tcg_debug_assert(pi <= ARRAY_SIZE(op->args));
Richard Henderson2bece2c2010-06-14 17:35:27 -07001616
Richard Henderson34b1a492014-03-04 13:39:48 -08001617#if defined(__sparc__) && !defined(__arch64__) \
1618 && !defined(CONFIG_TCG_INTERPRETER)
1619 /* Free all of the parts we allocated above. */
1620 for (i = real_args = 0; i < orig_nargs; ++i) {
Richard Henderson7319d832021-03-18 10:01:01 -06001621 int argtype = extract32(orig_typemask, (i + 1) * 3, 3);
1622 bool is_64bit = (argtype & ~1) == dh_typecode_i64;
1623
Richard Henderson34b1a492014-03-04 13:39:48 -08001624 if (is_64bit) {
Richard Henderson085272b2017-10-20 00:05:45 -07001625 tcg_temp_free_internal(args[real_args++]);
1626 tcg_temp_free_internal(args[real_args++]);
Richard Henderson34b1a492014-03-04 13:39:48 -08001627 } else {
1628 real_args++;
1629 }
1630 }
Richard Henderson7319d832021-03-18 10:01:01 -06001631 if ((orig_typemask & 6) == dh_typecode_i64) {
Richard Henderson34b1a492014-03-04 13:39:48 -08001632 /* The 32-bit ABI returned two 32-bit pieces. Re-assemble them.
1633 Note that describing these as TCGv_i64 eliminates an unnecessary
1634 zero-extension that tcg_gen_concat_i32_i64 would create. */
Richard Henderson085272b2017-10-20 00:05:45 -07001635 tcg_gen_concat32_i64(temp_tcgv_i64(ret), retl, reth);
Richard Henderson34b1a492014-03-04 13:39:48 -08001636 tcg_temp_free_i64(retl);
1637 tcg_temp_free_i64(reth);
1638 }
1639#elif defined(TCG_TARGET_EXTEND_ARGS) && TCG_TARGET_REG_BITS == 64
Richard Henderson2bece2c2010-06-14 17:35:27 -07001640 for (i = 0; i < nargs; ++i) {
Richard Henderson7319d832021-03-18 10:01:01 -06001641 int argtype = extract32(typemask, (i + 1) * 3, 3);
1642 bool is_32bit = (argtype & ~1) == dh_typecode_i32;
1643
1644 if (is_32bit) {
Richard Henderson085272b2017-10-20 00:05:45 -07001645 tcg_temp_free_internal(args[i]);
Richard Henderson2bece2c2010-06-14 17:35:27 -07001646 }
1647 }
1648#endif /* TCG_TARGET_EXTEND_ARGS */
bellardc896fe22008-02-01 10:05:41 +00001649}
bellardc896fe22008-02-01 10:05:41 +00001650
blueswir18fcd3692008-08-17 20:26:25 +00001651static void tcg_reg_alloc_start(TCGContext *s)
bellardc896fe22008-02-01 10:05:41 +00001652{
Richard Hendersonac3b8892016-11-02 11:21:44 -06001653 int i, n;
Richard Hendersonac3b8892016-11-02 11:21:44 -06001654
Richard Hendersonee17db82020-03-29 10:11:56 -07001655 for (i = 0, n = s->nb_temps; i < n; i++) {
1656 TCGTemp *ts = &s->temps[i];
1657 TCGTempVal val = TEMP_VAL_MEM;
1658
1659 switch (ts->kind) {
Richard Hendersonc0522132020-03-29 18:55:52 -07001660 case TEMP_CONST:
1661 val = TEMP_VAL_CONST;
1662 break;
Richard Hendersonee17db82020-03-29 10:11:56 -07001663 case TEMP_FIXED:
1664 val = TEMP_VAL_REG;
1665 break;
1666 case TEMP_GLOBAL:
1667 break;
1668 case TEMP_NORMAL:
1669 val = TEMP_VAL_DEAD;
1670 /* fall through */
1671 case TEMP_LOCAL:
1672 ts->mem_allocated = 0;
1673 break;
1674 default:
1675 g_assert_not_reached();
1676 }
1677 ts->val_type = val;
bellarde8996ee2008-05-23 17:33:39 +00001678 }
Richard Hendersonf8b2f202013-09-18 15:21:56 -07001679
1680 memset(s->reg_to_temp, 0, sizeof(s->reg_to_temp));
bellardc896fe22008-02-01 10:05:41 +00001681}
1682
Richard Hendersonf8b2f202013-09-18 15:21:56 -07001683static char *tcg_get_arg_str_ptr(TCGContext *s, char *buf, int buf_size,
1684 TCGTemp *ts)
bellardc896fe22008-02-01 10:05:41 +00001685{
Richard Henderson1807f4c2017-06-20 12:24:57 -07001686 int idx = temp_idx(ts);
pbrookac56dd42008-02-03 19:56:33 +00001687
Richard Hendersonee17db82020-03-29 10:11:56 -07001688 switch (ts->kind) {
1689 case TEMP_FIXED:
1690 case TEMP_GLOBAL:
pbrookac56dd42008-02-03 19:56:33 +00001691 pstrcpy(buf, buf_size, ts->name);
Richard Hendersonee17db82020-03-29 10:11:56 -07001692 break;
1693 case TEMP_LOCAL:
Richard Hendersonf8b2f202013-09-18 15:21:56 -07001694 snprintf(buf, buf_size, "loc%d", idx - s->nb_globals);
Richard Hendersonee17db82020-03-29 10:11:56 -07001695 break;
1696 case TEMP_NORMAL:
Richard Hendersonf8b2f202013-09-18 15:21:56 -07001697 snprintf(buf, buf_size, "tmp%d", idx - s->nb_globals);
Richard Hendersonee17db82020-03-29 10:11:56 -07001698 break;
Richard Hendersonc0522132020-03-29 18:55:52 -07001699 case TEMP_CONST:
1700 switch (ts->type) {
1701 case TCG_TYPE_I32:
1702 snprintf(buf, buf_size, "$0x%x", (int32_t)ts->val);
1703 break;
1704#if TCG_TARGET_REG_BITS > 32
1705 case TCG_TYPE_I64:
1706 snprintf(buf, buf_size, "$0x%" PRIx64, ts->val);
1707 break;
1708#endif
1709 case TCG_TYPE_V64:
1710 case TCG_TYPE_V128:
1711 case TCG_TYPE_V256:
1712 snprintf(buf, buf_size, "v%d$0x%" PRIx64,
1713 64 << (ts->type - TCG_TYPE_V64), ts->val);
1714 break;
1715 default:
1716 g_assert_not_reached();
1717 }
1718 break;
bellardc896fe22008-02-01 10:05:41 +00001719 }
1720 return buf;
1721}
1722
Richard Henderson43439132017-06-19 23:18:10 -07001723static char *tcg_get_arg_str(TCGContext *s, char *buf,
1724 int buf_size, TCGArg arg)
Richard Hendersonf8b2f202013-09-18 15:21:56 -07001725{
Richard Henderson43439132017-06-19 23:18:10 -07001726 return tcg_get_arg_str_ptr(s, buf, buf_size, arg_temp(arg));
Richard Hendersonf8b2f202013-09-18 15:21:56 -07001727}
1728
blueswir1f48f3ed2008-09-14 07:45:17 +00001729static const char * const cond_name[] =
1730{
Richard Henderson0aed2572012-09-24 14:21:40 -07001731 [TCG_COND_NEVER] = "never",
1732 [TCG_COND_ALWAYS] = "always",
blueswir1f48f3ed2008-09-14 07:45:17 +00001733 [TCG_COND_EQ] = "eq",
1734 [TCG_COND_NE] = "ne",
1735 [TCG_COND_LT] = "lt",
1736 [TCG_COND_GE] = "ge",
1737 [TCG_COND_LE] = "le",
1738 [TCG_COND_GT] = "gt",
1739 [TCG_COND_LTU] = "ltu",
1740 [TCG_COND_GEU] = "geu",
1741 [TCG_COND_LEU] = "leu",
1742 [TCG_COND_GTU] = "gtu"
1743};
1744
Richard Hendersonf713d6a2013-09-04 08:11:05 -07001745static const char * const ldst_name[] =
1746{
1747 [MO_UB] = "ub",
1748 [MO_SB] = "sb",
1749 [MO_LEUW] = "leuw",
1750 [MO_LESW] = "lesw",
1751 [MO_LEUL] = "leul",
1752 [MO_LESL] = "lesl",
1753 [MO_LEQ] = "leq",
1754 [MO_BEUW] = "beuw",
1755 [MO_BESW] = "besw",
1756 [MO_BEUL] = "beul",
1757 [MO_BESL] = "besl",
1758 [MO_BEQ] = "beq",
1759};
1760
Sergey Sorokin1f00b272016-06-23 21:16:46 +03001761static const char * const alignment_name[(MO_AMASK >> MO_ASHIFT) + 1] = {
tony.nguyen@bt.com52bf9772019-07-18 06:01:31 +00001762#ifdef TARGET_ALIGNED_ONLY
Sergey Sorokin1f00b272016-06-23 21:16:46 +03001763 [MO_UNALN >> MO_ASHIFT] = "un+",
1764 [MO_ALIGN >> MO_ASHIFT] = "",
1765#else
1766 [MO_UNALN >> MO_ASHIFT] = "",
1767 [MO_ALIGN >> MO_ASHIFT] = "al+",
1768#endif
1769 [MO_ALIGN_2 >> MO_ASHIFT] = "al2+",
1770 [MO_ALIGN_4 >> MO_ASHIFT] = "al4+",
1771 [MO_ALIGN_8 >> MO_ASHIFT] = "al8+",
1772 [MO_ALIGN_16 >> MO_ASHIFT] = "al16+",
1773 [MO_ALIGN_32 >> MO_ASHIFT] = "al32+",
1774 [MO_ALIGN_64 >> MO_ASHIFT] = "al64+",
1775};
1776
Richard Henderson587195b2021-06-12 21:32:27 -07001777static const char bswap_flag_name[][6] = {
1778 [TCG_BSWAP_IZ] = "iz",
1779 [TCG_BSWAP_OZ] = "oz",
1780 [TCG_BSWAP_OS] = "os",
1781 [TCG_BSWAP_IZ | TCG_BSWAP_OZ] = "iz,oz",
1782 [TCG_BSWAP_IZ | TCG_BSWAP_OS] = "iz,os",
1783};
1784
Richard Hendersonb0164862018-11-27 07:16:21 -08001785static inline bool tcg_regset_single(TCGRegSet d)
1786{
1787 return (d & (d - 1)) == 0;
1788}
1789
1790static inline TCGReg tcg_regset_first(TCGRegSet d)
1791{
1792 if (TCG_TARGET_NB_REGS <= 32) {
1793 return ctz32(d);
1794 } else {
1795 return ctz64(d);
1796 }
1797}
1798
Richard Henderson1894f692018-11-27 12:46:00 -08001799static void tcg_dump_ops(TCGContext *s, bool have_prefs)
bellardc896fe22008-02-01 10:05:41 +00001800{
bellardc896fe22008-02-01 10:05:41 +00001801 char buf[128];
Richard Hendersonc45cb8b2014-09-19 13:49:15 -07001802 TCGOp *op;
bellardc896fe22008-02-01 10:05:41 +00001803
Richard Henderson15fa08f2017-11-02 15:19:14 +01001804 QTAILQ_FOREACH(op, &s->ops, link) {
Richard Hendersonc45cb8b2014-09-19 13:49:15 -07001805 int i, k, nb_oargs, nb_iargs, nb_cargs;
1806 const TCGOpDef *def;
Richard Hendersonc45cb8b2014-09-19 13:49:15 -07001807 TCGOpcode c;
Richard Hendersonbdfb4602016-06-23 19:15:55 -07001808 int col = 0;
Richard Hendersonc45cb8b2014-09-19 13:49:15 -07001809
Richard Hendersonc45cb8b2014-09-19 13:49:15 -07001810 c = op->opc;
bellardc896fe22008-02-01 10:05:41 +00001811 def = &tcg_op_defs[c];
Richard Hendersonc45cb8b2014-09-19 13:49:15 -07001812
Richard Henderson765b8422015-08-29 12:37:33 -07001813 if (c == INDEX_op_insn_start) {
Richard Hendersonb0164862018-11-27 07:16:21 -08001814 nb_oargs = 0;
Richard Henderson15fa08f2017-11-02 15:19:14 +01001815 col += qemu_log("\n ----");
Richard Henderson9aef40e2015-08-30 09:21:33 -07001816
1817 for (i = 0; i < TARGET_INSN_START_WORDS; ++i) {
1818 target_ulong a;
bellard7e4597d2008-05-22 16:56:05 +00001819#if TARGET_LONG_BITS > TCG_TARGET_REG_BITS
Richard Hendersonefee3742016-12-08 13:12:08 -08001820 a = deposit64(op->args[i * 2], 32, 32, op->args[i * 2 + 1]);
bellard7e4597d2008-05-22 16:56:05 +00001821#else
Richard Hendersonefee3742016-12-08 13:12:08 -08001822 a = op->args[i];
bellard7e4597d2008-05-22 16:56:05 +00001823#endif
Richard Hendersonbdfb4602016-06-23 19:15:55 -07001824 col += qemu_log(" " TARGET_FMT_lx, a);
Blue Swirleeacee42012-06-03 16:35:32 +00001825 }
bellard7e4597d2008-05-22 16:56:05 +00001826 } else if (c == INDEX_op_call) {
Richard Henderson3e92aa32021-03-18 11:29:50 -06001827 const TCGHelperInfo *info = tcg_call_info(op);
Richard Hendersonfa52e662021-03-18 16:40:07 -06001828 void *func = tcg_call_func(op);
Richard Henderson3e92aa32021-03-18 11:29:50 -06001829
bellardc896fe22008-02-01 10:05:41 +00001830 /* variable number of arguments */
Richard Hendersoncd9090a2017-11-14 13:02:51 +01001831 nb_oargs = TCGOP_CALLO(op);
1832 nb_iargs = TCGOP_CALLI(op);
bellardc896fe22008-02-01 10:05:41 +00001833 nb_cargs = def->nb_cargs;
bellardc896fe22008-02-01 10:05:41 +00001834
Richard Henderson3e92aa32021-03-18 11:29:50 -06001835 col += qemu_log(" %s ", def->name);
1836
1837 /*
1838 * Print the function name from TCGHelperInfo, if available.
1839 * Note that plugins have a template function for the info,
1840 * but the actual function pointer comes from the plugin.
1841 */
Richard Henderson3e92aa32021-03-18 11:29:50 -06001842 if (func == info->func) {
1843 col += qemu_log("%s", info->name);
1844 } else {
1845 col += qemu_log("plugin(%p)", func);
1846 }
1847
Richard Henderson8973fe42021-07-06 17:03:32 -07001848 col += qemu_log(",$0x%x,$%d", info->flags, nb_oargs);
Richard Hendersoncf066672014-03-22 20:06:52 -07001849 for (i = 0; i < nb_oargs; i++) {
Richard Henderson43439132017-06-19 23:18:10 -07001850 col += qemu_log(",%s", tcg_get_arg_str(s, buf, sizeof(buf),
1851 op->args[i]));
bellardb03cce82008-05-10 10:52:05 +00001852 }
Richard Hendersoncf066672014-03-22 20:06:52 -07001853 for (i = 0; i < nb_iargs; i++) {
Richard Hendersonefee3742016-12-08 13:12:08 -08001854 TCGArg arg = op->args[nb_oargs + i];
Richard Hendersoncf066672014-03-22 20:06:52 -07001855 const char *t = "<dummy>";
1856 if (arg != TCG_CALL_DUMMY_ARG) {
Richard Henderson43439132017-06-19 23:18:10 -07001857 t = tcg_get_arg_str(s, buf, sizeof(buf), arg);
bellard39cf05d2008-05-22 14:59:57 +00001858 }
Richard Hendersonbdfb4602016-06-23 19:15:55 -07001859 col += qemu_log(",%s", t);
bellarde8996ee2008-05-23 17:33:39 +00001860 }
bellardb03cce82008-05-10 10:52:05 +00001861 } else {
Richard Hendersonbdfb4602016-06-23 19:15:55 -07001862 col += qemu_log(" %s ", def->name);
Richard Hendersonc45cb8b2014-09-19 13:49:15 -07001863
1864 nb_oargs = def->nb_oargs;
1865 nb_iargs = def->nb_iargs;
1866 nb_cargs = def->nb_cargs;
1867
Richard Hendersond2fd7452017-09-14 13:53:46 -07001868 if (def->flags & TCG_OPF_VECTOR) {
1869 col += qemu_log("v%d,e%d,", 64 << TCGOP_VECL(op),
1870 8 << TCGOP_VECE(op));
1871 }
1872
bellardb03cce82008-05-10 10:52:05 +00001873 k = 0;
Richard Hendersonc45cb8b2014-09-19 13:49:15 -07001874 for (i = 0; i < nb_oargs; i++) {
Blue Swirleeacee42012-06-03 16:35:32 +00001875 if (k != 0) {
Richard Hendersonbdfb4602016-06-23 19:15:55 -07001876 col += qemu_log(",");
Blue Swirleeacee42012-06-03 16:35:32 +00001877 }
Richard Henderson43439132017-06-19 23:18:10 -07001878 col += qemu_log("%s", tcg_get_arg_str(s, buf, sizeof(buf),
1879 op->args[k++]));
bellardb03cce82008-05-10 10:52:05 +00001880 }
Richard Hendersonc45cb8b2014-09-19 13:49:15 -07001881 for (i = 0; i < nb_iargs; i++) {
Blue Swirleeacee42012-06-03 16:35:32 +00001882 if (k != 0) {
Richard Hendersonbdfb4602016-06-23 19:15:55 -07001883 col += qemu_log(",");
Blue Swirleeacee42012-06-03 16:35:32 +00001884 }
Richard Henderson43439132017-06-19 23:18:10 -07001885 col += qemu_log("%s", tcg_get_arg_str(s, buf, sizeof(buf),
1886 op->args[k++]));
bellardb03cce82008-05-10 10:52:05 +00001887 }
Richard Hendersonbe210ac2010-01-07 10:13:31 -08001888 switch (c) {
1889 case INDEX_op_brcond_i32:
Richard Hendersonffc5ea02012-09-21 10:13:34 -07001890 case INDEX_op_setcond_i32:
1891 case INDEX_op_movcond_i32:
Richard Hendersonbe210ac2010-01-07 10:13:31 -08001892 case INDEX_op_brcond2_i32:
Richard Hendersonbe210ac2010-01-07 10:13:31 -08001893 case INDEX_op_setcond2_i32:
Richard Hendersonffc5ea02012-09-21 10:13:34 -07001894 case INDEX_op_brcond_i64:
Richard Hendersonbe210ac2010-01-07 10:13:31 -08001895 case INDEX_op_setcond_i64:
Richard Hendersonffc5ea02012-09-21 10:13:34 -07001896 case INDEX_op_movcond_i64:
Richard Henderson212be172017-11-17 20:47:42 +01001897 case INDEX_op_cmp_vec:
Richard Hendersonf75da292019-04-30 13:01:12 -07001898 case INDEX_op_cmpsel_vec:
Richard Hendersonefee3742016-12-08 13:12:08 -08001899 if (op->args[k] < ARRAY_SIZE(cond_name)
1900 && cond_name[op->args[k]]) {
1901 col += qemu_log(",%s", cond_name[op->args[k++]]);
Blue Swirleeacee42012-06-03 16:35:32 +00001902 } else {
Richard Hendersonefee3742016-12-08 13:12:08 -08001903 col += qemu_log(",$0x%" TCG_PRIlx, op->args[k++]);
Blue Swirleeacee42012-06-03 16:35:32 +00001904 }
blueswir1f48f3ed2008-09-14 07:45:17 +00001905 i = 1;
Richard Hendersonbe210ac2010-01-07 10:13:31 -08001906 break;
Richard Hendersonf713d6a2013-09-04 08:11:05 -07001907 case INDEX_op_qemu_ld_i32:
1908 case INDEX_op_qemu_st_i32:
Richard Henderson07ce0b02020-12-09 13:58:39 -06001909 case INDEX_op_qemu_st8_i32:
Richard Hendersonf713d6a2013-09-04 08:11:05 -07001910 case INDEX_op_qemu_ld_i64:
1911 case INDEX_op_qemu_st_i64:
Richard Henderson59227d52015-05-12 11:51:44 -07001912 {
Richard Hendersonefee3742016-12-08 13:12:08 -08001913 TCGMemOpIdx oi = op->args[k++];
Tony Nguyen14776ab2019-08-24 04:10:58 +10001914 MemOp op = get_memop(oi);
Richard Henderson59227d52015-05-12 11:51:44 -07001915 unsigned ix = get_mmuidx(oi);
1916
Richard Henderson59c4b7e2015-06-01 14:38:56 -07001917 if (op & ~(MO_AMASK | MO_BSWAP | MO_SSIZE)) {
Richard Hendersonbdfb4602016-06-23 19:15:55 -07001918 col += qemu_log(",$0x%x,%u", op, ix);
Richard Henderson59c4b7e2015-06-01 14:38:56 -07001919 } else {
Sergey Sorokin1f00b272016-06-23 21:16:46 +03001920 const char *s_al, *s_op;
1921 s_al = alignment_name[(op & MO_AMASK) >> MO_ASHIFT];
Richard Henderson59c4b7e2015-06-01 14:38:56 -07001922 s_op = ldst_name[op & (MO_BSWAP | MO_SSIZE)];
Richard Hendersonbdfb4602016-06-23 19:15:55 -07001923 col += qemu_log(",%s%s,%u", s_al, s_op, ix);
Richard Henderson59227d52015-05-12 11:51:44 -07001924 }
1925 i = 1;
Richard Hendersonf713d6a2013-09-04 08:11:05 -07001926 }
Richard Hendersonf713d6a2013-09-04 08:11:05 -07001927 break;
Richard Henderson587195b2021-06-12 21:32:27 -07001928 case INDEX_op_bswap16_i32:
1929 case INDEX_op_bswap16_i64:
1930 case INDEX_op_bswap32_i32:
1931 case INDEX_op_bswap32_i64:
1932 case INDEX_op_bswap64_i64:
1933 {
1934 TCGArg flags = op->args[k];
1935 const char *name = NULL;
1936
1937 if (flags < ARRAY_SIZE(bswap_flag_name)) {
1938 name = bswap_flag_name[flags];
1939 }
1940 if (name) {
1941 col += qemu_log(",%s", name);
1942 } else {
1943 col += qemu_log(",$0x%" TCG_PRIlx, flags);
1944 }
1945 i = k = 1;
1946 }
1947 break;
Richard Hendersonbe210ac2010-01-07 10:13:31 -08001948 default:
blueswir1f48f3ed2008-09-14 07:45:17 +00001949 i = 0;
Richard Hendersonbe210ac2010-01-07 10:13:31 -08001950 break;
1951 }
Richard Henderson51e39722015-02-13 18:51:05 -08001952 switch (c) {
1953 case INDEX_op_set_label:
1954 case INDEX_op_br:
1955 case INDEX_op_brcond_i32:
1956 case INDEX_op_brcond_i64:
1957 case INDEX_op_brcond2_i32:
Richard Hendersonefee3742016-12-08 13:12:08 -08001958 col += qemu_log("%s$L%d", k ? "," : "",
1959 arg_label(op->args[k])->id);
Richard Henderson51e39722015-02-13 18:51:05 -08001960 i++, k++;
1961 break;
1962 default:
1963 break;
1964 }
1965 for (; i < nb_cargs; i++, k++) {
Richard Hendersonefee3742016-12-08 13:12:08 -08001966 col += qemu_log("%s$0x%" TCG_PRIlx, k ? "," : "", op->args[k]);
Richard Hendersonbdfb4602016-06-23 19:15:55 -07001967 }
1968 }
Richard Hendersonbdfb4602016-06-23 19:15:55 -07001969
Richard Henderson1894f692018-11-27 12:46:00 -08001970 if (have_prefs || op->life) {
Robert Foley76064882019-11-18 16:15:27 -05001971
1972 QemuLogFile *logfile;
1973
1974 rcu_read_lock();
Stefan Hajnoczid73415a2020-09-23 11:56:46 +01001975 logfile = qatomic_rcu_read(&qemu_logfile);
Robert Foley76064882019-11-18 16:15:27 -05001976 if (logfile) {
1977 for (; col < 40; ++col) {
1978 putc(' ', logfile->fd);
1979 }
Richard Hendersonbdfb4602016-06-23 19:15:55 -07001980 }
Robert Foley76064882019-11-18 16:15:27 -05001981 rcu_read_unlock();
Richard Henderson1894f692018-11-27 12:46:00 -08001982 }
1983
1984 if (op->life) {
1985 unsigned life = op->life;
Richard Hendersonbdfb4602016-06-23 19:15:55 -07001986
1987 if (life & (SYNC_ARG * 3)) {
1988 qemu_log(" sync:");
1989 for (i = 0; i < 2; ++i) {
1990 if (life & (SYNC_ARG << i)) {
1991 qemu_log(" %d", i);
1992 }
1993 }
1994 }
1995 life /= DEAD_ARG;
1996 if (life) {
1997 qemu_log(" dead:");
1998 for (i = 0; life; ++i, life >>= 1) {
1999 if (life & 1) {
2000 qemu_log(" %d", i);
2001 }
2002 }
bellardb03cce82008-05-10 10:52:05 +00002003 }
bellardc896fe22008-02-01 10:05:41 +00002004 }
Richard Henderson1894f692018-11-27 12:46:00 -08002005
2006 if (have_prefs) {
2007 for (i = 0; i < nb_oargs; ++i) {
2008 TCGRegSet set = op->output_pref[i];
2009
2010 if (i == 0) {
2011 qemu_log(" pref=");
2012 } else {
2013 qemu_log(",");
2014 }
2015 if (set == 0) {
2016 qemu_log("none");
2017 } else if (set == MAKE_64BIT_MASK(0, TCG_TARGET_NB_REGS)) {
2018 qemu_log("all");
2019#ifdef CONFIG_DEBUG_TCG
2020 } else if (tcg_regset_single(set)) {
2021 TCGReg reg = tcg_regset_first(set);
2022 qemu_log("%s", tcg_target_reg_names[reg]);
2023#endif
2024 } else if (TCG_TARGET_NB_REGS <= 32) {
2025 qemu_log("%#x", (uint32_t)set);
2026 } else {
2027 qemu_log("%#" PRIx64, (uint64_t)set);
2028 }
2029 }
2030 }
2031
Blue Swirleeacee42012-06-03 16:35:32 +00002032 qemu_log("\n");
bellardc896fe22008-02-01 10:05:41 +00002033 }
2034}
2035
2036/* we give more priority to constraints with less registers */
2037static int get_constraint_priority(const TCGOpDef *def, int k)
2038{
Richard Henderson74a11792020-09-03 15:56:24 -07002039 const TCGArgConstraint *arg_ct = &def->args_ct[k];
2040 int n;
bellardc896fe22008-02-01 10:05:41 +00002041
Richard Hendersonbc2b17e2019-04-04 19:34:19 -07002042 if (arg_ct->oalias) {
bellardc896fe22008-02-01 10:05:41 +00002043 /* an alias is equivalent to a single register */
2044 n = 1;
2045 } else {
Richard Henderson74a11792020-09-03 15:56:24 -07002046 n = ctpop64(arg_ct->regs);
bellardc896fe22008-02-01 10:05:41 +00002047 }
2048 return TCG_TARGET_NB_REGS - n + 1;
2049}
2050
2051/* sort from highest priority to lowest */
2052static void sort_constraints(TCGOpDef *def, int start, int n)
2053{
Richard Henderson66792f92019-04-04 09:37:38 +07002054 int i, j;
2055 TCGArgConstraint *a = def->args_ct;
bellardc896fe22008-02-01 10:05:41 +00002056
Richard Henderson66792f92019-04-04 09:37:38 +07002057 for (i = 0; i < n; i++) {
2058 a[start + i].sort_index = start + i;
2059 }
2060 if (n <= 1) {
bellardc896fe22008-02-01 10:05:41 +00002061 return;
Richard Henderson66792f92019-04-04 09:37:38 +07002062 }
2063 for (i = 0; i < n - 1; i++) {
2064 for (j = i + 1; j < n; j++) {
2065 int p1 = get_constraint_priority(def, a[start + i].sort_index);
2066 int p2 = get_constraint_priority(def, a[start + j].sort_index);
bellardc896fe22008-02-01 10:05:41 +00002067 if (p1 < p2) {
Richard Henderson66792f92019-04-04 09:37:38 +07002068 int tmp = a[start + i].sort_index;
2069 a[start + i].sort_index = a[start + j].sort_index;
2070 a[start + j].sort_index = tmp;
bellardc896fe22008-02-01 10:05:41 +00002071 }
2072 }
2073 }
2074}
2075
Richard Hendersonf69d2772016-11-18 09:31:40 +01002076static void process_op_defs(TCGContext *s)
bellardc896fe22008-02-01 10:05:41 +00002077{
Richard Hendersona9751602010-03-19 11:12:29 -07002078 TCGOpcode op;
bellardc896fe22008-02-01 10:05:41 +00002079
Richard Hendersonf69d2772016-11-18 09:31:40 +01002080 for (op = 0; op < NB_OPS; op++) {
2081 TCGOpDef *def = &tcg_op_defs[op];
2082 const TCGTargetOpDef *tdefs;
Richard Henderson069ea732016-11-18 11:50:59 +01002083 int i, nb_args;
Richard Hendersonf69d2772016-11-18 09:31:40 +01002084
2085 if (def->flags & TCG_OPF_NOT_PRESENT) {
2086 continue;
2087 }
2088
bellardc896fe22008-02-01 10:05:41 +00002089 nb_args = def->nb_iargs + def->nb_oargs;
Richard Hendersonf69d2772016-11-18 09:31:40 +01002090 if (nb_args == 0) {
2091 continue;
2092 }
2093
Richard Henderson4c22e842020-10-16 22:20:55 -07002094 /*
2095 * Macro magic should make it impossible, but double-check that
2096 * the array index is in range. Since the signness of an enum
2097 * is implementation defined, force the result to unsigned.
2098 */
2099 unsigned con_set = tcg_target_op_def(op);
2100 tcg_debug_assert(con_set < ARRAY_SIZE(constraint_sets));
2101 tdefs = &constraint_sets[con_set];
Richard Hendersonf69d2772016-11-18 09:31:40 +01002102
2103 for (i = 0; i < nb_args; i++) {
2104 const char *ct_str = tdefs->args_ct_str[i];
2105 /* Incomplete TCGTargetOpDef entry. */
Aurelien Jarnoeabb7b92016-04-21 10:48:49 +02002106 tcg_debug_assert(ct_str != NULL);
Richard Hendersonf69d2772016-11-18 09:31:40 +01002107
Richard Henderson17280ff2016-11-18 17:41:24 +01002108 while (*ct_str != '\0') {
2109 switch(*ct_str) {
2110 case '0' ... '9':
2111 {
2112 int oarg = *ct_str - '0';
2113 tcg_debug_assert(ct_str == tdefs->args_ct_str[i]);
2114 tcg_debug_assert(oarg < def->nb_oargs);
Richard Henderson74a11792020-09-03 15:56:24 -07002115 tcg_debug_assert(def->args_ct[oarg].regs != 0);
Richard Henderson17280ff2016-11-18 17:41:24 +01002116 def->args_ct[i] = def->args_ct[oarg];
Richard Hendersonbc2b17e2019-04-04 19:34:19 -07002117 /* The output sets oalias. */
2118 def->args_ct[oarg].oalias = true;
Richard Henderson17280ff2016-11-18 17:41:24 +01002119 def->args_ct[oarg].alias_index = i;
Richard Hendersonbc2b17e2019-04-04 19:34:19 -07002120 /* The input sets ialias. */
2121 def->args_ct[i].ialias = true;
Richard Henderson17280ff2016-11-18 17:41:24 +01002122 def->args_ct[i].alias_index = oarg;
bellardc896fe22008-02-01 10:05:41 +00002123 }
Richard Henderson17280ff2016-11-18 17:41:24 +01002124 ct_str++;
2125 break;
2126 case '&':
Richard Hendersonbc2b17e2019-04-04 19:34:19 -07002127 def->args_ct[i].newreg = true;
Richard Henderson17280ff2016-11-18 17:41:24 +01002128 ct_str++;
2129 break;
2130 case 'i':
2131 def->args_ct[i].ct |= TCG_CT_CONST;
2132 ct_str++;
2133 break;
Richard Henderson358b4922020-10-16 15:27:46 -07002134
Richard Henderson358b4922020-10-16 15:27:46 -07002135 /* Include all of the target-specific constraints. */
2136
2137#undef CONST
2138#define CONST(CASE, MASK) \
2139 case CASE: def->args_ct[i].ct |= MASK; ct_str++; break;
2140#define REGS(CASE, MASK) \
2141 case CASE: def->args_ct[i].regs |= MASK; ct_str++; break;
2142
2143#include "tcg-target-con-str.h"
2144
2145#undef REGS
2146#undef CONST
Richard Henderson17280ff2016-11-18 17:41:24 +01002147 default:
Richard Henderson17280ff2016-11-18 17:41:24 +01002148 /* Typo in TCGTargetOpDef constraint. */
Richard Henderson358b4922020-10-16 15:27:46 -07002149 g_assert_not_reached();
bellardc896fe22008-02-01 10:05:41 +00002150 }
2151 }
2152 }
2153
Stefan Weilc68aaa12010-02-15 17:17:21 +01002154 /* TCGTargetOpDef entry with too much information? */
Aurelien Jarnoeabb7b92016-04-21 10:48:49 +02002155 tcg_debug_assert(i == TCG_MAX_OP_ARGS || tdefs->args_ct_str[i] == NULL);
Stefan Weilc68aaa12010-02-15 17:17:21 +01002156
bellardc896fe22008-02-01 10:05:41 +00002157 /* sort the constraints (XXX: this is just an heuristic) */
2158 sort_constraints(def, 0, def->nb_oargs);
2159 sort_constraints(def, def->nb_oargs, def->nb_iargs);
bellardc896fe22008-02-01 10:05:41 +00002160 }
bellardc896fe22008-02-01 10:05:41 +00002161}
2162
Richard Henderson0c627cd2014-03-30 16:51:54 -07002163void tcg_op_remove(TCGContext *s, TCGOp *op)
2164{
Richard Hendersond88a1172018-11-26 12:47:28 -08002165 TCGLabel *label;
2166
2167 switch (op->opc) {
2168 case INDEX_op_br:
2169 label = arg_label(op->args[0]);
2170 label->refs--;
2171 break;
2172 case INDEX_op_brcond_i32:
2173 case INDEX_op_brcond_i64:
2174 label = arg_label(op->args[3]);
2175 label->refs--;
2176 break;
2177 case INDEX_op_brcond2_i32:
2178 label = arg_label(op->args[5]);
2179 label->refs--;
2180 break;
2181 default:
2182 break;
2183 }
2184
Richard Henderson15fa08f2017-11-02 15:19:14 +01002185 QTAILQ_REMOVE(&s->ops, op, link);
2186 QTAILQ_INSERT_TAIL(&s->free_ops, op, link);
Richard Hendersonabebf922018-05-08 19:18:59 +00002187 s->nb_ops--;
Richard Henderson0c627cd2014-03-30 16:51:54 -07002188
2189#ifdef CONFIG_PROFILER
Stefan Hajnoczid73415a2020-09-23 11:56:46 +01002190 qatomic_set(&s->prof.del_op_count, s->prof.del_op_count + 1);
Richard Henderson0c627cd2014-03-30 16:51:54 -07002191#endif
2192}
2193
Richard Hendersona80cdd32021-06-04 14:26:45 -07002194void tcg_remove_ops_after(TCGOp *op)
2195{
2196 TCGContext *s = tcg_ctx;
2197
2198 while (true) {
2199 TCGOp *last = tcg_last_op();
2200 if (last == op) {
2201 return;
2202 }
2203 tcg_op_remove(s, last);
2204 }
2205}
2206
Richard Henderson15fa08f2017-11-02 15:19:14 +01002207static TCGOp *tcg_op_alloc(TCGOpcode opc)
2208{
2209 TCGContext *s = tcg_ctx;
2210 TCGOp *op;
2211
2212 if (likely(QTAILQ_EMPTY(&s->free_ops))) {
2213 op = tcg_malloc(sizeof(TCGOp));
2214 } else {
2215 op = QTAILQ_FIRST(&s->free_ops);
2216 QTAILQ_REMOVE(&s->free_ops, op, link);
2217 }
2218 memset(op, 0, offsetof(TCGOp, link));
2219 op->opc = opc;
Richard Hendersonabebf922018-05-08 19:18:59 +00002220 s->nb_ops++;
Richard Henderson15fa08f2017-11-02 15:19:14 +01002221
2222 return op;
2223}
2224
2225TCGOp *tcg_emit_op(TCGOpcode opc)
2226{
2227 TCGOp *op = tcg_op_alloc(opc);
2228 QTAILQ_INSERT_TAIL(&tcg_ctx->ops, op, link);
2229 return op;
2230}
2231
Emilio G. Cotaac1043f2018-12-09 14:37:19 -05002232TCGOp *tcg_op_insert_before(TCGContext *s, TCGOp *old_op, TCGOpcode opc)
Richard Henderson5a184072016-06-23 20:34:33 -07002233{
Richard Henderson15fa08f2017-11-02 15:19:14 +01002234 TCGOp *new_op = tcg_op_alloc(opc);
2235 QTAILQ_INSERT_BEFORE(old_op, new_op, link);
Richard Henderson5a184072016-06-23 20:34:33 -07002236 return new_op;
2237}
2238
Emilio G. Cotaac1043f2018-12-09 14:37:19 -05002239TCGOp *tcg_op_insert_after(TCGContext *s, TCGOp *old_op, TCGOpcode opc)
Richard Henderson5a184072016-06-23 20:34:33 -07002240{
Richard Henderson15fa08f2017-11-02 15:19:14 +01002241 TCGOp *new_op = tcg_op_alloc(opc);
2242 QTAILQ_INSERT_AFTER(&s->ops, old_op, new_op, link);
Richard Henderson5a184072016-06-23 20:34:33 -07002243 return new_op;
2244}
2245
Richard Hendersonb4fc67c2018-11-26 14:28:28 -08002246/* Reachable analysis : remove unreachable code. */
2247static void reachable_code_pass(TCGContext *s)
2248{
2249 TCGOp *op, *op_next;
2250 bool dead = false;
2251
2252 QTAILQ_FOREACH_SAFE(op, &s->ops, link, op_next) {
2253 bool remove = dead;
2254 TCGLabel *label;
Richard Hendersonb4fc67c2018-11-26 14:28:28 -08002255
2256 switch (op->opc) {
2257 case INDEX_op_set_label:
2258 label = arg_label(op->args[0]);
2259 if (label->refs == 0) {
2260 /*
2261 * While there is an occasional backward branch, virtually
2262 * all branches generated by the translators are forward.
2263 * Which means that generally we will have already removed
2264 * all references to the label that will be, and there is
2265 * little to be gained by iterating.
2266 */
2267 remove = true;
2268 } else {
2269 /* Once we see a label, insns become live again. */
2270 dead = false;
2271 remove = false;
2272
2273 /*
2274 * Optimization can fold conditional branches to unconditional.
2275 * If we find a label with one reference which is preceded by
2276 * an unconditional branch to it, remove both. This needed to
2277 * wait until the dead code in between them was removed.
2278 */
2279 if (label->refs == 1) {
Paolo Bonzinieae3eb32018-12-06 13:10:34 +01002280 TCGOp *op_prev = QTAILQ_PREV(op, link);
Richard Hendersonb4fc67c2018-11-26 14:28:28 -08002281 if (op_prev->opc == INDEX_op_br &&
2282 label == arg_label(op_prev->args[0])) {
2283 tcg_op_remove(s, op_prev);
2284 remove = true;
2285 }
2286 }
2287 }
2288 break;
2289
2290 case INDEX_op_br:
2291 case INDEX_op_exit_tb:
2292 case INDEX_op_goto_ptr:
2293 /* Unconditional branches; everything following is dead. */
2294 dead = true;
2295 break;
2296
2297 case INDEX_op_call:
2298 /* Notice noreturn helper calls, raising exceptions. */
Richard Henderson90163902021-03-18 10:21:45 -06002299 if (tcg_call_flags(op) & TCG_CALL_NO_RETURN) {
Richard Hendersonb4fc67c2018-11-26 14:28:28 -08002300 dead = true;
2301 }
2302 break;
2303
2304 case INDEX_op_insn_start:
2305 /* Never remove -- we need to keep these for unwind. */
2306 remove = false;
2307 break;
2308
2309 default:
2310 break;
2311 }
2312
2313 if (remove) {
2314 tcg_op_remove(s, op);
2315 }
2316 }
2317}
2318
Richard Hendersonc70fbf02016-06-23 20:34:22 -07002319#define TS_DEAD 1
2320#define TS_MEM 2
2321
Richard Henderson5a184072016-06-23 20:34:33 -07002322#define IS_DEAD_ARG(n) (arg_life & (DEAD_ARG << (n)))
2323#define NEED_SYNC_ARG(n) (arg_life & (SYNC_ARG << (n)))
2324
Richard Henderson25f49c52018-11-27 12:45:26 -08002325/* For liveness_pass_1, the register preferences for a given temp. */
2326static inline TCGRegSet *la_temp_pref(TCGTemp *ts)
2327{
2328 return ts->state_ptr;
2329}
2330
2331/* For liveness_pass_1, reset the preferences for a given temp to the
2332 * maximal regset for its type.
2333 */
2334static inline void la_reset_pref(TCGTemp *ts)
2335{
2336 *la_temp_pref(ts)
2337 = (ts->state == TS_DEAD ? 0 : tcg_target_available_regs[ts->type]);
2338}
2339
Aurelien Jarno9c43b682012-10-09 21:53:07 +02002340/* liveness analysis: end of function: all temps are dead, and globals
2341 should be in memory. */
Richard Henderson2616c802018-11-27 13:37:24 -08002342static void la_func_end(TCGContext *s, int ng, int nt)
bellardc896fe22008-02-01 10:05:41 +00002343{
Richard Hendersonb83eabe2016-11-01 15:56:04 -06002344 int i;
2345
2346 for (i = 0; i < ng; ++i) {
2347 s->temps[i].state = TS_DEAD | TS_MEM;
Richard Henderson25f49c52018-11-27 12:45:26 -08002348 la_reset_pref(&s->temps[i]);
Richard Hendersonb83eabe2016-11-01 15:56:04 -06002349 }
2350 for (i = ng; i < nt; ++i) {
2351 s->temps[i].state = TS_DEAD;
Richard Henderson25f49c52018-11-27 12:45:26 -08002352 la_reset_pref(&s->temps[i]);
Richard Hendersonb83eabe2016-11-01 15:56:04 -06002353 }
bellardc896fe22008-02-01 10:05:41 +00002354}
2355
Aurelien Jarno9c43b682012-10-09 21:53:07 +02002356/* liveness analysis: end of basic block: all temps are dead, globals
2357 and local temps should be in memory. */
Richard Henderson2616c802018-11-27 13:37:24 -08002358static void la_bb_end(TCGContext *s, int ng, int nt)
bellard641d5fb2008-05-25 17:24:00 +00002359{
Richard Hendersonb83eabe2016-11-01 15:56:04 -06002360 int i;
bellard641d5fb2008-05-25 17:24:00 +00002361
Richard Hendersonee17db82020-03-29 10:11:56 -07002362 for (i = 0; i < nt; ++i) {
2363 TCGTemp *ts = &s->temps[i];
2364 int state;
2365
2366 switch (ts->kind) {
2367 case TEMP_FIXED:
2368 case TEMP_GLOBAL:
2369 case TEMP_LOCAL:
2370 state = TS_DEAD | TS_MEM;
2371 break;
2372 case TEMP_NORMAL:
Richard Hendersonc0522132020-03-29 18:55:52 -07002373 case TEMP_CONST:
Richard Hendersonee17db82020-03-29 10:11:56 -07002374 state = TS_DEAD;
2375 break;
2376 default:
2377 g_assert_not_reached();
2378 }
2379 ts->state = state;
2380 la_reset_pref(ts);
bellard641d5fb2008-05-25 17:24:00 +00002381 }
2382}
2383
Richard Hendersonf65a0612018-11-27 14:00:35 -08002384/* liveness analysis: sync globals back to memory. */
2385static void la_global_sync(TCGContext *s, int ng)
2386{
2387 int i;
2388
2389 for (i = 0; i < ng; ++i) {
Richard Henderson25f49c52018-11-27 12:45:26 -08002390 int state = s->temps[i].state;
2391 s->temps[i].state = state | TS_MEM;
2392 if (state == TS_DEAD) {
2393 /* If the global was previously dead, reset prefs. */
2394 la_reset_pref(&s->temps[i]);
2395 }
Richard Hendersonf65a0612018-11-27 14:00:35 -08002396 }
2397}
2398
Richard Hendersonb4cb76e2020-10-08 15:21:43 -05002399/*
2400 * liveness analysis: conditional branch: all temps are dead,
2401 * globals and local temps should be synced.
2402 */
2403static void la_bb_sync(TCGContext *s, int ng, int nt)
2404{
2405 la_global_sync(s, ng);
2406
2407 for (int i = ng; i < nt; ++i) {
Richard Hendersonc0522132020-03-29 18:55:52 -07002408 TCGTemp *ts = &s->temps[i];
2409 int state;
2410
2411 switch (ts->kind) {
2412 case TEMP_LOCAL:
2413 state = ts->state;
2414 ts->state = state | TS_MEM;
Richard Hendersonb4cb76e2020-10-08 15:21:43 -05002415 if (state != TS_DEAD) {
2416 continue;
2417 }
Richard Hendersonc0522132020-03-29 18:55:52 -07002418 break;
2419 case TEMP_NORMAL:
Richard Hendersonb4cb76e2020-10-08 15:21:43 -05002420 s->temps[i].state = TS_DEAD;
Richard Hendersonc0522132020-03-29 18:55:52 -07002421 break;
2422 case TEMP_CONST:
2423 continue;
2424 default:
2425 g_assert_not_reached();
Richard Hendersonb4cb76e2020-10-08 15:21:43 -05002426 }
2427 la_reset_pref(&s->temps[i]);
2428 }
2429}
2430
Richard Hendersonf65a0612018-11-27 14:00:35 -08002431/* liveness analysis: sync globals back to memory and kill. */
2432static void la_global_kill(TCGContext *s, int ng)
2433{
2434 int i;
2435
2436 for (i = 0; i < ng; i++) {
2437 s->temps[i].state = TS_DEAD | TS_MEM;
Richard Henderson25f49c52018-11-27 12:45:26 -08002438 la_reset_pref(&s->temps[i]);
2439 }
2440}
2441
2442/* liveness analysis: note live globals crossing calls. */
2443static void la_cross_call(TCGContext *s, int nt)
2444{
2445 TCGRegSet mask = ~tcg_target_call_clobber_regs;
2446 int i;
2447
2448 for (i = 0; i < nt; i++) {
2449 TCGTemp *ts = &s->temps[i];
2450 if (!(ts->state & TS_DEAD)) {
2451 TCGRegSet *pset = la_temp_pref(ts);
2452 TCGRegSet set = *pset;
2453
2454 set &= mask;
2455 /* If the combination is not possible, restart. */
2456 if (set == 0) {
2457 set = tcg_target_available_regs[ts->type] & mask;
2458 }
2459 *pset = set;
2460 }
Richard Hendersonf65a0612018-11-27 14:00:35 -08002461 }
2462}
2463
Richard Hendersona1b3c482016-06-22 15:46:09 -07002464/* Liveness analysis : update the opc_arg_life array to tell if a
bellardc896fe22008-02-01 10:05:41 +00002465 given input arguments is dead. Instructions updating dead
2466 temporaries are removed. */
Richard Hendersonb83eabe2016-11-01 15:56:04 -06002467static void liveness_pass_1(TCGContext *s)
bellardc896fe22008-02-01 10:05:41 +00002468{
Richard Hendersonc70fbf02016-06-23 20:34:22 -07002469 int nb_globals = s->nb_globals;
Richard Henderson2616c802018-11-27 13:37:24 -08002470 int nb_temps = s->nb_temps;
Richard Henderson15fa08f2017-11-02 15:19:14 +01002471 TCGOp *op, *op_prev;
Richard Henderson25f49c52018-11-27 12:45:26 -08002472 TCGRegSet *prefs;
2473 int i;
2474
2475 prefs = tcg_malloc(sizeof(TCGRegSet) * nb_temps);
2476 for (i = 0; i < nb_temps; ++i) {
2477 s->temps[i].state_ptr = prefs + i;
2478 }
Richard Hendersona1b3c482016-06-22 15:46:09 -07002479
Richard Hendersonae36a242018-11-27 13:45:08 -08002480 /* ??? Should be redundant with the exit_tb that ends the TB. */
Richard Henderson2616c802018-11-27 13:37:24 -08002481 la_func_end(s, nb_globals, nb_temps);
bellardc896fe22008-02-01 10:05:41 +00002482
Paolo Bonzinieae3eb32018-12-06 13:10:34 +01002483 QTAILQ_FOREACH_REVERSE_SAFE(op, &s->ops, link, op_prev) {
Richard Henderson25f49c52018-11-27 12:45:26 -08002484 int nb_iargs, nb_oargs;
Richard Hendersonc45cb8b2014-09-19 13:49:15 -07002485 TCGOpcode opc_new, opc_new2;
2486 bool have_opc_new2;
Richard Hendersona1b3c482016-06-22 15:46:09 -07002487 TCGLifeData arg_life = 0;
Richard Henderson25f49c52018-11-27 12:45:26 -08002488 TCGTemp *ts;
Richard Hendersonc45cb8b2014-09-19 13:49:15 -07002489 TCGOpcode opc = op->opc;
2490 const TCGOpDef *def = &tcg_op_defs[opc];
2491
Richard Hendersonc45cb8b2014-09-19 13:49:15 -07002492 switch (opc) {
bellardc896fe22008-02-01 10:05:41 +00002493 case INDEX_op_call:
bellardc6e113f2008-05-17 12:42:15 +00002494 {
2495 int call_flags;
Richard Henderson25f49c52018-11-27 12:45:26 -08002496 int nb_call_regs;
bellardc896fe22008-02-01 10:05:41 +00002497
Richard Hendersoncd9090a2017-11-14 13:02:51 +01002498 nb_oargs = TCGOP_CALLO(op);
2499 nb_iargs = TCGOP_CALLI(op);
Richard Henderson90163902021-03-18 10:21:45 -06002500 call_flags = tcg_call_flags(op);
bellardc896fe22008-02-01 10:05:41 +00002501
Richard Hendersonc45cb8b2014-09-19 13:49:15 -07002502 /* pure functions can be removed if their result is unused */
Aurelien Jarno78505272012-10-09 21:53:08 +02002503 if (call_flags & TCG_CALL_NO_SIDE_EFFECTS) {
Richard Hendersoncf066672014-03-22 20:06:52 -07002504 for (i = 0; i < nb_oargs; i++) {
Richard Henderson25f49c52018-11-27 12:45:26 -08002505 ts = arg_temp(op->args[i]);
2506 if (ts->state != TS_DEAD) {
bellardc6e113f2008-05-17 12:42:15 +00002507 goto do_not_remove_call;
Aurelien Jarno9c43b682012-10-09 21:53:07 +02002508 }
bellardc6e113f2008-05-17 12:42:15 +00002509 }
Richard Hendersonc45cb8b2014-09-19 13:49:15 -07002510 goto do_remove;
Richard Henderson152c35a2018-11-27 13:32:33 -08002511 }
2512 do_not_remove_call:
bellardc6e113f2008-05-17 12:42:15 +00002513
Richard Henderson25f49c52018-11-27 12:45:26 -08002514 /* Output args are dead. */
Richard Henderson152c35a2018-11-27 13:32:33 -08002515 for (i = 0; i < nb_oargs; i++) {
Richard Henderson25f49c52018-11-27 12:45:26 -08002516 ts = arg_temp(op->args[i]);
2517 if (ts->state & TS_DEAD) {
Richard Henderson152c35a2018-11-27 13:32:33 -08002518 arg_life |= DEAD_ARG << i;
bellardc6e113f2008-05-17 12:42:15 +00002519 }
Richard Henderson25f49c52018-11-27 12:45:26 -08002520 if (ts->state & TS_MEM) {
Richard Henderson152c35a2018-11-27 13:32:33 -08002521 arg_life |= SYNC_ARG << i;
2522 }
Richard Henderson25f49c52018-11-27 12:45:26 -08002523 ts->state = TS_DEAD;
2524 la_reset_pref(ts);
2525
2526 /* Not used -- it will be tcg_target_call_oarg_regs[i]. */
2527 op->output_pref[i] = 0;
Richard Henderson152c35a2018-11-27 13:32:33 -08002528 }
Aurelien Jarno78505272012-10-09 21:53:08 +02002529
Richard Henderson152c35a2018-11-27 13:32:33 -08002530 if (!(call_flags & (TCG_CALL_NO_WRITE_GLOBALS |
2531 TCG_CALL_NO_READ_GLOBALS))) {
Richard Hendersonf65a0612018-11-27 14:00:35 -08002532 la_global_kill(s, nb_globals);
Richard Henderson152c35a2018-11-27 13:32:33 -08002533 } else if (!(call_flags & TCG_CALL_NO_READ_GLOBALS)) {
Richard Hendersonf65a0612018-11-27 14:00:35 -08002534 la_global_sync(s, nb_globals);
Richard Henderson152c35a2018-11-27 13:32:33 -08002535 }
aurel32b9c18f52009-04-06 12:33:59 +00002536
Richard Henderson25f49c52018-11-27 12:45:26 -08002537 /* Record arguments that die in this helper. */
Richard Henderson152c35a2018-11-27 13:32:33 -08002538 for (i = nb_oargs; i < nb_iargs + nb_oargs; i++) {
Richard Henderson25f49c52018-11-27 12:45:26 -08002539 ts = arg_temp(op->args[i]);
2540 if (ts && ts->state & TS_DEAD) {
Richard Henderson152c35a2018-11-27 13:32:33 -08002541 arg_life |= DEAD_ARG << i;
bellardc6e113f2008-05-17 12:42:15 +00002542 }
Richard Henderson152c35a2018-11-27 13:32:33 -08002543 }
Richard Henderson25f49c52018-11-27 12:45:26 -08002544
2545 /* For all live registers, remove call-clobbered prefs. */
2546 la_cross_call(s, nb_temps);
2547
2548 nb_call_regs = ARRAY_SIZE(tcg_target_call_iarg_regs);
2549
2550 /* Input arguments are live for preceding opcodes. */
2551 for (i = 0; i < nb_iargs; i++) {
2552 ts = arg_temp(op->args[i + nb_oargs]);
2553 if (ts && ts->state & TS_DEAD) {
2554 /* For those arguments that die, and will be allocated
2555 * in registers, clear the register set for that arg,
2556 * to be filled in below. For args that will be on
2557 * the stack, reset to any available reg.
2558 */
2559 *la_temp_pref(ts)
2560 = (i < nb_call_regs ? 0 :
2561 tcg_target_available_regs[ts->type]);
2562 ts->state &= ~TS_DEAD;
2563 }
2564 }
2565
2566 /* For each input argument, add its input register to prefs.
2567 If a temp is used once, this produces a single set bit. */
2568 for (i = 0; i < MIN(nb_call_regs, nb_iargs); i++) {
2569 ts = arg_temp(op->args[i + nb_oargs]);
2570 if (ts) {
2571 tcg_regset_set_reg(*la_temp_pref(ts),
2572 tcg_target_call_iarg_regs[i]);
Aurelien Jarnoc19f47b2015-06-04 21:47:08 +02002573 }
bellardc896fe22008-02-01 10:05:41 +00002574 }
bellardc896fe22008-02-01 10:05:41 +00002575 }
bellardc896fe22008-02-01 10:05:41 +00002576 break;
Richard Henderson765b8422015-08-29 12:37:33 -07002577 case INDEX_op_insn_start:
bellardc896fe22008-02-01 10:05:41 +00002578 break;
bellard5ff9d6a2008-02-04 00:37:54 +00002579 case INDEX_op_discard:
bellard5ff9d6a2008-02-04 00:37:54 +00002580 /* mark the temporary as dead */
Richard Henderson25f49c52018-11-27 12:45:26 -08002581 ts = arg_temp(op->args[0]);
2582 ts->state = TS_DEAD;
2583 la_reset_pref(ts);
bellard5ff9d6a2008-02-04 00:37:54 +00002584 break;
Richard Henderson1305c452012-10-02 11:32:29 -07002585
2586 case INDEX_op_add2_i32:
Richard Hendersonc45cb8b2014-09-19 13:49:15 -07002587 opc_new = INDEX_op_add_i32;
Richard Hendersonf1fae402013-02-19 23:52:02 -08002588 goto do_addsub2;
Richard Henderson1305c452012-10-02 11:32:29 -07002589 case INDEX_op_sub2_i32:
Richard Hendersonc45cb8b2014-09-19 13:49:15 -07002590 opc_new = INDEX_op_sub_i32;
Richard Hendersonf1fae402013-02-19 23:52:02 -08002591 goto do_addsub2;
2592 case INDEX_op_add2_i64:
Richard Hendersonc45cb8b2014-09-19 13:49:15 -07002593 opc_new = INDEX_op_add_i64;
Richard Hendersonf1fae402013-02-19 23:52:02 -08002594 goto do_addsub2;
2595 case INDEX_op_sub2_i64:
Richard Hendersonc45cb8b2014-09-19 13:49:15 -07002596 opc_new = INDEX_op_sub_i64;
Richard Hendersonf1fae402013-02-19 23:52:02 -08002597 do_addsub2:
Richard Henderson1305c452012-10-02 11:32:29 -07002598 nb_iargs = 4;
2599 nb_oargs = 2;
2600 /* Test if the high part of the operation is dead, but not
2601 the low part. The result can be optimized to a simple
2602 add or sub. This happens often for x86_64 guest when the
2603 cpu mode is set to 32 bit. */
Richard Hendersonb83eabe2016-11-01 15:56:04 -06002604 if (arg_temp(op->args[1])->state == TS_DEAD) {
2605 if (arg_temp(op->args[0])->state == TS_DEAD) {
Richard Henderson1305c452012-10-02 11:32:29 -07002606 goto do_remove;
2607 }
Richard Hendersonc45cb8b2014-09-19 13:49:15 -07002608 /* Replace the opcode and adjust the args in place,
2609 leaving 3 unused args at the end. */
2610 op->opc = opc = opc_new;
Richard Hendersonefee3742016-12-08 13:12:08 -08002611 op->args[1] = op->args[2];
2612 op->args[2] = op->args[4];
Richard Henderson1305c452012-10-02 11:32:29 -07002613 /* Fall through and mark the single-word operation live. */
2614 nb_iargs = 2;
2615 nb_oargs = 1;
2616 }
2617 goto do_not_remove;
2618
Richard Henderson14149682012-10-02 11:32:30 -07002619 case INDEX_op_mulu2_i32:
Richard Hendersonc45cb8b2014-09-19 13:49:15 -07002620 opc_new = INDEX_op_mul_i32;
2621 opc_new2 = INDEX_op_muluh_i32;
2622 have_opc_new2 = TCG_TARGET_HAS_muluh_i32;
Richard Henderson03271522013-08-14 14:35:56 -07002623 goto do_mul2;
Richard Hendersonf1fae402013-02-19 23:52:02 -08002624 case INDEX_op_muls2_i32:
Richard Hendersonc45cb8b2014-09-19 13:49:15 -07002625 opc_new = INDEX_op_mul_i32;
2626 opc_new2 = INDEX_op_mulsh_i32;
2627 have_opc_new2 = TCG_TARGET_HAS_mulsh_i32;
Richard Hendersonf1fae402013-02-19 23:52:02 -08002628 goto do_mul2;
2629 case INDEX_op_mulu2_i64:
Richard Hendersonc45cb8b2014-09-19 13:49:15 -07002630 opc_new = INDEX_op_mul_i64;
2631 opc_new2 = INDEX_op_muluh_i64;
2632 have_opc_new2 = TCG_TARGET_HAS_muluh_i64;
Richard Henderson03271522013-08-14 14:35:56 -07002633 goto do_mul2;
Richard Hendersonf1fae402013-02-19 23:52:02 -08002634 case INDEX_op_muls2_i64:
Richard Hendersonc45cb8b2014-09-19 13:49:15 -07002635 opc_new = INDEX_op_mul_i64;
2636 opc_new2 = INDEX_op_mulsh_i64;
2637 have_opc_new2 = TCG_TARGET_HAS_mulsh_i64;
Richard Henderson03271522013-08-14 14:35:56 -07002638 goto do_mul2;
Richard Hendersonf1fae402013-02-19 23:52:02 -08002639 do_mul2:
Richard Henderson14149682012-10-02 11:32:30 -07002640 nb_iargs = 2;
2641 nb_oargs = 2;
Richard Hendersonb83eabe2016-11-01 15:56:04 -06002642 if (arg_temp(op->args[1])->state == TS_DEAD) {
2643 if (arg_temp(op->args[0])->state == TS_DEAD) {
Richard Henderson03271522013-08-14 14:35:56 -07002644 /* Both parts of the operation are dead. */
Richard Henderson14149682012-10-02 11:32:30 -07002645 goto do_remove;
2646 }
Richard Henderson03271522013-08-14 14:35:56 -07002647 /* The high part of the operation is dead; generate the low. */
Richard Hendersonc45cb8b2014-09-19 13:49:15 -07002648 op->opc = opc = opc_new;
Richard Hendersonefee3742016-12-08 13:12:08 -08002649 op->args[1] = op->args[2];
2650 op->args[2] = op->args[3];
Richard Hendersonb83eabe2016-11-01 15:56:04 -06002651 } else if (arg_temp(op->args[0])->state == TS_DEAD && have_opc_new2) {
Richard Hendersonc45cb8b2014-09-19 13:49:15 -07002652 /* The low part of the operation is dead; generate the high. */
2653 op->opc = opc = opc_new2;
Richard Hendersonefee3742016-12-08 13:12:08 -08002654 op->args[0] = op->args[1];
2655 op->args[1] = op->args[2];
2656 op->args[2] = op->args[3];
Richard Henderson03271522013-08-14 14:35:56 -07002657 } else {
2658 goto do_not_remove;
Richard Henderson14149682012-10-02 11:32:30 -07002659 }
Richard Henderson03271522013-08-14 14:35:56 -07002660 /* Mark the single-word operation live. */
2661 nb_oargs = 1;
Richard Henderson14149682012-10-02 11:32:30 -07002662 goto do_not_remove;
2663
bellardc896fe22008-02-01 10:05:41 +00002664 default:
Richard Henderson1305c452012-10-02 11:32:29 -07002665 /* XXX: optimize by hardcoding common cases (e.g. triadic ops) */
aurel3249516bc2008-12-07 18:15:45 +00002666 nb_iargs = def->nb_iargs;
2667 nb_oargs = def->nb_oargs;
bellardc896fe22008-02-01 10:05:41 +00002668
aurel3249516bc2008-12-07 18:15:45 +00002669 /* Test if the operation can be removed because all
2670 its outputs are dead. We assume that nb_oargs == 0
2671 implies side effects */
2672 if (!(def->flags & TCG_OPF_SIDE_EFFECTS) && nb_oargs != 0) {
Richard Hendersonc45cb8b2014-09-19 13:49:15 -07002673 for (i = 0; i < nb_oargs; i++) {
Richard Hendersonb83eabe2016-11-01 15:56:04 -06002674 if (arg_temp(op->args[i])->state != TS_DEAD) {
aurel3249516bc2008-12-07 18:15:45 +00002675 goto do_not_remove;
Aurelien Jarno9c43b682012-10-09 21:53:07 +02002676 }
bellardc896fe22008-02-01 10:05:41 +00002677 }
Richard Henderson152c35a2018-11-27 13:32:33 -08002678 goto do_remove;
2679 }
2680 goto do_not_remove;
aurel3249516bc2008-12-07 18:15:45 +00002681
Richard Henderson152c35a2018-11-27 13:32:33 -08002682 do_remove:
2683 tcg_op_remove(s, op);
2684 break;
aurel3249516bc2008-12-07 18:15:45 +00002685
Richard Henderson152c35a2018-11-27 13:32:33 -08002686 do_not_remove:
Richard Henderson152c35a2018-11-27 13:32:33 -08002687 for (i = 0; i < nb_oargs; i++) {
Richard Henderson25f49c52018-11-27 12:45:26 -08002688 ts = arg_temp(op->args[i]);
2689
2690 /* Remember the preference of the uses that followed. */
2691 op->output_pref[i] = *la_temp_pref(ts);
2692
2693 /* Output args are dead. */
2694 if (ts->state & TS_DEAD) {
Richard Henderson152c35a2018-11-27 13:32:33 -08002695 arg_life |= DEAD_ARG << i;
Aurelien Jarnoc19f47b2015-06-04 21:47:08 +02002696 }
Richard Henderson25f49c52018-11-27 12:45:26 -08002697 if (ts->state & TS_MEM) {
Richard Henderson152c35a2018-11-27 13:32:33 -08002698 arg_life |= SYNC_ARG << i;
aurel3249516bc2008-12-07 18:15:45 +00002699 }
Richard Henderson25f49c52018-11-27 12:45:26 -08002700 ts->state = TS_DEAD;
2701 la_reset_pref(ts);
Richard Henderson152c35a2018-11-27 13:32:33 -08002702 }
2703
Richard Henderson25f49c52018-11-27 12:45:26 -08002704 /* If end of basic block, update. */
Richard Hendersonae36a242018-11-27 13:45:08 -08002705 if (def->flags & TCG_OPF_BB_EXIT) {
2706 la_func_end(s, nb_globals, nb_temps);
Richard Hendersonb4cb76e2020-10-08 15:21:43 -05002707 } else if (def->flags & TCG_OPF_COND_BRANCH) {
2708 la_bb_sync(s, nb_globals, nb_temps);
Richard Hendersonae36a242018-11-27 13:45:08 -08002709 } else if (def->flags & TCG_OPF_BB_END) {
Richard Henderson2616c802018-11-27 13:37:24 -08002710 la_bb_end(s, nb_globals, nb_temps);
Richard Henderson152c35a2018-11-27 13:32:33 -08002711 } else if (def->flags & TCG_OPF_SIDE_EFFECTS) {
Richard Hendersonf65a0612018-11-27 14:00:35 -08002712 la_global_sync(s, nb_globals);
Richard Henderson25f49c52018-11-27 12:45:26 -08002713 if (def->flags & TCG_OPF_CALL_CLOBBER) {
2714 la_cross_call(s, nb_temps);
2715 }
Richard Henderson152c35a2018-11-27 13:32:33 -08002716 }
2717
Richard Henderson25f49c52018-11-27 12:45:26 -08002718 /* Record arguments that die in this opcode. */
Richard Henderson152c35a2018-11-27 13:32:33 -08002719 for (i = nb_oargs; i < nb_oargs + nb_iargs; i++) {
Richard Henderson25f49c52018-11-27 12:45:26 -08002720 ts = arg_temp(op->args[i]);
2721 if (ts->state & TS_DEAD) {
Richard Henderson152c35a2018-11-27 13:32:33 -08002722 arg_life |= DEAD_ARG << i;
2723 }
2724 }
Richard Henderson25f49c52018-11-27 12:45:26 -08002725
2726 /* Input arguments are live for preceding opcodes. */
Richard Henderson152c35a2018-11-27 13:32:33 -08002727 for (i = nb_oargs; i < nb_oargs + nb_iargs; i++) {
Richard Henderson25f49c52018-11-27 12:45:26 -08002728 ts = arg_temp(op->args[i]);
2729 if (ts->state & TS_DEAD) {
2730 /* For operands that were dead, initially allow
2731 all regs for the type. */
2732 *la_temp_pref(ts) = tcg_target_available_regs[ts->type];
2733 ts->state &= ~TS_DEAD;
2734 }
2735 }
2736
2737 /* Incorporate constraints for this operand. */
2738 switch (opc) {
2739 case INDEX_op_mov_i32:
2740 case INDEX_op_mov_i64:
2741 /* Note that these are TCG_OPF_NOT_PRESENT and do not
2742 have proper constraints. That said, special case
2743 moves to propagate preferences backward. */
2744 if (IS_DEAD_ARG(1)) {
2745 *la_temp_pref(arg_temp(op->args[0]))
2746 = *la_temp_pref(arg_temp(op->args[1]));
2747 }
2748 break;
2749
2750 default:
2751 for (i = nb_oargs; i < nb_oargs + nb_iargs; i++) {
2752 const TCGArgConstraint *ct = &def->args_ct[i];
2753 TCGRegSet set, *pset;
2754
2755 ts = arg_temp(op->args[i]);
2756 pset = la_temp_pref(ts);
2757 set = *pset;
2758
Richard Henderson9be0d082020-09-03 15:19:03 -07002759 set &= ct->regs;
Richard Hendersonbc2b17e2019-04-04 19:34:19 -07002760 if (ct->ialias) {
Richard Henderson25f49c52018-11-27 12:45:26 -08002761 set &= op->output_pref[ct->alias_index];
2762 }
2763 /* If the combination is not possible, restart. */
2764 if (set == 0) {
Richard Henderson9be0d082020-09-03 15:19:03 -07002765 set = ct->regs;
Richard Henderson25f49c52018-11-27 12:45:26 -08002766 }
2767 *pset = set;
2768 }
2769 break;
bellardc896fe22008-02-01 10:05:41 +00002770 }
2771 break;
2772 }
Richard Hendersonbee158c2016-06-22 20:43:29 -07002773 op->life = arg_life;
Evgeny Voevodin1ff0a2c2012-11-12 13:27:48 +04002774 }
bellardc896fe22008-02-01 10:05:41 +00002775}
bellardc896fe22008-02-01 10:05:41 +00002776
Richard Henderson5a184072016-06-23 20:34:33 -07002777/* Liveness analysis: Convert indirect regs to direct temporaries. */
Richard Hendersonb83eabe2016-11-01 15:56:04 -06002778static bool liveness_pass_2(TCGContext *s)
Richard Henderson5a184072016-06-23 20:34:33 -07002779{
2780 int nb_globals = s->nb_globals;
Richard Henderson15fa08f2017-11-02 15:19:14 +01002781 int nb_temps, i;
Richard Henderson5a184072016-06-23 20:34:33 -07002782 bool changes = false;
Richard Henderson15fa08f2017-11-02 15:19:14 +01002783 TCGOp *op, *op_next;
Richard Henderson5a184072016-06-23 20:34:33 -07002784
Richard Henderson5a184072016-06-23 20:34:33 -07002785 /* Create a temporary for each indirect global. */
2786 for (i = 0; i < nb_globals; ++i) {
2787 TCGTemp *its = &s->temps[i];
2788 if (its->indirect_reg) {
2789 TCGTemp *dts = tcg_temp_alloc(s);
2790 dts->type = its->type;
2791 dts->base_type = its->base_type;
Richard Hendersonb83eabe2016-11-01 15:56:04 -06002792 its->state_ptr = dts;
2793 } else {
2794 its->state_ptr = NULL;
Richard Henderson5a184072016-06-23 20:34:33 -07002795 }
Richard Hendersonb83eabe2016-11-01 15:56:04 -06002796 /* All globals begin dead. */
2797 its->state = TS_DEAD;
Richard Henderson5a184072016-06-23 20:34:33 -07002798 }
Richard Hendersonb83eabe2016-11-01 15:56:04 -06002799 for (nb_temps = s->nb_temps; i < nb_temps; ++i) {
2800 TCGTemp *its = &s->temps[i];
2801 its->state_ptr = NULL;
2802 its->state = TS_DEAD;
2803 }
Richard Henderson5a184072016-06-23 20:34:33 -07002804
Richard Henderson15fa08f2017-11-02 15:19:14 +01002805 QTAILQ_FOREACH_SAFE(op, &s->ops, link, op_next) {
Richard Henderson5a184072016-06-23 20:34:33 -07002806 TCGOpcode opc = op->opc;
2807 const TCGOpDef *def = &tcg_op_defs[opc];
2808 TCGLifeData arg_life = op->life;
2809 int nb_iargs, nb_oargs, call_flags;
Richard Hendersonb83eabe2016-11-01 15:56:04 -06002810 TCGTemp *arg_ts, *dir_ts;
Richard Henderson5a184072016-06-23 20:34:33 -07002811
Richard Henderson5a184072016-06-23 20:34:33 -07002812 if (opc == INDEX_op_call) {
Richard Hendersoncd9090a2017-11-14 13:02:51 +01002813 nb_oargs = TCGOP_CALLO(op);
2814 nb_iargs = TCGOP_CALLI(op);
Richard Henderson90163902021-03-18 10:21:45 -06002815 call_flags = tcg_call_flags(op);
Richard Henderson5a184072016-06-23 20:34:33 -07002816 } else {
2817 nb_iargs = def->nb_iargs;
2818 nb_oargs = def->nb_oargs;
2819
2820 /* Set flags similar to how calls require. */
Richard Hendersonb4cb76e2020-10-08 15:21:43 -05002821 if (def->flags & TCG_OPF_COND_BRANCH) {
2822 /* Like reading globals: sync_globals */
2823 call_flags = TCG_CALL_NO_WRITE_GLOBALS;
2824 } else if (def->flags & TCG_OPF_BB_END) {
Richard Henderson5a184072016-06-23 20:34:33 -07002825 /* Like writing globals: save_globals */
2826 call_flags = 0;
2827 } else if (def->flags & TCG_OPF_SIDE_EFFECTS) {
2828 /* Like reading globals: sync_globals */
2829 call_flags = TCG_CALL_NO_WRITE_GLOBALS;
2830 } else {
2831 /* No effect on globals. */
2832 call_flags = (TCG_CALL_NO_READ_GLOBALS |
2833 TCG_CALL_NO_WRITE_GLOBALS);
2834 }
2835 }
2836
2837 /* Make sure that input arguments are available. */
2838 for (i = nb_oargs; i < nb_iargs + nb_oargs; i++) {
Richard Hendersonb83eabe2016-11-01 15:56:04 -06002839 arg_ts = arg_temp(op->args[i]);
2840 if (arg_ts) {
2841 dir_ts = arg_ts->state_ptr;
2842 if (dir_ts && arg_ts->state == TS_DEAD) {
2843 TCGOpcode lopc = (arg_ts->type == TCG_TYPE_I32
Richard Henderson5a184072016-06-23 20:34:33 -07002844 ? INDEX_op_ld_i32
2845 : INDEX_op_ld_i64);
Emilio G. Cotaac1043f2018-12-09 14:37:19 -05002846 TCGOp *lop = tcg_op_insert_before(s, op, lopc);
Richard Henderson5a184072016-06-23 20:34:33 -07002847
Richard Hendersonb83eabe2016-11-01 15:56:04 -06002848 lop->args[0] = temp_arg(dir_ts);
2849 lop->args[1] = temp_arg(arg_ts->mem_base);
2850 lop->args[2] = arg_ts->mem_offset;
Richard Henderson5a184072016-06-23 20:34:33 -07002851
2852 /* Loaded, but synced with memory. */
Richard Hendersonb83eabe2016-11-01 15:56:04 -06002853 arg_ts->state = TS_MEM;
Richard Henderson5a184072016-06-23 20:34:33 -07002854 }
2855 }
2856 }
2857
2858 /* Perform input replacement, and mark inputs that became dead.
2859 No action is required except keeping temp_state up to date
2860 so that we reload when needed. */
2861 for (i = nb_oargs; i < nb_iargs + nb_oargs; i++) {
Richard Hendersonb83eabe2016-11-01 15:56:04 -06002862 arg_ts = arg_temp(op->args[i]);
2863 if (arg_ts) {
2864 dir_ts = arg_ts->state_ptr;
2865 if (dir_ts) {
2866 op->args[i] = temp_arg(dir_ts);
Richard Henderson5a184072016-06-23 20:34:33 -07002867 changes = true;
2868 if (IS_DEAD_ARG(i)) {
Richard Hendersonb83eabe2016-11-01 15:56:04 -06002869 arg_ts->state = TS_DEAD;
Richard Henderson5a184072016-06-23 20:34:33 -07002870 }
2871 }
2872 }
2873 }
2874
2875 /* Liveness analysis should ensure that the following are
2876 all correct, for call sites and basic block end points. */
2877 if (call_flags & TCG_CALL_NO_READ_GLOBALS) {
2878 /* Nothing to do */
2879 } else if (call_flags & TCG_CALL_NO_WRITE_GLOBALS) {
2880 for (i = 0; i < nb_globals; ++i) {
2881 /* Liveness should see that globals are synced back,
2882 that is, either TS_DEAD or TS_MEM. */
Richard Hendersonb83eabe2016-11-01 15:56:04 -06002883 arg_ts = &s->temps[i];
2884 tcg_debug_assert(arg_ts->state_ptr == 0
2885 || arg_ts->state != 0);
Richard Henderson5a184072016-06-23 20:34:33 -07002886 }
2887 } else {
2888 for (i = 0; i < nb_globals; ++i) {
2889 /* Liveness should see that globals are saved back,
2890 that is, TS_DEAD, waiting to be reloaded. */
Richard Hendersonb83eabe2016-11-01 15:56:04 -06002891 arg_ts = &s->temps[i];
2892 tcg_debug_assert(arg_ts->state_ptr == 0
2893 || arg_ts->state == TS_DEAD);
Richard Henderson5a184072016-06-23 20:34:33 -07002894 }
2895 }
2896
2897 /* Outputs become available. */
Richard Henderson61f15c42020-04-23 12:27:53 -07002898 if (opc == INDEX_op_mov_i32 || opc == INDEX_op_mov_i64) {
2899 arg_ts = arg_temp(op->args[0]);
Richard Hendersonb83eabe2016-11-01 15:56:04 -06002900 dir_ts = arg_ts->state_ptr;
Richard Henderson61f15c42020-04-23 12:27:53 -07002901 if (dir_ts) {
2902 op->args[0] = temp_arg(dir_ts);
2903 changes = true;
2904
2905 /* The output is now live and modified. */
2906 arg_ts->state = 0;
2907
2908 if (NEED_SYNC_ARG(0)) {
2909 TCGOpcode sopc = (arg_ts->type == TCG_TYPE_I32
2910 ? INDEX_op_st_i32
2911 : INDEX_op_st_i64);
2912 TCGOp *sop = tcg_op_insert_after(s, op, sopc);
2913 TCGTemp *out_ts = dir_ts;
2914
2915 if (IS_DEAD_ARG(0)) {
2916 out_ts = arg_temp(op->args[1]);
2917 arg_ts->state = TS_DEAD;
2918 tcg_op_remove(s, op);
2919 } else {
2920 arg_ts->state = TS_MEM;
2921 }
2922
2923 sop->args[0] = temp_arg(out_ts);
2924 sop->args[1] = temp_arg(arg_ts->mem_base);
2925 sop->args[2] = arg_ts->mem_offset;
2926 } else {
2927 tcg_debug_assert(!IS_DEAD_ARG(0));
2928 }
Richard Henderson5a184072016-06-23 20:34:33 -07002929 }
Richard Henderson61f15c42020-04-23 12:27:53 -07002930 } else {
2931 for (i = 0; i < nb_oargs; i++) {
2932 arg_ts = arg_temp(op->args[i]);
2933 dir_ts = arg_ts->state_ptr;
2934 if (!dir_ts) {
2935 continue;
2936 }
2937 op->args[i] = temp_arg(dir_ts);
2938 changes = true;
Richard Henderson5a184072016-06-23 20:34:33 -07002939
Richard Henderson61f15c42020-04-23 12:27:53 -07002940 /* The output is now live and modified. */
2941 arg_ts->state = 0;
Richard Henderson5a184072016-06-23 20:34:33 -07002942
Richard Henderson61f15c42020-04-23 12:27:53 -07002943 /* Sync outputs upon their last write. */
2944 if (NEED_SYNC_ARG(i)) {
2945 TCGOpcode sopc = (arg_ts->type == TCG_TYPE_I32
2946 ? INDEX_op_st_i32
2947 : INDEX_op_st_i64);
2948 TCGOp *sop = tcg_op_insert_after(s, op, sopc);
Richard Henderson5a184072016-06-23 20:34:33 -07002949
Richard Henderson61f15c42020-04-23 12:27:53 -07002950 sop->args[0] = temp_arg(dir_ts);
2951 sop->args[1] = temp_arg(arg_ts->mem_base);
2952 sop->args[2] = arg_ts->mem_offset;
Richard Henderson5a184072016-06-23 20:34:33 -07002953
Richard Henderson61f15c42020-04-23 12:27:53 -07002954 arg_ts->state = TS_MEM;
2955 }
2956 /* Drop outputs that are dead. */
2957 if (IS_DEAD_ARG(i)) {
2958 arg_ts->state = TS_DEAD;
2959 }
Richard Henderson5a184072016-06-23 20:34:33 -07002960 }
2961 }
2962 }
2963
2964 return changes;
2965}
2966
Aurelien Jarno8d8fdba2016-04-21 10:48:50 +02002967#ifdef CONFIG_DEBUG_TCG
bellardc896fe22008-02-01 10:05:41 +00002968static void dump_regs(TCGContext *s)
2969{
2970 TCGTemp *ts;
2971 int i;
2972 char buf[64];
2973
2974 for(i = 0; i < s->nb_temps; i++) {
2975 ts = &s->temps[i];
Richard Henderson43439132017-06-19 23:18:10 -07002976 printf(" %10s: ", tcg_get_arg_str_ptr(s, buf, sizeof(buf), ts));
bellardc896fe22008-02-01 10:05:41 +00002977 switch(ts->val_type) {
2978 case TEMP_VAL_REG:
2979 printf("%s", tcg_target_reg_names[ts->reg]);
2980 break;
2981 case TEMP_VAL_MEM:
Richard Hendersonb3a62932013-09-18 14:12:53 -07002982 printf("%d(%s)", (int)ts->mem_offset,
2983 tcg_target_reg_names[ts->mem_base->reg]);
bellardc896fe22008-02-01 10:05:41 +00002984 break;
2985 case TEMP_VAL_CONST:
Richard Hendersonbdb38b92020-09-06 11:31:44 -07002986 printf("$0x%" PRIx64, ts->val);
bellardc896fe22008-02-01 10:05:41 +00002987 break;
2988 case TEMP_VAL_DEAD:
2989 printf("D");
2990 break;
2991 default:
2992 printf("???");
2993 break;
2994 }
2995 printf("\n");
2996 }
2997
2998 for(i = 0; i < TCG_TARGET_NB_REGS; i++) {
Richard Hendersonf8b2f202013-09-18 15:21:56 -07002999 if (s->reg_to_temp[i] != NULL) {
bellardc896fe22008-02-01 10:05:41 +00003000 printf("%s: %s\n",
3001 tcg_target_reg_names[i],
Richard Hendersonf8b2f202013-09-18 15:21:56 -07003002 tcg_get_arg_str_ptr(s, buf, sizeof(buf), s->reg_to_temp[i]));
bellardc896fe22008-02-01 10:05:41 +00003003 }
3004 }
3005}
3006
3007static void check_regs(TCGContext *s)
3008{
Richard Henderson869938a2016-02-10 05:20:16 +11003009 int reg;
Richard Hendersonb6638662013-09-18 14:54:45 -07003010 int k;
bellardc896fe22008-02-01 10:05:41 +00003011 TCGTemp *ts;
3012 char buf[64];
3013
Richard Hendersonf8b2f202013-09-18 15:21:56 -07003014 for (reg = 0; reg < TCG_TARGET_NB_REGS; reg++) {
3015 ts = s->reg_to_temp[reg];
3016 if (ts != NULL) {
3017 if (ts->val_type != TEMP_VAL_REG || ts->reg != reg) {
bellardc896fe22008-02-01 10:05:41 +00003018 printf("Inconsistency for register %s:\n",
3019 tcg_target_reg_names[reg]);
bellardb03cce82008-05-10 10:52:05 +00003020 goto fail;
bellardc896fe22008-02-01 10:05:41 +00003021 }
3022 }
3023 }
Richard Hendersonf8b2f202013-09-18 15:21:56 -07003024 for (k = 0; k < s->nb_temps; k++) {
bellardc896fe22008-02-01 10:05:41 +00003025 ts = &s->temps[k];
Richard Hendersonee17db82020-03-29 10:11:56 -07003026 if (ts->val_type == TEMP_VAL_REG
3027 && ts->kind != TEMP_FIXED
Richard Hendersonf8b2f202013-09-18 15:21:56 -07003028 && s->reg_to_temp[ts->reg] != ts) {
3029 printf("Inconsistency for temp %s:\n",
3030 tcg_get_arg_str_ptr(s, buf, sizeof(buf), ts));
bellardb03cce82008-05-10 10:52:05 +00003031 fail:
Richard Hendersonf8b2f202013-09-18 15:21:56 -07003032 printf("reg state:\n");
3033 dump_regs(s);
3034 tcg_abort();
bellardc896fe22008-02-01 10:05:41 +00003035 }
3036 }
3037}
3038#endif
3039
Richard Henderson2272e4a2016-11-09 15:25:09 +01003040static void temp_allocate_frame(TCGContext *s, TCGTemp *ts)
bellardc896fe22008-02-01 10:05:41 +00003041{
Richard Hendersonc1c09192021-06-18 21:53:27 -07003042 intptr_t off, size, align;
3043
3044 switch (ts->type) {
3045 case TCG_TYPE_I32:
3046 size = align = 4;
3047 break;
3048 case TCG_TYPE_I64:
3049 case TCG_TYPE_V64:
3050 size = align = 8;
3051 break;
3052 case TCG_TYPE_V128:
3053 size = align = 16;
3054 break;
3055 case TCG_TYPE_V256:
3056 /* Note that we do not require aligned storage for V256. */
3057 size = 32, align = 16;
3058 break;
3059 default:
3060 g_assert_not_reached();
Blue Swirlb591dc52011-05-14 14:03:22 +00003061 }
Richard Hendersonc1c09192021-06-18 21:53:27 -07003062
3063 assert(align <= TCG_TARGET_STACK_ALIGN);
3064 off = ROUND_UP(s->current_frame_offset, align);
Richard Henderson732d5892021-06-19 06:32:03 -07003065
3066 /* If we've exhausted the stack frame, restart with a smaller TB. */
3067 if (off + size > s->frame_end) {
3068 tcg_raise_tb_overflow(s);
3069 }
Richard Hendersonc1c09192021-06-18 21:53:27 -07003070 s->current_frame_offset = off + size;
3071
3072 ts->mem_offset = off;
Richard Henderson9defd1b2021-06-18 16:49:26 -07003073#if defined(__sparc__)
3074 ts->mem_offset += TCG_TARGET_STACK_BIAS;
3075#endif
Richard Hendersonb3a62932013-09-18 14:12:53 -07003076 ts->mem_base = s->frame_temp;
bellardc896fe22008-02-01 10:05:41 +00003077 ts->mem_allocated = 1;
bellardc896fe22008-02-01 10:05:41 +00003078}
3079
Richard Hendersonb7224522018-11-27 07:48:06 -08003080static void temp_load(TCGContext *, TCGTemp *, TCGRegSet, TCGRegSet, TCGRegSet);
Richard Hendersonb3915db2013-09-19 10:36:18 -07003081
Richard Henderson59d7c142016-06-19 22:59:13 -07003082/* Mark a temporary as free or dead. If 'free_or_dead' is negative,
3083 mark it free; otherwise mark it dead. */
3084static void temp_free_or_dead(TCGContext *s, TCGTemp *ts, int free_or_dead)
bellardc896fe22008-02-01 10:05:41 +00003085{
Richard Hendersonc0522132020-03-29 18:55:52 -07003086 TCGTempVal new_type;
3087
3088 switch (ts->kind) {
3089 case TEMP_FIXED:
Richard Henderson59d7c142016-06-19 22:59:13 -07003090 return;
Richard Hendersonc0522132020-03-29 18:55:52 -07003091 case TEMP_GLOBAL:
3092 case TEMP_LOCAL:
3093 new_type = TEMP_VAL_MEM;
3094 break;
3095 case TEMP_NORMAL:
3096 new_type = free_or_dead < 0 ? TEMP_VAL_MEM : TEMP_VAL_DEAD;
3097 break;
3098 case TEMP_CONST:
3099 new_type = TEMP_VAL_CONST;
3100 break;
3101 default:
3102 g_assert_not_reached();
Richard Henderson59d7c142016-06-19 22:59:13 -07003103 }
3104 if (ts->val_type == TEMP_VAL_REG) {
3105 s->reg_to_temp[ts->reg] = NULL;
3106 }
Richard Hendersonc0522132020-03-29 18:55:52 -07003107 ts->val_type = new_type;
Richard Henderson59d7c142016-06-19 22:59:13 -07003108}
bellardc896fe22008-02-01 10:05:41 +00003109
Richard Henderson59d7c142016-06-19 22:59:13 -07003110/* Mark a temporary as dead. */
3111static inline void temp_dead(TCGContext *s, TCGTemp *ts)
3112{
3113 temp_free_or_dead(s, ts, 1);
3114}
3115
3116/* Sync a temporary to memory. 'allocated_regs' is used in case a temporary
3117 registers needs to be allocated to store a constant. If 'free_or_dead'
3118 is non-zero, subsequently release the temporary; if it is positive, the
3119 temp is dead; if it is negative, the temp is free. */
Richard Henderson98b4e182018-11-27 15:35:04 -08003120static void temp_sync(TCGContext *s, TCGTemp *ts, TCGRegSet allocated_regs,
3121 TCGRegSet preferred_regs, int free_or_dead)
Richard Henderson59d7c142016-06-19 22:59:13 -07003122{
Richard Hendersonc0522132020-03-29 18:55:52 -07003123 if (!temp_readonly(ts) && !ts->mem_coherent) {
Aurelien Jarno7f6ceed2012-10-09 21:53:06 +02003124 if (!ts->mem_allocated) {
Richard Henderson2272e4a2016-11-09 15:25:09 +01003125 temp_allocate_frame(s, ts);
Richard Henderson59d7c142016-06-19 22:59:13 -07003126 }
Richard Henderson59d7c142016-06-19 22:59:13 -07003127 switch (ts->val_type) {
3128 case TEMP_VAL_CONST:
3129 /* If we're going to free the temp immediately, then we won't
3130 require it later in a register, so attempt to store the
3131 constant to memory directly. */
3132 if (free_or_dead
3133 && tcg_out_sti(s, ts->type, ts->val,
3134 ts->mem_base->reg, ts->mem_offset)) {
3135 break;
3136 }
3137 temp_load(s, ts, tcg_target_available_regs[ts->type],
Richard Henderson98b4e182018-11-27 15:35:04 -08003138 allocated_regs, preferred_regs);
Richard Henderson59d7c142016-06-19 22:59:13 -07003139 /* fallthrough */
3140
3141 case TEMP_VAL_REG:
3142 tcg_out_st(s, ts->type, ts->reg,
3143 ts->mem_base->reg, ts->mem_offset);
3144 break;
3145
3146 case TEMP_VAL_MEM:
3147 break;
3148
3149 case TEMP_VAL_DEAD:
3150 default:
3151 tcg_abort();
3152 }
3153 ts->mem_coherent = 1;
Aurelien Jarno7f6ceed2012-10-09 21:53:06 +02003154 }
Richard Henderson59d7c142016-06-19 22:59:13 -07003155 if (free_or_dead) {
3156 temp_free_or_dead(s, ts, free_or_dead);
3157 }
Aurelien Jarno7f6ceed2012-10-09 21:53:06 +02003158}
3159
3160/* free register 'reg' by spilling the corresponding temporary if necessary */
Richard Hendersonb3915db2013-09-19 10:36:18 -07003161static void tcg_reg_free(TCGContext *s, TCGReg reg, TCGRegSet allocated_regs)
Aurelien Jarno7f6ceed2012-10-09 21:53:06 +02003162{
Richard Hendersonf8b2f202013-09-18 15:21:56 -07003163 TCGTemp *ts = s->reg_to_temp[reg];
Richard Hendersonf8b2f202013-09-18 15:21:56 -07003164 if (ts != NULL) {
Richard Henderson98b4e182018-11-27 15:35:04 -08003165 temp_sync(s, ts, allocated_regs, 0, -1);
bellardc896fe22008-02-01 10:05:41 +00003166 }
3167}
3168
Richard Hendersonb0164862018-11-27 07:16:21 -08003169/**
3170 * tcg_reg_alloc:
3171 * @required_regs: Set of registers in which we must allocate.
3172 * @allocated_regs: Set of registers which must be avoided.
3173 * @preferred_regs: Set of registers we should prefer.
3174 * @rev: True if we search the registers in "indirect" order.
3175 *
3176 * The allocated register must be in @required_regs & ~@allocated_regs,
3177 * but if we can put it in @preferred_regs we may save a move later.
3178 */
3179static TCGReg tcg_reg_alloc(TCGContext *s, TCGRegSet required_regs,
3180 TCGRegSet allocated_regs,
3181 TCGRegSet preferred_regs, bool rev)
bellardc896fe22008-02-01 10:05:41 +00003182{
Richard Hendersonb0164862018-11-27 07:16:21 -08003183 int i, j, f, n = ARRAY_SIZE(tcg_target_reg_alloc_order);
3184 TCGRegSet reg_ct[2];
Richard Henderson91478ce2015-08-18 23:23:08 -07003185 const int *order;
bellardc896fe22008-02-01 10:05:41 +00003186
Richard Hendersonb0164862018-11-27 07:16:21 -08003187 reg_ct[1] = required_regs & ~allocated_regs;
3188 tcg_debug_assert(reg_ct[1] != 0);
3189 reg_ct[0] = reg_ct[1] & preferred_regs;
3190
3191 /* Skip the preferred_regs option if it cannot be satisfied,
3192 or if the preference made no difference. */
3193 f = reg_ct[0] == 0 || reg_ct[0] == reg_ct[1];
3194
Richard Henderson91478ce2015-08-18 23:23:08 -07003195 order = rev ? indirect_reg_alloc_order : tcg_target_reg_alloc_order;
bellardc896fe22008-02-01 10:05:41 +00003196
Richard Hendersonb0164862018-11-27 07:16:21 -08003197 /* Try free registers, preferences first. */
3198 for (j = f; j < 2; j++) {
3199 TCGRegSet set = reg_ct[j];
3200
3201 if (tcg_regset_single(set)) {
3202 /* One register in the set. */
3203 TCGReg reg = tcg_regset_first(set);
3204 if (s->reg_to_temp[reg] == NULL) {
3205 return reg;
3206 }
3207 } else {
3208 for (i = 0; i < n; i++) {
3209 TCGReg reg = order[i];
3210 if (s->reg_to_temp[reg] == NULL &&
3211 tcg_regset_test_reg(set, reg)) {
3212 return reg;
3213 }
3214 }
3215 }
bellardc896fe22008-02-01 10:05:41 +00003216 }
3217
Richard Hendersonb0164862018-11-27 07:16:21 -08003218 /* We must spill something. */
3219 for (j = f; j < 2; j++) {
3220 TCGRegSet set = reg_ct[j];
3221
3222 if (tcg_regset_single(set)) {
3223 /* One register in the set. */
3224 TCGReg reg = tcg_regset_first(set);
Richard Hendersonb3915db2013-09-19 10:36:18 -07003225 tcg_reg_free(s, reg, allocated_regs);
bellardc896fe22008-02-01 10:05:41 +00003226 return reg;
Richard Hendersonb0164862018-11-27 07:16:21 -08003227 } else {
3228 for (i = 0; i < n; i++) {
3229 TCGReg reg = order[i];
3230 if (tcg_regset_test_reg(set, reg)) {
3231 tcg_reg_free(s, reg, allocated_regs);
3232 return reg;
3233 }
3234 }
bellardc896fe22008-02-01 10:05:41 +00003235 }
3236 }
3237
3238 tcg_abort();
3239}
3240
Richard Henderson40ae5c62013-09-19 08:02:05 -07003241/* Make sure the temporary is in a register. If needed, allocate the register
3242 from DESIRED while avoiding ALLOCATED. */
3243static void temp_load(TCGContext *s, TCGTemp *ts, TCGRegSet desired_regs,
Richard Hendersonb7224522018-11-27 07:48:06 -08003244 TCGRegSet allocated_regs, TCGRegSet preferred_regs)
Richard Henderson40ae5c62013-09-19 08:02:05 -07003245{
3246 TCGReg reg;
3247
3248 switch (ts->val_type) {
3249 case TEMP_VAL_REG:
3250 return;
3251 case TEMP_VAL_CONST:
Richard Hendersonb0164862018-11-27 07:16:21 -08003252 reg = tcg_reg_alloc(s, desired_regs, allocated_regs,
Richard Hendersonb7224522018-11-27 07:48:06 -08003253 preferred_regs, ts->indirect_base);
Richard Henderson0a6a8bc2020-03-31 05:43:23 -07003254 if (ts->type <= TCG_TYPE_I64) {
3255 tcg_out_movi(s, ts->type, reg, ts->val);
3256 } else {
Richard Henderson4e186172020-03-31 01:02:08 -07003257 uint64_t val = ts->val;
3258 MemOp vece = MO_64;
3259
3260 /*
3261 * Find the minimal vector element that matches the constant.
3262 * The targets will, in general, have to do this search anyway,
3263 * do this generically.
3264 */
Richard Henderson4e186172020-03-31 01:02:08 -07003265 if (val == dup_const(MO_8, val)) {
3266 vece = MO_8;
3267 } else if (val == dup_const(MO_16, val)) {
3268 vece = MO_16;
Richard Henderson0b4286d2020-09-06 17:33:18 -07003269 } else if (val == dup_const(MO_32, val)) {
Richard Henderson4e186172020-03-31 01:02:08 -07003270 vece = MO_32;
3271 }
3272
3273 tcg_out_dupi_vec(s, ts->type, vece, reg, ts->val);
Richard Henderson0a6a8bc2020-03-31 05:43:23 -07003274 }
Richard Henderson40ae5c62013-09-19 08:02:05 -07003275 ts->mem_coherent = 0;
3276 break;
3277 case TEMP_VAL_MEM:
Richard Hendersonb0164862018-11-27 07:16:21 -08003278 reg = tcg_reg_alloc(s, desired_regs, allocated_regs,
Richard Hendersonb7224522018-11-27 07:48:06 -08003279 preferred_regs, ts->indirect_base);
Richard Henderson40ae5c62013-09-19 08:02:05 -07003280 tcg_out_ld(s, ts->type, reg, ts->mem_base->reg, ts->mem_offset);
3281 ts->mem_coherent = 1;
3282 break;
3283 case TEMP_VAL_DEAD:
3284 default:
3285 tcg_abort();
3286 }
3287 ts->reg = reg;
3288 ts->val_type = TEMP_VAL_REG;
3289 s->reg_to_temp[reg] = ts;
3290}
3291
Richard Henderson59d7c142016-06-19 22:59:13 -07003292/* Save a temporary to memory. 'allocated_regs' is used in case a
3293 temporary registers needs to be allocated to store a constant. */
3294static void temp_save(TCGContext *s, TCGTemp *ts, TCGRegSet allocated_regs)
Aurelien Jarno1ad80722012-10-09 21:53:06 +02003295{
Richard Henderson5a184072016-06-23 20:34:33 -07003296 /* The liveness analysis already ensures that globals are back
3297 in memory. Keep an tcg_debug_assert for safety. */
Richard Hendersone01fa972020-03-29 10:40:49 -07003298 tcg_debug_assert(ts->val_type == TEMP_VAL_MEM || temp_readonly(ts));
Aurelien Jarno1ad80722012-10-09 21:53:06 +02003299}
3300
Dong Xu Wang9814dd22011-11-22 18:06:22 +08003301/* save globals to their canonical location and assume they can be
bellarde8996ee2008-05-23 17:33:39 +00003302 modified be the following code. 'allocated_regs' is used in case a
3303 temporary registers needs to be allocated to store a constant. */
3304static void save_globals(TCGContext *s, TCGRegSet allocated_regs)
bellardc896fe22008-02-01 10:05:41 +00003305{
Richard Hendersonac3b8892016-11-02 11:21:44 -06003306 int i, n;
bellardc896fe22008-02-01 10:05:41 +00003307
Richard Hendersonac3b8892016-11-02 11:21:44 -06003308 for (i = 0, n = s->nb_globals; i < n; i++) {
Richard Hendersonb13eb722013-09-18 15:35:32 -07003309 temp_save(s, &s->temps[i], allocated_regs);
bellardc896fe22008-02-01 10:05:41 +00003310 }
bellarde5097dc2008-05-21 16:24:20 +00003311}
3312
Aurelien Jarno3d5c5f82012-10-09 21:53:08 +02003313/* sync globals to their canonical location and assume they can be
3314 read by the following code. 'allocated_regs' is used in case a
3315 temporary registers needs to be allocated to store a constant. */
3316static void sync_globals(TCGContext *s, TCGRegSet allocated_regs)
3317{
Richard Hendersonac3b8892016-11-02 11:21:44 -06003318 int i, n;
Aurelien Jarno3d5c5f82012-10-09 21:53:08 +02003319
Richard Hendersonac3b8892016-11-02 11:21:44 -06003320 for (i = 0, n = s->nb_globals; i < n; i++) {
Richard Henderson12b9b112013-09-18 15:33:00 -07003321 TCGTemp *ts = &s->temps[i];
Richard Henderson5a184072016-06-23 20:34:33 -07003322 tcg_debug_assert(ts->val_type != TEMP_VAL_REG
Richard Hendersonee17db82020-03-29 10:11:56 -07003323 || ts->kind == TEMP_FIXED
Richard Henderson5a184072016-06-23 20:34:33 -07003324 || ts->mem_coherent);
Aurelien Jarno3d5c5f82012-10-09 21:53:08 +02003325 }
3326}
3327
bellarde5097dc2008-05-21 16:24:20 +00003328/* at the end of a basic block, we assume all temporaries are dead and
bellarde8996ee2008-05-23 17:33:39 +00003329 all globals are stored at their canonical location. */
3330static void tcg_reg_alloc_bb_end(TCGContext *s, TCGRegSet allocated_regs)
bellarde5097dc2008-05-21 16:24:20 +00003331{
bellarde5097dc2008-05-21 16:24:20 +00003332 int i;
3333
Richard Hendersonb13eb722013-09-18 15:35:32 -07003334 for (i = s->nb_globals; i < s->nb_temps; i++) {
3335 TCGTemp *ts = &s->temps[i];
Richard Hendersonc0522132020-03-29 18:55:52 -07003336
3337 switch (ts->kind) {
3338 case TEMP_LOCAL:
Richard Hendersonb13eb722013-09-18 15:35:32 -07003339 temp_save(s, ts, allocated_regs);
Richard Hendersonc0522132020-03-29 18:55:52 -07003340 break;
3341 case TEMP_NORMAL:
Richard Henderson5a184072016-06-23 20:34:33 -07003342 /* The liveness analysis already ensures that temps are dead.
3343 Keep an tcg_debug_assert for safety. */
3344 tcg_debug_assert(ts->val_type == TEMP_VAL_DEAD);
Richard Hendersonc0522132020-03-29 18:55:52 -07003345 break;
3346 case TEMP_CONST:
3347 /* Similarly, we should have freed any allocated register. */
3348 tcg_debug_assert(ts->val_type == TEMP_VAL_CONST);
3349 break;
3350 default:
3351 g_assert_not_reached();
bellardc896fe22008-02-01 10:05:41 +00003352 }
3353 }
bellarde8996ee2008-05-23 17:33:39 +00003354
3355 save_globals(s, allocated_regs);
bellardc896fe22008-02-01 10:05:41 +00003356}
3357
Richard Hendersonbab16712019-03-18 11:20:27 -07003358/*
Richard Hendersonb4cb76e2020-10-08 15:21:43 -05003359 * At a conditional branch, we assume all temporaries are dead and
3360 * all globals and local temps are synced to their location.
3361 */
3362static void tcg_reg_alloc_cbranch(TCGContext *s, TCGRegSet allocated_regs)
3363{
3364 sync_globals(s, allocated_regs);
3365
3366 for (int i = s->nb_globals; i < s->nb_temps; i++) {
3367 TCGTemp *ts = &s->temps[i];
3368 /*
3369 * The liveness analysis already ensures that temps are dead.
3370 * Keep tcg_debug_asserts for safety.
3371 */
Richard Hendersonc0522132020-03-29 18:55:52 -07003372 switch (ts->kind) {
3373 case TEMP_LOCAL:
Richard Hendersonb4cb76e2020-10-08 15:21:43 -05003374 tcg_debug_assert(ts->val_type != TEMP_VAL_REG || ts->mem_coherent);
Richard Hendersonc0522132020-03-29 18:55:52 -07003375 break;
3376 case TEMP_NORMAL:
Richard Hendersonb4cb76e2020-10-08 15:21:43 -05003377 tcg_debug_assert(ts->val_type == TEMP_VAL_DEAD);
Richard Hendersonc0522132020-03-29 18:55:52 -07003378 break;
3379 case TEMP_CONST:
3380 break;
3381 default:
3382 g_assert_not_reached();
Richard Hendersonb4cb76e2020-10-08 15:21:43 -05003383 }
3384 }
3385}
3386
3387/*
Richard Hendersonc58f4c92020-04-17 13:22:43 -07003388 * Specialized code generation for INDEX_op_mov_* with a constant.
Richard Hendersonbab16712019-03-18 11:20:27 -07003389 */
Paolo Bonzini0fe4fca2016-09-15 15:16:00 +02003390static void tcg_reg_alloc_do_movi(TCGContext *s, TCGTemp *ots,
Richard Hendersonba877192018-11-27 15:39:21 -08003391 tcg_target_ulong val, TCGLifeData arg_life,
3392 TCGRegSet preferred_regs)
bellarde8996ee2008-05-23 17:33:39 +00003393{
Richard Hendersond63e3b62019-03-16 17:48:02 +00003394 /* ENV should not be modified. */
Richard Hendersone01fa972020-03-29 10:40:49 -07003395 tcg_debug_assert(!temp_readonly(ots));
Richard Henderson59d7c142016-06-19 22:59:13 -07003396
3397 /* The movi is not explicitly generated here. */
3398 if (ots->val_type == TEMP_VAL_REG) {
3399 s->reg_to_temp[ots->reg] = NULL;
3400 }
3401 ots->val_type = TEMP_VAL_CONST;
3402 ots->val = val;
3403 ots->mem_coherent = 0;
Aurelien Jarnoec7a8692012-10-09 21:53:07 +02003404 if (NEED_SYNC_ARG(0)) {
Richard Hendersonba877192018-11-27 15:39:21 -08003405 temp_sync(s, ots, s->reserved_regs, preferred_regs, IS_DEAD_ARG(0));
Richard Henderson59d7c142016-06-19 22:59:13 -07003406 } else if (IS_DEAD_ARG(0)) {
Richard Hendersonf8bf00f2013-09-18 15:29:18 -07003407 temp_dead(s, ots);
Aurelien Jarno4c4e1ab2012-10-09 21:53:07 +02003408 }
bellarde8996ee2008-05-23 17:33:39 +00003409}
3410
Richard Hendersonbab16712019-03-18 11:20:27 -07003411/*
3412 * Specialized code generation for INDEX_op_mov_*.
3413 */
Richard Hendersondd186292016-12-08 13:42:08 -08003414static void tcg_reg_alloc_mov(TCGContext *s, const TCGOp *op)
bellardc896fe22008-02-01 10:05:41 +00003415{
Richard Hendersondd186292016-12-08 13:42:08 -08003416 const TCGLifeData arg_life = op->life;
Richard Henderson69e37062018-11-27 07:44:51 -08003417 TCGRegSet allocated_regs, preferred_regs;
bellardc896fe22008-02-01 10:05:41 +00003418 TCGTemp *ts, *ots;
Richard Henderson450445d2014-05-13 14:50:18 -07003419 TCGType otype, itype;
bellardc896fe22008-02-01 10:05:41 +00003420
Richard Hendersond21369f2017-09-11 11:58:44 -07003421 allocated_regs = s->reserved_regs;
Richard Henderson69e37062018-11-27 07:44:51 -08003422 preferred_regs = op->output_pref[0];
Richard Henderson43439132017-06-19 23:18:10 -07003423 ots = arg_temp(op->args[0]);
3424 ts = arg_temp(op->args[1]);
Richard Henderson450445d2014-05-13 14:50:18 -07003425
Richard Hendersond63e3b62019-03-16 17:48:02 +00003426 /* ENV should not be modified. */
Richard Hendersone01fa972020-03-29 10:40:49 -07003427 tcg_debug_assert(!temp_readonly(ots));
Richard Hendersond63e3b62019-03-16 17:48:02 +00003428
Richard Henderson450445d2014-05-13 14:50:18 -07003429 /* Note that otype != itype for no-op truncation. */
3430 otype = ots->type;
3431 itype = ts->type;
bellardc896fe22008-02-01 10:05:41 +00003432
Paolo Bonzini0fe4fca2016-09-15 15:16:00 +02003433 if (ts->val_type == TEMP_VAL_CONST) {
3434 /* propagate constant or generate sti */
3435 tcg_target_ulong val = ts->val;
3436 if (IS_DEAD_ARG(1)) {
3437 temp_dead(s, ts);
3438 }
Richard Henderson69e37062018-11-27 07:44:51 -08003439 tcg_reg_alloc_do_movi(s, ots, val, arg_life, preferred_regs);
Paolo Bonzini0fe4fca2016-09-15 15:16:00 +02003440 return;
3441 }
3442
3443 /* If the source value is in memory we're going to be forced
3444 to have it in a register in order to perform the copy. Copy
3445 the SOURCE value into its own register first, that way we
3446 don't have to reload SOURCE the next time it is used. */
3447 if (ts->val_type == TEMP_VAL_MEM) {
Richard Henderson69e37062018-11-27 07:44:51 -08003448 temp_load(s, ts, tcg_target_available_regs[itype],
3449 allocated_regs, preferred_regs);
Aurelien Jarnoc29c1d72012-10-09 21:53:07 +02003450 }
3451
Paolo Bonzini0fe4fca2016-09-15 15:16:00 +02003452 tcg_debug_assert(ts->val_type == TEMP_VAL_REG);
Richard Hendersond63e3b62019-03-16 17:48:02 +00003453 if (IS_DEAD_ARG(0)) {
Aurelien Jarnoc29c1d72012-10-09 21:53:07 +02003454 /* mov to a non-saved dead register makes no sense (even with
3455 liveness analysis disabled). */
Aurelien Jarnoeabb7b92016-04-21 10:48:49 +02003456 tcg_debug_assert(NEED_SYNC_ARG(0));
Aurelien Jarnoc29c1d72012-10-09 21:53:07 +02003457 if (!ots->mem_allocated) {
Richard Henderson2272e4a2016-11-09 15:25:09 +01003458 temp_allocate_frame(s, ots);
Aurelien Jarnoc29c1d72012-10-09 21:53:07 +02003459 }
Richard Hendersonb3a62932013-09-18 14:12:53 -07003460 tcg_out_st(s, otype, ts->reg, ots->mem_base->reg, ots->mem_offset);
Aurelien Jarnoc29c1d72012-10-09 21:53:07 +02003461 if (IS_DEAD_ARG(1)) {
Richard Hendersonf8bf00f2013-09-18 15:29:18 -07003462 temp_dead(s, ts);
Aurelien Jarnoc29c1d72012-10-09 21:53:07 +02003463 }
Richard Hendersonf8bf00f2013-09-18 15:29:18 -07003464 temp_dead(s, ots);
Aurelien Jarnoc29c1d72012-10-09 21:53:07 +02003465 } else {
Richard Hendersonee17db82020-03-29 10:11:56 -07003466 if (IS_DEAD_ARG(1) && ts->kind != TEMP_FIXED) {
bellardc896fe22008-02-01 10:05:41 +00003467 /* the mov can be suppressed */
Aurelien Jarnoc29c1d72012-10-09 21:53:07 +02003468 if (ots->val_type == TEMP_VAL_REG) {
Richard Hendersonf8b2f202013-09-18 15:21:56 -07003469 s->reg_to_temp[ots->reg] = NULL;
Aurelien Jarnoc29c1d72012-10-09 21:53:07 +02003470 }
3471 ots->reg = ts->reg;
Richard Hendersonf8bf00f2013-09-18 15:29:18 -07003472 temp_dead(s, ts);
bellardc896fe22008-02-01 10:05:41 +00003473 } else {
Aurelien Jarnoc29c1d72012-10-09 21:53:07 +02003474 if (ots->val_type != TEMP_VAL_REG) {
3475 /* When allocating a new register, make sure to not spill the
3476 input one. */
3477 tcg_regset_set_reg(allocated_regs, ts->reg);
Richard Henderson450445d2014-05-13 14:50:18 -07003478 ots->reg = tcg_reg_alloc(s, tcg_target_available_regs[otype],
Richard Henderson69e37062018-11-27 07:44:51 -08003479 allocated_regs, preferred_regs,
Richard Hendersonb0164862018-11-27 07:16:21 -08003480 ots->indirect_base);
bellardc896fe22008-02-01 10:05:41 +00003481 }
Richard Henderson78113e82019-03-16 17:48:18 +00003482 if (!tcg_out_mov(s, otype, ots->reg, ts->reg)) {
Richard Henderson240c08d2019-03-16 17:48:32 +00003483 /*
3484 * Cross register class move not supported.
3485 * Store the source register into the destination slot
3486 * and leave the destination temp as TEMP_VAL_MEM.
3487 */
Richard Hendersone01fa972020-03-29 10:40:49 -07003488 assert(!temp_readonly(ots));
Richard Henderson240c08d2019-03-16 17:48:32 +00003489 if (!ts->mem_allocated) {
3490 temp_allocate_frame(s, ots);
3491 }
3492 tcg_out_st(s, ts->type, ts->reg,
3493 ots->mem_base->reg, ots->mem_offset);
3494 ots->mem_coherent = 1;
3495 temp_free_or_dead(s, ots, -1);
3496 return;
Richard Henderson78113e82019-03-16 17:48:18 +00003497 }
bellardc896fe22008-02-01 10:05:41 +00003498 }
Aurelien Jarnoc29c1d72012-10-09 21:53:07 +02003499 ots->val_type = TEMP_VAL_REG;
3500 ots->mem_coherent = 0;
Richard Hendersonf8b2f202013-09-18 15:21:56 -07003501 s->reg_to_temp[ots->reg] = ots;
Aurelien Jarnoc29c1d72012-10-09 21:53:07 +02003502 if (NEED_SYNC_ARG(0)) {
Richard Henderson98b4e182018-11-27 15:35:04 -08003503 temp_sync(s, ots, allocated_regs, 0, 0);
bellardc896fe22008-02-01 10:05:41 +00003504 }
Aurelien Jarnoec7a8692012-10-09 21:53:07 +02003505 }
bellardc896fe22008-02-01 10:05:41 +00003506}
3507
Richard Hendersonbab16712019-03-18 11:20:27 -07003508/*
3509 * Specialized code generation for INDEX_op_dup_vec.
3510 */
3511static void tcg_reg_alloc_dup(TCGContext *s, const TCGOp *op)
3512{
3513 const TCGLifeData arg_life = op->life;
3514 TCGRegSet dup_out_regs, dup_in_regs;
3515 TCGTemp *its, *ots;
3516 TCGType itype, vtype;
Richard Hendersond6ecb4a2019-03-18 12:00:39 -07003517 intptr_t endian_fixup;
Richard Hendersonbab16712019-03-18 11:20:27 -07003518 unsigned vece;
3519 bool ok;
3520
3521 ots = arg_temp(op->args[0]);
3522 its = arg_temp(op->args[1]);
3523
3524 /* ENV should not be modified. */
Richard Hendersone01fa972020-03-29 10:40:49 -07003525 tcg_debug_assert(!temp_readonly(ots));
Richard Hendersonbab16712019-03-18 11:20:27 -07003526
3527 itype = its->type;
3528 vece = TCGOP_VECE(op);
3529 vtype = TCGOP_VECL(op) + TCG_TYPE_V64;
3530
3531 if (its->val_type == TEMP_VAL_CONST) {
3532 /* Propagate constant via movi -> dupi. */
3533 tcg_target_ulong val = its->val;
3534 if (IS_DEAD_ARG(1)) {
3535 temp_dead(s, its);
3536 }
3537 tcg_reg_alloc_do_movi(s, ots, val, arg_life, op->output_pref[0]);
3538 return;
3539 }
3540
Richard Henderson9be0d082020-09-03 15:19:03 -07003541 dup_out_regs = tcg_op_defs[INDEX_op_dup_vec].args_ct[0].regs;
3542 dup_in_regs = tcg_op_defs[INDEX_op_dup_vec].args_ct[1].regs;
Richard Hendersonbab16712019-03-18 11:20:27 -07003543
3544 /* Allocate the output register now. */
3545 if (ots->val_type != TEMP_VAL_REG) {
3546 TCGRegSet allocated_regs = s->reserved_regs;
3547
3548 if (!IS_DEAD_ARG(1) && its->val_type == TEMP_VAL_REG) {
3549 /* Make sure to not spill the input register. */
3550 tcg_regset_set_reg(allocated_regs, its->reg);
3551 }
3552 ots->reg = tcg_reg_alloc(s, dup_out_regs, allocated_regs,
3553 op->output_pref[0], ots->indirect_base);
3554 ots->val_type = TEMP_VAL_REG;
3555 ots->mem_coherent = 0;
3556 s->reg_to_temp[ots->reg] = ots;
3557 }
3558
3559 switch (its->val_type) {
3560 case TEMP_VAL_REG:
3561 /*
3562 * The dup constriaints must be broad, covering all possible VECE.
3563 * However, tcg_op_dup_vec() gets to see the VECE and we allow it
3564 * to fail, indicating that extra moves are required for that case.
3565 */
3566 if (tcg_regset_test_reg(dup_in_regs, its->reg)) {
3567 if (tcg_out_dup_vec(s, vtype, vece, ots->reg, its->reg)) {
3568 goto done;
3569 }
3570 /* Try again from memory or a vector input register. */
3571 }
3572 if (!its->mem_coherent) {
3573 /*
3574 * The input register is not synced, and so an extra store
3575 * would be required to use memory. Attempt an integer-vector
3576 * register move first. We do not have a TCGRegSet for this.
3577 */
3578 if (tcg_out_mov(s, itype, ots->reg, its->reg)) {
3579 break;
3580 }
3581 /* Sync the temp back to its slot and load from there. */
3582 temp_sync(s, its, s->reserved_regs, 0, 0);
3583 }
3584 /* fall through */
3585
3586 case TEMP_VAL_MEM:
Richard Hendersond6ecb4a2019-03-18 12:00:39 -07003587#ifdef HOST_WORDS_BIGENDIAN
3588 endian_fixup = itype == TCG_TYPE_I32 ? 4 : 8;
3589 endian_fixup -= 1 << vece;
3590#else
3591 endian_fixup = 0;
3592#endif
3593 if (tcg_out_dupm_vec(s, vtype, vece, ots->reg, its->mem_base->reg,
3594 its->mem_offset + endian_fixup)) {
3595 goto done;
3596 }
Richard Hendersonbab16712019-03-18 11:20:27 -07003597 tcg_out_ld(s, itype, ots->reg, its->mem_base->reg, its->mem_offset);
3598 break;
3599
3600 default:
3601 g_assert_not_reached();
3602 }
3603
3604 /* We now have a vector input register, so dup must succeed. */
3605 ok = tcg_out_dup_vec(s, vtype, vece, ots->reg, ots->reg);
3606 tcg_debug_assert(ok);
3607
3608 done:
3609 if (IS_DEAD_ARG(1)) {
3610 temp_dead(s, its);
3611 }
3612 if (NEED_SYNC_ARG(0)) {
3613 temp_sync(s, ots, s->reserved_regs, 0, 0);
3614 }
3615 if (IS_DEAD_ARG(0)) {
3616 temp_dead(s, ots);
3617 }
3618}
3619
Richard Hendersondd186292016-12-08 13:42:08 -08003620static void tcg_reg_alloc_op(TCGContext *s, const TCGOp *op)
bellardc896fe22008-02-01 10:05:41 +00003621{
Richard Hendersondd186292016-12-08 13:42:08 -08003622 const TCGLifeData arg_life = op->life;
3623 const TCGOpDef * const def = &tcg_op_defs[op->opc];
Richard Henderson82790a82016-11-18 08:35:03 +01003624 TCGRegSet i_allocated_regs;
3625 TCGRegSet o_allocated_regs;
Richard Hendersonb6638662013-09-18 14:54:45 -07003626 int i, k, nb_iargs, nb_oargs;
3627 TCGReg reg;
bellardc896fe22008-02-01 10:05:41 +00003628 TCGArg arg;
3629 const TCGArgConstraint *arg_ct;
3630 TCGTemp *ts;
3631 TCGArg new_args[TCG_MAX_OP_ARGS];
3632 int const_args[TCG_MAX_OP_ARGS];
3633
3634 nb_oargs = def->nb_oargs;
3635 nb_iargs = def->nb_iargs;
3636
3637 /* copy constants */
3638 memcpy(new_args + nb_oargs + nb_iargs,
Richard Hendersondd186292016-12-08 13:42:08 -08003639 op->args + nb_oargs + nb_iargs,
bellardc896fe22008-02-01 10:05:41 +00003640 sizeof(TCGArg) * def->nb_cargs);
3641
Richard Hendersond21369f2017-09-11 11:58:44 -07003642 i_allocated_regs = s->reserved_regs;
3643 o_allocated_regs = s->reserved_regs;
Richard Henderson82790a82016-11-18 08:35:03 +01003644
bellardc896fe22008-02-01 10:05:41 +00003645 /* satisfy input constraints */
Richard Hendersondd186292016-12-08 13:42:08 -08003646 for (k = 0; k < nb_iargs; k++) {
Richard Hendersond62816f2018-11-27 20:21:31 -08003647 TCGRegSet i_preferred_regs, o_preferred_regs;
3648
Richard Henderson66792f92019-04-04 09:37:38 +07003649 i = def->args_ct[nb_oargs + k].sort_index;
Richard Hendersondd186292016-12-08 13:42:08 -08003650 arg = op->args[i];
bellardc896fe22008-02-01 10:05:41 +00003651 arg_ct = &def->args_ct[i];
Richard Henderson43439132017-06-19 23:18:10 -07003652 ts = arg_temp(arg);
Richard Henderson40ae5c62013-09-19 08:02:05 -07003653
3654 if (ts->val_type == TEMP_VAL_CONST
Richard Hendersona4fbbd72021-05-03 16:47:37 -07003655 && tcg_target_const_match(ts->val, ts->type, arg_ct->ct)) {
Richard Henderson40ae5c62013-09-19 08:02:05 -07003656 /* constant is OK for instruction */
3657 const_args[i] = 1;
3658 new_args[i] = ts->val;
Richard Hendersond62816f2018-11-27 20:21:31 -08003659 continue;
bellardc896fe22008-02-01 10:05:41 +00003660 }
Richard Henderson40ae5c62013-09-19 08:02:05 -07003661
Richard Hendersond62816f2018-11-27 20:21:31 -08003662 i_preferred_regs = o_preferred_regs = 0;
Richard Hendersonbc2b17e2019-04-04 19:34:19 -07003663 if (arg_ct->ialias) {
Richard Hendersond62816f2018-11-27 20:21:31 -08003664 o_preferred_regs = op->output_pref[arg_ct->alias_index];
Richard Hendersond62816f2018-11-27 20:21:31 -08003665
Richard Hendersonc0522132020-03-29 18:55:52 -07003666 /*
3667 * If the input is readonly, then it cannot also be an
3668 * output and aliased to itself. If the input is not
3669 * dead after the instruction, we must allocate a new
3670 * register and move it.
3671 */
3672 if (temp_readonly(ts) || !IS_DEAD_ARG(i)) {
3673 goto allocate_in_reg;
3674 }
3675
3676 /*
3677 * Check if the current register has already been allocated
3678 * for another input aliased to an output.
3679 */
3680 if (ts->val_type == TEMP_VAL_REG) {
3681 reg = ts->reg;
3682 for (int k2 = 0; k2 < k; k2++) {
3683 int i2 = def->args_ct[nb_oargs + k2].sort_index;
3684 if (def->args_ct[i2].ialias && reg == new_args[i2]) {
3685 goto allocate_in_reg;
Aurelien Jarno7e1df262015-06-04 21:47:07 +02003686 }
3687 }
bellard5ff9d6a2008-02-04 00:37:54 +00003688 }
Richard Hendersonc0522132020-03-29 18:55:52 -07003689 i_preferred_regs = o_preferred_regs;
bellardc896fe22008-02-01 10:05:41 +00003690 }
Richard Hendersond62816f2018-11-27 20:21:31 -08003691
Richard Henderson9be0d082020-09-03 15:19:03 -07003692 temp_load(s, ts, arg_ct->regs, i_allocated_regs, i_preferred_regs);
bellardc896fe22008-02-01 10:05:41 +00003693 reg = ts->reg;
Richard Hendersond62816f2018-11-27 20:21:31 -08003694
Richard Hendersonc0522132020-03-29 18:55:52 -07003695 if (!tcg_regset_test_reg(arg_ct->regs, reg)) {
3696 allocate_in_reg:
3697 /*
3698 * Allocate a new register matching the constraint
3699 * and move the temporary register into it.
3700 */
Richard Hendersond62816f2018-11-27 20:21:31 -08003701 temp_load(s, ts, tcg_target_available_regs[ts->type],
3702 i_allocated_regs, 0);
Richard Henderson9be0d082020-09-03 15:19:03 -07003703 reg = tcg_reg_alloc(s, arg_ct->regs, i_allocated_regs,
Richard Hendersond62816f2018-11-27 20:21:31 -08003704 o_preferred_regs, ts->indirect_base);
Richard Henderson78113e82019-03-16 17:48:18 +00003705 if (!tcg_out_mov(s, ts->type, reg, ts->reg)) {
Richard Henderson240c08d2019-03-16 17:48:32 +00003706 /*
3707 * Cross register class move not supported. Sync the
3708 * temp back to its slot and load from there.
3709 */
3710 temp_sync(s, ts, i_allocated_regs, 0, 0);
3711 tcg_out_ld(s, ts->type, reg,
3712 ts->mem_base->reg, ts->mem_offset);
Richard Henderson78113e82019-03-16 17:48:18 +00003713 }
bellardc896fe22008-02-01 10:05:41 +00003714 }
bellardc896fe22008-02-01 10:05:41 +00003715 new_args[i] = reg;
3716 const_args[i] = 0;
Richard Henderson82790a82016-11-18 08:35:03 +01003717 tcg_regset_set_reg(i_allocated_regs, reg);
bellardc896fe22008-02-01 10:05:41 +00003718 }
3719
Aurelien Jarnoa52ad072012-10-09 21:53:07 +02003720 /* mark dead temporaries and free the associated registers */
3721 for (i = nb_oargs; i < nb_oargs + nb_iargs; i++) {
3722 if (IS_DEAD_ARG(i)) {
Richard Henderson43439132017-06-19 23:18:10 -07003723 temp_dead(s, arg_temp(op->args[i]));
Aurelien Jarnoa52ad072012-10-09 21:53:07 +02003724 }
3725 }
3726
Richard Hendersonb4cb76e2020-10-08 15:21:43 -05003727 if (def->flags & TCG_OPF_COND_BRANCH) {
3728 tcg_reg_alloc_cbranch(s, i_allocated_regs);
3729 } else if (def->flags & TCG_OPF_BB_END) {
Richard Henderson82790a82016-11-18 08:35:03 +01003730 tcg_reg_alloc_bb_end(s, i_allocated_regs);
bellarde8996ee2008-05-23 17:33:39 +00003731 } else {
bellarde8996ee2008-05-23 17:33:39 +00003732 if (def->flags & TCG_OPF_CALL_CLOBBER) {
3733 /* XXX: permit generic clobber register list ? */
Richard Hendersonc8074022016-02-09 10:43:42 +11003734 for (i = 0; i < TCG_TARGET_NB_REGS; i++) {
3735 if (tcg_regset_test_reg(tcg_target_call_clobber_regs, i)) {
Richard Henderson82790a82016-11-18 08:35:03 +01003736 tcg_reg_free(s, i, i_allocated_regs);
bellarde8996ee2008-05-23 17:33:39 +00003737 }
3738 }
Aurelien Jarno3d5c5f82012-10-09 21:53:08 +02003739 }
3740 if (def->flags & TCG_OPF_SIDE_EFFECTS) {
3741 /* sync globals if the op has side effects and might trigger
3742 an exception. */
Richard Henderson82790a82016-11-18 08:35:03 +01003743 sync_globals(s, i_allocated_regs);
bellarde8996ee2008-05-23 17:33:39 +00003744 }
3745
3746 /* satisfy the output constraints */
bellarde8996ee2008-05-23 17:33:39 +00003747 for(k = 0; k < nb_oargs; k++) {
Richard Henderson66792f92019-04-04 09:37:38 +07003748 i = def->args_ct[k].sort_index;
Richard Hendersondd186292016-12-08 13:42:08 -08003749 arg = op->args[i];
bellarde8996ee2008-05-23 17:33:39 +00003750 arg_ct = &def->args_ct[i];
Richard Henderson43439132017-06-19 23:18:10 -07003751 ts = arg_temp(arg);
Richard Hendersond63e3b62019-03-16 17:48:02 +00003752
3753 /* ENV should not be modified. */
Richard Hendersone01fa972020-03-29 10:40:49 -07003754 tcg_debug_assert(!temp_readonly(ts));
Richard Hendersond63e3b62019-03-16 17:48:02 +00003755
Richard Hendersonbc2b17e2019-04-04 19:34:19 -07003756 if (arg_ct->oalias && !const_args[arg_ct->alias_index]) {
bellarde8996ee2008-05-23 17:33:39 +00003757 reg = new_args[arg_ct->alias_index];
Richard Hendersonbc2b17e2019-04-04 19:34:19 -07003758 } else if (arg_ct->newreg) {
Richard Henderson9be0d082020-09-03 15:19:03 -07003759 reg = tcg_reg_alloc(s, arg_ct->regs,
Richard Henderson82790a82016-11-18 08:35:03 +01003760 i_allocated_regs | o_allocated_regs,
Richard Henderson69e37062018-11-27 07:44:51 -08003761 op->output_pref[k], ts->indirect_base);
bellarde8996ee2008-05-23 17:33:39 +00003762 } else {
Richard Henderson9be0d082020-09-03 15:19:03 -07003763 reg = tcg_reg_alloc(s, arg_ct->regs, o_allocated_regs,
Richard Henderson69e37062018-11-27 07:44:51 -08003764 op->output_pref[k], ts->indirect_base);
bellarde8996ee2008-05-23 17:33:39 +00003765 }
Richard Henderson82790a82016-11-18 08:35:03 +01003766 tcg_regset_set_reg(o_allocated_regs, reg);
Richard Hendersond63e3b62019-03-16 17:48:02 +00003767 if (ts->val_type == TEMP_VAL_REG) {
3768 s->reg_to_temp[ts->reg] = NULL;
bellardc896fe22008-02-01 10:05:41 +00003769 }
Richard Hendersond63e3b62019-03-16 17:48:02 +00003770 ts->val_type = TEMP_VAL_REG;
3771 ts->reg = reg;
3772 /*
3773 * Temp value is modified, so the value kept in memory is
3774 * potentially not the same.
3775 */
3776 ts->mem_coherent = 0;
3777 s->reg_to_temp[reg] = ts;
bellarde8996ee2008-05-23 17:33:39 +00003778 new_args[i] = reg;
bellardc896fe22008-02-01 10:05:41 +00003779 }
3780 }
3781
bellardc896fe22008-02-01 10:05:41 +00003782 /* emit instruction */
Richard Hendersond2fd7452017-09-14 13:53:46 -07003783 if (def->flags & TCG_OPF_VECTOR) {
3784 tcg_out_vec_op(s, op->opc, TCGOP_VECL(op), TCGOP_VECE(op),
3785 new_args, const_args);
3786 } else {
3787 tcg_out_op(s, op->opc, new_args, const_args);
3788 }
3789
bellardc896fe22008-02-01 10:05:41 +00003790 /* move the outputs in the correct register if needed */
3791 for(i = 0; i < nb_oargs; i++) {
Richard Henderson43439132017-06-19 23:18:10 -07003792 ts = arg_temp(op->args[i]);
Richard Hendersond63e3b62019-03-16 17:48:02 +00003793
3794 /* ENV should not be modified. */
Richard Hendersone01fa972020-03-29 10:40:49 -07003795 tcg_debug_assert(!temp_readonly(ts));
Richard Hendersond63e3b62019-03-16 17:48:02 +00003796
Aurelien Jarnoec7a8692012-10-09 21:53:07 +02003797 if (NEED_SYNC_ARG(i)) {
Richard Henderson98b4e182018-11-27 15:35:04 -08003798 temp_sync(s, ts, o_allocated_regs, 0, IS_DEAD_ARG(i));
Richard Henderson59d7c142016-06-19 22:59:13 -07003799 } else if (IS_DEAD_ARG(i)) {
Richard Hendersonf8bf00f2013-09-18 15:29:18 -07003800 temp_dead(s, ts);
Aurelien Jarnoec7a8692012-10-09 21:53:07 +02003801 }
bellardc896fe22008-02-01 10:05:41 +00003802 }
3803}
3804
Richard Hendersonefe86b22020-03-31 02:33:21 -07003805static bool tcg_reg_alloc_dup2(TCGContext *s, const TCGOp *op)
3806{
3807 const TCGLifeData arg_life = op->life;
3808 TCGTemp *ots, *itsl, *itsh;
3809 TCGType vtype = TCGOP_VECL(op) + TCG_TYPE_V64;
3810
3811 /* This opcode is only valid for 32-bit hosts, for 64-bit elements. */
3812 tcg_debug_assert(TCG_TARGET_REG_BITS == 32);
3813 tcg_debug_assert(TCGOP_VECE(op) == MO_64);
3814
3815 ots = arg_temp(op->args[0]);
3816 itsl = arg_temp(op->args[1]);
3817 itsh = arg_temp(op->args[2]);
3818
3819 /* ENV should not be modified. */
3820 tcg_debug_assert(!temp_readonly(ots));
3821
3822 /* Allocate the output register now. */
3823 if (ots->val_type != TEMP_VAL_REG) {
3824 TCGRegSet allocated_regs = s->reserved_regs;
3825 TCGRegSet dup_out_regs =
3826 tcg_op_defs[INDEX_op_dup_vec].args_ct[0].regs;
3827
3828 /* Make sure to not spill the input registers. */
3829 if (!IS_DEAD_ARG(1) && itsl->val_type == TEMP_VAL_REG) {
3830 tcg_regset_set_reg(allocated_regs, itsl->reg);
3831 }
3832 if (!IS_DEAD_ARG(2) && itsh->val_type == TEMP_VAL_REG) {
3833 tcg_regset_set_reg(allocated_regs, itsh->reg);
3834 }
3835
3836 ots->reg = tcg_reg_alloc(s, dup_out_regs, allocated_regs,
3837 op->output_pref[0], ots->indirect_base);
3838 ots->val_type = TEMP_VAL_REG;
3839 ots->mem_coherent = 0;
3840 s->reg_to_temp[ots->reg] = ots;
3841 }
3842
3843 /* Promote dup2 of immediates to dupi_vec. */
3844 if (itsl->val_type == TEMP_VAL_CONST && itsh->val_type == TEMP_VAL_CONST) {
3845 uint64_t val = deposit64(itsl->val, 32, 32, itsh->val);
3846 MemOp vece = MO_64;
3847
3848 if (val == dup_const(MO_8, val)) {
3849 vece = MO_8;
3850 } else if (val == dup_const(MO_16, val)) {
3851 vece = MO_16;
3852 } else if (val == dup_const(MO_32, val)) {
3853 vece = MO_32;
3854 }
3855
3856 tcg_out_dupi_vec(s, vtype, vece, ots->reg, val);
3857 goto done;
3858 }
3859
3860 /* If the two inputs form one 64-bit value, try dupm_vec. */
3861 if (itsl + 1 == itsh && itsl->base_type == TCG_TYPE_I64) {
3862 if (!itsl->mem_coherent) {
3863 temp_sync(s, itsl, s->reserved_regs, 0, 0);
3864 }
3865 if (!itsh->mem_coherent) {
3866 temp_sync(s, itsh, s->reserved_regs, 0, 0);
3867 }
3868#ifdef HOST_WORDS_BIGENDIAN
3869 TCGTemp *its = itsh;
3870#else
3871 TCGTemp *its = itsl;
3872#endif
3873 if (tcg_out_dupm_vec(s, vtype, MO_64, ots->reg,
3874 its->mem_base->reg, its->mem_offset)) {
3875 goto done;
3876 }
3877 }
3878
3879 /* Fall back to generic expansion. */
3880 return false;
3881
3882 done:
3883 if (IS_DEAD_ARG(1)) {
3884 temp_dead(s, itsl);
3885 }
3886 if (IS_DEAD_ARG(2)) {
3887 temp_dead(s, itsh);
3888 }
3889 if (NEED_SYNC_ARG(0)) {
3890 temp_sync(s, ots, s->reserved_regs, 0, IS_DEAD_ARG(0));
3891 } else if (IS_DEAD_ARG(0)) {
3892 temp_dead(s, ots);
3893 }
3894 return true;
3895}
3896
bellardb03cce82008-05-10 10:52:05 +00003897#ifdef TCG_TARGET_STACK_GROWSUP
3898#define STACK_DIR(x) (-(x))
3899#else
3900#define STACK_DIR(x) (x)
3901#endif
3902
Richard Hendersondd186292016-12-08 13:42:08 -08003903static void tcg_reg_alloc_call(TCGContext *s, TCGOp *op)
bellardc896fe22008-02-01 10:05:41 +00003904{
Richard Hendersoncd9090a2017-11-14 13:02:51 +01003905 const int nb_oargs = TCGOP_CALLO(op);
3906 const int nb_iargs = TCGOP_CALLI(op);
Richard Hendersondd186292016-12-08 13:42:08 -08003907 const TCGLifeData arg_life = op->life;
Richard Henderson7b7d8b22021-01-30 14:24:25 -08003908 const TCGHelperInfo *info;
Richard Hendersonb6638662013-09-18 14:54:45 -07003909 int flags, nb_regs, i;
3910 TCGReg reg;
Richard Hendersoncf066672014-03-22 20:06:52 -07003911 TCGArg arg;
bellardc896fe22008-02-01 10:05:41 +00003912 TCGTemp *ts;
Richard Hendersond3452f12013-08-20 17:12:38 -07003913 intptr_t stack_offset;
3914 size_t call_stack_size;
Richard Hendersoncf066672014-03-22 20:06:52 -07003915 tcg_insn_unit *func_addr;
3916 int allocate_args;
bellardc896fe22008-02-01 10:05:41 +00003917 TCGRegSet allocated_regs;
bellardc896fe22008-02-01 10:05:41 +00003918
Richard Hendersonfa52e662021-03-18 16:40:07 -06003919 func_addr = tcg_call_func(op);
Richard Henderson7b7d8b22021-01-30 14:24:25 -08003920 info = tcg_call_info(op);
3921 flags = info->flags;
bellardc896fe22008-02-01 10:05:41 +00003922
Stefan Weil6e17d0c2012-09-13 19:37:46 +02003923 nb_regs = ARRAY_SIZE(tcg_target_call_iarg_regs);
Richard Hendersonc45cb8b2014-09-19 13:49:15 -07003924 if (nb_regs > nb_iargs) {
3925 nb_regs = nb_iargs;
Richard Hendersoncf066672014-03-22 20:06:52 -07003926 }
bellardc896fe22008-02-01 10:05:41 +00003927
3928 /* assign stack slots first */
Richard Hendersonc45cb8b2014-09-19 13:49:15 -07003929 call_stack_size = (nb_iargs - nb_regs) * sizeof(tcg_target_long);
bellardc896fe22008-02-01 10:05:41 +00003930 call_stack_size = (call_stack_size + TCG_TARGET_STACK_ALIGN - 1) &
3931 ~(TCG_TARGET_STACK_ALIGN - 1);
bellardb03cce82008-05-10 10:52:05 +00003932 allocate_args = (call_stack_size > TCG_STATIC_CALL_ARGS_SIZE);
3933 if (allocate_args) {
Blue Swirl345649c2011-05-28 07:13:05 +00003934 /* XXX: if more than TCG_STATIC_CALL_ARGS_SIZE is needed,
3935 preallocate call stack */
3936 tcg_abort();
bellardb03cce82008-05-10 10:52:05 +00003937 }
bellard39cf05d2008-05-22 14:59:57 +00003938
3939 stack_offset = TCG_TARGET_CALL_STACK_OFFSET;
Richard Hendersondd186292016-12-08 13:42:08 -08003940 for (i = nb_regs; i < nb_iargs; i++) {
3941 arg = op->args[nb_oargs + i];
bellard39cf05d2008-05-22 14:59:57 +00003942#ifdef TCG_TARGET_STACK_GROWSUP
3943 stack_offset -= sizeof(tcg_target_long);
3944#endif
3945 if (arg != TCG_CALL_DUMMY_ARG) {
Richard Henderson43439132017-06-19 23:18:10 -07003946 ts = arg_temp(arg);
Richard Henderson40ae5c62013-09-19 08:02:05 -07003947 temp_load(s, ts, tcg_target_available_regs[ts->type],
Richard Hendersonb7224522018-11-27 07:48:06 -08003948 s->reserved_regs, 0);
Richard Henderson40ae5c62013-09-19 08:02:05 -07003949 tcg_out_st(s, ts->type, ts->reg, TCG_REG_CALL_STACK, stack_offset);
bellardc896fe22008-02-01 10:05:41 +00003950 }
bellard39cf05d2008-05-22 14:59:57 +00003951#ifndef TCG_TARGET_STACK_GROWSUP
3952 stack_offset += sizeof(tcg_target_long);
3953#endif
bellardc896fe22008-02-01 10:05:41 +00003954 }
3955
3956 /* assign input registers */
Richard Hendersond21369f2017-09-11 11:58:44 -07003957 allocated_regs = s->reserved_regs;
Richard Hendersondd186292016-12-08 13:42:08 -08003958 for (i = 0; i < nb_regs; i++) {
3959 arg = op->args[nb_oargs + i];
bellard39cf05d2008-05-22 14:59:57 +00003960 if (arg != TCG_CALL_DUMMY_ARG) {
Richard Henderson43439132017-06-19 23:18:10 -07003961 ts = arg_temp(arg);
bellard39cf05d2008-05-22 14:59:57 +00003962 reg = tcg_target_call_iarg_regs[i];
Richard Henderson40ae5c62013-09-19 08:02:05 -07003963
bellard39cf05d2008-05-22 14:59:57 +00003964 if (ts->val_type == TEMP_VAL_REG) {
3965 if (ts->reg != reg) {
Richard Henderson4250da12018-12-11 10:25:02 -06003966 tcg_reg_free(s, reg, allocated_regs);
Richard Henderson78113e82019-03-16 17:48:18 +00003967 if (!tcg_out_mov(s, ts->type, reg, ts->reg)) {
Richard Henderson240c08d2019-03-16 17:48:32 +00003968 /*
3969 * Cross register class move not supported. Sync the
3970 * temp back to its slot and load from there.
3971 */
3972 temp_sync(s, ts, allocated_regs, 0, 0);
3973 tcg_out_ld(s, ts->type, reg,
3974 ts->mem_base->reg, ts->mem_offset);
Richard Henderson78113e82019-03-16 17:48:18 +00003975 }
bellard39cf05d2008-05-22 14:59:57 +00003976 }
bellard39cf05d2008-05-22 14:59:57 +00003977 } else {
Richard Hendersonccb1bb62017-09-11 11:25:55 -07003978 TCGRegSet arg_set = 0;
Richard Henderson40ae5c62013-09-19 08:02:05 -07003979
Richard Henderson4250da12018-12-11 10:25:02 -06003980 tcg_reg_free(s, reg, allocated_regs);
Richard Henderson40ae5c62013-09-19 08:02:05 -07003981 tcg_regset_set_reg(arg_set, reg);
Richard Hendersonb7224522018-11-27 07:48:06 -08003982 temp_load(s, ts, arg_set, allocated_regs, 0);
bellardc896fe22008-02-01 10:05:41 +00003983 }
Richard Henderson40ae5c62013-09-19 08:02:05 -07003984
bellard39cf05d2008-05-22 14:59:57 +00003985 tcg_regset_set_reg(allocated_regs, reg);
bellardc896fe22008-02-01 10:05:41 +00003986 }
bellardc896fe22008-02-01 10:05:41 +00003987 }
3988
bellardc896fe22008-02-01 10:05:41 +00003989 /* mark dead temporaries and free the associated registers */
Richard Hendersondd186292016-12-08 13:42:08 -08003990 for (i = nb_oargs; i < nb_iargs + nb_oargs; i++) {
Aurelien Jarno866cb6c2011-05-17 18:25:45 +02003991 if (IS_DEAD_ARG(i)) {
Richard Henderson43439132017-06-19 23:18:10 -07003992 temp_dead(s, arg_temp(op->args[i]));
bellardc896fe22008-02-01 10:05:41 +00003993 }
3994 }
3995
3996 /* clobber call registers */
Richard Hendersonc8074022016-02-09 10:43:42 +11003997 for (i = 0; i < TCG_TARGET_NB_REGS; i++) {
3998 if (tcg_regset_test_reg(tcg_target_call_clobber_regs, i)) {
Richard Hendersonb3915db2013-09-19 10:36:18 -07003999 tcg_reg_free(s, i, allocated_regs);
bellardc896fe22008-02-01 10:05:41 +00004000 }
4001 }
Aurelien Jarno78505272012-10-09 21:53:08 +02004002
4003 /* Save globals if they might be written by the helper, sync them if
4004 they might be read. */
4005 if (flags & TCG_CALL_NO_READ_GLOBALS) {
4006 /* Nothing to do */
4007 } else if (flags & TCG_CALL_NO_WRITE_GLOBALS) {
4008 sync_globals(s, allocated_regs);
4009 } else {
aurel32b9c18f52009-04-06 12:33:59 +00004010 save_globals(s, allocated_regs);
4011 }
bellardc896fe22008-02-01 10:05:41 +00004012
Richard Henderson7b7d8b22021-01-30 14:24:25 -08004013#ifdef CONFIG_TCG_INTERPRETER
4014 {
4015 gpointer hash = (gpointer)(uintptr_t)info->typemask;
4016 ffi_cif *cif = g_hash_table_lookup(ffi_table, hash);
4017 assert(cif != NULL);
4018 tcg_out_call(s, func_addr, cif);
4019 }
4020#else
Richard Hendersoncf066672014-03-22 20:06:52 -07004021 tcg_out_call(s, func_addr);
Richard Henderson7b7d8b22021-01-30 14:24:25 -08004022#endif
bellardc896fe22008-02-01 10:05:41 +00004023
4024 /* assign output registers and emit moves if needed */
4025 for(i = 0; i < nb_oargs; i++) {
Richard Hendersondd186292016-12-08 13:42:08 -08004026 arg = op->args[i];
Richard Henderson43439132017-06-19 23:18:10 -07004027 ts = arg_temp(arg);
Richard Hendersond63e3b62019-03-16 17:48:02 +00004028
4029 /* ENV should not be modified. */
Richard Hendersone01fa972020-03-29 10:40:49 -07004030 tcg_debug_assert(!temp_readonly(ts));
Richard Hendersond63e3b62019-03-16 17:48:02 +00004031
bellardc896fe22008-02-01 10:05:41 +00004032 reg = tcg_target_call_oarg_regs[i];
Aurelien Jarnoeabb7b92016-04-21 10:48:49 +02004033 tcg_debug_assert(s->reg_to_temp[reg] == NULL);
Richard Hendersond63e3b62019-03-16 17:48:02 +00004034 if (ts->val_type == TEMP_VAL_REG) {
4035 s->reg_to_temp[ts->reg] = NULL;
4036 }
4037 ts->val_type = TEMP_VAL_REG;
4038 ts->reg = reg;
4039 ts->mem_coherent = 0;
4040 s->reg_to_temp[reg] = ts;
4041 if (NEED_SYNC_ARG(i)) {
4042 temp_sync(s, ts, allocated_regs, 0, IS_DEAD_ARG(i));
4043 } else if (IS_DEAD_ARG(i)) {
4044 temp_dead(s, ts);
bellardc896fe22008-02-01 10:05:41 +00004045 }
4046 }
bellardc896fe22008-02-01 10:05:41 +00004047}
4048
4049#ifdef CONFIG_PROFILER
4050
Emilio G. Cotac3fac112017-07-05 19:35:06 -04004051/* avoid copy/paste errors */
4052#define PROF_ADD(to, from, field) \
4053 do { \
Stefan Hajnoczid73415a2020-09-23 11:56:46 +01004054 (to)->field += qatomic_read(&((from)->field)); \
Emilio G. Cotac3fac112017-07-05 19:35:06 -04004055 } while (0)
4056
4057#define PROF_MAX(to, from, field) \
4058 do { \
Stefan Hajnoczid73415a2020-09-23 11:56:46 +01004059 typeof((from)->field) val__ = qatomic_read(&((from)->field)); \
Emilio G. Cotac3fac112017-07-05 19:35:06 -04004060 if (val__ > (to)->field) { \
4061 (to)->field = val__; \
4062 } \
4063 } while (0)
4064
4065/* Pass in a zero'ed @prof */
4066static inline
4067void tcg_profile_snapshot(TCGProfile *prof, bool counters, bool table)
4068{
Richard Henderson0e2d61c2021-03-09 23:06:32 -06004069 unsigned int n_ctxs = qatomic_read(&tcg_cur_ctxs);
Emilio G. Cotac3fac112017-07-05 19:35:06 -04004070 unsigned int i;
4071
Emilio G. Cota3468b592017-07-19 18:57:58 -04004072 for (i = 0; i < n_ctxs; i++) {
Stefan Hajnoczid73415a2020-09-23 11:56:46 +01004073 TCGContext *s = qatomic_read(&tcg_ctxs[i]);
Emilio G. Cota3468b592017-07-19 18:57:58 -04004074 const TCGProfile *orig = &s->prof;
Emilio G. Cotac3fac112017-07-05 19:35:06 -04004075
4076 if (counters) {
Emilio G. Cota72fd2ef2018-10-10 10:48:53 -04004077 PROF_ADD(prof, orig, cpu_exec_time);
Emilio G. Cotac3fac112017-07-05 19:35:06 -04004078 PROF_ADD(prof, orig, tb_count1);
4079 PROF_ADD(prof, orig, tb_count);
4080 PROF_ADD(prof, orig, op_count);
4081 PROF_MAX(prof, orig, op_count_max);
4082 PROF_ADD(prof, orig, temp_count);
4083 PROF_MAX(prof, orig, temp_count_max);
4084 PROF_ADD(prof, orig, del_op_count);
4085 PROF_ADD(prof, orig, code_in_len);
4086 PROF_ADD(prof, orig, code_out_len);
4087 PROF_ADD(prof, orig, search_out_len);
4088 PROF_ADD(prof, orig, interm_time);
4089 PROF_ADD(prof, orig, code_time);
4090 PROF_ADD(prof, orig, la_time);
4091 PROF_ADD(prof, orig, opt_time);
4092 PROF_ADD(prof, orig, restore_count);
4093 PROF_ADD(prof, orig, restore_time);
4094 }
4095 if (table) {
4096 int i;
4097
4098 for (i = 0; i < NB_OPS; i++) {
4099 PROF_ADD(prof, orig, table_op_count[i]);
4100 }
4101 }
4102 }
4103}
4104
4105#undef PROF_ADD
4106#undef PROF_MAX
4107
4108static void tcg_profile_snapshot_counters(TCGProfile *prof)
4109{
4110 tcg_profile_snapshot(prof, true, false);
4111}
4112
4113static void tcg_profile_snapshot_table(TCGProfile *prof)
4114{
4115 tcg_profile_snapshot(prof, false, true);
4116}
bellardc896fe22008-02-01 10:05:41 +00004117
Markus Armbrusterd4c51a02019-04-17 21:17:51 +02004118void tcg_dump_op_count(void)
bellardc896fe22008-02-01 10:05:41 +00004119{
Emilio G. Cotac3fac112017-07-05 19:35:06 -04004120 TCGProfile prof = {};
bellardc896fe22008-02-01 10:05:41 +00004121 int i;
zhanghailiangd70724c2014-08-18 15:58:08 +08004122
Emilio G. Cotac3fac112017-07-05 19:35:06 -04004123 tcg_profile_snapshot_table(&prof);
Richard Henderson15fc7da2014-03-30 20:40:35 -07004124 for (i = 0; i < NB_OPS; i++) {
Markus Armbrusterd4c51a02019-04-17 21:17:51 +02004125 qemu_printf("%s %" PRId64 "\n", tcg_op_defs[i].name,
Emilio G. Cotac3fac112017-07-05 19:35:06 -04004126 prof.table_op_count[i]);
bellardc896fe22008-02-01 10:05:41 +00004127 }
bellardc896fe22008-02-01 10:05:41 +00004128}
Emilio G. Cota72fd2ef2018-10-10 10:48:53 -04004129
4130int64_t tcg_cpu_exec_time(void)
4131{
Richard Henderson0e2d61c2021-03-09 23:06:32 -06004132 unsigned int n_ctxs = qatomic_read(&tcg_cur_ctxs);
Emilio G. Cota72fd2ef2018-10-10 10:48:53 -04004133 unsigned int i;
4134 int64_t ret = 0;
4135
4136 for (i = 0; i < n_ctxs; i++) {
Stefan Hajnoczid73415a2020-09-23 11:56:46 +01004137 const TCGContext *s = qatomic_read(&tcg_ctxs[i]);
Emilio G. Cota72fd2ef2018-10-10 10:48:53 -04004138 const TCGProfile *prof = &s->prof;
4139
Stefan Hajnoczid73415a2020-09-23 11:56:46 +01004140 ret += qatomic_read(&prof->cpu_exec_time);
Emilio G. Cota72fd2ef2018-10-10 10:48:53 -04004141 }
4142 return ret;
4143}
Max Filippov246ae242014-11-02 11:04:18 +03004144#else
Markus Armbrusterd4c51a02019-04-17 21:17:51 +02004145void tcg_dump_op_count(void)
Max Filippov246ae242014-11-02 11:04:18 +03004146{
Markus Armbrusterd4c51a02019-04-17 21:17:51 +02004147 qemu_printf("[TCG profiler not compiled]\n");
Max Filippov246ae242014-11-02 11:04:18 +03004148}
Emilio G. Cota72fd2ef2018-10-10 10:48:53 -04004149
4150int64_t tcg_cpu_exec_time(void)
4151{
4152 error_report("%s: TCG profiler not compiled", __func__);
4153 exit(EXIT_FAILURE);
4154}
bellardc896fe22008-02-01 10:05:41 +00004155#endif
4156
4157
Alex Bennée5bd2ec32016-03-15 14:30:16 +00004158int tcg_gen_code(TCGContext *s, TranslationBlock *tb)
bellardc896fe22008-02-01 10:05:41 +00004159{
Emilio G. Cotac3fac112017-07-05 19:35:06 -04004160#ifdef CONFIG_PROFILER
4161 TCGProfile *prof = &s->prof;
4162#endif
Richard Henderson15fa08f2017-11-02 15:19:14 +01004163 int i, num_insns;
4164 TCGOp *op;
bellardc896fe22008-02-01 10:05:41 +00004165
Richard Henderson04fe6402015-09-01 20:07:48 -07004166#ifdef CONFIG_PROFILER
4167 {
Emilio G. Cotac1f543b2018-10-10 10:48:51 -04004168 int n = 0;
Richard Henderson04fe6402015-09-01 20:07:48 -07004169
Richard Henderson15fa08f2017-11-02 15:19:14 +01004170 QTAILQ_FOREACH(op, &s->ops, link) {
4171 n++;
4172 }
Stefan Hajnoczid73415a2020-09-23 11:56:46 +01004173 qatomic_set(&prof->op_count, prof->op_count + n);
Emilio G. Cotac3fac112017-07-05 19:35:06 -04004174 if (n > prof->op_count_max) {
Stefan Hajnoczid73415a2020-09-23 11:56:46 +01004175 qatomic_set(&prof->op_count_max, n);
Richard Henderson04fe6402015-09-01 20:07:48 -07004176 }
4177
4178 n = s->nb_temps;
Stefan Hajnoczid73415a2020-09-23 11:56:46 +01004179 qatomic_set(&prof->temp_count, prof->temp_count + n);
Emilio G. Cotac3fac112017-07-05 19:35:06 -04004180 if (n > prof->temp_count_max) {
Stefan Hajnoczid73415a2020-09-23 11:56:46 +01004181 qatomic_set(&prof->temp_count_max, n);
Richard Henderson04fe6402015-09-01 20:07:48 -07004182 }
4183 }
4184#endif
4185
bellardc896fe22008-02-01 10:05:41 +00004186#ifdef DEBUG_DISAS
Alex Bennéed977e1c2016-03-15 14:30:21 +00004187 if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP)
4188 && qemu_log_in_addr_range(tb->pc))) {
Robert Foleyfc59d2d2019-11-18 16:15:26 -05004189 FILE *logfile = qemu_log_lock();
aliguori93fcfe32009-01-15 22:34:14 +00004190 qemu_log("OP:\n");
Richard Henderson1894f692018-11-27 12:46:00 -08004191 tcg_dump_ops(s, false);
aliguori93fcfe32009-01-15 22:34:14 +00004192 qemu_log("\n");
Robert Foleyfc59d2d2019-11-18 16:15:26 -05004193 qemu_log_unlock(logfile);
bellardc896fe22008-02-01 10:05:41 +00004194 }
4195#endif
4196
Richard Hendersonbef16ab2019-02-07 13:26:40 +00004197#ifdef CONFIG_DEBUG_TCG
4198 /* Ensure all labels referenced have been emitted. */
4199 {
4200 TCGLabel *l;
4201 bool error = false;
4202
4203 QSIMPLEQ_FOREACH(l, &s->labels, next) {
4204 if (unlikely(!l->present) && l->refs) {
4205 qemu_log_mask(CPU_LOG_TB_OP,
4206 "$L%d referenced but not present.\n", l->id);
4207 error = true;
4208 }
4209 }
4210 assert(!error);
4211 }
4212#endif
4213
Aurelien Jarnoc5cc28f2012-09-06 16:47:13 +02004214#ifdef CONFIG_PROFILER
Stefan Hajnoczid73415a2020-09-23 11:56:46 +01004215 qatomic_set(&prof->opt_time, prof->opt_time - profile_getclock());
Aurelien Jarnoc5cc28f2012-09-06 16:47:13 +02004216#endif
4217
Kirill Batuzov8f2e8c02011-07-07 16:37:12 +04004218#ifdef USE_TCG_OPTIMIZATIONS
Richard Hendersonc45cb8b2014-09-19 13:49:15 -07004219 tcg_optimize(s);
Kirill Batuzov8f2e8c02011-07-07 16:37:12 +04004220#endif
4221
bellarda23a9ec2008-05-23 09:52:20 +00004222#ifdef CONFIG_PROFILER
Stefan Hajnoczid73415a2020-09-23 11:56:46 +01004223 qatomic_set(&prof->opt_time, prof->opt_time + profile_getclock());
4224 qatomic_set(&prof->la_time, prof->la_time - profile_getclock());
bellarda23a9ec2008-05-23 09:52:20 +00004225#endif
Aurelien Jarnoc5cc28f2012-09-06 16:47:13 +02004226
Richard Hendersonb4fc67c2018-11-26 14:28:28 -08004227 reachable_code_pass(s);
Richard Hendersonb83eabe2016-11-01 15:56:04 -06004228 liveness_pass_1(s);
Richard Henderson5a184072016-06-23 20:34:33 -07004229
Richard Hendersonb83eabe2016-11-01 15:56:04 -06004230 if (s->nb_indirects > 0) {
Richard Henderson5a184072016-06-23 20:34:33 -07004231#ifdef DEBUG_DISAS
Richard Hendersonb83eabe2016-11-01 15:56:04 -06004232 if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP_IND)
4233 && qemu_log_in_addr_range(tb->pc))) {
Robert Foleyfc59d2d2019-11-18 16:15:26 -05004234 FILE *logfile = qemu_log_lock();
Richard Hendersonb83eabe2016-11-01 15:56:04 -06004235 qemu_log("OP before indirect lowering:\n");
Richard Henderson1894f692018-11-27 12:46:00 -08004236 tcg_dump_ops(s, false);
Richard Hendersonb83eabe2016-11-01 15:56:04 -06004237 qemu_log("\n");
Robert Foleyfc59d2d2019-11-18 16:15:26 -05004238 qemu_log_unlock(logfile);
Richard Hendersonb83eabe2016-11-01 15:56:04 -06004239 }
Richard Henderson5a184072016-06-23 20:34:33 -07004240#endif
Richard Hendersonb83eabe2016-11-01 15:56:04 -06004241 /* Replace indirect temps with direct temps. */
4242 if (liveness_pass_2(s)) {
4243 /* If changes were made, re-run liveness. */
4244 liveness_pass_1(s);
Richard Henderson5a184072016-06-23 20:34:33 -07004245 }
4246 }
Aurelien Jarnoc5cc28f2012-09-06 16:47:13 +02004247
bellarda23a9ec2008-05-23 09:52:20 +00004248#ifdef CONFIG_PROFILER
Stefan Hajnoczid73415a2020-09-23 11:56:46 +01004249 qatomic_set(&prof->la_time, prof->la_time + profile_getclock());
bellarda23a9ec2008-05-23 09:52:20 +00004250#endif
bellardc896fe22008-02-01 10:05:41 +00004251
4252#ifdef DEBUG_DISAS
Alex Bennéed977e1c2016-03-15 14:30:21 +00004253 if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP_OPT)
4254 && qemu_log_in_addr_range(tb->pc))) {
Robert Foleyfc59d2d2019-11-18 16:15:26 -05004255 FILE *logfile = qemu_log_lock();
Aurelien Jarnoc5cc28f2012-09-06 16:47:13 +02004256 qemu_log("OP after optimization and liveness analysis:\n");
Richard Henderson1894f692018-11-27 12:46:00 -08004257 tcg_dump_ops(s, true);
aliguori93fcfe32009-01-15 22:34:14 +00004258 qemu_log("\n");
Robert Foleyfc59d2d2019-11-18 16:15:26 -05004259 qemu_log_unlock(logfile);
bellardc896fe22008-02-01 10:05:41 +00004260 }
4261#endif
4262
4263 tcg_reg_alloc_start(s);
4264
Richard Hendersondb0c51a2020-10-28 12:05:44 -07004265 /*
4266 * Reset the buffer pointers when restarting after overflow.
4267 * TODO: Move this into translate-all.c with the rest of the
4268 * buffer management. Having only this done here is confusing.
4269 */
4270 s->code_buf = tcg_splitwx_to_rw(tb->tc.ptr);
4271 s->code_ptr = s->code_buf;
bellardc896fe22008-02-01 10:05:41 +00004272
Richard Henderson659ef5c2017-07-30 12:30:41 -07004273#ifdef TCG_TARGET_NEED_LDST_LABELS
Laurent Vivier6001f772018-04-30 01:58:40 +02004274 QSIMPLEQ_INIT(&s->ldst_labels);
Richard Henderson659ef5c2017-07-30 12:30:41 -07004275#endif
Richard Henderson57a26942017-07-30 13:13:21 -07004276#ifdef TCG_TARGET_NEED_POOL_LABELS
4277 s->pool_labels = NULL;
4278#endif
Richard Henderson9ecefc82013-10-03 14:51:24 -05004279
Richard Hendersonfca8a502015-09-01 19:11:45 -07004280 num_insns = -1;
Richard Henderson15fa08f2017-11-02 15:19:14 +01004281 QTAILQ_FOREACH(op, &s->ops, link) {
Richard Hendersonc45cb8b2014-09-19 13:49:15 -07004282 TCGOpcode opc = op->opc;
blueswir1b3db8752008-03-08 13:33:42 +00004283
bellardc896fe22008-02-01 10:05:41 +00004284#ifdef CONFIG_PROFILER
Stefan Hajnoczid73415a2020-09-23 11:56:46 +01004285 qatomic_set(&prof->table_op_count[opc], prof->table_op_count[opc] + 1);
bellardc896fe22008-02-01 10:05:41 +00004286#endif
Richard Hendersonc45cb8b2014-09-19 13:49:15 -07004287
4288 switch (opc) {
bellardc896fe22008-02-01 10:05:41 +00004289 case INDEX_op_mov_i32:
bellardc896fe22008-02-01 10:05:41 +00004290 case INDEX_op_mov_i64:
Richard Hendersond2fd7452017-09-14 13:53:46 -07004291 case INDEX_op_mov_vec:
Richard Hendersondd186292016-12-08 13:42:08 -08004292 tcg_reg_alloc_mov(s, op);
bellardc896fe22008-02-01 10:05:41 +00004293 break;
Richard Hendersonbab16712019-03-18 11:20:27 -07004294 case INDEX_op_dup_vec:
4295 tcg_reg_alloc_dup(s, op);
4296 break;
Richard Henderson765b8422015-08-29 12:37:33 -07004297 case INDEX_op_insn_start:
Richard Hendersonfca8a502015-09-01 19:11:45 -07004298 if (num_insns >= 0) {
Richard Henderson9f754622018-06-14 19:57:03 -10004299 size_t off = tcg_current_code_size(s);
4300 s->gen_insn_end_off[num_insns] = off;
4301 /* Assert that we do not overflow our stored offset. */
4302 assert(s->gen_insn_end_off[num_insns] == off);
Richard Hendersonfca8a502015-09-01 19:11:45 -07004303 }
4304 num_insns++;
Richard Hendersonbad729e2015-09-01 15:51:12 -07004305 for (i = 0; i < TARGET_INSN_START_WORDS; ++i) {
4306 target_ulong a;
4307#if TARGET_LONG_BITS > TCG_TARGET_REG_BITS
Richard Hendersonefee3742016-12-08 13:12:08 -08004308 a = deposit64(op->args[i * 2], 32, 32, op->args[i * 2 + 1]);
Richard Hendersonbad729e2015-09-01 15:51:12 -07004309#else
Richard Hendersonefee3742016-12-08 13:12:08 -08004310 a = op->args[i];
Richard Hendersonbad729e2015-09-01 15:51:12 -07004311#endif
Richard Hendersonfca8a502015-09-01 19:11:45 -07004312 s->gen_insn_data[num_insns][i] = a;
Richard Hendersonbad729e2015-09-01 15:51:12 -07004313 }
bellardc896fe22008-02-01 10:05:41 +00004314 break;
bellard5ff9d6a2008-02-04 00:37:54 +00004315 case INDEX_op_discard:
Richard Henderson43439132017-06-19 23:18:10 -07004316 temp_dead(s, arg_temp(op->args[0]));
bellard5ff9d6a2008-02-04 00:37:54 +00004317 break;
bellardc896fe22008-02-01 10:05:41 +00004318 case INDEX_op_set_label:
bellarde8996ee2008-05-23 17:33:39 +00004319 tcg_reg_alloc_bb_end(s, s->reserved_regs);
Richard Henderson92ab8e72020-10-28 18:55:50 -07004320 tcg_out_label(s, arg_label(op->args[0]));
bellardc896fe22008-02-01 10:05:41 +00004321 break;
4322 case INDEX_op_call:
Richard Hendersondd186292016-12-08 13:42:08 -08004323 tcg_reg_alloc_call(s, op);
Richard Hendersonc45cb8b2014-09-19 13:49:15 -07004324 break;
Richard Hendersonefe86b22020-03-31 02:33:21 -07004325 case INDEX_op_dup2_vec:
4326 if (tcg_reg_alloc_dup2(s, op)) {
4327 break;
4328 }
4329 /* fall through */
bellardc896fe22008-02-01 10:05:41 +00004330 default:
Richard Henderson25c4d9c2011-08-17 14:11:46 -07004331 /* Sanity check that we've not introduced any unhandled opcodes. */
Richard Hendersonbe0f34b2017-08-17 07:43:20 -07004332 tcg_debug_assert(tcg_op_supported(opc));
bellardc896fe22008-02-01 10:05:41 +00004333 /* Note: in order to speed up the code, it would be much
4334 faster to have specialized register allocator functions for
4335 some common argument patterns */
Richard Hendersondd186292016-12-08 13:42:08 -08004336 tcg_reg_alloc_op(s, op);
bellardc896fe22008-02-01 10:05:41 +00004337 break;
4338 }
Aurelien Jarno8d8fdba2016-04-21 10:48:50 +02004339#ifdef CONFIG_DEBUG_TCG
bellardc896fe22008-02-01 10:05:41 +00004340 check_regs(s);
4341#endif
Richard Hendersonb125f9d2015-09-22 13:01:15 -07004342 /* Test for (pending) buffer overflow. The assumption is that any
4343 one operation beginning below the high water mark cannot overrun
4344 the buffer completely. Thus we can test for overflow after
4345 generating code without having to check during generation. */
John Clarke644da9b2015-11-19 10:30:50 +01004346 if (unlikely((void *)s->code_ptr > s->code_gen_highwater)) {
Richard Hendersonb125f9d2015-09-22 13:01:15 -07004347 return -1;
4348 }
Richard Henderson6e6c4ef2019-04-15 22:06:39 -10004349 /* Test for TB overflow, as seen by gen_insn_end_off. */
4350 if (unlikely(tcg_current_code_size(s) > UINT16_MAX)) {
4351 return -2;
4352 }
bellardc896fe22008-02-01 10:05:41 +00004353 }
Richard Hendersonfca8a502015-09-01 19:11:45 -07004354 tcg_debug_assert(num_insns >= 0);
4355 s->gen_insn_end_off[num_insns] = tcg_current_code_size(s);
Richard Hendersonc45cb8b2014-09-19 13:49:15 -07004356
Yeongkyoon Leeb76f0d82012-10-31 16:04:25 +09004357 /* Generate TB finalization at the end of block */
Richard Henderson659ef5c2017-07-30 12:30:41 -07004358#ifdef TCG_TARGET_NEED_LDST_LABELS
Richard Hendersonaeee05f2019-04-21 14:51:00 -07004359 i = tcg_out_ldst_finalize(s);
4360 if (i < 0) {
4361 return i;
Richard Henderson23dceda2015-12-02 13:59:59 -08004362 }
Richard Henderson659ef5c2017-07-30 12:30:41 -07004363#endif
Richard Henderson57a26942017-07-30 13:13:21 -07004364#ifdef TCG_TARGET_NEED_POOL_LABELS
Richard Henderson17689872019-04-21 13:51:56 -07004365 i = tcg_out_pool_finalize(s);
4366 if (i < 0) {
4367 return i;
Richard Henderson57a26942017-07-30 13:13:21 -07004368 }
4369#endif
Richard Henderson7ecd02a2019-04-21 13:34:35 -07004370 if (!tcg_resolve_relocs(s)) {
4371 return -2;
4372 }
bellardc896fe22008-02-01 10:05:41 +00004373
Richard Hendersondf5d2b12020-12-12 09:08:02 -06004374#ifndef CONFIG_TCG_INTERPRETER
bellardc896fe22008-02-01 10:05:41 +00004375 /* flush instruction cache */
Richard Hendersondb0c51a2020-10-28 12:05:44 -07004376 flush_idcache_range((uintptr_t)tcg_splitwx_to_rx(s->code_buf),
4377 (uintptr_t)s->code_buf,
Richard Henderson1da8de32020-12-12 10:38:21 -06004378 tcg_ptr_byte_diff(s->code_ptr, s->code_buf));
Richard Hendersondf5d2b12020-12-12 09:08:02 -06004379#endif
Stefan Weil2aeabc02012-03-02 23:30:07 +01004380
Richard Henderson1813e172014-03-28 12:56:22 -07004381 return tcg_current_code_size(s);
bellardc896fe22008-02-01 10:05:41 +00004382}
4383
bellarda23a9ec2008-05-23 09:52:20 +00004384#ifdef CONFIG_PROFILER
Markus Armbruster3de2faa2019-04-17 21:17:52 +02004385void tcg_dump_info(void)
bellarda23a9ec2008-05-23 09:52:20 +00004386{
Emilio G. Cotac3fac112017-07-05 19:35:06 -04004387 TCGProfile prof = {};
4388 const TCGProfile *s;
4389 int64_t tb_count;
4390 int64_t tb_div_count;
4391 int64_t tot;
4392
4393 tcg_profile_snapshot_counters(&prof);
4394 s = &prof;
4395 tb_count = s->tb_count;
4396 tb_div_count = tb_count ? tb_count : 1;
4397 tot = s->interm_time + s->code_time;
bellarda23a9ec2008-05-23 09:52:20 +00004398
Markus Armbruster3de2faa2019-04-17 21:17:52 +02004399 qemu_printf("JIT cycles %" PRId64 " (%0.3f s at 2.4 GHz)\n",
bellarda23a9ec2008-05-23 09:52:20 +00004400 tot, tot / 2.4e9);
Markus Armbruster3de2faa2019-04-17 21:17:52 +02004401 qemu_printf("translated TBs %" PRId64 " (aborted=%" PRId64
4402 " %0.1f%%)\n",
Richard Hendersonfca8a502015-09-01 19:11:45 -07004403 tb_count, s->tb_count1 - tb_count,
4404 (double)(s->tb_count1 - s->tb_count)
4405 / (s->tb_count1 ? s->tb_count1 : 1) * 100.0);
Markus Armbruster3de2faa2019-04-17 21:17:52 +02004406 qemu_printf("avg ops/TB %0.1f max=%d\n",
Richard Hendersonfca8a502015-09-01 19:11:45 -07004407 (double)s->op_count / tb_div_count, s->op_count_max);
Markus Armbruster3de2faa2019-04-17 21:17:52 +02004408 qemu_printf("deleted ops/TB %0.2f\n",
Richard Hendersonfca8a502015-09-01 19:11:45 -07004409 (double)s->del_op_count / tb_div_count);
Markus Armbruster3de2faa2019-04-17 21:17:52 +02004410 qemu_printf("avg temps/TB %0.2f max=%d\n",
Richard Hendersonfca8a502015-09-01 19:11:45 -07004411 (double)s->temp_count / tb_div_count, s->temp_count_max);
Markus Armbruster3de2faa2019-04-17 21:17:52 +02004412 qemu_printf("avg host code/TB %0.1f\n",
Richard Hendersonfca8a502015-09-01 19:11:45 -07004413 (double)s->code_out_len / tb_div_count);
Markus Armbruster3de2faa2019-04-17 21:17:52 +02004414 qemu_printf("avg search data/TB %0.1f\n",
Richard Hendersonfca8a502015-09-01 19:11:45 -07004415 (double)s->search_out_len / tb_div_count);
bellarda23a9ec2008-05-23 09:52:20 +00004416
Markus Armbruster3de2faa2019-04-17 21:17:52 +02004417 qemu_printf("cycles/op %0.1f\n",
bellarda23a9ec2008-05-23 09:52:20 +00004418 s->op_count ? (double)tot / s->op_count : 0);
Markus Armbruster3de2faa2019-04-17 21:17:52 +02004419 qemu_printf("cycles/in byte %0.1f\n",
bellarda23a9ec2008-05-23 09:52:20 +00004420 s->code_in_len ? (double)tot / s->code_in_len : 0);
Markus Armbruster3de2faa2019-04-17 21:17:52 +02004421 qemu_printf("cycles/out byte %0.1f\n",
bellarda23a9ec2008-05-23 09:52:20 +00004422 s->code_out_len ? (double)tot / s->code_out_len : 0);
Markus Armbruster3de2faa2019-04-17 21:17:52 +02004423 qemu_printf("cycles/search byte %0.1f\n",
Richard Hendersonfca8a502015-09-01 19:11:45 -07004424 s->search_out_len ? (double)tot / s->search_out_len : 0);
4425 if (tot == 0) {
bellarda23a9ec2008-05-23 09:52:20 +00004426 tot = 1;
Richard Hendersonfca8a502015-09-01 19:11:45 -07004427 }
Markus Armbruster3de2faa2019-04-17 21:17:52 +02004428 qemu_printf(" gen_interm time %0.1f%%\n",
bellarda23a9ec2008-05-23 09:52:20 +00004429 (double)s->interm_time / tot * 100.0);
Markus Armbruster3de2faa2019-04-17 21:17:52 +02004430 qemu_printf(" gen_code time %0.1f%%\n",
bellarda23a9ec2008-05-23 09:52:20 +00004431 (double)s->code_time / tot * 100.0);
Markus Armbruster3de2faa2019-04-17 21:17:52 +02004432 qemu_printf("optim./code time %0.1f%%\n",
Aurelien Jarnoc5cc28f2012-09-06 16:47:13 +02004433 (double)s->opt_time / (s->code_time ? s->code_time : 1)
4434 * 100.0);
Markus Armbruster3de2faa2019-04-17 21:17:52 +02004435 qemu_printf("liveness/code time %0.1f%%\n",
bellarda23a9ec2008-05-23 09:52:20 +00004436 (double)s->la_time / (s->code_time ? s->code_time : 1) * 100.0);
Markus Armbruster3de2faa2019-04-17 21:17:52 +02004437 qemu_printf("cpu_restore count %" PRId64 "\n",
bellarda23a9ec2008-05-23 09:52:20 +00004438 s->restore_count);
Markus Armbruster3de2faa2019-04-17 21:17:52 +02004439 qemu_printf(" avg cycles %0.1f\n",
bellarda23a9ec2008-05-23 09:52:20 +00004440 s->restore_count ? (double)s->restore_time / s->restore_count : 0);
bellarda23a9ec2008-05-23 09:52:20 +00004441}
4442#else
Markus Armbruster3de2faa2019-04-17 21:17:52 +02004443void tcg_dump_info(void)
bellarda23a9ec2008-05-23 09:52:20 +00004444{
Markus Armbruster3de2faa2019-04-17 21:17:52 +02004445 qemu_printf("[TCG profiler not compiled]\n");
bellarda23a9ec2008-05-23 09:52:20 +00004446}
4447#endif
Richard Henderson813da622012-03-19 12:25:11 -07004448
4449#ifdef ELF_HOST_MACHINE
Richard Henderson5872bbf2012-03-24 10:47:36 -07004450/* In order to use this feature, the backend needs to do three things:
4451
4452 (1) Define ELF_HOST_MACHINE to indicate both what value to
4453 put into the ELF image and to indicate support for the feature.
4454
4455 (2) Define tcg_register_jit. This should create a buffer containing
4456 the contents of a .debug_frame section that describes the post-
4457 prologue unwind info for the tcg machine.
4458
4459 (3) Call tcg_register_jit_int, with the constructed .debug_frame.
4460*/
Richard Henderson813da622012-03-19 12:25:11 -07004461
4462/* Begin GDB interface. THE FOLLOWING MUST MATCH GDB DOCS. */
4463typedef enum {
4464 JIT_NOACTION = 0,
4465 JIT_REGISTER_FN,
4466 JIT_UNREGISTER_FN
4467} jit_actions_t;
4468
4469struct jit_code_entry {
4470 struct jit_code_entry *next_entry;
4471 struct jit_code_entry *prev_entry;
4472 const void *symfile_addr;
4473 uint64_t symfile_size;
4474};
4475
4476struct jit_descriptor {
4477 uint32_t version;
4478 uint32_t action_flag;
4479 struct jit_code_entry *relevant_entry;
4480 struct jit_code_entry *first_entry;
4481};
4482
4483void __jit_debug_register_code(void) __attribute__((noinline));
4484void __jit_debug_register_code(void)
4485{
4486 asm("");
4487}
4488
4489/* Must statically initialize the version, because GDB may check
4490 the version before we can set it. */
4491struct jit_descriptor __jit_debug_descriptor = { 1, 0, 0, 0 };
4492
4493/* End GDB interface. */
4494
4495static int find_string(const char *strtab, const char *str)
4496{
4497 const char *p = strtab + 1;
4498
4499 while (1) {
4500 if (strcmp(p, str) == 0) {
4501 return p - strtab;
4502 }
4503 p += strlen(p) + 1;
4504 }
4505}
4506
Richard Henderson755bf9e2020-10-29 09:17:30 -07004507static void tcg_register_jit_int(const void *buf_ptr, size_t buf_size,
Richard Henderson2c907842014-05-15 12:48:01 -07004508 const void *debug_frame,
4509 size_t debug_frame_size)
Richard Henderson813da622012-03-19 12:25:11 -07004510{
Richard Henderson5872bbf2012-03-24 10:47:36 -07004511 struct __attribute__((packed)) DebugInfo {
4512 uint32_t len;
4513 uint16_t version;
4514 uint32_t abbrev;
4515 uint8_t ptr_size;
4516 uint8_t cu_die;
4517 uint16_t cu_lang;
4518 uintptr_t cu_low_pc;
4519 uintptr_t cu_high_pc;
4520 uint8_t fn_die;
4521 char fn_name[16];
4522 uintptr_t fn_low_pc;
4523 uintptr_t fn_high_pc;
4524 uint8_t cu_eoc;
4525 };
Richard Henderson813da622012-03-19 12:25:11 -07004526
4527 struct ElfImage {
4528 ElfW(Ehdr) ehdr;
4529 ElfW(Phdr) phdr;
Richard Henderson5872bbf2012-03-24 10:47:36 -07004530 ElfW(Shdr) shdr[7];
4531 ElfW(Sym) sym[2];
4532 struct DebugInfo di;
4533 uint8_t da[24];
4534 char str[80];
4535 };
4536
4537 struct ElfImage *img;
4538
4539 static const struct ElfImage img_template = {
4540 .ehdr = {
4541 .e_ident[EI_MAG0] = ELFMAG0,
4542 .e_ident[EI_MAG1] = ELFMAG1,
4543 .e_ident[EI_MAG2] = ELFMAG2,
4544 .e_ident[EI_MAG3] = ELFMAG3,
4545 .e_ident[EI_CLASS] = ELF_CLASS,
4546 .e_ident[EI_DATA] = ELF_DATA,
4547 .e_ident[EI_VERSION] = EV_CURRENT,
4548 .e_type = ET_EXEC,
4549 .e_machine = ELF_HOST_MACHINE,
4550 .e_version = EV_CURRENT,
4551 .e_phoff = offsetof(struct ElfImage, phdr),
4552 .e_shoff = offsetof(struct ElfImage, shdr),
4553 .e_ehsize = sizeof(ElfW(Shdr)),
4554 .e_phentsize = sizeof(ElfW(Phdr)),
4555 .e_phnum = 1,
4556 .e_shentsize = sizeof(ElfW(Shdr)),
4557 .e_shnum = ARRAY_SIZE(img->shdr),
4558 .e_shstrndx = ARRAY_SIZE(img->shdr) - 1,
Richard Hendersonabbb3ea2012-03-24 10:47:37 -07004559#ifdef ELF_HOST_FLAGS
4560 .e_flags = ELF_HOST_FLAGS,
4561#endif
4562#ifdef ELF_OSABI
4563 .e_ident[EI_OSABI] = ELF_OSABI,
4564#endif
Richard Henderson5872bbf2012-03-24 10:47:36 -07004565 },
4566 .phdr = {
4567 .p_type = PT_LOAD,
4568 .p_flags = PF_X,
4569 },
4570 .shdr = {
4571 [0] = { .sh_type = SHT_NULL },
4572 /* Trick: The contents of code_gen_buffer are not present in
4573 this fake ELF file; that got allocated elsewhere. Therefore
4574 we mark .text as SHT_NOBITS (similar to .bss) so that readers
4575 will not look for contents. We can record any address. */
4576 [1] = { /* .text */
4577 .sh_type = SHT_NOBITS,
4578 .sh_flags = SHF_EXECINSTR | SHF_ALLOC,
4579 },
4580 [2] = { /* .debug_info */
4581 .sh_type = SHT_PROGBITS,
4582 .sh_offset = offsetof(struct ElfImage, di),
4583 .sh_size = sizeof(struct DebugInfo),
4584 },
4585 [3] = { /* .debug_abbrev */
4586 .sh_type = SHT_PROGBITS,
4587 .sh_offset = offsetof(struct ElfImage, da),
4588 .sh_size = sizeof(img->da),
4589 },
4590 [4] = { /* .debug_frame */
4591 .sh_type = SHT_PROGBITS,
4592 .sh_offset = sizeof(struct ElfImage),
4593 },
4594 [5] = { /* .symtab */
4595 .sh_type = SHT_SYMTAB,
4596 .sh_offset = offsetof(struct ElfImage, sym),
4597 .sh_size = sizeof(img->sym),
4598 .sh_info = 1,
4599 .sh_link = ARRAY_SIZE(img->shdr) - 1,
4600 .sh_entsize = sizeof(ElfW(Sym)),
4601 },
4602 [6] = { /* .strtab */
4603 .sh_type = SHT_STRTAB,
4604 .sh_offset = offsetof(struct ElfImage, str),
4605 .sh_size = sizeof(img->str),
4606 }
4607 },
4608 .sym = {
4609 [1] = { /* code_gen_buffer */
4610 .st_info = ELF_ST_INFO(STB_GLOBAL, STT_FUNC),
4611 .st_shndx = 1,
4612 }
4613 },
4614 .di = {
4615 .len = sizeof(struct DebugInfo) - 4,
4616 .version = 2,
4617 .ptr_size = sizeof(void *),
4618 .cu_die = 1,
4619 .cu_lang = 0x8001, /* DW_LANG_Mips_Assembler */
4620 .fn_die = 2,
4621 .fn_name = "code_gen_buffer"
4622 },
4623 .da = {
4624 1, /* abbrev number (the cu) */
4625 0x11, 1, /* DW_TAG_compile_unit, has children */
4626 0x13, 0x5, /* DW_AT_language, DW_FORM_data2 */
4627 0x11, 0x1, /* DW_AT_low_pc, DW_FORM_addr */
4628 0x12, 0x1, /* DW_AT_high_pc, DW_FORM_addr */
4629 0, 0, /* end of abbrev */
4630 2, /* abbrev number (the fn) */
4631 0x2e, 0, /* DW_TAG_subprogram, no children */
4632 0x3, 0x8, /* DW_AT_name, DW_FORM_string */
4633 0x11, 0x1, /* DW_AT_low_pc, DW_FORM_addr */
4634 0x12, 0x1, /* DW_AT_high_pc, DW_FORM_addr */
4635 0, 0, /* end of abbrev */
4636 0 /* no more abbrev */
4637 },
4638 .str = "\0" ".text\0" ".debug_info\0" ".debug_abbrev\0"
4639 ".debug_frame\0" ".symtab\0" ".strtab\0" "code_gen_buffer",
Richard Henderson813da622012-03-19 12:25:11 -07004640 };
4641
4642 /* We only need a single jit entry; statically allocate it. */
4643 static struct jit_code_entry one_entry;
4644
Richard Henderson5872bbf2012-03-24 10:47:36 -07004645 uintptr_t buf = (uintptr_t)buf_ptr;
Richard Henderson813da622012-03-19 12:25:11 -07004646 size_t img_size = sizeof(struct ElfImage) + debug_frame_size;
Richard Henderson2c907842014-05-15 12:48:01 -07004647 DebugFrameHeader *dfh;
Richard Henderson813da622012-03-19 12:25:11 -07004648
Richard Henderson5872bbf2012-03-24 10:47:36 -07004649 img = g_malloc(img_size);
4650 *img = img_template;
Richard Henderson813da622012-03-19 12:25:11 -07004651
Richard Henderson5872bbf2012-03-24 10:47:36 -07004652 img->phdr.p_vaddr = buf;
4653 img->phdr.p_paddr = buf;
4654 img->phdr.p_memsz = buf_size;
Richard Henderson813da622012-03-19 12:25:11 -07004655
Richard Henderson5872bbf2012-03-24 10:47:36 -07004656 img->shdr[1].sh_name = find_string(img->str, ".text");
4657 img->shdr[1].sh_addr = buf;
4658 img->shdr[1].sh_size = buf_size;
Richard Henderson813da622012-03-19 12:25:11 -07004659
Richard Henderson5872bbf2012-03-24 10:47:36 -07004660 img->shdr[2].sh_name = find_string(img->str, ".debug_info");
4661 img->shdr[3].sh_name = find_string(img->str, ".debug_abbrev");
4662
4663 img->shdr[4].sh_name = find_string(img->str, ".debug_frame");
4664 img->shdr[4].sh_size = debug_frame_size;
4665
4666 img->shdr[5].sh_name = find_string(img->str, ".symtab");
4667 img->shdr[6].sh_name = find_string(img->str, ".strtab");
4668
4669 img->sym[1].st_name = find_string(img->str, "code_gen_buffer");
4670 img->sym[1].st_value = buf;
4671 img->sym[1].st_size = buf_size;
4672
4673 img->di.cu_low_pc = buf;
Richard Henderson45aba092013-05-24 14:16:14 -07004674 img->di.cu_high_pc = buf + buf_size;
Richard Henderson5872bbf2012-03-24 10:47:36 -07004675 img->di.fn_low_pc = buf;
Richard Henderson45aba092013-05-24 14:16:14 -07004676 img->di.fn_high_pc = buf + buf_size;
Richard Henderson813da622012-03-19 12:25:11 -07004677
Richard Henderson2c907842014-05-15 12:48:01 -07004678 dfh = (DebugFrameHeader *)(img + 1);
4679 memcpy(dfh, debug_frame, debug_frame_size);
4680 dfh->fde.func_start = buf;
4681 dfh->fde.func_len = buf_size;
4682
Richard Henderson813da622012-03-19 12:25:11 -07004683#ifdef DEBUG_JIT
4684 /* Enable this block to be able to debug the ELF image file creation.
4685 One can use readelf, objdump, or other inspection utilities. */
4686 {
4687 FILE *f = fopen("/tmp/qemu.jit", "w+b");
4688 if (f) {
Richard Henderson5872bbf2012-03-24 10:47:36 -07004689 if (fwrite(img, img_size, 1, f) != img_size) {
Richard Henderson813da622012-03-19 12:25:11 -07004690 /* Avoid stupid unused return value warning for fwrite. */
4691 }
4692 fclose(f);
4693 }
4694 }
4695#endif
4696
4697 one_entry.symfile_addr = img;
4698 one_entry.symfile_size = img_size;
4699
4700 __jit_debug_descriptor.action_flag = JIT_REGISTER_FN;
4701 __jit_debug_descriptor.relevant_entry = &one_entry;
4702 __jit_debug_descriptor.first_entry = &one_entry;
4703 __jit_debug_register_code();
4704}
4705#else
Richard Henderson5872bbf2012-03-24 10:47:36 -07004706/* No support for the feature. Provide the entry point expected by exec.c,
4707 and implement the internal function we declared earlier. */
Richard Henderson813da622012-03-19 12:25:11 -07004708
Richard Henderson755bf9e2020-10-29 09:17:30 -07004709static void tcg_register_jit_int(const void *buf, size_t size,
Richard Henderson2c907842014-05-15 12:48:01 -07004710 const void *debug_frame,
4711 size_t debug_frame_size)
Richard Henderson813da622012-03-19 12:25:11 -07004712{
4713}
4714
Richard Henderson755bf9e2020-10-29 09:17:30 -07004715void tcg_register_jit(const void *buf, size_t buf_size)
Richard Henderson813da622012-03-19 12:25:11 -07004716{
4717}
4718#endif /* ELF_HOST_MACHINE */
Richard Hendersondb432672017-09-15 14:11:45 -07004719
4720#if !TCG_TARGET_MAYBE_vec
4721void tcg_expand_vec_op(TCGOpcode o, TCGType t, unsigned e, TCGArg a0, ...)
4722{
4723 g_assert_not_reached();
4724}
4725#endif