blob: e9ef16b3c6bd7677e77f327e85ce3e968847cf35 [file] [log] [blame]
Kirill Batuzov8f2e8c02011-07-07 16:37:12 +04001/*
2 * Optimizations for Tiny Code Generator for QEMU
3 *
4 * Copyright (c) 2010 Samsung Electronics.
5 * Contributed by Kirill Batuzov <batuzovk@ispras.ru>
6 *
7 * Permission is hereby granted, free of charge, to any person obtaining a copy
8 * of this software and associated documentation files (the "Software"), to deal
9 * in the Software without restriction, including without limitation the rights
10 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
11 * copies of the Software, and to permit persons to whom the Software is
12 * furnished to do so, subject to the following conditions:
13 *
14 * The above copyright notice and this permission notice shall be included in
15 * all copies or substantial portions of the Software.
16 *
17 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
18 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
19 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
20 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
21 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
22 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
23 * THE SOFTWARE.
24 */
25
Peter Maydell757e7252016-01-26 18:17:08 +000026#include "qemu/osdep.h"
Richard Henderson9531c072021-08-26 06:51:39 -070027#include "qemu/int128.h"
Richard Hendersonab84dc32023-08-23 23:04:24 -070028#include "qemu/interval-tree.h"
Richard Hendersonad3d0e42023-03-28 18:17:24 -070029#include "tcg/tcg-op-common.h"
Richard Henderson90163902021-03-18 10:21:45 -060030#include "tcg-internal.h"
Kirill Batuzov8f2e8c02011-07-07 16:37:12 +040031
Kirill Batuzov8f2e8c02011-07-07 16:37:12 +040032#define CASE_OP_32_64(x) \
33 glue(glue(case INDEX_op_, x), _i32): \
34 glue(glue(case INDEX_op_, x), _i64)
Kirill Batuzov8f2e8c02011-07-07 16:37:12 +040035
Richard Henderson170ba882017-11-22 09:07:11 +010036#define CASE_OP_32_64_VEC(x) \
37 glue(glue(case INDEX_op_, x), _i32): \
38 glue(glue(case INDEX_op_, x), _i64): \
39 glue(glue(case INDEX_op_, x), _vec)
40
Richard Hendersonab84dc32023-08-23 23:04:24 -070041typedef struct MemCopyInfo {
42 IntervalTreeNode itree;
43 QSIMPLEQ_ENTRY (MemCopyInfo) next;
44 TCGTemp *ts;
45 TCGType type;
46} MemCopyInfo;
47
Richard Henderson6fcb98e2020-03-30 17:44:30 -070048typedef struct TempOptInfo {
Aurelien Jarnob41059d2015-07-27 12:41:44 +020049 bool is_const;
Richard Henderson63490392017-06-20 13:43:15 -070050 TCGTemp *prev_copy;
51 TCGTemp *next_copy;
Richard Hendersonab84dc32023-08-23 23:04:24 -070052 QSIMPLEQ_HEAD(, MemCopyInfo) mem_copy;
Richard Henderson54795542020-09-06 16:21:32 -070053 uint64_t val;
Richard Hendersonb1fde412021-08-23 13:07:49 -070054 uint64_t z_mask; /* mask bit is 0 if and only if value bit is 0 */
Richard Henderson57fe5c62021-08-26 12:04:46 -070055 uint64_t s_mask; /* a left-aligned mask of clrsb(value) bits. */
Richard Henderson6fcb98e2020-03-30 17:44:30 -070056} TempOptInfo;
Kirill Batuzov22613af2011-07-07 16:37:13 +040057
Richard Henderson3b3f8472021-08-23 22:06:31 -070058typedef struct OptContext {
Richard Hendersondc849882021-08-24 07:13:45 -070059 TCGContext *tcg;
Richard Hendersond0ed5152021-08-24 07:38:39 -070060 TCGOp *prev_mb;
Richard Henderson3b3f8472021-08-23 22:06:31 -070061 TCGTempSet temps_used;
Richard Henderson137f1f42021-08-24 08:49:25 -070062
Richard Hendersonab84dc32023-08-23 23:04:24 -070063 IntervalTreeRoot mem_copy;
64 QSIMPLEQ_HEAD(, MemCopyInfo) mem_free;
65
Richard Henderson137f1f42021-08-24 08:49:25 -070066 /* In flight values from optimization. */
Richard Hendersonfae450b2021-08-25 22:42:19 -070067 uint64_t a_mask; /* mask bit is 0 iff value identical to first input */
68 uint64_t z_mask; /* mask bit is 0 iff value bit is 0 */
Richard Henderson57fe5c62021-08-26 12:04:46 -070069 uint64_t s_mask; /* mask of clrsb(value) bits */
Richard Henderson67f84c92021-08-25 08:00:20 -070070 TCGType type;
Richard Henderson3b3f8472021-08-23 22:06:31 -070071} OptContext;
72
Richard Henderson57fe5c62021-08-26 12:04:46 -070073/* Calculate the smask for a specific value. */
74static uint64_t smask_from_value(uint64_t value)
75{
76 int rep = clrsb64(value);
77 return ~(~0ull >> rep);
78}
79
80/*
81 * Calculate the smask for a given set of known-zeros.
82 * If there are lots of zeros on the left, we can consider the remainder
83 * an unsigned field, and thus the corresponding signed field is one bit
84 * larger.
85 */
86static uint64_t smask_from_zmask(uint64_t zmask)
87{
88 /*
89 * Only the 0 bits are significant for zmask, thus the msb itself
90 * must be zero, else we have no sign information.
91 */
92 int rep = clz64(zmask);
93 if (rep == 0) {
94 return 0;
95 }
96 rep -= 1;
97 return ~(~0ull >> rep);
98}
99
Richard Henderson93a967f2021-08-26 13:24:59 -0700100/*
101 * Recreate a properly left-aligned smask after manipulation.
102 * Some bit-shuffling, particularly shifts and rotates, may
103 * retain sign bits on the left, but may scatter disconnected
104 * sign bits on the right. Retain only what remains to the left.
105 */
106static uint64_t smask_from_smask(int64_t smask)
107{
108 /* Only the 1 bits are significant for smask */
109 return smask_from_zmask(~smask);
110}
111
Richard Henderson6fcb98e2020-03-30 17:44:30 -0700112static inline TempOptInfo *ts_info(TCGTemp *ts)
Aurelien Jarnod9c769c2015-07-27 12:41:44 +0200113{
Richard Henderson63490392017-06-20 13:43:15 -0700114 return ts->state_ptr;
Aurelien Jarnod9c769c2015-07-27 12:41:44 +0200115}
116
Richard Henderson6fcb98e2020-03-30 17:44:30 -0700117static inline TempOptInfo *arg_info(TCGArg arg)
Aurelien Jarnod9c769c2015-07-27 12:41:44 +0200118{
Richard Henderson63490392017-06-20 13:43:15 -0700119 return ts_info(arg_temp(arg));
120}
121
122static inline bool ts_is_const(TCGTemp *ts)
123{
124 return ts_info(ts)->is_const;
125}
126
Richard Henderson27cdb852023-10-23 11:38:00 -0700127static inline bool ts_is_const_val(TCGTemp *ts, uint64_t val)
128{
129 TempOptInfo *ti = ts_info(ts);
130 return ti->is_const && ti->val == val;
131}
132
Richard Henderson63490392017-06-20 13:43:15 -0700133static inline bool arg_is_const(TCGArg arg)
134{
135 return ts_is_const(arg_temp(arg));
136}
137
Richard Henderson27cdb852023-10-23 11:38:00 -0700138static inline bool arg_is_const_val(TCGArg arg, uint64_t val)
139{
140 return ts_is_const_val(arg_temp(arg), val);
141}
142
Richard Henderson63490392017-06-20 13:43:15 -0700143static inline bool ts_is_copy(TCGTemp *ts)
144{
145 return ts_info(ts)->next_copy != ts;
Aurelien Jarnod9c769c2015-07-27 12:41:44 +0200146}
147
Richard Henderson9f75e522023-11-02 13:37:46 -0700148static TCGTemp *cmp_better_copy(TCGTemp *a, TCGTemp *b)
149{
150 return a->kind < b->kind ? b : a;
151}
152
Aurelien Jarno1208d7d2015-07-27 12:41:44 +0200153/* Initialize and activate a temporary. */
Richard Henderson3b3f8472021-08-23 22:06:31 -0700154static void init_ts_info(OptContext *ctx, TCGTemp *ts)
Aurelien Jarno1208d7d2015-07-27 12:41:44 +0200155{
Richard Henderson63490392017-06-20 13:43:15 -0700156 size_t idx = temp_idx(ts);
Richard Henderson8f17a972020-03-30 19:52:02 -0700157 TempOptInfo *ti;
Richard Henderson63490392017-06-20 13:43:15 -0700158
Richard Henderson3b3f8472021-08-23 22:06:31 -0700159 if (test_bit(idx, ctx->temps_used.l)) {
Richard Henderson8f17a972020-03-30 19:52:02 -0700160 return;
161 }
Richard Henderson3b3f8472021-08-23 22:06:31 -0700162 set_bit(idx, ctx->temps_used.l);
Richard Henderson8f17a972020-03-30 19:52:02 -0700163
164 ti = ts->state_ptr;
165 if (ti == NULL) {
166 ti = tcg_malloc(sizeof(TempOptInfo));
Richard Henderson63490392017-06-20 13:43:15 -0700167 ts->state_ptr = ti;
Richard Henderson8f17a972020-03-30 19:52:02 -0700168 }
169
170 ti->next_copy = ts;
171 ti->prev_copy = ts;
Richard Hendersonab84dc32023-08-23 23:04:24 -0700172 QSIMPLEQ_INIT(&ti->mem_copy);
Richard Henderson8f17a972020-03-30 19:52:02 -0700173 if (ts->kind == TEMP_CONST) {
174 ti->is_const = true;
175 ti->val = ts->val;
Richard Hendersonb1fde412021-08-23 13:07:49 -0700176 ti->z_mask = ts->val;
Richard Henderson57fe5c62021-08-26 12:04:46 -0700177 ti->s_mask = smask_from_value(ts->val);
Richard Henderson8f17a972020-03-30 19:52:02 -0700178 } else {
179 ti->is_const = false;
Richard Hendersonb1fde412021-08-23 13:07:49 -0700180 ti->z_mask = -1;
Richard Henderson57fe5c62021-08-26 12:04:46 -0700181 ti->s_mask = 0;
Aurelien Jarno1208d7d2015-07-27 12:41:44 +0200182 }
183}
184
Richard Hendersonab84dc32023-08-23 23:04:24 -0700185static MemCopyInfo *mem_copy_first(OptContext *ctx, intptr_t s, intptr_t l)
186{
187 IntervalTreeNode *r = interval_tree_iter_first(&ctx->mem_copy, s, l);
188 return r ? container_of(r, MemCopyInfo, itree) : NULL;
189}
190
191static MemCopyInfo *mem_copy_next(MemCopyInfo *mem, intptr_t s, intptr_t l)
192{
193 IntervalTreeNode *r = interval_tree_iter_next(&mem->itree, s, l);
194 return r ? container_of(r, MemCopyInfo, itree) : NULL;
195}
196
197static void remove_mem_copy(OptContext *ctx, MemCopyInfo *mc)
198{
199 TCGTemp *ts = mc->ts;
200 TempOptInfo *ti = ts_info(ts);
201
202 interval_tree_remove(&mc->itree, &ctx->mem_copy);
203 QSIMPLEQ_REMOVE(&ti->mem_copy, mc, MemCopyInfo, next);
204 QSIMPLEQ_INSERT_TAIL(&ctx->mem_free, mc, next);
205}
206
207static void remove_mem_copy_in(OptContext *ctx, intptr_t s, intptr_t l)
208{
209 while (true) {
210 MemCopyInfo *mc = mem_copy_first(ctx, s, l);
211 if (!mc) {
212 break;
213 }
214 remove_mem_copy(ctx, mc);
215 }
216}
217
218static void remove_mem_copy_all(OptContext *ctx)
219{
220 remove_mem_copy_in(ctx, 0, -1);
221 tcg_debug_assert(interval_tree_is_empty(&ctx->mem_copy));
222}
223
Richard Henderson9f75e522023-11-02 13:37:46 -0700224static TCGTemp *find_better_copy(TCGTemp *ts)
Aurelien Jarnoe590d4e2012-09-11 12:31:21 +0200225{
Richard Henderson9f75e522023-11-02 13:37:46 -0700226 TCGTemp *i, *ret;
Aurelien Jarnoe590d4e2012-09-11 12:31:21 +0200227
Richard Henderson4c868ce2020-04-23 09:02:23 -0700228 /* If this is already readonly, we can't do better. */
229 if (temp_readonly(ts)) {
Richard Henderson63490392017-06-20 13:43:15 -0700230 return ts;
Aurelien Jarnoe590d4e2012-09-11 12:31:21 +0200231 }
232
Richard Henderson9f75e522023-11-02 13:37:46 -0700233 ret = ts;
Richard Henderson63490392017-06-20 13:43:15 -0700234 for (i = ts_info(ts)->next_copy; i != ts; i = ts_info(i)->next_copy) {
Richard Henderson9f75e522023-11-02 13:37:46 -0700235 ret = cmp_better_copy(ret, i);
Aurelien Jarnoe590d4e2012-09-11 12:31:21 +0200236 }
Richard Henderson9f75e522023-11-02 13:37:46 -0700237 return ret;
Aurelien Jarnoe590d4e2012-09-11 12:31:21 +0200238}
239
Richard Hendersonab84dc32023-08-23 23:04:24 -0700240static void move_mem_copies(TCGTemp *dst_ts, TCGTemp *src_ts)
241{
242 TempOptInfo *si = ts_info(src_ts);
243 TempOptInfo *di = ts_info(dst_ts);
244 MemCopyInfo *mc;
245
246 QSIMPLEQ_FOREACH(mc, &si->mem_copy, next) {
247 tcg_debug_assert(mc->ts == src_ts);
248 mc->ts = dst_ts;
249 }
250 QSIMPLEQ_CONCAT(&di->mem_copy, &si->mem_copy);
251}
252
253/* Reset TEMP's state, possibly removing the temp for the list of copies. */
254static void reset_ts(OptContext *ctx, TCGTemp *ts)
255{
256 TempOptInfo *ti = ts_info(ts);
257 TCGTemp *pts = ti->prev_copy;
258 TCGTemp *nts = ti->next_copy;
259 TempOptInfo *pi = ts_info(pts);
260 TempOptInfo *ni = ts_info(nts);
261
262 ni->prev_copy = ti->prev_copy;
263 pi->next_copy = ti->next_copy;
264 ti->next_copy = ts;
265 ti->prev_copy = ts;
266 ti->is_const = false;
267 ti->z_mask = -1;
268 ti->s_mask = 0;
269
270 if (!QSIMPLEQ_EMPTY(&ti->mem_copy)) {
271 if (ts == nts) {
272 /* Last temp copy being removed, the mem copies die. */
273 MemCopyInfo *mc;
274 QSIMPLEQ_FOREACH(mc, &ti->mem_copy, next) {
275 interval_tree_remove(&mc->itree, &ctx->mem_copy);
276 }
277 QSIMPLEQ_CONCAT(&ctx->mem_free, &ti->mem_copy);
278 } else {
279 move_mem_copies(find_better_copy(nts), ts);
280 }
281 }
282}
283
284static void reset_temp(OptContext *ctx, TCGArg arg)
285{
286 reset_ts(ctx, arg_temp(arg));
287}
288
289static void record_mem_copy(OptContext *ctx, TCGType type,
290 TCGTemp *ts, intptr_t start, intptr_t last)
291{
292 MemCopyInfo *mc;
293 TempOptInfo *ti;
294
295 mc = QSIMPLEQ_FIRST(&ctx->mem_free);
296 if (mc) {
297 QSIMPLEQ_REMOVE_HEAD(&ctx->mem_free, next);
298 } else {
299 mc = tcg_malloc(sizeof(*mc));
300 }
301
302 memset(mc, 0, sizeof(*mc));
303 mc->itree.start = start;
304 mc->itree.last = last;
305 mc->type = type;
306 interval_tree_insert(&mc->itree, &ctx->mem_copy);
307
308 ts = find_better_copy(ts);
309 ti = ts_info(ts);
310 mc->ts = ts;
311 QSIMPLEQ_INSERT_TAIL(&ti->mem_copy, mc, next);
312}
313
Richard Henderson63490392017-06-20 13:43:15 -0700314static bool ts_are_copies(TCGTemp *ts1, TCGTemp *ts2)
Aurelien Jarnoe590d4e2012-09-11 12:31:21 +0200315{
Richard Henderson63490392017-06-20 13:43:15 -0700316 TCGTemp *i;
Aurelien Jarnoe590d4e2012-09-11 12:31:21 +0200317
Richard Henderson63490392017-06-20 13:43:15 -0700318 if (ts1 == ts2) {
Aurelien Jarnoe590d4e2012-09-11 12:31:21 +0200319 return true;
320 }
321
Richard Henderson63490392017-06-20 13:43:15 -0700322 if (!ts_is_copy(ts1) || !ts_is_copy(ts2)) {
Aurelien Jarnoe590d4e2012-09-11 12:31:21 +0200323 return false;
324 }
325
Richard Henderson63490392017-06-20 13:43:15 -0700326 for (i = ts_info(ts1)->next_copy; i != ts1; i = ts_info(i)->next_copy) {
327 if (i == ts2) {
Aurelien Jarnoe590d4e2012-09-11 12:31:21 +0200328 return true;
329 }
330 }
331
332 return false;
333}
334
Richard Henderson63490392017-06-20 13:43:15 -0700335static bool args_are_copies(TCGArg arg1, TCGArg arg2)
336{
337 return ts_are_copies(arg_temp(arg1), arg_temp(arg2));
338}
339
Richard Hendersonab84dc32023-08-23 23:04:24 -0700340static TCGTemp *find_mem_copy_for(OptContext *ctx, TCGType type, intptr_t s)
341{
342 MemCopyInfo *mc;
343
344 for (mc = mem_copy_first(ctx, s, s); mc; mc = mem_copy_next(mc, s, s)) {
345 if (mc->itree.start == s && mc->type == type) {
346 return find_better_copy(mc->ts);
347 }
348 }
349 return NULL;
350}
351
Richard Henderson26aac972023-10-23 12:31:57 -0700352static TCGArg arg_new_constant(OptContext *ctx, uint64_t val)
353{
354 TCGType type = ctx->type;
355 TCGTemp *ts;
356
357 if (type == TCG_TYPE_I32) {
358 val = (int32_t)val;
359 }
360
361 ts = tcg_constant_internal(type, val);
362 init_ts_info(ctx, ts);
363
364 return temp_arg(ts);
365}
366
Richard Hendersonfb04ab72024-01-10 18:21:58 +1100367static TCGArg arg_new_temp(OptContext *ctx)
368{
369 TCGTemp *ts = tcg_temp_new_internal(ctx->type, TEMP_EBB);
370 init_ts_info(ctx, ts);
371 return temp_arg(ts);
372}
373
Richard Henderson6b99d5b2021-08-24 10:57:56 -0700374static bool tcg_opt_gen_mov(OptContext *ctx, TCGOp *op, TCGArg dst, TCGArg src)
Kirill Batuzov22613af2011-07-07 16:37:13 +0400375{
Richard Henderson63490392017-06-20 13:43:15 -0700376 TCGTemp *dst_ts = arg_temp(dst);
377 TCGTemp *src_ts = arg_temp(src);
Richard Henderson6fcb98e2020-03-30 17:44:30 -0700378 TempOptInfo *di;
379 TempOptInfo *si;
Richard Henderson63490392017-06-20 13:43:15 -0700380 TCGOpcode new_op;
381
382 if (ts_are_copies(dst_ts, src_ts)) {
Richard Hendersondc849882021-08-24 07:13:45 -0700383 tcg_op_remove(ctx->tcg, op);
Richard Henderson6b99d5b2021-08-24 10:57:56 -0700384 return true;
Aurelien Jarno53657182015-06-04 21:53:25 +0200385 }
386
Richard Henderson986cac12023-01-09 13:59:35 -0800387 reset_ts(ctx, dst_ts);
Richard Henderson63490392017-06-20 13:43:15 -0700388 di = ts_info(dst_ts);
389 si = ts_info(src_ts);
Richard Henderson67f84c92021-08-25 08:00:20 -0700390
391 switch (ctx->type) {
392 case TCG_TYPE_I32:
Richard Henderson170ba882017-11-22 09:07:11 +0100393 new_op = INDEX_op_mov_i32;
Richard Henderson67f84c92021-08-25 08:00:20 -0700394 break;
395 case TCG_TYPE_I64:
396 new_op = INDEX_op_mov_i64;
397 break;
398 case TCG_TYPE_V64:
399 case TCG_TYPE_V128:
400 case TCG_TYPE_V256:
401 /* TCGOP_VECL and TCGOP_VECE remain unchanged. */
402 new_op = INDEX_op_mov_vec;
403 break;
404 default:
405 g_assert_not_reached();
Richard Henderson170ba882017-11-22 09:07:11 +0100406 }
Richard Hendersonc45cb8b2014-09-19 13:49:15 -0700407 op->opc = new_op;
Richard Henderson63490392017-06-20 13:43:15 -0700408 op->args[0] = dst;
409 op->args[1] = src;
Richard Hendersona62f6f52014-05-22 10:59:12 -0700410
Richard Hendersonfaa2e102021-08-26 09:03:59 -0700411 di->z_mask = si->z_mask;
Richard Henderson57fe5c62021-08-26 12:04:46 -0700412 di->s_mask = si->s_mask;
Richard Henderson24666ba2014-05-22 11:14:10 -0700413
Richard Henderson63490392017-06-20 13:43:15 -0700414 if (src_ts->type == dst_ts->type) {
Richard Henderson6fcb98e2020-03-30 17:44:30 -0700415 TempOptInfo *ni = ts_info(si->next_copy);
Richard Henderson63490392017-06-20 13:43:15 -0700416
417 di->next_copy = si->next_copy;
418 di->prev_copy = src_ts;
419 ni->prev_copy = dst_ts;
420 si->next_copy = dst_ts;
421 di->is_const = si->is_const;
422 di->val = si->val;
Richard Hendersonab84dc32023-08-23 23:04:24 -0700423
424 if (!QSIMPLEQ_EMPTY(&si->mem_copy)
425 && cmp_better_copy(src_ts, dst_ts) == dst_ts) {
426 move_mem_copies(dst_ts, src_ts);
427 }
Paolo Bonzini3a9d8b12013-01-11 15:42:52 -0800428 }
Richard Henderson6b99d5b2021-08-24 10:57:56 -0700429 return true;
Kirill Batuzov22613af2011-07-07 16:37:13 +0400430}
431
Richard Henderson6b99d5b2021-08-24 10:57:56 -0700432static bool tcg_opt_gen_movi(OptContext *ctx, TCGOp *op,
Richard Hendersondc849882021-08-24 07:13:45 -0700433 TCGArg dst, uint64_t val)
Richard Henderson8fe35e02020-03-30 20:42:43 -0700434{
Richard Hendersonfaa2e102021-08-26 09:03:59 -0700435 /* Convert movi to mov with constant temp. */
Richard Henderson26aac972023-10-23 12:31:57 -0700436 return tcg_opt_gen_mov(ctx, op, dst, arg_new_constant(ctx, val));
Richard Henderson8fe35e02020-03-30 20:42:43 -0700437}
438
Richard Henderson54795542020-09-06 16:21:32 -0700439static uint64_t do_constant_folding_2(TCGOpcode op, uint64_t x, uint64_t y)
Kirill Batuzov53108fb2011-07-07 16:37:14 +0400440{
Richard Henderson03271522013-08-14 14:35:56 -0700441 uint64_t l64, h64;
442
Kirill Batuzov53108fb2011-07-07 16:37:14 +0400443 switch (op) {
444 CASE_OP_32_64(add):
445 return x + y;
446
447 CASE_OP_32_64(sub):
448 return x - y;
449
450 CASE_OP_32_64(mul):
451 return x * y;
452
Richard Hendersonc578ff12021-12-16 06:07:25 -0800453 CASE_OP_32_64_VEC(and):
Kirill Batuzov9a810902011-07-07 16:37:15 +0400454 return x & y;
455
Richard Hendersonc578ff12021-12-16 06:07:25 -0800456 CASE_OP_32_64_VEC(or):
Kirill Batuzov9a810902011-07-07 16:37:15 +0400457 return x | y;
458
Richard Hendersonc578ff12021-12-16 06:07:25 -0800459 CASE_OP_32_64_VEC(xor):
Kirill Batuzov9a810902011-07-07 16:37:15 +0400460 return x ^ y;
461
Kirill Batuzov55c09752011-07-07 16:37:16 +0400462 case INDEX_op_shl_i32:
Richard Henderson50c5c4d2014-03-18 07:45:39 -0700463 return (uint32_t)x << (y & 31);
Kirill Batuzov55c09752011-07-07 16:37:16 +0400464
Kirill Batuzov55c09752011-07-07 16:37:16 +0400465 case INDEX_op_shl_i64:
Richard Henderson50c5c4d2014-03-18 07:45:39 -0700466 return (uint64_t)x << (y & 63);
Kirill Batuzov55c09752011-07-07 16:37:16 +0400467
468 case INDEX_op_shr_i32:
Richard Henderson50c5c4d2014-03-18 07:45:39 -0700469 return (uint32_t)x >> (y & 31);
Kirill Batuzov55c09752011-07-07 16:37:16 +0400470
Kirill Batuzov55c09752011-07-07 16:37:16 +0400471 case INDEX_op_shr_i64:
Richard Henderson50c5c4d2014-03-18 07:45:39 -0700472 return (uint64_t)x >> (y & 63);
Kirill Batuzov55c09752011-07-07 16:37:16 +0400473
474 case INDEX_op_sar_i32:
Richard Henderson50c5c4d2014-03-18 07:45:39 -0700475 return (int32_t)x >> (y & 31);
Kirill Batuzov55c09752011-07-07 16:37:16 +0400476
Kirill Batuzov55c09752011-07-07 16:37:16 +0400477 case INDEX_op_sar_i64:
Richard Henderson50c5c4d2014-03-18 07:45:39 -0700478 return (int64_t)x >> (y & 63);
Kirill Batuzov55c09752011-07-07 16:37:16 +0400479
480 case INDEX_op_rotr_i32:
Richard Henderson50c5c4d2014-03-18 07:45:39 -0700481 return ror32(x, y & 31);
Kirill Batuzov55c09752011-07-07 16:37:16 +0400482
Kirill Batuzov55c09752011-07-07 16:37:16 +0400483 case INDEX_op_rotr_i64:
Richard Henderson50c5c4d2014-03-18 07:45:39 -0700484 return ror64(x, y & 63);
Kirill Batuzov55c09752011-07-07 16:37:16 +0400485
486 case INDEX_op_rotl_i32:
Richard Henderson50c5c4d2014-03-18 07:45:39 -0700487 return rol32(x, y & 31);
Kirill Batuzov55c09752011-07-07 16:37:16 +0400488
Kirill Batuzov55c09752011-07-07 16:37:16 +0400489 case INDEX_op_rotl_i64:
Richard Henderson50c5c4d2014-03-18 07:45:39 -0700490 return rol64(x, y & 63);
Kirill Batuzov55c09752011-07-07 16:37:16 +0400491
Richard Hendersonc578ff12021-12-16 06:07:25 -0800492 CASE_OP_32_64_VEC(not):
Kirill Batuzova640f032011-07-07 16:37:17 +0400493 return ~x;
494
Richard Hendersoncb25c802011-08-17 14:11:47 -0700495 CASE_OP_32_64(neg):
496 return -x;
497
Richard Hendersonc578ff12021-12-16 06:07:25 -0800498 CASE_OP_32_64_VEC(andc):
Richard Hendersoncb25c802011-08-17 14:11:47 -0700499 return x & ~y;
500
Richard Hendersonc578ff12021-12-16 06:07:25 -0800501 CASE_OP_32_64_VEC(orc):
Richard Hendersoncb25c802011-08-17 14:11:47 -0700502 return x | ~y;
503
Richard Hendersoned523472021-12-16 11:17:46 -0800504 CASE_OP_32_64_VEC(eqv):
Richard Hendersoncb25c802011-08-17 14:11:47 -0700505 return ~(x ^ y);
506
Richard Hendersoned523472021-12-16 11:17:46 -0800507 CASE_OP_32_64_VEC(nand):
Richard Hendersoncb25c802011-08-17 14:11:47 -0700508 return ~(x & y);
509
Richard Hendersoned523472021-12-16 11:17:46 -0800510 CASE_OP_32_64_VEC(nor):
Richard Hendersoncb25c802011-08-17 14:11:47 -0700511 return ~(x | y);
512
Richard Henderson0e28d002016-11-16 09:23:28 +0100513 case INDEX_op_clz_i32:
514 return (uint32_t)x ? clz32(x) : y;
515
516 case INDEX_op_clz_i64:
517 return x ? clz64(x) : y;
518
519 case INDEX_op_ctz_i32:
520 return (uint32_t)x ? ctz32(x) : y;
521
522 case INDEX_op_ctz_i64:
523 return x ? ctz64(x) : y;
524
Richard Hendersona768e4e2016-11-21 11:13:39 +0100525 case INDEX_op_ctpop_i32:
526 return ctpop32(x);
527
528 case INDEX_op_ctpop_i64:
529 return ctpop64(x);
530
Richard Henderson25c4d9c2011-08-17 14:11:46 -0700531 CASE_OP_32_64(ext8s):
Kirill Batuzova640f032011-07-07 16:37:17 +0400532 return (int8_t)x;
533
Richard Henderson25c4d9c2011-08-17 14:11:46 -0700534 CASE_OP_32_64(ext16s):
Kirill Batuzova640f032011-07-07 16:37:17 +0400535 return (int16_t)x;
536
Richard Henderson25c4d9c2011-08-17 14:11:46 -0700537 CASE_OP_32_64(ext8u):
Kirill Batuzova640f032011-07-07 16:37:17 +0400538 return (uint8_t)x;
539
Richard Henderson25c4d9c2011-08-17 14:11:46 -0700540 CASE_OP_32_64(ext16u):
Kirill Batuzova640f032011-07-07 16:37:17 +0400541 return (uint16_t)x;
542
Richard Henderson64985942018-11-20 08:53:34 +0100543 CASE_OP_32_64(bswap16):
Richard Henderson0b76ff82021-06-13 13:04:00 -0700544 x = bswap16(x);
545 return y & TCG_BSWAP_OS ? (int16_t)x : x;
Richard Henderson64985942018-11-20 08:53:34 +0100546
547 CASE_OP_32_64(bswap32):
Richard Henderson0b76ff82021-06-13 13:04:00 -0700548 x = bswap32(x);
549 return y & TCG_BSWAP_OS ? (int32_t)x : x;
Richard Henderson64985942018-11-20 08:53:34 +0100550
551 case INDEX_op_bswap64_i64:
552 return bswap64(x);
553
Aurelien Jarno8bcb5c82015-07-27 12:41:45 +0200554 case INDEX_op_ext_i32_i64:
Kirill Batuzova640f032011-07-07 16:37:17 +0400555 case INDEX_op_ext32s_i64:
556 return (int32_t)x;
557
Aurelien Jarno8bcb5c82015-07-27 12:41:45 +0200558 case INDEX_op_extu_i32_i64:
Richard Henderson609ad702015-07-24 07:16:00 -0700559 case INDEX_op_extrl_i64_i32:
Kirill Batuzova640f032011-07-07 16:37:17 +0400560 case INDEX_op_ext32u_i64:
561 return (uint32_t)x;
Kirill Batuzova640f032011-07-07 16:37:17 +0400562
Richard Henderson609ad702015-07-24 07:16:00 -0700563 case INDEX_op_extrh_i64_i32:
564 return (uint64_t)x >> 32;
565
Richard Henderson03271522013-08-14 14:35:56 -0700566 case INDEX_op_muluh_i32:
567 return ((uint64_t)(uint32_t)x * (uint32_t)y) >> 32;
568 case INDEX_op_mulsh_i32:
569 return ((int64_t)(int32_t)x * (int32_t)y) >> 32;
570
571 case INDEX_op_muluh_i64:
572 mulu64(&l64, &h64, x, y);
573 return h64;
574 case INDEX_op_mulsh_i64:
575 muls64(&l64, &h64, x, y);
576 return h64;
577
Richard Henderson01547f72013-08-14 15:22:46 -0700578 case INDEX_op_div_i32:
579 /* Avoid crashing on divide by zero, otherwise undefined. */
580 return (int32_t)x / ((int32_t)y ? : 1);
581 case INDEX_op_divu_i32:
582 return (uint32_t)x / ((uint32_t)y ? : 1);
583 case INDEX_op_div_i64:
584 return (int64_t)x / ((int64_t)y ? : 1);
585 case INDEX_op_divu_i64:
586 return (uint64_t)x / ((uint64_t)y ? : 1);
587
588 case INDEX_op_rem_i32:
589 return (int32_t)x % ((int32_t)y ? : 1);
590 case INDEX_op_remu_i32:
591 return (uint32_t)x % ((uint32_t)y ? : 1);
592 case INDEX_op_rem_i64:
593 return (int64_t)x % ((int64_t)y ? : 1);
594 case INDEX_op_remu_i64:
595 return (uint64_t)x % ((uint64_t)y ? : 1);
596
Kirill Batuzov53108fb2011-07-07 16:37:14 +0400597 default:
Richard Henderson732e89f2023-04-05 12:09:14 -0700598 g_assert_not_reached();
Kirill Batuzov53108fb2011-07-07 16:37:14 +0400599 }
600}
601
Richard Henderson67f84c92021-08-25 08:00:20 -0700602static uint64_t do_constant_folding(TCGOpcode op, TCGType type,
603 uint64_t x, uint64_t y)
Kirill Batuzov53108fb2011-07-07 16:37:14 +0400604{
Richard Henderson54795542020-09-06 16:21:32 -0700605 uint64_t res = do_constant_folding_2(op, x, y);
Richard Henderson67f84c92021-08-25 08:00:20 -0700606 if (type == TCG_TYPE_I32) {
Aurelien Jarno29f3ff82015-07-10 18:03:31 +0200607 res = (int32_t)res;
Kirill Batuzov53108fb2011-07-07 16:37:14 +0400608 }
Kirill Batuzov53108fb2011-07-07 16:37:14 +0400609 return res;
610}
611
Richard Henderson9519da72012-10-02 11:32:26 -0700612static bool do_constant_folding_cond_32(uint32_t x, uint32_t y, TCGCond c)
613{
614 switch (c) {
615 case TCG_COND_EQ:
616 return x == y;
617 case TCG_COND_NE:
618 return x != y;
619 case TCG_COND_LT:
620 return (int32_t)x < (int32_t)y;
621 case TCG_COND_GE:
622 return (int32_t)x >= (int32_t)y;
623 case TCG_COND_LE:
624 return (int32_t)x <= (int32_t)y;
625 case TCG_COND_GT:
626 return (int32_t)x > (int32_t)y;
627 case TCG_COND_LTU:
628 return x < y;
629 case TCG_COND_GEU:
630 return x >= y;
631 case TCG_COND_LEU:
632 return x <= y;
633 case TCG_COND_GTU:
634 return x > y;
Richard Hendersonceb9ee02023-10-23 23:44:27 -0700635 case TCG_COND_TSTEQ:
636 return (x & y) == 0;
637 case TCG_COND_TSTNE:
638 return (x & y) != 0;
639 case TCG_COND_ALWAYS:
640 case TCG_COND_NEVER:
641 break;
Richard Henderson9519da72012-10-02 11:32:26 -0700642 }
Richard Hendersonceb9ee02023-10-23 23:44:27 -0700643 g_assert_not_reached();
Richard Henderson9519da72012-10-02 11:32:26 -0700644}
645
646static bool do_constant_folding_cond_64(uint64_t x, uint64_t y, TCGCond c)
647{
648 switch (c) {
649 case TCG_COND_EQ:
650 return x == y;
651 case TCG_COND_NE:
652 return x != y;
653 case TCG_COND_LT:
654 return (int64_t)x < (int64_t)y;
655 case TCG_COND_GE:
656 return (int64_t)x >= (int64_t)y;
657 case TCG_COND_LE:
658 return (int64_t)x <= (int64_t)y;
659 case TCG_COND_GT:
660 return (int64_t)x > (int64_t)y;
661 case TCG_COND_LTU:
662 return x < y;
663 case TCG_COND_GEU:
664 return x >= y;
665 case TCG_COND_LEU:
666 return x <= y;
667 case TCG_COND_GTU:
668 return x > y;
Richard Hendersonceb9ee02023-10-23 23:44:27 -0700669 case TCG_COND_TSTEQ:
670 return (x & y) == 0;
671 case TCG_COND_TSTNE:
672 return (x & y) != 0;
673 case TCG_COND_ALWAYS:
674 case TCG_COND_NEVER:
675 break;
Richard Henderson9519da72012-10-02 11:32:26 -0700676 }
Richard Hendersonceb9ee02023-10-23 23:44:27 -0700677 g_assert_not_reached();
Richard Henderson9519da72012-10-02 11:32:26 -0700678}
679
Richard Hendersonceb9ee02023-10-23 23:44:27 -0700680static int do_constant_folding_cond_eq(TCGCond c)
Richard Henderson9519da72012-10-02 11:32:26 -0700681{
682 switch (c) {
683 case TCG_COND_GT:
684 case TCG_COND_LTU:
685 case TCG_COND_LT:
686 case TCG_COND_GTU:
687 case TCG_COND_NE:
688 return 0;
689 case TCG_COND_GE:
690 case TCG_COND_GEU:
691 case TCG_COND_LE:
692 case TCG_COND_LEU:
693 case TCG_COND_EQ:
694 return 1;
Richard Hendersonceb9ee02023-10-23 23:44:27 -0700695 case TCG_COND_TSTEQ:
696 case TCG_COND_TSTNE:
697 return -1;
698 case TCG_COND_ALWAYS:
699 case TCG_COND_NEVER:
700 break;
Richard Henderson9519da72012-10-02 11:32:26 -0700701 }
Richard Hendersonceb9ee02023-10-23 23:44:27 -0700702 g_assert_not_reached();
Richard Henderson9519da72012-10-02 11:32:26 -0700703}
704
Richard Henderson8d57bf12021-08-24 08:34:27 -0700705/*
706 * Return -1 if the condition can't be simplified,
707 * and the result of the condition (0 or 1) if it can.
708 */
Richard Henderson67f84c92021-08-25 08:00:20 -0700709static int do_constant_folding_cond(TCGType type, TCGArg x,
Richard Henderson8d57bf12021-08-24 08:34:27 -0700710 TCGArg y, TCGCond c)
Aurelien Jarnof8dd19e2012-09-06 16:47:14 +0200711{
Richard Henderson63490392017-06-20 13:43:15 -0700712 if (arg_is_const(x) && arg_is_const(y)) {
Alex Bennée9becc362022-02-09 11:21:42 +0000713 uint64_t xv = arg_info(x)->val;
714 uint64_t yv = arg_info(y)->val;
715
Richard Henderson67f84c92021-08-25 08:00:20 -0700716 switch (type) {
717 case TCG_TYPE_I32:
Richard Henderson170ba882017-11-22 09:07:11 +0100718 return do_constant_folding_cond_32(xv, yv, c);
Richard Henderson67f84c92021-08-25 08:00:20 -0700719 case TCG_TYPE_I64:
720 return do_constant_folding_cond_64(xv, yv, c);
721 default:
722 /* Only scalar comparisons are optimizable */
723 return -1;
Aurelien Jarnof8dd19e2012-09-06 16:47:14 +0200724 }
Richard Henderson63490392017-06-20 13:43:15 -0700725 } else if (args_are_copies(x, y)) {
Richard Henderson9519da72012-10-02 11:32:26 -0700726 return do_constant_folding_cond_eq(c);
Richard Henderson27cdb852023-10-23 11:38:00 -0700727 } else if (arg_is_const_val(y, 0)) {
Aurelien Jarnob336ceb2012-09-18 19:37:00 +0200728 switch (c) {
729 case TCG_COND_LTU:
Richard Hendersonceb9ee02023-10-23 23:44:27 -0700730 case TCG_COND_TSTNE:
Aurelien Jarnob336ceb2012-09-18 19:37:00 +0200731 return 0;
732 case TCG_COND_GEU:
Richard Hendersonceb9ee02023-10-23 23:44:27 -0700733 case TCG_COND_TSTEQ:
Aurelien Jarnob336ceb2012-09-18 19:37:00 +0200734 return 1;
735 default:
Richard Henderson8d57bf12021-08-24 08:34:27 -0700736 return -1;
Aurelien Jarnob336ceb2012-09-18 19:37:00 +0200737 }
Aurelien Jarnof8dd19e2012-09-06 16:47:14 +0200738 }
Richard Henderson8d57bf12021-08-24 08:34:27 -0700739 return -1;
Aurelien Jarnof8dd19e2012-09-06 16:47:14 +0200740}
741
Richard Henderson7a2f7082021-08-26 07:06:39 -0700742/**
743 * swap_commutative:
744 * @dest: TCGArg of the destination argument, or NO_DEST.
745 * @p1: first paired argument
746 * @p2: second paired argument
747 *
748 * If *@p1 is a constant and *@p2 is not, swap.
749 * If *@p2 matches @dest, swap.
750 * Return true if a swap was performed.
751 */
752
753#define NO_DEST temp_arg(NULL)
754
Richard Henderson24c9ae42012-10-02 11:32:21 -0700755static bool swap_commutative(TCGArg dest, TCGArg *p1, TCGArg *p2)
756{
757 TCGArg a1 = *p1, a2 = *p2;
758 int sum = 0;
Richard Henderson63490392017-06-20 13:43:15 -0700759 sum += arg_is_const(a1);
760 sum -= arg_is_const(a2);
Richard Henderson24c9ae42012-10-02 11:32:21 -0700761
762 /* Prefer the constant in second argument, and then the form
763 op a, a, b, which is better handled on non-RISC hosts. */
764 if (sum > 0 || (sum == 0 && dest == a2)) {
765 *p1 = a2;
766 *p2 = a1;
767 return true;
768 }
769 return false;
770}
771
Richard Henderson0bfcb862012-10-02 11:32:23 -0700772static bool swap_commutative2(TCGArg *p1, TCGArg *p2)
773{
774 int sum = 0;
Richard Henderson63490392017-06-20 13:43:15 -0700775 sum += arg_is_const(p1[0]);
776 sum += arg_is_const(p1[1]);
777 sum -= arg_is_const(p2[0]);
778 sum -= arg_is_const(p2[1]);
Richard Henderson0bfcb862012-10-02 11:32:23 -0700779 if (sum > 0) {
780 TCGArg t;
781 t = p1[0], p1[0] = p2[0], p2[0] = t;
782 t = p1[1], p1[1] = p2[1], p2[1] = t;
783 return true;
784 }
785 return false;
786}
787
Richard Henderson7e64b112023-10-24 16:53:56 -0700788/*
789 * Return -1 if the condition can't be simplified,
790 * and the result of the condition (0 or 1) if it can.
791 */
Richard Hendersonfb04ab72024-01-10 18:21:58 +1100792static int do_constant_folding_cond1(OptContext *ctx, TCGOp *op, TCGArg dest,
Richard Henderson246c4b72023-10-24 16:36:50 -0700793 TCGArg *p1, TCGArg *p2, TCGArg *pcond)
794{
795 TCGCond cond;
796 bool swap;
797 int r;
798
799 swap = swap_commutative(dest, p1, p2);
800 cond = *pcond;
801 if (swap) {
802 *pcond = cond = tcg_swap_cond(cond);
803 }
804
805 r = do_constant_folding_cond(ctx->type, *p1, *p2, cond);
Richard Hendersonceb9ee02023-10-23 23:44:27 -0700806 if (r >= 0) {
807 return r;
808 }
809 if (!is_tst_cond(cond)) {
810 return -1;
811 }
812
813 /*
814 * TSTNE x,x -> NE x,0
815 * TSTNE x,-1 -> NE x,0
816 */
817 if (args_are_copies(*p1, *p2) || arg_is_const_val(*p2, -1)) {
818 *p2 = arg_new_constant(ctx, 0);
819 *pcond = tcg_tst_eqne_cond(cond);
820 return -1;
821 }
822
823 /* TSTNE x,sign -> LT x,0 */
824 if (arg_is_const_val(*p2, (ctx->type == TCG_TYPE_I32
825 ? INT32_MIN : INT64_MIN))) {
826 *p2 = arg_new_constant(ctx, 0);
827 *pcond = tcg_tst_ltge_cond(cond);
Richard Hendersonfb04ab72024-01-10 18:21:58 +1100828 return -1;
829 }
830
831 /* Expand to AND with a temporary if no backend support. */
832 if (!TCG_TARGET_HAS_tst) {
833 TCGOpcode and_opc = (ctx->type == TCG_TYPE_I32
834 ? INDEX_op_and_i32 : INDEX_op_and_i64);
835 TCGOp *op2 = tcg_op_insert_before(ctx->tcg, op, and_opc, 3);
836 TCGArg tmp = arg_new_temp(ctx);
837
838 op2->args[0] = tmp;
839 op2->args[1] = *p1;
840 op2->args[2] = *p2;
841
842 *p1 = tmp;
843 *p2 = arg_new_constant(ctx, 0);
844 *pcond = tcg_tst_eqne_cond(cond);
Richard Hendersonceb9ee02023-10-23 23:44:27 -0700845 }
846 return -1;
Richard Henderson246c4b72023-10-24 16:36:50 -0700847}
848
Richard Hendersonfb04ab72024-01-10 18:21:58 +1100849static int do_constant_folding_cond2(OptContext *ctx, TCGOp *op, TCGArg *args)
Richard Henderson7e64b112023-10-24 16:53:56 -0700850{
851 TCGArg al, ah, bl, bh;
852 TCGCond c;
853 bool swap;
Richard Hendersonceb9ee02023-10-23 23:44:27 -0700854 int r;
Richard Henderson7e64b112023-10-24 16:53:56 -0700855
856 swap = swap_commutative2(args, args + 2);
857 c = args[4];
858 if (swap) {
859 args[4] = c = tcg_swap_cond(c);
860 }
861
862 al = args[0];
863 ah = args[1];
864 bl = args[2];
865 bh = args[3];
866
867 if (arg_is_const(bl) && arg_is_const(bh)) {
868 tcg_target_ulong blv = arg_info(bl)->val;
869 tcg_target_ulong bhv = arg_info(bh)->val;
870 uint64_t b = deposit64(blv, 32, 32, bhv);
871
872 if (arg_is_const(al) && arg_is_const(ah)) {
873 tcg_target_ulong alv = arg_info(al)->val;
874 tcg_target_ulong ahv = arg_info(ah)->val;
875 uint64_t a = deposit64(alv, 32, 32, ahv);
Richard Hendersonceb9ee02023-10-23 23:44:27 -0700876
877 r = do_constant_folding_cond_64(a, b, c);
878 if (r >= 0) {
879 return r;
880 }
Richard Henderson7e64b112023-10-24 16:53:56 -0700881 }
Richard Hendersonceb9ee02023-10-23 23:44:27 -0700882
Richard Henderson7e64b112023-10-24 16:53:56 -0700883 if (b == 0) {
884 switch (c) {
885 case TCG_COND_LTU:
Richard Hendersonceb9ee02023-10-23 23:44:27 -0700886 case TCG_COND_TSTNE:
Richard Henderson7e64b112023-10-24 16:53:56 -0700887 return 0;
888 case TCG_COND_GEU:
Richard Hendersonceb9ee02023-10-23 23:44:27 -0700889 case TCG_COND_TSTEQ:
Richard Henderson7e64b112023-10-24 16:53:56 -0700890 return 1;
891 default:
892 break;
893 }
894 }
Richard Hendersonceb9ee02023-10-23 23:44:27 -0700895
896 /* TSTNE x,-1 -> NE x,0 */
897 if (b == -1 && is_tst_cond(c)) {
898 args[3] = args[2] = arg_new_constant(ctx, 0);
899 args[4] = tcg_tst_eqne_cond(c);
900 return -1;
901 }
902
903 /* TSTNE x,sign -> LT x,0 */
904 if (b == INT64_MIN && is_tst_cond(c)) {
905 /* bl must be 0, so copy that to bh */
906 args[3] = bl;
907 args[4] = tcg_tst_ltge_cond(c);
908 return -1;
909 }
Richard Henderson7e64b112023-10-24 16:53:56 -0700910 }
Richard Hendersonceb9ee02023-10-23 23:44:27 -0700911
Richard Henderson7e64b112023-10-24 16:53:56 -0700912 if (args_are_copies(al, bl) && args_are_copies(ah, bh)) {
Richard Hendersonceb9ee02023-10-23 23:44:27 -0700913 r = do_constant_folding_cond_eq(c);
914 if (r >= 0) {
915 return r;
916 }
917
918 /* TSTNE x,x -> NE x,0 */
919 if (is_tst_cond(c)) {
920 args[3] = args[2] = arg_new_constant(ctx, 0);
921 args[4] = tcg_tst_eqne_cond(c);
922 return -1;
923 }
Richard Henderson7e64b112023-10-24 16:53:56 -0700924 }
Richard Hendersonfb04ab72024-01-10 18:21:58 +1100925
926 /* Expand to AND with a temporary if no backend support. */
927 if (!TCG_TARGET_HAS_tst && is_tst_cond(c)) {
928 TCGOp *op1 = tcg_op_insert_before(ctx->tcg, op, INDEX_op_and_i32, 3);
929 TCGOp *op2 = tcg_op_insert_before(ctx->tcg, op, INDEX_op_and_i32, 3);
930 TCGArg t1 = arg_new_temp(ctx);
931 TCGArg t2 = arg_new_temp(ctx);
932
933 op1->args[0] = t1;
934 op1->args[1] = al;
935 op1->args[2] = bl;
936 op2->args[0] = t2;
937 op2->args[1] = ah;
938 op2->args[2] = bh;
939
940 args[0] = t1;
941 args[1] = t2;
942 args[3] = args[2] = arg_new_constant(ctx, 0);
943 args[4] = tcg_tst_eqne_cond(c);
944 }
Richard Henderson7e64b112023-10-24 16:53:56 -0700945 return -1;
946}
947
Richard Hendersone2577ea2021-08-24 08:00:48 -0700948static void init_arguments(OptContext *ctx, TCGOp *op, int nb_args)
949{
950 for (int i = 0; i < nb_args; i++) {
951 TCGTemp *ts = arg_temp(op->args[i]);
Richard Henderson39004a72022-11-11 10:09:37 +1000952 init_ts_info(ctx, ts);
Richard Hendersone2577ea2021-08-24 08:00:48 -0700953 }
954}
955
Richard Henderson8774dde2021-08-24 08:04:47 -0700956static void copy_propagate(OptContext *ctx, TCGOp *op,
957 int nb_oargs, int nb_iargs)
958{
Richard Henderson8774dde2021-08-24 08:04:47 -0700959 for (int i = nb_oargs; i < nb_oargs + nb_iargs; i++) {
960 TCGTemp *ts = arg_temp(op->args[i]);
Richard Henderson39004a72022-11-11 10:09:37 +1000961 if (ts_is_copy(ts)) {
Richard Henderson9f75e522023-11-02 13:37:46 -0700962 op->args[i] = temp_arg(find_better_copy(ts));
Richard Henderson8774dde2021-08-24 08:04:47 -0700963 }
964 }
965}
966
Richard Henderson137f1f42021-08-24 08:49:25 -0700967static void finish_folding(OptContext *ctx, TCGOp *op)
968{
969 const TCGOpDef *def = &tcg_op_defs[op->opc];
970 int i, nb_oargs;
971
972 /*
Richard Hendersond97f8f32023-10-16 19:10:42 -0700973 * We only optimize extended basic blocks. If the opcode ends a BB
974 * and is not a conditional branch, reset all temp data.
Richard Henderson137f1f42021-08-24 08:49:25 -0700975 */
976 if (def->flags & TCG_OPF_BB_END) {
Richard Henderson137f1f42021-08-24 08:49:25 -0700977 ctx->prev_mb = NULL;
Richard Hendersond97f8f32023-10-16 19:10:42 -0700978 if (!(def->flags & TCG_OPF_COND_BRANCH)) {
979 memset(&ctx->temps_used, 0, sizeof(ctx->temps_used));
Richard Hendersonab84dc32023-08-23 23:04:24 -0700980 remove_mem_copy_all(ctx);
Richard Hendersond97f8f32023-10-16 19:10:42 -0700981 }
Richard Henderson137f1f42021-08-24 08:49:25 -0700982 return;
983 }
984
985 nb_oargs = def->nb_oargs;
986 for (i = 0; i < nb_oargs; i++) {
Richard Henderson57fe5c62021-08-26 12:04:46 -0700987 TCGTemp *ts = arg_temp(op->args[i]);
Richard Henderson986cac12023-01-09 13:59:35 -0800988 reset_ts(ctx, ts);
Richard Henderson137f1f42021-08-24 08:49:25 -0700989 /*
Richard Henderson57fe5c62021-08-26 12:04:46 -0700990 * Save the corresponding known-zero/sign bits mask for the
Richard Henderson137f1f42021-08-24 08:49:25 -0700991 * first output argument (only one supported so far).
992 */
993 if (i == 0) {
Richard Henderson57fe5c62021-08-26 12:04:46 -0700994 ts_info(ts)->z_mask = ctx->z_mask;
995 ts_info(ts)->s_mask = ctx->s_mask;
Richard Henderson137f1f42021-08-24 08:49:25 -0700996 }
997 }
998}
999
Richard Henderson2f9f08b2021-08-25 12:03:48 -07001000/*
1001 * The fold_* functions return true when processing is complete,
1002 * usually by folding the operation to a constant or to a copy,
1003 * and calling tcg_opt_gen_{mov,movi}. They may do other things,
1004 * like collect information about the value produced, for use in
1005 * optimizing a subsequent operation.
1006 *
1007 * These first fold_* functions are all helpers, used by other
1008 * folders for more specific operations.
1009 */
1010
1011static bool fold_const1(OptContext *ctx, TCGOp *op)
1012{
1013 if (arg_is_const(op->args[1])) {
1014 uint64_t t;
1015
1016 t = arg_info(op->args[1])->val;
Richard Henderson67f84c92021-08-25 08:00:20 -07001017 t = do_constant_folding(op->opc, ctx->type, t, 0);
Richard Henderson2f9f08b2021-08-25 12:03:48 -07001018 return tcg_opt_gen_movi(ctx, op, op->args[0], t);
1019 }
1020 return false;
1021}
1022
1023static bool fold_const2(OptContext *ctx, TCGOp *op)
1024{
1025 if (arg_is_const(op->args[1]) && arg_is_const(op->args[2])) {
1026 uint64_t t1 = arg_info(op->args[1])->val;
1027 uint64_t t2 = arg_info(op->args[2])->val;
1028
Richard Henderson67f84c92021-08-25 08:00:20 -07001029 t1 = do_constant_folding(op->opc, ctx->type, t1, t2);
Richard Henderson2f9f08b2021-08-25 12:03:48 -07001030 return tcg_opt_gen_movi(ctx, op, op->args[0], t1);
1031 }
1032 return false;
1033}
1034
Richard Hendersonc578ff12021-12-16 06:07:25 -08001035static bool fold_commutative(OptContext *ctx, TCGOp *op)
1036{
1037 swap_commutative(op->args[0], &op->args[1], &op->args[2]);
1038 return false;
1039}
1040
Richard Henderson7a2f7082021-08-26 07:06:39 -07001041static bool fold_const2_commutative(OptContext *ctx, TCGOp *op)
1042{
1043 swap_commutative(op->args[0], &op->args[1], &op->args[2]);
1044 return fold_const2(ctx, op);
1045}
1046
Richard Hendersonfae450b2021-08-25 22:42:19 -07001047static bool fold_masks(OptContext *ctx, TCGOp *op)
1048{
1049 uint64_t a_mask = ctx->a_mask;
1050 uint64_t z_mask = ctx->z_mask;
Richard Henderson57fe5c62021-08-26 12:04:46 -07001051 uint64_t s_mask = ctx->s_mask;
Richard Hendersonfae450b2021-08-25 22:42:19 -07001052
1053 /*
Richard Hendersonfaa2e102021-08-26 09:03:59 -07001054 * 32-bit ops generate 32-bit results, which for the purpose of
1055 * simplifying tcg are sign-extended. Certainly that's how we
1056 * represent our constants elsewhere. Note that the bits will
1057 * be reset properly for a 64-bit value when encountering the
1058 * type changing opcodes.
Richard Hendersonfae450b2021-08-25 22:42:19 -07001059 */
1060 if (ctx->type == TCG_TYPE_I32) {
Richard Hendersonfaa2e102021-08-26 09:03:59 -07001061 a_mask = (int32_t)a_mask;
1062 z_mask = (int32_t)z_mask;
Richard Henderson57fe5c62021-08-26 12:04:46 -07001063 s_mask |= MAKE_64BIT_MASK(32, 32);
Richard Hendersonfaa2e102021-08-26 09:03:59 -07001064 ctx->z_mask = z_mask;
Richard Henderson57fe5c62021-08-26 12:04:46 -07001065 ctx->s_mask = s_mask;
Richard Hendersonfae450b2021-08-25 22:42:19 -07001066 }
1067
1068 if (z_mask == 0) {
1069 return tcg_opt_gen_movi(ctx, op, op->args[0], 0);
1070 }
1071 if (a_mask == 0) {
1072 return tcg_opt_gen_mov(ctx, op, op->args[0], op->args[1]);
1073 }
1074 return false;
1075}
1076
Richard Henderson0e0a32b2021-08-24 13:18:01 -07001077/*
1078 * Convert @op to NOT, if NOT is supported by the host.
1079 * Return true f the conversion is successful, which will still
1080 * indicate that the processing is complete.
1081 */
1082static bool fold_not(OptContext *ctx, TCGOp *op);
1083static bool fold_to_not(OptContext *ctx, TCGOp *op, int idx)
1084{
1085 TCGOpcode not_op;
1086 bool have_not;
1087
1088 switch (ctx->type) {
1089 case TCG_TYPE_I32:
1090 not_op = INDEX_op_not_i32;
1091 have_not = TCG_TARGET_HAS_not_i32;
1092 break;
1093 case TCG_TYPE_I64:
1094 not_op = INDEX_op_not_i64;
1095 have_not = TCG_TARGET_HAS_not_i64;
1096 break;
1097 case TCG_TYPE_V64:
1098 case TCG_TYPE_V128:
1099 case TCG_TYPE_V256:
1100 not_op = INDEX_op_not_vec;
1101 have_not = TCG_TARGET_HAS_not_vec;
1102 break;
1103 default:
1104 g_assert_not_reached();
1105 }
1106 if (have_not) {
1107 op->opc = not_op;
1108 op->args[1] = op->args[idx];
1109 return fold_not(ctx, op);
1110 }
1111 return false;
1112}
1113
Richard Hendersonda48e272021-08-25 20:42:04 -07001114/* If the binary operation has first argument @i, fold to @i. */
1115static bool fold_ix_to_i(OptContext *ctx, TCGOp *op, uint64_t i)
1116{
Richard Henderson27cdb852023-10-23 11:38:00 -07001117 if (arg_is_const_val(op->args[1], i)) {
Richard Hendersonda48e272021-08-25 20:42:04 -07001118 return tcg_opt_gen_movi(ctx, op, op->args[0], i);
1119 }
1120 return false;
1121}
1122
Richard Henderson0e0a32b2021-08-24 13:18:01 -07001123/* If the binary operation has first argument @i, fold to NOT. */
1124static bool fold_ix_to_not(OptContext *ctx, TCGOp *op, uint64_t i)
1125{
Richard Henderson27cdb852023-10-23 11:38:00 -07001126 if (arg_is_const_val(op->args[1], i)) {
Richard Henderson0e0a32b2021-08-24 13:18:01 -07001127 return fold_to_not(ctx, op, 2);
1128 }
1129 return false;
1130}
1131
Richard Hendersone8679952021-08-25 13:19:52 -07001132/* If the binary operation has second argument @i, fold to @i. */
1133static bool fold_xi_to_i(OptContext *ctx, TCGOp *op, uint64_t i)
1134{
Richard Henderson27cdb852023-10-23 11:38:00 -07001135 if (arg_is_const_val(op->args[2], i)) {
Richard Hendersone8679952021-08-25 13:19:52 -07001136 return tcg_opt_gen_movi(ctx, op, op->args[0], i);
1137 }
1138 return false;
1139}
1140
Richard Hendersona63ce0e2021-08-25 20:28:53 -07001141/* If the binary operation has second argument @i, fold to identity. */
1142static bool fold_xi_to_x(OptContext *ctx, TCGOp *op, uint64_t i)
1143{
Richard Henderson27cdb852023-10-23 11:38:00 -07001144 if (arg_is_const_val(op->args[2], i)) {
Richard Hendersona63ce0e2021-08-25 20:28:53 -07001145 return tcg_opt_gen_mov(ctx, op, op->args[0], op->args[1]);
1146 }
1147 return false;
1148}
1149
Richard Henderson0e0a32b2021-08-24 13:18:01 -07001150/* If the binary operation has second argument @i, fold to NOT. */
1151static bool fold_xi_to_not(OptContext *ctx, TCGOp *op, uint64_t i)
1152{
Richard Henderson27cdb852023-10-23 11:38:00 -07001153 if (arg_is_const_val(op->args[2], i)) {
Richard Henderson0e0a32b2021-08-24 13:18:01 -07001154 return fold_to_not(ctx, op, 1);
1155 }
1156 return false;
1157}
1158
Richard Hendersoncbe42fb2021-08-25 13:02:00 -07001159/* If the binary operation has both arguments equal, fold to @i. */
1160static bool fold_xx_to_i(OptContext *ctx, TCGOp *op, uint64_t i)
1161{
1162 if (args_are_copies(op->args[1], op->args[2])) {
1163 return tcg_opt_gen_movi(ctx, op, op->args[0], i);
1164 }
1165 return false;
1166}
1167
Richard Hendersonca7bb042021-08-25 13:14:21 -07001168/* If the binary operation has both arguments equal, fold to identity. */
1169static bool fold_xx_to_x(OptContext *ctx, TCGOp *op)
1170{
1171 if (args_are_copies(op->args[1], op->args[2])) {
1172 return tcg_opt_gen_mov(ctx, op, op->args[0], op->args[1]);
1173 }
1174 return false;
1175}
1176
Richard Henderson2f9f08b2021-08-25 12:03:48 -07001177/*
1178 * These outermost fold_<op> functions are sorted alphabetically.
Richard Hendersonca7bb042021-08-25 13:14:21 -07001179 *
1180 * The ordering of the transformations should be:
1181 * 1) those that produce a constant
1182 * 2) those that produce a copy
1183 * 3) those that produce information about the result value.
Richard Henderson2f9f08b2021-08-25 12:03:48 -07001184 */
1185
1186static bool fold_add(OptContext *ctx, TCGOp *op)
1187{
Richard Henderson7a2f7082021-08-26 07:06:39 -07001188 if (fold_const2_commutative(ctx, op) ||
Richard Hendersona63ce0e2021-08-25 20:28:53 -07001189 fold_xi_to_x(ctx, op, 0)) {
1190 return true;
1191 }
1192 return false;
Richard Henderson2f9f08b2021-08-25 12:03:48 -07001193}
1194
Richard Hendersonc578ff12021-12-16 06:07:25 -08001195/* We cannot as yet do_constant_folding with vectors. */
1196static bool fold_add_vec(OptContext *ctx, TCGOp *op)
1197{
1198 if (fold_commutative(ctx, op) ||
1199 fold_xi_to_x(ctx, op, 0)) {
1200 return true;
1201 }
1202 return false;
1203}
1204
Richard Henderson9531c072021-08-26 06:51:39 -07001205static bool fold_addsub2(OptContext *ctx, TCGOp *op, bool add)
Richard Hendersone3f7dc22021-08-24 10:30:38 -07001206{
Richard Hendersonf2457572023-10-25 18:39:44 -07001207 bool a_const = arg_is_const(op->args[2]) && arg_is_const(op->args[3]);
1208 bool b_const = arg_is_const(op->args[4]) && arg_is_const(op->args[5]);
1209
1210 if (a_const && b_const) {
Richard Henderson9531c072021-08-26 06:51:39 -07001211 uint64_t al = arg_info(op->args[2])->val;
1212 uint64_t ah = arg_info(op->args[3])->val;
1213 uint64_t bl = arg_info(op->args[4])->val;
1214 uint64_t bh = arg_info(op->args[5])->val;
Richard Hendersone3f7dc22021-08-24 10:30:38 -07001215 TCGArg rl, rh;
Richard Henderson9531c072021-08-26 06:51:39 -07001216 TCGOp *op2;
Richard Hendersone3f7dc22021-08-24 10:30:38 -07001217
Richard Henderson9531c072021-08-26 06:51:39 -07001218 if (ctx->type == TCG_TYPE_I32) {
1219 uint64_t a = deposit64(al, 32, 32, ah);
1220 uint64_t b = deposit64(bl, 32, 32, bh);
1221
1222 if (add) {
1223 a += b;
1224 } else {
1225 a -= b;
1226 }
1227
1228 al = sextract64(a, 0, 32);
1229 ah = sextract64(a, 32, 32);
Richard Hendersone3f7dc22021-08-24 10:30:38 -07001230 } else {
Richard Henderson9531c072021-08-26 06:51:39 -07001231 Int128 a = int128_make128(al, ah);
1232 Int128 b = int128_make128(bl, bh);
1233
1234 if (add) {
1235 a = int128_add(a, b);
1236 } else {
1237 a = int128_sub(a, b);
1238 }
1239
1240 al = int128_getlo(a);
1241 ah = int128_gethi(a);
Richard Hendersone3f7dc22021-08-24 10:30:38 -07001242 }
1243
1244 rl = op->args[0];
1245 rh = op->args[1];
Richard Henderson9531c072021-08-26 06:51:39 -07001246
1247 /* The proper opcode is supplied by tcg_opt_gen_mov. */
Philippe Mathieu-Daudéd4478942022-12-18 22:18:31 +01001248 op2 = tcg_op_insert_before(ctx->tcg, op, 0, 2);
Richard Henderson9531c072021-08-26 06:51:39 -07001249
1250 tcg_opt_gen_movi(ctx, op, rl, al);
1251 tcg_opt_gen_movi(ctx, op2, rh, ah);
Richard Hendersone3f7dc22021-08-24 10:30:38 -07001252 return true;
1253 }
Richard Hendersonf2457572023-10-25 18:39:44 -07001254
1255 /* Fold sub2 r,x,i to add2 r,x,-i */
1256 if (!add && b_const) {
1257 uint64_t bl = arg_info(op->args[4])->val;
1258 uint64_t bh = arg_info(op->args[5])->val;
1259
1260 /* Negate the two parts without assembling and disassembling. */
1261 bl = -bl;
1262 bh = ~bh + !bl;
1263
1264 op->opc = (ctx->type == TCG_TYPE_I32
1265 ? INDEX_op_add2_i32 : INDEX_op_add2_i64);
1266 op->args[4] = arg_new_constant(ctx, bl);
1267 op->args[5] = arg_new_constant(ctx, bh);
1268 }
Richard Hendersone3f7dc22021-08-24 10:30:38 -07001269 return false;
1270}
1271
Richard Henderson9531c072021-08-26 06:51:39 -07001272static bool fold_add2(OptContext *ctx, TCGOp *op)
Richard Hendersone3f7dc22021-08-24 10:30:38 -07001273{
Richard Henderson7a2f7082021-08-26 07:06:39 -07001274 /* Note that the high and low parts may be independently swapped. */
1275 swap_commutative(op->args[0], &op->args[2], &op->args[4]);
1276 swap_commutative(op->args[1], &op->args[3], &op->args[5]);
1277
Richard Henderson9531c072021-08-26 06:51:39 -07001278 return fold_addsub2(ctx, op, true);
Richard Hendersone3f7dc22021-08-24 10:30:38 -07001279}
1280
Richard Henderson2f9f08b2021-08-25 12:03:48 -07001281static bool fold_and(OptContext *ctx, TCGOp *op)
1282{
Richard Hendersonfae450b2021-08-25 22:42:19 -07001283 uint64_t z1, z2;
1284
Richard Henderson7a2f7082021-08-26 07:06:39 -07001285 if (fold_const2_commutative(ctx, op) ||
Richard Hendersone8679952021-08-25 13:19:52 -07001286 fold_xi_to_i(ctx, op, 0) ||
Richard Hendersona63ce0e2021-08-25 20:28:53 -07001287 fold_xi_to_x(ctx, op, -1) ||
Richard Hendersonca7bb042021-08-25 13:14:21 -07001288 fold_xx_to_x(ctx, op)) {
1289 return true;
1290 }
Richard Hendersonfae450b2021-08-25 22:42:19 -07001291
1292 z1 = arg_info(op->args[1])->z_mask;
1293 z2 = arg_info(op->args[2])->z_mask;
1294 ctx->z_mask = z1 & z2;
1295
1296 /*
Richard Henderson3f2b1f82021-08-26 13:08:54 -07001297 * Sign repetitions are perforce all identical, whether they are 1 or 0.
1298 * Bitwise operations preserve the relative quantity of the repetitions.
1299 */
1300 ctx->s_mask = arg_info(op->args[1])->s_mask
1301 & arg_info(op->args[2])->s_mask;
1302
1303 /*
Richard Hendersonfae450b2021-08-25 22:42:19 -07001304 * Known-zeros does not imply known-ones. Therefore unless
1305 * arg2 is constant, we can't infer affected bits from it.
1306 */
1307 if (arg_is_const(op->args[2])) {
1308 ctx->a_mask = z1 & ~z2;
1309 }
1310
1311 return fold_masks(ctx, op);
Richard Henderson2f9f08b2021-08-25 12:03:48 -07001312}
1313
1314static bool fold_andc(OptContext *ctx, TCGOp *op)
1315{
Richard Hendersonfae450b2021-08-25 22:42:19 -07001316 uint64_t z1;
1317
Richard Hendersoncbe42fb2021-08-25 13:02:00 -07001318 if (fold_const2(ctx, op) ||
Richard Henderson0e0a32b2021-08-24 13:18:01 -07001319 fold_xx_to_i(ctx, op, 0) ||
Richard Hendersona63ce0e2021-08-25 20:28:53 -07001320 fold_xi_to_x(ctx, op, 0) ||
Richard Henderson0e0a32b2021-08-24 13:18:01 -07001321 fold_ix_to_not(ctx, op, -1)) {
Richard Hendersoncbe42fb2021-08-25 13:02:00 -07001322 return true;
1323 }
Richard Hendersonfae450b2021-08-25 22:42:19 -07001324
1325 z1 = arg_info(op->args[1])->z_mask;
1326
1327 /*
1328 * Known-zeros does not imply known-ones. Therefore unless
1329 * arg2 is constant, we can't infer anything from it.
1330 */
1331 if (arg_is_const(op->args[2])) {
1332 uint64_t z2 = ~arg_info(op->args[2])->z_mask;
1333 ctx->a_mask = z1 & ~z2;
1334 z1 &= z2;
1335 }
1336 ctx->z_mask = z1;
1337
Richard Henderson3f2b1f82021-08-26 13:08:54 -07001338 ctx->s_mask = arg_info(op->args[1])->s_mask
1339 & arg_info(op->args[2])->s_mask;
Richard Hendersonfae450b2021-08-25 22:42:19 -07001340 return fold_masks(ctx, op);
Richard Henderson2f9f08b2021-08-25 12:03:48 -07001341}
1342
Richard Henderson079b0802021-08-24 09:30:59 -07001343static bool fold_brcond(OptContext *ctx, TCGOp *op)
1344{
Richard Hendersonfb04ab72024-01-10 18:21:58 +11001345 int i = do_constant_folding_cond1(ctx, op, NO_DEST, &op->args[0],
Richard Henderson246c4b72023-10-24 16:36:50 -07001346 &op->args[1], &op->args[2]);
Richard Henderson079b0802021-08-24 09:30:59 -07001347 if (i == 0) {
1348 tcg_op_remove(ctx->tcg, op);
1349 return true;
1350 }
1351 if (i > 0) {
1352 op->opc = INDEX_op_br;
1353 op->args[0] = op->args[3];
1354 }
1355 return false;
1356}
1357
Richard Henderson764d2ab2021-08-24 09:22:11 -07001358static bool fold_brcond2(OptContext *ctx, TCGOp *op)
1359{
Richard Henderson7e64b112023-10-24 16:53:56 -07001360 TCGCond cond;
1361 TCGArg label;
Richard Henderson7a2f7082021-08-26 07:06:39 -07001362 int i, inv = 0;
Richard Henderson764d2ab2021-08-24 09:22:11 -07001363
Richard Hendersonfb04ab72024-01-10 18:21:58 +11001364 i = do_constant_folding_cond2(ctx, op, &op->args[0]);
Richard Henderson7e64b112023-10-24 16:53:56 -07001365 cond = op->args[4];
1366 label = op->args[5];
Richard Henderson764d2ab2021-08-24 09:22:11 -07001367 if (i >= 0) {
1368 goto do_brcond_const;
1369 }
1370
1371 switch (cond) {
1372 case TCG_COND_LT:
1373 case TCG_COND_GE:
1374 /*
1375 * Simplify LT/GE comparisons vs zero to a single compare
1376 * vs the high word of the input.
1377 */
Richard Henderson27cdb852023-10-23 11:38:00 -07001378 if (arg_is_const_val(op->args[2], 0) &&
1379 arg_is_const_val(op->args[3], 0)) {
Richard Henderson764d2ab2021-08-24 09:22:11 -07001380 goto do_brcond_high;
1381 }
1382 break;
1383
1384 case TCG_COND_NE:
1385 inv = 1;
1386 QEMU_FALLTHROUGH;
1387 case TCG_COND_EQ:
1388 /*
1389 * Simplify EQ/NE comparisons where one of the pairs
1390 * can be simplified.
1391 */
Richard Henderson67f84c92021-08-25 08:00:20 -07001392 i = do_constant_folding_cond(TCG_TYPE_I32, op->args[0],
Richard Henderson764d2ab2021-08-24 09:22:11 -07001393 op->args[2], cond);
1394 switch (i ^ inv) {
1395 case 0:
1396 goto do_brcond_const;
1397 case 1:
1398 goto do_brcond_high;
1399 }
1400
Richard Henderson67f84c92021-08-25 08:00:20 -07001401 i = do_constant_folding_cond(TCG_TYPE_I32, op->args[1],
Richard Henderson764d2ab2021-08-24 09:22:11 -07001402 op->args[3], cond);
1403 switch (i ^ inv) {
1404 case 0:
1405 goto do_brcond_const;
1406 case 1:
Richard Hendersonceb9ee02023-10-23 23:44:27 -07001407 goto do_brcond_low;
1408 }
1409 break;
1410
1411 case TCG_COND_TSTEQ:
1412 case TCG_COND_TSTNE:
1413 if (arg_is_const_val(op->args[2], 0)) {
1414 goto do_brcond_high;
1415 }
1416 if (arg_is_const_val(op->args[3], 0)) {
1417 goto do_brcond_low;
Richard Henderson764d2ab2021-08-24 09:22:11 -07001418 }
1419 break;
1420
1421 default:
1422 break;
1423
Richard Hendersonceb9ee02023-10-23 23:44:27 -07001424 do_brcond_low:
1425 op->opc = INDEX_op_brcond_i32;
1426 op->args[1] = op->args[2];
1427 op->args[2] = cond;
1428 op->args[3] = label;
1429 return fold_brcond(ctx, op);
1430
Richard Henderson764d2ab2021-08-24 09:22:11 -07001431 do_brcond_high:
1432 op->opc = INDEX_op_brcond_i32;
1433 op->args[0] = op->args[1];
1434 op->args[1] = op->args[3];
1435 op->args[2] = cond;
1436 op->args[3] = label;
Richard Hendersonceb9ee02023-10-23 23:44:27 -07001437 return fold_brcond(ctx, op);
Richard Henderson764d2ab2021-08-24 09:22:11 -07001438
1439 do_brcond_const:
1440 if (i == 0) {
1441 tcg_op_remove(ctx->tcg, op);
1442 return true;
1443 }
1444 op->opc = INDEX_op_br;
1445 op->args[0] = label;
1446 break;
1447 }
1448 return false;
1449}
1450
Richard Henderson09bacdc2021-08-24 11:58:12 -07001451static bool fold_bswap(OptContext *ctx, TCGOp *op)
1452{
Richard Henderson57fe5c62021-08-26 12:04:46 -07001453 uint64_t z_mask, s_mask, sign;
Richard Hendersonfae450b2021-08-25 22:42:19 -07001454
Richard Henderson09bacdc2021-08-24 11:58:12 -07001455 if (arg_is_const(op->args[1])) {
1456 uint64_t t = arg_info(op->args[1])->val;
1457
Richard Henderson67f84c92021-08-25 08:00:20 -07001458 t = do_constant_folding(op->opc, ctx->type, t, op->args[2]);
Richard Henderson09bacdc2021-08-24 11:58:12 -07001459 return tcg_opt_gen_movi(ctx, op, op->args[0], t);
1460 }
Richard Hendersonfae450b2021-08-25 22:42:19 -07001461
1462 z_mask = arg_info(op->args[1])->z_mask;
Richard Henderson57fe5c62021-08-26 12:04:46 -07001463
Richard Hendersonfae450b2021-08-25 22:42:19 -07001464 switch (op->opc) {
1465 case INDEX_op_bswap16_i32:
1466 case INDEX_op_bswap16_i64:
1467 z_mask = bswap16(z_mask);
1468 sign = INT16_MIN;
1469 break;
1470 case INDEX_op_bswap32_i32:
1471 case INDEX_op_bswap32_i64:
1472 z_mask = bswap32(z_mask);
1473 sign = INT32_MIN;
1474 break;
1475 case INDEX_op_bswap64_i64:
1476 z_mask = bswap64(z_mask);
1477 sign = INT64_MIN;
1478 break;
1479 default:
1480 g_assert_not_reached();
1481 }
Richard Henderson57fe5c62021-08-26 12:04:46 -07001482 s_mask = smask_from_zmask(z_mask);
Richard Hendersonfae450b2021-08-25 22:42:19 -07001483
1484 switch (op->args[2] & (TCG_BSWAP_OZ | TCG_BSWAP_OS)) {
1485 case TCG_BSWAP_OZ:
1486 break;
1487 case TCG_BSWAP_OS:
1488 /* If the sign bit may be 1, force all the bits above to 1. */
1489 if (z_mask & sign) {
1490 z_mask |= sign;
Richard Henderson57fe5c62021-08-26 12:04:46 -07001491 s_mask = sign << 1;
Richard Hendersonfae450b2021-08-25 22:42:19 -07001492 }
1493 break;
1494 default:
1495 /* The high bits are undefined: force all bits above the sign to 1. */
1496 z_mask |= sign << 1;
Richard Henderson57fe5c62021-08-26 12:04:46 -07001497 s_mask = 0;
Richard Hendersonfae450b2021-08-25 22:42:19 -07001498 break;
1499 }
1500 ctx->z_mask = z_mask;
Richard Henderson57fe5c62021-08-26 12:04:46 -07001501 ctx->s_mask = s_mask;
Richard Hendersonfae450b2021-08-25 22:42:19 -07001502
1503 return fold_masks(ctx, op);
Richard Henderson09bacdc2021-08-24 11:58:12 -07001504}
1505
Richard Henderson5cf32be2021-08-24 08:17:08 -07001506static bool fold_call(OptContext *ctx, TCGOp *op)
1507{
1508 TCGContext *s = ctx->tcg;
1509 int nb_oargs = TCGOP_CALLO(op);
1510 int nb_iargs = TCGOP_CALLI(op);
1511 int flags, i;
1512
1513 init_arguments(ctx, op, nb_oargs + nb_iargs);
1514 copy_propagate(ctx, op, nb_oargs, nb_iargs);
1515
1516 /* If the function reads or writes globals, reset temp data. */
1517 flags = tcg_call_flags(op);
1518 if (!(flags & (TCG_CALL_NO_READ_GLOBALS | TCG_CALL_NO_WRITE_GLOBALS))) {
1519 int nb_globals = s->nb_globals;
1520
1521 for (i = 0; i < nb_globals; i++) {
1522 if (test_bit(i, ctx->temps_used.l)) {
Richard Henderson986cac12023-01-09 13:59:35 -08001523 reset_ts(ctx, &ctx->tcg->temps[i]);
Richard Henderson5cf32be2021-08-24 08:17:08 -07001524 }
1525 }
1526 }
1527
Richard Hendersonab84dc32023-08-23 23:04:24 -07001528 /* If the function has side effects, reset mem data. */
1529 if (!(flags & TCG_CALL_NO_SIDE_EFFECTS)) {
1530 remove_mem_copy_all(ctx);
1531 }
1532
Richard Henderson5cf32be2021-08-24 08:17:08 -07001533 /* Reset temp data for outputs. */
1534 for (i = 0; i < nb_oargs; i++) {
Richard Henderson986cac12023-01-09 13:59:35 -08001535 reset_temp(ctx, op->args[i]);
Richard Henderson5cf32be2021-08-24 08:17:08 -07001536 }
1537
1538 /* Stop optimizing MB across calls. */
1539 ctx->prev_mb = NULL;
1540 return true;
1541}
1542
Richard Henderson30dd0bf2021-08-24 10:51:34 -07001543static bool fold_count_zeros(OptContext *ctx, TCGOp *op)
1544{
Richard Hendersonfae450b2021-08-25 22:42:19 -07001545 uint64_t z_mask;
1546
Richard Henderson30dd0bf2021-08-24 10:51:34 -07001547 if (arg_is_const(op->args[1])) {
1548 uint64_t t = arg_info(op->args[1])->val;
1549
1550 if (t != 0) {
Richard Henderson67f84c92021-08-25 08:00:20 -07001551 t = do_constant_folding(op->opc, ctx->type, t, 0);
Richard Henderson30dd0bf2021-08-24 10:51:34 -07001552 return tcg_opt_gen_movi(ctx, op, op->args[0], t);
1553 }
1554 return tcg_opt_gen_mov(ctx, op, op->args[0], op->args[2]);
1555 }
Richard Hendersonfae450b2021-08-25 22:42:19 -07001556
1557 switch (ctx->type) {
1558 case TCG_TYPE_I32:
1559 z_mask = 31;
1560 break;
1561 case TCG_TYPE_I64:
1562 z_mask = 63;
1563 break;
1564 default:
1565 g_assert_not_reached();
1566 }
1567 ctx->z_mask = arg_info(op->args[2])->z_mask | z_mask;
Richard Henderson2b9d0c52021-08-26 13:24:17 -07001568 ctx->s_mask = smask_from_zmask(ctx->z_mask);
Richard Henderson30dd0bf2021-08-24 10:51:34 -07001569 return false;
1570}
1571
Richard Henderson2f9f08b2021-08-25 12:03:48 -07001572static bool fold_ctpop(OptContext *ctx, TCGOp *op)
1573{
Richard Hendersonfae450b2021-08-25 22:42:19 -07001574 if (fold_const1(ctx, op)) {
1575 return true;
1576 }
1577
1578 switch (ctx->type) {
1579 case TCG_TYPE_I32:
1580 ctx->z_mask = 32 | 31;
1581 break;
1582 case TCG_TYPE_I64:
1583 ctx->z_mask = 64 | 63;
1584 break;
1585 default:
1586 g_assert_not_reached();
1587 }
Richard Henderson2b9d0c52021-08-26 13:24:17 -07001588 ctx->s_mask = smask_from_zmask(ctx->z_mask);
Richard Hendersonfae450b2021-08-25 22:42:19 -07001589 return false;
Richard Henderson2f9f08b2021-08-25 12:03:48 -07001590}
1591
Richard Henderson1b1907b2021-08-24 10:47:04 -07001592static bool fold_deposit(OptContext *ctx, TCGOp *op)
1593{
Richard Henderson8f7a8402023-08-13 11:03:05 -07001594 TCGOpcode and_opc;
1595
Richard Henderson1b1907b2021-08-24 10:47:04 -07001596 if (arg_is_const(op->args[1]) && arg_is_const(op->args[2])) {
1597 uint64_t t1 = arg_info(op->args[1])->val;
1598 uint64_t t2 = arg_info(op->args[2])->val;
1599
1600 t1 = deposit64(t1, op->args[3], op->args[4], t2);
1601 return tcg_opt_gen_movi(ctx, op, op->args[0], t1);
1602 }
Richard Hendersonfae450b2021-08-25 22:42:19 -07001603
Richard Henderson8f7a8402023-08-13 11:03:05 -07001604 switch (ctx->type) {
1605 case TCG_TYPE_I32:
1606 and_opc = INDEX_op_and_i32;
1607 break;
1608 case TCG_TYPE_I64:
1609 and_opc = INDEX_op_and_i64;
1610 break;
1611 default:
1612 g_assert_not_reached();
1613 }
1614
1615 /* Inserting a value into zero at offset 0. */
Richard Henderson27cdb852023-10-23 11:38:00 -07001616 if (arg_is_const_val(op->args[1], 0) && op->args[3] == 0) {
Richard Henderson8f7a8402023-08-13 11:03:05 -07001617 uint64_t mask = MAKE_64BIT_MASK(0, op->args[4]);
1618
1619 op->opc = and_opc;
1620 op->args[1] = op->args[2];
Richard Henderson26aac972023-10-23 12:31:57 -07001621 op->args[2] = arg_new_constant(ctx, mask);
Richard Henderson8f7a8402023-08-13 11:03:05 -07001622 ctx->z_mask = mask & arg_info(op->args[1])->z_mask;
1623 return false;
1624 }
1625
1626 /* Inserting zero into a value. */
Richard Henderson27cdb852023-10-23 11:38:00 -07001627 if (arg_is_const_val(op->args[2], 0)) {
Richard Henderson8f7a8402023-08-13 11:03:05 -07001628 uint64_t mask = deposit64(-1, op->args[3], op->args[4], 0);
1629
1630 op->opc = and_opc;
Richard Henderson26aac972023-10-23 12:31:57 -07001631 op->args[2] = arg_new_constant(ctx, mask);
Richard Henderson8f7a8402023-08-13 11:03:05 -07001632 ctx->z_mask = mask & arg_info(op->args[1])->z_mask;
1633 return false;
1634 }
1635
Richard Hendersonfae450b2021-08-25 22:42:19 -07001636 ctx->z_mask = deposit64(arg_info(op->args[1])->z_mask,
1637 op->args[3], op->args[4],
1638 arg_info(op->args[2])->z_mask);
Richard Henderson1b1907b2021-08-24 10:47:04 -07001639 return false;
1640}
1641
Richard Henderson2f9f08b2021-08-25 12:03:48 -07001642static bool fold_divide(OptContext *ctx, TCGOp *op)
1643{
Richard Henderson2f9d9a32021-10-25 11:30:14 -07001644 if (fold_const2(ctx, op) ||
1645 fold_xi_to_x(ctx, op, 1)) {
1646 return true;
1647 }
1648 return false;
Richard Henderson2f9f08b2021-08-25 12:03:48 -07001649}
1650
Richard Henderson8cdb3fc2021-08-24 12:06:33 -07001651static bool fold_dup(OptContext *ctx, TCGOp *op)
1652{
1653 if (arg_is_const(op->args[1])) {
1654 uint64_t t = arg_info(op->args[1])->val;
1655 t = dup_const(TCGOP_VECE(op), t);
1656 return tcg_opt_gen_movi(ctx, op, op->args[0], t);
1657 }
1658 return false;
1659}
1660
1661static bool fold_dup2(OptContext *ctx, TCGOp *op)
1662{
1663 if (arg_is_const(op->args[1]) && arg_is_const(op->args[2])) {
1664 uint64_t t = deposit64(arg_info(op->args[1])->val, 32, 32,
1665 arg_info(op->args[2])->val);
1666 return tcg_opt_gen_movi(ctx, op, op->args[0], t);
1667 }
1668
1669 if (args_are_copies(op->args[1], op->args[2])) {
1670 op->opc = INDEX_op_dup_vec;
1671 TCGOP_VECE(op) = MO_32;
1672 }
1673 return false;
1674}
1675
Richard Henderson2f9f08b2021-08-25 12:03:48 -07001676static bool fold_eqv(OptContext *ctx, TCGOp *op)
1677{
Richard Henderson7a2f7082021-08-26 07:06:39 -07001678 if (fold_const2_commutative(ctx, op) ||
Richard Hendersona63ce0e2021-08-25 20:28:53 -07001679 fold_xi_to_x(ctx, op, -1) ||
Richard Henderson0e0a32b2021-08-24 13:18:01 -07001680 fold_xi_to_not(ctx, op, 0)) {
1681 return true;
1682 }
Richard Henderson3f2b1f82021-08-26 13:08:54 -07001683
1684 ctx->s_mask = arg_info(op->args[1])->s_mask
1685 & arg_info(op->args[2])->s_mask;
Richard Henderson0e0a32b2021-08-24 13:18:01 -07001686 return false;
Richard Henderson2f9f08b2021-08-25 12:03:48 -07001687}
1688
Richard Hendersonb6617c82021-08-24 10:44:53 -07001689static bool fold_extract(OptContext *ctx, TCGOp *op)
1690{
Richard Hendersonfae450b2021-08-25 22:42:19 -07001691 uint64_t z_mask_old, z_mask;
Richard Henderson57fe5c62021-08-26 12:04:46 -07001692 int pos = op->args[2];
1693 int len = op->args[3];
Richard Hendersonfae450b2021-08-25 22:42:19 -07001694
Richard Hendersonb6617c82021-08-24 10:44:53 -07001695 if (arg_is_const(op->args[1])) {
1696 uint64_t t;
1697
1698 t = arg_info(op->args[1])->val;
Richard Henderson57fe5c62021-08-26 12:04:46 -07001699 t = extract64(t, pos, len);
Richard Hendersonb6617c82021-08-24 10:44:53 -07001700 return tcg_opt_gen_movi(ctx, op, op->args[0], t);
1701 }
Richard Hendersonfae450b2021-08-25 22:42:19 -07001702
1703 z_mask_old = arg_info(op->args[1])->z_mask;
Richard Henderson57fe5c62021-08-26 12:04:46 -07001704 z_mask = extract64(z_mask_old, pos, len);
1705 if (pos == 0) {
Richard Hendersonfae450b2021-08-25 22:42:19 -07001706 ctx->a_mask = z_mask_old ^ z_mask;
1707 }
1708 ctx->z_mask = z_mask;
Richard Henderson57fe5c62021-08-26 12:04:46 -07001709 ctx->s_mask = smask_from_zmask(z_mask);
Richard Hendersonfae450b2021-08-25 22:42:19 -07001710
1711 return fold_masks(ctx, op);
Richard Hendersonb6617c82021-08-24 10:44:53 -07001712}
1713
Richard Hendersondcd08992021-08-24 10:41:39 -07001714static bool fold_extract2(OptContext *ctx, TCGOp *op)
1715{
1716 if (arg_is_const(op->args[1]) && arg_is_const(op->args[2])) {
1717 uint64_t v1 = arg_info(op->args[1])->val;
1718 uint64_t v2 = arg_info(op->args[2])->val;
1719 int shr = op->args[3];
1720
1721 if (op->opc == INDEX_op_extract2_i64) {
1722 v1 >>= shr;
1723 v2 <<= 64 - shr;
1724 } else {
1725 v1 = (uint32_t)v1 >> shr;
Richard Henderson225bec02021-11-09 23:17:59 +01001726 v2 = (uint64_t)((int32_t)v2 << (32 - shr));
Richard Hendersondcd08992021-08-24 10:41:39 -07001727 }
1728 return tcg_opt_gen_movi(ctx, op, op->args[0], v1 | v2);
1729 }
1730 return false;
1731}
1732
Richard Henderson2f9f08b2021-08-25 12:03:48 -07001733static bool fold_exts(OptContext *ctx, TCGOp *op)
1734{
Richard Henderson57fe5c62021-08-26 12:04:46 -07001735 uint64_t s_mask_old, s_mask, z_mask, sign;
Richard Hendersonfae450b2021-08-25 22:42:19 -07001736 bool type_change = false;
1737
1738 if (fold_const1(ctx, op)) {
1739 return true;
1740 }
1741
Richard Henderson57fe5c62021-08-26 12:04:46 -07001742 z_mask = arg_info(op->args[1])->z_mask;
1743 s_mask = arg_info(op->args[1])->s_mask;
1744 s_mask_old = s_mask;
Richard Hendersonfae450b2021-08-25 22:42:19 -07001745
1746 switch (op->opc) {
1747 CASE_OP_32_64(ext8s):
1748 sign = INT8_MIN;
1749 z_mask = (uint8_t)z_mask;
1750 break;
1751 CASE_OP_32_64(ext16s):
1752 sign = INT16_MIN;
1753 z_mask = (uint16_t)z_mask;
1754 break;
1755 case INDEX_op_ext_i32_i64:
1756 type_change = true;
1757 QEMU_FALLTHROUGH;
1758 case INDEX_op_ext32s_i64:
1759 sign = INT32_MIN;
1760 z_mask = (uint32_t)z_mask;
1761 break;
1762 default:
1763 g_assert_not_reached();
1764 }
1765
1766 if (z_mask & sign) {
1767 z_mask |= sign;
Richard Hendersonfae450b2021-08-25 22:42:19 -07001768 }
Richard Henderson57fe5c62021-08-26 12:04:46 -07001769 s_mask |= sign << 1;
1770
Richard Hendersonfae450b2021-08-25 22:42:19 -07001771 ctx->z_mask = z_mask;
Richard Henderson57fe5c62021-08-26 12:04:46 -07001772 ctx->s_mask = s_mask;
1773 if (!type_change) {
1774 ctx->a_mask = s_mask & ~s_mask_old;
1775 }
Richard Hendersonfae450b2021-08-25 22:42:19 -07001776
1777 return fold_masks(ctx, op);
Richard Henderson2f9f08b2021-08-25 12:03:48 -07001778}
1779
1780static bool fold_extu(OptContext *ctx, TCGOp *op)
1781{
Richard Hendersonfae450b2021-08-25 22:42:19 -07001782 uint64_t z_mask_old, z_mask;
1783 bool type_change = false;
1784
1785 if (fold_const1(ctx, op)) {
1786 return true;
1787 }
1788
1789 z_mask_old = z_mask = arg_info(op->args[1])->z_mask;
1790
1791 switch (op->opc) {
1792 CASE_OP_32_64(ext8u):
1793 z_mask = (uint8_t)z_mask;
1794 break;
1795 CASE_OP_32_64(ext16u):
1796 z_mask = (uint16_t)z_mask;
1797 break;
1798 case INDEX_op_extrl_i64_i32:
1799 case INDEX_op_extu_i32_i64:
1800 type_change = true;
1801 QEMU_FALLTHROUGH;
1802 case INDEX_op_ext32u_i64:
1803 z_mask = (uint32_t)z_mask;
1804 break;
1805 case INDEX_op_extrh_i64_i32:
1806 type_change = true;
1807 z_mask >>= 32;
1808 break;
1809 default:
1810 g_assert_not_reached();
1811 }
1812
1813 ctx->z_mask = z_mask;
Richard Henderson57fe5c62021-08-26 12:04:46 -07001814 ctx->s_mask = smask_from_zmask(z_mask);
Richard Hendersonfae450b2021-08-25 22:42:19 -07001815 if (!type_change) {
1816 ctx->a_mask = z_mask_old ^ z_mask;
1817 }
1818 return fold_masks(ctx, op);
Richard Henderson2f9f08b2021-08-25 12:03:48 -07001819}
1820
Richard Henderson3eefdf22021-08-25 11:06:43 -07001821static bool fold_mb(OptContext *ctx, TCGOp *op)
1822{
1823 /* Eliminate duplicate and redundant fence instructions. */
1824 if (ctx->prev_mb) {
1825 /*
1826 * Merge two barriers of the same type into one,
1827 * or a weaker barrier into a stronger one,
1828 * or two weaker barriers into a stronger one.
1829 * mb X; mb Y => mb X|Y
1830 * mb; strl => mb; st
1831 * ldaq; mb => ld; mb
1832 * ldaq; strl => ld; mb; st
1833 * Other combinations are also merged into a strong
1834 * barrier. This is stricter than specified but for
1835 * the purposes of TCG is better than not optimizing.
1836 */
1837 ctx->prev_mb->args[0] |= op->args[0];
1838 tcg_op_remove(ctx->tcg, op);
1839 } else {
1840 ctx->prev_mb = op;
1841 }
1842 return true;
1843}
1844
Richard Henderson2cfac7f2021-08-25 13:05:43 -07001845static bool fold_mov(OptContext *ctx, TCGOp *op)
1846{
1847 return tcg_opt_gen_mov(ctx, op, op->args[0], op->args[1]);
1848}
1849
Richard Henderson0c310a32021-08-24 10:37:24 -07001850static bool fold_movcond(OptContext *ctx, TCGOp *op)
1851{
Richard Henderson7a2f7082021-08-26 07:06:39 -07001852 int i;
Richard Henderson0c310a32021-08-24 10:37:24 -07001853
Richard Henderson141125e2024-09-06 21:00:10 -07001854 /* If true and false values are the same, eliminate the cmp. */
1855 if (args_are_copies(op->args[3], op->args[4])) {
1856 return tcg_opt_gen_mov(ctx, op, op->args[0], op->args[3]);
1857 }
1858
Richard Henderson7a2f7082021-08-26 07:06:39 -07001859 /*
1860 * Canonicalize the "false" input reg to match the destination reg so
1861 * that the tcg backend can implement a "move if true" operation.
1862 */
1863 if (swap_commutative(op->args[0], &op->args[4], &op->args[3])) {
Richard Henderson246c4b72023-10-24 16:36:50 -07001864 op->args[5] = tcg_invert_cond(op->args[5]);
Richard Henderson7a2f7082021-08-26 07:06:39 -07001865 }
1866
Richard Hendersonfb04ab72024-01-10 18:21:58 +11001867 i = do_constant_folding_cond1(ctx, op, NO_DEST, &op->args[1],
Richard Henderson246c4b72023-10-24 16:36:50 -07001868 &op->args[2], &op->args[5]);
Richard Henderson0c310a32021-08-24 10:37:24 -07001869 if (i >= 0) {
1870 return tcg_opt_gen_mov(ctx, op, op->args[0], op->args[4 - i]);
1871 }
1872
Richard Hendersonfae450b2021-08-25 22:42:19 -07001873 ctx->z_mask = arg_info(op->args[3])->z_mask
1874 | arg_info(op->args[4])->z_mask;
Richard Henderson3f2b1f82021-08-26 13:08:54 -07001875 ctx->s_mask = arg_info(op->args[3])->s_mask
1876 & arg_info(op->args[4])->s_mask;
Richard Hendersonfae450b2021-08-25 22:42:19 -07001877
Richard Henderson0c310a32021-08-24 10:37:24 -07001878 if (arg_is_const(op->args[3]) && arg_is_const(op->args[4])) {
1879 uint64_t tv = arg_info(op->args[3])->val;
1880 uint64_t fv = arg_info(op->args[4])->val;
Richard Henderson36355022023-08-04 23:24:04 +00001881 TCGOpcode opc, negopc = 0;
Richard Henderson246c4b72023-10-24 16:36:50 -07001882 TCGCond cond = op->args[5];
Richard Henderson0c310a32021-08-24 10:37:24 -07001883
Richard Henderson67f84c92021-08-25 08:00:20 -07001884 switch (ctx->type) {
1885 case TCG_TYPE_I32:
1886 opc = INDEX_op_setcond_i32;
Richard Henderson36355022023-08-04 23:24:04 +00001887 if (TCG_TARGET_HAS_negsetcond_i32) {
1888 negopc = INDEX_op_negsetcond_i32;
1889 }
1890 tv = (int32_t)tv;
1891 fv = (int32_t)fv;
Richard Henderson67f84c92021-08-25 08:00:20 -07001892 break;
1893 case TCG_TYPE_I64:
1894 opc = INDEX_op_setcond_i64;
Richard Henderson36355022023-08-04 23:24:04 +00001895 if (TCG_TARGET_HAS_negsetcond_i64) {
1896 negopc = INDEX_op_negsetcond_i64;
1897 }
Richard Henderson67f84c92021-08-25 08:00:20 -07001898 break;
1899 default:
1900 g_assert_not_reached();
1901 }
Richard Henderson0c310a32021-08-24 10:37:24 -07001902
1903 if (tv == 1 && fv == 0) {
1904 op->opc = opc;
1905 op->args[3] = cond;
1906 } else if (fv == 1 && tv == 0) {
1907 op->opc = opc;
1908 op->args[3] = tcg_invert_cond(cond);
Richard Henderson36355022023-08-04 23:24:04 +00001909 } else if (negopc) {
1910 if (tv == -1 && fv == 0) {
1911 op->opc = negopc;
1912 op->args[3] = cond;
1913 } else if (fv == -1 && tv == 0) {
1914 op->opc = negopc;
1915 op->args[3] = tcg_invert_cond(cond);
1916 }
Richard Henderson0c310a32021-08-24 10:37:24 -07001917 }
1918 }
1919 return false;
1920}
1921
Richard Henderson2f9f08b2021-08-25 12:03:48 -07001922static bool fold_mul(OptContext *ctx, TCGOp *op)
1923{
Richard Hendersone8679952021-08-25 13:19:52 -07001924 if (fold_const2(ctx, op) ||
Richard Henderson5b5cf472021-10-25 11:19:14 -07001925 fold_xi_to_i(ctx, op, 0) ||
1926 fold_xi_to_x(ctx, op, 1)) {
Richard Hendersone8679952021-08-25 13:19:52 -07001927 return true;
1928 }
1929 return false;
Richard Henderson2f9f08b2021-08-25 12:03:48 -07001930}
1931
1932static bool fold_mul_highpart(OptContext *ctx, TCGOp *op)
1933{
Richard Henderson7a2f7082021-08-26 07:06:39 -07001934 if (fold_const2_commutative(ctx, op) ||
Richard Hendersone8679952021-08-25 13:19:52 -07001935 fold_xi_to_i(ctx, op, 0)) {
1936 return true;
1937 }
1938 return false;
Richard Henderson2f9f08b2021-08-25 12:03:48 -07001939}
1940
Richard Henderson407112b2021-08-26 06:33:04 -07001941static bool fold_multiply2(OptContext *ctx, TCGOp *op)
Richard Henderson6b8ac0d2021-08-24 10:24:12 -07001942{
Richard Henderson7a2f7082021-08-26 07:06:39 -07001943 swap_commutative(op->args[0], &op->args[2], &op->args[3]);
1944
Richard Henderson6b8ac0d2021-08-24 10:24:12 -07001945 if (arg_is_const(op->args[2]) && arg_is_const(op->args[3])) {
Richard Henderson407112b2021-08-26 06:33:04 -07001946 uint64_t a = arg_info(op->args[2])->val;
1947 uint64_t b = arg_info(op->args[3])->val;
1948 uint64_t h, l;
Richard Henderson6b8ac0d2021-08-24 10:24:12 -07001949 TCGArg rl, rh;
Richard Henderson407112b2021-08-26 06:33:04 -07001950 TCGOp *op2;
1951
1952 switch (op->opc) {
1953 case INDEX_op_mulu2_i32:
1954 l = (uint64_t)(uint32_t)a * (uint32_t)b;
1955 h = (int32_t)(l >> 32);
1956 l = (int32_t)l;
1957 break;
1958 case INDEX_op_muls2_i32:
1959 l = (int64_t)(int32_t)a * (int32_t)b;
1960 h = l >> 32;
1961 l = (int32_t)l;
1962 break;
1963 case INDEX_op_mulu2_i64:
1964 mulu64(&l, &h, a, b);
1965 break;
1966 case INDEX_op_muls2_i64:
1967 muls64(&l, &h, a, b);
1968 break;
1969 default:
1970 g_assert_not_reached();
1971 }
Richard Henderson6b8ac0d2021-08-24 10:24:12 -07001972
1973 rl = op->args[0];
1974 rh = op->args[1];
Richard Henderson407112b2021-08-26 06:33:04 -07001975
1976 /* The proper opcode is supplied by tcg_opt_gen_mov. */
Philippe Mathieu-Daudéd4478942022-12-18 22:18:31 +01001977 op2 = tcg_op_insert_before(ctx->tcg, op, 0, 2);
Richard Henderson407112b2021-08-26 06:33:04 -07001978
1979 tcg_opt_gen_movi(ctx, op, rl, l);
1980 tcg_opt_gen_movi(ctx, op2, rh, h);
Richard Henderson6b8ac0d2021-08-24 10:24:12 -07001981 return true;
1982 }
1983 return false;
1984}
1985
Richard Henderson2f9f08b2021-08-25 12:03:48 -07001986static bool fold_nand(OptContext *ctx, TCGOp *op)
1987{
Richard Henderson7a2f7082021-08-26 07:06:39 -07001988 if (fold_const2_commutative(ctx, op) ||
Richard Henderson0e0a32b2021-08-24 13:18:01 -07001989 fold_xi_to_not(ctx, op, -1)) {
1990 return true;
1991 }
Richard Henderson3f2b1f82021-08-26 13:08:54 -07001992
1993 ctx->s_mask = arg_info(op->args[1])->s_mask
1994 & arg_info(op->args[2])->s_mask;
Richard Henderson0e0a32b2021-08-24 13:18:01 -07001995 return false;
Richard Henderson2f9f08b2021-08-25 12:03:48 -07001996}
1997
Richard Hendersone25fe882024-04-04 20:53:50 +00001998static bool fold_neg_no_const(OptContext *ctx, TCGOp *op)
Richard Henderson2f9f08b2021-08-25 12:03:48 -07001999{
Richard Hendersonfae450b2021-08-25 22:42:19 -07002000 /* Set to 1 all bits to the left of the rightmost. */
Richard Hendersone25fe882024-04-04 20:53:50 +00002001 uint64_t z_mask = arg_info(op->args[1])->z_mask;
Richard Hendersonfae450b2021-08-25 22:42:19 -07002002 ctx->z_mask = -(z_mask & -z_mask);
2003
Richard Henderson9caca882021-08-24 13:30:32 -07002004 /*
2005 * Because of fold_sub_to_neg, we want to always return true,
2006 * via finish_folding.
2007 */
2008 finish_folding(ctx, op);
2009 return true;
Richard Henderson2f9f08b2021-08-25 12:03:48 -07002010}
2011
Richard Hendersone25fe882024-04-04 20:53:50 +00002012static bool fold_neg(OptContext *ctx, TCGOp *op)
2013{
2014 return fold_const1(ctx, op) || fold_neg_no_const(ctx, op);
2015}
2016
Richard Henderson2f9f08b2021-08-25 12:03:48 -07002017static bool fold_nor(OptContext *ctx, TCGOp *op)
2018{
Richard Henderson7a2f7082021-08-26 07:06:39 -07002019 if (fold_const2_commutative(ctx, op) ||
Richard Henderson0e0a32b2021-08-24 13:18:01 -07002020 fold_xi_to_not(ctx, op, 0)) {
2021 return true;
2022 }
Richard Henderson3f2b1f82021-08-26 13:08:54 -07002023
2024 ctx->s_mask = arg_info(op->args[1])->s_mask
2025 & arg_info(op->args[2])->s_mask;
Richard Henderson0e0a32b2021-08-24 13:18:01 -07002026 return false;
Richard Henderson2f9f08b2021-08-25 12:03:48 -07002027}
2028
2029static bool fold_not(OptContext *ctx, TCGOp *op)
2030{
Richard Henderson0e0a32b2021-08-24 13:18:01 -07002031 if (fold_const1(ctx, op)) {
2032 return true;
2033 }
2034
Richard Henderson3f2b1f82021-08-26 13:08:54 -07002035 ctx->s_mask = arg_info(op->args[1])->s_mask;
2036
Richard Henderson0e0a32b2021-08-24 13:18:01 -07002037 /* Because of fold_to_not, we want to always return true, via finish. */
2038 finish_folding(ctx, op);
2039 return true;
Richard Henderson2f9f08b2021-08-25 12:03:48 -07002040}
2041
2042static bool fold_or(OptContext *ctx, TCGOp *op)
2043{
Richard Henderson7a2f7082021-08-26 07:06:39 -07002044 if (fold_const2_commutative(ctx, op) ||
Richard Hendersona63ce0e2021-08-25 20:28:53 -07002045 fold_xi_to_x(ctx, op, 0) ||
Richard Hendersonca7bb042021-08-25 13:14:21 -07002046 fold_xx_to_x(ctx, op)) {
2047 return true;
2048 }
Richard Hendersonfae450b2021-08-25 22:42:19 -07002049
2050 ctx->z_mask = arg_info(op->args[1])->z_mask
2051 | arg_info(op->args[2])->z_mask;
Richard Henderson3f2b1f82021-08-26 13:08:54 -07002052 ctx->s_mask = arg_info(op->args[1])->s_mask
2053 & arg_info(op->args[2])->s_mask;
Richard Hendersonfae450b2021-08-25 22:42:19 -07002054 return fold_masks(ctx, op);
Richard Henderson2f9f08b2021-08-25 12:03:48 -07002055}
2056
2057static bool fold_orc(OptContext *ctx, TCGOp *op)
2058{
Richard Henderson0e0a32b2021-08-24 13:18:01 -07002059 if (fold_const2(ctx, op) ||
Richard Henderson4e858d92021-08-26 07:31:13 -07002060 fold_xx_to_i(ctx, op, -1) ||
Richard Hendersona63ce0e2021-08-25 20:28:53 -07002061 fold_xi_to_x(ctx, op, -1) ||
Richard Henderson0e0a32b2021-08-24 13:18:01 -07002062 fold_ix_to_not(ctx, op, 0)) {
2063 return true;
2064 }
Richard Henderson3f2b1f82021-08-26 13:08:54 -07002065
2066 ctx->s_mask = arg_info(op->args[1])->s_mask
2067 & arg_info(op->args[2])->s_mask;
Richard Henderson0e0a32b2021-08-24 13:18:01 -07002068 return false;
Richard Henderson2f9f08b2021-08-25 12:03:48 -07002069}
2070
Richard Henderson3eefdf22021-08-25 11:06:43 -07002071static bool fold_qemu_ld(OptContext *ctx, TCGOp *op)
2072{
Richard Hendersonfae450b2021-08-25 22:42:19 -07002073 const TCGOpDef *def = &tcg_op_defs[op->opc];
2074 MemOpIdx oi = op->args[def->nb_oargs + def->nb_iargs];
2075 MemOp mop = get_memop(oi);
2076 int width = 8 * memop_size(mop);
2077
Richard Henderson57fe5c62021-08-26 12:04:46 -07002078 if (width < 64) {
2079 ctx->s_mask = MAKE_64BIT_MASK(width, 64 - width);
2080 if (!(mop & MO_SIGN)) {
2081 ctx->z_mask = MAKE_64BIT_MASK(0, width);
2082 ctx->s_mask <<= 1;
2083 }
Richard Hendersonfae450b2021-08-25 22:42:19 -07002084 }
2085
Richard Henderson3eefdf22021-08-25 11:06:43 -07002086 /* Opcodes that touch guest memory stop the mb optimization. */
2087 ctx->prev_mb = NULL;
2088 return false;
2089}
2090
2091static bool fold_qemu_st(OptContext *ctx, TCGOp *op)
2092{
2093 /* Opcodes that touch guest memory stop the mb optimization. */
2094 ctx->prev_mb = NULL;
2095 return false;
2096}
2097
Richard Henderson2f9f08b2021-08-25 12:03:48 -07002098static bool fold_remainder(OptContext *ctx, TCGOp *op)
2099{
Richard Henderson267c17e2021-10-25 11:30:33 -07002100 if (fold_const2(ctx, op) ||
2101 fold_xx_to_i(ctx, op, 0)) {
2102 return true;
2103 }
2104 return false;
Richard Henderson2f9f08b2021-08-25 12:03:48 -07002105}
2106
Richard Henderson8d65cda2024-03-26 16:00:40 -10002107static bool fold_setcond_zmask(OptContext *ctx, TCGOp *op, bool neg)
2108{
2109 uint64_t a_zmask, b_val;
2110 TCGCond cond;
2111
2112 if (!arg_is_const(op->args[2])) {
2113 return false;
2114 }
2115
2116 a_zmask = arg_info(op->args[1])->z_mask;
2117 b_val = arg_info(op->args[2])->val;
2118 cond = op->args[3];
2119
2120 if (ctx->type == TCG_TYPE_I32) {
2121 a_zmask = (uint32_t)a_zmask;
2122 b_val = (uint32_t)b_val;
2123 }
2124
2125 /*
2126 * A with only low bits set vs B with high bits set means that A < B.
2127 */
2128 if (a_zmask < b_val) {
2129 bool inv = false;
2130
2131 switch (cond) {
2132 case TCG_COND_NE:
2133 case TCG_COND_LEU:
2134 case TCG_COND_LTU:
2135 inv = true;
2136 /* fall through */
2137 case TCG_COND_GTU:
2138 case TCG_COND_GEU:
2139 case TCG_COND_EQ:
2140 return tcg_opt_gen_movi(ctx, op, op->args[0], neg ? -inv : inv);
2141 default:
2142 break;
2143 }
2144 }
2145
2146 /*
2147 * A with only lsb set is already boolean.
2148 */
2149 if (a_zmask <= 1) {
2150 bool convert = false;
2151 bool inv = false;
2152
2153 switch (cond) {
2154 case TCG_COND_EQ:
2155 inv = true;
2156 /* fall through */
2157 case TCG_COND_NE:
2158 convert = (b_val == 0);
2159 break;
2160 case TCG_COND_LTU:
2161 case TCG_COND_TSTEQ:
2162 inv = true;
2163 /* fall through */
2164 case TCG_COND_GEU:
2165 case TCG_COND_TSTNE:
2166 convert = (b_val == 1);
2167 break;
2168 default:
2169 break;
2170 }
2171 if (convert) {
2172 TCGOpcode add_opc, xor_opc, neg_opc;
2173
2174 if (!inv && !neg) {
2175 return tcg_opt_gen_mov(ctx, op, op->args[0], op->args[1]);
2176 }
2177
2178 switch (ctx->type) {
2179 case TCG_TYPE_I32:
2180 add_opc = INDEX_op_add_i32;
2181 neg_opc = INDEX_op_neg_i32;
2182 xor_opc = INDEX_op_xor_i32;
2183 break;
2184 case TCG_TYPE_I64:
2185 add_opc = INDEX_op_add_i64;
2186 neg_opc = INDEX_op_neg_i64;
2187 xor_opc = INDEX_op_xor_i64;
2188 break;
2189 default:
2190 g_assert_not_reached();
2191 }
2192
2193 if (!inv) {
2194 op->opc = neg_opc;
2195 } else if (neg) {
2196 op->opc = add_opc;
2197 op->args[2] = arg_new_constant(ctx, -1);
2198 } else {
2199 op->opc = xor_opc;
2200 op->args[2] = arg_new_constant(ctx, 1);
2201 }
2202 return false;
2203 }
2204 }
2205
2206 return false;
2207}
2208
Richard Hendersonceb9ee02023-10-23 23:44:27 -07002209static void fold_setcond_tst_pow2(OptContext *ctx, TCGOp *op, bool neg)
2210{
Paolo Bonziniff202812024-02-28 12:06:41 +01002211 TCGOpcode and_opc, sub_opc, xor_opc, neg_opc, shr_opc;
2212 TCGOpcode uext_opc = 0, sext_opc = 0;
Richard Hendersonceb9ee02023-10-23 23:44:27 -07002213 TCGCond cond = op->args[3];
2214 TCGArg ret, src1, src2;
2215 TCGOp *op2;
2216 uint64_t val;
2217 int sh;
2218 bool inv;
2219
2220 if (!is_tst_cond(cond) || !arg_is_const(op->args[2])) {
2221 return;
2222 }
2223
2224 src2 = op->args[2];
2225 val = arg_info(src2)->val;
2226 if (!is_power_of_2(val)) {
2227 return;
2228 }
2229 sh = ctz64(val);
2230
2231 switch (ctx->type) {
2232 case TCG_TYPE_I32:
2233 and_opc = INDEX_op_and_i32;
2234 sub_opc = INDEX_op_sub_i32;
2235 xor_opc = INDEX_op_xor_i32;
2236 shr_opc = INDEX_op_shr_i32;
2237 neg_opc = INDEX_op_neg_i32;
2238 if (TCG_TARGET_extract_i32_valid(sh, 1)) {
2239 uext_opc = TCG_TARGET_HAS_extract_i32 ? INDEX_op_extract_i32 : 0;
2240 sext_opc = TCG_TARGET_HAS_sextract_i32 ? INDEX_op_sextract_i32 : 0;
2241 }
2242 break;
2243 case TCG_TYPE_I64:
2244 and_opc = INDEX_op_and_i64;
2245 sub_opc = INDEX_op_sub_i64;
2246 xor_opc = INDEX_op_xor_i64;
2247 shr_opc = INDEX_op_shr_i64;
2248 neg_opc = INDEX_op_neg_i64;
2249 if (TCG_TARGET_extract_i64_valid(sh, 1)) {
2250 uext_opc = TCG_TARGET_HAS_extract_i64 ? INDEX_op_extract_i64 : 0;
2251 sext_opc = TCG_TARGET_HAS_sextract_i64 ? INDEX_op_sextract_i64 : 0;
2252 }
2253 break;
2254 default:
2255 g_assert_not_reached();
2256 }
2257
2258 ret = op->args[0];
2259 src1 = op->args[1];
2260 inv = cond == TCG_COND_TSTEQ;
2261
2262 if (sh && sext_opc && neg && !inv) {
2263 op->opc = sext_opc;
2264 op->args[1] = src1;
2265 op->args[2] = sh;
2266 op->args[3] = 1;
2267 return;
2268 } else if (sh && uext_opc) {
2269 op->opc = uext_opc;
2270 op->args[1] = src1;
2271 op->args[2] = sh;
2272 op->args[3] = 1;
2273 } else {
2274 if (sh) {
2275 op2 = tcg_op_insert_before(ctx->tcg, op, shr_opc, 3);
2276 op2->args[0] = ret;
2277 op2->args[1] = src1;
2278 op2->args[2] = arg_new_constant(ctx, sh);
2279 src1 = ret;
2280 }
2281 op->opc = and_opc;
2282 op->args[1] = src1;
2283 op->args[2] = arg_new_constant(ctx, 1);
2284 }
2285
2286 if (neg && inv) {
2287 op2 = tcg_op_insert_after(ctx->tcg, op, sub_opc, 3);
2288 op2->args[0] = ret;
2289 op2->args[1] = ret;
2290 op2->args[2] = arg_new_constant(ctx, 1);
2291 } else if (inv) {
2292 op2 = tcg_op_insert_after(ctx->tcg, op, xor_opc, 3);
2293 op2->args[0] = ret;
2294 op2->args[1] = ret;
2295 op2->args[2] = arg_new_constant(ctx, 1);
2296 } else if (neg) {
2297 op2 = tcg_op_insert_after(ctx->tcg, op, neg_opc, 2);
2298 op2->args[0] = ret;
2299 op2->args[1] = ret;
2300 }
2301}
2302
Richard Hendersonc63ff552021-08-24 09:35:30 -07002303static bool fold_setcond(OptContext *ctx, TCGOp *op)
2304{
Richard Hendersonfb04ab72024-01-10 18:21:58 +11002305 int i = do_constant_folding_cond1(ctx, op, op->args[0], &op->args[1],
Richard Henderson246c4b72023-10-24 16:36:50 -07002306 &op->args[2], &op->args[3]);
Richard Hendersonc63ff552021-08-24 09:35:30 -07002307 if (i >= 0) {
2308 return tcg_opt_gen_movi(ctx, op, op->args[0], i);
2309 }
Richard Henderson8d65cda2024-03-26 16:00:40 -10002310
2311 if (fold_setcond_zmask(ctx, op, false)) {
2312 return true;
2313 }
Richard Hendersonceb9ee02023-10-23 23:44:27 -07002314 fold_setcond_tst_pow2(ctx, op, false);
Richard Hendersonfae450b2021-08-25 22:42:19 -07002315
2316 ctx->z_mask = 1;
Richard Henderson275d7d82021-08-26 13:20:39 -07002317 ctx->s_mask = smask_from_zmask(1);
Richard Hendersonc63ff552021-08-24 09:35:30 -07002318 return false;
2319}
2320
Richard Henderson36355022023-08-04 23:24:04 +00002321static bool fold_negsetcond(OptContext *ctx, TCGOp *op)
2322{
Richard Hendersonfb04ab72024-01-10 18:21:58 +11002323 int i = do_constant_folding_cond1(ctx, op, op->args[0], &op->args[1],
Richard Henderson246c4b72023-10-24 16:36:50 -07002324 &op->args[2], &op->args[3]);
Richard Henderson36355022023-08-04 23:24:04 +00002325 if (i >= 0) {
2326 return tcg_opt_gen_movi(ctx, op, op->args[0], -i);
2327 }
Richard Henderson8d65cda2024-03-26 16:00:40 -10002328
2329 if (fold_setcond_zmask(ctx, op, true)) {
2330 return true;
2331 }
Richard Hendersonceb9ee02023-10-23 23:44:27 -07002332 fold_setcond_tst_pow2(ctx, op, true);
Richard Henderson36355022023-08-04 23:24:04 +00002333
2334 /* Value is {0,-1} so all bits are repetitions of the sign. */
2335 ctx->s_mask = -1;
2336 return false;
2337}
2338
Richard Hendersonbc47b1a2021-08-24 09:09:35 -07002339static bool fold_setcond2(OptContext *ctx, TCGOp *op)
2340{
Richard Henderson7e64b112023-10-24 16:53:56 -07002341 TCGCond cond;
Richard Henderson7a2f7082021-08-26 07:06:39 -07002342 int i, inv = 0;
Richard Hendersonbc47b1a2021-08-24 09:09:35 -07002343
Richard Hendersonfb04ab72024-01-10 18:21:58 +11002344 i = do_constant_folding_cond2(ctx, op, &op->args[1]);
Richard Henderson7e64b112023-10-24 16:53:56 -07002345 cond = op->args[5];
Richard Hendersonbc47b1a2021-08-24 09:09:35 -07002346 if (i >= 0) {
2347 goto do_setcond_const;
2348 }
2349
2350 switch (cond) {
2351 case TCG_COND_LT:
2352 case TCG_COND_GE:
2353 /*
2354 * Simplify LT/GE comparisons vs zero to a single compare
2355 * vs the high word of the input.
2356 */
Richard Henderson27cdb852023-10-23 11:38:00 -07002357 if (arg_is_const_val(op->args[3], 0) &&
2358 arg_is_const_val(op->args[4], 0)) {
Richard Hendersonbc47b1a2021-08-24 09:09:35 -07002359 goto do_setcond_high;
2360 }
2361 break;
2362
2363 case TCG_COND_NE:
2364 inv = 1;
2365 QEMU_FALLTHROUGH;
2366 case TCG_COND_EQ:
2367 /*
2368 * Simplify EQ/NE comparisons where one of the pairs
2369 * can be simplified.
2370 */
Richard Henderson67f84c92021-08-25 08:00:20 -07002371 i = do_constant_folding_cond(TCG_TYPE_I32, op->args[1],
Richard Hendersonbc47b1a2021-08-24 09:09:35 -07002372 op->args[3], cond);
2373 switch (i ^ inv) {
2374 case 0:
2375 goto do_setcond_const;
2376 case 1:
2377 goto do_setcond_high;
2378 }
2379
Richard Henderson67f84c92021-08-25 08:00:20 -07002380 i = do_constant_folding_cond(TCG_TYPE_I32, op->args[2],
Richard Hendersonbc47b1a2021-08-24 09:09:35 -07002381 op->args[4], cond);
2382 switch (i ^ inv) {
2383 case 0:
2384 goto do_setcond_const;
2385 case 1:
Richard Hendersonceb9ee02023-10-23 23:44:27 -07002386 goto do_setcond_low;
2387 }
2388 break;
2389
2390 case TCG_COND_TSTEQ:
2391 case TCG_COND_TSTNE:
Richard Hendersona71d9df2024-06-30 19:46:23 -07002392 if (arg_is_const_val(op->args[3], 0)) {
Richard Hendersonceb9ee02023-10-23 23:44:27 -07002393 goto do_setcond_high;
2394 }
2395 if (arg_is_const_val(op->args[4], 0)) {
2396 goto do_setcond_low;
Richard Hendersonbc47b1a2021-08-24 09:09:35 -07002397 }
2398 break;
2399
2400 default:
2401 break;
2402
Richard Hendersonceb9ee02023-10-23 23:44:27 -07002403 do_setcond_low:
2404 op->args[2] = op->args[3];
2405 op->args[3] = cond;
2406 op->opc = INDEX_op_setcond_i32;
2407 return fold_setcond(ctx, op);
2408
Richard Hendersonbc47b1a2021-08-24 09:09:35 -07002409 do_setcond_high:
2410 op->args[1] = op->args[2];
2411 op->args[2] = op->args[4];
2412 op->args[3] = cond;
2413 op->opc = INDEX_op_setcond_i32;
Richard Hendersonceb9ee02023-10-23 23:44:27 -07002414 return fold_setcond(ctx, op);
Richard Hendersonbc47b1a2021-08-24 09:09:35 -07002415 }
Richard Hendersonfae450b2021-08-25 22:42:19 -07002416
2417 ctx->z_mask = 1;
Richard Henderson275d7d82021-08-26 13:20:39 -07002418 ctx->s_mask = smask_from_zmask(1);
Richard Hendersonbc47b1a2021-08-24 09:09:35 -07002419 return false;
2420
2421 do_setcond_const:
2422 return tcg_opt_gen_movi(ctx, op, op->args[0], i);
2423}
2424
Richard Henderson1f106542024-09-06 12:22:41 -07002425static bool fold_cmp_vec(OptContext *ctx, TCGOp *op)
2426{
2427 /* Canonicalize the comparison to put immediate second. */
2428 if (swap_commutative(NO_DEST, &op->args[1], &op->args[2])) {
2429 op->args[3] = tcg_swap_cond(op->args[3]);
2430 }
2431 return false;
2432}
2433
2434static bool fold_cmpsel_vec(OptContext *ctx, TCGOp *op)
2435{
2436 /* If true and false values are the same, eliminate the cmp. */
2437 if (args_are_copies(op->args[3], op->args[4])) {
2438 return tcg_opt_gen_mov(ctx, op, op->args[0], op->args[3]);
2439 }
2440
2441 /* Canonicalize the comparison to put immediate second. */
2442 if (swap_commutative(NO_DEST, &op->args[1], &op->args[2])) {
2443 op->args[5] = tcg_swap_cond(op->args[5]);
2444 }
2445 /*
2446 * Canonicalize the "false" input reg to match the destination,
2447 * so that the tcg backend can implement "move if true".
2448 */
2449 if (swap_commutative(op->args[0], &op->args[4], &op->args[3])) {
2450 op->args[5] = tcg_invert_cond(op->args[5]);
2451 }
2452 return false;
2453}
2454
Richard Hendersonb6617c82021-08-24 10:44:53 -07002455static bool fold_sextract(OptContext *ctx, TCGOp *op)
2456{
Richard Henderson57fe5c62021-08-26 12:04:46 -07002457 uint64_t z_mask, s_mask, s_mask_old;
2458 int pos = op->args[2];
2459 int len = op->args[3];
Richard Hendersonfae450b2021-08-25 22:42:19 -07002460
Richard Hendersonb6617c82021-08-24 10:44:53 -07002461 if (arg_is_const(op->args[1])) {
2462 uint64_t t;
2463
2464 t = arg_info(op->args[1])->val;
Richard Henderson57fe5c62021-08-26 12:04:46 -07002465 t = sextract64(t, pos, len);
Richard Hendersonb6617c82021-08-24 10:44:53 -07002466 return tcg_opt_gen_movi(ctx, op, op->args[0], t);
2467 }
Richard Hendersonfae450b2021-08-25 22:42:19 -07002468
Richard Henderson57fe5c62021-08-26 12:04:46 -07002469 z_mask = arg_info(op->args[1])->z_mask;
2470 z_mask = sextract64(z_mask, pos, len);
Richard Hendersonfae450b2021-08-25 22:42:19 -07002471 ctx->z_mask = z_mask;
2472
Richard Henderson57fe5c62021-08-26 12:04:46 -07002473 s_mask_old = arg_info(op->args[1])->s_mask;
2474 s_mask = sextract64(s_mask_old, pos, len);
2475 s_mask |= MAKE_64BIT_MASK(len, 64 - len);
2476 ctx->s_mask = s_mask;
2477
2478 if (pos == 0) {
2479 ctx->a_mask = s_mask & ~s_mask_old;
2480 }
2481
Richard Hendersonfae450b2021-08-25 22:42:19 -07002482 return fold_masks(ctx, op);
Richard Hendersonb6617c82021-08-24 10:44:53 -07002483}
2484
Richard Henderson2f9f08b2021-08-25 12:03:48 -07002485static bool fold_shift(OptContext *ctx, TCGOp *op)
2486{
Richard Henderson93a967f2021-08-26 13:24:59 -07002487 uint64_t s_mask, z_mask, sign;
2488
Richard Hendersona63ce0e2021-08-25 20:28:53 -07002489 if (fold_const2(ctx, op) ||
Richard Hendersonda48e272021-08-25 20:42:04 -07002490 fold_ix_to_i(ctx, op, 0) ||
Richard Hendersona63ce0e2021-08-25 20:28:53 -07002491 fold_xi_to_x(ctx, op, 0)) {
2492 return true;
2493 }
Richard Hendersonfae450b2021-08-25 22:42:19 -07002494
Richard Henderson93a967f2021-08-26 13:24:59 -07002495 s_mask = arg_info(op->args[1])->s_mask;
2496 z_mask = arg_info(op->args[1])->z_mask;
2497
Richard Hendersonfae450b2021-08-25 22:42:19 -07002498 if (arg_is_const(op->args[2])) {
Richard Henderson93a967f2021-08-26 13:24:59 -07002499 int sh = arg_info(op->args[2])->val;
2500
2501 ctx->z_mask = do_constant_folding(op->opc, ctx->type, z_mask, sh);
2502
2503 s_mask = do_constant_folding(op->opc, ctx->type, s_mask, sh);
2504 ctx->s_mask = smask_from_smask(s_mask);
2505
Richard Hendersonfae450b2021-08-25 22:42:19 -07002506 return fold_masks(ctx, op);
2507 }
Richard Henderson93a967f2021-08-26 13:24:59 -07002508
2509 switch (op->opc) {
2510 CASE_OP_32_64(sar):
2511 /*
2512 * Arithmetic right shift will not reduce the number of
2513 * input sign repetitions.
2514 */
2515 ctx->s_mask = s_mask;
2516 break;
2517 CASE_OP_32_64(shr):
2518 /*
2519 * If the sign bit is known zero, then logical right shift
2520 * will not reduced the number of input sign repetitions.
2521 */
2522 sign = (s_mask & -s_mask) >> 1;
Richard Henderson2911e9b2024-03-26 11:21:38 -10002523 if (sign && !(z_mask & sign)) {
Richard Henderson93a967f2021-08-26 13:24:59 -07002524 ctx->s_mask = s_mask;
2525 }
2526 break;
2527 default:
2528 break;
2529 }
2530
Richard Hendersona63ce0e2021-08-25 20:28:53 -07002531 return false;
Richard Henderson2f9f08b2021-08-25 12:03:48 -07002532}
2533
Richard Henderson9caca882021-08-24 13:30:32 -07002534static bool fold_sub_to_neg(OptContext *ctx, TCGOp *op)
2535{
2536 TCGOpcode neg_op;
2537 bool have_neg;
2538
2539 if (!arg_is_const(op->args[1]) || arg_info(op->args[1])->val != 0) {
2540 return false;
2541 }
2542
2543 switch (ctx->type) {
2544 case TCG_TYPE_I32:
2545 neg_op = INDEX_op_neg_i32;
Richard Hendersonb701f192023-10-25 21:14:04 -07002546 have_neg = true;
Richard Henderson9caca882021-08-24 13:30:32 -07002547 break;
2548 case TCG_TYPE_I64:
2549 neg_op = INDEX_op_neg_i64;
Richard Hendersonb701f192023-10-25 21:14:04 -07002550 have_neg = true;
Richard Henderson9caca882021-08-24 13:30:32 -07002551 break;
2552 case TCG_TYPE_V64:
2553 case TCG_TYPE_V128:
2554 case TCG_TYPE_V256:
2555 neg_op = INDEX_op_neg_vec;
2556 have_neg = (TCG_TARGET_HAS_neg_vec &&
2557 tcg_can_emit_vec_op(neg_op, ctx->type, TCGOP_VECE(op)) > 0);
2558 break;
2559 default:
2560 g_assert_not_reached();
2561 }
2562 if (have_neg) {
2563 op->opc = neg_op;
2564 op->args[1] = op->args[2];
Richard Hendersone25fe882024-04-04 20:53:50 +00002565 return fold_neg_no_const(ctx, op);
Richard Henderson9caca882021-08-24 13:30:32 -07002566 }
2567 return false;
2568}
2569
Richard Hendersonc578ff12021-12-16 06:07:25 -08002570/* We cannot as yet do_constant_folding with vectors. */
2571static bool fold_sub_vec(OptContext *ctx, TCGOp *op)
Richard Henderson2f9f08b2021-08-25 12:03:48 -07002572{
Richard Hendersonc578ff12021-12-16 06:07:25 -08002573 if (fold_xx_to_i(ctx, op, 0) ||
Richard Hendersona63ce0e2021-08-25 20:28:53 -07002574 fold_xi_to_x(ctx, op, 0) ||
Richard Henderson9caca882021-08-24 13:30:32 -07002575 fold_sub_to_neg(ctx, op)) {
Richard Hendersoncbe42fb2021-08-25 13:02:00 -07002576 return true;
2577 }
2578 return false;
Richard Henderson2f9f08b2021-08-25 12:03:48 -07002579}
2580
Richard Hendersonc578ff12021-12-16 06:07:25 -08002581static bool fold_sub(OptContext *ctx, TCGOp *op)
2582{
Richard Henderson6334a962023-10-25 18:39:43 -07002583 if (fold_const2(ctx, op) || fold_sub_vec(ctx, op)) {
2584 return true;
2585 }
2586
2587 /* Fold sub r,x,i to add r,x,-i */
2588 if (arg_is_const(op->args[2])) {
2589 uint64_t val = arg_info(op->args[2])->val;
2590
2591 op->opc = (ctx->type == TCG_TYPE_I32
2592 ? INDEX_op_add_i32 : INDEX_op_add_i64);
2593 op->args[2] = arg_new_constant(ctx, -val);
2594 }
2595 return false;
Richard Hendersonc578ff12021-12-16 06:07:25 -08002596}
2597
Richard Henderson9531c072021-08-26 06:51:39 -07002598static bool fold_sub2(OptContext *ctx, TCGOp *op)
Richard Hendersone3f7dc22021-08-24 10:30:38 -07002599{
Richard Henderson9531c072021-08-26 06:51:39 -07002600 return fold_addsub2(ctx, op, false);
Richard Hendersone3f7dc22021-08-24 10:30:38 -07002601}
2602
Richard Hendersonfae450b2021-08-25 22:42:19 -07002603static bool fold_tcg_ld(OptContext *ctx, TCGOp *op)
2604{
2605 /* We can't do any folding with a load, but we can record bits. */
2606 switch (op->opc) {
Richard Henderson57fe5c62021-08-26 12:04:46 -07002607 CASE_OP_32_64(ld8s):
2608 ctx->s_mask = MAKE_64BIT_MASK(8, 56);
2609 break;
Richard Hendersonfae450b2021-08-25 22:42:19 -07002610 CASE_OP_32_64(ld8u):
2611 ctx->z_mask = MAKE_64BIT_MASK(0, 8);
Richard Henderson57fe5c62021-08-26 12:04:46 -07002612 ctx->s_mask = MAKE_64BIT_MASK(9, 55);
2613 break;
2614 CASE_OP_32_64(ld16s):
2615 ctx->s_mask = MAKE_64BIT_MASK(16, 48);
Richard Hendersonfae450b2021-08-25 22:42:19 -07002616 break;
2617 CASE_OP_32_64(ld16u):
2618 ctx->z_mask = MAKE_64BIT_MASK(0, 16);
Richard Henderson57fe5c62021-08-26 12:04:46 -07002619 ctx->s_mask = MAKE_64BIT_MASK(17, 47);
2620 break;
2621 case INDEX_op_ld32s_i64:
2622 ctx->s_mask = MAKE_64BIT_MASK(32, 32);
Richard Hendersonfae450b2021-08-25 22:42:19 -07002623 break;
2624 case INDEX_op_ld32u_i64:
2625 ctx->z_mask = MAKE_64BIT_MASK(0, 32);
Richard Henderson57fe5c62021-08-26 12:04:46 -07002626 ctx->s_mask = MAKE_64BIT_MASK(33, 31);
Richard Hendersonfae450b2021-08-25 22:42:19 -07002627 break;
2628 default:
2629 g_assert_not_reached();
2630 }
2631 return false;
2632}
2633
Richard Hendersonab84dc32023-08-23 23:04:24 -07002634static bool fold_tcg_ld_memcopy(OptContext *ctx, TCGOp *op)
2635{
2636 TCGTemp *dst, *src;
2637 intptr_t ofs;
2638 TCGType type;
2639
2640 if (op->args[1] != tcgv_ptr_arg(tcg_env)) {
2641 return false;
2642 }
2643
2644 type = ctx->type;
2645 ofs = op->args[2];
2646 dst = arg_temp(op->args[0]);
2647 src = find_mem_copy_for(ctx, type, ofs);
2648 if (src && src->base_type == type) {
2649 return tcg_opt_gen_mov(ctx, op, temp_arg(dst), temp_arg(src));
2650 }
2651
2652 reset_ts(ctx, dst);
2653 record_mem_copy(ctx, type, dst, ofs, ofs + tcg_type_size(type) - 1);
2654 return true;
2655}
2656
2657static bool fold_tcg_st(OptContext *ctx, TCGOp *op)
2658{
2659 intptr_t ofs = op->args[2];
2660 intptr_t lm1;
2661
2662 if (op->args[1] != tcgv_ptr_arg(tcg_env)) {
2663 remove_mem_copy_all(ctx);
2664 return false;
2665 }
2666
2667 switch (op->opc) {
2668 CASE_OP_32_64(st8):
2669 lm1 = 0;
2670 break;
2671 CASE_OP_32_64(st16):
2672 lm1 = 1;
2673 break;
2674 case INDEX_op_st32_i64:
2675 case INDEX_op_st_i32:
2676 lm1 = 3;
2677 break;
2678 case INDEX_op_st_i64:
2679 lm1 = 7;
2680 break;
2681 case INDEX_op_st_vec:
2682 lm1 = tcg_type_size(ctx->type) - 1;
2683 break;
2684 default:
2685 g_assert_not_reached();
2686 }
2687 remove_mem_copy_in(ctx, ofs, ofs + lm1);
2688 return false;
2689}
2690
2691static bool fold_tcg_st_memcopy(OptContext *ctx, TCGOp *op)
2692{
2693 TCGTemp *src;
2694 intptr_t ofs, last;
2695 TCGType type;
2696
2697 if (op->args[1] != tcgv_ptr_arg(tcg_env)) {
2698 fold_tcg_st(ctx, op);
2699 return false;
2700 }
2701
2702 src = arg_temp(op->args[0]);
2703 ofs = op->args[2];
2704 type = ctx->type;
Richard Henderson3eaadae2023-08-23 23:13:06 -07002705
2706 /*
2707 * Eliminate duplicate stores of a constant.
2708 * This happens frequently when the target ISA zero-extends.
2709 */
2710 if (ts_is_const(src)) {
2711 TCGTemp *prev = find_mem_copy_for(ctx, type, ofs);
2712 if (src == prev) {
2713 tcg_op_remove(ctx->tcg, op);
2714 return true;
2715 }
2716 }
2717
Richard Hendersonab84dc32023-08-23 23:04:24 -07002718 last = ofs + tcg_type_size(type) - 1;
2719 remove_mem_copy_in(ctx, ofs, last);
2720 record_mem_copy(ctx, type, src, ofs, last);
2721 return false;
2722}
2723
Richard Henderson2f9f08b2021-08-25 12:03:48 -07002724static bool fold_xor(OptContext *ctx, TCGOp *op)
2725{
Richard Henderson7a2f7082021-08-26 07:06:39 -07002726 if (fold_const2_commutative(ctx, op) ||
Richard Henderson0e0a32b2021-08-24 13:18:01 -07002727 fold_xx_to_i(ctx, op, 0) ||
Richard Hendersona63ce0e2021-08-25 20:28:53 -07002728 fold_xi_to_x(ctx, op, 0) ||
Richard Henderson0e0a32b2021-08-24 13:18:01 -07002729 fold_xi_to_not(ctx, op, -1)) {
Richard Hendersoncbe42fb2021-08-25 13:02:00 -07002730 return true;
2731 }
Richard Hendersonfae450b2021-08-25 22:42:19 -07002732
2733 ctx->z_mask = arg_info(op->args[1])->z_mask
2734 | arg_info(op->args[2])->z_mask;
Richard Henderson3f2b1f82021-08-26 13:08:54 -07002735 ctx->s_mask = arg_info(op->args[1])->s_mask
2736 & arg_info(op->args[2])->s_mask;
Richard Hendersonfae450b2021-08-25 22:42:19 -07002737 return fold_masks(ctx, op);
Richard Henderson2f9f08b2021-08-25 12:03:48 -07002738}
2739
Richard Hendersone58b9772024-09-06 22:30:01 -07002740static bool fold_bitsel_vec(OptContext *ctx, TCGOp *op)
2741{
2742 /* If true and false values are the same, eliminate the cmp. */
2743 if (args_are_copies(op->args[2], op->args[3])) {
2744 return tcg_opt_gen_mov(ctx, op, op->args[0], op->args[2]);
2745 }
2746
2747 if (arg_is_const(op->args[2]) && arg_is_const(op->args[3])) {
2748 uint64_t tv = arg_info(op->args[2])->val;
2749 uint64_t fv = arg_info(op->args[3])->val;
2750
2751 if (tv == -1 && fv == 0) {
2752 return tcg_opt_gen_mov(ctx, op, op->args[0], op->args[1]);
2753 }
2754 if (tv == 0 && fv == -1) {
2755 if (TCG_TARGET_HAS_not_vec) {
2756 op->opc = INDEX_op_not_vec;
2757 return fold_not(ctx, op);
2758 } else {
2759 op->opc = INDEX_op_xor_vec;
2760 op->args[2] = arg_new_constant(ctx, -1);
2761 return fold_xor(ctx, op);
2762 }
2763 }
2764 }
2765 if (arg_is_const(op->args[2])) {
2766 uint64_t tv = arg_info(op->args[2])->val;
2767 if (tv == -1) {
2768 op->opc = INDEX_op_or_vec;
2769 op->args[2] = op->args[3];
2770 return fold_or(ctx, op);
2771 }
2772 if (tv == 0 && TCG_TARGET_HAS_andc_vec) {
2773 op->opc = INDEX_op_andc_vec;
2774 op->args[2] = op->args[1];
2775 op->args[1] = op->args[3];
2776 return fold_andc(ctx, op);
2777 }
2778 }
2779 if (arg_is_const(op->args[3])) {
2780 uint64_t fv = arg_info(op->args[3])->val;
2781 if (fv == 0) {
2782 op->opc = INDEX_op_and_vec;
2783 return fold_and(ctx, op);
2784 }
2785 if (fv == -1 && TCG_TARGET_HAS_orc_vec) {
2786 op->opc = INDEX_op_orc_vec;
2787 op->args[2] = op->args[1];
2788 op->args[1] = op->args[3];
2789 return fold_orc(ctx, op);
2790 }
2791 }
2792 return false;
2793}
2794
Kirill Batuzov22613af2011-07-07 16:37:13 +04002795/* Propagate constants and copies, fold constant expressions. */
Aurelien Jarno36e60ef2015-06-04 21:53:27 +02002796void tcg_optimize(TCGContext *s)
Kirill Batuzov8f2e8c02011-07-07 16:37:12 +04002797{
Richard Henderson5cf32be2021-08-24 08:17:08 -07002798 int nb_temps, i;
Richard Hendersond0ed5152021-08-24 07:38:39 -07002799 TCGOp *op, *op_next;
Richard Hendersondc849882021-08-24 07:13:45 -07002800 OptContext ctx = { .tcg = s };
Richard Henderson5d8f5362012-09-21 10:13:38 -07002801
Richard Hendersonab84dc32023-08-23 23:04:24 -07002802 QSIMPLEQ_INIT(&ctx.mem_free);
2803
Kirill Batuzov22613af2011-07-07 16:37:13 +04002804 /* Array VALS has an element for each temp.
2805 If this temp holds a constant then its value is kept in VALS' element.
Aurelien Jarnoe590d4e2012-09-11 12:31:21 +02002806 If this temp is a copy of other ones then the other copies are
2807 available through the doubly linked circular list. */
Kirill Batuzov8f2e8c02011-07-07 16:37:12 +04002808
2809 nb_temps = s->nb_temps;
Richard Henderson8f17a972020-03-30 19:52:02 -07002810 for (i = 0; i < nb_temps; ++i) {
2811 s->temps[i].state_ptr = NULL;
2812 }
Kirill Batuzov8f2e8c02011-07-07 16:37:12 +04002813
Richard Henderson15fa08f2017-11-02 15:19:14 +01002814 QTAILQ_FOREACH_SAFE(op, &s->ops, link, op_next) {
Richard Hendersonc45cb8b2014-09-19 13:49:15 -07002815 TCGOpcode opc = op->opc;
Richard Henderson5cf32be2021-08-24 08:17:08 -07002816 const TCGOpDef *def;
Richard Henderson404a1482021-08-24 11:08:21 -07002817 bool done = false;
Richard Hendersonc45cb8b2014-09-19 13:49:15 -07002818
Richard Henderson5cf32be2021-08-24 08:17:08 -07002819 /* Calls are special. */
Richard Hendersonc45cb8b2014-09-19 13:49:15 -07002820 if (opc == INDEX_op_call) {
Richard Henderson5cf32be2021-08-24 08:17:08 -07002821 fold_call(&ctx, op);
2822 continue;
Richard Hendersoncf066672014-03-22 20:06:52 -07002823 }
Richard Henderson5cf32be2021-08-24 08:17:08 -07002824
2825 def = &tcg_op_defs[opc];
Richard Hendersonec5d4cb2021-08-24 08:20:27 -07002826 init_arguments(&ctx, op, def->nb_oargs + def->nb_iargs);
2827 copy_propagate(&ctx, op, def->nb_oargs, def->nb_iargs);
Kirill Batuzov22613af2011-07-07 16:37:13 +04002828
Richard Henderson67f84c92021-08-25 08:00:20 -07002829 /* Pre-compute the type of the operation. */
2830 if (def->flags & TCG_OPF_VECTOR) {
2831 ctx.type = TCG_TYPE_V64 + TCGOP_VECL(op);
2832 } else if (def->flags & TCG_OPF_64BIT) {
2833 ctx.type = TCG_TYPE_I64;
2834 } else {
2835 ctx.type = TCG_TYPE_I32;
2836 }
2837
Richard Henderson57fe5c62021-08-26 12:04:46 -07002838 /* Assume all bits affected, no bits known zero, no sign reps. */
Richard Hendersonfae450b2021-08-25 22:42:19 -07002839 ctx.a_mask = -1;
2840 ctx.z_mask = -1;
Richard Henderson57fe5c62021-08-26 12:04:46 -07002841 ctx.s_mask = 0;
Paolo Bonzini633f6502013-01-11 15:42:53 -08002842
Richard Henderson2cfac7f2021-08-25 13:05:43 -07002843 /*
2844 * Process each opcode.
2845 * Sorted alphabetically by opcode as much as possible.
2846 */
Richard Hendersonc45cb8b2014-09-19 13:49:15 -07002847 switch (opc) {
Richard Hendersonc578ff12021-12-16 06:07:25 -08002848 CASE_OP_32_64(add):
Richard Henderson2f9f08b2021-08-25 12:03:48 -07002849 done = fold_add(&ctx, op);
2850 break;
Richard Hendersonc578ff12021-12-16 06:07:25 -08002851 case INDEX_op_add_vec:
2852 done = fold_add_vec(&ctx, op);
2853 break;
Richard Henderson9531c072021-08-26 06:51:39 -07002854 CASE_OP_32_64(add2):
2855 done = fold_add2(&ctx, op);
Richard Hendersone3f7dc22021-08-24 10:30:38 -07002856 break;
Richard Henderson2f9f08b2021-08-25 12:03:48 -07002857 CASE_OP_32_64_VEC(and):
2858 done = fold_and(&ctx, op);
2859 break;
2860 CASE_OP_32_64_VEC(andc):
2861 done = fold_andc(&ctx, op);
2862 break;
Richard Henderson079b0802021-08-24 09:30:59 -07002863 CASE_OP_32_64(brcond):
2864 done = fold_brcond(&ctx, op);
2865 break;
Richard Henderson764d2ab2021-08-24 09:22:11 -07002866 case INDEX_op_brcond2_i32:
2867 done = fold_brcond2(&ctx, op);
2868 break;
Richard Henderson09bacdc2021-08-24 11:58:12 -07002869 CASE_OP_32_64(bswap16):
2870 CASE_OP_32_64(bswap32):
2871 case INDEX_op_bswap64_i64:
2872 done = fold_bswap(&ctx, op);
2873 break;
Richard Henderson30dd0bf2021-08-24 10:51:34 -07002874 CASE_OP_32_64(clz):
2875 CASE_OP_32_64(ctz):
2876 done = fold_count_zeros(&ctx, op);
2877 break;
Richard Henderson2f9f08b2021-08-25 12:03:48 -07002878 CASE_OP_32_64(ctpop):
2879 done = fold_ctpop(&ctx, op);
2880 break;
Richard Henderson1b1907b2021-08-24 10:47:04 -07002881 CASE_OP_32_64(deposit):
2882 done = fold_deposit(&ctx, op);
2883 break;
Richard Henderson2f9f08b2021-08-25 12:03:48 -07002884 CASE_OP_32_64(div):
2885 CASE_OP_32_64(divu):
2886 done = fold_divide(&ctx, op);
2887 break;
Richard Henderson8cdb3fc2021-08-24 12:06:33 -07002888 case INDEX_op_dup_vec:
2889 done = fold_dup(&ctx, op);
2890 break;
2891 case INDEX_op_dup2_vec:
2892 done = fold_dup2(&ctx, op);
2893 break;
Richard Hendersoned523472021-12-16 11:17:46 -08002894 CASE_OP_32_64_VEC(eqv):
Richard Henderson2f9f08b2021-08-25 12:03:48 -07002895 done = fold_eqv(&ctx, op);
2896 break;
Richard Hendersonb6617c82021-08-24 10:44:53 -07002897 CASE_OP_32_64(extract):
2898 done = fold_extract(&ctx, op);
2899 break;
Richard Hendersondcd08992021-08-24 10:41:39 -07002900 CASE_OP_32_64(extract2):
2901 done = fold_extract2(&ctx, op);
2902 break;
Richard Henderson2f9f08b2021-08-25 12:03:48 -07002903 CASE_OP_32_64(ext8s):
2904 CASE_OP_32_64(ext16s):
2905 case INDEX_op_ext32s_i64:
2906 case INDEX_op_ext_i32_i64:
2907 done = fold_exts(&ctx, op);
2908 break;
2909 CASE_OP_32_64(ext8u):
2910 CASE_OP_32_64(ext16u):
2911 case INDEX_op_ext32u_i64:
2912 case INDEX_op_extu_i32_i64:
2913 case INDEX_op_extrl_i64_i32:
2914 case INDEX_op_extrh_i64_i32:
2915 done = fold_extu(&ctx, op);
2916 break;
Richard Henderson57fe5c62021-08-26 12:04:46 -07002917 CASE_OP_32_64(ld8s):
Richard Hendersonfae450b2021-08-25 22:42:19 -07002918 CASE_OP_32_64(ld8u):
Richard Henderson57fe5c62021-08-26 12:04:46 -07002919 CASE_OP_32_64(ld16s):
Richard Hendersonfae450b2021-08-25 22:42:19 -07002920 CASE_OP_32_64(ld16u):
Richard Henderson57fe5c62021-08-26 12:04:46 -07002921 case INDEX_op_ld32s_i64:
Richard Hendersonfae450b2021-08-25 22:42:19 -07002922 case INDEX_op_ld32u_i64:
2923 done = fold_tcg_ld(&ctx, op);
2924 break;
Richard Hendersonab84dc32023-08-23 23:04:24 -07002925 case INDEX_op_ld_i32:
2926 case INDEX_op_ld_i64:
2927 case INDEX_op_ld_vec:
2928 done = fold_tcg_ld_memcopy(&ctx, op);
2929 break;
2930 CASE_OP_32_64(st8):
2931 CASE_OP_32_64(st16):
2932 case INDEX_op_st32_i64:
2933 done = fold_tcg_st(&ctx, op);
2934 break;
2935 case INDEX_op_st_i32:
2936 case INDEX_op_st_i64:
2937 case INDEX_op_st_vec:
2938 done = fold_tcg_st_memcopy(&ctx, op);
2939 break;
Richard Henderson3eefdf22021-08-25 11:06:43 -07002940 case INDEX_op_mb:
2941 done = fold_mb(&ctx, op);
2942 break;
Richard Henderson2cfac7f2021-08-25 13:05:43 -07002943 CASE_OP_32_64_VEC(mov):
2944 done = fold_mov(&ctx, op);
2945 break;
Richard Henderson0c310a32021-08-24 10:37:24 -07002946 CASE_OP_32_64(movcond):
2947 done = fold_movcond(&ctx, op);
2948 break;
Richard Henderson2f9f08b2021-08-25 12:03:48 -07002949 CASE_OP_32_64(mul):
2950 done = fold_mul(&ctx, op);
2951 break;
2952 CASE_OP_32_64(mulsh):
2953 CASE_OP_32_64(muluh):
2954 done = fold_mul_highpart(&ctx, op);
2955 break;
Richard Henderson407112b2021-08-26 06:33:04 -07002956 CASE_OP_32_64(muls2):
2957 CASE_OP_32_64(mulu2):
2958 done = fold_multiply2(&ctx, op);
Richard Henderson6b8ac0d2021-08-24 10:24:12 -07002959 break;
Richard Hendersoned523472021-12-16 11:17:46 -08002960 CASE_OP_32_64_VEC(nand):
Richard Henderson2f9f08b2021-08-25 12:03:48 -07002961 done = fold_nand(&ctx, op);
2962 break;
2963 CASE_OP_32_64(neg):
2964 done = fold_neg(&ctx, op);
2965 break;
Richard Hendersoned523472021-12-16 11:17:46 -08002966 CASE_OP_32_64_VEC(nor):
Richard Henderson2f9f08b2021-08-25 12:03:48 -07002967 done = fold_nor(&ctx, op);
2968 break;
2969 CASE_OP_32_64_VEC(not):
2970 done = fold_not(&ctx, op);
2971 break;
2972 CASE_OP_32_64_VEC(or):
2973 done = fold_or(&ctx, op);
2974 break;
2975 CASE_OP_32_64_VEC(orc):
2976 done = fold_orc(&ctx, op);
2977 break;
Richard Hendersonfecccfc2023-05-16 20:07:20 -07002978 case INDEX_op_qemu_ld_a32_i32:
2979 case INDEX_op_qemu_ld_a64_i32:
2980 case INDEX_op_qemu_ld_a32_i64:
2981 case INDEX_op_qemu_ld_a64_i64:
2982 case INDEX_op_qemu_ld_a32_i128:
2983 case INDEX_op_qemu_ld_a64_i128:
Richard Henderson3eefdf22021-08-25 11:06:43 -07002984 done = fold_qemu_ld(&ctx, op);
2985 break;
Richard Hendersonfecccfc2023-05-16 20:07:20 -07002986 case INDEX_op_qemu_st8_a32_i32:
2987 case INDEX_op_qemu_st8_a64_i32:
2988 case INDEX_op_qemu_st_a32_i32:
2989 case INDEX_op_qemu_st_a64_i32:
2990 case INDEX_op_qemu_st_a32_i64:
2991 case INDEX_op_qemu_st_a64_i64:
2992 case INDEX_op_qemu_st_a32_i128:
2993 case INDEX_op_qemu_st_a64_i128:
Richard Henderson3eefdf22021-08-25 11:06:43 -07002994 done = fold_qemu_st(&ctx, op);
2995 break;
Richard Henderson2f9f08b2021-08-25 12:03:48 -07002996 CASE_OP_32_64(rem):
2997 CASE_OP_32_64(remu):
2998 done = fold_remainder(&ctx, op);
2999 break;
3000 CASE_OP_32_64(rotl):
3001 CASE_OP_32_64(rotr):
3002 CASE_OP_32_64(sar):
3003 CASE_OP_32_64(shl):
3004 CASE_OP_32_64(shr):
3005 done = fold_shift(&ctx, op);
3006 break;
Richard Hendersonc63ff552021-08-24 09:35:30 -07003007 CASE_OP_32_64(setcond):
3008 done = fold_setcond(&ctx, op);
3009 break;
Richard Henderson36355022023-08-04 23:24:04 +00003010 CASE_OP_32_64(negsetcond):
3011 done = fold_negsetcond(&ctx, op);
3012 break;
Richard Hendersonbc47b1a2021-08-24 09:09:35 -07003013 case INDEX_op_setcond2_i32:
3014 done = fold_setcond2(&ctx, op);
3015 break;
Richard Henderson1f106542024-09-06 12:22:41 -07003016 case INDEX_op_cmp_vec:
3017 done = fold_cmp_vec(&ctx, op);
3018 break;
3019 case INDEX_op_cmpsel_vec:
3020 done = fold_cmpsel_vec(&ctx, op);
3021 break;
Richard Hendersone58b9772024-09-06 22:30:01 -07003022 case INDEX_op_bitsel_vec:
3023 done = fold_bitsel_vec(&ctx, op);
3024 break;
Richard Hendersonb6617c82021-08-24 10:44:53 -07003025 CASE_OP_32_64(sextract):
3026 done = fold_sextract(&ctx, op);
3027 break;
Richard Hendersonc578ff12021-12-16 06:07:25 -08003028 CASE_OP_32_64(sub):
Richard Henderson2f9f08b2021-08-25 12:03:48 -07003029 done = fold_sub(&ctx, op);
3030 break;
Richard Hendersonc578ff12021-12-16 06:07:25 -08003031 case INDEX_op_sub_vec:
3032 done = fold_sub_vec(&ctx, op);
3033 break;
Richard Henderson9531c072021-08-26 06:51:39 -07003034 CASE_OP_32_64(sub2):
3035 done = fold_sub2(&ctx, op);
Richard Hendersone3f7dc22021-08-24 10:30:38 -07003036 break;
Richard Henderson2f9f08b2021-08-25 12:03:48 -07003037 CASE_OP_32_64_VEC(xor):
3038 done = fold_xor(&ctx, op);
Richard Hendersonb10f3832021-08-23 22:30:17 -07003039 break;
Richard Henderson2cfac7f2021-08-25 13:05:43 -07003040 default:
3041 break;
Richard Hendersonb10f3832021-08-23 22:30:17 -07003042 }
3043
Richard Henderson404a1482021-08-24 11:08:21 -07003044 if (!done) {
3045 finish_folding(&ctx, op);
3046 }
Kirill Batuzov8f2e8c02011-07-07 16:37:12 +04003047 }
Kirill Batuzov8f2e8c02011-07-07 16:37:12 +04003048}