blob: 84af210bc0b9f931e4f874bccdb43fb55af0181a [file] [log] [blame]
Richard Hendersond2fd7452017-09-14 13:53:46 -07001/*
2 * Tiny Code Generator for QEMU
3 *
4 * Copyright (c) 2018 Linaro, Inc.
5 *
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
Thomas Huthfb0343d2019-01-23 15:08:56 +01009 * version 2.1 of the License, or (at your option) any later version.
Richard Hendersond2fd7452017-09-14 13:53:46 -070010 *
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
15 *
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
18 */
19
20#include "qemu/osdep.h"
Philippe Mathieu-Daudédcb32f12020-01-01 12:23:00 +010021#include "tcg/tcg.h"
Richard Henderson47f73132023-02-24 22:45:43 -100022#include "tcg/tcg-temp-internal.h"
Richard Hendersonad3d0e42023-03-28 18:17:24 -070023#include "tcg/tcg-op-common.h"
Philippe Mathieu-Daudédcb32f12020-01-01 12:23:00 +010024#include "tcg/tcg-mo.h"
Richard Hendersond56fea72022-10-17 11:07:39 +100025#include "tcg-internal.h"
26
Richard Henderson53229a72019-03-17 00:27:29 +000027/*
28 * Vector optional opcode tracking.
29 * Except for the basic logical operations (and, or, xor), and
30 * data movement (mov, ld, st, dupi), many vector opcodes are
31 * optional and may not be supported on the host. Thank Intel
32 * for the irregularity in their instruction set.
33 *
34 * The gvec expanders allow custom vector operations to be composed,
35 * generally via the .fniv callback in the GVecGen* structures. At
36 * the same time, in deciding whether to use this hook we need to
37 * know if the host supports the required operations. This is
38 * presented as an array of opcodes, terminated by 0. Each opcode
39 * is assumed to be expanded with the given VECE.
40 *
41 * For debugging, we want to validate this array. Therefore, when
42 * tcg_ctx->vec_opt_opc is non-NULL, the tcg_gen_*_vec expanders
43 * will validate that their opcode is present in the list.
44 */
Philippe Mathieu-Daudéec2297b2023-06-29 11:11:07 +020045static void tcg_assert_listed_vecop(TCGOpcode op)
Richard Henderson53229a72019-03-17 00:27:29 +000046{
Philippe Mathieu-Daudéec2297b2023-06-29 11:11:07 +020047#ifdef CONFIG_DEBUG_TCG
Richard Henderson53229a72019-03-17 00:27:29 +000048 const TCGOpcode *p = tcg_ctx->vecop_list;
49 if (p) {
50 for (; *p; ++p) {
51 if (*p == op) {
52 return;
53 }
54 }
55 g_assert_not_reached();
56 }
Richard Henderson53229a72019-03-17 00:27:29 +000057#endif
Philippe Mathieu-Daudéec2297b2023-06-29 11:11:07 +020058}
Richard Henderson53229a72019-03-17 00:27:29 +000059
60bool tcg_can_emit_vecop_list(const TCGOpcode *list,
61 TCGType type, unsigned vece)
62{
63 if (list == NULL) {
64 return true;
65 }
66
67 for (; *list; ++list) {
68 TCGOpcode opc = *list;
69
70#ifdef CONFIG_DEBUG_TCG
71 switch (opc) {
72 case INDEX_op_and_vec:
73 case INDEX_op_or_vec:
74 case INDEX_op_xor_vec:
75 case INDEX_op_mov_vec:
76 case INDEX_op_dup_vec:
Richard Henderson53229a72019-03-17 00:27:29 +000077 case INDEX_op_dup2_vec:
78 case INDEX_op_ld_vec:
79 case INDEX_op_st_vec:
Richard Henderson38dc1292019-04-30 11:02:23 -070080 case INDEX_op_bitsel_vec:
Richard Henderson53229a72019-03-17 00:27:29 +000081 /* These opcodes are mandatory and should not be listed. */
82 g_assert_not_reached();
Richard Henderson11978f62019-06-27 17:34:47 +000083 case INDEX_op_not_vec:
84 /* These opcodes have generic expansions using the above. */
85 g_assert_not_reached();
Richard Henderson53229a72019-03-17 00:27:29 +000086 default:
87 break;
88 }
89#endif
90
91 if (tcg_can_emit_vec_op(opc, type, vece)) {
92 continue;
93 }
94
95 /*
96 * The opcode list is created by front ends based on what they
97 * actually invoke. We must mirror the logic in the routines
98 * below for generic expansions using other opcodes.
99 */
100 switch (opc) {
101 case INDEX_op_neg_vec:
102 if (tcg_can_emit_vec_op(INDEX_op_sub_vec, type, vece)) {
103 continue;
104 }
105 break;
Richard Hendersonbcefc902019-04-17 13:53:02 -1000106 case INDEX_op_abs_vec:
107 if (tcg_can_emit_vec_op(INDEX_op_sub_vec, type, vece)
108 && (tcg_can_emit_vec_op(INDEX_op_smax_vec, type, vece) > 0
109 || tcg_can_emit_vec_op(INDEX_op_sari_vec, type, vece) > 0
110 || tcg_can_emit_vec_op(INDEX_op_cmp_vec, type, vece))) {
111 continue;
112 }
113 break;
Richard Henderson2552d602020-09-14 19:25:58 -0700114 case INDEX_op_usadd_vec:
115 if (tcg_can_emit_vec_op(INDEX_op_umin_vec, type, vece) ||
116 tcg_can_emit_vec_op(INDEX_op_cmp_vec, type, vece)) {
117 continue;
118 }
119 break;
120 case INDEX_op_ussub_vec:
121 if (tcg_can_emit_vec_op(INDEX_op_umax_vec, type, vece) ||
122 tcg_can_emit_vec_op(INDEX_op_cmp_vec, type, vece)) {
123 continue;
124 }
125 break;
Richard Hendersonf75da292019-04-30 13:01:12 -0700126 case INDEX_op_cmpsel_vec:
Richard Henderson72b4c792019-04-20 03:26:09 +0000127 case INDEX_op_smin_vec:
128 case INDEX_op_smax_vec:
129 case INDEX_op_umin_vec:
130 case INDEX_op_umax_vec:
Richard Hendersonf75da292019-04-30 13:01:12 -0700131 if (tcg_can_emit_vec_op(INDEX_op_cmp_vec, type, vece)) {
132 continue;
133 }
134 break;
Richard Henderson53229a72019-03-17 00:27:29 +0000135 default:
136 break;
137 }
138 return false;
139 }
140 return true;
141}
142
Richard Hendersond2fd7452017-09-14 13:53:46 -0700143void vec_gen_2(TCGOpcode opc, TCGType type, unsigned vece, TCGArg r, TCGArg a)
144{
Philippe Mathieu-Daudéd4478942022-12-18 22:18:31 +0100145 TCGOp *op = tcg_emit_op(opc, 2);
Richard Hendersond2fd7452017-09-14 13:53:46 -0700146 TCGOP_VECL(op) = type - TCG_TYPE_V64;
147 TCGOP_VECE(op) = vece;
148 op->args[0] = r;
149 op->args[1] = a;
150}
151
152void vec_gen_3(TCGOpcode opc, TCGType type, unsigned vece,
153 TCGArg r, TCGArg a, TCGArg b)
154{
Philippe Mathieu-Daudéd4478942022-12-18 22:18:31 +0100155 TCGOp *op = tcg_emit_op(opc, 3);
Richard Hendersond2fd7452017-09-14 13:53:46 -0700156 TCGOP_VECL(op) = type - TCG_TYPE_V64;
157 TCGOP_VECE(op) = vece;
158 op->args[0] = r;
159 op->args[1] = a;
160 op->args[2] = b;
161}
162
163void vec_gen_4(TCGOpcode opc, TCGType type, unsigned vece,
164 TCGArg r, TCGArg a, TCGArg b, TCGArg c)
165{
Philippe Mathieu-Daudéd4478942022-12-18 22:18:31 +0100166 TCGOp *op = tcg_emit_op(opc, 4);
Richard Hendersond2fd7452017-09-14 13:53:46 -0700167 TCGOP_VECL(op) = type - TCG_TYPE_V64;
168 TCGOP_VECE(op) = vece;
169 op->args[0] = r;
170 op->args[1] = a;
171 op->args[2] = b;
172 op->args[3] = c;
173}
174
Richard Hendersonf75da292019-04-30 13:01:12 -0700175static void vec_gen_6(TCGOpcode opc, TCGType type, unsigned vece, TCGArg r,
176 TCGArg a, TCGArg b, TCGArg c, TCGArg d, TCGArg e)
177{
Philippe Mathieu-Daudéd4478942022-12-18 22:18:31 +0100178 TCGOp *op = tcg_emit_op(opc, 6);
Richard Hendersonf75da292019-04-30 13:01:12 -0700179 TCGOP_VECL(op) = type - TCG_TYPE_V64;
180 TCGOP_VECE(op) = vece;
181 op->args[0] = r;
182 op->args[1] = a;
183 op->args[2] = b;
184 op->args[3] = c;
185 op->args[4] = d;
186 op->args[5] = e;
187}
188
Richard Hendersond2fd7452017-09-14 13:53:46 -0700189static void vec_gen_op2(TCGOpcode opc, unsigned vece, TCGv_vec r, TCGv_vec a)
190{
191 TCGTemp *rt = tcgv_vec_temp(r);
192 TCGTemp *at = tcgv_vec_temp(a);
193 TCGType type = rt->base_type;
194
Richard Hendersondb432672017-09-15 14:11:45 -0700195 /* Must enough inputs for the output. */
196 tcg_debug_assert(at->base_type >= type);
Richard Hendersond2fd7452017-09-14 13:53:46 -0700197 vec_gen_2(opc, type, vece, temp_arg(rt), temp_arg(at));
198}
199
200static void vec_gen_op3(TCGOpcode opc, unsigned vece,
201 TCGv_vec r, TCGv_vec a, TCGv_vec b)
202{
203 TCGTemp *rt = tcgv_vec_temp(r);
204 TCGTemp *at = tcgv_vec_temp(a);
205 TCGTemp *bt = tcgv_vec_temp(b);
206 TCGType type = rt->base_type;
207
Richard Hendersondb432672017-09-15 14:11:45 -0700208 /* Must enough inputs for the output. */
209 tcg_debug_assert(at->base_type >= type);
210 tcg_debug_assert(bt->base_type >= type);
Richard Hendersond2fd7452017-09-14 13:53:46 -0700211 vec_gen_3(opc, type, vece, temp_arg(rt), temp_arg(at), temp_arg(bt));
212}
213
214void tcg_gen_mov_vec(TCGv_vec r, TCGv_vec a)
215{
216 if (r != a) {
217 vec_gen_op2(INDEX_op_mov_vec, 0, r, a);
218 }
219}
220
Richard Hendersondb432672017-09-15 14:11:45 -0700221void tcg_gen_dupi_vec(unsigned vece, TCGv_vec r, uint64_t a)
222{
Richard Henderson0b4286d2020-09-06 17:33:18 -0700223 TCGTemp *rt = tcgv_vec_temp(r);
224 tcg_gen_mov_vec(r, tcg_constant_vec(rt->base_type, vece, a));
Richard Hendersond2fd7452017-09-14 13:53:46 -0700225}
226
227void tcg_gen_dup_i64_vec(unsigned vece, TCGv_vec r, TCGv_i64 a)
228{
229 TCGArg ri = tcgv_vec_arg(r);
230 TCGTemp *rt = arg_temp(ri);
231 TCGType type = rt->base_type;
232
233 if (TCG_TARGET_REG_BITS == 64) {
234 TCGArg ai = tcgv_i64_arg(a);
Richard Hendersondb432672017-09-15 14:11:45 -0700235 vec_gen_2(INDEX_op_dup_vec, type, vece, ri, ai);
Richard Hendersond2fd7452017-09-14 13:53:46 -0700236 } else if (vece == MO_64) {
237 TCGArg al = tcgv_i32_arg(TCGV_LOW(a));
238 TCGArg ah = tcgv_i32_arg(TCGV_HIGH(a));
239 vec_gen_3(INDEX_op_dup2_vec, type, MO_64, ri, al, ah);
240 } else {
241 TCGArg ai = tcgv_i32_arg(TCGV_LOW(a));
Richard Hendersondb432672017-09-15 14:11:45 -0700242 vec_gen_2(INDEX_op_dup_vec, type, vece, ri, ai);
Richard Hendersond2fd7452017-09-14 13:53:46 -0700243 }
244}
245
246void tcg_gen_dup_i32_vec(unsigned vece, TCGv_vec r, TCGv_i32 a)
247{
248 TCGArg ri = tcgv_vec_arg(r);
249 TCGArg ai = tcgv_i32_arg(a);
250 TCGTemp *rt = arg_temp(ri);
251 TCGType type = rt->base_type;
252
253 vec_gen_2(INDEX_op_dup_vec, type, vece, ri, ai);
254}
255
Richard Henderson37ee55a2019-03-17 01:55:22 +0000256void tcg_gen_dup_mem_vec(unsigned vece, TCGv_vec r, TCGv_ptr b,
257 tcg_target_long ofs)
258{
259 TCGArg ri = tcgv_vec_arg(r);
260 TCGArg bi = tcgv_ptr_arg(b);
261 TCGTemp *rt = arg_temp(ri);
262 TCGType type = rt->base_type;
263
264 vec_gen_3(INDEX_op_dupm_vec, type, vece, ri, bi, ofs);
265}
266
Richard Hendersond2fd7452017-09-14 13:53:46 -0700267static void vec_gen_ldst(TCGOpcode opc, TCGv_vec r, TCGv_ptr b, TCGArg o)
268{
269 TCGArg ri = tcgv_vec_arg(r);
270 TCGArg bi = tcgv_ptr_arg(b);
271 TCGTemp *rt = arg_temp(ri);
272 TCGType type = rt->base_type;
273
274 vec_gen_3(opc, type, 0, ri, bi, o);
275}
276
277void tcg_gen_ld_vec(TCGv_vec r, TCGv_ptr b, TCGArg o)
278{
279 vec_gen_ldst(INDEX_op_ld_vec, r, b, o);
280}
281
282void tcg_gen_st_vec(TCGv_vec r, TCGv_ptr b, TCGArg o)
283{
284 vec_gen_ldst(INDEX_op_st_vec, r, b, o);
285}
286
287void tcg_gen_stl_vec(TCGv_vec r, TCGv_ptr b, TCGArg o, TCGType low_type)
288{
289 TCGArg ri = tcgv_vec_arg(r);
290 TCGArg bi = tcgv_ptr_arg(b);
291 TCGTemp *rt = arg_temp(ri);
292 TCGType type = rt->base_type;
293
294 tcg_debug_assert(low_type >= TCG_TYPE_V64);
295 tcg_debug_assert(low_type <= type);
296 vec_gen_3(INDEX_op_st_vec, low_type, 0, ri, bi, o);
297}
298
Richard Hendersond2fd7452017-09-14 13:53:46 -0700299void tcg_gen_and_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
300{
301 vec_gen_op3(INDEX_op_and_vec, 0, r, a, b);
302}
303
304void tcg_gen_or_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
305{
306 vec_gen_op3(INDEX_op_or_vec, 0, r, a, b);
307}
308
309void tcg_gen_xor_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
310{
311 vec_gen_op3(INDEX_op_xor_vec, 0, r, a, b);
312}
313
314void tcg_gen_andc_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
315{
316 if (TCG_TARGET_HAS_andc_vec) {
317 vec_gen_op3(INDEX_op_andc_vec, 0, r, a, b);
318 } else {
319 TCGv_vec t = tcg_temp_new_vec_matching(r);
320 tcg_gen_not_vec(0, t, b);
321 tcg_gen_and_vec(0, r, a, t);
322 tcg_temp_free_vec(t);
323 }
324}
325
326void tcg_gen_orc_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
327{
328 if (TCG_TARGET_HAS_orc_vec) {
329 vec_gen_op3(INDEX_op_orc_vec, 0, r, a, b);
330 } else {
331 TCGv_vec t = tcg_temp_new_vec_matching(r);
332 tcg_gen_not_vec(0, t, b);
333 tcg_gen_or_vec(0, r, a, t);
334 tcg_temp_free_vec(t);
335 }
336}
337
Richard Hendersonf5508052018-12-17 13:22:06 -0800338void tcg_gen_nand_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
339{
Richard Hendersoned523472021-12-16 11:17:46 -0800340 if (TCG_TARGET_HAS_nand_vec) {
341 vec_gen_op3(INDEX_op_nand_vec, 0, r, a, b);
342 } else {
343 tcg_gen_and_vec(0, r, a, b);
344 tcg_gen_not_vec(0, r, r);
345 }
Richard Hendersonf5508052018-12-17 13:22:06 -0800346}
347
348void tcg_gen_nor_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
349{
Richard Hendersoned523472021-12-16 11:17:46 -0800350 if (TCG_TARGET_HAS_nor_vec) {
351 vec_gen_op3(INDEX_op_nor_vec, 0, r, a, b);
352 } else {
353 tcg_gen_or_vec(0, r, a, b);
354 tcg_gen_not_vec(0, r, r);
355 }
Richard Hendersonf5508052018-12-17 13:22:06 -0800356}
357
358void tcg_gen_eqv_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
359{
Richard Hendersoned523472021-12-16 11:17:46 -0800360 if (TCG_TARGET_HAS_eqv_vec) {
361 vec_gen_op3(INDEX_op_eqv_vec, 0, r, a, b);
362 } else {
363 tcg_gen_xor_vec(0, r, a, b);
364 tcg_gen_not_vec(0, r, r);
365 }
Richard Hendersonf5508052018-12-17 13:22:06 -0800366}
367
Richard Hendersonce27c5d2019-03-16 21:44:56 +0000368static bool do_op2(unsigned vece, TCGv_vec r, TCGv_vec a, TCGOpcode opc)
369{
370 TCGTemp *rt = tcgv_vec_temp(r);
371 TCGTemp *at = tcgv_vec_temp(a);
372 TCGArg ri = temp_arg(rt);
373 TCGArg ai = temp_arg(at);
374 TCGType type = rt->base_type;
375 int can;
376
377 tcg_debug_assert(at->base_type >= type);
Richard Henderson53229a72019-03-17 00:27:29 +0000378 tcg_assert_listed_vecop(opc);
Richard Hendersonce27c5d2019-03-16 21:44:56 +0000379 can = tcg_can_emit_vec_op(opc, type, vece);
380 if (can > 0) {
381 vec_gen_2(opc, type, vece, ri, ai);
382 } else if (can < 0) {
Richard Henderson53229a72019-03-17 00:27:29 +0000383 const TCGOpcode *hold_list = tcg_swap_vecop_list(NULL);
Richard Hendersonce27c5d2019-03-16 21:44:56 +0000384 tcg_expand_vec_op(opc, type, vece, ri, ai);
Richard Henderson53229a72019-03-17 00:27:29 +0000385 tcg_swap_vecop_list(hold_list);
Richard Hendersonce27c5d2019-03-16 21:44:56 +0000386 } else {
387 return false;
388 }
389 return true;
390}
391
Richard Hendersond2fd7452017-09-14 13:53:46 -0700392void tcg_gen_not_vec(unsigned vece, TCGv_vec r, TCGv_vec a)
393{
Richard Henderson77fafcb2023-08-28 12:15:35 -0700394 if (TCG_TARGET_HAS_not_vec) {
395 vec_gen_op2(INDEX_op_not_vec, 0, r, a);
396 } else {
Richard Hendersonf6ff9c22023-02-26 13:57:36 -1000397 tcg_gen_xor_vec(0, r, a, tcg_constant_vec_matching(r, 0, -1));
Richard Hendersond2fd7452017-09-14 13:53:46 -0700398 }
399}
400
401void tcg_gen_neg_vec(unsigned vece, TCGv_vec r, TCGv_vec a)
402{
Richard Henderson53229a72019-03-17 00:27:29 +0000403 const TCGOpcode *hold_list;
404
405 tcg_assert_listed_vecop(INDEX_op_neg_vec);
406 hold_list = tcg_swap_vecop_list(NULL);
407
Richard Hendersonce27c5d2019-03-16 21:44:56 +0000408 if (!TCG_TARGET_HAS_neg_vec || !do_op2(vece, r, a, INDEX_op_neg_vec)) {
Richard Hendersonf6ff9c22023-02-26 13:57:36 -1000409 tcg_gen_sub_vec(vece, r, tcg_constant_vec_matching(r, vece, 0), a);
Richard Hendersond2fd7452017-09-14 13:53:46 -0700410 }
Richard Henderson53229a72019-03-17 00:27:29 +0000411 tcg_swap_vecop_list(hold_list);
Richard Hendersond2fd7452017-09-14 13:53:46 -0700412}
Richard Hendersond0ec9792017-11-17 14:35:11 +0100413
Richard Hendersonbcefc902019-04-17 13:53:02 -1000414void tcg_gen_abs_vec(unsigned vece, TCGv_vec r, TCGv_vec a)
415{
416 const TCGOpcode *hold_list;
417
418 tcg_assert_listed_vecop(INDEX_op_abs_vec);
419 hold_list = tcg_swap_vecop_list(NULL);
420
421 if (!do_op2(vece, r, a, INDEX_op_abs_vec)) {
422 TCGType type = tcgv_vec_temp(r)->base_type;
423 TCGv_vec t = tcg_temp_new_vec(type);
424
425 tcg_debug_assert(tcg_can_emit_vec_op(INDEX_op_sub_vec, type, vece));
426 if (tcg_can_emit_vec_op(INDEX_op_smax_vec, type, vece) > 0) {
427 tcg_gen_neg_vec(vece, t, a);
428 tcg_gen_smax_vec(vece, r, a, t);
429 } else {
430 if (tcg_can_emit_vec_op(INDEX_op_sari_vec, type, vece) > 0) {
431 tcg_gen_sari_vec(vece, t, a, (8 << vece) - 1);
432 } else {
Richard Henderson0b4286d2020-09-06 17:33:18 -0700433 tcg_gen_cmp_vec(TCG_COND_LT, vece, t, a,
434 tcg_constant_vec(type, vece, 0));
Richard Hendersonbcefc902019-04-17 13:53:02 -1000435 }
436 tcg_gen_xor_vec(vece, r, a, t);
437 tcg_gen_sub_vec(vece, r, r, t);
438 }
439
440 tcg_temp_free_vec(t);
441 }
442 tcg_swap_vecop_list(hold_list);
443}
444
Richard Hendersond0ec9792017-11-17 14:35:11 +0100445static void do_shifti(TCGOpcode opc, unsigned vece,
446 TCGv_vec r, TCGv_vec a, int64_t i)
447{
448 TCGTemp *rt = tcgv_vec_temp(r);
449 TCGTemp *at = tcgv_vec_temp(a);
450 TCGArg ri = temp_arg(rt);
451 TCGArg ai = temp_arg(at);
452 TCGType type = rt->base_type;
453 int can;
454
455 tcg_debug_assert(at->base_type == type);
456 tcg_debug_assert(i >= 0 && i < (8 << vece));
Richard Henderson53229a72019-03-17 00:27:29 +0000457 tcg_assert_listed_vecop(opc);
Richard Hendersond0ec9792017-11-17 14:35:11 +0100458
459 if (i == 0) {
460 tcg_gen_mov_vec(r, a);
461 return;
462 }
463
464 can = tcg_can_emit_vec_op(opc, type, vece);
465 if (can > 0) {
466 vec_gen_3(opc, type, vece, ri, ai, i);
467 } else {
468 /* We leave the choice of expansion via scalar or vector shift
469 to the target. Often, but not always, dupi can feed a vector
470 shift easier than a scalar. */
Richard Henderson53229a72019-03-17 00:27:29 +0000471 const TCGOpcode *hold_list = tcg_swap_vecop_list(NULL);
Richard Hendersond0ec9792017-11-17 14:35:11 +0100472 tcg_debug_assert(can < 0);
473 tcg_expand_vec_op(opc, type, vece, ri, ai, i);
Richard Henderson53229a72019-03-17 00:27:29 +0000474 tcg_swap_vecop_list(hold_list);
Richard Hendersond0ec9792017-11-17 14:35:11 +0100475 }
476}
477
478void tcg_gen_shli_vec(unsigned vece, TCGv_vec r, TCGv_vec a, int64_t i)
479{
480 do_shifti(INDEX_op_shli_vec, vece, r, a, i);
481}
482
483void tcg_gen_shri_vec(unsigned vece, TCGv_vec r, TCGv_vec a, int64_t i)
484{
485 do_shifti(INDEX_op_shri_vec, vece, r, a, i);
486}
487
488void tcg_gen_sari_vec(unsigned vece, TCGv_vec r, TCGv_vec a, int64_t i)
489{
490 do_shifti(INDEX_op_sari_vec, vece, r, a, i);
491}
Richard Henderson212be172017-11-17 20:47:42 +0100492
Richard Hendersonb0f7e742020-04-19 18:01:52 -0700493void tcg_gen_rotli_vec(unsigned vece, TCGv_vec r, TCGv_vec a, int64_t i)
494{
495 do_shifti(INDEX_op_rotli_vec, vece, r, a, i);
496}
497
498void tcg_gen_rotri_vec(unsigned vece, TCGv_vec r, TCGv_vec a, int64_t i)
499{
500 int bits = 8 << vece;
501 tcg_debug_assert(i >= 0 && i < bits);
502 do_shifti(INDEX_op_rotli_vec, vece, r, a, -i & (bits - 1));
503}
504
Richard Henderson212be172017-11-17 20:47:42 +0100505void tcg_gen_cmp_vec(TCGCond cond, unsigned vece,
506 TCGv_vec r, TCGv_vec a, TCGv_vec b)
507{
508 TCGTemp *rt = tcgv_vec_temp(r);
509 TCGTemp *at = tcgv_vec_temp(a);
510 TCGTemp *bt = tcgv_vec_temp(b);
Richard Henderson6975cc42024-05-15 15:08:16 +0200511 TCGTemp *tt = NULL;
Richard Henderson212be172017-11-17 20:47:42 +0100512 TCGArg ri = temp_arg(rt);
513 TCGArg ai = temp_arg(at);
514 TCGArg bi = temp_arg(bt);
Richard Henderson6975cc42024-05-15 15:08:16 +0200515 TCGArg ti;
Richard Henderson212be172017-11-17 20:47:42 +0100516 TCGType type = rt->base_type;
517 int can;
518
Richard Henderson9a938d82018-04-17 11:35:42 -1000519 tcg_debug_assert(at->base_type >= type);
520 tcg_debug_assert(bt->base_type >= type);
Richard Henderson53229a72019-03-17 00:27:29 +0000521 tcg_assert_listed_vecop(INDEX_op_cmp_vec);
Richard Henderson212be172017-11-17 20:47:42 +0100522 can = tcg_can_emit_vec_op(INDEX_op_cmp_vec, type, vece);
Richard Henderson6975cc42024-05-15 15:08:16 +0200523
524 if (!TCG_TARGET_HAS_tst_vec && is_tst_cond(cond)) {
525 tt = tcg_temp_new_internal(type, TEMP_EBB);
526 ti = temp_arg(tt);
527 vec_gen_3(INDEX_op_and_vec, type, 0, ti, ai, bi);
528 at = tt;
529 ai = ti;
530 bt = tcg_constant_internal(type, 0);
531 bi = temp_arg(bt);
532 cond = tcg_tst_eqne_cond(cond);
533 }
534
Richard Henderson212be172017-11-17 20:47:42 +0100535 if (can > 0) {
536 vec_gen_4(INDEX_op_cmp_vec, type, vece, ri, ai, bi, cond);
537 } else {
Richard Henderson53229a72019-03-17 00:27:29 +0000538 const TCGOpcode *hold_list = tcg_swap_vecop_list(NULL);
Richard Henderson212be172017-11-17 20:47:42 +0100539 tcg_debug_assert(can < 0);
540 tcg_expand_vec_op(INDEX_op_cmp_vec, type, vece, ri, ai, bi, cond);
Richard Henderson53229a72019-03-17 00:27:29 +0000541 tcg_swap_vecop_list(hold_list);
Richard Henderson212be172017-11-17 20:47:42 +0100542 }
Richard Henderson6975cc42024-05-15 15:08:16 +0200543
544 if (tt) {
545 tcg_temp_free_internal(tt);
546 }
Richard Henderson212be172017-11-17 20:47:42 +0100547}
Richard Henderson37740302017-11-21 10:11:14 +0100548
Richard Henderson17f79942019-04-20 03:13:26 +0000549static bool do_op3(unsigned vece, TCGv_vec r, TCGv_vec a,
Richard Henderson8afaf052018-12-17 18:01:47 -0800550 TCGv_vec b, TCGOpcode opc)
Richard Henderson37740302017-11-21 10:11:14 +0100551{
552 TCGTemp *rt = tcgv_vec_temp(r);
553 TCGTemp *at = tcgv_vec_temp(a);
554 TCGTemp *bt = tcgv_vec_temp(b);
555 TCGArg ri = temp_arg(rt);
556 TCGArg ai = temp_arg(at);
557 TCGArg bi = temp_arg(bt);
558 TCGType type = rt->base_type;
559 int can;
560
Richard Henderson9a938d82018-04-17 11:35:42 -1000561 tcg_debug_assert(at->base_type >= type);
562 tcg_debug_assert(bt->base_type >= type);
Richard Henderson53229a72019-03-17 00:27:29 +0000563 tcg_assert_listed_vecop(opc);
Richard Henderson8afaf052018-12-17 18:01:47 -0800564 can = tcg_can_emit_vec_op(opc, type, vece);
Richard Henderson37740302017-11-21 10:11:14 +0100565 if (can > 0) {
Richard Henderson8afaf052018-12-17 18:01:47 -0800566 vec_gen_3(opc, type, vece, ri, ai, bi);
Richard Henderson17f79942019-04-20 03:13:26 +0000567 } else if (can < 0) {
Richard Henderson53229a72019-03-17 00:27:29 +0000568 const TCGOpcode *hold_list = tcg_swap_vecop_list(NULL);
Richard Henderson8afaf052018-12-17 18:01:47 -0800569 tcg_expand_vec_op(opc, type, vece, ri, ai, bi);
Richard Henderson53229a72019-03-17 00:27:29 +0000570 tcg_swap_vecop_list(hold_list);
Richard Henderson17f79942019-04-20 03:13:26 +0000571 } else {
572 return false;
Richard Henderson37740302017-11-21 10:11:14 +0100573 }
Richard Henderson17f79942019-04-20 03:13:26 +0000574 return true;
575}
576
577static void do_op3_nofail(unsigned vece, TCGv_vec r, TCGv_vec a,
578 TCGv_vec b, TCGOpcode opc)
579{
580 bool ok = do_op3(vece, r, a, b, opc);
581 tcg_debug_assert(ok);
Richard Henderson37740302017-11-21 10:11:14 +0100582}
Richard Henderson8afaf052018-12-17 18:01:47 -0800583
Richard Hendersonce27c5d2019-03-16 21:44:56 +0000584void tcg_gen_add_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
585{
Richard Henderson17f79942019-04-20 03:13:26 +0000586 do_op3_nofail(vece, r, a, b, INDEX_op_add_vec);
Richard Hendersonce27c5d2019-03-16 21:44:56 +0000587}
588
589void tcg_gen_sub_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
590{
Richard Henderson17f79942019-04-20 03:13:26 +0000591 do_op3_nofail(vece, r, a, b, INDEX_op_sub_vec);
Richard Hendersonce27c5d2019-03-16 21:44:56 +0000592}
593
Richard Henderson8afaf052018-12-17 18:01:47 -0800594void tcg_gen_mul_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
595{
Richard Henderson17f79942019-04-20 03:13:26 +0000596 do_op3_nofail(vece, r, a, b, INDEX_op_mul_vec);
Richard Henderson8afaf052018-12-17 18:01:47 -0800597}
598
599void tcg_gen_ssadd_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
600{
Richard Henderson17f79942019-04-20 03:13:26 +0000601 do_op3_nofail(vece, r, a, b, INDEX_op_ssadd_vec);
Richard Henderson8afaf052018-12-17 18:01:47 -0800602}
603
604void tcg_gen_usadd_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
605{
Richard Henderson2552d602020-09-14 19:25:58 -0700606 if (!do_op3(vece, r, a, b, INDEX_op_usadd_vec)) {
607 const TCGOpcode *hold_list = tcg_swap_vecop_list(NULL);
608 TCGv_vec t = tcg_temp_new_vec_matching(r);
609
610 /* usadd(a, b) = min(a, ~b) + b */
611 tcg_gen_not_vec(vece, t, b);
612 tcg_gen_umin_vec(vece, t, t, a);
613 tcg_gen_add_vec(vece, r, t, b);
614
615 tcg_temp_free_vec(t);
616 tcg_swap_vecop_list(hold_list);
617 }
Richard Henderson8afaf052018-12-17 18:01:47 -0800618}
619
620void tcg_gen_sssub_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
621{
Richard Henderson17f79942019-04-20 03:13:26 +0000622 do_op3_nofail(vece, r, a, b, INDEX_op_sssub_vec);
Richard Henderson8afaf052018-12-17 18:01:47 -0800623}
624
625void tcg_gen_ussub_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
626{
Richard Henderson2552d602020-09-14 19:25:58 -0700627 if (!do_op3(vece, r, a, b, INDEX_op_ussub_vec)) {
628 const TCGOpcode *hold_list = tcg_swap_vecop_list(NULL);
629 TCGv_vec t = tcg_temp_new_vec_matching(r);
630
631 /* ussub(a, b) = max(a, b) - b */
632 tcg_gen_umax_vec(vece, t, a, b);
633 tcg_gen_sub_vec(vece, r, t, b);
634
635 tcg_temp_free_vec(t);
636 tcg_swap_vecop_list(hold_list);
637 }
Richard Henderson8afaf052018-12-17 18:01:47 -0800638}
Richard Hendersondd0a0fc2018-12-17 19:35:46 -0800639
Richard Henderson72b4c792019-04-20 03:26:09 +0000640static void do_minmax(unsigned vece, TCGv_vec r, TCGv_vec a,
641 TCGv_vec b, TCGOpcode opc, TCGCond cond)
642{
643 if (!do_op3(vece, r, a, b, opc)) {
Richard Henderson69c918d2020-06-09 16:32:09 -0700644 const TCGOpcode *hold_list = tcg_swap_vecop_list(NULL);
Richard Henderson72b4c792019-04-20 03:26:09 +0000645 tcg_gen_cmpsel_vec(cond, vece, r, a, b, a, b);
Richard Henderson69c918d2020-06-09 16:32:09 -0700646 tcg_swap_vecop_list(hold_list);
Richard Henderson72b4c792019-04-20 03:26:09 +0000647 }
648}
649
Richard Hendersondd0a0fc2018-12-17 19:35:46 -0800650void tcg_gen_smin_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
651{
Richard Henderson72b4c792019-04-20 03:26:09 +0000652 do_minmax(vece, r, a, b, INDEX_op_smin_vec, TCG_COND_LT);
Richard Hendersondd0a0fc2018-12-17 19:35:46 -0800653}
654
655void tcg_gen_umin_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
656{
Richard Henderson72b4c792019-04-20 03:26:09 +0000657 do_minmax(vece, r, a, b, INDEX_op_umin_vec, TCG_COND_LTU);
Richard Hendersondd0a0fc2018-12-17 19:35:46 -0800658}
659
660void tcg_gen_smax_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
661{
Richard Henderson72b4c792019-04-20 03:26:09 +0000662 do_minmax(vece, r, a, b, INDEX_op_smax_vec, TCG_COND_GT);
Richard Hendersondd0a0fc2018-12-17 19:35:46 -0800663}
664
665void tcg_gen_umax_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
666{
Richard Henderson72b4c792019-04-20 03:26:09 +0000667 do_minmax(vece, r, a, b, INDEX_op_umax_vec, TCG_COND_GTU);
Richard Hendersondd0a0fc2018-12-17 19:35:46 -0800668}
Richard Henderson5ee5c142019-04-13 20:42:37 -1000669
670void tcg_gen_shlv_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
671{
Richard Henderson17f79942019-04-20 03:13:26 +0000672 do_op3_nofail(vece, r, a, b, INDEX_op_shlv_vec);
Richard Henderson5ee5c142019-04-13 20:42:37 -1000673}
674
675void tcg_gen_shrv_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
676{
Richard Henderson17f79942019-04-20 03:13:26 +0000677 do_op3_nofail(vece, r, a, b, INDEX_op_shrv_vec);
Richard Henderson5ee5c142019-04-13 20:42:37 -1000678}
679
680void tcg_gen_sarv_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
681{
Richard Henderson17f79942019-04-20 03:13:26 +0000682 do_op3_nofail(vece, r, a, b, INDEX_op_sarv_vec);
Richard Henderson5ee5c142019-04-13 20:42:37 -1000683}
Richard Hendersonb4578cd2019-04-18 18:19:38 -1000684
Richard Henderson5d0ceda2020-04-19 19:47:59 -0700685void tcg_gen_rotlv_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
686{
687 do_op3_nofail(vece, r, a, b, INDEX_op_rotlv_vec);
688}
689
690void tcg_gen_rotrv_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
691{
692 do_op3_nofail(vece, r, a, b, INDEX_op_rotrv_vec);
693}
694
Richard Hendersonb4578cd2019-04-18 18:19:38 -1000695static void do_shifts(unsigned vece, TCGv_vec r, TCGv_vec a,
Richard Henderson3d5bb2e2020-04-20 07:56:36 -0700696 TCGv_i32 s, TCGOpcode opc)
Richard Hendersonb4578cd2019-04-18 18:19:38 -1000697{
698 TCGTemp *rt = tcgv_vec_temp(r);
699 TCGTemp *at = tcgv_vec_temp(a);
700 TCGTemp *st = tcgv_i32_temp(s);
701 TCGArg ri = temp_arg(rt);
702 TCGArg ai = temp_arg(at);
703 TCGArg si = temp_arg(st);
704 TCGType type = rt->base_type;
Richard Hendersonb4578cd2019-04-18 18:19:38 -1000705 int can;
706
707 tcg_debug_assert(at->base_type >= type);
Richard Henderson3d5bb2e2020-04-20 07:56:36 -0700708 tcg_assert_listed_vecop(opc);
709 can = tcg_can_emit_vec_op(opc, type, vece);
Richard Hendersonb4578cd2019-04-18 18:19:38 -1000710 if (can > 0) {
Richard Henderson3d5bb2e2020-04-20 07:56:36 -0700711 vec_gen_3(opc, type, vece, ri, ai, si);
Richard Hendersonb4578cd2019-04-18 18:19:38 -1000712 } else if (can < 0) {
Richard Henderson3d5bb2e2020-04-20 07:56:36 -0700713 const TCGOpcode *hold_list = tcg_swap_vecop_list(NULL);
714 tcg_expand_vec_op(opc, type, vece, ri, ai, si);
715 tcg_swap_vecop_list(hold_list);
Richard Hendersonb4578cd2019-04-18 18:19:38 -1000716 } else {
Richard Henderson3d5bb2e2020-04-20 07:56:36 -0700717 g_assert_not_reached();
Richard Hendersonb4578cd2019-04-18 18:19:38 -1000718 }
Richard Hendersonb4578cd2019-04-18 18:19:38 -1000719}
720
721void tcg_gen_shls_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_i32 b)
722{
Richard Henderson3d5bb2e2020-04-20 07:56:36 -0700723 do_shifts(vece, r, a, b, INDEX_op_shls_vec);
Richard Hendersonb4578cd2019-04-18 18:19:38 -1000724}
725
726void tcg_gen_shrs_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_i32 b)
727{
Richard Henderson3d5bb2e2020-04-20 07:56:36 -0700728 do_shifts(vece, r, a, b, INDEX_op_shrs_vec);
Richard Hendersonb4578cd2019-04-18 18:19:38 -1000729}
730
731void tcg_gen_sars_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_i32 b)
732{
Richard Henderson3d5bb2e2020-04-20 07:56:36 -0700733 do_shifts(vece, r, a, b, INDEX_op_sars_vec);
Richard Hendersonb4578cd2019-04-18 18:19:38 -1000734}
Richard Henderson38dc1292019-04-30 11:02:23 -0700735
Richard Henderson23850a72020-04-20 08:22:44 -0700736void tcg_gen_rotls_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_i32 s)
737{
738 do_shifts(vece, r, a, s, INDEX_op_rotls_vec);
739}
740
Richard Henderson38dc1292019-04-30 11:02:23 -0700741void tcg_gen_bitsel_vec(unsigned vece, TCGv_vec r, TCGv_vec a,
742 TCGv_vec b, TCGv_vec c)
743{
744 TCGTemp *rt = tcgv_vec_temp(r);
745 TCGTemp *at = tcgv_vec_temp(a);
746 TCGTemp *bt = tcgv_vec_temp(b);
747 TCGTemp *ct = tcgv_vec_temp(c);
748 TCGType type = rt->base_type;
749
750 tcg_debug_assert(at->base_type >= type);
751 tcg_debug_assert(bt->base_type >= type);
752 tcg_debug_assert(ct->base_type >= type);
753
754 if (TCG_TARGET_HAS_bitsel_vec) {
755 vec_gen_4(INDEX_op_bitsel_vec, type, MO_8,
756 temp_arg(rt), temp_arg(at), temp_arg(bt), temp_arg(ct));
757 } else {
758 TCGv_vec t = tcg_temp_new_vec(type);
759 tcg_gen_and_vec(MO_8, t, a, b);
760 tcg_gen_andc_vec(MO_8, r, c, a);
761 tcg_gen_or_vec(MO_8, r, r, t);
762 tcg_temp_free_vec(t);
763 }
764}
Richard Hendersonf75da292019-04-30 13:01:12 -0700765
766void tcg_gen_cmpsel_vec(TCGCond cond, unsigned vece, TCGv_vec r,
767 TCGv_vec a, TCGv_vec b, TCGv_vec c, TCGv_vec d)
768{
769 TCGTemp *rt = tcgv_vec_temp(r);
770 TCGTemp *at = tcgv_vec_temp(a);
771 TCGTemp *bt = tcgv_vec_temp(b);
772 TCGTemp *ct = tcgv_vec_temp(c);
773 TCGTemp *dt = tcgv_vec_temp(d);
774 TCGArg ri = temp_arg(rt);
775 TCGArg ai = temp_arg(at);
776 TCGArg bi = temp_arg(bt);
777 TCGArg ci = temp_arg(ct);
778 TCGArg di = temp_arg(dt);
779 TCGType type = rt->base_type;
780 const TCGOpcode *hold_list;
781 int can;
782
783 tcg_debug_assert(at->base_type >= type);
784 tcg_debug_assert(bt->base_type >= type);
785 tcg_debug_assert(ct->base_type >= type);
786 tcg_debug_assert(dt->base_type >= type);
787
788 tcg_assert_listed_vecop(INDEX_op_cmpsel_vec);
789 hold_list = tcg_swap_vecop_list(NULL);
790 can = tcg_can_emit_vec_op(INDEX_op_cmpsel_vec, type, vece);
791
792 if (can > 0) {
793 vec_gen_6(INDEX_op_cmpsel_vec, type, vece, ri, ai, bi, ci, di, cond);
794 } else if (can < 0) {
795 tcg_expand_vec_op(INDEX_op_cmpsel_vec, type, vece,
796 ri, ai, bi, ci, di, cond);
797 } else {
798 TCGv_vec t = tcg_temp_new_vec(type);
799 tcg_gen_cmp_vec(cond, vece, t, a, b);
800 tcg_gen_bitsel_vec(vece, r, t, c, d);
801 tcg_temp_free_vec(t);
802 }
803 tcg_swap_vecop_list(hold_list);
804}