| #------------------------------------------------------------------------------ | |
| # | |
| # Copyright (c) 2020, Arm, Limited. All rights reserved.<BR> | |
| # | |
| # SPDX-License-Identifier: BSD-2-Clause-Patent | |
| # | |
| #------------------------------------------------------------------------------ | |
| /* | |
| * Provide the GCC intrinsics that are required when using GCC 9 or | |
| * later with the -moutline-atomics options (which became the default | |
| * in GCC 10) | |
| */ | |
| .arch armv8-a | |
| .macro reg_alias, pfx, sz | |
| r0_\sz .req \pfx\()0 | |
| r1_\sz .req \pfx\()1 | |
| tmp0_\sz .req \pfx\()16 | |
| tmp1_\sz .req \pfx\()17 | |
| .endm | |
| /* | |
| * Define register aliases of the right type for each size | |
| * (xN for 8 bytes, wN for everything smaller) | |
| */ | |
| reg_alias w, 1 | |
| reg_alias w, 2 | |
| reg_alias w, 4 | |
| reg_alias x, 8 | |
| .macro fn_start, name:req | |
| .section .text.\name | |
| .globl \name | |
| .type \name, %function | |
| \name\(): | |
| .endm | |
| .macro fn_end, name:req | |
| .size \name, . - \name | |
| .endm | |
| /* | |
| * Emit an atomic helper for \model with operands of size \sz, using | |
| * the operation specified by \insn (which is the LSE name), and which | |
| * can be implemented using the generic load-locked/store-conditional | |
| * (LL/SC) sequence below, using the arithmetic operation given by | |
| * \opc. | |
| */ | |
| .macro emit_ld_sz, sz:req, insn:req, opc:req, model:req, s, a, l | |
| fn_start __aarch64_\insn\()\sz\()\model | |
| mov tmp0_\sz, r0_\sz | |
| 0: ld\a\()xr\s r0_\sz, [x1] | |
| .ifnc \insn, swp | |
| \opc tmp1_\sz, r0_\sz, tmp0_\sz | |
| st\l\()xr\s w15, tmp1_\sz, [x1] | |
| .else | |
| st\l\()xr\s w15, tmp0_\sz, [x1] | |
| .endif | |
| cbnz w15, 0b | |
| ret | |
| fn_end __aarch64_\insn\()\sz\()\model | |
| .endm | |
| /* | |
| * Emit atomic helpers for \model for operand sizes in the | |
| * set {1, 2, 4, 8}, for the instruction pattern given by | |
| * \insn. (This is the LSE name, but this implementation uses | |
| * the generic LL/SC sequence using \opc as the arithmetic | |
| * operation on the target.) | |
| */ | |
| .macro emit_ld, insn:req, opc:req, model:req, a, l | |
| emit_ld_sz 1, \insn, \opc, \model, b, \a, \l | |
| emit_ld_sz 2, \insn, \opc, \model, h, \a, \l | |
| emit_ld_sz 4, \insn, \opc, \model, , \a, \l | |
| emit_ld_sz 8, \insn, \opc, \model, , \a, \l | |
| .endm | |
| /* | |
| * Emit the compare and swap helper for \model and size \sz | |
| * using LL/SC instructions. | |
| */ | |
| .macro emit_cas_sz, sz:req, model:req, uxt:req, s, a, l | |
| fn_start __aarch64_cas\sz\()\model | |
| \uxt tmp0_\sz, r0_\sz | |
| 0: ld\a\()xr\s r0_\sz, [x2] | |
| cmp r0_\sz, tmp0_\sz | |
| bne 1f | |
| st\l\()xr\s w15, r1_\sz, [x2] | |
| cbnz w15, 0b | |
| 1: ret | |
| fn_end __aarch64_cas\sz\()\model | |
| .endm | |
| /* | |
| * Emit compare-and-swap helpers for \model for operand sizes in the | |
| * set {1, 2, 4, 8, 16}. | |
| */ | |
| .macro emit_cas, model:req, a, l | |
| emit_cas_sz 1, \model, uxtb, b, \a, \l | |
| emit_cas_sz 2, \model, uxth, h, \a, \l | |
| emit_cas_sz 4, \model, mov , , \a, \l | |
| emit_cas_sz 8, \model, mov , , \a, \l | |
| /* | |
| * We cannot use the parameterized sequence for 16 byte CAS, so we | |
| * need to define it explicitly. | |
| */ | |
| fn_start __aarch64_cas16\model | |
| mov x16, x0 | |
| mov x17, x1 | |
| 0: ld\a\()xp x0, x1, [x4] | |
| cmp x0, x16 | |
| ccmp x1, x17, #0, eq | |
| bne 1f | |
| st\l\()xp w15, x16, x17, [x4] | |
| cbnz w15, 0b | |
| 1: ret | |
| fn_end __aarch64_cas16\model | |
| .endm | |
| /* | |
| * Emit the set of GCC outline atomic helper functions for | |
| * the memory ordering model given by \model: | |
| * - relax unordered loads and stores | |
| * - acq load-acquire, unordered store | |
| * - rel unordered load, store-release | |
| * - acq_rel load-acquire, store-release | |
| */ | |
| .macro emit_model, model:req, a, l | |
| emit_ld ldadd, add, \model, \a, \l | |
| emit_ld ldclr, bic, \model, \a, \l | |
| emit_ld ldeor, eor, \model, \a, \l | |
| emit_ld ldset, orr, \model, \a, \l | |
| emit_ld swp, mov, \model, \a, \l | |
| emit_cas \model, \a, \l | |
| .endm | |
| emit_model _relax | |
| emit_model _acq, a | |
| emit_model _rel,, l | |
| emit_model _acq_rel, a, l |