/*
 *  Copyright(c) 2019-2022 Qualcomm Innovation Center, Inc. All Rights Reserved.
 *
 *  This program is free software; you can redistribute it and/or modify
 *  it under the terms of the GNU General Public License as published by
 *  the Free Software Foundation; either version 2 of the License, or
 *  (at your option) any later version.
 *
 *  This program is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU General Public License for more details.
 *
 *  You should have received a copy of the GNU General Public License
 *  along with this program; if not, see <http://www.gnu.org/licenses/>.
 */

#include <stdio.h>

/*
 *  Make sure that the :mem_noshuf packet attribute is honored.
 *  This is important when the addresses overlap.
 *  The store instruction in slot 1 effectively executes first,
 *  followed by the load instruction in slot 0.
 */

#define MEM_NOSHUF32(NAME, ST_TYPE, LD_TYPE, ST_OP, LD_OP) \
static inline unsigned int NAME(ST_TYPE * p, LD_TYPE * q, ST_TYPE x) \
{ \
    unsigned int ret; \
    asm volatile("{\n\t" \
                 "    " #ST_OP "(%1) = %3\n\t" \
                 "    %0 = " #LD_OP "(%2)\n\t" \
                 "}:mem_noshuf\n" \
                 : "=r"(ret) \
                 : "r"(p), "r"(q), "r"(x) \
                 : "memory"); \
    return ret; \
}

#define MEM_NOSHUF64(NAME, ST_TYPE, LD_TYPE, ST_OP, LD_OP) \
static inline unsigned long long NAME(ST_TYPE * p, LD_TYPE * q, ST_TYPE x) \
{ \
    unsigned long long ret; \
    asm volatile("{\n\t" \
                 "    " #ST_OP "(%1) = %3\n\t" \
                 "    %0 = " #LD_OP "(%2)\n\t" \
                 "}:mem_noshuf\n" \
                 : "=r"(ret) \
                 : "r"(p), "r"(q), "r"(x) \
                 : "memory"); \
    return ret; \
}

/* Store byte combinations */
MEM_NOSHUF32(mem_noshuf_sb_lb,  signed char,  signed char,      memb, memb)
MEM_NOSHUF32(mem_noshuf_sb_lub, signed char,  unsigned char,    memb, memub)
MEM_NOSHUF32(mem_noshuf_sb_lh,  signed char,  signed short,     memb, memh)
MEM_NOSHUF32(mem_noshuf_sb_luh, signed char,  unsigned short,   memb, memuh)
MEM_NOSHUF32(mem_noshuf_sb_lw,  signed char,  signed int,       memb, memw)
MEM_NOSHUF64(mem_noshuf_sb_ld,  signed char,  signed long long, memb, memd)

/* Store half combinations */
MEM_NOSHUF32(mem_noshuf_sh_lb,  signed short, signed char,      memh, memb)
MEM_NOSHUF32(mem_noshuf_sh_lub, signed short, unsigned char,    memh, memub)
MEM_NOSHUF32(mem_noshuf_sh_lh,  signed short, signed short,     memh, memh)
MEM_NOSHUF32(mem_noshuf_sh_luh, signed short, unsigned short,   memh, memuh)
MEM_NOSHUF32(mem_noshuf_sh_lw,  signed short, signed int,       memh, memw)
MEM_NOSHUF64(mem_noshuf_sh_ld,  signed short, signed long long, memh, memd)

/* Store word combinations */
MEM_NOSHUF32(mem_noshuf_sw_lb,  signed int,   signed char,      memw, memb)
MEM_NOSHUF32(mem_noshuf_sw_lub, signed int,   unsigned char,    memw, memub)
MEM_NOSHUF32(mem_noshuf_sw_lh,  signed int,   signed short,     memw, memh)
MEM_NOSHUF32(mem_noshuf_sw_luh, signed int,   unsigned short,   memw, memuh)
MEM_NOSHUF32(mem_noshuf_sw_lw,  signed int,   signed int,       memw, memw)
MEM_NOSHUF64(mem_noshuf_sw_ld,  signed int,   signed long long, memw, memd)

/* Store double combinations */
MEM_NOSHUF32(mem_noshuf_sd_lb,  long long,    signed char,      memd, memb)
MEM_NOSHUF32(mem_noshuf_sd_lub, long long,    unsigned char,    memd, memub)
MEM_NOSHUF32(mem_noshuf_sd_lh,  long long,    signed short,     memd, memh)
MEM_NOSHUF32(mem_noshuf_sd_luh, long long,    unsigned short,   memd, memuh)
MEM_NOSHUF32(mem_noshuf_sd_lw,  long long,    signed int,       memd, memw)
MEM_NOSHUF64(mem_noshuf_sd_ld,  long long,    signed long long, memd, memd)

static inline int pred_lw_sw(int pred, int *p, int *q, int x, int y)
{
    int ret;
    asm volatile("p0 = cmp.eq(%5, #0)\n\t"
                 "%0 = %3\n\t"
                 "{\n\t"
                 "    memw(%1) = %4\n\t"
                 "    if (!p0) %0 = memw(%2)\n\t"
                 "}:mem_noshuf\n"
                 : "=&r"(ret)
                 : "r"(p), "r"(q), "r"(x), "r"(y), "r"(pred)
                 : "p0", "memory");
    return ret;
}

static inline int pred_lw_sw_pi(int pred, int *p, int *q, int x, int y)
{
    int ret;
    asm volatile("p0 = cmp.eq(%5, #0)\n\t"
                 "%0 = %3\n\t"
                 "r7 = %2\n\t"
                 "{\n\t"
                 "    memw(%1) = %4\n\t"
                 "    if (!p0) %0 = memw(r7++#4)\n\t"
                 "}:mem_noshuf\n"
                 : "=&r"(ret)
                 : "r"(p), "r"(q), "r"(x), "r"(y), "r"(pred)
                 : "r7", "p0", "memory");
    return ret;
}

static inline long long pred_ld_sd(int pred, long long *p, long long *q,
                                   long long x, long long y)
{
    unsigned long long ret;
    asm volatile("p0 = cmp.eq(%5, #0)\n\t"
                 "%0 = %3\n\t"
                 "{\n\t"
                 "    memd(%1) = %4\n\t"
                 "    if (!p0) %0 = memd(%2)\n\t"
                 "}:mem_noshuf\n"
                 : "=&r"(ret)
                 : "r"(p), "r"(q), "r"(x), "r"(y), "r"(pred)
                 : "p0", "memory");
    return ret;
}

static inline long long pred_ld_sd_pi(int pred, long long *p, long long *q,
                                      long long x, long long y)
{
    long long ret;
    asm volatile("p0 = cmp.eq(%5, #0)\n\t"
                 "%0 = %3\n\t"
                 "r7 = %2\n\t"
                 "{\n\t"
                 "    memd(%1) = %4\n\t"
                 "    if (!p0) %0 = memd(r7++#8)\n\t"
                 "}:mem_noshuf\n"
                 : "=&r"(ret)
                 : "r"(p), "r"(q), "r"(x), "r"(y), "r"(pred)
                 : "r7", "p0", "memory");
    return ret;
}

static inline unsigned int cancel_sw_lb(int pred, int *p, signed char *q, int x)
{
    unsigned int ret;
    asm volatile("p0 = cmp.eq(%4, #0)\n\t"
                 "{\n\t"
                 "    if (!p0) memw(%1) = %3\n\t"
                 "    %0 = memb(%2)\n\t"
                 "}:mem_noshuf\n"
                 : "=r"(ret)
                 : "r"(p), "r"(q), "r"(x), "r"(pred)
                 : "p0", "memory");
    return ret;
}

static inline
unsigned long long cancel_sw_ld(int pred, int *p, long long *q, int x)
{
    long long ret;
    asm volatile("p0 = cmp.eq(%4, #0)\n\t"
                 "{\n\t"
                 "    if (!p0) memw(%1) = %3\n\t"
                 "    %0 = memd(%2)\n\t"
                 "}:mem_noshuf\n"
                 : "=r"(ret)
                 : "r"(p), "r"(q), "r"(x), "r"(pred)
                 : "p0", "memory");
    return ret;
}

typedef union {
    signed long long d[2];
    unsigned long long ud[2];
    signed int w[4];
    unsigned int uw[4];
    signed short h[8];
    unsigned short uh[8];
    signed char b[16];
    unsigned char ub[16];
} Memory;

int err;

#define check32(n, expect) check32_(n, expect, __LINE__)

static void check32_(int n, int expect, int line)
{
    if (n != expect) {
        printf("ERROR: 0x%08x != 0x%08x, line %d\n", n, expect, line);
        err++;
    }
}

#define check64(n, expect) check64_(n, expect, __LINE__)

static void check64_(long long n, long long expect, int line)
{
    if (n != expect) {
        printf("ERROR: 0x%08llx != 0x%08llx, line %d\n", n, expect, line);
        err++;
    }
}

int main()
{
    Memory n;
    unsigned int res32;
    unsigned long long res64;

    /*
     * Store byte combinations
     */
    n.w[0] = ~0;
    res32 = mem_noshuf_sb_lb(&n.b[0], &n.b[0], 0x87);
    check32(res32, 0xffffff87);

    n.w[0] = ~0;
    res32 = mem_noshuf_sb_lub(&n.b[0], &n.ub[0], 0x87);
    check32(res32, 0x00000087);

    n.w[0] = ~0;
    res32 = mem_noshuf_sb_lh(&n.b[0], &n.h[0], 0x87);
    check32(res32, 0xffffff87);

    n.w[0] = ~0;
    res32 = mem_noshuf_sb_luh(&n.b[0], &n.uh[0], 0x87);
    check32(res32, 0x0000ff87);

    n.w[0] = ~0;
    res32 = mem_noshuf_sb_lw(&n.b[0], &n.w[0], 0x87);
    check32(res32, 0xffffff87);

    n.d[0] = ~0LL;
    res64 = mem_noshuf_sb_ld(&n.b[0], &n.d[0], 0x87);
    check64(res64, 0xffffffffffffff87LL);

    /*
     * Store half combinations
     */
    n.w[0] = ~0;
    res32 = mem_noshuf_sh_lb(&n.h[0], &n.b[0], 0x8787);
    check32(res32, 0xffffff87);

    n.w[0] = ~0;
    res32 = mem_noshuf_sh_lub(&n.h[0], &n.ub[1], 0x8f87);
    check32(res32, 0x0000008f);

    n.w[0] = ~0;
    res32 = mem_noshuf_sh_lh(&n.h[0], &n.h[0], 0x8a87);
    check32(res32, 0xffff8a87);

    n.w[0] = ~0;
    res32 = mem_noshuf_sh_luh(&n.h[0], &n.uh[0], 0x8a87);
    check32(res32, 0x8a87);

    n.w[0] = ~0;
    res32 = mem_noshuf_sh_lw(&n.h[1], &n.w[0], 0x8a87);
    check32(res32, 0x8a87ffff);

    n.w[0] = ~0;
    res64 = mem_noshuf_sh_ld(&n.h[1], &n.d[0], 0x8a87);
    check64(res64, 0xffffffff8a87ffffLL);

    /*
     * Store word combinations
     */
    n.w[0] = ~0;
    res32 = mem_noshuf_sw_lb(&n.w[0], &n.b[0], 0x12345687);
    check32(res32, 0xffffff87);

    n.w[0] = ~0;
    res32 = mem_noshuf_sw_lub(&n.w[0], &n.ub[0], 0x12345687);
    check32(res32, 0x00000087);

    n.w[0] = ~0;
    res32 = mem_noshuf_sw_lh(&n.w[0], &n.h[0], 0x1234f678);
    check32(res32, 0xfffff678);

    n.w[0] = ~0;
    res32 = mem_noshuf_sw_luh(&n.w[0], &n.uh[0], 0x12345678);
    check32(res32, 0x00005678);

    n.w[0] = ~0;
    res32 = mem_noshuf_sw_lw(&n.w[0], &n.w[0], 0x12345678);
    check32(res32, 0x12345678);

    n.d[0] = ~0LL;
    res64 = mem_noshuf_sw_ld(&n.w[0], &n.d[0], 0x12345678);
    check64(res64, 0xffffffff12345678LL);

    /*
     * Store double combinations
     */
    n.d[0] = ~0LL;
    res32 = mem_noshuf_sd_lb(&n.d[0], &n.b[1], 0x123456789abcdef0);
    check32(res32, 0xffffffde);

    n.d[0] = ~0LL;
    res32 = mem_noshuf_sd_lub(&n.d[0], &n.ub[1], 0x123456789abcdef0);
    check32(res32, 0x000000de);

    n.d[0] = ~0LL;
    res32 = mem_noshuf_sd_lh(&n.d[0], &n.h[1], 0x123456789abcdef0);
    check32(res32, 0xffff9abc);

    n.d[0] = ~0LL;
    res32 = mem_noshuf_sd_luh(&n.d[0], &n.uh[1], 0x123456789abcdef0);
    check32(res32, 0x00009abc);

    n.d[0] = ~0LL;
    res32 = mem_noshuf_sd_lw(&n.d[0], &n.w[1], 0x123456789abcdef0);
    check32(res32, 0x12345678);

    n.d[0] = ~0LL;
    res64 = mem_noshuf_sd_ld(&n.d[0], &n.d[0], 0x123456789abcdef0);
    check64(res64, 0x123456789abcdef0LL);

    /*
     * Predicated word stores
     */
    n.w[0] = ~0;
    res32 = cancel_sw_lb(0, &n.w[0], &n.b[0], 0x12345678);
    check32(res32, 0xffffffff);

    n.w[0] = ~0;
    res32 = cancel_sw_lb(1, &n.w[0], &n.b[0], 0x12345687);
    check32(res32, 0xffffff87);

    /*
     * Predicated double stores
     */
    n.d[0] = ~0LL;
    res64 = cancel_sw_ld(0, &n.w[0], &n.d[0], 0x12345678);
    check64(res64, 0xffffffffffffffffLL);

    n.d[0] = ~0LL;
    res64 = cancel_sw_ld(1, &n.w[0], &n.d[0], 0x12345678);
    check64(res64, 0xffffffff12345678LL);

    n.d[0] = ~0LL;
    res64 = cancel_sw_ld(0, &n.w[1], &n.d[0], 0x12345678);
    check64(res64, 0xffffffffffffffffLL);

    n.d[0] = ~0LL;
    res64 = cancel_sw_ld(1, &n.w[1], &n.d[0], 0x12345678);
    check64(res64, 0x12345678ffffffffLL);

    /*
     * No overlap tests
     */
    n.w[0] = ~0;
    res32 = mem_noshuf_sb_lb(&n.b[1], &n.b[0], 0x87);
    check32(res32, 0xffffffff);

    n.w[0] = ~0;
    res32 = mem_noshuf_sb_lb(&n.b[0], &n.b[1], 0x87);
    check32(res32, 0xffffffff);

    n.w[0] = ~0;
    res32 = mem_noshuf_sh_lh(&n.h[1], &n.h[0], 0x8787);
    check32(res32, 0xffffffff);

    n.w[0] = ~0;
    res32 = mem_noshuf_sh_lh(&n.h[0], &n.h[1], 0x8787);
    check32(res32, 0xffffffff);

    n.d[0] = ~0LL;
    res32 = mem_noshuf_sw_lw(&n.w[0], &n.w[1], 0x12345678);
    check32(res32, 0xffffffff);

    n.d[0] = ~0LL;
    res32 = mem_noshuf_sw_lw(&n.w[1], &n.w[0], 0x12345678);
    check32(res32, 0xffffffff);

    n.d[0] = ~0LL;
    n.d[1] = ~0LL;
    res64 = mem_noshuf_sd_ld(&n.d[1], &n.d[0], 0x123456789abcdef0LL);
    check64(res64, 0xffffffffffffffffLL);

    n.d[0] = ~0LL;
    n.d[1] = ~0LL;
    res64 = mem_noshuf_sd_ld(&n.d[0], &n.d[1], 0x123456789abcdef0LL);
    check64(res64, 0xffffffffffffffffLL);

    n.w[0] = ~0;
    res32 = pred_lw_sw(0, &n.w[0], &n.w[0], 0x12345678, 0xc0ffeeda);
    check32(res32, 0x12345678);
    check32(n.w[0], 0xc0ffeeda);

    n.w[0] = ~0;
    res32 = pred_lw_sw(1, &n.w[0], &n.w[0], 0x12345678, 0xc0ffeeda);
    check32(res32, 0xc0ffeeda);
    check32(n.w[0], 0xc0ffeeda);

    n.w[0] = ~0;
    res32 = pred_lw_sw_pi(0, &n.w[0], &n.w[0], 0x12345678, 0xc0ffeeda);
    check32(res32, 0x12345678);
    check32(n.w[0], 0xc0ffeeda);

    n.w[0] = ~0;
    res32 = pred_lw_sw_pi(1, &n.w[0], &n.w[0], 0x12345678, 0xc0ffeeda);
    check32(res32, 0xc0ffeeda);
    check32(n.w[0], 0xc0ffeeda);

    n.d[0] = ~0LL;
    res64 = pred_ld_sd(0, &n.d[0], &n.d[0],
                       0x1234567812345678LL, 0xc0ffeedac0ffeedaLL);
    check64(res64, 0x1234567812345678LL);
    check64(n.d[0], 0xc0ffeedac0ffeedaLL);

    n.d[0] = ~0LL;
    res64 = pred_ld_sd(1, &n.d[0], &n.d[0],
                       0x1234567812345678LL, 0xc0ffeedac0ffeedaLL);
    check64(res64, 0xc0ffeedac0ffeedaLL);
    check64(n.d[0], 0xc0ffeedac0ffeedaLL);

    n.d[0] = ~0LL;
    res64 = pred_ld_sd_pi(0, &n.d[0], &n.d[0],
                          0x1234567812345678LL, 0xc0ffeedac0ffeedaLL);
    check64(res64, 0x1234567812345678LL);
    check64(n.d[0], 0xc0ffeedac0ffeedaLL);

    n.d[0] = ~0LL;
    res64 = pred_ld_sd_pi(1, &n.d[0], &n.d[0],
                          0x1234567812345678LL, 0xc0ffeedac0ffeedaLL);
    check64(res64, 0xc0ffeedac0ffeedaLL);
    check64(n.d[0], 0xc0ffeedac0ffeedaLL);

    puts(err ? "FAIL" : "PASS");
    return err;
}