/*
 * QEMU Plugin Core code
 *
 * This is the core code that deals with injecting instrumentation into the code
 *
 * Copyright (C) 2017, Emilio G. Cota <cota@braap.org>
 * Copyright (C) 2019, Linaro
 *
 * License: GNU GPL, version 2 or later.
 *   See the COPYING file in the top-level directory.
 *
 * SPDX-License-Identifier: GPL-2.0-or-later
 */
#include "qemu/osdep.h"
#include "qemu/lockable.h"
#include "qemu/option.h"
#include "qemu/plugin.h"
#include "plugins/qemu-plugin.h"
#include "qemu/queue.h"
#include "qemu/rcu_queue.h"
#include "qemu/rcu.h"
#include "exec/tb-flush.h"
#include "tcg/tcg-op-common.h"
#include "plugin.h"

struct qemu_plugin_cb {
    struct qemu_plugin_ctx *ctx;
    union qemu_plugin_cb_sig f;
    void *udata;
    QLIST_ENTRY(qemu_plugin_cb) entry;
};

struct qemu_plugin_state plugin;

struct qemu_plugin_ctx *plugin_id_to_ctx_locked(qemu_plugin_id_t id)
{
    struct qemu_plugin_ctx *ctx;
    qemu_plugin_id_t *id_p;

    id_p = g_hash_table_lookup(plugin.id_ht, &id);
    ctx = container_of(id_p, struct qemu_plugin_ctx, id);
    if (ctx == NULL) {
        error_report("plugin: invalid plugin id %" PRIu64, id);
        abort();
    }
    return ctx;
}

static void plugin_cpu_update__async(CPUState *cpu, run_on_cpu_data data)
{
    bitmap_copy(cpu->plugin_state->event_mask,
                &data.host_ulong, QEMU_PLUGIN_EV_MAX);
    tcg_flush_jmp_cache(cpu);
}

static void plugin_cpu_update__locked(gpointer k, gpointer v, gpointer udata)
{
    CPUState *cpu = container_of(k, CPUState, cpu_index);
    run_on_cpu_data mask = RUN_ON_CPU_HOST_ULONG(*plugin.mask);

    async_run_on_cpu(cpu, plugin_cpu_update__async, mask);
}

void plugin_unregister_cb__locked(struct qemu_plugin_ctx *ctx,
                                  enum qemu_plugin_event ev)
{
    struct qemu_plugin_cb *cb = ctx->callbacks[ev];

    if (cb == NULL) {
        return;
    }
    QLIST_REMOVE_RCU(cb, entry);
    g_free(cb);
    ctx->callbacks[ev] = NULL;
    if (QLIST_EMPTY_RCU(&plugin.cb_lists[ev])) {
        clear_bit(ev, plugin.mask);
        g_hash_table_foreach(plugin.cpu_ht, plugin_cpu_update__locked, NULL);
    }
}

/*
 * Disable CFI checks.
 * The callback function has been loaded from an external library so we do not
 * have type information
 */
QEMU_DISABLE_CFI
static void plugin_vcpu_cb__udata(CPUState *cpu, enum qemu_plugin_event ev)
{
    struct qemu_plugin_cb *cb, *next;

    switch (ev) {
    case QEMU_PLUGIN_EV_VCPU_INIT:
    case QEMU_PLUGIN_EV_VCPU_IDLE:
    case QEMU_PLUGIN_EV_VCPU_RESUME:
    case QEMU_PLUGIN_EV_VCPU_EXIT:
        QLIST_FOREACH_SAFE_RCU(cb, &plugin.cb_lists[ev], entry, next) {
            qemu_plugin_vcpu_udata_cb_t func = cb->f.vcpu_udata;
            func(cpu->cpu_index, cb->udata);
        }
        break;
    default:
        g_assert_not_reached();
    }
}

/*
 * Disable CFI checks.
 * The callback function has been loaded from an external library so we do not
 * have type information
 */
QEMU_DISABLE_CFI
static void plugin_vcpu_cb__discon(CPUState *cpu,
                                   enum qemu_plugin_event ev,
                                   enum qemu_plugin_discon_type type,
                                   uint64_t from)
{
    struct qemu_plugin_cb *cb, *next;
    uint64_t to = cpu->cc->get_pc(cpu);

    qemu_plugin_set_cb_flags(cpu, QEMU_PLUGIN_CB_RW_REGS_PC);
    if (cpu->cpu_index < plugin.num_vcpus) {
        /* iterate safely; plugins might uninstall themselves at any time */
        QLIST_FOREACH_SAFE_RCU(cb, &plugin.cb_lists[ev], entry, next) {
            qemu_plugin_vcpu_discon_cb_t func = cb->f.vcpu_discon;
            func(cpu->cpu_index, type, from, to, cb->udata);
        }
    }
    qemu_plugin_set_cb_flags(cpu, QEMU_PLUGIN_CB_NO_REGS);
}

/*
 * Disable CFI checks.
 * The callback function has been loaded from an external library so we do not
 * have type information
 */
QEMU_DISABLE_CFI
static void plugin_cb__udata(enum qemu_plugin_event ev)
{
    struct qemu_plugin_cb *cb, *next;

    switch (ev) {
    case QEMU_PLUGIN_EV_ATEXIT:
    case QEMU_PLUGIN_EV_FLUSH:
        QLIST_FOREACH_SAFE_RCU(cb, &plugin.cb_lists[ev], entry, next) {
            qemu_plugin_udata_cb_t func = cb->f.udata;

            func(cb->udata);
        }
        break;
    default:
        g_assert_not_reached();
    }
}

static void
do_plugin_register_cb(qemu_plugin_id_t id, enum qemu_plugin_event ev,
                      void *func, void *udata)
{
    struct qemu_plugin_ctx *ctx;

    QEMU_LOCK_GUARD(&plugin.lock);
    ctx = plugin_id_to_ctx_locked(id);
    /* if the plugin is on its way out, ignore this request */
    if (unlikely(ctx->uninstalling)) {
        return;
    }
    if (func) {
        struct qemu_plugin_cb *cb = ctx->callbacks[ev];

        if (cb) {
            cb->f.generic = func;
            cb->udata = udata;
        } else {
            cb = g_new(struct qemu_plugin_cb, 1);
            cb->ctx = ctx;
            cb->f.generic = func;
            cb->udata = udata;
            ctx->callbacks[ev] = cb;
            QLIST_INSERT_HEAD_RCU(&plugin.cb_lists[ev], cb, entry);
            if (!test_bit(ev, plugin.mask)) {
                set_bit(ev, plugin.mask);
                g_hash_table_foreach(plugin.cpu_ht, plugin_cpu_update__locked,
                                     NULL);
            }
        }
    } else {
        plugin_unregister_cb__locked(ctx, ev);
    }
}

void plugin_register_cb(qemu_plugin_id_t id, enum qemu_plugin_event ev,
                        void *func)
{
    do_plugin_register_cb(id, ev, func, NULL);
}

void
plugin_register_cb_udata(qemu_plugin_id_t id, enum qemu_plugin_event ev,
                         void *func, void *udata)
{
    do_plugin_register_cb(id, ev, func, udata);
}

CPUPluginState *qemu_plugin_create_vcpu_state(void)
{
    return g_new0(CPUPluginState, 1);
}

static void plugin_grow_scoreboards__locked(CPUState *cpu)
{
    size_t scoreboard_size = plugin.scoreboard_alloc_size;
    bool need_realloc = false;

    if (cpu->cpu_index < scoreboard_size) {
        return;
    }

    while (cpu->cpu_index >= scoreboard_size) {
        scoreboard_size *= 2;
        need_realloc = true;
    }

    if (!need_realloc) {
        return;
    }

    if (QLIST_EMPTY(&plugin.scoreboards)) {
        /* just update size for future scoreboards */
        plugin.scoreboard_alloc_size = scoreboard_size;
        return;
    }

    /*
     * A scoreboard creation/deletion might be in progress. If a new vcpu is
     * initialized at the same time, we are safe, as the new
     * plugin.scoreboard_alloc_size was not yet written.
     */
    qemu_rec_mutex_unlock(&plugin.lock);

    /* cpus must be stopped, as tb might still use an existing scoreboard. */
    start_exclusive();
    /* re-acquire lock */
    qemu_rec_mutex_lock(&plugin.lock);
    /* in case another vcpu is created between unlock and exclusive section. */
    if (scoreboard_size > plugin.scoreboard_alloc_size) {
        struct qemu_plugin_scoreboard *score;
        QLIST_FOREACH(score, &plugin.scoreboards, entry) {
            g_array_set_size(score->data, scoreboard_size);
        }
        plugin.scoreboard_alloc_size = scoreboard_size;
        /* force all tb to be flushed, as scoreboard pointers were changed. */
        tb_flush__exclusive_or_serial();
    }
    end_exclusive();
}

static void qemu_plugin_vcpu_init__async(CPUState *cpu, run_on_cpu_data unused)
{
    bool success;

    assert(cpu->cpu_index != UNASSIGNED_CPU_INDEX);
    qemu_rec_mutex_lock(&plugin.lock);
    plugin.num_vcpus = MAX(plugin.num_vcpus, cpu->cpu_index + 1);
    plugin_cpu_update__locked(&cpu->cpu_index, NULL, NULL);
    success = g_hash_table_insert(plugin.cpu_ht, &cpu->cpu_index,
                                  &cpu->cpu_index);
    g_assert(success);
    plugin_grow_scoreboards__locked(cpu);
    qemu_rec_mutex_unlock(&plugin.lock);

    qemu_plugin_set_cb_flags(cpu, QEMU_PLUGIN_CB_RW_REGS);
    plugin_vcpu_cb__udata(cpu, QEMU_PLUGIN_EV_VCPU_INIT);
    qemu_plugin_set_cb_flags(cpu, QEMU_PLUGIN_CB_NO_REGS);
}

void qemu_plugin_vcpu_init_hook(CPUState *cpu)
{
    /* Plugin initialization must wait until the cpu start executing code */
    async_run_on_cpu(cpu, qemu_plugin_vcpu_init__async, RUN_ON_CPU_NULL);
}

void qemu_plugin_vcpu_exit_hook(CPUState *cpu)
{
    bool success;

    qemu_plugin_set_cb_flags(cpu, QEMU_PLUGIN_CB_RW_REGS);
    plugin_vcpu_cb__udata(cpu, QEMU_PLUGIN_EV_VCPU_EXIT);
    qemu_plugin_set_cb_flags(cpu, QEMU_PLUGIN_CB_NO_REGS);

    assert(cpu->cpu_index != UNASSIGNED_CPU_INDEX);
    qemu_rec_mutex_lock(&plugin.lock);
    success = g_hash_table_remove(plugin.cpu_ht, &cpu->cpu_index);
    g_assert(success);
    qemu_rec_mutex_unlock(&plugin.lock);
}

struct plugin_for_each_args {
    struct qemu_plugin_ctx *ctx;
    qemu_plugin_vcpu_udata_cb_t cb;
    void *userdata;
};

static void plugin_vcpu_for_each(gpointer k, gpointer v, gpointer udata)
{
    struct plugin_for_each_args *args = udata;
    int cpu_index = *(int *)k;
    args->cb(cpu_index, args->userdata);
}

void qemu_plugin_vcpu_for_each(qemu_plugin_id_t id,
                               qemu_plugin_vcpu_udata_cb_t cb,
                               void *userdata)
{
    struct plugin_for_each_args args;

    if (cb == NULL) {
        return;
    }
    qemu_rec_mutex_lock(&plugin.lock);
    args.ctx = plugin_id_to_ctx_locked(id);
    args.cb = cb;
    args.userdata = userdata;
    g_hash_table_foreach(plugin.cpu_ht, plugin_vcpu_for_each, &args);
    qemu_rec_mutex_unlock(&plugin.lock);
}

/* Allocate and return a callback record */
static struct qemu_plugin_dyn_cb *plugin_get_dyn_cb(GArray **arr)
{
    GArray *cbs = *arr;

    if (!cbs) {
        cbs = g_array_sized_new(false, true,
                                sizeof(struct qemu_plugin_dyn_cb), 1);
        *arr = cbs;
    }

    g_array_set_size(cbs, cbs->len + 1);
    return &g_array_index(cbs, struct qemu_plugin_dyn_cb, cbs->len - 1);
}

static enum plugin_dyn_cb_type op_to_cb_type(enum qemu_plugin_op op)
{
    switch (op) {
    case QEMU_PLUGIN_INLINE_ADD_U64:
        return PLUGIN_CB_INLINE_ADD_U64;
    case QEMU_PLUGIN_INLINE_STORE_U64:
        return PLUGIN_CB_INLINE_STORE_U64;
    default:
        g_assert_not_reached();
    }
}

void plugin_register_inline_op_on_entry(GArray **arr,
                                        enum qemu_plugin_mem_rw rw,
                                        enum qemu_plugin_op op,
                                        qemu_plugin_u64 entry,
                                        uint64_t imm)
{
    struct qemu_plugin_dyn_cb *dyn_cb;

    struct qemu_plugin_inline_cb inline_cb = { .rw = rw,
                                               .entry = entry,
                                               .imm = imm };
    dyn_cb = plugin_get_dyn_cb(arr);
    dyn_cb->type = op_to_cb_type(op);
    dyn_cb->inline_insn = inline_cb;
}

void plugin_register_dyn_cb__udata(GArray **arr,
                                   qemu_plugin_vcpu_udata_cb_t cb,
                                   enum qemu_plugin_cb_flags flags,
                                   void *udata)
{
    static TCGHelperInfo info[4] = {
        [QEMU_PLUGIN_CB_NO_REGS].flags = TCG_CALL_NO_RWG,
        [QEMU_PLUGIN_CB_R_REGS].flags = TCG_CALL_NO_WG,
        [QEMU_PLUGIN_CB_RW_REGS].flags = 0,
        [QEMU_PLUGIN_CB_RW_REGS_PC].flags = 0,
        /*
         * Match qemu_plugin_vcpu_udata_cb_t:
         *   void (*)(uint32_t, void *)
         */
        [0 ... 3].typemask = (dh_typemask(void, 0) |
                              dh_typemask(i32, 1) |
                              dh_typemask(ptr, 2))
    };
    assert((unsigned)flags < ARRAY_SIZE(info));

    struct qemu_plugin_dyn_cb *dyn_cb = plugin_get_dyn_cb(arr);
    struct qemu_plugin_regular_cb regular_cb = { .f.vcpu_udata = cb,
                                                 .userp = udata,
                                                 .info = &info[flags] };
    dyn_cb->type = PLUGIN_CB_REGULAR;
    dyn_cb->regular = regular_cb;
}

void plugin_register_dyn_cond_cb__udata(GArray **arr,
                                        qemu_plugin_vcpu_udata_cb_t cb,
                                        enum qemu_plugin_cb_flags flags,
                                        enum qemu_plugin_cond cond,
                                        qemu_plugin_u64 entry,
                                        uint64_t imm,
                                        void *udata)
{
    static TCGHelperInfo info[4] = {
        [QEMU_PLUGIN_CB_NO_REGS].flags = TCG_CALL_NO_RWG,
        [QEMU_PLUGIN_CB_R_REGS].flags = TCG_CALL_NO_WG,
        [QEMU_PLUGIN_CB_RW_REGS].flags = 0,
        [QEMU_PLUGIN_CB_RW_REGS_PC].flags = 0,
        /*
         * Match qemu_plugin_vcpu_udata_cb_t:
         *   void (*)(uint32_t, void *)
         */
        [0 ... 3].typemask = (dh_typemask(void, 0) |
                              dh_typemask(i32, 1) |
                              dh_typemask(ptr, 2))
    };
    assert((unsigned)flags < ARRAY_SIZE(info));

    struct qemu_plugin_dyn_cb *dyn_cb = plugin_get_dyn_cb(arr);
    struct qemu_plugin_conditional_cb cond_cb = { .userp = udata,
                                                  .f.vcpu_udata = cb,
                                                  .cond = cond,
                                                  .entry = entry,
                                                  .imm = imm,
                                                  .info = &info[flags] };
    dyn_cb->type = PLUGIN_CB_COND;
    dyn_cb->cond = cond_cb;
}

void plugin_register_vcpu_mem_cb(GArray **arr,
                                 void *cb,
                                 enum qemu_plugin_cb_flags flags,
                                 enum qemu_plugin_mem_rw rw,
                                 void *udata)
{
    /*
     * Expect that the underlying type for enum qemu_plugin_meminfo_t
     * is either int32_t or uint32_t, aka int or unsigned int.
     */
    QEMU_BUILD_BUG_ON(
        !__builtin_types_compatible_p(qemu_plugin_meminfo_t, uint32_t) &&
        !__builtin_types_compatible_p(qemu_plugin_meminfo_t, int32_t));

    static TCGHelperInfo info[4] = {
        [QEMU_PLUGIN_CB_NO_REGS].flags = TCG_CALL_NO_RWG,
        [QEMU_PLUGIN_CB_R_REGS].flags = TCG_CALL_NO_WG,
        [QEMU_PLUGIN_CB_RW_REGS].flags = 0,
        [QEMU_PLUGIN_CB_RW_REGS_PC].flags = 0,
        /*
         * Match qemu_plugin_vcpu_mem_cb_t:
         *   void (*)(uint32_t, qemu_plugin_meminfo_t, uint64_t, void *)
         */
        [0 ... 3].typemask =
            (dh_typemask(void, 0) |
             dh_typemask(i32, 1) |
             (__builtin_types_compatible_p(qemu_plugin_meminfo_t, uint32_t)
              ? dh_typemask(i32, 2) : dh_typemask(s32, 2)) |
             dh_typemask(i64, 3) |
             dh_typemask(ptr, 4))
    };
    assert((unsigned)flags < ARRAY_SIZE(info));

    struct qemu_plugin_dyn_cb *dyn_cb = plugin_get_dyn_cb(arr);
    struct qemu_plugin_regular_cb regular_cb = { .userp = udata,
                                                 .rw = rw,
                                                 .f.vcpu_mem = cb,
                                                 .info = &info[flags] };
    dyn_cb->type = PLUGIN_CB_MEM_REGULAR;
    dyn_cb->regular = regular_cb;
}

/*
 * Disable CFI checks.
 * The callback function has been loaded from an external library so we do not
 * have type information
 */
QEMU_DISABLE_CFI
void qemu_plugin_tb_trans_cb(CPUState *cpu, struct qemu_plugin_tb *tb)
{
    struct qemu_plugin_cb *cb, *next;
    enum qemu_plugin_event ev = QEMU_PLUGIN_EV_VCPU_TB_TRANS;

    /* no plugin_state->event_mask check here; caller should have checked */

    QLIST_FOREACH_SAFE_RCU(cb, &plugin.cb_lists[ev], entry, next) {
        qemu_plugin_vcpu_tb_trans_cb_t func = cb->f.vcpu_tb_trans;

        qemu_plugin_set_cb_flags(cpu, QEMU_PLUGIN_CB_RW_REGS);
        func(tb, cb->udata);
        qemu_plugin_set_cb_flags(cpu, QEMU_PLUGIN_CB_NO_REGS);
    }
}

static void clamp_syscall_arguments(uint64_t *a1, uint64_t *a2, uint64_t *a3,
                                    uint64_t *a4, uint64_t *a5, uint64_t *a6,
                                    uint64_t *a7, uint64_t *a8)
{
    if (target_long_bits() == 32) {
        const uint64_t mask = UINT32_MAX;
        *a1 &= mask;
        *a2 &= mask;
        *a3 &= mask;
        *a4 &= mask;
        *a5 &= mask;
        *a6 &= mask;
        *a7 &= mask;
        *a8 &= mask;
    }
}

/*
 * Disable CFI checks.
 * The callback function has been loaded from an external library so we do not
 * have type information
 */
QEMU_DISABLE_CFI
void
qemu_plugin_vcpu_syscall(CPUState *cpu, int64_t num, uint64_t a1, uint64_t a2,
                         uint64_t a3, uint64_t a4, uint64_t a5,
                         uint64_t a6, uint64_t a7, uint64_t a8)
{
    struct qemu_plugin_cb *cb, *next;
    enum qemu_plugin_event ev = QEMU_PLUGIN_EV_VCPU_SYSCALL;

    if (!test_bit(ev, cpu->plugin_state->event_mask)) {
        return;
    }

    clamp_syscall_arguments(&a1, &a2, &a3, &a4, &a5, &a6, &a7, &a8);

    QLIST_FOREACH_SAFE_RCU(cb, &plugin.cb_lists[ev], entry, next) {
        qemu_plugin_vcpu_syscall_cb_t func = cb->f.vcpu_syscall;

        qemu_plugin_set_cb_flags(cpu, QEMU_PLUGIN_CB_RW_REGS_PC);
        func(cpu->cpu_index, num, a1, a2, a3, a4, a5, a6, a7, a8, cb->udata);
        qemu_plugin_set_cb_flags(cpu, QEMU_PLUGIN_CB_NO_REGS);
    }
}

/*
 * Disable CFI checks.
 * The callback function has been loaded from an external library so we do not
 * have type information
 */
QEMU_DISABLE_CFI
void qemu_plugin_vcpu_syscall_ret(CPUState *cpu, int64_t num, int64_t ret)
{
    struct qemu_plugin_cb *cb, *next;
    enum qemu_plugin_event ev = QEMU_PLUGIN_EV_VCPU_SYSCALL_RET;

    if (!test_bit(ev, cpu->plugin_state->event_mask)) {
        return;
    }

    QLIST_FOREACH_SAFE_RCU(cb, &plugin.cb_lists[ev], entry, next) {
        qemu_plugin_vcpu_syscall_ret_cb_t func = cb->f.vcpu_syscall_ret;

        qemu_plugin_set_cb_flags(cpu, QEMU_PLUGIN_CB_RW_REGS_PC);
        func(cpu->cpu_index, num, ret, cb->udata);
        qemu_plugin_set_cb_flags(cpu, QEMU_PLUGIN_CB_NO_REGS);
    }
}

/*
 * Disable CFI checks.
 * The callback function has been loaded from an external library so we do not
 * have type information
 */
QEMU_DISABLE_CFI
bool
qemu_plugin_vcpu_syscall_filter(CPUState *cpu, int64_t num, uint64_t a1,
                                uint64_t a2, uint64_t a3, uint64_t a4,
                                uint64_t a5, uint64_t a6, uint64_t a7,
                                uint64_t a8, int64_t *sysret)
{
    struct qemu_plugin_cb *cb, *next;
    enum qemu_plugin_event ev = QEMU_PLUGIN_EV_VCPU_SYSCALL_FILTER;
    bool filtered = false;

    if (!test_bit(ev, cpu->plugin_state->event_mask)) {
        return false;
    }

    clamp_syscall_arguments(&a1, &a2, &a3, &a4, &a5, &a6, &a7, &a8);

    qemu_plugin_set_cb_flags(cpu, QEMU_PLUGIN_CB_RW_REGS_PC);

    QLIST_FOREACH_SAFE_RCU(cb, &plugin.cb_lists[ev], entry, next) {
        qemu_plugin_vcpu_syscall_filter_cb_t func = cb->f.vcpu_syscall_filter;

        if (func(cpu->cpu_index, num, a1, a2, a3, a4,
                 a5, a6, a7, a8, sysret, cb->udata)) {
            filtered = true;
            break;
        }
    }

    qemu_plugin_set_cb_flags(cpu, QEMU_PLUGIN_CB_NO_REGS);

    return filtered;
}

void qemu_plugin_vcpu_idle_cb(CPUState *cpu)
{
    /* idle and resume cb may be called before init, ignore in this case */
    if (cpu->cpu_index < plugin.num_vcpus) {
        qemu_plugin_set_cb_flags(cpu, QEMU_PLUGIN_CB_RW_REGS_PC);
        plugin_vcpu_cb__udata(cpu, QEMU_PLUGIN_EV_VCPU_IDLE);
        qemu_plugin_set_cb_flags(cpu, QEMU_PLUGIN_CB_NO_REGS);
    }
}

void qemu_plugin_vcpu_resume_cb(CPUState *cpu)
{
    if (cpu->cpu_index < plugin.num_vcpus) {
        qemu_plugin_set_cb_flags(cpu, QEMU_PLUGIN_CB_RW_REGS_PC);
        plugin_vcpu_cb__udata(cpu, QEMU_PLUGIN_EV_VCPU_RESUME);
        qemu_plugin_set_cb_flags(cpu, QEMU_PLUGIN_CB_NO_REGS);
    }
}

void qemu_plugin_vcpu_interrupt_cb(CPUState *cpu, uint64_t from)
{
    plugin_vcpu_cb__discon(cpu, QEMU_PLUGIN_EV_VCPU_INTERRUPT,
                           QEMU_PLUGIN_DISCON_INTERRUPT, from);
}

void qemu_plugin_vcpu_exception_cb(CPUState *cpu, uint64_t from)
{
    plugin_vcpu_cb__discon(cpu, QEMU_PLUGIN_EV_VCPU_EXCEPTION,
                           QEMU_PLUGIN_DISCON_EXCEPTION, from);
}

void qemu_plugin_vcpu_hostcall_cb(CPUState *cpu, uint64_t from)
{
    plugin_vcpu_cb__discon(cpu, QEMU_PLUGIN_EV_VCPU_HOSTCALL,
                           QEMU_PLUGIN_DISCON_HOSTCALL, from);
}

void qemu_plugin_register_vcpu_idle_cb(qemu_plugin_id_t id,
                                       qemu_plugin_vcpu_udata_cb_t cb,
                                       void *userdata)
{
    plugin_register_cb_udata(id, QEMU_PLUGIN_EV_VCPU_IDLE, cb, userdata);
}

void qemu_plugin_register_vcpu_resume_cb(qemu_plugin_id_t id,
                                         qemu_plugin_vcpu_udata_cb_t cb,
                                         void *userdata)
{
    plugin_register_cb_udata(id, QEMU_PLUGIN_EV_VCPU_RESUME, cb, userdata);
}

void qemu_plugin_register_vcpu_discon_cb(qemu_plugin_id_t id,
                                         enum qemu_plugin_discon_type type,
                                         qemu_plugin_vcpu_discon_cb_t cb,
                                         void *userdata)
{
    if (type & QEMU_PLUGIN_DISCON_INTERRUPT) {
        plugin_register_cb_udata(id, QEMU_PLUGIN_EV_VCPU_INTERRUPT, cb, userdata);
    }
    if (type & QEMU_PLUGIN_DISCON_EXCEPTION) {
        plugin_register_cb_udata(id, QEMU_PLUGIN_EV_VCPU_EXCEPTION, cb, userdata);
    }
    if (type & QEMU_PLUGIN_DISCON_HOSTCALL) {
        plugin_register_cb_udata(id, QEMU_PLUGIN_EV_VCPU_HOSTCALL, cb, userdata);
    }
}

void qemu_plugin_register_flush_cb(qemu_plugin_id_t id,
                                   qemu_plugin_udata_cb_t cb,
                                   void *userdata)
{
    plugin_register_cb_udata(id, QEMU_PLUGIN_EV_FLUSH, cb, userdata);
}

static bool free_dyn_cb_arr(void *p, uint32_t h, void *userp)
{
    g_array_free((GArray *) p, true);
    return true;
}

void qemu_plugin_flush_cb(void)
{
    qht_iter_remove(&plugin.dyn_cb_arr_ht, free_dyn_cb_arr, NULL);
    qht_reset(&plugin.dyn_cb_arr_ht);

    plugin_cb__udata(QEMU_PLUGIN_EV_FLUSH);
}

void exec_inline_op(enum plugin_dyn_cb_type type,
                    struct qemu_plugin_inline_cb *cb,
                    int cpu_index)
{
    char *ptr = cb->entry.score->data->data;
    size_t elem_size = g_array_get_element_size(
        cb->entry.score->data);
    size_t offset = cb->entry.offset;
    uint64_t *val = (uint64_t *)(ptr + offset + cpu_index * elem_size);

    switch (type) {
    case PLUGIN_CB_INLINE_ADD_U64:
        *val += cb->imm;
        break;
    case PLUGIN_CB_INLINE_STORE_U64:
        *val = cb->imm;
        break;
    default:
        g_assert_not_reached();
    }
}

QEMU_DISABLE_CFI
void qemu_plugin_vcpu_mem_cb(CPUState *cpu, uint64_t vaddr,
                             uint64_t value_low,
                             uint64_t value_high,
                             MemOpIdx oi, enum qemu_plugin_mem_rw rw)
{
    GArray *arr = cpu->neg.plugin_mem_cbs;
    size_t i;

    if (arr == NULL) {
        return;
    }

    cpu->neg.plugin_mem_value_low = value_low;
    cpu->neg.plugin_mem_value_high = value_high;

    for (i = 0; i < arr->len; i++) {
        struct qemu_plugin_dyn_cb *cb =
            &g_array_index(arr, struct qemu_plugin_dyn_cb, i);

        switch (cb->type) {
        case PLUGIN_CB_MEM_REGULAR:
            if (rw & cb->regular.rw) {
                qemu_plugin_set_cb_flags(cpu,
                    tcg_call_to_qemu_plugin_cb_flags(cb->regular.info->flags));

                cb->regular.f.vcpu_mem(cpu->cpu_index,
                                       make_plugin_meminfo(oi, rw),
                                       vaddr, cb->regular.userp);
                qemu_plugin_set_cb_flags(cpu, QEMU_PLUGIN_CB_NO_REGS);
            }
            break;
        case PLUGIN_CB_INLINE_ADD_U64:
        case PLUGIN_CB_INLINE_STORE_U64:
            if (rw & cb->inline_insn.rw) {
                exec_inline_op(cb->type, &cb->inline_insn, cpu->cpu_index);
            }
            break;
        default:
            g_assert_not_reached();
        }
    }
}

void qemu_plugin_atexit_cb(void)
{
    plugin_cb__udata(QEMU_PLUGIN_EV_ATEXIT);
}

void qemu_plugin_register_atexit_cb(qemu_plugin_id_t id,
                                    qemu_plugin_udata_cb_t cb,
                                    void *udata)
{
    plugin_register_cb_udata(id, QEMU_PLUGIN_EV_ATEXIT, cb, udata);
}

/*
 * Handle exit from linux-user. Unlike the normal atexit() mechanism
 * we need to handle the clean-up manually as it's possible threads
 * are still running. We need to remove all callbacks from code
 * generation, flush the current translations and then we can safely
 * trigger the exit callbacks.
 */

void qemu_plugin_user_exit(void)
{
    enum qemu_plugin_event ev;
    CPUState *cpu;

    /*
     * Locking order: we must acquire locks in an order that is consistent
     * with the one in fork_start(). That is:
     * - start_exclusive(), which acquires qemu_cpu_list_lock,
     *   must be called before acquiring plugin.lock.
     */
    start_exclusive();

    qemu_rec_mutex_lock(&plugin.lock);
    /* un-register all callbacks except the final AT_EXIT one */
    for (ev = 0; ev < QEMU_PLUGIN_EV_MAX; ev++) {
        if (ev != QEMU_PLUGIN_EV_ATEXIT) {
            struct qemu_plugin_cb *cb, *next;

            QLIST_FOREACH_SAFE_RCU(cb, &plugin.cb_lists[ev], entry, next) {
                plugin_unregister_cb__locked(cb->ctx, ev);
            }
        }
    }
    CPU_FOREACH(cpu) {
        qemu_plugin_disable_mem_helpers(cpu);
    }
    qemu_rec_mutex_unlock(&plugin.lock);

    tb_flush__exclusive_or_serial();
    end_exclusive();

    /* now it's safe to handle the exit case */
    qemu_plugin_atexit_cb();
}

/*
 * Helpers for *-user to ensure locks are sane across fork() events.
 */

void qemu_plugin_user_prefork_lock(void)
{
    qemu_rec_mutex_lock(&plugin.lock);
}

void qemu_plugin_user_postfork(bool is_child)
{
    if (is_child) {
        /* should we just reset via plugin_init? */
        qemu_rec_mutex_init(&plugin.lock);
    } else {
        qemu_rec_mutex_unlock(&plugin.lock);
    }
}

static bool plugin_dyn_cb_arr_cmp(const void *ap, const void *bp)
{
    return ap == bp;
}

static void __attribute__((__constructor__)) plugin_init(void)
{
    int i;

    for (i = 0; i < QEMU_PLUGIN_EV_MAX; i++) {
        QLIST_INIT(&plugin.cb_lists[i]);
    }
    qemu_rec_mutex_init(&plugin.lock);
    plugin.id_ht = g_hash_table_new(g_int64_hash, g_int64_equal);
    plugin.cpu_ht = g_hash_table_new(g_int_hash, g_int_equal);
    QLIST_INIT(&plugin.scoreboards);
    plugin.scoreboard_alloc_size = 16; /* avoid frequent reallocation */
    QTAILQ_INIT(&plugin.ctxs);
    qht_init(&plugin.dyn_cb_arr_ht, plugin_dyn_cb_arr_cmp, 16,
             QHT_MODE_AUTO_RESIZE);
    atexit(qemu_plugin_atexit_cb);
}

int plugin_num_vcpus(void)
{
    return plugin.num_vcpus;
}

struct qemu_plugin_scoreboard *plugin_scoreboard_new(size_t element_size)
{
    struct qemu_plugin_scoreboard *score =
        g_malloc0(sizeof(struct qemu_plugin_scoreboard));
    score->data = g_array_new(FALSE, TRUE, element_size);
    g_array_set_size(score->data, plugin.scoreboard_alloc_size);

    qemu_rec_mutex_lock(&plugin.lock);
    QLIST_INSERT_HEAD(&plugin.scoreboards, score, entry);
    qemu_rec_mutex_unlock(&plugin.lock);

    return score;
}

void plugin_scoreboard_free(struct qemu_plugin_scoreboard *score)
{
    qemu_rec_mutex_lock(&plugin.lock);
    QLIST_REMOVE(score, entry);
    qemu_rec_mutex_unlock(&plugin.lock);

    g_array_free(score->data, TRUE);
    g_free(score);
}

enum qemu_plugin_cb_flags tcg_call_to_qemu_plugin_cb_flags(int flags)
{
    if (flags & TCG_CALL_NO_RWG) {
        return QEMU_PLUGIN_CB_NO_REGS;
    } else if (flags & TCG_CALL_NO_WG) {
        return QEMU_PLUGIN_CB_R_REGS;
    } else {
        return QEMU_PLUGIN_CB_RW_REGS_PC;
    }
}