blob: 2898a45ced7cddd1c2413655c305f012d1351000 [file] [log] [blame]
// SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later
/*
* Base support for OPAL calls
*
* Copyright 2013-2019 IBM Corp.
*/
#include <skiboot.h>
#include <opal.h>
#include <stack.h>
#include <lock.h>
#include <fsp.h>
#include <cpu.h>
#include <interrupts.h>
#include <op-panel.h>
#include <device.h>
#include <console.h>
#include <trace.h>
#include <timebase.h>
#include <affinity.h>
#include <opal-msg.h>
#include <timer.h>
#include <elf-abi.h>
#include <errorlog.h>
#include <occ.h>
/* Pending events to signal via opal_poll_events */
uint64_t opal_pending_events;
/* OPAL dispatch table defined in head.S */
extern const uint64_t opal_branch_table[];
/* Number of args expected for each call. */
static const u8 opal_num_args[OPAL_LAST+1];
/* OPAL anchor node */
struct dt_node *opal_node;
/* mask of dynamic vs fixed events; opal_allocate_dynamic_event will
* only allocate from this range */
static const uint64_t opal_dynamic_events_mask = 0xffffffff00000000ul;
static uint64_t opal_dynamic_events;
extern uint32_t attn_trigger;
extern uint32_t hir_trigger;
void opal_table_init(void)
{
struct opal_table_entry *s = __opal_table_start;
struct opal_table_entry *e = __opal_table_end;
prlog(PR_DEBUG, "OPAL table: %p .. %p, branch table: %p\n",
s, e, opal_branch_table);
while(s < e) {
((uint64_t *)opal_branch_table)[s->token] = function_entry_address(s->func);
((u8 *)opal_num_args)[s->token] = s->nargs;
s++;
}
}
/* Called from head.S, thus no prototype */
long opal_bad_token(uint64_t token);
long opal_bad_token(uint64_t token)
{
/**
* @fwts-label OPALBadToken
* @fwts-advice OPAL was called with a bad token. On POWER8 and
* earlier, Linux kernels had a bug where they wouldn't check
* if firmware supported particular OPAL calls before making them.
* It is, in fact, harmless for these cases. On systems newer than
* POWER8, this should never happen and indicates a kernel bug
* where OPAL_CHECK_TOKEN isn't being called where it should be.
*/
prlog(PR_ERR, "OPAL: Called with bad token %lld !\n", token);
return OPAL_PARAMETER;
}
#ifdef OPAL_TRACE_ENTRY
static void opal_trace_entry(struct stack_frame *eframe __unused)
{
union trace t;
unsigned nargs, i;
if (eframe->gpr[0] > OPAL_LAST)
nargs = 0;
else
nargs = opal_num_args[eframe->gpr[0]];
t.opal.token = cpu_to_be64(eframe->gpr[0]);
t.opal.lr = cpu_to_be64(eframe->lr);
t.opal.sp = cpu_to_be64(eframe->gpr[1]);
for(i=0; i<nargs; i++)
t.opal.r3_to_11[i] = cpu_to_be64(eframe->gpr[3+i]);
trace_add(&t, TRACE_OPAL, offsetof(struct trace_opal, r3_to_11[nargs]));
}
#endif
/*
* opal_quiesce_state is used as a lock. Don't use an actual lock to avoid
* lock busting.
*/
static uint32_t opal_quiesce_state; /* 0 or QUIESCE_HOLD/QUIESCE_REJECT */
static int32_t opal_quiesce_owner; /* PIR */
static int32_t opal_quiesce_target; /* -1 or PIR */
static int64_t opal_check_token(uint64_t token);
/* Called from head.S, thus no prototype */
int64_t opal_entry_check(struct stack_frame *eframe);
int64_t opal_entry_check(struct stack_frame *eframe)
{
struct cpu_thread *cpu = this_cpu();
uint64_t token = eframe->gpr[0];
if (cpu->pir != mfspr(SPR_PIR)) {
printf("CPU MISMATCH ! PIR=%04lx cpu @%p -> pir=%04x token=%llu\n",
mfspr(SPR_PIR), cpu, cpu->pir, token);
abort();
}
#ifdef OPAL_TRACE_ENTRY
opal_trace_entry(eframe);
#endif
if (!opal_check_token(token))
return opal_bad_token(token);
if (!opal_quiesce_state && cpu->in_opal_call > 1) {
disable_fast_reboot("Kernel re-entered OPAL");
switch (token) {
case OPAL_CONSOLE_READ:
case OPAL_CONSOLE_WRITE:
case OPAL_CONSOLE_WRITE_BUFFER_SPACE:
case OPAL_CONSOLE_FLUSH:
case OPAL_POLL_EVENTS:
case OPAL_CHECK_TOKEN:
case OPAL_CEC_REBOOT:
case OPAL_CEC_REBOOT2:
case OPAL_SIGNAL_SYSTEM_RESET:
break;
default:
printf("CPU ATTEMPT TO RE-ENTER FIRMWARE! PIR=%04lx cpu @%p -> pir=%04x token=%llu\n",
mfspr(SPR_PIR), cpu, cpu->pir, token);
if (cpu->in_opal_call > 2) {
printf("Emergency stack is destroyed, can't continue.\n");
abort();
}
return OPAL_INTERNAL_ERROR;
}
}
cpu->entered_opal_call_at = mftb();
return OPAL_SUCCESS;
}
int64_t opal_exit_check(int64_t retval, struct stack_frame *eframe);
int64_t opal_exit_check(int64_t retval, struct stack_frame *eframe)
{
struct cpu_thread *cpu = this_cpu();
uint64_t token = eframe->gpr[0];
uint64_t now = mftb();
uint64_t call_time = tb_to_msecs(now - cpu->entered_opal_call_at);
if (!cpu->in_opal_call) {
disable_fast_reboot("Un-accounted firmware entry");
printf("CPU UN-ACCOUNTED FIRMWARE ENTRY! PIR=%04lx cpu @%p -> pir=%04x token=%llu retval=%lld\n",
mfspr(SPR_PIR), cpu, cpu->pir, token, retval);
cpu->in_opal_call++; /* avoid exit path underflowing */
} else {
if (cpu->in_opal_call > 2) {
printf("Emergency stack is destroyed, can't continue.\n");
abort();
}
if (!list_empty(&cpu->locks_held)) {
prlog(PR_ERR, "OPAL exiting with locks held, pir=%04x token=%llu retval=%lld\n",
cpu->pir, token, retval);
drop_my_locks(true);
}
}
if (call_time > 100 && token != OPAL_RESYNC_TIMEBASE) {
prlog((call_time < 1000) ? PR_DEBUG : PR_WARNING,
"Spent %llu msecs in OPAL call %llu!\n",
call_time, token);
}
cpu->current_token = 0;
return retval;
}
int64_t opal_quiesce(uint32_t quiesce_type, int32_t cpu_target)
{
struct cpu_thread *cpu = this_cpu();
struct cpu_thread *target = NULL;
struct cpu_thread *c;
uint64_t end;
bool stuck = false;
if (cpu_target >= 0) {
target = find_cpu_by_server(cpu_target);
if (!target)
return OPAL_PARAMETER;
} else if (cpu_target != -1) {
return OPAL_PARAMETER;
}
if (quiesce_type == QUIESCE_HOLD || quiesce_type == QUIESCE_REJECT) {
if (cmpxchg32(&opal_quiesce_state, 0, quiesce_type) != 0) {
if (opal_quiesce_owner != cpu->pir) {
/*
* Nested is allowed for now just for
* internal uses, so an error is returned
* for OS callers, but no error message
* printed if we are nested.
*/
printf("opal_quiesce already quiescing\n");
}
return OPAL_BUSY;
}
opal_quiesce_owner = cpu->pir;
opal_quiesce_target = cpu_target;
}
if (opal_quiesce_owner != cpu->pir) {
printf("opal_quiesce CPU does not own quiesce state (must call QUIESCE_HOLD or QUIESCE_REJECT)\n");
return OPAL_BUSY;
}
/* Okay now we own the quiesce state */
if (quiesce_type == QUIESCE_RESUME ||
quiesce_type == QUIESCE_RESUME_FAST_REBOOT) {
bust_locks = false;
sync(); /* release barrier vs opal entry */
if (target) {
target->quiesce_opal_call = 0;
} else {
for_each_cpu(c) {
if (quiesce_type == QUIESCE_RESUME_FAST_REBOOT)
c->in_opal_call = 0;
if (c == cpu) {
assert(!c->quiesce_opal_call);
continue;
}
c->quiesce_opal_call = 0;
}
}
sync();
opal_quiesce_state = 0;
return OPAL_SUCCESS;
}
if (quiesce_type == QUIESCE_LOCK_BREAK) {
if (opal_quiesce_target != -1) {
printf("opal_quiesce has not quiesced all CPUs (must target -1)\n");
return OPAL_BUSY;
}
bust_locks = true;
return OPAL_SUCCESS;
}
if (target) {
target->quiesce_opal_call = quiesce_type;
} else {
for_each_cpu(c) {
if (c == cpu)
continue;
c->quiesce_opal_call = quiesce_type;
}
}
sync(); /* Order stores to quiesce_opal_call vs loads of in_opal_call */
end = mftb() + msecs_to_tb(1000);
smt_lowest();
if (target) {
while (target->in_opal_call) {
if (tb_compare(mftb(), end) == TB_AAFTERB) {
printf("OPAL quiesce CPU:%04x stuck in OPAL\n", target->pir);
stuck = true;
break;
}
barrier();
}
} else {
for_each_cpu(c) {
if (c == cpu)
continue;
while (c->in_opal_call) {
if (tb_compare(mftb(), end) == TB_AAFTERB) {
printf("OPAL quiesce CPU:%04x stuck in OPAL\n", c->pir);
stuck = true;
break;
}
barrier();
}
}
}
smt_medium();
sync(); /* acquire barrier vs opal entry */
if (stuck) {
printf("OPAL quiesce could not kick all CPUs out of OPAL\n");
return OPAL_PARTIAL;
}
return OPAL_SUCCESS;
}
opal_call(OPAL_QUIESCE, opal_quiesce, 2);
void __opal_register(uint64_t token, void *func, unsigned int nargs)
{
assert(token <= OPAL_LAST);
((uint64_t *)opal_branch_table)[token] = function_entry_address(func);
((u8 *)opal_num_args)[token] = nargs;
}
/*
* add_opal_firmware_exports_node: adds properties to the device-tree which
* the OS will then change into sysfs nodes.
* The properties must be placed under /ibm,opal/firmware/exports.
* The new sysfs nodes are created under /opal/exports.
* To be correctly exported the properties must contain:
* name
* base memory location (u64)
* size (u64)
*/
static void add_opal_firmware_exports_node(struct dt_node *node)
{
struct dt_node *exports = dt_new(node, "exports");
uint64_t sym_start = (uint64_t)__sym_map_start;
uint64_t sym_size = (uint64_t)__sym_map_end - sym_start;
/*
* These property names will be used by Linux as the user-visible file
* name, so make them meaningful if possible. We use _ as the separator
* here to remain consistent with existing file names in /sys/opal.
*/
dt_add_property_u64s(exports, "symbol_map", sym_start, sym_size);
dt_add_property_u64s(exports, "hdat_map", SPIRA_HEAP_BASE,
SPIRA_HEAP_SIZE);
#ifdef SKIBOOT_GCOV
dt_add_property_u64s(exports, "gcov", SKIBOOT_BASE,
HEAP_BASE - SKIBOOT_BASE);
#endif
}
static void add_opal_firmware_node(void)
{
struct dt_node *firmware = dt_new(opal_node, "firmware");
uint64_t sym_start = (uint64_t)__sym_map_start;
uint64_t sym_size = (uint64_t)__sym_map_end - sym_start;
dt_add_property_string(firmware, "compatible", "ibm,opal-firmware");
dt_add_property_string(firmware, "name", "firmware");
dt_add_property_string(firmware, "version", version);
/*
* As previous OS versions use symbol-map located at
* /ibm,opal/firmware we will keep a copy of symbol-map here
* for backwards compatibility
*/
dt_add_property_u64s(firmware, "symbol-map", sym_start, sym_size);
add_opal_firmware_exports_node(firmware);
}
void add_opal_node(void)
{
uint64_t base, entry, size;
extern uint32_t opal_entry;
extern uint32_t boot_entry;
struct dt_node *opal_event;
/* XXX TODO: Reorg this. We should create the base OPAL
* node early on, and have the various sub modules populate
* their own entries (console etc...)
*
* The logic of which console backend to use should be
* extracted
*/
entry = (uint64_t)&opal_entry;
base = SKIBOOT_BASE;
size = (CPU_STACKS_BASE +
(uint64_t)(cpu_max_pir + 1) * STACK_SIZE) - SKIBOOT_BASE;
opal_node = dt_new_check(dt_root, "ibm,opal");
dt_add_property_cells(opal_node, "#address-cells", 0);
dt_add_property_cells(opal_node, "#size-cells", 0);
if (proc_gen < proc_gen_p9)
dt_add_property_strings(opal_node, "compatible", "ibm,opal-v2",
"ibm,opal-v3");
else
dt_add_property_strings(opal_node, "compatible", "ibm,opal-v3");
dt_add_property_cells(opal_node, "opal-msg-async-num", OPAL_MAX_ASYNC_COMP);
dt_add_property_cells(opal_node, "opal-msg-size", OPAL_MSG_SIZE);
dt_add_property_u64(opal_node, "opal-base-address", base);
dt_add_property_u64(opal_node, "opal-entry-address", entry);
dt_add_property_u64(opal_node, "opal-boot-address", (uint64_t)&boot_entry);
dt_add_property_u64(opal_node, "opal-runtime-size", size);
/* Add irqchip interrupt controller */
opal_event = dt_new(opal_node, "event");
dt_add_property_strings(opal_event, "compatible", "ibm,opal-event");
dt_add_property_cells(opal_event, "#interrupt-cells", 0x1);
dt_add_property(opal_event, "interrupt-controller", NULL, 0);
add_opal_firmware_node();
add_associativity_ref_point();
memcons_add_properties();
}
static struct lock evt_lock = LOCK_UNLOCKED;
void opal_update_pending_evt(uint64_t evt_mask, uint64_t evt_values)
{
uint64_t new_evts;
lock(&evt_lock);
new_evts = (opal_pending_events & ~evt_mask) | evt_values;
if (opal_pending_events != new_evts) {
uint64_t tok;
#ifdef OPAL_TRACE_EVT_CHG
printf("OPAL: Evt change: 0x%016llx -> 0x%016llx\n",
opal_pending_events, new_evts);
#endif
/*
* If an event gets *set* while we are in a different call chain
* than opal_handle_interrupt() or opal_handle_hmi(), then we
* artificially generate an interrupt (OCC interrupt specifically)
* to ensure that Linux properly broadcast the event change internally
*/
if ((new_evts & ~opal_pending_events) != 0) {
tok = this_cpu()->current_token;
if (tok != OPAL_HANDLE_INTERRUPT && tok != OPAL_HANDLE_HMI)
occ_send_dummy_interrupt();
}
opal_pending_events = new_evts;
}
unlock(&evt_lock);
}
uint64_t opal_dynamic_event_alloc(void)
{
uint64_t new_event;
int n;
lock(&evt_lock);
/* Create the event mask. This set-bit will be within the event mask
* iff there are free events, or out of the mask if there are no free
* events. If opal_dynamic_events is all ones (ie, all events are
* dynamic, and allocated), then ilog2 will return -1, and we'll have a
* zero mask.
*/
n = ilog2(~opal_dynamic_events);
new_event = 1ull << n;
/* Ensure we're still within the allocatable dynamic events range */
if (new_event & opal_dynamic_events_mask)
opal_dynamic_events |= new_event;
else
new_event = 0;
unlock(&evt_lock);
return new_event;
}
void opal_dynamic_event_free(uint64_t event)
{
lock(&evt_lock);
opal_dynamic_events &= ~event;
unlock(&evt_lock);
}
static uint64_t opal_test_func(uint64_t arg)
{
printf("OPAL: Test function called with arg 0x%llx\n", arg);
return 0xfeedf00d;
}
opal_call(OPAL_TEST, opal_test_func, 1);
struct opal_poll_entry {
struct list_node link;
void (*poller)(void *data);
void *data;
};
static struct list_head opal_pollers = LIST_HEAD_INIT(opal_pollers);
static struct lock opal_poll_lock = LOCK_UNLOCKED;
void opal_add_poller(void (*poller)(void *data), void *data)
{
struct opal_poll_entry *ent;
ent = zalloc(sizeof(struct opal_poll_entry));
assert(ent);
ent->poller = poller;
ent->data = data;
lock(&opal_poll_lock);
list_add_tail(&opal_pollers, &ent->link);
unlock(&opal_poll_lock);
}
void opal_del_poller(void (*poller)(void *data))
{
struct opal_poll_entry *ent;
/* XXX This is currently unused. To solve various "interesting"
* locking issues, the pollers are run locklessly, so if we were
* to free them, we would have to be careful, using something
* akin to RCU to synchronize with other OPAL entries. For now
* if anybody uses it, print a warning and leak the entry, don't
* free it.
*/
/**
* @fwts-label UnsupportedOPALdelpoller
* @fwts-advice Currently removing a poller is DANGEROUS and
* MUST NOT be done in production firmware.
*/
prlog(PR_ALERT, "WARNING: Unsupported opal_del_poller."
" Interesting locking issues, don't call this.\n");
lock(&opal_poll_lock);
list_for_each(&opal_pollers, ent, link) {
if (ent->poller == poller) {
list_del(&ent->link);
/* free(ent); */
break;
}
}
unlock(&opal_poll_lock);
}
void opal_run_pollers(void)
{
static int pollers_with_lock_warnings = 0;
static int poller_recursion = 0;
struct opal_poll_entry *poll_ent;
bool was_in_poller;
/* Don't re-enter on this CPU, unless it was an OPAL re-entry */
if (this_cpu()->in_opal_call == 1 && this_cpu()->in_poller) {
/**
* @fwts-label OPALPollerRecursion
* @fwts-advice Recursion detected in opal_run_pollers(). This
* indicates a bug in OPAL where a poller ended up running
* pollers, which doesn't lead anywhere good.
*/
poller_recursion++;
if (poller_recursion <= 16) {
disable_fast_reboot("Poller recursion detected.");
prlog(PR_ERR, "OPAL: Poller recursion detected.\n");
backtrace();
}
if (poller_recursion == 16)
prlog(PR_ERR, "OPAL: Squashing future poller recursion warnings (>16).\n");
return;
}
was_in_poller = this_cpu()->in_poller;
this_cpu()->in_poller = true;
if (!list_empty(&this_cpu()->locks_held) && pollers_with_lock_warnings < 64) {
/**
* @fwts-label OPALPollerWithLock
* @fwts-advice opal_run_pollers() was called with a lock
* held, which could lead to deadlock if not excessively
* lucky/careful.
*/
prlog(PR_ERR, "Running pollers with lock held !\n");
dump_locks_list();
backtrace();
pollers_with_lock_warnings++;
if (pollers_with_lock_warnings == 64) {
/**
* @fwts-label OPALPollerWithLock64
* @fwts-advice Your firmware is buggy, see the 64
* messages complaining about opal_run_pollers with
* lock held.
*/
prlog(PR_ERR, "opal_run_pollers with lock run 64 "
"times, disabling warning.\n");
}
}
/* We run the timers first */
check_timers(false);
/* The pollers are run lokelessly, see comment in opal_del_poller */
list_for_each(&opal_pollers, poll_ent, link)
poll_ent->poller(poll_ent->data);
/* Disable poller flag */
this_cpu()->in_poller = was_in_poller;
/* On debug builds, print max stack usage */
check_stacks();
}
static int64_t opal_poll_events(__be64 *outstanding_event_mask)
{
if (!opal_addr_valid(outstanding_event_mask))
return OPAL_PARAMETER;
/* Check if we need to trigger an attn for test use */
if (attn_trigger == 0xdeadbeef) {
prlog(PR_EMERG, "Triggering attn\n");
assert(false);
}
opal_run_pollers();
if (outstanding_event_mask)
*outstanding_event_mask = cpu_to_be64(opal_pending_events);
return OPAL_SUCCESS;
}
opal_call(OPAL_POLL_EVENTS, opal_poll_events, 1);
static int64_t opal_check_token(uint64_t token)
{
if (token > OPAL_LAST)
return OPAL_TOKEN_ABSENT;
if (opal_branch_table[token])
return OPAL_TOKEN_PRESENT;
return OPAL_TOKEN_ABSENT;
}
opal_call(OPAL_CHECK_TOKEN, opal_check_token, 1);
struct opal_sync_entry {
struct list_node link;
bool (*notify)(void *data);
void *data;
};
static struct list_head opal_syncers = LIST_HEAD_INIT(opal_syncers);
void opal_add_host_sync_notifier(bool (*notify)(void *data), void *data)
{
struct opal_sync_entry *ent;
ent = zalloc(sizeof(struct opal_sync_entry));
assert(ent);
ent->notify = notify;
ent->data = data;
list_add_tail(&opal_syncers, &ent->link);
}
/*
* Remove a host sync notifier for given callback and data
*/
void opal_del_host_sync_notifier(bool (*notify)(void *data), void *data)
{
struct opal_sync_entry *ent;
list_for_each(&opal_syncers, ent, link) {
if (ent->notify == notify && ent->data == data) {
list_del(&ent->link);
free(ent);
return;
}
}
}
/*
* OPAL call to handle host kexec'ing scenario
*/
static int64_t opal_sync_host_reboot(void)
{
struct opal_sync_entry *ent, *nxt;
int ret = OPAL_SUCCESS;
list_for_each_safe(&opal_syncers, ent, nxt, link)
if (! ent->notify(ent->data))
ret = OPAL_BUSY_EVENT;
return ret;
}
opal_call(OPAL_SYNC_HOST_REBOOT, opal_sync_host_reboot, 0);