blob: f803b7f285d767df62940cad6322a1b265e10291 [file] [log] [blame]
// SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later
/*
* run something, but later.
*
* Timers are run when the SBE timer interrupt triggers (based on us setting
* it) or when the regular heartbeat call from the OS occurs and there's a
* timer that's expired.
*
* Copyright 2014-2019 IBM Corp.
*/
#include <timer.h>
#include <timebase.h>
#include <lock.h>
#include <fsp.h>
#include <device.h>
#include <opal.h>
#include <sbe.h>
#ifdef __TEST__
#define this_cpu() ((void *)-1)
#define cpu_relax()
#else
#include <cpu.h>
#endif
/* Heartbeat requested from Linux */
#define HEARTBEAT_DEFAULT_MS 200
static struct lock timer_lock = LOCK_UNLOCKED;
static LIST_HEAD(timer_list);
static LIST_HEAD(timer_poll_list);
static bool timer_in_poll;
static uint64_t timer_poll_gen;
static inline void update_timer_expiry(uint64_t target)
{
if (sbe_timer_ok())
sbe_update_timer_expiry(target);
}
void init_timer(struct timer *t, timer_func_t expiry, void *data)
{
t->link.next = t->link.prev = NULL;
t->target = 0;
t->expiry = expiry;
t->user_data = data;
t->running = NULL;
}
static void __remove_timer(struct timer *t)
{
list_del(&t->link);
t->link.next = t->link.prev = NULL;
}
static void __sync_timer(struct timer *t)
{
sync();
/* Guard against re-entrancy */
assert(t->running != this_cpu());
while (t->running) {
unlock(&timer_lock);
smt_lowest();
while (t->running)
barrier();
smt_medium();
/* Should we call the pollers here ? */
lock(&timer_lock);
}
}
void sync_timer(struct timer *t)
{
lock(&timer_lock);
__sync_timer(t);
unlock(&timer_lock);
}
void cancel_timer(struct timer *t)
{
lock(&timer_lock);
__sync_timer(t);
if (t->link.next)
__remove_timer(t);
unlock(&timer_lock);
}
void cancel_timer_async(struct timer *t)
{
lock(&timer_lock);
if (t->link.next)
__remove_timer(t);
unlock(&timer_lock);
}
static void __schedule_timer_at(struct timer *t, uint64_t when)
{
struct timer *lt;
/* If the timer is already scheduled, take it out */
if (t->link.next)
__remove_timer(t);
/* Update target */
t->target = when;
if (when == TIMER_POLL) {
/* It's a poller, add it to the poller list */
t->gen = timer_poll_gen;
list_add_tail(&timer_poll_list, &t->link);
} else {
/* It's a real timer, add it in the right spot in the
* ordered timer list
*/
list_for_each(&timer_list, lt, link) {
if (when >= lt->target)
continue;
list_add_before(&timer_list, &lt->link, &t->link);
goto bail;
}
list_add_tail(&timer_list, &t->link);
}
bail:
/* Pick up the next timer and upddate the SBE HW timer */
lt = list_top(&timer_list, struct timer, link);
if (lt) {
update_timer_expiry(lt->target);
}
}
void schedule_timer_at(struct timer *t, uint64_t when)
{
lock(&timer_lock);
__schedule_timer_at(t, when);
unlock(&timer_lock);
}
uint64_t schedule_timer(struct timer *t, uint64_t how_long)
{
uint64_t now = mftb();
if (how_long == TIMER_POLL)
schedule_timer_at(t, TIMER_POLL);
else
schedule_timer_at(t, now + how_long);
return now;
}
static void __check_poll_timers(uint64_t now)
{
struct timer *t;
/* Don't call this from multiple CPUs at once */
if (timer_in_poll)
return;
timer_in_poll = true;
/*
* Poll timers might re-enqueue themselves and don't have an
* expiry so we can't do like normal timers and just run until
* we hit a wall. Instead, each timer has a generation count,
* which we set to the current global gen count when we schedule
* it and update when we run it. It will only be considered if
* the generation count is different than the current one. We
* don't try to compare generations being larger or smaller
* because at boot, this can be called quite quickly and I want
* to be safe vs. wraps.
*/
timer_poll_gen++;
for (;;) {
t = list_top(&timer_poll_list, struct timer, link);
/* Top timer has a different generation than current ? Must
* be older, we are done.
*/
if (!t || t->gen == timer_poll_gen)
break;
/* Top of list still running, we have to delay handling it,
* let's reprogram the SLW with a small delay. We chose
* arbitrarily 1us.
*/
if (t->running) {
update_timer_expiry(now + usecs_to_tb(1));
break;
}
/* Allright, first remove it and mark it running */
__remove_timer(t);
t->running = this_cpu();
/* Now we can unlock and call it's expiry */
unlock(&timer_lock);
t->expiry(t, t->user_data, now);
/* Re-lock and mark not running */
lock(&timer_lock);
t->running = NULL;
}
timer_in_poll = false;
}
static void __check_timers(uint64_t now)
{
struct timer *t;
for (;;) {
t = list_top(&timer_list, struct timer, link);
/* Top of list not expired ? that's it ... */
if (!t || t->target > now)
break;
/* Top of list still running, we have to delay handling
* it. For now just skip until the next poll, when we have
* SLW interrupts, we'll probably want to trip another one
* ASAP
*/
if (t->running)
break;
/* Allright, first remove it and mark it running */
__remove_timer(t);
t->running = this_cpu();
/* Now we can unlock and call it's expiry */
unlock(&timer_lock);
t->expiry(t, t->user_data, now);
/* Re-lock and mark not running */
lock(&timer_lock);
t->running = NULL;
/* Update time stamp */
now = mftb();
}
}
void check_timers(bool from_interrupt)
{
uint64_t now = mftb();
/* This is the polling variant, the SLW interrupt path, when it
* exists, will use a slight variant of this that doesn't call
* the pollers
*/
/* Lockless "peek", a bit racy but shouldn't be a problem as
* we are only looking at whether the list is empty
*/
if (list_empty_nocheck(&timer_poll_list) &&
list_empty_nocheck(&timer_list))
return;
/* Take lock and try again */
lock(&timer_lock);
if (!from_interrupt)
__check_poll_timers(now);
__check_timers(now);
unlock(&timer_lock);
}
#ifndef __TEST__
void late_init_timers(void)
{
int heartbeat = HEARTBEAT_DEFAULT_MS;
/* Add a property requesting the OS to call opal_poll_event() at
* a specified interval in order for us to run our background
* low priority pollers.
*
* If a platform quirk exists, use that, else use the default.
*
* If we have an SBE timer facility, we run this 10 times slower,
* we could possibly completely get rid of it.
*
* We use a value in milliseconds, we don't want this to ever be
* faster than that.
*/
if (platform.heartbeat_time) {
heartbeat = platform.heartbeat_time();
} else if (sbe_timer_ok()) {
heartbeat = HEARTBEAT_DEFAULT_MS * 10;
}
dt_add_property_cells(opal_node, "ibm,heartbeat-ms", heartbeat);
}
#endif