blob: d0fc6ccd53fa8f4f15ea78b3f978f2abefa90e73 [file] [log] [blame]
// SPDX-License-Identifier: Apache-2.0
/* Copyright 2013-2019 IBM Corp. */
#ifndef __CPU_H
#define __CPU_H
#include <processor.h>
#include <ccan/list/list.h>
#include <lock.h>
#include <device.h>
#include <opal.h>
#include <stack.h>
#include <timer.h>
/*
* cpu_thread is our internal structure representing each
* thread in the system
*/
enum cpu_thread_state {
cpu_state_no_cpu = 0, /* Nothing there */
cpu_state_unknown, /* In PACA, not called in yet */
cpu_state_unavailable, /* Not available */
cpu_state_fast_reboot_entry, /* Called back into OPAL, real mode */
cpu_state_present, /* Assumed to spin in asm entry */
cpu_state_active, /* Secondary called in */
cpu_state_os, /* Under OS control */
cpu_state_disabled, /* Disabled by us due to error */
cpu_state_rvwinkle, /* Doing an rvwinkle cycle */
};
struct cpu_job;
struct xive_cpu_state;
struct cpu_thread {
/*
* "stack_guard" must be at offset 0 to match the
* -mstack-protector-guard-offset=0 statement in the Makefile
*/
uint64_t stack_guard;
uint32_t pir;
uint32_t server_no;
uint32_t chip_id;
bool is_secondary;
bool is_fused_core;
struct cpu_thread *primary;
struct cpu_thread *ec_primary;
enum cpu_thread_state state;
struct dt_node *node;
struct trace_info *trace;
uint64_t save_r1;
void *icp_regs;
uint32_t in_opal_call;
uint32_t quiesce_opal_call;
uint64_t entered_opal_call_at;
uint32_t con_suspend;
struct list_head locks_held;
bool con_need_flush;
bool in_mcount;
bool in_poller;
bool in_reinit;
bool in_fast_sleep;
bool in_idle; /* any idle state, even busy wait */
bool in_sleep; /* idle which requires IPI */
bool in_job_sleep; /* requires IPI and cpu_wake_on_job */
uint32_t hbrt_spec_wakeup; /* primary only */
uint64_t save_l2_fir_action1;
uint64_t current_token;
#ifdef STACK_CHECK_ENABLED
int64_t stack_bot_mark;
uint64_t stack_bot_pc;
uint64_t stack_bot_tok;
#define CPU_BACKTRACE_SIZE 60
struct bt_entry stack_bot_bt[CPU_BACKTRACE_SIZE];
struct bt_metadata stack_bot_bt_metadata;
#endif
struct lock job_lock;
struct list_head job_queue;
uint32_t job_count;
bool job_has_no_return;
/*
* Per-core mask tracking for threads in HMI handler and
* a cleanup done bit.
* [D][TTTTTTTT]
*
* The member 'core_hmi_state' is primary only.
* The 'core_hmi_state_ptr' member from all secondry cpus will point
* to 'core_hmi_state' member in primary cpu.
*/
uint32_t core_hmi_state; /* primary only */
uint32_t *core_hmi_state_ptr;
bool tb_invalid;
bool tb_resynced;
/* For use by XICS emulation on XIVE */
struct xive_cpu_state *xstate;
/*
* For direct controls scoms, including special wakeup.
*/
struct lock dctl_lock; /* primary only */
bool dctl_stopped; /* per thread */
uint32_t special_wakeup_count; /* primary */
/*
* For reading DTS sensors async
*/
struct lock dts_lock;
struct timer dts_timer;
__be64 *sensor_data;
u32 sensor_attr;
u32 token;
bool dts_read_in_progress;
#ifdef DEBUG_LOCKS
/* The lock requested by this cpu, used for deadlock detection */
struct lock *requested_lock;
#endif
};
/* This global is set to 1 to allow secondaries to callin,
* typically set after the primary has allocated the cpu_thread
* array and stacks
*/
extern unsigned long cpu_secondary_start;
/* Max PIR in the system */
extern unsigned int cpu_max_pir;
/* Max # of threads per core */
extern unsigned int cpu_thread_count;
/* Boot CPU. */
extern struct cpu_thread *boot_cpu;
extern void __nomcount cpu_relax(void);
/* Initialize CPUs */
void pre_init_boot_cpu(void);
void init_boot_cpu(void);
void init_cpu_max_pir(void);
void init_all_cpus(void);
/* This brings up our secondaries */
extern void cpu_bringup(void);
/* This is called by secondaries as they call in */
extern void cpu_callin(struct cpu_thread *cpu);
/* For cpus which fail to call in. */
extern void cpu_remove_node(const struct cpu_thread *t);
/* Find CPUs using different methods */
extern struct cpu_thread *find_cpu_by_chip_id(u32 chip_id);
extern struct cpu_thread *find_cpu_by_node(struct dt_node *cpu);
extern struct cpu_thread *find_cpu_by_server(u32 server_no);
extern struct cpu_thread *find_cpu_by_pir(u32 pir);
/* Used for lock internals to avoid re-entrancy */
extern struct cpu_thread __nomcount *find_cpu_by_pir_nomcount(u32 pir);
extern struct dt_node *get_cpu_node(u32 pir);
/* Iterator */
extern struct cpu_thread *first_cpu(void);
extern struct cpu_thread *next_cpu(struct cpu_thread *cpu);
/* WARNING: CPUs that have been picked up by the OS are no longer
* appearing as available and can not have jobs scheduled
* on them. Essentially that means that after the OS is
* fully started, all CPUs are seen as unavailable from
* this API standpoint.
*/
static inline bool cpu_is_present(struct cpu_thread *cpu)
{
return cpu->state >= cpu_state_present;
}
static inline bool cpu_is_available(struct cpu_thread *cpu)
{
return cpu->state == cpu_state_active ||
cpu->state == cpu_state_rvwinkle;
}
extern struct cpu_thread *first_available_cpu(void);
extern struct cpu_thread *next_available_cpu(struct cpu_thread *cpu);
extern struct cpu_thread *first_present_cpu(void);
extern struct cpu_thread *next_present_cpu(struct cpu_thread *cpu);
extern struct cpu_thread *first_ungarded_cpu(void);
extern struct cpu_thread *next_ungarded_cpu(struct cpu_thread *cpu);
extern struct cpu_thread *first_ungarded_primary(void);
extern struct cpu_thread *next_ungarded_primary(struct cpu_thread *cpu);
#define for_each_cpu(cpu) \
for (cpu = first_cpu(); cpu; cpu = next_cpu(cpu))
#define for_each_available_cpu(cpu) \
for (cpu = first_available_cpu(); cpu; cpu = next_available_cpu(cpu))
#define for_each_present_cpu(cpu) \
for (cpu = first_present_cpu(); cpu; cpu = next_present_cpu(cpu))
#define for_each_ungarded_cpu(cpu) \
for (cpu = first_ungarded_cpu(); cpu; cpu = next_ungarded_cpu(cpu))
#define for_each_ungarded_primary(cpu) \
for (cpu = first_ungarded_primary(); cpu; cpu = next_ungarded_primary(cpu))
extern struct cpu_thread *first_available_core_in_chip(u32 chip_id);
extern struct cpu_thread *next_available_core_in_chip(struct cpu_thread *cpu, u32 chip_id);
extern u8 get_available_nr_cores_in_chip(u32 chip_id);
#define for_each_available_core_in_chip(core, chip_id) \
for (core = first_available_core_in_chip(chip_id); core; \
core = next_available_core_in_chip(core, chip_id))
/* Return the caller CPU (only after init_cpu_threads) */
#ifndef __TEST__
register struct cpu_thread *__this_cpu asm("r16");
#else
struct cpu_thread *__this_cpu;
#endif
static inline __nomcount struct cpu_thread *this_cpu(void)
{
return __this_cpu;
}
/*
* Note: On POWER9 fused core, cpu_get_thread_index() and cpu_get_core_index()
* return respectively the thread number within a fused core (0..7) and
* the fused core number. If you want the EC (small core) number, you have
* to use the low level pir_to_core_id() and pir_to_thread_id().
*/
/* Get the thread # of a cpu within the core */
static inline uint32_t cpu_get_thread_index(struct cpu_thread *cpu)
{
return cpu->pir - cpu->primary->pir;
}
/* Get the core # of a cpu within the core */
extern uint32_t cpu_get_core_index(struct cpu_thread *cpu);
/* Get the PIR of thread 0 of the same core */
static inline uint32_t cpu_get_thread0(struct cpu_thread *cpu)
{
return cpu->primary->pir;
}
static inline bool cpu_is_thread0(struct cpu_thread *cpu)
{
return cpu->primary == cpu;
}
static inline bool cpu_is_core_chiplet_primary(struct cpu_thread *cpu)
{
return cpu->is_fused_core & (cpu_get_thread_index(cpu) == 1);
}
static inline bool cpu_is_sibling(struct cpu_thread *cpu1,
struct cpu_thread *cpu2)
{
return cpu1->primary == cpu2->primary;
}
/* Called when some error condition requires disabling a core */
void cpu_disable_all_threads(struct cpu_thread *cpu);
/* Allocate & queue a job on target CPU */
extern struct cpu_job *__cpu_queue_job(struct cpu_thread *cpu,
const char *name,
void (*func)(void *data), void *data,
bool no_return);
static inline struct cpu_job *cpu_queue_job(struct cpu_thread *cpu,
const char *name,
void (*func)(void *data),
void *data)
{
return __cpu_queue_job(cpu, name, func, data, false);
}
extern struct cpu_job *cpu_queue_job_on_node(uint32_t chip_id,
const char *name,
void (*func)(void *data), void *data);
/* Poll job status, returns true if completed */
extern bool cpu_poll_job(struct cpu_job *job);
/* Synchronously wait for a job to complete, this will
* continue handling the FSP mailbox if called from the
* boot CPU. Set free_it to free it automatically.
*/
extern void cpu_wait_job(struct cpu_job *job, bool free_it);
/* Called by init to process jobs */
extern void cpu_process_jobs(void);
/* Fallback to running jobs synchronously for global jobs */
extern void cpu_process_local_jobs(void);
/* Check if there's any job pending */
bool cpu_check_jobs(struct cpu_thread *cpu);
/* Set/clear HILE on all CPUs */
void cpu_set_hile_mode(bool hile);
/* OPAL sreset vector in place at 0x100 */
void cpu_set_sreset_enable(bool sreset_enabled);
/* IPI for PM modes is enabled */
void cpu_set_ipi_enable(bool sreset_enabled);
static inline void cpu_give_self_os(void)
{
__this_cpu->state = cpu_state_os;
}
extern unsigned long __attrconst cpu_stack_bottom(unsigned int pir);
extern unsigned long __attrconst cpu_stack_top(unsigned int pir);
extern unsigned long __attrconst cpu_emergency_stack_top(unsigned int pir);
extern void cpu_idle_job(void);
extern void cpu_idle_delay(unsigned long delay);
extern void cpu_fast_reboot_complete(void);
int dctl_set_special_wakeup(struct cpu_thread *t);
int dctl_clear_special_wakeup(struct cpu_thread *t);
int dctl_core_is_gated(struct cpu_thread *t);
extern void exit_uv_mode(int);
void cpu_disable_pef(void);
#endif /* __CPU_H */