| /* Copyright 2016 IBM Corp. |
| * |
| * Licensed under the Apache License, Version 2.0 (the "License"); |
| * you may not use this file except in compliance with the License. |
| * You may obtain a copy of the License at |
| * |
| * http://www.apache.org/licenses/LICENSE-2.0 |
| * |
| * Unless required by applicable law or agreed to in writing, software |
| * distributed under the License is distributed on an "AS IS" BASIS, |
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or |
| * implied. |
| * See the License for the specific language governing permissions and |
| * limitations under the License. |
| */ |
| #include <skiboot.h> |
| #include <xscom.h> |
| #include <chip.h> |
| #include <io.h> |
| #include <xive.h> |
| #include <xscom-p9-regs.h> |
| #include <interrupts.h> |
| #include <timebase.h> |
| #include <bitmap.h> |
| #include <buddy.h> |
| #include <phys-map.h> |
| |
| /* Use Block group mode to move chip_id into block .... */ |
| #define USE_BLOCK_GROUP_MODE |
| |
| /* Indirect mode */ |
| #define USE_INDIRECT |
| |
| /* Always notify from EQ to VP (no EOI on EQs). Will speed up |
| * EOIs at the expense of potentially higher powerbus traffic. |
| */ |
| #define EQ_ALWAYS_NOTIFY |
| |
| /* Verbose debug */ |
| #undef XIVE_VERBOSE_DEBUG |
| |
| /* Extra debug options used in debug builds */ |
| #ifdef DEBUG |
| #define XIVE_DEBUG_DUPLICATES |
| #define XIVE_PERCPU_LOG |
| #define XIVE_DEBUG_INIT_CACHE_UPDATES |
| #define XIVE_EXTRA_CHECK_INIT_CACHE |
| #undef XIVE_CHECK_MISROUTED_IPI |
| #define XIVE_CHECK_LOCKS |
| #define XIVE_INT_SAFETY_GAP 0x1000 |
| #else |
| #undef XIVE_DEBUG_DUPLICATES |
| #undef XIVE_PERCPU_LOG |
| #undef XIVE_DEBUG_INIT_CACHE_UPDATES |
| #undef XIVE_EXTRA_CHECK_INIT_CACHE |
| #undef XIVE_CHECK_MISROUTED_IPI |
| #undef XIVE_CHECK_LOCKS |
| #define XIVE_INT_SAFETY_GAP 0x10 |
| #endif |
| |
| /* |
| * |
| * VSDs, blocks, set translation etc... |
| * |
| * This stuff confused me to no end so here's an attempt at explaining |
| * my understanding of it and how I use it in OPAL & Linux |
| * |
| * For the following data structures, the XIVE use a mechanism called |
| * Virtualization Structure Tables (VST) to manage the memory layout |
| * and access: ESBs (Event State Buffers, aka IPI sources), EAS/IVT |
| * (Event assignment structures), END/EQs (Notification descriptors |
| * aka event queues) and NVT/VPD (Notification Virtual Targets). |
| * |
| * These structures divide those tables into 16 "blocks". Each XIVE |
| * instance has a definition for all 16 blocks that can either represent |
| * an actual table in memory or a remote XIVE MMIO port to access a |
| * block that is owned by that remote XIVE. |
| * |
| * Our SW design will consist of allocating one block per chip (and thus |
| * per XIVE instance) for now, thus giving us up to 16 supported chips in |
| * the system. We may have to revisit that if we ever support systems with |
| * more than 16 chips but that isn't on our radar at the moment or if we |
| * want to do like pHyp on some machines and dedicate 2 blocks per chip |
| * for some structures. |
| * |
| * Thus we need to be careful that we never expose to Linux the concept |
| * of block and block boundaries, but instead we provide full number ranges |
| * so that consecutive blocks can be supported. |
| * |
| * We will pre-allocate some of the tables in order to support a "fallback" |
| * mode operations where an old-style XICS is emulated via OPAL calls. This |
| * is achieved by having a default of one VP per physical thread associated |
| * with one EQ and one IPI. There is also enought EATs to cover all the PHBs. |
| * |
| * Similarily, for MMIO access, the BARs support what is called "set |
| * translation" which allows tyhe BAR to be devided into a certain |
| * number of sets. The VC BAR (ESBs, ENDs, ...) supports 64 sets and |
| * the PC BAT supports 16. Each "set" can be routed to a specific |
| * block and offset within a block. |
| * |
| * For now, we will not use much of that functionality. We will use a |
| * fixed split between ESB and ENDs for the VC BAR as defined by the |
| * constants below and we will allocate all the PC BARs set to the |
| * local block of that chip |
| */ |
| |
| |
| /* BAR default values (should be initialized by HostBoot but for |
| * now we do it). Based on the memory map document by Dave Larson |
| * |
| * Fixed IC and TM BARs first. |
| */ |
| /* Use 64K for everything by default */ |
| #define IC_PAGE_SIZE 0x10000 |
| #define TM_PAGE_SIZE 0x10000 |
| #define IPI_ESB_SHIFT (16 + 1) |
| #define EQ_ESB_SHIFT (16 + 1) |
| |
| /* VC BAR contains set translations for the ESBs and the EQs. |
| * |
| * It's divided in 64 sets, each of which can be either ESB pages or EQ pages. |
| * The table configuring this is the EDT |
| * |
| * Additionally, the ESB pages come in pair of Linux_Trig_Mode isn't enabled |
| * (which we won't enable for now as it assumes write-only permission which |
| * the MMU doesn't support). |
| * |
| * To get started we just hard wire the following setup: |
| * |
| * VC_BAR size is 512G. We split it into 384G of ESBs (48 sets) and 128G |
| * of ENDs (16 sets) for the time being. IE. Each set is thus 8GB |
| */ |
| |
| #define VC_ESB_SETS 48 |
| #define VC_END_SETS 16 |
| #define VC_MAX_SETS 64 |
| |
| /* The table configuring the PC set translation (16 sets) is the VDT */ |
| #define PC_MAX_SETS 16 |
| |
| /* XXX This is the currently top limit of number of ESB/SBE entries |
| * and EAS/IVT entries pre-allocated per chip. This should probably |
| * turn into a device-tree property or NVRAM setting, or maybe |
| * calculated from the amount of system RAM... |
| * |
| * This is currently set to 1M |
| * |
| * This is independent of the sizing of the MMIO space. |
| * |
| * WARNING: Due to how XICS emulation works, we cannot support more |
| * interrupts per chip at this stage as the full interrupt number |
| * (block + index) has to fit in a 24-bit number. |
| * |
| * That gives us a pre-allocated space of 256KB per chip for the state |
| * bits and 8M per chip for the EAS/IVT. |
| * |
| * Note: The HW interrupts from PCIe and similar other entities that |
| * use their own state bit array will have to share that IVT space, |
| * so we could potentially make the IVT size twice as big, but for now |
| * we will simply share it and ensure we don't hand out IPIs that |
| * overlap the HW interrupts. |
| */ |
| #define MAX_INT_ENTRIES (1 * 1024 * 1024) |
| |
| /* Corresponding direct table sizes */ |
| #define SBE_SIZE (MAX_INT_ENTRIES / 4) |
| #define IVT_SIZE (MAX_INT_ENTRIES * 8) |
| |
| /* Max number of EQs. We allocate an indirect table big enough so |
| * that when fully populated we can have that many EQs. |
| * |
| * The max number of EQs we support in our MMIO space is 128G/128K |
| * ie. 1M. Since one EQ is 8 words (32 bytes), a 64K page can hold |
| * 2K EQs. We need 512 pointers, ie, 4K of memory for the indirect |
| * table. |
| * |
| * XXX Adjust that based on BAR value ? |
| */ |
| #ifdef USE_INDIRECT |
| #define MAX_EQ_COUNT (1 * 1024 * 1024) |
| #define EQ_PER_PAGE (0x10000 / 32) // Use sizeof ? |
| #define IND_EQ_TABLE_SIZE ((MAX_EQ_COUNT / EQ_PER_PAGE) * 8) |
| #else |
| #define MAX_EQ_COUNT (4 * 1024 * 64) |
| #define EQT_SIZE (MAX_EQ_COUNT * 32) |
| #endif |
| |
| /* Number of priorities (and thus EQDs) we allocate for each VP */ |
| #define NUM_INT_PRIORITIES 8 |
| |
| /* Priority used for the one queue in XICS emulation */ |
| #define XIVE_EMULATION_PRIO 7 |
| |
| /* Max number of VPs. We allocate an indirect table big enough so |
| * that when fully populated we can have that many VPs. |
| * |
| * The max number of VPs we support in our MMIO space is 64G/64K |
| * ie. 1M. Since one VP is 16 words (64 bytes), a 64K page can hold |
| * 1K EQ. We need 1024 pointers, ie, 8K of memory for the indirect |
| * table. |
| * |
| * HOWEVER: A block supports only up to 512K VPs (19 bits of target |
| * in the EQ). Since we currently only support 1 block per chip, |
| * we will allocate half of the above. We might add support for |
| * 2 blocks per chip later if necessary. |
| * |
| * XXX Adjust that based on BAR value ? |
| */ |
| #ifdef USE_INDIRECT |
| #define MAX_VP_ORDER 19 /* 512k */ |
| #define MAX_VP_COUNT (1ul << MAX_VP_ORDER) |
| #define VP_PER_PAGE (0x10000 / 64) // Use sizeof ? |
| #define IND_VP_TABLE_SIZE ((MAX_VP_COUNT / VP_PER_PAGE) * 8) |
| #else |
| #define MAX_VP_ORDER 13 /* 8k */ |
| #define MAX_VP_COUNT (1ul << MAX_VP_ORDER) |
| #define VPT_SIZE (MAX_VP_COUNT * 64) |
| #endif |
| |
| #ifdef USE_BLOCK_GROUP_MODE |
| |
| /* Initial number of VPs (XXX Make it a variable ?). Round things |
| * up to a max of 32 cores per chip |
| */ |
| #define INITIAL_VP_BASE 0x80 |
| #define INITIAL_VP_COUNT 0x80 |
| |
| #else |
| |
| /* Initial number of VPs on block 0 only */ |
| #define INITIAL_BLK0_VP_BASE 0x800 |
| #define INITIAL_BLK0_VP_COUNT 0x800 |
| |
| #endif |
| |
| /* The xive operation mode indicates the active "API" and corresponds |
| * to the "mode" parameter of the opal_xive_reset() call |
| */ |
| static enum { |
| XIVE_MODE_EMU = OPAL_XIVE_MODE_EMU, |
| XIVE_MODE_EXPL = OPAL_XIVE_MODE_EXPL, |
| } xive_mode; |
| |
| |
| /* Each source controller has one of these. There's one embedded |
| * in the XIVE struct for IPIs |
| */ |
| struct xive_src { |
| struct irq_source is; |
| const struct irq_source_ops *orig_ops; |
| struct xive *xive; |
| void *esb_mmio; |
| uint32_t esb_base; |
| uint32_t esb_shift; |
| uint32_t flags; |
| }; |
| |
| #define LOG_TYPE_XIRR 0 |
| #define LOG_TYPE_XIRR2 1 |
| #define LOG_TYPE_POPQ 2 |
| #define LOG_TYPE_EOI 3 |
| #define LOG_TYPE_EQD 4 |
| |
| struct xive_log_ent { |
| uint8_t type; |
| uint8_t cnt; |
| uint64_t tb; |
| #define MAX_LOG_DATA 8 |
| uint32_t data[MAX_LOG_DATA]; |
| }; |
| #define MAX_LOG_ENT 32 |
| |
| struct xive_cpu_state { |
| struct xive *xive; |
| void *tm_ring1; |
| |
| #ifdef XIVE_PERCPU_LOG |
| struct xive_log_ent log[MAX_LOG_ENT]; |
| uint32_t log_pos; |
| #endif |
| /* Base HW VP and associated queues */ |
| uint32_t vp_blk; |
| uint32_t vp_idx; |
| uint32_t eq_blk; |
| uint32_t eq_idx; /* Base eq index of a block of 8 */ |
| void *eq_page; |
| |
| /* Pre-allocated IPI */ |
| uint32_t ipi_irq; |
| |
| /* Use for XICS emulation */ |
| struct lock lock; |
| uint8_t cppr; |
| uint8_t mfrr; |
| uint8_t pending; |
| uint8_t prev_cppr; |
| uint32_t *eqbuf; |
| uint32_t eqptr; |
| uint32_t eqmsk; |
| uint8_t eqgen; |
| void *eqmmio; |
| uint64_t total_irqs; |
| }; |
| |
| #ifdef XIVE_PERCPU_LOG |
| |
| static void log_add(struct xive_cpu_state *xs, uint8_t type, |
| uint8_t count, ...) |
| { |
| struct xive_log_ent *e = &xs->log[xs->log_pos]; |
| va_list args; |
| int i; |
| |
| e->type = type; |
| e->cnt = count; |
| e->tb = mftb(); |
| va_start(args, count); |
| for (i = 0; i < count; i++) |
| e->data[i] = va_arg(args, u32); |
| va_end(args); |
| xs->log_pos = xs->log_pos + 1; |
| if (xs->log_pos == MAX_LOG_ENT) |
| xs->log_pos = 0; |
| } |
| |
| static void log_print(struct xive_cpu_state *xs) |
| { |
| uint32_t pos = xs->log_pos; |
| uint8_t buf[256]; |
| int i, j; |
| static const char *lts[] = { |
| ">XIRR", |
| "<XIRR", |
| " POPQ", |
| " EOI", |
| " EQD" |
| }; |
| for (i = 0; i < MAX_LOG_ENT; i++) { |
| struct xive_log_ent *e = &xs->log[pos]; |
| uint8_t *b = buf, *eb = &buf[255]; |
| |
| b += snprintf(b, eb-b, "%08llx %s ", e->tb, |
| lts[e->type]); |
| for (j = 0; j < e->cnt && b < eb; j++) |
| b += snprintf(b, eb-b, "%08x ", e->data[j]); |
| printf("%s\n", buf); |
| pos = pos + 1; |
| if (pos == MAX_LOG_ENT) |
| pos = 0; |
| } |
| } |
| |
| #else /* XIVE_PERCPU_LOG */ |
| |
| static inline void log_add(struct xive_cpu_state *xs __unused, |
| uint8_t type __unused, |
| uint8_t count __unused, ...) { } |
| static inline void log_print(struct xive_cpu_state *xs __unused) { } |
| |
| #endif /* XIVE_PERCPU_LOG */ |
| |
| struct xive { |
| uint32_t chip_id; |
| uint32_t block_id; |
| struct dt_node *x_node; |
| int rev; |
| #define XIVE_REV_UNKNOWN 0 /* Unknown version */ |
| #define XIVE_REV_1 1 /* P9 (Nimbus) DD1.x */ |
| #define XIVE_REV_2 2 /* P9 (Nimbus) DD2.x or Cumulus */ |
| |
| uint64_t xscom_base; |
| |
| /* MMIO regions */ |
| void *ic_base; |
| uint64_t ic_size; |
| uint32_t ic_shift; |
| void *tm_base; |
| uint64_t tm_size; |
| uint32_t tm_shift; |
| void *pc_base; |
| uint64_t pc_size; |
| void *vc_base; |
| uint64_t vc_size; |
| |
| void *esb_mmio; |
| void *eq_mmio; |
| |
| /* Set on XSCOM register access error */ |
| bool last_reg_error; |
| |
| /* Per-XIVE mutex */ |
| struct lock lock; |
| |
| /* Pre-allocated tables. |
| * |
| * We setup all the VDS for actual tables (ie, by opposition to |
| * forwarding ports) as either direct pre-allocated or indirect |
| * and partially populated. |
| * |
| * Currently, the ESB/SBE and the EAS/IVT tables are direct and |
| * fully pre-allocated based on MAX_INT_ENTRIES. |
| * |
| * The other tables are indirect, we thus pre-allocate the indirect |
| * table (ie, pages of pointers) and populate enough of the pages |
| * for our basic setup using 64K pages. |
| * |
| * The size of the indirect tables are driven by MAX_VP_COUNT and |
| * MAX_EQ_COUNT. The number of pre-allocated ones are driven by |
| * INITIAL_VP_COUNT (number of EQ depends on number of VP) in block |
| * mode, otherwise we only preallocate INITIAL_BLK0_VP_COUNT on |
| * block 0. |
| */ |
| |
| /* Direct SBE and IVT tables */ |
| void *sbe_base; |
| void *ivt_base; |
| |
| #ifdef USE_INDIRECT |
| /* Indirect END/EQ table. NULL entries are unallocated, count is |
| * the numbre of pointers (ie, sub page placeholders). |
| */ |
| uint64_t *eq_ind_base; |
| uint32_t eq_ind_count; |
| #else |
| void *eq_base; |
| #endif |
| /* EQ allocation bitmap. Each bit represent 8 EQs */ |
| bitmap_t *eq_map; |
| |
| #ifdef USE_INDIRECT |
| /* Indirect NVT/VP table. NULL entries are unallocated, count is |
| * the numbre of pointers (ie, sub page placeholders). |
| */ |
| uint64_t *vp_ind_base; |
| uint32_t vp_ind_count; |
| #else |
| void *vp_base; |
| #endif |
| |
| #ifndef USE_BLOCK_GROUP_MODE |
| /* VP allocation buddy when not using block group mode */ |
| struct buddy *vp_buddy; |
| #endif |
| |
| #ifdef USE_INDIRECT |
| /* Pool of donated pages for provisioning indirect EQ and VP pages */ |
| struct list_head donated_pages; |
| #endif |
| |
| /* To ease a possible change to supporting more than one block of |
| * interrupts per chip, we store here the "base" global number |
| * and max number of interrupts for this chip. The global number |
| * encompass the block number and index. |
| */ |
| uint32_t int_base; |
| uint32_t int_max; |
| |
| /* Due to the overlap between IPIs and HW sources in the IVT table, |
| * we keep some kind of top-down allocator. It is used for HW sources |
| * to "allocate" interrupt entries and will limit what can be handed |
| * out as IPIs. Of course this assumes we "allocate" all HW sources |
| * before we start handing out IPIs. |
| * |
| * Note: The numbers here are global interrupt numbers so that we can |
| * potentially handle more than one block per chip in the future. |
| */ |
| uint32_t int_hw_bot; /* Bottom of HW allocation */ |
| uint32_t int_ipi_top; /* Highest IPI handed out so far + 1 */ |
| |
| /* The IPI allocation bitmap */ |
| bitmap_t *ipi_alloc_map; |
| |
| /* We keep track of which interrupts were ever enabled to |
| * speed up xive_reset |
| */ |
| bitmap_t *int_enabled_map; |
| |
| /* Embedded source IPIs */ |
| struct xive_src ipis; |
| |
| /* Embedded escalation interrupts */ |
| struct xive_src esc_irqs; |
| |
| /* In memory queue overflow */ |
| void *q_ovf; |
| }; |
| |
| /* Global DT node */ |
| static struct dt_node *xive_dt_node; |
| |
| |
| /* Block <-> Chip conversions. |
| * |
| * As chipIDs may not be within the range of 16 block IDs supported by XIVE, |
| * we have a 2 way conversion scheme. |
| * |
| * From block to chip, use the global table below. |
| * |
| * From chip to block, a field in struct proc_chip contains the first block |
| * of that chip. For now we only support one block per chip but that might |
| * change in the future |
| */ |
| #define XIVE_INVALID_CHIP 0xffffffff |
| #define XIVE_MAX_CHIPS 16 |
| static uint32_t xive_block_to_chip[XIVE_MAX_CHIPS]; |
| static uint32_t xive_block_count; |
| |
| #ifdef USE_BLOCK_GROUP_MODE |
| static uint32_t xive_chip_to_block(uint32_t chip_id) |
| { |
| struct proc_chip *c = get_chip(chip_id); |
| |
| assert(c); |
| assert(c->xive); |
| return c->xive->block_id; |
| } |
| #endif |
| |
| /* Conversion between GIRQ and block/index. |
| * |
| * ------------------------------------ |
| * |0000000E|BLOC| INDEX| |
| * ------------------------------------ |
| * 8 4 20 |
| * |
| * the E bit indicates that this is an escalation interrupt, in |
| * that case, the BLOC/INDEX represents the EQ containig the |
| * corresponding escalation descriptor. |
| * |
| * Global interrupt numbers for non-escalation interrupts are thus |
| * limited to 24 bits which is necessary for our XICS emulation since |
| * the top 8 bits are reserved for the CPPR value. |
| * |
| */ |
| #define GIRQ_TO_BLK(__g) (((__g) >> 20) & 0xf) |
| #define GIRQ_TO_IDX(__g) ((__g) & 0x000fffff) |
| #define BLKIDX_TO_GIRQ(__b,__i) (((uint32_t)(__b)) << 20 | (__i)) |
| #define GIRQ_IS_ESCALATION(__g) ((__g) & 0x01000000) |
| #define MAKE_ESCALATION_GIRQ(__b,__i)(BLKIDX_TO_GIRQ(__b,__i) | 0x01000000) |
| |
| /* Block/IRQ to chip# conversions */ |
| #define PC_BLK_TO_CHIP(__b) (xive_block_to_chip[__b]) |
| #define VC_BLK_TO_CHIP(__b) (xive_block_to_chip[__b]) |
| #define GIRQ_TO_CHIP(__isn) (VC_BLK_TO_CHIP(GIRQ_TO_BLK(__isn))) |
| |
| /* Routing of physical processors to VPs */ |
| #ifdef USE_BLOCK_GROUP_MODE |
| #define PIR2VP_IDX(__pir) (0x80 | P9_PIR2LOCALCPU(__pir)) |
| #define PIR2VP_BLK(__pir) (xive_chip_to_block(P9_PIR2GCID(__pir))) |
| #define VP2PIR(__blk, __idx) (P9_PIRFROMLOCALCPU(VC_BLK_TO_CHIP(__blk), (__idx) & 0x7f)) |
| #else |
| #define PIR2VP_IDX(__pir) (0x800 | (P9_PIR2GCID(__pir) << 7) | P9_PIR2LOCALCPU(__pir)) |
| #define PIR2VP_BLK(__pir) (0) |
| #define VP2PIR(__blk, __idx) (P9_PIRFROMLOCALCPU(((__idx) >> 7) & 0xf, (__idx) & 0x7f)) |
| #endif |
| |
| /* Decoding of OPAL API VP IDs. The VP IDs are encoded as follow |
| * |
| * Block group mode: |
| * |
| * ----------------------------------- |
| * |GVEOOOOO| INDEX| |
| * ----------------------------------- |
| * || | |
| * || Order |
| * |Virtual |
| * Group |
| * |
| * G (Group) : Set to 1 for a group VP (not currently supported) |
| * V (Virtual) : Set to 1 for an allocated VP (vs. a physical processor ID) |
| * E (Error) : Should never be 1, used internally for errors |
| * O (Order) : Allocation order of the VP block |
| * |
| * The conversion is thus done as follow (groups aren't implemented yet) |
| * |
| * If V=0, O must be 0 and 24-bit INDEX value is the PIR |
| * If V=1, the order O group is allocated such that if N is the number of |
| * chip bits considered for allocation (*) |
| * then the INDEX is constructed as follow (bit numbers such as 0=LSB) |
| * - bottom O-N bits is the index within the "VP block" |
| * - next N bits is the XIVE blockID of the VP |
| * - the remaining bits is the per-chip "base" |
| * so the conversion consists of "extracting" the block ID and moving |
| * down the upper bits by N bits. |
| * |
| * In non-block-group mode, the difference is that the blockID is |
| * on the left of the index (the entire VP block is in a single |
| * block ID) |
| */ |
| #ifdef USE_BLOCK_GROUP_MODE |
| |
| /* VP allocation */ |
| static uint32_t xive_chips_alloc_bits = 0; |
| struct buddy *xive_vp_buddy; |
| struct lock xive_buddy_lock = LOCK_UNLOCKED; |
| |
| /* VP# decoding/encoding */ |
| static bool xive_decode_vp(uint32_t vp, uint32_t *blk, uint32_t *idx, |
| uint8_t *order, bool *group) |
| { |
| uint32_t o = (vp >> 24) & 0x1f; |
| uint32_t n = xive_chips_alloc_bits; |
| uint32_t index = vp & 0x00ffffff; |
| uint32_t imask = (1 << (o - n)) - 1; |
| |
| /* Groups not supported yet */ |
| if ((vp >> 31) & 1) |
| return false; |
| if (group) |
| *group = false; |
| |
| /* PIR case */ |
| if (((vp >> 30) & 1) == 0) { |
| if (find_cpu_by_pir(index) == NULL) |
| return false; |
| if (blk) |
| *blk = PIR2VP_BLK(index); |
| if (idx) |
| *idx = PIR2VP_IDX(index); |
| return true; |
| } |
| |
| /* Ensure o > n, we have *at least* 2 VPs per block */ |
| if (o <= n) |
| return false; |
| |
| /* Combine the index base and index */ |
| if (idx) |
| *idx = ((index >> n) & ~imask) | (index & imask); |
| /* Extract block ID */ |
| if (blk) |
| *blk = (index >> (o - n)) & ((1 << n) - 1); |
| |
| /* Return order as well if asked for */ |
| if (order) |
| *order = o; |
| |
| return true; |
| } |
| |
| static uint32_t xive_encode_vp(uint32_t blk, uint32_t idx, uint32_t order) |
| { |
| uint32_t vp = 0x40000000 | (order << 24); |
| uint32_t n = xive_chips_alloc_bits; |
| uint32_t imask = (1 << (order - n)) - 1; |
| |
| vp |= (idx & ~imask) << n; |
| vp |= blk << (order - n); |
| vp |= idx & imask; |
| return vp; |
| } |
| |
| #else /* USE_BLOCK_GROUP_MODE */ |
| |
| /* VP# decoding/encoding */ |
| static bool xive_decode_vp(uint32_t vp, uint32_t *blk, uint32_t *idx, |
| uint8_t *order, bool *group) |
| { |
| uint32_t o = (vp >> 24) & 0x1f; |
| uint32_t index = vp & 0x00ffffff; |
| uint32_t imask = (1 << o) - 1; |
| |
| /* Groups not supported yet */ |
| if ((vp >> 31) & 1) |
| return false; |
| if (group) |
| *group = false; |
| |
| /* PIR case */ |
| if (((vp >> 30) & 1) == 0) { |
| if (find_cpu_by_pir(index) == NULL) |
| return false; |
| if (blk) |
| *blk = PIR2VP_BLK(index); |
| if (idx) |
| *idx = PIR2VP_IDX(index); |
| return true; |
| } |
| |
| /* Ensure o > 0, we have *at least* 2 VPs per block */ |
| if (o == 0) |
| return false; |
| |
| /* Extract index */ |
| if (idx) |
| *idx = index & imask; |
| /* Extract block ID */ |
| if (blk) |
| *blk = index >> o; |
| |
| /* Return order as well if asked for */ |
| if (order) |
| *order = o; |
| |
| return true; |
| } |
| |
| static uint32_t xive_encode_vp(uint32_t blk, uint32_t idx, uint32_t order) |
| { |
| return 0x40000000 | (order << 24) | (blk << order) | idx; |
| } |
| |
| #endif /* !USE_BLOCK_GROUP_MODE */ |
| |
| #define xive_regw(__x, __r, __v) \ |
| __xive_regw(__x, __r, X_##__r, __v, #__r) |
| #define xive_regr(__x, __r) \ |
| __xive_regr(__x, __r, X_##__r, #__r) |
| #define xive_regwx(__x, __r, __v) \ |
| __xive_regw(__x, 0, X_##__r, __v, #__r) |
| #define xive_regrx(__x, __r) \ |
| __xive_regr(__x, 0, X_##__r, #__r) |
| |
| #ifdef XIVE_VERBOSE_DEBUG |
| #define xive_vdbg(__x,__fmt,...) prlog(PR_DEBUG,"XIVE[ IC %02x ] " __fmt, (__x)->chip_id, ##__VA_ARGS__) |
| #define xive_cpu_vdbg(__c,__fmt,...) prlog(PR_DEBUG,"XIVE[CPU %04x] " __fmt, (__c)->pir, ##__VA_ARGS__) |
| #else |
| #define xive_vdbg(x,fmt,...) do { } while(0) |
| #define xive_cpu_vdbg(x,fmt,...) do { } while(0) |
| #endif |
| |
| #define xive_dbg(__x,__fmt,...) prlog(PR_DEBUG,"XIVE[ IC %02x ] " __fmt, (__x)->chip_id, ##__VA_ARGS__) |
| #define xive_cpu_dbg(__c,__fmt,...) prlog(PR_DEBUG,"XIVE[CPU %04x] " __fmt, (__c)->pir, ##__VA_ARGS__) |
| #define xive_warn(__x,__fmt,...) prlog(PR_WARNING,"XIVE[ IC %02x ] " __fmt, (__x)->chip_id, ##__VA_ARGS__) |
| #define xive_cpu_warn(__c,__fmt,...) prlog(PR_WARNING,"XIVE[CPU %04x] " __fmt, (__c)->pir, ##__VA_ARGS__) |
| #define xive_err(__x,__fmt,...) prlog(PR_ERR,"XIVE[ IC %02x ] " __fmt, (__x)->chip_id, ##__VA_ARGS__) |
| #define xive_cpu_err(__c,__fmt,...) prlog(PR_ERR,"XIVE[CPU %04x] " __fmt, (__c)->pir, ##__VA_ARGS__) |
| |
| static void __xive_regw(struct xive *x, uint32_t m_reg, uint32_t x_reg, uint64_t v, |
| const char *rname) |
| { |
| bool use_xscom = (m_reg == 0) || !x->ic_base; |
| int64_t rc; |
| |
| x->last_reg_error = false; |
| |
| if (use_xscom) { |
| assert(x_reg != 0); |
| rc = xscom_write(x->chip_id, x->xscom_base + x_reg, v); |
| if (rc) { |
| if (!rname) |
| rname = "???"; |
| xive_err(x, "Error writing register %s\n", rname); |
| /* Anything else we can do here ? */ |
| x->last_reg_error = true; |
| } |
| } else { |
| out_be64(x->ic_base + m_reg, v); |
| } |
| } |
| |
| static uint64_t __xive_regr(struct xive *x, uint32_t m_reg, uint32_t x_reg, |
| const char *rname) |
| { |
| bool use_xscom = (m_reg == 0) || !x->ic_base; |
| int64_t rc; |
| uint64_t val; |
| |
| x->last_reg_error = false; |
| |
| if (use_xscom) { |
| assert(x_reg != 0); |
| rc = xscom_read(x->chip_id, x->xscom_base + x_reg, &val); |
| if (rc) { |
| if (!rname) |
| rname = "???"; |
| xive_err(x, "Error reading register %s\n", rname); |
| /* Anything else we can do here ? */ |
| x->last_reg_error = true; |
| return -1ull; |
| } |
| } else { |
| val = in_be64(x->ic_base + m_reg); |
| } |
| return val; |
| } |
| |
| /* Locate a controller from an IRQ number */ |
| static struct xive *xive_from_isn(uint32_t isn) |
| { |
| uint32_t chip_id = GIRQ_TO_CHIP(isn); |
| struct proc_chip *c = get_chip(chip_id); |
| |
| if (!c) |
| return NULL; |
| return c->xive; |
| } |
| |
| static struct xive *xive_from_pc_blk(uint32_t blk) |
| { |
| uint32_t chip_id = PC_BLK_TO_CHIP(blk); |
| struct proc_chip *c = get_chip(chip_id); |
| |
| if (!c) |
| return NULL; |
| return c->xive; |
| } |
| |
| static struct xive *xive_from_vc_blk(uint32_t blk) |
| { |
| uint32_t chip_id = VC_BLK_TO_CHIP(blk); |
| struct proc_chip *c = get_chip(chip_id); |
| |
| if (!c) |
| return NULL; |
| return c->xive; |
| } |
| |
| static struct xive_eq *xive_get_eq(struct xive *x, unsigned int idx) |
| { |
| struct xive_eq *p; |
| |
| #ifdef USE_INDIRECT |
| if (idx >= (x->eq_ind_count * EQ_PER_PAGE)) |
| return NULL; |
| p = (struct xive_eq *)(x->eq_ind_base[idx / EQ_PER_PAGE] & |
| VSD_ADDRESS_MASK); |
| if (!p) |
| return NULL; |
| |
| return &p[idx % EQ_PER_PAGE]; |
| #else |
| if (idx >= MAX_EQ_COUNT) |
| return NULL; |
| if (!x->eq_base) |
| return NULL; |
| p = x->eq_base; |
| return p + idx; |
| #endif |
| } |
| |
| static struct xive_ive *xive_get_ive(struct xive *x, unsigned int isn) |
| { |
| struct xive_ive *ivt; |
| uint32_t idx = GIRQ_TO_IDX(isn); |
| |
| if (GIRQ_IS_ESCALATION(isn)) { |
| /* Allright, an escalation IVE is buried inside an EQ, let's |
| * try to find it |
| */ |
| struct xive_eq *eq; |
| |
| if (x->chip_id != VC_BLK_TO_CHIP(GIRQ_TO_BLK(isn))) { |
| xive_err(x, "xive_get_ive, ESC ISN 0x%x not on right chip\n", isn); |
| return NULL; |
| } |
| eq = xive_get_eq(x, idx); |
| if (!eq) { |
| xive_err(x, "xive_get_ive, ESC ISN 0x%x EQ not found\n", isn); |
| return NULL; |
| } |
| return (struct xive_ive *)(char *)&eq->w4; |
| } else { |
| /* Check the block matches */ |
| if (isn < x->int_base || isn >= x->int_max) { |
| xive_err(x, "xive_get_ive, ISN 0x%x not on right chip\n", isn); |
| return NULL; |
| } |
| assert (idx < MAX_INT_ENTRIES); |
| |
| /* If we support >1 block per chip, this should still work as |
| * we are likely to make the table contiguous anyway |
| */ |
| ivt = x->ivt_base; |
| assert(ivt); |
| |
| return ivt + idx; |
| } |
| } |
| |
| static struct xive_vp *xive_get_vp(struct xive *x, unsigned int idx) |
| { |
| struct xive_vp *p; |
| |
| #ifdef USE_INDIRECT |
| assert(idx < (x->vp_ind_count * VP_PER_PAGE)); |
| p = (struct xive_vp *)(x->vp_ind_base[idx / VP_PER_PAGE] & |
| VSD_ADDRESS_MASK); |
| if (!p) |
| return NULL; |
| |
| return &p[idx % VP_PER_PAGE]; |
| #else |
| assert(idx < MAX_VP_COUNT); |
| p = x->vp_base; |
| return p + idx; |
| #endif |
| } |
| |
| static void xive_init_default_vp(struct xive_vp *vp, |
| uint32_t eq_blk, uint32_t eq_idx) |
| { |
| memset(vp, 0, sizeof(struct xive_vp)); |
| |
| /* Stash the EQ base in the pressure relief interrupt field */ |
| vp->w1 = (eq_blk << 28) | eq_idx; |
| vp->w0 = VP_W0_VALID; |
| } |
| |
| static void xive_init_emu_eq(uint32_t vp_blk, uint32_t vp_idx, |
| struct xive_eq *eq, void *backing_page, |
| uint8_t prio) |
| { |
| memset(eq, 0, sizeof(struct xive_eq)); |
| |
| eq->w1 = EQ_W1_GENERATION; |
| eq->w3 = ((uint64_t)backing_page) & 0xffffffff; |
| eq->w2 = (((uint64_t)backing_page)) >> 32 & 0x0fffffff; |
| eq->w6 = SETFIELD(EQ_W6_NVT_BLOCK, 0ul, vp_blk) | |
| SETFIELD(EQ_W6_NVT_INDEX, 0ul, vp_idx); |
| eq->w7 = SETFIELD(EQ_W7_F0_PRIORITY, 0ul, prio); |
| eq->w0 = EQ_W0_VALID | EQ_W0_ENQUEUE | |
| SETFIELD(EQ_W0_QSIZE, 0ul, EQ_QSIZE_64K) | |
| EQ_W0_FIRMWARE; |
| #ifdef EQ_ALWAYS_NOTIFY |
| eq->w0 |= EQ_W0_UCOND_NOTIFY; |
| #endif |
| } |
| |
| static uint32_t *xive_get_eq_buf(uint32_t eq_blk, uint32_t eq_idx) |
| { |
| struct xive *x = xive_from_vc_blk(eq_blk); |
| struct xive_eq *eq; |
| uint64_t addr; |
| |
| assert(x); |
| eq = xive_get_eq(x, eq_idx); |
| assert(eq); |
| assert(eq->w0 & EQ_W0_VALID); |
| addr = (((uint64_t)eq->w2) & 0x0fffffff) << 32 | eq->w3; |
| |
| return (uint32_t *)addr; |
| } |
| |
| #ifdef USE_INDIRECT |
| static void *xive_get_donated_page(struct xive *x __unused) |
| { |
| return (void *)list_pop_(&x->donated_pages, 0); |
| } |
| #endif |
| |
| #define XIVE_ALLOC_IS_ERR(_idx) ((_idx) >= 0xfffffff0) |
| |
| #define XIVE_ALLOC_NO_SPACE 0xffffffff /* No possible space */ |
| #define XIVE_ALLOC_NO_IND 0xfffffffe /* Indirect need provisioning */ |
| #define XIVE_ALLOC_NO_MEM 0xfffffffd /* Local allocation failed */ |
| |
| static uint32_t xive_alloc_eq_set(struct xive *x, bool alloc_indirect __unused) |
| { |
| uint32_t ind_idx __unused; |
| int idx; |
| |
| xive_vdbg(x, "Allocating EQ set...\n"); |
| |
| assert(x->eq_map); |
| |
| /* Allocate from the EQ bitmap. Each bit is 8 EQs */ |
| idx = bitmap_find_zero_bit(*x->eq_map, 0, MAX_EQ_COUNT >> 3); |
| if (idx < 0) { |
| xive_dbg(x, "Allocation from EQ bitmap failed !\n"); |
| return XIVE_ALLOC_NO_SPACE; |
| } |
| bitmap_set_bit(*x->eq_map, idx); |
| |
| idx <<= 3; |
| |
| xive_vdbg(x, "Got EQs 0x%x..0x%x\n", idx, idx + 7); |
| |
| #ifdef USE_INDIRECT |
| /* Calculate the indirect page where the EQs reside */ |
| ind_idx = idx / EQ_PER_PAGE; |
| |
| /* Is there an indirect page ? If not, check if we can provision it */ |
| if (!x->eq_ind_base[ind_idx]) { |
| /* Default flags */ |
| uint64_t vsd_flags = SETFIELD(VSD_TSIZE, 0ull, 4) | |
| SETFIELD(VSD_MODE, 0ull, VSD_MODE_EXCLUSIVE); |
| void *page; |
| |
| /* If alloc_indirect is set, allocate the memory from OPAL own, |
| * otherwise try to provision from the donated pool |
| */ |
| if (alloc_indirect) { |
| /* Allocate/provision indirect page during boot only */ |
| xive_dbg(x, "Indirect empty, provisioning from local pool\n"); |
| page = local_alloc(x->chip_id, 0x10000, 0x10000); |
| if (!page) { |
| xive_dbg(x, "provisioning failed !\n"); |
| return XIVE_ALLOC_NO_MEM; |
| } |
| vsd_flags |= VSD_FIRMWARE; |
| } else { |
| xive_dbg(x, "Indirect empty, provisioning from donated pages\n"); |
| page = xive_get_donated_page(x); |
| if (!page) { |
| xive_dbg(x, "none available !\n"); |
| return XIVE_ALLOC_NO_IND; |
| } |
| } |
| memset(page, 0, 0x10000); |
| x->eq_ind_base[ind_idx] = vsd_flags | |
| (((uint64_t)page) & VSD_ADDRESS_MASK); |
| /* Any cache scrub needed ? */ |
| } |
| #endif /* USE_INDIRECT */ |
| |
| return idx; |
| } |
| |
| static void xive_free_eq_set(struct xive *x, uint32_t eqs) |
| { |
| uint32_t idx; |
| |
| xive_vdbg(x, "Freeing EQ set...\n"); |
| |
| assert((eqs & 7) == 0); |
| assert(x->eq_map); |
| |
| idx = eqs >> 3; |
| bitmap_clr_bit(*x->eq_map, idx); |
| } |
| |
| #ifdef USE_INDIRECT |
| static bool xive_provision_vp_ind(struct xive *x, uint32_t vp_idx, uint32_t order) |
| { |
| uint32_t pbase, pend, i; |
| |
| pbase = vp_idx / VP_PER_PAGE; |
| pend = (vp_idx + (1 << order)) / VP_PER_PAGE; |
| |
| for (i = pbase; i <= pend; i++) { |
| void *page; |
| u64 vsd; |
| |
| /* Already provisioned ? */ |
| if (x->vp_ind_base[i]) |
| continue; |
| |
| /* Try to grab a donated page */ |
| page = xive_get_donated_page(x); |
| if (!page) |
| return false; |
| |
| /* Install the page */ |
| memset(page, 0, 0x10000); |
| vsd = ((uint64_t)page) & VSD_ADDRESS_MASK; |
| vsd |= SETFIELD(VSD_TSIZE, 0ull, 4); |
| vsd |= SETFIELD(VSD_MODE, 0ull, VSD_MODE_EXCLUSIVE); |
| x->vp_ind_base[i] = vsd; |
| } |
| return true; |
| } |
| #else |
| static inline bool xive_provision_vp_ind(struct xive *x __unused, |
| uint32_t vp_idx __unused, |
| uint32_t order __unused) |
| { |
| return true; |
| } |
| #endif /* USE_INDIRECT */ |
| |
| #ifdef USE_BLOCK_GROUP_MODE |
| |
| static void xive_init_vp_allocator(void) |
| { |
| /* Initialize chip alloc bits */ |
| xive_chips_alloc_bits = ilog2(xive_block_count); |
| |
| prlog(PR_INFO, "XIVE: %d chips considered for VP allocations\n", |
| 1 << xive_chips_alloc_bits); |
| |
| /* Allocate a buddy big enough for MAX_VP_ORDER allocations. |
| * |
| * each bit in the buddy represents 1 << xive_chips_alloc_bits |
| * VPs. |
| */ |
| xive_vp_buddy = buddy_create(MAX_VP_ORDER); |
| assert(xive_vp_buddy); |
| |
| /* We reserve the whole range of VPs representing HW chips. |
| * |
| * These are 0x80..0xff, so order 7 starting at 0x80. This will |
| * reserve that range on each chip. |
| * |
| * XXX This can go away if we just call xive_reset .. |
| */ |
| assert(buddy_reserve(xive_vp_buddy, 0x80, 7)); |
| } |
| |
| static uint32_t xive_alloc_vps(uint32_t order) |
| { |
| uint32_t local_order, i; |
| int vp; |
| |
| /* The minimum order is 2 VPs per chip */ |
| if (order < (xive_chips_alloc_bits + 1)) |
| order = xive_chips_alloc_bits + 1; |
| |
| /* We split the allocation */ |
| local_order = order - xive_chips_alloc_bits; |
| |
| /* We grab that in the global buddy */ |
| assert(xive_vp_buddy); |
| lock(&xive_buddy_lock); |
| vp = buddy_alloc(xive_vp_buddy, local_order); |
| unlock(&xive_buddy_lock); |
| if (vp < 0) |
| return XIVE_ALLOC_NO_SPACE; |
| |
| /* Provision on every chip considered for allocation */ |
| for (i = 0; i < (1 << xive_chips_alloc_bits); i++) { |
| struct xive *x = xive_from_pc_blk(i); |
| bool success; |
| |
| /* Return internal error & log rather than assert ? */ |
| assert(x); |
| lock(&x->lock); |
| success = xive_provision_vp_ind(x, vp, local_order); |
| unlock(&x->lock); |
| if (!success) { |
| lock(&xive_buddy_lock); |
| buddy_free(xive_vp_buddy, vp, local_order); |
| unlock(&xive_buddy_lock); |
| return XIVE_ALLOC_NO_IND; |
| } |
| } |
| |
| /* Encode the VP number. "blk" is 0 as this represents |
| * all blocks and the allocation always starts at 0 |
| */ |
| return xive_encode_vp(0, vp, order); |
| } |
| |
| static void xive_free_vps(uint32_t vp) |
| { |
| uint32_t idx; |
| uint8_t order, local_order; |
| |
| assert(xive_decode_vp(vp, NULL, &idx, &order, NULL)); |
| |
| /* We split the allocation */ |
| local_order = order - xive_chips_alloc_bits; |
| |
| /* Free that in the buddy */ |
| lock(&xive_buddy_lock); |
| buddy_free(xive_vp_buddy, idx, local_order); |
| unlock(&xive_buddy_lock); |
| } |
| |
| #else /* USE_BLOCK_GROUP_MODE */ |
| |
| static void xive_init_vp_allocator(void) |
| { |
| struct proc_chip *chip; |
| |
| for_each_chip(chip) { |
| struct xive *x = chip->xive; |
| if (!x) |
| continue; |
| /* Each chip has a MAX_VP_ORDER buddy */ |
| x->vp_buddy = buddy_create(MAX_VP_ORDER); |
| assert(x->vp_buddy); |
| |
| /* We reserve the whole range of VPs representing HW chips. |
| * |
| * These are 0x800..0xfff on block 0 only, so order 11 |
| * starting at 0x800. |
| */ |
| if (x->block_id == 0) |
| assert(buddy_reserve(x->vp_buddy, 0x800, 11)); |
| } |
| } |
| |
| static uint32_t xive_alloc_vps(uint32_t order) |
| { |
| struct proc_chip *chip; |
| struct xive *x = NULL; |
| int vp = -1; |
| |
| /* Minimum order is 1 */ |
| if (order < 1) |
| order = 1; |
| |
| /* Try on every chip */ |
| for_each_chip(chip) { |
| x = chip->xive; |
| if (!x) |
| continue; |
| assert(x->vp_buddy); |
| lock(&x->lock); |
| vp = buddy_alloc(x->vp_buddy, order); |
| unlock(&x->lock); |
| if (vp >= 0) |
| break; |
| } |
| if (vp < 0) |
| return XIVE_ALLOC_NO_SPACE; |
| |
| /* We have VPs, make sure we have backing for the |
| * NVTs on that block |
| */ |
| if (!xive_provision_vp_ind(x, vp, order)) { |
| lock(&x->lock); |
| buddy_free(x->vp_buddy, vp, order); |
| unlock(&x->lock); |
| return XIVE_ALLOC_NO_IND; |
| } |
| |
| /* Encode the VP number */ |
| return xive_encode_vp(x->block_id, vp, order); |
| } |
| |
| static void xive_free_vps(uint32_t vp) |
| { |
| uint32_t idx, blk; |
| uint8_t order; |
| struct xive *x; |
| |
| assert(xive_decode_vp(vp, &blk, &idx, &order, NULL)); |
| |
| /* Grab appropriate xive */ |
| x = xive_from_pc_blk(blk); |
| /* XXX Return error instead ? */ |
| assert(x); |
| |
| /* Free that in the buddy */ |
| lock(&x->lock); |
| buddy_free(x->vp_buddy, idx, order); |
| unlock(&x->lock); |
| } |
| |
| #endif /* ndef USE_BLOCK_GROUP_MODE */ |
| |
| enum xive_cache_type { |
| xive_cache_ivc, |
| xive_cache_sbc, |
| xive_cache_eqc, |
| xive_cache_vpc, |
| }; |
| |
| static int64_t __xive_cache_watch(struct xive *x, enum xive_cache_type ctype, |
| uint64_t block, uint64_t idx, |
| uint32_t start_dword, uint32_t dword_count, |
| void *new_data, bool light_watch, |
| bool synchronous); |
| |
| static void xive_scrub_workaround_vp(struct xive *x, uint32_t block, uint32_t idx __unused) |
| { |
| /* VP variant of the workaround described in __xive_cache_scrub(), |
| * we need to be careful to use for that workaround an NVT that |
| * sits on the same xive but isn NOT part of a donated indirect |
| * entry. |
| * |
| * The reason is that the dummy cache watch will re-create a |
| * dirty entry in the cache, even if the entry is marked |
| * invalid. |
| * |
| * Thus if we are about to dispose of the indirect entry backing |
| * it, we'll cause a checkstop later on when trying to write it |
| * out. |
| * |
| * Note: This means the workaround only works for block group |
| * mode. |
| */ |
| #ifdef USE_BLOCK_GROUP_MODE |
| __xive_cache_watch(x, xive_cache_vpc, block, INITIAL_VP_BASE, 0, |
| 0, NULL, true, false); |
| #else |
| /* WARNING: Some workarounds related to cache scrubs require us to |
| * have at least one firmware owned (permanent) indirect entry for |
| * each XIVE instance. This currently only happens in block group |
| * mode |
| */ |
| #warning Block group mode should not be disabled |
| #endif |
| } |
| |
| static void xive_scrub_workaround_eq(struct xive *x, uint32_t block __unused, uint32_t idx) |
| { |
| void *mmio; |
| |
| /* EQ variant of the workaround described in __xive_cache_scrub(), |
| * a simple non-side effect load from ESn will do |
| */ |
| mmio = x->eq_mmio + idx * 0x20000; |
| |
| /* Ensure the above has returned before we do anything else |
| * the XIVE store queue is completely empty |
| */ |
| load_wait(in_be64(mmio + 0x800)); |
| } |
| |
| static int64_t __xive_cache_scrub(struct xive *x, enum xive_cache_type ctype, |
| uint64_t block, uint64_t idx, |
| bool want_inval, bool want_disable) |
| { |
| uint64_t sreg, sregx, mreg, mregx; |
| uint64_t mval, sval; |
| |
| #ifdef XIVE_CHECK_LOCKS |
| assert(lock_held_by_me(&x->lock)); |
| #endif |
| |
| /* Workaround a HW bug in XIVE where the scrub completion |
| * isn't ordered by loads, thus the data might still be |
| * in a queue and may not have reached coherency. |
| * |
| * The workaround is two folds: We force the scrub to also |
| * invalidate, then after the scrub, we do a dummy cache |
| * watch which will make the HW read the data back, which |
| * should be ordered behind all the preceding stores. |
| * |
| * Update: For EQs we can do a non-side effect ESB load instead |
| * which is faster. |
| */ |
| want_inval = true; |
| |
| switch (ctype) { |
| case xive_cache_ivc: |
| sreg = VC_IVC_SCRUB_TRIG; |
| sregx = X_VC_IVC_SCRUB_TRIG; |
| mreg = VC_IVC_SCRUB_MASK; |
| mregx = X_VC_IVC_SCRUB_MASK; |
| break; |
| case xive_cache_sbc: |
| sreg = VC_SBC_SCRUB_TRIG; |
| sregx = X_VC_SBC_SCRUB_TRIG; |
| mreg = VC_SBC_SCRUB_MASK; |
| mregx = X_VC_SBC_SCRUB_MASK; |
| break; |
| case xive_cache_eqc: |
| sreg = VC_EQC_SCRUB_TRIG; |
| sregx = X_VC_EQC_SCRUB_TRIG; |
| mreg = VC_EQC_SCRUB_MASK; |
| mregx = X_VC_EQC_SCRUB_MASK; |
| break; |
| case xive_cache_vpc: |
| sreg = PC_VPC_SCRUB_TRIG; |
| sregx = X_PC_VPC_SCRUB_TRIG; |
| mreg = PC_VPC_SCRUB_MASK; |
| mregx = X_PC_VPC_SCRUB_MASK; |
| break; |
| default: |
| return OPAL_INTERNAL_ERROR; |
| } |
| if (ctype == xive_cache_vpc) { |
| mval = PC_SCRUB_BLOCK_ID | PC_SCRUB_OFFSET; |
| sval = SETFIELD(PC_SCRUB_BLOCK_ID, idx, block) | |
| PC_SCRUB_VALID; |
| } else { |
| mval = VC_SCRUB_BLOCK_ID | VC_SCRUB_OFFSET; |
| sval = SETFIELD(VC_SCRUB_BLOCK_ID, idx, block) | |
| VC_SCRUB_VALID; |
| } |
| if (want_inval) |
| sval |= PC_SCRUB_WANT_INVAL; |
| if (want_disable) |
| sval |= PC_SCRUB_WANT_DISABLE; |
| |
| __xive_regw(x, mreg, mregx, mval, NULL); |
| __xive_regw(x, sreg, sregx, sval, NULL); |
| |
| /* XXX Add timeout !!! */ |
| for (;;) { |
| sval = __xive_regr(x, sreg, sregx, NULL); |
| if (!(sval & VC_SCRUB_VALID)) |
| break; |
| /* Small delay */ |
| time_wait(100); |
| } |
| sync(); |
| |
| /* Workaround for HW bug described above (only applies to |
| * EQC and VPC |
| */ |
| if (ctype == xive_cache_eqc) |
| xive_scrub_workaround_eq(x, block, idx); |
| else if (ctype == xive_cache_vpc) |
| xive_scrub_workaround_vp(x, block, idx); |
| |
| return 0; |
| } |
| |
| static int64_t xive_ivc_scrub(struct xive *x, uint64_t block, uint64_t idx) |
| { |
| /* IVC has no "want_inval" bit, it always invalidates */ |
| return __xive_cache_scrub(x, xive_cache_ivc, block, idx, false, false); |
| } |
| |
| static int64_t xive_vpc_scrub_clean(struct xive *x, uint64_t block, uint64_t idx) |
| { |
| return __xive_cache_scrub(x, xive_cache_vpc, block, idx, true, false); |
| } |
| |
| static int64_t __xive_cache_watch(struct xive *x, enum xive_cache_type ctype, |
| uint64_t block, uint64_t idx, |
| uint32_t start_dword, uint32_t dword_count, |
| void *new_data, bool light_watch, |
| bool synchronous) |
| { |
| uint64_t sreg, sregx, dreg0, dreg0x; |
| uint64_t dval0, sval, status; |
| int64_t i; |
| |
| #ifdef XIVE_CHECK_LOCKS |
| assert(lock_held_by_me(&x->lock)); |
| #endif |
| switch (ctype) { |
| case xive_cache_eqc: |
| sreg = VC_EQC_CWATCH_SPEC; |
| sregx = X_VC_EQC_CWATCH_SPEC; |
| dreg0 = VC_EQC_CWATCH_DAT0; |
| dreg0x = X_VC_EQC_CWATCH_DAT0; |
| sval = SETFIELD(VC_EQC_CWATCH_BLOCKID, idx, block); |
| break; |
| case xive_cache_vpc: |
| sreg = PC_VPC_CWATCH_SPEC; |
| sregx = X_PC_VPC_CWATCH_SPEC; |
| dreg0 = PC_VPC_CWATCH_DAT0; |
| dreg0x = X_PC_VPC_CWATCH_DAT0; |
| sval = SETFIELD(PC_VPC_CWATCH_BLOCKID, idx, block); |
| break; |
| default: |
| return OPAL_INTERNAL_ERROR; |
| } |
| |
| /* The full bit is in the same position for EQC and VPC */ |
| if (!light_watch) |
| sval |= VC_EQC_CWATCH_FULL; |
| |
| for (;;) { |
| /* Write the cache watch spec */ |
| __xive_regw(x, sreg, sregx, sval, NULL); |
| |
| /* Load data0 register to populate the watch */ |
| dval0 = __xive_regr(x, dreg0, dreg0x, NULL); |
| |
| /* If new_data is NULL, this is a dummy watch used as a |
| * workaround for a HW bug |
| */ |
| if (!new_data) { |
| __xive_regw(x, dreg0, dreg0x, dval0, NULL); |
| return 0; |
| } |
| |
| /* Write the words into the watch facility. We write in reverse |
| * order in case word 0 is part of it as it must be the last |
| * one written. |
| */ |
| for (i = start_dword + dword_count - 1; i >= start_dword ;i--) { |
| uint64_t dw = ((uint64_t *)new_data)[i - start_dword]; |
| __xive_regw(x, dreg0 + i * 8, dreg0x + i, dw, NULL); |
| } |
| |
| /* Write data0 register to trigger the update if word 0 wasn't |
| * written above |
| */ |
| if (start_dword > 0) |
| __xive_regw(x, dreg0, dreg0x, dval0, NULL); |
| |
| /* This may not be necessary for light updates (it's possible |
| * that a sync in sufficient, TBD). Ensure the above is |
| * complete and check the status of the watch. |
| */ |
| status = __xive_regr(x, sreg, sregx, NULL); |
| |
| /* Bits FULL and CONFLICT are in the same position in |
| * EQC and VPC |
| */ |
| if (!(status & VC_EQC_CWATCH_FULL) || |
| !(status & VC_EQC_CWATCH_CONFLICT)) |
| break; |
| if (!synchronous) |
| return OPAL_BUSY; |
| |
| /* XXX Add timeout ? */ |
| } |
| |
| /* Perform a scrub with "want_invalidate" set to false to push the |
| * cache updates to memory as well |
| */ |
| return __xive_cache_scrub(x, ctype, block, idx, false, false); |
| } |
| |
| static int64_t xive_eqc_cache_update(struct xive *x, uint64_t block, |
| uint64_t idx, uint32_t start_dword, |
| uint32_t dword_count, void *new_data, |
| bool light_watch, bool synchronous) |
| { |
| return __xive_cache_watch(x, xive_cache_eqc, block, idx, |
| start_dword, dword_count, |
| new_data, light_watch, synchronous); |
| } |
| |
| static int64_t xive_vpc_cache_update(struct xive *x, uint64_t block, |
| uint64_t idx, uint32_t start_dword, |
| uint32_t dword_count, void *new_data, |
| bool light_watch, bool synchronous) |
| { |
| return __xive_cache_watch(x, xive_cache_vpc, block, idx, |
| start_dword, dword_count, |
| new_data, light_watch, synchronous); |
| } |
| |
| static bool xive_set_vsd(struct xive *x, uint32_t tbl, uint32_t idx, uint64_t v) |
| { |
| /* Set VC version */ |
| xive_regw(x, VC_VSD_TABLE_ADDR, |
| SETFIELD(VST_TABLE_SELECT, 0ull, tbl) | |
| SETFIELD(VST_TABLE_OFFSET, 0ull, idx)); |
| if (x->last_reg_error) |
| return false; |
| /* Hack to workaround DD1 issue with NVT in VC in DD1 */ |
| if (tbl == VST_TSEL_VPDT) |
| xive_regw(x, VC_VSD_TABLE_DATA, v | VSD_TSIZE); |
| else |
| xive_regw(x, VC_VSD_TABLE_DATA, v); |
| if (x->last_reg_error) |
| return false; |
| |
| /* Except for IRQ table, also set PC version */ |
| if (tbl == VST_TSEL_IRQ) |
| return true; |
| |
| xive_regw(x, PC_VSD_TABLE_ADDR, |
| SETFIELD(VST_TABLE_SELECT, 0ull, tbl) | |
| SETFIELD(VST_TABLE_OFFSET, 0ull, idx)); |
| if (x->last_reg_error) |
| return false; |
| xive_regw(x, PC_VSD_TABLE_DATA, v); |
| if (x->last_reg_error) |
| return false; |
| return true; |
| } |
| |
| static bool xive_set_local_tables(struct xive *x) |
| { |
| uint64_t base, i; |
| |
| /* These have to be power of 2 sized */ |
| assert(is_pow2(SBE_SIZE)); |
| assert(is_pow2(IVT_SIZE)); |
| |
| /* All tables set as exclusive */ |
| base = SETFIELD(VSD_MODE, 0ull, VSD_MODE_EXCLUSIVE); |
| |
| /* Set IVT as direct mode */ |
| if (!xive_set_vsd(x, VST_TSEL_IVT, x->block_id, base | |
| (((uint64_t)x->ivt_base) & VSD_ADDRESS_MASK) | |
| SETFIELD(VSD_TSIZE, 0ull, ilog2(IVT_SIZE) - 12))) |
| return false; |
| |
| /* Set SBE as direct mode */ |
| if (!xive_set_vsd(x, VST_TSEL_SBE, x->block_id, base | |
| (((uint64_t)x->sbe_base) & VSD_ADDRESS_MASK) | |
| SETFIELD(VSD_TSIZE, 0ull, ilog2(SBE_SIZE) - 12))) |
| return false; |
| |
| #ifdef USE_INDIRECT |
| /* Set EQDT as indirect mode with 64K subpages */ |
| if (!xive_set_vsd(x, VST_TSEL_EQDT, x->block_id, base | |
| (((uint64_t)x->eq_ind_base) & VSD_ADDRESS_MASK) | |
| VSD_INDIRECT | SETFIELD(VSD_TSIZE, 0ull, 4))) |
| return false; |
| |
| /* Set VPDT as indirect mode with 64K subpages */ |
| if (!xive_set_vsd(x, VST_TSEL_VPDT, x->block_id, base | |
| (((uint64_t)x->vp_ind_base) & VSD_ADDRESS_MASK) | |
| VSD_INDIRECT | SETFIELD(VSD_TSIZE, 0ull, 4))) |
| return false; |
| #else |
| /* Set EQDT as direct mode */ |
| if (!xive_set_vsd(x, VST_TSEL_EQDT, x->block_id, base | |
| (((uint64_t)x->eq_base) & VSD_ADDRESS_MASK) | |
| SETFIELD(VSD_TSIZE, 0ull, ilog2(EQT_SIZE) - 12))) |
| return false; |
| |
| /* Set VPDT as direct mode */ |
| if (!xive_set_vsd(x, VST_TSEL_VPDT, x->block_id, base | |
| (((uint64_t)x->vp_base) & VSD_ADDRESS_MASK) | |
| SETFIELD(VSD_TSIZE, 0ull, ilog2(VPT_SIZE) - 12))) |
| return false; |
| #endif |
| |
| /* Setup quue overflows */ |
| for (i = 0; i < VC_QUEUE_OVF_COUNT; i++) { |
| u64 addr = ((uint64_t)x->q_ovf) + i * 0x10000; |
| u64 cfg, sreg, sregx; |
| |
| if (!xive_set_vsd(x, VST_TSEL_IRQ, i, base | |
| (addr & VSD_ADDRESS_MASK) | |
| SETFIELD(VSD_TSIZE, 0ull, 4))) |
| return false; |
| sreg = VC_IRQ_CONFIG_IPI + i * 8; |
| sregx = X_VC_IRQ_CONFIG_IPI + i; |
| cfg = __xive_regr(x, sreg, sregx, NULL); |
| cfg |= VC_IRQ_CONFIG_MEMB_EN; |
| cfg = SETFIELD(VC_IRQ_CONFIG_MEMB_SZ, cfg, 4); |
| __xive_regw(x, sreg, sregx, cfg, NULL); |
| } |
| |
| return true; |
| } |
| |
| static bool xive_configure_bars(struct xive *x) |
| { |
| uint64_t chip_id = x->chip_id; |
| uint64_t val; |
| |
| /* IC BAR */ |
| phys_map_get(chip_id, XIVE_IC, 0, (uint64_t *)&x->ic_base, &x->ic_size); |
| val = (uint64_t)x->ic_base | CQ_IC_BAR_VALID; |
| if (IC_PAGE_SIZE == 0x10000) { |
| val |= CQ_IC_BAR_64K; |
| x->ic_shift = 16; |
| } else |
| x->ic_shift = 12; |
| xive_regwx(x, CQ_IC_BAR, val); |
| if (x->last_reg_error) |
| return false; |
| |
| /* TM BAR, only configure TM1. Note that this has the same address |
| * for each chip !!! Hence we create a fake chip 0 and use that for |
| * all phys_map_get(XIVE_TM) calls. |
| */ |
| phys_map_get(0, XIVE_TM, 0, (uint64_t *)&x->tm_base, &x->tm_size); |
| val = (uint64_t)x->tm_base | CQ_TM_BAR_VALID; |
| if (TM_PAGE_SIZE == 0x10000) { |
| x->tm_shift = 16; |
| val |= CQ_TM_BAR_64K; |
| } else |
| x->tm_shift = 12; |
| xive_regwx(x, CQ_TM1_BAR, val); |
| if (x->last_reg_error) |
| return false; |
| xive_regwx(x, CQ_TM2_BAR, 0); |
| if (x->last_reg_error) |
| return false; |
| |
| /* PC BAR. Clear first, write mask, then write value */ |
| phys_map_get(chip_id, XIVE_PC, 0, (uint64_t *)&x->pc_base, &x->pc_size); |
| xive_regwx(x, CQ_PC_BAR, 0); |
| if (x->last_reg_error) |
| return false; |
| val = ~(x->pc_size - 1) & CQ_PC_BARM_MASK; |
| xive_regwx(x, CQ_PC_BARM, val); |
| if (x->last_reg_error) |
| return false; |
| val = (uint64_t)x->pc_base | CQ_PC_BAR_VALID; |
| xive_regwx(x, CQ_PC_BAR, val); |
| if (x->last_reg_error) |
| return false; |
| |
| /* VC BAR. Clear first, write mask, then write value */ |
| phys_map_get(chip_id, XIVE_VC, 0, (uint64_t *)&x->vc_base, &x->vc_size); |
| xive_regwx(x, CQ_VC_BAR, 0); |
| if (x->last_reg_error) |
| return false; |
| val = ~(x->vc_size - 1) & CQ_VC_BARM_MASK; |
| xive_regwx(x, CQ_VC_BARM, val); |
| if (x->last_reg_error) |
| return false; |
| val = (uint64_t)x->vc_base | CQ_VC_BAR_VALID; |
| xive_regwx(x, CQ_VC_BAR, val); |
| if (x->last_reg_error) |
| return false; |
| |
| /* Calculate some MMIO bases in the VC BAR */ |
| x->esb_mmio = x->vc_base; |
| x->eq_mmio = x->vc_base + (x->vc_size / VC_MAX_SETS) * VC_ESB_SETS; |
| |
| /* Print things out */ |
| xive_dbg(x, "IC: %14p [0x%012llx/%d]\n", x->ic_base, x->ic_size, |
| x->ic_shift); |
| xive_dbg(x, "TM: %14p [0x%012llx/%d]\n", x->tm_base, x->tm_size, |
| x->tm_shift); |
| xive_dbg(x, "PC: %14p [0x%012llx]\n", x->pc_base, x->pc_size); |
| xive_dbg(x, "VC: %14p [0x%012llx]\n", x->vc_base, x->vc_size); |
| |
| return true; |
| } |
| |
| static void xive_dump_mmio(struct xive *x) |
| { |
| prlog(PR_DEBUG, " CQ_CFG_PB_GEN = %016llx\n", |
| in_be64(x->ic_base + CQ_CFG_PB_GEN)); |
| prlog(PR_DEBUG, " CQ_MSGSND = %016llx\n", |
| in_be64(x->ic_base + CQ_MSGSND)); |
| } |
| |
| static bool xive_config_init(struct xive *x) |
| { |
| uint64_t val __unused; |
| |
| /* Configure PC and VC page sizes and disable Linux trigger mode */ |
| xive_regwx(x, CQ_PBI_CTL, CQ_PBI_PC_64K | CQ_PBI_VC_64K | CQ_PBI_FORCE_TM_LOCAL); |
| if (x->last_reg_error) |
| return false; |
| |
| /*** The rest can use MMIO ***/ |
| |
| #ifdef USE_INDIRECT |
| /* Enable indirect mode in VC config */ |
| val = xive_regr(x, VC_GLOBAL_CONFIG); |
| val |= VC_GCONF_INDIRECT; |
| xive_regw(x, VC_GLOBAL_CONFIG, val); |
| #endif |
| |
| /* Enable indirect mode in PC config */ |
| val = xive_regr(x, PC_GLOBAL_CONFIG); |
| #ifdef USE_INDIRECT |
| val |= PC_GCONF_INDIRECT; |
| #endif |
| val |= PC_GCONF_CHIPID_OVR; |
| val = SETFIELD(PC_GCONF_CHIPID, val, x->block_id); |
| xive_regw(x, PC_GLOBAL_CONFIG, val); |
| xive_dbg(x, "PC_GLOBAL_CONFIG=%016llx\n", val); |
| |
| val = xive_regr(x, PC_TCTXT_CFG); |
| #ifdef USE_BLOCK_GROUP_MODE |
| val |= PC_TCTXT_CFG_BLKGRP_EN | PC_TCTXT_CFG_HARD_CHIPID_BLK; |
| #endif |
| val |= PC_TCTXT_CHIPID_OVERRIDE; |
| val |= PC_TCTXT_CFG_TARGET_EN; |
| val = SETFIELD(PC_TCTXT_CHIPID, val, x->block_id); |
| if (x->rev >= XIVE_REV_2) { |
| val = SETFIELD(PC_TCTXT_INIT_AGE, val, 0x2); |
| val |= PC_TCTXT_CFG_LGS_EN; |
| /* Disable pressure relief as we hijack the field in the VPs */ |
| val &= ~PC_TCTXT_CFG_STORE_ACK; |
| } |
| xive_regw(x, PC_TCTXT_CFG, val); |
| xive_dbg(x, "PC_TCTXT_CFG=%016llx\n", val); |
| |
| /* Subsequent inits are DD2 only */ |
| if (x->rev < XIVE_REV_2) |
| return true; |
| |
| val = xive_regr(x, CQ_CFG_PB_GEN); |
| /* 1-block-per-chip mode */ |
| val = SETFIELD(CQ_INT_ADDR_OPT, val, 2); |
| xive_regw(x, CQ_CFG_PB_GEN, val); |
| |
| /* Enable StoreEOI */ |
| val = xive_regr(x, VC_SBC_CONFIG); |
| val |= VC_SBC_CONF_CPLX_CIST | VC_SBC_CONF_CIST_BOTH; |
| val |= VC_SBC_CONF_NO_UPD_PRF; |
| xive_regw(x, VC_SBC_CONFIG, val); |
| |
| /* Enable block tracking */ |
| val = xive_regr(x, PC_TCTXT_TRACK); |
| val |= PC_TCTXT_TRACK_EN; |
| xive_regw(x, PC_TCTXT_TRACK, val); |
| |
| /* Enable relaxed ordering of trigger forwarding */ |
| val = xive_regr(x, VC_AIB_TX_ORDER_TAG2); |
| val |= VC_AIB_TX_ORDER_TAG2_REL_TF; |
| xive_regw(x, VC_AIB_TX_ORDER_TAG2, val); |
| |
| /* Enable new END s and u bits for silent escalate */ |
| val = xive_regr(x, VC_EQC_CONFIG); |
| val |= VC_EQC_CONF_ENABLE_END_s_BIT; |
| val |= VC_EQC_CONF_ENABLE_END_u_BIT; |
| xive_regw(x, VC_EQC_CONFIG, val); |
| |
| /* Disable error reporting in the FIR for info errors |
| * from the VC. |
| */ |
| xive_regw(x, CQ_FIRMASK_OR, 3ull); |
| |
| return true; |
| } |
| |
| static bool xive_setup_set_xlate(struct xive *x) |
| { |
| unsigned int i; |
| |
| /* Configure EDT for ESBs (aka IPIs) */ |
| xive_regw(x, CQ_TAR, CQ_TAR_TBL_AUTOINC | CQ_TAR_TSEL_EDT); |
| if (x->last_reg_error) |
| return false; |
| for (i = 0; i < VC_ESB_SETS; i++) { |
| xive_regw(x, CQ_TDR, |
| /* IPI type */ |
| (1ull << 62) | |
| /* block ID */ |
| (((uint64_t)x->block_id) << 48) | |
| /* offset */ |
| (((uint64_t)i) << 32)); |
| if (x->last_reg_error) |
| return false; |
| } |
| |
| /* Configure EDT for ENDs (aka EQs) */ |
| for (i = 0; i < VC_END_SETS; i++) { |
| xive_regw(x, CQ_TDR, |
| /* EQ type */ |
| (2ull << 62) | |
| /* block ID */ |
| (((uint64_t)x->block_id) << 48) | |
| /* offset */ |
| (((uint64_t)i) << 32)); |
| if (x->last_reg_error) |
| return false; |
| } |
| |
| /* Configure VDT */ |
| xive_regw(x, CQ_TAR, CQ_TAR_TBL_AUTOINC | CQ_TAR_TSEL_VDT); |
| if (x->last_reg_error) |
| return false; |
| for (i = 0; i < PC_MAX_SETS; i++) { |
| xive_regw(x, CQ_TDR, |
| /* Valid bit */ |
| (1ull << 63) | |
| /* block ID */ |
| (((uint64_t)x->block_id) << 48) | |
| /* offset */ |
| (((uint64_t)i) << 32)); |
| if (x->last_reg_error) |
| return false; |
| } |
| return true; |
| } |
| |
| static bool xive_prealloc_tables(struct xive *x) |
| { |
| uint32_t i __unused, vp_init_count __unused, vp_init_base __unused; |
| uint32_t pbase __unused, pend __unused; |
| uint64_t al __unused; |
| |
| /* ESB/SBE has 4 entries per byte */ |
| x->sbe_base = local_alloc(x->chip_id, SBE_SIZE, SBE_SIZE); |
| if (!x->sbe_base) { |
| xive_err(x, "Failed to allocate SBE\n"); |
| return false; |
| } |
| /* SBEs are initialized to 0b01 which corresponds to "ints off" */ |
| memset(x->sbe_base, 0x55, SBE_SIZE); |
| xive_dbg(x, "SBE at %p size 0x%x\n", x->sbe_base, IVT_SIZE); |
| |
| /* EAS/IVT entries are 8 bytes */ |
| x->ivt_base = local_alloc(x->chip_id, IVT_SIZE, IVT_SIZE); |
| if (!x->ivt_base) { |
| xive_err(x, "Failed to allocate IVT\n"); |
| return false; |
| } |
| /* We clear the entries (non-valid). They will be initialized |
| * when actually used |
| */ |
| memset(x->ivt_base, 0, IVT_SIZE); |
| xive_dbg(x, "IVT at %p size 0x%x\n", x->ivt_base, IVT_SIZE); |
| |
| #ifdef USE_INDIRECT |
| /* Indirect EQ table. (XXX Align to 64K until I figure out the |
| * HW requirements) |
| */ |
| al = (IND_EQ_TABLE_SIZE + 0xffff) & ~0xffffull; |
| x->eq_ind_base = local_alloc(x->chip_id, al, al); |
| if (!x->eq_ind_base) { |
| xive_err(x, "Failed to allocate EQ indirect table\n"); |
| return false; |
| } |
| memset(x->eq_ind_base, 0, al); |
| xive_dbg(x, "EQi at %p size 0x%llx\n", x->eq_ind_base, al); |
| x->eq_ind_count = IND_EQ_TABLE_SIZE / 8; |
| |
| /* Indirect VP table. (XXX Align to 64K until I figure out the |
| * HW requirements) |
| */ |
| al = (IND_VP_TABLE_SIZE + 0xffff) & ~0xffffull; |
| x->vp_ind_base = local_alloc(x->chip_id, al, al); |
| if (!x->vp_ind_base) { |
| xive_err(x, "Failed to allocate VP indirect table\n"); |
| return false; |
| } |
| xive_dbg(x, "VPi at %p size 0x%llx\n", x->vp_ind_base, al); |
| x->vp_ind_count = IND_VP_TABLE_SIZE / 8; |
| memset(x->vp_ind_base, 0, al); |
| |
| /* Populate/initialize VP/EQs indirect backing */ |
| #ifdef USE_BLOCK_GROUP_MODE |
| vp_init_count = INITIAL_VP_COUNT; |
| vp_init_base = INITIAL_VP_BASE; |
| #else |
| vp_init_count = x->block_id == 0 ? INITIAL_BLK0_VP_COUNT : 0; |
| vp_init_base = INITIAL_BLK0_VP_BASE; |
| #endif |
| |
| /* Allocate pages for some VPs in indirect mode */ |
| pbase = vp_init_base / VP_PER_PAGE; |
| pend = (vp_init_base + vp_init_count) / VP_PER_PAGE; |
| |
| xive_dbg(x, "Allocating pages %d to %d of VPs (for %d VPs)\n", |
| pbase, pend, vp_init_count); |
| for (i = pbase; i <= pend; i++) { |
| void *page; |
| u64 vsd; |
| |
| /* Indirect entries have a VSD format */ |
| page = local_alloc(x->chip_id, 0x10000, 0x10000); |
| if (!page) { |
| xive_err(x, "Failed to allocate VP page\n"); |
| return false; |
| } |
| xive_dbg(x, "VP%d at %p size 0x%x\n", i, page, 0x10000); |
| memset(page, 0, 0x10000); |
| vsd = ((uint64_t)page) & VSD_ADDRESS_MASK; |
| |
| vsd |= SETFIELD(VSD_TSIZE, 0ull, 4); |
| vsd |= SETFIELD(VSD_MODE, 0ull, VSD_MODE_EXCLUSIVE); |
| vsd |= VSD_FIRMWARE; |
| x->vp_ind_base[i] = vsd; |
| } |
| |
| #else /* USE_INDIRECT */ |
| |
| /* Allocate direct EQ and VP tables */ |
| x->eq_base = local_alloc(x->chip_id, EQT_SIZE, EQT_SIZE); |
| if (!x->eq_base) { |
| xive_err(x, "Failed to allocate EQ table\n"); |
| return false; |
| } |
| memset(x->eq_base, 0, EQT_SIZE); |
| x->vp_base = local_alloc(x->chip_id, VPT_SIZE, VPT_SIZE); |
| if (!x->vp_base) { |
| xive_err(x, "Failed to allocate VP table\n"); |
| return false; |
| } |
| /* We clear the entries (non-valid). They will be initialized |
| * when actually used |
| */ |
| memset(x->vp_base, 0, VPT_SIZE); |
| #endif /* USE_INDIRECT */ |
| |
| /* Allocate the queue overflow pages */ |
| x->q_ovf = local_alloc(x->chip_id, VC_QUEUE_OVF_COUNT * 0x10000, 0x10000); |
| if (!x->q_ovf) { |
| xive_err(x, "Failed to allocate queue overflow\n"); |
| return false; |
| } |
| return true; |
| } |
| |
| #ifdef USE_INDIRECT |
| static void xive_add_provisioning_properties(void) |
| { |
| uint32_t chips[XIVE_MAX_CHIPS]; |
| uint32_t i, count; |
| |
| dt_add_property_cells(xive_dt_node, |
| "ibm,xive-provision-page-size", 0x10000); |
| |
| #ifdef USE_BLOCK_GROUP_MODE |
| count = 1 << xive_chips_alloc_bits; |
| #else |
| count = xive_block_count; |
| #endif |
| for (i = 0; i < count; i++) |
| chips[i] = xive_block_to_chip[i]; |
| dt_add_property(xive_dt_node, "ibm,xive-provision-chips", |
| chips, 4 * count); |
| } |
| #else |
| static inline void xive_add_provisioning_properties(void) { } |
| #endif |
| |
| static void xive_create_mmio_dt_node(struct xive *x) |
| { |
| uint64_t tb = (uint64_t)x->tm_base; |
| uint32_t stride = 1u << x->tm_shift; |
| |
| xive_dt_node = dt_new_addr(dt_root, "interrupt-controller", tb); |
| assert(xive_dt_node); |
| |
| dt_add_property_u64s(xive_dt_node, "reg", |
| tb + 0 * stride, stride, |
| tb + 1 * stride, stride, |
| tb + 2 * stride, stride, |
| tb + 3 * stride, stride); |
| |
| dt_add_property_strings(xive_dt_node, "compatible", |
| "ibm,opal-xive-pe", "ibm,opal-intc"); |
| |
| dt_add_property_cells(xive_dt_node, "ibm,xive-eq-sizes", |
| 12, 16, 21, 24); |
| |
| dt_add_property_cells(xive_dt_node, "ibm,xive-#priorities", 8); |
| |
| xive_add_provisioning_properties(); |
| } |
| |
| static void xive_setup_forward_ports(struct xive *x, struct proc_chip *remote_chip) |
| { |
| struct xive *remote_xive = remote_chip->xive; |
| uint64_t base = SETFIELD(VSD_MODE, 0ull, VSD_MODE_FORWARD); |
| uint32_t remote_id = remote_xive->block_id; |
| uint64_t nport; |
| |
| /* ESB(SBE), EAS(IVT) and END(EQ) point to the notify port */ |
| nport = ((uint64_t)remote_xive->ic_base) + (1ul << remote_xive->ic_shift); |
| if (!xive_set_vsd(x, VST_TSEL_IVT, remote_id, base | nport)) |
| goto error; |
| if (!xive_set_vsd(x, VST_TSEL_SBE, remote_id, base | nport)) |
| goto error; |
| if (!xive_set_vsd(x, VST_TSEL_EQDT, remote_id, base | nport)) |
| goto error; |
| |
| /* NVT/VPD points to the remote NVT MMIO sets */ |
| if (!xive_set_vsd(x, VST_TSEL_VPDT, remote_id, |
| base | ((uint64_t)remote_xive->pc_base) | |
| SETFIELD(VSD_TSIZE, 0ull, ilog2(x->pc_size) - 12))) |
| goto error; |
| |
| return; |
| |
| error: |
| xive_err(x, "Failure configuring forwarding ports\n"); |
| } |
| |
| static void late_init_one_xive(struct xive *x) |
| { |
| struct proc_chip *chip; |
| |
| /* We need to setup the cross-chip forward ports. Let's |
| * iterate all chip and set them up accordingly |
| */ |
| for_each_chip(chip) { |
| /* We skip ourselves or chips without a xive */ |
| if (chip->xive == x || !chip->xive) |
| continue; |
| |
| /* Setup our forward ports to that chip */ |
| xive_setup_forward_ports(x, chip); |
| } |
| } |
| |
| static bool xive_check_ipi_free(struct xive *x, uint32_t irq, uint32_t count) |
| { |
| uint32_t i, idx = GIRQ_TO_IDX(irq); |
| |
| for (i = 0; i < count; i++) |
| if (bitmap_tst_bit(*x->ipi_alloc_map, idx + i)) |
| return false; |
| return true; |
| } |
| |
| uint32_t xive_alloc_hw_irqs(uint32_t chip_id, uint32_t count, uint32_t align) |
| { |
| struct proc_chip *chip = get_chip(chip_id); |
| struct xive *x; |
| uint32_t base, i; |
| |
| assert(chip); |
| assert(is_pow2(align)); |
| |
| x = chip->xive; |
| assert(x); |
| |
| lock(&x->lock); |
| |
| /* Allocate the HW interrupts */ |
| base = x->int_hw_bot - count; |
| base &= ~(align - 1); |
| if (base < x->int_ipi_top) { |
| xive_err(x, |
| "HW alloc request for %d interrupts aligned to %d failed\n", |
| count, align); |
| unlock(&x->lock); |
| return XIVE_IRQ_ERROR; |
| } |
| if (!xive_check_ipi_free(x, base, count)) { |
| xive_err(x, "HWIRQ boot allocator request overlaps dynamic allocator\n"); |
| unlock(&x->lock); |
| return XIVE_IRQ_ERROR; |
| } |
| |
| x->int_hw_bot = base; |
| |
| /* Initialize the corresponding IVT entries to sane defaults, |
| * IE entry is valid, not routed and masked, EQ data is set |
| * to the GIRQ number. |
| */ |
| for (i = 0; i < count; i++) { |
| struct xive_ive *ive = xive_get_ive(x, base + i); |
| |
| ive->w = IVE_VALID | IVE_MASKED | SETFIELD(IVE_EQ_DATA, 0ul, base + i); |
| } |
| |
| unlock(&x->lock); |
| return base; |
| } |
| |
| uint32_t xive_alloc_ipi_irqs(uint32_t chip_id, uint32_t count, uint32_t align) |
| { |
| struct proc_chip *chip = get_chip(chip_id); |
| struct xive *x; |
| uint32_t base, i; |
| |
| assert(chip); |
| assert(is_pow2(align)); |
| |
| x = chip->xive; |
| assert(x); |
| |
| lock(&x->lock); |
| |
| /* Allocate the IPI interrupts */ |
| base = x->int_ipi_top + (align - 1); |
| base &= ~(align - 1); |
| if (base >= x->int_hw_bot) { |
| xive_err(x, |
| "IPI alloc request for %d interrupts aligned to %d failed\n", |
| count, align); |
| unlock(&x->lock); |
| return XIVE_IRQ_ERROR; |
| } |
| if (!xive_check_ipi_free(x, base, count)) { |
| xive_err(x, "IPI boot allocator request overlaps dynamic allocator\n"); |
| unlock(&x->lock); |
| return XIVE_IRQ_ERROR; |
| } |
| |
| x->int_ipi_top = base + count; |
| |
| /* Initialize the corresponding IVT entries to sane defaults, |
| * IE entry is valid, not routed and masked, EQ data is set |
| * to the GIRQ number. |
| */ |
| for (i = 0; i < count; i++) { |
| struct xive_ive *ive = xive_get_ive(x, base + i); |
| |
| ive->w = IVE_VALID | IVE_MASKED | |
| SETFIELD(IVE_EQ_DATA, 0ul, base + i); |
| } |
| |
| unlock(&x->lock); |
| return base; |
| } |
| |
| void *xive_get_trigger_port(uint32_t girq) |
| { |
| uint32_t idx = GIRQ_TO_IDX(girq); |
| struct xive *x; |
| |
| /* Find XIVE on which the IVE resides */ |
| x = xive_from_isn(girq); |
| if (!x) |
| return NULL; |
| |
| if (GIRQ_IS_ESCALATION(girq)) { |
| /* There is no trigger page for escalation interrupts */ |
| return NULL; |
| } else { |
| /* Make sure it's an IPI on that chip */ |
| if (girq < x->int_base || |
| girq >= x->int_ipi_top) |
| return NULL; |
| |
| return x->esb_mmio + idx * 0x20000; |
| } |
| } |
| |
| uint64_t xive_get_notify_port(uint32_t chip_id, uint32_t ent) |
| { |
| struct proc_chip *chip = get_chip(chip_id); |
| struct xive *x; |
| uint32_t offset = 0; |
| |
| assert(chip); |
| x = chip->xive; |
| assert(x); |
| |
| /* This is where we can assign a different HW queue to a different |
| * source by offsetting into the cache lines of the notify port |
| * |
| * For now we keep it very basic, this will have to be looked at |
| * again on real HW with some proper performance analysis. |
| * |
| * Here's what Florian says on the matter: |
| * |
| * << |
| * The first 2k of the notify port page can all be used for PCIe triggers |
| * |
| * However the idea would be that we try to use the first 4 cache lines to |
| * balance the PCIe Interrupt requests to use the least used snoop buses |
| * (we went from 2 to 4 snoop buses for P9). snoop 0 is heavily used |
| * (I think TLBIs are using that in addition to the normal addresses), |
| * snoop 3 is used for all Int commands, so I think snoop 2 (CL 2 in the |
| * page) is the least used overall. So we probably should that one for |
| * the Int commands from PCIe. |
| * |
| * In addition, our EAS cache supports hashing to provide "private" cache |
| * areas for the PHBs in the shared 1k EAS cache. This allows e.g. to avoid |
| * that one "thrashing" PHB thrashes the EAS cache for everyone, or provide |
| * a PHB with a private area that would allow high cache hits in case of a |
| * device using very few interrupts. The hashing is based on the offset within |
| * the cache line. So using that, you can e.g. set the EAS cache up so that |
| * IPIs use 512 entries, the x16 PHB uses 256 entries and the x8 PHBs 128 |
| * entries each - or IPIs using all entries and sharing with PHBs, so PHBs |
| * would use 512 entries and 256 entries respectively. |
| * |
| * This is a tuning we would probably do later in the lab, but as a "prep" |
| * we should set up the different PHBs such that they are using different |
| * 8B-aligned offsets within the cache line, so e.g. |
| * PH4_0 addr 0x100 (CL 2 DW0 |
| * PH4_1 addr 0x108 (CL 2 DW1) |
| * PH4_2 addr 0x110 (CL 2 DW2) |
| * etc. |
| * >> |
| * |
| * I'm using snoop1 for PHB0 and snoop2 for everybody else. |
| */ |
| switch(ent) { |
| case XIVE_HW_SRC_PHBn(0): |
| offset = 0x100; |
| break; |
| case XIVE_HW_SRC_PHBn(1): |
| offset = 0x208; |
| break; |
| case XIVE_HW_SRC_PHBn(2): |
| offset = 0x210; |
| break; |
| case XIVE_HW_SRC_PHBn(3): |
| offset = 0x218; |
| break; |
| case XIVE_HW_SRC_PHBn(4): |
| offset = 0x220; |
| break; |
| case XIVE_HW_SRC_PHBn(5): |
| offset = 0x228; |
| break; |
| case XIVE_HW_SRC_PSI: |
| offset = 0x230; |
| break; |
| default: |
| assert(false); |
| return 0; |
| } |
| |
| /* Notify port is the second page of the IC BAR */ |
| return ((uint64_t)x->ic_base) + (1ul << x->ic_shift) + offset; |
| } |
| |
| /* Manufacture the powerbus packet bits 32:63 */ |
| __attrconst uint32_t xive_get_notify_base(uint32_t girq) |
| { |
| return (GIRQ_TO_BLK(girq) << 28) | GIRQ_TO_IDX(girq); |
| } |
| |
| static bool xive_get_irq_targetting(uint32_t isn, uint32_t *out_target, |
| uint8_t *out_prio, uint32_t *out_lirq) |
| { |
| struct xive_ive *ive; |
| struct xive *x, *eq_x; |
| struct xive_eq *eq; |
| uint32_t eq_blk, eq_idx; |
| uint32_t vp_blk __unused, vp_idx; |
| uint32_t prio, server; |
| bool is_escalation = GIRQ_IS_ESCALATION(isn); |
| |
| /* Find XIVE on which the IVE resides */ |
| x = xive_from_isn(isn); |
| if (!x) |
| return false; |
| /* Grab the IVE */ |
| ive = xive_get_ive(x, isn); |
| if (!ive) |
| return false; |
| if (!(ive->w & IVE_VALID) && !is_escalation) { |
| xive_err(x, "ISN %x lead to invalid IVE !\n", isn); |
| return false; |
| } |
| |
| if (out_lirq) |
| *out_lirq = GETFIELD(IVE_EQ_DATA, ive->w); |
| |
| /* Find the EQ and its xive instance */ |
| eq_blk = GETFIELD(IVE_EQ_BLOCK, ive->w); |
| eq_idx = GETFIELD(IVE_EQ_INDEX, ive->w); |
| eq_x = xive_from_vc_blk(eq_blk); |
| |
| /* This can fail if the interrupt hasn't been initialized yet |
| * but it should also be masked, so fail silently |
| */ |
| if (!eq_x) |
| goto pick_default; |
| eq = xive_get_eq(eq_x, eq_idx); |
| if (!eq) |
| goto pick_default; |
| |
| /* XXX Check valid and format 0 */ |
| |
| /* No priority conversion, return the actual one ! */ |
| if (ive->w & IVE_MASKED) |
| prio = 0xff; |
| else |
| prio = GETFIELD(EQ_W7_F0_PRIORITY, eq->w7); |
| if (out_prio) |
| *out_prio = prio; |
| |
| vp_blk = GETFIELD(EQ_W6_NVT_BLOCK, eq->w6); |
| vp_idx = GETFIELD(EQ_W6_NVT_INDEX, eq->w6); |
| server = VP2PIR(vp_blk, vp_idx); |
| |
| if (out_target) |
| *out_target = server; |
| |
| xive_vdbg(eq_x, "EQ info for ISN %x: prio=%d, server=0x%x (VP %x/%x)\n", |
| isn, prio, server, vp_blk, vp_idx); |
| return true; |
| |
| pick_default: |
| xive_vdbg(eq_x, "EQ info for ISN %x: Using masked defaults\n", isn); |
| |
| if (out_prio) |
| *out_prio = 0xff; |
| /* Pick a random default, me will be fine ... */ |
| if (out_target) |
| *out_target = mfspr(SPR_PIR); |
| return true; |
| } |
| |
| static inline bool xive_eq_for_target(uint32_t target, uint8_t prio, |
| uint32_t *out_eq_blk, |
| uint32_t *out_eq_idx) |
| { |
| struct xive *x; |
| struct xive_vp *vp; |
| uint32_t vp_blk, vp_idx; |
| uint32_t eq_blk, eq_idx; |
| |
| if (prio > 7) |
| return false; |
| |
| /* Get the VP block/index from the target word */ |
| if (!xive_decode_vp(target, &vp_blk, &vp_idx, NULL, NULL)) |
| return false; |
| |
| /* Grab the target VP's XIVE */ |
| x = xive_from_pc_blk(vp_blk); |
| if (!x) |
| return false; |
| |
| /* Find the VP structrure where we stashed the EQ number */ |
| vp = xive_get_vp(x, vp_idx); |
| if (!vp) |
| return false; |
| |
| /* Grab it, it's in the pressure relief interrupt field, |
| * top 4 bits are the block (word 1). |
| */ |
| eq_blk = vp->w1 >> 28; |
| eq_idx = vp->w1 & 0x0fffffff; |
| |
| /* Currently the EQ block and VP block should be the same */ |
| if (eq_blk != vp_blk) { |
| xive_err(x, "eq_blk != vp_blk (%d vs. %d) for target 0x%08x/%d\n", |
| eq_blk, vp_blk, target, prio); |
| assert(false); |
| } |
| |
| if (out_eq_blk) |
| *out_eq_blk = eq_blk; |
| if (out_eq_idx) |
| *out_eq_idx = eq_idx + prio; |
| |
| return true; |
| } |
| |
| static int64_t xive_set_irq_targetting(uint32_t isn, uint32_t target, |
| uint8_t prio, uint32_t lirq, |
| bool synchronous) |
| { |
| struct xive *x; |
| struct xive_ive *ive; |
| uint32_t eq_blk, eq_idx; |
| bool is_escalation = GIRQ_IS_ESCALATION(isn); |
| uint64_t new_ive; |
| int64_t rc; |
| |
| /* Find XIVE on which the IVE resides */ |
| x = xive_from_isn(isn); |
| if (!x) |
| return OPAL_PARAMETER; |
| /* Grab the IVE */ |
| ive = xive_get_ive(x, isn); |
| if (!ive) |
| return OPAL_PARAMETER; |
| if (!(ive->w & IVE_VALID) && !is_escalation) { |
| xive_err(x, "ISN %x lead to invalid IVE !\n", isn); |
| return OPAL_PARAMETER; |
| } |
| |
| lock(&x->lock); |
| |
| /* If using emulation mode, fixup prio to the only supported one */ |
| if (xive_mode == XIVE_MODE_EMU && prio != 0xff) |
| prio = XIVE_EMULATION_PRIO; |
| |
| /* Read existing IVE */ |
| new_ive = ive->w; |
| |
| /* Are we masking ? */ |
| if (prio == 0xff && !is_escalation) { |
| new_ive |= IVE_MASKED; |
| xive_vdbg(x, "ISN %x masked !\n", isn); |
| |
| /* Put prio 7 in the EQ */ |
| prio = 7; |
| } else { |
| /* Unmasking */ |
| new_ive = ive->w & ~IVE_MASKED; |
| xive_vdbg(x, "ISN %x unmasked !\n", isn); |
| |
| /* For normal interrupt sources, keep track of which ones |
| * we ever enabled since the last reset |
| */ |
| if (!is_escalation) |
| bitmap_set_bit(*x->int_enabled_map, GIRQ_TO_IDX(isn)); |
| } |
| |
| /* If prio isn't 0xff, re-target the IVE. First find the EQ |
| * correponding to the target |
| */ |
| if (prio != 0xff) { |
| if (!xive_eq_for_target(target, prio, &eq_blk, &eq_idx)) { |
| xive_err(x, "Can't find EQ for target/prio 0x%x/%d\n", |
| target, prio); |
| unlock(&x->lock); |
| return OPAL_PARAMETER; |
| } |
| |
| /* Try to update it atomically to avoid an intermediary |
| * stale state |
| */ |
| new_ive = SETFIELD(IVE_EQ_BLOCK, new_ive, eq_blk); |
| new_ive = SETFIELD(IVE_EQ_INDEX, new_ive, eq_idx); |
| } |
| new_ive = SETFIELD(IVE_EQ_DATA, new_ive, lirq); |
| |
| xive_vdbg(x,"ISN %x routed to eq %x/%x lirq=%08x IVE=%016llx !\n", |
| isn, eq_blk, eq_idx, lirq, new_ive); |
| |
| /* Updating the cache differs between real IVEs and escalation |
| * IVEs inside an EQ |
| */ |
| if (is_escalation) { |
| rc = xive_eqc_cache_update(x, x->block_id, GIRQ_TO_IDX(isn), |
| 2, 1, &new_ive, true, synchronous); |
| } else { |
| sync(); |
| ive->w = new_ive; |
| rc = xive_ivc_scrub(x, x->block_id, GIRQ_TO_IDX(isn)); |
| } |
| |
| unlock(&x->lock); |
| return rc; |
| } |
| |
| static int64_t xive_source_get_xive(struct irq_source *is __unused, |
| uint32_t isn, uint16_t *server, |
| uint8_t *prio) |
| { |
| uint32_t target_id; |
| |
| if (xive_get_irq_targetting(isn, &target_id, prio, NULL)) { |
| *server = target_id << 2; |
| return OPAL_SUCCESS; |
| } else |
| return OPAL_PARAMETER; |
| } |
| |
| static void xive_update_irq_mask(struct xive_src *s, uint32_t idx, bool masked) |
| { |
| void *mmio_base = s->esb_mmio + (1ul << s->esb_shift) * idx; |
| uint32_t offset; |
| |
| /* XXX FIXME: A quick mask/umask can make us shoot an interrupt |
| * more than once to a queue. We need to keep track better |
| */ |
| if (s->flags & XIVE_SRC_EOI_PAGE1) |
| mmio_base += 1ull << (s->esb_shift - 1); |
| if (masked) |
| offset = 0xd00; /* PQ = 01 */ |
| else |
| offset = 0xc00; /* PQ = 00 */ |
| |
| if (s->flags & XIVE_SRC_SHIFT_BUG) |
| offset <<= 4; |
| |
| in_be64(mmio_base + offset); |
| } |
| |
| static int64_t xive_sync(struct xive *x) |
| { |
| uint64_t r; |
| void *p; |
| |
| lock(&x->lock); |
| |
| /* Second 2K range of second page */ |
| p = x->ic_base + (1 << x->ic_shift) + 0x800; |
| |
| /* TODO: Make this more fine grained */ |
| out_be64(p + (10 << 7), 0); /* Sync OS escalations */ |
| out_be64(p + (11 << 7), 0); /* Sync Hyp escalations */ |
| out_be64(p + (12 << 7), 0); /* Sync Redistribution */ |
| out_be64(p + ( 8 << 7), 0); /* Sync IPI */ |
| out_be64(p + ( 9 << 7), 0); /* Sync HW */ |
| |
| #define SYNC_MASK \ |
| (VC_EQC_CONF_SYNC_IPI | \ |
| VC_EQC_CONF_SYNC_HW | \ |
| VC_EQC_CONF_SYNC_ESC1 | \ |
| VC_EQC_CONF_SYNC_ESC2 | \ |
| VC_EQC_CONF_SYNC_REDI) |
| |
| /* XXX Add timeout */ |
| for (;;) { |
| r = xive_regrx(x, VC_EQC_CONFIG); |
| if ((r & SYNC_MASK) == SYNC_MASK) |
| break; |
| cpu_relax(); |
| } |
| xive_regw(x, VC_EQC_CONFIG, r & ~SYNC_MASK); |
| |
| /* Workaround HW issue, read back before allowing a new sync */ |
| xive_regr(x, VC_GLOBAL_CONFIG); |
| |
| unlock(&x->lock); |
| |
| return 0; |
| } |
| |
| static int64_t __xive_set_irq_config(struct irq_source *is, uint32_t girq, |
| uint64_t vp, uint8_t prio, uint32_t lirq, |
| bool update_esb, bool no_sync) |
| { |
| struct xive_src *s = container_of(is, struct xive_src, is); |
| uint32_t old_target, vp_blk; |
| u8 old_prio; |
| int64_t rc; |
| |
| /* Grab existing target */ |
| if (!xive_get_irq_targetting(girq, &old_target, &old_prio, NULL)) |
| return OPAL_PARAMETER; |
| |
| /* Let XIVE configure the EQ. We do the update without the |
| * synchronous flag, thus a cache update failure will result |
| * in us returning OPAL_BUSY |
| */ |
| rc = xive_set_irq_targetting(girq, vp, prio, lirq, false); |
| if (rc) |
| return rc; |
| |
| /* Do we need to update the mask ? */ |
| if (old_prio != prio && (old_prio == 0xff || prio == 0xff)) { |
| /* The source has special variants of masking/unmasking */ |
| if (s->orig_ops && s->orig_ops->set_xive) { |
| /* We don't pass as server on source ops ! Targetting |
| * is handled by the XIVE |
| */ |
| rc = s->orig_ops->set_xive(is, girq, 0, prio); |
| } else if (update_esb) { |
| /* Ensure it's enabled/disabled in the source |
| * controller |
| */ |
| xive_update_irq_mask(s, girq - s->esb_base, |
| prio == 0xff); |
| } |
| } |
| |
| /* |
| * Synchronize the source and old target XIVEs to ensure that |
| * all pending interrupts to the old target have reached their |
| * respective queue. |
| * |
| * WARNING: This assumes the VP and it's queues are on the same |
| * XIVE instance ! |
| */ |
| if (no_sync) |
| return OPAL_SUCCESS; |
| xive_sync(s->xive); |
| if (xive_decode_vp(old_target, &vp_blk, NULL, NULL, NULL)) { |
| struct xive *x = xive_from_pc_blk(vp_blk); |
| if (x) |
| xive_sync(x); |
| } |
| |
| return OPAL_SUCCESS; |
| } |
| |
| static int64_t xive_set_irq_config(uint32_t girq, uint64_t vp, uint8_t prio, |
| uint32_t lirq, bool update_esb) |
| { |
| struct irq_source *is = irq_find_source(girq); |
| |
| return __xive_set_irq_config(is, girq, vp, prio, lirq, update_esb, |
| false); |
| } |
| |
| static int64_t xive_source_set_xive(struct irq_source *is, |
| uint32_t isn, uint16_t server, uint8_t prio) |
| { |
| /* |
| * WARNING: There is an inherent race with the use of the |
| * mask bit in the EAS/IVT. When masked, interrupts are "lost" |
| * but their P/Q bits are still set. So when unmasking, one has |
| * to check the P bit and possibly trigger a resend. |
| * |
| * We "deal" with it by relying on the fact that the OS will |
| * lazy disable MSIs. Thus mask will only be called if the |
| * interrupt occurred while already logically masked. Thus |
| * losing subsequent occurrences is of no consequences, we just |
| * need to "cleanup" P and Q when unmasking. |
| * |
| * This needs to be documented in the OPAL APIs |
| */ |
| |
| /* Unmangle server */ |
| server >>= 2; |
| |
| /* Set logical irq to match isn */ |
| return __xive_set_irq_config(is, isn, server, prio, isn, true, false); |
| } |
| |
| void __xive_source_eoi(struct irq_source *is, uint32_t isn) |
| { |
| struct xive_src *s = container_of(is, struct xive_src, is); |
| uint32_t idx = isn - s->esb_base; |
| struct xive_ive *ive; |
| void *mmio_base; |
| uint64_t eoi_val; |
| |
| /* Grab the IVE */ |
| ive = s->xive->ivt_base; |
| if (!ive) |
| return; |
| ive += GIRQ_TO_IDX(isn); |
| |
| /* XXX To fix the races with mask/unmask potentially causing |
| * multiple queue entries, we need to keep track of EOIs here, |
| * before the masked test below |
| */ |
| |
| /* If it's invalid or masked, don't do anything */ |
| if ((ive->w & IVE_MASKED) || !(ive->w & IVE_VALID)) |
| return; |
| |
| /* Grab MMIO control address for that ESB */ |
| mmio_base = s->esb_mmio + (1ull << s->esb_shift) * idx; |
| |
| /* If the XIVE supports the new "store EOI facility, use it */ |
| if (s->flags & XIVE_SRC_STORE_EOI) |
| out_be64(mmio_base + 0x400, 0); |
| else { |
| uint64_t offset; |
| |
| /* Otherwise for EOI, we use the special MMIO that does |
| * a clear of both P and Q and returns the old Q. |
| * |
| * This allows us to then do a re-trigger if Q was set |
| * rather than synthetizing an interrupt in software |
| */ |
| if (s->flags & XIVE_SRC_EOI_PAGE1) |
| mmio_base += 1ull << (s->esb_shift - 1); |
| |
| /* LSIs don't need anything special, just EOI */ |
| if (s->flags & XIVE_SRC_LSI) |
| in_be64(mmio_base); |
| else { |
| offset = 0xc00; |
| if (s->flags & XIVE_SRC_SHIFT_BUG) |
| offset <<= 4; |
| eoi_val = in_be64(mmio_base + offset); |
| xive_vdbg(s->xive, "ISN: %08x EOI=%llx\n", |
| isn, eoi_val); |
| if (!(eoi_val & 1)) |
| return; |
| |
| /* Re-trigger always on page0 or page1 ? */ |
| out_be64(mmio_base, 0); |
| } |
| } |
| } |
| |
| static void xive_source_eoi(struct irq_source *is, uint32_t isn) |
| { |
| struct xive_src *s = container_of(is, struct xive_src, is); |
| |
| if (s->orig_ops && s->orig_ops->eoi) |
| s->orig_ops->eoi(is, isn); |
| else |
| __xive_source_eoi(is, isn); |
| } |
| |
| static void xive_source_interrupt(struct irq_source *is, uint32_t isn) |
| { |
| struct xive_src *s = container_of(is, struct xive_src, is); |
| |
| if (!s->orig_ops || !s->orig_ops->interrupt) |
| return; |
| s->orig_ops->interrupt(is, isn); |
| } |
| |
| static uint64_t xive_source_attributes(struct irq_source *is, uint32_t isn) |
| { |
| struct xive_src *s = container_of(is, struct xive_src, is); |
| |
| if (!s->orig_ops || !s->orig_ops->attributes) |
| return IRQ_ATTR_TARGET_LINUX; |
| return s->orig_ops->attributes(is, isn); |
| } |
| |
| static char *xive_source_name(struct irq_source *is, uint32_t isn) |
| { |
| struct xive_src *s = container_of(is, struct xive_src, is); |
| |
| if (!s->orig_ops || !s->orig_ops->name) |
| return NULL; |
| return s->orig_ops->name(is, isn); |
| } |
| |
| static const struct irq_source_ops xive_irq_source_ops = { |
| .get_xive = xive_source_get_xive, |
| .set_xive = xive_source_set_xive, |
| .eoi = xive_source_eoi, |
| .interrupt = xive_source_interrupt, |
| .attributes = xive_source_attributes, |
| .name = xive_source_name, |
| }; |
| |
| static void __xive_register_source(struct xive *x, struct xive_src *s, |
| uint32_t base, uint32_t count, |
| uint32_t shift, void *mmio, uint32_t flags, |
| bool secondary, void *data, |
| const struct irq_source_ops *orig_ops) |
| { |
| s->esb_base = base; |
| s->esb_shift = shift; |
| s->esb_mmio = mmio; |
| s->flags = flags; |
| s->orig_ops = orig_ops; |
| s->xive = x; |
| s->is.start = base; |
| s->is.end = base + count; |
| s->is.ops = &xive_irq_source_ops; |
| s->is.data = data; |
| |
| __register_irq_source(&s->is, secondary); |
| } |
| |
| void xive_register_hw_source(uint32_t base, uint32_t count, uint32_t shift, |
| void *mmio, uint32_t flags, void *data, |
| const struct irq_source_ops *ops) |
| { |
| struct xive_src *s; |
| struct xive *x = xive_from_isn(base); |
| |
| assert(x); |
| |
| s = malloc(sizeof(struct xive_src)); |
| assert(s); |
| __xive_register_source(x, s, base, count, shift, mmio, flags, |
| false, data, ops); |
| } |
| |
| void xive_register_ipi_source(uint32_t base, uint32_t count, void *data, |
| const struct irq_source_ops *ops) |
| { |
| struct xive_src *s; |
| struct xive *x = xive_from_isn(base); |
| uint32_t base_idx = GIRQ_TO_IDX(base); |
| void *mmio_base; |
| uint32_t flags = XIVE_SRC_EOI_PAGE1 | XIVE_SRC_TRIGGER_PAGE; |
| |
| assert(x); |
| assert(base >= x->int_base && (base + count) <= x->int_ipi_top); |
| |
| s = malloc(sizeof(struct xive_src)); |
| assert(s); |
| |
| /* Store EOI supported on DD2.0 */ |
| if (x->rev >= XIVE_REV_2) |
| flags |= XIVE_SRC_STORE_EOI; |
| |
| /* Callbacks assume the MMIO base corresponds to the first |
| * interrupt of that source structure so adjust it |
| */ |
| mmio_base = x->esb_mmio + (1ul << IPI_ESB_SHIFT) * base_idx; |
| __xive_register_source(x, s, base, count, IPI_ESB_SHIFT, mmio_base, |
| flags, false, data, ops); |
| } |
| |
| static struct xive *init_one_xive(struct dt_node *np) |
| { |
| struct xive *x; |
| struct proc_chip *chip; |
| uint32_t flags; |
| |
| x = zalloc(sizeof(struct xive)); |
| assert(x); |
| x->x_node = np; |
| x->xscom_base = dt_get_address(np, 0, NULL); |
| x->chip_id = dt_get_chip_id(np); |
| |
| /* "Allocate" a new block ID for the chip */ |
| x->block_id = xive_block_count++; |
| assert (x->block_id < XIVE_MAX_CHIPS); |
| xive_block_to_chip[x->block_id] = x->chip_id; |
| init_lock(&x->lock); |
| |
| chip = get_chip(x->chip_id); |
| assert(chip); |
| |
| x->rev = XIVE_REV_UNKNOWN; |
| if (chip->type == PROC_CHIP_P9_NIMBUS) { |
| if ((chip->ec_level & 0xf0) == 0x10) |
| x->rev = XIVE_REV_1; |
| else if ((chip->ec_level & 0xf0) == 0x20) |
| x->rev = XIVE_REV_2; |
| } else if (chip->type == PROC_CHIP_P9_CUMULUS) |
| x->rev = XIVE_REV_2; |
| |
| xive_dbg(x, "Initializing rev %d block ID %d...\n", |
| x->rev, x->block_id); |
| chip->xive = x; |
| |
| #ifdef USE_INDIRECT |
| list_head_init(&x->donated_pages); |
| #endif |
| /* Base interrupt numbers and allocator init */ |
| /* XXX Consider allocating half as many ESBs than MMIO space |
| * so that HW sources land outside of ESB space... |
| */ |
| x->int_base = BLKIDX_TO_GIRQ(x->block_id, 0); |
| x->int_max = x->int_base + MAX_INT_ENTRIES; |
| x->int_hw_bot = x->int_max; |
| x->int_ipi_top = x->int_base; |
| |
| /* Make sure we never hand out "2" as it's reserved for XICS emulation |
| * IPI returns. Generally start handing out at 0x10 |
| */ |
| if (x->int_ipi_top < XIVE_INT_SAFETY_GAP) |
| x->int_ipi_top = XIVE_INT_SAFETY_GAP; |
| |
| /* Allocate a few bitmaps */ |
| x->eq_map = zalloc(BITMAP_BYTES(MAX_EQ_COUNT >> 3)); |
| assert(x->eq_map); |
| /* Make sure we don't hand out 0 */ |
| bitmap_set_bit(*x->eq_map, 0); |
| |
| x->int_enabled_map = zalloc(BITMAP_BYTES(MAX_INT_ENTRIES)); |
| assert(x->int_enabled_map); |
| x->ipi_alloc_map = zalloc(BITMAP_BYTES(MAX_INT_ENTRIES)); |
| assert(x->ipi_alloc_map); |
| |
| xive_dbg(x, "Handling interrupts [%08x..%08x]\n", |
| x->int_base, x->int_max - 1); |
| |
| /* System dependant values that must be set before BARs */ |
| //xive_regwx(x, CQ_CFG_PB_GEN, xx); |
| //xive_regwx(x, CQ_MSGSND, xx); |
| |
| /* Setup the BARs */ |
| if (!xive_configure_bars(x)) |
| goto fail; |
| |
| /* Some basic global inits such as page sizes etc... */ |
| if (!xive_config_init(x)) |
| goto fail; |
| |
| /* Configure the set translations for MMIO */ |
| if (!xive_setup_set_xlate(x)) |
| goto fail; |
| |
| /* Dump some MMIO registers for diagnostics */ |
| xive_dump_mmio(x); |
| |
| /* Pre-allocate a number of tables */ |
| if (!xive_prealloc_tables(x)) |
| goto fail; |
| |
| /* Configure local tables in VSDs (forward ports will be |
| * handled later) |
| */ |
| if (!xive_set_local_tables(x)) |
| goto fail; |
| |
| /* Register built-in source controllers (aka IPIs) */ |
| flags = XIVE_SRC_EOI_PAGE1 | XIVE_SRC_TRIGGER_PAGE; |
| if (x->rev >= XIVE_REV_2) |
| flags |= XIVE_SRC_STORE_EOI; |
| __xive_register_source(x, &x->ipis, x->int_base, |
| x->int_hw_bot - x->int_base, IPI_ESB_SHIFT, |
| x->esb_mmio, flags, true, NULL, NULL); |
| |
| /* Register escalation sources */ |
| __xive_register_source(x, &x->esc_irqs, |
| MAKE_ESCALATION_GIRQ(x->block_id, 0), |
| MAX_EQ_COUNT, EQ_ESB_SHIFT, |
| x->eq_mmio, XIVE_SRC_EOI_PAGE1, |
| false, NULL, NULL); |
| |
| |
| return x; |
| fail: |
| xive_err(x, "Initialization failed...\n"); |
| |
| /* Should this be fatal ? */ |
| //assert(false); |
| return NULL; |
| } |
| |
| /* |
| * XICS emulation |
| */ |
| static void xive_ipi_init(struct xive *x, struct cpu_thread *cpu) |
| { |
| struct xive_cpu_state *xs = cpu->xstate; |
| |
| assert(xs); |
| |
| __xive_set_irq_config(&x->ipis.is, xs->ipi_irq, cpu->pir, |
| XIVE_EMULATION_PRIO, xs->ipi_irq, |
| true, false); |
| } |
| |
| static void xive_ipi_eoi(struct xive *x, uint32_t idx) |
| { |
| uint8_t *mm = x->esb_mmio + idx * 0x20000; |
| uint8_t eoi_val; |
| |
| /* For EOI, we use the special MMIO that does a clear of both |
| * P and Q and returns the old Q. |
| * |
| * This allows us to then do a re-trigger if Q was set rather |
| * than synthetizing an interrupt in software |
| */ |
| eoi_val = in_8(mm + 0x10c00); |
| if (eoi_val & 1) { |
| out_8(mm, 0); |
| } |
| } |
| |
| static void xive_ipi_trigger(struct xive *x, uint32_t idx) |
| { |
| uint8_t *mm = x->esb_mmio + idx * 0x20000; |
| |
| xive_vdbg(x, "Trigger IPI 0x%x\n", idx); |
| |
| out_8(mm, 0); |
| } |
| |
| |
| static void xive_reset_enable_thread(struct cpu_thread *c) |
| { |
| struct proc_chip *chip = get_chip(c->chip_id); |
| struct xive *x = chip->xive; |
| uint32_t fc, bit; |
| |
| /* Get fused core number */ |
| fc = (c->pir >> 3) & 0xf; |
| |
| /* Get bit in register */ |
| bit = c->pir & 0x3f; |
| |
| /* Get which register to access */ |
| if (fc < 8) { |
| xive_regw(x, PC_THREAD_EN_REG0_CLR, PPC_BIT(bit)); |
| xive_regw(x, PC_THREAD_EN_REG0_SET, PPC_BIT(bit)); |
| } else { |
| xive_regw(x, PC_THREAD_EN_REG1_CLR, PPC_BIT(bit)); |
| xive_regw(x, PC_THREAD_EN_REG1_SET, PPC_BIT(bit)); |
| } |
| } |
| |
| void xive_cpu_callin(struct cpu_thread *cpu) |
| { |
| struct xive_cpu_state *xs = cpu->xstate; |
| uint8_t old_w2, w2; |
| |
| if (!xs) |
| return; |
| |
| /* Reset the HW thread context and enable it */ |
| xive_reset_enable_thread(cpu); |
| |
| /* Set VT to 1 */ |
| old_w2 = in_8(xs->tm_ring1 + TM_QW3_HV_PHYS + TM_WORD2); |
| out_8(xs->tm_ring1 + TM_QW3_HV_PHYS + TM_WORD2, 0x80); |
| w2 = in_8(xs->tm_ring1 + TM_QW3_HV_PHYS + TM_WORD2); |
| |
| xive_cpu_dbg(cpu, "Initialized TIMA VP=%x/%x W01=%016llx W2=%02x->%02x\n", |
| xs->vp_blk, xs->vp_idx, |
| in_be64(xs->tm_ring1 + TM_QW3_HV_PHYS), |
| old_w2, w2); |
| } |
| |
| #ifdef XIVE_DEBUG_INIT_CACHE_UPDATES |
| static bool xive_check_eq_update(struct xive *x, uint32_t idx, struct xive_eq *eq) |
| { |
| struct xive_eq *eq_p = xive_get_eq(x, idx); |
| struct xive_eq eq2; |
| |
| assert(eq_p); |
| eq2 = *eq_p; |
| if (memcmp(eq, &eq2, sizeof(eq)) != 0) { |
| xive_err(x, "EQ update mismatch idx %d\n", idx); |
| xive_err(x, "want: %08x %08x %08x %08x\n", |
| eq->w0, eq->w1, eq->w2, eq->w3); |
| xive_err(x, " %08x %08x %08x %08x\n", |
| eq->w4, eq->w5, eq->w6, eq->w7); |
| xive_err(x, "got : %08x %08x %08x %08x\n", |
| eq2.w0, eq2.w1, eq2.w2, eq2.w3); |
| xive_err(x, " %08x %08x %08x %08x\n", |
| eq2.w4, eq2.w5, eq2.w6, eq2.w7); |
| return false; |
| } |
| return true; |
| } |
| |
| static bool xive_check_vpc_update(struct xive *x, uint32_t idx, struct xive_vp *vp) |
| { |
| struct xive_vp *vp_p = xive_get_vp(x, idx); |
| struct xive_vp vp2; |
| |
| assert(vp_p); |
| vp2 = *vp_p; |
| if (memcmp(vp, &vp2, sizeof(vp)) != 0) { |
| xive_err(x, "VP update mismatch idx %d\n", idx); |
| xive_err(x, "want: %08x %08x %08x %08x\n", |
| vp->w0, vp->w1, vp->w2, vp->w3); |
| xive_err(x, " %08x %08x %08x %08x\n", |
| vp->w4, vp->w5, vp->w6, vp->w7); |
| xive_err(x, "got : %08x %08x %08x %08x\n", |
| vp2.w0, vp2.w1, vp2.w2, vp2.w3); |
| xive_err(x, " %08x %08x %08x %08x\n", |
| vp2.w4, vp2.w5, vp2.w6, vp2.w7); |
| return false; |
| } |
| return true; |
| } |
| #else |
| static inline bool xive_check_eq_update(struct xive *x __unused, |
| uint32_t idx __unused, |
| struct xive_eq *eq __unused) |
| { |
| return true; |
| } |
| |
| static inline bool xive_check_vpc_update(struct xive *x __unused, |
| uint32_t idx __unused, |
| struct xive_vp *vp __unused) |
| { |
| return true; |
| } |
| #endif |
| |
| #ifdef XIVE_EXTRA_CHECK_INIT_CACHE |
| static void xive_special_cache_check(struct xive *x, uint32_t blk, uint32_t idx) |
| { |
| struct xive_vp vp = {}; |
| uint32_t i; |
| |
| for (i = 0; i < 1000; i++) { |
| struct xive_vp *vp_m = xive_get_vp(x, idx); |
| |
| memset(vp_m, (~i) & 0xff, sizeof(*vp_m)); |
| sync(); |
| vp.w1 = (i << 16) | i; |
| xive_vpc_cache_update(x, blk, idx, |
| 0, 8, &vp, false, true); |
| if (!xive_check_vpc_update(x, idx, &vp)) { |
| xive_dbg(x, "Test failed at %d iterations\n", i); |
| return; |
| } |
| } |
| xive_dbg(x, "1000 iterations test success at %d/0x%x\n", blk, idx); |
| } |
| #else |
| static inline void xive_special_cache_check(struct xive *x __unused, |
| uint32_t blk __unused, |
| uint32_t idx __unused) |
| { |
| } |
| #endif |
| |
| static void xive_setup_hw_for_emu(struct xive_cpu_state *xs) |
| { |
| struct xive_eq eq; |
| struct xive_vp vp; |
| struct xive *x_eq, *x_vp; |
| |
| /* Grab the XIVE where the VP resides. It could be different from |
| * the local chip XIVE if not using block group mode |
| */ |
| x_vp = xive_from_pc_blk(xs->vp_blk); |
| assert(x_vp); |
| |
| /* Grab the XIVE where the EQ resides. It will be the same as the |
| * VP one with the current provisioning but I prefer not making |
| * this code depend on it. |
| */ |
| x_eq = xive_from_vc_blk(xs->eq_blk); |
| assert(x_eq); |
| |
| /* Initialize the structure */ |
| xive_init_emu_eq(xs->vp_blk, xs->vp_idx, &eq, |
| xs->eq_page, XIVE_EMULATION_PRIO); |
| |
| /* Use the cache watch to write it out */ |
| lock(&x_eq->lock); |
| xive_eqc_cache_update(x_eq, xs->eq_blk, |
| xs->eq_idx + XIVE_EMULATION_PRIO, |
| 0, 4, &eq, false, true); |
| xive_check_eq_update(x_eq, xs->eq_idx + XIVE_EMULATION_PRIO, &eq); |
| |
| /* Extra testing of cache watch & scrub facilities */ |
| xive_special_cache_check(x_vp, xs->vp_blk, xs->vp_idx); |
| unlock(&x_eq->lock); |
| |
| /* Initialize/enable the VP */ |
| xive_init_default_vp(&vp, xs->eq_blk, xs->eq_idx); |
| |
| /* Use the cache watch to write it out */ |
| lock(&x_vp->lock); |
| xive_vpc_cache_update(x_vp, xs->vp_blk, xs->vp_idx, |
| 0, 8, &vp, false, true); |
| xive_check_vpc_update(x_vp, xs->vp_idx, &vp); |
| unlock(&x_vp->lock); |
| } |
| |
| static void xive_init_cpu_emulation(struct xive_cpu_state *xs, |
| struct cpu_thread *cpu) |
| { |
| struct xive *x; |
| |
| /* Setup HW EQ and VP */ |
| xive_setup_hw_for_emu(xs); |
| |
| /* Setup and unmask the IPI */ |
| xive_ipi_init(xs->xive, cpu); |
| |
| /* Initialize remaining state */ |
| xs->cppr = 0; |
| xs->mfrr = 0xff; |
| xs->eqbuf = xive_get_eq_buf(xs->vp_blk, |
| xs->eq_idx + XIVE_EMULATION_PRIO); |
| assert(xs->eqbuf); |
| memset(xs->eqbuf, 0, 0x10000); |
| |
| xs->eqptr = 0; |
| xs->eqmsk = (0x10000/4) - 1; |
| xs->eqgen = 0; |
| x = xive_from_vc_blk(xs->eq_blk); |
| assert(x); |
| xs->eqmmio = x->eq_mmio + (xs->eq_idx + XIVE_EMULATION_PRIO) * 0x20000; |
| } |
| |
| static void xive_init_cpu_exploitation(struct xive_cpu_state *xs) |
| { |
| struct xive_vp vp; |
| struct xive *x_vp; |
| |
| /* Grab the XIVE where the VP resides. It could be different from |
| * the local chip XIVE if not using block group mode |
| */ |
| x_vp = xive_from_pc_blk(xs->vp_blk); |
| assert(x_vp); |
| |
| /* Initialize/enable the VP */ |
| xive_init_default_vp(&vp, xs->eq_blk, xs->eq_idx); |
| |
| /* Use the cache watch to write it out */ |
| lock(&x_vp->lock); |
| xive_vpc_cache_update(x_vp, xs->vp_blk, xs->vp_idx, |
| 0, 8, &vp, false, true); |
| unlock(&x_vp->lock); |
| |
| /* Clenaup remaining state */ |
| xs->cppr = 0; |
| xs->mfrr = 0xff; |
| xs->eqbuf = NULL; |
| xs->eqptr = 0; |
| xs->eqmsk = 0; |
| xs->eqgen = 0; |
| xs->eqmmio = NULL; |
| } |
| |
| static void xive_configure_ex_special_bar(struct xive *x, struct cpu_thread *c) |
| { |
| uint64_t xa, val; |
| int64_t rc; |
| |
| xive_cpu_dbg(c, "Setting up special BAR\n"); |
| xa = XSCOM_ADDR_P9_EX(pir_to_core_id(c->pir), P9X_EX_NCU_SPEC_BAR); |
| val = (uint64_t)x->tm_base | P9X_EX_NCU_SPEC_BAR_ENABLE; |
| if (x->tm_shift == 16) |
| val |= P9X_EX_NCU_SPEC_BAR_256K; |
| xive_cpu_dbg(c, "NCU_SPEC_BAR_XA[%08llx]=%016llx\n", xa, val); |
| rc = xscom_write(c->chip_id, xa, val); |
| if (rc) { |
| xive_cpu_err(c, "Failed to setup NCU_SPEC_BAR\n"); |
| /* XXXX what do do now ? */ |
| } |
| } |
| |
| static void xive_provision_cpu(struct xive_cpu_state *xs, struct cpu_thread *c) |
| { |
| struct xive *x; |
| void *p; |
| |
| /* Physical VPs are pre-allocated */ |
| xs->vp_blk = PIR2VP_BLK(c->pir); |
| xs->vp_idx = PIR2VP_IDX(c->pir); |
| |
| /* For now we use identical block IDs for VC and PC but that might |
| * change. We allocate the EQs on the same XIVE as the VP. |
| */ |
| xs->eq_blk = xs->vp_blk; |
| |
| /* Grab the XIVE where the EQ resides. It could be different from |
| * the local chip XIVE if not using block group mode |
| */ |
| x = xive_from_vc_blk(xs->eq_blk); |
| assert(x); |
| |
| /* Allocate a set of EQs for that VP */ |
| xs->eq_idx = xive_alloc_eq_set(x, true); |
| assert(!XIVE_ALLOC_IS_ERR(xs->eq_idx)); |
| |
| /* Provision one of the queues. Allocate the memory on the |
| * chip where the CPU resides |
| */ |
| p = local_alloc(c->chip_id, 0x10000, 0x10000); |
| if (!p) { |
| xive_err(x, "Failed to allocate EQ backing store\n"); |
| assert(false); |
| } |
| xs->eq_page = p; |
| } |
| |
| static void xive_init_cpu(struct cpu_thread *c) |
| { |
| struct proc_chip *chip = get_chip(c->chip_id); |
| struct xive *x = chip->xive; |
| struct xive_cpu_state *xs; |
| |
| if (!x) |
| return; |
| |
| /* |
| * Each core pair (EX) needs this special BAR setup to have the |
| * right powerbus cycle for the TM area (as it has the same address |
| * on all chips so it's somewhat special). |
| * |
| * Because we don't want to bother trying to figure out which core |
| * of a pair is present we just do the setup for each of them, which |
| * is harmless. |
| */ |
| if (cpu_is_thread0(c)) |
| xive_configure_ex_special_bar(x, c); |
| |
| /* Initialize the state structure */ |
| c->xstate = xs = local_alloc(c->chip_id, sizeof(struct xive_cpu_state), 1); |
| assert(xs); |
| memset(xs, 0, sizeof(struct xive_cpu_state)); |
| xs->xive = x; |
| |
| init_lock(&xs->lock); |
| |
| /* Shortcut to TM HV ring */ |
| xs->tm_ring1 = x->tm_base + (1u << x->tm_shift); |
| |
| /* Allocate an IPI */ |
| xs->ipi_irq = xive_alloc_ipi_irqs(c->chip_id, 1, 1); |
| |
| xive_cpu_dbg(c, "CPU IPI is irq %08x\n", xs->ipi_irq); |
| |
| /* Provision a VP and some EQDs for a physical CPU */ |
| xive_provision_cpu(xs, c); |
| |
| /* Initialize the XICS emulation related fields */ |
| xive_init_cpu_emulation(xs, c); |
| } |
| |
| static void xive_init_cpu_properties(struct cpu_thread *cpu) |
| { |
| struct cpu_thread *t; |
| uint32_t iprop[8][2] = { }; |
| uint32_t i; |
| |
| assert(cpu_thread_count <= 8); |
| |
| if (!cpu->node) |
| return; |
| for (i = 0; i < cpu_thread_count; i++) { |
| t = (i == 0) ? cpu : find_cpu_by_pir(cpu->pir + i); |
| if (!t) |
| continue; |
| iprop[i][0] = t->xstate->ipi_irq; |
| iprop[i][1] = 0; /* Edge */ |
| } |
| dt_add_property(cpu->node, "interrupts", iprop, cpu_thread_count * 8); |
| dt_add_property_cells(cpu->node, "interrupt-parent", get_ics_phandle()); |
| } |
| |
| #ifdef XIVE_DEBUG_DUPLICATES |
| static uint32_t xive_count_irq_copies(struct xive_cpu_state *xs, uint32_t ref) |
| { |
| uint32_t i, irq; |
| uint32_t cnt = 0; |
| uint32_t pos = xs->eqptr; |
| uint32_t gen = xs->eqgen; |
| |
| for (i = 0; i < 0x3fff; i++) { |
| irq = xs->eqbuf[pos]; |
| if ((irq >> 31) == gen) |
| break; |
| if (irq == ref) |
| cnt++; |
| pos = (pos + 1) & xs->eqmsk; |
| if (!pos) |
| gen ^= 1; |
| } |
| return cnt; |
| } |
| #else |
| static inline uint32_t xive_count_irq_copies(struct xive_cpu_state *xs __unused, |
| uint32_t ref __unused) |
| { |
| return 1; |
| } |
| #endif |
| |
| static uint32_t xive_read_eq(struct xive_cpu_state *xs, bool just_peek) |
| { |
| uint32_t cur, copies; |
| |
| xive_cpu_vdbg(this_cpu(), " EQ %s... IDX=%x MSK=%x G=%d\n", |
| just_peek ? "peek" : "read", |
| xs->eqptr, xs->eqmsk, xs->eqgen); |
| cur = xs->eqbuf[xs->eqptr]; |
| xive_cpu_vdbg(this_cpu(), " cur: %08x [%08x %08x %08x ...]\n", cur, |
| xs->eqbuf[(xs->eqptr + 1) & xs->eqmsk], |
| xs->eqbuf[(xs->eqptr + 2) & xs->eqmsk], |
| xs->eqbuf[(xs->eqptr + 3) & xs->eqmsk]); |
| if ((cur >> 31) == xs->eqgen) |
| return 0; |
| |
| /* Debug: check for duplicate interrupts in the queue */ |
| copies = xive_count_irq_copies(xs, cur); |
| if (copies > 1) { |
| struct xive_eq *eq; |
| |
| prerror("Wow ! Dups of irq %x, found %d copies !\n", |
| cur & 0x7fffffff, copies); |
| prerror("[%08x > %08x %08x %08x %08x ...] eqgen=%x eqptr=%x jp=%d\n", |
| xs->eqbuf[(xs->eqptr - 1) & xs->eqmsk], |
| xs->eqbuf[(xs->eqptr + 0) & xs->eqmsk], |
| xs->eqbuf[(xs->eqptr + 1) & xs->eqmsk], |
| xs->eqbuf[(xs->eqptr + 2) & xs->eqmsk], |
| xs->eqbuf[(xs->eqptr + 3) & xs->eqmsk], |
| xs->eqgen, xs->eqptr, just_peek); |
| lock(&xs->xive->lock); |
| __xive_cache_scrub(xs->xive, xive_cache_eqc, xs->eq_blk, |
| xs->eq_idx + XIVE_EMULATION_PRIO, |
| false, false); |
| unlock(&xs->xive->lock); |
| eq = xive_get_eq(xs->xive, xs->eq_idx + XIVE_EMULATION_PRIO); |
| prerror("EQ @%p W0=%08x W1=%08x qbuf @%p\n", |
| eq, eq->w0, eq->w1, xs->eqbuf); |
| } |
| log_add(xs, LOG_TYPE_POPQ, 7, cur, |
| xs->eqbuf[(xs->eqptr + 1) & xs->eqmsk], |
| xs->eqbuf[(xs->eqptr + 2) & xs->eqmsk], |
| copies, |
| xs->eqptr, xs->eqgen, just_peek); |
| if (!just_peek) { |
| xs->eqptr = (xs->eqptr + 1) & xs->eqmsk; |
| if (xs->eqptr == 0) |
| xs->eqgen ^= 1; |
| xs->total_irqs++; |
| } |
| return cur & 0x00ffffff; |
| } |
| |
| static uint8_t xive_sanitize_cppr(uint8_t cppr) |
| { |
| if (cppr == 0xff || cppr == 0) |
| return cppr; |
| else |
| return XIVE_EMULATION_PRIO; |
| } |
| |
| static inline uint8_t opal_xive_check_pending(struct xive_cpu_state *xs, |
| uint8_t cppr) |
| { |
| uint8_t mask = (cppr > 7) ? 0xff : ~((0x100 >> cppr) - 1); |
| |
| return xs->pending & mask; |
| } |
| |
| static void opal_xive_update_cppr(struct xive_cpu_state *xs, u8 cppr) |
| { |
| /* Peform the update */ |
| xs->cppr = cppr; |
| out_8(xs->tm_ring1 + TM_QW3_HV_PHYS + TM_CPPR, cppr); |
| |
| /* Trigger the IPI if it's still more favored than the CPPR |
| * |
| * This can lead to a bunch of spurrious retriggers if the |
| * IPI is queued up behind other interrupts but that's not |
| * a big deal and keeps the code simpler |
| */ |
| if (xs->mfrr < cppr) |
| xive_ipi_trigger(xs->xive, GIRQ_TO_IDX(xs->ipi_irq)); |
| } |
| |
| static int64_t opal_xive_eoi(uint32_t xirr) |
| { |
| struct cpu_thread *c = this_cpu(); |
| struct xive_cpu_state *xs = c->xstate; |
| uint32_t isn = xirr & 0x00ffffff; |
| struct xive *src_x; |
| bool special_ipi = false; |
| uint8_t cppr; |
| |
| /* |
| * In exploitation mode, this is supported as a way to perform |
| * an EOI via a FW calls. This can be needed to workaround HW |
| * implementation bugs for example. In this case interrupts will |
| * have the OPAL_XIVE_IRQ_EOI_VIA_FW flag set. |
| * |
| * In that mode the entire "xirr" argument is interpreterd as |
| * a global IRQ number (including the escalation bit), ther is |
| * no split between the top 8 bits for CPPR and bottom 24 for |
| * the interrupt number. |
| */ |
| if (xive_mode != XIVE_MODE_EMU) |
| return irq_source_eoi(xirr) ? OPAL_SUCCESS : OPAL_PARAMETER; |
| |
| if (!xs) |
| return OPAL_INTERNAL_ERROR; |
| |
| xive_cpu_vdbg(c, "EOI xirr=%08x cur_cppr=%d\n", xirr, xs->cppr); |
| |
| /* Limit supported CPPR values from OS */ |
| cppr = xive_sanitize_cppr(xirr >> 24); |
| |
| lock(&xs->lock); |
| |
| log_add(xs, LOG_TYPE_EOI, 3, isn, xs->eqptr, xs->eqgen); |
| |
| /* If this was our magic IPI, convert to IRQ number */ |
| if (isn == 2) { |
| isn = xs->ipi_irq; |
| special_ipi = true; |
| xive_cpu_vdbg(c, "User EOI for IPI !\n"); |
| } |
| |
| /* First check if we have stuff in that queue. If we do, don't bother with |
| * doing an EOI on the EQ. Just mark that priority pending, we'll come |
| * back later. |
| * |
| * If/when supporting multiple queues we would have to check them all |
| * in ascending prio order up to the passed-in CPPR value (exclusive). |
| */ |
| if (xive_read_eq(xs, true)) { |
| xive_cpu_vdbg(c, " isn %08x, skip, queue non empty\n", xirr); |
| xs->pending |= 1 << XIVE_EMULATION_PRIO; |
| } |
| #ifndef EQ_ALWAYS_NOTIFY |
| else { |
| uint8_t eoi_val; |
| |
| /* Perform EQ level EOI. Only one EQ for now ... |
| * |
| * Note: We aren't doing an actual EOI. Instead we are clearing |
| * both P and Q and will re-check the queue if Q was set. |
| */ |
| eoi_val = in_8(xs->eqmmio + 0xc00); |
| xive_cpu_vdbg(c, " isn %08x, eoi_val=%02x\n", xirr, eoi_val); |
| |
| /* Q was set ? Check EQ again after doing a sync to ensure |
| * ordering. |
| */ |
| if (eoi_val & 1) { |
| sync(); |
| if (xive_read_eq(xs, true)) |
| xs->pending |= 1 << XIVE_EMULATION_PRIO; |
| } |
| } |
| #endif |
| |
| /* Perform source level EOI if it's not our emulated MFRR IPI |
| * otherwise EOI ourselves |
| */ |
| src_x = xive_from_isn(isn); |
| if (src_x) { |
| uint32_t idx = GIRQ_TO_IDX(isn); |
| |
| /* Is it an IPI ? */ |
| if (special_ipi) { |
| xive_ipi_eoi(src_x, idx); |
| } else { |
| /* Otherwise go through the source mechanism */ |
| xive_vdbg(src_x, "EOI of IDX %x in EXT range\n", idx); |
| irq_source_eoi(isn); |
| } |
| } else { |
| xive_cpu_err(c, " EOI unknown ISN %08x\n", isn); |
| } |
| |
| /* Finally restore CPPR */ |
| opal_xive_update_cppr(xs, cppr); |
| |
| xive_cpu_vdbg(c, " pending=0x%x cppr=%d\n", xs->pending, cppr); |
| |
| unlock(&xs->lock); |
| |
| /* Return whether something is pending that is suitable for |
| * delivery considering the new CPPR value. This can be done |
| * without lock as these fields are per-cpu. |
| */ |
| return opal_xive_check_pending(xs, cppr) ? 1 : 0; |
| } |
| |
| #ifdef XIVE_CHECK_MISROUTED_IPI |
| static void xive_dump_eq(uint32_t eq_blk, uint32_t eq_idx) |
| { |
| struct cpu_thread *me = this_cpu(); |
| struct xive *x; |
| struct xive_eq *eq; |
| |
| x = xive_from_vc_blk(eq_blk); |
| if (!x) |
| return; |
| eq = xive_get_eq(x, eq_idx); |
| if (!eq) |
| return; |
| xive_cpu_err(me, "EQ: %08x %08x %08x %08x (@%p)\n", |
| eq->w0, eq->w1, eq->w2, eq->w3, eq); |
| xive_cpu_err(me, " %08x %08x %08x %08x\n", |
| eq->w4, eq->w5, eq->w6, eq->w7); |
| } |
| static int64_t __opal_xive_dump_emu(struct xive_cpu_state *xs, uint32_t pir); |
| |
| static bool check_misrouted_ipi(struct cpu_thread *me, uint32_t irq) |
| { |
| struct cpu_thread *c; |
| |
| for_each_present_cpu(c) { |
| struct xive_cpu_state *xs = c->xstate; |
| struct xive_ive *ive; |
| uint32_t ipi_target, i, eq_blk, eq_idx; |
| struct proc_chip *chip; |
| struct xive *x; |
| |
| if (!xs) |
| continue; |
| if (irq == xs->ipi_irq) { |
| xive_cpu_err(me, "misrouted IPI 0x%x, should" |
| " be aimed at CPU 0x%x\n", |
| irq, c->pir); |
| xive_cpu_err(me, " my eq_page=%p eqbuff=%p eq=0x%x/%x\n", |
| me->xstate->eq_page, me->xstate->eqbuf, |
| me->xstate->eq_blk, me->xstate->eq_idx + XIVE_EMULATION_PRIO); |
| xive_cpu_err(me, "tgt eq_page=%p eqbuff=%p eq=0x%x/%x\n", |
| c->xstate->eq_page, c->xstate->eqbuf, |
| c->xstate->eq_blk, c->xstate->eq_idx + XIVE_EMULATION_PRIO); |
| __opal_xive_dump_emu(me->xstate, me->pir); |
| __opal_xive_dump_emu(c->xstate, c->pir); |
| if (xive_get_irq_targetting(xs->ipi_irq, &ipi_target, NULL, NULL)) |
| xive_cpu_err(me, "target=%08x\n", ipi_target); |
| else |
| xive_cpu_err(me, "target=???\n"); |
| /* Find XIVE on which the IVE resides */ |
| x = xive_from_isn(irq); |
| if (!x) { |
| xive_cpu_err(me, "no xive attached\n"); |
| return true; |
| } |
| ive = xive_get_ive(x, irq); |
| if (!ive) { |
| xive_cpu_err(me, "no ive attached\n"); |
| return true; |
| } |
| xive_cpu_err(me, "ive=%016llx\n", ive->w); |
| for_each_chip(chip) { |
| x = chip->xive; |
| if (!x) |
| continue; |
| ive = x->ivt_base; |
| for (i = 0; i < MAX_INT_ENTRIES; i++) { |
| if ((ive[i].w & IVE_EQ_DATA) == irq) { |
| eq_blk = GETFIELD(IVE_EQ_BLOCK, ive[i].w); |
| eq_idx = GETFIELD(IVE_EQ_INDEX, ive[i].w); |
| xive_cpu_err(me, "Found source: 0x%x ive=%016llx\n" |
| " eq 0x%x/%x", |
| BLKIDX_TO_GIRQ(x->block_id, i), |
| ive[i].w, eq_blk, eq_idx); |
| xive_dump_eq(eq_blk, eq_idx); |
| } |
| } |
| } |
| return true; |
| } |
| } |
| return false; |
| } |
| #else |
| static inline bool check_misrouted_ipi(struct cpu_thread *c __unused, |
| uint32_t irq __unused) |
| { |
| return false; |
| } |
| #endif |
| |
| static int64_t opal_xive_get_xirr(uint32_t *out_xirr, bool just_poll) |
| { |
| struct cpu_thread *c = this_cpu(); |
| struct xive_cpu_state *xs = c->xstate; |
| uint16_t ack; |
| uint8_t active, old_cppr; |
| |
| if (xive_mode != XIVE_MODE_EMU) |
| return OPAL_WRONG_STATE; |
| if (!xs) |
| return OPAL_INTERNAL_ERROR; |
| if (!out_xirr) |
| return OPAL_PARAMETER; |
| |
| *out_xirr = 0; |
| |
| lock(&xs->lock); |
| |
| /* |
| * Due to the need to fetch multiple interrupts from the EQ, we |
| * need to play some tricks. |
| * |
| * The "pending" byte in "xs" keeps track of the priorities that |
| * are known to have stuff to read (currently we only use one). |
| * |
| * It is set in EOI and cleared when consumed here. We don't bother |
| * looking ahead here, EOI will do it. |
| * |
| * We do need to still do an ACK every time in case a higher prio |
| * exception occurred (though we don't do prio yet... right ? still |
| * let's get the basic design right !). |
| * |
| * Note that if we haven't found anything via ack, but did find |
| * something in the queue, we must also raise CPPR back. |
| */ |
| |
| xive_cpu_vdbg(c, "get_xirr W01=%016llx W2=%08x\n", |
| __in_be64(xs->tm_ring1 + TM_QW3_HV_PHYS), |
| __in_be32(xs->tm_ring1 + TM_QW3_HV_PHYS + 8)); |
| |
| /* Perform the HV Ack cycle */ |
| if (just_poll) |
| ack = __in_be64(xs->tm_ring1 + TM_QW3_HV_PHYS) >> 48; |
| else |
| ack = __in_be16(xs->tm_ring1 + TM_SPC_ACK_HV_REG); |
| sync(); |
| xive_cpu_vdbg(c, "get_xirr,%s=%04x\n", just_poll ? "POLL" : "ACK", ack); |
| |
| /* Capture the old CPPR which we will return with the interrupt */ |
| old_cppr = xs->cppr; |
| |
| switch(GETFIELD(TM_QW3_NSR_HE, (ack >> 8))) { |
| case TM_QW3_NSR_HE_NONE: |
| break; |
| case TM_QW3_NSR_HE_POOL: |
| break; |
| case TM_QW3_NSR_HE_PHYS: |
| /* Mark pending and keep track of the CPPR update */ |
| if (!just_poll && (ack & 0xff) != 0xff) { |
| xs->cppr = ack & 0xff; |
| xs->pending |= 1 << xs->cppr; |
| } |
| break; |
| case TM_QW3_NSR_HE_LSI: |
| break; |
| } |
| |
| /* Calculate "active" lines as being the pending interrupts |
| * masked by the "old" CPPR |
| */ |
| active = opal_xive_check_pending(xs, old_cppr); |
| |
| log_add(xs, LOG_TYPE_XIRR, 6, old_cppr, xs->cppr, xs->pending, active, |
| xs->eqptr, xs->eqgen); |
| |
| #ifdef XIVE_PERCPU_LOG |
| { |
| struct xive_eq *eq; |
| lock(&xs->xive->lock); |
| __xive_cache_scrub(xs->xive, xive_cache_eqc, xs->eq_blk, |
| xs->eq_idx + XIVE_EMULATION_PRIO, |
| false, false); |
| unlock(&xs->xive->lock); |
| eq = xive_get_eq(xs->xive, xs->eq_idx + XIVE_EMULATION_PRIO); |
| log_add(xs, LOG_TYPE_EQD, 2, eq->w0, eq->w1); |
| } |
| #endif /* XIVE_PERCPU_LOG */ |
| |
| xive_cpu_vdbg(c, " cppr=%d->%d pending=0x%x active=%x\n", |
| old_cppr, xs->cppr, xs->pending, active); |
| if (active) { |
| /* Find highest pending */ |
| uint8_t prio = ffs(active) - 1; |
| uint32_t val; |
| |
| /* XXX Use "p" to select queue */ |
| val = xive_read_eq(xs, just_poll); |
| |
| if (val && val < XIVE_INT_SAFETY_GAP) |
| xive_cpu_err(c, "Bogus interrupt 0x%x received !\n", val); |
| |
| /* Convert to magic IPI if needed */ |
| if (val == xs->ipi_irq) |
| val = 2; |
| if (check_misrouted_ipi(c, val)) |
| val = 2; |
| |
| *out_xirr = (old_cppr << 24) | val; |
| |
| /* If we are polling, that's it */ |
| if (just_poll) |
| goto skip; |
| |
| /* Clear the pending bit. EOI will set it again if needed. We |
| * could check the queue but that's not really critical here. |
| */ |
| xs->pending &= ~(1 << prio); |
| |
| /* Spurrious IPB bit, nothing to fetch, bring CPPR back */ |
| if (!val) |
| prio = old_cppr; |
| |
| /* We could have fetched a pending interrupt left over |
| * by a previous EOI, so the CPPR might need adjusting |
| * Also if we had a spurrious one as well. |
| */ |
| if (xs->cppr != prio) { |
| xs->cppr = prio; |
| out_8(xs->tm_ring1 + TM_QW3_HV_PHYS + TM_CPPR, prio); |
| xive_cpu_vdbg(c, " adjusted CPPR to %d\n", prio); |
| } |
| |
| if (val) |
| xive_cpu_vdbg(c, " found irq, prio=%d\n", prio); |
| |
| } else { |
| /* Nothing was active, this is a fluke, restore CPPR */ |
| opal_xive_update_cppr(xs, old_cppr); |
| xive_cpu_vdbg(c, " nothing active, restored CPPR to %d\n", |
| old_cppr); |
| } |
| skip: |
| |
| log_add(xs, LOG_TYPE_XIRR2, 5, xs->cppr, xs->pending, |
| *out_xirr, xs->eqptr, xs->eqgen); |
| xive_cpu_vdbg(c, " returning XIRR=%08x, pending=0x%x\n", |
| *out_xirr, xs->pending); |
| |
| unlock(&xs->lock); |
| |
| return OPAL_SUCCESS; |
| } |
| |
| static int64_t opal_xive_set_cppr(uint8_t cppr) |
| { |
| struct cpu_thread *c = this_cpu(); |
| struct xive_cpu_state *xs = c->xstate; |
| |
| if (xive_mode != XIVE_MODE_EMU) |
| return OPAL_WRONG_STATE; |
| |
| /* Limit supported CPPR values */ |
| cppr = xive_sanitize_cppr(cppr); |
| |
| if (!xs) |
| return OPAL_INTERNAL_ERROR; |
| xive_cpu_vdbg(c, "CPPR setting to %d\n", cppr); |
| |
| lock(&xs->lock); |
| opal_xive_update_cppr(xs, cppr); |
| unlock(&xs->lock); |
| |
| return OPAL_SUCCESS; |
| } |
| |
| static int64_t opal_xive_set_mfrr(uint32_t cpu, uint8_t mfrr) |
| { |
| struct cpu_thread *c = find_cpu_by_server(cpu); |
| struct xive_cpu_state *xs; |
| uint8_t old_mfrr; |
| |
| if (xive_mode != XIVE_MODE_EMU) |
| return OPAL_WRONG_STATE; |
| if (!c) |
| return OPAL_PARAMETER; |
| xs = c->xstate; |
| if (!xs) |
| return OPAL_INTERNAL_ERROR; |
| |
| lock(&xs->lock); |
| old_mfrr = xs->mfrr; |
| xive_cpu_vdbg(c, " Setting MFRR to %x, old is %x\n", mfrr, old_mfrr); |
| xs->mfrr = mfrr; |
| if (old_mfrr > mfrr && mfrr < xs->cppr) |
| xive_ipi_trigger(xs->xive, GIRQ_TO_IDX(xs->ipi_irq)); |
| unlock(&xs->lock); |
| |
| return OPAL_SUCCESS; |
| } |
| |
| static uint64_t xive_convert_irq_flags(uint64_t iflags) |
| { |
| uint64_t oflags = 0; |
| |
| if (iflags & XIVE_SRC_STORE_EOI) |
| oflags |= OPAL_XIVE_IRQ_STORE_EOI; |
| |
| /* OPAL_XIVE_IRQ_TRIGGER_PAGE is only meant to be set if |
| * the interrupt has a *separate* trigger page. |
| */ |
| if ((iflags & XIVE_SRC_EOI_PAGE1) && |
| (iflags & XIVE_SRC_TRIGGER_PAGE)) |
| oflags |= OPAL_XIVE_IRQ_TRIGGER_PAGE; |
| |
| if (iflags & XIVE_SRC_LSI) |
| oflags |= OPAL_XIVE_IRQ_LSI; |
| if (iflags & XIVE_SRC_SHIFT_BUG) |
| oflags |= OPAL_XIVE_IRQ_SHIFT_BUG; |
| return oflags; |
| } |
| |
| static int64_t opal_xive_get_irq_info(uint32_t girq, |
| uint64_t *out_flags, |
| uint64_t *out_eoi_page, |
| uint64_t *out_trig_page, |
| uint32_t *out_esb_shift, |
| uint32_t *out_src_chip) |
| { |
| struct irq_source *is = irq_find_source(girq); |
| struct xive_src *s = container_of(is, struct xive_src, is); |
| uint32_t idx; |
| uint64_t mm_base; |
| uint64_t eoi_page = 0, trig_page = 0; |
| |
| if (xive_mode != XIVE_MODE_EXPL) |
| return OPAL_WRONG_STATE; |
| if (is == NULL || out_flags == NULL) |
| return OPAL_PARAMETER; |
| assert(is->ops == &xive_irq_source_ops); |
| |
| if (out_flags) |
| *out_flags = xive_convert_irq_flags(s->flags); |
| |
| /* |
| * If the orig source has a set_xive callback, then set |
| * OPAL_XIVE_IRQ_MASK_VIA_FW as masking/unmasking requires |
| * source specific workarounds. Same with EOI. |
| */ |
| if (out_flags && s->orig_ops) { |
| if (s->orig_ops->set_xive) |
| *out_flags |= OPAL_XIVE_IRQ_MASK_VIA_FW; |
| if (s->orig_ops->eoi) |
| *out_flags |= OPAL_XIVE_IRQ_EOI_VIA_FW; |
| } |
| |
| idx = girq - s->esb_base; |
| |
| if (out_esb_shift) |
| *out_esb_shift = s->esb_shift; |
| |
| mm_base = (uint64_t)s->esb_mmio + (1ull << s->esb_shift) * idx; |
| |
| /* The EOI page can either be the first or second page */ |
| if (s->flags & XIVE_SRC_EOI_PAGE1) { |
| uint64_t p1off = 1ull << (s->esb_shift - 1); |
| eoi_page = mm_base + p1off; |
| } else |
| eoi_page = mm_base; |
| |
| /* The trigger page, if it exists, is always the first page */ |
| if (s->flags & XIVE_SRC_TRIGGER_PAGE) |
| trig_page = mm_base; |
| |
| if (out_eoi_page) |
| *out_eoi_page = eoi_page; |
| if (out_trig_page) |
| *out_trig_page = trig_page; |
| if (out_src_chip) |
| *out_src_chip = GIRQ_TO_CHIP(girq); |
| |
| return OPAL_SUCCESS; |
| } |
| |
| static int64_t opal_xive_get_irq_config(uint32_t girq, |
| uint64_t *out_vp, |
| uint8_t *out_prio, |
| uint32_t *out_lirq) |
| { |
| uint32_t vp; |
| |
| if (xive_mode != XIVE_MODE_EXPL) |
| return OPAL_WRONG_STATE; |
| |
| if (xive_get_irq_targetting(girq, &vp, out_prio, out_lirq)) { |
| *out_vp = vp; |
| return OPAL_SUCCESS; |
| } else |
| return OPAL_PARAMETER; |
| } |
| |
| static int64_t opal_xive_set_irq_config(uint32_t girq, |
| uint64_t vp, |
| uint8_t prio, |
| uint32_t lirq) |
| { |
| /* |
| * This variant is meant for a XIVE-aware OS, thus it will |
| * *not* affect the ESB state of the interrupt. If used with |
| * a prio of FF, the IVT/EAS will be mased. In that case the |
| * races have to be handled by the OS. |
| * |
| * The exception to this rule is interrupts for which masking |
| * and unmasking is handled by firmware. In that case the ESB |
| * state isn't under OS control and will be dealt here. This |
| * is currently only the case of LSIs and on P9 DD1.0 only so |
| * isn't an issue. |
| */ |
| |
| if (xive_mode != XIVE_MODE_EXPL) |
| return OPAL_WRONG_STATE; |
| |
| return xive_set_irq_config(girq, vp, prio, lirq, false); |
| } |
| |
| static int64_t opal_xive_get_queue_info(uint64_t vp, uint32_t prio, |
| uint64_t *out_qpage, |
| uint64_t *out_qsize, |
| uint64_t *out_qeoi_page, |
| uint32_t *out_escalate_irq, |
| uint64_t *out_qflags) |
| { |
| uint32_t blk, idx; |
| struct xive *x; |
| struct xive_eq *eq; |
| |
| if (xive_mode != XIVE_MODE_EXPL) |
| return OPAL_WRONG_STATE; |
| |
| if (!xive_eq_for_target(vp, prio, &blk, &idx)) |
| return OPAL_PARAMETER; |
| |
| x = xive_from_vc_blk(blk); |
| if (!x) |
| return OPAL_PARAMETER; |
| |
| eq = xive_get_eq(x, idx); |
| if (!eq) |
| return OPAL_PARAMETER; |
| |
| if (out_escalate_irq) { |
| *out_escalate_irq = |
| MAKE_ESCALATION_GIRQ(blk, idx); |
| } |
| if (out_qpage) { |
| if (eq->w0 & EQ_W0_ENQUEUE) |
| *out_qpage = |
| (((uint64_t)(eq->w2 & 0x0fffffff)) << 32) | eq->w3; |
| else |
| *out_qpage = 0; |
| } |
| if (out_qsize) { |
| if (eq->w0 & EQ_W0_ENQUEUE) |
| *out_qsize = GETFIELD(EQ_W0_QSIZE, eq->w0) + 12; |
| else |
| *out_qsize = 0; |
| } |
| if (out_qeoi_page) { |
| *out_qeoi_page = |
| (uint64_t)x->eq_mmio + idx * 0x20000; |
| } |
| if (out_qflags) { |
| *out_qflags = 0; |
| if (eq->w0 & EQ_W0_VALID) |
| *out_qflags |= OPAL_XIVE_EQ_ENABLED; |
| if (eq->w0 & EQ_W0_UCOND_NOTIFY) |
| *out_qflags |= OPAL_XIVE_EQ_ALWAYS_NOTIFY; |
| if (eq->w0 & EQ_W0_ESCALATE_CTL) |
| *out_qflags |= OPAL_XIVE_EQ_ESCALATE; |
| } |
| |
| return OPAL_SUCCESS; |
| } |
| |
| static int64_t opal_xive_set_queue_info(uint64_t vp, uint32_t prio, |
| uint64_t qpage, |
| uint64_t qsize, |
| uint64_t qflags) |
| { |
| uint32_t blk, idx; |
| struct xive *x; |
| struct xive_eq *old_eq; |
| struct xive_eq eq; |
| uint32_t vp_blk, vp_idx; |
| bool group; |
| int64_t rc; |
| |
| if (xive_mode != XIVE_MODE_EXPL) |
| return OPAL_WRONG_STATE; |
| if (!xive_eq_for_target(vp, prio, &blk, &idx)) |
| return OPAL_PARAMETER; |
| |
| x = xive_from_vc_blk(blk); |
| if (!x) |
| return OPAL_PARAMETER; |
| |
| old_eq = xive_get_eq(x, idx); |
| if (!old_eq) |
| return OPAL_PARAMETER; |
| |
| /* This shouldn't fail or xive_eq_for_target would have |
| * failed already |
| */ |
| if (!xive_decode_vp(vp, &vp_blk, &vp_idx, NULL, &group)) |
| return OPAL_PARAMETER; |
| |
| /* |
| * Make a local copy which we will later try to commit using |
| * the cache watch facility |
| */ |
| eq = *old_eq; |
| |
| switch(qsize) { |
| /* Supported sizes */ |
| case 12: |
| case 16: |
| case 21: |
| case 24: |
| eq.w3 = ((uint64_t)qpage) & 0xffffffff; |
| eq.w2 = (((uint64_t)qpage)) >> 32 & 0x0fffffff; |
| eq.w0 |= EQ_W0_ENQUEUE; |
| eq.w0 = SETFIELD(EQ_W0_QSIZE, eq.w0, qsize - 12); |
| break; |
| case 0: |
| eq.w2 = eq.w3 = 0; |
| eq.w0 &= ~EQ_W0_ENQUEUE; |
| break; |
| default: |
| return OPAL_PARAMETER; |
| } |
| |
| /* Ensure the priority and target are correctly set (they will |
| * not be right after allocation |
| */ |
| eq.w6 = SETFIELD(EQ_W6_NVT_BLOCK, 0ul, vp_blk) | |
| SETFIELD(EQ_W6_NVT_INDEX, 0ul, vp_idx); |
| eq.w7 = SETFIELD(EQ_W7_F0_PRIORITY, 0ul, prio); |
| /* XXX Handle group i bit when needed */ |
| |
| /* Always notify flag */ |
| if (qflags & OPAL_XIVE_EQ_ALWAYS_NOTIFY) |
| eq.w0 |= EQ_W0_UCOND_NOTIFY; |
| else |
| eq.w0 &= ~EQ_W0_UCOND_NOTIFY; |
| |
| /* Escalation flag */ |
| if (qflags & OPAL_XIVE_EQ_ESCALATE) |
| eq.w0 |= EQ_W0_ESCALATE_CTL; |
| else |
| eq.w0 &= ~EQ_W0_ESCALATE_CTL; |
| |
| /* Unconditionally clear the current queue pointer, set |
| * generation to 1 and disable escalation interrupts. |
| */ |
| eq.w1 = EQ_W1_GENERATION | |
| (old_eq->w1 & (EQ_W1_ESe_P | EQ_W1_ESe_Q | |
| EQ_W1_ESn_P | EQ_W1_ESn_Q)); |
| |
| /* Enable or disable. We always enable backlog for an |
| * enabled queue otherwise escalations won't work. |
| */ |
| if (qflags & OPAL_XIVE_EQ_ENABLED) |
| eq.w0 |= EQ_W0_VALID | EQ_W0_BACKLOG; |
| else { |
| eq.w0 &= ~EQ_W0_VALID; |
| eq.w1 &= ~(EQ_W1_ESe_P | EQ_W1_ESn_P); |
| eq.w1 |= EQ_W1_ESe_Q | EQ_W1_ESn_Q; |
| } |
| |
| /* Update EQ, non-synchronous */ |
| lock(&x->lock); |
| rc = xive_eqc_cache_update(x, blk, idx, 0, 4, &eq, false, false); |
| unlock(&x->lock); |
| |
| return rc; |
| } |
| |
| static int64_t opal_xive_donate_page(uint32_t chip_id, uint64_t addr) |
| { |
| struct proc_chip *c = get_chip(chip_id); |
| struct list_node *n __unused; |
| |
| if (xive_mode != XIVE_MODE_EXPL) |
| return OPAL_WRONG_STATE; |
| if (!c) |
| return OPAL_PARAMETER; |
| if (!c->xive) |
| return OPAL_PARAMETER; |
| if (addr & 0xffff) |
| return OPAL_PARAMETER; |
| #ifdef USE_INDIRECT |
| n = (struct list_node *)addr; |
| lock(&c->xive->lock); |
| list_add(&c->xive->donated_pages, n); |
| unlock(&c->xive->lock); |
| #endif |
| return OPAL_SUCCESS; |
| } |
| |
| static int64_t opal_xive_get_vp_info(uint64_t vp_id, |
| uint64_t *out_flags, |
| uint64_t *out_cam_value, |
| uint64_t *out_report_cl_pair, |
| uint32_t *out_chip_id) |
| { |
| struct xive *x; |
| struct xive_vp *vp; |
| uint32_t blk, idx; |
| bool group; |
| |
| if (!xive_decode_vp(vp_id, &blk, &idx, NULL, &group)) |
| return OPAL_PARAMETER; |
| /* We don't do groups yet */ |
| if (group) |
| return OPAL_PARAMETER; |
| x = xive_from_pc_blk(blk); |
| if (!x) |
| return OPAL_PARAMETER; |
| vp = xive_get_vp(x, idx); |
| if (!vp) |
| return OPAL_PARAMETER; |
| |
| if (out_flags) { |
| *out_flags = 0; |
| if (vp->w0 & VP_W0_VALID) |
| *out_flags |= OPAL_XIVE_VP_ENABLED; |
| } |
| |
| if (out_cam_value) |
| *out_cam_value = (blk << 19) | idx; |
| |
| if (out_report_cl_pair) { |
| *out_report_cl_pair = ((uint64_t)(vp->w6 & 0x0fffffff)) << 32; |
| *out_report_cl_pair |= vp->w7 & 0xffffff00; |
| } |
| |
| if (out_chip_id) |
| *out_chip_id = xive_block_to_chip[blk]; |
| |
| return OPAL_SUCCESS; |
| } |
| |
| static int64_t opal_xive_set_vp_info(uint64_t vp_id, |
| uint64_t flags, |
| uint64_t report_cl_pair) |
| { |
| struct xive *x; |
| struct xive_vp *vp, vp_new; |
| uint32_t blk, idx; |
| bool group; |
| int64_t rc; |
| |
| if (!xive_decode_vp(vp_id, &blk, &idx, NULL, &group)) |
| return OPAL_PARAMETER; |
| /* We don't do groups yet */ |
| if (group) |
| return OPAL_PARAMETER; |
| if (report_cl_pair & 0xff) |
| return OPAL_PARAMETER; |
| x = xive_from_pc_blk(blk); |
| if (!x) |
| return OPAL_PARAMETER; |
| vp = xive_get_vp(x, idx); |
| if (!vp) |
| return OPAL_PARAMETER; |
| |
| vp_new = *vp; |
| if (flags & OPAL_XIVE_VP_ENABLED) |
| vp_new.w0 |= VP_W0_VALID; |
| else |
| vp_new.w0 &= ~VP_W0_VALID; |
| |
| vp_new.w7 = report_cl_pair & 0xffffffff; |
| vp_new.w6 = report_cl_pair >> 32; |
| |
| lock(&x->lock); |
| rc = xive_vpc_cache_update(x, blk, idx, 0, 8, &vp_new, false, false); |
| if (rc) { |
| unlock(&x->lock); |
| return rc; |
| } |
| |
| /* When disabling, we scrub clean (invalidate the entry) so |
| * we can avoid cache ops in alloc/free |
| */ |
| if (!(flags & OPAL_XIVE_VP_ENABLED)) |
| xive_vpc_scrub_clean(x, blk, idx); |
| |
| unlock(&x->lock); |
| |
| return OPAL_SUCCESS; |
| } |
| |
| static void xive_cleanup_cpu_tima(struct cpu_thread *c) |
| { |
| struct xive_cpu_state *xs = c->xstate; |
| struct xive *x = xs->xive; |
| void *ind_tm_base = x->ic_base + (4 << x->ic_shift); |
| uint8_t old_w2, w2; |
| |
| /* Reset the HW context */ |
| xive_reset_enable_thread(c); |
| |
| /* Setup indirect access to the corresponding thread */ |
| xive_regw(x, PC_TCTXT_INDIR0, |
| PC_TCTXT_INDIR_VALID | |
| SETFIELD(PC_TCTXT_INDIR_THRDID, 0ull, c->pir & 0xff)); |
| |
| /* Workaround for HW issue: Need to read the above register |
| * back before doing the subsequent accesses |
| */ |
| xive_regr(x, PC_TCTXT_INDIR0); |
| |
| /* Set VT to 1 */ |
| old_w2 = in_8(ind_tm_base + TM_QW3_HV_PHYS + TM_WORD2); |
| out_8(ind_tm_base + TM_QW3_HV_PHYS + TM_WORD2, 0x80); |
| w2 = in_8(ind_tm_base + TM_QW3_HV_PHYS + TM_WORD2); |
| |
| /* Dump HV state */ |
| xive_cpu_dbg(c, "[reset] VP TIMA VP=%x/%x W01=%016llx W2=%02x->%02x\n", |
| xs->vp_blk, xs->vp_idx, |
| in_be64(ind_tm_base + TM_QW3_HV_PHYS), |
| old_w2, w2); |
| |
| /* Reset indirect access */ |
| xive_regw(x, PC_TCTXT_INDIR0, 0); |
| } |
| |
| #ifdef USE_INDIRECT |
| static int64_t xive_vc_ind_cache_kill(struct xive *x, uint64_t type) |
| { |
| uint64_t val; |
| |
| /* We clear the whole thing */ |
| xive_regw(x, VC_AT_MACRO_KILL_MASK, 0); |
| xive_regw(x, VC_AT_MACRO_KILL, VC_KILL_VALID | |
| SETFIELD(VC_KILL_TYPE, 0ull, type)); |
| |
| /* XXX SIMICS problem ? */ |
| if (chip_quirk(QUIRK_SIMICS)) |
| return 0; |
| |
| /* XXX Add timeout */ |
| for (;;) { |
| val = xive_regr(x, VC_AT_MACRO_KILL); |
| if (!(val & VC_KILL_VALID)) |
| break; |
| } |
| return 0; |
| } |
| |
| static int64_t xive_pc_ind_cache_kill(struct xive *x) |
| { |
| uint64_t val; |
| |
| /* We clear the whole thing */ |
| xive_regw(x, PC_AT_KILL_MASK, 0); |
| xive_regw(x, PC_AT_KILL, PC_AT_KILL_VALID); |
| |
| /* XXX SIMICS problem ? */ |
| if (chip_quirk(QUIRK_SIMICS)) |
| return 0; |
| |
| /* XXX Add timeout */ |
| for (;;) { |
| val = xive_regr(x, PC_AT_KILL); |
| if (!(val & PC_AT_KILL_VALID)) |
| break; |
| } |
| return 0; |
| } |
| |
| static void xive_cleanup_vp_ind(struct xive *x) |
| { |
| int i; |
| |
| xive_dbg(x, "Cleaning up %d VP ind entries...\n", x->vp_ind_count); |
| for (i = 0; i < x->vp_ind_count; i++) { |
| if (x->vp_ind_base[i] & VSD_FIRMWARE) { |
| xive_dbg(x, " %04x ... skip (firmware)\n", i); |
| continue; |
| } |
| if (x->vp_ind_base[i] != 0) { |
| x->vp_ind_base[i] = 0; |
| xive_dbg(x, " %04x ... cleaned\n", i); |
| } |
| } |
| xive_pc_ind_cache_kill(x); |
| } |
| |
| static void xive_cleanup_eq_ind(struct xive *x) |
| { |
| int i; |
| |
| xive_dbg(x, "Cleaning up %d EQ ind entries...\n", x->eq_ind_count); |
| for (i = 0; i < x->eq_ind_count; i++) { |
| if (x->eq_ind_base[i] & VSD_FIRMWARE) { |
| xive_dbg(x, " %04x ... skip (firmware)\n", i); |
| continue; |
| } |
| if (x->eq_ind_base[i] != 0) { |
| x->eq_ind_base[i] = 0; |
| xive_dbg(x, " %04x ... cleaned\n", i); |
| } |
| } |
| xive_vc_ind_cache_kill(x, VC_KILL_EQD); |
| } |
| #endif /* USE_INDIRECT */ |
| |
| static void xive_reset_one(struct xive *x) |
| { |
| struct cpu_thread *c; |
| bool eq_firmware; |
| int i; |
| |
| xive_dbg(x, "Resetting one xive...\n"); |
| |
| lock(&x->lock); |
| |
| /* Check all interrupts are disabled */ |
| i = bitmap_find_one_bit(*x->int_enabled_map, 0, MAX_INT_ENTRIES); |
| if (i >= 0) |
| xive_warn(x, "Interrupt %d (and maybe more) not disabled" |
| " at reset !\n", i); |
| |
| /* Reset IPI allocation */ |
| xive_dbg(x, "freeing alloc map %p/%p\n", |
| x->ipi_alloc_map, *x->ipi_alloc_map); |
| memset(x->ipi_alloc_map, 0, BITMAP_BYTES(MAX_INT_ENTRIES)); |
| |
| xive_dbg(x, "Resetting EQs...\n"); |
| |
| /* Reset all allocated EQs and free the user ones */ |
| bitmap_for_each_one(*x->eq_map, MAX_EQ_COUNT >> 3, i) { |
| struct xive_eq eq0; |
| struct xive_eq *eq; |
| int j; |
| |
| if (i == 0) |
| continue; |
| eq_firmware = false; |
| memset(&eq0, 0, sizeof(eq0)); |
| for (j = 0; j < 8; j++) { |
| uint32_t idx = (i << 3) | j; |
| |
| eq = xive_get_eq(x, idx); |
| if (!eq) |
| continue; |
| |
| /* We need to preserve the firmware bit, otherwise |
| * we will incorrectly free the EQs that are reserved |
| * for the physical CPUs |
| */ |
| eq0.w0 = eq->w0 & EQ_W0_FIRMWARE; |
| xive_eqc_cache_update(x, x->block_id, |
| idx, 0, 4, &eq0, false, true); |
| if (eq->w0 & EQ_W0_FIRMWARE) |
| eq_firmware = true; |
| } |
| if (!eq_firmware) |
| bitmap_clr_bit(*x->eq_map, i); |
| } |
| |
| /* Take out all VPs from HW and reset all CPPRs to 0 */ |
| for_each_present_cpu(c) { |
| if (c->chip_id != x->chip_id) |
| continue; |
| if (!c->xstate) |
| continue; |
| xive_cleanup_cpu_tima(c); |
| } |
| |
| /* Reset all user-allocated VPs. This is inefficient, we should |
| * either keep a bitmap of allocated VPs or add an iterator to |
| * the buddy which is trickier but doable. |
| */ |
| for (i = 0; i < MAX_VP_COUNT; i++) { |
| struct xive_vp *vp; |
| struct xive_vp vp0 = {0}; |
| |
| /* Ignore the physical CPU VPs */ |
| #ifdef USE_BLOCK_GROUP_MODE |
| if (i >= INITIAL_VP_BASE && |
| i < (INITIAL_VP_BASE + INITIAL_VP_COUNT)) |
| continue; |
| #else |
| if (x->block_id == 0 && |
| i >= INITIAL_BLK0_VP_BASE && |
| i < (INITIAL_BLK0_VP_BASE + INITIAL_BLK0_VP_BASE)) |
| continue; |
| #endif |
| /* Is the VP valid ? */ |
| vp = xive_get_vp(x, i); |
| if (!vp || !(vp->w0 & VP_W0_VALID)) |
| continue; |
| |
| /* Clear it */ |
| xive_vpc_cache_update(x, x->block_id, |
| i, 0, 8, &vp0, false, true); |
| } |
| |
| #ifndef USE_BLOCK_GROUP_MODE |
| /* If block group mode isn't enabled, reset VP alloc buddy */ |
| buddy_reset(x->vp_buddy); |
| #endif |
| |
| #ifdef USE_INDIRECT |
| /* Forget about remaining donated pages */ |
| list_head_init(&x->donated_pages); |
| |
| /* And cleanup donated indirect VP and EQ pages */ |
| xive_cleanup_vp_ind(x); |
| xive_cleanup_eq_ind(x); |
| #endif |
| |
| /* The rest must not be called with the lock held */ |
| unlock(&x->lock); |
| |
| /* Re-configure VPs and emulation */ |
| for_each_present_cpu(c) { |
| struct xive_cpu_state *xs = c->xstate; |
| |
| if (c->chip_id != x->chip_id || !xs) |
| continue; |
| |
| if (xive_mode == XIVE_MODE_EMU) |
| xive_init_cpu_emulation(xs, c); |
| else |
| xive_init_cpu_exploitation(xs); |
| } |
| } |
| |
| static void xive_reset_mask_source_cb(struct irq_source *is, |
| void *data __unused) |
| { |
| struct xive_src *s = container_of(is, struct xive_src, is); |
| struct xive *x; |
| uint32_t isn; |
| |
| if (is->ops != &xive_irq_source_ops) |
| return; |
| |
| /* Skip escalation sources */ |
| if (GIRQ_IS_ESCALATION(is->start)) |
| return; |
| |
| x = s->xive; |
| |
| /* Iterate all interrupts */ |
| for (isn = is->start; isn < is->end; isn++) { |
| /* Has it ever been enabled ? */ |
| if (!bitmap_tst_bit(*x->int_enabled_map, GIRQ_TO_IDX(isn))) |
| continue; |
| /* Mask it and clear the enabled map bit */ |
| xive_dbg(x, "[reset] disabling source 0x%x\n", isn); |
| __xive_set_irq_config(is, isn, 0, 0xff, isn, true, true); |
| bitmap_clr_bit(*x->int_enabled_map, GIRQ_TO_IDX(isn)); |
| } |
| } |
| |
| static int64_t opal_xive_reset(uint64_t version) |
| { |
| struct proc_chip *chip; |
| |
| prlog(PR_DEBUG, "XIVE reset, version: %d...\n", (int)version); |
| |
| if (version > 1) |
| return OPAL_PARAMETER; |
| |
| xive_mode = version; |
| |
| /* Mask all interrupt sources */ |
| irq_for_each_source(xive_reset_mask_source_cb, NULL); |
| |
| /* For each XIVE do a sync... */ |
| for_each_chip(chip) { |
| if (!chip->xive) |
| continue; |
| xive_sync(chip->xive); |
| } |
| |
| /* For each XIVE reset everything else... */ |
| for_each_chip(chip) { |
| if (!chip->xive) |
| continue; |
| xive_reset_one(chip->xive); |
| } |
| |
| #ifdef USE_BLOCK_GROUP_MODE |
| /* Cleanup global VP allocator */ |
| buddy_reset(xive_vp_buddy); |
| |
| /* We reserve the whole range of VPs representing HW chips. |
| * |
| * These are 0x80..0xff, so order 7 starting at 0x80. This will |
| * reserve that range on each chip. |
| */ |
| assert(buddy_reserve(xive_vp_buddy, 0x80, 7)); |
| #endif /* USE_BLOCK_GROUP_MODE */ |
| |
| return OPAL_SUCCESS; |
| } |
| |
| static int64_t opal_xive_free_vp_block(uint64_t vp_base) |
| { |
| uint32_t blk, idx, i, count; |
| uint8_t order; |
| bool group; |
| |
| if (xive_mode != XIVE_MODE_EXPL) |
| return OPAL_WRONG_STATE; |
| |
| if (!xive_decode_vp(vp_base, &blk, &idx, &order, &group)) |
| return OPAL_PARAMETER; |
| if (group) |
| return OPAL_PARAMETER; |
| #ifdef USE_BLOCK_GROUP_MODE |
| if (blk) |
| return OPAL_PARAMETER; |
| if (order < (xive_chips_alloc_bits + 1)) |
| return OPAL_PARAMETER; |
| if (idx & ((1 << (order - xive_chips_alloc_bits)) - 1)) |
| return OPAL_PARAMETER; |
| #else |
| if (order < 1) |
| return OPAL_PARAMETER; |
| if (idx & ((1 << order) - 1)) |
| return OPAL_PARAMETER; |
| #endif |
| |
| count = 1 << order; |
| for (i = 0; i < count; i++) { |
| uint32_t vp_id = vp_base + i; |
| uint32_t blk, idx, eq_blk, eq_idx; |
| struct xive *x; |
| struct xive_vp *vp; |
| |
| if (!xive_decode_vp(vp_id, &blk, &idx, NULL, NULL)) { |
| prerror("XIVE: Couldn't decode VP id %u\n", vp_id); |
| return OPAL_INTERNAL_ERROR; |
| } |
| x = xive_from_pc_blk(blk); |
| if (!x) { |
| prerror("XIVE: Instance not found for deallocated VP" |
| " block %d\n", blk); |
| return OPAL_INTERNAL_ERROR; |
| } |
| vp = xive_get_vp(x, idx); |
| if (!vp) { |
| prerror("XIVE: VP not found for deallocation !"); |
| return OPAL_INTERNAL_ERROR; |
| } |
| |
| /* VP must be disabled */ |
| if (vp->w0 & VP_W0_VALID) { |
| prerror("XIVE: Freeing enabled VP !\n"); |
| // XXX Disable it synchronously |
| } |
| |
| /* Not populated */ |
| if (vp->w1 == 0) |
| continue; |
| eq_blk = vp->w1 >> 28; |
| eq_idx = vp->w1 & 0x0fffffff; |
| vp->w1 = 0; |
| |
| if (eq_blk != blk) { |
| prerror("XIVE: Block mismatch trying to free EQs\n"); |
| return OPAL_INTERNAL_ERROR; |
| } |
| |
| /* XX Ensure the EQs are disabled */ |
| lock(&x->lock); |
| xive_free_eq_set(x, eq_idx); |
| unlock(&x->lock); |
| } |
| |
| xive_free_vps(vp_base); |
| |
| return OPAL_SUCCESS; |
| } |
| |
| static int64_t opal_xive_alloc_vp_block(uint32_t alloc_order) |
| { |
| uint32_t vp_base, eqs, count, i; |
| int64_t rc; |
| |
| if (xive_mode != XIVE_MODE_EXPL) |
| return OPAL_WRONG_STATE; |
| |
| prlog(PR_DEBUG, "opal_xive_alloc_vp_block(%d)\n", alloc_order); |
| |
| vp_base = xive_alloc_vps(alloc_order); |
| if (XIVE_ALLOC_IS_ERR(vp_base)) { |
| if (vp_base == XIVE_ALLOC_NO_IND) |
| return OPAL_XIVE_PROVISIONING; |
| return OPAL_RESOURCE; |
| } |
| |
| /* Allocate EQs and initialize VPs */ |
| count = 1 << alloc_order; |
| for (i = 0; i < count; i++) { |
| uint32_t vp_id = vp_base + i; |
| uint32_t blk, idx; |
| struct xive *x; |
| struct xive_vp *vp; |
| |
| if (!xive_decode_vp(vp_id, &blk, &idx, NULL, NULL)) { |
| prerror("XIVE: Couldn't decode VP id %u\n", vp_id); |
| return OPAL_INTERNAL_ERROR; |
| } |
| x = xive_from_pc_blk(blk); |
| if (!x) { |
| prerror("XIVE: Instance not found for allocated VP" |
| " block %d\n", blk); |
| rc = OPAL_INTERNAL_ERROR; |
| goto fail; |
| } |
| vp = xive_get_vp(x, idx); |
| if (!vp) { |
| prerror("XIVE: VP not found after allocation !"); |
| rc = OPAL_INTERNAL_ERROR; |
| goto fail; |
| } |
| |
| /* Allocate EQs, if fails, free the VPs and return */ |
| lock(&x->lock); |
| eqs = xive_alloc_eq_set(x, false); |
| unlock(&x->lock); |
| if (XIVE_ALLOC_IS_ERR(eqs)) { |
| if (eqs == XIVE_ALLOC_NO_IND) |
| rc = OPAL_XIVE_PROVISIONING; |
| else |
| rc = OPAL_RESOURCE; |
| goto fail; |
| } |
| |
| /* Initialize the VP structure. We don't use a cache watch |
| * as we have made sure when freeing the entries to scrub |
| * it out of the cache. |
| */ |
| memset(vp, 0, sizeof(*vp)); |
| vp->w1 = (blk << 28) | eqs; |
| vp->w5 = 0xff000000; |
| } |
| return vp_base; |
| fail: |
| opal_xive_free_vp_block(vp_base); |
| |
| return rc; |
| } |
| |
| static int64_t xive_try_allocate_irq(struct xive *x) |
| { |
| int idx, base_idx, max_count, girq; |
| struct xive_ive *ive; |
| |
| lock(&x->lock); |
| |
| base_idx = x->int_ipi_top - x->int_base; |
| max_count = x->int_hw_bot - x->int_ipi_top; |
| |
| idx = bitmap_find_zero_bit(*x->ipi_alloc_map, base_idx, max_count); |
| if (idx < 0) { |
| unlock(&x->lock); |
| return XIVE_ALLOC_NO_SPACE; |
| } |
| bitmap_set_bit(*x->ipi_alloc_map, idx); |
| girq = x->int_base + idx; |
| |
| /* Mark the IVE valid. Don't bother with the HW cache, it's |
| * still masked anyway, the cache will be updated when unmasked |
| * and configured. |
| */ |
| ive = xive_get_ive(x, girq); |
| if (!ive) { |
| bitmap_clr_bit(*x->ipi_alloc_map, idx); |
| unlock(&x->lock); |
| return OPAL_PARAMETER; |
| } |
| ive->w = IVE_VALID | IVE_MASKED | SETFIELD(IVE_EQ_DATA, 0ul, girq); |
| unlock(&x->lock); |
| |
| return girq; |
| } |
| |
| static int64_t opal_xive_allocate_irq(uint32_t chip_id) |
| { |
| struct proc_chip *chip; |
| bool try_all = false; |
| int64_t rc; |
| |
| if (xive_mode != XIVE_MODE_EXPL) |
| return OPAL_WRONG_STATE; |
| |
| if (chip_id == OPAL_XIVE_ANY_CHIP) { |
| try_all = true; |
| chip_id = this_cpu()->chip_id; |
| } |
| chip = get_chip(chip_id); |
| if (!chip) |
| return OPAL_PARAMETER; |
| |
| /* Try initial target chip */ |
| if (!chip->xive) |
| rc = OPAL_PARAMETER; |
| else |
| rc = xive_try_allocate_irq(chip->xive); |
| if (rc >= 0 || !try_all) |
| return rc; |
| |
| /* Failed and we try all... do so */ |
| for_each_chip(chip) { |
| if (!chip->xive) |
| continue; |
| rc = xive_try_allocate_irq(chip->xive); |
| if (rc >= 0) |
| break; |
| } |
| return rc; |
| } |
| |
| static int64_t opal_xive_free_irq(uint32_t girq) |
| { |
| struct irq_source *is = irq_find_source(girq); |
| struct xive_src *s = container_of(is, struct xive_src, is); |
| struct xive *x = xive_from_isn(girq); |
| struct xive_ive *ive; |
| uint32_t idx; |
| |
| if (xive_mode != XIVE_MODE_EXPL) |
| return OPAL_WRONG_STATE; |
| if (!x || !is) |
| return OPAL_PARAMETER; |
| |
| idx = GIRQ_TO_IDX(girq); |
| |
| lock(&x->lock); |
| |
| ive = xive_get_ive(x, girq); |
| if (!ive) { |
| unlock(&x->lock); |
| return OPAL_PARAMETER; |
| } |
| |
| /* Mask the interrupt source */ |
| xive_update_irq_mask(s, girq - s->esb_base, true); |
| |
| /* Mark the IVE masked and invalid */ |
| ive->w = IVE_MASKED | IVE_VALID; |
| xive_ivc_scrub(x, x->block_id, idx); |
| |
| /* Free it */ |
| if (!bitmap_tst_bit(*x->ipi_alloc_map, idx)) { |
| unlock(&x->lock); |
| return OPAL_PARAMETER; |
| } |
| bitmap_clr_bit(*x->ipi_alloc_map, idx); |
| bitmap_clr_bit(*x->int_enabled_map, idx); |
| unlock(&x->lock); |
| |
| return OPAL_SUCCESS; |
| } |
| |
| static int64_t opal_xive_dump_tm(uint32_t offset, const char *n, uint32_t pir) |
| { |
| struct cpu_thread *c = find_cpu_by_pir(pir); |
| struct xive_cpu_state *xs; |
| struct xive *x; |
| void *ind_tm_base; |
| uint64_t v0,v1; |
| |
| if (!c) |
| return OPAL_PARAMETER; |
| xs = c->xstate; |
| if (!xs || !xs->tm_ring1) |
| return OPAL_INTERNAL_ERROR; |
| x = xs->xive; |
| ind_tm_base = x->ic_base + (4 << x->ic_shift); |
| |
| lock(&x->lock); |
| |
| /* Setup indirect access to the corresponding thread */ |
| xive_regw(x, PC_TCTXT_INDIR0, |
| PC_TCTXT_INDIR_VALID | |
| SETFIELD(PC_TCTXT_INDIR_THRDID, 0ull, pir & 0xff)); |
| |
| /* Workaround for HW issue: Need to read the above register |
| * back before doing the subsequent accesses |
| */ |
| xive_regr(x, PC_TCTXT_INDIR0); |
| |
| v0 = in_be64(ind_tm_base + offset); |
| if (offset == TM_QW3_HV_PHYS) { |
| v1 = in_8(ind_tm_base + offset + 8); |
| v1 <<= 56; |
| } else { |
| v1 = in_be32(ind_tm_base + offset + 8); |
| v1 <<= 32; |
| } |
| prlog(PR_INFO, "CPU[%04x]: TM state for QW %s\n", pir, n); |
| prlog(PR_INFO, "CPU[%04x]: NSR CPPR IPB LSMFB ACK# INC AGE PIPR" |
| " W2 W3\n", pir); |
| prlog(PR_INFO, "CPU[%04x]: %02x %02x %02x %02x %02x " |
| "%02x %02x %02x %08x %08x\n", pir, |
| (uint8_t)(v0 >> 58) & 0xff, (uint8_t)(v0 >> 48) & 0xff, |
| (uint8_t)(v0 >> 40) & 0xff, (uint8_t)(v0 >> 32) & 0xff, |
| (uint8_t)(v0 >> 24) & 0xff, (uint8_t)(v0 >> 16) & 0xff, |
| (uint8_t)(v0 >> 8) & 0xff, (uint8_t)(v0 ) & 0xff, |
| (uint32_t)(v1 >> 32) & 0xffffffff, |
| (uint32_t)(v1 & 0xffffffff)); |
| |
| |
| xive_regw(x, PC_TCTXT_INDIR0, 0); |
| unlock(&x->lock); |
| |
| return OPAL_SUCCESS; |
| } |
| |
| static int64_t opal_xive_dump_vp(uint32_t vp_id) |
| { |
| uint32_t blk, idx; |
| uint8_t order; |
| bool group; |
| struct xive *x; |
| struct xive_vp *vp; |
| uint32_t *vpw; |
| |
| if (!xive_decode_vp(vp_id, &blk, &idx, &order, &group)) |
| return OPAL_PARAMETER; |
| |
| x = xive_from_vc_blk(blk); |
| if (!x) |
| return OPAL_PARAMETER; |
| vp = xive_get_vp(x, idx); |
| if (!vp) |
| return OPAL_PARAMETER; |
| lock(&x->lock); |
| |
| xive_vpc_scrub_clean(x, blk, idx); |
| |
| vpw = ((uint32_t *)vp) + (group ? 8 : 0); |
| prlog(PR_INFO, "VP[%08x]: 0..3: %08x %08x %08x %08x\n", vp_id, |
| vpw[0], vpw[1], vpw[2], vpw[3]); |
| prlog(PR_INFO, "VP[%08x]: 4..7: %08x %08x %08x %08x\n", vp_id, |
| vpw[4], vpw[5], vpw[6], vpw[7]); |
| unlock(&x->lock); |
| |
| return OPAL_SUCCESS; |
| } |
| |
| static int64_t __opal_xive_dump_emu(struct xive_cpu_state *xs, uint32_t pir) |
| { |
| struct xive_eq *eq; |
| uint32_t ipi_target; |
| uint8_t *mm, pq; |
| |
| prlog(PR_INFO, "CPU[%04x]: XIVE emulation state\n", pir); |
| |
| prlog(PR_INFO, "CPU[%04x]: cppr=%02x mfrr=%02x pend=%02x" |
| " prev_cppr=%02x total_irqs=%llx\n", pir, |
| xs->cppr, xs->mfrr, xs->pending, xs->prev_cppr, xs->total_irqs); |
| |
| prlog(PR_INFO, "CPU[%04x]: EQ IDX=%x MSK=%x G=%d [%08x %08x %08x > %08x %08x %08x %08x ...]\n", |
| pir, xs->eqptr, xs->eqmsk, xs->eqgen, |
| xs->eqbuf[(xs->eqptr - 3) & xs->eqmsk], |
| xs->eqbuf[(xs->eqptr - 2) & xs->eqmsk], |
| xs->eqbuf[(xs->eqptr - 1) & xs->eqmsk], |
| xs->eqbuf[(xs->eqptr + 0) & xs->eqmsk], |
| xs->eqbuf[(xs->eqptr + 1) & xs->eqmsk], |
| xs->eqbuf[(xs->eqptr + 2) & xs->eqmsk], |
| xs->eqbuf[(xs->eqptr + 3) & xs->eqmsk]); |
| |
| mm = xs->xive->esb_mmio + GIRQ_TO_IDX(xs->ipi_irq) * 0x20000; |
| pq = in_8(mm + 0x10800); |
| if (xive_get_irq_targetting(xs->ipi_irq, &ipi_target, NULL, NULL)) |
| prlog(PR_INFO, "CPU[%04x]: IPI #%08x PQ=%x target=%08x\n", |
| pir, xs->ipi_irq, pq, ipi_target); |
| else |
| prlog(PR_INFO, "CPU[%04x]: IPI #%08x PQ=%x target=??\n", |
| pir, xs->ipi_irq, pq); |
| |
| |
| |
| __xive_cache_scrub(xs->xive, xive_cache_eqc, xs->eq_blk, |
| xs->eq_idx + XIVE_EMULATION_PRIO, |
| false, false); |
| eq = xive_get_eq(xs->xive, xs->eq_idx + XIVE_EMULATION_PRIO); |
| prlog(PR_INFO, "CPU[%04x]: EQ @%p W0=%08x W1=%08x qbuf @%p\n", |
| pir, eq, eq->w0, eq->w1, xs->eqbuf); |
| |
| return OPAL_SUCCESS; |
| } |
| |
| static int64_t opal_xive_dump_emu(uint32_t pir) |
| { |
| struct cpu_thread *c = find_cpu_by_pir(pir); |
| struct xive_cpu_state *xs; |
| int64_t rc; |
| |
| if (!c) |
| return OPAL_PARAMETER; |
| |
| xs = c->xstate; |
| if (!xs) { |
| prlog(PR_INFO, " <none>\n"); |
| return OPAL_SUCCESS; |
| } |
| lock(&xs->lock); |
| rc = __opal_xive_dump_emu(xs, pir); |
| log_print(xs); |
| unlock(&xs->lock); |
| |
| return rc; |
| } |
| |
| static int64_t opal_xive_sync_irq_src(uint32_t girq) |
| { |
| struct xive *x = xive_from_isn(girq); |
| |
| if (!x) |
| return OPAL_PARAMETER; |
| return xive_sync(x); |
| } |
| |
| static int64_t opal_xive_sync_irq_target(uint32_t girq) |
| { |
| uint32_t target, vp_blk; |
| struct xive *x; |
| |
| if (!xive_get_irq_targetting(girq, &target, NULL, NULL)) |
| return OPAL_PARAMETER; |
| if (!xive_decode_vp(target, &vp_blk, NULL, NULL, NULL)) |
| return OPAL_PARAMETER; |
| x = xive_from_pc_blk(vp_blk); |
| if (!x) |
| return OPAL_PARAMETER; |
| return xive_sync(x); |
| } |
| |
| static int64_t opal_xive_sync(uint32_t type, uint32_t id) |
| { |
| int64_t rc = OPAL_SUCCESS;; |
| |
| if (type & XIVE_SYNC_EAS) |
| rc = opal_xive_sync_irq_src(id); |
| if (rc) |
| return rc; |
| if (type & XIVE_SYNC_QUEUE) |
| rc = opal_xive_sync_irq_target(id); |
| if (rc) |
| return rc; |
| |
| /* Add more ... */ |
| |
| return rc; |
| } |
| |
| static int64_t opal_xive_dump(uint32_t type, uint32_t id) |
| { |
| switch (type) { |
| case XIVE_DUMP_TM_HYP: |
| return opal_xive_dump_tm(TM_QW3_HV_PHYS, "PHYS", id); |
| case XIVE_DUMP_TM_POOL: |
| return opal_xive_dump_tm(TM_QW2_HV_POOL, "POOL", id); |
| case XIVE_DUMP_TM_OS: |
| return opal_xive_dump_tm(TM_QW1_OS, "OS ", id); |
| case XIVE_DUMP_TM_USER: |
| return opal_xive_dump_tm(TM_QW0_USER, "USER", id); |
| case XIVE_DUMP_VP: |
| return opal_xive_dump_vp(id); |
| case XIVE_DUMP_EMU_STATE: |
| return opal_xive_dump_emu(id); |
| default: |
| return OPAL_PARAMETER; |
| } |
| } |
| |
| static void xive_init_globals(void) |
| { |
| uint32_t i; |
| |
| for (i = 0; i < XIVE_MAX_CHIPS; i++) |
| xive_block_to_chip[i] = XIVE_INVALID_CHIP; |
| } |
| |
| void init_xive(void) |
| { |
| struct dt_node *np; |
| struct proc_chip *chip; |
| struct cpu_thread *cpu; |
| struct xive *one_xive; |
| bool first = true; |
| |
| /* Look for xive nodes and do basic inits */ |
| dt_for_each_compatible(dt_root, np, "ibm,power9-xive-x") { |
| struct xive *x; |
| |
| /* Initialize some global stuff */ |
| if (first) |
| xive_init_globals(); |
| |
| /* Create/initialize the xive instance */ |
| x = init_one_xive(np); |
| if (first) |
| one_xive = x; |
| first = false; |
| } |
| if (first) |
| return; |
| |
| /* Init VP allocator */ |
| xive_init_vp_allocator(); |
| |
| /* Create a device-tree node for Linux use */ |
| xive_create_mmio_dt_node(one_xive); |
| |
| /* Some inits must be done after all xive have been created |
| * such as setting up the forwarding ports |
| */ |
| for_each_chip(chip) { |
| if (chip->xive) |
| late_init_one_xive(chip->xive); |
| } |
| |
| /* Initialize XICS emulation per-cpu structures */ |
| for_each_present_cpu(cpu) { |
| xive_init_cpu(cpu); |
| } |
| /* Add interrupts propertie to each CPU node */ |
| for_each_present_cpu(cpu) { |
| if (cpu_is_thread0(cpu)) |
| xive_init_cpu_properties(cpu); |
| } |
| |
| /* Calling boot CPU */ |
| xive_cpu_callin(this_cpu()); |
| |
| /* Register XICS emulation calls */ |
| opal_register(OPAL_INT_GET_XIRR, opal_xive_get_xirr, 2); |
| opal_register(OPAL_INT_SET_CPPR, opal_xive_set_cppr, 1); |
| opal_register(OPAL_INT_EOI, opal_xive_eoi, 1); |
| opal_register(OPAL_INT_SET_MFRR, opal_xive_set_mfrr, 2); |
| |
| /* Register XIVE exploitation calls */ |
| opal_register(OPAL_XIVE_RESET, opal_xive_reset, 1); |
| opal_register(OPAL_XIVE_GET_IRQ_INFO, opal_xive_get_irq_info, 6); |
| opal_register(OPAL_XIVE_GET_IRQ_CONFIG, opal_xive_get_irq_config, 4); |
| opal_register(OPAL_XIVE_SET_IRQ_CONFIG, opal_xive_set_irq_config, 4); |
| opal_register(OPAL_XIVE_GET_QUEUE_INFO, opal_xive_get_queue_info, 7); |
| opal_register(OPAL_XIVE_SET_QUEUE_INFO, opal_xive_set_queue_info, 5); |
| opal_register(OPAL_XIVE_DONATE_PAGE, opal_xive_donate_page, 2); |
| opal_register(OPAL_XIVE_ALLOCATE_IRQ, opal_xive_allocate_irq, 1); |
| opal_register(OPAL_XIVE_FREE_IRQ, opal_xive_free_irq, 1); |
| opal_register(OPAL_XIVE_ALLOCATE_VP_BLOCK, opal_xive_alloc_vp_block, 1); |
| opal_register(OPAL_XIVE_FREE_VP_BLOCK, opal_xive_free_vp_block, 1); |
| opal_register(OPAL_XIVE_GET_VP_INFO, opal_xive_get_vp_info, 5); |
| opal_register(OPAL_XIVE_SET_VP_INFO, opal_xive_set_vp_info, 3); |
| opal_register(OPAL_XIVE_SYNC, opal_xive_sync, 2); |
| opal_register(OPAL_XIVE_DUMP, opal_xive_dump, 2); |
| } |
| |