| // SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later |
| /* |
| * NVLink1, supported by the NPU (POWER8) |
| * |
| * Copyright 2013-2019 IBM Corp. |
| */ |
| |
| #include <skiboot.h> |
| #include <io.h> |
| #include <timebase.h> |
| #include <pci.h> |
| #include <pci-cfg.h> |
| #include <pci-virt.h> |
| #include <pci-slot.h> |
| #include <interrupts.h> |
| #include <opal.h> |
| #include <opal-api.h> |
| #include <cpu.h> |
| #include <device.h> |
| #include <ccan/str/str.h> |
| #include <ccan/array_size/array_size.h> |
| #include <ccan/build_assert/build_assert.h> |
| #include <affinity.h> |
| #include <npu-regs.h> |
| #include <npu.h> |
| #include <xscom.h> |
| #include <string.h> |
| |
| /* |
| * Terminology: |
| * |
| * Brick - A group of either 8 TX or 8 RX lanes |
| * Link - A group of 8 TX and 8 RX lanes |
| * |
| * Each link is represented in system software as an emulated PCI |
| * device. Garrison has two chips each with 4 links, therefore there |
| * are 8 emulated PCI devices in total. |
| * |
| * +----------------------------------------------------------------+ |
| * | PBCQ3 (SCOM Base Address 0x2012c00) | |
| * | PHB3 (SCOM Base Address 0x9012c00) | |
| * +----------------------------------------------------------------+ |
| * |||||||| |||||||| |
| * |||||||| |||||||| |
| * |||||||| |||||||| |
| * |||||||| |||||||| |
| * +----------------------------------------------------------------+ |
| * | PCIe x8 | |
| * +----------------------------------------------------------------+ |
| * | GPU0 | |
| * +--------------------------------+-------------------------------+ |
| * | NV Link 1 | NV Link 0 | |
| * +---------------+----------------+---------------+---------------+ |
| * | RX | TX | RX | TX | |
| * +---------------+----------------+---------------+---------------+ |
| * |||||||| |||||||| |||||||| |||||||| |
| * |||||||| |||||||| |||||||| |||||||| |
| * |||||||| |||||||| |||||||| |||||||| |
| * |||||||| |||||||| |||||||| |||||||| |
| * +---------------+----------------+---------------+---------------+ |
| * | TX | RX | TX | RX | |
| * +---------------+----------------+---------------+---------------+ |
| * | Lanes [0:7] PHY 0 Lanes [8:15] | |
| * | SCOM Base Address 0x8000080008010c3f | |
| * +--------------------------------+-------------------------------+ |
| * | Link 0 NDL/NTL | Link 1 NTL/NDL | |
| * | SCOM Base Address 0x8013c00 | SCOM Base Address 0x8013c40 | |
| * +--------------------------------+-------------------------------+ |
| * | | |
| * | Address Translation/AT (shared for all links) | |
| * | SCOM Base Address 0x8013d80 | |
| * | | |
| * +--------------------------------+-------------------------------+ |
| * | Link 3 NDL/NTL | Link 4 NTL/NDL | |
| * | SCOM Base Address 0x8013d00 | SCOM Base Address 0x8013d40 | |
| * +--------------------------------+-------------------------------+ |
| * | Lanes [8:15] PHY 1 Lanes [0:7] | |
| * | SCOM Base Address 0x8000080008010c7f | |
| * +---------------+----------------+---------------+---------------+ |
| * | TX | RX | TX | RX | |
| * +---------------+----------------+---------------+---------------+ |
| * |||||||| |||||||| |||||||| |||||||| |
| * |||||||| |||||||| |||||||| |||||||| |
| * |||||||| |||||||| |||||||| |||||||| |
| * |||||||| |||||||| |||||||| |||||||| |
| * +---------------+----------------+---------------+---------------+ |
| * | RX | TX | RX | TX | |
| * +---------------+----------------+---------------+---------------+ |
| * | NV Link 2 | NV Link 3 | |
| * +--------------------------------+-------------------------------+ |
| * | GPU1 | |
| * +----------------------------------------------------------------+ |
| * | PCIe x8 | |
| * +----------------------------------------------------------------+ |
| * |||||||| |||||||| |
| * |||||||| |||||||| |
| * |||||||| |||||||| |
| * |||||||| |||||||| |
| * +----------------------------------------------------------------+ |
| * | PHB2 (SCOM Base Address 0x9012800) | |
| * | PBCQ2 (SCOM Base Address 0x2012800) | |
| * +----------------------------------------------------------------+ |
| * |
| */ |
| |
| static struct npu_dev_cap *npu_dev_find_capability(struct npu_dev *dev, |
| uint16_t id); |
| |
| #define OPAL_NPU_VERSION 0x02 |
| |
| #define PCIE_CAP_START 0x40 |
| #define PCIE_CAP_END 0x80 |
| #define VENDOR_CAP_START 0x80 |
| #define VENDOR_CAP_END 0x90 |
| |
| #define VENDOR_CAP_PCI_DEV_OFFSET 0x0d |
| |
| /* Returns the scom base for the given link index */ |
| static uint64_t npu_link_scom_base(struct dt_node *dn, uint32_t scom_base, |
| int index) |
| { |
| struct dt_node *link; |
| uint32_t link_index; |
| char namebuf[32]; |
| |
| snprintf(namebuf, sizeof(namebuf), "link@%x", index); |
| link = dt_find_by_name(dn, namebuf); |
| assert(link); |
| link_index = dt_prop_get_u32(link, "ibm,npu-link-index"); |
| return scom_base + (link_index * NPU_LINK_SIZE); |
| } |
| |
| static uint64_t get_bar_size(uint64_t bar) |
| { |
| return (1 << GETFIELD(NX_MMIO_BAR_SIZE, bar)) * 0x10000; |
| } |
| |
| /* Update the changes of the device BAR to link BARs */ |
| static void npu_dev_bar_update(uint32_t gcid, struct npu_dev_bar *bar, |
| bool enable) |
| { |
| uint64_t val; |
| |
| if (!bar->xscom) |
| return; |
| |
| val = bar->base; |
| val = SETFIELD(NX_MMIO_BAR_SIZE, val, ilog2(bar->size / 0x10000)); |
| if (enable) |
| val |= NX_MMIO_BAR_ENABLE; |
| xscom_write(gcid, bar->xscom, val); |
| } |
| |
| /* Trap for PCI command (0x4) to enable or disable device's BARs */ |
| static int64_t npu_dev_cfg_write_cmd(void *dev, |
| struct pci_cfg_reg_filter *pcrf __unused, |
| uint32_t offset, uint32_t size, |
| uint32_t *data, bool write) |
| { |
| struct pci_virt_device *pvd = dev; |
| struct npu_dev *ndev = pvd->data; |
| bool enable; |
| |
| if (!write) |
| return OPAL_PARTIAL; |
| |
| if (offset != PCI_CFG_CMD) |
| return OPAL_PARAMETER; |
| if (size != 1 && size != 2 && size != 4) |
| return OPAL_PARAMETER; |
| |
| /* Update device BARs and link BARs will be syncrhonized |
| * with hardware automatically. |
| */ |
| enable = !!(*data & PCI_CFG_CMD_MEM_EN); |
| npu_dev_bar_update(ndev->npu->chip_id, &ndev->bar, enable); |
| |
| /* Normal path to update PCI config buffer */ |
| return OPAL_PARTIAL; |
| } |
| |
| /* |
| * Trap for memory BARs: 0xFF's should be written to BAR register |
| * prior to getting its size. |
| */ |
| static int64_t npu_dev_cfg_bar_read(struct npu_dev *dev __unused, |
| struct pci_cfg_reg_filter *pcrf, |
| uint32_t offset, uint32_t size, |
| uint32_t *data) |
| { |
| struct npu_dev_bar *bar = (struct npu_dev_bar *)(pcrf->data); |
| |
| /* Revert to normal path if we weren't trapped for BAR size */ |
| if (!bar->trapped) |
| return OPAL_PARTIAL; |
| |
| if (offset != pcrf->start && |
| offset != pcrf->start + 4) |
| return OPAL_PARAMETER; |
| if (size != 4) |
| return OPAL_PARAMETER; |
| |
| bar->trapped = false; |
| *data = bar->bar_sz; |
| return OPAL_SUCCESS; |
| } |
| |
| static int64_t npu_dev_cfg_bar_write(struct npu_dev *dev, |
| struct pci_cfg_reg_filter *pcrf, |
| uint32_t offset, uint32_t size, |
| uint32_t data) |
| { |
| struct pci_virt_device *pvd = dev->pvd; |
| struct npu_dev_bar *bar = (struct npu_dev_bar *)(pcrf->data); |
| uint32_t pci_cmd; |
| |
| if (offset != pcrf->start && |
| offset != pcrf->start + 4) |
| return OPAL_PARAMETER; |
| if (size != 4) |
| return OPAL_PARAMETER; |
| |
| /* Return BAR size on next read */ |
| if (data == 0xffffffff) { |
| bar->trapped = true; |
| if (offset == pcrf->start) |
| bar->bar_sz = (bar->size & 0xffffffff); |
| else |
| bar->bar_sz = (bar->size >> 32); |
| |
| return OPAL_SUCCESS; |
| } |
| |
| /* Update BAR base address */ |
| if (offset == pcrf->start) { |
| bar->base &= 0xffffffff00000000UL; |
| bar->base |= (data & 0xfffffff0); |
| } else { |
| bar->base &= 0x00000000ffffffffUL; |
| bar->base |= ((uint64_t)data << 32); |
| |
| PCI_VIRT_CFG_NORMAL_RD(pvd, PCI_CFG_CMD, 4, &pci_cmd); |
| npu_dev_bar_update(dev->npu->chip_id, bar, |
| !!(pci_cmd & PCI_CFG_CMD_MEM_EN)); |
| } |
| |
| /* We still depend on the normal path to update the |
| * cached config buffer. |
| */ |
| return OPAL_PARAMETER; |
| } |
| |
| static int64_t npu_dev_cfg_bar(void *dev, struct pci_cfg_reg_filter *pcrf, |
| uint32_t offset, uint32_t len, uint32_t *data, |
| bool write) |
| { |
| struct pci_virt_device *pvd = dev; |
| struct npu_dev *ndev = pvd->data; |
| |
| if (write) |
| return npu_dev_cfg_bar_write(ndev, pcrf, offset, len, *data); |
| |
| return npu_dev_cfg_bar_read(ndev, pcrf, offset, len, data); |
| } |
| |
| static int64_t npu_dev_cfg_exp_devcap(void *dev, |
| struct pci_cfg_reg_filter *pcrf __unused, |
| uint32_t offset, uint32_t size, |
| uint32_t *data, bool write) |
| { |
| struct pci_virt_device *pvd = dev; |
| struct npu_dev *ndev = pvd->data; |
| |
| assert(write); |
| |
| if ((size != 2) || (offset & 1)) { |
| /* Short config writes are not supported */ |
| prlog(PR_ERR, "NPU%d: Unsupported write to pcie control register\n", |
| ndev->phb->opal_id); |
| return OPAL_PARAMETER; |
| } |
| |
| if (*data & PCICAP_EXP_DEVCTL_FUNC_RESET) |
| npu_dev_procedure_reset(ndev); |
| |
| return OPAL_PARTIAL; |
| } |
| |
| static struct npu_dev *bdfn_to_npu_dev(struct npu *p, uint32_t bdfn) |
| { |
| struct pci_virt_device *pvd; |
| |
| /* Sanity check */ |
| if (bdfn & ~0xff) |
| return NULL; |
| |
| pvd = pci_virt_find_device(&p->phb, bdfn); |
| if (pvd) |
| return pvd->data; |
| |
| return NULL; |
| } |
| |
| #define NPU_CFG_READ(size, type) \ |
| static int64_t npu_cfg_read##size(struct phb *phb, uint32_t bdfn, \ |
| uint32_t offset, type *data) \ |
| { \ |
| uint32_t val; \ |
| int64_t ret; \ |
| \ |
| ret = pci_virt_cfg_read(phb, bdfn, offset, sizeof(*data), &val); \ |
| *data = (type)val; \ |
| return ret; \ |
| } |
| #define NPU_CFG_WRITE(size, type) \ |
| static int64_t npu_cfg_write##size(struct phb *phb, uint32_t bdfn, \ |
| uint32_t offset, type data) \ |
| { \ |
| uint32_t val = data; \ |
| \ |
| return pci_virt_cfg_write(phb, bdfn, offset, sizeof(data), val); \ |
| } |
| |
| NPU_CFG_READ(8, u8); |
| NPU_CFG_READ(16, u16); |
| NPU_CFG_READ(32, u32); |
| NPU_CFG_WRITE(8, u8); |
| NPU_CFG_WRITE(16, u16); |
| NPU_CFG_WRITE(32, u32); |
| |
| static int __npu_dev_bind_pci_dev(struct phb *phb __unused, |
| struct pci_device *pd, |
| void *data) |
| { |
| struct npu_dev *dev = data; |
| struct dt_node *pci_dt_node; |
| char *pcislot; |
| |
| /* Ignore non-nvidia PCI devices */ |
| if ((pd->vdid & 0xffff) != 0x10de) |
| return 0; |
| |
| /* Find the PCI device's slot location */ |
| for (pci_dt_node = pd->dn; |
| pci_dt_node && !dt_find_property(pci_dt_node, "ibm,slot-label"); |
| pci_dt_node = pci_dt_node->parent); |
| |
| if (!pci_dt_node) |
| return 0; |
| |
| pcislot = (char *)dt_prop_get(pci_dt_node, "ibm,slot-label"); |
| |
| prlog(PR_DEBUG, "NPU: comparing GPU %s and NPU %s\n", |
| pcislot, dev->slot_label); |
| |
| if (streq(pcislot, dev->slot_label)) |
| return 1; |
| |
| return 0; |
| } |
| |
| static void npu_dev_bind_pci_dev(struct npu_dev *dev) |
| { |
| struct phb *phb; |
| uint32_t i; |
| |
| if (dev->pd) |
| return; |
| |
| for (i = 0; i < 64; i++) { |
| if (dev->npu->phb.opal_id == i) |
| continue; |
| |
| phb = pci_get_phb(i); |
| if (!phb) |
| continue; |
| |
| dev->pd = pci_walk_dev(phb, NULL, __npu_dev_bind_pci_dev, dev); |
| if (dev->pd) { |
| dev->phb = phb; |
| /* Found the device, set the bit in config space */ |
| PCI_VIRT_CFG_INIT_RO(dev->pvd, VENDOR_CAP_START + |
| VENDOR_CAP_PCI_DEV_OFFSET, 1, 0x01); |
| return; |
| } |
| } |
| |
| prlog(PR_INFO, "%s: No PCI device for NPU device %04x:%02x:%02x.%x to bind to. If you expect a GPU to be there, this is a problem.\n", |
| __func__, dev->npu->phb.opal_id, |
| dev->pvd->bdfn >> 8 & 0xff, |
| dev->pvd->bdfn >> 3 & 0x1f, |
| dev->pvd->bdfn & 0x7); |
| |
| } |
| |
| static struct lock pci_npu_phandle_lock = LOCK_UNLOCKED; |
| |
| /* Appends an NPU phandle to the given PCI device node ibm,npu |
| * property */ |
| static void npu_append_pci_phandle(struct dt_node *dn, u32 phandle) |
| { |
| uint32_t *npu_phandles; |
| struct dt_property *pci_npu_phandle_prop; |
| size_t prop_len; |
| |
| /* Use a lock to make sure no one else has a reference to an |
| * ibm,npu property (this assumes this is the only function |
| * that holds a reference to it). */ |
| lock(&pci_npu_phandle_lock); |
| |
| /* This function shouldn't be called unless ibm,npu exists */ |
| pci_npu_phandle_prop = (struct dt_property *) |
| dt_require_property(dn, "ibm,npu", -1); |
| |
| /* Need to append to the properties */ |
| prop_len = pci_npu_phandle_prop->len; |
| prop_len += sizeof(*npu_phandles); |
| dt_resize_property(&pci_npu_phandle_prop, prop_len); |
| |
| npu_phandles = (uint32_t *) pci_npu_phandle_prop->prop; |
| npu_phandles[prop_len/sizeof(*npu_phandles) - 1] = phandle; |
| unlock(&pci_npu_phandle_lock); |
| } |
| |
| static int npu_dn_fixup(struct phb *phb, |
| struct pci_device *pd, |
| void *data __unused) |
| { |
| struct npu *p = phb_to_npu(phb); |
| struct npu_dev *dev; |
| |
| dev = bdfn_to_npu_dev(p, pd->bdfn); |
| assert(dev); |
| |
| if (dev->phb || dev->pd) |
| return 0; |
| |
| /* NPU devices require a slot location to associate with GPUs */ |
| dev->slot_label = dt_prop_get(pd->dn, "ibm,slot-label"); |
| |
| /* Bind the emulated PCI device with the real one, which can't |
| * be done until the PCI devices are populated. Once the real |
| * PCI device is identified, we also need fix the device-tree |
| * for it |
| */ |
| npu_dev_bind_pci_dev(dev); |
| if (dev->phb && dev->pd && dev->pd->dn) { |
| if (dt_find_property(dev->pd->dn, "ibm,npu")) |
| npu_append_pci_phandle(dev->pd->dn, pd->dn->phandle); |
| else |
| dt_add_property_cells(dev->pd->dn, "ibm,npu", pd->dn->phandle); |
| |
| dt_add_property_cells(pd->dn, "ibm,gpu", dev->pd->dn->phandle); |
| } |
| |
| return 0; |
| } |
| |
| static void npu_phb_final_fixup(struct phb *phb) |
| { |
| pci_walk_dev(phb, NULL, npu_dn_fixup, NULL); |
| } |
| |
| static void npu_ioda_init(struct npu *p) |
| { |
| uint64_t *data64; |
| uint32_t i; |
| |
| /* LXIVT - Disable all LSIs */ |
| for (i = 0; i < ARRAY_SIZE(p->lxive_cache); i++) { |
| data64 = &p->lxive_cache[i]; |
| *data64 = SETFIELD(NPU_IODA_LXIVT_PRIORITY, 0ul, 0xff); |
| *data64 = SETFIELD(NPU_IODA_LXIVT_SERVER, *data64, 0); |
| } |
| |
| /* PCT - Reset to reserved PE# */ |
| for (i = 0; i < ARRAY_SIZE(p->pce_cache); i++) { |
| data64 = &p->pce_cache[i]; |
| *data64 = SETFIELD(NPU_IODA_PCT_PE, 0ul, 0ul); |
| *data64 |= NPU_IODA_PCT_LINK_ENABLED; |
| } |
| |
| /* Clear TVT */ |
| memset(p->tve_cache, 0, sizeof(p->tve_cache)); |
| } |
| |
| static int64_t npu_ioda_reset(struct phb *phb, bool purge) |
| { |
| struct npu *p = phb_to_npu(phb); |
| uint32_t i; |
| |
| if (purge) { |
| NPUDBG(p, "Purging all IODA tables...\n"); |
| npu_ioda_init(p); |
| } |
| |
| /* LIST */ |
| npu_ioda_sel(p, NPU_IODA_TBL_LIST, 0, true); |
| for (i = 0; i < 8; i++) |
| out_be64(p->at_regs + NPU_IODA_DATA0, 0x1); |
| |
| /* LIXVT */ |
| npu_ioda_sel(p, NPU_IODA_TBL_LXIVT, 0, true); |
| for (i = 0; i < ARRAY_SIZE(p->lxive_cache); i++) |
| out_be64(p->at_regs + NPU_IODA_DATA0, p->lxive_cache[i]); |
| |
| /* PCT */ |
| npu_ioda_sel(p, NPU_IODA_TBL_PCT, 0, true); |
| for (i = 0; i < ARRAY_SIZE(p->pce_cache); i++) |
| out_be64(p->at_regs + NPU_IODA_DATA0, p->pce_cache[i]); |
| |
| /* TVT */ |
| npu_ioda_sel(p, NPU_IODA_TBL_TVT, 0, true); |
| for (i = 0; i < ARRAY_SIZE(p->tve_cache); i++) |
| out_be64(p->at_regs + NPU_IODA_DATA0, p->tve_cache[i]); |
| |
| return OPAL_SUCCESS; |
| } |
| |
| static int npu_isn_valid(struct npu *p, uint32_t isn) |
| { |
| if (p->chip_id != p8_irq_to_chip(isn) || p->index != 0 || |
| NPU_IRQ_NUM(isn) < NPU_LSI_IRQ_MIN || |
| NPU_IRQ_NUM(isn) > NPU_LSI_IRQ_MAX) { |
| /** |
| * @fwts-label NPUisnInvalid |
| * @fwts-advice NVLink not functional |
| */ |
| prlog(PR_ERR, "NPU%d: isn 0x%x not valid for this NPU\n", |
| p->phb.opal_id, isn); |
| return false; |
| } |
| |
| return true; |
| } |
| |
| static int64_t npu_lsi_get_xive(struct irq_source *is, uint32_t isn, |
| uint16_t *server, uint8_t *prio) |
| { |
| struct npu *p = is->data; |
| uint32_t irq = NPU_IRQ_NUM(isn); |
| uint64_t lxive; |
| |
| if (!npu_isn_valid(p, isn)) |
| return OPAL_PARAMETER; |
| |
| /* The content is fetched from the cache, which requires |
| * that the initial cache should be initialized with the |
| * default values |
| */ |
| irq -= NPU_LSI_IRQ_MIN; |
| lxive = p->lxive_cache[irq]; |
| *server = GETFIELD(NPU_IODA_LXIVT_SERVER, lxive); |
| *prio = GETFIELD(NPU_IODA_LXIVT_PRIORITY, lxive); |
| |
| return OPAL_SUCCESS; |
| } |
| |
| static int64_t npu_lsi_set_xive(struct irq_source *is, uint32_t isn, |
| uint16_t server, uint8_t prio) |
| { |
| struct npu *p = is->data; |
| uint32_t irq = NPU_IRQ_NUM(isn); |
| uint64_t lxive; |
| |
| if (!npu_isn_valid(p, isn)) |
| return OPAL_PARAMETER; |
| |
| /* Figure out LXIVT entry */ |
| lxive = SETFIELD(NPU_IODA_LXIVT_SERVER, 0ul, server); |
| lxive = SETFIELD(NPU_IODA_LXIVT_PRIORITY, lxive, prio); |
| |
| /* Cache LXIVT entry */ |
| irq -= NPU_LSI_IRQ_MIN; |
| p->lxive_cache[irq] = lxive; |
| |
| /* Update to LXIVT entry */ |
| npu_ioda_sel(p, NPU_IODA_TBL_LXIVT, irq, false); |
| lxive = in_be64(p->at_regs + NPU_IODA_DATA0); |
| lxive = SETFIELD(NPU_IODA_LXIVT_SERVER, lxive, server); |
| lxive = SETFIELD(NPU_IODA_LXIVT_PRIORITY, lxive, prio); |
| out_be64(p->at_regs + NPU_IODA_DATA0, lxive); |
| |
| return OPAL_SUCCESS; |
| } |
| |
| static void npu_err_interrupt(struct irq_source *is, uint32_t isn) |
| { |
| struct npu *p = is->data; |
| uint32_t irq = NPU_IRQ_NUM(isn); |
| |
| if (!npu_isn_valid(p, isn)) |
| return; |
| |
| /* There're 4 LSIs used for error reporting: 4/5 for data |
| * link error reporting while 6/7 for frozen PE detection |
| */ |
| irq -= NPU_LSI_IRQ_MIN; |
| switch (irq) { |
| case 4 ... 5: |
| prerror("Invalid NPU error interrupt received\n"); |
| break; |
| case 6 ... 7: |
| opal_update_pending_evt(OPAL_EVENT_PCI_ERROR, |
| OPAL_EVENT_PCI_ERROR); |
| } |
| } |
| |
| static uint64_t npu_lsi_attributes(struct irq_source *is, uint32_t isn) |
| { |
| struct npu *p = is->data; |
| uint32_t idx = isn - p->base_lsi; |
| |
| if (idx >= 4) |
| return IRQ_ATTR_TARGET_OPAL | IRQ_ATTR_TARGET_RARE | IRQ_ATTR_TYPE_LSI; |
| return IRQ_ATTR_TARGET_LINUX; |
| } |
| |
| /* Error LSIs (skiboot owned) */ |
| static const struct irq_source_ops npu_lsi_irq_ops = { |
| .get_xive = npu_lsi_get_xive, |
| .set_xive = npu_lsi_set_xive, |
| .attributes = npu_lsi_attributes, |
| .interrupt = npu_err_interrupt, |
| }; |
| |
| static void npu_register_irq(struct npu *p) |
| { |
| register_irq_source(&npu_lsi_irq_ops, p, p->base_lsi, 8); |
| } |
| |
| static void npu_hw_init(struct npu *p) |
| { |
| /* 3 MMIO setup for AT */ |
| out_be64(p->at_regs + NPU_LSI_SOURCE_ID, |
| SETFIELD(NPU_LSI_SRC_ID_BASE, 0ul, NPU_LSI_IRQ_MIN >> 4)); |
| BUILD_ASSERT((NPU_LSI_IRQ_MIN & 0x07F0) == NPU_LSI_IRQ_MIN); |
| out_be64(p->at_regs + NPU_INTREP_TIMER, 0x0ul); |
| npu_ioda_reset(&p->phb, false); |
| } |
| |
| static int64_t npu_map_pe_dma_window_real(struct phb *phb, |
| uint64_t pe_number, |
| uint16_t window_id, |
| uint64_t pci_start_addr, |
| uint64_t pci_mem_size) |
| { |
| struct npu *p = phb_to_npu(phb); |
| uint64_t end; |
| uint64_t tve; |
| |
| /* Sanity check. Each PE has one corresponding TVE */ |
| if (pe_number >= NPU_NUM_OF_PES || |
| window_id != pe_number) |
| return OPAL_PARAMETER; |
| |
| if (pci_mem_size) { |
| /* Enable */ |
| |
| end = pci_start_addr + pci_mem_size; |
| |
| /* We have to be 16M aligned */ |
| if ((pci_start_addr & 0x00ffffff) || |
| (pci_mem_size & 0x00ffffff)) |
| return OPAL_PARAMETER; |
| |
| /* |
| * It *looks* like this is the max we can support (we need |
| * to verify this. Also we are not checking for rollover, |
| * but then we aren't trying too hard to protect ourselves |
| * againt a completely broken OS. |
| */ |
| if (end > 0x0003ffffffffffffull) |
| return OPAL_PARAMETER; |
| |
| /* |
| * Put start address bits 49:24 into TVE[52:53]||[0:23] |
| * and end address bits 49:24 into TVE[54:55]||[24:47] |
| * and set TVE[51] |
| */ |
| tve = (pci_start_addr << 16) & (0xffffffull << 48); |
| tve |= (pci_start_addr >> 38) & (3ull << 10); |
| tve |= (end >> 8) & (0xfffffful << 16); |
| tve |= (end >> 40) & (3ull << 8); |
| tve |= PPC_BIT(51); |
| } else { |
| /* Disable */ |
| tve = 0; |
| } |
| |
| npu_ioda_sel(p, NPU_IODA_TBL_TVT, window_id, false); |
| out_be64(p->at_regs + NPU_IODA_DATA0, tve); |
| p->tve_cache[window_id] = tve; |
| |
| return OPAL_SUCCESS; |
| } |
| |
| static int64_t npu_map_pe_dma_window(struct phb *phb, |
| uint64_t pe_number, |
| uint16_t window_id, |
| uint16_t tce_levels, |
| uint64_t tce_table_addr, |
| uint64_t tce_table_size, |
| uint64_t tce_page_size) |
| { |
| struct npu *p = phb_to_npu(phb); |
| uint64_t tts_encoded; |
| uint64_t data64 = 0; |
| |
| /* Sanity check. Each PE has one corresponding TVE */ |
| if (pe_number >= NPU_NUM_OF_PES || |
| window_id != pe_number) |
| return OPAL_PARAMETER; |
| |
| /* Special condition, zero TCE table size used to disable |
| * the TVE. |
| */ |
| if (!tce_table_size) { |
| npu_ioda_sel(p, NPU_IODA_TBL_TVT, window_id, false); |
| out_be64(p->at_regs + NPU_IODA_DATA0, 0ul); |
| p->tve_cache[window_id] = 0ul; |
| return OPAL_SUCCESS; |
| } |
| |
| /* Additional arguments validation */ |
| if (tce_levels < 1 || |
| tce_levels > 4 || |
| !is_pow2(tce_table_size) || |
| tce_table_size < 0x1000) |
| return OPAL_PARAMETER; |
| |
| /* TCE table size */ |
| data64 = SETFIELD(NPU_IODA_TVT_TTA, 0ul, tce_table_addr >> 12); |
| tts_encoded = ilog2(tce_table_size) - 11; |
| if (tts_encoded > 39) |
| return OPAL_PARAMETER; |
| data64 = SETFIELD(NPU_IODA_TVT_SIZE, data64, tts_encoded); |
| |
| /* TCE page size */ |
| switch (tce_page_size) { |
| case 0x10000: /* 64K */ |
| data64 = SETFIELD(NPU_IODA_TVT_PSIZE, data64, 5); |
| break; |
| case 0x1000000: /* 16M */ |
| data64 = SETFIELD(NPU_IODA_TVT_PSIZE, data64, 13); |
| break; |
| case 0x10000000: /* 256M */ |
| data64 = SETFIELD(NPU_IODA_TVT_PSIZE, data64, 17); |
| break; |
| case 0x1000: /* 4K */ |
| default: |
| data64 = SETFIELD(NPU_IODA_TVT_PSIZE, data64, 1); |
| } |
| |
| /* Number of levels */ |
| data64 = SETFIELD(NPU_IODA_TVT_LEVELS, data64, tce_levels - 1); |
| |
| /* Update to hardware */ |
| npu_ioda_sel(p, NPU_IODA_TBL_TVT, window_id, false); |
| out_be64(p->at_regs + NPU_IODA_DATA0, data64); |
| p->tve_cache[window_id] = data64; |
| |
| return OPAL_SUCCESS; |
| } |
| |
| static int64_t npu_set_pe(struct phb *phb, |
| uint64_t pe_number, |
| uint64_t bdfn, |
| uint8_t bcompare, |
| uint8_t dcompare, |
| uint8_t fcompare, |
| uint8_t action) |
| { |
| struct npu *p = phb_to_npu(phb); |
| struct npu_dev *dev; |
| uint32_t link_idx; |
| uint64_t *data64; |
| |
| /* Sanity check */ |
| if (action != OPAL_MAP_PE && |
| action != OPAL_UNMAP_PE) |
| return OPAL_PARAMETER; |
| if (pe_number >= NPU_NUM_OF_PES) |
| return OPAL_PARAMETER; |
| |
| /* All emulated PCI devices hooked to root bus, whose |
| * bus number is zero. |
| */ |
| dev = bdfn_to_npu_dev(p, bdfn); |
| if (PCI_BUS_NUM(bdfn) || !dev) |
| return OPAL_PARAMETER; |
| |
| link_idx = dev->index; |
| dev->pe_number = pe_number; |
| |
| /* Separate links will be mapped to different PEs */ |
| if (bcompare != OpalPciBusAll || |
| dcompare != OPAL_COMPARE_RID_DEVICE_NUMBER || |
| fcompare != OPAL_COMPARE_RID_FUNCTION_NUMBER) |
| return OPAL_UNSUPPORTED; |
| |
| /* Map the link to the corresponding PE */ |
| data64 = &p->pce_cache[link_idx]; |
| if (action == OPAL_MAP_PE) |
| *data64 = SETFIELD(NPU_IODA_PCT_PE, *data64, |
| pe_number); |
| else |
| *data64 = SETFIELD(NPU_IODA_PCT_PE, *data64, |
| NPU_NUM_OF_PES); |
| |
| *data64 |= NPU_IODA_PCT_LINK_ENABLED; |
| |
| npu_ioda_sel(p, NPU_IODA_TBL_PCT, link_idx, false); |
| out_be64(p->at_regs + NPU_IODA_DATA0, *data64); |
| |
| return OPAL_SUCCESS; |
| } |
| |
| static int64_t npu_get_link_state(struct pci_slot *slot __unused, uint8_t *val) |
| { |
| /* As we're emulating all PCI stuff, the link bandwidth |
| * isn't big deal anyway. |
| */ |
| *val = OPAL_SHPC_LINK_UP_x1; |
| return OPAL_SUCCESS; |
| } |
| |
| static int64_t npu_get_power_state(struct pci_slot *slot __unused, uint8_t *val) |
| { |
| *val = PCI_SLOT_POWER_ON; |
| return OPAL_SUCCESS; |
| } |
| |
| static int64_t npu_hreset(struct pci_slot *slot __unused) |
| { |
| prlog(PR_DEBUG, "NPU: driver should call reset procedure here\n"); |
| |
| return OPAL_SUCCESS; |
| } |
| |
| static int64_t npu_freset(struct pci_slot *slot __unused) |
| { |
| /* FIXME: PHB fundamental reset, which need to be |
| * figured out later. It's used by EEH recovery |
| * upon fenced AT. |
| */ |
| return OPAL_SUCCESS; |
| } |
| |
| static struct pci_slot *npu_slot_create(struct phb *phb) |
| { |
| struct pci_slot *slot; |
| |
| slot = pci_slot_alloc(phb, NULL); |
| if (!slot) |
| return slot; |
| |
| /* Elementary functions */ |
| slot->ops.get_presence_state = NULL; |
| slot->ops.get_link_state = npu_get_link_state; |
| slot->ops.get_power_state = npu_get_power_state; |
| slot->ops.get_attention_state = NULL; |
| slot->ops.get_latch_state = NULL; |
| slot->ops.set_power_state = NULL; |
| slot->ops.set_attention_state = NULL; |
| |
| slot->ops.prepare_link_change = NULL; |
| slot->ops.poll_link = NULL; |
| slot->ops.hreset = npu_hreset; |
| slot->ops.freset = npu_freset; |
| slot->ops.creset = NULL; |
| |
| return slot; |
| } |
| |
| static int64_t npu_freeze_status(struct phb *phb, |
| uint64_t pe_number __unused, |
| uint8_t *freeze_state, |
| uint16_t *pci_error_type __unused, |
| uint16_t *severity __unused) |
| { |
| /* FIXME: When it's called by skiboot PCI config accessor, |
| * the PE number is fixed to 0, which is incorrect. We need |
| * introduce another PHB callback to translate it. For now, |
| * it keeps the skiboot PCI enumeration going. |
| */ |
| struct npu *p = phb_to_npu(phb); |
| if (p->fenced) |
| *freeze_state = OPAL_EEH_STOPPED_MMIO_DMA_FREEZE; |
| else |
| *freeze_state = OPAL_EEH_STOPPED_NOT_FROZEN; |
| return OPAL_SUCCESS; |
| } |
| |
| static int64_t npu_eeh_next_error(struct phb *phb, |
| uint64_t *first_frozen_pe, |
| uint16_t *pci_error_type, |
| uint16_t *severity) |
| { |
| struct npu *p = phb_to_npu(phb); |
| int i; |
| uint64_t result = 0; |
| *first_frozen_pe = -1; |
| *pci_error_type = OPAL_EEH_NO_ERROR; |
| *severity = OPAL_EEH_SEV_NO_ERROR; |
| |
| if (p->fenced) { |
| *pci_error_type = OPAL_EEH_PHB_ERROR; |
| *severity = OPAL_EEH_SEV_PHB_FENCED; |
| return OPAL_SUCCESS; |
| } |
| |
| npu_ioda_sel(p, NPU_IODA_TBL_PESTB, 0, true); |
| for (i = 0; i < NPU_NUM_OF_PES; i++) { |
| result = in_be64(p->at_regs + NPU_IODA_DATA0); |
| if (result > 0) { |
| *first_frozen_pe = i; |
| *pci_error_type = OPAL_EEH_PE_ERROR; |
| *severity = OPAL_EEH_SEV_PE_ER; |
| break; |
| } |
| } |
| |
| return OPAL_SUCCESS; |
| } |
| |
| /* For use in error injection and handling. */ |
| void npu_set_fence_state(struct npu *p, bool fence) { |
| p->fenced = fence; |
| |
| if (fence) |
| prlog(PR_ERR, "NPU: Chip %x is fenced, reboot required.\n", |
| p->chip_id); |
| else |
| prlog(PR_WARNING, "NPU: un-fencing is dangerous and should \ |
| only be used for development purposes."); |
| } |
| |
| /* Sets the NPU to trigger an error when a DMA occurs */ |
| static int64_t npu_err_inject(struct phb *phb, uint64_t pe_number, |
| uint32_t type, uint32_t func __unused, |
| uint64_t addr __unused, uint64_t mask __unused) |
| { |
| struct npu *p = phb_to_npu(phb); |
| struct npu_dev *dev = NULL; |
| int i; |
| |
| if (pe_number >= NPU_NUM_OF_PES) { |
| prlog(PR_ERR, "NPU: error injection failed, bad PE given\n"); |
| return OPAL_PARAMETER; |
| } |
| |
| for (i = 0; i < p->total_devices; i++) { |
| if (p->devices[i].pe_number == pe_number) { |
| dev = &p->devices[i]; |
| break; |
| } |
| } |
| |
| if (!dev) { |
| prlog(PR_ERR, "NPU: couldn't find device with PE%llx\n", pe_number); |
| return OPAL_PARAMETER; |
| } |
| |
| /* TODO: extend this to conform to OPAL injection standards */ |
| if (type > 1) { |
| prlog(PR_ERR, "NPU: invalid error injection type\n"); |
| return OPAL_PARAMETER; |
| } else if (type == 1) { |
| /* Emulate fence mode. */ |
| npu_set_fence_state(p, true); |
| } else { |
| /* Cause a freeze with an invalid MMIO read. If the BAR is not |
| * enabled, this will checkstop the machine. |
| */ |
| npu_dev_bar_update(p->chip_id, &dev->bar, true); |
| in_be64((void *)dev->bar.base); |
| } |
| |
| return OPAL_SUCCESS; |
| } |
| |
| static const struct phb_ops npu_ops = { |
| .cfg_read8 = npu_cfg_read8, |
| .cfg_read16 = npu_cfg_read16, |
| .cfg_read32 = npu_cfg_read32, |
| .cfg_write8 = npu_cfg_write8, |
| .cfg_write16 = npu_cfg_write16, |
| .cfg_write32 = npu_cfg_write32, |
| .get_reserved_pe_number = NULL, |
| .device_init = NULL, |
| .phb_final_fixup = npu_phb_final_fixup, |
| .ioda_reset = npu_ioda_reset, |
| .papr_errinjct_reset = NULL, |
| .pci_reinit = NULL, |
| .set_phb_mem_window = NULL, |
| .phb_mmio_enable = NULL, |
| .map_pe_mmio_window = NULL, |
| .map_pe_dma_window = npu_map_pe_dma_window, |
| .map_pe_dma_window_real = npu_map_pe_dma_window_real, |
| .pci_msi_eoi = NULL, |
| .set_xive_pe = NULL, |
| .get_msi_32 = NULL, |
| .get_msi_64 = NULL, |
| .set_pe = npu_set_pe, |
| .set_peltv = NULL, |
| .eeh_freeze_status = npu_freeze_status, |
| .eeh_freeze_clear = NULL, |
| .eeh_freeze_set = NULL, |
| .next_error = npu_eeh_next_error, |
| .err_inject = npu_err_inject, |
| .get_diag_data2 = NULL, |
| .set_capi_mode = NULL, |
| .set_capp_recovery = NULL, |
| }; |
| |
| static void assign_mmio_bars(uint32_t gcid, uint32_t xscom, |
| struct dt_node *npu_dn, uint64_t mm_win[2], |
| uint64_t at_bar[2]) |
| { |
| uint64_t mem_start, mem_end; |
| struct npu_dev_bar bar; |
| struct dt_node *link; |
| |
| /* Configure BAR selection. |
| * |
| * Currently, each PHY contains 2 links and each link has 2 |
| * BARs. The first BAR is assigned to the DLTL region which is |
| * what the kernel uses. The second BAR is either assigned to |
| * either the PL or AT region or unassigned. The PL0/PL1/AT |
| * MMIO regions are not exposed to the kernel so we assigned |
| * them at the start of the available memory area followed by |
| * the DLTL regions. So we end up with the following memory |
| * map (assuming we're given a memory region starting at |
| * 0x3fff000000000): |
| * |
| * Link#0-BAR#0: NTL/NDL BAR (128KB) - 0x3fff000420000 |
| * Link#0-BAR#1: PL0 BAR ( 2MB) - 0x3fff000000000 |
| * Link#1-BAR#0: NTL/NDL BAR (128KB) - 0x3fff000440000 |
| * Link#1-BAR#1: AT BAR ( 64KB) - 0x3fff000400000 |
| * Link#2-BAR#0: NTL/NDL BAR (128KB) - 0x3fff000460000 |
| * Link#2-BAR#1: PL1 BAR ( 2MB) - 0x3fff000200000 |
| * Link#3-BAR#0: NTL/NDL BAR (128KB) - 0x3fff000480000 |
| * Link#3-BAR#1: UNASSIGNED |
| */ |
| xscom_write(gcid, xscom + NPU_AT_SCOM_OFFSET + NX_BAR, |
| 0x0211000043500000UL); |
| |
| xscom_read(gcid, npu_link_scom_base(npu_dn, xscom, 0) + NX_MMIO_BAR_0, |
| &mem_start); |
| mem_start = GETFIELD(NX_MMIO_BAR_BASE, mem_start) << 12; |
| |
| xscom_read(gcid, npu_link_scom_base(npu_dn, xscom, 5) + NX_MMIO_BAR_0, |
| &mem_end); |
| mem_end = (GETFIELD(NX_MMIO_BAR_BASE, mem_end) << 12) + |
| get_bar_size(mem_end); |
| |
| /* PL0 BAR comes first at 0x3fff000000000 */ |
| bar.xscom = npu_link_scom_base(npu_dn, xscom, 0) + NX_MMIO_BAR_1; |
| bar.base = mem_start; |
| bar.size = NX_MMIO_PL_SIZE; |
| npu_dev_bar_update(gcid, &bar, true); |
| |
| /* PL1 BAR */ |
| bar.xscom = npu_link_scom_base(npu_dn, xscom, 4) + NX_MMIO_BAR_1; |
| bar.base += bar.size; |
| bar.size = NX_MMIO_PL_SIZE; |
| npu_dev_bar_update(gcid, &bar, true); |
| |
| /* Then the AT BAR */ |
| bar.xscom = npu_link_scom_base(npu_dn, xscom, 1) + NX_MMIO_BAR_1; |
| bar.base += bar.size; |
| bar.size = NX_MMIO_AT_SIZE; |
| at_bar[0] = bar.base; |
| at_bar[1] = NX_MMIO_AT_SIZE; |
| npu_dev_bar_update(gcid, &bar, true); |
| |
| /* Now we configure all the DLTL BARs. These are the ones |
| * actually exposed to the kernel. */ |
| mm_win[0] = bar.base + bar.size; |
| dt_for_each_node(npu_dn, link) { |
| uint32_t index; |
| |
| index = dt_prop_get_u32(link, "ibm,npu-link-index"); |
| bar.xscom = npu_link_scom_base(npu_dn, xscom, index) + |
| NX_MMIO_BAR_0; |
| bar.base += bar.size; |
| bar.size = NX_MMIO_DL_SIZE; |
| bar.base = ALIGN_UP(bar.base, bar.size); |
| npu_dev_bar_update(gcid, &bar, false); |
| } |
| mm_win[1] = (bar.base + bar.size) - mm_win[0]; |
| |
| /* If we weren't given enough room to setup all the BARs we |
| * require it's better to crash here than risk creating |
| * overlapping BARs which will xstop the machine randomly in |
| * the future.*/ |
| assert(bar.base + bar.size <= mem_end); |
| } |
| |
| /* Probe NPU device node and create PCI root device node |
| * accordingly. The NPU deivce node should specify number |
| * of links and xscom base address to access links. |
| */ |
| static void npu_probe_phb(struct dt_node *dn) |
| { |
| struct dt_node *np; |
| uint32_t gcid, index, phb_index, xscom; |
| uint64_t at_bar[2], mm_win[2]; |
| uint32_t links; |
| char *path; |
| |
| /* Retrieve chip id */ |
| path = dt_get_path(dn); |
| gcid = dt_get_chip_id(dn); |
| index = dt_prop_get_u32(dn, "ibm,npu-index"); |
| phb_index = dt_prop_get_u32(dn, "ibm,phb-index"); |
| links = dt_prop_get_u32(dn, "ibm,npu-links"); |
| prlog(PR_INFO, "Chip %d Found NPU%d (%d links) at %s\n", |
| gcid, index, links, path); |
| free(path); |
| |
| /* Retrieve xscom base addr */ |
| xscom = dt_get_address(dn, 0, NULL); |
| prlog(PR_INFO, " XSCOM Base: %08x\n", xscom); |
| |
| assign_mmio_bars(gcid, xscom, dn, mm_win, at_bar); |
| prlog(PR_INFO, " AT BAR: %016llx (%lldKB)\n", |
| at_bar[0], at_bar[1] / 0x400); |
| |
| /* Create PCI root device node */ |
| np = dt_new_addr(dt_root, "pciex", at_bar[0]); |
| assert(np); |
| |
| dt_add_property_strings(np, "compatible", |
| "ibm,power8-npu-pciex", "ibm,ioda2-npu-phb"); |
| dt_add_property_strings(np, "device_type", "pciex"); |
| dt_add_property(np, "reg", at_bar, sizeof(at_bar)); |
| |
| dt_add_property_cells(np, "ibm,phb-index", phb_index); |
| dt_add_property_cells(np, "ibm,npu-index", index); |
| dt_add_property_cells(np, "ibm,chip-id", gcid); |
| dt_add_property_cells(np, "ibm,xscom-base", xscom); |
| dt_add_property_cells(np, "ibm,npcq", dn->phandle); |
| dt_add_property_cells(np, "ibm,links", links); |
| dt_add_property(np, "ibm,mmio-window", mm_win, sizeof(mm_win)); |
| dt_add_property_cells(np, "ibm,phb-diag-data-size", 0); |
| |
| /* Disable fast reboot - not currently supported */ |
| disable_fast_reboot("NVLink device enabled"); |
| } |
| |
| static void npu_dev_populate_vendor_cap(struct npu_dev_cap *cap) |
| { |
| struct npu_dev *dev = cap->dev; |
| struct pci_virt_device *pvd = dev->pvd; |
| uint32_t offset = cap->start; |
| uint8_t val; |
| |
| /* Add length and version information */ |
| val = cap->end - cap->start; |
| PCI_VIRT_CFG_INIT_RO(pvd, offset + 2, 1, val); |
| PCI_VIRT_CFG_INIT_RO(pvd, offset + 3, 1, OPAL_NPU_VERSION); |
| offset += 4; |
| |
| /* Defaults when the trap can't handle the read/write (eg. due |
| * to reading/writing less than 4 bytes). */ |
| val = 0x0; |
| PCI_VIRT_CFG_INIT_RO(pvd, offset, 4, val); |
| PCI_VIRT_CFG_INIT_RO(pvd, offset + 4, 4, val); |
| |
| /* Create a trap for AT/PL procedures */ |
| pci_virt_add_filter(pvd, offset, 8, |
| PCI_REG_FLAG_READ | PCI_REG_FLAG_WRITE, |
| npu_dev_procedure, NULL); |
| offset += 8; |
| |
| PCI_VIRT_CFG_INIT_RO(pvd, offset, 1, dev->index); |
| } |
| |
| static void npu_dev_populate_pcie_cap(struct npu_dev_cap *cap) |
| { |
| struct npu_dev *dev = cap->dev; |
| struct pci_virt_device *pvd = dev->pvd; |
| uint32_t base = cap->start; |
| uint32_t val; |
| |
| /* Sanity check on capability ID */ |
| if (cap->id != PCI_CFG_CAP_ID_EXP) { |
| prlog(PR_NOTICE, "%s: Invalid capability ID %d (%d)\n", |
| __func__, cap->id, PCI_CFG_CAP_ID_EXP); |
| return; |
| } |
| |
| /* Sanity check on spanned registers */ |
| if ((cap->end - cap->start) < PCIE_CAP_START) { |
| prlog(PR_NOTICE, "%s: Invalid reg region [%x, %x] for cap %d\n", |
| __func__, cap->start, cap->end, cap->id); |
| return; |
| } |
| |
| /* 0x00 - ID/PCIE capability */ |
| val = cap->id; |
| val |= ((0x2 << 16) | (PCIE_TYPE_ENDPOINT << 20)); |
| PCI_VIRT_CFG_INIT_RO(pvd, base, 4, val); |
| |
| /* 0x04 - Device capability |
| * |
| * We should support FLR. Otherwise, it might have |
| * problem passing it through to userland via Linux |
| * VFIO infrastructure |
| */ |
| val = ((PCIE_MPSS_128) | |
| (PCIE_PHANTOM_NONE << 3) | |
| (PCIE_L0SL_MAX_NO_LIMIT << 6) | |
| (PCIE_L1L_MAX_NO_LIMIT << 9) | |
| (PCICAP_EXP_DEVCAP_FUNC_RESET)); |
| PCI_VIRT_CFG_INIT_RO(pvd, base + PCICAP_EXP_DEVCAP, 4, val); |
| |
| pci_virt_add_filter(pvd, base + PCICAP_EXP_DEVCTL, 2, |
| PCI_REG_FLAG_WRITE, |
| npu_dev_cfg_exp_devcap, NULL); |
| |
| /* 0x08 - Device control and status */ |
| PCI_VIRT_CFG_INIT(pvd, base + PCICAP_EXP_DEVCTL, 4, 0x00002810, |
| 0xffff0000, 0x000f0000); |
| |
| /* 0x0c - Link capability */ |
| val = (PCIE_LSPEED_VECBIT_2 | (PCIE_LWIDTH_1X << 4)); |
| PCI_VIRT_CFG_INIT_RO(pvd, base + PCICAP_EXP_LCAP, 4, val); |
| |
| /* 0x10 - Link control and status */ |
| PCI_VIRT_CFG_INIT(pvd, base + PCICAP_EXP_LCTL, 4, 0x00130000, |
| 0xfffff000, 0xc0000000); |
| |
| /* 0x14 - Slot capability */ |
| PCI_VIRT_CFG_INIT_RO(pvd, base + PCICAP_EXP_SLOTCAP, 4, 0x00000000); |
| |
| /* 0x18 - Slot control and status */ |
| PCI_VIRT_CFG_INIT_RO(pvd, base + PCICAP_EXP_SLOTCTL, 4, 0x00000000); |
| |
| /* 0x1c - Root control and capability */ |
| PCI_VIRT_CFG_INIT(pvd, base + PCICAP_EXP_RC, 4, 0x00000000, |
| 0xffffffe0, 0x00000000); |
| |
| /* 0x20 - Root status */ |
| PCI_VIRT_CFG_INIT(pvd, base + PCICAP_EXP_RSTAT, 4, 0x00000000, |
| 0xffffffff, 0x00010000); |
| |
| /* 0x24 - Device capability 2 */ |
| PCI_VIRT_CFG_INIT_RO(pvd, base + PCIECAP_EXP_DCAP2, 4, 0x00000000); |
| |
| /* 0x28 - Device Control and status 2 */ |
| PCI_VIRT_CFG_INIT(pvd, base + PCICAP_EXP_DCTL2, 4, 0x00070000, |
| 0xffff0000, 0x00000000); |
| |
| /* 0x2c - Link capability 2 */ |
| PCI_VIRT_CFG_INIT_RO(pvd, base + PCICAP_EXP_LCAP2, 4, 0x00000007); |
| |
| /* 0x30 - Link control and status 2 */ |
| PCI_VIRT_CFG_INIT(pvd, base + PCICAP_EXP_LCTL2, 4, 0x00000003, |
| 0xffff0000, 0x00200000); |
| |
| /* 0x34 - Slot capability 2 */ |
| PCI_VIRT_CFG_INIT_RO(pvd, base + PCICAP_EXP_SCAP2, 4, 0x00000000); |
| |
| /* 0x38 - Slot control and status 2 */ |
| PCI_VIRT_CFG_INIT_RO(pvd, base + PCICAP_EXP_SCTL2, 4, 0x00000000); |
| } |
| |
| static struct npu_dev_cap *npu_dev_create_capability(struct npu_dev *dev, |
| void (*populate)(struct npu_dev_cap *), |
| uint16_t id, |
| uint16_t start, |
| uint16_t end) |
| { |
| struct npu_dev_cap *cap; |
| |
| /* Check if the capability is existing */ |
| cap = npu_dev_find_capability(dev, id); |
| if (cap) |
| return cap; |
| |
| /* Allocate new one */ |
| cap = zalloc(sizeof(struct npu_dev_cap)); |
| assert(cap); |
| |
| /* Put it into the pool */ |
| cap->id = id; |
| cap->start = start; |
| cap->end = end; |
| cap->dev = dev; |
| cap->populate = populate; |
| list_add_tail(&dev->capabilities, &cap->link); |
| |
| return cap; |
| } |
| |
| static struct npu_dev_cap *npu_dev_find_capability(struct npu_dev *dev, |
| uint16_t id) |
| { |
| struct npu_dev_cap *cap; |
| |
| list_for_each(&dev->capabilities, cap, link) { |
| if (cap->id == id) |
| return cap; |
| } |
| |
| return NULL; |
| } |
| |
| /* |
| * All capabilities should be put into the device capability |
| * list according to register offset in ascending order for |
| * easy access at later point. |
| */ |
| static void npu_dev_create_capabilities(struct npu_dev *dev) |
| { |
| list_head_init(&dev->capabilities); |
| |
| /* PCI express capability */ |
| npu_dev_create_capability(dev, npu_dev_populate_pcie_cap, |
| PCI_CFG_CAP_ID_EXP, PCIE_CAP_START, |
| PCIE_CAP_END); |
| |
| /* Vendor specific capability */ |
| npu_dev_create_capability(dev, npu_dev_populate_vendor_cap, |
| PCI_CFG_CAP_ID_VENDOR, VENDOR_CAP_START, |
| VENDOR_CAP_END); |
| } |
| |
| static void npu_dev_create_cfg(struct npu_dev *dev) |
| { |
| struct pci_virt_device *pvd = dev->pvd; |
| struct npu_dev_cap *cap; |
| uint32_t offset; |
| uint32_t last_cap_offset; |
| |
| /* 0x00 - Vendor/Device ID */ |
| PCI_VIRT_CFG_INIT_RO(pvd, PCI_CFG_VENDOR_ID, 4, 0x04ea1014); |
| |
| /* 0x04 - Command/Status |
| * |
| * Create one trap to trace toggling memory BAR enable bit |
| */ |
| PCI_VIRT_CFG_INIT(pvd, PCI_CFG_CMD, 4, 0x00100000, 0xffb802b8, |
| 0xf9000000); |
| |
| pci_virt_add_filter(pvd, PCI_CFG_CMD, 1, PCI_REG_FLAG_WRITE, |
| npu_dev_cfg_write_cmd, NULL); |
| |
| /* 0x08 - Rev/Class/Cache */ |
| PCI_VIRT_CFG_INIT_RO(pvd, PCI_CFG_REV_ID, 4, 0x06800100); |
| |
| /* 0x0c - CLS/Latency Timer/Header/BIST */ |
| PCI_VIRT_CFG_INIT_RO(pvd, PCI_CFG_CACHE_LINE_SIZE, 4, 0x00800000); |
| |
| /* 0x10 - BARs, always 64-bits non-prefetchable |
| * |
| * Each emulated device represents one link and therefore |
| * there is one BAR for the associated DLTL region. |
| */ |
| |
| /* Low 32-bits */ |
| PCI_VIRT_CFG_INIT(pvd, PCI_CFG_BAR0, 4, |
| (dev->bar.base & 0xfffffff0) | dev->bar.flags, |
| 0x0000000f, 0x00000000); |
| |
| /* High 32-bits */ |
| PCI_VIRT_CFG_INIT(pvd, PCI_CFG_BAR1, 4, (dev->bar.base >> 32), |
| 0x00000000, 0x00000000); |
| |
| /* |
| * Create trap. Writting 0xFF's to BAR registers should be |
| * trapped and return size on next read |
| */ |
| pci_virt_add_filter(pvd, PCI_CFG_BAR0, 8, |
| PCI_REG_FLAG_READ | PCI_REG_FLAG_WRITE, |
| npu_dev_cfg_bar, &dev->bar); |
| |
| /* 0x18/1c/20/24 - Disabled BAR#2/3/4/5 |
| * |
| * Mark those BARs readonly so that 0x0 will be returned when |
| * probing the length and the BARs will be skipped. |
| */ |
| PCI_VIRT_CFG_INIT_RO(pvd, PCI_CFG_BAR2, 4, 0x00000000); |
| PCI_VIRT_CFG_INIT_RO(pvd, PCI_CFG_BAR3, 4, 0x00000000); |
| PCI_VIRT_CFG_INIT_RO(pvd, PCI_CFG_BAR4, 4, 0x00000000); |
| PCI_VIRT_CFG_INIT_RO(pvd, PCI_CFG_BAR5, 4, 0x00000000); |
| |
| /* 0x28 - Cardbus CIS pointer */ |
| PCI_VIRT_CFG_INIT_RO(pvd, PCI_CFG_CARDBUS_CIS, 4, 0x00000000); |
| |
| /* 0x2c - Subsystem ID */ |
| PCI_VIRT_CFG_INIT_RO(pvd, PCI_CFG_SUBSYS_VENDOR_ID, 4, 0x00000000); |
| |
| /* 0x30 - ROM BAR |
| * |
| * Force its size to be zero so that the kernel will skip |
| * probing the ROM BAR. We needn't emulate ROM BAR. |
| */ |
| PCI_VIRT_CFG_INIT_RO(pvd, PCI_CFG_ROMBAR, 4, 0xffffffff); |
| |
| /* 0x34 - PCI Capability |
| * |
| * By default, we don't have any capabilities |
| */ |
| PCI_VIRT_CFG_INIT_RO(pvd, PCI_CFG_CAP, 4, 0x00000000); |
| |
| last_cap_offset = PCI_CFG_CAP - 1; |
| list_for_each(&dev->capabilities, cap, link) { |
| offset = cap->start; |
| |
| /* Initialize config space for the capability */ |
| if (cap->populate) |
| cap->populate(cap); |
| |
| /* Add capability header */ |
| PCI_VIRT_CFG_INIT_RO(pvd, offset, 2, cap->id); |
| |
| /* Update the next capability pointer */ |
| PCI_VIRT_CFG_NORMAL_WR(pvd, last_cap_offset + 1, 1, offset); |
| |
| last_cap_offset = offset; |
| } |
| |
| /* 0x38 - Reserved */ |
| PCI_VIRT_CFG_INIT_RO(pvd, 0x38, 4, 0x00000000); |
| |
| /* 0x3c - INT line/pin/Minimal grant/Maximal latency */ |
| if (!(dev->index % 2)) |
| PCI_VIRT_CFG_INIT_RO(pvd, PCI_CFG_INT_LINE, 4, 0x00000100); |
| else |
| PCI_VIRT_CFG_INIT_RO(pvd, PCI_CFG_INT_LINE, 4, 0x00000200); |
| } |
| |
| static uint32_t npu_allocate_bdfn(struct npu *p, uint32_t group) |
| { |
| int i; |
| int bdfn = (group << 3); |
| |
| for (i = 0; i < p->total_devices; i++) { |
| if ((p->devices[i].pvd->bdfn & 0xf8) == (bdfn & 0xf8)) |
| bdfn++; |
| } |
| |
| return bdfn; |
| } |
| |
| static void npu_create_devices(struct dt_node *dn, struct npu *p) |
| { |
| struct npu_dev *dev; |
| struct dt_node *npu_dn, *link; |
| uint32_t bdfn, npu_phandle, index = 0; |
| uint64_t buid_reg; |
| uint64_t lsisrcid; |
| uint64_t buid; |
| |
| |
| /* The bits in the LSI ID Base register are always compared and |
| * can be set to 0 in the buid base and mask fields. The |
| * buid (bus unit id) is the full irq minus the last 4 bits. */ |
| lsisrcid = GETFIELD(NPU_LSI_SRC_ID_BASE, NPU_LSI_SRC_ID_BASE); |
| buid = p8_chip_irq_block_base(p->chip_id, P8_IRQ_BLOCK_MISC) >> 4; |
| |
| buid_reg = SETFIELD(NP_IRQ_LEVELS, NP_BUID_ENABLE, ~0); |
| buid_reg = SETFIELD(NP_BUID_MASK, buid_reg, ~lsisrcid); |
| buid_reg = SETFIELD(NP_BUID_BASE, buid_reg, (buid & ~lsisrcid)); |
| |
| /* Get the npu node which has the links which we expand here |
| * into pci like devices attached to our emulated phb. */ |
| npu_phandle = dt_prop_get_u32(dn, "ibm,npcq"); |
| npu_dn = dt_find_by_phandle(dt_root, npu_phandle); |
| assert(npu_dn); |
| |
| /* Walk the link@x nodes to initialize devices */ |
| p->total_devices = 0; |
| p->phb.scan_map = 0; |
| list_head_init(&p->phb.virt_devices); |
| dt_for_each_compatible(npu_dn, link, "ibm,npu-link") { |
| struct npu_dev_bar *bar; |
| uint32_t group_id; |
| uint64_t val; |
| |
| dev = &p->devices[index]; |
| dev->index = dt_prop_get_u32(link, "ibm,npu-link-index"); |
| dev->xscom = npu_link_scom_base(npu_dn, p->xscom_base, |
| dev->index); |
| |
| dev->npu = p; |
| dev->dt_node = link; |
| |
| /* We don't support MMIO PHY access yet */ |
| dev->pl_base = NULL; |
| |
| group_id = dt_prop_get_u32(link, "ibm,npu-group-id"); |
| bdfn = npu_allocate_bdfn(p, group_id); |
| |
| /* This must be done after calling |
| * npu_allocate_bdfn() */ |
| p->total_devices++; |
| p->phb.scan_map |= 0x1 << ((bdfn & 0xf8) >> 3); |
| |
| dev->pl_xscom_base = dt_prop_get_u64(link, "ibm,npu-phy"); |
| dev->lane_mask = dt_prop_get_u32(link, "ibm,npu-lane-mask"); |
| |
| /* Setup BUID/ISRN */ |
| xscom_write(p->chip_id, dev->xscom + NX_NP_BUID, buid_reg); |
| |
| /* Create PCI virtual device */ |
| dev->pvd = pci_virt_add_device(&p->phb, bdfn, NPU_DEV_CFG_SIZE, dev); |
| assert(dev->pvd); |
| bar = &dev->bar; |
| bar->flags = (PCI_CFG_BAR_TYPE_MEM | |
| PCI_CFG_BAR_MEM64); |
| |
| /* Update BAR info */ |
| bar->xscom = dev->xscom + NX_MMIO_BAR_0; |
| xscom_read(p->chip_id, bar->xscom, &val); |
| bar->base = GETFIELD(NX_MMIO_BAR_BASE, val) << 12; |
| bar->size = get_bar_size(val); |
| |
| /* |
| * The config space is initialised with the BARs |
| * disabled, so make sure it is actually disabled in |
| * hardware. |
| */ |
| npu_dev_bar_update(p->chip_id, bar, false); |
| |
| /* Initialize capabilities */ |
| npu_dev_create_capabilities(dev); |
| |
| /* Initialize config space */ |
| npu_dev_create_cfg(dev); |
| |
| index++; |
| } |
| } |
| |
| static void npu_add_phb_properties(struct npu *p) |
| { |
| struct dt_node *np = p->phb.dt_node; |
| uint32_t icsp = get_ics_phandle(); |
| uint64_t tkill, mm_base, mm_size; |
| uint32_t base_lsi = p->base_lsi; |
| uint32_t map[] = { |
| /* Dev 0 INT#A (used by fn0) */ |
| 0x0000, 0x0, 0x0, 0x1, icsp, base_lsi + NPU_LSI_INT_DL0, 1, |
| /* Dev 0 INT#B (used by fn1) */ |
| 0x0000, 0x0, 0x0, 0x2, icsp, base_lsi + NPU_LSI_INT_DL1, 1, |
| /* Dev 1 INT#A (used by fn0) */ |
| 0x0800, 0x0, 0x0, 0x1, icsp, base_lsi + NPU_LSI_INT_DL2, 1, |
| /* Dev 1 INT#B (used by fn1) */ |
| 0x0800, 0x0, 0x0, 0x2, icsp, base_lsi + NPU_LSI_INT_DL3, 1, |
| }; |
| /* Mask is bus, device and INT# */ |
| uint32_t mask[] = {0xf800, 0x0, 0x0, 0x7}; |
| char slotbuf[32]; |
| |
| /* Add various properties that HB doesn't have to |
| * add, some of them simply because they result from |
| * policy decisions made in skiboot rather than in HB |
| * such as the MMIO windows going to PCI, interrupts, |
| * etc. |
| */ |
| dt_add_property_cells(np, "#address-cells", 3); |
| dt_add_property_cells(np, "#size-cells", 2); |
| dt_add_property_cells(np, "#interrupt-cells", 1); |
| dt_add_property_cells(np, "bus-range", 0, 0xff); |
| dt_add_property_cells(np, "clock-frequency", 0x200, 0); |
| dt_add_property_cells(np, "interrupt-parent", icsp); |
| |
| /* DLPL Interrupts, we don't use the standard swizzle */ |
| p->phb.lstate.int_size = 0; |
| dt_add_property(np, "interrupt-map", map, sizeof(map)); |
| dt_add_property(np, "interrupt-map-mask", mask, sizeof(mask)); |
| |
| /* NPU PHB properties */ |
| /* TODO: Due to an errata TCE KILL only works when DMA traffic |
| * has been stopped. We need to implement the work around |
| * which is to do a TCE kill all instead. */ |
| tkill = cleanup_addr((uint64_t)p->at_regs) + NPU_TCE_KILL; |
| dt_add_property_cells(np, "ibm,opal-num-pes", |
| NPU_NUM_OF_PES); |
| dt_add_property_cells(np, "ibm,opal-reserved-pe", |
| 0); |
| dt_add_property_u64(np, "ibm,opal-tce-kill", tkill); |
| |
| /* Memory window is exposed as 32-bits non-prefetchable |
| * one because 64-bits prefetchable one is kind of special |
| * to kernel. |
| */ |
| mm_base = p->mm_base; |
| mm_size = p->mm_size; |
| dt_add_property_cells(np, "ranges", 0x02000000, |
| hi32(mm_base), lo32(mm_base), |
| hi32(mm_base), lo32(mm_base), |
| hi32(mm_size), lo32(mm_size)); |
| |
| /* Set the slot location on the NPU PHB. This PHB can contain |
| * devices that correlate with multiple physical slots, so |
| * present the chip ID instead. |
| */ |
| snprintf(slotbuf, sizeof(slotbuf), "NPU Chip %d", p->chip_id); |
| dt_add_property_string(np, "ibm,io-base-loc-code", slotbuf); |
| } |
| |
| static void npu_create_phb(struct dt_node *dn) |
| { |
| const struct dt_property *prop; |
| struct npu *p; |
| struct pci_slot *slot; |
| uint32_t links; |
| void *pmem; |
| |
| /* Retrieve number of devices */ |
| links = dt_prop_get_u32(dn, "ibm,links"); |
| pmem = zalloc(sizeof(struct npu) + links * sizeof(struct npu_dev)); |
| assert(pmem); |
| |
| /* Populate PHB */ |
| p = pmem; |
| p->index = dt_prop_get_u32(dn, "ibm,npu-index"); |
| p->chip_id = dt_prop_get_u32(dn, "ibm,chip-id"); |
| p->xscom_base = dt_prop_get_u32(dn, "ibm,xscom-base"); |
| p->total_devices = links; |
| |
| /* TODO: When hardware fences are implemented, detect them here */ |
| p->fenced = false; |
| |
| /* This is the AT base */ |
| p->at_xscom = p->xscom_base + NPU_AT_SCOM_OFFSET; |
| p->at_regs = (void *)dt_get_address(dn, 0, NULL); |
| |
| prop = dt_require_property(dn, "ibm,mmio-window", -1); |
| assert(prop->len >= (2 * sizeof(uint64_t))); |
| p->mm_base = ((const uint64_t *)prop->prop)[0]; |
| p->mm_size = ((const uint64_t *)prop->prop)[1]; |
| |
| p->devices = pmem + sizeof(struct npu); |
| |
| /* Interrupt */ |
| p->base_lsi = p8_chip_irq_block_base(p->chip_id, P8_IRQ_BLOCK_MISC) + |
| NPU_LSI_IRQ_MIN; |
| |
| /* Generic PHB */ |
| p->phb.dt_node = dn; |
| p->phb.ops = &npu_ops; |
| p->phb.phb_type = phb_type_pcie_v3; |
| |
| /* Populate devices */ |
| npu_create_devices(dn, p); |
| |
| /* Populate extra properties */ |
| npu_add_phb_properties(p); |
| |
| /* Create PHB slot */ |
| slot = npu_slot_create(&p->phb); |
| if (!slot) |
| { |
| /** |
| * @fwts-label NPUCannotCreatePHBSlot |
| * @fwts-advice Firmware probably ran out of memory creating |
| * NPU slot. NVLink functionality could be broken. |
| */ |
| prlog(PR_ERR, "NPU: Cannot create PHB slot\n"); |
| } |
| |
| /* Register PHB */ |
| pci_register_phb(&p->phb, OPAL_DYNAMIC_PHB_ID); |
| |
| /* Initialize IODA cache */ |
| npu_ioda_init(p); |
| |
| /* Register interrupt source */ |
| npu_register_irq(p); |
| |
| /* Initialize hardware */ |
| npu_hw_init(p); |
| } |
| |
| void probe_npu(void) |
| { |
| struct dt_node *np; |
| |
| /* Scan NPU XSCOM nodes */ |
| dt_for_each_compatible(dt_root, np, "ibm,power8-npu") |
| npu_probe_phb(np); |
| |
| /* Scan newly created PHB nodes */ |
| dt_for_each_compatible(dt_root, np, "ibm,power8-npu-pciex") |
| npu_create_phb(np); |
| } |