Google Git
Sign in
qemu / skiboot / 455b723d3dce90494355b6072bb144129e4cb361 / . / hw / npu2.c
blob: cf57eeb0c7e3765aab34ee6b88bb0611461b3135 [file] [log] [blame]
// SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later
/*
* NPU - NVlink and OpenCAPI
*
* Copyright 2013-2019 IBM Corp.
*/
#include <skiboot.h>
#include <io.h>
#include <timebase.h>
#include <pci-cfg.h>
#include <pci.h>
#include <pci-slot.h>
#include <pci-virt.h>
#include <opal.h>
#include <opal-api.h>
#include <cpu.h>
#include <device.h>
#include <ccan/str/str.h>
#include <ccan/array_size/array_size.h>
#include <affinity.h>
#include <npu2.h>
#include <lock.h>
#include <xscom.h>
#include <bitutils.h>
#include <chip.h>
#include <phys-map.h>
#include <nvram.h>
#include <xscom-p9-regs.h>
#include <phb4.h>
#include <cache-p9.h>
#define VENDOR_CAP_START 0x80
#define VENDOR_CAP_END 0x90
#define VENDOR_CAP_LEN 0x10
#define VENDOR_CAP_VERSION 0x01
#define VENDOR_CAP_PCI_DEV_OFFSET 0x0d
/*
* NPU2 BAR layout definition. We have 3 stacks and each of them
* contains 2 bricks. So every NPU2 has 6 bricks in total. There are 2
* PHY BARs and each of them is shared by 3 bricks. Every brick has
* one NTL BAR and two bricks share one GENID BAR. There is also a
* global MMIO BAR. We only expose DL and GENID BARs to the OS and all
* other BARs will be hidden in skiboot.
*
* Before the global MMIO BAR is configured, scom is the only way to
* access the BAR registers. At NPU2 PHB probing time, we rely on scom
* to assign all BARs until the global MMIO BAR is established.
*
* We need to access 4 SM registers in the same stack in order to
* configure one particular BAR.
*/
/* Set a specific flag in the vendor config space */
void npu2_set_link_flag(struct npu2_dev *ndev, uint8_t flag)
{
ndev->nvlink.link_flags |= flag;
PCI_VIRT_CFG_INIT_RO(ndev->nvlink.pvd, VENDOR_CAP_START +
VENDOR_CAP_PCI_DEV_OFFSET, 1, ndev->nvlink.link_flags);
}
void npu2_clear_link_flag(struct npu2_dev *ndev, uint8_t flag)
{
ndev->nvlink.link_flags &= ~flag;
PCI_VIRT_CFG_INIT_RO(ndev->nvlink.pvd, VENDOR_CAP_START +
VENDOR_CAP_PCI_DEV_OFFSET, 1, ndev->nvlink.link_flags);
}
static inline void npu2_ioda_sel(struct npu2 *p, uint32_t table,
uint32_t index, bool autoinc)
{
out_be64(p->regs + NPU2_ATS_IODA_TBL,
(autoinc ? NPU2_ATS_IODA_TBL_AUTOINC : 0ul) |
SETFIELD(NPU2_ATS_IODA_TBL_SELECT, 0ul, table) |
SETFIELD(NPU2_ATS_IODA_TBL_INDEX, 0ul, index));
}
static struct npu2_dev *npu2_bdf_to_dev(struct npu2 *p,
uint32_t bdfn)
{
struct pci_virt_device *pvd;
/* All emulated devices are attached to root bus */
if (bdfn & ~0xff)
return NULL;
pvd = pci_virt_find_device(&p->phb_nvlink, bdfn);
if (pvd)
return pvd->data;
return NULL;
}
static inline void npu2_get_bar(uint32_t gcid, struct npu2_bar *bar)
{
phys_map_get(gcid, bar->type, bar->index, &bar->base, &bar->size);
}
static void npu2_read_bar(struct npu2 *p, struct npu2_bar *bar)
{
uint64_t reg, val;
int enabled;
reg = NPU2_REG_OFFSET(0, NPU2_BLOCK_SM_0, bar->reg);
val = npu2_read(p, reg);
switch (NPU2_REG(bar->reg)) {
case NPU2_PHY_BAR:
bar->base = GETFIELD(NPU2_PHY_BAR_ADDR, val) << 21;
enabled = GETFIELD(NPU2_PHY_BAR_ENABLE, val);
if (NPU2_REG_STACK(reg) == NPU2_STACK_STCK_2)
/* This is the global MMIO BAR */
bar->size = 0x1000000;
else
bar->size = 0x200000;
break;
case NPU2_NTL0_BAR:
case NPU2_NTL1_BAR:
bar->base = GETFIELD(NPU2_NTL_BAR_ADDR, val) << 16;
enabled = GETFIELD(NPU2_NTL_BAR_ENABLE, val);
bar->size = 0x10000 << GETFIELD(NPU2_NTL_BAR_SIZE, val);
break;
case NPU2_GENID_BAR:
bar->base = GETFIELD(NPU2_GENID_BAR_ADDR, val) << 16;
enabled = GETFIELD(NPU2_GENID_BAR_ENABLE, val);
bar->size = 0x20000;
break;
default:
bar->base = 0ul;
enabled = 0;
bar->size = 0;
break;
}
bar->flags = SETFIELD(NPU2_BAR_FLAG_ENABLED, bar->flags, enabled);
}
static void npu2_write_bar(struct npu2 *p,
struct npu2_bar *bar,
uint32_t gcid,
uint32_t scom)
{
uint64_t reg, val, enable = !!(bar->flags & NPU2_BAR_FLAG_ENABLED);
int block;
switch (NPU2_REG(bar->reg)) {
case NPU2_PHY_BAR:
val = SETFIELD(NPU2_PHY_BAR_ADDR, 0ul, bar->base >> 21);
val = SETFIELD(NPU2_PHY_BAR_ENABLE, val, enable);
break;
case NPU2_NTL0_BAR:
case NPU2_NTL1_BAR:
val = SETFIELD(NPU2_NTL_BAR_ADDR, 0ul, bar->base >> 16);
val = SETFIELD(NPU2_NTL_BAR_ENABLE, val, enable);
val = SETFIELD(NPU2_NTL_BAR_SIZE, val, 1);
break;
case NPU2_GENID_BAR:
val = SETFIELD(NPU2_GENID_BAR_ADDR, 0ul, bar->base >> 16);
val = SETFIELD(NPU2_GENID_BAR_ENABLE, val, enable);
break;
default:
val = 0ul;
}
for (block = NPU2_BLOCK_SM_0; block <= NPU2_BLOCK_SM_3; block++) {
reg = NPU2_REG_OFFSET(0, block, bar->reg);
if (p)
npu2_write(p, reg, val);
else
npu2_scom_write(gcid, scom, reg, NPU2_MISC_DA_LEN_8B, val);
}
}
/* Trap for PCI command (0x4) to enable or disable device's BARs */
static int64_t npu2_cfg_write_cmd(void *dev,
struct pci_cfg_reg_filter *pcrf __unused,
uint32_t offset, uint32_t size,
uint32_t *data, bool write)
{
struct pci_virt_device *pvd = dev;
struct npu2_dev *ndev = pvd->data;
struct npu2_bar *ntl_npu_bar, *genid_npu_bar;
bool enabled;
if (!write)
return OPAL_PARTIAL;
if (offset != PCI_CFG_CMD)
return OPAL_PARAMETER;
if (size != 1 && size != 2 && size != 4)
return OPAL_PARAMETER;
/*
* Enable or disable NTL and GENID BAR. Two bricks share
* one GENID BAR, which is exposed via the first brick.
*/
enabled = !!(*data & PCI_CFG_CMD_MEM_EN);
ntl_npu_bar = &ndev->bars[0].npu2_bar;
genid_npu_bar = &ndev->bars[1].npu2_bar;
ntl_npu_bar->flags = SETFIELD(NPU2_BAR_FLAG_ENABLED, ntl_npu_bar->flags, enabled);
npu2_write_bar(ndev->npu, ntl_npu_bar, 0, 0);
/*
* Enable/disable the GENID BAR. Two bricks share one GENID
* BAR which is exposed via the first brick so we need to
* track the enables separately.
*/
if (NPU2DEV_BRICK(ndev))
genid_npu_bar->flags = SETFIELD(NPU2_BAR_FLAG_ENABLED1, genid_npu_bar->flags,
enabled);
else
genid_npu_bar->flags = SETFIELD(NPU2_BAR_FLAG_ENABLED0, genid_npu_bar->flags,
enabled);
/* Enable the BAR if either device requests it enabled, otherwise disable it */
genid_npu_bar->flags = SETFIELD(NPU2_BAR_FLAG_ENABLED, genid_npu_bar->flags,
!!(genid_npu_bar->flags & (NPU2_BAR_FLAG_ENABLED0 |
NPU2_BAR_FLAG_ENABLED1)));
npu2_write_bar(ndev->npu, genid_npu_bar, 0, 0);
return OPAL_PARTIAL;
}
static int64_t npu2_cfg_read_bar(struct npu2_dev *dev __unused,
struct pci_cfg_reg_filter *pcrf,
uint32_t offset, uint32_t size,
uint32_t *data)
{
struct npu2_pcie_bar *bar = (struct npu2_pcie_bar *) pcrf->data;
if (!(bar->flags & NPU2_PCIE_BAR_FLAG_TRAPPED))
return OPAL_PARTIAL;
if ((size != 4) ||
(offset != pcrf->start && offset != pcrf->start + 4))
return OPAL_PARAMETER;
if (bar->flags & NPU2_PCIE_BAR_FLAG_SIZE_HI)
*data = bar->npu2_bar.size >> 32;
else
*data = bar->npu2_bar.size;
bar->flags &= ~(NPU2_PCIE_BAR_FLAG_TRAPPED | NPU2_PCIE_BAR_FLAG_SIZE_HI);
return OPAL_SUCCESS;
}
static int64_t npu2_cfg_write_bar(struct npu2_dev *dev,
struct pci_cfg_reg_filter *pcrf,
uint32_t offset, uint32_t size,
uint32_t data)
{
struct npu2_pcie_bar *bar = (struct npu2_pcie_bar *) pcrf->data;
struct npu2_bar old_bar, *npu2_bar = &bar->npu2_bar;
if ((size != 4) ||
(offset != pcrf->start && offset != pcrf->start + 4))
return OPAL_PARAMETER;
/* Return BAR size on next read */
if (data == 0xffffffff) {
bar->flags |= NPU2_PCIE_BAR_FLAG_TRAPPED;
if (offset == pcrf->start + 4)
bar->flags |= NPU2_PCIE_BAR_FLAG_SIZE_HI;
return OPAL_SUCCESS;
}
if (offset == pcrf->start) {
npu2_bar->base &= 0xffffffff00000000UL;
npu2_bar->base |= (data & 0xfffffff0);
} else {
npu2_bar->base &= 0x00000000ffffffffUL;
npu2_bar->base |= ((uint64_t)data << 32);
if (NPU2_REG(npu2_bar->reg) == NPU2_GENID_BAR && NPU2DEV_BRICK(dev))
npu2_bar->base -= 0x10000;
old_bar.reg = npu2_bar->reg;
npu2_read_bar(dev->npu, &old_bar);
/* Only allow changing the base address if the BAR is not enabled */
if ((npu2_bar->flags & NPU2_BAR_FLAG_ENABLED) &&
(npu2_bar->base != old_bar.base)) {
npu2_bar->base = old_bar.base;
return OPAL_HARDWARE;
}
npu2_write_bar(dev->npu, &bar->npu2_bar, 0, 0);
}
/* To update the config cache */
return OPAL_PARTIAL;
}
static int64_t npu2_dev_cfg_bar(void *dev, struct pci_cfg_reg_filter *pcrf,
uint32_t offset, uint32_t len, uint32_t *data,
bool write)
{
struct pci_virt_device *pvd = dev;
struct npu2_dev *ndev = (struct npu2_dev *) pvd->data;
if (write)
return npu2_cfg_write_bar(ndev, pcrf, offset, len, *data);
return npu2_cfg_read_bar(ndev, pcrf, offset, len, data);
}
static int64_t npu2_dev_cfg_exp_devcap(void *dev,
struct pci_cfg_reg_filter *pcrf __unused,
uint32_t offset, uint32_t size,
uint32_t *data, bool write)
{
struct pci_virt_device *pvd = dev;
struct npu2_dev *ndev = pvd->data;
int rc;
assert(write);
if ((size != 2) || (offset & 1)) {
/* Short config writes are not supported */
prlog(PR_ERR, "NPU%d: Unsupported write to pcie control register\n",
ndev->nvlink.phb->opal_id);
return OPAL_PARAMETER;
}
if (*data & PCICAP_EXP_DEVCTL_FUNC_RESET)
npu2_dev_procedure_reset(ndev);
rc = purge_l2_l3_caches();
if (rc)
return rc;
return OPAL_PARTIAL;
}
#define NPU2_CFG_READ(size, type) \
static int64_t npu2_cfg_read##size(struct phb *phb, uint32_t bdfn, \
uint32_t offset, type *data) \
{ \
uint32_t val; \
int64_t ret; \
\
ret = pci_virt_cfg_read(phb, bdfn, offset, \
sizeof(*data), &val); \
*data = (type)val; \
return ret; \
}
#define NPU2_CFG_WRITE(size, type) \
static int64_t npu2_cfg_write##size(struct phb *phb, uint32_t bdfn, \
uint32_t offset, type data) \
{ \
uint32_t val = data; \
int64_t ret; \
\
ret = pci_virt_cfg_write(phb, bdfn, offset, \
sizeof(data), val); \
return ret; \
}
NPU2_CFG_READ(8, u8);
NPU2_CFG_READ(16, u16);
NPU2_CFG_READ(32, u32);
NPU2_CFG_WRITE(8, u8);
NPU2_CFG_WRITE(16, u16);
NPU2_CFG_WRITE(32, u32);
static int __npu2_dev_bind_pci_dev(struct phb *phb __unused,
struct pci_device *pd,
void *data)
{
struct npu2_dev *dev = data;
struct dt_node *pci_dt_node;
char *pcislot;
/* Ignore non-nvidia PCI devices */
if ((pd->vdid & 0xffff) != 0x10de)
return 0;
/* Find the PCI device's slot location */
for (pci_dt_node = pd->dn;
pci_dt_node && !dt_find_property(pci_dt_node, "ibm,loc-code");
pci_dt_node = pci_dt_node->parent);
if (!pci_dt_node)
return 0;
pcislot = (char *)dt_prop_get(pci_dt_node, "ibm,loc-code");
NPU2DEVDBG(dev, "Comparing GPU '%s' and NPU2 '%s'\n",
pcislot, dev->nvlink.slot_label);
if (streq(pcislot, dev->nvlink.slot_label))
return 1;
return 0;
}
static int64_t npu2_gpu_bridge_sec_bus_reset(void *dev,
struct pci_cfg_reg_filter *pcrf __unused,
uint32_t offset, uint32_t len,
uint32_t *data, bool write)
{
struct pci_device *pd = dev;
struct pci_device *gpu;
struct phb *npphb;
struct npu2 *npu;
struct dt_node *np;
struct npu2_dev *ndev;
int i;
assert(write);
if ((len != 2) || (offset & 1)) {
/* Short config writes are not supported */
PCIERR(pd->phb, pd->bdfn,
"Unsupported write to bridge control register\n");
return OPAL_PARAMETER;
}
gpu = list_top(&pd->children, struct pci_device, link);
if (gpu && (*data & PCI_CFG_BRCTL_SECONDARY_RESET)) {
int64_t rc;
dt_for_each_compatible(dt_root, np, "ibm,power9-npu-pciex") {
npphb = pci_get_phb(dt_prop_get_cell(np,
"ibm,opal-phbid", 1));
if (!npphb || npphb->phb_type != phb_type_npu_v2)
continue;
npu = phb_to_npu2_nvlink(npphb);
for (i = 0; i < npu->total_devices; ++i) {
ndev = &npu->devices[i];
if (ndev->nvlink.pd == gpu)
npu2_dev_procedure_reset(ndev);
}
}
rc = purge_l2_l3_caches();
if (rc)
return rc;
}
return OPAL_PARTIAL;
}
static void npu2_dev_bind_pci_dev(struct npu2_dev *dev)
{
struct phb *phb;
uint32_t i;
if (dev->nvlink.pd)
return;
for (i = 0; i < 64; i++) {
if (dev->npu->phb_nvlink.opal_id == i)
continue;
phb = pci_get_phb(i);
if (!phb)
continue;
dev->nvlink.pd = pci_walk_dev(phb, NULL, __npu2_dev_bind_pci_dev, dev);
if (dev->nvlink.pd) {
dev->nvlink.phb = phb;
/* Found the device, set the bit in config space */
npu2_set_link_flag(dev, NPU2_DEV_PCI_LINKED);
/*
* We define a custom sec bus reset handler for a slot
* with an NVLink-connected GPU to prevent HMIs which
* will otherwise happen if we reset GPU before
* resetting NVLinks.
*/
if (dev->nvlink.pd->parent &&
dev->nvlink.pd->parent->slot)
pci_add_cfg_reg_filter(dev->nvlink.pd->parent,
PCI_CFG_BRCTL, 2,
PCI_REG_FLAG_WRITE,
npu2_gpu_bridge_sec_bus_reset);
return;
}
}
NPU2DEVINF(dev, "No PCI device found for slot '%s'\n",
dev->nvlink.slot_label);
}
static struct lock pci_npu_phandle_lock = LOCK_UNLOCKED;
static void npu2_append_phandle(struct dt_node *dn,
u32 phandle)
{
struct dt_property *prop;
uint32_t *npu_phandles;
size_t len;
/*
* Use a lock to make sure no one else has a reference to an
* ibm,npu property (this assumes this is the only function
* that holds a reference to it)
*/
lock(&pci_npu_phandle_lock);
/* This function shouldn't be called unless ibm,npu exists */
prop = (struct dt_property *)dt_require_property(dn, "ibm,npu", -1);
/* Need to append to the properties */
len = prop->len + sizeof(*npu_phandles);
dt_resize_property(&prop, len);
npu_phandles = (uint32_t *)prop->prop;
npu_phandles[len / sizeof(*npu_phandles) - 1] = phandle;
unlock(&pci_npu_phandle_lock);
}
static struct dt_node *npu2_create_memory_dn(uint64_t addr, uint64_t size)
{
struct dt_node *mem;
static u32 chip_id = 255;
mem = dt_find_by_name_addr(dt_root, "memory", addr);
if (mem)
return mem;
mem = dt_new_addr(dt_root, "memory", addr);
if (!mem)
return NULL;
dt_add_property_string(mem, "device_type", "memory");
dt_add_property_string(mem, "compatible", "ibm,coherent-device-memory");
dt_add_property_u64s(mem, "reg", addr, size);
dt_add_property_cells(mem, "ibm,chip-id", chip_id);
dt_add_property_u64s(mem, "linux,usable-memory", addr, 0);
dt_add_property_cells(mem, "ibm,associativity", 4, chip_id, chip_id, chip_id, chip_id);
chip_id--;
assert(chip_id);
return mem;
}
/* There are potentially multiple links per GPU, so lookup the GPU memory based
* on bdfn. */
static void npu2_get_gpu_base(struct npu2_dev *ndev, uint64_t *addr, uint64_t *size)
{
struct npu2 *p = ndev->npu;
int group;
group = PCI_DEV(ndev->bdfn);
phys_map_get(ndev->npu->chip_id, p->gpu_map_type, group, addr, size);
}
static void npu2_dn_fixup_gmb(struct dt_node *pd_dn, struct npu2_dev *ndev)
{
uint64_t gpu_base, gpu_size, gta;
struct dt_node *mem_dn;
npu2_get_gpu_base(ndev, &gpu_base, &gpu_size);
mem_dn = npu2_create_memory_dn(gpu_base, gpu_size);
assert(mem_dn);
dt_add_property_cells(pd_dn, "memory-region", mem_dn->phandle);
/* Coral mode address compression. This is documented in Figure 3.5
* "P9->GPU RA Compression (Coral) of the NPU2 workbook". */
gta = ((gpu_base >> 42) & 0x1) << 42;
gta |= ((gpu_base >> 45) & 0x3) << 43;
gta |= ((gpu_base >> 49) & 0x3) << 45;
gta |= gpu_base & ((1UL << 43) - 1);
dt_add_property_u64s(pd_dn, "ibm,device-tgt-addr", gta);
}
static int npu2_assign_gmb(struct npu2_dev *ndev)
{
struct npu2 *p = ndev->npu;
int peers, mode;
uint32_t bdfn;
uint64_t base, size, reg, val, gmb;
/* Need to work out number of link peers. This amount to
* working out the maximum function number. So work start at
* the highest bdfn (fn = 6) and count back until we find a
* npu2_dev. */
for (bdfn = (ndev->bdfn & ~0x7) | NPU2_LINKS_PER_CHIP;
PCI_FUNC(bdfn) != 0x7; bdfn = (bdfn & ~0x7) | (PCI_FUNC(bdfn) - 1))
if (npu2_bdf_to_dev(p, bdfn))
break;
peers = PCI_FUNC(bdfn);
npu2_get_gpu_base(ndev, &base, &size);
NPU2DBG(p, "Setting BAR region dt:%llx\n", base);
val = SETFIELD(NPU2_MEM_BAR_EN, 0ULL, 1);
val = SETFIELD(NPU2_MEM_BAR_SEL_MEM, val, base >> (63-14));
val = SETFIELD(NPU2_MEM_BAR_GROUP, val, base >> (63-18));
val = SETFIELD(NPU2_MEM_BAR_CHIP, val, base >> (63-21));
val = SETFIELD(NPU2_MEM_BAR_NODE_ADDR, val, base >> (63-33));
val = SETFIELD(NPU2_MEM_BAR_POISON, val, 1);
val = SETFIELD(NPU2_MEM_BAR_GRANULE, val, 0);
/* We don't know how much memory the GPU has, so we may as well just
* pass the whole aperture through at this point. */
val = SETFIELD(NPU2_MEM_BAR_BAR_SIZE, val, ilog2(size >> 30));
switch (peers) {
case 0:
mode = 0;
break;
case 1:
mode = 1;
break;
case 2:
mode = 3;
break;
case 3:
mode = 6;
break;
case 5:
mode = 10;
break;
default:
/* Hardware does not support this configuration */
assert(0);
}
mode += PCI_FUNC(ndev->bdfn);
val = SETFIELD(NPU2_MEM_BAR_MODE, val, mode);
gmb = NPU2_GPU0_MEM_BAR;
if (NPU2DEV_BRICK(ndev))
gmb = NPU2_GPU1_MEM_BAR;
reg = NPU2_REG_OFFSET(NPU2_STACK_STCK_0 + NPU2DEV_STACK(ndev),
NPU2_BLOCK_SM_0, gmb);
npu2_write(p, reg, val);
reg = NPU2_REG_OFFSET(NPU2_STACK_STCK_0 + NPU2DEV_STACK(ndev),
NPU2_BLOCK_SM_1, gmb);
npu2_write(p, reg, val);
reg = NPU2_REG_OFFSET(NPU2_STACK_STCK_0 + NPU2DEV_STACK(ndev),
NPU2_BLOCK_SM_2, gmb);
npu2_write(p, reg, val);
reg = NPU2_REG_OFFSET(NPU2_STACK_STCK_0 + NPU2DEV_STACK(ndev),
NPU2_BLOCK_SM_3, gmb);
npu2_write(p, reg, val);
return 0;
}
static int npu2_dn_fixup(struct phb *phb,
struct pci_device *pd,
void *data __unused)
{
struct npu2 *p = phb_to_npu2_nvlink(phb);
struct npu2_dev *dev;
uint32_t speed;
const char *label;
dev = npu2_bdf_to_dev(p, pd->bdfn);
assert(dev);
if (dev->nvlink.phb || dev->nvlink.pd)
return 0;
npu2_assign_gmb(dev);
npu2_dn_fixup_gmb(pd->dn, dev);
dt_add_property_cells(pd->dn, "ibm,nvlink", dev->dt_node->phandle);
/*
* NVLink supports multiple speeds and device drivers need to know what
* speed has been set by firmware. Hostboot does the inits that set the
* link speed and tell us via HDAT and we need to copy that from the
* link node.
*/
speed = dt_prop_get_u32_def(dev->dt_node, "nvidia,link-speed", 0xff);
if (speed != 0xff)
dt_add_property_cells(pd->dn, "ibm,nvlink-speed", speed);
/*
* NPU2 devices have a slot label that indicates which GPU slot
* this NPU is connected to. Add a location code to the NVlink
* device node based on the slot label.
*/
label = dt_prop_get_def(dev->dt_node, "ibm,slot-label", NULL);
if (!label) {
/**
* @fwts-label NPUNoPHBSlotLabel
* @fwts-advice No GPU/NPU2 slot information was found.
* NVLink2 functionality will not work.
*/
prlog(PR_ERR, "NPU: Cannot find GPU slot information\n");
return 0;
}
dt_add_property_string(pd->dn, "ibm,loc-code", label);
dev->nvlink.slot_label = label;
/*
* Bind the emulated PCI device with the real one, which can't
* be done until the PCI devices are populated. Once the real
* PCI device is identified, we also need fix the device-tree
* for it
*/
npu2_dev_bind_pci_dev(dev);
if (dev->nvlink.phb && dev->nvlink.pd && dev->nvlink.pd->dn) {
if (dt_find_property(dev->nvlink.pd->dn, "ibm,npu"))
npu2_append_phandle(dev->nvlink.pd->dn, pd->dn->phandle);
else
dt_add_property_cells(dev->nvlink.pd->dn, "ibm,npu", pd->dn->phandle);
dt_add_property_cells(pd->dn, "ibm,gpu", dev->nvlink.pd->dn->phandle);
dev->nvlink.gpu_bdfn = dev->nvlink.pd->bdfn;
}
return 0;
}
static int npu2_links_per_gpu(struct phb *phb,
struct pci_device *pd,
void *data)
{
struct npu2 *p = phb_to_npu2_nvlink(phb);
struct npu2_dev *dev;
int *nlinks = (int *)data;
dev = npu2_bdf_to_dev(p, pd->bdfn);
assert(dev);
if (dev->nvlink.phb && dev->nvlink.pd && dev->nvlink.pd->dn) {
const struct dt_property *prop;
int n;
/* The link count is the number of phandles in "ibm,npu" */
prop = dt_find_property(dev->nvlink.pd->dn, "ibm,npu");
if (!prop)
return 0;
/* Count could vary by gpu, so find the max */
n = prop->len / sizeof(uint32_t);
if (n > *nlinks)
*nlinks = n;
}
return 0;
}
static void npu2_phb_fixup_scominit(struct dt_node *dn, int links_per_gpu)
{
uint32_t gcid = dt_get_chip_id(dn);
uint64_t val, mask;
/*
* MRBSP settings for 2- and 3-link GPU systems. These can improve
* GPU peer-to-peer fully ordered write performance.
*/
if (links_per_gpu == 3) {
val = PPC_BIT(30) | PPC_BIT(34) | PPC_BIT(36) | PPC_BIT(37) |
PPC_BIT(44) | PPC_BIT(45);
mask = PPC_BITMASK(28,39) | PPC_BITMASK(44,47);
} else if (links_per_gpu == 2) {
val = PPC_BIT(46) | PPC_BIT(47);
mask = PPC_BITMASK(44,47);
} else
return;
xscom_write_mask(gcid, 0x50110c0, val, mask);
xscom_write_mask(gcid, 0x50112c0, val, mask);
xscom_write_mask(gcid, 0x50114c0, val, mask);
}
static void npu2_phb_final_fixup(struct phb *phb)
{
int links_per_gpu = 0;
struct dt_node *np;
pci_walk_dev(phb, NULL, npu2_dn_fixup, NULL);
/*
* Now that the emulated devices are bound to the real ones, we can
* determine links_per_gpu and do some final init.
*/
pci_walk_dev(phb, NULL, npu2_links_per_gpu, &links_per_gpu);
dt_for_each_compatible(dt_root, np, "ibm,power9-npu")
npu2_phb_fixup_scominit(np, links_per_gpu);
}
static void npu2_init_ioda_cache(struct npu2 *p)
{
/* TVT */
memset(p->tve_cache, 0, sizeof(p->tve_cache));
}
static int64_t npu2_ioda_reset(struct phb *phb, bool purge)
{
struct npu2 *p = phb_to_npu2_nvlink(phb);
uint32_t i;
if (purge) {
NPU2DBG(p, "Purging all IODA tables...\n");
npu2_init_ioda_cache(p);
}
/* TVT */
npu2_ioda_sel(p, NPU2_ATS_IODA_TBL_TVT, 0, true);
for (i = 0; i < ARRAY_SIZE(p->tve_cache); i++)
out_be64(p->regs + NPU2_ATS_IODA_DATA, p->tve_cache[i]);
return OPAL_SUCCESS;
}
static void npu2_write_mcd(struct npu2 *p, uint64_t pcb_addr, uint64_t addr,
uint64_t size)
{
uint64_t val;
NPU2DBG(p, "Setting MCD addr:%llx\n", pcb_addr);
assert(is_pow2(size));
val = MCD_BANK_CN_VALID;
val = SETFIELD(MCD_BANK_CN_SIZE, val, (size >> 25) - 1);
val = SETFIELD(MCD_BANK_CN_ADDR, val, addr >> 25);
xscom_write(p->chip_id, pcb_addr, val);
}
static void npu2_mcd_init(struct npu2 *p)
{
int i;
uint64_t size, addr, gpu_min_addr, gpu_max_addr, total_size;
/* Init memory cache directory (MCD) registers. */
phys_map_get(p->chip_id, p->gpu_map_type, NPU2_LINKS_PER_CHIP - 1,
&gpu_min_addr, NULL);
phys_map_get(p->chip_id, p->gpu_map_type, 0, &gpu_max_addr, &size);
gpu_max_addr += size;
/* We assume GPU memory is contiguous from the first possible GPU to the
* last and that the size is the same so best to check that. */
for (i = 0; i < NPU2_LINKS_PER_CHIP; i++) {
uint64_t tmp;
phys_map_get(p->chip_id, p->gpu_map_type, i, &addr, &tmp);
assert((addr >= gpu_min_addr) && (addr + tmp <= gpu_max_addr));
assert(tmp == size);
}
/* We have two MCDs, so if neccessary we can split the region covered
* across both if total_size is not a power of two. */
total_size = gpu_max_addr - gpu_min_addr;
size = 1ull << ilog2(total_size);
/* Allocate the biggest chunk first as we assume gpu_max_addr has the
* highest alignment. */
addr = gpu_max_addr - size;
npu2_write_mcd(p, MCD0_BANK0_CN3, addr, size);
total_size -= size;
if (total_size) {
/* total_size was not a power of two, but the remainder should
* be if all GPUs were assigned the same size. */
assert(is_pow2(total_size));
size = 1ull << ilog2(total_size);
addr -= size;
assert(addr <= gpu_min_addr);
npu2_write_mcd(p, MCD1_BANK0_CN3, addr, size);
}
}
static void npu2_hw_init(struct npu2 *p)
{
uint64_t reg, val;
int s, b;
npu2_ioda_reset(&p->phb_nvlink, false);
/* Enable XTS retry mode */
val = npu2_read(p, NPU2_XTS_CFG);
npu2_write(p, NPU2_XTS_CFG, val | NPU2_XTS_CFG_MMIOSD | NPU2_XTS_CFG_TRY_ATR_RO);
val = npu2_read(p, NPU2_XTS_CFG2);
npu2_write(p, NPU2_XTS_CFG2, val | NPU2_XTS_CFG2_NO_FLUSH_ENA);
/*
* There are three different ways we configure the MCD and memory map.
* 1) Old way
* Skiboot configures the MCD and puts GPUs at 4TB and below
* 2) New way with MCD
* Hostboot configures the MCD and skiboot puts GPU at 4TB and above
* 3) New way without MCD
* No one configures the MCD and skiboot puts GPU at 4TB and below
*
* 1) Will go away evenutally as it's a configuration that can
* cause an xstop or data integrity problems. We are keeping
* it around to support existing hostboot. Print error
* message if used.
* 2) Is for smaller memory configurations and will be used
* initially for GPUs on Witherspoon. Supports only to
* 512GB of memory and 4 GPUs per socket.
* 3) Is for fully populated configurations of 4TB of memory
* and 6GPUs per socket. May have performance impacts.
*
* The different configurations can be detected via the following scoms:
* 1) 0x5011c0c bit 2 = 1, 0x5011c0a bits 42:48 = 0
* 2) 0x5011c0c bit 2 = 1, 0x5011c0a bits 42:48 = 7
* 3) 0x5011c0c bit 2 = 0, 0x5011c0a bits 42:48 = 0
*/
/* Get 0x05011c0c bit 2 = 1 */
xscom_read(p->chip_id, PB_CENT_HP_MODE_CURR, &val);
if ((val & PB_CFG_CHG_RATE_GP_MASTER) != 0) {
/* Get 0x05011c0a bits 42:48 */
xscom_read(p->chip_id, PB_CENT_MODE, &val);
if (GETFIELD(PB_CFG_CHIP_ADDR_EXTENSION_MASK_CENT, val) == 0) {
/* 1) */
NPU2DBG(p, "Using old memory map + MCD enabled in skiboot\n");
NPU2ERR(p, "!!! Old firmware detected. Update hostboot for new MCD mapping !!!\n");
p->gpu_map_type = GPU_MEM_4T_DOWN;
npu2_mcd_init(p);
} else if (GETFIELD(PB_CFG_CHIP_ADDR_EXTENSION_MASK_CENT, val) == 7) {
/* 2) */
NPU2DBG(p, "Using small memory map + MCD enabled\n");
p->gpu_map_type = GPU_MEM_4T_UP;
} else
NPU2ERR(p, "!!! Unsupported NPU2 configuration. "
"0x%llx!!!\n", val);
} else {
/* 3) */
NPU2DBG(p, "Using large memory map + MCD disabled\n");
p->gpu_map_type = GPU_MEM_4T_DOWN;
}
/* Static initialization of every relaxed-ordering cfg[2] register */
val = NPU2_RELAXED_ORDERING_CMD_CL_DMA_W |
NPU2_RELAXED_ORDERING_CMD_CL_DMA_W_HP |
NPU2_RELAXED_ORDERING_CMD_CL_DMA_INJ |
NPU2_RELAXED_ORDERING_CMD_PR_DMA_INJ |
NPU2_RELAXED_ORDERING_CMD_DMA_PR_W |
NPU2_RELAXED_ORDERING_CMD_CL_RD_NC_F0 |
NPU2_RELAXED_ORDERING_SOURCE4_RDENA;
for (s = NPU2_STACK_STCK_0; s <= NPU2_STACK_STCK_2; s++) {
for (b = NPU2_BLOCK_SM_0; b <= NPU2_BLOCK_SM_3; b++) {
reg = NPU2_REG_OFFSET(s, b, NPU2_RELAXED_ORDERING_CFG(2));
npu2_write(p, reg, val);
}
}
}
static int64_t npu2_map_pe_dma_window_real(struct phb *phb,
uint64_t pe_num,
uint16_t window_id,
uint64_t pci_start_addr __unused,
uint64_t pci_mem_size __unused)
{
struct npu2 *p = phb_to_npu2_nvlink(phb);
uint64_t tve;
/* Sanity check. Each PE has one corresponding TVE */
if (pe_num >= NPU2_MAX_PE_NUM ||
window_id != pe_num)
return OPAL_PARAMETER;
if (pci_mem_size) {
/* GPUs need to be able to access the MMIO memory space as well.
* On POWER9 this is above the top of ram so disable the TVT
* range check allowing access to all memory addresses. */
tve = 0;
} else {
/* Disable */
tve = PPC_BIT(51);
}
npu2_ioda_sel(p, NPU2_ATS_IODA_TBL_TVT, window_id, false);
out_be64(p->regs + NPU2_ATS_IODA_DATA, tve);
p->tve_cache[window_id] = tve;
return OPAL_SUCCESS;
}
static int64_t npu2_map_pe_dma_window(struct phb *phb,
uint64_t pe_num,
uint16_t window_id,
uint16_t tce_levels,
uint64_t tce_table_addr,
uint64_t tce_table_size,
uint64_t tce_page_size)
{
struct npu2 *p = phb_to_npu2_nvlink(phb);
uint64_t tts_encoded;
uint64_t data64 = 0;
/* Sanity check. Each PE has one corresponding TVE */
if (pe_num >= NPU2_MAX_PE_NUM ||
window_id != pe_num)
return OPAL_PARAMETER;
/*
* Special condition, zero TCE table size used to disable
* the TVE.
*/
if (!tce_table_size) {
npu2_ioda_sel(p, NPU2_ATS_IODA_TBL_TVT, window_id, false);
out_be64(p->regs + NPU2_ATS_IODA_DATA, 0ul);
p->tve_cache[window_id] = 0ul;
return OPAL_SUCCESS;
}
/* Additional arguments validation */
if (tce_levels < 1 ||
tce_levels > 4 ||
!is_pow2(tce_table_size) ||
tce_table_size < 0x1000)
return OPAL_PARAMETER;
/* TCE table size */
data64 = SETFIELD(NPU2_ATS_IODA_TBL_TVT_TTA, 0ul, tce_table_addr >> 12);
tts_encoded = ilog2(tce_table_size) - 11;
if (tts_encoded > 39)
return OPAL_PARAMETER;
data64 = SETFIELD(NPU2_ATS_IODA_TBL_TVT_SIZE, data64, tts_encoded);
/* TCE page size */
switch (tce_page_size) {
case 0x10000: /* 64K */
data64 = SETFIELD(NPU2_ATS_IODA_TBL_TVT_PSIZE, data64, 5);
break;
case 0x1000000: /* 16M */
data64 = SETFIELD(NPU2_ATS_IODA_TBL_TVT_PSIZE, data64, 13);
break;
case 0x10000000: /* 256M */
data64 = SETFIELD(NPU2_ATS_IODA_TBL_TVT_PSIZE, data64, 17);
break;
case 0x1000: /* 4K */
default:
data64 = SETFIELD(NPU2_ATS_IODA_TBL_TVT_PSIZE, data64, 1);
}
/* Number of levels */
data64 = SETFIELD(NPU2_ATS_IODA_TBL_TVT_LEVEL, data64, tce_levels - 1);
/* Update to hardware */
npu2_ioda_sel(p, NPU2_ATS_IODA_TBL_TVT, window_id, false);
out_be64(p->regs + NPU2_ATS_IODA_DATA, data64);
p->tve_cache[window_id] = data64;
return OPAL_SUCCESS;
}
static int64_t npu2_set_pe(struct phb *phb,
uint64_t pe_num,
uint64_t bdfn,
uint8_t bcompare,
uint8_t dcompare,
uint8_t fcompare,
uint8_t action)
{
struct npu2 *p;
struct npu2_dev *dev;
uint64_t reg, val;
/* Sanity check */
if (action != OPAL_MAP_PE && action != OPAL_UNMAP_PE)
return OPAL_PARAMETER;
if (pe_num >= NPU2_MAX_PE_NUM)
return OPAL_PARAMETER;
if (bdfn >> 8)
return OPAL_PARAMETER;
if (bcompare != OpalPciBusAll ||
dcompare != OPAL_COMPARE_RID_DEVICE_NUMBER ||
fcompare != OPAL_COMPARE_RID_FUNCTION_NUMBER)
return OPAL_UNSUPPORTED;
if (phb->phb_type != phb_type_npu_v2)
return OPAL_PARAMETER;
p = phb_to_npu2_nvlink(phb);
if (!p)
return OPAL_PARAMETER;
dev = npu2_bdf_to_dev(p, bdfn);
if (!dev)
return OPAL_PARAMETER;
val = NPU2_CQ_BRICK_BDF2PE_MAP_ENABLE;
val = SETFIELD(NPU2_CQ_BRICK_BDF2PE_MAP_PE, val, pe_num);
val = SETFIELD(NPU2_CQ_BRICK_BDF2PE_MAP_BDF, val, dev->nvlink.gpu_bdfn);
if (!NPU2DEV_BRICK(dev))
reg = NPU2_REG_OFFSET(NPU2_STACK_STCK_0 + dev->brick_index/2,
NPU2_BLOCK_CTL, NPU2_CQ_BRICK0_BDF2PE_MAP0);
else
reg = NPU2_REG_OFFSET(NPU2_STACK_STCK_0 + dev->brick_index/2,
NPU2_BLOCK_CTL, NPU2_CQ_BRICK1_BDF2PE_MAP0);
npu2_write(p, reg, val);
val = NPU2_MISC_BRICK_BDF2PE_MAP_ENABLE;
val = SETFIELD(NPU2_MISC_BRICK_BDF2PE_MAP_PE, val, pe_num);
val = SETFIELD(NPU2_MISC_BRICK_BDF2PE_MAP_BDF, val, dev->nvlink.gpu_bdfn);
reg = NPU2_REG_OFFSET(NPU2_STACK_MISC, NPU2_BLOCK_MISC,
NPU2_MISC_BRICK0_BDF2PE_MAP0 + (dev->brick_index * 0x18));
npu2_write(p, reg, val);
return OPAL_SUCCESS;
}
static int64_t npu2_get_link_state(struct pci_slot *slot __unused, uint8_t *val)
{
/*
* As we're emulating all PCI stuff, the link bandwidth
* isn't big deal anyway.
*/
*val = OPAL_SHPC_LINK_UP_x1;
return OPAL_SUCCESS;
}
static int64_t npu2_get_power_state(struct pci_slot *slot __unused, uint8_t *val)
{
*val = PCI_SLOT_POWER_ON;
return OPAL_SUCCESS;
}
static int64_t npu2_hreset(struct pci_slot *slot __unused)
{
struct npu2 *p;
int i;
struct npu2_dev *ndev;
p = phb_to_npu2_nvlink(slot->phb);
NPU2INF(p, "Hreset PHB state\n");
for (i = 0; i < p->total_devices; i++) {
ndev = &p->devices[i];
if (ndev) {
NPU2DEVINF(ndev, "Resetting device\n");
reset_ntl(ndev);
}
}
return purge_l2_l3_caches();
}
static int64_t npu2_freset(struct pci_slot *slot __unused)
{
return OPAL_SUCCESS;
}
static int64_t npu2_creset(struct pci_slot *slot)
{
struct npu2 *p;
int i;
struct npu2_dev *ndev;
p = phb_to_npu2_nvlink(slot->phb);
NPU2INF(p, "Creset PHB state\n");
for (i = 0; i < p->total_devices; i++) {
ndev = &p->devices[i];
if (ndev) {
NPU2DEVINF(ndev, "Resetting device\n");
reset_ntl(ndev);
}
}
return OPAL_SUCCESS;
}
static struct pci_slot *npu2_slot_create(struct phb *phb)
{
struct pci_slot *slot;
slot = pci_slot_alloc(phb, NULL);
if (!slot)
return slot;
/* Elementary functions */
slot->ops.get_presence_state = NULL;
slot->ops.get_link_state = npu2_get_link_state;
slot->ops.get_power_state = npu2_get_power_state;
slot->ops.get_attention_state = NULL;
slot->ops.get_latch_state = NULL;
slot->ops.set_power_state = NULL;
slot->ops.set_attention_state = NULL;
slot->ops.prepare_link_change = NULL;
slot->ops.poll_link = NULL;
slot->ops.hreset = npu2_hreset;
slot->ops.freset = npu2_freset;
slot->ops.creset = npu2_creset;
return slot;
}
int64_t npu2_freeze_status(struct phb *phb __unused,
uint64_t pe_number __unused,
uint8_t *freeze_state,
uint16_t *pci_error_type,
uint16_t *severity)
{
/*
* FIXME: When it's called by skiboot PCI config accessor,
* the PE number is fixed to 0, which is incorrect. We need
* introduce another PHB callback to translate it. For now,
* it keeps the skiboot PCI enumeration going.
*/
*freeze_state = OPAL_EEH_STOPPED_NOT_FROZEN;
*pci_error_type = OPAL_EEH_NO_ERROR;
if (severity)
*severity = OPAL_EEH_SEV_NO_ERROR;
return OPAL_SUCCESS;
}
static int64_t npu2_eeh_next_error(struct phb *phb,
uint64_t *first_frozen_pe,
uint16_t *pci_error_type,
uint16_t *severity)
{
struct npu2 *p = phb_to_npu2_nvlink(phb);
int i;
uint64_t result = 0;
if (!first_frozen_pe || !pci_error_type || !severity)
return OPAL_PARAMETER;
*first_frozen_pe = -1;
*pci_error_type = OPAL_EEH_NO_ERROR;
*severity = OPAL_EEH_SEV_NO_ERROR;
for (i = 0; i < NPU2_MAX_PE_NUM; i++) {
result = npu2_read(p, NPU2_MISC_PESTB(i));
if (result > 0) {
*first_frozen_pe = i;
*pci_error_type = OPAL_EEH_PE_ERROR;
*severity = OPAL_EEH_SEV_PE_ER;
break;
}
}
return OPAL_SUCCESS;
}
static int64_t npu2_tce_kill(struct phb *phb, uint32_t kill_type,
uint64_t pe_number, uint32_t tce_size,
uint64_t dma_addr, uint32_t npages)
{
struct npu2 *npu = phb_to_npu2_nvlink(phb);
uint32_t tce_page_size;
uint64_t val;
if (pe_number > NPU2_MAX_PE_NUM)
return OPAL_PARAMETER;
sync();
switch(kill_type) {
case OPAL_PCI_TCE_KILL_PAGES:
tce_page_size = 1ULL << (
11 + GETFIELD(npu->tve_cache[pe_number],
NPU2_ATS_IODA_TBL_TVT_PSIZE));
if (tce_page_size != tce_size) {
NPU2ERR(npu, "npu2_tce_kill: Unexpected TCE size (got 0x%x expected 0x%x)\n",
tce_size, tce_page_size);
return OPAL_PARAMETER;
}
if (npages < 128) {
while (npages--) {
val = SETFIELD(NPU2_ATS_TCE_KILL_PENUM, dma_addr, pe_number);
npu2_write(npu, NPU2_ATS_TCE_KILL, NPU2_ATS_TCE_KILL_ONE | val);
dma_addr += tce_size;
}
break;
}
/*
* For too many TCEs do not bother with the loop above and simply
* flush everything, going to be lot faster.
*/
/* Fall through */
case OPAL_PCI_TCE_KILL_PE:
/*
* NPU2 doesn't support killing a PE so fall through
* and do a kill all instead.
*/
case OPAL_PCI_TCE_KILL_ALL:
npu2_write(npu, NPU2_ATS_TCE_KILL, NPU2_ATS_TCE_KILL_ALL);
break;
default:
return OPAL_PARAMETER;
}
return OPAL_SUCCESS;
}
static const struct phb_ops npu_ops = {
.cfg_read8 = npu2_cfg_read8,
.cfg_read16 = npu2_cfg_read16,
.cfg_read32 = npu2_cfg_read32,
.cfg_write8 = npu2_cfg_write8,
.cfg_write16 = npu2_cfg_write16,
.cfg_write32 = npu2_cfg_write32,
.device_init = NULL,
.phb_final_fixup = npu2_phb_final_fixup,
.ioda_reset = npu2_ioda_reset,
.papr_errinjct_reset = NULL,
.pci_reinit = NULL,
.set_phb_mem_window = NULL,
.phb_mmio_enable = NULL,
.map_pe_mmio_window = NULL,
.map_pe_dma_window = npu2_map_pe_dma_window,
.map_pe_dma_window_real = npu2_map_pe_dma_window_real,
.pci_msi_eoi = NULL,
.set_xive_pe = NULL,
.get_msi_32 = NULL,
.get_msi_64 = NULL,
.set_pe = npu2_set_pe,
.set_peltv = NULL,
.eeh_freeze_status = npu2_freeze_status,
.eeh_freeze_clear = NULL,
.eeh_freeze_set = NULL,
.next_error = npu2_eeh_next_error,
.err_inject = NULL,
.get_diag_data2 = NULL,
.set_capi_mode = NULL,
.set_capp_recovery = NULL,
.tce_kill = npu2_tce_kill,
};
static void assign_mmio_bars(uint64_t gcid, uint32_t scom, uint64_t reg[2], uint64_t mm_win[2])
{
uint32_t i;
struct npu2_bar *bar;
struct npu2_bar npu2_bars[] = {
/* NPU_REGS must be first in this list */
{ .type = NPU_REGS, .index = 0,
.reg = NPU2_REG_OFFSET(NPU2_STACK_STCK_0, 0, NPU2_PHY_BAR),
.flags = NPU2_BAR_FLAG_ENABLED },
{ .type = NPU_PHY, .index = 0,
.reg = NPU2_REG_OFFSET(NPU2_STACK_STCK_1, 0, NPU2_PHY_BAR),
.flags = NPU2_BAR_FLAG_ENABLED },
{ .type = NPU_PHY, .index = 1,
.reg = NPU2_REG_OFFSET(NPU2_STACK_STCK_2, 0, NPU2_PHY_BAR),
.flags = NPU2_BAR_FLAG_ENABLED },
{ .type = NPU_NTL, .index = 0,
.reg = NPU2_REG_OFFSET(NPU2_STACK_STCK_0, 0, NPU2_NTL0_BAR) },
{ .type = NPU_NTL, .index = 1,
.reg = NPU2_REG_OFFSET(NPU2_STACK_STCK_0, 0, NPU2_NTL1_BAR) },
{ .type = NPU_NTL, .index = 2,
.reg = NPU2_REG_OFFSET(NPU2_STACK_STCK_1, 0, NPU2_NTL0_BAR) },
{ .type = NPU_NTL, .index = 3,
.reg = NPU2_REG_OFFSET(NPU2_STACK_STCK_1, 0, NPU2_NTL1_BAR) },
{ .type = NPU_NTL, .index = 4,
.reg = NPU2_REG_OFFSET(NPU2_STACK_STCK_2, 0, NPU2_NTL0_BAR) },
{ .type = NPU_NTL, .index = 5,
.reg = NPU2_REG_OFFSET(NPU2_STACK_STCK_2, 0, NPU2_NTL1_BAR) },
{ .type = NPU_GENID, .index = 0,
.reg = NPU2_REG_OFFSET(NPU2_STACK_STCK_0, 0, NPU2_GENID_BAR) },
{ .type = NPU_GENID, .index = 1,
.reg = NPU2_REG_OFFSET(NPU2_STACK_STCK_1, 0, NPU2_GENID_BAR) },
{ .type = NPU_GENID, .index = 2,
.reg = NPU2_REG_OFFSET(NPU2_STACK_STCK_2, 0, NPU2_GENID_BAR) },
};
for (i = 0; i < ARRAY_SIZE(npu2_bars); i++) {
bar = &npu2_bars[i];
npu2_get_bar(gcid, bar);
npu2_write_bar(NULL, bar, gcid, scom);
}
/* Global MMIO BAR */
reg[0] = npu2_bars[0].base;
reg[1] = npu2_bars[0].size;
/* NTL and GENID BARs are exposed to kernel via the mm
* window */
mm_win[0] = npu2_bars[3].base;
mm_win[1] = npu2_bars[ARRAY_SIZE(npu2_bars) - 1].base +
npu2_bars[ARRAY_SIZE(npu2_bars) - 1].size -
mm_win[0];
}
/*
* Set up NPU for NVLink and create PCI root device node
* accordingly.
*/
int npu2_nvlink_init_npu(struct npu2 *npu)
{
struct dt_node *np;
uint64_t reg[2], mm_win[2], val, mask;
/* TODO: Clean this up with register names, etc. when we get
* time. This just turns NVLink mode on in each brick and should
* get replaced with a patch from ajd once we've worked out how
* things are going to work there.
*
* Obviously if the year is now 2020 that didn't happen and you
* should fix this :-) */
val = PPC_BIT(58);
mask = PPC_BIT(58) | /* CONFIG_NVLINK_MODE */
PPC_BIT(40); /* CONFIG_ENABLE_SNARF_CPM */
/*
* V100 GPUs are known to violate NVLink2 protocol if some GPU memory
* mapped by a CPU was also "linear-block" mapped by a GPU. When this
* happens, it breaks the NPU2 cache coherency state machine and
* it throws machine checkstop. Disabling snarfing fixes this so let's
* disable it by default.
*/
if (nvram_query_eq_dangerous("opal-npu2-snarf-cpm", "enable")) {
prlog(PR_WARNING, "NPU2#%d: enabling Probe.I.MO snarfing, a bad GPU driver may crash the system!\n",
npu->index);
val |= PPC_BIT(40); /* CONFIG_ENABLE_SNARF_CPM */
}
xscom_write_mask(npu->chip_id, NPU_STCK0_CS_SM0_MISC_CONFIG0,
val, mask);
xscom_write_mask(npu->chip_id, NPU_STCK0_CS_SM1_MISC_CONFIG0,
val, mask);
xscom_write_mask(npu->chip_id, NPU_STCK0_CS_SM2_MISC_CONFIG0,
val, mask);
xscom_write_mask(npu->chip_id, NPU_STCK0_CS_SM3_MISC_CONFIG0,
val, mask);
xscom_write_mask(npu->chip_id, NPU_STCK1_CS_SM0_MISC_CONFIG0,
val, mask);
xscom_write_mask(npu->chip_id, NPU_STCK1_CS_SM1_MISC_CONFIG0,
val, mask);
xscom_write_mask(npu->chip_id, NPU_STCK1_CS_SM2_MISC_CONFIG0,
val, mask);
xscom_write_mask(npu->chip_id, NPU_STCK1_CS_SM3_MISC_CONFIG0,
val, mask);
xscom_write_mask(npu->chip_id, NPU_STCK2_CS_SM0_MISC_CONFIG0,
val, mask);
xscom_write_mask(npu->chip_id, NPU_STCK2_CS_SM1_MISC_CONFIG0,
val, mask);
xscom_write_mask(npu->chip_id, NPU_STCK2_CS_SM2_MISC_CONFIG0,
val, mask);
xscom_write_mask(npu->chip_id, NPU_STCK2_CS_SM3_MISC_CONFIG0,
val, mask);
xscom_write_mask(npu->chip_id, 0x50110c0, PPC_BIT(53), PPC_BIT(53));
xscom_write_mask(npu->chip_id, 0x50112c0, PPC_BIT(53), PPC_BIT(53));
xscom_write_mask(npu->chip_id, 0x50114c0, PPC_BIT(53), PPC_BIT(53));
xscom_write_mask(npu->chip_id, 0x50110f1, PPC_BIT(41), PPC_BIT(41));
xscom_write_mask(npu->chip_id, 0x50112f1, PPC_BIT(41), PPC_BIT(41));
xscom_write_mask(npu->chip_id, 0x50114f1, PPC_BIT(41), PPC_BIT(41));
val = NPU2_NTL_MISC_CFG2_BRICK_ENABLE |
NPU2_NTL_MISC_CFG2_NDL_TX_PARITY_ENA |
NPU2_NTL_MISC_CFG2_NDL_PRI_PARITY_ENA |
NPU2_NTL_MISC_CFG2_RCV_CREDIT_OVERFLOW_ENA;
xscom_write_mask(npu->chip_id, 0x5011110, val, val);
xscom_write_mask(npu->chip_id, 0x5011130, val, val);
xscom_write_mask(npu->chip_id, 0x5011310, val, val);
xscom_write_mask(npu->chip_id, 0x5011330, val, val);
xscom_write_mask(npu->chip_id, 0x5011510, val, val);
xscom_write_mask(npu->chip_id, 0x5011530, val, val);
val = PPC_BIT(6) | PPC_BIT(7) | PPC_BIT(11);
xscom_write_mask(npu->chip_id, 0x5011009, val, PPC_BITMASK(6,11));
xscom_write_mask(npu->chip_id, 0x5011039, val, PPC_BITMASK(6,11));
xscom_write_mask(npu->chip_id, 0x5011069, val, PPC_BITMASK(6,11));
xscom_write_mask(npu->chip_id, 0x5011099, val, PPC_BITMASK(6,11));
xscom_write_mask(npu->chip_id, 0x5011209, val, PPC_BITMASK(6,11));
xscom_write_mask(npu->chip_id, 0x5011239, val, PPC_BITMASK(6,11));
xscom_write_mask(npu->chip_id, 0x5011269, val, PPC_BITMASK(6,11));
xscom_write_mask(npu->chip_id, 0x5011299, val, PPC_BITMASK(6,11));
xscom_write_mask(npu->chip_id, 0x5011409, val, PPC_BITMASK(6,11));
xscom_write_mask(npu->chip_id, 0x5011439, val, PPC_BITMASK(6,11));
xscom_write_mask(npu->chip_id, 0x5011469, val, PPC_BITMASK(6,11));
xscom_write_mask(npu->chip_id, 0x5011499, val, PPC_BITMASK(6,11));
/* Reassign the BARs */
assign_mmio_bars(npu->chip_id, npu->xscom_base, reg, mm_win);
npu->regs = (void *)reg[0];
npu->mm_base = mm_win[0];
npu->mm_size = mm_win[1];
if (reg[0] && reg[1])
prlog(PR_INFO, " Global MMIO BAR: %016llx (%lldMB)\n",
reg[0], reg[1] >> 20);
else
prlog(PR_ERR, " Global MMIO BAR: Disabled\n");
/* Populate PCI root device node */
np = dt_new_addr(dt_root, "pciex", reg[0]);
assert(np);
dt_add_property_strings(np,
"compatible",
"ibm,power9-npu-pciex",
"ibm,ioda2-npu2-phb");
dt_add_property_strings(np, "device_type", "pciex");
dt_add_property(np, "reg", reg, sizeof(reg));
dt_add_property_cells(np, "ibm,phb-index", npu2_get_phb_index(0));
dt_add_property_cells(np, "ibm,npu-index", npu->index);
dt_add_property_cells(np, "ibm,chip-id", npu->chip_id);
dt_add_property_cells(np, "ibm,xscom-base", npu->xscom_base);
dt_add_property_cells(np, "ibm,npcq", npu->dt_node->phandle);
dt_add_property_cells(np, "ibm,links", npu->total_devices);
dt_add_property(np, "ibm,mmio-window", mm_win, sizeof(mm_win));
dt_add_property_cells(np, "ibm,phb-diag-data-size", 0);
/* Disable fast reboot - not currently supported */
disable_fast_reboot("NVLink device enabled");
npu2_nvlink_create_phb(npu, np);
return 0;
}
static uint32_t npu2_populate_pcie_cap(struct npu2_dev *dev,
uint32_t start,
uint32_t prev_cap)
{
struct pci_virt_device *pvd = dev->nvlink.pvd;
uint32_t val;
/* Add capability list */
PCI_VIRT_CFG_INIT_RO(pvd, prev_cap, 1, start);
PCI_VIRT_CFG_INIT_RO(pvd, start, 1, PCI_CFG_CAP_ID_EXP);
/* 0x00 - ID/PCIE capability */
val = PCI_CFG_CAP_ID_EXP;
val |= ((0x2 << 16) | (PCIE_TYPE_ENDPOINT << 20));
PCI_VIRT_CFG_INIT_RO(pvd, start, 4, val);
/* 0x04 - Device capability
*
* We should support FLR. Otherwise, it might have
* problem passing it through to userland via Linux
* VFIO infrastructure
*/
val = ((PCIE_MPSS_128) |
(PCIE_PHANTOM_NONE << 3) |
(PCIE_L0SL_MAX_NO_LIMIT << 6) |
(PCIE_L1L_MAX_NO_LIMIT << 9) |
(PCICAP_EXP_DEVCAP_FUNC_RESET));
PCI_VIRT_CFG_INIT_RO(pvd, start + PCICAP_EXP_DEVCAP, 4, val);
pci_virt_add_filter(pvd, start + PCICAP_EXP_DEVCTL, 2,
PCI_REG_FLAG_WRITE,
npu2_dev_cfg_exp_devcap, NULL);
/* 0x08 - Device control and status */
PCI_VIRT_CFG_INIT(pvd, start + PCICAP_EXP_DEVCTL, 4, 0x00002810,
0xffff0000, 0x000f0000);
/* 0x0c - Link capability */
val = (PCIE_LSPEED_VECBIT_2 | (PCIE_LWIDTH_1X << 4));
PCI_VIRT_CFG_INIT_RO(pvd, start + PCICAP_EXP_LCAP, 4, val);
/* 0x10 - Link control and status */
PCI_VIRT_CFG_INIT(pvd, start + PCICAP_EXP_LCTL, 4, 0x00130000,
0xfffff000, 0xc0000000);
/* 0x14 - Slot capability */
PCI_VIRT_CFG_INIT_RO(pvd, start + PCICAP_EXP_SLOTCAP, 4, 0x00000000);
/* 0x18 - Slot control and status */
PCI_VIRT_CFG_INIT_RO(pvd, start + PCICAP_EXP_SLOTCTL, 4, 0x00000000);
/* 0x1c - Root control and capability */
PCI_VIRT_CFG_INIT(pvd, start + PCICAP_EXP_RC, 4, 0x00000000,
0xffffffe0, 0x00000000);
/* 0x20 - Root status */
PCI_VIRT_CFG_INIT(pvd, start + PCICAP_EXP_RSTAT, 4, 0x00000000,
0xffffffff, 0x00010000);
/* 0x24 - Device capability 2 */
PCI_VIRT_CFG_INIT_RO(pvd, start + PCIECAP_EXP_DCAP2, 4, 0x00000000);
/* 0x28 - Device Control and status 2 */
PCI_VIRT_CFG_INIT(pvd, start + PCICAP_EXP_DCTL2, 4, 0x00070000,
0xffff0000, 0x00000000);
/* 0x2c - Link capability 2 */
PCI_VIRT_CFG_INIT_RO(pvd, start + PCICAP_EXP_LCAP2, 4, 0x00000007);
/* 0x30 - Link control and status 2 */
PCI_VIRT_CFG_INIT(pvd, start + PCICAP_EXP_LCTL2, 4, 0x00000003,
0xffff0000, 0x00200000);
/* 0x34 - Slot capability 2 */
PCI_VIRT_CFG_INIT_RO(pvd, start + PCICAP_EXP_SCAP2, 4, 0x00000000);
/* 0x38 - Slot control and status 2 */
PCI_VIRT_CFG_INIT_RO(pvd, start + PCICAP_EXP_SCTL2, 4, 0x00000000);
return start + PCICAP_EXP_SCTL2 + 8;
}
static uint32_t npu2_populate_vendor_cap(struct npu2_dev *dev,
uint32_t start,
uint32_t prev_cap)
{
struct pci_virt_device *pvd = dev->nvlink.pvd;
/* Capbility list */
PCI_VIRT_CFG_INIT_RO(pvd, prev_cap, 1, start);
PCI_VIRT_CFG_INIT_RO(pvd, start, 1, PCI_CFG_CAP_ID_VENDOR);
/* Length and version */
PCI_VIRT_CFG_INIT_RO(pvd, start + 2, 1, VENDOR_CAP_LEN);
PCI_VIRT_CFG_INIT_RO(pvd, start + 3, 1, VENDOR_CAP_VERSION);
/*
* Defaults when the trap can't handle the read/write (eg. due
* to reading/writing less than 4 bytes).
*/
PCI_VIRT_CFG_INIT_RO(pvd, start + 4, 4, 0);
PCI_VIRT_CFG_INIT_RO(pvd, start + 8, 4, 0);
/* Add NVLink2 PHY procedures trap */
pci_virt_add_filter(pvd, start + 4, 8,
PCI_REG_FLAG_READ | PCI_REG_FLAG_WRITE,
npu2_dev_procedure,
NULL);
/* Link index */
PCI_VIRT_CFG_INIT_RO(pvd, start + 0xc, 1, dev->link_index);
return start + VENDOR_CAP_LEN;
}
static void npu2_populate_cfg(struct npu2_dev *dev)
{
struct pci_virt_device *pvd = dev->nvlink.pvd;
struct npu2_pcie_bar *bar;
uint32_t pos;
/* 0x00 - Vendor/Device ID */
PCI_VIRT_CFG_INIT_RO(pvd, PCI_CFG_VENDOR_ID, 4, 0x04ea1014);
/* 0x04 - Command/Status */
PCI_VIRT_CFG_INIT(pvd, PCI_CFG_CMD, 4, 0x00100000, 0xffb802b8,
0xf9000000);
pci_virt_add_filter(pvd, PCI_CFG_CMD, 1, PCI_REG_FLAG_WRITE,
npu2_cfg_write_cmd, NULL);
/* 0x08 - Rev/Class/Cache */
PCI_VIRT_CFG_INIT_RO(pvd, PCI_CFG_REV_ID, 4, 0x06800101);
/* 0x0c - CLS/Latency Timer/Header/BIST */
PCI_VIRT_CFG_INIT_RO(pvd, PCI_CFG_CACHE_LINE_SIZE, 4, 0x00800000);
/* 0x10/14 - BAR#0, NTL BAR */
bar = &dev->bars[0];
PCI_VIRT_CFG_INIT(pvd, PCI_CFG_BAR0, 4,
(bar->npu2_bar.base & 0xfffffff0) | (bar->flags & 0xF),
0x0000000f, 0x00000000);
PCI_VIRT_CFG_INIT(pvd, PCI_CFG_BAR1, 4, (bar->npu2_bar.base >> 32),
0x00000000, 0x00000000);
pci_virt_add_filter(pvd, PCI_CFG_BAR0, 8,
PCI_REG_FLAG_READ | PCI_REG_FLAG_WRITE,
npu2_dev_cfg_bar, bar);
/* 0x18/1c - BAR#1, GENID BAR */
bar = &dev->bars[1];
if (NPU2DEV_BRICK(dev) == 0)
PCI_VIRT_CFG_INIT(pvd, PCI_CFG_BAR2, 4, (bar->npu2_bar.base & 0xfffffff0) |
(bar->flags & 0xF),
0x0000000f, 0x00000000);
else
/* Brick 1 gets the upper portion of the generation id register */
PCI_VIRT_CFG_INIT(pvd, PCI_CFG_BAR2, 4, ((bar->npu2_bar.base + 0x10000) & 0xfffffff0) |
(bar->flags & 0xF),
0x0000000f, 0x00000000);
PCI_VIRT_CFG_INIT(pvd, PCI_CFG_BAR3, 4, (bar->npu2_bar.base >> 32), 0x00000000,
0x00000000);
pci_virt_add_filter(pvd, PCI_CFG_BAR2, 8,
PCI_REG_FLAG_READ | PCI_REG_FLAG_WRITE,
npu2_dev_cfg_bar, bar);
/* 0x20/0x24 - BARs, disabled */
PCI_VIRT_CFG_INIT_RO(pvd, PCI_CFG_BAR4, 4, 0x00000000);
PCI_VIRT_CFG_INIT_RO(pvd, PCI_CFG_BAR5, 4, 0x00000000);
/* 0x28 - Cardbus CIS pointer */
PCI_VIRT_CFG_INIT_RO(pvd, PCI_CFG_CARDBUS_CIS, 4, 0x00000000);
/* 0x2c - Subsystem ID */
PCI_VIRT_CFG_INIT_RO(pvd, PCI_CFG_SUBSYS_VENDOR_ID, 4, 0x00000000);
/* 0x30 - ROM BAR, zero sized */
PCI_VIRT_CFG_INIT_RO(pvd, PCI_CFG_ROMBAR, 4, 0xffffffff);
/* 0x34 - PCI Capability */
PCI_VIRT_CFG_INIT_RO(pvd, PCI_CFG_CAP, 4, 0x00000000);
/* 0x38 - Reserved */
PCI_VIRT_CFG_INIT_RO(pvd, 0x38, 4, 0x00000000);
/* 0x3c - INT line/pin/Minimal grant/Maximal latency */
PCI_VIRT_CFG_INIT_RO(pvd, PCI_CFG_INT_LINE, 4, 0x00000100); /* INT A */
/* PCIE and vendor specific capability */
pos = npu2_populate_pcie_cap(dev, 0x40, PCI_CFG_CAP);
pos = npu2_populate_vendor_cap(dev, pos, 0x41);
PCI_VIRT_CFG_INIT_RO(pvd, pos + 1, 1, 0);
}
static uint32_t npu_allocate_bdfn(struct npu2 *p, uint32_t group)
{
int i;
int bdfn = (group << 3);
for (i = 0; i < p->total_devices; i++) {
if ((p->devices[i].bdfn & 0xf8) == (bdfn & 0xf8))
bdfn++;
}
return bdfn;
}
static void npu2_populate_devices(struct npu2 *p,
struct dt_node *dn)
{
struct npu2_dev *dev;
struct dt_node *npu2_dn, *link;
uint32_t npu_phandle, index = 0;
int stack;
/*
* Get the npu node which has the links which we expand here
* into pci like devices attached to our emulated phb.
*/
npu_phandle = dt_prop_get_u32(dn, "ibm,npcq");
npu2_dn = dt_find_by_phandle(dt_root, npu_phandle);
assert(npu2_dn);
/* Walk the link@x nodes to initialize devices */
p->total_devices = 0;
p->phb_nvlink.scan_map = 0;
dt_for_each_compatible(npu2_dn, link, "ibm,npu-link") {
uint32_t group_id;
struct npu2_bar *npu2_bar;
dev = &p->devices[index];
dev->type = NPU2_DEV_TYPE_NVLINK;
dev->npu = p;
dev->dt_node = link;
dev->link_index = dt_prop_get_u32(link, "ibm,npu-link-index");
dev->brick_index = dev->link_index;
group_id = dt_prop_get_u32(link, "ibm,npu-group-id");
dev->bdfn = npu_allocate_bdfn(p, group_id);
/* This must be done after calling
* npu_allocate_bdfn() */
p->total_devices++;
p->phb_nvlink.scan_map |= 0x1 << ((dev->bdfn & 0xf8) >> 3);
dev->pl_xscom_base = dt_prop_get_u64(link, "ibm,npu-phy");
dev->lane_mask = dt_prop_get_u32(link, "ibm,npu-lane-mask");
/* Populate BARs. BAR0/1 is the NTL bar. */
stack = NPU2_STACK_STCK_0 + NPU2DEV_STACK(dev);
npu2_bar = &dev->bars[0].npu2_bar;
npu2_bar->type = NPU_NTL;
npu2_bar->index = dev->brick_index;
npu2_bar->reg = NPU2_REG_OFFSET(stack, 0, NPU2DEV_BRICK(dev) == 0 ?
NPU2_NTL0_BAR : NPU2_NTL1_BAR);
npu2_get_bar(p->chip_id, npu2_bar);
dev->bars[0].flags = PCI_CFG_BAR_TYPE_MEM | PCI_CFG_BAR_MEM64;
/* BAR2/3 is the GENID bar. */
npu2_bar = &dev->bars[1].npu2_bar;
npu2_bar->type = NPU_GENID;
npu2_bar->index = NPU2DEV_STACK(dev);
npu2_bar->reg = NPU2_REG_OFFSET(stack, 0, NPU2_GENID_BAR);
npu2_get_bar(p->chip_id, npu2_bar);
/* The GENID is a single physical BAR that we split
* for each emulated device */
npu2_bar->size = 0x10000;
if (NPU2DEV_BRICK(dev))
npu2_bar->base += 0x10000;
dev->bars[1].flags = PCI_CFG_BAR_TYPE_MEM | PCI_CFG_BAR_MEM64;
/* Initialize PCI virtual device */
dev->nvlink.pvd = pci_virt_add_device(&p->phb_nvlink, dev->bdfn, 0x100, dev);
if (dev->nvlink.pvd)
npu2_populate_cfg(dev);
index++;
}
}
static void npu2_add_interrupt_map(struct npu2 *p,
struct dt_node *dn)
{
struct dt_node *npu2_dn, *link, *phb_dn;
uint32_t npu2_phandle, index = 0, i;
uint32_t icsp = get_ics_phandle();
uint32_t *map;
size_t map_size;
uint32_t mask[] = {0xff00, 0x0, 0x0, 0x7};
assert(p->phb_nvlink.dt_node);
phb_dn = p->phb_nvlink.dt_node;
npu2_phandle = dt_prop_get_u32(dn, "ibm,npcq");
npu2_dn = dt_find_by_phandle(dt_root, npu2_phandle);
assert(npu2_dn);
map_size = 7 * sizeof(*map) * p->total_devices;
map = malloc(map_size);
index = 0;
dt_for_each_compatible(npu2_dn, link, "ibm,npu-link") {
i = index * 7;
map[i + 0] = (p->devices[index].bdfn << 8);
map[i + 1] = 0;
map[i + 2] = 0;
map[i + 3] = 1; /* INT A */
map[i + 4] = icsp; /* interrupt-parent */
map[i + 5] = p->base_lsi + (index * 2) + 1; /* NDL No-Stall Event */
map[i + 6] = 0; /* 0 = EDGE, 1 = LEVEL. */
index++;
}
dt_add_property(phb_dn, "interrupt-map", map, map_size);
free(map);
dt_add_property(phb_dn, "interrupt-map-mask", mask, sizeof(mask));
}
static void npu2_add_phb_properties(struct npu2 *p)
{
struct dt_node *np = p->phb_nvlink.dt_node;
uint32_t icsp = get_ics_phandle();
uint64_t mm_base, mm_size;
/*
* Add various properties that HB doesn't have to
* add, some of them simply because they result from
* policy decisions made in skiboot rather than in HB
* such as the MMIO windows going to PCI, interrupts,
* etc.
*/
dt_add_property_cells(np, "#address-cells", 3);
dt_add_property_cells(np, "#size-cells", 2);
dt_add_property_cells(np, "#interrupt-cells", 1);
dt_add_property_cells(np, "bus-range", 0, 0xff);
dt_add_property_cells(np, "clock-frequency", 0x200, 0);
dt_add_property_cells(np, "interrupt-parent", icsp);
/* NPU2 PHB properties */
dt_add_property_cells(np, "ibm,opal-num-pes",
NPU2_MAX_PE_NUM);
dt_add_property_cells(np, "ibm,opal-reserved-pe",
NPU2_RESERVED_PE_NUM);
dt_add_property_cells(np, "ibm,supported-tce-sizes",
12, // 4K
16, // 64K
24, // 16M
28); // 256M
dt_add_property_u64s(np, "ibm,mmio-atsd",
MMIO_ATSD_ADDR(p->regs, 0),
MMIO_ATSD_ADDR(p->regs, 1),
MMIO_ATSD_ADDR(p->regs, 2),
MMIO_ATSD_ADDR(p->regs, 3),
MMIO_ATSD_ADDR(p->regs, 4),
MMIO_ATSD_ADDR(p->regs, 5),
MMIO_ATSD_ADDR(p->regs, 6),
MMIO_ATSD_ADDR(p->regs, 7));
/*
* Memory window is exposed as 64-bits non-prefetchable
* one because 64-bits prefetchable one is kind of special
* to kernel.
*/
mm_base = p->mm_base;
mm_size = p->mm_size;
dt_add_property_cells(np, "ranges", 0x02000000,
hi32(mm_base), lo32(mm_base),
hi32(mm_base), lo32(mm_base),
hi32(mm_size), lo32(mm_size));
}
void npu2_nvlink_create_phb(struct npu2 *npu, struct dt_node *dn)
{
struct pci_slot *slot;
/* Generic PHB */
npu->phb_nvlink.dt_node = dn;
npu->phb_nvlink.ops = &npu_ops;
npu->phb_nvlink.phb_type = phb_type_npu_v2;
init_lock(&npu->lock);
init_lock(&npu->phb_nvlink.lock);
list_head_init(&npu->phb_nvlink.devices);
list_head_init(&npu->phb_nvlink.virt_devices);
npu2_populate_devices(npu, dn);
npu2_add_interrupt_map(npu, dn);
npu2_add_phb_properties(npu);
slot = npu2_slot_create(&npu->phb_nvlink);
if (!slot)
{
/**
* @fwts-label NPUCannotCreatePHBSlot
* @fwts-advice Firmware probably ran out of memory creating
* NPU2 slot. NVLink functionality could be broken.
*/
prlog(PR_ERR, "NPU: Cannot create PHB slot\n");
}
pci_register_phb(&npu->phb_nvlink, OPAL_DYNAMIC_PHB_ID);
npu2_init_ioda_cache(npu);
npu2_hw_init(npu);
}
/*
* Search a table for an entry with matching value under mask. Returns
* the index and the current value in *value.
*/
static int npu_table_search(struct npu2 *p, uint64_t table_addr, int stride,
int table_size, uint64_t *value, uint64_t mask)
{
int i;
uint64_t val;
assert(value);
for (i = 0; i < table_size; i++) {
val = npu2_read(p, table_addr + i*stride);
if ((val & mask) == *value) {
*value = val;
return i;
}
}
return -1;
}
/*
* Allocate a context ID and initialise the tables with the relevant
* information. Returns the ID on or error if one couldn't be
* allocated.
*/
#define NPU2_VALID_ATS_MSR_BITS (MSR_DR | MSR_HV | MSR_PR | MSR_SF)
int64_t npu2_init_context(struct phb *phb, uint64_t msr, uint64_t bdf)
{
struct npu2 *p;
uint64_t xts_bdf, old_xts_bdf_pid, xts_bdf_pid;
int id;
/*
* MSR bits should be masked by the caller to allow for future
* expansion if required.
*/
if (msr & ~NPU2_VALID_ATS_MSR_BITS)
return OPAL_UNSUPPORTED;
/*
* Need to get LPARSHORT.
*/
p = phb_to_npu2_nvlink(phb);
lock(&p->lock);
xts_bdf = SETFIELD(NPU2_XTS_BDF_MAP_BDF, 0ul, bdf);
if (npu_table_search(p, NPU2_XTS_BDF_MAP, 8, NPU2_XTS_BDF_MAP_SIZE,
&xts_bdf, NPU2_XTS_BDF_MAP_BDF) < 0) {
NPU2ERR(p, "LPARID not associated with any GPU\n");
id = OPAL_PARAMETER;
goto out;
}
id = GETFIELD(NPU2_XTS_BDF_MAP_LPARSHORT, xts_bdf);
NPU2DBG(p, "Found LPARSHORT = 0x%x for BDF = 0x%03llx\n", id, bdf);
/* Enable this mapping for both real and virtual addresses */
xts_bdf_pid = SETFIELD(NPU2_XTS_PID_MAP_VALID_ATRGPA0, 0UL, 1);
xts_bdf_pid = SETFIELD(NPU2_XTS_PID_MAP_VALID_ATRGPA1, xts_bdf_pid, 1);
/* Enables TLBIE/MMIOSD forwarding for this entry */
xts_bdf_pid = SETFIELD(NPU2_XTS_PID_MAP_VALID_ATSD, xts_bdf_pid, 1);
xts_bdf_pid = SETFIELD(NPU2_XTS_PID_MAP_LPARSHORT, xts_bdf_pid, id);
/* Set the relevant MSR bits */
xts_bdf_pid = SETFIELD(NPU2_XTS_PID_MAP_MSR_DR, xts_bdf_pid,
!!(msr & MSR_DR));
xts_bdf_pid = SETFIELD(NPU2_XTS_PID_MAP_MSR_HV, xts_bdf_pid,
!!(msr & MSR_HV));
xts_bdf_pid = SETFIELD(NPU2_XTS_PID_MAP_MSR_PR, xts_bdf_pid,
!!(msr & MSR_PR));
/* We don't support anything other than 64-bit so we can safely hardcode
* it here */
xts_bdf_pid = SETFIELD(NPU2_XTS_PID_MAP_MSR_SF, xts_bdf_pid, 1);
/*
* Throw an error if the wildcard entry for this bdf is already set
* with different msr bits.
*/
old_xts_bdf_pid = npu2_read(p, NPU2_XTS_PID_MAP + id*0x20);
if (old_xts_bdf_pid) {
if (GETFIELD(NPU2_XTS_PID_MAP_MSR, old_xts_bdf_pid) !=
GETFIELD(NPU2_XTS_PID_MAP_MSR, xts_bdf_pid)) {
NPU2ERR(p, "%s: Unexpected MSR value\n", __func__);
id = OPAL_PARAMETER;
goto out;
} else if (!p->ctx_ref[id]) {
NPU2ERR(p, "%s: Unexpected mapping\n", __func__);
id = OPAL_INTERNAL_ERROR;
goto out;
}
}
/* Write the entry */
if (!p->ctx_ref[id]) {
NPU2DBG(p, "XTS_PID_MAP[%03d] = 0x%08llx\n", id, xts_bdf_pid);
npu2_write(p, NPU2_XTS_PID_MAP + id*0x20, xts_bdf_pid);
if (!GETFIELD(NPU2_XTS_BDF_MAP_VALID, xts_bdf)) {
xts_bdf = SETFIELD(NPU2_XTS_BDF_MAP_VALID, xts_bdf, 1);
npu2_write(p, NPU2_XTS_BDF_MAP + id*8, xts_bdf);
}
}
++p->ctx_ref[id];
out:
unlock(&p->lock);
return id;
}
int64_t npu2_destroy_context(struct phb *phb, uint64_t bdf)
{
struct npu2 *p;
uint64_t xts_bdf;
int rc = OPAL_PARAMETER, id;
p = phb_to_npu2_nvlink(phb);
lock(&p->lock);
/* Need to find lparshort for this bdf */
xts_bdf = SETFIELD(NPU2_XTS_BDF_MAP_BDF, 0ul, bdf);
if (npu_table_search(p, NPU2_XTS_BDF_MAP, 8, NPU2_XTS_BDF_MAP_SIZE,
&xts_bdf, NPU2_XTS_BDF_MAP_BDF) < 0) {
NPU2ERR(p, "LPARID not associated with any GPU\n");
} else {
/*
* The bdf/pid table contains wildcard entries and MSR bits
* which we need to clear between switching a device from
* a host to a guest or vice versa.
*/
id = GETFIELD(NPU2_XTS_BDF_MAP_LPARSHORT, xts_bdf);
if (p->ctx_ref[id]) {
--p->ctx_ref[id];
if (!p->ctx_ref[id]) {
NPU2DBG(p, "XTS_PID_MAP[%03d] = 0 (destroy)\n",
id);
npu2_write(p, NPU2_XTS_PID_MAP + id*0x20, 0);
}
rc = OPAL_SUCCESS;
}
}
unlock(&p->lock);
return rc;
}
/*
* Map the given virtual bdf to lparid with given lpcr.
*/
int64_t npu2_map_lpar(struct phb *phb, uint64_t bdf, uint64_t lparid,
uint64_t lpcr)
{
struct npu2 *p;
struct npu2_dev *ndev = NULL;
uint64_t xts_bdf_lpar, atsd_lpar, rc = OPAL_SUCCESS;
int i;
int id;
static uint64_t atsd_lpar_regs[] = {
NPU2_XTS_MMIO_ATSD0_LPARID, NPU2_XTS_MMIO_ATSD1_LPARID,
NPU2_XTS_MMIO_ATSD2_LPARID, NPU2_XTS_MMIO_ATSD3_LPARID,
NPU2_XTS_MMIO_ATSD4_LPARID, NPU2_XTS_MMIO_ATSD5_LPARID,
NPU2_XTS_MMIO_ATSD6_LPARID, NPU2_XTS_MMIO_ATSD7_LPARID
};
if (lpcr)
/* The LPCR bits are only required for hash based ATS,
* which we don't currently support but may need to in
* future. */
return OPAL_UNSUPPORTED;
p = phb_to_npu2_nvlink(phb);
lock(&p->lock);
/* Find any existing entries and update them */
xts_bdf_lpar = SETFIELD(NPU2_XTS_BDF_MAP_BDF, 0L, bdf);
id = npu_table_search(p, NPU2_XTS_BDF_MAP, 8, NPU2_XTS_BDF_MAP_SIZE,
&xts_bdf_lpar, NPU2_XTS_BDF_MAP_BDF);
if (id < 0) {
/* No existing mapping found, find space for a new one */
xts_bdf_lpar = 0;
id = npu_table_search(p, NPU2_XTS_BDF_MAP, 8, NPU2_XTS_BDF_MAP_SIZE,
&xts_bdf_lpar, -1UL);
}
if (id < 0) {
/* Unable to find a free mapping */
NPU2ERR(p, "No free XTS_BDF[] entry\n");
rc = OPAL_RESOURCE;
goto out;
}
xts_bdf_lpar = SETFIELD(NPU2_XTS_BDF_MAP_UNFILT, 0UL, 1);
xts_bdf_lpar = SETFIELD(NPU2_XTS_BDF_MAP_BDF, xts_bdf_lpar, bdf);
/* We only support radix for the moment */
xts_bdf_lpar = SETFIELD(NPU2_XTS_BDF_MAP_XLAT, xts_bdf_lpar, 0x3);
xts_bdf_lpar = SETFIELD(NPU2_XTS_BDF_MAP_LPARID, xts_bdf_lpar, lparid);
xts_bdf_lpar = SETFIELD(NPU2_XTS_BDF_MAP_LPARSHORT, xts_bdf_lpar, id);
/* Need to find an NVLink to send the ATSDs for this device over */
for (i = 0; i < p->total_devices; i++) {
if (p->devices[i].nvlink.gpu_bdfn == bdf) {
ndev = &p->devices[i];
break;
}
}
if (!ndev) {
NPU2ERR(p, "Unable to find nvlink for bdf %llx\n", bdf);
rc = OPAL_PARAMETER;
goto out;
}
/*
* We need to allocate an ATSD per NVLink bridge if possible,
* use the ibm,npu-link-index property for that.
*/
atsd_lpar = SETFIELD(NPU2_XTS_MMIO_ATSD_LPARID, 0, lparid);
if (!lparid)
atsd_lpar = SETFIELD(NPU2_XTS_MMIO_ATSD_MSR_HV, atsd_lpar, 1);
if (ndev->link_index < ARRAY_SIZE(atsd_lpar_regs))
npu2_write(p, atsd_lpar_regs[ndev->link_index], atsd_lpar);
else
NPU2ERR(p, "Unable to assign ATSD for link index %u\n",
ndev->link_index);
xts_bdf_lpar = SETFIELD(NPU2_XTS_BDF_MAP_STACK, xts_bdf_lpar,
0x4 >> (ndev->brick_index / 2));
xts_bdf_lpar = SETFIELD(NPU2_XTS_BDF_MAP_BRICK, xts_bdf_lpar,
(ndev->brick_index % 2));
NPU2DBG(p, "XTS_BDF_MAP[%03d] = 0x%08llx\n", id, xts_bdf_lpar);
npu2_write(p, NPU2_XTS_BDF_MAP + id*8, xts_bdf_lpar);
/* Reset wildcard in the PID map and the refcounter */
if (npu2_read(p, NPU2_XTS_PID_MAP + id*0x20) || p->ctx_ref[id]) {
prlog(PR_INFO, "Resetting PID MAP for LPID %lld\n", lparid);
p->ctx_ref[id] = 0;
npu2_write(p, NPU2_XTS_PID_MAP + id*0x20, 0);
}
out:
unlock(&p->lock);
return rc;
}
static inline uint32_t npu2_relaxed_ordering_source_grpchp(uint32_t gcid)
{
if (gcid & ~0x1b)
return OPAL_PARAMETER;
/* Repack 0bGGGGCCC to 0bGGCC */
return ((gcid & 0x18) >> 1) | (gcid & 0x3);
}
static uint64_t npu2_relaxed_ordering_cfg_read(struct npu2_dev *ndev, int n)
{
uint64_t reg = NPU2_SM_REG_OFFSET(ndev, 0, NPU2_RELAXED_ORDERING_CFG(n));
return npu2_read(ndev->npu, reg);
}
static void npu2_relaxed_ordering_cfg_write(struct npu2_dev *ndev, int n,
uint64_t val)
{
uint64_t reg;
int sm;
/* Set every register on our stack */
for (sm = NPU2_BLOCK_SM_0; sm <= NPU2_BLOCK_SM_3; sm++) {
reg = NPU2_SM_REG_OFFSET(ndev, sm, NPU2_RELAXED_ORDERING_CFG(n));
npu2_write(ndev->npu, reg, val);
}
}
/*
* Parse the value of a relaxed ordering config register. Returns SOURCE0 or
* SOURCE1 register mask if relaxed ordering is set for the given chip/pec.
* Returns 0 if unset.
*/
static uint64_t npu2_relaxed_ordering_cfg_enabled(uint64_t val, uint32_t gcid,
int pec)
{
uint32_t src, grpchp;
uint64_t mask;
int i;
for (i = 0; i < 2; i++) {
mask = NPU2_RELAXED_ORDERING_SOURCE(i);
src = GETFIELD(mask, val);
if (!GETFIELD(NPU2_RELAXED_ORDERING_SOURCE_ENA, src))
continue;
if (GETFIELD(NPU2_RELAXED_ORDERING_SOURCE_PECSEL, src) != pec)
continue;
grpchp = GETFIELD(NPU2_RELAXED_ORDERING_SOURCE_GRPCHP, src);
if (grpchp == npu2_relaxed_ordering_source_grpchp(gcid))
return mask;
if (grpchp == 0xf) /* match all */
return mask;
}
return 0;
}
static int npu2_enable_relaxed_ordering(struct npu2_dev *ndev, uint32_t gcid,
int pec)
{
uint64_t val, mask;
uint32_t src;
int rc = OPAL_RESOURCE;
int i;
NPU2DEVINF(ndev, "Enabling relaxed ordering for PEC %d on chip %d\n", pec, gcid);
lock(&ndev->npu->lock);
for (i = 0; i < 2; i++) {
val = npu2_relaxed_ordering_cfg_read(ndev, i);
if (!npu2_relaxed_ordering_cfg_enabled(val, gcid, pec))
continue;
/* Already enabled */
rc = OPAL_SUCCESS;
goto out;
}
src = NPU2_RELAXED_ORDERING_SOURCE_WRENA |
NPU2_RELAXED_ORDERING_SOURCE_RDENA;
src = SETFIELD(NPU2_RELAXED_ORDERING_SOURCE_PECSEL, src, pec);
src = SETFIELD(NPU2_RELAXED_ORDERING_SOURCE_GRPCHP, src,
npu2_relaxed_ordering_source_grpchp(gcid));
src = SETFIELD(NPU2_RELAXED_ORDERING_SOURCE_WRMIN, src, 0);
src = SETFIELD(NPU2_RELAXED_ORDERING_SOURCE_WRMAX, src, 23);
src = SETFIELD(NPU2_RELAXED_ORDERING_SOURCE_RDMIN, src, 0);
src = SETFIELD(NPU2_RELAXED_ORDERING_SOURCE_RDMAX, src, 47);
/* Find somewhere to write this config */
for (i = 0; i < 2; i++) {
val = npu2_relaxed_ordering_cfg_read(ndev, i);
if (!GETFIELD(NPU2_RELAXED_ORDERING_SOURCE_ENA << 32, val))
mask = NPU2_RELAXED_ORDERING_SOURCE(0);
else if (!GETFIELD(NPU2_RELAXED_ORDERING_SOURCE_ENA, val))
mask = NPU2_RELAXED_ORDERING_SOURCE(1);
else
continue;
val = SETFIELD(mask, val, src);
npu2_relaxed_ordering_cfg_write(ndev, i, val);
rc = OPAL_SUCCESS;
break;
}
out:
unlock(&ndev->npu->lock);
return rc;
}
static void npu2_disable_relaxed_ordering(struct npu2_dev *ndev, uint32_t gcid,
int pec)
{
uint64_t val, mask;
int i;
NPU2DEVINF(ndev, "Disabling relaxed ordering for PEC %d on chip %d\n", pec, gcid);
lock(&ndev->npu->lock);
for (i = 0; i < 2; i++) {
val = npu2_relaxed_ordering_cfg_read(ndev, i);
mask = npu2_relaxed_ordering_cfg_enabled(val, gcid, pec);
if (!mask)
continue;
val = SETFIELD(mask, val, 0);
npu2_relaxed_ordering_cfg_write(ndev, i, val);
}
unlock(&ndev->npu->lock);
}
/*
* Enable or disable relaxed ordering on all nvlinks for a given PEC. May leave
* relaxed ordering partially enabled if there are insufficient HW resources to
* enable it on all links.
*/
int64_t npu2_set_relaxed_order(struct phb *phb, uint32_t gcid, int pec,
bool enable)
{
struct npu2 *npu = phb_to_npu2_nvlink(phb);
struct npu2_dev *ndev;
int64_t rc = OPAL_SUCCESS;
for (int i = 0; i < npu->total_devices; i++) {
ndev = &npu->devices[i];
if (enable)
rc = npu2_enable_relaxed_ordering(ndev, gcid, pec);
else
npu2_disable_relaxed_ordering(ndev, gcid, pec);
if (rc != OPAL_SUCCESS) {
NPU2DEVINF(ndev, "Insufficient resources to activate relaxed ordering mode\n");
return OPAL_RESOURCE;
}
}
return OPAL_SUCCESS;
}