s390x/pci: assign msix io region for each pci device

For efficiency we now assign one msix io region for each pci device
and provide it with the pointer to the zPCI device as opaque
parameter. In addition, we remove msix address space and add msix io
region as a subregion to the root memory region of pci device.

Signed-off-by: Yi Min Zhao <zyimin@linux.vnet.ibm.com>
Reviewed-by: Pierre Morel <pmorel@linux.vnet.ibm.com>
Signed-off-by: Christian Borntraeger <borntraeger@de.ibm.com>
diff --git a/hw/s390x/s390-pci-bus.c b/hw/s390x/s390-pci-bus.c
index ac8f06d..fe5fc4e 100644
--- a/hw/s390x/s390-pci-bus.c
+++ b/hw/s390x/s390-pci-bus.c
@@ -383,7 +383,6 @@
     uint64_t pte;
     uint32_t flags;
     S390PCIBusDevice *pbdev = container_of(iommu, S390PCIBusDevice, iommu_mr);
-    S390pciState *s;
     IOMMUTLBEntry ret = {
         .target_as = &address_space_memory,
         .iova = 0,
@@ -405,12 +404,10 @@
 
     DPRINTF("iommu trans addr 0x%" PRIx64 "\n", addr);
 
-    s = S390_PCI_HOST_BRIDGE(pci_device_root_bus(pbdev->pdev)->qbus.parent);
     /* s390 does not have an APIC mapped to main storage so we use
      * a separate AddressSpace only for msix notifications
      */
     if (addr == ZPCI_MSI_ADDR) {
-        ret.target_as = &s->msix_notify_as;
         ret.iova = addr;
         ret.translated_addr = addr;
         ret.addr_mask = 0xfff;
@@ -476,7 +473,7 @@
 static void s390_msi_ctrl_write(void *opaque, hwaddr addr, uint64_t data,
                                 unsigned int size)
 {
-    S390PCIBusDevice *pbdev;
+    S390PCIBusDevice *pbdev = opaque;
     uint32_t io_int_word;
     uint32_t idx = data >> ZPCI_MSI_VEC_BITS;
     uint32_t vec = data & ZPCI_MSI_VEC_MASK;
@@ -486,7 +483,6 @@
 
     DPRINTF("write_msix data 0x%" PRIx64 " idx %d vec 0x%x\n", data, idx, vec);
 
-    pbdev = s390_pci_find_dev_by_idx(idx);
     if (!pbdev) {
         e |= (vec << ERR_EVENT_MVN_OFFSET);
         s390_pci_generate_error_event(ERR_EVENT_NOMSI, idx, 0, addr, e);
@@ -548,10 +544,6 @@
 
         s->iommu[i] = iommu;
     }
-
-    memory_region_init_io(&s->msix_notify_mr, OBJECT(s),
-                          &s390_msi_ctrl_ops, s, "msix-s390", UINT64_MAX);
-    address_space_init(&s->msix_notify_as, &s->msix_notify_mr, "msix-pci");
 }
 
 static int s390_pcihost_init(SysBusDevice *dev)
@@ -581,7 +573,7 @@
     return 0;
 }
 
-static int s390_pcihost_setup_msix(S390PCIBusDevice *pbdev)
+static int s390_pci_setup_msix(S390PCIBusDevice *pbdev)
 {
     uint8_t pos;
     uint16_t ctrl;
@@ -609,6 +601,26 @@
     return 0;
 }
 
+static void s390_pci_msix_init(S390PCIBusDevice *pbdev)
+{
+    char *name;
+
+    name = g_strdup_printf("msix-s390-%04x", pbdev->uid);
+
+    memory_region_init_io(&pbdev->msix_notify_mr, OBJECT(pbdev),
+                          &s390_msi_ctrl_ops, pbdev, name, PAGE_SIZE);
+    memory_region_add_subregion(&pbdev->iommu->mr, ZPCI_MSI_ADDR,
+                                &pbdev->msix_notify_mr);
+
+    g_free(name);
+}
+
+static void s390_pci_msix_free(S390PCIBusDevice *pbdev)
+{
+    memory_region_del_subregion(&pbdev->iommu->mr, &pbdev->msix_notify_mr);
+    object_unparent(OBJECT(&pbdev->msix_notify_mr));
+}
+
 static S390PCIBusDevice *s390_pci_device_new(const char *target)
 {
     DeviceState *dev = NULL;
@@ -662,7 +674,9 @@
         pbdev->pdev = pdev;
         pbdev->iommu = s->iommu[PCI_SLOT(pdev->devfn)];
         pbdev->state = ZPCI_FS_STANDBY;
-        s390_pcihost_setup_msix(pbdev);
+
+        s390_pci_msix_init(pbdev);
+        s390_pci_setup_msix(pbdev);
 
         if (dev->hotplugged) {
             s390_pci_generate_plug_event(HP_EVENT_RESERVED_TO_STANDBY,
@@ -749,6 +763,7 @@
     s390_pci_generate_plug_event(HP_EVENT_STANDBY_TO_RESERVED,
                                  pbdev->fh, pbdev->fid);
     object_unparent(OBJECT(pci_dev));
+    s390_pci_msix_free(pbdev);
     pbdev->pdev = NULL;
     pbdev->state = ZPCI_FS_RESERVED;
 out:
diff --git a/hw/s390x/s390-pci-bus.h b/hw/s390x/s390-pci-bus.h
index 4f564e0..7f27013 100644
--- a/hw/s390x/s390-pci-bus.h
+++ b/hw/s390x/s390-pci-bus.h
@@ -82,6 +82,7 @@
 #define ZPCI_EDMA_ADDR 0x1ffffffffffffffULL
 
 #define PAGE_SHIFT      12
+#define PAGE_SIZE       (1 << PAGE_SHIFT)
 #define PAGE_MASK       (~(PAGE_SIZE-1))
 #define PAGE_DEFAULT_ACC        0
 #define PAGE_DEFAULT_KEY        (PAGE_DEFAULT_ACC << 4)
@@ -283,6 +284,7 @@
     AdapterRoutes routes;
     S390PCIIOMMU *iommu;
     MemoryRegion iommu_mr;
+    MemoryRegion msix_notify_mr;
     IndAddr *summary_ind;
     IndAddr *indicator;
     QEMUTimer *release_timer;
@@ -297,8 +299,6 @@
     S390PCIBus *bus;
     S390PCIBusDevice *pbdev[PCI_SLOT_MAX];
     S390PCIIOMMU *iommu[PCI_SLOT_MAX];
-    AddressSpace msix_notify_as;
-    MemoryRegion msix_notify_mr;
     QTAILQ_HEAD(, SeiContainer) pending_sei;
 } S390pciState;