numa: Enable numa for SGX EPC sections
The basic SGX did not enable numa for SGX EPC sections, which
result in all EPC sections located in numa node 0. This patch
enable SGX numa function in the guest and the EPC section can
work with RAM as one numa node.
The Guest kernel related log:
[ 0.009981] ACPI: SRAT: Node 0 PXM 0 [mem 0x180000000-0x183ffffff]
[ 0.009982] ACPI: SRAT: Node 1 PXM 1 [mem 0x184000000-0x185bfffff]
The SRAT table can normally show SGX EPC sections menory info in different
numa nodes.
The SGX EPC numa related command:
......
-m 4G,maxmem=20G \
-smp sockets=2,cores=2 \
-cpu host,+sgx-provisionkey \
-object memory-backend-ram,size=2G,host-nodes=0,policy=bind,id=node0 \
-object memory-backend-epc,id=mem0,size=64M,prealloc=on,host-nodes=0,policy=bind \
-numa node,nodeid=0,cpus=0-1,memdev=node0 \
-object memory-backend-ram,size=2G,host-nodes=1,policy=bind,id=node1 \
-object memory-backend-epc,id=mem1,size=28M,prealloc=on,host-nodes=1,policy=bind \
-numa node,nodeid=1,cpus=2-3,memdev=node1 \
-M sgx-epc.0.memdev=mem0,sgx-epc.0.node=0,sgx-epc.1.memdev=mem1,sgx-epc.1.node=1 \
......
Signed-off-by: Yang Zhong <yang.zhong@intel.com>
Message-Id: <20211101162009.62161-2-yang.zhong@intel.com>
Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>
diff --git a/hw/core/numa.c b/hw/core/numa.c
index e6050b2..1aa05dc 100644
--- a/hw/core/numa.c
+++ b/hw/core/numa.c
@@ -784,9 +784,8 @@
break;
case MEMORY_DEVICE_INFO_KIND_SGX_EPC:
se = value->u.sgx_epc.data;
- /* TODO: once we support numa, assign to right node */
- node_mem[0].node_mem += se->size;
- node_mem[0].node_plugged_mem += se->size;
+ node_mem[se->node].node_mem += se->size;
+ node_mem[se->node].node_plugged_mem = 0;
break;
default:
g_assert_not_reached();
diff --git a/hw/i386/acpi-build.c b/hw/i386/acpi-build.c
index a99c6e4..8383b83 100644
--- a/hw/i386/acpi-build.c
+++ b/hw/i386/acpi-build.c
@@ -2068,6 +2068,8 @@
nvdimm_build_srat(table_data);
}
+ sgx_epc_build_srat(table_data);
+
/*
* TODO: this part is not in ACPI spec and current linux kernel boots fine
* without these entries. But I recall there were issues the last time I
diff --git a/hw/i386/sgx-epc.c b/hw/i386/sgx-epc.c
index e508827..96b2940 100644
--- a/hw/i386/sgx-epc.c
+++ b/hw/i386/sgx-epc.c
@@ -21,6 +21,7 @@
static Property sgx_epc_properties[] = {
DEFINE_PROP_UINT64(SGX_EPC_ADDR_PROP, SGXEPCDevice, addr, 0),
+ DEFINE_PROP_UINT32(SGX_EPC_NUMA_NODE_PROP, SGXEPCDevice, node, 0),
DEFINE_PROP_LINK(SGX_EPC_MEMDEV_PROP, SGXEPCDevice, hostmem,
TYPE_MEMORY_BACKEND_EPC, HostMemoryBackendEpc *),
DEFINE_PROP_END_OF_LIST(),
@@ -139,6 +140,8 @@
se->memaddr = epc->addr;
se->size = object_property_get_uint(OBJECT(epc), SGX_EPC_SIZE_PROP,
NULL);
+ se->node = object_property_get_uint(OBJECT(epc), SGX_EPC_NUMA_NODE_PROP,
+ NULL);
se->memdev = object_get_canonical_path(OBJECT(epc->hostmem));
info->u.sgx_epc.data = se;
diff --git a/hw/i386/sgx-stub.c b/hw/i386/sgx-stub.c
index c9b379e..26833eb 100644
--- a/hw/i386/sgx-stub.c
+++ b/hw/i386/sgx-stub.c
@@ -6,6 +6,10 @@
#include "qapi/error.h"
#include "qapi/qapi-commands-misc-target.h"
+void sgx_epc_build_srat(GArray *table_data)
+{
+}
+
SGXInfo *qmp_query_sgx(Error **errp)
{
error_setg(errp, "SGX support is not compiled in");
diff --git a/hw/i386/sgx.c b/hw/i386/sgx.c
index 8fef3dd..d042999 100644
--- a/hw/i386/sgx.c
+++ b/hw/i386/sgx.c
@@ -23,6 +23,7 @@
#include "sysemu/hw_accel.h"
#include "sysemu/reset.h"
#include <sys/ioctl.h>
+#include "hw/acpi/aml-build.h"
#define SGX_MAX_EPC_SECTIONS 8
#define SGX_CPUID_EPC_INVALID 0x0
@@ -36,6 +37,46 @@
#define RETRY_NUM 2
+static int sgx_epc_device_list(Object *obj, void *opaque)
+{
+ GSList **list = opaque;
+
+ if (object_dynamic_cast(obj, TYPE_SGX_EPC)) {
+ *list = g_slist_append(*list, DEVICE(obj));
+ }
+
+ object_child_foreach(obj, sgx_epc_device_list, opaque);
+ return 0;
+}
+
+static GSList *sgx_epc_get_device_list(void)
+{
+ GSList *list = NULL;
+
+ object_child_foreach(qdev_get_machine(), sgx_epc_device_list, &list);
+ return list;
+}
+
+void sgx_epc_build_srat(GArray *table_data)
+{
+ GSList *device_list = sgx_epc_get_device_list();
+
+ for (; device_list; device_list = device_list->next) {
+ DeviceState *dev = device_list->data;
+ Object *obj = OBJECT(dev);
+ uint64_t addr, size;
+ int node;
+
+ node = object_property_get_uint(obj, SGX_EPC_NUMA_NODE_PROP,
+ &error_abort);
+ addr = object_property_get_uint(obj, SGX_EPC_ADDR_PROP, &error_abort);
+ size = object_property_get_uint(obj, SGX_EPC_SIZE_PROP, &error_abort);
+
+ build_srat_memory(table_data, addr, size, node, MEM_AFFINITY_ENABLED);
+ }
+ g_slist_free(device_list);
+}
+
static uint64_t sgx_calc_section_metric(uint64_t low, uint64_t high)
{
return (low & MAKE_64BIT_MASK(12, 20)) +
@@ -226,6 +267,9 @@
/* set the memdev link with memory backend */
object_property_parse(obj, SGX_EPC_MEMDEV_PROP, list->value->memdev,
&error_fatal);
+ /* set the numa node property for sgx epc object */
+ object_property_set_uint(obj, SGX_EPC_NUMA_NODE_PROP, list->value->node,
+ &error_fatal);
object_property_set_bool(obj, "realized", true, &error_fatal);
object_unref(obj);
}
diff --git a/include/hw/i386/sgx-epc.h b/include/hw/i386/sgx-epc.h
index a6a65be..581fac3 100644
--- a/include/hw/i386/sgx-epc.h
+++ b/include/hw/i386/sgx-epc.h
@@ -25,6 +25,7 @@
#define SGX_EPC_ADDR_PROP "addr"
#define SGX_EPC_SIZE_PROP "size"
#define SGX_EPC_MEMDEV_PROP "memdev"
+#define SGX_EPC_NUMA_NODE_PROP "node"
/**
* SGXEPCDevice:
@@ -38,6 +39,7 @@
/* public */
uint64_t addr;
+ uint32_t node;
HostMemoryBackendEpc *hostmem;
} SGXEPCDevice;
@@ -56,6 +58,7 @@
} SGXEPCState;
bool sgx_epc_get_section(int section_nr, uint64_t *addr, uint64_t *size);
+void sgx_epc_build_srat(GArray *table_data);
static inline uint64_t sgx_epc_above_4g_end(SGXEPCState *sgx_epc)
{
diff --git a/monitor/hmp-cmds.c b/monitor/hmp-cmds.c
index 9c91bf9..2669156 100644
--- a/monitor/hmp-cmds.c
+++ b/monitor/hmp-cmds.c
@@ -1810,6 +1810,7 @@
se->id ? se->id : "");
monitor_printf(mon, " memaddr: 0x%" PRIx64 "\n", se->memaddr);
monitor_printf(mon, " size: %" PRIu64 "\n", se->size);
+ monitor_printf(mon, " node: %" PRId64 "\n", se->node);
monitor_printf(mon, " memdev: %s\n", se->memdev);
break;
default:
diff --git a/qapi/machine.json b/qapi/machine.json
index f1839ac..edeab60 100644
--- a/qapi/machine.json
+++ b/qapi/machine.json
@@ -1207,12 +1207,15 @@
#
# @memdev: memory backend linked with device
#
+# @node: the numa node
+#
# Since: 6.2
##
{ 'struct': 'SgxEPCDeviceInfo',
'data': { '*id': 'str',
'memaddr': 'size',
'size': 'size',
+ 'node': 'int',
'memdev': 'str'
}
}
@@ -1285,10 +1288,15 @@
#
# @memdev: memory backend linked with device
#
+# @node: the numa node
+#
# Since: 6.2
##
{ 'struct': 'SgxEPC',
- 'data': { 'memdev': 'str' } }
+ 'data': { 'memdev': 'str',
+ 'node': 'int'
+ }
+}
##
# @SgxEPCProperties:
diff --git a/qemu-options.hx b/qemu-options.hx
index ae2c6db..489b58e 100644
--- a/qemu-options.hx
+++ b/qemu-options.hx
@@ -127,11 +127,11 @@
ERST
DEF("M", HAS_ARG, QEMU_OPTION_M,
- " sgx-epc.0.memdev=memid\n",
+ " sgx-epc.0.memdev=memid,sgx-epc.0.node=numaid\n",
QEMU_ARCH_ALL)
SRST
-``sgx-epc.0.memdev=@var{memid}``
+``sgx-epc.0.memdev=@var{memid},sgx-epc.0.node=@var{numaid}``
Define an SGX EPC section.
ERST
