/** @file | |
SMM CPU Sync lib implementation. | |
The lib provides 3 sets of APIs: | |
1. ContextInit/ContextDeinit/ContextReset: | |
ContextInit() is called in driver's entrypoint to allocate and initialize the SMM CPU Sync context. | |
ContextDeinit() is called in driver's unload function to deinitialize the SMM CPU Sync context. | |
ContextReset() is called by one of CPUs after all CPUs are ready to exit SMI, which allows CPU to | |
check into the next SMI from this point. | |
2. GetArrivedCpuCount/CheckInCpu/CheckOutCpu/LockDoor: | |
When SMI happens, all processors including BSP enter to SMM mode by calling CheckInCpu(). | |
CheckOutCpu() can be called in error handling flow for the CPU who calls CheckInCpu() earlier. | |
The elected BSP calls LockDoor() so that CheckInCpu() and CheckOutCpu() will return the error code after that. | |
GetArrivedCpuCount() returns the number of checked-in CPUs. | |
3. WaitForAPs/ReleaseOneAp/WaitForBsp/ReleaseBsp | |
WaitForAPs() & ReleaseOneAp() are called from BSP to wait the number of APs and release one specific AP. | |
WaitForBsp() & ReleaseBsp() are called from APs to wait and release BSP. | |
The 4 APIs are used to synchronize the running flow among BSP and APs. | |
BSP and AP Sync flow can be easy understand as below: | |
BSP: ReleaseOneAp --> AP: WaitForBsp | |
BSP: WaitForAPs <-- AP: ReleaseBsp | |
Copyright (c) 2023, Intel Corporation. All rights reserved.<BR> | |
SPDX-License-Identifier: BSD-2-Clause-Patent | |
**/ | |
#include <Library/BaseLib.h> | |
#include <Library/DebugLib.h> | |
#include <Library/MemoryAllocationLib.h> | |
#include <Library/SafeIntLib.h> | |
#include <Library/SmmCpuSyncLib.h> | |
#include <Library/SynchronizationLib.h> | |
#include <Uefi.h> | |
/// | |
/// The implementation shall place one semaphore on exclusive cache line for good performance. | |
/// | |
typedef volatile UINT32 SMM_CPU_SYNC_SEMAPHORE; | |
typedef struct { | |
/// | |
/// Used for control each CPU continue run or wait for signal | |
/// | |
SMM_CPU_SYNC_SEMAPHORE *Run; | |
} SMM_CPU_SYNC_SEMAPHORE_FOR_EACH_CPU; | |
struct SMM_CPU_SYNC_CONTEXT { | |
/// | |
/// Indicate all CPUs in the system. | |
/// | |
UINTN NumberOfCpus; | |
/// | |
/// Address of semaphores. | |
/// | |
VOID *SemBuffer; | |
/// | |
/// Size of semaphores. | |
/// | |
UINTN SemBufferPages; | |
/// | |
/// Before the door is locked, CpuCount stores the arrived CPU count. | |
/// After the door is locked, CpuCount is set to -1 indicating the door is locked. | |
/// ArrivedCpuCountUponLock stores the arrived CPU count then. | |
/// | |
UINTN ArrivedCpuCountUponLock; | |
/// | |
/// Indicate CPUs entered SMM before lock door. | |
/// | |
SMM_CPU_SYNC_SEMAPHORE *CpuCount; | |
/// | |
/// Define an array of structure for each CPU semaphore due to the size alignment | |
/// requirement. With the array of structure for each CPU semaphore, it's easy to | |
/// reach the specific CPU with CPU Index for its own semaphore access: CpuSem[CpuIndex]. | |
/// | |
SMM_CPU_SYNC_SEMAPHORE_FOR_EACH_CPU CpuSem[]; | |
}; | |
/** | |
Performs an atomic compare exchange operation to get semaphore. | |
The compare exchange operation must be performed using MP safe | |
mechanisms. | |
@param[in,out] Sem IN: 32-bit unsigned integer | |
OUT: original integer - 1 if Sem is not locked. | |
OUT: MAX_UINT32 if Sem is locked. | |
@retval Original integer - 1 if Sem is not locked. | |
MAX_UINT32 if Sem is locked. | |
**/ | |
STATIC | |
UINT32 | |
InternalWaitForSemaphore ( | |
IN OUT volatile UINT32 *Sem | |
) | |
{ | |
UINT32 Value; | |
for ( ; ;) { | |
Value = *Sem; | |
if (Value == MAX_UINT32) { | |
return Value; | |
} | |
if ((Value != 0) && | |
(InterlockedCompareExchange32 ( | |
(UINT32 *)Sem, | |
Value, | |
Value - 1 | |
) == Value)) | |
{ | |
break; | |
} | |
CpuPause (); | |
} | |
return Value - 1; | |
} | |
/** | |
Performs an atomic compare exchange operation to release semaphore. | |
The compare exchange operation must be performed using MP safe | |
mechanisms. | |
@param[in,out] Sem IN: 32-bit unsigned integer | |
OUT: original integer + 1 if Sem is not locked. | |
OUT: MAX_UINT32 if Sem is locked. | |
@retval Original integer + 1 if Sem is not locked. | |
MAX_UINT32 if Sem is locked. | |
**/ | |
STATIC | |
UINT32 | |
InternalReleaseSemaphore ( | |
IN OUT volatile UINT32 *Sem | |
) | |
{ | |
UINT32 Value; | |
do { | |
Value = *Sem; | |
} while (Value + 1 != 0 && | |
InterlockedCompareExchange32 ( | |
(UINT32 *)Sem, | |
Value, | |
Value + 1 | |
) != Value); | |
if (Value == MAX_UINT32) { | |
return Value; | |
} | |
return Value + 1; | |
} | |
/** | |
Performs an atomic compare exchange operation to lock semaphore. | |
The compare exchange operation must be performed using MP safe | |
mechanisms. | |
@param[in,out] Sem IN: 32-bit unsigned integer | |
OUT: -1 | |
@retval Original integer | |
**/ | |
STATIC | |
UINT32 | |
InternalLockdownSemaphore ( | |
IN OUT volatile UINT32 *Sem | |
) | |
{ | |
UINT32 Value; | |
do { | |
Value = *Sem; | |
} while (InterlockedCompareExchange32 ( | |
(UINT32 *)Sem, | |
Value, | |
(UINT32)-1 | |
) != Value); | |
return Value; | |
} | |
/** | |
Create and initialize the SMM CPU Sync context. It is to allocate and initialize the | |
SMM CPU Sync context. | |
If Context is NULL, then ASSERT(). | |
@param[in] NumberOfCpus The number of Logical Processors in the system. | |
@param[out] Context Pointer to the new created and initialized SMM CPU Sync context object. | |
NULL will be returned if any error happen during init. | |
@retval RETURN_SUCCESS The SMM CPU Sync context was successful created and initialized. | |
@retval RETURN_OUT_OF_RESOURCES There are not enough resources available to create and initialize SMM CPU Sync context. | |
@retval RETURN_BUFFER_TOO_SMALL Overflow happen | |
**/ | |
RETURN_STATUS | |
EFIAPI | |
SmmCpuSyncContextInit ( | |
IN UINTN NumberOfCpus, | |
OUT SMM_CPU_SYNC_CONTEXT **Context | |
) | |
{ | |
RETURN_STATUS Status; | |
UINTN ContextSize; | |
UINTN OneSemSize; | |
UINTN NumSem; | |
UINTN TotalSemSize; | |
UINTN SemAddr; | |
UINTN CpuIndex; | |
SMM_CPU_SYNC_SEMAPHORE_FOR_EACH_CPU *CpuSem; | |
ASSERT (Context != NULL); | |
// | |
// Calculate ContextSize | |
// | |
Status = SafeUintnMult (NumberOfCpus, sizeof (SMM_CPU_SYNC_SEMAPHORE_FOR_EACH_CPU), &ContextSize); | |
if (RETURN_ERROR (Status)) { | |
return Status; | |
} | |
Status = SafeUintnAdd (ContextSize, sizeof (SMM_CPU_SYNC_CONTEXT), &ContextSize); | |
if (RETURN_ERROR (Status)) { | |
return Status; | |
} | |
// | |
// Allocate Buffer for Context | |
// | |
*Context = AllocatePool (ContextSize); | |
if (*Context == NULL) { | |
return RETURN_OUT_OF_RESOURCES; | |
} | |
(*Context)->ArrivedCpuCountUponLock = 0; | |
// | |
// Save NumberOfCpus | |
// | |
(*Context)->NumberOfCpus = NumberOfCpus; | |
// | |
// Calculate total semaphore size | |
// | |
OneSemSize = GetSpinLockProperties (); | |
ASSERT (sizeof (SMM_CPU_SYNC_SEMAPHORE) <= OneSemSize); | |
Status = SafeUintnAdd (1, NumberOfCpus, &NumSem); | |
if (RETURN_ERROR (Status)) { | |
goto ON_ERROR; | |
} | |
Status = SafeUintnMult (NumSem, OneSemSize, &TotalSemSize); | |
if (RETURN_ERROR (Status)) { | |
goto ON_ERROR; | |
} | |
// | |
// Allocate for Semaphores in the *Context | |
// | |
(*Context)->SemBufferPages = EFI_SIZE_TO_PAGES (TotalSemSize); | |
(*Context)->SemBuffer = AllocatePages ((*Context)->SemBufferPages); | |
if ((*Context)->SemBuffer == NULL) { | |
Status = RETURN_OUT_OF_RESOURCES; | |
goto ON_ERROR; | |
} | |
// | |
// Assign Global Semaphore pointer | |
// | |
SemAddr = (UINTN)(*Context)->SemBuffer; | |
(*Context)->CpuCount = (SMM_CPU_SYNC_SEMAPHORE *)SemAddr; | |
*(*Context)->CpuCount = 0; | |
SemAddr += OneSemSize; | |
// | |
// Assign CPU Semaphore pointer | |
// | |
CpuSem = (*Context)->CpuSem; | |
for (CpuIndex = 0; CpuIndex < NumberOfCpus; CpuIndex++) { | |
CpuSem->Run = (SMM_CPU_SYNC_SEMAPHORE *)SemAddr; | |
*CpuSem->Run = 0; | |
CpuSem++; | |
SemAddr += OneSemSize; | |
} | |
return RETURN_SUCCESS; | |
ON_ERROR: | |
FreePool (*Context); | |
return Status; | |
} | |
/** | |
Deinit an allocated SMM CPU Sync context. The resources allocated in SmmCpuSyncContextInit() will | |
be freed. | |
If Context is NULL, then ASSERT(). | |
@param[in,out] Context Pointer to the SMM CPU Sync context object to be deinitialized. | |
**/ | |
VOID | |
EFIAPI | |
SmmCpuSyncContextDeinit ( | |
IN OUT SMM_CPU_SYNC_CONTEXT *Context | |
) | |
{ | |
ASSERT (Context != NULL); | |
FreePages (Context->SemBuffer, Context->SemBufferPages); | |
FreePool (Context); | |
} | |
/** | |
Reset SMM CPU Sync context. SMM CPU Sync context will be reset to the initialized state. | |
This function is called by one of CPUs after all CPUs are ready to exit SMI, which allows CPU to | |
check into the next SMI from this point. | |
If Context is NULL, then ASSERT(). | |
@param[in,out] Context Pointer to the SMM CPU Sync context object to be reset. | |
**/ | |
VOID | |
EFIAPI | |
SmmCpuSyncContextReset ( | |
IN OUT SMM_CPU_SYNC_CONTEXT *Context | |
) | |
{ | |
ASSERT (Context != NULL); | |
Context->ArrivedCpuCountUponLock = 0; | |
*Context->CpuCount = 0; | |
} | |
/** | |
Get current number of arrived CPU in SMI. | |
BSP might need to know the current number of arrived CPU in SMI to make sure all APs | |
in SMI. This API can be for that purpose. | |
If Context is NULL, then ASSERT(). | |
@param[in] Context Pointer to the SMM CPU Sync context object. | |
@retval Current number of arrived CPU in SMI. | |
**/ | |
UINTN | |
EFIAPI | |
SmmCpuSyncGetArrivedCpuCount ( | |
IN SMM_CPU_SYNC_CONTEXT *Context | |
) | |
{ | |
UINT32 Value; | |
ASSERT (Context != NULL); | |
Value = *Context->CpuCount; | |
if (Value == (UINT32)-1) { | |
return Context->ArrivedCpuCountUponLock; | |
} | |
return Value; | |
} | |
/** | |
Performs an atomic operation to check in CPU. | |
When SMI happens, all processors including BSP enter to SMM mode by calling SmmCpuSyncCheckInCpu(). | |
If Context is NULL, then ASSERT(). | |
If CpuIndex exceeds the range of all CPUs in the system, then ASSERT(). | |
@param[in,out] Context Pointer to the SMM CPU Sync context object. | |
@param[in] CpuIndex Check in CPU index. | |
@retval RETURN_SUCCESS Check in CPU (CpuIndex) successfully. | |
@retval RETURN_ABORTED Check in CPU failed due to SmmCpuSyncLockDoor() has been called by one elected CPU. | |
**/ | |
RETURN_STATUS | |
EFIAPI | |
SmmCpuSyncCheckInCpu ( | |
IN OUT SMM_CPU_SYNC_CONTEXT *Context, | |
IN UINTN CpuIndex | |
) | |
{ | |
ASSERT (Context != NULL); | |
ASSERT (CpuIndex < Context->NumberOfCpus); | |
// | |
// Check to return if CpuCount has already been locked. | |
// | |
if (InternalReleaseSemaphore (Context->CpuCount) == MAX_UINT32) { | |
return RETURN_ABORTED; | |
} | |
return RETURN_SUCCESS; | |
} | |
/** | |
Performs an atomic operation to check out CPU. | |
This function can be called in error handling flow for the CPU who calls CheckInCpu() earlier. | |
The caller shall make sure the CPU specified by CpuIndex has already checked-in. | |
If Context is NULL, then ASSERT(). | |
If CpuIndex exceeds the range of all CPUs in the system, then ASSERT(). | |
@param[in,out] Context Pointer to the SMM CPU Sync context object. | |
@param[in] CpuIndex Check out CPU index. | |
@retval RETURN_SUCCESS Check out CPU (CpuIndex) successfully. | |
@retval RETURN_ABORTED Check out CPU failed due to SmmCpuSyncLockDoor() has been called by one elected CPU. | |
**/ | |
RETURN_STATUS | |
EFIAPI | |
SmmCpuSyncCheckOutCpu ( | |
IN OUT SMM_CPU_SYNC_CONTEXT *Context, | |
IN UINTN CpuIndex | |
) | |
{ | |
ASSERT (Context != NULL); | |
ASSERT (CpuIndex < Context->NumberOfCpus); | |
if (InternalWaitForSemaphore (Context->CpuCount) == MAX_UINT32) { | |
return RETURN_ABORTED; | |
} | |
return RETURN_SUCCESS; | |
} | |
/** | |
Performs an atomic operation lock door for CPU checkin and checkout. After this function: | |
CPU can not check in via SmmCpuSyncCheckInCpu(). | |
CPU can not check out via SmmCpuSyncCheckOutCpu(). | |
The CPU specified by CpuIndex is elected to lock door. The caller shall make sure the CpuIndex | |
is the actual CPU calling this function to avoid the undefined behavior. | |
If Context is NULL, then ASSERT(). | |
If CpuCount is NULL, then ASSERT(). | |
If CpuIndex exceeds the range of all CPUs in the system, then ASSERT(). | |
@param[in,out] Context Pointer to the SMM CPU Sync context object. | |
@param[in] CpuIndex Indicate which CPU to lock door. | |
@param[out] CpuCount Number of arrived CPU in SMI after look door. | |
**/ | |
VOID | |
EFIAPI | |
SmmCpuSyncLockDoor ( | |
IN OUT SMM_CPU_SYNC_CONTEXT *Context, | |
IN UINTN CpuIndex, | |
OUT UINTN *CpuCount | |
) | |
{ | |
ASSERT (Context != NULL); | |
ASSERT (CpuCount != NULL); | |
ASSERT (CpuIndex < Context->NumberOfCpus); | |
// | |
// Temporarily record the CpuCount into the ArrivedCpuCountUponLock before lock door. | |
// Recording before lock door is to avoid the Context->CpuCount is locked but possible | |
// Context->ArrivedCpuCountUponLock is not updated. | |
// | |
Context->ArrivedCpuCountUponLock = *Context->CpuCount; | |
// | |
// Lock door operation | |
// | |
*CpuCount = InternalLockdownSemaphore (Context->CpuCount); | |
// | |
// Update the ArrivedCpuCountUponLock | |
// | |
Context->ArrivedCpuCountUponLock = *CpuCount; | |
} | |
/** | |
Used by the BSP to wait for APs. | |
The number of APs need to be waited is specified by NumberOfAPs. The BSP is specified by BspIndex. | |
The caller shall make sure the BspIndex is the actual CPU calling this function to avoid the undefined behavior. | |
The caller shall make sure the NumberOfAPs have already checked-in to avoid the undefined behavior. | |
If Context is NULL, then ASSERT(). | |
If NumberOfAPs >= All CPUs in system, then ASSERT(). | |
If BspIndex exceeds the range of all CPUs in the system, then ASSERT(). | |
Note: | |
This function is blocking mode, and it will return only after the number of APs released by | |
calling SmmCpuSyncReleaseBsp(): | |
BSP: WaitForAPs <-- AP: ReleaseBsp | |
@param[in,out] Context Pointer to the SMM CPU Sync context object. | |
@param[in] NumberOfAPs Number of APs need to be waited by BSP. | |
@param[in] BspIndex The BSP Index to wait for APs. | |
**/ | |
VOID | |
EFIAPI | |
SmmCpuSyncWaitForAPs ( | |
IN OUT SMM_CPU_SYNC_CONTEXT *Context, | |
IN UINTN NumberOfAPs, | |
IN UINTN BspIndex | |
) | |
{ | |
UINTN Arrived; | |
ASSERT (Context != NULL); | |
ASSERT (NumberOfAPs < Context->NumberOfCpus); | |
ASSERT (BspIndex < Context->NumberOfCpus); | |
for (Arrived = 0; Arrived < NumberOfAPs; Arrived++) { | |
InternalWaitForSemaphore (Context->CpuSem[BspIndex].Run); | |
} | |
} | |
/** | |
Used by the BSP to release one AP. | |
The AP is specified by CpuIndex. The BSP is specified by BspIndex. | |
The caller shall make sure the BspIndex is the actual CPU calling this function to avoid the undefined behavior. | |
The caller shall make sure the CpuIndex has already checked-in to avoid the undefined behavior. | |
If Context is NULL, then ASSERT(). | |
If CpuIndex == BspIndex, then ASSERT(). | |
If BspIndex or CpuIndex exceed the range of all CPUs in the system, then ASSERT(). | |
@param[in,out] Context Pointer to the SMM CPU Sync context object. | |
@param[in] CpuIndex Indicate which AP need to be released. | |
@param[in] BspIndex The BSP Index to release AP. | |
**/ | |
VOID | |
EFIAPI | |
SmmCpuSyncReleaseOneAp ( | |
IN OUT SMM_CPU_SYNC_CONTEXT *Context, | |
IN UINTN CpuIndex, | |
IN UINTN BspIndex | |
) | |
{ | |
ASSERT (Context != NULL); | |
ASSERT (BspIndex != CpuIndex); | |
ASSERT (CpuIndex < Context->NumberOfCpus); | |
ASSERT (BspIndex < Context->NumberOfCpus); | |
InternalReleaseSemaphore (Context->CpuSem[CpuIndex].Run); | |
} | |
/** | |
Used by the AP to wait BSP. | |
The AP is specified by CpuIndex. | |
The caller shall make sure the CpuIndex is the actual CPU calling this function to avoid the undefined behavior. | |
The BSP is specified by BspIndex. | |
If Context is NULL, then ASSERT(). | |
If CpuIndex == BspIndex, then ASSERT(). | |
If BspIndex or CpuIndex exceed the range of all CPUs in the system, then ASSERT(). | |
Note: | |
This function is blocking mode, and it will return only after the AP released by | |
calling SmmCpuSyncReleaseOneAp(): | |
BSP: ReleaseOneAp --> AP: WaitForBsp | |
@param[in,out] Context Pointer to the SMM CPU Sync context object. | |
@param[in] CpuIndex Indicate which AP wait BSP. | |
@param[in] BspIndex The BSP Index to be waited. | |
**/ | |
VOID | |
EFIAPI | |
SmmCpuSyncWaitForBsp ( | |
IN OUT SMM_CPU_SYNC_CONTEXT *Context, | |
IN UINTN CpuIndex, | |
IN UINTN BspIndex | |
) | |
{ | |
ASSERT (Context != NULL); | |
ASSERT (BspIndex != CpuIndex); | |
ASSERT (CpuIndex < Context->NumberOfCpus); | |
ASSERT (BspIndex < Context->NumberOfCpus); | |
InternalWaitForSemaphore (Context->CpuSem[CpuIndex].Run); | |
} | |
/** | |
Used by the AP to release BSP. | |
The AP is specified by CpuIndex. | |
The caller shall make sure the CpuIndex is the actual CPU calling this function to avoid the undefined behavior. | |
The BSP is specified by BspIndex. | |
If Context is NULL, then ASSERT(). | |
If CpuIndex == BspIndex, then ASSERT(). | |
If BspIndex or CpuIndex exceed the range of all CPUs in the system, then ASSERT(). | |
@param[in,out] Context Pointer to the SMM CPU Sync context object. | |
@param[in] CpuIndex Indicate which AP release BSP. | |
@param[in] BspIndex The BSP Index to be released. | |
**/ | |
VOID | |
EFIAPI | |
SmmCpuSyncReleaseBsp ( | |
IN OUT SMM_CPU_SYNC_CONTEXT *Context, | |
IN UINTN CpuIndex, | |
IN UINTN BspIndex | |
) | |
{ | |
ASSERT (Context != NULL); | |
ASSERT (BspIndex != CpuIndex); | |
ASSERT (CpuIndex < Context->NumberOfCpus); | |
ASSERT (BspIndex < Context->NumberOfCpus); | |
InternalReleaseSemaphore (Context->CpuSem[BspIndex].Run); | |
} |