| /** @file | |
| NvmExpressDxe driver is used to manage non-volatile memory subsystem which follows | |
| NVM Express specification. | |
| Copyright (c) 2013 - 2019, Intel Corporation. All rights reserved.<BR> | |
| SPDX-License-Identifier: BSD-2-Clause-Patent | |
| **/ | |
| #include "NvmExpress.h" | |
| #define NVME_SHUTDOWN_PROCESS_TIMEOUT 45 | |
| // | |
| // The number of NVME controllers managed by this driver, used by | |
| // NvmeRegisterShutdownNotification() and NvmeUnregisterShutdownNotification(). | |
| // | |
| UINTN mNvmeControllerNumber = 0; | |
| /** | |
| Read Nvm Express controller capability register. | |
| @param Private The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure. | |
| @param Cap The buffer used to store capability register content. | |
| @return EFI_SUCCESS Successfully read the controller capability register content. | |
| @return EFI_DEVICE_ERROR Fail to read the controller capability register. | |
| **/ | |
| EFI_STATUS | |
| ReadNvmeControllerCapabilities ( | |
| IN NVME_CONTROLLER_PRIVATE_DATA *Private, | |
| IN NVME_CAP *Cap | |
| ) | |
| { | |
| EFI_PCI_IO_PROTOCOL *PciIo; | |
| EFI_STATUS Status; | |
| UINT64 Data; | |
| PciIo = Private->PciIo; | |
| Status = PciIo->Mem.Read ( | |
| PciIo, | |
| EfiPciIoWidthUint32, | |
| NVME_BAR, | |
| NVME_CAP_OFFSET, | |
| 2, | |
| &Data | |
| ); | |
| if (EFI_ERROR (Status)) { | |
| return Status; | |
| } | |
| WriteUnaligned64 ((UINT64 *)Cap, Data); | |
| return EFI_SUCCESS; | |
| } | |
| /** | |
| Read Nvm Express controller configuration register. | |
| @param Private The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure. | |
| @param Cc The buffer used to store configuration register content. | |
| @return EFI_SUCCESS Successfully read the controller configuration register content. | |
| @return EFI_DEVICE_ERROR Fail to read the controller configuration register. | |
| **/ | |
| EFI_STATUS | |
| ReadNvmeControllerConfiguration ( | |
| IN NVME_CONTROLLER_PRIVATE_DATA *Private, | |
| IN NVME_CC *Cc | |
| ) | |
| { | |
| EFI_PCI_IO_PROTOCOL *PciIo; | |
| EFI_STATUS Status; | |
| UINT32 Data; | |
| PciIo = Private->PciIo; | |
| Status = PciIo->Mem.Read ( | |
| PciIo, | |
| EfiPciIoWidthUint32, | |
| NVME_BAR, | |
| NVME_CC_OFFSET, | |
| 1, | |
| &Data | |
| ); | |
| if (EFI_ERROR (Status)) { | |
| return Status; | |
| } | |
| WriteUnaligned32 ((UINT32 *)Cc, Data); | |
| return EFI_SUCCESS; | |
| } | |
| /** | |
| Write Nvm Express controller configuration register. | |
| @param Private The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure. | |
| @param Cc The buffer used to store the content to be written into configuration register. | |
| @return EFI_SUCCESS Successfully write data into the controller configuration register. | |
| @return EFI_DEVICE_ERROR Fail to write data into the controller configuration register. | |
| **/ | |
| EFI_STATUS | |
| WriteNvmeControllerConfiguration ( | |
| IN NVME_CONTROLLER_PRIVATE_DATA *Private, | |
| IN NVME_CC *Cc | |
| ) | |
| { | |
| EFI_PCI_IO_PROTOCOL *PciIo; | |
| EFI_STATUS Status; | |
| UINT32 Data; | |
| PciIo = Private->PciIo; | |
| Data = ReadUnaligned32 ((UINT32 *)Cc); | |
| Status = PciIo->Mem.Write ( | |
| PciIo, | |
| EfiPciIoWidthUint32, | |
| NVME_BAR, | |
| NVME_CC_OFFSET, | |
| 1, | |
| &Data | |
| ); | |
| if (EFI_ERROR (Status)) { | |
| return Status; | |
| } | |
| DEBUG ((DEBUG_INFO, "Cc.En: %d\n", Cc->En)); | |
| DEBUG ((DEBUG_INFO, "Cc.Css: %d\n", Cc->Css)); | |
| DEBUG ((DEBUG_INFO, "Cc.Mps: %d\n", Cc->Mps)); | |
| DEBUG ((DEBUG_INFO, "Cc.Ams: %d\n", Cc->Ams)); | |
| DEBUG ((DEBUG_INFO, "Cc.Shn: %d\n", Cc->Shn)); | |
| DEBUG ((DEBUG_INFO, "Cc.Iosqes: %d\n", Cc->Iosqes)); | |
| DEBUG ((DEBUG_INFO, "Cc.Iocqes: %d\n", Cc->Iocqes)); | |
| return EFI_SUCCESS; | |
| } | |
| /** | |
| Read Nvm Express controller status register. | |
| @param Private The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure. | |
| @param Csts The buffer used to store status register content. | |
| @return EFI_SUCCESS Successfully read the controller status register content. | |
| @return EFI_DEVICE_ERROR Fail to read the controller status register. | |
| **/ | |
| EFI_STATUS | |
| ReadNvmeControllerStatus ( | |
| IN NVME_CONTROLLER_PRIVATE_DATA *Private, | |
| IN NVME_CSTS *Csts | |
| ) | |
| { | |
| EFI_PCI_IO_PROTOCOL *PciIo; | |
| EFI_STATUS Status; | |
| UINT32 Data; | |
| PciIo = Private->PciIo; | |
| Status = PciIo->Mem.Read ( | |
| PciIo, | |
| EfiPciIoWidthUint32, | |
| NVME_BAR, | |
| NVME_CSTS_OFFSET, | |
| 1, | |
| &Data | |
| ); | |
| if (EFI_ERROR (Status)) { | |
| return Status; | |
| } | |
| WriteUnaligned32 ((UINT32 *)Csts, Data); | |
| return EFI_SUCCESS; | |
| } | |
| /** | |
| Write Nvm Express admin queue attributes register. | |
| @param Private The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure. | |
| @param Aqa The buffer used to store the content to be written into admin queue attributes register. | |
| @return EFI_SUCCESS Successfully write data into the admin queue attributes register. | |
| @return EFI_DEVICE_ERROR Fail to write data into the admin queue attributes register. | |
| **/ | |
| EFI_STATUS | |
| WriteNvmeAdminQueueAttributes ( | |
| IN NVME_CONTROLLER_PRIVATE_DATA *Private, | |
| IN NVME_AQA *Aqa | |
| ) | |
| { | |
| EFI_PCI_IO_PROTOCOL *PciIo; | |
| EFI_STATUS Status; | |
| UINT32 Data; | |
| PciIo = Private->PciIo; | |
| Data = ReadUnaligned32 ((UINT32 *)Aqa); | |
| Status = PciIo->Mem.Write ( | |
| PciIo, | |
| EfiPciIoWidthUint32, | |
| NVME_BAR, | |
| NVME_AQA_OFFSET, | |
| 1, | |
| &Data | |
| ); | |
| if (EFI_ERROR (Status)) { | |
| return Status; | |
| } | |
| DEBUG ((DEBUG_INFO, "Aqa.Asqs: %d\n", Aqa->Asqs)); | |
| DEBUG ((DEBUG_INFO, "Aqa.Acqs: %d\n", Aqa->Acqs)); | |
| return EFI_SUCCESS; | |
| } | |
| /** | |
| Write Nvm Express admin submission queue base address register. | |
| @param Private The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure. | |
| @param Asq The buffer used to store the content to be written into admin submission queue base address register. | |
| @return EFI_SUCCESS Successfully write data into the admin submission queue base address register. | |
| @return EFI_DEVICE_ERROR Fail to write data into the admin submission queue base address register. | |
| **/ | |
| EFI_STATUS | |
| WriteNvmeAdminSubmissionQueueBaseAddress ( | |
| IN NVME_CONTROLLER_PRIVATE_DATA *Private, | |
| IN NVME_ASQ *Asq | |
| ) | |
| { | |
| EFI_PCI_IO_PROTOCOL *PciIo; | |
| EFI_STATUS Status; | |
| UINT64 Data; | |
| PciIo = Private->PciIo; | |
| Data = ReadUnaligned64 ((UINT64 *)Asq); | |
| Status = PciIo->Mem.Write ( | |
| PciIo, | |
| EfiPciIoWidthUint32, | |
| NVME_BAR, | |
| NVME_ASQ_OFFSET, | |
| 2, | |
| &Data | |
| ); | |
| if (EFI_ERROR (Status)) { | |
| return Status; | |
| } | |
| DEBUG ((DEBUG_INFO, "Asq: %lx\n", *Asq)); | |
| return EFI_SUCCESS; | |
| } | |
| /** | |
| Write Nvm Express admin completion queue base address register. | |
| @param Private The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure. | |
| @param Acq The buffer used to store the content to be written into admin completion queue base address register. | |
| @return EFI_SUCCESS Successfully write data into the admin completion queue base address register. | |
| @return EFI_DEVICE_ERROR Fail to write data into the admin completion queue base address register. | |
| **/ | |
| EFI_STATUS | |
| WriteNvmeAdminCompletionQueueBaseAddress ( | |
| IN NVME_CONTROLLER_PRIVATE_DATA *Private, | |
| IN NVME_ACQ *Acq | |
| ) | |
| { | |
| EFI_PCI_IO_PROTOCOL *PciIo; | |
| EFI_STATUS Status; | |
| UINT64 Data; | |
| PciIo = Private->PciIo; | |
| Data = ReadUnaligned64 ((UINT64 *)Acq); | |
| Status = PciIo->Mem.Write ( | |
| PciIo, | |
| EfiPciIoWidthUint32, | |
| NVME_BAR, | |
| NVME_ACQ_OFFSET, | |
| 2, | |
| &Data | |
| ); | |
| if (EFI_ERROR (Status)) { | |
| return Status; | |
| } | |
| DEBUG ((DEBUG_INFO, "Acq: %lxh\n", *Acq)); | |
| return EFI_SUCCESS; | |
| } | |
| /** | |
| Disable the Nvm Express controller. | |
| @param Private The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure. | |
| @return EFI_SUCCESS Successfully disable the controller. | |
| @return EFI_DEVICE_ERROR Fail to disable the controller. | |
| **/ | |
| EFI_STATUS | |
| NvmeDisableController ( | |
| IN NVME_CONTROLLER_PRIVATE_DATA *Private | |
| ) | |
| { | |
| NVME_CC Cc; | |
| NVME_CSTS Csts; | |
| EFI_STATUS Status; | |
| UINT32 Index; | |
| UINT8 Timeout; | |
| // | |
| // Read Controller Configuration Register. | |
| // | |
| Status = ReadNvmeControllerConfiguration (Private, &Cc); | |
| if (EFI_ERROR (Status)) { | |
| return Status; | |
| } | |
| Cc.En = 0; | |
| // | |
| // Disable the controller. | |
| // | |
| Status = WriteNvmeControllerConfiguration (Private, &Cc); | |
| if (EFI_ERROR (Status)) { | |
| return Status; | |
| } | |
| // | |
| // Cap.To specifies max delay time in 500ms increments for Csts.Rdy to transition from 1 to 0 after | |
| // Cc.Enable transition from 1 to 0. Loop produces a 1 millisecond delay per itteration, up to 500 * Cap.To. | |
| // | |
| if (Private->Cap.To == 0) { | |
| Timeout = 1; | |
| } else { | |
| Timeout = Private->Cap.To; | |
| } | |
| for (Index = (Timeout * 500); Index != 0; --Index) { | |
| gBS->Stall (1000); | |
| // | |
| // Check if the controller is initialized | |
| // | |
| Status = ReadNvmeControllerStatus (Private, &Csts); | |
| if (EFI_ERROR (Status)) { | |
| return Status; | |
| } | |
| if (Csts.Rdy == 0) { | |
| break; | |
| } | |
| } | |
| if (Index == 0) { | |
| Status = EFI_DEVICE_ERROR; | |
| REPORT_STATUS_CODE ( | |
| (EFI_ERROR_CODE | EFI_ERROR_MAJOR), | |
| (EFI_IO_BUS_SCSI | EFI_IOB_EC_INTERFACE_ERROR) | |
| ); | |
| } | |
| DEBUG ((DEBUG_INFO, "NVMe controller is disabled with status [%r].\n", Status)); | |
| return Status; | |
| } | |
| /** | |
| Enable the Nvm Express controller. | |
| @param Private The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure. | |
| @return EFI_SUCCESS Successfully enable the controller. | |
| @return EFI_DEVICE_ERROR Fail to enable the controller. | |
| @return EFI_TIMEOUT Fail to enable the controller in given time slot. | |
| **/ | |
| EFI_STATUS | |
| NvmeEnableController ( | |
| IN NVME_CONTROLLER_PRIVATE_DATA *Private | |
| ) | |
| { | |
| NVME_CC Cc; | |
| NVME_CSTS Csts; | |
| EFI_STATUS Status; | |
| UINT32 Index; | |
| UINT8 Timeout; | |
| // | |
| // Enable the controller. | |
| // CC.AMS, CC.MPS and CC.CSS are all set to 0. | |
| // | |
| ZeroMem (&Cc, sizeof (NVME_CC)); | |
| Cc.En = 1; | |
| Cc.Iosqes = 6; | |
| Cc.Iocqes = 4; | |
| Status = WriteNvmeControllerConfiguration (Private, &Cc); | |
| if (EFI_ERROR (Status)) { | |
| return Status; | |
| } | |
| // | |
| // Cap.To specifies max delay time in 500ms increments for Csts.Rdy to set after | |
| // Cc.Enable. Loop produces a 1 millisecond delay per itteration, up to 500 * Cap.To. | |
| // | |
| if (Private->Cap.To == 0) { | |
| Timeout = 1; | |
| } else { | |
| Timeout = Private->Cap.To; | |
| } | |
| for (Index = (Timeout * 500); Index != 0; --Index) { | |
| gBS->Stall (1000); | |
| // | |
| // Check if the controller is initialized | |
| // | |
| Status = ReadNvmeControllerStatus (Private, &Csts); | |
| if (EFI_ERROR (Status)) { | |
| return Status; | |
| } | |
| if (Csts.Rdy) { | |
| break; | |
| } | |
| } | |
| if (Index == 0) { | |
| Status = EFI_TIMEOUT; | |
| REPORT_STATUS_CODE ( | |
| (EFI_ERROR_CODE | EFI_ERROR_MAJOR), | |
| (EFI_IO_BUS_SCSI | EFI_IOB_EC_INTERFACE_ERROR) | |
| ); | |
| } | |
| DEBUG ((DEBUG_INFO, "NVMe controller is enabled with status [%r].\n", Status)); | |
| return Status; | |
| } | |
| /** | |
| Get identify controller data. | |
| @param Private The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure. | |
| @param Buffer The buffer used to store the identify controller data. | |
| @return EFI_SUCCESS Successfully get the identify controller data. | |
| @return EFI_DEVICE_ERROR Fail to get the identify controller data. | |
| **/ | |
| EFI_STATUS | |
| NvmeIdentifyController ( | |
| IN NVME_CONTROLLER_PRIVATE_DATA *Private, | |
| IN VOID *Buffer | |
| ) | |
| { | |
| EFI_NVM_EXPRESS_PASS_THRU_COMMAND_PACKET CommandPacket; | |
| EFI_NVM_EXPRESS_COMMAND Command; | |
| EFI_NVM_EXPRESS_COMPLETION Completion; | |
| EFI_STATUS Status; | |
| ZeroMem (&CommandPacket, sizeof (EFI_NVM_EXPRESS_PASS_THRU_COMMAND_PACKET)); | |
| ZeroMem (&Command, sizeof (EFI_NVM_EXPRESS_COMMAND)); | |
| ZeroMem (&Completion, sizeof (EFI_NVM_EXPRESS_COMPLETION)); | |
| Command.Cdw0.Opcode = NVME_ADMIN_IDENTIFY_CMD; | |
| // | |
| // According to Nvm Express 1.1 spec Figure 38, When not used, the field shall be cleared to 0h. | |
| // For the Identify command, the Namespace Identifier is only used for the Namespace data structure. | |
| // | |
| Command.Nsid = 0; | |
| CommandPacket.NvmeCmd = &Command; | |
| CommandPacket.NvmeCompletion = &Completion; | |
| CommandPacket.TransferBuffer = Buffer; | |
| CommandPacket.TransferLength = sizeof (NVME_ADMIN_CONTROLLER_DATA); | |
| CommandPacket.CommandTimeout = NVME_GENERIC_TIMEOUT; | |
| CommandPacket.QueueType = NVME_ADMIN_QUEUE; | |
| // | |
| // Set bit 0 (Cns bit) to 1 to identify a controller | |
| // | |
| Command.Cdw10 = 1; | |
| Command.Flags = CDW10_VALID; | |
| Status = Private->Passthru.PassThru ( | |
| &Private->Passthru, | |
| NVME_CONTROLLER_ID, | |
| &CommandPacket, | |
| NULL | |
| ); | |
| return Status; | |
| } | |
| /** | |
| Get specified identify namespace data. | |
| @param Private The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure. | |
| @param NamespaceId The specified namespace identifier. | |
| @param Buffer The buffer used to store the identify namespace data. | |
| @return EFI_SUCCESS Successfully get the identify namespace data. | |
| @return EFI_DEVICE_ERROR Fail to get the identify namespace data. | |
| **/ | |
| EFI_STATUS | |
| NvmeIdentifyNamespace ( | |
| IN NVME_CONTROLLER_PRIVATE_DATA *Private, | |
| IN UINT32 NamespaceId, | |
| IN VOID *Buffer | |
| ) | |
| { | |
| EFI_NVM_EXPRESS_PASS_THRU_COMMAND_PACKET CommandPacket; | |
| EFI_NVM_EXPRESS_COMMAND Command; | |
| EFI_NVM_EXPRESS_COMPLETION Completion; | |
| EFI_STATUS Status; | |
| ZeroMem (&CommandPacket, sizeof (EFI_NVM_EXPRESS_PASS_THRU_COMMAND_PACKET)); | |
| ZeroMem (&Command, sizeof (EFI_NVM_EXPRESS_COMMAND)); | |
| ZeroMem (&Completion, sizeof (EFI_NVM_EXPRESS_COMPLETION)); | |
| CommandPacket.NvmeCmd = &Command; | |
| CommandPacket.NvmeCompletion = &Completion; | |
| Command.Cdw0.Opcode = NVME_ADMIN_IDENTIFY_CMD; | |
| Command.Nsid = NamespaceId; | |
| CommandPacket.TransferBuffer = Buffer; | |
| CommandPacket.TransferLength = sizeof (NVME_ADMIN_NAMESPACE_DATA); | |
| CommandPacket.CommandTimeout = NVME_GENERIC_TIMEOUT; | |
| CommandPacket.QueueType = NVME_ADMIN_QUEUE; | |
| // | |
| // Set bit 0 (Cns bit) to 1 to identify a namespace | |
| // | |
| CommandPacket.NvmeCmd->Cdw10 = 0; | |
| CommandPacket.NvmeCmd->Flags = CDW10_VALID; | |
| Status = Private->Passthru.PassThru ( | |
| &Private->Passthru, | |
| NamespaceId, | |
| &CommandPacket, | |
| NULL | |
| ); | |
| return Status; | |
| } | |
| /** | |
| Create io completion queue. | |
| @param Private The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure. | |
| @return EFI_SUCCESS Successfully create io completion queue. | |
| @return EFI_DEVICE_ERROR Fail to create io completion queue. | |
| **/ | |
| EFI_STATUS | |
| NvmeCreateIoCompletionQueue ( | |
| IN NVME_CONTROLLER_PRIVATE_DATA *Private | |
| ) | |
| { | |
| EFI_NVM_EXPRESS_PASS_THRU_COMMAND_PACKET CommandPacket; | |
| EFI_NVM_EXPRESS_COMMAND Command; | |
| EFI_NVM_EXPRESS_COMPLETION Completion; | |
| EFI_STATUS Status; | |
| NVME_ADMIN_CRIOCQ CrIoCq; | |
| UINT32 Index; | |
| UINT16 QueueSize; | |
| Status = EFI_SUCCESS; | |
| Private->CreateIoQueue = TRUE; | |
| for (Index = 1; Index < NVME_MAX_QUEUES; Index++) { | |
| ZeroMem (&CommandPacket, sizeof (EFI_NVM_EXPRESS_PASS_THRU_COMMAND_PACKET)); | |
| ZeroMem (&Command, sizeof (EFI_NVM_EXPRESS_COMMAND)); | |
| ZeroMem (&Completion, sizeof (EFI_NVM_EXPRESS_COMPLETION)); | |
| ZeroMem (&CrIoCq, sizeof (NVME_ADMIN_CRIOCQ)); | |
| CommandPacket.NvmeCmd = &Command; | |
| CommandPacket.NvmeCompletion = &Completion; | |
| Command.Cdw0.Opcode = NVME_ADMIN_CRIOCQ_CMD; | |
| CommandPacket.TransferBuffer = Private->CqBufferPciAddr[Index]; | |
| CommandPacket.TransferLength = EFI_PAGE_SIZE; | |
| CommandPacket.CommandTimeout = NVME_GENERIC_TIMEOUT; | |
| CommandPacket.QueueType = NVME_ADMIN_QUEUE; | |
| if (Index == 1) { | |
| QueueSize = NVME_CCQ_SIZE; | |
| } else { | |
| if (Private->Cap.Mqes > NVME_ASYNC_CCQ_SIZE) { | |
| QueueSize = NVME_ASYNC_CCQ_SIZE; | |
| } else { | |
| QueueSize = Private->Cap.Mqes; | |
| } | |
| } | |
| CrIoCq.Qid = Index; | |
| CrIoCq.Qsize = QueueSize; | |
| CrIoCq.Pc = 1; | |
| CopyMem (&CommandPacket.NvmeCmd->Cdw10, &CrIoCq, sizeof (NVME_ADMIN_CRIOCQ)); | |
| CommandPacket.NvmeCmd->Flags = CDW10_VALID | CDW11_VALID; | |
| Status = Private->Passthru.PassThru ( | |
| &Private->Passthru, | |
| 0, | |
| &CommandPacket, | |
| NULL | |
| ); | |
| if (EFI_ERROR (Status)) { | |
| break; | |
| } | |
| } | |
| Private->CreateIoQueue = FALSE; | |
| return Status; | |
| } | |
| /** | |
| Create io submission queue. | |
| @param Private The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure. | |
| @return EFI_SUCCESS Successfully create io submission queue. | |
| @return EFI_DEVICE_ERROR Fail to create io submission queue. | |
| **/ | |
| EFI_STATUS | |
| NvmeCreateIoSubmissionQueue ( | |
| IN NVME_CONTROLLER_PRIVATE_DATA *Private | |
| ) | |
| { | |
| EFI_NVM_EXPRESS_PASS_THRU_COMMAND_PACKET CommandPacket; | |
| EFI_NVM_EXPRESS_COMMAND Command; | |
| EFI_NVM_EXPRESS_COMPLETION Completion; | |
| EFI_STATUS Status; | |
| NVME_ADMIN_CRIOSQ CrIoSq; | |
| UINT32 Index; | |
| UINT16 QueueSize; | |
| Status = EFI_SUCCESS; | |
| Private->CreateIoQueue = TRUE; | |
| for (Index = 1; Index < NVME_MAX_QUEUES; Index++) { | |
| ZeroMem (&CommandPacket, sizeof (EFI_NVM_EXPRESS_PASS_THRU_COMMAND_PACKET)); | |
| ZeroMem (&Command, sizeof (EFI_NVM_EXPRESS_COMMAND)); | |
| ZeroMem (&Completion, sizeof (EFI_NVM_EXPRESS_COMPLETION)); | |
| ZeroMem (&CrIoSq, sizeof (NVME_ADMIN_CRIOSQ)); | |
| CommandPacket.NvmeCmd = &Command; | |
| CommandPacket.NvmeCompletion = &Completion; | |
| Command.Cdw0.Opcode = NVME_ADMIN_CRIOSQ_CMD; | |
| CommandPacket.TransferBuffer = Private->SqBufferPciAddr[Index]; | |
| CommandPacket.TransferLength = EFI_PAGE_SIZE; | |
| CommandPacket.CommandTimeout = NVME_GENERIC_TIMEOUT; | |
| CommandPacket.QueueType = NVME_ADMIN_QUEUE; | |
| if (Index == 1) { | |
| QueueSize = NVME_CSQ_SIZE; | |
| } else { | |
| if (Private->Cap.Mqes > NVME_ASYNC_CSQ_SIZE) { | |
| QueueSize = NVME_ASYNC_CSQ_SIZE; | |
| } else { | |
| QueueSize = Private->Cap.Mqes; | |
| } | |
| } | |
| CrIoSq.Qid = Index; | |
| CrIoSq.Qsize = QueueSize; | |
| CrIoSq.Pc = 1; | |
| CrIoSq.Cqid = Index; | |
| CrIoSq.Qprio = 0; | |
| CopyMem (&CommandPacket.NvmeCmd->Cdw10, &CrIoSq, sizeof (NVME_ADMIN_CRIOSQ)); | |
| CommandPacket.NvmeCmd->Flags = CDW10_VALID | CDW11_VALID; | |
| Status = Private->Passthru.PassThru ( | |
| &Private->Passthru, | |
| 0, | |
| &CommandPacket, | |
| NULL | |
| ); | |
| if (EFI_ERROR (Status)) { | |
| break; | |
| } | |
| } | |
| Private->CreateIoQueue = FALSE; | |
| return Status; | |
| } | |
| /** | |
| Initialize the Nvm Express controller. | |
| @param[in] Private The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure. | |
| @retval EFI_SUCCESS The NVM Express Controller is initialized successfully. | |
| @retval Others A device error occurred while initializing the controller. | |
| **/ | |
| EFI_STATUS | |
| NvmeControllerInit ( | |
| IN NVME_CONTROLLER_PRIVATE_DATA *Private | |
| ) | |
| { | |
| EFI_STATUS Status; | |
| EFI_PCI_IO_PROTOCOL *PciIo; | |
| UINT64 Supports; | |
| NVME_AQA Aqa; | |
| NVME_ASQ Asq; | |
| NVME_ACQ Acq; | |
| UINT8 Sn[21]; | |
| UINT8 Mn[41]; | |
| // | |
| // Enable this controller. | |
| // | |
| PciIo = Private->PciIo; | |
| Status = PciIo->Attributes ( | |
| PciIo, | |
| EfiPciIoAttributeOperationSupported, | |
| 0, | |
| &Supports | |
| ); | |
| if (!EFI_ERROR (Status)) { | |
| Supports &= (UINT64)EFI_PCI_DEVICE_ENABLE; | |
| Status = PciIo->Attributes ( | |
| PciIo, | |
| EfiPciIoAttributeOperationEnable, | |
| Supports, | |
| NULL | |
| ); | |
| } | |
| if (EFI_ERROR (Status)) { | |
| DEBUG ((DEBUG_INFO, "NvmeControllerInit: failed to enable controller\n")); | |
| return Status; | |
| } | |
| // | |
| // Read the Controller Capabilities register and verify that the NVM command set is supported | |
| // | |
| Status = ReadNvmeControllerCapabilities (Private, &Private->Cap); | |
| if (EFI_ERROR (Status)) { | |
| return Status; | |
| } | |
| if ((Private->Cap.Css & BIT0) == 0) { | |
| DEBUG ((DEBUG_INFO, "NvmeControllerInit: the controller doesn't support NVMe command set\n")); | |
| return EFI_UNSUPPORTED; | |
| } | |
| // | |
| // Currently the driver only supports 4k page size. | |
| // | |
| ASSERT ((Private->Cap.Mpsmin + 12) <= EFI_PAGE_SHIFT); | |
| Private->Cid[0] = 0; | |
| Private->Cid[1] = 0; | |
| Private->Cid[2] = 0; | |
| Private->Pt[0] = 0; | |
| Private->Pt[1] = 0; | |
| Private->Pt[2] = 0; | |
| Private->SqTdbl[0].Sqt = 0; | |
| Private->SqTdbl[1].Sqt = 0; | |
| Private->SqTdbl[2].Sqt = 0; | |
| Private->CqHdbl[0].Cqh = 0; | |
| Private->CqHdbl[1].Cqh = 0; | |
| Private->CqHdbl[2].Cqh = 0; | |
| Private->AsyncSqHead = 0; | |
| Status = NvmeDisableController (Private); | |
| if (EFI_ERROR (Status)) { | |
| return Status; | |
| } | |
| // | |
| // set number of entries admin submission & completion queues. | |
| // | |
| Aqa.Asqs = NVME_ASQ_SIZE; | |
| Aqa.Rsvd1 = 0; | |
| Aqa.Acqs = NVME_ACQ_SIZE; | |
| Aqa.Rsvd2 = 0; | |
| // | |
| // Address of admin submission queue. | |
| // | |
| Asq = (UINT64)(UINTN)(Private->BufferPciAddr) & ~0xFFF; | |
| // | |
| // Address of admin completion queue. | |
| // | |
| Acq = (UINT64)(UINTN)(Private->BufferPciAddr + EFI_PAGE_SIZE) & ~0xFFF; | |
| // | |
| // Address of I/O submission & completion queue. | |
| // | |
| ZeroMem (Private->Buffer, EFI_PAGES_TO_SIZE (6)); | |
| Private->SqBuffer[0] = (NVME_SQ *)(UINTN)(Private->Buffer); | |
| Private->SqBufferPciAddr[0] = (NVME_SQ *)(UINTN)(Private->BufferPciAddr); | |
| Private->CqBuffer[0] = (NVME_CQ *)(UINTN)(Private->Buffer + 1 * EFI_PAGE_SIZE); | |
| Private->CqBufferPciAddr[0] = (NVME_CQ *)(UINTN)(Private->BufferPciAddr + 1 * EFI_PAGE_SIZE); | |
| Private->SqBuffer[1] = (NVME_SQ *)(UINTN)(Private->Buffer + 2 * EFI_PAGE_SIZE); | |
| Private->SqBufferPciAddr[1] = (NVME_SQ *)(UINTN)(Private->BufferPciAddr + 2 * EFI_PAGE_SIZE); | |
| Private->CqBuffer[1] = (NVME_CQ *)(UINTN)(Private->Buffer + 3 * EFI_PAGE_SIZE); | |
| Private->CqBufferPciAddr[1] = (NVME_CQ *)(UINTN)(Private->BufferPciAddr + 3 * EFI_PAGE_SIZE); | |
| Private->SqBuffer[2] = (NVME_SQ *)(UINTN)(Private->Buffer + 4 * EFI_PAGE_SIZE); | |
| Private->SqBufferPciAddr[2] = (NVME_SQ *)(UINTN)(Private->BufferPciAddr + 4 * EFI_PAGE_SIZE); | |
| Private->CqBuffer[2] = (NVME_CQ *)(UINTN)(Private->Buffer + 5 * EFI_PAGE_SIZE); | |
| Private->CqBufferPciAddr[2] = (NVME_CQ *)(UINTN)(Private->BufferPciAddr + 5 * EFI_PAGE_SIZE); | |
| DEBUG ((DEBUG_INFO, "Private->Buffer = [%016X]\n", (UINT64)(UINTN)Private->Buffer)); | |
| DEBUG ((DEBUG_INFO, "Admin Submission Queue size (Aqa.Asqs) = [%08X]\n", Aqa.Asqs)); | |
| DEBUG ((DEBUG_INFO, "Admin Completion Queue size (Aqa.Acqs) = [%08X]\n", Aqa.Acqs)); | |
| DEBUG ((DEBUG_INFO, "Admin Submission Queue (SqBuffer[0]) = [%016X]\n", Private->SqBuffer[0])); | |
| DEBUG ((DEBUG_INFO, "Admin Completion Queue (CqBuffer[0]) = [%016X]\n", Private->CqBuffer[0])); | |
| DEBUG ((DEBUG_INFO, "Sync I/O Submission Queue (SqBuffer[1]) = [%016X]\n", Private->SqBuffer[1])); | |
| DEBUG ((DEBUG_INFO, "Sync I/O Completion Queue (CqBuffer[1]) = [%016X]\n", Private->CqBuffer[1])); | |
| DEBUG ((DEBUG_INFO, "Async I/O Submission Queue (SqBuffer[2]) = [%016X]\n", Private->SqBuffer[2])); | |
| DEBUG ((DEBUG_INFO, "Async I/O Completion Queue (CqBuffer[2]) = [%016X]\n", Private->CqBuffer[2])); | |
| // | |
| // Program admin queue attributes. | |
| // | |
| Status = WriteNvmeAdminQueueAttributes (Private, &Aqa); | |
| if (EFI_ERROR (Status)) { | |
| return Status; | |
| } | |
| // | |
| // Program admin submission queue address. | |
| // | |
| Status = WriteNvmeAdminSubmissionQueueBaseAddress (Private, &Asq); | |
| if (EFI_ERROR (Status)) { | |
| return Status; | |
| } | |
| // | |
| // Program admin completion queue address. | |
| // | |
| Status = WriteNvmeAdminCompletionQueueBaseAddress (Private, &Acq); | |
| if (EFI_ERROR (Status)) { | |
| return Status; | |
| } | |
| Status = NvmeEnableController (Private); | |
| if (EFI_ERROR (Status)) { | |
| return Status; | |
| } | |
| // | |
| // Allocate buffer for Identify Controller data | |
| // | |
| if (Private->ControllerData == NULL) { | |
| Private->ControllerData = (NVME_ADMIN_CONTROLLER_DATA *)AllocateZeroPool (sizeof (NVME_ADMIN_CONTROLLER_DATA)); | |
| if (Private->ControllerData == NULL) { | |
| return EFI_OUT_OF_RESOURCES; | |
| } | |
| } | |
| // | |
| // Get current Identify Controller Data | |
| // | |
| Status = NvmeIdentifyController (Private, Private->ControllerData); | |
| if (EFI_ERROR (Status)) { | |
| FreePool (Private->ControllerData); | |
| Private->ControllerData = NULL; | |
| return EFI_NOT_FOUND; | |
| } | |
| // | |
| // Dump NvmExpress Identify Controller Data | |
| // | |
| CopyMem (Sn, Private->ControllerData->Sn, sizeof (Private->ControllerData->Sn)); | |
| Sn[20] = 0; | |
| CopyMem (Mn, Private->ControllerData->Mn, sizeof (Private->ControllerData->Mn)); | |
| Mn[40] = 0; | |
| DEBUG ((DEBUG_INFO, " == NVME IDENTIFY CONTROLLER DATA ==\n")); | |
| DEBUG ((DEBUG_INFO, " PCI VID : 0x%x\n", Private->ControllerData->Vid)); | |
| DEBUG ((DEBUG_INFO, " PCI SSVID : 0x%x\n", Private->ControllerData->Ssvid)); | |
| DEBUG ((DEBUG_INFO, " SN : %a\n", Sn)); | |
| DEBUG ((DEBUG_INFO, " MN : %a\n", Mn)); | |
| DEBUG ((DEBUG_INFO, " FR : 0x%x\n", *((UINT64 *)Private->ControllerData->Fr))); | |
| DEBUG ((DEBUG_INFO, " TNVMCAP (high 8-byte) : 0x%lx\n", *((UINT64 *)(Private->ControllerData->Tnvmcap + 8)))); | |
| DEBUG ((DEBUG_INFO, " TNVMCAP (low 8-byte) : 0x%lx\n", *((UINT64 *)Private->ControllerData->Tnvmcap))); | |
| DEBUG ((DEBUG_INFO, " RAB : 0x%x\n", Private->ControllerData->Rab)); | |
| DEBUG ((DEBUG_INFO, " IEEE : 0x%x\n", *(UINT32 *)Private->ControllerData->Ieee_oui)); | |
| DEBUG ((DEBUG_INFO, " AERL : 0x%x\n", Private->ControllerData->Aerl)); | |
| DEBUG ((DEBUG_INFO, " SQES : 0x%x\n", Private->ControllerData->Sqes)); | |
| DEBUG ((DEBUG_INFO, " CQES : 0x%x\n", Private->ControllerData->Cqes)); | |
| DEBUG ((DEBUG_INFO, " NN : 0x%x\n", Private->ControllerData->Nn)); | |
| // | |
| // Create two I/O completion queues. | |
| // One for blocking I/O, one for non-blocking I/O. | |
| // | |
| Status = NvmeCreateIoCompletionQueue (Private); | |
| if (EFI_ERROR (Status)) { | |
| return Status; | |
| } | |
| // | |
| // Create two I/O Submission queues. | |
| // One for blocking I/O, one for non-blocking I/O. | |
| // | |
| Status = NvmeCreateIoSubmissionQueue (Private); | |
| return Status; | |
| } | |
| /** | |
| This routine is called to properly shutdown the Nvm Express controller per NVMe spec. | |
| @param[in] ResetType The type of reset to perform. | |
| @param[in] ResetStatus The status code for the reset. | |
| @param[in] DataSize The size, in bytes, of ResetData. | |
| @param[in] ResetData For a ResetType of EfiResetCold, EfiResetWarm, or | |
| EfiResetShutdown the data buffer starts with a Null-terminated | |
| string, optionally followed by additional binary data. | |
| The string is a description that the caller may use to further | |
| indicate the reason for the system reset. | |
| For a ResetType of EfiResetPlatformSpecific the data buffer | |
| also starts with a Null-terminated string that is followed | |
| by an EFI_GUID that describes the specific type of reset to perform. | |
| **/ | |
| VOID | |
| EFIAPI | |
| NvmeShutdownAllControllers ( | |
| IN EFI_RESET_TYPE ResetType, | |
| IN EFI_STATUS ResetStatus, | |
| IN UINTN DataSize, | |
| IN VOID *ResetData OPTIONAL | |
| ) | |
| { | |
| EFI_STATUS Status; | |
| EFI_HANDLE *Handles; | |
| UINTN HandleCount; | |
| UINTN HandleIndex; | |
| EFI_OPEN_PROTOCOL_INFORMATION_ENTRY *OpenInfos; | |
| UINTN OpenInfoCount; | |
| UINTN OpenInfoIndex; | |
| EFI_NVM_EXPRESS_PASS_THRU_PROTOCOL *NvmePassThru; | |
| NVME_CC Cc; | |
| NVME_CSTS Csts; | |
| UINTN Index; | |
| NVME_CONTROLLER_PRIVATE_DATA *Private; | |
| Status = gBS->LocateHandleBuffer ( | |
| ByProtocol, | |
| &gEfiPciIoProtocolGuid, | |
| NULL, | |
| &HandleCount, | |
| &Handles | |
| ); | |
| if (EFI_ERROR (Status)) { | |
| HandleCount = 0; | |
| } | |
| for (HandleIndex = 0; HandleIndex < HandleCount; HandleIndex++) { | |
| Status = gBS->OpenProtocolInformation ( | |
| Handles[HandleIndex], | |
| &gEfiPciIoProtocolGuid, | |
| &OpenInfos, | |
| &OpenInfoCount | |
| ); | |
| if (EFI_ERROR (Status)) { | |
| continue; | |
| } | |
| for (OpenInfoIndex = 0; OpenInfoIndex < OpenInfoCount; OpenInfoIndex++) { | |
| // | |
| // Find all the NVME controller managed by this driver. | |
| // gImageHandle equals to DriverBinding handle for this driver. | |
| // | |
| if (((OpenInfos[OpenInfoIndex].Attributes & EFI_OPEN_PROTOCOL_BY_DRIVER) != 0) && | |
| (OpenInfos[OpenInfoIndex].AgentHandle == gImageHandle)) | |
| { | |
| Status = gBS->OpenProtocol ( | |
| OpenInfos[OpenInfoIndex].ControllerHandle, | |
| &gEfiNvmExpressPassThruProtocolGuid, | |
| (VOID **)&NvmePassThru, | |
| NULL, | |
| NULL, | |
| EFI_OPEN_PROTOCOL_GET_PROTOCOL | |
| ); | |
| if (EFI_ERROR (Status)) { | |
| continue; | |
| } | |
| Private = NVME_CONTROLLER_PRIVATE_DATA_FROM_PASS_THRU (NvmePassThru); | |
| // | |
| // Read Controller Configuration Register. | |
| // | |
| Status = ReadNvmeControllerConfiguration (Private, &Cc); | |
| if (EFI_ERROR (Status)) { | |
| continue; | |
| } | |
| // | |
| // The host should set the Shutdown Notification (CC.SHN) field to 01b | |
| // to indicate a normal shutdown operation. | |
| // | |
| Cc.Shn = NVME_CC_SHN_NORMAL_SHUTDOWN; | |
| Status = WriteNvmeControllerConfiguration (Private, &Cc); | |
| if (EFI_ERROR (Status)) { | |
| continue; | |
| } | |
| // | |
| // The controller indicates when shutdown processing is completed by updating the | |
| // Shutdown Status (CSTS.SHST) field to 10b. | |
| // Wait up to 45 seconds (break down to 4500 x 10ms) for the shutdown to complete. | |
| // | |
| for (Index = 0; Index < NVME_SHUTDOWN_PROCESS_TIMEOUT * 100; Index++) { | |
| Status = ReadNvmeControllerStatus (Private, &Csts); | |
| if (!EFI_ERROR (Status) && (Csts.Shst == NVME_CSTS_SHST_SHUTDOWN_COMPLETED)) { | |
| DEBUG ((DEBUG_INFO, "NvmeShutdownController: shutdown processing is completed after %dms.\n", Index * 10)); | |
| break; | |
| } | |
| // | |
| // Stall for 10ms | |
| // | |
| gBS->Stall (10 * 1000); | |
| } | |
| if (Index == NVME_SHUTDOWN_PROCESS_TIMEOUT * 100) { | |
| DEBUG ((DEBUG_ERROR, "NvmeShutdownController: shutdown processing is timed out\n")); | |
| } | |
| } | |
| } | |
| } | |
| } | |
| /** | |
| Register the shutdown notification through the ResetNotification protocol. | |
| Register the shutdown notification when mNvmeControllerNumber increased from 0 to 1. | |
| **/ | |
| VOID | |
| NvmeRegisterShutdownNotification ( | |
| VOID | |
| ) | |
| { | |
| EFI_STATUS Status; | |
| EFI_RESET_NOTIFICATION_PROTOCOL *ResetNotify; | |
| mNvmeControllerNumber++; | |
| if (mNvmeControllerNumber == 1) { | |
| Status = gBS->LocateProtocol (&gEfiResetNotificationProtocolGuid, NULL, (VOID **)&ResetNotify); | |
| if (!EFI_ERROR (Status)) { | |
| Status = ResetNotify->RegisterResetNotify (ResetNotify, NvmeShutdownAllControllers); | |
| ASSERT_EFI_ERROR (Status); | |
| } else { | |
| DEBUG ((DEBUG_WARN, "NVME: ResetNotification absent! Shutdown notification cannot be performed!\n")); | |
| } | |
| } | |
| } | |
| /** | |
| Unregister the shutdown notification through the ResetNotification protocol. | |
| Unregister the shutdown notification when mNvmeControllerNumber decreased from 1 to 0. | |
| **/ | |
| VOID | |
| NvmeUnregisterShutdownNotification ( | |
| VOID | |
| ) | |
| { | |
| EFI_STATUS Status; | |
| EFI_RESET_NOTIFICATION_PROTOCOL *ResetNotify; | |
| mNvmeControllerNumber--; | |
| if (mNvmeControllerNumber == 0) { | |
| Status = gBS->LocateProtocol (&gEfiResetNotificationProtocolGuid, NULL, (VOID **)&ResetNotify); | |
| if (!EFI_ERROR (Status)) { | |
| Status = ResetNotify->UnregisterResetNotify (ResetNotify, NvmeShutdownAllControllers); | |
| ASSERT_EFI_ERROR (Status); | |
| } | |
| } | |
| } |