| // SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later |
| /* |
| * This code will enable retrieving of error log from FSP -> Sapphire in |
| * sequence. |
| * Here, FSP would send next log only when Sapphire sends a new log notification |
| * response to FSP. On Completion of reading the log from FSP, |
| * OPAL_EVENT_ERROR_LOG_AVAIL is signaled. This will remain raised until a call |
| * to opal_elog_read() is made and OPAL_SUCCESS is returned. Upon which, the |
| * operation is complete and the event is cleared. This is READ action from FSP. |
| * |
| * Copyright 2013-2017 IBM Corp. |
| */ |
| |
| /* |
| * Design of READ error log : |
| * When we receive a new error log entry notification from FSP, we queue it into |
| * the "pending" list. If the "pending" list is not empty, then we start |
| * fetching log from FSP. |
| * |
| * When Linux reads a log entry, we dequeue it from the "pending" list and |
| * enqueue it to another "processed" list. At this point, if the "pending" |
| * list is not empty, we continue to fetch the next log. |
| * |
| * When Linux calls opal_resend_pending_logs(), we fetch the log corresponding |
| * to the head of the pending list and move it to the processed list, and |
| * continue this process until the pending list is empty. If the pending list |
| * was empty earlier and is currently non-empty, we initiate an error log fetch. |
| * |
| * When Linux acks an error log, we remove it from processed list. |
| */ |
| |
| #include <errno.h> |
| #include <fsp.h> |
| #include <fsp-elog.h> |
| #include <lock.h> |
| #include <opal-api.h> |
| #include <psi.h> |
| #include <skiboot.h> |
| |
| /* |
| * Maximum number of entries that are pre-allocated |
| * to keep track of pending elogs to be fetched. |
| */ |
| #define ELOG_READ_MAX_RECORD 128 |
| |
| /* Structure to maintain log-id, log-size, pending and processed list. */ |
| struct fsp_log_entry { |
| uint32_t log_id; |
| size_t log_size; |
| struct list_node link; |
| }; |
| |
| static LIST_HEAD(elog_read_pending); |
| static LIST_HEAD(elog_read_processed); |
| static LIST_HEAD(elog_read_free); |
| /* |
| * Lock is used to protect overwriting of processed and pending list |
| * and also used while updating state of each log. |
| */ |
| static struct lock elog_read_lock = LOCK_UNLOCKED; |
| |
| #define ELOG_READ_BUFFER_SIZE 0x00004000 |
| /* Log buffer to copy FSP log for read */ |
| static void *elog_read_buffer; |
| static uint32_t elog_head_id; /* FSP entry ID */ |
| static size_t elog_head_size; /* Actual FSP log size */ |
| static uint32_t elog_read_retries; /* Bad response status count */ |
| |
| /* Initialize the state of the log */ |
| static enum elog_head_state elog_read_from_fsp_head_state = ELOG_STATE_NONE; |
| |
| static bool elog_enabled = false; |
| |
| /* Need forward declaration because of circular dependency. */ |
| static void fsp_elog_queue_fetch(void); |
| |
| /* |
| * Check the response message for mbox acknowledgement |
| * command send to FSP. |
| */ |
| static void fsp_elog_ack_complete(struct fsp_msg *msg) |
| { |
| uint8_t val; |
| |
| val = (msg->resp->word1 >> 8) & 0xff; |
| if (val != 0) |
| prerror("ELOG: Acknowledgement error\n"); |
| |
| fsp_freemsg(msg); |
| } |
| |
| /* Send error log PHYP acknowledgement to FSP with entry ID. */ |
| static int64_t fsp_send_elog_ack(uint32_t log_id) |
| { |
| struct fsp_msg *ack_msg; |
| |
| ack_msg = fsp_mkmsg(FSP_CMD_ERRLOG_PHYP_ACK, 1, log_id); |
| if (!ack_msg) { |
| prerror("ELOG: Failed to allocate ack message\n"); |
| return OPAL_INTERNAL_ERROR; |
| } |
| |
| if (fsp_queue_msg(ack_msg, fsp_elog_ack_complete)) { |
| fsp_freemsg(ack_msg); |
| ack_msg = NULL; |
| prerror("ELOG: Error queueing elog ack complete\n"); |
| return OPAL_INTERNAL_ERROR; |
| } |
| |
| return OPAL_SUCCESS; |
| } |
| |
| /* Retrieve error log from FSP with TCE for the data transfer. */ |
| static void fsp_elog_check_and_fetch_head(void) |
| { |
| lock(&elog_read_lock); |
| if (elog_read_from_fsp_head_state != ELOG_STATE_NONE || |
| list_empty(&elog_read_pending)) { |
| unlock(&elog_read_lock); |
| return; |
| } |
| |
| elog_read_retries = 0; |
| /* Start fetching first entry from the pending list */ |
| fsp_elog_queue_fetch(); |
| unlock(&elog_read_lock); |
| } |
| |
| void elog_set_head_state(bool opal_logs, enum elog_head_state state) |
| { |
| static enum elog_head_state opal_logs_state = ELOG_STATE_NONE; |
| static enum elog_head_state fsp_logs_state = ELOG_STATE_NONE; |
| |
| /* ELOG disabled */ |
| if (!elog_enabled) |
| return; |
| |
| if (opal_logs) |
| opal_logs_state = state; |
| else |
| fsp_logs_state = state; |
| |
| if (fsp_logs_state == ELOG_STATE_FETCHED_DATA || |
| opal_logs_state == ELOG_STATE_FETCHED_DATA) |
| opal_update_pending_evt(OPAL_EVENT_ERROR_LOG_AVAIL, |
| OPAL_EVENT_ERROR_LOG_AVAIL); |
| else |
| opal_update_pending_evt(OPAL_EVENT_ERROR_LOG_AVAIL, 0); |
| } |
| |
| /* This function should be called with the lock held. */ |
| static inline void fsp_elog_set_head_state(enum elog_head_state state) |
| { |
| elog_set_head_state(false, state); |
| elog_read_from_fsp_head_state = state; |
| } |
| |
| /* |
| * When, we try maximum time of fetching log from FSP |
| * we call following function to delete log from the |
| * pending list and update the state to fetch next log. |
| * |
| * This function should be called with the lock held. |
| */ |
| static void fsp_elog_fetch_failure(uint8_t fsp_status) |
| { |
| struct fsp_log_entry *log_data; |
| |
| /* Read top list and delete the node */ |
| log_data = list_top(&elog_read_pending, struct fsp_log_entry, link); |
| if (!log_data) { |
| /** |
| * @fwts-label ElogFetchFailureInconsistent |
| * @fwts-advice Inconsistent state between OPAL and FSP |
| * in code path for handling failure of fetching error log |
| * from FSP. Likely a bug in interaction between FSP and OPAL. |
| */ |
| prlog(PR_ERR, "%s: Inconsistent internal list state !\n", |
| __func__); |
| } else { |
| list_del(&log_data->link); |
| list_add(&elog_read_free, &log_data->link); |
| prerror("ELOG: received invalid data: %x FSP status: 0x%x\n", |
| log_data->log_id, fsp_status); |
| } |
| |
| fsp_elog_set_head_state(ELOG_STATE_NONE); |
| } |
| |
| /* Read response value from FSP for fetch sp data mbox command */ |
| static void fsp_elog_read_complete(struct fsp_msg *read_msg) |
| { |
| uint8_t val; |
| |
| lock(&elog_read_lock); |
| val = (read_msg->resp->word1 >> 8) & 0xff; |
| fsp_freemsg(read_msg); |
| if (elog_read_from_fsp_head_state == ELOG_STATE_REJECTED) { |
| fsp_elog_set_head_state(ELOG_STATE_NONE); |
| goto elog_read_out; |
| } |
| |
| switch (val) { |
| case FSP_STATUS_SUCCESS: |
| fsp_elog_set_head_state(ELOG_STATE_FETCHED_DATA); |
| break; |
| |
| case FSP_STATUS_DMA_ERROR: |
| if (elog_read_retries++ < MAX_RETRIES) { |
| /* |
| * For a error response value from FSP, we try to |
| * send fetch sp data mbox command again for three |
| * times if response from FSP is still not valid |
| * we send generic error response to FSP. |
| */ |
| fsp_elog_queue_fetch(); |
| break; |
| } |
| |
| fsp_elog_fetch_failure(val); |
| break; |
| |
| default: |
| fsp_elog_fetch_failure(val); |
| } |
| |
| elog_read_out: |
| unlock(&elog_read_lock); |
| |
| /* Check if a new log needs fetching */ |
| fsp_elog_check_and_fetch_head(); |
| } |
| |
| /* Read error log from FSP through mbox commands */ |
| static void fsp_elog_queue_fetch(void) |
| { |
| int rc; |
| uint8_t flags = 0; |
| struct fsp_log_entry *entry; |
| |
| entry = list_top(&elog_read_pending, struct fsp_log_entry, link); |
| if (!entry) { |
| /** |
| * @fwts-label ElogQueueInconsistent |
| * @fwts-advice Bug in interaction between FSP and OPAL. We |
| * expected there to be a pending read from FSP but the list |
| * was empty. |
| */ |
| prlog(PR_ERR, "%s: Inconsistent internal list state !\n", |
| __func__); |
| fsp_elog_set_head_state(ELOG_STATE_NONE); |
| return; |
| } |
| |
| fsp_elog_set_head_state(ELOG_STATE_FETCHING); |
| elog_head_id = entry->log_id; |
| elog_head_size = entry->log_size; |
| rc = fsp_fetch_data_queue(flags, FSP_DATASET_ERRLOG, elog_head_id, |
| 0, (void *)PSI_DMA_ERRLOG_READ_BUF, |
| &elog_head_size, fsp_elog_read_complete); |
| if (rc) { |
| prerror("ELOG: failed to queue read message: %d\n", rc); |
| fsp_elog_set_head_state(ELOG_STATE_NONE); |
| } |
| } |
| |
| /* OPAL interface for PowerNV to read log size and log ID from Sapphire. */ |
| static int64_t fsp_opal_elog_info(__be64 *opal_elog_id, |
| __be64 *opal_elog_size, __be64 *elog_type) |
| { |
| struct fsp_log_entry *log_data; |
| |
| /* Copy type of the error log */ |
| *elog_type = cpu_to_be64(ELOG_TYPE_PEL); |
| |
| /* Check if any OPAL log needs to be reported to the host */ |
| if (opal_elog_info(opal_elog_id, opal_elog_size)) |
| return OPAL_SUCCESS; |
| |
| lock(&elog_read_lock); |
| if (elog_read_from_fsp_head_state != ELOG_STATE_FETCHED_DATA) { |
| unlock(&elog_read_lock); |
| return OPAL_WRONG_STATE; |
| } |
| |
| log_data = list_top(&elog_read_pending, struct fsp_log_entry, link); |
| if (!log_data) { |
| /** |
| * @fwts-label ElogInfoInconsistentState |
| * @fwts-advice We expected there to be an entry in the list |
| * of error logs for the error log we're fetching information |
| * for. There wasn't. This means there's a bug. |
| */ |
| prlog(PR_ERR, "%s: Inconsistent internal list state !\n", |
| __func__); |
| fsp_elog_set_head_state(ELOG_STATE_NONE); |
| unlock(&elog_read_lock); |
| return OPAL_WRONG_STATE; |
| } |
| |
| *opal_elog_id = cpu_to_be64(log_data->log_id); |
| *opal_elog_size = cpu_to_be64(log_data->log_size); |
| fsp_elog_set_head_state(ELOG_STATE_HOST_INFO); |
| unlock(&elog_read_lock); |
| return OPAL_SUCCESS; |
| } |
| |
| /* OPAL interface for PowerNV to read log from Sapphire. */ |
| static int64_t fsp_opal_elog_read(void *buffer, uint64_t opal_elog_size, |
| uint64_t opal_elog_id) |
| { |
| int size = opal_elog_size; |
| struct fsp_log_entry *log_data; |
| |
| /* Check if any OPAL log needs to be reported to the PowerNV */ |
| if (opal_elog_read(buffer, opal_elog_size, opal_elog_id)) |
| return OPAL_SUCCESS; |
| |
| /* |
| * Read top entry from list. |
| * As we know always top record of the list is fetched from FSP |
| */ |
| lock(&elog_read_lock); |
| if (elog_read_from_fsp_head_state != ELOG_STATE_HOST_INFO) { |
| unlock(&elog_read_lock); |
| return OPAL_WRONG_STATE; |
| } |
| |
| log_data = list_top(&elog_read_pending, struct fsp_log_entry, link); |
| if (!log_data) { |
| /** |
| * @fwts-label ElogReadInconsistentState |
| * @fwts-advice Inconsistent state while reading error log |
| * from FSP. Bug in OPAL and FSP interaction. |
| */ |
| prlog(PR_ERR, "%s: Inconsistent internal list state !\n", |
| __func__); |
| fsp_elog_set_head_state(ELOG_STATE_NONE); |
| unlock(&elog_read_lock); |
| return OPAL_WRONG_STATE; |
| } |
| |
| /* Check log ID and then read log from buffer */ |
| if (opal_elog_id != log_data->log_id) { |
| unlock(&elog_read_lock); |
| return OPAL_PARAMETER; |
| } |
| |
| /* Do not copy more than actual log size */ |
| if (opal_elog_size > log_data->log_size) |
| size = log_data->log_size; |
| |
| memset(buffer, 0, opal_elog_size); |
| memcpy(buffer, elog_read_buffer, size); |
| |
| /* |
| * Once log is read from linux move record from pending |
| * to processed list and delete record from pending list |
| * and change state of the log to fetch next record. |
| */ |
| list_del(&log_data->link); |
| list_add(&elog_read_processed, &log_data->link); |
| fsp_elog_set_head_state(ELOG_STATE_NONE); |
| unlock(&elog_read_lock); |
| |
| /* Read error log from FSP */ |
| fsp_elog_check_and_fetch_head(); |
| |
| return OPAL_SUCCESS; |
| } |
| |
| /* Set state of the log head before fetching the log. */ |
| static void elog_reject_head(void) |
| { |
| if (elog_read_from_fsp_head_state == ELOG_STATE_FETCHING) |
| fsp_elog_set_head_state(ELOG_STATE_REJECTED); |
| else |
| fsp_elog_set_head_state(ELOG_STATE_NONE); |
| } |
| |
| /* OPAL interface for PowerNV to send ack to FSP with log ID */ |
| static int64_t fsp_opal_elog_ack(uint64_t ack_id) |
| { |
| int rc = 0; |
| struct fsp_log_entry *record, *next_record; |
| |
| if (opal_elog_ack(ack_id)) |
| return rc; |
| |
| /* Send acknowledgement to FSP */ |
| rc = fsp_send_elog_ack(ack_id); |
| if (rc != OPAL_SUCCESS) { |
| prerror("ELOG: failed to send acknowledgement: %d\n", rc); |
| return rc; |
| } |
| |
| lock(&elog_read_lock); |
| list_for_each_safe(&elog_read_processed, record, next_record, link) { |
| if (record->log_id != ack_id) |
| continue; |
| |
| list_del(&record->link); |
| list_add(&elog_read_free, &record->link); |
| unlock(&elog_read_lock); |
| return rc; |
| } |
| |
| list_for_each_safe(&elog_read_pending, record, next_record, link) { |
| if (record->log_id != ack_id) |
| continue; |
| /* |
| * It means PowerNV has sent ACK without reading actual data. |
| * Because of this elog_read_from_fsp_head_state may be |
| * stuck in wrong state (ELOG_STATE_HOST_INFO) and not able |
| * to send remaining ELOGs to PowerNV. Hence reset ELOG state |
| * and start sending remaining ELOGs. |
| */ |
| list_del(&record->link); |
| list_add(&elog_read_free, &record->link); |
| elog_reject_head(); |
| unlock(&elog_read_lock); |
| fsp_elog_check_and_fetch_head(); |
| return rc; |
| } |
| |
| unlock(&elog_read_lock); |
| return OPAL_PARAMETER; |
| } |
| |
| /* |
| * Once Linux kexec's it ask to resend all logs which |
| * are not acknowledged from Linux. |
| */ |
| static void fsp_opal_resend_pending_logs(void) |
| { |
| struct fsp_log_entry *entry; |
| |
| lock(&elog_read_lock); |
| elog_enabled = true; |
| unlock(&elog_read_lock); |
| |
| /* Check if any Sapphire logs are pending. */ |
| opal_resend_pending_logs(); |
| |
| lock(&elog_read_lock); |
| /* |
| * If processed list is not empty add all record from |
| * processed list to pending list at head of the list |
| * and delete records from processed list. |
| */ |
| while (!list_empty(&elog_read_processed)) { |
| entry = list_pop(&elog_read_processed, |
| struct fsp_log_entry, link); |
| list_add(&elog_read_pending, &entry->link); |
| } |
| |
| unlock(&elog_read_lock); |
| |
| /* Read error log from FSP */ |
| elog_reject_head(); |
| fsp_elog_check_and_fetch_head(); |
| } |
| |
| /* Disable ELOG event flag until PowerNV is ready to receive event */ |
| static bool opal_kexec_elog_notify(void *data __unused) |
| { |
| lock(&elog_read_lock); |
| elog_enabled = false; |
| opal_update_pending_evt(OPAL_EVENT_ERROR_LOG_AVAIL, 0); |
| unlock(&elog_read_lock); |
| |
| return true; |
| } |
| |
| /* FSP elog notify function */ |
| static bool fsp_elog_msg(uint32_t cmd_sub_mod, struct fsp_msg *msg) |
| { |
| int rc = 0; |
| struct fsp_log_entry *record; |
| uint32_t log_id; |
| uint32_t log_size; |
| |
| if (cmd_sub_mod != FSP_CMD_ERRLOG_NOTIFICATION) |
| return false; |
| |
| log_id = fsp_msg_get_data_word(msg, 0); |
| log_size = fsp_msg_get_data_word(msg, 1); |
| |
| prlog(PR_TRACE, "ELOG: Notified of log 0x%08x (size: %d)\n", |
| log_id, log_size); |
| |
| /* Make sure we don't cross read buffer size */ |
| if (log_size > ELOG_READ_BUFFER_SIZE) { |
| log_size = ELOG_READ_BUFFER_SIZE; |
| printf("ELOG: Truncated log (0x%08x) to 0x%x\n", |
| log_id, log_size); |
| } |
| |
| /* Take a lock until we take out the node from elog_read_free */ |
| lock(&elog_read_lock); |
| if (!list_empty(&elog_read_free)) { |
| /* Create a new entry in the pending list. */ |
| record = list_pop(&elog_read_free, struct fsp_log_entry, link); |
| record->log_id = log_id; |
| record->log_size = log_size; |
| list_add_tail(&elog_read_pending, &record->link); |
| unlock(&elog_read_lock); |
| |
| /* Send response back to FSP for a new elog notify message. */ |
| rc = fsp_queue_msg(fsp_mkmsg(FSP_RSP_ERRLOG_NOTIFICATION, |
| 1, log_id), fsp_freemsg); |
| if (rc) |
| prerror("ELOG: Failed to queue errlog notification" |
| " response: %d\n", rc); |
| |
| /* Read error log from FSP */ |
| fsp_elog_check_and_fetch_head(); |
| |
| } else { |
| prlog(PR_TRACE, "ELOG: Log entry 0x%08x discarded\n", log_id); |
| |
| /* Unlock if elog_read_free is empty. */ |
| unlock(&elog_read_lock); |
| |
| rc = fsp_queue_msg(fsp_mkmsg(FSP_RSP_ERRLOG_NOTIFICATION, |
| 1, log_id), fsp_freemsg); |
| if (rc) |
| prerror("ELOG: Failed to queue errlog notification" |
| " response: %d\n", rc); |
| |
| /* |
| * If list is full with max record then we send discarded by |
| * phyp (condition full) ack to FSP. |
| * |
| * At some point in the future, we'll get notified again. |
| * This is largely up to FSP as to when they tell us about |
| * the log again. |
| */ |
| rc = fsp_queue_msg(fsp_mkmsg(FSP_CMD_ERRLOG_PHYP_ACK | 0x02, |
| 1, log_id), fsp_freemsg); |
| if (rc) |
| prerror("ELOG: Failed to queue errlog ack" |
| " response: %d\n", rc); |
| } |
| |
| return true; |
| } |
| |
| static struct fsp_client fsp_get_elog_notify = { |
| .message = fsp_elog_msg, |
| }; |
| |
| /* Pre-allocate memory for reading error log from FSP */ |
| static int init_elog_read_free_list(uint32_t num_entries) |
| { |
| struct fsp_log_entry *entry; |
| int i; |
| |
| entry = zalloc(sizeof(struct fsp_log_entry) * num_entries); |
| if (!entry) |
| goto out_err; |
| |
| for (i = 0; i < num_entries; ++i) { |
| list_add_tail(&elog_read_free, &entry->link); |
| entry++; |
| } |
| |
| return 0; |
| |
| out_err: |
| return -ENOMEM; |
| } |
| |
| /* FSP elog read init function */ |
| void fsp_elog_read_init(void) |
| { |
| int val = 0; |
| |
| if (!fsp_present()) |
| return; |
| |
| elog_read_buffer = memalign(TCE_PSIZE, ELOG_READ_BUFFER_SIZE); |
| if (!elog_read_buffer) { |
| prerror("FSP: could not allocate FSP ELOG_READ_BUFFER!\n"); |
| return; |
| } |
| |
| /* Map TCEs */ |
| fsp_tce_map(PSI_DMA_ERRLOG_READ_BUF, elog_read_buffer, |
| PSI_DMA_ERRLOG_READ_BUF_SZ); |
| |
| /* Pre allocate memory for 128 record */ |
| val = init_elog_read_free_list(ELOG_READ_MAX_RECORD); |
| if (val != 0) |
| return; |
| |
| /* Register error log class D2 */ |
| fsp_register_client(&fsp_get_elog_notify, FSP_MCLASS_ERR_LOG); |
| |
| /* Register for sync on PowerNV reboot call */ |
| opal_add_host_sync_notifier(opal_kexec_elog_notify, NULL); |
| |
| /* Register OPAL interface */ |
| opal_register(OPAL_ELOG_READ, fsp_opal_elog_read, 3); |
| opal_register(OPAL_ELOG_ACK, fsp_opal_elog_ack, 1); |
| opal_register(OPAL_ELOG_RESEND, fsp_opal_resend_pending_logs, 0); |
| opal_register(OPAL_ELOG_SIZE, fsp_opal_elog_info, 3); |
| } |