src/drivers/net/ena.h

#ifndef _ENA_H
#define _ENA_H

/** @file
 *
 * Amazon ENA network driver
 *
 */

FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );

#include <stdint.h>
#include <ipxe/if_ether.h>

/** Register BAR */
#define ENA_REGS_BAR PCI_BASE_ADDRESS_0

/** Register BAR size */
#define ENA_REGS_SIZE 16384

/** On-device memory BAR */
#define ENA_MEM_BAR PCI_BASE_ADDRESS_2

/** Queue alignment */
#define ENA_ALIGN 4096

/** Number of admin queue entries */
#define ENA_AQ_COUNT 2

/** Number of admin completion queue entries */
#define ENA_ACQ_COUNT 2

/** Number of async event notification queue entries */
#define ENA_AENQ_COUNT 2

/** Number of transmit queue entries */
#define ENA_TX_COUNT 32

/** Number of receive queue entries */
#define ENA_RX_COUNT 32

/** Base address low register offset */
#define ENA_BASE_LO 0x0

/** Base address high register offset */
#define ENA_BASE_HI 0x4

/** Capability register value */
#define ENA_CAPS( count, size ) ( ( (size) << 16 ) | ( (count) << 0 ) )

/** Admin queue base address register */
#define ENA_AQ_BASE 0x10

/** Admin queue capabilities register */
#define ENA_AQ_CAPS 0x18

/** Admin completion queue base address register */
#define ENA_ACQ_BASE 0x20

/** Admin completion queue capabilities register */
#define ENA_ACQ_CAPS 0x28

/** Admin queue doorbell register */
#define ENA_AQ_DB 0x2c

/** Maximum time to wait for admin requests */
#define ENA_ADMIN_MAX_WAIT_MS 5000

/** Async event notification queue capabilities register */
#define ENA_AENQ_CAPS 0x34

/** Async event notification queue base address register */
#define ENA_AENQ_BASE 0x38

/** Device control register */
#define ENA_CTRL 0x54
#define ENA_CTRL_RESET 0x00000001UL	/**< Reset */

/** Maximum time to wait for reset */
#define ENA_RESET_MAX_WAIT_MS 1000

/** Device status register */
#define ENA_STAT 0x58
#define ENA_STAT_RESET 0x00000008UL	/**< Reset in progress */

/** Admin queue entry header */
struct ena_aq_header {
	/** Request identifier */
	uint8_t id;
	/** Reserved */
	uint8_t reserved;
	/** Opcode */
	uint8_t opcode;
	/** Flags */
	uint8_t flags;
} __attribute__ (( packed ));

/** Admin queue ownership phase flag */
#define ENA_AQ_PHASE 0x01

/** Admin completion queue entry header */
struct ena_acq_header {
	/** Request identifier */
	uint8_t id;
	/** Reserved */
	uint8_t reserved;
	/** Status */
	uint8_t status;
	/** Flags */
	uint8_t flags;
	/** Extended status */
	uint16_t ext;
	/** Consumer index */
	uint16_t cons;
} __attribute__ (( packed ));

/** Admin completion queue ownership phase flag */
#define ENA_ACQ_PHASE 0x01

/** Device attributes */
#define ENA_DEVICE_ATTRIBUTES 1

/** Device attributes */
struct ena_device_attributes {
	/** Implementation */
	uint32_t implementation;
	/** Device version */
	uint32_t version;
	/** Supported features */
	uint32_t features;
	/** Reserved */
	uint8_t reserved_a[4];
	/** Physical address width */
	uint32_t physical;
	/** Virtual address width */
	uint32_t virtual;
	/** MAC address */
	uint8_t mac[ETH_ALEN];
	/** Reserved */
	uint8_t reserved_b[2];
	/** Maximum MTU */
	uint32_t mtu;
} __attribute__ (( packed ));

/** Device supports low latency queues */
#define ENA_FEATURE_LLQ 0x00000010

/** Low latency queue config */
#define ENA_LLQ_CONFIG 4

/** A low latency queue option */
struct ena_llq_option {
	/** Bitmask of supported option values */
	uint16_t supported;
	/** Single-entry bitmask of the enabled option value */
	uint16_t enabled;
} __attribute__ (( packed ));

/** Low latency queue config */
struct ena_llq_config {
	/** Maximum number of low latency queues */
	uint32_t queues;
	/** Maximum queue depth */
	uint32_t count;
	/** Header locations */
	struct ena_llq_option header;
	/** Entry sizes */
	struct ena_llq_option size;
	/** Descriptor counts */
	struct ena_llq_option desc;
	/** Descriptor strides */
	struct ena_llq_option stride;
	/** Reserved */
	uint8_t reserved_a[4];
	/** Acceleration mode */
	uint16_t mode;
	/** Maximum burst size */
	uint16_t burst;
	/** Reserved */
	uint8_t reserved_b[4];
} __attribute__ (( packed ));

/** Low latency queue header locations */
enum ena_llq_header {
	/** Headers are placed inline immediately after descriptors */
	ENA_LLQ_HEADER_INLINE = 0x0001,
};

/** Low latency queue entry sizes */
enum ena_llq_size {
	/** Entries are 128 bytes */
	ENA_LLQ_SIZE_128 = 0x0001,
};

/** Low latency queue descriptor count */
enum ena_llq_desc {
	/** Two descriptors before inline headers */
	ENA_LLQ_DESC_2 = 0x0002,
};

/** Async event notification queue config */
#define ENA_AENQ_CONFIG 26

/** Async event notification queue config */
struct ena_aenq_config {
	/** Bitmask of supported AENQ groups (device -> host) */
	uint32_t supported;
	/** Bitmask of enabled AENQ groups (host -> device) */
	uint32_t enabled;
} __attribute__ (( packed ));

/** Host attributes */
#define ENA_HOST_ATTRIBUTES 28

/** Host attributes */
struct ena_host_attributes {
	/** Host info base address */
	uint64_t info;
	/** Debug area base address */
	uint64_t debug;
	/** Debug area size */
	uint32_t debug_len;
} __attribute__ (( packed ));

/** Host information */
struct ena_host_info {
	/** Operating system type */
	uint32_t type;
	/** Operating system distribution (string) */
	char dist_str[128];
	/** Operating system distribution (numeric) */
	uint32_t dist;
	/** Kernel version (string) */
	char kernel_str[32];
	/** Kernel version (numeric) */
	uint32_t kernel;
	/** Driver version */
	uint32_t version;
	/** Linux network device features */
	uint64_t linux_features;
	/** ENA specification version */
	uint16_t spec;
	/** PCI bus:dev.fn address */
	uint16_t busdevfn;
	/** Number of CPUs */
	uint16_t cpus;
	/** Reserved */
	uint8_t reserved_a[2];
	/** Supported features */
	uint32_t features;
} __attribute__ (( packed ));

/** Operating system type
 *
 * Some very broken older versions of the ENA firmware will refuse to
 * allow a completion queue to be created if "iPXE" (type 5) is used,
 * and require us to pretend that we are "Linux" (type 1) instead.
 *
 * The ENA team at AWS assures us that the entire AWS fleet has been
 * upgraded to fix this bug, and that we are now safe to use the
 * correct operating system type value.
 */
#define ENA_HOST_INFO_TYPE_IPXE 5

/** Driver version
 *
 * The driver version field is nominally used to report a version
 * number outside of the VM for consumption by humans (and potentially
 * by automated monitoring tools that could e.g. check for outdated
 * versions with known security flaws).
 *
 * However, at some point in the development of the ENA firmware, some
 * unknown person at AWS thought it would be sensible to apply a
 * machine interpretation to this field and adjust the behaviour of
 * the firmware based on its value, thereby creating a maintenance and
 * debugging nightmare for all existing and future drivers.
 *
 * Hint to engineers: if you ever find yourself writing code of the
 * form "if (version == SOME_MAGIC_NUMBER)" then something has gone
 * very, very wrong.  This *always* indicates that something is
 * broken, either in your own code or in the code with which you are
 * forced to interact.
 */
#define ENA_HOST_INFO_VERSION_WTF 0x00000002UL

/** ENA specification version */
#define ENA_HOST_INFO_SPEC_2_0 0x0200

/** Feature */
union ena_feature {
	/** Device attributes */
	struct ena_device_attributes device;
	/** Low latency queue configuration */
	struct ena_llq_config llq;
	/** Async event notification queue config */
	struct ena_aenq_config aenq;
	/** Host attributes */
	struct ena_host_attributes host;
};

/** Submission queue direction */
enum ena_sq_direction {
	/** Transmit */
	ENA_SQ_TX = 0x20,
	/** Receive */
	ENA_SQ_RX = 0x40,
};

/** Create submission queue */
#define ENA_CREATE_SQ 1

/** Create submission queue request */
struct ena_create_sq_req {
	/** Header */
	struct ena_aq_header header;
	/** Direction */
	uint8_t direction;
	/** Reserved */
	uint8_t reserved_a;
	/** Policy */
	uint16_t policy;
	/** Completion queue identifier */
	uint16_t cq_id;
	/** Number of entries */
	uint16_t count;
	/** Base address */
	uint64_t address;
	/** Writeback address */
	uint64_t writeback;
	/** Reserved */
	uint8_t reserved_b[8];
} __attribute__ (( packed ));

/** Submission queue policy */
enum ena_sq_policy {
	/** Use host memory */
	ENA_SQ_HOST_MEMORY = 0x0001,
	/** Use on-device memory (must be used in addition to host memory) */
	ENA_SQ_DEVICE_MEMORY = 0x0002,
	/** Memory is contiguous */
	ENA_SQ_CONTIGUOUS = 0x0100,
};

/** Create submission queue response */
struct ena_create_sq_rsp {
	/** Header */
	struct ena_acq_header header;
	/** Submission queue identifier */
	uint16_t id;
	/** Reserved */
	uint8_t reserved_a[2];
	/** Doorbell register offset */
	uint32_t doorbell;
	/** LLQ descriptor ring offset */
	uint32_t llqe;
	/** Reserved */
	uint8_t reserved_b[4];
} __attribute__ (( packed ));

/** Destroy submission queue */
#define ENA_DESTROY_SQ 2

/** Destroy submission queue request */
struct ena_destroy_sq_req {
	/** Header */
	struct ena_aq_header header;
	/** Submission queue identifier */
	uint16_t id;
	/** Direction */
	uint8_t direction;
	/** Reserved */
	uint8_t reserved;
} __attribute__ (( packed ));

/** Destroy submission queue response */
struct ena_destroy_sq_rsp {
	/** Header */
	struct ena_acq_header header;
} __attribute__ (( packed ));

/** Create completion queue */
#define ENA_CREATE_CQ 3

/** Create completion queue request */
struct ena_create_cq_req {
	/** Header */
	struct ena_aq_header header;
	/** Interrupts enabled */
	uint8_t intr;
	/** Entry size (in 32-bit words) */
	uint8_t size;
	/** Number of entries */
	uint16_t count;
	/** MSI-X vector */
	uint32_t vector;
	/** Base address */
	uint64_t address;
} __attribute__ (( packed ));

/** Empty MSI-X vector
 *
 * Some versions of the ENA firmware will complain if the completion
 * queue's MSI-X vector field is left empty, even though the queue
 * configuration specifies that interrupts are not used.
 */
#define ENA_MSIX_NONE 0xffffffffUL

/** Create completion queue response */
struct ena_create_cq_rsp {
	/** Header */
	struct ena_acq_header header;
	/** Completion queue identifier */
	uint16_t id;
	/** Actual number of entries */
	uint16_t count;
	/** NUMA node register offset */
	uint32_t node;
	/** Doorbell register offset */
	uint32_t doorbell;
	/** Interrupt unmask register offset */
	uint32_t intr;
} __attribute__ (( packed ));

/** Destroy completion queue */
#define ENA_DESTROY_CQ 4

/** Destroy completion queue request */
struct ena_destroy_cq_req {
	/** Header */
	struct ena_aq_header header;
	/** Completion queue identifier */
	uint16_t id;
	/** Reserved */
	uint8_t reserved[2];
} __attribute__ (( packed ));

/** Destroy completion queue response */
struct ena_destroy_cq_rsp {
	/** Header */
	struct ena_acq_header header;
} __attribute__ (( packed ));

/** Get feature */
#define ENA_GET_FEATURE 8

/** Get feature request */
struct ena_get_feature_req {
	/** Header */
	struct ena_aq_header header;
	/** Length */
	uint32_t len;
	/** Address */
	uint64_t address;
	/** Flags */
	uint8_t flags;
	/** Feature identifier */
	uint8_t id;
	/** Reserved */
	uint8_t reserved[2];
} __attribute__ (( packed ));

/** Get feature response */
struct ena_get_feature_rsp {
	/** Header */
	struct ena_acq_header header;
	/** Feature */
	union ena_feature feature;
} __attribute__ (( packed ));

/** Set feature */
#define ENA_SET_FEATURE 9

/** Set feature request */
struct ena_set_feature_req {
	/** Header */
	struct ena_aq_header header;
	/** Length */
	uint32_t len;
	/** Address */
	uint64_t address;
	/** Flags */
	uint8_t flags;
	/** Feature identifier */
	uint8_t id;
	/** Reserved */
	uint8_t reserved[2];
	/** Feature */
	union ena_feature feature;
} __attribute__ (( packed ));

/** Get statistics */
#define ENA_GET_STATS 11

/** Get statistics request */
struct ena_get_stats_req {
	/** Header */
	struct ena_aq_header header;
	/** Reserved */
	uint8_t reserved_a[12];
	/** Type */
	uint8_t type;
	/** Scope */
	uint8_t scope;
	/** Reserved */
	uint8_t reserved_b[2];
	/** Queue ID */
	uint16_t queue;
	/** Device ID */
	uint16_t device;
} __attribute__ (( packed ));

/** Basic statistics */
#define ENA_STATS_TYPE_BASIC 0

/** Ethernet statistics */
#define ENA_STATS_SCOPE_ETH 1

/** My device */
#define ENA_DEVICE_MINE 0xffff

/** Get statistics response */
struct ena_get_stats_rsp {
	/** Header */
	struct ena_acq_header header;
	/** Transmit byte count */
	uint64_t tx_bytes;
	/** Transmit packet count */
	uint64_t tx_packets;
	/** Receive byte count */
	uint64_t rx_bytes;
	/** Receive packet count */
	uint64_t rx_packets;
	/** Receive drop count */
	uint64_t rx_drops;
} __attribute__ (( packed ));

/** Admin queue request */
union ena_aq_req {
	/** Header */
	struct ena_aq_header header;
	/** Create submission queue */
	struct ena_create_sq_req create_sq;
	/** Destroy submission queue */
	struct ena_destroy_sq_req destroy_sq;
	/** Create completion queue */
	struct ena_create_cq_req create_cq;
	/** Destroy completion queue */
	struct ena_destroy_cq_req destroy_cq;
	/** Get feature */
	struct ena_get_feature_req get_feature;
	/** Set feature */
	struct ena_set_feature_req set_feature;
	/** Get statistics */
	struct ena_get_stats_req get_stats;
	/** Padding */
	uint8_t pad[64];
};

/** Admin completion queue response */
union ena_acq_rsp {
	/** Header */
	struct ena_acq_header header;
	/** Create submission queue */
	struct ena_create_sq_rsp create_sq;
	/** Destroy submission queue */
	struct ena_destroy_sq_rsp destroy_sq;
	/** Create completion queue */
	struct ena_create_cq_rsp create_cq;
	/** Destroy completion queue */
	struct ena_destroy_cq_rsp destroy_cq;
	/** Get feature */
	struct ena_get_feature_rsp get_feature;
	/** Get statistics */
	struct ena_get_stats_rsp get_stats;
	/** Padding */
	uint8_t pad[64];
};

/** Admin queue */
struct ena_aq {
	/** Requests */
	union ena_aq_req *req;
	/** Producer counter */
	unsigned int prod;
};

/** Admin completion queue */
struct ena_acq {
	/** Responses */
	union ena_acq_rsp *rsp;
	/** Consumer counter */
	unsigned int cons;
	/** Phase */
	unsigned int phase;
};

/** Async event notification queue event */
struct ena_aenq_event {
	/** Type of event */
	uint16_t group;
	/** ID of event */
	uint16_t syndrome;
	/** Phase */
	uint8_t flags;
	/** Reserved */
	uint8_t reserved[3];
	/** Timestamp */
	uint64_t timestamp;
	/** Additional event data */
	uint8_t data[48];
} __attribute__ (( packed ));

/** Async event notification queue */
struct ena_aenq {
	/** Events */
	struct ena_aenq_event *evt;
};

/** Transmit submission queue entry */
struct ena_tx_sqe {
	/** Length */
	uint16_t len;
	/** Metadata flags */
	uint8_t meta;
	/** Flags */
	uint8_t flags;
	/** Reserved */
	uint8_t reserved_b[3];
	/** Request identifier */
	uint8_t id;
	/** Address and inlined length */
	union {
		/** Address */
		uint64_t address;
		/** Inlined length */
		struct {
			/** Reserved */
			uint8_t reserved[7];
			/** Inlined length */
			uint8_t inlined;
		} __attribute__ (( packed ));
	} __attribute__ (( packed ));
} __attribute__ (( packed ));

/** This is a metadata entry */
#define ENA_TX_SQE_META 0x80

/** Receive submission queue entry */
struct ena_rx_sqe {
	/** Length */
	uint16_t len;
	/** Reserved */
	uint8_t reserved_a;
	/** Flags */
	uint8_t flags;
	/** Request identifier */
	uint16_t id;
	/** Reserved */
	uint8_t reserved_b[2];
	/** Address */
	uint64_t address;
} __attribute__ (( packed ));

/** Submission queue ownership phase flag */
#define ENA_SQE_PHASE 0x01

/** This is the first descriptor */
#define ENA_SQE_FIRST 0x04

/** This is the last descriptor */
#define ENA_SQE_LAST 0x08

/** Request completion */
#define ENA_SQE_CPL 0x10

/** Transmit completion queue entry */
struct ena_tx_cqe {
	/** Request identifier */
	uint16_t id;
	/** Status */
	uint8_t status;
	/** Flags */
	uint8_t flags;
	/** Reserved */
	uint8_t reserved[2];
	/** Consumer index */
	uint16_t cons;
} __attribute__ (( packed ));

/** Transmit completion request identifier */
#define ENA_TX_CQE_ID(id) ( (id) >> 2 )

/** Receive completion queue entry */
struct ena_rx_cqe {
	/** Reserved */
	uint8_t reserved_a[3];
	/** Flags */
	uint8_t flags;
	/** Length */
	uint16_t len;
	/** Request identifier */
	uint16_t id;
	/** Reserved */
	uint8_t reserved_b[8];
} __attribute__ (( packed ));

/** Completion queue ownership phase flag */
#define ENA_CQE_PHASE 0x01

/** Low latency transmit queue bounce buffer */
struct ena_tx_llqe {
	/** Pointless metadata descriptor */
	struct ena_tx_sqe meta;
	/** Transmit descriptor */
	struct ena_tx_sqe sqe;
	/** Inlined header data */
	uint8_t inlined[96];
} __attribute__ (( packed ));

/** Submission queue */
struct ena_sq {
	/** Entries */
	union {
		/** Transmit submission queue entries */
		struct ena_tx_sqe *tx;
		/** Receive submission queue entries */
		struct ena_rx_sqe *rx;
		/** Low latency queue bounce buffer */
		struct ena_tx_llqe *llq;
		/** Raw data */
		void *raw;
	} sqe;
	/** Buffer IDs */
	uint8_t *ids;
	/** Low latency queue base */
	void *llqe;
	/** Doorbell register offset */
	unsigned int doorbell;
	/** Total length of entries */
	size_t len;
	/** Producer counter */
	unsigned int prod;
	/** Phase */
	unsigned int phase;
	/** Queue policy */
	uint16_t policy;
	/** Submission queue identifier */
	uint16_t id;
	/** Direction */
	uint8_t direction;
	/** Number of entries */
	uint8_t count;
	/** Fill level (limited to completion queue size) */
	uint8_t fill;
	/** Maximum inline header length */
	uint8_t inlined;
};

/**
 * Initialise submission queue
 *
 * @v sq		Submission queue
 * @v direction		Direction
 * @v count		Number of entries
 * @v size		Size of each entry
 * @v ids		Buffer IDs
 */
static inline __attribute__ (( always_inline )) void
ena_sq_init ( struct ena_sq *sq, unsigned int direction, unsigned int count,
	      size_t size, uint8_t *ids ) {

	sq->len = ( count * size );
	sq->policy = ( ENA_SQ_HOST_MEMORY | ENA_SQ_CONTIGUOUS );
	sq->direction = direction;
	sq->count = count;
	sq->ids = ids;
}

/** Completion queue */
struct ena_cq {
	/** Entries */
	union {
		/** Transmit completion queue entries */
		struct ena_tx_cqe *tx;
		/** Receive completion queue entries */
		struct ena_rx_cqe *rx;
		/** Raw data */
		void *raw;
	} cqe;
	/** Doorbell register offset */
	unsigned int doorbell;
	/** Total length of entries */
	size_t len;
	/** Consumer counter */
	unsigned int cons;
	/** Phase */
	unsigned int phase;
	/** Completion queue identifier */
	uint16_t id;
	/** Entry size (in 32-bit words) */
	uint8_t size;
	/** Requested number of entries */
	uint8_t requested;
	/** Actual number of entries */
	uint8_t actual;
	/** Actual number of entries minus one */
	uint8_t mask;
};

/**
 * Initialise completion queue
 *
 * @v cq		Completion queue
 * @v count		Number of entries
 * @v size		Size of each entry
 */
static inline __attribute__ (( always_inline )) void
ena_cq_init ( struct ena_cq *cq, unsigned int count, size_t size ) {

	cq->len = ( count * size );
	cq->size = ( size / sizeof ( uint32_t ) );
	cq->requested = count;
}

/** Queue pair */
struct ena_qp {
	/** Submission queue */
	struct ena_sq sq;
	/** Completion queue */
	struct ena_cq cq;
};

/** An ENA network card */
struct ena_nic {
	/** Registers */
	void *regs;
	/** On-device memory */
	void *mem;
	/** Device features */
	uint32_t features;
	/** Host info */
	struct ena_host_info *info;
	/** Admin queue */
	struct ena_aq aq;
	/** Admin completion queue */
	struct ena_acq acq;
	/** Async event notification queue */
	struct ena_aenq aenq;
	/** Transmit queue */
	struct ena_qp tx;
	/** Receive queue */
	struct ena_qp rx;
	/** Transmit buffer IDs */
	uint8_t tx_ids[ENA_TX_COUNT];
	/** Transmit I/O buffers, indexed by buffer ID */
	struct io_buffer *tx_iobuf[ENA_TX_COUNT];
	/** Receive buffer IDs */
	uint8_t rx_ids[ENA_RX_COUNT];
	/** Receive I/O buffers, indexed by buffer ID */
	struct io_buffer *rx_iobuf[ENA_RX_COUNT];
};

#endif /* _ENA_H */