src/drivers/net/intelxl.c

/*
 * Copyright (C) 2018 Michael Brown <mbrown@fensystems.co.uk>.
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public License as
 * published by the Free Software Foundation; either version 2 of the
 * License, or (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
 * 02110-1301, USA.
 *
 * You can also choose to distribute this program under the terms of
 * the Unmodified Binary Distribution Licence (as given in the file
 * COPYING.UBDL), provided that you have satisfied its requirements.
 */

FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
FILE_SECBOOT ( PERMITTED );

#include <stdint.h>
#include <string.h>
#include <stdio.h>
#include <unistd.h>
#include <errno.h>
#include <byteswap.h>
#include <ipxe/netdevice.h>
#include <ipxe/ethernet.h>
#include <ipxe/if_ether.h>
#include <ipxe/vlan.h>
#include <ipxe/iobuf.h>
#include <ipxe/pci.h>
#include <ipxe/version.h>
#include "intelxl.h"

/** @file
 *
 * Intel 40 Gigabit Ethernet network card driver
 *
 */

/******************************************************************************
 *
 * MSI-X interrupts
 *
 ******************************************************************************
 */

/**
 * Enable MSI-X dummy interrupt
 *
 * @v intelxl		Intel device
 * @v pci		PCI device
 * @v vector		MSI-X vector
 * @ret rc		Return status code
 */
int intelxl_msix_enable ( struct intelxl_nic *intelxl,
			  struct pci_device *pci, unsigned int vector ) {
	int rc;

	/* Enable MSI-X capability */
	if ( ( rc = pci_msix_enable ( pci, &intelxl->msix ) ) != 0 ) {
		DBGC ( intelxl, "INTELXL %p could not enable MSI-X: %s\n",
		       intelxl, strerror ( rc ) );
		goto err_enable;
	}

	/* Enable dummy interrupt */
	pci_msix_unmask ( &intelxl->msix, vector );

	return 0;

	pci_msix_disable ( pci, &intelxl->msix );
 err_enable:
	return rc;
}

/**
 * Disable MSI-X dummy interrupt
 *
 * @v intelxl		Intel device
 * @v pci		PCI device
 * @v vector		MSI-X vector
 */
void intelxl_msix_disable ( struct intelxl_nic *intelxl,
			    struct pci_device *pci, unsigned int vector ) {

	/* Disable dummy interrupts */
	pci_msix_mask ( &intelxl->msix, vector );

	/* Disable MSI-X capability */
	pci_msix_disable ( pci, &intelxl->msix );
}

/******************************************************************************
 *
 * Admin queue
 *
 ******************************************************************************
 */

/** Admin queue register offsets */
const struct intelxl_admin_offsets intelxl_admin_offsets = {
	.bal = INTELXL_ADMIN_BAL,
	.bah = INTELXL_ADMIN_BAH,
	.len = INTELXL_ADMIN_LEN,
	.head = INTELXL_ADMIN_HEAD,
	.tail = INTELXL_ADMIN_TAIL,
};

/**
 * Allocate admin queue
 *
 * @v intelxl		Intel device
 * @v admin		Admin queue
 * @ret rc		Return status code
 */
static int intelxl_alloc_admin ( struct intelxl_nic *intelxl,
				 struct intelxl_admin *admin ) {
	size_t buf_len = ( sizeof ( admin->buf[0] ) * INTELXL_ADMIN_NUM_DESC );
	size_t len = ( sizeof ( admin->desc[0] ) * INTELXL_ADMIN_NUM_DESC );

	/* Allocate admin queue */
	admin->buf = dma_alloc ( intelxl->dma, &admin->map, ( buf_len + len ),
				 INTELXL_ALIGN );
	if ( ! admin->buf )
		return -ENOMEM;
	admin->desc = ( ( ( void * ) admin->buf ) + buf_len );

	DBGC ( intelxl, "INTELXL %p A%cQ is at [%08lx,%08lx) buf "
	       "[%08lx,%08lx)\n", intelxl,
	       ( ( admin == &intelxl->command ) ? 'T' : 'R' ),
	       virt_to_phys ( admin->desc ),
	       ( virt_to_phys ( admin->desc ) + len ),
	       virt_to_phys ( admin->buf ),
	       ( virt_to_phys ( admin->buf ) + buf_len ) );
	return 0;
}

/**
 * Enable admin queue
 *
 * @v intelxl		Intel device
 * @v admin		Admin queue
 */
static void intelxl_enable_admin ( struct intelxl_nic *intelxl,
				   struct intelxl_admin *admin ) {
	size_t len = ( sizeof ( admin->desc[0] ) * INTELXL_ADMIN_NUM_DESC );
	const struct intelxl_admin_offsets *regs = admin->regs;
	void *admin_regs = ( intelxl->regs + admin->base );
	physaddr_t address;

	/* Initialise admin queue */
	memset ( admin->desc, 0, len );

	/* Reset head and tail registers */
	writel ( 0, admin_regs + regs->head );
	writel ( 0, admin_regs + regs->tail );

	/* Reset queue index */
	admin->index = 0;

	/* Program queue address */
	address = dma ( &admin->map, admin->desc );
	writel ( ( address & 0xffffffffUL ), admin_regs + regs->bal );
	if ( sizeof ( physaddr_t ) > sizeof ( uint32_t ) ) {
		writel ( ( ( ( uint64_t ) address ) >> 32 ),
			 admin_regs + regs->bah );
	} else {
		writel ( 0, admin_regs + regs->bah );
	}

	/* Program queue length and enable queue */
	writel ( ( INTELXL_ADMIN_LEN_LEN ( INTELXL_ADMIN_NUM_DESC ) |
		   INTELXL_ADMIN_LEN_ENABLE ),
		 admin_regs + regs->len );
}

/**
 * Disable admin queue
 *
 * @v intelxl		Intel device
 * @v admin		Admin queue
 */
static void intelxl_disable_admin ( struct intelxl_nic *intelxl,
				    struct intelxl_admin *admin ) {
	const struct intelxl_admin_offsets *regs = admin->regs;
	void *admin_regs = ( intelxl->regs + admin->base );

	/* Disable queue */
	writel ( 0, admin_regs + regs->len );
}

/**
 * Free admin queue
 *
 * @v intelxl		Intel device
 * @v admin		Admin queue
 */
static void intelxl_free_admin ( struct intelxl_nic *intelxl __unused,
				 struct intelxl_admin *admin ) {
	size_t buf_len = ( sizeof ( admin->buf[0] ) * INTELXL_ADMIN_NUM_DESC );
	size_t len = ( sizeof ( admin->desc[0] ) * INTELXL_ADMIN_NUM_DESC );

	/* Free queue */
	dma_free ( &admin->map, admin->buf, ( buf_len + len ) );
}

/**
 * Get next admin command queue descriptor
 *
 * @v intelxl		Intel device
 * @ret cmd		Command descriptor
 */
struct intelxl_admin_descriptor *
intelxl_admin_command_descriptor ( struct intelxl_nic *intelxl ) {
	struct intelxl_admin *admin = &intelxl->command;
	struct intelxl_admin_descriptor *cmd;

	/* Get and initialise next descriptor */
	cmd = &admin->desc[ admin->index % INTELXL_ADMIN_NUM_DESC ];
	memset ( cmd, 0, sizeof ( *cmd ) );
	return cmd;
}

/**
 * Get next admin command queue data buffer
 *
 * @v intelxl		Intel device
 * @ret buf		Data buffer
 */
union intelxl_admin_buffer *
intelxl_admin_command_buffer ( struct intelxl_nic *intelxl ) {
	struct intelxl_admin *admin = &intelxl->command;
	union intelxl_admin_buffer *buf;

	/* Get next data buffer */
	buf = &admin->buf[ admin->index % INTELXL_ADMIN_NUM_DESC ];
	memset ( buf, 0, sizeof ( *buf ) );
	return buf;
}

/**
 * Initialise admin event queue descriptor
 *
 * @v intelxl		Intel device
 * @v index		Event queue index
 */
static void intelxl_admin_event_init ( struct intelxl_nic *intelxl,
				       unsigned int index ) {
	struct intelxl_admin *admin = &intelxl->event;
	struct intelxl_admin_descriptor *evt;
	union intelxl_admin_buffer *buf;
	uint64_t address;

	/* Initialise descriptor */
	evt = &admin->desc[ index % INTELXL_ADMIN_NUM_DESC ];
	buf = &admin->buf[ index % INTELXL_ADMIN_NUM_DESC ];
	address = dma ( &admin->map, buf );
	evt->flags = cpu_to_le16 ( INTELXL_ADMIN_FL_BUF );
	evt->len = cpu_to_le16 ( sizeof ( *buf ) );
	evt->params.buffer.high = cpu_to_le32 ( address >> 32 );
	evt->params.buffer.low = cpu_to_le32 ( address & 0xffffffffUL );
}

/**
 * Issue admin queue command
 *
 * @v intelxl		Intel device
 * @ret rc		Return status code
 */
int intelxl_admin_command ( struct intelxl_nic *intelxl ) {
	struct intelxl_admin *admin = &intelxl->command;
	const struct intelxl_admin_offsets *regs = admin->regs;
	void *admin_regs = ( intelxl->regs + admin->base );
	struct intelxl_admin_descriptor *cmd;
	union intelxl_admin_buffer *buf;
	uint64_t address;
	uint32_t cookie;
	uint16_t silence;
	unsigned int index;
	unsigned int tail;
	unsigned int i;
	int rc;

	/* Get next queue entry */
	index = admin->index++;
	tail = ( admin->index % INTELXL_ADMIN_NUM_DESC );
	cmd = &admin->desc[ index % INTELXL_ADMIN_NUM_DESC ];
	buf = &admin->buf[ index % INTELXL_ADMIN_NUM_DESC ];
	DBGC2 ( intelxl, "INTELXL %p admin command %#x opcode %#04x",
		intelxl, index, le16_to_cpu ( cmd->opcode ) );
	if ( cmd->cookie )
		DBGC2 ( intelxl, "/%#08x", le32_to_cpu ( cmd->cookie ) );
	DBGC2 ( intelxl, ":\n" );

	/* Allow expected errors to be silenced */
	silence = cmd->ret;
	cmd->ret = 0;

	/* Sanity checks */
	assert ( ! ( cmd->flags & cpu_to_le16 ( INTELXL_ADMIN_FL_DD ) ) );
	assert ( ! ( cmd->flags & cpu_to_le16 ( INTELXL_ADMIN_FL_CMP ) ) );
	assert ( ! ( cmd->flags & cpu_to_le16 ( INTELXL_ADMIN_FL_ERR ) ) );

	/* Populate data buffer address if applicable */
	if ( cmd->flags & cpu_to_le16 ( INTELXL_ADMIN_FL_BUF ) ) {
		address = dma ( &admin->map, buf );
		cmd->params.buffer.high = cpu_to_le32 ( address >> 32 );
		cmd->params.buffer.low = cpu_to_le32 ( address & 0xffffffffUL );
	}

	/* Populate cookie, if not being (ab)used for VF opcode */
	if ( ! cmd->cookie )
		cmd->cookie = cpu_to_le32 ( index );

	/* Record cookie */
	cookie = cmd->cookie;

	/* Post command descriptor */
	DBGC2_HDA ( intelxl, virt_to_phys ( cmd ), cmd, sizeof ( *cmd ) );
	if ( cmd->flags & cpu_to_le16 ( INTELXL_ADMIN_FL_RD ) ) {
		DBGC2_HDA ( intelxl, virt_to_phys ( buf ), buf,
			    le16_to_cpu ( cmd->len ) );
	}
	wmb();
	writel ( tail, admin_regs + regs->tail );

	/* Wait for completion */
	for ( i = 0 ; i < INTELXL_ADMIN_MAX_WAIT_MS ; i++ ) {

		/* If response is not complete, delay 1ms and retry */
		if ( ! ( cmd->flags & cpu_to_le16 ( INTELXL_ADMIN_FL_DD ) ) ) {
			mdelay ( 1 );
			continue;
		}
		DBGC2 ( intelxl, "INTELXL %p admin command %#x response:\n",
			intelxl, index );
		DBGC2_HDA ( intelxl, virt_to_phys ( cmd ), cmd,
			    sizeof ( *cmd ) );
		if ( cmd->flags & cpu_to_le16 ( INTELXL_ADMIN_FL_BUF ) ) {
			DBGC2_HDA ( intelxl, virt_to_phys ( buf ), buf,
				    le16_to_cpu ( cmd->len ) );
		}

		/* Check for cookie mismatch */
		if ( cmd->cookie != cookie ) {
			DBGC ( intelxl, "INTELXL %p admin command %#x bad "
			       "cookie %#x\n", intelxl, index,
			       le32_to_cpu ( cmd->cookie ) );
			rc = -EPROTO;
			goto err;
		}

		/* Check for unexpected errors */
		if ( ( cmd->ret != 0 ) && ( cmd->ret != silence ) ) {
			DBGC ( intelxl, "INTELXL %p admin command %#x error "
			       "%d\n", intelxl, index,
			       le16_to_cpu ( cmd->ret ) );
			rc = -EIO;
			goto err;
		}

		/* Success */
		return 0;
	}

	rc = -ETIMEDOUT;
	DBGC ( intelxl, "INTELXL %p timed out waiting for admin command %#x:\n",
	       intelxl, index );
 err:
	DBGC_HDA ( intelxl, virt_to_phys ( cmd ), cmd, sizeof ( *cmd ) );
	return rc;
}

/**
 * Get firmware version
 *
 * @v intelxl		Intel device
 * @ret rc		Return status code
 */
static int intelxl_admin_version ( struct intelxl_nic *intelxl ) {
	struct intelxl_admin_descriptor *cmd;
	struct intelxl_admin_version_params *version;
	unsigned int api;
	int rc;

	/* Populate descriptor */
	cmd = intelxl_admin_command_descriptor ( intelxl );
	cmd->opcode = cpu_to_le16 ( INTELXL_ADMIN_VERSION );
	version = &cmd->params.version;

	/* Issue command */
	if ( ( rc = intelxl_admin_command ( intelxl ) ) != 0 )
		return rc;
	api = le16_to_cpu ( version->api.major );
	DBGC ( intelxl, "INTELXL %p firmware v%d.%d API v%d.%d\n",
	       intelxl, le16_to_cpu ( version->firmware.major ),
	       le16_to_cpu ( version->firmware.minor ),
	       api, le16_to_cpu ( version->api.minor ) );

	/* Check for API compatibility */
	if ( api > INTELXL_ADMIN_API_MAJOR ) {
		DBGC ( intelxl, "INTELXL %p unsupported API v%d\n",
		       intelxl, api );
		return -ENOTSUP;
	}

	return 0;
}

/**
 * Report driver version
 *
 * @v intelxl		Intel device
 * @ret rc		Return status code
 */
static int intelxl_admin_driver ( struct intelxl_nic *intelxl ) {
	struct intelxl_admin_descriptor *cmd;
	struct intelxl_admin_driver_params *driver;
	union intelxl_admin_buffer *buf;
	int rc;

	/* Populate descriptor */
	cmd = intelxl_admin_command_descriptor ( intelxl );
	cmd->opcode = cpu_to_le16 ( INTELXL_ADMIN_DRIVER );
	cmd->flags = cpu_to_le16 ( INTELXL_ADMIN_FL_RD | INTELXL_ADMIN_FL_BUF );
	cmd->len = cpu_to_le16 ( sizeof ( buf->driver ) );
	driver = &cmd->params.driver;
	driver->major = product_major_version;
	driver->minor = product_minor_version;
	buf = intelxl_admin_command_buffer ( intelxl );
	snprintf ( buf->driver.name, sizeof ( buf->driver.name ), "%s",
		   ( product_name[0] ? product_name : product_short_name ) );

	/* Issue command */
	if ( ( rc = intelxl_admin_command ( intelxl ) ) != 0 )
		return rc;

	return 0;
}

/**
 * Shutdown admin queues
 *
 * @v intelxl		Intel device
 * @ret rc		Return status code
 */
static int intelxl_admin_shutdown ( struct intelxl_nic *intelxl ) {
	struct intelxl_admin_descriptor *cmd;
	struct intelxl_admin_shutdown_params *shutdown;
	int rc;

	/* Populate descriptor */
	cmd = intelxl_admin_command_descriptor ( intelxl );
	cmd->opcode = cpu_to_le16 ( INTELXL_ADMIN_SHUTDOWN );
	shutdown = &cmd->params.shutdown;
	shutdown->unloading = INTELXL_ADMIN_SHUTDOWN_UNLOADING;

	/* Issue command */
	if ( ( rc = intelxl_admin_command ( intelxl ) ) != 0 )
		return rc;

	return 0;
}

/**
 * Get MAC address
 *
 * @v netdev		Network device
 * @ret rc		Return status code
 */
static int intelxl_admin_mac_read ( struct net_device *netdev ) {
	struct intelxl_nic *intelxl = netdev->priv;
	struct intelxl_admin_descriptor *cmd;
	struct intelxl_admin_mac_read_params *read;
	union intelxl_admin_buffer *buf;
	uint8_t *mac;
	int rc;

	/* Populate descriptor */
	cmd = intelxl_admin_command_descriptor ( intelxl );
	cmd->opcode = cpu_to_le16 ( INTELXL_ADMIN_MAC_READ );
	cmd->flags = cpu_to_le16 ( INTELXL_ADMIN_FL_BUF );
	cmd->len = cpu_to_le16 ( sizeof ( buf->mac_read ) );
	read = &cmd->params.mac_read;
	buf = intelxl_admin_command_buffer ( intelxl );
	mac = buf->mac_read.pf;

	/* Issue command */
	if ( ( rc = intelxl_admin_command ( intelxl ) ) != 0 )
		return rc;

	/* Check that MAC address is present in response */
	if ( ! ( read->valid & INTELXL_ADMIN_MAC_READ_VALID_LAN ) ) {
		DBGC ( intelxl, "INTELXL %p has no MAC address\n", intelxl );
		return -ENOENT;
	}

	/* Check that address is valid */
	if ( ! is_valid_ether_addr ( mac ) ) {
		DBGC ( intelxl, "INTELXL %p has invalid MAC address (%s)\n",
		       intelxl, eth_ntoa ( mac ) );
		return -ENOENT;
	}

	/* Copy MAC address */
	DBGC ( intelxl, "INTELXL %p has MAC address %s\n",
	       intelxl, eth_ntoa ( mac ) );
	memcpy ( netdev->hw_addr, mac, ETH_ALEN );

	return 0;
}

/**
 * Set MAC address
 *
 * @v netdev		Network device
 * @ret rc		Return status code
 */
static int intelxl_admin_mac_write ( struct net_device *netdev ) {
	struct intelxl_nic *intelxl = netdev->priv;
	struct intelxl_admin_descriptor *cmd;
	struct intelxl_admin_mac_write_params *write;
	union {
		uint8_t raw[ETH_ALEN];
		struct {
			uint16_t high;
			uint32_t low;
		} __attribute__ (( packed ));
	} mac;
	int rc;

	/* Populate descriptor */
	cmd = intelxl_admin_command_descriptor ( intelxl );
	cmd->opcode = cpu_to_le16 ( INTELXL_ADMIN_MAC_WRITE );
	write = &cmd->params.mac_write;
	memcpy ( mac.raw, netdev->ll_addr, ETH_ALEN );
	write->high = bswap_16 ( mac.high );
	write->low = bswap_32 ( mac.low );

	/* Issue command */
	if ( ( rc = intelxl_admin_command ( intelxl ) ) != 0 )
		return rc;

	return 0;
}

/**
 * Clear PXE mode
 *
 * @v intelxl		Intel device
 * @ret rc		Return status code
 */
int intelxl_admin_clear_pxe ( struct intelxl_nic *intelxl ) {
	struct intelxl_admin_descriptor *cmd;
	struct intelxl_admin_clear_pxe_params *pxe;
	int rc;

	/* Populate descriptor */
	cmd = intelxl_admin_command_descriptor ( intelxl );
	cmd->opcode = cpu_to_le16 ( INTELXL_ADMIN_CLEAR_PXE );
	cmd->ret = cpu_to_le16 ( INTELXL_ADMIN_EEXIST );
	pxe = &cmd->params.pxe;
	pxe->magic = INTELXL_ADMIN_CLEAR_PXE_MAGIC;

	/* Issue command */
	if ( ( rc = intelxl_admin_command ( intelxl ) ) != 0 )
		return rc;

	/* Check for expected errors */
	if ( cmd->ret == cpu_to_le16 ( INTELXL_ADMIN_EEXIST ) ) {
		DBGC ( intelxl, "INTELXL %p already in non-PXE mode\n",
		       intelxl );
		return 0;
	}

	return 0;
}

/**
 * Get switch configuration
 *
 * @v intelxl		Intel device
 * @ret rc		Return status code
 */
static int intelxl_admin_switch ( struct intelxl_nic *intelxl ) {
	struct intelxl_admin_descriptor *cmd;
	struct intelxl_admin_switch_params *sw;
	union intelxl_admin_buffer *buf;
	uint16_t next = 0;
	int rc;

	/* Get each configuration in turn */
	do {
		/* Populate descriptor */
		cmd = intelxl_admin_command_descriptor ( intelxl );
		cmd->opcode = cpu_to_le16 ( INTELXL_ADMIN_SWITCH );
		cmd->flags = cpu_to_le16 ( INTELXL_ADMIN_FL_BUF );
		cmd->len = cpu_to_le16 ( sizeof ( buf->sw ) );
		sw = &cmd->params.sw;
		sw->next = next;
		buf = intelxl_admin_command_buffer ( intelxl );

		/* Issue command */
		if ( ( rc = intelxl_admin_command ( intelxl ) ) != 0 )
			return rc;

		/* Dump raw configuration */
		DBGC2 ( intelxl, "INTELXL %p SEID %#04x:\n",
			intelxl, le16_to_cpu ( buf->sw.cfg.seid ) );
		DBGC2_HDA ( intelxl, 0, &buf->sw.cfg, sizeof ( buf->sw.cfg ) );

		/* Parse response */
		if ( buf->sw.cfg.type == INTELXL_ADMIN_SWITCH_TYPE_VSI ) {
			intelxl->vsi = le16_to_cpu ( buf->sw.cfg.seid );
			DBGC ( intelxl, "INTELXL %p VSI %#04x uplink %#04x "
			       "downlink %#04x conn %#02x\n", intelxl,
			       intelxl->vsi, le16_to_cpu ( buf->sw.cfg.uplink ),
			       le16_to_cpu ( buf->sw.cfg.downlink ),
			       buf->sw.cfg.connection );
		}

	} while ( ( next = sw->next ) );

	/* Check that we found a VSI */
	if ( ! intelxl->vsi ) {
		DBGC ( intelxl, "INTELXL %p has no VSI\n", intelxl );
		return -ENOENT;
	}

	return 0;
}

/**
 * Get VSI parameters
 *
 * @v intelxl		Intel device
 * @ret rc		Return status code
 */
static int intelxl_admin_vsi ( struct intelxl_nic *intelxl ) {
	struct intelxl_admin_descriptor *cmd;
	struct intelxl_admin_vsi_params *vsi;
	union intelxl_admin_buffer *buf;
	int rc;

	/* Populate descriptor */
	cmd = intelxl_admin_command_descriptor ( intelxl );
	cmd->opcode = cpu_to_le16 ( INTELXL_ADMIN_VSI );
	cmd->flags = cpu_to_le16 ( INTELXL_ADMIN_FL_BUF );
	cmd->len = cpu_to_le16 ( sizeof ( buf->vsi ) );
	vsi = &cmd->params.vsi;
	vsi->vsi = cpu_to_le16 ( intelxl->vsi );
	buf = intelxl_admin_command_buffer ( intelxl );

	/* Issue command */
	if ( ( rc = intelxl_admin_command ( intelxl ) ) != 0 )
		return rc;

	/* Parse response */
	intelxl->queue = le16_to_cpu ( buf->vsi.queue[0] );
	intelxl->qset = le16_to_cpu ( buf->vsi.qset[0] );
	DBGC ( intelxl, "INTELXL %p VSI %#04x queue %#04x qset %#04x\n",
	       intelxl, intelxl->vsi, intelxl->queue, intelxl->qset );

	return 0;
}

/**
 * Set VSI promiscuous modes
 *
 * @v intelxl		Intel device
 * @ret rc		Return status code
 */
static int intelxl_admin_promisc ( struct intelxl_nic *intelxl ) {
	struct intelxl_admin_descriptor *cmd;
	struct intelxl_admin_promisc_params *promisc;
	uint16_t flags;
	int rc;

	/* Populate descriptor */
	cmd = intelxl_admin_command_descriptor ( intelxl );
	cmd->opcode = cpu_to_le16 ( INTELXL_ADMIN_PROMISC );
	flags = ( INTELXL_ADMIN_PROMISC_FL_UNICAST |
		  INTELXL_ADMIN_PROMISC_FL_MULTICAST |
		  INTELXL_ADMIN_PROMISC_FL_BROADCAST |
		  INTELXL_ADMIN_PROMISC_FL_VLAN );
	promisc = &cmd->params.promisc;
	promisc->flags = cpu_to_le16 ( flags );
	promisc->valid = cpu_to_le16 ( flags );
	promisc->vsi = cpu_to_le16 ( intelxl->vsi );

	/* Issue command */
	if ( ( rc = intelxl_admin_command ( intelxl ) ) != 0 )
		return rc;

	return 0;
}

/**
 * Set MAC configuration
 *
 * @v intelxl		Intel device
 * @ret rc		Return status code
 */
int intelxl_admin_mac_config ( struct intelxl_nic *intelxl ) {
	struct intelxl_admin_descriptor *cmd;
	struct intelxl_admin_mac_config_params *config;
	int rc;

	/* Populate descriptor */
	cmd = intelxl_admin_command_descriptor ( intelxl );
	cmd->opcode = cpu_to_le16 ( INTELXL_ADMIN_MAC_CONFIG );
	config = &cmd->params.mac_config;
	config->mfs = cpu_to_le16 ( intelxl->mfs );
	config->flags = INTELXL_ADMIN_MAC_CONFIG_FL_CRC;

	/* Issue command */
	if ( ( rc = intelxl_admin_command ( intelxl ) ) != 0 )
		return rc;

	return 0;
}

/**
 * Restart autonegotiation
 *
 * @v intelxl		Intel device
 * @ret rc		Return status code
 */
static int intelxl_admin_autoneg ( struct intelxl_nic *intelxl ) {
	struct intelxl_admin_descriptor *cmd;
	struct intelxl_admin_autoneg_params *autoneg;
	int rc;

	/* Populate descriptor */
	cmd = intelxl_admin_command_descriptor ( intelxl );
	cmd->opcode = cpu_to_le16 ( INTELXL_ADMIN_AUTONEG );
	autoneg = &cmd->params.autoneg;
	autoneg->flags = ( INTELXL_ADMIN_AUTONEG_FL_RESTART |
			   INTELXL_ADMIN_AUTONEG_FL_ENABLE );

	/* Issue command */
	if ( ( rc = intelxl_admin_command ( intelxl ) ) != 0 )
		return rc;

	return 0;
}

/**
 * Get link status
 *
 * @v netdev		Network device
 * @ret rc		Return status code
 */
static int intelxl_admin_link ( struct net_device *netdev ) {
	struct intelxl_nic *intelxl = netdev->priv;
	struct intelxl_admin_descriptor *cmd;
	struct intelxl_admin_link_params *link;
	int rc;

	/* Populate descriptor */
	cmd = intelxl_admin_command_descriptor ( intelxl );
	cmd->opcode = cpu_to_le16 ( INTELXL_ADMIN_LINK );
	link = &cmd->params.link;
	link->notify = INTELXL_ADMIN_LINK_NOTIFY;

	/* Issue command */
	if ( ( rc = intelxl_admin_command ( intelxl ) ) != 0 )
		return rc;
	DBGC ( intelxl, "INTELXL %p PHY %#02x speed %#02x status %#02x\n",
	       intelxl, link->phy, link->speed, link->status );

	/* Update network device */
	if ( link->status & INTELXL_ADMIN_LINK_UP ) {
		netdev_link_up ( netdev );
	} else {
		netdev_link_down ( netdev );
	}

	return 0;
}

/**
 * Handle admin event
 *
 * @v netdev		Network device
 * @v evt		Event descriptor
 * @v buf		Data buffer
 */
static void intelxl_admin_event ( struct net_device *netdev,
				  struct intelxl_admin_descriptor *evt,
				  union intelxl_admin_buffer *buf __unused ) {
	struct intelxl_nic *intelxl = netdev->priv;

	/* Ignore unrecognised events */
	if ( evt->opcode != cpu_to_le16 ( INTELXL_ADMIN_LINK ) ) {
		DBGC ( intelxl, "INTELXL %p unrecognised event opcode "
		       "%#04x\n", intelxl, le16_to_cpu ( evt->opcode ) );
		return;
	}

	/* Update link status */
	intelxl_admin_link ( netdev );
}

/**
 * Refill admin event queue
 *
 * @v intelxl		Intel device
 */
static void intelxl_refill_admin ( struct intelxl_nic *intelxl ) {
	struct intelxl_admin *admin = &intelxl->event;
	const struct intelxl_admin_offsets *regs = admin->regs;
	void *admin_regs = ( intelxl->regs + admin->base );
	unsigned int tail;

	/* Update tail pointer */
	tail = ( ( admin->index + INTELXL_ADMIN_NUM_DESC - 1 ) %
		 INTELXL_ADMIN_NUM_DESC );
	wmb();
	writel ( tail, admin_regs + regs->tail );
}

/**
 * Poll admin event queue
 *
 * @v netdev		Network device
 */
void intelxl_poll_admin ( struct net_device *netdev ) {
	struct intelxl_nic *intelxl = netdev->priv;
	struct intelxl_admin *admin = &intelxl->event;
	struct intelxl_admin_descriptor *evt;
	union intelxl_admin_buffer *buf;

	/* Check for events */
	while ( 1 ) {

		/* Get next event descriptor and data buffer */
		evt = &admin->desc[ admin->index % INTELXL_ADMIN_NUM_DESC ];
		buf = &admin->buf[ admin->index % INTELXL_ADMIN_NUM_DESC ];

		/* Stop if descriptor is not yet completed */
		if ( ! ( evt->flags & cpu_to_le16 ( INTELXL_ADMIN_FL_DD ) ) )
			return;
		DBGC2 ( intelxl, "INTELXL %p admin event %#x:\n",
			intelxl, admin->index );
		DBGC2_HDA ( intelxl, virt_to_phys ( evt ), evt,
			    sizeof ( *evt ) );
		if ( evt->flags & cpu_to_le16 ( INTELXL_ADMIN_FL_BUF ) ) {
			DBGC2_HDA ( intelxl, virt_to_phys ( buf ), buf,
				    le16_to_cpu ( evt->len ) );
		}

		/* Handle event */
		intelxl->handle ( netdev, evt, buf );

		/* Reset descriptor and refill queue */
		intelxl_admin_event_init ( intelxl, admin->index );
		admin->index++;
		intelxl_refill_admin ( intelxl );
	}
}

/**
 * Open admin queues
 *
 * @v intelxl		Intel device
 * @ret rc		Return status code
 */
int intelxl_open_admin ( struct intelxl_nic *intelxl ) {
	int rc;

	/* Allocate admin event queue */
	if ( ( rc = intelxl_alloc_admin ( intelxl, &intelxl->event ) ) != 0 )
		goto err_alloc_event;

	/* Allocate admin command queue */
	if ( ( rc = intelxl_alloc_admin ( intelxl, &intelxl->command ) ) != 0 )
		goto err_alloc_command;

	/* (Re)open admin queues */
	intelxl_reopen_admin ( intelxl );

	return 0;

	intelxl_disable_admin ( intelxl, &intelxl->command );
	intelxl_disable_admin ( intelxl, &intelxl->event );
	intelxl_free_admin ( intelxl, &intelxl->command );
 err_alloc_command:
	intelxl_free_admin ( intelxl, &intelxl->event );
 err_alloc_event:
	return rc;
}

/**
 * Reopen admin queues (after virtual function reset)
 *
 * @v intelxl		Intel device
 */
void intelxl_reopen_admin ( struct intelxl_nic *intelxl ) {
	unsigned int i;

	/* Enable admin event queue */
	intelxl_enable_admin ( intelxl, &intelxl->event );

	/* Enable admin command queue */
	intelxl_enable_admin ( intelxl, &intelxl->command );

	/* Initialise all admin event queue descriptors */
	for ( i = 0 ; i < INTELXL_ADMIN_NUM_DESC ; i++ )
		intelxl_admin_event_init ( intelxl, i );

	/* Post all descriptors to event queue */
	intelxl_refill_admin ( intelxl );
}

/**
 * Close admin queues
 *
 * @v intelxl		Intel device
 */
void intelxl_close_admin ( struct intelxl_nic *intelxl ) {

	/* Shut down admin queues */
	intelxl_admin_shutdown ( intelxl );

	/* Disable admin queues */
	intelxl_disable_admin ( intelxl, &intelxl->command );
	intelxl_disable_admin ( intelxl, &intelxl->event );

	/* Free admin queues */
	intelxl_free_admin ( intelxl, &intelxl->command );
	intelxl_free_admin ( intelxl, &intelxl->event );
}

/******************************************************************************
 *
 * Descriptor rings
 *
 ******************************************************************************
 */

/**
 * Allocate descriptor ring
 *
 * @v intelxl		Intel device
 * @v ring		Descriptor ring
 * @ret rc		Return status code
 */
int intelxl_alloc_ring ( struct intelxl_nic *intelxl,
			 struct intelxl_ring *ring ) {
	int rc;

	/* Allocate descriptor ring */
	ring->desc.raw = dma_alloc ( intelxl->dma, &ring->map, ring->len,
				     INTELXL_ALIGN );
	if ( ! ring->desc.raw ) {
		rc = -ENOMEM;
		goto err_alloc;
	}

	/* Initialise descriptor ring */
	memset ( ring->desc.raw, 0, ring->len );

	/* Reset tail pointer */
	writel ( 0, ( intelxl->regs + ring->tail ) );

	/* Reset counters */
	ring->prod = 0;
	ring->cons = 0;

	DBGC ( intelxl, "INTELXL %p ring %06x is at [%08lx,%08lx)\n",
	       intelxl, ring->tail, virt_to_phys ( ring->desc.raw ),
	       ( virt_to_phys ( ring->desc.raw ) + ring->len ) );

	return 0;

	dma_free ( &ring->map, ring->desc.raw, ring->len );
 err_alloc:
	return rc;
}

/**
 * Free descriptor ring
 *
 * @v intelxl		Intel device
 * @v ring		Descriptor ring
 */
void intelxl_free_ring ( struct intelxl_nic *intelxl __unused,
			 struct intelxl_ring *ring ) {

	/* Free descriptor ring */
	dma_free ( &ring->map, ring->desc.raw, ring->len );
	ring->desc.raw = NULL;
}

/**
 * Dump queue context (for debugging)
 *
 * @v intelxl		Intel device
 * @v op		Context operation
 * @v len		Size of context
 */
static __attribute__ (( unused )) void
intelxl_context_dump ( struct intelxl_nic *intelxl, uint32_t op, size_t len ) {
	struct intelxl_context_line line;
	uint32_t pfcm_lanctxctl;
	uint32_t pfcm_lanctxstat;
	unsigned int queue;
	unsigned int index;
	unsigned int i;

	/* Do nothing unless debug output is enabled */
	if ( ! DBG_EXTRA )
		return;

	/* Dump context */
	DBGC2 ( intelxl, "INTELXL %p context %#08x:\n", intelxl, op );
	for ( index = 0 ; ( sizeof ( line ) * index ) < len ; index++ ) {

		/* Start context operation */
		queue = ( intelxl->base + intelxl->queue );
		pfcm_lanctxctl =
			( INTELXL_PFCM_LANCTXCTL_QUEUE_NUM ( queue ) |
			  INTELXL_PFCM_LANCTXCTL_SUB_LINE ( index ) |
			  INTELXL_PFCM_LANCTXCTL_OP_CODE_READ | op );
		writel ( pfcm_lanctxctl,
			 intelxl->regs + INTELXL_PFCM_LANCTXCTL );

		/* Wait for operation to complete */
		for ( i = 0 ; i < INTELXL_CTX_MAX_WAIT_MS ; i++ ) {

			/* Check if operation is complete */
			pfcm_lanctxstat = readl ( intelxl->regs +
						  INTELXL_PFCM_LANCTXSTAT );
			if ( pfcm_lanctxstat & INTELXL_PFCM_LANCTXSTAT_DONE )
				break;

			/* Delay */
			mdelay ( 1 );
		}

		/* Read context data */
		for ( i = 0 ; i < ( sizeof ( line ) /
				    sizeof ( line.raw[0] ) ) ; i++ ) {
			line.raw[i] = readl ( intelxl->regs +
					      INTELXL_PFCM_LANCTXDATA ( i ) );
		}
		DBGC2_HDA ( intelxl, ( sizeof ( line ) * index ),
			    &line, sizeof ( line ) );
	}
}

/**
 * Program queue context line
 *
 * @v intelxl		Intel device
 * @v line		Queue context line
 * @v index		Line number
 * @v op		Context operation
 * @ret rc		Return status code
 */
static int intelxl_context_line ( struct intelxl_nic *intelxl,
				  struct intelxl_context_line *line,
				  unsigned int index, uint32_t op ) {
	uint32_t pfcm_lanctxctl;
	uint32_t pfcm_lanctxstat;
	unsigned int queue;
	unsigned int i;

	/* Write context data */
	for ( i = 0; i < ( sizeof ( *line ) / sizeof ( line->raw[0] ) ); i++ ) {
		writel ( le32_to_cpu ( line->raw[i] ),
			 intelxl->regs + INTELXL_PFCM_LANCTXDATA ( i ) );
	}

	/* Start context operation */
	queue = ( intelxl->base + intelxl->queue );
	pfcm_lanctxctl = ( INTELXL_PFCM_LANCTXCTL_QUEUE_NUM ( queue ) |
			   INTELXL_PFCM_LANCTXCTL_SUB_LINE ( index ) |
			   INTELXL_PFCM_LANCTXCTL_OP_CODE_WRITE | op );
	writel ( pfcm_lanctxctl, intelxl->regs + INTELXL_PFCM_LANCTXCTL );

	/* Wait for operation to complete */
	for ( i = 0 ; i < INTELXL_CTX_MAX_WAIT_MS ; i++ ) {

		/* Check if operation is complete */
		pfcm_lanctxstat = readl ( intelxl->regs +
					  INTELXL_PFCM_LANCTXSTAT );
		if ( pfcm_lanctxstat & INTELXL_PFCM_LANCTXSTAT_DONE )
			return 0;

		/* Delay */
		mdelay ( 1 );
	}

	DBGC ( intelxl, "INTELXL %p timed out waiting for context: %#08x\n",
	       intelxl, pfcm_lanctxctl );
	return -ETIMEDOUT;
}

/**
 * Program queue context
 *
 * @v intelxl		Intel device
 * @v line		Queue context lines
 * @v len		Size of context
 * @v op		Context operation
 * @ret rc		Return status code
 */
static int intelxl_context ( struct intelxl_nic *intelxl,
			     struct intelxl_context_line *line,
			     size_t len, uint32_t op ) {
	unsigned int index;
	int rc;

	DBGC2 ( intelxl, "INTELXL %p context %#08x len %#zx:\n",
		intelxl, op, len );
	DBGC2_HDA ( intelxl, 0, line, len );

	/* Program one line at a time */
	for ( index = 0 ; ( sizeof ( *line ) * index ) < len ; index++ ) {
		if ( ( rc = intelxl_context_line ( intelxl, line++, index,
						   op ) ) != 0 )
			return rc;
	}

	return 0;
}

/**
 * Program transmit queue context
 *
 * @v intelxl		Intel device
 * @v address		Descriptor ring base address
 * @ret rc		Return status code
 */
static int intelxl_context_tx ( struct intelxl_nic *intelxl,
				physaddr_t address ) {
	union {
		struct intelxl_context_tx tx;
		struct intelxl_context_line line;
	} ctx;
	int rc;

	/* Initialise context */
	memset ( &ctx, 0, sizeof ( ctx ) );
	ctx.tx.flags = cpu_to_le16 ( INTELXL_CTX_TX_FL_NEW );
	ctx.tx.base = cpu_to_le64 ( INTELXL_CTX_TX_BASE ( address ) );
	ctx.tx.count =
		cpu_to_le16 ( INTELXL_CTX_TX_COUNT ( INTELXL_TX_NUM_DESC ) );
	ctx.tx.qset = INTELXL_CTX_TX_QSET ( intelxl->qset );

	/* Program context */
	if ( ( rc = intelxl_context ( intelxl, &ctx.line, sizeof ( ctx ),
				      INTELXL_PFCM_LANCTXCTL_TYPE_TX ) ) != 0 )
		return rc;

	return 0;
}

/**
 * Program receive queue context
 *
 * @v intelxl		Intel device
 * @v address		Descriptor ring base address
 * @ret rc		Return status code
 */
static int intelxl_context_rx ( struct intelxl_nic *intelxl,
				physaddr_t address ) {
	union {
		struct intelxl_context_rx rx;
		struct intelxl_context_line line;
	} ctx;
	uint64_t base_count;
	int rc;

	/* Initialise context */
	memset ( &ctx, 0, sizeof ( ctx ) );
	base_count = INTELXL_CTX_RX_BASE_COUNT ( address, INTELXL_RX_NUM_DESC );
	ctx.rx.base_count = cpu_to_le64 ( base_count );
	ctx.rx.len = cpu_to_le16 ( INTELXL_CTX_RX_LEN ( intelxl->mfs ) );
	ctx.rx.flags = ( INTELXL_CTX_RX_FL_DSIZE | INTELXL_CTX_RX_FL_CRCSTRIP );
	ctx.rx.mfs = cpu_to_le16 ( INTELXL_CTX_RX_MFS ( intelxl->mfs ) );

	/* Program context */
	if ( ( rc = intelxl_context ( intelxl, &ctx.line, sizeof ( ctx ),
				      INTELXL_PFCM_LANCTXCTL_TYPE_RX ) ) != 0 )
		return rc;

	return 0;
}

/**
 * Enable descriptor ring
 *
 * @v intelxl		Intel device
 * @v ring		Descriptor ring
 * @ret rc		Return status code
 */
static int intelxl_enable_ring ( struct intelxl_nic *intelxl,
				 struct intelxl_ring *ring ) {
	void *ring_regs = ( intelxl->regs + ring->reg );
	uint32_t qxx_ena;

	/* Enable ring */
	writel ( INTELXL_QXX_ENA_REQ, ( ring_regs + INTELXL_QXX_ENA ) );
	udelay ( INTELXL_QUEUE_ENABLE_DELAY_US );
	qxx_ena = readl ( ring_regs + INTELXL_QXX_ENA );
	if ( ! ( qxx_ena & INTELXL_QXX_ENA_STAT ) ) {
		DBGC ( intelxl, "INTELXL %p ring %06x failed to enable: "
		       "%#08x\n", intelxl, ring->tail, qxx_ena );
		return -EIO;
	}

	return 0;
}

/**
 * Disable descriptor ring
 *
 * @v intelxl		Intel device
 * @v ring		Descriptor ring
 * @ret rc		Return status code
 */
static int intelxl_disable_ring ( struct intelxl_nic *intelxl,
				  struct intelxl_ring *ring ) {
	void *ring_regs = ( intelxl->regs + ring->reg );
	uint32_t qxx_ena;
	unsigned int i;

	/* Disable ring */
	writel ( 0, ( ring_regs + INTELXL_QXX_ENA ) );

	/* Wait for ring to be disabled */
	for ( i = 0 ; i < INTELXL_QUEUE_DISABLE_MAX_WAIT_MS ; i++ ) {

		/* Check if ring is disabled */
		qxx_ena = readl ( ring_regs + INTELXL_QXX_ENA );
		if ( ! ( qxx_ena & INTELXL_QXX_ENA_STAT ) )
			return 0;

		/* Delay */
		mdelay ( 1 );
	}

	DBGC ( intelxl, "INTELXL %p ring %06x timed out waiting for disable: "
	       "%#08x\n", intelxl, ring->tail, qxx_ena );
	return -ETIMEDOUT;
}

/**
 * Create descriptor ring
 *
 * @v intelxl		Intel device
 * @v ring		Descriptor ring
 * @ret rc		Return status code
 */
int intelxl_create_ring ( struct intelxl_nic *intelxl,
			  struct intelxl_ring *ring ) {
	physaddr_t address;
	int rc;

	/* Allocate descriptor ring */
	if ( ( rc = intelxl_alloc_ring ( intelxl, ring ) ) != 0 )
		goto err_alloc;

	/* Program queue context */
	address = dma ( &ring->map, ring->desc.raw );
	if ( ( rc = ring->context ( intelxl, address ) ) != 0 )
		goto err_context;

	/* Enable ring */
	if ( ( rc = intelxl_enable_ring ( intelxl, ring ) ) != 0 )
		goto err_enable;

	return 0;

	intelxl_disable_ring ( intelxl, ring );
 err_enable:
 err_context:
	intelxl_free_ring ( intelxl, ring );
 err_alloc:
	return rc;
}

/**
 * Destroy descriptor ring
 *
 * @v intelxl		Intel device
 * @v ring		Descriptor ring
 */
void intelxl_destroy_ring ( struct intelxl_nic *intelxl,
			    struct intelxl_ring *ring ) {
	int rc;

	/* Disable ring */
	if ( ( rc = intelxl_disable_ring ( intelxl, ring ) ) != 0 ) {
		/* Leak memory; there's nothing else we can do */
		return;
	}

	/* Free descriptor ring */
	intelxl_free_ring ( intelxl, ring );
}

/**
 * Refill receive descriptor ring
 *
 * @v intelxl		Intel device
 */
static void intelxl_refill_rx ( struct intelxl_nic *intelxl ) {
	struct intelxl_rx_data_descriptor *rx;
	struct io_buffer *iobuf;
	unsigned int rx_idx;
	unsigned int rx_tail;
	unsigned int refilled = 0;

	/* Refill ring */
	while ( ( intelxl->rx.prod - intelxl->rx.cons ) < INTELXL_RX_FILL ) {

		/* Allocate I/O buffer */
		iobuf = alloc_rx_iob ( intelxl->mfs, intelxl->dma );
		if ( ! iobuf ) {
			/* Wait for next refill */
			break;
		}

		/* Get next receive descriptor */
		rx_idx = ( intelxl->rx.prod++ % INTELXL_RX_NUM_DESC );
		rx = &intelxl->rx.desc.rx[rx_idx].data;

		/* Populate receive descriptor */
		rx->address = cpu_to_le64 ( iob_dma ( iobuf ) );
		rx->flags = 0;

		/* Record I/O buffer */
		assert ( intelxl->rx_iobuf[rx_idx] == NULL );
		intelxl->rx_iobuf[rx_idx] = iobuf;

		DBGC2 ( intelxl, "INTELXL %p RX %d is [%08lx,%08lx)\n",
			intelxl, rx_idx, virt_to_phys ( iobuf->data ),
			( virt_to_phys ( iobuf->data ) +  intelxl->mfs ) );
		refilled++;
	}

	/* Push descriptors to card, if applicable */
	if ( refilled ) {
		wmb();
		rx_tail = ( intelxl->rx.prod % INTELXL_RX_NUM_DESC );
		writel ( rx_tail, ( intelxl->regs + intelxl->rx.tail ) );
	}
}

/**
 * Discard unused receive I/O buffers
 *
 * @v intelxl		Intel device
 */
void intelxl_empty_rx ( struct intelxl_nic *intelxl ) {
	unsigned int i;

	/* Discard any unused receive buffers */
	for ( i = 0 ; i < INTELXL_RX_NUM_DESC ; i++ ) {
		if ( intelxl->rx_iobuf[i] )
			free_rx_iob ( intelxl->rx_iobuf[i] );
		intelxl->rx_iobuf[i] = NULL;
	}
}

/******************************************************************************
 *
 * Network device interface
 *
 ******************************************************************************
 */

/**
 * Open network device
 *
 * @v netdev		Network device
 * @ret rc		Return status code
 */
static int intelxl_open ( struct net_device *netdev ) {
	struct intelxl_nic *intelxl = netdev->priv;
	unsigned int queue;
	int rc;

	/* Calculate maximum frame size */
	intelxl->mfs = ( ( ETH_HLEN + netdev->mtu + 4 /* CRC */ +
			   INTELXL_ALIGN - 1 ) & ~( INTELXL_ALIGN - 1 ) );

	/* Set MAC address */
	if ( ( rc = intelxl_admin_mac_write ( netdev ) ) != 0 )
		goto err_mac_write;

	/* Set maximum frame size */
	if ( ( rc = intelxl_admin_mac_config ( intelxl ) ) != 0 )
		goto err_mac_config;

	/* Associate transmit queue to PF */
	writel ( ( INTELXL_QXX_CTL_PFVF_Q_PF |
		   INTELXL_QXX_CTL_PFVF_PF_INDX ( intelxl->pf ) ),
		 ( intelxl->regs + intelxl->tx.reg + INTELXL_QXX_CTL ) );

	/* Clear transmit pre queue disable */
	queue = ( intelxl->base + intelxl->queue );
	writel ( ( INTELXL_GLLAN_TXPRE_QDIS_CLEAR_QDIS |
		   INTELXL_GLLAN_TXPRE_QDIS_QINDX ( queue ) ),
		 ( intelxl->regs + INTELXL_GLLAN_TXPRE_QDIS ( queue ) ) );

	/* Reset transmit queue head */
	writel ( 0, ( intelxl->regs + INTELXL_QTX_HEAD ( intelxl->queue ) ) );

	/* Create receive descriptor ring */
	if ( ( rc = intelxl_create_ring ( intelxl, &intelxl->rx ) ) != 0 )
		goto err_create_rx;

	/* Create transmit descriptor ring */
	if ( ( rc = intelxl_create_ring ( intelxl, &intelxl->tx ) ) != 0 )
		goto err_create_tx;

	/* Fill receive ring */
	intelxl_refill_rx ( intelxl );

	/* Restart autonegotiation */
	intelxl_admin_autoneg ( intelxl );

	/* Update link state */
	intelxl_admin_link ( netdev );

	return 0;

	writel ( ( INTELXL_GLLAN_TXPRE_QDIS_SET_QDIS |
		   INTELXL_GLLAN_TXPRE_QDIS_QINDX ( queue ) ),
		 ( intelxl->regs + INTELXL_GLLAN_TXPRE_QDIS ( queue ) ) );
	udelay ( INTELXL_QUEUE_PRE_DISABLE_DELAY_US );
	intelxl_destroy_ring ( intelxl, &intelxl->tx );
 err_create_tx:
	intelxl_destroy_ring ( intelxl, &intelxl->rx );
 err_create_rx:
 err_mac_config:
 err_mac_write:
	return rc;
}

/**
 * Close network device
 *
 * @v netdev		Network device
 */
static void intelxl_close ( struct net_device *netdev ) {
	struct intelxl_nic *intelxl = netdev->priv;
	unsigned int queue;

	/* Dump contexts (for debugging) */
	intelxl_context_dump ( intelxl, INTELXL_PFCM_LANCTXCTL_TYPE_TX,
			       sizeof ( struct intelxl_context_tx ) );
	intelxl_context_dump ( intelxl, INTELXL_PFCM_LANCTXCTL_TYPE_RX,
			       sizeof ( struct intelxl_context_rx ) );

	/* Pre-disable transmit queue */
	queue = ( intelxl->base + intelxl->queue );
	writel ( ( INTELXL_GLLAN_TXPRE_QDIS_SET_QDIS |
		   INTELXL_GLLAN_TXPRE_QDIS_QINDX ( queue ) ),
		 ( intelxl->regs + INTELXL_GLLAN_TXPRE_QDIS ( queue ) ) );
	udelay ( INTELXL_QUEUE_PRE_DISABLE_DELAY_US );

	/* Destroy transmit descriptor ring */
	intelxl_destroy_ring ( intelxl, &intelxl->tx );

	/* Destroy receive descriptor ring */
	intelxl_destroy_ring ( intelxl, &intelxl->rx );

	/* Discard any unused receive buffers */
	intelxl_empty_rx ( intelxl );
}

/**
 * Transmit packet
 *
 * @v netdev		Network device
 * @v iobuf		I/O buffer
 * @ret rc		Return status code
 */
int intelxl_transmit ( struct net_device *netdev, struct io_buffer *iobuf ) {
	struct intelxl_nic *intelxl = netdev->priv;
	struct intelxl_tx_data_descriptor *tx;
	unsigned int tx_idx;
	unsigned int tx_tail;
	size_t len;

	/* Get next transmit descriptor */
	if ( ( intelxl->tx.prod - intelxl->tx.cons ) >= INTELXL_TX_FILL ) {
		DBGC ( intelxl, "INTELXL %p out of transmit descriptors\n",
		       intelxl );
		return -ENOBUFS;
	}
	tx_idx = ( intelxl->tx.prod++ % INTELXL_TX_NUM_DESC );
	tx_tail = ( intelxl->tx.prod % INTELXL_TX_NUM_DESC );
	tx = &intelxl->tx.desc.tx[tx_idx].data;

	/* Populate transmit descriptor */
	len = iob_len ( iobuf );
	tx->address = cpu_to_le64 ( iob_dma ( iobuf ) );
	tx->len = cpu_to_le32 ( INTELXL_TX_DATA_LEN ( len ) );
	tx->flags = cpu_to_le32 ( INTELXL_TX_DATA_DTYP | INTELXL_TX_DATA_EOP |
				  INTELXL_TX_DATA_RS | INTELXL_TX_DATA_JFDI );
	wmb();

	/* Notify card that there are packets ready to transmit */
	writel ( tx_tail, ( intelxl->regs + intelxl->tx.tail ) );

	DBGC2 ( intelxl, "INTELXL %p TX %d is [%08lx,%08lx)\n",
		intelxl, tx_idx, virt_to_phys ( iobuf->data ),
		( virt_to_phys ( iobuf->data ) + len ) );
	return 0;
}

/**
 * Poll for completed packets
 *
 * @v netdev		Network device
 */
static void intelxl_poll_tx ( struct net_device *netdev ) {
	struct intelxl_nic *intelxl = netdev->priv;
	struct intelxl_tx_writeback_descriptor *tx_wb;
	unsigned int tx_idx;

	/* Check for completed packets */
	while ( intelxl->tx.cons != intelxl->tx.prod ) {

		/* Get next transmit descriptor */
		tx_idx = ( intelxl->tx.cons % INTELXL_TX_NUM_DESC );
		tx_wb = &intelxl->tx.desc.tx[tx_idx].wb;

		/* Stop if descriptor is still in use */
		if ( ! ( tx_wb->flags & INTELXL_TX_WB_FL_DD ) )
			return;
		DBGC2 ( intelxl, "INTELXL %p TX %d complete\n",
			intelxl, tx_idx );

		/* Complete TX descriptor */
		netdev_tx_complete_next ( netdev );
		intelxl->tx.cons++;
	}
}

/**
 * Poll for received packets
 *
 * @v netdev		Network device
 */
static void intelxl_poll_rx ( struct net_device *netdev ) {
	struct intelxl_nic *intelxl = netdev->priv;
	struct intelxl_rx_writeback_descriptor *rx_wb;
	struct io_buffer *iobuf;
	unsigned int rx_idx;
	unsigned int tag;
	size_t len;

	/* Check for received packets */
	while ( intelxl->rx.cons != intelxl->rx.prod ) {

		/* Get next receive descriptor */
		rx_idx = ( intelxl->rx.cons % INTELXL_RX_NUM_DESC );
		rx_wb = &intelxl->rx.desc.rx[rx_idx].wb;

		/* Stop if descriptor is still in use */
		if ( ! ( rx_wb->flags & cpu_to_le32 ( INTELXL_RX_WB_FL_DD ) ) )
			return;

		/* Populate I/O buffer */
		iobuf = intelxl->rx_iobuf[rx_idx];
		intelxl->rx_iobuf[rx_idx] = NULL;
		len = INTELXL_RX_WB_LEN ( le32_to_cpu ( rx_wb->len ) );
		iob_put ( iobuf, len );

		/* Find VLAN device, if applicable */
		if ( rx_wb->flags & cpu_to_le32 ( INTELXL_RX_WB_FL_VLAN ) ) {
			tag = VLAN_TAG ( le16_to_cpu ( rx_wb->vlan ) );
		} else {
			tag = 0;
		}

		/* Hand off to network stack */
		if ( rx_wb->flags & cpu_to_le32 ( INTELXL_RX_WB_FL_RXE ) ) {
			DBGC ( intelxl, "INTELXL %p RX %d error (length %zd, "
			       "flags %08x)\n", intelxl, rx_idx, len,
			       le32_to_cpu ( rx_wb->flags ) );
			vlan_netdev_rx_err ( netdev, tag, iobuf, -EIO );
		} else {
			DBGC2 ( intelxl, "INTELXL %p RX %d complete (length "
				"%zd)\n", intelxl, rx_idx, len );
			vlan_netdev_rx ( netdev, tag, iobuf );
		}
		intelxl->rx.cons++;
	}
}

/**
 * Poll for completed and received packets
 *
 * @v netdev		Network device
 */
void intelxl_poll ( struct net_device *netdev ) {
	struct intelxl_nic *intelxl = netdev->priv;

	/* Poll for completed packets */
	intelxl_poll_tx ( netdev );

	/* Poll for received packets */
	intelxl_poll_rx ( netdev );

	/* Poll for admin events */
	intelxl_poll_admin ( netdev );

	/* Refill RX ring */
	intelxl_refill_rx ( intelxl );

	/* Rearm interrupt, since otherwise receive descriptors will
	 * be written back only after a complete cacheline (four
	 * packets) have been received.
	 *
	 * There is unfortunately no efficient way to determine
	 * whether or not rearming the interrupt is necessary.  If we
	 * are running inside a hypervisor (e.g. using a VF or PF as a
	 * passed-through PCI device), then the MSI-X write is
	 * redirected by the hypervisor to the real host APIC and the
	 * host ISR then raises an interrupt within the guest.  We
	 * therefore cannot poll the nominal MSI-X target location to
	 * watch for the value being written.  We could read from the
	 * INT_DYN_CTL register, but this is even less efficient than
	 * just unconditionally rearming the interrupt.
	 */
	writel ( INTELXL_INT_DYN_CTL_INTENA, intelxl->regs + intelxl->intr );
}

/** Network device operations */
static struct net_device_operations intelxl_operations = {
	.open		= intelxl_open,
	.close		= intelxl_close,
	.transmit	= intelxl_transmit,
	.poll		= intelxl_poll,
};

/******************************************************************************
 *
 * PCI interface
 *
 ******************************************************************************
 */

/**
 * Probe PCI device
 *
 * @v pci		PCI device
 * @ret rc		Return status code
 */
static int intelxl_probe ( struct pci_device *pci ) {
	struct net_device *netdev;
	struct intelxl_nic *intelxl;
	uint32_t pffunc_rid;
	uint32_t pfgen_portnum;
	uint32_t pflan_qalloc;
	int rc;

	/* Allocate and initialise net device */
	netdev = alloc_etherdev ( sizeof ( *intelxl ) );
	if ( ! netdev ) {
		rc = -ENOMEM;
		goto err_alloc;
	}
	netdev_init ( netdev, &intelxl_operations );
	netdev->max_pkt_len = INTELXL_MAX_PKT_LEN;
	intelxl = netdev->priv;
	pci_set_drvdata ( pci, netdev );
	netdev->dev = &pci->dev;
	memset ( intelxl, 0, sizeof ( *intelxl ) );
	intelxl->intr = INTELXL_PFINT_DYN_CTL0;
	intelxl->handle = intelxl_admin_event;
	intelxl_init_admin ( &intelxl->command, INTELXL_ADMIN_CMD,
			     &intelxl_admin_offsets );
	intelxl_init_admin ( &intelxl->event, INTELXL_ADMIN_EVT,
			     &intelxl_admin_offsets );
	intelxl_init_ring ( &intelxl->tx, INTELXL_TX_NUM_DESC,
			    sizeof ( intelxl->tx.desc.tx[0] ),
			    intelxl_context_tx );
	intelxl_init_ring ( &intelxl->rx, INTELXL_RX_NUM_DESC,
			    sizeof ( intelxl->rx.desc.rx[0] ),
			    intelxl_context_rx );

	/* Fix up PCI device */
	adjust_pci_device ( pci );

	/* Map registers */
	intelxl->regs = pci_ioremap ( pci, pci->membase, INTELXL_BAR_SIZE );
	if ( ! intelxl->regs ) {
		rc = -ENODEV;
		goto err_ioremap;
	}

	/* Configure DMA */
	intelxl->dma = &pci->dma;
	dma_set_mask_64bit ( intelxl->dma );
	netdev->dma = intelxl->dma;

	/* Locate PCI Express capability */
	intelxl->exp = pci_find_capability ( pci, PCI_CAP_ID_EXP );
	if ( ! intelxl->exp ) {
		DBGC ( intelxl, "INTELXL %p missing PCIe capability\n",
		       intelxl );
		rc = -ENXIO;
		goto err_exp;
	}

	/* Reset the function via PCIe FLR */
	pci_reset ( pci, intelxl->exp );

	/* Get function number, port number and base queue number */
	pffunc_rid = readl ( intelxl->regs + INTELXL_PFFUNC_RID );
	intelxl->pf = INTELXL_PFFUNC_RID_FUNC_NUM ( pffunc_rid );
	pfgen_portnum = readl ( intelxl->regs + INTELXL_PFGEN_PORTNUM );
	intelxl->port = INTELXL_PFGEN_PORTNUM_PORT_NUM ( pfgen_portnum );
	pflan_qalloc = readl ( intelxl->regs + INTELXL_PFLAN_QALLOC );
	intelxl->base = INTELXL_PFLAN_QALLOC_FIRSTQ ( pflan_qalloc );
	DBGC ( intelxl, "INTELXL %p PF %d using port %d queues [%#04x-%#04x]\n",
	       intelxl, intelxl->pf, intelxl->port, intelxl->base,
	       INTELXL_PFLAN_QALLOC_LASTQ ( pflan_qalloc ) );

	/* Enable MSI-X dummy interrupt */
	if ( ( rc = intelxl_msix_enable ( intelxl, pci,
					  INTELXL_MSIX_VECTOR ) ) != 0 )
		goto err_msix;

	/* Open admin queues */
	if ( ( rc = intelxl_open_admin ( intelxl ) ) != 0 )
		goto err_open_admin;

	/* Get firmware version */
	if ( ( rc = intelxl_admin_version ( intelxl ) ) != 0 )
		goto err_admin_version;

	/* Report driver version */
	if ( ( rc = intelxl_admin_driver ( intelxl ) ) != 0 )
		goto err_admin_driver;

	/* Clear PXE mode */
	if ( ( rc = intelxl_admin_clear_pxe ( intelxl ) ) != 0 )
		goto err_admin_clear_pxe;

	/* Get switch configuration */
	if ( ( rc = intelxl_admin_switch ( intelxl ) ) != 0 )
		goto err_admin_switch;

	/* Get VSI configuration */
	if ( ( rc = intelxl_admin_vsi ( intelxl ) ) != 0 )
		goto err_admin_vsi;

	/* Configure switch for promiscuous mode */
	if ( ( rc = intelxl_admin_promisc ( intelxl ) ) != 0 )
		goto err_admin_promisc;

	/* Get MAC address */
	if ( ( rc = intelxl_admin_mac_read ( netdev ) ) != 0 )
		goto err_admin_mac_read;

	/* Configure queue register addresses */
	intelxl->tx.reg = INTELXL_QTX ( intelxl->queue );
	intelxl->tx.tail = ( intelxl->tx.reg + INTELXL_QXX_TAIL );
	intelxl->rx.reg = INTELXL_QRX ( intelxl->queue );
	intelxl->rx.tail = ( intelxl->rx.reg + INTELXL_QXX_TAIL );

	/* Configure interrupt causes */
	writel ( ( INTELXL_QINT_TQCTL_NEXTQ_INDX_NONE |
		   INTELXL_QINT_TQCTL_CAUSE_ENA ),
		 intelxl->regs + INTELXL_QINT_TQCTL ( intelxl->queue ) );
	writel ( ( INTELXL_QINT_RQCTL_NEXTQ_INDX ( intelxl->queue ) |
		   INTELXL_QINT_RQCTL_NEXTQ_TYPE_TX |
		   INTELXL_QINT_RQCTL_CAUSE_ENA ),
		 intelxl->regs + INTELXL_QINT_RQCTL ( intelxl->queue ) );
	writel ( ( INTELXL_PFINT_LNKLST0_FIRSTQ_INDX ( intelxl->queue ) |
		   INTELXL_PFINT_LNKLST0_FIRSTQ_TYPE_RX ),
		 intelxl->regs + INTELXL_PFINT_LNKLST0 );
	writel ( INTELXL_PFINT_ICR0_ENA_ADMINQ,
		 intelxl->regs + INTELXL_PFINT_ICR0_ENA );

	/* Register network device */
	if ( ( rc = register_netdev ( netdev ) ) != 0 )
		goto err_register_netdev;

	/* Set initial link state */
	intelxl_admin_link ( netdev );

	return 0;

	unregister_netdev ( netdev );
 err_register_netdev:
 err_admin_mac_read:
 err_admin_promisc:
 err_admin_vsi:
 err_admin_switch:
 err_admin_clear_pxe:
 err_admin_driver:
 err_admin_version:
	intelxl_close_admin ( intelxl );
 err_open_admin:
	intelxl_msix_disable ( intelxl, pci, INTELXL_MSIX_VECTOR );
 err_msix:
	pci_reset ( pci, intelxl->exp );
 err_exp:
	iounmap ( intelxl->regs );
 err_ioremap:
	netdev_nullify ( netdev );
	netdev_put ( netdev );
 err_alloc:
	return rc;
}

/**
 * Remove PCI device
 *
 * @v pci		PCI device
 */
static void intelxl_remove ( struct pci_device *pci ) {
	struct net_device *netdev = pci_get_drvdata ( pci );
	struct intelxl_nic *intelxl = netdev->priv;

	/* Unregister network device */
	unregister_netdev ( netdev );

	/* Close admin queues */
	intelxl_close_admin ( intelxl );

	/* Disable MSI-X dummy interrupt */
	intelxl_msix_disable ( intelxl, pci, INTELXL_MSIX_VECTOR );

	/* Reset the NIC */
	pci_reset ( pci, intelxl->exp );

	/* Free network device */
	iounmap ( intelxl->regs );
	netdev_nullify ( netdev );
	netdev_put ( netdev );
}

/** PCI device IDs */
static struct pci_device_id intelxl_nics[] = {
	PCI_ROM ( 0x8086, 0x0cf8, "x710-n3000", "X710 FPGA N3000", 0 ),
	PCI_ROM ( 0x8086, 0x0d58, "xxv710-n3000", "XXV710 FPGA N3000", 0 ),
	PCI_ROM ( 0x8086, 0x104e, "x710-sfp-b", "X710 10GbE SFP+", 0 ),
	PCI_ROM ( 0x8086, 0x104f, "x710-kx-b", "X710 10GbE backplane", 0 ),
	PCI_ROM ( 0x8086, 0x1572, "x710-sfp", "X710 10GbE SFP+", 0 ),
	PCI_ROM ( 0x8086, 0x1574, "xl710-qemu", "Virtual XL710", 0 ),
	PCI_ROM ( 0x8086, 0x1580, "xl710-kx-b", "XL710 40GbE backplane", 0 ),
	PCI_ROM ( 0x8086, 0x1581, "xl710-kx-c", "XL710 10GbE backplane", 0 ),
	PCI_ROM ( 0x8086, 0x1583, "xl710-qda2", "XL710 40GbE QSFP+", 0 ),
	PCI_ROM ( 0x8086, 0x1584, "xl710-qda1", "XL710 40GbE QSFP+", 0 ),
	PCI_ROM ( 0x8086, 0x1585, "x710-qsfp", "X710 10GbE QSFP+", 0 ),
	PCI_ROM ( 0x8086, 0x1586, "x710-10gt", "X710 10GBASE-T", 0 ),
	PCI_ROM ( 0x8086, 0x1587, "x710-kr2", "XL710 20GbE backplane", 0 ),
	PCI_ROM ( 0x8086, 0x1588, "x710-kr2-a", "XL710 20GbE backplane", 0 ),
	PCI_ROM ( 0x8086, 0x1589, "x710-10gt4", "X710 10GBASE-T4", 0 ),
	PCI_ROM ( 0x8086, 0x158a, "xxv710", "XXV710 25GbE backplane", 0 ),
	PCI_ROM ( 0x8086, 0x158b, "xxv710-sfp28", "XXV710 25GbE SFP28", 0 ),
	PCI_ROM ( 0x8086, 0x15ff, "x710-10gt-b", "X710 10GBASE-T", 0 ),
	PCI_ROM ( 0x8086, 0x37ce, "x722-kx", "X722 10GbE backplane", 0 ),
	PCI_ROM ( 0x8086, 0x37cf, "x722-qsfp", "X722 10GbE QSFP+", 0 ),
	PCI_ROM ( 0x8086, 0x37d0, "x722-sfp", "X722 10GbE SFP+", 0 ),
	PCI_ROM ( 0x8086, 0x37d1, "x722-1gt", "X722 1GBASE-T", 0 ),
	PCI_ROM ( 0x8086, 0x37d2, "x722-10gt", "X722 10GBASE-T", 0 ),
	PCI_ROM ( 0x8086, 0x37d3, "x722-sfp-i", "X722 10GbE SFP+", 0 ),
	PCI_ROM ( 0x8086, 0x57b0, "e610-10gt", "E610 10GBASE T",  0 ),
	PCI_ROM ( 0x8086, 0x57b1, "e610-2gt", "E610 2.5GBASE T", 0 ),
	PCI_ROM ( 0x8086, 0x57b2, "e610-sgmii", "E610 SGMII", 0 ),
};

/** PCI driver */
struct pci_driver intelxl_driver __pci_driver = {
	.ids = intelxl_nics,
	.id_count = ( sizeof ( intelxl_nics ) / sizeof ( intelxl_nics[0] ) ),
	.probe = intelxl_probe,
	.remove = intelxl_remove,
};