src/drivers/net/sfc/efx_hunt.c - ipxe - Git at Google

 /**************************************************************************
  *
  * Driver datapath for Solarflare network cards
  *
  * Written by Shradha Shah, maintained by <pre-boot-drivers@xilinx.com>
  *
  * Copyright 2012-2019 Solarflare Communications Inc.
  * Copyright 2019-2020 Xilinx Inc.
  *
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public License as
  * published by the Free Software Foundation; either version 2 of the
  * License, or any later version.
  *
  * You can also choose to distribute this program under the terms of
  * the Unmodified Binary Distribution Licence (as given in the file
  * COPYING.UBDL), provided that you have satisfied its requirements.
  *
  ***************************************************************************/

 #include <stdint.h>
 #include <stdlib.h>
 #include <stdio.h>
 #include <unistd.h>
 #include <errno.h>
 #include <assert.h>
 #include <byteswap.h>
 #include <ipxe/io.h>
 #include <ipxe/pci.h>
 #include <ipxe/malloc.h>
 #include <ipxe/iobuf.h>
 #include <ipxe/netdevice.h>
 #include "efx_hunt.h"
 #include "efx_bitfield.h"
 #include "ef10_regs.h"

 FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );

 void efx_hunt_free_special_buffer(void *buf, int bytes)
 {
 	free_phys(buf, bytes);
 }

 static void *efx_hunt_alloc_special_buffer(int bytes,
 					   struct efx_special_buffer *entry)
 {
 	void *buffer;
 	dma_addr_t dma_addr;

 	/* Allocate the buffer, aligned on a buffer address boundary.  This
 	 * buffer will be passed into an MC_CMD_INIT_*Q command to setup the
 	 * appropriate type of queue via MCDI.
 	 */
 	buffer = malloc_phys(bytes, EFX_BUF_ALIGN);
 	if (!buffer)
 		return NULL;

 	entry->dma_addr = dma_addr = virt_to_bus(buffer);
 	assert((dma_addr & (EFX_BUF_ALIGN - 1)) == 0);

 	/* Buffer table entries aren't allocated, so set id to zero */
 	entry->id = 0;
 	DBGP("Allocated 0x%x bytes at %p\n", bytes, buffer);

 	return buffer;
 }

 /*******************************************************************************
  *
  *
  * TX
  *
  *
  ******************************************************************************/
 static void
 efx_hunt_build_tx_desc(efx_tx_desc_t *txd, struct io_buffer *iob)
 {
 	dma_addr_t dma_addr;

 	dma_addr = virt_to_bus(iob->data);

 	EFX_POPULATE_QWORD_4(*txd,
 			     ESF_DZ_TX_KER_TYPE, 0,
 			     ESF_DZ_TX_KER_CONT, 0,
 			     ESF_DZ_TX_KER_BYTE_CNT, iob_len(iob),
 			     ESF_DZ_TX_KER_BUF_ADDR, dma_addr);
 }

 static void
 efx_hunt_notify_tx_desc(struct efx_nic *efx)
 {
 	struct efx_tx_queue *txq = &efx->txq;
 	int ptr = txq->write_ptr & EFX_TXD_MASK;
 	efx_dword_t reg;

 	EFX_POPULATE_DWORD_1(reg, ERF_DZ_TX_DESC_WPTR_DWORD, ptr);
 	efx_writel_page(efx, &reg, 0, ER_DZ_TX_DESC_UPD_DWORD);
 }

 int
 efx_hunt_transmit(struct net_device *netdev, struct io_buffer *iob)
 {
 	struct efx_nic *efx = netdev_priv(netdev);
 	struct efx_tx_queue *txq = &efx->txq;
 	int fill_level, space;
 	efx_tx_desc_t *txd;
 	int buf_id;

 	fill_level = txq->write_ptr - txq->read_ptr;
 	space = EFX_TXD_SIZE - fill_level - 1;
 	if (space < 1)
 		return -ENOBUFS;

 	/* Save the iobuffer for later completion */
 	buf_id = txq->write_ptr & EFX_TXD_MASK;
 	assert(txq->buf[buf_id] == NULL);
 	txq->buf[buf_id] = iob;

 	DBGCIO(efx, "tx_buf[%d] for iob %p data %p len %zd\n",
 	       buf_id, iob, iob->data, iob_len(iob));

 	/* Form the descriptor, and push it to hardware */
 	txd = txq->ring + buf_id;
 	efx_hunt_build_tx_desc(txd, iob);
 	++txq->write_ptr;
 	efx_hunt_notify_tx_desc(efx);

 	return 0;
 }

 static void
 efx_hunt_transmit_done(struct efx_nic *efx, int id)
 {
 	struct efx_tx_queue *txq = &efx->txq;
 	unsigned int read_ptr, stop;

 	/* Complete all buffers from read_ptr up to and including id */
 	read_ptr = txq->read_ptr & EFX_TXD_MASK;
 	stop = (id + 1) & EFX_TXD_MASK;

 	while (read_ptr != stop) {
 		struct io_buffer *iob = txq->buf[read_ptr];

 		assert(iob);
 		/* Complete the tx buffer */
 		if (iob)
 			netdev_tx_complete(efx->netdev, iob);
 		DBGCIO(efx, "tx_buf[%d] for iob %p done\n", read_ptr, iob);
 		txq->buf[read_ptr] = NULL;

 		++txq->read_ptr;
 		read_ptr = txq->read_ptr & EFX_TXD_MASK;
 	}
 }

 int efx_hunt_tx_init(struct net_device *netdev, dma_addr_t *dma_addr)
 {
 	struct efx_nic *efx = netdev_priv(netdev);
 	struct efx_tx_queue *txq = &efx->txq;
 	size_t bytes;

 	/* Allocate hardware transmit queue */
 	bytes = sizeof(efx_tx_desc_t) * EFX_TXD_SIZE;
 	txq->ring = efx_hunt_alloc_special_buffer(bytes, &txq->entry);
 	if (!txq->ring)
 		return -ENOMEM;

 	txq->read_ptr = txq->write_ptr = 0;
 	*dma_addr = txq->entry.dma_addr;
 	return 0;
 }

 /*******************************************************************************
  *
  *
  * RX
  *
  *
  ******************************************************************************/
 static void
 efx_hunt_build_rx_desc(efx_rx_desc_t *rxd, struct io_buffer *iob)
 {
 	dma_addr_t dma_addr = virt_to_bus(iob->data);

 	EFX_POPULATE_QWORD_2(*rxd,
 			     ESF_DZ_RX_KER_BYTE_CNT, EFX_RX_BUF_SIZE,
 			     ESF_DZ_RX_KER_BUF_ADDR, dma_addr);
 }

 static void
 efx_hunt_notify_rx_desc(struct efx_nic *efx)
 {
 	struct efx_rx_queue *rxq = &efx->rxq;
 	int ptr = rxq->write_ptr & EFX_RXD_MASK;
 	efx_dword_t reg;

 	EFX_POPULATE_DWORD_1(reg, ERF_DZ_RX_DESC_WPTR, ptr);
 	efx_writel_page(efx, &reg, 0, ER_DZ_RX_DESC_UPD);
 }

 static void
 efx_hunt_rxq_fill(struct efx_nic *efx)
 {
 	struct efx_rx_queue *rxq = &efx->rxq;
 	int fill_level = rxq->write_ptr - rxq->read_ptr;
 	int space = EFX_NUM_RX_DESC - fill_level - 1;
 	int pushed = 0;

 	while (space) {
 		int buf_id = rxq->write_ptr & (EFX_NUM_RX_DESC - 1);
 		int desc_id = rxq->write_ptr & EFX_RXD_MASK;
 		struct io_buffer *iob;
 		efx_rx_desc_t *rxd;

 		assert(rxq->buf[buf_id] == NULL);
 		iob = alloc_iob(EFX_RX_BUF_SIZE);
 		if (!iob)
 			break;

 		DBGCP(efx, "pushing rx_buf[%d] iob %p data %p\n",
 		      buf_id, iob, iob->data);

 		rxq->buf[buf_id] = iob;
 		rxd = rxq->ring + desc_id;
 		efx_hunt_build_rx_desc(rxd, iob);
 		++rxq->write_ptr;
 		++pushed;
 		--space;
 	}

 	/* Push the ptr to hardware */
 	if (pushed > 0) {
 		efx_hunt_notify_rx_desc(efx);

 		DBGCP(efx, "pushed %d rx buffers to fill level %d\n",
 		      pushed, rxq->write_ptr - rxq->read_ptr);
 	}
 }

 static void
 efx_hunt_receive(struct efx_nic *efx, unsigned int id, int len, int drop)
 {
 	struct efx_rx_queue *rxq = &efx->rxq;
 	unsigned int read_ptr = rxq->read_ptr & EFX_RXD_MASK;
 	unsigned int buf_ptr = rxq->read_ptr & EFX_NUM_RX_DESC_MASK;
 	struct io_buffer *iob;

 	/* id is the lower 4 bits of the desc index + 1 in huntington*/
 	/* hence anding with 15 */
 	assert((id & 15) == ((read_ptr + (len != 0)) & 15));

 	/* Pop this rx buffer out of the software ring */
 	iob = rxq->buf[buf_ptr];
 	rxq->buf[buf_ptr] = NULL;

 	DBGCIO(efx, "popping rx_buf[%d] iob %p data %p with %d bytes %s %x\n",
 	       read_ptr, iob, iob->data, len, drop ? "bad" : "ok", drop);

 	/* Pass the packet up if required */
 	if (drop)
 		netdev_rx_err(efx->netdev, iob, EBADMSG);
 	else {
 		iob_put(iob, len);
 		iob_pull(iob, efx->rx_prefix_size);
 		netdev_rx(efx->netdev, iob);
 	}

 	++rxq->read_ptr;
 }

 int efx_hunt_rx_init(struct net_device *netdev, dma_addr_t *dma_addr)
 {
 	struct efx_nic *efx = netdev_priv(netdev);
 	struct efx_rx_queue *rxq = &efx->rxq;
 	size_t bytes;

 	/* Allocate hardware receive queue */
 	bytes = sizeof(efx_rx_desc_t) * EFX_RXD_SIZE;
 	rxq->ring = efx_hunt_alloc_special_buffer(bytes, &rxq->entry);
 	if (rxq->ring == NULL)
 		return -ENOMEM;

 	rxq->read_ptr = rxq->write_ptr = 0;
 	*dma_addr = rxq->entry.dma_addr;
 	return 0;
 }

 /*******************************************************************************
  *
  *
  * Event queues and interrupts
  *
  *
  ******************************************************************************/
 int efx_hunt_ev_init(struct net_device *netdev, dma_addr_t *dma_addr)
 {
 	struct efx_nic *efx = netdev_priv(netdev);
 	struct efx_ev_queue *evq = &efx->evq;
 	size_t bytes;

 	/* Allocate the hardware event queue */
 	bytes = sizeof(efx_event_t) * EFX_EVQ_SIZE;
 	evq->ring = efx_hunt_alloc_special_buffer(bytes, &evq->entry);
 	if (evq->ring == NULL)
 		return -ENOMEM;

 	memset(evq->ring, 0xff, bytes);
 	evq->read_ptr = 0;
 	*dma_addr = evq->entry.dma_addr;
 	return 0;
 }

 static void
 efx_hunt_clear_interrupts(struct efx_nic *efx)
 {
 	efx_dword_t reg;
 	/* read the ISR */
 	efx_readl(efx, &reg, ER_DZ_BIU_INT_ISR);
 }

 /**
  * See if an event is present
  *
  * @v event            EFX event structure
  * @ret True           An event is pending
  * @ret False          No event is pending
  *
  * We check both the high and low dword of the event for all ones.  We
  * wrote all ones when we cleared the event, and no valid event can
  * have all ones in either its high or low dwords.  This approach is
  * robust against reordering.
  *
  * Note that using a single 64-bit comparison is incorrect; even
  * though the CPU read will be atomic, the DMA write may not be.
  */
 static inline int
 efx_hunt_event_present(efx_event_t *event)
 {
 	return (!(EFX_DWORD_IS_ALL_ONES(event->dword[0]) |
 		  EFX_DWORD_IS_ALL_ONES(event->dword[1])));
 }

 static void
 efx_hunt_evq_read_ack(struct efx_nic *efx)
 {
 	struct efx_ev_queue *evq = &efx->evq;
 	efx_dword_t reg;

 	if (efx->workaround_35388) {
 		EFX_POPULATE_DWORD_2(reg, ERF_DD_EVQ_IND_RPTR_FLAGS,
 				     EFE_DD_EVQ_IND_RPTR_FLAGS_HIGH,
 				     ERF_DD_EVQ_IND_RPTR,
 				    evq->read_ptr >> ERF_DD_EVQ_IND_RPTR_WIDTH);
 		efx_writel_page(efx, &reg, 0, ER_DD_EVQ_INDIRECT);
 		EFX_POPULATE_DWORD_2(reg, ERF_DD_EVQ_IND_RPTR_FLAGS,
 				     EFE_DD_EVQ_IND_RPTR_FLAGS_LOW,
 				     ERF_DD_EVQ_IND_RPTR, evq->read_ptr &
 				     ((1 << ERF_DD_EVQ_IND_RPTR_WIDTH) - 1));
 		efx_writel_page(efx, &reg, 0, ER_DD_EVQ_INDIRECT);
 	} else {
 		EFX_POPULATE_DWORD_1(reg, ERF_DZ_EVQ_RPTR, evq->read_ptr);
 		efx_writel_table(efx, &reg, 0, ER_DZ_EVQ_RPTR);
 	}
 }

 static unsigned int
 efx_hunt_handle_event(struct efx_nic *efx, efx_event_t *evt)
 {
 	struct efx_rx_queue *rxq = &efx->rxq;
 	int ev_code, desc_ptr, len;
 	int next_ptr_lbits, packet_drop;
 	int rx_cont;

 	/* Decode event */
 	ev_code = EFX_QWORD_FIELD(*evt, ESF_DZ_EV_CODE);

 	switch (ev_code) {
 	case ESE_DZ_EV_CODE_TX_EV:
 		desc_ptr = EFX_QWORD_FIELD(*evt, ESF_DZ_TX_DESCR_INDX);
 		efx_hunt_transmit_done(efx, desc_ptr);
 		break;

 	case ESE_DZ_EV_CODE_RX_EV:
 		len = EFX_QWORD_FIELD(*evt, ESF_DZ_RX_BYTES);
 		next_ptr_lbits = EFX_QWORD_FIELD(*evt, ESF_DZ_RX_DSC_PTR_LBITS);
 		rx_cont = EFX_QWORD_FIELD(*evt, ESF_DZ_RX_CONT);

 		/* We don't expect to receive scattered packets, so drop the
 		 * packet if RX_CONT is set on the current or previous event, or
 		 * if len is zero.
 		 */
 		packet_drop = (len == 0) | (rx_cont << 1) |
 			      (rxq->rx_cont_prev << 2);
 		efx_hunt_receive(efx, next_ptr_lbits, len, packet_drop);
 		rxq->rx_cont_prev = rx_cont;
 		return 1;

 	default:
 		DBGCP(efx, "Unknown event type %d\n", ev_code);
 		break;
 	}
 	return 0;
 }

 void efx_hunt_poll(struct net_device *netdev)
 {
 	struct efx_nic *efx = netdev_priv(netdev);
 	struct efx_ev_queue *evq = &efx->evq;
 	efx_event_t *evt;
 	int budget = 10;

 	/* Read the event queue by directly looking for events
 	 * (we don't even bother to read the eventq write ptr)
 	 */
 	evt = evq->ring + evq->read_ptr;
 	while (efx_hunt_event_present(evt) && (budget > 0)) {
 		DBGCP(efx, "Event at index 0x%x address %p is "
 		      EFX_QWORD_FMT "\n", evq->read_ptr,
 		      evt, EFX_QWORD_VAL(*evt));

 		budget -= efx_hunt_handle_event(efx, evt);

 		/* Clear the event */
 		EFX_SET_QWORD(*evt);

 		/* Move to the next event. We don't ack the event
 		 * queue until the end
 		 */
 		evq->read_ptr = ((evq->read_ptr + 1) & EFX_EVQ_MASK);
 		evt = evq->ring + evq->read_ptr;
 	}

 	/* Push more rx buffers if needed */
 	efx_hunt_rxq_fill(efx);

 	/* Clear any pending interrupts */
 	efx_hunt_clear_interrupts(efx);

 	/* Ack the event queue if interrupts are enabled */
 	if (efx->int_en)
 		efx_hunt_evq_read_ack(efx);
 }

 void efx_hunt_irq(struct net_device *netdev, int enable)
 {
 	struct efx_nic *efx = netdev_priv(netdev);

 	efx->int_en = enable;

 	/* If interrupts are enabled, prime the event queue.  Otherwise ack any
 	 * pending interrupts
 	 */
 	if (enable)
 		efx_hunt_evq_read_ack(efx);
 	else if (efx->netdev->state & NETDEV_OPEN)
 		efx_hunt_clear_interrupts(efx);
 }

 /*******************************************************************************
  *
  *
  * Initialization and Close
  *
  *
  ******************************************************************************/
 int efx_hunt_open(struct net_device *netdev)
 {
 	struct efx_nic *efx = netdev_priv(netdev);
 	efx_dword_t cmd;

 	/* Set interrupt moderation to 0*/
 	EFX_POPULATE_DWORD_2(cmd,
 			     ERF_DZ_TC_TIMER_MODE, 0,
 			     ERF_DZ_TC_TIMER_VAL, 0);
 	efx_writel_page(efx, &cmd, 0, ER_DZ_EVQ_TMR);

 	/* Ack the eventq */
 	if (efx->int_en)
 		efx_hunt_evq_read_ack(efx);

 	/* Push receive buffers */
 	efx_hunt_rxq_fill(efx);

 	return 0;
 }

 void efx_hunt_close(struct net_device *netdev)
 {
 	struct efx_nic *efx = netdev_priv(netdev);
 	struct efx_rx_queue *rxq = &efx->rxq;
 	struct efx_tx_queue *txq = &efx->txq;
 	int i;

 	/* Complete outstanding descriptors */
 	for (i = 0; i < EFX_NUM_RX_DESC; i++) {
 		if (rxq->buf[i]) {
 			free_iob(rxq->buf[i]);
 			rxq->buf[i] = NULL;
 		}
 	}

 	for (i = 0; i < EFX_TXD_SIZE; i++) {
 		if (txq->buf[i]) {
 			netdev_tx_complete(efx->netdev, txq->buf[i]);
 			txq->buf[i] = NULL;
 		}
 	}

 	/* Clear interrupts */
 	efx_hunt_clear_interrupts(efx);
 }
	/**************************************************************************
	*
	* Driver datapath for Solarflare network cards
	*
	* Written by Shradha Shah, maintained by <pre-boot-drivers@xilinx.com>
	*
	* Copyright 2012-2019 Solarflare Communications Inc.
	* Copyright 2019-2020 Xilinx Inc.
	*
	* This program is free software; you can redistribute it and/or
	* modify it under the terms of the GNU General Public License as
	* published by the Free Software Foundation; either version 2 of the
	* License, or any later version.
	*
	* You can also choose to distribute this program under the terms of
	* the Unmodified Binary Distribution Licence (as given in the file
	* COPYING.UBDL), provided that you have satisfied its requirements.
	*
	***************************************************************************/

	#include <stdint.h>
	#include <stdlib.h>
	#include <stdio.h>
	#include <unistd.h>
	#include <errno.h>
	#include <assert.h>
	#include <byteswap.h>
	#include <ipxe/io.h>
	#include <ipxe/pci.h>
	#include <ipxe/malloc.h>
	#include <ipxe/iobuf.h>
	#include <ipxe/netdevice.h>
	#include "efx_hunt.h"
	#include "efx_bitfield.h"
	#include "ef10_regs.h"

	FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );

	void efx_hunt_free_special_buffer(void *buf, int bytes)
	{
	free_phys(buf, bytes);
	}

	static void *efx_hunt_alloc_special_buffer(int bytes,
	struct efx_special_buffer *entry)
	{
	void *buffer;
	dma_addr_t dma_addr;

	/* Allocate the buffer, aligned on a buffer address boundary. This
	* buffer will be passed into an MC_CMD_INIT_*Q command to setup the
	* appropriate type of queue via MCDI.
	*/
	buffer = malloc_phys(bytes, EFX_BUF_ALIGN);
	if (!buffer)
	return NULL;

	entry->dma_addr = dma_addr = virt_to_bus(buffer);
	assert((dma_addr & (EFX_BUF_ALIGN - 1)) == 0);

	/* Buffer table entries aren't allocated, so set id to zero */
	entry->id = 0;
	DBGP("Allocated 0x%x bytes at %p\n", bytes, buffer);

	return buffer;
	}

	/*******************************************************************************
	*
	*
	* TX
	*
	*
	******************************************************************************/
	static void
	efx_hunt_build_tx_desc(efx_tx_desc_t txd, struct io_buffer iob)
	{
	dma_addr_t dma_addr;

	dma_addr = virt_to_bus(iob->data);

	EFX_POPULATE_QWORD_4(*txd,
	ESF_DZ_TX_KER_TYPE, 0,
	ESF_DZ_TX_KER_CONT, 0,
	ESF_DZ_TX_KER_BYTE_CNT, iob_len(iob),
	ESF_DZ_TX_KER_BUF_ADDR, dma_addr);
	}

	static void
	efx_hunt_notify_tx_desc(struct efx_nic *efx)
	{
	struct efx_tx_queue *txq = &efx->txq;
	int ptr = txq->write_ptr & EFX_TXD_MASK;
	efx_dword_t reg;

	EFX_POPULATE_DWORD_1(reg, ERF_DZ_TX_DESC_WPTR_DWORD, ptr);
	efx_writel_page(efx, &reg, 0, ER_DZ_TX_DESC_UPD_DWORD);
	}

	int
	efx_hunt_transmit(struct net_device netdev, struct io_buffer iob)
	{
	struct efx_nic *efx = netdev_priv(netdev);
	struct efx_tx_queue *txq = &efx->txq;
	int fill_level, space;
	efx_tx_desc_t *txd;
	int buf_id;

	fill_level = txq->write_ptr - txq->read_ptr;
	space = EFX_TXD_SIZE - fill_level - 1;
	if (space < 1)
	return -ENOBUFS;

	/* Save the iobuffer for later completion */
	buf_id = txq->write_ptr & EFX_TXD_MASK;
	assert(txq->buf[buf_id] == NULL);
	txq->buf[buf_id] = iob;

	DBGCIO(efx, "tx_buf[%d] for iob %p data %p len %zd\n",
	buf_id, iob, iob->data, iob_len(iob));

	/* Form the descriptor, and push it to hardware */
	txd = txq->ring + buf_id;
	efx_hunt_build_tx_desc(txd, iob);
	++txq->write_ptr;
	efx_hunt_notify_tx_desc(efx);

	return 0;
	}

	static void
	efx_hunt_transmit_done(struct efx_nic *efx, int id)
	{
	struct efx_tx_queue *txq = &efx->txq;
	unsigned int read_ptr, stop;

	/* Complete all buffers from read_ptr up to and including id */
	read_ptr = txq->read_ptr & EFX_TXD_MASK;
	stop = (id + 1) & EFX_TXD_MASK;

	while (read_ptr != stop) {
	struct io_buffer *iob = txq->buf[read_ptr];

	assert(iob);
	/* Complete the tx buffer */
	if (iob)
	netdev_tx_complete(efx->netdev, iob);
	DBGCIO(efx, "tx_buf[%d] for iob %p done\n", read_ptr, iob);
	txq->buf[read_ptr] = NULL;

	++txq->read_ptr;
	read_ptr = txq->read_ptr & EFX_TXD_MASK;
	}
	}

	int efx_hunt_tx_init(struct net_device netdev, dma_addr_t dma_addr)
	{
	struct efx_nic *efx = netdev_priv(netdev);
	struct efx_tx_queue *txq = &efx->txq;
	size_t bytes;

	/* Allocate hardware transmit queue */
	bytes = sizeof(efx_tx_desc_t) * EFX_TXD_SIZE;
	txq->ring = efx_hunt_alloc_special_buffer(bytes, &txq->entry);
	if (!txq->ring)
	return -ENOMEM;

	txq->read_ptr = txq->write_ptr = 0;
	*dma_addr = txq->entry.dma_addr;
	return 0;
	}

	/*******************************************************************************
	*
	*
	* RX
	*
	*
	******************************************************************************/
	static void
	efx_hunt_build_rx_desc(efx_rx_desc_t rxd, struct io_buffer iob)
	{
	dma_addr_t dma_addr = virt_to_bus(iob->data);

	EFX_POPULATE_QWORD_2(*rxd,
	ESF_DZ_RX_KER_BYTE_CNT, EFX_RX_BUF_SIZE,
	ESF_DZ_RX_KER_BUF_ADDR, dma_addr);
	}

	static void
	efx_hunt_notify_rx_desc(struct efx_nic *efx)
	{
	struct efx_rx_queue *rxq = &efx->rxq;
	int ptr = rxq->write_ptr & EFX_RXD_MASK;
	efx_dword_t reg;

	EFX_POPULATE_DWORD_1(reg, ERF_DZ_RX_DESC_WPTR, ptr);
	efx_writel_page(efx, &reg, 0, ER_DZ_RX_DESC_UPD);
	}

	static void
	efx_hunt_rxq_fill(struct efx_nic *efx)
	{
	struct efx_rx_queue *rxq = &efx->rxq;
	int fill_level = rxq->write_ptr - rxq->read_ptr;
	int space = EFX_NUM_RX_DESC - fill_level - 1;
	int pushed = 0;

	while (space) {
	int buf_id = rxq->write_ptr & (EFX_NUM_RX_DESC - 1);
	int desc_id = rxq->write_ptr & EFX_RXD_MASK;
	struct io_buffer *iob;
	efx_rx_desc_t *rxd;

	assert(rxq->buf[buf_id] == NULL);
	iob = alloc_iob(EFX_RX_BUF_SIZE);
	if (!iob)
	break;

	DBGCP(efx, "pushing rx_buf[%d] iob %p data %p\n",
	buf_id, iob, iob->data);

	rxq->buf[buf_id] = iob;
	rxd = rxq->ring + desc_id;
	efx_hunt_build_rx_desc(rxd, iob);
	++rxq->write_ptr;
	++pushed;
	--space;
	}

	/* Push the ptr to hardware */
	if (pushed > 0) {
	efx_hunt_notify_rx_desc(efx);

	DBGCP(efx, "pushed %d rx buffers to fill level %d\n",
	pushed, rxq->write_ptr - rxq->read_ptr);
	}
	}

	static void
	efx_hunt_receive(struct efx_nic *efx, unsigned int id, int len, int drop)
	{
	struct efx_rx_queue *rxq = &efx->rxq;
	unsigned int read_ptr = rxq->read_ptr & EFX_RXD_MASK;
	unsigned int buf_ptr = rxq->read_ptr & EFX_NUM_RX_DESC_MASK;
	struct io_buffer *iob;

	/* id is the lower 4 bits of the desc index + 1 in huntington*/
	/* hence anding with 15 */
	assert((id & 15) == ((read_ptr + (len != 0)) & 15));

	/* Pop this rx buffer out of the software ring */
	iob = rxq->buf[buf_ptr];
	rxq->buf[buf_ptr] = NULL;

	DBGCIO(efx, "popping rx_buf[%d] iob %p data %p with %d bytes %s %x\n",
	read_ptr, iob, iob->data, len, drop ? "bad" : "ok", drop);

	/* Pass the packet up if required */
	if (drop)
	netdev_rx_err(efx->netdev, iob, EBADMSG);
	else {
	iob_put(iob, len);
	iob_pull(iob, efx->rx_prefix_size);
	netdev_rx(efx->netdev, iob);
	}

	++rxq->read_ptr;
	}

	int efx_hunt_rx_init(struct net_device netdev, dma_addr_t dma_addr)
	{
	struct efx_nic *efx = netdev_priv(netdev);
	struct efx_rx_queue *rxq = &efx->rxq;
	size_t bytes;

	/* Allocate hardware receive queue */
	bytes = sizeof(efx_rx_desc_t) * EFX_RXD_SIZE;
	rxq->ring = efx_hunt_alloc_special_buffer(bytes, &rxq->entry);
	if (rxq->ring == NULL)
	return -ENOMEM;

	rxq->read_ptr = rxq->write_ptr = 0;
	*dma_addr = rxq->entry.dma_addr;
	return 0;
	}

	/*******************************************************************************
	*
	*
	* Event queues and interrupts
	*
	*
	******************************************************************************/
	int efx_hunt_ev_init(struct net_device netdev, dma_addr_t dma_addr)
	{
	struct efx_nic *efx = netdev_priv(netdev);
	struct efx_ev_queue *evq = &efx->evq;
	size_t bytes;

	/* Allocate the hardware event queue */
	bytes = sizeof(efx_event_t) * EFX_EVQ_SIZE;
	evq->ring = efx_hunt_alloc_special_buffer(bytes, &evq->entry);
	if (evq->ring == NULL)
	return -ENOMEM;

	memset(evq->ring, 0xff, bytes);
	evq->read_ptr = 0;
	*dma_addr = evq->entry.dma_addr;
	return 0;
	}

	static void
	efx_hunt_clear_interrupts(struct efx_nic *efx)
	{
	efx_dword_t reg;
	/* read the ISR */
	efx_readl(efx, &reg, ER_DZ_BIU_INT_ISR);
	}

	/**
	* See if an event is present
	*
	* @v event EFX event structure
	* @ret True An event is pending
	* @ret False No event is pending
	*
	* We check both the high and low dword of the event for all ones. We
	* wrote all ones when we cleared the event, and no valid event can
	* have all ones in either its high or low dwords. This approach is
	* robust against reordering.
	*
	* Note that using a single 64-bit comparison is incorrect; even
	* though the CPU read will be atomic, the DMA write may not be.
	*/
	static inline int
	efx_hunt_event_present(efx_event_t *event)
	{
	return (!(EFX_DWORD_IS_ALL_ONES(event->dword[0]) \|
	EFX_DWORD_IS_ALL_ONES(event->dword[1])));
	}

	static void
	efx_hunt_evq_read_ack(struct efx_nic *efx)
	{
	struct efx_ev_queue *evq = &efx->evq;
	efx_dword_t reg;

	if (efx->workaround_35388) {
	EFX_POPULATE_DWORD_2(reg, ERF_DD_EVQ_IND_RPTR_FLAGS,
	EFE_DD_EVQ_IND_RPTR_FLAGS_HIGH,
	ERF_DD_EVQ_IND_RPTR,
	evq->read_ptr >> ERF_DD_EVQ_IND_RPTR_WIDTH);
	efx_writel_page(efx, &reg, 0, ER_DD_EVQ_INDIRECT);
	EFX_POPULATE_DWORD_2(reg, ERF_DD_EVQ_IND_RPTR_FLAGS,
	EFE_DD_EVQ_IND_RPTR_FLAGS_LOW,
	ERF_DD_EVQ_IND_RPTR, evq->read_ptr &
	((1 << ERF_DD_EVQ_IND_RPTR_WIDTH) - 1));
	efx_writel_page(efx, &reg, 0, ER_DD_EVQ_INDIRECT);
	} else {
	EFX_POPULATE_DWORD_1(reg, ERF_DZ_EVQ_RPTR, evq->read_ptr);
	efx_writel_table(efx, &reg, 0, ER_DZ_EVQ_RPTR);
	}
	}

	static unsigned int
	efx_hunt_handle_event(struct efx_nic efx, efx_event_t evt)
	{
	struct efx_rx_queue *rxq = &efx->rxq;
	int ev_code, desc_ptr, len;
	int next_ptr_lbits, packet_drop;
	int rx_cont;

	/* Decode event */
	ev_code = EFX_QWORD_FIELD(*evt, ESF_DZ_EV_CODE);

	switch (ev_code) {
	case ESE_DZ_EV_CODE_TX_EV:
	desc_ptr = EFX_QWORD_FIELD(*evt, ESF_DZ_TX_DESCR_INDX);
	efx_hunt_transmit_done(efx, desc_ptr);
	break;

	case ESE_DZ_EV_CODE_RX_EV:
	len = EFX_QWORD_FIELD(*evt, ESF_DZ_RX_BYTES);
	next_ptr_lbits = EFX_QWORD_FIELD(*evt, ESF_DZ_RX_DSC_PTR_LBITS);
	rx_cont = EFX_QWORD_FIELD(*evt, ESF_DZ_RX_CONT);

	/* We don't expect to receive scattered packets, so drop the
	* packet if RX_CONT is set on the current or previous event, or
	* if len is zero.
	*/
	packet_drop = (len == 0) \| (rx_cont << 1) \|
	(rxq->rx_cont_prev << 2);
	efx_hunt_receive(efx, next_ptr_lbits, len, packet_drop);
	rxq->rx_cont_prev = rx_cont;
	return 1;

	default:
	DBGCP(efx, "Unknown event type %d\n", ev_code);
	break;
	}
	return 0;
	}

	void efx_hunt_poll(struct net_device *netdev)
	{
	struct efx_nic *efx = netdev_priv(netdev);
	struct efx_ev_queue *evq = &efx->evq;
	efx_event_t *evt;
	int budget = 10;

	/* Read the event queue by directly looking for events
	* (we don't even bother to read the eventq write ptr)
	*/
	evt = evq->ring + evq->read_ptr;
	while (efx_hunt_event_present(evt) && (budget > 0)) {
	DBGCP(efx, "Event at index 0x%x address %p is "
	EFX_QWORD_FMT "\n", evq->read_ptr,
	evt, EFX_QWORD_VAL(*evt));

	budget -= efx_hunt_handle_event(efx, evt);

	/* Clear the event */
	EFX_SET_QWORD(*evt);

	/* Move to the next event. We don't ack the event
	* queue until the end
	*/
	evq->read_ptr = ((evq->read_ptr + 1) & EFX_EVQ_MASK);
	evt = evq->ring + evq->read_ptr;
	}

	/* Push more rx buffers if needed */
	efx_hunt_rxq_fill(efx);

	/* Clear any pending interrupts */
	efx_hunt_clear_interrupts(efx);

	/* Ack the event queue if interrupts are enabled */
	if (efx->int_en)
	efx_hunt_evq_read_ack(efx);
	}

	void efx_hunt_irq(struct net_device *netdev, int enable)
	{
	struct efx_nic *efx = netdev_priv(netdev);

	efx->int_en = enable;

	/* If interrupts are enabled, prime the event queue. Otherwise ack any
	* pending interrupts
	*/
	if (enable)
	efx_hunt_evq_read_ack(efx);
	else if (efx->netdev->state & NETDEV_OPEN)
	efx_hunt_clear_interrupts(efx);
	}

	/*******************************************************************************
	*
	*
	* Initialization and Close
	*
	*
	******************************************************************************/
	int efx_hunt_open(struct net_device *netdev)
	{
	struct efx_nic *efx = netdev_priv(netdev);
	efx_dword_t cmd;

	/* Set interrupt moderation to 0*/
	EFX_POPULATE_DWORD_2(cmd,
	ERF_DZ_TC_TIMER_MODE, 0,
	ERF_DZ_TC_TIMER_VAL, 0);
	efx_writel_page(efx, &cmd, 0, ER_DZ_EVQ_TMR);

	/* Ack the eventq */
	if (efx->int_en)
	efx_hunt_evq_read_ack(efx);

	/* Push receive buffers */
	efx_hunt_rxq_fill(efx);

	return 0;
	}

	void efx_hunt_close(struct net_device *netdev)
	{
	struct efx_nic *efx = netdev_priv(netdev);
	struct efx_rx_queue *rxq = &efx->rxq;
	struct efx_tx_queue *txq = &efx->txq;
	int i;

	/* Complete outstanding descriptors */
	for (i = 0; i < EFX_NUM_RX_DESC; i++) {
	if (rxq->buf[i]) {
	free_iob(rxq->buf[i]);
	rxq->buf[i] = NULL;
	}
	}

	for (i = 0; i < EFX_TXD_SIZE; i++) {
	if (txq->buf[i]) {
	netdev_tx_complete(efx->netdev, txq->buf[i]);
	txq->buf[i] = NULL;
	}
	}

	/* Clear interrupts */
	efx_hunt_clear_interrupts(efx);
	}