| /* |
| * Copyright (C) 2016 Michael Brown <mbrown@fensystems.co.uk>. |
| * |
| * This program is free software; you can redistribute it and/or |
| * modify it under the terms of the GNU General Public License as |
| * published by the Free Software Foundation; either version 2 of the |
| * License, or any later version. |
| * |
| * This program is distributed in the hope that it will be useful, but |
| * WITHOUT ANY WARRANTY; without even the implied warranty of |
| * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU |
| * General Public License for more details. |
| * |
| * You should have received a copy of the GNU General Public License |
| * along with this program; if not, write to the Free Software |
| * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA |
| * 02110-1301, USA. |
| * |
| * You can also choose to distribute this program under the terms of |
| * the Unmodified Binary Distribution Licence (as given in the file |
| * COPYING.UBDL), provided that you have satisfied its requirements. |
| */ |
| |
| FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); |
| |
| #include <string.h> |
| #include <stdio.h> |
| #include <errno.h> |
| #include <byteswap.h> |
| #include <ipxe/version.h> |
| #include <ipxe/timer.h> |
| #include <ipxe/malloc.h> |
| #include <ipxe/iobuf.h> |
| #include <ipxe/retry.h> |
| #include <ipxe/process.h> |
| #include <ipxe/settings.h> |
| #include <ipxe/infiniband.h> |
| #include <ipxe/ib_service.h> |
| #include <ipxe/ib_cmrc.h> |
| #include <ipxe/if_ether.h> |
| #include <ipxe/ethernet.h> |
| #include <ipxe/eoib.h> |
| #include <ipxe/xsigo.h> |
| |
| /** @file |
| * |
| * Xsigo virtual Ethernet devices |
| * |
| */ |
| |
| /** A Xsigo device */ |
| struct xsigo_device { |
| /** Reference count */ |
| struct refcnt refcnt; |
| /** Underlying Infiniband device */ |
| struct ib_device *ibdev; |
| /** List of Xsigo devices */ |
| struct list_head list; |
| /** Device name */ |
| const char *name; |
| |
| /** Link opener timer */ |
| struct retry_timer opener; |
| |
| /** Discovery timer */ |
| struct retry_timer discovery; |
| /** Discovery management transaction (if any) */ |
| struct ib_mad_transaction *madx; |
| |
| /** List of configuration managers */ |
| struct list_head managers; |
| }; |
| |
| /** A Xsigo configuration manager */ |
| struct xsigo_manager { |
| /** Reference count */ |
| struct refcnt refcnt; |
| /** Xsigo device */ |
| struct xsigo_device *xdev; |
| /** List of managers */ |
| struct list_head list; |
| /** Device name */ |
| char name[16]; |
| /** Manager ID */ |
| struct xsigo_manager_id id; |
| |
| /** Data transfer interface */ |
| struct interface xfer; |
| /** Connection timer */ |
| struct retry_timer reopen; |
| /** Keepalive timer */ |
| struct retry_timer keepalive; |
| /** Transmission process */ |
| struct process process; |
| /** Pending transmissions */ |
| unsigned int pending; |
| /** Transmit sequence number */ |
| uint32_t seq; |
| |
| /** List of virtual Ethernet devices */ |
| struct list_head nics; |
| }; |
| |
| /** Configuration manager pending transmissions */ |
| enum xsigo_manager_pending { |
| /** Send connection request */ |
| XCM_TX_CONNECT = 0x0001, |
| /** Send registration message */ |
| XCM_TX_REGISTER = 0x0002, |
| }; |
| |
| /** A Xsigo virtual Ethernet device */ |
| struct xsigo_nic { |
| /** Configuration manager */ |
| struct xsigo_manager *xcm; |
| /** List of virtual Ethernet devices */ |
| struct list_head list; |
| /** Device name */ |
| char name[16]; |
| |
| /** Resource identifier */ |
| union ib_guid resource; |
| /** MAC address */ |
| uint8_t mac[ETH_ALEN]; |
| /** Network ID */ |
| unsigned long network; |
| }; |
| |
| /** Configuration manager service ID */ |
| static union ib_guid xcm_service_id = { |
| .bytes = XCM_SERVICE_ID, |
| }; |
| |
| /** List of all Xsigo devices */ |
| static LIST_HEAD ( xsigo_devices ); |
| |
| /** |
| * Free Xsigo device |
| * |
| * @v refcnt Reference count |
| */ |
| static void xsigo_free ( struct refcnt *refcnt ) { |
| struct xsigo_device *xdev = |
| container_of ( refcnt, struct xsigo_device, refcnt ); |
| |
| /* Sanity checks */ |
| assert ( ! timer_running ( &xdev->opener ) ); |
| assert ( ! timer_running ( &xdev->discovery ) ); |
| assert ( xdev->madx == NULL ); |
| assert ( list_empty ( &xdev->managers ) ); |
| |
| /* Drop reference to Infiniband device */ |
| ibdev_put ( xdev->ibdev ); |
| |
| /* Free device */ |
| free ( xdev ); |
| } |
| |
| /** |
| * Free configuration manager |
| * |
| * @v refcnt Reference count |
| */ |
| static void xcm_free ( struct refcnt *refcnt ) { |
| struct xsigo_manager *xcm = |
| container_of ( refcnt, struct xsigo_manager, refcnt ); |
| |
| /* Sanity checks */ |
| assert ( ! timer_running ( &xcm->reopen ) ); |
| assert ( ! timer_running ( &xcm->keepalive ) ); |
| assert ( ! process_running ( &xcm->process ) ); |
| assert ( list_empty ( &xcm->nics ) ); |
| |
| /* Drop reference to Xsigo device */ |
| ref_put ( &xcm->xdev->refcnt ); |
| |
| /* Free manager */ |
| free ( xcm ); |
| } |
| |
| /**************************************************************************** |
| * |
| * Virtual Ethernet (XVE) devices |
| * |
| **************************************************************************** |
| */ |
| |
| /** |
| * Create virtual Ethernet device |
| * |
| * @v xcm Configuration manager |
| * @v resource Resource identifier |
| * @v mac Ethernet MAC |
| * @v network Network identifier |
| * @v name Device name |
| * @ret rc Return status code |
| */ |
| static int xve_create ( struct xsigo_manager *xcm, union ib_guid *resource, |
| const uint8_t *mac, unsigned long network, |
| unsigned long qkey, const char *name ) { |
| struct xsigo_device *xdev = xcm->xdev; |
| struct ib_device *ibdev = xdev->ibdev; |
| struct xsigo_nic *xve; |
| struct ib_address_vector broadcast; |
| int rc; |
| |
| /* Allocate and initialise structure */ |
| xve = zalloc ( sizeof ( *xve ) ); |
| if ( ! xve ) { |
| rc = -ENOMEM; |
| goto err_alloc; |
| } |
| xve->xcm = xcm; |
| snprintf ( xve->name, sizeof ( xve->name ), "%s", name ); |
| memcpy ( &xve->resource, resource, sizeof ( xve->resource ) ); |
| memcpy ( xve->mac, mac, ETH_ALEN ); |
| xve->network = network; |
| DBGC ( xve, "XVE %s created for %s " IB_GUID_FMT "\n", |
| xve->name, xcm->name, IB_GUID_ARGS ( resource ) ); |
| DBGC ( xve, "XVE %s is MAC %s on network %ld\n", |
| xve->name, eth_ntoa ( mac ), network ); |
| |
| /* Construct broadcast address vector */ |
| memset ( &broadcast, 0, sizeof ( broadcast ) ); |
| broadcast.qpn = IB_QPN_BROADCAST; |
| broadcast.qkey = qkey; |
| broadcast.gid_present = 1; |
| broadcast.gid.dwords[0] = htonl ( XVE_PREFIX ); |
| broadcast.gid.words[2] = htons ( ibdev->pkey ); |
| broadcast.gid.dwords[3] = htonl ( network ); |
| |
| /* Create EoIB device */ |
| if ( ( rc = eoib_create ( ibdev, xve->mac, &broadcast, |
| xve->name ) ) != 0 ) { |
| DBGC ( xve, "XVE %s could not create EoIB device: %s\n", |
| xve->name, strerror ( rc ) ); |
| goto err_create; |
| } |
| |
| /* Add to list of virtual Ethernet devices. Do this only |
| * after creating the EoIB device, so that our net device |
| * notifier won't attempt to send an operational state update |
| * before we have acknowledged the installation. |
| */ |
| list_add ( &xve->list, &xcm->nics ); |
| |
| return 0; |
| |
| list_del ( &xve->list ); |
| err_create: |
| free ( xve ); |
| err_alloc: |
| return rc; |
| } |
| |
| /** |
| * Find virtual Ethernet device |
| * |
| * @v xcm Configuration manager |
| * @v resource Resource identifier |
| * @ret xve Virtual Ethernet device, or NULL |
| */ |
| static struct xsigo_nic * xve_find ( struct xsigo_manager *xcm, |
| union ib_guid *resource ) { |
| struct xsigo_nic *xve; |
| |
| list_for_each_entry ( xve, &xcm->nics, list ) { |
| if ( memcmp ( resource, &xve->resource, |
| sizeof ( *resource ) ) == 0 ) |
| return xve; |
| } |
| return NULL; |
| } |
| |
| /** |
| * Destroy virtual Ethernet device |
| * |
| * @v xve Virtual Ethernet device |
| */ |
| static void xve_destroy ( struct xsigo_nic *xve ) { |
| struct xsigo_manager *xcm = xve->xcm; |
| struct xsigo_device *xdev = xcm->xdev; |
| struct ib_device *ibdev = xdev->ibdev; |
| struct eoib_device *eoib; |
| |
| /* Destroy corresponding EoIB device, if any */ |
| if ( ( eoib = eoib_find ( ibdev, xve->mac ) ) ) |
| eoib_destroy ( eoib ); |
| |
| /* Remove from list of virtual Ethernet devices */ |
| list_del ( &xve->list ); |
| |
| /* Free virtual Ethernet device */ |
| DBGC ( xve, "XVE %s destroyed\n", xve->name ); |
| free ( xve ); |
| } |
| |
| /** |
| * Update virtual Ethernet device MTU |
| * |
| * @v xve Virtual Ethernet device |
| * @v eoib EoIB device |
| * @v mtu New MTU (excluding Ethernet and EoIB headers) |
| * @ret rc Return status code |
| */ |
| static int xve_update_mtu ( struct xsigo_nic *xve, struct eoib_device *eoib, |
| size_t mtu ) { |
| struct net_device *netdev = eoib->netdev; |
| size_t max; |
| |
| /* Check that we can support this MTU */ |
| max = ( IB_MAX_PAYLOAD_SIZE - ( sizeof ( struct ethhdr ) + |
| sizeof ( struct eoib_header ) ) ); |
| if ( mtu > max ) { |
| DBGC ( xve, "XVE %s cannot support MTU %zd (max %zd)\n", |
| xve->name, mtu, max ); |
| return -ERANGE; |
| } |
| |
| /* Update MTU. No need to close/reopen the network device, |
| * since our Infiniband stack uses a fixed MTU anyway. Note |
| * that the network device sees the Ethernet frame header but |
| * not the EoIB header. |
| */ |
| netdev->max_pkt_len = ( mtu + sizeof ( struct ethhdr ) ); |
| netdev->mtu = mtu; |
| DBGC ( xve, "XVE %s has MTU %zd\n", xve->name, mtu ); |
| |
| return 0; |
| } |
| |
| /** |
| * Open virtual Ethernet device |
| * |
| * @v xve Virtual Ethernet device |
| * @v eoib EoIB device |
| * @v open New administrative state |
| * @ret rc Return status code |
| */ |
| static int xve_open ( struct xsigo_nic *xve, struct eoib_device *eoib ) { |
| struct net_device *netdev = eoib->netdev; |
| int rc; |
| |
| /* Do nothing if network device is already open */ |
| if ( netdev_is_open ( netdev ) ) |
| return 0; |
| DBGC ( xve, "XVE %s opening network device\n", xve->name ); |
| |
| /* Open network device */ |
| if ( ( rc = netdev_open ( netdev ) ) != 0 ) { |
| DBGC ( xve, "XVE %s could not open: %s\n", |
| xve->name, strerror ( rc ) ); |
| return rc; |
| } |
| |
| return 0; |
| } |
| |
| /** |
| * Close virtual Ethernet device |
| * |
| * @v xve Virtual Ethernet device |
| * @v eoib EoIB device |
| */ |
| static void xve_close ( struct xsigo_nic *xve, struct eoib_device *eoib ) { |
| struct net_device *netdev = eoib->netdev; |
| |
| /* Do nothing if network device is already closed */ |
| if ( ! netdev_is_open ( netdev ) ) |
| return; |
| |
| /* Close network device */ |
| netdev_close ( netdev ); |
| DBGC ( xve, "XVE %s closed network device\n", xve->name ); |
| } |
| |
| /** |
| * Update virtual Ethernet device administrative state |
| * |
| * @v xve Virtual Ethernet device |
| * @v eoib EoIB device |
| * @v open New administrative state |
| * @ret rc Return status code |
| */ |
| static int xve_update_state ( struct xsigo_nic *xve, struct eoib_device *eoib, |
| int open ) { |
| |
| /* Open or close device, as applicable */ |
| if ( open ) { |
| return xve_open ( xve, eoib ); |
| } else { |
| xve_close ( xve, eoib ); |
| return 0; |
| } |
| } |
| |
| /** |
| * Update gateway (TCA) |
| * |
| * @v xve Virtual Ethernet device |
| * @v eoib EoIB device |
| * @v av Address vector, or NULL if no gateway |
| * @ret rc Return status code |
| */ |
| static int xve_update_tca ( struct xsigo_nic *xve, struct eoib_device *eoib, |
| struct ib_address_vector *av ) { |
| |
| /* Update gateway address */ |
| eoib_set_gateway ( eoib, av ); |
| if ( av ) { |
| DBGC ( xve, "XVE %s has TCA " IB_GID_FMT " data %#lx qkey " |
| "%#lx\n", xve->name, IB_GID_ARGS ( &av->gid ), av->qpn, |
| av->qkey ); |
| } else { |
| DBGC ( xve, "XVE %s has no TCA\n", xve->name ); |
| } |
| |
| /* The Linux driver will modify the local device's link state |
| * to reflect the EoIB-to-Ethernet gateway's link state, but |
| * this seems philosophically incorrect since communication |
| * within the EoIB broadcast domain still works regardless of |
| * the state of the gateway. |
| */ |
| |
| return 0; |
| } |
| |
| /**************************************************************************** |
| * |
| * Server management protocol (XSMP) session messages |
| * |
| **************************************************************************** |
| */ |
| |
| /** |
| * Get session message name (for debugging) |
| * |
| * @v type Message type |
| * @ret name Message name |
| */ |
| static const char * xsmp_session_type ( unsigned int type ) { |
| static char buf[16]; |
| |
| switch ( type ) { |
| case XSMP_SESSION_TYPE_HELLO: return "HELLO"; |
| case XSMP_SESSION_TYPE_REGISTER: return "REGISTER"; |
| case XSMP_SESSION_TYPE_CONFIRM: return "CONFIRM"; |
| case XSMP_SESSION_TYPE_REJECT: return "REJECT"; |
| case XSMP_SESSION_TYPE_SHUTDOWN: return "SHUTDOWN"; |
| default: |
| snprintf ( buf, sizeof ( buf ), "UNKNOWN<%d>", type ); |
| return buf; |
| } |
| } |
| |
| /** |
| * Extract chassis name (for debugging) |
| * |
| * @v msg Session message |
| * @ret chassis Chassis name |
| */ |
| static const char * xsmp_chassis_name ( struct xsmp_session_message *msg ) { |
| static char chassis[ sizeof ( msg->chassis ) + 1 /* NUL */ ]; |
| |
| memcpy ( chassis, msg->chassis, sizeof ( msg->chassis ) ); |
| return chassis; |
| } |
| |
| /** |
| * Extract session name (for debugging) |
| * |
| * @v msg Session message |
| * @ret session Session name |
| */ |
| static const char * xsmp_session_name ( struct xsmp_session_message *msg ) { |
| static char session[ sizeof ( msg->session ) + 1 /* NUL */ ]; |
| |
| memcpy ( session, msg->session, sizeof ( msg->session ) ); |
| return session; |
| } |
| |
| /** |
| * Send session message |
| * |
| * @v xcm Configuration manager |
| * @v type Message type |
| * @ret rc Return status code |
| */ |
| static int xsmp_tx_session ( struct xsigo_manager *xcm, unsigned int type ) { |
| struct xsigo_device *xdev = xcm->xdev; |
| struct ib_device *ibdev = xdev->ibdev; |
| struct xsmp_session_message msg; |
| int rc; |
| |
| /* Construct session message */ |
| memset ( &msg, 0, sizeof ( msg ) ); |
| msg.hdr.type = XSMP_TYPE_SESSION; |
| msg.hdr.len = htons ( sizeof ( msg ) ); |
| msg.hdr.seq = htonl ( ++xcm->seq ); |
| memcpy ( &msg.hdr.src.guid, &ibdev->gid.s.guid, |
| sizeof ( msg.hdr.src.guid ) ); |
| memcpy ( &msg.hdr.dst.guid, &xcm->id.guid, |
| sizeof ( msg.hdr.dst.guid ) ); |
| msg.type = type; |
| msg.len = htons ( sizeof ( msg ) - sizeof ( msg.hdr ) ); |
| msg.os_type = XSIGO_OS_TYPE_GENERIC; |
| msg.resources = htons ( XSIGO_RESOURCE_XVE | |
| XSIGO_RESOURCE_NO_HA ); |
| msg.boot = htonl ( XSMP_BOOT_PXE ); |
| DBGCP ( xcm, "XCM %s TX[%d] session %s\n", xcm->name, |
| ntohl ( msg.hdr.seq ), xsmp_session_type ( msg.type ) ); |
| DBGCP_HDA ( xcm, 0, &msg, sizeof ( msg ) ); |
| |
| /* Send session message */ |
| if ( ( rc = xfer_deliver_raw ( &xcm->xfer, &msg, |
| sizeof ( msg ) ) ) != 0 ) { |
| DBGC ( xcm, "XCM %s TX session %s failed: %s\n", xcm->name, |
| xsmp_session_type ( msg.type ), strerror ( rc ) ); |
| return rc; |
| } |
| |
| return 0; |
| } |
| |
| /** |
| * Send registration message |
| * |
| * @v xcm Configuration manager |
| * @ret rc Return status code |
| */ |
| static inline int xsmp_tx_session_register ( struct xsigo_manager *xcm ) { |
| |
| DBGC ( xcm, "XCM %s registering with " IB_GUID_FMT "\n", |
| xcm->name, IB_GUID_ARGS ( &xcm->id.guid ) ); |
| |
| /* Send registration message */ |
| return xsmp_tx_session ( xcm, XSMP_SESSION_TYPE_REGISTER ); |
| } |
| |
| /** |
| * Send keepalive message |
| * |
| * @v xcm Configuration manager |
| * @ret rc Return status code |
| */ |
| static int xsmp_tx_session_hello ( struct xsigo_manager *xcm ) { |
| |
| /* Send keepalive message */ |
| return xsmp_tx_session ( xcm, XSMP_SESSION_TYPE_HELLO ); |
| } |
| |
| /** |
| * Handle received keepalive message |
| * |
| * @v xcm Configuration manager |
| * @v msg Keepalive message |
| * @ret rc Return status code |
| */ |
| static int xsmp_rx_session_hello ( struct xsigo_manager *xcm, |
| struct xsmp_session_message *msg __unused ) { |
| |
| /* Respond to keepalive message. Note that the XCM doesn't |
| * seem to actually ever send these. |
| */ |
| return xsmp_tx_session_hello ( xcm ); |
| } |
| |
| /** |
| * Handle received registration confirmation message |
| * |
| * @v xcm Configuration manager |
| * @v msg Registration confirmation message |
| * @ret rc Return status code |
| */ |
| static int xsmp_rx_session_confirm ( struct xsigo_manager *xcm, |
| struct xsmp_session_message *msg ) { |
| |
| DBGC ( xcm, "XCM %s registered with \"%s\" as \"%s\"\n", xcm->name, |
| xsmp_chassis_name ( msg ), xsmp_session_name ( msg ) ); |
| |
| return 0; |
| } |
| |
| /** |
| * Handle received registration rejection message |
| * |
| * @v xcm Configuration manager |
| * @v msg Registration confirmation message |
| * @ret rc Return status code |
| */ |
| static int xsmp_rx_session_reject ( struct xsigo_manager *xcm, |
| struct xsmp_session_message *msg ) { |
| |
| DBGC ( xcm, "XCM %s rejected by \"%s\":\n", |
| xcm->name, xsmp_chassis_name ( msg ) ); |
| DBGC_HDA ( xcm, 0, msg, sizeof ( *msg ) ); |
| |
| return -EPERM; |
| } |
| |
| /** |
| * Handle received shutdown message |
| * |
| * @v xcm Configuration manager |
| * @v msg Registration confirmation message |
| * @ret rc Return status code |
| */ |
| static int xsmp_rx_session_shutdown ( struct xsigo_manager *xcm, |
| struct xsmp_session_message *msg ) { |
| |
| DBGC ( xcm, "XCM %s shut down by \"%s\":\n", |
| xcm->name, xsmp_chassis_name ( msg ) ); |
| DBGC_HDA ( xcm, 0, msg, sizeof ( *msg ) ); |
| |
| return -ENOTCONN; |
| } |
| |
| /** |
| * Handle received session message |
| * |
| * @v xcm Configuration manager |
| * @v msg Session message |
| * @ret rc Return status code |
| */ |
| static int xsmp_rx_session ( struct xsigo_manager *xcm, |
| struct xsmp_session_message *msg ) { |
| |
| DBGCP ( xcm, "XCM %s RX[%d] session %s\n", xcm->name, |
| ntohl ( msg->hdr.seq ), xsmp_session_type ( msg->type ) ); |
| DBGCP_HDA ( xcm, 0, msg, sizeof ( *msg ) ); |
| |
| /* Handle message according to type */ |
| switch ( msg->type ) { |
| case XSMP_SESSION_TYPE_HELLO: |
| return xsmp_rx_session_hello ( xcm, msg ); |
| case XSMP_SESSION_TYPE_CONFIRM: |
| return xsmp_rx_session_confirm ( xcm, msg ); |
| case XSMP_SESSION_TYPE_REJECT: |
| return xsmp_rx_session_reject ( xcm, msg ); |
| case XSMP_SESSION_TYPE_SHUTDOWN: |
| return xsmp_rx_session_shutdown ( xcm, msg ); |
| default: |
| DBGC ( xcm, "XCM %s RX[%d] session unexpected %s:\n", xcm->name, |
| ntohl ( msg->hdr.seq ), xsmp_session_type ( msg->type )); |
| DBGC_HDA ( xcm, 0, msg, sizeof ( *msg ) ); |
| return -EPROTO; |
| } |
| } |
| |
| /**************************************************************************** |
| * |
| * Server management protocol (XSMP) virtual Ethernet (XVE) messages |
| * |
| **************************************************************************** |
| */ |
| |
| /** |
| * Get virtual Ethernet message name (for debugging) |
| * |
| * @v type Message type |
| * @ret name Message name |
| */ |
| static const char * xsmp_xve_type ( unsigned int type ) { |
| static char buf[16]; |
| |
| switch ( type ) { |
| case XSMP_XVE_TYPE_INSTALL: return "INSTALL"; |
| case XSMP_XVE_TYPE_DELETE: return "DELETE"; |
| case XSMP_XVE_TYPE_UPDATE: return "UPDATE"; |
| case XSMP_XVE_TYPE_OPER_UP: return "OPER_UP"; |
| case XSMP_XVE_TYPE_OPER_DOWN: return "OPER_DOWN"; |
| case XSMP_XVE_TYPE_OPER_REQ: return "OPER_REQ"; |
| case XSMP_XVE_TYPE_READY: return "READY"; |
| default: |
| snprintf ( buf, sizeof ( buf ), "UNKNOWN<%d>", type ); |
| return buf; |
| } |
| } |
| |
| /** |
| * Send virtual Ethernet message |
| * |
| * @v xcm Configuration manager |
| * @v msg Partial message |
| * @ret rc Return status code |
| */ |
| static int xsmp_tx_xve ( struct xsigo_manager *xcm, |
| struct xsmp_xve_message *msg ) { |
| struct xsigo_device *xdev = xcm->xdev; |
| struct ib_device *ibdev = xdev->ibdev; |
| int rc; |
| |
| /* Fill in common header fields */ |
| msg->hdr.type = XSMP_TYPE_XVE; |
| msg->hdr.len = htons ( sizeof ( *msg ) ); |
| msg->hdr.seq = htonl ( ++xcm->seq ); |
| memcpy ( &msg->hdr.src.guid, &ibdev->gid.s.guid, |
| sizeof ( msg->hdr.src.guid ) ); |
| memcpy ( &msg->hdr.dst.guid, &xcm->id.guid, |
| sizeof ( msg->hdr.dst.guid ) ); |
| msg->len = htons ( sizeof ( *msg ) - sizeof ( msg->hdr ) ); |
| DBGCP ( xcm, "XCM %s TX[%d] xve %s code %#02x\n", xcm->name, |
| ntohl ( msg->hdr.seq ), xsmp_xve_type ( msg->type ), |
| msg->code ); |
| DBGCP_HDA ( xcm, 0, msg, sizeof ( *msg ) ); |
| |
| /* Send virtual Ethernet message */ |
| if ( ( rc = xfer_deliver_raw ( &xcm->xfer, msg, |
| sizeof ( *msg ) ) ) != 0 ) { |
| DBGC ( xcm, "XCM %s TX xve %s failed: %s\n", xcm->name, |
| xsmp_xve_type ( msg->type ), strerror ( rc ) ); |
| return rc; |
| } |
| |
| return 0; |
| } |
| |
| /** |
| * Send virtual Ethernet message including current device parameters |
| * |
| * @v xcm Configuration manager |
| * @v msg Partial virtual Ethernet message |
| * @v xve Virtual Ethernet device |
| * @v eoib EoIB device |
| * @ret rc Return status code |
| */ |
| static int xsmp_tx_xve_params ( struct xsigo_manager *xcm, |
| struct xsmp_xve_message *msg, |
| struct xsigo_nic *xve, |
| struct eoib_device *eoib ) { |
| struct xsigo_device *xdev = xcm->xdev; |
| struct ib_device *ibdev = xdev->ibdev; |
| struct net_device *netdev = eoib->netdev; |
| |
| /* Set successful response code */ |
| msg->code = 0; |
| |
| /* Include network identifier, MTU, and current HCA parameters */ |
| msg->network = htonl ( xve->network ); |
| msg->mtu = htons ( netdev->max_pkt_len - sizeof ( struct ethhdr ) ); |
| msg->hca.prefix_le.qword = bswap_64 ( ibdev->gid.s.prefix.qword ); |
| msg->hca.pkey = htons ( ibdev->pkey ); |
| msg->hca.qkey = msg->tca.qkey; |
| if ( eoib->qp ) { |
| msg->hca.data = htonl ( eoib->qp->ext_qpn ); |
| msg->hca.qkey = htons ( eoib->qp->qkey ); |
| } |
| |
| /* The message type field is (ab)used to return the current |
| * operational status. |
| */ |
| if ( msg->type == XSMP_XVE_TYPE_OPER_REQ ) { |
| msg->type = ( netdev_is_open ( netdev ) ? |
| XSMP_XVE_TYPE_OPER_UP : XSMP_XVE_TYPE_OPER_DOWN ); |
| } |
| |
| /* Send message */ |
| DBGC ( xve, "XVE %s network %d MTU %d ctrl %#x data %#x qkey %#04x " |
| "%s\n", xve->name, ntohl ( msg->network ), ntohs ( msg->mtu ), |
| ntohl ( msg->hca.ctrl ), ntohl ( msg->hca.data ), |
| ntohs ( msg->hca.qkey ), xsmp_xve_type ( msg->type ) ); |
| |
| return xsmp_tx_xve ( xcm, msg ); |
| } |
| |
| /** |
| * Send virtual Ethernet error response |
| * |
| * @v xcm Configuration manager |
| * @v msg Partial virtual Ethernet message |
| * @ret rc Return status code |
| */ |
| static inline int xsmp_tx_xve_nack ( struct xsigo_manager *xcm, |
| struct xsmp_xve_message *msg ) { |
| |
| /* Set error response code. (There aren't any meaningful |
| * detailed response codes defined by the wire protocol.) |
| */ |
| msg->code = XSMP_XVE_CODE_ERROR; |
| |
| /* Send message */ |
| return xsmp_tx_xve ( xcm, msg ); |
| } |
| |
| /** |
| * Send virtual Ethernet notification |
| * |
| * @v xcm Configuration manager |
| * @v type Message type |
| * @v xve Virtual Ethernet device |
| * @v eoib EoIB device |
| * @ret rc Return status code |
| */ |
| static int xsmp_tx_xve_notify ( struct xsigo_manager *xcm, |
| unsigned int type, |
| struct xsigo_nic *xve, |
| struct eoib_device *eoib ) { |
| struct xsmp_xve_message msg; |
| |
| /* Construct message */ |
| memset ( &msg, 0, sizeof ( msg ) ); |
| msg.type = type; |
| memcpy ( &msg.resource, &xve->resource, sizeof ( msg.resource ) ); |
| |
| /* Send message */ |
| return xsmp_tx_xve_params ( xcm, &msg, xve, eoib ); |
| } |
| |
| /** |
| * Send virtual Ethernet current operational state |
| * |
| * @v xcm Configuration manager |
| * @v xve Virtual Ethernet device |
| * @v eoib EoIB device |
| * @ret rc Return status code |
| */ |
| static inline int xsmp_tx_xve_oper ( struct xsigo_manager *xcm, |
| struct xsigo_nic *xve, |
| struct eoib_device *eoib ) { |
| |
| /* Send notification */ |
| return xsmp_tx_xve_notify ( xcm, XSMP_XVE_TYPE_OPER_REQ, xve, eoib ); |
| } |
| |
| /** |
| * Handle received virtual Ethernet modification message |
| * |
| * @v xcm Configuration manager |
| * @v msg Virtual Ethernet message |
| * @v update Update bitmask |
| * @ret rc Return status code |
| */ |
| static int xsmp_rx_xve_modify ( struct xsigo_manager *xcm, |
| struct xsmp_xve_message *msg, |
| unsigned int update ) { |
| struct xsigo_device *xdev = xcm->xdev; |
| struct ib_device *ibdev = xdev->ibdev; |
| struct xsigo_nic *xve; |
| struct eoib_device *eoib; |
| struct ib_address_vector tca; |
| size_t mtu; |
| int rc; |
| |
| /* Avoid returning uninitialised HCA parameters in response */ |
| memset ( &msg->hca, 0, sizeof ( msg->hca ) ); |
| |
| /* Find virtual Ethernet device */ |
| xve = xve_find ( xcm, &msg->resource ); |
| if ( ! xve ) { |
| DBGC ( xcm, "XCM %s unrecognised resource " IB_GUID_FMT "\n", |
| xcm->name, IB_GUID_ARGS ( &msg->resource ) ); |
| rc = -ENOENT; |
| goto err_no_xve; |
| } |
| |
| /* Find corresponding EoIB device */ |
| eoib = eoib_find ( ibdev, xve->mac ); |
| if ( ! eoib ) { |
| DBGC ( xve, "XVE %s has no EoIB device\n", xve->name ); |
| rc = -EPIPE; |
| goto err_no_eoib; |
| } |
| |
| /* The Xsigo management software fails to create the EoIB |
| * multicast group. This is a fundamental design flaw. |
| */ |
| eoib_force_group_creation ( eoib ); |
| |
| /* Extract modifiable parameters. Note that the TCA GID is |
| * erroneously transmitted as little-endian. |
| */ |
| mtu = ntohs ( msg->mtu ); |
| tca.qpn = ntohl ( msg->tca.data ); |
| tca.qkey = ntohs ( msg->tca.qkey ); |
| tca.gid_present = 1; |
| tca.gid.s.prefix.qword = bswap_64 ( msg->tca.prefix_le.qword ); |
| tca.gid.s.guid.qword = bswap_64 ( msg->guid_le.qword ); |
| |
| /* Update MTU, if applicable */ |
| if ( ( update & XSMP_XVE_UPDATE_MTU ) && |
| ( ( rc = xve_update_mtu ( xve, eoib, mtu ) ) != 0 ) ) |
| goto err_mtu; |
| update &= ~XSMP_XVE_UPDATE_MTU; |
| |
| /* Update admin state, if applicable */ |
| if ( ( update & XSMP_XVE_UPDATE_STATE ) && |
| ( ( rc = xve_update_state ( xve, eoib, msg->state ) ) != 0 ) ) |
| goto err_state; |
| update &= ~XSMP_XVE_UPDATE_STATE; |
| |
| /* Remove gateway, if applicable */ |
| if ( ( update & XSMP_XVE_UPDATE_GW_DOWN ) && |
| ( ( rc = xve_update_tca ( xve, eoib, NULL ) ) != 0 ) ) |
| goto err_gw_down; |
| update &= ~XSMP_XVE_UPDATE_GW_DOWN; |
| |
| /* Update gateway, if applicable */ |
| if ( ( update & XSMP_XVE_UPDATE_GW_CHANGE ) && |
| ( ( rc = xve_update_tca ( xve, eoib, &tca ) ) != 0 ) ) |
| goto err_gw_change; |
| update &= ~XSMP_XVE_UPDATE_GW_CHANGE; |
| |
| /* Warn about unexpected updates */ |
| if ( update ) { |
| DBGC ( xve, "XVE %s unrecognised update(s) %#08x\n", |
| xve->name, update ); |
| } |
| |
| xsmp_tx_xve_params ( xcm, msg, xve, eoib ); |
| return 0; |
| |
| err_gw_change: |
| err_gw_down: |
| err_state: |
| err_mtu: |
| err_no_eoib: |
| err_no_xve: |
| /* Send NACK */ |
| xsmp_tx_xve_nack ( xcm, msg ); |
| return rc; |
| } |
| |
| /** |
| * Handle received virtual Ethernet installation message |
| * |
| * @v xcm Configuration manager |
| * @v msg Virtual Ethernet message |
| * @ret rc Return status code |
| */ |
| static int xsmp_rx_xve_install ( struct xsigo_manager *xcm, |
| struct xsmp_xve_message *msg ) { |
| union { |
| struct xsmp_xve_mac msg; |
| uint8_t raw[ETH_ALEN]; |
| } mac; |
| char name[ sizeof ( msg->name ) + 1 /* NUL */ ]; |
| unsigned long network; |
| unsigned long qkey; |
| unsigned int update; |
| int rc; |
| |
| /* Demangle MAC address (which is erroneously transmitted as |
| * little-endian). |
| */ |
| mac.msg.high = bswap_16 ( msg->mac_le.high ); |
| mac.msg.low = bswap_32 ( msg->mac_le.low ); |
| |
| /* Extract interface name (which may not be NUL-terminated) */ |
| memcpy ( name, msg->name, ( sizeof ( name ) - 1 /* NUL */ ) ); |
| name[ sizeof ( name ) - 1 /* NUL */ ] = '\0'; |
| |
| /* Extract remaining message parameters */ |
| network = ntohl ( msg->network ); |
| qkey = ntohs ( msg->tca.qkey ); |
| DBGC2 ( xcm, "XCM %s " IB_GUID_FMT " install \"%s\" %s net %ld qkey " |
| "%#lx\n", xcm->name, IB_GUID_ARGS ( &msg->resource ), name, |
| eth_ntoa ( mac.raw ), network, qkey ); |
| |
| /* Create virtual Ethernet device, if applicable */ |
| if ( ( xve_find ( xcm, &msg->resource ) == NULL ) && |
| ( ( rc = xve_create ( xcm, &msg->resource, mac.raw, network, |
| qkey, name ) ) != 0 ) ) |
| goto err_create; |
| |
| /* Handle remaining parameters as for a modification message */ |
| update = XSMP_XVE_UPDATE_MTU; |
| if ( msg->uplink == XSMP_XVE_UPLINK ) |
| update |= XSMP_XVE_UPDATE_GW_CHANGE; |
| return xsmp_rx_xve_modify ( xcm, msg, update ); |
| |
| err_create: |
| /* Send NACK */ |
| xsmp_tx_xve_nack ( xcm, msg ); |
| return rc; |
| } |
| |
| /** |
| * Handle received virtual Ethernet deletion message |
| * |
| * @v xcm Configuration manager |
| * @v msg Virtual Ethernet message |
| * @ret rc Return status code |
| */ |
| static int xsmp_rx_xve_delete ( struct xsigo_manager *xcm, |
| struct xsmp_xve_message *msg ) { |
| struct xsigo_nic *xve; |
| |
| DBGC2 ( xcm, "XCM %s " IB_GUID_FMT " delete\n", |
| xcm->name, IB_GUID_ARGS ( &msg->resource ) ); |
| |
| /* Destroy virtual Ethernet device (if any) */ |
| if ( ( xve = xve_find ( xcm, &msg->resource ) ) ) |
| xve_destroy ( xve ); |
| |
| /* Send ACK */ |
| msg->code = 0; |
| xsmp_tx_xve ( xcm, msg ); |
| |
| return 0; |
| } |
| |
| /** |
| * Handle received virtual Ethernet update message |
| * |
| * @v xcm Configuration manager |
| * @v msg Virtual Ethernet message |
| * @ret rc Return status code |
| */ |
| static int xsmp_rx_xve_update ( struct xsigo_manager *xcm, |
| struct xsmp_xve_message *msg ) { |
| unsigned int update = ntohl ( msg->update ); |
| |
| DBGC2 ( xcm, "XCM %s " IB_GUID_FMT " update (%08x)\n", |
| xcm->name, IB_GUID_ARGS ( &msg->resource ), update ); |
| |
| /* Handle as a modification message */ |
| return xsmp_rx_xve_modify ( xcm, msg, update ); |
| } |
| |
| /** |
| * Handle received virtual Ethernet operational request message |
| * |
| * @v xcm Configuration manager |
| * @v msg Virtual Ethernet message |
| * @ret rc Return status code |
| */ |
| static int xsmp_rx_xve_oper_req ( struct xsigo_manager *xcm, |
| struct xsmp_xve_message *msg ) { |
| |
| DBGC2 ( xcm, "XCM %s " IB_GUID_FMT " operational request\n", |
| xcm->name, IB_GUID_ARGS ( &msg->resource ) ); |
| |
| /* Handle as a nullipotent modification message */ |
| return xsmp_rx_xve_modify ( xcm, msg, 0 ); |
| } |
| |
| /** |
| * Handle received virtual Ethernet readiness message |
| * |
| * @v xcm Configuration manager |
| * @v msg Virtual Ethernet message |
| * @ret rc Return status code |
| */ |
| static int xsmp_rx_xve_ready ( struct xsigo_manager *xcm, |
| struct xsmp_xve_message *msg ) { |
| int rc; |
| |
| DBGC2 ( xcm, "XCM %s " IB_GUID_FMT " ready\n", |
| xcm->name, IB_GUID_ARGS ( &msg->resource ) ); |
| |
| /* Handle as a nullipotent modification message */ |
| if ( ( rc = xsmp_rx_xve_modify ( xcm, msg, 0 ) ) != 0 ) |
| return rc; |
| |
| /* Send an unsolicited operational state update, since there |
| * is no other way to convey the current operational state. |
| */ |
| msg->type = XSMP_XVE_TYPE_OPER_REQ; |
| if ( ( rc = xsmp_rx_xve_modify ( xcm, msg, 0 ) ) != 0 ) |
| return rc; |
| |
| return 0; |
| } |
| |
| /** |
| * Handle received virtual Ethernet message |
| * |
| * @v xcm Configuration manager |
| * @v msg Virtual Ethernet message |
| * @ret rc Return status code |
| */ |
| static int xsmp_rx_xve ( struct xsigo_manager *xcm, |
| struct xsmp_xve_message *msg ) { |
| |
| DBGCP ( xcm, "XCM %s RX[%d] xve %s\n", xcm->name, |
| ntohl ( msg->hdr.seq ), xsmp_xve_type ( msg->type ) ); |
| DBGCP_HDA ( xcm, 0, msg, sizeof ( *msg ) ); |
| |
| /* Handle message according to type */ |
| switch ( msg->type ) { |
| case XSMP_XVE_TYPE_INSTALL: |
| return xsmp_rx_xve_install ( xcm, msg ); |
| case XSMP_XVE_TYPE_DELETE: |
| return xsmp_rx_xve_delete ( xcm, msg ); |
| case XSMP_XVE_TYPE_UPDATE: |
| return xsmp_rx_xve_update ( xcm, msg ); |
| case XSMP_XVE_TYPE_OPER_REQ: |
| return xsmp_rx_xve_oper_req ( xcm, msg ); |
| case XSMP_XVE_TYPE_READY: |
| return xsmp_rx_xve_ready ( xcm, msg ); |
| default: |
| DBGC ( xcm, "XCM %s RX[%d] xve unexpected %s:\n", xcm->name, |
| ntohl ( msg->hdr.seq ), xsmp_xve_type ( msg->type ) ); |
| DBGC_HDA ( xcm, 0, msg, sizeof ( *msg ) ); |
| return -EPROTO; |
| } |
| } |
| |
| /**************************************************************************** |
| * |
| * Configuration managers (XCM) |
| * |
| **************************************************************************** |
| */ |
| |
| /** |
| * Close configuration manager connection |
| * |
| * @v xcm Configuration manager |
| * @v rc Reason for close |
| */ |
| static void xcm_close ( struct xsigo_manager *xcm, int rc ) { |
| |
| DBGC ( xcm, "XCM %s closed: %s\n", xcm->name, strerror ( rc ) ); |
| |
| /* Stop transmission process */ |
| process_del ( &xcm->process ); |
| |
| /* Stop keepalive timer */ |
| stop_timer ( &xcm->keepalive ); |
| |
| /* Restart data transfer interface */ |
| intf_restart ( &xcm->xfer, rc ); |
| |
| /* Schedule reconnection attempt */ |
| start_timer ( &xcm->reopen ); |
| } |
| |
| /** |
| * Send data to configuration manager |
| * |
| * @v xcm Configuration manager |
| */ |
| static void xcm_step ( struct xsigo_manager *xcm ) { |
| int rc; |
| |
| /* Do nothing unless we have something to send */ |
| if ( ! xcm->pending ) |
| return; |
| |
| /* Send (empty) connection request, if applicable */ |
| if ( xcm->pending & XCM_TX_CONNECT ) { |
| if ( ( rc = xfer_deliver_raw ( &xcm->xfer, NULL, 0 ) ) != 0 ) { |
| DBGC ( xcm, "XCM %s could not send connection request: " |
| "%s\n", xcm->name, strerror ( rc ) ); |
| goto err; |
| } |
| xcm->pending &= ~XCM_TX_CONNECT; |
| return; |
| } |
| |
| /* Wait until data transfer interface is connected */ |
| if ( ! xfer_window ( &xcm->xfer ) ) |
| return; |
| |
| /* Send registration message, if applicable */ |
| if ( xcm->pending & XCM_TX_REGISTER ) { |
| if ( ( rc = xsmp_tx_session_register ( xcm ) ) != 0 ) |
| goto err; |
| xcm->pending &= ~XCM_TX_REGISTER; |
| return; |
| } |
| |
| return; |
| |
| err: |
| xcm_close ( xcm, rc ); |
| } |
| |
| /** |
| * Receive data from configuration manager |
| * |
| * @v xcm Configuration manager |
| * @v iobuf I/O buffer |
| * @v meta Data transfer metadata |
| * @ret rc Return status code |
| */ |
| static int xcm_deliver ( struct xsigo_manager *xcm, struct io_buffer *iobuf, |
| struct xfer_metadata *meta __unused ) { |
| union xsmp_message *msg; |
| size_t len = iob_len ( iobuf ); |
| int rc; |
| |
| /* Sanity check */ |
| if ( len < sizeof ( msg->hdr ) ) { |
| DBGC ( xcm, "XCM %s underlength message:\n", xcm->name ); |
| DBGC_HDA ( xcm, 0, iobuf->data, iob_len ( iobuf ) ); |
| rc = -EPROTO; |
| goto out; |
| } |
| msg = iobuf->data; |
| |
| /* Handle message according to type */ |
| if ( ! msg->hdr.type ) { |
| |
| /* Ignore unused communication manager private data blocks */ |
| rc = 0; |
| |
| } else if ( ( msg->hdr.type == XSMP_TYPE_SESSION ) && |
| ( len >= sizeof ( msg->sess ) ) ) { |
| |
| /* Session message */ |
| rc = xsmp_rx_session ( xcm, &msg->sess ); |
| |
| } else if ( ( msg->hdr.type == XSMP_TYPE_XVE ) && |
| ( len >= sizeof ( msg->xve ) ) ) { |
| |
| /* Virtual Ethernet message */ |
| xsmp_rx_xve ( xcm, &msg->xve ); |
| |
| /* Virtual Ethernet message errors are non-fatal */ |
| rc = 0; |
| |
| } else { |
| |
| /* Unknown message */ |
| DBGC ( xcm, "XCM %s unexpected message type %d:\n", |
| xcm->name, msg->hdr.type ); |
| DBGC_HDA ( xcm, 0, iobuf->data, iob_len ( iobuf ) ); |
| rc = -EPROTO; |
| } |
| |
| out: |
| free_iob ( iobuf ); |
| if ( rc != 0 ) |
| xcm_close ( xcm, rc ); |
| return rc; |
| } |
| |
| /** Configuration manager data transfer interface operations */ |
| static struct interface_operation xcm_xfer_op[] = { |
| INTF_OP ( xfer_deliver, struct xsigo_manager *, xcm_deliver ), |
| INTF_OP ( xfer_window_changed, struct xsigo_manager *, xcm_step ), |
| INTF_OP ( intf_close, struct xsigo_manager *, xcm_close ), |
| }; |
| |
| /** Configuration manager data transfer interface descriptor */ |
| static struct interface_descriptor xcm_xfer_desc = |
| INTF_DESC ( struct xsigo_manager, xfer, xcm_xfer_op ); |
| |
| /** Configuration manager process descriptor */ |
| static struct process_descriptor xcm_process_desc = |
| PROC_DESC_ONCE ( struct xsigo_manager, process, xcm_step ); |
| |
| /** |
| * Handle configuration manager connection timer expiry |
| * |
| * @v timer Connection timer |
| * @v fail Failure indicator |
| */ |
| static void xcm_reopen ( struct retry_timer *timer, int fail __unused ) { |
| struct xsigo_manager *xcm = |
| container_of ( timer, struct xsigo_manager, reopen ); |
| struct xsigo_device *xdev = xcm->xdev; |
| struct ib_device *ibdev = xdev->ibdev; |
| union ib_gid gid; |
| int rc; |
| |
| /* Stop transmission process */ |
| process_del ( &xcm->process ); |
| |
| /* Stop keepalive timer */ |
| stop_timer ( &xcm->keepalive ); |
| |
| /* Restart data transfer interface */ |
| intf_restart ( &xcm->xfer, -ECANCELED ); |
| |
| /* Reset sequence number */ |
| xcm->seq = 0; |
| |
| /* Construct GID */ |
| memcpy ( &gid.s.prefix, &ibdev->gid.s.prefix, sizeof ( gid.s.prefix ) ); |
| memcpy ( &gid.s.guid, &xcm->id.guid, sizeof ( gid.s.guid ) ); |
| DBGC ( xcm, "XCM %s connecting to " IB_GID_FMT "\n", |
| xcm->name, IB_GID_ARGS ( &gid ) ); |
| |
| /* Open CMRC connection */ |
| if ( ( rc = ib_cmrc_open ( &xcm->xfer, ibdev, &gid, |
| &xcm_service_id, xcm->name ) ) != 0 ) { |
| DBGC ( xcm, "XCM %s could not open CMRC connection: %s\n", |
| xcm->name, strerror ( rc ) ); |
| start_timer ( &xcm->reopen ); |
| return; |
| } |
| |
| /* Schedule transmissions */ |
| xcm->pending |= ( XCM_TX_CONNECT | XCM_TX_REGISTER ); |
| process_add ( &xcm->process ); |
| |
| /* Start keepalive timer */ |
| start_timer_fixed ( &xcm->keepalive, XSIGO_KEEPALIVE_INTERVAL ); |
| |
| return; |
| } |
| |
| /** |
| * Handle configuration manager keepalive timer expiry |
| * |
| * @v timer Connection timer |
| * @v fail Failure indicator |
| */ |
| static void xcm_keepalive ( struct retry_timer *timer, int fail __unused ) { |
| struct xsigo_manager *xcm = |
| container_of ( timer, struct xsigo_manager, keepalive ); |
| int rc; |
| |
| /* Send keepalive message. The server won't actually respond |
| * to these, but it gives the RC queue pair a chance to |
| * complain if it doesn't ever at least get an ACK. |
| */ |
| if ( ( rc = xsmp_tx_session_hello ( xcm ) ) != 0 ) { |
| xcm_close ( xcm, rc ); |
| return; |
| } |
| |
| /* Restart keepalive timer */ |
| start_timer_fixed ( &xcm->keepalive, XSIGO_KEEPALIVE_INTERVAL ); |
| } |
| |
| /** |
| * Create configuration manager |
| * |
| * @v xsigo Xsigo device |
| * @v id Configuration manager ID |
| * @ret rc Return status code |
| */ |
| static int xcm_create ( struct xsigo_device *xdev, |
| struct xsigo_manager_id *id ) { |
| struct xsigo_manager *xcm; |
| |
| /* Allocate and initialise structure */ |
| xcm = zalloc ( sizeof ( *xcm ) ); |
| if ( ! xcm ) |
| return -ENOMEM; |
| ref_init ( &xcm->refcnt, xcm_free ); |
| xcm->xdev = xdev; |
| ref_get ( &xcm->xdev->refcnt ); |
| snprintf ( xcm->name, sizeof ( xcm->name ), "%s:xcm-%d", |
| xdev->name, ntohs ( id->lid ) ); |
| memcpy ( &xcm->id, id, sizeof ( xcm->id ) ); |
| intf_init ( &xcm->xfer, &xcm_xfer_desc, &xcm->refcnt ); |
| timer_init ( &xcm->keepalive, xcm_keepalive, &xcm->refcnt ); |
| timer_init ( &xcm->reopen, xcm_reopen, &xcm->refcnt ); |
| process_init_stopped ( &xcm->process, &xcm_process_desc, &xcm->refcnt ); |
| INIT_LIST_HEAD ( &xcm->nics ); |
| |
| /* Start timer to open connection */ |
| start_timer_nodelay ( &xcm->reopen ); |
| |
| /* Add to list of managers and transfer reference to list */ |
| list_add ( &xcm->list, &xdev->managers ); |
| DBGC ( xcm, "XCM %s created for " IB_GUID_FMT " (LID %d)\n", xcm->name, |
| IB_GUID_ARGS ( &xcm->id.guid ), ntohs ( id->lid ) ); |
| return 0; |
| } |
| |
| /** |
| * Find configuration manager |
| * |
| * @v xsigo Xsigo device |
| * @v id Configuration manager ID |
| * @ret xcm Configuration manager, or NULL |
| */ |
| static struct xsigo_manager * xcm_find ( struct xsigo_device *xdev, |
| struct xsigo_manager_id *id ) { |
| struct xsigo_manager *xcm; |
| union ib_guid *guid = &id->guid; |
| |
| /* Find configuration manager */ |
| list_for_each_entry ( xcm, &xdev->managers, list ) { |
| if ( memcmp ( guid, &xcm->id.guid, sizeof ( *guid ) ) == 0 ) |
| return xcm; |
| } |
| return NULL; |
| } |
| |
| /** |
| * Destroy configuration manager |
| * |
| * @v xcm Configuration manager |
| */ |
| static void xcm_destroy ( struct xsigo_manager *xcm ) { |
| struct xsigo_nic *xve; |
| |
| /* Remove all EoIB NICs */ |
| while ( ( xve = list_first_entry ( &xcm->nics, struct xsigo_nic, |
| list ) ) ) { |
| xve_destroy ( xve ); |
| } |
| |
| /* Stop transmission process */ |
| process_del ( &xcm->process ); |
| |
| /* Stop timers */ |
| stop_timer ( &xcm->keepalive ); |
| stop_timer ( &xcm->reopen ); |
| |
| /* Shut down data transfer interface */ |
| intf_shutdown ( &xcm->xfer, 0 ); |
| |
| /* Remove from list of managers and drop list's reference */ |
| DBGC ( xcm, "XCM %s destroyed\n", xcm->name ); |
| list_del ( &xcm->list ); |
| ref_put ( &xcm->refcnt ); |
| } |
| |
| /** |
| * Synchronise list of configuration managers |
| * |
| * @v xdev Xsigo device |
| * @v ids List of manager IDs |
| * @v count Number of manager IDs |
| * @ret rc Return status code |
| */ |
| static int xcm_list ( struct xsigo_device *xdev, struct xsigo_manager_id *ids, |
| unsigned int count ) { |
| struct xsigo_manager_id *id; |
| struct xsigo_manager *xcm; |
| struct xsigo_manager *tmp; |
| struct list_head list; |
| unsigned int i; |
| int rc; |
| |
| /* Create list of managers to be retained */ |
| INIT_LIST_HEAD ( &list ); |
| for ( i = 0, id = ids ; i < count ; i++, id++ ) { |
| if ( ( xcm = xcm_find ( xdev, id ) ) ) { |
| list_del ( &xcm->list ); |
| list_add_tail ( &xcm->list, &list ); |
| } |
| } |
| |
| /* Destroy any managers not in the list */ |
| list_for_each_entry_safe ( xcm, tmp, &xdev->managers, list ) |
| xcm_destroy ( xcm ); |
| list_splice ( &list, &xdev->managers ); |
| |
| /* Create any new managers in the list, and force reconnection |
| * for any changed LIDs. |
| */ |
| for ( i = 0, id = ids ; i < count ; i++, id++ ) { |
| if ( ( xcm = xcm_find ( xdev, id ) ) ) { |
| if ( xcm->id.lid != id->lid ) |
| start_timer_nodelay ( &xcm->reopen ); |
| continue; |
| } |
| if ( ( rc = xcm_create ( xdev, id ) ) != 0 ) { |
| DBGC ( xdev, "XDEV %s could not create manager: %s\n", |
| xdev->name, strerror ( rc ) ); |
| return rc; |
| } |
| } |
| |
| return 0; |
| } |
| |
| /**************************************************************************** |
| * |
| * Configuration manager discovery |
| * |
| **************************************************************************** |
| */ |
| |
| /** A stage of discovery */ |
| struct xsigo_discovery { |
| /** Name */ |
| const char *name; |
| /** Management transaction operations */ |
| struct ib_mad_transaction_operations op; |
| }; |
| |
| /** |
| * Handle configuration manager lookup completion |
| * |
| * @v ibdev Infiniband device |
| * @v mi Management interface |
| * @v madx Management transaction |
| * @v rc Status code |
| * @v mad Received MAD (or NULL on error) |
| * @v av Source address vector (or NULL on error) |
| */ |
| static void xsigo_xcm_complete ( struct ib_device *ibdev, |
| struct ib_mad_interface *mi __unused, |
| struct ib_mad_transaction *madx, |
| int rc, union ib_mad *mad, |
| struct ib_address_vector *av __unused ) { |
| struct xsigo_device *xdev = ib_madx_get_ownerdata ( madx ); |
| union xsigo_mad *xsmad = container_of ( mad, union xsigo_mad, mad ); |
| struct xsigo_managers_reply *reply = &xsmad->reply; |
| |
| /* Check for failures */ |
| if ( ( rc == 0 ) && ( mad->hdr.status != htons ( IB_MGMT_STATUS_OK ) ) ) |
| rc = -ENODEV; |
| if ( rc != 0 ) { |
| DBGC ( xdev, "XDEV %s manager lookup failed: %s\n", |
| xdev->name, strerror ( rc ) ); |
| goto out; |
| } |
| |
| /* Sanity checks */ |
| if ( reply->count > ( sizeof ( reply->manager ) / |
| sizeof ( reply->manager[0] ) ) ) { |
| DBGC ( xdev, "XDEV %s has too many managers (%d)\n", |
| xdev->name, reply->count ); |
| goto out; |
| } |
| |
| /* Synchronise list of managers */ |
| if ( ( rc = xcm_list ( xdev, reply->manager, reply->count ) ) != 0 ) |
| goto out; |
| |
| /* Report an empty list of managers */ |
| if ( reply->count == 0 ) |
| DBGC ( xdev, "XDEV %s has no managers\n", xdev->name ); |
| |
| /* Delay next discovery attempt */ |
| start_timer_fixed ( &xdev->discovery, XSIGO_DISCOVERY_SUCCESS_DELAY ); |
| |
| out: |
| /* Destroy the completed transaction */ |
| ib_destroy_madx ( ibdev, ibdev->gsi, madx ); |
| xdev->madx = NULL; |
| } |
| |
| /** Configuration manager lookup discovery stage */ |
| static struct xsigo_discovery xsigo_xcm_discovery = { |
| .name = "manager", |
| .op = { |
| .complete = xsigo_xcm_complete, |
| }, |
| }; |
| |
| /** |
| * Handle directory service lookup completion |
| * |
| * @v ibdev Infiniband device |
| * @v mi Management interface |
| * @v madx Management transaction |
| * @v rc Status code |
| * @v mad Received MAD (or NULL on error) |
| * @v av Source address vector (or NULL on error) |
| */ |
| static void xsigo_xds_complete ( struct ib_device *ibdev, |
| struct ib_mad_interface *mi __unused, |
| struct ib_mad_transaction *madx, |
| int rc, union ib_mad *mad, |
| struct ib_address_vector *av __unused ) { |
| struct xsigo_device *xdev = ib_madx_get_ownerdata ( madx ); |
| union xsigo_mad *xsmad = container_of ( mad, union xsigo_mad, mad ); |
| struct xsigo_managers_request *request = &xsmad->request; |
| struct ib_service_record *svc; |
| struct ib_address_vector dest; |
| union ib_guid *guid; |
| |
| /* Allow for reuse of transaction pointer */ |
| xdev->madx = NULL; |
| |
| /* Check for failures */ |
| if ( ( rc == 0 ) && ( mad->hdr.status != htons ( IB_MGMT_STATUS_OK ) ) ) |
| rc = -ENODEV; |
| if ( rc != 0 ) { |
| DBGC ( xdev, "XDEV %s directory lookup failed: %s\n", |
| xdev->name, strerror ( rc ) ); |
| goto out; |
| } |
| |
| /* Construct address vector */ |
| memset ( &dest, 0, sizeof ( dest ) ); |
| svc = &mad->sa.sa_data.service_record; |
| dest.lid = ntohs ( svc->data16[0] ); |
| dest.sl = ibdev->sm_sl; |
| dest.qpn = IB_QPN_GSI; |
| dest.qkey = IB_QKEY_GSI; |
| guid = ( ( union ib_guid * ) &svc->data64[0] ); |
| DBGC2 ( xdev, "XDEV %s found directory at LID %d GUID " IB_GUID_FMT |
| "\n", xdev->name, dest.lid, IB_GUID_ARGS ( guid ) ); |
| |
| /* Construct request (reusing MAD buffer) */ |
| memset ( request, 0, sizeof ( *request ) ); |
| request->mad_hdr.mgmt_class = XSIGO_MGMT_CLASS; |
| request->mad_hdr.class_version = XSIGO_MGMT_CLASS_VERSION; |
| request->mad_hdr.method = IB_MGMT_METHOD_GET; |
| request->mad_hdr.attr_id = htons ( XSIGO_ATTR_XCM_REQUEST ); |
| memcpy ( &request->server.guid, &ibdev->gid.s.guid, |
| sizeof ( request->server.guid ) ); |
| snprintf ( request->os_version, sizeof ( request->os_version ), |
| "%s %s", product_short_name, product_version ); |
| snprintf ( request->arch, sizeof ( request->arch ), _S2 ( ARCH ) ); |
| request->os_type = XSIGO_OS_TYPE_GENERIC; |
| request->resources = htons ( XSIGO_RESOURCES_PRESENT | |
| XSIGO_RESOURCE_XVE | |
| XSIGO_RESOURCE_NO_HA ); |
| |
| /* The handling of this request on the server side is a |
| * textbook example of how not to design a wire protocol. The |
| * server uses the _driver_ version number to determine which |
| * fields are present. |
| */ |
| request->driver_version = htonl ( 0x2a2a2a ); |
| |
| /* The build version field is ignored unless it happens to |
| * contain the substring "xg-". |
| */ |
| snprintf ( request->build, sizeof ( request->build ), |
| "not-xg-%08lx", build_id ); |
| |
| /* The server side user interface occasionally has no way to |
| * refer to an entry with an empty hostname. |
| */ |
| fetch_string_setting ( NULL, &hostname_setting, request->hostname, |
| sizeof ( request->hostname ) ); |
| if ( ! request->hostname[0] ) { |
| snprintf ( request->hostname, sizeof ( request->hostname ), |
| "%s-" IB_GUID_FMT, product_short_name, |
| IB_GUID_ARGS ( &ibdev->gid.s.guid ) ); |
| } |
| |
| /* Start configuration manager lookup */ |
| xdev->madx = ib_create_madx ( ibdev, ibdev->gsi, mad, &dest, |
| &xsigo_xcm_discovery.op ); |
| if ( ! xdev->madx ) { |
| DBGC ( xdev, "XDEV %s could not start manager lookup\n", |
| xdev->name ); |
| goto out; |
| } |
| ib_madx_set_ownerdata ( xdev->madx, xdev ); |
| |
| out: |
| /* Destroy the completed transaction */ |
| ib_destroy_madx ( ibdev, ibdev->gsi, madx ); |
| } |
| |
| /** Directory service lookup discovery stage */ |
| static struct xsigo_discovery xsigo_xds_discovery = { |
| .name = "directory", |
| .op = { |
| .complete = xsigo_xds_complete, |
| }, |
| }; |
| |
| /** |
| * Discover configuration managers |
| * |
| * @v timer Retry timer |
| * @v over Failure indicator |
| */ |
| static void xsigo_discover ( struct retry_timer *timer, int over __unused ) { |
| struct xsigo_device *xdev = |
| container_of ( timer, struct xsigo_device, discovery ); |
| struct ib_device *ibdev = xdev->ibdev; |
| struct xsigo_discovery *discovery; |
| |
| /* Restart timer */ |
| start_timer_fixed ( &xdev->discovery, XSIGO_DISCOVERY_FAILURE_DELAY ); |
| |
| /* Cancel any pending discovery transaction */ |
| if ( xdev->madx ) { |
| discovery = container_of ( xdev->madx->op, |
| struct xsigo_discovery, op ); |
| DBGC ( xdev, "XDEV %s timed out waiting for %s lookup\n", |
| xdev->name, discovery->name ); |
| ib_destroy_madx ( ibdev, ibdev->gsi, xdev->madx ); |
| xdev->madx = NULL; |
| } |
| |
| /* Start directory service lookup */ |
| xdev->madx = ib_create_service_madx ( ibdev, ibdev->gsi, |
| XDS_SERVICE_NAME, |
| &xsigo_xds_discovery.op ); |
| if ( ! xdev->madx ) { |
| DBGC ( xdev, "XDEV %s could not start directory lookup\n", |
| xdev->name ); |
| return; |
| } |
| ib_madx_set_ownerdata ( xdev->madx, xdev ); |
| } |
| |
| /**************************************************************************** |
| * |
| * Infiniband device driver |
| * |
| **************************************************************************** |
| */ |
| |
| /** |
| * Open link and start discovery |
| * |
| * @v opener Link opener |
| * @v over Failure indicator |
| */ |
| static void xsigo_ib_open ( struct retry_timer *opener, int over __unused ) { |
| struct xsigo_device *xdev = |
| container_of ( opener, struct xsigo_device, opener ); |
| struct ib_device *ibdev = xdev->ibdev; |
| int rc; |
| |
| /* Open Infiniband device */ |
| if ( ( rc = ib_open ( ibdev ) ) != 0 ) { |
| DBGC ( xdev, "XDEV %s could not open: %s\n", |
| xdev->name, strerror ( rc ) ); |
| /* Delay and try again */ |
| start_timer_fixed ( &xdev->opener, XSIGO_OPEN_RETRY_DELAY ); |
| return; |
| } |
| |
| /* If link is already up, then start discovery */ |
| if ( ib_link_ok ( ibdev ) ) |
| start_timer_nodelay ( &xdev->discovery ); |
| } |
| |
| /** |
| * Probe Xsigo device |
| * |
| * @v ibdev Infiniband device |
| * @ret rc Return status code |
| */ |
| static int xsigo_ib_probe ( struct ib_device *ibdev ) { |
| struct xsigo_device *xdev; |
| |
| /* Allocate and initialise structure */ |
| xdev = zalloc ( sizeof ( *xdev ) ); |
| if ( ! xdev ) |
| return -ENOMEM; |
| ref_init ( &xdev->refcnt, xsigo_free ); |
| xdev->ibdev = ibdev_get ( ibdev ); |
| xdev->name = ibdev->name; |
| timer_init ( &xdev->opener, xsigo_ib_open, &xdev->refcnt ); |
| timer_init ( &xdev->discovery, xsigo_discover, &xdev->refcnt ); |
| INIT_LIST_HEAD ( &xdev->managers ); |
| |
| /* Start timer to open Infiniband device. (We are currently |
| * within the Infiniband device probe callback list; opening |
| * the device here would have interesting side-effects.) |
| */ |
| start_timer_nodelay ( &xdev->opener ); |
| |
| /* Add to list of devices and transfer reference to list */ |
| list_add_tail ( &xdev->list, &xsigo_devices ); |
| DBGC ( xdev, "XDEV %s created for " IB_GUID_FMT "\n", |
| xdev->name, IB_GUID_ARGS ( &ibdev->gid.s.guid ) ); |
| return 0; |
| } |
| |
| /** |
| * Handle device or link status change |
| * |
| * @v ibdev Infiniband device |
| */ |
| static void xsigo_ib_notify ( struct ib_device *ibdev ) { |
| struct xsigo_device *xdev; |
| |
| /* Stop/restart discovery on any attached devices */ |
| list_for_each_entry ( xdev, &xsigo_devices, list ) { |
| |
| /* Skip non-attached devices */ |
| if ( xdev->ibdev != ibdev ) |
| continue; |
| |
| /* Stop any ongoing discovery */ |
| if ( xdev->madx ) { |
| ib_destroy_madx ( ibdev, ibdev->gsi, xdev->madx ); |
| xdev->madx = NULL; |
| } |
| stop_timer ( &xdev->discovery ); |
| |
| /* If link is up, then start discovery */ |
| if ( ib_link_ok ( ibdev ) ) |
| start_timer_nodelay ( &xdev->discovery ); |
| } |
| } |
| |
| /** |
| * Remove Xsigo device |
| * |
| * @v ibdev Infiniband device |
| */ |
| static void xsigo_ib_remove ( struct ib_device *ibdev ) { |
| struct xsigo_device *xdev; |
| struct xsigo_device *tmp; |
| |
| /* Remove any attached Xsigo devices */ |
| list_for_each_entry_safe ( xdev, tmp, &xsigo_devices, list ) { |
| |
| /* Skip non-attached devices */ |
| if ( xdev->ibdev != ibdev ) |
| continue; |
| |
| /* Stop any ongoing discovery */ |
| if ( xdev->madx ) { |
| ib_destroy_madx ( ibdev, ibdev->gsi, xdev->madx ); |
| xdev->madx = NULL; |
| } |
| stop_timer ( &xdev->discovery ); |
| |
| /* Destroy all configuration managers */ |
| xcm_list ( xdev, NULL, 0 ); |
| |
| /* Close Infiniband device, if applicable */ |
| if ( ! timer_running ( &xdev->opener ) ) |
| ib_close ( xdev->ibdev ); |
| |
| /* Stop link opener */ |
| stop_timer ( &xdev->opener ); |
| |
| /* Remove from list of devices and drop list's reference */ |
| DBGC ( xdev, "XDEV %s destroyed\n", xdev->name ); |
| list_del ( &xdev->list ); |
| ref_put ( &xdev->refcnt ); |
| } |
| } |
| |
| /** Xsigo Infiniband driver */ |
| struct ib_driver xsigo_ib_driver __ib_driver = { |
| .name = "Xsigo", |
| .probe = xsigo_ib_probe, |
| .notify = xsigo_ib_notify, |
| .remove = xsigo_ib_remove, |
| }; |
| |
| /**************************************************************************** |
| * |
| * Network device driver |
| * |
| **************************************************************************** |
| */ |
| |
| /** |
| * Handle device or link status change |
| * |
| * @v netdev Network device |
| */ |
| static void xsigo_net_notify ( struct net_device *netdev ) { |
| struct xsigo_device *xdev; |
| struct ib_device *ibdev; |
| struct xsigo_manager *xcm; |
| struct xsigo_nic *xve; |
| struct eoib_device *eoib; |
| |
| /* Send current operational state to XCM, if applicable */ |
| list_for_each_entry ( xdev, &xsigo_devices, list ) { |
| ibdev = xdev->ibdev; |
| list_for_each_entry ( xcm, &xdev->managers, list ) { |
| list_for_each_entry ( xve, &xcm->nics, list ) { |
| eoib = eoib_find ( ibdev, xve->mac ); |
| if ( ! eoib ) |
| continue; |
| if ( eoib->netdev != netdev ) |
| continue; |
| xsmp_tx_xve_oper ( xcm, xve, eoib ); |
| } |
| } |
| } |
| } |
| |
| /** Xsigo network driver */ |
| struct net_driver xsigo_net_driver __net_driver = { |
| .name = "Xsigo", |
| .notify = xsigo_net_notify, |
| }; |