#include <gpxe/iobuf.h>
#include <gpxe/netdevice.h>
#include <gpxe/infiniband.h>
-#include <gpxe/ipoib.h>
#include "arbel.h"
/**
}
static inline int
-arbel_cmd_mad_ifc ( struct arbel *arbel, union arbelprm_mad *mad ) {
+arbel_cmd_mad_ifc ( struct arbel *arbel, unsigned int port,
+ union arbelprm_mad *mad ) {
return arbel_cmd ( arbel,
ARBEL_HCR_INOUT_CMD ( ARBEL_HCR_MAD_IFC,
1, sizeof ( *mad ),
1, sizeof ( *mad ) ),
- 0x03, mad, PXE_IB_PORT, mad );
+ 0x03, mad, port, mad );
}
static inline int
MLX_FILL_1 ( &qpctx, 5,
qpc_eec_data.usr_page, arbel->limits.reserved_uars );
MLX_FILL_1 ( &qpctx, 10, qpc_eec_data.primary_address_path.port_number,
- PXE_IB_PORT );
+ ibdev->port );
MLX_FILL_1 ( &qpctx, 27, qpc_eec_data.pd, ARBEL_GLOBAL_PD );
MLX_FILL_1 ( &qpctx, 29, qpc_eec_data.wqe_lkey, arbel->reserved_lkey );
MLX_FILL_1 ( &qpctx, 30, qpc_eec_data.ssc, 1 );
memset ( &wqe->ud, 0, sizeof ( wqe->ud ) );
MLX_FILL_2 ( &wqe->ud, 0,
ud_address_vector.pd, ARBEL_GLOBAL_PD,
- ud_address_vector.port_number, PXE_IB_PORT );
+ ud_address_vector.port_number, ibdev->port );
MLX_FILL_2 ( &wqe->ud, 1,
ud_address_vector.rlid, av->dlid,
ud_address_vector.g, av->gid_present );
}
}
+/***************************************************************************
+ *
+ * Infiniband link-layer operations
+ *
+ ***************************************************************************
+ */
+
+/**
+ * Initialise Infiniband link
+ *
+ * @v ibdev Infiniband device
+ * @ret rc Return status code
+ */
+static int arbel_open ( struct ib_device *ibdev ) {
+ struct arbel *arbel = ibdev->dev_priv;
+ struct arbelprm_init_ib init_ib;
+ int rc;
+
+ memset ( &init_ib, 0, sizeof ( init_ib ) );
+ MLX_FILL_3 ( &init_ib, 0,
+ mtu_cap, ARBEL_MTU_2048,
+ port_width_cap, 3,
+ vl_cap, 1 );
+ MLX_FILL_1 ( &init_ib, 1, max_gid, 1 );
+ MLX_FILL_1 ( &init_ib, 2, max_pkey, 64 );
+ if ( ( rc = arbel_cmd_init_ib ( arbel, ibdev->port,
+ &init_ib ) ) != 0 ) {
+ DBGC ( arbel, "Arbel %p could not intialise IB: %s\n",
+ arbel, strerror ( rc ) );
+ return rc;
+ }
+
+ return 0;
+}
+
+/**
+ * Close Infiniband link
+ *
+ * @v ibdev Infiniband device
+ */
+static void arbel_close ( struct ib_device *ibdev ) {
+ struct arbel *arbel = ibdev->dev_priv;
+ int rc;
+
+ if ( ( rc = arbel_cmd_close_ib ( arbel, ibdev->port ) ) != 0 ) {
+ DBGC ( arbel, "Arbel %p could not close IB: %s\n",
+ arbel, strerror ( rc ) );
+ /* Nothing we can do about this */
+ }
+}
+
/***************************************************************************
*
* Multicast group operations
}
}
-/** Arbel Infiniband operations */
-static struct ib_device_operations arbel_ib_operations = {
- .create_cq = arbel_create_cq,
- .destroy_cq = arbel_destroy_cq,
- .create_qp = arbel_create_qp,
- .destroy_qp = arbel_destroy_qp,
- .post_send = arbel_post_send,
- .post_recv = arbel_post_recv,
- .poll_cq = arbel_poll_cq,
- .mcast_attach = arbel_mcast_attach,
- .mcast_detach = arbel_mcast_detach,
-};
-
/***************************************************************************
*
- * MAD IFC operations
+ * MAD operations
*
***************************************************************************
*/
-static int arbel_mad_ifc ( struct arbel *arbel,
- union arbelprm_mad *mad ) {
- struct ib_mad_hdr *hdr = &mad->mad.mad_hdr;
- int rc;
-
- hdr->base_version = IB_MGMT_BASE_VERSION;
- if ( ( rc = arbel_cmd_mad_ifc ( arbel, mad ) ) != 0 ) {
- DBGC ( arbel, "Arbel %p could not issue MAD IFC: %s\n",
- arbel, strerror ( rc ) );
- return rc;
- }
- if ( hdr->status != 0 ) {
- DBGC ( arbel, "Arbel %p MAD IFC status %04x\n",
- arbel, ntohs ( hdr->status ) );
- return -EIO;
- }
- return 0;
-}
-
-static int arbel_get_port_info ( struct arbel *arbel,
- struct ib_mad_port_info *port_info ) {
- union arbelprm_mad mad;
- struct ib_mad_hdr *hdr = &mad.mad.mad_hdr;
- int rc;
-
- memset ( &mad, 0, sizeof ( mad ) );
- hdr->mgmt_class = IB_MGMT_CLASS_SUBN_LID_ROUTED;
- hdr->class_version = 1;
- hdr->method = IB_MGMT_METHOD_GET;
- hdr->attr_id = htons ( IB_SMP_ATTR_PORT_INFO );
- hdr->attr_mod = htonl ( PXE_IB_PORT );
- if ( ( rc = arbel_mad_ifc ( arbel, &mad ) ) != 0 ) {
- DBGC ( arbel, "Arbel %p could not get port info: %s\n",
- arbel, strerror ( rc ) );
- return rc;
- }
- memcpy ( port_info, &mad.mad.port_info, sizeof ( *port_info ) );
- return 0;
-}
-
-static int arbel_get_guid_info ( struct arbel *arbel,
- struct ib_mad_guid_info *guid_info ) {
- union arbelprm_mad mad;
- struct ib_mad_hdr *hdr = &mad.mad.mad_hdr;
- int rc;
-
- memset ( &mad, 0, sizeof ( mad ) );
- hdr->mgmt_class = IB_MGMT_CLASS_SUBN_LID_ROUTED;
- hdr->class_version = 1;
- hdr->method = IB_MGMT_METHOD_GET;
- hdr->attr_id = htons ( IB_SMP_ATTR_GUID_INFO );
- if ( ( rc = arbel_mad_ifc ( arbel, &mad ) ) != 0 ) {
- DBGC ( arbel, "Arbel %p could not get GUID info: %s\n",
- arbel, strerror ( rc ) );
- return rc;
- }
- memcpy ( guid_info, &mad.mad.guid_info, sizeof ( *guid_info ) );
- return 0;
-}
-
-static int arbel_get_pkey_table ( struct arbel *arbel,
- struct ib_mad_pkey_table *pkey_table ) {
- union arbelprm_mad mad;
- struct ib_mad_hdr *hdr = &mad.mad.mad_hdr;
- int rc;
-
- memset ( &mad, 0, sizeof ( mad ) );
- hdr->mgmt_class = IB_MGMT_CLASS_SUBN_LID_ROUTED;
- hdr->class_version = 1;
- hdr->method = IB_MGMT_METHOD_GET;
- hdr->attr_id = htons ( IB_SMP_ATTR_PKEY_TABLE );
- if ( ( rc = arbel_mad_ifc ( arbel, &mad ) ) != 0 ) {
- DBGC ( arbel, "Arbel %p could not get pkey table: %s\n",
- arbel, strerror ( rc ) );
- return rc;
- }
- memcpy ( pkey_table, &mad.mad.pkey_table, sizeof ( *pkey_table ) );
- return 0;
-}
-
-static int arbel_get_port_gid ( struct arbel *arbel,
- struct ib_gid *port_gid ) {
- union {
- /* This union exists just to save stack space */
- struct ib_mad_port_info port_info;
- struct ib_mad_guid_info guid_info;
- } u;
- int rc;
-
- /* Port info gives us the first half of the port GID */
- if ( ( rc = arbel_get_port_info ( arbel, &u.port_info ) ) != 0 )
- return rc;
- memcpy ( &port_gid->u.bytes[0], u.port_info.gid_prefix, 8 );
-
- /* GUID info gives us the second half of the port GID */
- if ( ( rc = arbel_get_guid_info ( arbel, &u.guid_info ) ) != 0 )
- return rc;
- memcpy ( &port_gid->u.bytes[8], u.guid_info.gid_local, 8 );
-
- return 0;
-}
-
-static int arbel_get_sm_lid ( struct arbel *arbel,
- unsigned long *sm_lid ) {
- struct ib_mad_port_info port_info;
- int rc;
-
- if ( ( rc = arbel_get_port_info ( arbel, &port_info ) ) != 0 )
- return rc;
- *sm_lid = ntohs ( port_info.mastersm_lid );
- return 0;
-}
-
-static int arbel_get_pkey ( struct arbel *arbel, unsigned int *pkey ) {
- struct ib_mad_pkey_table pkey_table;
- int rc;
-
- if ( ( rc = arbel_get_pkey_table ( arbel, &pkey_table ) ) != 0 )
- return rc;
- *pkey = ntohs ( pkey_table.pkey[0][0] );
- return 0;
-}
-
-/**
- * Wait for link up
- *
- * @v arbel Arbel device
- * @ret rc Return status code
- *
- * This function shouldn't really exist. Unfortunately, IB links take
- * a long time to come up, and we can't get various key parameters
- * e.g. our own IPoIB MAC address without information from the subnet
- * manager). We should eventually make link-up an asynchronous event.
- */
-static int arbel_wait_for_link ( struct arbel *arbel ) {
- struct ib_mad_port_info port_info;
- unsigned int retries;
- int rc;
-
- printf ( "Waiting for Infiniband link-up..." );
- for ( retries = 20 ; retries ; retries-- ) {
- if ( ( rc = arbel_get_port_info ( arbel, &port_info ) ) != 0 )
- continue;
- if ( ( ( port_info.port_state__link_speed_supported ) & 0xf )
- == 4 ) {
- printf ( "ok\n" );
- return 0;
- }
- printf ( "." );
- sleep ( 1 );
- }
- printf ( "failed\n" );
- return -ENODEV;
-};
-
/**
- * Get MAD parameters
+ * Issue management datagram
*
- * @v arbel Arbel device
+ * @v ibdev Infiniband device
+ * @v mad Management datagram
+ * @v len Length of management datagram
* @ret rc Return status code
*/
-static int arbel_get_mad_params ( struct ib_device *ibdev ) {
+static int arbel_mad ( struct ib_device *ibdev, struct ib_mad_hdr *mad,
+ size_t len ) {
struct arbel *arbel = ibdev->dev_priv;
+ union arbelprm_mad mad_ifc;
int rc;
- /* Get subnet manager LID */
- if ( ( rc = arbel_get_sm_lid ( arbel, &ibdev->sm_lid ) ) != 0 ) {
- DBGC ( arbel, "Arbel %p could not determine subnet manager "
- "LID: %s\n", arbel, strerror ( rc ) );
- return rc;
- }
+ /* Copy in request packet */
+ memset ( &mad_ifc, 0, sizeof ( mad_ifc ) );
+ assert ( len <= sizeof ( mad_ifc.mad ) );
+ memcpy ( &mad_ifc.mad, mad, len );
- /* Get port GID */
- if ( ( rc = arbel_get_port_gid ( arbel, &ibdev->port_gid ) ) != 0 ) {
- DBGC ( arbel, "Arbel %p could not determine port GID: %s\n",
+ /* Issue MAD */
+ if ( ( rc = arbel_cmd_mad_ifc ( arbel, ibdev->port,
+ &mad_ifc ) ) != 0 ) {
+ DBGC ( arbel, "Arbel %p could not issue MAD IFC: %s\n",
arbel, strerror ( rc ) );
return rc;
}
- /* Get partition key */
- if ( ( rc = arbel_get_pkey ( arbel, &ibdev->pkey ) ) != 0 ) {
- DBGC ( arbel, "Arbel %p could not determine partition key: "
- "%s\n", arbel, strerror ( rc ) );
- return rc;
- }
+ /* Copy out reply packet */
+ memcpy ( mad, &mad_ifc.mad, len );
+ if ( mad->status != 0 ) {
+ DBGC ( arbel, "Arbel %p MAD IFC status %04x\n",
+ arbel, ntohs ( mad->status ) );
+ return -EIO;
+ }
return 0;
}
+/** Arbel Infiniband operations */
+static struct ib_device_operations arbel_ib_operations = {
+ .create_cq = arbel_create_cq,
+ .destroy_cq = arbel_destroy_cq,
+ .create_qp = arbel_create_qp,
+ .destroy_qp = arbel_destroy_qp,
+ .post_send = arbel_post_send,
+ .post_recv = arbel_post_recv,
+ .poll_cq = arbel_poll_cq,
+ .open = arbel_open,
+ .close = arbel_close,
+ .mcast_attach = arbel_mcast_attach,
+ .mcast_detach = arbel_mcast_detach,
+ .mad = arbel_mad,
+};
+
/***************************************************************************
*
* Firmware control
arbel->icm = UNULL;
}
-/***************************************************************************
- *
- * Infiniband link-layer operations
- *
- ***************************************************************************
- */
-
-/**
- * Initialise Infiniband link
- *
- * @v arbel Arbel device
- * @ret rc Return status code
- */
-static int arbel_init_ib ( struct arbel *arbel ) {
- struct arbelprm_init_ib init_ib;
- int rc;
-
- memset ( &init_ib, 0, sizeof ( init_ib ) );
- MLX_FILL_3 ( &init_ib, 0,
- mtu_cap, ARBEL_MTU_2048,
- port_width_cap, 3,
- vl_cap, 1 );
- MLX_FILL_1 ( &init_ib, 1, max_gid, 1 );
- MLX_FILL_1 ( &init_ib, 2, max_pkey, 64 );
- if ( ( rc = arbel_cmd_init_ib ( arbel, PXE_IB_PORT,
- &init_ib ) ) != 0 ) {
- DBGC ( arbel, "Arbel %p could not intialise IB: %s\n",
- arbel, strerror ( rc ) );
- return rc;
- }
-
- return 0;
-}
-
-/**
- * Close Infiniband link
- *
- * @v arbel Arbel device
- */
-static void arbel_close_ib ( struct arbel *arbel ) {
- int rc;
-
- if ( ( rc = arbel_cmd_close_ib ( arbel, PXE_IB_PORT ) ) != 0 ) {
- DBGC ( arbel, "Arbel %p could not close IB: %s\n",
- arbel, strerror ( rc ) );
- /* Nothing we can do about this */
- }
-}
-
/***************************************************************************
*
* PCI interface
ibdev->op = &arbel_ib_operations;
pci_set_drvdata ( pci, ibdev );
ibdev->dev = &pci->dev;
+ ibdev->port = PXE_IB_PORT;
arbel = ibdev->dev_priv;
memset ( arbel, 0, sizeof ( *arbel ) );
if ( ( rc = arbel_setup_mpt ( arbel ) ) != 0 )
goto err_setup_mpt;
- /* Bring up IB layer */
- if ( ( rc = arbel_init_ib ( arbel ) ) != 0 )
- goto err_init_ib;
-
- /* Wait for link */
- if ( ( rc = arbel_wait_for_link ( arbel ) ) != 0 )
- goto err_wait_for_link;
-
- /* Get MAD parameters */
- if ( ( rc = arbel_get_mad_params ( ibdev ) ) != 0 )
- goto err_get_mad_params;
-
- DBGC ( arbel, "Arbel %p port GID is %08lx:%08lx:%08lx:%08lx\n", arbel,
- htonl ( ibdev->port_gid.u.dwords[0] ),
- htonl ( ibdev->port_gid.u.dwords[1] ),
- htonl ( ibdev->port_gid.u.dwords[2] ),
- htonl ( ibdev->port_gid.u.dwords[3] ) );
-
- /* Add IPoIB device */
- if ( ( rc = ipoib_probe ( ibdev ) ) != 0 ) {
- DBGC ( arbel, "Arbel %p could not add IPoIB device: %s\n",
+ /* Register Infiniband device */
+ if ( ( rc = register_ibdev ( ibdev ) ) != 0 ) {
+ DBGC ( arbel, "Arbel %p could not register IB device: %s\n",
arbel, strerror ( rc ) );
- goto err_ipoib_probe;
+ goto err_register_ibdev;
}
return 0;
- err_ipoib_probe:
- err_get_mad_params:
- err_wait_for_link:
- arbel_close_ib ( arbel );
- err_init_ib:
+ err_register_ibdev:
err_setup_mpt:
arbel_cmd_close_hca ( arbel );
err_init_hca:
struct ib_device *ibdev = pci_get_drvdata ( pci );
struct arbel *arbel = ibdev->dev_priv;
- ipoib_remove ( ibdev );
- arbel_close_ib ( arbel );
+ unregister_ibdev ( ibdev );
arbel_cmd_close_hca ( arbel );
arbel_free_icm ( arbel );
arbel_stop_firmware ( arbel );
#include <gpxe/iobuf.h>
#include <gpxe/netdevice.h>
#include <gpxe/infiniband.h>
-#include <gpxe/ipoib.h>
#include "hermon.h"
/**
}
static inline int
-hermon_cmd_mad_ifc ( struct hermon *hermon, union hermonprm_mad *mad ) {
+hermon_cmd_mad_ifc ( struct hermon *hermon, unsigned int port,
+ union hermonprm_mad *mad ) {
return hermon_cmd ( hermon,
HERMON_HCR_INOUT_CMD ( HERMON_HCR_MAD_IFC,
1, sizeof ( *mad ),
1, sizeof ( *mad ) ),
- 0x03, mad, PXE_IB_PORT, mad );
+ 0x03, mad, port, mad );
}
static inline int
goto err_alloc_mtt;
}
- /* Hand queue over to hardware */
+ /* Transition queue to INIT state */
memset ( &qpctx, 0, sizeof ( qpctx ) );
MLX_FILL_2 ( &qpctx, 2,
qpc_eec_data.pm_state, 0x03 /* Always 0x03 for UD */,
goto err_rst2init_qp;
}
+ /* Transition queue to RTR state */
memset ( &qpctx, 0, sizeof ( qpctx ) );
MLX_FILL_2 ( &qpctx, 4,
qpc_eec_data.mtu, HERMON_MTU_2048,
MLX_FILL_1 ( &qpctx, 16,
qpc_eec_data.primary_address_path.sched_queue,
( 0x83 /* default policy */ |
- ( ( PXE_IB_PORT - 1 ) << 6 ) ) );
+ ( ( ibdev->port - 1 ) << 6 ) ) );
if ( ( rc = hermon_cmd_init2rtr_qp ( hermon, qp->qpn,
&qpctx ) ) != 0 ) {
DBGC ( hermon, "Hermon %p INIT2RTR_QP failed: %s\n",
MLX_FILL_1 ( &wqe->ctrl, 2, c, 0x03 /* generate completion */ );
MLX_FILL_2 ( &wqe->ud, 0,
ud_address_vector.pd, HERMON_GLOBAL_PD,
- ud_address_vector.port_number, PXE_IB_PORT );
+ ud_address_vector.port_number, ibdev->port );
MLX_FILL_2 ( &wqe->ud, 1,
ud_address_vector.rlid, av->dlid,
ud_address_vector.g, av->gid_present );
}
}
+/***************************************************************************
+ *
+ * Infiniband link-layer operations
+ *
+ ***************************************************************************
+ */
+
+/**
+ * Initialise Infiniband link
+ *
+ * @v ibdev Infiniband device
+ * @ret rc Return status code
+ */
+static int hermon_open ( struct ib_device *ibdev ) {
+ struct hermon *hermon = ibdev->dev_priv;
+ struct hermonprm_init_port init_port;
+ int rc;
+
+ memset ( &init_port, 0, sizeof ( init_port ) );
+ MLX_FILL_2 ( &init_port, 0,
+ port_width_cap, 3,
+ vl_cap, 1 );
+ MLX_FILL_2 ( &init_port, 1,
+ mtu, HERMON_MTU_2048,
+ max_gid, 1 );
+ MLX_FILL_1 ( &init_port, 2, max_pkey, 64 );
+ if ( ( rc = hermon_cmd_init_port ( hermon, ibdev->port,
+ &init_port ) ) != 0 ) {
+ DBGC ( hermon, "Hermon %p could not intialise port: %s\n",
+ hermon, strerror ( rc ) );
+ return rc;
+ }
+
+ return 0;
+}
+
+/**
+ * Close Infiniband link
+ *
+ * @v ibdev Infiniband device
+ */
+static void hermon_close ( struct ib_device *ibdev ) {
+ struct hermon *hermon = ibdev->dev_priv;
+ int rc;
+
+ if ( ( rc = hermon_cmd_close_port ( hermon, ibdev->port ) ) != 0 ) {
+ DBGC ( hermon, "Hermon %p could not close port: %s\n",
+ hermon, strerror ( rc ) );
+ /* Nothing we can do about this */
+ }
+}
+
/***************************************************************************
*
* Multicast group operations
}
}
-/** Hermon Infiniband operations */
-static struct ib_device_operations hermon_ib_operations = {
- .create_cq = hermon_create_cq,
- .destroy_cq = hermon_destroy_cq,
- .create_qp = hermon_create_qp,
- .destroy_qp = hermon_destroy_qp,
- .post_send = hermon_post_send,
- .post_recv = hermon_post_recv,
- .poll_cq = hermon_poll_cq,
- .mcast_attach = hermon_mcast_attach,
- .mcast_detach = hermon_mcast_detach,
-};
-
/***************************************************************************
*
- * MAD IFC operations
+ * MAD operations
*
***************************************************************************
*/
-static int hermon_mad_ifc ( struct hermon *hermon,
- union hermonprm_mad *mad ) {
- struct ib_mad_hdr *hdr = &mad->mad.mad_hdr;
- int rc;
-
- hdr->base_version = IB_MGMT_BASE_VERSION;
- if ( ( rc = hermon_cmd_mad_ifc ( hermon, mad ) ) != 0 ) {
- DBGC ( hermon, "Hermon %p could not issue MAD IFC: %s\n",
- hermon, strerror ( rc ) );
- return rc;
- }
- if ( hdr->status != 0 ) {
- DBGC ( hermon, "Hermon %p MAD IFC status %04x\n",
- hermon, ntohs ( hdr->status ) );
- return -EIO;
- }
- return 0;
-}
-
-static int hermon_get_port_info ( struct hermon *hermon,
- struct ib_mad_port_info *port_info ) {
- union hermonprm_mad mad;
- struct ib_mad_hdr *hdr = &mad.mad.mad_hdr;
- int rc;
-
- memset ( &mad, 0, sizeof ( mad ) );
- hdr->mgmt_class = IB_MGMT_CLASS_SUBN_LID_ROUTED;
- hdr->class_version = 1;
- hdr->method = IB_MGMT_METHOD_GET;
- hdr->attr_id = htons ( IB_SMP_ATTR_PORT_INFO );
- hdr->attr_mod = htonl ( PXE_IB_PORT );
- if ( ( rc = hermon_mad_ifc ( hermon, &mad ) ) != 0 ) {
- DBGC ( hermon, "Hermon %p could not get port info: %s\n",
- hermon, strerror ( rc ) );
- return rc;
- }
- memcpy ( port_info, &mad.mad.port_info, sizeof ( *port_info ) );
- return 0;
-}
-
-static int hermon_get_guid_info ( struct hermon *hermon,
- struct ib_mad_guid_info *guid_info ) {
- union hermonprm_mad mad;
- struct ib_mad_hdr *hdr = &mad.mad.mad_hdr;
- int rc;
-
- memset ( &mad, 0, sizeof ( mad ) );
- hdr->mgmt_class = IB_MGMT_CLASS_SUBN_LID_ROUTED;
- hdr->class_version = 1;
- hdr->method = IB_MGMT_METHOD_GET;
- hdr->attr_id = htons ( IB_SMP_ATTR_GUID_INFO );
- if ( ( rc = hermon_mad_ifc ( hermon, &mad ) ) != 0 ) {
- DBGC ( hermon, "Hermon %p could not get GUID info: %s\n",
- hermon, strerror ( rc ) );
- return rc;
- }
- memcpy ( guid_info, &mad.mad.guid_info, sizeof ( *guid_info ) );
- return 0;
-}
-
-static int hermon_get_pkey_table ( struct hermon *hermon,
- struct ib_mad_pkey_table *pkey_table ) {
- union hermonprm_mad mad;
- struct ib_mad_hdr *hdr = &mad.mad.mad_hdr;
- int rc;
-
- memset ( &mad, 0, sizeof ( mad ) );
- hdr->mgmt_class = IB_MGMT_CLASS_SUBN_LID_ROUTED;
- hdr->class_version = 1;
- hdr->method = IB_MGMT_METHOD_GET;
- hdr->attr_id = htons ( IB_SMP_ATTR_PKEY_TABLE );
- if ( ( rc = hermon_mad_ifc ( hermon, &mad ) ) != 0 ) {
- DBGC ( hermon, "Hermon %p could not get pkey table: %s\n",
- hermon, strerror ( rc ) );
- return rc;
- }
- memcpy ( pkey_table, &mad.mad.pkey_table, sizeof ( *pkey_table ) );
- return 0;
-}
-
-static int hermon_get_port_gid ( struct hermon *hermon,
- struct ib_gid *port_gid ) {
- union {
- /* This union exists just to save stack space */
- struct ib_mad_port_info port_info;
- struct ib_mad_guid_info guid_info;
- } u;
- int rc;
-
- /* Port info gives us the first half of the port GID */
- if ( ( rc = hermon_get_port_info ( hermon, &u.port_info ) ) != 0 )
- return rc;
- memcpy ( &port_gid->u.bytes[0], u.port_info.gid_prefix, 8 );
-
- /* GUID info gives us the second half of the port GID */
- if ( ( rc = hermon_get_guid_info ( hermon, &u.guid_info ) ) != 0 )
- return rc;
- memcpy ( &port_gid->u.bytes[8], u.guid_info.gid_local, 8 );
-
- return 0;
-}
-
-static int hermon_get_sm_lid ( struct hermon *hermon,
- unsigned long *sm_lid ) {
- struct ib_mad_port_info port_info;
- int rc;
-
- if ( ( rc = hermon_get_port_info ( hermon, &port_info ) ) != 0 )
- return rc;
- *sm_lid = ntohs ( port_info.mastersm_lid );
- return 0;
-}
-
-static int hermon_get_pkey ( struct hermon *hermon, unsigned int *pkey ) {
- struct ib_mad_pkey_table pkey_table;
- int rc;
-
- if ( ( rc = hermon_get_pkey_table ( hermon, &pkey_table ) ) != 0 )
- return rc;
- *pkey = ntohs ( pkey_table.pkey[0][0] );
- return 0;
-}
-
/**
- * Wait for link up
+ * Issue management datagram
*
- * @v hermon Hermon device
- * @ret rc Return status code
- *
- * This function shouldn't really exist. Unfortunately, IB links take
- * a long time to come up, and we can't get various key parameters
- * e.g. our own IPoIB MAC address without information from the subnet
- * manager). We should eventually make link-up an asynchronous event.
- */
-static int hermon_wait_for_link ( struct hermon *hermon ) {
- struct ib_mad_port_info port_info;
- unsigned int retries;
- int rc;
-
- printf ( "Waiting for Infiniband link-up..." );
- for ( retries = 20 ; retries ; retries-- ) {
- if ( ( rc = hermon_get_port_info ( hermon,
- &port_info ) ) != 0 )
- continue;
- if ( ( ( port_info.port_state__link_speed_supported ) & 0xf )
- == 4 ) {
- printf ( "ok\n" );
- return 0;
- }
- printf ( "." );
- sleep ( 1 );
- }
- printf ( "failed\n" );
- return -ENODEV;
-};
-
-/**
- * Get MAD parameters
- *
- * @v hermon Hermon device
+ * @v ibdev Infiniband device
+ * @v mad Management datagram
+ * @v len Length of management datagram
* @ret rc Return status code
*/
-static int hermon_get_mad_params ( struct ib_device *ibdev ) {
+static int hermon_mad ( struct ib_device *ibdev, struct ib_mad_hdr *mad,
+ size_t len ) {
struct hermon *hermon = ibdev->dev_priv;
+ union hermonprm_mad mad_ifc;
int rc;
- /* Get subnet manager LID */
- if ( ( rc = hermon_get_sm_lid ( hermon, &ibdev->sm_lid ) ) != 0 ) {
- DBGC ( hermon, "Hermon %p could not determine subnet manager "
- "LID: %s\n", hermon, strerror ( rc ) );
- return rc;
- }
+ /* Copy in request packet */
+ memset ( &mad_ifc, 0, sizeof ( mad_ifc ) );
+ assert ( len <= sizeof ( mad_ifc.mad ) );
+ memcpy ( &mad_ifc.mad, mad, len );
- /* Get port GID */
- if ( ( rc = hermon_get_port_gid ( hermon, &ibdev->port_gid ) ) != 0 ) {
- DBGC ( hermon, "Hermon %p could not determine port GID: %s\n",
+ /* Issue MAD */
+ if ( ( rc = hermon_cmd_mad_ifc ( hermon, ibdev->port,
+ &mad_ifc ) ) != 0 ) {
+ DBGC ( hermon, "Hermon %p could not issue MAD IFC: %s\n",
hermon, strerror ( rc ) );
return rc;
}
- /* Get partition key */
- if ( ( rc = hermon_get_pkey ( hermon, &ibdev->pkey ) ) != 0 ) {
- DBGC ( hermon, "Hermon %p could not determine partition key: "
- "%s\n", hermon, strerror ( rc ) );
- return rc;
- }
+ /* Copy out reply packet */
+ memcpy ( mad, &mad_ifc.mad, len );
+ if ( mad->status != 0 ) {
+ DBGC ( hermon, "Hermon %p MAD IFC status %04x\n",
+ hermon, ntohs ( mad->status ) );
+ return -EIO;
+ }
return 0;
}
+/** Hermon Infiniband operations */
+static struct ib_device_operations hermon_ib_operations = {
+ .create_cq = hermon_create_cq,
+ .destroy_cq = hermon_destroy_cq,
+ .create_qp = hermon_create_qp,
+ .destroy_qp = hermon_destroy_qp,
+ .post_send = hermon_post_send,
+ .post_recv = hermon_post_recv,
+ .poll_cq = hermon_poll_cq,
+ .open = hermon_open,
+ .close = hermon_close,
+ .mcast_attach = hermon_mcast_attach,
+ .mcast_detach = hermon_mcast_detach,
+ .mad = hermon_mad,
+};
+
/***************************************************************************
*
* Firmware control
hermon->icm = UNULL;
}
-/***************************************************************************
- *
- * Infiniband link-layer operations
- *
- ***************************************************************************
- */
-
-/**
- * Initialise Infiniband link
- *
- * @v hermon Hermon device
- * @ret rc Return status code
- */
-static int hermon_init_port ( struct hermon *hermon ) {
- struct hermonprm_init_port init_port;
- int rc;
-
- memset ( &init_port, 0, sizeof ( init_port ) );
- MLX_FILL_2 ( &init_port, 0,
- port_width_cap, 3,
- vl_cap, 1 );
- MLX_FILL_2 ( &init_port, 1,
- mtu, HERMON_MTU_2048,
- max_gid, 1 );
- MLX_FILL_1 ( &init_port, 2, max_pkey, 64 );
- if ( ( rc = hermon_cmd_init_port ( hermon, PXE_IB_PORT,
- &init_port ) ) != 0 ) {
- DBGC ( hermon, "Hermon %p could not intialise port: %s\n",
- hermon, strerror ( rc ) );
- return rc;
- }
-
- return 0;
-}
-
-/**
- * Close Infiniband link
- *
- * @v hermon Hermon device
- */
-static void hermon_close_port ( struct hermon *hermon ) {
- int rc;
-
- if ( ( rc = hermon_cmd_close_port ( hermon, PXE_IB_PORT ) ) != 0 ) {
- DBGC ( hermon, "Hermon %p could not close port: %s\n",
- hermon, strerror ( rc ) );
- /* Nothing we can do about this */
- }
-}
-
/***************************************************************************
*
* PCI interface
ibdev->op = &hermon_ib_operations;
pci_set_drvdata ( pci, ibdev );
ibdev->dev = &pci->dev;
+ ibdev->port = PXE_IB_PORT;
hermon = ibdev->dev_priv;
memset ( hermon, 0, sizeof ( *hermon ) );
if ( ( rc = hermon_setup_mpt ( hermon ) ) != 0 )
goto err_setup_mpt;
- /* Bring up IB layer */
- if ( ( rc = hermon_init_port ( hermon ) ) != 0 )
- goto err_init_port;
-
- /* Wait for link */
- if ( ( rc = hermon_wait_for_link ( hermon ) ) != 0 )
- goto err_wait_for_link;
-
- /* Get MAD parameters */
- if ( ( rc = hermon_get_mad_params ( ibdev ) ) != 0 )
- goto err_get_mad_params;
-
- DBGC ( hermon, "Hermon %p port GID is %08lx:%08lx:%08lx:%08lx\n",
- hermon, htonl ( ibdev->port_gid.u.dwords[0] ),
- htonl ( ibdev->port_gid.u.dwords[1] ),
- htonl ( ibdev->port_gid.u.dwords[2] ),
- htonl ( ibdev->port_gid.u.dwords[3] ) );
-
- /* Add IPoIB device */
- if ( ( rc = ipoib_probe ( ibdev ) ) != 0 ) {
- DBGC ( hermon, "Hermon %p could not add IPoIB device: %s\n",
+ /* Register Infiniband device */
+ if ( ( rc = register_ibdev ( ibdev ) ) != 0 ) {
+ DBGC ( hermon, "Hermon %p could not register IB device: %s\n",
hermon, strerror ( rc ) );
- goto err_ipoib_probe;
+ goto err_register_ibdev;
}
return 0;
- err_ipoib_probe:
- err_get_mad_params:
- err_wait_for_link:
- hermon_close_port ( hermon );
- err_init_port:
+ err_register_ibdev:
err_setup_mpt:
hermon_cmd_close_hca ( hermon );
err_init_hca:
struct ib_device *ibdev = pci_get_drvdata ( pci );
struct hermon *hermon = ibdev->dev_priv;
- ipoib_remove ( ibdev );
- hermon_close_port ( hermon );
+ unregister_ibdev ( ibdev );
hermon_cmd_close_hca ( hermon );
hermon_free_icm ( hermon );
hermon_stop_firmware ( hermon );
struct ib_gid gid;
};
+struct ib_mad_hdr;
+
/**
* Infiniband device operations
*
struct ib_completion_queue *cq,
ib_completer_t complete_send,
ib_completer_t complete_recv );
+ /**
+ * Open port
+ *
+ * @v ibdev Infiniband device
+ * @ret rc Return status code
+ */
+ int ( * open ) ( struct ib_device *ibdev );
+ /**
+ * Close port
+ *
+ * @v ibdev Infiniband device
+ */
+ void ( * close ) ( struct ib_device *ibdev );
/** Attach to multicast group
*
* @v ibdev Infiniband device
void ( * mcast_detach ) ( struct ib_device *ibdev,
struct ib_queue_pair *qp,
struct ib_gid *gid );
+ /**
+ * Issue management datagram
+ *
+ * @v ibdev Infiniband device
+ * @v mad Management datagram
+ * @v len Length of management datagram
+ * @ret rc Return status code
+ */
+ int ( * mad ) ( struct ib_device *ibdev, struct ib_mad_hdr *mad,
+ size_t len );
};
/** An Infiniband device */
struct ib_device {
+ /** Underlying device */
+ struct device *dev;
+ /** Infiniband operations */
+ struct ib_device_operations *op;
+ /** Port number */
+ unsigned int port;
/** Port GID */
struct ib_gid port_gid;
/** Subnet manager LID */
unsigned long sm_lid;
/** Partition key */
unsigned int pkey;
- /** Underlying device */
- struct device *dev;
- /** Infiniband operations */
- struct ib_device_operations *op;
/** Device private data */
void *dev_priv;
/** Owner private data */
extern struct ib_work_queue * ib_find_wq ( struct ib_completion_queue *cq,
unsigned long qpn, int is_send );
extern struct ib_device * alloc_ibdev ( size_t priv_size );
+extern int register_ibdev ( struct ib_device *ibdev );
+extern void unregister_ibdev ( struct ib_device *ibdev );
extern void free_ibdev ( struct ib_device *ibdev );
/**
ibdev->op->poll_cq ( ibdev, cq, complete_send, complete_recv );
}
+/**
+ * Open port
+ *
+ * @v ibdev Infiniband device
+ * @ret rc Return status code
+ */
+static inline __attribute__ (( always_inline )) int
+ib_open ( struct ib_device *ibdev ) {
+ return ibdev->op->open ( ibdev );
+}
+
+/**
+ * Close port
+ *
+ * @v ibdev Infiniband device
+ */
+static inline __attribute__ (( always_inline )) void
+ib_close ( struct ib_device *ibdev ) {
+ ibdev->op->close ( ibdev );
+}
/**
* Attach to multicast group
ibdev->op->mcast_detach ( ibdev, qp, gid );
}
+/**
+ * Issue management datagram
+ *
+ * @v ibdev Infiniband device
+ * @v mad Management datagram
+ * @v len Length of management datagram
+ * @ret rc Return status code
+ */
+static inline __attribute__ (( always_inline )) int
+ib_mad ( struct ib_device *ibdev, struct ib_mad_hdr *mad, size_t len ) {
+ return ibdev->op->mad ( ibdev, mad, len );
+}
+
/**
* Set Infiniband owner-private data
*
* @v pci Infiniband device
* @v priv Private data
*/
-static inline void ib_set_ownerdata ( struct ib_device *ibdev,
- void *owner_priv ) {
+static inline __attribute__ (( always_inline )) void
+ib_set_ownerdata ( struct ib_device *ibdev, void *owner_priv ) {
ibdev->owner_priv = owner_priv;
}
* @v pci Infiniband device
* @ret priv Private data
*/
-static inline void * ib_get_ownerdata ( struct ib_device *ibdev ) {
+static inline __attribute__ (( always_inline )) void *
+ib_get_ownerdata ( struct ib_device *ibdev ) {
return ibdev->owner_priv;
}
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
+#include <unistd.h>
#include <byteswap.h>
#include <errno.h>
#include <assert.h>
#include <gpxe/if_arp.h>
#include <gpxe/netdevice.h>
#include <gpxe/iobuf.h>
+#include <gpxe/ipoib.h>
#include <gpxe/infiniband.h>
/** @file
return NULL;
}
+/***************************************************************************
+ *
+ * Management datagram operations
+ *
+ ***************************************************************************
+ */
+
+/**
+ * Get port information
+ *
+ * @v ibdev Infiniband device
+ * @v port_info Port information datagram to fill in
+ * @ret rc Return status code
+ */
+static int ib_get_port_info ( struct ib_device *ibdev,
+ struct ib_mad_port_info *port_info ) {
+ struct ib_mad_hdr *hdr = &port_info->mad_hdr;
+ int rc;
+
+ /* Construct MAD */
+ memset ( port_info, 0, sizeof ( *port_info ) );
+ hdr->base_version = IB_MGMT_BASE_VERSION;
+ hdr->mgmt_class = IB_MGMT_CLASS_SUBN_LID_ROUTED;
+ hdr->class_version = 1;
+ hdr->method = IB_MGMT_METHOD_GET;
+ hdr->attr_id = htons ( IB_SMP_ATTR_PORT_INFO );
+ hdr->attr_mod = htonl ( ibdev->port );
+
+ if ( ( rc = ib_mad ( ibdev, hdr, sizeof ( *port_info ) ) ) != 0 ) {
+ DBGC ( ibdev, "IBDEV %p could not get port info: %s\n",
+ ibdev, strerror ( rc ) );
+ return rc;
+ }
+ return 0;
+}
+
+/**
+ * Get GUID information
+ *
+ * @v ibdev Infiniband device
+ * @v guid_info GUID information datagram to fill in
+ * @ret rc Return status code
+ */
+static int ib_get_guid_info ( struct ib_device *ibdev,
+ struct ib_mad_guid_info *guid_info ) {
+ struct ib_mad_hdr *hdr = &guid_info->mad_hdr;
+ int rc;
+
+ /* Construct MAD */
+ memset ( guid_info, 0, sizeof ( *guid_info ) );
+ hdr->base_version = IB_MGMT_BASE_VERSION;
+ hdr->mgmt_class = IB_MGMT_CLASS_SUBN_LID_ROUTED;
+ hdr->class_version = 1;
+ hdr->method = IB_MGMT_METHOD_GET;
+ hdr->attr_id = htons ( IB_SMP_ATTR_GUID_INFO );
+
+ if ( ( rc = ib_mad ( ibdev, hdr, sizeof ( *guid_info ) ) ) != 0 ) {
+ DBGC ( ibdev, "IBDEV %p could not get GUID info: %s\n",
+ ibdev, strerror ( rc ) );
+ return rc;
+ }
+ return 0;
+}
+
+/**
+ * Get partition key table
+ *
+ * @v ibdev Infiniband device
+ * @v guid_info Partition key table datagram to fill in
+ * @ret rc Return status code
+ */
+static int ib_get_pkey_table ( struct ib_device *ibdev,
+ struct ib_mad_pkey_table *pkey_table ) {
+ struct ib_mad_hdr *hdr = &pkey_table->mad_hdr;
+ int rc;
+
+ /* Construct MAD */
+ memset ( pkey_table, 0, sizeof ( *pkey_table ) );
+ hdr->base_version = IB_MGMT_BASE_VERSION;
+ hdr->mgmt_class = IB_MGMT_CLASS_SUBN_LID_ROUTED;
+ hdr->class_version = 1;
+ hdr->method = IB_MGMT_METHOD_GET;
+ hdr->attr_id = htons ( IB_SMP_ATTR_PKEY_TABLE );
+
+ if ( ( rc = ib_mad ( ibdev, hdr, sizeof ( *pkey_table ) ) ) != 0 ) {
+ DBGC ( ibdev, "IBDEV %p could not get pkey table: %s\n",
+ ibdev, strerror ( rc ) );
+ return rc;
+ }
+ return 0;
+}
+
+/**
+ * Wait for link up
+ *
+ * @v ibdev Infiniband device
+ * @ret rc Return status code
+ *
+ * This function shouldn't really exist. Unfortunately, IB links take
+ * a long time to come up, and we can't get various key parameters
+ * e.g. our own IPoIB MAC address without information from the subnet
+ * manager). We should eventually make link-up an asynchronous event.
+ */
+static int ib_wait_for_link ( struct ib_device *ibdev ) {
+ struct ib_mad_port_info port_info;
+ unsigned int retries;
+ int rc;
+
+ printf ( "Waiting for Infiniband link-up..." );
+ for ( retries = 20 ; retries ; retries-- ) {
+ if ( ( rc = ib_get_port_info ( ibdev, &port_info ) ) != 0 )
+ continue;
+ if ( ( ( port_info.port_state__link_speed_supported ) & 0xf )
+ == 4 ) {
+ printf ( "ok\n" );
+ return 0;
+ }
+ printf ( "." );
+ sleep ( 1 );
+ }
+ printf ( "failed\n" );
+ return -ENODEV;
+};
+
+/**
+ * Get MAD parameters
+ *
+ * @v ibdev Infiniband device
+ * @ret rc Return status code
+ */
+static int ib_get_mad_params ( struct ib_device *ibdev ) {
+ union {
+ /* This union exists just to save stack space */
+ struct ib_mad_port_info port_info;
+ struct ib_mad_guid_info guid_info;
+ struct ib_mad_pkey_table pkey_table;
+ } u;
+ int rc;
+
+ /* Port info gives us the first half of the port GID and the SM LID */
+ if ( ( rc = ib_get_port_info ( ibdev, &u.port_info ) ) != 0 )
+ return rc;
+ memcpy ( &ibdev->port_gid.u.bytes[0], u.port_info.gid_prefix, 8 );
+ ibdev->sm_lid = ntohs ( u.port_info.mastersm_lid );
+
+ /* GUID info gives us the second half of the port GID */
+ if ( ( rc = ib_get_guid_info ( ibdev, &u.guid_info ) ) != 0 )
+ return rc;
+ memcpy ( &ibdev->port_gid.u.bytes[8], u.guid_info.gid_local, 8 );
+
+ /* Get partition key */
+ if ( ( rc = ib_get_pkey_table ( ibdev, &u.pkey_table ) ) != 0 )
+ return rc;
+ ibdev->pkey = ntohs ( u.pkey_table.pkey[0][0] );
+
+ DBGC ( ibdev, "IBDEV %p port GID is %08lx:%08lx:%08lx:%08lx\n",
+ ibdev, htonl ( ibdev->port_gid.u.dwords[0] ),
+ htonl ( ibdev->port_gid.u.dwords[1] ),
+ htonl ( ibdev->port_gid.u.dwords[2] ),
+ htonl ( ibdev->port_gid.u.dwords[3] ) );
+
+ return 0;
+}
+
+/***************************************************************************
+ *
+ * Infiniband device creation/destruction
+ *
+ ***************************************************************************
+ */
+
/**
* Allocate Infiniband device
*
return ibdev;
}
+/**
+ * Register Infiniband device
+ *
+ * @v ibdev Infiniband device
+ * @ret rc Return status code
+ */
+int register_ibdev ( struct ib_device *ibdev ) {
+ int rc;
+
+ /* Open link */
+ if ( ( rc = ib_open ( ibdev ) ) != 0 )
+ goto err_open;
+
+ /* Wait for link */
+ if ( ( rc = ib_wait_for_link ( ibdev ) ) != 0 )
+ goto err_wait_for_link;
+
+ /* Get MAD parameters */
+ if ( ( rc = ib_get_mad_params ( ibdev ) ) != 0 )
+ goto err_get_mad_params;
+
+ /* Add IPoIB device */
+ if ( ( rc = ipoib_probe ( ibdev ) ) != 0 ) {
+ DBGC ( ibdev, "IBDEV %p could not add IPoIB device: %s\n",
+ ibdev, strerror ( rc ) );
+ goto err_ipoib_probe;
+ }
+
+ return 0;
+
+ err_ipoib_probe:
+ err_get_mad_params:
+ err_wait_for_link:
+ ib_close ( ibdev );
+ err_open:
+ return rc;
+}
+
+/**
+ * Unregister Infiniband device
+ *
+ * @v ibdev Infiniband device
+ */
+void unregister_ibdev ( struct ib_device *ibdev ) {
+ ipoib_remove ( ibdev );
+ ib_close ( ibdev );
+}
+
/**
* Free Infiniband device
*
void free_ibdev ( struct ib_device *ibdev ) {
free ( ibdev );
}
+
http://git.etherboot.org / people / balajirrao / gpxe.git / commitdiff