[infiniband] Allow external QPN to differ from real QPN

Most IB hardware seems not to allow allocation of the genuine QPNs 0
and 1, so allow for the externally-visible QPN (as constructed and
parsed by ib_packet, where used) to differ from the real
hardware-allocated QPN.

diff --git a/src/include/gpxe/infiniband.h b/src/include/gpxe/infiniband.h
index 9c1f57a..dacb13e 100644 (file)
--- a/src/include/gpxe/infiniband.h
+++ b/src/include/gpxe/infiniband.h
@@ -96,8 +96,15 @@ struct ib_queue_pair {
        struct ib_device *ibdev;
        /** List of queue pairs on this Infiniband device */
        struct list_head list;
-       /** Queue Pair Number */
+       /** Queue pair number */
        unsigned long qpn;
+       /** Externally-visible queue pair number
+        *
+        * This may differ from the real queue pair number (e.g. when
+        * the HCA cannot use the management QPNs 0 and 1 as hardware
+        * QPNs and needs to remap them).
+        */
+       unsigned long ext_qpn;
        /** Queue pair type */
        enum ib_queue_pair_type type;
        /** Queue key */

diff --git a/src/net/infiniband.c b/src/net/infiniband.c
index 1444a12..b15dcc6 100644 (file)
--- a/src/net/infiniband.c
+++ b/src/net/infiniband.c
@@ -196,7 +196,6 @@ struct ib_queue_pair * ib_create_qp ( struct ib_device *ibdev,
                       "%s\n", ibdev, strerror ( rc ) );
                goto err_dev_create_qp;
        }
-
        DBGC ( ibdev, "IBDEV %p created queue pair %p (%p) with QPN %#lx\n",
               ibdev, qp, ib_qp_get_drvdata ( qp ), qp->qpn );
        DBGC ( ibdev, "IBDEV %p QPN %#lx has %d send entries at [%p,%p)\n",
@@ -205,6 +204,24 @@ struct ib_queue_pair * ib_create_qp ( struct ib_device *ibdev,
        DBGC ( ibdev, "IBDEV %p QPN %#lx has %d receive entries at [%p,%p)\n",
               ibdev, qp->qpn, num_recv_wqes, qp->recv.iobufs,
               ( ( ( void * ) qp ) + total_size ) );
+
+       /* Calculate externally-visible QPN */
+       switch ( type ) {
+       case IB_QPT_SMA:
+               qp->ext_qpn = IB_QPN_SMA;
+               break;
+       case IB_QPT_GMA:
+               qp->ext_qpn = IB_QPN_GMA;
+               break;
+       default:
+               qp->ext_qpn = qp->qpn;
+               break;
+       }
+       if ( qp->ext_qpn != qp->qpn ) {
+               DBGC ( ibdev, "IBDEV %p QPN %#lx has external QPN %#lx\n",
+                      ibdev, qp->qpn, qp->ext_qpn );
+       }
+
        return qp;
 
        ibdev->op->destroy_qp ( ibdev, qp );
@@ -295,7 +312,7 @@ struct ib_queue_pair * ib_find_qp_qpn ( struct ib_device *ibdev,
        struct ib_queue_pair *qp;
 
        list_for_each_entry ( qp, &ibdev->qps, list ) {
-               if ( qp->qpn == qpn )
+               if ( ( qpn == qp->qpn ) || ( qpn == qp->ext_qpn ) )
                        return qp;
        }
        return NULL;

diff --git a/src/net/infiniband/ib_packet.c b/src/net/infiniband/ib_packet.c
index 409ef2f..689e00d 100644 (file)
--- a/src/net/infiniband/ib_packet.c
+++ b/src/net/infiniband/ib_packet.c
@@ -58,7 +58,7 @@ int ib_push ( struct ib_device *ibdev, struct io_buffer *iobuf,
        unsigned int lnh;
 
        DBGC2 ( ibdev, "IBDEV %p TX %04x:%08lx => %04x:%08lx (key %08lx)\n",
-               ibdev, ibdev->lid, qp->qpn, av->lid, av->qpn, av->qkey );
+               ibdev, ibdev->lid, qp->ext_qpn, av->lid, av->qpn, av->qkey );
 
        /* Calculate packet length */
        pad_len = ( (-payload_len) & 0x3 );
@@ -76,7 +76,7 @@ int ib_push ( struct ib_device *ibdev, struct io_buffer *iobuf,
        lrh_len = ( payload_len + iob_len ( iobuf ) - orig_iob_len );
 
        /* Construct LRH */
-       vl = ( ( av->qpn == IB_QPN_SMA ) ? IB_VL_SMP : IB_VL_DEFAULT );
+       vl = ( ( qp->ext_qpn == IB_QPN_SMA ) ? IB_VL_SMP : IB_VL_DEFAULT );
        lrh->vl__lver = ( vl << 4 );
        lnh = ( grh ? IB_LNH_GRH : IB_LNH_BTH );
        lrh->sl__lnh = ( ( av->sl << 4 ) | lnh );
@@ -104,7 +104,7 @@ int ib_push ( struct ib_device *ibdev, struct io_buffer *iobuf,
 
        /* Construct DETH */
        deth->qkey = htonl ( av->qkey );
-       deth->src_qp = htonl ( qp->qpn );
+       deth->src_qp = htonl ( qp->ext_qpn );
 
        DBGCP_HDA ( ibdev, 0, iobuf->data,
                    ( iob_len ( iobuf ) - orig_iob_len ) );
@@ -233,8 +233,8 @@ int ib_pull ( struct ib_device *ibdev, struct io_buffer *iobuf,
        }
 
        DBGC2 ( ibdev, "IBDEV %p RX %04x:%08lx <= %04x:%08lx (key %08x)\n",
-               ibdev, lid,
-               ( IB_LID_MULTICAST( lid ) ? ( qp ? (*qp)->qpn : -1UL ) : qpn ),
+               ibdev, lid, ( IB_LID_MULTICAST( lid ) ?
+                             ( qp ? (*qp)->ext_qpn : -1UL ) : qpn ),
                av->lid, av->qpn, ntohl ( deth->qkey ) );
        DBGCP_HDA ( ibdev, 0,
                    ( iobuf->data - ( orig_iob_len - iob_len ( iobuf ) ) ),