[infiniband] Poll completion queues automatically

Currently, all Infiniband users must create a process for polling
their completion queues (or rely on a regular hook such as
netdev_poll() in ipoib.c).

Move instead to a model whereby the Infiniband core maintains a single
process calling ib_poll_eq(), and polling the event queue triggers
polls of the applicable completion queues.  (At present, the
Infiniband core simply polls all of the device's completion queues.)
Polling a completion queue will now implicitly refill all attached
receive work queues; this is analogous to the way that netdev_poll()
implicitly refills the RX ring.

Infiniband users no longer need to create a process just to poll their
completion queues and refill their receive rings.

diff --git a/src/drivers/net/ipoib.c b/src/drivers/net/ipoib.c
index 03dfa53..a559d01 100644 (file)
--- a/src/drivers/net/ipoib.c
+++ b/src/drivers/net/ipoib.c
@@ -743,10 +743,7 @@ static void ipoib_poll ( struct net_device *netdev ) {
        struct ipoib_device *ipoib = netdev->priv;
        struct ib_device *ibdev = ipoib->ibdev;
 
-       ib_poll_cq ( ibdev, ipoib->meta.cq );
-       ib_poll_cq ( ibdev, ipoib->data.cq );
-       ib_qset_refill_recv ( ibdev, &ipoib->meta );
-       ib_qset_refill_recv ( ibdev, &ipoib->data );
+       ib_poll_eq ( ibdev );
 }
 
 /**
@@ -861,8 +858,8 @@ static int ipoib_open ( struct net_device *netdev ) {
        mac->qpn = htonl ( ipoib->data.qp->qpn );
 
        /* Fill receive rings */
-       ib_qset_refill_recv ( ibdev, &ipoib->meta );
-       ib_qset_refill_recv ( ibdev, &ipoib->data );
+       ib_refill_recv ( ibdev, ipoib->meta.qp );
+       ib_refill_recv ( ibdev, ipoib->data.qp );
 
        /* Join broadcast group */
        if ( ( rc = ipoib_join_broadcast_group ( ipoib ) ) != 0 ) {

diff --git a/src/include/gpxe/ib_qset.h b/src/include/gpxe/ib_qset.h
index d7a7617..a111697 100644 (file)
--- a/src/include/gpxe/ib_qset.h
+++ b/src/include/gpxe/ib_qset.h
@@ -18,8 +18,6 @@ struct ib_queue_set {
        struct ib_completion_queue *cq;
        /** Queue pair */
        struct ib_queue_pair *qp;
-       /** Receive work queue maximum fill level */
-       unsigned int recv_max_fill;
 };
 
 extern int ib_create_qset ( struct ib_device *ibdev,
@@ -27,8 +25,6 @@ extern int ib_create_qset ( struct ib_device *ibdev,
                            struct ib_completion_queue_operations *cq_op,
                            unsigned int num_send_wqes,
                            unsigned int num_recv_wqes, unsigned long qkey );
-extern void ib_qset_refill_recv ( struct ib_device *ibdev,
-                                 struct ib_queue_set *qset );
 extern void ib_destroy_qset ( struct ib_device *ibdev,
                              struct ib_queue_set *qset );
 

diff --git a/src/include/gpxe/ib_sma.h b/src/include/gpxe/ib_sma.h
index 6d98480..dadcdff 100644 (file)
--- a/src/include/gpxe/ib_sma.h
+++ b/src/include/gpxe/ib_sma.h
@@ -10,7 +10,6 @@
 FILE_LICENCE ( GPL2_OR_LATER );
 
 #include <gpxe/infiniband.h>
-#include <gpxe/process.h>
 
 /** Infiniband Subnet Management Agent operations */
 struct ib_sma_operations {
@@ -33,8 +32,6 @@ struct ib_sma {
        struct ib_completion_queue *cq;
        /** SMA queue pair */
        struct ib_queue_pair *qp;
-       /** Poll process */
-       struct process poll;
 };
 
 /** SMA number of send WQEs

diff --git a/src/include/gpxe/infiniband.h b/src/include/gpxe/infiniband.h
index 6cfceda..41c55ac 100644 (file)
--- a/src/include/gpxe/infiniband.h
+++ b/src/include/gpxe/infiniband.h
@@ -154,6 +154,10 @@ struct ib_completion_queue_operations {
 
 /** An Infiniband Completion Queue */
 struct ib_completion_queue {
+       /** Containing Infiniband device */
+       struct ib_device *ibdev;
+       /** List of completion queues on this Infiniband device */
+       struct list_head list;
        /** Completion queue number */
        unsigned long cqn;
        /** Number of completion queue entries */
@@ -310,6 +314,8 @@ struct ib_device {
        struct list_head list;
        /** Underlying device */
        struct device *dev;
+       /** List of completion queues */
+       struct list_head cqs;
        /** List of queue pairs */
        struct list_head qps;
        /** Infiniband operations */
@@ -350,6 +356,8 @@ ib_create_cq ( struct ib_device *ibdev, unsigned int num_cqes,
               struct ib_completion_queue_operations *op );
 extern void ib_destroy_cq ( struct ib_device *ibdev,
                            struct ib_completion_queue *cq );
+extern void ib_poll_cq ( struct ib_device *ibdev,
+                        struct ib_completion_queue *cq );
 extern struct ib_queue_pair *
 ib_create_qp ( struct ib_device *ibdev, unsigned int num_send_wqes,
               struct ib_completion_queue *send_cq, unsigned int num_recv_wqes,
@@ -376,6 +384,8 @@ extern void ib_complete_recv ( struct ib_device *ibdev,
                               struct ib_queue_pair *qp,
                               struct ib_address_vector *av,
                               struct io_buffer *iobuf, int rc );
+extern void ib_refill_recv ( struct ib_device *ibdev,
+                            struct ib_queue_pair *qp );
 extern int ib_open ( struct ib_device *ibdev );
 extern void ib_close ( struct ib_device *ibdev );
 extern int ib_mcast_attach ( struct ib_device *ibdev, struct ib_queue_pair *qp,
@@ -388,23 +398,13 @@ extern struct ib_device * alloc_ibdev ( size_t priv_size );
 extern int register_ibdev ( struct ib_device *ibdev );
 extern void unregister_ibdev ( struct ib_device *ibdev );
 extern void ib_link_state_changed ( struct ib_device *ibdev );
+extern void ib_poll_eq ( struct ib_device *ibdev );
 extern struct list_head ib_devices;
 
 /** Iterate over all network devices */
 #define for_each_ibdev( ibdev ) \
        list_for_each_entry ( (ibdev), &ib_devices, list )
 
-/**
- * Poll completion queue
- *
- * @v ibdev            Infiniband device
- * @v cq               Completion queue
- */
-static inline __always_inline void
-ib_poll_cq ( struct ib_device *ibdev, struct ib_completion_queue *cq ) {
-       ibdev->op->poll_cq ( ibdev, cq );
-}
-
 /**
  * Check link state
  *

diff --git a/src/net/infiniband.c b/src/net/infiniband.c
index 4784f40..48572e0 100644 (file)
--- a/src/net/infiniband.c
+++ b/src/net/infiniband.c
@@ -43,6 +43,13 @@ FILE_LICENCE ( GPL2_OR_LATER );
 /** List of Infiniband devices */
 struct list_head ib_devices = LIST_HEAD_INIT ( ib_devices );
 
+/***************************************************************************
+ *
+ * Completion queues
+ *
+ ***************************************************************************
+ */
+
 /**
  * Create completion queue
  *
@@ -63,6 +70,8 @@ ib_create_cq ( struct ib_device *ibdev, unsigned int num_cqes,
        cq = zalloc ( sizeof ( *cq ) );
        if ( ! cq )
                goto err_alloc_cq;
+       cq->ibdev = ibdev;
+       list_add ( &cq->list, &ibdev->cqs );
        cq->num_cqes = num_cqes;
        INIT_LIST_HEAD ( &cq->work_queues );
        cq->op = op;
@@ -81,6 +90,7 @@ ib_create_cq ( struct ib_device *ibdev, unsigned int num_cqes,
 
        ibdev->op->destroy_cq ( ibdev, cq );
  err_dev_create_cq:
+       list_del ( &cq->list );
        free ( cq );
  err_alloc_cq:
        return NULL;
@@ -98,9 +108,37 @@ void ib_destroy_cq ( struct ib_device *ibdev,
               ibdev, cq->cqn );
        assert ( list_empty ( &cq->work_queues ) );
        ibdev->op->destroy_cq ( ibdev, cq );
+       list_del ( &cq->list );
        free ( cq );
 }
 
+/**
+ * Poll completion queue
+ *
+ * @v ibdev            Infiniband device
+ * @v cq               Completion queue
+ */
+void ib_poll_cq ( struct ib_device *ibdev,
+                 struct ib_completion_queue *cq ) {
+       struct ib_work_queue *wq;
+
+       /* Poll completion queue */
+       ibdev->op->poll_cq ( ibdev, cq );
+
+       /* Refill receive work queues */
+       list_for_each_entry ( wq, &cq->work_queues, list ) {
+               if ( ! wq->is_send )
+                       ib_refill_recv ( ibdev, wq->qp );
+       }
+}
+
+/***************************************************************************
+ *
+ * Work queues
+ *
+ ***************************************************************************
+ */
+
 /**
  * Create queue pair
  *
@@ -400,6 +438,44 @@ void ib_complete_recv ( struct ib_device *ibdev, struct ib_queue_pair *qp,
        qp->recv.fill--;
 }
 
+/**
+ * Refill receive work queue
+ *
+ * @v ibdev            Infiniband device
+ * @v qp               Queue pair
+ */
+void ib_refill_recv ( struct ib_device *ibdev, struct ib_queue_pair *qp ) {
+       struct io_buffer *iobuf;
+       int rc;
+
+       /* Keep filling while unfilled entries remain */
+       while ( qp->recv.fill < qp->recv.num_wqes ) {
+
+               /* Allocate I/O buffer */
+               iobuf = alloc_iob ( IB_MAX_PAYLOAD_SIZE );
+               if ( ! iobuf ) {
+                       /* Non-fatal; we will refill on next attempt */
+                       return;
+               }
+
+               /* Post I/O buffer */
+               if ( ( rc = ib_post_recv ( ibdev, qp, iobuf ) ) != 0 ) {
+                       DBGC ( ibdev, "IBDEV %p could not refill: %s\n",
+                              ibdev, strerror ( rc ) );
+                       free_iob ( iobuf );
+                       /* Give up */
+                       return;
+               }
+       }
+}
+
+/***************************************************************************
+ *
+ * Link control
+ *
+ ***************************************************************************
+ */
+
 /**
  * Open port
  *
@@ -436,6 +512,13 @@ void ib_close ( struct ib_device *ibdev ) {
                ibdev->op->close ( ibdev );
 }
 
+/***************************************************************************
+ *
+ * Multicast
+ *
+ ***************************************************************************
+ */
+
 /**
  * Attach to multicast group
  *
@@ -495,6 +578,13 @@ void ib_mcast_detach ( struct ib_device *ibdev, struct ib_queue_pair *qp,
        }
 }
 
+/***************************************************************************
+ *
+ * Miscellaneous
+ *
+ ***************************************************************************
+ */
+
 /**
  * Get Infiniband HCA information
  *
@@ -540,6 +630,22 @@ void ib_link_state_changed ( struct ib_device *ibdev ) {
        ipoib_link_state_changed ( ibdev );
 }
 
+/**
+ * Poll event queue
+ *
+ * @v ibdev            Infiniband device
+ */
+void ib_poll_eq ( struct ib_device *ibdev ) {
+       struct ib_completion_queue *cq;
+
+       /* Poll device's event queue */
+       ibdev->op->poll_eq ( ibdev );
+
+       /* Poll all completion queues */
+       list_for_each_entry ( cq, &ibdev->cqs, list )
+               ib_poll_cq ( ibdev, cq );
+}
+
 /**
  * Single-step the Infiniband event queue
  *
@@ -548,9 +654,8 @@ void ib_link_state_changed ( struct ib_device *ibdev ) {
 static void ib_step ( struct process *process __unused ) {
        struct ib_device *ibdev;
 
-       list_for_each_entry ( ibdev, &ib_devices, list ) {
-               ibdev->op->poll_eq ( ibdev );
-       }
+       for_each_ibdev ( ibdev )
+               ib_poll_eq ( ibdev );
 }
 
 /** Infiniband event queue process */
@@ -581,6 +686,7 @@ struct ib_device * alloc_ibdev ( size_t priv_size ) {
        if ( ibdev ) {
                drv_priv = ( ( ( void * ) ibdev ) + sizeof ( *ibdev ) );
                ib_set_drvdata ( ibdev, drv_priv );
+               INIT_LIST_HEAD ( &ibdev->cqs );
                INIT_LIST_HEAD ( &ibdev->qps );
                ibdev->lid = IB_LID_NONE;
                ibdev->pkey = IB_PKEY_NONE;

diff --git a/src/net/infiniband/ib_qset.c b/src/net/infiniband/ib_qset.c
index 799489f..0a1e1f9 100644 (file)
--- a/src/net/infiniband/ib_qset.c
+++ b/src/net/infiniband/ib_qset.c
@@ -54,9 +54,6 @@ int ib_create_qset ( struct ib_device *ibdev, struct ib_queue_set *qset,
        assert ( qset->cq == NULL );
        assert ( qset->qp == NULL );
 
-       /* Store queue parameters */
-       qset->recv_max_fill = num_recv_wqes;
-
        /* Allocate completion queue */
        qset->cq = ib_create_cq ( ibdev, num_cqes, cq_op );
        if ( ! qset->cq ) {
@@ -83,37 +80,6 @@ int ib_create_qset ( struct ib_device *ibdev, struct ib_queue_set *qset,
        return rc;
 }
 
-/**
- * Refill IPoIB receive ring
- *
- * @v ibdev            Infiniband device
- * @v qset             Queue set
- */
-void ib_qset_refill_recv ( struct ib_device *ibdev,
-                          struct ib_queue_set *qset ) {
-       struct io_buffer *iobuf;
-       int rc;
-
-       while ( qset->qp->recv.fill < qset->recv_max_fill ) {
-
-               /* Allocate I/O buffer */
-               iobuf = alloc_iob ( IB_MAX_PAYLOAD_SIZE );
-               if ( ! iobuf ) {
-                       /* Non-fatal; we will refill on next attempt */
-                       return;
-               }
-
-               /* Post I/O buffer */
-               if ( ( rc = ib_post_recv ( ibdev, qset->qp, iobuf ) ) != 0 ) {
-                       DBGC ( ibdev, "IBDEV %p could not refill: %s\n",
-                              ibdev, strerror ( rc ) );
-                       free_iob ( iobuf );
-                       /* Give up */
-                       return;
-               }
-       }
-}
-
 /**
  * Destroy queue set
  *

diff --git a/src/net/infiniband/ib_sma.c b/src/net/infiniband/ib_sma.c
index b83d20e..5fd1319 100644 (file)
--- a/src/net/infiniband/ib_sma.c
+++ b/src/net/infiniband/ib_sma.c
@@ -27,7 +27,6 @@ FILE_LICENCE ( GPL2_OR_LATER );
 #include <byteswap.h>
 #include <gpxe/infiniband.h>
 #include <gpxe/iobuf.h>
-#include <gpxe/process.h>
 #include <gpxe/ib_sma.h>
 
 /**
@@ -348,36 +347,6 @@ static int ib_sma_mad ( struct ib_sma *sma, union ib_mad *mad ) {
        return 0;
 }
 
-/**
- * Refill SMA receive ring
- *
- * @v sma              Subnet management agent
- */
-static void ib_sma_refill_recv ( struct ib_sma *sma ) {
-       struct ib_device *ibdev = sma->ibdev;
-       struct io_buffer *iobuf;
-       int rc;
-
-       while ( sma->qp->recv.fill < IB_SMA_NUM_RECV_WQES ) {
-
-               /* Allocate I/O buffer */
-               iobuf = alloc_iob ( IB_MAX_PAYLOAD_SIZE );
-               if ( ! iobuf ) {
-                       /* Non-fatal; we will refill on next attempt */
-                       return;
-               }
-
-               /* Post I/O buffer */
-               if ( ( rc = ib_post_recv ( ibdev, sma->qp, iobuf ) ) != 0 ) {
-                       DBGC ( sma, "SMA %p could not refill: %s\n",
-                              sma, strerror ( rc ) );
-                       free_iob ( iobuf );
-                       /* Give up */
-                       return;
-               }
-       }
-}
-
 /**
  * Complete SMA send
  *
@@ -456,23 +425,6 @@ static struct ib_completion_queue_operations ib_sma_completion_ops = {
        .complete_recv = ib_sma_complete_recv,
 };
 
-/**
- * Poll SMA
- *
- * @v process          Process
- */
-static void ib_sma_step ( struct process *process ) {
-       struct ib_sma *sma =
-               container_of ( process, struct ib_sma, poll );
-       struct ib_device *ibdev = sma->ibdev;
-
-       /* Poll the kernel completion queue */
-       ib_poll_cq ( ibdev, sma->cq );
-
-       /* Refill the receive ring */
-       ib_sma_refill_recv ( sma );
-}
-
 /**
  * Create SMA
  *
@@ -489,7 +441,6 @@ int ib_create_sma ( struct ib_sma *sma, struct ib_device *ibdev,
        memset ( sma, 0, sizeof ( *sma ) );
        sma->ibdev = ibdev;
        sma->op = op;
-       process_init ( &sma->poll, ib_sma_step, &ibdev->refcnt );
 
        /* Create completion queue */
        sma->cq = ib_create_cq ( ibdev, IB_SMA_NUM_CQES,
@@ -517,7 +468,7 @@ int ib_create_sma ( struct ib_sma *sma, struct ib_device *ibdev,
        }
 
        /* Fill receive ring */
-       ib_sma_refill_recv ( sma );
+       ib_refill_recv ( ibdev, sma->qp );
        return 0;
 
  err_not_qp0:
@@ -525,7 +476,6 @@ int ib_create_sma ( struct ib_sma *sma, struct ib_device *ibdev,
  err_create_qp:
        ib_destroy_cq ( ibdev, sma->cq );
  err_create_cq:
-       process_del ( &sma->poll );
        return rc;
 }
 
@@ -539,5 +489,4 @@ void ib_destroy_sma ( struct ib_sma *sma ) {
 
        ib_destroy_qp ( ibdev, sma->qp );
        ib_destroy_cq ( ibdev, sma->cq );
-       process_del ( &sma->poll );
 }