2 * Copyright (C) 2007 Michael Brown <mbrown@fensystems.co.uk>.
4 * This program is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU General Public License as
6 * published by the Free Software Foundation; either version 2 of the
7 * License, or any later version.
9 * This program is distributed in the hope that it will be useful, but
10 * WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12 * General Public License for more details.
14 * You should have received a copy of the GNU General Public License
15 * along with this program; if not, write to the Free Software
16 * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
27 #include <gpxe/list.h>
28 #include <gpxe/if_arp.h>
29 #include <gpxe/netdevice.h>
30 #include <gpxe/iobuf.h>
31 #include <gpxe/ipoib.h>
32 #include <gpxe/infiniband.h>
41 * Create completion queue
43 * @v ibdev Infiniband device
44 * @v num_cqes Number of completion queue entries
45 * @ret cq New completion queue
47 struct ib_completion_queue * ib_create_cq ( struct ib_device *ibdev,
48 unsigned int num_cqes ) {
49 struct ib_completion_queue *cq;
52 DBGC ( ibdev, "IBDEV %p creating completion queue\n", ibdev );
54 /* Allocate and initialise data structure */
55 cq = zalloc ( sizeof ( *cq ) );
58 cq->num_cqes = num_cqes;
59 INIT_LIST_HEAD ( &cq->work_queues );
61 /* Perform device-specific initialisation and get CQN */
62 if ( ( rc = ibdev->op->create_cq ( ibdev, cq ) ) != 0 ) {
63 DBGC ( ibdev, "IBDEV %p could not initialise completion "
64 "queue: %s\n", ibdev, strerror ( rc ) );
69 DBGC ( ibdev, "IBDEV %p created %d-entry completion queue %p (%p) "
70 "with CQN %#lx\n", ibdev, num_cqes, cq,
71 ib_cq_get_drvdata ( cq ), cq->cqn );
76 * Destroy completion queue
78 * @v ibdev Infiniband device
79 * @v cq Completion queue
81 void ib_destroy_cq ( struct ib_device *ibdev,
82 struct ib_completion_queue *cq ) {
83 DBGC ( ibdev, "IBDEV %p destroying completion queue %#lx\n",
85 assert ( list_empty ( &cq->work_queues ) );
86 ibdev->op->destroy_cq ( ibdev, cq );
93 * @v ibdev Infiniband device
94 * @v num_send_wqes Number of send work queue entries
95 * @v send_cq Send completion queue
96 * @v num_recv_wqes Number of receive work queue entries
97 * @v recv_cq Receive completion queue
101 struct ib_queue_pair * ib_create_qp ( struct ib_device *ibdev,
102 unsigned int num_send_wqes,
103 struct ib_completion_queue *send_cq,
104 unsigned int num_recv_wqes,
105 struct ib_completion_queue *recv_cq,
106 unsigned long qkey ) {
107 struct ib_queue_pair *qp;
111 DBGC ( ibdev, "IBDEV %p creating queue pair\n", ibdev );
113 /* Allocate and initialise data structure */
114 total_size = ( sizeof ( *qp ) +
115 ( num_send_wqes * sizeof ( qp->send.iobufs[0] ) ) +
116 ( num_recv_wqes * sizeof ( qp->recv.iobufs[0] ) ) );
117 qp = zalloc ( total_size );
122 qp->send.is_send = 1;
123 qp->send.cq = send_cq;
124 list_add ( &qp->send.list, &send_cq->work_queues );
125 qp->send.num_wqes = num_send_wqes;
126 qp->send.iobufs = ( ( ( void * ) qp ) + sizeof ( *qp ) );
128 qp->recv.cq = recv_cq;
129 list_add ( &qp->recv.list, &recv_cq->work_queues );
130 qp->recv.num_wqes = num_recv_wqes;
131 qp->recv.iobufs = ( ( ( void * ) qp ) + sizeof ( *qp ) +
132 ( num_send_wqes * sizeof ( qp->send.iobufs[0] ) ));
134 /* Perform device-specific initialisation and get QPN */
135 if ( ( rc = ibdev->op->create_qp ( ibdev, qp ) ) != 0 ) {
136 DBGC ( ibdev, "IBDEV %p could not initialise queue pair: "
137 "%s\n", ibdev, strerror ( rc ) );
138 list_del ( &qp->send.list );
139 list_del ( &qp->recv.list );
144 DBGC ( ibdev, "IBDEV %p created queue pair %p (%p) with QPN %#lx\n",
145 ibdev, qp, ib_qp_get_drvdata ( qp ), qp->qpn );
146 DBGC ( ibdev, "IBDEV %p QPN %#lx has %d send entries at [%p,%p)\n",
147 ibdev, qp->qpn, num_send_wqes, qp->send.iobufs,
149 DBGC ( ibdev, "IBDEV %p QPN %#lx has %d receive entries at [%p,%p)\n",
150 ibdev, qp->qpn, num_recv_wqes, qp->recv.iobufs,
151 ( ( ( void * ) qp ) + total_size ) );
158 * @v ibdev Infiniband device
160 * @v mod_list Modification list
161 * @v qkey New queue key, if applicable
162 * @ret rc Return status code
164 int ib_modify_qp ( struct ib_device *ibdev, struct ib_queue_pair *qp,
165 unsigned long mod_list, unsigned long qkey ) {
168 DBGC ( ibdev, "IBDEV %p modifying QPN %#lx\n", ibdev, qp->qpn );
170 if ( mod_list & IB_MODIFY_QKEY )
173 if ( ( rc = ibdev->op->modify_qp ( ibdev, qp, mod_list ) ) != 0 ) {
174 DBGC ( ibdev, "IBDEV %p could not modify QPN %#lx: %s\n",
175 ibdev, qp->qpn, strerror ( rc ) );
185 * @v ibdev Infiniband device
188 void ib_destroy_qp ( struct ib_device *ibdev, struct ib_queue_pair *qp ) {
189 DBGC ( ibdev, "IBDEV %p destroying QPN %#lx\n",
191 ibdev->op->destroy_qp ( ibdev, qp );
192 list_del ( &qp->send.list );
193 list_del ( &qp->recv.list );
198 * Find work queue belonging to completion queue
200 * @v cq Completion queue
201 * @v qpn Queue pair number
202 * @v is_send Find send work queue (rather than receive)
203 * @ret wq Work queue, or NULL if not found
205 struct ib_work_queue * ib_find_wq ( struct ib_completion_queue *cq,
206 unsigned long qpn, int is_send ) {
207 struct ib_work_queue *wq;
209 list_for_each_entry ( wq, &cq->work_queues, list ) {
210 if ( ( wq->qp->qpn == qpn ) && ( wq->is_send == is_send ) )
216 /***************************************************************************
218 * Management datagram operations
220 ***************************************************************************
224 * Get port information
226 * @v ibdev Infiniband device
227 * @v port_info Port information datagram to fill in
228 * @ret rc Return status code
230 static int ib_get_port_info ( struct ib_device *ibdev,
231 struct ib_mad_port_info *port_info ) {
232 struct ib_mad_hdr *hdr = &port_info->mad_hdr;
236 memset ( port_info, 0, sizeof ( *port_info ) );
237 hdr->base_version = IB_MGMT_BASE_VERSION;
238 hdr->mgmt_class = IB_MGMT_CLASS_SUBN_LID_ROUTED;
239 hdr->class_version = 1;
240 hdr->method = IB_MGMT_METHOD_GET;
241 hdr->attr_id = htons ( IB_SMP_ATTR_PORT_INFO );
242 hdr->attr_mod = htonl ( ibdev->port );
244 if ( ( rc = ib_mad ( ibdev, hdr, sizeof ( *port_info ) ) ) != 0 ) {
245 DBGC ( ibdev, "IBDEV %p could not get port info: %s\n",
246 ibdev, strerror ( rc ) );
253 * Get GUID information
255 * @v ibdev Infiniband device
256 * @v guid_info GUID information datagram to fill in
257 * @ret rc Return status code
259 static int ib_get_guid_info ( struct ib_device *ibdev,
260 struct ib_mad_guid_info *guid_info ) {
261 struct ib_mad_hdr *hdr = &guid_info->mad_hdr;
265 memset ( guid_info, 0, sizeof ( *guid_info ) );
266 hdr->base_version = IB_MGMT_BASE_VERSION;
267 hdr->mgmt_class = IB_MGMT_CLASS_SUBN_LID_ROUTED;
268 hdr->class_version = 1;
269 hdr->method = IB_MGMT_METHOD_GET;
270 hdr->attr_id = htons ( IB_SMP_ATTR_GUID_INFO );
272 if ( ( rc = ib_mad ( ibdev, hdr, sizeof ( *guid_info ) ) ) != 0 ) {
273 DBGC ( ibdev, "IBDEV %p could not get GUID info: %s\n",
274 ibdev, strerror ( rc ) );
281 * Get partition key table
283 * @v ibdev Infiniband device
284 * @v guid_info Partition key table datagram to fill in
285 * @ret rc Return status code
287 static int ib_get_pkey_table ( struct ib_device *ibdev,
288 struct ib_mad_pkey_table *pkey_table ) {
289 struct ib_mad_hdr *hdr = &pkey_table->mad_hdr;
293 memset ( pkey_table, 0, sizeof ( *pkey_table ) );
294 hdr->base_version = IB_MGMT_BASE_VERSION;
295 hdr->mgmt_class = IB_MGMT_CLASS_SUBN_LID_ROUTED;
296 hdr->class_version = 1;
297 hdr->method = IB_MGMT_METHOD_GET;
298 hdr->attr_id = htons ( IB_SMP_ATTR_PKEY_TABLE );
300 if ( ( rc = ib_mad ( ibdev, hdr, sizeof ( *pkey_table ) ) ) != 0 ) {
301 DBGC ( ibdev, "IBDEV %p could not get pkey table: %s\n",
302 ibdev, strerror ( rc ) );
311 * @v ibdev Infiniband device
312 * @ret rc Return status code
314 static int ib_get_mad_params ( struct ib_device *ibdev ) {
316 /* This union exists just to save stack space */
317 struct ib_mad_port_info port_info;
318 struct ib_mad_guid_info guid_info;
319 struct ib_mad_pkey_table pkey_table;
323 /* Port info gives us the link state, the first half of the
324 * port GID and the SM LID.
326 if ( ( rc = ib_get_port_info ( ibdev, &u.port_info ) ) != 0 )
328 ibdev->link_up = ( ( u.port_info.port_state__link_speed_supported
330 memcpy ( &ibdev->port_gid.u.bytes[0], u.port_info.gid_prefix, 8 );
331 ibdev->sm_lid = ntohs ( u.port_info.mastersm_lid );
333 /* GUID info gives us the second half of the port GID */
334 if ( ( rc = ib_get_guid_info ( ibdev, &u.guid_info ) ) != 0 )
336 memcpy ( &ibdev->port_gid.u.bytes[8], u.guid_info.gid_local, 8 );
338 /* Get partition key */
339 if ( ( rc = ib_get_pkey_table ( ibdev, &u.pkey_table ) ) != 0 )
341 ibdev->pkey = ntohs ( u.pkey_table.pkey[0][0] );
343 DBGC ( ibdev, "IBDEV %p port GID is %08lx:%08lx:%08lx:%08lx\n",
344 ibdev, htonl ( ibdev->port_gid.u.dwords[0] ),
345 htonl ( ibdev->port_gid.u.dwords[1] ),
346 htonl ( ibdev->port_gid.u.dwords[2] ),
347 htonl ( ibdev->port_gid.u.dwords[3] ) );
352 /***************************************************************************
354 * Infiniband device creation/destruction
356 ***************************************************************************
360 * Allocate Infiniband device
362 * @v priv_size Size of driver private data area
363 * @ret ibdev Infiniband device, or NULL
365 struct ib_device * alloc_ibdev ( size_t priv_size ) {
366 struct ib_device *ibdev;
370 total_len = ( sizeof ( *ibdev ) + priv_size );
371 ibdev = zalloc ( total_len );
373 drv_priv = ( ( ( void * ) ibdev ) + sizeof ( *ibdev ) );
374 ib_set_drvdata ( ibdev, drv_priv );
380 * Register Infiniband device
382 * @v ibdev Infiniband device
383 * @ret rc Return status code
385 int register_ibdev ( struct ib_device *ibdev ) {
389 if ( ( rc = ib_open ( ibdev ) ) != 0 )
392 /* Get MAD parameters */
393 if ( ( rc = ib_get_mad_params ( ibdev ) ) != 0 )
394 goto err_get_mad_params;
396 /* Add IPoIB device */
397 if ( ( rc = ipoib_probe ( ibdev ) ) != 0 ) {
398 DBGC ( ibdev, "IBDEV %p could not add IPoIB device: %s\n",
399 ibdev, strerror ( rc ) );
400 goto err_ipoib_probe;
413 * Unregister Infiniband device
415 * @v ibdev Infiniband device
417 void unregister_ibdev ( struct ib_device *ibdev ) {
418 ipoib_remove ( ibdev );
423 * Free Infiniband device
425 * @v ibdev Infiniband device
427 void free_ibdev ( struct ib_device *ibdev ) {
432 * Handle Infiniband link state change
434 * @v ibdev Infiniband device
436 void ib_link_state_changed ( struct ib_device *ibdev ) {
439 /* Update MAD parameters */
440 if ( ( rc = ib_get_mad_params ( ibdev ) ) != 0 ) {
441 DBGC ( ibdev, "IBDEV %p could not update MAD parameters: %s\n",
442 ibdev, strerror ( rc ) );
446 /* Notify IPoIB of link state change */
447 ipoib_link_state_changed ( ibdev );