1 #ifndef _GPXE_INFINIBAND_H
2 #define _GPXE_INFINIBAND_H
11 #include <gpxe/device.h>
13 /** Subnet administrator QPN */
17 #define IB_BROADCAST_QPN 0xffffffUL
19 /** Subnet administrator queue key */
20 #define IB_GLOBAL_QKEY 0x80010000UL
22 /** An Infiniband Global Identifier */
31 /** An Infiniband Global Route Header */
32 struct ib_global_route_header {
33 /** IP version, traffic class, and flow label
35 * 4 bits : Version of the GRH
36 * 8 bits : Traffic class
37 * 20 bits : Flow label
39 uint32_t ipver_tclass_flowlabel;
48 /** Destiniation GID */
50 } __attribute__ (( packed ));
54 struct ib_completion_queue;
56 /** An Infiniband Work Queue */
57 struct ib_work_queue {
58 /** Containing queue pair */
59 struct ib_queue_pair *qp;
60 /** "Is a send queue" flag */
62 /** Associated completion queue */
63 struct ib_completion_queue *cq;
64 /** List of work queues on this completion queue */
65 struct list_head list;
66 /** Number of work queue entries */
67 unsigned int num_wqes;
68 /** Next work queue entry index
70 * This is the index of the next entry to be filled (i.e. the
71 * first empty entry). This value is not bounded by num_wqes;
72 * users must logical-AND with (num_wqes-1) to generate an
75 unsigned long next_idx;
76 /** I/O buffers assigned to work queue */
77 struct io_buffer **iobufs;
78 /** Driver private data */
82 /** An Infiniband Queue Pair */
83 struct ib_queue_pair {
84 /** Queue Pair Number */
89 struct ib_work_queue send;
91 struct ib_work_queue recv;
92 /** Driver private data */
94 /** Queue owner private data */
98 /** Infiniband queue pair modification flags */
99 enum ib_queue_pair_mods {
100 IB_MODIFY_QKEY = 0x0001,
103 /** An Infiniband Completion Queue */
104 struct ib_completion_queue {
105 /** Completion queue number */
107 /** Number of completion queue entries */
108 unsigned int num_cqes;
109 /** Next completion queue entry index
111 * This is the index of the next entry to be filled (i.e. the
112 * first empty entry). This value is not bounded by num_wqes;
113 * users must logical-AND with (num_wqes-1) to generate an
116 unsigned long next_idx;
117 /** List of work queues completing to this queue */
118 struct list_head work_queues;
119 /** Driver private data */
123 /** An Infiniband completion */
124 struct ib_completion {
127 * If non-zero, then the completion is in error.
129 unsigned int syndrome;
134 /** An Infiniband completion handler
136 * @v ibdev Infiniband device
138 * @v completion Completion
139 * @v iobuf I/O buffer
141 typedef void ( * ib_completer_t ) ( struct ib_device *ibdev,
142 struct ib_queue_pair *qp,
143 struct ib_completion *completion,
144 struct io_buffer *iobuf );
146 /** An Infiniband Address Vector */
147 struct ib_address_vector {
148 /** Destination Queue Pair */
149 unsigned int dest_qp;
152 /** Destination Local ID */
158 /** GID is present */
159 unsigned int gid_present;
167 * Infiniband device operations
169 * These represent a subset of the Infiniband Verbs.
171 struct ib_device_operations {
172 /** Create completion queue
174 * @v ibdev Infiniband device
175 * @v cq Completion queue
176 * @ret rc Return status code
178 int ( * create_cq ) ( struct ib_device *ibdev,
179 struct ib_completion_queue *cq );
180 /** Destroy completion queue
182 * @v ibdev Infiniband device
183 * @v cq Completion queue
185 void ( * destroy_cq ) ( struct ib_device *ibdev,
186 struct ib_completion_queue *cq );
187 /** Create queue pair
189 * @v ibdev Infiniband device
191 * @ret rc Return status code
193 int ( * create_qp ) ( struct ib_device *ibdev,
194 struct ib_queue_pair *qp );
195 /** Modify queue pair
197 * @v ibdev Infiniband device
199 * @v mod_list Modification list
200 * @ret rc Return status code
202 int ( * modify_qp ) ( struct ib_device *ibdev,
203 struct ib_queue_pair *qp,
204 unsigned long mod_list );
205 /** Destroy queue pair
207 * @v ibdev Infiniband device
210 void ( * destroy_qp ) ( struct ib_device *ibdev,
211 struct ib_queue_pair *qp );
212 /** Post send work queue entry
214 * @v ibdev Infiniband device
216 * @v av Address vector
217 * @v iobuf I/O buffer
218 * @ret rc Return status code
220 * If this method returns success, the I/O buffer remains
221 * owned by the queue pair. If this method returns failure,
222 * the I/O buffer is immediately released; the failure is
223 * interpreted as "failure to enqueue buffer".
225 int ( * post_send ) ( struct ib_device *ibdev,
226 struct ib_queue_pair *qp,
227 struct ib_address_vector *av,
228 struct io_buffer *iobuf );
229 /** Post receive work queue entry
231 * @v ibdev Infiniband device
233 * @v iobuf I/O buffer
234 * @ret rc Return status code
236 * If this method returns success, the I/O buffer remains
237 * owned by the queue pair. If this method returns failure,
238 * the I/O buffer is immediately released; the failure is
239 * interpreted as "failure to enqueue buffer".
241 int ( * post_recv ) ( struct ib_device *ibdev,
242 struct ib_queue_pair *qp,
243 struct io_buffer *iobuf );
244 /** Poll completion queue
246 * @v ibdev Infiniband device
247 * @v cq Completion queue
248 * @v complete_send Send completion handler
249 * @v complete_recv Receive completion handler
251 * The completion handler takes ownership of the I/O buffer.
253 void ( * poll_cq ) ( struct ib_device *ibdev,
254 struct ib_completion_queue *cq,
255 ib_completer_t complete_send,
256 ib_completer_t complete_recv );
260 * @v ibdev Infiniband device
261 * @ret rc Return status code
263 int ( * open ) ( struct ib_device *ibdev );
267 * @v ibdev Infiniband device
269 void ( * close ) ( struct ib_device *ibdev );
270 /** Attach to multicast group
272 * @v ibdev Infiniband device
274 * @v gid Multicast GID
275 * @ret rc Return status code
277 int ( * mcast_attach ) ( struct ib_device *ibdev,
278 struct ib_queue_pair *qp,
279 struct ib_gid *gid );
280 /** Detach from multicast group
282 * @v ibdev Infiniband device
284 * @v gid Multicast GID
286 void ( * mcast_detach ) ( struct ib_device *ibdev,
287 struct ib_queue_pair *qp,
288 struct ib_gid *gid );
290 * Issue management datagram
292 * @v ibdev Infiniband device
293 * @v mad Management datagram
294 * @v len Length of management datagram
295 * @ret rc Return status code
297 int ( * mad ) ( struct ib_device *ibdev, struct ib_mad_hdr *mad,
301 /** An Infiniband device */
303 /** Underlying device */
305 /** Infiniband operations */
306 struct ib_device_operations *op;
312 struct ib_gid port_gid;
313 /** Subnet manager LID */
314 unsigned long sm_lid;
317 /** Driver private data */
319 /** Owner private data */
323 extern struct ib_completion_queue * ib_create_cq ( struct ib_device *ibdev,
324 unsigned int num_cqes );
325 extern void ib_destroy_cq ( struct ib_device *ibdev,
326 struct ib_completion_queue *cq );
327 extern struct ib_queue_pair *
328 ib_create_qp ( struct ib_device *ibdev, unsigned int num_send_wqes,
329 struct ib_completion_queue *send_cq, unsigned int num_recv_wqes,
330 struct ib_completion_queue *recv_cq, unsigned long qkey );
331 extern int ib_modify_qp ( struct ib_device *ibdev, struct ib_queue_pair *qp,
332 unsigned long mod_list, unsigned long qkey );
333 extern void ib_destroy_qp ( struct ib_device *ibdev,
334 struct ib_queue_pair *qp );
335 extern struct ib_work_queue * ib_find_wq ( struct ib_completion_queue *cq,
336 unsigned long qpn, int is_send );
337 extern struct ib_device * alloc_ibdev ( size_t priv_size );
338 extern int register_ibdev ( struct ib_device *ibdev );
339 extern void unregister_ibdev ( struct ib_device *ibdev );
340 extern void free_ibdev ( struct ib_device *ibdev );
341 extern void ib_link_state_changed ( struct ib_device *ibdev );
344 * Post send work queue entry
346 * @v ibdev Infiniband device
348 * @v av Address vector
349 * @v iobuf I/O buffer
350 * @ret rc Return status code
352 static inline __attribute__ (( always_inline )) int
353 ib_post_send ( struct ib_device *ibdev, struct ib_queue_pair *qp,
354 struct ib_address_vector *av, struct io_buffer *iobuf ) {
355 return ibdev->op->post_send ( ibdev, qp, av, iobuf );
359 * Post receive work queue entry
361 * @v ibdev Infiniband device
363 * @v iobuf I/O buffer
364 * @ret rc Return status code
366 static inline __attribute__ (( always_inline )) int
367 ib_post_recv ( struct ib_device *ibdev, struct ib_queue_pair *qp,
368 struct io_buffer *iobuf ) {
369 return ibdev->op->post_recv ( ibdev, qp, iobuf );
373 * Poll completion queue
375 * @v ibdev Infiniband device
376 * @v cq Completion queue
377 * @v complete_send Send completion handler
378 * @v complete_recv Receive completion handler
380 static inline __attribute__ (( always_inline )) void
381 ib_poll_cq ( struct ib_device *ibdev, struct ib_completion_queue *cq,
382 ib_completer_t complete_send, ib_completer_t complete_recv ) {
383 ibdev->op->poll_cq ( ibdev, cq, complete_send, complete_recv );
389 * @v ibdev Infiniband device
390 * @ret rc Return status code
392 static inline __attribute__ (( always_inline )) int
393 ib_open ( struct ib_device *ibdev ) {
394 return ibdev->op->open ( ibdev );
400 * @v ibdev Infiniband device
402 static inline __attribute__ (( always_inline )) void
403 ib_close ( struct ib_device *ibdev ) {
404 ibdev->op->close ( ibdev );
408 * Attach to multicast group
410 * @v ibdev Infiniband device
412 * @v gid Multicast GID
413 * @ret rc Return status code
415 static inline __attribute__ (( always_inline )) int
416 ib_mcast_attach ( struct ib_device *ibdev, struct ib_queue_pair *qp,
417 struct ib_gid *gid ) {
418 return ibdev->op->mcast_attach ( ibdev, qp, gid );
422 * Detach from multicast group
424 * @v ibdev Infiniband device
426 * @v gid Multicast GID
428 static inline __attribute__ (( always_inline )) void
429 ib_mcast_detach ( struct ib_device *ibdev, struct ib_queue_pair *qp,
430 struct ib_gid *gid ) {
431 ibdev->op->mcast_detach ( ibdev, qp, gid );
435 * Issue management datagram
437 * @v ibdev Infiniband device
438 * @v mad Management datagram
439 * @v len Length of management datagram
440 * @ret rc Return status code
442 static inline __attribute__ (( always_inline )) int
443 ib_mad ( struct ib_device *ibdev, struct ib_mad_hdr *mad, size_t len ) {
444 return ibdev->op->mad ( ibdev, mad, len );
448 * Set Infiniband work queue driver-private data
451 * @v priv Private data
453 static inline __attribute__ (( always_inline )) void
454 ib_wq_set_drvdata ( struct ib_work_queue *wq, void *priv ) {
459 * Get Infiniband work queue driver-private data
462 * @ret priv Private data
464 static inline __attribute__ (( always_inline )) void *
465 ib_wq_get_drvdata ( struct ib_work_queue *wq ) {
470 * Set Infiniband queue pair driver-private data
473 * @v priv Private data
475 static inline __attribute__ (( always_inline )) void
476 ib_qp_set_drvdata ( struct ib_queue_pair *qp, void *priv ) {
481 * Get Infiniband queue pair driver-private data
484 * @ret priv Private data
486 static inline __attribute__ (( always_inline )) void *
487 ib_qp_get_drvdata ( struct ib_queue_pair *qp ) {
492 * Set Infiniband queue pair owner-private data
495 * @v priv Private data
497 static inline __attribute__ (( always_inline )) void
498 ib_qp_set_ownerdata ( struct ib_queue_pair *qp, void *priv ) {
499 qp->owner_priv = priv;
503 * Get Infiniband queue pair owner-private data
506 * @ret priv Private data
508 static inline __attribute__ (( always_inline )) void *
509 ib_qp_get_ownerdata ( struct ib_queue_pair *qp ) {
510 return qp->owner_priv;
514 * Set Infiniband completion queue driver-private data
516 * @v cq Completion queue
517 * @v priv Private data
519 static inline __attribute__ (( always_inline )) void
520 ib_cq_set_drvdata ( struct ib_completion_queue *cq, void *priv ) {
525 * Get Infiniband completion queue driver-private data
527 * @v cq Completion queue
528 * @ret priv Private data
530 static inline __attribute__ (( always_inline )) void *
531 ib_cq_get_drvdata ( struct ib_completion_queue *cq ) {
536 * Set Infiniband device driver-private data
538 * @v ibdev Infiniband device
539 * @v priv Private data
541 static inline __attribute__ (( always_inline )) void
542 ib_set_drvdata ( struct ib_device *ibdev, void *priv ) {
543 ibdev->drv_priv = priv;
547 * Get Infiniband device driver-private data
549 * @v ibdev Infiniband device
550 * @ret priv Private data
552 static inline __attribute__ (( always_inline )) void *
553 ib_get_drvdata ( struct ib_device *ibdev ) {
554 return ibdev->drv_priv;
558 * Set Infiniband device owner-private data
560 * @v ibdev Infiniband device
561 * @v priv Private data
563 static inline __attribute__ (( always_inline )) void
564 ib_set_ownerdata ( struct ib_device *ibdev, void *priv ) {
565 ibdev->owner_priv = priv;
569 * Get Infiniband device owner-private data
571 * @v ibdev Infiniband device
572 * @ret priv Private data
574 static inline __attribute__ (( always_inline )) void *
575 ib_get_ownerdata ( struct ib_device *ibdev ) {
576 return ibdev->owner_priv;
579 /*****************************************************************************
581 * Management datagrams
583 * Portions Copyright (c) 2004 Mellanox Technologies Ltd. All rights
588 /* Management base version */
589 #define IB_MGMT_BASE_VERSION 1
591 /* Management classes */
592 #define IB_MGMT_CLASS_SUBN_LID_ROUTED 0x01
593 #define IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE 0x81
594 #define IB_MGMT_CLASS_SUBN_ADM 0x03
595 #define IB_MGMT_CLASS_PERF_MGMT 0x04
596 #define IB_MGMT_CLASS_BM 0x05
597 #define IB_MGMT_CLASS_DEVICE_MGMT 0x06
598 #define IB_MGMT_CLASS_CM 0x07
599 #define IB_MGMT_CLASS_SNMP 0x08
600 #define IB_MGMT_CLASS_VENDOR_RANGE2_START 0x30
601 #define IB_MGMT_CLASS_VENDOR_RANGE2_END 0x4F
603 /* Management methods */
604 #define IB_MGMT_METHOD_GET 0x01
605 #define IB_MGMT_METHOD_SET 0x02
606 #define IB_MGMT_METHOD_GET_RESP 0x81
607 #define IB_MGMT_METHOD_SEND 0x03
608 #define IB_MGMT_METHOD_TRAP 0x05
609 #define IB_MGMT_METHOD_REPORT 0x06
610 #define IB_MGMT_METHOD_REPORT_RESP 0x86
611 #define IB_MGMT_METHOD_TRAP_REPRESS 0x07
612 #define IB_MGMT_METHOD_DELETE 0x15
613 #define IB_MGMT_METHOD_RESP 0x80
615 /* Subnet management attributes */
616 #define IB_SMP_ATTR_NOTICE 0x0002
617 #define IB_SMP_ATTR_NODE_DESC 0x0010
618 #define IB_SMP_ATTR_NODE_INFO 0x0011
619 #define IB_SMP_ATTR_SWITCH_INFO 0x0012
620 #define IB_SMP_ATTR_GUID_INFO 0x0014
621 #define IB_SMP_ATTR_PORT_INFO 0x0015
622 #define IB_SMP_ATTR_PKEY_TABLE 0x0016
623 #define IB_SMP_ATTR_SL_TO_VL_TABLE 0x0017
624 #define IB_SMP_ATTR_VL_ARB_TABLE 0x0018
625 #define IB_SMP_ATTR_LINEAR_FORWARD_TABLE 0x0019
626 #define IB_SMP_ATTR_RANDOM_FORWARD_TABLE 0x001A
627 #define IB_SMP_ATTR_MCAST_FORWARD_TABLE 0x001B
628 #define IB_SMP_ATTR_SM_INFO 0x0020
629 #define IB_SMP_ATTR_VENDOR_DIAG 0x0030
630 #define IB_SMP_ATTR_LED_INFO 0x0031
631 #define IB_SMP_ATTR_VENDOR_MASK 0xFF00
633 #define IB_SA_ATTR_MC_MEMBER_REC 0x38
634 #define IB_SA_ATTR_PATH_REC 0x35
636 #define IB_SA_MCMEMBER_REC_MGID (1<<0)
637 #define IB_SA_MCMEMBER_REC_PORT_GID (1<<1)
638 #define IB_SA_MCMEMBER_REC_QKEY (1<<2)
639 #define IB_SA_MCMEMBER_REC_MLID (1<<3)
640 #define IB_SA_MCMEMBER_REC_MTU_SELECTOR (1<<4)
641 #define IB_SA_MCMEMBER_REC_MTU (1<<5)
642 #define IB_SA_MCMEMBER_REC_TRAFFIC_CLASS (1<<6)
643 #define IB_SA_MCMEMBER_REC_PKEY (1<<7)
644 #define IB_SA_MCMEMBER_REC_RATE_SELECTOR (1<<8)
645 #define IB_SA_MCMEMBER_REC_RATE (1<<9)
646 #define IB_SA_MCMEMBER_REC_PACKET_LIFE_TIME_SELECTOR (1<<10)
647 #define IB_SA_MCMEMBER_REC_PACKET_LIFE_TIME (1<<11)
648 #define IB_SA_MCMEMBER_REC_SL (1<<12)
649 #define IB_SA_MCMEMBER_REC_FLOW_LABEL (1<<13)
650 #define IB_SA_MCMEMBER_REC_HOP_LIMIT (1<<14)
651 #define IB_SA_MCMEMBER_REC_SCOPE (1<<15)
652 #define IB_SA_MCMEMBER_REC_JOIN_STATE (1<<16)
653 #define IB_SA_MCMEMBER_REC_PROXY_JOIN (1<<17)
655 #define IB_SA_PATH_REC_DGID (1<<2)
656 #define IB_SA_PATH_REC_SGID (1<<3)
659 uint8_t base_version;
661 uint8_t class_version;
664 uint16_t class_specific;
669 } __attribute__ (( packed ));
674 uint16_t attrib_offset;
675 uint32_t comp_mask[2];
676 } __attribute__ (( packed ));
680 } __attribute__ (( packed ));
683 struct ib_mad_hdr mad_hdr;
685 } __attribute__ (( packed ));
687 struct ib_mad_guid_info {
688 struct ib_mad_hdr mad_hdr;
690 uint32_t reserved[8];
691 uint8_t gid_local[8];
692 } __attribute__ (( packed ));
694 struct ib_mad_port_info {
695 struct ib_mad_hdr mad_hdr;
697 uint32_t reserved[8];
699 uint8_t gid_prefix[8];
701 uint16_t mastersm_lid;
704 uint16_t mkey_lease_period;
705 uint8_t local_port_num;
706 uint8_t link_width_enabled;
707 uint8_t link_width_supported;
708 uint8_t link_width_active;
709 uint8_t port_state__link_speed_supported;
710 uint8_t link_down_def_state__port_phys_state;
711 uint8_t lmc__r1__mkey_prot_bits;
712 uint8_t link_speed_enabled__link_speed_active;
713 } __attribute__ (( packed ));
715 struct ib_mad_pkey_table {
716 struct ib_mad_hdr mad_hdr;
718 uint32_t reserved[8];
719 uint16_t pkey[16][2];
720 } __attribute__ (( packed ));
722 struct ib_mad_path_record {
723 struct ib_mad_hdr mad_hdr;
724 struct ib_rmpp_hdr rmpp_hdr;
725 struct ib_sa_hdr sa_hdr;
726 uint32_t reserved0[2];
731 uint32_t hop_limit__flow_label__raw_traffic;
732 uint32_t pkey__numb_path__reversible__tclass;
734 uint8_t reserved__sl;
735 uint8_t mtu_selector__mtu;
736 uint8_t rate_selector__rate;
737 uint32_t preference__packet_lifetime__packet_lifetime_selector;
738 uint32_t reserved2[35];
739 } __attribute__ (( packed ));
741 struct ib_mad_mc_member_record {
742 struct ib_mad_hdr mad_hdr;
743 struct ib_rmpp_hdr rmpp_hdr;
744 struct ib_sa_hdr sa_hdr;
746 struct ib_gid port_gid;
749 uint8_t mtu_selector__mtu;
752 uint8_t rate_selector__rate;
753 uint8_t packet_lifetime_selector__packet_lifetime;
754 uint32_t sl__flow_label__hop_limit;
755 uint8_t scope__join_state;
756 uint8_t proxy_join__reserved;
758 uint32_t reserved1[37];
759 } __attribute__ (( packed ));
762 struct ib_mad_hdr mad_hdr;
763 struct ib_mad_data data;
764 struct ib_mad_guid_info guid_info;
765 struct ib_mad_port_info port_info;
766 struct ib_mad_pkey_table pkey_table;
767 struct ib_mad_path_record path_record;
768 struct ib_mad_mc_member_record mc_member_record;
769 } __attribute__ (( packed ));
771 #endif /* _GPXE_INFINIBAND_H */