2 * Copyright (c) 2004-2007 Voltaire Inc. All rights reserved.
\r
3 * Copyright (c) 2008 Intel Corp., Inc. All rights reserved.
\r
5 * This software is available to you under a choice of one of two
\r
6 * licenses. You may choose to be licensed under the terms of the GNU
\r
7 * General Public License (GPL) Version 2, available from the file
\r
8 * COPYING in the main directory of this source tree, or the
\r
9 * OpenIB.org BSD license below:
\r
11 * Redistribution and use in source and binary forms, with or
\r
12 * without modification, are permitted provided that the following
\r
13 * conditions are met:
\r
15 * - Redistributions of source code must retain the above
\r
16 * copyright notice, this list of conditions and the following
\r
19 * - Redistributions in binary form must reproduce the above
\r
20 * copyright notice, this list of conditions and the following
\r
21 * disclaimer in the documentation and/or other materials
\r
22 * provided with the distribution.
\r
24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
\r
25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
\r
26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
\r
27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
\r
28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
\r
29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
\r
30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
\r
35 #include <windows.h>
\r
38 #include <infiniband/umad.h>
\r
39 #include <infiniband/verbs.h>
\r
40 #include <rdma/wvstatus.h>
\r
43 #define IB_OPENIB_OUI (0x001405)
\r
45 #define UMAD_MAX_PKEYS 16
\r
47 typedef struct um_port
\r
57 CRITICAL_SECTION crit_sec;
\r
58 um_port_t ports[UMAD_MAX_PORTS];
\r
61 __declspec(dllexport)
\r
64 InitializeCriticalSection(&crit_sec);
\r
68 __declspec(dllexport)
\r
71 DeleteCriticalSection(&crit_sec);
\r
75 __declspec(dllexport)
\r
76 int umad_get_cas_names(char cas[][UMAD_CA_NAME_LEN], int max)
\r
78 struct ibv_device **list;
\r
81 list = ibv_get_device_list(&cnt);
\r
86 for (i = 0; i < min(cnt, max); i++) {
\r
87 strcpy(cas[i], ibv_get_device_name(list[i]));
\r
90 ibv_free_device_list(list);
\r
94 __declspec(dllexport)
\r
95 int umad_get_ca_portguids(char *ca_name, uint64_t *portguids, int max)
\r
100 if (umad_get_ca(ca_name, &ca) < 0)
\r
103 if (ca.numports + 1 > max) {
\r
104 umad_release_ca(&ca);
\r
108 portguids[ports++] = 0;
\r
109 for (i = 1; i <= ca.numports; i++)
\r
110 portguids[ports++] = ca.ports[i]->port_guid;
\r
112 umad_release_ca(&ca);
\r
116 static void umad_convert_ca_attr(umad_ca_t *ca, ibv_device_attr *attr)
\r
118 ca->node_type = 1; // HCA
\r
119 ca->numports = attr->phys_port_cnt;
\r
120 strncpy(ca->fw_ver, attr->fw_ver, 20);
\r
121 memset(ca->ca_type, 0, 40); // TODO: determine what this should be
\r
122 sprintf(ca->hw_ver, "0x%x", attr->hw_ver);
\r
123 ca->node_guid = attr->node_guid;
\r
124 ca->system_guid = attr->sys_image_guid;
\r
127 static int umad_query_port(struct ibv_context *context, umad_port_t *port)
\r
129 ibv_port_attr attr;
\r
133 ret = ibv_query_port(context, (uint8_t) port->portnum, &attr);
\r
138 port->base_lid = attr.lid;
\r
139 port->lmc = attr.lmc;
\r
140 port->sm_lid = attr.sm_lid;
\r
141 port->sm_sl = attr.sm_sl;
\r
142 port->state = attr.state;
\r
143 port->phys_state = attr.phys_state;
\r
144 port->rate = attr.active_speed;
\r
145 port->capmask = attr.port_cap_flags;
\r
147 // Assume GID 0 contains port GUID and gid prefix
\r
148 ret = ibv_query_gid(context, (uint8_t) port->portnum, 0, &gid);
\r
153 port->gid_prefix = gid.global.subnet_prefix;
\r
154 port->port_guid = gid.global.interface_id;
\r
156 port->pkeys_size = min(UMAD_MAX_PKEYS, attr.pkey_tbl_len);
\r
157 for (i = 0; i < (int) port->pkeys_size; i++) {
\r
158 ret = ibv_query_pkey(context,(uint8_t) port->portnum, i, &port->pkeys[i]);
\r
167 __declspec(dllexport)
\r
168 int umad_get_ca(char *ca_name, umad_ca_t *ca)
\r
170 struct ibv_device **list;
\r
171 struct ibv_context *context;
\r
172 ibv_device_attr dev_attr;
\r
173 int cnt, i, ret = 0;
\r
177 list = ibv_get_device_list(&cnt);
\r
178 if (list == NULL) {
\r
182 for (i = 0; i < cnt; i++) {
\r
183 if (!strcmp(ca_name, ibv_get_device_name(list[i]))) {
\r
193 context = ibv_open_device(list[i]);
\r
194 if (context == NULL) {
\r
199 ret = ibv_query_device(context, &dev_attr);
\r
204 port_size = sizeof(umad_port_t) + sizeof(uint16_t) * UMAD_MAX_PKEYS;
\r
205 ports = new uint8_t[port_size * dev_attr.phys_port_cnt];
\r
206 if (ports == NULL) {
\r
211 strcpy(ca->ca_name, ca_name);
\r
212 umad_convert_ca_attr(ca, &dev_attr);
\r
213 memset(ca->ports, 0, sizeof(ca->ports));
\r
215 for (i = 1; i <= dev_attr.phys_port_cnt; i++, ports += port_size) {
\r
217 ca->ports[i] = (umad_port_t *) ports;
\r
218 strcpy(ca->ports[i]->ca_name, ca_name);
\r
219 ca->ports[i]->portnum = i;
\r
220 ca->ports[i]->pkeys = (uint16_t *) (ports + sizeof(umad_port_t));
\r
222 ret = umad_query_port(context, ca->ports[i]);
\r
230 ibv_close_device(context);
\r
232 ibv_free_device_list(list);
\r
236 __declspec(dllexport)
\r
237 int umad_release_ca(umad_ca_t *ca)
\r
239 delete ca->ports[0];
\r
243 static uint64_t umad_get_ca_guid(char *ca_name)
\r
249 ret = umad_get_ca(ca_name, &ca);
\r
254 guid = ca.node_guid;
\r
255 umad_release_ca(&ca);
\r
259 __declspec(dllexport)
\r
260 int umad_get_port(char *ca_name, int portnum, umad_port_t *port)
\r
265 ret = umad_get_ca(ca_name, &ca);
\r
270 memcpy(port, ca.ports[portnum], sizeof(umad_port_t));
\r
272 port->pkeys = new uint16_t[ca.ports[portnum]->pkeys_size];
\r
273 if (port->pkeys == NULL) {
\r
278 memcpy(port->pkeys, ca.ports[portnum]->pkeys,
\r
279 sizeof(uint16_t) * ca.ports[portnum]->pkeys_size);
\r
281 umad_release_ca(&ca);
\r
285 __declspec(dllexport)
\r
286 int umad_release_port(umad_port_t *port)
\r
288 delete port->pkeys;
\r
292 __declspec(dllexport)
\r
293 int umad_get_issm_path(char *ca_name, int portnum, char path[], int max)
\r
298 static uint8_t umad_find_port(char *ca_name, enum ibv_port_state state)
\r
303 ret = umad_get_ca(ca_name, &ca);
\r
308 for (i = 1; i <= ca.numports; i++) {
\r
309 if (ca.ports[i]->state == state) {
\r
310 i = ca.ports[i]->portnum;
\r
311 umad_release_ca(&ca);
\r
312 return (uint8_t) i;
\r
316 umad_release_ca(&ca);
\r
320 static int umad_find_ca_port(enum ibv_port_state state, char *ca_name, uint8_t *port)
\r
322 char names[8][UMAD_CA_NAME_LEN];
\r
325 cnt = umad_get_cas_names(names, 8);
\r
327 for (i = 0; i < cnt; i++) {
\r
328 *port = umad_find_port(names[i], state);
\r
330 strcpy(ca_name, names[i]);
\r
337 static int umad_resolve_ca_port(char *ca_name, uint8_t *port)
\r
341 if (ca_name[0] != NULL) {
\r
346 *port = umad_find_port(ca_name, IBV_PORT_ACTIVE);
\r
350 *port = umad_find_port(ca_name, IBV_PORT_INIT);
\r
354 *port = umad_find_port(ca_name, IBV_PORT_DOWN);
\r
355 return (*port == 0);
\r
358 ret = umad_find_ca_port(IBV_PORT_ACTIVE, ca_name, port);
\r
362 ret = umad_find_ca_port(IBV_PORT_INIT, ca_name, port);
\r
366 ret = umad_find_ca_port(IBV_PORT_DOWN, ca_name, port);
\r
370 __declspec(dllexport)
\r
371 int umad_open_port(char *ca_name, int portnum)
\r
373 char name[UMAD_CA_NAME_LEN];
\r
378 if (ca_name != NULL) {
\r
379 strcpy(name, ca_name);
\r
380 port = (uint8_t) portnum;
\r
386 hr = umad_resolve_ca_port(name, &port);
\r
391 EnterCriticalSection(&crit_sec);
\r
392 for (portid = 0; portid < UMAD_MAX_PORTS; portid++) {
\r
393 if (ports[portid].prov == NULL) {
\r
398 if (portid == UMAD_MAX_PORTS) {
\r
403 ports[portid].overlap.hEvent = CreateEvent(NULL, FALSE, FALSE, NULL);
\r
404 if (ports[portid].overlap.hEvent == NULL) {
\r
409 hr = WmGetObject(IID_IWMProvider, (LPVOID*) &ports[portid].prov);
\r
411 CloseHandle(ports[portid].overlap.hEvent);
\r
412 portid = GetLastError() & 0x80000000;
\r
416 ports[portid].dev_guid = umad_get_ca_guid(name);
\r
417 ports[portid].port_num = port;
\r
420 LeaveCriticalSection(&crit_sec);
\r
424 __declspec(dllexport)
\r
425 int umad_close_port(int portid)
\r
427 CloseHandle(ports[portid].overlap.hEvent);
\r
428 ports[portid].prov->Release();
\r
430 EnterCriticalSection(&crit_sec);
\r
431 ports[portid].prov = NULL;
\r
432 LeaveCriticalSection(&crit_sec);
\r
437 __declspec(dllexport)
\r
438 int umad_set_grh_net(void *umad, void *mad_addr)
\r
440 struct ib_user_mad *mad = (struct ib_user_mad *) umad;
\r
441 struct ib_mad_addr *addr = (struct ib_mad_addr *) mad_addr;
\r
444 mad->addr.grh_present = 1;
\r
445 memcpy(mad->addr.gid, addr->gid, 16);
\r
446 mad->addr.flow_label = addr->flow_label;
\r
447 mad->addr.hop_limit = addr->hop_limit;
\r
448 mad->addr.traffic_class = addr->traffic_class;
\r
450 mad->addr.grh_present = 0;
\r
454 __declspec(dllexport)
\r
455 int umad_set_grh(void *umad, void *mad_addr)
\r
457 struct ib_user_mad *mad = (struct ib_user_mad *) umad;
\r
458 struct ib_mad_addr *addr = (struct ib_mad_addr *) mad_addr;
\r
461 mad->addr.grh_present = 1;
\r
462 memcpy(mad->addr.gid, addr->gid, 16);
\r
463 mad->addr.flow_label = htonl(addr->flow_label);
\r
464 mad->addr.hop_limit = addr->hop_limit;
\r
465 mad->addr.traffic_class = addr->traffic_class;
\r
467 mad->addr.grh_present = 0;
\r
471 __declspec(dllexport)
\r
472 int umad_set_pkey(void *umad, int pkey_index)
\r
474 struct ib_user_mad *mad = (struct ib_user_mad *) umad;
\r
476 mad->addr.pkey_index = (uint16_t) pkey_index;
\r
480 __declspec(dllexport)
\r
481 int umad_get_pkey(void *umad)
\r
483 struct ib_user_mad *mad = (struct ib_user_mad *) umad;
\r
485 return mad->addr.pkey_index;
\r
488 __declspec(dllexport)
\r
489 int umad_set_addr(void *umad, int dlid, int dqp, int sl, int qkey)
\r
491 struct ib_user_mad *mad = (struct ib_user_mad *) umad;
\r
493 mad->addr.qpn = htonl(dqp);
\r
494 mad->addr.lid = htons((uint16_t) dlid);
\r
495 mad->addr.qkey = htonl(qkey);
\r
496 mad->addr.sl = (uint8_t) sl;
\r
500 __declspec(dllexport)
\r
501 int umad_set_addr_net(void *umad, int dlid, int dqp, int sl, int qkey)
\r
503 struct ib_user_mad *mad = (struct ib_user_mad *) umad;
\r
505 mad->addr.qpn = dqp;
\r
506 mad->addr.lid = (uint16_t) dlid;
\r
507 mad->addr.qkey = qkey;
\r
508 mad->addr.sl = (uint8_t) sl;
\r
513 __declspec(dllexport)
\r
514 int umad_status(void *umad)
\r
516 return ((struct ib_user_mad *) umad)->status;
\r
519 __declspec(dllexport)
\r
520 ib_mad_addr_t *umad_get_mad_addr(void *umad)
\r
522 return &((struct ib_user_mad *) umad)->addr;
\r
525 __declspec(dllexport)
\r
526 void *umad_get_mad(void *umad)
\r
528 return ((struct ib_user_mad *)umad)->data;
\r
531 __declspec(dllexport)
\r
532 void *umad_alloc(int num, size_t size)
\r
534 return new uint8_t[num * size];
\r
537 __declspec(dllexport)
\r
538 void umad_free(void *umad)
\r
543 __declspec(dllexport)
\r
544 size_t umad_size(void)
\r
546 return sizeof(struct ib_user_mad);
\r
549 __declspec(dllexport)
\r
550 int umad_send(int portid, int agentid, void *umad, int length,
\r
551 int timeout_ms, int retries)
\r
553 struct ib_user_mad *mad = (struct ib_user_mad *) umad;
\r
556 mad->agent_id = agentid;
\r
557 mad->reserved_id = 0;
\r
559 mad->timeout_ms = (uint32_t) timeout_ms;
\r
560 mad->retries = (uint32_t) retries;
\r
561 mad->length = (uint32_t) length;
\r
563 hr = ports[portid].prov->Send((WM_MAD *) mad, NULL);
\r
565 return GetLastError();
\r
571 __declspec(dllexport)
\r
572 int umad_recv(int portid, void *umad, int *length, int timeout_ms)
\r
574 WM_MAD *mad = (WM_MAD *) umad;
\r
578 port = &ports[portid];
\r
579 hr = port->prov->Receive(mad, sizeof(WM_MAD) + (size_t) *length, &port->overlap);
\r
581 if (hr == WV_IO_PENDING) {
\r
582 if (port->pending && timeout_ms == 0) {
\r
584 hr = WaitForSingleObject(port->overlap.hEvent, 250);
\r
585 } while (hr == WAIT_TIMEOUT && port->pending);
\r
587 hr = WaitForSingleObject(port->overlap.hEvent, (DWORD) timeout_ms);
\r
588 if (hr == WAIT_TIMEOUT) {
\r
589 return -EWOULDBLOCK;
\r
598 if (mad->Length <= (UINT32) *length) {
\r
599 port->pending = FALSE;
\r
600 hr = (HRESULT) mad->Id;
\r
606 *length = mad->Length;
\r
610 __declspec(dllexport)
\r
611 int umad_poll(int portid, int timeout_ms)
\r
617 port = &ports[portid];
\r
618 hr = port->prov->Receive(&mad, sizeof mad, &port->overlap);
\r
620 if (hr == WV_IO_PENDING) {
\r
621 hr = WaitForSingleObject(port->overlap.hEvent, (DWORD) timeout_ms);
\r
622 if (hr == WAIT_TIMEOUT) {
\r
627 if (FAILED(hr) && hr != ERROR_MORE_DATA) {
\r
631 port->pending = TRUE;
\r
635 __declspec(dllexport)
\r
636 int umad_register_oui(int portid, int mgmt_class, uint8_t rmpp_version,
\r
637 uint8_t oui[3], long method_mask[16/sizeof(long)])
\r
642 UNREFERENCED_PARAMETER(rmpp_version);
\r
644 reg.Guid = ports[portid].dev_guid;
\r
645 reg.Qpn = (mgmt_class == 0x01 || mgmt_class == 0x81) ? 0 : htonl(1);
\r
646 reg.Port = ports[portid].port_num;
\r
647 reg.Class = (uint8_t) mgmt_class;
\r
649 memset(reg.Reserved, 0, sizeof(reg.Reserved));
\r
650 memcpy(reg.Oui, oui, sizeof(oui));
\r
651 if (method_mask != NULL) {
\r
652 memcpy(reg.Methods, method_mask, sizeof(reg.Methods));
\r
654 memset(reg.Methods, 0, sizeof(reg.Methods));
\r
656 ports[portid].prov->Register(®, &id);
\r
661 __declspec(dllexport)
\r
662 int umad_register(int portid, int mgmt_class, int mgmt_version,
\r
663 uint8_t rmpp_version, long method_mask[16/sizeof(long)])
\r
668 return umad_register_oui(portid, mgmt_class, rmpp_version, oui, method_mask);
\r
671 __declspec(dllexport)
\r
672 int umad_unregister(int portid, int agentid)
\r
674 ports[portid].pending = FALSE;
\r
675 return ports[portid].prov->Deregister((UINT64) agentid);
\r
678 HANDLE umad_get_fd(int portid)
\r
680 return ports[portid].prov->GetFileHandle();
\r
683 __declspec(dllexport)
\r
684 int umad_debug(int level)
\r
686 UNREFERENCED_PARAMETER(level);
\r
690 __declspec(dllexport)
\r
691 void umad_addr_dump(ib_mad_addr_t *addr)
\r
693 printf("umad_addr:\n");
\r
694 printf("\tqpn 0x%x\n", addr->qpn);
\r
695 printf("\tqkey 0x%x\n", addr->qkey);
\r
696 printf("\tsl 0x%x\n", addr->sl);
\r
697 printf("\tlid 0x%x\n", addr->lid);
\r
698 printf("\tpkey_index 0x%x\n", addr->pkey_index);
\r
699 printf("\tpath_bits 0x%x\n", addr->path_bits);
\r
700 printf("\trate 0x%x\n", addr->reserved_rate);
\r
702 printf("\tgrh_present 0x%x\n", addr->grh_present);
\r
703 if (addr->grh_present) {
\r
704 printf("\tgid_index 0x%x\n", addr->gid_index);
\r
705 printf("\tgid 0x%x %x\n",
\r
706 (uint64_t) addr->gid, (uint64_t) (addr->gid + 8));
\r
707 printf("\tflow_lable 0x%x\n", addr->flow_label);
\r
708 printf("\thop_limit 0x%x\n", addr->hop_limit);
\r
709 printf("\ttraffic_class 0x%x\n", addr->qpn);
\r
713 __declspec(dllexport)
\r
714 void umad_dump(void *umad)
\r
716 struct ib_user_mad *mad = (struct ib_user_mad *) umad;
\r
719 umad_addr_dump(&mad->addr);
\r
720 printf("umad_data\n");
\r
721 printf("offset: hex data\n");
\r
722 for (i = 0; i < 256; i += 4) {
\r
723 printf("%03d: ", i);
\r
724 printf("%02x %02x %02x %02x\n", mad->data[i], mad->data[i + 1],
\r
725 mad->data[i + 2], mad->data[i + 3]);
\r