2 * Copyright (c) 2004-2007 Voltaire Inc. All rights reserved.
\r
3 * Copyright (c) 2008 Intel Corp., Inc. All rights reserved.
\r
5 * This software is available to you under a choice of one of two
\r
6 * licenses. You may choose to be licensed under the terms of the GNU
\r
7 * General Public License (GPL) Version 2, available from the file
\r
8 * COPYING in the main directory of this source tree, or the
\r
9 * OpenIB.org BSD license below:
\r
11 * Redistribution and use in source and binary forms, with or
\r
12 * without modification, are permitted provided that the following
\r
13 * conditions are met:
\r
15 * - Redistributions of source code must retain the above
\r
16 * copyright notice, this list of conditions and the following
\r
19 * - Redistributions in binary form must reproduce the above
\r
20 * copyright notice, this list of conditions and the following
\r
21 * disclaimer in the documentation and/or other materials
\r
22 * provided with the distribution.
\r
24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
\r
25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
\r
26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
\r
27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
\r
28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
\r
29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
\r
30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
\r
35 #include <windows.h>
\r
38 #include <infiniband/umad.h>
\r
39 #include <infiniband/verbs.h>
\r
40 #include <rdma/wvstatus.h>
\r
43 #define IB_OPENIB_OUI (0x001405)
\r
45 #define UMAD_MAX_PKEYS 16
\r
47 typedef struct um_port
\r
57 CRITICAL_SECTION crit_sec;
\r
58 um_port_t ports[UMAD_MAX_PORTS];
\r
61 __declspec(dllexport)
\r
64 InitializeCriticalSection(&crit_sec);
\r
68 __declspec(dllexport)
\r
71 DeleteCriticalSection(&crit_sec);
\r
75 __declspec(dllexport)
\r
76 int umad_get_cas_names(char cas[][UMAD_CA_NAME_LEN], int max)
\r
78 struct ibv_device **list;
\r
81 list = ibv_get_device_list(&cnt);
\r
86 for (i = 0; i < min(cnt, max); i++) {
\r
87 strcpy(cas[i], ibv_get_device_name(list[i]));
\r
90 ibv_free_device_list(list);
\r
94 __declspec(dllexport)
\r
95 int umad_get_ca_portguids(char *ca_name, uint64_t *portguids, int max)
\r
100 if (umad_get_ca(ca_name, &ca) < 0)
\r
103 if (ca.numports + 1 > max) {
\r
104 umad_release_ca(&ca);
\r
108 portguids[ports++] = 0;
\r
109 for (i = 1; i <= ca.numports; i++)
\r
110 portguids[ports++] = ca.ports[i]->port_guid;
\r
112 umad_release_ca(&ca);
\r
116 static void umad_convert_ca_attr(umad_ca_t *ca, ibv_device_attr *attr)
\r
118 ca->node_type = 1; // HCA
\r
119 ca->numports = attr->phys_port_cnt;
\r
120 strncpy(ca->fw_ver, attr->fw_ver, 20);
\r
121 memset(ca->ca_type, 0, 40); // TODO: determine what this should be
\r
122 sprintf(ca->hw_ver, "0x%x", attr->hw_ver);
\r
123 ca->node_guid = attr->node_guid;
\r
124 ca->system_guid = attr->sys_image_guid;
\r
127 static int umad_query_port(struct ibv_context *context, umad_port_t *port)
\r
129 ibv_port_attr attr;
\r
133 ret = ibv_query_port(context, (uint8_t) port->portnum, &attr);
\r
138 port->base_lid = attr.lid;
\r
139 port->lmc = attr.lmc;
\r
140 port->sm_lid = attr.sm_lid;
\r
141 port->sm_sl = attr.sm_sl;
\r
142 port->state = attr.state;
\r
143 port->phys_state = attr.phys_state;
\r
144 port->rate = attr.active_speed;
\r
145 port->capmask = attr.port_cap_flags;
\r
147 // Assume GID 0 contains port GUID and gid prefix
\r
148 ret = ibv_query_gid(context, (uint8_t) port->portnum, 0, &gid);
\r
153 port->gid_prefix = gid.global.subnet_prefix;
\r
154 port->port_guid = gid.global.interface_id;
\r
156 port->pkeys_size = min(UMAD_MAX_PKEYS, attr.pkey_tbl_len);
\r
157 for (i = 0; i < (int) port->pkeys_size; i++) {
\r
158 ret = ibv_query_pkey(context,(uint8_t) port->portnum, i, &port->pkeys[i]);
\r
167 __declspec(dllexport)
\r
168 int umad_get_ca(char *ca_name, umad_ca_t *ca)
\r
170 struct ibv_device **list;
\r
171 struct ibv_context *context;
\r
172 ibv_device_attr dev_attr;
\r
173 int cnt, i, ret = 0;
\r
177 list = ibv_get_device_list(&cnt);
\r
178 if (list == NULL) {
\r
182 for (i = 0; i < cnt; i++) {
\r
183 if (!strcmp(ca_name, ibv_get_device_name(list[i]))) {
\r
193 context = ibv_open_device(list[i]);
\r
194 if (context == NULL) {
\r
199 ret = ibv_query_device(context, &dev_attr);
\r
204 port_size = sizeof(umad_port_t) + sizeof(uint16_t) * UMAD_MAX_PKEYS;
\r
205 ports = new uint8_t[port_size * dev_attr.phys_port_cnt];
\r
206 if (ports == NULL) {
\r
211 strcpy(ca->ca_name, ca_name);
\r
212 umad_convert_ca_attr(ca, &dev_attr);
\r
213 memset(ca->ports, 0, sizeof(ca->ports));
\r
215 for (i = 1; i <= dev_attr.phys_port_cnt; i++, ports += port_size) {
\r
217 ca->ports[i] = (umad_port_t *) ports;
\r
218 strcpy(ca->ports[i]->ca_name, ca_name);
\r
219 ca->ports[i]->portnum = i;
\r
220 ca->ports[i]->pkeys = (uint16_t *) (ports + sizeof(umad_port_t));
\r
222 ret = umad_query_port(context, ca->ports[i]);
\r
229 ibv_close_device(context);
\r
231 ibv_free_device_list(list);
\r
238 __declspec(dllexport)
\r
239 int umad_release_ca(umad_ca_t *ca)
\r
241 delete ca->ports[0];
\r
245 static uint64_t umad_get_ca_guid(char *ca_name)
\r
251 ret = umad_get_ca(ca_name, &ca);
\r
256 guid = ca.node_guid;
\r
257 umad_release_ca(&ca);
\r
261 __declspec(dllexport)
\r
262 int umad_get_port(char *ca_name, int portnum, umad_port_t *port)
\r
267 ret = umad_get_ca(ca_name, &ca);
\r
272 memcpy(port, ca.ports[portnum], sizeof(umad_port_t));
\r
274 port->pkeys = new uint16_t[ca.ports[portnum]->pkeys_size];
\r
275 if (port->pkeys == NULL) {
\r
280 memcpy(port->pkeys, ca.ports[portnum]->pkeys,
\r
281 sizeof(uint16_t) * ca.ports[portnum]->pkeys_size);
\r
283 umad_release_ca(&ca);
\r
287 __declspec(dllexport)
\r
288 int umad_release_port(umad_port_t *port)
\r
290 delete port->pkeys;
\r
294 __declspec(dllexport)
\r
295 int umad_get_issm_path(char *ca_name, int portnum, char path[], int max)
\r
300 static uint8_t umad_find_port(char *ca_name, enum ibv_port_state state)
\r
305 ret = umad_get_ca(ca_name, &ca);
\r
310 for (i = 1; i <= ca.numports; i++) {
\r
311 if (ca.ports[i]->state == state) {
\r
312 i = ca.ports[i]->portnum;
\r
313 umad_release_ca(&ca);
\r
314 return (uint8_t) i;
\r
318 umad_release_ca(&ca);
\r
322 static int umad_find_ca_port(enum ibv_port_state state, char *ca_name, uint8_t *port)
\r
324 char names[8][UMAD_CA_NAME_LEN];
\r
327 cnt = umad_get_cas_names(names, 8);
\r
329 for (i = 0; i < cnt; i++) {
\r
330 *port = umad_find_port(names[i], state);
\r
332 strcpy(ca_name, names[i]);
\r
339 static int umad_resolve_ca_port(char *ca_name, uint8_t *port)
\r
343 if (ca_name[0] != NULL) {
\r
348 *port = umad_find_port(ca_name, IBV_PORT_ACTIVE);
\r
352 *port = umad_find_port(ca_name, IBV_PORT_INIT);
\r
356 *port = umad_find_port(ca_name, IBV_PORT_DOWN);
\r
357 return (*port == 0);
\r
360 ret = umad_find_ca_port(IBV_PORT_ACTIVE, ca_name, port);
\r
364 ret = umad_find_ca_port(IBV_PORT_INIT, ca_name, port);
\r
368 ret = umad_find_ca_port(IBV_PORT_DOWN, ca_name, port);
\r
372 __declspec(dllexport)
\r
373 int umad_open_port(char *ca_name, int portnum)
\r
375 char name[UMAD_CA_NAME_LEN];
\r
380 if (ca_name != NULL) {
\r
381 strcpy(name, ca_name);
\r
382 port = (uint8_t) portnum;
\r
388 hr = umad_resolve_ca_port(name, &port);
\r
393 EnterCriticalSection(&crit_sec);
\r
394 for (portid = 0; portid < UMAD_MAX_PORTS; portid++) {
\r
395 if (ports[portid].prov == NULL) {
\r
400 if (portid == UMAD_MAX_PORTS) {
\r
405 ports[portid].overlap.hEvent = CreateEvent(NULL, FALSE, FALSE, NULL);
\r
406 if (ports[portid].overlap.hEvent == NULL) {
\r
411 hr = WmGetObject(IID_IWMProvider, (LPVOID*) &ports[portid].prov);
\r
413 CloseHandle(ports[portid].overlap.hEvent);
\r
414 portid = GetLastError() & 0x80000000;
\r
418 ports[portid].dev_guid = umad_get_ca_guid(name);
\r
419 ports[portid].port_num = port;
\r
422 LeaveCriticalSection(&crit_sec);
\r
426 __declspec(dllexport)
\r
427 int umad_close_port(int portid)
\r
429 CloseHandle(ports[portid].overlap.hEvent);
\r
430 ports[portid].prov->Release();
\r
432 EnterCriticalSection(&crit_sec);
\r
433 ports[portid].prov = NULL;
\r
434 LeaveCriticalSection(&crit_sec);
\r
439 __declspec(dllexport)
\r
440 int umad_set_grh_net(void *umad, void *mad_addr)
\r
442 struct ib_user_mad *mad = (struct ib_user_mad *) umad;
\r
443 struct ib_mad_addr *addr = (struct ib_mad_addr *) mad_addr;
\r
446 mad->addr.grh_present = 1;
\r
447 memcpy(mad->addr.gid, addr->gid, 16);
\r
448 mad->addr.flow_label = addr->flow_label;
\r
449 mad->addr.hop_limit = addr->hop_limit;
\r
450 mad->addr.traffic_class = addr->traffic_class;
\r
452 mad->addr.grh_present = 0;
\r
456 __declspec(dllexport)
\r
457 int umad_set_grh(void *umad, void *mad_addr)
\r
459 struct ib_user_mad *mad = (struct ib_user_mad *) umad;
\r
460 struct ib_mad_addr *addr = (struct ib_mad_addr *) mad_addr;
\r
463 mad->addr.grh_present = 1;
\r
464 memcpy(mad->addr.gid, addr->gid, 16);
\r
465 mad->addr.flow_label = htonl(addr->flow_label);
\r
466 mad->addr.hop_limit = addr->hop_limit;
\r
467 mad->addr.traffic_class = addr->traffic_class;
\r
469 mad->addr.grh_present = 0;
\r
473 __declspec(dllexport)
\r
474 int umad_set_pkey(void *umad, int pkey_index)
\r
476 struct ib_user_mad *mad = (struct ib_user_mad *) umad;
\r
478 mad->addr.pkey_index = (uint16_t) pkey_index;
\r
482 __declspec(dllexport)
\r
483 int umad_get_pkey(void *umad)
\r
485 struct ib_user_mad *mad = (struct ib_user_mad *) umad;
\r
487 return mad->addr.pkey_index;
\r
490 __declspec(dllexport)
\r
491 int umad_set_addr(void *umad, int dlid, int dqp, int sl, int qkey)
\r
493 struct ib_user_mad *mad = (struct ib_user_mad *) umad;
\r
495 mad->addr.qpn = htonl(dqp);
\r
496 mad->addr.lid = htons((uint16_t) dlid);
\r
497 mad->addr.qkey = htonl(qkey);
\r
498 mad->addr.sl = (uint8_t) sl;
\r
502 __declspec(dllexport)
\r
503 int umad_set_addr_net(void *umad, int dlid, int dqp, int sl, int qkey)
\r
505 struct ib_user_mad *mad = (struct ib_user_mad *) umad;
\r
507 mad->addr.qpn = dqp;
\r
508 mad->addr.lid = (uint16_t) dlid;
\r
509 mad->addr.qkey = qkey;
\r
510 mad->addr.sl = (uint8_t) sl;
\r
515 __declspec(dllexport)
\r
516 int umad_status(void *umad)
\r
518 return ((struct ib_user_mad *) umad)->status;
\r
521 __declspec(dllexport)
\r
522 ib_mad_addr_t *umad_get_mad_addr(void *umad)
\r
524 return &((struct ib_user_mad *) umad)->addr;
\r
527 __declspec(dllexport)
\r
528 void *umad_get_mad(void *umad)
\r
530 return ((struct ib_user_mad *)umad)->data;
\r
533 __declspec(dllexport)
\r
534 void *umad_alloc(int num, size_t size)
\r
536 return new uint8_t[num * size];
\r
539 __declspec(dllexport)
\r
540 void umad_free(void *umad)
\r
545 __declspec(dllexport)
\r
546 size_t umad_size(void)
\r
548 return sizeof(struct ib_user_mad);
\r
551 __declspec(dllexport)
\r
552 int umad_send(int portid, int agentid, void *umad, int length,
\r
553 int timeout_ms, int retries)
\r
555 struct ib_user_mad *mad = (struct ib_user_mad *) umad;
\r
558 mad->agent_id = agentid;
\r
559 mad->reserved_id = 0;
\r
561 mad->timeout_ms = (uint32_t) timeout_ms;
\r
562 mad->retries = (uint32_t) retries;
\r
563 mad->length = (uint32_t) length;
\r
565 hr = ports[portid].prov->Send((WM_MAD *) mad, NULL);
\r
567 return GetLastError();
\r
573 __declspec(dllexport)
\r
574 int umad_recv(int portid, void *umad, int *length, int timeout_ms)
\r
576 WM_MAD *mad = (WM_MAD *) umad;
\r
580 port = &ports[portid];
\r
581 hr = port->prov->Receive(mad, sizeof(WM_MAD) + (size_t) *length, &port->overlap);
\r
583 if (hr == WV_IO_PENDING) {
\r
584 if (port->pending && timeout_ms == 0) {
\r
586 hr = WaitForSingleObject(port->overlap.hEvent, 250);
\r
587 } while (hr == WAIT_TIMEOUT && port->pending);
\r
589 hr = WaitForSingleObject(port->overlap.hEvent, (DWORD) timeout_ms);
\r
590 if (hr == WAIT_TIMEOUT) {
\r
591 return -EWOULDBLOCK;
\r
600 if (mad->Length <= (UINT32) *length) {
\r
601 port->pending = FALSE;
\r
602 hr = (HRESULT) mad->Id;
\r
608 *length = mad->Length;
\r
612 __declspec(dllexport)
\r
613 int umad_poll(int portid, int timeout_ms)
\r
619 port = &ports[portid];
\r
620 hr = port->prov->Receive(&mad, sizeof mad, &port->overlap);
\r
622 if (hr == WV_IO_PENDING) {
\r
623 hr = WaitForSingleObject(port->overlap.hEvent, (DWORD) timeout_ms);
\r
624 if (hr == WAIT_TIMEOUT) {
\r
629 if (FAILED(hr) && hr != ERROR_MORE_DATA) {
\r
633 port->pending = TRUE;
\r
637 __declspec(dllexport)
\r
638 int umad_register_oui(int portid, int mgmt_class, uint8_t rmpp_version,
\r
639 uint8_t oui[3], long method_mask[16/sizeof(long)])
\r
644 UNREFERENCED_PARAMETER(rmpp_version);
\r
646 reg.Guid = ports[portid].dev_guid;
\r
647 reg.Qpn = (mgmt_class == 0x01 || mgmt_class == 0x81) ? 0 : htonl(1);
\r
648 reg.Port = ports[portid].port_num;
\r
649 reg.Class = (uint8_t) mgmt_class;
\r
651 memset(reg.Reserved, 0, sizeof(reg.Reserved));
\r
652 memcpy(reg.Oui, oui, sizeof(oui));
\r
653 if (method_mask != NULL) {
\r
654 memcpy(reg.Methods, method_mask, sizeof(reg.Methods));
\r
656 memset(reg.Methods, 0, sizeof(reg.Methods));
\r
658 ports[portid].prov->Register(®, &id);
\r
663 __declspec(dllexport)
\r
664 int umad_register(int portid, int mgmt_class, int mgmt_version,
\r
665 uint8_t rmpp_version, long method_mask[16/sizeof(long)])
\r
670 return umad_register_oui(portid, mgmt_class, rmpp_version, oui, method_mask);
\r
673 __declspec(dllexport)
\r
674 int umad_unregister(int portid, int agentid)
\r
676 ports[portid].pending = FALSE;
\r
677 return ports[portid].prov->Deregister((UINT64) agentid);
\r
680 HANDLE umad_get_fd(int portid)
\r
682 return ports[portid].prov->GetFileHandle();
\r
685 __declspec(dllexport)
\r
686 int umad_debug(int level)
\r
688 UNREFERENCED_PARAMETER(level);
\r
692 __declspec(dllexport)
\r
693 void umad_addr_dump(ib_mad_addr_t *addr)
\r
695 printf("umad_addr:\n");
\r
696 printf("\tqpn 0x%x\n", addr->qpn);
\r
697 printf("\tqkey 0x%x\n", addr->qkey);
\r
698 printf("\tsl 0x%x\n", addr->sl);
\r
699 printf("\tlid 0x%x\n", addr->lid);
\r
700 printf("\tpkey_index 0x%x\n", addr->pkey_index);
\r
701 printf("\tpath_bits 0x%x\n", addr->path_bits);
\r
702 printf("\trate 0x%x\n", addr->reserved_rate);
\r
704 printf("\tgrh_present 0x%x\n", addr->grh_present);
\r
705 if (addr->grh_present) {
\r
706 printf("\tgid_index 0x%x\n", addr->gid_index);
\r
707 printf("\tgid 0x%x %x\n",
\r
708 (uint64_t) addr->gid, (uint64_t) (addr->gid + 8));
\r
709 printf("\tflow_lable 0x%x\n", addr->flow_label);
\r
710 printf("\thop_limit 0x%x\n", addr->hop_limit);
\r
711 printf("\ttraffic_class 0x%x\n", addr->qpn);
\r
715 __declspec(dllexport)
\r
716 void umad_dump(void *umad)
\r
718 struct ib_user_mad *mad = (struct ib_user_mad *) umad;
\r
721 umad_addr_dump(&mad->addr);
\r
722 printf("umad_data\n");
\r
723 printf("offset: hex data\n");
\r
724 for (i = 0; i < 256; i += 4) {
\r
725 printf("%03d: ", i);
\r
726 printf("%02x %02x %02x %02x\n", mad->data[i], mad->data[i + 1],
\r
727 mad->data[i + 2], mad->data[i + 3]);
\r