2 * Copyright (c) 2004-2007 Voltaire Inc. All rights reserved.
\r
3 * Copyright (c) 2008 Intel Corp., Inc. All rights reserved.
\r
5 * This software is available to you under a choice of one of two
\r
6 * licenses. You may choose to be licensed under the terms of the GNU
\r
7 * General Public License (GPL) Version 2, available from the file
\r
8 * COPYING in the main directory of this source tree, or the
\r
9 * OpenIB.org BSD license below:
\r
11 * Redistribution and use in source and binary forms, with or
\r
12 * without modification, are permitted provided that the following
\r
13 * conditions are met:
\r
15 * - Redistributions of source code must retain the above
\r
16 * copyright notice, this list of conditions and the following
\r
19 * - Redistributions in binary form must reproduce the above
\r
20 * copyright notice, this list of conditions and the following
\r
21 * disclaimer in the documentation and/or other materials
\r
22 * provided with the distribution.
\r
24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
\r
25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
\r
26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
\r
27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
\r
28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
\r
29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
\r
30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
\r
35 #include <windows.h>
\r
38 #include <infiniband/umad.h>
\r
39 #include <infiniband/verbs.h>
\r
40 #include <rdma/wvstatus.h>
\r
43 #define IB_OPENIB_OUI (0x001405)
\r
45 #define UMAD_MAX_PKEYS 16
\r
47 typedef struct um_port
\r
57 CRITICAL_SECTION crit_sec;
\r
58 um_port_t ports[UMAD_MAX_PORTS];
\r
61 __declspec(dllexport)
\r
64 InitializeCriticalSection(&crit_sec);
\r
68 __declspec(dllexport)
\r
74 __declspec(dllexport)
\r
75 int umad_get_cas_names(char cas[][UMAD_CA_NAME_LEN], int max)
\r
77 struct ibv_device **list;
\r
80 list = ibv_get_device_list(&cnt);
\r
85 for (i = 0; i < min(cnt, max); i++) {
\r
86 strcpy(cas[i], ibv_get_device_name(list[i]));
\r
89 ibv_free_device_list(list);
\r
93 __declspec(dllexport)
\r
94 int umad_get_ca_portguids(char *ca_name, uint64_t *portguids, int max)
\r
99 if (umad_get_ca(ca_name, &ca) < 0)
\r
102 if (ca.numports + 1 > max) {
\r
103 umad_release_ca(&ca);
\r
107 portguids[ports++] = 0;
\r
108 for (i = 1; i <= ca.numports; i++)
\r
109 portguids[ports++] = ca.ports[i]->port_guid;
\r
111 umad_release_ca(&ca);
\r
115 static void umad_convert_ca_attr(umad_ca_t *ca, ibv_device_attr *attr)
\r
117 ca->node_type = 1; // HCA
\r
118 ca->numports = attr->phys_port_cnt;
\r
119 strncpy(ca->fw_ver, attr->fw_ver, 20);
\r
120 memset(ca->ca_type, 0, 40); // TODO: determine what this should be
\r
121 sprintf(ca->hw_ver, "0x%x", attr->hw_ver);
\r
122 ca->node_guid = attr->node_guid;
\r
123 ca->system_guid = attr->sys_image_guid;
\r
126 static int umad_query_port(struct ibv_context *context, umad_port_t *port)
\r
128 ibv_port_attr attr;
\r
132 ret = ibv_query_port(context, (uint8_t) port->portnum, &attr);
\r
137 port->base_lid = attr.lid;
\r
138 port->lmc = attr.lmc;
\r
139 port->sm_lid = attr.sm_lid;
\r
140 port->sm_sl = attr.sm_sl;
\r
141 port->state = attr.state;
\r
142 port->phys_state = attr.phys_state;
\r
143 port->rate = attr.active_speed;
\r
144 port->capmask = attr.port_cap_flags;
\r
146 // Assume GID 0 contains port GUID and gid prefix
\r
147 ret = ibv_query_gid(context, (uint8_t) port->portnum, 0, &gid);
\r
152 port->gid_prefix = gid.global.subnet_prefix;
\r
153 port->port_guid = gid.global.interface_id;
\r
155 port->pkeys_size = min(UMAD_MAX_PKEYS, attr.pkey_tbl_len);
\r
156 for (i = 0; i < (int) port->pkeys_size; i++) {
\r
157 ret = ibv_query_pkey(context,(uint8_t) port->portnum, i, &port->pkeys[i]);
\r
166 __declspec(dllexport)
\r
167 int umad_get_ca(char *ca_name, umad_ca_t *ca)
\r
169 struct ibv_device **list;
\r
170 struct ibv_context *context;
\r
171 ibv_device_attr dev_attr;
\r
172 int cnt, i, ret = 0;
\r
176 list = ibv_get_device_list(&cnt);
\r
177 if (list == NULL) {
\r
181 for (i = 0; i < cnt; i++) {
\r
182 if (!strcmp(ca_name, ibv_get_device_name(list[i]))) {
\r
192 context = ibv_open_device(list[i]);
\r
193 if (context == NULL) {
\r
198 ret = ibv_query_device(context, &dev_attr);
\r
203 port_size = sizeof(umad_port_t) + sizeof(uint16_t) * UMAD_MAX_PKEYS;
\r
204 ports = new uint8_t[port_size * dev_attr.phys_port_cnt];
\r
205 if (ports == NULL) {
\r
210 strcpy(ca->ca_name, ca_name);
\r
211 umad_convert_ca_attr(ca, &dev_attr);
\r
212 memset(ca->ports, 0, sizeof(ca->ports));
\r
214 for (i = 1; i <= dev_attr.phys_port_cnt; i++, ports += port_size) {
\r
216 ca->ports[i] = (umad_port_t *) ports;
\r
217 strcpy(ca->ports[i]->ca_name, ca_name);
\r
218 ca->ports[i]->portnum = i;
\r
219 ca->ports[i]->pkeys = (uint16_t *) (ports + sizeof(umad_port_t));
\r
221 ret = umad_query_port(context, ca->ports[i]);
\r
228 ibv_close_device(context);
\r
230 ibv_free_device_list(list);
\r
237 __declspec(dllexport)
\r
238 int umad_release_ca(umad_ca_t *ca)
\r
240 delete ca->ports[0];
\r
244 static uint64_t umad_get_ca_guid(char *ca_name)
\r
250 ret = umad_get_ca(ca_name, &ca);
\r
255 guid = ca.node_guid;
\r
256 umad_release_ca(&ca);
\r
260 __declspec(dllexport)
\r
261 int umad_get_port(char *ca_name, int portnum, umad_port_t *port)
\r
266 ret = umad_get_ca(ca_name, &ca);
\r
271 memcpy(port, ca.ports[portnum], sizeof(umad_port_t));
\r
273 port->pkeys = new uint16_t[ca.ports[portnum]->pkeys_size];
\r
274 if (port->pkeys == NULL) {
\r
279 memcpy(port->pkeys, ca.ports[portnum]->pkeys,
\r
280 sizeof(uint16_t) * ca.ports[portnum]->pkeys_size);
\r
282 umad_release_ca(&ca);
\r
286 __declspec(dllexport)
\r
287 int umad_release_port(umad_port_t *port)
\r
289 delete port->pkeys;
\r
293 __declspec(dllexport)
\r
294 int umad_get_issm_path(char *ca_name, int portnum, char path[], int max)
\r
299 static uint8_t umad_find_port(char *ca_name, enum ibv_port_state state)
\r
304 ret = umad_get_ca(ca_name, &ca);
\r
309 for (i = 1; i <= ca.numports; i++) {
\r
310 if (ca.ports[i]->state == state) {
\r
311 i = ca.ports[i]->portnum;
\r
312 umad_release_ca(&ca);
\r
313 return (uint8_t) i;
\r
317 umad_release_ca(&ca);
\r
321 static int umad_find_ca_port(enum ibv_port_state state, char *ca_name, uint8_t *port)
\r
323 char names[8][UMAD_CA_NAME_LEN];
\r
326 cnt = umad_get_cas_names(names, 8);
\r
328 for (i = 0; i < cnt; i++) {
\r
329 *port = umad_find_port(names[i], state);
\r
331 strcpy(ca_name, names[i]);
\r
338 static int umad_resolve_ca_port(char *ca_name, uint8_t *port)
\r
342 if (ca_name[0] != NULL) {
\r
347 *port = umad_find_port(ca_name, IBV_PORT_ACTIVE);
\r
351 *port = umad_find_port(ca_name, IBV_PORT_INIT);
\r
355 *port = umad_find_port(ca_name, IBV_PORT_DOWN);
\r
356 return (*port == 0);
\r
359 ret = umad_find_ca_port(IBV_PORT_ACTIVE, ca_name, port);
\r
363 ret = umad_find_ca_port(IBV_PORT_INIT, ca_name, port);
\r
367 ret = umad_find_ca_port(IBV_PORT_DOWN, ca_name, port);
\r
371 __declspec(dllexport)
\r
372 int umad_open_port(char *ca_name, int portnum)
\r
374 char name[UMAD_CA_NAME_LEN];
\r
379 if (ca_name != NULL) {
\r
380 strcpy(name, ca_name);
\r
381 port = (uint8_t) portnum;
\r
387 hr = umad_resolve_ca_port(name, &port);
\r
392 EnterCriticalSection(&crit_sec);
\r
393 for (portid = 0; portid < UMAD_MAX_PORTS; portid++) {
\r
394 if (ports[portid].prov == NULL) {
\r
399 if (portid == UMAD_MAX_PORTS) {
\r
404 ports[portid].overlap.hEvent = CreateEvent(NULL, FALSE, FALSE, NULL);
\r
405 if (ports[portid].overlap.hEvent == NULL) {
\r
410 hr = WmGetObject(IID_IWMProvider, (LPVOID*) &ports[portid].prov);
\r
412 CloseHandle(ports[portid].overlap.hEvent);
\r
413 portid = GetLastError() & 0x80000000;
\r
417 ports[portid].dev_guid = umad_get_ca_guid(name);
\r
418 ports[portid].port_num = port;
\r
421 LeaveCriticalSection(&crit_sec);
\r
425 __declspec(dllexport)
\r
426 int umad_close_port(int portid)
\r
428 CloseHandle(ports[portid].overlap.hEvent);
\r
429 ports[portid].prov->Release();
\r
431 EnterCriticalSection(&crit_sec);
\r
432 ports[portid].prov = NULL;
\r
433 LeaveCriticalSection(&crit_sec);
\r
438 __declspec(dllexport)
\r
439 int umad_set_grh_net(void *umad, void *mad_addr)
\r
441 struct ib_user_mad *mad = (struct ib_user_mad *) umad;
\r
442 struct ib_mad_addr *addr = (struct ib_mad_addr *) mad_addr;
\r
445 mad->addr.grh_present = 1;
\r
446 memcpy(mad->addr.gid, addr->gid, 16);
\r
447 mad->addr.flow_label = addr->flow_label;
\r
448 mad->addr.hop_limit = addr->hop_limit;
\r
449 mad->addr.traffic_class = addr->traffic_class;
\r
451 mad->addr.grh_present = 0;
\r
455 __declspec(dllexport)
\r
456 int umad_set_grh(void *umad, void *mad_addr)
\r
458 struct ib_user_mad *mad = (struct ib_user_mad *) umad;
\r
459 struct ib_mad_addr *addr = (struct ib_mad_addr *) mad_addr;
\r
462 mad->addr.grh_present = 1;
\r
463 memcpy(mad->addr.gid, addr->gid, 16);
\r
464 mad->addr.flow_label = htonl(addr->flow_label);
\r
465 mad->addr.hop_limit = addr->hop_limit;
\r
466 mad->addr.traffic_class = addr->traffic_class;
\r
468 mad->addr.grh_present = 0;
\r
472 __declspec(dllexport)
\r
473 int umad_set_pkey(void *umad, int pkey_index)
\r
475 struct ib_user_mad *mad = (struct ib_user_mad *) umad;
\r
477 mad->addr.pkey_index = (uint16_t) pkey_index;
\r
481 __declspec(dllexport)
\r
482 int umad_get_pkey(void *umad)
\r
484 struct ib_user_mad *mad = (struct ib_user_mad *) umad;
\r
486 return mad->addr.pkey_index;
\r
489 __declspec(dllexport)
\r
490 int umad_set_addr(void *umad, int dlid, int dqp, int sl, int qkey)
\r
492 struct ib_user_mad *mad = (struct ib_user_mad *) umad;
\r
494 mad->addr.qpn = htonl(dqp);
\r
495 mad->addr.lid = htons((uint16_t) dlid);
\r
496 mad->addr.qkey = htonl(qkey);
\r
497 mad->addr.sl = (uint8_t) sl;
\r
501 __declspec(dllexport)
\r
502 int umad_set_addr_net(void *umad, int dlid, int dqp, int sl, int qkey)
\r
504 struct ib_user_mad *mad = (struct ib_user_mad *) umad;
\r
506 mad->addr.qpn = dqp;
\r
507 mad->addr.lid = (uint16_t) dlid;
\r
508 mad->addr.qkey = qkey;
\r
509 mad->addr.sl = (uint8_t) sl;
\r
514 __declspec(dllexport)
\r
515 int umad_status(void *umad)
\r
517 return ((struct ib_user_mad *) umad)->status;
\r
520 __declspec(dllexport)
\r
521 ib_mad_addr_t *umad_get_mad_addr(void *umad)
\r
523 return &((struct ib_user_mad *) umad)->addr;
\r
526 __declspec(dllexport)
\r
527 void *umad_get_mad(void *umad)
\r
529 return ((struct ib_user_mad *)umad)->data;
\r
532 __declspec(dllexport)
\r
533 void *umad_alloc(int num, size_t size)
\r
535 return new uint8_t[num * size];
\r
538 __declspec(dllexport)
\r
539 void umad_free(void *umad)
\r
544 __declspec(dllexport)
\r
545 size_t umad_size(void)
\r
547 return sizeof(struct ib_user_mad);
\r
550 __declspec(dllexport)
\r
551 int umad_send(int portid, int agentid, void *umad, int length,
\r
552 int timeout_ms, int retries)
\r
554 struct ib_user_mad *mad = (struct ib_user_mad *) umad;
\r
557 mad->agent_id = agentid;
\r
558 mad->reserved_id = 0;
\r
560 mad->timeout_ms = (uint32_t) timeout_ms;
\r
561 mad->retries = (uint32_t) retries;
\r
562 mad->length = (uint32_t) length;
\r
564 hr = ports[portid].prov->Send((WM_MAD *) mad, NULL);
\r
566 return GetLastError();
\r
572 __declspec(dllexport)
\r
573 int umad_recv(int portid, void *umad, int *length, int timeout_ms)
\r
575 WM_MAD *mad = (WM_MAD *) umad;
\r
579 port = &ports[portid];
\r
580 hr = port->prov->Receive(mad, sizeof(WM_MAD) + (size_t) *length, &port->overlap);
\r
582 if (hr == WV_IO_PENDING) {
\r
583 if (port->pending && timeout_ms == 0) {
\r
585 hr = WaitForSingleObject(port->overlap.hEvent, 250);
\r
586 } while (hr == WAIT_TIMEOUT && port->pending);
\r
588 hr = WaitForSingleObject(port->overlap.hEvent, (DWORD) timeout_ms);
\r
589 if (hr == WAIT_TIMEOUT) {
\r
590 return -EWOULDBLOCK;
\r
599 if (mad->Length <= (UINT32) *length) {
\r
600 port->pending = FALSE;
\r
601 hr = (HRESULT) mad->Id;
\r
607 *length = mad->Length;
\r
611 __declspec(dllexport)
\r
612 int umad_poll(int portid, int timeout_ms)
\r
618 port = &ports[portid];
\r
619 hr = port->prov->Receive(&mad, sizeof mad, &port->overlap);
\r
621 if (hr == WV_IO_PENDING) {
\r
622 hr = WaitForSingleObject(port->overlap.hEvent, (DWORD) timeout_ms);
\r
623 if (hr == WAIT_TIMEOUT) {
\r
628 if (FAILED(hr) && hr != ERROR_MORE_DATA) {
\r
632 port->pending = TRUE;
\r
636 __declspec(dllexport)
\r
637 int umad_register_oui(int portid, int mgmt_class, uint8_t rmpp_version,
\r
638 uint8_t oui[3], long method_mask[16/sizeof(long)])
\r
643 UNREFERENCED_PARAMETER(rmpp_version);
\r
645 reg.Guid = ports[portid].dev_guid;
\r
646 reg.Qpn = (mgmt_class == 0x01 || mgmt_class == 0x81) ? 0 : htonl(1);
\r
647 reg.Port = ports[portid].port_num;
\r
648 reg.Class = (uint8_t) mgmt_class;
\r
650 memset(reg.Reserved, 0, sizeof(reg.Reserved));
\r
651 memcpy(reg.Oui, oui, sizeof(oui));
\r
652 if (method_mask != NULL) {
\r
653 memcpy(reg.Methods, method_mask, sizeof(reg.Methods));
\r
655 memset(reg.Methods, 0, sizeof(reg.Methods));
\r
657 ports[portid].prov->Register(®, &id);
\r
662 __declspec(dllexport)
\r
663 int umad_register(int portid, int mgmt_class, int mgmt_version,
\r
664 uint8_t rmpp_version, long method_mask[16/sizeof(long)])
\r
669 return umad_register_oui(portid, mgmt_class, rmpp_version, oui, method_mask);
\r
672 __declspec(dllexport)
\r
673 int umad_unregister(int portid, int agentid)
\r
675 ports[portid].pending = FALSE;
\r
676 return ports[portid].prov->Deregister((UINT64) agentid);
\r
679 HANDLE umad_get_fd(int portid)
\r
681 return ports[portid].prov->GetFileHandle();
\r
684 __declspec(dllexport)
\r
685 int umad_debug(int level)
\r
687 UNREFERENCED_PARAMETER(level);
\r
691 __declspec(dllexport)
\r
692 void umad_addr_dump(ib_mad_addr_t *addr)
\r
694 printf("umad_addr:\n");
\r
695 printf("\tqpn 0x%x\n", addr->qpn);
\r
696 printf("\tqkey 0x%x\n", addr->qkey);
\r
697 printf("\tsl 0x%x\n", addr->sl);
\r
698 printf("\tlid 0x%x\n", addr->lid);
\r
699 printf("\tpkey_index 0x%x\n", addr->pkey_index);
\r
700 printf("\tpath_bits 0x%x\n", addr->path_bits);
\r
701 printf("\trate 0x%x\n", addr->reserved_rate);
\r
703 printf("\tgrh_present 0x%x\n", addr->grh_present);
\r
704 if (addr->grh_present) {
\r
705 printf("\tgid_index 0x%x\n", addr->gid_index);
\r
706 printf("\tgid 0x%x %x\n",
\r
707 (uint64_t) addr->gid, (uint64_t) (addr->gid + 8));
\r
708 printf("\tflow_lable 0x%x\n", addr->flow_label);
\r
709 printf("\thop_limit 0x%x\n", addr->hop_limit);
\r
710 printf("\ttraffic_class 0x%x\n", addr->qpn);
\r
714 __declspec(dllexport)
\r
715 void umad_dump(void *umad)
\r
717 struct ib_user_mad *mad = (struct ib_user_mad *) umad;
\r
720 umad_addr_dump(&mad->addr);
\r
721 printf("umad_data\n");
\r
722 printf("offset: hex data\n");
\r
723 for (i = 0; i < 256; i += 4) {
\r
724 printf("%03d: ", i);
\r
725 printf("%02x %02x %02x %02x\n", mad->data[i], mad->data[i + 1],
\r
726 mad->data[i + 2], mad->data[i + 3]);
\r