2 * Copyright (c) 2004-2007 Voltaire Inc. All rights reserved.
\r
3 * Copyright (c) 2008 Intel Corp., Inc. All rights reserved.
\r
5 * This software is available to you under a choice of one of two
\r
6 * licenses. You may choose to be licensed under the terms of the GNU
\r
7 * General Public License (GPL) Version 2, available from the file
\r
8 * COPYING in the main directory of this source tree, or the
\r
9 * OpenIB.org BSD license below:
\r
11 * Redistribution and use in source and binary forms, with or
\r
12 * without modification, are permitted provided that the following
\r
13 * conditions are met:
\r
15 * - Redistributions of source code must retain the above
\r
16 * copyright notice, this list of conditions and the following
\r
19 * - Redistributions in binary form must reproduce the above
\r
20 * copyright notice, this list of conditions and the following
\r
21 * disclaimer in the documentation and/or other materials
\r
22 * provided with the distribution.
\r
24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
\r
25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
\r
26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
\r
27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
\r
28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
\r
29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
\r
30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
\r
35 #include <windows.h>
\r
38 #include <infiniband/umad.h>
\r
39 #include <infiniband/verbs.h>
\r
40 #include <rdma/wvstatus.h>
\r
43 #define IB_OPENIB_OUI (0x001405)
\r
45 #define UMAD_MAX_PKEYS 16
\r
47 typedef struct um_port
\r
57 CRITICAL_SECTION crit_sec;
\r
58 um_port_t ports[UMAD_MAX_PORTS];
\r
61 __declspec(dllexport)
\r
64 InitializeCriticalSection(&crit_sec);
\r
68 __declspec(dllexport)
\r
74 __declspec(dllexport)
\r
75 int umad_get_cas_names(char cas[][UMAD_CA_NAME_LEN], int max)
\r
77 struct ibv_device **list;
\r
80 list = ibv_get_device_list(&cnt);
\r
85 for (i = 0; i < min(cnt, max); i++) {
\r
86 strcpy(cas[i], ibv_get_device_name(list[i]));
\r
89 ibv_free_device_list(list);
\r
93 __declspec(dllexport)
\r
94 int umad_get_ca_portguids(char *ca_name, uint64_t *portguids, int max)
\r
99 if (umad_get_ca(ca_name, &ca) < 0)
\r
102 if (ca.numports + 1 > max) {
\r
103 umad_release_ca(&ca);
\r
107 for (i = 0; i <= ca.numports; i++)
\r
108 portguids[ports++] = ca.ports[i]->port_guid;
\r
110 umad_release_ca(&ca);
\r
114 static void umad_convert_ca_attr(umad_ca_t *ca, ibv_device_attr *attr)
\r
116 ca->node_type = 1; // HCA
\r
117 ca->numports = attr->phys_port_cnt;
\r
118 strncpy(ca->fw_ver, attr->fw_ver, 20);
\r
119 memset(ca->ca_type, 0, 40); // TODO: determine what this should be
\r
120 sprintf(ca->hw_ver, "0x%x", attr->hw_ver);
\r
121 ca->node_guid = attr->node_guid;
\r
122 ca->system_guid = attr->sys_image_guid;
\r
125 static int umad_query_port(struct ibv_context *context, umad_port_t *port)
\r
127 ibv_port_attr attr;
\r
131 ret = ibv_query_port(context, (uint8_t) port->portnum, &attr);
\r
136 port->base_lid = attr.lid;
\r
137 port->lmc = attr.lmc;
\r
138 port->sm_lid = attr.sm_lid;
\r
139 port->sm_sl = attr.sm_sl;
\r
140 port->state = attr.state;
\r
141 port->phys_state = attr.phys_state;
\r
142 port->rate = attr.active_speed;
\r
143 port->capmask = attr.port_cap_flags;
\r
145 // Assume GID 0 contains port GUID and gid prefix
\r
146 ret = ibv_query_gid(context, (uint8_t) port->portnum, 0, &gid);
\r
151 port->gid_prefix = gid.global.subnet_prefix;
\r
152 port->port_guid = gid.global.interface_id;
\r
154 port->pkeys_size = min(UMAD_MAX_PKEYS, attr.pkey_tbl_len);
\r
155 for (i = 0; i < (int) port->pkeys_size; i++) {
\r
156 ret = ibv_query_pkey(context,(uint8_t) port->portnum, i, &port->pkeys[i]);
\r
165 __declspec(dllexport)
\r
166 int umad_get_ca(char *ca_name, umad_ca_t *ca)
\r
168 struct ibv_device **list;
\r
169 struct ibv_context *context;
\r
170 ibv_device_attr dev_attr;
\r
171 int cnt, i, ret = 0;
\r
175 list = ibv_get_device_list(&cnt);
\r
176 if (list == NULL) {
\r
180 for (i = 0; i < cnt; i++) {
\r
181 if (!strcmp(ca_name, ibv_get_device_name(list[i]))) {
\r
191 context = ibv_open_device(list[i]);
\r
192 if (context == NULL) {
\r
197 ret = ibv_query_device(context, &dev_attr);
\r
202 port_size = sizeof(umad_port_t) + sizeof(uint16_t) * UMAD_MAX_PKEYS;
\r
203 ports = new uint8_t[port_size * dev_attr.phys_port_cnt];
\r
204 if (ports == NULL) {
\r
209 strcpy(ca->ca_name, ca_name);
\r
210 umad_convert_ca_attr(ca, &dev_attr);
\r
212 for (i = 0; i < dev_attr.phys_port_cnt; i++, ports += port_size) {
\r
214 ca->ports[i] = (umad_port_t *) ports;
\r
215 strcpy(ca->ports[i]->ca_name, ca_name);
\r
216 ca->ports[i]->portnum = i + 1;
\r
217 ca->ports[i]->pkeys = (uint16_t *) (ports + sizeof(umad_port_t));
\r
219 ret = umad_query_port(context, ca->ports[i]);
\r
226 ibv_close_device(context);
\r
228 ibv_free_device_list(list);
\r
235 __declspec(dllexport)
\r
236 int umad_release_ca(umad_ca_t *ca)
\r
238 delete ca->ports[0];
\r
242 static uint64_t umad_get_ca_guid(char *ca_name)
\r
248 ret = umad_get_ca(ca_name, &ca);
\r
253 guid = ca.node_guid;
\r
254 umad_release_ca(&ca);
\r
258 __declspec(dllexport)
\r
259 int umad_get_port(char *ca_name, int portnum, umad_port_t *port)
\r
264 ret = umad_get_ca(ca_name, &ca);
\r
269 memcpy(port, ca.ports[portnum - 1], sizeof(umad_port_t));
\r
271 port->pkeys = new uint16_t[ca.ports[portnum - 1]->pkeys_size];
\r
272 if (port->pkeys == NULL) {
\r
277 memcpy(port->pkeys, ca.ports[portnum - 1]->pkeys,
\r
278 sizeof(uint16_t) * ca.ports[portnum - 1]->pkeys_size);
\r
280 umad_release_ca(&ca);
\r
284 __declspec(dllexport)
\r
285 int umad_release_port(umad_port_t *port)
\r
287 delete port->pkeys;
\r
291 __declspec(dllexport)
\r
292 int umad_get_issm_path(char *ca_name, int portnum, char path[], int max)
\r
297 static uint8_t umad_find_port(char *ca_name, enum ibv_port_state state)
\r
302 ret = umad_get_ca(ca_name, &ca);
\r
307 for (i = 0; i < ca.numports; i++) {
\r
308 if (ca.ports[i]->state == state) {
\r
309 i = ca.ports[i]->portnum;
\r
310 umad_release_ca(&ca);
\r
311 return (uint8_t) i;
\r
315 umad_release_ca(&ca);
\r
319 static int umad_find_ca_port(enum ibv_port_state state, char *ca_name, uint8_t *port)
\r
321 char names[8][UMAD_CA_NAME_LEN];
\r
324 cnt = umad_get_cas_names(names, 8);
\r
326 for (i = 0; i < cnt; i++) {
\r
327 *port = umad_find_port(names[i], state);
\r
329 strcpy(ca_name, names[i]);
\r
336 static int umad_resolve_ca_port(char *ca_name, uint8_t *port)
\r
340 if (ca_name[0] != NULL) {
\r
345 *port = umad_find_port(ca_name, IBV_PORT_ACTIVE);
\r
349 *port = umad_find_port(ca_name, IBV_PORT_INIT);
\r
353 *port = umad_find_port(ca_name, IBV_PORT_DOWN);
\r
354 return (*port == 0);
\r
357 ret = umad_find_ca_port(IBV_PORT_ACTIVE, ca_name, port);
\r
361 ret = umad_find_ca_port(IBV_PORT_INIT, ca_name, port);
\r
365 ret = umad_find_ca_port(IBV_PORT_DOWN, ca_name, port);
\r
369 __declspec(dllexport)
\r
370 int umad_open_port(char *ca_name, int portnum)
\r
372 char name[UMAD_CA_NAME_LEN];
\r
377 if (ca_name != NULL) {
\r
378 strcpy(name, ca_name);
\r
379 port = (uint8_t) portnum;
\r
385 hr = umad_resolve_ca_port(name, &port);
\r
390 EnterCriticalSection(&crit_sec);
\r
391 for (portid = 0; portid < UMAD_MAX_PORTS; portid++) {
\r
392 if (ports[portid].prov == NULL) {
\r
397 if (portid == UMAD_MAX_PORTS) {
\r
402 ports[portid].overlap.hEvent = CreateEvent(NULL, FALSE, FALSE, NULL);
\r
403 if (ports[portid].overlap.hEvent == NULL) {
\r
408 hr = WmGetObject(IID_IWMProvider, (LPVOID*) &ports[portid].prov);
\r
410 CloseHandle(ports[portid].overlap.hEvent);
\r
411 portid = GetLastError() & 0x80000000;
\r
415 ports[portid].dev_guid = umad_get_ca_guid(name);
\r
416 ports[portid].port_num = port;
\r
419 LeaveCriticalSection(&crit_sec);
\r
423 __declspec(dllexport)
\r
424 int umad_close_port(int portid)
\r
426 CloseHandle(ports[portid].overlap.hEvent);
\r
427 ports[portid].prov->Release();
\r
429 EnterCriticalSection(&crit_sec);
\r
430 ports[portid].prov = NULL;
\r
431 LeaveCriticalSection(&crit_sec);
\r
436 __declspec(dllexport)
\r
437 int umad_set_grh_net(void *umad, void *mad_addr)
\r
439 struct ib_user_mad *mad = (struct ib_user_mad *) umad;
\r
440 struct ib_mad_addr *addr = (struct ib_mad_addr *) mad_addr;
\r
443 mad->addr.grh_present = 1;
\r
444 memcpy(mad->addr.gid, addr->gid, 16);
\r
445 mad->addr.flow_label = addr->flow_label;
\r
446 mad->addr.hop_limit = addr->hop_limit;
\r
447 mad->addr.traffic_class = addr->traffic_class;
\r
449 mad->addr.grh_present = 0;
\r
453 __declspec(dllexport)
\r
454 int umad_set_grh(void *umad, void *mad_addr)
\r
456 struct ib_user_mad *mad = (struct ib_user_mad *) umad;
\r
457 struct ib_mad_addr *addr = (struct ib_mad_addr *) mad_addr;
\r
460 mad->addr.grh_present = 1;
\r
461 memcpy(mad->addr.gid, addr->gid, 16);
\r
462 mad->addr.flow_label = htonl(addr->flow_label);
\r
463 mad->addr.hop_limit = addr->hop_limit;
\r
464 mad->addr.traffic_class = addr->traffic_class;
\r
466 mad->addr.grh_present = 0;
\r
470 __declspec(dllexport)
\r
471 int umad_set_pkey(void *umad, int pkey_index)
\r
473 struct ib_user_mad *mad = (struct ib_user_mad *) umad;
\r
475 mad->addr.pkey_index = (uint16_t) pkey_index;
\r
479 __declspec(dllexport)
\r
480 int umad_get_pkey(void *umad)
\r
482 struct ib_user_mad *mad = (struct ib_user_mad *) umad;
\r
484 return mad->addr.pkey_index;
\r
487 __declspec(dllexport)
\r
488 int umad_set_addr(void *umad, int dlid, int dqp, int sl, int qkey)
\r
490 struct ib_user_mad *mad = (struct ib_user_mad *) umad;
\r
492 mad->addr.qpn = htonl(dqp);
\r
493 mad->addr.lid = htons((uint16_t) dlid);
\r
494 mad->addr.qkey = htonl(qkey);
\r
495 mad->addr.sl = (uint8_t) sl;
\r
499 __declspec(dllexport)
\r
500 int umad_set_addr_net(void *umad, int dlid, int dqp, int sl, int qkey)
\r
502 struct ib_user_mad *mad = (struct ib_user_mad *) umad;
\r
504 mad->addr.qpn = dqp;
\r
505 mad->addr.lid = (uint16_t) dlid;
\r
506 mad->addr.qkey = qkey;
\r
507 mad->addr.sl = (uint8_t) sl;
\r
512 __declspec(dllexport)
\r
513 int umad_status(void *umad)
\r
515 return ((struct ib_user_mad *) umad)->status;
\r
518 __declspec(dllexport)
\r
519 ib_mad_addr_t *umad_get_mad_addr(void *umad)
\r
521 return &((struct ib_user_mad *) umad)->addr;
\r
524 __declspec(dllexport)
\r
525 void *umad_get_mad(void *umad)
\r
527 return ((struct ib_user_mad *)umad)->data;
\r
530 __declspec(dllexport)
\r
531 void *umad_alloc(int num, size_t size)
\r
533 return new uint8_t[num * size];
\r
536 __declspec(dllexport)
\r
537 void umad_free(void *umad)
\r
542 __declspec(dllexport)
\r
543 size_t umad_size(void)
\r
545 return sizeof(struct ib_user_mad);
\r
548 __declspec(dllexport)
\r
549 int umad_send(int portid, int agentid, void *umad, int length,
\r
550 int timeout_ms, int retries)
\r
552 struct ib_user_mad *mad = (struct ib_user_mad *) umad;
\r
555 mad->agent_id = agentid;
\r
556 mad->reserved_id = 0;
\r
558 mad->timeout_ms = (uint32_t) timeout_ms;
\r
559 mad->retries = (uint32_t) retries;
\r
560 mad->length = (uint32_t) length;
\r
562 hr = ports[portid].prov->Send((WM_MAD *) mad, NULL);
\r
564 return GetLastError();
\r
570 __declspec(dllexport)
\r
571 int umad_recv(int portid, void *umad, int *length, int timeout_ms)
\r
573 WM_MAD *mad = (WM_MAD *) umad;
\r
577 port = &ports[portid];
\r
578 hr = port->prov->Receive(mad, sizeof(WM_MAD) + (size_t) *length, &port->overlap);
\r
580 if (hr == WV_IO_PENDING) {
\r
581 if (port->pending && timeout_ms == 0) {
\r
583 hr = WaitForSingleObject(port->overlap.hEvent, 250);
\r
584 } while (hr == WAIT_TIMEOUT && port->pending);
\r
586 hr = WaitForSingleObject(port->overlap.hEvent, (DWORD) timeout_ms);
\r
587 if (hr == WAIT_TIMEOUT) {
\r
588 return -EWOULDBLOCK;
\r
597 if (mad->Length <= (UINT32) *length) {
\r
598 port->pending = FALSE;
\r
599 hr = (HRESULT) mad->Id;
\r
605 *length = mad->Length;
\r
609 __declspec(dllexport)
\r
610 int umad_poll(int portid, int timeout_ms)
\r
616 port = &ports[portid];
\r
617 hr = port->prov->Receive(&mad, sizeof mad, &port->overlap);
\r
619 if (hr == WV_IO_PENDING) {
\r
620 hr = WaitForSingleObject(port->overlap.hEvent, (DWORD) timeout_ms);
\r
621 if (hr == WAIT_TIMEOUT) {
\r
626 if (FAILED(hr) && hr != ERROR_MORE_DATA) {
\r
630 port->pending = TRUE;
\r
634 __declspec(dllexport)
\r
635 int umad_register_oui(int portid, int mgmt_class, uint8_t rmpp_version,
\r
636 uint8_t oui[3], long method_mask[16/sizeof(long)])
\r
641 UNREFERENCED_PARAMETER(rmpp_version);
\r
643 reg.Guid = ports[portid].dev_guid;
\r
644 reg.Qpn = (mgmt_class == 0x01 || mgmt_class == 0x81) ? 0 : htonl(1);
\r
645 reg.Port = ports[portid].port_num;
\r
646 reg.Class = (uint8_t) mgmt_class;
\r
648 memset(reg.Reserved, 0, sizeof(reg.Reserved));
\r
649 memcpy(reg.Oui, oui, sizeof(oui));
\r
650 if (method_mask != NULL) {
\r
651 memcpy(reg.Methods, method_mask, sizeof(reg.Methods));
\r
653 memset(reg.Methods, 0, sizeof(reg.Methods));
\r
655 ports[portid].prov->Register(®, &id);
\r
660 __declspec(dllexport)
\r
661 int umad_register(int portid, int mgmt_class, int mgmt_version,
\r
662 uint8_t rmpp_version, long method_mask[16/sizeof(long)])
\r
667 return umad_register_oui(portid, mgmt_class, rmpp_version, oui, method_mask);
\r
670 __declspec(dllexport)
\r
671 int umad_unregister(int portid, int agentid)
\r
673 ports[portid].pending = FALSE;
\r
674 return ports[portid].prov->Deregister((UINT64) agentid);
\r
677 HANDLE umad_get_fd(int portid)
\r
679 return ports[portid].prov->GetFileHandle();
\r
682 __declspec(dllexport)
\r
683 int umad_debug(int level)
\r
685 UNREFERENCED_PARAMETER(level);
\r
689 __declspec(dllexport)
\r
690 void umad_addr_dump(ib_mad_addr_t *addr)
\r
692 UNREFERENCED_PARAMETER(addr);
\r
695 __declspec(dllexport)
\r
696 void umad_dump(void *umad)
\r
698 UNREFERENCED_PARAMETER(umad);
\r