librdmacm: provide OFED compatibility library.
[mirror/winof/.git] / ulp / librdmacm / examples / mckey / mckey.c
1 /*\r
2  * Copyright (c) 2005-2007 Intel Corporation.  All rights reserved.\r
3  *\r
4  * This software is available to you under a choice of one of two\r
5  * licenses.  You may choose to be licensed under the terms of the GNU\r
6  * General Public License (GPL) Version 2, available from the file\r
7  * COPYING in the main directory of this source tree, or the\r
8  * OpenIB.org BSD license below:\r
9  *\r
10  *     Redistribution and use in source and binary forms, with or\r
11  *     without modification, are permitted provided that the following\r
12  *     conditions are met:\r
13  *\r
14  *      - Redistributions of source code must retain the above\r
15  *        copyright notice, this list of conditions and the following\r
16  *        disclaimer.\r
17  *\r
18  *      - Redistributions in binary form must reproduce the above\r
19  *        copyright notice, this list of conditions and the following\r
20  *        disclaimer in the documentation and/or other materials\r
21  *        provided with the distribution.\r
22  *\r
23  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,\r
24  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF\r
25  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND\r
26  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS\r
27  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN\r
28  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN\r
29  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\r
30  * SOFTWARE.\r
31  *\r
32  * $Id$\r
33  */\r
34 \r
35 #include <stdlib.h>\r
36 #include <string.h>\r
37 #include <stdio.h>\r
38 #include <errno.h>\r
39 #include <sys/types.h>\r
40 #include <netinet/in.h>\r
41 #include <arpa/inet.h>\r
42 #include <sys/socket.h>\r
43 #include <netdb.h>\r
44 #include <byteswap.h>\r
45 #include <unistd.h>\r
46 #include <getopt.h>\r
47 \r
48 #include <rdma/rdma_cma.h>\r
49 \r
50 struct cmatest_node {\r
51         int                     id;\r
52         struct rdma_cm_id       *cma_id;\r
53         int                     connected;\r
54         struct ibv_pd           *pd;\r
55         struct ibv_cq           *cq;\r
56         struct ibv_mr           *mr;\r
57         struct ibv_ah           *ah;\r
58         uint32_t                remote_qpn;\r
59         uint32_t                remote_qkey;\r
60         void                    *mem;\r
61 };\r
62 \r
63 struct cmatest {\r
64         struct rdma_event_channel *channel;\r
65         struct cmatest_node     *nodes;\r
66         int                     conn_index;\r
67         int                     connects_left;\r
68 \r
69         struct sockaddr_in6     dst_in;\r
70         struct sockaddr         *dst_addr;\r
71         struct sockaddr_in6     src_in;\r
72         struct sockaddr         *src_addr;\r
73 };\r
74 \r
75 static struct cmatest test;\r
76 static int connections = 1;\r
77 static int message_size = 100;\r
78 static int message_count = 10;\r
79 static int is_sender;\r
80 static int unmapped_addr;\r
81 static char *dst_addr;\r
82 static char *src_addr;\r
83 static enum rdma_port_space port_space = RDMA_PS_UDP;\r
84 \r
85 static int create_message(struct cmatest_node *node)\r
86 {\r
87         if (!message_size)\r
88                 message_count = 0;\r
89 \r
90         if (!message_count)\r
91                 return 0;\r
92 \r
93         node->mem = malloc(message_size + sizeof(struct ibv_grh));\r
94         if (!node->mem) {\r
95                 printf("failed message allocation\n");\r
96                 return -1;\r
97         }\r
98         node->mr = ibv_reg_mr(node->pd, node->mem,\r
99                               message_size + sizeof(struct ibv_grh),\r
100                               IBV_ACCESS_LOCAL_WRITE);\r
101         if (!node->mr) {\r
102                 printf("failed to reg MR\n");\r
103                 goto err;\r
104         }\r
105         return 0;\r
106 err:\r
107         free(node->mem);\r
108         return -1;\r
109 }\r
110 \r
111 static int verify_test_params(struct cmatest_node *node)\r
112 {\r
113         struct ibv_port_attr port_attr;\r
114         int ret;\r
115 \r
116         ret = ibv_query_port(node->cma_id->verbs, node->cma_id->port_num,\r
117                              &port_attr);\r
118         if (ret)\r
119                 return ret;\r
120 \r
121         if (message_count && message_size > (1 << (port_attr.active_mtu + 7))) {\r
122                 printf("mckey: message_size %d is larger than active mtu %d\n",\r
123                        message_size, 1 << (port_attr.active_mtu + 7));\r
124                 return -EINVAL;\r
125         }\r
126 \r
127         return 0;\r
128 }\r
129 \r
130 static int init_node(struct cmatest_node *node)\r
131 {\r
132         struct ibv_qp_init_attr init_qp_attr;\r
133         int cqe, ret;\r
134 \r
135         node->pd = ibv_alloc_pd(node->cma_id->verbs);\r
136         if (!node->pd) {\r
137                 ret = -ENOMEM;\r
138                 printf("mckey: unable to allocate PD\n");\r
139                 goto out;\r
140         }\r
141 \r
142         cqe = message_count ? message_count * 2 : 2;\r
143         node->cq = ibv_create_cq(node->cma_id->verbs, cqe, node, 0, 0);\r
144         if (!node->cq) {\r
145                 ret = -ENOMEM;\r
146                 printf("mckey: unable to create CQ\n");\r
147                 goto out;\r
148         }\r
149 \r
150         memset(&init_qp_attr, 0, sizeof init_qp_attr);\r
151         init_qp_attr.cap.max_send_wr = message_count ? message_count : 1;\r
152         init_qp_attr.cap.max_recv_wr = message_count ? message_count : 1;\r
153         init_qp_attr.cap.max_send_sge = 1;\r
154         init_qp_attr.cap.max_recv_sge = 1;\r
155         init_qp_attr.qp_context = node;\r
156         init_qp_attr.sq_sig_all = 0;\r
157         init_qp_attr.qp_type = IBV_QPT_UD;\r
158         init_qp_attr.send_cq = node->cq;\r
159         init_qp_attr.recv_cq = node->cq;\r
160         ret = rdma_create_qp(node->cma_id, node->pd, &init_qp_attr);\r
161         if (ret) {\r
162                 printf("mckey: unable to create QP: %d\n", ret);\r
163                 goto out;\r
164         }\r
165 \r
166         ret = create_message(node);\r
167         if (ret) {\r
168                 printf("mckey: failed to create messages: %d\n", ret);\r
169                 goto out;\r
170         }\r
171 out:\r
172         return ret;\r
173 }\r
174 \r
175 static int post_recvs(struct cmatest_node *node)\r
176 {\r
177         struct ibv_recv_wr recv_wr, *recv_failure;\r
178         struct ibv_sge sge;\r
179         int i, ret = 0;\r
180 \r
181         if (!message_count)\r
182                 return 0;\r
183 \r
184         recv_wr.next = NULL;\r
185         recv_wr.sg_list = &sge;\r
186         recv_wr.num_sge = 1;\r
187         recv_wr.wr_id = (uintptr_t) node;\r
188 \r
189         sge.length = message_size + sizeof(struct ibv_grh);\r
190         sge.lkey = node->mr->lkey;\r
191         sge.addr = (uintptr_t) node->mem;\r
192 \r
193         for (i = 0; i < message_count && !ret; i++ ) {\r
194                 ret = ibv_post_recv(node->cma_id->qp, &recv_wr, &recv_failure);\r
195                 if (ret) {\r
196                         printf("failed to post receives: %d\n", ret);\r
197                         break;\r
198                 }\r
199         }\r
200         return ret;\r
201 }\r
202 \r
203 static int post_sends(struct cmatest_node *node, int signal_flag)\r
204 {\r
205         struct ibv_send_wr send_wr, *bad_send_wr;\r
206         struct ibv_sge sge;\r
207         int i, ret = 0;\r
208 \r
209         if (!node->connected || !message_count)\r
210                 return 0;\r
211 \r
212         send_wr.next = NULL;\r
213         send_wr.sg_list = &sge;\r
214         send_wr.num_sge = 1;\r
215         send_wr.opcode = IBV_WR_SEND_WITH_IMM;\r
216         send_wr.send_flags = signal_flag;\r
217         send_wr.wr_id = (unsigned long)node;\r
218         send_wr.imm_data = htonl(node->cma_id->qp->qp_num);\r
219 \r
220         send_wr.wr.ud.ah = node->ah;\r
221         send_wr.wr.ud.remote_qpn = node->remote_qpn;\r
222         send_wr.wr.ud.remote_qkey = node->remote_qkey;\r
223 \r
224         sge.length = message_size;\r
225         sge.lkey = node->mr->lkey;\r
226         sge.addr = (uintptr_t) node->mem;\r
227 \r
228         for (i = 0; i < message_count && !ret; i++) {\r
229                 ret = ibv_post_send(node->cma_id->qp, &send_wr, &bad_send_wr);\r
230                 if (ret)\r
231                         printf("failed to post sends: %d\n", ret);\r
232         }\r
233         return ret;\r
234 }\r
235 \r
236 static void connect_error(void)\r
237 {\r
238         test.connects_left--;\r
239 }\r
240 \r
241 static int addr_handler(struct cmatest_node *node)\r
242 {\r
243         int ret;\r
244 \r
245         ret = verify_test_params(node);\r
246         if (ret)\r
247                 goto err;\r
248 \r
249         ret = init_node(node);\r
250         if (ret)\r
251                 goto err;\r
252 \r
253         if (!is_sender) {\r
254                 ret = post_recvs(node);\r
255                 if (ret)\r
256                         goto err;\r
257         }\r
258 \r
259         ret = rdma_join_multicast(node->cma_id, test.dst_addr, node);\r
260         if (ret) {\r
261                 printf("mckey: failure joining: %d\n", ret);\r
262                 goto err;\r
263         }\r
264         return 0;\r
265 err:\r
266         connect_error();\r
267         return ret;\r
268 }\r
269 \r
270 static int join_handler(struct cmatest_node *node,\r
271                         struct rdma_ud_param *param)\r
272 {\r
273         char buf[40];\r
274 \r
275         inet_ntop(AF_INET6, param->ah_attr.grh.dgid.raw, buf, 40);\r
276         printf("mckey: joined dgid: %s\n", buf);\r
277 \r
278         node->remote_qpn = param->qp_num;\r
279         node->remote_qkey = param->qkey;\r
280         node->ah = ibv_create_ah(node->pd, &param->ah_attr);\r
281         if (!node->ah) {\r
282                 printf("mckey: failure creating address handle\n");\r
283                 goto err;\r
284         }\r
285 \r
286         node->connected = 1;\r
287         test.connects_left--;\r
288         return 0;\r
289 err:\r
290         connect_error();\r
291         return -1;\r
292 }\r
293 \r
294 static int cma_handler(struct rdma_cm_id *cma_id, struct rdma_cm_event *event)\r
295 {\r
296         int ret = 0;\r
297 \r
298         switch (event->event) {\r
299         case RDMA_CM_EVENT_ADDR_RESOLVED:\r
300                 ret = addr_handler(cma_id->context);\r
301                 break;\r
302         case RDMA_CM_EVENT_MULTICAST_JOIN:\r
303                 ret = join_handler(cma_id->context, &event->param.ud);\r
304                 break;\r
305         case RDMA_CM_EVENT_ADDR_ERROR:\r
306         case RDMA_CM_EVENT_ROUTE_ERROR:\r
307         case RDMA_CM_EVENT_MULTICAST_ERROR:\r
308                 printf("mckey: event: %s, error: %d\n",\r
309                        rdma_event_str(event->event), event->status);\r
310                 connect_error();\r
311                 ret = event->status;\r
312                 break;\r
313         case RDMA_CM_EVENT_DEVICE_REMOVAL:\r
314                 /* Cleanup will occur after test completes. */\r
315                 break;\r
316         default:\r
317                 break;\r
318         }\r
319         return ret;\r
320 }\r
321 \r
322 static void destroy_node(struct cmatest_node *node)\r
323 {\r
324         if (!node->cma_id)\r
325                 return;\r
326 \r
327         if (node->ah)\r
328                 ibv_destroy_ah(node->ah);\r
329 \r
330         if (node->cma_id->qp)\r
331                 rdma_destroy_qp(node->cma_id);\r
332 \r
333         if (node->cq)\r
334                 ibv_destroy_cq(node->cq);\r
335 \r
336         if (node->mem) {\r
337                 ibv_dereg_mr(node->mr);\r
338                 free(node->mem);\r
339         }\r
340 \r
341         if (node->pd)\r
342                 ibv_dealloc_pd(node->pd);\r
343 \r
344         /* Destroy the RDMA ID after all device resources */\r
345         rdma_destroy_id(node->cma_id);\r
346 }\r
347 \r
348 static int alloc_nodes(void)\r
349 {\r
350         int ret, i;\r
351 \r
352         test.nodes = malloc(sizeof *test.nodes * connections);\r
353         if (!test.nodes) {\r
354                 printf("mckey: unable to allocate memory for test nodes\n");\r
355                 return -ENOMEM;\r
356         }\r
357         memset(test.nodes, 0, sizeof *test.nodes * connections);\r
358 \r
359         for (i = 0; i < connections; i++) {\r
360                 test.nodes[i].id = i;\r
361                 ret = rdma_create_id(test.channel, &test.nodes[i].cma_id,\r
362                                      &test.nodes[i], port_space);\r
363                 if (ret)\r
364                         goto err;\r
365         }\r
366         return 0;\r
367 err:\r
368         while (--i >= 0)\r
369                 rdma_destroy_id(test.nodes[i].cma_id);\r
370         free(test.nodes);\r
371         return ret;\r
372 }\r
373 \r
374 static void destroy_nodes(void)\r
375 {\r
376         int i;\r
377 \r
378         for (i = 0; i < connections; i++)\r
379                 destroy_node(&test.nodes[i]);\r
380         free(test.nodes);\r
381 }\r
382 \r
383 static int poll_cqs(void)\r
384 {\r
385         struct ibv_wc wc[8];\r
386         int done, i, ret;\r
387 \r
388         for (i = 0; i < connections; i++) {\r
389                 if (!test.nodes[i].connected)\r
390                         continue;\r
391 \r
392                 for (done = 0; done < message_count; done += ret) {\r
393                         ret = ibv_poll_cq(test.nodes[i].cq, 8, wc);\r
394                         if (ret < 0) {\r
395                                 printf("mckey: failed polling CQ: %d\n", ret);\r
396                                 return ret;\r
397                         }\r
398                 }\r
399         }\r
400         return 0;\r
401 }\r
402 \r
403 static int connect_events(void)\r
404 {\r
405         struct rdma_cm_event *event;\r
406         int ret = 0;\r
407 \r
408         while (test.connects_left && !ret) {\r
409                 ret = rdma_get_cm_event(test.channel, &event);\r
410                 if (!ret) {\r
411                         ret = cma_handler(event->id, event);\r
412                         rdma_ack_cm_event(event);\r
413                 }\r
414         }\r
415         return ret;\r
416 }\r
417 \r
418 static int get_addr(char *dst, struct sockaddr *addr)\r
419 {\r
420         struct addrinfo *res;\r
421         int ret;\r
422 \r
423         ret = getaddrinfo(dst, NULL, NULL, &res);\r
424         if (ret) {\r
425                 printf("getaddrinfo failed - invalid hostname or IP address\n");\r
426                 return ret;\r
427         }\r
428 \r
429         memcpy(addr, res->ai_addr, res->ai_addrlen);\r
430         freeaddrinfo(res);\r
431         return ret;\r
432 }\r
433 \r
434 static int run(void)\r
435 {\r
436         int i, ret;\r
437 \r
438         printf("mckey: starting %s\n", is_sender ? "client" : "server");\r
439         if (src_addr) {\r
440                 ret = get_addr(src_addr, (struct sockaddr *) &test.src_in);\r
441                 if (ret)\r
442                         return ret;\r
443         }\r
444 \r
445         ret = get_addr(dst_addr, (struct sockaddr *) &test.dst_in);\r
446         if (ret)\r
447                 return ret;\r
448 \r
449         printf("mckey: joining\n");\r
450         for (i = 0; i < connections; i++) {\r
451                 if (src_addr) {\r
452                         ret = rdma_bind_addr(test.nodes[i].cma_id,\r
453                                              test.src_addr);\r
454                         if (ret) {\r
455                                 printf("mckey: addr bind failure: %d\n", ret);\r
456                                 connect_error();\r
457                                 return ret;\r
458                         }\r
459                 }\r
460 \r
461                 if (unmapped_addr)\r
462                         ret = addr_handler(&test.nodes[i]);\r
463                 else\r
464                         ret = rdma_resolve_addr(test.nodes[i].cma_id,\r
465                                                 test.src_addr, test.dst_addr,\r
466                                                 2000);\r
467                 if (ret) {\r
468                         printf("mckey: resolve addr failure: %d\n", ret);\r
469                         connect_error();\r
470                         return ret;\r
471                 }\r
472         }\r
473 \r
474         ret = connect_events();\r
475         if (ret)\r
476                 goto out;\r
477 \r
478         /*\r
479          * Pause to give SM chance to configure switches.  We don't want to\r
480          * handle reliability issue in this simple test program.\r
481          */\r
482         sleep(3);\r
483 \r
484         if (message_count) {\r
485                 if (is_sender) {\r
486                         printf("initiating data transfers\n");\r
487                         for (i = 0; i < connections; i++) {\r
488                                 ret = post_sends(&test.nodes[i], 0);\r
489                                 if (ret)\r
490                                         goto out;\r
491                         }\r
492                 } else {\r
493                         printf("receiving data transfers\n");\r
494                         ret = poll_cqs();\r
495                         if (ret)\r
496                                 goto out;\r
497                 }\r
498                 printf("data transfers complete\n");\r
499         }\r
500 out:\r
501         for (i = 0; i < connections; i++) {\r
502                 ret = rdma_leave_multicast(test.nodes[i].cma_id,\r
503                                            test.dst_addr);\r
504                 if (ret)\r
505                         printf("mckey: failure leaving: %d\n", ret);\r
506         }\r
507         return ret;\r
508 }\r
509 \r
510 int main(int argc, char **argv)\r
511 {\r
512         int op, ret;\r
513 \r
514 \r
515         while ((op = getopt(argc, argv, "m:M:sb:c:C:S:p:")) != -1) {\r
516                 switch (op) {\r
517                 case 'm':\r
518                         dst_addr = optarg;\r
519                         break;\r
520                 case 'M':\r
521                         unmapped_addr = 1;\r
522                         dst_addr = optarg;\r
523                         break;\r
524                 case 's':\r
525                         is_sender = 1;\r
526                         break;\r
527                 case 'b':\r
528                         src_addr = optarg;\r
529                         test.src_addr = (struct sockaddr *) &test.src_in;\r
530                         break;\r
531                 case 'c':\r
532                         connections = atoi(optarg);\r
533                         break;\r
534                 case 'C':\r
535                         message_count = atoi(optarg);\r
536                         break;\r
537                 case 'S':\r
538                         message_size = atoi(optarg);\r
539                         break;\r
540                 case 'p':\r
541                         port_space = strtol(optarg, NULL, 0);\r
542                         break;\r
543                 default:\r
544                         printf("usage: %s\n", argv[0]);\r
545                         printf("\t-m multicast_address\n");\r
546                         printf("\t[-M unmapped_multicast_address]\n"\r
547                                "\t replaces -m and requires -b\n");\r
548                         printf("\t[-s(ender)]\n");\r
549                         printf("\t[-b bind_address]\n");\r
550                         printf("\t[-c connections]\n");\r
551                         printf("\t[-C message_count]\n");\r
552                         printf("\t[-S message_size]\n");\r
553                         printf("\t[-p port_space - %#x for UDP (default), "\r
554                                "%#x for IPOIB]\n", RDMA_PS_UDP, RDMA_PS_IPOIB);\r
555                         exit(1);\r
556                 }\r
557         }\r
558 \r
559         test.dst_addr = (struct sockaddr *) &test.dst_in;\r
560         test.connects_left = connections;\r
561 \r
562         test.channel = rdma_create_event_channel();\r
563         if (!test.channel) {\r
564                 printf("failed to create event channel\n");\r
565                 exit(1);\r
566         }\r
567 \r
568         if (alloc_nodes())\r
569                 exit(1);\r
570 \r
571         ret = run();\r
572 \r
573         printf("test complete\n");\r
574         destroy_nodes();\r
575         rdma_destroy_event_channel(test.channel);\r
576 \r
577         printf("return status %d\n", ret);\r
578         return ret;\r
579 }\r