[IPoIB] Avoid the SM. [branches\WOF2-0: 1610-1613, 1617]
[mirror/winof/.git] / tests / limits / user / limits_main.c
1 /*\r
2  * Copyright (c) 2005 SilverStorm Technologies.  All rights reserved.\r
3  * Portions Copyright (c) 2008 Microsoft Corporation.  All rights reserved.\r
4  *\r
5  * This software is available to you under the OpenIB.org BSD license\r
6  * below:\r
7  *\r
8  *     Redistribution and use in source and binary forms, with or\r
9  *     without modification, are permitted provided that the following\r
10  *     conditions are met:\r
11  *\r
12  *      - Redistributions of source code must retain the above\r
13  *        copyright notice, this list of conditions and the following\r
14  *        disclaimer.\r
15  *\r
16  *      - Redistributions in binary form must reproduce the above\r
17  *        copyright notice, this list of conditions and the following\r
18  *        disclaimer in the documentation and/or other materials\r
19  *        provided with the distribution.\r
20  *\r
21  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,\r
22  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF\r
23  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND\r
24  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS\r
25  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN\r
26  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN\r
27  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\r
28  * SOFTWARE.\r
29  *\r
30  * $Id$\r
31  */\r
32 \r
33 \r
34 /*\r
35  * Abstract:\r
36  *      Test limits for:\r
37  *              - memory registration\r
38  *              - CQ creation\r
39  *              - CQ resize\r
40  *              - QP creation\r
41  *\r
42  * Environment:\r
43  *      User Mode\r
44  */\r
45 \r
46 #include <stdio.h>\r
47 #include <stdlib.h>\r
48 #include <stdarg.h>\r
49 #include <ctype.h>\r
50 #include <complib/cl_atomic.h>\r
51 #include <complib/cl_debug.h>\r
52 #include <complib/cl_event.h>\r
53 #include <complib/cl_math.h>\r
54 #include <complib/cl_mutex.h>\r
55 #include <complib/cl_qlist.h>\r
56 #include <complib/cl_thread.h>\r
57 #include <complib/cl_timer.h>\r
58 #include <iba/ib_types.h>\r
59 #include <iba/ib_al.h>\r
60 \r
61 \r
62 /* Globals */\r
63 #define CMT_DBG_VERBOSE         1\r
64 \r
65 \r
66 uint32_t        cmt_dbg_lvl = 0x80000000;\r
67 \r
68 \r
69 /**********************************************************************\r
70  **********************************************************************/\r
71 static void\r
72 __show_usage()\r
73 {\r
74         printf( "\n------- ib_limits - Usage and options ----------------------\n" );\r
75         printf( "Usage:   ib_limits [options]\n");\r
76         printf( "Options:\n" );\r
77         printf( "-m\n"\r
78                         "--memory\n"\r
79                         "\tThis option directs ib_limits to test memory registration\n" );\r
80         printf( "-c\n"\r
81                         "--cq\n"\r
82                         "\tThis option directs ib_limits to test CQ creation\n" );\r
83         printf( "-r\n"\r
84                         "--resize_cq\n"\r
85                         "\tThis option directs ib_limits to test CQ resize\n" );\r
86         printf( "-q\n"\r
87                         "--qp\n"\r
88                         "\tThis option directs ib_limits to test QP creation\n" );\r
89         printf( "-v\n"\r
90                         "--verbose\n"\r
91                         "          This option enables verbosity level to debug console.\n" );\r
92         printf( "-h\n"\r
93                         "--help\n"\r
94                         "          Display this usage info then exit.\n\n" );\r
95 }\r
96 \r
97 \r
98 /* Windows support. */\r
99 struct option\r
100 {\r
101         const char              *long_name;\r
102         unsigned long   flag;\r
103         void                    *pfn_handler;\r
104         char                    short_name;\r
105 };\r
106 \r
107 static char                     *optarg;\r
108 \r
109 #define strtoull        strtoul\r
110 \r
111 \r
112 boolean_t       test_mr, test_cq, test_resize, test_qp;\r
113 \r
114 \r
115 char\r
116 getopt_long(\r
117         int                                     argc,\r
118         char                            *argv[],\r
119         const char                      *short_option,\r
120         const struct option *long_option,\r
121         void                            *unused )\r
122 {\r
123         static int i = 1;\r
124         int j;\r
125         char            ret = 0;\r
126 \r
127         UNUSED_PARAM( unused );\r
128 \r
129         if( i == argc )\r
130                 return -1;\r
131 \r
132         if( argv[i][0] != '-' )\r
133                 return ret;\r
134 \r
135         /* find the first character of the value. */\r
136         for( j = 1; isalpha( argv[i][j] ); j++ )\r
137                 ;\r
138         optarg = &argv[i][j];\r
139 \r
140         if( argv[i][1] == '-' )\r
141         {\r
142                 /* Long option. */\r
143                 for( j = 0; long_option[j].long_name; j++ )\r
144                 {\r
145                         if( strncmp( &argv[i][2], long_option[j].long_name,\r
146                                 optarg - argv[i] - 2 ) )\r
147                         {\r
148                                 continue;\r
149                         }\r
150 \r
151                         switch( long_option[j].flag )\r
152                         {\r
153                         case 1:\r
154                                 if( *optarg == '\0' )\r
155                                         return 0;\r
156                         default:\r
157                                 break;\r
158                         }\r
159                         ret = long_option[j].short_name;\r
160                         break;\r
161                 }\r
162         }\r
163         else\r
164         {\r
165                 for( j = 0; short_option[j] != '\0'; j++ )\r
166                 {\r
167                         if( !isalpha( short_option[j] ) )\r
168                                 return 0;\r
169 \r
170                         if( short_option[j] == argv[i][1] )\r
171                         {\r
172                                 ret = short_option[j];\r
173                                 break;\r
174                         }\r
175 \r
176                         if( short_option[j+1] == ':' )\r
177                         {\r
178                                 if( *optarg == '\0' )\r
179                                         return 0;\r
180                                 j++;\r
181                         }\r
182                 }\r
183         }\r
184         i++;\r
185         return ret;\r
186 }\r
187 \r
188 \r
189 static boolean_t\r
190 __parse_options(\r
191         int                                                     argc,\r
192         char*                                           argv[] )\r
193 {\r
194         uint32_t                                        next_option;\r
195         const char* const                       short_option = "mcrq:vh";\r
196 \r
197         /*\r
198                 In the array below, the 2nd parameter specified the number\r
199                 of arguments as follows:\r
200                 0: no arguments\r
201                 1: argument\r
202                 2: optional\r
203         */\r
204         const struct option long_option[] =\r
205         {\r
206                 {       "memory",       2,      NULL,   'm'},\r
207                 {       "cq",           2,      NULL,   'c'},\r
208                 {       "resize_cq",2,  NULL,   'r'},\r
209                 {       "qp",           2,      NULL,   'q'},\r
210                 {       "verbose",      0,      NULL,   'v'},\r
211                 {       "help",         0,      NULL,   'h'},\r
212                 {       NULL,           0,      NULL,    0 }    /* Required at end of array */\r
213         };\r
214 \r
215         test_mr = FALSE;\r
216         test_cq = FALSE;\r
217         test_resize = FALSE;\r
218         test_qp = FALSE;\r
219 \r
220         /* parse cmd line arguments as input params */\r
221         do\r
222         {\r
223                 next_option = getopt_long( argc, argv, short_option,\r
224                         long_option, NULL );\r
225 \r
226                 switch( next_option )\r
227                 {\r
228                 case 'm':\r
229                         test_mr = TRUE;\r
230                         printf( "\tTest Memory Registration\n" );\r
231                         break;\r
232 \r
233                 case 'c':\r
234                         test_cq = TRUE;\r
235                         printf( "\tTest CQ\n" );\r
236                         break;\r
237 \r
238                 case 'r':\r
239                         test_resize = TRUE;\r
240                         printf( "\tTest CQ Resize\n" );\r
241                         break;\r
242 \r
243                 case 'q':\r
244                         test_qp = TRUE;\r
245                         printf( "\tTest QP\n" );\r
246                         break;\r
247 \r
248                 case 'v':\r
249                         cmt_dbg_lvl = 0xFFFFFFFF;\r
250                         printf( "\tverbose\n" );\r
251                         break;\r
252 \r
253                 case 'h':\r
254                         __show_usage();\r
255                         return FALSE;\r
256 \r
257                 case -1:\r
258                         break;\r
259 \r
260                 default: /* something wrong */\r
261                         __show_usage();\r
262                         return FALSE;\r
263                 }\r
264         } while( next_option != -1 );\r
265 \r
266         return TRUE;\r
267 }\r
268 \r
269 \r
270 struct __mr_buf\r
271 {\r
272         cl_list_item_t  list_item;\r
273         ib_mr_handle_t  h_mr;\r
274         char                    buf[8192 - sizeof(ib_mr_handle_t) - sizeof(cl_list_item_t)];\r
275 };\r
276 \r
277 static void\r
278 __test_mr(\r
279         ib_pd_handle_t                          h_pd )\r
280 {\r
281         ib_api_status_t         status = IB_SUCCESS;\r
282         struct __mr_buf         *p_mr;\r
283         int                                     i = 0;\r
284         ib_mr_create_t          mr_create;\r
285         cl_qlist_t                      mr_list;\r
286         net32_t                         lkey, rkey;\r
287         int64_t                         reg_time, dereg_time, tmp_time, cnt;\r
288 \r
289         printf( "MR testing [\n" );\r
290 \r
291         cl_qlist_init( &mr_list );\r
292         reg_time = 0;\r
293         dereg_time = 0;\r
294         cnt = 0;\r
295 \r
296         do\r
297         {\r
298                 p_mr = cl_malloc( sizeof(struct __mr_buf) );\r
299                 if( !p_mr )\r
300                 {\r
301                         i++;\r
302                         printf( "Failed to allocate memory.\n" );\r
303                         continue;\r
304                 }\r
305 \r
306                 mr_create.vaddr = p_mr->buf;\r
307                 mr_create.length = sizeof(p_mr->buf);\r
308                 mr_create.access_ctrl =\r
309                         IB_AC_LOCAL_WRITE | IB_AC_RDMA_READ | IB_AC_RDMA_WRITE;\r
310 \r
311                 tmp_time = cl_get_time_stamp();\r
312                 status = ib_reg_mem( h_pd, &mr_create, &lkey, &rkey, &p_mr->h_mr );\r
313                 if( status != IB_SUCCESS )\r
314                 {\r
315                         i++;\r
316                         printf( "ib_reg_mem returned %s\n", ib_get_err_str( status ) );\r
317                         cl_free( p_mr );\r
318                         continue;\r
319                 }\r
320                 reg_time += cl_get_time_stamp() - tmp_time;\r
321                 cnt++;\r
322 \r
323                 cl_qlist_insert_tail( &mr_list, &p_mr->list_item );\r
324 \r
325         }       while( status == IB_SUCCESS || i < 1000 );\r
326 \r
327         while( cl_qlist_count( &mr_list ) )\r
328         {\r
329                 p_mr = PARENT_STRUCT( cl_qlist_remove_head( &mr_list ),\r
330                         struct __mr_buf, list_item );\r
331 \r
332                 tmp_time = cl_get_time_stamp();\r
333                 status = ib_dereg_mr( p_mr->h_mr );\r
334                 if( status != IB_SUCCESS )\r
335                         printf( "ib_dereg_mr returned %s\n", ib_get_err_str( status ) );\r
336                 dereg_time += cl_get_time_stamp() - tmp_time;\r
337 \r
338                 cl_free( p_mr );\r
339         }\r
340 \r
341         printf( "reg time %f, dereg time %f\n", (double)reg_time/(double)cnt,\r
342                 (double)dereg_time/(double)cnt );\r
343         printf( "MR testing ]\n" );\r
344 }\r
345 \r
346 \r
347 struct __cq\r
348 {\r
349         cl_list_item_t          list_item;\r
350         ib_cq_handle_t          h_cq;\r
351 };\r
352 \r
353 static void\r
354 __test_cq(\r
355         ib_ca_handle_t                          h_ca,\r
356         boolean_t                                       resize )\r
357 {\r
358         ib_api_status_t         status = IB_SUCCESS;\r
359         struct __cq                     *p_cq;\r
360         int                                     i = 0, j;\r
361         ib_cq_create_t          cq_create;\r
362         cl_qlist_t                      cq_list;\r
363         cl_waitobj_handle_t     h_waitobj;\r
364         uint32_t                        size;\r
365 \r
366         printf( "CQ %stesting [\n", resize?"resize ":"" );\r
367 \r
368         cl_qlist_init( &cq_list );\r
369 \r
370         if( cl_waitobj_create( FALSE, &h_waitobj ) != CL_SUCCESS )\r
371         {\r
372                 printf( "Failed to allocate CQ wait object.\n" );\r
373                 return;\r
374         }\r
375 \r
376         do\r
377         {\r
378                 p_cq = cl_malloc( sizeof(*p_cq) );\r
379                 if( !p_cq )\r
380                 {\r
381                         i++;\r
382                         printf( "Failed to allocate memory.\n" );\r
383                         continue;\r
384                 }\r
385 \r
386                 cq_create.h_wait_obj = h_waitobj;\r
387                 cq_create.pfn_comp_cb = NULL;\r
388                 if( resize )\r
389                         cq_create.size = 32;\r
390                 else\r
391                         cq_create.size = 4096;\r
392 \r
393                 status = ib_create_cq( h_ca, &cq_create, NULL, NULL, &p_cq->h_cq );\r
394                 if( status != IB_SUCCESS )\r
395                 {\r
396                         i++;\r
397                         printf( "ib_create_cq returned %s\n", ib_get_err_str( status ) );\r
398                         cl_free( p_cq );\r
399                         continue;\r
400                 }\r
401 \r
402                 if( resize )\r
403                 {\r
404                         size = 256;\r
405                         j = 0;\r
406 \r
407                         do\r
408                         {\r
409                                 status = ib_modify_cq( p_cq->h_cq, &size );\r
410                                 if( status == IB_SUCCESS )\r
411                                 {\r
412                                         size += 256;\r
413                                 }\r
414                                 else\r
415                                 {\r
416                                         j++;\r
417                                         printf( "ib_modify_cq returned %s\n",\r
418                                                 ib_get_err_str( status ) );\r
419                                 }\r
420 \r
421                         } while( status == IB_SUCCESS || j < 100 );\r
422                 }\r
423 \r
424                 cl_qlist_insert_tail( &cq_list, &p_cq->list_item );\r
425 \r
426         }       while( status == IB_SUCCESS || i < 1000 );\r
427 \r
428         while( cl_qlist_count( &cq_list ) )\r
429         {\r
430                 p_cq = PARENT_STRUCT( cl_qlist_remove_head( &cq_list ),\r
431                         struct __cq, list_item );\r
432 \r
433                 status = ib_destroy_cq( p_cq->h_cq, NULL );\r
434                 if( status != IB_SUCCESS )\r
435                         printf( "ib_destroy_cq returned %s\n", ib_get_err_str( status ) );\r
436 \r
437                 cl_free( p_cq );\r
438         }\r
439 \r
440         printf( "CQ %stesting ]\n", resize?"resize ":"" );\r
441 }\r
442 \r
443 /**********************************************************************\r
444  **********************************************************************/\r
445 int __cdecl\r
446 main(\r
447         int                                                     argc,\r
448         char*                                           argv[] )\r
449 {\r
450         ib_api_status_t         status;\r
451         ib_al_handle_t          h_al;\r
452         ib_ca_handle_t          h_ca;\r
453         ib_pd_handle_t          h_pd;\r
454         size_t                          size;\r
455         net64_t                         *ca_guids;\r
456 \r
457         /* Set defaults. */\r
458         if( !__parse_options( argc, argv ) )\r
459                 return 1;\r
460 \r
461         status = ib_open_al( &h_al );\r
462         if( status != IB_SUCCESS )\r
463         {\r
464                 printf( "ib_open_al returned %s\n", ib_get_err_str( status ) );\r
465                 return 1;\r
466         }\r
467 \r
468         size = 0;\r
469         status = ib_get_ca_guids( h_al, NULL, &size );\r
470         if( status != IB_INSUFFICIENT_MEMORY )\r
471         {\r
472                 printf( "ib_get_ca_guids for array size returned %s",\r
473                         ib_get_err_str( status ) );\r
474                 goto done;\r
475         }\r
476 \r
477         if( size == 0 )\r
478         {\r
479                 printf( "No CAs installed.\n" );\r
480                 goto done;\r
481         }\r
482 \r
483         ca_guids = malloc( sizeof(net64_t) * size );\r
484         if( !ca_guids )\r
485         {\r
486                 printf( "Failed to allocate CA GUID array.\n" );\r
487                 goto done;\r
488         }\r
489 \r
490         status = ib_get_ca_guids( h_al, ca_guids, &size );\r
491         if( status != IB_SUCCESS )\r
492         {\r
493                 printf( "ib_get_ca_guids for CA guids returned %s",\r
494                         ib_get_err_str( status ) );\r
495                 free( ca_guids );\r
496                 goto done;\r
497         }\r
498 \r
499         status = ib_open_ca( h_al, ca_guids[0], NULL, NULL, &h_ca );\r
500         free( ca_guids );\r
501         if( status != IB_SUCCESS )\r
502         {\r
503                 printf( "ib_open_ca returned %s", ib_get_err_str( status ) );\r
504                 goto done;\r
505         }\r
506 \r
507         status = ib_alloc_pd( h_ca, IB_PDT_NORMAL, NULL, &h_pd );\r
508         if( status != IB_SUCCESS )\r
509         {\r
510                 printf( "ib_alloc_pd returned %s", ib_get_err_str( status ) );\r
511                 goto done;\r
512         }\r
513 \r
514         if( test_mr )\r
515                 __test_mr( h_pd );\r
516 \r
517         if( test_cq )\r
518                 __test_cq( h_ca, FALSE );\r
519 \r
520         if( test_resize )\r
521                 __test_cq( h_ca, TRUE );\r
522 \r
523         //if( test_qp )\r
524         //      __test_qp( h_ca, h_pd );\r
525 \r
526 done:\r
527         ib_close_al( h_al );\r
528 \r
529         return 0;\r
530 }\r