[TOOLS] add read_bw test and cosmetics
authorsleybo <sleybo@ad392aa1-c5ef-ae45-8dd8-e69d62a5ef86>
Mon, 19 Feb 2007 15:45:06 +0000 (15:45 +0000)
committersleybo <sleybo@ad392aa1-c5ef-ae45-8dd8-e69d62a5ef86>
Mon, 19 Feb 2007 15:45:06 +0000 (15:45 +0000)
git-svn-id: svn://openib.tc.cornell.edu/gen1/trunk@598 ad392aa1-c5ef-ae45-8dd8-e69d62a5ef86

tools/perftests/user/dirs
tools/perftests/user/read_bw/SOURCES [new file with mode: 0644]
tools/perftests/user/read_bw/makefile [new file with mode: 0644]
tools/perftests/user/read_bw/read_bw.c [new file with mode: 0644]
tools/perftests/user/read_bw/read_bw.rc [new file with mode: 0644]
tools/perftests/user/send_lat/send_lat.c
tools/perftests/user/write_bw/write_bw.c

index 3e83a60..615798e 100644 (file)
@@ -2,6 +2,7 @@ DIRS=\
        send_bw \ \r
        send_lat \\r
        write_lat \\r
-       write_bw \r
+       write_bw  \\r
+  read_bw\r
 \r
 #      write_bw_postlist rdma_lat rdma_bw send_lat  write_lat write_bw read_lat read_bwr\r
diff --git a/tools/perftests/user/read_bw/SOURCES b/tools/perftests/user/read_bw/SOURCES
new file mode 100644 (file)
index 0000000..e08f73a
--- /dev/null
@@ -0,0 +1,28 @@
+TARGETNAME=ib_read_bw\r
+TARGETPATH=..\..\..\..\bin\user\obj$(BUILD_ALT_DIR)\r
+TARGETTYPE=PROGRAM\r
+UMTYPE=console\r
+USE_CRTDLL=1\r
+\r
+C_DEFINES=$(C_DEFINES) /D__WIN__ \r
+\r
+SOURCES=read_bw.rc \\r
+       ..\getopt.c \\r
+       ..\perf_utils.c \\r
+       read_bw.c \r
+\r
+INCLUDES=..;..\..\..\..\inc;..\..\..\..\inc\user\r
+\r
+RCOPTIONS=/I..\..\win\include\r
+\r
+TARGETLIBS= \\r
+                       $(DDK_LIB_PATH)\Ws2_32.lib \\r
+!if $(FREEBUILD)\r
+                       $(TARGETPATH)\*\complib.lib \\r
+                       $(TARGETPATH)\*\ibal.lib\r
+!else\r
+                       $(TARGETPATH)\*\complibd.lib \\r
+                       $(TARGETPATH)\*\ibald.lib\r
+!endif\r
+\r
+MSC_WARNING_LEVEL= /W3\r
diff --git a/tools/perftests/user/read_bw/makefile b/tools/perftests/user/read_bw/makefile
new file mode 100644 (file)
index 0000000..a0c0627
--- /dev/null
@@ -0,0 +1,7 @@
+#\r
+# DO NOT EDIT THIS FILE!!!  Edit .\sources. if you want to add a new source\r
+# file to this component.  This file merely indirects to the real make file\r
+# that is shared by all the driver components of the OpenIB Windows project.\r
+#\r
+\r
+!INCLUDE ..\..\..\..\inc\openib.def\r
diff --git a/tools/perftests/user/read_bw/read_bw.c b/tools/perftests/user/read_bw/read_bw.c
new file mode 100644 (file)
index 0000000..797f635
--- /dev/null
@@ -0,0 +1,787 @@
+/*
+ * Copyright (c) 2005 Topspin Communications.  All rights reserved.
+ * Copyright (c) 2005 Mellanox Technologies Ltd.  All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses.  You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and/or other materials
+ *        provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ * $Id: read_bw.c 1955 2007-02-19 14:46:04Z sleybo $
+ */
+#include "getopt.h"
+#include "perf_defs.h"
+#include "get_clock.h"
+
+struct user_parameters {
+       const char              *servername;
+       int connection_type;
+       int mtu;
+       int all; /* run all msg size */
+       int iters;
+       int tx_depth;
+       int max_out_read;
+};
+
+static int page_size;
+
+cycles_t       *tposted;
+cycles_t       *tcompleted;
+
+
+void
+pp_cq_comp_cb(
+       IN              const   ib_cq_handle_t                          h_cq,
+       IN                              void                                            *cq_context )
+{
+       UNUSED_PARAM( h_cq );
+       UNUSED_PARAM( cq_context);
+       return ;
+}
+
+static struct pingpong_context *pp_init_ctx(unsigned size,int port, struct user_parameters *user_parm)
+{
+
+       struct pingpong_context *ctx;
+       ib_api_status_t                         ib_status = IB_SUCCESS; 
+       size_t                                  guid_count;
+       ib_net64_t                              *ca_guid_array;
+
+
+       ctx = malloc(sizeof *ctx);
+       if (!ctx){
+               perror("malloc");
+               return NULL;
+       }
+
+       memset(ctx, 0, sizeof(struct pingpong_context));
+       ctx->size = size;
+       ctx->tx_depth = user_parm->tx_depth;
+
+       ctx->qp = malloc(sizeof (ib_qp_handle_t));
+       if (!ctx->qp) {
+               perror("malloc");
+               return NULL;
+       }
+       
+       ctx->qp_attr = malloc(sizeof (ib_qp_attr_t));
+       if (!ctx->qp_attr) {
+               perror("malloc");
+               return NULL;
+       }
+       
+       ctx->buf = malloc( size * 2 );
+       if (!ctx->buf) {
+               fprintf(stderr, "Couldn't allocate work buf.\n");
+               return NULL;
+       }
+
+       memset(ctx->buf, 0, size * 2 );
+
+
+       /*
+        * Open the AL instance
+        */
+       ib_status = ib_open_al(&ctx->al);
+       if(ib_status != IB_SUCCESS)
+       {
+               fprintf(stderr,"ib_open_al failed status = %d\n", ib_status);
+               return NULL;
+       }
+
+       /*
+        * Get the Local CA Guids
+        */
+       ib_status = ib_get_ca_guids(ctx->al, NULL, &guid_count);
+       if(ib_status != IB_INSUFFICIENT_MEMORY)
+       {
+               fprintf(stderr,"ib_get_ca_guids1 failed status = %d\n", (uint32_t)ib_status);
+               return NULL;
+       }
+       
+       /*
+        * If no CA's Present then return
+        */
+
+       if(guid_count == 0)
+               return NULL;
+
+       
+       ca_guid_array = (ib_net64_t*)malloc(sizeof(ib_net64_t) * guid_count);
+       
+       ib_status = ib_get_ca_guids(ctx->al, ca_guid_array, &guid_count);
+       if(ib_status != IB_SUCCESS)
+       {
+               fprintf(stderr,"ib_get_ca_guids2 failed with status = %d\n", ib_status);
+               return NULL;
+       }
+
+       /*
+        * Open only the first HCA
+        */
+       /* Open the CA */
+       ib_status = ib_open_ca(ctx->al ,ca_guid_array[0] ,NULL,
+               NULL,   //ca_context
+               &ctx->ca);
+
+       if(ib_status != IB_SUCCESS)
+       {
+               fprintf(stderr,"ib_open_ca failed with status = %d\n", ib_status);
+               return NULL;
+       }
+
+       //xxx
+       //printf("ib_open_ca passed i=%d\n",i); 
+       //xxx
+
+       
+
+       {
+               /* Query the CA */
+               uint32_t bsize = 0;
+               ib_status = ib_query_ca(ctx->ca, NULL, &bsize);
+               if(ib_status != IB_INSUFFICIENT_MEMORY)
+               {
+                       fprintf(stderr, "Failed to query device props");
+                       return NULL;
+               }
+
+               ctx->ca_attr = (ib_ca_attr_t *)malloc(bsize);
+
+               ib_status = ib_query_ca(ctx->ca, ctx->ca_attr, &bsize);
+               if(ib_status != IB_SUCCESS)
+               {
+                       printf("ib_query_ca failed with status = %d\n", ib_status);
+                       return NULL;
+               }
+               if (user_parm->mtu == 0) {/*user did not ask for specific mtu */
+                       if (ctx->ca_attr->dev_id == 23108) {
+                               user_parm->mtu = 1024;
+                       } else {
+                               user_parm->mtu = 2048;
+                       }
+               }
+       }
+
+       ib_status = ib_alloc_pd(ctx->ca ,
+                                               IB_PDT_NORMAL,
+                                               ctx, //pd_context
+                                               &ctx->pd);
+       if (ib_status != IB_SUCCESS) {
+               fprintf(stderr, "Couldn't allocate PD\n");
+               return NULL;
+       }
+
+
+       /* We dont really want IBV_ACCESS_LOCAL_WRITE, but IB spec says:
+        * The Consumer is not allowed to assign Remote Write or Remote Atomic to
+        * a Memory Region that has not been assigned Local Write. */
+
+
+       {
+               ib_mr_create_t                  mr_create;
+
+               mr_create.length = size * 2;
+                       
+               mr_create.vaddr = ctx->buf;
+               mr_create.access_ctrl = IB_AC_RDMA_WRITE| IB_AC_LOCAL_WRITE|IB_AC_RDMA_READ;
+               
+               ib_status = ib_reg_mem(ctx->pd ,&mr_create ,&ctx->lkey ,&ctx->rkey ,&ctx->mr);
+               if (ib_status != IB_SUCCESS) {
+                       fprintf(stderr, "Couldn't allocate MR\n");
+                       return NULL;
+               }
+       }
+
+       {
+               ib_cq_create_t                  cq_create;
+               
+               cq_create.size = user_parm->tx_depth;
+               cq_create.h_wait_obj = NULL;
+               cq_create.pfn_comp_cb = pp_cq_comp_cb;
+               ib_status = ib_create_cq(ctx->ca,&cq_create ,ctx, NULL, &ctx->scq);
+               if (ib_status != IB_SUCCESS) {
+                       fprintf(stderr, "Couldn't create CQ ib_status = %d\n",ib_status);
+                       return NULL;
+               }
+       }
+
+       {
+
+               ib_qp_create_t  qp_create;
+               ib_qp_mod_t             qp_modify;
+               ib_qp_attr_t            qp_attr;
+               
+               memset(&qp_create, 0, sizeof(ib_qp_create_t));
+               qp_create.h_sq_cq       = ctx->scq;
+               qp_create.h_rq_cq       = ctx->scq;
+               qp_create.sq_depth      = user_parm->tx_depth;
+               qp_create.rq_depth      = user_parm->tx_depth;
+               qp_create.sq_sge        = 1;
+               qp_create.rq_sge        = 1;
+               //TODO MAX_INLINE
+               qp_create.qp_type= IB_QPT_RELIABLE_CONN;
+               qp_create.sq_signaled = FALSE;
+               /*attr.sq_sig_all = 0;*/
+
+               ib_status = ib_create_qp(ctx->pd, &qp_create,NULL,NULL,&ctx->qp[0]);
+               if (ib_status != IB_SUCCESS){
+                       fprintf(stderr, "Couldn't create QP\n");
+                       return NULL;
+               }
+       
+       
+
+       
+               memset(&qp_modify, 0, sizeof(ib_qp_mod_t));
+               qp_modify.req_state = IB_QPS_INIT;
+               qp_modify.state.init.pkey_index = 0 ;
+               qp_modify.state.init.primary_port = (uint8_t)port;
+               qp_modify.state.init.access_ctrl = IB_AC_RDMA_WRITE | IB_AC_LOCAL_WRITE|IB_AC_RDMA_READ;
+
+               
+               ib_status = ib_modify_qp(ctx->qp[0], &qp_modify);
+               if (ib_status != IB_SUCCESS){
+                       fprintf(stderr, "Failed to modify QP to INIT\n");
+                       return NULL;
+               }
+
+               memset(&qp_attr, 0, sizeof(ib_qp_attr_t));
+               ib_status = ib_query_qp(ctx->qp[0],&ctx->qp_attr[0]);
+               if (ib_status != IB_SUCCESS){
+                       fprintf(stderr, "Failed to modify QP to INIT\n");
+                       return NULL;
+               }
+               fprintf(stderr, "max inline size %d\n",ctx->qp_attr[0].sq_max_inline);
+       }
+       return ctx;
+}
+
+
+
+
+
+static int pp_connect_ctx(struct pingpong_context *ctx, int port, int my_psn,
+                         struct pingpong_dest *dest, struct user_parameters *user_parm,int qpindex)
+{
+
+       ib_api_status_t ib_status;
+       ib_qp_mod_t     attr;
+       memset(&attr, 0, sizeof(ib_qp_mod_t));
+
+       attr.req_state          = IB_QPS_RTR;
+       switch (user_parm->mtu) {
+       case 256 : 
+               attr.state.rtr.primary_av.conn.path_mtu = IB_MTU_LEN_256;
+               break;
+       case 512 :
+               attr.state.rtr.primary_av.conn.path_mtu = IB_MTU_LEN_512;
+               break;
+       case 1024 :
+               attr.state.rtr.primary_av.conn.path_mtu = IB_MTU_LEN_1024;
+               break;
+       case 2048 :
+               attr.state.rtr.primary_av.conn.path_mtu = IB_MTU_LEN_2048;
+               break;
+       }
+       printf("Mtu : %d\n", user_parm->mtu);
+       attr.state.rtr.dest_qp  = dest->qpn;;
+       attr.state.rtr.rq_psn   = dest->psn;
+       attr.state.rtr.resp_res = (uint8_t)user_parm->max_out_read;
+       attr.state.rtr.rnr_nak_timeout = 12;
+       attr.state.rtr.primary_av.grh_valid = 0;
+       attr.state.rtr.primary_av.dlid = dest->lid;
+       attr.state.rtr.primary_av.sl = 0;
+       attr.state.rtr.primary_av.path_bits = 0;
+       attr.state.rtr.primary_av.port_num = (uint8_t)port;
+       attr.state.rtr.primary_av.static_rate = IB_PATH_RECORD_RATE_10_GBS;
+       attr.state.rtr.opts = IB_MOD_QP_LOCAL_ACK_TIMEOUT |
+                                       IB_MOD_QP_RESP_RES |
+                                       IB_MOD_QP_PRIMARY_AV;
+
+
+       ib_status = ib_modify_qp(ctx->qp[0], &attr);
+       if(ib_status != IB_SUCCESS){
+                       fprintf(stderr, "Failed to modify QP to RTR\n");
+                       return 1;
+       }
+
+
+
+       memset(&attr, 0, sizeof(ib_qp_mod_t));
+       attr.req_state  = IB_QPS_RTS;
+       attr.state.rts.sq_psn = my_psn;
+       attr.state.rts.init_depth = (uint8_t)user_parm->max_out_read;
+       attr.state.rts.local_ack_timeout = 14;
+       attr.state.rts.retry_cnt = 7;
+       attr.state.rts.rnr_retry_cnt = 7;
+       attr.state.rts.opts = IB_MOD_QP_RNR_RETRY_CNT |
+                                       IB_MOD_QP_RETRY_CNT |
+                                       IB_MOD_QP_INIT_DEPTH |
+                                       IB_MOD_QP_LOCAL_ACK_TIMEOUT;
+
+       ib_status = ib_modify_qp(ctx->qp[0], &attr);
+       if(ib_status != IB_SUCCESS){
+               fprintf(stderr, "Failed to modify QP to RTS\n");
+               return 1;
+       }
+       return 0;
+}
+
+
+static SOCKET pp_open_port(struct pingpong_context *ctx, const char * servername,
+                       int ib_port, int port, struct pingpong_dest **p_my_dest,
+                       struct pingpong_dest **p_rem_dest,struct user_parameters *user_parm)
+{
+       struct pingpong_dest    *my_dest;
+       struct pingpong_dest    *rem_dest;
+       SOCKET                          sockfd;
+       int                                     rc;
+       int                                     i;
+       int                                     numofqps = 1;
+       
+       /* Create connection between client and server.
+        * We do it by exchanging data over a TCP socket connection. */
+
+       
+       my_dest = malloc( sizeof (struct pingpong_dest) * numofqps);
+       if (!my_dest){
+               perror("malloc");
+               return INVALID_SOCKET;
+       }
+
+       rem_dest = malloc(sizeof (struct pingpong_dest) * numofqps );
+       if (!rem_dest){
+               perror("malloc");
+               return INVALID_SOCKET;
+       }
+
+       sockfd = servername ? pp_client_connect(servername, port) :
+               pp_server_connect(port);
+
+       if (sockfd  == INVALID_SOCKET) {
+               printf("pp_connect_sock(%s,%d) failed (%d)!\n",
+                      servername, port, sockfd);
+               return INVALID_SOCKET;
+       }
+
+       
+       for (i =0 ;i<numofqps;i ++) 
+       {
+               /* Create connection between client and server.
+               * We do it by exchanging data over a TCP socket connection. */
+               
+               my_dest[i].lid = ctx->ca_attr->p_port_attr[ib_port-1].lid;
+               my_dest[i].psn = rand() & 0xffffff;
+               if (!my_dest[i].lid) {
+                       fprintf(stderr, "Local lid 0x0 detected. Is an SM running?\n");
+                       return 1;
+               }
+               my_dest[i].qpn = ctx->qp_attr[i].num;
+               /* TBD this should be changed inot VA and different key to each qp */
+               my_dest[i].rkey = ctx->rkey;
+               my_dest[i].vaddr = (uintptr_t)ctx->buf + ctx->size;
+
+               printf("  local address:  LID %#04x, QPN %#06x, PSN %#06x, "
+               "RKey %#08x VAddr %#016Lx\n",
+               my_dest[i].lid, my_dest[i].qpn, my_dest[i].psn,
+               my_dest[i].rkey, my_dest[i].vaddr);
+
+               rc = servername ? pp_client_exch_dest(sockfd, &my_dest[i],&rem_dest[i]):
+                                               pp_server_exch_dest(sockfd, &my_dest[i],&rem_dest[i]);
+               if (rc)
+                       return INVALID_SOCKET;
+               printf("  remote address: LID %#04x, QPN %#06x, PSN %#06x, "
+               "RKey %#08x VAddr %#016Lx\n",
+               rem_dest[i].lid, rem_dest[i].qpn, rem_dest[i].psn,
+               rem_dest[i].rkey, rem_dest[i].vaddr);
+
+               if (pp_connect_ctx(ctx, ib_port, my_dest[i].psn, &rem_dest[i], user_parm,i))
+                       return INVALID_SOCKET;
+               /* An additional handshake is required *after* moving qp to RTR.
+               Arbitrarily reuse exch_dest for this purpose. */
+               rc = servername ? pp_client_exch_dest(sockfd, &my_dest[i],&rem_dest[i]):
+                                               pp_server_exch_dest(sockfd, &my_dest[i],&rem_dest[i]);
+               if (rc)
+                       return INVALID_SOCKET;
+       }
+       *p_rem_dest = rem_dest;
+       *p_my_dest = my_dest;
+       return sockfd;
+}
+
+static void usage(const char *argv0)
+{
+       printf("Usage:\n");
+       printf("  %s            start a server and wait for connection\n", argv0);
+       printf("  %s <host>     connect to server at <host>\n", argv0);
+       printf("\n");
+       printf("Options:\n");
+       printf("  -p, --port=<port>      listen on/connect to port <port> (default 18515)\n");
+       printf("  -d, --ib-dev=<dev>     use IB device <dev> (default first device found)\n");
+       printf("  -i, --ib-port=<port>   use port <port> of IB device (default 1)\n");
+       printf("  -m, --mtu=<mtu>        mtu size (default 1024)\n");
+       printf("  -o, --outs=<num>       num of outstanding read/atom(default 4)\n");
+       printf("  -s, --size=<size>      size of message to exchange (default 65536)\n");
+       printf("  -a, --all              Run sizes from 2 till 2^23\n");
+       printf("  -t, --tx-depth=<dep>   size of tx queue (default 100)\n");
+       printf("  -n, --iters=<iters>    number of exchanges (at least 2, default 1000)\n");
+       printf("  -b, --bidirectional    measure bidirectional bandwidth (default unidirectional)\n");
+       printf("  -V, --version          display version number\n");
+       printf("  -e, --events           sleep on CQ events (default poll)\n");
+}
+
+static void print_report(unsigned int iters, unsigned size, int duplex,
+                        cycles_t *tposted, cycles_t *tcompleted)
+{
+       double                  cycles_to_units;
+       uint64_t                        tsize;  /* Transferred size, in megabytes */
+       unsigned int            i, j;
+       int                             opt_posted = 0, opt_completed = 0;
+       cycles_t                        opt_delta;
+       cycles_t                        t;
+
+
+       opt_delta = tcompleted[opt_posted] - tposted[opt_completed];
+
+       /* Find the peak bandwidth */
+       for (i = 0; i < iters; ++i)
+               for (j = i; j < iters; ++j) {
+                       t = (tcompleted[j] - tposted[i]) / (j - i + 1);
+                       if (t < opt_delta) {
+                               opt_delta  = t;
+                               opt_posted = i;
+                               opt_completed = j;
+                       }
+               }
+
+       cycles_to_units = get_cpu_mhz() ;
+
+       tsize = duplex ? 2 : 1;
+       tsize = tsize * size;
+       printf("%7d        %d            %7.2f               %7.2f\n",
+              size,iters,tsize * cycles_to_units / opt_delta / 0x100000,
+              (uint64_t)tsize * iters * cycles_to_units /(tcompleted[iters - 1] - tposted[0]) / 0x100000);
+}
+
+
+
+int run_iter(struct pingpong_context *ctx, struct user_parameters *user_param,
+            struct pingpong_dest *rem_dest, int size)
+{
+       ib_api_status_t         ib_status;
+       int                                     scnt, ccnt ;
+       ib_send_wr_t                    *bad_wr;
+               
+       ctx->list.vaddr = (uintptr_t) ctx->buf;
+       ctx->list.length = size;
+       ctx->list.lkey = ctx->lkey;
+       ctx->wr.remote_ops.vaddr = rem_dest->vaddr;
+       ctx->wr.remote_ops.rkey = rem_dest->rkey;
+       ctx->wr.wr_id      = PINGPONG_RDMA_WRID;
+       ctx->wr.ds_array  = &ctx->list;
+       ctx->wr.num_ds = 1;
+       ctx->wr.wr_type = WR_RDMA_READ;
+       ctx->wr.send_opt = IB_SEND_OPT_SIGNALED;
+       ctx->wr.p_next = NULL;
+
+       scnt = 0;
+       ccnt = 0;
+       
+       /* Done with setup. Start the test. */
+       while (scnt < user_param->iters || ccnt < user_param->iters) {
+
+               while (scnt < user_param->iters && (scnt - ccnt) < user_param->tx_depth ) {
+
+                       tposted[scnt] = get_cycles();
+                       ib_status = ib_post_send(ctx->qp[0], &ctx->wr, &bad_wr);
+                       if (ib_status != IB_SUCCESS) 
+                       {
+                               fprintf(stderr, "Couldn't post send: scnt %d ccnt %d\n",scnt,ccnt);
+                               return 1;
+                       }
+                       ++scnt;
+                       PERF_DEBUG("scnt = %d \n",scnt);
+
+               }
+
+               if (ccnt < user_param->iters) 
+               {
+                       ib_wc_t wc;
+                       ib_wc_t *p_wc_done,*p_wc_free;
+
+                       p_wc_free = &wc;
+                       p_wc_done = NULL;
+                       p_wc_free->p_next = NULL;
+
+
+                       do {
+                               ib_status = ib_poll_cq(ctx->scq, &p_wc_free, &p_wc_done);
+                               if (ib_status == IB_SUCCESS) {
+                                       tcompleted[ccnt] = get_cycles();
+                                       if (p_wc_done->status != IB_WCS_SUCCESS) {
+                                               fprintf(stderr, "Completion wth error at %s:\n",
+                                               user_param->servername ? "client" : "server");
+                                               fprintf(stderr, "Failed status %d: wr_id %d syndrom 0x%x\n",
+                                               p_wc_done->status, (int) p_wc_done->wr_id, p_wc_done->vendor_specific);
+                                               return 1;
+                                       }
+
+                                       /*here the id is the index to the qp num */
+                                       ++ccnt;
+                                       PERF_DEBUG("ccnt = %d \n",ccnt);
+                                       p_wc_free = p_wc_done;
+                                       p_wc_free->p_next = NULL;
+                                       p_wc_done = NULL;
+                               }
+
+                       
+                       } while (ib_status == IB_SUCCESS);
+
+                       if (ib_status != IB_NOT_FOUND) {
+                               fprintf(stderr, "Poll Recieve CQ failed %d\n", ib_status);
+                               return 12;
+                       }
+
+               }
+       }
+
+       return(0);
+}
+
+int __cdecl main(int argc, char *argv[])
+{
+       struct pingpong_context *ctx;
+       struct pingpong_dest     *my_dest;
+       struct pingpong_dest     *rem_dest;
+       struct user_parameters  user_param;
+       char                            *ib_devname = NULL;
+       int                             port = 18515;
+       int                             ib_port = 1;
+       unsigned                        size = 65536;
+       SOCKET                  sockfd = INVALID_SOCKET;
+       WSADATA                 wsaData;
+       int                             iResult; 
+       int                             i = 0;
+       int                             duplex = 0;
+
+       /* init default values to user's parameters */
+       memset(&user_param, 0, sizeof(struct user_parameters));
+       user_param.mtu = 0; /* signal choose default by device */
+       user_param.iters = 1000;
+       user_param.tx_depth = 100;
+       user_param.servername = NULL;
+       user_param.connection_type = RC;
+       user_param.max_out_read = 4; /* the device capability on gen2 */
+       /* Parameter parsing. */
+       while (1) {
+               int c;
+
+               static struct option long_options[] = {
+                       { "port",                       1,      NULL,   'p' },
+                       { "ib-dev",             1,      NULL,   'd' },
+                       { "ib-port",            1,      NULL,   'i' },
+                       { "mtu",                        1,      NULL,   'm' },
+                       { "outs",                       1,      NULL,   'o' },
+                       { "size",                       1,      NULL,   's' },
+                       { "iters",                      1,      NULL,   'n' },
+                       { "tx-depth",           1,      NULL,   't' },
+                       { "all",                        0,      NULL,   'a' },
+                       { "bidirectional",      0,      NULL,   'b' },
+                       { "version",            0,      NULL,   'V' },
+                       { 0 }
+               };
+               c = getopt_long(argc, argv, "p:d:i:m:o:s:n:t:ba", long_options, NULL);
+               if (c == -1)
+                       break;
+
+               switch (c) {
+               case 'p':
+                       port = strtol(optarg, NULL, 0);
+                       if (port < 0 || port > 65535) {
+                               usage(argv[0]);
+                               return 1;
+                       }
+                       break;
+
+               case 'd':
+                       ib_devname = _strdup(optarg);
+                       break;
+               case 'm':
+                       user_param.mtu = strtol(optarg, NULL, 0);
+                       break;
+               case 'o':
+                       user_param.max_out_read = strtol(optarg, NULL, 0);
+                       break;
+               case 'a':
+                       user_param.all = ALL;
+                       break;
+               case 'V':
+                       printf("read_bw version : %.2f\n",VERSION);
+                       return 0;
+                       break;
+               case 'i':
+                       ib_port = strtol(optarg, NULL, 0);
+                       if (ib_port < 0) {
+                               usage(argv[0]);
+                               return 1;
+                       }
+                       break;
+
+               case 's':
+                       size = strtol(optarg, NULL, 0);
+                       if (size < 1 || size > UINT_MAX / 2) {
+                               usage(argv[0]);
+                               return 1;
+                       }
+                       break;
+
+               case 't':
+                       user_param.tx_depth = strtol(optarg, NULL, 0);
+                       if (user_param.tx_depth < 1) { usage(argv[0]); return 1; }
+                       break;
+
+               case 'n':
+                       user_param.iters = strtol(optarg, NULL, 0);
+                       if (user_param.iters < 2) {
+                               usage(argv[0]);
+                               return 1;
+                       }
+
+                       break;
+
+               case 'b':
+                       duplex = 1;
+                       break;
+
+               default:
+                       usage(argv[0]);
+                       return 1;
+               }
+       }
+
+       if (optind == argc - 1)
+               user_param.servername = _strdup(argv[optind]);
+       else if (optind < argc) {
+               usage(argv[0]);
+               return 6;
+       }
+       printf("------------------------------------------------------------------\n");
+       if (duplex == 1) {
+               printf("                    RDMA_Read Bidirectional BW Test\n");
+       } else {
+               printf("                    RDMA_Read BW Test\n");
+       }
+       printf("Connection type : RC\n");
+       /* Done with parameter parsing. Perform setup. */
+
+       // Initialize Winsock
+       iResult = WSAStartup(MAKEWORD(2,2), &wsaData);
+       if (iResult != NO_ERROR) {
+               printf("Error at WSAStartup()\n");
+               return 1;
+       }
+
+       
+       if (user_param.all == ALL) {
+               /*since we run all sizes */
+               size = 8388608; /*2^23 */
+       }
+       
+       srand(GetCurrentProcessId() * GetTickCount());
+
+       //TODO: get pagesize from sysinfo
+       page_size = 4096;
+
+       //TODO:get the device names
+
+
+       ctx = pp_init_ctx(size,ib_port, &user_param);
+       if (!ctx)
+               return 8;
+
+
+       sockfd = pp_open_port(ctx, user_param.servername, ib_port, port,&my_dest,&rem_dest,&user_param);
+       if (sockfd == INVALID_SOCKET)
+               return 9;
+       
+       
+       printf("------------------------------------------------------------------\n");
+       printf(" #bytes #iterations    BW peak[MB/sec]    BW average[MB/sec]  \n");
+       /* For half duplex tests, server just waits for client to exit */
+       /* use dummy my_dest struct*/
+       if (!user_param.servername && !duplex) {
+               pp_server_exch_dest(sockfd, my_dest,rem_dest);
+               send(sockfd, "done", sizeof "done",0);
+               closesocket(sockfd);
+               return 0;
+       }
+
+       tposted = malloc(user_param.iters * sizeof *tposted);
+
+       if (!tposted) {
+               perror("malloc");
+               return 1;
+       }
+
+       tcompleted = malloc(user_param.iters  * sizeof *tcompleted);
+
+       if (!tcompleted) {
+               perror("malloc");
+               return 1;
+       }
+
+
+
+       if (user_param.all == ALL) {
+               for (i = 1; i < 24 ; ++i) {
+                       size = 1 << i;
+                       if(run_iter(ctx, &user_param, rem_dest, size))
+                               return 17;
+                       print_report(user_param.iters, size, duplex, tposted, tcompleted);
+               }
+       } else {
+               if(run_iter(ctx, &user_param, rem_dest, size))
+                       return 18;
+               print_report(user_param.iters, size, duplex, tposted, tcompleted);
+       }
+
+       if (user_param.servername) {
+               pp_client_exch_dest(sockfd, my_dest,rem_dest);
+       } else {
+               pp_server_exch_dest(sockfd, my_dest,rem_dest);
+       }
+
+       send(sockfd, "done", sizeof "done",0);
+       closesocket(sockfd);
+
+       free(tposted);
+       free(tcompleted);
+
+       printf("------------------------------------------------------------------\n");
+       return 0;
+}
diff --git a/tools/perftests/user/read_bw/read_bw.rc b/tools/perftests/user/read_bw/read_bw.rc
new file mode 100644 (file)
index 0000000..13e5f43
--- /dev/null
@@ -0,0 +1,47 @@
+/*\r
+ * Copyright (c) 2005 Mellanox Technologies.  All rights reserved.\r
+ *\r
+ * This software is available to you under the OpenIB.org BSD license\r
+ * below:\r
+ *\r
+ *     Redistribution and use in source and binary forms, with or\r
+ *     without modification, are permitted provided that the following\r
+ *     conditions are met:\r
+ *\r
+ *      - Redistributions of source code must retain the above\r
+ *        copyright notice, this list of conditions and the following\r
+ *        disclaimer.\r
+ *\r
+ *      - Redistributions in binary form must reproduce the above\r
+ *        copyright notice, this list of conditions and the following\r
+ *        disclaimer in the documentation and/or other materials\r
+ *        provided with the distribution.\r
+ *\r
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,\r
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF\r
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND\r
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS\r
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN\r
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN\r
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\r
+ * SOFTWARE.\r
+ *\r
+ * $Id: write_bw.rc 1611 2006-08-20 14:48:55Z sleybo $\r
+ */\r
+\r
+\r
+#include <oib_ver.h>\r
+\r
+#define VER_FILETYPE                           VFT_APP\r
+#define VER_FILESUBTYPE                                VFT2_UNKNOWN\r
+\r
+#ifdef _DEBUG_\r
+#define VER_FILEDESCRIPTION_STR                "RDMA read Bandwidth Test (Debug)"\r
+#else\r
+#define VER_FILEDESCRIPTION_STR                "RDMA read Bandwidth Test "\r
+#endif\r
+\r
+#define VER_INTERNALNAME_STR           "ib_read_bw.exe"\r
+#define VER_ORIGINALFILENAME_STR       "ib_read_bw.exe"\r
+\r
+#include <common.ver>\r
index d519c1c..79e8126 100644 (file)
@@ -54,7 +54,6 @@ struct user_parameters {
        int all; /* run all msg size */
        int iters;
        int tx_depth;
-       int use_event;
 };
 
 struct report_options {
@@ -208,15 +207,6 @@ static struct pingpong_context *pp_init_ctx(unsigned int size,int port,struct us
                        }
                }
        }
-       if (user_parm->use_event) {
-//PORTED               ctx->channel = ibv_create_comp_channel(ctx->context);
-               ctx->channel = NULL;//remove when PORTED
-               if (!ctx->channel) {
-                       fprintf(stderr, "Couldn't create completion channel\n");
-                       return NULL;
-               }
-       } else
-               ctx->channel = NULL;                  
 
        ib_status = ib_alloc_pd(ctx->ca ,
                                                IB_PDT_NORMAL,
@@ -727,7 +717,7 @@ int run_iter(struct pingpong_context *ctx, struct user_parameters *user_param,
 
                        do {
                                ib_status = ib_poll_cq(ctx->rcq,&p_wc_free, &p_wc_done);
-                       } while (!user_param->use_event && ib_status == IB_NOT_FOUND);
+                       } while (ib_status == IB_NOT_FOUND);
 
                        if (ib_status != IB_SUCCESS) {
                                fprintf(stderr, "Poll Recieve CQ failed %d\n", ib_status);
@@ -781,33 +771,11 @@ int run_iter(struct pingpong_context *ctx, struct user_parameters *user_param,
                        p_wc_done = NULL;
                        p_wc_free->p_next = NULL;
 
-#if PORTED
-                       if (user_param->use_event) {
-                               struct ibv_cq *ev_cq;
-                               void          *ev_ctx;
-
-                               if (ibv_get_cq_event(ctx->channel, &ev_cq, &ev_ctx)) {
-                                       fprintf(stderr, "Failed to get send cq_event\n");
-                                       return 1;
-                               }
-
-                               if (ev_cq != ctx->scq) {
-                                       fprintf(stderr, "CQ event for unknown SCQ %p\n", ev_cq);
-                                       return 1;
-                               }
-
-                               if (ibv_req_notify_cq(ctx->scq, 0)) {
-                                       fprintf(stderr, "Couldn't request SCQ notification\n");
-                                       return 1;
-                               }
-                       }
-
-#endif
 
                        /* poll on scq */
                        do {
                                ib_status = ib_poll_cq(ctx->scq, &p_wc_free, &p_wc_done);
-                       } while (!user_param->use_event && ib_status == IB_NOT_FOUND);
+                       } while (ib_status == IB_NOT_FOUND);
 
                        if (ib_status != IB_SUCCESS) {
                                fprintf(stderr, "Poll Recieve CQ failed %d\n", ib_status);
@@ -866,7 +834,6 @@ int __cdecl main(int argc, char *argv[])
        user_param.iters = 1000;
        user_param.tx_depth = 50;
        user_param.servername = NULL;
-       user_param.use_event = 0;
        /* Parameter parsing. */
        while (1) {
                int c;
@@ -886,7 +853,6 @@ int __cdecl main(int argc, char *argv[])
                        { "report-histogram",   0,NULL, 'H' },
                        { "report-unsorted",    0,NULL, 'U' },
                        { "version",                    0,NULL, 'V' },
-                       { "events",                     0,NULL, 'e' },
                        { 0 }
                };
 
@@ -909,9 +875,6 @@ int __cdecl main(int argc, char *argv[])
                                user_param.connection_type=UD;
                        /* default is 0 for any other option RC*/
                        break;
-               case 'e':
-                       ++user_param.use_event;
-                       break;
                case 'm':
                        user_param.mtu = strtol(optarg, NULL, 0);
                        break;
index 951975c..05a4e71 100644 (file)
@@ -73,7 +73,7 @@ static struct pingpong_context *pp_init_ctx(unsigned size, int port, struct user
        ib_api_status_t                         ib_status = IB_SUCCESS; 
        size_t                                  guid_count;
        ib_net64_t                              *ca_guid_array;
-       int counter;
+       int                                             counter;
 
        ctx = malloc(sizeof *ctx);
        if (!ctx){
@@ -651,8 +651,8 @@ int __cdecl main(int argc, char *argv[])
        unsigned                        size = 65536;
        SOCKET                  sockfd = INVALID_SOCKET;
        WSADATA         wsaData;
+       int                             iResult;
        int                             i = 0;
-       int                             iResult; 
        int                             duplex = 0;
 
 
@@ -777,7 +777,6 @@ int __cdecl main(int argc, char *argv[])
          printf("                    RDMA_Write BW Test\n");
        }
        
-       printf("Inline data is used up to 400 bytes message\n");
        printf("Number of qp's running          %d\n",user_param.numofqps);
        printf("Number of iterations                    %d\n",user_param.iters);
        printf("Massege size                            %d\n",size);
@@ -787,10 +786,10 @@ int __cdecl main(int argc, char *argv[])
                printf("Connection type : UC\n");
        }
        if (user_param.maxpostsofqpiniteration > user_param.tx_depth ) {
-         printf("Can not post more than tx_depth , adjusting number of post to tx_depth\n");
-         user_param.maxpostsofqpiniteration = user_param.tx_depth;
+               printf("Can not post more than tx_depth , adjusting number of post to tx_depth\n");
+               user_param.maxpostsofqpiniteration = user_param.tx_depth;
        } else {
-         printf("Each Qp will post up to %d messages each time\n",user_param.maxpostsofqpiniteration);
+               printf("Each Qp will post up to %d messages each time\n",user_param.maxpostsofqpiniteration);
        }
        /* Done with parameter parsing. Perform setup. */