[TOOLS] add read_lat test and many cosmetics issues
authorsleybo <sleybo@ad392aa1-c5ef-ae45-8dd8-e69d62a5ef86>
Mon, 19 Feb 2007 15:46:31 +0000 (15:46 +0000)
committersleybo <sleybo@ad392aa1-c5ef-ae45-8dd8-e69d62a5ef86>
Mon, 19 Feb 2007 15:46:31 +0000 (15:46 +0000)
git-svn-id: svn://openib.tc.cornell.edu/gen1/trunk@599 ad392aa1-c5ef-ae45-8dd8-e69d62a5ef86

tools/perftests/user/dirs
tools/perftests/user/get_clock.h
tools/perftests/user/perf_defs.h
tools/perftests/user/read_lat/SOURCES [new file with mode: 0644]
tools/perftests/user/read_lat/makefile [new file with mode: 0644]
tools/perftests/user/read_lat/read_lat.c [new file with mode: 0644]
tools/perftests/user/read_lat/read_lat.rc [new file with mode: 0644]
tools/perftests/user/send_lat/send_lat.c
tools/perftests/user/write_bw/write_bw.c
tools/perftests/user/write_lat/write_lat.c

index 615798e..efc41e7 100644 (file)
@@ -3,6 +3,7 @@ DIRS=\
        send_lat \\r
        write_lat \\r
        write_bw  \\r
+  read_lat  \\r
   read_bw\r
 \r
 #      write_bw_postlist rdma_lat rdma_bw send_lat  write_lat write_bw read_lat read_bwr\r
index ee61458..d46cb2c 100644 (file)
@@ -50,6 +50,21 @@ static int __cdecl cycles_compare(const void * aptr, const void * bptr)
        return 0;
 
 }
+/*
+ * When there is an
+ *     odd number of samples, the median is the middle number.
+ *     even number of samples, the median is the mean of the
+ *             two middle numbers.
+ *
+ */
+inline cycles_t get_median(int n, cycles_t delta[])
+{
+       if (n % 2)
+               return(delta[n / 2] + delta[n / 2 - 1]) / 2;
+       else
+               return delta[n / 2];
+}
+
 
 static inline cycles_t get_cycles()
 {
index f6ce49c..0719b12 100644 (file)
 #define PINGPONG_RDMA_WRID     3
 
 
+#define SIGNAL 1
+#define MAX_INLINE 400
+
+
 #if 0
 #define PERF_ENTER     printf("%s: ===>\n",__FUNCTION__);
 #define PERF_EXIT printf("%s: <===\n",__FUNCTION__);
@@ -121,17 +125,31 @@ struct pingpong_dest {
 };
 
 
-static int pp_write_keys(SOCKET sockfd, const struct pingpong_dest *my_dest);
-static int pp_read_keys(SOCKET sockfd, struct pingpong_dest *rem_dest);
- SOCKET pp_client_connect(const char *servername, int port);
+struct report_options {
+       int unsorted;
+       int histogram;
+       int cycles;   /* report delta's in cycles, not microsec's */
+};
 
- int pp_client_exch_dest(SOCKET sockfd, const struct pingpong_dest *my_dest,
-                               struct pingpong_dest *rem_dest);
 
-SOCKET pp_server_connect(int port);
-int pp_server_exch_dest(SOCKET sockfd, const struct pingpong_dest *my_dest,
-       struct pingpong_dest* rem_dest);
+static int 
+pp_write_keys(SOCKET sockfd, const struct pingpong_dest *my_dest);
 
+static int 
+pp_read_keys(SOCKET sockfd, struct pingpong_dest *rem_dest);
 
+ SOCKET 
+ pp_client_connect(const char *servername, int port);
+
+int
+pp_client_exch_dest(SOCKET sockfd, const struct pingpong_dest *my_dest,
+                               struct pingpong_dest *rem_dest);
+
+SOCKET
+pp_server_connect(int port);
+
+int
+pp_server_exch_dest(SOCKET sockfd, const struct pingpong_dest *my_dest,
+       struct pingpong_dest* rem_dest);
 
 #endif
diff --git a/tools/perftests/user/read_lat/SOURCES b/tools/perftests/user/read_lat/SOURCES
new file mode 100644 (file)
index 0000000..25df9e4
--- /dev/null
@@ -0,0 +1,28 @@
+TARGETNAME=ib_read_lat\r
+TARGETPATH=..\..\..\..\bin\user\obj$(BUILD_ALT_DIR)\r
+TARGETTYPE=PROGRAM\r
+UMTYPE=console\r
+USE_CRTDLL=1\r
+\r
+C_DEFINES=$(C_DEFINES) /D__WIN__ \r
+\r
+SOURCES=read_lat.rc \\r
+       ..\getopt.c \\r
+       ..\perf_utils.c \\r
+       read_lat.c \r
+\r
+INCLUDES=..;..\..\..\..\inc;..\..\..\..\inc\user\r
+\r
+RCOPTIONS=/I..\..\win\include\r
+\r
+TARGETLIBS= \\r
+                       $(DDK_LIB_PATH)\Ws2_32.lib \\r
+!if $(FREEBUILD)\r
+                       $(TARGETPATH)\*\complib.lib \\r
+                       $(TARGETPATH)\*\ibal.lib\r
+!else\r
+                       $(TARGETPATH)\*\complibd.lib \\r
+                       $(TARGETPATH)\*\ibald.lib\r
+!endif\r
+\r
+MSC_WARNING_LEVEL= /W3\r
diff --git a/tools/perftests/user/read_lat/makefile b/tools/perftests/user/read_lat/makefile
new file mode 100644 (file)
index 0000000..a0c0627
--- /dev/null
@@ -0,0 +1,7 @@
+#\r
+# DO NOT EDIT THIS FILE!!!  Edit .\sources. if you want to add a new source\r
+# file to this component.  This file merely indirects to the real make file\r
+# that is shared by all the driver components of the OpenIB Windows project.\r
+#\r
+\r
+!INCLUDE ..\..\..\..\inc\openib.def\r
diff --git a/tools/perftests/user/read_lat/read_lat.c b/tools/perftests/user/read_lat/read_lat.c
new file mode 100644 (file)
index 0000000..3cb1cd8
--- /dev/null
@@ -0,0 +1,810 @@
+/*
+ * Copyright (c) 2005 Topspin Communications.  All rights reserved.
+ * Copyright (c) 2005 Mellanox Technologies Ltd.  All rights reserved.
+ * Copyright (c) 2005 Hewlett Packard, Inc (Grant Grundler)
+ *
+ * This software is available to you under a choice of one of two
+ * licenses.  You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and/or other materials
+ *        provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ * $Id: read_lat.c 1955 2007-02-19 14:46:04Z sleybo $
+ */
+
+#include "getopt.h"
+#include "perf_defs.h"
+#include "get_clock.h"
+
+
+
+
+struct user_parameters {
+       const char              *servername;
+       int connection_type;
+       int mtu;
+       int all; /* run all msg size */
+       int iters;
+       int tx_depth;
+       int max_out_read;
+};
+
+static int page_size;
+
+cycles_t                *tstamp;
+
+
+
+void
+pp_cq_comp_cb(
+       IN              const   ib_cq_handle_t                          h_cq,
+       IN                              void                                            *cq_context )
+{
+       UNUSED_PARAM( h_cq );
+       UNUSED_PARAM( cq_context);
+       return ;
+}
+
+static struct pingpong_context *pp_init_ctx(unsigned size,int port, struct user_parameters *user_parm)
+{
+
+
+       struct pingpong_context *ctx;
+       ib_api_status_t                         ib_status = IB_SUCCESS; 
+       size_t                                  guid_count;
+       ib_net64_t                              *ca_guid_array;
+
+
+       ctx = malloc(sizeof *ctx);
+       if (!ctx){
+               perror("malloc");
+               return NULL;
+       }
+       memset(ctx, 0, sizeof(struct pingpong_context));
+       ctx->size = size;
+       ctx->tx_depth = user_parm->tx_depth;
+
+       ctx->qp = malloc(sizeof (ib_qp_handle_t));
+       if (!ctx->qp) {
+               perror("malloc");
+               return NULL;
+       }
+       
+       ctx->qp_attr = malloc(sizeof (ib_qp_attr_t));
+       if (!ctx->qp_attr) {
+               perror("malloc");
+               return NULL;
+       }
+
+       ctx->buf = malloc( size * 2);
+       if (!ctx->buf) {
+               fprintf(stderr, "Couldn't allocate work buf.\n");
+               return NULL;
+       }
+
+       memset(ctx->buf, 0, size * 2);
+
+
+       /*
+        * Open the AL instance
+        */
+       ib_status = ib_open_al(&ctx->al);
+       if(ib_status != IB_SUCCESS)
+       {
+               fprintf(stderr,"ib_open_al failed status = %d\n", ib_status);
+               return NULL;
+       }
+
+       /*
+        * Get the Local CA Guids
+        */
+       ib_status = ib_get_ca_guids(ctx->al, NULL, &guid_count);
+       if(ib_status != IB_INSUFFICIENT_MEMORY)
+       {
+               fprintf(stderr,"ib_get_ca_guids1 failed status = %d\n", (uint32_t)ib_status);
+               return NULL;
+       }
+       
+       /*
+        * If no CA's Present then return
+        */
+
+       if(guid_count == 0)
+               return NULL;
+
+       
+       ca_guid_array = (ib_net64_t*)malloc(sizeof(ib_net64_t) * guid_count);
+       
+       ib_status = ib_get_ca_guids(ctx->al, ca_guid_array, &guid_count);
+       if(ib_status != IB_SUCCESS)
+       {
+               fprintf(stderr,"ib_get_ca_guids2 failed with status = %d\n", ib_status);
+               return NULL;
+       }
+
+       /*
+        * Open only the first HCA
+        */
+       /* Open the CA */
+       ib_status = ib_open_ca(ctx->al ,ca_guid_array[0] ,NULL,
+               NULL,   //ca_context
+               &ctx->ca);
+
+       if(ib_status != IB_SUCCESS)
+       {
+               fprintf(stderr,"ib_open_ca failed with status = %d\n", ib_status);
+               return NULL;
+       }
+
+       //xxx
+       //printf("ib_open_ca passed i=%d\n",i); 
+       //xxx
+
+
+       
+
+       {
+               /* Query the CA */
+               uint32_t bsize = 0;
+               ib_status = ib_query_ca(ctx->ca, NULL, &bsize);
+               if(ib_status != IB_INSUFFICIENT_MEMORY)
+               {
+                       fprintf(stderr, "Failed to query device props");
+                       return NULL;
+               }
+
+               ctx->ca_attr = (ib_ca_attr_t *)malloc(bsize);
+
+               ib_status = ib_query_ca(ctx->ca, ctx->ca_attr, &bsize);
+               if(ib_status != IB_SUCCESS)
+               {
+                       printf("ib_query_ca failed with status = %d\n", ib_status);
+                       return NULL;
+               }
+               if (user_parm->mtu == 0) {/*user did not ask for specific mtu */
+                       if (ctx->ca_attr->dev_id == 23108) {
+                               user_parm->mtu = 1024;
+                       } else {
+                               user_parm->mtu = 2048;
+                       }
+               }
+       }
+
+
+
+       ib_status = ib_alloc_pd(ctx->ca ,
+                                               IB_PDT_NORMAL,
+                                               ctx, //pd_context
+                                               &ctx->pd);
+       if (ib_status != IB_SUCCESS) {
+               fprintf(stderr, "Couldn't allocate PD\n");
+               return NULL;
+       }
+
+
+       /* We dont really want IBV_ACCESS_LOCAL_WRITE, but IB spec says:
+        * The Consumer is not allowed to assign Remote Write or Remote Atomic to
+        * a Memory Region that has not been assigned Local Write. */
+
+
+       {
+               ib_mr_create_t                  mr_create;
+
+               mr_create.length = size * 2;
+                       
+               mr_create.vaddr = ctx->buf;
+               mr_create.access_ctrl = IB_AC_RDMA_WRITE| IB_AC_LOCAL_WRITE|IB_AC_RDMA_READ;
+               
+               ib_status = ib_reg_mem(ctx->pd ,&mr_create ,&ctx->lkey ,&ctx->rkey ,&ctx->mr);
+               if (ib_status != IB_SUCCESS) {
+                       fprintf(stderr, "Couldn't allocate MR\n");
+                       return NULL;
+               }
+       }
+       
+       {
+               ib_cq_create_t                  cq_create;
+               
+               cq_create.size = user_parm->tx_depth;
+               cq_create.h_wait_obj = NULL;
+               cq_create.pfn_comp_cb = pp_cq_comp_cb;
+               ib_status = ib_create_cq(ctx->ca,&cq_create ,ctx, NULL, &ctx->scq);
+               if (ib_status != IB_SUCCESS) {
+                       fprintf(stderr, "Couldn't create CQ ib_status = %d\n",ib_status);
+                       return NULL;
+               }
+       }
+
+       {
+
+               ib_qp_create_t  qp_create;
+               ib_qp_mod_t     qp_modify;
+               ib_qp_attr_t    qp_attr;
+               
+               memset(&qp_create, 0, sizeof(ib_qp_create_t));
+               qp_create.h_sq_cq       = ctx->scq;
+               qp_create.h_rq_cq       = ctx->scq;
+               qp_create.sq_depth      = user_parm->tx_depth;
+               qp_create.rq_depth      = 1;
+               qp_create.sq_sge        = 1;
+               qp_create.rq_sge        = 1;
+               //TODO MAX_INLINE
+               qp_create.qp_type= IB_QPT_RELIABLE_CONN;
+               qp_create.sq_signaled = FALSE;
+               /*attr.sq_sig_all = 0;*/
+
+               ib_status = ib_create_qp(ctx->pd, &qp_create,NULL,NULL,&ctx->qp[0]);
+               if (ib_status != IB_SUCCESS){
+                       fprintf(stderr, "Couldn't create QP\n");
+                       return NULL;
+               }
+
+       
+
+       
+               memset(&qp_modify, 0, sizeof(ib_qp_mod_t));
+               qp_modify.req_state = IB_QPS_INIT;
+               qp_modify.state.init.pkey_index = 0 ;
+               qp_modify.state.init.primary_port = (uint8_t)port;
+               qp_modify.state.init.access_ctrl = IB_AC_RDMA_WRITE | IB_AC_LOCAL_WRITE|IB_AC_RDMA_READ;
+
+               
+               ib_status = ib_modify_qp(ctx->qp[0], &qp_modify);
+               if (ib_status != IB_SUCCESS){
+                       fprintf(stderr, "Failed to modify QP to INIT\n");
+                       return NULL;
+               }
+
+
+               memset(&qp_attr, 0, sizeof(ib_qp_attr_t));
+               ib_status = ib_query_qp(ctx->qp[0], &ctx->qp_attr[0]);
+               if (ib_status != IB_SUCCESS){
+                       fprintf(stderr, "Failed to modify QP to INIT\n");
+                       return NULL;
+               }
+               fprintf(stderr, "max inline size %d\n",ctx->qp_attr[0].sq_max_inline);
+       }
+       return ctx;
+}
+
+               
+
+       
+
+static int pp_connect_ctx(struct pingpong_context *ctx, int port, int my_psn,
+                         struct pingpong_dest *dest, struct user_parameters *user_parm,int qpindex)
+{
+       
+       ib_api_status_t ib_status;
+       ib_qp_mod_t     attr;
+       memset(&attr, 0, sizeof(ib_qp_mod_t));
+
+       attr.req_state          = IB_QPS_RTR;
+       switch (user_parm->mtu) {
+       case 256 : 
+               attr.state.rtr.primary_av.conn.path_mtu = IB_MTU_LEN_256;
+               break;
+       case 512 :
+               attr.state.rtr.primary_av.conn.path_mtu = IB_MTU_LEN_512;
+               break;
+       case 1024 :
+               attr.state.rtr.primary_av.conn.path_mtu = IB_MTU_LEN_1024;
+               break;
+       case 2048 :
+               attr.state.rtr.primary_av.conn.path_mtu = IB_MTU_LEN_2048;
+               break;
+       }
+       printf("Mtu : %d\n", user_parm->mtu);
+       attr.state.rtr.dest_qp  = dest->qpn;;
+       attr.state.rtr.rq_psn   = dest->psn;
+       attr.state.rtr.resp_res = (uint8_t)user_parm->max_out_read;
+       attr.state.rtr.rnr_nak_timeout = 12;
+       attr.state.rtr.primary_av.grh_valid = 0;
+       attr.state.rtr.primary_av.dlid = dest->lid;
+       attr.state.rtr.primary_av.sl = 0;
+       attr.state.rtr.primary_av.path_bits = 0;
+       attr.state.rtr.primary_av.port_num = (uint8_t)port;
+       attr.state.rtr.primary_av.static_rate = IB_PATH_RECORD_RATE_10_GBS;
+       attr.state.rtr.opts = IB_MOD_QP_LOCAL_ACK_TIMEOUT |
+                                       IB_MOD_QP_RESP_RES |
+                                       IB_MOD_QP_PRIMARY_AV;
+
+
+       ib_status = ib_modify_qp(ctx->qp[0], &attr);
+       if(ib_status != IB_SUCCESS){
+                       fprintf(stderr, "Failed to modify QP to RTR\n");
+                       return 1;
+       }
+
+
+
+       memset(&attr, 0, sizeof(ib_qp_mod_t));
+       attr.req_state  = IB_QPS_RTS;
+       attr.state.rts.sq_psn = my_psn;
+
+       attr.state.rts.init_depth = (uint8_t)user_parm->max_out_read;
+       attr.state.rts.local_ack_timeout = 14;
+       attr.state.rts.retry_cnt = 7;
+       attr.state.rts.rnr_retry_cnt = 7;
+       attr.state.rts.opts = IB_MOD_QP_RNR_RETRY_CNT |
+                                       IB_MOD_QP_RETRY_CNT |
+                                       IB_MOD_QP_INIT_DEPTH |
+                                       IB_MOD_QP_LOCAL_ACK_TIMEOUT;
+                                               
+       ib_status = ib_modify_qp(ctx->qp[0], &attr);
+       if(ib_status != IB_SUCCESS){
+               fprintf(stderr, "Failed to modify QP to RTS\n");
+               return 1;
+       }
+
+       return 0;
+}
+
+
+static SOCKET pp_open_port(struct pingpong_context *ctx, const char * servername,
+                       int ib_port, int port, struct pingpong_dest **p_my_dest,
+                       struct pingpong_dest **p_rem_dest,struct user_parameters *user_parm)
+{
+       //char addr_fmt[] = "%8s address: LID %#04x QPN %#06x PSN %#06x RKey %#08x VAddr %#016Lx\n";
+       struct pingpong_dest    *my_dest;
+       struct pingpong_dest    *rem_dest;
+       SOCKET                          sockfd;
+       int                                     rc;
+       int                                     i;
+       int                                     numofqps = 1;
+       
+       /* Create connection between client and server.
+        * We do it by exchanging data over a TCP socket connection. */
+
+       
+       my_dest = malloc( sizeof (struct pingpong_dest) * numofqps);
+       if (!my_dest){
+               perror("malloc");
+               return INVALID_SOCKET;
+       }
+
+       rem_dest = malloc(sizeof (struct pingpong_dest) * numofqps );
+       if (!rem_dest){
+               perror("malloc");
+               return INVALID_SOCKET;
+       }
+
+       sockfd = servername ? pp_client_connect(servername, port) :
+               pp_server_connect(port);
+
+       if (sockfd  == INVALID_SOCKET) {
+               printf("pp_connect_sock(%s,%d) failed (%d)!\n",
+                      servername, port, sockfd);
+               return INVALID_SOCKET;
+       }
+       
+       
+       for (i =0 ;i<numofqps;i ++) 
+       {
+               /* Create connection between client and server.
+               * We do it by exchanging data over a TCP socket connection. */
+               
+               my_dest[i].lid = ctx->ca_attr->p_port_attr[ib_port-1].lid;
+               my_dest[i].psn = rand() & 0xffffff;
+               if (!my_dest[i].lid) {
+                       fprintf(stderr, "Local lid 0x0 detected. Is an SM running?\n");
+                       return 1;
+               }
+               my_dest[i].qpn = ctx->qp_attr[i].num;
+               /* TBD this should be changed inot VA and different key to each qp */
+               my_dest[i].rkey = ctx->rkey;
+               my_dest[i].vaddr = (uintptr_t)ctx->buf + ctx->size;
+
+               printf("  local address:  LID %#04x, QPN %#06x, PSN %#06x, "
+               "RKey %#08x VAddr %#016Lx\n",
+               my_dest[i].lid, my_dest[i].qpn, my_dest[i].psn,
+               my_dest[i].rkey, my_dest[i].vaddr);
+
+               rc = servername ? pp_client_exch_dest(sockfd, &my_dest[i],&rem_dest[i]):
+                                               pp_server_exch_dest(sockfd, &my_dest[i],&rem_dest[i]);
+               if (rc)
+                       return INVALID_SOCKET;
+               printf("  remote address: LID %#04x, QPN %#06x, PSN %#06x, "
+               "RKey %#08x VAddr %#016Lx\n",
+               rem_dest[i].lid, rem_dest[i].qpn, rem_dest[i].psn,
+               rem_dest[i].rkey, rem_dest[i].vaddr);
+
+               if (pp_connect_ctx(ctx, ib_port, my_dest[i].psn, &rem_dest[i], user_parm, i))
+                       return INVALID_SOCKET;
+               /* An additional handshake is required *after* moving qp to RTR.
+               Arbitrarily reuse exch_dest for this purpose. */
+               rc = servername ? pp_client_exch_dest(sockfd, &my_dest[i],&rem_dest[i]):
+                                               pp_server_exch_dest(sockfd, &my_dest[i],&rem_dest[i]);
+               if (rc)
+                       return INVALID_SOCKET;
+       }
+       *p_rem_dest = rem_dest;
+       *p_my_dest = my_dest;
+       return sockfd;
+}
+
+static void usage(const char *argv0)
+{
+       printf("Usage:\n");
+       printf("  %s            start a server and wait for connection\n", argv0);
+       printf("  %s <host>     connect to server at <host>\n", argv0);
+       printf("\n");
+       printf("Options:\n");
+       printf("  -p, --port=<port>            listen on/connect to port <port> (default 18515)\n");
+       printf("  -c, --connection=<RC/UC>     connection type RC/UC (default RC)\n");
+       printf("  -m, --mtu=<mtu>              mtu size (default 256)\n");
+       printf("  -d, --ib-dev=<dev>           use IB device <dev> (default first device found)\n");
+       printf("  -i, --ib-port=<port>         use port <port> of IB device (default 1)\n");
+       printf("  -s, --size=<size>            size of message to exchange (default 1)\n");
+       printf("  -t, --tx-depth=<dep>         size of tx queue (default 50)\n");
+       printf("  -n, --iters=<iters>          number of exchanges (at least 2, default 1000)\n");
+       printf("  -o, --outs=<num>             num of outstanding read/atom(default 4)\n");
+       printf("  -a, --all                    Run sizes from 2 till 2^23\n");
+       printf("  -C, --report-cycles          report times in cpu cycle units (default microseconds)\n");
+       printf("  -H, --report-histogram       print out all results (default print summary only)\n");
+       printf("  -U, --report-unsorted        (implies -H) print out unsorted results (default sorted)\n");
+       printf("  -V, --version                display version number\n");
+       printf("  -e, --events                 sleep on CQ events (default poll)\n");
+}
+
+
+
+static void print_report(struct report_options * options,
+                        unsigned int iters, cycles_t *tstamp,int size)
+{
+       double cycles_to_units;
+       cycles_t median;
+       unsigned int i;
+       const char* units;
+       cycles_t *delta = malloc(iters * sizeof *delta);
+
+       if (!delta) {
+               perror("malloc");
+               return;
+       }
+
+       for (i = 0; i < iters - 1; ++i)
+               delta[i] = tstamp[i + 1] - tstamp[i];
+
+
+       if (options->cycles) {
+               cycles_to_units = 1;
+               units = "cycles";
+       } else {
+               cycles_to_units = get_cpu_mhz()/1000000;
+               units = "usec";
+       }
+
+       if (options->unsorted) {
+               printf("#, %s\n", units);
+               for (i = 0; i < iters - 1; ++i)
+                       printf("%d, %g\n", i + 1, delta[i] / cycles_to_units );
+       }
+
+       qsort(delta, iters - 1, sizeof *delta, cycles_compare);
+
+       if (options->histogram) {
+               printf("#, %s\n", units);
+               for (i = 0; i < iters - 1; ++i)
+                       printf("%d, %g\n", i + 1, delta[i] / cycles_to_units );
+       }
+
+       median = get_median(iters - 1, delta);
+       printf("%7d        %d        %7.2f        %7.2f          %7.2f\n",
+              size,iters,delta[0] / cycles_to_units ,
+              delta[iters - 3] / cycles_to_units ,median / cycles_to_units );
+
+       free(delta);
+}
+
+int run_iter(struct pingpong_context *ctx, struct user_parameters *user_param,
+            struct pingpong_dest *rem_dest, int size)
+{
+       ib_api_status_t         ib_status;
+       int                                     scnt, ccnt ;
+       ib_send_wr_t                    *bad_wr;
+
+       ctx->list.vaddr = (uintptr_t) ctx->buf ;
+       ctx->list.length = size;
+       ctx->list.lkey = ctx->lkey;
+       ctx->wr.remote_ops.vaddr = rem_dest->vaddr;
+       ctx->wr.remote_ops.rkey = rem_dest->rkey;
+       ctx->wr.wr_id      = PINGPONG_RDMA_WRID;
+       ctx->wr.ds_array    = &ctx->list;
+       ctx->wr.num_ds    = 1;
+       ctx->wr.wr_type     = WR_RDMA_READ;
+       ctx->wr.send_opt = IB_SEND_OPT_SIGNALED;
+       ctx->wr.p_next       = NULL;
+       
+       scnt = 0;
+       ccnt = 0;
+
+
+       /* Done with setup. Start the test. */
+
+       if(user_param->servername) {
+               while (scnt < user_param->iters ) {
+
+                       tstamp[scnt] = get_cycles();
+                       ib_status = ib_post_send(ctx->qp[0], &ctx->wr, &bad_wr);
+                       if (ib_status != IB_SUCCESS) 
+                       {
+                               fprintf(stderr, "Couldn't post send: scnt %d ccnt %d\n",scnt,ccnt);
+                               return 1;
+                       }
+                       ++scnt;
+                       PERF_DEBUG("scnt = %d \n",scnt);
+
+                       {
+                               ib_wc_t wc;
+                               ib_wc_t *p_wc_done,*p_wc_free;
+
+                               p_wc_free = &wc;
+                               p_wc_done = NULL;
+                               p_wc_free->p_next = NULL;
+
+                               do{
+                                       ib_status = ib_poll_cq(ctx->scq, &p_wc_free, &p_wc_done);
+                               } while (ib_status == IB_NOT_FOUND);
+                               
+                               if (ib_status != IB_SUCCESS) {
+                                       fprintf(stderr, "Poll Send CQ failed %d\n", ib_status);
+                                       return 12;
+                               }
+
+                               if (p_wc_done->status != IB_WCS_SUCCESS) {
+                                       fprintf(stderr, "Completion wth error at %s:\n",
+                                       user_param->servername ? "client" : "server");
+                                       fprintf(stderr, "Failed status %d: wr_id %d syndrom 0x%x\n",
+                                       p_wc_done->status, (int) p_wc_done->wr_id, p_wc_done->vendor_specific);
+                                       return 1;
+                               }
+                               ++ccnt;
+                               PERF_DEBUG("ccnt = %d \n",ccnt);
+                       
+                       }
+
+               } 
+       }
+       return(0);
+}
+
+
+int __cdecl main(int argc, char *argv[])
+{
+       struct pingpong_context *ctx;
+       struct pingpong_dest     *my_dest;
+       struct pingpong_dest     *rem_dest;
+       struct user_parameters  user_param;
+       char                            *ib_devname = NULL;
+       int                             port = 18515;
+       int                             ib_port = 1;
+       unsigned                        tmp_size,size = 2;
+       SOCKET                  sockfd = INVALID_SOCKET;
+       WSADATA                 wsaData;
+       int                             iResult; 
+       int                             i = 0;
+       struct report_options    report = {0};
+
+
+       /* init default values to user's parameters */
+       memset(&user_param, 0, sizeof(struct user_parameters));
+       user_param.mtu = 0; /* signal choose default by device */
+       user_param.iters = 1000;
+       user_param.tx_depth = 50;
+       user_param.servername = NULL;
+       user_param.connection_type = RC;
+       user_param.max_out_read = 4; /* the device capability on gen2 */
+/* Parameter parsing. */
+       while (1) {
+               int c;
+
+               static struct option long_options[] = {
+                       { "port",                               1, NULL, 'p' },
+                       { "mtu",                                1, NULL, 'm' },
+                       { "outs",                               1, NULL, 'o' },
+                       {  "ib-dev",                    1, NULL, 'd' },
+                       { "ib-port",                    1, NULL, 'i' },
+                       { "size",                               1, NULL, 's' },
+                       { "iters",                              1, NULL, 'n' },
+                       { "tx-depth",                   1, NULL, 't' },
+                       { "all",                                0, NULL, 'a' },
+                       { "report-cycles",              0, NULL, 'C' },
+                       { "report-histogram",   0, NULL, 'H' },
+                       { "report-unsorted",    0, NULL, 'U' },
+                       { "version",                    0, NULL, 'V' },
+                       { 0 }
+               };
+       
+
+               c = getopt_long(argc, argv, "p:m:d:i:s:n:t:aHUV", long_options, NULL);
+               if (c == -1)
+                       break;
+
+               switch (c) {
+               case 'p':
+                       port = strtol(optarg, NULL, 0);
+                       if (port < 0 || port > 65535) {
+                               usage(argv[0]);
+                               return 1;
+                       }
+                       break;
+               case 'm':
+                       user_param.mtu = strtol(optarg, NULL, 0);
+                       break;
+               case 'o':
+                       user_param.max_out_read = strtol(optarg, NULL, 0);
+                       break;
+               case 'a':
+                       user_param.all = ALL;
+                       break;
+               case 'V':
+                       printf("read_lat version : %.2f\n",VERSION);
+                       return 0;
+                       break;
+               case 'd':
+                       ib_devname = _strdup(optarg);
+                       break;
+
+               case 'i':
+                       ib_port = strtol(optarg, NULL, 0);
+                       if (ib_port < 0) {
+                               usage(argv[0]);
+                               return 2;
+                       }
+                       break;
+
+               case 's':
+                       size = strtol(optarg, NULL, 0);
+                       if (size < 1) {
+                               usage(argv[0]); return 3;
+                       }
+                       break;
+
+               case 't':
+                       user_param.tx_depth = strtol(optarg, NULL, 0);
+                       if (user_param.tx_depth < 1) {
+                               usage(argv[0]); return 4;
+                       }
+                       break;
+
+               case 'n':
+                       user_param.iters = strtol(optarg, NULL, 0);
+                       if (user_param.iters < 2) {
+                               usage(argv[0]);
+                               return 5;
+                       }
+
+                       break;
+
+               case 'C':
+                       report.cycles = 1;
+                       break;
+
+               case 'H':
+                       report.histogram = 1;
+                       break;
+
+               case 'U':
+                       report.unsorted = 1;
+                       break;
+
+               default:
+                       usage(argv[0]);
+                       return 5;
+               }
+       }
+
+       if (optind == argc - 1)
+               user_param.servername = _strdup(argv[optind]);
+       else if (optind < argc) {
+               usage(argv[0]);
+               return 6;
+       }
+
+       /*
+        *  Done with parameter parsing. Perform setup.
+        */
+       tstamp = malloc(user_param.iters * sizeof *tstamp);
+       if (!tstamp) {
+               perror("malloc");
+               return 10;
+       }
+       printf("------------------------------------------------------------------\n");
+       printf("                    RDMA_Read Latency Test\n");
+
+
+       // Initialize Winsock
+       iResult = WSAStartup(MAKEWORD(2,2), &wsaData);
+       if (iResult != NO_ERROR) {
+               printf("Error at WSAStartup()\n");
+               return 1;
+       }
+
+       tmp_size = size;
+       /* anyway make sure the connection is RC */
+       if (user_param.all == ALL) {
+               /*since we run all sizes */
+               size = 8388608; /*2^23 */
+       } else if (size < 128) {
+               /* can cut up to 70 nsec probably related to cache line size */        
+               size = 128;
+       }
+       
+       srand(GetCurrentProcessId() * GetTickCount());
+
+       //TODO: get pagesize from sysinfo
+       page_size = 4096;
+
+       //TODO:get the device names
+
+
+       ctx = pp_init_ctx(size,ib_port, &user_param);
+       if (!ctx)
+               return 8;
+
+       
+       sockfd = pp_open_port(ctx, user_param.servername, ib_port, port,&my_dest,&rem_dest,&user_param);
+       if (sockfd == INVALID_SOCKET)
+               return 9;
+
+       
+       printf("------------------------------------------------------------------\n");
+       printf(" #bytes #iterations    t_min[usec]    t_max[usec]  t_typical[usec]\n");
+       /* For half duplex tests, server just waits for client to exit */
+       /* use dummy my_dest struct*/
+       if (!user_param.servername) {
+               pp_server_exch_dest(sockfd, my_dest,rem_dest);
+               send(sockfd, "done", sizeof "done",0);
+               closesocket(sockfd);
+               return 0; 
+       }
+
+       /* fix for true size in small msg size */
+       if (tmp_size < 128) {
+               size = tmp_size ;
+       }
+
+       if (user_param.all == ALL) {
+               for (i = 1; i < 24 ; ++i) {
+                       size = 1 << i;
+                       if(run_iter(ctx, &user_param, rem_dest, size))
+                               return 17;
+                       print_report(&report, user_param.iters, tstamp, size);
+               }
+       } else {
+               if(run_iter(ctx, &user_param, rem_dest, size))
+                       return 18;
+               print_report(&report, user_param.iters, tstamp, size);
+       }
+
+       pp_client_exch_dest(sockfd, my_dest,rem_dest);
+       send(sockfd, "done", sizeof "done",0);
+       closesocket(sockfd);
+
+       printf("------------------------------------------------------------------\n");
+       free(tstamp);
+       return 0;
+}
diff --git a/tools/perftests/user/read_lat/read_lat.rc b/tools/perftests/user/read_lat/read_lat.rc
new file mode 100644 (file)
index 0000000..2dd3fb9
--- /dev/null
@@ -0,0 +1,47 @@
+/*\r
+ * Copyright (c) 2005 Mellanox Technologies.  All rights reserved.\r
+ *\r
+ * This software is available to you under the OpenIB.org BSD license\r
+ * below:\r
+ *\r
+ *     Redistribution and use in source and binary forms, with or\r
+ *     without modification, are permitted provided that the following\r
+ *     conditions are met:\r
+ *\r
+ *      - Redistributions of source code must retain the above\r
+ *        copyright notice, this list of conditions and the following\r
+ *        disclaimer.\r
+ *\r
+ *      - Redistributions in binary form must reproduce the above\r
+ *        copyright notice, this list of conditions and the following\r
+ *        disclaimer in the documentation and/or other materials\r
+ *        provided with the distribution.\r
+ *\r
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,\r
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF\r
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND\r
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS\r
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN\r
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN\r
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\r
+ * SOFTWARE.\r
+ *\r
+ * $Id: write_bw.rc 1611 2006-08-20 14:48:55Z sleybo $\r
+ */\r
+\r
+\r
+#include <oib_ver.h>\r
+\r
+#define VER_FILETYPE                           VFT_APP\r
+#define VER_FILESUBTYPE                                VFT2_UNKNOWN\r
+\r
+#ifdef _DEBUG_\r
+#define VER_FILEDESCRIPTION_STR                "RDMA read Latency Test (Debug)"\r
+#else\r
+#define VER_FILEDESCRIPTION_STR                "RDMA read Latency Test "\r
+#endif\r
+\r
+#define VER_INTERNALNAME_STR           "ib_read_lat.exe"\r
+#define VER_ORIGINALFILENAME_STR       "ib_read_lat.exe"\r
+\r
+#include <common.ver>\r
index 79e8126..28bbf92 100644 (file)
@@ -39,9 +39,6 @@
 #include "perf_defs.h"
 
 
-#define SIGNAL 1
-#define MAX_INLINE 400
-
 
 static int page_size;
 cycles_t                *tstamp;
@@ -56,12 +53,6 @@ struct user_parameters {
        int tx_depth;
 };
 
-struct report_options {
-       int unsorted;
-       int histogram;
-       int cycles;   /* report delta's in cycles, not microsec's */
-};
-
 
 
 void
@@ -558,20 +549,6 @@ static void usage(const char *argv0)
     printf("  -e, --events                 sleep on CQ events (default poll)\n");
 }
 
-/*
- * When there is an
- *     odd number of samples, the median is the middle number.
- *     even number of samples, the median is the mean of the
- *             two middle numbers.
- *
- */
-static inline cycles_t get_median(int n, cycles_t delta[])
-{
-       if (n % 2)
-               return(delta[n / 2] + delta[n / 2 - 1]) / 2;
-       else
-               return delta[n / 2];
-}
 
 
 static void print_report(struct report_options * options,
index 05a4e71..1931c68 100644 (file)
@@ -37,8 +37,7 @@
 #include "perf_defs.h"
 #include "get_clock.h"
 
-#define RC 0
-#define UC 1
+
 
 struct user_parameters {
        const char              *servername;
@@ -237,7 +236,7 @@ static struct pingpong_context *pp_init_ctx(unsigned size, int port, struct user
                cq_create.pfn_comp_cb = pp_cq_comp_cb;
                ib_status = ib_create_cq(ctx->ca,&cq_create ,ctx, NULL, &ctx->scq);
                if (ib_status != IB_SUCCESS) {
-                       fprintf(stderr, "Couldn't create CQ\n");
+                       fprintf(stderr, "Couldn't create CQ ib_status = %d\n",ib_status);
                        return NULL;
                }
        }
@@ -284,6 +283,7 @@ static struct pingpong_context *pp_init_ctx(unsigned size, int port, struct user
        
 
        
+       
                memset(&qp_modify, 0, sizeof(ib_qp_mod_t));
                qp_modify.req_state = IB_QPS_INIT;
                qp_modify.state.init.pkey_index = 0 ;
@@ -363,12 +363,13 @@ static int pp_connect_ctx(struct pingpong_context *ctx, int port, int my_psn,
        attr.state.rts.sq_psn = my_psn;
 
        if (user_parm->connection_type == RC) {
-               attr.state.rts.resp_res = 1;
+               attr.state.rts.init_depth = 1;
                attr.state.rts.local_ack_timeout = 14;
                attr.state.rts.retry_cnt = 7;
                attr.state.rts.rnr_retry_cnt = 7;
                attr.state.rts.opts = IB_MOD_QP_RNR_RETRY_CNT |
                                                IB_MOD_QP_RETRY_CNT |
+                                               IB_MOD_QP_INIT_DEPTH |
                                                IB_MOD_QP_LOCAL_ACK_TIMEOUT;
                                                
        }       
@@ -727,7 +728,7 @@ int __cdecl main(int argc, char *argv[])
                        ib_port = strtol(optarg, NULL, 0);
                        if (ib_port < 0) {
                                usage(argv[0]);
-                               return 1;
+                               return 2;
                        }
                        break;
 
@@ -741,14 +742,16 @@ int __cdecl main(int argc, char *argv[])
 
                case 't':
                        user_param.tx_depth = strtol(optarg, NULL, 0);
-                       if (user_param.tx_depth < 1) { usage(argv[0]); return 1; }
+                       if (user_param.tx_depth < 1) {
+                               usage(argv[0]); return 4;
+                       }
                        break;
 
                case 'n':
                        user_param.iters = strtol(optarg, NULL, 0);
                        if (user_param.iters < 2) {
                                usage(argv[0]);
-                               return 1;
+                               return 5;
                        }
 
                        break;
@@ -759,7 +762,7 @@ int __cdecl main(int argc, char *argv[])
 
                default:
                        usage(argv[0]);
-                       return 1;
+                       return 5;
                }
        }
 
@@ -767,7 +770,7 @@ int __cdecl main(int argc, char *argv[])
                user_param.servername = _strdup(argv[optind]);
        else if (optind < argc) {
                usage(argv[0]);
-               return 1;
+               return 6;
        }
        
        printf("------------------------------------------------------------------\n");
@@ -802,22 +805,20 @@ int __cdecl main(int argc, char *argv[])
 
        
        if (user_param.all == ALL) {
-               /*since we run all sizes */
+               /*since we run all sizes lets allocate big enough buffer */
                size = 8388608; /*2^23 */
        }
-       
        srand(GetCurrentProcessId() * GetTickCount());
 
        //TODO: get pagesize from sysinfo
        page_size = 4096;
 
-       //TODO:get the device names
+       //TODO  get the device names
 
 
        ctx = pp_init_ctx(size, ib_port, &user_param);
        if (!ctx)
-               return 1;
-
+               return 8;
        sockfd = pp_open_port(ctx, user_param.servername, ib_port, port,&my_dest,&rem_dest,&user_param);
        if (sockfd == INVALID_SOCKET)
                return 9;
index fe1c128..329cbe2 100644 (file)
 #include "get_clock.h"
 
 
-#define RC 0
-#define UC 1
 
-
-static int             page_size;
-cycles_t               *tstamp;
 struct user_parameters {
        const char              *servername;
        int connection_type;
@@ -54,11 +49,10 @@ struct user_parameters {
        int tx_depth;
 };
 
-struct report_options {
-       int unsorted;
-       int histogram;
-       int cycles;   /* report delta's in cycles, not microsec's */
-};
+static int page_size;
+
+cycles_t                *tstamp;
+
 
 
 void
@@ -232,15 +226,11 @@ static struct pingpong_context *pp_init_ctx(unsigned size, int port, struct user
                cq_create.pfn_comp_cb = pp_cq_comp_cb;
                ib_status = ib_create_cq(ctx->ca,&cq_create ,ctx, NULL, &ctx->scq);
                if (ib_status != IB_SUCCESS) {
-                       fprintf(stderr, "Couldn't create CQ\n");
+                       fprintf(stderr, "Couldn't create CQ ib_status = %d\n",ib_status);
                        return NULL;
                }
        }
        
-
-
-       
-
        {
 
                ib_qp_create_t  qp_create;
@@ -279,6 +269,7 @@ static struct pingpong_context *pp_init_ctx(unsigned size, int port, struct user
        
 
        
+       
                memset(&qp_modify, 0, sizeof(ib_qp_mod_t));
                qp_modify.req_state = IB_QPS_INIT;
                qp_modify.state.init.pkey_index = 0 ;
@@ -302,13 +293,6 @@ static struct pingpong_context *pp_init_ctx(unsigned size, int port, struct user
                fprintf(stderr, "max inline size %d\n",ctx->qp_attr[0].sq_max_inline);
        }
 
-       
-       ctx->wr.wr_id      = PINGPONG_RDMA_WRID;
-       ctx->wr.ds_array    = &ctx->list;
-       ctx->wr.num_ds    = 1;
-       ctx->wr.wr_type     = WR_RDMA_WRITE;
-       ctx->wr.p_next       = NULL;
-       
        return ctx;
 }
 
@@ -316,7 +300,7 @@ static struct pingpong_context *pp_init_ctx(unsigned size, int port, struct user
 
 
 static int pp_connect_ctx(struct pingpong_context *ctx, int port, int my_psn,
-                         struct pingpong_dest *dest, struct user_parameters *user_parm,int index)
+                         struct pingpong_dest *dest, struct user_parameters *user_parm,int qpindex)
 {
 
        ib_api_status_t ib_status;
@@ -356,7 +340,7 @@ static int pp_connect_ctx(struct pingpong_context *ctx, int port, int my_psn,
                                        IB_MOD_QP_PRIMARY_AV;
 
 
-       ib_status = ib_modify_qp(ctx->qp[index], &attr);
+       ib_status = ib_modify_qp(ctx->qp[0], &attr);
        if(ib_status != IB_SUCCESS){
                        fprintf(stderr, "Failed to modify QP to RTR\n");
                        return 1;
@@ -367,16 +351,17 @@ static int pp_connect_ctx(struct pingpong_context *ctx, int port, int my_psn,
        attr.state.rts.sq_psn = my_psn;
 
        if (user_parm->connection_type == RC) {
-               attr.state.rts.resp_res = 1;
+               attr.state.rts.init_depth = 1;
                attr.state.rts.local_ack_timeout = 14;
                attr.state.rts.retry_cnt = 7;
                attr.state.rts.rnr_retry_cnt = 7;
                attr.state.rts.opts = IB_MOD_QP_RNR_RETRY_CNT |
                                                IB_MOD_QP_RETRY_CNT |
+                                               IB_MOD_QP_INIT_DEPTH |
                                                IB_MOD_QP_LOCAL_ACK_TIMEOUT;
                                                
-       }       
-       ib_status = ib_modify_qp(ctx->qp[index], &attr);
+       }               
+       ib_status = ib_modify_qp(ctx->qp[0], &attr);
        if(ib_status != IB_SUCCESS){
                fprintf(stderr, "Failed to modify QP to RTS\n");
                return 1;
@@ -490,20 +475,6 @@ static void usage(const char *argv0)
        printf("  -V, --version                display version number\n");
 }
 
-/*
- * When there is an
- *     odd number of samples, the median is the middle number.
- *     even number of samples, the median is the mean of the
- *             two middle numbers.
- *
- */
-static inline cycles_t get_median(int n, cycles_t delta[])
-{
-       if (n % 2)
-               return(delta[n / 2] + delta[n / 2 - 1]) / 2;
-       else
-               return delta[n / 2];
-}
 
 
 static void print_report(struct report_options * options,
@@ -560,17 +531,13 @@ int run_iter(struct pingpong_context *ctx, struct user_parameters *user_param,
             struct pingpong_dest *rem_dest, int size)
 {
        ib_api_status_t         ib_status;
-       ib_qp_handle_t          qp;
+       int                      scnt, ccnt, rcnt;
        ib_send_wr_t                    *bad_wr;
        volatile char           *poll_buf; 
        volatile char           *post_buf;
 
-       int                      scnt, ccnt, rcnt;
-       int                      iters;
-       int                      tx_depth;
 
-       iters = user_param->iters;
-       tx_depth = user_param->tx_depth;
+
 
 
        ctx->list.vaddr = (uintptr_t) ctx->buf ;
@@ -578,6 +545,10 @@ int run_iter(struct pingpong_context *ctx, struct user_parameters *user_param,
        ctx->list.lkey = ctx->lkey;
        ctx->wr.remote_ops.vaddr = rem_dest->vaddr;
        ctx->wr.remote_ops.rkey = rem_dest->rkey;
+       ctx->wr.wr_id      = PINGPONG_RDMA_WRID;
+       ctx->wr.ds_array    = &ctx->list;
+       ctx->wr.num_ds    = 1;
+       ctx->wr.wr_type     = WR_RDMA_WRITE;
 
        if ((uint32_t)size > ctx->qp_attr[0].sq_max_inline) {/* complaince to perf_main */
                ctx->wr.send_opt = IB_SEND_OPT_SIGNALED;
@@ -595,10 +566,9 @@ int run_iter(struct pingpong_context *ctx, struct user_parameters *user_param,
                poll_buf = ctx->poll_buf;
                post_buf = ctx->post_buf;
        }    
-       qp = ctx->qp[0];
 
        /* Done with setup. Start the test. */
-       while (scnt < iters || ccnt < iters || rcnt < iters) {
+       while (scnt < user_param->iters || ccnt < user_param->iters || rcnt < user_param->iters) {
 
                /* Wait till buffer changes. */
                if (rcnt < user_param->iters && !(scnt < 1 && user_param->servername)) {
@@ -615,7 +585,7 @@ int run_iter(struct pingpong_context *ctx, struct user_parameters *user_param,
                        tstamp[scnt] = get_cycles();
                        *post_buf = (char)++scnt;
 
-                       ib_status = ib_post_send(qp, &ctx->wr, &bad_wr);
+                       ib_status = ib_post_send(ctx->qp[0], &ctx->wr, &bad_wr);
                        if (ib_status != IB_SUCCESS) 
                        {
                                fprintf(stderr, "Couldn't post send:scnt %d ccnt=%d \n",scnt,ccnt);
@@ -662,10 +632,6 @@ int run_iter(struct pingpong_context *ctx, struct user_parameters *user_param,
 
 int __cdecl main(int argc, char *argv[])
 {
-
-
-
-
        struct pingpong_context *ctx;
        struct pingpong_dest     *my_dest;
        struct pingpong_dest     *rem_dest;
@@ -692,19 +658,19 @@ int __cdecl main(int argc, char *argv[])
                int c;
 
                static struct option long_options[] = {
-                       {  "port",            1, NULL, 'p' },
-                       {  "connection",      1, NULL, 'c' },
-                       {  "mtu",             1, NULL, 'm' },
-                       {  "ib-dev",          1, NULL, 'd' },
-                       {  "ib-port",         1, NULL, 'i' },
-                       {  "size",            1, NULL, 's' },
-                       {  "iters",           1, NULL, 'n' },
-                       {  "tx-depth",        1, NULL, 't' },
-                       {  "all",             0, NULL, 'a' },
-                       {  "report-cycles",   0, NULL, 'C' },
-                       {  "report-histogram", 0, NULL, 'H' },
-                       {  "report-unsorted", 0, NULL, 'U' },
-                       {  "version",         0, NULL, 'V' },
+                       {  "port",                              1, NULL, 'p' },
+                       {  "connection",                1, NULL, 'c' },
+                       {  "mtu",                               1, NULL, 'm' },
+                       {  "ib-dev",                    1, NULL, 'd' },
+                       {  "ib-port",                   1, NULL, 'i' },
+                       {  "size",                              1, NULL, 's' },
+                       {  "iters",                             1, NULL, 'n' },
+                       {  "tx-depth",                  1, NULL, 't' },
+                       {  "all",                               0, NULL, 'a' },
+                       {  "report-cycles",             0, NULL, 'C' },
+                       {  "report-histogram",  0, NULL, 'H' },
+                       {  "report-unsorted",   0, NULL, 'U' },
+                       {  "version",                   0, NULL, 'V' },
                        { 0 }
                };
 
@@ -722,8 +688,7 @@ int __cdecl main(int argc, char *argv[])
                        break;
                case 'c':
                        if (strcmp("UC",optarg)==0)
-                               user_param.connection_type=1;
-                       /* default is 0 for any other option RC*/
+                               user_param.connection_type=UC;
                        break;
 
                case 'm':
@@ -807,14 +772,13 @@ int __cdecl main(int argc, char *argv[])
        }
        printf("------------------------------------------------------------------\n");
        printf("                    RDMA_Write Latency Test\n");
-       printf("Inline data is used up to 400 bytes message\n");
        if (user_param.connection_type==0) {
                printf("Connection type : RC\n");
        } else {
                printf("Connection type : UC\n");
        }
 
-       /* Done with parameter parsing. Perform setup. */
+
 
        // Initialize Winsock
        iResult = WSAStartup(MAKEWORD(2,2), &wsaData);
@@ -842,6 +806,7 @@ int __cdecl main(int argc, char *argv[])
        sockfd = pp_open_port(ctx, user_param.servername, ib_port, port,&my_dest,&rem_dest,&user_param);
        if (sockfd == INVALID_SOCKET)
                return 9;
+
        printf("------------------------------------------------------------------\n");
        printf(" #bytes #iterations    t_min[usec]    t_max[usec]  t_typical[usec]\n");