[DAPL2] more debug - Lindent + replace %s for %d in EVD_flush.
authorstansmith <stansmith@ad392aa1-c5ef-ae45-8dd8-e69d62a5ef86>
Sun, 26 Apr 2009 21:35:57 +0000 (21:35 +0000)
committerstansmith <stansmith@ad392aa1-c5ef-ae45-8dd8-e69d62a5ef86>
Sun, 26 Apr 2009 21:35:57 +0000 (21:35 +0000)
git-svn-id: svn://openib.tc.cornell.edu/gen1/trunk@2137 ad392aa1-c5ef-ae45-8dd8-e69d62a5ef86

ulp/dapl2/test/dtest/dtest.c

index 0ab17dc..85b5ae0 100644 (file)
@@ -50,7 +50,7 @@
 #include "..\..\..\..\etc\user\getopt.c"
 
 #define getpid() ((int)GetCurrentProcessId())
-#define F64x "%I64x" 
+#define F64x "%I64x"
 
 #ifdef DBG
 #define DAPL_PROVIDER "ibnic0v2d"
@@ -89,7 +89,7 @@
 #define ntohll(x)  bswap_64(x)
 #endif
 
-#endif  // _WIN32 || _WIN64
+#endif // _WIN32 || _WIN64
 
 /* Debug: 1 == connect & close only, otherwise full-meal deal */
 #define CONNECT_ONLY 0
 #define RDMA_BUFFER_SIZE  (64)
 
 /* Global DAT vars */
-static DAT_IA_HANDLE      h_ia = DAT_HANDLE_NULL;
-static DAT_PZ_HANDLE      h_pz = DAT_HANDLE_NULL;
-static DAT_EP_HANDLE      h_ep = DAT_HANDLE_NULL;
-static DAT_PSP_HANDLE     h_psp = DAT_HANDLE_NULL;
-static DAT_CR_HANDLE      h_cr = DAT_HANDLE_NULL;
-
-static DAT_EVD_HANDLE     h_async_evd = DAT_HANDLE_NULL;
-static DAT_EVD_HANDLE     h_dto_req_evd = DAT_HANDLE_NULL;
-static DAT_EVD_HANDLE     h_dto_rcv_evd = DAT_HANDLE_NULL;
-static DAT_EVD_HANDLE     h_cr_evd = DAT_HANDLE_NULL;
-static DAT_EVD_HANDLE     h_conn_evd = DAT_HANDLE_NULL;
-static DAT_CNO_HANDLE     h_dto_cno = DAT_HANDLE_NULL;
+static DAT_IA_HANDLE h_ia = DAT_HANDLE_NULL;
+static DAT_PZ_HANDLE h_pz = DAT_HANDLE_NULL;
+static DAT_EP_HANDLE h_ep = DAT_HANDLE_NULL;
+static DAT_PSP_HANDLE h_psp = DAT_HANDLE_NULL;
+static DAT_CR_HANDLE h_cr = DAT_HANDLE_NULL;
+
+static DAT_EVD_HANDLE h_async_evd = DAT_HANDLE_NULL;
+static DAT_EVD_HANDLE h_dto_req_evd = DAT_HANDLE_NULL;
+static DAT_EVD_HANDLE h_dto_rcv_evd = DAT_HANDLE_NULL;
+static DAT_EVD_HANDLE h_cr_evd = DAT_HANDLE_NULL;
+static DAT_EVD_HANDLE h_conn_evd = DAT_HANDLE_NULL;
+static DAT_CNO_HANDLE h_dto_cno = DAT_HANDLE_NULL;
 
 /* RDMA buffers */
-static DAT_LMR_HANDLE     h_lmr_send = DAT_HANDLE_NULL;
-static DAT_LMR_HANDLE     h_lmr_recv = DAT_HANDLE_NULL;
-static DAT_LMR_CONTEXT    lmr_context_send;
-static DAT_LMR_CONTEXT    lmr_context_recv;
-static DAT_RMR_CONTEXT    rmr_context_send;
-static DAT_RMR_CONTEXT    rmr_context_recv;
-static DAT_VLEN           registered_size_send;
-static DAT_VLEN           registered_size_recv;
-static DAT_VADDR          registered_addr_send;
-static DAT_VADDR          registered_addr_recv;
+static DAT_LMR_HANDLE h_lmr_send = DAT_HANDLE_NULL;
+static DAT_LMR_HANDLE h_lmr_recv = DAT_HANDLE_NULL;
+static DAT_LMR_CONTEXT lmr_context_send;
+static DAT_LMR_CONTEXT lmr_context_recv;
+static DAT_RMR_CONTEXT rmr_context_send;
+static DAT_RMR_CONTEXT rmr_context_recv;
+static DAT_VLEN registered_size_send;
+static DAT_VLEN registered_size_recv;
+static DAT_VADDR registered_addr_send;
+static DAT_VADDR registered_addr_recv;
 
 /* Initial msg receive buf, RMR exchange, and Rdma-write notification */
 #define MSG_BUF_COUNT     3
 #define MSG_IOV_COUNT     2
-static DAT_RMR_TRIPLET    rmr_recv_msg[MSG_BUF_COUNT];
-static DAT_LMR_HANDLE     h_lmr_recv_msg = DAT_HANDLE_NULL;
-static DAT_LMR_CONTEXT    lmr_context_recv_msg;
-static DAT_RMR_CONTEXT    rmr_context_recv_msg;
-static DAT_VLEN           registered_size_recv_msg;
-static DAT_VADDR          registered_addr_recv_msg;
+static DAT_RMR_TRIPLET rmr_recv_msg[MSG_BUF_COUNT];
+static DAT_LMR_HANDLE h_lmr_recv_msg = DAT_HANDLE_NULL;
+static DAT_LMR_CONTEXT lmr_context_recv_msg;
+static DAT_RMR_CONTEXT rmr_context_recv_msg;
+static DAT_VLEN registered_size_recv_msg;
+static DAT_VADDR registered_addr_recv_msg;
 
 /* message send buffer */
-static DAT_RMR_TRIPLET    rmr_send_msg;
-static DAT_LMR_HANDLE     h_lmr_send_msg = DAT_HANDLE_NULL;
-static DAT_LMR_CONTEXT    lmr_context_send_msg;
-static DAT_RMR_CONTEXT    rmr_context_send_msg;
-static DAT_VLEN           registered_size_send_msg;
-static DAT_VADDR          registered_addr_send_msg;
-static DAT_EP_ATTR        ep_attr;
-char                      hostname[256] = {0};
-char                      provider[64] = DAPL_PROVIDER;
-char                      addr_str[INET_ADDRSTRLEN];
+static DAT_RMR_TRIPLET rmr_send_msg;
+static DAT_LMR_HANDLE h_lmr_send_msg = DAT_HANDLE_NULL;
+static DAT_LMR_CONTEXT lmr_context_send_msg;
+static DAT_RMR_CONTEXT rmr_context_send_msg;
+static DAT_VLEN registered_size_send_msg;
+static DAT_VADDR registered_addr_send_msg;
+static DAT_EP_ATTR ep_attr;
+char hostname[256] = { 0 };
+char provider[64] = DAPL_PROVIDER;
+char addr_str[INET_ADDRSTRLEN];
 
 /* rdma pointers */
-char   *rbuf = NULL;
-char   *sbuf = NULL;
-int     status;
+char *rbuf = NULL;
+char *sbuf = NULL;
+int status;
 
 /* timers */
-double start,stop,total_us,total_sec;
+double start, stop, total_us, total_sec;
 
 struct dt_time {
-    double  total;
-    double  open;
-    double  reg;
-    double  unreg;
-    double  pzc;
-    double  pzf;
-    double  evdc;
-    double  evdf;
-    double  cnoc;
-    double  cnof;
-    double  epc;
-    double  epf;
-    double  rdma_wr;
-    double  rdma_rd[MAX_RDMA_RD];
-    double  rdma_rd_total;
-    double  rtt;
-    double  close;
+       double total;
+       double open;
+       double reg;
+       double unreg;
+       double pzc;
+       double pzf;
+       double evdc;
+       double evdf;
+       double cnoc;
+       double cnof;
+       double epc;
+       double epf;
+       double rdma_wr;
+       double rdma_rd[MAX_RDMA_RD];
+       double rdma_rd_total;
+       double rtt;
+       double close;
 };
 
 struct dt_time time;
 
 /* defaults */
-static int  failed=0;
-static int  performance_times=0;
-static int  connected=0;
-static int  burst=10;
-static int  server=1;
-static int  verbose=0;
-static int  polling=0;
-static int  poll_count=0;
-static int  rdma_wr_poll_count=0;
-static int  rdma_rd_poll_count[MAX_RDMA_RD]={0};
-static int  delay=0;
-static int  buf_len=RDMA_BUFFER_SIZE;
-static int  use_cno=0;
-static int  recv_msg_index=0;
-static int  burst_msg_posted=0;
-static int  burst_msg_index=0;
+static int failed = 0;
+static int performance_times = 0;
+static int connected = 0;
+static int burst = 10;
+static int server = 1;
+static int verbose = 0;
+static int polling = 0;
+static int poll_count = 0;
+static int rdma_wr_poll_count = 0;
+static int rdma_rd_poll_count[MAX_RDMA_RD] = { 0 };
+static int delay = 0;
+static int buf_len = RDMA_BUFFER_SIZE;
+static int use_cno = 0;
+static int recv_msg_index = 0;
+static int burst_msg_posted = 0;
+static int burst_msg_index = 0;
 
 /* forward prototypes */
-const char * DT_RetToString (DAT_RETURN ret_value);
-const char * DT_EventToSTr (DAT_EVENT_NUMBER event_code);
-void       print_usage(void);
-double     get_time(void);
-void       init_data(void);
-
-DAT_RETURN     send_msg(   void                    *data,
-                           DAT_COUNT               size,
-                           DAT_LMR_CONTEXT         context,
-                           DAT_DTO_COOKIE          cookie,
-                           DAT_COMPLETION_FLAGS    flags );
-
-DAT_RETURN     connect_ep( char *hostname, DAT_CONN_QUAL conn_id );
-void           disconnect_ep( void );
-DAT_RETURN     register_rdma_memory( void );
-DAT_RETURN     unregister_rdma_memory( void );
-DAT_RETURN     create_events( void );
-DAT_RETURN     destroy_events(void);
-DAT_RETURN     do_rdma_write_with_msg( void );
-DAT_RETURN     do_rdma_read_with_msg( void );
-DAT_RETURN     do_ping_pong_msg( void );
+const char *DT_RetToString(DAT_RETURN ret_value);
+const char *DT_EventToSTr(DAT_EVENT_NUMBER event_code);
+void print_usage(void);
+double get_time(void);
+void init_data(void);
+
+DAT_RETURN send_msg(void *data,
+                   DAT_COUNT size,
+                   DAT_LMR_CONTEXT context,
+                   DAT_DTO_COOKIE cookie, DAT_COMPLETION_FLAGS flags);
+
+DAT_RETURN connect_ep(char *hostname, DAT_CONN_QUAL conn_id);
+void disconnect_ep(void);
+DAT_RETURN register_rdma_memory(void);
+DAT_RETURN unregister_rdma_memory(void);
+DAT_RETURN create_events(void);
+DAT_RETURN destroy_events(void);
+DAT_RETURN do_rdma_write_with_msg(void);
+DAT_RETURN do_rdma_read_with_msg(void);
+DAT_RETURN do_ping_pong_msg(void);
 
 #define LOGPRINTF if (verbose) printf
 
-int
-main(int argc, char **argv)
+void flush_evds(void)
 {
-       int i,c;
-       DAT_RETURN  ret;
-       DAT_EP_PARAM ep_param;
-
-       /* parse arguments */
-       while ((c = getopt(argc, argv, "tscvpb:d:B:h:P:")) != -1)
-       {
-               switch(c)
-               {
-                      case 't':
-                               performance_times = 1;
-                               fflush(stdout);
-                               break;
-                       case 's':
-                               server = 1;
-                               fflush(stdout);
-                               break;
-                       case 'c':
-                               use_cno = 1;
-                               printf("%d Creating CNO for DTO EVD's\n",getpid());
-                               fflush(stdout);
-                               break;
-                       case 'v':
-                               verbose = 1;
-                               printf("%d Verbose\n",getpid());
-                               fflush(stdout);
-                               break;
-                       case 'p':
-                               polling = 1;
-                               printf("%d Polling\n",getpid());
-                               fflush(stdout);
-                               break;
-                       case 'B':
-                               burst = atoi(optarg);
-                               break;
-                       case 'd':
-                               delay = atoi(optarg);
-                               break;
-                       case 'b':
-                               buf_len = atoi(optarg);
-                               break;
-                       case 'h':
-                               server = 0;
-                               strcpy (hostname, optarg);
-                               break;
-                       case 'P':
-                               strcpy (provider, optarg);
-                               break;
-                       default:
-                               print_usage();
-                               exit(-12);
-               }
-       }
+       DAT_EVENT event;
+
+       /* Flush async error queue */
+       printf("%d ERR: Checking ASYNC EVD...\n", getpid());
+       while (dat_evd_dequeue(h_async_evd, &event) == DAT_SUCCESS) {
+               printf(" ASYNC EVD ENTRY: handle=%p reason=%d\n",
+                       event.event_data.asynch_error_event_data.dat_handle,
+                       event.event_data.asynch_error_event_data.reason);
+       }
+       /* Flush receive queue */
+       printf("%d ERR: Checking RECEIVE EVD...\n", getpid());
+       while (dat_evd_dequeue(h_dto_rcv_evd, &event) == DAT_SUCCESS) {
+               printf(" RCV EVD ENTRY: op=%d stat=%d ln=%d ck="F64x"\n",
+                       event.event_data.dto_completion_event_data.operation,
+                       event.event_data.dto_completion_event_data.status,
+                       event.event_data.dto_completion_event_data.transfered_length,
+                       event.event_data.dto_completion_event_data.user_cookie.as_64);
+       }
+       /* Flush request queue */
+       printf("%d ERR: Checking REQUEST EVD...\n", getpid());
+       while (dat_evd_dequeue(h_dto_req_evd, &event) == DAT_SUCCESS) {
+               printf(" REQ EVD ENTRY: op=%d stat=%d ln=%d ck="F64x"\n",
+                       event.event_data.dto_completion_event_data.operation,
+                       event.event_data.dto_completion_event_data.status,
+                       event.event_data.dto_completion_event_data.transfered_length,
+                       event.event_data.dto_completion_event_data.user_cookie.as_64);
+       }
+}
+
+int main(int argc, char **argv)
+{
+       int i, c;
+       DAT_RETURN ret;
+       DAT_EP_PARAM ep_param;
+
+       /* parse arguments */
+       while ((c = getopt(argc, argv, "tscvpb:d:B:h:P:")) != -1) {
+               switch (c) {
+               case 't':
+                       performance_times = 1;
+                       fflush(stdout);
+                       break;
+               case 's':
+                       server = 1;
+                       fflush(stdout);
+                       break;
+               case 'c':
+                       use_cno = 1;
+                       printf("%d Creating CNO for DTO EVD's\n", getpid());
+                       fflush(stdout);
+                       break;
+               case 'v':
+                       verbose = 1;
+                       printf("%d Verbose\n", getpid());
+                       fflush(stdout);
+                       break;
+               case 'p':
+                       polling = 1;
+                       printf("%d Polling\n", getpid());
+                       fflush(stdout);
+                       break;
+               case 'B':
+                       burst = atoi(optarg);
+                       break;
+               case 'd':
+                       delay = atoi(optarg);
+                       break;
+               case 'b':
+                       buf_len = atoi(optarg);
+                       break;
+               case 'h':
+                       server = 0;
+                       strcpy(hostname, optarg);
+                       break;
+               case 'P':
+                       strcpy(provider, optarg);
+                       break;
+               default:
+                       print_usage();
+                       exit(-12);
+               }
+       }
 
 #if defined(_WIN32) || defined(_WIN64)
-    {
-        WSADATA wsaData; 
-
-        i = WSAStartup ( MAKEWORD(2,2), &wsaData );
-        if ( i != 0 ) {
-               printf("%s WSAStartup(2.2) failed? (0x%x)\n",argv[0],i);
-               fflush(stdout);
-               exit(1);
-        }
-    }
+       {
+               WSADATA wsaData;
+
+               i = WSAStartup(MAKEWORD(2, 2), &wsaData);
+               if (i != 0) {
+                       printf("%s WSAStartup(2.2) failed? (0x%x)\n", argv[0],
+                              i);
+                       fflush(stdout);
+                       exit(1);
+               }
+       }
 #endif
 
-    if (!server) {
-            printf("%d Running as client - %s\n",getpid(),provider);
-    } else {
-            printf("%d Running as server - %s\n",getpid(),provider);
-    }
-    fflush(stdout);
-
-    /* allocate send and receive buffers */
-    if (((rbuf = malloc(buf_len*burst)) == NULL) ||
-           ((sbuf = malloc(buf_len*burst)) == NULL)) {
-               perror("malloc");
-               exit(1);
-    }
-    memset( &time, 0, sizeof(struct dt_time) );
-    LOGPRINTF("%d Allocated RDMA buffers (r:%p,s:%p) len %d \n",
-                       getpid(), rbuf, sbuf, buf_len);
-
-    /* dat_ia_open, dat_pz_create */
-    h_async_evd = DAT_HANDLE_NULL;
-    start = get_time();
-    ret = dat_ia_open( provider, 8, &h_async_evd, &h_ia );
-    stop = get_time();
-    time.open += ((stop - start)*1.0e6);
-    if(ret != DAT_SUCCESS) {
-         fprintf(stderr, "%d: Error Adaptor open: %s\n",
-                       getpid(),DT_RetToString(ret));
-         exit(1);
-    } else
-       LOGPRINTF("%d Opened Interface Adaptor\n",getpid());
-
-    /* Create Protection Zone */
-    start = get_time();
-    LOGPRINTF("%d Create Protection Zone\n",getpid());
-    ret = dat_pz_create(h_ia, &h_pz);
-    stop = get_time();
-    time.pzc += ((stop - start)*1.0e6);
-    if(ret != DAT_SUCCESS) {
-         fprintf(stderr, "%d Error creating Protection Zone: %s\n",
-                       getpid(),DT_RetToString(ret));
-         exit(1);
-    } else
-         LOGPRINTF("%d Created Protection Zone\n",getpid());
-
-    /* Register memory */
-    LOGPRINTF("%d Register RDMA memory\n", getpid());
-    ret = register_rdma_memory();
-    if(ret != DAT_SUCCESS) {
-         fprintf(stderr, "%d Error registering RDMA memory: %s\n",
-                               getpid(),DT_RetToString(ret));
-         goto cleanup;
-    } else
-         LOGPRINTF("%d Register RDMA memory done\n", getpid());
-
-    LOGPRINTF("%d Create events\n", getpid());
-    ret = create_events();
-    if(ret != DAT_SUCCESS) {
-         fprintf(stderr, "%d Error creating events: %s\n",
-                       getpid(),DT_RetToString(ret));
-         goto cleanup;
-    } else {
-         LOGPRINTF("%d Create events done\n", getpid());
-    }
-
-    /* create EP */
-    memset( &ep_attr, 0, sizeof(ep_attr) );
-    ep_attr.service_type                = DAT_SERVICE_TYPE_RC;
-    ep_attr.max_rdma_size               = 0x10000;
-    ep_attr.qos                         = 0;
-    ep_attr.recv_completion_flags       = 0;
-    ep_attr.max_recv_dtos               = MSG_BUF_COUNT + (burst*3);
-    ep_attr.max_request_dtos            = MSG_BUF_COUNT + (burst*3) + MAX_RDMA_RD; 
-    ep_attr.max_recv_iov                = MSG_IOV_COUNT;
-    ep_attr.max_request_iov             = MSG_IOV_COUNT;
-    ep_attr.max_rdma_read_in            = MAX_RDMA_RD;
-    ep_attr.max_rdma_read_out           = MAX_RDMA_RD;
-    ep_attr.request_completion_flags    = DAT_COMPLETION_DEFAULT_FLAG;
-    ep_attr.ep_transport_specific_count = 0;
-    ep_attr.ep_transport_specific       = NULL;
-    ep_attr.ep_provider_specific_count  = 0;
-    ep_attr.ep_provider_specific        = NULL;
-
-    start = get_time();
-    ret = dat_ep_create( h_ia, h_pz, h_dto_rcv_evd, 
-                         h_dto_req_evd, h_conn_evd, &ep_attr, &h_ep );
-    stop = get_time();
-    time.epc += ((stop - start)*1.0e6);
-    time.total += time.epc;
-    if(ret != DAT_SUCCESS) {
-         fprintf(stderr, "%d Error dat_ep_create: %s\n",
-                       getpid(),DT_RetToString(ret));
-         goto cleanup;
-    } else
-         LOGPRINTF("%d EP created %p \n", getpid(), h_ep);
-
-    /*
-     * register message buffers, establish connection, and
-     * exchange DMA RMR information info via messages
-     */
-    ret = connect_ep( hostname, SERVER_CONN_QUAL );
-    if(ret != DAT_SUCCESS) {
-         fprintf(stderr, "%d Error connect_ep: %s\n",
-                               getpid(),DT_RetToString(ret));
-         goto cleanup;
-    } else
-         LOGPRINTF("%d connect_ep complete\n", getpid());
-
-    /* Query EP for local and remote address information, print */
-    ret = dat_ep_query(h_ep, DAT_EP_FIELD_ALL, &ep_param);
-    if(ret != DAT_SUCCESS) {
-        fprintf(stderr, "%d Error dat_ep_query: %s\n",
-                getpid(),DT_RetToString(ret));
-        goto cleanup;
-    } else
-        LOGPRINTF("%d EP queried %p \n", getpid(), h_ep);
+       if (!server) {
+               printf("%d Running as client - %s\n", getpid(), provider);
+       } else {
+               printf("%d Running as server - %s\n", getpid(), provider);
+       }
+       fflush(stdout);
+
+       /* allocate send and receive buffers */
+       if (((rbuf = malloc(buf_len * (burst+1))) == NULL) ||
+           ((sbuf = malloc(buf_len * (burst+1))) == NULL)) {
+               perror("malloc");
+               exit(1);
+       }
+       memset(&time, 0, sizeof(struct dt_time));
+       LOGPRINTF("%d Allocated RDMA buffers (r:%p,s:%p) len %d \n",
+                 getpid(), rbuf, sbuf, buf_len);
+
+       /* dat_ia_open, dat_pz_create */
+       h_async_evd = DAT_HANDLE_NULL;
+       start = get_time();
+       ret = dat_ia_open(provider, 8, &h_async_evd, &h_ia);
+       stop = get_time();
+       time.open += ((stop - start) * 1.0e6);
+       if (ret != DAT_SUCCESS) {
+               fprintf(stderr, "%d: Error Adaptor open: %s\n",
+                       getpid(), DT_RetToString(ret));
+               exit(1);
+       } else
+               LOGPRINTF("%d Opened Interface Adaptor\n", getpid());
+
+       /* Create Protection Zone */
+       start = get_time();
+       LOGPRINTF("%d Create Protection Zone\n", getpid());
+       ret = dat_pz_create(h_ia, &h_pz);
+       stop = get_time();
+       time.pzc += ((stop - start) * 1.0e6);
+       if (ret != DAT_SUCCESS) {
+               fprintf(stderr, "%d Error creating Protection Zone: %s\n",
+                       getpid(), DT_RetToString(ret));
+               exit(1);
+       } else
+               LOGPRINTF("%d Created Protection Zone\n", getpid());
+
+       /* Register memory */
+       LOGPRINTF("%d Register RDMA memory\n", getpid());
+       ret = register_rdma_memory();
+       if (ret != DAT_SUCCESS) {
+               fprintf(stderr, "%d Error registering RDMA memory: %s\n",
+                       getpid(), DT_RetToString(ret));
+               goto cleanup;
+       } else
+               LOGPRINTF("%d Register RDMA memory done\n", getpid());
+
+       LOGPRINTF("%d Create events\n", getpid());
+       ret = create_events();
+       if (ret != DAT_SUCCESS) {
+               fprintf(stderr, "%d Error creating events: %s\n",
+                       getpid(), DT_RetToString(ret));
+               goto cleanup;
+       } else {
+               LOGPRINTF("%d Create events done\n", getpid());
+       }
+
+       /* create EP */
+       memset(&ep_attr, 0, sizeof(ep_attr));
+       ep_attr.service_type = DAT_SERVICE_TYPE_RC;
+       ep_attr.max_rdma_size = 0x10000;
+       ep_attr.qos = 0;
+       ep_attr.recv_completion_flags = 0;
+       ep_attr.max_recv_dtos = MSG_BUF_COUNT + (burst * 3);
+       ep_attr.max_request_dtos = MSG_BUF_COUNT + (burst * 3) + MAX_RDMA_RD;
+       ep_attr.max_recv_iov = MSG_IOV_COUNT;
+       ep_attr.max_request_iov = MSG_IOV_COUNT;
+       ep_attr.max_rdma_read_in = MAX_RDMA_RD;
+       ep_attr.max_rdma_read_out = MAX_RDMA_RD;
+       ep_attr.request_completion_flags = DAT_COMPLETION_DEFAULT_FLAG;
+       ep_attr.ep_transport_specific_count = 0;
+       ep_attr.ep_transport_specific = NULL;
+       ep_attr.ep_provider_specific_count = 0;
+       ep_attr.ep_provider_specific = NULL;
+
+       start = get_time();
+       ret = dat_ep_create(h_ia, h_pz, h_dto_rcv_evd,
+                           h_dto_req_evd, h_conn_evd, &ep_attr, &h_ep);
+       stop = get_time();
+       time.epc += ((stop - start) * 1.0e6);
+       time.total += time.epc;
+       if (ret != DAT_SUCCESS) {
+               fprintf(stderr, "%d Error dat_ep_create: %s\n",
+                       getpid(), DT_RetToString(ret));
+               goto cleanup;
+       } else
+               LOGPRINTF("%d EP created %p \n", getpid(), h_ep);
+
+       /*
+        * register message buffers, establish connection, and
+        * exchange DMA RMR information info via messages
+        */
+       ret = connect_ep(hostname, SERVER_CONN_QUAL);
+       if (ret != DAT_SUCCESS) {
+               fprintf(stderr, "%d Error connect_ep: %s\n",
+                       getpid(), DT_RetToString(ret));
+               goto cleanup;
+       } else
+               LOGPRINTF("%d connect_ep complete\n", getpid());
+
+       /* Query EP for local and remote address information, print */
+       ret = dat_ep_query(h_ep, DAT_EP_FIELD_ALL, &ep_param);
+       if (ret != DAT_SUCCESS) {
+               fprintf(stderr, "%d Error dat_ep_query: %s\n",
+                       getpid(), DT_RetToString(ret));
+               goto cleanup;
+       } else
+               LOGPRINTF("%d EP queried %p \n", getpid(), h_ep);
 #if defined(_WIN32)
-    printf("\n%d Query EP: LOCAL addr %s port %lld\n", getpid(),
-         inet_ntoa(((struct sockaddr_in *)
-                    ep_param.local_ia_address_ptr)->sin_addr),
-         (ep_param.local_port_qual));
+       printf("\n%d Query EP: LOCAL addr %s port %lld\n", getpid(),
+              inet_ntoa(((struct sockaddr_in *)
+                         ep_param.local_ia_address_ptr)->sin_addr),
+              (ep_param.local_port_qual));
 #else
-    inet_ntop(AF_INET,
-         &((struct sockaddr_in *)ep_param.local_ia_address_ptr)->sin_addr,
-         addr_str, sizeof(addr_str));
-    printf("\n%d Query EP: LOCAL addr %s port "F64x"\n", getpid(),
-         addr_str, (ep_param.local_port_qual));
+       inet_ntop(AF_INET,
+                 &((struct sockaddr_in *)ep_param.local_ia_address_ptr)->
+                 sin_addr, addr_str, sizeof(addr_str));
+       printf("\n%d Query EP: LOCAL addr %s port " F64x "\n", getpid(),
+              addr_str, (ep_param.local_port_qual));
 #endif
 #if defined(_WIN32)
-    printf("%d Query EP: REMOTE addr %s port %lld\n", getpid(),
-         inet_ntoa(((struct sockaddr_in *)
-                    ep_param.local_ia_address_ptr)->sin_addr),
-         (ep_param.remote_port_qual));
+       printf("%d Query EP: REMOTE addr %s port %lld\n", getpid(),
+              inet_ntoa(((struct sockaddr_in *)
+                         ep_param.local_ia_address_ptr)->sin_addr),
+              (ep_param.remote_port_qual));
 #else
-    inet_ntop(AF_INET,
-        &((struct sockaddr_in *)ep_param.remote_ia_address_ptr)->sin_addr,
-        addr_str, sizeof(addr_str));
-    printf("%d Query EP: REMOTE addr %s port "F64x"\n", getpid(),
-           addr_str, (ep_param.remote_port_qual));
+       inet_ntop(AF_INET,
+                 &((struct sockaddr_in *)ep_param.remote_ia_address_ptr)->
+                 sin_addr, addr_str, sizeof(addr_str));
+       printf("%d Query EP: REMOTE addr %s port " F64x "\n", getpid(),
+              addr_str, (ep_param.remote_port_qual));
 #endif
-    fflush(stdout);
+       fflush(stdout);
 
 #if CONNECT_ONLY
 #if defined(_WIN32) || defined(_WIN64)
-    Sleep(1*1000);
+       Sleep(1 * 1000);
 #else
-    sleep(1);
+       sleep(1);
 #endif
-    goto cleanup;
+       goto cleanup;
 #endif
 
-    /*********** RDMA write data *************/
-    ret = do_rdma_write_with_msg();
-    if(ret != DAT_SUCCESS) {
-         fprintf(stderr, "%d Error do_rdma_write_with_msg: %s\n",
-                               getpid(),DT_RetToString(ret));
-         goto cleanup;
-    } else
-         LOGPRINTF("%d do_rdma_write_with_msg complete\n", getpid());
-
-    /*********** RDMA read data *************/
-    ret = do_rdma_read_with_msg();
-    if(ret != DAT_SUCCESS) {
-         fprintf(stderr, "%d Error do_rdma_read_with_msg: %s\n",
-                               getpid(),DT_RetToString(ret));
-         goto cleanup;
-    } else
-         LOGPRINTF("%d do_rdma_read_with_msg complete\n", getpid());
-
-    /*********** PING PING messages ************/
-    ret = do_ping_pong_msg();
-    if(ret != DAT_SUCCESS) {
-         fprintf(stderr, "%d Error do_ping_pong_msg: %s\n",
-                               getpid(),DT_RetToString(ret));
-         goto cleanup;
-    } else {
-         LOGPRINTF("%d do_ping_pong_msg complete\n", getpid());
-        goto complete;
-    }
+       /*********** RDMA write data *************/
+       ret = do_rdma_write_with_msg();
+       if (ret != DAT_SUCCESS) {
+               fprintf(stderr, "%d Error do_rdma_write_with_msg: %s\n",
+                       getpid(), DT_RetToString(ret));
+               goto cleanup;
+       } else
+               LOGPRINTF("%d do_rdma_write_with_msg complete\n", getpid());
+
+       /*********** RDMA read data *************/
+       ret = do_rdma_read_with_msg();
+       if (ret != DAT_SUCCESS) {
+               fprintf(stderr, "%d Error do_rdma_read_with_msg: %s\n",
+                       getpid(), DT_RetToString(ret));
+               goto cleanup;
+       } else
+               LOGPRINTF("%d do_rdma_read_with_msg complete\n", getpid());
+
+       /*********** PING PING messages ************/
+       ret = do_ping_pong_msg();
+       if (ret != DAT_SUCCESS) {
+               fprintf(stderr, "%d Error do_ping_pong_msg: %s\n",
+                       getpid(), DT_RetToString(ret));
+               goto cleanup;
+       } else {
+               LOGPRINTF("%d do_ping_pong_msg complete\n", getpid());
+               goto complete;
+       }
 
 cleanup:
-       failed++;
+
+       fflush(stderr);
+       fflush(stdout);
+       flush_evds();
+       failed++;
+       fflush(stderr);
+       fflush(stdout);
+
 complete:
 
-    /* disconnect and free EP resources */
-    if ( h_ep != DAT_HANDLE_NULL ) {
-         /* unregister message buffers and tear down connection */
-         LOGPRINTF("%d Disconnect and Free EP %p \n",getpid(),h_ep);
-         disconnect_ep();
-    
-        /* free EP */
-        LOGPRINTF("%d Free EP %p \n",getpid(),h_ep);
-        start = get_time();
-        ret = dat_ep_free( h_ep );
-        stop = get_time();
-        time.epf += ((stop - start)*1.0e6);
-        time.total += time.epf;
-        if(ret != DAT_SUCCESS) {
-            fprintf(stderr, "%d Error freeing EP: %s\n",
-                    getpid(), DT_RetToString(ret));
-        } else {
-            LOGPRINTF("%d Freed EP\n",getpid());
-            h_ep = DAT_HANDLE_NULL;
-        }   
-    }
-    
-    /* free EVDs */
-    LOGPRINTF("%d destroy events\n", getpid());
-    ret = destroy_events();
-    if(ret != DAT_SUCCESS)
-         fprintf(stderr, "%d Error destroy_events: %s\n",
-                       getpid(),DT_RetToString(ret));
-    else
-         LOGPRINTF("%d destroy events done\n", getpid());
-
-
-    ret = unregister_rdma_memory();
-    LOGPRINTF("%d unregister_rdma_memory \n", getpid());
-    if(ret != DAT_SUCCESS)
-         fprintf(stderr, "%d Error unregister_rdma_memory: %s\n",
-                       getpid(),DT_RetToString(ret));
-    else
-         LOGPRINTF("%d unregister_rdma_memory done\n", getpid());
-
-    /* Free protection domain */
-    LOGPRINTF("%d Freeing pz\n",getpid());
-    start = get_time();
-    ret = dat_pz_free( h_pz );
-    stop = get_time();
-    time.pzf += ((stop - start)*1.0e6);
-    if (ret != DAT_SUCCESS) {
-         fprintf(stderr, "%d Error freeing PZ: %s\n",
-               getpid(), DT_RetToString(ret));
-    } else {
-         LOGPRINTF("%d Freed pz\n",getpid());
-         h_pz = NULL;
-    }
-
-    /* close the device */
-    LOGPRINTF("%d Closing Interface Adaptor\n",getpid());
-    start = get_time();
-    ret = dat_ia_close( h_ia, DAT_CLOSE_ABRUPT_FLAG );
-    stop = get_time();
-    time.close += ((stop - start)*1.0e6);
-    if(ret != DAT_SUCCESS) {
-         fprintf(stderr, "%d: Error Adaptor close: %s\n",
-                       getpid(),DT_RetToString(ret));
-         exit(1);
-    } else
-         LOGPRINTF("%d Closed Interface Adaptor\n",getpid());
-
-    /* free rdma buffers */
-    free(rbuf);
-    free(sbuf);
-
-    printf("\n%d: DAPL Test Complete. %s\n\n",
-           getpid(), failed?"FAILED":"PASSED");
-    
-    if (!performance_times) 
-        exit(0);
-
-    printf( "\n%d: DAPL Test Complete.\n\n",getpid());
-    printf( "%d: Message RTT: Total=%10.2lf usec, %d bursts, itime=%10.2lf"
-               " usec, pc=%d\n", 
-               getpid(), time.rtt, burst, time.rtt/burst, poll_count );
-    printf( "%d: RDMA write:  Total=%10.2lf usec, %d bursts, itime=%10.2lf"
-               " usec, pc=%d\n", 
-               getpid(), time.rdma_wr, burst, 
-               time.rdma_wr/burst, rdma_wr_poll_count );
-    for(i=0;i<MAX_RDMA_RD;i++) {
-           printf( "%d: RDMA read:   Total=%10.2lf usec,   %d bursts, "
-                   "itime=%10.2lf usec, pc=%d\n", 
-                  getpid(),time.rdma_rd_total,MAX_RDMA_RD,
-                  time.rdma_rd[i],rdma_rd_poll_count[i] );
-    }
-    printf( "%d: open:      %10.2lf usec\n", getpid(), time.open  );
-    printf( "%d: close:     %10.2lf usec\n", getpid(), time.close );
-    printf( "%d: PZ create: %10.2lf usec\n", getpid(), time.pzc );
-    printf( "%d: PZ free:   %10.2lf usec\n", getpid(), time.pzf );
-    printf( "%d: LMR create:%10.2lf usec\n", getpid(), time.reg );
-    printf( "%d: LMR free:  %10.2lf usec\n", getpid(), time.unreg );
-    printf( "%d: EVD create:%10.2lf usec\n", getpid(), time.evdc );
-    printf( "%d: EVD free:  %10.2lf usec\n", getpid(), time.evdf );
-    if (use_cno) {
-       printf( "%d: CNO create:  %10.2lf usec\n", getpid(), time.cnoc );
-       printf( "%d: CNO free:    %10.2lf usec\n", getpid(), time.cnof );
-    }
-    printf( "%d: EP create: %10.2lf usec\n",getpid(), time.epc );
-    printf( "%d: EP free:   %10.2lf usec\n",getpid(), time.epf );
-    printf( "%d: TOTAL:     %10.2lf usec\n",getpid(), time.total );
+       /* disconnect and free EP resources */
+       if (h_ep != DAT_HANDLE_NULL) {
+               /* unregister message buffers and tear down connection */
+               LOGPRINTF("%d Disconnect and Free EP %p \n", getpid(), h_ep);
+               disconnect_ep();
+
+               /* free EP */
+               LOGPRINTF("%d Free EP %p \n", getpid(), h_ep);
+               start = get_time();
+               ret = dat_ep_free(h_ep);
+               stop = get_time();
+               time.epf += ((stop - start) * 1.0e6);
+               time.total += time.epf;
+               if (ret != DAT_SUCCESS) {
+                       fprintf(stderr, "%d Error freeing EP: %s\n",
+                               getpid(), DT_RetToString(ret));
+               } else {
+                       LOGPRINTF("%d Freed EP\n", getpid());
+                       h_ep = DAT_HANDLE_NULL;
+               }
+       }
+
+       /* free EVDs */
+       LOGPRINTF("%d destroy events\n", getpid());
+       ret = destroy_events();
+       if (ret != DAT_SUCCESS)
+               fprintf(stderr, "%d Error destroy_events: %s\n",
+                       getpid(), DT_RetToString(ret));
+       else
+               LOGPRINTF("%d destroy events done\n", getpid());
+
+       ret = unregister_rdma_memory();
+       LOGPRINTF("%d unregister_rdma_memory \n", getpid());
+       if (ret != DAT_SUCCESS)
+               fprintf(stderr, "%d Error unregister_rdma_memory: %s\n",
+                       getpid(), DT_RetToString(ret));
+       else
+               LOGPRINTF("%d unregister_rdma_memory done\n", getpid());
+
+       /* Free protection domain */
+       LOGPRINTF("%d Freeing pz\n", getpid());
+       start = get_time();
+       ret = dat_pz_free(h_pz);
+       stop = get_time();
+       time.pzf += ((stop - start) * 1.0e6);
+       if (ret != DAT_SUCCESS) {
+               fprintf(stderr, "%d Error freeing PZ: %s\n",
+                       getpid(), DT_RetToString(ret));
+       } else {
+               LOGPRINTF("%d Freed pz\n", getpid());
+               h_pz = NULL;
+       }
+
+       /* close the device */
+       LOGPRINTF("%d Closing Interface Adaptor\n", getpid());
+       start = get_time();
+       ret = dat_ia_close(h_ia, DAT_CLOSE_ABRUPT_FLAG);
+       stop = get_time();
+       time.close += ((stop - start) * 1.0e6);
+       if (ret != DAT_SUCCESS) {
+               fprintf(stderr, "%d: Error Adaptor close: %s\n",
+                       getpid(), DT_RetToString(ret));
+       } else
+               LOGPRINTF("%d Closed Interface Adaptor\n", getpid());
+
+       /* free rdma buffers */
+       free(rbuf);
+       free(sbuf);
+
+       printf("\n%d: DAPL Test Complete. %s\n\n",
+              getpid(), failed ? "FAILED" : "PASSED");
+
+       if (!performance_times)
+               exit(0);
+
+       printf("\n%d: DAPL Test Complete.\n\n", getpid());
+       printf("%d: Message RTT: Total=%10.2lf usec, %d bursts, itime=%10.2lf"
+              " usec, pc=%d\n",
+              getpid(), time.rtt, burst, time.rtt / burst, poll_count);
+       printf("%d: RDMA write:  Total=%10.2lf usec, %d bursts, itime=%10.2lf"
+              " usec, pc=%d\n",
+              getpid(), time.rdma_wr, burst,
+              time.rdma_wr / burst, rdma_wr_poll_count);
+       for (i = 0; i < MAX_RDMA_RD; i++) {
+               printf("%d: RDMA read:   Total=%10.2lf usec,   %d bursts, "
+                      "itime=%10.2lf usec, pc=%d\n",
+                      getpid(), time.rdma_rd_total, MAX_RDMA_RD,
+                      time.rdma_rd[i], rdma_rd_poll_count[i]);
+       }
+       printf("%d: open:      %10.2lf usec\n", getpid(), time.open);
+       printf("%d: close:     %10.2lf usec\n", getpid(), time.close);
+       printf("%d: PZ create: %10.2lf usec\n", getpid(), time.pzc);
+       printf("%d: PZ free:   %10.2lf usec\n", getpid(), time.pzf);
+       printf("%d: LMR create:%10.2lf usec\n", getpid(), time.reg);
+       printf("%d: LMR free:  %10.2lf usec\n", getpid(), time.unreg);
+       printf("%d: EVD create:%10.2lf usec\n", getpid(), time.evdc);
+       printf("%d: EVD free:  %10.2lf usec\n", getpid(), time.evdf);
+       if (use_cno) {
+               printf("%d: CNO create:  %10.2lf usec\n", getpid(), time.cnoc);
+               printf("%d: CNO free:    %10.2lf usec\n", getpid(), time.cnof);
+       }
+       printf("%d: EP create: %10.2lf usec\n", getpid(), time.epc);
+       printf("%d: EP free:   %10.2lf usec\n", getpid(), time.epf);
+       printf("%d: TOTAL:     %10.2lf usec\n", getpid(), time.total);
 
 #if defined(_WIN32) || defined(_WIN64)
-    WSACleanup();
+       WSACleanup();
 #endif
-    return(0);
+       return (0);
 }
 
 #if defined(_WIN32) || defined(_WIN64)
 
-void gettimeofday( struct timeval *t, char *jnk)
+void gettimeofday(struct timeval *t, char *jnk)
 {
        SYSTEMTIME now;
        GetLocalTime(&now);
@@ -610,1489 +643,1508 @@ void gettimeofday( struct timeval *t, char *jnk)
 
 double get_time(void)
 {
-       struct timeval tp;
+       struct timeval tp;
 
-       gettimeofday(&tp, NULL);
-       return ((double) tp.tv_sec + (double) tp.tv_usec * 1e-6);
+       gettimeofday(&tp, NULL);
+       return ((double)tp.tv_sec + (double)tp.tv_usec * 1e-6);
 }
 
 void init_data(void)
 {
-       memset(rbuf, 'a', buf_len);
-       memset(sbuf, 'b', buf_len);
+       memset(rbuf, 'a', buf_len);
+       memset(sbuf, 'b', buf_len);
 }
 
-
 DAT_RETURN
-send_msg(  void                   *data,
-           DAT_COUNT               size,
-           DAT_LMR_CONTEXT         context,
-           DAT_DTO_COOKIE          cookie,
-           DAT_COMPLETION_FLAGS    flags )
+send_msg(void *data,
+        DAT_COUNT size,
+        DAT_LMR_CONTEXT context,
+        DAT_DTO_COOKIE cookie, DAT_COMPLETION_FLAGS flags)
 {
-    DAT_LMR_TRIPLET    iov;
-    DAT_EVENT          event;
-    DAT_COUNT          nmore;
-    DAT_RETURN         ret;
+       DAT_LMR_TRIPLET iov;
+       DAT_EVENT event;
+       DAT_COUNT nmore;
+       DAT_RETURN ret;
 
-    iov.lmr_context     = context;
+       iov.lmr_context = context;
 #if defined(_WIN32)
-    iov.virtual_address = (DAT_VADDR)data;
+       iov.virtual_address = (DAT_VADDR) data;
 #else
-    iov.virtual_address = (DAT_VADDR)(unsigned long)data;
+       iov.virtual_address = (DAT_VADDR) (unsigned long)data;
 #endif
-    iov.segment_length  = size;
-    
-    LOGPRINTF("%d calling post_send\n", getpid());
-    cookie.as_64 = 0xaaaa;
-    ret = dat_ep_post_send( h_ep,
-                           1,
-                           &iov,
-                           cookie,
-                           flags );
-
-    if (ret != DAT_SUCCESS) {
-        fprintf(stderr, "%d: ERROR: dat_ep_post_send() %s\n",
-                           getpid(),DT_RetToString(ret));
-        return ret;
-    }
-
-    if (!(flags & DAT_COMPLETION_SUPPRESS_FLAG)) {
-       if ( polling ) {
-           printf("%d Polling post send completion...\n",getpid());
-           while (dat_evd_dequeue(h_dto_req_evd, &event) == DAT_QUEUE_EMPTY) ;
-       }
-       else {
-           LOGPRINTF("%d waiting for post_send completion event\n", getpid());
-           if (use_cno) {
-               DAT_EVD_HANDLE evd = DAT_HANDLE_NULL;
-               ret = dat_cno_wait( h_dto_cno, DTO_TIMEOUT, &evd );
-               LOGPRINTF("%d cno wait return evd_handle=%p\n", getpid(),evd);
-               if ( evd != h_dto_req_evd ) {
-                   fprintf(stderr,
-                       "%d Error waiting on h_dto_cno: evd != h_dto_req_evd\n",
-                       getpid());
-                   return( DAT_ABORT );
-               }
-           }
-           /* use wait to dequeue */
-           ret = dat_evd_wait( h_dto_req_evd, DTO_TIMEOUT, 1, &event, &nmore );
-           if (ret != DAT_SUCCESS) {
-               fprintf(stderr, "%d: ERROR: DTO dat_evd_wait() %s\n",
-                       getpid(),DT_RetToString(ret));
-               return ret;
-           }
-       }
-
-       /* validate event number, len, cookie, and status */
-       if ( event.event_number != DAT_DTO_COMPLETION_EVENT ) {
-           fprintf(stderr, "%d: ERROR: DTO event number %s\n",
-                   getpid(),DT_EventToSTr(event.event_number));
-           return( DAT_ABORT );
-       }
-
-       if ((event.event_data.dto_completion_event_data.transfered_length != size ) ||
-           (event.event_data.dto_completion_event_data.user_cookie.as_64 != 0xaaaa )) {
-           fprintf(stderr, "%d: ERROR: DTO len %d or cookie "F64x" \n",
-               getpid(),
-               event.event_data.dto_completion_event_data.transfered_length,
-               event.event_data.dto_completion_event_data.user_cookie.as_64 );
-           return( DAT_ABORT );
-
-       }
-       if (event.event_data.dto_completion_event_data.status != DAT_SUCCESS) {
-           fprintf(stderr, "%d: ERROR: DTO event status %s\n",
-                   getpid(),DT_RetToString(ret));
-           return( DAT_ABORT );
-       }
-    }
-
-    return DAT_SUCCESS;
+       iov.segment_length = size;
+
+       LOGPRINTF("%d calling post_send\n", getpid());
+       cookie.as_64 = 0xaaaa;
+       ret = dat_ep_post_send(h_ep, 1, &iov, cookie, flags);
+
+       if (ret != DAT_SUCCESS) {
+               fprintf(stderr, "%d: ERROR: dat_ep_post_send() %s\n",
+                       getpid(), DT_RetToString(ret));
+               return ret;
+       }
+
+       if (!(flags & DAT_COMPLETION_SUPPRESS_FLAG)) {
+               if (polling) {
+                       printf("%d Polling post send completion...\n",
+                              getpid());
+                       while (dat_evd_dequeue(h_dto_req_evd, &event) ==
+                              DAT_QUEUE_EMPTY) ;
+               } else {
+                       LOGPRINTF("%d waiting for post_send completion event\n",
+                                 getpid());
+                       if (use_cno) {
+                               DAT_EVD_HANDLE evd = DAT_HANDLE_NULL;
+                               ret =
+                                   dat_cno_wait(h_dto_cno, DTO_TIMEOUT, &evd);
+                               LOGPRINTF("%d cno wait return evd_handle=%p\n",
+                                         getpid(), evd);
+                               if (evd != h_dto_req_evd) {
+                                       fprintf(stderr,
+                                               "%d Error waiting on h_dto_cno: evd != h_dto_req_evd\n",
+                                               getpid());
+                                       return (DAT_ABORT);
+                               }
+                       }
+                       /* use wait to dequeue */
+                       ret =
+                           dat_evd_wait(h_dto_req_evd, DTO_TIMEOUT, 1, &event,
+                                        &nmore);
+                       if (ret != DAT_SUCCESS) {
+                               fprintf(stderr,
+                                       "%d: ERROR: DTO dat_evd_wait() %s\n",
+                                       getpid(), DT_RetToString(ret));
+                               return ret;
+                       }
+               }
+
+               /* validate event number, len, cookie, and status */
+               if (event.event_number != DAT_DTO_COMPLETION_EVENT) {
+                       fprintf(stderr, "%d: ERROR: DTO event number %s\n",
+                               getpid(), DT_EventToSTr(event.event_number));
+                       return (DAT_ABORT);
+               }
+
+               if ((event.event_data.dto_completion_event_data.
+                    transfered_length != size)
+                   || (event.event_data.dto_completion_event_data.user_cookie.
+                       as_64 != 0xaaaa)) {
+                       fprintf(stderr,
+                               "%d: ERROR: DTO len %d or cookie " F64x " \n",
+                               getpid(),
+                               event.event_data.dto_completion_event_data.
+                               transfered_length,
+                               event.event_data.dto_completion_event_data.
+                               user_cookie.as_64);
+                       return (DAT_ABORT);
+
+               }
+               if (event.event_data.dto_completion_event_data.status !=
+                   DAT_SUCCESS) {
+                       fprintf(stderr, "%d: ERROR: DTO event status %s\n",
+                               getpid(), DT_RetToString(ret));
+                       return (DAT_ABORT);
+               }
+       }
+
+       return DAT_SUCCESS;
 }
 
-DAT_RETURN
-connect_ep( char *hostname, DAT_CONN_QUAL conn_id )
+DAT_RETURN connect_ep(char *hostname, DAT_CONN_QUAL conn_id)
 {
-    DAT_SOCK_ADDR           remote_addr;
-    DAT_RETURN              ret;
-    DAT_REGION_DESCRIPTION  region;
-    DAT_EVENT               event;
-    DAT_COUNT               nmore;
-    DAT_LMR_TRIPLET         l_iov;
-    DAT_RMR_TRIPLET         r_iov;
-    DAT_DTO_COOKIE          cookie;
-    int                     i;
-    unsigned char           *buf;
-    DAT_CR_PARAM            cr_param = { 0 };
-    unsigned char          pdata[48] = { 0 };
-    
-     /* Register send message buffer */
-    LOGPRINTF("%d Registering send Message Buffer %p, len %d\n",
-               getpid(), &rmr_send_msg, (int)sizeof(DAT_RMR_TRIPLET) );
-    region.for_va = &rmr_send_msg;
-    ret = dat_lmr_create(  h_ia,
-                           DAT_MEM_TYPE_VIRTUAL,
-                           region,
-                           sizeof(DAT_RMR_TRIPLET),
-                           h_pz,
-                           DAT_MEM_PRIV_LOCAL_WRITE_FLAG,
-                          DAT_VA_TYPE_VA,
-                           &h_lmr_send_msg,
-                           &lmr_context_send_msg,
-                           &rmr_context_send_msg,
-                           &registered_size_send_msg,
-                           &registered_addr_send_msg );
-
-     if (ret != DAT_SUCCESS) {
-       fprintf(stderr, "%d Error registering send msg buffer: %s\n",
-               getpid(),DT_RetToString(ret));
-       return(ret);
-    }
-    else
-       LOGPRINTF("%d Registered send Message Buffer %p \n",
-               getpid(),region.for_va );
-
-    /* Register Receive buffers */
-    LOGPRINTF("%d Registering Receive Message Buffer %p\n",
-               getpid(), rmr_recv_msg );
-    region.for_va = rmr_recv_msg;
-    ret = dat_lmr_create(  h_ia,
-                           DAT_MEM_TYPE_VIRTUAL,
-                           region,
-                           sizeof(DAT_RMR_TRIPLET)*MSG_BUF_COUNT,
-                           h_pz,
-                           DAT_MEM_PRIV_LOCAL_WRITE_FLAG,
-                          DAT_VA_TYPE_VA,
-                           &h_lmr_recv_msg,
-                           &lmr_context_recv_msg,
-                           &rmr_context_recv_msg,
-                           &registered_size_recv_msg,
-                           &registered_addr_recv_msg );
-    if(ret != DAT_SUCCESS) {
-       fprintf(stderr, "%d Error registering recv msg buffer: %s\n",
-               getpid(),DT_RetToString(ret));
-       return(ret);
-    }
-    else
-       LOGPRINTF("%d Registered Receive Message Buffer %p\n",
-               getpid(),region.for_va);
-
-    for ( i = 0; i < MSG_BUF_COUNT; i++ ) {
-       cookie.as_64          = i;
-       l_iov.lmr_context     = lmr_context_recv_msg;
+       DAT_SOCK_ADDR remote_addr;
+       DAT_RETURN ret;
+       DAT_REGION_DESCRIPTION region;
+       DAT_EVENT event;
+       DAT_COUNT nmore;
+       DAT_LMR_TRIPLET l_iov;
+       DAT_RMR_TRIPLET r_iov;
+       DAT_DTO_COOKIE cookie;
+       int i;
+       unsigned char *buf;
+       DAT_CR_PARAM cr_param = { 0 };
+       unsigned char pdata[48] = { 0 };
+
+       /* Register send message buffer */
+       LOGPRINTF("%d Registering send Message Buffer %p, len %d\n",
+                 getpid(), &rmr_send_msg, (int)sizeof(DAT_RMR_TRIPLET));
+       region.for_va = &rmr_send_msg;
+       ret = dat_lmr_create(h_ia,
+                            DAT_MEM_TYPE_VIRTUAL,
+                            region,
+                            sizeof(DAT_RMR_TRIPLET),
+                            h_pz,
+                            DAT_MEM_PRIV_LOCAL_WRITE_FLAG,
+                            DAT_VA_TYPE_VA,
+                            &h_lmr_send_msg,
+                            &lmr_context_send_msg,
+                            &rmr_context_send_msg,
+                            &registered_size_send_msg,
+                            &registered_addr_send_msg);
+
+       if (ret != DAT_SUCCESS) {
+               fprintf(stderr, "%d Error registering send msg buffer: %s\n",
+                       getpid(), DT_RetToString(ret));
+               return (ret);
+       } else
+               LOGPRINTF("%d Registered send Message Buffer %p \n",
+                         getpid(), region.for_va);
+
+       /* Register Receive buffers */
+       LOGPRINTF("%d Registering Receive Message Buffer %p\n",
+                 getpid(), rmr_recv_msg);
+       region.for_va = rmr_recv_msg;
+       ret = dat_lmr_create(h_ia,
+                            DAT_MEM_TYPE_VIRTUAL,
+                            region,
+                            sizeof(DAT_RMR_TRIPLET) * MSG_BUF_COUNT,
+                            h_pz,
+                            DAT_MEM_PRIV_LOCAL_WRITE_FLAG,
+                            DAT_VA_TYPE_VA,
+                            &h_lmr_recv_msg,
+                            &lmr_context_recv_msg,
+                            &rmr_context_recv_msg,
+                            &registered_size_recv_msg,
+                            &registered_addr_recv_msg);
+       if (ret != DAT_SUCCESS) {
+               fprintf(stderr, "%d Error registering recv msg buffer: %s\n",
+                       getpid(), DT_RetToString(ret));
+               return (ret);
+       } else
+               LOGPRINTF("%d Registered Receive Message Buffer %p\n",
+                         getpid(), region.for_va);
+
+       for (i = 0; i < MSG_BUF_COUNT; i++) {
+               cookie.as_64 = i;
+               l_iov.lmr_context = lmr_context_recv_msg;
 #if defined(_WIN32)
-       l_iov.virtual_address = (DAT_VADDR)&rmr_recv_msg[ i ];
+               l_iov.virtual_address = (DAT_VADDR) & rmr_recv_msg[i];
 #else
-       l_iov.virtual_address = (DAT_VADDR)(unsigned long)&rmr_recv_msg[ i ];
+               l_iov.virtual_address =
+                   (DAT_VADDR) (unsigned long)&rmr_recv_msg[i];
 #endif
-       l_iov.segment_length  = sizeof(DAT_RMR_TRIPLET);
-
-       LOGPRINTF("%d Posting Receive Message Buffer %p\n",
-                   getpid(), &rmr_recv_msg[ i ]);
-       ret = dat_ep_post_recv( h_ep,
-                               1,
-                               &l_iov,
-                               cookie,
-                               DAT_COMPLETION_DEFAULT_FLAG );
-
-        if(ret != DAT_SUCCESS) {
-           fprintf(stderr, "%d Error registering recv msg buffer: %s\n",
-                           getpid(),DT_RetToString(ret));
-           return(ret);
-        }
-       else
-           LOGPRINTF("%d Registered Receive Message Buffer %p\n",
-                                   getpid(),region.for_va);
-
-    }
-
-    /* setup receive rdma buffer to initial string to be overwritten */
-    strcpy( (char*)rbuf, "blah, blah, blah\n" );
-
-    if ( server ) {  /* SERVER */
-
-        /* create the service point for server listen */
-        LOGPRINTF("%d Creating service point for listen\n",getpid());
-       ret = dat_psp_create(   h_ia,
-                               conn_id,
-                               h_cr_evd,
-                               DAT_PSP_CONSUMER_FLAG,
-                               &h_psp );
-       if(ret != DAT_SUCCESS) {
-           fprintf(stderr, "%d Error dat_psp_create: %s\n",
-                           getpid(),DT_RetToString(ret));
-           return(ret);
-       }
-       else
-           LOGPRINTF("%d dat_psp_created for server listen\n", getpid());
-
-       printf("%d Server waiting for connect request on port "F64x"\n", 
-               getpid(), conn_id);
-       
-       ret = dat_evd_wait( h_cr_evd, SERVER_TIMEOUT, 1, &event, &nmore );
-       if(ret != DAT_SUCCESS) {
-           fprintf(stderr, "%d Error dat_evd_wait: %s\n",
-                           getpid(),DT_RetToString(ret));
-           return(ret);
-       }
-       else
-           LOGPRINTF("%d dat_evd_wait for cr_evd completed\n", getpid());
-
-       if ( event.event_number != DAT_CONNECTION_REQUEST_EVENT ) {
-            fprintf(stderr, "%d Error unexpected cr event : %s\n",
-                                   getpid(),DT_EventToSTr(event.event_number));
-           return( DAT_ABORT );
-       }
-       if ( (event.event_data.cr_arrival_event_data.conn_qual != SERVER_CONN_QUAL) ||
-            (event.event_data.cr_arrival_event_data.sp_handle.psp_handle != h_psp) ) {
-            fprintf(stderr, "%d Error wrong cr event data : %s\n",
-                   getpid(),DT_EventToSTr(event.event_number));
-           return( DAT_ABORT );
-       }
-       
-       /* use to test rdma_cma timeout logic */
+               l_iov.segment_length = sizeof(DAT_RMR_TRIPLET);
+
+               LOGPRINTF("%d Posting Receive Message Buffer %p\n",
+                         getpid(), &rmr_recv_msg[i]);
+               ret = dat_ep_post_recv(h_ep,
+                                      1,
+                                      &l_iov,
+                                      cookie, DAT_COMPLETION_DEFAULT_FLAG);
+
+               if (ret != DAT_SUCCESS) {
+                       fprintf(stderr,
+                               "%d Error registering recv msg buffer: %s\n",
+                               getpid(), DT_RetToString(ret));
+                       return (ret);
+               } else
+                       LOGPRINTF("%d Registered Receive Message Buffer %p\n",
+                                 getpid(), region.for_va);
+
+       }
+
+       /* setup receive rdma buffer to initial string to be overwritten */
+       strcpy((char *)rbuf, "blah, blah, blah\n");
+
+       if (server) {           /* SERVER */
+
+               /* create the service point for server listen */
+               LOGPRINTF("%d Creating service point for listen\n", getpid());
+               ret = dat_psp_create(h_ia,
+                                    conn_id,
+                                    h_cr_evd, DAT_PSP_CONSUMER_FLAG, &h_psp);
+               if (ret != DAT_SUCCESS) {
+                       fprintf(stderr, "%d Error dat_psp_create: %s\n",
+                               getpid(), DT_RetToString(ret));
+                       return (ret);
+               } else
+                       LOGPRINTF("%d dat_psp_created for server listen\n",
+                                 getpid());
+
+               printf("%d Server waiting for connect request on port " F64x
+                      "\n", getpid(), conn_id);
+
+               ret = dat_evd_wait(h_cr_evd, SERVER_TIMEOUT, 1, &event, &nmore);
+               if (ret != DAT_SUCCESS) {
+                       fprintf(stderr, "%d Error dat_evd_wait: %s\n",
+                               getpid(), DT_RetToString(ret));
+                       return (ret);
+               } else
+                       LOGPRINTF("%d dat_evd_wait for cr_evd completed\n",
+                                 getpid());
+
+               if (event.event_number != DAT_CONNECTION_REQUEST_EVENT) {
+                       fprintf(stderr, "%d Error unexpected cr event : %s\n",
+                               getpid(), DT_EventToSTr(event.event_number));
+                       return (DAT_ABORT);
+               }
+               if ((event.event_data.cr_arrival_event_data.conn_qual !=
+                    SERVER_CONN_QUAL)
+                   || (event.event_data.cr_arrival_event_data.sp_handle.
+                       psp_handle != h_psp)) {
+                       fprintf(stderr, "%d Error wrong cr event data : %s\n",
+                               getpid(), DT_EventToSTr(event.event_number));
+                       return (DAT_ABORT);
+               }
+
+               /* use to test rdma_cma timeout logic */
 #if defined(_WIN32) || defined(_WIN64)
-       if (delay) Sleep(delay*1000);
+               if (delay)
+                       Sleep(delay * 1000);
 #else
-       if (delay) sleep(delay);
+               if (delay)
+                       sleep(delay);
 #endif
 
-        /* accept connect request from client */
-       h_cr = event.event_data.cr_arrival_event_data.cr_handle;
-       LOGPRINTF("%d Accepting connect request from client\n",getpid());
-
-       /* private data - check and send it back */
-       dat_cr_query( h_cr, DAT_CSP_FIELD_ALL, &cr_param); 
-
-       buf = (unsigned char*)cr_param.private_data;
-       LOGPRINTF("%d CONN REQUEST Private Data %p[0]=%d [47]=%d\n",
-                 getpid(),buf,buf[0],buf[47]);
-       for (i=0;i<48;i++) {
-           if (buf[i] != i+1) {
-               fprintf(stderr, "%d Error with CONNECT REQUEST"
-                       " private data: %p[%d]=%d s/be %d\n",
-                       getpid(), buf, i, buf[i], i+1);
-               dat_cr_reject(h_cr, 0, NULL);
-               return(DAT_ABORT);
-           }
-          buf[i]++; /* change for trip back */
-       }       
+               /* accept connect request from client */
+               h_cr = event.event_data.cr_arrival_event_data.cr_handle;
+               LOGPRINTF("%d Accepting connect request from client\n",
+                         getpid());
+
+               /* private data - check and send it back */
+               dat_cr_query(h_cr, DAT_CSP_FIELD_ALL, &cr_param);
+
+               buf = (unsigned char *)cr_param.private_data;
+               LOGPRINTF("%d CONN REQUEST Private Data %p[0]=%d [47]=%d\n",
+                         getpid(), buf, buf[0], buf[47]);
+               for (i = 0; i < 48; i++) {
+                       if (buf[i] != i + 1) {
+                               fprintf(stderr, "%d Error with CONNECT REQUEST"
+                                       " private data: %p[%d]=%d s/be %d\n",
+                                       getpid(), buf, i, buf[i], i + 1);
+                               dat_cr_reject(h_cr, 0, NULL);
+                               return (DAT_ABORT);
+                       }
+                       buf[i]++;       /* change for trip back */
+               }
 
 #ifdef TEST_REJECT_WITH_PRIVATE_DATA
-       printf("%d REJECT request with 48 bytes of private data\n", getpid());
-       ret = dat_cr_reject(h_cr, 48, cr_param.private_data);
-       printf("\n%d: DAPL Test Complete. %s\n\n",
-             getpid(), ret?"FAILED":"PASSED");
-       exit(0);
+               printf("%d REJECT request with 48 bytes of private data\n",
+                      getpid());
+               ret = dat_cr_reject(h_cr, 48, cr_param.private_data);
+               printf("\n%d: DAPL Test Complete. %s\n\n",
+                      getpid(), ret ? "FAILED" : "PASSED");
+               exit(0);
 #endif
 
-       ret = dat_cr_accept(h_cr, h_ep, 48, cr_param.private_data);
+               ret = dat_cr_accept(h_cr, h_ep, 48, cr_param.private_data);
 
-       if(ret != DAT_SUCCESS) {
-           fprintf(stderr, "%d Error dat_cr_accept: %s\n",
-                   getpid(),DT_RetToString(ret));
-           return(ret);
-       }
-       else
-           LOGPRINTF("%d dat_cr_accept completed\n", getpid());
-    }
-    else {  /* CLIENT */
-       struct addrinfo *target;
-       int             rval;
+               if (ret != DAT_SUCCESS) {
+                       fprintf(stderr, "%d Error dat_cr_accept: %s\n",
+                               getpid(), DT_RetToString(ret));
+                       return (ret);
+               } else
+                       LOGPRINTF("%d dat_cr_accept completed\n", getpid());
+       } else {                /* CLIENT */
+               struct addrinfo *target;
+               int rval;
 
 #if defined(_WIN32) || defined(_WIN64)
-       if ((rval=getaddrinfo (hostname, "ftp", NULL, &target)) != 0) {
-           printf("\n remote name resolution failed! %s\n",gai_strerror(rval));
-           exit ( 1 );
-       }
-       rval = ((struct sockaddr_in *)target->ai_addr)->sin_addr.s_addr;
+               if ((rval = getaddrinfo(hostname, "ftp", NULL, &target)) != 0) {
+                       printf("\n remote name resolution failed! %s\n",
+                              gai_strerror(rval));
+                       exit(1);
+               }
+               rval = ((struct sockaddr_in *)target->ai_addr)->sin_addr.s_addr;
 #else
-       if (getaddrinfo (hostname, NULL, NULL, &target) != 0) {
-           perror("\n remote name resolution failed!");
-           exit ( 1 );
-       }
-       rval = ((struct sockaddr_in *)target->ai_addr)->sin_addr.s_addr;
+               if (getaddrinfo(hostname, NULL, NULL, &target) != 0) {
+                       perror("\n remote name resolution failed!");
+                       exit(1);
+               }
+               rval = ((struct sockaddr_in *)target->ai_addr)->sin_addr.s_addr;
 #endif
-       printf ("%d Server Name: %s \n", getpid(), hostname);
-       printf ("%d Server Net Address: %d.%d.%d.%d port "F64x"\n", getpid(),
-               (rval >>  0) & 0xff, (rval >>  8) & 0xff,
-               (rval >> 16) & 0xff, (rval >> 24) & 0xff, conn_id);
-
-       remote_addr = *((DAT_IA_ADDRESS_PTR)target->ai_addr);
-       freeaddrinfo(target);
-
-       for (i=0;i<48;i++) /* simple pattern in private data */
-           pdata[i]=i+1;
-
-       LOGPRINTF("%d Connecting to server\n",getpid());
-       ret = dat_ep_connect(   h_ep,
-                               &remote_addr,
-                               conn_id,
-                               CONN_TIMEOUT,
-                               48,
-                               (DAT_PVOID)pdata,
-                               0,
-                               DAT_CONNECT_DEFAULT_FLAG  );
-       if(ret != DAT_SUCCESS) {
-           fprintf(stderr, "%d Error dat_ep_connect: %s\n",
-                               getpid(), DT_RetToString(ret));
-           return(ret);
-       }
-       else
-           LOGPRINTF("%d dat_ep_connect completed\n", getpid());
-    }
-
-    printf("%d Waiting for connect response\n",getpid());
-
-    ret = dat_evd_wait( h_conn_evd, DAT_TIMEOUT_INFINITE, 1, &event, &nmore );
-    if(ret != DAT_SUCCESS) {
-           fprintf(stderr, "%d Error dat_evd_wait: %s\n",
-                           getpid(),DT_RetToString(ret));
-           return(ret);
-    }
-    else
-           LOGPRINTF("%d dat_evd_wait for h_conn_evd completed\n", getpid());
+               printf("%d Server Name: %s \n", getpid(), hostname);
+               printf("%d Server Net Address: %d.%d.%d.%d port " F64x "\n",
+                      getpid(), (rval >> 0) & 0xff, (rval >> 8) & 0xff,
+                      (rval >> 16) & 0xff, (rval >> 24) & 0xff, conn_id);
+
+               remote_addr = *((DAT_IA_ADDRESS_PTR) target->ai_addr);
+               freeaddrinfo(target);
+
+               for (i = 0; i < 48; i++)        /* simple pattern in private data */
+                       pdata[i] = i + 1;
+
+               LOGPRINTF("%d Connecting to server\n", getpid());
+               ret = dat_ep_connect(h_ep,
+                                    &remote_addr,
+                                    conn_id,
+                                    CONN_TIMEOUT,
+                                    48,
+                                    (DAT_PVOID) pdata,
+                                    0, DAT_CONNECT_DEFAULT_FLAG);
+               if (ret != DAT_SUCCESS) {
+                       fprintf(stderr, "%d Error dat_ep_connect: %s\n",
+                               getpid(), DT_RetToString(ret));
+                       return (ret);
+               } else
+                       LOGPRINTF("%d dat_ep_connect completed\n", getpid());
+       }
+
+       printf("%d Waiting for connect response\n", getpid());
+
+       ret = dat_evd_wait(h_conn_evd, DAT_TIMEOUT_INFINITE, 1, &event, &nmore);
+       if (ret != DAT_SUCCESS) {
+               fprintf(stderr, "%d Error dat_evd_wait: %s\n",
+                       getpid(), DT_RetToString(ret));
+               return (ret);
+       } else
+               LOGPRINTF("%d dat_evd_wait for h_conn_evd completed\n",
+                         getpid());
 
 #ifdef TEST_REJECT_WITH_PRIVATE_DATA
-    if (event.event_number != DAT_CONNECTION_EVENT_PEER_REJECTED) {
-           fprintf(stderr, "%d expected conn reject event : %s\n",
-                               getpid(),DT_EventToSTr(event.event_number));
-           return( DAT_ABORT );
-    }
-    /* get the reject private data and validate */
-    buf = (unsigned char*)event.event_data.connect_event_data.private_data;
-    printf("%d Received REJECT with private data %p[0]=%d [47]=%d\n",
-           getpid(),buf,buf[0],buf[47]);
-    for (i=0;i<48;i++) {
-        if (buf[i] != i+2) {
-            fprintf(stderr, "%d client: Error with REJECT event"
-                    " private data: %p[%d]=%d s/be %d\n",
-                    getpid(), buf, i, buf[i], i+2);
-            dat_ep_disconnect( h_ep, DAT_CLOSE_ABRUPT_FLAG);
-            return(DAT_ABORT);
-        }
-    }
-    printf("\n%d: DAPL Test Complete. PASSED\n\n", getpid());
-    exit(0);
+       if (event.event_number != DAT_CONNECTION_EVENT_PEER_REJECTED) {
+               fprintf(stderr, "%d expected conn reject event : %s\n",
+                       getpid(), DT_EventToSTr(event.event_number));
+               return (DAT_ABORT);
+       }
+       /* get the reject private data and validate */
+       buf = (unsigned char *)event.event_data.connect_event_data.private_data;
+       printf("%d Received REJECT with private data %p[0]=%d [47]=%d\n",
+              getpid(), buf, buf[0], buf[47]);
+       for (i = 0; i < 48; i++) {
+               if (buf[i] != i + 2) {
+                       fprintf(stderr, "%d client: Error with REJECT event"
+                               " private data: %p[%d]=%d s/be %d\n",
+                               getpid(), buf, i, buf[i], i + 2);
+                       dat_ep_disconnect(h_ep, DAT_CLOSE_ABRUPT_FLAG);
+                       return (DAT_ABORT);
+               }
+       }
+       printf("\n%d: DAPL Test Complete. PASSED\n\n", getpid());
+       exit(0);
 #endif
 
-    if ( event.event_number != DAT_CONNECTION_EVENT_ESTABLISHED ) {
-           fprintf(stderr, "%d Error unexpected conn event : %s\n",
-                               getpid(),DT_EventToSTr(event.event_number));
-           return( DAT_ABORT );
-    }
-
-    /* check private data back from server  */
-    if (!server) {
-        buf = (unsigned char*)event.event_data.connect_event_data.private_data;
-        LOGPRINTF("%d CONN Private Data %p[0]=%d [47]=%d\n",
-                  getpid(),buf,buf[0],buf[47]);
-        for (i=0;i<48;i++) {
-            if (buf[i] != i+2) {
-                fprintf(stderr, "%d Error with CONNECT event"
-                        " private data: %p[%d]=%d s/be %d\n",
-                        getpid(), buf, i, buf[i], i+2);
-                dat_ep_disconnect(h_ep, DAT_CLOSE_ABRUPT_FLAG);
-                LOGPRINTF("%d waiting for disconnect event...\n", getpid());
-                dat_evd_wait(h_conn_evd, DAT_TIMEOUT_INFINITE, 1, &event, &nmore);
-                return(DAT_ABORT);
-            }
-        }
-    }
-
-    printf("\n%d CONNECTED!\n\n",getpid());
-    connected = 1;
+       if (event.event_number != DAT_CONNECTION_EVENT_ESTABLISHED) {
+               fprintf(stderr, "%d Error unexpected conn event : %s\n",
+                       getpid(), DT_EventToSTr(event.event_number));
+               return (DAT_ABORT);
+       }
+
+       /* check private data back from server  */
+       if (!server) {
+               buf =
+                   (unsigned char *)event.event_data.connect_event_data.
+                   private_data;
+               LOGPRINTF("%d CONN Private Data %p[0]=%d [47]=%d\n", getpid(),
+                         buf, buf[0], buf[47]);
+               for (i = 0; i < 48; i++) {
+                       if (buf[i] != i + 2) {
+                               fprintf(stderr, "%d Error with CONNECT event"
+                                       " private data: %p[%d]=%d s/be %d\n",
+                                       getpid(), buf, i, buf[i], i + 2);
+                               dat_ep_disconnect(h_ep, DAT_CLOSE_ABRUPT_FLAG);
+                               LOGPRINTF
+                                   ("%d waiting for disconnect event...\n",
+                                    getpid());
+                               dat_evd_wait(h_conn_evd, DAT_TIMEOUT_INFINITE,
+                                            1, &event, &nmore);
+                               return (DAT_ABORT);
+                       }
+               }
+       }
+
+       printf("\n%d CONNECTED!\n\n", getpid());
+       connected = 1;
 
 #if CONNECT_ONLY
-    return 0;
+       return 0;
 #endif
 
-    /*
-     *  Setup our remote memory and tell the other side about it
-     */
-    rmr_send_msg.virtual_address = htonll((DAT_VADDR)(uintptr_t)rbuf);
-    rmr_send_msg.segment_length  = htonl(RDMA_BUFFER_SIZE);
-    rmr_send_msg.rmr_context     = htonl(rmr_context_recv);
-
-    printf("%d Send RMR msg to remote: r_key_ctx=0x%x,va=%p,len=0x%x\n",
-           getpid(), rmr_context_recv, rbuf, RDMA_BUFFER_SIZE );
-
-    ret = send_msg(&rmr_send_msg,
-                   sizeof( DAT_RMR_TRIPLET ),
-                   lmr_context_send_msg,
-                   cookie,
-                   DAT_COMPLETION_SUPPRESS_FLAG);
-
-    if(ret != DAT_SUCCESS) {
-        fprintf(stderr, "%d Error send_msg: %s\n",
-               getpid(),DT_RetToString(ret));
-       return(ret);
-    }
-    else
-       LOGPRINTF("%d send_msg completed\n", getpid());
-
-    /*
-     *  Wait for remote RMR information for RDMA
-     */
-    if ( polling ) {
-       printf("%d Polling for remote to send RMR data\n",getpid());
-       while (  dat_evd_dequeue( h_dto_rcv_evd, &event ) == DAT_QUEUE_EMPTY );
-    }
-    else  {
-       printf("%d Waiting for remote to send RMR data\n",getpid());
-       if (use_cno)
-       {
-           DAT_EVD_HANDLE evd = DAT_HANDLE_NULL;
-           ret = dat_cno_wait( h_dto_cno, DTO_TIMEOUT, &evd );
-           LOGPRINTF("%d cno wait return evd_handle=%p\n", getpid(),evd);
-           if ( evd != h_dto_rcv_evd ) {
-               fprintf(stderr,
-                       "%d Error waiting on h_dto_cno: evd != h_dto_rcv_evd\n",
-                       getpid());
-               return( DAT_ABORT );
-           }
-       }
-       /* use wait to dequeue */
-       ret = dat_evd_wait( h_dto_rcv_evd, DTO_TIMEOUT, 1, &event, &nmore );
-       if(ret != DAT_SUCCESS) {
-           fprintf(stderr, "%d Error waiting on h_dto_rcv_evd: %s\n",
-                   getpid(),DT_RetToString(ret));
-           return(ret);
-       }
-       else {
-           LOGPRINTF("%d dat_evd_wait h_dto_rcv_evd completed\n", getpid());
-       }
-    }
-
-    printf("%d remote RMR data arrived!\n",getpid());
-
-    if ( event.event_number != DAT_DTO_COMPLETION_EVENT ) {
-        fprintf(stderr, "%d Error unexpected DTO event : %s\n",
-               getpid(),DT_EventToSTr(event.event_number));
-        return( DAT_ABORT );
-    }
-    if ((event.event_data.dto_completion_event_data.transfered_length !=
-               sizeof( DAT_RMR_TRIPLET )) ||
-       (event.event_data.dto_completion_event_data.user_cookie.as_64 !=
-               recv_msg_index) ) {
-       fprintf(stderr,"ERR recv event: len=%d cookie="F64x" expected %d/%d\n",
-           (int)event.event_data.dto_completion_event_data.transfered_length,
-           event.event_data.dto_completion_event_data.user_cookie.as_64,
-           (int)sizeof(DAT_RMR_TRIPLET), recv_msg_index );
-       return( DAT_ABORT );
-    }
-
-    /* swap received RMR msg: network order to host order */
-    r_iov = rmr_recv_msg[recv_msg_index];
-    rmr_recv_msg[recv_msg_index].rmr_context = 
-           ntohl(r_iov.rmr_context);
-    rmr_recv_msg[recv_msg_index].virtual_address = 
+       /*
+        *  Setup our remote memory and tell the other side about it
+        */
+       rmr_send_msg.virtual_address = htonll((DAT_VADDR) (uintptr_t) rbuf);
+       rmr_send_msg.segment_length = htonl(RDMA_BUFFER_SIZE);
+       rmr_send_msg.rmr_context = htonl(rmr_context_recv);
+
+       printf("%d Send RMR msg to remote: r_key_ctx=0x%x,va=%p,len=0x%x\n",
+              getpid(), rmr_context_recv, rbuf, RDMA_BUFFER_SIZE);
+
+       ret = send_msg(&rmr_send_msg,
+                      sizeof(DAT_RMR_TRIPLET),
+                      lmr_context_send_msg,
+                      cookie, DAT_COMPLETION_SUPPRESS_FLAG);
+
+       if (ret != DAT_SUCCESS) {
+               fprintf(stderr, "%d Error send_msg: %s\n",
+                       getpid(), DT_RetToString(ret));
+               return (ret);
+       } else
+               LOGPRINTF("%d send_msg completed\n", getpid());
+
+       /*
+        *  Wait for remote RMR information for RDMA
+        */
+       if (polling) {
+               printf("%d Polling for remote to send RMR data\n", getpid());
+               while (dat_evd_dequeue(h_dto_rcv_evd, &event) ==
+                      DAT_QUEUE_EMPTY) ;
+       } else {
+               printf("%d Waiting for remote to send RMR data\n", getpid());
+               if (use_cno) {
+                       DAT_EVD_HANDLE evd = DAT_HANDLE_NULL;
+                       ret = dat_cno_wait(h_dto_cno, DTO_TIMEOUT, &evd);
+                       LOGPRINTF("%d cno wait return evd_handle=%p\n",
+                                 getpid(), evd);
+                       if (evd != h_dto_rcv_evd) {
+                               fprintf(stderr,
+                                       "%d Error waiting on h_dto_cno: evd != h_dto_rcv_evd\n",
+                                       getpid());
+                               return (DAT_ABORT);
+                       }
+               }
+               /* use wait to dequeue */
+               ret =
+                   dat_evd_wait(h_dto_rcv_evd, DTO_TIMEOUT, 1, &event, &nmore);
+               if (ret != DAT_SUCCESS) {
+                       fprintf(stderr,
+                               "%d Error waiting on h_dto_rcv_evd: %s\n",
+                               getpid(), DT_RetToString(ret));
+                       return (ret);
+               } else {
+                       LOGPRINTF("%d dat_evd_wait h_dto_rcv_evd completed\n",
+                                 getpid());
+               }
+       }
+
+       printf("%d remote RMR data arrived!\n", getpid());
+
+       if (event.event_number != DAT_DTO_COMPLETION_EVENT) {
+               fprintf(stderr, "%d Error unexpected DTO event : %s\n",
+                       getpid(), DT_EventToSTr(event.event_number));
+               return (DAT_ABORT);
+       }
+       if ((event.event_data.dto_completion_event_data.transfered_length !=
+            sizeof(DAT_RMR_TRIPLET)) ||
+           (event.event_data.dto_completion_event_data.user_cookie.as_64 !=
+            recv_msg_index)) {
+               fprintf(stderr,
+                       "ERR recv event: len=%d cookie=" F64x
+                       " expected %d/%d\n",
+                       (int)event.event_data.dto_completion_event_data.
+                       transfered_length,
+                       event.event_data.dto_completion_event_data.user_cookie.
+                       as_64, (int)sizeof(DAT_RMR_TRIPLET), recv_msg_index);
+               return (DAT_ABORT);
+       }
+
+       /* swap received RMR msg: network order to host order */
+       r_iov = rmr_recv_msg[recv_msg_index];
+       rmr_recv_msg[recv_msg_index].rmr_context = ntohl(r_iov.rmr_context);
+       rmr_recv_msg[recv_msg_index].virtual_address =
            ntohll(r_iov.virtual_address);
-    rmr_recv_msg[recv_msg_index].segment_length = 
+       rmr_recv_msg[recv_msg_index].segment_length =
            ntohl(r_iov.segment_length);
 
-    printf("%d Received RMR from remote: "
-           "r_iov: r_key_ctx=%x,va="F64x",len=0x%x\n",
-           getpid(), rmr_recv_msg[recv_msg_index].rmr_context,
-           rmr_recv_msg[recv_msg_index].virtual_address,
-           rmr_recv_msg[recv_msg_index].segment_length );
+       printf("%d Received RMR from remote: "
+              "r_iov: r_key_ctx=%x,va=" F64x ",len=0x%x\n",
+              getpid(), rmr_recv_msg[recv_msg_index].rmr_context,
+              rmr_recv_msg[recv_msg_index].virtual_address,
+              rmr_recv_msg[recv_msg_index].segment_length);
 
-    recv_msg_index++;
+       recv_msg_index++;
 
-    return ( DAT_SUCCESS );
+       return (DAT_SUCCESS);
 }
 
-void
-disconnect_ep( void )
+void disconnect_ep(void)
 {
-    DAT_RETURN ret;
-    DAT_EVENT  event;
-    DAT_COUNT  nmore;
-
-    if (connected) {
-
-       /* 
-        * Only the client needs to call disconnect. The server _should_ be able
-        * to just wait on the EVD associated with connection events for a
-        * disconnect request and then exit.
-        */
-       if ( !server ) {
-           LOGPRINTF("%d dat_ep_disconnect\n", getpid());
-           ret = dat_ep_disconnect( h_ep, DAT_CLOSE_DEFAULT );
-           if(ret != DAT_SUCCESS)  {
-                   fprintf(stderr, "%d Error dat_ep_disconnect: %s\n",
-                           getpid(),DT_RetToString(ret));
-           }
-           else {
-               LOGPRINTF("%d dat_ep_disconnect completed\n", getpid());
-           }
-       }
-       else {
-           LOGPRINTF("%d Server waiting for disconnect...\n", getpid());
-       }
-
-       ret = dat_evd_wait(h_conn_evd, DAT_TIMEOUT_INFINITE, 1, &event, &nmore);
-       if(ret != DAT_SUCCESS) {
-           fprintf(stderr, "%d Error dat_evd_wait: %s\n",
-                               getpid(),DT_RetToString(ret));
-       }
-       else {
-           LOGPRINTF("%d dat_evd_wait for h_conn_evd completed\n", getpid());
-       }
-    }
-
-    /* destroy service point */
-    if (( server ) && ( h_psp != DAT_HANDLE_NULL )) {
-       ret = dat_psp_free( h_psp );
-        if(ret != DAT_SUCCESS) {
-           fprintf(stderr, "%d Error dat_psp_free: %s\n",
-                   getpid(),DT_RetToString(ret));
-       }
-       else {
-           LOGPRINTF("%d dat_psp_free completed\n", getpid());
-       }
-    }
-
-    /* Unregister Send message Buffer */
-    if ( h_lmr_send_msg != DAT_HANDLE_NULL ) {
-       LOGPRINTF("%d Unregister send message h_lmr %p \n",getpid(),h_lmr_send_msg);
-       ret = dat_lmr_free(h_lmr_send_msg);
-        if(ret != DAT_SUCCESS) {
-           fprintf(stderr, "%d Error deregistering send msg mr: %s\n",
-           getpid(), DT_RetToString(ret));
-       } else {
-           LOGPRINTF("%d Unregistered send message Buffer\n",getpid());
-           h_lmr_send_msg = NULL;
-       }
-    }
-
-    /* Unregister recv message Buffer */
-    if ( h_lmr_recv_msg != DAT_HANDLE_NULL ) {
-       LOGPRINTF("%d Unregister recv message h_lmr %p \n",getpid(),h_lmr_recv_msg);
-       ret = dat_lmr_free(h_lmr_recv_msg);
-        if(ret != DAT_SUCCESS) {
-           fprintf(stderr, "%d Error deregistering recv msg mr: %s\n",
-                           getpid(), DT_RetToString(ret));
-       } else {
-           LOGPRINTF("%d Unregistered recv message Buffer\n",getpid());
-           h_lmr_recv_msg = NULL;
-       }
-    }
-    return;
-}
+       DAT_RETURN ret;
+       DAT_EVENT event;
+       DAT_COUNT nmore;
+
+       if (connected) {
+
+               /* 
+                * Only the client needs to call disconnect. The server _should_ be able
+                * to just wait on the EVD associated with connection events for a
+                * disconnect request and then exit.
+                */
+               if (!server) {
+                       LOGPRINTF("%d dat_ep_disconnect\n", getpid());
+                       ret = dat_ep_disconnect(h_ep, DAT_CLOSE_DEFAULT);
+                       if (ret != DAT_SUCCESS) {
+                               fprintf(stderr,
+                                       "%d Error dat_ep_disconnect: %s\n",
+                                       getpid(), DT_RetToString(ret));
+                       } else {
+                               LOGPRINTF("%d dat_ep_disconnect completed\n",
+                                         getpid());
+                       }
+               } else {
+                       LOGPRINTF("%d Server waiting for disconnect...\n",
+                                 getpid());
+               }
 
+               ret =
+                   dat_evd_wait(h_conn_evd, DAT_TIMEOUT_INFINITE, 1, &event,
+                                &nmore);
+               if (ret != DAT_SUCCESS) {
+                       fprintf(stderr, "%d Error dat_evd_wait: %s\n",
+                               getpid(), DT_RetToString(ret));
+               } else {
+                       LOGPRINTF("%d dat_evd_wait for h_conn_evd completed\n",
+                                 getpid());
+               }
+       }
 
-DAT_RETURN
-do_rdma_write_with_msg( void )
-{
-       DAT_EVENT               event;
-       DAT_COUNT               nmore;
-       DAT_LMR_TRIPLET         l_iov[MSG_IOV_COUNT];
-       DAT_RMR_TRIPLET         r_iov;
-       DAT_DTO_COOKIE          cookie;
-       DAT_RETURN              ret;
-       int                     i;
-
-       printf("\n %d RDMA WRITE DATA with SEND MSG\n\n",getpid());
-
-       cookie.as_64 = 0x5555;
-
-       if ( recv_msg_index >= MSG_BUF_COUNT )
-               return( DAT_ABORT );
-
-       /* get RMR information from previously received message */
-       r_iov = rmr_recv_msg[ recv_msg_index-1 ];
-
-       if ( server )
-           strcpy( (char*)sbuf, "server RDMA write data..." );
-       else
-           strcpy( (char*)sbuf, "client RDMA write data..." );
-
-       for (i=0;i<MSG_IOV_COUNT;i++) {
-          l_iov[i].lmr_context     = lmr_context_send;
-          l_iov[i].segment_length  = buf_len/MSG_IOV_COUNT;
-          l_iov[i].virtual_address = (DAT_VADDR)(uintptr_t)
-                                       (&sbuf[l_iov[i].segment_length*i]);
-
-          LOGPRINTF("%d rdma_write iov[%d] buf=%p,len=%d\n", 
-                       getpid(), i, &sbuf[l_iov[i].segment_length*i],
-                       l_iov[i].segment_length);
-       }
-
-       start = get_time();
-       for (i=0;i<burst;i++) {
-           cookie.as_64 = 0x9999;
-           ret = dat_ep_post_rdma_write(   h_ep,               // ep_handle
-                                           MSG_IOV_COUNT,      // num_segments
-                                           l_iov,              // LMR
-                                           cookie,             // user_cookie
-                                           &r_iov,             // RMR
-                                           DAT_COMPLETION_SUPPRESS_FLAG );
-           if (ret != DAT_SUCCESS) {
-               fprintf(stderr, "%d: ERROR: dat_ep_post_rdma_write() %s\n",
-                                       getpid(),DT_RetToString(ret));
-               return( DAT_ABORT );
-           }
-           LOGPRINTF("%d rdma_write # %d completed\n", getpid(),i+1);
-       }
-
-       /*
-        *  Send RMR information a 2nd time to indicate completion
-        *  NOTE: already swapped to network order in connect_ep
-        */
-       printf("%d Sending RDMA WRITE completion message\n",getpid());
-
-       ret = send_msg( &rmr_send_msg,
-                       sizeof( DAT_RMR_TRIPLET ),
-                       lmr_context_send_msg,
-                       cookie,
-                       DAT_COMPLETION_SUPPRESS_FLAG );
-
-       if(ret != DAT_SUCCESS) {
-               fprintf(stderr, "%d Error send_msg: %s\n",
-                               getpid(),DT_RetToString(ret));
-               return(ret);
-       } else {
-               LOGPRINTF("%d send_msg completed\n", getpid());
-       }
-
-       /*
-        *  Collect first event, write completion or the inbound recv 
-        */
-       if (polling) {
-           while (dat_evd_dequeue(h_dto_rcv_evd, &event) == DAT_QUEUE_EMPTY)
-               rdma_wr_poll_count++;
-       }
-       else {
-           LOGPRINTF("%d waiting for message receive event\n", getpid());
-           if (use_cno)  {
-                   DAT_EVD_HANDLE evd = DAT_HANDLE_NULL;
-                   ret = dat_cno_wait( h_dto_cno, DTO_TIMEOUT, &evd );
-                   LOGPRINTF("%d cno wait return evd_handle=%p\n",
-                               getpid(),evd);
-                   if ( evd != h_dto_rcv_evd ) {
-                           fprintf(stderr, "%d Error waiting on h_dto_cno: "
-                                  "evd != h_dto_rcv_evd\n",
-                                   getpid());
-                           return( ret );
-                   }
-           }
-           /* use wait to dequeue */
-           ret = dat_evd_wait(h_dto_rcv_evd, DTO_TIMEOUT, 1, &event, &nmore);
-           if (ret != DAT_SUCCESS) {
-                   fprintf(stderr, "%d: ERROR: DTO dat_evd_wait() %s\n",
-                                           getpid(),DT_RetToString(ret));
-                   return( ret );
-           }
-       }
-       stop = get_time();
-       time.rdma_wr = ((stop - start)*1.0e6);
-
-       /* validate event number and status */
-       printf("%d inbound rdma_write; send message arrived!\n",getpid());
-       if ( event.event_number != DAT_DTO_COMPLETION_EVENT ) {
-           fprintf(stderr, "%d Error unexpected DTO event : %s\n",
-                               getpid(),DT_EventToSTr(event.event_number));
-           return( DAT_ABORT );
-       }
-
-       if ( (event.event_data.dto_completion_event_data.transfered_length != sizeof( DAT_RMR_TRIPLET )) ||
-            (event.event_data.dto_completion_event_data.user_cookie.as_64 != recv_msg_index) ) {
-           fprintf(stderr,"unexpected event data for receive: len=%d cookie="F64x" exp %d/%d\n",
-               (int)event.event_data.dto_completion_event_data.transfered_length,
-               event.event_data.dto_completion_event_data.user_cookie.as_64,
-               (int)sizeof(DAT_RMR_TRIPLET), recv_msg_index );
-
-           return( DAT_ABORT );
-       }
-
-       /* swap received RMR msg: network order to host order */
-       r_iov = rmr_recv_msg[recv_msg_index];
-       rmr_recv_msg[recv_msg_index].virtual_address =
-               ntohll(rmr_recv_msg[recv_msg_index].virtual_address);
-       rmr_recv_msg[recv_msg_index].segment_length =
-               ntohl(rmr_recv_msg[recv_msg_index].segment_length);
-       rmr_recv_msg[recv_msg_index].rmr_context =
-               ntohl(rmr_recv_msg[recv_msg_index].rmr_context);
-
-       printf("%d Received RMR from remote: "
-              "r_iov: r_key_ctx=%x,va="F64x",len=0x%x\n",
-              getpid(), rmr_recv_msg[recv_msg_index].rmr_context,
-              rmr_recv_msg[recv_msg_index].virtual_address,
-              rmr_recv_msg[recv_msg_index].segment_length);
-
-       LOGPRINTF("%d inbound rdma_write; send msg event SUCCESS!!\n",
-                getpid());
-
-       printf("%d %s RDMA write buffer contains: %s\n",
-                       getpid(),
-                       server ? "SERVER:" : "CLIENT:",
-                       rbuf );
-
-       recv_msg_index++;
-
-       return ( DAT_SUCCESS );
+       /* destroy service point */
+       if ((server) && (h_psp != DAT_HANDLE_NULL)) {
+               ret = dat_psp_free(h_psp);
+               if (ret != DAT_SUCCESS) {
+                       fprintf(stderr, "%d Error dat_psp_free: %s\n",
+                               getpid(), DT_RetToString(ret));
+               } else {
+                       LOGPRINTF("%d dat_psp_free completed\n", getpid());
+               }
+       }
+
+       /* Unregister Send message Buffer */
+       if (h_lmr_send_msg != DAT_HANDLE_NULL) {
+               LOGPRINTF("%d Unregister send message h_lmr %p \n", getpid(),
+                         h_lmr_send_msg);
+               ret = dat_lmr_free(h_lmr_send_msg);
+               if (ret != DAT_SUCCESS) {
+                       fprintf(stderr,
+                               "%d Error deregistering send msg mr: %s\n",
+                               getpid(), DT_RetToString(ret));
+               } else {
+                       LOGPRINTF("%d Unregistered send message Buffer\n",
+                                 getpid());
+                       h_lmr_send_msg = NULL;
+               }
+       }
+
+       /* Unregister recv message Buffer */
+       if (h_lmr_recv_msg != DAT_HANDLE_NULL) {
+               LOGPRINTF("%d Unregister recv message h_lmr %p \n", getpid(),
+                         h_lmr_recv_msg);
+               ret = dat_lmr_free(h_lmr_recv_msg);
+               if (ret != DAT_SUCCESS) {
+                       fprintf(stderr,
+                               "%d Error deregistering recv msg mr: %s\n",
+                               getpid(), DT_RetToString(ret));
+               } else {
+                       LOGPRINTF("%d Unregistered recv message Buffer\n",
+                                 getpid());
+                       h_lmr_recv_msg = NULL;
+               }
+       }
+       return;
 }
 
-DAT_RETURN
-do_rdma_read_with_msg( void )
+DAT_RETURN do_rdma_write_with_msg(void)
 {
-       DAT_EVENT               event;
-       DAT_COUNT               nmore;
-       DAT_LMR_TRIPLET         l_iov;
-       DAT_RMR_TRIPLET         r_iov;
-       DAT_DTO_COOKIE          cookie;
-       DAT_RETURN              ret;
-       int                     i;
-
-       printf("\n %d RDMA READ DATA with SEND MSG\n\n",getpid());
-
-       if (recv_msg_index >= MSG_BUF_COUNT)
-               return( DAT_ABORT );
-
-       /* get RMR information from previously received message */
-       r_iov = rmr_recv_msg[recv_msg_index-1];
-
-       /* setup rdma read buffer to initial string to be overwritten */
-       strcpy((char*)sbuf, "blah, blah, blah\n");
-
-       if (server)
-           strcpy((char*)rbuf, "server RDMA read data...");
-       else
-           strcpy((char*)rbuf, "client RDMA read data...");
-
-       l_iov.lmr_context     = lmr_context_send;
-       l_iov.virtual_address = (DAT_VADDR)(uintptr_t)sbuf;
-       l_iov.segment_length  = buf_len;
-       
-       for (i=0;i<MAX_RDMA_RD;i++) {
-           cookie.as_64 = 0x9999;
-           start = get_time();
-           ret = dat_ep_post_rdma_read(h_ep,           // ep_handle
-                                       1,              // num_segments
-                                       &l_iov,         // LMR
-                                       cookie,         // user_cookie
-                                       &r_iov,         // RMR
-                                       DAT_COMPLETION_DEFAULT_FLAG );
-           if (ret != DAT_SUCCESS) {
-               fprintf(stderr, "%d: ERROR: dat_ep_post_rdma_read() %s\n", 
-                                       getpid(),DT_RetToString(ret));
-               return(DAT_ABORT);
-           }
-           
-           if (polling) {
-               while (dat_evd_dequeue(h_dto_req_evd, &event) == DAT_QUEUE_EMPTY)
-                       rdma_rd_poll_count[i]++;
-           } 
-           else {
-               LOGPRINTF("%d waiting for rdma_read completion event\n", getpid());
+       DAT_EVENT event;
+       DAT_COUNT nmore;
+       DAT_LMR_TRIPLET l_iov[MSG_IOV_COUNT];
+       DAT_RMR_TRIPLET r_iov;
+       DAT_DTO_COOKIE cookie;
+       DAT_RETURN ret;
+       int i;
+
+       printf("\n %d RDMA WRITE DATA with SEND MSG\n\n", getpid());
+
+       cookie.as_64 = 0x5555;
+
+       if (recv_msg_index >= MSG_BUF_COUNT)
+               return (DAT_ABORT);
+
+       /* get RMR information from previously received message */
+       r_iov = rmr_recv_msg[recv_msg_index - 1];
+
+       if (server)
+               strcpy((char *)sbuf, "server RDMA write data...");
+       else
+               strcpy((char *)sbuf, "client RDMA write data...");
+
+       for (i = 0; i < MSG_IOV_COUNT; i++) {
+               l_iov[i].lmr_context = lmr_context_send;
+               l_iov[i].segment_length = buf_len / MSG_IOV_COUNT;
+               l_iov[i].virtual_address = (DAT_VADDR) (uintptr_t)
+                   (&sbuf[l_iov[i].segment_length * i]);
+
+               LOGPRINTF("%d rdma_write iov[%d] buf=%p,len=%d\n",
+                         getpid(), i, &sbuf[l_iov[i].segment_length * i],
+                         l_iov[i].segment_length);
+       }
+
+       start = get_time();
+       for (i = 0; i < burst; i++) {
+               cookie.as_64 = 0x9999;
+               ret = dat_ep_post_rdma_write(h_ep,      // ep_handle
+                                            MSG_IOV_COUNT,     // num_segments
+                                            l_iov,     // LMR
+                                            cookie,    // user_cookie
+                                            &r_iov,    // RMR
+                                            DAT_COMPLETION_SUPPRESS_FLAG);
+               if (ret != DAT_SUCCESS) {
+                       fprintf(stderr,
+                               "%d: ERROR: dat_ep_post_rdma_write() %s\n",
+                               getpid(), DT_RetToString(ret));
+                       return (DAT_ABORT);
+               }
+               LOGPRINTF("%d rdma_write # %d completed\n", getpid(), i + 1);
+       }
+
+       /*
+        *  Send RMR information a 2nd time to indicate completion
+        *  NOTE: already swapped to network order in connect_ep
+        */
+       printf("%d Sending RDMA WRITE completion message\n", getpid());
+
+       ret = send_msg(&rmr_send_msg,
+                      sizeof(DAT_RMR_TRIPLET),
+                      lmr_context_send_msg,
+                      cookie, DAT_COMPLETION_SUPPRESS_FLAG);
+
+       if (ret != DAT_SUCCESS) {
+               fprintf(stderr, "%d Error send_msg: %s\n",
+                       getpid(), DT_RetToString(ret));
+               return (ret);
+       } else {
+               LOGPRINTF("%d send_msg completed\n", getpid());
+       }
+
+       /*
+        *  Collect first event, write completion or the inbound recv 
+        */
+       if (polling) {
+               while (dat_evd_dequeue(h_dto_rcv_evd, &event) ==
+                      DAT_QUEUE_EMPTY)
+                       rdma_wr_poll_count++;
+       } else {
+               LOGPRINTF("%d waiting for message receive event\n", getpid());
                if (use_cno) {
                        DAT_EVD_HANDLE evd = DAT_HANDLE_NULL;
-                       ret = dat_cno_wait( h_dto_cno, DTO_TIMEOUT, &evd );
-                       LOGPRINTF("%d cno wait return evd_handle=%p\n", getpid(),evd);
-                       if ( evd != h_dto_req_evd ) {
-                               fprintf(stderr, 
-                               "%d Error waiting on h_dto_cno: evd != h_dto_req_evd\n", 
-                               getpid());
-                               return( DAT_ABORT );
+                       ret = dat_cno_wait(h_dto_cno, DTO_TIMEOUT, &evd);
+                       LOGPRINTF("%d cno wait return evd_handle=%p\n",
+                                 getpid(), evd);
+                       if (evd != h_dto_rcv_evd) {
+                               fprintf(stderr,
+                                       "%d Error waiting on h_dto_cno: "
+                                       "evd != h_dto_rcv_evd\n", getpid());
+                               return (ret);
                        }
                }
                /* use wait to dequeue */
-               ret = dat_evd_wait( h_dto_req_evd, DTO_TIMEOUT, 1, &event, &nmore );
+               ret =
+                   dat_evd_wait(h_dto_rcv_evd, DTO_TIMEOUT, 1, &event, &nmore);
                if (ret != DAT_SUCCESS) {
-                       fprintf(stderr, "%d: ERROR: DTO dat_evd_wait() %s\n", 
-                               getpid(),DT_RetToString(ret));
-                       return ret;
+                       fprintf(stderr, "%d: ERROR: DTO dat_evd_wait() %s\n",
+                               getpid(), DT_RetToString(ret));
+                       return (ret);
                }
-           }
-           /* validate event number, len, cookie, and status */
-           if (event.event_number != DAT_DTO_COMPLETION_EVENT) {
-               fprintf(stderr, "%d: ERROR: DTO event number %s\n", 
-                       getpid(),DT_EventToSTr(event.event_number));
-               return( DAT_ABORT );
-           }
-           if ((event.event_data.dto_completion_event_data.transfered_length != buf_len ) ||
-               (event.event_data.dto_completion_event_data.user_cookie.as_64 != 0x9999 )) {
-               fprintf(stderr, "%d: ERROR: DTO len %d or cookie "F64x"\n", 
-                       getpid(),
-                       event.event_data.dto_completion_event_data.transfered_length,
-                       event.event_data.dto_completion_event_data.user_cookie.as_64 );
-               return( DAT_ABORT );
-           }
-           if (event.event_data.dto_completion_event_data.status != DAT_SUCCESS) {
-               fprintf(stderr, "%d: ERROR: DTO event status %s\n", 
-                       getpid(),DT_RetToString(ret));
-               return( DAT_ABORT );
-           }
-           stop = get_time();
-           time.rdma_rd[i] = ((stop - start)*1.0e6);
-           time.rdma_rd_total += time.rdma_rd[i];
-
-           LOGPRINTF("%d rdma_read # %d completed\n", getpid(),i+1);
-       }
-
-       /*
-        *  Send RMR information a 3rd time to indicate completion
-        *  NOTE: already swapped to network order in connect_ep
-        */
-       printf("%d Sending RDMA read completion message\n",getpid());
-
-       ret = send_msg( &rmr_send_msg,
-                       sizeof( DAT_RMR_TRIPLET ),
-                       lmr_context_send_msg,
-                       cookie,
-                       DAT_COMPLETION_SUPPRESS_FLAG );
-
-       if(ret != DAT_SUCCESS) {
-               fprintf(stderr, "%d Error send_msg: %s\n",
-                               getpid(),DT_RetToString(ret));
-               return(ret);
-       } else {
-               LOGPRINTF("%d send_msg completed\n", getpid());
-       }
-
-       /*
-        *  Collect first event, write completion or the inbound recv with immed
-        */
-       printf("%d Waiting for inbound message....\n",getpid());
-       if ( polling ) {
-           while (  dat_evd_dequeue( h_dto_rcv_evd, &event ) == DAT_QUEUE_EMPTY );
-       }
-       else {
-           LOGPRINTF("%d waiting for message receive event\n", getpid());
-           if (use_cno) {
-                   DAT_EVD_HANDLE evd = DAT_HANDLE_NULL;
-                   ret = dat_cno_wait( h_dto_cno, DTO_TIMEOUT, &evd );
-                   LOGPRINTF("%d cno wait return evd_handle=%p\n", getpid(),evd);
-                   if ( evd != h_dto_rcv_evd ) {
-                           fprintf(stderr, 
-                                  "%d Error waiting on h_dto_cno: evd != h_dto_rcv_evd\n",
-                                   getpid());
-                           return( ret );
-                   }
-           }
-           /* use wait to dequeue */
-           ret = dat_evd_wait( h_dto_rcv_evd, DTO_TIMEOUT, 1, &event, &nmore );
-           if (ret != DAT_SUCCESS) {
-                   fprintf(stderr, "%d: ERROR: DTO dat_evd_wait() %s\n",
-                                           getpid(),DT_RetToString(ret));
-                   return( ret );
-           }
-       }
-
-       /* validate event number and status */
-       printf("%d inbound rdma_read; send message arrived!\n",getpid());
-       if (event.event_number != DAT_DTO_COMPLETION_EVENT ) {
-           fprintf(stderr, "%d Error unexpected DTO event : %s\n",
-                               getpid(),DT_EventToSTr(event.event_number));
-           return( DAT_ABORT );
-       }
-
-       if ((event.event_data.dto_completion_event_data.transfered_length != sizeof( DAT_RMR_TRIPLET)) ||
-            (event.event_data.dto_completion_event_data.user_cookie.as_64 != recv_msg_index)) {
-
-           fprintf(stderr,"unexpected event data for receive: len=%d cookie="F64x" exp %d/%d\n",
-               (int)event.event_data.dto_completion_event_data.transfered_length,
-               event.event_data.dto_completion_event_data.user_cookie.as_64,
-               (int)sizeof(DAT_RMR_TRIPLET), recv_msg_index );
-
-           return( DAT_ABORT );
-       }
-
-       /* swap received RMR msg: network order to host order */
-       r_iov = rmr_recv_msg[recv_msg_index];
-       rmr_recv_msg[recv_msg_index].virtual_address =
-               ntohll(rmr_recv_msg[recv_msg_index].virtual_address);
-       rmr_recv_msg[recv_msg_index].segment_length =
-               ntohl(rmr_recv_msg[ recv_msg_index].segment_length);
-       rmr_recv_msg[recv_msg_index].rmr_context =
-               ntohl(rmr_recv_msg[recv_msg_index].rmr_context);
-
-       printf("%d Received RMR from remote: "
-              "r_iov: r_key_ctx=%x,va="F64x",len=0x%x\n",
-              getpid(), rmr_recv_msg[recv_msg_index].rmr_context,
-              rmr_recv_msg[recv_msg_index].virtual_address,
-              rmr_recv_msg[recv_msg_index].segment_length);
-
-       LOGPRINTF("%d inbound rdma_write; send msg event SUCCESS!!\n",getpid());
-
-       printf("%d %s RCV RDMA read buffer contains: %s\n",
-                       getpid(),
-                       server ? "SERVER:" : "CLIENT:",
-                       sbuf );
-
-       recv_msg_index++;
-
-       return ( DAT_SUCCESS );
+       }
+       stop = get_time();
+       time.rdma_wr = ((stop - start) * 1.0e6);
+
+       /* validate event number and status */
+       printf("%d inbound rdma_write; send message arrived!\n", getpid());
+       if (event.event_number != DAT_DTO_COMPLETION_EVENT) {
+               fprintf(stderr, "%d Error unexpected DTO event : %s\n",
+                       getpid(), DT_EventToSTr(event.event_number));
+               return (DAT_ABORT);
+       }
+
+       if ((event.event_data.dto_completion_event_data.transfered_length !=
+            sizeof(DAT_RMR_TRIPLET))
+           || (event.event_data.dto_completion_event_data.user_cookie.as_64 !=
+               recv_msg_index)) {
+               fprintf(stderr,
+                       "unexpected event data for receive: len=%d cookie=" F64x
+                       " exp %d/%d\n",
+                       (int)event.event_data.dto_completion_event_data.
+                       transfered_length,
+                       event.event_data.dto_completion_event_data.user_cookie.
+                       as_64, (int)sizeof(DAT_RMR_TRIPLET), recv_msg_index);
+
+               return (DAT_ABORT);
+       }
+
+       /* swap received RMR msg: network order to host order */
+       r_iov = rmr_recv_msg[recv_msg_index];
+       rmr_recv_msg[recv_msg_index].virtual_address =
+           ntohll(rmr_recv_msg[recv_msg_index].virtual_address);
+       rmr_recv_msg[recv_msg_index].segment_length =
+           ntohl(rmr_recv_msg[recv_msg_index].segment_length);
+       rmr_recv_msg[recv_msg_index].rmr_context =
+           ntohl(rmr_recv_msg[recv_msg_index].rmr_context);
+
+       printf("%d Received RMR from remote: "
+              "r_iov: r_key_ctx=%x,va=" F64x ",len=0x%x\n",
+              getpid(), rmr_recv_msg[recv_msg_index].rmr_context,
+              rmr_recv_msg[recv_msg_index].virtual_address,
+              rmr_recv_msg[recv_msg_index].segment_length);
+
+       LOGPRINTF("%d inbound rdma_write; send msg event SUCCESS!!\n",
+                 getpid());
+
+       printf("%d %s RDMA write buffer contains: %s\n",
+              getpid(), server ? "SERVER:" : "CLIENT:", rbuf);
+
+       recv_msg_index++;
+
+       return (DAT_SUCCESS);
 }
 
+DAT_RETURN do_rdma_read_with_msg(void)
+{
+       DAT_EVENT event;
+       DAT_COUNT nmore;
+       DAT_LMR_TRIPLET l_iov;
+       DAT_RMR_TRIPLET r_iov;
+       DAT_DTO_COOKIE cookie;
+       DAT_RETURN ret;
+       int i;
+
+       printf("\n %d RDMA READ DATA with SEND MSG\n\n", getpid());
+
+       if (recv_msg_index >= MSG_BUF_COUNT)
+               return (DAT_ABORT);
+
+       /* get RMR information from previously received message */
+       r_iov = rmr_recv_msg[recv_msg_index - 1];
+
+       /* setup rdma read buffer to initial string to be overwritten */
+       strcpy((char *)sbuf, "blah, blah, blah\n");
+
+       if (server)
+               strcpy((char *)rbuf, "server RDMA read data...");
+       else
+               strcpy((char *)rbuf, "client RDMA read data...");
+
+       l_iov.lmr_context = lmr_context_send;
+       l_iov.virtual_address = (DAT_VADDR) (uintptr_t) sbuf;
+       l_iov.segment_length = buf_len;
+
+       for (i = 0; i < MAX_RDMA_RD; i++) {
+               cookie.as_64 = 0x9999;
+               start = get_time();
+               ret = dat_ep_post_rdma_read(h_ep,       // ep_handle
+                                           1,  // num_segments
+                                           &l_iov,     // LMR
+                                           cookie,     // user_cookie
+                                           &r_iov,     // RMR
+                                           DAT_COMPLETION_DEFAULT_FLAG);
+               if (ret != DAT_SUCCESS) {
+                       fprintf(stderr,
+                               "%d: ERROR: dat_ep_post_rdma_read() %s\n",
+                               getpid(), DT_RetToString(ret));
+                       return (DAT_ABORT);
+               }
 
-DAT_RETURN
-do_ping_pong_msg( )
+               if (polling) {
+                       while (dat_evd_dequeue(h_dto_req_evd, &event) ==
+                              DAT_QUEUE_EMPTY)
+                               rdma_rd_poll_count[i]++;
+               } else {
+                       LOGPRINTF("%d waiting for rdma_read completion event\n",
+                                 getpid());
+                       if (use_cno) {
+                               DAT_EVD_HANDLE evd = DAT_HANDLE_NULL;
+                               ret =
+                                   dat_cno_wait(h_dto_cno, DTO_TIMEOUT, &evd);
+                               LOGPRINTF("%d cno wait return evd_handle=%p\n",
+                                         getpid(), evd);
+                               if (evd != h_dto_req_evd) {
+                                       fprintf(stderr,
+                                               "%d Error waiting on h_dto_cno: evd != h_dto_req_evd\n",
+                                               getpid());
+                                       return (DAT_ABORT);
+                               }
+                       }
+                       /* use wait to dequeue */
+                       ret =
+                           dat_evd_wait(h_dto_req_evd, DTO_TIMEOUT, 1, &event,
+                                        &nmore);
+                       if (ret != DAT_SUCCESS) {
+                               fprintf(stderr,
+                                       "%d: ERROR: DTO dat_evd_wait() %s\n",
+                                       getpid(), DT_RetToString(ret));
+                               return ret;
+                       }
+               }
+               /* validate event number, len, cookie, and status */
+               if (event.event_number != DAT_DTO_COMPLETION_EVENT) {
+                       fprintf(stderr, "%d: ERROR: DTO event number %s\n",
+                               getpid(), DT_EventToSTr(event.event_number));
+                       return (DAT_ABORT);
+               }
+               if ((event.event_data.dto_completion_event_data.
+                    transfered_length != buf_len)
+                   || (event.event_data.dto_completion_event_data.user_cookie.
+                       as_64 != 0x9999)) {
+                       fprintf(stderr,
+                               "%d: ERROR: DTO len %d or cookie " F64x "\n",
+                               getpid(),
+                               event.event_data.dto_completion_event_data.
+                               transfered_length,
+                               event.event_data.dto_completion_event_data.
+                               user_cookie.as_64);
+                       return (DAT_ABORT);
+               }
+               if (event.event_data.dto_completion_event_data.status !=
+                   DAT_SUCCESS) {
+                       fprintf(stderr, "%d: ERROR: DTO event status %s\n",
+                               getpid(), DT_RetToString(ret));
+                       return (DAT_ABORT);
+               }
+               stop = get_time();
+               time.rdma_rd[i] = ((stop - start) * 1.0e6);
+               time.rdma_rd_total += time.rdma_rd[i];
+
+               LOGPRINTF("%d rdma_read # %d completed\n", getpid(), i + 1);
+       }
+
+       /*
+        *  Send RMR information a 3rd time to indicate completion
+        *  NOTE: already swapped to network order in connect_ep
+        */
+       printf("%d Sending RDMA read completion message\n", getpid());
+
+       ret = send_msg(&rmr_send_msg,
+                      sizeof(DAT_RMR_TRIPLET),
+                      lmr_context_send_msg,
+                      cookie, DAT_COMPLETION_SUPPRESS_FLAG);
+
+       if (ret != DAT_SUCCESS) {
+               fprintf(stderr, "%d Error send_msg: %s\n",
+                       getpid(), DT_RetToString(ret));
+               return (ret);
+       } else {
+               LOGPRINTF("%d send_msg completed\n", getpid());
+       }
+
+       /*
+        *  Collect first event, write completion or the inbound recv with immed
+        */
+       printf("%d Waiting for inbound message....\n", getpid());
+       if (polling) {
+               while (dat_evd_dequeue(h_dto_rcv_evd, &event) ==
+                      DAT_QUEUE_EMPTY) ;
+       } else {
+               LOGPRINTF("%d waiting for message receive event\n", getpid());
+               if (use_cno) {
+                       DAT_EVD_HANDLE evd = DAT_HANDLE_NULL;
+                       ret = dat_cno_wait(h_dto_cno, DTO_TIMEOUT, &evd);
+                       LOGPRINTF("%d cno wait return evd_handle=%p\n",
+                                 getpid(), evd);
+                       if (evd != h_dto_rcv_evd) {
+                               fprintf(stderr,
+                                       "%d Error waiting on h_dto_cno: evd != h_dto_rcv_evd\n",
+                                       getpid());
+                               return (ret);
+                       }
+               }
+               /* use wait to dequeue */
+               ret =
+                   dat_evd_wait(h_dto_rcv_evd, DTO_TIMEOUT, 1, &event, &nmore);
+               if (ret != DAT_SUCCESS) {
+                       fprintf(stderr, "%d: ERROR: DTO dat_evd_wait() %s\n",
+                               getpid(), DT_RetToString(ret));
+                       return (ret);
+               }
+       }
+
+       /* validate event number and status */
+       printf("%d inbound rdma_read; send message arrived!\n", getpid());
+       if (event.event_number != DAT_DTO_COMPLETION_EVENT) {
+               fprintf(stderr, "%d Error unexpected DTO event : %s\n",
+                       getpid(), DT_EventToSTr(event.event_number));
+               return (DAT_ABORT);
+       }
+
+       if ((event.event_data.dto_completion_event_data.transfered_length !=
+            sizeof(DAT_RMR_TRIPLET))
+           || (event.event_data.dto_completion_event_data.user_cookie.as_64 !=
+               recv_msg_index)) {
+
+               fprintf(stderr,
+                       "unexpected event data for receive: len=%d cookie=" F64x
+                       " exp %d/%d\n",
+                       (int)event.event_data.dto_completion_event_data.
+                       transfered_length,
+                       event.event_data.dto_completion_event_data.user_cookie.
+                       as_64, (int)sizeof(DAT_RMR_TRIPLET), recv_msg_index);
+
+               return (DAT_ABORT);
+       }
+
+       /* swap received RMR msg: network order to host order */
+       r_iov = rmr_recv_msg[recv_msg_index];
+       rmr_recv_msg[recv_msg_index].virtual_address =
+           ntohll(rmr_recv_msg[recv_msg_index].virtual_address);
+       rmr_recv_msg[recv_msg_index].segment_length =
+           ntohl(rmr_recv_msg[recv_msg_index].segment_length);
+       rmr_recv_msg[recv_msg_index].rmr_context =
+           ntohl(rmr_recv_msg[recv_msg_index].rmr_context);
+
+       printf("%d Received RMR from remote: "
+              "r_iov: r_key_ctx=%x,va=" F64x ",len=0x%x\n",
+              getpid(), rmr_recv_msg[recv_msg_index].rmr_context,
+              rmr_recv_msg[recv_msg_index].virtual_address,
+              rmr_recv_msg[recv_msg_index].segment_length);
+
+       LOGPRINTF("%d inbound rdma_write; send msg event SUCCESS!!\n",
+                 getpid());
+
+       printf("%d %s RCV RDMA read buffer contains: %s\n",
+              getpid(), server ? "SERVER:" : "CLIENT:", sbuf);
+
+       recv_msg_index++;
+
+       return (DAT_SUCCESS);
+}
+
+DAT_RETURN do_ping_pong_msg()
 {
-    DAT_EVENT          event;
-    DAT_COUNT          nmore;
-    DAT_DTO_COOKIE     cookie;
-    DAT_LMR_TRIPLET    l_iov;
-    DAT_RETURN         ret;
-    int                i;
-    char               *snd_buf;
-    char               *rcv_buf;
-
-    printf("\n %d PING DATA with SEND MSG\n\n",getpid());
-
-    snd_buf = sbuf;
-    rcv_buf = rbuf;
-
-    /* pre-post all buffers */
-    for ( i=0; i < burst; i++ ) {
-       burst_msg_posted++;
-       cookie.as_64          = i;
-       l_iov.lmr_context     = lmr_context_recv;
-       l_iov.virtual_address = (DAT_VADDR)(uintptr_t)rcv_buf;
-       l_iov.segment_length  = buf_len;
-
-       LOGPRINTF("%d Pre-posting Receive Message Buffers %p\n",
-                   getpid(), rcv_buf );
-
-       ret = dat_ep_post_recv( h_ep,
-                               1,
-                               &l_iov,
-                               cookie,
-                               DAT_COMPLETION_DEFAULT_FLAG );
-
-        if(ret != DAT_SUCCESS) {
-           fprintf(stderr, "%d Error posting recv msg buffer: %s\n",
-                           getpid(),DT_RetToString(ret));
-           return(ret);
-        }
-       else {
-           LOGPRINTF("%d Posted Receive Message Buffer %p\n",getpid(),rcv_buf);
-        }
-
-       /* next buffer */
-       rcv_buf += buf_len;
-    }
+       DAT_EVENT event;
+       DAT_COUNT nmore;
+       DAT_DTO_COOKIE cookie;
+       DAT_LMR_TRIPLET l_iov;
+       DAT_RETURN ret;
+       int i;
+       char *snd_buf;
+       char *rcv_buf;
+
+       printf("\n %d PING DATA with SEND MSG\n\n", getpid());
+
+       snd_buf = sbuf;
+       rcv_buf = rbuf;
+
+       /* pre-post all buffers */
+       for (i = 0; i < burst; i++) {
+               burst_msg_posted++;
+               cookie.as_64 = i;
+               l_iov.lmr_context = lmr_context_recv;
+               l_iov.virtual_address = (DAT_VADDR) (uintptr_t) rcv_buf;
+               l_iov.segment_length = buf_len;
+
+               LOGPRINTF("%d Pre-posting Receive Message Buffers %p\n",
+                         getpid(), rcv_buf);
+
+               ret = dat_ep_post_recv(h_ep,
+                                      1,
+                                      &l_iov,
+                                      cookie, DAT_COMPLETION_DEFAULT_FLAG);
+
+               if (ret != DAT_SUCCESS) {
+                       fprintf(stderr,
+                               "%d Error posting recv msg buffer: %s\n",
+                               getpid(), DT_RetToString(ret));
+                       return (ret);
+               } else {
+                       LOGPRINTF("%d Posted Receive Message Buffer %p\n",
+                                 getpid(), rcv_buf);
+               }
+
+               /* next buffer */
+               rcv_buf += buf_len;
+       }
 #if defined(_WIN32) || defined(_WIN64)
-    Sleep(1000);
+       Sleep(1000);
 #else
-    sleep(1);
+       sleep(1);
 #endif
 
-    /* Initialize recv_buf and index to beginning */
-    rcv_buf = rbuf;
-    burst_msg_index=0;
-
-    /* client ping 0x55, server pong 0xAA in first byte */
-    start = get_time();
-    for ( i=0;i<burst;i++ ) {
-       /* walk the send and recv buffers */
-       if ( !server ) {
-           *snd_buf = 0x55;
-
-           LOGPRINTF("%d %s SND buffer %p contains: 0x%x len=%d\n",
-                   getpid(), server ? "SERVER:" : "CLIENT:",
-                   snd_buf, *snd_buf, buf_len );
-
-           ret = send_msg( snd_buf,
-                           buf_len,
-                           lmr_context_send,
-                           cookie,
-                           DAT_COMPLETION_SUPPRESS_FLAG );
-
-           if(ret != DAT_SUCCESS) {
-               fprintf(stderr, "%d Error send_msg: %s\n",
-                                   getpid(),DT_RetToString(ret));
-               return(ret);
-           }
-           else {
-               LOGPRINTF("%d send_msg completed\n", getpid());
-           }
-       }
-
-       /* Wait for recv message */
-       if ( polling ) {
-           poll_count=0;
-           LOGPRINTF("%d Polling for message receive event\n", getpid());
-           while (  dat_evd_dequeue( h_dto_rcv_evd, &event ) == DAT_QUEUE_EMPTY )
-               poll_count++;
-       }
-       else {
-           LOGPRINTF("%d waiting for message receive event\n", getpid());
-           if (use_cno) {
-               DAT_EVD_HANDLE evd = DAT_HANDLE_NULL;
-               ret = dat_cno_wait( h_dto_cno, DTO_TIMEOUT, &evd );
-               LOGPRINTF("%d cno wait return evd_handle=%p\n", getpid(),evd);
-               if ( evd != h_dto_rcv_evd )
-               {
-                   fprintf(stderr, 
-                          "%d Error waiting on h_dto_cno: evd != h_dto_rcv_evd\n",
-                           getpid());
-                   return( ret );
-               }
-           }
-           /* use wait to dequeue */
-           ret = dat_evd_wait( h_dto_rcv_evd, DTO_TIMEOUT, 1, &event, &nmore );
-           if (ret != DAT_SUCCESS) {
-               fprintf(stderr, "%d: ERROR: DTO dat_evd_wait() %s\n",
-                                       getpid(),DT_RetToString(ret));
-               return( ret );
-           }
-       }
-       /* start timer after first message arrives on server */
-       if ( i == 0) {
-           start = get_time();
-       }
-       /* validate event number and status */
-       LOGPRINTF("%d inbound message; message arrived!\n",getpid());
-       if ( event.event_number != DAT_DTO_COMPLETION_EVENT ) {
-           fprintf(stderr, "%d Error unexpected DTO event : %s\n",
-                   getpid(),DT_EventToSTr(event.event_number));
-           return( DAT_ABORT );
-       }
-       if ((event.event_data.dto_completion_event_data.transfered_length
-           != buf_len) ||
-           (event.event_data.dto_completion_event_data.user_cookie.as_64
-           != burst_msg_index) )  {
-           fprintf(stderr,"ERR: recv event: len=%d cookie="F64x" exp %d/%d\n",
-               (int)event.event_data.dto_completion_event_data.transfered_length,
-               event.event_data.dto_completion_event_data.user_cookie.as_64,
-               (int)buf_len, (int)burst_msg_index );
-
-           return( DAT_ABORT );
-       }
-
-       LOGPRINTF("%d %s RCV buffer %p contains: 0x%x len=%d\n",
-                   getpid(), server ? "SERVER:" : "CLIENT:",
-                   rcv_buf, *rcv_buf, buf_len );
-
-       burst_msg_index++;
-
-       /* If server, change data and send it back to client */
-       if ( server ) {
-           *snd_buf = 0xaa;
-
-           LOGPRINTF("%d %s SND buffer %p contains: 0x%x len=%d\n",
-                   getpid(), server ? "SERVER:" : "CLIENT:",
-                   snd_buf, *snd_buf, buf_len );
-
-           ret = send_msg( snd_buf,
-                           buf_len,
-                           lmr_context_send,
-                           cookie,
-                           DAT_COMPLETION_SUPPRESS_FLAG );
-
-           if(ret != DAT_SUCCESS) {
-               fprintf(stderr, "%d Error send_msg: %s\n",
-                                   getpid(),DT_RetToString(ret));
-               return(ret);
-           }
-           else {
-               LOGPRINTF("%d send_msg completed\n", getpid());
-           }
-       }
-
-       /* next buffers */
-       rcv_buf += buf_len;
-       snd_buf += buf_len;
-    }
-    stop = get_time();
-    time.rtt = ((stop - start)*1.0e6);
-
-    return ( DAT_SUCCESS );
+       /* Initialize recv_buf and index to beginning */
+       rcv_buf = rbuf;
+       burst_msg_index = 0;
+
+       /* client ping 0x55, server pong 0xAA in first byte */
+       start = get_time();
+       for (i = 0; i < burst; i++) {
+               /* walk the send and recv buffers */
+               if (!server) {
+                       *snd_buf = 0x55;
+
+                       LOGPRINTF("%d %s SND buffer %p contains: 0x%x len=%d\n",
+                                 getpid(), server ? "SERVER:" : "CLIENT:",
+                                 snd_buf, *snd_buf, buf_len);
+
+                       ret = send_msg(snd_buf,
+                                      buf_len,
+                                      lmr_context_send,
+                                      cookie, DAT_COMPLETION_SUPPRESS_FLAG);
+
+                       if (ret != DAT_SUCCESS) {
+                               fprintf(stderr, "%d Error send_msg: %s\n",
+                                       getpid(), DT_RetToString(ret));
+                               return (ret);
+                       } else {
+                               LOGPRINTF("%d send_msg completed\n", getpid());
+                       }
+               }
+
+               /* Wait for recv message */
+               if (polling) {
+                       poll_count = 0;
+                       LOGPRINTF("%d Polling for message receive event\n",
+                                 getpid());
+                       while (dat_evd_dequeue(h_dto_rcv_evd, &event) ==
+                              DAT_QUEUE_EMPTY)
+                               poll_count++;
+               } else {
+                       LOGPRINTF("%d waiting for message receive event\n",
+                                 getpid());
+                       if (use_cno) {
+                               DAT_EVD_HANDLE evd = DAT_HANDLE_NULL;
+                               ret =
+                                   dat_cno_wait(h_dto_cno, DTO_TIMEOUT, &evd);
+                               LOGPRINTF("%d cno wait return evd_handle=%p\n",
+                                         getpid(), evd);
+                               if (evd != h_dto_rcv_evd) {
+                                       fprintf(stderr,
+                                               "%d Error waiting on h_dto_cno: evd != h_dto_rcv_evd\n",
+                                               getpid());
+                                       return (ret);
+                               }
+                       }
+                       /* use wait to dequeue */
+                       ret =
+                           dat_evd_wait(h_dto_rcv_evd, DTO_TIMEOUT, 1, &event,
+                                        &nmore);
+                       if (ret != DAT_SUCCESS) {
+                               fprintf(stderr,
+                                       "%d: ERROR: DTO dat_evd_wait() %s\n",
+                                       getpid(), DT_RetToString(ret));
+                               return (ret);
+                       }
+               }
+               /* start timer after first message arrives on server */
+               if (i == 0) {
+                       start = get_time();
+               }
+               /* validate event number and status */
+               LOGPRINTF("%d inbound message; message arrived!\n", getpid());
+               if (event.event_number != DAT_DTO_COMPLETION_EVENT) {
+                       fprintf(stderr, "%d Error unexpected DTO event : %s\n",
+                               getpid(), DT_EventToSTr(event.event_number));
+                       return (DAT_ABORT);
+               }
+               if ((event.event_data.dto_completion_event_data.
+                    transfered_length != buf_len)
+                   || (event.event_data.dto_completion_event_data.user_cookie.
+                       as_64 != burst_msg_index)) {
+                       fprintf(stderr,
+                               "ERR: recv event: len=%d cookie=" F64x
+                               " exp %d/%d\n",
+                               (int)event.event_data.dto_completion_event_data.
+                               transfered_length,
+                               event.event_data.dto_completion_event_data.
+                               user_cookie.as_64, (int)buf_len,
+                               (int)burst_msg_index);
+
+                       return (DAT_ABORT);
+               }
+
+               LOGPRINTF("%d %s RCV buffer %p contains: 0x%x len=%d\n",
+                         getpid(), server ? "SERVER:" : "CLIENT:",
+                         rcv_buf, *rcv_buf, buf_len);
+
+               burst_msg_index++;
+
+               /* If server, change data and send it back to client */
+               if (server) {
+                       *snd_buf = 0xaa;
+
+                       LOGPRINTF("%d %s SND buffer %p contains: 0x%x len=%d\n",
+                                 getpid(), server ? "SERVER:" : "CLIENT:",
+                                 snd_buf, *snd_buf, buf_len);
+
+                       ret = send_msg(snd_buf,
+                                      buf_len,
+                                      lmr_context_send,
+                                      cookie, DAT_COMPLETION_SUPPRESS_FLAG);
+
+                       if (ret != DAT_SUCCESS) {
+                               fprintf(stderr, "%d Error send_msg: %s\n",
+                                       getpid(), DT_RetToString(ret));
+                               return (ret);
+                       } else {
+                               LOGPRINTF("%d send_msg completed\n", getpid());
+                       }
+               }
+
+               /* next buffers */
+               rcv_buf += buf_len;
+               snd_buf += buf_len;
+       }
+       stop = get_time();
+       time.rtt = ((stop - start) * 1.0e6);
+
+       return (DAT_SUCCESS);
 }
 
 /* Register RDMA Receive buffer */
-DAT_RETURN
-register_rdma_memory(void)
+DAT_RETURN register_rdma_memory(void)
 {
-    DAT_RETURN             ret;
-    DAT_REGION_DESCRIPTION region;
-
-    region.for_va = rbuf;
-    start = get_time();
-    ret = dat_lmr_create(  h_ia,
-                           DAT_MEM_TYPE_VIRTUAL,
-                           region,
-                           buf_len*burst,
-                           h_pz,
-                           DAT_MEM_PRIV_ALL_FLAG,
-                          DAT_VA_TYPE_VA,
-                           &h_lmr_recv,
-                           &lmr_context_recv,
-                           &rmr_context_recv,
-                           &registered_size_recv,
-                           &registered_addr_recv );
-    stop = get_time();
-    time.reg += ((stop - start)*1.0e6);
-    time.total += time.reg;
-
-    if(ret != DAT_SUCCESS) {
-       fprintf(stderr, "%d Error registering Receive RDMA buffer: %s\n",
-               getpid(),DT_RetToString(ret));
-       return (ret);
-    } else {
-           LOGPRINTF("%d Registered Receive RDMA Buffer %p\n",
-                       getpid(),region.for_va);
-    }
-
-    /* Register RDMA Send buffer */
-    region.for_va = sbuf;
-    ret = dat_lmr_create(   h_ia,
-                           DAT_MEM_TYPE_VIRTUAL,
-                           region,
-                           buf_len*burst,
-                           h_pz,
-                           DAT_MEM_PRIV_ALL_FLAG,
-                          DAT_VA_TYPE_VA,
-                           &h_lmr_send,
-                           &lmr_context_send,
-                           &rmr_context_send,
-                           &registered_size_send,
-                           &registered_addr_send );
-    if(ret != DAT_SUCCESS) {
-           fprintf(stderr, "%d Error registering send RDMA buffer: %s\n",
-                           getpid(),DT_RetToString(ret));
-           return (ret);
-    } else {
-           LOGPRINTF("%d Registered Send RDMA Buffer %p\n",
-                   getpid(),region.for_va);
-    }
-
-    return DAT_SUCCESS;
+       DAT_RETURN ret;
+       DAT_REGION_DESCRIPTION region;
+
+       region.for_va = rbuf;
+       start = get_time();
+       ret = dat_lmr_create(h_ia,
+                            DAT_MEM_TYPE_VIRTUAL,
+                            region,
+                            buf_len * (burst+1),
+                            h_pz,
+                            DAT_MEM_PRIV_ALL_FLAG,
+                            DAT_VA_TYPE_VA,
+                            &h_lmr_recv,
+                            &lmr_context_recv,
+                            &rmr_context_recv,
+                            &registered_size_recv, &registered_addr_recv);
+       stop = get_time();
+       time.reg += ((stop - start) * 1.0e6);
+       time.total += time.reg;
+
+       if (ret != DAT_SUCCESS) {
+               fprintf(stderr,
+                       "%d Error registering Receive RDMA buffer: %s\n",
+                       getpid(), DT_RetToString(ret));
+               return (ret);
+       } else {
+               LOGPRINTF("%d Registered Receive RDMA Buffer %p\n",
+                         getpid(), region.for_va);
+       }
+
+       /* Register RDMA Send buffer */
+       region.for_va = sbuf;
+       ret = dat_lmr_create(h_ia,
+                            DAT_MEM_TYPE_VIRTUAL,
+                            region,
+                            buf_len * (burst + 1),
+                            h_pz,
+                            DAT_MEM_PRIV_ALL_FLAG,
+                            DAT_VA_TYPE_VA,
+                            &h_lmr_send,
+                            &lmr_context_send,
+                            &rmr_context_send,
+                            &registered_size_send, &registered_addr_send);
+       if (ret != DAT_SUCCESS) {
+               fprintf(stderr, "%d Error registering send RDMA buffer: %s\n",
+                       getpid(), DT_RetToString(ret));
+               return (ret);
+       } else {
+               LOGPRINTF("%d Registered Send RDMA Buffer %p\n",
+                         getpid(), region.for_va);
+       }
+
+       return DAT_SUCCESS;
 }
 
 /*
  * Unregister RDMA memory
  */
-DAT_RETURN
-unregister_rdma_memory(void)
+DAT_RETURN unregister_rdma_memory(void)
 {
-    DAT_RETURN ret;
-
-    /* Unregister Recv Buffer */
-    if ( h_lmr_recv != DAT_HANDLE_NULL )
-    {
-       LOGPRINTF("%d Unregister h_lmr %p \n",getpid(),h_lmr_recv);
-        start = get_time();
-       ret = dat_lmr_free(h_lmr_recv);
-        stop = get_time();
-       time.unreg += ((stop - start)*1.0e6);
-       time.total += time.unreg;
-       if(ret != DAT_SUCCESS) {
-           fprintf(stderr, "%d Error deregistering recv mr: %s\n",
-                       getpid(), DT_RetToString(ret));
-           return (ret);
-       }
-       else {
-           LOGPRINTF("%d Unregistered Recv Buffer\n",getpid());
-           h_lmr_recv = NULL;
-       }
-    }
-
-    /* Unregister Send Buffer */
-    if ( h_lmr_send != DAT_HANDLE_NULL ) {
-       LOGPRINTF("%d Unregister h_lmr %p \n",getpid(),h_lmr_send);
-       ret = dat_lmr_free(h_lmr_send);
-        if(ret != DAT_SUCCESS) {
-           fprintf(stderr, "%d Error deregistering send mr: %s\n",
-                           getpid(), DT_RetToString(ret));
-           return (ret);
-       }
-       else {
-           LOGPRINTF("%d Unregistered send Buffer\n",getpid());
-           h_lmr_send = NULL;
-       }
-    }
-    return DAT_SUCCESS;
+       DAT_RETURN ret;
+
+       /* Unregister Recv Buffer */
+       if (h_lmr_recv != DAT_HANDLE_NULL) {
+               LOGPRINTF("%d Unregister h_lmr %p \n", getpid(), h_lmr_recv);
+               start = get_time();
+               ret = dat_lmr_free(h_lmr_recv);
+               stop = get_time();
+               time.unreg += ((stop - start) * 1.0e6);
+               time.total += time.unreg;
+               if (ret != DAT_SUCCESS) {
+                       fprintf(stderr, "%d Error deregistering recv mr: %s\n",
+                               getpid(), DT_RetToString(ret));
+                       return (ret);
+               } else {
+                       LOGPRINTF("%d Unregistered Recv Buffer\n", getpid());
+                       h_lmr_recv = NULL;
+               }
+       }
+
+       /* Unregister Send Buffer */
+       if (h_lmr_send != DAT_HANDLE_NULL) {
+               LOGPRINTF("%d Unregister h_lmr %p \n", getpid(), h_lmr_send);
+               ret = dat_lmr_free(h_lmr_send);
+               if (ret != DAT_SUCCESS) {
+                       fprintf(stderr, "%d Error deregistering send mr: %s\n",
+                               getpid(), DT_RetToString(ret));
+                       return (ret);
+               } else {
+                       LOGPRINTF("%d Unregistered send Buffer\n", getpid());
+                       h_lmr_send = NULL;
+               }
+       }
+       return DAT_SUCCESS;
 }
 
  /*
   * Create CNO, CR, CONN, and DTO events
   */
-DAT_RETURN
-create_events(void)
+DAT_RETURN create_events(void)
 {
-    DAT_RETURN ret;
-    DAT_EVD_PARAM param;
+       DAT_RETURN ret;
+       DAT_EVD_PARAM param;
 
-    /* create CNO */
-    if (use_cno) {
-        start = get_time();
+       /* create CNO */
+       if (use_cno) {
+               start = get_time();
 #if defined(_WIN32) || defined(_WIN64)
-        { DAT_OS_WAIT_PROXY_AGENT pa = {NULL, NULL};
-          ret = dat_cno_create(h_ia, pa, &h_dto_cno);
-        }
+               {
+                       DAT_OS_WAIT_PROXY_AGENT pa = { NULL, NULL };
+                       ret = dat_cno_create(h_ia, pa, &h_dto_cno);
+               }
 #else
-        ret = dat_cno_create(h_ia, DAT_OS_WAIT_PROXY_AGENT_NULL, &h_dto_cno);
+               ret =
+                   dat_cno_create(h_ia, DAT_OS_WAIT_PROXY_AGENT_NULL,
+                                  &h_dto_cno);
 #endif
-        stop = get_time();
-       time.cnoc += ((stop - start)*1.0e6);
-       time.total += time.cnoc;
-       if(ret != DAT_SUCCESS) {
-           fprintf(stderr, "%d Error dat_cno_create: %s\n",
-                               getpid(),DT_RetToString(ret));
-           return (ret);
-       }
-       else {
-           LOGPRINTF("%d cr_evd created, %p\n", getpid(), h_dto_cno);
-       }
-    }
-
-    /* create cr EVD */
-    start = get_time();
-    ret = dat_evd_create(h_ia, 10, DAT_HANDLE_NULL, DAT_EVD_CR_FLAG, &h_cr_evd);
-    stop = get_time();
-    time.evdc += ((stop - start)*1.0e6);
-    time.total += time.evdc;
-    if(ret != DAT_SUCCESS) {
-        fprintf(stderr, "%d Error dat_evd_create: %s\n",
-                           getpid(),DT_RetToString(ret));
-        return (ret);
-    }
-    else {
-        LOGPRINTF("%d cr_evd created %p\n", getpid(),h_cr_evd);
-    }
-
-    /* create conn EVD */
-    ret = dat_evd_create( h_ia,
-                          10,
-                          DAT_HANDLE_NULL,
-                          DAT_EVD_CONNECTION_FLAG,
-                          &h_conn_evd );
-    if(ret != DAT_SUCCESS) {
-        fprintf(stderr, "%d Error dat_evd_create: %s\n",
-                           getpid(),DT_RetToString(ret));
-        return (ret);
-    }
-    else {
-        LOGPRINTF("%d con_evd created %p\n", getpid(),h_conn_evd);
-    }
-
-    /* create dto SND EVD, with CNO if use_cno was set */
-    ret = dat_evd_create( h_ia,
-                         (MSG_BUF_COUNT+MAX_RDMA_RD+burst)*2,
-                         h_dto_cno,
-                         DAT_EVD_DTO_FLAG,
-                         &h_dto_req_evd  );
-    if(ret != DAT_SUCCESS) {
-        fprintf(stderr, "%d Error dat_evd_create REQ: %s\n",
-                           getpid(),DT_RetToString(ret));
-        return (ret);
-    }
-    else {
-        LOGPRINTF("%d dto_req_evd created %p\n", getpid(), h_dto_req_evd );
-    }
-
-    /* create dto RCV EVD, with CNO if use_cno was set */
-    ret = dat_evd_create( h_ia,
-                         MSG_BUF_COUNT,
-                         h_dto_cno,
-                         DAT_EVD_DTO_FLAG,
-                         &h_dto_rcv_evd  );
-    if(ret != DAT_SUCCESS) {
-        fprintf(stderr, "%d Error dat_evd_create RCV: %s\n",
-                           getpid(),DT_RetToString(ret));
-        return (ret);
-    }
-    else {
-        LOGPRINTF("%d dto_rcv_evd created %p\n", getpid(), h_dto_rcv_evd );
-    }
-
-    /* query DTO req EVD and check size */
-    ret = dat_evd_query( h_dto_req_evd,
-                          DAT_EVD_FIELD_EVD_QLEN,
-                          &param  );
-    if(ret != DAT_SUCCESS) {
-         fprintf(stderr, "%d Error dat_evd_query request evd: %s\n",
-                            getpid(),DT_RetToString(ret));
-         return (ret);
-    }
-    else if (param.evd_qlen < (MSG_BUF_COUNT+MAX_RDMA_RD+burst)*2) {
-         fprintf(stderr, "%d Error dat_evd qsize too small: %d < %d\n",
-                 getpid(), param.evd_qlen, 
-                 (MSG_BUF_COUNT+MAX_RDMA_RD+burst)*2);
-         return (ret);
-    }
-  
-    LOGPRINTF("%d dto_req_evd QLEN - requested %d and actual %d\n", 
-              getpid(), (MSG_BUF_COUNT+MAX_RDMA_RD+burst)*2,
-              param.evd_qlen);
-
-    return DAT_SUCCESS;
+               stop = get_time();
+               time.cnoc += ((stop - start) * 1.0e6);
+               time.total += time.cnoc;
+               if (ret != DAT_SUCCESS) {
+                       fprintf(stderr, "%d Error dat_cno_create: %s\n",
+                               getpid(), DT_RetToString(ret));
+                       return (ret);
+               } else {
+                       LOGPRINTF("%d cr_evd created, %p\n", getpid(),
+                                 h_dto_cno);
+               }
+       }
+
+       /* create cr EVD */
+       start = get_time();
+       ret =
+           dat_evd_create(h_ia, 10, DAT_HANDLE_NULL, DAT_EVD_CR_FLAG,
+                          &h_cr_evd);
+       stop = get_time();
+       time.evdc += ((stop - start) * 1.0e6);
+       time.total += time.evdc;
+       if (ret != DAT_SUCCESS) {
+               fprintf(stderr, "%d Error dat_evd_create: %s\n",
+                       getpid(), DT_RetToString(ret));
+               return (ret);
+       } else {
+               LOGPRINTF("%d cr_evd created %p\n", getpid(), h_cr_evd);
+       }
+
+       /* create conn EVD */
+       ret = dat_evd_create(h_ia,
+                            10,
+                            DAT_HANDLE_NULL,
+                            DAT_EVD_CONNECTION_FLAG, &h_conn_evd);
+       if (ret != DAT_SUCCESS) {
+               fprintf(stderr, "%d Error dat_evd_create: %s\n",
+                       getpid(), DT_RetToString(ret));
+               return (ret);
+       } else {
+               LOGPRINTF("%d con_evd created %p\n", getpid(), h_conn_evd);
+       }
+
+       /* create dto SND EVD, with CNO if use_cno was set */
+       ret = dat_evd_create(h_ia,
+                            (MSG_BUF_COUNT + MAX_RDMA_RD + burst) * 2,
+                            h_dto_cno, DAT_EVD_DTO_FLAG, &h_dto_req_evd);
+       if (ret != DAT_SUCCESS) {
+               fprintf(stderr, "%d Error dat_evd_create REQ: %s\n",
+                       getpid(), DT_RetToString(ret));
+               return (ret);
+       } else {
+               LOGPRINTF("%d dto_req_evd created %p\n", getpid(),
+                         h_dto_req_evd);
+       }
+
+       /* create dto RCV EVD, with CNO if use_cno was set */
+       ret = dat_evd_create(h_ia,
+                            MSG_BUF_COUNT + burst,
+                            h_dto_cno, DAT_EVD_DTO_FLAG, &h_dto_rcv_evd);
+       if (ret != DAT_SUCCESS) {
+               fprintf(stderr, "%d Error dat_evd_create RCV: %s\n",
+                       getpid(), DT_RetToString(ret));
+               return (ret);
+       } else {
+               LOGPRINTF("%d dto_rcv_evd created %p\n", getpid(),
+                         h_dto_rcv_evd);
+       }
+
+       /* query DTO req EVD and check size */
+       ret = dat_evd_query(h_dto_req_evd, DAT_EVD_FIELD_EVD_QLEN, &param);
+       if (ret != DAT_SUCCESS) {
+               fprintf(stderr, "%d Error dat_evd_query request evd: %s\n",
+                       getpid(), DT_RetToString(ret));
+               return (ret);
+       } else if (param.evd_qlen < (MSG_BUF_COUNT + MAX_RDMA_RD + burst) * 2) {
+               fprintf(stderr, "%d Error dat_evd qsize too small: %d < %d\n",
+                       getpid(), param.evd_qlen,
+                       (MSG_BUF_COUNT + MAX_RDMA_RD + burst) * 2);
+               return (ret);
+       }
+
+       LOGPRINTF("%d dto_req_evd QLEN - requested %d and actual %d\n",
+                 getpid(), (MSG_BUF_COUNT + MAX_RDMA_RD + burst) * 2,
+                 param.evd_qlen);
+
+       return DAT_SUCCESS;
 }
 
 /*
  * Destroy CR, CONN, CNO, and DTO events
  */
 
-DAT_RETURN
-destroy_events(void)
+DAT_RETURN destroy_events(void)
 {
-    DAT_RETURN      ret;
-
-       /* free cr EVD */
-    if ( h_cr_evd != DAT_HANDLE_NULL ) {
-        LOGPRINTF("%d Free cr EVD %p \n",getpid(),h_cr_evd);
-        ret = dat_evd_free( h_cr_evd );
-        if(ret != DAT_SUCCESS) {
-                   fprintf(stderr, "%d Error freeing cr EVD: %s\n",
-                           getpid(), DT_RetToString(ret));
-                       return (ret);
-           } else {
-                   LOGPRINTF("%d Freed cr EVD\n",getpid());
-            h_cr_evd = DAT_HANDLE_NULL;
-           }
-    }
-
-    /* free conn EVD */
-    if ( h_conn_evd != DAT_HANDLE_NULL ) {
-        LOGPRINTF("%d Free conn EVD %p \n",getpid(),h_conn_evd);
-        ret = dat_evd_free( h_conn_evd );
-        if(ret != DAT_SUCCESS) {
-           fprintf(stderr, "%d Error freeing conn EVD: %s\n",
-                       getpid(), DT_RetToString(ret));
-           return (ret);
-       }
-       else {
-           LOGPRINTF("%d Freed conn EVD\n",getpid());
-           h_conn_evd = DAT_HANDLE_NULL;
-       }
-    }
-
-    /* free RCV dto EVD */
-    if ( h_dto_rcv_evd != DAT_HANDLE_NULL ) {
-        LOGPRINTF("%d Free RCV dto EVD %p \n",getpid(),h_dto_rcv_evd);
-        start = get_time();
-        ret = dat_evd_free( h_dto_rcv_evd );
-        stop = get_time();
-        time.evdf += ((stop - start)*1.0e6);
-        time.total += time.evdf;
-        if(ret != DAT_SUCCESS) {
-           fprintf(stderr, "%d Error freeing dto EVD: %s\n",
-                           getpid(), DT_RetToString(ret));
-           return (ret);
-        }
-        else {
-           LOGPRINTF("%d Freed dto EVD\n",getpid());
-            h_dto_rcv_evd = DAT_HANDLE_NULL;
-        }
-     }
-
-    /* free REQ dto EVD */
-    if ( h_dto_req_evd != DAT_HANDLE_NULL ) {
-        LOGPRINTF("%d Free REQ dto EVD %p \n",getpid(),h_dto_req_evd);
-        ret = dat_evd_free( h_dto_req_evd );
-        if(ret != DAT_SUCCESS) {
-           fprintf(stderr, "%d Error freeing dto EVD: %s\n",
-                           getpid(), DT_RetToString(ret));
-           return (ret);
-       }
-       else {
-           LOGPRINTF("%d Freed dto EVD\n",getpid());
-            h_dto_req_evd = DAT_HANDLE_NULL;
-       }
-    }
-
-    /* free CNO */
-    if ( h_dto_cno != DAT_HANDLE_NULL ) {
-        LOGPRINTF("%d Free dto CNO %p \n",getpid(),h_dto_cno);
-        start = get_time();
-        ret = dat_cno_free( h_dto_cno );
-        stop = get_time();
-        time.cnof += ((stop - start)*1.0e6);
-        time.total += time.cnof;
-        if(ret != DAT_SUCCESS) {
-           fprintf(stderr, "%d Error freeing dto CNO: %s\n",
-                   getpid(), DT_RetToString(ret));
-           return (ret);
-       }
-       else {
-           LOGPRINTF("%d Freed dto CNO\n",getpid());
-            h_dto_cno = DAT_HANDLE_NULL;
-       }
-    }
-    return DAT_SUCCESS;
+       DAT_RETURN ret;
+
+       /* free cr EVD */
+       if (h_cr_evd != DAT_HANDLE_NULL) {
+               LOGPRINTF("%d Free cr EVD %p \n", getpid(), h_cr_evd);
+               ret = dat_evd_free(h_cr_evd);
+               if (ret != DAT_SUCCESS) {
+                       fprintf(stderr, "%d Error freeing cr EVD: %s\n",
+                               getpid(), DT_RetToString(ret));
+                       return (ret);
+               } else {
+                       LOGPRINTF("%d Freed cr EVD\n", getpid());
+                       h_cr_evd = DAT_HANDLE_NULL;
+               }
+       }
+
+       /* free conn EVD */
+       if (h_conn_evd != DAT_HANDLE_NULL) {
+               LOGPRINTF("%d Free conn EVD %p \n", getpid(), h_conn_evd);
+               ret = dat_evd_free(h_conn_evd);
+               if (ret != DAT_SUCCESS) {
+                       fprintf(stderr, "%d Error freeing conn EVD: %s\n",
+                               getpid(), DT_RetToString(ret));
+                       return (ret);
+               } else {
+                       LOGPRINTF("%d Freed conn EVD\n", getpid());
+                       h_conn_evd = DAT_HANDLE_NULL;
+               }
+       }
+
+       /* free RCV dto EVD */
+       if (h_dto_rcv_evd != DAT_HANDLE_NULL) {
+               LOGPRINTF("%d Free RCV dto EVD %p \n", getpid(), h_dto_rcv_evd);
+               start = get_time();
+               ret = dat_evd_free(h_dto_rcv_evd);
+               stop = get_time();
+               time.evdf += ((stop - start) * 1.0e6);
+               time.total += time.evdf;
+               if (ret != DAT_SUCCESS) {
+                       fprintf(stderr, "%d Error freeing dto EVD: %s\n",
+                               getpid(), DT_RetToString(ret));
+                       return (ret);
+               } else {
+                       LOGPRINTF("%d Freed dto EVD\n", getpid());
+                       h_dto_rcv_evd = DAT_HANDLE_NULL;
+               }
+       }
+
+       /* free REQ dto EVD */
+       if (h_dto_req_evd != DAT_HANDLE_NULL) {
+               LOGPRINTF("%d Free REQ dto EVD %p \n", getpid(), h_dto_req_evd);
+               ret = dat_evd_free(h_dto_req_evd);
+               if (ret != DAT_SUCCESS) {
+                       fprintf(stderr, "%d Error freeing dto EVD: %s\n",
+                               getpid(), DT_RetToString(ret));
+                       return (ret);
+               } else {
+                       LOGPRINTF("%d Freed dto EVD\n", getpid());
+                       h_dto_req_evd = DAT_HANDLE_NULL;
+               }
+       }
+
+       /* free CNO */
+       if (h_dto_cno != DAT_HANDLE_NULL) {
+               LOGPRINTF("%d Free dto CNO %p \n", getpid(), h_dto_cno);
+               start = get_time();
+               ret = dat_cno_free(h_dto_cno);
+               stop = get_time();
+               time.cnof += ((stop - start) * 1.0e6);
+               time.total += time.cnof;
+               if (ret != DAT_SUCCESS) {
+                       fprintf(stderr, "%d Error freeing dto CNO: %s\n",
+                               getpid(), DT_RetToString(ret));
+                       return (ret);
+               } else {
+                       LOGPRINTF("%d Freed dto CNO\n", getpid());
+                       h_dto_cno = DAT_HANDLE_NULL;
+               }
+       }
+       return DAT_SUCCESS;
 }
 
 /*
  * Map DAT_RETURN values to readable strings,
  * but don't assume the values are zero-based or contiguous.
  */
-char    errmsg[512] = {0};
-const char *
-DT_RetToString (DAT_RETURN ret_value)
+char errmsg[512] = { 0 };
+const char *DT_RetToString(DAT_RETURN ret_value)
 {
-    const char *major_msg, *minor_msg;
+       const char *major_msg, *minor_msg;
 
-    dat_strerror (ret_value, &major_msg, &minor_msg);
+       dat_strerror(ret_value, &major_msg, &minor_msg);
 
-    strcpy(errmsg, major_msg);
-    strcat(errmsg, " ");
-    strcat(errmsg, minor_msg);
+       strcpy(errmsg, major_msg);
+       strcat(errmsg, " ");
+       strcat(errmsg, minor_msg);
 
-    return errmsg;
+       return errmsg;
 }
 
 /*
  * Map DAT_EVENT_CODE values to readable strings
  */
-const char *
-DT_EventToSTr (DAT_EVENT_NUMBER event_code)
+const char *DT_EventToSTr(DAT_EVENT_NUMBER event_code)
 {
-    unsigned int i;
-    static struct {
-           const char  *name;
-           DAT_RETURN  value;
-    }
-    dat_events[] =
-    {
-           #   define DATxx(x) { # x, x }
-           DATxx (DAT_DTO_COMPLETION_EVENT),
-           DATxx (DAT_RMR_BIND_COMPLETION_EVENT),
-           DATxx (DAT_CONNECTION_REQUEST_EVENT),
-           DATxx (DAT_CONNECTION_EVENT_ESTABLISHED),
-           DATxx (DAT_CONNECTION_EVENT_PEER_REJECTED),
-           DATxx (DAT_CONNECTION_EVENT_NON_PEER_REJECTED),
-           DATxx (DAT_CONNECTION_EVENT_ACCEPT_COMPLETION_ERROR),
-           DATxx (DAT_CONNECTION_EVENT_DISCONNECTED),
-           DATxx (DAT_CONNECTION_EVENT_BROKEN),
-           DATxx (DAT_CONNECTION_EVENT_TIMED_OUT),
-           DATxx (DAT_CONNECTION_EVENT_UNREACHABLE),
-           DATxx (DAT_ASYNC_ERROR_EVD_OVERFLOW),
-           DATxx (DAT_ASYNC_ERROR_IA_CATASTROPHIC),
-           DATxx (DAT_ASYNC_ERROR_EP_BROKEN),
-           DATxx (DAT_ASYNC_ERROR_TIMED_OUT),
-           DATxx (DAT_ASYNC_ERROR_PROVIDER_INTERNAL_ERROR),
-           DATxx (DAT_SOFTWARE_EVENT)
-           #   undef DATxx
-    };
-    #   define NUM_EVENTS (sizeof(dat_events)/sizeof(dat_events[0]))
-
-    for (i = 0;  i < NUM_EVENTS;  i++) {
-           if (dat_events[i].value == event_code)
-           {
-               return ( dat_events[i].name );
-           }
-    }
-
-    return ( "Invalid_DAT_EVENT_NUMBER" );
-}
+       unsigned int i;
+       static struct {
+               const char *name;
+               DAT_RETURN value;
+       } dat_events[] = {
+#   define DATxx(x) { # x, x }
+               DATxx(DAT_DTO_COMPLETION_EVENT),
+                   DATxx(DAT_RMR_BIND_COMPLETION_EVENT),
+                   DATxx(DAT_CONNECTION_REQUEST_EVENT),
+                   DATxx(DAT_CONNECTION_EVENT_ESTABLISHED),
+                   DATxx(DAT_CONNECTION_EVENT_PEER_REJECTED),
+                   DATxx(DAT_CONNECTION_EVENT_NON_PEER_REJECTED),
+                   DATxx(DAT_CONNECTION_EVENT_ACCEPT_COMPLETION_ERROR),
+                   DATxx(DAT_CONNECTION_EVENT_DISCONNECTED),
+                   DATxx(DAT_CONNECTION_EVENT_BROKEN),
+                   DATxx(DAT_CONNECTION_EVENT_TIMED_OUT),
+                   DATxx(DAT_CONNECTION_EVENT_UNREACHABLE),
+                   DATxx(DAT_ASYNC_ERROR_EVD_OVERFLOW),
+                   DATxx(DAT_ASYNC_ERROR_IA_CATASTROPHIC),
+                   DATxx(DAT_ASYNC_ERROR_EP_BROKEN),
+                   DATxx(DAT_ASYNC_ERROR_TIMED_OUT),
+                   DATxx(DAT_ASYNC_ERROR_PROVIDER_INTERNAL_ERROR),
+                   DATxx(DAT_SOFTWARE_EVENT)
+#   undef DATxx
+       };
+#   define NUM_EVENTS (sizeof(dat_events)/sizeof(dat_events[0]))
+
+       for (i = 0; i < NUM_EVENTS; i++) {
+               if (dat_events[i].value == event_code) {
+                       return (dat_events[i].name);
+               }
+       }
 
+       return ("Invalid_DAT_EVENT_NUMBER");
+}
 
 void print_usage(void)
 {
-    printf("\n DAPL USAGE \n\n");
-    printf("s: server\n");
-    printf("t: performance times\n");
-    printf("c: use cno\n");
-    printf("v: verbose\n");
-    printf("p: polling\n");
-    printf("d: delay before accept\n");
-    printf("b: buf length to allocate\n");
-    printf("B: burst count, rdma and msgs \n");
-    printf("h: hostname/address of server, specified on client\n");
-    printf("P: provider name (default = OpenIB-cma)\n");
-    printf("\n");
+       printf("\n DAPL USAGE \n\n");
+       printf("s: server\n");
+       printf("t: performance times\n");
+       printf("c: use cno\n");
+       printf("v: verbose\n");
+       printf("p: polling\n");
+       printf("d: delay before accept\n");
+       printf("b: buf length to allocate\n");
+       printf("B: burst count, rdma and msgs \n");
+       printf("h: hostname/address of server, specified on client\n");
+       printf("P: provider name (default = OpenIB-cma)\n");
+       printf("\n");
 }