#include "..\..\..\..\etc\user\getopt.c"
#define getpid() ((int)GetCurrentProcessId())
-#define F64x "%I64x"
+#define F64x "%I64x"
#ifdef DBG
#define DAPL_PROVIDER "ibnic0v2d"
#define ntohll(x) bswap_64(x)
#endif
-#endif // _WIN32 || _WIN64
+#endif // _WIN32 || _WIN64
/* Debug: 1 == connect & close only, otherwise full-meal deal */
#define CONNECT_ONLY 0
#define RDMA_BUFFER_SIZE (64)
/* Global DAT vars */
-static DAT_IA_HANDLE h_ia = DAT_HANDLE_NULL;
-static DAT_PZ_HANDLE h_pz = DAT_HANDLE_NULL;
-static DAT_EP_HANDLE h_ep = DAT_HANDLE_NULL;
-static DAT_PSP_HANDLE h_psp = DAT_HANDLE_NULL;
-static DAT_CR_HANDLE h_cr = DAT_HANDLE_NULL;
-
-static DAT_EVD_HANDLE h_async_evd = DAT_HANDLE_NULL;
-static DAT_EVD_HANDLE h_dto_req_evd = DAT_HANDLE_NULL;
-static DAT_EVD_HANDLE h_dto_rcv_evd = DAT_HANDLE_NULL;
-static DAT_EVD_HANDLE h_cr_evd = DAT_HANDLE_NULL;
-static DAT_EVD_HANDLE h_conn_evd = DAT_HANDLE_NULL;
-static DAT_CNO_HANDLE h_dto_cno = DAT_HANDLE_NULL;
+static DAT_IA_HANDLE h_ia = DAT_HANDLE_NULL;
+static DAT_PZ_HANDLE h_pz = DAT_HANDLE_NULL;
+static DAT_EP_HANDLE h_ep = DAT_HANDLE_NULL;
+static DAT_PSP_HANDLE h_psp = DAT_HANDLE_NULL;
+static DAT_CR_HANDLE h_cr = DAT_HANDLE_NULL;
+
+static DAT_EVD_HANDLE h_async_evd = DAT_HANDLE_NULL;
+static DAT_EVD_HANDLE h_dto_req_evd = DAT_HANDLE_NULL;
+static DAT_EVD_HANDLE h_dto_rcv_evd = DAT_HANDLE_NULL;
+static DAT_EVD_HANDLE h_cr_evd = DAT_HANDLE_NULL;
+static DAT_EVD_HANDLE h_conn_evd = DAT_HANDLE_NULL;
+static DAT_CNO_HANDLE h_dto_cno = DAT_HANDLE_NULL;
/* RDMA buffers */
-static DAT_LMR_HANDLE h_lmr_send = DAT_HANDLE_NULL;
-static DAT_LMR_HANDLE h_lmr_recv = DAT_HANDLE_NULL;
-static DAT_LMR_CONTEXT lmr_context_send;
-static DAT_LMR_CONTEXT lmr_context_recv;
-static DAT_RMR_CONTEXT rmr_context_send;
-static DAT_RMR_CONTEXT rmr_context_recv;
-static DAT_VLEN registered_size_send;
-static DAT_VLEN registered_size_recv;
-static DAT_VADDR registered_addr_send;
-static DAT_VADDR registered_addr_recv;
+static DAT_LMR_HANDLE h_lmr_send = DAT_HANDLE_NULL;
+static DAT_LMR_HANDLE h_lmr_recv = DAT_HANDLE_NULL;
+static DAT_LMR_CONTEXT lmr_context_send;
+static DAT_LMR_CONTEXT lmr_context_recv;
+static DAT_RMR_CONTEXT rmr_context_send;
+static DAT_RMR_CONTEXT rmr_context_recv;
+static DAT_VLEN registered_size_send;
+static DAT_VLEN registered_size_recv;
+static DAT_VADDR registered_addr_send;
+static DAT_VADDR registered_addr_recv;
/* Initial msg receive buf, RMR exchange, and Rdma-write notification */
#define MSG_BUF_COUNT 3
#define MSG_IOV_COUNT 2
-static DAT_RMR_TRIPLET rmr_recv_msg[MSG_BUF_COUNT];
-static DAT_LMR_HANDLE h_lmr_recv_msg = DAT_HANDLE_NULL;
-static DAT_LMR_CONTEXT lmr_context_recv_msg;
-static DAT_RMR_CONTEXT rmr_context_recv_msg;
-static DAT_VLEN registered_size_recv_msg;
-static DAT_VADDR registered_addr_recv_msg;
+static DAT_RMR_TRIPLET rmr_recv_msg[MSG_BUF_COUNT];
+static DAT_LMR_HANDLE h_lmr_recv_msg = DAT_HANDLE_NULL;
+static DAT_LMR_CONTEXT lmr_context_recv_msg;
+static DAT_RMR_CONTEXT rmr_context_recv_msg;
+static DAT_VLEN registered_size_recv_msg;
+static DAT_VADDR registered_addr_recv_msg;
/* message send buffer */
-static DAT_RMR_TRIPLET rmr_send_msg;
-static DAT_LMR_HANDLE h_lmr_send_msg = DAT_HANDLE_NULL;
-static DAT_LMR_CONTEXT lmr_context_send_msg;
-static DAT_RMR_CONTEXT rmr_context_send_msg;
-static DAT_VLEN registered_size_send_msg;
-static DAT_VADDR registered_addr_send_msg;
-static DAT_EP_ATTR ep_attr;
-char hostname[256] = { 0 };
-char provider[64] = DAPL_PROVIDER;
-char addr_str[INET_ADDRSTRLEN];
+static DAT_RMR_TRIPLET rmr_send_msg;
+static DAT_LMR_HANDLE h_lmr_send_msg = DAT_HANDLE_NULL;
+static DAT_LMR_CONTEXT lmr_context_send_msg;
+static DAT_RMR_CONTEXT rmr_context_send_msg;
+static DAT_VLEN registered_size_send_msg;
+static DAT_VADDR registered_addr_send_msg;
+static DAT_EP_ATTR ep_attr;
+char hostname[256] = {0};
+char provider[64] = DAPL_PROVIDER;
+char addr_str[INET_ADDRSTRLEN];
/* rdma pointers */
-char *rbuf = NULL;
-char *sbuf = NULL;
-int status;
+char *rbuf = NULL;
+char *sbuf = NULL;
+int status;
/* timers */
-double start, stop, total_us, total_sec;
+double start,stop,total_us,total_sec;
struct dt_time {
- double total;
- double open;
- double reg;
- double unreg;
- double pzc;
- double pzf;
- double evdc;
- double evdf;
- double cnoc;
- double cnof;
- double epc;
- double epf;
- double rdma_wr;
- double rdma_rd[MAX_RDMA_RD];
- double rdma_rd_total;
- double rtt;
- double close;
+ double total;
+ double open;
+ double reg;
+ double unreg;
+ double pzc;
+ double pzf;
+ double evdc;
+ double evdf;
+ double cnoc;
+ double cnof;
+ double epc;
+ double epf;
+ double rdma_wr;
+ double rdma_rd[MAX_RDMA_RD];
+ double rdma_rd_total;
+ double rtt;
+ double close;
};
struct dt_time time;
/* defaults */
-static int failed = 0;
-static int performance_times = 0;
-static int connected = 0;
-static int burst = 10;
-static int server = 1;
-static int verbose = 0;
-static int polling = 0;
-static int poll_count = 0;
-static int rdma_wr_poll_count = 0;
-static int rdma_rd_poll_count[MAX_RDMA_RD] = { 0 };
-static int delay = 0;
-static int buf_len = RDMA_BUFFER_SIZE;
-static int use_cno = 0;
-static int recv_msg_index = 0;
-static int burst_msg_posted = 0;
-static int burst_msg_index = 0;
+static int failed=0;
+static int performance_times=0;
+static int connected=0;
+static int burst=10;
+static int server=1;
+static int verbose=0;
+static int polling=0;
+static int poll_count=0;
+static int rdma_wr_poll_count=0;
+static int rdma_rd_poll_count[MAX_RDMA_RD]={0};
+static int delay=0;
+static int buf_len=RDMA_BUFFER_SIZE;
+static int use_cno=0;
+static int recv_msg_index=0;
+static int burst_msg_posted=0;
+static int burst_msg_index=0;
/* forward prototypes */
-const char *DT_RetToString(DAT_RETURN ret_value);
-const char *DT_EventToSTr(DAT_EVENT_NUMBER event_code);
-void print_usage(void);
-double get_time(void);
-void init_data(void);
-
-DAT_RETURN send_msg(void *data,
- DAT_COUNT size,
- DAT_LMR_CONTEXT context,
- DAT_DTO_COOKIE cookie, DAT_COMPLETION_FLAGS flags);
-
-DAT_RETURN connect_ep(char *hostname, DAT_CONN_QUAL conn_id);
-void disconnect_ep(void);
-DAT_RETURN register_rdma_memory(void);
-DAT_RETURN unregister_rdma_memory(void);
-DAT_RETURN create_events(void);
-DAT_RETURN destroy_events(void);
-DAT_RETURN do_rdma_write_with_msg(void);
-DAT_RETURN do_rdma_read_with_msg(void);
-DAT_RETURN do_ping_pong_msg(void);
+const char * DT_RetToString (DAT_RETURN ret_value);
+const char * DT_EventToSTr (DAT_EVENT_NUMBER event_code);
+void print_usage(void);
+double get_time(void);
+void init_data(void);
+
+DAT_RETURN send_msg( void *data,
+ DAT_COUNT size,
+ DAT_LMR_CONTEXT context,
+ DAT_DTO_COOKIE cookie,
+ DAT_COMPLETION_FLAGS flags );
+
+DAT_RETURN connect_ep( char *hostname, DAT_CONN_QUAL conn_id );
+void disconnect_ep( void );
+DAT_RETURN register_rdma_memory( void );
+DAT_RETURN unregister_rdma_memory( void );
+DAT_RETURN create_events( void );
+DAT_RETURN destroy_events(void);
+DAT_RETURN do_rdma_write_with_msg( void );
+DAT_RETURN do_rdma_read_with_msg( void );
+DAT_RETURN do_ping_pong_msg( void );
#define LOGPRINTF if (verbose) printf
-void flush_evds(void)
+int
+main(int argc, char **argv)
{
- DAT_EVENT event;
-
- /* Flush async error queue */
- printf("%d ERR: Checking ASYNC EVD...\n", getpid());
- while (dat_evd_dequeue(h_async_evd, &event) == DAT_SUCCESS) {
- printf(" ASYNC EVD ENTRY: handle=%p reason=%d\n",
- event.event_data.asynch_error_event_data.dat_handle,
- event.event_data.asynch_error_event_data.reason);
- }
- /* Flush receive queue */
- printf("%d ERR: Checking RECEIVE EVD...\n", getpid());
- while (dat_evd_dequeue(h_dto_rcv_evd, &event) == DAT_SUCCESS) {
- printf(" RCV EVD ENTRY: op=%d stat=%d ln=%d ck="F64x"\n",
- event.event_data.dto_completion_event_data.operation,
- event.event_data.dto_completion_event_data.status,
- event.event_data.dto_completion_event_data.transfered_length,
- event.event_data.dto_completion_event_data.user_cookie.as_64);
- }
- /* Flush request queue */
- printf("%d ERR: Checking REQUEST EVD...\n", getpid());
- while (dat_evd_dequeue(h_dto_req_evd, &event) == DAT_SUCCESS) {
- printf(" REQ EVD ENTRY: op=%d stat=%d ln=%d ck="F64x"\n",
- event.event_data.dto_completion_event_data.operation,
- event.event_data.dto_completion_event_data.status,
- event.event_data.dto_completion_event_data.transfered_length,
- event.event_data.dto_completion_event_data.user_cookie.as_64);
- }
-}
-
-int main(int argc, char **argv)
-{
- int i, c;
- DAT_RETURN ret;
- DAT_EP_PARAM ep_param;
-
- /* parse arguments */
- while ((c = getopt(argc, argv, "tscvpb:d:B:h:P:")) != -1) {
- switch (c) {
- case 't':
- performance_times = 1;
- fflush(stdout);
- break;
- case 's':
- server = 1;
- fflush(stdout);
- break;
- case 'c':
- use_cno = 1;
- printf("%d Creating CNO for DTO EVD's\n", getpid());
- fflush(stdout);
- break;
- case 'v':
- verbose = 1;
- printf("%d Verbose\n", getpid());
- fflush(stdout);
- break;
- case 'p':
- polling = 1;
- printf("%d Polling\n", getpid());
- fflush(stdout);
- break;
- case 'B':
- burst = atoi(optarg);
- break;
- case 'd':
- delay = atoi(optarg);
- break;
- case 'b':
- buf_len = atoi(optarg);
- break;
- case 'h':
- server = 0;
- strcpy(hostname, optarg);
- break;
- case 'P':
- strcpy(provider, optarg);
- break;
- default:
- print_usage();
- exit(-12);
- }
- }
+ int i,c;
+ DAT_RETURN ret;
+ DAT_EP_PARAM ep_param;
+
+ /* parse arguments */
+ while ((c = getopt(argc, argv, "tscvpb:d:B:h:P:")) != -1)
+ {
+ switch(c)
+ {
+ case 't':
+ performance_times = 1;
+ fflush(stdout);
+ break;
+ case 's':
+ server = 1;
+ fflush(stdout);
+ break;
+ case 'c':
+ use_cno = 1;
+ printf("%d Creating CNO for DTO EVD's\n",getpid());
+ fflush(stdout);
+ break;
+ case 'v':
+ verbose = 1;
+ printf("%d Verbose\n",getpid());
+ fflush(stdout);
+ break;
+ case 'p':
+ polling = 1;
+ printf("%d Polling\n",getpid());
+ fflush(stdout);
+ break;
+ case 'B':
+ burst = atoi(optarg);
+ break;
+ case 'd':
+ delay = atoi(optarg);
+ break;
+ case 'b':
+ buf_len = atoi(optarg);
+ break;
+ case 'h':
+ server = 0;
+ strcpy (hostname, optarg);
+ break;
+ case 'P':
+ strcpy (provider, optarg);
+ break;
+ default:
+ print_usage();
+ exit(-12);
+ }
+ }
#if defined(_WIN32) || defined(_WIN64)
- {
- WSADATA wsaData;
-
- i = WSAStartup(MAKEWORD(2, 2), &wsaData);
- if (i != 0) {
- printf("%s WSAStartup(2.2) failed? (0x%x)\n", argv[0],
- i);
- fflush(stdout);
- exit(1);
- }
- }
+ {
+ WSADATA wsaData;
+
+ i = WSAStartup ( MAKEWORD(2,2), &wsaData );
+ if ( i != 0 ) {
+ printf("%s WSAStartup(2.2) failed? (0x%x)\n",argv[0],i);
+ fflush(stdout);
+ exit(1);
+ }
+ }
#endif
- if (!server) {
- printf("%d Running as client - %s\n", getpid(), provider);
- } else {
- printf("%d Running as server - %s\n", getpid(), provider);
- }
- fflush(stdout);
-
- /* allocate send and receive buffers */
- if (((rbuf = malloc(buf_len * (burst+1))) == NULL) ||
- ((sbuf = malloc(buf_len * (burst+1))) == NULL)) {
- perror("malloc");
- exit(1);
- }
- memset(&time, 0, sizeof(struct dt_time));
- LOGPRINTF("%d Allocated RDMA buffers (r:%p,s:%p) len %d \n",
- getpid(), rbuf, sbuf, buf_len);
-
- /* dat_ia_open, dat_pz_create */
- h_async_evd = DAT_HANDLE_NULL;
- start = get_time();
- ret = dat_ia_open(provider, 8, &h_async_evd, &h_ia);
- stop = get_time();
- time.open += ((stop - start) * 1.0e6);
- if (ret != DAT_SUCCESS) {
- fprintf(stderr, "%d: Error Adaptor open: %s\n",
- getpid(), DT_RetToString(ret));
- exit(1);
- } else
- LOGPRINTF("%d Opened Interface Adaptor\n", getpid());
-
- /* Create Protection Zone */
- start = get_time();
- LOGPRINTF("%d Create Protection Zone\n", getpid());
- ret = dat_pz_create(h_ia, &h_pz);
- stop = get_time();
- time.pzc += ((stop - start) * 1.0e6);
- if (ret != DAT_SUCCESS) {
- fprintf(stderr, "%d Error creating Protection Zone: %s\n",
- getpid(), DT_RetToString(ret));
- exit(1);
- } else
- LOGPRINTF("%d Created Protection Zone\n", getpid());
-
- /* Register memory */
- LOGPRINTF("%d Register RDMA memory\n", getpid());
- ret = register_rdma_memory();
- if (ret != DAT_SUCCESS) {
- fprintf(stderr, "%d Error registering RDMA memory: %s\n",
- getpid(), DT_RetToString(ret));
- goto cleanup;
- } else
- LOGPRINTF("%d Register RDMA memory done\n", getpid());
-
- LOGPRINTF("%d Create events\n", getpid());
- ret = create_events();
- if (ret != DAT_SUCCESS) {
- fprintf(stderr, "%d Error creating events: %s\n",
- getpid(), DT_RetToString(ret));
- goto cleanup;
- } else {
- LOGPRINTF("%d Create events done\n", getpid());
- }
-
- /* create EP */
- memset(&ep_attr, 0, sizeof(ep_attr));
- ep_attr.service_type = DAT_SERVICE_TYPE_RC;
- ep_attr.max_rdma_size = 0x10000;
- ep_attr.qos = 0;
- ep_attr.recv_completion_flags = 0;
- ep_attr.max_recv_dtos = MSG_BUF_COUNT + (burst * 3);
- ep_attr.max_request_dtos = MSG_BUF_COUNT + (burst * 3) + MAX_RDMA_RD;
- ep_attr.max_recv_iov = MSG_IOV_COUNT;
- ep_attr.max_request_iov = MSG_IOV_COUNT;
- ep_attr.max_rdma_read_in = MAX_RDMA_RD;
- ep_attr.max_rdma_read_out = MAX_RDMA_RD;
- ep_attr.request_completion_flags = DAT_COMPLETION_DEFAULT_FLAG;
- ep_attr.ep_transport_specific_count = 0;
- ep_attr.ep_transport_specific = NULL;
- ep_attr.ep_provider_specific_count = 0;
- ep_attr.ep_provider_specific = NULL;
-
- start = get_time();
- ret = dat_ep_create(h_ia, h_pz, h_dto_rcv_evd,
- h_dto_req_evd, h_conn_evd, &ep_attr, &h_ep);
- stop = get_time();
- time.epc += ((stop - start) * 1.0e6);
- time.total += time.epc;
- if (ret != DAT_SUCCESS) {
- fprintf(stderr, "%d Error dat_ep_create: %s\n",
- getpid(), DT_RetToString(ret));
- goto cleanup;
- } else
- LOGPRINTF("%d EP created %p \n", getpid(), h_ep);
-
- /*
- * register message buffers, establish connection, and
- * exchange DMA RMR information info via messages
- */
- ret = connect_ep(hostname, SERVER_CONN_QUAL);
- if (ret != DAT_SUCCESS) {
- fprintf(stderr, "%d Error connect_ep: %s\n",
- getpid(), DT_RetToString(ret));
- goto cleanup;
- } else
- LOGPRINTF("%d connect_ep complete\n", getpid());
-
- /* Query EP for local and remote address information, print */
- ret = dat_ep_query(h_ep, DAT_EP_FIELD_ALL, &ep_param);
- if (ret != DAT_SUCCESS) {
- fprintf(stderr, "%d Error dat_ep_query: %s\n",
- getpid(), DT_RetToString(ret));
- goto cleanup;
- } else
- LOGPRINTF("%d EP queried %p \n", getpid(), h_ep);
+ if (!server) {
+ printf("%d Running as client - %s\n",getpid(),provider);
+ } else {
+ printf("%d Running as server - %s\n",getpid(),provider);
+ }
+ fflush(stdout);
+
+ /* allocate send and receive buffers */
+ if (((rbuf = malloc(buf_len*burst)) == NULL) ||
+ ((sbuf = malloc(buf_len*burst)) == NULL)) {
+ perror("malloc");
+ exit(1);
+ }
+ memset( &time, 0, sizeof(struct dt_time) );
+ LOGPRINTF("%d Allocated RDMA buffers (r:%p,s:%p) len %d \n",
+ getpid(), rbuf, sbuf, buf_len);
+
+ /* dat_ia_open, dat_pz_create */
+ h_async_evd = DAT_HANDLE_NULL;
+ start = get_time();
+ ret = dat_ia_open( provider, 8, &h_async_evd, &h_ia );
+ stop = get_time();
+ time.open += ((stop - start)*1.0e6);
+ if(ret != DAT_SUCCESS) {
+ fprintf(stderr, "%d: Error Adaptor open: %s\n",
+ getpid(),DT_RetToString(ret));
+ exit(1);
+ } else
+ LOGPRINTF("%d Opened Interface Adaptor\n",getpid());
+
+ /* Create Protection Zone */
+ start = get_time();
+ LOGPRINTF("%d Create Protection Zone\n",getpid());
+ ret = dat_pz_create(h_ia, &h_pz);
+ stop = get_time();
+ time.pzc += ((stop - start)*1.0e6);
+ if(ret != DAT_SUCCESS) {
+ fprintf(stderr, "%d Error creating Protection Zone: %s\n",
+ getpid(),DT_RetToString(ret));
+ exit(1);
+ } else
+ LOGPRINTF("%d Created Protection Zone\n",getpid());
+
+ /* Register memory */
+ LOGPRINTF("%d Register RDMA memory\n", getpid());
+ ret = register_rdma_memory();
+ if(ret != DAT_SUCCESS) {
+ fprintf(stderr, "%d Error registering RDMA memory: %s\n",
+ getpid(),DT_RetToString(ret));
+ goto cleanup;
+ } else
+ LOGPRINTF("%d Register RDMA memory done\n", getpid());
+
+ LOGPRINTF("%d Create events\n", getpid());
+ ret = create_events();
+ if(ret != DAT_SUCCESS) {
+ fprintf(stderr, "%d Error creating events: %s\n",
+ getpid(),DT_RetToString(ret));
+ goto cleanup;
+ } else {
+ LOGPRINTF("%d Create events done\n", getpid());
+ }
+
+ /* create EP */
+ memset( &ep_attr, 0, sizeof(ep_attr) );
+ ep_attr.service_type = DAT_SERVICE_TYPE_RC;
+ ep_attr.max_rdma_size = 0x10000;
+ ep_attr.qos = 0;
+ ep_attr.recv_completion_flags = 0;
+ ep_attr.max_recv_dtos = MSG_BUF_COUNT + (burst*3);
+ ep_attr.max_request_dtos = MSG_BUF_COUNT + (burst*3) + MAX_RDMA_RD;
+ ep_attr.max_recv_iov = MSG_IOV_COUNT;
+ ep_attr.max_request_iov = MSG_IOV_COUNT;
+ ep_attr.max_rdma_read_in = MAX_RDMA_RD;
+ ep_attr.max_rdma_read_out = MAX_RDMA_RD;
+ ep_attr.request_completion_flags = DAT_COMPLETION_DEFAULT_FLAG;
+ ep_attr.ep_transport_specific_count = 0;
+ ep_attr.ep_transport_specific = NULL;
+ ep_attr.ep_provider_specific_count = 0;
+ ep_attr.ep_provider_specific = NULL;
+
+ start = get_time();
+ ret = dat_ep_create( h_ia, h_pz, h_dto_rcv_evd,
+ h_dto_req_evd, h_conn_evd, &ep_attr, &h_ep );
+ stop = get_time();
+ time.epc += ((stop - start)*1.0e6);
+ time.total += time.epc;
+ if(ret != DAT_SUCCESS) {
+ fprintf(stderr, "%d Error dat_ep_create: %s\n",
+ getpid(),DT_RetToString(ret));
+ goto cleanup;
+ } else
+ LOGPRINTF("%d EP created %p \n", getpid(), h_ep);
+
+ /*
+ * register message buffers, establish connection, and
+ * exchange DMA RMR information info via messages
+ */
+ ret = connect_ep( hostname, SERVER_CONN_QUAL );
+ if(ret != DAT_SUCCESS) {
+ fprintf(stderr, "%d Error connect_ep: %s\n",
+ getpid(),DT_RetToString(ret));
+ goto cleanup;
+ } else
+ LOGPRINTF("%d connect_ep complete\n", getpid());
+
+ /* Query EP for local and remote address information, print */
+ ret = dat_ep_query(h_ep, DAT_EP_FIELD_ALL, &ep_param);
+ if(ret != DAT_SUCCESS) {
+ fprintf(stderr, "%d Error dat_ep_query: %s\n",
+ getpid(),DT_RetToString(ret));
+ goto cleanup;
+ } else
+ LOGPRINTF("%d EP queried %p \n", getpid(), h_ep);
#if defined(_WIN32)
- printf("\n%d Query EP: LOCAL addr %s port %lld\n", getpid(),
- inet_ntoa(((struct sockaddr_in *)
- ep_param.local_ia_address_ptr)->sin_addr),
- (ep_param.local_port_qual));
+ printf("\n%d Query EP: LOCAL addr %s port %lld\n", getpid(),
+ inet_ntoa(((struct sockaddr_in *)
+ ep_param.local_ia_address_ptr)->sin_addr),
+ (ep_param.local_port_qual));
#else
- inet_ntop(AF_INET,
- &((struct sockaddr_in *)ep_param.local_ia_address_ptr)->
- sin_addr, addr_str, sizeof(addr_str));
- printf("\n%d Query EP: LOCAL addr %s port " F64x "\n", getpid(),
- addr_str, (ep_param.local_port_qual));
+ inet_ntop(AF_INET,
+ &((struct sockaddr_in *)ep_param.local_ia_address_ptr)->sin_addr,
+ addr_str, sizeof(addr_str));
+ printf("\n%d Query EP: LOCAL addr %s port "F64x"\n", getpid(),
+ addr_str, (ep_param.local_port_qual));
#endif
#if defined(_WIN32)
- printf("%d Query EP: REMOTE addr %s port %lld\n", getpid(),
- inet_ntoa(((struct sockaddr_in *)
- ep_param.local_ia_address_ptr)->sin_addr),
- (ep_param.remote_port_qual));
+ printf("%d Query EP: REMOTE addr %s port %lld\n", getpid(),
+ inet_ntoa(((struct sockaddr_in *)
+ ep_param.local_ia_address_ptr)->sin_addr),
+ (ep_param.remote_port_qual));
#else
- inet_ntop(AF_INET,
- &((struct sockaddr_in *)ep_param.remote_ia_address_ptr)->
- sin_addr, addr_str, sizeof(addr_str));
- printf("%d Query EP: REMOTE addr %s port " F64x "\n", getpid(),
- addr_str, (ep_param.remote_port_qual));
+ inet_ntop(AF_INET,
+ &((struct sockaddr_in *)ep_param.remote_ia_address_ptr)->sin_addr,
+ addr_str, sizeof(addr_str));
+ printf("%d Query EP: REMOTE addr %s port "F64x"\n", getpid(),
+ addr_str, (ep_param.remote_port_qual));
#endif
- fflush(stdout);
+ fflush(stdout);
#if CONNECT_ONLY
#if defined(_WIN32) || defined(_WIN64)
- Sleep(1 * 1000);
+ Sleep(1*1000);
#else
- sleep(1);
+ sleep(1);
#endif
- goto cleanup;
+ goto cleanup;
#endif
- /*********** RDMA write data *************/
- ret = do_rdma_write_with_msg();
- if (ret != DAT_SUCCESS) {
- fprintf(stderr, "%d Error do_rdma_write_with_msg: %s\n",
- getpid(), DT_RetToString(ret));
- goto cleanup;
- } else
- LOGPRINTF("%d do_rdma_write_with_msg complete\n", getpid());
-
- /*********** RDMA read data *************/
- ret = do_rdma_read_with_msg();
- if (ret != DAT_SUCCESS) {
- fprintf(stderr, "%d Error do_rdma_read_with_msg: %s\n",
- getpid(), DT_RetToString(ret));
- goto cleanup;
- } else
- LOGPRINTF("%d do_rdma_read_with_msg complete\n", getpid());
-
- /*********** PING PING messages ************/
- ret = do_ping_pong_msg();
- if (ret != DAT_SUCCESS) {
- fprintf(stderr, "%d Error do_ping_pong_msg: %s\n",
- getpid(), DT_RetToString(ret));
- goto cleanup;
- } else {
- LOGPRINTF("%d do_ping_pong_msg complete\n", getpid());
- goto complete;
- }
+ /*********** RDMA write data *************/
+ ret = do_rdma_write_with_msg();
+ if(ret != DAT_SUCCESS) {
+ fprintf(stderr, "%d Error do_rdma_write_with_msg: %s\n",
+ getpid(),DT_RetToString(ret));
+ goto cleanup;
+ } else
+ LOGPRINTF("%d do_rdma_write_with_msg complete\n", getpid());
+
+ /*********** RDMA read data *************/
+ ret = do_rdma_read_with_msg();
+ if(ret != DAT_SUCCESS) {
+ fprintf(stderr, "%d Error do_rdma_read_with_msg: %s\n",
+ getpid(),DT_RetToString(ret));
+ goto cleanup;
+ } else
+ LOGPRINTF("%d do_rdma_read_with_msg complete\n", getpid());
+
+ /*********** PING PING messages ************/
+ ret = do_ping_pong_msg();
+ if(ret != DAT_SUCCESS) {
+ fprintf(stderr, "%d Error do_ping_pong_msg: %s\n",
+ getpid(),DT_RetToString(ret));
+ goto cleanup;
+ } else {
+ LOGPRINTF("%d do_ping_pong_msg complete\n", getpid());
+ goto complete;
+ }
cleanup:
-
- fflush(stderr);
- fflush(stdout);
- flush_evds();
- failed++;
- fflush(stderr);
- fflush(stdout);
-
+ failed++;
complete:
- /* disconnect and free EP resources */
- if (h_ep != DAT_HANDLE_NULL) {
- /* unregister message buffers and tear down connection */
- LOGPRINTF("%d Disconnect and Free EP %p \n", getpid(), h_ep);
- disconnect_ep();
-
- /* free EP */
- LOGPRINTF("%d Free EP %p \n", getpid(), h_ep);
- start = get_time();
- ret = dat_ep_free(h_ep);
- stop = get_time();
- time.epf += ((stop - start) * 1.0e6);
- time.total += time.epf;
- if (ret != DAT_SUCCESS) {
- fprintf(stderr, "%d Error freeing EP: %s\n",
- getpid(), DT_RetToString(ret));
- } else {
- LOGPRINTF("%d Freed EP\n", getpid());
- h_ep = DAT_HANDLE_NULL;
- }
- }
-
- /* free EVDs */
- LOGPRINTF("%d destroy events\n", getpid());
- ret = destroy_events();
- if (ret != DAT_SUCCESS)
- fprintf(stderr, "%d Error destroy_events: %s\n",
- getpid(), DT_RetToString(ret));
- else
- LOGPRINTF("%d destroy events done\n", getpid());
-
- ret = unregister_rdma_memory();
- LOGPRINTF("%d unregister_rdma_memory \n", getpid());
- if (ret != DAT_SUCCESS)
- fprintf(stderr, "%d Error unregister_rdma_memory: %s\n",
- getpid(), DT_RetToString(ret));
- else
- LOGPRINTF("%d unregister_rdma_memory done\n", getpid());
-
- /* Free protection domain */
- LOGPRINTF("%d Freeing pz\n", getpid());
- start = get_time();
- ret = dat_pz_free(h_pz);
- stop = get_time();
- time.pzf += ((stop - start) * 1.0e6);
- if (ret != DAT_SUCCESS) {
- fprintf(stderr, "%d Error freeing PZ: %s\n",
- getpid(), DT_RetToString(ret));
- } else {
- LOGPRINTF("%d Freed pz\n", getpid());
- h_pz = NULL;
- }
-
- /* close the device */
- LOGPRINTF("%d Closing Interface Adaptor\n", getpid());
- start = get_time();
- ret = dat_ia_close(h_ia, DAT_CLOSE_ABRUPT_FLAG);
- stop = get_time();
- time.close += ((stop - start) * 1.0e6);
- if (ret != DAT_SUCCESS) {
- fprintf(stderr, "%d: Error Adaptor close: %s\n",
- getpid(), DT_RetToString(ret));
- } else
- LOGPRINTF("%d Closed Interface Adaptor\n", getpid());
-
- /* free rdma buffers */
- free(rbuf);
- free(sbuf);
-
- printf("\n%d: DAPL Test Complete. %s\n\n",
- getpid(), failed ? "FAILED" : "PASSED");
-
- if (!performance_times)
- exit(0);
-
- printf("\n%d: DAPL Test Complete.\n\n", getpid());
- printf("%d: Message RTT: Total=%10.2lf usec, %d bursts, itime=%10.2lf"
- " usec, pc=%d\n",
- getpid(), time.rtt, burst, time.rtt / burst, poll_count);
- printf("%d: RDMA write: Total=%10.2lf usec, %d bursts, itime=%10.2lf"
- " usec, pc=%d\n",
- getpid(), time.rdma_wr, burst,
- time.rdma_wr / burst, rdma_wr_poll_count);
- for (i = 0; i < MAX_RDMA_RD; i++) {
- printf("%d: RDMA read: Total=%10.2lf usec, %d bursts, "
- "itime=%10.2lf usec, pc=%d\n",
- getpid(), time.rdma_rd_total, MAX_RDMA_RD,
- time.rdma_rd[i], rdma_rd_poll_count[i]);
- }
- printf("%d: open: %10.2lf usec\n", getpid(), time.open);
- printf("%d: close: %10.2lf usec\n", getpid(), time.close);
- printf("%d: PZ create: %10.2lf usec\n", getpid(), time.pzc);
- printf("%d: PZ free: %10.2lf usec\n", getpid(), time.pzf);
- printf("%d: LMR create:%10.2lf usec\n", getpid(), time.reg);
- printf("%d: LMR free: %10.2lf usec\n", getpid(), time.unreg);
- printf("%d: EVD create:%10.2lf usec\n", getpid(), time.evdc);
- printf("%d: EVD free: %10.2lf usec\n", getpid(), time.evdf);
- if (use_cno) {
- printf("%d: CNO create: %10.2lf usec\n", getpid(), time.cnoc);
- printf("%d: CNO free: %10.2lf usec\n", getpid(), time.cnof);
- }
- printf("%d: EP create: %10.2lf usec\n", getpid(), time.epc);
- printf("%d: EP free: %10.2lf usec\n", getpid(), time.epf);
- printf("%d: TOTAL: %10.2lf usec\n", getpid(), time.total);
+ /* disconnect and free EP resources */
+ if ( h_ep != DAT_HANDLE_NULL ) {
+ /* unregister message buffers and tear down connection */
+ LOGPRINTF("%d Disconnect and Free EP %p \n",getpid(),h_ep);
+ disconnect_ep();
+
+ /* free EP */
+ LOGPRINTF("%d Free EP %p \n",getpid(),h_ep);
+ start = get_time();
+ ret = dat_ep_free( h_ep );
+ stop = get_time();
+ time.epf += ((stop - start)*1.0e6);
+ time.total += time.epf;
+ if(ret != DAT_SUCCESS) {
+ fprintf(stderr, "%d Error freeing EP: %s\n",
+ getpid(), DT_RetToString(ret));
+ } else {
+ LOGPRINTF("%d Freed EP\n",getpid());
+ h_ep = DAT_HANDLE_NULL;
+ }
+ }
+
+ /* free EVDs */
+ LOGPRINTF("%d destroy events\n", getpid());
+ ret = destroy_events();
+ if(ret != DAT_SUCCESS)
+ fprintf(stderr, "%d Error destroy_events: %s\n",
+ getpid(),DT_RetToString(ret));
+ else
+ LOGPRINTF("%d destroy events done\n", getpid());
+
+
+ ret = unregister_rdma_memory();
+ LOGPRINTF("%d unregister_rdma_memory \n", getpid());
+ if(ret != DAT_SUCCESS)
+ fprintf(stderr, "%d Error unregister_rdma_memory: %s\n",
+ getpid(),DT_RetToString(ret));
+ else
+ LOGPRINTF("%d unregister_rdma_memory done\n", getpid());
+
+ /* Free protection domain */
+ LOGPRINTF("%d Freeing pz\n",getpid());
+ start = get_time();
+ ret = dat_pz_free( h_pz );
+ stop = get_time();
+ time.pzf += ((stop - start)*1.0e6);
+ if (ret != DAT_SUCCESS) {
+ fprintf(stderr, "%d Error freeing PZ: %s\n",
+ getpid(), DT_RetToString(ret));
+ } else {
+ LOGPRINTF("%d Freed pz\n",getpid());
+ h_pz = NULL;
+ }
+
+ /* close the device */
+ LOGPRINTF("%d Closing Interface Adaptor\n",getpid());
+ start = get_time();
+ ret = dat_ia_close( h_ia, DAT_CLOSE_ABRUPT_FLAG );
+ stop = get_time();
+ time.close += ((stop - start)*1.0e6);
+ if(ret != DAT_SUCCESS) {
+ fprintf(stderr, "%d: Error Adaptor close: %s\n",
+ getpid(),DT_RetToString(ret));
+ exit(1);
+ } else
+ LOGPRINTF("%d Closed Interface Adaptor\n",getpid());
+
+ /* free rdma buffers */
+ free(rbuf);
+ free(sbuf);
+
+ printf("\n%d: DAPL Test Complete. %s\n\n",
+ getpid(), failed?"FAILED":"PASSED");
+
+ if (!performance_times)
+ exit(0);
+
+ printf( "\n%d: DAPL Test Complete.\n\n",getpid());
+ printf( "%d: Message RTT: Total=%10.2lf usec, %d bursts, itime=%10.2lf"
+ " usec, pc=%d\n",
+ getpid(), time.rtt, burst, time.rtt/burst, poll_count );
+ printf( "%d: RDMA write: Total=%10.2lf usec, %d bursts, itime=%10.2lf"
+ " usec, pc=%d\n",
+ getpid(), time.rdma_wr, burst,
+ time.rdma_wr/burst, rdma_wr_poll_count );
+ for(i=0;i<MAX_RDMA_RD;i++) {
+ printf( "%d: RDMA read: Total=%10.2lf usec, %d bursts, "
+ "itime=%10.2lf usec, pc=%d\n",
+ getpid(),time.rdma_rd_total,MAX_RDMA_RD,
+ time.rdma_rd[i],rdma_rd_poll_count[i] );
+ }
+ printf( "%d: open: %10.2lf usec\n", getpid(), time.open );
+ printf( "%d: close: %10.2lf usec\n", getpid(), time.close );
+ printf( "%d: PZ create: %10.2lf usec\n", getpid(), time.pzc );
+ printf( "%d: PZ free: %10.2lf usec\n", getpid(), time.pzf );
+ printf( "%d: LMR create:%10.2lf usec\n", getpid(), time.reg );
+ printf( "%d: LMR free: %10.2lf usec\n", getpid(), time.unreg );
+ printf( "%d: EVD create:%10.2lf usec\n", getpid(), time.evdc );
+ printf( "%d: EVD free: %10.2lf usec\n", getpid(), time.evdf );
+ if (use_cno) {
+ printf( "%d: CNO create: %10.2lf usec\n", getpid(), time.cnoc );
+ printf( "%d: CNO free: %10.2lf usec\n", getpid(), time.cnof );
+ }
+ printf( "%d: EP create: %10.2lf usec\n",getpid(), time.epc );
+ printf( "%d: EP free: %10.2lf usec\n",getpid(), time.epf );
+ printf( "%d: TOTAL: %10.2lf usec\n",getpid(), time.total );
#if defined(_WIN32) || defined(_WIN64)
- WSACleanup();
+ WSACleanup();
#endif
- return (0);
+ return(0);
}
#if defined(_WIN32) || defined(_WIN64)
-void gettimeofday(struct timeval *t, char *jnk)
+void gettimeofday( struct timeval *t, char *jnk)
{
SYSTEMTIME now;
GetLocalTime(&now);
double get_time(void)
{
- struct timeval tp;
+ struct timeval tp;
- gettimeofday(&tp, NULL);
- return ((double)tp.tv_sec + (double)tp.tv_usec * 1e-6);
+ gettimeofday(&tp, NULL);
+ return ((double) tp.tv_sec + (double) tp.tv_usec * 1e-6);
}
void init_data(void)
{
- memset(rbuf, 'a', buf_len);
- memset(sbuf, 'b', buf_len);
+ memset(rbuf, 'a', buf_len);
+ memset(sbuf, 'b', buf_len);
}
+
DAT_RETURN
-send_msg(void *data,
- DAT_COUNT size,
- DAT_LMR_CONTEXT context,
- DAT_DTO_COOKIE cookie, DAT_COMPLETION_FLAGS flags)
+send_msg( void *data,
+ DAT_COUNT size,
+ DAT_LMR_CONTEXT context,
+ DAT_DTO_COOKIE cookie,
+ DAT_COMPLETION_FLAGS flags )
{
- DAT_LMR_TRIPLET iov;
- DAT_EVENT event;
- DAT_COUNT nmore;
- DAT_RETURN ret;
+ DAT_LMR_TRIPLET iov;
+ DAT_EVENT event;
+ DAT_COUNT nmore;
+ DAT_RETURN ret;
- iov.lmr_context = context;
+ iov.lmr_context = context;
#if defined(_WIN32)
- iov.virtual_address = (DAT_VADDR) data;
+ iov.virtual_address = (DAT_VADDR)data;
#else
- iov.virtual_address = (DAT_VADDR) (unsigned long)data;
+ iov.virtual_address = (DAT_VADDR)(unsigned long)data;
#endif
- iov.segment_length = size;
-
- LOGPRINTF("%d calling post_send\n", getpid());
- cookie.as_64 = 0xaaaa;
- ret = dat_ep_post_send(h_ep, 1, &iov, cookie, flags);
-
- if (ret != DAT_SUCCESS) {
- fprintf(stderr, "%d: ERROR: dat_ep_post_send() %s\n",
- getpid(), DT_RetToString(ret));
- return ret;
- }
-
- if (!(flags & DAT_COMPLETION_SUPPRESS_FLAG)) {
- if (polling) {
- printf("%d Polling post send completion...\n",
- getpid());
- while (dat_evd_dequeue(h_dto_req_evd, &event) ==
- DAT_QUEUE_EMPTY) ;
- } else {
- LOGPRINTF("%d waiting for post_send completion event\n",
- getpid());
- if (use_cno) {
- DAT_EVD_HANDLE evd = DAT_HANDLE_NULL;
- ret =
- dat_cno_wait(h_dto_cno, DTO_TIMEOUT, &evd);
- LOGPRINTF("%d cno wait return evd_handle=%p\n",
- getpid(), evd);
- if (evd != h_dto_req_evd) {
- fprintf(stderr,
- "%d Error waiting on h_dto_cno: evd != h_dto_req_evd\n",
- getpid());
- return (DAT_ABORT);
- }
- }
- /* use wait to dequeue */
- ret =
- dat_evd_wait(h_dto_req_evd, DTO_TIMEOUT, 1, &event,
- &nmore);
- if (ret != DAT_SUCCESS) {
- fprintf(stderr,
- "%d: ERROR: DTO dat_evd_wait() %s\n",
- getpid(), DT_RetToString(ret));
- return ret;
- }
- }
-
- /* validate event number, len, cookie, and status */
- if (event.event_number != DAT_DTO_COMPLETION_EVENT) {
- fprintf(stderr, "%d: ERROR: DTO event number %s\n",
- getpid(), DT_EventToSTr(event.event_number));
- return (DAT_ABORT);
- }
-
- if ((event.event_data.dto_completion_event_data.
- transfered_length != size)
- || (event.event_data.dto_completion_event_data.user_cookie.
- as_64 != 0xaaaa)) {
- fprintf(stderr,
- "%d: ERROR: DTO len %d or cookie " F64x " \n",
- getpid(),
- event.event_data.dto_completion_event_data.
- transfered_length,
- event.event_data.dto_completion_event_data.
- user_cookie.as_64);
- return (DAT_ABORT);
-
- }
- if (event.event_data.dto_completion_event_data.status !=
- DAT_SUCCESS) {
- fprintf(stderr, "%d: ERROR: DTO event status %s\n",
- getpid(), DT_RetToString(ret));
- return (DAT_ABORT);
- }
- }
-
- return DAT_SUCCESS;
+ iov.segment_length = size;
+
+ LOGPRINTF("%d calling post_send\n", getpid());
+ cookie.as_64 = 0xaaaa;
+ ret = dat_ep_post_send( h_ep,
+ 1,
+ &iov,
+ cookie,
+ flags );
+
+ if (ret != DAT_SUCCESS) {
+ fprintf(stderr, "%d: ERROR: dat_ep_post_send() %s\n",
+ getpid(),DT_RetToString(ret));
+ return ret;
+ }
+
+ if (!(flags & DAT_COMPLETION_SUPPRESS_FLAG)) {
+ if ( polling ) {
+ printf("%d Polling post send completion...\n",getpid());
+ while (dat_evd_dequeue(h_dto_req_evd, &event) == DAT_QUEUE_EMPTY) ;
+ }
+ else {
+ LOGPRINTF("%d waiting for post_send completion event\n", getpid());
+ if (use_cno) {
+ DAT_EVD_HANDLE evd = DAT_HANDLE_NULL;
+ ret = dat_cno_wait( h_dto_cno, DTO_TIMEOUT, &evd );
+ LOGPRINTF("%d cno wait return evd_handle=%p\n", getpid(),evd);
+ if ( evd != h_dto_req_evd ) {
+ fprintf(stderr,
+ "%d Error waiting on h_dto_cno: evd != h_dto_req_evd\n",
+ getpid());
+ return( DAT_ABORT );
+ }
+ }
+ /* use wait to dequeue */
+ ret = dat_evd_wait( h_dto_req_evd, DTO_TIMEOUT, 1, &event, &nmore );
+ if (ret != DAT_SUCCESS) {
+ fprintf(stderr, "%d: ERROR: DTO dat_evd_wait() %s\n",
+ getpid(),DT_RetToString(ret));
+ return ret;
+ }
+ }
+
+ /* validate event number, len, cookie, and status */
+ if ( event.event_number != DAT_DTO_COMPLETION_EVENT ) {
+ fprintf(stderr, "%d: ERROR: DTO event number %s\n",
+ getpid(),DT_EventToSTr(event.event_number));
+ return( DAT_ABORT );
+ }
+
+ if ((event.event_data.dto_completion_event_data.transfered_length != size ) ||
+ (event.event_data.dto_completion_event_data.user_cookie.as_64 != 0xaaaa )) {
+ fprintf(stderr, "%d: ERROR: DTO len %d or cookie "F64x" \n",
+ getpid(),
+ event.event_data.dto_completion_event_data.transfered_length,
+ event.event_data.dto_completion_event_data.user_cookie.as_64 );
+ return( DAT_ABORT );
+
+ }
+ if (event.event_data.dto_completion_event_data.status != DAT_SUCCESS) {
+ fprintf(stderr, "%d: ERROR: DTO event status %s\n",
+ getpid(),DT_RetToString(ret));
+ return( DAT_ABORT );
+ }
+ }
+
+ return DAT_SUCCESS;
}
-DAT_RETURN connect_ep(char *hostname, DAT_CONN_QUAL conn_id)
+DAT_RETURN
+connect_ep( char *hostname, DAT_CONN_QUAL conn_id )
{
- DAT_SOCK_ADDR remote_addr;
- DAT_RETURN ret;
- DAT_REGION_DESCRIPTION region;
- DAT_EVENT event;
- DAT_COUNT nmore;
- DAT_LMR_TRIPLET l_iov;
- DAT_RMR_TRIPLET r_iov;
- DAT_DTO_COOKIE cookie;
- int i;
- unsigned char *buf;
- DAT_CR_PARAM cr_param = { 0 };
- unsigned char pdata[48] = { 0 };
-
- /* Register send message buffer */
- LOGPRINTF("%d Registering send Message Buffer %p, len %d\n",
- getpid(), &rmr_send_msg, (int)sizeof(DAT_RMR_TRIPLET));
- region.for_va = &rmr_send_msg;
- ret = dat_lmr_create(h_ia,
- DAT_MEM_TYPE_VIRTUAL,
- region,
- sizeof(DAT_RMR_TRIPLET),
- h_pz,
- DAT_MEM_PRIV_LOCAL_WRITE_FLAG,
- DAT_VA_TYPE_VA,
- &h_lmr_send_msg,
- &lmr_context_send_msg,
- &rmr_context_send_msg,
- ®istered_size_send_msg,
- ®istered_addr_send_msg);
-
- if (ret != DAT_SUCCESS) {
- fprintf(stderr, "%d Error registering send msg buffer: %s\n",
- getpid(), DT_RetToString(ret));
- return (ret);
- } else
- LOGPRINTF("%d Registered send Message Buffer %p \n",
- getpid(), region.for_va);
-
- /* Register Receive buffers */
- LOGPRINTF("%d Registering Receive Message Buffer %p\n",
- getpid(), rmr_recv_msg);
- region.for_va = rmr_recv_msg;
- ret = dat_lmr_create(h_ia,
- DAT_MEM_TYPE_VIRTUAL,
- region,
- sizeof(DAT_RMR_TRIPLET) * MSG_BUF_COUNT,
- h_pz,
- DAT_MEM_PRIV_LOCAL_WRITE_FLAG,
- DAT_VA_TYPE_VA,
- &h_lmr_recv_msg,
- &lmr_context_recv_msg,
- &rmr_context_recv_msg,
- ®istered_size_recv_msg,
- ®istered_addr_recv_msg);
- if (ret != DAT_SUCCESS) {
- fprintf(stderr, "%d Error registering recv msg buffer: %s\n",
- getpid(), DT_RetToString(ret));
- return (ret);
- } else
- LOGPRINTF("%d Registered Receive Message Buffer %p\n",
- getpid(), region.for_va);
-
- for (i = 0; i < MSG_BUF_COUNT; i++) {
- cookie.as_64 = i;
- l_iov.lmr_context = lmr_context_recv_msg;
+ DAT_SOCK_ADDR remote_addr;
+ DAT_RETURN ret;
+ DAT_REGION_DESCRIPTION region;
+ DAT_EVENT event;
+ DAT_COUNT nmore;
+ DAT_LMR_TRIPLET l_iov;
+ DAT_RMR_TRIPLET r_iov;
+ DAT_DTO_COOKIE cookie;
+ int i;
+ unsigned char *buf;
+ DAT_CR_PARAM cr_param = { 0 };
+ unsigned char pdata[48] = { 0 };
+
+ /* Register send message buffer */
+ LOGPRINTF("%d Registering send Message Buffer %p, len %d\n",
+ getpid(), &rmr_send_msg, (int)sizeof(DAT_RMR_TRIPLET) );
+ region.for_va = &rmr_send_msg;
+ ret = dat_lmr_create( h_ia,
+ DAT_MEM_TYPE_VIRTUAL,
+ region,
+ sizeof(DAT_RMR_TRIPLET),
+ h_pz,
+ DAT_MEM_PRIV_LOCAL_WRITE_FLAG,
+ DAT_VA_TYPE_VA,
+ &h_lmr_send_msg,
+ &lmr_context_send_msg,
+ &rmr_context_send_msg,
+ ®istered_size_send_msg,
+ ®istered_addr_send_msg );
+
+ if (ret != DAT_SUCCESS) {
+ fprintf(stderr, "%d Error registering send msg buffer: %s\n",
+ getpid(),DT_RetToString(ret));
+ return(ret);
+ }
+ else
+ LOGPRINTF("%d Registered send Message Buffer %p \n",
+ getpid(),region.for_va );
+
+ /* Register Receive buffers */
+ LOGPRINTF("%d Registering Receive Message Buffer %p\n",
+ getpid(), rmr_recv_msg );
+ region.for_va = rmr_recv_msg;
+ ret = dat_lmr_create( h_ia,
+ DAT_MEM_TYPE_VIRTUAL,
+ region,
+ sizeof(DAT_RMR_TRIPLET)*MSG_BUF_COUNT,
+ h_pz,
+ DAT_MEM_PRIV_LOCAL_WRITE_FLAG,
+ DAT_VA_TYPE_VA,
+ &h_lmr_recv_msg,
+ &lmr_context_recv_msg,
+ &rmr_context_recv_msg,
+ ®istered_size_recv_msg,
+ ®istered_addr_recv_msg );
+ if(ret != DAT_SUCCESS) {
+ fprintf(stderr, "%d Error registering recv msg buffer: %s\n",
+ getpid(),DT_RetToString(ret));
+ return(ret);
+ }
+ else
+ LOGPRINTF("%d Registered Receive Message Buffer %p\n",
+ getpid(),region.for_va);
+
+ for ( i = 0; i < MSG_BUF_COUNT; i++ ) {
+ cookie.as_64 = i;
+ l_iov.lmr_context = lmr_context_recv_msg;
#if defined(_WIN32)
- l_iov.virtual_address = (DAT_VADDR) & rmr_recv_msg[i];
+ l_iov.virtual_address = (DAT_VADDR)&rmr_recv_msg[ i ];
#else
- l_iov.virtual_address =
- (DAT_VADDR) (unsigned long)&rmr_recv_msg[i];
+ l_iov.virtual_address = (DAT_VADDR)(unsigned long)&rmr_recv_msg[ i ];
#endif
- l_iov.segment_length = sizeof(DAT_RMR_TRIPLET);
-
- LOGPRINTF("%d Posting Receive Message Buffer %p\n",
- getpid(), &rmr_recv_msg[i]);
- ret = dat_ep_post_recv(h_ep,
- 1,
- &l_iov,
- cookie, DAT_COMPLETION_DEFAULT_FLAG);
-
- if (ret != DAT_SUCCESS) {
- fprintf(stderr,
- "%d Error registering recv msg buffer: %s\n",
- getpid(), DT_RetToString(ret));
- return (ret);
- } else
- LOGPRINTF("%d Registered Receive Message Buffer %p\n",
- getpid(), region.for_va);
-
- }
-
- /* setup receive rdma buffer to initial string to be overwritten */
- strcpy((char *)rbuf, "blah, blah, blah\n");
-
- if (server) { /* SERVER */
-
- /* create the service point for server listen */
- LOGPRINTF("%d Creating service point for listen\n", getpid());
- ret = dat_psp_create(h_ia,
- conn_id,
- h_cr_evd, DAT_PSP_CONSUMER_FLAG, &h_psp);
- if (ret != DAT_SUCCESS) {
- fprintf(stderr, "%d Error dat_psp_create: %s\n",
- getpid(), DT_RetToString(ret));
- return (ret);
- } else
- LOGPRINTF("%d dat_psp_created for server listen\n",
- getpid());
-
- printf("%d Server waiting for connect request on port " F64x
- "\n", getpid(), conn_id);
-
- ret = dat_evd_wait(h_cr_evd, SERVER_TIMEOUT, 1, &event, &nmore);
- if (ret != DAT_SUCCESS) {
- fprintf(stderr, "%d Error dat_evd_wait: %s\n",
- getpid(), DT_RetToString(ret));
- return (ret);
- } else
- LOGPRINTF("%d dat_evd_wait for cr_evd completed\n",
- getpid());
-
- if (event.event_number != DAT_CONNECTION_REQUEST_EVENT) {
- fprintf(stderr, "%d Error unexpected cr event : %s\n",
- getpid(), DT_EventToSTr(event.event_number));
- return (DAT_ABORT);
- }
- if ((event.event_data.cr_arrival_event_data.conn_qual !=
- SERVER_CONN_QUAL)
- || (event.event_data.cr_arrival_event_data.sp_handle.
- psp_handle != h_psp)) {
- fprintf(stderr, "%d Error wrong cr event data : %s\n",
- getpid(), DT_EventToSTr(event.event_number));
- return (DAT_ABORT);
- }
-
- /* use to test rdma_cma timeout logic */
+ l_iov.segment_length = sizeof(DAT_RMR_TRIPLET);
+
+ LOGPRINTF("%d Posting Receive Message Buffer %p\n",
+ getpid(), &rmr_recv_msg[ i ]);
+ ret = dat_ep_post_recv( h_ep,
+ 1,
+ &l_iov,
+ cookie,
+ DAT_COMPLETION_DEFAULT_FLAG );
+
+ if(ret != DAT_SUCCESS) {
+ fprintf(stderr, "%d Error registering recv msg buffer: %s\n",
+ getpid(),DT_RetToString(ret));
+ return(ret);
+ }
+ else
+ LOGPRINTF("%d Registered Receive Message Buffer %p\n",
+ getpid(),region.for_va);
+
+ }
+
+ /* setup receive rdma buffer to initial string to be overwritten */
+ strcpy( (char*)rbuf, "blah, blah, blah\n" );
+
+ if ( server ) { /* SERVER */
+
+ /* create the service point for server listen */
+ LOGPRINTF("%d Creating service point for listen\n",getpid());
+ ret = dat_psp_create( h_ia,
+ conn_id,
+ h_cr_evd,
+ DAT_PSP_CONSUMER_FLAG,
+ &h_psp );
+ if(ret != DAT_SUCCESS) {
+ fprintf(stderr, "%d Error dat_psp_create: %s\n",
+ getpid(),DT_RetToString(ret));
+ return(ret);
+ }
+ else
+ LOGPRINTF("%d dat_psp_created for server listen\n", getpid());
+
+ printf("%d Server waiting for connect request on port "F64x"\n",
+ getpid(), conn_id);
+
+ ret = dat_evd_wait( h_cr_evd, SERVER_TIMEOUT, 1, &event, &nmore );
+ if(ret != DAT_SUCCESS) {
+ fprintf(stderr, "%d Error dat_evd_wait: %s\n",
+ getpid(),DT_RetToString(ret));
+ return(ret);
+ }
+ else
+ LOGPRINTF("%d dat_evd_wait for cr_evd completed\n", getpid());
+
+ if ( event.event_number != DAT_CONNECTION_REQUEST_EVENT ) {
+ fprintf(stderr, "%d Error unexpected cr event : %s\n",
+ getpid(),DT_EventToSTr(event.event_number));
+ return( DAT_ABORT );
+ }
+ if ( (event.event_data.cr_arrival_event_data.conn_qual != SERVER_CONN_QUAL) ||
+ (event.event_data.cr_arrival_event_data.sp_handle.psp_handle != h_psp) ) {
+ fprintf(stderr, "%d Error wrong cr event data : %s\n",
+ getpid(),DT_EventToSTr(event.event_number));
+ return( DAT_ABORT );
+ }
+
+ /* use to test rdma_cma timeout logic */
#if defined(_WIN32) || defined(_WIN64)
- if (delay)
- Sleep(delay * 1000);
+ if (delay) Sleep(delay*1000);
#else
- if (delay)
- sleep(delay);
+ if (delay) sleep(delay);
#endif
- /* accept connect request from client */
- h_cr = event.event_data.cr_arrival_event_data.cr_handle;
- LOGPRINTF("%d Accepting connect request from client\n",
- getpid());
-
- /* private data - check and send it back */
- dat_cr_query(h_cr, DAT_CSP_FIELD_ALL, &cr_param);
-
- buf = (unsigned char *)cr_param.private_data;
- LOGPRINTF("%d CONN REQUEST Private Data %p[0]=%d [47]=%d\n",
- getpid(), buf, buf[0], buf[47]);
- for (i = 0; i < 48; i++) {
- if (buf[i] != i + 1) {
- fprintf(stderr, "%d Error with CONNECT REQUEST"
- " private data: %p[%d]=%d s/be %d\n",
- getpid(), buf, i, buf[i], i + 1);
- dat_cr_reject(h_cr, 0, NULL);
- return (DAT_ABORT);
- }
- buf[i]++; /* change for trip back */
- }
+ /* accept connect request from client */
+ h_cr = event.event_data.cr_arrival_event_data.cr_handle;
+ LOGPRINTF("%d Accepting connect request from client\n",getpid());
+
+ /* private data - check and send it back */
+ dat_cr_query( h_cr, DAT_CSP_FIELD_ALL, &cr_param);
+
+ buf = (unsigned char*)cr_param.private_data;
+ LOGPRINTF("%d CONN REQUEST Private Data %p[0]=%d [47]=%d\n",
+ getpid(),buf,buf[0],buf[47]);
+ for (i=0;i<48;i++) {
+ if (buf[i] != i+1) {
+ fprintf(stderr, "%d Error with CONNECT REQUEST"
+ " private data: %p[%d]=%d s/be %d\n",
+ getpid(), buf, i, buf[i], i+1);
+ dat_cr_reject(h_cr, 0, NULL);
+ return(DAT_ABORT);
+ }
+ buf[i]++; /* change for trip back */
+ }
#ifdef TEST_REJECT_WITH_PRIVATE_DATA
- printf("%d REJECT request with 48 bytes of private data\n",
- getpid());
- ret = dat_cr_reject(h_cr, 48, cr_param.private_data);
- printf("\n%d: DAPL Test Complete. %s\n\n",
- getpid(), ret ? "FAILED" : "PASSED");
- exit(0);
+ printf("%d REJECT request with 48 bytes of private data\n", getpid());
+ ret = dat_cr_reject(h_cr, 48, cr_param.private_data);
+ printf("\n%d: DAPL Test Complete. %s\n\n",
+ getpid(), ret?"FAILED":"PASSED");
+ exit(0);
#endif
- ret = dat_cr_accept(h_cr, h_ep, 48, cr_param.private_data);
+ ret = dat_cr_accept(h_cr, h_ep, 48, cr_param.private_data);
- if (ret != DAT_SUCCESS) {
- fprintf(stderr, "%d Error dat_cr_accept: %s\n",
- getpid(), DT_RetToString(ret));
- return (ret);
- } else
- LOGPRINTF("%d dat_cr_accept completed\n", getpid());
- } else { /* CLIENT */
- struct addrinfo *target;
- int rval;
+ if(ret != DAT_SUCCESS) {
+ fprintf(stderr, "%d Error dat_cr_accept: %s\n",
+ getpid(),DT_RetToString(ret));
+ return(ret);
+ }
+ else
+ LOGPRINTF("%d dat_cr_accept completed\n", getpid());
+ }
+ else { /* CLIENT */
+ struct addrinfo *target;
+ int rval;
#if defined(_WIN32) || defined(_WIN64)
- if ((rval = getaddrinfo(hostname, "ftp", NULL, &target)) != 0) {
- printf("\n remote name resolution failed! %s\n",
- gai_strerror(rval));
- exit(1);
- }
- rval = ((struct sockaddr_in *)target->ai_addr)->sin_addr.s_addr;
+ if ((rval=getaddrinfo (hostname, "ftp", NULL, &target)) != 0) {
+ printf("\n remote name resolution failed! %s\n",gai_strerror(rval));
+ exit ( 1 );
+ }
+ rval = ((struct sockaddr_in *)target->ai_addr)->sin_addr.s_addr;
#else
- if (getaddrinfo(hostname, NULL, NULL, &target) != 0) {
- perror("\n remote name resolution failed!");
- exit(1);
- }
- rval = ((struct sockaddr_in *)target->ai_addr)->sin_addr.s_addr;
+ if (getaddrinfo (hostname, NULL, NULL, &target) != 0) {
+ perror("\n remote name resolution failed!");
+ exit ( 1 );
+ }
+ rval = ((struct sockaddr_in *)target->ai_addr)->sin_addr.s_addr;
#endif
- printf("%d Server Name: %s \n", getpid(), hostname);
- printf("%d Server Net Address: %d.%d.%d.%d port " F64x "\n",
- getpid(), (rval >> 0) & 0xff, (rval >> 8) & 0xff,
- (rval >> 16) & 0xff, (rval >> 24) & 0xff, conn_id);
-
- remote_addr = *((DAT_IA_ADDRESS_PTR) target->ai_addr);
- freeaddrinfo(target);
-
- for (i = 0; i < 48; i++) /* simple pattern in private data */
- pdata[i] = i + 1;
-
- LOGPRINTF("%d Connecting to server\n", getpid());
- ret = dat_ep_connect(h_ep,
- &remote_addr,
- conn_id,
- CONN_TIMEOUT,
- 48,
- (DAT_PVOID) pdata,
- 0, DAT_CONNECT_DEFAULT_FLAG);
- if (ret != DAT_SUCCESS) {
- fprintf(stderr, "%d Error dat_ep_connect: %s\n",
- getpid(), DT_RetToString(ret));
- return (ret);
- } else
- LOGPRINTF("%d dat_ep_connect completed\n", getpid());
- }
-
- printf("%d Waiting for connect response\n", getpid());
-
- ret = dat_evd_wait(h_conn_evd, DAT_TIMEOUT_INFINITE, 1, &event, &nmore);
- if (ret != DAT_SUCCESS) {
- fprintf(stderr, "%d Error dat_evd_wait: %s\n",
- getpid(), DT_RetToString(ret));
- return (ret);
- } else
- LOGPRINTF("%d dat_evd_wait for h_conn_evd completed\n",
- getpid());
+ printf ("%d Server Name: %s \n", getpid(), hostname);
+ printf ("%d Server Net Address: %d.%d.%d.%d port "F64x"\n", getpid(),
+ (rval >> 0) & 0xff, (rval >> 8) & 0xff,
+ (rval >> 16) & 0xff, (rval >> 24) & 0xff, conn_id);
+
+ remote_addr = *((DAT_IA_ADDRESS_PTR)target->ai_addr);
+ freeaddrinfo(target);
+
+ for (i=0;i<48;i++) /* simple pattern in private data */
+ pdata[i]=i+1;
+
+ LOGPRINTF("%d Connecting to server\n",getpid());
+ ret = dat_ep_connect( h_ep,
+ &remote_addr,
+ conn_id,
+ CONN_TIMEOUT,
+ 48,
+ (DAT_PVOID)pdata,
+ 0,
+ DAT_CONNECT_DEFAULT_FLAG );
+ if(ret != DAT_SUCCESS) {
+ fprintf(stderr, "%d Error dat_ep_connect: %s\n",
+ getpid(), DT_RetToString(ret));
+ return(ret);
+ }
+ else
+ LOGPRINTF("%d dat_ep_connect completed\n", getpid());
+ }
+
+ printf("%d Waiting for connect response\n",getpid());
+
+ ret = dat_evd_wait( h_conn_evd, DAT_TIMEOUT_INFINITE, 1, &event, &nmore );
+ if(ret != DAT_SUCCESS) {
+ fprintf(stderr, "%d Error dat_evd_wait: %s\n",
+ getpid(),DT_RetToString(ret));
+ return(ret);
+ }
+ else
+ LOGPRINTF("%d dat_evd_wait for h_conn_evd completed\n", getpid());
#ifdef TEST_REJECT_WITH_PRIVATE_DATA
- if (event.event_number != DAT_CONNECTION_EVENT_PEER_REJECTED) {
- fprintf(stderr, "%d expected conn reject event : %s\n",
- getpid(), DT_EventToSTr(event.event_number));
- return (DAT_ABORT);
- }
- /* get the reject private data and validate */
- buf = (unsigned char *)event.event_data.connect_event_data.private_data;
- printf("%d Received REJECT with private data %p[0]=%d [47]=%d\n",
- getpid(), buf, buf[0], buf[47]);
- for (i = 0; i < 48; i++) {
- if (buf[i] != i + 2) {
- fprintf(stderr, "%d client: Error with REJECT event"
- " private data: %p[%d]=%d s/be %d\n",
- getpid(), buf, i, buf[i], i + 2);
- dat_ep_disconnect(h_ep, DAT_CLOSE_ABRUPT_FLAG);
- return (DAT_ABORT);
- }
- }
- printf("\n%d: DAPL Test Complete. PASSED\n\n", getpid());
- exit(0);
+ if (event.event_number != DAT_CONNECTION_EVENT_PEER_REJECTED) {
+ fprintf(stderr, "%d expected conn reject event : %s\n",
+ getpid(),DT_EventToSTr(event.event_number));
+ return( DAT_ABORT );
+ }
+ /* get the reject private data and validate */
+ buf = (unsigned char*)event.event_data.connect_event_data.private_data;
+ printf("%d Received REJECT with private data %p[0]=%d [47]=%d\n",
+ getpid(),buf,buf[0],buf[47]);
+ for (i=0;i<48;i++) {
+ if (buf[i] != i+2) {
+ fprintf(stderr, "%d client: Error with REJECT event"
+ " private data: %p[%d]=%d s/be %d\n",
+ getpid(), buf, i, buf[i], i+2);
+ dat_ep_disconnect( h_ep, DAT_CLOSE_ABRUPT_FLAG);
+ return(DAT_ABORT);
+ }
+ }
+ printf("\n%d: DAPL Test Complete. PASSED\n\n", getpid());
+ exit(0);
#endif
- if (event.event_number != DAT_CONNECTION_EVENT_ESTABLISHED) {
- fprintf(stderr, "%d Error unexpected conn event : %s\n",
- getpid(), DT_EventToSTr(event.event_number));
- return (DAT_ABORT);
- }
-
- /* check private data back from server */
- if (!server) {
- buf =
- (unsigned char *)event.event_data.connect_event_data.
- private_data;
- LOGPRINTF("%d CONN Private Data %p[0]=%d [47]=%d\n", getpid(),
- buf, buf[0], buf[47]);
- for (i = 0; i < 48; i++) {
- if (buf[i] != i + 2) {
- fprintf(stderr, "%d Error with CONNECT event"
- " private data: %p[%d]=%d s/be %d\n",
- getpid(), buf, i, buf[i], i + 2);
- dat_ep_disconnect(h_ep, DAT_CLOSE_ABRUPT_FLAG);
- LOGPRINTF
- ("%d waiting for disconnect event...\n",
- getpid());
- dat_evd_wait(h_conn_evd, DAT_TIMEOUT_INFINITE,
- 1, &event, &nmore);
- return (DAT_ABORT);
- }
- }
- }
-
- printf("\n%d CONNECTED!\n\n", getpid());
- connected = 1;
+ if ( event.event_number != DAT_CONNECTION_EVENT_ESTABLISHED ) {
+ fprintf(stderr, "%d Error unexpected conn event : %s\n",
+ getpid(),DT_EventToSTr(event.event_number));
+ return( DAT_ABORT );
+ }
+
+ /* check private data back from server */
+ if (!server) {
+ buf = (unsigned char*)event.event_data.connect_event_data.private_data;
+ LOGPRINTF("%d CONN Private Data %p[0]=%d [47]=%d\n",
+ getpid(),buf,buf[0],buf[47]);
+ for (i=0;i<48;i++) {
+ if (buf[i] != i+2) {
+ fprintf(stderr, "%d Error with CONNECT event"
+ " private data: %p[%d]=%d s/be %d\n",
+ getpid(), buf, i, buf[i], i+2);
+ dat_ep_disconnect(h_ep, DAT_CLOSE_ABRUPT_FLAG);
+ LOGPRINTF("%d waiting for disconnect event...\n", getpid());
+ dat_evd_wait(h_conn_evd, DAT_TIMEOUT_INFINITE, 1, &event, &nmore);
+ return(DAT_ABORT);
+ }
+ }
+ }
+
+ printf("\n%d CONNECTED!\n\n",getpid());
+ connected = 1;
#if CONNECT_ONLY
- return 0;
+ return 0;
#endif
- /*
- * Setup our remote memory and tell the other side about it
- */
- rmr_send_msg.virtual_address = htonll((DAT_VADDR) (uintptr_t) rbuf);
- rmr_send_msg.segment_length = htonl(RDMA_BUFFER_SIZE);
- rmr_send_msg.rmr_context = htonl(rmr_context_recv);
-
- printf("%d Send RMR msg to remote: r_key_ctx=0x%x,va=%p,len=0x%x\n",
- getpid(), rmr_context_recv, rbuf, RDMA_BUFFER_SIZE);
-
- ret = send_msg(&rmr_send_msg,
- sizeof(DAT_RMR_TRIPLET),
- lmr_context_send_msg,
- cookie, DAT_COMPLETION_SUPPRESS_FLAG);
-
- if (ret != DAT_SUCCESS) {
- fprintf(stderr, "%d Error send_msg: %s\n",
- getpid(), DT_RetToString(ret));
- return (ret);
- } else
- LOGPRINTF("%d send_msg completed\n", getpid());
-
- /*
- * Wait for remote RMR information for RDMA
- */
- if (polling) {
- printf("%d Polling for remote to send RMR data\n", getpid());
- while (dat_evd_dequeue(h_dto_rcv_evd, &event) ==
- DAT_QUEUE_EMPTY) ;
- } else {
- printf("%d Waiting for remote to send RMR data\n", getpid());
- if (use_cno) {
- DAT_EVD_HANDLE evd = DAT_HANDLE_NULL;
- ret = dat_cno_wait(h_dto_cno, DTO_TIMEOUT, &evd);
- LOGPRINTF("%d cno wait return evd_handle=%p\n",
- getpid(), evd);
- if (evd != h_dto_rcv_evd) {
- fprintf(stderr,
- "%d Error waiting on h_dto_cno: evd != h_dto_rcv_evd\n",
- getpid());
- return (DAT_ABORT);
- }
- }
- /* use wait to dequeue */
- ret =
- dat_evd_wait(h_dto_rcv_evd, DTO_TIMEOUT, 1, &event, &nmore);
- if (ret != DAT_SUCCESS) {
- fprintf(stderr,
- "%d Error waiting on h_dto_rcv_evd: %s\n",
- getpid(), DT_RetToString(ret));
- return (ret);
- } else {
- LOGPRINTF("%d dat_evd_wait h_dto_rcv_evd completed\n",
- getpid());
- }
- }
-
- printf("%d remote RMR data arrived!\n", getpid());
-
- if (event.event_number != DAT_DTO_COMPLETION_EVENT) {
- fprintf(stderr, "%d Error unexpected DTO event : %s\n",
- getpid(), DT_EventToSTr(event.event_number));
- return (DAT_ABORT);
- }
- if ((event.event_data.dto_completion_event_data.transfered_length !=
- sizeof(DAT_RMR_TRIPLET)) ||
- (event.event_data.dto_completion_event_data.user_cookie.as_64 !=
- recv_msg_index)) {
- fprintf(stderr,
- "ERR recv event: len=%d cookie=" F64x
- " expected %d/%d\n",
- (int)event.event_data.dto_completion_event_data.
- transfered_length,
- event.event_data.dto_completion_event_data.user_cookie.
- as_64, (int)sizeof(DAT_RMR_TRIPLET), recv_msg_index);
- return (DAT_ABORT);
- }
-
- /* swap received RMR msg: network order to host order */
- r_iov = rmr_recv_msg[recv_msg_index];
- rmr_recv_msg[recv_msg_index].rmr_context = ntohl(r_iov.rmr_context);
- rmr_recv_msg[recv_msg_index].virtual_address =
+ /*
+ * Setup our remote memory and tell the other side about it
+ */
+ rmr_send_msg.virtual_address = htonll((DAT_VADDR)(uintptr_t)rbuf);
+ rmr_send_msg.segment_length = htonl(RDMA_BUFFER_SIZE);
+ rmr_send_msg.rmr_context = htonl(rmr_context_recv);
+
+ printf("%d Send RMR msg to remote: r_key_ctx=0x%x,va=%p,len=0x%x\n",
+ getpid(), rmr_context_recv, rbuf, RDMA_BUFFER_SIZE );
+
+ ret = send_msg(&rmr_send_msg,
+ sizeof( DAT_RMR_TRIPLET ),
+ lmr_context_send_msg,
+ cookie,
+ DAT_COMPLETION_SUPPRESS_FLAG);
+
+ if(ret != DAT_SUCCESS) {
+ fprintf(stderr, "%d Error send_msg: %s\n",
+ getpid(),DT_RetToString(ret));
+ return(ret);
+ }
+ else
+ LOGPRINTF("%d send_msg completed\n", getpid());
+
+ /*
+ * Wait for remote RMR information for RDMA
+ */
+ if ( polling ) {
+ printf("%d Polling for remote to send RMR data\n",getpid());
+ while ( dat_evd_dequeue( h_dto_rcv_evd, &event ) == DAT_QUEUE_EMPTY );
+ }
+ else {
+ printf("%d Waiting for remote to send RMR data\n",getpid());
+ if (use_cno)
+ {
+ DAT_EVD_HANDLE evd = DAT_HANDLE_NULL;
+ ret = dat_cno_wait( h_dto_cno, DTO_TIMEOUT, &evd );
+ LOGPRINTF("%d cno wait return evd_handle=%p\n", getpid(),evd);
+ if ( evd != h_dto_rcv_evd ) {
+ fprintf(stderr,
+ "%d Error waiting on h_dto_cno: evd != h_dto_rcv_evd\n",
+ getpid());
+ return( DAT_ABORT );
+ }
+ }
+ /* use wait to dequeue */
+ ret = dat_evd_wait( h_dto_rcv_evd, DTO_TIMEOUT, 1, &event, &nmore );
+ if(ret != DAT_SUCCESS) {
+ fprintf(stderr, "%d Error waiting on h_dto_rcv_evd: %s\n",
+ getpid(),DT_RetToString(ret));
+ return(ret);
+ }
+ else {
+ LOGPRINTF("%d dat_evd_wait h_dto_rcv_evd completed\n", getpid());
+ }
+ }
+
+ printf("%d remote RMR data arrived!\n",getpid());
+
+ if ( event.event_number != DAT_DTO_COMPLETION_EVENT ) {
+ fprintf(stderr, "%d Error unexpected DTO event : %s\n",
+ getpid(),DT_EventToSTr(event.event_number));
+ return( DAT_ABORT );
+ }
+ if ((event.event_data.dto_completion_event_data.transfered_length !=
+ sizeof( DAT_RMR_TRIPLET )) ||
+ (event.event_data.dto_completion_event_data.user_cookie.as_64 !=
+ recv_msg_index) ) {
+ fprintf(stderr,"ERR recv event: len=%d cookie="F64x" expected %d/%d\n",
+ (int)event.event_data.dto_completion_event_data.transfered_length,
+ event.event_data.dto_completion_event_data.user_cookie.as_64,
+ (int)sizeof(DAT_RMR_TRIPLET), recv_msg_index );
+ return( DAT_ABORT );
+ }
+
+ /* swap received RMR msg: network order to host order */
+ r_iov = rmr_recv_msg[recv_msg_index];
+ rmr_recv_msg[recv_msg_index].rmr_context =
+ ntohl(r_iov.rmr_context);
+ rmr_recv_msg[recv_msg_index].virtual_address =
ntohll(r_iov.virtual_address);
- rmr_recv_msg[recv_msg_index].segment_length =
+ rmr_recv_msg[recv_msg_index].segment_length =
ntohl(r_iov.segment_length);
- printf("%d Received RMR from remote: "
- "r_iov: r_key_ctx=%x,va=" F64x ",len=0x%x\n",
- getpid(), rmr_recv_msg[recv_msg_index].rmr_context,
- rmr_recv_msg[recv_msg_index].virtual_address,
- rmr_recv_msg[recv_msg_index].segment_length);
+ printf("%d Received RMR from remote: "
+ "r_iov: r_key_ctx=%x,va="F64x",len=0x%x\n",
+ getpid(), rmr_recv_msg[recv_msg_index].rmr_context,
+ rmr_recv_msg[recv_msg_index].virtual_address,
+ rmr_recv_msg[recv_msg_index].segment_length );
- recv_msg_index++;
+ recv_msg_index++;
- return (DAT_SUCCESS);
+ return ( DAT_SUCCESS );
}
-void disconnect_ep(void)
+void
+disconnect_ep( void )
{
- DAT_RETURN ret;
- DAT_EVENT event;
- DAT_COUNT nmore;
-
- if (connected) {
-
- /*
- * Only the client needs to call disconnect. The server _should_ be able
- * to just wait on the EVD associated with connection events for a
- * disconnect request and then exit.
- */
- if (!server) {
- LOGPRINTF("%d dat_ep_disconnect\n", getpid());
- ret = dat_ep_disconnect(h_ep, DAT_CLOSE_DEFAULT);
- if (ret != DAT_SUCCESS) {
- fprintf(stderr,
- "%d Error dat_ep_disconnect: %s\n",
- getpid(), DT_RetToString(ret));
- } else {
- LOGPRINTF("%d dat_ep_disconnect completed\n",
- getpid());
- }
- } else {
- LOGPRINTF("%d Server waiting for disconnect...\n",
- getpid());
- }
-
- ret =
- dat_evd_wait(h_conn_evd, DAT_TIMEOUT_INFINITE, 1, &event,
- &nmore);
- if (ret != DAT_SUCCESS) {
- fprintf(stderr, "%d Error dat_evd_wait: %s\n",
- getpid(), DT_RetToString(ret));
- } else {
- LOGPRINTF("%d dat_evd_wait for h_conn_evd completed\n",
- getpid());
- }
- }
-
- /* destroy service point */
- if ((server) && (h_psp != DAT_HANDLE_NULL)) {
- ret = dat_psp_free(h_psp);
- if (ret != DAT_SUCCESS) {
- fprintf(stderr, "%d Error dat_psp_free: %s\n",
- getpid(), DT_RetToString(ret));
- } else {
- LOGPRINTF("%d dat_psp_free completed\n", getpid());
- }
- }
-
- /* Unregister Send message Buffer */
- if (h_lmr_send_msg != DAT_HANDLE_NULL) {
- LOGPRINTF("%d Unregister send message h_lmr %p \n", getpid(),
- h_lmr_send_msg);
- ret = dat_lmr_free(h_lmr_send_msg);
- if (ret != DAT_SUCCESS) {
- fprintf(stderr,
- "%d Error deregistering send msg mr: %s\n",
- getpid(), DT_RetToString(ret));
- } else {
- LOGPRINTF("%d Unregistered send message Buffer\n",
- getpid());
- h_lmr_send_msg = NULL;
- }
- }
-
- /* Unregister recv message Buffer */
- if (h_lmr_recv_msg != DAT_HANDLE_NULL) {
- LOGPRINTF("%d Unregister recv message h_lmr %p \n", getpid(),
- h_lmr_recv_msg);
- ret = dat_lmr_free(h_lmr_recv_msg);
- if (ret != DAT_SUCCESS) {
- fprintf(stderr,
- "%d Error deregistering recv msg mr: %s\n",
- getpid(), DT_RetToString(ret));
- } else {
- LOGPRINTF("%d Unregistered recv message Buffer\n",
- getpid());
- h_lmr_recv_msg = NULL;
- }
- }
- return;
+ DAT_RETURN ret;
+ DAT_EVENT event;
+ DAT_COUNT nmore;
+
+ if (connected) {
+
+ /*
+ * Only the client needs to call disconnect. The server _should_ be able
+ * to just wait on the EVD associated with connection events for a
+ * disconnect request and then exit.
+ */
+ if ( !server ) {
+ LOGPRINTF("%d dat_ep_disconnect\n", getpid());
+ ret = dat_ep_disconnect( h_ep, DAT_CLOSE_DEFAULT );
+ if(ret != DAT_SUCCESS) {
+ fprintf(stderr, "%d Error dat_ep_disconnect: %s\n",
+ getpid(),DT_RetToString(ret));
+ }
+ else {
+ LOGPRINTF("%d dat_ep_disconnect completed\n", getpid());
+ }
+ }
+ else {
+ LOGPRINTF("%d Server waiting for disconnect...\n", getpid());
+ }
+
+ ret = dat_evd_wait(h_conn_evd, DAT_TIMEOUT_INFINITE, 1, &event, &nmore);
+ if(ret != DAT_SUCCESS) {
+ fprintf(stderr, "%d Error dat_evd_wait: %s\n",
+ getpid(),DT_RetToString(ret));
+ }
+ else {
+ LOGPRINTF("%d dat_evd_wait for h_conn_evd completed\n", getpid());
+ }
+ }
+
+ /* destroy service point */
+ if (( server ) && ( h_psp != DAT_HANDLE_NULL )) {
+ ret = dat_psp_free( h_psp );
+ if(ret != DAT_SUCCESS) {
+ fprintf(stderr, "%d Error dat_psp_free: %s\n",
+ getpid(),DT_RetToString(ret));
+ }
+ else {
+ LOGPRINTF("%d dat_psp_free completed\n", getpid());
+ }
+ }
+
+ /* Unregister Send message Buffer */
+ if ( h_lmr_send_msg != DAT_HANDLE_NULL ) {
+ LOGPRINTF("%d Unregister send message h_lmr %p \n",getpid(),h_lmr_send_msg);
+ ret = dat_lmr_free(h_lmr_send_msg);
+ if(ret != DAT_SUCCESS) {
+ fprintf(stderr, "%d Error deregistering send msg mr: %s\n",
+ getpid(), DT_RetToString(ret));
+ } else {
+ LOGPRINTF("%d Unregistered send message Buffer\n",getpid());
+ h_lmr_send_msg = NULL;
+ }
+ }
+
+ /* Unregister recv message Buffer */
+ if ( h_lmr_recv_msg != DAT_HANDLE_NULL ) {
+ LOGPRINTF("%d Unregister recv message h_lmr %p \n",getpid(),h_lmr_recv_msg);
+ ret = dat_lmr_free(h_lmr_recv_msg);
+ if(ret != DAT_SUCCESS) {
+ fprintf(stderr, "%d Error deregistering recv msg mr: %s\n",
+ getpid(), DT_RetToString(ret));
+ } else {
+ LOGPRINTF("%d Unregistered recv message Buffer\n",getpid());
+ h_lmr_recv_msg = NULL;
+ }
+ }
+ return;
}
-DAT_RETURN do_rdma_write_with_msg(void)
+
+DAT_RETURN
+do_rdma_write_with_msg( void )
{
- DAT_EVENT event;
- DAT_COUNT nmore;
- DAT_LMR_TRIPLET l_iov[MSG_IOV_COUNT];
- DAT_RMR_TRIPLET r_iov;
- DAT_DTO_COOKIE cookie;
- DAT_RETURN ret;
- int i;
-
- printf("\n %d RDMA WRITE DATA with SEND MSG\n\n", getpid());
-
- cookie.as_64 = 0x5555;
-
- if (recv_msg_index >= MSG_BUF_COUNT)
- return (DAT_ABORT);
-
- /* get RMR information from previously received message */
- r_iov = rmr_recv_msg[recv_msg_index - 1];
-
- if (server)
- strcpy((char *)sbuf, "server RDMA write data...");
- else
- strcpy((char *)sbuf, "client RDMA write data...");
-
- for (i = 0; i < MSG_IOV_COUNT; i++) {
- l_iov[i].lmr_context = lmr_context_send;
- l_iov[i].segment_length = buf_len / MSG_IOV_COUNT;
- l_iov[i].virtual_address = (DAT_VADDR) (uintptr_t)
- (&sbuf[l_iov[i].segment_length * i]);
-
- LOGPRINTF("%d rdma_write iov[%d] buf=%p,len=%d\n",
- getpid(), i, &sbuf[l_iov[i].segment_length * i],
- l_iov[i].segment_length);
- }
-
- start = get_time();
- for (i = 0; i < burst; i++) {
- cookie.as_64 = 0x9999;
- ret = dat_ep_post_rdma_write(h_ep, // ep_handle
- MSG_IOV_COUNT, // num_segments
- l_iov, // LMR
- cookie, // user_cookie
- &r_iov, // RMR
- DAT_COMPLETION_SUPPRESS_FLAG);
- if (ret != DAT_SUCCESS) {
- fprintf(stderr,
- "%d: ERROR: dat_ep_post_rdma_write() %s\n",
- getpid(), DT_RetToString(ret));
- return (DAT_ABORT);
- }
- LOGPRINTF("%d rdma_write # %d completed\n", getpid(), i + 1);
- }
-
- /*
- * Send RMR information a 2nd time to indicate completion
- * NOTE: already swapped to network order in connect_ep
- */
- printf("%d Sending RDMA WRITE completion message\n", getpid());
-
- ret = send_msg(&rmr_send_msg,
- sizeof(DAT_RMR_TRIPLET),
- lmr_context_send_msg,
- cookie, DAT_COMPLETION_SUPPRESS_FLAG);
-
- if (ret != DAT_SUCCESS) {
- fprintf(stderr, "%d Error send_msg: %s\n",
- getpid(), DT_RetToString(ret));
- return (ret);
- } else {
- LOGPRINTF("%d send_msg completed\n", getpid());
- }
-
- /*
- * Collect first event, write completion or the inbound recv
- */
- if (polling) {
- while (dat_evd_dequeue(h_dto_rcv_evd, &event) ==
- DAT_QUEUE_EMPTY)
- rdma_wr_poll_count++;
- } else {
- LOGPRINTF("%d waiting for message receive event\n", getpid());
- if (use_cno) {
- DAT_EVD_HANDLE evd = DAT_HANDLE_NULL;
- ret = dat_cno_wait(h_dto_cno, DTO_TIMEOUT, &evd);
- LOGPRINTF("%d cno wait return evd_handle=%p\n",
- getpid(), evd);
- if (evd != h_dto_rcv_evd) {
- fprintf(stderr,
- "%d Error waiting on h_dto_cno: "
- "evd != h_dto_rcv_evd\n", getpid());
- return (ret);
- }
- }
- /* use wait to dequeue */
- ret =
- dat_evd_wait(h_dto_rcv_evd, DTO_TIMEOUT, 1, &event, &nmore);
- if (ret != DAT_SUCCESS) {
- fprintf(stderr, "%d: ERROR: DTO dat_evd_wait() %s\n",
- getpid(), DT_RetToString(ret));
- return (ret);
- }
- }
- stop = get_time();
- time.rdma_wr = ((stop - start) * 1.0e6);
-
- /* validate event number and status */
- printf("%d inbound rdma_write; send message arrived!\n", getpid());
- if (event.event_number != DAT_DTO_COMPLETION_EVENT) {
- fprintf(stderr, "%d Error unexpected DTO event : %s\n",
- getpid(), DT_EventToSTr(event.event_number));
- return (DAT_ABORT);
- }
-
- if ((event.event_data.dto_completion_event_data.transfered_length !=
- sizeof(DAT_RMR_TRIPLET))
- || (event.event_data.dto_completion_event_data.user_cookie.as_64 !=
- recv_msg_index)) {
- fprintf(stderr,
- "unexpected event data for receive: len=%d cookie=" F64x
- " exp %d/%d\n",
- (int)event.event_data.dto_completion_event_data.
- transfered_length,
- event.event_data.dto_completion_event_data.user_cookie.
- as_64, (int)sizeof(DAT_RMR_TRIPLET), recv_msg_index);
-
- return (DAT_ABORT);
- }
-
- /* swap received RMR msg: network order to host order */
- r_iov = rmr_recv_msg[recv_msg_index];
- rmr_recv_msg[recv_msg_index].virtual_address =
- ntohll(rmr_recv_msg[recv_msg_index].virtual_address);
- rmr_recv_msg[recv_msg_index].segment_length =
- ntohl(rmr_recv_msg[recv_msg_index].segment_length);
- rmr_recv_msg[recv_msg_index].rmr_context =
- ntohl(rmr_recv_msg[recv_msg_index].rmr_context);
-
- printf("%d Received RMR from remote: "
- "r_iov: r_key_ctx=%x,va=" F64x ",len=0x%x\n",
- getpid(), rmr_recv_msg[recv_msg_index].rmr_context,
- rmr_recv_msg[recv_msg_index].virtual_address,
- rmr_recv_msg[recv_msg_index].segment_length);
-
- LOGPRINTF("%d inbound rdma_write; send msg event SUCCESS!!\n",
- getpid());
-
- printf("%d %s RDMA write buffer contains: %s\n",
- getpid(), server ? "SERVER:" : "CLIENT:", rbuf);
-
- recv_msg_index++;
-
- return (DAT_SUCCESS);
+ DAT_EVENT event;
+ DAT_COUNT nmore;
+ DAT_LMR_TRIPLET l_iov[MSG_IOV_COUNT];
+ DAT_RMR_TRIPLET r_iov;
+ DAT_DTO_COOKIE cookie;
+ DAT_RETURN ret;
+ int i;
+
+ printf("\n %d RDMA WRITE DATA with SEND MSG\n\n",getpid());
+
+ cookie.as_64 = 0x5555;
+
+ if ( recv_msg_index >= MSG_BUF_COUNT )
+ return( DAT_ABORT );
+
+ /* get RMR information from previously received message */
+ r_iov = rmr_recv_msg[ recv_msg_index-1 ];
+
+ if ( server )
+ strcpy( (char*)sbuf, "server RDMA write data..." );
+ else
+ strcpy( (char*)sbuf, "client RDMA write data..." );
+
+ for (i=0;i<MSG_IOV_COUNT;i++) {
+ l_iov[i].lmr_context = lmr_context_send;
+ l_iov[i].segment_length = buf_len/MSG_IOV_COUNT;
+ l_iov[i].virtual_address = (DAT_VADDR)(uintptr_t)
+ (&sbuf[l_iov[i].segment_length*i]);
+
+ LOGPRINTF("%d rdma_write iov[%d] buf=%p,len=%d\n",
+ getpid(), i, &sbuf[l_iov[i].segment_length*i],
+ l_iov[i].segment_length);
+ }
+
+ start = get_time();
+ for (i=0;i<burst;i++) {
+ cookie.as_64 = 0x9999;
+ ret = dat_ep_post_rdma_write( h_ep, // ep_handle
+ MSG_IOV_COUNT, // num_segments
+ l_iov, // LMR
+ cookie, // user_cookie
+ &r_iov, // RMR
+ DAT_COMPLETION_SUPPRESS_FLAG );
+ if (ret != DAT_SUCCESS) {
+ fprintf(stderr, "%d: ERROR: dat_ep_post_rdma_write() %s\n",
+ getpid(),DT_RetToString(ret));
+ return( DAT_ABORT );
+ }
+ LOGPRINTF("%d rdma_write # %d completed\n", getpid(),i+1);
+ }
+
+ /*
+ * Send RMR information a 2nd time to indicate completion
+ * NOTE: already swapped to network order in connect_ep
+ */
+ printf("%d Sending RDMA WRITE completion message\n",getpid());
+
+ ret = send_msg( &rmr_send_msg,
+ sizeof( DAT_RMR_TRIPLET ),
+ lmr_context_send_msg,
+ cookie,
+ DAT_COMPLETION_SUPPRESS_FLAG );
+
+ if(ret != DAT_SUCCESS) {
+ fprintf(stderr, "%d Error send_msg: %s\n",
+ getpid(),DT_RetToString(ret));
+ return(ret);
+ } else {
+ LOGPRINTF("%d send_msg completed\n", getpid());
+ }
+
+ /*
+ * Collect first event, write completion or the inbound recv
+ */
+ if (polling) {
+ while (dat_evd_dequeue(h_dto_rcv_evd, &event) == DAT_QUEUE_EMPTY)
+ rdma_wr_poll_count++;
+ }
+ else {
+ LOGPRINTF("%d waiting for message receive event\n", getpid());
+ if (use_cno) {
+ DAT_EVD_HANDLE evd = DAT_HANDLE_NULL;
+ ret = dat_cno_wait( h_dto_cno, DTO_TIMEOUT, &evd );
+ LOGPRINTF("%d cno wait return evd_handle=%p\n",
+ getpid(),evd);
+ if ( evd != h_dto_rcv_evd ) {
+ fprintf(stderr, "%d Error waiting on h_dto_cno: "
+ "evd != h_dto_rcv_evd\n",
+ getpid());
+ return( ret );
+ }
+ }
+ /* use wait to dequeue */
+ ret = dat_evd_wait(h_dto_rcv_evd, DTO_TIMEOUT, 1, &event, &nmore);
+ if (ret != DAT_SUCCESS) {
+ fprintf(stderr, "%d: ERROR: DTO dat_evd_wait() %s\n",
+ getpid(),DT_RetToString(ret));
+ return( ret );
+ }
+ }
+ stop = get_time();
+ time.rdma_wr = ((stop - start)*1.0e6);
+
+ /* validate event number and status */
+ printf("%d inbound rdma_write; send message arrived!\n",getpid());
+ if ( event.event_number != DAT_DTO_COMPLETION_EVENT ) {
+ fprintf(stderr, "%d Error unexpected DTO event : %s\n",
+ getpid(),DT_EventToSTr(event.event_number));
+ return( DAT_ABORT );
+ }
+
+ if ( (event.event_data.dto_completion_event_data.transfered_length != sizeof( DAT_RMR_TRIPLET )) ||
+ (event.event_data.dto_completion_event_data.user_cookie.as_64 != recv_msg_index) ) {
+ fprintf(stderr,"unexpected event data for receive: len=%d cookie="F64x" exp %d/%d\n",
+ (int)event.event_data.dto_completion_event_data.transfered_length,
+ event.event_data.dto_completion_event_data.user_cookie.as_64,
+ (int)sizeof(DAT_RMR_TRIPLET), recv_msg_index );
+
+ return( DAT_ABORT );
+ }
+
+ /* swap received RMR msg: network order to host order */
+ r_iov = rmr_recv_msg[recv_msg_index];
+ rmr_recv_msg[recv_msg_index].virtual_address =
+ ntohll(rmr_recv_msg[recv_msg_index].virtual_address);
+ rmr_recv_msg[recv_msg_index].segment_length =
+ ntohl(rmr_recv_msg[recv_msg_index].segment_length);
+ rmr_recv_msg[recv_msg_index].rmr_context =
+ ntohl(rmr_recv_msg[recv_msg_index].rmr_context);
+
+ printf("%d Received RMR from remote: "
+ "r_iov: r_key_ctx=%x,va="F64x",len=0x%x\n",
+ getpid(), rmr_recv_msg[recv_msg_index].rmr_context,
+ rmr_recv_msg[recv_msg_index].virtual_address,
+ rmr_recv_msg[recv_msg_index].segment_length);
+
+ LOGPRINTF("%d inbound rdma_write; send msg event SUCCESS!!\n",
+ getpid());
+
+ printf("%d %s RDMA write buffer contains: %s\n",
+ getpid(),
+ server ? "SERVER:" : "CLIENT:",
+ rbuf );
+
+ recv_msg_index++;
+
+ return ( DAT_SUCCESS );
}
-DAT_RETURN do_rdma_read_with_msg(void)
+DAT_RETURN
+do_rdma_read_with_msg( void )
{
- DAT_EVENT event;
- DAT_COUNT nmore;
- DAT_LMR_TRIPLET l_iov;
- DAT_RMR_TRIPLET r_iov;
- DAT_DTO_COOKIE cookie;
- DAT_RETURN ret;
- int i;
-
- printf("\n %d RDMA READ DATA with SEND MSG\n\n", getpid());
-
- if (recv_msg_index >= MSG_BUF_COUNT)
- return (DAT_ABORT);
-
- /* get RMR information from previously received message */
- r_iov = rmr_recv_msg[recv_msg_index - 1];
-
- /* setup rdma read buffer to initial string to be overwritten */
- strcpy((char *)sbuf, "blah, blah, blah\n");
-
- if (server)
- strcpy((char *)rbuf, "server RDMA read data...");
- else
- strcpy((char *)rbuf, "client RDMA read data...");
-
- l_iov.lmr_context = lmr_context_send;
- l_iov.virtual_address = (DAT_VADDR) (uintptr_t) sbuf;
- l_iov.segment_length = buf_len;
-
- for (i = 0; i < MAX_RDMA_RD; i++) {
- cookie.as_64 = 0x9999;
- start = get_time();
- ret = dat_ep_post_rdma_read(h_ep, // ep_handle
- 1, // num_segments
- &l_iov, // LMR
- cookie, // user_cookie
- &r_iov, // RMR
- DAT_COMPLETION_DEFAULT_FLAG);
- if (ret != DAT_SUCCESS) {
- fprintf(stderr,
- "%d: ERROR: dat_ep_post_rdma_read() %s\n",
- getpid(), DT_RetToString(ret));
- return (DAT_ABORT);
- }
-
- if (polling) {
- while (dat_evd_dequeue(h_dto_req_evd, &event) ==
- DAT_QUEUE_EMPTY)
- rdma_rd_poll_count[i]++;
- } else {
- LOGPRINTF("%d waiting for rdma_read completion event\n",
- getpid());
- if (use_cno) {
- DAT_EVD_HANDLE evd = DAT_HANDLE_NULL;
- ret =
- dat_cno_wait(h_dto_cno, DTO_TIMEOUT, &evd);
- LOGPRINTF("%d cno wait return evd_handle=%p\n",
- getpid(), evd);
- if (evd != h_dto_req_evd) {
- fprintf(stderr,
- "%d Error waiting on h_dto_cno: evd != h_dto_req_evd\n",
- getpid());
- return (DAT_ABORT);
- }
- }
- /* use wait to dequeue */
- ret =
- dat_evd_wait(h_dto_req_evd, DTO_TIMEOUT, 1, &event,
- &nmore);
- if (ret != DAT_SUCCESS) {
- fprintf(stderr,
- "%d: ERROR: DTO dat_evd_wait() %s\n",
- getpid(), DT_RetToString(ret));
- return ret;
- }
- }
- /* validate event number, len, cookie, and status */
- if (event.event_number != DAT_DTO_COMPLETION_EVENT) {
- fprintf(stderr, "%d: ERROR: DTO event number %s\n",
- getpid(), DT_EventToSTr(event.event_number));
- return (DAT_ABORT);
- }
- if ((event.event_data.dto_completion_event_data.
- transfered_length != buf_len)
- || (event.event_data.dto_completion_event_data.user_cookie.
- as_64 != 0x9999)) {
- fprintf(stderr,
- "%d: ERROR: DTO len %d or cookie " F64x "\n",
- getpid(),
- event.event_data.dto_completion_event_data.
- transfered_length,
- event.event_data.dto_completion_event_data.
- user_cookie.as_64);
- return (DAT_ABORT);
- }
- if (event.event_data.dto_completion_event_data.status !=
- DAT_SUCCESS) {
- fprintf(stderr, "%d: ERROR: DTO event status %s\n",
- getpid(), DT_RetToString(ret));
- return (DAT_ABORT);
- }
- stop = get_time();
- time.rdma_rd[i] = ((stop - start) * 1.0e6);
- time.rdma_rd_total += time.rdma_rd[i];
-
- LOGPRINTF("%d rdma_read # %d completed\n", getpid(), i + 1);
- }
-
- /*
- * Send RMR information a 3rd time to indicate completion
- * NOTE: already swapped to network order in connect_ep
- */
- printf("%d Sending RDMA read completion message\n", getpid());
-
- ret = send_msg(&rmr_send_msg,
- sizeof(DAT_RMR_TRIPLET),
- lmr_context_send_msg,
- cookie, DAT_COMPLETION_SUPPRESS_FLAG);
-
- if (ret != DAT_SUCCESS) {
- fprintf(stderr, "%d Error send_msg: %s\n",
- getpid(), DT_RetToString(ret));
- return (ret);
- } else {
- LOGPRINTF("%d send_msg completed\n", getpid());
- }
-
- /*
- * Collect first event, write completion or the inbound recv with immed
- */
- printf("%d Waiting for inbound message....\n", getpid());
- if (polling) {
- while (dat_evd_dequeue(h_dto_rcv_evd, &event) ==
- DAT_QUEUE_EMPTY) ;
- } else {
- LOGPRINTF("%d waiting for message receive event\n", getpid());
+ DAT_EVENT event;
+ DAT_COUNT nmore;
+ DAT_LMR_TRIPLET l_iov;
+ DAT_RMR_TRIPLET r_iov;
+ DAT_DTO_COOKIE cookie;
+ DAT_RETURN ret;
+ int i;
+
+ printf("\n %d RDMA READ DATA with SEND MSG\n\n",getpid());
+
+ if (recv_msg_index >= MSG_BUF_COUNT)
+ return( DAT_ABORT );
+
+ /* get RMR information from previously received message */
+ r_iov = rmr_recv_msg[recv_msg_index-1];
+
+ /* setup rdma read buffer to initial string to be overwritten */
+ strcpy((char*)sbuf, "blah, blah, blah\n");
+
+ if (server)
+ strcpy((char*)rbuf, "server RDMA read data...");
+ else
+ strcpy((char*)rbuf, "client RDMA read data...");
+
+ l_iov.lmr_context = lmr_context_send;
+ l_iov.virtual_address = (DAT_VADDR)(uintptr_t)sbuf;
+ l_iov.segment_length = buf_len;
+
+ for (i=0;i<MAX_RDMA_RD;i++) {
+ cookie.as_64 = 0x9999;
+ start = get_time();
+ ret = dat_ep_post_rdma_read(h_ep, // ep_handle
+ 1, // num_segments
+ &l_iov, // LMR
+ cookie, // user_cookie
+ &r_iov, // RMR
+ DAT_COMPLETION_DEFAULT_FLAG );
+ if (ret != DAT_SUCCESS) {
+ fprintf(stderr, "%d: ERROR: dat_ep_post_rdma_read() %s\n",
+ getpid(),DT_RetToString(ret));
+ return(DAT_ABORT);
+ }
+
+ if (polling) {
+ while (dat_evd_dequeue(h_dto_req_evd, &event) == DAT_QUEUE_EMPTY)
+ rdma_rd_poll_count[i]++;
+ }
+ else {
+ LOGPRINTF("%d waiting for rdma_read completion event\n", getpid());
if (use_cno) {
DAT_EVD_HANDLE evd = DAT_HANDLE_NULL;
- ret = dat_cno_wait(h_dto_cno, DTO_TIMEOUT, &evd);
- LOGPRINTF("%d cno wait return evd_handle=%p\n",
- getpid(), evd);
- if (evd != h_dto_rcv_evd) {
- fprintf(stderr,
- "%d Error waiting on h_dto_cno: evd != h_dto_rcv_evd\n",
- getpid());
- return (ret);
+ ret = dat_cno_wait( h_dto_cno, DTO_TIMEOUT, &evd );
+ LOGPRINTF("%d cno wait return evd_handle=%p\n", getpid(),evd);
+ if ( evd != h_dto_req_evd ) {
+ fprintf(stderr,
+ "%d Error waiting on h_dto_cno: evd != h_dto_req_evd\n",
+ getpid());
+ return( DAT_ABORT );
}
}
/* use wait to dequeue */
- ret =
- dat_evd_wait(h_dto_rcv_evd, DTO_TIMEOUT, 1, &event, &nmore);
+ ret = dat_evd_wait( h_dto_req_evd, DTO_TIMEOUT, 1, &event, &nmore );
if (ret != DAT_SUCCESS) {
- fprintf(stderr, "%d: ERROR: DTO dat_evd_wait() %s\n",
- getpid(), DT_RetToString(ret));
- return (ret);
+ fprintf(stderr, "%d: ERROR: DTO dat_evd_wait() %s\n",
+ getpid(),DT_RetToString(ret));
+ return ret;
}
- }
-
- /* validate event number and status */
- printf("%d inbound rdma_read; send message arrived!\n", getpid());
- if (event.event_number != DAT_DTO_COMPLETION_EVENT) {
- fprintf(stderr, "%d Error unexpected DTO event : %s\n",
- getpid(), DT_EventToSTr(event.event_number));
- return (DAT_ABORT);
- }
-
- if ((event.event_data.dto_completion_event_data.transfered_length !=
- sizeof(DAT_RMR_TRIPLET))
- || (event.event_data.dto_completion_event_data.user_cookie.as_64 !=
- recv_msg_index)) {
-
- fprintf(stderr,
- "unexpected event data for receive: len=%d cookie=" F64x
- " exp %d/%d\n",
- (int)event.event_data.dto_completion_event_data.
- transfered_length,
- event.event_data.dto_completion_event_data.user_cookie.
- as_64, (int)sizeof(DAT_RMR_TRIPLET), recv_msg_index);
-
- return (DAT_ABORT);
- }
-
- /* swap received RMR msg: network order to host order */
- r_iov = rmr_recv_msg[recv_msg_index];
- rmr_recv_msg[recv_msg_index].virtual_address =
- ntohll(rmr_recv_msg[recv_msg_index].virtual_address);
- rmr_recv_msg[recv_msg_index].segment_length =
- ntohl(rmr_recv_msg[recv_msg_index].segment_length);
- rmr_recv_msg[recv_msg_index].rmr_context =
- ntohl(rmr_recv_msg[recv_msg_index].rmr_context);
-
- printf("%d Received RMR from remote: "
- "r_iov: r_key_ctx=%x,va=" F64x ",len=0x%x\n",
- getpid(), rmr_recv_msg[recv_msg_index].rmr_context,
- rmr_recv_msg[recv_msg_index].virtual_address,
- rmr_recv_msg[recv_msg_index].segment_length);
-
- LOGPRINTF("%d inbound rdma_write; send msg event SUCCESS!!\n",
- getpid());
-
- printf("%d %s RCV RDMA read buffer contains: %s\n",
- getpid(), server ? "SERVER:" : "CLIENT:", sbuf);
-
- recv_msg_index++;
-
- return (DAT_SUCCESS);
+ }
+ /* validate event number, len, cookie, and status */
+ if (event.event_number != DAT_DTO_COMPLETION_EVENT) {
+ fprintf(stderr, "%d: ERROR: DTO event number %s\n",
+ getpid(),DT_EventToSTr(event.event_number));
+ return( DAT_ABORT );
+ }
+ if ((event.event_data.dto_completion_event_data.transfered_length != buf_len ) ||
+ (event.event_data.dto_completion_event_data.user_cookie.as_64 != 0x9999 )) {
+ fprintf(stderr, "%d: ERROR: DTO len %d or cookie "F64x"\n",
+ getpid(),
+ event.event_data.dto_completion_event_data.transfered_length,
+ event.event_data.dto_completion_event_data.user_cookie.as_64 );
+ return( DAT_ABORT );
+ }
+ if (event.event_data.dto_completion_event_data.status != DAT_SUCCESS) {
+ fprintf(stderr, "%d: ERROR: DTO event status %s\n",
+ getpid(),DT_RetToString(ret));
+ return( DAT_ABORT );
+ }
+ stop = get_time();
+ time.rdma_rd[i] = ((stop - start)*1.0e6);
+ time.rdma_rd_total += time.rdma_rd[i];
+
+ LOGPRINTF("%d rdma_read # %d completed\n", getpid(),i+1);
+ }
+
+ /*
+ * Send RMR information a 3rd time to indicate completion
+ * NOTE: already swapped to network order in connect_ep
+ */
+ printf("%d Sending RDMA read completion message\n",getpid());
+
+ ret = send_msg( &rmr_send_msg,
+ sizeof( DAT_RMR_TRIPLET ),
+ lmr_context_send_msg,
+ cookie,
+ DAT_COMPLETION_SUPPRESS_FLAG );
+
+ if(ret != DAT_SUCCESS) {
+ fprintf(stderr, "%d Error send_msg: %s\n",
+ getpid(),DT_RetToString(ret));
+ return(ret);
+ } else {
+ LOGPRINTF("%d send_msg completed\n", getpid());
+ }
+
+ /*
+ * Collect first event, write completion or the inbound recv with immed
+ */
+ printf("%d Waiting for inbound message....\n",getpid());
+ if ( polling ) {
+ while ( dat_evd_dequeue( h_dto_rcv_evd, &event ) == DAT_QUEUE_EMPTY );
+ }
+ else {
+ LOGPRINTF("%d waiting for message receive event\n", getpid());
+ if (use_cno) {
+ DAT_EVD_HANDLE evd = DAT_HANDLE_NULL;
+ ret = dat_cno_wait( h_dto_cno, DTO_TIMEOUT, &evd );
+ LOGPRINTF("%d cno wait return evd_handle=%p\n", getpid(),evd);
+ if ( evd != h_dto_rcv_evd ) {
+ fprintf(stderr,
+ "%d Error waiting on h_dto_cno: evd != h_dto_rcv_evd\n",
+ getpid());
+ return( ret );
+ }
+ }
+ /* use wait to dequeue */
+ ret = dat_evd_wait( h_dto_rcv_evd, DTO_TIMEOUT, 1, &event, &nmore );
+ if (ret != DAT_SUCCESS) {
+ fprintf(stderr, "%d: ERROR: DTO dat_evd_wait() %s\n",
+ getpid(),DT_RetToString(ret));
+ return( ret );
+ }
+ }
+
+ /* validate event number and status */
+ printf("%d inbound rdma_read; send message arrived!\n",getpid());
+ if (event.event_number != DAT_DTO_COMPLETION_EVENT ) {
+ fprintf(stderr, "%d Error unexpected DTO event : %s\n",
+ getpid(),DT_EventToSTr(event.event_number));
+ return( DAT_ABORT );
+ }
+
+ if ((event.event_data.dto_completion_event_data.transfered_length != sizeof( DAT_RMR_TRIPLET)) ||
+ (event.event_data.dto_completion_event_data.user_cookie.as_64 != recv_msg_index)) {
+
+ fprintf(stderr,"unexpected event data for receive: len=%d cookie="F64x" exp %d/%d\n",
+ (int)event.event_data.dto_completion_event_data.transfered_length,
+ event.event_data.dto_completion_event_data.user_cookie.as_64,
+ (int)sizeof(DAT_RMR_TRIPLET), recv_msg_index );
+
+ return( DAT_ABORT );
+ }
+
+ /* swap received RMR msg: network order to host order */
+ r_iov = rmr_recv_msg[recv_msg_index];
+ rmr_recv_msg[recv_msg_index].virtual_address =
+ ntohll(rmr_recv_msg[recv_msg_index].virtual_address);
+ rmr_recv_msg[recv_msg_index].segment_length =
+ ntohl(rmr_recv_msg[ recv_msg_index].segment_length);
+ rmr_recv_msg[recv_msg_index].rmr_context =
+ ntohl(rmr_recv_msg[recv_msg_index].rmr_context);
+
+ printf("%d Received RMR from remote: "
+ "r_iov: r_key_ctx=%x,va="F64x",len=0x%x\n",
+ getpid(), rmr_recv_msg[recv_msg_index].rmr_context,
+ rmr_recv_msg[recv_msg_index].virtual_address,
+ rmr_recv_msg[recv_msg_index].segment_length);
+
+ LOGPRINTF("%d inbound rdma_write; send msg event SUCCESS!!\n",getpid());
+
+ printf("%d %s RCV RDMA read buffer contains: %s\n",
+ getpid(),
+ server ? "SERVER:" : "CLIENT:",
+ sbuf );
+
+ recv_msg_index++;
+
+ return ( DAT_SUCCESS );
}
-DAT_RETURN do_ping_pong_msg()
-{
- DAT_EVENT event;
- DAT_COUNT nmore;
- DAT_DTO_COOKIE cookie;
- DAT_LMR_TRIPLET l_iov;
- DAT_RETURN ret;
- int i;
- char *snd_buf;
- char *rcv_buf;
-
- printf("\n %d PING DATA with SEND MSG\n\n", getpid());
-
- snd_buf = sbuf;
- rcv_buf = rbuf;
-
- /* pre-post all buffers */
- for (i = 0; i < burst; i++) {
- burst_msg_posted++;
- cookie.as_64 = i;
- l_iov.lmr_context = lmr_context_recv;
- l_iov.virtual_address = (DAT_VADDR) (uintptr_t) rcv_buf;
- l_iov.segment_length = buf_len;
-
- LOGPRINTF("%d Pre-posting Receive Message Buffers %p\n",
- getpid(), rcv_buf);
-
- ret = dat_ep_post_recv(h_ep,
- 1,
- &l_iov,
- cookie, DAT_COMPLETION_DEFAULT_FLAG);
- if (ret != DAT_SUCCESS) {
- fprintf(stderr,
- "%d Error posting recv msg buffer: %s\n",
- getpid(), DT_RetToString(ret));
- return (ret);
- } else {
- LOGPRINTF("%d Posted Receive Message Buffer %p\n",
- getpid(), rcv_buf);
- }
-
- /* next buffer */
- rcv_buf += buf_len;
- }
+DAT_RETURN
+do_ping_pong_msg( )
+{
+ DAT_EVENT event;
+ DAT_COUNT nmore;
+ DAT_DTO_COOKIE cookie;
+ DAT_LMR_TRIPLET l_iov;
+ DAT_RETURN ret;
+ int i;
+ char *snd_buf;
+ char *rcv_buf;
+
+ printf("\n %d PING DATA with SEND MSG\n\n",getpid());
+
+ snd_buf = sbuf;
+ rcv_buf = rbuf;
+
+ /* pre-post all buffers */
+ for ( i=0; i < burst; i++ ) {
+ burst_msg_posted++;
+ cookie.as_64 = i;
+ l_iov.lmr_context = lmr_context_recv;
+ l_iov.virtual_address = (DAT_VADDR)(uintptr_t)rcv_buf;
+ l_iov.segment_length = buf_len;
+
+ LOGPRINTF("%d Pre-posting Receive Message Buffers %p\n",
+ getpid(), rcv_buf );
+
+ ret = dat_ep_post_recv( h_ep,
+ 1,
+ &l_iov,
+ cookie,
+ DAT_COMPLETION_DEFAULT_FLAG );
+
+ if(ret != DAT_SUCCESS) {
+ fprintf(stderr, "%d Error posting recv msg buffer: %s\n",
+ getpid(),DT_RetToString(ret));
+ return(ret);
+ }
+ else {
+ LOGPRINTF("%d Posted Receive Message Buffer %p\n",getpid(),rcv_buf);
+ }
+
+ /* next buffer */
+ rcv_buf += buf_len;
+ }
#if defined(_WIN32) || defined(_WIN64)
- Sleep(1000);
+ Sleep(1000);
#else
- sleep(1);
+ sleep(1);
#endif
- /* Initialize recv_buf and index to beginning */
- rcv_buf = rbuf;
- burst_msg_index = 0;
-
- /* client ping 0x55, server pong 0xAA in first byte */
- start = get_time();
- for (i = 0; i < burst; i++) {
- /* walk the send and recv buffers */
- if (!server) {
- *snd_buf = 0x55;
-
- LOGPRINTF("%d %s SND buffer %p contains: 0x%x len=%d\n",
- getpid(), server ? "SERVER:" : "CLIENT:",
- snd_buf, *snd_buf, buf_len);
-
- ret = send_msg(snd_buf,
- buf_len,
- lmr_context_send,
- cookie, DAT_COMPLETION_SUPPRESS_FLAG);
-
- if (ret != DAT_SUCCESS) {
- fprintf(stderr, "%d Error send_msg: %s\n",
- getpid(), DT_RetToString(ret));
- return (ret);
- } else {
- LOGPRINTF("%d send_msg completed\n", getpid());
- }
- }
-
- /* Wait for recv message */
- if (polling) {
- poll_count = 0;
- LOGPRINTF("%d Polling for message receive event\n",
- getpid());
- while (dat_evd_dequeue(h_dto_rcv_evd, &event) ==
- DAT_QUEUE_EMPTY)
- poll_count++;
- } else {
- LOGPRINTF("%d waiting for message receive event\n",
- getpid());
- if (use_cno) {
- DAT_EVD_HANDLE evd = DAT_HANDLE_NULL;
- ret =
- dat_cno_wait(h_dto_cno, DTO_TIMEOUT, &evd);
- LOGPRINTF("%d cno wait return evd_handle=%p\n",
- getpid(), evd);
- if (evd != h_dto_rcv_evd) {
- fprintf(stderr,
- "%d Error waiting on h_dto_cno: evd != h_dto_rcv_evd\n",
- getpid());
- return (ret);
- }
- }
- /* use wait to dequeue */
- ret =
- dat_evd_wait(h_dto_rcv_evd, DTO_TIMEOUT, 1, &event,
- &nmore);
- if (ret != DAT_SUCCESS) {
- fprintf(stderr,
- "%d: ERROR: DTO dat_evd_wait() %s\n",
- getpid(), DT_RetToString(ret));
- return (ret);
- }
- }
- /* start timer after first message arrives on server */
- if (i == 0) {
- start = get_time();
- }
- /* validate event number and status */
- LOGPRINTF("%d inbound message; message arrived!\n", getpid());
- if (event.event_number != DAT_DTO_COMPLETION_EVENT) {
- fprintf(stderr, "%d Error unexpected DTO event : %s\n",
- getpid(), DT_EventToSTr(event.event_number));
- return (DAT_ABORT);
- }
- if ((event.event_data.dto_completion_event_data.
- transfered_length != buf_len)
- || (event.event_data.dto_completion_event_data.user_cookie.
- as_64 != burst_msg_index)) {
- fprintf(stderr,
- "ERR: recv event: len=%d cookie=" F64x
- " exp %d/%d\n",
- (int)event.event_data.dto_completion_event_data.
- transfered_length,
- event.event_data.dto_completion_event_data.
- user_cookie.as_64, (int)buf_len,
- (int)burst_msg_index);
-
- return (DAT_ABORT);
- }
-
- LOGPRINTF("%d %s RCV buffer %p contains: 0x%x len=%d\n",
- getpid(), server ? "SERVER:" : "CLIENT:",
- rcv_buf, *rcv_buf, buf_len);
-
- burst_msg_index++;
-
- /* If server, change data and send it back to client */
- if (server) {
- *snd_buf = 0xaa;
-
- LOGPRINTF("%d %s SND buffer %p contains: 0x%x len=%d\n",
- getpid(), server ? "SERVER:" : "CLIENT:",
- snd_buf, *snd_buf, buf_len);
-
- ret = send_msg(snd_buf,
- buf_len,
- lmr_context_send,
- cookie, DAT_COMPLETION_SUPPRESS_FLAG);
-
- if (ret != DAT_SUCCESS) {
- fprintf(stderr, "%d Error send_msg: %s\n",
- getpid(), DT_RetToString(ret));
- return (ret);
- } else {
- LOGPRINTF("%d send_msg completed\n", getpid());
- }
- }
-
- /* next buffers */
- rcv_buf += buf_len;
- snd_buf += buf_len;
- }
- stop = get_time();
- time.rtt = ((stop - start) * 1.0e6);
-
- return (DAT_SUCCESS);
+ /* Initialize recv_buf and index to beginning */
+ rcv_buf = rbuf;
+ burst_msg_index=0;
+
+ /* client ping 0x55, server pong 0xAA in first byte */
+ start = get_time();
+ for ( i=0;i<burst;i++ ) {
+ /* walk the send and recv buffers */
+ if ( !server ) {
+ *snd_buf = 0x55;
+
+ LOGPRINTF("%d %s SND buffer %p contains: 0x%x len=%d\n",
+ getpid(), server ? "SERVER:" : "CLIENT:",
+ snd_buf, *snd_buf, buf_len );
+
+ ret = send_msg( snd_buf,
+ buf_len,
+ lmr_context_send,
+ cookie,
+ DAT_COMPLETION_SUPPRESS_FLAG );
+
+ if(ret != DAT_SUCCESS) {
+ fprintf(stderr, "%d Error send_msg: %s\n",
+ getpid(),DT_RetToString(ret));
+ return(ret);
+ }
+ else {
+ LOGPRINTF("%d send_msg completed\n", getpid());
+ }
+ }
+
+ /* Wait for recv message */
+ if ( polling ) {
+ poll_count=0;
+ LOGPRINTF("%d Polling for message receive event\n", getpid());
+ while ( dat_evd_dequeue( h_dto_rcv_evd, &event ) == DAT_QUEUE_EMPTY )
+ poll_count++;
+ }
+ else {
+ LOGPRINTF("%d waiting for message receive event\n", getpid());
+ if (use_cno) {
+ DAT_EVD_HANDLE evd = DAT_HANDLE_NULL;
+ ret = dat_cno_wait( h_dto_cno, DTO_TIMEOUT, &evd );
+ LOGPRINTF("%d cno wait return evd_handle=%p\n", getpid(),evd);
+ if ( evd != h_dto_rcv_evd )
+ {
+ fprintf(stderr,
+ "%d Error waiting on h_dto_cno: evd != h_dto_rcv_evd\n",
+ getpid());
+ return( ret );
+ }
+ }
+ /* use wait to dequeue */
+ ret = dat_evd_wait( h_dto_rcv_evd, DTO_TIMEOUT, 1, &event, &nmore );
+ if (ret != DAT_SUCCESS) {
+ fprintf(stderr, "%d: ERROR: DTO dat_evd_wait() %s\n",
+ getpid(),DT_RetToString(ret));
+ return( ret );
+ }
+ }
+ /* start timer after first message arrives on server */
+ if ( i == 0) {
+ start = get_time();
+ }
+ /* validate event number and status */
+ LOGPRINTF("%d inbound message; message arrived!\n",getpid());
+ if ( event.event_number != DAT_DTO_COMPLETION_EVENT ) {
+ fprintf(stderr, "%d Error unexpected DTO event : %s\n",
+ getpid(),DT_EventToSTr(event.event_number));
+ return( DAT_ABORT );
+ }
+ if ((event.event_data.dto_completion_event_data.transfered_length
+ != buf_len) ||
+ (event.event_data.dto_completion_event_data.user_cookie.as_64
+ != burst_msg_index) ) {
+ fprintf(stderr,"ERR: recv event: len=%d cookie="F64x" exp %d/%d\n",
+ (int)event.event_data.dto_completion_event_data.transfered_length,
+ event.event_data.dto_completion_event_data.user_cookie.as_64,
+ (int)buf_len, (int)burst_msg_index );
+
+ return( DAT_ABORT );
+ }
+
+ LOGPRINTF("%d %s RCV buffer %p contains: 0x%x len=%d\n",
+ getpid(), server ? "SERVER:" : "CLIENT:",
+ rcv_buf, *rcv_buf, buf_len );
+
+ burst_msg_index++;
+
+ /* If server, change data and send it back to client */
+ if ( server ) {
+ *snd_buf = 0xaa;
+
+ LOGPRINTF("%d %s SND buffer %p contains: 0x%x len=%d\n",
+ getpid(), server ? "SERVER:" : "CLIENT:",
+ snd_buf, *snd_buf, buf_len );
+
+ ret = send_msg( snd_buf,
+ buf_len,
+ lmr_context_send,
+ cookie,
+ DAT_COMPLETION_SUPPRESS_FLAG );
+
+ if(ret != DAT_SUCCESS) {
+ fprintf(stderr, "%d Error send_msg: %s\n",
+ getpid(),DT_RetToString(ret));
+ return(ret);
+ }
+ else {
+ LOGPRINTF("%d send_msg completed\n", getpid());
+ }
+ }
+
+ /* next buffers */
+ rcv_buf += buf_len;
+ snd_buf += buf_len;
+ }
+ stop = get_time();
+ time.rtt = ((stop - start)*1.0e6);
+
+ return ( DAT_SUCCESS );
}
/* Register RDMA Receive buffer */
-DAT_RETURN register_rdma_memory(void)
+DAT_RETURN
+register_rdma_memory(void)
{
- DAT_RETURN ret;
- DAT_REGION_DESCRIPTION region;
-
- region.for_va = rbuf;
- start = get_time();
- ret = dat_lmr_create(h_ia,
- DAT_MEM_TYPE_VIRTUAL,
- region,
- buf_len * (burst+1),
- h_pz,
- DAT_MEM_PRIV_ALL_FLAG,
- DAT_VA_TYPE_VA,
- &h_lmr_recv,
- &lmr_context_recv,
- &rmr_context_recv,
- ®istered_size_recv, ®istered_addr_recv);
- stop = get_time();
- time.reg += ((stop - start) * 1.0e6);
- time.total += time.reg;
-
- if (ret != DAT_SUCCESS) {
- fprintf(stderr,
- "%d Error registering Receive RDMA buffer: %s\n",
- getpid(), DT_RetToString(ret));
- return (ret);
- } else {
- LOGPRINTF("%d Registered Receive RDMA Buffer %p\n",
- getpid(), region.for_va);
- }
-
- /* Register RDMA Send buffer */
- region.for_va = sbuf;
- ret = dat_lmr_create(h_ia,
- DAT_MEM_TYPE_VIRTUAL,
- region,
- buf_len * (burst + 1),
- h_pz,
- DAT_MEM_PRIV_ALL_FLAG,
- DAT_VA_TYPE_VA,
- &h_lmr_send,
- &lmr_context_send,
- &rmr_context_send,
- ®istered_size_send, ®istered_addr_send);
- if (ret != DAT_SUCCESS) {
- fprintf(stderr, "%d Error registering send RDMA buffer: %s\n",
- getpid(), DT_RetToString(ret));
- return (ret);
- } else {
- LOGPRINTF("%d Registered Send RDMA Buffer %p\n",
- getpid(), region.for_va);
- }
-
- return DAT_SUCCESS;
+ DAT_RETURN ret;
+ DAT_REGION_DESCRIPTION region;
+
+ region.for_va = rbuf;
+ start = get_time();
+ ret = dat_lmr_create( h_ia,
+ DAT_MEM_TYPE_VIRTUAL,
+ region,
+ buf_len*burst,
+ h_pz,
+ DAT_MEM_PRIV_ALL_FLAG,
+ DAT_VA_TYPE_VA,
+ &h_lmr_recv,
+ &lmr_context_recv,
+ &rmr_context_recv,
+ ®istered_size_recv,
+ ®istered_addr_recv );
+ stop = get_time();
+ time.reg += ((stop - start)*1.0e6);
+ time.total += time.reg;
+
+ if(ret != DAT_SUCCESS) {
+ fprintf(stderr, "%d Error registering Receive RDMA buffer: %s\n",
+ getpid(),DT_RetToString(ret));
+ return (ret);
+ } else {
+ LOGPRINTF("%d Registered Receive RDMA Buffer %p\n",
+ getpid(),region.for_va);
+ }
+
+ /* Register RDMA Send buffer */
+ region.for_va = sbuf;
+ ret = dat_lmr_create( h_ia,
+ DAT_MEM_TYPE_VIRTUAL,
+ region,
+ buf_len*burst,
+ h_pz,
+ DAT_MEM_PRIV_ALL_FLAG,
+ DAT_VA_TYPE_VA,
+ &h_lmr_send,
+ &lmr_context_send,
+ &rmr_context_send,
+ ®istered_size_send,
+ ®istered_addr_send );
+ if(ret != DAT_SUCCESS) {
+ fprintf(stderr, "%d Error registering send RDMA buffer: %s\n",
+ getpid(),DT_RetToString(ret));
+ return (ret);
+ } else {
+ LOGPRINTF("%d Registered Send RDMA Buffer %p\n",
+ getpid(),region.for_va);
+ }
+
+ return DAT_SUCCESS;
}
/*
* Unregister RDMA memory
*/
-DAT_RETURN unregister_rdma_memory(void)
+DAT_RETURN
+unregister_rdma_memory(void)
{
- DAT_RETURN ret;
-
- /* Unregister Recv Buffer */
- if (h_lmr_recv != DAT_HANDLE_NULL) {
- LOGPRINTF("%d Unregister h_lmr %p \n", getpid(), h_lmr_recv);
- start = get_time();
- ret = dat_lmr_free(h_lmr_recv);
- stop = get_time();
- time.unreg += ((stop - start) * 1.0e6);
- time.total += time.unreg;
- if (ret != DAT_SUCCESS) {
- fprintf(stderr, "%d Error deregistering recv mr: %s\n",
- getpid(), DT_RetToString(ret));
- return (ret);
- } else {
- LOGPRINTF("%d Unregistered Recv Buffer\n", getpid());
- h_lmr_recv = NULL;
- }
- }
-
- /* Unregister Send Buffer */
- if (h_lmr_send != DAT_HANDLE_NULL) {
- LOGPRINTF("%d Unregister h_lmr %p \n", getpid(), h_lmr_send);
- ret = dat_lmr_free(h_lmr_send);
- if (ret != DAT_SUCCESS) {
- fprintf(stderr, "%d Error deregistering send mr: %s\n",
- getpid(), DT_RetToString(ret));
- return (ret);
- } else {
- LOGPRINTF("%d Unregistered send Buffer\n", getpid());
- h_lmr_send = NULL;
- }
- }
- return DAT_SUCCESS;
+ DAT_RETURN ret;
+
+ /* Unregister Recv Buffer */
+ if ( h_lmr_recv != DAT_HANDLE_NULL )
+ {
+ LOGPRINTF("%d Unregister h_lmr %p \n",getpid(),h_lmr_recv);
+ start = get_time();
+ ret = dat_lmr_free(h_lmr_recv);
+ stop = get_time();
+ time.unreg += ((stop - start)*1.0e6);
+ time.total += time.unreg;
+ if(ret != DAT_SUCCESS) {
+ fprintf(stderr, "%d Error deregistering recv mr: %s\n",
+ getpid(), DT_RetToString(ret));
+ return (ret);
+ }
+ else {
+ LOGPRINTF("%d Unregistered Recv Buffer\n",getpid());
+ h_lmr_recv = NULL;
+ }
+ }
+
+ /* Unregister Send Buffer */
+ if ( h_lmr_send != DAT_HANDLE_NULL ) {
+ LOGPRINTF("%d Unregister h_lmr %p \n",getpid(),h_lmr_send);
+ ret = dat_lmr_free(h_lmr_send);
+ if(ret != DAT_SUCCESS) {
+ fprintf(stderr, "%d Error deregistering send mr: %s\n",
+ getpid(), DT_RetToString(ret));
+ return (ret);
+ }
+ else {
+ LOGPRINTF("%d Unregistered send Buffer\n",getpid());
+ h_lmr_send = NULL;
+ }
+ }
+ return DAT_SUCCESS;
}
/*
* Create CNO, CR, CONN, and DTO events
*/
-DAT_RETURN create_events(void)
+DAT_RETURN
+create_events(void)
{
- DAT_RETURN ret;
- DAT_EVD_PARAM param;
+ DAT_RETURN ret;
+ DAT_EVD_PARAM param;
- /* create CNO */
- if (use_cno) {
- start = get_time();
+ /* create CNO */
+ if (use_cno) {
+ start = get_time();
#if defined(_WIN32) || defined(_WIN64)
- {
- DAT_OS_WAIT_PROXY_AGENT pa = { NULL, NULL };
- ret = dat_cno_create(h_ia, pa, &h_dto_cno);
- }
+ { DAT_OS_WAIT_PROXY_AGENT pa = {NULL, NULL};
+ ret = dat_cno_create(h_ia, pa, &h_dto_cno);
+ }
#else
- ret =
- dat_cno_create(h_ia, DAT_OS_WAIT_PROXY_AGENT_NULL,
- &h_dto_cno);
+ ret = dat_cno_create(h_ia, DAT_OS_WAIT_PROXY_AGENT_NULL, &h_dto_cno);
#endif
- stop = get_time();
- time.cnoc += ((stop - start) * 1.0e6);
- time.total += time.cnoc;
- if (ret != DAT_SUCCESS) {
- fprintf(stderr, "%d Error dat_cno_create: %s\n",
- getpid(), DT_RetToString(ret));
- return (ret);
- } else {
- LOGPRINTF("%d cr_evd created, %p\n", getpid(),
- h_dto_cno);
- }
- }
-
- /* create cr EVD */
- start = get_time();
- ret =
- dat_evd_create(h_ia, 10, DAT_HANDLE_NULL, DAT_EVD_CR_FLAG,
- &h_cr_evd);
- stop = get_time();
- time.evdc += ((stop - start) * 1.0e6);
- time.total += time.evdc;
- if (ret != DAT_SUCCESS) {
- fprintf(stderr, "%d Error dat_evd_create: %s\n",
- getpid(), DT_RetToString(ret));
- return (ret);
- } else {
- LOGPRINTF("%d cr_evd created %p\n", getpid(), h_cr_evd);
- }
-
- /* create conn EVD */
- ret = dat_evd_create(h_ia,
- 10,
- DAT_HANDLE_NULL,
- DAT_EVD_CONNECTION_FLAG, &h_conn_evd);
- if (ret != DAT_SUCCESS) {
- fprintf(stderr, "%d Error dat_evd_create: %s\n",
- getpid(), DT_RetToString(ret));
- return (ret);
- } else {
- LOGPRINTF("%d con_evd created %p\n", getpid(), h_conn_evd);
- }
-
- /* create dto SND EVD, with CNO if use_cno was set */
- ret = dat_evd_create(h_ia,
- (MSG_BUF_COUNT + MAX_RDMA_RD + burst) * 2,
- h_dto_cno, DAT_EVD_DTO_FLAG, &h_dto_req_evd);
- if (ret != DAT_SUCCESS) {
- fprintf(stderr, "%d Error dat_evd_create REQ: %s\n",
- getpid(), DT_RetToString(ret));
- return (ret);
- } else {
- LOGPRINTF("%d dto_req_evd created %p\n", getpid(),
- h_dto_req_evd);
- }
-
- /* create dto RCV EVD, with CNO if use_cno was set */
- ret = dat_evd_create(h_ia,
- MSG_BUF_COUNT + burst,
- h_dto_cno, DAT_EVD_DTO_FLAG, &h_dto_rcv_evd);
- if (ret != DAT_SUCCESS) {
- fprintf(stderr, "%d Error dat_evd_create RCV: %s\n",
- getpid(), DT_RetToString(ret));
- return (ret);
- } else {
- LOGPRINTF("%d dto_rcv_evd created %p\n", getpid(),
- h_dto_rcv_evd);
- }
-
- /* query DTO req EVD and check size */
- ret = dat_evd_query(h_dto_req_evd, DAT_EVD_FIELD_EVD_QLEN, ¶m);
- if (ret != DAT_SUCCESS) {
- fprintf(stderr, "%d Error dat_evd_query request evd: %s\n",
- getpid(), DT_RetToString(ret));
- return (ret);
- } else if (param.evd_qlen < (MSG_BUF_COUNT + MAX_RDMA_RD + burst) * 2) {
- fprintf(stderr, "%d Error dat_evd qsize too small: %d < %d\n",
- getpid(), param.evd_qlen,
- (MSG_BUF_COUNT + MAX_RDMA_RD + burst) * 2);
- return (ret);
- }
-
- LOGPRINTF("%d dto_req_evd QLEN - requested %d and actual %d\n",
- getpid(), (MSG_BUF_COUNT + MAX_RDMA_RD + burst) * 2,
- param.evd_qlen);
-
- return DAT_SUCCESS;
+ stop = get_time();
+ time.cnoc += ((stop - start)*1.0e6);
+ time.total += time.cnoc;
+ if(ret != DAT_SUCCESS) {
+ fprintf(stderr, "%d Error dat_cno_create: %s\n",
+ getpid(),DT_RetToString(ret));
+ return (ret);
+ }
+ else {
+ LOGPRINTF("%d cr_evd created, %p\n", getpid(), h_dto_cno);
+ }
+ }
+
+ /* create cr EVD */
+ start = get_time();
+ ret = dat_evd_create(h_ia, 10, DAT_HANDLE_NULL, DAT_EVD_CR_FLAG, &h_cr_evd);
+ stop = get_time();
+ time.evdc += ((stop - start)*1.0e6);
+ time.total += time.evdc;
+ if(ret != DAT_SUCCESS) {
+ fprintf(stderr, "%d Error dat_evd_create: %s\n",
+ getpid(),DT_RetToString(ret));
+ return (ret);
+ }
+ else {
+ LOGPRINTF("%d cr_evd created %p\n", getpid(),h_cr_evd);
+ }
+
+ /* create conn EVD */
+ ret = dat_evd_create( h_ia,
+ 10,
+ DAT_HANDLE_NULL,
+ DAT_EVD_CONNECTION_FLAG,
+ &h_conn_evd );
+ if(ret != DAT_SUCCESS) {
+ fprintf(stderr, "%d Error dat_evd_create: %s\n",
+ getpid(),DT_RetToString(ret));
+ return (ret);
+ }
+ else {
+ LOGPRINTF("%d con_evd created %p\n", getpid(),h_conn_evd);
+ }
+
+ /* create dto SND EVD, with CNO if use_cno was set */
+ ret = dat_evd_create( h_ia,
+ (MSG_BUF_COUNT+MAX_RDMA_RD+burst)*2,
+ h_dto_cno,
+ DAT_EVD_DTO_FLAG,
+ &h_dto_req_evd );
+ if(ret != DAT_SUCCESS) {
+ fprintf(stderr, "%d Error dat_evd_create REQ: %s\n",
+ getpid(),DT_RetToString(ret));
+ return (ret);
+ }
+ else {
+ LOGPRINTF("%d dto_req_evd created %p\n", getpid(), h_dto_req_evd );
+ }
+
+ /* create dto RCV EVD, with CNO if use_cno was set */
+ ret = dat_evd_create( h_ia,
+ MSG_BUF_COUNT,
+ h_dto_cno,
+ DAT_EVD_DTO_FLAG,
+ &h_dto_rcv_evd );
+ if(ret != DAT_SUCCESS) {
+ fprintf(stderr, "%d Error dat_evd_create RCV: %s\n",
+ getpid(),DT_RetToString(ret));
+ return (ret);
+ }
+ else {
+ LOGPRINTF("%d dto_rcv_evd created %p\n", getpid(), h_dto_rcv_evd );
+ }
+
+ /* query DTO req EVD and check size */
+ ret = dat_evd_query( h_dto_req_evd,
+ DAT_EVD_FIELD_EVD_QLEN,
+ ¶m );
+ if(ret != DAT_SUCCESS) {
+ fprintf(stderr, "%d Error dat_evd_query request evd: %s\n",
+ getpid(),DT_RetToString(ret));
+ return (ret);
+ }
+ else if (param.evd_qlen < (MSG_BUF_COUNT+MAX_RDMA_RD+burst)*2) {
+ fprintf(stderr, "%d Error dat_evd qsize too small: %d < %d\n",
+ getpid(), param.evd_qlen,
+ (MSG_BUF_COUNT+MAX_RDMA_RD+burst)*2);
+ return (ret);
+ }
+
+ LOGPRINTF("%d dto_req_evd QLEN - requested %d and actual %d\n",
+ getpid(), (MSG_BUF_COUNT+MAX_RDMA_RD+burst)*2,
+ param.evd_qlen);
+
+ return DAT_SUCCESS;
}
/*
* Destroy CR, CONN, CNO, and DTO events
*/
-DAT_RETURN destroy_events(void)
+DAT_RETURN
+destroy_events(void)
{
- DAT_RETURN ret;
-
- /* free cr EVD */
- if (h_cr_evd != DAT_HANDLE_NULL) {
- LOGPRINTF("%d Free cr EVD %p \n", getpid(), h_cr_evd);
- ret = dat_evd_free(h_cr_evd);
- if (ret != DAT_SUCCESS) {
- fprintf(stderr, "%d Error freeing cr EVD: %s\n",
- getpid(), DT_RetToString(ret));
- return (ret);
- } else {
- LOGPRINTF("%d Freed cr EVD\n", getpid());
- h_cr_evd = DAT_HANDLE_NULL;
- }
- }
-
- /* free conn EVD */
- if (h_conn_evd != DAT_HANDLE_NULL) {
- LOGPRINTF("%d Free conn EVD %p \n", getpid(), h_conn_evd);
- ret = dat_evd_free(h_conn_evd);
- if (ret != DAT_SUCCESS) {
- fprintf(stderr, "%d Error freeing conn EVD: %s\n",
- getpid(), DT_RetToString(ret));
- return (ret);
- } else {
- LOGPRINTF("%d Freed conn EVD\n", getpid());
- h_conn_evd = DAT_HANDLE_NULL;
- }
- }
-
- /* free RCV dto EVD */
- if (h_dto_rcv_evd != DAT_HANDLE_NULL) {
- LOGPRINTF("%d Free RCV dto EVD %p \n", getpid(), h_dto_rcv_evd);
- start = get_time();
- ret = dat_evd_free(h_dto_rcv_evd);
- stop = get_time();
- time.evdf += ((stop - start) * 1.0e6);
- time.total += time.evdf;
- if (ret != DAT_SUCCESS) {
- fprintf(stderr, "%d Error freeing dto EVD: %s\n",
- getpid(), DT_RetToString(ret));
- return (ret);
- } else {
- LOGPRINTF("%d Freed dto EVD\n", getpid());
- h_dto_rcv_evd = DAT_HANDLE_NULL;
- }
- }
-
- /* free REQ dto EVD */
- if (h_dto_req_evd != DAT_HANDLE_NULL) {
- LOGPRINTF("%d Free REQ dto EVD %p \n", getpid(), h_dto_req_evd);
- ret = dat_evd_free(h_dto_req_evd);
- if (ret != DAT_SUCCESS) {
- fprintf(stderr, "%d Error freeing dto EVD: %s\n",
- getpid(), DT_RetToString(ret));
- return (ret);
- } else {
- LOGPRINTF("%d Freed dto EVD\n", getpid());
- h_dto_req_evd = DAT_HANDLE_NULL;
- }
- }
-
- /* free CNO */
- if (h_dto_cno != DAT_HANDLE_NULL) {
- LOGPRINTF("%d Free dto CNO %p \n", getpid(), h_dto_cno);
- start = get_time();
- ret = dat_cno_free(h_dto_cno);
- stop = get_time();
- time.cnof += ((stop - start) * 1.0e6);
- time.total += time.cnof;
- if (ret != DAT_SUCCESS) {
- fprintf(stderr, "%d Error freeing dto CNO: %s\n",
- getpid(), DT_RetToString(ret));
- return (ret);
- } else {
- LOGPRINTF("%d Freed dto CNO\n", getpid());
- h_dto_cno = DAT_HANDLE_NULL;
- }
- }
- return DAT_SUCCESS;
+ DAT_RETURN ret;
+
+ /* free cr EVD */
+ if ( h_cr_evd != DAT_HANDLE_NULL ) {
+ LOGPRINTF("%d Free cr EVD %p \n",getpid(),h_cr_evd);
+ ret = dat_evd_free( h_cr_evd );
+ if(ret != DAT_SUCCESS) {
+ fprintf(stderr, "%d Error freeing cr EVD: %s\n",
+ getpid(), DT_RetToString(ret));
+ return (ret);
+ } else {
+ LOGPRINTF("%d Freed cr EVD\n",getpid());
+ h_cr_evd = DAT_HANDLE_NULL;
+ }
+ }
+
+ /* free conn EVD */
+ if ( h_conn_evd != DAT_HANDLE_NULL ) {
+ LOGPRINTF("%d Free conn EVD %p \n",getpid(),h_conn_evd);
+ ret = dat_evd_free( h_conn_evd );
+ if(ret != DAT_SUCCESS) {
+ fprintf(stderr, "%d Error freeing conn EVD: %s\n",
+ getpid(), DT_RetToString(ret));
+ return (ret);
+ }
+ else {
+ LOGPRINTF("%d Freed conn EVD\n",getpid());
+ h_conn_evd = DAT_HANDLE_NULL;
+ }
+ }
+
+ /* free RCV dto EVD */
+ if ( h_dto_rcv_evd != DAT_HANDLE_NULL ) {
+ LOGPRINTF("%d Free RCV dto EVD %p \n",getpid(),h_dto_rcv_evd);
+ start = get_time();
+ ret = dat_evd_free( h_dto_rcv_evd );
+ stop = get_time();
+ time.evdf += ((stop - start)*1.0e6);
+ time.total += time.evdf;
+ if(ret != DAT_SUCCESS) {
+ fprintf(stderr, "%d Error freeing dto EVD: %s\n",
+ getpid(), DT_RetToString(ret));
+ return (ret);
+ }
+ else {
+ LOGPRINTF("%d Freed dto EVD\n",getpid());
+ h_dto_rcv_evd = DAT_HANDLE_NULL;
+ }
+ }
+
+ /* free REQ dto EVD */
+ if ( h_dto_req_evd != DAT_HANDLE_NULL ) {
+ LOGPRINTF("%d Free REQ dto EVD %p \n",getpid(),h_dto_req_evd);
+ ret = dat_evd_free( h_dto_req_evd );
+ if(ret != DAT_SUCCESS) {
+ fprintf(stderr, "%d Error freeing dto EVD: %s\n",
+ getpid(), DT_RetToString(ret));
+ return (ret);
+ }
+ else {
+ LOGPRINTF("%d Freed dto EVD\n",getpid());
+ h_dto_req_evd = DAT_HANDLE_NULL;
+ }
+ }
+
+ /* free CNO */
+ if ( h_dto_cno != DAT_HANDLE_NULL ) {
+ LOGPRINTF("%d Free dto CNO %p \n",getpid(),h_dto_cno);
+ start = get_time();
+ ret = dat_cno_free( h_dto_cno );
+ stop = get_time();
+ time.cnof += ((stop - start)*1.0e6);
+ time.total += time.cnof;
+ if(ret != DAT_SUCCESS) {
+ fprintf(stderr, "%d Error freeing dto CNO: %s\n",
+ getpid(), DT_RetToString(ret));
+ return (ret);
+ }
+ else {
+ LOGPRINTF("%d Freed dto CNO\n",getpid());
+ h_dto_cno = DAT_HANDLE_NULL;
+ }
+ }
+ return DAT_SUCCESS;
}
/*
* Map DAT_RETURN values to readable strings,
* but don't assume the values are zero-based or contiguous.
*/
-char errmsg[512] = { 0 };
-const char *DT_RetToString(DAT_RETURN ret_value)
+char errmsg[512] = {0};
+const char *
+DT_RetToString (DAT_RETURN ret_value)
{
- const char *major_msg, *minor_msg;
+ const char *major_msg, *minor_msg;
- dat_strerror(ret_value, &major_msg, &minor_msg);
+ dat_strerror (ret_value, &major_msg, &minor_msg);
- strcpy(errmsg, major_msg);
- strcat(errmsg, " ");
- strcat(errmsg, minor_msg);
+ strcpy(errmsg, major_msg);
+ strcat(errmsg, " ");
+ strcat(errmsg, minor_msg);
- return errmsg;
+ return errmsg;
}
/*
* Map DAT_EVENT_CODE values to readable strings
*/
-const char *DT_EventToSTr(DAT_EVENT_NUMBER event_code)
+const char *
+DT_EventToSTr (DAT_EVENT_NUMBER event_code)
{
- unsigned int i;
- static struct {
- const char *name;
- DAT_RETURN value;
- } dat_events[] = {
-# define DATxx(x) { # x, x }
- DATxx(DAT_DTO_COMPLETION_EVENT),
- DATxx(DAT_RMR_BIND_COMPLETION_EVENT),
- DATxx(DAT_CONNECTION_REQUEST_EVENT),
- DATxx(DAT_CONNECTION_EVENT_ESTABLISHED),
- DATxx(DAT_CONNECTION_EVENT_PEER_REJECTED),
- DATxx(DAT_CONNECTION_EVENT_NON_PEER_REJECTED),
- DATxx(DAT_CONNECTION_EVENT_ACCEPT_COMPLETION_ERROR),
- DATxx(DAT_CONNECTION_EVENT_DISCONNECTED),
- DATxx(DAT_CONNECTION_EVENT_BROKEN),
- DATxx(DAT_CONNECTION_EVENT_TIMED_OUT),
- DATxx(DAT_CONNECTION_EVENT_UNREACHABLE),
- DATxx(DAT_ASYNC_ERROR_EVD_OVERFLOW),
- DATxx(DAT_ASYNC_ERROR_IA_CATASTROPHIC),
- DATxx(DAT_ASYNC_ERROR_EP_BROKEN),
- DATxx(DAT_ASYNC_ERROR_TIMED_OUT),
- DATxx(DAT_ASYNC_ERROR_PROVIDER_INTERNAL_ERROR),
- DATxx(DAT_SOFTWARE_EVENT)
-# undef DATxx
- };
-# define NUM_EVENTS (sizeof(dat_events)/sizeof(dat_events[0]))
-
- for (i = 0; i < NUM_EVENTS; i++) {
- if (dat_events[i].value == event_code) {
- return (dat_events[i].name);
- }
- }
-
- return ("Invalid_DAT_EVENT_NUMBER");
+ unsigned int i;
+ static struct {
+ const char *name;
+ DAT_RETURN value;
+ }
+ dat_events[] =
+ {
+ # define DATxx(x) { # x, x }
+ DATxx (DAT_DTO_COMPLETION_EVENT),
+ DATxx (DAT_RMR_BIND_COMPLETION_EVENT),
+ DATxx (DAT_CONNECTION_REQUEST_EVENT),
+ DATxx (DAT_CONNECTION_EVENT_ESTABLISHED),
+ DATxx (DAT_CONNECTION_EVENT_PEER_REJECTED),
+ DATxx (DAT_CONNECTION_EVENT_NON_PEER_REJECTED),
+ DATxx (DAT_CONNECTION_EVENT_ACCEPT_COMPLETION_ERROR),
+ DATxx (DAT_CONNECTION_EVENT_DISCONNECTED),
+ DATxx (DAT_CONNECTION_EVENT_BROKEN),
+ DATxx (DAT_CONNECTION_EVENT_TIMED_OUT),
+ DATxx (DAT_CONNECTION_EVENT_UNREACHABLE),
+ DATxx (DAT_ASYNC_ERROR_EVD_OVERFLOW),
+ DATxx (DAT_ASYNC_ERROR_IA_CATASTROPHIC),
+ DATxx (DAT_ASYNC_ERROR_EP_BROKEN),
+ DATxx (DAT_ASYNC_ERROR_TIMED_OUT),
+ DATxx (DAT_ASYNC_ERROR_PROVIDER_INTERNAL_ERROR),
+ DATxx (DAT_SOFTWARE_EVENT)
+ # undef DATxx
+ };
+ # define NUM_EVENTS (sizeof(dat_events)/sizeof(dat_events[0]))
+
+ for (i = 0; i < NUM_EVENTS; i++) {
+ if (dat_events[i].value == event_code)
+ {
+ return ( dat_events[i].name );
+ }
+ }
+
+ return ( "Invalid_DAT_EVENT_NUMBER" );
}
+
void print_usage(void)
{
- printf("\n DAPL USAGE \n\n");
- printf("s: server\n");
- printf("t: performance times\n");
- printf("c: use cno\n");
- printf("v: verbose\n");
- printf("p: polling\n");
- printf("d: delay before accept\n");
- printf("b: buf length to allocate\n");
- printf("B: burst count, rdma and msgs \n");
- printf("h: hostname/address of server, specified on client\n");
- printf("P: provider name (default = OpenIB-cma)\n");
- printf("\n");
+ printf("\n DAPL USAGE \n\n");
+ printf("s: server\n");
+ printf("t: performance times\n");
+ printf("c: use cno\n");
+ printf("v: verbose\n");
+ printf("p: polling\n");
+ printf("d: delay before accept\n");
+ printf("b: buf length to allocate\n");
+ printf("B: burst count, rdma and msgs \n");
+ printf("h: hostname/address of server, specified on client\n");
+ printf("P: provider name (default = OpenIB-cma)\n");
+ printf("\n");
}
http://git.etherboot.org / mirror / winof / .git / commitdiff