[DAPL2] fix a problem in socket-cm with FD_SETSIZE not being uniformly defined. Add...
authorstansmith <stansmith@ad392aa1-c5ef-ae45-8dd8-e69d62a5ef86>
Thu, 9 Jul 2009 17:53:47 +0000 (17:53 +0000)
committerstansmith <stansmith@ad392aa1-c5ef-ae45-8dd8-e69d62a5ef86>
Thu, 9 Jul 2009 17:53:47 +0000 (17:53 +0000)
git-svn-id: svn://openib.tc.cornell.edu/gen1/trunk@2279 ad392aa1-c5ef-ae45-8dd8-e69d62a5ef86

13 files changed:
ulp/dapl2/dapl/openib_cma/cm.c
ulp/dapl2/dapl/openib_scm/SOURCES
ulp/dapl2/dapl/openib_scm/cm.c
ulp/dapl2/dapl/openib_scm/windows/openib_osd.h
ulp/dapl2/test/dapltest/scripts/dt-cli.bat
ulp/dapl2/test/dapltest/scripts/dt-svr.bat
ulp/dapl2/test/dtest/dtest.c
ulp/dapl2/test/dtest/dtestcm.c
ulp/dapl2/test/dtest/windows/dtest/SOURCES
ulp/dapl2/test/dtest/windows/dtest/dtest.c
ulp/dapl2/test/dtest/windows/dtestcm/SOURCES
ulp/dapl2/test/dtest/windows/dtestcm/dtestcm.c
ulp/dapl2/test/dtest/windows/dtestx/SOURCES

index 163aa58..5631fe7 100644 (file)
@@ -1317,6 +1317,8 @@ void dapli_cma_event_cb(void)
                                dapli_cm_active_cb(conn, event);\r
                        break;\r
                case RDMA_CM_EVENT_CONNECT_RESPONSE:\r
+               case RDMA_CM_EVENT_TIMEWAIT_EXIT:\r
+                       break;\r
                default:\r
                        dapl_dbg_log(DAPL_DBG_TYPE_WARN,\r
                                     " cm_event: UNEXPECTED EVENT=%p ID=%p CTX=%p\n",\r
index c2f890e..6e4ad30 100644 (file)
@@ -28,7 +28,7 @@ INCLUDES = ..\include;..\openib_common\;..\common;windows;..\..\dat\include;\
                   ..\..\dat\udat\windows;..\udapl\windows;\\r
                   ..\..\..\..\inc;..\..\..\..\inc\user;..\..\..\libibverbs\include\r
 \r
-DAPL_OPTS = -DEXPORT_DAPL_SYMBOLS -DDAT_EXTENSIONS=1 -DSOCK_CM -DOPENIB -DCQ_WAIT_OBJECT\r
+DAPL_OPTS = -DEXPORT_DAPL_SYMBOLS -DDAT_EXTENSIONS -DSOCK_CM -DOPENIB -DCQ_WAIT_OBJECT\r
 \r
 USER_C_FLAGS = $(USER_C_FLAGS) $(DAPL_OPTS)\r
 \r
index cf9302e..3639f15 100644 (file)
  *\r
  **************************************************************************/\r
 \r
+#if defined(_WIN32)\r
+#define FD_SETSIZE 1024\r
+#define DAPL_FD_SETSIZE FD_SETSIZE\r
+#endif\r
+\r
 #include "dapl.h"\r
 #include "dapl_adapter_util.h"\r
 #include "dapl_evd_util.h"\r
@@ -274,19 +279,24 @@ void dapls_ib_cm_free(dp_ib_cm_handle_t cm_ptr, DAPL_EP *ep)
        hca_ptr = cm_ptr->hca;\r
 \r
        /* cleanup, never made it to work queue */\r
+       dapl_os_lock(&cm_ptr->lock);\r
        if (cm_ptr->state == DCM_INIT) {\r
                if (cm_ptr->socket != DAPL_INVALID_SOCKET) {\r
                        shutdown(cm_ptr->socket, SHUT_RDWR);\r
                        closesocket(cm_ptr->socket);\r
                }\r
+               dapl_os_unlock(&cm_ptr->lock);\r
                dapl_os_free(cm_ptr, sizeof(*cm_ptr));\r
                return;\r
        }\r
 \r
-       /* free could be called before disconnect */\r
-       dapli_socket_disconnect(cm_ptr);\r
+       /* free could be called before disconnect, disc_clean will destroy */\r
+       if (cm_ptr->state == DCM_CONNECTED) {\r
+               dapl_os_unlock(&cm_ptr->lock);\r
+               dapli_socket_disconnect(cm_ptr);\r
+               return;\r
+       }\r
 \r
-       dapl_os_lock(&cm_ptr->lock);\r
        cm_ptr->state = DCM_DESTROY;\r
        if ((cm_ptr->ep) && (cm_ptr->ep->cm_handle == cm_ptr)) {\r
                cm_ptr->ep->cm_handle = IB_INVALID_HANDLE;\r
@@ -427,7 +437,7 @@ DAT_RETURN dapli_socket_disconnect(dp_ib_cm_handle_t cm_ptr)
  */\r
 static void dapli_socket_connected(dp_ib_cm_handle_t cm_ptr, int err)\r
 {\r
-       int len, opt = 1;\r
+       int ret, len, opt = 1;\r
        struct iovec iov[2];\r
        struct dapl_ep *ep_ptr = cm_ptr->ep;\r
 \r
@@ -447,8 +457,12 @@ static void dapli_socket_connected(dp_ib_cm_handle_t cm_ptr, int err)
                     " socket connected, write QP and private data\n");\r
 \r
        /* no delay for small packets */\r
-       setsockopt(cm_ptr->socket, IPPROTO_TCP, TCP_NODELAY,\r
-                  (char *)&opt, sizeof(opt));\r
+       ret = setsockopt(cm_ptr->socket, IPPROTO_TCP, TCP_NODELAY,\r
+                        (char *)&opt, sizeof(opt));\r
+       if (ret)\r
+               dapl_log(DAPL_DBG_TYPE_WARN,\r
+                        " connected: NODELAY setsockopt: %s\n",\r
+                        strerror(errno));\r
 \r
        /* send qp info and pdata to remote peer */\r
        iov[0].iov_base = (void *)&cm_ptr->dst;\r
@@ -798,6 +812,7 @@ dapli_socket_listen(DAPL_IA * ia_ptr, DAT_CONN_QUAL serviceID, DAPL_SP * sp_ptr)
 \r
        setsockopt(cm_ptr->socket, SOL_SOCKET, SO_REUSEADDR,\r
                   (char *)&opt, sizeof(opt));\r
+\r
        addr.sin_port = htons(serviceID);\r
        addr.sin_family = AF_INET;\r
        addr.sin_addr.s_addr = INADDR_ANY;\r
@@ -839,7 +854,7 @@ dapli_socket_listen(DAPL_IA * ia_ptr, DAT_CONN_QUAL serviceID, DAPL_SP * sp_ptr)
 static void dapli_socket_accept(ib_cm_srvc_handle_t cm_ptr)\r
 {\r
        dp_ib_cm_handle_t acm_ptr;\r
-       int len;\r
+       int ret, len, opt = 1;\r
 \r
        dapl_dbg_log(DAPL_DBG_TYPE_EP, " socket_accept\n");\r
        \r
@@ -868,6 +883,14 @@ static void dapli_socket_accept(ib_cm_srvc_handle_t cm_ptr)
                        return;\r
                }\r
 \r
+               /* no delay for small packets */\r
+               ret = setsockopt(acm_ptr->socket, IPPROTO_TCP, TCP_NODELAY,\r
+                          (char *)&opt, sizeof(opt));\r
+               if (ret)\r
+                       dapl_log(DAPL_DBG_TYPE_WARN,\r
+                                " accept: NODELAY setsockopt: %s\n",\r
+                                strerror(errno));\r
+\r
                acm_ptr->state = DCM_ACCEPTING;\r
                dapli_cm_queue(acm_ptr);\r
        \r
index 9b0f089..31fe32f 100644 (file)
@@ -1,36 +1,39 @@
-#ifndef OPENIB_OSD_H
-#define OPENIB_OSD_H
-
-#ifndef FD_SETSIZE
-#define FD_SETSIZE 1024 /* Set before including winsock2 - see select help */
-#define DAPL_FD_SETSIZE FD_SETSIZE
-#endif
-
-#include <winsock2.h>
-#include <ws2tcpip.h>
-#include <io.h>
-#include <fcntl.h>
-
-#define ntohll _byteswap_uint64
-#define htonll _byteswap_uint64
-
-#define DAPL_SOCKET SOCKET
-#define DAPL_INVALID_SOCKET INVALID_SOCKET
-#define SHUT_RDWR SD_BOTH
-
-/* allow casting to WSABUF */
-struct iovec
-{
-       u_long iov_len;
-       char FAR* iov_base;
-};
-
-static int writev(DAPL_SOCKET s, struct iovec *vector, int count)
-{
-       int len, ret;
-
-       ret = WSASend(s, (WSABUF *) vector, count, &len, 0, NULL, NULL);
-       return ret ? ret : len;
-}
-
-#endif // OPENIB_OSD_H
+#ifndef OPENIB_OSD_H\r
+#define OPENIB_OSD_H\r
+\r
+#if defined(FD_SETSIZE) && FD_SETSIZE != 1024\r
+#undef FD_SETSIZE\r
+#undef DAPL_FD_SETSIZE\r
+#endif\r
+\r
+#define FD_SETSIZE 1024 /* Set before including winsock2 - see select help */\r
+#define DAPL_FD_SETSIZE FD_SETSIZE\r
+\r
+#include <winsock2.h>\r
+#include <ws2tcpip.h>\r
+#include <io.h>\r
+#include <fcntl.h>\r
+\r
+#define ntohll _byteswap_uint64\r
+#define htonll _byteswap_uint64\r
+\r
+#define DAPL_SOCKET SOCKET\r
+#define DAPL_INVALID_SOCKET INVALID_SOCKET\r
+#define SHUT_RDWR SD_BOTH\r
+\r
+/* allow casting to WSABUF */\r
+struct iovec\r
+{\r
+       u_long iov_len;\r
+       char FAR* iov_base;\r
+};\r
+\r
+static int writev(DAPL_SOCKET s, struct iovec *vector, int count)\r
+{\r
+       int len, ret;\r
+\r
+       ret = WSASend(s, (WSABUF *) vector, count, &len, 0, NULL, NULL);\r
+       return ret ? ret : len;\r
+}\r
+\r
+#endif // OPENIB_OSD_H\r
index 9cbb202..c445d5a 100644 (file)
@@ -308,7 +308,7 @@ if "%T%" == "interop" (
 )\r
 \r
 if "%T%" == "stop" (\r
-    %DT% -T Q -s %S% -D %D% > nul\r
+    %DT% -T Q -s %S% -D %D%\r
     goto rxt\r
 )\r
 \r
index 1d3760d..c876d41 100644 (file)
@@ -37,10 +37,10 @@ if "%DEV%" == ""    set DEV=%1
 rem '-D' enables full debug output?\r
 rem '-D hex-bit-mask' enables selective debug output - see manual.htm for details.\r
 if "%2" == "-D" (\r
-    if "%2" == "" (\r
+    if "%3" == "" (\r
         set X=0xfffff\r
     ) else (\r
-        set X=%2\r
+        set X=%3\r
     )\r
 ) else ( set X= )\r
 \r
index f1f0f2b..77d78b2 100644 (file)
@@ -630,19 +630,6 @@ complete:
        return (0);
 }
 
-#if defined(_WIN32) || defined(_WIN64)
-
-void gettimeofday(struct timeval *t, char *jnk)
-{
-       SYSTEMTIME now;
-       GetLocalTime(&now);
-       t->tv_sec = now.wMinute * 60;
-       t->tv_sec += now.wSecond;
-       t->tv_usec = now.wMilliseconds;
-}
-
-#endif
-
 double get_time(void)
 {
        struct timeval tp;
index abc7bb5..b6dcccf 100644 (file)
@@ -97,6 +97,7 @@
 \r
 /* Header files needed for DAT/uDAPL */\r
 #include    "dat2/udat.h"\r
+#include    "dat2/dat_ib_extensions.h"\r
 \r
 /* definitions */\r
 #define SERVER_CONN_QUAL  45248\r
@@ -149,6 +150,7 @@ struct dt_time time;
 /* defaults */\r
 static int connected = 0;\r
 static int multi_listens = 0;\r
+static int ud_test = 0;\r
 static int server = 1;\r
 static int waiting = 0;\r
 static int verbose = 0;\r
@@ -191,7 +193,7 @@ int main(int argc, char **argv)
        DAT_RETURN ret;\r
        \r
        /* parse arguments */\r
-       while ((c = getopt(argc, argv, "smwvb:c:d:h:P:p:")) != -1) {\r
+       while ((c = getopt(argc, argv, "smwvub:c:d:h:P:p:")) != -1) {\r
                switch (c) {\r
                case 's':\r
                        server = 1;\r
@@ -202,6 +204,9 @@ int main(int argc, char **argv)
                case 'w':\r
                        waiting = 1;\r
                        break;\r
+               case 'u':\r
+                       ud_test = 1;\r
+                       break;\r
                case 'c':\r
                        connections = atoi(optarg);\r
                        break;\r
@@ -246,11 +251,11 @@ int main(int argc, char **argv)
 #endif\r
 \r
        if (!server) {\r
-               printf(" Running client on %s with %d connections\n", \r
-                       provider, connections);\r
+               printf(" Running client on %s with %d %s connections\n", \r
+                       provider, connections, ud_test ? "UD" : "RC");\r
        } else {\r
-               printf(" Running server on %s with %d connections\n", \r
-                       provider, connections);\r
+               printf(" Running server on %s with %d %s connections\n", \r
+                       provider, connections, ud_test ? "UD" : "RC");\r
        }\r
        fflush(stdout);\r
        \r
@@ -318,8 +323,15 @@ int main(int argc, char **argv)
 \r
        /* create EP */\r
        memset(&ep_attr, 0, sizeof(ep_attr));\r
-       ep_attr.service_type = DAT_SERVICE_TYPE_RC;\r
-       ep_attr.max_rdma_size = 0x10000;\r
+       if (ud_test) {\r
+               ep_attr.service_type = DAT_IB_SERVICE_TYPE_UD;\r
+               ep_attr.max_message_size = 2048;\r
+       } else {\r
+               ep_attr.service_type = DAT_SERVICE_TYPE_RC;\r
+               ep_attr.max_rdma_size = 0x10000;\r
+               ep_attr.max_rdma_read_in = 4;\r
+               ep_attr.max_rdma_read_out = 4;\r
+       }\r
        ep_attr.max_recv_dtos = 1;\r
        ep_attr.max_request_dtos = 1;\r
        ep_attr.max_recv_iov = 1;\r
@@ -489,19 +501,6 @@ bail:
        return (0);\r
 }\r
 \r
-#if defined(_WIN32) || defined(_WIN64)\r
-\r
-void gettimeofday(struct timeval *t, char *jnk)\r
-{\r
-       SYSTEMTIME now;\r
-       GetLocalTime(&now);\r
-       t->tv_sec = now.wMinute * 60;\r
-       t->tv_sec += now.wSecond;\r
-       t->tv_usec = now.wMilliseconds;\r
-}\r
-\r
-#endif\r
-\r
 double get_time(void)\r
 {\r
        struct timeval tp;\r
@@ -515,10 +514,10 @@ DAT_RETURN conn_server()
        DAT_RETURN ret;\r
        DAT_EVENT event;\r
        DAT_COUNT nmore;\r
-       DAT_CR_ARRIVAL_EVENT_DATA *cr_event =\r
-               &event.event_data.cr_arrival_event_data;\r
        int i,bi;\r
        unsigned char *buf;\r
+       DAT_CR_ARRIVAL_EVENT_DATA *cr_event =\r
+               &event.event_data.cr_arrival_event_data;\r
        DAT_CR_PARAM cr_param = { 0 };\r
        \r
        printf(" Accepting...\n");\r
@@ -541,24 +540,26 @@ DAT_RETURN conn_server()
                        }\r
                }\r
        \r
-               if (event.event_number != DAT_CONNECTION_REQUEST_EVENT) {\r
-                       fprintf(stderr, " Error unexpected CR event : %s\n",\r
-                               DT_EventToSTr(event.event_number));\r
-                       return (DAT_ABORT);\r
+               if ((event.event_number != DAT_CONNECTION_REQUEST_EVENT) &&\r
+                   (ud_test && event.event_number != \r
+                    DAT_IB_UD_CONNECTION_REQUEST_EVENT)) {\r
+                               fprintf(stderr, " Error unexpected CR event : %s\n",\r
+                                       DT_EventToSTr(event.event_number));\r
+                               return (DAT_ABORT);\r
                }\r
 \r
-\r
+               \r
                /* use to test rdma_cma timeout logic */\r
 #if defined(_WIN32) || defined(_WIN64)\r
-                   if (delay) {\r
+               if (delay) {\r
                        printf(" Accept delayed by %d seconds...\n", delay);\r
                        Sleep(delay * 1000);\r
-                   }\r
+               }\r
 #else\r
-                   if (delay) {\r
+               if (delay) {\r
                        printf(" Accept delayed by %d seconds...\n", delay);\r
                        sleep(delay);\r
-                   }\r
+               }\r
 #endif\r
                /* accept connect request from client */\r
                h_cr = cr_event->cr_handle;\r
@@ -622,8 +623,10 @@ DAT_RETURN conn_server()
                                return ret;\r
                        }\r
                }\r
+               if ((event.event_number != DAT_CONNECTION_EVENT_ESTABLISHED) &&\r
+                   (ud_test && event.event_number != \r
+                    DAT_IB_UD_CONNECTION_EVENT_ESTABLISHED)) {\r
 \r
-               if (event.event_number != DAT_CONNECTION_EVENT_ESTABLISHED) {\r
                        fprintf(stderr, " Error unexpected CR EST "\r
                                "event : 0x%x %s\n",\r
                                 event.event_number,\r
@@ -748,8 +751,10 @@ DAT_RETURN conn_client()
                        printf("\n Rej Test Done. PASSED\n\n");\r
                        exit(0);\r
 #endif\r
-                       if (event.event_number != \r
-                               DAT_CONNECTION_EVENT_ESTABLISHED) {\r
+                       if ((event.event_number != \r
+                           DAT_CONNECTION_EVENT_ESTABLISHED) &&\r
+                           (ud_test && event.event_number != \r
+                           DAT_IB_UD_CONNECTION_EVENT_ESTABLISHED)) {\r
                                fprintf(stderr, " Error unexpected conn "\r
                                        "event : 0x%x %s\n",\r
                                         event.event_number,\r
@@ -802,8 +807,28 @@ DAT_RETURN disconnect_eps(void)
        DAT_CONNECTION_EVENT_DATA *conn_event =\r
                &event.event_data.connect_event_data;\r
 \r
-       if (!connected) \r
+       if (!connected)\r
+               return DAT_SUCCESS;\r
+\r
+       /* UD, no connection to disconnect, just free EP's */\r
+       if (ud_test) {\r
+               for (i = 0; i < connections; i++) {\r
+                       ret = dat_ep_free(h_ep[i]);\r
+                       if (ret != DAT_SUCCESS) {\r
+                               fprintf(stderr, \r
+                                       " ERR free EP[%d] %p: %s\n",\r
+                                       i, h_ep[i], DT_RetToString(ret));\r
+                       } else {\r
+                               LOGPRINTF(" Freed EP[%d] %p\n", \r
+                                         i, h_ep[i]);\r
+                               h_ep[i] = DAT_HANDLE_NULL;\r
+                       }\r
+               }\r
+               stop = get_time();\r
+               time.epf += ((stop - start) * 1.0e6);\r
+               time.total += time.epf;\r
                return DAT_SUCCESS;\r
+       }\r
        \r
        /* \r
         * Only the client needs to call disconnect. The server _should_ be able\r
@@ -811,6 +836,7 @@ DAT_RETURN disconnect_eps(void)
         * disconnect request and then exit.\r
         */\r
        if (!server) {\r
+               start = get_time();\r
                for (i = 0; i < connections; i++) {\r
                        LOGPRINTF(" dat_ep_disconnect\n");\r
                        ret = dat_ep_disconnect(h_ep[i], \r
index 58f5c66..e880beb 100644 (file)
@@ -19,7 +19,7 @@ INCLUDES = ..\..\..\..\dat\include;..\..\..\..\..\..\inc;\
 RCOPTIONS=/I..\..\..\..\..\..\inc;
 
 # Set defines particular to the driver.
-#USER_C_FLAGS = $(USER_C_FLAGS) /DDAT_EXTENSIONS
+USER_C_FLAGS = $(USER_C_FLAGS) /DDAT_EXTENSIONS /DFD_SETSIZE=1024
 
 !if $(FREEBUILD)
 DATLIB = dat2.lib
index ae69c20..2bdfae3 100644 (file)
@@ -1,2 +1,3 @@
+#include "..\..\..\..\..\..\etc\user\gtod.c"
 #include "..\..\dtest.c"
 
index 7b4264e..662da89 100644 (file)
@@ -19,7 +19,7 @@ INCLUDES = ..\..\..\..\dat\include;..\..\..\..\..\..\inc;\
 RCOPTIONS=/I..\..\..\..\..\..\inc;
 
 # Set defines particular to the driver.
-#USER_C_FLAGS = $(USER_C_FLAGS) /DDAT_EXTENSIONS
+USER_C_FLAGS = $(USER_C_FLAGS) /DDAT_EXTENSIONS /DFD_SETSIZE=1024
 
 !if $(FREEBUILD)
 DATLIB = dat2.lib
index f9a6db0..c734a6b 100644 (file)
@@ -1,2 +1,3 @@
+#include "..\..\..\..\..\..\etc\user\gtod.c"\r
 #include "..\..\dtestcm.c"\r
 \r
index ee63817..2a7f1f6 100644 (file)
@@ -16,6 +16,9 @@ SOURCES = \
 INCLUDES = ..\..\..\..\dat\include;..\..\..\..\..\..\inc;\\r
           ..\..\..\..\..\..\inc\user;\r
 \r
+# Set defines particular to the driver.\r
+USER_C_FLAGS = $(USER_C_FLAGS) /DDAT_EXTENSIONS /DFD_SETSIZE=1024\r
+\r
 !if $(FREEBUILD)\r
 DATLIB = dat2.lib\r
 !else\r