winverbs: convert RDMA address to network-byte order
authorshefty <shefty@ad392aa1-c5ef-ae45-8dd8-e69d62a5ef86>
Thu, 11 Jun 2009 04:02:27 +0000 (04:02 +0000)
committershefty <shefty@ad392aa1-c5ef-ae45-8dd8-e69d62a5ef86>
Thu, 11 Jun 2009 04:02:27 +0000 (04:02 +0000)
Change the winverbs interface to accept remote RDMA addresses in network
byte order.  This keeps it consistent with all other remote/wire data fields.

Unfortunately, this means that the remote address must be swizzled between
winverbs and the HCA libraries, which take the address in host order, but
the other fields in network order.  Longer term, the HCA libraries can provide
optimized interfaces for winverbs, like posting a single work request.

For now, to avoid compatibility issues later, define the winverbs interface
using what's most reasonable.  Update libibverbs accordingly.

Signed-off-by: Sean Hefty <sean.hefty@intel.com>
git-svn-id: svn://openib.tc.cornell.edu/gen1/trunk@2245 ad392aa1-c5ef-ae45-8dd8-e69d62a5ef86

core/winverbs/user/wv_base.h
core/winverbs/user/wv_qp.cpp
inc/user/rdma/winverbs.h
ulp/libibverbs/src/verbs.cpp

index ef4aa2e..592fc18 100644 (file)
@@ -70,7 +70,6 @@ protected:
 \r
 #if defined(_WIN64)\r
        #define WvConvertSgl(pSgl, nSge)        ((ib_local_ds_t *) pSgl)\r
-       #define WvConvertSends(pSend)           ((ib_send_wr_t *) pSend)\r
 #else\r
        static inline ib_local_ds_t *WvConvertSgl(WV_SGE* pSgl, SIZE_T nSge)\r
        {\r
@@ -81,16 +80,6 @@ protected:
                }\r
                return (ib_local_ds_t *) pSgl;\r
        }\r
-\r
-       static inline ib_send_wr_t *WvConvertSends(WV_SEND_REQUEST *pSend)\r
-       {\r
-               WV_SEND_REQUEST *preq;\r
-\r
-               for (preq = pSend; preq != NULL; preq = preq->pNext) {\r
-                       WvConvertSgl(preq->pSgl, preq->nSge);\r
-               }\r
-               return (ib_send_wr_t *) pSend;\r
-       }\r
 #endif (_WIN64)\r
 \r
 #endif // _WV_BASE_H_
\ No newline at end of file
index 297436a..dd1038c 100644 (file)
@@ -421,18 +421,31 @@ PostReceive(UINT64 WrId, WV_SGE* pSgl, SIZE_T nSge)
 STDMETHODIMP CWVQueuePair::\r
 PostSend(WV_SEND_REQUEST *pSend, WV_SEND_REQUEST **ppFailed)\r
 {\r
-       ib_send_wr_t            *wr;\r
+       WV_SEND_REQUEST         *wr;\r
        ib_api_status_t         stat;\r
        HRESULT                         hr;\r
 \r
-       stat = m_pVerbs->post_send(m_hVerbsQp, WvConvertSends(pSend),\r
-                                                          (ppFailed == NULL) ? &wr : (ib_send_wr_t **) ppFailed);\r
+       for (wr = pSend; wr != NULL; wr = wr->pNext) {\r
+               if (wr->Opcode != WvSend) {\r
+                       wr->Wr.Rdma.RemoteAddress = _byteswap_uint64(wr->Wr.Rdma.RemoteAddress);\r
+               }\r
+               WvConvertSgl(wr->pSgl, wr->nSge);\r
+       }\r
+\r
+       stat = m_pVerbs->post_send(m_hVerbsQp, (ib_send_wr_t *) pSend,\r
+                                                          (ib_send_wr_t **) (ppFailed ? ppFailed : &wr));\r
        if (stat == IB_SUCCESS) {\r
                hr = WV_SUCCESS;\r
        } else {\r
                hr = WvConvertIbStatus(stat);\r
        }\r
 \r
+       for (wr = pSend; wr != NULL; wr = wr->pNext) {\r
+               if (wr->Opcode != WvSend) {\r
+                       wr->Wr.Rdma.RemoteAddress = _byteswap_uint64(wr->Wr.Rdma.RemoteAddress);\r
+               }\r
+       }\r
+\r
        return hr;\r
 }\r
 \r
index e707cd9..4deef0a 100644 (file)
@@ -291,14 +291,14 @@ typedef struct _WV_SEND_REQUEST
        {\r
                struct\r
                {\r
-                       UINT64                  RemoteAddress;\r
+                       NET64                   RemoteAddress;\r
                        NET32                   Rkey;\r
 \r
                }       Rdma;\r
 \r
                struct\r
                {\r
-                       UINT64                  RemoteAddress;\r
+                       NET64                   RemoteAddress;\r
                        NET32                   Rkey;\r
                        NET64                   Compare;\r
                        NET64                   Exchange;\r
@@ -307,7 +307,7 @@ typedef struct _WV_SEND_REQUEST
 \r
                struct\r
                {\r
-                       UINT64                  RemoteAddress;\r
+                       NET64                   RemoteAddress;\r
                        NET32                   Rkey;\r
                        NET64                   Add;\r
                        UINT64                  Reserved;\r
@@ -739,9 +739,9 @@ DECLARE_INTERFACE_(IWVSharedReceiveQueue, IWVOverlapped)
 \r
 #undef INTERFACE\r
 #define INTERFACE IWVQueuePair\r
-// {a847c13c-e617-489c-b0ab-2da73eb0adfd}\r
-DEFINE_GUID(IID_IWVQueuePair, 0xa847c13c, 0xe617, 0x489c,\r
-                       0xb0, 0xab, 0x2d, 0xa7, 0x3e, 0xb0, 0xad, 0xfd);\r
+// {35C64226-6A1F-4c8d-9465-C6FEE8053CDD}\r
+DEFINE_GUID(IID_IWVQueuePair, 0x35c64226, 0x6a1f, 0x4c8d,\r
+                       0x94, 0x65, 0xc6, 0xfe, 0xe8, 0x5, 0x3c, 0xdd);\r
 \r
 DECLARE_INTERFACE_(IWVQueuePair, IWVOverlapped)\r
 {\r
@@ -802,9 +802,9 @@ DECLARE_INTERFACE_(IWVQueuePair, IWVOverlapped)
 \r
 #undef INTERFACE\r
 #define INTERFACE IWVConnectQueuePair\r
-// {A791309C-4244-4194-AD63-7D372CADE5E0}\r
-DEFINE_GUID(IID_IWVConnectQueuePair, 0xa791309c, 0x4244, 0x4194,\r
-                       0xad, 0x63, 0x7d, 0x37, 0x2c, 0xad, 0xe5, 0xe0);\r
+// {DF905570-AEEB-4114-B30E-4DC3EB5A9AD6}\r
+DEFINE_GUID(IID_IWVConnectQueuePair, 0xdf905570, 0xaeeb, 0x4114,\r
+                       0xb3, 0xe, 0x4d, 0xc3, 0xeb, 0x5a, 0x9a, 0xd6);\r
 \r
 DECLARE_INTERFACE_(IWVConnectQueuePair, IWVQueuePair)\r
 {\r
index fd9f318..e5dffc6 100644 (file)
@@ -735,6 +735,7 @@ int ibv_post_send(struct ibv_qp *qp, struct ibv_send_wr *wr,
                }\r
 \r
                if (cur_wr->opcode != 0) {\r
+                       cur_wr->wr.rdma.remote_addr = htonll(cur_wr->wr.rdma.remote_addr);\r
                        cur_wr->wr.rdma.rkey = htonl(cur_wr->wr.rdma.rkey);\r
                }\r
        }\r
@@ -744,6 +745,7 @@ int ibv_post_send(struct ibv_qp *qp, struct ibv_send_wr *wr,
        for (cur_wr = wr; cur_wr != NULL; cur_wr = cur_wr->next) {\r
                if (cur_wr->opcode != 0) {\r
                        cur_wr->wr.rdma.rkey = ntohl(cur_wr->wr.rdma.rkey);\r
+                       cur_wr->wr.rdma.remote_addr = htonll(cur_wr->wr.rdma.remote_addr);\r
                }\r
 \r
                if ((cur_wr->send_flags & WV_SEND_IMMEDIATE) != 0) {\r