[VNIC] Initial checkin of VNIC code. Not yet fully functional.
[mirror/winof/.git] / ulp / inic / kernel / vnic_data.c
1 /*\r
2  * Copyright (c) 2005 SilverStorm Technologies.  All rights reserved.\r
3  *\r
4  * This software is available to you under the OpenIB.org BSD license\r
5  * below:\r
6  *\r
7  *     Redistribution and use in source and binary forms, with or\r
8  *     without modification, are permitted provided that the following\r
9  *     conditions are met:\r
10  *\r
11  *      - Redistributions of source code must retain the above\r
12  *        copyright notice, this list of conditions and the following\r
13  *        disclaimer.\r
14  *\r
15  *      - Redistributions in binary form must reproduce the above\r
16  *        copyright notice, this list of conditions and the following\r
17  *        disclaimer in the documentation and/or other materials\r
18  *        provided with the distribution.\r
19  *\r
20  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,\r
21  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF\r
22  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND\r
23  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS\r
24  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN\r
25  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN\r
26  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\r
27  * SOFTWARE.\r
28  *\r
29  * $Id$\r
30  */\r
31 #include <complib/comp_lib.h>\r
32 #include "vnic_driver.h"\r
33 \r
34 static void    data_postRecvs(Data_t *pData);\r
35 static void    _data_receivedKick(Io_t *pIo);\r
36 static void    _data_xmitComplete(Io_t *pIo);\r
37 static void    data_sendKickMessage(Data_t *pData);\r
38 static void    _data_kickTimeoutHandler( void *context );\r
39 static BOOLEAN data_allocXmitBuffer(Data_t *pData,\r
40                                         BufferPoolEntry_t **ppBpe, RdmaIo_t **ppRdmaIo, BOOLEAN *pLast);\r
41 static void    data_checkXmitBuffers(Data_t *pData);\r
42 static void    data_rdmaPacket(Data_t *pData, BufferPoolEntry_t *pBpe, RdmaIo_t *pRdmaIo);\r
43 \r
44 static NDIS_PACKET *\r
45 _data_recv_to_ndis_pkt( Data_t *pData, RdmaDest_t *pRdmaDest );\r
46 \r
47 static void\r
48 _data_allocBuffers(\r
49                                         Data_t          *pData,\r
50                                         BOOLEAN         initialAllocation );\r
51 \r
52 static void\r
53 _data_addFreeBuffer(\r
54                                         Data_t          *pData,\r
55                                         int                     index,\r
56                                         RdmaDest_t      *pRdmaDest );\r
57 \r
58 static uint32_t\r
59 _data_incomingRecv(\r
60                                         Data_t          *pData );\r
61 \r
62 static void   \r
63 _data_sendFreeRecvBuffers(\r
64                                         Data_t          *pData );\r
65 \r
66 static uint8_t\r
67 _tx_chksum_flags(\r
68  IN             NDIS_PACKET* const              p_packet );\r
69 \r
70 \r
71 static void\r
72 _get_first_buffer(\r
73                 IN              NDIS_PACKET             *p_packet,\r
74                 IN OUT  NDIS_BUFFER             **pp_buf_desc,\r
75                 OUT             void                    **pp_buf,\r
76                 OUT             ULONG                   *p_packet_sz );\r
77 \r
78 static void\r
79 _data_return_recv(\r
80           IN    NDIS_PACKET             *p_packet );\r
81 \r
82 static void\r
83 _data_kickTimer_start(\r
84                   IN    Data_t          *pData,\r
85                   IN    uint32_t        microseconds );\r
86 \r
87 static void\r
88 _data_kickTimer_stop(\r
89                   IN    Data_t          *pData );\r
90 \r
91 #define LOCAL_IO(x) PTR64((x))\r
92 \r
93 #define INBOUND_COPY\r
94 \r
95 #ifdef VNIC_STATISTIC\r
96 int64_t recvRef;\r
97 #endif /* VNIC_STATISTIC */\r
98 \r
99 void\r
100 data_construct(\r
101         IN              Data_t                  *pData,\r
102         IN              viport_t                *pViport )\r
103 {\r
104         VNIC_ENTER( VNIC_DBG_DATA | VNIC_DBG_INFO );\r
105 \r
106         RtlZeroMemory( pData, sizeof(*pData) );\r
107 \r
108         pData->p_viport = pViport;\r
109         pData->p_phy_region = &pViport->p_adapter->ca.region;\r
110         InitializeListHead( &pData->recvIos );\r
111         KeInitializeSpinLock ( &pData->recvIosLock );\r
112         KeInitializeSpinLock ( &pData->xmitBufLock );\r
113         cl_timer_construct( &pData->kickTimer );\r
114 \r
115         ibqp_construct( &pData->qp, pViport );\r
116 \r
117         VNIC_EXIT( VNIC_DBG_DATA | VNIC_DBG_INFO );\r
118 }\r
119 \r
120 \r
121 ib_api_status_t\r
122 data_init(\r
123         IN              Data_t                  *pData,\r
124         IN              DataConfig_t    *p_conf,\r
125         IN              uint64_t                guid )\r
126 {\r
127         ib_api_status_t ib_status;\r
128 \r
129         VNIC_ENTER( VNIC_DBG_DATA | VNIC_DBG_INFO );\r
130 \r
131         ASSERT( pData->p_viport != NULL );\r
132         pData->p_conf = p_conf;\r
133 \r
134         cl_timer_init( &pData->kickTimer, _data_kickTimeoutHandler, pData );\r
135 \r
136         ib_status = ibqp_init(&pData->qp, guid, &p_conf->ibConfig );\r
137         if( ib_status != IB_SUCCESS )\r
138                 VNIC_TRACE( VNIC_DBG_ERROR, ("data ibqp_init failed\n") );\r
139 \r
140         VNIC_EXIT( VNIC_DBG_DATA | VNIC_DBG_INFO );\r
141         return ib_status;\r
142 }\r
143 \r
144 \r
145 ib_api_status_t\r
146 data_connect(\r
147                 IN              Data_t          *pData )\r
148 {\r
149         NDIS_STATUS                     status;\r
150         ib_api_status_t         ib_status;\r
151         XmitPool_t        *pXmitPool = &pData->xmitPool;\r
152         RecvPool_t        *pRecvPool = &pData->recvPool;\r
153         RecvIo_t          *pRecvIo;\r
154         SendIo_t          *pSendIo;\r
155         RdmaIo_t          *pRdmaIo;\r
156         RdmaDest_t        *pRdmaDest;\r
157         uint8_t           *pRegionData;\r
158         int               sz, regionSz;\r
159         unsigned int      i, j;\r
160 \r
161         VNIC_ENTER( VNIC_DBG_DATA | VNIC_DBG_INFO );\r
162 \r
163         pRecvPool->poolSz           = pData->p_conf->hostRecvPoolEntries;\r
164         pRecvPool->eiocPoolSz       = pData->hostPoolParms.numRecvPoolEntries;\r
165 \r
166         if ( pRecvPool->poolSz > pRecvPool->eiocPoolSz )\r
167         {\r
168                 pRecvPool->poolSz = pData->hostPoolParms.numRecvPoolEntries;\r
169         }\r
170         pRecvPool->szFreeBundle     =\r
171                         pData->hostPoolParms.freeRecvPoolEntriesPerUpdate;\r
172         pRecvPool->numFreeBufs      = 0;\r
173         pRecvPool->numPostedBufs    = 0;\r
174         pRecvPool->nextFullBuf      = 0;\r
175         pRecvPool->nextFreeBuf      = 0;\r
176         pRecvPool->kickOnFree       = FALSE;\r
177         pRecvPool->bufferSz         = pData->hostPoolParms.sizeRecvPoolEntry;\r
178 \r
179         pXmitPool->bufferSz         = pData->eiocPoolParms.sizeRecvPoolEntry;\r
180         pXmitPool->poolSz           = pData->eiocPoolParms.numRecvPoolEntries;\r
181         pXmitPool->notifyCount      = 0;\r
182         pXmitPool->notifyBundle     = pData->p_conf->notifyBundle;\r
183         pXmitPool->nextXmitPool     = 0;\r
184 \r
185 #if TRUE // LIMIT_OUTSTANDING_SENDS\r
186         pXmitPool->numXmitBufs      = pXmitPool->notifyBundle * 2;\r
187 #else /* !LIMIT_OUTSTANDING_SENDS */\r
188         pXmitPool->numXmitBufs      = pXmitPool->poolSz;\r
189 #endif /* LIMIT_OUTSTANDING_SENDS */\r
190 \r
191         pXmitPool->nextXmitBuf      = 0;\r
192         pXmitPool->lastCompBuf      = pXmitPool->numXmitBufs - 1;\r
193         pXmitPool->kickCount        = 0;\r
194         pXmitPool->kickByteCount    = 0;\r
195         pXmitPool->sendKicks        =\r
196                 (BOOLEAN)(( pData->eiocPoolParms.numRecvPoolEntriesBeforeKick != 0 )\r
197                 || ( pData->eiocPoolParms.numRecvPoolBytesBeforeKick != 0 ));\r
198         pXmitPool->kickBundle       =\r
199                 pData->eiocPoolParms.numRecvPoolEntriesBeforeKick;\r
200         pXmitPool->kickByteBundle   =\r
201                 pData->eiocPoolParms.numRecvPoolBytesBeforeKick;\r
202         pXmitPool->needBuffers      = TRUE;\r
203 \r
204         sz  = sizeof(RdmaDest_t) * pRecvPool->poolSz;\r
205         sz += sizeof(RecvIo_t) * pData->p_conf->numRecvs;\r
206         sz += sizeof(RdmaIo_t) * pXmitPool->numXmitBufs;\r
207 \r
208         regionSz  = 4 * pData->p_conf->numRecvs;\r
209         regionSz += sizeof(BufferPoolEntry_t) * pRecvPool->eiocPoolSz;\r
210         regionSz += sizeof(BufferPoolEntry_t) * pXmitPool->poolSz;\r
211         sz       += regionSz;\r
212 \r
213         status = NdisAllocateMemoryWithTag( &pData->pLocalStorage,\r
214                                                                                 (UINT)sz,\r
215                                                                                 'grtS' );\r
216         if ( status != NDIS_STATUS_SUCCESS )\r
217         {\r
218                 VNIC_TRACE_EXIT( VNIC_DBG_ERROR,\r
219                         ("Failed allocating %d bytes local storage\n", sz ) );\r
220                 ib_status = IB_INSUFFICIENT_MEMORY;\r
221                 goto err1;\r
222         }\r
223 \r
224         NdisZeroMemory( pData->pLocalStorage, sz );\r
225         pData->localStorageSz = sz;\r
226 \r
227         pRecvPool->pRecvBufs = (RdmaDest_t *)pData->pLocalStorage;\r
228         sz                   = sizeof(RdmaDest_t) * pRecvPool->poolSz;\r
229         pRecvIo              = (RecvIo_t *)(pData->pLocalStorage + sz);\r
230         sz                  += sizeof(RecvIo_t) * pData->p_conf->numRecvs;\r
231 \r
232         pXmitPool->pXmitBufs = (RdmaIo_t *)(pData->pLocalStorage + sz);\r
233         sz                  += sizeof(RdmaIo_t) * pXmitPool->numXmitBufs;\r
234 \r
235         pRegionData          = pData->pLocalStorage + sz;\r
236         sz                  += 4 * pData->p_conf->numRecvs;\r
237 \r
238         pRecvPool->bufPool   = (BufferPoolEntry_t *)(pData->pLocalStorage + sz);\r
239         sz                  += sizeof(BufferPoolEntry_t) * pRecvPool->eiocPoolSz;\r
240         pXmitPool->bufPool   = (BufferPoolEntry_t *)(pData->pLocalStorage + sz);\r
241 \r
242         ib_status = ibregion_init( pData->p_viport, &pData->region,\r
243                 pData->p_viport->p_adapter->ca.hPd, pRegionData, regionSz,\r
244                 ( IB_AC_LOCAL_WRITE | IB_AC_RDMA_WRITE ) );\r
245         if( ib_status != IB_SUCCESS )\r
246         {\r
247                 VNIC_TRACE_EXIT( VNIC_DBG_ERROR, ("ib_region_init failed\n") );\r
248                 goto err2;\r
249         }\r
250 \r
251         pRdmaIo = &pData->freeBufsIo;\r
252         pRdmaIo->io.pViport          = pData->p_viport;\r
253         pRdmaIo->io.pRoutine         = NULL;\r
254         pRdmaIo->io.wrq.p_next       = NULL;\r
255         pRdmaIo->io.wrq.wr_type      = WR_RDMA_WRITE;\r
256         pRdmaIo->io.wrq.wr_id        = PTR64( pRdmaIo );\r
257         pRdmaIo->io.wrq.num_ds       = 1;\r
258         pRdmaIo->io.wrq.ds_array     = pRdmaIo->dsList;\r
259         pRdmaIo->dsList[0].lkey      = pData->region.lkey;\r
260         pRdmaIo->io.wrq.send_opt     = IB_SEND_OPT_SIGNALED;\r
261 \r
262         pSendIo = &pData->kickIo;\r
263         pSendIo->io.pViport          = pData->p_viport;\r
264         pSendIo->io.pRoutine         = NULL;\r
265         pSendIo->io.wrq.p_next       = NULL;\r
266         pSendIo->io.wrq.wr_type      = WR_SEND;\r
267         pSendIo->io.wrq.wr_id        = PTR64( pSendIo );\r
268         pSendIo->io.wrq.num_ds       = 1;\r
269         pSendIo->io.wrq.ds_array     = &pSendIo->dsList;\r
270 \r
271         pSendIo->io.wrq.send_opt     = IB_SEND_OPT_SIGNALED;\r
272 \r
273         pSendIo->dsList.length       = 0;\r
274         pSendIo->dsList.vaddr        = PTR64( pRegionData );\r
275         pSendIo->dsList.lkey         = pData->region.lkey;\r
276 \r
277         for ( i = 0; i < pData->p_conf->numRecvs; i++ )\r
278         {\r
279                 pRecvIo[i].io.pViport         = pData->p_viport;\r
280                 pRecvIo[i].io.pRoutine        = _data_receivedKick;\r
281                 pRecvIo[i].io.r_wrq.wr_id     = PTR64( &pRecvIo[i].io );\r
282                 pRecvIo[i].io.r_wrq.p_next    = NULL;\r
283                 pRecvIo[i].io.r_wrq.num_ds    = 1;\r
284                 pRecvIo[i].io.r_wrq.ds_array  = &pRecvIo[i].dsList;\r
285                 pRecvIo[i].dsList.length      = 4;\r
286                 pRecvIo[i].dsList.vaddr       = PTR64( pRegionData );\r
287                 pRecvIo[i].dsList.lkey        = pData->region.lkey;\r
288         \r
289                 ExInterlockedInsertTailList( &pData->recvIos, &pRecvIo[i].io.listPtrs, &pData->recvIosLock );\r
290         /* Do not need to move pointer since the receive info\r
291          * is not read.  Note, we could reduce the amount\r
292          * of memory allocated and the size of the region.\r
293                  * pRegionData                  += 4;\r
294          * */\r
295         }\r
296 \r
297         sz = pRecvPool->poolSz * pRecvPool->bufferSz;\r
298         status = NdisAllocateMemoryWithTag(&pData->p_recv_bufs,\r
299                                                                                 sz, 'fubr');\r
300         if( status != NDIS_STATUS_SUCCESS )\r
301         {\r
302                 ib_status = IB_INSUFFICIENT_MEMORY;\r
303                 goto err3;\r
304         }\r
305         NdisZeroMemory( pData->p_recv_bufs, sz );\r
306 \r
307         pData->recv_bufs_sz = sz;\r
308 \r
309         ib_status = ibregion_init( pData->p_viport, &pData->rbuf_region,\r
310                 pData->p_viport->p_adapter->ca.hPd, pData->p_recv_bufs, sz,\r
311                 (IB_AC_LOCAL_WRITE | IB_AC_RDMA_WRITE) );\r
312         if( ib_status != IB_SUCCESS )\r
313         {\r
314                 goto err4;\r
315         }\r
316 \r
317         NdisAllocatePacketPool( &status,\r
318                                                         &pData->h_recv_pkt_pool,\r
319                                                         pRecvPool->poolSz,\r
320                                                         PROTOCOL_RESERVED_SIZE_IN_PACKET );\r
321 \r
322         if( status != NDIS_STATUS_SUCCESS )\r
323         {\r
324                 VNIC_TRACE_EXIT( VNIC_DBG_ERROR,\r
325                         ("Allocate packet pool failed status %#x\n", status ));\r
326                 ib_status = IB_INSUFFICIENT_MEMORY;\r
327                 goto err5;\r
328         }\r
329 \r
330         NdisAllocateBufferPool(\r
331                 &status, &pData->h_recv_buf_pool, pRecvPool->poolSz );\r
332 \r
333         if( status != NDIS_STATUS_SUCCESS )\r
334         {\r
335                 VNIC_TRACE_EXIT( VNIC_DBG_ERROR,\r
336                         ("Allocate packet pool failed status %#x\n", status ));\r
337                 ib_status = IB_INSUFFICIENT_MEMORY;\r
338                 goto err6;\r
339         }\r
340         pData->recvPool.recv_pkt_array =\r
341                 cl_zalloc( sizeof(NDIS_PACKET*)* pRecvPool->poolSz );\r
342         if( !pData->recvPool.recv_pkt_array )\r
343         {\r
344                 VNIC_TRACE_EXIT( VNIC_DBG_ERROR,\r
345                                 ("Allocate packet array failed\n" ) );\r
346                 ib_status = IB_INSUFFICIENT_MEMORY;\r
347                 goto err7;\r
348         }\r
349 \r
350         InitializeListHead( &pRecvPool->availRecvBufs );\r
351 \r
352         for ( i = 0; i < pRecvPool->poolSz; i++ )\r
353         {\r
354                 /* Allocate a region for each possible receive\r
355                  * buffer. Make sure each is large enough for\r
356                  * the maximum packet size, but initially point\r
357                  * each to a random piece of memory. As buffers\r
358                  * are allocated, the region will be modified to\r
359                  * reflect the memory space of the actual buffers.\r
360                  */\r
361                 pRdmaDest = &pRecvPool->pRecvBufs[i];\r
362                 pRdmaDest->data = pData->p_recv_bufs + (i * pRecvPool->bufferSz );\r
363                 pRdmaDest->region = pData->rbuf_region;\r
364                 InsertTailList( &pRecvPool->availRecvBufs, &pRdmaDest->listPtrs );\r
365         }\r
366 \r
367         for ( i = 0; i < pXmitPool->numXmitBufs; i++ )\r
368         {\r
369                 pRdmaIo = &pXmitPool->pXmitBufs[i];\r
370                 pRdmaIo->index               = (uint16_t)i;\r
371                 pRdmaIo->io.pViport          = pData->p_viport;\r
372                 pRdmaIo->io.pRoutine         = _data_xmitComplete;\r
373                 pRdmaIo->io.wrq.p_next       = NULL;\r
374                 pRdmaIo->io.wrq.wr_type      = WR_RDMA_WRITE;\r
375                 pRdmaIo->io.wrq.wr_id        = PTR64(pRdmaIo);\r
376                 pRdmaIo->io.wrq.num_ds       = MAX_NUM_SGE; // will set actual number when transmit\r
377                 pRdmaIo->io.wrq.ds_array     = pRdmaIo->dsList;\r
378                 pRdmaIo->p_trailer                      =  (ViportTrailer_t *)&pRdmaIo->data[0];\r
379                 for( j = 0; j < MAX_NUM_SGE; j++ )\r
380                 {\r
381                         pRdmaIo->dsList[j].lkey      = pData->p_phy_region->lkey;\r
382                 }\r
383         }\r
384 \r
385         pXmitPool->rdmaRKey      = pData->region.rkey;\r
386         pXmitPool->rdmaAddr      = PTR64( pXmitPool->bufPool );\r
387 \r
388         data_postRecvs( pData );\r
389 \r
390         ib_status = ibqp_connect( &pData->qp );\r
391         if( ib_status != IB_SUCCESS )\r
392         {\r
393                 VNIC_TRACE( VNIC_DBG_ERROR, ("ibqp_connect returned %s\n",\r
394                         pData->p_viport->p_adapter->ifc.get_err_str( ib_status )) );\r
395 err7:\r
396                 NdisFreeBufferPool( pData->h_recv_buf_pool );\r
397 err6:\r
398                 NdisFreePacketPool( pData->h_recv_pkt_pool );\r
399 err5:\r
400                 ibregion_cleanup( pData->p_viport, &pData->rbuf_region );\r
401 err4:\r
402                 NdisFreeMemory( pData->p_recv_bufs, pData->recv_bufs_sz, 0 );\r
403                 pData->p_recv_bufs = NULL;\r
404 err3:\r
405                 ibregion_cleanup(pData->p_viport, &pData->region );\r
406 err2:\r
407                 NdisFreeMemory( pData->pLocalStorage, pData->localStorageSz, 0 );\r
408                 pData->pLocalStorage = NULL;\r
409 err1:\r
410                 pRecvPool->poolSz = 0;\r
411         }\r
412 \r
413         VNIC_EXIT( VNIC_DBG_DATA | VNIC_DBG_INFO );\r
414         return ib_status;\r
415 }\r
416 \r
417 \r
418 void\r
419 data_connected(\r
420                 IN              Data_t          *pData )\r
421 {\r
422         VNIC_ENTER( VNIC_DBG_DATA | VNIC_DBG_INFO );\r
423 \r
424         pData->freeBufsIo.io.wrq.remote_ops.rkey =\r
425                                                         pData->recvPool.eiocRdmaRkey;\r
426 \r
427         _data_allocBuffers(pData, TRUE);\r
428         _data_sendFreeRecvBuffers(pData);\r
429         pData->connected = TRUE;\r
430 \r
431         VNIC_EXIT( VNIC_DBG_DATA | VNIC_DBG_INFO );\r
432         return;\r
433 }\r
434 \r
435 void\r
436 data_disconnect(\r
437                 IN              Data_t          *pData )\r
438 {\r
439         RecvPool_t *pRecvPool = &pData->recvPool;\r
440         viport_t        *p_viport = pData->p_viport;\r
441         NDIS_PACKET             *p_packet;\r
442         cl_list_item_t  *p_list_item;\r
443         unsigned int        i;\r
444 \r
445         VNIC_ENTER( VNIC_DBG_DATA | VNIC_DBG_INFO );\r
446 \r
447         pData->connected = FALSE;\r
448         _data_kickTimer_stop ( pData );\r
449 \r
450         ibqp_detach( &pData->qp );\r
451 \r
452         ibregion_cleanup( pData->p_viport, &pData->rbuf_region );\r
453         ibregion_cleanup( pData->p_viport, &pData->region );\r
454 \r
455         for ( i = 0; i < pRecvPool->poolSz; i++ )\r
456         {\r
457                 p_packet = pRecvPool->pRecvBufs[i].p_packet;\r
458                 pRecvPool->pRecvBufs[i].p_packet = NULL;\r
459 \r
460                 if ( p_packet != NULL )\r
461                 {\r
462                         _data_return_recv( p_packet );\r
463                 }\r
464         }\r
465         /* clear pending queue if any */\r
466         if( cl_qlist_count( &p_viport->send_pending_list ) )\r
467         {\r
468                 for( p_list_item = cl_qlist_remove_head( &p_viport->send_pending_list );\r
469                         p_list_item != cl_qlist_end( &p_viport->send_pending_list );\r
470                         p_list_item = cl_qlist_remove_head( &p_viport->send_pending_list ) )\r
471                 {\r
472                         p_packet = VNIC_PACKET_FROM_LIST_ITEM( p_list_item );\r
473                         NdisMSendComplete( pData->p_viport->p_adapter->h_handle,\r
474                                                                 p_packet, NDIS_STATUS_RESET_IN_PROGRESS );\r
475                 }\r
476         }\r
477 \r
478         VNIC_EXIT( VNIC_DBG_DATA | VNIC_DBG_INFO );\r
479         return;\r
480 }\r
481 \r
482 BOOLEAN\r
483 data_xmitPacket(\r
484                 IN              Data_t                          *pData,\r
485                 IN              NDIS_PACKET* const      p_packet )\r
486 {\r
487         XmitPool_t              *p_xmitPool = &pData->xmitPool;\r
488         RdmaIo_t                *pRdmaIo;\r
489         BufferPoolEntry_t *pBpe;\r
490         BOOLEAN                 last;\r
491         uint8_t                 *p_buf;\r
492         uint32_t                buf_len;\r
493         NDIS_BUFFER             *p_buf_desc;\r
494         eth_hdr_t*              p_eth_hdr;\r
495         int                             pad;\r
496         SCATTER_GATHER_LIST             *p_sgl;\r
497         uint32_t                i;\r
498         PHYSICAL_ADDRESS phy_addr;\r
499 \r
500         VNIC_ENTER( VNIC_DBG_DATA | VNIC_DBG_INFO );\r
501 \r
502         if( !data_allocXmitBuffer( pData, &pBpe, &pRdmaIo, &last ) )\r
503         {\r
504                 return FALSE;\r
505         }\r
506         p_sgl = NDIS_PER_PACKET_INFO_FROM_PACKET( p_packet,\r
507                                                                                         ScatterGatherListPacketInfo );\r
508         if ( p_sgl == NULL )\r
509         {\r
510                 return FALSE;\r
511         }\r
512 \r
513         NdisGetFirstBufferFromPacketSafe( p_packet,\r
514                                                                         &p_buf_desc,\r
515                                                                         &p_buf,\r
516                                                                         &buf_len,\r
517                                                                         &pRdmaIo->packet_sz,\r
518                                                                         NormalPagePriority );\r
519 \r
520         if( pRdmaIo->packet_sz > p_xmitPool->bufferSz )\r
521         {\r
522                 VNIC_TRACE_EXIT( VNIC_DBG_ERROR,\r
523                         ("Outbound packet too large, size = %d\n", pRdmaIo->packet_sz ) );\r
524                 return FALSE;\r
525         }\r
526 \r
527         if ( p_sgl->NumberOfElements > (ULONG)MAX_NUM_SGE - 1 )\r
528         {\r
529                 VNIC_TRACE( VNIC_DBG_DATA | VNIC_DBG_INFO,\r
530                         (" Xmit packet exceeded SGE limit - %d\n",\r
531                                                                                 p_sgl->NumberOfElements ) );\r
532                 return FALSE;\r
533         }\r
534 \r
535         for( i=0; i < p_sgl->NumberOfElements; i++ )\r
536         {\r
537                 pRdmaIo->dsList[i].vaddr = p_sgl->Elements[i].Address.QuadPart;\r
538                 pRdmaIo->dsList[i].length = p_sgl->Elements[i].Length;\r
539         }\r
540 \r
541         pRdmaIo->len = (uint32_t)ROUNDUPP2(\r
542                 max(60, pRdmaIo->packet_sz), VIPORT_TRAILER_ALIGNMENT );\r
543         pad = pRdmaIo->len - pRdmaIo->packet_sz;\r
544 \r
545         p_eth_hdr = (eth_hdr_t *)p_buf;\r
546 \r
547         pRdmaIo->p_trailer = (ViportTrailer_t *)&pRdmaIo->data[pad];\r
548         cl_memclr( pRdmaIo->data, pad + sizeof( ViportTrailer_t ) );\r
549         cl_memcpy( pRdmaIo->p_trailer->destMacAddr, p_eth_hdr->dst.addr, MAC_ADDR_LEN );\r
550 \r
551         pRdmaIo->p_trailer->dataLength =\r
552                                         hton16( (uint16_t)max( 60, pRdmaIo->packet_sz ) );\r
553 \r
554         NdisGetNextBuffer( p_buf_desc, &p_buf_desc );\r
555 \r
556         /* should handle VLAN tag */\r
557         if( pRdmaIo->packet_sz > 16 )\r
558         {\r
559                 if(     p_eth_hdr->type == hton16(0x8100) )\r
560                 {\r
561                         if( p_sgl->Elements[0].Length > sizeof(eth_hdr_t) )\r
562                         {\r
563                                 pRdmaIo->p_trailer->vLan = *(uint16_t *)((uint8_t *)p_eth_hdr + 14 );\r
564                                 pRdmaIo->p_trailer->pktFlags |= PF_VLAN_INSERT;\r
565                         }\r
566                         else\r
567                         {\r
568                                 if( p_buf_desc )\r
569                                 {\r
570                                         NdisQueryBufferSafe( p_buf_desc, &p_buf, &buf_len, NormalPagePriority );\r
571 \r
572                                         pad = sizeof(eth_hdr_t) - p_sgl->Elements[0].Length;\r
573                                         pRdmaIo->p_trailer->vLan = *(uint16_t *)(p_buf + pad + 2);\r
574                                         pRdmaIo->p_trailer->pktFlags |= PF_VLAN_INSERT;\r
575                                 }\r
576                         }\r
577                 }\r
578                 else if( p_eth_hdr->type == ETH_PROT_TYPE_IP &&\r
579                                 !( p_eth_hdr->dst.addr[0] & 0x01 ) )\r
580                 {\r
581                         if( p_buf_desc )\r
582                         {\r
583                                 NdisQueryBufferSafe( p_buf_desc, &p_buf, &buf_len, NormalPagePriority );\r
584 \r
585                                 if( ((ip_pkt_t*)p_buf)->hdr.prot == IP_PROT_UDP ||\r
586                                         ((ip_pkt_t*)p_buf)->hdr.prot == IP_PROT_TCP )\r
587                                 {\r
588                                         /* use socket src port + dest port to generate hash value\r
589                                         * for link aggregation distribution function.\r
590                                         */\r
591                                         pRdmaIo->p_trailer->connectionHashAndValid = 0x40 |\r
592                                                 ((uint8_t)((ip_pkt_t*)p_buf)->prot.tcp.src_port +\r
593                                                 (uint8_t)((ip_pkt_t*)p_buf)->prot.tcp.dst_port ) & 0x3f;\r
594                                 }\r
595                         }\r
596                 }\r
597         }\r
598 \r
599         pRdmaIo->p_trailer->txChksumFlags = _tx_chksum_flags( p_packet );\r
600         pRdmaIo->p_trailer->connectionHashAndValid |= CHV_VALID;\r
601 \r
602         if( last )\r
603                 pRdmaIo->p_trailer->pktFlags |= PF_KICK;\r
604 \r
605         /* fill last data segment with trailer and pad */\r
606         phy_addr = MmGetPhysicalAddress( pRdmaIo->data );\r
607 \r
608         pRdmaIo->dsList[p_sgl->NumberOfElements].vaddr = phy_addr.QuadPart;\r
609         pRdmaIo->dsList[p_sgl->NumberOfElements].length = pRdmaIo->len -\r
610                                                                                                           pRdmaIo->packet_sz +\r
611                                                                                                           sizeof( ViportTrailer_t );\r
612         //pRdmaIo->io.wrq.ds_array = pRdmaIo->dsList;\r
613     pRdmaIo->io.wrq.num_ds =p_sgl->NumberOfElements + 1;\r
614 \r
615         data_rdmaPacket( pData, pBpe, pRdmaIo );\r
616 \r
617         if( p_xmitPool->sendKicks )\r
618         {\r
619                 /* EIOC needs kicks to inform it of sent packets */\r
620 \r
621                 p_xmitPool->kickCount++;\r
622                 p_xmitPool->kickByteCount += pRdmaIo->packet_sz;\r
623                 if( ( p_xmitPool->kickCount >= p_xmitPool->kickBundle )\r
624                          || ( p_xmitPool->kickByteCount >= p_xmitPool->kickByteBundle ) )\r
625                 {\r
626                         data_sendKickMessage( pData );\r
627                 }\r
628                 else if( p_xmitPool->kickCount == 1 )\r
629                 {\r
630                         _data_kickTimer_start( pData, pData->eiocPoolParms.timeoutBeforeKick );\r
631                 }\r
632         }\r
633         return TRUE;\r
634 }\r
635 static uint8_t\r
636 _tx_chksum_flags(\r
637                   IN    NDIS_PACKET* const              p_packet )\r
638 \r
639 {\r
640         NDIS_TCP_IP_CHECKSUM_PACKET_INFO        *p_packet_info;\r
641         ULONG                                                           packet_info;\r
642         uint8_t                 txChksumFlags = 0;\r
643 \r
644         if( NDIS_PROTOCOL_ID_TCP_IP == NDIS_GET_PACKET_PROTOCOL_TYPE(p_packet) )\r
645         {\r
646                 packet_info = PtrToUlong( NDIS_PER_PACKET_INFO_FROM_PACKET( p_packet, TcpIpChecksumPacketInfo));\r
647                 p_packet_info = ( NDIS_TCP_IP_CHECKSUM_PACKET_INFO *)&packet_info;\r
648 \r
649                 if( p_packet_info &&\r
650                         p_packet_info->Transmit.NdisPacketChecksumV4 )\r
651                 {\r
652                         txChksumFlags = TX_CHKSUM_FLAGS_CHECKSUM_V4\r
653                         | ( p_packet_info->Transmit.NdisPacketIpChecksum ? TX_CHKSUM_FLAGS_IP_CHECKSUM: 0 )\r
654                         | ( p_packet_info->Transmit.NdisPacketTcpChecksum ? TX_CHKSUM_FLAGS_TCP_CHECKSUM: 0 )\r
655                         | ( p_packet_info->Transmit.NdisPacketUdpChecksum ? TX_CHKSUM_FLAGS_UDP_CHECKSUM: 0 );\r
656                 }\r
657         }\r
658         \r
659         VNIC_TRACE( VNIC_DBG_DATA | VNIC_DBG_INFO ,\r
660                                 ("txChksumFlags = %d: V4 %c, V6 %c, IP %c, TCP %c, UDP %c\n",\r
661                                 txChksumFlags,\r
662                                 ((txChksumFlags & TX_CHKSUM_FLAGS_CHECKSUM_V4 )? '+': '-'),\r
663                                 ((txChksumFlags & TX_CHKSUM_FLAGS_CHECKSUM_V6 )? '+': '-'),\r
664                                 ((txChksumFlags & TX_CHKSUM_FLAGS_IP_CHECKSUM )? '+': '-'),\r
665                                 ((txChksumFlags & TX_CHKSUM_FLAGS_TCP_CHECKSUM )? '+': '-'),\r
666                                 ((txChksumFlags & TX_CHKSUM_FLAGS_UDP_CHECKSUM )? '+': '-') ));\r
667 \r
668         return txChksumFlags;\r
669 }\r
670 \r
671 static void\r
672 _get_first_buffer(\r
673                 IN              NDIS_PACKET             *p_packet,\r
674                 IN OUT  NDIS_BUFFER             **pp_buf_desc,\r
675                 OUT             void                    **pp_buf,\r
676                 OUT             ULONG                   *p_packet_sz )\r
677 {\r
678         UINT            buf_len;\r
679         VNIC_ENTER( VNIC_DBG_DATA | VNIC_DBG_INFO );\r
680 \r
681         NdisGetFirstBufferFromPacketSafe( p_packet,\r
682                                                                         pp_buf_desc,\r
683                                                                         pp_buf,\r
684                                                                         &buf_len,\r
685                                                                         p_packet_sz,\r
686                                                                         NormalPagePriority );\r
687         VNIC_EXIT( VNIC_DBG_DATA | VNIC_DBG_INFO );\r
688 }\r
689 \r
690 static void\r
691 data_postRecvs(\r
692                 IN              Data_t          *pData )\r
693 {\r
694         RecvIo_t                *pRecvIo;\r
695         LIST_ENTRY              *p_list_entry;\r
696         ib_api_status_t ib_status;\r
697 \r
698         VNIC_ENTER ( VNIC_DBG_DATA | VNIC_DBG_INFO );\r
699 \r
700         while( ( p_list_entry = ExInterlockedRemoveHeadList( &pData->recvIos,\r
701                                                                                                                  &pData->recvIosLock ))\r
702                                                                                                                 != NULL )\r
703         {\r
704                 pRecvIo = (RecvIo_t *)p_list_entry;\r
705 \r
706                 ib_status = ibqp_postRecv( &pData->qp, &pRecvIo->io );\r
707                 if( ib_status != IB_SUCCESS )\r
708                 {\r
709                         VNIC_TRACE_EXIT( VNIC_DBG_ERROR, ("ibqp_postRecv returned %s\n",\r
710                                 pData->p_viport->p_adapter->ifc.get_err_str( ib_status )) );\r
711                         viport_failure( pData->p_viport );\r
712                         return;\r
713                 }\r
714         }\r
715 \r
716         VNIC_EXIT( VNIC_DBG_DATA | VNIC_DBG_INFO );\r
717         return;\r
718 }\r
719 \r
720 static void\r
721 _data_receivedKick(\r
722                         IN              Io_t            *pIo )\r
723 {\r
724         Data_t        *pData = &pIo->pViport->data;\r
725         uint32_t                num_pkts = 0;\r
726 \r
727         VNIC_ENTER( VNIC_DBG_DATA | VNIC_DBG_INFO );\r
728 \r
729 #ifdef VNIC_STATISTIC\r
730         recvRef = cl_get_tick_count();\r
731 #endif /* VNIC_STATISTIC */\r
732 \r
733         ExInterlockedInsertTailList( &pData->recvIos, &pIo->listPtrs, &pData->recvIosLock );\r
734 \r
735         data_postRecvs( pData );\r
736 \r
737 #ifdef VNIC_STATISTIC\r
738         pData->statistics.kickRecvs++;\r
739 #endif /* VNIC_STATISTIC */\r
740 \r
741         data_checkXmitBuffers( pData );\r
742 \r
743         num_pkts = _data_incomingRecv( pData );\r
744 \r
745         if( num_pkts )\r
746         {\r
747                 NdisMIndicateReceivePacket( pData->p_viport->p_adapter->h_handle,\r
748                                                                         pData->recvPool.recv_pkt_array,\r
749                                                                         num_pkts );\r
750                 pData->p_viport->stats.ifInOk++;\r
751         }\r
752 \r
753         VNIC_EXIT( VNIC_DBG_DATA | VNIC_DBG_INFO );\r
754         return;\r
755 }\r
756 \r
757 static void\r
758 _data_xmitComplete(\r
759                         IN              Io_t            *pIo )\r
760 {\r
761         RdmaIo_t       *pRdmaIo = (RdmaIo_t *)pIo;\r
762         Data_t         *pData = &pIo->pViport->data;\r
763         XmitPool_t     *p_xmitPool = &pData->xmitPool;\r
764         vnic_adapter_t  *p_adapter = pIo->pViport->p_adapter;\r
765         NDIS_PACKET             *p_packet;\r
766         NDIS_STATUS             ndis_status;\r
767         cl_list_item_t  *p_list_item;\r
768 \r
769         VNIC_ENTER( VNIC_DBG_DATA | VNIC_DBG_INFO );\r
770 \r
771         while ( p_xmitPool->lastCompBuf != pRdmaIo->index )\r
772         {\r
773                 INC(p_xmitPool->lastCompBuf, 1, p_xmitPool->numXmitBufs);\r
774                 p_packet = p_xmitPool->pXmitBufs[p_xmitPool->lastCompBuf].p_packet;\r
775         \r
776                 p_xmitPool->pXmitBufs[p_xmitPool->lastCompBuf].p_packet = NULL;\r
777 \r
778                 if( p_packet != NULL )\r
779                 {\r
780                         if( pIo->wc_status != IB_WCS_SUCCESS )\r
781                         {\r
782                                 ndis_status = NDIS_STATUS_FAILURE;\r
783                                 p_adapter->p_viport->stats.ifOutErrors++;\r
784                         }\r
785                         else\r
786                         {\r
787                                 ndis_status = NDIS_STATUS_SUCCESS;\r
788                                 p_adapter->p_viport->stats.ifOutOk++;\r
789                         }\r
790                         NDIS_SET_PACKET_STATUS( p_packet, ndis_status );\r
791                         NdisMSendComplete( pIo->pViport->p_adapter->h_handle,\r
792                                                                 p_packet, ndis_status );\r
793                 }\r
794         }\r
795         pIo->wc_status = IB_WCS_SUCCESS;\r
796         if( !p_adapter->packet_filter )\r
797         {\r
798                 if( cl_qlist_count( &pIo->pViport->send_pending_list ) )\r
799                 {\r
800                         for( p_list_item = cl_qlist_remove_head( &pIo->pViport->send_pending_list );\r
801                                 p_list_item != cl_qlist_end( &pIo->pViport->send_pending_list );\r
802                                 p_list_item = cl_qlist_remove_head( &pIo->pViport->send_pending_list ) )\r
803                         {\r
804                                 p_packet = VNIC_PACKET_FROM_LIST_ITEM( p_list_item );\r
805                                 NdisMSendComplete( p_adapter->h_handle, p_packet,\r
806                                                                         NDIS_STATUS_RESET_IN_PROGRESS );\r
807                         }\r
808                 }\r
809         }\r
810         data_checkXmitBuffers(pData);\r
811 \r
812         VNIC_EXIT( VNIC_DBG_DATA | VNIC_DBG_INFO );\r
813         return;\r
814 }\r
815 \r
816 static void\r
817 data_sendKickMessage(\r
818                 IN              Data_t          *pData )\r
819 {\r
820         XmitPool_t *pPool = &pData->xmitPool;\r
821 \r
822         VNIC_ENTER( VNIC_DBG_DATA | VNIC_DBG_INFO );\r
823 \r
824         /* stop timer for BundleTimeout */\r
825         _data_kickTimer_stop( pData );\r
826 \r
827         pPool->kickCount = 0;\r
828         pPool->kickByteCount = 0;\r
829 \r
830         /* TBD: Keep track of when kick is outstanding, and\r
831          * don't reuse until complete\r
832          */\r
833         if ( ibqp_postSend( &pData->qp, &pData->kickIo.io ) != IB_SUCCESS )\r
834         {\r
835                 VNIC_TRACE( VNIC_DBG_ERROR,\r
836                                         ("Unable to send kick to EIOC\n") );\r
837                 viport_failure( pData->p_viport );\r
838         }\r
839 \r
840         VNIC_EXIT( VNIC_DBG_DATA | VNIC_DBG_INFO );\r
841 }\r
842 \r
843 static void\r
844 _data_kickTimeoutHandler( void * context )\r
845 {\r
846         Data_t* pData = (Data_t *)context;\r
847 \r
848         VNIC_ENTER( VNIC_DBG_DATA | VNIC_DBG_INFO );\r
849 \r
850         pData->kickTimerOn = FALSE;\r
851         data_sendKickMessage( pData );\r
852 \r
853         VNIC_EXIT( VNIC_DBG_DATA | VNIC_DBG_INFO );\r
854 \r
855         return;\r
856 }\r
857 \r
858 static BOOLEAN\r
859 data_allocXmitBuffer(\r
860                 IN              Data_t                          *pData,\r
861                 OUT             BufferPoolEntry_t       **ppBpe,\r
862                 OUT             RdmaIo_t                        **ppRdmaIo,\r
863                 OUT             BOOLEAN                         *pLast )\r
864 {\r
865         XmitPool_t    *p_xmitPool = &pData->xmitPool;\r
866         KIRQL flags;\r
867 \r
868         VNIC_ENTER( VNIC_DBG_DATA | VNIC_DBG_INFO );\r
869 \r
870         KeAcquireSpinLock( &pData->xmitBufLock, &flags );\r
871 \r
872         *pLast = FALSE;\r
873         *ppRdmaIo = &p_xmitPool->pXmitBufs[p_xmitPool->nextXmitBuf];\r
874         *ppBpe = &p_xmitPool->bufPool[p_xmitPool->nextXmitPool];\r
875 \r
876         if ( (*ppBpe)->valid && p_xmitPool->nextXmitBuf != p_xmitPool->lastCompBuf )\r
877         {\r
878                 INC(p_xmitPool->nextXmitBuf, 1, p_xmitPool->numXmitBufs);\r
879                 INC(p_xmitPool->nextXmitPool, 1, p_xmitPool->poolSz);\r
880 \r
881                 if ( !p_xmitPool->bufPool[p_xmitPool->nextXmitPool].valid )\r
882                 {\r
883                         VNIC_TRACE( VNIC_DBG_DATA | VNIC_DBG_INFO,\r
884                                 ("Just used the last EIOU receive buffer\n") );\r
885 \r
886                         *pLast = TRUE;\r
887                         p_xmitPool->needBuffers = TRUE;\r
888                         viport_stopXmit( pData->p_viport );\r
889 #ifdef VNIC_STATISTIC\r
890                         pData->statistics.kickReqs++;\r
891 #endif /* VNIC_STATISTIC */\r
892                 }\r
893                 else if ( p_xmitPool->nextXmitBuf == p_xmitPool->lastCompBuf )\r
894                 {\r
895                         VNIC_TRACE( VNIC_DBG_DATA | VNIC_DBG_INFO,\r
896                                                 ("Just used our last xmit buffer\n") );\r
897                 \r
898                         p_xmitPool->needBuffers = TRUE;\r
899                         viport_stopXmit( pData->p_viport );\r
900                 }\r
901 \r
902                 (*ppBpe)->valid  = 0;\r
903 \r
904                 KeReleaseSpinLock( &pData->xmitBufLock, flags );\r
905                 return TRUE;\r
906         }\r
907         else\r
908         {\r
909 #ifdef VNIC_STATISTIC\r
910                 pData->statistics.noXmitBufs++;\r
911 #endif /* VNIC_STATISTIC */\r
912         \r
913                 VNIC_TRACE( VNIC_DBG_ERROR,\r
914                                         ("Out of xmit buffers\n") );\r
915 \r
916                 viport_stopXmit( pData->p_viport );\r
917 \r
918                 KeReleaseSpinLock( &pData->xmitBufLock, flags );\r
919                 return FALSE;\r
920         }\r
921 }\r
922 \r
923 static void\r
924 data_checkXmitBuffers(\r
925                 IN                      Data_t          *pData )\r
926 {\r
927         XmitPool_t      *p_xmitPool = &pData->xmitPool;\r
928         KIRQL           flags;\r
929 \r
930         VNIC_ENTER( VNIC_DBG_DATA | VNIC_DBG_INFO );\r
931 \r
932         KeAcquireSpinLock( &pData->xmitBufLock, &flags );\r
933 \r
934         if ( pData->xmitPool.needBuffers\r
935                 && p_xmitPool->bufPool[p_xmitPool->nextXmitPool].valid\r
936                 && p_xmitPool->nextXmitBuf != p_xmitPool->lastCompBuf )\r
937         {\r
938                 pData->xmitPool.needBuffers = FALSE;\r
939                 viport_restartXmit( pData->p_viport );\r
940 \r
941                 VNIC_TRACE( VNIC_DBG_DATA | VNIC_DBG_INFO,\r
942                                                 ("There are free xmit buffers\n") );\r
943         }\r
944 \r
945         KeReleaseSpinLock( &pData->xmitBufLock, flags );\r
946 \r
947         VNIC_EXIT( VNIC_DBG_DATA | VNIC_DBG_INFO );\r
948         return;\r
949 }\r
950 \r
951 static void\r
952 data_rdmaPacket(\r
953                         IN                      Data_t                          *pData,\r
954                         IN                      BufferPoolEntry_t       *pBpe,\r
955                         IN                      RdmaIo_t                        *pRdmaIo )\r
956 {\r
957         ib_send_wr_t    *pWrq;\r
958         uint64_t                remote_addr;\r
959 \r
960         VNIC_ENTER( VNIC_DBG_DATA | VNIC_DBG_INFO );\r
961 \r
962         pWrq = &pRdmaIo->io.wrq;\r
963 \r
964         remote_addr  = ntoh64( pBpe->remoteAddr );\r
965         remote_addr += pData->xmitPool.bufferSz;\r
966         remote_addr -= ( pRdmaIo->len + sizeof(ViportTrailer_t) );\r
967 \r
968         pWrq->remote_ops.vaddr          = remote_addr;\r
969         pWrq->remote_ops.rkey           = pBpe->rKey;\r
970 \r
971         pData->xmitPool.notifyCount++;\r
972 \r
973         if( pData->xmitPool.notifyCount >= pData->xmitPool.notifyBundle )\r
974         {\r
975                 pData->xmitPool.notifyCount = 0;\r
976                 pWrq->send_opt = IB_SEND_OPT_SIGNALED;\r
977         }\r
978         else\r
979         {\r
980                 pWrq->send_opt &= ~IB_SEND_OPT_SIGNALED;\r
981         }\r
982         pWrq->send_opt = IB_SEND_OPT_SIGNALED;\r
983 \r
984         if( ibqp_postSend( &pData->qp, &pRdmaIo->io ) != IB_SUCCESS )\r
985         {\r
986                 VNIC_TRACE(VNIC_DBG_ERROR,\r
987                                         ("Failed sending data to EIOC\n") );\r
988                 viport_failure( pData->p_viport );\r
989                 return;\r
990         }\r
991 #ifdef VNIC_STATISTIC\r
992         pData->statistics.xmitNum++;\r
993 #endif /* VNIC_STATISTIC */\r
994 \r
995         VNIC_EXIT( VNIC_DBG_DATA | VNIC_DBG_INFO );\r
996 }\r
997 \r
998 static NDIS_PACKET *\r
999 _data_recv_to_ndis_pkt(\r
1000                 IN              Data_t                  *pData,\r
1001                 IN              RdmaDest_t              *pRdmaDest )\r
1002 {\r
1003         struct ViportTrailer *pTrailer;\r
1004         NDIS_PACKET             *p_packet;\r
1005         NDIS_STATUS             ndis_status;\r
1006         int                  start;\r
1007         unsigned int         len;\r
1008         uint8_t rxChksumFlags;\r
1009         NDIS_TCP_IP_CHECKSUM_PACKET_INFO  packet_info;\r
1010 \r
1011         VNIC_ENTER( VNIC_DBG_DATA | VNIC_DBG_INFO );\r
1012 \r
1013         pTrailer = pRdmaDest->pTrailer;\r
1014         start = (int)data_offset(pData, pTrailer);\r
1015         len = data_len(pData, pTrailer);\r
1016 \r
1017         NdisAllocatePacket( &ndis_status,\r
1018                                                 &p_packet,\r
1019                                                 pData->h_recv_pkt_pool );\r
1020 \r
1021         if ( ndis_status != NDIS_STATUS_SUCCESS )\r
1022         {\r
1023                 VNIC_TRACE_EXIT( VNIC_DBG_ERROR,\r
1024                         ( "NdisAllocatePacket failed %#x\n", ndis_status ) );\r
1025                 return NULL;\r
1026         }\r
1027         NdisAllocateBuffer( &ndis_status,\r
1028                                                 &pRdmaDest->p_buf,\r
1029                                                 pData->h_recv_buf_pool,\r
1030                                                 pRdmaDest->data + start,\r
1031                                                 len );\r
1032 \r
1033         if ( ndis_status != NDIS_STATUS_SUCCESS )\r
1034         {\r
1035                 VNIC_TRACE_EXIT( VNIC_DBG_ERROR,\r
1036                         ( "NdisAllocateBuffer failed %#x\n", ndis_status ) );\r
1037                 NdisFreePacket( p_packet );\r
1038                 return NULL;\r
1039         }\r
1040 \r
1041         NdisChainBufferAtFront( p_packet, pRdmaDest->p_buf );\r
1042         pRdmaDest->p_packet = p_packet;\r
1043 \r
1044         if ( pTrailer->pktFlags & PF_VLAN_INSERT )\r
1045         {\r
1046                 /*      TODO:\r
1047                  *      add OID_GEN_VLAN_ID\r
1048                  *      handle VLAN     tag insertion\r
1049                  *      set packet header size = eth_hdr + 4\r
1050                  */\r
1051         }\r
1052 \r
1053         NDIS_SET_PACKET_HEADER_SIZE( p_packet, sizeof(eth_hdr_t) );\r
1054 \r
1055         rxChksumFlags = pTrailer->rxChksumFlags;\r
1056 \r
1057         VNIC_TRACE( VNIC_DBG_DATA | VNIC_DBG_INFO,\r
1058                 ("rxChksumFlags = %d, LOOP = %c, IP = %c, TCP = %c, UDP = %c\n",\r
1059                 rxChksumFlags,\r
1060                 (rxChksumFlags & RX_CHKSUM_FLAGS_LOOPBACK)? 'Y': 'N',\r
1061                 (rxChksumFlags & RX_CHKSUM_FLAGS_IP_CHECKSUM_SUCCEEDED)? 'Y':\r
1062                 (rxChksumFlags & RX_CHKSUM_FLAGS_IP_CHECKSUM_FAILED)? 'N': '-',\r
1063                 (rxChksumFlags & RX_CHKSUM_FLAGS_TCP_CHECKSUM_SUCCEEDED)? 'Y':\r
1064                 (rxChksumFlags & RX_CHKSUM_FLAGS_TCP_CHECKSUM_FAILED)? 'N': '-',\r
1065                 (rxChksumFlags & RX_CHKSUM_FLAGS_UDP_CHECKSUM_SUCCEEDED)? 'Y':\r
1066                 (rxChksumFlags & RX_CHKSUM_FLAGS_UDP_CHECKSUM_FAILED)? 'N': '-') );\r
1067 \r
1068         packet_info.Value = 0;\r
1069 \r
1070         if( rxChksumFlags & RX_CHKSUM_FLAGS_IP_CHECKSUM_SUCCEEDED )\r
1071                 packet_info.Receive.NdisPacketIpChecksumSucceeded = TRUE;\r
1072         else if( rxChksumFlags & RX_CHKSUM_FLAGS_IP_CHECKSUM_FAILED )\r
1073                 packet_info.Receive.NdisPacketIpChecksumFailed = TRUE;\r
1074 \r
1075         if( rxChksumFlags & RX_CHKSUM_FLAGS_TCP_CHECKSUM_SUCCEEDED )\r
1076                 packet_info.Receive.NdisPacketTcpChecksumSucceeded = TRUE;\r
1077         else if( rxChksumFlags & RX_CHKSUM_FLAGS_TCP_CHECKSUM_FAILED )\r
1078                 packet_info.Receive.NdisPacketTcpChecksumFailed = TRUE;\r
1079 \r
1080         if( rxChksumFlags & RX_CHKSUM_FLAGS_TCP_CHECKSUM_SUCCEEDED )\r
1081                 packet_info.Receive.NdisPacketUdpChecksumSucceeded = TRUE;\r
1082         else if( rxChksumFlags & RX_CHKSUM_FLAGS_TCP_CHECKSUM_FAILED )\r
1083                 packet_info.Receive.NdisPacketUdpChecksumFailed = TRUE;\r
1084 \r
1085         NDIS_PER_PACKET_INFO_FROM_PACKET( p_packet, TcpIpChecksumPacketInfo )=\r
1086                                                                                 (void *)(uintn_t)packet_info.Value;\r
1087 \r
1088         VNIC_RECV_FROM_PACKET( p_packet ) = pRdmaDest;\r
1089         NDIS_SET_PACKET_STATUS( p_packet, NDIS_STATUS_SUCCESS );\r
1090 \r
1091         VNIC_EXIT( VNIC_DBG_DATA | VNIC_DBG_INFO );\r
1092         return p_packet;\r
1093 }\r
1094 \r
1095 /* NOTE: This routine is not reentrant */\r
1096 static void\r
1097 _data_allocBuffers(\r
1098                 IN                      Data_t          *pData,\r
1099                 IN                      BOOLEAN         initialAllocation )\r
1100 {\r
1101         RecvPool_t     *p_recvPool = &pData->recvPool;\r
1102         RdmaDest_t     *pRdmaDest;\r
1103         LIST_ENTRY              *p_list_entry;\r
1104         int            index;\r
1105 \r
1106         VNIC_ENTER( VNIC_DBG_DATA | VNIC_DBG_INFO );\r
1107 \r
1108         index = ADD(p_recvPool->nextFreeBuf, p_recvPool->numFreeBufs, p_recvPool->eiocPoolSz);\r
1109 \r
1110         while ( !IsListEmpty( &p_recvPool->availRecvBufs ) )\r
1111         {\r
1112                 p_list_entry = RemoveHeadList( &p_recvPool->availRecvBufs );\r
1113                 pRdmaDest       = (RdmaDest_t*)p_list_entry;\r
1114         \r
1115                 if( initialAllocation )\r
1116                 {\r
1117                         pRdmaDest->buf_sz = p_recvPool->bufferSz;\r
1118                         pRdmaDest->pTrailer =\r
1119                                 (struct ViportTrailer*)(pRdmaDest->data + pRdmaDest->buf_sz\r
1120                                 - sizeof(struct ViportTrailer));\r
1121                         pRdmaDest->pTrailer->connectionHashAndValid = 0;\r
1122                 }\r
1123 \r
1124                 pRdmaDest->p_packet = NULL;\r
1125                 _data_addFreeBuffer(pData, index, pRdmaDest);\r
1126                 index = NEXT(index,p_recvPool->eiocPoolSz);\r
1127         }\r
1128 \r
1129         VNIC_EXIT( VNIC_DBG_DATA | VNIC_DBG_INFO );\r
1130         return;\r
1131 }\r
1132 \r
1133 static void\r
1134 _data_addFreeBuffer(\r
1135                 IN              Data_t                  *pData,\r
1136                 IN              int                             index,\r
1137             IN          RdmaDest_t              *pRdmaDest )\r
1138 {\r
1139         RecvPool_t        *p_recvPool = &pData->recvPool;\r
1140         BufferPoolEntry_t *pBpe;\r
1141 \r
1142         pRdmaDest->pTrailer->connectionHashAndValid = 0;\r
1143         pBpe = &p_recvPool->bufPool[index];\r
1144 \r
1145         pBpe->rKey       = pRdmaDest->region.rkey;\r
1146         pBpe->remoteAddr = hton64( PTR64( pRdmaDest->data ) );\r
1147         pBpe->valid      = (uint32_t)(pRdmaDest - &p_recvPool->pRecvBufs[0]) + 1;\r
1148         ++p_recvPool->numFreeBufs;\r
1149 \r
1150         return;\r
1151 }\r
1152 \r
1153 static uint32_t\r
1154 _data_incomingRecv(\r
1155                 IN              Data_t          *pData )\r
1156 {\r
1157         RecvPool_t        *p_recvPool = &pData->recvPool;\r
1158         RdmaDest_t        *pRdmaDest;\r
1159         ViportTrailer_t   *pTrailer;\r
1160         BufferPoolEntry_t *pBpe;\r
1161         NDIS_PACKET             *p_packet = NULL;\r
1162         uint32_t                idx = 0;\r
1163         BOOLEAN status = FALSE;\r
1164 \r
1165         VNIC_ENTER( VNIC_DBG_DATA | VNIC_DBG_INFO );\r
1166 \r
1167         while( !status )\r
1168         {\r
1169                 if ( p_recvPool->nextFullBuf == p_recvPool->nextFreeBuf )\r
1170                         return idx;\r
1171 \r
1172                 pBpe = &p_recvPool->bufPool[p_recvPool->nextFullBuf];\r
1173                 pRdmaDest = &p_recvPool->pRecvBufs[pBpe->valid - 1];\r
1174                 pTrailer = pRdmaDest->pTrailer;\r
1175 \r
1176                 if ( ( pTrailer != NULL ) &&\r
1177                         ( pTrailer->connectionHashAndValid & CHV_VALID ) )\r
1178                 {\r
1179                         /* received a packet */\r
1180                         if ( pTrailer->pktFlags & PF_KICK )\r
1181                         {\r
1182                                 p_recvPool->kickOnFree = TRUE;\r
1183                         }\r
1184                         /* we do not want to indicate packet if  no filter is set */\r
1185                         if( pData->p_viport->p_adapter->packet_filter )\r
1186                         {\r
1187                                 p_packet = _data_recv_to_ndis_pkt( pData, pRdmaDest );\r
1188                                 if ( p_packet != NULL )\r
1189                                 {\r
1190                                         p_recvPool->recv_pkt_array[idx++] = p_packet;\r
1191                                 }\r
1192                         }\r
1193                         else\r
1194                         {       /* put back to free buffers pool */\r
1195                                 InsertTailList( &p_recvPool->availRecvBufs,\r
1196                                         &pRdmaDest->listPtrs );\r
1197                         }\r
1198                         pBpe->valid = 0;\r
1199                         INC( p_recvPool->nextFullBuf, 1, p_recvPool->eiocPoolSz );\r
1200                         p_recvPool->numPostedBufs--;\r
1201 #ifdef VNIC_STATISTIC\r
1202                         pData->statistics.recvNum++;\r
1203 #endif /* VNIC_STATISTIC */\r
1204                 }\r
1205                 else\r
1206                         break;\r
1207         }\r
1208         return idx;\r
1209 }\r
1210 \r
1211 \r
1212 void\r
1213 vnic_return_packet(\r
1214         IN              NDIS_HANDLE             adapter_context,\r
1215         IN              NDIS_PACKET     * const p_packet )\r
1216 {\r
1217 \r
1218 \r
1219         vnic_adapter_t  *p_adapter = (vnic_adapter_t *)adapter_context;\r
1220         viport_t        *p_viport = p_adapter->p_currentPath->pViport;\r
1221         RdmaDest_t      *p_rdma_dest = VNIC_RECV_FROM_PACKET( p_packet );\r
1222 \r
1223         ASSERT( p_rdma_dest->p_packet == p_packet );\r
1224         _data_return_recv( p_packet );\r
1225         p_rdma_dest->p_packet = NULL;\r
1226 \r
1227         InsertTailList( &p_viport->data.recvPool.availRecvBufs,\r
1228                                         &p_rdma_dest->listPtrs );\r
1229 \r
1230         if( p_viport->data.connected == TRUE )\r
1231         {\r
1232                 _data_allocBuffers( &p_viport->data, FALSE );\r
1233                 _data_sendFreeRecvBuffers( &p_viport->data );\r
1234         }\r
1235 }\r
1236 static void\r
1237 _data_return_recv(\r
1238           IN    NDIS_PACKET             *p_packet )\r
1239 {\r
1240         NDIS_BUFFER             *p_buf;\r
1241         /* Unchain the NDIS buffer. */\r
1242         NdisUnchainBufferAtFront( p_packet, &p_buf );\r
1243         CL_ASSERT( p_buf );\r
1244         /* Return the NDIS packet and NDIS buffer to their pools. */\r
1245         NdisFreeBuffer( p_buf );\r
1246         NdisFreePacket( p_packet );\r
1247 }\r
1248 \r
1249 static void\r
1250 _data_sendFreeRecvBuffers(\r
1251                         IN                      Data_t          *pData )\r
1252 {\r
1253         RecvPool_t  *p_recvPool = &pData->recvPool;\r
1254         ib_send_wr_t *pWrq = &pData->freeBufsIo.io.wrq;\r
1255         BOOLEAN     bufsSent = FALSE;\r
1256         uint64_t    rdmaAddr;\r
1257         uint32_t    offset;\r
1258         uint32_t    sz;\r
1259         unsigned int numToSend,\r
1260                     nextIncrement;\r
1261 \r
1262         VNIC_ENTER( VNIC_DBG_DATA | VNIC_DBG_INFO );\r
1263 \r
1264         for (   numToSend = p_recvPool->szFreeBundle;\r
1265                         numToSend <= p_recvPool->numFreeBufs;\r
1266                         numToSend += p_recvPool->szFreeBundle )\r
1267         {\r
1268                 /* Handle multiple bundles as one when possible. */\r
1269                 nextIncrement = numToSend + p_recvPool->szFreeBundle;\r
1270                 if (    ( nextIncrement <= p_recvPool->numFreeBufs )\r
1271                                 && ( p_recvPool->nextFreeBuf + nextIncrement <= p_recvPool->eiocPoolSz ) )\r
1272                 {\r
1273                         continue;\r
1274                 }\r
1275 \r
1276                 offset   = p_recvPool->nextFreeBuf * sizeof(BufferPoolEntry_t);\r
1277                 sz       = numToSend * sizeof(BufferPoolEntry_t);\r
1278                 rdmaAddr = p_recvPool->eiocRdmaAddr + offset;\r
1279         \r
1280                 //pWrq->MessageLen      = sz;\r
1281                 pWrq->ds_array->length  = sz;\r
1282                 pWrq->ds_array->vaddr = PTR64((uint8_t *)p_recvPool->bufPool + offset);\r
1283                 pWrq->remote_ops.vaddr = rdmaAddr;\r
1284 \r
1285                 if ( ibqp_postSend( &pData->qp, &pData->freeBufsIo.io ) != IB_SUCCESS )\r
1286                 {\r
1287                         VNIC_TRACE(VNIC_DBG_ERROR,\r
1288                                         ("Unable to rdma free buffers to EIOC\n") );\r
1289 \r
1290                         viport_failure( pData->p_viport );\r
1291                         break;\r
1292                 }\r
1293 \r
1294                 INC( p_recvPool->nextFreeBuf, numToSend, p_recvPool->eiocPoolSz );\r
1295                 p_recvPool->numFreeBufs -= numToSend;\r
1296                 p_recvPool->numPostedBufs += numToSend;\r
1297                 bufsSent = TRUE;\r
1298         }\r
1299 \r
1300         if( bufsSent )\r
1301         {\r
1302                 if( p_recvPool->kickOnFree )\r
1303                 {\r
1304                         data_sendKickMessage( pData );\r
1305                 }\r
1306         }\r
1307         if( p_recvPool->numPostedBufs == 0 )\r
1308         {\r
1309                 VNIC_TRACE( VNIC_DBG_ERROR,\r
1310                                 ("%s: Unable to allocate receive buffers\n",\r
1311                                                         pData->p_viport->p_adapter->name ) );\r
1312         \r
1313                 viport_failure( pData->p_viport );\r
1314         }\r
1315         VNIC_EXIT( VNIC_DBG_DATA | VNIC_DBG_INFO );\r
1316 }\r
1317 \r
1318 \r
1319 void\r
1320 data_cleanup(\r
1321         IN                      Data_t  *pData )\r
1322 {\r
1323         VNIC_ENTER( VNIC_DBG_DATA | VNIC_DBG_INFO );\r
1324 \r
1325         if( pData->recvPool.recv_pkt_array )\r
1326         {\r
1327                 cl_free( pData->recvPool.recv_pkt_array );\r
1328                 pData->recvPool.recv_pkt_array = NULL;\r
1329                 pData->recvPool.poolSz = 0;\r
1330         }\r
1331 \r
1332         if ( pData->pLocalStorage )\r
1333         {\r
1334                 NdisFreeMemory( pData->pLocalStorage, pData->localStorageSz, 0 );\r
1335                 pData->pLocalStorage = NULL;\r
1336         }\r
1337         if(  pData->h_recv_buf_pool )\r
1338         {\r
1339                 NdisFreeBufferPool( pData->h_recv_buf_pool );\r
1340                 pData->h_recv_buf_pool = NULL;\r
1341         }\r
1342         if ( pData->h_recv_pkt_pool )\r
1343         {\r
1344                 NdisFreePacketPool( pData->h_recv_pkt_pool );\r
1345                 pData->h_recv_pkt_pool = NULL;\r
1346         }\r
1347         // clear Qp struct for reuse\r
1348         cl_memclr( &pData->qp, sizeof( IbQp_t) );\r
1349 \r
1350         cl_timer_destroy( &pData->kickTimer );\r
1351 \r
1352         VNIC_EXIT( VNIC_DBG_CTRL );\r
1353 }\r
1354 \r
1355 \r
1356 static void\r
1357 _data_kickTimer_start(\r
1358                   IN    Data_t          *pData,\r
1359                   IN    uint32_t        microseconds )\r
1360 {\r
1361         VNIC_ENTER( VNIC_DBG_DATA | VNIC_DBG_INFO );\r
1362 \r
1363         InterlockedExchange( (LONG *)&pData->kickTimerOn, TRUE );\r
1364 \r
1365         usec_timer_start(&pData->kickTimer, microseconds );\r
1366 \r
1367         VNIC_EXIT( VNIC_DBG_DATA | VNIC_DBG_INFO );\r
1368         return;\r
1369 }\r
1370 \r
1371 static void\r
1372 _data_kickTimer_stop(\r
1373                   IN    Data_t          *pData )\r
1374 {\r
1375         VNIC_ENTER( VNIC_DBG_DATA | VNIC_DBG_INFO );\r
1376 \r
1377         if( InterlockedExchange( &pData->kickTimerOn, FALSE ) == TRUE )\r
1378         {\r
1379                 cl_timer_stop( &pData->kickTimer );\r
1380         }\r
1381 \r
1382         VNIC_EXIT( VNIC_DBG_DATA | VNIC_DBG_INFO );\r
1383 }\r