378053b0b34f3838e90999d1f676a25d304c4b29
[mirror/winof/.git] / ulp / inic / kernel / vnic_data.c
1 /*\r
2  * Copyright (c) 2005 SilverStorm Technologies.  All rights reserved.\r
3  *\r
4  * This software is available to you under the OpenIB.org BSD license\r
5  * below:\r
6  *\r
7  *     Redistribution and use in source and binary forms, with or\r
8  *     without modification, are permitted provided that the following\r
9  *     conditions are met:\r
10  *\r
11  *      - Redistributions of source code must retain the above\r
12  *        copyright notice, this list of conditions and the following\r
13  *        disclaimer.\r
14  *\r
15  *      - Redistributions in binary form must reproduce the above\r
16  *        copyright notice, this list of conditions and the following\r
17  *        disclaimer in the documentation and/or other materials\r
18  *        provided with the distribution.\r
19  *\r
20  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,\r
21  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF\r
22  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND\r
23  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS\r
24  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN\r
25  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN\r
26  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\r
27  * SOFTWARE.\r
28  *\r
29  * $Id$\r
30  */\r
31 #include <complib/comp_lib.h>\r
32 #include "vnic_driver.h"\r
33 \r
34 static void    data_postRecvs(Data_t *pData);\r
35 static void    _data_receivedKick(Io_t *pIo);\r
36 static void    _data_xmitComplete(Io_t *pIo);\r
37 static void    data_sendKickMessage(Data_t *pData);\r
38 static void    _data_kickTimeoutHandler( void *context );\r
39 static BOOLEAN data_allocXmitBuffer(Data_t *pData,\r
40                                         BufferPoolEntry_t **ppBpe, RdmaIo_t **ppRdmaIo, BOOLEAN *pLast);\r
41 static void    data_checkXmitBuffers(Data_t *pData);\r
42 static void    data_rdmaPacket(Data_t *pData, BufferPoolEntry_t *pBpe, RdmaIo_t *pRdmaIo);\r
43 \r
44 static NDIS_PACKET *\r
45 _data_recv_to_ndis_pkt( Data_t *pData, RdmaDest_t *pRdmaDest );\r
46 \r
47 static void\r
48 _data_allocBuffers(\r
49                                         Data_t          *pData,\r
50                                         BOOLEAN         initialAllocation );\r
51 \r
52 static void\r
53 _data_addFreeBuffer(\r
54                                         Data_t          *pData,\r
55                                         int                     index,\r
56                                         RdmaDest_t      *pRdmaDest );\r
57 \r
58 static uint32_t\r
59 _data_incomingRecv(\r
60                                         Data_t          *pData );\r
61 \r
62 static void   \r
63 _data_sendFreeRecvBuffers(\r
64                                         Data_t          *pData );\r
65 \r
66 static uint8_t\r
67 _tx_chksum_flags(\r
68  IN             NDIS_PACKET* const              p_packet );\r
69 \r
70 \r
71 static void\r
72 _get_first_buffer(\r
73                 IN              NDIS_PACKET             *p_packet,\r
74                 IN OUT  NDIS_BUFFER             **pp_buf_desc,\r
75                 OUT             void                    **pp_buf,\r
76                 OUT             ULONG                   *p_packet_sz );\r
77 \r
78 static void\r
79 _data_return_recv(\r
80           IN    NDIS_PACKET             *p_packet );\r
81 \r
82 static void\r
83 _data_kickTimer_start(\r
84                   IN    Data_t          *pData,\r
85                   IN    uint32_t        microseconds );\r
86 \r
87 static void\r
88 _data_kickTimer_stop(\r
89                   IN    Data_t          *pData );\r
90 \r
91 #define LOCAL_IO(x) PTR64((x))\r
92 \r
93 #define INBOUND_COPY\r
94 \r
95 #ifdef VNIC_STATISTIC\r
96 int64_t recvRef;\r
97 #endif /* VNIC_STATISTIC */\r
98 \r
99 void\r
100 data_construct(\r
101         IN              Data_t                  *pData,\r
102         IN              viport_t                *pViport )\r
103 {\r
104         VNIC_ENTER( VNIC_DBG_DATA );\r
105 \r
106         RtlZeroMemory( pData, sizeof(*pData) );\r
107 \r
108         pData->p_viport = pViport;\r
109         pData->p_phy_region = &pViport->p_adapter->ca.region;\r
110         InitializeListHead( &pData->recvIos );\r
111         KeInitializeSpinLock ( &pData->recvIosLock );\r
112         KeInitializeSpinLock ( &pData->xmitBufLock );\r
113         cl_timer_construct( &pData->kickTimer );\r
114 \r
115         ibqp_construct( &pData->qp, pViport );\r
116 \r
117         VNIC_EXIT( VNIC_DBG_DATA );\r
118 }\r
119 \r
120 \r
121 ib_api_status_t\r
122 data_init(\r
123         IN              Data_t                  *pData,\r
124         IN              DataConfig_t    *p_conf,\r
125         IN              uint64_t                guid )\r
126 {\r
127         ib_api_status_t ib_status;\r
128 \r
129         VNIC_ENTER( VNIC_DBG_DATA );\r
130 \r
131         ASSERT( pData->p_viport != NULL );\r
132         pData->p_conf = p_conf;\r
133 \r
134         cl_timer_init( &pData->kickTimer, _data_kickTimeoutHandler, pData );\r
135 \r
136         ib_status = ibqp_init(&pData->qp, guid, &p_conf->ibConfig );\r
137         if( ib_status != IB_SUCCESS )\r
138                 VNIC_TRACE( VNIC_DBG_ERROR, ("data ibqp_init failed\n") );\r
139 \r
140         VNIC_EXIT( VNIC_DBG_DATA );\r
141         return ib_status;\r
142 }\r
143 \r
144 \r
145 ib_api_status_t\r
146 data_connect(\r
147                 IN              Data_t          *pData )\r
148 {\r
149         NDIS_STATUS                     status;\r
150         ib_api_status_t         ib_status;\r
151         XmitPool_t        *pXmitPool = &pData->xmitPool;\r
152         RecvPool_t        *pRecvPool = &pData->recvPool;\r
153         RecvIo_t          *pRecvIo;\r
154         SendIo_t          *pSendIo;\r
155         RdmaIo_t          *pRdmaIo;\r
156         RdmaDest_t        *pRdmaDest;\r
157         uint8_t           *pRegionData;\r
158         int               sz, regionSz;\r
159         unsigned int      i, j;\r
160 \r
161         VNIC_ENTER( VNIC_DBG_DATA );\r
162 \r
163         pRecvPool->poolSz           = pData->p_conf->hostRecvPoolEntries;\r
164         pRecvPool->eiocPoolSz       = pData->hostPoolParms.numRecvPoolEntries;\r
165 \r
166         if ( pRecvPool->poolSz > pRecvPool->eiocPoolSz )\r
167         {\r
168                 pRecvPool->poolSz = pData->hostPoolParms.numRecvPoolEntries;\r
169         }\r
170         pRecvPool->szFreeBundle     =\r
171                         pData->hostPoolParms.freeRecvPoolEntriesPerUpdate;\r
172         pRecvPool->numFreeBufs      = 0;\r
173         pRecvPool->numPostedBufs    = 0;\r
174         pRecvPool->nextFullBuf      = 0;\r
175         pRecvPool->nextFreeBuf      = 0;\r
176         pRecvPool->kickOnFree       = FALSE;\r
177         pRecvPool->bufferSz         = pData->hostPoolParms.sizeRecvPoolEntry;\r
178 \r
179         pXmitPool->bufferSz         = pData->eiocPoolParms.sizeRecvPoolEntry;\r
180         pXmitPool->poolSz           = pData->eiocPoolParms.numRecvPoolEntries;\r
181         pXmitPool->notifyCount      = 0;\r
182         pXmitPool->notifyBundle     = pData->p_conf->notifyBundle;\r
183         pXmitPool->nextXmitPool     = 0;\r
184 \r
185 #if TRUE // LIMIT_OUTSTANDING_SENDS\r
186         pXmitPool->numXmitBufs      = pXmitPool->notifyBundle * 2;\r
187 #else /* !LIMIT_OUTSTANDING_SENDS */\r
188         pXmitPool->numXmitBufs      = pXmitPool->poolSz;\r
189 #endif /* LIMIT_OUTSTANDING_SENDS */\r
190 \r
191         pXmitPool->nextXmitBuf      = 0;\r
192         pXmitPool->lastCompBuf      = pXmitPool->numXmitBufs - 1;\r
193         pXmitPool->kickCount        = 0;\r
194         pXmitPool->kickByteCount    = 0;\r
195         pXmitPool->sendKicks        =\r
196                 (BOOLEAN)(( pData->eiocPoolParms.numRecvPoolEntriesBeforeKick != 0 )\r
197                 || ( pData->eiocPoolParms.numRecvPoolBytesBeforeKick != 0 ));\r
198         pXmitPool->kickBundle       =\r
199                 pData->eiocPoolParms.numRecvPoolEntriesBeforeKick;\r
200         pXmitPool->kickByteBundle   =\r
201                 pData->eiocPoolParms.numRecvPoolBytesBeforeKick;\r
202         pXmitPool->needBuffers      = TRUE;\r
203 \r
204         sz  = sizeof(RdmaDest_t) * pRecvPool->poolSz;\r
205         sz += sizeof(RecvIo_t) * pData->p_conf->numRecvs;\r
206         sz += sizeof(RdmaIo_t) * pXmitPool->numXmitBufs;\r
207 \r
208         regionSz  = 4 * pData->p_conf->numRecvs;\r
209         regionSz += sizeof(BufferPoolEntry_t) * pRecvPool->eiocPoolSz;\r
210         regionSz += sizeof(BufferPoolEntry_t) * pXmitPool->poolSz;\r
211         sz       += regionSz;\r
212 \r
213         status = NdisAllocateMemoryWithTag( &pData->pLocalStorage,\r
214                                                                                 (UINT)sz,\r
215                                                                                 'grtS' );\r
216         if ( status != NDIS_STATUS_SUCCESS )\r
217         {\r
218                 VNIC_TRACE_EXIT( VNIC_DBG_ERROR,\r
219                         ("Failed allocating %d bytes local storage\n", sz ) );\r
220                 ib_status = IB_INSUFFICIENT_MEMORY;\r
221                 goto err1;\r
222         }\r
223 \r
224         NdisZeroMemory( pData->pLocalStorage, sz );\r
225         pData->localStorageSz = sz;\r
226 \r
227         pRecvPool->pRecvBufs = (RdmaDest_t *)pData->pLocalStorage;\r
228         sz                   = sizeof(RdmaDest_t) * pRecvPool->poolSz;\r
229         pRecvIo              = (RecvIo_t *)(pData->pLocalStorage + sz);\r
230         sz                  += sizeof(RecvIo_t) * pData->p_conf->numRecvs;\r
231 \r
232         pXmitPool->pXmitBufs = (RdmaIo_t *)(pData->pLocalStorage + sz);\r
233         sz                  += sizeof(RdmaIo_t) * pXmitPool->numXmitBufs;\r
234 \r
235         pRegionData          = pData->pLocalStorage + sz;\r
236         sz                  += 4 * pData->p_conf->numRecvs;\r
237 \r
238         pRecvPool->bufPool   = (BufferPoolEntry_t *)(pData->pLocalStorage + sz);\r
239         sz                  += sizeof(BufferPoolEntry_t) * pRecvPool->eiocPoolSz;\r
240         pXmitPool->bufPool   = (BufferPoolEntry_t *)(pData->pLocalStorage + sz);\r
241 \r
242         ib_status = ibregion_init( pData->p_viport, &pData->region,\r
243                 pData->p_viport->p_adapter->ca.hPd, pRegionData, regionSz,\r
244                 ( IB_AC_LOCAL_WRITE | IB_AC_RDMA_WRITE ) );\r
245         if( ib_status != IB_SUCCESS )\r
246         {\r
247                 VNIC_TRACE_EXIT( VNIC_DBG_ERROR, ("ib_region_init failed\n") );\r
248                 goto err2;\r
249         }\r
250 \r
251         pRdmaIo = &pData->freeBufsIo;\r
252         pRdmaIo->io.pViport          = pData->p_viport;\r
253         pRdmaIo->io.pRoutine         = NULL;\r
254         pRdmaIo->io.wrq.p_next       = NULL;\r
255         pRdmaIo->io.wrq.wr_type      = WR_RDMA_WRITE;\r
256         pRdmaIo->io.wrq.wr_id        = PTR64( pRdmaIo );\r
257         pRdmaIo->io.wrq.num_ds       = 1;\r
258         pRdmaIo->io.wrq.ds_array     = pRdmaIo->dsList;\r
259         pRdmaIo->dsList[0].lkey      = pData->region.lkey;\r
260         pRdmaIo->io.wrq.send_opt     = IB_SEND_OPT_SIGNALED;\r
261 \r
262         pSendIo = &pData->kickIo;\r
263         pSendIo->io.pViport          = pData->p_viport;\r
264         pSendIo->io.pRoutine         = NULL;\r
265         pSendIo->io.wrq.p_next       = NULL;\r
266         pSendIo->io.wrq.wr_type      = WR_SEND;\r
267         pSendIo->io.wrq.wr_id        = PTR64( pSendIo );\r
268         pSendIo->io.wrq.num_ds       = 1;\r
269         pSendIo->io.wrq.ds_array     = &pSendIo->dsList;\r
270 \r
271         pSendIo->io.wrq.send_opt     = IB_SEND_OPT_SIGNALED;\r
272 \r
273         pSendIo->dsList.length       = 0;\r
274         pSendIo->dsList.vaddr        = PTR64( pRegionData );\r
275         pSendIo->dsList.lkey         = pData->region.lkey;\r
276 \r
277         for ( i = 0; i < pData->p_conf->numRecvs; i++ )\r
278         {\r
279                 pRecvIo[i].io.pViport         = pData->p_viport;\r
280                 pRecvIo[i].io.pRoutine        = _data_receivedKick;\r
281                 pRecvIo[i].io.r_wrq.wr_id     = PTR64( &pRecvIo[i].io );\r
282                 pRecvIo[i].io.r_wrq.p_next    = NULL;\r
283                 pRecvIo[i].io.r_wrq.num_ds    = 1;\r
284                 pRecvIo[i].io.r_wrq.ds_array  = &pRecvIo[i].dsList;\r
285                 pRecvIo[i].dsList.length      = 4;\r
286                 pRecvIo[i].dsList.vaddr       = PTR64( pRegionData );\r
287                 pRecvIo[i].dsList.lkey        = pData->region.lkey;\r
288         \r
289                 ExInterlockedInsertTailList( &pData->recvIos, &pRecvIo[i].io.listPtrs, &pData->recvIosLock );\r
290         /* Do not need to move pointer since the receive info\r
291          * is not read.  Note, we could reduce the amount\r
292          * of memory allocated and the size of the region.\r
293                  * pRegionData                  += 4;\r
294          * */\r
295         }\r
296 \r
297         sz = pRecvPool->poolSz * pRecvPool->bufferSz;\r
298         status = NdisAllocateMemoryWithTag(&pData->p_recv_bufs,\r
299                                                                                 sz, 'fubr');\r
300         if( status != NDIS_STATUS_SUCCESS )\r
301         {\r
302                 VNIC_TRACE_EXIT( VNIC_DBG_ERROR,\r
303                                 ("Allocate recv buffers failed\n"));\r
304                 ib_status = IB_INSUFFICIENT_MEMORY;\r
305                 goto err3;\r
306         }\r
307         NdisZeroMemory( pData->p_recv_bufs, sz );\r
308 \r
309         pData->recv_bufs_sz = sz;\r
310 \r
311         ib_status = ibregion_init( pData->p_viport, &pData->rbuf_region,\r
312                 pData->p_viport->p_adapter->ca.hPd, pData->p_recv_bufs, sz,\r
313                 (IB_AC_LOCAL_WRITE | IB_AC_RDMA_WRITE) );\r
314         if( ib_status != IB_SUCCESS )\r
315         {\r
316                 goto err4;\r
317         }\r
318 \r
319         NdisAllocatePacketPool( &status,\r
320                                                         &pData->h_recv_pkt_pool,\r
321                                                         pRecvPool->poolSz,\r
322                                                         PROTOCOL_RESERVED_SIZE_IN_PACKET );\r
323 \r
324         if( status != NDIS_STATUS_SUCCESS )\r
325         {\r
326                 VNIC_TRACE_EXIT( VNIC_DBG_ERROR,\r
327                         ("Allocate packet pool failed status %#x\n", status ));\r
328                 ib_status = IB_INSUFFICIENT_MEMORY;\r
329                 goto err5;\r
330         }\r
331 \r
332         NdisAllocateBufferPool(\r
333                 &status, &pData->h_recv_buf_pool, pRecvPool->poolSz );\r
334 \r
335         if( status != NDIS_STATUS_SUCCESS )\r
336         {\r
337                 VNIC_TRACE_EXIT( VNIC_DBG_ERROR,\r
338                         ("Allocate packet pool failed status %#x\n", status ));\r
339                 ib_status = IB_INSUFFICIENT_MEMORY;\r
340                 goto err6;\r
341         }\r
342         pData->recvPool.recv_pkt_array =\r
343                 cl_zalloc( sizeof(NDIS_PACKET*)* pRecvPool->poolSz );\r
344         if( !pData->recvPool.recv_pkt_array )\r
345         {\r
346                 VNIC_TRACE_EXIT( VNIC_DBG_ERROR,\r
347                                 ("Allocate packet array failed\n" ) );\r
348                 ib_status = IB_INSUFFICIENT_MEMORY;\r
349                 goto err7;\r
350         }\r
351 \r
352         InitializeListHead( &pRecvPool->availRecvBufs );\r
353 \r
354         for ( i = 0; i < pRecvPool->poolSz; i++ )\r
355         {\r
356                 pRdmaDest = &pRecvPool->pRecvBufs[i];\r
357                 pRdmaDest->data = pData->p_recv_bufs + (i * pRecvPool->bufferSz );\r
358                 pRdmaDest->region = pData->rbuf_region;\r
359                 InsertTailList( &pRecvPool->availRecvBufs, &pRdmaDest->listPtrs );\r
360         }\r
361 \r
362         for ( i = 0; i < pXmitPool->numXmitBufs; i++ )\r
363         {\r
364                 pRdmaIo = &pXmitPool->pXmitBufs[i];\r
365                 pRdmaIo->index               = (uint16_t)i;\r
366                 pRdmaIo->io.pViport          = pData->p_viport;\r
367                 pRdmaIo->io.pRoutine         = _data_xmitComplete;\r
368                 pRdmaIo->io.wrq.p_next       = NULL;\r
369                 pRdmaIo->io.wrq.wr_type      = WR_RDMA_WRITE;\r
370                 pRdmaIo->io.wrq.wr_id        = PTR64(pRdmaIo);\r
371                 pRdmaIo->io.wrq.num_ds       = MAX_NUM_SGE; // will set actual number when transmit\r
372                 pRdmaIo->io.wrq.ds_array     = pRdmaIo->dsList;\r
373                 pRdmaIo->p_trailer                      =  (ViportTrailer_t *)&pRdmaIo->data[0];\r
374                 for( j = 0; j < MAX_NUM_SGE; j++ )\r
375                 {\r
376                         pRdmaIo->dsList[j].lkey      = pData->p_phy_region->lkey;\r
377                 }\r
378         }\r
379 \r
380         pXmitPool->rdmaRKey      = pData->region.rkey;\r
381         pXmitPool->rdmaAddr      = PTR64( pXmitPool->bufPool );\r
382 \r
383         data_postRecvs( pData );\r
384 \r
385         ib_status = ibqp_connect( &pData->qp );\r
386         if( ib_status != IB_SUCCESS )\r
387         {\r
388                 VNIC_TRACE( VNIC_DBG_ERROR, ("ibqp_connect returned %s\n",\r
389                         pData->p_viport->p_adapter->ifc.get_err_str( ib_status )) );\r
390 err7:\r
391                 NdisFreeBufferPool( pData->h_recv_buf_pool );\r
392                 pData->h_recv_buf_pool = NULL;\r
393 err6:\r
394                 NdisFreePacketPool( pData->h_recv_pkt_pool );\r
395                 pData->h_recv_pkt_pool = NULL;\r
396 err5:\r
397                 ibregion_cleanup( pData->p_viport, &pData->rbuf_region );\r
398 err4:\r
399                 NdisFreeMemory( pData->p_recv_bufs, pData->recv_bufs_sz, 0 );\r
400                 pData->p_recv_bufs = NULL;\r
401 err3:\r
402                 ibregion_cleanup(pData->p_viport, &pData->region );\r
403 err2:\r
404                 NdisFreeMemory( pData->pLocalStorage, pData->localStorageSz, 0 );\r
405                 pData->pLocalStorage = NULL;\r
406 err1:\r
407                 pRecvPool->poolSz = 0;\r
408         }\r
409 \r
410         VNIC_EXIT( VNIC_DBG_DATA );\r
411         return ib_status;\r
412 }\r
413 \r
414 \r
415 void\r
416 data_connected(\r
417                 IN              Data_t          *pData )\r
418 {\r
419         VNIC_ENTER( VNIC_DBG_DATA );\r
420 \r
421         pData->freeBufsIo.io.wrq.remote_ops.rkey =\r
422                                                         pData->recvPool.eiocRdmaRkey;\r
423 \r
424         _data_allocBuffers(pData, TRUE);\r
425         _data_sendFreeRecvBuffers(pData);\r
426         pData->connected = TRUE;\r
427 \r
428         VNIC_EXIT( VNIC_DBG_DATA );\r
429         return;\r
430 }\r
431 \r
432 void\r
433 data_disconnect(\r
434                 IN              Data_t          *pData )\r
435 {\r
436         RecvPool_t *pRecvPool = &pData->recvPool;\r
437         viport_t        *p_viport = pData->p_viport;\r
438         NDIS_PACKET             *p_packet;\r
439         LIST_ENTRY              *p_list_item;\r
440         unsigned int        i;\r
441 \r
442         VNIC_ENTER( VNIC_DBG_DATA );\r
443 \r
444         _data_kickTimer_stop ( pData );\r
445 \r
446         pData->connected = FALSE;\r
447 \r
448         ibqp_detach( &pData->qp );\r
449 \r
450         ibregion_cleanup( p_viport, &pData->rbuf_region );\r
451         ibregion_cleanup( p_viport, &pData->region );\r
452 \r
453         for ( i = 0; i < pRecvPool->poolSz; i++ )\r
454         {\r
455                 p_packet = pRecvPool->pRecvBufs[i].p_packet;\r
456                 if ( p_packet != NULL )\r
457                 {\r
458                         pRecvPool->pRecvBufs[i].p_packet = NULL;\r
459                         _data_return_recv( p_packet );\r
460                 }\r
461         }\r
462         /* clear pending queue if any */\r
463         while( ( p_list_item = NdisInterlockedRemoveHeadList(\r
464                 &p_viport->send_pending_list,\r
465                 &p_viport->pending_list_lock )) != NULL )\r
466         {\r
467                 p_packet = VNIC_PACKET_FROM_LIST_ITEM( p_list_item );\r
468                 if ( p_packet )\r
469                 {\r
470                         NDIS_SET_PACKET_STATUS( p_packet, NDIS_STATUS_FAILURE );\r
471                         NdisMSendComplete( p_viport->p_adapter->h_handle,\r
472                                 p_packet, NDIS_STATUS_FAILURE );\r
473                         p_viport->stats.ifOutErrors++;\r
474                 }\r
475         }\r
476 \r
477         VNIC_EXIT( VNIC_DBG_DATA );\r
478         return;\r
479 }\r
480 \r
481 BOOLEAN\r
482 data_xmitPacket(\r
483                 IN              Data_t                          *pData,\r
484                 IN              NDIS_PACKET* const      p_packet )\r
485 {\r
486         XmitPool_t              *p_xmitPool = &pData->xmitPool;\r
487         RdmaIo_t                *pRdmaIo;\r
488         BufferPoolEntry_t *pBpe;\r
489         BOOLEAN                 last;\r
490         uint8_t                 *p_buf;\r
491         uint32_t                buf_len;\r
492         NDIS_BUFFER             *p_buf_desc;\r
493         eth_hdr_t*              p_eth_hdr;\r
494         int                             pad;\r
495         SCATTER_GATHER_LIST             *p_sgl;\r
496         uint32_t                i;\r
497         PHYSICAL_ADDRESS phy_addr;\r
498 \r
499         VNIC_ENTER( VNIC_DBG_DATA );\r
500 \r
501         if( !data_allocXmitBuffer( pData, &pBpe, &pRdmaIo, &last ) )\r
502         {\r
503                 return FALSE;\r
504         }\r
505         p_sgl = NDIS_PER_PACKET_INFO_FROM_PACKET( p_packet,\r
506                                                                                         ScatterGatherListPacketInfo );\r
507         if ( p_sgl == NULL )\r
508         {\r
509                 return FALSE;\r
510         }\r
511 \r
512         NdisGetFirstBufferFromPacketSafe( p_packet,\r
513                                                                         &p_buf_desc,\r
514                                                                         &p_buf,\r
515                                                                         &buf_len,\r
516                                                                         &pRdmaIo->packet_sz,\r
517                                                                         NormalPagePriority );\r
518         if( pRdmaIo->packet_sz > p_xmitPool->bufferSz )\r
519         {\r
520                 VNIC_TRACE_EXIT( VNIC_DBG_ERROR,\r
521                         ("Outbound packet too large, size = %d\n", pRdmaIo->packet_sz ) );\r
522                 return FALSE;\r
523         }\r
524 \r
525         if ( p_sgl->NumberOfElements > (ULONG)MAX_NUM_SGE - 1 )\r
526         {\r
527                 VNIC_TRACE( VNIC_DBG_DATA,\r
528                         (" Xmit packet exceeded SGE limit - %d\n",\r
529                                                                                 p_sgl->NumberOfElements ) );\r
530                 return FALSE;\r
531         }\r
532 \r
533         for( i=0; i < p_sgl->NumberOfElements; i++ )\r
534         {\r
535                 pRdmaIo->dsList[i].vaddr = p_sgl->Elements[i].Address.QuadPart;\r
536                 pRdmaIo->dsList[i].length = p_sgl->Elements[i].Length;\r
537         }\r
538 \r
539         pRdmaIo->len = (uint32_t)ROUNDUPP2(\r
540                 max(60, pRdmaIo->packet_sz), VIPORT_TRAILER_ALIGNMENT );\r
541         pad = pRdmaIo->len - pRdmaIo->packet_sz;\r
542 \r
543         p_eth_hdr = (eth_hdr_t *)p_buf;\r
544 \r
545         pRdmaIo->p_trailer = (ViportTrailer_t *)&pRdmaIo->data[pad];\r
546         cl_memclr( pRdmaIo->data, pad + sizeof( ViportTrailer_t ) );\r
547         cl_memcpy( pRdmaIo->p_trailer->destMacAddr, p_eth_hdr->dst.addr, MAC_ADDR_LEN );\r
548 \r
549         pRdmaIo->p_trailer->dataLength =\r
550                                         hton16( (uint16_t)max( 60, pRdmaIo->packet_sz ) );\r
551 \r
552         NdisGetNextBuffer( p_buf_desc, &p_buf_desc );\r
553 \r
554         /* should handle VLAN tag */\r
555         if( pRdmaIo->packet_sz > 16 )\r
556         {\r
557                 if(     p_eth_hdr->type == hton16(0x8100) )\r
558                 {\r
559                         if( p_sgl->Elements[0].Length > sizeof(eth_hdr_t) )\r
560                         {\r
561                                 pRdmaIo->p_trailer->vLan = *(uint16_t *)((uint8_t *)p_eth_hdr + 14 );\r
562                                 pRdmaIo->p_trailer->pktFlags |= PF_VLAN_INSERT;\r
563                         }\r
564                         else\r
565                         {\r
566                                 if( p_buf_desc )\r
567                                 {\r
568                                         NdisQueryBufferSafe( p_buf_desc, &p_buf, &buf_len, NormalPagePriority );\r
569 \r
570                                         pad = sizeof(eth_hdr_t) - p_sgl->Elements[0].Length;\r
571                                         pRdmaIo->p_trailer->vLan = *(uint16_t *)(p_buf + pad + 2);\r
572                                         pRdmaIo->p_trailer->pktFlags |= PF_VLAN_INSERT;\r
573                                 }\r
574                         }\r
575                 }\r
576                 else if( p_eth_hdr->type == ETH_PROT_TYPE_IP &&\r
577                                 !( p_eth_hdr->dst.addr[0] & 0x01 ) )\r
578                 {\r
579                         if( p_buf_desc )\r
580                         {\r
581                                 NdisQueryBufferSafe( p_buf_desc, &p_buf, &buf_len, NormalPagePriority );\r
582 \r
583                                 if( ((ip_pkt_t*)p_buf)->hdr.prot == IP_PROT_UDP ||\r
584                                         ((ip_pkt_t*)p_buf)->hdr.prot == IP_PROT_TCP )\r
585                                 {\r
586                                         /* use socket src port + dest port to generate hash value\r
587                                         * for link aggregation distribution function.\r
588                                         */\r
589                                         pRdmaIo->p_trailer->connectionHashAndValid = 0x40 |\r
590                                                 ((uint8_t)((ip_pkt_t*)p_buf)->prot.tcp.src_port +\r
591                                                 (uint8_t)((ip_pkt_t*)p_buf)->prot.tcp.dst_port ) & 0x3f;\r
592                                 }\r
593                         }\r
594                 }\r
595         }\r
596 \r
597         pRdmaIo->p_trailer->txChksumFlags = _tx_chksum_flags( p_packet );\r
598         pRdmaIo->p_trailer->connectionHashAndValid |= CHV_VALID;\r
599 \r
600         if( last )\r
601                 pRdmaIo->p_trailer->pktFlags |= PF_KICK;\r
602 \r
603         /* fill last data segment with trailer and pad */\r
604         phy_addr = MmGetPhysicalAddress( pRdmaIo->data );\r
605 \r
606         pRdmaIo->dsList[p_sgl->NumberOfElements].vaddr = phy_addr.QuadPart;\r
607         pRdmaIo->dsList[p_sgl->NumberOfElements].length = pRdmaIo->len -\r
608                                                                                                           pRdmaIo->packet_sz +\r
609                                                                                                           sizeof( ViportTrailer_t );\r
610         //pRdmaIo->io.wrq.ds_array = pRdmaIo->dsList;\r
611     pRdmaIo->io.wrq.num_ds =p_sgl->NumberOfElements + 1;\r
612 \r
613         data_rdmaPacket( pData, pBpe, pRdmaIo );\r
614 \r
615         if( p_xmitPool->sendKicks )\r
616         {\r
617                 /* EIOC needs kicks to inform it of sent packets */\r
618 \r
619                 p_xmitPool->kickCount++;\r
620                 p_xmitPool->kickByteCount += pRdmaIo->packet_sz;\r
621                 if( ( p_xmitPool->kickCount >= p_xmitPool->kickBundle )\r
622                          || ( p_xmitPool->kickByteCount >= p_xmitPool->kickByteBundle ) )\r
623                 {\r
624                         data_sendKickMessage( pData );\r
625                 }\r
626                 else if( p_xmitPool->kickCount == 1 )\r
627                 {\r
628                         _data_kickTimer_start( pData, pData->eiocPoolParms.timeoutBeforeKick );\r
629                 }\r
630         }\r
631         return TRUE;\r
632 }\r
633 static uint8_t\r
634 _tx_chksum_flags(\r
635                   IN    NDIS_PACKET* const              p_packet )\r
636 \r
637 {\r
638         NDIS_TCP_IP_CHECKSUM_PACKET_INFO        *p_packet_info;\r
639         ULONG                                                           packet_info;\r
640         uint8_t                 txChksumFlags = 0;\r
641 \r
642         if( NDIS_PROTOCOL_ID_TCP_IP == NDIS_GET_PACKET_PROTOCOL_TYPE(p_packet) )\r
643         {\r
644                 packet_info = PtrToUlong( NDIS_PER_PACKET_INFO_FROM_PACKET( p_packet, TcpIpChecksumPacketInfo));\r
645                 p_packet_info = ( NDIS_TCP_IP_CHECKSUM_PACKET_INFO *)&packet_info;\r
646 \r
647                 if( p_packet_info &&\r
648                         p_packet_info->Transmit.NdisPacketChecksumV4 )\r
649                 {\r
650                         txChksumFlags = TX_CHKSUM_FLAGS_CHECKSUM_V4\r
651                         | ( p_packet_info->Transmit.NdisPacketIpChecksum ? TX_CHKSUM_FLAGS_IP_CHECKSUM: 0 )\r
652                         | ( p_packet_info->Transmit.NdisPacketTcpChecksum ? TX_CHKSUM_FLAGS_TCP_CHECKSUM: 0 )\r
653                         | ( p_packet_info->Transmit.NdisPacketUdpChecksum ? TX_CHKSUM_FLAGS_UDP_CHECKSUM: 0 );\r
654                 }\r
655         }\r
656         \r
657         VNIC_TRACE( VNIC_DBG_DATA ,\r
658                                 ("txChksumFlags = %d: V4 %c, V6 %c, IP %c, TCP %c, UDP %c\n",\r
659                                 txChksumFlags,\r
660                                 ((txChksumFlags & TX_CHKSUM_FLAGS_CHECKSUM_V4 )? '+': '-'),\r
661                                 ((txChksumFlags & TX_CHKSUM_FLAGS_CHECKSUM_V6 )? '+': '-'),\r
662                                 ((txChksumFlags & TX_CHKSUM_FLAGS_IP_CHECKSUM )? '+': '-'),\r
663                                 ((txChksumFlags & TX_CHKSUM_FLAGS_TCP_CHECKSUM )? '+': '-'),\r
664                                 ((txChksumFlags & TX_CHKSUM_FLAGS_UDP_CHECKSUM )? '+': '-') ));\r
665 \r
666         return txChksumFlags;\r
667 }\r
668 \r
669 static void\r
670 _get_first_buffer(\r
671                 IN              NDIS_PACKET             *p_packet,\r
672                 IN OUT  NDIS_BUFFER             **pp_buf_desc,\r
673                 OUT             void                    **pp_buf,\r
674                 OUT             ULONG                   *p_packet_sz )\r
675 {\r
676         UINT            buf_len;\r
677         VNIC_ENTER( VNIC_DBG_DATA );\r
678 \r
679         NdisGetFirstBufferFromPacketSafe( p_packet,\r
680                                                                         pp_buf_desc,\r
681                                                                         pp_buf,\r
682                                                                         &buf_len,\r
683                                                                         p_packet_sz,\r
684                                                                         NormalPagePriority );\r
685         VNIC_EXIT( VNIC_DBG_DATA );\r
686 }\r
687 \r
688 static void\r
689 data_postRecvs(\r
690                 IN              Data_t          *pData )\r
691 {\r
692         RecvIo_t                *pRecvIo;\r
693         LIST_ENTRY              *p_list_entry;\r
694         ib_api_status_t ib_status;\r
695 \r
696         VNIC_ENTER ( VNIC_DBG_DATA );\r
697 \r
698         while( ( p_list_entry = ExInterlockedRemoveHeadList( &pData->recvIos,\r
699                                                                                                                  &pData->recvIosLock ))\r
700                                                                                                                 != NULL )\r
701         {\r
702                 pRecvIo = (RecvIo_t *)p_list_entry;\r
703 \r
704                 ib_status = ibqp_postRecv( &pData->qp, &pRecvIo->io );\r
705                 if( ib_status != IB_SUCCESS )\r
706                 {\r
707                         VNIC_TRACE_EXIT( VNIC_DBG_ERROR, ("ibqp_postRecv returned %s\n",\r
708                                 pData->p_viport->p_adapter->ifc.get_err_str( ib_status )) );\r
709                         viport_failure( pData->p_viport );\r
710                         return;\r
711                 }\r
712         }\r
713 \r
714         VNIC_EXIT( VNIC_DBG_DATA );\r
715         return;\r
716 }\r
717 \r
718 static void\r
719 _data_receivedKick(\r
720                         IN              Io_t            *pIo )\r
721 {\r
722         Data_t        *pData = &pIo->pViport->data;\r
723         uint32_t                num_pkts = 0;\r
724 \r
725         VNIC_ENTER( VNIC_DBG_DATA );\r
726 \r
727 #ifdef VNIC_STATISTIC\r
728         recvRef = cl_get_tick_count();\r
729 #endif /* VNIC_STATISTIC */\r
730 \r
731         ExInterlockedInsertTailList( &pData->recvIos, &pIo->listPtrs, &pData->recvIosLock );\r
732 \r
733         data_postRecvs( pData );\r
734 \r
735 #ifdef VNIC_STATISTIC\r
736         pData->statistics.kickRecvs++;\r
737 #endif /* VNIC_STATISTIC */\r
738 \r
739         data_checkXmitBuffers( pData );\r
740 \r
741         num_pkts = _data_incomingRecv( pData );\r
742 \r
743         if( num_pkts )\r
744         {\r
745                 NdisMIndicateReceivePacket( pData->p_viport->p_adapter->h_handle,\r
746                                                                         pData->recvPool.recv_pkt_array,\r
747                                                                         num_pkts );\r
748                 pData->p_viport->stats.ifInOk += num_pkts;\r
749         }\r
750 \r
751         VNIC_EXIT( VNIC_DBG_DATA );\r
752         return;\r
753 }\r
754 \r
755 static void\r
756 _data_xmitComplete(\r
757                         IN              Io_t            *pIo )\r
758 {\r
759         RdmaIo_t       *pRdmaIo = (RdmaIo_t *)pIo;\r
760         Data_t         *pData = &pIo->pViport->data;\r
761         XmitPool_t     *p_xmitPool = &pData->xmitPool;\r
762         NDIS_PACKET             *p_packet;\r
763         NDIS_STATUS             ndis_status;\r
764         LIST_ENTRY              *p_list_item;\r
765 \r
766         VNIC_ENTER( VNIC_DBG_DATA );\r
767 \r
768         while ( p_xmitPool->lastCompBuf != pRdmaIo->index )\r
769         {\r
770                 INC(p_xmitPool->lastCompBuf, 1, p_xmitPool->numXmitBufs);\r
771                 p_packet = p_xmitPool->pXmitBufs[p_xmitPool->lastCompBuf].p_packet;\r
772         \r
773                 p_xmitPool->pXmitBufs[p_xmitPool->lastCompBuf].p_packet = NULL;\r
774 \r
775                 if( p_packet != NULL )\r
776                 {\r
777                         if( pIo->wc_status != IB_WCS_SUCCESS )\r
778                         {\r
779                                 ndis_status = NDIS_STATUS_FAILURE;\r
780                                 pIo->pViport->stats.ifOutErrors++;\r
781                                 pIo->wc_status = IB_WCS_SUCCESS;\r
782                         }\r
783                         else\r
784                         {\r
785                                 ndis_status = NDIS_STATUS_SUCCESS;\r
786                                 pIo->pViport->stats.ifOutOk++;\r
787                         }\r
788                         NDIS_SET_PACKET_STATUS( p_packet, ndis_status );\r
789                         NdisMSendComplete( pIo->pViport->p_adapter->h_handle,\r
790                                                                 p_packet, ndis_status );\r
791                 }\r
792         }\r
793 \r
794         if( !pIo->pViport->p_netpath->carrier )\r
795         {\r
796                 while( ( p_list_item = NdisInterlockedRemoveHeadList(\r
797                         &pIo->pViport->send_pending_list,\r
798                         &pIo->pViport->pending_list_lock ) ) != NULL )\r
799                 {\r
800                         p_packet = VNIC_PACKET_FROM_LIST_ITEM( p_list_item );\r
801                         NDIS_SET_PACKET_STATUS( p_packet, NDIS_STATUS_FAILURE );\r
802                         NdisMSendComplete( &pIo->pViport->p_adapter->h_handle,\r
803                                                                 p_packet, NDIS_STATUS_FAILURE );\r
804                         pIo->pViport->stats.ifOutErrors++;\r
805                 }\r
806         }\r
807 \r
808         data_checkXmitBuffers( pData );\r
809 \r
810         VNIC_EXIT( VNIC_DBG_DATA );\r
811         return;\r
812 }\r
813 \r
814 static void\r
815 data_sendKickMessage(\r
816                 IN              Data_t          *pData )\r
817 {\r
818         XmitPool_t *pPool = &pData->xmitPool;\r
819 \r
820         VNIC_ENTER( VNIC_DBG_DATA );\r
821 \r
822         /* stop timer for BundleTimeout */\r
823         _data_kickTimer_stop( pData );\r
824 \r
825         pPool->kickCount = 0;\r
826         pPool->kickByteCount = 0;\r
827 \r
828         /* TBD: Keep track of when kick is outstanding, and\r
829          * don't reuse until complete\r
830          */\r
831         if ( ibqp_postSend( &pData->qp, &pData->kickIo.io ) != IB_SUCCESS )\r
832         {\r
833                 VNIC_TRACE( VNIC_DBG_ERROR,\r
834                                         ("Unable to send kick to EIOC\n") );\r
835                 viport_failure( pData->p_viport );\r
836         }\r
837 \r
838         VNIC_EXIT( VNIC_DBG_DATA );\r
839 }\r
840 \r
841 static void\r
842 _data_kickTimeoutHandler( void * context )\r
843 {\r
844         Data_t* pData = (Data_t *)context;\r
845 \r
846         VNIC_ENTER( VNIC_DBG_DATA );\r
847         \r
848         InterlockedExchange( &pData->kickTimerOn, FALSE );\r
849         data_sendKickMessage( pData );\r
850 \r
851         VNIC_EXIT( VNIC_DBG_DATA );\r
852 \r
853         return;\r
854 }\r
855 \r
856 static BOOLEAN\r
857 data_allocXmitBuffer(\r
858                 IN              Data_t                          *pData,\r
859                 OUT             BufferPoolEntry_t       **ppBpe,\r
860                 OUT             RdmaIo_t                        **ppRdmaIo,\r
861                 OUT             BOOLEAN                         *pLast )\r
862 {\r
863         XmitPool_t    *p_xmitPool = &pData->xmitPool;\r
864         KIRQL flags;\r
865 \r
866         VNIC_ENTER( VNIC_DBG_DATA );\r
867 \r
868         KeAcquireSpinLock( &pData->xmitBufLock, &flags );\r
869 \r
870         *pLast = FALSE;\r
871         *ppRdmaIo = &p_xmitPool->pXmitBufs[p_xmitPool->nextXmitBuf];\r
872         *ppBpe = &p_xmitPool->bufPool[p_xmitPool->nextXmitPool];\r
873 \r
874         if ( (*ppBpe)->valid && p_xmitPool->nextXmitBuf != p_xmitPool->lastCompBuf )\r
875         {\r
876                 INC(p_xmitPool->nextXmitBuf, 1, p_xmitPool->numXmitBufs);\r
877                 INC(p_xmitPool->nextXmitPool, 1, p_xmitPool->poolSz);\r
878 \r
879                 if ( !p_xmitPool->bufPool[p_xmitPool->nextXmitPool].valid )\r
880                 {\r
881                         VNIC_TRACE( VNIC_DBG_DATA,\r
882                                 ("Just used the last EIOU receive buffer\n") );\r
883 \r
884                         *pLast = TRUE;\r
885                         p_xmitPool->needBuffers = TRUE;\r
886                         viport_stopXmit( pData->p_viport );\r
887 #ifdef VNIC_STATISTIC\r
888                         pData->statistics.kickReqs++;\r
889 #endif /* VNIC_STATISTIC */\r
890                 }\r
891                 else if ( p_xmitPool->nextXmitBuf == p_xmitPool->lastCompBuf )\r
892                 {\r
893                         VNIC_TRACE( VNIC_DBG_DATA,\r
894                                                 ("Just used our last xmit buffer\n") );\r
895                 \r
896                         p_xmitPool->needBuffers = TRUE;\r
897                         viport_stopXmit( pData->p_viport );\r
898                 }\r
899 \r
900                 (*ppBpe)->valid  = 0;\r
901 \r
902                 KeReleaseSpinLock( &pData->xmitBufLock, flags );\r
903                 return TRUE;\r
904         }\r
905         else\r
906         {\r
907 #ifdef VNIC_STATISTIC\r
908                 pData->statistics.noXmitBufs++;\r
909 #endif /* VNIC_STATISTIC */\r
910         \r
911                 VNIC_TRACE( VNIC_DBG_ERROR,\r
912                                         ("Out of xmit buffers\n") );\r
913 \r
914                 viport_stopXmit( pData->p_viport );\r
915 \r
916                 KeReleaseSpinLock( &pData->xmitBufLock, flags );\r
917                 return FALSE;\r
918         }\r
919 }\r
920 \r
921 static void\r
922 data_checkXmitBuffers(\r
923                 IN                      Data_t          *pData )\r
924 {\r
925         XmitPool_t      *p_xmitPool = &pData->xmitPool;\r
926         KIRQL           flags;\r
927 \r
928         VNIC_ENTER( VNIC_DBG_DATA );\r
929 \r
930         KeAcquireSpinLock( &pData->xmitBufLock, &flags );\r
931 \r
932         if ( pData->xmitPool.needBuffers\r
933                 && p_xmitPool->bufPool[p_xmitPool->nextXmitPool].valid\r
934                 && p_xmitPool->nextXmitBuf != p_xmitPool->lastCompBuf )\r
935         {\r
936                 pData->xmitPool.needBuffers = FALSE;\r
937                 viport_restartXmit( pData->p_viport );\r
938 \r
939                 VNIC_TRACE( VNIC_DBG_DATA,\r
940                                                 ("There are free xmit buffers\n") );\r
941         }\r
942 \r
943         KeReleaseSpinLock( &pData->xmitBufLock, flags );\r
944 \r
945         VNIC_EXIT( VNIC_DBG_DATA );\r
946         return;\r
947 }\r
948 \r
949 static void\r
950 data_rdmaPacket(\r
951                         IN                      Data_t                          *pData,\r
952                         IN                      BufferPoolEntry_t       *pBpe,\r
953                         IN                      RdmaIo_t                        *pRdmaIo )\r
954 {\r
955         ib_send_wr_t    *pWrq;\r
956         uint64_t                remote_addr;\r
957 \r
958         VNIC_ENTER( VNIC_DBG_DATA );\r
959 \r
960         pWrq = &pRdmaIo->io.wrq;\r
961 \r
962         remote_addr  = ntoh64( pBpe->remoteAddr );\r
963         remote_addr += pData->xmitPool.bufferSz;\r
964         remote_addr -= ( pRdmaIo->len + sizeof(ViportTrailer_t) );\r
965 \r
966         pWrq->remote_ops.vaddr          = remote_addr;\r
967         pWrq->remote_ops.rkey           = pBpe->rKey;\r
968 \r
969         pData->xmitPool.notifyCount++;\r
970 \r
971         if( pData->xmitPool.notifyCount >= pData->xmitPool.notifyBundle )\r
972         {\r
973                 pData->xmitPool.notifyCount = 0;\r
974                 pWrq->send_opt = IB_SEND_OPT_SIGNALED;\r
975         }\r
976         else\r
977         {\r
978                 pWrq->send_opt &= ~IB_SEND_OPT_SIGNALED;\r
979         }\r
980         pWrq->send_opt = IB_SEND_OPT_SIGNALED;\r
981 \r
982         if( ibqp_postSend( &pData->qp, &pRdmaIo->io ) != IB_SUCCESS )\r
983         {\r
984                 VNIC_TRACE(VNIC_DBG_ERROR,\r
985                                         ("Failed sending data to EIOC\n") );\r
986                 viport_failure( pData->p_viport );\r
987                 return;\r
988         }\r
989 #ifdef VNIC_STATISTIC\r
990         pData->statistics.xmitNum++;\r
991 #endif /* VNIC_STATISTIC */\r
992 \r
993         VNIC_EXIT( VNIC_DBG_DATA );\r
994 }\r
995 \r
996 static NDIS_PACKET *\r
997 _data_recv_to_ndis_pkt(\r
998                 IN              Data_t                  *pData,\r
999                 IN              RdmaDest_t              *pRdmaDest )\r
1000 {\r
1001         struct ViportTrailer *pTrailer;\r
1002         NDIS_PACKET             *p_packet;\r
1003         NDIS_STATUS             ndis_status;\r
1004         int                  start;\r
1005         unsigned int         len;\r
1006         uint8_t rxChksumFlags;\r
1007         NDIS_TCP_IP_CHECKSUM_PACKET_INFO  packet_info;\r
1008 \r
1009         VNIC_ENTER( VNIC_DBG_DATA );\r
1010 \r
1011         pTrailer = pRdmaDest->pTrailer;\r
1012         start = (int)data_offset(pData, pTrailer);\r
1013         len = data_len(pData, pTrailer);\r
1014 \r
1015         NdisAllocatePacket( &ndis_status,\r
1016                                                 &p_packet,\r
1017                                                 pData->h_recv_pkt_pool );\r
1018 \r
1019         if ( ndis_status != NDIS_STATUS_SUCCESS )\r
1020         {\r
1021                 VNIC_TRACE_EXIT( VNIC_DBG_ERROR,\r
1022                         ( "NdisAllocatePacket failed %#x\n", ndis_status ) );\r
1023                 return NULL;\r
1024         }\r
1025         NdisAllocateBuffer( &ndis_status,\r
1026                                                 &pRdmaDest->p_buf,\r
1027                                                 pData->h_recv_buf_pool,\r
1028                                                 pRdmaDest->data + start,\r
1029                                                 len );\r
1030 \r
1031         if ( ndis_status != NDIS_STATUS_SUCCESS )\r
1032         {\r
1033                 VNIC_TRACE_EXIT( VNIC_DBG_ERROR,\r
1034                         ( "NdisAllocateBuffer failed %#x\n", ndis_status ) );\r
1035                 NdisFreePacket( p_packet );\r
1036                 return NULL;\r
1037         }\r
1038 \r
1039         NdisChainBufferAtFront( p_packet, pRdmaDest->p_buf );\r
1040         pRdmaDest->p_packet = p_packet;\r
1041 \r
1042         if ( pTrailer->pktFlags & PF_VLAN_INSERT )\r
1043         {\r
1044                 /*      TODO:\r
1045                  *      add OID_GEN_VLAN_ID\r
1046                  *      handle VLAN     tag insertion\r
1047                  *      set packet header size = eth_hdr + 4\r
1048                  */\r
1049         }\r
1050 \r
1051         NDIS_SET_PACKET_HEADER_SIZE( p_packet, sizeof(eth_hdr_t) );\r
1052 \r
1053         rxChksumFlags = pTrailer->rxChksumFlags;\r
1054 \r
1055         VNIC_TRACE( VNIC_DBG_DATA,\r
1056                 ("rxChksumFlags = %d, LOOP = %c, IP = %c, TCP = %c, UDP = %c\n",\r
1057                 rxChksumFlags,\r
1058                 (rxChksumFlags & RX_CHKSUM_FLAGS_LOOPBACK)? 'Y': 'N',\r
1059                 (rxChksumFlags & RX_CHKSUM_FLAGS_IP_CHECKSUM_SUCCEEDED)? 'Y':\r
1060                 (rxChksumFlags & RX_CHKSUM_FLAGS_IP_CHECKSUM_FAILED)? 'N': '-',\r
1061                 (rxChksumFlags & RX_CHKSUM_FLAGS_TCP_CHECKSUM_SUCCEEDED)? 'Y':\r
1062                 (rxChksumFlags & RX_CHKSUM_FLAGS_TCP_CHECKSUM_FAILED)? 'N': '-',\r
1063                 (rxChksumFlags & RX_CHKSUM_FLAGS_UDP_CHECKSUM_SUCCEEDED)? 'Y':\r
1064                 (rxChksumFlags & RX_CHKSUM_FLAGS_UDP_CHECKSUM_FAILED)? 'N': '-') );\r
1065 \r
1066         packet_info.Value = 0;\r
1067 \r
1068         if( rxChksumFlags & RX_CHKSUM_FLAGS_IP_CHECKSUM_SUCCEEDED )\r
1069                 packet_info.Receive.NdisPacketIpChecksumSucceeded = TRUE;\r
1070         else if( rxChksumFlags & RX_CHKSUM_FLAGS_IP_CHECKSUM_FAILED )\r
1071                 packet_info.Receive.NdisPacketIpChecksumFailed = TRUE;\r
1072 \r
1073         if( rxChksumFlags & RX_CHKSUM_FLAGS_TCP_CHECKSUM_SUCCEEDED )\r
1074                 packet_info.Receive.NdisPacketTcpChecksumSucceeded = TRUE;\r
1075         else if( rxChksumFlags & RX_CHKSUM_FLAGS_TCP_CHECKSUM_FAILED )\r
1076                 packet_info.Receive.NdisPacketTcpChecksumFailed = TRUE;\r
1077 \r
1078         if( rxChksumFlags & RX_CHKSUM_FLAGS_TCP_CHECKSUM_SUCCEEDED )\r
1079                 packet_info.Receive.NdisPacketUdpChecksumSucceeded = TRUE;\r
1080         else if( rxChksumFlags & RX_CHKSUM_FLAGS_TCP_CHECKSUM_FAILED )\r
1081                 packet_info.Receive.NdisPacketUdpChecksumFailed = TRUE;\r
1082 \r
1083         NDIS_PER_PACKET_INFO_FROM_PACKET( p_packet, TcpIpChecksumPacketInfo )=\r
1084                                                                                 (void *)(uintn_t)packet_info.Value;\r
1085 \r
1086         VNIC_RECV_FROM_PACKET( p_packet ) = pRdmaDest;\r
1087         NDIS_SET_PACKET_STATUS( p_packet, NDIS_STATUS_SUCCESS );\r
1088 \r
1089         VNIC_EXIT( VNIC_DBG_DATA );\r
1090         return p_packet;\r
1091 }\r
1092 \r
1093 /* NOTE: This routine is not reentrant */\r
1094 static void\r
1095 _data_allocBuffers(\r
1096                 IN                      Data_t          *pData,\r
1097                 IN                      BOOLEAN         initialAllocation )\r
1098 {\r
1099         RecvPool_t     *p_recvPool = &pData->recvPool;\r
1100         RdmaDest_t     *pRdmaDest;\r
1101         LIST_ENTRY              *p_list_entry;\r
1102         int            index;\r
1103 \r
1104         VNIC_ENTER( VNIC_DBG_DATA );\r
1105 \r
1106         index = ADD(p_recvPool->nextFreeBuf, p_recvPool->numFreeBufs, p_recvPool->eiocPoolSz);\r
1107 \r
1108         while ( !IsListEmpty( &p_recvPool->availRecvBufs ) )\r
1109         {\r
1110                 p_list_entry = RemoveHeadList( &p_recvPool->availRecvBufs );\r
1111                 pRdmaDest       = (RdmaDest_t*)p_list_entry;\r
1112         \r
1113                 if( initialAllocation )\r
1114                 {\r
1115                         pRdmaDest->buf_sz = p_recvPool->bufferSz;\r
1116                         pRdmaDest->pTrailer =\r
1117                                 (struct ViportTrailer*)(pRdmaDest->data + pRdmaDest->buf_sz\r
1118                                 - sizeof(struct ViportTrailer));\r
1119                         pRdmaDest->pTrailer->connectionHashAndValid = 0;\r
1120                 }\r
1121 \r
1122                 pRdmaDest->p_packet = NULL;\r
1123                 _data_addFreeBuffer(pData, index, pRdmaDest);\r
1124                 index = NEXT(index,p_recvPool->eiocPoolSz);\r
1125         }\r
1126 \r
1127         VNIC_EXIT( VNIC_DBG_DATA );\r
1128         return;\r
1129 }\r
1130 \r
1131 static void\r
1132 _data_addFreeBuffer(\r
1133                 IN              Data_t                  *pData,\r
1134                 IN              int                             index,\r
1135             IN          RdmaDest_t              *pRdmaDest )\r
1136 {\r
1137         RecvPool_t        *p_recvPool = &pData->recvPool;\r
1138         BufferPoolEntry_t *pBpe;\r
1139 \r
1140         pRdmaDest->pTrailer->connectionHashAndValid = 0;\r
1141         pBpe = &p_recvPool->bufPool[index];\r
1142 \r
1143         pBpe->rKey       = pRdmaDest->region.rkey;\r
1144         pBpe->remoteAddr = hton64( PTR64( pRdmaDest->data ) );\r
1145         pBpe->valid      = (uint32_t)(pRdmaDest - &p_recvPool->pRecvBufs[0]) + 1;\r
1146         ++p_recvPool->numFreeBufs;\r
1147 \r
1148         return;\r
1149 }\r
1150 \r
1151 static uint32_t\r
1152 _data_incomingRecv(\r
1153                 IN              Data_t          *pData )\r
1154 {\r
1155         RecvPool_t        *p_recvPool = &pData->recvPool;\r
1156         RdmaDest_t        *pRdmaDest;\r
1157         ViportTrailer_t   *pTrailer;\r
1158         BufferPoolEntry_t *pBpe;\r
1159         NDIS_PACKET             *p_packet = NULL;\r
1160         uint32_t                idx = 0;\r
1161         BOOLEAN status = FALSE;\r
1162 \r
1163         VNIC_ENTER( VNIC_DBG_DATA );\r
1164 \r
1165         while( !status )\r
1166         {\r
1167                 if ( p_recvPool->nextFullBuf == p_recvPool->nextFreeBuf )\r
1168                         return idx;\r
1169 \r
1170                 pBpe = &p_recvPool->bufPool[p_recvPool->nextFullBuf];\r
1171                 pRdmaDest = &p_recvPool->pRecvBufs[pBpe->valid - 1];\r
1172                 pTrailer = pRdmaDest->pTrailer;\r
1173 \r
1174                 if ( ( pTrailer != NULL ) &&\r
1175                         ( pTrailer->connectionHashAndValid & CHV_VALID ) )\r
1176                 {\r
1177                         /* received a packet */\r
1178                         if ( pTrailer->pktFlags & PF_KICK )\r
1179                         {\r
1180                                 p_recvPool->kickOnFree = TRUE;\r
1181                         }\r
1182                         /* we do not want to indicate packet if  no filter is set */\r
1183                         if( pData->p_viport->p_adapter->packet_filter )\r
1184                         {\r
1185                                 p_packet = _data_recv_to_ndis_pkt( pData, pRdmaDest );\r
1186                                 if ( p_packet != NULL )\r
1187                                 {\r
1188                                         p_recvPool->recv_pkt_array[idx++] = p_packet;\r
1189                                 }\r
1190                         }\r
1191                         else\r
1192                         {       /* put back to free buffers pool */\r
1193                                 InsertTailList( &p_recvPool->availRecvBufs,\r
1194                                         &pRdmaDest->listPtrs );\r
1195                         }\r
1196                         pBpe->valid = 0;\r
1197                         INC( p_recvPool->nextFullBuf, 1, p_recvPool->eiocPoolSz );\r
1198                         p_recvPool->numPostedBufs--;\r
1199 #ifdef VNIC_STATISTIC\r
1200                         pData->statistics.recvNum++;\r
1201 #endif /* VNIC_STATISTIC */\r
1202                 }\r
1203                 else\r
1204                         break;\r
1205         }\r
1206         return idx;\r
1207 }\r
1208 \r
1209 \r
1210 void\r
1211 vnic_return_packet(\r
1212         IN              NDIS_HANDLE             adapter_context,\r
1213         IN              NDIS_PACKET     * const p_packet )\r
1214 {\r
1215 \r
1216 \r
1217         vnic_adapter_t  *p_adapter = (vnic_adapter_t *)adapter_context;\r
1218         viport_t        *p_viport = p_adapter->p_currentPath->pViport;\r
1219         RdmaDest_t      *p_rdma_dest = VNIC_RECV_FROM_PACKET( p_packet );\r
1220 \r
1221         ASSERT( p_rdma_dest->p_packet == p_packet );\r
1222         _data_return_recv( p_packet );\r
1223         p_rdma_dest->p_packet = NULL;\r
1224 \r
1225         InsertTailList( &p_viport->data.recvPool.availRecvBufs,\r
1226                                         &p_rdma_dest->listPtrs );\r
1227 \r
1228         if( p_viport->data.connected == TRUE &&\r
1229                 !p_viport->errored )\r
1230         {\r
1231                 _data_allocBuffers( &p_viport->data, FALSE );\r
1232                 _data_sendFreeRecvBuffers( &p_viport->data );\r
1233         }\r
1234 }\r
1235 static void\r
1236 _data_return_recv(\r
1237           IN    NDIS_PACKET             *p_packet )\r
1238 {\r
1239         NDIS_BUFFER             *p_buf;\r
1240         /* Unchain the NDIS buffer. */\r
1241         NdisUnchainBufferAtFront( p_packet, &p_buf );\r
1242         CL_ASSERT( p_buf );\r
1243         /* Return the NDIS packet and NDIS buffer to their pools. */\r
1244         NdisFreeBuffer( p_buf );\r
1245         NdisFreePacket( p_packet );\r
1246 }\r
1247 \r
1248 static void\r
1249 _data_sendFreeRecvBuffers(\r
1250                         IN                      Data_t          *pData )\r
1251 {\r
1252         RecvPool_t  *p_recvPool = &pData->recvPool;\r
1253         ib_send_wr_t *pWrq = &pData->freeBufsIo.io.wrq;\r
1254         BOOLEAN     bufsSent = FALSE;\r
1255         uint64_t    rdmaAddr;\r
1256         uint32_t    offset;\r
1257         uint32_t    sz;\r
1258         unsigned int numToSend,\r
1259                     nextIncrement;\r
1260 \r
1261         VNIC_ENTER( VNIC_DBG_DATA );\r
1262 \r
1263         for (   numToSend = p_recvPool->szFreeBundle;\r
1264                         numToSend <= p_recvPool->numFreeBufs;\r
1265                         numToSend += p_recvPool->szFreeBundle )\r
1266         {\r
1267                 /* Handle multiple bundles as one when possible. */\r
1268                 nextIncrement = numToSend + p_recvPool->szFreeBundle;\r
1269                 if (    ( nextIncrement <= p_recvPool->numFreeBufs )\r
1270                                 && ( p_recvPool->nextFreeBuf + nextIncrement <= p_recvPool->eiocPoolSz ) )\r
1271                 {\r
1272                         continue;\r
1273                 }\r
1274 \r
1275                 offset   = p_recvPool->nextFreeBuf * sizeof(BufferPoolEntry_t);\r
1276                 sz       = numToSend * sizeof(BufferPoolEntry_t);\r
1277                 rdmaAddr = p_recvPool->eiocRdmaAddr + offset;\r
1278         \r
1279                 pWrq->ds_array->length  = sz;\r
1280                 pWrq->ds_array->vaddr = PTR64((uint8_t *)p_recvPool->bufPool + offset);\r
1281                 pWrq->remote_ops.vaddr = rdmaAddr;\r
1282 \r
1283                 if ( ibqp_postSend( &pData->qp, &pData->freeBufsIo.io ) != IB_SUCCESS )\r
1284                 {\r
1285                         VNIC_TRACE(VNIC_DBG_ERROR,\r
1286                                         ("Unable to rdma free buffers to EIOC\n") );\r
1287 \r
1288                         viport_failure( pData->p_viport );\r
1289                         break;\r
1290                 }\r
1291 \r
1292                 INC( p_recvPool->nextFreeBuf, numToSend, p_recvPool->eiocPoolSz );\r
1293                 p_recvPool->numFreeBufs -= numToSend;\r
1294                 p_recvPool->numPostedBufs += numToSend;\r
1295                 bufsSent = TRUE;\r
1296         }\r
1297 \r
1298         if( bufsSent )\r
1299         {\r
1300                 if( p_recvPool->kickOnFree )\r
1301                 {\r
1302                         data_sendKickMessage( pData );\r
1303                 }\r
1304         }\r
1305         if( p_recvPool->numPostedBufs == 0 )\r
1306         {\r
1307                 VNIC_TRACE( VNIC_DBG_ERROR,\r
1308                                 ("IOC %d: Unable to allocate receive buffers\n",\r
1309                                         pData->p_viport->ioc_num ) );\r
1310         \r
1311                 viport_failure( pData->p_viport );\r
1312         }\r
1313         VNIC_EXIT( VNIC_DBG_DATA );\r
1314 }\r
1315 \r
1316 \r
1317 void\r
1318 data_cleanup(\r
1319         IN                      Data_t  *pData )\r
1320 {\r
1321         VNIC_ENTER( VNIC_DBG_DATA );\r
1322 \r
1323         if( pData->recvPool.recv_pkt_array )\r
1324         {\r
1325                 cl_free( pData->recvPool.recv_pkt_array );\r
1326                 pData->recvPool.recv_pkt_array = NULL;\r
1327                 pData->recvPool.poolSz = 0;\r
1328         }\r
1329 \r
1330         if ( pData->pLocalStorage )\r
1331         {\r
1332                 NdisFreeMemory( pData->pLocalStorage, pData->localStorageSz, 0 );\r
1333                 pData->pLocalStorage = NULL;\r
1334         }\r
1335 \r
1336         if(  pData->h_recv_buf_pool )\r
1337         {\r
1338                 NdisFreeBufferPool( pData->h_recv_buf_pool );\r
1339                 pData->h_recv_buf_pool = NULL;\r
1340         }\r
1341 \r
1342         if ( pData->h_recv_pkt_pool )\r
1343         {\r
1344                 while( NdisPacketPoolUsage(pData->h_recv_pkt_pool) != 0)\r
1345                 {\r
1346                         VNIC_TRACE( VNIC_DBG_WARN,\r
1347                                 ("Recv packet pool is not empty!!!\n") );\r
1348                         NdisMSleep(100);\r
1349                 }\r
1350                 NdisFreePacketPool( pData->h_recv_pkt_pool );\r
1351                 pData->h_recv_pkt_pool = NULL;\r
1352         }\r
1353         if( pData->p_recv_bufs )\r
1354         {\r
1355                 NdisFreeMemory( pData->p_recv_bufs, pData->recv_bufs_sz, 0 );\r
1356                 pData->p_recv_bufs = NULL;\r
1357         }\r
1358         // clear Qp struct for reuse\r
1359         cl_memclr( &pData->qp, sizeof( IbQp_t) );\r
1360 \r
1361         cl_timer_destroy( &pData->kickTimer );\r
1362 \r
1363         VNIC_EXIT( VNIC_DBG_DATA );\r
1364 }\r
1365 \r
1366 \r
1367 static void\r
1368 _data_kickTimer_start(\r
1369                   IN    Data_t          *pData,\r
1370                   IN    uint32_t        microseconds )\r
1371 {\r
1372         VNIC_ENTER( VNIC_DBG_DATA );\r
1373 \r
1374         InterlockedExchange( (LONG *)&pData->kickTimerOn, TRUE );\r
1375 \r
1376         usec_timer_start(&pData->kickTimer, microseconds );\r
1377 \r
1378         VNIC_EXIT( VNIC_DBG_DATA );\r
1379         return;\r
1380 }\r
1381 \r
1382 static void\r
1383 _data_kickTimer_stop(\r
1384                   IN    Data_t          *pData )\r
1385 {\r
1386         VNIC_ENTER( VNIC_DBG_DATA );\r
1387 \r
1388         if( InterlockedExchange( &pData->kickTimerOn, FALSE ) == TRUE )\r
1389         {\r
1390                 cl_timer_stop( &pData->kickTimer );\r
1391         }\r
1392 \r
1393         VNIC_EXIT( VNIC_DBG_DATA );\r
1394 }\r