[VNIC] nasty bug.
[mirror/winof/.git] / ulp / inic / kernel / vnic_data.c
1 /*\r
2  * Copyright (c) 2005 SilverStorm Technologies.  All rights reserved.\r
3  *\r
4  * This software is available to you under the OpenIB.org BSD license\r
5  * below:\r
6  *\r
7  *     Redistribution and use in source and binary forms, with or\r
8  *     without modification, are permitted provided that the following\r
9  *     conditions are met:\r
10  *\r
11  *      - Redistributions of source code must retain the above\r
12  *        copyright notice, this list of conditions and the following\r
13  *        disclaimer.\r
14  *\r
15  *      - Redistributions in binary form must reproduce the above\r
16  *        copyright notice, this list of conditions and the following\r
17  *        disclaimer in the documentation and/or other materials\r
18  *        provided with the distribution.\r
19  *\r
20  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,\r
21  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF\r
22  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND\r
23  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS\r
24  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN\r
25  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN\r
26  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\r
27  * SOFTWARE.\r
28  *\r
29  * $Id$\r
30  */\r
31 #include <complib/comp_lib.h>\r
32 #include "vnic_driver.h"\r
33 \r
34 static void    data_postRecvs(Data_t *pData);\r
35 static void    _data_receivedKick(Io_t *pIo);\r
36 static void    _data_xmitComplete(Io_t *pIo);\r
37 static void    data_sendKickMessage(Data_t *pData);\r
38 static void    _data_kickTimeoutHandler( void *context );\r
39 static BOOLEAN data_allocXmitBuffer(Data_t *pData,\r
40                                         BufferPoolEntry_t **ppBpe, RdmaIo_t **ppRdmaIo, BOOLEAN *pLast);\r
41 static void    data_checkXmitBuffers(Data_t *pData);\r
42 static void    data_rdmaPacket(Data_t *pData, BufferPoolEntry_t *pBpe, RdmaIo_t *pRdmaIo);\r
43 \r
44 static NDIS_PACKET *\r
45 _data_recv_to_ndis_pkt( Data_t *pData, RdmaDest_t *pRdmaDest );\r
46 \r
47 static void\r
48 _data_allocBuffers(\r
49                                         Data_t          *pData,\r
50                                         BOOLEAN         initialAllocation );\r
51 \r
52 static void\r
53 _data_addFreeBuffer(\r
54                                         Data_t          *pData,\r
55                                         int                     index,\r
56                                         RdmaDest_t      *pRdmaDest );\r
57 \r
58 static uint32_t\r
59 _data_incomingRecv(\r
60                                         Data_t          *pData );\r
61 \r
62 static void   \r
63 _data_sendFreeRecvBuffers(\r
64                                         Data_t          *pData );\r
65 \r
66 static uint8_t\r
67 _tx_chksum_flags(\r
68  IN             NDIS_PACKET* const              p_packet );\r
69 \r
70 \r
71 static void\r
72 _get_first_buffer(\r
73                 IN              NDIS_PACKET             *p_packet,\r
74                 IN OUT  NDIS_BUFFER             **pp_buf_desc,\r
75                 OUT             void                    **pp_buf,\r
76                 OUT             ULONG                   *p_packet_sz );\r
77 \r
78 static void\r
79 _data_return_recv(\r
80           IN    NDIS_PACKET             *p_packet );\r
81 \r
82 static void\r
83 _data_kickTimer_start(\r
84                   IN    Data_t          *pData,\r
85                   IN    uint32_t        microseconds );\r
86 \r
87 static void\r
88 _data_kickTimer_stop(\r
89                   IN    Data_t          *pData );\r
90 \r
91 #define LOCAL_IO(x) PTR64((x))\r
92 \r
93 #define INBOUND_COPY\r
94 \r
95 #ifdef VNIC_STATISTIC\r
96 int64_t recvRef;\r
97 #endif /* VNIC_STATISTIC */\r
98 \r
99 void\r
100 data_construct(\r
101         IN              Data_t                  *pData,\r
102         IN              viport_t                *pViport )\r
103 {\r
104         VNIC_ENTER( VNIC_DBG_DATA );\r
105 \r
106         RtlZeroMemory( pData, sizeof(*pData) );\r
107 \r
108         pData->p_viport = pViport;\r
109         pData->p_phy_region = &pViport->p_adapter->ca.region;\r
110         InitializeListHead( &pData->recvIos );\r
111         KeInitializeSpinLock ( &pData->recvIosLock );\r
112         KeInitializeSpinLock ( &pData->xmitBufLock );\r
113         cl_timer_construct( &pData->kickTimer );\r
114 \r
115         ibqp_construct( &pData->qp, pViport );\r
116 \r
117         VNIC_EXIT( VNIC_DBG_DATA );\r
118 }\r
119 \r
120 \r
121 ib_api_status_t\r
122 data_init(\r
123         IN              Data_t                  *pData,\r
124         IN              DataConfig_t    *p_conf,\r
125         IN              uint64_t                guid )\r
126 {\r
127         ib_api_status_t ib_status;\r
128 \r
129         VNIC_ENTER( VNIC_DBG_DATA );\r
130 \r
131         ASSERT( pData->p_viport != NULL );\r
132         pData->p_conf = p_conf;\r
133 \r
134         cl_timer_init( &pData->kickTimer, _data_kickTimeoutHandler, pData );\r
135 \r
136         ib_status = ibqp_init(&pData->qp, guid, &p_conf->ibConfig );\r
137         if( ib_status != IB_SUCCESS )\r
138                 VNIC_TRACE( VNIC_DBG_ERROR, ("data ibqp_init failed\n") );\r
139 \r
140         VNIC_EXIT( VNIC_DBG_DATA );\r
141         return ib_status;\r
142 }\r
143 \r
144 \r
145 ib_api_status_t\r
146 data_connect(\r
147                 IN              Data_t          *pData )\r
148 {\r
149         NDIS_STATUS                     status;\r
150         ib_api_status_t         ib_status;\r
151         XmitPool_t        *pXmitPool = &pData->xmitPool;\r
152         RecvPool_t        *pRecvPool = &pData->recvPool;\r
153         RecvIo_t          *pRecvIo;\r
154         SendIo_t          *pSendIo;\r
155         RdmaIo_t          *pRdmaIo;\r
156         RdmaDest_t        *pRdmaDest;\r
157         uint8_t           *pRegionData;\r
158         int               sz, regionSz;\r
159         unsigned int      i, j;\r
160 \r
161         VNIC_ENTER( VNIC_DBG_DATA );\r
162 \r
163         pRecvPool->poolSz           = pData->p_conf->hostRecvPoolEntries;\r
164         pRecvPool->eiocPoolSz       = pData->hostPoolParms.numRecvPoolEntries;\r
165 \r
166         if ( pRecvPool->poolSz > pRecvPool->eiocPoolSz )\r
167         {\r
168                 pRecvPool->poolSz = pData->hostPoolParms.numRecvPoolEntries;\r
169         }\r
170         pRecvPool->szFreeBundle     =\r
171                         pData->hostPoolParms.freeRecvPoolEntriesPerUpdate;\r
172         pRecvPool->numFreeBufs      = 0;\r
173         pRecvPool->numPostedBufs    = 0;\r
174         pRecvPool->nextFullBuf      = 0;\r
175         pRecvPool->nextFreeBuf      = 0;\r
176         pRecvPool->kickOnFree       = FALSE;\r
177         pRecvPool->bufferSz         = pData->hostPoolParms.sizeRecvPoolEntry;\r
178 \r
179         pXmitPool->bufferSz         = pData->eiocPoolParms.sizeRecvPoolEntry;\r
180         pXmitPool->poolSz           = pData->eiocPoolParms.numRecvPoolEntries;\r
181         pXmitPool->notifyCount      = 0;\r
182         pXmitPool->notifyBundle     = pData->p_conf->notifyBundle;\r
183         pXmitPool->nextXmitPool     = 0;\r
184 \r
185 #if TRUE // LIMIT_OUTSTANDING_SENDS\r
186         pXmitPool->numXmitBufs      = pXmitPool->notifyBundle * 2;\r
187 #else /* !LIMIT_OUTSTANDING_SENDS */\r
188         pXmitPool->numXmitBufs      = pXmitPool->poolSz;\r
189 #endif /* LIMIT_OUTSTANDING_SENDS */\r
190 \r
191         pXmitPool->nextXmitBuf      = 0;\r
192         pXmitPool->lastCompBuf      = pXmitPool->numXmitBufs - 1;\r
193         pXmitPool->kickCount        = 0;\r
194         pXmitPool->kickByteCount    = 0;\r
195         pXmitPool->sendKicks        =\r
196                 (BOOLEAN)(( pData->eiocPoolParms.numRecvPoolEntriesBeforeKick != 0 )\r
197                 || ( pData->eiocPoolParms.numRecvPoolBytesBeforeKick != 0 ));\r
198         pXmitPool->kickBundle       =\r
199                 pData->eiocPoolParms.numRecvPoolEntriesBeforeKick;\r
200         pXmitPool->kickByteBundle   =\r
201                 pData->eiocPoolParms.numRecvPoolBytesBeforeKick;\r
202         pXmitPool->needBuffers      = TRUE;\r
203 \r
204         sz  = sizeof(RdmaDest_t) * pRecvPool->poolSz;\r
205         sz += sizeof(RecvIo_t) * pData->p_conf->numRecvs;\r
206         sz += sizeof(RdmaIo_t) * pXmitPool->numXmitBufs;\r
207 \r
208         regionSz  = 4 * pData->p_conf->numRecvs;\r
209         regionSz += sizeof(BufferPoolEntry_t) * pRecvPool->eiocPoolSz;\r
210         regionSz += sizeof(BufferPoolEntry_t) * pXmitPool->poolSz;\r
211         sz       += regionSz;\r
212 \r
213         status = NdisAllocateMemoryWithTag( &pData->pLocalStorage,\r
214                                                                                 (UINT)sz,\r
215                                                                                 'grtS' );\r
216         if ( status != NDIS_STATUS_SUCCESS )\r
217         {\r
218                 VNIC_TRACE_EXIT( VNIC_DBG_ERROR,\r
219                         ("Failed allocating %d bytes local storage\n", sz ) );\r
220                 ib_status = IB_INSUFFICIENT_MEMORY;\r
221                 goto err1;\r
222         }\r
223 \r
224         NdisZeroMemory( pData->pLocalStorage, sz );\r
225         pData->localStorageSz = sz;\r
226 \r
227         pRecvPool->pRecvBufs = (RdmaDest_t *)pData->pLocalStorage;\r
228         sz                   = sizeof(RdmaDest_t) * pRecvPool->poolSz;\r
229         pRecvIo              = (RecvIo_t *)(pData->pLocalStorage + sz);\r
230         sz                  += sizeof(RecvIo_t) * pData->p_conf->numRecvs;\r
231 \r
232         pXmitPool->pXmitBufs = (RdmaIo_t *)(pData->pLocalStorage + sz);\r
233         sz                  += sizeof(RdmaIo_t) * pXmitPool->numXmitBufs;\r
234 \r
235         pRegionData          = pData->pLocalStorage + sz;\r
236         sz                  += 4 * pData->p_conf->numRecvs;\r
237 \r
238         pRecvPool->bufPool   = (BufferPoolEntry_t *)(pData->pLocalStorage + sz);\r
239         sz                  += sizeof(BufferPoolEntry_t) * pRecvPool->eiocPoolSz;\r
240         pXmitPool->bufPool   = (BufferPoolEntry_t *)(pData->pLocalStorage + sz);\r
241 \r
242         ib_status = ibregion_init( pData->p_viport, &pData->region,\r
243                 pData->p_viport->p_adapter->ca.hPd, pRegionData, regionSz,\r
244                 ( IB_AC_LOCAL_WRITE | IB_AC_RDMA_WRITE ) );\r
245         if( ib_status != IB_SUCCESS )\r
246         {\r
247                 VNIC_TRACE_EXIT( VNIC_DBG_ERROR, ("ib_region_init failed\n") );\r
248                 goto err2;\r
249         }\r
250 \r
251         pRdmaIo = &pData->freeBufsIo;\r
252         pRdmaIo->io.pViport          = pData->p_viport;\r
253         pRdmaIo->io.pRoutine         = NULL;\r
254         pRdmaIo->io.wrq.p_next       = NULL;\r
255         pRdmaIo->io.wrq.wr_type      = WR_RDMA_WRITE;\r
256         pRdmaIo->io.wrq.wr_id        = PTR64( pRdmaIo );\r
257         pRdmaIo->io.wrq.num_ds       = 1;\r
258         pRdmaIo->io.wrq.ds_array     = pRdmaIo->dsList;\r
259         pRdmaIo->dsList[0].lkey      = pData->region.lkey;\r
260         pRdmaIo->io.wrq.send_opt     = IB_SEND_OPT_SIGNALED;\r
261 \r
262         pSendIo = &pData->kickIo;\r
263         pSendIo->io.pViport          = pData->p_viport;\r
264         pSendIo->io.pRoutine         = NULL;\r
265         pSendIo->io.wrq.p_next       = NULL;\r
266         pSendIo->io.wrq.wr_type      = WR_SEND;\r
267         pSendIo->io.wrq.wr_id        = PTR64( pSendIo );\r
268         pSendIo->io.wrq.num_ds       = 1;\r
269         pSendIo->io.wrq.ds_array     = &pSendIo->dsList;\r
270 \r
271         pSendIo->io.wrq.send_opt     = IB_SEND_OPT_SIGNALED;\r
272 \r
273         pSendIo->dsList.length       = 0;\r
274         pSendIo->dsList.vaddr        = PTR64( pRegionData );\r
275         pSendIo->dsList.lkey         = pData->region.lkey;\r
276 \r
277         for ( i = 0; i < pData->p_conf->numRecvs; i++ )\r
278         {\r
279                 pRecvIo[i].io.pViport         = pData->p_viport;\r
280                 pRecvIo[i].io.pRoutine        = _data_receivedKick;\r
281                 pRecvIo[i].io.r_wrq.wr_id     = PTR64( &pRecvIo[i].io );\r
282                 pRecvIo[i].io.r_wrq.p_next    = NULL;\r
283                 pRecvIo[i].io.r_wrq.num_ds    = 1;\r
284                 pRecvIo[i].io.r_wrq.ds_array  = &pRecvIo[i].dsList;\r
285                 pRecvIo[i].dsList.length      = 4;\r
286                 pRecvIo[i].dsList.vaddr       = PTR64( pRegionData );\r
287                 pRecvIo[i].dsList.lkey        = pData->region.lkey;\r
288         \r
289                 ExInterlockedInsertTailList( &pData->recvIos, &pRecvIo[i].io.listPtrs, &pData->recvIosLock );\r
290         /* Do not need to move pointer since the receive info\r
291          * is not read.  Note, we could reduce the amount\r
292          * of memory allocated and the size of the region.\r
293                  * pRegionData                  += 4;\r
294          * */\r
295         }\r
296 \r
297         sz = pRecvPool->poolSz * pRecvPool->bufferSz;\r
298         status = NdisAllocateMemoryWithTag(&pData->p_recv_bufs,\r
299                                                                                 sz, 'fubr');\r
300         if( status != NDIS_STATUS_SUCCESS )\r
301         {\r
302                 VNIC_TRACE_EXIT( VNIC_DBG_ERROR,\r
303                                 ("Allocate recv buffers failed\n"));\r
304                 ib_status = IB_INSUFFICIENT_MEMORY;\r
305                 goto err3;\r
306         }\r
307         NdisZeroMemory( pData->p_recv_bufs, sz );\r
308 \r
309         pData->recv_bufs_sz = sz;\r
310 \r
311         ib_status = ibregion_init( pData->p_viport, &pData->rbuf_region,\r
312                 pData->p_viport->p_adapter->ca.hPd, pData->p_recv_bufs, sz,\r
313                 (IB_AC_LOCAL_WRITE | IB_AC_RDMA_WRITE) );\r
314         if( ib_status != IB_SUCCESS )\r
315         {\r
316                 goto err4;\r
317         }\r
318 \r
319         NdisAllocatePacketPool( &status,\r
320                                                         &pData->h_recv_pkt_pool,\r
321                                                         pRecvPool->poolSz,\r
322                                                         PROTOCOL_RESERVED_SIZE_IN_PACKET );\r
323 \r
324         if( status != NDIS_STATUS_SUCCESS )\r
325         {\r
326                 VNIC_TRACE_EXIT( VNIC_DBG_ERROR,\r
327                         ("Allocate packet pool failed status %#x\n", status ));\r
328                 ib_status = IB_INSUFFICIENT_MEMORY;\r
329                 goto err5;\r
330         }\r
331 \r
332         NdisAllocateBufferPool(\r
333                 &status, &pData->h_recv_buf_pool, pRecvPool->poolSz );\r
334 \r
335         if( status != NDIS_STATUS_SUCCESS )\r
336         {\r
337                 VNIC_TRACE_EXIT( VNIC_DBG_ERROR,\r
338                         ("Allocate packet pool failed status %#x\n", status ));\r
339                 ib_status = IB_INSUFFICIENT_MEMORY;\r
340                 goto err6;\r
341         }\r
342         pData->recvPool.recv_pkt_array =\r
343                 cl_zalloc( sizeof(NDIS_PACKET*)* pRecvPool->poolSz );\r
344         if( !pData->recvPool.recv_pkt_array )\r
345         {\r
346                 VNIC_TRACE_EXIT( VNIC_DBG_ERROR,\r
347                                 ("Allocate packet array failed\n" ) );\r
348                 ib_status = IB_INSUFFICIENT_MEMORY;\r
349                 goto err7;\r
350         }\r
351 \r
352         InitializeListHead( &pRecvPool->availRecvBufs );\r
353 \r
354         for ( i = 0; i < pRecvPool->poolSz; i++ )\r
355         {\r
356                 pRdmaDest = &pRecvPool->pRecvBufs[i];\r
357                 pRdmaDest->data = pData->p_recv_bufs + (i * pRecvPool->bufferSz );\r
358                 pRdmaDest->region = pData->rbuf_region;\r
359                 InsertTailList( &pRecvPool->availRecvBufs, &pRdmaDest->listPtrs );\r
360         }\r
361 \r
362         for ( i = 0; i < pXmitPool->numXmitBufs; i++ )\r
363         {\r
364                 pRdmaIo = &pXmitPool->pXmitBufs[i];\r
365                 pRdmaIo->index               = (uint16_t)i;\r
366                 pRdmaIo->io.pViport          = pData->p_viport;\r
367                 pRdmaIo->io.pRoutine         = _data_xmitComplete;\r
368                 pRdmaIo->io.wrq.p_next       = NULL;\r
369                 pRdmaIo->io.wrq.wr_type      = WR_RDMA_WRITE;\r
370                 pRdmaIo->io.wrq.wr_id        = PTR64(pRdmaIo);\r
371                 pRdmaIo->io.wrq.num_ds       = MAX_NUM_SGE; // will set actual number when transmit\r
372                 pRdmaIo->io.wrq.ds_array     = pRdmaIo->dsList;\r
373                 pRdmaIo->p_trailer                      =  (ViportTrailer_t *)&pRdmaIo->data[0];\r
374                 for( j = 0; j < MAX_NUM_SGE; j++ )\r
375                 {\r
376                         pRdmaIo->dsList[j].lkey      = pData->p_phy_region->lkey;\r
377                 }\r
378         }\r
379 \r
380         pXmitPool->rdmaRKey      = pData->region.rkey;\r
381         pXmitPool->rdmaAddr      = PTR64( pXmitPool->bufPool );\r
382 \r
383         data_postRecvs( pData );\r
384 \r
385         ib_status = ibqp_connect( &pData->qp );\r
386         if( ib_status != IB_SUCCESS )\r
387         {\r
388                 VNIC_TRACE( VNIC_DBG_ERROR, ("ibqp_connect returned %s\n",\r
389                         pData->p_viport->p_adapter->ifc.get_err_str( ib_status )) );\r
390 err7:\r
391                 NdisFreeBufferPool( pData->h_recv_buf_pool );\r
392                 pData->h_recv_buf_pool = NULL;\r
393 err6:\r
394                 NdisFreePacketPool( pData->h_recv_pkt_pool );\r
395                 pData->h_recv_pkt_pool = NULL;\r
396 err5:\r
397                 ibregion_cleanup( pData->p_viport, &pData->rbuf_region );\r
398 err4:\r
399                 NdisFreeMemory( pData->p_recv_bufs, pData->recv_bufs_sz, 0 );\r
400                 pData->p_recv_bufs = NULL;\r
401 err3:\r
402                 ibregion_cleanup(pData->p_viport, &pData->region );\r
403 err2:\r
404                 NdisFreeMemory( pData->pLocalStorage, pData->localStorageSz, 0 );\r
405                 pData->pLocalStorage = NULL;\r
406 err1:\r
407                 pRecvPool->poolSz = 0;\r
408         }\r
409 \r
410         VNIC_EXIT( VNIC_DBG_DATA );\r
411         return ib_status;\r
412 }\r
413 \r
414 \r
415 void\r
416 data_connected(\r
417                 IN              Data_t          *pData )\r
418 {\r
419         VNIC_ENTER( VNIC_DBG_DATA );\r
420 \r
421         pData->freeBufsIo.io.wrq.remote_ops.rkey =\r
422                                                         pData->recvPool.eiocRdmaRkey;\r
423 \r
424         _data_allocBuffers(pData, TRUE);\r
425         _data_sendFreeRecvBuffers(pData);\r
426         pData->connected = TRUE;\r
427 \r
428         VNIC_EXIT( VNIC_DBG_DATA );\r
429         return;\r
430 }\r
431 \r
432 void\r
433 data_disconnect(\r
434                 IN              Data_t          *pData )\r
435 {\r
436         RecvPool_t *pRecvPool = &pData->recvPool;\r
437         viport_t        *p_viport = pData->p_viport;\r
438         NDIS_PACKET             *p_packet;\r
439         LIST_ENTRY              *p_list_item;\r
440         unsigned int        i;\r
441 \r
442         VNIC_ENTER( VNIC_DBG_DATA );\r
443 \r
444         _data_kickTimer_stop ( pData );\r
445 \r
446         pData->connected = FALSE;\r
447 \r
448         ibqp_detach( &pData->qp );\r
449 \r
450         ibregion_cleanup( p_viport, &pData->rbuf_region );\r
451         ibregion_cleanup( p_viport, &pData->region );\r
452 \r
453         for ( i = 0; i < pRecvPool->poolSz; i++ )\r
454         {\r
455                 p_packet = pRecvPool->pRecvBufs[i].p_packet;\r
456                 if ( p_packet != NULL )\r
457                 {\r
458                         pRecvPool->pRecvBufs[i].p_packet = NULL;\r
459                         _data_return_recv( p_packet );\r
460                 }\r
461         }\r
462         /* clear pending queue if any */\r
463         while( ( p_list_item = NdisInterlockedRemoveHeadList(\r
464                 &p_viport->send_pending_list,\r
465                 &p_viport->pending_list_lock )) != NULL )\r
466         {\r
467                 p_packet = VNIC_PACKET_FROM_LIST_ITEM( p_list_item );\r
468                 if ( p_packet )\r
469                 {\r
470                         NDIS_SET_PACKET_STATUS( p_packet, NDIS_STATUS_FAILURE );\r
471                         NdisMSendComplete( p_viport->p_adapter->h_handle,\r
472                                 p_packet, NDIS_STATUS_FAILURE );\r
473                         p_viport->stats.ifOutErrors++;\r
474                 }\r
475         }\r
476 \r
477         VNIC_EXIT( VNIC_DBG_DATA );\r
478         return;\r
479 }\r
480 \r
481 BOOLEAN\r
482 data_xmitPacket(\r
483                 IN              Data_t                          *pData,\r
484                 IN              NDIS_PACKET* const      p_packet )\r
485 {\r
486         XmitPool_t              *p_xmitPool = &pData->xmitPool;\r
487         RdmaIo_t                *pRdmaIo;\r
488         BufferPoolEntry_t *pBpe;\r
489         BOOLEAN                 last;\r
490         uint8_t                 *p_buf;\r
491         uint32_t                buf_len;\r
492         NDIS_BUFFER             *p_buf_desc;\r
493         eth_hdr_t*              p_eth_hdr;\r
494         int                             pad;\r
495         SCATTER_GATHER_LIST             *p_sgl;\r
496         uint32_t                i;\r
497         PHYSICAL_ADDRESS phy_addr;\r
498 \r
499         VNIC_ENTER( VNIC_DBG_DATA );\r
500 \r
501         if( !data_allocXmitBuffer( pData, &pBpe, &pRdmaIo, &last ) )\r
502         {\r
503                 return FALSE;\r
504         }\r
505         p_sgl = NDIS_PER_PACKET_INFO_FROM_PACKET( p_packet,\r
506                                                                                         ScatterGatherListPacketInfo );\r
507         if ( p_sgl == NULL )\r
508         {\r
509                 return FALSE;\r
510         }\r
511 \r
512         NdisGetFirstBufferFromPacketSafe( p_packet,\r
513                                                                         &p_buf_desc,\r
514                                                                         &p_buf,\r
515                                                                         &buf_len,\r
516                                                                         &pRdmaIo->packet_sz,\r
517                                                                         NormalPagePriority );\r
518         if( pRdmaIo->packet_sz > p_xmitPool->bufferSz )\r
519         {\r
520                 VNIC_TRACE_EXIT( VNIC_DBG_ERROR,\r
521                         ("Outbound packet too large, size = %d\n", pRdmaIo->packet_sz ) );\r
522                 return FALSE;\r
523         }\r
524 \r
525         if ( p_sgl->NumberOfElements > (ULONG)MAX_NUM_SGE - 1 )\r
526         {\r
527                 VNIC_TRACE( VNIC_DBG_DATA,\r
528                         (" Xmit packet exceeded SGE limit - %d\n",\r
529                                                                                 p_sgl->NumberOfElements ) );\r
530                 return FALSE;\r
531         }\r
532         pRdmaIo->p_packet = p_packet;\r
533 \r
534         for( i=0; i < p_sgl->NumberOfElements; i++ )\r
535         {\r
536                 pRdmaIo->dsList[i].vaddr = p_sgl->Elements[i].Address.QuadPart;\r
537                 pRdmaIo->dsList[i].length = p_sgl->Elements[i].Length;\r
538         }\r
539 \r
540         pRdmaIo->len = (uint32_t)ROUNDUPP2(\r
541                 max(60, pRdmaIo->packet_sz), VIPORT_TRAILER_ALIGNMENT );\r
542         pad = pRdmaIo->len - pRdmaIo->packet_sz;\r
543 \r
544         p_eth_hdr = (eth_hdr_t *)p_buf;\r
545 \r
546         pRdmaIo->p_trailer = (ViportTrailer_t *)&pRdmaIo->data[pad];\r
547         cl_memclr( pRdmaIo->data, pad + sizeof( ViportTrailer_t ) );\r
548         cl_memcpy( pRdmaIo->p_trailer->destMacAddr, p_eth_hdr->dst.addr, MAC_ADDR_LEN );\r
549 \r
550         pRdmaIo->p_trailer->dataLength =\r
551                                         hton16( (uint16_t)max( 60, pRdmaIo->packet_sz ) );\r
552 \r
553         NdisGetNextBuffer( p_buf_desc, &p_buf_desc );\r
554 \r
555         /* should handle VLAN tag */\r
556         if( pRdmaIo->packet_sz > 16 )\r
557         {\r
558                 if(     p_eth_hdr->type == hton16(0x8100) )\r
559                 {\r
560                         if( p_sgl->Elements[0].Length > sizeof(eth_hdr_t) )\r
561                         {\r
562                                 pRdmaIo->p_trailer->vLan = *(uint16_t *)((uint8_t *)p_eth_hdr + 14 );\r
563                                 pRdmaIo->p_trailer->pktFlags |= PF_VLAN_INSERT;\r
564                         }\r
565                         else\r
566                         {\r
567                                 if( p_buf_desc )\r
568                                 {\r
569                                         NdisQueryBufferSafe( p_buf_desc, &p_buf, &buf_len, NormalPagePriority );\r
570 \r
571                                         pad = sizeof(eth_hdr_t) - p_sgl->Elements[0].Length;\r
572                                         pRdmaIo->p_trailer->vLan = *(uint16_t *)(p_buf + pad + 2);\r
573                                         pRdmaIo->p_trailer->pktFlags |= PF_VLAN_INSERT;\r
574                                 }\r
575                         }\r
576                 }\r
577                 else if( p_eth_hdr->type == ETH_PROT_TYPE_IP &&\r
578                                 !( p_eth_hdr->dst.addr[0] & 0x01 ) )\r
579                 {\r
580                         if( p_buf_desc )\r
581                         {\r
582                                 NdisQueryBufferSafe( p_buf_desc, &p_buf, &buf_len, NormalPagePriority );\r
583 \r
584                                 if( ((ip_pkt_t*)p_buf)->hdr.prot == IP_PROT_UDP ||\r
585                                         ((ip_pkt_t*)p_buf)->hdr.prot == IP_PROT_TCP )\r
586                                 {\r
587                                         /* use socket src port + dest port to generate hash value\r
588                                         * for link aggregation distribution function.\r
589                                         */\r
590                                         pRdmaIo->p_trailer->connectionHashAndValid = 0x40 |\r
591                                                 ((uint8_t)((ip_pkt_t*)p_buf)->prot.tcp.src_port +\r
592                                                 (uint8_t)((ip_pkt_t*)p_buf)->prot.tcp.dst_port ) & 0x3f;\r
593                                 }\r
594                         }\r
595                 }\r
596         }\r
597 \r
598         pRdmaIo->p_trailer->txChksumFlags = _tx_chksum_flags( p_packet );\r
599         pRdmaIo->p_trailer->connectionHashAndValid |= CHV_VALID;\r
600 \r
601         if( last )\r
602                 pRdmaIo->p_trailer->pktFlags |= PF_KICK;\r
603 \r
604         /* fill last data segment with trailer and pad */\r
605         phy_addr = MmGetPhysicalAddress( pRdmaIo->data );\r
606 \r
607         pRdmaIo->dsList[p_sgl->NumberOfElements].vaddr = phy_addr.QuadPart;\r
608         pRdmaIo->dsList[p_sgl->NumberOfElements].length = pRdmaIo->len -\r
609                                                                                                           pRdmaIo->packet_sz +\r
610                                                                                                           sizeof( ViportTrailer_t );\r
611 \r
612         pRdmaIo->io.wrq.num_ds =p_sgl->NumberOfElements + 1;\r
613 \r
614         data_rdmaPacket( pData, pBpe, pRdmaIo );\r
615 \r
616         if( p_xmitPool->sendKicks )\r
617         {\r
618                 /* EIOC needs kicks to inform it of sent packets */\r
619 \r
620                 p_xmitPool->kickCount++;\r
621                 p_xmitPool->kickByteCount += pRdmaIo->packet_sz;\r
622                 if( ( p_xmitPool->kickCount >= p_xmitPool->kickBundle )\r
623                          || ( p_xmitPool->kickByteCount >= p_xmitPool->kickByteBundle ) )\r
624                 {\r
625                         data_sendKickMessage( pData );\r
626                 }\r
627                 else if( p_xmitPool->kickCount == 1 )\r
628                 {\r
629                         _data_kickTimer_start( pData, pData->eiocPoolParms.timeoutBeforeKick );\r
630                 }\r
631         }\r
632         return TRUE;\r
633 }\r
634 static uint8_t\r
635 _tx_chksum_flags(\r
636                   IN    NDIS_PACKET* const              p_packet )\r
637 \r
638 {\r
639         NDIS_TCP_IP_CHECKSUM_PACKET_INFO        *p_packet_info;\r
640         ULONG                                                           packet_info;\r
641         uint8_t                 txChksumFlags = 0;\r
642 \r
643         if( NDIS_PROTOCOL_ID_TCP_IP == NDIS_GET_PACKET_PROTOCOL_TYPE(p_packet) )\r
644         {\r
645                 packet_info = PtrToUlong( NDIS_PER_PACKET_INFO_FROM_PACKET( p_packet, TcpIpChecksumPacketInfo));\r
646                 p_packet_info = ( NDIS_TCP_IP_CHECKSUM_PACKET_INFO *)&packet_info;\r
647 \r
648                 if( p_packet_info &&\r
649                         p_packet_info->Transmit.NdisPacketChecksumV4 )\r
650                 {\r
651                         txChksumFlags = TX_CHKSUM_FLAGS_CHECKSUM_V4\r
652                         | ( p_packet_info->Transmit.NdisPacketIpChecksum ? TX_CHKSUM_FLAGS_IP_CHECKSUM: 0 )\r
653                         | ( p_packet_info->Transmit.NdisPacketTcpChecksum ? TX_CHKSUM_FLAGS_TCP_CHECKSUM: 0 )\r
654                         | ( p_packet_info->Transmit.NdisPacketUdpChecksum ? TX_CHKSUM_FLAGS_UDP_CHECKSUM: 0 );\r
655                 }\r
656         }\r
657         \r
658         VNIC_TRACE( VNIC_DBG_DATA ,\r
659                                 ("txChksumFlags = %d: V4 %c, V6 %c, IP %c, TCP %c, UDP %c\n",\r
660                                 txChksumFlags,\r
661                                 ((txChksumFlags & TX_CHKSUM_FLAGS_CHECKSUM_V4 )? '+': '-'),\r
662                                 ((txChksumFlags & TX_CHKSUM_FLAGS_CHECKSUM_V6 )? '+': '-'),\r
663                                 ((txChksumFlags & TX_CHKSUM_FLAGS_IP_CHECKSUM )? '+': '-'),\r
664                                 ((txChksumFlags & TX_CHKSUM_FLAGS_TCP_CHECKSUM )? '+': '-'),\r
665                                 ((txChksumFlags & TX_CHKSUM_FLAGS_UDP_CHECKSUM )? '+': '-') ));\r
666 \r
667         return txChksumFlags;\r
668 }\r
669 \r
670 static void\r
671 _get_first_buffer(\r
672                 IN              NDIS_PACKET             *p_packet,\r
673                 IN OUT  NDIS_BUFFER             **pp_buf_desc,\r
674                 OUT             void                    **pp_buf,\r
675                 OUT             ULONG                   *p_packet_sz )\r
676 {\r
677         UINT            buf_len;\r
678         VNIC_ENTER( VNIC_DBG_DATA );\r
679 \r
680         NdisGetFirstBufferFromPacketSafe( p_packet,\r
681                                                                         pp_buf_desc,\r
682                                                                         pp_buf,\r
683                                                                         &buf_len,\r
684                                                                         p_packet_sz,\r
685                                                                         NormalPagePriority );\r
686         VNIC_EXIT( VNIC_DBG_DATA );\r
687 }\r
688 \r
689 static void\r
690 data_postRecvs(\r
691                 IN              Data_t          *pData )\r
692 {\r
693         RecvIo_t                *pRecvIo;\r
694         LIST_ENTRY              *p_list_entry;\r
695         ib_api_status_t ib_status;\r
696 \r
697         VNIC_ENTER ( VNIC_DBG_DATA );\r
698 \r
699         while( ( p_list_entry = ExInterlockedRemoveHeadList( &pData->recvIos,\r
700                                                                                                                  &pData->recvIosLock ))\r
701                                                                                                                 != NULL )\r
702         {\r
703                 pRecvIo = (RecvIo_t *)p_list_entry;\r
704 \r
705                 ib_status = ibqp_postRecv( &pData->qp, &pRecvIo->io );\r
706                 if( ib_status != IB_SUCCESS )\r
707                 {\r
708                         VNIC_TRACE_EXIT( VNIC_DBG_ERROR, ("ibqp_postRecv returned %s\n",\r
709                                 pData->p_viport->p_adapter->ifc.get_err_str( ib_status )) );\r
710                         viport_failure( pData->p_viport );\r
711                         return;\r
712                 }\r
713         }\r
714 \r
715         VNIC_EXIT( VNIC_DBG_DATA );\r
716         return;\r
717 }\r
718 \r
719 static void\r
720 _data_receivedKick(\r
721                         IN              Io_t            *pIo )\r
722 {\r
723         Data_t        *pData = &pIo->pViport->data;\r
724         uint32_t                num_pkts = 0;\r
725 \r
726         VNIC_ENTER( VNIC_DBG_DATA );\r
727 \r
728 #ifdef VNIC_STATISTIC\r
729         recvRef = cl_get_tick_count();\r
730 #endif /* VNIC_STATISTIC */\r
731 \r
732         ExInterlockedInsertTailList( &pData->recvIos, &pIo->listPtrs, &pData->recvIosLock );\r
733 \r
734         data_postRecvs( pData );\r
735 \r
736 #ifdef VNIC_STATISTIC\r
737         pData->statistics.kickRecvs++;\r
738 #endif /* VNIC_STATISTIC */\r
739 \r
740         data_checkXmitBuffers( pData );\r
741 \r
742         num_pkts = _data_incomingRecv( pData );\r
743 \r
744         if( num_pkts )\r
745         {\r
746                 NdisMIndicateReceivePacket( pData->p_viport->p_adapter->h_handle,\r
747                                                                         pData->recvPool.recv_pkt_array,\r
748                                                                         num_pkts );\r
749                 pData->p_viport->stats.ifInOk += num_pkts;\r
750         }\r
751 \r
752         VNIC_EXIT( VNIC_DBG_DATA );\r
753         return;\r
754 }\r
755 \r
756 static void\r
757 _data_xmitComplete(\r
758                         IN              Io_t            *pIo )\r
759 {\r
760         RdmaIo_t       *pRdmaIo = (RdmaIo_t *)pIo;\r
761         Data_t         *pData = &pIo->pViport->data;\r
762         XmitPool_t     *p_xmitPool = &pData->xmitPool;\r
763         NDIS_PACKET             *p_packet;\r
764         NDIS_STATUS             ndis_status;\r
765         LIST_ENTRY              *p_list_item;\r
766 \r
767         VNIC_ENTER( VNIC_DBG_DATA );\r
768 \r
769         while ( p_xmitPool->lastCompBuf != pRdmaIo->index )\r
770         {\r
771                 INC(p_xmitPool->lastCompBuf, 1, p_xmitPool->numXmitBufs);\r
772                 p_packet = p_xmitPool->pXmitBufs[p_xmitPool->lastCompBuf].p_packet;\r
773         \r
774                 p_xmitPool->pXmitBufs[p_xmitPool->lastCompBuf].p_packet = NULL;\r
775 \r
776                 if( p_packet != NULL )\r
777                 {\r
778                         if( pIo->wc_status != IB_WCS_SUCCESS )\r
779                         {\r
780                                 ndis_status = NDIS_STATUS_FAILURE;\r
781                                 pIo->pViport->stats.ifOutErrors++;\r
782                                 pIo->wc_status = IB_WCS_SUCCESS;\r
783                         }\r
784                         else\r
785                         {\r
786                                 ndis_status = NDIS_STATUS_SUCCESS;\r
787                                 pIo->pViport->stats.ifOutOk++;\r
788                         }\r
789                         NDIS_SET_PACKET_STATUS( p_packet, ndis_status );\r
790                         NdisMSendComplete( pIo->pViport->p_adapter->h_handle,\r
791                                                                 p_packet, ndis_status );\r
792                 }\r
793         }\r
794 \r
795         if( !pIo->pViport->p_netpath->carrier )\r
796         {\r
797                 while( ( p_list_item = NdisInterlockedRemoveHeadList(\r
798                         &pIo->pViport->send_pending_list,\r
799                         &pIo->pViport->pending_list_lock ) ) != NULL )\r
800                 {\r
801                         p_packet = VNIC_PACKET_FROM_LIST_ITEM( p_list_item );\r
802                         NDIS_SET_PACKET_STATUS( p_packet, NDIS_STATUS_FAILURE );\r
803                         NdisMSendComplete( &pIo->pViport->p_adapter->h_handle,\r
804                                                                 p_packet, NDIS_STATUS_FAILURE );\r
805                         pIo->pViport->stats.ifOutErrors++;\r
806                 }\r
807         }\r
808 \r
809         data_checkXmitBuffers( pData );\r
810 \r
811         VNIC_EXIT( VNIC_DBG_DATA );\r
812         return;\r
813 }\r
814 \r
815 static void\r
816 data_sendKickMessage(\r
817                 IN              Data_t          *pData )\r
818 {\r
819         XmitPool_t *pPool = &pData->xmitPool;\r
820 \r
821         VNIC_ENTER( VNIC_DBG_DATA );\r
822 \r
823         /* stop timer for BundleTimeout */\r
824         _data_kickTimer_stop( pData );\r
825 \r
826         pPool->kickCount = 0;\r
827         pPool->kickByteCount = 0;\r
828 \r
829         /* TBD: Keep track of when kick is outstanding, and\r
830          * don't reuse until complete\r
831          */\r
832         if ( ibqp_postSend( &pData->qp, &pData->kickIo.io ) != IB_SUCCESS )\r
833         {\r
834                 VNIC_TRACE( VNIC_DBG_ERROR,\r
835                                         ("Unable to send kick to EIOC\n") );\r
836                 viport_failure( pData->p_viport );\r
837         }\r
838 \r
839         VNIC_EXIT( VNIC_DBG_DATA );\r
840 }\r
841 \r
842 static void\r
843 _data_kickTimeoutHandler( void * context )\r
844 {\r
845         Data_t* pData = (Data_t *)context;\r
846 \r
847         VNIC_ENTER( VNIC_DBG_DATA );\r
848         \r
849         InterlockedExchange( &pData->kickTimerOn, FALSE );\r
850         data_sendKickMessage( pData );\r
851 \r
852         VNIC_EXIT( VNIC_DBG_DATA );\r
853 \r
854         return;\r
855 }\r
856 \r
857 static BOOLEAN\r
858 data_allocXmitBuffer(\r
859                 IN              Data_t                          *pData,\r
860                 OUT             BufferPoolEntry_t       **ppBpe,\r
861                 OUT             RdmaIo_t                        **ppRdmaIo,\r
862                 OUT             BOOLEAN                         *pLast )\r
863 {\r
864         XmitPool_t    *p_xmitPool = &pData->xmitPool;\r
865         KIRQL flags;\r
866 \r
867         VNIC_ENTER( VNIC_DBG_DATA );\r
868 \r
869         KeAcquireSpinLock( &pData->xmitBufLock, &flags );\r
870 \r
871         *pLast = FALSE;\r
872         *ppRdmaIo = &p_xmitPool->pXmitBufs[p_xmitPool->nextXmitBuf];\r
873         *ppBpe = &p_xmitPool->bufPool[p_xmitPool->nextXmitPool];\r
874 \r
875         if ( (*ppBpe)->valid && p_xmitPool->nextXmitBuf != p_xmitPool->lastCompBuf )\r
876         {\r
877                 INC(p_xmitPool->nextXmitBuf, 1, p_xmitPool->numXmitBufs);\r
878                 INC(p_xmitPool->nextXmitPool, 1, p_xmitPool->poolSz);\r
879 \r
880                 if ( !p_xmitPool->bufPool[p_xmitPool->nextXmitPool].valid )\r
881                 {\r
882                         VNIC_TRACE( VNIC_DBG_DATA,\r
883                                 ("Just used the last EIOU receive buffer\n") );\r
884 \r
885                         *pLast = TRUE;\r
886                         p_xmitPool->needBuffers = TRUE;\r
887                         viport_stopXmit( pData->p_viport );\r
888 #ifdef VNIC_STATISTIC\r
889                         pData->statistics.kickReqs++;\r
890 #endif /* VNIC_STATISTIC */\r
891                 }\r
892                 else if ( p_xmitPool->nextXmitBuf == p_xmitPool->lastCompBuf )\r
893                 {\r
894                         VNIC_TRACE( VNIC_DBG_DATA,\r
895                                                 ("Just used our last xmit buffer\n") );\r
896                 \r
897                         p_xmitPool->needBuffers = TRUE;\r
898                         viport_stopXmit( pData->p_viport );\r
899                 }\r
900 \r
901                 (*ppBpe)->valid  = 0;\r
902 \r
903                 KeReleaseSpinLock( &pData->xmitBufLock, flags );\r
904                 return TRUE;\r
905         }\r
906         else\r
907         {\r
908 #ifdef VNIC_STATISTIC\r
909                 pData->statistics.noXmitBufs++;\r
910 #endif /* VNIC_STATISTIC */\r
911         \r
912                 VNIC_TRACE( VNIC_DBG_ERROR,\r
913                                         ("Out of xmit buffers\n") );\r
914 \r
915                 viport_stopXmit( pData->p_viport );\r
916 \r
917                 KeReleaseSpinLock( &pData->xmitBufLock, flags );\r
918                 return FALSE;\r
919         }\r
920 }\r
921 \r
922 static void\r
923 data_checkXmitBuffers(\r
924                 IN                      Data_t          *pData )\r
925 {\r
926         XmitPool_t      *p_xmitPool = &pData->xmitPool;\r
927         KIRQL           flags;\r
928 \r
929         VNIC_ENTER( VNIC_DBG_DATA );\r
930 \r
931         KeAcquireSpinLock( &pData->xmitBufLock, &flags );\r
932 \r
933         if ( pData->xmitPool.needBuffers\r
934                 && p_xmitPool->bufPool[p_xmitPool->nextXmitPool].valid\r
935                 && p_xmitPool->nextXmitBuf != p_xmitPool->lastCompBuf )\r
936         {\r
937                 pData->xmitPool.needBuffers = FALSE;\r
938                 viport_restartXmit( pData->p_viport );\r
939 \r
940                 VNIC_TRACE( VNIC_DBG_DATA,\r
941                                                 ("There are free xmit buffers\n") );\r
942         }\r
943 \r
944         KeReleaseSpinLock( &pData->xmitBufLock, flags );\r
945 \r
946         VNIC_EXIT( VNIC_DBG_DATA );\r
947         return;\r
948 }\r
949 \r
950 static void\r
951 data_rdmaPacket(\r
952                         IN                      Data_t                          *pData,\r
953                         IN                      BufferPoolEntry_t       *pBpe,\r
954                         IN                      RdmaIo_t                        *pRdmaIo )\r
955 {\r
956         ib_send_wr_t    *pWrq;\r
957         uint64_t                remote_addr;\r
958 \r
959         VNIC_ENTER( VNIC_DBG_DATA );\r
960 \r
961         pWrq = &pRdmaIo->io.wrq;\r
962 \r
963         remote_addr  = ntoh64( pBpe->remoteAddr );\r
964         remote_addr += pData->xmitPool.bufferSz;\r
965         remote_addr -= ( pRdmaIo->len + sizeof(ViportTrailer_t) );\r
966 \r
967         pWrq->remote_ops.vaddr          = remote_addr;\r
968         pWrq->remote_ops.rkey           = pBpe->rKey;\r
969 \r
970         pData->xmitPool.notifyCount++;\r
971 \r
972         if( pData->xmitPool.notifyCount >= pData->xmitPool.notifyBundle )\r
973         {\r
974                 pData->xmitPool.notifyCount = 0;\r
975                 pWrq->send_opt = IB_SEND_OPT_SIGNALED;\r
976         }\r
977         else\r
978         {\r
979                 pWrq->send_opt &= ~IB_SEND_OPT_SIGNALED;\r
980         }\r
981         pWrq->send_opt = IB_SEND_OPT_SIGNALED;\r
982 \r
983         if( ibqp_postSend( &pData->qp, &pRdmaIo->io ) != IB_SUCCESS )\r
984         {\r
985                 VNIC_TRACE(VNIC_DBG_ERROR,\r
986                                         ("Failed sending data to EIOC\n") );\r
987                 viport_failure( pData->p_viport );\r
988                 return;\r
989         }\r
990 #ifdef VNIC_STATISTIC\r
991         pData->statistics.xmitNum++;\r
992 #endif /* VNIC_STATISTIC */\r
993 \r
994         VNIC_EXIT( VNIC_DBG_DATA );\r
995 }\r
996 \r
997 static NDIS_PACKET *\r
998 _data_recv_to_ndis_pkt(\r
999                 IN              Data_t                  *pData,\r
1000                 IN              RdmaDest_t              *pRdmaDest )\r
1001 {\r
1002         struct ViportTrailer *pTrailer;\r
1003         NDIS_PACKET             *p_packet;\r
1004         NDIS_STATUS             ndis_status;\r
1005         int                  start;\r
1006         unsigned int         len;\r
1007         uint8_t rxChksumFlags;\r
1008         NDIS_TCP_IP_CHECKSUM_PACKET_INFO  packet_info;\r
1009 \r
1010         VNIC_ENTER( VNIC_DBG_DATA );\r
1011 \r
1012         pTrailer = pRdmaDest->pTrailer;\r
1013         start = (int)data_offset(pData, pTrailer);\r
1014         len = data_len(pData, pTrailer);\r
1015 \r
1016         NdisAllocatePacket( &ndis_status,\r
1017                                                 &p_packet,\r
1018                                                 pData->h_recv_pkt_pool );\r
1019 \r
1020         if ( ndis_status != NDIS_STATUS_SUCCESS )\r
1021         {\r
1022                 VNIC_TRACE_EXIT( VNIC_DBG_ERROR,\r
1023                         ( "NdisAllocatePacket failed %#x\n", ndis_status ) );\r
1024                 return NULL;\r
1025         }\r
1026         NdisAllocateBuffer( &ndis_status,\r
1027                                                 &pRdmaDest->p_buf,\r
1028                                                 pData->h_recv_buf_pool,\r
1029                                                 pRdmaDest->data + start,\r
1030                                                 len );\r
1031 \r
1032         if ( ndis_status != NDIS_STATUS_SUCCESS )\r
1033         {\r
1034                 VNIC_TRACE_EXIT( VNIC_DBG_ERROR,\r
1035                         ( "NdisAllocateBuffer failed %#x\n", ndis_status ) );\r
1036                 NdisFreePacket( p_packet );\r
1037                 return NULL;\r
1038         }\r
1039 \r
1040         NdisChainBufferAtFront( p_packet, pRdmaDest->p_buf );\r
1041         pRdmaDest->p_packet = p_packet;\r
1042 \r
1043         if ( pTrailer->pktFlags & PF_VLAN_INSERT )\r
1044         {\r
1045                 /*      TODO:\r
1046                  *      add OID_GEN_VLAN_ID\r
1047                  *      handle VLAN     tag insertion\r
1048                  *      set packet header size = eth_hdr + 4\r
1049                  */\r
1050         }\r
1051 \r
1052         NDIS_SET_PACKET_HEADER_SIZE( p_packet, sizeof(eth_hdr_t) );\r
1053 \r
1054         rxChksumFlags = pTrailer->rxChksumFlags;\r
1055 \r
1056         VNIC_TRACE( VNIC_DBG_DATA,\r
1057                 ("rxChksumFlags = %d, LOOP = %c, IP = %c, TCP = %c, UDP = %c\n",\r
1058                 rxChksumFlags,\r
1059                 (rxChksumFlags & RX_CHKSUM_FLAGS_LOOPBACK)? 'Y': 'N',\r
1060                 (rxChksumFlags & RX_CHKSUM_FLAGS_IP_CHECKSUM_SUCCEEDED)? 'Y':\r
1061                 (rxChksumFlags & RX_CHKSUM_FLAGS_IP_CHECKSUM_FAILED)? 'N': '-',\r
1062                 (rxChksumFlags & RX_CHKSUM_FLAGS_TCP_CHECKSUM_SUCCEEDED)? 'Y':\r
1063                 (rxChksumFlags & RX_CHKSUM_FLAGS_TCP_CHECKSUM_FAILED)? 'N': '-',\r
1064                 (rxChksumFlags & RX_CHKSUM_FLAGS_UDP_CHECKSUM_SUCCEEDED)? 'Y':\r
1065                 (rxChksumFlags & RX_CHKSUM_FLAGS_UDP_CHECKSUM_FAILED)? 'N': '-') );\r
1066 \r
1067         packet_info.Value = 0;\r
1068 \r
1069         if( rxChksumFlags & RX_CHKSUM_FLAGS_IP_CHECKSUM_SUCCEEDED )\r
1070                 packet_info.Receive.NdisPacketIpChecksumSucceeded = TRUE;\r
1071         else if( rxChksumFlags & RX_CHKSUM_FLAGS_IP_CHECKSUM_FAILED )\r
1072                 packet_info.Receive.NdisPacketIpChecksumFailed = TRUE;\r
1073 \r
1074         if( rxChksumFlags & RX_CHKSUM_FLAGS_TCP_CHECKSUM_SUCCEEDED )\r
1075                 packet_info.Receive.NdisPacketTcpChecksumSucceeded = TRUE;\r
1076         else if( rxChksumFlags & RX_CHKSUM_FLAGS_TCP_CHECKSUM_FAILED )\r
1077                 packet_info.Receive.NdisPacketTcpChecksumFailed = TRUE;\r
1078 \r
1079         if( rxChksumFlags & RX_CHKSUM_FLAGS_TCP_CHECKSUM_SUCCEEDED )\r
1080                 packet_info.Receive.NdisPacketUdpChecksumSucceeded = TRUE;\r
1081         else if( rxChksumFlags & RX_CHKSUM_FLAGS_TCP_CHECKSUM_FAILED )\r
1082                 packet_info.Receive.NdisPacketUdpChecksumFailed = TRUE;\r
1083 \r
1084         NDIS_PER_PACKET_INFO_FROM_PACKET( p_packet, TcpIpChecksumPacketInfo )=\r
1085                                                                                 (void *)(uintn_t)packet_info.Value;\r
1086 \r
1087         VNIC_RECV_FROM_PACKET( p_packet ) = pRdmaDest;\r
1088         NDIS_SET_PACKET_STATUS( p_packet, NDIS_STATUS_SUCCESS );\r
1089 \r
1090         VNIC_EXIT( VNIC_DBG_DATA );\r
1091         return p_packet;\r
1092 }\r
1093 \r
1094 /* NOTE: This routine is not reentrant */\r
1095 static void\r
1096 _data_allocBuffers(\r
1097                 IN                      Data_t          *pData,\r
1098                 IN                      BOOLEAN         initialAllocation )\r
1099 {\r
1100         RecvPool_t     *p_recvPool = &pData->recvPool;\r
1101         RdmaDest_t     *pRdmaDest;\r
1102         LIST_ENTRY              *p_list_entry;\r
1103         int            index;\r
1104 \r
1105         VNIC_ENTER( VNIC_DBG_DATA );\r
1106 \r
1107         index = ADD(p_recvPool->nextFreeBuf, p_recvPool->numFreeBufs, p_recvPool->eiocPoolSz);\r
1108 \r
1109         while ( !IsListEmpty( &p_recvPool->availRecvBufs ) )\r
1110         {\r
1111                 p_list_entry = RemoveHeadList( &p_recvPool->availRecvBufs );\r
1112                 pRdmaDest       = (RdmaDest_t*)p_list_entry;\r
1113         \r
1114                 if( initialAllocation )\r
1115                 {\r
1116                         pRdmaDest->buf_sz = p_recvPool->bufferSz;\r
1117                         pRdmaDest->pTrailer =\r
1118                                 (struct ViportTrailer*)(pRdmaDest->data + pRdmaDest->buf_sz\r
1119                                 - sizeof(struct ViportTrailer));\r
1120                         pRdmaDest->pTrailer->connectionHashAndValid = 0;\r
1121                 }\r
1122 \r
1123                 pRdmaDest->p_packet = NULL;\r
1124                 _data_addFreeBuffer(pData, index, pRdmaDest);\r
1125                 index = NEXT(index,p_recvPool->eiocPoolSz);\r
1126         }\r
1127 \r
1128         VNIC_EXIT( VNIC_DBG_DATA );\r
1129         return;\r
1130 }\r
1131 \r
1132 static void\r
1133 _data_addFreeBuffer(\r
1134                 IN              Data_t                  *pData,\r
1135                 IN              int                             index,\r
1136             IN          RdmaDest_t              *pRdmaDest )\r
1137 {\r
1138         RecvPool_t        *p_recvPool = &pData->recvPool;\r
1139         BufferPoolEntry_t *pBpe;\r
1140 \r
1141         pRdmaDest->pTrailer->connectionHashAndValid = 0;\r
1142         pBpe = &p_recvPool->bufPool[index];\r
1143 \r
1144         pBpe->rKey       = pRdmaDest->region.rkey;\r
1145         pBpe->remoteAddr = hton64( PTR64( pRdmaDest->data ) );\r
1146         pBpe->valid      = (uint32_t)(pRdmaDest - &p_recvPool->pRecvBufs[0]) + 1;\r
1147         ++p_recvPool->numFreeBufs;\r
1148 \r
1149         return;\r
1150 }\r
1151 \r
1152 static uint32_t\r
1153 _data_incomingRecv(\r
1154                 IN              Data_t          *pData )\r
1155 {\r
1156         RecvPool_t        *p_recvPool = &pData->recvPool;\r
1157         RdmaDest_t        *pRdmaDest;\r
1158         ViportTrailer_t   *pTrailer;\r
1159         BufferPoolEntry_t *pBpe;\r
1160         NDIS_PACKET             *p_packet = NULL;\r
1161         uint32_t                idx = 0;\r
1162         BOOLEAN status = FALSE;\r
1163 \r
1164         VNIC_ENTER( VNIC_DBG_DATA );\r
1165 \r
1166         while( !status )\r
1167         {\r
1168                 if ( p_recvPool->nextFullBuf == p_recvPool->nextFreeBuf )\r
1169                         return idx;\r
1170 \r
1171                 pBpe = &p_recvPool->bufPool[p_recvPool->nextFullBuf];\r
1172                 pRdmaDest = &p_recvPool->pRecvBufs[pBpe->valid - 1];\r
1173                 pTrailer = pRdmaDest->pTrailer;\r
1174 \r
1175                 if ( ( pTrailer != NULL ) &&\r
1176                         ( pTrailer->connectionHashAndValid & CHV_VALID ) )\r
1177                 {\r
1178                         /* received a packet */\r
1179                         if ( pTrailer->pktFlags & PF_KICK )\r
1180                         {\r
1181                                 p_recvPool->kickOnFree = TRUE;\r
1182                         }\r
1183                         /* we do not want to indicate packet if  no filter is set */\r
1184                         if( pData->p_viport->p_adapter->packet_filter )\r
1185                         {\r
1186                                 p_packet = _data_recv_to_ndis_pkt( pData, pRdmaDest );\r
1187                                 if ( p_packet != NULL )\r
1188                                 {\r
1189                                         p_recvPool->recv_pkt_array[idx++] = p_packet;\r
1190                                 }\r
1191                         }\r
1192                         else\r
1193                         {       /* put back to free buffers pool */\r
1194                                 InsertTailList( &p_recvPool->availRecvBufs,\r
1195                                         &pRdmaDest->listPtrs );\r
1196                         }\r
1197                         pBpe->valid = 0;\r
1198                         INC( p_recvPool->nextFullBuf, 1, p_recvPool->eiocPoolSz );\r
1199                         p_recvPool->numPostedBufs--;\r
1200 #ifdef VNIC_STATISTIC\r
1201                         pData->statistics.recvNum++;\r
1202 #endif /* VNIC_STATISTIC */\r
1203                 }\r
1204                 else\r
1205                         break;\r
1206         }\r
1207         return idx;\r
1208 }\r
1209 \r
1210 \r
1211 void\r
1212 vnic_return_packet(\r
1213         IN              NDIS_HANDLE             adapter_context,\r
1214         IN              NDIS_PACKET     * const p_packet )\r
1215 {\r
1216 \r
1217 \r
1218         vnic_adapter_t  *p_adapter = (vnic_adapter_t *)adapter_context;\r
1219         viport_t        *p_viport = p_adapter->p_currentPath->pViport;\r
1220         RdmaDest_t      *p_rdma_dest = VNIC_RECV_FROM_PACKET( p_packet );\r
1221 \r
1222         ASSERT( p_rdma_dest->p_packet == p_packet );\r
1223         _data_return_recv( p_packet );\r
1224         p_rdma_dest->p_packet = NULL;\r
1225 \r
1226         InsertTailList( &p_viport->data.recvPool.availRecvBufs,\r
1227                                         &p_rdma_dest->listPtrs );\r
1228 \r
1229         if( p_viport->data.connected == TRUE &&\r
1230                 !p_viport->errored )\r
1231         {\r
1232                 _data_allocBuffers( &p_viport->data, FALSE );\r
1233                 _data_sendFreeRecvBuffers( &p_viport->data );\r
1234         }\r
1235 }\r
1236 static void\r
1237 _data_return_recv(\r
1238           IN    NDIS_PACKET             *p_packet )\r
1239 {\r
1240         NDIS_BUFFER             *p_buf;\r
1241         /* Unchain the NDIS buffer. */\r
1242         NdisUnchainBufferAtFront( p_packet, &p_buf );\r
1243         CL_ASSERT( p_buf );\r
1244         /* Return the NDIS packet and NDIS buffer to their pools. */\r
1245         NdisFreeBuffer( p_buf );\r
1246         NdisFreePacket( p_packet );\r
1247 }\r
1248 \r
1249 static void\r
1250 _data_sendFreeRecvBuffers(\r
1251                         IN                      Data_t          *pData )\r
1252 {\r
1253         RecvPool_t  *p_recvPool = &pData->recvPool;\r
1254         ib_send_wr_t *pWrq = &pData->freeBufsIo.io.wrq;\r
1255         BOOLEAN     bufsSent = FALSE;\r
1256         uint64_t    rdmaAddr;\r
1257         uint32_t    offset;\r
1258         uint32_t    sz;\r
1259         unsigned int numToSend,\r
1260                     nextIncrement;\r
1261 \r
1262         VNIC_ENTER( VNIC_DBG_DATA );\r
1263 \r
1264         for (   numToSend = p_recvPool->szFreeBundle;\r
1265                         numToSend <= p_recvPool->numFreeBufs;\r
1266                         numToSend += p_recvPool->szFreeBundle )\r
1267         {\r
1268                 /* Handle multiple bundles as one when possible. */\r
1269                 nextIncrement = numToSend + p_recvPool->szFreeBundle;\r
1270                 if (    ( nextIncrement <= p_recvPool->numFreeBufs )\r
1271                                 && ( p_recvPool->nextFreeBuf + nextIncrement <= p_recvPool->eiocPoolSz ) )\r
1272                 {\r
1273                         continue;\r
1274                 }\r
1275 \r
1276                 offset   = p_recvPool->nextFreeBuf * sizeof(BufferPoolEntry_t);\r
1277                 sz       = numToSend * sizeof(BufferPoolEntry_t);\r
1278                 rdmaAddr = p_recvPool->eiocRdmaAddr + offset;\r
1279         \r
1280                 pWrq->ds_array->length  = sz;\r
1281                 pWrq->ds_array->vaddr = PTR64((uint8_t *)p_recvPool->bufPool + offset);\r
1282                 pWrq->remote_ops.vaddr = rdmaAddr;\r
1283 \r
1284                 if ( ibqp_postSend( &pData->qp, &pData->freeBufsIo.io ) != IB_SUCCESS )\r
1285                 {\r
1286                         VNIC_TRACE(VNIC_DBG_ERROR,\r
1287                                         ("Unable to rdma free buffers to EIOC\n") );\r
1288 \r
1289                         viport_failure( pData->p_viport );\r
1290                         break;\r
1291                 }\r
1292 \r
1293                 INC( p_recvPool->nextFreeBuf, numToSend, p_recvPool->eiocPoolSz );\r
1294                 p_recvPool->numFreeBufs -= numToSend;\r
1295                 p_recvPool->numPostedBufs += numToSend;\r
1296                 bufsSent = TRUE;\r
1297         }\r
1298 \r
1299         if( bufsSent )\r
1300         {\r
1301                 if( p_recvPool->kickOnFree )\r
1302                 {\r
1303                         data_sendKickMessage( pData );\r
1304                 }\r
1305         }\r
1306         if( p_recvPool->numPostedBufs == 0 )\r
1307         {\r
1308                 VNIC_TRACE( VNIC_DBG_ERROR,\r
1309                                 ("IOC %d: Unable to allocate receive buffers\n",\r
1310                                         pData->p_viport->ioc_num ) );\r
1311         \r
1312                 viport_failure( pData->p_viport );\r
1313         }\r
1314         VNIC_EXIT( VNIC_DBG_DATA );\r
1315 }\r
1316 \r
1317 \r
1318 void\r
1319 data_cleanup(\r
1320         IN                      Data_t  *pData )\r
1321 {\r
1322         VNIC_ENTER( VNIC_DBG_DATA );\r
1323 \r
1324         if( pData->recvPool.recv_pkt_array )\r
1325         {\r
1326                 cl_free( pData->recvPool.recv_pkt_array );\r
1327                 pData->recvPool.recv_pkt_array = NULL;\r
1328                 pData->recvPool.poolSz = 0;\r
1329         }\r
1330 \r
1331         if ( pData->pLocalStorage )\r
1332         {\r
1333                 NdisFreeMemory( pData->pLocalStorage, pData->localStorageSz, 0 );\r
1334                 pData->pLocalStorage = NULL;\r
1335         }\r
1336 \r
1337         if(  pData->h_recv_buf_pool )\r
1338         {\r
1339                 NdisFreeBufferPool( pData->h_recv_buf_pool );\r
1340                 pData->h_recv_buf_pool = NULL;\r
1341         }\r
1342 \r
1343         if ( pData->h_recv_pkt_pool )\r
1344         {\r
1345                 while( NdisPacketPoolUsage(pData->h_recv_pkt_pool) != 0)\r
1346                 {\r
1347                         VNIC_TRACE( VNIC_DBG_WARN,\r
1348                                 ("Recv packet pool is not empty!!!\n") );\r
1349                         NdisMSleep(100);\r
1350                 }\r
1351                 NdisFreePacketPool( pData->h_recv_pkt_pool );\r
1352                 pData->h_recv_pkt_pool = NULL;\r
1353         }\r
1354         if( pData->p_recv_bufs )\r
1355         {\r
1356                 NdisFreeMemory( pData->p_recv_bufs, pData->recv_bufs_sz, 0 );\r
1357                 pData->p_recv_bufs = NULL;\r
1358         }\r
1359         // clear Qp struct for reuse\r
1360         cl_memclr( &pData->qp, sizeof( IbQp_t) );\r
1361 \r
1362         cl_timer_destroy( &pData->kickTimer );\r
1363 \r
1364         VNIC_EXIT( VNIC_DBG_DATA );\r
1365 }\r
1366 \r
1367 \r
1368 static void\r
1369 _data_kickTimer_start(\r
1370                   IN    Data_t          *pData,\r
1371                   IN    uint32_t        microseconds )\r
1372 {\r
1373         VNIC_ENTER( VNIC_DBG_DATA );\r
1374 \r
1375         InterlockedExchange( (LONG *)&pData->kickTimerOn, TRUE );\r
1376 \r
1377         usec_timer_start(&pData->kickTimer, microseconds );\r
1378 \r
1379         VNIC_EXIT( VNIC_DBG_DATA );\r
1380         return;\r
1381 }\r
1382 \r
1383 static void\r
1384 _data_kickTimer_stop(\r
1385                   IN    Data_t          *pData )\r
1386 {\r
1387         VNIC_ENTER( VNIC_DBG_DATA );\r
1388 \r
1389         if( InterlockedExchange( &pData->kickTimerOn, FALSE ) == TRUE )\r
1390         {\r
1391                 cl_timer_stop( &pData->kickTimer );\r
1392         }\r
1393 \r
1394         VNIC_EXIT( VNIC_DBG_DATA );\r
1395 }\r