2 * Copyright (c) 2005 SilverStorm Technologies. All rights reserved.
\r
4 * This software is available to you under the OpenIB.org BSD license
\r
7 * Redistribution and use in source and binary forms, with or
\r
8 * without modification, are permitted provided that the following
\r
9 * conditions are met:
\r
11 * - Redistributions of source code must retain the above
\r
12 * copyright notice, this list of conditions and the following
\r
15 * - Redistributions in binary form must reproduce the above
\r
16 * copyright notice, this list of conditions and the following
\r
17 * disclaimer in the documentation and/or other materials
\r
18 * provided with the distribution.
\r
20 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
\r
21 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
\r
22 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
\r
23 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
\r
24 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
\r
25 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
\r
26 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
\r
31 #include <complib/comp_lib.h>
\r
32 #include "vnic_driver.h"
\r
34 static void data_postRecvs(Data_t *pData);
\r
35 static void _data_receivedKick(Io_t *pIo);
\r
36 static void _data_xmitComplete(Io_t *pIo);
\r
37 static void data_sendKickMessage(Data_t *pData);
\r
38 static void _data_kickTimeoutHandler( void *context );
\r
39 static BOOLEAN data_allocXmitBuffer(Data_t *pData,
\r
40 BufferPoolEntry_t **ppBpe, RdmaIo_t **ppRdmaIo, BOOLEAN *pLast);
\r
41 static void data_checkXmitBuffers(Data_t *pData);
\r
42 static void data_rdmaPacket(Data_t *pData, BufferPoolEntry_t *pBpe, RdmaIo_t *pRdmaIo);
\r
44 static NDIS_PACKET *
\r
45 _data_recv_to_ndis_pkt( Data_t *pData, RdmaDest_t *pRdmaDest );
\r
50 BOOLEAN initialAllocation );
\r
53 _data_addFreeBuffer(
\r
56 RdmaDest_t *pRdmaDest );
\r
63 _data_sendFreeRecvBuffers(
\r
68 IN NDIS_PACKET* const p_packet );
\r
73 IN NDIS_PACKET *p_packet,
\r
74 IN OUT NDIS_BUFFER **pp_buf_desc,
\r
76 OUT ULONG *p_packet_sz );
\r
80 IN NDIS_PACKET *p_packet );
\r
83 _data_kickTimer_start(
\r
85 IN uint32_t microseconds );
\r
88 _data_kickTimer_stop(
\r
91 #define LOCAL_IO(x) PTR64((x))
\r
93 #define INBOUND_COPY
\r
95 #ifdef VNIC_STATISTIC
\r
97 #endif /* VNIC_STATISTIC */
\r
102 IN viport_t *pViport )
\r
104 VNIC_ENTER( VNIC_DBG_DATA );
\r
106 RtlZeroMemory( pData, sizeof(*pData) );
\r
108 pData->p_viport = pViport;
\r
109 pData->p_phy_region = &pViport->p_adapter->ca.region;
\r
110 InitializeListHead( &pData->recvIos );
\r
111 KeInitializeSpinLock ( &pData->recvIosLock );
\r
112 KeInitializeSpinLock ( &pData->xmitBufLock );
\r
113 cl_timer_construct( &pData->kickTimer );
\r
115 ibqp_construct( &pData->qp, pViport );
\r
117 VNIC_EXIT( VNIC_DBG_DATA );
\r
124 IN DataConfig_t *p_conf,
\r
127 ib_api_status_t ib_status;
\r
129 VNIC_ENTER( VNIC_DBG_DATA );
\r
131 ASSERT( pData->p_viport != NULL );
\r
132 pData->p_conf = p_conf;
\r
134 cl_timer_init( &pData->kickTimer, _data_kickTimeoutHandler, pData );
\r
136 ib_status = ibqp_init(&pData->qp, guid, &p_conf->ibConfig );
\r
137 if( ib_status != IB_SUCCESS )
\r
138 VNIC_TRACE( VNIC_DBG_ERROR, ("data ibqp_init failed\n") );
\r
140 VNIC_EXIT( VNIC_DBG_DATA );
\r
149 NDIS_STATUS status;
\r
150 ib_api_status_t ib_status;
\r
151 XmitPool_t *pXmitPool = &pData->xmitPool;
\r
152 RecvPool_t *pRecvPool = &pData->recvPool;
\r
156 RdmaDest_t *pRdmaDest;
\r
157 uint8_t *pRegionData;
\r
161 VNIC_ENTER( VNIC_DBG_DATA );
\r
163 pRecvPool->poolSz = pData->p_conf->hostRecvPoolEntries;
\r
164 pRecvPool->eiocPoolSz = pData->hostPoolParms.numRecvPoolEntries;
\r
166 if ( pRecvPool->poolSz > pRecvPool->eiocPoolSz )
\r
168 pRecvPool->poolSz = pData->hostPoolParms.numRecvPoolEntries;
\r
170 pRecvPool->szFreeBundle =
\r
171 pData->hostPoolParms.freeRecvPoolEntriesPerUpdate;
\r
172 pRecvPool->numFreeBufs = 0;
\r
173 pRecvPool->numPostedBufs = 0;
\r
174 pRecvPool->nextFullBuf = 0;
\r
175 pRecvPool->nextFreeBuf = 0;
\r
176 pRecvPool->kickOnFree = FALSE;
\r
177 pRecvPool->bufferSz = pData->hostPoolParms.sizeRecvPoolEntry;
\r
179 pXmitPool->bufferSz = pData->eiocPoolParms.sizeRecvPoolEntry;
\r
180 pXmitPool->poolSz = pData->eiocPoolParms.numRecvPoolEntries;
\r
181 pXmitPool->notifyCount = 0;
\r
182 pXmitPool->notifyBundle = pData->p_conf->notifyBundle;
\r
183 pXmitPool->nextXmitPool = 0;
\r
185 #if TRUE // LIMIT_OUTSTANDING_SENDS
\r
186 pXmitPool->numXmitBufs = pXmitPool->notifyBundle * 2;
\r
187 #else /* !LIMIT_OUTSTANDING_SENDS */
\r
188 pXmitPool->numXmitBufs = pXmitPool->poolSz;
\r
189 #endif /* LIMIT_OUTSTANDING_SENDS */
\r
191 pXmitPool->nextXmitBuf = 0;
\r
192 pXmitPool->lastCompBuf = pXmitPool->numXmitBufs - 1;
\r
193 pXmitPool->kickCount = 0;
\r
194 pXmitPool->kickByteCount = 0;
\r
195 pXmitPool->sendKicks =
\r
196 (BOOLEAN)(( pData->eiocPoolParms.numRecvPoolEntriesBeforeKick != 0 )
\r
197 || ( pData->eiocPoolParms.numRecvPoolBytesBeforeKick != 0 ));
\r
198 pXmitPool->kickBundle =
\r
199 pData->eiocPoolParms.numRecvPoolEntriesBeforeKick;
\r
200 pXmitPool->kickByteBundle =
\r
201 pData->eiocPoolParms.numRecvPoolBytesBeforeKick;
\r
202 pXmitPool->needBuffers = TRUE;
\r
204 sz = sizeof(RdmaDest_t) * pRecvPool->poolSz;
\r
205 sz += sizeof(RecvIo_t) * pData->p_conf->numRecvs;
\r
206 sz += sizeof(RdmaIo_t) * pXmitPool->numXmitBufs;
\r
208 regionSz = 4 * pData->p_conf->numRecvs;
\r
209 regionSz += sizeof(BufferPoolEntry_t) * pRecvPool->eiocPoolSz;
\r
210 regionSz += sizeof(BufferPoolEntry_t) * pXmitPool->poolSz;
\r
213 status = NdisAllocateMemoryWithTag( &pData->pLocalStorage,
\r
216 if ( status != NDIS_STATUS_SUCCESS )
\r
218 VNIC_TRACE_EXIT( VNIC_DBG_ERROR,
\r
219 ("Failed allocating %d bytes local storage\n", sz ) );
\r
220 ib_status = IB_INSUFFICIENT_MEMORY;
\r
224 NdisZeroMemory( pData->pLocalStorage, sz );
\r
225 pData->localStorageSz = sz;
\r
227 pRecvPool->pRecvBufs = (RdmaDest_t *)pData->pLocalStorage;
\r
228 sz = sizeof(RdmaDest_t) * pRecvPool->poolSz;
\r
229 pRecvIo = (RecvIo_t *)(pData->pLocalStorage + sz);
\r
230 sz += sizeof(RecvIo_t) * pData->p_conf->numRecvs;
\r
232 pXmitPool->pXmitBufs = (RdmaIo_t *)(pData->pLocalStorage + sz);
\r
233 sz += sizeof(RdmaIo_t) * pXmitPool->numXmitBufs;
\r
235 pRegionData = pData->pLocalStorage + sz;
\r
236 sz += 4 * pData->p_conf->numRecvs;
\r
238 pRecvPool->bufPool = (BufferPoolEntry_t *)(pData->pLocalStorage + sz);
\r
239 sz += sizeof(BufferPoolEntry_t) * pRecvPool->eiocPoolSz;
\r
240 pXmitPool->bufPool = (BufferPoolEntry_t *)(pData->pLocalStorage + sz);
\r
242 ib_status = ibregion_init( pData->p_viport, &pData->region,
\r
243 pData->p_viport->p_adapter->ca.hPd, pRegionData, regionSz,
\r
244 ( IB_AC_LOCAL_WRITE | IB_AC_RDMA_WRITE ) );
\r
245 if( ib_status != IB_SUCCESS )
\r
247 VNIC_TRACE_EXIT( VNIC_DBG_ERROR, ("ib_region_init failed\n") );
\r
251 pRdmaIo = &pData->freeBufsIo;
\r
252 pRdmaIo->io.pViport = pData->p_viport;
\r
253 pRdmaIo->io.pRoutine = NULL;
\r
254 pRdmaIo->io.wrq.p_next = NULL;
\r
255 pRdmaIo->io.wrq.wr_type = WR_RDMA_WRITE;
\r
256 pRdmaIo->io.wrq.wr_id = PTR64( pRdmaIo );
\r
257 pRdmaIo->io.wrq.num_ds = 1;
\r
258 pRdmaIo->io.wrq.ds_array = pRdmaIo->dsList;
\r
259 pRdmaIo->dsList[0].lkey = pData->region.lkey;
\r
260 pRdmaIo->io.wrq.send_opt = IB_SEND_OPT_SIGNALED;
\r
262 pSendIo = &pData->kickIo;
\r
263 pSendIo->io.pViport = pData->p_viport;
\r
264 pSendIo->io.pRoutine = NULL;
\r
265 pSendIo->io.wrq.p_next = NULL;
\r
266 pSendIo->io.wrq.wr_type = WR_SEND;
\r
267 pSendIo->io.wrq.wr_id = PTR64( pSendIo );
\r
268 pSendIo->io.wrq.num_ds = 1;
\r
269 pSendIo->io.wrq.ds_array = &pSendIo->dsList;
\r
271 pSendIo->io.wrq.send_opt = IB_SEND_OPT_SIGNALED;
\r
273 pSendIo->dsList.length = 0;
\r
274 pSendIo->dsList.vaddr = PTR64( pRegionData );
\r
275 pSendIo->dsList.lkey = pData->region.lkey;
\r
277 for ( i = 0; i < pData->p_conf->numRecvs; i++ )
\r
279 pRecvIo[i].io.pViport = pData->p_viport;
\r
280 pRecvIo[i].io.pRoutine = _data_receivedKick;
\r
281 pRecvIo[i].io.r_wrq.wr_id = PTR64( &pRecvIo[i].io );
\r
282 pRecvIo[i].io.r_wrq.p_next = NULL;
\r
283 pRecvIo[i].io.r_wrq.num_ds = 1;
\r
284 pRecvIo[i].io.r_wrq.ds_array = &pRecvIo[i].dsList;
\r
285 pRecvIo[i].dsList.length = 4;
\r
286 pRecvIo[i].dsList.vaddr = PTR64( pRegionData );
\r
287 pRecvIo[i].dsList.lkey = pData->region.lkey;
\r
289 ExInterlockedInsertTailList( &pData->recvIos, &pRecvIo[i].io.listPtrs, &pData->recvIosLock );
\r
290 /* Do not need to move pointer since the receive info
\r
291 * is not read. Note, we could reduce the amount
\r
292 * of memory allocated and the size of the region.
\r
293 * pRegionData += 4;
\r
297 sz = pRecvPool->poolSz * pRecvPool->bufferSz;
\r
298 status = NdisAllocateMemoryWithTag(&pData->p_recv_bufs,
\r
300 if( status != NDIS_STATUS_SUCCESS )
\r
302 VNIC_TRACE_EXIT( VNIC_DBG_ERROR,
\r
303 ("Allocate recv buffers failed\n"));
\r
304 ib_status = IB_INSUFFICIENT_MEMORY;
\r
307 NdisZeroMemory( pData->p_recv_bufs, sz );
\r
309 pData->recv_bufs_sz = sz;
\r
311 ib_status = ibregion_init( pData->p_viport, &pData->rbuf_region,
\r
312 pData->p_viport->p_adapter->ca.hPd, pData->p_recv_bufs, sz,
\r
313 (IB_AC_LOCAL_WRITE | IB_AC_RDMA_WRITE) );
\r
314 if( ib_status != IB_SUCCESS )
\r
319 NdisAllocatePacketPool( &status,
\r
320 &pData->h_recv_pkt_pool,
\r
322 PROTOCOL_RESERVED_SIZE_IN_PACKET );
\r
324 if( status != NDIS_STATUS_SUCCESS )
\r
326 VNIC_TRACE_EXIT( VNIC_DBG_ERROR,
\r
327 ("Allocate packet pool failed status %#x\n", status ));
\r
328 ib_status = IB_INSUFFICIENT_MEMORY;
\r
332 NdisAllocateBufferPool(
\r
333 &status, &pData->h_recv_buf_pool, pRecvPool->poolSz );
\r
335 if( status != NDIS_STATUS_SUCCESS )
\r
337 VNIC_TRACE_EXIT( VNIC_DBG_ERROR,
\r
338 ("Allocate packet pool failed status %#x\n", status ));
\r
339 ib_status = IB_INSUFFICIENT_MEMORY;
\r
342 pData->recvPool.recv_pkt_array =
\r
343 cl_zalloc( sizeof(NDIS_PACKET*)* pRecvPool->poolSz );
\r
344 if( !pData->recvPool.recv_pkt_array )
\r
346 VNIC_TRACE_EXIT( VNIC_DBG_ERROR,
\r
347 ("Allocate packet array failed\n" ) );
\r
348 ib_status = IB_INSUFFICIENT_MEMORY;
\r
352 InitializeListHead( &pRecvPool->availRecvBufs );
\r
354 for ( i = 0; i < pRecvPool->poolSz; i++ )
\r
356 pRdmaDest = &pRecvPool->pRecvBufs[i];
\r
357 pRdmaDest->data = pData->p_recv_bufs + (i * pRecvPool->bufferSz );
\r
358 pRdmaDest->region = pData->rbuf_region;
\r
359 InsertTailList( &pRecvPool->availRecvBufs, &pRdmaDest->listPtrs );
\r
362 for ( i = 0; i < pXmitPool->numXmitBufs; i++ )
\r
364 pRdmaIo = &pXmitPool->pXmitBufs[i];
\r
365 pRdmaIo->index = (uint16_t)i;
\r
366 pRdmaIo->io.pViport = pData->p_viport;
\r
367 pRdmaIo->io.pRoutine = _data_xmitComplete;
\r
368 pRdmaIo->io.wrq.p_next = NULL;
\r
369 pRdmaIo->io.wrq.wr_type = WR_RDMA_WRITE;
\r
370 pRdmaIo->io.wrq.wr_id = PTR64(pRdmaIo);
\r
371 pRdmaIo->io.wrq.num_ds = MAX_NUM_SGE; // will set actual number when transmit
\r
372 pRdmaIo->io.wrq.ds_array = pRdmaIo->dsList;
\r
373 pRdmaIo->p_trailer = (ViportTrailer_t *)&pRdmaIo->data[0];
\r
374 for( j = 0; j < MAX_NUM_SGE; j++ )
\r
376 pRdmaIo->dsList[j].lkey = pData->p_phy_region->lkey;
\r
380 pXmitPool->rdmaRKey = pData->region.rkey;
\r
381 pXmitPool->rdmaAddr = PTR64( pXmitPool->bufPool );
\r
383 data_postRecvs( pData );
\r
385 ib_status = ibqp_connect( &pData->qp );
\r
386 if( ib_status != IB_SUCCESS )
\r
388 VNIC_TRACE( VNIC_DBG_ERROR, ("ibqp_connect returned %s\n",
\r
389 pData->p_viport->p_adapter->ifc.get_err_str( ib_status )) );
\r
391 NdisFreeBufferPool( pData->h_recv_buf_pool );
\r
392 pData->h_recv_buf_pool = NULL;
\r
394 NdisFreePacketPool( pData->h_recv_pkt_pool );
\r
395 pData->h_recv_pkt_pool = NULL;
\r
397 ibregion_cleanup( pData->p_viport, &pData->rbuf_region );
\r
399 NdisFreeMemory( pData->p_recv_bufs, pData->recv_bufs_sz, 0 );
\r
400 pData->p_recv_bufs = NULL;
\r
402 ibregion_cleanup(pData->p_viport, &pData->region );
\r
404 NdisFreeMemory( pData->pLocalStorage, pData->localStorageSz, 0 );
\r
405 pData->pLocalStorage = NULL;
\r
407 pRecvPool->poolSz = 0;
\r
410 VNIC_EXIT( VNIC_DBG_DATA );
\r
419 VNIC_ENTER( VNIC_DBG_DATA );
\r
421 pData->freeBufsIo.io.wrq.remote_ops.rkey =
\r
422 pData->recvPool.eiocRdmaRkey;
\r
424 _data_allocBuffers(pData, TRUE);
\r
425 _data_sendFreeRecvBuffers(pData);
\r
426 pData->connected = TRUE;
\r
428 VNIC_EXIT( VNIC_DBG_DATA );
\r
436 RecvPool_t *pRecvPool = &pData->recvPool;
\r
437 viport_t *p_viport = pData->p_viport;
\r
438 NDIS_PACKET *p_packet;
\r
439 LIST_ENTRY *p_list_item;
\r
442 VNIC_ENTER( VNIC_DBG_DATA );
\r
444 _data_kickTimer_stop ( pData );
\r
446 pData->connected = FALSE;
\r
448 ibqp_detach( &pData->qp );
\r
450 ibregion_cleanup( p_viport, &pData->rbuf_region );
\r
451 ibregion_cleanup( p_viport, &pData->region );
\r
453 for ( i = 0; i < pRecvPool->poolSz; i++ )
\r
455 p_packet = pRecvPool->pRecvBufs[i].p_packet;
\r
456 if ( p_packet != NULL )
\r
458 pRecvPool->pRecvBufs[i].p_packet = NULL;
\r
459 _data_return_recv( p_packet );
\r
462 /* clear pending queue if any */
\r
463 while( ( p_list_item = NdisInterlockedRemoveHeadList(
\r
464 &p_viport->send_pending_list,
\r
465 &p_viport->pending_list_lock )) != NULL )
\r
467 p_packet = VNIC_PACKET_FROM_LIST_ITEM( p_list_item );
\r
470 NDIS_SET_PACKET_STATUS( p_packet, NDIS_STATUS_FAILURE );
\r
471 NdisMSendComplete( p_viport->p_adapter->h_handle,
\r
472 p_packet, NDIS_STATUS_FAILURE );
\r
473 p_viport->stats.ifOutErrors++;
\r
477 VNIC_EXIT( VNIC_DBG_DATA );
\r
484 IN NDIS_PACKET* const p_packet )
\r
486 XmitPool_t *p_xmitPool = &pData->xmitPool;
\r
488 BufferPoolEntry_t *pBpe;
\r
492 NDIS_BUFFER *p_buf_desc;
\r
493 eth_hdr_t* p_eth_hdr;
\r
495 SCATTER_GATHER_LIST *p_sgl;
\r
497 PHYSICAL_ADDRESS phy_addr;
\r
499 VNIC_ENTER( VNIC_DBG_DATA );
\r
501 if( !data_allocXmitBuffer( pData, &pBpe, &pRdmaIo, &last ) )
\r
505 p_sgl = NDIS_PER_PACKET_INFO_FROM_PACKET( p_packet,
\r
506 ScatterGatherListPacketInfo );
\r
507 if ( p_sgl == NULL )
\r
512 NdisGetFirstBufferFromPacketSafe( p_packet,
\r
516 &pRdmaIo->packet_sz,
\r
517 NormalPagePriority );
\r
518 if( pRdmaIo->packet_sz > p_xmitPool->bufferSz )
\r
520 VNIC_TRACE_EXIT( VNIC_DBG_ERROR,
\r
521 ("Outbound packet too large, size = %d\n", pRdmaIo->packet_sz ) );
\r
525 if ( p_sgl->NumberOfElements > (ULONG)MAX_NUM_SGE - 1 )
\r
527 VNIC_TRACE( VNIC_DBG_DATA,
\r
528 (" Xmit packet exceeded SGE limit - %d\n",
\r
529 p_sgl->NumberOfElements ) );
\r
533 for( i=0; i < p_sgl->NumberOfElements; i++ )
\r
535 pRdmaIo->dsList[i].vaddr = p_sgl->Elements[i].Address.QuadPart;
\r
536 pRdmaIo->dsList[i].length = p_sgl->Elements[i].Length;
\r
539 pRdmaIo->len = (uint32_t)ROUNDUPP2(
\r
540 max(60, pRdmaIo->packet_sz), VIPORT_TRAILER_ALIGNMENT );
\r
541 pad = pRdmaIo->len - pRdmaIo->packet_sz;
\r
543 p_eth_hdr = (eth_hdr_t *)p_buf;
\r
545 pRdmaIo->p_trailer = (ViportTrailer_t *)&pRdmaIo->data[pad];
\r
546 cl_memclr( pRdmaIo->data, pad + sizeof( ViportTrailer_t ) );
\r
547 cl_memcpy( pRdmaIo->p_trailer->destMacAddr, p_eth_hdr->dst.addr, MAC_ADDR_LEN );
\r
549 pRdmaIo->p_trailer->dataLength =
\r
550 hton16( (uint16_t)max( 60, pRdmaIo->packet_sz ) );
\r
552 NdisGetNextBuffer( p_buf_desc, &p_buf_desc );
\r
554 /* should handle VLAN tag */
\r
555 if( pRdmaIo->packet_sz > 16 )
\r
557 if( p_eth_hdr->type == hton16(0x8100) )
\r
559 if( p_sgl->Elements[0].Length > sizeof(eth_hdr_t) )
\r
561 pRdmaIo->p_trailer->vLan = *(uint16_t *)((uint8_t *)p_eth_hdr + 14 );
\r
562 pRdmaIo->p_trailer->pktFlags |= PF_VLAN_INSERT;
\r
568 NdisQueryBufferSafe( p_buf_desc, &p_buf, &buf_len, NormalPagePriority );
\r
570 pad = sizeof(eth_hdr_t) - p_sgl->Elements[0].Length;
\r
571 pRdmaIo->p_trailer->vLan = *(uint16_t *)(p_buf + pad + 2);
\r
572 pRdmaIo->p_trailer->pktFlags |= PF_VLAN_INSERT;
\r
576 else if( p_eth_hdr->type == ETH_PROT_TYPE_IP &&
\r
577 !( p_eth_hdr->dst.addr[0] & 0x01 ) )
\r
581 NdisQueryBufferSafe( p_buf_desc, &p_buf, &buf_len, NormalPagePriority );
\r
583 if( ((ip_pkt_t*)p_buf)->hdr.prot == IP_PROT_UDP ||
\r
584 ((ip_pkt_t*)p_buf)->hdr.prot == IP_PROT_TCP )
\r
586 /* use socket src port + dest port to generate hash value
\r
587 * for link aggregation distribution function.
\r
589 pRdmaIo->p_trailer->connectionHashAndValid = 0x40 |
\r
590 ((uint8_t)((ip_pkt_t*)p_buf)->prot.tcp.src_port +
\r
591 (uint8_t)((ip_pkt_t*)p_buf)->prot.tcp.dst_port ) & 0x3f;
\r
597 pRdmaIo->p_trailer->txChksumFlags = _tx_chksum_flags( p_packet );
\r
598 pRdmaIo->p_trailer->connectionHashAndValid |= CHV_VALID;
\r
601 pRdmaIo->p_trailer->pktFlags |= PF_KICK;
\r
603 /* fill last data segment with trailer and pad */
\r
604 phy_addr = MmGetPhysicalAddress( pRdmaIo->data );
\r
606 pRdmaIo->dsList[p_sgl->NumberOfElements].vaddr = phy_addr.QuadPart;
\r
607 pRdmaIo->dsList[p_sgl->NumberOfElements].length = pRdmaIo->len -
\r
608 pRdmaIo->packet_sz +
\r
609 sizeof( ViportTrailer_t );
\r
610 //pRdmaIo->io.wrq.ds_array = pRdmaIo->dsList;
\r
611 pRdmaIo->io.wrq.num_ds =p_sgl->NumberOfElements + 1;
\r
613 data_rdmaPacket( pData, pBpe, pRdmaIo );
\r
615 if( p_xmitPool->sendKicks )
\r
617 /* EIOC needs kicks to inform it of sent packets */
\r
619 p_xmitPool->kickCount++;
\r
620 p_xmitPool->kickByteCount += pRdmaIo->packet_sz;
\r
621 if( ( p_xmitPool->kickCount >= p_xmitPool->kickBundle )
\r
622 || ( p_xmitPool->kickByteCount >= p_xmitPool->kickByteBundle ) )
\r
624 data_sendKickMessage( pData );
\r
626 else if( p_xmitPool->kickCount == 1 )
\r
628 _data_kickTimer_start( pData, pData->eiocPoolParms.timeoutBeforeKick );
\r
635 IN NDIS_PACKET* const p_packet )
\r
638 NDIS_TCP_IP_CHECKSUM_PACKET_INFO *p_packet_info;
\r
640 uint8_t txChksumFlags = 0;
\r
642 if( NDIS_PROTOCOL_ID_TCP_IP == NDIS_GET_PACKET_PROTOCOL_TYPE(p_packet) )
\r
644 packet_info = PtrToUlong( NDIS_PER_PACKET_INFO_FROM_PACKET( p_packet, TcpIpChecksumPacketInfo));
\r
645 p_packet_info = ( NDIS_TCP_IP_CHECKSUM_PACKET_INFO *)&packet_info;
\r
647 if( p_packet_info &&
\r
648 p_packet_info->Transmit.NdisPacketChecksumV4 )
\r
650 txChksumFlags = TX_CHKSUM_FLAGS_CHECKSUM_V4
\r
651 | ( p_packet_info->Transmit.NdisPacketIpChecksum ? TX_CHKSUM_FLAGS_IP_CHECKSUM: 0 )
\r
652 | ( p_packet_info->Transmit.NdisPacketTcpChecksum ? TX_CHKSUM_FLAGS_TCP_CHECKSUM: 0 )
\r
653 | ( p_packet_info->Transmit.NdisPacketUdpChecksum ? TX_CHKSUM_FLAGS_UDP_CHECKSUM: 0 );
\r
657 VNIC_TRACE( VNIC_DBG_DATA ,
\r
658 ("txChksumFlags = %d: V4 %c, V6 %c, IP %c, TCP %c, UDP %c\n",
\r
660 ((txChksumFlags & TX_CHKSUM_FLAGS_CHECKSUM_V4 )? '+': '-'),
\r
661 ((txChksumFlags & TX_CHKSUM_FLAGS_CHECKSUM_V6 )? '+': '-'),
\r
662 ((txChksumFlags & TX_CHKSUM_FLAGS_IP_CHECKSUM )? '+': '-'),
\r
663 ((txChksumFlags & TX_CHKSUM_FLAGS_TCP_CHECKSUM )? '+': '-'),
\r
664 ((txChksumFlags & TX_CHKSUM_FLAGS_UDP_CHECKSUM )? '+': '-') ));
\r
666 return txChksumFlags;
\r
671 IN NDIS_PACKET *p_packet,
\r
672 IN OUT NDIS_BUFFER **pp_buf_desc,
\r
674 OUT ULONG *p_packet_sz )
\r
677 VNIC_ENTER( VNIC_DBG_DATA );
\r
679 NdisGetFirstBufferFromPacketSafe( p_packet,
\r
684 NormalPagePriority );
\r
685 VNIC_EXIT( VNIC_DBG_DATA );
\r
693 LIST_ENTRY *p_list_entry;
\r
694 ib_api_status_t ib_status;
\r
696 VNIC_ENTER ( VNIC_DBG_DATA );
\r
698 while( ( p_list_entry = ExInterlockedRemoveHeadList( &pData->recvIos,
\r
699 &pData->recvIosLock ))
\r
702 pRecvIo = (RecvIo_t *)p_list_entry;
\r
704 ib_status = ibqp_postRecv( &pData->qp, &pRecvIo->io );
\r
705 if( ib_status != IB_SUCCESS )
\r
707 VNIC_TRACE_EXIT( VNIC_DBG_ERROR, ("ibqp_postRecv returned %s\n",
\r
708 pData->p_viport->p_adapter->ifc.get_err_str( ib_status )) );
\r
709 viport_failure( pData->p_viport );
\r
714 VNIC_EXIT( VNIC_DBG_DATA );
\r
719 _data_receivedKick(
\r
722 Data_t *pData = &pIo->pViport->data;
\r
723 uint32_t num_pkts = 0;
\r
725 VNIC_ENTER( VNIC_DBG_DATA );
\r
727 #ifdef VNIC_STATISTIC
\r
728 recvRef = cl_get_tick_count();
\r
729 #endif /* VNIC_STATISTIC */
\r
731 ExInterlockedInsertTailList( &pData->recvIos, &pIo->listPtrs, &pData->recvIosLock );
\r
733 data_postRecvs( pData );
\r
735 #ifdef VNIC_STATISTIC
\r
736 pData->statistics.kickRecvs++;
\r
737 #endif /* VNIC_STATISTIC */
\r
739 data_checkXmitBuffers( pData );
\r
741 num_pkts = _data_incomingRecv( pData );
\r
745 NdisMIndicateReceivePacket( pData->p_viport->p_adapter->h_handle,
\r
746 pData->recvPool.recv_pkt_array,
\r
748 pData->p_viport->stats.ifInOk += num_pkts;
\r
751 VNIC_EXIT( VNIC_DBG_DATA );
\r
756 _data_xmitComplete(
\r
759 RdmaIo_t *pRdmaIo = (RdmaIo_t *)pIo;
\r
760 Data_t *pData = &pIo->pViport->data;
\r
761 XmitPool_t *p_xmitPool = &pData->xmitPool;
\r
762 NDIS_PACKET *p_packet;
\r
763 NDIS_STATUS ndis_status;
\r
764 LIST_ENTRY *p_list_item;
\r
766 VNIC_ENTER( VNIC_DBG_DATA );
\r
768 while ( p_xmitPool->lastCompBuf != pRdmaIo->index )
\r
770 INC(p_xmitPool->lastCompBuf, 1, p_xmitPool->numXmitBufs);
\r
771 p_packet = p_xmitPool->pXmitBufs[p_xmitPool->lastCompBuf].p_packet;
\r
773 p_xmitPool->pXmitBufs[p_xmitPool->lastCompBuf].p_packet = NULL;
\r
775 if( p_packet != NULL )
\r
777 if( pIo->wc_status != IB_WCS_SUCCESS )
\r
779 ndis_status = NDIS_STATUS_FAILURE;
\r
780 pIo->pViport->stats.ifOutErrors++;
\r
781 pIo->wc_status = IB_WCS_SUCCESS;
\r
785 ndis_status = NDIS_STATUS_SUCCESS;
\r
786 pIo->pViport->stats.ifOutOk++;
\r
788 NDIS_SET_PACKET_STATUS( p_packet, ndis_status );
\r
789 NdisMSendComplete( pIo->pViport->p_adapter->h_handle,
\r
790 p_packet, ndis_status );
\r
794 if( !pIo->pViport->p_netpath->carrier )
\r
796 while( ( p_list_item = NdisInterlockedRemoveHeadList(
\r
797 &pIo->pViport->send_pending_list,
\r
798 &pIo->pViport->pending_list_lock ) ) != NULL )
\r
800 p_packet = VNIC_PACKET_FROM_LIST_ITEM( p_list_item );
\r
801 NDIS_SET_PACKET_STATUS( p_packet, NDIS_STATUS_FAILURE );
\r
802 NdisMSendComplete( &pIo->pViport->p_adapter->h_handle,
\r
803 p_packet, NDIS_STATUS_FAILURE );
\r
804 pIo->pViport->stats.ifOutErrors++;
\r
808 data_checkXmitBuffers( pData );
\r
810 VNIC_EXIT( VNIC_DBG_DATA );
\r
815 data_sendKickMessage(
\r
818 XmitPool_t *pPool = &pData->xmitPool;
\r
820 VNIC_ENTER( VNIC_DBG_DATA );
\r
822 /* stop timer for BundleTimeout */
\r
823 _data_kickTimer_stop( pData );
\r
825 pPool->kickCount = 0;
\r
826 pPool->kickByteCount = 0;
\r
828 /* TBD: Keep track of when kick is outstanding, and
\r
829 * don't reuse until complete
\r
831 if ( ibqp_postSend( &pData->qp, &pData->kickIo.io ) != IB_SUCCESS )
\r
833 VNIC_TRACE( VNIC_DBG_ERROR,
\r
834 ("Unable to send kick to EIOC\n") );
\r
835 viport_failure( pData->p_viport );
\r
838 VNIC_EXIT( VNIC_DBG_DATA );
\r
842 _data_kickTimeoutHandler( void * context )
\r
844 Data_t* pData = (Data_t *)context;
\r
846 VNIC_ENTER( VNIC_DBG_DATA );
\r
848 InterlockedExchange( &pData->kickTimerOn, FALSE );
\r
849 data_sendKickMessage( pData );
\r
851 VNIC_EXIT( VNIC_DBG_DATA );
\r
857 data_allocXmitBuffer(
\r
859 OUT BufferPoolEntry_t **ppBpe,
\r
860 OUT RdmaIo_t **ppRdmaIo,
\r
861 OUT BOOLEAN *pLast )
\r
863 XmitPool_t *p_xmitPool = &pData->xmitPool;
\r
866 VNIC_ENTER( VNIC_DBG_DATA );
\r
868 KeAcquireSpinLock( &pData->xmitBufLock, &flags );
\r
871 *ppRdmaIo = &p_xmitPool->pXmitBufs[p_xmitPool->nextXmitBuf];
\r
872 *ppBpe = &p_xmitPool->bufPool[p_xmitPool->nextXmitPool];
\r
874 if ( (*ppBpe)->valid && p_xmitPool->nextXmitBuf != p_xmitPool->lastCompBuf )
\r
876 INC(p_xmitPool->nextXmitBuf, 1, p_xmitPool->numXmitBufs);
\r
877 INC(p_xmitPool->nextXmitPool, 1, p_xmitPool->poolSz);
\r
879 if ( !p_xmitPool->bufPool[p_xmitPool->nextXmitPool].valid )
\r
881 VNIC_TRACE( VNIC_DBG_DATA,
\r
882 ("Just used the last EIOU receive buffer\n") );
\r
885 p_xmitPool->needBuffers = TRUE;
\r
886 viport_stopXmit( pData->p_viport );
\r
887 #ifdef VNIC_STATISTIC
\r
888 pData->statistics.kickReqs++;
\r
889 #endif /* VNIC_STATISTIC */
\r
891 else if ( p_xmitPool->nextXmitBuf == p_xmitPool->lastCompBuf )
\r
893 VNIC_TRACE( VNIC_DBG_DATA,
\r
894 ("Just used our last xmit buffer\n") );
\r
896 p_xmitPool->needBuffers = TRUE;
\r
897 viport_stopXmit( pData->p_viport );
\r
900 (*ppBpe)->valid = 0;
\r
902 KeReleaseSpinLock( &pData->xmitBufLock, flags );
\r
907 #ifdef VNIC_STATISTIC
\r
908 pData->statistics.noXmitBufs++;
\r
909 #endif /* VNIC_STATISTIC */
\r
911 VNIC_TRACE( VNIC_DBG_ERROR,
\r
912 ("Out of xmit buffers\n") );
\r
914 viport_stopXmit( pData->p_viport );
\r
916 KeReleaseSpinLock( &pData->xmitBufLock, flags );
\r
922 data_checkXmitBuffers(
\r
925 XmitPool_t *p_xmitPool = &pData->xmitPool;
\r
928 VNIC_ENTER( VNIC_DBG_DATA );
\r
930 KeAcquireSpinLock( &pData->xmitBufLock, &flags );
\r
932 if ( pData->xmitPool.needBuffers
\r
933 && p_xmitPool->bufPool[p_xmitPool->nextXmitPool].valid
\r
934 && p_xmitPool->nextXmitBuf != p_xmitPool->lastCompBuf )
\r
936 pData->xmitPool.needBuffers = FALSE;
\r
937 viport_restartXmit( pData->p_viport );
\r
939 VNIC_TRACE( VNIC_DBG_DATA,
\r
940 ("There are free xmit buffers\n") );
\r
943 KeReleaseSpinLock( &pData->xmitBufLock, flags );
\r
945 VNIC_EXIT( VNIC_DBG_DATA );
\r
952 IN BufferPoolEntry_t *pBpe,
\r
953 IN RdmaIo_t *pRdmaIo )
\r
955 ib_send_wr_t *pWrq;
\r
956 uint64_t remote_addr;
\r
958 VNIC_ENTER( VNIC_DBG_DATA );
\r
960 pWrq = &pRdmaIo->io.wrq;
\r
962 remote_addr = ntoh64( pBpe->remoteAddr );
\r
963 remote_addr += pData->xmitPool.bufferSz;
\r
964 remote_addr -= ( pRdmaIo->len + sizeof(ViportTrailer_t) );
\r
966 pWrq->remote_ops.vaddr = remote_addr;
\r
967 pWrq->remote_ops.rkey = pBpe->rKey;
\r
969 pData->xmitPool.notifyCount++;
\r
971 if( pData->xmitPool.notifyCount >= pData->xmitPool.notifyBundle )
\r
973 pData->xmitPool.notifyCount = 0;
\r
974 pWrq->send_opt = IB_SEND_OPT_SIGNALED;
\r
978 pWrq->send_opt &= ~IB_SEND_OPT_SIGNALED;
\r
980 pWrq->send_opt = IB_SEND_OPT_SIGNALED;
\r
982 if( ibqp_postSend( &pData->qp, &pRdmaIo->io ) != IB_SUCCESS )
\r
984 VNIC_TRACE(VNIC_DBG_ERROR,
\r
985 ("Failed sending data to EIOC\n") );
\r
986 viport_failure( pData->p_viport );
\r
989 #ifdef VNIC_STATISTIC
\r
990 pData->statistics.xmitNum++;
\r
991 #endif /* VNIC_STATISTIC */
\r
993 VNIC_EXIT( VNIC_DBG_DATA );
\r
996 static NDIS_PACKET *
\r
997 _data_recv_to_ndis_pkt(
\r
999 IN RdmaDest_t *pRdmaDest )
\r
1001 struct ViportTrailer *pTrailer;
\r
1002 NDIS_PACKET *p_packet;
\r
1003 NDIS_STATUS ndis_status;
\r
1006 uint8_t rxChksumFlags;
\r
1007 NDIS_TCP_IP_CHECKSUM_PACKET_INFO packet_info;
\r
1009 VNIC_ENTER( VNIC_DBG_DATA );
\r
1011 pTrailer = pRdmaDest->pTrailer;
\r
1012 start = (int)data_offset(pData, pTrailer);
\r
1013 len = data_len(pData, pTrailer);
\r
1015 NdisAllocatePacket( &ndis_status,
\r
1017 pData->h_recv_pkt_pool );
\r
1019 if ( ndis_status != NDIS_STATUS_SUCCESS )
\r
1021 VNIC_TRACE_EXIT( VNIC_DBG_ERROR,
\r
1022 ( "NdisAllocatePacket failed %#x\n", ndis_status ) );
\r
1025 NdisAllocateBuffer( &ndis_status,
\r
1026 &pRdmaDest->p_buf,
\r
1027 pData->h_recv_buf_pool,
\r
1028 pRdmaDest->data + start,
\r
1031 if ( ndis_status != NDIS_STATUS_SUCCESS )
\r
1033 VNIC_TRACE_EXIT( VNIC_DBG_ERROR,
\r
1034 ( "NdisAllocateBuffer failed %#x\n", ndis_status ) );
\r
1035 NdisFreePacket( p_packet );
\r
1039 NdisChainBufferAtFront( p_packet, pRdmaDest->p_buf );
\r
1040 pRdmaDest->p_packet = p_packet;
\r
1042 if ( pTrailer->pktFlags & PF_VLAN_INSERT )
\r
1045 * add OID_GEN_VLAN_ID
\r
1046 * handle VLAN tag insertion
\r
1047 * set packet header size = eth_hdr + 4
\r
1051 NDIS_SET_PACKET_HEADER_SIZE( p_packet, sizeof(eth_hdr_t) );
\r
1053 rxChksumFlags = pTrailer->rxChksumFlags;
\r
1055 VNIC_TRACE( VNIC_DBG_DATA,
\r
1056 ("rxChksumFlags = %d, LOOP = %c, IP = %c, TCP = %c, UDP = %c\n",
\r
1058 (rxChksumFlags & RX_CHKSUM_FLAGS_LOOPBACK)? 'Y': 'N',
\r
1059 (rxChksumFlags & RX_CHKSUM_FLAGS_IP_CHECKSUM_SUCCEEDED)? 'Y':
\r
1060 (rxChksumFlags & RX_CHKSUM_FLAGS_IP_CHECKSUM_FAILED)? 'N': '-',
\r
1061 (rxChksumFlags & RX_CHKSUM_FLAGS_TCP_CHECKSUM_SUCCEEDED)? 'Y':
\r
1062 (rxChksumFlags & RX_CHKSUM_FLAGS_TCP_CHECKSUM_FAILED)? 'N': '-',
\r
1063 (rxChksumFlags & RX_CHKSUM_FLAGS_UDP_CHECKSUM_SUCCEEDED)? 'Y':
\r
1064 (rxChksumFlags & RX_CHKSUM_FLAGS_UDP_CHECKSUM_FAILED)? 'N': '-') );
\r
1066 packet_info.Value = 0;
\r
1068 if( rxChksumFlags & RX_CHKSUM_FLAGS_IP_CHECKSUM_SUCCEEDED )
\r
1069 packet_info.Receive.NdisPacketIpChecksumSucceeded = TRUE;
\r
1070 else if( rxChksumFlags & RX_CHKSUM_FLAGS_IP_CHECKSUM_FAILED )
\r
1071 packet_info.Receive.NdisPacketIpChecksumFailed = TRUE;
\r
1073 if( rxChksumFlags & RX_CHKSUM_FLAGS_TCP_CHECKSUM_SUCCEEDED )
\r
1074 packet_info.Receive.NdisPacketTcpChecksumSucceeded = TRUE;
\r
1075 else if( rxChksumFlags & RX_CHKSUM_FLAGS_TCP_CHECKSUM_FAILED )
\r
1076 packet_info.Receive.NdisPacketTcpChecksumFailed = TRUE;
\r
1078 if( rxChksumFlags & RX_CHKSUM_FLAGS_TCP_CHECKSUM_SUCCEEDED )
\r
1079 packet_info.Receive.NdisPacketUdpChecksumSucceeded = TRUE;
\r
1080 else if( rxChksumFlags & RX_CHKSUM_FLAGS_TCP_CHECKSUM_FAILED )
\r
1081 packet_info.Receive.NdisPacketUdpChecksumFailed = TRUE;
\r
1083 NDIS_PER_PACKET_INFO_FROM_PACKET( p_packet, TcpIpChecksumPacketInfo )=
\r
1084 (void *)(uintn_t)packet_info.Value;
\r
1086 VNIC_RECV_FROM_PACKET( p_packet ) = pRdmaDest;
\r
1087 NDIS_SET_PACKET_STATUS( p_packet, NDIS_STATUS_SUCCESS );
\r
1089 VNIC_EXIT( VNIC_DBG_DATA );
\r
1093 /* NOTE: This routine is not reentrant */
\r
1095 _data_allocBuffers(
\r
1097 IN BOOLEAN initialAllocation )
\r
1099 RecvPool_t *p_recvPool = &pData->recvPool;
\r
1100 RdmaDest_t *pRdmaDest;
\r
1101 LIST_ENTRY *p_list_entry;
\r
1104 VNIC_ENTER( VNIC_DBG_DATA );
\r
1106 index = ADD(p_recvPool->nextFreeBuf, p_recvPool->numFreeBufs, p_recvPool->eiocPoolSz);
\r
1108 while ( !IsListEmpty( &p_recvPool->availRecvBufs ) )
\r
1110 p_list_entry = RemoveHeadList( &p_recvPool->availRecvBufs );
\r
1111 pRdmaDest = (RdmaDest_t*)p_list_entry;
\r
1113 if( initialAllocation )
\r
1115 pRdmaDest->buf_sz = p_recvPool->bufferSz;
\r
1116 pRdmaDest->pTrailer =
\r
1117 (struct ViportTrailer*)(pRdmaDest->data + pRdmaDest->buf_sz
\r
1118 - sizeof(struct ViportTrailer));
\r
1119 pRdmaDest->pTrailer->connectionHashAndValid = 0;
\r
1122 pRdmaDest->p_packet = NULL;
\r
1123 _data_addFreeBuffer(pData, index, pRdmaDest);
\r
1124 index = NEXT(index,p_recvPool->eiocPoolSz);
\r
1127 VNIC_EXIT( VNIC_DBG_DATA );
\r
1132 _data_addFreeBuffer(
\r
1135 IN RdmaDest_t *pRdmaDest )
\r
1137 RecvPool_t *p_recvPool = &pData->recvPool;
\r
1138 BufferPoolEntry_t *pBpe;
\r
1140 pRdmaDest->pTrailer->connectionHashAndValid = 0;
\r
1141 pBpe = &p_recvPool->bufPool[index];
\r
1143 pBpe->rKey = pRdmaDest->region.rkey;
\r
1144 pBpe->remoteAddr = hton64( PTR64( pRdmaDest->data ) );
\r
1145 pBpe->valid = (uint32_t)(pRdmaDest - &p_recvPool->pRecvBufs[0]) + 1;
\r
1146 ++p_recvPool->numFreeBufs;
\r
1152 _data_incomingRecv(
\r
1153 IN Data_t *pData )
\r
1155 RecvPool_t *p_recvPool = &pData->recvPool;
\r
1156 RdmaDest_t *pRdmaDest;
\r
1157 ViportTrailer_t *pTrailer;
\r
1158 BufferPoolEntry_t *pBpe;
\r
1159 NDIS_PACKET *p_packet = NULL;
\r
1161 BOOLEAN status = FALSE;
\r
1163 VNIC_ENTER( VNIC_DBG_DATA );
\r
1167 if ( p_recvPool->nextFullBuf == p_recvPool->nextFreeBuf )
\r
1170 pBpe = &p_recvPool->bufPool[p_recvPool->nextFullBuf];
\r
1171 pRdmaDest = &p_recvPool->pRecvBufs[pBpe->valid - 1];
\r
1172 pTrailer = pRdmaDest->pTrailer;
\r
1174 if ( ( pTrailer != NULL ) &&
\r
1175 ( pTrailer->connectionHashAndValid & CHV_VALID ) )
\r
1177 /* received a packet */
\r
1178 if ( pTrailer->pktFlags & PF_KICK )
\r
1180 p_recvPool->kickOnFree = TRUE;
\r
1182 /* we do not want to indicate packet if no filter is set */
\r
1183 if( pData->p_viport->p_adapter->packet_filter )
\r
1185 p_packet = _data_recv_to_ndis_pkt( pData, pRdmaDest );
\r
1186 if ( p_packet != NULL )
\r
1188 p_recvPool->recv_pkt_array[idx++] = p_packet;
\r
1192 { /* put back to free buffers pool */
\r
1193 InsertTailList( &p_recvPool->availRecvBufs,
\r
1194 &pRdmaDest->listPtrs );
\r
1197 INC( p_recvPool->nextFullBuf, 1, p_recvPool->eiocPoolSz );
\r
1198 p_recvPool->numPostedBufs--;
\r
1199 #ifdef VNIC_STATISTIC
\r
1200 pData->statistics.recvNum++;
\r
1201 #endif /* VNIC_STATISTIC */
\r
1211 vnic_return_packet(
\r
1212 IN NDIS_HANDLE adapter_context,
\r
1213 IN NDIS_PACKET * const p_packet )
\r
1217 vnic_adapter_t *p_adapter = (vnic_adapter_t *)adapter_context;
\r
1218 viport_t *p_viport = p_adapter->p_currentPath->pViport;
\r
1219 RdmaDest_t *p_rdma_dest = VNIC_RECV_FROM_PACKET( p_packet );
\r
1221 ASSERT( p_rdma_dest->p_packet == p_packet );
\r
1222 _data_return_recv( p_packet );
\r
1223 p_rdma_dest->p_packet = NULL;
\r
1225 InsertTailList( &p_viport->data.recvPool.availRecvBufs,
\r
1226 &p_rdma_dest->listPtrs );
\r
1228 if( p_viport->data.connected == TRUE &&
\r
1229 !p_viport->errored )
\r
1231 _data_allocBuffers( &p_viport->data, FALSE );
\r
1232 _data_sendFreeRecvBuffers( &p_viport->data );
\r
1236 _data_return_recv(
\r
1237 IN NDIS_PACKET *p_packet )
\r
1239 NDIS_BUFFER *p_buf;
\r
1240 /* Unchain the NDIS buffer. */
\r
1241 NdisUnchainBufferAtFront( p_packet, &p_buf );
\r
1242 CL_ASSERT( p_buf );
\r
1243 /* Return the NDIS packet and NDIS buffer to their pools. */
\r
1244 NdisFreeBuffer( p_buf );
\r
1245 NdisFreePacket( p_packet );
\r
1249 _data_sendFreeRecvBuffers(
\r
1250 IN Data_t *pData )
\r
1252 RecvPool_t *p_recvPool = &pData->recvPool;
\r
1253 ib_send_wr_t *pWrq = &pData->freeBufsIo.io.wrq;
\r
1254 BOOLEAN bufsSent = FALSE;
\r
1255 uint64_t rdmaAddr;
\r
1258 unsigned int numToSend,
\r
1261 VNIC_ENTER( VNIC_DBG_DATA );
\r
1263 for ( numToSend = p_recvPool->szFreeBundle;
\r
1264 numToSend <= p_recvPool->numFreeBufs;
\r
1265 numToSend += p_recvPool->szFreeBundle )
\r
1267 /* Handle multiple bundles as one when possible. */
\r
1268 nextIncrement = numToSend + p_recvPool->szFreeBundle;
\r
1269 if ( ( nextIncrement <= p_recvPool->numFreeBufs )
\r
1270 && ( p_recvPool->nextFreeBuf + nextIncrement <= p_recvPool->eiocPoolSz ) )
\r
1275 offset = p_recvPool->nextFreeBuf * sizeof(BufferPoolEntry_t);
\r
1276 sz = numToSend * sizeof(BufferPoolEntry_t);
\r
1277 rdmaAddr = p_recvPool->eiocRdmaAddr + offset;
\r
1279 pWrq->ds_array->length = sz;
\r
1280 pWrq->ds_array->vaddr = PTR64((uint8_t *)p_recvPool->bufPool + offset);
\r
1281 pWrq->remote_ops.vaddr = rdmaAddr;
\r
1283 if ( ibqp_postSend( &pData->qp, &pData->freeBufsIo.io ) != IB_SUCCESS )
\r
1285 VNIC_TRACE(VNIC_DBG_ERROR,
\r
1286 ("Unable to rdma free buffers to EIOC\n") );
\r
1288 viport_failure( pData->p_viport );
\r
1292 INC( p_recvPool->nextFreeBuf, numToSend, p_recvPool->eiocPoolSz );
\r
1293 p_recvPool->numFreeBufs -= numToSend;
\r
1294 p_recvPool->numPostedBufs += numToSend;
\r
1300 if( p_recvPool->kickOnFree )
\r
1302 data_sendKickMessage( pData );
\r
1305 if( p_recvPool->numPostedBufs == 0 )
\r
1307 VNIC_TRACE( VNIC_DBG_ERROR,
\r
1308 ("IOC %d: Unable to allocate receive buffers\n",
\r
1309 pData->p_viport->ioc_num ) );
\r
1311 viport_failure( pData->p_viport );
\r
1313 VNIC_EXIT( VNIC_DBG_DATA );
\r
1319 IN Data_t *pData )
\r
1321 VNIC_ENTER( VNIC_DBG_DATA );
\r
1323 if( pData->recvPool.recv_pkt_array )
\r
1325 cl_free( pData->recvPool.recv_pkt_array );
\r
1326 pData->recvPool.recv_pkt_array = NULL;
\r
1327 pData->recvPool.poolSz = 0;
\r
1330 if ( pData->pLocalStorage )
\r
1332 NdisFreeMemory( pData->pLocalStorage, pData->localStorageSz, 0 );
\r
1333 pData->pLocalStorage = NULL;
\r
1336 if( pData->h_recv_buf_pool )
\r
1338 NdisFreeBufferPool( pData->h_recv_buf_pool );
\r
1339 pData->h_recv_buf_pool = NULL;
\r
1342 if ( pData->h_recv_pkt_pool )
\r
1344 while( NdisPacketPoolUsage(pData->h_recv_pkt_pool) != 0)
\r
1346 VNIC_TRACE( VNIC_DBG_WARN,
\r
1347 ("Recv packet pool is not empty!!!\n") );
\r
1350 NdisFreePacketPool( pData->h_recv_pkt_pool );
\r
1351 pData->h_recv_pkt_pool = NULL;
\r
1353 if( pData->p_recv_bufs )
\r
1355 NdisFreeMemory( pData->p_recv_bufs, pData->recv_bufs_sz, 0 );
\r
1356 pData->p_recv_bufs = NULL;
\r
1358 // clear Qp struct for reuse
\r
1359 cl_memclr( &pData->qp, sizeof( IbQp_t) );
\r
1361 cl_timer_destroy( &pData->kickTimer );
\r
1363 VNIC_EXIT( VNIC_DBG_DATA );
\r
1368 _data_kickTimer_start(
\r
1370 IN uint32_t microseconds )
\r
1372 VNIC_ENTER( VNIC_DBG_DATA );
\r
1374 InterlockedExchange( (LONG *)&pData->kickTimerOn, TRUE );
\r
1376 usec_timer_start(&pData->kickTimer, microseconds );
\r
1378 VNIC_EXIT( VNIC_DBG_DATA );
\r
1383 _data_kickTimer_stop(
\r
1384 IN Data_t *pData )
\r
1386 VNIC_ENTER( VNIC_DBG_DATA );
\r
1388 if( InterlockedExchange( &pData->kickTimerOn, FALSE ) == TRUE )
\r
1390 cl_timer_stop( &pData->kickTimer );
\r
1393 VNIC_EXIT( VNIC_DBG_DATA );
\r