[IPOIB] Prevent a BSOD which happens when restarting the opensm more than once (if...
[mirror/winof/.git] / ulp / ipoib / kernel / ipoib_port.c
1 /*\r
2  * Copyright (c) 2005 SilverStorm Technologies.  All rights reserved.\r
3  * Copyright (c) 2006 Mellanox Technologies.  All rights reserved.\r
4  * Portions Copyright (c) 2008 Microsoft Corporation.  All rights reserved.\r
5  *\r
6  * This software is available to you under the OpenIB.org BSD license\r
7  * below:\r
8  *\r
9  *     Redistribution and use in source and binary forms, with or\r
10  *     without modification, are permitted provided that the following\r
11  *     conditions are met:\r
12  *\r
13  *      - Redistributions of source code must retain the above\r
14  *        copyright notice, this list of conditions and the following\r
15  *        disclaimer.\r
16  *\r
17  *      - Redistributions in binary form must reproduce the above\r
18  *        copyright notice, this list of conditions and the following\r
19  *        disclaimer in the documentation and/or other materials\r
20  *        provided with the distribution.\r
21  *\r
22  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,\r
23  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF\r
24  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND\r
25  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS\r
26  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN\r
27  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN\r
28  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\r
29  * SOFTWARE.\r
30  *\r
31  * $Id$\r
32  */\r
33 \r
34 \r
35 \r
36 #include "ipoib_port.h"\r
37 #include "ipoib_adapter.h"\r
38 #include "ipoib_debug.h"\r
39 #if defined(EVENT_TRACING)\r
40 #ifdef offsetof\r
41 #undef offsetof\r
42 #endif\r
43 #include "ipoib_port.tmh"\r
44 #endif\r
45 #include <offload.h>\r
46 \r
47 \r
48 /* Amount of physical memory to register. */\r
49 #define MEM_REG_SIZE    0xFFFFFFFFFFFFFFFF\r
50 \r
51 /* Number of work completions to chain for send and receive polling. */\r
52 #define MAX_SEND_WC             8\r
53 #define MAX_RECV_WC             16\r
54 \r
55 \r
56 ib_gid_t        bcast_mgid_template = {\r
57         0xff,                                                           /* multicast field */\r
58         0x12,                                                           /* scope (to be filled in) */\r
59         0x40, 0x1b,                                                     /* IPv4 signature */\r
60         0xff, 0xff,                                                     /* 16 bits of P_Key (to be filled in) */\r
61         0x00, 0x00, 0x00, 0x00, 0x00, 0x00,     /* 48 bits of zeros */\r
62         0xff, 0xff, 0xff, 0xff,                         /* 32 bit IPv4 broadcast address */\r
63 };\r
64 \r
65 \r
66 #ifdef _DEBUG_\r
67 /* Handy pointer for debug use. */\r
68 ipoib_port_t    *gp_ipoib_port;\r
69 #endif\r
70 \r
71 static void __port_mcast_garbage_dpc(KDPC *p_gc_dpc,void *context,void *s_arg1, void *s_arg2);\r
72 static void __port_do_mcast_garbage(ipoib_port_t* const p_port );\r
73 \r
74 \r
75 static void __recv_cb_dpc(KDPC *p_gc_dpc,void *context,void *s_arg1, void *s_arg2);\r
76 \r
77 \r
78 /******************************************************************************\r
79 *\r
80 * Declarations\r
81 *\r
82 ******************************************************************************/\r
83 static void\r
84 __port_construct(\r
85         IN                              ipoib_port_t* const                     p_port );\r
86 \r
87 static ib_api_status_t\r
88 __port_init(\r
89         IN                              ipoib_port_t* const                     p_port,\r
90         IN                              ipoib_adapter_t* const          p_adapter,\r
91         IN                              ib_pnp_port_rec_t* const        p_pnp_rec );\r
92 \r
93 static void\r
94 __port_destroying(\r
95         IN                              cl_obj_t* const                         p_obj );\r
96 \r
97 static void\r
98 __port_cleanup(\r
99         IN                              cl_obj_t* const                         p_obj );\r
100 \r
101 static void\r
102 __port_free(\r
103         IN                              cl_obj_t* const                         p_obj );\r
104 \r
105 \r
106 /******************************************************************************\r
107 *\r
108 * IB resource manager operations\r
109 *\r
110 ******************************************************************************/\r
111 static void\r
112 __ib_mgr_construct(\r
113         IN                              ipoib_port_t* const                     p_port );\r
114 \r
115 static ib_api_status_t\r
116 __ib_mgr_init(\r
117         IN                              ipoib_port_t* const                     p_port );\r
118 \r
119 static void\r
120 __ib_mgr_destroy(\r
121         IN                              ipoib_port_t* const                     p_port );\r
122 \r
123 static void\r
124 __qp_event(\r
125         IN                              ib_async_event_rec_t            *p_event_rec );\r
126 \r
127 static void\r
128 __cq_event(\r
129         IN                              ib_async_event_rec_t            *p_event_rec );\r
130 \r
131 static ib_api_status_t\r
132 __ib_mgr_activate(\r
133         IN                              ipoib_port_t* const                     p_port );\r
134 \r
135 /******************************************************************************\r
136 *\r
137 * Buffer manager operations.\r
138 *\r
139 ******************************************************************************/\r
140 static void\r
141 __buf_mgr_construct(\r
142         IN                              ipoib_port_t* const                     p_port );\r
143 \r
144 static ib_api_status_t\r
145 __buf_mgr_init(\r
146         IN                              ipoib_port_t* const                     p_port );\r
147 \r
148 static void\r
149 __buf_mgr_destroy(\r
150         IN                              ipoib_port_t* const                     p_port );\r
151 \r
152 static cl_status_t\r
153 __recv_ctor(\r
154         IN                              void* const                                     p_object,\r
155         IN                              void*                                           context,\r
156                 OUT                     cl_pool_item_t** const          pp_pool_item );\r
157 \r
158 #if !IPOIB_INLINE_RECV\r
159 static void\r
160 __recv_dtor(\r
161         IN              const   cl_pool_item_t* const           p_pool_item,\r
162         IN                              void                                            *context );\r
163 #endif  /* IPOIB_INLINE_RECV */\r
164 \r
165 static inline ipoib_send_desc_t*\r
166 __buf_mgr_get_send(\r
167         IN                              ipoib_port_t* const                     p_port );\r
168 \r
169 static inline void\r
170 __buf_mgr_put_send(\r
171         IN                              ipoib_port_t* const                     p_port,\r
172         IN                              ipoib_send_desc_t* const        p_desc );\r
173 \r
174 static inline ipoib_recv_desc_t*\r
175 __buf_mgr_get_recv(\r
176         IN                              ipoib_port_t* const                     p_port );\r
177 \r
178 static inline void\r
179 __buf_mgr_put_recv(\r
180         IN                              ipoib_port_t* const                     p_port,\r
181         IN                              ipoib_recv_desc_t* const        p_desc,\r
182         IN                              NDIS_PACKET* const                      p_packet OPTIONAL );\r
183 \r
184 static inline void\r
185 __buf_mgr_put_recv_list(\r
186         IN                              ipoib_port_t* const                     p_port,\r
187         IN                              cl_qlist_t* const                       p_list );\r
188 \r
189 static inline NDIS_PACKET*\r
190 __buf_mgr_get_ndis_pkt(\r
191         IN                              ipoib_port_t* const                     p_port,\r
192         IN                              ipoib_recv_desc_t* const        p_desc );\r
193 \r
194 \r
195 /******************************************************************************\r
196 *\r
197 * Receive manager operations.\r
198 *\r
199 ******************************************************************************/\r
200 static void\r
201 __recv_mgr_construct(\r
202         IN                              ipoib_port_t* const                     p_port );\r
203 \r
204 static ib_api_status_t\r
205 __recv_mgr_init(\r
206         IN                              ipoib_port_t* const                     p_port );\r
207 \r
208 static void\r
209 __recv_mgr_destroy(\r
210         IN                              ipoib_port_t* const                     p_port );\r
211 \r
212 /* Posts receive buffers to the receive queue. */\r
213 static ib_api_status_t\r
214 __recv_mgr_repost(\r
215         IN                              ipoib_port_t* const                     p_port );\r
216 \r
217 static void\r
218 __recv_cb(\r
219         IN              const   ib_cq_handle_t                          h_cq,\r
220         IN                              void                                            *cq_context );\r
221 \r
222 static void\r
223 __recv_get_endpts(\r
224         IN                              ipoib_port_t* const                     p_port,\r
225         IN                              ipoib_recv_desc_t* const        p_desc,\r
226         IN                              ib_wc_t* const                          p_wc,\r
227                 OUT                     ipoib_endpt_t** const           pp_src,\r
228                 OUT                     ipoib_endpt_t** const           pp_dst );\r
229 \r
230 static int32_t\r
231 __recv_mgr_filter(\r
232         IN                              ipoib_port_t* const                     p_port,\r
233         IN                              ib_wc_t* const                          p_done_wc_list,\r
234                 OUT                     cl_qlist_t* const                       p_done_list,\r
235                 OUT                     cl_qlist_t* const                       p_bad_list );\r
236 \r
237 static ib_api_status_t\r
238 __recv_gen(\r
239         IN              const   ipoib_pkt_t* const                      p_ipoib,\r
240                 OUT                     eth_pkt_t* const                        p_eth,\r
241         IN                              ipoib_endpt_t* const            p_src,\r
242         IN                              ipoib_endpt_t* const            p_dst );\r
243 \r
244 static ib_api_status_t\r
245 __recv_dhcp(\r
246         IN                              ipoib_port_t* const                     p_port,\r
247         IN              const   ipoib_pkt_t* const                      p_ipoib,\r
248                 OUT                     eth_pkt_t* const                        p_eth,\r
249         IN                              ipoib_endpt_t* const            p_src,\r
250         IN                              ipoib_endpt_t* const            p_dst );\r
251 \r
252 static ib_api_status_t\r
253 __recv_arp(\r
254         IN                              ipoib_port_t* const                     p_port,\r
255         IN                              ib_wc_t* const                          p_wc,\r
256         IN              const   ipoib_pkt_t* const                      p_ipoib,\r
257                 OUT                     eth_pkt_t* const                        p_eth,\r
258         IN                              ipoib_endpt_t** const           p_src,\r
259         IN                              ipoib_endpt_t* const            p_dst );\r
260 \r
261 static ib_api_status_t\r
262 __recv_mgr_prepare_pkt(\r
263         IN                              ipoib_port_t* const                     p_port,\r
264         IN                              ipoib_recv_desc_t* const        p_desc,\r
265                 OUT                     NDIS_PACKET** const                     pp_packet );\r
266 \r
267 static uint32_t\r
268 __recv_mgr_build_pkt_array(\r
269         IN                              ipoib_port_t* const                     p_port,\r
270         IN                              int32_t                                         shortage,\r
271                 OUT                     cl_qlist_t* const                       p_done_list,\r
272                 OUT                     int32_t* const                          p_discarded );\r
273 \r
274 /******************************************************************************\r
275 *\r
276 * Send manager operations.\r
277 *\r
278 ******************************************************************************/\r
279 static void\r
280 __send_mgr_construct(\r
281         IN                              ipoib_port_t* const                     p_port );\r
282 \r
283 static void\r
284 __send_mgr_destroy(\r
285         IN                              ipoib_port_t* const                     p_port );\r
286 \r
287 static NDIS_STATUS\r
288 __send_gen(\r
289         IN                              ipoib_port_t* const                     p_port,\r
290         IN                              ipoib_send_desc_t* const        p_desc,\r
291         IN                              INT                                             lso_data_index);\r
292 \r
293 static NDIS_STATUS\r
294 __send_mgr_filter_ip(\r
295         IN                              ipoib_port_t* const                     p_port,\r
296         IN              const   eth_hdr_t* const                        p_eth_hdr,\r
297         IN                              NDIS_BUFFER*                            p_buf,\r
298         IN                              size_t                                          buf_len,\r
299         IN      OUT                     ipoib_send_desc_t* const        p_desc );\r
300 \r
301 static NDIS_STATUS\r
302 __send_mgr_filter_igmp_v2(\r
303         IN                              ipoib_port_t* const                     p_port,\r
304     IN          const   ip_hdr_t* const                         p_ip_hdr,\r
305         IN                              size_t                                          iph_options_size,\r
306         IN                              NDIS_BUFFER*                            p_buf,\r
307         IN                              size_t                                          buf_len );\r
308 \r
309 static NDIS_STATUS\r
310 __send_mgr_filter_udp(\r
311         IN                              ipoib_port_t* const                     p_port,\r
312         IN              const   ip_hdr_t* const                         p_ip_hdr,\r
313         IN                              NDIS_BUFFER*                            p_buf,\r
314         IN                              size_t                                          buf_len,\r
315         IN      OUT                     ipoib_send_desc_t* const        p_desc );\r
316 \r
317 static NDIS_STATUS\r
318 __send_mgr_filter_dhcp(\r
319         IN                              ipoib_port_t* const                     p_port,\r
320         IN              const   udp_hdr_t* const                        p_udp_hdr,\r
321         IN                              NDIS_BUFFER*                            p_buf,\r
322         IN                              size_t                                          buf_len,\r
323         IN      OUT                     ipoib_send_desc_t* const        p_desc );\r
324 \r
325 static NDIS_STATUS\r
326 __send_mgr_filter_arp(\r
327         IN                              ipoib_port_t* const                     p_port,\r
328         IN              const   eth_hdr_t* const                        p_eth_hdr,\r
329         IN                              NDIS_BUFFER*                            p_buf,\r
330         IN                              size_t                                          buf_len,\r
331         IN      OUT                     ipoib_send_desc_t* const        p_desc );\r
332 \r
333 static void\r
334 __process_failed_send(\r
335         IN                              ipoib_port_t* const                     p_port,\r
336         IN                              ipoib_send_desc_t* const        p_desc,\r
337         IN              const   NDIS_STATUS                                     status );\r
338 \r
339 static void\r
340 __send_cb(\r
341         IN              const   ib_cq_handle_t                          h_cq,\r
342         IN                              void                                            *cq_context );\r
343 \r
344 static NDIS_STATUS GetLsoHeaderSize(\r
345         IN      ipoib_port_t* const pPort,\r
346         IN      PNDIS_BUFFER  CurrBuffer,\r
347         IN      LsoData *pLsoData,\r
348         OUT     uint16_t *pSize,\r
349         OUT     INT  *IndexOfData,\r
350         IN              ipoib_hdr_t *ipoib_hdr\r
351         );\r
352 \r
353 /******************************************************************************\r
354 *\r
355 * Endpoint manager operations\r
356 *\r
357 ******************************************************************************/\r
358 static void\r
359 __endpt_mgr_construct(\r
360         IN                              ipoib_port_t* const                     p_port );\r
361 \r
362 static ib_api_status_t\r
363 __endpt_mgr_init(\r
364         IN                              ipoib_port_t* const                     p_port );\r
365 \r
366 static void\r
367 __endpt_mgr_destroy(\r
368         IN                              ipoib_port_t* const                     p_port );\r
369 \r
370 /****f* IPoIB/__endpt_mgr_remove_all\r
371 * NAME\r
372 *       __endpt_mgr_remove_all\r
373 *\r
374 * DESCRIPTION\r
375 *       Removes all enpoints from the port, dereferencing them to initiate\r
376 *       destruction.\r
377 *\r
378 * SYNOPSIS\r
379 */\r
380 static void\r
381 __endpt_mgr_remove_all(\r
382         IN                              ipoib_port_t* const                     p_port );\r
383 /*\r
384 ********/\r
385 \r
386 static void\r
387 __endpt_mgr_remove(\r
388         IN                              ipoib_port_t* const                     p_port,\r
389         IN                              ipoib_endpt_t* const            p_endpt );\r
390 \r
391 static void\r
392 __endpt_mgr_reset_all(\r
393         IN                              ipoib_port_t* const                     p_port );\r
394 \r
395 static inline NDIS_STATUS\r
396 __endpt_mgr_ref(\r
397         IN                              ipoib_port_t* const                     p_port,\r
398         IN              const   mac_addr_t                                      mac,\r
399                 OUT                     ipoib_endpt_t** const           pp_endpt );\r
400 \r
401 static inline NDIS_STATUS\r
402 __endpt_mgr_get_gid_qpn(\r
403         IN                              ipoib_port_t* const                     p_port,\r
404         IN              const   mac_addr_t                                      mac,\r
405                 OUT                     ib_gid_t* const                         p_gid,\r
406                 OUT                     UNALIGNED net32_t* const        p_qpn );\r
407 \r
408 static inline ipoib_endpt_t*\r
409 __endpt_mgr_get_by_gid(\r
410         IN                              ipoib_port_t* const                     p_port,\r
411         IN              const   ib_gid_t* const                         p_gid );\r
412 \r
413 static inline ipoib_endpt_t*\r
414 __endpt_mgr_get_by_lid(\r
415         IN                              ipoib_port_t* const                     p_port,\r
416         IN              const   net16_t                                         lid );\r
417 \r
418 static inline ib_api_status_t\r
419 __endpt_mgr_insert_locked(\r
420         IN                              ipoib_port_t* const                     p_port,\r
421         IN              const   mac_addr_t                                      mac,\r
422         IN                              ipoib_endpt_t* const            p_endpt );\r
423 \r
424 static inline ib_api_status_t\r
425 __endpt_mgr_insert(\r
426         IN                              ipoib_port_t* const                     p_port,\r
427         IN              const   mac_addr_t                                      mac,\r
428         IN                              ipoib_endpt_t* const            p_endpt );\r
429 \r
430 static ib_api_status_t\r
431 __endpt_mgr_add_local(\r
432         IN                              ipoib_port_t* const                     p_port,\r
433         IN                              ib_port_info_t* const           p_port_info );\r
434 \r
435 static ib_api_status_t\r
436 __endpt_mgr_add_bcast(\r
437         IN                              ipoib_port_t* const                     p_port,\r
438         IN                              ib_mcast_rec_t                          *p_mcast_rec );\r
439 \r
440 /******************************************************************************\r
441 *\r
442 * MCast operations.\r
443 *\r
444 ******************************************************************************/\r
445 static ib_api_status_t\r
446 __port_get_bcast(\r
447         IN                              ipoib_port_t* const                     p_port );\r
448 \r
449 static ib_api_status_t\r
450 __port_join_bcast(\r
451         IN                              ipoib_port_t* const                     p_port,\r
452         IN                              ib_member_rec_t* const          p_member_rec );\r
453 \r
454 static ib_api_status_t\r
455 __port_create_bcast(\r
456         IN                              ipoib_port_t* const                     p_port );\r
457 \r
458 \r
459 \r
460 static void\r
461 __bcast_get_cb(\r
462         IN                              ib_query_rec_t                          *p_query_rec );\r
463 \r
464 \r
465 static void\r
466 __bcast_cb(\r
467         IN                              ib_mcast_rec_t                          *p_mcast_rec );\r
468 \r
469 \r
470 static void\r
471 __mcast_cb(\r
472         IN                              ib_mcast_rec_t                          *p_mcast_rec );\r
473 \r
474 void\r
475 __leave_error_mcast_cb(\r
476         IN                              void                            *context );\r
477 \r
478 \r
479 static intn_t\r
480 __gid_cmp(\r
481         IN              const   void* const                                     p_key1,\r
482         IN              const   void* const                                     p_key2 )\r
483 {\r
484         return cl_memcmp( p_key1, p_key2, sizeof(ib_gid_t) );\r
485 }\r
486 \r
487 \r
488 inline void ipoib_port_ref( ipoib_port_t * p_port, int type )\r
489 {\r
490         cl_obj_ref( &p_port->obj );\r
491 #if DBG\r
492         cl_atomic_inc( &p_port->ref[type % ref_mask] );\r
493         IPOIB_PRINT( TRACE_LEVEL_INFORMATION, IPOIB_DBG_OBJ,\r
494                 ("ref type %d ref_cnt %d\n", type, p_port->obj.ref_cnt) );\r
495 #else\r
496         UNREFERENCED_PARAMETER(type);\r
497 #endif\r
498 }\r
499 \r
500 \r
501 inline void ipoib_port_deref(ipoib_port_t * p_port, int type)\r
502 {\r
503 #if DBG\r
504         cl_atomic_dec( &p_port->ref[type % ref_mask] );\r
505         IPOIB_PRINT( TRACE_LEVEL_INFORMATION, IPOIB_DBG_OBJ,\r
506                 ("deref type %d ref_cnt %d\n", type, p_port->obj.ref_cnt) );\r
507 #else\r
508         UNREFERENCED_PARAMETER(type);\r
509 #endif\r
510         cl_obj_deref( &p_port->obj );\r
511 \r
512 }\r
513 \r
514 /* function returns pointer to payload that is going after IP header.\r
515 *  asssuming that payload and IP header are in the same buffer\r
516 */\r
517 static void* GetIpPayloadPtr(const      ip_hdr_t* const p_ip_hdr)\r
518 {\r
519         return (void*)((uint8_t*)p_ip_hdr + 4*(p_ip_hdr->ver_hl & 0xf));\r
520 }\r
521 \r
522 /******************************************************************************\r
523 *\r
524 * Implementation\r
525 *\r
526 ******************************************************************************/\r
527 ib_api_status_t\r
528 ipoib_create_port(\r
529         IN                              ipoib_adapter_t* const          p_adapter,\r
530         IN                              ib_pnp_port_rec_t* const        p_pnp_rec,\r
531                 OUT                     ipoib_port_t** const            pp_port )\r
532 {\r
533         ib_api_status_t         status;\r
534         ipoib_port_t            *p_port;\r
535 \r
536         IPOIB_ENTER( IPOIB_DBG_INIT );\r
537 \r
538         CL_ASSERT( !p_adapter->p_port );\r
539 \r
540         p_port = cl_zalloc( sizeof(ipoib_port_t) +\r
541                 (sizeof(ipoib_hdr_t) * (p_adapter->params.sq_depth - 1)) );\r
542         if( !p_port )\r
543         {\r
544                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
545                         ("Failed to allocate ipoib_port_t (%d bytes)\n",\r
546                         sizeof(ipoib_port_t)) );\r
547                 return IB_INSUFFICIENT_MEMORY;\r
548         }\r
549 \r
550 #ifdef _DEBUG_\r
551         gp_ipoib_port = p_port;\r
552 #endif\r
553 \r
554         __port_construct( p_port );\r
555 \r
556         status = __port_init( p_port, p_adapter, p_pnp_rec );\r
557         if( status != IB_SUCCESS )\r
558         {\r
559                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
560                         ("ipoib_port_init returned %s.\n", \r
561                         p_adapter->p_ifc->get_err_str( status )) );\r
562                 __port_cleanup( &p_port->obj );\r
563                 __port_free( &p_port->obj );\r
564                 return status;\r
565         }\r
566 \r
567         *pp_port = p_port;\r
568         IPOIB_EXIT( IPOIB_DBG_INIT );\r
569         return IB_SUCCESS;\r
570 }\r
571 \r
572 \r
573 void\r
574 ipoib_port_destroy(\r
575         IN                              ipoib_port_t* const                     p_port )\r
576 {\r
577         IPOIB_ENTER( IPOIB_DBG_INIT );\r
578 \r
579         CL_ASSERT( p_port );\r
580         CL_ASSERT( p_port->p_adapter );\r
581         CL_ASSERT( !p_port->p_adapter->p_port );\r
582 \r
583         cl_obj_destroy( &p_port->obj );\r
584 \r
585         IPOIB_EXIT( IPOIB_DBG_INIT );\r
586 }\r
587 \r
588 \r
589 static void\r
590 __port_construct(\r
591         IN                              ipoib_port_t* const                     p_port )\r
592 {\r
593         IPOIB_ENTER( IPOIB_DBG_INIT );\r
594 \r
595         p_port->state = IB_QPS_RESET;\r
596 \r
597         cl_obj_construct( &p_port->obj, IPOIB_OBJ_PORT );\r
598         cl_spinlock_construct( &p_port->send_lock );\r
599         cl_spinlock_construct( &p_port->recv_lock );\r
600         __ib_mgr_construct( p_port );\r
601         __buf_mgr_construct( p_port );\r
602 \r
603         __recv_mgr_construct( p_port );\r
604         __send_mgr_construct( p_port );\r
605 \r
606         __endpt_mgr_construct( p_port );\r
607 \r
608         KeInitializeEvent( &p_port->sa_event, NotificationEvent, TRUE );\r
609         KeInitializeEvent( &p_port->leave_mcast_event, NotificationEvent, TRUE );\r
610         \r
611         IPOIB_EXIT( IPOIB_DBG_INIT );\r
612 }\r
613 \r
614 \r
615 static ib_api_status_t\r
616 __port_init(\r
617         IN                              ipoib_port_t* const                     p_port,\r
618         IN                              ipoib_adapter_t* const          p_adapter,\r
619         IN                              ib_pnp_port_rec_t* const        p_pnp_rec )\r
620 {\r
621         cl_status_t                     cl_status;\r
622         ib_api_status_t         status;\r
623 \r
624         IPOIB_ENTER( IPOIB_DBG_INIT );\r
625 \r
626         p_port->port_num = p_pnp_rec->p_port_attr->port_num;\r
627         p_port->p_adapter = p_adapter;\r
628 \r
629         cl_status = cl_spinlock_init( &p_port->send_lock );\r
630         if( cl_status != CL_SUCCESS )\r
631         {\r
632                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
633                         ("cl_spinlock_init returned %#x\n", cl_status) );\r
634                 return IB_ERROR;\r
635         }\r
636 \r
637         cl_status = cl_spinlock_init( &p_port->recv_lock );\r
638         if( cl_status != CL_SUCCESS )\r
639         {\r
640                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
641                         ("cl_spinlock_init returned %#x\n", cl_status) );\r
642                 return IB_ERROR;\r
643         }\r
644 \r
645         /* Initialize the IB resource manager. */\r
646         status = __ib_mgr_init( p_port );\r
647         if( status != IB_SUCCESS )\r
648         {\r
649                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
650                         ("__ib_mgr_init returned %s\n", \r
651                         p_adapter->p_ifc->get_err_str( status )) );\r
652                 return status;\r
653         }\r
654 \r
655         /* Initialize the buffer manager. */\r
656         status = __buf_mgr_init( p_port );\r
657         if( status != IB_SUCCESS )\r
658         {\r
659                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
660                         ("__buf_mgr_init returned %s\n", \r
661                         p_adapter->p_ifc->get_err_str( status )) );\r
662                 return status;\r
663         }\r
664 \r
665         /* Initialize the receive manager. */\r
666         status = __recv_mgr_init( p_port );\r
667         if( status != IB_SUCCESS )\r
668         {\r
669                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
670                         ("__recv_mgr_init returned %s\n", \r
671                         p_adapter->p_ifc->get_err_str( status )) );\r
672                 return status;\r
673         }\r
674 \r
675         /* Initialize the endpoint manager. */\r
676         status = __endpt_mgr_init( p_port );\r
677         if( status != IB_SUCCESS )\r
678         {\r
679                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
680                         ("__endpt_mgr_init returned %s\n", \r
681                         p_adapter->p_ifc->get_err_str( status )) );\r
682                 return status;\r
683         }\r
684 \r
685          KeInitializeDpc(&p_port->recv_dpc,(PKDEFERRED_ROUTINE)__recv_cb_dpc,p_port);\r
686 \r
687 \r
688          /* Initialize multicast garbage collector timer and DPC object */\r
689          KeInitializeDpc(&p_port->gc_dpc,(PKDEFERRED_ROUTINE)__port_mcast_garbage_dpc,p_port);\r
690          KeInitializeTimerEx(&p_port->gc_timer,SynchronizationTimer);\r
691 \r
692         /* We only ever destroy from the PnP callback thread. */\r
693         cl_status = cl_obj_init( &p_port->obj, CL_DESTROY_SYNC,\r
694                 __port_destroying, __port_cleanup, __port_free );\r
695 \r
696 #if DBG\r
697         cl_atomic_inc( &p_port->ref[ref_init] );\r
698         IPOIB_PRINT( TRACE_LEVEL_INFORMATION, IPOIB_DBG_OBJ,\r
699                 ("ref type %d ref_cnt %d\n", ref_init, p_port->obj.ref_cnt) );\r
700 #endif\r
701 \r
702         if( cl_status != CL_SUCCESS )\r
703         {\r
704                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
705                         ("cl_obj_init returned %#x\n", cl_status) );\r
706                 return IB_ERROR;\r
707         }\r
708 \r
709         cl_status = cl_obj_insert_rel( &p_port->rel, &p_adapter->obj, &p_port->obj );\r
710         if( cl_status != CL_SUCCESS )\r
711         {\r
712                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
713                         ("cl_obj_insert_rel returned %#x\n", cl_status) );\r
714                 cl_obj_destroy( &p_port->obj );\r
715                 return IB_ERROR;\r
716         }\r
717 \r
718 #if DBG\r
719         cl_atomic_inc( &p_port->ref[ref_init] );\r
720         IPOIB_PRINT( TRACE_LEVEL_ERROR, IPOIB_DBG_OBJ,\r
721                 ("ref type %d ref_cnt %d\n", ref_init, p_port->obj.ref_cnt) );\r
722 #endif\r
723 \r
724         IPOIB_EXIT( IPOIB_DBG_INIT );\r
725         return IB_SUCCESS;\r
726 }\r
727 \r
728 \r
729 static void\r
730 __port_destroying(\r
731         IN                              cl_obj_t* const                         p_obj )\r
732 {\r
733         ipoib_port_t    *p_port;\r
734 \r
735         IPOIB_ENTER( IPOIB_DBG_INIT );\r
736 \r
737         CL_ASSERT( p_obj );\r
738 \r
739         p_port = PARENT_STRUCT( p_obj, ipoib_port_t, obj );\r
740 \r
741         ipoib_port_down( p_port );\r
742 \r
743         __endpt_mgr_remove_all( p_port );\r
744 \r
745         ipoib_port_resume( p_port );\r
746 \r
747         IPOIB_EXIT( IPOIB_DBG_INIT );\r
748 }\r
749 \r
750 \r
751 static void\r
752 __port_cleanup(\r
753         IN                              cl_obj_t* const                         p_obj )\r
754 {\r
755         ipoib_port_t    *p_port;\r
756 \r
757         IPOIB_ENTER( IPOIB_DBG_INIT );\r
758 \r
759         CL_ASSERT( p_obj );\r
760 \r
761         p_port = PARENT_STRUCT( p_obj, ipoib_port_t, obj );\r
762 \r
763         /* Wait for all sends and receives to get flushed. */\r
764         while( p_port->send_mgr.depth || p_port->recv_mgr.depth )\r
765                 cl_thread_suspend( 0 );\r
766 \r
767         /* Destroy the send and receive managers before closing the CA. */\r
768         __ib_mgr_destroy( p_port );\r
769 \r
770         IPOIB_EXIT( IPOIB_DBG_INIT );\r
771 }\r
772 \r
773 \r
774 static void\r
775 __port_free(\r
776         IN                              cl_obj_t* const                         p_obj )\r
777 {\r
778         ipoib_port_t    *p_port;\r
779 \r
780         IPOIB_ENTER( IPOIB_DBG_INIT );\r
781 \r
782         CL_ASSERT( p_obj );\r
783 \r
784         p_port = PARENT_STRUCT( p_obj, ipoib_port_t, obj );\r
785 \r
786         KeCancelTimer(&p_port->gc_timer);\r
787         KeFlushQueuedDpcs();\r
788         __endpt_mgr_destroy( p_port );\r
789         __recv_mgr_destroy( p_port );\r
790         __send_mgr_destroy( p_port );\r
791         __buf_mgr_destroy( p_port );\r
792 \r
793         cl_spinlock_destroy( &p_port->send_lock );\r
794         cl_spinlock_destroy( &p_port->recv_lock );\r
795 \r
796         cl_obj_deinit( p_obj );\r
797 \r
798         cl_free( p_port );\r
799 \r
800         IPOIB_EXIT( IPOIB_DBG_INIT );\r
801 }\r
802 \r
803 \r
804 \r
805 /******************************************************************************\r
806 *\r
807 * IB resource manager implementation.\r
808 *\r
809 ******************************************************************************/\r
810 static void\r
811 __ib_mgr_construct(\r
812         IN                              ipoib_port_t* const                     p_port )\r
813 {\r
814         IPOIB_ENTER( IPOIB_DBG_INIT );\r
815 \r
816         cl_memclr( &p_port->ib_mgr, sizeof(ipoib_ib_mgr_t) );\r
817 \r
818         IPOIB_EXIT( IPOIB_DBG_INIT );\r
819 }\r
820 \r
821 \r
822 static ib_api_status_t\r
823 __ib_mgr_init(\r
824         IN                              ipoib_port_t* const                     p_port )\r
825 {\r
826         ib_api_status_t         status;\r
827         ib_cq_create_t          cq_create;\r
828         ib_qp_create_t          qp_create;\r
829         ib_phys_create_t        phys_create;\r
830         ib_phys_range_t         phys_range;\r
831         uint64_t                        vaddr;\r
832         net32_t                         rkey;\r
833         ib_qp_attr_t            qp_attr;\r
834         ib_ca_attr_t *          p_ca_attr;\r
835         uint32_t                        attr_size;\r
836 \r
837         IPOIB_ENTER( IPOIB_DBG_INIT );\r
838 \r
839         /* Open the CA. */\r
840         status = p_port->p_adapter->p_ifc->open_ca(\r
841                 p_port->p_adapter->h_al, p_port->p_adapter->guids.ca_guid,\r
842                 NULL, p_port, &p_port->ib_mgr.h_ca );\r
843         if( status != IB_SUCCESS )\r
844         {\r
845                 NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
846                         EVENT_IPOIB_OPEN_CA, 1, status );\r
847                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
848                         ("ib_open_ca returned %s\n", \r
849                         p_port->p_adapter->p_ifc->get_err_str( status )) );\r
850                 return status;\r
851         }\r
852 \r
853         /* Allocate the PD. */\r
854         status = p_port->p_adapter->p_ifc->alloc_pd(\r
855                 p_port->ib_mgr.h_ca, IB_PDT_UD, p_port, &p_port->ib_mgr.h_pd );\r
856         if( status != IB_SUCCESS )\r
857         {\r
858                 NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
859                         EVENT_IPOIB_ALLOC_PD, 1, status );\r
860                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
861                         ("ib_alloc_pd returned %s\n", \r
862                         p_port->p_adapter->p_ifc->get_err_str( status )) );\r
863                 return status;\r
864         }\r
865 \r
866         /* Allocate receive CQ. */\r
867         cq_create.size = p_port->p_adapter->params.rq_depth;\r
868         cq_create.pfn_comp_cb = __recv_cb;\r
869         cq_create.h_wait_obj = NULL;\r
870 \r
871         status = p_port->p_adapter->p_ifc->create_cq(\r
872                 p_port->ib_mgr.h_ca, &cq_create, p_port,\r
873                 __cq_event, &p_port->ib_mgr.h_recv_cq );\r
874         if( status != IB_SUCCESS )\r
875         {\r
876                 NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
877                         EVENT_IPOIB_CREATE_RECV_CQ, 1, status );\r
878                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
879                         ("ib_create_cq returned %s.\n", \r
880                         p_port->p_adapter->p_ifc->get_err_str( status )) );\r
881                 return status;\r
882         }\r
883 \r
884         /* Allocate send CQ. */\r
885         cq_create.size = p_port->p_adapter->params.sq_depth;\r
886         cq_create.pfn_comp_cb = __send_cb;\r
887 \r
888         status = p_port->p_adapter->p_ifc->create_cq(\r
889                 p_port->ib_mgr.h_ca, &cq_create, p_port,\r
890                 __cq_event, &p_port->ib_mgr.h_send_cq );\r
891         if( status != IB_SUCCESS )\r
892         {\r
893                 NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
894                         EVENT_IPOIB_CREATE_SEND_CQ, 1, status );\r
895                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
896                         ("ib_create_cq returned %s.\n", \r
897                         p_port->p_adapter->p_ifc->get_err_str( status )) );\r
898                 return status;\r
899         }\r
900         \r
901         /* Allocate the QP. */\r
902         cl_memclr( &qp_create, sizeof(qp_create) );\r
903         qp_create.qp_type = IB_QPT_UNRELIABLE_DGRM;\r
904         qp_create.rq_depth = p_port->p_adapter->params.rq_depth;\r
905         qp_create.rq_sge = 2;   /* To support buffers spanning pages. */\r
906         qp_create.h_rq_cq = p_port->ib_mgr.h_recv_cq;\r
907         qp_create.sq_depth = p_port->p_adapter->params.sq_depth;\r
908         \r
909         //Figure out the right number of SGE entries for sends.\r
910         /* Get the size of the CA attribute structure. */\r
911         status = p_port->p_adapter->p_ifc->query_ca( p_port->ib_mgr.h_ca, NULL, &attr_size );\r
912         if( status != IB_INSUFFICIENT_MEMORY )\r
913         {\r
914                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
915                         ("ib_query_ca failed with status %s.\n", p_port->p_adapter->p_ifc->get_err_str(status)) );\r
916                 return status;\r
917         }\r
918 \r
919         /* Allocate enough space to store the attribute structure. */\r
920         p_ca_attr = cl_malloc( attr_size );\r
921         if( !p_ca_attr )\r
922         {\r
923                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
924                         ("cl_malloc failed to allocate p_ca_attr!\n") );\r
925                 return IB_INSUFFICIENT_RESOURCES;\r
926         }\r
927 \r
928         /* Query the CA attributes. */\r
929         status = p_port->p_adapter->p_ifc->query_ca(p_port->ib_mgr.h_ca, p_ca_attr, &attr_size );\r
930         if( status != IB_SUCCESS )\r
931         {\r
932                 cl_free( p_ca_attr );\r
933 \r
934                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
935                         ("ib_query_ca failed with status %s.\n", p_port->p_adapter->p_ifc->get_err_str(status)) );\r
936                 return status;\r
937         }\r
938 #define UD_QP_USED_SGE 3\r
939         qp_create.sq_sge = MAX_SEND_SGE < p_ca_attr->max_sges ? MAX_SEND_SGE  : (p_ca_attr->max_sges - UD_QP_USED_SGE);\r
940         if (!p_ca_attr->ipoib_csum) { \r
941                 //checksum is not supported by device\r
942                 //user must specify BYPASS to explicitly cancel checksum calculation\r
943                 if (p_port->p_adapter->params.send_chksum_offload == CSUM_ENABLED)\r
944                         p_port->p_adapter->params.send_chksum_offload = CSUM_DISABLED;\r
945                 if (p_port->p_adapter->params.recv_chksum_offload == CSUM_ENABLED)\r
946                         p_port->p_adapter->params.recv_chksum_offload = CSUM_DISABLED;\r
947         }\r
948         cl_free( p_ca_attr );\r
949         \r
950         qp_create.h_sq_cq = p_port->ib_mgr.h_send_cq;\r
951         qp_create.sq_signaled = TRUE;\r
952         status = p_port->p_adapter->p_ifc->create_qp(\r
953                 p_port->ib_mgr.h_pd, &qp_create, p_port,\r
954                 __qp_event, &p_port->ib_mgr.h_qp );\r
955         if( status != IB_SUCCESS )\r
956         {\r
957                 NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
958                         EVENT_IPOIB_CREATE_QP, 1, status );\r
959                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
960                         ("ib_create_qp returned %s\n", \r
961                         p_port->p_adapter->p_ifc->get_err_str( status )) );\r
962                 return status;\r
963         }\r
964         /* Query the QP so we can get our QPN. */\r
965         status = p_port->p_adapter->p_ifc->query_qp(\r
966                 p_port->ib_mgr.h_qp, &qp_attr );\r
967         if( status != IB_SUCCESS )\r
968         {\r
969                 NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
970                         EVENT_IPOIB_QUERY_QP, 1, status );\r
971                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
972                         ("ib_query_qp returned %s\n", \r
973                         p_port->p_adapter->p_ifc->get_err_str( status )) );\r
974                 return status;\r
975         }\r
976         p_port->ib_mgr.qpn = qp_attr.num;\r
977 \r
978         /* Register all of physical memory */\r
979         phys_create.length = MEM_REG_SIZE;\r
980         phys_create.num_ranges = 1;\r
981         phys_create.range_array = &phys_range;\r
982         phys_create.buf_offset = 0;\r
983         phys_create.hca_page_size = PAGE_SIZE;\r
984         phys_create.access_ctrl = IB_AC_LOCAL_WRITE;\r
985         phys_range.base_addr = 0;\r
986         phys_range.size = MEM_REG_SIZE;\r
987         vaddr = 0;\r
988         status = p_port->p_adapter->p_ifc->reg_phys(\r
989                 p_port->ib_mgr.h_pd, &phys_create, &vaddr,\r
990                 &p_port->ib_mgr.lkey, &rkey, &p_port->ib_mgr.h_mr );\r
991         if( status != IB_SUCCESS )\r
992         {\r
993                 NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
994                         EVENT_IPOIB_REG_PHYS, 1, status );\r
995                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
996                         ("ib_reg_phys returned %s\n", \r
997                         p_port->p_adapter->p_ifc->get_err_str( status )) );\r
998                 return status;\r
999         }\r
1000 \r
1001         IPOIB_EXIT( IPOIB_DBG_INIT );\r
1002         return IB_SUCCESS;\r
1003 }\r
1004 \r
1005 \r
1006 static void\r
1007 __ib_mgr_destroy(\r
1008         IN                              ipoib_port_t* const                     p_port )\r
1009 {\r
1010         ib_api_status_t status;\r
1011 \r
1012         IPOIB_ENTER( IPOIB_DBG_INIT );\r
1013 \r
1014         if( p_port->ib_mgr.h_ca )\r
1015         {\r
1016                 status =\r
1017                         p_port->p_adapter->p_ifc->close_ca( p_port->ib_mgr.h_ca, NULL );\r
1018                 CL_ASSERT( status == IB_SUCCESS );\r
1019                 p_port->ib_mgr.h_ca = NULL;\r
1020         }\r
1021 \r
1022         IPOIB_EXIT( IPOIB_DBG_INIT );\r
1023 }\r
1024 \r
1025 \r
1026 \r
1027 /******************************************************************************\r
1028 *\r
1029 * Buffer manager implementation.\r
1030 *\r
1031 ******************************************************************************/\r
1032 static void\r
1033 __buf_mgr_construct(\r
1034         IN                              ipoib_port_t* const                     p_port )\r
1035 {\r
1036         IPOIB_ENTER( IPOIB_DBG_INIT );\r
1037 \r
1038         cl_qpool_construct( &p_port->buf_mgr.recv_pool );\r
1039 \r
1040         p_port->buf_mgr.h_packet_pool = NULL;\r
1041         p_port->buf_mgr.h_buffer_pool = NULL;\r
1042 \r
1043         ExInitializeNPagedLookasideList( &p_port->buf_mgr.send_buf_list,\r
1044                 NULL, NULL, 0, MAX_XFER_BLOCK_SIZE, 'bipi', 0 );\r
1045 \r
1046         p_port->buf_mgr.h_send_pkt_pool = NULL;\r
1047         p_port->buf_mgr.h_send_buf_pool = NULL;\r
1048 \r
1049         IPOIB_EXIT( IPOIB_DBG_INIT );\r
1050 }\r
1051 \r
1052 \r
1053 static ib_api_status_t\r
1054 __buf_mgr_init(\r
1055         IN                              ipoib_port_t* const                     p_port )\r
1056 {\r
1057         cl_status_t             cl_status;\r
1058         NDIS_STATUS             ndis_status;\r
1059         ipoib_params_t  *p_params;\r
1060 \r
1061         IPOIB_ENTER(IPOIB_DBG_INIT );\r
1062 \r
1063         CL_ASSERT( p_port );\r
1064         CL_ASSERT( p_port->p_adapter );\r
1065 \r
1066         p_params = &p_port->p_adapter->params;\r
1067 \r
1068         /* Allocate the receive descriptor pool */\r
1069         cl_status = cl_qpool_init( &p_port->buf_mgr.recv_pool,\r
1070                 p_params->rq_depth * p_params->recv_pool_ratio,\r
1071 #if IPOIB_INLINE_RECV\r
1072                 0, 0, sizeof(ipoib_recv_desc_t), __recv_ctor, NULL, p_port );\r
1073 #else   /* IPOIB_INLINE_RECV */\r
1074                 0, 0, sizeof(ipoib_recv_desc_t), __recv_ctor, __recv_dtor, p_port );\r
1075 #endif  /* IPOIB_INLINE_RECV */\r
1076         if( cl_status != CL_SUCCESS )\r
1077         {\r
1078                 NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
1079                         EVENT_IPOIB_RECV_POOL, 1, cl_status );\r
1080                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1081                         ("cl_qpool_init for recvs returned %#x\n",\r
1082                         cl_status) );\r
1083                 return IB_INSUFFICIENT_MEMORY;\r
1084         }\r
1085 \r
1086         /* Allocate the NDIS buffer and packet pools for receive indication. */\r
1087         NdisAllocatePacketPool( &ndis_status, &p_port->buf_mgr.h_packet_pool,\r
1088                 p_params->rq_depth, PROTOCOL_RESERVED_SIZE_IN_PACKET );\r
1089         if( ndis_status != NDIS_STATUS_SUCCESS )\r
1090         {\r
1091                 NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
1092                         EVENT_IPOIB_RECV_PKT_POOL, 1, ndis_status );\r
1093                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1094                         ("NdisAllocatePacketPool returned %08X\n", ndis_status) );\r
1095                 return IB_INSUFFICIENT_RESOURCES;\r
1096         }\r
1097 \r
1098         NdisAllocateBufferPool( &ndis_status, &p_port->buf_mgr.h_buffer_pool,\r
1099                 p_params->rq_depth );\r
1100         if( ndis_status != NDIS_STATUS_SUCCESS )\r
1101         {\r
1102                 NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
1103                         EVENT_IPOIB_RECV_BUF_POOL, 1, ndis_status );\r
1104                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1105                         ("NdisAllocateBufferPool returned %08X\n", ndis_status) );\r
1106                 return IB_INSUFFICIENT_RESOURCES;\r
1107         }\r
1108 \r
1109         /* Allocate the NDIS buffer and packet pools for send formatting. */\r
1110         NdisAllocatePacketPool( &ndis_status, &p_port->buf_mgr.h_send_pkt_pool,\r
1111                 1, PROTOCOL_RESERVED_SIZE_IN_PACKET );\r
1112         if( ndis_status != NDIS_STATUS_SUCCESS )\r
1113         {\r
1114                 NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
1115                         EVENT_IPOIB_SEND_PKT_POOL, 1, ndis_status );\r
1116                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1117                         ("NdisAllocatePacketPool returned %08X\n", ndis_status) );\r
1118                 return IB_INSUFFICIENT_RESOURCES;\r
1119         }\r
1120 \r
1121         NdisAllocateBufferPool( &ndis_status,\r
1122                 &p_port->buf_mgr.h_send_buf_pool, 1 );\r
1123         if( ndis_status != NDIS_STATUS_SUCCESS )\r
1124         {\r
1125                 NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
1126                         EVENT_IPOIB_SEND_BUF_POOL, 1, ndis_status );\r
1127                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1128                         ("NdisAllocateBufferPool returned %08X\n", ndis_status) );\r
1129                 return IB_INSUFFICIENT_RESOURCES;\r
1130         }\r
1131 \r
1132         IPOIB_EXIT( IPOIB_DBG_INIT );\r
1133         return IB_SUCCESS;\r
1134 }\r
1135 \r
1136 \r
1137 static void\r
1138 __buf_mgr_destroy(\r
1139         IN                              ipoib_port_t* const                     p_port )\r
1140 {\r
1141         IPOIB_ENTER(IPOIB_DBG_INIT );\r
1142 \r
1143         CL_ASSERT( p_port );\r
1144 \r
1145         /* Destroy the send packet and buffer pools. */\r
1146         if( p_port->buf_mgr.h_send_buf_pool )\r
1147                 NdisFreeBufferPool( p_port->buf_mgr.h_send_buf_pool );\r
1148         if( p_port->buf_mgr.h_send_pkt_pool )\r
1149                 NdisFreePacketPool( p_port->buf_mgr.h_send_pkt_pool );\r
1150 \r
1151         /* Destroy the receive packet and buffer pools. */\r
1152         if( p_port->buf_mgr.h_buffer_pool )\r
1153                 NdisFreeBufferPool( p_port->buf_mgr.h_buffer_pool );\r
1154         if( p_port->buf_mgr.h_packet_pool )\r
1155                 NdisFreePacketPool( p_port->buf_mgr.h_packet_pool );\r
1156 \r
1157         /* Free the receive and send descriptors. */\r
1158         cl_qpool_destroy( &p_port->buf_mgr.recv_pool );\r
1159 \r
1160         /* Free the lookaside list of scratch buffers. */\r
1161         ExDeleteNPagedLookasideList( &p_port->buf_mgr.send_buf_list );\r
1162 \r
1163         IPOIB_EXIT(  IPOIB_DBG_INIT );\r
1164 }\r
1165 \r
1166 \r
1167 static cl_status_t\r
1168 __recv_ctor(\r
1169         IN                              void* const                                     p_object,\r
1170         IN                              void*                                           context,\r
1171                 OUT                     cl_pool_item_t** const          pp_pool_item )\r
1172 {\r
1173         ipoib_recv_desc_t       *p_desc;\r
1174         ipoib_port_t            *p_port;\r
1175         uint32_t                        ds0_len;\r
1176 \r
1177         IPOIB_ENTER( IPOIB_DBG_ALLOC );\r
1178 \r
1179         CL_ASSERT( p_object );\r
1180         CL_ASSERT( context );\r
1181 \r
1182         p_desc = (ipoib_recv_desc_t*)p_object;\r
1183         p_port = (ipoib_port_t*)context;\r
1184 \r
1185         /* Setup the work request. */\r
1186         p_desc->wr.ds_array = p_desc->local_ds;\r
1187         p_desc->wr.wr_id = (uintn_t)p_desc;\r
1188 \r
1189 #if IPOIB_INLINE_RECV\r
1190         /* Sanity check on the receive buffer layout */\r
1191         CL_ASSERT( (void*)&p_desc->buf.eth.pkt.type ==\r
1192                 (void*)&p_desc->buf.ib.pkt.type );\r
1193         CL_ASSERT( sizeof(recv_buf_t) == sizeof(ipoib_pkt_t) + sizeof(ib_grh_t) );\r
1194 \r
1195         /* Setup the local data segment. */\r
1196         p_desc->local_ds[0].vaddr = cl_get_physaddr( &p_desc->buf );\r
1197         p_desc->local_ds[0].lkey = p_port->ib_mgr.lkey;\r
1198         ds0_len =\r
1199                 PAGE_SIZE - ((uint32_t)p_desc->local_ds[0].vaddr & (PAGE_SIZE - 1));\r
1200         if( ds0_len >= sizeof(recv_buf_t) )\r
1201         {\r
1202                 /* The whole buffer is within a page. */\r
1203                 p_desc->local_ds[0].length = ds0_len;\r
1204                 p_desc->wr.num_ds = 1;\r
1205         }\r
1206         else\r
1207         {\r
1208                 /* The buffer crosses page boundaries. */\r
1209                 p_desc->local_ds[0].length = ds0_len;\r
1210                 p_desc->local_ds[1].vaddr = cl_get_physaddr( \r
1211                         ((uint8_t*)&p_desc->buf) + ds0_len );\r
1212                 p_desc->local_ds[1].lkey = p_port->ib_mgr.lkey;\r
1213                 p_desc->local_ds[1].length = sizeof(recv_buf_t) - ds0_len;\r
1214                 p_desc->wr.num_ds = 2;\r
1215         }\r
1216 #else   /* IPOIB_INLINE_RECV */\r
1217         /* Allocate the receive buffer. */\r
1218         p_desc->p_buf = (recv_buf_t*)cl_zalloc( sizeof(recv_buf_t) );\r
1219         if( !p_desc->p_buf )\r
1220         {\r
1221                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1222                         ("Failed to allocate receive buffer.\n") );\r
1223                 return CL_INSUFFICIENT_MEMORY;\r
1224         }\r
1225 \r
1226         /* Sanity check on the receive buffer layout */\r
1227         CL_ASSERT( (void*)&p_desc->p_buf->eth.pkt.type ==\r
1228                 (void*)&p_desc->p_buf->ib.pkt.type );\r
1229 \r
1230         /* Setup the local data segment. */\r
1231         p_desc->local_ds[0].vaddr = cl_get_physaddr( p_desc->p_buf );\r
1232         p_desc->local_ds[0].length = sizeof(ipoib_pkt_t) + sizeof(ib_grh_t);\r
1233         p_desc->local_ds[0].lkey = p_port->ib_mgr.lkey;\r
1234 #endif  /* IPOIB_INLINE_RECV */\r
1235 \r
1236         *pp_pool_item = &p_desc->item;\r
1237 \r
1238         IPOIB_EXIT( IPOIB_DBG_ALLOC );\r
1239         return CL_SUCCESS;\r
1240 }\r
1241 \r
1242 \r
1243 #if !IPOIB_INLINE_RECV\r
1244 static void\r
1245 __recv_dtor(\r
1246         IN              const   cl_pool_item_t* const           p_pool_item,\r
1247         IN                              void                                            *context )\r
1248 {\r
1249         ipoib_recv_desc_t       *p_desc;\r
1250 \r
1251         IPOIB_ENTER(  IPOIB_DBG_ALLOC );\r
1252 \r
1253         UNUSED_PARAM( context );\r
1254 \r
1255         p_desc = PARENT_STRUCT( p_pool_item, ipoib_recv_desc_t, item );\r
1256 \r
1257         if( p_desc->p_buf )\r
1258                 cl_free( p_desc->p_buf );\r
1259 \r
1260         IPOIB_EXIT( IPOIB_DBG_ALLOC );\r
1261 }\r
1262 #endif\r
1263 \r
1264 \r
1265 static inline ipoib_recv_desc_t*\r
1266 __buf_mgr_get_recv(\r
1267         IN                              ipoib_port_t* const                     p_port )\r
1268 {\r
1269         ipoib_recv_desc_t       *p_desc;\r
1270         IPOIB_ENTER( IPOIB_DBG_RECV );\r
1271         p_desc = (ipoib_recv_desc_t*)cl_qpool_get( &p_port->buf_mgr.recv_pool );\r
1272         /* Reference the port object for the send. */\r
1273         if( p_desc )\r
1274         {\r
1275                 ipoib_port_ref( p_port, ref_get_recv );\r
1276                 CL_ASSERT( p_desc->wr.wr_id == (uintn_t)p_desc );\r
1277 #if IPOIB_INLINE_RECV\r
1278                 CL_ASSERT( p_desc->local_ds[0].vaddr ==\r
1279                         cl_get_physaddr( &p_desc->buf ) );\r
1280 #else   /* IPOIB_INLINE_RECV */\r
1281                 CL_ASSERT( p_desc->local_ds[0].vaddr ==\r
1282                         cl_get_physaddr( p_desc->p_buf ) );\r
1283                 CL_ASSERT( p_desc->local_ds[0].length ==\r
1284                         (sizeof(ipoib_pkt_t) + sizeof(ib_grh_t)) );\r
1285 #endif  /* IPOIB_INLINE_RECV */\r
1286                 CL_ASSERT( p_desc->local_ds[0].lkey == p_port->ib_mgr.lkey );\r
1287         }\r
1288         IPOIB_EXIT( IPOIB_DBG_RECV );\r
1289         return p_desc;\r
1290 }\r
1291 \r
1292 \r
1293 static inline void\r
1294 __buf_mgr_put_recv(\r
1295         IN                              ipoib_port_t* const                     p_port,\r
1296         IN                              ipoib_recv_desc_t* const        p_desc,\r
1297         IN                              NDIS_PACKET* const                      p_packet OPTIONAL )\r
1298 {\r
1299         NDIS_BUFFER             *p_buf;\r
1300 \r
1301         IPOIB_ENTER(IPOIB_DBG_RECV );\r
1302 \r
1303         if( p_packet )\r
1304         {\r
1305                 /* Unchain the NDIS buffer. */\r
1306                 NdisUnchainBufferAtFront( p_packet, &p_buf );\r
1307                 CL_ASSERT( p_buf );\r
1308                 /* Return the NDIS packet and NDIS buffer to their pools. */\r
1309                 NdisDprFreePacketNonInterlocked( p_packet );\r
1310                 NdisFreeBuffer( p_buf );\r
1311         }\r
1312 \r
1313         /* Return the descriptor to its pools. */\r
1314         cl_qpool_put( &p_port->buf_mgr.recv_pool, &p_desc->item );\r
1315 \r
1316         /*\r
1317          * Dereference the port object since the receive is no longer outstanding.\r
1318          */\r
1319         ipoib_port_deref( p_port, ref_get_recv );\r
1320         IPOIB_EXIT(  IPOIB_DBG_RECV );\r
1321 }\r
1322 \r
1323 \r
1324 static inline void\r
1325 __buf_mgr_put_recv_list(\r
1326         IN                              ipoib_port_t* const                     p_port,\r
1327         IN                              cl_qlist_t* const                       p_list )\r
1328 {\r
1329         //IPOIB_ENTER(  IPOIB_DBG_RECV );\r
1330         cl_qpool_put_list( &p_port->buf_mgr.recv_pool, p_list );\r
1331         //IPOIB_EXIT(  IPOIB_DBG_RECV );\r
1332 }\r
1333 \r
1334 \r
1335 static inline NDIS_PACKET*\r
1336 __buf_mgr_get_ndis_pkt(\r
1337         IN                              ipoib_port_t* const                     p_port,\r
1338         IN                              ipoib_recv_desc_t* const        p_desc )\r
1339 {\r
1340         NDIS_STATUS                             status;\r
1341         NDIS_PACKET                             *p_packet;\r
1342         NDIS_BUFFER                             *p_buffer;\r
1343 \r
1344         IPOIB_ENTER(  IPOIB_DBG_RECV );\r
1345 \r
1346         NdisDprAllocatePacketNonInterlocked( &status, &p_packet,\r
1347                 p_port->buf_mgr.h_packet_pool );\r
1348         if( status != NDIS_STATUS_SUCCESS )\r
1349         {\r
1350                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1351                         ("Failed to allocate NDIS_PACKET: %08x\n", status) );\r
1352                 return NULL;\r
1353         }\r
1354 \r
1355         IPOIB_PORT_FROM_PACKET( p_packet ) = p_port;\r
1356         IPOIB_RECV_FROM_PACKET( p_packet ) = p_desc;\r
1357 \r
1358         NdisAllocateBuffer( &status, &p_buffer,\r
1359 #if IPOIB_INLINE_RECV\r
1360                 p_port->buf_mgr.h_buffer_pool, &p_desc->buf.eth.pkt, p_desc->len );\r
1361 #else   /* IPOIB_INLINE_RECV */\r
1362                 p_port->buf_mgr.h_buffer_pool, &p_desc->p_buf->eth.pkt, p_desc->len );\r
1363 #endif  /* IPOIB_INLINE_RECV */\r
1364         if( status != NDIS_STATUS_SUCCESS )\r
1365         {\r
1366                 IPOIB_PRINT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1367                         ("Failed to allocate NDIS_BUFFER: %08x\n", status) );\r
1368                 NdisDprFreePacketNonInterlocked( p_packet );\r
1369                 return NULL;\r
1370         }\r
1371 \r
1372         NdisChainBufferAtFront( p_packet, p_buffer );\r
1373         NDIS_SET_PACKET_HEADER_SIZE( p_packet, sizeof(eth_hdr_t) );\r
1374 \r
1375         IPOIB_EXIT(  IPOIB_DBG_RECV );\r
1376         return p_packet;\r
1377 }\r
1378 \r
1379 /******************************************************************************\r
1380 *\r
1381 * Receive manager implementation.\r
1382 *\r
1383 ******************************************************************************/\r
1384 static void\r
1385 __recv_mgr_construct(\r
1386         IN                              ipoib_port_t* const                     p_port )\r
1387 {\r
1388         IPOIB_ENTER( IPOIB_DBG_INIT );\r
1389 \r
1390         cl_qlist_init( &p_port->recv_mgr.done_list );\r
1391 \r
1392         p_port->recv_mgr.recv_pkt_array = NULL;\r
1393 \r
1394         IPOIB_EXIT( IPOIB_DBG_INIT );\r
1395 }\r
1396 \r
1397 \r
1398 static ib_api_status_t\r
1399 __recv_mgr_init(\r
1400         IN                              ipoib_port_t* const                     p_port )\r
1401 {\r
1402         IPOIB_ENTER( IPOIB_DBG_INIT );\r
1403 \r
1404         /* Allocate the NDIS_PACKET pointer array for indicating receives. */\r
1405         p_port->recv_mgr.recv_pkt_array = cl_malloc(\r
1406                 sizeof(NDIS_PACKET*) * p_port->p_adapter->params.rq_depth );\r
1407         if( !p_port->recv_mgr.recv_pkt_array )\r
1408         {\r
1409                 NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
1410                         EVENT_IPOIB_RECV_PKT_ARRAY, 0 );\r
1411                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1412                         ("cl_malloc for PNDIS_PACKET array failed.\n") );\r
1413                 return IB_INSUFFICIENT_MEMORY;\r
1414         }\r
1415 \r
1416         IPOIB_EXIT( IPOIB_DBG_INIT );\r
1417         return IB_SUCCESS;\r
1418 }\r
1419 \r
1420 \r
1421 static void\r
1422 __recv_mgr_destroy(\r
1423         IN                              ipoib_port_t* const                     p_port )\r
1424 {\r
1425         IPOIB_ENTER( IPOIB_DBG_INIT );\r
1426 \r
1427         CL_ASSERT( cl_is_qlist_empty( &p_port->recv_mgr.done_list ) );\r
1428         CL_ASSERT( !p_port->recv_mgr.depth );\r
1429 \r
1430         if( p_port->recv_mgr.recv_pkt_array )\r
1431                 cl_free( p_port->recv_mgr.recv_pkt_array );\r
1432 \r
1433         IPOIB_EXIT( IPOIB_DBG_INIT );\r
1434 }\r
1435 \r
1436 \r
1437 /*\r
1438  * Posts receive buffers to the receive queue and returns the number\r
1439  * of receives needed to bring the RQ to its low water mark.  Note\r
1440  * that the value is signed, and can go negative.  All tests must\r
1441  * be for > 0.\r
1442  */\r
1443 static int32_t\r
1444 __recv_mgr_repost(\r
1445         IN                              ipoib_port_t* const                     p_port )\r
1446 {\r
1447         ipoib_recv_desc_t       *p_head = NULL, *p_tail = NULL, *p_next;\r
1448         ib_api_status_t         status;\r
1449         ib_recv_wr_t            *p_failed;\r
1450         PERF_DECLARE( GetRecv );\r
1451         PERF_DECLARE( PostRecv );\r
1452 \r
1453         IPOIB_ENTER( IPOIB_DBG_RECV );\r
1454 \r
1455         CL_ASSERT( p_port );\r
1456         cl_obj_lock( &p_port->obj );\r
1457         if( p_port->state != IB_QPS_RTS )\r
1458         {\r
1459                 cl_obj_unlock( &p_port->obj );\r
1460                 IPOIB_PRINT_EXIT( TRACE_LEVEL_INFORMATION, IPOIB_DBG_RECV,\r
1461                         ("Port in invalid state.  Not reposting.\n") );\r
1462                 return 0;\r
1463         }\r
1464         ipoib_port_ref( p_port, ref_repost );\r
1465         cl_obj_unlock( &p_port->obj );\r
1466 \r
1467         while( p_port->recv_mgr.depth < p_port->p_adapter->params.rq_depth )\r
1468         {\r
1469                 /* Pull receives out of the pool and chain them up. */\r
1470                 cl_perf_start( GetRecv );\r
1471                 p_next = __buf_mgr_get_recv( p_port );\r
1472                 cl_perf_stop( &p_port->p_adapter->perf, GetRecv );\r
1473                 if( !p_next )\r
1474                 {\r
1475                         IPOIB_PRINT(TRACE_LEVEL_INFORMATION, IPOIB_DBG_RECV,\r
1476                                 ("Out of receive descriptors! recv queue depath 0x%x\n",p_port->recv_mgr.depth) );\r
1477                         break;\r
1478                 }\r
1479 \r
1480                 if( !p_tail )\r
1481                 {\r
1482                         p_tail = p_next;\r
1483                         p_next->wr.p_next = NULL;\r
1484                 }\r
1485                 else\r
1486                 {\r
1487                         p_next->wr.p_next = &p_head->wr;\r
1488                 }\r
1489 \r
1490                 p_head = p_next;\r
1491 \r
1492                 p_port->recv_mgr.depth++;\r
1493         }\r
1494 \r
1495         if( p_head )\r
1496         {\r
1497                 cl_perf_start( PostRecv );\r
1498                 status = p_port->p_adapter->p_ifc->post_recv(\r
1499                         p_port->ib_mgr.h_qp, &p_head->wr, &p_failed );\r
1500                 cl_perf_stop( &p_port->p_adapter->perf, PostRecv );\r
1501 \r
1502                 if( status != IB_SUCCESS )\r
1503                 {\r
1504                         IPOIB_PRINT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1505                                 ("ip_post_recv returned %s\n", \r
1506                                 p_port->p_adapter->p_ifc->get_err_str( status )) );\r
1507                         /* return the descriptors to the pool */\r
1508                         while( p_failed )\r
1509                         {\r
1510                                 p_head = PARENT_STRUCT( p_failed, ipoib_recv_desc_t, wr );\r
1511                                 p_failed = p_failed->p_next;\r
1512 \r
1513                                 __buf_mgr_put_recv( p_port, p_head, NULL );\r
1514                                 p_port->recv_mgr.depth--;\r
1515                         }\r
1516                 }\r
1517         }\r
1518 \r
1519         ipoib_port_deref( p_port, ref_repost );\r
1520         IPOIB_EXIT( IPOIB_DBG_RECV );\r
1521         return p_port->p_adapter->params.rq_low_watermark - p_port->recv_mgr.depth;\r
1522 }\r
1523 \r
1524 \r
1525 void\r
1526 ipoib_return_packet(\r
1527         IN                              NDIS_HANDLE                                     adapter_context,\r
1528         IN                              NDIS_PACKET                                     *p_packet )\r
1529 {\r
1530         cl_list_item_t          *p_item;\r
1531         ipoib_port_t            *p_port;\r
1532         ipoib_recv_desc_t       *p_desc;\r
1533         ib_api_status_t         status = IB_NOT_DONE;\r
1534         int32_t                         shortage;\r
1535         PERF_DECLARE( ReturnPacket );\r
1536         PERF_DECLARE( ReturnPutRecv );\r
1537         PERF_DECLARE( ReturnRepostRecv );\r
1538         PERF_DECLARE( ReturnPreparePkt );\r
1539         PERF_DECLARE( ReturnNdisIndicate );\r
1540 \r
1541         IPOIB_ENTER( IPOIB_DBG_RECV );\r
1542 \r
1543         UNUSED_PARAM( adapter_context );\r
1544         CL_ASSERT( p_packet );\r
1545 \r
1546         cl_perf_start( ReturnPacket );\r
1547 \r
1548         /* Get the port and descriptor from the packet. */\r
1549         p_port = IPOIB_PORT_FROM_PACKET( p_packet );\r
1550         p_desc = IPOIB_RECV_FROM_PACKET( p_packet );\r
1551 \r
1552         cl_spinlock_acquire( &p_port->recv_lock );\r
1553 \r
1554         cl_perf_start( ReturnPutRecv );\r
1555         __buf_mgr_put_recv( p_port, p_desc, p_packet );\r
1556         cl_perf_stop( &p_port->p_adapter->perf, ReturnPutRecv );\r
1557 \r
1558         /* Repost buffers. */\r
1559         cl_perf_start( ReturnRepostRecv );\r
1560         shortage = __recv_mgr_repost( p_port );\r
1561         cl_perf_stop( &p_port->p_adapter->perf, ReturnRepostRecv );\r
1562 \r
1563         for( p_item = cl_qlist_remove_head( &p_port->recv_mgr.done_list );\r
1564                 p_item != cl_qlist_end( &p_port->recv_mgr.done_list );\r
1565                 p_item = cl_qlist_remove_head( &p_port->recv_mgr.done_list ) )\r
1566         {\r
1567                 p_desc = (ipoib_recv_desc_t*)p_item;\r
1568 \r
1569                 cl_perf_start( ReturnPreparePkt );\r
1570                 status = __recv_mgr_prepare_pkt( p_port, p_desc, &p_packet );\r
1571                 cl_perf_stop( &p_port->p_adapter->perf, ReturnPreparePkt );\r
1572                 if( status == IB_SUCCESS )\r
1573                 {\r
1574                         if( shortage > 0 )\r
1575                                 NDIS_SET_PACKET_STATUS( p_packet, NDIS_STATUS_RESOURCES );\r
1576                         else\r
1577                                 NDIS_SET_PACKET_STATUS( p_packet, NDIS_STATUS_SUCCESS );\r
1578 \r
1579                         cl_spinlock_release( &p_port->recv_lock );\r
1580                         cl_perf_start( ReturnNdisIndicate );\r
1581                         NdisMIndicateReceivePacket( p_port->p_adapter->h_adapter,\r
1582                                 &p_packet, 1 );\r
1583                         cl_perf_stop( &p_port->p_adapter->perf, ReturnNdisIndicate );\r
1584                         cl_spinlock_acquire( &p_port->recv_lock );\r
1585 \r
1586                         if( shortage > 0 )\r
1587                         {\r
1588                                 cl_perf_start( ReturnPutRecv );\r
1589                                 __buf_mgr_put_recv( p_port, p_desc, p_packet );\r
1590                                 cl_perf_stop( &p_port->p_adapter->perf, ReturnPutRecv );\r
1591 \r
1592                                 /* Repost buffers. */\r
1593                                 cl_perf_start( ReturnRepostRecv );\r
1594                                 shortage = __recv_mgr_repost( p_port );\r
1595                                 cl_perf_stop( &p_port->p_adapter->perf, ReturnRepostRecv );\r
1596                         }\r
1597                 }\r
1598                 else if( status != IB_NOT_DONE )\r
1599                 {\r
1600                         IPOIB_PRINT(TRACE_LEVEL_INFORMATION, IPOIB_DBG_RECV,\r
1601                                 ("__recv_mgr_prepare_pkt returned %s\n",\r
1602                                 p_port->p_adapter->p_ifc->get_err_str( status )) );\r
1603                         /* Return the item to the head of the list. */\r
1604                         cl_qlist_insert_head( &p_port->recv_mgr.done_list, p_item );\r
1605                         break;\r
1606                 }\r
1607         }\r
1608         cl_spinlock_release( &p_port->recv_lock );\r
1609         cl_perf_stop( &p_port->p_adapter->perf, ReturnPacket );\r
1610 \r
1611         IPOIB_EXIT( IPOIB_DBG_RECV );\r
1612 }\r
1613 \r
1614 static void __recv_cb_dpc(KDPC *p_gc_dpc,void *context,void * s_arg1 , void * s_arg2)\r
1615 {\r
1616 \r
1617         ipoib_port_t *p_port = context;\r
1618 \r
1619         UNREFERENCED_PARAMETER(p_gc_dpc);\r
1620         UNREFERENCED_PARAMETER(s_arg1);\r
1621         UNREFERENCED_PARAMETER(s_arg2);\r
1622 \r
1623 \r
1624         __recv_cb(NULL, p_port);\r
1625         ipoib_port_deref( p_port, ref_recv_cb );\r
1626 \r
1627 \r
1628 }\r
1629 \r
1630 \r
1631 static void\r
1632 __recv_cb(\r
1633         IN              const   ib_cq_handle_t                          h_cq,\r
1634         IN                              void                                            *cq_context )\r
1635 {\r
1636         ipoib_port_t            *p_port;\r
1637         ib_api_status_t         status;\r
1638         ib_wc_t                         wc[MAX_RECV_WC], *p_free, *p_wc;\r
1639         int32_t                         pkt_cnt, recv_cnt = 0, shortage, discarded;\r
1640         cl_qlist_t                      done_list, bad_list;\r
1641         size_t                          i;\r
1642         PERF_DECLARE( RecvCompBundle );\r
1643         PERF_DECLARE( RecvCb );\r
1644         PERF_DECLARE( PollRecv );\r
1645         PERF_DECLARE( RepostRecv );\r
1646         PERF_DECLARE( FilterRecv );\r
1647         PERF_DECLARE( BuildPktArray );\r
1648         PERF_DECLARE( RecvNdisIndicate );\r
1649         PERF_DECLARE( RearmRecv );\r
1650         PERF_DECLARE( PutRecvList );\r
1651 \r
1652         IPOIB_ENTER( IPOIB_DBG_RECV );\r
1653 \r
1654         cl_perf_clr( RecvCompBundle );\r
1655 \r
1656         cl_perf_start( RecvCb );\r
1657 \r
1658         UNUSED_PARAM( h_cq );\r
1659 \r
1660         p_port = (ipoib_port_t*)cq_context;\r
1661 \r
1662         cl_qlist_init( &done_list );\r
1663         cl_qlist_init( &bad_list );\r
1664 \r
1665         ipoib_port_ref( p_port, ref_recv_cb );\r
1666         for( i = 0; i < MAX_RECV_WC; i++ )\r
1667                 wc[i].p_next = &wc[i + 1];\r
1668         wc[MAX_RECV_WC - 1].p_next = NULL;\r
1669 \r
1670         /*\r
1671          * We'll be accessing the endpoint map so take a reference\r
1672          * on it to prevent modifications.\r
1673          */\r
1674         cl_obj_lock( &p_port->obj );\r
1675         cl_atomic_inc( &p_port->endpt_rdr );\r
1676         cl_obj_unlock( &p_port->obj );\r
1677 \r
1678         do\r
1679         {\r
1680                 /* If we get here, then the list of WCs is intact. */\r
1681                 p_free = wc;\r
1682 \r
1683                 cl_perf_start( PollRecv );\r
1684                 status = p_port->p_adapter->p_ifc->poll_cq(\r
1685                         p_port->ib_mgr.h_recv_cq, &p_free, &p_wc );\r
1686                 cl_perf_stop( &p_port->p_adapter->perf, PollRecv );\r
1687                 CL_ASSERT( status == IB_SUCCESS || status == IB_NOT_FOUND );\r
1688 \r
1689                 /* Look at the payload now and filter ARP and DHCP packets. */\r
1690                 cl_perf_start( FilterRecv );\r
1691                 recv_cnt += __recv_mgr_filter( p_port, p_wc, &done_list, &bad_list );\r
1692                 cl_perf_stop( &p_port->p_adapter->perf, FilterRecv );\r
1693 \r
1694         } while( (!p_free) && (recv_cnt < 128));\r
1695 \r
1696         /* We're done looking at the endpoint map, release the reference. */\r
1697         cl_atomic_dec( &p_port->endpt_rdr );\r
1698 \r
1699         cl_perf_log( &p_port->p_adapter->perf, RecvCompBundle, recv_cnt );\r
1700 \r
1701         cl_spinlock_acquire( &p_port->recv_lock );\r
1702 \r
1703         /* Update our posted depth. */\r
1704         p_port->recv_mgr.depth -= recv_cnt;\r
1705 \r
1706         /* Return any discarded receives to the pool */\r
1707         cl_perf_start( PutRecvList );\r
1708         __buf_mgr_put_recv_list( p_port, &bad_list );\r
1709         cl_perf_stop( &p_port->p_adapter->perf, PutRecvList );\r
1710 \r
1711         do\r
1712         {\r
1713                 /* Repost ASAP so we don't starve the RQ. */\r
1714                 cl_perf_start( RepostRecv );\r
1715                 shortage = __recv_mgr_repost( p_port );\r
1716                 cl_perf_stop( &p_port->p_adapter->perf, RepostRecv );\r
1717 \r
1718                 cl_perf_start( BuildPktArray );\r
1719                 /* Notify NDIS of any and all possible receive buffers. */\r
1720                 pkt_cnt = __recv_mgr_build_pkt_array(\r
1721                         p_port, shortage, &done_list, &discarded );\r
1722                 cl_perf_stop( &p_port->p_adapter->perf, BuildPktArray );\r
1723 \r
1724                 /* Only indicate receives if we actually had any. */\r
1725                 if( discarded && shortage > 0 )\r
1726                 {\r
1727                         /* We may have thrown away packets, and have a shortage */\r
1728                         cl_perf_start( RepostRecv );\r
1729                         __recv_mgr_repost( p_port );\r
1730                         cl_perf_stop( &p_port->p_adapter->perf, RepostRecv );\r
1731                 }\r
1732 \r
1733                 if( !pkt_cnt )\r
1734                         break;\r
1735 \r
1736                 cl_spinlock_release( &p_port->recv_lock );\r
1737 \r
1738                 cl_perf_start( RecvNdisIndicate );\r
1739                 NdisMIndicateReceivePacket( p_port->p_adapter->h_adapter,\r
1740                         p_port->recv_mgr.recv_pkt_array, pkt_cnt );\r
1741                 cl_perf_stop( &p_port->p_adapter->perf, RecvNdisIndicate );\r
1742 \r
1743                 /*\r
1744                  * Cap the number of receives to put back to what we just indicated\r
1745                  * with NDIS_STATUS_RESOURCES.\r
1746                  */\r
1747                 if( shortage > 0 )\r
1748                 {\r
1749                         if( pkt_cnt < shortage )\r
1750                                 shortage = pkt_cnt;\r
1751 \r
1752                         /* Return all but the last packet to the pool. */\r
1753                         cl_spinlock_acquire( &p_port->recv_lock );\r
1754                         while( shortage-- > 1 )\r
1755                         {\r
1756                                 __buf_mgr_put_recv( p_port,\r
1757                                         IPOIB_RECV_FROM_PACKET( p_port->recv_mgr.recv_pkt_array[shortage] ),\r
1758                                         p_port->recv_mgr.recv_pkt_array[shortage] );\r
1759                         }\r
1760                         cl_spinlock_release( &p_port->recv_lock );\r
1761 \r
1762                         /*\r
1763                          * Return the last packet as if NDIS returned it, so that we repost\r
1764                          * and report any other pending receives.\r
1765                          */\r
1766                         ipoib_return_packet( NULL, p_port->recv_mgr.recv_pkt_array[0] );\r
1767                 }\r
1768                 cl_spinlock_acquire( &p_port->recv_lock );\r
1769 \r
1770         } while( pkt_cnt );\r
1771         cl_spinlock_release( &p_port->recv_lock );\r
1772 \r
1773         if (p_free ) {\r
1774                 /*\r
1775                  * Rearm after filtering to prevent contention on the enpoint maps\r
1776                  * and eliminate the possibility of having a call to\r
1777                  * __endpt_mgr_insert find a duplicate.\r
1778                  */\r
1779                 cl_perf_start( RearmRecv );\r
1780                 status = p_port->p_adapter->p_ifc->rearm_cq(\r
1781                         p_port->ib_mgr.h_recv_cq, FALSE );\r
1782                 cl_perf_stop( &p_port->p_adapter->perf, RearmRecv );\r
1783                 CL_ASSERT( status == IB_SUCCESS );\r
1784 \r
1785                 ipoib_port_deref( p_port, ref_recv_cb );\r
1786         } else {\r
1787                 // Please note the reference is still up\r
1788                 KeInsertQueueDpc(&p_port->recv_dpc, NULL, NULL);\r
1789         }\r
1790 \r
1791         cl_perf_stop( &p_port->p_adapter->perf, RecvCb );\r
1792 \r
1793         IPOIB_EXIT( IPOIB_DBG_RECV );\r
1794 }\r
1795 \r
1796 \r
1797 static void\r
1798 __recv_get_endpts(\r
1799         IN                              ipoib_port_t* const                     p_port,\r
1800         IN                              ipoib_recv_desc_t* const        p_desc,\r
1801         IN                              ib_wc_t* const                          p_wc,\r
1802                 OUT                     ipoib_endpt_t** const           pp_src,\r
1803                 OUT                     ipoib_endpt_t** const           pp_dst )\r
1804 {\r
1805         ib_api_status_t         status;\r
1806         mac_addr_t                      mac;\r
1807         PERF_DECLARE( GetEndptByGid );\r
1808         PERF_DECLARE( GetEndptByLid );\r
1809         PERF_DECLARE( EndptInsert );\r
1810 \r
1811         IPOIB_ENTER( IPOIB_DBG_RECV );\r
1812 \r
1813         /* Setup our shortcut pointers based on whether GRH is valid. */\r
1814         if( p_wc->recv.ud.recv_opt & IB_RECV_OPT_GRH_VALID )\r
1815         {\r
1816                 /* Lookup the source endpoints based on GID. */\r
1817                 cl_perf_start( GetEndptByGid );\r
1818                 *pp_src =\r
1819 #if IPOIB_INLINE_RECV\r
1820                         __endpt_mgr_get_by_gid( p_port, &p_desc->buf.ib.grh.src_gid );\r
1821 #else   /* IPOIB_INLINE_RECV */\r
1822                         __endpt_mgr_get_by_gid( p_port, &p_desc->p_buf->ib.grh.src_gid );\r
1823 #endif  /* IPOIB_INLINE_RECV */\r
1824                 cl_perf_stop( &p_port->p_adapter->perf, GetEndptByGid );\r
1825 \r
1826                 /*\r
1827                  * Lookup the destination endpoint based on GID.\r
1828                  * This is used along with the packet filter to determine\r
1829                  * whether to report this to NDIS.\r
1830                  */\r
1831                 cl_perf_start( GetEndptByGid );\r
1832                 *pp_dst =\r
1833 #if IPOIB_INLINE_RECV\r
1834                         __endpt_mgr_get_by_gid( p_port, &p_desc->buf.ib.grh.dest_gid );\r
1835 #else   /* IPOIB_INLINE_RECV */\r
1836                         __endpt_mgr_get_by_gid( p_port, &p_desc->p_buf->ib.grh.dest_gid );\r
1837 #endif  /* IPOIB_INLINE_RECV */\r
1838                 cl_perf_stop( &p_port->p_adapter->perf, GetEndptByGid );\r
1839 \r
1840                 /*\r
1841                  * Create the source endpoint if it does not exist.  Note that we\r
1842                  * can only do this for globally routed traffic since we need the\r
1843                  * information from the GRH to generate the MAC.\r
1844                  */\r
1845                 if( !*pp_src )\r
1846                 {\r
1847                         status = ipoib_mac_from_guid(\r
1848 #if IPOIB_INLINE_RECV\r
1849                                 p_desc->buf.ib.grh.src_gid.unicast.interface_id, p_port->p_adapter->params.guid_mask, &mac );\r
1850 #else   /* IPOIB_INLINE_RECV */\r
1851                                 p_desc->p_buf->ib.grh.src_gid.unicast.interface_id, p_port->p_adapter->params.guid_mask, &mac );\r
1852 #endif  /* IPOIB_INLINE_RECV */\r
1853                         if( status != IB_SUCCESS )\r
1854                         {\r
1855                                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1856                                         ("ipoib_mac_from_guid returned %s\n",\r
1857                                         p_port->p_adapter->p_ifc->get_err_str( status )) );\r
1858                                 return;\r
1859                         }\r
1860 \r
1861                         /* Create the endpoint. */\r
1862 #if IPOIB_INLINE_RECV\r
1863                         *pp_src = ipoib_endpt_create( &p_desc->buf.ib.grh.src_gid,\r
1864 #else   /* IPOIB_INLINE_RECV */\r
1865                         *pp_src = ipoib_endpt_create( &p_desc->p_buf->ib.grh.src_gid,\r
1866 #endif  /* IPOIB_INLINE_RECV */\r
1867                                 p_wc->recv.ud.remote_lid, p_wc->recv.ud.remote_qp );\r
1868                         if( !*pp_src )\r
1869                         {\r
1870                                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1871                                         ("ipoib_endpt_create failed\n") );\r
1872                                 return;\r
1873                         }\r
1874                         cl_perf_start( EndptInsert );\r
1875                         cl_obj_lock( &p_port->obj );\r
1876                         status = __endpt_mgr_insert( p_port, mac, *pp_src );\r
1877                         if( status != IB_SUCCESS )\r
1878                         {\r
1879                                 cl_obj_unlock( &p_port->obj );\r
1880                                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1881                                         ("__endpt_mgr_insert returned %s\n",\r
1882                                         p_port->p_adapter->p_ifc->get_err_str( status )) );\r
1883                                 *pp_src = NULL;\r
1884                                 return;\r
1885                         }\r
1886                         cl_obj_unlock( &p_port->obj );\r
1887                         cl_perf_stop( &p_port->p_adapter->perf, EndptInsert );\r
1888                 }\r
1889         }\r
1890         else\r
1891         {\r
1892                 /*\r
1893                  * Lookup the remote endpoint based on LID.  Note that only\r
1894                  * unicast traffic can be LID routed.\r
1895                  */\r
1896                 cl_perf_start( GetEndptByLid );\r
1897                 *pp_src = __endpt_mgr_get_by_lid( p_port, p_wc->recv.ud.remote_lid );\r
1898                 cl_perf_stop( &p_port->p_adapter->perf, GetEndptByLid );\r
1899                 *pp_dst = p_port->p_local_endpt;\r
1900                 CL_ASSERT( *pp_dst );\r
1901         }\r
1902 \r
1903         if( *pp_src && !ipoib_is_voltaire_router_gid( &(*pp_src)->dgid ) &&\r
1904                 (*pp_src)->qpn != p_wc->recv.ud.remote_qp )\r
1905         {\r
1906                 /* Update the QPN for the endpoint. */\r
1907                 IPOIB_PRINT(TRACE_LEVEL_INFORMATION, IPOIB_DBG_RECV,\r
1908                         ("Updating QPN for MAC: %02X-%02X-%02X-%02X-%02X-%02X\n",\r
1909                         (*pp_src )->mac.addr[0], (*pp_src )->mac.addr[1],\r
1910                         (*pp_src )->mac.addr[2], (*pp_src )->mac.addr[3],\r
1911                         (*pp_src )->mac.addr[4], (*pp_src )->mac.addr[5]) );\r
1912                 (*pp_src)->qpn = p_wc->recv.ud.remote_qp;\r
1913         }\r
1914 \r
1915         if( *pp_src && *pp_dst )\r
1916         {\r
1917                 IPOIB_PRINT(TRACE_LEVEL_INFORMATION, IPOIB_DBG_RECV,\r
1918                         ("Recv:\n"\r
1919                         "\tsrc MAC: %02X-%02X-%02X-%02X-%02X-%02X\n"\r
1920                         "\tdst MAC: %02X-%02X-%02X-%02X-%02X-%02X\n",\r
1921                         (*pp_src )->mac.addr[0], (*pp_src )->mac.addr[1],\r
1922                         (*pp_src )->mac.addr[2], (*pp_src )->mac.addr[3],\r
1923                         (*pp_src )->mac.addr[4], (*pp_src )->mac.addr[5],\r
1924                         (*pp_dst )->mac.addr[0], (*pp_dst )->mac.addr[1],\r
1925                         (*pp_dst )->mac.addr[2], (*pp_dst )->mac.addr[3],\r
1926                         (*pp_dst )->mac.addr[4], (*pp_dst )->mac.addr[5]) );\r
1927         }\r
1928 \r
1929         IPOIB_EXIT( IPOIB_DBG_RECV );\r
1930 }\r
1931 \r
1932 \r
1933 static int32_t\r
1934 __recv_mgr_filter(\r
1935         IN                              ipoib_port_t* const                     p_port,\r
1936         IN                              ib_wc_t* const                          p_done_wc_list,\r
1937                 OUT                     cl_qlist_t* const                       p_done_list,\r
1938                 OUT                     cl_qlist_t* const                       p_bad_list )\r
1939 {\r
1940         ipoib_recv_desc_t               *p_desc;\r
1941         ib_wc_t                                 *p_wc;\r
1942         ipoib_pkt_t                             *p_ipoib;\r
1943         eth_pkt_t                               *p_eth;\r
1944         ipoib_endpt_t                   *p_src, *p_dst;\r
1945         ib_api_status_t                 status;\r
1946         uint32_t                                len;\r
1947         int32_t                                 recv_cnt = 0;\r
1948         PERF_DECLARE( GetRecvEndpts );\r
1949         PERF_DECLARE( RecvGen );\r
1950         PERF_DECLARE( RecvTcp );\r
1951         PERF_DECLARE( RecvUdp );\r
1952         PERF_DECLARE( RecvDhcp );\r
1953         PERF_DECLARE( RecvArp );\r
1954 \r
1955         IPOIB_ENTER( IPOIB_DBG_RECV );\r
1956 \r
1957         for( p_wc = p_done_wc_list; p_wc; p_wc = p_wc->p_next )\r
1958         {\r
1959                 CL_ASSERT( p_wc->status != IB_WCS_SUCCESS || p_wc->wc_type == IB_WC_RECV );\r
1960                 p_desc = (ipoib_recv_desc_t*)(uintn_t)p_wc->wr_id;\r
1961                 recv_cnt++;\r
1962 \r
1963                 if( p_wc->status != IB_WCS_SUCCESS )\r
1964                 {\r
1965                         if( p_wc->status != IB_WCS_WR_FLUSHED_ERR )\r
1966                         {\r
1967                                 IPOIB_PRINT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1968                                         ("Failed completion %s  (vendor specific %#x)\n",\r
1969                                         p_port->p_adapter->p_ifc->get_wc_status_str( p_wc->status ),\r
1970                                         (int)p_wc->vendor_specific) );\r
1971                                 ipoib_inc_recv_stat( p_port->p_adapter, IP_STAT_ERROR, 0 );\r
1972                         }\r
1973                         else\r
1974                         {\r
1975                                 IPOIB_PRINT(TRACE_LEVEL_INFORMATION, IPOIB_DBG_RECV,\r
1976                                         ("Flushed completion %s\n",\r
1977                                         p_port->p_adapter->p_ifc->get_wc_status_str( p_wc->status )) );\r
1978                                 ipoib_inc_recv_stat( p_port->p_adapter, IP_STAT_DROPPED, 0 );\r
1979                         }\r
1980                         cl_qlist_insert_tail( p_bad_list, &p_desc->item.list_item );\r
1981                         /* Dereference the port object on behalf of the failed receive. */\r
1982                         ipoib_port_deref( p_port, ref_failed_recv_wc );\r
1983                         continue;\r
1984                 }\r
1985 \r
1986                 len = p_wc->length - sizeof(ib_grh_t);\r
1987 \r
1988                 if( len < sizeof(ipoib_hdr_t) )\r
1989                 {\r
1990                         IPOIB_PRINT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1991                                 ("Received ETH packet < min size\n") );\r
1992                         ipoib_inc_recv_stat( p_port->p_adapter, IP_STAT_ERROR, 0 );\r
1993                         cl_qlist_insert_tail( p_bad_list, &p_desc->item.list_item );\r
1994                         ipoib_port_deref( p_port, ref_recv_inv_len );\r
1995                         continue;\r
1996                 }\r
1997 \r
1998                 if((len - sizeof(ipoib_hdr_t)) > p_port->p_adapter->params.payload_mtu)\r
1999                 {\r
2000                         IPOIB_PRINT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2001                                 ("Received ETH packet > payload MTU (%d)\n",\r
2002                                 p_port->p_adapter->params.payload_mtu) );\r
2003                         ipoib_inc_recv_stat( p_port->p_adapter, IP_STAT_ERROR, 0 );\r
2004                         cl_qlist_insert_tail( p_bad_list, &p_desc->item.list_item );\r
2005                         ipoib_port_deref( p_port, ref_recv_inv_len );\r
2006                         continue;\r
2007                         \r
2008                 }\r
2009                 /* Successful completion.  Get the receive information. */\r
2010                 p_desc->ndis_csum.Value = ( (p_wc->recv.ud.recv_opt & IB_RECV_OPT_CSUM_MASK ) >> 8 );\r
2011                 cl_perf_start( GetRecvEndpts );\r
2012                 __recv_get_endpts( p_port, p_desc, p_wc, &p_src, &p_dst );\r
2013                 cl_perf_stop( &p_port->p_adapter->perf, GetRecvEndpts );\r
2014 \r
2015 #if IPOIB_INLINE_RECV\r
2016                 p_ipoib = &p_desc->buf.ib.pkt;\r
2017                 p_eth = &p_desc->buf.eth.pkt;\r
2018 #else   /* IPOIB_INLINE_RECV */\r
2019                 p_ipoib = &p_desc->p_buf->ib.pkt;\r
2020                 p_eth = &p_desc->p_buf->eth.pkt;\r
2021 #endif  /*IPOIB_INLINE_RECV */\r
2022 \r
2023                 if( p_src )\r
2024                 {\r
2025                         /* Don't report loopback traffic - we requested SW loopback. */\r
2026                         if( !cl_memcmp( &p_port->p_adapter->params.conf_mac,\r
2027                                 &p_src->mac, sizeof(p_port->p_adapter->params.conf_mac) ) )\r
2028                         {\r
2029                                 /*\r
2030                                  * "This is not the packet you're looking for" - don't update\r
2031                                  * receive statistics, the packet never happened.\r
2032                                  */\r
2033                                 cl_qlist_insert_tail( p_bad_list, &p_desc->item.list_item );\r
2034                                 /* Dereference the port object on behalf of the failed recv. */\r
2035                                 ipoib_port_deref( p_port, ref_recv_loopback );\r
2036                                 continue;\r
2037                         }\r
2038                 }\r
2039 \r
2040                 switch( p_ipoib->hdr.type )\r
2041                 {\r
2042                 case ETH_PROT_TYPE_IP:\r
2043                         if( len < (sizeof(ipoib_hdr_t) + sizeof(ip_hdr_t)) )\r
2044                         {\r
2045                                 IPOIB_PRINT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2046                                         ("Received IP packet < min size\n") );\r
2047                                 status = IB_INVALID_SETTING;\r
2048                                 break;\r
2049                         }\r
2050 \r
2051                         if( p_ipoib->type.ip.hdr.offset ||\r
2052                                 p_ipoib->type.ip.hdr.prot != IP_PROT_UDP )\r
2053                         {\r
2054                                 /* Unfiltered.  Setup the ethernet header and report. */\r
2055                                 cl_perf_start( RecvTcp );\r
2056                                 status = __recv_gen( p_ipoib, p_eth, p_src, p_dst );\r
2057                                 cl_perf_stop( &p_port->p_adapter->perf, RecvTcp );\r
2058                                 break;\r
2059                         }\r
2060 \r
2061                         /* First packet of a UDP transfer. */\r
2062                         if( len <\r
2063                                 (sizeof(ipoib_hdr_t) + sizeof(ip_hdr_t) + sizeof(udp_hdr_t)) )\r
2064                         {\r
2065                                 IPOIB_PRINT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2066                                         ("Received UDP packet < min size\n") );\r
2067                                 status = IB_INVALID_SETTING;\r
2068                                 break;\r
2069                         }\r
2070 \r
2071                         /* Check if DHCP conversion is required. */\r
2072                         if( (p_ipoib->type.ip.prot.udp.hdr.dst_port == DHCP_PORT_SERVER &&\r
2073                                 p_ipoib->type.ip.prot.udp.hdr.src_port == DHCP_PORT_CLIENT) ||\r
2074                                 (p_ipoib->type.ip.prot.udp.hdr.dst_port == DHCP_PORT_CLIENT &&\r
2075                                 p_ipoib->type.ip.prot.udp.hdr.src_port == DHCP_PORT_SERVER) ||\r
2076                                 (p_ipoib->type.ip.prot.udp.hdr.dst_port == DHCP_PORT_PROXY_SERVER &&\r
2077                                 p_ipoib->type.ip.prot.udp.hdr.src_port == DHCP_PORT_CLIENT) ||\r
2078                                 (p_ipoib->type.ip.prot.udp.hdr.dst_port == DHCP_PORT_CLIENT &&\r
2079                                 p_ipoib->type.ip.prot.udp.hdr.src_port == DHCP_PORT_PROXY_SERVER) )\r
2080                         {\r
2081                                 if( len < (sizeof(ipoib_hdr_t) + sizeof(ip_hdr_t) +\r
2082                                         sizeof(udp_hdr_t) + DHCP_MIN_SIZE) )\r
2083                                 {\r
2084                                         IPOIB_PRINT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2085                                                 ("Received DHCP < min size\n") );\r
2086                                         status = IB_INVALID_SETTING;\r
2087                                         break;\r
2088                                 }\r
2089                                 if ((p_ipoib->type.ip.hdr.ver_hl & 0x0f) != 5 ) {\r
2090                                         // If there are IP options in this message, we are in trouble in any case\r
2091                                         status = IB_INVALID_SETTING;\r
2092                                         break;                                  \r
2093                                 }\r
2094                                 /* UDP packet with BOOTP ports in src/dst port numbers. */\r
2095                                 cl_perf_start( RecvDhcp );\r
2096                                 status = __recv_dhcp( p_port, p_ipoib, p_eth, p_src, p_dst );\r
2097                                 cl_perf_stop( &p_port->p_adapter->perf, RecvDhcp );\r
2098                         }\r
2099                         else\r
2100                         {\r
2101                                 /* Unfiltered.  Setup the ethernet header and report. */\r
2102                                 cl_perf_start( RecvUdp );\r
2103                                 status = __recv_gen( p_ipoib, p_eth, p_src, p_dst );\r
2104                                 cl_perf_stop( &p_port->p_adapter->perf, RecvUdp );\r
2105                         }\r
2106                         break;\r
2107 \r
2108                 case ETH_PROT_TYPE_ARP:\r
2109                         if( len < (sizeof(ipoib_hdr_t) + sizeof(ipoib_arp_pkt_t)) )\r
2110                         {\r
2111                                 IPOIB_PRINT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2112                                         ("Received ARP < min size\n") );\r
2113                                 status = IB_INVALID_SETTING;\r
2114                                 break;\r
2115                         }\r
2116                         cl_perf_start( RecvArp );\r
2117                         status = __recv_arp( p_port, p_wc, p_ipoib, p_eth, &p_src, p_dst );\r
2118                         cl_perf_stop( &p_port->p_adapter->perf, RecvArp );\r
2119                         len = sizeof(ipoib_hdr_t) + sizeof(arp_pkt_t);\r
2120                         break;\r
2121 \r
2122                 default:\r
2123                         /* Unfiltered.  Setup the ethernet header and report. */\r
2124                         cl_perf_start( RecvGen );\r
2125                         status = __recv_gen( p_ipoib, p_eth, p_src, p_dst );\r
2126                         cl_perf_stop( &p_port->p_adapter->perf, RecvGen );\r
2127                 }\r
2128 \r
2129                 if( status != IB_SUCCESS )\r
2130                 {\r
2131                         /* Update stats. */\r
2132                         ipoib_inc_recv_stat( p_port->p_adapter, IP_STAT_ERROR, 0 );\r
2133                         cl_qlist_insert_tail( p_bad_list, &p_desc->item.list_item );\r
2134                         /* Dereference the port object on behalf of the failed receive. */\r
2135                         ipoib_port_deref( p_port, ref_recv_filter );\r
2136                 }\r
2137                 else\r
2138                 {\r
2139                         ip_stat_sel_t               ip_stat;\r
2140                         p_desc->len =\r
2141                                 len + sizeof(eth_hdr_t) - sizeof(ipoib_hdr_t);\r
2142                         if( p_dst->h_mcast)\r
2143                         {\r
2144                                 if( p_dst->dgid.multicast.raw_group_id[10] == 0xFF &&\r
2145                                         p_dst->dgid.multicast.raw_group_id[11] == 0xFF &&\r
2146                                         p_dst->dgid.multicast.raw_group_id[12] == 0xFF &&\r
2147                                         p_dst->dgid.multicast.raw_group_id[13] == 0xFF )\r
2148                                 {\r
2149                                         p_desc->type = PKT_TYPE_BCAST;\r
2150                                         ip_stat = IP_STAT_BCAST_BYTES;\r
2151                                 }\r
2152                                 else\r
2153                                 {\r
2154                                         p_desc->type = PKT_TYPE_MCAST;\r
2155                                         ip_stat = IP_STAT_MCAST_BYTES;\r
2156                                 }\r
2157                         }\r
2158                         else\r
2159                         {\r
2160                                 p_desc->type = PKT_TYPE_UCAST;\r
2161                                 ip_stat = IP_STAT_UCAST_BYTES;\r
2162                                 \r
2163                         }\r
2164                         cl_qlist_insert_tail( p_done_list, &p_desc->item.list_item );\r
2165                         ipoib_inc_recv_stat( p_port->p_adapter,ip_stat , len );  \r
2166                 }\r
2167         }\r
2168 \r
2169         IPOIB_EXIT( IPOIB_DBG_RECV );\r
2170         return recv_cnt;\r
2171 }\r
2172 \r
2173 \r
2174 static ib_api_status_t\r
2175 __recv_gen(\r
2176         IN              const   ipoib_pkt_t* const                      p_ipoib,\r
2177                 OUT                     eth_pkt_t* const                        p_eth,\r
2178         IN                              ipoib_endpt_t* const            p_src,\r
2179         IN                              ipoib_endpt_t* const            p_dst )\r
2180 {\r
2181         IPOIB_ENTER( IPOIB_DBG_RECV );\r
2182 \r
2183         if( !p_src || !p_dst )\r
2184         {\r
2185                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2186                         ("Received packet with no matching endpoints.\n") );\r
2187                 return IB_NOT_DONE;\r
2188         }\r
2189 \r
2190         /*\r
2191          * Fill in the ethernet header.  Note that doing so will overwrite\r
2192          * the IPoIB header, so start by moving the information from the IPoIB\r
2193          * header.\r
2194          */\r
2195         p_eth->hdr.type = p_ipoib->hdr.type;\r
2196         p_eth->hdr.src = p_src->mac;\r
2197         p_eth->hdr.dst = p_dst->mac;\r
2198 \r
2199         if ( p_eth->hdr.dst.addr[0] == 1 && \r
2200                  p_eth->hdr.type == ETH_PROT_TYPE_IP &&\r
2201                  p_eth->hdr.dst.addr[2] == 0x5E)  \r
2202         {\r
2203                 p_eth->hdr.dst.addr[1] = 0;\r
2204                 p_eth->hdr.dst.addr[3] = p_eth->hdr.dst.addr[3] & 0x7f;\r
2205         }\r
2206         if (p_dst->h_mcast)\r
2207                 p_dst->is_in_use = TRUE;\r
2208 \r
2209         IPOIB_EXIT( IPOIB_DBG_RECV );\r
2210         return IB_SUCCESS;\r
2211 }\r
2212 \r
2213 \r
2214 static ib_api_status_t\r
2215 __recv_dhcp(\r
2216         IN                              ipoib_port_t* const                     p_port,\r
2217         IN              const   ipoib_pkt_t* const                      p_ipoib,\r
2218                 OUT                     eth_pkt_t* const                        p_eth,\r
2219         IN                              ipoib_endpt_t* const            p_src,\r
2220         IN                              ipoib_endpt_t* const            p_dst )\r
2221 {\r
2222         ib_api_status_t         status;\r
2223         dhcp_pkt_t                      *p_dhcp;\r
2224         uint8_t                         *p_option;\r
2225         uint8_t                         *p_cid = NULL;\r
2226         uint8_t                         msg = 0;\r
2227 \r
2228         IPOIB_ENTER( IPOIB_DBG_RECV );\r
2229 \r
2230         UNUSED_PARAM( p_port );\r
2231 \r
2232         /* Create the ethernet header. */\r
2233         status = __recv_gen( p_ipoib, p_eth, p_src, p_dst );\r
2234         if( status != IB_SUCCESS )\r
2235         {\r
2236                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2237                         ("__recv_gen returned %s.\n", \r
2238                         p_port->p_adapter->p_ifc->get_err_str( status )) );\r
2239                 return status;\r
2240         }\r
2241 \r
2242         /* Fixup the payload. */\r
2243         p_dhcp = &p_eth->type.ip.prot.udp.dhcp;\r
2244         if( p_dhcp->op != DHCP_REQUEST && p_dhcp->op != DHCP_REPLY )\r
2245         {\r
2246                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2247                         ("Invalid DHCP op code.\n") );\r
2248                 return IB_INVALID_SETTING;\r
2249         }\r
2250 \r
2251         /*\r
2252          * Find the client identifier option, making sure to skip\r
2253          * the "magic cookie".\r
2254          */\r
2255         p_option = &p_dhcp->options[0];\r
2256         if ( *(uint32_t *)p_option != DHCP_COOKIE )\r
2257         {\r
2258                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2259                         ("DHCP cookie corrupted.\n") );\r
2260                 return IB_INVALID_PARAMETER;\r
2261         }\r
2262 \r
2263         p_option = &p_dhcp->options[DHCP_COOKIE_SIZE];\r
2264         while( *p_option != DHCP_OPT_END && p_option < &p_dhcp->options[312] )\r
2265         {\r
2266                 switch( *p_option )\r
2267                 {\r
2268                 case DHCP_OPT_PAD:\r
2269                         p_option++;\r
2270                         break;\r
2271 \r
2272                 case DHCP_OPT_MSG:\r
2273                         msg = p_option[2];\r
2274                         p_option += 3;\r
2275                         break;\r
2276 \r
2277                 case DHCP_OPT_CLIENT_ID:\r
2278                         p_cid = p_option;\r
2279                         /* Fall through. */\r
2280 \r
2281                 default:\r
2282                         /*\r
2283                          * All other options have a length byte following the option code.\r
2284                          * Offset by the length to get to the next option.\r
2285                          */\r
2286                         p_option += (p_option[1] + 2);\r
2287                 }\r
2288         }\r
2289 \r
2290         switch( msg )\r
2291         {\r
2292         /* message from client */\r
2293         case DHCPDISCOVER:\r
2294         case DHCPREQUEST:\r
2295         case DHCPDECLINE:\r
2296         case DHCPRELEASE:\r
2297         case DHCPINFORM:\r
2298                 if( !p_cid )\r
2299                 {\r
2300                         IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2301                                 ("Failed to find required Client-identifier option.\n") );\r
2302                         return IB_INVALID_SETTING;\r
2303                 }\r
2304                 if( p_dhcp->htype != DHCP_HW_TYPE_IB )\r
2305                 {\r
2306                         IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2307                                 ("Invalid hardware address type.\n") );\r
2308                         return IB_INVALID_SETTING;\r
2309                 }\r
2310                 break;\r
2311         /* message from DHCP server */\r
2312         case DHCPOFFER:\r
2313         case DHCPACK:\r
2314         case DHCPNAK:\r
2315                 break;\r
2316 \r
2317         default:\r
2318                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2319                         ("Invalide message type.\n") );\r
2320                 return IB_INVALID_PARAMETER;\r
2321         }\r
2322         p_eth->type.ip.prot.udp.hdr.chksum = 0;\r
2323         p_dhcp->htype = DHCP_HW_TYPE_ETH;\r
2324         p_dhcp->hlen = HW_ADDR_LEN;\r
2325 \r
2326         if( p_cid ) /* from client */\r
2327         {\r
2328                 /* Validate that the length and type of the option is as required. */\r
2329                 if( p_cid[1] != 21 )\r
2330                 {\r
2331                         IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2332                                 ("Client-identifier length not 21 as required.\n") );\r
2333                         return IB_INVALID_SETTING;\r
2334                 }\r
2335                 if( p_cid[2] != DHCP_HW_TYPE_IB )\r
2336                 {\r
2337                         IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2338                                 ("Client-identifier type is wrong.\n") );\r
2339                         return IB_INVALID_SETTING;\r
2340                 }\r
2341                 /*\r
2342                  * Copy the GID value from the option so that we can make aligned\r
2343                  * accesses to the contents.\r
2344                  * Recover CID to standard type.\r
2345                  */\r
2346                 p_cid[1] =  sizeof (ib_net64_t) + 1;// CID length \r
2347                 p_cid[2] =  DHCP_HW_TYPE_ETH;// CID type\r
2348                 RtlMoveMemory( &p_cid[3], &p_cid[15], sizeof (ib_net64_t) );\r
2349                 RtlFillMemory(&p_cid[11], 12, 0);\r
2350 \r
2351                 RtlCopyMemory( p_dhcp->chaddr, &p_src->mac, sizeof(p_src->mac) );\r
2352         }\r
2353         IPOIB_EXIT( IPOIB_DBG_RECV );\r
2354         return status;\r
2355 }\r
2356 \r
2357 \r
2358 static ib_api_status_t\r
2359 __recv_arp(\r
2360         IN                              ipoib_port_t* const                     p_port,\r
2361         IN                              ib_wc_t* const                          p_wc,\r
2362         IN              const   ipoib_pkt_t* const                      p_ipoib,\r
2363                 OUT                     eth_pkt_t* const                        p_eth,\r
2364         IN                              ipoib_endpt_t** const           pp_src,\r
2365         IN                              ipoib_endpt_t* const            p_dst )\r
2366 {\r
2367         ib_api_status_t                 status;\r
2368         arp_pkt_t                               *p_arp;\r
2369         const ipoib_arp_pkt_t   *p_ib_arp;\r
2370         ib_gid_t                                gid;\r
2371         mac_addr_t                              mac;\r
2372         ipoib_hw_addr_t                 null_hw = {0};\r
2373 \r
2374         IPOIB_ENTER( IPOIB_DBG_RECV );\r
2375 \r
2376         if( !p_dst )\r
2377         {\r
2378                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2379                         ("Unknown destination endpoint\n") );\r
2380                 return IB_INVALID_SETTING;\r
2381         }\r
2382 \r
2383         p_ib_arp = &p_ipoib->type.arp;\r
2384         p_arp = &p_eth->type.arp;\r
2385 \r
2386         if( p_ib_arp->hw_type != ARP_HW_TYPE_IB )\r
2387         {\r
2388                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2389                         ("ARP hardware type is not IB\n") );\r
2390                 return IB_INVALID_SETTING;\r
2391         }\r
2392 \r
2393         if( p_ib_arp->hw_size != sizeof(ipoib_hw_addr_t) )\r
2394         {\r
2395                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2396                         ("ARP hardware address size is not sizeof(ipoib_hw_addr_t)\n") );\r
2397                 return IB_INVALID_SETTING;\r
2398         }\r
2399 \r
2400         if( p_ib_arp->prot_type != ETH_PROT_TYPE_IP )\r
2401         {\r
2402                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2403                         ("ARP protocal type not IP\n") );\r
2404                 return IB_INVALID_SETTING;\r
2405         }\r
2406 \r
2407         /*\r
2408          * If we don't have a source, lookup the endpoint specified in the payload.\r
2409          */\r
2410         if( !*pp_src )\r
2411                 *pp_src = __endpt_mgr_get_by_gid( p_port, &p_ib_arp->src_hw.gid );\r
2412 \r
2413         /*\r
2414          * If the endpoint exists for the GID, make sure\r
2415          * the dlid and qpn match the arp.\r
2416          */\r
2417         if( *pp_src )\r
2418         {\r
2419                 if( cl_memcmp( &(*pp_src)->dgid, &p_ib_arp->src_hw.gid,\r
2420                         sizeof(ib_gid_t) ) )\r
2421                 {\r
2422                         /*\r
2423                          * GIDs for the endpoint are different.  The ARP must\r
2424                          * have been proxied.  Dereference it.\r
2425                          */\r
2426                         *pp_src = NULL;\r
2427                 }\r
2428                 else if( (*pp_src)->dlid &&\r
2429                         (*pp_src)->dlid != p_wc->recv.ud.remote_lid )\r
2430                 {\r
2431                         /* Out of date!  Destroy the endpoint and replace it. */\r
2432                         __endpt_mgr_remove( p_port, *pp_src );\r
2433                         *pp_src = NULL;\r
2434                 }\r
2435                 else if ( ! ((*pp_src)->dlid)) {\r
2436                         /* Out of date!  Destroy the endpoint and replace it. */\r
2437                         __endpt_mgr_remove( p_port, *pp_src );\r
2438                         *pp_src = NULL;\r
2439                 }\r
2440                 else if( ipoib_is_voltaire_router_gid( &(*pp_src)->dgid ) )\r
2441                 {\r
2442                         if( (*pp_src)->qpn !=\r
2443                                 (p_ib_arp->src_hw.flags_qpn & CL_HTON32(0x00FFFFFF)) &&\r
2444                                 p_wc->recv.ud.remote_qp !=\r
2445                                 (p_ib_arp->src_hw.flags_qpn & CL_HTON32(0x00FFFFFF)) )\r
2446                         {\r
2447                                 /* Out of date!  Destroy the endpoint and replace it. */\r
2448                                 __endpt_mgr_remove( p_port, *pp_src );\r
2449                                 *pp_src = NULL;\r
2450                         }\r
2451                 }\r
2452                 else if( (*pp_src)->qpn != p_wc->recv.ud.remote_qp )\r
2453                 {\r
2454                         /* Out of date!  Destroy the endpoint and replace it. */\r
2455                         __endpt_mgr_remove( p_port, *pp_src );\r
2456                         *pp_src = NULL;\r
2457                 }\r
2458         }\r
2459 \r
2460         /* Do we need to create an endpoint for this GID? */\r
2461         if( !*pp_src )\r
2462         {\r
2463                 /* Copy the src GID to allow aligned access */\r
2464                 cl_memcpy( &gid, &p_ib_arp->src_hw.gid, sizeof(ib_gid_t) );\r
2465                 status = ipoib_mac_from_guid( gid.unicast.interface_id, p_port->p_adapter->params.guid_mask, &mac );\r
2466                 if (status == IB_INVALID_GUID_MASK)\r
2467                 {\r
2468                         IPOIB_PRINT( TRACE_LEVEL_WARNING, IPOIB_DBG_ERROR,\r
2469                                 ("Invalid GUID mask received, rejecting it") );\r
2470                         ipoib_create_log(p_port->p_adapter->h_adapter, GUID_MASK_LOG_INDEX, EVENT_IPOIB_WRONG_PARAMETER_WRN);\r
2471                 }\r
2472                 else if( status != IB_SUCCESS )\r
2473                 {\r
2474                         IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2475                                 ("ipoib_mac_from_guid returned %s\n",\r
2476                                 p_port->p_adapter->p_ifc->get_err_str( status )) );\r
2477                         return status;\r
2478                 }\r
2479                 /*\r
2480                  * Create the endpoint.\r
2481                  */\r
2482                 *pp_src = ipoib_endpt_create( &p_ib_arp->src_hw.gid,\r
2483                         p_wc->recv.ud.remote_lid, (p_ib_arp->src_hw.flags_qpn & CL_HTON32(0x00FFFFFF)) );\r
2484 \r
2485                 if( !*pp_src )\r
2486                 {\r
2487                         IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2488                                 ("ipoib_endpt_create failed\n") );\r
2489                         return status;\r
2490                 }\r
2491 \r
2492                 cl_obj_lock( &p_port->obj );\r
2493                 status = __endpt_mgr_insert( p_port, mac, *pp_src );\r
2494                 if( status != IB_SUCCESS )\r
2495                 {\r
2496                         cl_obj_unlock( &p_port->obj );\r
2497                         IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2498                                 ("__endpt_mgr_insert return %s \n",\r
2499                                 p_port->p_adapter->p_ifc->get_err_str( status )) );\r
2500                         return status;\r
2501                 }\r
2502 \r
2503                 cl_obj_unlock( &p_port->obj );\r
2504         }\r
2505 \r
2506         CL_ASSERT( !cl_memcmp(\r
2507                 &(*pp_src)->dgid, &p_ib_arp->src_hw.gid, sizeof(ib_gid_t) ) );\r
2508         CL_ASSERT( ipoib_is_voltaire_router_gid( &(*pp_src)->dgid ) ||\r
2509                 (*pp_src)->qpn ==\r
2510                 (p_ib_arp->src_hw.flags_qpn & CL_HTON32(0x00FFFFFF)) );\r
2511         /* Now swizzle the data. */\r
2512         p_arp->hw_type = ARP_HW_TYPE_ETH;\r
2513         p_arp->hw_size = sizeof(mac_addr_t);\r
2514         p_arp->src_hw = (*pp_src)->mac;\r
2515         p_arp->src_ip = p_ib_arp->src_ip;\r
2516 \r
2517         if( cl_memcmp( &p_ib_arp->dst_hw, &null_hw, sizeof(ipoib_hw_addr_t) ) )\r
2518         {\r
2519                 if( cl_memcmp( &p_dst->dgid, &p_ib_arp->dst_hw.gid, sizeof(ib_gid_t) ) )\r
2520                 {\r
2521                         /*\r
2522                          * We received bcast ARP packet that means\r
2523                          * remote port lets everyone know it was changed IP/MAC\r
2524                          * or just activated\r
2525                          */\r
2526 \r
2527                         /* Guy: TODO: Check why this check fails in case of Voltaire IPR */\r
2528 \r
2529                         if ( !ipoib_is_voltaire_router_gid( &(*pp_src)->dgid ) &&\r
2530                                  !ib_gid_is_multicast( (const ib_gid_t*)&p_dst->dgid ) )\r
2531                         {\r
2532                                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2533                                         ("ARP: is not ARP MCAST\n") );\r
2534                                 return IB_INVALID_SETTING;\r
2535                         }\r
2536 \r
2537                         p_arp->dst_hw = p_port->p_local_endpt->mac;\r
2538                         p_dst->mac = p_port->p_local_endpt->mac;\r
2539                         /*\r
2540                          * we don't care what receiver ip addr is,\r
2541                          * as long as OS' ARP table is global  ???\r
2542                          */\r
2543                         p_arp->dst_ip = (net32_t)0;\r
2544                 }\r
2545                 else /* we've got reply to our ARP request */\r
2546                 {\r
2547                         p_arp->dst_hw = p_dst->mac;\r
2548                         p_arp->dst_ip = p_ib_arp->dst_ip;\r
2549                         CL_ASSERT( p_dst->qpn == \r
2550                                 (p_ib_arp->dst_hw.flags_qpn & CL_HTON32(0x00FFFFFF)) );\r
2551                 }\r
2552         }\r
2553         else /* we got ARP reqeust */\r
2554         {\r
2555                 cl_memclr( &p_arp->dst_hw, sizeof(mac_addr_t) );\r
2556                 p_arp->dst_ip = p_ib_arp->dst_ip;\r
2557         }\r
2558 \r
2559         /*\r
2560          * Create the ethernet header.  Note that this is done last so that\r
2561          * we have a chance to create a new endpoint.\r
2562          */\r
2563         status = __recv_gen( p_ipoib, p_eth, *pp_src, p_dst );\r
2564         if( status != IB_SUCCESS )\r
2565         {\r
2566                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2567                         ("__recv_gen returned %s.\n", \r
2568                         p_port->p_adapter->p_ifc->get_err_str( status )) );\r
2569                 return status;\r
2570         }\r
2571 \r
2572         IPOIB_EXIT( IPOIB_DBG_RECV );\r
2573         return IB_SUCCESS;\r
2574 }\r
2575 \r
2576 \r
2577 static ib_api_status_t\r
2578 __recv_mgr_prepare_pkt(\r
2579         IN                              ipoib_port_t* const                     p_port,\r
2580         IN                              ipoib_recv_desc_t* const        p_desc,\r
2581                 OUT                     NDIS_PACKET** const                     pp_packet )\r
2582 {\r
2583         NDIS_STATUS                                                     status;\r
2584         uint32_t                                                        pkt_filter;\r
2585         ip_stat_sel_t                                           type;\r
2586         NDIS_TCP_IP_CHECKSUM_PACKET_INFO        chksum;\r
2587 \r
2588         PERF_DECLARE( GetNdisPkt );\r
2589 \r
2590         IPOIB_ENTER( IPOIB_DBG_RECV );\r
2591 \r
2592         pkt_filter = p_port->p_adapter->packet_filter;\r
2593         /* Check the packet filter. */\r
2594         switch( p_desc->type )\r
2595         {\r
2596         default:\r
2597         case PKT_TYPE_UCAST:\r
2598                 if( pkt_filter & NDIS_PACKET_TYPE_PROMISCUOUS ||\r
2599                         pkt_filter & NDIS_PACKET_TYPE_ALL_FUNCTIONAL ||\r
2600                         pkt_filter & NDIS_PACKET_TYPE_SOURCE_ROUTING ||\r
2601                         pkt_filter & NDIS_PACKET_TYPE_DIRECTED )\r
2602                 {\r
2603                         /* OK to report. */\r
2604                         type = IP_STAT_UCAST_BYTES;\r
2605                         status = NDIS_STATUS_SUCCESS;\r
2606                 }\r
2607                 else\r
2608                 {\r
2609                         type = IP_STAT_DROPPED;\r
2610                         status = NDIS_STATUS_FAILURE;\r
2611                 }\r
2612                 break;\r
2613         case PKT_TYPE_BCAST:\r
2614                 if( pkt_filter & NDIS_PACKET_TYPE_PROMISCUOUS ||\r
2615                         pkt_filter & NDIS_PACKET_TYPE_BROADCAST )\r
2616                 {\r
2617                         /* OK to report. */\r
2618                         type = IP_STAT_BCAST_BYTES;\r
2619                         status = NDIS_STATUS_SUCCESS;\r
2620                 }\r
2621                 else\r
2622                 {\r
2623                         type = IP_STAT_DROPPED;\r
2624                         status = NDIS_STATUS_FAILURE;\r
2625                 }\r
2626                 break;\r
2627         case PKT_TYPE_MCAST:\r
2628                 if( pkt_filter & NDIS_PACKET_TYPE_PROMISCUOUS ||\r
2629                         pkt_filter & NDIS_PACKET_TYPE_ALL_MULTICAST ||\r
2630                         pkt_filter & NDIS_PACKET_TYPE_MULTICAST )\r
2631                 {\r
2632                         /* OK to report. */\r
2633                         type = IP_STAT_MCAST_BYTES;\r
2634                         status = NDIS_STATUS_SUCCESS;\r
2635                 }\r
2636                 else\r
2637                 {\r
2638                         type = IP_STAT_DROPPED;\r
2639                         status = NDIS_STATUS_FAILURE;\r
2640                 }\r
2641                 break;\r
2642         }\r
2643 \r
2644         if( status != NDIS_STATUS_SUCCESS )\r
2645         {\r
2646                 ipoib_inc_recv_stat( p_port->p_adapter, type, 0 );\r
2647                 /* Return the receive descriptor to the pool. */\r
2648                 __buf_mgr_put_recv( p_port, p_desc, NULL );\r
2649                 IPOIB_PRINT_EXIT( TRACE_LEVEL_INFORMATION, IPOIB_DBG_RECV,\r
2650                         ("Packet filter doesn't match receive.  Dropping.\n") );\r
2651                 /*\r
2652                  * Return IB_NOT_DONE since the packet has been completed,\r
2653                  * but has not consumed an array entry.\r
2654                  */\r
2655                 return IB_NOT_DONE;\r
2656         }\r
2657 \r
2658         cl_perf_start( GetNdisPkt );\r
2659         *pp_packet = __buf_mgr_get_ndis_pkt( p_port, p_desc );\r
2660         cl_perf_stop( &p_port->p_adapter->perf, GetNdisPkt );\r
2661         if( !*pp_packet )\r
2662         {\r
2663                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2664                         ("__buf_mgr_get_ndis_pkt failed\n") );\r
2665                 return IB_INSUFFICIENT_RESOURCES;\r
2666         }\r
2667 \r
2668         chksum.Value = 0;\r
2669         switch (p_port->p_adapter->params.recv_chksum_offload) {\r
2670           case CSUM_DISABLED:\r
2671                 NDIS_PER_PACKET_INFO_FROM_PACKET( *pp_packet, TcpIpChecksumPacketInfo ) =\r
2672                 (void*)(uintn_t)chksum.Value;\r
2673                 break;\r
2674           case CSUM_ENABLED:\r
2675                 /* Get the checksums directly from packet information. */\r
2676                 /* In this case, no one of cheksum's cat get false value */\r
2677                 /* If hardware checksum failed or wasn't calculated, NDIS will recalculate it again */\r
2678                 NDIS_PER_PACKET_INFO_FROM_PACKET( *pp_packet, TcpIpChecksumPacketInfo ) = \r
2679                         (void*)(uintn_t)(p_desc->ndis_csum.Value);\r
2680                 break;\r
2681           case CSUM_BYPASS:\r
2682                 /* Flag the checksums as having been calculated. */\r
2683                 chksum.Receive.NdisPacketTcpChecksumSucceeded = TRUE;\r
2684                 chksum.Receive.NdisPacketUdpChecksumSucceeded = TRUE;\r
2685                 chksum.Receive.NdisPacketIpChecksumSucceeded = TRUE;\r
2686                 NDIS_PER_PACKET_INFO_FROM_PACKET( *pp_packet, TcpIpChecksumPacketInfo ) =\r
2687                 (void*)(uintn_t)chksum.Value;\r
2688                 break;\r
2689           default:\r
2690                 ASSERT(FALSE);\r
2691                 NDIS_PER_PACKET_INFO_FROM_PACKET( *pp_packet, TcpIpChecksumPacketInfo ) =\r
2692                 (void*)(uintn_t)chksum.Value;\r
2693         }\r
2694         ipoib_inc_recv_stat( p_port->p_adapter, type, p_desc->len );\r
2695 \r
2696         IPOIB_EXIT( IPOIB_DBG_RECV );\r
2697         return IB_SUCCESS;\r
2698 }\r
2699 \r
2700 \r
2701 static uint32_t\r
2702 __recv_mgr_build_pkt_array(\r
2703         IN                              ipoib_port_t* const                     p_port,\r
2704         IN                              int32_t                                         shortage,\r
2705                 OUT                     cl_qlist_t* const                       p_done_list,\r
2706                 OUT                     int32_t* const                          p_discarded )\r
2707 {\r
2708         cl_list_item_t                  *p_item;\r
2709         ipoib_recv_desc_t               *p_desc;\r
2710         uint32_t                                i = 0;\r
2711         ib_api_status_t                 status;\r
2712         PERF_DECLARE( PreparePkt );\r
2713 \r
2714         IPOIB_ENTER( IPOIB_DBG_RECV );\r
2715 \r
2716         *p_discarded = 0;\r
2717 \r
2718         /* Move any existing receives to the head to preserve ordering. */\r
2719         cl_qlist_insert_list_head( p_done_list, &p_port->recv_mgr.done_list );\r
2720         p_item = cl_qlist_remove_head( p_done_list );\r
2721         while( p_item != cl_qlist_end( p_done_list ) )\r
2722         {\r
2723                 p_desc = (ipoib_recv_desc_t*)p_item;\r
2724 \r
2725                 cl_perf_start( PreparePkt );\r
2726                 status = __recv_mgr_prepare_pkt( p_port, p_desc,\r
2727                         &p_port->recv_mgr.recv_pkt_array[i] );\r
2728                 cl_perf_stop( &p_port->p_adapter->perf, PreparePkt );\r
2729                 if( status == IB_SUCCESS )\r
2730                 {\r
2731                         CL_ASSERT( p_port->recv_mgr.recv_pkt_array[i] );\r
2732                         if( shortage-- > 0 )\r
2733                         {\r
2734                                 NDIS_SET_PACKET_STATUS(\r
2735                                         p_port->recv_mgr.recv_pkt_array[i], NDIS_STATUS_RESOURCES );\r
2736                         }\r
2737                         else\r
2738                         {\r
2739                                 NDIS_SET_PACKET_STATUS(\r
2740                                         p_port->recv_mgr.recv_pkt_array[i], NDIS_STATUS_SUCCESS );\r
2741                         }\r
2742                         i++;\r
2743                 }\r
2744                 else if( status == IB_NOT_DONE )\r
2745                 {\r
2746                         (*p_discarded)++;\r
2747                 }\r
2748                 else\r
2749                 {\r
2750                         IPOIB_PRINT(TRACE_LEVEL_INFORMATION, IPOIB_DBG_RECV,\r
2751                                 ("__recv_mgr_prepare_pkt returned %s\n",\r
2752                                 p_port->p_adapter->p_ifc->get_err_str( status )) );\r
2753                         /* Put all completed receives on the port's done list. */\r
2754                         cl_qlist_insert_tail( &p_port->recv_mgr.done_list, p_item );\r
2755                         cl_qlist_insert_list_tail( &p_port->recv_mgr.done_list, p_done_list );\r
2756                         break;\r
2757                 }\r
2758 \r
2759                 p_item = cl_qlist_remove_head( p_done_list );\r
2760         }\r
2761 \r
2762         IPOIB_EXIT( IPOIB_DBG_RECV );\r
2763         return i;\r
2764 }\r
2765 \r
2766 \r
2767 \r
2768 \r
2769 /******************************************************************************\r
2770 *\r
2771 * Send manager implementation.\r
2772 *\r
2773 ******************************************************************************/\r
2774 static void\r
2775 __send_mgr_construct(\r
2776         IN                              ipoib_port_t* const                     p_port )\r
2777 {\r
2778         IPOIB_ENTER( IPOIB_DBG_SEND );\r
2779         p_port->send_mgr.depth = 0;\r
2780         cl_qlist_init( &p_port->send_mgr.pending_list );\r
2781         IPOIB_EXIT( IPOIB_DBG_SEND );\r
2782 }\r
2783 \r
2784 \r
2785 static void \r
2786 __pending_list_destroy(\r
2787         IN                              ipoib_port_t* const                     p_port )\r
2788 {\r
2789         cl_list_item_t  *p_item;\r
2790         NDIS_PACKET             *p_packet;\r
2791         \r
2792         cl_spinlock_acquire( &p_port->send_lock );\r
2793         /* Complete any pending packets. */\r
2794         for( p_item = cl_qlist_remove_head( &p_port->send_mgr.pending_list );\r
2795                 p_item != cl_qlist_end( &p_port->send_mgr.pending_list );\r
2796                 p_item = cl_qlist_remove_head( &p_port->send_mgr.pending_list ) )\r
2797         {\r
2798                 p_packet = IPOIB_PACKET_FROM_LIST_ITEM( p_item );\r
2799                 NdisMSendCompleteX( p_port->p_adapter->h_adapter, p_packet,\r
2800                         NDIS_STATUS_RESET_IN_PROGRESS );\r
2801         }\r
2802         cl_spinlock_release( &p_port->send_lock );\r
2803 }\r
2804 \r
2805 static void\r
2806 __send_mgr_destroy(\r
2807         IN                              ipoib_port_t* const                     p_port )\r
2808 {\r
2809         IPOIB_ENTER( IPOIB_DBG_SEND );\r
2810         __pending_list_destroy(p_port);\r
2811 \r
2812         IPOIB_EXIT( IPOIB_DBG_SEND );\r
2813 }\r
2814 \r
2815 \r
2816 static NDIS_STATUS\r
2817 __send_mgr_filter(\r
2818         IN                              ipoib_port_t* const                     p_port,\r
2819         IN              const   eth_hdr_t* const                        p_eth_hdr,\r
2820         IN                              NDIS_BUFFER* const                      p_buf,\r
2821         IN                              size_t                                          buf_len,\r
2822         IN      OUT                     ipoib_send_desc_t* const        p_desc )\r
2823 {\r
2824         NDIS_STATUS             status;\r
2825 \r
2826         PERF_DECLARE( FilterIp );\r
2827         PERF_DECLARE( FilterArp );\r
2828         PERF_DECLARE( SendGen );\r
2829 \r
2830         IPOIB_ENTER( IPOIB_DBG_SEND );\r
2831 \r
2832         /*\r
2833          * We already checked the ethernet header length, so we know it's safe\r
2834          * to decrement the buf_len without underflowing.\r
2835          */\r
2836         buf_len -= sizeof(eth_hdr_t);\r
2837 \r
2838         switch( p_eth_hdr->type )\r
2839         {\r
2840         case ETH_PROT_TYPE_IP:\r
2841                 cl_perf_start( FilterIp );\r
2842                 status = __send_mgr_filter_ip(\r
2843                         p_port, p_eth_hdr, p_buf, buf_len, p_desc );\r
2844                 cl_perf_stop( &p_port->p_adapter->perf, FilterIp );\r
2845                 break;\r
2846 \r
2847         case ETH_PROT_TYPE_ARP:\r
2848                 cl_perf_start( FilterArp );\r
2849                 status = __send_mgr_filter_arp(\r
2850                         p_port, p_eth_hdr, p_buf, buf_len, p_desc );\r
2851                 cl_perf_stop( &p_port->p_adapter->perf, FilterArp );\r
2852                 break;\r
2853 \r
2854         default:\r
2855                 /*\r
2856                  * The IPoIB spec doesn't define how to send non IP or ARP packets.\r
2857                  * Just send the payload and hope for the best.\r
2858                  */\r
2859                 cl_perf_start( SendGen );\r
2860                 status = __send_gen( p_port, p_desc, 0 );\r
2861                 cl_perf_stop( &p_port->p_adapter->perf, SendGen );\r
2862                 break;\r
2863         }\r
2864 \r
2865         IPOIB_EXIT( IPOIB_DBG_SEND );\r
2866         return status;\r
2867 }\r
2868 \r
2869 \r
2870 static NDIS_STATUS\r
2871 __send_copy(\r
2872         IN                              ipoib_port_t* const                     p_port,\r
2873         IN                              ipoib_send_desc_t* const        p_desc )\r
2874 {\r
2875         NDIS_PACKET                             *p_packet;\r
2876         NDIS_BUFFER                             *p_buf;\r
2877         NDIS_STATUS                             status;\r
2878         UINT                                    tot_len, bytes_copied = 0;\r
2879 \r
2880         IPOIB_ENTER( IPOIB_DBG_SEND );\r
2881 \r
2882         p_desc->p_buf = \r
2883                 ExAllocateFromNPagedLookasideList( &p_port->buf_mgr.send_buf_list );\r
2884         if( !p_desc->p_buf )\r
2885         {\r
2886                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2887                         ("Failed to allocate buffer for packet copy.\n") );\r
2888                 return NDIS_STATUS_RESOURCES;\r
2889         }\r
2890 \r
2891         NdisAllocatePacket( &status, &p_packet, p_port->buf_mgr.h_send_pkt_pool );\r
2892         if( status != NDIS_STATUS_SUCCESS )\r
2893         {\r
2894                 IPOIB_PRINT_EXIT( TRACE_LEVEL_WARNING, IPOIB_DBG_SEND,\r
2895                         ("Failed to allocate NDIS_PACKET for copy.\n") );\r
2896                 return status;\r
2897         }\r
2898 \r
2899         NdisAllocateBuffer( &status, &p_buf, p_port->buf_mgr.h_send_buf_pool,\r
2900                 p_desc->p_buf, p_port->p_adapter->params.xfer_block_size );\r
2901         if( status != NDIS_STATUS_SUCCESS )\r
2902         {\r
2903                 NdisFreePacket( p_packet );\r
2904                 IPOIB_PRINT_EXIT( TRACE_LEVEL_WARNING, IPOIB_DBG_SEND,\r
2905                         ("Failed to allocate NDIS_BUFFER for copy.\n") );\r
2906                 return status;\r
2907         }\r
2908 \r
2909         NdisChainBufferAtFront( p_packet, p_buf );\r
2910 \r
2911         NdisQueryPacketLength( p_desc->p_pkt, &tot_len );\r
2912 \r
2913         /* Setup the work request. */\r
2914         p_desc->local_ds[1].vaddr = cl_get_physaddr(\r
2915                 ((uint8_t*)p_desc->p_buf) + sizeof(eth_hdr_t) );\r
2916         p_desc->local_ds[1].length = tot_len - sizeof(eth_hdr_t);\r
2917         p_desc->local_ds[1].lkey = p_port->ib_mgr.lkey;\r
2918         p_desc->wr.num_ds = 2;\r
2919 \r
2920         /* Copy the packet. */\r
2921         NdisCopyFromPacketToPacketSafe( p_packet, bytes_copied, tot_len,\r
2922                 p_desc->p_pkt, bytes_copied, &bytes_copied,\r
2923                 NormalPagePriority );\r
2924 \r
2925         /* Free our temp packet now that the data is copied. */\r
2926         NdisUnchainBufferAtFront( p_packet, &p_buf );\r
2927         NdisFreeBuffer( p_buf );\r
2928         NdisFreePacket( p_packet );\r
2929 \r
2930         if( bytes_copied != tot_len )\r
2931         {\r
2932                 /* Something went wrong.  Drop the packet. */\r
2933                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2934                         ("Failed to copy full packet: %d of %d bytes copied.\n",\r
2935                         bytes_copied, tot_len) );\r
2936                 return NDIS_STATUS_RESOURCES;\r
2937         }\r
2938 \r
2939         IPOIB_EXIT( IPOIB_DBG_SEND );\r
2940         return NDIS_STATUS_SUCCESS;\r
2941 }\r
2942 \r
2943 \r
2944 #if !IPOIB_USE_DMA\r
2945 /* Send using the MDL's page information rather than the SGL. */\r
2946 static ib_api_status_t\r
2947 __send_gen(\r
2948         IN                              ipoib_port_t* const                     p_port,\r
2949         IN                              ipoib_send_desc_t* const        p_desc )\r
2950 {\r
2951         uint32_t                                i, j = 1;\r
2952         ULONG                                   offset;\r
2953         MDL                                             *p_mdl;\r
2954         UINT                                    num_pages, tot_len;\r
2955         ULONG                                   buf_len;\r
2956         PPFN_NUMBER                             page_array;\r
2957         boolean_t                               hdr_done = FALSE;\r
2958         ib_api_status_t                 status;\r
2959 \r
2960         IPOIB_ENTER( IPOIB_DBG_SEND );\r
2961 \r
2962         NdisQueryPacket( p_desc->p_pkt, &num_pages, NULL, &p_mdl,\r
2963                 &tot_len );\r
2964 \r
2965         if( !p_mdl )\r
2966         {\r
2967                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2968                         ("No buffers associated with packet.\n") );\r
2969                 return IB_ERROR;\r
2970         }\r
2971 \r
2972         /* Remember that one of the DS entries is reserved for the IPoIB header. */\r
2973         if( num_pages >= MAX_SEND_SGE )\r
2974         {\r
2975                 IPOIB_PRINT(TRACE_LEVEL_INFORMATION, IPOIB_DBG_SEND,\r
2976                         ("Too many buffers to fit in WR ds_array.  Copying data.\n") );\r
2977                 status = __send_copy( p_port, p_desc );\r
2978                 IPOIB_EXIT( IPOIB_DBG_SEND );\r
2979                 return status;\r
2980         }\r
2981 \r
2982         CL_ASSERT( tot_len > sizeof(eth_hdr_t) );\r
2983         CL_ASSERT( tot_len <= p_port->p_adapter->params.xfer_block_size );\r
2984         /*\r
2985          * Assume that the ethernet header is always fully contained\r
2986          * in the first page of the first MDL.  This makes for much\r
2987          * simpler code.\r
2988          */\r
2989         offset = MmGetMdlByteOffset( p_mdl ) + sizeof(eth_hdr_t);\r
2990         CL_ASSERT( offset <= PAGE_SIZE );\r
2991 \r
2992         while( tot_len )\r
2993         {\r
2994                 buf_len = MmGetMdlByteCount( p_mdl );\r
2995                 page_array = MmGetMdlPfnArray( p_mdl );\r
2996                 CL_ASSERT( page_array );\r
2997                 i = 0;\r
2998                 if( !hdr_done )\r
2999                 {\r
3000                         CL_ASSERT( buf_len >= sizeof(eth_hdr_t) );\r
3001                         /* Skip the ethernet header. */\r
3002                         buf_len -= sizeof(eth_hdr_t);\r
3003                         CL_ASSERT( buf_len <= p_port->p_adapter->params.payload_mtu );\r
3004                         if( buf_len )\r
3005                         {\r
3006                                 /* The ethernet header is a subset of this MDL. */\r
3007                                 CL_ASSERT( i == 0 );\r
3008                                 if( offset < PAGE_SIZE )\r
3009                                 {\r
3010                                         p_desc->local_ds[j].lkey = p_port->ib_mgr.lkey;\r
3011                                         p_desc->local_ds[j].vaddr = (page_array[i] << PAGE_SHIFT);\r
3012                                         /* Add the byte offset since we're on the 1st page. */\r
3013                                         p_desc->local_ds[j].vaddr += offset;\r
3014                                         if( offset + buf_len > PAGE_SIZE )\r
3015                                         {\r
3016                                                 p_desc->local_ds[j].length = PAGE_SIZE - offset;\r