[ipoix] Fix two races where ipoib_port_up was called twice.
[mirror/winof/.git] / ulp / ipoib / kernel / ipoib_port.c
1 /*\r
2  * Copyright (c) 2005 SilverStorm Technologies.  All rights reserved.\r
3  * Copyright (c) 2006 Mellanox Technologies.  All rights reserved.\r
4  * Portions Copyright (c) 2008 Microsoft Corporation.  All rights reserved.\r
5  *\r
6  * This software is available to you under the OpenIB.org BSD license\r
7  * below:\r
8  *\r
9  *     Redistribution and use in source and binary forms, with or\r
10  *     without modification, are permitted provided that the following\r
11  *     conditions are met:\r
12  *\r
13  *      - Redistributions of source code must retain the above\r
14  *        copyright notice, this list of conditions and the following\r
15  *        disclaimer.\r
16  *\r
17  *      - Redistributions in binary form must reproduce the above\r
18  *        copyright notice, this list of conditions and the following\r
19  *        disclaimer in the documentation and/or other materials\r
20  *        provided with the distribution.\r
21  *\r
22  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,\r
23  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF\r
24  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND\r
25  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS\r
26  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN\r
27  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN\r
28  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\r
29  * SOFTWARE.\r
30  *\r
31  * $Id$\r
32  */\r
33 \r
34 \r
35 \r
36 #include "ipoib_port.h"\r
37 #include "ipoib_adapter.h"\r
38 #include "ipoib_debug.h"\r
39 #if defined(EVENT_TRACING)\r
40 #ifdef offsetof\r
41 #undef offsetof\r
42 #endif\r
43 #include "ipoib_port.tmh"\r
44 #endif\r
45 #include <offload.h>\r
46 \r
47 \r
48 /* Amount of physical memory to register. */\r
49 #define MEM_REG_SIZE    0xFFFFFFFFFFFFFFFF\r
50 \r
51 /* Number of work completions to chain for send and receive polling. */\r
52 #define MAX_SEND_WC             8\r
53 #define MAX_RECV_WC             16\r
54 \r
55 \r
56 ib_gid_t        bcast_mgid_template = {\r
57         0xff,                                                           /* multicast field */\r
58         0x12,                                                           /* scope (to be filled in) */\r
59         0x40, 0x1b,                                                     /* IPv4 signature */\r
60         0xff, 0xff,                                                     /* 16 bits of P_Key (to be filled in) */\r
61         0x00, 0x00, 0x00, 0x00, 0x00, 0x00,     /* 48 bits of zeros */\r
62         0xff, 0xff, 0xff, 0xff,                         /* 32 bit IPv4 broadcast address */\r
63 };\r
64 \r
65 \r
66 #ifdef _DEBUG_\r
67 /* Handy pointer for debug use. */\r
68 ipoib_port_t    *gp_ipoib_port;\r
69 #endif\r
70 \r
71 static void __port_mcast_garbage_dpc(KDPC *p_gc_dpc,void *context,void *s_arg1, void *s_arg2);\r
72 static void __port_do_mcast_garbage(ipoib_port_t* const p_port );\r
73 /******************************************************************************\r
74 *\r
75 * Declarations\r
76 *\r
77 ******************************************************************************/\r
78 static void\r
79 __port_construct(\r
80         IN                              ipoib_port_t* const                     p_port );\r
81 \r
82 static ib_api_status_t\r
83 __port_init(\r
84         IN                              ipoib_port_t* const                     p_port,\r
85         IN                              ipoib_adapter_t* const          p_adapter,\r
86         IN                              ib_pnp_port_rec_t* const        p_pnp_rec );\r
87 \r
88 static void\r
89 __port_destroying(\r
90         IN                              cl_obj_t* const                         p_obj );\r
91 \r
92 static void\r
93 __port_cleanup(\r
94         IN                              cl_obj_t* const                         p_obj );\r
95 \r
96 static void\r
97 __port_free(\r
98         IN                              cl_obj_t* const                         p_obj );\r
99 \r
100 \r
101 /******************************************************************************\r
102 *\r
103 * IB resource manager operations\r
104 *\r
105 ******************************************************************************/\r
106 static void\r
107 __ib_mgr_construct(\r
108         IN                              ipoib_port_t* const                     p_port );\r
109 \r
110 static ib_api_status_t\r
111 __ib_mgr_init(\r
112         IN                              ipoib_port_t* const                     p_port );\r
113 \r
114 static void\r
115 __ib_mgr_destroy(\r
116         IN                              ipoib_port_t* const                     p_port );\r
117 \r
118 static void\r
119 __qp_event(\r
120         IN                              ib_async_event_rec_t            *p_event_rec );\r
121 \r
122 static void\r
123 __cq_event(\r
124         IN                              ib_async_event_rec_t            *p_event_rec );\r
125 \r
126 static ib_api_status_t\r
127 __ib_mgr_activate(\r
128         IN                              ipoib_port_t* const                     p_port );\r
129 \r
130 /******************************************************************************\r
131 *\r
132 * Buffer manager operations.\r
133 *\r
134 ******************************************************************************/\r
135 static void\r
136 __buf_mgr_construct(\r
137         IN                              ipoib_port_t* const                     p_port );\r
138 \r
139 static ib_api_status_t\r
140 __buf_mgr_init(\r
141         IN                              ipoib_port_t* const                     p_port );\r
142 \r
143 static void\r
144 __buf_mgr_destroy(\r
145         IN                              ipoib_port_t* const                     p_port );\r
146 \r
147 static cl_status_t\r
148 __recv_ctor(\r
149         IN                              void* const                                     p_object,\r
150         IN                              void*                                           context,\r
151                 OUT                     cl_pool_item_t** const          pp_pool_item );\r
152 \r
153 #if !IPOIB_INLINE_RECV\r
154 static void\r
155 __recv_dtor(\r
156         IN              const   cl_pool_item_t* const           p_pool_item,\r
157         IN                              void                                            *context );\r
158 #endif  /* IPOIB_INLINE_RECV */\r
159 \r
160 static inline ipoib_send_desc_t*\r
161 __buf_mgr_get_send(\r
162         IN                              ipoib_port_t* const                     p_port );\r
163 \r
164 static inline void\r
165 __buf_mgr_put_send(\r
166         IN                              ipoib_port_t* const                     p_port,\r
167         IN                              ipoib_send_desc_t* const        p_desc );\r
168 \r
169 static inline ipoib_recv_desc_t*\r
170 __buf_mgr_get_recv(\r
171         IN                              ipoib_port_t* const                     p_port );\r
172 \r
173 static inline void\r
174 __buf_mgr_put_recv(\r
175         IN                              ipoib_port_t* const                     p_port,\r
176         IN                              ipoib_recv_desc_t* const        p_desc,\r
177         IN                              NDIS_PACKET* const                      p_packet OPTIONAL );\r
178 \r
179 static inline void\r
180 __buf_mgr_put_recv_list(\r
181         IN                              ipoib_port_t* const                     p_port,\r
182         IN                              cl_qlist_t* const                       p_list );\r
183 \r
184 static inline NDIS_PACKET*\r
185 __buf_mgr_get_ndis_pkt(\r
186         IN                              ipoib_port_t* const                     p_port,\r
187         IN                              ipoib_recv_desc_t* const        p_desc );\r
188 \r
189 \r
190 /******************************************************************************\r
191 *\r
192 * Receive manager operations.\r
193 *\r
194 ******************************************************************************/\r
195 static void\r
196 __recv_mgr_construct(\r
197         IN                              ipoib_port_t* const                     p_port );\r
198 \r
199 static ib_api_status_t\r
200 __recv_mgr_init(\r
201         IN                              ipoib_port_t* const                     p_port );\r
202 \r
203 static void\r
204 __recv_mgr_destroy(\r
205         IN                              ipoib_port_t* const                     p_port );\r
206 \r
207 /* Posts receive buffers to the receive queue. */\r
208 static ib_api_status_t\r
209 __recv_mgr_repost(\r
210         IN                              ipoib_port_t* const                     p_port );\r
211 \r
212 static void\r
213 __recv_cb(\r
214         IN              const   ib_cq_handle_t                          h_cq,\r
215         IN                              void                                            *cq_context );\r
216 \r
217 static void\r
218 __recv_get_endpts(\r
219         IN                              ipoib_port_t* const                     p_port,\r
220         IN                              ipoib_recv_desc_t* const        p_desc,\r
221         IN                              ib_wc_t* const                          p_wc,\r
222                 OUT                     ipoib_endpt_t** const           pp_src,\r
223                 OUT                     ipoib_endpt_t** const           pp_dst );\r
224 \r
225 static int32_t\r
226 __recv_mgr_filter(\r
227         IN                              ipoib_port_t* const                     p_port,\r
228         IN                              ib_wc_t* const                          p_done_wc_list,\r
229                 OUT                     cl_qlist_t* const                       p_done_list,\r
230                 OUT                     cl_qlist_t* const                       p_bad_list );\r
231 \r
232 static ib_api_status_t\r
233 __recv_gen(\r
234         IN              const   ipoib_pkt_t* const                      p_ipoib,\r
235                 OUT                     eth_pkt_t* const                        p_eth,\r
236         IN                              ipoib_endpt_t* const            p_src,\r
237         IN                              ipoib_endpt_t* const            p_dst );\r
238 \r
239 static ib_api_status_t\r
240 __recv_dhcp(\r
241         IN                              ipoib_port_t* const                     p_port,\r
242         IN              const   ipoib_pkt_t* const                      p_ipoib,\r
243                 OUT                     eth_pkt_t* const                        p_eth,\r
244         IN                              ipoib_endpt_t* const            p_src,\r
245         IN                              ipoib_endpt_t* const            p_dst );\r
246 \r
247 static ib_api_status_t\r
248 __recv_arp(\r
249         IN                              ipoib_port_t* const                     p_port,\r
250         IN                              ib_wc_t* const                          p_wc,\r
251         IN              const   ipoib_pkt_t* const                      p_ipoib,\r
252                 OUT                     eth_pkt_t* const                        p_eth,\r
253         IN                              ipoib_endpt_t** const           p_src,\r
254         IN                              ipoib_endpt_t* const            p_dst );\r
255 \r
256 static ib_api_status_t\r
257 __recv_mgr_prepare_pkt(\r
258         IN                              ipoib_port_t* const                     p_port,\r
259         IN                              ipoib_recv_desc_t* const        p_desc,\r
260                 OUT                     NDIS_PACKET** const                     pp_packet );\r
261 \r
262 static uint32_t\r
263 __recv_mgr_build_pkt_array(\r
264         IN                              ipoib_port_t* const                     p_port,\r
265         IN                              int32_t                                         shortage,\r
266                 OUT                     cl_qlist_t* const                       p_done_list,\r
267                 OUT                     int32_t* const                          p_discarded );\r
268 \r
269 /******************************************************************************\r
270 *\r
271 * Send manager operations.\r
272 *\r
273 ******************************************************************************/\r
274 static void\r
275 __send_mgr_construct(\r
276         IN                              ipoib_port_t* const                     p_port );\r
277 \r
278 static void\r
279 __send_mgr_destroy(\r
280         IN                              ipoib_port_t* const                     p_port );\r
281 \r
282 static NDIS_STATUS\r
283 __send_gen(\r
284         IN                              ipoib_port_t* const                     p_port,\r
285         IN                              ipoib_send_desc_t* const        p_desc,\r
286         IN                              INT                                             lso_data_index);\r
287 \r
288 static NDIS_STATUS\r
289 __send_mgr_filter_ip(\r
290         IN                              ipoib_port_t* const                     p_port,\r
291         IN              const   eth_hdr_t* const                        p_eth_hdr,\r
292         IN                              NDIS_BUFFER*                            p_buf,\r
293         IN                              size_t                                          buf_len,\r
294         IN      OUT                     ipoib_send_desc_t* const        p_desc );\r
295 \r
296 static NDIS_STATUS\r
297 __send_mgr_filter_igmp_v2(\r
298         IN                              ipoib_port_t* const                     p_port,\r
299     IN          const   ip_hdr_t* const                         p_ip_hdr,\r
300         IN                              size_t                                          iph_options_size,\r
301         IN                              NDIS_BUFFER*                            p_buf,\r
302         IN                              size_t                                          buf_len );\r
303 \r
304 static NDIS_STATUS\r
305 __send_mgr_filter_udp(\r
306         IN                              ipoib_port_t* const                     p_port,\r
307         IN              const   ip_hdr_t* const                         p_ip_hdr,\r
308         IN                              NDIS_BUFFER*                            p_buf,\r
309         IN                              size_t                                          buf_len,\r
310         IN      OUT                     ipoib_send_desc_t* const        p_desc );\r
311 \r
312 static NDIS_STATUS\r
313 __send_mgr_filter_dhcp(\r
314         IN                              ipoib_port_t* const                     p_port,\r
315         IN              const   udp_hdr_t* const                        p_udp_hdr,\r
316         IN                              NDIS_BUFFER*                            p_buf,\r
317         IN                              size_t                                          buf_len,\r
318         IN      OUT                     ipoib_send_desc_t* const        p_desc );\r
319 \r
320 static NDIS_STATUS\r
321 __send_mgr_filter_arp(\r
322         IN                              ipoib_port_t* const                     p_port,\r
323         IN              const   eth_hdr_t* const                        p_eth_hdr,\r
324         IN                              NDIS_BUFFER*                            p_buf,\r
325         IN                              size_t                                          buf_len,\r
326         IN      OUT                     ipoib_send_desc_t* const        p_desc );\r
327 \r
328 static void\r
329 __process_failed_send(\r
330         IN                              ipoib_port_t* const                     p_port,\r
331         IN                              ipoib_send_desc_t* const        p_desc,\r
332         IN              const   NDIS_STATUS                                     status );\r
333 \r
334 static void\r
335 __send_cb(\r
336         IN              const   ib_cq_handle_t                          h_cq,\r
337         IN                              void                                            *cq_context );\r
338 \r
339 static NDIS_STATUS GetLsoHeaderSize(\r
340         IN      ipoib_port_t* const pPort,\r
341         IN      PNDIS_BUFFER  CurrBuffer,\r
342         IN      LsoData *pLsoData,\r
343         OUT     uint16_t *pSize,\r
344         OUT     INT  *IndexOfData,\r
345         IN              ipoib_hdr_t *ipoib_hdr\r
346         );\r
347 \r
348 /******************************************************************************\r
349 *\r
350 * Endpoint manager operations\r
351 *\r
352 ******************************************************************************/\r
353 static void\r
354 __endpt_mgr_construct(\r
355         IN                              ipoib_port_t* const                     p_port );\r
356 \r
357 static ib_api_status_t\r
358 __endpt_mgr_init(\r
359         IN                              ipoib_port_t* const                     p_port );\r
360 \r
361 static void\r
362 __endpt_mgr_destroy(\r
363         IN                              ipoib_port_t* const                     p_port );\r
364 \r
365 /****f* IPoIB/__endpt_mgr_remove_all\r
366 * NAME\r
367 *       __endpt_mgr_remove_all\r
368 *\r
369 * DESCRIPTION\r
370 *       Removes all enpoints from the port, dereferencing them to initiate\r
371 *       destruction.\r
372 *\r
373 * SYNOPSIS\r
374 */\r
375 static void\r
376 __endpt_mgr_remove_all(\r
377         IN                              ipoib_port_t* const                     p_port );\r
378 /*\r
379 ********/\r
380 \r
381 static void\r
382 __endpt_mgr_remove(\r
383         IN                              ipoib_port_t* const                     p_port,\r
384         IN                              ipoib_endpt_t* const            p_endpt );\r
385 \r
386 static void\r
387 __endpt_mgr_reset_all(\r
388         IN                              ipoib_port_t* const                     p_port );\r
389 \r
390 static inline NDIS_STATUS\r
391 __endpt_mgr_ref(\r
392         IN                              ipoib_port_t* const                     p_port,\r
393         IN              const   mac_addr_t                                      mac,\r
394                 OUT                     ipoib_endpt_t** const           pp_endpt );\r
395 \r
396 static inline NDIS_STATUS\r
397 __endpt_mgr_get_gid_qpn(\r
398         IN                              ipoib_port_t* const                     p_port,\r
399         IN              const   mac_addr_t                                      mac,\r
400                 OUT                     ib_gid_t* const                         p_gid,\r
401                 OUT                     UNALIGNED net32_t* const        p_qpn );\r
402 \r
403 static inline ipoib_endpt_t*\r
404 __endpt_mgr_get_by_gid(\r
405         IN                              ipoib_port_t* const                     p_port,\r
406         IN              const   ib_gid_t* const                         p_gid );\r
407 \r
408 static inline ipoib_endpt_t*\r
409 __endpt_mgr_get_by_lid(\r
410         IN                              ipoib_port_t* const                     p_port,\r
411         IN              const   net16_t                                         lid );\r
412 \r
413 static inline ib_api_status_t\r
414 __endpt_mgr_insert_locked(\r
415         IN                              ipoib_port_t* const                     p_port,\r
416         IN              const   mac_addr_t                                      mac,\r
417         IN                              ipoib_endpt_t* const            p_endpt );\r
418 \r
419 static inline ib_api_status_t\r
420 __endpt_mgr_insert(\r
421         IN                              ipoib_port_t* const                     p_port,\r
422         IN              const   mac_addr_t                                      mac,\r
423         IN                              ipoib_endpt_t* const            p_endpt );\r
424 \r
425 static ib_api_status_t\r
426 __endpt_mgr_add_local(\r
427         IN                              ipoib_port_t* const                     p_port,\r
428         IN                              ib_port_info_t* const           p_port_info );\r
429 \r
430 static ib_api_status_t\r
431 __endpt_mgr_add_bcast(\r
432         IN                              ipoib_port_t* const                     p_port,\r
433         IN                              ib_mcast_rec_t                          *p_mcast_rec );\r
434 \r
435 /******************************************************************************\r
436 *\r
437 * MCast operations.\r
438 *\r
439 ******************************************************************************/\r
440 static ib_api_status_t\r
441 __port_get_bcast(\r
442         IN                              ipoib_port_t* const                     p_port );\r
443 \r
444 static ib_api_status_t\r
445 __port_join_bcast(\r
446         IN                              ipoib_port_t* const                     p_port,\r
447         IN                              ib_member_rec_t* const          p_member_rec );\r
448 \r
449 static ib_api_status_t\r
450 __port_create_bcast(\r
451         IN                              ipoib_port_t* const                     p_port );\r
452 \r
453 \r
454 \r
455 static void\r
456 __bcast_get_cb(\r
457         IN                              ib_query_rec_t                          *p_query_rec );\r
458 \r
459 \r
460 static void\r
461 __bcast_cb(\r
462         IN                              ib_mcast_rec_t                          *p_mcast_rec );\r
463 \r
464 \r
465 static void\r
466 __mcast_cb(\r
467         IN                              ib_mcast_rec_t                          *p_mcast_rec );\r
468 \r
469 void\r
470 __leave_error_mcast_cb(\r
471         IN                              void                            *context );\r
472 \r
473 \r
474 static intn_t\r
475 __gid_cmp(\r
476         IN              const   void* const                                     p_key1,\r
477         IN              const   void* const                                     p_key2 )\r
478 {\r
479         return cl_memcmp( p_key1, p_key2, sizeof(ib_gid_t) );\r
480 }\r
481 \r
482 \r
483 inline void ipoib_port_ref( ipoib_port_t * p_port, int type )\r
484 {\r
485         cl_obj_ref( &p_port->obj );\r
486 #if DBG\r
487         cl_atomic_inc( &p_port->ref[type % ref_mask] );\r
488         IPOIB_PRINT( TRACE_LEVEL_INFORMATION, IPOIB_DBG_OBJ,\r
489                 ("ref type %d ref_cnt %d\n", type, p_port->obj.ref_cnt) );\r
490 #else\r
491         UNREFERENCED_PARAMETER(type);\r
492 #endif\r
493 }\r
494 \r
495 \r
496 inline void ipoib_port_deref(ipoib_port_t * p_port, int type)\r
497 {\r
498         cl_obj_deref( &p_port->obj );\r
499 \r
500 #if DBG\r
501         cl_atomic_dec( &p_port->ref[type % ref_mask] );\r
502         IPOIB_PRINT( TRACE_LEVEL_INFORMATION, IPOIB_DBG_OBJ,\r
503                 ("deref type %d ref_cnt %d\n", type, p_port->obj.ref_cnt) );\r
504 #else\r
505         UNREFERENCED_PARAMETER(type);\r
506 #endif\r
507 }\r
508 \r
509 /* function returns pointer to payload that is going after IP header.\r
510 *  asssuming that payload and IP header are in the same buffer\r
511 */\r
512 static void* GetIpPayloadPtr(const      ip_hdr_t* const p_ip_hdr)\r
513 {\r
514         return (void*)((uint8_t*)p_ip_hdr + 4*(p_ip_hdr->ver_hl & 0xf));\r
515 }\r
516 \r
517 /******************************************************************************\r
518 *\r
519 * Implementation\r
520 *\r
521 ******************************************************************************/\r
522 ib_api_status_t\r
523 ipoib_create_port(\r
524         IN                              ipoib_adapter_t* const          p_adapter,\r
525         IN                              ib_pnp_port_rec_t* const        p_pnp_rec,\r
526                 OUT                     ipoib_port_t** const            pp_port )\r
527 {\r
528         ib_api_status_t         status;\r
529         ipoib_port_t            *p_port;\r
530 \r
531         IPOIB_ENTER( IPOIB_DBG_INIT );\r
532 \r
533         CL_ASSERT( !p_adapter->p_port );\r
534 \r
535         p_port = cl_zalloc( sizeof(ipoib_port_t) +\r
536                 (sizeof(ipoib_hdr_t) * (p_adapter->params.sq_depth - 1)) );\r
537         if( !p_port )\r
538         {\r
539                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
540                         ("Failed to allocate ipoib_port_t (%d bytes)\n",\r
541                         sizeof(ipoib_port_t)) );\r
542                 return IB_INSUFFICIENT_MEMORY;\r
543         }\r
544 \r
545 #ifdef _DEBUG_\r
546         gp_ipoib_port = p_port;\r
547 #endif\r
548 \r
549         __port_construct( p_port );\r
550 \r
551         status = __port_init( p_port, p_adapter, p_pnp_rec );\r
552         if( status != IB_SUCCESS )\r
553         {\r
554                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
555                         ("ipoib_port_init returned %s.\n", \r
556                         p_adapter->p_ifc->get_err_str( status )) );\r
557                 __port_cleanup( &p_port->obj );\r
558                 __port_free( &p_port->obj );\r
559                 return status;\r
560         }\r
561 \r
562         *pp_port = p_port;\r
563         IPOIB_EXIT( IPOIB_DBG_INIT );\r
564         return IB_SUCCESS;\r
565 }\r
566 \r
567 \r
568 void\r
569 ipoib_port_destroy(\r
570         IN                              ipoib_port_t* const                     p_port )\r
571 {\r
572         IPOIB_ENTER( IPOIB_DBG_INIT );\r
573 \r
574         CL_ASSERT( p_port );\r
575         CL_ASSERT( p_port->p_adapter );\r
576         CL_ASSERT( !p_port->p_adapter->p_port );\r
577 \r
578         cl_obj_destroy( &p_port->obj );\r
579 \r
580         IPOIB_EXIT( IPOIB_DBG_INIT );\r
581 }\r
582 \r
583 \r
584 static void\r
585 __port_construct(\r
586         IN                              ipoib_port_t* const                     p_port )\r
587 {\r
588         IPOIB_ENTER( IPOIB_DBG_INIT );\r
589 \r
590         p_port->state = IB_QPS_RESET;\r
591 \r
592         cl_obj_construct( &p_port->obj, IPOIB_OBJ_PORT );\r
593         cl_spinlock_construct( &p_port->send_lock );\r
594         cl_spinlock_construct( &p_port->recv_lock );\r
595         __ib_mgr_construct( p_port );\r
596         __buf_mgr_construct( p_port );\r
597 \r
598         __recv_mgr_construct( p_port );\r
599         __send_mgr_construct( p_port );\r
600 \r
601         __endpt_mgr_construct( p_port );\r
602 \r
603         KeInitializeEvent( &p_port->sa_event, NotificationEvent, TRUE );\r
604         KeInitializeEvent( &p_port->leave_mcast_event, NotificationEvent, TRUE );\r
605         \r
606         IPOIB_EXIT( IPOIB_DBG_INIT );\r
607 }\r
608 \r
609 \r
610 static ib_api_status_t\r
611 __port_init(\r
612         IN                              ipoib_port_t* const                     p_port,\r
613         IN                              ipoib_adapter_t* const          p_adapter,\r
614         IN                              ib_pnp_port_rec_t* const        p_pnp_rec )\r
615 {\r
616         cl_status_t                     cl_status;\r
617         ib_api_status_t         status;\r
618 \r
619         IPOIB_ENTER( IPOIB_DBG_INIT );\r
620 \r
621         p_port->port_num = p_pnp_rec->p_port_attr->port_num;\r
622         p_port->p_adapter = p_adapter;\r
623 \r
624         cl_status = cl_spinlock_init( &p_port->send_lock );\r
625         if( cl_status != CL_SUCCESS )\r
626         {\r
627                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
628                         ("cl_spinlock_init returned %#x\n", cl_status) );\r
629                 return IB_ERROR;\r
630         }\r
631 \r
632         cl_status = cl_spinlock_init( &p_port->recv_lock );\r
633         if( cl_status != CL_SUCCESS )\r
634         {\r
635                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
636                         ("cl_spinlock_init returned %#x\n", cl_status) );\r
637                 return IB_ERROR;\r
638         }\r
639 \r
640         /* Initialize the IB resource manager. */\r
641         status = __ib_mgr_init( p_port );\r
642         if( status != IB_SUCCESS )\r
643         {\r
644                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
645                         ("__ib_mgr_init returned %s\n", \r
646                         p_adapter->p_ifc->get_err_str( status )) );\r
647                 return status;\r
648         }\r
649 \r
650         /* Initialize the buffer manager. */\r
651         status = __buf_mgr_init( p_port );\r
652         if( status != IB_SUCCESS )\r
653         {\r
654                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
655                         ("__buf_mgr_init returned %s\n", \r
656                         p_adapter->p_ifc->get_err_str( status )) );\r
657                 return status;\r
658         }\r
659 \r
660         /* Initialize the receive manager. */\r
661         status = __recv_mgr_init( p_port );\r
662         if( status != IB_SUCCESS )\r
663         {\r
664                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
665                         ("__recv_mgr_init returned %s\n", \r
666                         p_adapter->p_ifc->get_err_str( status )) );\r
667                 return status;\r
668         }\r
669 \r
670         /* Initialize the endpoint manager. */\r
671         status = __endpt_mgr_init( p_port );\r
672         if( status != IB_SUCCESS )\r
673         {\r
674                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
675                         ("__endpt_mgr_init returned %s\n", \r
676                         p_adapter->p_ifc->get_err_str( status )) );\r
677                 return status;\r
678         }\r
679          /* Initialize multicast garbage collector timer and DPC object */\r
680          KeInitializeDpc(&p_port->gc_dpc,(PKDEFERRED_ROUTINE)__port_mcast_garbage_dpc,p_port);\r
681          KeInitializeTimerEx(&p_port->gc_timer,SynchronizationTimer);\r
682 \r
683         /* We only ever destroy from the PnP callback thread. */\r
684         cl_status = cl_obj_init( &p_port->obj, CL_DESTROY_SYNC,\r
685                 __port_destroying, __port_cleanup, __port_free );\r
686 \r
687 #if DBG\r
688         cl_atomic_inc( &p_port->ref[ref_init] );\r
689         IPOIB_PRINT( TRACE_LEVEL_INFORMATION, IPOIB_DBG_OBJ,\r
690                 ("ref type %d ref_cnt %d\n", ref_init, p_port->obj.ref_cnt) );\r
691 #endif\r
692 \r
693         if( cl_status != CL_SUCCESS )\r
694         {\r
695                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
696                         ("cl_obj_init returned %#x\n", cl_status) );\r
697                 return IB_ERROR;\r
698         }\r
699 \r
700         cl_status = cl_obj_insert_rel( &p_port->rel, &p_adapter->obj, &p_port->obj );\r
701         if( cl_status != CL_SUCCESS )\r
702         {\r
703                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
704                         ("cl_obj_insert_rel returned %#x\n", cl_status) );\r
705                 cl_obj_destroy( &p_port->obj );\r
706                 return IB_ERROR;\r
707         }\r
708 \r
709 #if DBG\r
710         cl_atomic_inc( &p_port->ref[ref_init] );\r
711         IPOIB_PRINT( TRACE_LEVEL_ERROR, IPOIB_DBG_OBJ,\r
712                 ("ref type %d ref_cnt %d\n", ref_init, p_port->obj.ref_cnt) );\r
713 #endif\r
714 \r
715         IPOIB_EXIT( IPOIB_DBG_INIT );\r
716         return IB_SUCCESS;\r
717 }\r
718 \r
719 \r
720 static void\r
721 __port_destroying(\r
722         IN                              cl_obj_t* const                         p_obj )\r
723 {\r
724         ipoib_port_t    *p_port;\r
725 \r
726         IPOIB_ENTER( IPOIB_DBG_INIT );\r
727 \r
728         CL_ASSERT( p_obj );\r
729 \r
730         p_port = PARENT_STRUCT( p_obj, ipoib_port_t, obj );\r
731 \r
732         ipoib_port_down( p_port );\r
733 \r
734         __endpt_mgr_remove_all( p_port );\r
735 \r
736         ipoib_port_resume( p_port );\r
737 \r
738         IPOIB_EXIT( IPOIB_DBG_INIT );\r
739 }\r
740 \r
741 \r
742 static void\r
743 __port_cleanup(\r
744         IN                              cl_obj_t* const                         p_obj )\r
745 {\r
746         ipoib_port_t    *p_port;\r
747 \r
748         IPOIB_ENTER( IPOIB_DBG_INIT );\r
749 \r
750         CL_ASSERT( p_obj );\r
751 \r
752         p_port = PARENT_STRUCT( p_obj, ipoib_port_t, obj );\r
753 \r
754         /* Wait for all sends and receives to get flushed. */\r
755         while( p_port->send_mgr.depth || p_port->recv_mgr.depth )\r
756                 cl_thread_suspend( 0 );\r
757 \r
758         /* Destroy the send and receive managers before closing the CA. */\r
759         __ib_mgr_destroy( p_port );\r
760 \r
761         IPOIB_EXIT( IPOIB_DBG_INIT );\r
762 }\r
763 \r
764 \r
765 static void\r
766 __port_free(\r
767         IN                              cl_obj_t* const                         p_obj )\r
768 {\r
769         ipoib_port_t    *p_port;\r
770 \r
771         IPOIB_ENTER( IPOIB_DBG_INIT );\r
772 \r
773         CL_ASSERT( p_obj );\r
774 \r
775         p_port = PARENT_STRUCT( p_obj, ipoib_port_t, obj );\r
776 \r
777         KeCancelTimer(&p_port->gc_timer);\r
778         KeFlushQueuedDpcs();\r
779         __endpt_mgr_destroy( p_port );\r
780         __recv_mgr_destroy( p_port );\r
781         __send_mgr_destroy( p_port );\r
782         __buf_mgr_destroy( p_port );\r
783 \r
784         cl_spinlock_destroy( &p_port->send_lock );\r
785         cl_spinlock_destroy( &p_port->recv_lock );\r
786 \r
787         cl_obj_deinit( p_obj );\r
788 \r
789         cl_free( p_port );\r
790 \r
791         IPOIB_EXIT( IPOIB_DBG_INIT );\r
792 }\r
793 \r
794 \r
795 \r
796 /******************************************************************************\r
797 *\r
798 * IB resource manager implementation.\r
799 *\r
800 ******************************************************************************/\r
801 static void\r
802 __ib_mgr_construct(\r
803         IN                              ipoib_port_t* const                     p_port )\r
804 {\r
805         IPOIB_ENTER( IPOIB_DBG_INIT );\r
806 \r
807         cl_memclr( &p_port->ib_mgr, sizeof(ipoib_ib_mgr_t) );\r
808 \r
809         IPOIB_EXIT( IPOIB_DBG_INIT );\r
810 }\r
811 \r
812 \r
813 static ib_api_status_t\r
814 __ib_mgr_init(\r
815         IN                              ipoib_port_t* const                     p_port )\r
816 {\r
817         ib_api_status_t         status;\r
818         ib_cq_create_t          cq_create;\r
819         ib_qp_create_t          qp_create;\r
820         ib_phys_create_t        phys_create;\r
821         ib_phys_range_t         phys_range;\r
822         uint64_t                        vaddr;\r
823         net32_t                         rkey;\r
824         ib_qp_attr_t            qp_attr;\r
825         ib_ca_attr_t *          p_ca_attr;\r
826         uint32_t                        attr_size;\r
827 \r
828         IPOIB_ENTER( IPOIB_DBG_INIT );\r
829 \r
830         /* Open the CA. */\r
831         status = p_port->p_adapter->p_ifc->open_ca(\r
832                 p_port->p_adapter->h_al, p_port->p_adapter->guids.ca_guid,\r
833                 NULL, p_port, &p_port->ib_mgr.h_ca );\r
834         if( status != IB_SUCCESS )\r
835         {\r
836                 NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
837                         EVENT_IPOIB_OPEN_CA, 1, status );\r
838                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
839                         ("ib_open_ca returned %s\n", \r
840                         p_port->p_adapter->p_ifc->get_err_str( status )) );\r
841                 return status;\r
842         }\r
843 \r
844         /* Allocate the PD. */\r
845         status = p_port->p_adapter->p_ifc->alloc_pd(\r
846                 p_port->ib_mgr.h_ca, IB_PDT_UD, p_port, &p_port->ib_mgr.h_pd );\r
847         if( status != IB_SUCCESS )\r
848         {\r
849                 NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
850                         EVENT_IPOIB_ALLOC_PD, 1, status );\r
851                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
852                         ("ib_alloc_pd returned %s\n", \r
853                         p_port->p_adapter->p_ifc->get_err_str( status )) );\r
854                 return status;\r
855         }\r
856 \r
857         /* Allocate receive CQ. */\r
858         cq_create.size = p_port->p_adapter->params.rq_depth;\r
859         cq_create.pfn_comp_cb = __recv_cb;\r
860         cq_create.h_wait_obj = NULL;\r
861 \r
862         status = p_port->p_adapter->p_ifc->create_cq(\r
863                 p_port->ib_mgr.h_ca, &cq_create, p_port,\r
864                 __cq_event, &p_port->ib_mgr.h_recv_cq );\r
865         if( status != IB_SUCCESS )\r
866         {\r
867                 NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
868                         EVENT_IPOIB_CREATE_RECV_CQ, 1, status );\r
869                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
870                         ("ib_create_cq returned %s.\n", \r
871                         p_port->p_adapter->p_ifc->get_err_str( status )) );\r
872                 return status;\r
873         }\r
874 \r
875         /* Allocate send CQ. */\r
876         cq_create.size = p_port->p_adapter->params.sq_depth;\r
877         cq_create.pfn_comp_cb = __send_cb;\r
878 \r
879         status = p_port->p_adapter->p_ifc->create_cq(\r
880                 p_port->ib_mgr.h_ca, &cq_create, p_port,\r
881                 __cq_event, &p_port->ib_mgr.h_send_cq );\r
882         if( status != IB_SUCCESS )\r
883         {\r
884                 NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
885                         EVENT_IPOIB_CREATE_SEND_CQ, 1, status );\r
886                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
887                         ("ib_create_cq returned %s.\n", \r
888                         p_port->p_adapter->p_ifc->get_err_str( status )) );\r
889                 return status;\r
890         }\r
891         \r
892         /* Allocate the QP. */\r
893         cl_memclr( &qp_create, sizeof(qp_create) );\r
894         qp_create.qp_type = IB_QPT_UNRELIABLE_DGRM;\r
895         qp_create.rq_depth = p_port->p_adapter->params.rq_depth;\r
896         qp_create.rq_sge = 2;   /* To support buffers spanning pages. */\r
897         qp_create.h_rq_cq = p_port->ib_mgr.h_recv_cq;\r
898         qp_create.sq_depth = p_port->p_adapter->params.sq_depth;\r
899         \r
900         //Figure out the right number of SGE entries for sends.\r
901         /* Get the size of the CA attribute structure. */\r
902         status = p_port->p_adapter->p_ifc->query_ca( p_port->ib_mgr.h_ca, NULL, &attr_size );\r
903         if( status != IB_INSUFFICIENT_MEMORY )\r
904         {\r
905                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
906                         ("ib_query_ca failed with status %s.\n", p_port->p_adapter->p_ifc->get_err_str(status)) );\r
907                 return status;\r
908         }\r
909 \r
910         /* Allocate enough space to store the attribute structure. */\r
911         p_ca_attr = cl_malloc( attr_size );\r
912         if( !p_ca_attr )\r
913         {\r
914                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
915                         ("cl_malloc failed to allocate p_ca_attr!\n") );\r
916                 return IB_INSUFFICIENT_RESOURCES;\r
917         }\r
918 \r
919         /* Query the CA attributes. */\r
920         status = p_port->p_adapter->p_ifc->query_ca(p_port->ib_mgr.h_ca, p_ca_attr, &attr_size );\r
921         if( status != IB_SUCCESS )\r
922         {\r
923                 cl_free( p_ca_attr );\r
924 \r
925                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
926                         ("ib_query_ca failed with status %s.\n", p_port->p_adapter->p_ifc->get_err_str(status)) );\r
927                 return status;\r
928         }\r
929 #define UD_QP_USED_SGE 3\r
930         qp_create.sq_sge = MAX_SEND_SGE < p_ca_attr->max_sges ? MAX_SEND_SGE  : (p_ca_attr->max_sges - UD_QP_USED_SGE);\r
931         if (!p_ca_attr->ipoib_csum) { \r
932                 //checksum is not supported by device\r
933                 //user must specify BYPASS to explicitly cancel checksum calculation\r
934                 if (p_port->p_adapter->params.send_chksum_offload == CSUM_ENABLED)\r
935                         p_port->p_adapter->params.send_chksum_offload = CSUM_DISABLED;\r
936                 if (p_port->p_adapter->params.recv_chksum_offload == CSUM_ENABLED)\r
937                         p_port->p_adapter->params.recv_chksum_offload = CSUM_DISABLED;\r
938         }\r
939         cl_free( p_ca_attr );\r
940         \r
941         qp_create.h_sq_cq = p_port->ib_mgr.h_send_cq;\r
942         qp_create.sq_signaled = TRUE;\r
943         status = p_port->p_adapter->p_ifc->create_qp(\r
944                 p_port->ib_mgr.h_pd, &qp_create, p_port,\r
945                 __qp_event, &p_port->ib_mgr.h_qp );\r
946         if( status != IB_SUCCESS )\r
947         {\r
948                 NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
949                         EVENT_IPOIB_CREATE_QP, 1, status );\r
950                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
951                         ("ib_create_qp returned %s\n", \r
952                         p_port->p_adapter->p_ifc->get_err_str( status )) );\r
953                 return status;\r
954         }\r
955         /* Query the QP so we can get our QPN. */\r
956         status = p_port->p_adapter->p_ifc->query_qp(\r
957                 p_port->ib_mgr.h_qp, &qp_attr );\r
958         if( status != IB_SUCCESS )\r
959         {\r
960                 NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
961                         EVENT_IPOIB_QUERY_QP, 1, status );\r
962                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
963                         ("ib_query_qp returned %s\n", \r
964                         p_port->p_adapter->p_ifc->get_err_str( status )) );\r
965                 return status;\r
966         }\r
967         p_port->ib_mgr.qpn = qp_attr.num;\r
968 \r
969         /* Register all of physical memory */\r
970         phys_create.length = MEM_REG_SIZE;\r
971         phys_create.num_ranges = 1;\r
972         phys_create.range_array = &phys_range;\r
973         phys_create.buf_offset = 0;\r
974         phys_create.hca_page_size = PAGE_SIZE;\r
975         phys_create.access_ctrl = IB_AC_LOCAL_WRITE;\r
976         phys_range.base_addr = 0;\r
977         phys_range.size = MEM_REG_SIZE;\r
978         vaddr = 0;\r
979         status = p_port->p_adapter->p_ifc->reg_phys(\r
980                 p_port->ib_mgr.h_pd, &phys_create, &vaddr,\r
981                 &p_port->ib_mgr.lkey, &rkey, &p_port->ib_mgr.h_mr );\r
982         if( status != IB_SUCCESS )\r
983         {\r
984                 NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
985                         EVENT_IPOIB_REG_PHYS, 1, status );\r
986                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
987                         ("ib_reg_phys returned %s\n", \r
988                         p_port->p_adapter->p_ifc->get_err_str( status )) );\r
989                 return status;\r
990         }\r
991 \r
992         IPOIB_EXIT( IPOIB_DBG_INIT );\r
993         return IB_SUCCESS;\r
994 }\r
995 \r
996 \r
997 static void\r
998 __ib_mgr_destroy(\r
999         IN                              ipoib_port_t* const                     p_port )\r
1000 {\r
1001         ib_api_status_t status;\r
1002 \r
1003         IPOIB_ENTER( IPOIB_DBG_INIT );\r
1004 \r
1005         if( p_port->ib_mgr.h_ca )\r
1006         {\r
1007                 status =\r
1008                         p_port->p_adapter->p_ifc->close_ca( p_port->ib_mgr.h_ca, NULL );\r
1009                 CL_ASSERT( status == IB_SUCCESS );\r
1010                 p_port->ib_mgr.h_ca = NULL;\r
1011         }\r
1012 \r
1013         IPOIB_EXIT( IPOIB_DBG_INIT );\r
1014 }\r
1015 \r
1016 \r
1017 \r
1018 /******************************************************************************\r
1019 *\r
1020 * Buffer manager implementation.\r
1021 *\r
1022 ******************************************************************************/\r
1023 static void\r
1024 __buf_mgr_construct(\r
1025         IN                              ipoib_port_t* const                     p_port )\r
1026 {\r
1027         IPOIB_ENTER( IPOIB_DBG_INIT );\r
1028 \r
1029         cl_qpool_construct( &p_port->buf_mgr.recv_pool );\r
1030 \r
1031         p_port->buf_mgr.h_packet_pool = NULL;\r
1032         p_port->buf_mgr.h_buffer_pool = NULL;\r
1033 \r
1034         ExInitializeNPagedLookasideList( &p_port->buf_mgr.send_buf_list,\r
1035                 NULL, NULL, 0, MAX_XFER_BLOCK_SIZE, 'bipi', 0 );\r
1036 \r
1037         p_port->buf_mgr.h_send_pkt_pool = NULL;\r
1038         p_port->buf_mgr.h_send_buf_pool = NULL;\r
1039 \r
1040         IPOIB_EXIT( IPOIB_DBG_INIT );\r
1041 }\r
1042 \r
1043 \r
1044 static ib_api_status_t\r
1045 __buf_mgr_init(\r
1046         IN                              ipoib_port_t* const                     p_port )\r
1047 {\r
1048         cl_status_t             cl_status;\r
1049         NDIS_STATUS             ndis_status;\r
1050         ipoib_params_t  *p_params;\r
1051 \r
1052         IPOIB_ENTER(IPOIB_DBG_INIT );\r
1053 \r
1054         CL_ASSERT( p_port );\r
1055         CL_ASSERT( p_port->p_adapter );\r
1056 \r
1057         p_params = &p_port->p_adapter->params;\r
1058 \r
1059         /* Allocate the receive descriptor pool */\r
1060         cl_status = cl_qpool_init( &p_port->buf_mgr.recv_pool,\r
1061                 p_params->rq_depth * p_params->recv_pool_ratio,\r
1062 #if IPOIB_INLINE_RECV\r
1063                 0, 0, sizeof(ipoib_recv_desc_t), __recv_ctor, NULL, p_port );\r
1064 #else   /* IPOIB_INLINE_RECV */\r
1065                 0, 0, sizeof(ipoib_recv_desc_t), __recv_ctor, __recv_dtor, p_port );\r
1066 #endif  /* IPOIB_INLINE_RECV */\r
1067         if( cl_status != CL_SUCCESS )\r
1068         {\r
1069                 NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
1070                         EVENT_IPOIB_RECV_POOL, 1, cl_status );\r
1071                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1072                         ("cl_qpool_init for recvs returned %#x\n",\r
1073                         cl_status) );\r
1074                 return IB_INSUFFICIENT_MEMORY;\r
1075         }\r
1076 \r
1077         /* Allocate the NDIS buffer and packet pools for receive indication. */\r
1078         NdisAllocatePacketPool( &ndis_status, &p_port->buf_mgr.h_packet_pool,\r
1079                 p_params->rq_depth, PROTOCOL_RESERVED_SIZE_IN_PACKET );\r
1080         if( ndis_status != NDIS_STATUS_SUCCESS )\r
1081         {\r
1082                 NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
1083                         EVENT_IPOIB_RECV_PKT_POOL, 1, ndis_status );\r
1084                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1085                         ("NdisAllocatePacketPool returned %08X\n", ndis_status) );\r
1086                 return IB_INSUFFICIENT_RESOURCES;\r
1087         }\r
1088 \r
1089         NdisAllocateBufferPool( &ndis_status, &p_port->buf_mgr.h_buffer_pool,\r
1090                 p_params->rq_depth );\r
1091         if( ndis_status != NDIS_STATUS_SUCCESS )\r
1092         {\r
1093                 NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
1094                         EVENT_IPOIB_RECV_BUF_POOL, 1, ndis_status );\r
1095                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1096                         ("NdisAllocateBufferPool returned %08X\n", ndis_status) );\r
1097                 return IB_INSUFFICIENT_RESOURCES;\r
1098         }\r
1099 \r
1100         /* Allocate the NDIS buffer and packet pools for send formatting. */\r
1101         NdisAllocatePacketPool( &ndis_status, &p_port->buf_mgr.h_send_pkt_pool,\r
1102                 1, PROTOCOL_RESERVED_SIZE_IN_PACKET );\r
1103         if( ndis_status != NDIS_STATUS_SUCCESS )\r
1104         {\r
1105                 NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
1106                         EVENT_IPOIB_SEND_PKT_POOL, 1, ndis_status );\r
1107                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1108                         ("NdisAllocatePacketPool returned %08X\n", ndis_status) );\r
1109                 return IB_INSUFFICIENT_RESOURCES;\r
1110         }\r
1111 \r
1112         NdisAllocateBufferPool( &ndis_status,\r
1113                 &p_port->buf_mgr.h_send_buf_pool, 1 );\r
1114         if( ndis_status != NDIS_STATUS_SUCCESS )\r
1115         {\r
1116                 NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
1117                         EVENT_IPOIB_SEND_BUF_POOL, 1, ndis_status );\r
1118                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1119                         ("NdisAllocateBufferPool returned %08X\n", ndis_status) );\r
1120                 return IB_INSUFFICIENT_RESOURCES;\r
1121         }\r
1122 \r
1123         IPOIB_EXIT( IPOIB_DBG_INIT );\r
1124         return IB_SUCCESS;\r
1125 }\r
1126 \r
1127 \r
1128 static void\r
1129 __buf_mgr_destroy(\r
1130         IN                              ipoib_port_t* const                     p_port )\r
1131 {\r
1132         IPOIB_ENTER(IPOIB_DBG_INIT );\r
1133 \r
1134         CL_ASSERT( p_port );\r
1135 \r
1136         /* Destroy the send packet and buffer pools. */\r
1137         if( p_port->buf_mgr.h_send_buf_pool )\r
1138                 NdisFreeBufferPool( p_port->buf_mgr.h_send_buf_pool );\r
1139         if( p_port->buf_mgr.h_send_pkt_pool )\r
1140                 NdisFreePacketPool( p_port->buf_mgr.h_send_pkt_pool );\r
1141 \r
1142         /* Destroy the receive packet and buffer pools. */\r
1143         if( p_port->buf_mgr.h_buffer_pool )\r
1144                 NdisFreeBufferPool( p_port->buf_mgr.h_buffer_pool );\r
1145         if( p_port->buf_mgr.h_packet_pool )\r
1146                 NdisFreePacketPool( p_port->buf_mgr.h_packet_pool );\r
1147 \r
1148         /* Free the receive and send descriptors. */\r
1149         cl_qpool_destroy( &p_port->buf_mgr.recv_pool );\r
1150 \r
1151         /* Free the lookaside list of scratch buffers. */\r
1152         ExDeleteNPagedLookasideList( &p_port->buf_mgr.send_buf_list );\r
1153 \r
1154         IPOIB_EXIT(  IPOIB_DBG_INIT );\r
1155 }\r
1156 \r
1157 \r
1158 static cl_status_t\r
1159 __recv_ctor(\r
1160         IN                              void* const                                     p_object,\r
1161         IN                              void*                                           context,\r
1162                 OUT                     cl_pool_item_t** const          pp_pool_item )\r
1163 {\r
1164         ipoib_recv_desc_t       *p_desc;\r
1165         ipoib_port_t            *p_port;\r
1166         uint32_t                        ds0_len;\r
1167 \r
1168         IPOIB_ENTER( IPOIB_DBG_ALLOC );\r
1169 \r
1170         CL_ASSERT( p_object );\r
1171         CL_ASSERT( context );\r
1172 \r
1173         p_desc = (ipoib_recv_desc_t*)p_object;\r
1174         p_port = (ipoib_port_t*)context;\r
1175 \r
1176         /* Setup the work request. */\r
1177         p_desc->wr.ds_array = p_desc->local_ds;\r
1178         p_desc->wr.wr_id = (uintn_t)p_desc;\r
1179 \r
1180 #if IPOIB_INLINE_RECV\r
1181         /* Sanity check on the receive buffer layout */\r
1182         CL_ASSERT( (void*)&p_desc->buf.eth.pkt.type ==\r
1183                 (void*)&p_desc->buf.ib.pkt.type );\r
1184         CL_ASSERT( sizeof(recv_buf_t) == sizeof(ipoib_pkt_t) + sizeof(ib_grh_t) );\r
1185 \r
1186         /* Setup the local data segment. */\r
1187         p_desc->local_ds[0].vaddr = cl_get_physaddr( &p_desc->buf );\r
1188         p_desc->local_ds[0].lkey = p_port->ib_mgr.lkey;\r
1189         ds0_len =\r
1190                 PAGE_SIZE - ((uint32_t)p_desc->local_ds[0].vaddr & (PAGE_SIZE - 1));\r
1191         if( ds0_len >= sizeof(recv_buf_t) )\r
1192         {\r
1193                 /* The whole buffer is within a page. */\r
1194                 p_desc->local_ds[0].length = ds0_len;\r
1195                 p_desc->wr.num_ds = 1;\r
1196         }\r
1197         else\r
1198         {\r
1199                 /* The buffer crosses page boundaries. */\r
1200                 p_desc->local_ds[0].length = ds0_len;\r
1201                 p_desc->local_ds[1].vaddr = cl_get_physaddr( \r
1202                         ((uint8_t*)&p_desc->buf) + ds0_len );\r
1203                 p_desc->local_ds[1].lkey = p_port->ib_mgr.lkey;\r
1204                 p_desc->local_ds[1].length = sizeof(recv_buf_t) - ds0_len;\r
1205                 p_desc->wr.num_ds = 2;\r
1206         }\r
1207 #else   /* IPOIB_INLINE_RECV */\r
1208         /* Allocate the receive buffer. */\r
1209         p_desc->p_buf = (recv_buf_t*)cl_zalloc( sizeof(recv_buf_t) );\r
1210         if( !p_desc->p_buf )\r
1211         {\r
1212                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1213                         ("Failed to allocate receive buffer.\n") );\r
1214                 return CL_INSUFFICIENT_MEMORY;\r
1215         }\r
1216 \r
1217         /* Sanity check on the receive buffer layout */\r
1218         CL_ASSERT( (void*)&p_desc->p_buf->eth.pkt.type ==\r
1219                 (void*)&p_desc->p_buf->ib.pkt.type );\r
1220 \r
1221         /* Setup the local data segment. */\r
1222         p_desc->local_ds[0].vaddr = cl_get_physaddr( p_desc->p_buf );\r
1223         p_desc->local_ds[0].length = sizeof(ipoib_pkt_t) + sizeof(ib_grh_t);\r
1224         p_desc->local_ds[0].lkey = p_port->ib_mgr.lkey;\r
1225 #endif  /* IPOIB_INLINE_RECV */\r
1226 \r
1227         *pp_pool_item = &p_desc->item;\r
1228 \r
1229         IPOIB_EXIT( IPOIB_DBG_ALLOC );\r
1230         return CL_SUCCESS;\r
1231 }\r
1232 \r
1233 \r
1234 #if !IPOIB_INLINE_RECV\r
1235 static void\r
1236 __recv_dtor(\r
1237         IN              const   cl_pool_item_t* const           p_pool_item,\r
1238         IN                              void                                            *context )\r
1239 {\r
1240         ipoib_recv_desc_t       *p_desc;\r
1241 \r
1242         IPOIB_ENTER(  IPOIB_DBG_ALLOC );\r
1243 \r
1244         UNUSED_PARAM( context );\r
1245 \r
1246         p_desc = PARENT_STRUCT( p_pool_item, ipoib_recv_desc_t, item );\r
1247 \r
1248         if( p_desc->p_buf )\r
1249                 cl_free( p_desc->p_buf );\r
1250 \r
1251         IPOIB_EXIT( IPOIB_DBG_ALLOC );\r
1252 }\r
1253 #endif\r
1254 \r
1255 \r
1256 static inline ipoib_recv_desc_t*\r
1257 __buf_mgr_get_recv(\r
1258         IN                              ipoib_port_t* const                     p_port )\r
1259 {\r
1260         ipoib_recv_desc_t       *p_desc;\r
1261         IPOIB_ENTER( IPOIB_DBG_RECV );\r
1262         p_desc = (ipoib_recv_desc_t*)cl_qpool_get( &p_port->buf_mgr.recv_pool );\r
1263         /* Reference the port object for the send. */\r
1264         if( p_desc )\r
1265         {\r
1266                 ipoib_port_ref( p_port, ref_get_recv );\r
1267                 CL_ASSERT( p_desc->wr.wr_id == (uintn_t)p_desc );\r
1268 #if IPOIB_INLINE_RECV\r
1269                 CL_ASSERT( p_desc->local_ds[0].vaddr ==\r
1270                         cl_get_physaddr( &p_desc->buf ) );\r
1271 #else   /* IPOIB_INLINE_RECV */\r
1272                 CL_ASSERT( p_desc->local_ds[0].vaddr ==\r
1273                         cl_get_physaddr( p_desc->p_buf ) );\r
1274                 CL_ASSERT( p_desc->local_ds[0].length ==\r
1275                         (sizeof(ipoib_pkt_t) + sizeof(ib_grh_t)) );\r
1276 #endif  /* IPOIB_INLINE_RECV */\r
1277                 CL_ASSERT( p_desc->local_ds[0].lkey == p_port->ib_mgr.lkey );\r
1278         }\r
1279         IPOIB_EXIT( IPOIB_DBG_RECV );\r
1280         return p_desc;\r
1281 }\r
1282 \r
1283 \r
1284 static inline void\r
1285 __buf_mgr_put_recv(\r
1286         IN                              ipoib_port_t* const                     p_port,\r
1287         IN                              ipoib_recv_desc_t* const        p_desc,\r
1288         IN                              NDIS_PACKET* const                      p_packet OPTIONAL )\r
1289 {\r
1290         NDIS_BUFFER             *p_buf;\r
1291 \r
1292         IPOIB_ENTER(IPOIB_DBG_RECV );\r
1293 \r
1294         if( p_packet )\r
1295         {\r
1296                 /* Unchain the NDIS buffer. */\r
1297                 NdisUnchainBufferAtFront( p_packet, &p_buf );\r
1298                 CL_ASSERT( p_buf );\r
1299                 /* Return the NDIS packet and NDIS buffer to their pools. */\r
1300                 NdisDprFreePacketNonInterlocked( p_packet );\r
1301                 NdisFreeBuffer( p_buf );\r
1302         }\r
1303 \r
1304         /* Return the descriptor to its pools. */\r
1305         cl_qpool_put( &p_port->buf_mgr.recv_pool, &p_desc->item );\r
1306 \r
1307         /*\r
1308          * Dereference the port object since the receive is no longer outstanding.\r
1309          */\r
1310         ipoib_port_deref( p_port, ref_get_recv );\r
1311         IPOIB_EXIT(  IPOIB_DBG_RECV );\r
1312 }\r
1313 \r
1314 \r
1315 static inline void\r
1316 __buf_mgr_put_recv_list(\r
1317         IN                              ipoib_port_t* const                     p_port,\r
1318         IN                              cl_qlist_t* const                       p_list )\r
1319 {\r
1320         //IPOIB_ENTER(  IPOIB_DBG_RECV );\r
1321         cl_qpool_put_list( &p_port->buf_mgr.recv_pool, p_list );\r
1322         //IPOIB_EXIT(  IPOIB_DBG_RECV );\r
1323 }\r
1324 \r
1325 \r
1326 static inline NDIS_PACKET*\r
1327 __buf_mgr_get_ndis_pkt(\r
1328         IN                              ipoib_port_t* const                     p_port,\r
1329         IN                              ipoib_recv_desc_t* const        p_desc )\r
1330 {\r
1331         NDIS_STATUS                             status;\r
1332         NDIS_PACKET                             *p_packet;\r
1333         NDIS_BUFFER                             *p_buffer;\r
1334 \r
1335         IPOIB_ENTER(  IPOIB_DBG_RECV );\r
1336 \r
1337         NdisDprAllocatePacketNonInterlocked( &status, &p_packet,\r
1338                 p_port->buf_mgr.h_packet_pool );\r
1339         if( status != NDIS_STATUS_SUCCESS )\r
1340         {\r
1341                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1342                         ("Failed to allocate NDIS_PACKET: %08x\n", status) );\r
1343                 return NULL;\r
1344         }\r
1345 \r
1346         IPOIB_PORT_FROM_PACKET( p_packet ) = p_port;\r
1347         IPOIB_RECV_FROM_PACKET( p_packet ) = p_desc;\r
1348 \r
1349         NdisAllocateBuffer( &status, &p_buffer,\r
1350 #if IPOIB_INLINE_RECV\r
1351                 p_port->buf_mgr.h_buffer_pool, &p_desc->buf.eth.pkt, p_desc->len );\r
1352 #else   /* IPOIB_INLINE_RECV */\r
1353                 p_port->buf_mgr.h_buffer_pool, &p_desc->p_buf->eth.pkt, p_desc->len );\r
1354 #endif  /* IPOIB_INLINE_RECV */\r
1355         if( status != NDIS_STATUS_SUCCESS )\r
1356         {\r
1357                 IPOIB_PRINT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1358                         ("Failed to allocate NDIS_BUFFER: %08x\n", status) );\r
1359                 NdisDprFreePacketNonInterlocked( p_packet );\r
1360                 return NULL;\r
1361         }\r
1362 \r
1363         NdisChainBufferAtFront( p_packet, p_buffer );\r
1364         NDIS_SET_PACKET_HEADER_SIZE( p_packet, sizeof(eth_hdr_t) );\r
1365 \r
1366         IPOIB_EXIT(  IPOIB_DBG_RECV );\r
1367         return p_packet;\r
1368 }\r
1369 \r
1370 /******************************************************************************\r
1371 *\r
1372 * Receive manager implementation.\r
1373 *\r
1374 ******************************************************************************/\r
1375 static void\r
1376 __recv_mgr_construct(\r
1377         IN                              ipoib_port_t* const                     p_port )\r
1378 {\r
1379         IPOIB_ENTER( IPOIB_DBG_INIT );\r
1380 \r
1381         cl_qlist_init( &p_port->recv_mgr.done_list );\r
1382 \r
1383         p_port->recv_mgr.recv_pkt_array = NULL;\r
1384 \r
1385         IPOIB_EXIT( IPOIB_DBG_INIT );\r
1386 }\r
1387 \r
1388 \r
1389 static ib_api_status_t\r
1390 __recv_mgr_init(\r
1391         IN                              ipoib_port_t* const                     p_port )\r
1392 {\r
1393         IPOIB_ENTER( IPOIB_DBG_INIT );\r
1394 \r
1395         /* Allocate the NDIS_PACKET pointer array for indicating receives. */\r
1396         p_port->recv_mgr.recv_pkt_array = cl_malloc(\r
1397                 sizeof(NDIS_PACKET*) * p_port->p_adapter->params.rq_depth );\r
1398         if( !p_port->recv_mgr.recv_pkt_array )\r
1399         {\r
1400                 NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
1401                         EVENT_IPOIB_RECV_PKT_ARRAY, 0 );\r
1402                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1403                         ("cl_malloc for PNDIS_PACKET array failed.\n") );\r
1404                 return IB_INSUFFICIENT_MEMORY;\r
1405         }\r
1406 \r
1407         IPOIB_EXIT( IPOIB_DBG_INIT );\r
1408         return IB_SUCCESS;\r
1409 }\r
1410 \r
1411 \r
1412 static void\r
1413 __recv_mgr_destroy(\r
1414         IN                              ipoib_port_t* const                     p_port )\r
1415 {\r
1416         IPOIB_ENTER( IPOIB_DBG_INIT );\r
1417 \r
1418         CL_ASSERT( cl_is_qlist_empty( &p_port->recv_mgr.done_list ) );\r
1419         CL_ASSERT( !p_port->recv_mgr.depth );\r
1420 \r
1421         if( p_port->recv_mgr.recv_pkt_array )\r
1422                 cl_free( p_port->recv_mgr.recv_pkt_array );\r
1423 \r
1424         IPOIB_EXIT( IPOIB_DBG_INIT );\r
1425 }\r
1426 \r
1427 \r
1428 /*\r
1429  * Posts receive buffers to the receive queue and returns the number\r
1430  * of receives needed to bring the RQ to its low water mark.  Note\r
1431  * that the value is signed, and can go negative.  All tests must\r
1432  * be for > 0.\r
1433  */\r
1434 static int32_t\r
1435 __recv_mgr_repost(\r
1436         IN                              ipoib_port_t* const                     p_port )\r
1437 {\r
1438         ipoib_recv_desc_t       *p_head = NULL, *p_tail = NULL, *p_next;\r
1439         ib_api_status_t         status;\r
1440         ib_recv_wr_t            *p_failed;\r
1441         PERF_DECLARE( GetRecv );\r
1442         PERF_DECLARE( PostRecv );\r
1443 \r
1444         IPOIB_ENTER( IPOIB_DBG_RECV );\r
1445 \r
1446         CL_ASSERT( p_port );\r
1447         cl_obj_lock( &p_port->obj );\r
1448         if( p_port->state != IB_QPS_RTS )\r
1449         {\r
1450                 cl_obj_unlock( &p_port->obj );\r
1451                 IPOIB_PRINT_EXIT( TRACE_LEVEL_INFORMATION, IPOIB_DBG_RECV,\r
1452                         ("Port in invalid state.  Not reposting.\n") );\r
1453                 return 0;\r
1454         }\r
1455         ipoib_port_ref( p_port, ref_repost );\r
1456         cl_obj_unlock( &p_port->obj );\r
1457 \r
1458         while( p_port->recv_mgr.depth < p_port->p_adapter->params.rq_depth )\r
1459         {\r
1460                 /* Pull receives out of the pool and chain them up. */\r
1461                 cl_perf_start( GetRecv );\r
1462                 p_next = __buf_mgr_get_recv( p_port );\r
1463                 cl_perf_stop( &p_port->p_adapter->perf, GetRecv );\r
1464                 if( !p_next )\r
1465                 {\r
1466                         IPOIB_PRINT(TRACE_LEVEL_INFORMATION, IPOIB_DBG_RECV,\r
1467                                 ("Out of receive descriptors! recv queue depath 0x%x\n",p_port->recv_mgr.depth) );\r
1468                         break;\r
1469                 }\r
1470 \r
1471                 if( !p_tail )\r
1472                 {\r
1473                         p_tail = p_next;\r
1474                         p_next->wr.p_next = NULL;\r
1475                 }\r
1476                 else\r
1477                 {\r
1478                         p_next->wr.p_next = &p_head->wr;\r
1479                 }\r
1480 \r
1481                 p_head = p_next;\r
1482 \r
1483                 p_port->recv_mgr.depth++;\r
1484         }\r
1485 \r
1486         if( p_head )\r
1487         {\r
1488                 cl_perf_start( PostRecv );\r
1489                 status = p_port->p_adapter->p_ifc->post_recv(\r
1490                         p_port->ib_mgr.h_qp, &p_head->wr, &p_failed );\r
1491                 cl_perf_stop( &p_port->p_adapter->perf, PostRecv );\r
1492 \r
1493                 if( status != IB_SUCCESS )\r
1494                 {\r
1495                         IPOIB_PRINT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1496                                 ("ip_post_recv returned %s\n", \r
1497                                 p_port->p_adapter->p_ifc->get_err_str( status )) );\r
1498                         /* return the descriptors to the pool */\r
1499                         while( p_failed )\r
1500                         {\r
1501                                 p_head = PARENT_STRUCT( p_failed, ipoib_recv_desc_t, wr );\r
1502                                 p_failed = p_failed->p_next;\r
1503 \r
1504                                 __buf_mgr_put_recv( p_port, p_head, NULL );\r
1505                                 p_port->recv_mgr.depth--;\r
1506                         }\r
1507                 }\r
1508         }\r
1509 \r
1510         ipoib_port_deref( p_port, ref_repost );\r
1511         IPOIB_EXIT( IPOIB_DBG_RECV );\r
1512         return p_port->p_adapter->params.rq_low_watermark - p_port->recv_mgr.depth;\r
1513 }\r
1514 \r
1515 \r
1516 void\r
1517 ipoib_return_packet(\r
1518         IN                              NDIS_HANDLE                                     adapter_context,\r
1519         IN                              NDIS_PACKET                                     *p_packet )\r
1520 {\r
1521         cl_list_item_t          *p_item;\r
1522         ipoib_port_t            *p_port;\r
1523         ipoib_recv_desc_t       *p_desc;\r
1524         ib_api_status_t         status = IB_NOT_DONE;\r
1525         int32_t                         shortage;\r
1526         PERF_DECLARE( ReturnPacket );\r
1527         PERF_DECLARE( ReturnPutRecv );\r
1528         PERF_DECLARE( ReturnRepostRecv );\r
1529         PERF_DECLARE( ReturnPreparePkt );\r
1530         PERF_DECLARE( ReturnNdisIndicate );\r
1531 \r
1532         IPOIB_ENTER( IPOIB_DBG_RECV );\r
1533 \r
1534         UNUSED_PARAM( adapter_context );\r
1535         CL_ASSERT( p_packet );\r
1536 \r
1537         cl_perf_start( ReturnPacket );\r
1538 \r
1539         /* Get the port and descriptor from the packet. */\r
1540         p_port = IPOIB_PORT_FROM_PACKET( p_packet );\r
1541         p_desc = IPOIB_RECV_FROM_PACKET( p_packet );\r
1542 \r
1543         cl_spinlock_acquire( &p_port->recv_lock );\r
1544 \r
1545         cl_perf_start( ReturnPutRecv );\r
1546         __buf_mgr_put_recv( p_port, p_desc, p_packet );\r
1547         cl_perf_stop( &p_port->p_adapter->perf, ReturnPutRecv );\r
1548 \r
1549         /* Repost buffers. */\r
1550         cl_perf_start( ReturnRepostRecv );\r
1551         shortage = __recv_mgr_repost( p_port );\r
1552         cl_perf_stop( &p_port->p_adapter->perf, ReturnRepostRecv );\r
1553 \r
1554         for( p_item = cl_qlist_remove_head( &p_port->recv_mgr.done_list );\r
1555                 p_item != cl_qlist_end( &p_port->recv_mgr.done_list );\r
1556                 p_item = cl_qlist_remove_head( &p_port->recv_mgr.done_list ) )\r
1557         {\r
1558                 p_desc = (ipoib_recv_desc_t*)p_item;\r
1559 \r
1560                 cl_perf_start( ReturnPreparePkt );\r
1561                 status = __recv_mgr_prepare_pkt( p_port, p_desc, &p_packet );\r
1562                 cl_perf_stop( &p_port->p_adapter->perf, ReturnPreparePkt );\r
1563                 if( status == IB_SUCCESS )\r
1564                 {\r
1565                         if( shortage > 0 )\r
1566                                 NDIS_SET_PACKET_STATUS( p_packet, NDIS_STATUS_RESOURCES );\r
1567                         else\r
1568                                 NDIS_SET_PACKET_STATUS( p_packet, NDIS_STATUS_SUCCESS );\r
1569 \r
1570                         cl_spinlock_release( &p_port->recv_lock );\r
1571                         cl_perf_start( ReturnNdisIndicate );\r
1572                         NdisMIndicateReceivePacket( p_port->p_adapter->h_adapter,\r
1573                                 &p_packet, 1 );\r
1574                         cl_perf_stop( &p_port->p_adapter->perf, ReturnNdisIndicate );\r
1575                         cl_spinlock_acquire( &p_port->recv_lock );\r
1576 \r
1577                         if( shortage > 0 )\r
1578                         {\r
1579                                 cl_perf_start( ReturnPutRecv );\r
1580                                 __buf_mgr_put_recv( p_port, p_desc, p_packet );\r
1581                                 cl_perf_stop( &p_port->p_adapter->perf, ReturnPutRecv );\r
1582 \r
1583                                 /* Repost buffers. */\r
1584                                 cl_perf_start( ReturnRepostRecv );\r
1585                                 shortage = __recv_mgr_repost( p_port );\r
1586                                 cl_perf_stop( &p_port->p_adapter->perf, ReturnRepostRecv );\r
1587                         }\r
1588                 }\r
1589                 else if( status != IB_NOT_DONE )\r
1590                 {\r
1591                         IPOIB_PRINT(TRACE_LEVEL_INFORMATION, IPOIB_DBG_RECV,\r
1592                                 ("__recv_mgr_prepare_pkt returned %s\n",\r
1593                                 p_port->p_adapter->p_ifc->get_err_str( status )) );\r
1594                         /* Return the item to the head of the list. */\r
1595                         cl_qlist_insert_head( &p_port->recv_mgr.done_list, p_item );\r
1596                         break;\r
1597                 }\r
1598         }\r
1599         cl_spinlock_release( &p_port->recv_lock );\r
1600         cl_perf_stop( &p_port->p_adapter->perf, ReturnPacket );\r
1601 \r
1602         IPOIB_EXIT( IPOIB_DBG_RECV );\r
1603 }\r
1604 \r
1605 \r
1606 static void\r
1607 __recv_cb(\r
1608         IN              const   ib_cq_handle_t                          h_cq,\r
1609         IN                              void                                            *cq_context )\r
1610 {\r
1611         ipoib_port_t            *p_port;\r
1612         ib_api_status_t         status;\r
1613         ib_wc_t                         wc[MAX_RECV_WC], *p_free, *p_wc;\r
1614         int32_t                         pkt_cnt, recv_cnt = 0, shortage, discarded;\r
1615         cl_qlist_t                      done_list, bad_list;\r
1616         size_t                          i;\r
1617         PERF_DECLARE( RecvCompBundle );\r
1618         PERF_DECLARE( RecvCb );\r
1619         PERF_DECLARE( PollRecv );\r
1620         PERF_DECLARE( RepostRecv );\r
1621         PERF_DECLARE( FilterRecv );\r
1622         PERF_DECLARE( BuildPktArray );\r
1623         PERF_DECLARE( RecvNdisIndicate );\r
1624         PERF_DECLARE( RearmRecv );\r
1625         PERF_DECLARE( PutRecvList );\r
1626 \r
1627         IPOIB_ENTER( IPOIB_DBG_RECV );\r
1628 \r
1629         cl_perf_clr( RecvCompBundle );\r
1630 \r
1631         cl_perf_start( RecvCb );\r
1632 \r
1633         UNUSED_PARAM( h_cq );\r
1634 \r
1635         p_port = (ipoib_port_t*)cq_context;\r
1636 \r
1637         cl_qlist_init( &done_list );\r
1638         cl_qlist_init( &bad_list );\r
1639 \r
1640         ipoib_port_ref( p_port, ref_recv_cb );\r
1641         for( i = 0; i < MAX_RECV_WC; i++ )\r
1642                 wc[i].p_next = &wc[i + 1];\r
1643         wc[MAX_RECV_WC - 1].p_next = NULL;\r
1644 \r
1645         /*\r
1646          * We'll be accessing the endpoint map so take a reference\r
1647          * on it to prevent modifications.\r
1648          */\r
1649         cl_obj_lock( &p_port->obj );\r
1650         cl_atomic_inc( &p_port->endpt_rdr );\r
1651         cl_obj_unlock( &p_port->obj );\r
1652 \r
1653         do\r
1654         {\r
1655                 /* If we get here, then the list of WCs is intact. */\r
1656                 p_free = wc;\r
1657 \r
1658                 cl_perf_start( PollRecv );\r
1659                 status = p_port->p_adapter->p_ifc->poll_cq(\r
1660                         p_port->ib_mgr.h_recv_cq, &p_free, &p_wc );\r
1661                 cl_perf_stop( &p_port->p_adapter->perf, PollRecv );\r
1662                 CL_ASSERT( status == IB_SUCCESS || status == IB_NOT_FOUND );\r
1663 \r
1664                 /* Look at the payload now and filter ARP and DHCP packets. */\r
1665                 cl_perf_start( FilterRecv );\r
1666                 recv_cnt += __recv_mgr_filter( p_port, p_wc, &done_list, &bad_list );\r
1667                 cl_perf_stop( &p_port->p_adapter->perf, FilterRecv );\r
1668 \r
1669         } while( !p_free );\r
1670 \r
1671         /* We're done looking at the endpoint map, release the reference. */\r
1672         cl_atomic_dec( &p_port->endpt_rdr );\r
1673 \r
1674         cl_perf_log( &p_port->p_adapter->perf, RecvCompBundle, recv_cnt );\r
1675 \r
1676         cl_spinlock_acquire( &p_port->recv_lock );\r
1677 \r
1678         /* Update our posted depth. */\r
1679         p_port->recv_mgr.depth -= recv_cnt;\r
1680 \r
1681         /* Return any discarded receives to the pool */\r
1682         cl_perf_start( PutRecvList );\r
1683         __buf_mgr_put_recv_list( p_port, &bad_list );\r
1684         cl_perf_stop( &p_port->p_adapter->perf, PutRecvList );\r
1685 \r
1686         do\r
1687         {\r
1688                 /* Repost ASAP so we don't starve the RQ. */\r
1689                 cl_perf_start( RepostRecv );\r
1690                 shortage = __recv_mgr_repost( p_port );\r
1691                 cl_perf_stop( &p_port->p_adapter->perf, RepostRecv );\r
1692 \r
1693                 cl_perf_start( BuildPktArray );\r
1694                 /* Notify NDIS of any and all possible receive buffers. */\r
1695                 pkt_cnt = __recv_mgr_build_pkt_array(\r
1696                         p_port, shortage, &done_list, &discarded );\r
1697                 cl_perf_stop( &p_port->p_adapter->perf, BuildPktArray );\r
1698 \r
1699                 /* Only indicate receives if we actually had any. */\r
1700                 if( discarded && shortage > 0 )\r
1701                 {\r
1702                         /* We may have thrown away packets, and have a shortage */\r
1703                         cl_perf_start( RepostRecv );\r
1704                         __recv_mgr_repost( p_port );\r
1705                         cl_perf_stop( &p_port->p_adapter->perf, RepostRecv );\r
1706                 }\r
1707 \r
1708                 if( !pkt_cnt )\r
1709                         break;\r
1710 \r
1711                 cl_spinlock_release( &p_port->recv_lock );\r
1712 \r
1713                 cl_perf_start( RecvNdisIndicate );\r
1714                 NdisMIndicateReceivePacket( p_port->p_adapter->h_adapter,\r
1715                         p_port->recv_mgr.recv_pkt_array, pkt_cnt );\r
1716                 cl_perf_stop( &p_port->p_adapter->perf, RecvNdisIndicate );\r
1717 \r
1718                 /*\r
1719                  * Cap the number of receives to put back to what we just indicated\r
1720                  * with NDIS_STATUS_RESOURCES.\r
1721                  */\r
1722                 if( shortage > 0 )\r
1723                 {\r
1724                         if( pkt_cnt < shortage )\r
1725                                 shortage = pkt_cnt;\r
1726 \r
1727                         /* Return all but the last packet to the pool. */\r
1728                         cl_spinlock_acquire( &p_port->recv_lock );\r
1729                         while( shortage-- > 1 )\r
1730                         {\r
1731                                 __buf_mgr_put_recv( p_port,\r
1732                                         IPOIB_RECV_FROM_PACKET( p_port->recv_mgr.recv_pkt_array[shortage] ),\r
1733                                         p_port->recv_mgr.recv_pkt_array[shortage] );\r
1734                         }\r
1735                         cl_spinlock_release( &p_port->recv_lock );\r
1736 \r
1737                         /*\r
1738                          * Return the last packet as if NDIS returned it, so that we repost\r
1739                          * and report any other pending receives.\r
1740                          */\r
1741                         ipoib_return_packet( NULL, p_port->recv_mgr.recv_pkt_array[0] );\r
1742                 }\r
1743                 cl_spinlock_acquire( &p_port->recv_lock );\r
1744 \r
1745         } while( pkt_cnt );\r
1746         cl_spinlock_release( &p_port->recv_lock );\r
1747 \r
1748         /*\r
1749          * Rearm after filtering to prevent contention on the enpoint maps\r
1750          * and eliminate the possibility of having a call to\r
1751          * __endpt_mgr_insert find a duplicate.\r
1752          */\r
1753         cl_perf_start( RearmRecv );\r
1754         status = p_port->p_adapter->p_ifc->rearm_cq(\r
1755                 p_port->ib_mgr.h_recv_cq, FALSE );\r
1756         cl_perf_stop( &p_port->p_adapter->perf, RearmRecv );\r
1757         CL_ASSERT( status == IB_SUCCESS );\r
1758 \r
1759         ipoib_port_deref( p_port, ref_recv_cb );\r
1760 \r
1761         cl_perf_stop( &p_port->p_adapter->perf, RecvCb );\r
1762 \r
1763         IPOIB_EXIT( IPOIB_DBG_RECV );\r
1764 }\r
1765 \r
1766 \r
1767 static void\r
1768 __recv_get_endpts(\r
1769         IN                              ipoib_port_t* const                     p_port,\r
1770         IN                              ipoib_recv_desc_t* const        p_desc,\r
1771         IN                              ib_wc_t* const                          p_wc,\r
1772                 OUT                     ipoib_endpt_t** const           pp_src,\r
1773                 OUT                     ipoib_endpt_t** const           pp_dst )\r
1774 {\r
1775         ib_api_status_t         status;\r
1776         mac_addr_t                      mac;\r
1777         PERF_DECLARE( GetEndptByGid );\r
1778         PERF_DECLARE( GetEndptByLid );\r
1779         PERF_DECLARE( EndptInsert );\r
1780 \r
1781         IPOIB_ENTER( IPOIB_DBG_RECV );\r
1782 \r
1783         /* Setup our shortcut pointers based on whether GRH is valid. */\r
1784         if( p_wc->recv.ud.recv_opt & IB_RECV_OPT_GRH_VALID )\r
1785         {\r
1786                 /* Lookup the source endpoints based on GID. */\r
1787                 cl_perf_start( GetEndptByGid );\r
1788                 *pp_src =\r
1789 #if IPOIB_INLINE_RECV\r
1790                         __endpt_mgr_get_by_gid( p_port, &p_desc->buf.ib.grh.src_gid );\r
1791 #else   /* IPOIB_INLINE_RECV */\r
1792                         __endpt_mgr_get_by_gid( p_port, &p_desc->p_buf->ib.grh.src_gid );\r
1793 #endif  /* IPOIB_INLINE_RECV */\r
1794                 cl_perf_stop( &p_port->p_adapter->perf, GetEndptByGid );\r
1795 \r
1796                 /*\r
1797                  * Lookup the destination endpoint based on GID.\r
1798                  * This is used along with the packet filter to determine\r
1799                  * whether to report this to NDIS.\r
1800                  */\r
1801                 cl_perf_start( GetEndptByGid );\r
1802                 *pp_dst =\r
1803 #if IPOIB_INLINE_RECV\r
1804                         __endpt_mgr_get_by_gid( p_port, &p_desc->buf.ib.grh.dest_gid );\r
1805 #else   /* IPOIB_INLINE_RECV */\r
1806                         __endpt_mgr_get_by_gid( p_port, &p_desc->p_buf->ib.grh.dest_gid );\r
1807 #endif  /* IPOIB_INLINE_RECV */\r
1808                 cl_perf_stop( &p_port->p_adapter->perf, GetEndptByGid );\r
1809 \r
1810                 /*\r
1811                  * Create the source endpoint if it does not exist.  Note that we\r
1812                  * can only do this for globally routed traffic since we need the\r
1813                  * information from the GRH to generate the MAC.\r
1814                  */\r
1815                 if( !*pp_src )\r
1816                 {\r
1817                         status = ipoib_mac_from_guid(\r
1818 #if IPOIB_INLINE_RECV\r
1819                                 p_desc->buf.ib.grh.src_gid.unicast.interface_id, p_port->p_adapter->params.guid_mask, &mac );\r
1820 #else   /* IPOIB_INLINE_RECV */\r
1821                                 p_desc->p_buf->ib.grh.src_gid.unicast.interface_id, p_port->p_adapter->params.guid_mask, &mac );\r
1822 #endif  /* IPOIB_INLINE_RECV */\r
1823                         if( status != IB_SUCCESS )\r
1824                         {\r
1825                                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1826                                         ("ipoib_mac_from_guid returned %s\n",\r
1827                                         p_port->p_adapter->p_ifc->get_err_str( status )) );\r
1828                                 return;\r
1829                         }\r
1830 \r
1831                         /* Create the endpoint. */\r
1832 #if IPOIB_INLINE_RECV\r
1833                         *pp_src = ipoib_endpt_create( &p_desc->buf.ib.grh.src_gid,\r
1834 #else   /* IPOIB_INLINE_RECV */\r
1835                         *pp_src = ipoib_endpt_create( &p_desc->p_buf->ib.grh.src_gid,\r
1836 #endif  /* IPOIB_INLINE_RECV */\r
1837                                 p_wc->recv.ud.remote_lid, p_wc->recv.ud.remote_qp );\r
1838                         if( !*pp_src )\r
1839                         {\r
1840                                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1841                                         ("ipoib_endpt_create failed\n") );\r
1842                                 return;\r
1843                         }\r
1844                         cl_perf_start( EndptInsert );\r
1845                         cl_obj_lock( &p_port->obj );\r
1846                         status = __endpt_mgr_insert( p_port, mac, *pp_src );\r
1847                         if( status != IB_SUCCESS )\r
1848                         {\r
1849                                 cl_obj_unlock( &p_port->obj );\r
1850                                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1851                                         ("__endpt_mgr_insert returned %s\n",\r
1852                                         p_port->p_adapter->p_ifc->get_err_str( status )) );\r
1853                                 return;\r
1854                         }\r
1855                         cl_obj_unlock( &p_port->obj );\r
1856                         cl_perf_stop( &p_port->p_adapter->perf, EndptInsert );\r
1857                 }\r
1858         }\r
1859         else\r
1860         {\r
1861                 /*\r
1862                  * Lookup the remote endpoint based on LID.  Note that only\r
1863                  * unicast traffic can be LID routed.\r
1864                  */\r
1865                 cl_perf_start( GetEndptByLid );\r
1866                 *pp_src = __endpt_mgr_get_by_lid( p_port, p_wc->recv.ud.remote_lid );\r
1867                 cl_perf_stop( &p_port->p_adapter->perf, GetEndptByLid );\r
1868                 *pp_dst = p_port->p_local_endpt;\r
1869                 CL_ASSERT( *pp_dst );\r
1870         }\r
1871 \r
1872         if( *pp_src && !ipoib_is_voltaire_router_gid( &(*pp_src)->dgid ) &&\r
1873                 (*pp_src)->qpn != p_wc->recv.ud.remote_qp )\r
1874         {\r
1875                 /* Update the QPN for the endpoint. */\r
1876                 IPOIB_PRINT(TRACE_LEVEL_INFORMATION, IPOIB_DBG_RECV,\r
1877                         ("Updating QPN for MAC: %02X-%02X-%02X-%02X-%02X-%02X\n",\r
1878                         (*pp_src )->mac.addr[0], (*pp_src )->mac.addr[1],\r
1879                         (*pp_src )->mac.addr[2], (*pp_src )->mac.addr[3],\r
1880                         (*pp_src )->mac.addr[4], (*pp_src )->mac.addr[5]) );\r
1881                 (*pp_src)->qpn = p_wc->recv.ud.remote_qp;\r
1882         }\r
1883 \r
1884         if( *pp_src && *pp_dst )\r
1885         {\r
1886                 IPOIB_PRINT(TRACE_LEVEL_INFORMATION, IPOIB_DBG_RECV,\r
1887                         ("Recv:\n"\r
1888                         "\tsrc MAC: %02X-%02X-%02X-%02X-%02X-%02X\n"\r
1889                         "\tdst MAC: %02X-%02X-%02X-%02X-%02X-%02X\n",\r
1890                         (*pp_src )->mac.addr[0], (*pp_src )->mac.addr[1],\r
1891                         (*pp_src )->mac.addr[2], (*pp_src )->mac.addr[3],\r
1892                         (*pp_src )->mac.addr[4], (*pp_src )->mac.addr[5],\r
1893                         (*pp_dst )->mac.addr[0], (*pp_dst )->mac.addr[1],\r
1894                         (*pp_dst )->mac.addr[2], (*pp_dst )->mac.addr[3],\r
1895                         (*pp_dst )->mac.addr[4], (*pp_dst )->mac.addr[5]) );\r
1896         }\r
1897 \r
1898         IPOIB_EXIT( IPOIB_DBG_RECV );\r
1899 }\r
1900 \r
1901 \r
1902 static int32_t\r
1903 __recv_mgr_filter(\r
1904         IN                              ipoib_port_t* const                     p_port,\r
1905         IN                              ib_wc_t* const                          p_done_wc_list,\r
1906                 OUT                     cl_qlist_t* const                       p_done_list,\r
1907                 OUT                     cl_qlist_t* const                       p_bad_list )\r
1908 {\r
1909         ipoib_recv_desc_t               *p_desc;\r
1910         ib_wc_t                                 *p_wc;\r
1911         ipoib_pkt_t                             *p_ipoib;\r
1912         eth_pkt_t                               *p_eth;\r
1913         ipoib_endpt_t                   *p_src, *p_dst;\r
1914         ib_api_status_t                 status;\r
1915         uint32_t                                len;\r
1916         int32_t                                 recv_cnt = 0;\r
1917         PERF_DECLARE( GetRecvEndpts );\r
1918         PERF_DECLARE( RecvGen );\r
1919         PERF_DECLARE( RecvTcp );\r
1920         PERF_DECLARE( RecvUdp );\r
1921         PERF_DECLARE( RecvDhcp );\r
1922         PERF_DECLARE( RecvArp );\r
1923 \r
1924         IPOIB_ENTER( IPOIB_DBG_RECV );\r
1925 \r
1926         for( p_wc = p_done_wc_list; p_wc; p_wc = p_wc->p_next )\r
1927         {\r
1928                 CL_ASSERT( p_wc->status != IB_WCS_SUCCESS || p_wc->wc_type == IB_WC_RECV );\r
1929                 p_desc = (ipoib_recv_desc_t*)(uintn_t)p_wc->wr_id;\r
1930                 recv_cnt++;\r
1931 \r
1932                 if( p_wc->status != IB_WCS_SUCCESS )\r
1933                 {\r
1934                         if( p_wc->status != IB_WCS_WR_FLUSHED_ERR )\r
1935                         {\r
1936                                 IPOIB_PRINT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1937                                         ("Failed completion %s  (vendor specific %#x)\n",\r
1938                                         p_port->p_adapter->p_ifc->get_wc_status_str( p_wc->status ),\r
1939                                         (int)p_wc->vendor_specific) );\r
1940                                 ipoib_inc_recv_stat( p_port->p_adapter, IP_STAT_ERROR, 0 );\r
1941                         }\r
1942                         else\r
1943                         {\r
1944                                 IPOIB_PRINT(TRACE_LEVEL_INFORMATION, IPOIB_DBG_RECV,\r
1945                                         ("Flushed completion %s\n",\r
1946                                         p_port->p_adapter->p_ifc->get_wc_status_str( p_wc->status )) );\r
1947                                 ipoib_inc_recv_stat( p_port->p_adapter, IP_STAT_DROPPED, 0 );\r
1948                         }\r
1949                         cl_qlist_insert_tail( p_bad_list, &p_desc->item.list_item );\r
1950                         /* Dereference the port object on behalf of the failed receive. */\r
1951                         ipoib_port_deref( p_port, ref_failed_recv_wc );\r
1952                         continue;\r
1953                 }\r
1954 \r
1955                 len = p_wc->length - sizeof(ib_grh_t);\r
1956 \r
1957                 if( len < sizeof(ipoib_hdr_t) )\r
1958                 {\r
1959                         IPOIB_PRINT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1960                                 ("Received ETH packet < min size\n") );\r
1961                         ipoib_inc_recv_stat( p_port->p_adapter, IP_STAT_ERROR, 0 );\r
1962                         cl_qlist_insert_tail( p_bad_list, &p_desc->item.list_item );\r
1963                         ipoib_port_deref( p_port, ref_recv_inv_len );\r
1964                         continue;\r
1965                 }\r
1966 \r
1967                 if((len - sizeof(ipoib_hdr_t)) > p_port->p_adapter->params.payload_mtu)\r
1968                 {\r
1969                         IPOIB_PRINT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1970                                 ("Received ETH packet > payload MTU (%d)\n",\r
1971                                 p_port->p_adapter->params.payload_mtu) );\r
1972                         ipoib_inc_recv_stat( p_port->p_adapter, IP_STAT_ERROR, 0 );\r
1973                         cl_qlist_insert_tail( p_bad_list, &p_desc->item.list_item );\r
1974                         ipoib_port_deref( p_port, ref_recv_inv_len );\r
1975                         continue;\r
1976                         \r
1977                 }\r
1978                 /* Successful completion.  Get the receive information. */\r
1979                 p_desc->ndis_csum.Value = ( (p_wc->recv.ud.recv_opt & IB_RECV_OPT_CSUM_MASK ) >> 8 );\r
1980                 cl_perf_start( GetRecvEndpts );\r
1981                 __recv_get_endpts( p_port, p_desc, p_wc, &p_src, &p_dst );\r
1982                 cl_perf_stop( &p_port->p_adapter->perf, GetRecvEndpts );\r
1983 \r
1984 #if IPOIB_INLINE_RECV\r
1985                 p_ipoib = &p_desc->buf.ib.pkt;\r
1986                 p_eth = &p_desc->buf.eth.pkt;\r
1987 #else   /* IPOIB_INLINE_RECV */\r
1988                 p_ipoib = &p_desc->p_buf->ib.pkt;\r
1989                 p_eth = &p_desc->p_buf->eth.pkt;\r
1990 #endif  /*IPOIB_INLINE_RECV */\r
1991 \r
1992                 if( p_src )\r
1993                 {\r
1994                         /* Don't report loopback traffic - we requested SW loopback. */\r
1995                         if( !cl_memcmp( &p_port->p_adapter->params.conf_mac,\r
1996                                 &p_src->mac, sizeof(p_port->p_adapter->params.conf_mac) ) )\r
1997                         {\r
1998                                 /*\r
1999                                  * "This is not the packet you're looking for" - don't update\r
2000                                  * receive statistics, the packet never happened.\r
2001                                  */\r
2002                                 cl_qlist_insert_tail( p_bad_list, &p_desc->item.list_item );\r
2003                                 /* Dereference the port object on behalf of the failed recv. */\r
2004                                 ipoib_port_deref( p_port, ref_recv_loopback );\r
2005                                 continue;\r
2006                         }\r
2007                 }\r
2008 \r
2009                 switch( p_ipoib->hdr.type )\r
2010                 {\r
2011                 case ETH_PROT_TYPE_IP:\r
2012                         if( len < (sizeof(ipoib_hdr_t) + sizeof(ip_hdr_t)) )\r
2013                         {\r
2014                                 IPOIB_PRINT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2015                                         ("Received IP packet < min size\n") );\r
2016                                 status = IB_INVALID_SETTING;\r
2017                                 break;\r
2018                         }\r
2019 \r
2020                         if( p_ipoib->type.ip.hdr.offset ||\r
2021                                 p_ipoib->type.ip.hdr.prot != IP_PROT_UDP )\r
2022                         {\r
2023                                 /* Unfiltered.  Setup the ethernet header and report. */\r
2024                                 cl_perf_start( RecvTcp );\r
2025                                 status = __recv_gen( p_ipoib, p_eth, p_src, p_dst );\r
2026                                 cl_perf_stop( &p_port->p_adapter->perf, RecvTcp );\r
2027                                 break;\r
2028                         }\r
2029 \r
2030                         /* First packet of a UDP transfer. */\r
2031                         if( len <\r
2032                                 (sizeof(ipoib_hdr_t) + sizeof(ip_hdr_t) + sizeof(udp_hdr_t)) )\r
2033                         {\r
2034                                 IPOIB_PRINT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2035                                         ("Received UDP packet < min size\n") );\r
2036                                 status = IB_INVALID_SETTING;\r
2037                                 break;\r
2038                         }\r
2039 \r
2040                         /* Check if DHCP conversion is required. */\r
2041                         if( (p_ipoib->type.ip.prot.udp.hdr.dst_port == DHCP_PORT_SERVER &&\r
2042                                 p_ipoib->type.ip.prot.udp.hdr.src_port == DHCP_PORT_CLIENT) ||\r
2043                                 (p_ipoib->type.ip.prot.udp.hdr.dst_port == DHCP_PORT_CLIENT &&\r
2044                                 p_ipoib->type.ip.prot.udp.hdr.src_port == DHCP_PORT_SERVER) )\r
2045                         {\r
2046                                 if( len < (sizeof(ipoib_hdr_t) + sizeof(ip_hdr_t) +\r
2047                                         sizeof(udp_hdr_t) + DHCP_MIN_SIZE) )\r
2048                                 {\r
2049                                         IPOIB_PRINT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2050                                                 ("Received DHCP < min size\n") );\r
2051                                         status = IB_INVALID_SETTING;\r
2052                                         break;\r
2053                                 }\r
2054                                 if ((p_ipoib->type.ip.hdr.ver_hl & 0x0f) != 5 ) {\r
2055                                         // If there are IP options in this message, we are in trouble in any case\r
2056                                         status = IB_INVALID_SETTING;\r
2057                                         break;                                  \r
2058                                 }\r
2059                                 /* UDP packet with BOOTP ports in src/dst port numbers. */\r
2060                                 cl_perf_start( RecvDhcp );\r
2061                                 status = __recv_dhcp( p_port, p_ipoib, p_eth, p_src, p_dst );\r
2062                                 cl_perf_stop( &p_port->p_adapter->perf, RecvDhcp );\r
2063                         }\r
2064                         else\r
2065                         {\r
2066                                 /* Unfiltered.  Setup the ethernet header and report. */\r
2067                                 cl_perf_start( RecvUdp );\r
2068                                 status = __recv_gen( p_ipoib, p_eth, p_src, p_dst );\r
2069                                 cl_perf_stop( &p_port->p_adapter->perf, RecvUdp );\r
2070                         }\r
2071                         break;\r
2072 \r
2073                 case ETH_PROT_TYPE_ARP:\r
2074                         if( len < (sizeof(ipoib_hdr_t) + sizeof(ipoib_arp_pkt_t)) )\r
2075                         {\r
2076                                 IPOIB_PRINT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2077                                         ("Received ARP < min size\n") );\r
2078                                 status = IB_INVALID_SETTING;\r
2079                                 break;\r
2080                         }\r
2081                         cl_perf_start( RecvArp );\r
2082                         status = __recv_arp( p_port, p_wc, p_ipoib, p_eth, &p_src, p_dst );\r
2083                         cl_perf_stop( &p_port->p_adapter->perf, RecvArp );\r
2084                         len = sizeof(ipoib_hdr_t) + sizeof(arp_pkt_t);\r
2085                         break;\r
2086 \r
2087                 default:\r
2088                         /* Unfiltered.  Setup the ethernet header and report. */\r
2089                         cl_perf_start( RecvGen );\r
2090                         status = __recv_gen( p_ipoib, p_eth, p_src, p_dst );\r
2091                         cl_perf_stop( &p_port->p_adapter->perf, RecvGen );\r
2092                 }\r
2093 \r
2094                 if( status != IB_SUCCESS )\r
2095                 {\r
2096                         /* Update stats. */\r
2097                         ipoib_inc_recv_stat( p_port->p_adapter, IP_STAT_ERROR, 0 );\r
2098                         cl_qlist_insert_tail( p_bad_list, &p_desc->item.list_item );\r
2099                         /* Dereference the port object on behalf of the failed receive. */\r
2100                         ipoib_port_deref( p_port, ref_recv_filter );\r
2101                 }\r
2102                 else\r
2103                 {\r
2104                         ip_stat_sel_t               ip_stat;\r
2105                         p_desc->len =\r
2106                                 len + sizeof(eth_hdr_t) - sizeof(ipoib_hdr_t);\r
2107                         if( p_dst->h_mcast)\r
2108                         {\r
2109                                 if( p_dst->dgid.multicast.raw_group_id[10] == 0xFF &&\r
2110                                         p_dst->dgid.multicast.raw_group_id[11] == 0xFF &&\r
2111                                         p_dst->dgid.multicast.raw_group_id[12] == 0xFF &&\r
2112                                         p_dst->dgid.multicast.raw_group_id[13] == 0xFF )\r
2113                                 {\r
2114                                         p_desc->type = PKT_TYPE_BCAST;\r
2115                                         ip_stat = IP_STAT_BCAST_BYTES;\r
2116                                 }\r
2117                                 else\r
2118                                 {\r
2119                                         p_desc->type = PKT_TYPE_MCAST;\r
2120                                         ip_stat = IP_STAT_MCAST_BYTES;\r
2121                                 }\r
2122                         }\r
2123                         else\r
2124                         {\r
2125                                 p_desc->type = PKT_TYPE_UCAST;\r
2126                                 ip_stat = IP_STAT_UCAST_BYTES;\r
2127                                 \r
2128                         }\r
2129                         cl_qlist_insert_tail( p_done_list, &p_desc->item.list_item );\r
2130                         ipoib_inc_recv_stat( p_port->p_adapter,ip_stat , len );  \r
2131                 }\r
2132         }\r
2133 \r
2134         IPOIB_EXIT( IPOIB_DBG_RECV );\r
2135         return recv_cnt;\r
2136 }\r
2137 \r
2138 \r
2139 static ib_api_status_t\r
2140 __recv_gen(\r
2141         IN              const   ipoib_pkt_t* const                      p_ipoib,\r
2142                 OUT                     eth_pkt_t* const                        p_eth,\r
2143         IN                              ipoib_endpt_t* const            p_src,\r
2144         IN                              ipoib_endpt_t* const            p_dst )\r
2145 {\r
2146         IPOIB_ENTER( IPOIB_DBG_RECV );\r
2147 \r
2148         if( !p_src || !p_dst )\r
2149         {\r
2150                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2151                         ("Received packet with no matching endpoints.\n") );\r
2152                 return IB_NOT_DONE;\r
2153         }\r
2154 \r
2155         /*\r
2156          * Fill in the ethernet header.  Note that doing so will overwrite\r
2157          * the IPoIB header, so start by moving the information from the IPoIB\r
2158          * header.\r
2159          */\r
2160         p_eth->hdr.type = p_ipoib->hdr.type;\r
2161         p_eth->hdr.src = p_src->mac;\r
2162         p_eth->hdr.dst = p_dst->mac;\r
2163 \r
2164         if ( p_eth->hdr.dst.addr[0] == 1 && \r
2165                  p_eth->hdr.type == ETH_PROT_TYPE_IP &&\r
2166                  p_eth->hdr.dst.addr[2] == 0x5E)  \r
2167         {\r
2168                 p_eth->hdr.dst.addr[1] = 0;\r
2169                 p_eth->hdr.dst.addr[3] = p_eth->hdr.dst.addr[3] & 0x7f;\r
2170         }\r
2171         if (p_dst->h_mcast)\r
2172                 p_dst->is_in_use = TRUE;\r
2173 \r
2174         IPOIB_EXIT( IPOIB_DBG_RECV );\r
2175         return IB_SUCCESS;\r
2176 }\r
2177 \r
2178 \r
2179 static ib_api_status_t\r
2180 __recv_dhcp(\r
2181         IN                              ipoib_port_t* const                     p_port,\r
2182         IN              const   ipoib_pkt_t* const                      p_ipoib,\r
2183                 OUT                     eth_pkt_t* const                        p_eth,\r
2184         IN                              ipoib_endpt_t* const            p_src,\r
2185         IN                              ipoib_endpt_t* const            p_dst )\r
2186 {\r
2187         ib_api_status_t         status;\r
2188         dhcp_pkt_t                      *p_dhcp;\r
2189         uint8_t                         *p_option;\r
2190         uint8_t                         *p_cid = NULL;\r
2191         ib_gid_t                        gid;\r
2192         uint8_t                         msg = 0;\r
2193 \r
2194         IPOIB_ENTER( IPOIB_DBG_RECV );\r
2195 \r
2196         UNUSED_PARAM( p_port );\r
2197 \r
2198         /* Create the ethernet header. */\r
2199         status = __recv_gen( p_ipoib, p_eth, p_src, p_dst );\r
2200         if( status != IB_SUCCESS )\r
2201         {\r
2202                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2203                         ("__recv_gen returned %s.\n", \r
2204                         p_port->p_adapter->p_ifc->get_err_str( status )) );\r
2205                 return status;\r
2206         }\r
2207 \r
2208         /* Fixup the payload. */\r
2209         p_dhcp = &p_eth->type.ip.prot.udp.dhcp;\r
2210         if( p_dhcp->op != DHCP_REQUEST && p_dhcp->op != DHCP_REPLY )\r
2211         {\r
2212                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2213                         ("Invalid DHCP op code.\n") );\r
2214                 return IB_INVALID_SETTING;\r
2215         }\r
2216 \r
2217         /*\r
2218          * Find the client identifier option, making sure to skip\r
2219          * the "magic cookie".\r
2220          */\r
2221         p_option = &p_dhcp->options[0];\r
2222         if ( *(uint32_t *)p_option != DHCP_COOKIE )\r
2223         {\r
2224                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2225                         ("DHCP cookie corrupted.\n") );\r
2226                 return IB_INVALID_PARAMETER;\r
2227         }\r
2228 \r
2229         p_option = &p_dhcp->options[4];\r
2230         while( *p_option != DHCP_OPT_END && p_option < &p_dhcp->options[312] )\r
2231         {\r
2232                 switch( *p_option )\r
2233                 {\r
2234                 case DHCP_OPT_PAD:\r
2235                         p_option++;\r
2236                         break;\r
2237 \r
2238                 case DHCP_OPT_MSG:\r
2239                         msg = p_option[2];\r
2240                         p_option += 3;\r
2241                         break;\r
2242 \r
2243                 case DHCP_OPT_CLIENT_ID:\r
2244                         p_cid = p_option;\r
2245                         /* Fall through. */\r
2246 \r
2247                 default:\r
2248                         /*\r
2249                          * All other options have a length byte following the option code.\r
2250                          * Offset by the length to get to the next option.\r
2251                          */\r
2252                         p_option += (p_option[1] + 2);\r
2253                 }\r
2254         }\r
2255 \r
2256         switch( msg )\r
2257         {\r
2258         /* message from client */\r
2259         case DHCPDISCOVER:\r
2260         case DHCPREQUEST:\r
2261         case DHCPDECLINE:\r
2262         case DHCPRELEASE:\r
2263         case DHCPINFORM:\r
2264                 if( !p_cid )\r
2265                 {\r
2266                         IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2267                                 ("Failed to find required Client-identifier option.\n") );\r
2268                         return IB_INVALID_SETTING;\r
2269                 }\r
2270                 if( p_dhcp->htype != DHCP_HW_TYPE_IB )\r
2271                 {\r
2272                         IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2273                                 ("Invalid hardware address type.\n") );\r
2274                         return IB_INVALID_SETTING;\r
2275                 }\r
2276                 break;\r
2277         /* message from DHCP server */\r
2278         case DHCPOFFER:\r
2279         case DHCPACK:\r
2280         case DHCPNAK:\r
2281                 break;\r
2282 \r
2283         default:\r
2284                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2285                         ("Invalide message type.\n") );\r
2286                 return IB_INVALID_PARAMETER;\r
2287         }\r
2288         p_eth->type.ip.prot.udp.hdr.chksum = 0;\r
2289         p_dhcp->htype = DHCP_HW_TYPE_ETH;\r
2290         p_dhcp->hlen = HW_ADDR_LEN;\r
2291 \r
2292         if( p_cid ) /* from client */\r
2293         {\r
2294                 /* Validate that the length and type of the option is as required. */\r
2295                 if( p_cid[1] != 21 )\r
2296                 {\r
2297                         IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2298                                 ("Client-identifier length not 21 as required.\n") );\r
2299                         return IB_INVALID_SETTING;\r
2300                 }\r
2301                 if( p_cid[2] != DHCP_HW_TYPE_IB )\r
2302                 {\r
2303                         IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2304                                 ("Client-identifier type is wrong.\n") );\r
2305                         return IB_INVALID_SETTING;\r
2306                 }\r
2307                 /*\r
2308                  * Copy the GID value from the option so that we can make aligned\r
2309                  * accesses to the contents.\r
2310                  * Recover CID to standard type.\r
2311                  */\r
2312                 cl_memcpy( &gid, &p_cid[7], sizeof(ib_gid_t) );\r
2313                 p_cid[1] =  HW_ADDR_LEN +1;// CID length \r
2314                 p_cid[2] =  DHCP_HW_TYPE_ETH;// CID type \r
2315                 status = ipoib_mac_from_guid( gid.unicast.interface_id, p_port->p_adapter->params.guid_mask, (mac_addr_t*)&p_cid[3] );\r
2316                 if (status == IB_INVALID_GUID_MASK)\r
2317                 {\r
2318                         IPOIB_PRINT( TRACE_LEVEL_WARNING, IPOIB_DBG_ERROR,\r
2319                                 ("Invalid GUID mask received, rejecting it") );\r
2320                         ipoib_create_log(p_port->p_adapter->h_adapter, GUID_MASK_LOG_INDEX, EVENT_IPOIB_WRONG_PARAMETER_WRN);\r
2321                         status = IB_SUCCESS;\r
2322                 }\r
2323                 p_cid[HW_ADDR_LEN + 3] = DHCP_OPT_END; //terminate tag\r
2324         }\r
2325         IPOIB_EXIT( IPOIB_DBG_RECV );\r
2326         return status;\r
2327 }\r
2328 \r
2329 \r
2330 static ib_api_status_t\r
2331 __recv_arp(\r
2332         IN                              ipoib_port_t* const                     p_port,\r
2333         IN                              ib_wc_t* const                          p_wc,\r
2334         IN              const   ipoib_pkt_t* const                      p_ipoib,\r
2335                 OUT                     eth_pkt_t* const                        p_eth,\r
2336         IN                              ipoib_endpt_t** const           pp_src,\r
2337         IN                              ipoib_endpt_t* const            p_dst )\r
2338 {\r
2339         ib_api_status_t                 status;\r
2340         arp_pkt_t                               *p_arp;\r
2341         const ipoib_arp_pkt_t   *p_ib_arp;\r
2342         ib_gid_t                                gid;\r
2343         mac_addr_t                              mac;\r
2344         ipoib_hw_addr_t                 null_hw = {0};\r
2345 \r
2346         IPOIB_ENTER( IPOIB_DBG_RECV );\r
2347 \r
2348         if( !p_dst )\r
2349         {\r
2350                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2351                         ("Unknown destination endpoint\n") );\r
2352                 return IB_INVALID_SETTING;\r
2353         }\r
2354 \r
2355         p_ib_arp = &p_ipoib->type.arp;\r
2356         p_arp = &p_eth->type.arp;\r
2357 \r
2358         if( p_ib_arp->hw_type != ARP_HW_TYPE_IB )\r
2359         {\r
2360                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2361                         ("ARP hardware type is not IB\n") );\r
2362                 return IB_INVALID_SETTING;\r
2363         }\r
2364 \r
2365         if( p_ib_arp->hw_size != sizeof(ipoib_hw_addr_t) )\r
2366         {\r
2367                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2368                         ("ARP hardware address size is not sizeof(ipoib_hw_addr_t)\n") );\r
2369                 return IB_INVALID_SETTING;\r
2370         }\r
2371 \r
2372         if( p_ib_arp->prot_type != ETH_PROT_TYPE_IP )\r
2373         {\r
2374                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2375                         ("ARP protocal type not IP\n") );\r
2376                 return IB_INVALID_SETTING;\r
2377         }\r
2378 \r
2379         /*\r
2380          * If we don't have a source, lookup the endpoint specified in the payload.\r
2381          */\r
2382         if( !*pp_src )\r
2383                 *pp_src = __endpt_mgr_get_by_gid( p_port, &p_ib_arp->src_hw.gid );\r
2384 \r
2385         /*\r
2386          * If the endpoint exists for the GID, make sure\r
2387          * the dlid and qpn match the arp.\r
2388          */\r
2389         if( *pp_src )\r
2390         {\r
2391                 if( cl_memcmp( &(*pp_src)->dgid, &p_ib_arp->src_hw.gid,\r
2392                         sizeof(ib_gid_t) ) )\r
2393                 {\r
2394                         /*\r
2395                          * GIDs for the endpoint are different.  The ARP must\r
2396                          * have been proxied.  Dereference it.\r
2397                          */\r
2398                         *pp_src = NULL;\r
2399                 }\r
2400                 else if( (*pp_src)->dlid &&\r
2401                         (*pp_src)->dlid != p_wc->recv.ud.remote_lid )\r
2402                 {\r
2403                         /* Out of date!  Destroy the endpoint and replace it. */\r
2404                         __endpt_mgr_remove( p_port, *pp_src );\r
2405                         *pp_src = NULL;\r
2406                 }\r
2407                 else if ( ! ((*pp_src)->dlid)) {\r
2408                         /* Out of date!  Destroy the endpoint and replace it. */\r
2409                         __endpt_mgr_remove( p_port, *pp_src );\r
2410                         *pp_src = NULL;\r
2411                 }\r
2412                 else if( ipoib_is_voltaire_router_gid( &(*pp_src)->dgid ) )\r
2413                 {\r
2414                         if( (*pp_src)->qpn !=\r
2415                                 (p_ib_arp->src_hw.flags_qpn & CL_HTON32(0x00FFFFFF)) &&\r
2416                                 p_wc->recv.ud.remote_qp !=\r
2417                                 (p_ib_arp->src_hw.flags_qpn & CL_HTON32(0x00FFFFFF)) )\r
2418                         {\r
2419                                 /* Out of date!  Destroy the endpoint and replace it. */\r
2420                                 __endpt_mgr_remove( p_port, *pp_src );\r
2421                                 *pp_src = NULL;\r
2422                         }\r
2423                 }\r
2424                 else if( (*pp_src)->qpn != p_wc->recv.ud.remote_qp )\r
2425                 {\r
2426                         /* Out of date!  Destroy the endpoint and replace it. */\r
2427                         __endpt_mgr_remove( p_port, *pp_src );\r
2428                         *pp_src = NULL;\r
2429                 }\r
2430         }\r
2431 \r
2432         /* Do we need to create an endpoint for this GID? */\r
2433         if( !*pp_src )\r
2434         {\r
2435                 /* Copy the src GID to allow aligned access */\r
2436                 cl_memcpy( &gid, &p_ib_arp->src_hw.gid, sizeof(ib_gid_t) );\r
2437                 status = ipoib_mac_from_guid( gid.unicast.interface_id, p_port->p_adapter->params.guid_mask, &mac );\r
2438                 if (status == IB_INVALID_GUID_MASK)\r
2439                 {\r
2440                         IPOIB_PRINT( TRACE_LEVEL_WARNING, IPOIB_DBG_ERROR,\r
2441                                 ("Invalid GUID mask received, rejecting it") );\r
2442                         ipoib_create_log(p_port->p_adapter->h_adapter, GUID_MASK_LOG_INDEX, EVENT_IPOIB_WRONG_PARAMETER_WRN);\r
2443                 }\r
2444                 else if( status != IB_SUCCESS )\r
2445                 {\r
2446                         IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2447                                 ("ipoib_mac_from_guid returned %s\n",\r
2448                                 p_port->p_adapter->p_ifc->get_err_str( status )) );\r
2449                         return status;\r
2450                 }\r
2451                 /*\r
2452                  * Create the endpoint.\r
2453                  */\r
2454                 *pp_src = ipoib_endpt_create( &p_ib_arp->src_hw.gid,\r
2455                         p_wc->recv.ud.remote_lid, (p_ib_arp->src_hw.flags_qpn & CL_HTON32(0x00FFFFFF)) );\r
2456 \r
2457                 if( !*pp_src )\r
2458                 {\r
2459                         IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2460                                 ("ipoib_endpt_create failed\n") );\r
2461                         return status;\r
2462                 }\r
2463 \r
2464                 cl_obj_lock( &p_port->obj );\r
2465                 status = __endpt_mgr_insert( p_port, mac, *pp_src );\r
2466                 if( status != IB_SUCCESS )\r
2467                 {\r
2468                         cl_obj_unlock( &p_port->obj );\r
2469                         IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2470                                 ("__endpt_mgr_insert return %s \n",\r
2471                                 p_port->p_adapter->p_ifc->get_err_str( status )) );\r
2472                         return status;\r
2473                 }\r
2474 \r
2475                 cl_obj_unlock( &p_port->obj );\r
2476         }\r
2477 \r
2478         CL_ASSERT( !cl_memcmp(\r
2479                 &(*pp_src)->dgid, &p_ib_arp->src_hw.gid, sizeof(ib_gid_t) ) );\r
2480         CL_ASSERT( ipoib_is_voltaire_router_gid( &(*pp_src)->dgid ) ||\r
2481                 (*pp_src)->qpn ==\r
2482                 (p_ib_arp->src_hw.flags_qpn & CL_HTON32(0x00FFFFFF)) );\r
2483         /* Now swizzle the data. */\r
2484         p_arp->hw_type = ARP_HW_TYPE_ETH;\r
2485         p_arp->hw_size = sizeof(mac_addr_t);\r
2486         p_arp->src_hw = (*pp_src)->mac;\r
2487         p_arp->src_ip = p_ib_arp->src_ip;\r
2488 \r
2489         if( cl_memcmp( &p_ib_arp->dst_hw, &null_hw, sizeof(ipoib_hw_addr_t) ) )\r
2490         {\r
2491                 if( cl_memcmp( &p_dst->dgid, &p_ib_arp->dst_hw.gid, sizeof(ib_gid_t) ) )\r
2492                 {\r
2493                         /*\r
2494                          * We received bcast ARP packet that means\r
2495                          * remote port lets everyone know it was changed IP/MAC\r
2496                          * or just activated\r
2497                          */\r
2498 \r
2499                         /* Guy: TODO: Check why this check fails in case of Voltaire IPR */\r
2500 \r
2501                         if ( !ipoib_is_voltaire_router_gid( &(*pp_src)->dgid ) &&\r
2502                                  !ib_gid_is_multicast( (const ib_gid_t*)&p_dst->dgid ) )\r
2503                         {\r
2504                                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2505                                         ("ARP: is not ARP MCAST\n") );\r
2506                                 return IB_INVALID_SETTING;\r
2507                         }\r
2508 \r
2509                         p_arp->dst_hw = p_port->p_local_endpt->mac;\r
2510                         p_dst->mac = p_port->p_local_endpt->mac;\r
2511                         /*\r
2512                          * we don't care what receiver ip addr is,\r
2513                          * as long as OS' ARP table is global  ???\r
2514                          */\r
2515                         p_arp->dst_ip = (net32_t)0;\r
2516                 }\r
2517                 else /* we've got reply to our ARP request */\r
2518                 {\r
2519                         p_arp->dst_hw = p_dst->mac;\r
2520                         p_arp->dst_ip = p_ib_arp->dst_ip;\r
2521                         CL_ASSERT( p_dst->qpn == \r
2522                                 (p_ib_arp->dst_hw.flags_qpn & CL_HTON32(0x00FFFFFF)) );\r
2523                 }\r
2524         }\r
2525         else /* we got ARP reqeust */\r
2526         {\r
2527                 cl_memclr( &p_arp->dst_hw, sizeof(mac_addr_t) );\r
2528                 p_arp->dst_ip = p_ib_arp->dst_ip;\r
2529         }\r
2530 \r
2531         /*\r
2532          * Create the ethernet header.  Note that this is done last so that\r
2533          * we have a chance to create a new endpoint.\r
2534          */\r
2535         status = __recv_gen( p_ipoib, p_eth, *pp_src, p_dst );\r
2536         if( status != IB_SUCCESS )\r
2537         {\r
2538                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2539                         ("__recv_gen returned %s.\n", \r
2540                         p_port->p_adapter->p_ifc->get_err_str( status )) );\r
2541                 return status;\r
2542         }\r
2543 \r
2544         IPOIB_EXIT( IPOIB_DBG_RECV );\r
2545         return IB_SUCCESS;\r
2546 }\r
2547 \r
2548 \r
2549 static ib_api_status_t\r
2550 __recv_mgr_prepare_pkt(\r
2551         IN                              ipoib_port_t* const                     p_port,\r
2552         IN                              ipoib_recv_desc_t* const        p_desc,\r
2553                 OUT                     NDIS_PACKET** const                     pp_packet )\r
2554 {\r
2555         NDIS_STATUS                                                     status;\r
2556         uint32_t                                                        pkt_filter;\r
2557         ip_stat_sel_t                                           type;\r
2558         NDIS_TCP_IP_CHECKSUM_PACKET_INFO        chksum;\r
2559 \r
2560         PERF_DECLARE( GetNdisPkt );\r
2561 \r
2562         IPOIB_ENTER( IPOIB_DBG_RECV );\r
2563 \r
2564         pkt_filter = p_port->p_adapter->packet_filter;\r
2565         /* Check the packet filter. */\r
2566         switch( p_desc->type )\r
2567         {\r
2568         default:\r
2569         case PKT_TYPE_UCAST:\r
2570                 if( pkt_filter & NDIS_PACKET_TYPE_PROMISCUOUS ||\r
2571                         pkt_filter & NDIS_PACKET_TYPE_ALL_FUNCTIONAL ||\r
2572                         pkt_filter & NDIS_PACKET_TYPE_SOURCE_ROUTING ||\r
2573                         pkt_filter & NDIS_PACKET_TYPE_DIRECTED )\r
2574                 {\r
2575                         /* OK to report. */\r
2576                         type = IP_STAT_UCAST_BYTES;\r
2577                         status = NDIS_STATUS_SUCCESS;\r
2578                 }\r
2579                 else\r
2580                 {\r
2581                         type = IP_STAT_DROPPED;\r
2582                         status = NDIS_STATUS_FAILURE;\r
2583                 }\r
2584                 break;\r
2585         case PKT_TYPE_BCAST:\r
2586                 if( pkt_filter & NDIS_PACKET_TYPE_PROMISCUOUS ||\r
2587                         pkt_filter & NDIS_PACKET_TYPE_BROADCAST )\r
2588                 {\r
2589                         /* OK to report. */\r
2590                         type = IP_STAT_BCAST_BYTES;\r
2591                         status = NDIS_STATUS_SUCCESS;\r
2592                 }\r
2593                 else\r
2594                 {\r
2595                         type = IP_STAT_DROPPED;\r
2596                         status = NDIS_STATUS_FAILURE;\r
2597                 }\r
2598                 break;\r
2599         case PKT_TYPE_MCAST:\r
2600                 if( pkt_filter & NDIS_PACKET_TYPE_PROMISCUOUS ||\r
2601                         pkt_filter & NDIS_PACKET_TYPE_ALL_MULTICAST ||\r
2602                         pkt_filter & NDIS_PACKET_TYPE_MULTICAST )\r
2603                 {\r
2604                         /* OK to report. */\r
2605                         type = IP_STAT_MCAST_BYTES;\r
2606                         status = NDIS_STATUS_SUCCESS;\r
2607                 }\r
2608                 else\r
2609                 {\r
2610                         type = IP_STAT_DROPPED;\r
2611                         status = NDIS_STATUS_FAILURE;\r
2612                 }\r
2613                 break;\r
2614         }\r
2615 \r
2616         if( status != NDIS_STATUS_SUCCESS )\r
2617         {\r
2618                 ipoib_inc_recv_stat( p_port->p_adapter, type, 0 );\r
2619                 /* Return the receive descriptor to the pool. */\r
2620                 __buf_mgr_put_recv( p_port, p_desc, NULL );\r
2621                 IPOIB_PRINT_EXIT( TRACE_LEVEL_INFORMATION, IPOIB_DBG_RECV,\r
2622                         ("Packet filter doesn't match receive.  Dropping.\n") );\r
2623                 /*\r
2624                  * Return IB_NOT_DONE since the packet has been completed,\r
2625                  * but has not consumed an array entry.\r
2626                  */\r
2627                 return IB_NOT_DONE;\r
2628         }\r
2629 \r
2630         cl_perf_start( GetNdisPkt );\r
2631         *pp_packet = __buf_mgr_get_ndis_pkt( p_port, p_desc );\r
2632         cl_perf_stop( &p_port->p_adapter->perf, GetNdisPkt );\r
2633         if( !*pp_packet )\r
2634         {\r
2635                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2636                         ("__buf_mgr_get_ndis_pkt failed\n") );\r
2637                 return IB_INSUFFICIENT_RESOURCES;\r
2638         }\r
2639 \r
2640         chksum.Value = 0;\r
2641         switch (p_port->p_adapter->params.recv_chksum_offload) {\r
2642           case CSUM_DISABLED:\r
2643                 NDIS_PER_PACKET_INFO_FROM_PACKET( *pp_packet, TcpIpChecksumPacketInfo ) =\r
2644                 (void*)(uintn_t)chksum.Value;\r
2645                 break;\r
2646           case CSUM_ENABLED:\r
2647                 /* Get the checksums directly from packet information. */\r
2648                 /* In this case, no one of cheksum's cat get false value */\r
2649                 /* If hardware checksum failed or wasn't calculated, NDIS will recalculate it again */\r
2650                 NDIS_PER_PACKET_INFO_FROM_PACKET( *pp_packet, TcpIpChecksumPacketInfo ) = \r
2651                         (void*)(uintn_t)(p_desc->ndis_csum.Value);\r
2652                 break;\r
2653           case CSUM_BYPASS:\r
2654                 /* Flag the checksums as having been calculated. */\r
2655                 chksum.Receive.NdisPacketTcpChecksumSucceeded = TRUE;\r
2656                 chksum.Receive.NdisPacketUdpChecksumSucceeded = TRUE;\r
2657                 chksum.Receive.NdisPacketIpChecksumSucceeded = TRUE;\r
2658                 NDIS_PER_PACKET_INFO_FROM_PACKET( *pp_packet, TcpIpChecksumPacketInfo ) =\r
2659                 (void*)(uintn_t)chksum.Value;\r
2660                 break;\r
2661           default:\r
2662                 ASSERT(FALSE);\r
2663                 NDIS_PER_PACKET_INFO_FROM_PACKET( *pp_packet, TcpIpChecksumPacketInfo ) =\r
2664                 (void*)(uintn_t)chksum.Value;\r
2665         }\r
2666         ipoib_inc_recv_stat( p_port->p_adapter, type, p_desc->len );\r
2667 \r
2668         IPOIB_EXIT( IPOIB_DBG_RECV );\r
2669         return IB_SUCCESS;\r
2670 }\r
2671 \r
2672 \r
2673 static uint32_t\r
2674 __recv_mgr_build_pkt_array(\r
2675         IN                              ipoib_port_t* const                     p_port,\r
2676         IN                              int32_t                                         shortage,\r
2677                 OUT                     cl_qlist_t* const                       p_done_list,\r
2678                 OUT                     int32_t* const                          p_discarded )\r
2679 {\r
2680         cl_list_item_t                  *p_item;\r
2681         ipoib_recv_desc_t               *p_desc;\r
2682         uint32_t                                i = 0;\r
2683         ib_api_status_t                 status;\r
2684         PERF_DECLARE( PreparePkt );\r
2685 \r
2686         IPOIB_ENTER( IPOIB_DBG_RECV );\r
2687 \r
2688         *p_discarded = 0;\r
2689 \r
2690         /* Move any existing receives to the head to preserve ordering. */\r
2691         cl_qlist_insert_list_head( p_done_list, &p_port->recv_mgr.done_list );\r
2692         p_item = cl_qlist_remove_head( p_done_list );\r
2693         while( p_item != cl_qlist_end( p_done_list ) )\r
2694         {\r
2695                 p_desc = (ipoib_recv_desc_t*)p_item;\r
2696 \r
2697                 cl_perf_start( PreparePkt );\r
2698                 status = __recv_mgr_prepare_pkt( p_port, p_desc,\r
2699                         &p_port->recv_mgr.recv_pkt_array[i] );\r
2700                 cl_perf_stop( &p_port->p_adapter->perf, PreparePkt );\r
2701                 if( status == IB_SUCCESS )\r
2702                 {\r
2703                         CL_ASSERT( p_port->recv_mgr.recv_pkt_array[i] );\r
2704                         if( shortage-- > 0 )\r
2705                         {\r
2706                                 NDIS_SET_PACKET_STATUS(\r
2707                                         p_port->recv_mgr.recv_pkt_array[i], NDIS_STATUS_RESOURCES );\r
2708                         }\r
2709                         else\r
2710                         {\r
2711                                 NDIS_SET_PACKET_STATUS(\r
2712                                         p_port->recv_mgr.recv_pkt_array[i], NDIS_STATUS_SUCCESS );\r
2713                         }\r
2714                         i++;\r
2715                 }\r
2716                 else if( status == IB_NOT_DONE )\r
2717                 {\r
2718                         (*p_discarded)++;\r
2719                 }\r
2720                 else\r
2721                 {\r
2722                         IPOIB_PRINT(TRACE_LEVEL_INFORMATION, IPOIB_DBG_RECV,\r
2723                                 ("__recv_mgr_prepare_pkt returned %s\n",\r
2724                                 p_port->p_adapter->p_ifc->get_err_str( status )) );\r
2725                         /* Put all completed receives on the port's done list. */\r
2726                         cl_qlist_insert_tail( &p_port->recv_mgr.done_list, p_item );\r
2727                         cl_qlist_insert_list_tail( &p_port->recv_mgr.done_list, p_done_list );\r
2728                         break;\r
2729                 }\r
2730 \r
2731                 p_item = cl_qlist_remove_head( p_done_list );\r
2732         }\r
2733 \r
2734         IPOIB_EXIT( IPOIB_DBG_RECV );\r
2735         return i;\r
2736 }\r
2737 \r
2738 \r
2739 \r
2740 \r
2741 /******************************************************************************\r
2742 *\r
2743 * Send manager implementation.\r
2744 *\r
2745 ******************************************************************************/\r
2746 static void\r
2747 __send_mgr_construct(\r
2748         IN                              ipoib_port_t* const                     p_port )\r
2749 {\r
2750         IPOIB_ENTER( IPOIB_DBG_SEND );\r
2751         p_port->send_mgr.depth = 0;\r
2752         cl_qlist_init( &p_port->send_mgr.pending_list );\r
2753         IPOIB_EXIT( IPOIB_DBG_SEND );\r
2754 }\r
2755 \r
2756 \r
2757 static void \r
2758 __pending_list_destroy(\r
2759         IN                              ipoib_port_t* const                     p_port )\r
2760 {\r
2761         cl_list_item_t  *p_item;\r
2762         NDIS_PACKET             *p_packet;\r
2763         \r
2764         cl_spinlock_acquire( &p_port->send_lock );\r
2765         /* Complete any pending packets. */\r
2766         for( p_item = cl_qlist_remove_head( &p_port->send_mgr.pending_list );\r
2767                 p_item != cl_qlist_end( &p_port->send_mgr.pending_list );\r
2768                 p_item = cl_qlist_remove_head( &p_port->send_mgr.pending_list ) )\r
2769         {\r
2770                 p_packet = IPOIB_PACKET_FROM_LIST_ITEM( p_item );\r
2771                 NdisMSendCompleteX( p_port->p_adapter->h_adapter, p_packet,\r
2772                         NDIS_STATUS_RESET_IN_PROGRESS );\r
2773         }\r
2774         cl_spinlock_release( &p_port->send_lock );\r
2775 }\r
2776 \r
2777 static void\r
2778 __send_mgr_destroy(\r
2779         IN                              ipoib_port_t* const                     p_port )\r
2780 {\r
2781         IPOIB_ENTER( IPOIB_DBG_SEND );\r
2782         __pending_list_destroy(p_port);\r
2783 \r
2784         IPOIB_EXIT( IPOIB_DBG_SEND );\r
2785 }\r
2786 \r
2787 \r
2788 static NDIS_STATUS\r
2789 __send_mgr_filter(\r
2790         IN                              ipoib_port_t* const                     p_port,\r
2791         IN              const   eth_hdr_t* const                        p_eth_hdr,\r
2792         IN                              NDIS_BUFFER* const                      p_buf,\r
2793         IN                              size_t                                          buf_len,\r
2794         IN      OUT                     ipoib_send_desc_t* const        p_desc )\r
2795 {\r
2796         NDIS_STATUS             status;\r
2797 \r
2798         PERF_DECLARE( FilterIp );\r
2799         PERF_DECLARE( FilterArp );\r
2800         PERF_DECLARE( SendGen );\r
2801 \r
2802         IPOIB_ENTER( IPOIB_DBG_SEND );\r
2803 \r
2804         /*\r
2805          * We already checked the ethernet header length, so we know it's safe\r
2806          * to decrement the buf_len without underflowing.\r
2807          */\r
2808         buf_len -= sizeof(eth_hdr_t);\r
2809 \r
2810         switch( p_eth_hdr->type )\r
2811         {\r
2812         case ETH_PROT_TYPE_IP:\r
2813                 cl_perf_start( FilterIp );\r
2814                 status = __send_mgr_filter_ip(\r
2815                         p_port, p_eth_hdr, p_buf, buf_len, p_desc );\r
2816                 cl_perf_stop( &p_port->p_adapter->perf, FilterIp );\r
2817                 break;\r
2818 \r
2819         case ETH_PROT_TYPE_ARP:\r
2820                 cl_perf_start( FilterArp );\r
2821                 status = __send_mgr_filter_arp(\r
2822                         p_port, p_eth_hdr, p_buf, buf_len, p_desc );\r
2823                 cl_perf_stop( &p_port->p_adapter->perf, FilterArp );\r
2824                 break;\r
2825 \r
2826         default:\r
2827                 /*\r
2828                  * The IPoIB spec doesn't define how to send non IP or ARP packets.\r
2829                  * Just send the payload and hope for the best.\r
2830                  */\r
2831                 cl_perf_start( SendGen );\r
2832                 status = __send_gen( p_port, p_desc, 0 );\r
2833                 cl_perf_stop( &p_port->p_adapter->perf, SendGen );\r
2834                 break;\r
2835         }\r
2836 \r
2837         IPOIB_EXIT( IPOIB_DBG_SEND );\r
2838         return status;\r
2839 }\r
2840 \r
2841 \r
2842 static NDIS_STATUS\r
2843 __send_copy(\r
2844         IN                              ipoib_port_t* const                     p_port,\r
2845         IN                              ipoib_send_desc_t* const        p_desc )\r
2846 {\r
2847         NDIS_PACKET                             *p_packet;\r
2848         NDIS_BUFFER                             *p_buf;\r
2849         NDIS_STATUS                             status;\r
2850         UINT                                    tot_len, bytes_copied = 0;\r
2851 \r
2852         IPOIB_ENTER( IPOIB_DBG_SEND );\r
2853 \r
2854         p_desc->p_buf = \r
2855                 ExAllocateFromNPagedLookasideList( &p_port->buf_mgr.send_buf_list );\r
2856         if( !p_desc->p_buf )\r
2857         {\r
2858                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2859                         ("Failed to allocate buffer for packet copy.\n") );\r
2860                 return NDIS_STATUS_RESOURCES;\r
2861         }\r
2862 \r
2863         NdisAllocatePacket( &status, &p_packet, p_port->buf_mgr.h_send_pkt_pool );\r
2864         if( status != NDIS_STATUS_SUCCESS )\r
2865         {\r
2866                 IPOIB_PRINT_EXIT( TRACE_LEVEL_WARNING, IPOIB_DBG_SEND,\r
2867                         ("Failed to allocate NDIS_PACKET for copy.\n") );\r
2868                 return status;\r
2869         }\r
2870 \r
2871         NdisAllocateBuffer( &status, &p_buf, p_port->buf_mgr.h_send_buf_pool,\r
2872                 p_desc->p_buf, p_port->p_adapter->params.xfer_block_size );\r
2873         if( status != NDIS_STATUS_SUCCESS )\r
2874         {\r
2875                 NdisFreePacket( p_packet );\r
2876                 IPOIB_PRINT_EXIT( TRACE_LEVEL_WARNING, IPOIB_DBG_SEND,\r
2877                         ("Failed to allocate NDIS_BUFFER for copy.\n") );\r
2878                 return status;\r
2879         }\r
2880 \r
2881         NdisChainBufferAtFront( p_packet, p_buf );\r
2882 \r
2883         NdisQueryPacketLength( p_desc->p_pkt, &tot_len );\r
2884 \r
2885         /* Setup the work request. */\r
2886         p_desc->local_ds[1].vaddr = cl_get_physaddr(\r
2887                 ((uint8_t*)p_desc->p_buf) + sizeof(eth_hdr_t) );\r
2888         p_desc->local_ds[1].length = tot_len - sizeof(eth_hdr_t);\r
2889         p_desc->local_ds[1].lkey = p_port->ib_mgr.lkey;\r
2890         p_desc->wr.num_ds = 2;\r
2891 \r
2892         /* Copy the packet. */\r
2893         NdisCopyFromPacketToPacketSafe( p_packet, bytes_copied, tot_len,\r
2894                 p_desc->p_pkt, bytes_copied, &bytes_copied,\r
2895                 NormalPagePriority );\r
2896 \r
2897         /* Free our temp packet now that the data is copied. */\r
2898         NdisUnchainBufferAtFront( p_packet, &p_buf );\r
2899         NdisFreeBuffer( p_buf );\r
2900         NdisFreePacket( p_packet );\r
2901 \r
2902         if( bytes_copied != tot_len )\r
2903         {\r
2904                 /* Something went wrong.  Drop the packet. */\r
2905                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2906                         ("Failed to copy full packet: %d of %d bytes copied.\n",\r
2907                         bytes_copied, tot_len) );\r
2908                 return NDIS_STATUS_RESOURCES;\r
2909         }\r
2910 \r
2911         IPOIB_EXIT( IPOIB_DBG_SEND );\r
2912         return NDIS_STATUS_SUCCESS;\r
2913 }\r
2914 \r
2915 \r
2916 #if !IPOIB_USE_DMA\r
2917 /* Send using the MDL's page information rather than the SGL. */\r
2918 static ib_api_status_t\r
2919 __send_gen(\r
2920         IN                              ipoib_port_t* const                     p_port,\r
2921         IN                              ipoib_send_desc_t* const        p_desc )\r
2922 {\r
2923         uint32_t                                i, j = 1;\r
2924         ULONG                                   offset;\r
2925         MDL                                             *p_mdl;\r
2926         UINT                                    num_pages, tot_len;\r
2927         ULONG                                   buf_len;\r
2928         PPFN_NUMBER                             page_array;\r
2929         boolean_t                               hdr_done = FALSE;\r
2930         ib_api_status_t                 status;\r
2931 \r
2932         IPOIB_ENTER( IPOIB_DBG_SEND );\r
2933 \r
2934         NdisQueryPacket( p_desc->p_pkt, &num_pages, NULL, &p_mdl,\r
2935                 &tot_len );\r
2936 \r
2937         if( !p_mdl )\r
2938         {\r
2939                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2940                         ("No buffers associated with packet.\n") );\r
2941                 return IB_ERROR;\r
2942         }\r
2943 \r
2944         /* Remember that one of the DS entries is reserved for the IPoIB header. */\r
2945         if( num_pages >= MAX_SEND_SGE )\r
2946         {\r
2947                 IPOIB_PRINT(TRACE_LEVEL_INFORMATION, IPOIB_DBG_SEND,\r
2948                         ("Too many buffers to fit in WR ds_array.  Copying data.\n") );\r
2949                 status = __send_copy( p_port, p_desc );\r
2950                 IPOIB_EXIT( IPOIB_DBG_SEND );\r
2951                 return status;\r
2952         }\r
2953 \r
2954         CL_ASSERT( tot_len > sizeof(eth_hdr_t) );\r
2955         CL_ASSERT( tot_len <= p_port->p_adapter->params.xfer_block_size );\r
2956         /*\r
2957          * Assume that the ethernet header is always fully contained\r
2958          * in the first page of the first MDL.  This makes for much\r
2959          * simpler code.\r
2960          */\r
2961         offset = MmGetMdlByteOffset( p_mdl ) + sizeof(eth_hdr_t);\r
2962         CL_ASSERT( offset <= PAGE_SIZE );\r
2963 \r
2964         while( tot_len )\r
2965         {\r
2966                 buf_len = MmGetMdlByteCount( p_mdl );\r
2967                 page_array = MmGetMdlPfnArray( p_mdl );\r
2968                 CL_ASSERT( page_array );\r
2969                 i = 0;\r
2970                 if( !hdr_done )\r
2971                 {\r
2972                         CL_ASSERT( buf_len >= sizeof(eth_hdr_t) );\r
2973                         /* Skip the ethernet header. */\r
2974                         buf_len -= sizeof(eth_hdr_t);\r
2975                         CL_ASSERT( buf_len <= p_port->p_adapter->params.payload_mtu );\r
2976                         if( buf_len )\r
2977                         {\r
2978                                 /* The ethernet header is a subset of this MDL. */\r
2979                                 CL_ASSERT( i == 0 );\r
2980                                 if( offset < PAGE_SIZE )\r
2981                                 {\r
2982                                         p_desc->local_ds[j].lkey = p_port->ib_mgr.lkey;\r
2983                                         p_desc->local_ds[j].vaddr = (page_array[i] << PAGE_SHIFT);\r
2984                                         /* Add the byte offset since we're on the 1st page. */\r
2985                                         p_desc->local_ds[j].vaddr += offset;\r
2986                                         if( offset + buf_len > PAGE_SIZE )\r
2987                                         {\r
2988                                                 p_desc->local_ds[j].length = PAGE_SIZE - offset;\r
2989                                                 buf_len -= p_desc->local_ds[j].length;\r
2990                                         }\r
2991                                         else\r
2992                                         {\r
2993                                                 p_desc->local_ds[j].length = buf_len;\r
2994                                                 buf_len = 0;\r
2995                                         }\r
2996                                         /* This data segment is done.  Move to the next. */\r
2997                                         j++;\r
2998                                 }\r
2999                                 /* This page is done.  Move to the next. */\r
3000                                 i++;\r
3001                         }\r
3002                         /* Done handling the ethernet header. */\r
3003                         hdr_done = TRUE;\r
3004                 }\r
3005 \r
3006                 /* Finish this MDL */\r
3007   &n