1e0f94698496f0d8b8236f54f47c3f9f8813e16c
[mirror/winof/.git] / ulp / ipoib / kernel / ipoib_port.c
1 /*\r
2  * Copyright (c) 2005 SilverStorm Technologies.  All rights reserved.\r
3  * Copyright (c) 2006 Mellanox Technologies.  All rights reserved.\r
4  * Portions Copyright (c) 2008 Microsoft Corporation.  All rights reserved.\r
5  *\r
6  * This software is available to you under the OpenIB.org BSD license\r
7  * below:\r
8  *\r
9  *     Redistribution and use in source and binary forms, with or\r
10  *     without modification, are permitted provided that the following\r
11  *     conditions are met:\r
12  *\r
13  *      - Redistributions of source code must retain the above\r
14  *        copyright notice, this list of conditions and the following\r
15  *        disclaimer.\r
16  *\r
17  *      - Redistributions in binary form must reproduce the above\r
18  *        copyright notice, this list of conditions and the following\r
19  *        disclaimer in the documentation and/or other materials\r
20  *        provided with the distribution.\r
21  *\r
22  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,\r
23  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF\r
24  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND\r
25  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS\r
26  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN\r
27  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN\r
28  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\r
29  * SOFTWARE.\r
30  *\r
31  * $Id$\r
32  */\r
33 \r
34 \r
35 \r
36 #include "ipoib_port.h"\r
37 #include "ipoib_adapter.h"\r
38 #include "ipoib_debug.h"\r
39 #if defined(EVENT_TRACING)\r
40 #ifdef offsetof\r
41 #undef offsetof\r
42 #endif\r
43 #include "ipoib_port.tmh"\r
44 #endif\r
45 \r
46 \r
47 /* Amount of physical memory to register. */\r
48 #define MEM_REG_SIZE    0xFFFFFFFFFFFFFFFF\r
49 \r
50 /* Number of work completions to chain for send and receive polling. */\r
51 #define MAX_SEND_WC             8\r
52 #define MAX_RECV_WC             16\r
53 \r
54 \r
55 ib_gid_t        bcast_mgid_template = {\r
56         0xff,                                                           /* multicast field */\r
57         0x12,                                                           /* scope (to be filled in) */\r
58         0x40, 0x1b,                                                     /* IPv4 signature */\r
59         0xff, 0xff,                                                     /* 16 bits of P_Key (to be filled in) */\r
60         0x00, 0x00, 0x00, 0x00, 0x00, 0x00,     /* 48 bits of zeros */\r
61         0xff, 0xff, 0xff, 0xff,                         /* 32 bit IPv4 broadcast address */\r
62 };\r
63 \r
64 \r
65 #ifdef _DEBUG_\r
66 /* Handy pointer for debug use. */\r
67 ipoib_port_t    *gp_ipoib_port;\r
68 #endif\r
69 \r
70 \r
71 /******************************************************************************\r
72 *\r
73 * Declarations\r
74 *\r
75 ******************************************************************************/\r
76 static void\r
77 __port_construct(\r
78         IN                              ipoib_port_t* const                     p_port );\r
79 \r
80 static ib_api_status_t\r
81 __port_init(\r
82         IN                              ipoib_port_t* const                     p_port,\r
83         IN                              ipoib_adapter_t* const          p_adapter,\r
84         IN                              ib_pnp_port_rec_t* const        p_pnp_rec );\r
85 \r
86 static void\r
87 __port_destroying(\r
88         IN                              cl_obj_t* const                         p_obj );\r
89 \r
90 static void\r
91 __port_cleanup(\r
92         IN                              cl_obj_t* const                         p_obj );\r
93 \r
94 static void\r
95 __port_free(\r
96         IN                              cl_obj_t* const                         p_obj );\r
97 \r
98 \r
99 /******************************************************************************\r
100 *\r
101 * IB resource manager operations\r
102 *\r
103 ******************************************************************************/\r
104 static void\r
105 __ib_mgr_construct(\r
106         IN                              ipoib_port_t* const                     p_port );\r
107 \r
108 static ib_api_status_t\r
109 __ib_mgr_init(\r
110         IN                              ipoib_port_t* const                     p_port );\r
111 \r
112 static void\r
113 __ib_mgr_destroy(\r
114         IN                              ipoib_port_t* const                     p_port );\r
115 \r
116 static void\r
117 __qp_event(\r
118         IN                              ib_async_event_rec_t            *p_event_rec );\r
119 \r
120 static void\r
121 __cq_event(\r
122         IN                              ib_async_event_rec_t            *p_event_rec );\r
123 \r
124 static ib_api_status_t\r
125 __ib_mgr_activate(\r
126         IN                              ipoib_port_t* const                     p_port );\r
127 \r
128 /******************************************************************************\r
129 *\r
130 * Buffer manager operations.\r
131 *\r
132 ******************************************************************************/\r
133 static void\r
134 __buf_mgr_construct(\r
135         IN                              ipoib_port_t* const                     p_port );\r
136 \r
137 static ib_api_status_t\r
138 __buf_mgr_init(\r
139         IN                              ipoib_port_t* const                     p_port );\r
140 \r
141 static void\r
142 __buf_mgr_destroy(\r
143         IN                              ipoib_port_t* const                     p_port );\r
144 \r
145 static cl_status_t\r
146 __recv_ctor(\r
147         IN                              void* const                                     p_object,\r
148         IN                              void*                                           context,\r
149                 OUT                     cl_pool_item_t** const          pp_pool_item );\r
150 \r
151 #if !IPOIB_INLINE_RECV\r
152 static void\r
153 __recv_dtor(\r
154         IN              const   cl_pool_item_t* const           p_pool_item,\r
155         IN                              void                                            *context );\r
156 #endif  /* IPOIB_INLINE_RECV */\r
157 \r
158 static inline ipoib_send_desc_t*\r
159 __buf_mgr_get_send(\r
160         IN                              ipoib_port_t* const                     p_port );\r
161 \r
162 static inline void\r
163 __buf_mgr_put_send(\r
164         IN                              ipoib_port_t* const                     p_port,\r
165         IN                              ipoib_send_desc_t* const        p_desc );\r
166 \r
167 static inline ipoib_recv_desc_t*\r
168 __buf_mgr_get_recv(\r
169         IN                              ipoib_port_t* const                     p_port );\r
170 \r
171 static inline void\r
172 __buf_mgr_put_recv(\r
173         IN                              ipoib_port_t* const                     p_port,\r
174         IN                              ipoib_recv_desc_t* const        p_desc,\r
175         IN                              NDIS_PACKET* const                      p_packet OPTIONAL );\r
176 \r
177 static inline void\r
178 __buf_mgr_put_recv_list(\r
179         IN                              ipoib_port_t* const                     p_port,\r
180         IN                              cl_qlist_t* const                       p_list );\r
181 \r
182 static inline NDIS_PACKET*\r
183 __buf_mgr_get_ndis_pkt(\r
184         IN                              ipoib_port_t* const                     p_port,\r
185         IN                              ipoib_recv_desc_t* const        p_desc );\r
186 \r
187 \r
188 /******************************************************************************\r
189 *\r
190 * Receive manager operations.\r
191 *\r
192 ******************************************************************************/\r
193 static void\r
194 __recv_mgr_construct(\r
195         IN                              ipoib_port_t* const                     p_port );\r
196 \r
197 static ib_api_status_t\r
198 __recv_mgr_init(\r
199         IN                              ipoib_port_t* const                     p_port );\r
200 \r
201 static void\r
202 __recv_mgr_destroy(\r
203         IN                              ipoib_port_t* const                     p_port );\r
204 \r
205 /* Posts receive buffers to the receive queue. */\r
206 static ib_api_status_t\r
207 __recv_mgr_repost(\r
208         IN                              ipoib_port_t* const                     p_port );\r
209 \r
210 static void\r
211 __recv_cb(\r
212         IN              const   ib_cq_handle_t                          h_cq,\r
213         IN                              void                                            *cq_context );\r
214 \r
215 static void\r
216 __recv_get_endpts(\r
217         IN                              ipoib_port_t* const                     p_port,\r
218         IN                              ipoib_recv_desc_t* const        p_desc,\r
219         IN                              ib_wc_t* const                          p_wc,\r
220                 OUT                     ipoib_endpt_t** const           pp_src,\r
221                 OUT                     ipoib_endpt_t** const           pp_dst );\r
222 \r
223 static int32_t\r
224 __recv_mgr_filter(\r
225         IN                              ipoib_port_t* const                     p_port,\r
226         IN                              ib_wc_t* const                          p_done_wc_list,\r
227                 OUT                     cl_qlist_t* const                       p_done_list,\r
228                 OUT                     cl_qlist_t* const                       p_bad_list );\r
229 \r
230 static ib_api_status_t\r
231 __recv_gen(\r
232         IN              const   ipoib_pkt_t* const                      p_ipoib,\r
233                 OUT                     eth_pkt_t* const                        p_eth,\r
234         IN                              ipoib_endpt_t* const            p_src,\r
235         IN                              ipoib_endpt_t* const            p_dst );\r
236 \r
237 static ib_api_status_t\r
238 __recv_dhcp(\r
239         IN                              ipoib_port_t* const                     p_port,\r
240         IN              const   ipoib_pkt_t* const                      p_ipoib,\r
241                 OUT                     eth_pkt_t* const                        p_eth,\r
242         IN                              ipoib_endpt_t* const            p_src,\r
243         IN                              ipoib_endpt_t* const            p_dst );\r
244 \r
245 static ib_api_status_t\r
246 __recv_arp(\r
247         IN                              ipoib_port_t* const                     p_port,\r
248         IN                              ib_wc_t* const                          p_wc,\r
249         IN              const   ipoib_pkt_t* const                      p_ipoib,\r
250                 OUT                     eth_pkt_t* const                        p_eth,\r
251         IN                              ipoib_endpt_t** const           p_src,\r
252         IN                              ipoib_endpt_t* const            p_dst );\r
253 \r
254 static ib_api_status_t\r
255 __recv_mgr_prepare_pkt(\r
256         IN                              ipoib_port_t* const                     p_port,\r
257         IN                              ipoib_recv_desc_t* const        p_desc,\r
258                 OUT                     NDIS_PACKET** const                     pp_packet );\r
259 \r
260 static uint32_t\r
261 __recv_mgr_build_pkt_array(\r
262         IN                              ipoib_port_t* const                     p_port,\r
263         IN                              int32_t                                         shortage,\r
264                 OUT                     cl_qlist_t* const                       p_done_list,\r
265                 OUT                     int32_t* const                          p_discarded );\r
266 \r
267 /******************************************************************************\r
268 *\r
269 * Send manager operations.\r
270 *\r
271 ******************************************************************************/\r
272 static void\r
273 __send_mgr_construct(\r
274         IN                              ipoib_port_t* const                     p_port );\r
275 \r
276 static void\r
277 __send_mgr_destroy(\r
278         IN                              ipoib_port_t* const                     p_port );\r
279 \r
280 static NDIS_STATUS\r
281 __send_gen(\r
282         IN                              ipoib_port_t* const                     p_port,\r
283         IN                              ipoib_send_desc_t* const        p_desc );\r
284 \r
285 static NDIS_STATUS\r
286 __send_mgr_filter_ip(\r
287         IN                              ipoib_port_t* const                     p_port,\r
288         IN              const   eth_hdr_t* const                        p_eth_hdr,\r
289         IN                              NDIS_BUFFER*                            p_buf,\r
290         IN                              size_t                                          buf_len,\r
291         IN      OUT                     ipoib_send_desc_t* const        p_desc );\r
292 \r
293 static NDIS_STATUS\r
294 __send_mgr_filter_udp(\r
295         IN                              ipoib_port_t* const                     p_port,\r
296         IN              const   ip_hdr_t* const                         p_ip_hdr,\r
297         IN                              NDIS_BUFFER*                            p_buf,\r
298         IN                              size_t                                          buf_len,\r
299         IN      OUT                     ipoib_send_desc_t* const        p_desc );\r
300 \r
301 static NDIS_STATUS\r
302 __send_mgr_filter_dhcp(\r
303         IN                              ipoib_port_t* const                     p_port,\r
304         IN              const   udp_hdr_t* const                        p_udp_hdr,\r
305         IN                              NDIS_BUFFER*                            p_buf,\r
306         IN                              size_t                                          buf_len,\r
307         IN      OUT                     ipoib_send_desc_t* const        p_desc );\r
308 \r
309 static NDIS_STATUS\r
310 __send_mgr_filter_arp(\r
311         IN                              ipoib_port_t* const                     p_port,\r
312         IN              const   eth_hdr_t* const                        p_eth_hdr,\r
313         IN                              NDIS_BUFFER*                            p_buf,\r
314         IN                              size_t                                          buf_len,\r
315         IN      OUT                     ipoib_send_desc_t* const        p_desc );\r
316 \r
317 static void\r
318 __process_failed_send(\r
319         IN                              ipoib_port_t* const                     p_port,\r
320         IN                              ipoib_send_desc_t* const        p_desc,\r
321         IN              const   NDIS_STATUS                                     status );\r
322 \r
323 static void\r
324 __send_cb(\r
325         IN              const   ib_cq_handle_t                          h_cq,\r
326         IN                              void                                            *cq_context );\r
327 \r
328 \r
329 /******************************************************************************\r
330 *\r
331 * Endpoint manager operations\r
332 *\r
333 ******************************************************************************/\r
334 static void\r
335 __endpt_mgr_construct(\r
336         IN                              ipoib_port_t* const                     p_port );\r
337 \r
338 static ib_api_status_t\r
339 __endpt_mgr_init(\r
340         IN                              ipoib_port_t* const                     p_port );\r
341 \r
342 static void\r
343 __endpt_mgr_destroy(\r
344         IN                              ipoib_port_t* const                     p_port );\r
345 \r
346 /****f* IPoIB/__endpt_mgr_remove_all\r
347 * NAME\r
348 *       __endpt_mgr_remove_all\r
349 *\r
350 * DESCRIPTION\r
351 *       Removes all enpoints from the port, dereferencing them to initiate\r
352 *       destruction.\r
353 *\r
354 * SYNOPSIS\r
355 */\r
356 static void\r
357 __endpt_mgr_remove_all(\r
358         IN                              ipoib_port_t* const                     p_port );\r
359 /*\r
360 ********/\r
361 \r
362 static void\r
363 __endpt_mgr_remove(\r
364         IN                              ipoib_port_t* const                     p_port,\r
365         IN                              ipoib_endpt_t* const            p_endpt );\r
366 \r
367 static void\r
368 __endpt_mgr_reset_all(\r
369         IN                              ipoib_port_t* const                     p_port );\r
370 \r
371 static inline NDIS_STATUS\r
372 __endpt_mgr_ref(\r
373         IN                              ipoib_port_t* const                     p_port,\r
374         IN              const   mac_addr_t                                      mac,\r
375                 OUT                     ipoib_endpt_t** const           pp_endpt );\r
376 \r
377 static inline NDIS_STATUS\r
378 __endpt_mgr_get_gid_qpn(\r
379         IN                              ipoib_port_t* const                     p_port,\r
380         IN              const   mac_addr_t                                      mac,\r
381                 OUT                     ib_gid_t* const                         p_gid,\r
382                 OUT                     UNALIGNED net32_t* const        p_qpn );\r
383 \r
384 static inline ipoib_endpt_t*\r
385 __endpt_mgr_get_by_gid(\r
386         IN                              ipoib_port_t* const                     p_port,\r
387         IN              const   ib_gid_t* const                         p_gid );\r
388 \r
389 static inline ipoib_endpt_t*\r
390 __endpt_mgr_get_by_lid(\r
391         IN                              ipoib_port_t* const                     p_port,\r
392         IN              const   net16_t                                         lid );\r
393 \r
394 static inline ib_api_status_t\r
395 __endpt_mgr_insert_locked(\r
396         IN                              ipoib_port_t* const                     p_port,\r
397         IN              const   mac_addr_t                                      mac,\r
398         IN                              ipoib_endpt_t* const            p_endpt );\r
399 \r
400 static inline ib_api_status_t\r
401 __endpt_mgr_insert(\r
402         IN                              ipoib_port_t* const                     p_port,\r
403         IN              const   mac_addr_t                                      mac,\r
404         IN                              ipoib_endpt_t* const            p_endpt );\r
405 \r
406 static ib_api_status_t\r
407 __endpt_mgr_add_local(\r
408         IN                              ipoib_port_t* const                     p_port,\r
409         IN                              ib_port_info_t* const           p_port_info );\r
410 \r
411 static ib_api_status_t\r
412 __endpt_mgr_add_bcast(\r
413         IN                              ipoib_port_t* const                     p_port,\r
414         IN                              ib_mcast_rec_t                          *p_mcast_rec );\r
415 \r
416 /******************************************************************************\r
417 *\r
418 * MCast operations.\r
419 *\r
420 ******************************************************************************/\r
421 static ib_api_status_t\r
422 __port_get_bcast(\r
423         IN                              ipoib_port_t* const                     p_port );\r
424 \r
425 static ib_api_status_t\r
426 __port_join_bcast(\r
427         IN                              ipoib_port_t* const                     p_port,\r
428         IN                              ib_member_rec_t* const          p_member_rec );\r
429 \r
430 static ib_api_status_t\r
431 __port_create_bcast(\r
432         IN                              ipoib_port_t* const                     p_port );\r
433 \r
434 static void\r
435 __port_info_cb(\r
436         IN                              ib_query_rec_t                          *p_query_rec );\r
437 \r
438 \r
439 static void\r
440 __bcast_get_cb(\r
441         IN                              ib_query_rec_t                          *p_query_rec );\r
442 \r
443 \r
444 static void\r
445 __bcast_cb(\r
446         IN                              ib_mcast_rec_t                          *p_mcast_rec );\r
447 \r
448 \r
449 static void\r
450 __mcast_cb(\r
451         IN                              ib_mcast_rec_t                          *p_mcast_rec );\r
452 \r
453 void\r
454 __leave_error_mcast_cb(\r
455         IN                              void                            *context );\r
456 \r
457 \r
458 static intn_t\r
459 __gid_cmp(\r
460         IN              const   void* const                                     p_key1,\r
461         IN              const   void* const                                     p_key2 )\r
462 {\r
463         return cl_memcmp( p_key1, p_key2, sizeof(ib_gid_t) );\r
464 }\r
465 \r
466 \r
467 inline void ipoib_port_ref( ipoib_port_t * p_port, int type )\r
468 {\r
469         cl_obj_ref( &p_port->obj );\r
470 #if DBG\r
471         cl_atomic_inc( &p_port->ref[type % ref_mask] );\r
472         IPOIB_PRINT( TRACE_LEVEL_INFORMATION, IPOIB_DBG_OBJ,\r
473                 ("ref type %d ref_cnt %d\n", type, p_port->obj.ref_cnt) );\r
474 #else\r
475         UNREFERENCED_PARAMETER(type);\r
476 #endif\r
477 }\r
478 \r
479 \r
480 inline void ipoib_port_deref(ipoib_port_t * p_port, int type)\r
481 {\r
482         cl_obj_deref( &p_port->obj );\r
483 \r
484 #if DBG\r
485         cl_atomic_dec( &p_port->ref[type % ref_mask] );\r
486         IPOIB_PRINT( TRACE_LEVEL_INFORMATION, IPOIB_DBG_OBJ,\r
487                 ("deref type %d ref_cnt %d\n", type, p_port->obj.ref_cnt) );\r
488 #else\r
489         UNREFERENCED_PARAMETER(type);\r
490 #endif\r
491 }\r
492 \r
493 \r
494 /******************************************************************************\r
495 *\r
496 * Implementation\r
497 *\r
498 ******************************************************************************/\r
499 ib_api_status_t\r
500 ipoib_create_port(\r
501         IN                              ipoib_adapter_t* const          p_adapter,\r
502         IN                              ib_pnp_port_rec_t* const        p_pnp_rec,\r
503                 OUT                     ipoib_port_t** const            pp_port )\r
504 {\r
505         ib_api_status_t         status;\r
506         ipoib_port_t            *p_port;\r
507 \r
508         IPOIB_ENTER( IPOIB_DBG_INIT );\r
509 \r
510         CL_ASSERT( !p_adapter->p_port );\r
511 \r
512         p_port = cl_zalloc( sizeof(ipoib_port_t) +\r
513                 (sizeof(ipoib_hdr_t) * (p_adapter->params.sq_depth - 1)) );\r
514         if( !p_port )\r
515         {\r
516                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
517                         ("Failed to allocate ipoib_port_t (%d bytes)\n",\r
518                         sizeof(ipoib_port_t)) );\r
519                 return IB_INSUFFICIENT_MEMORY;\r
520         }\r
521 \r
522 #ifdef _DEBUG_\r
523         gp_ipoib_port = p_port;\r
524 #endif\r
525 \r
526         __port_construct( p_port );\r
527 \r
528         status = __port_init( p_port, p_adapter, p_pnp_rec );\r
529         if( status != IB_SUCCESS )\r
530         {\r
531                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
532                         ("ipoib_port_init returned %s.\n", \r
533                         p_adapter->p_ifc->get_err_str( status )) );\r
534                 __port_cleanup( &p_port->obj );\r
535                 __port_free( &p_port->obj );\r
536                 return status;\r
537         }\r
538 \r
539         *pp_port = p_port;\r
540         IPOIB_EXIT( IPOIB_DBG_INIT );\r
541         return IB_SUCCESS;\r
542 }\r
543 \r
544 \r
545 void\r
546 ipoib_port_destroy(\r
547         IN                              ipoib_port_t* const                     p_port )\r
548 {\r
549         IPOIB_ENTER( IPOIB_DBG_INIT );\r
550 \r
551         CL_ASSERT( p_port );\r
552         CL_ASSERT( p_port->p_adapter );\r
553         CL_ASSERT( !p_port->p_adapter->p_port );\r
554 \r
555         cl_obj_destroy( &p_port->obj );\r
556 \r
557         IPOIB_EXIT( IPOIB_DBG_INIT );\r
558 }\r
559 \r
560 \r
561 static void\r
562 __port_construct(\r
563         IN                              ipoib_port_t* const                     p_port )\r
564 {\r
565         IPOIB_ENTER( IPOIB_DBG_INIT );\r
566 \r
567         p_port->state = IB_QPS_RESET;\r
568 \r
569         cl_obj_construct( &p_port->obj, IPOIB_OBJ_PORT );\r
570         cl_spinlock_construct( &p_port->send_lock );\r
571         cl_spinlock_construct( &p_port->recv_lock );\r
572         __ib_mgr_construct( p_port );\r
573         __buf_mgr_construct( p_port );\r
574 \r
575         __recv_mgr_construct( p_port );\r
576         __send_mgr_construct( p_port );\r
577 \r
578         __endpt_mgr_construct( p_port );\r
579 \r
580         KeInitializeEvent( &p_port->sa_event, NotificationEvent, TRUE );\r
581         KeInitializeEvent( &p_port->leave_mcast_event, NotificationEvent, TRUE );\r
582         \r
583         IPOIB_EXIT( IPOIB_DBG_INIT );\r
584 }\r
585 \r
586 \r
587 static ib_api_status_t\r
588 __port_init(\r
589         IN                              ipoib_port_t* const                     p_port,\r
590         IN                              ipoib_adapter_t* const          p_adapter,\r
591         IN                              ib_pnp_port_rec_t* const        p_pnp_rec )\r
592 {\r
593         cl_status_t                     cl_status;\r
594         ib_api_status_t         status;\r
595 \r
596         IPOIB_ENTER( IPOIB_DBG_INIT );\r
597 \r
598         p_port->port_num = p_pnp_rec->p_port_attr->port_num;\r
599         p_port->p_adapter = p_adapter;\r
600 \r
601         cl_status = cl_spinlock_init( &p_port->send_lock );\r
602         if( cl_status != CL_SUCCESS )\r
603         {\r
604                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
605                         ("cl_spinlock_init returned %#x\n", cl_status) );\r
606                 return IB_ERROR;\r
607         }\r
608 \r
609         cl_status = cl_spinlock_init( &p_port->recv_lock );\r
610         if( cl_status != CL_SUCCESS )\r
611         {\r
612                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
613                         ("cl_spinlock_init returned %#x\n", cl_status) );\r
614                 return IB_ERROR;\r
615         }\r
616 \r
617         /* Initialize the IB resource manager. */\r
618         status = __ib_mgr_init( p_port );\r
619         if( status != IB_SUCCESS )\r
620         {\r
621                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
622                         ("__ib_mgr_init returned %s\n", \r
623                         p_adapter->p_ifc->get_err_str( status )) );\r
624                 return status;\r
625         }\r
626 \r
627         /* Initialize the buffer manager. */\r
628         status = __buf_mgr_init( p_port );\r
629         if( status != IB_SUCCESS )\r
630         {\r
631                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
632                         ("__buf_mgr_init returned %s\n", \r
633                         p_adapter->p_ifc->get_err_str( status )) );\r
634                 return status;\r
635         }\r
636 \r
637         /* Initialize the receive manager. */\r
638         status = __recv_mgr_init( p_port );\r
639         if( status != IB_SUCCESS )\r
640         {\r
641                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
642                         ("__recv_mgr_init returned %s\n", \r
643                         p_adapter->p_ifc->get_err_str( status )) );\r
644                 return status;\r
645         }\r
646 \r
647         /* Initialize the endpoint manager. */\r
648         status = __endpt_mgr_init( p_port );\r
649         if( status != IB_SUCCESS )\r
650         {\r
651                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
652                         ("__endpt_mgr_init returned %s\n", \r
653                         p_adapter->p_ifc->get_err_str( status )) );\r
654                 return status;\r
655         }\r
656 \r
657         /* We only ever destroy from the PnP callback thread. */\r
658         cl_status = cl_obj_init( &p_port->obj, CL_DESTROY_SYNC,\r
659                 __port_destroying, __port_cleanup, __port_free );\r
660 \r
661 #if DBG\r
662         cl_atomic_inc( &p_port->ref[ref_init] );\r
663         IPOIB_PRINT( TRACE_LEVEL_INFORMATION, IPOIB_DBG_OBJ,\r
664                 ("ref type %d ref_cnt %d\n", ref_init, p_port->obj.ref_cnt) );\r
665 #endif\r
666 \r
667         if( cl_status != CL_SUCCESS )\r
668         {\r
669                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
670                         ("cl_obj_init returned %#x\n", cl_status) );\r
671                 return IB_ERROR;\r
672         }\r
673 \r
674         cl_status = cl_obj_insert_rel( &p_port->rel, &p_adapter->obj, &p_port->obj );\r
675         if( cl_status != CL_SUCCESS )\r
676         {\r
677                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
678                         ("cl_obj_insert_rel returned %#x\n", cl_status) );\r
679                 cl_obj_destroy( &p_port->obj );\r
680                 return IB_ERROR;\r
681         }\r
682 \r
683 #if DBG\r
684         cl_atomic_inc( &p_port->ref[ref_init] );\r
685         IPOIB_PRINT( TRACE_LEVEL_ERROR, IPOIB_DBG_OBJ,\r
686                 ("ref type %d ref_cnt %d\n", ref_init, p_port->obj.ref_cnt) );\r
687 #endif\r
688 \r
689         IPOIB_EXIT( IPOIB_DBG_INIT );\r
690         return IB_SUCCESS;\r
691 }\r
692 \r
693 \r
694 static void\r
695 __port_destroying(\r
696         IN                              cl_obj_t* const                         p_obj )\r
697 {\r
698         ipoib_port_t    *p_port;\r
699 \r
700         IPOIB_ENTER( IPOIB_DBG_INIT );\r
701 \r
702         CL_ASSERT( p_obj );\r
703 \r
704         p_port = PARENT_STRUCT( p_obj, ipoib_port_t, obj );\r
705 \r
706         ipoib_port_down( p_port );\r
707 \r
708         __endpt_mgr_remove_all( p_port );\r
709 \r
710         ipoib_port_resume( p_port );\r
711 \r
712         IPOIB_EXIT( IPOIB_DBG_INIT );\r
713 }\r
714 \r
715 \r
716 static void\r
717 __port_cleanup(\r
718         IN                              cl_obj_t* const                         p_obj )\r
719 {\r
720         ipoib_port_t    *p_port;\r
721 \r
722         IPOIB_ENTER( IPOIB_DBG_INIT );\r
723 \r
724         CL_ASSERT( p_obj );\r
725 \r
726         p_port = PARENT_STRUCT( p_obj, ipoib_port_t, obj );\r
727 \r
728         /* Wait for all sends and receives to get flushed. */\r
729         while( p_port->send_mgr.depth || p_port->recv_mgr.depth )\r
730                 cl_thread_suspend( 0 );\r
731 \r
732         /* Destroy the send and receive managers before closing the CA. */\r
733         __ib_mgr_destroy( p_port );\r
734 \r
735         IPOIB_EXIT( IPOIB_DBG_INIT );\r
736 }\r
737 \r
738 \r
739 static void\r
740 __port_free(\r
741         IN                              cl_obj_t* const                         p_obj )\r
742 {\r
743         ipoib_port_t    *p_port;\r
744 \r
745         IPOIB_ENTER( IPOIB_DBG_INIT );\r
746 \r
747         CL_ASSERT( p_obj );\r
748 \r
749         p_port = PARENT_STRUCT( p_obj, ipoib_port_t, obj );\r
750 \r
751         __endpt_mgr_destroy( p_port );\r
752         __recv_mgr_destroy( p_port );\r
753         __send_mgr_destroy( p_port );\r
754         __buf_mgr_destroy( p_port );\r
755 \r
756         cl_spinlock_destroy( &p_port->send_lock );\r
757         cl_spinlock_destroy( &p_port->recv_lock );\r
758 \r
759         cl_obj_deinit( p_obj );\r
760 \r
761         cl_free( p_port );\r
762 \r
763         IPOIB_EXIT( IPOIB_DBG_INIT );\r
764 }\r
765 \r
766 \r
767 \r
768 /******************************************************************************\r
769 *\r
770 * IB resource manager implementation.\r
771 *\r
772 ******************************************************************************/\r
773 static void\r
774 __ib_mgr_construct(\r
775         IN                              ipoib_port_t* const                     p_port )\r
776 {\r
777         IPOIB_ENTER( IPOIB_DBG_INIT );\r
778 \r
779         cl_memclr( &p_port->ib_mgr, sizeof(ipoib_ib_mgr_t) );\r
780 \r
781         IPOIB_EXIT( IPOIB_DBG_INIT );\r
782 }\r
783 \r
784 \r
785 static ib_api_status_t\r
786 __ib_mgr_init(\r
787         IN                              ipoib_port_t* const                     p_port )\r
788 {\r
789         ib_api_status_t         status;\r
790         ib_cq_create_t          cq_create;\r
791         ib_qp_create_t          qp_create;\r
792         ib_phys_create_t        phys_create;\r
793         ib_phys_range_t         phys_range;\r
794         uint64_t                        vaddr;\r
795         net32_t                         rkey;\r
796         ib_qp_attr_t            qp_attr;\r
797         ib_ca_attr_t            *ca_attr;\r
798         uint32_t                        ca_size;\r
799 \r
800         IPOIB_ENTER( IPOIB_DBG_INIT );\r
801 \r
802         /* Open the CA. */\r
803         status = p_port->p_adapter->p_ifc->open_ca(\r
804                 p_port->p_adapter->h_al, p_port->p_adapter->guids.ca_guid,\r
805                 NULL, p_port, &p_port->ib_mgr.h_ca );\r
806         if( status != IB_SUCCESS )\r
807         {\r
808                 NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
809                         EVENT_IPOIB_OPEN_CA, 1, status );\r
810                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
811                         ("ib_open_ca returned %s\n", \r
812                         p_port->p_adapter->p_ifc->get_err_str( status )) );\r
813                 return status;\r
814         }\r
815 \r
816         /* Query the CA for Pkey table */\r
817         status = p_port->p_adapter->p_ifc->query_ca(p_port->ib_mgr.h_ca, NULL, &ca_size);\r
818         if(status != IB_INSUFFICIENT_MEMORY)\r
819         {\r
820                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
821                         ("ib_query_ca failed\n"));\r
822                         return status;\r
823         }\r
824 \r
825         ca_attr = (ib_ca_attr_t*)cl_zalloc(ca_size);\r
826         if      (!ca_attr)\r
827         {\r
828                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
829                         ("cl_zalloc can't allocate %d\n",ca_size));\r
830                 return IB_INSUFFICIENT_RESOURCES;\r
831         }\r
832 \r
833         status = p_port->p_adapter->p_ifc->query_ca(p_port->ib_mgr.h_ca, ca_attr,&ca_size);     \r
834         if( status != IB_SUCCESS )\r
835         {\r
836                 cl_free(ca_attr);\r
837                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
838                         ("ib_query_ca returned %s\n", \r
839                         p_port->p_adapter->p_ifc->get_err_str( status )) );\r
840                 return status;\r
841         }\r
842         if( ca_attr->p_port_attr->link_state == IB_LINK_ACTIVE)\r
843         {\r
844                 uint16_t index;\r
845                 CL_ASSERT(ca_attr->p_port_attr->p_pkey_table[0] == IB_DEFAULT_PKEY);\r
846                 for(index = 0; index < ca_attr->p_port_attr->num_pkeys; index++)\r
847                 {\r
848                         if(p_port->p_adapter->guids.port_guid.pkey == ca_attr->p_port_attr->p_pkey_table[index])\r
849                                 break;\r
850                 }\r
851                 if(index >= ca_attr->p_port_attr->num_pkeys)\r
852                 {\r
853                         IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
854                         ("Pkey table is invalid, index not found\n"));\r
855                     return IB_NOT_FOUND;\r
856                 }\r
857                 p_port->pkey_index = index;\r
858                 IPOIB_PRINT_EXIT( TRACE_LEVEL_INFORMATION, IPOIB_DBG_IB,\r
859                         ("for PKEY = 0x%04X got index = %d\n",p_port->p_adapter->guids.port_guid.pkey,index));\r
860         }\r
861         cl_free(ca_attr);\r
862         /* Allocate the PD. */\r
863         status = p_port->p_adapter->p_ifc->alloc_pd(\r
864                 p_port->ib_mgr.h_ca, IB_PDT_UD, p_port, &p_port->ib_mgr.h_pd );\r
865         if( status != IB_SUCCESS )\r
866         {\r
867                 NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
868                         EVENT_IPOIB_ALLOC_PD, 1, status );\r
869                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
870                         ("ib_alloc_pd returned %s\n", \r
871                         p_port->p_adapter->p_ifc->get_err_str( status )) );\r
872                 return status;\r
873         }\r
874 \r
875         /* Allocate receive CQ. */\r
876         cq_create.size = p_port->p_adapter->params.rq_depth;\r
877         cq_create.pfn_comp_cb = __recv_cb;\r
878         cq_create.h_wait_obj = NULL;\r
879 \r
880         status = p_port->p_adapter->p_ifc->create_cq(\r
881                 p_port->ib_mgr.h_ca, &cq_create, p_port,\r
882                 __cq_event, &p_port->ib_mgr.h_recv_cq );\r
883         if( status != IB_SUCCESS )\r
884         {\r
885                 NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
886                         EVENT_IPOIB_CREATE_RECV_CQ, 1, status );\r
887                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
888                         ("ib_create_cq returned %s.\n", \r
889                         p_port->p_adapter->p_ifc->get_err_str( status )) );\r
890                 return status;\r
891         }\r
892 \r
893         /* Allocate send CQ. */\r
894         cq_create.size = p_port->p_adapter->params.sq_depth;\r
895         cq_create.pfn_comp_cb = __send_cb;\r
896 \r
897         status = p_port->p_adapter->p_ifc->create_cq(\r
898                 p_port->ib_mgr.h_ca, &cq_create, p_port,\r
899                 __cq_event, &p_port->ib_mgr.h_send_cq );\r
900         if( status != IB_SUCCESS )\r
901         {\r
902                 NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
903                         EVENT_IPOIB_CREATE_SEND_CQ, 1, status );\r
904                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
905                         ("ib_create_cq returned %s.\n", \r
906                         p_port->p_adapter->p_ifc->get_err_str( status )) );\r
907                 return status;\r
908         }\r
909 \r
910         /* Allocate the QP. */\r
911         cl_memclr( &qp_create, sizeof(qp_create) );\r
912         qp_create.qp_type = IB_QPT_UNRELIABLE_DGRM;\r
913         qp_create.rq_depth = p_port->p_adapter->params.rq_depth;\r
914         qp_create.rq_sge = 2;   /* To support buffers spanning pages. */\r
915         qp_create.h_rq_cq = p_port->ib_mgr.h_recv_cq;\r
916         qp_create.sq_depth = p_port->p_adapter->params.sq_depth;\r
917         //TODO: Figure out the right number of SGE entries for sends.\r
918         qp_create.sq_sge = MAX_SEND_SGE;\r
919         qp_create.h_sq_cq = p_port->ib_mgr.h_send_cq;\r
920         qp_create.sq_signaled = TRUE;\r
921         status = p_port->p_adapter->p_ifc->create_qp(\r
922                 p_port->ib_mgr.h_pd, &qp_create, p_port,\r
923                 __qp_event, &p_port->ib_mgr.h_qp );\r
924         if( status != IB_SUCCESS )\r
925         {\r
926                 NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
927                         EVENT_IPOIB_CREATE_QP, 1, status );\r
928                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
929                         ("ib_create_qp returned %s\n", \r
930                         p_port->p_adapter->p_ifc->get_err_str( status )) );\r
931                 return status;\r
932         }\r
933         /* Query the QP so we can get our QPN. */\r
934         status = p_port->p_adapter->p_ifc->query_qp(\r
935                 p_port->ib_mgr.h_qp, &qp_attr );\r
936         if( status != IB_SUCCESS )\r
937         {\r
938                 NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
939                         EVENT_IPOIB_QUERY_QP, 1, status );\r
940                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
941                         ("ib_query_qp returned %s\n", \r
942                         p_port->p_adapter->p_ifc->get_err_str( status )) );\r
943                 return status;\r
944         }\r
945         p_port->ib_mgr.qpn = qp_attr.num;\r
946 \r
947         /* Register all of physical memory */\r
948         phys_create.length = MEM_REG_SIZE;\r
949         phys_create.num_ranges = 1;\r
950         phys_create.range_array = &phys_range;\r
951         phys_create.buf_offset = 0;\r
952         phys_create.hca_page_size = PAGE_SIZE;\r
953         phys_create.access_ctrl = IB_AC_LOCAL_WRITE;\r
954         phys_range.base_addr = 0;\r
955         phys_range.size = MEM_REG_SIZE;\r
956         vaddr = 0;\r
957         status = p_port->p_adapter->p_ifc->reg_phys(\r
958                 p_port->ib_mgr.h_pd, &phys_create, &vaddr,\r
959                 &p_port->ib_mgr.lkey, &rkey, &p_port->ib_mgr.h_mr );\r
960         if( status != IB_SUCCESS )\r
961         {\r
962                 NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
963                         EVENT_IPOIB_REG_PHYS, 1, status );\r
964                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
965                         ("ib_reg_phys returned %s\n", \r
966                         p_port->p_adapter->p_ifc->get_err_str( status )) );\r
967                 return status;\r
968         }\r
969 \r
970         IPOIB_EXIT( IPOIB_DBG_INIT );\r
971         return IB_SUCCESS;\r
972 }\r
973 \r
974 \r
975 static void\r
976 __ib_mgr_destroy(\r
977         IN                              ipoib_port_t* const                     p_port )\r
978 {\r
979         ib_api_status_t status;\r
980 \r
981         IPOIB_ENTER( IPOIB_DBG_INIT );\r
982 \r
983         if( p_port->ib_mgr.h_ca )\r
984         {\r
985                 status =\r
986                         p_port->p_adapter->p_ifc->close_ca( p_port->ib_mgr.h_ca, NULL );\r
987                 CL_ASSERT( status == IB_SUCCESS );\r
988                 p_port->ib_mgr.h_ca = NULL;\r
989         }\r
990 \r
991         IPOIB_EXIT( IPOIB_DBG_INIT );\r
992 }\r
993 \r
994 \r
995 \r
996 /******************************************************************************\r
997 *\r
998 * Buffer manager implementation.\r
999 *\r
1000 ******************************************************************************/\r
1001 static void\r
1002 __buf_mgr_construct(\r
1003         IN                              ipoib_port_t* const                     p_port )\r
1004 {\r
1005         IPOIB_ENTER( IPOIB_DBG_INIT );\r
1006 \r
1007         cl_qpool_construct( &p_port->buf_mgr.recv_pool );\r
1008 \r
1009         p_port->buf_mgr.h_packet_pool = NULL;\r
1010         p_port->buf_mgr.h_buffer_pool = NULL;\r
1011 \r
1012         ExInitializeNPagedLookasideList( &p_port->buf_mgr.send_buf_list,\r
1013                 NULL, NULL, 0, MAX_XFER_BLOCK_SIZE, 'bipi', 0 );\r
1014 \r
1015         p_port->buf_mgr.h_send_pkt_pool = NULL;\r
1016         p_port->buf_mgr.h_send_buf_pool = NULL;\r
1017 \r
1018         IPOIB_EXIT( IPOIB_DBG_INIT );\r
1019 }\r
1020 \r
1021 \r
1022 static ib_api_status_t\r
1023 __buf_mgr_init(\r
1024         IN                              ipoib_port_t* const                     p_port )\r
1025 {\r
1026         cl_status_t             cl_status;\r
1027         NDIS_STATUS             ndis_status;\r
1028         ipoib_params_t  *p_params;\r
1029 \r
1030         IPOIB_ENTER(IPOIB_DBG_INIT );\r
1031 \r
1032         CL_ASSERT( p_port );\r
1033         CL_ASSERT( p_port->p_adapter );\r
1034 \r
1035         p_params = &p_port->p_adapter->params;\r
1036 \r
1037         /* Allocate the receive descriptor pool */\r
1038         cl_status = cl_qpool_init( &p_port->buf_mgr.recv_pool,\r
1039                 p_params->rq_depth * p_params->recv_pool_ratio,\r
1040 #if IPOIB_INLINE_RECV\r
1041                 0, 0, sizeof(ipoib_recv_desc_t), __recv_ctor, NULL, p_port );\r
1042 #else   /* IPOIB_INLINE_RECV */\r
1043                 0, 0, sizeof(ipoib_recv_desc_t), __recv_ctor, __recv_dtor, p_port );\r
1044 #endif  /* IPOIB_INLINE_RECV */\r
1045         if( cl_status != CL_SUCCESS )\r
1046         {\r
1047                 NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
1048                         EVENT_IPOIB_RECV_POOL, 1, cl_status );\r
1049                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1050                         ("cl_qpool_init for recvs returned %#x\n",\r
1051                         cl_status) );\r
1052                 return IB_INSUFFICIENT_MEMORY;\r
1053         }\r
1054 \r
1055         /* Allocate the NDIS buffer and packet pools for receive indication. */\r
1056         NdisAllocatePacketPool( &ndis_status, &p_port->buf_mgr.h_packet_pool,\r
1057                 p_params->rq_depth, PROTOCOL_RESERVED_SIZE_IN_PACKET );\r
1058         if( ndis_status != NDIS_STATUS_SUCCESS )\r
1059         {\r
1060                 NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
1061                         EVENT_IPOIB_RECV_PKT_POOL, 1, ndis_status );\r
1062                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1063                         ("NdisAllocatePacketPool returned %08X\n", ndis_status) );\r
1064                 return IB_INSUFFICIENT_RESOURCES;\r
1065         }\r
1066 \r
1067         NdisAllocateBufferPool( &ndis_status, &p_port->buf_mgr.h_buffer_pool,\r
1068                 p_params->rq_depth );\r
1069         if( ndis_status != NDIS_STATUS_SUCCESS )\r
1070         {\r
1071                 NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
1072                         EVENT_IPOIB_RECV_BUF_POOL, 1, ndis_status );\r
1073                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1074                         ("NdisAllocateBufferPool returned %08X\n", ndis_status) );\r
1075                 return IB_INSUFFICIENT_RESOURCES;\r
1076         }\r
1077 \r
1078         /* Allocate the NDIS buffer and packet pools for send formatting. */\r
1079         NdisAllocatePacketPool( &ndis_status, &p_port->buf_mgr.h_send_pkt_pool,\r
1080                 1, PROTOCOL_RESERVED_SIZE_IN_PACKET );\r
1081         if( ndis_status != NDIS_STATUS_SUCCESS )\r
1082         {\r
1083                 NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
1084                         EVENT_IPOIB_SEND_PKT_POOL, 1, ndis_status );\r
1085                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1086                         ("NdisAllocatePacketPool returned %08X\n", ndis_status) );\r
1087                 return IB_INSUFFICIENT_RESOURCES;\r
1088         }\r
1089 \r
1090         NdisAllocateBufferPool( &ndis_status,\r
1091                 &p_port->buf_mgr.h_send_buf_pool, 1 );\r
1092         if( ndis_status != NDIS_STATUS_SUCCESS )\r
1093         {\r
1094                 NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
1095                         EVENT_IPOIB_SEND_BUF_POOL, 1, ndis_status );\r
1096                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1097                         ("NdisAllocateBufferPool returned %08X\n", ndis_status) );\r
1098                 return IB_INSUFFICIENT_RESOURCES;\r
1099         }\r
1100 \r
1101         IPOIB_EXIT( IPOIB_DBG_INIT );\r
1102         return IB_SUCCESS;\r
1103 }\r
1104 \r
1105 \r
1106 static void\r
1107 __buf_mgr_destroy(\r
1108         IN                              ipoib_port_t* const                     p_port )\r
1109 {\r
1110         IPOIB_ENTER(IPOIB_DBG_INIT );\r
1111 \r
1112         CL_ASSERT( p_port );\r
1113 \r
1114         /* Destroy the send packet and buffer pools. */\r
1115         if( p_port->buf_mgr.h_send_buf_pool )\r
1116                 NdisFreeBufferPool( p_port->buf_mgr.h_send_buf_pool );\r
1117         if( p_port->buf_mgr.h_send_pkt_pool )\r
1118                 NdisFreePacketPool( p_port->buf_mgr.h_send_pkt_pool );\r
1119 \r
1120         /* Destroy the receive packet and buffer pools. */\r
1121         if( p_port->buf_mgr.h_buffer_pool )\r
1122                 NdisFreeBufferPool( p_port->buf_mgr.h_buffer_pool );\r
1123         if( p_port->buf_mgr.h_packet_pool )\r
1124                 NdisFreePacketPool( p_port->buf_mgr.h_packet_pool );\r
1125 \r
1126         /* Free the receive and send descriptors. */\r
1127         cl_qpool_destroy( &p_port->buf_mgr.recv_pool );\r
1128 \r
1129         /* Free the lookaside list of scratch buffers. */\r
1130         ExDeleteNPagedLookasideList( &p_port->buf_mgr.send_buf_list );\r
1131 \r
1132         IPOIB_EXIT(  IPOIB_DBG_INIT );\r
1133 }\r
1134 \r
1135 \r
1136 static cl_status_t\r
1137 __recv_ctor(\r
1138         IN                              void* const                                     p_object,\r
1139         IN                              void*                                           context,\r
1140                 OUT                     cl_pool_item_t** const          pp_pool_item )\r
1141 {\r
1142         ipoib_recv_desc_t       *p_desc;\r
1143         ipoib_port_t            *p_port;\r
1144         uint32_t                        ds0_len;\r
1145 \r
1146         IPOIB_ENTER( IPOIB_DBG_ALLOC );\r
1147 \r
1148         CL_ASSERT( p_object );\r
1149         CL_ASSERT( context );\r
1150 \r
1151         p_desc = (ipoib_recv_desc_t*)p_object;\r
1152         p_port = (ipoib_port_t*)context;\r
1153 \r
1154         /* Setup the work request. */\r
1155         p_desc->wr.ds_array = p_desc->local_ds;\r
1156         p_desc->wr.wr_id = (uintn_t)p_desc;\r
1157 \r
1158 #if IPOIB_INLINE_RECV\r
1159         /* Sanity check on the receive buffer layout */\r
1160         CL_ASSERT( (void*)&p_desc->buf.eth.pkt.type ==\r
1161                 (void*)&p_desc->buf.ib.pkt.type );\r
1162         CL_ASSERT( sizeof(recv_buf_t) == sizeof(ipoib_pkt_t) + sizeof(ib_grh_t) );\r
1163 \r
1164         /* Setup the local data segment. */\r
1165         p_desc->local_ds[0].vaddr = cl_get_physaddr( &p_desc->buf );\r
1166         p_desc->local_ds[0].lkey = p_port->ib_mgr.lkey;\r
1167         ds0_len =\r
1168                 PAGE_SIZE - ((uint32_t)p_desc->local_ds[0].vaddr & (PAGE_SIZE - 1));\r
1169         if( ds0_len >= sizeof(recv_buf_t) )\r
1170         {\r
1171                 /* The whole buffer is within a page. */\r
1172                 p_desc->local_ds[0].length = ds0_len;\r
1173                 p_desc->wr.num_ds = 1;\r
1174         }\r
1175         else\r
1176         {\r
1177                 /* The buffer crosses page boundaries. */\r
1178                 p_desc->local_ds[0].length = ds0_len;\r
1179                 p_desc->local_ds[1].vaddr = cl_get_physaddr( \r
1180                         ((uint8_t*)&p_desc->buf) + ds0_len );\r
1181                 p_desc->local_ds[1].lkey = p_port->ib_mgr.lkey;\r
1182                 p_desc->local_ds[1].length = sizeof(recv_buf_t) - ds0_len;\r
1183                 p_desc->wr.num_ds = 2;\r
1184         }\r
1185 #else   /* IPOIB_INLINE_RECV */\r
1186         /* Allocate the receive buffer. */\r
1187         p_desc->p_buf = (recv_buf_t*)cl_zalloc( sizeof(recv_buf_t) );\r
1188         if( !p_desc->p_buf )\r
1189         {\r
1190                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1191                         ("Failed to allocate receive buffer.\n") );\r
1192                 return CL_INSUFFICIENT_MEMORY;\r
1193         }\r
1194 \r
1195         /* Sanity check on the receive buffer layout */\r
1196         CL_ASSERT( (void*)&p_desc->p_buf->eth.pkt.type ==\r
1197                 (void*)&p_desc->p_buf->ib.pkt.type );\r
1198 \r
1199         /* Setup the local data segment. */\r
1200         p_desc->local_ds[0].vaddr = cl_get_physaddr( p_desc->p_buf );\r
1201         p_desc->local_ds[0].length = sizeof(ipoib_pkt_t) + sizeof(ib_grh_t);\r
1202         p_desc->local_ds[0].lkey = p_port->ib_mgr.lkey;\r
1203 #endif  /* IPOIB_INLINE_RECV */\r
1204 \r
1205         *pp_pool_item = &p_desc->item;\r
1206 \r
1207         IPOIB_EXIT( IPOIB_DBG_ALLOC );\r
1208         return CL_SUCCESS;\r
1209 }\r
1210 \r
1211 \r
1212 #if !IPOIB_INLINE_RECV\r
1213 static void\r
1214 __recv_dtor(\r
1215         IN              const   cl_pool_item_t* const           p_pool_item,\r
1216         IN                              void                                            *context )\r
1217 {\r
1218         ipoib_recv_desc_t       *p_desc;\r
1219 \r
1220         IPOIB_ENTER(  IPOIB_DBG_ALLOC );\r
1221 \r
1222         UNUSED_PARAM( context );\r
1223 \r
1224         p_desc = PARENT_STRUCT( p_pool_item, ipoib_recv_desc_t, item );\r
1225 \r
1226         if( p_desc->p_buf )\r
1227                 cl_free( p_desc->p_buf );\r
1228 \r
1229         IPOIB_EXIT( IPOIB_DBG_ALLOC );\r
1230 }\r
1231 #endif\r
1232 \r
1233 \r
1234 static inline ipoib_recv_desc_t*\r
1235 __buf_mgr_get_recv(\r
1236         IN                              ipoib_port_t* const                     p_port )\r
1237 {\r
1238         ipoib_recv_desc_t       *p_desc;\r
1239         IPOIB_ENTER( IPOIB_DBG_RECV );\r
1240         p_desc = (ipoib_recv_desc_t*)cl_qpool_get( &p_port->buf_mgr.recv_pool );\r
1241         /* Reference the port object for the send. */\r
1242         if( p_desc )\r
1243         {\r
1244                 ipoib_port_ref( p_port, ref_get_recv );\r
1245                 CL_ASSERT( p_desc->wr.wr_id == (uintn_t)p_desc );\r
1246 #if IPOIB_INLINE_RECV\r
1247                 CL_ASSERT( p_desc->local_ds[0].vaddr ==\r
1248                         cl_get_physaddr( &p_desc->buf ) );\r
1249 #else   /* IPOIB_INLINE_RECV */\r
1250                 CL_ASSERT( p_desc->local_ds[0].vaddr ==\r
1251                         cl_get_physaddr( p_desc->p_buf ) );\r
1252                 CL_ASSERT( p_desc->local_ds[0].length ==\r
1253                         (sizeof(ipoib_pkt_t) + sizeof(ib_grh_t)) );\r
1254 #endif  /* IPOIB_INLINE_RECV */\r
1255                 CL_ASSERT( p_desc->local_ds[0].lkey == p_port->ib_mgr.lkey );\r
1256         }\r
1257         IPOIB_EXIT( IPOIB_DBG_RECV );\r
1258         return p_desc;\r
1259 }\r
1260 \r
1261 \r
1262 static inline void\r
1263 __buf_mgr_put_recv(\r
1264         IN                              ipoib_port_t* const                     p_port,\r
1265         IN                              ipoib_recv_desc_t* const        p_desc,\r
1266         IN                              NDIS_PACKET* const                      p_packet OPTIONAL )\r
1267 {\r
1268         NDIS_BUFFER             *p_buf;\r
1269 \r
1270         IPOIB_ENTER(IPOIB_DBG_RECV );\r
1271 \r
1272         if( p_packet )\r
1273         {\r
1274                 /* Unchain the NDIS buffer. */\r
1275                 NdisUnchainBufferAtFront( p_packet, &p_buf );\r
1276                 CL_ASSERT( p_buf );\r
1277                 /* Return the NDIS packet and NDIS buffer to their pools. */\r
1278                 NdisDprFreePacketNonInterlocked( p_packet );\r
1279                 NdisFreeBuffer( p_buf );\r
1280         }\r
1281 \r
1282         /* Return the descriptor to its pools. */\r
1283         cl_qpool_put( &p_port->buf_mgr.recv_pool, &p_desc->item );\r
1284 \r
1285         /*\r
1286          * Dereference the port object since the receive is no longer outstanding.\r
1287          */\r
1288         ipoib_port_deref( p_port, ref_get_recv );\r
1289         IPOIB_EXIT(  IPOIB_DBG_RECV );\r
1290 }\r
1291 \r
1292 \r
1293 static inline void\r
1294 __buf_mgr_put_recv_list(\r
1295         IN                              ipoib_port_t* const                     p_port,\r
1296         IN                              cl_qlist_t* const                       p_list )\r
1297 {\r
1298         //IPOIB_ENTER(  IPOIB_DBG_RECV );\r
1299         cl_qpool_put_list( &p_port->buf_mgr.recv_pool, p_list );\r
1300         //IPOIB_EXIT(  IPOIB_DBG_RECV );\r
1301 }\r
1302 \r
1303 \r
1304 static inline NDIS_PACKET*\r
1305 __buf_mgr_get_ndis_pkt(\r
1306         IN                              ipoib_port_t* const                     p_port,\r
1307         IN                              ipoib_recv_desc_t* const        p_desc )\r
1308 {\r
1309         NDIS_STATUS                             status;\r
1310         NDIS_PACKET                             *p_packet;\r
1311         NDIS_BUFFER                             *p_buffer;\r
1312 \r
1313         IPOIB_ENTER(  IPOIB_DBG_RECV );\r
1314 \r
1315         NdisDprAllocatePacketNonInterlocked( &status, &p_packet,\r
1316                 p_port->buf_mgr.h_packet_pool );\r
1317         if( status != NDIS_STATUS_SUCCESS )\r
1318         {\r
1319                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1320                         ("Failed to allocate NDIS_PACKET: %08x\n", status) );\r
1321                 return NULL;\r
1322         }\r
1323 \r
1324         IPOIB_PORT_FROM_PACKET( p_packet ) = p_port;\r
1325         IPOIB_RECV_FROM_PACKET( p_packet ) = p_desc;\r
1326 \r
1327         NdisAllocateBuffer( &status, &p_buffer,\r
1328 #if IPOIB_INLINE_RECV\r
1329                 p_port->buf_mgr.h_buffer_pool, &p_desc->buf.eth.pkt, p_desc->len );\r
1330 #else   /* IPOIB_INLINE_RECV */\r
1331                 p_port->buf_mgr.h_buffer_pool, &p_desc->p_buf->eth.pkt, p_desc->len );\r
1332 #endif  /* IPOIB_INLINE_RECV */\r
1333         if( status != NDIS_STATUS_SUCCESS )\r
1334         {\r
1335                 IPOIB_PRINT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1336                         ("Failed to allocate NDIS_BUFFER: %08x\n", status) );\r
1337                 NdisDprFreePacketNonInterlocked( p_packet );\r
1338                 return NULL;\r
1339         }\r
1340 \r
1341         NdisChainBufferAtFront( p_packet, p_buffer );\r
1342         NDIS_SET_PACKET_HEADER_SIZE( p_packet, sizeof(eth_hdr_t) );\r
1343 \r
1344         IPOIB_EXIT(  IPOIB_DBG_RECV );\r
1345         return p_packet;\r
1346 }\r
1347 \r
1348 /******************************************************************************\r
1349 *\r
1350 * Receive manager implementation.\r
1351 *\r
1352 ******************************************************************************/\r
1353 static void\r
1354 __recv_mgr_construct(\r
1355         IN                              ipoib_port_t* const                     p_port )\r
1356 {\r
1357         IPOIB_ENTER( IPOIB_DBG_INIT );\r
1358 \r
1359         cl_qlist_init( &p_port->recv_mgr.done_list );\r
1360 \r
1361         p_port->recv_mgr.recv_pkt_array = NULL;\r
1362 \r
1363         IPOIB_EXIT( IPOIB_DBG_INIT );\r
1364 }\r
1365 \r
1366 \r
1367 static ib_api_status_t\r
1368 __recv_mgr_init(\r
1369         IN                              ipoib_port_t* const                     p_port )\r
1370 {\r
1371         IPOIB_ENTER( IPOIB_DBG_INIT );\r
1372 \r
1373         /* Allocate the NDIS_PACKET pointer array for indicating receives. */\r
1374         p_port->recv_mgr.recv_pkt_array = cl_malloc(\r
1375                 sizeof(NDIS_PACKET*) * p_port->p_adapter->params.rq_depth );\r
1376         if( !p_port->recv_mgr.recv_pkt_array )\r
1377         {\r
1378                 NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
1379                         EVENT_IPOIB_RECV_PKT_ARRAY, 0 );\r
1380                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1381                         ("cl_malloc for PNDIS_PACKET array failed.\n") );\r
1382                 return IB_INSUFFICIENT_MEMORY;\r
1383         }\r
1384 \r
1385         IPOIB_EXIT( IPOIB_DBG_INIT );\r
1386         return IB_SUCCESS;\r
1387 }\r
1388 \r
1389 \r
1390 static void\r
1391 __recv_mgr_destroy(\r
1392         IN                              ipoib_port_t* const                     p_port )\r
1393 {\r
1394         IPOIB_ENTER( IPOIB_DBG_INIT );\r
1395 \r
1396         CL_ASSERT( cl_is_qlist_empty( &p_port->recv_mgr.done_list ) );\r
1397         CL_ASSERT( !p_port->recv_mgr.depth );\r
1398 \r
1399         if( p_port->recv_mgr.recv_pkt_array )\r
1400                 cl_free( p_port->recv_mgr.recv_pkt_array );\r
1401 \r
1402         IPOIB_EXIT( IPOIB_DBG_INIT );\r
1403 }\r
1404 \r
1405 \r
1406 /*\r
1407  * Posts receive buffers to the receive queue and returns the number\r
1408  * of receives needed to bring the RQ to its low water mark.  Note\r
1409  * that the value is signed, and can go negative.  All tests must\r
1410  * be for > 0.\r
1411  */\r
1412 static int32_t\r
1413 __recv_mgr_repost(\r
1414         IN                              ipoib_port_t* const                     p_port )\r
1415 {\r
1416         ipoib_recv_desc_t       *p_head = NULL, *p_tail = NULL, *p_next;\r
1417         ib_api_status_t         status;\r
1418         ib_recv_wr_t            *p_failed;\r
1419         PERF_DECLARE( GetRecv );\r
1420         PERF_DECLARE( PostRecv );\r
1421 \r
1422         IPOIB_ENTER( IPOIB_DBG_RECV );\r
1423 \r
1424         CL_ASSERT( p_port );\r
1425         cl_obj_lock( &p_port->obj );\r
1426         if( p_port->state != IB_QPS_RTS )\r
1427         {\r
1428                 cl_obj_unlock( &p_port->obj );\r
1429                 IPOIB_PRINT_EXIT( TRACE_LEVEL_INFORMATION, IPOIB_DBG_RECV,\r
1430                         ("Port in invalid state.  Not reposting.\n") );\r
1431                 return 0;\r
1432         }\r
1433         ipoib_port_ref( p_port, ref_repost );\r
1434         cl_obj_unlock( &p_port->obj );\r
1435 \r
1436         while( p_port->recv_mgr.depth < p_port->p_adapter->params.rq_depth )\r
1437         {\r
1438                 /* Pull receives out of the pool and chain them up. */\r
1439                 cl_perf_start( GetRecv );\r
1440                 p_next = __buf_mgr_get_recv( p_port );\r
1441                 cl_perf_stop( &p_port->p_adapter->perf, GetRecv );\r
1442                 if( !p_next )\r
1443                 {\r
1444                         IPOIB_PRINT(TRACE_LEVEL_INFORMATION, IPOIB_DBG_RECV,\r
1445                                 ("Out of receive descriptors! recv queue depath 0x%x\n",p_port->recv_mgr.depth) );\r
1446                         break;\r
1447                 }\r
1448 \r
1449                 if( !p_tail )\r
1450                 {\r
1451                         p_tail = p_next;\r
1452                         p_next->wr.p_next = NULL;\r
1453                 }\r
1454                 else\r
1455                 {\r
1456                         p_next->wr.p_next = &p_head->wr;\r
1457                 }\r
1458 \r
1459                 p_head = p_next;\r
1460 \r
1461                 p_port->recv_mgr.depth++;\r
1462         }\r
1463 \r
1464         if( p_head )\r
1465         {\r
1466                 cl_perf_start( PostRecv );\r
1467                 status = p_port->p_adapter->p_ifc->post_recv(\r
1468                         p_port->ib_mgr.h_qp, &p_head->wr, &p_failed );\r
1469                 cl_perf_stop( &p_port->p_adapter->perf, PostRecv );\r
1470 \r
1471                 if( status != IB_SUCCESS )\r
1472                 {\r
1473                         IPOIB_PRINT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1474                                 ("ip_post_recv returned %s\n", \r
1475                                 p_port->p_adapter->p_ifc->get_err_str( status )) );\r
1476                         /* return the descriptors to the pool */\r
1477                         while( p_failed )\r
1478                         {\r
1479                                 p_head = PARENT_STRUCT( p_failed, ipoib_recv_desc_t, wr );\r
1480                                 p_failed = p_failed->p_next;\r
1481 \r
1482                                 __buf_mgr_put_recv( p_port, p_head, NULL );\r
1483                                 p_port->recv_mgr.depth--;\r
1484                         }\r
1485                 }\r
1486         }\r
1487 \r
1488         ipoib_port_deref( p_port, ref_repost );\r
1489         IPOIB_EXIT( IPOIB_DBG_RECV );\r
1490         return p_port->p_adapter->params.rq_low_watermark - p_port->recv_mgr.depth;\r
1491 }\r
1492 \r
1493 \r
1494 void\r
1495 ipoib_return_packet(\r
1496         IN                              NDIS_HANDLE                                     adapter_context,\r
1497         IN                              NDIS_PACKET                                     *p_packet )\r
1498 {\r
1499         cl_list_item_t          *p_item;\r
1500         ipoib_port_t            *p_port;\r
1501         ipoib_recv_desc_t       *p_desc;\r
1502         ib_api_status_t         status = IB_NOT_DONE;\r
1503         int32_t                         shortage;\r
1504         PERF_DECLARE( ReturnPacket );\r
1505         PERF_DECLARE( ReturnPutRecv );\r
1506         PERF_DECLARE( ReturnRepostRecv );\r
1507         PERF_DECLARE( ReturnPreparePkt );\r
1508         PERF_DECLARE( ReturnNdisIndicate );\r
1509 \r
1510         IPOIB_ENTER( IPOIB_DBG_RECV );\r
1511 \r
1512         UNUSED_PARAM( adapter_context );\r
1513         CL_ASSERT( p_packet );\r
1514 \r
1515         cl_perf_start( ReturnPacket );\r
1516 \r
1517         /* Get the port and descriptor from the packet. */\r
1518         p_port = IPOIB_PORT_FROM_PACKET( p_packet );\r
1519         p_desc = IPOIB_RECV_FROM_PACKET( p_packet );\r
1520 \r
1521         cl_spinlock_acquire( &p_port->recv_lock );\r
1522 \r
1523         cl_perf_start( ReturnPutRecv );\r
1524         __buf_mgr_put_recv( p_port, p_desc, p_packet );\r
1525         cl_perf_stop( &p_port->p_adapter->perf, ReturnPutRecv );\r
1526 \r
1527         /* Repost buffers. */\r
1528         cl_perf_start( ReturnRepostRecv );\r
1529         shortage = __recv_mgr_repost( p_port );\r
1530         cl_perf_stop( &p_port->p_adapter->perf, ReturnRepostRecv );\r
1531 \r
1532         for( p_item = cl_qlist_remove_head( &p_port->recv_mgr.done_list );\r
1533                 p_item != cl_qlist_end( &p_port->recv_mgr.done_list );\r
1534                 p_item = cl_qlist_remove_head( &p_port->recv_mgr.done_list ) )\r
1535         {\r
1536                 p_desc = (ipoib_recv_desc_t*)p_item;\r
1537 \r
1538                 cl_perf_start( ReturnPreparePkt );\r
1539                 status = __recv_mgr_prepare_pkt( p_port, p_desc, &p_packet );\r
1540                 cl_perf_stop( &p_port->p_adapter->perf, ReturnPreparePkt );\r
1541                 if( status == IB_SUCCESS )\r
1542                 {\r
1543                         if( shortage > 0 )\r
1544                                 NDIS_SET_PACKET_STATUS( p_packet, NDIS_STATUS_RESOURCES );\r
1545                         else\r
1546                                 NDIS_SET_PACKET_STATUS( p_packet, NDIS_STATUS_SUCCESS );\r
1547 \r
1548                         cl_spinlock_release( &p_port->recv_lock );\r
1549                         cl_perf_start( ReturnNdisIndicate );\r
1550                         NdisMIndicateReceivePacket( p_port->p_adapter->h_adapter,\r
1551                                 &p_packet, 1 );\r
1552                         cl_perf_stop( &p_port->p_adapter->perf, ReturnNdisIndicate );\r
1553                         cl_spinlock_acquire( &p_port->recv_lock );\r
1554 \r
1555                         if( shortage > 0 )\r
1556                         {\r
1557                                 cl_perf_start( ReturnPutRecv );\r
1558                                 __buf_mgr_put_recv( p_port, p_desc, p_packet );\r
1559                                 cl_perf_stop( &p_port->p_adapter->perf, ReturnPutRecv );\r
1560 \r
1561                                 /* Repost buffers. */\r
1562                                 cl_perf_start( ReturnRepostRecv );\r
1563                                 shortage = __recv_mgr_repost( p_port );\r
1564                                 cl_perf_stop( &p_port->p_adapter->perf, ReturnRepostRecv );\r
1565                         }\r
1566                 }\r
1567                 else if( status != IB_NOT_DONE )\r
1568                 {\r
1569                         IPOIB_PRINT(TRACE_LEVEL_INFORMATION, IPOIB_DBG_RECV,\r
1570                                 ("__recv_mgr_prepare_pkt returned %s\n",\r
1571                                 p_port->p_adapter->p_ifc->get_err_str( status )) );\r
1572                         /* Return the item to the head of the list. */\r
1573                         cl_qlist_insert_head( &p_port->recv_mgr.done_list, p_item );\r
1574                         break;\r
1575                 }\r
1576         }\r
1577         cl_spinlock_release( &p_port->recv_lock );\r
1578         cl_perf_stop( &p_port->p_adapter->perf, ReturnPacket );\r
1579 \r
1580         IPOIB_EXIT( IPOIB_DBG_RECV );\r
1581 }\r
1582 \r
1583 \r
1584 static void\r
1585 __recv_cb(\r
1586         IN              const   ib_cq_handle_t                          h_cq,\r
1587         IN                              void                                            *cq_context )\r
1588 {\r
1589         ipoib_port_t            *p_port;\r
1590         ib_api_status_t         status;\r
1591         ib_wc_t                         wc[MAX_RECV_WC], *p_free, *p_wc;\r
1592         int32_t                         pkt_cnt, recv_cnt = 0, shortage, discarded;\r
1593         cl_qlist_t                      done_list, bad_list;\r
1594         size_t                          i;\r
1595         PERF_DECLARE( RecvCompBundle );\r
1596         PERF_DECLARE( RecvCb );\r
1597         PERF_DECLARE( PollRecv );\r
1598         PERF_DECLARE( RepostRecv );\r
1599         PERF_DECLARE( FilterRecv );\r
1600         PERF_DECLARE( BuildPktArray );\r
1601         PERF_DECLARE( RecvNdisIndicate );\r
1602         PERF_DECLARE( RearmRecv );\r
1603         PERF_DECLARE( PutRecvList );\r
1604 \r
1605         IPOIB_ENTER( IPOIB_DBG_RECV );\r
1606 \r
1607         cl_perf_clr( RecvCompBundle );\r
1608 \r
1609         cl_perf_start( RecvCb );\r
1610 \r
1611         UNUSED_PARAM( h_cq );\r
1612 \r
1613         p_port = (ipoib_port_t*)cq_context;\r
1614 \r
1615         cl_qlist_init( &done_list );\r
1616         cl_qlist_init( &bad_list );\r
1617 \r
1618         ipoib_port_ref( p_port, ref_recv_cb );\r
1619         for( i = 0; i < MAX_RECV_WC; i++ )\r
1620                 wc[i].p_next = &wc[i + 1];\r
1621         wc[MAX_RECV_WC - 1].p_next = NULL;\r
1622 \r
1623         /*\r
1624          * We'll be accessing the endpoint map so take a reference\r
1625          * on it to prevent modifications.\r
1626          */\r
1627         cl_obj_lock( &p_port->obj );\r
1628         cl_atomic_inc( &p_port->endpt_rdr );\r
1629         cl_obj_unlock( &p_port->obj );\r
1630 \r
1631         do\r
1632         {\r
1633                 /* If we get here, then the list of WCs is intact. */\r
1634                 p_free = wc;\r
1635 \r
1636                 cl_perf_start( PollRecv );\r
1637                 status = p_port->p_adapter->p_ifc->poll_cq(\r
1638                         p_port->ib_mgr.h_recv_cq, &p_free, &p_wc );\r
1639                 cl_perf_stop( &p_port->p_adapter->perf, PollRecv );\r
1640                 CL_ASSERT( status == IB_SUCCESS || status == IB_NOT_FOUND );\r
1641 \r
1642                 /* Look at the payload now and filter ARP and DHCP packets. */\r
1643                 cl_perf_start( FilterRecv );\r
1644                 recv_cnt += __recv_mgr_filter( p_port, p_wc, &done_list, &bad_list );\r
1645                 cl_perf_stop( &p_port->p_adapter->perf, FilterRecv );\r
1646 \r
1647         } while( !p_free );\r
1648 \r
1649         /* We're done looking at the endpoint map, release the reference. */\r
1650         cl_atomic_dec( &p_port->endpt_rdr );\r
1651 \r
1652         cl_perf_log( &p_port->p_adapter->perf, RecvCompBundle, recv_cnt );\r
1653 \r
1654         cl_spinlock_acquire( &p_port->recv_lock );\r
1655 \r
1656         /* Update our posted depth. */\r
1657         p_port->recv_mgr.depth -= recv_cnt;\r
1658 \r
1659         /* Return any discarded receives to the pool */\r
1660         cl_perf_start( PutRecvList );\r
1661         __buf_mgr_put_recv_list( p_port, &bad_list );\r
1662         cl_perf_stop( &p_port->p_adapter->perf, PutRecvList );\r
1663 \r
1664         do\r
1665         {\r
1666                 /* Repost ASAP so we don't starve the RQ. */\r
1667                 cl_perf_start( RepostRecv );\r
1668                 shortage = __recv_mgr_repost( p_port );\r
1669                 cl_perf_stop( &p_port->p_adapter->perf, RepostRecv );\r
1670 \r
1671                 cl_perf_start( BuildPktArray );\r
1672                 /* Notify NDIS of any and all possible receive buffers. */\r
1673                 pkt_cnt = __recv_mgr_build_pkt_array(\r
1674                         p_port, shortage, &done_list, &discarded );\r
1675                 cl_perf_stop( &p_port->p_adapter->perf, BuildPktArray );\r
1676 \r
1677                 /* Only indicate receives if we actually had any. */\r
1678                 if( discarded && shortage > 0 )\r
1679                 {\r
1680                         /* We may have thrown away packets, and have a shortage */\r
1681                         cl_perf_start( RepostRecv );\r
1682                         __recv_mgr_repost( p_port );\r
1683                         cl_perf_stop( &p_port->p_adapter->perf, RepostRecv );\r
1684                 }\r
1685 \r
1686                 if( !pkt_cnt )\r
1687                         break;\r
1688 \r
1689                 cl_spinlock_release( &p_port->recv_lock );\r
1690 \r
1691                 cl_perf_start( RecvNdisIndicate );\r
1692                 NdisMIndicateReceivePacket( p_port->p_adapter->h_adapter,\r
1693                         p_port->recv_mgr.recv_pkt_array, pkt_cnt );\r
1694                 cl_perf_stop( &p_port->p_adapter->perf, RecvNdisIndicate );\r
1695 \r
1696                 /*\r
1697                  * Cap the number of receives to put back to what we just indicated\r
1698                  * with NDIS_STATUS_RESOURCES.\r
1699                  */\r
1700                 if( shortage > 0 )\r
1701                 {\r
1702                         if( pkt_cnt < shortage )\r
1703                                 shortage = pkt_cnt;\r
1704 \r
1705                         /* Return all but the last packet to the pool. */\r
1706                         cl_spinlock_acquire( &p_port->recv_lock );\r
1707                         while( shortage-- > 1 )\r
1708                         {\r
1709                                 __buf_mgr_put_recv( p_port,\r
1710                                         IPOIB_RECV_FROM_PACKET( p_port->recv_mgr.recv_pkt_array[shortage] ),\r
1711                                         p_port->recv_mgr.recv_pkt_array[shortage] );\r
1712                         }\r
1713                         cl_spinlock_release( &p_port->recv_lock );\r
1714 \r
1715                         /*\r
1716                          * Return the last packet as if NDIS returned it, so that we repost\r
1717                          * and report any other pending receives.\r
1718                          */\r
1719                         ipoib_return_packet( NULL, p_port->recv_mgr.recv_pkt_array[0] );\r
1720                 }\r
1721                 cl_spinlock_acquire( &p_port->recv_lock );\r
1722 \r
1723         } while( pkt_cnt );\r
1724         cl_spinlock_release( &p_port->recv_lock );\r
1725 \r
1726         /*\r
1727          * Rearm after filtering to prevent contention on the enpoint maps\r
1728          * and eliminate the possibility of having a call to\r
1729          * __endpt_mgr_insert find a duplicate.\r
1730          */\r
1731         cl_perf_start( RearmRecv );\r
1732         status = p_port->p_adapter->p_ifc->rearm_cq(\r
1733                 p_port->ib_mgr.h_recv_cq, FALSE );\r
1734         cl_perf_stop( &p_port->p_adapter->perf, RearmRecv );\r
1735         CL_ASSERT( status == IB_SUCCESS );\r
1736 \r
1737         ipoib_port_deref( p_port, ref_recv_cb );\r
1738 \r
1739         cl_perf_stop( &p_port->p_adapter->perf, RecvCb );\r
1740 \r
1741         IPOIB_EXIT( IPOIB_DBG_RECV );\r
1742 }\r
1743 \r
1744 \r
1745 static void\r
1746 __recv_get_endpts(\r
1747         IN                              ipoib_port_t* const                     p_port,\r
1748         IN                              ipoib_recv_desc_t* const        p_desc,\r
1749         IN                              ib_wc_t* const                          p_wc,\r
1750                 OUT                     ipoib_endpt_t** const           pp_src,\r
1751                 OUT                     ipoib_endpt_t** const           pp_dst )\r
1752 {\r
1753         ib_api_status_t         status;\r
1754         mac_addr_t                      mac;\r
1755         PERF_DECLARE( GetEndptByGid );\r
1756         PERF_DECLARE( GetEndptByLid );\r
1757         PERF_DECLARE( EndptInsert );\r
1758 \r
1759         IPOIB_ENTER( IPOIB_DBG_RECV );\r
1760 \r
1761         /* Setup our shortcut pointers based on whether GRH is valid. */\r
1762         if( p_wc->recv.ud.recv_opt & IB_RECV_OPT_GRH_VALID )\r
1763         {\r
1764                 /* Lookup the source endpoints based on GID. */\r
1765                 cl_perf_start( GetEndptByGid );\r
1766                 *pp_src =\r
1767 #if IPOIB_INLINE_RECV\r
1768                         __endpt_mgr_get_by_gid( p_port, &p_desc->buf.ib.grh.src_gid );\r
1769 #else   /* IPOIB_INLINE_RECV */\r
1770                         __endpt_mgr_get_by_gid( p_port, &p_desc->p_buf->ib.grh.src_gid );\r
1771 #endif  /* IPOIB_INLINE_RECV */\r
1772                 cl_perf_stop( &p_port->p_adapter->perf, GetEndptByGid );\r
1773 \r
1774                 /*\r
1775                  * Lookup the destination endpoint based on GID.\r
1776                  * This is used along with the packet filter to determine\r
1777                  * whether to report this to NDIS.\r
1778                  */\r
1779                 cl_perf_start( GetEndptByGid );\r
1780                 *pp_dst =\r
1781 #if IPOIB_INLINE_RECV\r
1782                         __endpt_mgr_get_by_gid( p_port, &p_desc->buf.ib.grh.dest_gid );\r
1783 #else   /* IPOIB_INLINE_RECV */\r
1784                         __endpt_mgr_get_by_gid( p_port, &p_desc->p_buf->ib.grh.dest_gid );\r
1785 #endif  /* IPOIB_INLINE_RECV */\r
1786                 cl_perf_stop( &p_port->p_adapter->perf, GetEndptByGid );\r
1787 \r
1788                 /*\r
1789                  * Create the source endpoint if it does not exist.  Note that we\r
1790                  * can only do this for globally routed traffic since we need the\r
1791                  * information from the GRH to generate the MAC.\r
1792                  */\r
1793                 if( !*pp_src )\r
1794                 {\r
1795                         status = ipoib_mac_from_guid(\r
1796 #if IPOIB_INLINE_RECV\r
1797                                 p_desc->buf.ib.grh.src_gid.unicast.interface_id, &mac );\r
1798 #else   /* IPOIB_INLINE_RECV */\r
1799                                 p_desc->p_buf->ib.grh.src_gid.unicast.interface_id, &mac );\r
1800 #endif  /* IPOIB_INLINE_RECV */\r
1801                         if( status != IB_SUCCESS )\r
1802                         {\r
1803                                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1804                                         ("ipoib_mac_from_guid returned %s\n",\r
1805                                         p_port->p_adapter->p_ifc->get_err_str( status )) );\r
1806                                 return;\r
1807                         }\r
1808 \r
1809                         /* Create the endpoint. */\r
1810 #if IPOIB_INLINE_RECV\r
1811                         *pp_src = ipoib_endpt_create( &p_desc->buf.ib.grh.src_gid,\r
1812 #else   /* IPOIB_INLINE_RECV */\r
1813                         *pp_src = ipoib_endpt_create( &p_desc->p_buf->ib.grh.src_gid,\r
1814 #endif  /* IPOIB_INLINE_RECV */\r
1815                                 0, p_wc->recv.ud.remote_qp );\r
1816                         if( !*pp_src )\r
1817                         {\r
1818                                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1819                                         ("ipoib_endpt_create failed\n") );\r
1820                                 return;\r
1821                         }\r
1822                         cl_perf_start( EndptInsert );\r
1823                         cl_obj_lock( &p_port->obj );\r
1824                         status = __endpt_mgr_insert( p_port, mac, *pp_src );\r
1825                         if( status != IB_SUCCESS )\r
1826                         {\r
1827                                 cl_obj_unlock( &p_port->obj );\r
1828                                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1829                                         ("__endpt_mgr_insert returned %s\n",\r
1830                                         p_port->p_adapter->p_ifc->get_err_str( status )) );\r
1831                                 return;\r
1832                         }\r
1833                         cl_obj_unlock( &p_port->obj );\r
1834                         cl_perf_stop( &p_port->p_adapter->perf, EndptInsert );\r
1835                 }\r
1836         }\r
1837         else\r
1838         {\r
1839                 /*\r
1840                  * Lookup the remote endpoint based on LID.  Note that only\r
1841                  * unicast traffic can be LID routed.\r
1842                  */\r
1843                 cl_perf_start( GetEndptByLid );\r
1844                 *pp_src = __endpt_mgr_get_by_lid( p_port, p_wc->recv.ud.remote_lid );\r
1845                 cl_perf_stop( &p_port->p_adapter->perf, GetEndptByLid );\r
1846                 *pp_dst = p_port->p_local_endpt;\r
1847                 CL_ASSERT( *pp_dst );\r
1848         }\r
1849 \r
1850         if( *pp_src && !ipoib_is_voltaire_router_gid( &(*pp_src)->dgid ) &&\r
1851                 (*pp_src)->qpn != p_wc->recv.ud.remote_qp )\r
1852         {\r
1853                 /* Update the QPN for the endpoint. */\r
1854                 IPOIB_PRINT(TRACE_LEVEL_INFORMATION, IPOIB_DBG_RECV,\r
1855                         ("Updating QPN for MAC: %02X-%02X-%02X-%02X-%02X-%02X\n",\r
1856                         (*pp_src )->mac.addr[0], (*pp_src )->mac.addr[1],\r
1857                         (*pp_src )->mac.addr[2], (*pp_src )->mac.addr[3],\r
1858                         (*pp_src )->mac.addr[4], (*pp_src )->mac.addr[5]) );\r
1859                 (*pp_src)->qpn = p_wc->recv.ud.remote_qp;\r
1860         }\r
1861 \r
1862         if( *pp_src && *pp_dst )\r
1863         {\r
1864                 IPOIB_PRINT(TRACE_LEVEL_INFORMATION, IPOIB_DBG_RECV,\r
1865                         ("Recv:\n"\r
1866                         "\tsrc MAC: %02X-%02X-%02X-%02X-%02X-%02X\n"\r
1867                         "\tdst MAC: %02X-%02X-%02X-%02X-%02X-%02X\n",\r
1868                         (*pp_src )->mac.addr[0], (*pp_src )->mac.addr[1],\r
1869                         (*pp_src )->mac.addr[2], (*pp_src )->mac.addr[3],\r
1870                         (*pp_src )->mac.addr[4], (*pp_src )->mac.addr[5],\r
1871                         (*pp_dst )->mac.addr[0], (*pp_dst )->mac.addr[1],\r
1872                         (*pp_dst )->mac.addr[2], (*pp_dst )->mac.addr[3],\r
1873                         (*pp_dst )->mac.addr[4], (*pp_dst )->mac.addr[5]) );\r
1874         }\r
1875 \r
1876         IPOIB_EXIT( IPOIB_DBG_RECV );\r
1877 }\r
1878 \r
1879 \r
1880 static int32_t\r
1881 __recv_mgr_filter(\r
1882         IN                              ipoib_port_t* const                     p_port,\r
1883         IN                              ib_wc_t* const                          p_done_wc_list,\r
1884                 OUT                     cl_qlist_t* const                       p_done_list,\r
1885                 OUT                     cl_qlist_t* const                       p_bad_list )\r
1886 {\r
1887         ipoib_recv_desc_t               *p_desc;\r
1888         ib_wc_t                                 *p_wc;\r
1889         ipoib_pkt_t                             *p_ipoib;\r
1890         eth_pkt_t                               *p_eth;\r
1891         ipoib_endpt_t                   *p_src, *p_dst;\r
1892         ib_api_status_t                 status;\r
1893         uint32_t                                len;\r
1894         int32_t                                 recv_cnt = 0;\r
1895         PERF_DECLARE( GetRecvEndpts );\r
1896         PERF_DECLARE( RecvGen );\r
1897         PERF_DECLARE( RecvTcp );\r
1898         PERF_DECLARE( RecvUdp );\r
1899         PERF_DECLARE( RecvDhcp );\r
1900         PERF_DECLARE( RecvArp );\r
1901 \r
1902         IPOIB_ENTER( IPOIB_DBG_RECV );\r
1903 \r
1904         for( p_wc = p_done_wc_list; p_wc; p_wc = p_wc->p_next )\r
1905         {\r
1906                 CL_ASSERT( p_wc->status != IB_WCS_SUCCESS || p_wc->wc_type == IB_WC_RECV );\r
1907                 p_desc = (ipoib_recv_desc_t*)(uintn_t)p_wc->wr_id;\r
1908                 recv_cnt++;\r
1909 \r
1910                 if( p_wc->status != IB_WCS_SUCCESS )\r
1911                 {\r
1912                         if( p_wc->status != IB_WCS_WR_FLUSHED_ERR )\r
1913                         {\r
1914                                 IPOIB_PRINT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1915                                         ("Failed completion %s  (vendor specific %#x)\n",\r
1916                                         p_port->p_adapter->p_ifc->get_wc_status_str( p_wc->status ),\r
1917                                         (int)p_wc->vendor_specific) );\r
1918                                 ipoib_inc_recv_stat( p_port->p_adapter, IP_STAT_ERROR, 0 );\r
1919                         }\r
1920                         else\r
1921                         {\r
1922                                 IPOIB_PRINT(TRACE_LEVEL_INFORMATION, IPOIB_DBG_RECV,\r
1923                                         ("Flushed completion %s\n",\r
1924                                         p_port->p_adapter->p_ifc->get_wc_status_str( p_wc->status )) );\r
1925                                 ipoib_inc_recv_stat( p_port->p_adapter, IP_STAT_DROPPED, 0 );\r
1926                         }\r
1927                         cl_qlist_insert_tail( p_bad_list, &p_desc->item.list_item );\r
1928                         /* Dereference the port object on behalf of the failed receive. */\r
1929                         ipoib_port_deref( p_port, ref_failed_recv_wc );\r
1930                         continue;\r
1931                 }\r
1932 \r
1933                 len = p_wc->length - sizeof(ib_grh_t);\r
1934 \r
1935                 if( len < sizeof(ipoib_hdr_t) )\r
1936                 {\r
1937                         IPOIB_PRINT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1938                                 ("Received ETH packet < min size\n") );\r
1939                         ipoib_inc_recv_stat( p_port->p_adapter, IP_STAT_ERROR, 0 );\r
1940                         cl_qlist_insert_tail( p_bad_list, &p_desc->item.list_item );\r
1941                         ipoib_port_deref( p_port, ref_recv_inv_len );\r
1942                         continue;\r
1943                 }\r
1944 \r
1945                 if((len - sizeof(ipoib_hdr_t)) > p_port->p_adapter->params.payload_mtu)\r
1946                 {\r
1947                         IPOIB_PRINT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1948                                 ("Received ETH packet > payload MTU (%d)\n",\r
1949                                 p_port->p_adapter->params.payload_mtu) );\r
1950                         ipoib_inc_recv_stat( p_port->p_adapter, IP_STAT_ERROR, 0 );\r
1951                         cl_qlist_insert_tail( p_bad_list, &p_desc->item.list_item );\r
1952                         ipoib_port_deref( p_port, ref_recv_inv_len );\r
1953                         continue;\r
1954                         \r
1955                 }\r
1956                 /* Successful completion.  Get the receive information. */\r
1957                 p_desc->ndis_csum.Value = (ULONG) p_wc->csum_ok;\r
1958                 cl_perf_start( GetRecvEndpts );\r
1959                 __recv_get_endpts( p_port, p_desc, p_wc, &p_src, &p_dst );\r
1960                 cl_perf_stop( &p_port->p_adapter->perf, GetRecvEndpts );\r
1961 \r
1962 #if IPOIB_INLINE_RECV\r
1963                 p_ipoib = &p_desc->buf.ib.pkt;\r
1964                 p_eth = &p_desc->buf.eth.pkt;\r
1965 #else   /* IPOIB_INLINE_RECV */\r
1966                 p_ipoib = &p_desc->p_buf->ib.pkt;\r
1967                 p_eth = &p_desc->p_buf->eth.pkt;\r
1968 #endif  /*IPOIB_INLINE_RECV */\r
1969 \r
1970                 if( p_src )\r
1971                 {\r
1972                         /* Don't report loopback traffic - we requested SW loopback. */\r
1973                         if( !cl_memcmp( &p_port->p_adapter->params.conf_mac,\r
1974                                 &p_src->mac, sizeof(p_port->p_adapter->params.conf_mac) ) )\r
1975                         {\r
1976                                 /*\r
1977                                  * "This is not the packet you're looking for" - don't update\r
1978                                  * receive statistics, the packet never happened.\r
1979                                  */\r
1980                                 cl_qlist_insert_tail( p_bad_list, &p_desc->item.list_item );\r
1981                                 /* Dereference the port object on behalf of the failed recv. */\r
1982                                 ipoib_port_deref( p_port, ref_recv_loopback );\r
1983                                 continue;\r
1984                         }\r
1985                 }\r
1986 \r
1987                 switch( p_ipoib->hdr.type )\r
1988                 {\r
1989                 case ETH_PROT_TYPE_IP:\r
1990                         if( len < (sizeof(ipoib_hdr_t) + sizeof(ip_hdr_t)) )\r
1991                         {\r
1992                                 IPOIB_PRINT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1993                                         ("Received IP packet < min size\n") );\r
1994                                 status = IB_INVALID_SETTING;\r
1995                                 break;\r
1996                         }\r
1997 \r
1998                         if( p_ipoib->type.ip.hdr.offset ||\r
1999                                 p_ipoib->type.ip.hdr.prot != IP_PROT_UDP )\r
2000                         {\r
2001                                 /* Unfiltered.  Setup the ethernet header and report. */\r
2002                                 cl_perf_start( RecvTcp );\r
2003                                 status = __recv_gen( p_ipoib, p_eth, p_src, p_dst );\r
2004                                 cl_perf_stop( &p_port->p_adapter->perf, RecvTcp );\r
2005                                 break;\r
2006                         }\r
2007 \r
2008                         /* First packet of a UDP transfer. */\r
2009                         if( len <\r
2010                                 (sizeof(ipoib_hdr_t) + sizeof(ip_hdr_t) + sizeof(udp_hdr_t)) )\r
2011                         {\r
2012                                 IPOIB_PRINT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2013                                         ("Received UDP packet < min size\n") );\r
2014                                 status = IB_INVALID_SETTING;\r
2015                                 break;\r
2016                         }\r
2017 \r
2018                         /* Check if DHCP conversion is required. */\r
2019                         if( (p_ipoib->type.ip.prot.udp.hdr.dst_port == DHCP_PORT_SERVER &&\r
2020                                 p_ipoib->type.ip.prot.udp.hdr.src_port == DHCP_PORT_CLIENT) ||\r
2021                                 (p_ipoib->type.ip.prot.udp.hdr.dst_port == DHCP_PORT_CLIENT &&\r
2022                                 p_ipoib->type.ip.prot.udp.hdr.src_port == DHCP_PORT_SERVER) )\r
2023                         {\r
2024                                 if( len < (sizeof(ipoib_hdr_t) + sizeof(ip_hdr_t) +\r
2025                                         sizeof(udp_hdr_t) + DHCP_MIN_SIZE) )\r
2026                                 {\r
2027                                         IPOIB_PRINT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2028                                                 ("Received DHCP < min size\n") );\r
2029                                         status = IB_INVALID_SETTING;\r
2030                                         break;\r
2031                                 }\r
2032                                 if ((p_ipoib->type.ip.hdr.ver_hl & 0x0f) != 5 ) {\r
2033                                         // If there are IP options in this message, we are in trouble in any case\r
2034                                         status = IB_INVALID_SETTING;\r
2035                                         break;                                  \r
2036                                 }\r
2037                                 /* UDP packet with BOOTP ports in src/dst port numbers. */\r
2038                                 cl_perf_start( RecvDhcp );\r
2039                                 status = __recv_dhcp( p_port, p_ipoib, p_eth, p_src, p_dst );\r
2040                                 cl_perf_stop( &p_port->p_adapter->perf, RecvDhcp );\r
2041                         }\r
2042                         else\r
2043                         {\r
2044                                 /* Unfiltered.  Setup the ethernet header and report. */\r
2045                                 cl_perf_start( RecvUdp );\r
2046                                 status = __recv_gen( p_ipoib, p_eth, p_src, p_dst );\r
2047                                 cl_perf_stop( &p_port->p_adapter->perf, RecvUdp );\r
2048                         }\r
2049                         break;\r
2050 \r
2051                 case ETH_PROT_TYPE_ARP:\r
2052                         if( len < (sizeof(ipoib_hdr_t) + sizeof(ipoib_arp_pkt_t)) )\r
2053                         {\r
2054                                 IPOIB_PRINT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2055                                         ("Received ARP < min size\n") );\r
2056                                 status = IB_INVALID_SETTING;\r
2057                                 break;\r
2058                         }\r
2059                         cl_perf_start( RecvArp );\r
2060                         status = __recv_arp( p_port, p_wc, p_ipoib, p_eth, &p_src, p_dst );\r
2061                         cl_perf_stop( &p_port->p_adapter->perf, RecvArp );\r
2062                         len = sizeof(ipoib_hdr_t) + sizeof(arp_pkt_t);\r
2063                         break;\r
2064 \r
2065                 default:\r
2066                         /* Unfiltered.  Setup the ethernet header and report. */\r
2067                         cl_perf_start( RecvGen );\r
2068                         status = __recv_gen( p_ipoib, p_eth, p_src, p_dst );\r
2069                         cl_perf_stop( &p_port->p_adapter->perf, RecvGen );\r
2070                 }\r
2071 \r
2072                 if( status != IB_SUCCESS )\r
2073                 {\r
2074                         /* Update stats. */\r
2075                         ipoib_inc_recv_stat( p_port->p_adapter, IP_STAT_ERROR, 0 );\r
2076                         cl_qlist_insert_tail( p_bad_list, &p_desc->item.list_item );\r
2077                         /* Dereference the port object on behalf of the failed receive. */\r
2078                         ipoib_port_deref( p_port, ref_recv_filter );\r
2079                 }\r
2080                 else\r
2081                 {\r
2082                         p_desc->len =\r
2083                                 len + sizeof(eth_hdr_t) - sizeof(ipoib_hdr_t);\r
2084                         if( p_dst->h_mcast)\r
2085                         {\r
2086                                 if( p_dst->dgid.multicast.raw_group_id[10] == 0xFF &&\r
2087                                         p_dst->dgid.multicast.raw_group_id[11] == 0xFF &&\r
2088                                         p_dst->dgid.multicast.raw_group_id[12] == 0xFF &&\r
2089                                         p_dst->dgid.multicast.raw_group_id[13] == 0xFF )\r
2090                                 {\r
2091                                         p_desc->type = PKT_TYPE_BCAST;\r
2092                                 }\r
2093                                 else\r
2094                                 {\r
2095                                         p_desc->type = PKT_TYPE_MCAST;\r
2096                                 }\r
2097                         }\r
2098                         else\r
2099                         {\r
2100                                 p_desc->type = PKT_TYPE_UCAST;\r
2101                         }\r
2102                         cl_qlist_insert_tail( p_done_list, &p_desc->item.list_item );\r
2103                 }\r
2104         }\r
2105 \r
2106         IPOIB_EXIT( IPOIB_DBG_RECV );\r
2107         return recv_cnt;\r
2108 }\r
2109 \r
2110 \r
2111 static ib_api_status_t\r
2112 __recv_gen(\r
2113         IN              const   ipoib_pkt_t* const                      p_ipoib,\r
2114                 OUT                     eth_pkt_t* const                        p_eth,\r
2115         IN                              ipoib_endpt_t* const            p_src,\r
2116         IN                              ipoib_endpt_t* const            p_dst )\r
2117 {\r
2118         IPOIB_ENTER( IPOIB_DBG_RECV );\r
2119 \r
2120         if( !p_src || !p_dst )\r
2121         {\r
2122                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2123                         ("Received packet with no matching endpoints.\n") );\r
2124                 return IB_NOT_DONE;\r
2125         }\r
2126 \r
2127         /*\r
2128          * Fill in the ethernet header.  Note that doing so will overwrite\r
2129          * the IPoIB header, so start by moving the information from the IPoIB\r
2130          * header.\r
2131          */\r
2132         p_eth->hdr.type = p_ipoib->hdr.type;\r
2133         p_eth->hdr.src = p_src->mac;\r
2134         p_eth->hdr.dst = p_dst->mac;\r
2135 \r
2136         IPOIB_EXIT( IPOIB_DBG_RECV );\r
2137         return IB_SUCCESS;\r
2138 }\r
2139 \r
2140 \r
2141 static ib_api_status_t\r
2142 __recv_dhcp(\r
2143         IN                              ipoib_port_t* const                     p_port,\r
2144         IN              const   ipoib_pkt_t* const                      p_ipoib,\r
2145                 OUT                     eth_pkt_t* const                        p_eth,\r
2146         IN                              ipoib_endpt_t* const            p_src,\r
2147         IN                              ipoib_endpt_t* const            p_dst )\r
2148 {\r
2149         ib_api_status_t         status;\r
2150         dhcp_pkt_t                      *p_dhcp;\r
2151         uint8_t                         *p_option;\r
2152         uint8_t                         *p_cid = NULL;\r
2153         ib_gid_t                        gid;\r
2154         uint8_t                         msg = 0;\r
2155 \r
2156         IPOIB_ENTER( IPOIB_DBG_RECV );\r
2157 \r
2158         UNUSED_PARAM( p_port );\r
2159 \r
2160         /* Create the ethernet header. */\r
2161         status = __recv_gen( p_ipoib, p_eth, p_src, p_dst );\r
2162         if( status != IB_SUCCESS )\r
2163         {\r
2164                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2165                         ("__recv_gen returned %s.\n", \r
2166                         p_port->p_adapter->p_ifc->get_err_str( status )) );\r
2167                 return status;\r
2168         }\r
2169 \r
2170         /* Fixup the payload. */\r
2171         p_dhcp = &p_eth->type.ip.prot.udp.dhcp;\r
2172         if( p_dhcp->op != DHCP_REQUEST && p_dhcp->op != DHCP_REPLY )\r
2173         {\r
2174                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2175                         ("Invalid DHCP op code.\n") );\r
2176                 return IB_INVALID_SETTING;\r
2177         }\r
2178 \r
2179         /*\r
2180          * Find the client identifier option, making sure to skip\r
2181          * the "magic cookie".\r
2182          */\r
2183         p_option = &p_dhcp->options[0];\r
2184         if ( *(uint32_t *)p_option != DHCP_COOKIE )\r
2185         {\r
2186                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2187                         ("DHCP cookie corrupted.\n") );\r
2188                 return IB_INVALID_PARAMETER;\r
2189         }\r
2190 \r
2191         p_option = &p_dhcp->options[4];\r
2192         while( *p_option != DHCP_OPT_END && p_option < &p_dhcp->options[312] )\r
2193         {\r
2194                 switch( *p_option )\r
2195                 {\r
2196                 case DHCP_OPT_PAD:\r
2197                         p_option++;\r
2198                         break;\r
2199 \r
2200                 case DHCP_OPT_MSG:\r
2201                         msg = p_option[2];\r
2202                         p_option += 3;\r
2203                         break;\r
2204 \r
2205                 case DHCP_OPT_CLIENT_ID:\r
2206                         p_cid = p_option;\r
2207                         /* Fall through. */\r
2208 \r
2209                 default:\r
2210                         /*\r
2211                          * All other options have a length byte following the option code.\r
2212                          * Offset by the length to get to the next option.\r
2213                          */\r
2214                         p_option += (p_option[1] + 2);\r
2215                 }\r
2216         }\r
2217 \r
2218         switch( msg )\r
2219         {\r
2220         /* message from client */\r
2221         case DHCPDISCOVER:\r
2222         case DHCPREQUEST:\r
2223         case DHCPDECLINE:\r
2224         case DHCPRELEASE:\r
2225         case DHCPINFORM:\r
2226                 if( !p_cid )\r
2227                 {\r
2228                         IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2229                                 ("Failed to find required Client-identifier option.\n") );\r
2230                         return IB_INVALID_SETTING;\r
2231                 }\r
2232                 if( p_dhcp->htype != DHCP_HW_TYPE_IB )\r
2233                 {\r
2234                         IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2235                                 ("Invalid hardware address type.\n") );\r
2236                         return IB_INVALID_SETTING;\r
2237                 }\r
2238                 break;\r
2239         /* message from DHCP server */\r
2240         case DHCPOFFER:\r
2241         case DHCPACK:\r
2242         case DHCPNAK:\r
2243                 break;\r
2244 \r
2245         default:\r
2246                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2247                         ("Invalide message type.\n") );\r
2248                 return IB_INVALID_PARAMETER;\r
2249         }\r
2250         p_eth->type.ip.prot.udp.hdr.chksum = 0;\r
2251         p_dhcp->htype = DHCP_HW_TYPE_ETH;\r
2252         p_dhcp->hlen = HW_ADDR_LEN;\r
2253 \r
2254         if( p_cid ) /* from client */\r
2255         {\r
2256                 /* Validate that the length and type of the option is as required. */\r
2257                 if( p_cid[1] != 21 )\r
2258                 {\r
2259                         IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2260                                 ("Client-identifier length not 21 as required.\n") );\r
2261                         return IB_INVALID_SETTING;\r
2262                 }\r
2263                 if( p_cid[2] != DHCP_HW_TYPE_IB )\r
2264                 {\r
2265                         IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2266                                 ("Client-identifier type is wrong.\n") );\r
2267                         return IB_INVALID_SETTING;\r
2268                 }\r
2269                 /*\r
2270                  * Copy the GID value from the option so that we can make aligned\r
2271                  * accesses to the contents.\r
2272                  * Recover CID to standard type.\r
2273                  */\r
2274                 cl_memcpy( &gid, &p_cid[7], sizeof(ib_gid_t) );\r
2275                 p_cid[1] =  HW_ADDR_LEN +1;// CID length \r
2276                 p_cid[2] =  DHCP_HW_TYPE_ETH;// CID type \r
2277                 status = ipoib_mac_from_guid( gid.unicast.interface_id, (mac_addr_t*)&p_cid[3] );\r
2278                 p_cid[HW_ADDR_LEN + 3] = DHCP_OPT_END; //terminate tag\r
2279         }\r
2280         IPOIB_EXIT( IPOIB_DBG_RECV );\r
2281         return status;\r
2282 }\r
2283 \r
2284 \r
2285 static ib_api_status_t\r
2286 __recv_arp(\r
2287         IN                              ipoib_port_t* const                     p_port,\r
2288         IN                              ib_wc_t* const                          p_wc,\r
2289         IN              const   ipoib_pkt_t* const                      p_ipoib,\r
2290                 OUT                     eth_pkt_t* const                        p_eth,\r
2291         IN                              ipoib_endpt_t** const           pp_src,\r
2292         IN                              ipoib_endpt_t* const            p_dst )\r
2293 {\r
2294         ib_api_status_t                 status;\r
2295         arp_pkt_t                               *p_arp;\r
2296         const ipoib_arp_pkt_t   *p_ib_arp;\r
2297         ib_gid_t                                gid;\r
2298         mac_addr_t                              mac;\r
2299         ipoib_hw_addr_t                 null_hw = {0};\r
2300 \r
2301         IPOIB_ENTER( IPOIB_DBG_RECV );\r
2302 \r
2303         if( !p_dst )\r
2304         {\r
2305                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2306                         ("Unknown destination endpoint\n") );\r
2307                 return IB_INVALID_SETTING;\r
2308         }\r
2309 \r
2310         p_ib_arp = &p_ipoib->type.arp;\r
2311         p_arp = &p_eth->type.arp;\r
2312 \r
2313         if( p_ib_arp->hw_type != ARP_HW_TYPE_IB )\r
2314         {\r
2315                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2316                         ("ARP hardware type is not IB\n") );\r
2317                 return IB_INVALID_SETTING;\r
2318         }\r
2319 \r
2320         if( p_ib_arp->hw_size != sizeof(ipoib_hw_addr_t) )\r
2321         {\r
2322                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2323                         ("ARP hardware address size is not sizeof(ipoib_hw_addr_t)\n") );\r
2324                 return IB_INVALID_SETTING;\r
2325         }\r
2326 \r
2327         if( p_ib_arp->prot_type != ETH_PROT_TYPE_IP )\r
2328         {\r
2329                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2330                         ("ARP protocal type not IP\n") );\r
2331                 return IB_INVALID_SETTING;\r
2332         }\r
2333 \r
2334         /*\r
2335          * If we don't have a source, lookup the endpoint specified in the payload.\r
2336          */\r
2337         if( !*pp_src )\r
2338                 *pp_src = __endpt_mgr_get_by_gid( p_port, &p_ib_arp->src_hw.gid );\r
2339 \r
2340         /*\r
2341          * If the endpoint exists for the GID, make sure\r
2342          * the dlid and qpn match the arp.\r
2343          */\r
2344         if( *pp_src )\r
2345         {\r
2346                 if( cl_memcmp( &(*pp_src)->dgid, &p_ib_arp->src_hw.gid,\r
2347                         sizeof(ib_gid_t) ) )\r
2348                 {\r
2349                         /*\r
2350                          * GIDs for the endpoint are different.  The ARP must\r
2351                          * have been proxied.  Dereference it.\r
2352                          */\r
2353                         *pp_src = NULL;\r
2354                 }\r
2355                 else if( (*pp_src)->dlid &&\r
2356                         (*pp_src)->dlid != p_wc->recv.ud.remote_lid )\r
2357                 {\r
2358                         /* Out of date!  Destroy the endpoint and replace it. */\r
2359                         __endpt_mgr_remove( p_port, *pp_src );\r
2360                         *pp_src = NULL;\r
2361                 }\r
2362                 else if( ipoib_is_voltaire_router_gid( &(*pp_src)->dgid ) )\r
2363                 {\r
2364                         if( (*pp_src)->qpn !=\r
2365                                 (p_ib_arp->src_hw.flags_qpn & CL_HTON32(0x00FFFFFF)) &&\r
2366                                 p_wc->recv.ud.remote_qp !=\r
2367                                 (p_ib_arp->src_hw.flags_qpn & CL_HTON32(0x00FFFFFF)) )\r
2368                         {\r
2369                                 /* Out of date!  Destroy the endpoint and replace it. */\r
2370                                 __endpt_mgr_remove( p_port, *pp_src );\r
2371                                 *pp_src = NULL;\r
2372                         }\r
2373                 }\r
2374                 else if( (*pp_src)->qpn != p_wc->recv.ud.remote_qp )\r
2375                 {\r
2376                         /* Out of date!  Destroy the endpoint and replace it. */\r
2377                         __endpt_mgr_remove( p_port, *pp_src );\r
2378                         *pp_src = NULL;\r
2379                 }\r
2380         }\r
2381 \r
2382         /* Do we need to create an endpoint for this GID? */\r
2383         if( !*pp_src )\r
2384         {\r
2385                 /* Copy the src GID to allow aligned access */\r
2386                 cl_memcpy( &gid, &p_ib_arp->src_hw.gid, sizeof(ib_gid_t) );\r
2387                 status = ipoib_mac_from_guid( gid.unicast.interface_id, &mac );\r
2388                 if( status != IB_SUCCESS )\r
2389                 {\r
2390                         IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2391                                 ("ipoib_mac_from_guid returned %s\n",\r
2392                                 p_port->p_adapter->p_ifc->get_err_str( status )) );\r
2393                         return status;\r
2394                 }\r
2395                 /*\r
2396                  * Create the endpoint.  Note that the LID is left blank and will be\r
2397                  * resolved by a path query as needed.  This is done because the\r
2398                  * remote LID/GID from the work completion may not be the original\r
2399                  * initiator.\r
2400                  */\r
2401                 *pp_src = ipoib_endpt_create( &p_ib_arp->src_hw.gid,\r
2402                         0, (p_ib_arp->src_hw.flags_qpn & CL_HTON32(0x00FFFFFF)) );\r
2403 \r
2404                 if( !*pp_src )\r
2405                 {\r
2406                         IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2407                                 ("ipoib_endpt_create failed\n") );\r
2408                         return status;\r
2409                 }\r
2410 \r
2411                 cl_obj_lock( &p_port->obj );\r
2412                 status = __endpt_mgr_insert( p_port, mac, *pp_src );\r
2413                 if( status != IB_SUCCESS )\r
2414                 {\r
2415                         cl_obj_unlock( &p_port->obj );\r
2416                         IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2417                                 ("__endpt_mgr_insert return %s \n",\r
2418                                 p_port->p_adapter->p_ifc->get_err_str( status )) );\r
2419                         return status;\r
2420                 }\r
2421 \r
2422                 cl_obj_unlock( &p_port->obj );\r
2423         }\r
2424 \r
2425         CL_ASSERT( !cl_memcmp(\r
2426                 &(*pp_src)->dgid, &p_ib_arp->src_hw.gid, sizeof(ib_gid_t) ) );\r
2427         CL_ASSERT( ipoib_is_voltaire_router_gid( &(*pp_src)->dgid ) ||\r
2428                 (*pp_src)->qpn ==\r
2429                 (p_ib_arp->src_hw.flags_qpn & CL_HTON32(0x00FFFFFF)) );\r
2430         /* Now swizzle the data. */\r
2431         p_arp->hw_type = ARP_HW_TYPE_ETH;\r
2432         p_arp->hw_size = sizeof(mac_addr_t);\r
2433         p_arp->src_hw = (*pp_src)->mac;\r
2434         p_arp->src_ip = p_ib_arp->src_ip;\r
2435 \r
2436         if( cl_memcmp( &p_ib_arp->dst_hw, &null_hw, sizeof(ipoib_hw_addr_t) ) )\r
2437         {\r
2438                 if( cl_memcmp( &p_dst->dgid, &p_ib_arp->dst_hw.gid, sizeof(ib_gid_t) ) )\r
2439                 {\r
2440                         /*\r
2441                          * We received bcast ARP packet that means\r
2442                          * remote port lets everyone know it was changed IP/MAC\r
2443                          * or just activated\r
2444                          */\r
2445 \r
2446                         /* Guy: TODO: Check why this check fails in case of Voltaire IPR */\r
2447 \r
2448                         if ( !ipoib_is_voltaire_router_gid( &(*pp_src)->dgid ) &&\r
2449                                  !ib_gid_is_multicast( (const ib_gid_t*)&p_dst->dgid ) )\r
2450                         {\r
2451                                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2452                                         ("ARP: is not ARP MCAST\n") );\r
2453                                 return IB_INVALID_SETTING;\r
2454                         }\r
2455 \r
2456                         p_arp->dst_hw = p_port->p_local_endpt->mac;\r
2457                         p_dst->mac = p_port->p_local_endpt->mac;\r
2458                         /*\r
2459                          * we don't care what receiver ip addr is,\r
2460                          * as long as OS' ARP table is global  ???\r
2461                          */\r
2462                         p_arp->dst_ip = (net32_t)0;\r
2463                 }\r
2464                 else /* we've got reply to our ARP request */\r
2465                 {\r
2466                         p_arp->dst_hw = p_dst->mac;\r
2467                         p_arp->dst_ip = p_ib_arp->dst_ip;\r
2468                         CL_ASSERT( p_dst->qpn == \r
2469                                 (p_ib_arp->dst_hw.flags_qpn & CL_HTON32(0x00FFFFFF)) );\r
2470                 }\r
2471         }\r
2472         else /* we got ARP reqeust */\r
2473         {\r
2474                 cl_memclr( &p_arp->dst_hw, sizeof(mac_addr_t) );\r
2475                 p_arp->dst_ip = p_ib_arp->dst_ip;\r
2476         }\r
2477 \r
2478         /*\r
2479          * Create the ethernet header.  Note that this is done last so that\r
2480          * we have a chance to create a new endpoint.\r
2481          */\r
2482         status = __recv_gen( p_ipoib, p_eth, *pp_src, p_dst );\r
2483         if( status != IB_SUCCESS )\r
2484         {\r
2485                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2486                         ("__recv_gen returned %s.\n", \r
2487                         p_port->p_adapter->p_ifc->get_err_str( status )) );\r
2488                 return status;\r
2489         }\r
2490 \r
2491         IPOIB_EXIT( IPOIB_DBG_RECV );\r
2492         return IB_SUCCESS;\r
2493 }\r
2494 \r
2495 \r
2496 static ib_api_status_t\r
2497 __recv_mgr_prepare_pkt(\r
2498         IN                              ipoib_port_t* const                     p_port,\r
2499         IN                              ipoib_recv_desc_t* const        p_desc,\r
2500                 OUT                     NDIS_PACKET** const                     pp_packet )\r
2501 {\r
2502         NDIS_STATUS                                                     status;\r
2503         uint32_t                                                        pkt_filter;\r
2504         ip_stat_sel_t                                           type;\r
2505         PERF_DECLARE( GetNdisPkt );\r
2506 \r
2507         IPOIB_ENTER( IPOIB_DBG_RECV );\r
2508 \r
2509         pkt_filter = p_port->p_adapter->packet_filter;\r
2510         /* Check the packet filter. */\r
2511         switch( p_desc->type )\r
2512         {\r
2513         default:\r
2514         case PKT_TYPE_UCAST:\r
2515                 if( pkt_filter & NDIS_PACKET_TYPE_PROMISCUOUS ||\r
2516                         pkt_filter & NDIS_PACKET_TYPE_ALL_FUNCTIONAL ||\r
2517                         pkt_filter & NDIS_PACKET_TYPE_SOURCE_ROUTING ||\r
2518                         pkt_filter & NDIS_PACKET_TYPE_DIRECTED )\r
2519                 {\r
2520                         /* OK to report. */\r
2521                         type = IP_STAT_UCAST_BYTES;\r
2522                         status = NDIS_STATUS_SUCCESS;\r
2523                 }\r
2524                 else\r
2525                 {\r
2526                         type = IP_STAT_DROPPED;\r
2527                         status = NDIS_STATUS_FAILURE;\r
2528                 }\r
2529                 break;\r
2530         case PKT_TYPE_BCAST:\r
2531                 if( pkt_filter & NDIS_PACKET_TYPE_PROMISCUOUS ||\r
2532                         pkt_filter & NDIS_PACKET_TYPE_BROADCAST )\r
2533                 {\r
2534                         /* OK to report. */\r
2535                         type = IP_STAT_BCAST_BYTES;\r
2536                         status = NDIS_STATUS_SUCCESS;\r
2537                 }\r
2538                 else\r
2539                 {\r
2540                         type = IP_STAT_DROPPED;\r
2541                         status = NDIS_STATUS_FAILURE;\r
2542                 }\r
2543                 break;\r
2544         case PKT_TYPE_MCAST:\r
2545                 if( pkt_filter & NDIS_PACKET_TYPE_PROMISCUOUS ||\r
2546                         pkt_filter & NDIS_PACKET_TYPE_ALL_MULTICAST ||\r
2547                         pkt_filter & NDIS_PACKET_TYPE_MULTICAST )\r
2548                 {\r
2549                         /* OK to report. */\r
2550                         type = IP_STAT_MCAST_BYTES;\r
2551                         status = NDIS_STATUS_SUCCESS;\r
2552                 }\r
2553                 else\r
2554                 {\r
2555                         type = IP_STAT_DROPPED;\r
2556                         status = NDIS_STATUS_FAILURE;\r
2557                 }\r
2558                 break;\r
2559         }\r
2560 \r
2561         if( status != NDIS_STATUS_SUCCESS )\r
2562         {\r
2563                 ipoib_inc_recv_stat( p_port->p_adapter, type, 0 );\r
2564                 /* Return the receive descriptor to the pool. */\r
2565                 __buf_mgr_put_recv( p_port, p_desc, NULL );\r
2566                 IPOIB_PRINT_EXIT( TRACE_LEVEL_INFORMATION, IPOIB_DBG_RECV,\r
2567                         ("Packet filter doesn't match receive.  Dropping.\n") );\r
2568                 /*\r
2569                  * Return IB_NOT_DONE since the packet has been completed,\r
2570                  * but has not consumed an array entry.\r
2571                  */\r
2572                 return IB_NOT_DONE;\r
2573         }\r
2574 \r
2575         cl_perf_start( GetNdisPkt );\r
2576         *pp_packet = __buf_mgr_get_ndis_pkt( p_port, p_desc );\r
2577         cl_perf_stop( &p_port->p_adapter->perf, GetNdisPkt );\r
2578         if( !*pp_packet )\r
2579         {\r
2580                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2581                         ("__buf_mgr_get_ndis_pkt failed\n") );\r
2582                 return IB_INSUFFICIENT_RESOURCES;\r
2583         }\r
2584 \r
2585         /* Get the checksums directly from packet information. */\r
2586         NDIS_PER_PACKET_INFO_FROM_PACKET( *pp_packet, TcpIpChecksumPacketInfo ) = \r
2587                 (PVOID) (uintn_t) (p_desc->ndis_csum.Value);\r
2588         ipoib_inc_recv_stat( p_port->p_adapter, type, p_desc->len );\r
2589 \r
2590         IPOIB_EXIT( IPOIB_DBG_RECV );\r
2591         return IB_SUCCESS;\r
2592 }\r
2593 \r
2594 \r
2595 static uint32_t\r
2596 __recv_mgr_build_pkt_array(\r
2597         IN                              ipoib_port_t* const                     p_port,\r
2598         IN                              int32_t                                         shortage,\r
2599                 OUT                     cl_qlist_t* const                       p_done_list,\r
2600                 OUT                     int32_t* const                          p_discarded )\r
2601 {\r
2602         cl_list_item_t                  *p_item;\r
2603         ipoib_recv_desc_t               *p_desc;\r
2604         uint32_t                                i = 0;\r
2605         ib_api_status_t                 status;\r
2606         PERF_DECLARE( PreparePkt );\r
2607 \r
2608         IPOIB_ENTER( IPOIB_DBG_RECV );\r
2609 \r
2610         *p_discarded = 0;\r
2611 \r
2612         /* Move any existing receives to the head to preserve ordering. */\r
2613         cl_qlist_insert_list_head( p_done_list, &p_port->recv_mgr.done_list );\r
2614         p_item = cl_qlist_remove_head( p_done_list );\r
2615         while( p_item != cl_qlist_end( p_done_list ) )\r
2616         {\r
2617                 p_desc = (ipoib_recv_desc_t*)p_item;\r
2618 \r
2619                 cl_perf_start( PreparePkt );\r
2620                 status = __recv_mgr_prepare_pkt( p_port, p_desc,\r
2621                         &p_port->recv_mgr.recv_pkt_array[i] );\r
2622                 cl_perf_stop( &p_port->p_adapter->perf, PreparePkt );\r
2623                 if( status == IB_SUCCESS )\r
2624                 {\r
2625                         CL_ASSERT( p_port->recv_mgr.recv_pkt_array[i] );\r
2626                         if( shortage-- > 0 )\r
2627                         {\r
2628                                 NDIS_SET_PACKET_STATUS(\r
2629                                         p_port->recv_mgr.recv_pkt_array[i], NDIS_STATUS_RESOURCES );\r
2630                         }\r
2631                         else\r
2632                         {\r
2633                                 NDIS_SET_PACKET_STATUS(\r
2634                                         p_port->recv_mgr.recv_pkt_array[i], NDIS_STATUS_SUCCESS );\r
2635                         }\r
2636                         i++;\r
2637                 }\r
2638                 else if( status == IB_NOT_DONE )\r
2639                 {\r
2640                         (*p_discarded)++;\r
2641                 }\r
2642                 else\r
2643                 {\r
2644                         IPOIB_PRINT(TRACE_LEVEL_INFORMATION, IPOIB_DBG_RECV,\r
2645                                 ("__recv_mgr_prepare_pkt returned %s\n",\r
2646                                 p_port->p_adapter->p_ifc->get_err_str( status )) );\r
2647                         /* Put all completed receives on the port's done list. */\r
2648                         cl_qlist_insert_tail( &p_port->recv_mgr.done_list, p_item );\r
2649                         cl_qlist_insert_list_tail( &p_port->recv_mgr.done_list, p_done_list );\r
2650                         break;\r
2651                 }\r
2652 \r
2653                 p_item = cl_qlist_remove_head( p_done_list );\r
2654         }\r
2655 \r
2656         IPOIB_EXIT( IPOIB_DBG_RECV );\r
2657         return i;\r
2658 }\r
2659 \r
2660 \r
2661 \r
2662 \r
2663 /******************************************************************************\r
2664 *\r
2665 * Send manager implementation.\r
2666 *\r
2667 ******************************************************************************/\r
2668 static void\r
2669 __send_mgr_construct(\r
2670         IN                              ipoib_port_t* const                     p_port )\r
2671 {\r
2672         IPOIB_ENTER( IPOIB_DBG_SEND );\r
2673         p_port->send_mgr.depth = 0;\r
2674         cl_qlist_init( &p_port->send_mgr.pending_list );\r
2675         IPOIB_EXIT( IPOIB_DBG_SEND );\r
2676 }\r
2677 \r
2678 \r
2679 static void \r
2680 __pending_list_destroy(\r
2681         IN                              ipoib_port_t* const                     p_port )\r
2682 {\r
2683         cl_list_item_t  *p_item;\r
2684         NDIS_PACKET             *p_packet;\r
2685         \r
2686         cl_spinlock_acquire( &p_port->send_lock );\r
2687         /* Complete any pending packets. */\r
2688         for( p_item = cl_qlist_remove_head( &p_port->send_mgr.pending_list );\r
2689                 p_item != cl_qlist_end( &p_port->send_mgr.pending_list );\r
2690                 p_item = cl_qlist_remove_head( &p_port->send_mgr.pending_list ) )\r
2691         {\r
2692                 p_packet = IPOIB_PACKET_FROM_LIST_ITEM( p_item );\r
2693                 NdisMSendComplete( p_port->p_adapter->h_adapter, p_packet,\r
2694                         NDIS_STATUS_RESET_IN_PROGRESS );\r
2695         }\r
2696         cl_spinlock_release( &p_port->send_lock );\r
2697 }\r
2698 \r
2699 static void\r
2700 __send_mgr_destroy(\r
2701         IN                              ipoib_port_t* const                     p_port )\r
2702 {\r
2703         IPOIB_ENTER( IPOIB_DBG_SEND );\r
2704         __pending_list_destroy(p_port);\r
2705 \r
2706         IPOIB_EXIT( IPOIB_DBG_SEND );\r
2707 }\r
2708 \r
2709 \r
2710 static NDIS_STATUS\r
2711 __send_mgr_filter(\r
2712         IN                              ipoib_port_t* const                     p_port,\r
2713         IN              const   eth_hdr_t* const                        p_eth_hdr,\r
2714         IN                              NDIS_BUFFER* const                      p_buf,\r
2715         IN                              size_t                                          buf_len,\r
2716         IN      OUT                     ipoib_send_desc_t* const        p_desc )\r
2717 {\r
2718         NDIS_STATUS             status;\r
2719 \r
2720         PERF_DECLARE( FilterIp );\r
2721         PERF_DECLARE( FilterArp );\r
2722         PERF_DECLARE( SendGen );\r
2723 \r
2724         IPOIB_ENTER( IPOIB_DBG_SEND );\r
2725 \r
2726         /*\r
2727          * We already checked the ethernet header length, so we know it's safe\r
2728          * to decrement the buf_len without underflowing.\r
2729          */\r
2730         buf_len -= sizeof(eth_hdr_t);\r
2731 \r
2732         switch( p_eth_hdr->type )\r
2733         {\r
2734         case ETH_PROT_TYPE_IP:\r
2735                 cl_perf_start( FilterIp );\r
2736                 status = __send_mgr_filter_ip(\r
2737                         p_port, p_eth_hdr, p_buf, buf_len, p_desc );\r
2738                 cl_perf_stop( &p_port->p_adapter->perf, FilterIp );\r
2739                 break;\r
2740 \r
2741         case ETH_PROT_TYPE_ARP:\r
2742                 cl_perf_start( FilterArp );\r
2743                 status = __send_mgr_filter_arp(\r
2744                         p_port, p_eth_hdr, p_buf, buf_len, p_desc );\r
2745                 cl_perf_stop( &p_port->p_adapter->perf, FilterArp );\r
2746                 break;\r
2747 \r
2748         default:\r
2749                 /*\r
2750                  * The IPoIB spec doesn't define how to send non IP or ARP packets.\r
2751                  * Just send the payload and hope for the best.\r
2752                  */\r
2753                 cl_perf_start( SendGen );\r
2754                 status = __send_gen( p_port, p_desc );\r
2755                 cl_perf_stop( &p_port->p_adapter->perf, SendGen );\r
2756                 break;\r
2757         }\r
2758 \r
2759         IPOIB_EXIT( IPOIB_DBG_SEND );\r
2760         return status;\r
2761 }\r
2762 \r
2763 \r
2764 static NDIS_STATUS\r
2765 __send_copy(\r
2766         IN                              ipoib_port_t* const                     p_port,\r
2767         IN                              ipoib_send_desc_t* const        p_desc )\r
2768 {\r
2769         NDIS_PACKET                             *p_packet;\r
2770         NDIS_BUFFER                             *p_buf;\r
2771         NDIS_STATUS                             status;\r
2772         UINT                                    tot_len, bytes_copied = 0;\r
2773 \r
2774         IPOIB_ENTER( IPOIB_DBG_SEND );\r
2775 \r
2776         p_desc->p_buf = \r
2777                 ExAllocateFromNPagedLookasideList( &p_port->buf_mgr.send_buf_list );\r
2778         if( !p_desc->p_buf )\r
2779         {\r
2780                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2781                         ("Failed to allocate buffer for packet copy.\n") );\r
2782                 return NDIS_STATUS_RESOURCES;\r
2783         }\r
2784 \r
2785         NdisAllocatePacket( &status, &p_packet, p_port->buf_mgr.h_send_pkt_pool );\r
2786         if( status != NDIS_STATUS_SUCCESS )\r
2787         {\r
2788                 IPOIB_PRINT_EXIT( TRACE_LEVEL_WARNING, IPOIB_DBG_SEND,\r
2789                         ("Failed to allocate NDIS_PACKET for copy.\n") );\r
2790                 return status;\r
2791         }\r
2792 \r
2793         NdisAllocateBuffer( &status, &p_buf, p_port->buf_mgr.h_send_buf_pool,\r
2794                 p_desc->p_buf, p_port->p_adapter->params.xfer_block_size );\r
2795         if( status != NDIS_STATUS_SUCCESS )\r
2796         {\r
2797                 NdisFreePacket( p_packet );\r
2798                 IPOIB_PRINT_EXIT( TRACE_LEVEL_WARNING, IPOIB_DBG_SEND,\r
2799                         ("Failed to allocate NDIS_BUFFER for copy.\n") );\r
2800                 return status;\r
2801         }\r
2802 \r
2803         NdisChainBufferAtFront( p_packet, p_buf );\r
2804 \r
2805         NdisQueryPacketLength( p_desc->p_pkt, &tot_len );\r
2806 \r
2807         /* Setup the work request. */\r
2808         p_desc->local_ds[1].vaddr = cl_get_physaddr(\r
2809                 ((uint8_t*)p_desc->p_buf) + sizeof(eth_hdr_t) );\r
2810         p_desc->local_ds[1].length = tot_len - sizeof(eth_hdr_t);\r
2811         p_desc->local_ds[1].lkey = p_port->ib_mgr.lkey;\r
2812         p_desc->wr.num_ds = 2;\r
2813 \r
2814         /* Copy the packet. */\r
2815         NdisCopyFromPacketToPacketSafe( p_packet, bytes_copied, tot_len,\r
2816                 p_desc->p_pkt, bytes_copied, &bytes_copied,\r
2817                 NormalPagePriority );\r
2818 \r
2819         /* Free our temp packet now that the data is copied. */\r
2820         NdisUnchainBufferAtFront( p_packet, &p_buf );\r
2821         NdisFreeBuffer( p_buf );\r
2822         NdisFreePacket( p_packet );\r
2823 \r
2824         if( bytes_copied != tot_len )\r
2825         {\r
2826                 /* Something went wrong.  Drop the packet. */\r
2827                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2828                         ("Failed to copy full packet: %d of %d bytes copied.\n",\r
2829                         bytes_copied, tot_len) );\r
2830                 return NDIS_STATUS_RESOURCES;\r
2831         }\r
2832 \r
2833         IPOIB_EXIT( IPOIB_DBG_SEND );\r
2834         return NDIS_STATUS_SUCCESS;\r
2835 }\r
2836 \r
2837 \r
2838 #if !IPOIB_USE_DMA\r
2839 /* Send using the MDL's page information rather than the SGL. */\r
2840 static ib_api_status_t\r
2841 __send_gen(\r
2842         IN                              ipoib_port_t* const                     p_port,\r
2843         IN                              ipoib_send_desc_t* const        p_desc )\r
2844 {\r
2845         uint32_t                                i, j = 1;\r
2846         ULONG                                   offset;\r
2847         MDL                                             *p_mdl;\r
2848         UINT                                    num_pages, tot_len;\r
2849         ULONG                                   buf_len;\r
2850         PPFN_NUMBER                             page_array;\r
2851         boolean_t                               hdr_done = FALSE;\r
2852         ib_api_status_t                 status;\r
2853 \r
2854         IPOIB_ENTER( IPOIB_DBG_SEND );\r
2855 \r
2856         NdisQueryPacket( p_desc->p_pkt, &num_pages, NULL, &p_mdl,\r
2857                 &tot_len );\r
2858 \r
2859         if( !p_mdl )\r
2860         {\r
2861                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2862                         ("No buffers associated with packet.\n") );\r
2863                 return IB_ERROR;\r
2864         }\r
2865 \r
2866         /* Remember that one of the DS entries is reserved for the IPoIB header. */\r
2867         if( num_pages >= MAX_SEND_SGE )\r
2868         {\r
2869                 IPOIB_PRINT(TRACE_LEVEL_INFORMATION, IPOIB_DBG_SEND,\r
2870                         ("Too many buffers to fit in WR ds_array.  Copying data.\n") );\r
2871                 status = __send_copy( p_port, p_desc );\r
2872                 IPOIB_EXIT( IPOIB_DBG_SEND );\r
2873                 return status;\r
2874         }\r
2875 \r
2876         CL_ASSERT( tot_len > sizeof(eth_hdr_t) );\r
2877         CL_ASSERT( tot_len <= p_port->p_adapter->params.xfer_block_size );\r
2878         /*\r
2879          * Assume that the ethernet header is always fully contained\r
2880          * in the first page of the first MDL.  This makes for much\r
2881          * simpler code.\r
2882          */\r
2883         offset = MmGetMdlByteOffset( p_mdl ) + sizeof(eth_hdr_t);\r
2884         CL_ASSERT( offset <= PAGE_SIZE );\r
2885 \r
2886         while( tot_len )\r
2887         {\r
2888                 buf_len = MmGetMdlByteCount( p_mdl );\r
2889                 page_array = MmGetMdlPfnArray( p_mdl );\r
2890                 CL_ASSERT( page_array );\r
2891                 i = 0;\r
2892                 if( !hdr_done )\r
2893                 {\r
2894                         CL_ASSERT( buf_len >= sizeof(eth_hdr_t) );\r
2895                         /* Skip the ethernet header. */\r
2896                         buf_len -= sizeof(eth_hdr_t);\r
2897                         CL_ASSERT( buf_len <= p_port->p_adapter->params.payload_mtu );\r
2898                         if( buf_len )\r
2899                         {\r
2900                                 /* The ethernet header is a subset of this MDL. */\r
2901                                 CL_ASSERT( i == 0 );\r
2902                                 if( offset < PAGE_SIZE )\r
2903                                 {\r
2904                                         p_desc->local_ds[j].lkey = p_port->ib_mgr.lkey;\r
2905                                         p_desc->local_ds[j].vaddr = (page_array[i] << PAGE_SHIFT);\r
2906                                         /* Add the byte offset since we're on the 1st page. */\r
2907                                         p_desc->local_ds[j].vaddr += offset;\r
2908                                         if( offset + buf_len > PAGE_SIZE )\r
2909                                         {\r
2910                                                 p_desc->local_ds[j].length = PAGE_SIZE - offset;\r
2911                                                 buf_len -= p_desc->local_ds[j].length;\r
2912                                         }\r
2913                                         else\r
2914                                         {\r
2915                                                 p_desc->local_ds[j].length = buf_len;\r
2916                                                 buf_len = 0;\r
2917                                         }\r
2918                                         /* This data segment is done.  Move to the next. */\r
2919                                         j++;\r
2920                                 }\r
2921                                 /* This page is done.  Move to the next. */\r
2922                                 i++;\r
2923                         }\r
2924                         /* Done handling the ethernet header. */\r
2925                         hdr_done = TRUE;\r
2926                 }\r
2927 \r
2928                 /* Finish this MDL */\r
2929                 while( buf_len )\r
2930                 {\r
2931                         p_desc->local_ds[j].lkey = p_port->ib_mgr.lkey;\r
2932                         p_desc->local_ds[j].vaddr = (page_array[i] << PAGE_SHIFT);\r
2933                         /* Add the first page's offset if we're on the first page. */\r
2934                         if( i == 0 )\r
2935                                 p_desc->local_ds[j].vaddr += MmGetMdlByteOffset( p_mdl );\r
2936 \r
2937                         if( i == 0 && (MmGetMdlByteOffset( p_mdl ) + buf_len) > PAGE_SIZE )\r
2938                         {\r
2939                                 /* Buffers spans pages. */\r
2940                                 p_desc->local_ds[j].length =\r
2941                                         PAGE_SIZE - MmGetMdlByteOffset( p_mdl );\r
2942                                 buf_len -= p_desc->local_ds[j].length;\r
2943                                 /* This page is done.  Move to the next. */\r
2944                                 i++;\r
2945                         }\r
2946                         else\r
2947                         {\r
2948                                 /* Last page of the buffer. */\r
2949                                 p_desc->local_ds[j].length = buf_len;\r
2950                                 buf_len = 0;\r
2951                         }\r
2952                         /* This data segment is done.  Move to the next. */\r
2953                         j++;\r
2954                 }\r
2955 \r
2956                 tot_len -= MmGetMdlByteCount( p_mdl );\r
2957                 if( !tot_len )\r
2958                         break;\r
2959 \r
2960                 NdisGetNextBuffer( p_mdl, &p_mdl );\r
2961                 if( !p_mdl )\r
2962                 {\r
2963                         IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2964                                 ("Failed to get next buffer.\n") );\r
2965                         return IB_ERROR;\r
2966                 }\r
2967         }\r
2968 \r
2969         /* Set the number of data segments. */\r
2970         p_desc->wr.num_ds = j;\r
2971 \r
2972         IPOIB_EXIT( IPOIB_DBG_SEND );\r
2973         return IB_SUCCESS;\r
2974 }\r
2975 \r
2976 #else\r
2977 \r
2978 static NDIS_STATUS\r
2979 __send_gen(\r
2980         IN                              ipoib_port_t* const                     p_port,\r
2981         IN                              ipoib_send_desc_t* const        p_desc )\r
2982 {\r
2983         ib_api_status_t                 status;\r
2984         SCATTER_GATHER_LIST             *p_sgl;\r
2985         uint32_t                                i, j = 1;\r
2986         uint32_t                                offset = sizeof(eth_hdr_t);\r
2987         PERF_DECLARE( SendCopy );\r
2988 \r
2989         IPOIB_ENTER( IPOIB_DBG_SEND );\r
2990 \r
2991         p_sgl = NDIS_PER_PACKET_INFO_FROM_PACKET( p_desc->p_pkt,\r
2992                 ScatterGatherListPacketInfo );\r
2993         if( !p_sgl )\r
2994         {\r
2995                 ASSERT( p_sgl );\r
2996                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2997                         ("Failed to get SGL from packet.\n") );\r
2998                 return NDIS_STATUS_FAILURE;\r
2999         }\r
3000 \r
3001         /* Remember that one of the DS entries is reserved for the IPoIB header. */\r
3002         if( ( p_sgl->NumberOfElements >= MAX_SEND_SGE &&\r
3003                 p_sgl->Elements[0].Length > sizeof(eth_hdr_t)) ||\r
3004                 ( p_sgl->NumberOfElements > MAX_SEND_SGE &&\r
3005                 p_sgl->Elements[0].Length <= sizeof(eth_hdr_t)) )\r