[ALL] fix drivers to return PKEY in BE order as IBAL requires [MLNX: 2551]
[mirror/winof/.git] / ulp / ipoib / kernel / ipoib_port.c
1 /*\r
2  * Copyright (c) 2005 SilverStorm Technologies.  All rights reserved.\r
3  * Copyright (c) 2006 Mellanox Technologies.  All rights reserved.\r
4  *\r
5  * This software is available to you under the OpenIB.org BSD license\r
6  * below:\r
7  *\r
8  *     Redistribution and use in source and binary forms, with or\r
9  *     without modification, are permitted provided that the following\r
10  *     conditions are met:\r
11  *\r
12  *      - Redistributions of source code must retain the above\r
13  *        copyright notice, this list of conditions and the following\r
14  *        disclaimer.\r
15  *\r
16  *      - Redistributions in binary form must reproduce the above\r
17  *        copyright notice, this list of conditions and the following\r
18  *        disclaimer in the documentation and/or other materials\r
19  *        provided with the distribution.\r
20  *\r
21  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,\r
22  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF\r
23  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND\r
24  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS\r
25  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN\r
26  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN\r
27  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\r
28  * SOFTWARE.\r
29  *\r
30  * $Id$\r
31  */\r
32 \r
33 \r
34 \r
35 #include "ipoib_port.h"\r
36 #include "ipoib_adapter.h"\r
37 #include "ipoib_debug.h"\r
38 #if defined(EVENT_TRACING)\r
39 #ifdef offsetof\r
40 #undef offsetof\r
41 #endif\r
42 #include "ipoib_port.tmh"\r
43 #endif\r
44 \r
45 \r
46 /* Amount of physical memory to register. */\r
47 #define MEM_REG_SIZE    0xFFFFFFFFFFFFFFFF\r
48 \r
49 /* Number of work completions to chain for send and receive polling. */\r
50 #define MAX_SEND_WC             8\r
51 #define MAX_RECV_WC             16\r
52 \r
53 \r
54 ib_gid_t        bcast_mgid_template = {\r
55         0xff,                                                           /* multicast field */\r
56         0x12,                                                           /* scope (to be filled in) */\r
57         0x40, 0x1b,                                                     /* IPv4 signature */\r
58         0xff, 0xff,                                                     /* 16 bits of P_Key (to be filled in) */\r
59         0x00, 0x00, 0x00, 0x00, 0x00, 0x00,     /* 48 bits of zeros */\r
60         0xff, 0xff, 0xff, 0xff,                         /* 32 bit IPv4 broadcast address */\r
61 };\r
62 \r
63 \r
64 #ifdef _DEBUG_\r
65 /* Handy pointer for debug use. */\r
66 ipoib_port_t    *gp_ipoib_port;\r
67 #endif\r
68 \r
69 \r
70 /******************************************************************************\r
71 *\r
72 * Declarations\r
73 *\r
74 ******************************************************************************/\r
75 static void\r
76 __port_construct(\r
77         IN                              ipoib_port_t* const                     p_port );\r
78 \r
79 static ib_api_status_t\r
80 __port_init(\r
81         IN                              ipoib_port_t* const                     p_port,\r
82         IN                              ipoib_adapter_t* const          p_adapter,\r
83         IN                              ib_pnp_port_rec_t* const        p_pnp_rec );\r
84 \r
85 static void\r
86 __port_destroying(\r
87         IN                              cl_obj_t* const                         p_obj );\r
88 \r
89 static void\r
90 __port_cleanup(\r
91         IN                              cl_obj_t* const                         p_obj );\r
92 \r
93 static void\r
94 __port_free(\r
95         IN                              cl_obj_t* const                         p_obj );\r
96 \r
97 \r
98 /******************************************************************************\r
99 *\r
100 * IB resource manager operations\r
101 *\r
102 ******************************************************************************/\r
103 static void\r
104 __ib_mgr_construct(\r
105         IN                              ipoib_port_t* const                     p_port );\r
106 \r
107 static ib_api_status_t\r
108 __ib_mgr_init(\r
109         IN                              ipoib_port_t* const                     p_port );\r
110 \r
111 static void\r
112 __ib_mgr_destroy(\r
113         IN                              ipoib_port_t* const                     p_port );\r
114 \r
115 static void\r
116 __qp_event(\r
117         IN                              ib_async_event_rec_t            *p_event_rec );\r
118 \r
119 static void\r
120 __cq_event(\r
121         IN                              ib_async_event_rec_t            *p_event_rec );\r
122 \r
123 static ib_api_status_t\r
124 __ib_mgr_activate(\r
125         IN                              ipoib_port_t* const                     p_port );\r
126 \r
127 /******************************************************************************\r
128 *\r
129 * Buffer manager operations.\r
130 *\r
131 ******************************************************************************/\r
132 static void\r
133 __buf_mgr_construct(\r
134         IN                              ipoib_port_t* const                     p_port );\r
135 \r
136 static ib_api_status_t\r
137 __buf_mgr_init(\r
138         IN                              ipoib_port_t* const                     p_port );\r
139 \r
140 static void\r
141 __buf_mgr_destroy(\r
142         IN                              ipoib_port_t* const                     p_port );\r
143 \r
144 static cl_status_t\r
145 __recv_ctor(\r
146         IN                              void* const                                     p_object,\r
147         IN                              void*                                           context,\r
148                 OUT                     cl_pool_item_t** const          pp_pool_item );\r
149 \r
150 #if !IPOIB_INLINE_RECV\r
151 static void\r
152 __recv_dtor(\r
153         IN              const   cl_pool_item_t* const           p_pool_item,\r
154         IN                              void                                            *context );\r
155 #endif  /* IPOIB_INLINE_RECV */\r
156 \r
157 static inline ipoib_send_desc_t*\r
158 __buf_mgr_get_send(\r
159         IN                              ipoib_port_t* const                     p_port );\r
160 \r
161 static inline void\r
162 __buf_mgr_put_send(\r
163         IN                              ipoib_port_t* const                     p_port,\r
164         IN                              ipoib_send_desc_t* const        p_desc );\r
165 \r
166 static inline ipoib_recv_desc_t*\r
167 __buf_mgr_get_recv(\r
168         IN                              ipoib_port_t* const                     p_port );\r
169 \r
170 static inline void\r
171 __buf_mgr_put_recv(\r
172         IN                              ipoib_port_t* const                     p_port,\r
173         IN                              ipoib_recv_desc_t* const        p_desc,\r
174         IN                              NDIS_PACKET* const                      p_packet OPTIONAL );\r
175 \r
176 static inline void\r
177 __buf_mgr_put_recv_list(\r
178         IN                              ipoib_port_t* const                     p_port,\r
179         IN                              cl_qlist_t* const                       p_list );\r
180 \r
181 static inline NDIS_PACKET*\r
182 __buf_mgr_get_ndis_pkt(\r
183         IN                              ipoib_port_t* const                     p_port,\r
184         IN                              ipoib_recv_desc_t* const        p_desc );\r
185 \r
186 \r
187 /******************************************************************************\r
188 *\r
189 * Receive manager operations.\r
190 *\r
191 ******************************************************************************/\r
192 static void\r
193 __recv_mgr_construct(\r
194         IN                              ipoib_port_t* const                     p_port );\r
195 \r
196 static ib_api_status_t\r
197 __recv_mgr_init(\r
198         IN                              ipoib_port_t* const                     p_port );\r
199 \r
200 static void\r
201 __recv_mgr_destroy(\r
202         IN                              ipoib_port_t* const                     p_port );\r
203 \r
204 /* Posts receive buffers to the receive queue. */\r
205 static ib_api_status_t\r
206 __recv_mgr_repost(\r
207         IN                              ipoib_port_t* const                     p_port );\r
208 \r
209 static void\r
210 __recv_cb(\r
211         IN              const   ib_cq_handle_t FUNC_PTR64                               h_cq,\r
212         IN                              void                                            *cq_context );\r
213 \r
214 static void\r
215 __recv_get_endpts(\r
216         IN                              ipoib_port_t* const                     p_port,\r
217         IN                              ipoib_recv_desc_t* const        p_desc,\r
218         IN                              ib_wc_t* const                          p_wc,\r
219                 OUT                     ipoib_endpt_t** const           pp_src,\r
220                 OUT                     ipoib_endpt_t** const           pp_dst );\r
221 \r
222 static int32_t\r
223 __recv_mgr_filter(\r
224         IN                              ipoib_port_t* const                     p_port,\r
225         IN                              ib_wc_t* const                          p_done_wc_list,\r
226                 OUT                     cl_qlist_t* const                       p_done_list,\r
227                 OUT                     cl_qlist_t* const                       p_bad_list );\r
228 \r
229 static ib_api_status_t\r
230 __recv_gen(\r
231         IN              const   ipoib_pkt_t* const                      p_ipoib,\r
232                 OUT                     eth_pkt_t* const                        p_eth,\r
233         IN                              ipoib_endpt_t* const            p_src,\r
234         IN                              ipoib_endpt_t* const            p_dst );\r
235 \r
236 static ib_api_status_t\r
237 __recv_dhcp(\r
238         IN                              ipoib_port_t* const                     p_port,\r
239         IN              const   ipoib_pkt_t* const                      p_ipoib,\r
240                 OUT                     eth_pkt_t* const                        p_eth,\r
241         IN                              ipoib_endpt_t* const            p_src,\r
242         IN                              ipoib_endpt_t* const            p_dst );\r
243 \r
244 static ib_api_status_t\r
245 __recv_arp(\r
246         IN                              ipoib_port_t* const                     p_port,\r
247         IN                              ib_wc_t* const                          p_wc,\r
248         IN              const   ipoib_pkt_t* const                      p_ipoib,\r
249                 OUT                     eth_pkt_t* const                        p_eth,\r
250         IN                              ipoib_endpt_t** const           p_src,\r
251         IN                              ipoib_endpt_t* const            p_dst );\r
252 \r
253 static ib_api_status_t\r
254 __recv_mgr_prepare_pkt(\r
255         IN                              ipoib_port_t* const                     p_port,\r
256         IN                              ipoib_recv_desc_t* const        p_desc,\r
257                 OUT                     NDIS_PACKET** const                     pp_packet );\r
258 \r
259 static uint32_t\r
260 __recv_mgr_build_pkt_array(\r
261         IN                              ipoib_port_t* const                     p_port,\r
262         IN                              int32_t                                         shortage,\r
263                 OUT                     cl_qlist_t* const                       p_done_list,\r
264                 OUT                     int32_t* const                          p_discarded );\r
265 \r
266 /******************************************************************************\r
267 *\r
268 * Send manager operations.\r
269 *\r
270 ******************************************************************************/\r
271 static void\r
272 __send_mgr_construct(\r
273         IN                              ipoib_port_t* const                     p_port );\r
274 \r
275 static void\r
276 __send_mgr_destroy(\r
277         IN                              ipoib_port_t* const                     p_port );\r
278 \r
279 static NDIS_STATUS\r
280 __send_gen(\r
281         IN                              ipoib_port_t* const                     p_port,\r
282         IN                              ipoib_send_desc_t* const        p_desc );\r
283 \r
284 static NDIS_STATUS\r
285 __send_mgr_filter_ip(\r
286         IN                              ipoib_port_t* const                     p_port,\r
287         IN              const   eth_hdr_t* const                        p_eth_hdr,\r
288         IN                              NDIS_BUFFER*                            p_buf,\r
289         IN                              size_t                                          buf_len,\r
290         IN      OUT                     ipoib_send_desc_t* const        p_desc );\r
291 \r
292 static NDIS_STATUS\r
293 __send_mgr_filter_udp(\r
294         IN                              ipoib_port_t* const                     p_port,\r
295         IN              const   ip_hdr_t* const                         p_ip_hdr,\r
296         IN                              NDIS_BUFFER*                            p_buf,\r
297         IN                              size_t                                          buf_len,\r
298         IN      OUT                     ipoib_send_desc_t* const        p_desc );\r
299 \r
300 static NDIS_STATUS\r
301 __send_mgr_filter_dhcp(\r
302         IN                              ipoib_port_t* const                     p_port,\r
303         IN              const   udp_hdr_t* const                        p_udp_hdr,\r
304         IN                              NDIS_BUFFER*                            p_buf,\r
305         IN                              size_t                                          buf_len,\r
306         IN      OUT                     ipoib_send_desc_t* const        p_desc );\r
307 \r
308 static NDIS_STATUS\r
309 __send_mgr_filter_arp(\r
310         IN                              ipoib_port_t* const                     p_port,\r
311         IN              const   eth_hdr_t* const                        p_eth_hdr,\r
312         IN                              NDIS_BUFFER*                            p_buf,\r
313         IN                              size_t                                          buf_len,\r
314         IN      OUT                     ipoib_send_desc_t* const        p_desc );\r
315 \r
316 static void\r
317 __process_failed_send(\r
318         IN                              ipoib_port_t* const                     p_port,\r
319         IN                              ipoib_send_desc_t* const        p_desc,\r
320         IN              const   NDIS_STATUS                                     status );\r
321 \r
322 static void\r
323 __send_cb(\r
324         IN              const   ib_cq_handle_t FUNC_PTR64                               h_cq,\r
325         IN                              void                                            *cq_context );\r
326 \r
327 \r
328 /******************************************************************************\r
329 *\r
330 * Endpoint manager operations\r
331 *\r
332 ******************************************************************************/\r
333 static void\r
334 __endpt_mgr_construct(\r
335         IN                              ipoib_port_t* const                     p_port );\r
336 \r
337 static ib_api_status_t\r
338 __endpt_mgr_init(\r
339         IN                              ipoib_port_t* const                     p_port );\r
340 \r
341 static void\r
342 __endpt_mgr_destroy(\r
343         IN                              ipoib_port_t* const                     p_port );\r
344 \r
345 /****f* IPoIB/__endpt_mgr_remove_all\r
346 * NAME\r
347 *       __endpt_mgr_remove_all\r
348 *\r
349 * DESCRIPTION\r
350 *       Removes all enpoints from the port, dereferencing them to initiate\r
351 *       destruction.\r
352 *\r
353 * SYNOPSIS\r
354 */\r
355 static void\r
356 __endpt_mgr_remove_all(\r
357         IN                              ipoib_port_t* const                     p_port );\r
358 /*\r
359 ********/\r
360 \r
361 static void\r
362 __endpt_mgr_remove(\r
363         IN                              ipoib_port_t* const                     p_port,\r
364         IN                              ipoib_endpt_t* const            p_endpt );\r
365 \r
366 static void\r
367 __endpt_mgr_reset_all(\r
368         IN                              ipoib_port_t* const                     p_port );\r
369 \r
370 static inline NDIS_STATUS\r
371 __endpt_mgr_ref(\r
372         IN                              ipoib_port_t* const                     p_port,\r
373         IN              const   mac_addr_t                                      mac,\r
374                 OUT                     ipoib_endpt_t** const           pp_endpt );\r
375 \r
376 static inline NDIS_STATUS\r
377 __endpt_mgr_get_gid_qpn(\r
378         IN                              ipoib_port_t* const                     p_port,\r
379         IN              const   mac_addr_t                                      mac,\r
380                 OUT                     ib_gid_t* const                         p_gid,\r
381                 OUT                     UNALIGNED net32_t* const        p_qpn );\r
382 \r
383 static inline ipoib_endpt_t*\r
384 __endpt_mgr_get_by_gid(\r
385         IN                              ipoib_port_t* const                     p_port,\r
386         IN              const   ib_gid_t* const                         p_gid );\r
387 \r
388 static inline ipoib_endpt_t*\r
389 __endpt_mgr_get_by_lid(\r
390         IN                              ipoib_port_t* const                     p_port,\r
391         IN              const   net16_t                                         lid );\r
392 \r
393 static inline ib_api_status_t\r
394 __endpt_mgr_insert_locked(\r
395         IN                              ipoib_port_t* const                     p_port,\r
396         IN              const   mac_addr_t                                      mac,\r
397         IN                              ipoib_endpt_t* const            p_endpt );\r
398 \r
399 static inline ib_api_status_t\r
400 __endpt_mgr_insert(\r
401         IN                              ipoib_port_t* const                     p_port,\r
402         IN              const   mac_addr_t                                      mac,\r
403         IN                              ipoib_endpt_t* const            p_endpt );\r
404 \r
405 static ib_api_status_t\r
406 __endpt_mgr_add_local(\r
407         IN                              ipoib_port_t* const                     p_port,\r
408         IN                              ib_port_info_t* const           p_port_info );\r
409 \r
410 static ib_api_status_t\r
411 __endpt_mgr_add_bcast(\r
412         IN                              ipoib_port_t* const                     p_port,\r
413         IN                              ib_mcast_rec_t                          *p_mcast_rec );\r
414 \r
415 /******************************************************************************\r
416 *\r
417 * MCast operations.\r
418 *\r
419 ******************************************************************************/\r
420 static ib_api_status_t\r
421 __port_get_bcast(\r
422         IN                              ipoib_port_t* const                     p_port );\r
423 \r
424 static ib_api_status_t\r
425 __port_join_bcast(\r
426         IN                              ipoib_port_t* const                     p_port,\r
427         IN                              ib_member_rec_t* const          p_member_rec );\r
428 \r
429 static ib_api_status_t\r
430 __port_create_bcast(\r
431         IN                              ipoib_port_t* const                     p_port );\r
432 \r
433 static void\r
434 __port_info_cb(\r
435         IN                              ib_query_rec_t                          *p_query_rec );\r
436 \r
437 \r
438 static void\r
439 __bcast_get_cb(\r
440         IN                              ib_query_rec_t                          *p_query_rec );\r
441 \r
442 \r
443 static void\r
444 __bcast_cb(\r
445         IN                              ib_mcast_rec_t                          *p_mcast_rec );\r
446 \r
447 \r
448 static void\r
449 __mcast_cb(\r
450         IN                              ib_mcast_rec_t                          *p_mcast_rec );\r
451 \r
452 void\r
453 __leave_error_mcast_cb(\r
454         IN                              void                            *context );\r
455 \r
456 \r
457 static intn_t\r
458 __gid_cmp(\r
459         IN              const   void* const                                     p_key1,\r
460         IN              const   void* const                                     p_key2 )\r
461 {\r
462         return cl_memcmp( p_key1, p_key2, sizeof(ib_gid_t) );\r
463 }\r
464 \r
465 \r
466 inline void ipoib_port_ref( ipoib_port_t * p_port, int type )\r
467 {\r
468         cl_obj_ref( &p_port->obj );\r
469 #if DBG\r
470         cl_atomic_inc( &p_port->ref[type % ref_mask] );\r
471         IPOIB_PRINT( TRACE_LEVEL_INFORMATION, IPOIB_DBG_OBJ,\r
472                 ("ref type %d ref_cnt %d\n", type, p_port->obj.ref_cnt) );\r
473 #else\r
474         UNREFERENCED_PARAMETER(type);\r
475 #endif\r
476 }\r
477 \r
478 \r
479 inline void ipoib_port_deref(ipoib_port_t * p_port, int type)\r
480 {\r
481         cl_obj_deref( &p_port->obj );\r
482 \r
483 #if DBG\r
484         cl_atomic_dec( &p_port->ref[type % ref_mask] );\r
485         IPOIB_PRINT( TRACE_LEVEL_INFORMATION, IPOIB_DBG_OBJ,\r
486                 ("deref type %d ref_cnt %d\n", type, p_port->obj.ref_cnt) );\r
487 #else\r
488         UNREFERENCED_PARAMETER(type);\r
489 #endif\r
490 }\r
491 \r
492 \r
493 /******************************************************************************\r
494 *\r
495 * Implementation\r
496 *\r
497 ******************************************************************************/\r
498 ib_api_status_t\r
499 ipoib_create_port(\r
500         IN                              ipoib_adapter_t* const          p_adapter,\r
501         IN                              ib_pnp_port_rec_t* const        p_pnp_rec,\r
502                 OUT                     ipoib_port_t** const            pp_port )\r
503 {\r
504         ib_api_status_t         status;\r
505         ipoib_port_t            *p_port;\r
506 \r
507         IPOIB_ENTER( IPOIB_DBG_INIT );\r
508 \r
509         CL_ASSERT( !p_adapter->p_port );\r
510 \r
511         p_port = cl_zalloc( sizeof(ipoib_port_t) +\r
512                 (sizeof(ipoib_hdr_t) * (p_adapter->params.sq_depth - 1)) );\r
513         if( !p_port )\r
514         {\r
515                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
516                         ("Failed to allocate ipoib_port_t (%d bytes)\n",\r
517                         sizeof(ipoib_port_t)) );\r
518                 return IB_INSUFFICIENT_MEMORY;\r
519         }\r
520 \r
521 #ifdef _DEBUG_\r
522         gp_ipoib_port = p_port;\r
523 #endif\r
524 \r
525         __port_construct( p_port );\r
526 \r
527         status = __port_init( p_port, p_adapter, p_pnp_rec );\r
528         if( status != IB_SUCCESS )\r
529         {\r
530                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
531                         ("ipoib_port_init returned %s.\n", \r
532                         p_adapter->p_ifc->get_err_str( status )) );\r
533                 __port_cleanup( &p_port->obj );\r
534                 __port_free( &p_port->obj );\r
535                 return status;\r
536         }\r
537 \r
538         *pp_port = p_port;\r
539         IPOIB_EXIT( IPOIB_DBG_INIT );\r
540         return IB_SUCCESS;\r
541 }\r
542 \r
543 \r
544 void\r
545 ipoib_port_destroy(\r
546         IN                              ipoib_port_t* const                     p_port )\r
547 {\r
548         IPOIB_ENTER( IPOIB_DBG_INIT );\r
549 \r
550         CL_ASSERT( p_port );\r
551         CL_ASSERT( p_port->p_adapter );\r
552         CL_ASSERT( !p_port->p_adapter->p_port );\r
553 \r
554         cl_obj_destroy( &p_port->obj );\r
555 \r
556         IPOIB_EXIT( IPOIB_DBG_INIT );\r
557 }\r
558 \r
559 \r
560 static void\r
561 __port_construct(\r
562         IN                              ipoib_port_t* const                     p_port )\r
563 {\r
564         IPOIB_ENTER( IPOIB_DBG_INIT );\r
565 \r
566         p_port->state = IB_QPS_RESET;\r
567 \r
568         cl_obj_construct( &p_port->obj, IPOIB_OBJ_PORT );\r
569         cl_spinlock_construct( &p_port->send_lock );\r
570         cl_spinlock_construct( &p_port->recv_lock );\r
571         __ib_mgr_construct( p_port );\r
572         __buf_mgr_construct( p_port );\r
573 \r
574         __recv_mgr_construct( p_port );\r
575         __send_mgr_construct( p_port );\r
576 \r
577         __endpt_mgr_construct( p_port );\r
578 \r
579         KeInitializeEvent( &p_port->sa_event, NotificationEvent, TRUE );\r
580         KeInitializeEvent( &p_port->leave_mcast_event, NotificationEvent, TRUE );\r
581         \r
582         IPOIB_EXIT( IPOIB_DBG_INIT );\r
583 }\r
584 \r
585 \r
586 static ib_api_status_t\r
587 __port_init(\r
588         IN                              ipoib_port_t* const                     p_port,\r
589         IN                              ipoib_adapter_t* const          p_adapter,\r
590         IN                              ib_pnp_port_rec_t* const        p_pnp_rec )\r
591 {\r
592         cl_status_t                     cl_status;\r
593         ib_api_status_t         status;\r
594 \r
595         IPOIB_ENTER( IPOIB_DBG_INIT );\r
596 \r
597         p_port->port_num = p_pnp_rec->p_port_attr->port_num;\r
598         p_port->p_adapter = p_adapter;\r
599 \r
600         cl_status = cl_spinlock_init( &p_port->send_lock );\r
601         if( cl_status != CL_SUCCESS )\r
602         {\r
603                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
604                         ("cl_spinlock_init returned %#x\n", cl_status) );\r
605                 return IB_ERROR;\r
606         }\r
607 \r
608         cl_status = cl_spinlock_init( &p_port->recv_lock );\r
609         if( cl_status != CL_SUCCESS )\r
610         {\r
611                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
612                         ("cl_spinlock_init returned %#x\n", cl_status) );\r
613                 return IB_ERROR;\r
614         }\r
615 \r
616         /* Initialize the IB resource manager. */\r
617         status = __ib_mgr_init( p_port );\r
618         if( status != IB_SUCCESS )\r
619         {\r
620                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
621                         ("__ib_mgr_init returned %s\n", \r
622                         p_adapter->p_ifc->get_err_str( status )) );\r
623                 return status;\r
624         }\r
625 \r
626         /* Initialize the buffer manager. */\r
627         status = __buf_mgr_init( p_port );\r
628         if( status != IB_SUCCESS )\r
629         {\r
630                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
631                         ("__buf_mgr_init returned %s\n", \r
632                         p_adapter->p_ifc->get_err_str( status )) );\r
633                 return status;\r
634         }\r
635 \r
636         /* Initialize the receive manager. */\r
637         status = __recv_mgr_init( p_port );\r
638         if( status != IB_SUCCESS )\r
639         {\r
640                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
641                         ("__recv_mgr_init returned %s\n", \r
642                         p_adapter->p_ifc->get_err_str( status )) );\r
643                 return status;\r
644         }\r
645 \r
646         /* Initialize the endpoint manager. */\r
647         status = __endpt_mgr_init( p_port );\r
648         if( status != IB_SUCCESS )\r
649         {\r
650                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
651                         ("__endpt_mgr_init returned %s\n", \r
652                         p_adapter->p_ifc->get_err_str( status )) );\r
653                 return status;\r
654         }\r
655 \r
656         /* We only ever destroy from the PnP callback thread. */\r
657         cl_status = cl_obj_init( &p_port->obj, CL_DESTROY_SYNC,\r
658                 __port_destroying, __port_cleanup, __port_free );\r
659 \r
660 #if DBG\r
661         cl_atomic_inc( &p_port->ref[ref_init] );\r
662         IPOIB_PRINT( TRACE_LEVEL_INFORMATION, IPOIB_DBG_OBJ,\r
663                 ("ref type %d ref_cnt %d\n", ref_init, p_port->obj.ref_cnt) );\r
664 #endif\r
665 \r
666         if( cl_status != CL_SUCCESS )\r
667         {\r
668                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
669                         ("cl_obj_init returned %#x\n", cl_status) );\r
670                 return IB_ERROR;\r
671         }\r
672 \r
673         cl_status = cl_obj_insert_rel( &p_port->rel, &p_adapter->obj, &p_port->obj );\r
674         if( cl_status != CL_SUCCESS )\r
675         {\r
676                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
677                         ("cl_obj_insert_rel returned %#x\n", cl_status) );\r
678                 cl_obj_destroy( &p_port->obj );\r
679                 return IB_ERROR;\r
680         }\r
681 \r
682 #if DBG\r
683         cl_atomic_inc( &p_port->ref[ref_init] );\r
684         IPOIB_PRINT( TRACE_LEVEL_ERROR, IPOIB_DBG_OBJ,\r
685                 ("ref type %d ref_cnt %d\n", ref_init, p_port->obj.ref_cnt) );\r
686 #endif\r
687 \r
688         IPOIB_EXIT( IPOIB_DBG_INIT );\r
689         return IB_SUCCESS;\r
690 }\r
691 \r
692 \r
693 static void\r
694 __port_destroying(\r
695         IN                              cl_obj_t* const                         p_obj )\r
696 {\r
697         ipoib_port_t    *p_port;\r
698 \r
699         IPOIB_ENTER( IPOIB_DBG_INIT );\r
700 \r
701         CL_ASSERT( p_obj );\r
702 \r
703         p_port = PARENT_STRUCT( p_obj, ipoib_port_t, obj );\r
704 \r
705         ipoib_port_down( p_port );\r
706 \r
707         __endpt_mgr_remove_all( p_port );\r
708 \r
709         ipoib_port_resume( p_port );\r
710 \r
711         IPOIB_EXIT( IPOIB_DBG_INIT );\r
712 }\r
713 \r
714 \r
715 static void\r
716 __port_cleanup(\r
717         IN                              cl_obj_t* const                         p_obj )\r
718 {\r
719         ipoib_port_t    *p_port;\r
720 \r
721         IPOIB_ENTER( IPOIB_DBG_INIT );\r
722 \r
723         CL_ASSERT( p_obj );\r
724 \r
725         p_port = PARENT_STRUCT( p_obj, ipoib_port_t, obj );\r
726 \r
727         /* Wait for all sends and receives to get flushed. */\r
728         while( p_port->send_mgr.depth || p_port->recv_mgr.depth )\r
729                 cl_thread_suspend( 0 );\r
730 \r
731         /* Destroy the send and receive managers before closing the CA. */\r
732         __ib_mgr_destroy( p_port );\r
733 \r
734         IPOIB_EXIT( IPOIB_DBG_INIT );\r
735 }\r
736 \r
737 \r
738 static void\r
739 __port_free(\r
740         IN                              cl_obj_t* const                         p_obj )\r
741 {\r
742         ipoib_port_t    *p_port;\r
743 \r
744         IPOIB_ENTER( IPOIB_DBG_INIT );\r
745 \r
746         CL_ASSERT( p_obj );\r
747 \r
748         p_port = PARENT_STRUCT( p_obj, ipoib_port_t, obj );\r
749 \r
750         __endpt_mgr_destroy( p_port );\r
751         __recv_mgr_destroy( p_port );\r
752         __send_mgr_destroy( p_port );\r
753         __buf_mgr_destroy( p_port );\r
754 \r
755         cl_spinlock_destroy( &p_port->send_lock );\r
756         cl_spinlock_destroy( &p_port->recv_lock );\r
757 \r
758         cl_obj_deinit( p_obj );\r
759 \r
760         cl_free( p_port );\r
761 \r
762         IPOIB_EXIT( IPOIB_DBG_INIT );\r
763 }\r
764 \r
765 \r
766 \r
767 /******************************************************************************\r
768 *\r
769 * IB resource manager implementation.\r
770 *\r
771 ******************************************************************************/\r
772 static void\r
773 __ib_mgr_construct(\r
774         IN                              ipoib_port_t* const                     p_port )\r
775 {\r
776         IPOIB_ENTER( IPOIB_DBG_INIT );\r
777 \r
778         cl_memclr( &p_port->ib_mgr, sizeof(ipoib_ib_mgr_t) );\r
779 \r
780         IPOIB_EXIT( IPOIB_DBG_INIT );\r
781 }\r
782 \r
783 \r
784 static ib_api_status_t\r
785 __ib_mgr_init(\r
786         IN                              ipoib_port_t* const                     p_port )\r
787 {\r
788         ib_api_status_t         status;\r
789         ib_cq_create_t          cq_create;\r
790         ib_qp_create_t          qp_create;\r
791         ib_phys_create_t        phys_create;\r
792         ib_phys_range_t         phys_range;\r
793         uint64_t                        vaddr;\r
794         net32_t                         rkey;\r
795         ib_qp_attr_t            qp_attr;\r
796         ib_ca_attr_t            *ca_attr;\r
797         uint32_t                        ca_size;\r
798 \r
799         IPOIB_ENTER( IPOIB_DBG_INIT );\r
800 \r
801         /* Open the CA. */\r
802         status = p_port->p_adapter->p_ifc->open_ca(\r
803                 p_port->p_adapter->h_al, p_port->p_adapter->guids.ca_guid,\r
804                 NULL, p_port, &p_port->ib_mgr.h_ca );\r
805         if( status != IB_SUCCESS )\r
806         {\r
807                 NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
808                         EVENT_IPOIB_OPEN_CA, 1, status );\r
809                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
810                         ("ib_open_ca returned %s\n", \r
811                         p_port->p_adapter->p_ifc->get_err_str( status )) );\r
812                 return status;\r
813         }\r
814 \r
815         /* Query the CA for Pkey table */\r
816         status = p_port->p_adapter->p_ifc->query_ca(p_port->ib_mgr.h_ca, NULL, &ca_size);\r
817         if(status != IB_INSUFFICIENT_MEMORY)\r
818         {\r
819                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
820                         ("ib_query_ca failed\n"));\r
821                         return status;\r
822         }\r
823 \r
824         ca_attr = (ib_ca_attr_t*)cl_zalloc(ca_size);\r
825         if      (!ca_attr)\r
826         {\r
827                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
828                         ("cl_zalloc can't allocate %d\n",ca_size));\r
829                 return IB_INSUFFICIENT_RESOURCES;\r
830         }\r
831 \r
832         status = p_port->p_adapter->p_ifc->query_ca(p_port->ib_mgr.h_ca, ca_attr,&ca_size);     \r
833         if( status != IB_SUCCESS )\r
834         {\r
835                 cl_free(ca_attr);\r
836                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
837                         ("ib_query_ca returned %s\n", \r
838                         p_port->p_adapter->p_ifc->get_err_str( status )) );\r
839                 return status;\r
840         }\r
841         if( ca_attr->p_port_attr->link_state == IB_LINK_ACTIVE)\r
842         {\r
843                 uint16_t index;\r
844                 CL_ASSERT(ca_attr->p_port_attr->p_pkey_table[0] == IB_DEFAULT_PKEY);\r
845                 for(index = 0; index < ca_attr->p_port_attr->num_pkeys; index++)\r
846                 {\r
847                         if(p_port->p_adapter->guids.port_guid.pkey == ca_attr->p_port_attr->p_pkey_table[index])\r
848                                 break;\r
849                 }\r
850                 if(index >= ca_attr->p_port_attr->num_pkeys)\r
851                 {\r
852                         IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
853                         ("Pkey table is invalid, index not found\n"));\r
854                     return IB_NOT_FOUND;\r
855                 }\r
856                 p_port->pkey_index = index;\r
857                 IPOIB_PRINT_EXIT( TRACE_LEVEL_INFORMATION, IPOIB_DBG_IB,\r
858                         ("for PKEY = 0x%04X got index = %d\n",p_port->p_adapter->guids.port_guid.pkey,index));\r
859         }\r
860         cl_free(ca_attr);\r
861         /* Allocate the PD. */\r
862         status = p_port->p_adapter->p_ifc->alloc_pd(\r
863                 p_port->ib_mgr.h_ca, IB_PDT_UD, p_port, &p_port->ib_mgr.h_pd );\r
864         if( status != IB_SUCCESS )\r
865         {\r
866                 NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
867                         EVENT_IPOIB_ALLOC_PD, 1, status );\r
868                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
869                         ("ib_alloc_pd returned %s\n", \r
870                         p_port->p_adapter->p_ifc->get_err_str( status )) );\r
871                 return status;\r
872         }\r
873 \r
874         /* Allocate receive CQ. */\r
875         cq_create.size = p_port->p_adapter->params.rq_depth;\r
876         cq_create.pfn_comp_cb = __recv_cb;\r
877         cq_create.h_wait_obj = NULL;\r
878 \r
879         status = p_port->p_adapter->p_ifc->create_cq(\r
880                 p_port->ib_mgr.h_ca, &cq_create, p_port,\r
881                 __cq_event, &p_port->ib_mgr.h_recv_cq );\r
882         if( status != IB_SUCCESS )\r
883         {\r
884                 NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
885                         EVENT_IPOIB_CREATE_RECV_CQ, 1, status );\r
886                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
887                         ("ib_create_cq returned %s.\n", \r
888                         p_port->p_adapter->p_ifc->get_err_str( status )) );\r
889                 return status;\r
890         }\r
891 \r
892         /* Allocate send CQ. */\r
893         cq_create.size = p_port->p_adapter->params.sq_depth;\r
894         cq_create.pfn_comp_cb = __send_cb;\r
895 \r
896         status = p_port->p_adapter->p_ifc->create_cq(\r
897                 p_port->ib_mgr.h_ca, &cq_create, p_port,\r
898                 __cq_event, &p_port->ib_mgr.h_send_cq );\r
899         if( status != IB_SUCCESS )\r
900         {\r
901                 NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
902                         EVENT_IPOIB_CREATE_SEND_CQ, 1, status );\r
903                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
904                         ("ib_create_cq returned %s.\n", \r
905                         p_port->p_adapter->p_ifc->get_err_str( status )) );\r
906                 return status;\r
907         }\r
908 \r
909         /* Allocate the QP. */\r
910         cl_memclr( &qp_create, sizeof(qp_create) );\r
911         qp_create.qp_type = IB_QPT_UNRELIABLE_DGRM;\r
912         qp_create.rq_depth = p_port->p_adapter->params.rq_depth;\r
913         qp_create.rq_sge = 2;   /* To support buffers spanning pages. */\r
914         qp_create.h_rq_cq = p_port->ib_mgr.h_recv_cq;\r
915         qp_create.sq_depth = p_port->p_adapter->params.sq_depth;\r
916         //TODO: Figure out the right number of SGE entries for sends.\r
917         qp_create.sq_sge = MAX_SEND_SGE;\r
918         qp_create.h_sq_cq = p_port->ib_mgr.h_send_cq;\r
919         qp_create.sq_signaled = TRUE;\r
920         status = p_port->p_adapter->p_ifc->create_qp(\r
921                 p_port->ib_mgr.h_pd, &qp_create, p_port,\r
922                 __qp_event, &p_port->ib_mgr.h_qp );\r
923         if( status != IB_SUCCESS )\r
924         {\r
925                 NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
926                         EVENT_IPOIB_CREATE_QP, 1, status );\r
927                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
928                         ("ib_create_qp returned %s\n", \r
929                         p_port->p_adapter->p_ifc->get_err_str( status )) );\r
930                 return status;\r
931         }\r
932         /* Query the QP so we can get our QPN. */\r
933         status = p_port->p_adapter->p_ifc->query_qp(\r
934                 p_port->ib_mgr.h_qp, &qp_attr );\r
935         if( status != IB_SUCCESS )\r
936         {\r
937                 NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
938                         EVENT_IPOIB_QUERY_QP, 1, status );\r
939                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
940                         ("ib_query_qp returned %s\n", \r
941                         p_port->p_adapter->p_ifc->get_err_str( status )) );\r
942                 return status;\r
943         }\r
944         p_port->ib_mgr.qpn = qp_attr.num;\r
945 \r
946         /* Register all of physical memory */\r
947         phys_create.length = MEM_REG_SIZE;\r
948         phys_create.num_ranges = 1;\r
949         phys_create.range_array = &phys_range;\r
950         phys_create.buf_offset = 0;\r
951         phys_create.hca_page_size = PAGE_SIZE;\r
952         phys_create.access_ctrl = IB_AC_LOCAL_WRITE;\r
953         phys_range.base_addr = 0;\r
954         phys_range.size = MEM_REG_SIZE;\r
955         vaddr = 0;\r
956         status = p_port->p_adapter->p_ifc->reg_phys(\r
957                 p_port->ib_mgr.h_pd, &phys_create, &vaddr,\r
958                 &p_port->ib_mgr.lkey, &rkey, &p_port->ib_mgr.h_mr );\r
959         if( status != IB_SUCCESS )\r
960         {\r
961                 NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
962                         EVENT_IPOIB_REG_PHYS, 1, status );\r
963                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
964                         ("ib_reg_phys returned %s\n", \r
965                         p_port->p_adapter->p_ifc->get_err_str( status )) );\r
966                 return status;\r
967         }\r
968 \r
969         IPOIB_EXIT( IPOIB_DBG_INIT );\r
970         return IB_SUCCESS;\r
971 }\r
972 \r
973 \r
974 static void\r
975 __ib_mgr_destroy(\r
976         IN                              ipoib_port_t* const                     p_port )\r
977 {\r
978         ib_api_status_t status;\r
979 \r
980         IPOIB_ENTER( IPOIB_DBG_INIT );\r
981 \r
982         if( p_port->ib_mgr.h_ca )\r
983         {\r
984                 status =\r
985                         p_port->p_adapter->p_ifc->close_ca( p_port->ib_mgr.h_ca, NULL );\r
986                 CL_ASSERT( status == IB_SUCCESS );\r
987                 p_port->ib_mgr.h_ca = NULL;\r
988         }\r
989 \r
990         IPOIB_EXIT( IPOIB_DBG_INIT );\r
991 }\r
992 \r
993 \r
994 \r
995 /******************************************************************************\r
996 *\r
997 * Buffer manager implementation.\r
998 *\r
999 ******************************************************************************/\r
1000 static void\r
1001 __buf_mgr_construct(\r
1002         IN                              ipoib_port_t* const                     p_port )\r
1003 {\r
1004         IPOIB_ENTER( IPOIB_DBG_INIT );\r
1005 \r
1006         cl_qpool_construct( &p_port->buf_mgr.recv_pool );\r
1007 \r
1008         p_port->buf_mgr.h_packet_pool = NULL;\r
1009         p_port->buf_mgr.h_buffer_pool = NULL;\r
1010 \r
1011         ExInitializeNPagedLookasideList( &p_port->buf_mgr.send_buf_list,\r
1012                 NULL, NULL, 0, MAX_XFER_BLOCK_SIZE, 'bipi', 0 );\r
1013 \r
1014         p_port->buf_mgr.h_send_pkt_pool = NULL;\r
1015         p_port->buf_mgr.h_send_buf_pool = NULL;\r
1016 \r
1017         IPOIB_EXIT( IPOIB_DBG_INIT );\r
1018 }\r
1019 \r
1020 \r
1021 static ib_api_status_t\r
1022 __buf_mgr_init(\r
1023         IN                              ipoib_port_t* const                     p_port )\r
1024 {\r
1025         cl_status_t             cl_status;\r
1026         NDIS_STATUS             ndis_status;\r
1027         ipoib_params_t  *p_params;\r
1028 \r
1029         IPOIB_ENTER(IPOIB_DBG_INIT );\r
1030 \r
1031         CL_ASSERT( p_port );\r
1032         CL_ASSERT( p_port->p_adapter );\r
1033 \r
1034         p_params = &p_port->p_adapter->params;\r
1035 \r
1036         /* Allocate the receive descriptor pool */\r
1037         cl_status = cl_qpool_init( &p_port->buf_mgr.recv_pool,\r
1038                 p_params->rq_depth * p_params->recv_pool_ratio,\r
1039 #if IPOIB_INLINE_RECV\r
1040                 0, 0, sizeof(ipoib_recv_desc_t), __recv_ctor, NULL, p_port );\r
1041 #else   /* IPOIB_INLINE_RECV */\r
1042                 0, 0, sizeof(ipoib_recv_desc_t), __recv_ctor, __recv_dtor, p_port );\r
1043 #endif  /* IPOIB_INLINE_RECV */\r
1044         if( cl_status != CL_SUCCESS )\r
1045         {\r
1046                 NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
1047                         EVENT_IPOIB_RECV_POOL, 1, cl_status );\r
1048                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1049                         ("cl_qpool_init for recvs returned %#x\n",\r
1050                         cl_status) );\r
1051                 return IB_INSUFFICIENT_MEMORY;\r
1052         }\r
1053 \r
1054         /* Allocate the NDIS buffer and packet pools for receive indication. */\r
1055         NdisAllocatePacketPool( &ndis_status, &p_port->buf_mgr.h_packet_pool,\r
1056                 p_params->rq_depth, PROTOCOL_RESERVED_SIZE_IN_PACKET );\r
1057         if( ndis_status != NDIS_STATUS_SUCCESS )\r
1058         {\r
1059                 NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
1060                         EVENT_IPOIB_RECV_PKT_POOL, 1, ndis_status );\r
1061                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1062                         ("NdisAllocatePacketPool returned %08X\n", ndis_status) );\r
1063                 return IB_INSUFFICIENT_RESOURCES;\r
1064         }\r
1065 \r
1066         NdisAllocateBufferPool( &ndis_status, &p_port->buf_mgr.h_buffer_pool,\r
1067                 p_params->rq_depth );\r
1068         if( ndis_status != NDIS_STATUS_SUCCESS )\r
1069         {\r
1070                 NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
1071                         EVENT_IPOIB_RECV_BUF_POOL, 1, ndis_status );\r
1072                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1073                         ("NdisAllocateBufferPool returned %08X\n", ndis_status) );\r
1074                 return IB_INSUFFICIENT_RESOURCES;\r
1075         }\r
1076 \r
1077         /* Allocate the NDIS buffer and packet pools for send formatting. */\r
1078         NdisAllocatePacketPool( &ndis_status, &p_port->buf_mgr.h_send_pkt_pool,\r
1079                 1, PROTOCOL_RESERVED_SIZE_IN_PACKET );\r
1080         if( ndis_status != NDIS_STATUS_SUCCESS )\r
1081         {\r
1082                 NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
1083                         EVENT_IPOIB_SEND_PKT_POOL, 1, ndis_status );\r
1084                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1085                         ("NdisAllocatePacketPool returned %08X\n", ndis_status) );\r
1086                 return IB_INSUFFICIENT_RESOURCES;\r
1087         }\r
1088 \r
1089         NdisAllocateBufferPool( &ndis_status,\r
1090                 &p_port->buf_mgr.h_send_buf_pool, 1 );\r
1091         if( ndis_status != NDIS_STATUS_SUCCESS )\r
1092         {\r
1093                 NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
1094                         EVENT_IPOIB_SEND_BUF_POOL, 1, ndis_status );\r
1095                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1096                         ("NdisAllocateBufferPool returned %08X\n", ndis_status) );\r
1097                 return IB_INSUFFICIENT_RESOURCES;\r
1098         }\r
1099 \r
1100         IPOIB_EXIT( IPOIB_DBG_INIT );\r
1101         return IB_SUCCESS;\r
1102 }\r
1103 \r
1104 \r
1105 static void\r
1106 __buf_mgr_destroy(\r
1107         IN                              ipoib_port_t* const                     p_port )\r
1108 {\r
1109         IPOIB_ENTER(IPOIB_DBG_INIT );\r
1110 \r
1111         CL_ASSERT( p_port );\r
1112 \r
1113         /* Destroy the send packet and buffer pools. */\r
1114         if( p_port->buf_mgr.h_send_buf_pool )\r
1115                 NdisFreeBufferPool( p_port->buf_mgr.h_send_buf_pool );\r
1116         if( p_port->buf_mgr.h_send_pkt_pool )\r
1117                 NdisFreePacketPool( p_port->buf_mgr.h_send_pkt_pool );\r
1118 \r
1119         /* Destroy the receive packet and buffer pools. */\r
1120         if( p_port->buf_mgr.h_buffer_pool )\r
1121                 NdisFreeBufferPool( p_port->buf_mgr.h_buffer_pool );\r
1122         if( p_port->buf_mgr.h_packet_pool )\r
1123                 NdisFreePacketPool( p_port->buf_mgr.h_packet_pool );\r
1124 \r
1125         /* Free the receive and send descriptors. */\r
1126         cl_qpool_destroy( &p_port->buf_mgr.recv_pool );\r
1127 \r
1128         /* Free the lookaside list of scratch buffers. */\r
1129         ExDeleteNPagedLookasideList( &p_port->buf_mgr.send_buf_list );\r
1130 \r
1131         IPOIB_EXIT(  IPOIB_DBG_INIT );\r
1132 }\r
1133 \r
1134 \r
1135 static cl_status_t\r
1136 __recv_ctor(\r
1137         IN                              void* const                                     p_object,\r
1138         IN                              void*                                           context,\r
1139                 OUT                     cl_pool_item_t** const          pp_pool_item )\r
1140 {\r
1141         ipoib_recv_desc_t       *p_desc;\r
1142         ipoib_port_t            *p_port;\r
1143         uint32_t                        ds0_len;\r
1144 \r
1145         IPOIB_ENTER( IPOIB_DBG_ALLOC );\r
1146 \r
1147         CL_ASSERT( p_object );\r
1148         CL_ASSERT( context );\r
1149 \r
1150         p_desc = (ipoib_recv_desc_t*)p_object;\r
1151         p_port = (ipoib_port_t*)context;\r
1152 \r
1153         /* Setup the work request. */\r
1154         p_desc->wr.ds_array = p_desc->local_ds;\r
1155         p_desc->wr.wr_id = (uintn_t)p_desc;\r
1156 \r
1157 #if IPOIB_INLINE_RECV\r
1158         /* Sanity check on the receive buffer layout */\r
1159         CL_ASSERT( (void*)&p_desc->buf.eth.pkt.type ==\r
1160                 (void*)&p_desc->buf.ib.pkt.type );\r
1161         CL_ASSERT( sizeof(recv_buf_t) == sizeof(ipoib_pkt_t) + sizeof(ib_grh_t) );\r
1162 \r
1163         /* Setup the local data segment. */\r
1164         p_desc->local_ds[0].vaddr = cl_get_physaddr( &p_desc->buf );\r
1165         p_desc->local_ds[0].lkey = p_port->ib_mgr.lkey;\r
1166         ds0_len =\r
1167                 PAGE_SIZE - ((uint32_t)p_desc->local_ds[0].vaddr & (PAGE_SIZE - 1));\r
1168         if( ds0_len >= sizeof(recv_buf_t) )\r
1169         {\r
1170                 /* The whole buffer is within a page. */\r
1171                 p_desc->local_ds[0].length = ds0_len;\r
1172                 p_desc->wr.num_ds = 1;\r
1173         }\r
1174         else\r
1175         {\r
1176                 /* The buffer crosses page boundaries. */\r
1177                 p_desc->local_ds[0].length = ds0_len;\r
1178                 p_desc->local_ds[1].vaddr = cl_get_physaddr( \r
1179                         ((uint8_t*)&p_desc->buf) + ds0_len );\r
1180                 p_desc->local_ds[1].lkey = p_port->ib_mgr.lkey;\r
1181                 p_desc->local_ds[1].length = sizeof(recv_buf_t) - ds0_len;\r
1182                 p_desc->wr.num_ds = 2;\r
1183         }\r
1184 #else   /* IPOIB_INLINE_RECV */\r
1185         /* Allocate the receive buffer. */\r
1186         p_desc->p_buf = (recv_buf_t*)cl_zalloc( sizeof(recv_buf_t) );\r
1187         if( !p_desc->p_buf )\r
1188         {\r
1189                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1190                         ("Failed to allocate receive buffer.\n") );\r
1191                 return CL_INSUFFICIENT_MEMORY;\r
1192         }\r
1193 \r
1194         /* Sanity check on the receive buffer layout */\r
1195         CL_ASSERT( (void*)&p_desc->p_buf->eth.pkt.type ==\r
1196                 (void*)&p_desc->p_buf->ib.pkt.type );\r
1197 \r
1198         /* Setup the local data segment. */\r
1199         p_desc->local_ds[0].vaddr = cl_get_physaddr( p_desc->p_buf );\r
1200         p_desc->local_ds[0].length = sizeof(ipoib_pkt_t) + sizeof(ib_grh_t);\r
1201         p_desc->local_ds[0].lkey = p_port->ib_mgr.lkey;\r
1202 #endif  /* IPOIB_INLINE_RECV */\r
1203 \r
1204         *pp_pool_item = &p_desc->item;\r
1205 \r
1206         IPOIB_EXIT( IPOIB_DBG_ALLOC );\r
1207         return CL_SUCCESS;\r
1208 }\r
1209 \r
1210 \r
1211 #if !IPOIB_INLINE_RECV\r
1212 static void\r
1213 __recv_dtor(\r
1214         IN              const   cl_pool_item_t* const           p_pool_item,\r
1215         IN                              void                                            *context )\r
1216 {\r
1217         ipoib_recv_desc_t       *p_desc;\r
1218 \r
1219         IPOIB_ENTER(  IPOIB_DBG_ALLOC );\r
1220 \r
1221         UNUSED_PARAM( context );\r
1222 \r
1223         p_desc = PARENT_STRUCT( p_pool_item, ipoib_recv_desc_t, item );\r
1224 \r
1225         if( p_desc->p_buf )\r
1226                 cl_free( p_desc->p_buf );\r
1227 \r
1228         IPOIB_EXIT( IPOIB_DBG_ALLOC );\r
1229 }\r
1230 #endif\r
1231 \r
1232 \r
1233 static inline ipoib_recv_desc_t*\r
1234 __buf_mgr_get_recv(\r
1235         IN                              ipoib_port_t* const                     p_port )\r
1236 {\r
1237         ipoib_recv_desc_t       *p_desc;\r
1238         IPOIB_ENTER( IPOIB_DBG_RECV );\r
1239         p_desc = (ipoib_recv_desc_t*)cl_qpool_get( &p_port->buf_mgr.recv_pool );\r
1240         /* Reference the port object for the send. */\r
1241         if( p_desc )\r
1242         {\r
1243                 ipoib_port_ref( p_port, ref_get_recv );\r
1244                 CL_ASSERT( p_desc->wr.wr_id == (uintn_t)p_desc );\r
1245 #if IPOIB_INLINE_RECV\r
1246                 CL_ASSERT( p_desc->local_ds[0].vaddr ==\r
1247                         cl_get_physaddr( &p_desc->buf ) );\r
1248 #else   /* IPOIB_INLINE_RECV */\r
1249                 CL_ASSERT( p_desc->local_ds[0].vaddr ==\r
1250                         cl_get_physaddr( p_desc->p_buf ) );\r
1251                 CL_ASSERT( p_desc->local_ds[0].length ==\r
1252                         (sizeof(ipoib_pkt_t) + sizeof(ib_grh_t)) );\r
1253 #endif  /* IPOIB_INLINE_RECV */\r
1254                 CL_ASSERT( p_desc->local_ds[0].lkey == p_port->ib_mgr.lkey );\r
1255         }\r
1256         IPOIB_EXIT( IPOIB_DBG_RECV );\r
1257         return p_desc;\r
1258 }\r
1259 \r
1260 \r
1261 static inline void\r
1262 __buf_mgr_put_recv(\r
1263         IN                              ipoib_port_t* const                     p_port,\r
1264         IN                              ipoib_recv_desc_t* const        p_desc,\r
1265         IN                              NDIS_PACKET* const                      p_packet OPTIONAL )\r
1266 {\r
1267         NDIS_BUFFER             *p_buf;\r
1268 \r
1269         IPOIB_ENTER(IPOIB_DBG_RECV );\r
1270 \r
1271         if( p_packet )\r
1272         {\r
1273                 /* Unchain the NDIS buffer. */\r
1274                 NdisUnchainBufferAtFront( p_packet, &p_buf );\r
1275                 CL_ASSERT( p_buf );\r
1276                 /* Return the NDIS packet and NDIS buffer to their pools. */\r
1277                 NdisDprFreePacketNonInterlocked( p_packet );\r
1278                 NdisFreeBuffer( p_buf );\r
1279         }\r
1280 \r
1281         /* Return the descriptor to its pools. */\r
1282         cl_qpool_put( &p_port->buf_mgr.recv_pool, &p_desc->item );\r
1283 \r
1284         /*\r
1285          * Dereference the port object since the receive is no longer outstanding.\r
1286          */\r
1287         ipoib_port_deref( p_port, ref_get_recv );\r
1288         IPOIB_EXIT(  IPOIB_DBG_RECV );\r
1289 }\r
1290 \r
1291 \r
1292 static inline void\r
1293 __buf_mgr_put_recv_list(\r
1294         IN                              ipoib_port_t* const                     p_port,\r
1295         IN                              cl_qlist_t* const                       p_list )\r
1296 {\r
1297         //IPOIB_ENTER(  IPOIB_DBG_RECV );\r
1298         cl_qpool_put_list( &p_port->buf_mgr.recv_pool, p_list );\r
1299         //IPOIB_EXIT(  IPOIB_DBG_RECV );\r
1300 }\r
1301 \r
1302 \r
1303 static inline NDIS_PACKET*\r
1304 __buf_mgr_get_ndis_pkt(\r
1305         IN                              ipoib_port_t* const                     p_port,\r
1306         IN                              ipoib_recv_desc_t* const        p_desc )\r
1307 {\r
1308         NDIS_STATUS                             status;\r
1309         NDIS_PACKET                             *p_packet;\r
1310         NDIS_BUFFER                             *p_buffer;\r
1311 \r
1312         IPOIB_ENTER(  IPOIB_DBG_RECV );\r
1313 \r
1314         NdisDprAllocatePacketNonInterlocked( &status, &p_packet,\r
1315                 p_port->buf_mgr.h_packet_pool );\r
1316         if( status != NDIS_STATUS_SUCCESS )\r
1317         {\r
1318                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1319                         ("Failed to allocate NDIS_PACKET: %08x\n", status) );\r
1320                 return NULL;\r
1321         }\r
1322 \r
1323         IPOIB_PORT_FROM_PACKET( p_packet ) = p_port;\r
1324         IPOIB_RECV_FROM_PACKET( p_packet ) = p_desc;\r
1325 \r
1326         NdisAllocateBuffer( &status, &p_buffer,\r
1327 #if IPOIB_INLINE_RECV\r
1328                 p_port->buf_mgr.h_buffer_pool, &p_desc->buf.eth.pkt, p_desc->len );\r
1329 #else   /* IPOIB_INLINE_RECV */\r
1330                 p_port->buf_mgr.h_buffer_pool, &p_desc->p_buf->eth.pkt, p_desc->len );\r
1331 #endif  /* IPOIB_INLINE_RECV */\r
1332         if( status != NDIS_STATUS_SUCCESS )\r
1333         {\r
1334                 IPOIB_PRINT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1335                         ("Failed to allocate NDIS_BUFFER: %08x\n", status) );\r
1336                 NdisDprFreePacketNonInterlocked( p_packet );\r
1337                 return NULL;\r
1338         }\r
1339 \r
1340         NdisChainBufferAtFront( p_packet, p_buffer );\r
1341         NDIS_SET_PACKET_HEADER_SIZE( p_packet, sizeof(eth_hdr_t) );\r
1342 \r
1343         IPOIB_EXIT(  IPOIB_DBG_RECV );\r
1344         return p_packet;\r
1345 }\r
1346 \r
1347 /******************************************************************************\r
1348 *\r
1349 * Receive manager implementation.\r
1350 *\r
1351 ******************************************************************************/\r
1352 static void\r
1353 __recv_mgr_construct(\r
1354         IN                              ipoib_port_t* const                     p_port )\r
1355 {\r
1356         IPOIB_ENTER( IPOIB_DBG_INIT );\r
1357 \r
1358         cl_qlist_init( &p_port->recv_mgr.done_list );\r
1359 \r
1360         p_port->recv_mgr.recv_pkt_array = NULL;\r
1361 \r
1362         IPOIB_EXIT( IPOIB_DBG_INIT );\r
1363 }\r
1364 \r
1365 \r
1366 static ib_api_status_t\r
1367 __recv_mgr_init(\r
1368         IN                              ipoib_port_t* const                     p_port )\r
1369 {\r
1370         IPOIB_ENTER( IPOIB_DBG_INIT );\r
1371 \r
1372         /* Allocate the NDIS_PACKET pointer array for indicating receives. */\r
1373         p_port->recv_mgr.recv_pkt_array = cl_malloc(\r
1374                 sizeof(NDIS_PACKET*) * p_port->p_adapter->params.rq_depth );\r
1375         if( !p_port->recv_mgr.recv_pkt_array )\r
1376         {\r
1377                 NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
1378                         EVENT_IPOIB_RECV_PKT_ARRAY, 0 );\r
1379                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1380                         ("cl_malloc for PNDIS_PACKET array failed.\n") );\r
1381                 return IB_INSUFFICIENT_MEMORY;\r
1382         }\r
1383 \r
1384         IPOIB_EXIT( IPOIB_DBG_INIT );\r
1385         return IB_SUCCESS;\r
1386 }\r
1387 \r
1388 \r
1389 static void\r
1390 __recv_mgr_destroy(\r
1391         IN                              ipoib_port_t* const                     p_port )\r
1392 {\r
1393         IPOIB_ENTER( IPOIB_DBG_INIT );\r
1394 \r
1395         CL_ASSERT( cl_is_qlist_empty( &p_port->recv_mgr.done_list ) );\r
1396         CL_ASSERT( !p_port->recv_mgr.depth );\r
1397 \r
1398         if( p_port->recv_mgr.recv_pkt_array )\r
1399                 cl_free( p_port->recv_mgr.recv_pkt_array );\r
1400 \r
1401         IPOIB_EXIT( IPOIB_DBG_INIT );\r
1402 }\r
1403 \r
1404 \r
1405 /*\r
1406  * Posts receive buffers to the receive queue and returns the number\r
1407  * of receives needed to bring the RQ to its low water mark.  Note\r
1408  * that the value is signed, and can go negative.  All tests must\r
1409  * be for > 0.\r
1410  */\r
1411 static int32_t\r
1412 __recv_mgr_repost(\r
1413         IN                              ipoib_port_t* const                     p_port )\r
1414 {\r
1415         ipoib_recv_desc_t       *p_head = NULL, *p_tail = NULL, *p_next;\r
1416         ib_api_status_t         status;\r
1417         ib_recv_wr_t            *p_failed;\r
1418         PERF_DECLARE( GetRecv );\r
1419         PERF_DECLARE( PostRecv );\r
1420 \r
1421         IPOIB_ENTER( IPOIB_DBG_RECV );\r
1422 \r
1423         CL_ASSERT( p_port );\r
1424         cl_obj_lock( &p_port->obj );\r
1425         if( p_port->state != IB_QPS_RTS )\r
1426         {\r
1427                 cl_obj_unlock( &p_port->obj );\r
1428                 IPOIB_PRINT_EXIT( TRACE_LEVEL_INFORMATION, IPOIB_DBG_RECV,\r
1429                         ("Port in invalid state.  Not reposting.\n") );\r
1430                 return 0;\r
1431         }\r
1432         ipoib_port_ref( p_port, ref_repost );\r
1433         cl_obj_unlock( &p_port->obj );\r
1434 \r
1435         while( p_port->recv_mgr.depth < p_port->p_adapter->params.rq_depth )\r
1436         {\r
1437                 /* Pull receives out of the pool and chain them up. */\r
1438                 cl_perf_start( GetRecv );\r
1439                 p_next = __buf_mgr_get_recv( p_port );\r
1440                 cl_perf_stop( &p_port->p_adapter->perf, GetRecv );\r
1441                 if( !p_next )\r
1442                 {\r
1443                         IPOIB_PRINT(TRACE_LEVEL_INFORMATION, IPOIB_DBG_RECV,\r
1444                                 ("Out of receive descriptors! recv queue depath 0x%x\n",p_port->recv_mgr.depth) );\r
1445                         break;\r
1446                 }\r
1447 \r
1448                 if( !p_tail )\r
1449                 {\r
1450                         p_tail = p_next;\r
1451                         p_next->wr.p_next = NULL;\r
1452                 }\r
1453                 else\r
1454                 {\r
1455                         p_next->wr.p_next = &p_head->wr;\r
1456                 }\r
1457 \r
1458                 p_head = p_next;\r
1459 \r
1460                 p_port->recv_mgr.depth++;\r
1461         }\r
1462 \r
1463         if( p_head )\r
1464         {\r
1465                 cl_perf_start( PostRecv );\r
1466                 status = p_port->p_adapter->p_ifc->post_recv(\r
1467                         p_port->ib_mgr.h_qp, &p_head->wr, &p_failed );\r
1468                 cl_perf_stop( &p_port->p_adapter->perf, PostRecv );\r
1469 \r
1470                 if( status != IB_SUCCESS )\r
1471                 {\r
1472                         IPOIB_PRINT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1473                                 ("ip_post_recv returned %s\n", \r
1474                                 p_port->p_adapter->p_ifc->get_err_str( status )) );\r
1475                         /* return the descriptors to the pool */\r
1476                         while( p_failed )\r
1477                         {\r
1478                                 p_head = PARENT_STRUCT( p_failed, ipoib_recv_desc_t, wr );\r
1479                                 p_failed = p_failed->p_next;\r
1480 \r
1481                                 __buf_mgr_put_recv( p_port, p_head, NULL );\r
1482                                 p_port->recv_mgr.depth--;\r
1483                         }\r
1484                 }\r
1485         }\r
1486 \r
1487         ipoib_port_deref( p_port, ref_repost );\r
1488         IPOIB_EXIT( IPOIB_DBG_RECV );\r
1489         return p_port->p_adapter->params.rq_low_watermark - p_port->recv_mgr.depth;\r
1490 }\r
1491 \r
1492 \r
1493 void\r
1494 ipoib_return_packet(\r
1495         IN                              NDIS_HANDLE                                     adapter_context,\r
1496         IN                              NDIS_PACKET                                     *p_packet )\r
1497 {\r
1498         cl_list_item_t          *p_item;\r
1499         ipoib_port_t            *p_port;\r
1500         ipoib_recv_desc_t       *p_desc;\r
1501         ib_api_status_t         status = IB_NOT_DONE;\r
1502         int32_t                         shortage;\r
1503         PERF_DECLARE( ReturnPacket );\r
1504         PERF_DECLARE( ReturnPutRecv );\r
1505         PERF_DECLARE( ReturnRepostRecv );\r
1506         PERF_DECLARE( ReturnPreparePkt );\r
1507         PERF_DECLARE( ReturnNdisIndicate );\r
1508 \r
1509         IPOIB_ENTER( IPOIB_DBG_RECV );\r
1510 \r
1511         UNUSED_PARAM( adapter_context );\r
1512         CL_ASSERT( p_packet );\r
1513 \r
1514         cl_perf_start( ReturnPacket );\r
1515 \r
1516         /* Get the port and descriptor from the packet. */\r
1517         p_port = IPOIB_PORT_FROM_PACKET( p_packet );\r
1518         p_desc = IPOIB_RECV_FROM_PACKET( p_packet );\r
1519 \r
1520         cl_spinlock_acquire( &p_port->recv_lock );\r
1521 \r
1522         cl_perf_start( ReturnPutRecv );\r
1523         __buf_mgr_put_recv( p_port, p_desc, p_packet );\r
1524         cl_perf_stop( &p_port->p_adapter->perf, ReturnPutRecv );\r
1525 \r
1526         /* Repost buffers. */\r
1527         cl_perf_start( ReturnRepostRecv );\r
1528         shortage = __recv_mgr_repost( p_port );\r
1529         cl_perf_stop( &p_port->p_adapter->perf, ReturnRepostRecv );\r
1530 \r
1531         for( p_item = cl_qlist_remove_head( &p_port->recv_mgr.done_list );\r
1532                 p_item != cl_qlist_end( &p_port->recv_mgr.done_list );\r
1533                 p_item = cl_qlist_remove_head( &p_port->recv_mgr.done_list ) )\r
1534         {\r
1535                 p_desc = (ipoib_recv_desc_t*)p_item;\r
1536 \r
1537                 cl_perf_start( ReturnPreparePkt );\r
1538                 status = __recv_mgr_prepare_pkt( p_port, p_desc, &p_packet );\r
1539                 cl_perf_stop( &p_port->p_adapter->perf, ReturnPreparePkt );\r
1540                 if( status == IB_SUCCESS )\r
1541                 {\r
1542                         if( shortage > 0 )\r
1543                                 NDIS_SET_PACKET_STATUS( p_packet, NDIS_STATUS_RESOURCES );\r
1544                         else\r
1545                                 NDIS_SET_PACKET_STATUS( p_packet, NDIS_STATUS_SUCCESS );\r
1546 \r
1547                         cl_spinlock_release( &p_port->recv_lock );\r
1548                         cl_perf_start( ReturnNdisIndicate );\r
1549                         NdisMIndicateReceivePacket( p_port->p_adapter->h_adapter,\r
1550                                 &p_packet, 1 );\r
1551                         cl_perf_stop( &p_port->p_adapter->perf, ReturnNdisIndicate );\r
1552                         cl_spinlock_acquire( &p_port->recv_lock );\r
1553 \r
1554                         if( shortage > 0 )\r
1555                         {\r
1556                                 cl_perf_start( ReturnPutRecv );\r
1557                                 __buf_mgr_put_recv( p_port, p_desc, p_packet );\r
1558                                 cl_perf_stop( &p_port->p_adapter->perf, ReturnPutRecv );\r
1559 \r
1560                                 /* Repost buffers. */\r
1561                                 cl_perf_start( ReturnRepostRecv );\r
1562                                 shortage = __recv_mgr_repost( p_port );\r
1563                                 cl_perf_stop( &p_port->p_adapter->perf, ReturnRepostRecv );\r
1564                         }\r
1565                 }\r
1566                 else if( status != IB_NOT_DONE )\r
1567                 {\r
1568                         IPOIB_PRINT(TRACE_LEVEL_INFORMATION, IPOIB_DBG_RECV,\r
1569                                 ("__recv_mgr_prepare_pkt returned %s\n",\r
1570                                 p_port->p_adapter->p_ifc->get_err_str( status )) );\r
1571                         /* Return the item to the head of the list. */\r
1572                         cl_qlist_insert_head( &p_port->recv_mgr.done_list, p_item );\r
1573                         break;\r
1574                 }\r
1575         }\r
1576         cl_spinlock_release( &p_port->recv_lock );\r
1577         cl_perf_stop( &p_port->p_adapter->perf, ReturnPacket );\r
1578 \r
1579         IPOIB_EXIT( IPOIB_DBG_RECV );\r
1580 }\r
1581 \r
1582 \r
1583 static void\r
1584 __recv_cb(\r
1585         IN              const   ib_cq_handle_t FUNC_PTR64                               h_cq,\r
1586         IN                              void                                            *cq_context )\r
1587 {\r
1588         ipoib_port_t            *p_port;\r
1589         ib_api_status_t         status;\r
1590         ib_wc_t                         wc[MAX_RECV_WC], *p_free, *p_wc;\r
1591         int32_t                         pkt_cnt, recv_cnt = 0, shortage, discarded;\r
1592         cl_qlist_t                      done_list, bad_list;\r
1593         size_t                          i;\r
1594         PERF_DECLARE( RecvCompBundle );\r
1595         PERF_DECLARE( RecvCb );\r
1596         PERF_DECLARE( PollRecv );\r
1597         PERF_DECLARE( RepostRecv );\r
1598         PERF_DECLARE( FilterRecv );\r
1599         PERF_DECLARE( BuildPktArray );\r
1600         PERF_DECLARE( RecvNdisIndicate );\r
1601         PERF_DECLARE( RearmRecv );\r
1602         PERF_DECLARE( PutRecvList );\r
1603 \r
1604         IPOIB_ENTER( IPOIB_DBG_RECV );\r
1605 \r
1606         cl_perf_clr( RecvCompBundle );\r
1607 \r
1608         cl_perf_start( RecvCb );\r
1609 \r
1610         UNUSED_PARAM( h_cq );\r
1611 \r
1612         p_port = (ipoib_port_t*)cq_context;\r
1613 \r
1614         cl_qlist_init( &done_list );\r
1615         cl_qlist_init( &bad_list );\r
1616 \r
1617         ipoib_port_ref( p_port, ref_recv_cb );\r
1618         for( i = 0; i < MAX_RECV_WC; i++ )\r
1619                 wc[i].p_next = &wc[i + 1];\r
1620         wc[MAX_RECV_WC - 1].p_next = NULL;\r
1621 \r
1622         /*\r
1623          * We'll be accessing the endpoint map so take a reference\r
1624          * on it to prevent modifications.\r
1625          */\r
1626         cl_obj_lock( &p_port->obj );\r
1627         cl_atomic_inc( &p_port->endpt_rdr );\r
1628         cl_obj_unlock( &p_port->obj );\r
1629 \r
1630         do\r
1631         {\r
1632                 /* If we get here, then the list of WCs is intact. */\r
1633                 p_free = wc;\r
1634 \r
1635                 cl_perf_start( PollRecv );\r
1636                 status = p_port->p_adapter->p_ifc->poll_cq(\r
1637                         p_port->ib_mgr.h_recv_cq, &p_free, &p_wc );\r
1638                 cl_perf_stop( &p_port->p_adapter->perf, PollRecv );\r
1639                 CL_ASSERT( status == IB_SUCCESS || status == IB_NOT_FOUND );\r
1640 \r
1641                 /* Look at the payload now and filter ARP and DHCP packets. */\r
1642                 cl_perf_start( FilterRecv );\r
1643                 recv_cnt += __recv_mgr_filter( p_port, p_wc, &done_list, &bad_list );\r
1644                 cl_perf_stop( &p_port->p_adapter->perf, FilterRecv );\r
1645 \r
1646         } while( !p_free );\r
1647 \r
1648         /* We're done looking at the endpoint map, release the reference. */\r
1649         cl_atomic_dec( &p_port->endpt_rdr );\r
1650 \r
1651         cl_perf_log( &p_port->p_adapter->perf, RecvCompBundle, recv_cnt );\r
1652 \r
1653         cl_spinlock_acquire( &p_port->recv_lock );\r
1654 \r
1655         /* Update our posted depth. */\r
1656         p_port->recv_mgr.depth -= recv_cnt;\r
1657 \r
1658         /* Return any discarded receives to the pool */\r
1659         cl_perf_start( PutRecvList );\r
1660         __buf_mgr_put_recv_list( p_port, &bad_list );\r
1661         cl_perf_stop( &p_port->p_adapter->perf, PutRecvList );\r
1662 \r
1663         do\r
1664         {\r
1665                 /* Repost ASAP so we don't starve the RQ. */\r
1666                 cl_perf_start( RepostRecv );\r
1667                 shortage = __recv_mgr_repost( p_port );\r
1668                 cl_perf_stop( &p_port->p_adapter->perf, RepostRecv );\r
1669 \r
1670                 cl_perf_start( BuildPktArray );\r
1671                 /* Notify NDIS of any and all possible receive buffers. */\r
1672                 pkt_cnt = __recv_mgr_build_pkt_array(\r
1673                         p_port, shortage, &done_list, &discarded );\r
1674                 cl_perf_stop( &p_port->p_adapter->perf, BuildPktArray );\r
1675 \r
1676                 /* Only indicate receives if we actually had any. */\r
1677                 if( discarded && shortage > 0 )\r
1678                 {\r
1679                         /* We may have thrown away packets, and have a shortage */\r
1680                         cl_perf_start( RepostRecv );\r
1681                         __recv_mgr_repost( p_port );\r
1682                         cl_perf_stop( &p_port->p_adapter->perf, RepostRecv );\r
1683                 }\r
1684 \r
1685                 if( !pkt_cnt )\r
1686                         break;\r
1687 \r
1688                 cl_spinlock_release( &p_port->recv_lock );\r
1689 \r
1690                 cl_perf_start( RecvNdisIndicate );\r
1691                 NdisMIndicateReceivePacket( p_port->p_adapter->h_adapter,\r
1692                         p_port->recv_mgr.recv_pkt_array, pkt_cnt );\r
1693                 cl_perf_stop( &p_port->p_adapter->perf, RecvNdisIndicate );\r
1694 \r
1695                 /*\r
1696                  * Cap the number of receives to put back to what we just indicated\r
1697                  * with NDIS_STATUS_RESOURCES.\r
1698                  */\r
1699                 if( shortage > 0 )\r
1700                 {\r
1701                         if( pkt_cnt < shortage )\r
1702                                 shortage = pkt_cnt;\r
1703 \r
1704                         /* Return all but the last packet to the pool. */\r
1705                         cl_spinlock_acquire( &p_port->recv_lock );\r
1706                         while( shortage-- > 1 )\r
1707                         {\r
1708                                 __buf_mgr_put_recv( p_port,\r
1709                                         IPOIB_RECV_FROM_PACKET( p_port->recv_mgr.recv_pkt_array[shortage] ),\r
1710                                         p_port->recv_mgr.recv_pkt_array[shortage] );\r
1711                         }\r
1712                         cl_spinlock_release( &p_port->recv_lock );\r
1713 \r
1714                         /*\r
1715                          * Return the last packet as if NDIS returned it, so that we repost\r
1716                          * and report any other pending receives.\r
1717                          */\r
1718                         ipoib_return_packet( NULL, p_port->recv_mgr.recv_pkt_array[0] );\r
1719                 }\r
1720                 cl_spinlock_acquire( &p_port->recv_lock );\r
1721 \r
1722         } while( pkt_cnt );\r
1723         cl_spinlock_release( &p_port->recv_lock );\r
1724 \r
1725         /*\r
1726          * Rearm after filtering to prevent contention on the enpoint maps\r
1727          * and eliminate the possibility of having a call to\r
1728          * __endpt_mgr_insert find a duplicate.\r
1729          */\r
1730         cl_perf_start( RearmRecv );\r
1731         status = p_port->p_adapter->p_ifc->rearm_cq(\r
1732                 p_port->ib_mgr.h_recv_cq, FALSE );\r
1733         cl_perf_stop( &p_port->p_adapter->perf, RearmRecv );\r
1734         CL_ASSERT( status == IB_SUCCESS );\r
1735 \r
1736         ipoib_port_deref( p_port, ref_recv_cb );\r
1737 \r
1738         cl_perf_stop( &p_port->p_adapter->perf, RecvCb );\r
1739 \r
1740         IPOIB_EXIT( IPOIB_DBG_RECV );\r
1741 }\r
1742 \r
1743 \r
1744 static void\r
1745 __recv_get_endpts(\r
1746         IN                              ipoib_port_t* const                     p_port,\r
1747         IN                              ipoib_recv_desc_t* const        p_desc,\r
1748         IN                              ib_wc_t* const                          p_wc,\r
1749                 OUT                     ipoib_endpt_t** const           pp_src,\r
1750                 OUT                     ipoib_endpt_t** const           pp_dst )\r
1751 {\r
1752         ib_api_status_t         status;\r
1753         mac_addr_t                      mac;\r
1754         PERF_DECLARE( GetEndptByGid );\r
1755         PERF_DECLARE( GetEndptByLid );\r
1756         PERF_DECLARE( EndptInsert );\r
1757 \r
1758         IPOIB_ENTER( IPOIB_DBG_RECV );\r
1759 \r
1760         /* Setup our shortcut pointers based on whether GRH is valid. */\r
1761         if( p_wc->recv.ud.recv_opt & IB_RECV_OPT_GRH_VALID )\r
1762         {\r
1763                 /* Lookup the source endpoints based on GID. */\r
1764                 cl_perf_start( GetEndptByGid );\r
1765                 *pp_src =\r
1766 #if IPOIB_INLINE_RECV\r
1767                         __endpt_mgr_get_by_gid( p_port, &p_desc->buf.ib.grh.src_gid );\r
1768 #else   /* IPOIB_INLINE_RECV */\r
1769                         __endpt_mgr_get_by_gid( p_port, &p_desc->p_buf->ib.grh.src_gid );\r
1770 #endif  /* IPOIB_INLINE_RECV */\r
1771                 cl_perf_stop( &p_port->p_adapter->perf, GetEndptByGid );\r
1772 \r
1773                 /*\r
1774                  * Lookup the destination endpoint based on GID.\r
1775                  * This is used along with the packet filter to determine\r
1776                  * whether to report this to NDIS.\r
1777                  */\r
1778                 cl_perf_start( GetEndptByGid );\r
1779                 *pp_dst =\r
1780 #if IPOIB_INLINE_RECV\r
1781                         __endpt_mgr_get_by_gid( p_port, &p_desc->buf.ib.grh.dest_gid );\r
1782 #else   /* IPOIB_INLINE_RECV */\r
1783                         __endpt_mgr_get_by_gid( p_port, &p_desc->p_buf->ib.grh.dest_gid );\r
1784 #endif  /* IPOIB_INLINE_RECV */\r
1785                 cl_perf_stop( &p_port->p_adapter->perf, GetEndptByGid );\r
1786 \r
1787                 /*\r
1788                  * Create the source endpoint if it does not exist.  Note that we\r
1789                  * can only do this for globally routed traffic since we need the\r
1790                  * information from the GRH to generate the MAC.\r
1791                  */\r
1792                 if( !*pp_src )\r
1793                 {\r
1794                         status = ipoib_mac_from_guid(\r
1795 #if IPOIB_INLINE_RECV\r
1796                                 p_desc->buf.ib.grh.src_gid.unicast.interface_id, &mac );\r
1797 #else   /* IPOIB_INLINE_RECV */\r
1798                                 p_desc->p_buf->ib.grh.src_gid.unicast.interface_id, &mac );\r
1799 #endif  /* IPOIB_INLINE_RECV */\r
1800                         if( status != IB_SUCCESS )\r
1801                         {\r
1802                                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1803                                         ("ipoib_mac_from_guid returned %s\n",\r
1804                                         p_port->p_adapter->p_ifc->get_err_str( status )) );\r
1805                                 return;\r
1806                         }\r
1807 \r
1808                         /* Create the endpoint. */\r
1809 #if IPOIB_INLINE_RECV\r
1810                         *pp_src = ipoib_endpt_create( &p_desc->buf.ib.grh.src_gid,\r
1811 #else   /* IPOIB_INLINE_RECV */\r
1812                         *pp_src = ipoib_endpt_create( &p_desc->p_buf->ib.grh.src_gid,\r
1813 #endif  /* IPOIB_INLINE_RECV */\r
1814                                 0, p_wc->recv.ud.remote_qp );\r
1815                         if( !*pp_src )\r
1816                         {\r
1817                                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1818                                         ("ipoib_endpt_create failed\n") );\r
1819                                 return;\r
1820                         }\r
1821                         cl_perf_start( EndptInsert );\r
1822                         cl_obj_lock( &p_port->obj );\r
1823                         status = __endpt_mgr_insert( p_port, mac, *pp_src );\r
1824                         if( status != IB_SUCCESS )\r
1825                         {\r
1826                                 cl_obj_unlock( &p_port->obj );\r
1827                                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1828                                         ("__endpt_mgr_insert returned %s\n",\r
1829                                         p_port->p_adapter->p_ifc->get_err_str( status )) );\r
1830                                 return;\r
1831                         }\r
1832                         cl_obj_unlock( &p_port->obj );\r
1833                         cl_perf_stop( &p_port->p_adapter->perf, EndptInsert );\r
1834                 }\r
1835         }\r
1836         else\r
1837         {\r
1838                 /*\r
1839                  * Lookup the remote endpoint based on LID.  Note that only\r
1840                  * unicast traffic can be LID routed.\r
1841                  */\r
1842                 cl_perf_start( GetEndptByLid );\r
1843                 *pp_src = __endpt_mgr_get_by_lid( p_port, p_wc->recv.ud.remote_lid );\r
1844                 cl_perf_stop( &p_port->p_adapter->perf, GetEndptByLid );\r
1845                 *pp_dst = p_port->p_local_endpt;\r
1846                 CL_ASSERT( *pp_dst );\r
1847         }\r
1848 \r
1849         if( *pp_src && !ipoib_is_voltaire_router_gid( &(*pp_src)->dgid ) &&\r
1850                 (*pp_src)->qpn != p_wc->recv.ud.remote_qp )\r
1851         {\r
1852                 /* Update the QPN for the endpoint. */\r
1853                 IPOIB_PRINT(TRACE_LEVEL_INFORMATION, IPOIB_DBG_RECV,\r
1854                         ("Updating QPN for MAC: %02X-%02X-%02X-%02X-%02X-%02X\n",\r
1855                         (*pp_src )->mac.addr[0], (*pp_src )->mac.addr[1],\r
1856                         (*pp_src )->mac.addr[2], (*pp_src )->mac.addr[3],\r
1857                         (*pp_src )->mac.addr[4], (*pp_src )->mac.addr[5]) );\r
1858                 (*pp_src)->qpn = p_wc->recv.ud.remote_qp;\r
1859         }\r
1860 \r
1861         if( *pp_src && *pp_dst )\r
1862         {\r
1863                 IPOIB_PRINT(TRACE_LEVEL_INFORMATION, IPOIB_DBG_RECV,\r
1864                         ("Recv:\n"\r
1865                         "\tsrc MAC: %02X-%02X-%02X-%02X-%02X-%02X\n"\r
1866                         "\tdst MAC: %02X-%02X-%02X-%02X-%02X-%02X\n",\r
1867                         (*pp_src )->mac.addr[0], (*pp_src )->mac.addr[1],\r
1868                         (*pp_src )->mac.addr[2], (*pp_src )->mac.addr[3],\r
1869                         (*pp_src )->mac.addr[4], (*pp_src )->mac.addr[5],\r
1870                         (*pp_dst )->mac.addr[0], (*pp_dst )->mac.addr[1],\r
1871                         (*pp_dst )->mac.addr[2], (*pp_dst )->mac.addr[3],\r
1872                         (*pp_dst )->mac.addr[4], (*pp_dst )->mac.addr[5]) );\r
1873         }\r
1874 \r
1875         IPOIB_EXIT( IPOIB_DBG_RECV );\r
1876 }\r
1877 \r
1878 \r
1879 static int32_t\r
1880 __recv_mgr_filter(\r
1881         IN                              ipoib_port_t* const                     p_port,\r
1882         IN                              ib_wc_t* const                          p_done_wc_list,\r
1883                 OUT                     cl_qlist_t* const                       p_done_list,\r
1884                 OUT                     cl_qlist_t* const                       p_bad_list )\r
1885 {\r
1886         ipoib_recv_desc_t               *p_desc;\r
1887         ib_wc_t                                 *p_wc;\r
1888         ipoib_pkt_t                             *p_ipoib;\r
1889         eth_pkt_t                               *p_eth;\r
1890         ipoib_endpt_t                   *p_src, *p_dst;\r
1891         ib_api_status_t                 status;\r
1892         uint32_t                                len;\r
1893         int32_t                                 recv_cnt = 0;\r
1894         PERF_DECLARE( GetRecvEndpts );\r
1895         PERF_DECLARE( RecvGen );\r
1896         PERF_DECLARE( RecvTcp );\r
1897         PERF_DECLARE( RecvUdp );\r
1898         PERF_DECLARE( RecvDhcp );\r
1899         PERF_DECLARE( RecvArp );\r
1900 \r
1901         IPOIB_ENTER( IPOIB_DBG_RECV );\r
1902 \r
1903         for( p_wc = p_done_wc_list; p_wc; p_wc = p_wc->p_next )\r
1904         {\r
1905                 CL_ASSERT( p_wc->status != IB_WCS_SUCCESS || p_wc->wc_type == IB_WC_RECV );\r
1906                 p_desc = (ipoib_recv_desc_t*)(uintn_t)p_wc->wr_id;\r
1907                 recv_cnt++;\r
1908 \r
1909                 if( p_wc->status != IB_WCS_SUCCESS )\r
1910                 {\r
1911                         if( p_wc->status != IB_WCS_WR_FLUSHED_ERR )\r
1912                         {\r
1913                                 IPOIB_PRINT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1914                                         ("Failed completion %s  (vendor specific %#x)\n",\r
1915                                         p_port->p_adapter->p_ifc->get_wc_status_str( p_wc->status ),\r
1916                                         (int)p_wc->vendor_specific) );\r
1917                                 ipoib_inc_recv_stat( p_port->p_adapter, IP_STAT_ERROR, 0 );\r
1918                         }\r
1919                         else\r
1920                         {\r
1921                                 IPOIB_PRINT(TRACE_LEVEL_INFORMATION, IPOIB_DBG_RECV,\r
1922                                         ("Flushed completion %s\n",\r
1923                                         p_port->p_adapter->p_ifc->get_wc_status_str( p_wc->status )) );\r
1924                                 ipoib_inc_recv_stat( p_port->p_adapter, IP_STAT_DROPPED, 0 );\r
1925                         }\r
1926                         cl_qlist_insert_tail( p_bad_list, &p_desc->item.list_item );\r
1927                         /* Dereference the port object on behalf of the failed receive. */\r
1928                         ipoib_port_deref( p_port, ref_failed_recv_wc );\r
1929                         continue;\r
1930                 }\r
1931 \r
1932                 len = p_wc->length - sizeof(ib_grh_t);\r
1933 \r
1934                 if( len < sizeof(ipoib_hdr_t) )\r
1935                 {\r
1936                         IPOIB_PRINT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1937                                 ("Received ETH packet < min size\n") );\r
1938                         ipoib_inc_recv_stat( p_port->p_adapter, IP_STAT_ERROR, 0 );\r
1939                         cl_qlist_insert_tail( p_bad_list, &p_desc->item.list_item );\r
1940                         ipoib_port_deref( p_port, ref_recv_inv_len );\r
1941                         continue;\r
1942                 }\r
1943 \r
1944                 if((len - sizeof(ipoib_hdr_t)) > p_port->p_adapter->params.payload_mtu)\r
1945                 {\r
1946                         IPOIB_PRINT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1947                                 ("Received ETH packet > payload MTU (%d)\n",\r
1948                                 p_port->p_adapter->params.payload_mtu) );\r
1949                         ipoib_inc_recv_stat( p_port->p_adapter, IP_STAT_ERROR, 0 );\r
1950                         cl_qlist_insert_tail( p_bad_list, &p_desc->item.list_item );\r
1951                         ipoib_port_deref( p_port, ref_recv_inv_len );\r
1952                         continue;\r
1953                         \r
1954                 }\r
1955                 /* Successful completion.  Get the receive information. */\r
1956                 cl_perf_start( GetRecvEndpts );\r
1957                 __recv_get_endpts( p_port, p_desc, p_wc, &p_src, &p_dst );\r
1958                 cl_perf_stop( &p_port->p_adapter->perf, GetRecvEndpts );\r
1959 \r
1960 #if IPOIB_INLINE_RECV\r
1961                 p_ipoib = &p_desc->buf.ib.pkt;\r
1962                 p_eth = &p_desc->buf.eth.pkt;\r
1963 #else   /* IPOIB_INLINE_RECV */\r
1964                 p_ipoib = &p_desc->p_buf->ib.pkt;\r
1965                 p_eth = &p_desc->p_buf->eth.pkt;\r
1966 #endif  /*IPOIB_INLINE_RECV */\r
1967 \r
1968                 if( p_src )\r
1969                 {\r
1970                         /* Don't report loopback traffic - we requested SW loopback. */\r
1971                         if( !cl_memcmp( &p_port->p_adapter->params.conf_mac,\r
1972                                 &p_src->mac, sizeof(p_port->p_adapter->params.conf_mac) ) )\r
1973                         {\r
1974                                 /*\r
1975                                  * "This is not the packet you're looking for" - don't update\r
1976                                  * receive statistics, the packet never happened.\r
1977                                  */\r
1978                                 cl_qlist_insert_tail( p_bad_list, &p_desc->item.list_item );\r
1979                                 /* Dereference the port object on behalf of the failed recv. */\r
1980                                 ipoib_port_deref( p_port, ref_recv_loopback );\r
1981                                 continue;\r
1982                         }\r
1983                 }\r
1984 \r
1985                 switch( p_ipoib->hdr.type )\r
1986                 {\r
1987                 case ETH_PROT_TYPE_IP:\r
1988                         if( len < (sizeof(ipoib_hdr_t) + sizeof(ip_hdr_t)) )\r
1989                         {\r
1990                                 IPOIB_PRINT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1991                                         ("Received IP packet < min size\n") );\r
1992                                 status = IB_INVALID_SETTING;\r
1993                                 break;\r
1994                         }\r
1995 \r
1996                         if( p_ipoib->type.ip.hdr.offset ||\r
1997                                 p_ipoib->type.ip.hdr.prot != IP_PROT_UDP )\r
1998                         {\r
1999                                 /* Unfiltered.  Setup the ethernet header and report. */\r
2000                                 cl_perf_start( RecvTcp );\r
2001                                 status = __recv_gen( p_ipoib, p_eth, p_src, p_dst );\r
2002                                 cl_perf_stop( &p_port->p_adapter->perf, RecvTcp );\r
2003                                 break;\r
2004                         }\r
2005 \r
2006                         /* First packet of a UDP transfer. */\r
2007                         if( len <\r
2008                                 (sizeof(ipoib_hdr_t) + sizeof(ip_hdr_t) + sizeof(udp_hdr_t)) )\r
2009                         {\r
2010                                 IPOIB_PRINT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2011                                         ("Received UDP packet < min size\n") );\r
2012                                 status = IB_INVALID_SETTING;\r
2013                                 break;\r
2014                         }\r
2015 \r
2016                         /* Check if DHCP conversion is required. */\r
2017                         if( (p_ipoib->type.ip.prot.udp.hdr.dst_port == DHCP_PORT_SERVER &&\r
2018                                 p_ipoib->type.ip.prot.udp.hdr.src_port == DHCP_PORT_CLIENT) ||\r
2019                                 (p_ipoib->type.ip.prot.udp.hdr.dst_port == DHCP_PORT_CLIENT &&\r
2020                                 p_ipoib->type.ip.prot.udp.hdr.src_port == DHCP_PORT_SERVER) )\r
2021                         {\r
2022                                 if( len < (sizeof(ipoib_hdr_t) + sizeof(ip_hdr_t) +\r
2023                                         sizeof(udp_hdr_t) + DHCP_MIN_SIZE) )\r
2024                                 {\r
2025                                         IPOIB_PRINT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2026                                                 ("Received DHCP < min size\n") );\r
2027                                         status = IB_INVALID_SETTING;\r
2028                                         break;\r
2029                                 }\r
2030                                 if ((p_ipoib->type.ip.hdr.ver_hl & 0x0f) != 5 ) {\r
2031                                         // If there are IP options in this message, we are in trouble in any case\r
2032                                         status = IB_INVALID_SETTING;\r
2033                                         break;                                  \r
2034                                 }\r
2035                                 /* UDP packet with BOOTP ports in src/dst port numbers. */\r
2036                                 cl_perf_start( RecvDhcp );\r
2037                                 status = __recv_dhcp( p_port, p_ipoib, p_eth, p_src, p_dst );\r
2038                                 cl_perf_stop( &p_port->p_adapter->perf, RecvDhcp );\r
2039                         }\r
2040                         else\r
2041                         {\r
2042                                 /* Unfiltered.  Setup the ethernet header and report. */\r
2043                                 cl_perf_start( RecvUdp );\r
2044                                 status = __recv_gen( p_ipoib, p_eth, p_src, p_dst );\r
2045                                 cl_perf_stop( &p_port->p_adapter->perf, RecvUdp );\r
2046                         }\r
2047                         break;\r
2048 \r
2049                 case ETH_PROT_TYPE_ARP:\r
2050                         if( len < (sizeof(ipoib_hdr_t) + sizeof(ipoib_arp_pkt_t)) )\r
2051                         {\r
2052                                 IPOIB_PRINT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2053                                         ("Received ARP < min size\n") );\r
2054                                 status = IB_INVALID_SETTING;\r
2055                                 break;\r
2056                         }\r
2057                         cl_perf_start( RecvArp );\r
2058                         status = __recv_arp( p_port, p_wc, p_ipoib, p_eth, &p_src, p_dst );\r
2059                         cl_perf_stop( &p_port->p_adapter->perf, RecvArp );\r
2060                         len = sizeof(ipoib_hdr_t) + sizeof(arp_pkt_t);\r
2061                         break;\r
2062 \r
2063                 default:\r
2064                         /* Unfiltered.  Setup the ethernet header and report. */\r
2065                         cl_perf_start( RecvGen );\r
2066                         status = __recv_gen( p_ipoib, p_eth, p_src, p_dst );\r
2067                         cl_perf_stop( &p_port->p_adapter->perf, RecvGen );\r
2068                 }\r
2069 \r
2070                 if( status != IB_SUCCESS )\r
2071                 {\r
2072                         /* Update stats. */\r
2073                         ipoib_inc_recv_stat( p_port->p_adapter, IP_STAT_ERROR, 0 );\r
2074                         cl_qlist_insert_tail( p_bad_list, &p_desc->item.list_item );\r
2075                         /* Dereference the port object on behalf of the failed receive. */\r
2076                         ipoib_port_deref( p_port, ref_recv_filter );\r
2077                 }\r
2078                 else\r
2079                 {\r
2080                         p_desc->len =\r
2081                                 len + sizeof(eth_hdr_t) - sizeof(ipoib_hdr_t);\r
2082                         if( p_dst->h_mcast)\r
2083                         {\r
2084                                 if( p_dst->dgid.multicast.raw_group_id[10] == 0xFF &&\r
2085                                         p_dst->dgid.multicast.raw_group_id[11] == 0xFF &&\r
2086                                         p_dst->dgid.multicast.raw_group_id[12] == 0xFF &&\r
2087                                         p_dst->dgid.multicast.raw_group_id[13] == 0xFF )\r
2088                                 {\r
2089                                         p_desc->type = PKT_TYPE_BCAST;\r
2090                                 }\r
2091                                 else\r
2092                                 {\r
2093                                         p_desc->type = PKT_TYPE_MCAST;\r
2094                                 }\r
2095                         }\r
2096                         else\r
2097                         {\r
2098                                 p_desc->type = PKT_TYPE_UCAST;\r
2099                         }\r
2100                         cl_qlist_insert_tail( p_done_list, &p_desc->item.list_item );\r
2101                 }\r
2102         }\r
2103 \r
2104         IPOIB_EXIT( IPOIB_DBG_RECV );\r
2105         return recv_cnt;\r
2106 }\r
2107 \r
2108 \r
2109 static ib_api_status_t\r
2110 __recv_gen(\r
2111         IN              const   ipoib_pkt_t* const                      p_ipoib,\r
2112                 OUT                     eth_pkt_t* const                        p_eth,\r
2113         IN                              ipoib_endpt_t* const            p_src,\r
2114         IN                              ipoib_endpt_t* const            p_dst )\r
2115 {\r
2116         IPOIB_ENTER( IPOIB_DBG_RECV );\r
2117 \r
2118         if( !p_src || !p_dst )\r
2119         {\r
2120                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2121                         ("Received packet with no matching endpoints.\n") );\r
2122                 return IB_NOT_DONE;\r
2123         }\r
2124 \r
2125         /*\r
2126          * Fill in the ethernet header.  Note that doing so will overwrite\r
2127          * the IPoIB header, so start by moving the information from the IPoIB\r
2128          * header.\r
2129          */\r
2130         p_eth->hdr.type = p_ipoib->hdr.type;\r
2131         p_eth->hdr.src = p_src->mac;\r
2132         p_eth->hdr.dst = p_dst->mac;\r
2133 \r
2134         IPOIB_EXIT( IPOIB_DBG_RECV );\r
2135         return IB_SUCCESS;\r
2136 }\r
2137 \r
2138 \r
2139 static ib_api_status_t\r
2140 __recv_dhcp(\r
2141         IN                              ipoib_port_t* const                     p_port,\r
2142         IN              const   ipoib_pkt_t* const                      p_ipoib,\r
2143                 OUT                     eth_pkt_t* const                        p_eth,\r
2144         IN                              ipoib_endpt_t* const            p_src,\r
2145         IN                              ipoib_endpt_t* const            p_dst )\r
2146 {\r
2147         ib_api_status_t         status;\r
2148         dhcp_pkt_t                      *p_dhcp;\r
2149         uint8_t                         *p_option;\r
2150         uint8_t                         *p_cid = NULL;\r
2151         ib_gid_t                        gid;\r
2152         uint8_t                         msg = 0;\r
2153 \r
2154         IPOIB_ENTER( IPOIB_DBG_RECV );\r
2155 \r
2156         UNUSED_PARAM( p_port );\r
2157 \r
2158         /* Create the ethernet header. */\r
2159         status = __recv_gen( p_ipoib, p_eth, p_src, p_dst );\r
2160         if( status != IB_SUCCESS )\r
2161         {\r
2162                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2163                         ("__recv_gen returned %s.\n", \r
2164                         p_port->p_adapter->p_ifc->get_err_str( status )) );\r
2165                 return status;\r
2166         }\r
2167 \r
2168         /* Fixup the payload. */\r
2169         p_dhcp = &p_eth->type.ip.prot.udp.dhcp;\r
2170         if( p_dhcp->op != DHCP_REQUEST && p_dhcp->op != DHCP_REPLY )\r
2171         {\r
2172                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2173                         ("Invalid DHCP op code.\n") );\r
2174                 return IB_INVALID_SETTING;\r
2175         }\r
2176 \r
2177         /*\r
2178          * Find the client identifier option, making sure to skip\r
2179          * the "magic cookie".\r
2180          */\r
2181         p_option = &p_dhcp->options[0];\r
2182         if ( *(uint32_t *)p_option != DHCP_COOKIE )\r
2183         {\r
2184                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2185                         ("DHCP cookie corrupted.\n") );\r
2186                 return IB_INVALID_PARAMETER;\r
2187         }\r
2188 \r
2189         p_option = &p_dhcp->options[4];\r
2190         while( *p_option != DHCP_OPT_END && p_option < &p_dhcp->options[312] )\r
2191         {\r
2192                 switch( *p_option )\r
2193                 {\r
2194                 case DHCP_OPT_PAD:\r
2195                         p_option++;\r
2196                         break;\r
2197 \r
2198                 case DHCP_OPT_MSG:\r
2199                         msg = p_option[2];\r
2200                         p_option += 3;\r
2201                         break;\r
2202 \r
2203                 case DHCP_OPT_CLIENT_ID:\r
2204                         p_cid = p_option;\r
2205                         /* Fall through. */\r
2206 \r
2207                 default:\r
2208                         /*\r
2209                          * All other options have a length byte following the option code.\r
2210                          * Offset by the length to get to the next option.\r
2211                          */\r
2212                         p_option += (p_option[1] + 2);\r
2213                 }\r
2214         }\r
2215 \r
2216         switch( msg )\r
2217         {\r
2218         /* message from client */\r
2219         case DHCPDISCOVER:\r
2220         case DHCPREQUEST:\r
2221         case DHCPDECLINE:\r
2222         case DHCPRELEASE:\r
2223         case DHCPINFORM:\r
2224                 if( !p_cid )\r
2225                 {\r
2226                         IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2227                                 ("Failed to find required Client-identifier option.\n") );\r
2228                         return IB_INVALID_SETTING;\r
2229                 }\r
2230                 if( p_dhcp->htype != DHCP_HW_TYPE_IB )\r
2231                 {\r
2232                         IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2233                                 ("Invalid hardware address type.\n") );\r
2234                         return IB_INVALID_SETTING;\r
2235                 }\r
2236                 break;\r
2237         /* message from DHCP server */\r
2238         case DHCPOFFER:\r
2239         case DHCPACK:\r
2240         case DHCPNAK:\r
2241                 break;\r
2242 \r
2243         default:\r
2244                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2245                         ("Invalide message type.\n") );\r
2246                 return IB_INVALID_PARAMETER;\r
2247         }\r
2248         p_eth->type.ip.prot.udp.hdr.chksum = 0;\r
2249         p_dhcp->htype = DHCP_HW_TYPE_ETH;\r
2250         p_dhcp->hlen = HW_ADDR_LEN;\r
2251 \r
2252         if( p_cid ) /* from client */\r
2253         {\r
2254                 /* Validate that the length and type of the option is as required. */\r
2255                 if( p_cid[1] != 21 )\r
2256                 {\r
2257                         IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2258                                 ("Client-identifier length not 21 as required.\n") );\r
2259                         return IB_INVALID_SETTING;\r
2260                 }\r
2261                 if( p_cid[2] != DHCP_HW_TYPE_IB )\r
2262                 {\r
2263                         IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2264                                 ("Client-identifier type is wrong.\n") );\r
2265                         return IB_INVALID_SETTING;\r
2266                 }\r
2267                 /*\r
2268                  * Copy the GID value from the option so that we can make aligned\r
2269                  * accesses to the contents.\r
2270                  * Recover CID to standard type.\r
2271                  */\r
2272                 cl_memcpy( &gid, &p_cid[7], sizeof(ib_gid_t) );\r
2273                 p_cid[1] =  HW_ADDR_LEN +1;// CID length \r
2274                 p_cid[2] =  DHCP_HW_TYPE_ETH;// CID type \r
2275                 status = ipoib_mac_from_guid( gid.unicast.interface_id, (mac_addr_t*)&p_cid[3] );\r
2276                 p_cid[HW_ADDR_LEN + 3] = DHCP_OPT_END; //terminate tag\r
2277         }\r
2278         IPOIB_EXIT( IPOIB_DBG_RECV );\r
2279         return status;\r
2280 }\r
2281 \r
2282 \r
2283 static ib_api_status_t\r
2284 __recv_arp(\r
2285         IN                              ipoib_port_t* const                     p_port,\r
2286         IN                              ib_wc_t* const                          p_wc,\r
2287         IN              const   ipoib_pkt_t* const                      p_ipoib,\r
2288                 OUT                     eth_pkt_t* const                        p_eth,\r
2289         IN                              ipoib_endpt_t** const           pp_src,\r
2290         IN                              ipoib_endpt_t* const            p_dst )\r
2291 {\r
2292         ib_api_status_t                 status;\r
2293         arp_pkt_t                               *p_arp;\r
2294         const ipoib_arp_pkt_t   *p_ib_arp;\r
2295         ib_gid_t                                gid;\r
2296         mac_addr_t                              mac;\r
2297         ipoib_hw_addr_t                 null_hw = {0};\r
2298 \r
2299         IPOIB_ENTER( IPOIB_DBG_RECV );\r
2300 \r
2301         if( !p_dst )\r
2302         {\r
2303                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2304                         ("Unknown destination endpoint\n") );\r
2305                 return IB_INVALID_SETTING;\r
2306         }\r
2307 \r
2308         p_ib_arp = &p_ipoib->type.arp;\r
2309         p_arp = &p_eth->type.arp;\r
2310 \r
2311         if( p_ib_arp->hw_type != ARP_HW_TYPE_IB )\r
2312         {\r
2313                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2314                         ("ARP hardware type is not IB\n") );\r
2315                 return IB_INVALID_SETTING;\r
2316         }\r
2317 \r
2318         if( p_ib_arp->hw_size != sizeof(ipoib_hw_addr_t) )\r
2319         {\r
2320                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2321                         ("ARP hardware address size is not sizeof(ipoib_hw_addr_t)\n") );\r
2322                 return IB_INVALID_SETTING;\r
2323         }\r
2324 \r
2325         if( p_ib_arp->prot_type != ETH_PROT_TYPE_IP )\r
2326         {\r
2327                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2328                         ("ARP protocal type not IP\n") );\r
2329                 return IB_INVALID_SETTING;\r
2330         }\r
2331 \r
2332         /*\r
2333          * If we don't have a source, lookup the endpoint specified in the payload.\r
2334          */\r
2335         if( !*pp_src )\r
2336                 *pp_src = __endpt_mgr_get_by_gid( p_port, &p_ib_arp->src_hw.gid );\r
2337 \r
2338         /*\r
2339          * If the endpoint exists for the GID, make sure\r
2340          * the dlid and qpn match the arp.\r
2341          */\r
2342         if( *pp_src )\r
2343         {\r
2344                 if( cl_memcmp( &(*pp_src)->dgid, &p_ib_arp->src_hw.gid,\r
2345                         sizeof(ib_gid_t) ) )\r
2346                 {\r
2347                         /*\r
2348                          * GIDs for the endpoint are different.  The ARP must\r
2349                          * have been proxied.  Dereference it.\r
2350                          */\r
2351                         *pp_src = NULL;\r
2352                 }\r
2353                 else if( (*pp_src)->dlid &&\r
2354                         (*pp_src)->dlid != p_wc->recv.ud.remote_lid )\r
2355                 {\r
2356                         /* Out of date!  Destroy the endpoint and replace it. */\r
2357                         __endpt_mgr_remove( p_port, *pp_src );\r
2358                         *pp_src = NULL;\r
2359                 }\r
2360                 else if( ipoib_is_voltaire_router_gid( &(*pp_src)->dgid ) )\r
2361                 {\r
2362                         if( (*pp_src)->qpn !=\r
2363                                 (p_ib_arp->src_hw.flags_qpn & CL_HTON32(0x00FFFFFF)) &&\r
2364                                 p_wc->recv.ud.remote_qp !=\r
2365                                 (p_ib_arp->src_hw.flags_qpn & CL_HTON32(0x00FFFFFF)) )\r
2366                         {\r
2367                                 /* Out of date!  Destroy the endpoint and replace it. */\r
2368                                 __endpt_mgr_remove( p_port, *pp_src );\r
2369                                 *pp_src = NULL;\r
2370                         }\r
2371                 }\r
2372                 else if( (*pp_src)->qpn != p_wc->recv.ud.remote_qp )\r
2373                 {\r
2374                         /* Out of date!  Destroy the endpoint and replace it. */\r
2375                         __endpt_mgr_remove( p_port, *pp_src );\r
2376                         *pp_src = NULL;\r
2377                 }\r
2378         }\r
2379 \r
2380         /* Do we need to create an endpoint for this GID? */\r
2381         if( !*pp_src )\r
2382         {\r
2383                 /* Copy the src GID to allow aligned access */\r
2384                 cl_memcpy( &gid, &p_ib_arp->src_hw.gid, sizeof(ib_gid_t) );\r
2385                 status = ipoib_mac_from_guid( gid.unicast.interface_id, &mac );\r
2386                 if( status != IB_SUCCESS )\r
2387                 {\r
2388                         IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2389                                 ("ipoib_mac_from_guid returned %s\n",\r
2390                                 p_port->p_adapter->p_ifc->get_err_str( status )) );\r
2391                         return status;\r
2392                 }\r
2393                 /*\r
2394                  * Create the endpoint.  Note that the LID is left blank and will be\r
2395                  * resolved by a path query as needed.  This is done because the\r
2396                  * remote LID/GID from the work completion may not be the original\r
2397                  * initiator.\r
2398                  */\r
2399                 *pp_src = ipoib_endpt_create( &p_ib_arp->src_hw.gid,\r
2400                         0, (p_ib_arp->src_hw.flags_qpn & CL_HTON32(0x00FFFFFF)) );\r
2401 \r
2402                 if( !*pp_src )\r
2403                 {\r
2404                         IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2405                                 ("ipoib_endpt_create failed\n") );\r
2406                         return status;\r
2407                 }\r
2408 \r
2409                 cl_obj_lock( &p_port->obj );\r
2410                 status = __endpt_mgr_insert( p_port, mac, *pp_src );\r
2411                 if( status != IB_SUCCESS )\r
2412                 {\r
2413                         cl_obj_unlock( &p_port->obj );\r
2414                         IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2415                                 ("__endpt_mgr_insert return %s \n",\r
2416                                 p_port->p_adapter->p_ifc->get_err_str( status )) );\r
2417                         return status;\r
2418                 }\r
2419 \r
2420                 cl_obj_unlock( &p_port->obj );\r
2421         }\r
2422 \r
2423         CL_ASSERT( !cl_memcmp(\r
2424                 &(*pp_src)->dgid, &p_ib_arp->src_hw.gid, sizeof(ib_gid_t) ) );\r
2425         CL_ASSERT( ipoib_is_voltaire_router_gid( &(*pp_src)->dgid ) ||\r
2426                 (*pp_src)->qpn ==\r
2427                 (p_ib_arp->src_hw.flags_qpn & CL_HTON32(0x00FFFFFF)) );\r
2428         /* Now swizzle the data. */\r
2429         p_arp->hw_type = ARP_HW_TYPE_ETH;\r
2430         p_arp->hw_size = sizeof(mac_addr_t);\r
2431         p_arp->src_hw = (*pp_src)->mac;\r
2432         p_arp->src_ip = p_ib_arp->src_ip;\r
2433 \r
2434         if( cl_memcmp( &p_ib_arp->dst_hw, &null_hw, sizeof(ipoib_hw_addr_t) ) )\r
2435         {\r
2436                 if( cl_memcmp( &p_dst->dgid, &p_ib_arp->dst_hw.gid, sizeof(ib_gid_t) ) )\r
2437                 {\r
2438                         /*\r
2439                          * We received bcast ARP packet that means\r
2440                          * remote port lets everyone know it was changed IP/MAC\r
2441                          * or just activated\r
2442                          */\r
2443 \r
2444                         /* Guy: TODO: Check why this check fails in case of Voltaire IPR */\r
2445 \r
2446                         if ( !ipoib_is_voltaire_router_gid( &(*pp_src)->dgid ) &&\r
2447                                  !ib_gid_is_multicast( (const ib_gid_t*)&p_dst->dgid ) )\r
2448                         {\r
2449                                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2450                                         ("ARP: is not ARP MCAST\n") );\r
2451                                 return IB_INVALID_SETTING;\r
2452                         }\r
2453 \r
2454                         p_arp->dst_hw = p_port->p_local_endpt->mac;\r
2455                         p_dst->mac = p_port->p_local_endpt->mac;\r
2456                         /*\r
2457                          * we don't care what receiver ip addr is,\r
2458                          * as long as OS' ARP table is global  ???\r
2459                          */\r
2460                         p_arp->dst_ip = (net32_t)0;\r
2461                 }\r
2462                 else /* we've got reply to our ARP request */\r
2463                 {\r
2464                         p_arp->dst_hw = p_dst->mac;\r
2465                         p_arp->dst_ip = p_ib_arp->dst_ip;\r
2466                         CL_ASSERT( p_dst->qpn == \r
2467                                 (p_ib_arp->dst_hw.flags_qpn & CL_HTON32(0x00FFFFFF)) );\r
2468                 }\r
2469         }\r
2470         else /* we got ARP reqeust */\r
2471         {\r
2472                 cl_memclr( &p_arp->dst_hw, sizeof(mac_addr_t) );\r
2473                 p_arp->dst_ip = p_ib_arp->dst_ip;\r
2474         }\r
2475 \r
2476         /*\r
2477          * Create the ethernet header.  Note that this is done last so that\r
2478          * we have a chance to create a new endpoint.\r
2479          */\r
2480         status = __recv_gen( p_ipoib, p_eth, *pp_src, p_dst );\r
2481         if( status != IB_SUCCESS )\r
2482         {\r
2483                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2484                         ("__recv_gen returned %s.\n", \r
2485                         p_port->p_adapter->p_ifc->get_err_str( status )) );\r
2486                 return status;\r
2487         }\r
2488 \r
2489         IPOIB_EXIT( IPOIB_DBG_RECV );\r
2490         return IB_SUCCESS;\r
2491 }\r
2492 \r
2493 \r
2494 static ib_api_status_t\r
2495 __recv_mgr_prepare_pkt(\r
2496         IN                              ipoib_port_t* const                     p_port,\r
2497         IN                              ipoib_recv_desc_t* const        p_desc,\r
2498                 OUT                     NDIS_PACKET** const                     pp_packet )\r
2499 {\r
2500         NDIS_STATUS                                                     status;\r
2501         uint32_t                                                        pkt_filter;\r
2502         ip_stat_sel_t                                           type;\r
2503         NDIS_TCP_IP_CHECKSUM_PACKET_INFO        chksum;\r
2504         PERF_DECLARE( GetNdisPkt );\r
2505 \r
2506         IPOIB_ENTER( IPOIB_DBG_RECV );\r
2507 \r
2508         pkt_filter = p_port->p_adapter->packet_filter;\r
2509         /* Check the packet filter. */\r
2510         switch( p_desc->type )\r
2511         {\r
2512         default:\r
2513         case PKT_TYPE_UCAST:\r
2514                 if( pkt_filter & NDIS_PACKET_TYPE_PROMISCUOUS ||\r
2515                         pkt_filter & NDIS_PACKET_TYPE_ALL_FUNCTIONAL ||\r
2516                         pkt_filter & NDIS_PACKET_TYPE_SOURCE_ROUTING ||\r
2517                         pkt_filter & NDIS_PACKET_TYPE_DIRECTED )\r
2518                 {\r
2519                         /* OK to report. */\r
2520                         type = IP_STAT_UCAST_BYTES;\r
2521                         status = NDIS_STATUS_SUCCESS;\r
2522                 }\r
2523                 else\r
2524                 {\r
2525                         type = IP_STAT_DROPPED;\r
2526                         status = NDIS_STATUS_FAILURE;\r
2527                 }\r
2528                 break;\r
2529         case PKT_TYPE_BCAST:\r
2530                 if( pkt_filter & NDIS_PACKET_TYPE_PROMISCUOUS ||\r
2531                         pkt_filter & NDIS_PACKET_TYPE_BROADCAST )\r
2532                 {\r
2533                         /* OK to report. */\r
2534                         type = IP_STAT_BCAST_BYTES;\r
2535                         status = NDIS_STATUS_SUCCESS;\r
2536                 }\r
2537                 else\r
2538                 {\r
2539                         type = IP_STAT_DROPPED;\r
2540                         status = NDIS_STATUS_FAILURE;\r
2541                 }\r
2542                 break;\r
2543         case PKT_TYPE_MCAST:\r
2544                 if( pkt_filter & NDIS_PACKET_TYPE_PROMISCUOUS ||\r
2545                         pkt_filter & NDIS_PACKET_TYPE_ALL_MULTICAST ||\r
2546                         pkt_filter & NDIS_PACKET_TYPE_MULTICAST )\r
2547                 {\r
2548                         /* OK to report. */\r
2549                         type = IP_STAT_MCAST_BYTES;\r
2550                         status = NDIS_STATUS_SUCCESS;\r
2551                 }\r
2552                 else\r
2553                 {\r
2554                         type = IP_STAT_DROPPED;\r
2555                         status = NDIS_STATUS_FAILURE;\r
2556                 }\r
2557                 break;\r
2558         }\r
2559 \r
2560         if( status != NDIS_STATUS_SUCCESS )\r
2561         {\r
2562                 ipoib_inc_recv_stat( p_port->p_adapter, type, 0 );\r
2563                 /* Return the receive descriptor to the pool. */\r
2564                 __buf_mgr_put_recv( p_port, p_desc, NULL );\r
2565                 IPOIB_PRINT_EXIT( TRACE_LEVEL_INFORMATION, IPOIB_DBG_RECV,\r
2566                         ("Packet filter doesn't match receive.  Dropping.\n") );\r
2567                 /*\r
2568                  * Return IB_NOT_DONE since the packet has been completed,\r
2569                  * but has not consumed an array entry.\r
2570                  */\r
2571                 return IB_NOT_DONE;\r
2572         }\r
2573 \r
2574         cl_perf_start( GetNdisPkt );\r
2575         *pp_packet = __buf_mgr_get_ndis_pkt( p_port, p_desc );\r
2576         cl_perf_stop( &p_port->p_adapter->perf, GetNdisPkt );\r
2577         if( !*pp_packet )\r
2578         {\r
2579                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2580                         ("__buf_mgr_get_ndis_pkt failed\n") );\r
2581                 return IB_INSUFFICIENT_RESOURCES;\r
2582         }\r
2583 \r
2584         /* Flag the checksums as having been calculated. */\r
2585         chksum.Value = 0;\r
2586         chksum.Receive.NdisPacketTcpChecksumSucceeded = TRUE;\r
2587         chksum.Receive.NdisPacketUdpChecksumSucceeded = TRUE;\r
2588         chksum.Receive.NdisPacketIpChecksumSucceeded = TRUE;\r
2589         NDIS_PER_PACKET_INFO_FROM_PACKET( *pp_packet, TcpIpChecksumPacketInfo ) =\r
2590                 (void*)(uintn_t)chksum.Value;\r
2591 \r
2592         ipoib_inc_recv_stat( p_port->p_adapter, type, p_desc->len );\r
2593 \r
2594         IPOIB_EXIT( IPOIB_DBG_RECV );\r
2595         return IB_SUCCESS;\r
2596 }\r
2597 \r
2598 \r
2599 static uint32_t\r
2600 __recv_mgr_build_pkt_array(\r
2601         IN                              ipoib_port_t* const                     p_port,\r
2602         IN                              int32_t                                         shortage,\r
2603                 OUT                     cl_qlist_t* const                       p_done_list,\r
2604                 OUT                     int32_t* const                          p_discarded )\r
2605 {\r
2606         cl_list_item_t                  *p_item;\r
2607         ipoib_recv_desc_t               *p_desc;\r
2608         uint32_t                                i = 0;\r
2609         ib_api_status_t                 status;\r
2610         PERF_DECLARE( PreparePkt );\r
2611 \r
2612         IPOIB_ENTER( IPOIB_DBG_RECV );\r
2613 \r
2614         *p_discarded = 0;\r
2615 \r
2616         /* Move any existing receives to the head to preserve ordering. */\r
2617         cl_qlist_insert_list_head( p_done_list, &p_port->recv_mgr.done_list );\r
2618         p_item = cl_qlist_remove_head( p_done_list );\r
2619         while( p_item != cl_qlist_end( p_done_list ) )\r
2620         {\r
2621                 p_desc = (ipoib_recv_desc_t*)p_item;\r
2622 \r
2623                 cl_perf_start( PreparePkt );\r
2624                 status = __recv_mgr_prepare_pkt( p_port, p_desc,\r
2625                         &p_port->recv_mgr.recv_pkt_array[i] );\r
2626                 cl_perf_stop( &p_port->p_adapter->perf, PreparePkt );\r
2627                 if( status == IB_SUCCESS )\r
2628                 {\r
2629                         CL_ASSERT( p_port->recv_mgr.recv_pkt_array[i] );\r
2630                         if( shortage-- > 0 )\r
2631                         {\r
2632                                 NDIS_SET_PACKET_STATUS(\r
2633                                         p_port->recv_mgr.recv_pkt_array[i], NDIS_STATUS_RESOURCES );\r
2634                         }\r
2635                         else\r
2636                         {\r
2637                                 NDIS_SET_PACKET_STATUS(\r
2638                                         p_port->recv_mgr.recv_pkt_array[i], NDIS_STATUS_SUCCESS );\r
2639                         }\r
2640                         i++;\r
2641                 }\r
2642                 else if( status == IB_NOT_DONE )\r
2643                 {\r
2644                         (*p_discarded)++;\r
2645                 }\r
2646                 else\r
2647                 {\r
2648                         IPOIB_PRINT(TRACE_LEVEL_INFORMATION, IPOIB_DBG_RECV,\r
2649                                 ("__recv_mgr_prepare_pkt returned %s\n",\r
2650                                 p_port->p_adapter->p_ifc->get_err_str( status )) );\r
2651                         /* Put all completed receives on the port's done list. */\r
2652                         cl_qlist_insert_tail( &p_port->recv_mgr.done_list, p_item );\r
2653                         cl_qlist_insert_list_tail( &p_port->recv_mgr.done_list, p_done_list );\r
2654                         break;\r
2655                 }\r
2656 \r
2657                 p_item = cl_qlist_remove_head( p_done_list );\r
2658         }\r
2659 \r
2660         IPOIB_EXIT( IPOIB_DBG_RECV );\r
2661         return i;\r
2662 }\r
2663 \r
2664 \r
2665 \r
2666 \r
2667 /******************************************************************************\r
2668 *\r
2669 * Send manager implementation.\r
2670 *\r
2671 ******************************************************************************/\r
2672 static void\r
2673 __send_mgr_construct(\r
2674         IN                              ipoib_port_t* const                     p_port )\r
2675 {\r
2676         IPOIB_ENTER( IPOIB_DBG_SEND );\r
2677         p_port->send_mgr.depth = 0;\r
2678         cl_qlist_init( &p_port->send_mgr.pending_list );\r
2679         IPOIB_EXIT( IPOIB_DBG_SEND );\r
2680 }\r
2681 \r
2682 \r
2683 static void \r
2684 __pending_list_destroy(\r
2685         IN                              ipoib_port_t* const                     p_port )\r
2686 {\r
2687         cl_list_item_t  *p_item;\r
2688         NDIS_PACKET             *p_packet;\r
2689         \r
2690         cl_spinlock_acquire( &p_port->send_lock );\r
2691         /* Complete any pending packets. */\r
2692         for( p_item = cl_qlist_remove_head( &p_port->send_mgr.pending_list );\r
2693                 p_item != cl_qlist_end( &p_port->send_mgr.pending_list );\r
2694                 p_item = cl_qlist_remove_head( &p_port->send_mgr.pending_list ) )\r
2695         {\r
2696                 p_packet = IPOIB_PACKET_FROM_LIST_ITEM( p_item );\r
2697                 NdisMSendComplete( p_port->p_adapter->h_adapter, p_packet,\r
2698                         NDIS_STATUS_RESET_IN_PROGRESS );\r
2699         }\r
2700         cl_spinlock_release( &p_port->send_lock );\r
2701 }\r
2702 \r
2703 static void\r
2704 __send_mgr_destroy(\r
2705         IN                              ipoib_port_t* const                     p_port )\r
2706 {\r
2707         IPOIB_ENTER( IPOIB_DBG_SEND );\r
2708         __pending_list_destroy(p_port);\r
2709 \r
2710         IPOIB_EXIT( IPOIB_DBG_SEND );\r
2711 }\r
2712 \r
2713 \r
2714 static NDIS_STATUS\r
2715 __send_mgr_filter(\r
2716         IN                              ipoib_port_t* const                     p_port,\r
2717         IN              const   eth_hdr_t* const                        p_eth_hdr,\r
2718         IN                              NDIS_BUFFER* const                      p_buf,\r
2719         IN                              size_t                                          buf_len,\r
2720         IN      OUT                     ipoib_send_desc_t* const        p_desc )\r
2721 {\r
2722         NDIS_STATUS             status;\r
2723 \r
2724         PERF_DECLARE( FilterIp );\r
2725         PERF_DECLARE( FilterArp );\r
2726         PERF_DECLARE( SendGen );\r
2727 \r
2728         IPOIB_ENTER( IPOIB_DBG_SEND );\r
2729 \r
2730         /*\r
2731          * We already checked the ethernet header length, so we know it's safe\r
2732          * to decrement the buf_len without underflowing.\r
2733          */\r
2734         buf_len -= sizeof(eth_hdr_t);\r
2735 \r
2736         switch( p_eth_hdr->type )\r
2737         {\r
2738         case ETH_PROT_TYPE_IP:\r
2739                 cl_perf_start( FilterIp );\r
2740                 status = __send_mgr_filter_ip(\r
2741                         p_port, p_eth_hdr, p_buf, buf_len, p_desc );\r
2742                 cl_perf_stop( &p_port->p_adapter->perf, FilterIp );\r
2743                 break;\r
2744 \r
2745         case ETH_PROT_TYPE_ARP:\r
2746                 cl_perf_start( FilterArp );\r
2747                 status = __send_mgr_filter_arp(\r
2748                         p_port, p_eth_hdr, p_buf, buf_len, p_desc );\r
2749                 cl_perf_stop( &p_port->p_adapter->perf, FilterArp );\r
2750                 break;\r
2751 \r
2752         default:\r
2753                 /*\r
2754                  * The IPoIB spec doesn't define how to send non IP or ARP packets.\r
2755                  * Just send the payload and hope for the best.\r
2756                  */\r
2757                 cl_perf_start( SendGen );\r
2758                 status = __send_gen( p_port, p_desc );\r
2759                 cl_perf_stop( &p_port->p_adapter->perf, SendGen );\r
2760                 break;\r
2761         }\r
2762 \r
2763         IPOIB_EXIT( IPOIB_DBG_SEND );\r
2764         return status;\r
2765 }\r
2766 \r
2767 \r
2768 static NDIS_STATUS\r
2769 __send_copy(\r
2770         IN                              ipoib_port_t* const                     p_port,\r
2771         IN                              ipoib_send_desc_t* const        p_desc )\r
2772 {\r
2773         NDIS_PACKET                             *p_packet;\r
2774         NDIS_BUFFER                             *p_buf;\r
2775         NDIS_STATUS                             status;\r
2776         UINT                                    tot_len, bytes_copied = 0;\r
2777 \r
2778         IPOIB_ENTER( IPOIB_DBG_SEND );\r
2779 \r
2780         p_desc->p_buf = \r
2781                 ExAllocateFromNPagedLookasideList( &p_port->buf_mgr.send_buf_list );\r
2782         if( !p_desc->p_buf )\r
2783         {\r
2784                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2785                         ("Failed to allocate buffer for packet copy.\n") );\r
2786                 return NDIS_STATUS_RESOURCES;\r
2787         }\r
2788 \r
2789         NdisAllocatePacket( &status, &p_packet, p_port->buf_mgr.h_send_pkt_pool );\r
2790         if( status != NDIS_STATUS_SUCCESS )\r
2791         {\r
2792                 IPOIB_PRINT_EXIT( TRACE_LEVEL_WARNING, IPOIB_DBG_SEND,\r
2793                         ("Failed to allocate NDIS_PACKET for copy.\n") );\r
2794                 return status;\r
2795         }\r
2796 \r
2797         NdisAllocateBuffer( &status, &p_buf, p_port->buf_mgr.h_send_buf_pool,\r
2798                 p_desc->p_buf, p_port->p_adapter->params.xfer_block_size );\r
2799         if( status != NDIS_STATUS_SUCCESS )\r
2800         {\r
2801                 NdisFreePacket( p_packet );\r
2802                 IPOIB_PRINT_EXIT( TRACE_LEVEL_WARNING, IPOIB_DBG_SEND,\r
2803                         ("Failed to allocate NDIS_BUFFER for copy.\n") );\r
2804                 return status;\r
2805         }\r
2806 \r
2807         NdisChainBufferAtFront( p_packet, p_buf );\r
2808 \r
2809         NdisQueryPacketLength( p_desc->p_pkt, &tot_len );\r
2810 \r
2811         /* Setup the work request. */\r
2812         p_desc->local_ds[1].vaddr = cl_get_physaddr(\r
2813                 ((uint8_t*)p_desc->p_buf) + sizeof(eth_hdr_t) );\r
2814         p_desc->local_ds[1].length = tot_len - sizeof(eth_hdr_t);\r
2815         p_desc->local_ds[1].lkey = p_port->ib_mgr.lkey;\r
2816         p_desc->wr.num_ds = 2;\r
2817 \r
2818         /* Copy the packet. */\r
2819         NdisCopyFromPacketToPacketSafe( p_packet, bytes_copied, tot_len,\r
2820                 p_desc->p_pkt, bytes_copied, &bytes_copied,\r
2821                 NormalPagePriority );\r
2822 \r
2823         /* Free our temp packet now that the data is copied. */\r
2824         NdisUnchainBufferAtFront( p_packet, &p_buf );\r
2825         NdisFreeBuffer( p_buf );\r
2826         NdisFreePacket( p_packet );\r
2827 \r
2828         if( bytes_copied != tot_len )\r
2829         {\r
2830                 /* Something went wrong.  Drop the packet. */\r
2831                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2832                         ("Failed to copy full packet: %d of %d bytes copied.\n",\r
2833                         bytes_copied, tot_len) );\r
2834                 return NDIS_STATUS_RESOURCES;\r
2835         }\r
2836 \r
2837         IPOIB_EXIT( IPOIB_DBG_SEND );\r
2838         return NDIS_STATUS_SUCCESS;\r
2839 }\r
2840 \r
2841 \r
2842 #if !IPOIB_USE_DMA\r
2843 /* Send using the MDL's page information rather than the SGL. */\r
2844 static ib_api_status_t\r
2845 __send_gen(\r
2846         IN                              ipoib_port_t* const                     p_port,\r
2847         IN                              ipoib_send_desc_t* const        p_desc )\r
2848 {\r
2849         uint32_t                                i, j = 1;\r
2850         ULONG                                   offset;\r
2851         MDL                                             *p_mdl;\r
2852         UINT                                    num_pages, tot_len;\r
2853         ULONG                                   buf_len;\r
2854         PPFN_NUMBER                             page_array;\r
2855         boolean_t                               hdr_done = FALSE;\r
2856         ib_api_status_t                 status;\r
2857 \r
2858         IPOIB_ENTER( IPOIB_DBG_SEND );\r
2859 \r
2860         NdisQueryPacket( p_desc->p_pkt, &num_pages, NULL, &p_mdl,\r
2861                 &tot_len );\r
2862 \r
2863         if( !p_mdl )\r
2864         {\r
2865                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2866                         ("No buffers associated with packet.\n") );\r
2867                 return IB_ERROR;\r
2868         }\r
2869 \r
2870         /* Remember that one of the DS entries is reserved for the IPoIB header. */\r
2871         if( num_pages >= MAX_SEND_SGE )\r
2872         {\r
2873                 IPOIB_PRINT(TRACE_LEVEL_INFORMATION, IPOIB_DBG_SEND,\r
2874                         ("Too many buffers to fit in WR ds_array.  Copying data.\n") );\r
2875                 status = __send_copy( p_port, p_desc );\r
2876                 IPOIB_EXIT( IPOIB_DBG_SEND );\r
2877                 return status;\r
2878         }\r
2879 \r
2880         CL_ASSERT( tot_len > sizeof(eth_hdr_t) );\r
2881         CL_ASSERT( tot_len <= p_port->p_adapter->params.xfer_block_size );\r
2882         /*\r
2883          * Assume that the ethernet header is always fully contained\r
2884          * in the first page of the first MDL.  This makes for much\r
2885          * simpler code.\r
2886          */\r
2887         offset = MmGetMdlByteOffset( p_mdl ) + sizeof(eth_hdr_t);\r
2888         CL_ASSERT( offset <= PAGE_SIZE );\r
2889 \r
2890         while( tot_len )\r
2891         {\r
2892                 buf_len = MmGetMdlByteCount( p_mdl );\r
2893                 page_array = MmGetMdlPfnArray( p_mdl );\r
2894                 CL_ASSERT( page_array );\r
2895                 i = 0;\r
2896                 if( !hdr_done )\r
2897                 {\r
2898                         CL_ASSERT( buf_len >= sizeof(eth_hdr_t) );\r
2899                         /* Skip the ethernet header. */\r
2900                         buf_len -= sizeof(eth_hdr_t);\r
2901                         CL_ASSERT( buf_len <= p_port->p_adapter->params.payload_mtu );\r
2902                         if( buf_len )\r
2903                         {\r
2904                                 /* The ethernet header is a subset of this MDL. */\r
2905                                 CL_ASSERT( i == 0 );\r
2906                                 if( offset < PAGE_SIZE )\r
2907                                 {\r
2908                                         p_desc->local_ds[j].lkey = p_port->ib_mgr.lkey;\r
2909                                         p_desc->local_ds[j].vaddr = (page_array[i] << PAGE_SHIFT);\r
2910                                         /* Add the byte offset since we're on the 1st page. */\r
2911                                         p_desc->local_ds[j].vaddr += offset;\r
2912                                         if( offset + buf_len > PAGE_SIZE )\r
2913                                         {\r
2914                                                 p_desc->local_ds[j].length = PAGE_SIZE - offset;\r
2915                                                 buf_len -= p_desc->local_ds[j].length;\r
2916                                         }\r
2917                                         else\r
2918                                         {\r
2919                                                 p_desc->local_ds[j].length = buf_len;\r
2920                                                 buf_len = 0;\r
2921                                         }\r
2922                                         /* This data segment is done.  Move to the next. */\r
2923                                         j++;\r
2924                                 }\r
2925                                 /* This page is done.  Move to the next. */\r
2926                                 i++;\r
2927                         }\r
2928                         /* Done handling the ethernet header. */\r
2929                         hdr_done = TRUE;\r
2930                 }\r
2931 \r
2932                 /* Finish this MDL */\r
2933                 while( buf_len )\r
2934                 {\r
2935                         p_desc->local_ds[j].lkey = p_port->ib_mgr.lkey;\r
2936                         p_desc->local_ds[j].vaddr = (page_array[i] << PAGE_SHIFT);\r
2937                         /* Add the first page's offset if we're on the first page. */\r
2938                         if( i == 0 )\r
2939                                 p_desc->local_ds[j].vaddr += MmGetMdlByteOffset( p_mdl );\r
2940 \r
2941                         if( i == 0 && (MmGetMdlByteOffset( p_mdl ) + buf_len) > PAGE_SIZE )\r
2942                         {\r
2943                                 /* Buffers spans pages. */\r
2944                                 p_desc->local_ds[j].length =\r
2945                                         PAGE_SIZE - MmGetMdlByteOffset( p_mdl );\r
2946                                 buf_len -= p_desc->local_ds[j].length;\r
2947                                 /* This page is done.  Move to the next. */\r
2948                                 i++;\r
2949                         }\r
2950                         else\r
2951                         {\r
2952                                 /* Last page of the buffer. */\r
2953                                 p_desc->local_ds[j].length = buf_len;\r
2954                                 buf_len = 0;\r
2955                         }\r
2956                         /* This data segment is done.  Move to the next. */\r
2957                         j++;\r
2958                 }\r
2959 \r
2960                 tot_len -= MmGetMdlByteCount( p_mdl );\r
2961                 if( !tot_len )\r
2962                         break;\r
2963 \r
2964                 NdisGetNextBuffer( p_mdl, &p_mdl );\r
2965                 if( !p_mdl )\r
2966                 {\r
2967                         IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2968                                 ("Failed to get next buffer.\n") );\r
2969                         return IB_ERROR;\r
2970                 }\r
2971         }\r
2972 \r
2973         /* Set the number of data segments. */\r
2974         p_desc->wr.num_ds = j;\r
2975 \r
2976         IPOIB_EXIT( IPOIB_DBG_SEND );\r
2977         return IB_SUCCESS;\r
2978 }\r
2979 \r
2980 #else\r
2981 \r
2982 static NDIS_STATUS\r
2983 __send_gen(\r
2984         IN                              ipoib_port_t* const                     p_port,\r
2985         IN                              ipoib_send_desc_t* const        p_desc )\r
2986 {\r
2987         ib_api_status_t                 status;\r
2988         SCATTER_GATHER_LIST             *p_sgl;\r
2989         uint32_t                                i, j = 1;\r
2990         uint32_t                                offset = sizeof(eth_hdr_t);\r
2991         PERF_DECLARE( SendCopy );\r
2992 \r
2993         IPOIB_ENTER( IPOIB_DBG_SEND );\r
2994 \r
2995         p_sgl = NDIS_PER_PACKET_INFO_FROM_PACKET( p_desc->p_pkt,\r
2996                 ScatterGatherListPacketInfo );\r
2997         if( !p_sgl )\r
2998         {\r
2999                 ASSERT( p_sgl );\r
3000                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
3001                         ("Failed to get SGL from packet.\n") );\r
3002                 return NDIS_STATUS_FAILURE;\r
3003         }\r
3004 \r
3005         /* Remember that one of the DS entries is reserved for the IPoIB header. */\r
3006         if( ( p_sgl->NumberOfElements >= MAX_SEND_SGE &&\r
3007                 p_sgl->Elements[0].Length > sizeof(eth_hdr_t)) ||\r
3008                 ( p_sgl->NumberOfElements > MAX_SEND_SGE &&\r
3009                 p_sgl->Elements[0].Length <= sizeof(eth_hdr_t)) )\r
3010         {\r
3011                 IPOIB