[IPOIB] filter out packets with payload MTU bigger then our adapter payload MTU
[mirror/winof/.git] / ulp / ipoib / kernel / ipoib_port.c
1 /*\r
2  * Copyright (c) 2005 SilverStorm Technologies.  All rights reserved.\r
3  *\r
4  * This software is available to you under the OpenIB.org BSD license\r
5  * below:\r
6  *\r
7  *     Redistribution and use in source and binary forms, with or\r
8  *     without modification, are permitted provided that the following\r
9  *     conditions are met:\r
10  *\r
11  *      - Redistributions of source code must retain the above\r
12  *        copyright notice, this list of conditions and the following\r
13  *        disclaimer.\r
14  *\r
15  *      - Redistributions in binary form must reproduce the above\r
16  *        copyright notice, this list of conditions and the following\r
17  *        disclaimer in the documentation and/or other materials\r
18  *        provided with the distribution.\r
19  *\r
20  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,\r
21  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF\r
22  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND\r
23  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS\r
24  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN\r
25  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN\r
26  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\r
27  * SOFTWARE.\r
28  *\r
29  * $Id$\r
30  */\r
31 \r
32 \r
33 \r
34 #include "ipoib_port.h"\r
35 #include "ipoib_adapter.h"\r
36 #include "ipoib_debug.h"\r
37 #if defined(EVENT_TRACING)\r
38 #ifdef offsetof\r
39 #undef offsetof\r
40 #endif\r
41 #include "ipoib_port.tmh"\r
42 #endif\r
43 \r
44 \r
45 /* Amount of physical memory to register. */\r
46 #define MEM_REG_SIZE    0xFFFFFFFFFFFFFFFF\r
47 \r
48 /* Number of work completions to chain for send and receive polling. */\r
49 #define MAX_SEND_WC             8\r
50 #define MAX_RECV_WC             16\r
51 \r
52 \r
53 ib_gid_t        bcast_mgid_template = {\r
54         0xff,                                                           /* multicast field */\r
55         0x12,                                                           /* scope (to be filled in) */\r
56         0x40, 0x1b,                                                     /* IPv4 signature */\r
57         0xff, 0xff,                                                     /* 16 bits of P_Key (to be filled in) */\r
58         0x00, 0x00, 0x00, 0x00, 0x00, 0x00,     /* 48 bits of zeros */\r
59         0xff, 0xff, 0xff, 0xff,                         /* 32 bit IPv4 broadcast address */\r
60 };\r
61 \r
62 \r
63 #ifdef _DEBUG_\r
64 /* Handy pointer for debug use. */\r
65 ipoib_port_t    *gp_ipoib_port;\r
66 #endif\r
67 \r
68 \r
69 /******************************************************************************\r
70 *\r
71 * Declarations\r
72 *\r
73 ******************************************************************************/\r
74 static void\r
75 __port_construct(\r
76         IN                              ipoib_port_t* const                     p_port );\r
77 \r
78 static ib_api_status_t\r
79 __port_init(\r
80         IN                              ipoib_port_t* const                     p_port,\r
81         IN                              ipoib_adapter_t* const          p_adapter,\r
82         IN                              ib_pnp_port_rec_t* const        p_pnp_rec );\r
83 \r
84 static void\r
85 __port_destroying(\r
86         IN                              cl_obj_t* const                         p_obj );\r
87 \r
88 static void\r
89 __port_cleanup(\r
90         IN                              cl_obj_t* const                         p_obj );\r
91 \r
92 static void\r
93 __port_free(\r
94         IN                              cl_obj_t* const                         p_obj );\r
95 \r
96 \r
97 /******************************************************************************\r
98 *\r
99 * IB resource manager operations\r
100 *\r
101 ******************************************************************************/\r
102 static void\r
103 __ib_mgr_construct(\r
104         IN                              ipoib_port_t* const                     p_port );\r
105 \r
106 static ib_api_status_t\r
107 __ib_mgr_init(\r
108         IN                              ipoib_port_t* const                     p_port );\r
109 \r
110 static void\r
111 __ib_mgr_destroy(\r
112         IN                              ipoib_port_t* const                     p_port );\r
113 \r
114 static void\r
115 __qp_event(\r
116         IN                              ib_async_event_rec_t            *p_event_rec );\r
117 \r
118 static void\r
119 __cq_event(\r
120         IN                              ib_async_event_rec_t            *p_event_rec );\r
121 \r
122 static ib_api_status_t\r
123 __ib_mgr_activate(\r
124         IN                              ipoib_port_t* const                     p_port );\r
125 \r
126 /******************************************************************************\r
127 *\r
128 * Buffer manager operations.\r
129 *\r
130 ******************************************************************************/\r
131 static void\r
132 __buf_mgr_construct(\r
133         IN                              ipoib_port_t* const                     p_port );\r
134 \r
135 static ib_api_status_t\r
136 __buf_mgr_init(\r
137         IN                              ipoib_port_t* const                     p_port );\r
138 \r
139 static void\r
140 __buf_mgr_destroy(\r
141         IN                              ipoib_port_t* const                     p_port );\r
142 \r
143 static cl_status_t\r
144 __recv_ctor(\r
145         IN                              void* const                                     p_object,\r
146         IN                              void*                                           context,\r
147                 OUT                     cl_pool_item_t** const          pp_pool_item );\r
148 \r
149 #if !IPOIB_INLINE_RECV\r
150 static void\r
151 __recv_dtor(\r
152         IN              const   cl_pool_item_t* const           p_pool_item,\r
153         IN                              void                                            *context );\r
154 #endif  /* IPOIB_INLINE_RECV */\r
155 \r
156 static inline ipoib_send_desc_t*\r
157 __buf_mgr_get_send(\r
158         IN                              ipoib_port_t* const                     p_port );\r
159 \r
160 static inline void\r
161 __buf_mgr_put_send(\r
162         IN                              ipoib_port_t* const                     p_port,\r
163         IN                              ipoib_send_desc_t* const        p_desc );\r
164 \r
165 static inline ipoib_recv_desc_t*\r
166 __buf_mgr_get_recv(\r
167         IN                              ipoib_port_t* const                     p_port );\r
168 \r
169 static inline void\r
170 __buf_mgr_put_recv(\r
171         IN                              ipoib_port_t* const                     p_port,\r
172         IN                              ipoib_recv_desc_t* const        p_desc,\r
173         IN                              NDIS_PACKET* const                      p_packet OPTIONAL );\r
174 \r
175 static inline void\r
176 __buf_mgr_put_recv_list(\r
177         IN                              ipoib_port_t* const                     p_port,\r
178         IN                              cl_qlist_t* const                       p_list );\r
179 \r
180 static inline NDIS_PACKET*\r
181 __buf_mgr_get_ndis_pkt(\r
182         IN                              ipoib_port_t* const                     p_port,\r
183         IN                              ipoib_recv_desc_t* const        p_desc );\r
184 \r
185 \r
186 /******************************************************************************\r
187 *\r
188 * Receive manager operations.\r
189 *\r
190 ******************************************************************************/\r
191 static void\r
192 __recv_mgr_construct(\r
193         IN                              ipoib_port_t* const                     p_port );\r
194 \r
195 static ib_api_status_t\r
196 __recv_mgr_init(\r
197         IN                              ipoib_port_t* const                     p_port );\r
198 \r
199 static void\r
200 __recv_mgr_destroy(\r
201         IN                              ipoib_port_t* const                     p_port );\r
202 \r
203 /* Posts receive buffers to the receive queue. */\r
204 static ib_api_status_t\r
205 __recv_mgr_repost(\r
206         IN                              ipoib_port_t* const                     p_port );\r
207 \r
208 static void\r
209 __recv_cb(\r
210         IN              const   ib_cq_handle_t                          h_cq,\r
211         IN                              void                                            *cq_context );\r
212 \r
213 static void\r
214 __recv_get_endpts(\r
215         IN                              ipoib_port_t* const                     p_port,\r
216         IN                              ipoib_recv_desc_t* const        p_desc,\r
217         IN                              ib_wc_t* const                          p_wc,\r
218                 OUT                     ipoib_endpt_t** const           pp_src,\r
219                 OUT                     ipoib_endpt_t** const           pp_dst );\r
220 \r
221 static int32_t\r
222 __recv_mgr_filter(\r
223         IN                              ipoib_port_t* const                     p_port,\r
224         IN                              ib_wc_t* const                          p_done_wc_list,\r
225                 OUT                     cl_qlist_t* const                       p_done_list,\r
226                 OUT                     cl_qlist_t* const                       p_bad_list );\r
227 \r
228 static ib_api_status_t\r
229 __recv_gen(\r
230         IN              const   ipoib_pkt_t* const                      p_ipoib,\r
231                 OUT                     eth_pkt_t* const                        p_eth,\r
232         IN                              ipoib_endpt_t* const            p_src,\r
233         IN                              ipoib_endpt_t* const            p_dst );\r
234 \r
235 static ib_api_status_t\r
236 __recv_dhcp(\r
237         IN                              ipoib_port_t* const                     p_port,\r
238         IN              const   ipoib_pkt_t* const                      p_ipoib,\r
239                 OUT                     eth_pkt_t* const                        p_eth,\r
240         IN                              ipoib_endpt_t* const            p_src,\r
241         IN                              ipoib_endpt_t* const            p_dst );\r
242 \r
243 static ib_api_status_t\r
244 __recv_arp(\r
245         IN                              ipoib_port_t* const                     p_port,\r
246         IN                              ib_wc_t* const                          p_wc,\r
247         IN              const   ipoib_pkt_t* const                      p_ipoib,\r
248                 OUT                     eth_pkt_t* const                        p_eth,\r
249         IN                              ipoib_endpt_t** const           p_src,\r
250         IN                              ipoib_endpt_t* const            p_dst );\r
251 \r
252 static ib_api_status_t\r
253 __recv_mgr_prepare_pkt(\r
254         IN                              ipoib_port_t* const                     p_port,\r
255         IN                              ipoib_recv_desc_t* const        p_desc,\r
256                 OUT                     NDIS_PACKET** const                     pp_packet );\r
257 \r
258 static uint32_t\r
259 __recv_mgr_build_pkt_array(\r
260         IN                              ipoib_port_t* const                     p_port,\r
261         IN                              int32_t                                         shortage,\r
262                 OUT                     cl_qlist_t* const                       p_done_list,\r
263                 OUT                     int32_t* const                          p_discarded );\r
264 \r
265 /******************************************************************************\r
266 *\r
267 * Send manager operations.\r
268 *\r
269 ******************************************************************************/\r
270 static void\r
271 __send_mgr_construct(\r
272         IN                              ipoib_port_t* const                     p_port );\r
273 \r
274 static void\r
275 __send_mgr_destroy(\r
276         IN                              ipoib_port_t* const                     p_port );\r
277 \r
278 static NDIS_STATUS\r
279 __send_gen(\r
280         IN                              ipoib_port_t* const                     p_port,\r
281         IN                              ipoib_send_desc_t* const        p_desc );\r
282 \r
283 static NDIS_STATUS\r
284 __send_mgr_filter_ip(\r
285         IN                              ipoib_port_t* const                     p_port,\r
286         IN              const   eth_hdr_t* const                        p_eth_hdr,\r
287         IN                              NDIS_BUFFER*                            p_buf,\r
288         IN                              size_t                                          buf_len,\r
289         IN      OUT                     ipoib_send_desc_t* const        p_desc );\r
290 \r
291 static NDIS_STATUS\r
292 __send_mgr_filter_udp(\r
293         IN                              ipoib_port_t* const                     p_port,\r
294         IN              const   ip_hdr_t* const                         p_ip_hdr,\r
295         IN                              NDIS_BUFFER*                            p_buf,\r
296         IN                              size_t                                          buf_len,\r
297         IN      OUT                     ipoib_send_desc_t* const        p_desc );\r
298 \r
299 static NDIS_STATUS\r
300 __send_mgr_filter_dhcp(\r
301         IN                              ipoib_port_t* const                     p_port,\r
302         IN              const   udp_hdr_t* const                        p_udp_hdr,\r
303         IN                              NDIS_BUFFER*                            p_buf,\r
304         IN                              size_t                                          buf_len,\r
305         IN      OUT                     ipoib_send_desc_t* const        p_desc );\r
306 \r
307 static NDIS_STATUS\r
308 __send_mgr_filter_arp(\r
309         IN                              ipoib_port_t* const                     p_port,\r
310         IN              const   eth_hdr_t* const                        p_eth_hdr,\r
311         IN                              NDIS_BUFFER*                            p_buf,\r
312         IN                              size_t                                          buf_len,\r
313         IN      OUT                     ipoib_send_desc_t* const        p_desc );\r
314 \r
315 static void\r
316 __process_failed_send(\r
317         IN                              ipoib_port_t* const                     p_port,\r
318         IN                              ipoib_send_desc_t* const        p_desc,\r
319         IN              const   NDIS_STATUS                                     status );\r
320 \r
321 static void\r
322 __send_cb(\r
323         IN              const   ib_cq_handle_t                          h_cq,\r
324         IN                              void                                            *cq_context );\r
325 \r
326 \r
327 /******************************************************************************\r
328 *\r
329 * Endpoint manager operations\r
330 *\r
331 ******************************************************************************/\r
332 static void\r
333 __endpt_mgr_construct(\r
334         IN                              ipoib_port_t* const                     p_port );\r
335 \r
336 static ib_api_status_t\r
337 __endpt_mgr_init(\r
338         IN                              ipoib_port_t* const                     p_port );\r
339 \r
340 static void\r
341 __endpt_mgr_destroy(\r
342         IN                              ipoib_port_t* const                     p_port );\r
343 \r
344 /****f* IPoIB/__endpt_mgr_remove_all\r
345 * NAME\r
346 *       __endpt_mgr_remove_all\r
347 *\r
348 * DESCRIPTION\r
349 *       Removes all enpoints from the port, dereferencing them to initiate\r
350 *       destruction.\r
351 *\r
352 * SYNOPSIS\r
353 */\r
354 static void\r
355 __endpt_mgr_remove_all(\r
356         IN                              ipoib_port_t* const                     p_port );\r
357 /*\r
358 ********/\r
359 \r
360 static void\r
361 __endpt_mgr_remove(\r
362         IN                              ipoib_port_t* const                     p_port,\r
363         IN                              ipoib_endpt_t* const            p_endpt );\r
364 \r
365 static void\r
366 __endpt_mgr_reset_all(\r
367         IN                              ipoib_port_t* const                     p_port );\r
368 \r
369 static inline NDIS_STATUS\r
370 __endpt_mgr_ref(\r
371         IN                              ipoib_port_t* const                     p_port,\r
372         IN              const   mac_addr_t                                      mac,\r
373                 OUT                     ipoib_endpt_t** const           pp_endpt );\r
374 \r
375 static inline NDIS_STATUS\r
376 __endpt_mgr_get_gid_qpn(\r
377         IN                              ipoib_port_t* const                     p_port,\r
378         IN              const   mac_addr_t                                      mac,\r
379                 OUT                     ib_gid_t* const                         p_gid,\r
380                 OUT                     UNALIGNED net32_t* const        p_qpn );\r
381 \r
382 static inline ipoib_endpt_t*\r
383 __endpt_mgr_get_by_gid(\r
384         IN                              ipoib_port_t* const                     p_port,\r
385         IN              const   ib_gid_t* const                         p_gid );\r
386 \r
387 static inline ipoib_endpt_t*\r
388 __endpt_mgr_get_by_lid(\r
389         IN                              ipoib_port_t* const                     p_port,\r
390         IN              const   net16_t                                         lid );\r
391 \r
392 static inline ib_api_status_t\r
393 __endpt_mgr_insert_locked(\r
394         IN                              ipoib_port_t* const                     p_port,\r
395         IN              const   mac_addr_t                                      mac,\r
396         IN                              ipoib_endpt_t* const            p_endpt );\r
397 \r
398 static inline ib_api_status_t\r
399 __endpt_mgr_insert(\r
400         IN                              ipoib_port_t* const                     p_port,\r
401         IN              const   mac_addr_t                                      mac,\r
402         IN                              ipoib_endpt_t* const            p_endpt );\r
403 \r
404 static ib_api_status_t\r
405 __endpt_mgr_add_local(\r
406         IN                              ipoib_port_t* const                     p_port,\r
407         IN                              ib_port_info_t* const           p_port_info );\r
408 \r
409 static ib_api_status_t\r
410 __endpt_mgr_add_bcast(\r
411         IN                              ipoib_port_t* const                     p_port,\r
412         IN                              ib_mcast_rec_t                          *p_mcast_rec );\r
413 \r
414 /******************************************************************************\r
415 *\r
416 * MCast operations.\r
417 *\r
418 ******************************************************************************/\r
419 static ib_api_status_t\r
420 __port_get_bcast(\r
421         IN                              ipoib_port_t* const                     p_port );\r
422 \r
423 static ib_api_status_t\r
424 __port_join_bcast(\r
425         IN                              ipoib_port_t* const                     p_port,\r
426         IN                              ib_member_rec_t* const          p_member_rec );\r
427 \r
428 static ib_api_status_t\r
429 __port_create_bcast(\r
430         IN                              ipoib_port_t* const                     p_port );\r
431 \r
432 static void\r
433 __port_info_cb(\r
434         IN                              ib_query_rec_t                          *p_query_rec );\r
435 \r
436 \r
437 static void\r
438 __bcast_get_cb(\r
439         IN                              ib_query_rec_t                          *p_query_rec );\r
440 \r
441 \r
442 static void\r
443 __bcast_cb(\r
444         IN                              ib_mcast_rec_t                          *p_mcast_rec );\r
445 \r
446 \r
447 static void\r
448 __mcast_cb(\r
449         IN                              ib_mcast_rec_t                          *p_mcast_rec );\r
450 \r
451 \r
452 static intn_t\r
453 __gid_cmp(\r
454         IN              const   void* const                                     p_key1,\r
455         IN              const   void* const                                     p_key2 )\r
456 {\r
457         return cl_memcmp( p_key1, p_key2, sizeof(ib_gid_t) );\r
458 }\r
459 \r
460 \r
461 inline void ipoib_port_ref( ipoib_port_t * p_port, int type )\r
462 {\r
463         cl_obj_ref( &p_port->obj );\r
464 #if DBG\r
465         cl_atomic_inc( &p_port->ref[type % ref_mask] );\r
466         IPOIB_PRINT( TRACE_LEVEL_INFORMATION, IPOIB_DBG_OBJ,\r
467                 ("ref type %d ref_cnt %d\n", type, p_port->obj.ref_cnt) );\r
468 #else\r
469         UNREFERENCED_PARAMETER(type);\r
470 #endif\r
471 }\r
472 \r
473 \r
474 inline void ipoib_port_deref(ipoib_port_t * p_port, int type)\r
475 {\r
476         cl_obj_deref( &p_port->obj );\r
477 \r
478 #if DBG\r
479         cl_atomic_dec( &p_port->ref[type % ref_mask] );\r
480         IPOIB_PRINT( TRACE_LEVEL_INFORMATION, IPOIB_DBG_OBJ,\r
481                 ("deref type %d ref_cnt %d\n", type, p_port->obj.ref_cnt) );\r
482 #else\r
483         UNREFERENCED_PARAMETER(type);\r
484 #endif\r
485 }\r
486 \r
487 \r
488 /******************************************************************************\r
489 *\r
490 * Implementation\r
491 *\r
492 ******************************************************************************/\r
493 ib_api_status_t\r
494 ipoib_create_port(\r
495         IN                              ipoib_adapter_t* const          p_adapter,\r
496         IN                              ib_pnp_port_rec_t* const        p_pnp_rec,\r
497                 OUT                     ipoib_port_t** const            pp_port )\r
498 {\r
499         ib_api_status_t         status;\r
500         ipoib_port_t            *p_port;\r
501 \r
502         IPOIB_ENTER( IPOIB_DBG_INIT );\r
503 \r
504         CL_ASSERT( !p_adapter->p_port );\r
505 \r
506         p_port = cl_zalloc( sizeof(ipoib_port_t) +\r
507                 (sizeof(ipoib_hdr_t) * (p_adapter->params.sq_depth - 1)) );\r
508         if( !p_port )\r
509         {\r
510                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
511                         ("Failed to allocate ipoib_port_t (%d bytes)\n",\r
512                         sizeof(ipoib_port_t)) );\r
513                 return IB_INSUFFICIENT_MEMORY;\r
514         }\r
515 \r
516 #ifdef _DEBUG_\r
517         gp_ipoib_port = p_port;\r
518 #endif\r
519 \r
520         __port_construct( p_port );\r
521 \r
522         status = __port_init( p_port, p_adapter, p_pnp_rec );\r
523         if( status != IB_SUCCESS )\r
524         {\r
525                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
526                         ("ipoib_port_init returned %s.\n", \r
527                         p_adapter->p_ifc->get_err_str( status )) );\r
528                 __port_cleanup( &p_port->obj );\r
529                 __port_free( &p_port->obj );\r
530                 return status;\r
531         }\r
532 \r
533         *pp_port = p_port;\r
534         IPOIB_EXIT( IPOIB_DBG_INIT );\r
535         return IB_SUCCESS;\r
536 }\r
537 \r
538 \r
539 void\r
540 ipoib_port_destroy(\r
541         IN                              ipoib_port_t* const                     p_port )\r
542 {\r
543         IPOIB_ENTER( IPOIB_DBG_INIT );\r
544 \r
545         CL_ASSERT( p_port );\r
546         CL_ASSERT( p_port->p_adapter );\r
547         CL_ASSERT( !p_port->p_adapter->p_port );\r
548 \r
549         cl_obj_destroy( &p_port->obj );\r
550 \r
551         IPOIB_EXIT( IPOIB_DBG_INIT );\r
552 }\r
553 \r
554 \r
555 static void\r
556 __port_construct(\r
557         IN                              ipoib_port_t* const                     p_port )\r
558 {\r
559         IPOIB_ENTER( IPOIB_DBG_INIT );\r
560 \r
561         p_port->state = IB_QPS_RESET;\r
562 \r
563         cl_obj_construct( &p_port->obj, IPOIB_OBJ_PORT );\r
564         cl_spinlock_construct( &p_port->send_lock );\r
565         cl_spinlock_construct( &p_port->recv_lock );\r
566         __ib_mgr_construct( p_port );\r
567         __buf_mgr_construct( p_port );\r
568 \r
569         __recv_mgr_construct( p_port );\r
570         __send_mgr_construct( p_port );\r
571 \r
572         __endpt_mgr_construct( p_port );\r
573 \r
574         KeInitializeEvent( &p_port->sa_event, NotificationEvent, TRUE );\r
575 \r
576         IPOIB_EXIT( IPOIB_DBG_INIT );\r
577 }\r
578 \r
579 \r
580 static ib_api_status_t\r
581 __port_init(\r
582         IN                              ipoib_port_t* const                     p_port,\r
583         IN                              ipoib_adapter_t* const          p_adapter,\r
584         IN                              ib_pnp_port_rec_t* const        p_pnp_rec )\r
585 {\r
586         cl_status_t                     cl_status;\r
587         ib_api_status_t         status;\r
588 \r
589         IPOIB_ENTER( IPOIB_DBG_INIT );\r
590 \r
591         p_port->port_num = p_pnp_rec->p_port_attr->port_num;\r
592         p_port->p_adapter = p_adapter;\r
593 \r
594         cl_status = cl_spinlock_init( &p_port->send_lock );\r
595         if( cl_status != CL_SUCCESS )\r
596         {\r
597                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
598                         ("cl_spinlock_init returned %s\n", cl_status_text[cl_status]) );\r
599                 return IB_ERROR;\r
600         }\r
601 \r
602         cl_status = cl_spinlock_init( &p_port->recv_lock );\r
603         if( cl_status != CL_SUCCESS )\r
604         {\r
605                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
606                         ("cl_spinlock_init returned %s\n", cl_status_text[cl_status]) );\r
607                 return IB_ERROR;\r
608         }\r
609 \r
610         /* Initialize the IB resource manager. */\r
611         status = __ib_mgr_init( p_port );\r
612         if( status != IB_SUCCESS )\r
613         {\r
614                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
615                         ("__ib_mgr_init returned %s\n", \r
616                         p_adapter->p_ifc->get_err_str( status )) );\r
617                 return status;\r
618         }\r
619 \r
620         /* Initialize the buffer manager. */\r
621         status = __buf_mgr_init( p_port );\r
622         if( status != IB_SUCCESS )\r
623         {\r
624                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
625                         ("__buf_mgr_init returned %s\n", \r
626                         p_adapter->p_ifc->get_err_str( status )) );\r
627                 return status;\r
628         }\r
629 \r
630         /* Initialize the receive manager. */\r
631         status = __recv_mgr_init( p_port );\r
632         if( status != IB_SUCCESS )\r
633         {\r
634                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
635                         ("__recv_mgr_init returned %s\n", \r
636                         p_adapter->p_ifc->get_err_str( status )) );\r
637                 return status;\r
638         }\r
639 \r
640         /* Initialize the endpoint manager. */\r
641         status = __endpt_mgr_init( p_port );\r
642         if( status != IB_SUCCESS )\r
643         {\r
644                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
645                         ("__endpt_mgr_init returned %s\n", \r
646                         p_adapter->p_ifc->get_err_str( status )) );\r
647                 return status;\r
648         }\r
649 \r
650         /* We only ever destroy from the PnP callback thread. */\r
651         cl_status = cl_obj_init( &p_port->obj, CL_DESTROY_SYNC,\r
652                 __port_destroying, __port_cleanup, __port_free );\r
653 \r
654 #if DBG\r
655         cl_atomic_inc( &p_port->ref[ref_init] );\r
656         IPOIB_PRINT( TRACE_LEVEL_INFORMATION, IPOIB_DBG_OBJ,\r
657                 ("ref type %d ref_cnt %d\n", ref_init, p_port->obj.ref_cnt) );\r
658 #endif\r
659 \r
660         if( cl_status != CL_SUCCESS )\r
661         {\r
662                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
663                         ("cl_obj_init returned %s\n", cl_status_text[cl_status]) );\r
664                 return IB_ERROR;\r
665         }\r
666 \r
667         cl_status = cl_obj_insert_rel( &p_port->rel, &p_adapter->obj, &p_port->obj );\r
668         if( cl_status != CL_SUCCESS )\r
669         {\r
670                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
671                         ("cl_obj_insert_rel returned %s\n", cl_status_text[cl_status]) );\r
672                 cl_obj_destroy( &p_port->obj );\r
673                 return IB_ERROR;\r
674         }\r
675 \r
676 #if DBG\r
677         cl_atomic_inc( &p_port->ref[ref_init] );\r
678         IPOIB_PRINT( TRACE_LEVEL_ERROR, IPOIB_DBG_OBJ,\r
679                 ("ref type %d ref_cnt %d\n", ref_init, p_port->obj.ref_cnt) );\r
680 #endif\r
681 \r
682         IPOIB_EXIT( IPOIB_DBG_INIT );\r
683         return IB_SUCCESS;\r
684 }\r
685 \r
686 \r
687 static void\r
688 __port_destroying(\r
689         IN                              cl_obj_t* const                         p_obj )\r
690 {\r
691         ipoib_port_t    *p_port;\r
692 \r
693         IPOIB_ENTER( IPOIB_DBG_INIT );\r
694 \r
695         CL_ASSERT( p_obj );\r
696 \r
697         p_port = PARENT_STRUCT( p_obj, ipoib_port_t, obj );\r
698 \r
699         ipoib_port_down( p_port );\r
700 \r
701         __endpt_mgr_remove_all( p_port );\r
702 \r
703         ipoib_port_resume( p_port );\r
704 \r
705         IPOIB_EXIT( IPOIB_DBG_INIT );\r
706 }\r
707 \r
708 \r
709 static void\r
710 __port_cleanup(\r
711         IN                              cl_obj_t* const                         p_obj )\r
712 {\r
713         ipoib_port_t    *p_port;\r
714 \r
715         IPOIB_ENTER( IPOIB_DBG_INIT );\r
716 \r
717         CL_ASSERT( p_obj );\r
718 \r
719         p_port = PARENT_STRUCT( p_obj, ipoib_port_t, obj );\r
720 \r
721         /* Wait for all sends and receives to get flushed. */\r
722         while( p_port->send_mgr.depth || p_port->recv_mgr.depth )\r
723                 cl_thread_suspend( 0 );\r
724 \r
725         /* Destroy the send and receive managers before closing the CA. */\r
726         __ib_mgr_destroy( p_port );\r
727 \r
728         IPOIB_EXIT( IPOIB_DBG_INIT );\r
729 }\r
730 \r
731 \r
732 static void\r
733 __port_free(\r
734         IN                              cl_obj_t* const                         p_obj )\r
735 {\r
736         ipoib_port_t    *p_port;\r
737 \r
738         IPOIB_ENTER( IPOIB_DBG_INIT );\r
739 \r
740         CL_ASSERT( p_obj );\r
741 \r
742         p_port = PARENT_STRUCT( p_obj, ipoib_port_t, obj );\r
743 \r
744         __endpt_mgr_destroy( p_port );\r
745         __recv_mgr_destroy( p_port );\r
746         __send_mgr_destroy( p_port );\r
747         __buf_mgr_destroy( p_port );\r
748 \r
749         cl_spinlock_destroy( &p_port->send_lock );\r
750         cl_spinlock_destroy( &p_port->recv_lock );\r
751 \r
752         cl_obj_deinit( p_obj );\r
753 \r
754         cl_free( p_port );\r
755 \r
756         IPOIB_EXIT( IPOIB_DBG_INIT );\r
757 }\r
758 \r
759 \r
760 \r
761 /******************************************************************************\r
762 *\r
763 * IB resource manager implementation.\r
764 *\r
765 ******************************************************************************/\r
766 static void\r
767 __ib_mgr_construct(\r
768         IN                              ipoib_port_t* const                     p_port )\r
769 {\r
770         IPOIB_ENTER( IPOIB_DBG_INIT );\r
771 \r
772         cl_memclr( &p_port->ib_mgr, sizeof(ipoib_ib_mgr_t) );\r
773 \r
774         IPOIB_EXIT( IPOIB_DBG_INIT );\r
775 }\r
776 \r
777 \r
778 static ib_api_status_t\r
779 __ib_mgr_init(\r
780         IN                              ipoib_port_t* const                     p_port )\r
781 {\r
782         ib_api_status_t         status;\r
783         ib_cq_create_t          cq_create;\r
784         ib_qp_create_t          qp_create;\r
785         ib_phys_create_t        phys_create;\r
786         ib_phys_range_t         phys_range;\r
787         uint64_t                        vaddr;\r
788         net32_t                         rkey;\r
789         ib_qp_attr_t            qp_attr;\r
790 \r
791         IPOIB_ENTER( IPOIB_DBG_INIT );\r
792 \r
793         /* Open the CA. */\r
794         status = p_port->p_adapter->p_ifc->open_ca(\r
795                 p_port->p_adapter->h_al, p_port->p_adapter->guids.ca_guid,\r
796                 NULL, p_port, &p_port->ib_mgr.h_ca );\r
797         if( status != IB_SUCCESS )\r
798         {\r
799                 NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
800                         EVENT_IPOIB_OPEN_CA, 1, status );\r
801                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
802                         ("ib_open_ca returned %s\n", \r
803                         p_port->p_adapter->p_ifc->get_err_str( status )) );\r
804                 return status;\r
805         }\r
806 \r
807         /* Allocate the PD. */\r
808         status = p_port->p_adapter->p_ifc->alloc_pd(\r
809                 p_port->ib_mgr.h_ca, IB_PDT_UD, p_port, &p_port->ib_mgr.h_pd );\r
810         if( status != IB_SUCCESS )\r
811         {\r
812                 NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
813                         EVENT_IPOIB_ALLOC_PD, 1, status );\r
814                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
815                         ("ib_alloc_pd returned %s\n", \r
816                         p_port->p_adapter->p_ifc->get_err_str( status )) );\r
817                 return status;\r
818         }\r
819 \r
820         /* Allocate receive CQ. */\r
821         cq_create.size = p_port->p_adapter->params.rq_depth;\r
822         cq_create.pfn_comp_cb = __recv_cb;\r
823         cq_create.h_wait_obj = NULL;\r
824 \r
825         status = p_port->p_adapter->p_ifc->create_cq(\r
826                 p_port->ib_mgr.h_ca, &cq_create, p_port,\r
827                 __cq_event, &p_port->ib_mgr.h_recv_cq );\r
828         if( status != IB_SUCCESS )\r
829         {\r
830                 NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
831                         EVENT_IPOIB_CREATE_RECV_CQ, 1, status );\r
832                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
833                         ("ib_create_cq returned %s.\n", \r
834                         p_port->p_adapter->p_ifc->get_err_str( status )) );\r
835                 return status;\r
836         }\r
837 \r
838         /* Allocate send CQ. */\r
839         cq_create.size = p_port->p_adapter->params.sq_depth;\r
840         cq_create.pfn_comp_cb = __send_cb;\r
841 \r
842         status = p_port->p_adapter->p_ifc->create_cq(\r
843                 p_port->ib_mgr.h_ca, &cq_create, p_port,\r
844                 __cq_event, &p_port->ib_mgr.h_send_cq );\r
845         if( status != IB_SUCCESS )\r
846         {\r
847                 NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
848                         EVENT_IPOIB_CREATE_SEND_CQ, 1, status );\r
849                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
850                         ("ib_create_cq returned %s.\n", \r
851                         p_port->p_adapter->p_ifc->get_err_str( status )) );\r
852                 return status;\r
853         }\r
854 \r
855         /* Allocate the QP. */\r
856         cl_memclr( &qp_create, sizeof(qp_create) );\r
857         qp_create.qp_type = IB_QPT_UNRELIABLE_DGRM;\r
858         qp_create.rq_depth = p_port->p_adapter->params.rq_depth;\r
859         qp_create.rq_sge = 2;   /* To support buffers spanning pages. */\r
860         qp_create.h_rq_cq = p_port->ib_mgr.h_recv_cq;\r
861         qp_create.sq_depth = p_port->p_adapter->params.sq_depth;\r
862         //TODO: Figure out the right number of SGE entries for sends.\r
863         qp_create.sq_sge = MAX_SEND_SGE;\r
864         qp_create.h_sq_cq = p_port->ib_mgr.h_send_cq;\r
865         qp_create.sq_signaled = TRUE;\r
866         status = p_port->p_adapter->p_ifc->create_qp(\r
867                 p_port->ib_mgr.h_pd, &qp_create, p_port,\r
868                 __qp_event, &p_port->ib_mgr.h_qp );\r
869         if( status != IB_SUCCESS )\r
870         {\r
871                 NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
872                         EVENT_IPOIB_CREATE_QP, 1, status );\r
873                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
874                         ("ib_create_qp returned %s\n", \r
875                         p_port->p_adapter->p_ifc->get_err_str( status )) );\r
876                 return status;\r
877         }\r
878         /* Query the QP so we can get our QPN. */\r
879         status = p_port->p_adapter->p_ifc->query_qp(\r
880                 p_port->ib_mgr.h_qp, &qp_attr );\r
881         if( status != IB_SUCCESS )\r
882         {\r
883                 NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
884                         EVENT_IPOIB_QUERY_QP, 1, status );\r
885                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
886                         ("ib_query_qp returned %s\n", \r
887                         p_port->p_adapter->p_ifc->get_err_str( status )) );\r
888                 return status;\r
889         }\r
890         p_port->ib_mgr.qpn = qp_attr.num;\r
891 \r
892         /* Register all of physical memory */\r
893         phys_create.length = MEM_REG_SIZE;\r
894         phys_create.num_ranges = 1;\r
895         phys_create.range_array = &phys_range;\r
896         phys_create.buf_offset = 0;\r
897         phys_create.hca_page_size = PAGE_SIZE;\r
898         phys_create.access_ctrl = IB_AC_LOCAL_WRITE;\r
899         phys_range.base_addr = 0;\r
900         phys_range.size = MEM_REG_SIZE;\r
901         vaddr = 0;\r
902         status = p_port->p_adapter->p_ifc->reg_phys(\r
903                 p_port->ib_mgr.h_pd, &phys_create, &vaddr,\r
904                 &p_port->ib_mgr.lkey, &rkey, &p_port->ib_mgr.h_mr );\r
905         if( status != IB_SUCCESS )\r
906         {\r
907                 NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
908                         EVENT_IPOIB_REG_PHYS, 1, status );\r
909                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
910                         ("ib_reg_phys returned %s\n", \r
911                         p_port->p_adapter->p_ifc->get_err_str( status )) );\r
912                 return status;\r
913         }\r
914 \r
915         IPOIB_EXIT( IPOIB_DBG_INIT );\r
916         return IB_SUCCESS;\r
917 }\r
918 \r
919 \r
920 static void\r
921 __ib_mgr_destroy(\r
922         IN                              ipoib_port_t* const                     p_port )\r
923 {\r
924         ib_api_status_t status;\r
925 \r
926         IPOIB_ENTER( IPOIB_DBG_INIT );\r
927 \r
928         if( p_port->ib_mgr.h_ca )\r
929         {\r
930                 status =\r
931                         p_port->p_adapter->p_ifc->close_ca( p_port->ib_mgr.h_ca, NULL );\r
932                 CL_ASSERT( status == IB_SUCCESS );\r
933                 p_port->ib_mgr.h_ca = NULL;\r
934         }\r
935 \r
936         IPOIB_EXIT( IPOIB_DBG_INIT );\r
937 }\r
938 \r
939 \r
940 \r
941 /******************************************************************************\r
942 *\r
943 * Buffer manager implementation.\r
944 *\r
945 ******************************************************************************/\r
946 static void\r
947 __buf_mgr_construct(\r
948         IN                              ipoib_port_t* const                     p_port )\r
949 {\r
950         IPOIB_ENTER( IPOIB_DBG_INIT );\r
951 \r
952         cl_qpool_construct( &p_port->buf_mgr.recv_pool );\r
953 \r
954         p_port->buf_mgr.h_packet_pool = NULL;\r
955         p_port->buf_mgr.h_buffer_pool = NULL;\r
956 \r
957         ExInitializeNPagedLookasideList( &p_port->buf_mgr.send_buf_list,\r
958                 NULL, NULL, 0, MAX_XFER_BLOCK_SIZE, 'bipi', 0 );\r
959 \r
960         p_port->buf_mgr.h_send_pkt_pool = NULL;\r
961         p_port->buf_mgr.h_send_buf_pool = NULL;\r
962 \r
963         IPOIB_EXIT( IPOIB_DBG_INIT );\r
964 }\r
965 \r
966 \r
967 static ib_api_status_t\r
968 __buf_mgr_init(\r
969         IN                              ipoib_port_t* const                     p_port )\r
970 {\r
971         cl_status_t             cl_status;\r
972         NDIS_STATUS             ndis_status;\r
973         ipoib_params_t  *p_params;\r
974 \r
975         IPOIB_ENTER(IPOIB_DBG_INIT );\r
976 \r
977         CL_ASSERT( p_port );\r
978         CL_ASSERT( p_port->p_adapter );\r
979 \r
980         p_params = &p_port->p_adapter->params;\r
981 \r
982         /* Allocate the receive descriptor pool */\r
983         cl_status = cl_qpool_init( &p_port->buf_mgr.recv_pool,\r
984                 p_params->rq_depth * p_params->recv_pool_ratio,\r
985 #if IPOIB_INLINE_RECV\r
986                 0, 0, sizeof(ipoib_recv_desc_t), __recv_ctor, NULL, p_port );\r
987 #else   /* IPOIB_INLINE_RECV */\r
988                 0, 0, sizeof(ipoib_recv_desc_t), __recv_ctor, __recv_dtor, p_port );\r
989 #endif  /* IPOIB_INLINE_RECV */\r
990         if( cl_status != CL_SUCCESS )\r
991         {\r
992                 NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
993                         EVENT_IPOIB_RECV_POOL, 1, cl_status );\r
994                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
995                         ("cl_qpool_init for recvs returned %s\n",\r
996                         cl_status_text[cl_status]) );\r
997                 return IB_INSUFFICIENT_MEMORY;\r
998         }\r
999 \r
1000         /* Allocate the NDIS buffer and packet pools for receive indication. */\r
1001         NdisAllocatePacketPool( &ndis_status, &p_port->buf_mgr.h_packet_pool,\r
1002                 p_params->rq_depth, PROTOCOL_RESERVED_SIZE_IN_PACKET );\r
1003         if( ndis_status != NDIS_STATUS_SUCCESS )\r
1004         {\r
1005                 NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
1006                         EVENT_IPOIB_RECV_PKT_POOL, 1, ndis_status );\r
1007                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1008                         ("NdisAllocatePacketPool returned %08X\n", ndis_status) );\r
1009                 return IB_INSUFFICIENT_RESOURCES;\r
1010         }\r
1011 \r
1012         NdisAllocateBufferPool( &ndis_status, &p_port->buf_mgr.h_buffer_pool,\r
1013                 p_params->rq_depth );\r
1014         if( ndis_status != NDIS_STATUS_SUCCESS )\r
1015         {\r
1016                 NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
1017                         EVENT_IPOIB_RECV_BUF_POOL, 1, ndis_status );\r
1018                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1019                         ("NdisAllocateBufferPool returned %08X\n", ndis_status) );\r
1020                 return IB_INSUFFICIENT_RESOURCES;\r
1021         }\r
1022 \r
1023         /* Allocate the NDIS buffer and packet pools for send formatting. */\r
1024         NdisAllocatePacketPool( &ndis_status, &p_port->buf_mgr.h_send_pkt_pool,\r
1025                 1, PROTOCOL_RESERVED_SIZE_IN_PACKET );\r
1026         if( ndis_status != NDIS_STATUS_SUCCESS )\r
1027         {\r
1028                 NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
1029                         EVENT_IPOIB_SEND_PKT_POOL, 1, ndis_status );\r
1030                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1031                         ("NdisAllocatePacketPool returned %08X\n", ndis_status) );\r
1032                 return IB_INSUFFICIENT_RESOURCES;\r
1033         }\r
1034 \r
1035         NdisAllocateBufferPool( &ndis_status,\r
1036                 &p_port->buf_mgr.h_send_buf_pool, 1 );\r
1037         if( ndis_status != NDIS_STATUS_SUCCESS )\r
1038         {\r
1039                 NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
1040                         EVENT_IPOIB_SEND_BUF_POOL, 1, ndis_status );\r
1041                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1042                         ("NdisAllocateBufferPool returned %08X\n", ndis_status) );\r
1043                 return IB_INSUFFICIENT_RESOURCES;\r
1044         }\r
1045 \r
1046         IPOIB_EXIT( IPOIB_DBG_INIT );\r
1047         return IB_SUCCESS;\r
1048 }\r
1049 \r
1050 \r
1051 static void\r
1052 __buf_mgr_destroy(\r
1053         IN                              ipoib_port_t* const                     p_port )\r
1054 {\r
1055         IPOIB_ENTER(IPOIB_DBG_INIT );\r
1056 \r
1057         CL_ASSERT( p_port );\r
1058 \r
1059         /* Destroy the send packet and buffer pools. */\r
1060         if( p_port->buf_mgr.h_send_buf_pool )\r
1061                 NdisFreeBufferPool( p_port->buf_mgr.h_send_buf_pool );\r
1062         if( p_port->buf_mgr.h_send_pkt_pool )\r
1063                 NdisFreePacketPool( p_port->buf_mgr.h_send_pkt_pool );\r
1064 \r
1065         /* Destroy the receive packet and buffer pools. */\r
1066         if( p_port->buf_mgr.h_buffer_pool )\r
1067                 NdisFreeBufferPool( p_port->buf_mgr.h_buffer_pool );\r
1068         if( p_port->buf_mgr.h_packet_pool )\r
1069                 NdisFreePacketPool( p_port->buf_mgr.h_packet_pool );\r
1070 \r
1071         /* Free the receive and send descriptors. */\r
1072         cl_qpool_destroy( &p_port->buf_mgr.recv_pool );\r
1073 \r
1074         /* Free the lookaside list of scratch buffers. */\r
1075         ExDeleteNPagedLookasideList( &p_port->buf_mgr.send_buf_list );\r
1076 \r
1077         IPOIB_EXIT(  IPOIB_DBG_INIT );\r
1078 }\r
1079 \r
1080 \r
1081 static cl_status_t\r
1082 __recv_ctor(\r
1083         IN                              void* const                                     p_object,\r
1084         IN                              void*                                           context,\r
1085                 OUT                     cl_pool_item_t** const          pp_pool_item )\r
1086 {\r
1087         ipoib_recv_desc_t       *p_desc;\r
1088         ipoib_port_t            *p_port;\r
1089         uint32_t                        ds0_len;\r
1090 \r
1091         IPOIB_ENTER( IPOIB_DBG_ALLOC );\r
1092 \r
1093         CL_ASSERT( p_object );\r
1094         CL_ASSERT( context );\r
1095 \r
1096         p_desc = (ipoib_recv_desc_t*)p_object;\r
1097         p_port = (ipoib_port_t*)context;\r
1098 \r
1099         /* Setup the work request. */\r
1100         p_desc->wr.ds_array = p_desc->local_ds;\r
1101         p_desc->wr.wr_id = (uintn_t)p_desc;\r
1102 \r
1103 #if IPOIB_INLINE_RECV\r
1104         /* Sanity check on the receive buffer layout */\r
1105         CL_ASSERT( (void*)&p_desc->buf.eth.pkt.type ==\r
1106                 (void*)&p_desc->buf.ib.pkt.type );\r
1107         CL_ASSERT( sizeof(recv_buf_t) == sizeof(ipoib_pkt_t) + sizeof(ib_grh_t) );\r
1108 \r
1109         /* Setup the local data segment. */\r
1110         p_desc->local_ds[0].vaddr = cl_get_physaddr( &p_desc->buf );\r
1111         p_desc->local_ds[0].lkey = p_port->ib_mgr.lkey;\r
1112         ds0_len =\r
1113                 PAGE_SIZE - ((uint32_t)p_desc->local_ds[0].vaddr & (PAGE_SIZE - 1));\r
1114         if( ds0_len >= sizeof(recv_buf_t) )\r
1115         {\r
1116                 /* The whole buffer is within a page. */\r
1117                 p_desc->local_ds[0].length = ds0_len;\r
1118                 p_desc->wr.num_ds = 1;\r
1119         }\r
1120         else\r
1121         {\r
1122                 /* The buffer crosses page boundaries. */\r
1123                 p_desc->local_ds[0].length = ds0_len;\r
1124                 p_desc->local_ds[1].vaddr = cl_get_physaddr( \r
1125                         ((uint8_t*)&p_desc->buf) + ds0_len );\r
1126                 p_desc->local_ds[1].lkey = p_port->ib_mgr.lkey;\r
1127                 p_desc->local_ds[1].length = sizeof(recv_buf_t) - ds0_len;\r
1128                 p_desc->wr.num_ds = 2;\r
1129         }\r
1130 #else   /* IPOIB_INLINE_RECV */\r
1131         /* Allocate the receive buffer. */\r
1132         p_desc->p_buf = (recv_buf_t*)cl_zalloc( sizeof(recv_buf_t) );\r
1133         if( !p_desc->p_buf )\r
1134         {\r
1135                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1136                         ("Failed to allocate receive buffer.\n") );\r
1137                 return CL_INSUFFICIENT_MEMORY;\r
1138         }\r
1139 \r
1140         /* Sanity check on the receive buffer layout */\r
1141         CL_ASSERT( (void*)&p_desc->p_buf->eth.pkt.type ==\r
1142                 (void*)&p_desc->p_buf->ib.pkt.type );\r
1143 \r
1144         /* Setup the local data segment. */\r
1145         p_desc->local_ds[0].vaddr = cl_get_physaddr( p_desc->p_buf );\r
1146         p_desc->local_ds[0].length = sizeof(ipoib_pkt_t) + sizeof(ib_grh_t);\r
1147         p_desc->local_ds[0].lkey = p_port->ib_mgr.lkey;\r
1148 #endif  /* IPOIB_INLINE_RECV */\r
1149 \r
1150         *pp_pool_item = &p_desc->item;\r
1151 \r
1152         IPOIB_EXIT( IPOIB_DBG_ALLOC );\r
1153         return CL_SUCCESS;\r
1154 }\r
1155 \r
1156 \r
1157 #if !IPOIB_INLINE_RECV\r
1158 static void\r
1159 __recv_dtor(\r
1160         IN              const   cl_pool_item_t* const           p_pool_item,\r
1161         IN                              void                                            *context )\r
1162 {\r
1163         ipoib_recv_desc_t       *p_desc;\r
1164 \r
1165         IPOIB_ENTER(  IPOIB_DBG_ALLOC );\r
1166 \r
1167         UNUSED_PARAM( context );\r
1168 \r
1169         p_desc = PARENT_STRUCT( p_pool_item, ipoib_recv_desc_t, item );\r
1170 \r
1171         if( p_desc->p_buf )\r
1172                 cl_free( p_desc->p_buf );\r
1173 \r
1174         IPOIB_EXIT( IPOIB_DBG_ALLOC );\r
1175 }\r
1176 #endif\r
1177 \r
1178 \r
1179 static inline ipoib_recv_desc_t*\r
1180 __buf_mgr_get_recv(\r
1181         IN                              ipoib_port_t* const                     p_port )\r
1182 {\r
1183         ipoib_recv_desc_t       *p_desc;\r
1184         IPOIB_ENTER( IPOIB_DBG_RECV );\r
1185         p_desc = (ipoib_recv_desc_t*)cl_qpool_get( &p_port->buf_mgr.recv_pool );\r
1186         /* Reference the port object for the send. */\r
1187         if( p_desc )\r
1188         {\r
1189                 ipoib_port_ref( p_port, ref_get_recv );\r
1190                 CL_ASSERT( p_desc->wr.wr_id == (uintn_t)p_desc );\r
1191 #if IPOIB_INLINE_RECV\r
1192                 CL_ASSERT( p_desc->local_ds[0].vaddr ==\r
1193                         cl_get_physaddr( &p_desc->buf ) );\r
1194 #else   /* IPOIB_INLINE_RECV */\r
1195                 CL_ASSERT( p_desc->local_ds[0].vaddr ==\r
1196                         cl_get_physaddr( p_desc->p_buf ) );\r
1197                 CL_ASSERT( p_desc->local_ds[0].length ==\r
1198                         (sizeof(ipoib_pkt_t) + sizeof(ib_grh_t)) );\r
1199 #endif  /* IPOIB_INLINE_RECV */\r
1200                 CL_ASSERT( p_desc->local_ds[0].lkey == p_port->ib_mgr.lkey );\r
1201         }\r
1202         IPOIB_EXIT( IPOIB_DBG_RECV );\r
1203         return p_desc;\r
1204 }\r
1205 \r
1206 \r
1207 static inline void\r
1208 __buf_mgr_put_recv(\r
1209         IN                              ipoib_port_t* const                     p_port,\r
1210         IN                              ipoib_recv_desc_t* const        p_desc,\r
1211         IN                              NDIS_PACKET* const                      p_packet OPTIONAL )\r
1212 {\r
1213         NDIS_BUFFER             *p_buf;\r
1214 \r
1215         IPOIB_ENTER(IPOIB_DBG_RECV );\r
1216 \r
1217         if( p_packet )\r
1218         {\r
1219                 /* Unchain the NDIS buffer. */\r
1220                 NdisUnchainBufferAtFront( p_packet, &p_buf );\r
1221                 CL_ASSERT( p_buf );\r
1222                 /* Return the NDIS packet and NDIS buffer to their pools. */\r
1223                 NdisDprFreePacketNonInterlocked( p_packet );\r
1224                 NdisFreeBuffer( p_buf );\r
1225         }\r
1226 \r
1227         /* Return the descriptor to its pools. */\r
1228         cl_qpool_put( &p_port->buf_mgr.recv_pool, &p_desc->item );\r
1229 \r
1230         /*\r
1231          * Dereference the port object since the receive is no longer outstanding.\r
1232          */\r
1233         ipoib_port_deref( p_port, ref_get_recv );\r
1234         IPOIB_EXIT(  IPOIB_DBG_RECV );\r
1235 }\r
1236 \r
1237 \r
1238 static inline void\r
1239 __buf_mgr_put_recv_list(\r
1240         IN                              ipoib_port_t* const                     p_port,\r
1241         IN                              cl_qlist_t* const                       p_list )\r
1242 {\r
1243         IPOIB_ENTER(  IPOIB_DBG_RECV );\r
1244         cl_qpool_put_list( &p_port->buf_mgr.recv_pool, p_list );\r
1245         IPOIB_EXIT(  IPOIB_DBG_RECV );\r
1246 }\r
1247 \r
1248 \r
1249 static inline NDIS_PACKET*\r
1250 __buf_mgr_get_ndis_pkt(\r
1251         IN                              ipoib_port_t* const                     p_port,\r
1252         IN                              ipoib_recv_desc_t* const        p_desc )\r
1253 {\r
1254         NDIS_STATUS                             status;\r
1255         NDIS_PACKET                             *p_packet;\r
1256         NDIS_BUFFER                             *p_buffer;\r
1257 \r
1258         IPOIB_ENTER(  IPOIB_DBG_RECV );\r
1259 \r
1260         NdisDprAllocatePacketNonInterlocked( &status, &p_packet,\r
1261                 p_port->buf_mgr.h_packet_pool );\r
1262         if( status != NDIS_STATUS_SUCCESS )\r
1263         {\r
1264                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1265                         ("Failed to allocate NDIS_PACKET: %08x\n", status) );\r
1266                 return NULL;\r
1267         }\r
1268 \r
1269         IPOIB_PORT_FROM_PACKET( p_packet ) = p_port;\r
1270         IPOIB_RECV_FROM_PACKET( p_packet ) = p_desc;\r
1271 \r
1272         NdisAllocateBuffer( &status, &p_buffer,\r
1273 #if IPOIB_INLINE_RECV\r
1274                 p_port->buf_mgr.h_buffer_pool, &p_desc->buf.eth.pkt, p_desc->len );\r
1275 #else   /* IPOIB_INLINE_RECV */\r
1276                 p_port->buf_mgr.h_buffer_pool, &p_desc->p_buf->eth.pkt, p_desc->len );\r
1277 #endif  /* IPOIB_INLINE_RECV */\r
1278         if( status != NDIS_STATUS_SUCCESS )\r
1279         {\r
1280                 IPOIB_PRINT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1281                         ("Failed to allocate NDIS_BUFFER: %08x\n", status) );\r
1282                 NdisDprFreePacketNonInterlocked( p_packet );\r
1283                 return NULL;\r
1284         }\r
1285 \r
1286         NdisChainBufferAtFront( p_packet, p_buffer );\r
1287         NDIS_SET_PACKET_HEADER_SIZE( p_packet, sizeof(eth_hdr_t) );\r
1288 \r
1289         IPOIB_EXIT(  IPOIB_DBG_RECV );\r
1290         return p_packet;\r
1291 }\r
1292 \r
1293 /******************************************************************************\r
1294 *\r
1295 * Receive manager implementation.\r
1296 *\r
1297 ******************************************************************************/\r
1298 static void\r
1299 __recv_mgr_construct(\r
1300         IN                              ipoib_port_t* const                     p_port )\r
1301 {\r
1302         IPOIB_ENTER( IPOIB_DBG_INIT );\r
1303 \r
1304         cl_qlist_init( &p_port->recv_mgr.done_list );\r
1305 \r
1306         p_port->recv_mgr.recv_pkt_array = NULL;\r
1307 \r
1308         IPOIB_EXIT( IPOIB_DBG_INIT );\r
1309 }\r
1310 \r
1311 \r
1312 static ib_api_status_t\r
1313 __recv_mgr_init(\r
1314         IN                              ipoib_port_t* const                     p_port )\r
1315 {\r
1316         IPOIB_ENTER( IPOIB_DBG_INIT );\r
1317 \r
1318         /* Allocate the NDIS_PACKET pointer array for indicating receives. */\r
1319         p_port->recv_mgr.recv_pkt_array = cl_malloc(\r
1320                 sizeof(NDIS_PACKET*) * p_port->p_adapter->params.rq_depth );\r
1321         if( !p_port->recv_mgr.recv_pkt_array )\r
1322         {\r
1323                 NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
1324                         EVENT_IPOIB_RECV_PKT_ARRAY, 0 );\r
1325                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1326                         ("cl_malloc for PNDIS_PACKET array failed.\n") );\r
1327                 return IB_INSUFFICIENT_MEMORY;\r
1328         }\r
1329 \r
1330         IPOIB_EXIT( IPOIB_DBG_INIT );\r
1331         return IB_SUCCESS;\r
1332 }\r
1333 \r
1334 \r
1335 static void\r
1336 __recv_mgr_destroy(\r
1337         IN                              ipoib_port_t* const                     p_port )\r
1338 {\r
1339         IPOIB_ENTER( IPOIB_DBG_INIT );\r
1340 \r
1341         CL_ASSERT( cl_is_qlist_empty( &p_port->recv_mgr.done_list ) );\r
1342         CL_ASSERT( !p_port->recv_mgr.depth );\r
1343 \r
1344         if( p_port->recv_mgr.recv_pkt_array )\r
1345                 cl_free( p_port->recv_mgr.recv_pkt_array );\r
1346 \r
1347         IPOIB_EXIT( IPOIB_DBG_INIT );\r
1348 }\r
1349 \r
1350 \r
1351 /*\r
1352  * Posts receive buffers to the receive queue and returns the number\r
1353  * of receives needed to bring the RQ to its low water mark.  Note\r
1354  * that the value is signed, and can go negative.  All tests must\r
1355  * be for > 0.\r
1356  */\r
1357 static int32_t\r
1358 __recv_mgr_repost(\r
1359         IN                              ipoib_port_t* const                     p_port )\r
1360 {\r
1361         ipoib_recv_desc_t       *p_head = NULL, *p_tail = NULL, *p_next;\r
1362         ib_api_status_t         status;\r
1363         ib_recv_wr_t            *p_failed;\r
1364         PERF_DECLARE( GetRecv );\r
1365         PERF_DECLARE( PostRecv );\r
1366 \r
1367         IPOIB_ENTER( IPOIB_DBG_RECV );\r
1368 \r
1369         CL_ASSERT( p_port );\r
1370         cl_obj_lock( &p_port->obj );\r
1371         if( p_port->state != IB_QPS_RTS )\r
1372         {\r
1373                 cl_obj_unlock( &p_port->obj );\r
1374                 IPOIB_PRINT_EXIT( TRACE_LEVEL_INFORMATION, IPOIB_DBG_RECV,\r
1375                         ("Port in invalid state.  Not reposting.\n") );\r
1376                 return 0;\r
1377         }\r
1378         ipoib_port_ref( p_port, ref_repost );\r
1379         cl_obj_unlock( &p_port->obj );\r
1380 \r
1381         while( p_port->recv_mgr.depth < p_port->p_adapter->params.rq_depth )\r
1382         {\r
1383                 /* Pull receives out of the pool and chain them up. */\r
1384                 cl_perf_start( GetRecv );\r
1385                 p_next = __buf_mgr_get_recv( p_port );\r
1386                 cl_perf_stop( &p_port->p_adapter->perf, GetRecv );\r
1387                 if( !p_next )\r
1388                 {\r
1389                         IPOIB_PRINT(TRACE_LEVEL_INFORMATION, IPOIB_DBG_RECV,\r
1390                                 ("Out of receive descriptors!\n") );\r
1391                         break;\r
1392                 }\r
1393 \r
1394                 if( !p_tail )\r
1395                 {\r
1396                         p_tail = p_next;\r
1397                         p_next->wr.p_next = NULL;\r
1398                 }\r
1399                 else\r
1400                 {\r
1401                         p_next->wr.p_next = &p_head->wr;\r
1402                 }\r
1403 \r
1404                 p_head = p_next;\r
1405 \r
1406                 p_port->recv_mgr.depth++;\r
1407         }\r
1408 \r
1409         if( p_head )\r
1410         {\r
1411                 cl_perf_start( PostRecv );\r
1412                 status = p_port->p_adapter->p_ifc->post_recv(\r
1413                         p_port->ib_mgr.h_qp, &p_head->wr, &p_failed );\r
1414                 cl_perf_stop( &p_port->p_adapter->perf, PostRecv );\r
1415 \r
1416                 if( status != IB_SUCCESS )\r
1417                 {\r
1418                         IPOIB_PRINT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1419                                 ("ip_post_recv returned %s\n", \r
1420                                 p_port->p_adapter->p_ifc->get_err_str( status )) );\r
1421                         /* return the descriptors to the pool */\r
1422                         while( p_failed )\r
1423                         {\r
1424                                 p_head = PARENT_STRUCT( p_failed, ipoib_recv_desc_t, wr );\r
1425                                 p_failed = p_failed->p_next;\r
1426 \r
1427                                 __buf_mgr_put_recv( p_port, p_head, NULL );\r
1428                                 p_port->recv_mgr.depth--;\r
1429                         }\r
1430                 }\r
1431         }\r
1432 \r
1433         ipoib_port_deref( p_port, ref_repost );\r
1434         IPOIB_EXIT( IPOIB_DBG_RECV );\r
1435         return p_port->p_adapter->params.rq_low_watermark - p_port->recv_mgr.depth;\r
1436 }\r
1437 \r
1438 \r
1439 void\r
1440 ipoib_return_packet(\r
1441         IN                              NDIS_HANDLE                                     adapter_context,\r
1442         IN                              NDIS_PACKET                                     *p_packet )\r
1443 {\r
1444         cl_list_item_t          *p_item;\r
1445         ipoib_port_t            *p_port;\r
1446         ipoib_recv_desc_t       *p_desc;\r
1447         ib_api_status_t         status = IB_NOT_DONE;\r
1448         int32_t                         shortage;\r
1449         PERF_DECLARE( ReturnPacket );\r
1450         PERF_DECLARE( ReturnPutRecv );\r
1451         PERF_DECLARE( ReturnRepostRecv );\r
1452         PERF_DECLARE( ReturnPreparePkt );\r
1453         PERF_DECLARE( ReturnNdisIndicate );\r
1454 \r
1455         IPOIB_ENTER( IPOIB_DBG_RECV );\r
1456 \r
1457         UNUSED_PARAM( adapter_context );\r
1458         CL_ASSERT( p_packet );\r
1459 \r
1460         cl_perf_start( ReturnPacket );\r
1461 \r
1462         /* Get the port and descriptor from the packet. */\r
1463         p_port = IPOIB_PORT_FROM_PACKET( p_packet );\r
1464         p_desc = IPOIB_RECV_FROM_PACKET( p_packet );\r
1465 \r
1466         cl_spinlock_acquire( &p_port->recv_lock );\r
1467 \r
1468         cl_perf_start( ReturnPutRecv );\r
1469         __buf_mgr_put_recv( p_port, p_desc, p_packet );\r
1470         cl_perf_stop( &p_port->p_adapter->perf, ReturnPutRecv );\r
1471 \r
1472         /* Repost buffers. */\r
1473         cl_perf_start( ReturnRepostRecv );\r
1474         shortage = __recv_mgr_repost( p_port );\r
1475         cl_perf_stop( &p_port->p_adapter->perf, ReturnRepostRecv );\r
1476 \r
1477         for( p_item = cl_qlist_remove_head( &p_port->recv_mgr.done_list );\r
1478                 p_item != cl_qlist_end( &p_port->recv_mgr.done_list );\r
1479                 p_item = cl_qlist_remove_head( &p_port->recv_mgr.done_list ) )\r
1480         {\r
1481                 p_desc = (ipoib_recv_desc_t*)p_item;\r
1482 \r
1483                 cl_perf_start( ReturnPreparePkt );\r
1484                 status = __recv_mgr_prepare_pkt( p_port, p_desc, &p_packet );\r
1485                 cl_perf_stop( &p_port->p_adapter->perf, ReturnPreparePkt );\r
1486                 if( status == IB_SUCCESS )\r
1487                 {\r
1488                         if( shortage > 0 )\r
1489                                 NDIS_SET_PACKET_STATUS( p_packet, NDIS_STATUS_RESOURCES );\r
1490                         else\r
1491                                 NDIS_SET_PACKET_STATUS( p_packet, NDIS_STATUS_SUCCESS );\r
1492 \r
1493                         cl_spinlock_release( &p_port->recv_lock );\r
1494                         cl_perf_start( ReturnNdisIndicate );\r
1495                         NdisMIndicateReceivePacket( p_port->p_adapter->h_adapter,\r
1496                                 &p_packet, 1 );\r
1497                         cl_perf_stop( &p_port->p_adapter->perf, ReturnNdisIndicate );\r
1498                         cl_spinlock_acquire( &p_port->recv_lock );\r
1499 \r
1500                         if( shortage > 0 )\r
1501                         {\r
1502                                 cl_perf_start( ReturnPutRecv );\r
1503                                 __buf_mgr_put_recv( p_port, p_desc, p_packet );\r
1504                                 cl_perf_stop( &p_port->p_adapter->perf, ReturnPutRecv );\r
1505 \r
1506                                 /* Repost buffers. */\r
1507                                 cl_perf_start( ReturnRepostRecv );\r
1508                                 shortage = __recv_mgr_repost( p_port );\r
1509                                 cl_perf_stop( &p_port->p_adapter->perf, ReturnRepostRecv );\r
1510                         }\r
1511                 }\r
1512                 else if( status != IB_NOT_DONE )\r
1513                 {\r
1514                         IPOIB_PRINT(TRACE_LEVEL_INFORMATION, IPOIB_DBG_RECV,\r
1515                                 ("__recv_mgr_prepare_pkt returned %s\n",\r
1516                                 p_port->p_adapter->p_ifc->get_err_str( status )) );\r
1517                         /* Return the item to the head of the list. */\r
1518                         cl_qlist_insert_head( &p_port->recv_mgr.done_list, p_item );\r
1519                         break;\r
1520                 }\r
1521         }\r
1522         cl_spinlock_release( &p_port->recv_lock );\r
1523         cl_perf_stop( &p_port->p_adapter->perf, ReturnPacket );\r
1524 \r
1525         IPOIB_EXIT( IPOIB_DBG_RECV );\r
1526 }\r
1527 \r
1528 \r
1529 static void\r
1530 __recv_cb(\r
1531         IN              const   ib_cq_handle_t                          h_cq,\r
1532         IN                              void                                            *cq_context )\r
1533 {\r
1534         ipoib_port_t            *p_port;\r
1535         ib_api_status_t         status;\r
1536         ib_wc_t                         wc[MAX_RECV_WC], *p_free, *p_wc;\r
1537         int32_t                         pkt_cnt, recv_cnt = 0, shortage, discarded;\r
1538         cl_qlist_t                      done_list, bad_list;\r
1539         size_t                          i;\r
1540         PERF_DECLARE( RecvCompBundle );\r
1541         PERF_DECLARE( RecvCb );\r
1542         PERF_DECLARE( PollRecv );\r
1543         PERF_DECLARE( RepostRecv );\r
1544         PERF_DECLARE( FilterRecv );\r
1545         PERF_DECLARE( BuildPktArray );\r
1546         PERF_DECLARE( RecvNdisIndicate );\r
1547         PERF_DECLARE( RearmRecv );\r
1548         PERF_DECLARE( PutRecvList );\r
1549 \r
1550         IPOIB_ENTER( IPOIB_DBG_RECV );\r
1551 \r
1552         cl_perf_clr( RecvCompBundle );\r
1553 \r
1554         cl_perf_start( RecvCb );\r
1555 \r
1556         UNUSED_PARAM( h_cq );\r
1557 \r
1558         p_port = (ipoib_port_t*)cq_context;\r
1559 \r
1560         cl_qlist_init( &done_list );\r
1561         cl_qlist_init( &bad_list );\r
1562 \r
1563         ipoib_port_ref( p_port, ref_recv_cb );\r
1564         for( i = 0; i < MAX_RECV_WC; i++ )\r
1565                 wc[i].p_next = &wc[i + 1];\r
1566         wc[MAX_RECV_WC - 1].p_next = NULL;\r
1567 \r
1568         /*\r
1569          * We'll be accessing the endpoint map so take a reference\r
1570          * on it to prevent modifications.\r
1571          */\r
1572         cl_obj_lock( &p_port->obj );\r
1573         cl_atomic_inc( &p_port->endpt_rdr );\r
1574         cl_obj_unlock( &p_port->obj );\r
1575 \r
1576         do\r
1577         {\r
1578                 /* If we get here, then the list of WCs is intact. */\r
1579                 p_free = wc;\r
1580 \r
1581                 cl_perf_start( PollRecv );\r
1582                 status = p_port->p_adapter->p_ifc->poll_cq(\r
1583                         p_port->ib_mgr.h_recv_cq, &p_free, &p_wc );\r
1584                 cl_perf_stop( &p_port->p_adapter->perf, PollRecv );\r
1585                 CL_ASSERT( status == IB_SUCCESS || status == IB_NOT_FOUND );\r
1586 \r
1587                 /* Look at the payload now and filter ARP and DHCP packets. */\r
1588                 cl_perf_start( FilterRecv );\r
1589                 recv_cnt += __recv_mgr_filter( p_port, p_wc, &done_list, &bad_list );\r
1590                 cl_perf_stop( &p_port->p_adapter->perf, FilterRecv );\r
1591 \r
1592         } while( !p_free );\r
1593 \r
1594         /* We're done looking at the endpoint map, release the reference. */\r
1595         cl_atomic_dec( &p_port->endpt_rdr );\r
1596 \r
1597         cl_perf_log( &p_port->p_adapter->perf, RecvCompBundle, recv_cnt );\r
1598 \r
1599         cl_spinlock_acquire( &p_port->recv_lock );\r
1600 \r
1601         /* Update our posted depth. */\r
1602         p_port->recv_mgr.depth -= recv_cnt;\r
1603 \r
1604         /* Return any discarded receives to the pool */\r
1605         cl_perf_start( PutRecvList );\r
1606         __buf_mgr_put_recv_list( p_port, &bad_list );\r
1607         cl_perf_stop( &p_port->p_adapter->perf, PutRecvList );\r
1608 \r
1609         do\r
1610         {\r
1611                 /* Repost ASAP so we don't starve the RQ. */\r
1612                 cl_perf_start( RepostRecv );\r
1613                 shortage = __recv_mgr_repost( p_port );\r
1614                 cl_perf_stop( &p_port->p_adapter->perf, RepostRecv );\r
1615 \r
1616                 cl_perf_start( BuildPktArray );\r
1617                 /* Notify NDIS of any and all possible receive buffers. */\r
1618                 pkt_cnt = __recv_mgr_build_pkt_array(\r
1619                         p_port, shortage, &done_list, &discarded );\r
1620                 cl_perf_stop( &p_port->p_adapter->perf, BuildPktArray );\r
1621 \r
1622                 /* Only indicate receives if we actually had any. */\r
1623                 if( discarded && shortage > 0 )\r
1624                 {\r
1625                         /* We may have thrown away packets, and have a shortage */\r
1626                         cl_perf_start( RepostRecv );\r
1627                         __recv_mgr_repost( p_port );\r
1628                         cl_perf_stop( &p_port->p_adapter->perf, RepostRecv );\r
1629                 }\r
1630 \r
1631                 if( !pkt_cnt )\r
1632                         break;\r
1633 \r
1634                 cl_spinlock_release( &p_port->recv_lock );\r
1635 \r
1636                 cl_perf_start( RecvNdisIndicate );\r
1637                 NdisMIndicateReceivePacket( p_port->p_adapter->h_adapter,\r
1638                         p_port->recv_mgr.recv_pkt_array, pkt_cnt );\r
1639                 cl_perf_stop( &p_port->p_adapter->perf, RecvNdisIndicate );\r
1640 \r
1641                 /*\r
1642                  * Cap the number of receives to put back to what we just indicated\r
1643                  * with NDIS_STATUS_RESOURCES.\r
1644                  */\r
1645                 if( shortage > 0 )\r
1646                 {\r
1647                         if( pkt_cnt < shortage )\r
1648                                 shortage = pkt_cnt;\r
1649 \r
1650                         /* Return all but the last packet to the pool. */\r
1651                         cl_spinlock_acquire( &p_port->recv_lock );\r
1652                         while( shortage-- > 1 )\r
1653                         {\r
1654                                 __buf_mgr_put_recv( p_port,\r
1655                                         IPOIB_RECV_FROM_PACKET( p_port->recv_mgr.recv_pkt_array[shortage] ),\r
1656                                         p_port->recv_mgr.recv_pkt_array[shortage] );\r
1657                         }\r
1658                         cl_spinlock_release( &p_port->recv_lock );\r
1659 \r
1660                         /*\r
1661                          * Return the last packet as if NDIS returned it, so that we repost\r
1662                          * and report any other pending receives.\r
1663                          */\r
1664                         ipoib_return_packet( NULL, p_port->recv_mgr.recv_pkt_array[0] );\r
1665                 }\r
1666                 cl_spinlock_acquire( &p_port->recv_lock );\r
1667 \r
1668         } while( pkt_cnt );\r
1669         cl_spinlock_release( &p_port->recv_lock );\r
1670 \r
1671         /*\r
1672          * Rearm after filtering to prevent contention on the enpoint maps\r
1673          * and eliminate the possibility of having a call to\r
1674          * __endpt_mgr_insert find a duplicate.\r
1675          */\r
1676         cl_perf_start( RearmRecv );\r
1677         status = p_port->p_adapter->p_ifc->rearm_cq(\r
1678                 p_port->ib_mgr.h_recv_cq, FALSE );\r
1679         cl_perf_stop( &p_port->p_adapter->perf, RearmRecv );\r
1680         CL_ASSERT( status == IB_SUCCESS );\r
1681 \r
1682         ipoib_port_deref( p_port, ref_recv_cb );\r
1683 \r
1684         cl_perf_stop( &p_port->p_adapter->perf, RecvCb );\r
1685 \r
1686         IPOIB_EXIT( IPOIB_DBG_RECV );\r
1687 }\r
1688 \r
1689 \r
1690 static void\r
1691 __recv_get_endpts(\r
1692         IN                              ipoib_port_t* const                     p_port,\r
1693         IN                              ipoib_recv_desc_t* const        p_desc,\r
1694         IN                              ib_wc_t* const                          p_wc,\r
1695                 OUT                     ipoib_endpt_t** const           pp_src,\r
1696                 OUT                     ipoib_endpt_t** const           pp_dst )\r
1697 {\r
1698         ib_api_status_t         status;\r
1699         mac_addr_t                      mac;\r
1700         PERF_DECLARE( GetEndptByGid );\r
1701         PERF_DECLARE( GetEndptByLid );\r
1702         PERF_DECLARE( EndptInsert );\r
1703 \r
1704         IPOIB_ENTER( IPOIB_DBG_RECV );\r
1705 \r
1706         /* Setup our shortcut pointers based on whether GRH is valid. */\r
1707         if( p_wc->recv.ud.recv_opt & IB_RECV_OPT_GRH_VALID )\r
1708         {\r
1709                 /* Lookup the source endpoints based on GID. */\r
1710                 cl_perf_start( GetEndptByGid );\r
1711                 *pp_src =\r
1712 #if IPOIB_INLINE_RECV\r
1713                         __endpt_mgr_get_by_gid( p_port, &p_desc->buf.ib.grh.src_gid );\r
1714 #else   /* IPOIB_INLINE_RECV */\r
1715                         __endpt_mgr_get_by_gid( p_port, &p_desc->p_buf->ib.grh.src_gid );\r
1716 #endif  /* IPOIB_INLINE_RECV */\r
1717                 cl_perf_stop( &p_port->p_adapter->perf, GetEndptByGid );\r
1718 \r
1719                 /*\r
1720                  * Lookup the destination endpoint based on GID.\r
1721                  * This is used along with the packet filter to determine\r
1722                  * whether to report this to NDIS.\r
1723                  */\r
1724                 cl_perf_start( GetEndptByGid );\r
1725                 *pp_dst =\r
1726 #if IPOIB_INLINE_RECV\r
1727                         __endpt_mgr_get_by_gid( p_port, &p_desc->buf.ib.grh.dest_gid );\r
1728 #else   /* IPOIB_INLINE_RECV */\r
1729                         __endpt_mgr_get_by_gid( p_port, &p_desc->p_buf->ib.grh.dest_gid );\r
1730 #endif  /* IPOIB_INLINE_RECV */\r
1731                 cl_perf_stop( &p_port->p_adapter->perf, GetEndptByGid );\r
1732 \r
1733                 /*\r
1734                  * Create the source endpoint if it does not exist.  Note that we\r
1735                  * can only do this for globally routed traffic since we need the\r
1736                  * information from the GRH to generate the MAC.\r
1737                  */\r
1738                 if( !*pp_src )\r
1739                 {\r
1740                         status = ipoib_mac_from_guid(\r
1741 #if IPOIB_INLINE_RECV\r
1742                                 p_desc->buf.ib.grh.src_gid.unicast.interface_id, &mac );\r
1743 #else   /* IPOIB_INLINE_RECV */\r
1744                                 p_desc->p_buf->ib.grh.src_gid.unicast.interface_id, &mac );\r
1745 #endif  /* IPOIB_INLINE_RECV */\r
1746                         if( status != IB_SUCCESS )\r
1747                         {\r
1748                                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1749                                         ("ipoib_mac_from_guid returned %s\n",\r
1750                                         p_port->p_adapter->p_ifc->get_err_str( status )) );\r
1751                                 return;\r
1752                         }\r
1753 \r
1754                         /* Create the endpoint. */\r
1755 #if IPOIB_INLINE_RECV\r
1756                         *pp_src = ipoib_endpt_create( &p_desc->buf.ib.grh.src_gid,\r
1757 #else   /* IPOIB_INLINE_RECV */\r
1758                         *pp_src = ipoib_endpt_create( &p_desc->p_buf->ib.grh.src_gid,\r
1759 #endif  /* IPOIB_INLINE_RECV */\r
1760                                 0, p_wc->recv.ud.remote_qp );\r
1761                         if( !*pp_src )\r
1762                         {\r
1763                                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1764                                         ("ipoib_endpt_create failed\n") );\r
1765                                 return;\r
1766                         }\r
1767                         cl_perf_start( EndptInsert );\r
1768                         cl_obj_lock( &p_port->obj );\r
1769                         status = __endpt_mgr_insert( p_port, mac, *pp_src );\r
1770                         if( status != IB_SUCCESS )\r
1771                         {\r
1772                                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1773                                         ("__endpt_mgr_insert returned %s\n",\r
1774                                         p_port->p_adapter->p_ifc->get_err_str( status )) );\r
1775                                 return;\r
1776                         }\r
1777                         cl_obj_unlock( &p_port->obj );\r
1778                         cl_perf_stop( &p_port->p_adapter->perf, EndptInsert );\r
1779                 }\r
1780         }\r
1781         else\r
1782         {\r
1783                 /*\r
1784                  * Lookup the remote endpoint based on LID.  Note that only\r
1785                  * unicast traffic can be LID routed.\r
1786                  */\r
1787                 cl_perf_start( GetEndptByLid );\r
1788                 *pp_src = __endpt_mgr_get_by_lid( p_port, p_wc->recv.ud.remote_lid );\r
1789                 cl_perf_stop( &p_port->p_adapter->perf, GetEndptByLid );\r
1790                 *pp_dst = p_port->p_local_endpt;\r
1791                 CL_ASSERT( *pp_dst );\r
1792         }\r
1793 \r
1794         if( *pp_src && !ipoib_is_voltaire_router_gid( &(*pp_src)->dgid ) &&\r
1795                 (*pp_src)->qpn != p_wc->recv.ud.remote_qp )\r
1796         {\r
1797                 /* Update the QPN for the endpoint. */\r
1798                 IPOIB_PRINT(TRACE_LEVEL_INFORMATION, IPOIB_DBG_RECV,\r
1799                         ("Updating QPN for MAC: %02X-%02X-%02X-%02X-%02X-%02X\n",\r
1800                         (*pp_src )->mac.addr[0], (*pp_src )->mac.addr[1],\r
1801                         (*pp_src )->mac.addr[2], (*pp_src )->mac.addr[3],\r
1802                         (*pp_src )->mac.addr[4], (*pp_src )->mac.addr[5]) );\r
1803                 (*pp_src)->qpn = p_wc->recv.ud.remote_qp;\r
1804         }\r
1805 \r
1806         if( *pp_src && *pp_dst )\r
1807         {\r
1808                 IPOIB_PRINT(TRACE_LEVEL_INFORMATION, IPOIB_DBG_RECV,\r
1809                         ("Recv:\n"\r
1810                         "\tsrc MAC: %02X-%02X-%02X-%02X-%02X-%02X\n"\r
1811                         "\tdst MAC: %02X-%02X-%02X-%02X-%02X-%02X\n",\r
1812                         (*pp_src )->mac.addr[0], (*pp_src )->mac.addr[1],\r
1813                         (*pp_src )->mac.addr[2], (*pp_src )->mac.addr[3],\r
1814                         (*pp_src )->mac.addr[4], (*pp_src )->mac.addr[5],\r
1815                         (*pp_dst )->mac.addr[0], (*pp_dst )->mac.addr[1],\r
1816                         (*pp_dst )->mac.addr[2], (*pp_dst )->mac.addr[3],\r
1817                         (*pp_dst )->mac.addr[4], (*pp_dst )->mac.addr[5]) );\r
1818         }\r
1819 \r
1820         IPOIB_EXIT( IPOIB_DBG_RECV );\r
1821 }\r
1822 \r
1823 \r
1824 static int32_t\r
1825 __recv_mgr_filter(\r
1826         IN                              ipoib_port_t* const                     p_port,\r
1827         IN                              ib_wc_t* const                          p_done_wc_list,\r
1828                 OUT                     cl_qlist_t* const                       p_done_list,\r
1829                 OUT                     cl_qlist_t* const                       p_bad_list )\r
1830 {\r
1831         ipoib_recv_desc_t               *p_desc;\r
1832         ib_wc_t                                 *p_wc;\r
1833         ipoib_pkt_t                             *p_ipoib;\r
1834         eth_pkt_t                               *p_eth;\r
1835         ipoib_endpt_t                   *p_src, *p_dst;\r
1836         ib_api_status_t                 status;\r
1837         uint32_t                                len;\r
1838         int32_t                                 recv_cnt = 0;\r
1839         PERF_DECLARE( GetRecvEndpts );\r
1840         PERF_DECLARE( RecvGen );\r
1841         PERF_DECLARE( RecvTcp );\r
1842         PERF_DECLARE( RecvUdp );\r
1843         PERF_DECLARE( RecvDhcp );\r
1844         PERF_DECLARE( RecvArp );\r
1845 \r
1846         IPOIB_ENTER( IPOIB_DBG_RECV );\r
1847 \r
1848         for( p_wc = p_done_wc_list; p_wc; p_wc = p_wc->p_next )\r
1849         {\r
1850                 CL_ASSERT( p_wc->wc_type == IB_WC_RECV );\r
1851                 p_desc = (ipoib_recv_desc_t*)(uintn_t)p_wc->wr_id;\r
1852                 recv_cnt++;\r
1853 \r
1854                 if( p_wc->status != IB_WCS_SUCCESS )\r
1855                 {\r
1856                         if( p_wc->status != IB_WCS_WR_FLUSHED_ERR )\r
1857                         {\r
1858                                 IPOIB_PRINT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1859                                         ("Failed completion %s  (vendor specific %#x)\n",\r
1860                                         p_port->p_adapter->p_ifc->get_wc_status_str( p_wc->status ),\r
1861                                         (int)p_wc->vendor_specific) );\r
1862                                 ipoib_inc_recv_stat( p_port->p_adapter, IP_STAT_ERROR, 0 );\r
1863                         }\r
1864                         else\r
1865                         {\r
1866                                 IPOIB_PRINT(TRACE_LEVEL_INFORMATION, IPOIB_DBG_RECV,\r
1867                                         ("Flushed completion %s\n",\r
1868                                         p_port->p_adapter->p_ifc->get_wc_status_str( p_wc->status )) );\r
1869                                 ipoib_inc_recv_stat( p_port->p_adapter, IP_STAT_DROPPED, 0 );\r
1870                         }\r
1871                         cl_qlist_insert_tail( p_bad_list, &p_desc->item.list_item );\r
1872                         /* Dereference the port object on behalf of the failed receive. */\r
1873                         ipoib_port_deref( p_port, ref_failed_recv_wc );\r
1874                         continue;\r
1875                 }\r
1876 \r
1877                 len = p_wc->length - sizeof(ib_grh_t);\r
1878 \r
1879                 if( len < sizeof(ipoib_hdr_t) )\r
1880                 {\r
1881                         IPOIB_PRINT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1882                                 ("Received ETH packet < min size\n") );\r
1883                         ipoib_inc_recv_stat( p_port->p_adapter, IP_STAT_ERROR, 0 );\r
1884                         cl_qlist_insert_tail( p_bad_list, &p_desc->item.list_item );\r
1885                         ipoib_port_deref( p_port, ref_recv_inv_len );\r
1886                         continue;\r
1887                 }\r
1888 \r
1889                 if((len - sizeof(ipoib_hdr_t)) > p_port->p_adapter->params.payload_mtu)\r
1890                 {\r
1891                         IPOIB_PRINT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1892                                 ("Received ETH packet > payload MTU (%d)\n",\r
1893                                 p_port->p_adapter->params.payload_mtu) );\r
1894                         ipoib_inc_recv_stat( p_port->p_adapter, IP_STAT_ERROR, 0 );\r
1895                         cl_qlist_insert_tail( p_bad_list, &p_desc->item.list_item );\r
1896                         ipoib_port_deref( p_port, ref_recv_inv_len );\r
1897                         continue;\r
1898                         \r
1899                 }\r
1900                 /* Successful completion.  Get the receive information. */\r
1901                 cl_perf_start( GetRecvEndpts );\r
1902                 __recv_get_endpts( p_port, p_desc, p_wc, &p_src, &p_dst );\r
1903                 cl_perf_stop( &p_port->p_adapter->perf, GetRecvEndpts );\r
1904 \r
1905 #if IPOIB_INLINE_RECV\r
1906                 p_ipoib = &p_desc->buf.ib.pkt;\r
1907                 p_eth = &p_desc->buf.eth.pkt;\r
1908 #else   /* IPOIB_INLINE_RECV */\r
1909                 p_ipoib = &p_desc->p_buf->ib.pkt;\r
1910                 p_eth = &p_desc->p_buf->eth.pkt;\r
1911 #endif  /*IPOIB_INLINE_RECV */\r
1912 \r
1913                 if( p_src )\r
1914                 {\r
1915                         /* Don't report loopback traffic - we requested SW loopback. */\r
1916                         if( !cl_memcmp( &p_port->p_adapter->params.conf_mac,\r
1917                                 &p_src->mac, sizeof(p_port->p_adapter->params.conf_mac) ) )\r
1918                         {\r
1919                                 /*\r
1920                                  * "This is not the packet you're looking for" - don't update\r
1921                                  * receive statistics, the packet never happened.\r
1922                                  */\r
1923                                 cl_qlist_insert_tail( p_bad_list, &p_desc->item.list_item );\r
1924                                 /* Dereference the port object on behalf of the failed recv. */\r
1925                                 ipoib_port_deref( p_port, ref_recv_loopback );\r
1926                                 continue;\r
1927                         }\r
1928                 }\r
1929 \r
1930                 switch( p_ipoib->hdr.type )\r
1931                 {\r
1932                 case ETH_PROT_TYPE_IP:\r
1933                         if( len < (sizeof(ipoib_hdr_t) + sizeof(ip_hdr_t)) )\r
1934                         {\r
1935                                 IPOIB_PRINT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1936                                         ("Received IP packet < min size\n") );\r
1937                                 status = IB_INVALID_SETTING;\r
1938                                 break;\r
1939                         }\r
1940 \r
1941                         if( p_ipoib->type.ip.hdr.offset ||\r
1942                                 p_ipoib->type.ip.hdr.prot != IP_PROT_UDP )\r
1943                         {\r
1944                                 /* Unfiltered.  Setup the ethernet header and report. */\r
1945                                 cl_perf_start( RecvTcp );\r
1946                                 status = __recv_gen( p_ipoib, p_eth, p_src, p_dst );\r
1947                                 cl_perf_stop( &p_port->p_adapter->perf, RecvTcp );\r
1948                                 break;\r
1949                         }\r
1950 \r
1951                         /* First packet of a UDP transfer. */\r
1952                         if( len <\r
1953                                 (sizeof(ipoib_hdr_t) + sizeof(ip_hdr_t) + sizeof(udp_hdr_t)) )\r
1954                         {\r
1955                                 IPOIB_PRINT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1956                                         ("Received UDP packet < min size\n") );\r
1957                                 status = IB_INVALID_SETTING;\r
1958                                 break;\r
1959                         }\r
1960 \r
1961                         /* Check if DHCP conversion is required. */\r
1962                         if( (p_ipoib->type.ip.prot.udp.hdr.dst_port == DHCP_PORT_SERVER &&\r
1963                                 p_ipoib->type.ip.prot.udp.hdr.src_port == DHCP_PORT_CLIENT) ||\r
1964                                 (p_ipoib->type.ip.prot.udp.hdr.dst_port == DHCP_PORT_CLIENT &&\r
1965                                 p_ipoib->type.ip.prot.udp.hdr.src_port == DHCP_PORT_SERVER) )\r
1966                         {\r
1967                                 if( len < (sizeof(ipoib_hdr_t) + sizeof(ip_hdr_t) +\r
1968                                         sizeof(udp_hdr_t) + DHCP_MIN_SIZE) )\r
1969                                 {\r
1970                                         IPOIB_PRINT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1971                                                 ("Received DHCP < min size\n") );\r
1972                                         status = IB_INVALID_SETTING;\r
1973                                         break;\r
1974                                 }\r
1975                                 /* UDP packet with BOOTP ports in src/dst port numbers. */\r
1976                                 cl_perf_start( RecvDhcp );\r
1977                                 status = __recv_dhcp( p_port, p_ipoib, p_eth, p_src, p_dst );\r
1978                                 cl_perf_stop( &p_port->p_adapter->perf, RecvDhcp );\r
1979                         }\r
1980                         else\r
1981                         {\r
1982                                 /* Unfiltered.  Setup the ethernet header and report. */\r
1983                                 cl_perf_start( RecvUdp );\r
1984                                 status = __recv_gen( p_ipoib, p_eth, p_src, p_dst );\r
1985                                 cl_perf_stop( &p_port->p_adapter->perf, RecvUdp );\r
1986                         }\r
1987                         break;\r
1988 \r
1989                 case ETH_PROT_TYPE_ARP:\r
1990                         if( len < (sizeof(ipoib_hdr_t) + sizeof(ipoib_arp_pkt_t)) )\r
1991                         {\r
1992                                 IPOIB_PRINT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1993                                         ("Received ARP < min size\n") );\r
1994                                 status = IB_INVALID_SETTING;\r
1995                                 break;\r
1996                         }\r
1997                         cl_perf_start( RecvArp );\r
1998                         status = __recv_arp( p_port, p_wc, p_ipoib, p_eth, &p_src, p_dst );\r
1999                         cl_perf_stop( &p_port->p_adapter->perf, RecvArp );\r
2000                         len = sizeof(ipoib_hdr_t) + sizeof(arp_pkt_t);\r
2001                         break;\r
2002 \r
2003                 default:\r
2004                         /* Unfiltered.  Setup the ethernet header and report. */\r
2005                         cl_perf_start( RecvGen );\r
2006                         status = __recv_gen( p_ipoib, p_eth, p_src, p_dst );\r
2007                         cl_perf_stop( &p_port->p_adapter->perf, RecvGen );\r
2008                 }\r
2009 \r
2010                 if( status != IB_SUCCESS )\r
2011                 {\r
2012                         /* Update stats. */\r
2013                         ipoib_inc_recv_stat( p_port->p_adapter, IP_STAT_ERROR, 0 );\r
2014                         cl_qlist_insert_tail( p_bad_list, &p_desc->item.list_item );\r
2015                         /* Dereference the port object on behalf of the failed receive. */\r
2016                         ipoib_port_deref( p_port, ref_recv_filter );\r
2017                 }\r
2018                 else\r
2019                 {\r
2020                         p_desc->len =\r
2021                                 len + sizeof(eth_hdr_t) - sizeof(ipoib_hdr_t);\r
2022                         if( p_dst->h_mcast)\r
2023                         {\r
2024                                 if( p_dst->dgid.multicast.raw_group_id[10] == 0xFF &&\r
2025                                         p_dst->dgid.multicast.raw_group_id[11] == 0xFF &&\r
2026                                         p_dst->dgid.multicast.raw_group_id[12] == 0xFF &&\r
2027                                         p_dst->dgid.multicast.raw_group_id[13] == 0xFF )\r
2028                                 {\r
2029                                         p_desc->type = PKT_TYPE_BCAST;\r
2030                                 }\r
2031                                 else\r
2032                                 {\r
2033                                         p_desc->type = PKT_TYPE_MCAST;\r
2034                                 }\r
2035                         }\r
2036                         else\r
2037                         {\r
2038                                 p_desc->type = PKT_TYPE_UCAST;\r
2039                         }\r
2040                         cl_qlist_insert_tail( p_done_list, &p_desc->item.list_item );\r
2041                 }\r
2042         }\r
2043 \r
2044         IPOIB_EXIT( IPOIB_DBG_RECV );\r
2045         return recv_cnt;\r
2046 }\r
2047 \r
2048 \r
2049 static ib_api_status_t\r
2050 __recv_gen(\r
2051         IN              const   ipoib_pkt_t* const                      p_ipoib,\r
2052                 OUT                     eth_pkt_t* const                        p_eth,\r
2053         IN                              ipoib_endpt_t* const            p_src,\r
2054         IN                              ipoib_endpt_t* const            p_dst )\r
2055 {\r
2056         IPOIB_ENTER( IPOIB_DBG_RECV );\r
2057 \r
2058         if( !p_src || !p_dst )\r
2059         {\r
2060                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2061                         ("Received packet with no matching endpoints.\n") );\r
2062                 return IB_NOT_DONE;\r
2063         }\r
2064 \r
2065         /*\r
2066          * Fill in the ethernet header.  Note that doing so will overwrite\r
2067          * the IPoIB header, so start by moving the information from the IPoIB\r
2068          * header.\r
2069          */\r
2070         p_eth->hdr.type = p_ipoib->hdr.type;\r
2071         p_eth->hdr.src = p_src->mac;\r
2072         p_eth->hdr.dst = p_dst->mac;\r
2073 \r
2074         IPOIB_EXIT( IPOIB_DBG_RECV );\r
2075         return IB_SUCCESS;\r
2076 }\r
2077 \r
2078 \r
2079 static ib_api_status_t\r
2080 __recv_dhcp(\r
2081         IN                              ipoib_port_t* const                     p_port,\r
2082         IN              const   ipoib_pkt_t* const                      p_ipoib,\r
2083                 OUT                     eth_pkt_t* const                        p_eth,\r
2084         IN                              ipoib_endpt_t* const            p_src,\r
2085         IN                              ipoib_endpt_t* const            p_dst )\r
2086 {\r
2087         ib_api_status_t         status;\r
2088         dhcp_pkt_t                      *p_dhcp;\r
2089         uint8_t                         *p_option;\r
2090         uint8_t                         *p_cid = NULL;\r
2091         ib_gid_t                        gid;\r
2092         uint8_t                         msg = 0;\r
2093 \r
2094         IPOIB_ENTER( IPOIB_DBG_RECV );\r
2095 \r
2096         UNUSED_PARAM( p_port );\r
2097 \r
2098         /* Create the ethernet header. */\r
2099         status = __recv_gen( p_ipoib, p_eth, p_src, p_dst );\r
2100         if( status != IB_SUCCESS )\r
2101         {\r
2102                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2103                         ("__recv_gen returned %s.\n", \r
2104                         p_port->p_adapter->p_ifc->get_err_str( status )) );\r
2105                 return status;\r
2106         }\r
2107 \r
2108         /* Fixup the payload. */\r
2109         p_dhcp = &p_eth->type.ip.prot.udp.dhcp;\r
2110         if( p_dhcp->op != DHCP_REQUEST && p_dhcp->op != DHCP_REPLY )\r
2111         {\r
2112                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2113                         ("Invalid DHCP op code.\n") );\r
2114                 return IB_INVALID_SETTING;\r
2115         }\r
2116 \r
2117         /*\r
2118          * Find the client identifier option, making sure to skip\r
2119          * the "magic cookie".\r
2120          */\r
2121         p_option = &p_dhcp->options[0];\r
2122         if ( *(uint32_t *)p_option != DHCP_COOKIE )\r
2123         {\r
2124                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2125                         ("DHCP cookie corrupted.\n") );\r
2126                 return IB_INVALID_PARAMETER;\r
2127         }\r
2128 \r
2129         p_option = &p_dhcp->options[4];\r
2130         while( *p_option != DHCP_OPT_END )\r
2131         {\r
2132                 switch( *p_option )\r
2133                 {\r
2134                 case DHCP_OPT_PAD:\r
2135                         p_option++;\r
2136                         break;\r
2137 \r
2138                 case DHCP_OPT_MSG:\r
2139                         msg = p_option[2];\r
2140                         p_option += 3;\r
2141                         break;\r
2142 \r
2143                 case DHCP_OPT_CLIENT_ID:\r
2144                         p_cid = p_option;\r
2145                         /* Fall through. */\r
2146 \r
2147                 default:\r
2148                         /*\r
2149                          * All other options have a length byte following the option code.\r
2150                          * Offset by the length to get to the next option.\r
2151                          */\r
2152                         p_option += (p_option[1] + 2);\r
2153                 }\r
2154         }\r
2155 \r
2156         switch( msg )\r
2157         {\r
2158         /* message from client */\r
2159         case DHCPDISCOVER:\r
2160         case DHCPREQUEST:\r
2161         case DHCPDECLINE:\r
2162         case DHCPRELEASE:\r
2163         case DHCPINFORM:\r
2164                 if( !p_cid )\r
2165                 {\r
2166                         IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2167                                 ("Failed to find required Client-identifier option.\n") );\r
2168                         return IB_INVALID_SETTING;\r
2169                 }\r
2170                 if( p_dhcp->htype != DHCP_HW_TYPE_IB )\r
2171                 {\r
2172                         IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2173                                 ("Invalid hardware address type.\n") );\r
2174                         return IB_INVALID_SETTING;\r
2175                 }\r
2176                 break;\r
2177         /* message from DHCP server */\r
2178         case DHCPOFFER:\r
2179         case DHCPACK:\r
2180         case DHCPNAK:\r
2181                 break;\r
2182 \r
2183         default:\r
2184                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2185                         ("Invalide message type.\n") );\r
2186                 return IB_INVALID_PARAMETER;\r
2187         }\r
2188         p_eth->type.ip.prot.udp.hdr.chksum = 0;\r
2189         p_dhcp->htype = DHCP_HW_TYPE_ETH;\r
2190         p_dhcp->hlen = HW_ADDR_LEN;\r
2191 \r
2192         if( p_cid ) /* from client */\r
2193         {\r
2194                 /* Validate that the length and type of the option is as required. */\r
2195                 if( p_cid[1] != 21 )\r
2196                 {\r
2197                         IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2198                                 ("Client-identifier length not 21 as required.\n") );\r
2199                         return IB_INVALID_SETTING;\r
2200                 }\r
2201                 if( p_cid[2] != DHCP_HW_TYPE_IB )\r
2202                 {\r
2203                         IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2204                                 ("Client-identifier type is wrong.\n") );\r
2205                         return IB_INVALID_SETTING;\r
2206                 }\r
2207                 /*\r
2208                  * Copy the GID value from the option so that we can make aligned\r
2209                  * accesses to the contents.\r
2210                  * Recover CID to standard type.\r
2211                  */\r
2212                 cl_memcpy( &gid, &p_cid[7], sizeof(ib_gid_t) );\r
2213                 p_cid[1] =  HW_ADDR_LEN +1;// CID length \r
2214                 p_cid[2] =  DHCP_HW_TYPE_ETH;// CID type \r
2215                 status = ipoib_mac_from_guid( gid.unicast.interface_id, (mac_addr_t*)&p_cid[3] );\r
2216                 p_cid[HW_ADDR_LEN + 3] = DHCP_OPT_END; //terminate tag\r
2217         }\r
2218         IPOIB_EXIT( IPOIB_DBG_RECV );\r
2219         return status;\r
2220 }\r
2221 \r
2222 \r
2223 static ib_api_status_t\r
2224 __recv_arp(\r
2225         IN                              ipoib_port_t* const                     p_port,\r
2226         IN                              ib_wc_t* const                          p_wc,\r
2227         IN              const   ipoib_pkt_t* const                      p_ipoib,\r
2228                 OUT                     eth_pkt_t* const                        p_eth,\r
2229         IN                              ipoib_endpt_t** const           pp_src,\r
2230         IN                              ipoib_endpt_t* const            p_dst )\r
2231 {\r
2232         ib_api_status_t                 status;\r
2233         arp_pkt_t                               *p_arp;\r
2234         const ipoib_arp_pkt_t   *p_ib_arp;\r
2235         ib_gid_t                                gid;\r
2236         mac_addr_t                              mac;\r
2237         ipoib_hw_addr_t                 null_hw = {0};\r
2238 \r
2239         IPOIB_ENTER( IPOIB_DBG_RECV );\r
2240 \r
2241         if( !p_dst )\r
2242         {\r
2243                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2244                         ("Unknown destination endpoint\n") );\r
2245                 return IB_INVALID_SETTING;\r
2246         }\r
2247 \r
2248         p_ib_arp = &p_ipoib->type.arp;\r
2249         p_arp = &p_eth->type.arp;\r
2250 \r
2251         if( p_ib_arp->hw_type != ARP_HW_TYPE_IB )\r
2252         {\r
2253                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2254                         ("ARP hardware type is not IB\n") );\r
2255                 return IB_INVALID_SETTING;\r
2256         }\r
2257 \r
2258         if( p_ib_arp->hw_size != sizeof(ipoib_hw_addr_t) )\r
2259         {\r
2260                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2261                         ("ARP hardware address size is not sizeof(ipoib_hw_addr_t)\n") );\r
2262                 return IB_INVALID_SETTING;\r
2263         }\r
2264 \r
2265         if( p_ib_arp->prot_type != ETH_PROT_TYPE_IP )\r
2266         {\r
2267                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2268                         ("ARP protocal type not IP\n") );\r
2269                 return IB_INVALID_SETTING;\r
2270         }\r
2271 \r
2272         /*\r
2273          * If we don't have a source, lookup the endpoint specified in the payload.\r
2274          */\r
2275         if( !*pp_src )\r
2276                 *pp_src = __endpt_mgr_get_by_gid( p_port, &p_ib_arp->src_hw.gid );\r
2277 \r
2278         /*\r
2279          * If the endpoint exists for the GID, make sure\r
2280          * the dlid and qpn match the arp.\r
2281          */\r
2282         if( *pp_src )\r
2283         {\r
2284                 if( cl_memcmp( &(*pp_src)->dgid, &p_ib_arp->src_hw.gid,\r
2285                         sizeof(ib_gid_t) ) )\r
2286                 {\r
2287                         /*\r
2288                          * GIDs for the endpoint are different.  The ARP must\r
2289                          * have been proxied.  Dereference it.\r
2290                          */\r
2291                         *pp_src = NULL;\r
2292                 }\r
2293                 else if( (*pp_src)->dlid &&\r
2294                         (*pp_src)->dlid != p_wc->recv.ud.remote_lid )\r
2295                 {\r
2296                         /* Out of date!  Destroy the endpoint and replace it. */\r
2297                         __endpt_mgr_remove( p_port, *pp_src );\r
2298                         *pp_src = NULL;\r
2299                 }\r
2300                 else if( ipoib_is_voltaire_router_gid( &(*pp_src)->dgid ) )\r
2301                 {\r
2302                         if( (*pp_src)->qpn !=\r
2303                                 (p_ib_arp->src_hw.flags_qpn & CL_HTON32(0x00FFFFFF)) &&\r
2304                                 p_wc->recv.ud.remote_qp !=\r
2305                                 (p_ib_arp->src_hw.flags_qpn & CL_HTON32(0x00FFFFFF)) )\r
2306                         {\r
2307                                 /* Out of date!  Destroy the endpoint and replace it. */\r
2308                                 __endpt_mgr_remove( p_port, *pp_src );\r
2309                                 *pp_src = NULL;\r
2310                         }\r
2311                 }\r
2312                 else if( (*pp_src)->qpn != p_wc->recv.ud.remote_qp )\r
2313                 {\r
2314                         /* Out of date!  Destroy the endpoint and replace it. */\r
2315                         __endpt_mgr_remove( p_port, *pp_src );\r
2316                         *pp_src = NULL;\r
2317                 }\r
2318         }\r
2319 \r
2320         /* Do we need to create an endpoint for this GID? */\r
2321         if( !*pp_src )\r
2322         {\r
2323                 /* Copy the src GID to allow aligned access */\r
2324                 cl_memcpy( &gid, &p_ib_arp->src_hw.gid, sizeof(ib_gid_t) );\r
2325                 status = ipoib_mac_from_guid( gid.unicast.interface_id, &mac );\r
2326                 if( status != IB_SUCCESS )\r
2327                 {\r
2328                         IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2329                                 ("ipoib_mac_from_guid returned %s\n",\r
2330                                 p_port->p_adapter->p_ifc->get_err_str( status )) );\r
2331                         return status;\r
2332                 }\r
2333                 /*\r
2334                  * Create the endpoint.  Note that the LID is left blank and will be\r
2335                  * resolved by a path query as needed.  This is done because the\r
2336                  * remote LID/GID from the work completion may not be the original\r
2337                  * initiator.\r
2338                  */\r
2339                 *pp_src = ipoib_endpt_create( &p_ib_arp->src_hw.gid,\r
2340                         0, (p_ib_arp->src_hw.flags_qpn & CL_HTON32(0x00FFFFFF)) );\r
2341 \r
2342                 if( !*pp_src )\r
2343                 {\r
2344                         IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2345                                 ("ipoib_endpt_create failed\n") );\r
2346                         return status;\r
2347                 }\r
2348 \r
2349                 cl_obj_lock( &p_port->obj );\r
2350                 status = __endpt_mgr_insert( p_port, mac, *pp_src );\r
2351                 if( status != IB_SUCCESS )\r
2352                 {\r
2353                         IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2354                                 ("__endpt_mgr_insert return %s \n",\r
2355                                 p_port->p_adapter->p_ifc->get_err_str( status )) );\r
2356                         return status;\r
2357                 }\r
2358 \r
2359                 cl_obj_unlock( &p_port->obj );\r
2360         }\r
2361 \r
2362         CL_ASSERT( !cl_memcmp(\r
2363                 &(*pp_src)->dgid, &p_ib_arp->src_hw.gid, sizeof(ib_gid_t) ) );\r
2364         CL_ASSERT( ipoib_is_voltaire_router_gid( &(*pp_src)->dgid ) ||\r
2365                 (*pp_src)->qpn ==\r
2366                 (p_ib_arp->src_hw.flags_qpn & CL_HTON32(0x00FFFFFF)) );\r
2367         /* Now swizzle the data. */\r
2368         p_arp->hw_type = ARP_HW_TYPE_ETH;\r
2369         p_arp->hw_size = sizeof(mac_addr_t);\r
2370         p_arp->src_hw = (*pp_src)->mac;\r
2371         p_arp->src_ip = p_ib_arp->src_ip;\r
2372 \r
2373         if( cl_memcmp( &p_ib_arp->dst_hw, &null_hw, sizeof(ipoib_hw_addr_t) ) )\r
2374         {\r
2375                 if( cl_memcmp( &p_dst->dgid, &p_ib_arp->dst_hw.gid, sizeof(ib_gid_t) ) )\r
2376                 {\r
2377                         /*\r
2378                          * We received bcast ARP packet that means\r
2379                          * remote port lets everyone know it was changed IP/MAC\r
2380                          * or just activated\r
2381                          */\r
2382 \r
2383                         /* Guy: TODO: Check why this check fails in case of Voltaire IPR */\r
2384 \r
2385                         if ( !ipoib_is_voltaire_router_gid( &(*pp_src)->dgid ) &&\r
2386                                  !ib_gid_is_multicast( (const ib_gid_t*)&p_dst->dgid ) )\r
2387                         {\r
2388                                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2389                                         ("ARP: is not ARP MCAST\n") );\r
2390                                 return IB_INVALID_SETTING;\r
2391                         }\r
2392 \r
2393                         p_arp->dst_hw = p_port->p_local_endpt->mac;\r
2394                         p_dst->mac = p_port->p_local_endpt->mac;\r
2395                         /*\r
2396                          * we don't care what receiver ip addr is,\r
2397                          * as long as OS' ARP table is global  ???\r
2398                          */\r
2399                         p_arp->dst_ip = (net32_t)0;\r
2400                 }\r
2401                 else /* we've got reply to our ARP request */\r
2402                 {\r
2403                         p_arp->dst_hw = p_dst->mac;\r
2404                         p_arp->dst_ip = p_ib_arp->dst_ip;\r
2405                         CL_ASSERT( p_dst->qpn == \r
2406                                 (p_ib_arp->dst_hw.flags_qpn & CL_HTON32(0x00FFFFFF)) );\r
2407                 }\r
2408         }\r
2409         else /* we got ARP reqeust */\r
2410         {\r
2411                 cl_memclr( &p_arp->dst_hw, sizeof(mac_addr_t) );\r
2412                 p_arp->dst_ip = p_ib_arp->dst_ip;\r
2413         }\r
2414 \r
2415         /*\r
2416          * Create the ethernet header.  Note that this is done last so that\r
2417          * we have a chance to create a new endpoint.\r
2418          */\r
2419         status = __recv_gen( p_ipoib, p_eth, *pp_src, p_dst );\r
2420         if( status != IB_SUCCESS )\r
2421         {\r
2422                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2423                         ("__recv_gen returned %s.\n", \r
2424                         p_port->p_adapter->p_ifc->get_err_str( status )) );\r
2425                 return status;\r
2426         }\r
2427 \r
2428         IPOIB_EXIT( IPOIB_DBG_RECV );\r
2429         return IB_SUCCESS;\r
2430 }\r
2431 \r
2432 \r
2433 static ib_api_status_t\r
2434 __recv_mgr_prepare_pkt(\r
2435         IN                              ipoib_port_t* const                     p_port,\r
2436         IN                              ipoib_recv_desc_t* const        p_desc,\r
2437                 OUT                     NDIS_PACKET** const                     pp_packet )\r
2438 {\r
2439         NDIS_STATUS                                                     status;\r
2440         uint32_t                                                        pkt_filter;\r
2441         ip_stat_sel_t                                           type;\r
2442         NDIS_TCP_IP_CHECKSUM_PACKET_INFO        chksum;\r
2443         PERF_DECLARE( GetNdisPkt );\r
2444 \r
2445         IPOIB_ENTER( IPOIB_DBG_RECV );\r
2446 \r
2447         pkt_filter = p_port->p_adapter->packet_filter;\r
2448         /* Check the packet filter. */\r
2449         switch( p_desc->type )\r
2450         {\r
2451         default:\r
2452         case PKT_TYPE_UCAST:\r
2453                 if( pkt_filter & NDIS_PACKET_TYPE_PROMISCUOUS ||\r
2454                         pkt_filter & NDIS_PACKET_TYPE_ALL_FUNCTIONAL ||\r
2455                         pkt_filter & NDIS_PACKET_TYPE_SOURCE_ROUTING ||\r
2456                         pkt_filter & NDIS_PACKET_TYPE_DIRECTED )\r
2457                 {\r
2458                         /* OK to report. */\r
2459                         type = IP_STAT_UCAST_BYTES;\r
2460                         status = NDIS_STATUS_SUCCESS;\r
2461                 }\r
2462                 else\r
2463                 {\r
2464                         type = IP_STAT_DROPPED;\r
2465                         status = NDIS_STATUS_FAILURE;\r
2466                 }\r
2467                 break;\r
2468         case PKT_TYPE_BCAST:\r
2469                 if( pkt_filter & NDIS_PACKET_TYPE_PROMISCUOUS ||\r
2470                         pkt_filter & NDIS_PACKET_TYPE_BROADCAST )\r
2471                 {\r
2472                         /* OK to report. */\r
2473                         type = IP_STAT_BCAST_BYTES;\r
2474                         status = NDIS_STATUS_SUCCESS;\r
2475                 }\r
2476                 else\r
2477                 {\r
2478                         type = IP_STAT_DROPPED;\r
2479                         status = NDIS_STATUS_FAILURE;\r
2480                 }\r
2481                 break;\r
2482         case PKT_TYPE_MCAST:\r
2483                 if( pkt_filter & NDIS_PACKET_TYPE_PROMISCUOUS ||\r
2484                         pkt_filter & NDIS_PACKET_TYPE_ALL_MULTICAST ||\r
2485                         pkt_filter & NDIS_PACKET_TYPE_MULTICAST )\r
2486                 {\r
2487                         /* OK to report. */\r
2488                         type = IP_STAT_MCAST_BYTES;\r
2489                         status = NDIS_STATUS_SUCCESS;\r
2490                 }\r
2491                 else\r
2492                 {\r
2493                         type = IP_STAT_DROPPED;\r
2494                         status = NDIS_STATUS_FAILURE;\r
2495                 }\r
2496                 break;\r
2497         }\r
2498 \r
2499         if( status != NDIS_STATUS_SUCCESS )\r
2500         {\r
2501                 ipoib_inc_recv_stat( p_port->p_adapter, type, 0 );\r
2502                 /* Return the receive descriptor to the pool. */\r
2503                 __buf_mgr_put_recv( p_port, p_desc, NULL );\r
2504                 IPOIB_PRINT_EXIT( TRACE_LEVEL_INFORMATION, IPOIB_DBG_RECV,\r
2505                         ("Packet filter doesn't match receive.  Dropping.\n") );\r
2506                 /*\r
2507                  * Return IB_NOT_DONE since the packet has been completed,\r
2508                  * but has not consumed an array entry.\r
2509                  */\r
2510                 return IB_NOT_DONE;\r
2511         }\r
2512 \r
2513         cl_perf_start( GetNdisPkt );\r
2514         *pp_packet = __buf_mgr_get_ndis_pkt( p_port, p_desc );\r
2515         cl_perf_stop( &p_port->p_adapter->perf, GetNdisPkt );\r
2516         if( !*pp_packet )\r
2517         {\r
2518                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2519                         ("__buf_mgr_get_ndis_pkt failed\n") );\r
2520                 return IB_INSUFFICIENT_RESOURCES;\r
2521         }\r
2522 \r
2523         /* Flag the checksums as having been calculated. */\r
2524         chksum.Value = 0;\r
2525         chksum.Receive.NdisPacketTcpChecksumSucceeded = TRUE;\r
2526         chksum.Receive.NdisPacketUdpChecksumSucceeded = TRUE;\r
2527         chksum.Receive.NdisPacketIpChecksumSucceeded = TRUE;\r
2528         NDIS_PER_PACKET_INFO_FROM_PACKET( *pp_packet, TcpIpChecksumPacketInfo ) =\r
2529                 (void*)(uintn_t)chksum.Value;\r
2530 \r
2531         ipoib_inc_recv_stat( p_port->p_adapter, type, p_desc->len );\r
2532 \r
2533         IPOIB_EXIT( IPOIB_DBG_RECV );\r
2534         return IB_SUCCESS;\r
2535 }\r
2536 \r
2537 \r
2538 static uint32_t\r
2539 __recv_mgr_build_pkt_array(\r
2540         IN                              ipoib_port_t* const                     p_port,\r
2541         IN                              int32_t                                         shortage,\r
2542                 OUT                     cl_qlist_t* const                       p_done_list,\r
2543                 OUT                     int32_t* const                          p_discarded )\r
2544 {\r
2545         cl_list_item_t                  *p_item;\r
2546         ipoib_recv_desc_t               *p_desc;\r
2547         uint32_t                                i = 0;\r
2548         ib_api_status_t                 status;\r
2549         PERF_DECLARE( PreparePkt );\r
2550 \r
2551         IPOIB_ENTER( IPOIB_DBG_RECV );\r
2552 \r
2553         *p_discarded = 0;\r
2554 \r
2555         /* Move any existing receives to the head to preserve ordering. */\r
2556         cl_qlist_insert_list_head( p_done_list, &p_port->recv_mgr.done_list );\r
2557         p_item = cl_qlist_remove_head( p_done_list );\r
2558         while( p_item != cl_qlist_end( p_done_list ) )\r
2559         {\r
2560                 p_desc = (ipoib_recv_desc_t*)p_item;\r
2561 \r
2562                 cl_perf_start( PreparePkt );\r
2563                 status = __recv_mgr_prepare_pkt( p_port, p_desc,\r
2564                         &p_port->recv_mgr.recv_pkt_array[i] );\r
2565                 cl_perf_stop( &p_port->p_adapter->perf, PreparePkt );\r
2566                 if( status == IB_SUCCESS )\r
2567                 {\r
2568                         CL_ASSERT( p_port->recv_mgr.recv_pkt_array[i] );\r
2569                         if( shortage-- > 0 )\r
2570                         {\r
2571                                 NDIS_SET_PACKET_STATUS(\r
2572                                         p_port->recv_mgr.recv_pkt_array[i], NDIS_STATUS_RESOURCES );\r
2573                         }\r
2574                         else\r
2575                         {\r
2576                                 NDIS_SET_PACKET_STATUS(\r
2577                                         p_port->recv_mgr.recv_pkt_array[i], NDIS_STATUS_SUCCESS );\r
2578                         }\r
2579                         i++;\r
2580                 }\r
2581                 else if( status == IB_NOT_DONE )\r
2582                 {\r
2583                         (*p_discarded)++;\r
2584                 }\r
2585                 else\r
2586                 {\r
2587                         IPOIB_PRINT(TRACE_LEVEL_INFORMATION, IPOIB_DBG_RECV,\r
2588                                 ("__recv_mgr_prepare_pkt returned %s\n",\r
2589                                 p_port->p_adapter->p_ifc->get_err_str( status )) );\r
2590                         /* Put all completed receives on the port's done list. */\r
2591                         cl_qlist_insert_tail( &p_port->recv_mgr.done_list, p_item );\r
2592                         cl_qlist_insert_list_tail( &p_port->recv_mgr.done_list, p_done_list );\r
2593                         break;\r
2594                 }\r
2595 \r
2596                 p_item = cl_qlist_remove_head( p_done_list );\r
2597         }\r
2598 \r
2599         IPOIB_EXIT( IPOIB_DBG_RECV );\r
2600         return i;\r
2601 }\r
2602 \r
2603 \r
2604 \r
2605 \r
2606 /******************************************************************************\r
2607 *\r
2608 * Send manager implementation.\r
2609 *\r
2610 ******************************************************************************/\r
2611 static void\r
2612 __send_mgr_construct(\r
2613         IN                              ipoib_port_t* const                     p_port )\r
2614 {\r
2615         IPOIB_ENTER( IPOIB_DBG_SEND );\r
2616         p_port->send_mgr.depth = 0;\r
2617         cl_qlist_init( &p_port->send_mgr.pending_list );\r
2618         IPOIB_EXIT( IPOIB_DBG_SEND );\r
2619 }\r
2620 \r
2621 \r
2622 static void\r
2623 __send_mgr_destroy(\r
2624         IN                              ipoib_port_t* const                     p_port )\r
2625 {\r
2626         cl_list_item_t  *p_item;\r
2627         NDIS_PACKET             *p_packet;\r
2628 \r
2629         IPOIB_ENTER( IPOIB_DBG_SEND );\r
2630 \r
2631         /* Complete any pending packets. */\r
2632         for( p_item = cl_qlist_remove_head( &p_port->send_mgr.pending_list );\r
2633                 p_item != cl_qlist_end( &p_port->send_mgr.pending_list );\r
2634                 p_item = cl_qlist_remove_head( &p_port->send_mgr.pending_list ) )\r
2635         {\r
2636                 p_packet = IPOIB_PACKET_FROM_LIST_ITEM( p_item );\r
2637                 NdisMSendComplete( p_port->p_adapter->h_adapter, p_packet,\r
2638                         NDIS_STATUS_RESET_IN_PROGRESS );\r
2639         }\r
2640 \r
2641         IPOIB_EXIT( IPOIB_DBG_SEND );\r
2642 }\r
2643 \r
2644 \r
2645 static NDIS_STATUS\r
2646 __send_mgr_filter(\r
2647         IN                              ipoib_port_t* const                     p_port,\r
2648         IN              const   eth_hdr_t* const                        p_eth_hdr,\r
2649         IN                              NDIS_BUFFER* const                      p_buf,\r
2650         IN                              size_t                                          buf_len,\r
2651         IN      OUT                     ipoib_send_desc_t* const        p_desc )\r
2652 {\r
2653         NDIS_STATUS             status;\r
2654 \r
2655         PERF_DECLARE( FilterIp );\r
2656         PERF_DECLARE( FilterArp );\r
2657         PERF_DECLARE( SendGen );\r
2658 \r
2659         IPOIB_ENTER( IPOIB_DBG_SEND );\r
2660 \r
2661         /*\r
2662          * We already checked the ethernet header length, so we know it's safe\r
2663          * to decrement the buf_len without underflowing.\r
2664          */\r
2665         buf_len -= sizeof(eth_hdr_t);\r
2666 \r
2667         switch( p_eth_hdr->type )\r
2668         {\r
2669         case ETH_PROT_TYPE_IP:\r
2670                 cl_perf_start( FilterIp );\r
2671                 status = __send_mgr_filter_ip(\r
2672                         p_port, p_eth_hdr, p_buf, buf_len, p_desc );\r
2673                 cl_perf_stop( &p_port->p_adapter->perf, FilterIp );\r
2674                 break;\r
2675 \r
2676         case ETH_PROT_TYPE_ARP:\r
2677                 cl_perf_start( FilterArp );\r
2678                 status = __send_mgr_filter_arp(\r
2679                         p_port, p_eth_hdr, p_buf, buf_len, p_desc );\r
2680                 cl_perf_stop( &p_port->p_adapter->perf, FilterArp );\r
2681                 break;\r
2682 \r
2683         default:\r
2684                 /*\r
2685                  * The IPoIB spec doesn't define how to send non IP or ARP packets.\r
2686                  * Just send the payload and hope for the best.\r
2687                  */\r
2688                 cl_perf_start( SendGen );\r
2689                 status = __send_gen( p_port, p_desc );\r
2690                 cl_perf_stop( &p_port->p_adapter->perf, SendGen );\r
2691                 break;\r
2692         }\r
2693 \r
2694         IPOIB_EXIT( IPOIB_DBG_SEND );\r
2695         return status;\r
2696 }\r
2697 \r
2698 \r
2699 static NDIS_STATUS\r
2700 __send_copy(\r
2701         IN                              ipoib_port_t* const                     p_port,\r
2702         IN                              ipoib_send_desc_t* const        p_desc )\r
2703 {\r
2704         NDIS_PACKET                             *p_packet;\r
2705         NDIS_BUFFER                             *p_buf;\r
2706         NDIS_STATUS                             status;\r
2707         UINT                                    tot_len, bytes_copied = 0;\r
2708 \r
2709         IPOIB_ENTER( IPOIB_DBG_SEND );\r
2710 \r
2711         p_desc->p_buf = \r
2712                 ExAllocateFromNPagedLookasideList( &p_port->buf_mgr.send_buf_list );\r
2713         if( !p_desc->p_buf )\r
2714         {\r
2715                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2716                         ("Failed to allocate buffer for packet copy.\n") );\r
2717                 return NDIS_STATUS_RESOURCES;\r
2718         }\r
2719 \r
2720         NdisAllocatePacket( &status, &p_packet, p_port->buf_mgr.h_send_pkt_pool );\r
2721         if( status != NDIS_STATUS_SUCCESS )\r
2722         {\r
2723                 IPOIB_PRINT_EXIT( TRACE_LEVEL_WARNING, IPOIB_DBG_SEND,\r
2724                         ("Failed to allocate NDIS_PACKET for copy.\n") );\r
2725                 return status;\r
2726         }\r
2727 \r
2728         NdisAllocateBuffer( &status, &p_buf, p_port->buf_mgr.h_send_buf_pool,\r
2729                 p_desc->p_buf, p_port->p_adapter->params.xfer_block_size );\r
2730         if( status != NDIS_STATUS_SUCCESS )\r
2731         {\r
2732                 NdisFreePacket( p_packet );\r
2733                 IPOIB_PRINT_EXIT( TRACE_LEVEL_WARNING, IPOIB_DBG_SEND,\r
2734                         ("Failed to allocate NDIS_BUFFER for copy.\n") );\r
2735                 return status;\r
2736         }\r
2737 \r
2738         NdisChainBufferAtFront( p_packet, p_buf );\r
2739 \r
2740         NdisQueryPacketLength( p_desc->p_pkt, &tot_len );\r
2741 \r
2742         /* Setup the work request. */\r
2743         p_desc->local_ds[1].vaddr = cl_get_physaddr(\r
2744                 ((uint8_t*)p_desc->p_buf) + sizeof(eth_hdr_t) );\r
2745         p_desc->local_ds[1].length = tot_len - sizeof(eth_hdr_t);\r
2746         p_desc->local_ds[1].lkey = p_port->ib_mgr.lkey;\r
2747         p_desc->wr.num_ds = 2;\r
2748 \r
2749         /* Copy the packet. */\r
2750         NdisCopyFromPacketToPacketSafe( p_packet, bytes_copied, tot_len,\r
2751                 p_desc->p_pkt, bytes_copied, &bytes_copied,\r
2752                 NormalPagePriority );\r
2753 \r
2754         /* Free our temp packet now that the data is copied. */\r
2755         NdisUnchainBufferAtFront( p_packet, &p_buf );\r
2756         NdisFreeBuffer( p_buf );\r
2757         NdisFreePacket( p_packet );\r
2758 \r
2759         if( bytes_copied != tot_len )\r
2760         {\r
2761                 /* Something went wrong.  Drop the packet. */\r
2762                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2763                         ("Failed to copy full packet: %d of %d bytes copied.\n",\r
2764                         bytes_copied, tot_len) );\r
2765                 return NDIS_STATUS_RESOURCES;\r
2766         }\r
2767 \r
2768         IPOIB_EXIT( IPOIB_DBG_SEND );\r
2769         return NDIS_STATUS_SUCCESS;\r
2770 }\r
2771 \r
2772 \r
2773 #if !IPOIB_USE_DMA\r
2774 /* Send using the MDL's page information rather than the SGL. */\r
2775 static ib_api_status_t\r
2776 __send_gen(\r
2777         IN                              ipoib_port_t* const                     p_port,\r
2778         IN                              ipoib_send_desc_t* const        p_desc )\r
2779 {\r
2780         uint32_t                                i, j = 1;\r
2781         ULONG                                   offset;\r
2782         MDL                                             *p_mdl;\r
2783         UINT                                    num_pages, tot_len;\r
2784         ULONG                                   buf_len;\r
2785         PPFN_NUMBER                             page_array;\r
2786         boolean_t                               hdr_done = FALSE;\r
2787         ib_api_status_t                 status;\r
2788 \r
2789         IPOIB_ENTER( IPOIB_DBG_SEND );\r
2790 \r
2791         NdisQueryPacket( p_desc->p_pkt, &num_pages, NULL, &p_mdl,\r
2792                 &tot_len );\r
2793 \r
2794         if( !p_mdl )\r
2795         {\r
2796                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2797                         ("No buffers associated with packet.\n") );\r
2798                 return IB_ERROR;\r
2799         }\r
2800 \r
2801         /* Remember that one of the DS entries is reserved for the IPoIB header. */\r
2802         if( num_pages >= MAX_SEND_SGE )\r
2803         {\r
2804                 IPOIB_PRINT(TRACE_LEVEL_INFORMATION, IPOIB_DBG_SEND,\r
2805                         ("Too many buffers to fit in WR ds_array.  Copying data.\n") );\r
2806                 status = __send_copy( p_port, p_desc );\r
2807                 IPOIB_EXIT( IPOIB_DBG_SEND );\r
2808                 return status;\r
2809         }\r
2810 \r
2811         CL_ASSERT( tot_len > sizeof(eth_hdr_t) );\r
2812         CL_ASSERT( tot_len <= p_port->p_adapter->params.xfer_block_size );\r
2813         /*\r
2814          * Assume that the ethernet header is always fully contained\r
2815          * in the first page of the first MDL.  This makes for much\r
2816          * simpler code.\r
2817          */\r
2818         offset = MmGetMdlByteOffset( p_mdl ) + sizeof(eth_hdr_t);\r
2819         CL_ASSERT( offset <= PAGE_SIZE );\r
2820 \r
2821         while( tot_len )\r
2822         {\r
2823                 buf_len = MmGetMdlByteCount( p_mdl );\r
2824                 page_array = MmGetMdlPfnArray( p_mdl );\r
2825                 CL_ASSERT( page_array );\r
2826                 i = 0;\r
2827                 if( !hdr_done )\r
2828                 {\r
2829                         CL_ASSERT( buf_len >= sizeof(eth_hdr_t) );\r
2830                         /* Skip the ethernet header. */\r
2831                         buf_len -= sizeof(eth_hdr_t);\r
2832                         CL_ASSERT( buf_len <= p_port->p_adapter->params.payload_mtu );\r
2833                         if( buf_len )\r
2834                         {\r
2835                                 /* The ethernet header is a subset of this MDL. */\r
2836                                 CL_ASSERT( i == 0 );\r
2837                                 if( offset < PAGE_SIZE )\r
2838                                 {\r
2839                                         p_desc->local_ds[j].lkey = p_port->ib_mgr.lkey;\r
2840                                         p_desc->local_ds[j].vaddr = (page_array[i] << PAGE_SHIFT);\r
2841                                         /* Add the byte offset since we're on the 1st page. */\r
2842                                         p_desc->local_ds[j].vaddr += offset;\r
2843                                         if( offset + buf_len > PAGE_SIZE )\r
2844                                         {\r
2845                                                 p_desc->local_ds[j].length = PAGE_SIZE - offset;\r
2846                                                 buf_len -= p_desc->local_ds[j].length;\r
2847                                         }\r
2848                                         else\r
2849                                         {\r
2850                                                 p_desc->local_ds[j].length = buf_len;\r
2851                                                 buf_len = 0;\r
2852                                         }\r
2853                                         /* This data segment is done.  Move to the next. */\r
2854                                         j++;\r
2855                                 }\r
2856                                 /* This page is done.  Move to the next. */\r
2857                                 i++;\r
2858                         }\r
2859                         /* Done handling the ethernet header. */\r
2860                         hdr_done = TRUE;\r
2861                 }\r
2862 \r
2863                 /* Finish this MDL */\r
2864                 while( buf_len )\r
2865                 {\r
2866                         p_desc->local_ds[j].lkey = p_port->ib_mgr.lkey;\r
2867                         p_desc->local_ds[j].vaddr = (page_array[i] << PAGE_SHIFT);\r
2868                         /* Add the first page's offset if we're on the first page. */\r
2869                         if( i == 0 )\r
2870                                 p_desc->local_ds[j].vaddr += MmGetMdlByteOffset( p_mdl );\r
2871 \r
2872                         if( i == 0 && (MmGetMdlByteOffset( p_mdl ) + buf_len) > PAGE_SIZE )\r
2873                         {\r
2874                                 /* Buffers spans pages. */\r
2875                                 p_desc->local_ds[j].length =\r
2876                                         PAGE_SIZE - MmGetMdlByteOffset( p_mdl );\r
2877                                 buf_len -= p_desc->local_ds[j].length;\r
2878                                 /* This page is done.  Move to the next. */\r
2879                                 i++;\r
2880                         }\r
2881                         else\r
2882                         {\r
2883                                 /* Last page of the buffer. */\r
2884                                 p_desc->local_ds[j].length = buf_len;\r
2885                                 buf_len = 0;\r
2886                         }\r
2887                         /* This data segment is done.  Move to the next. */\r
2888                         j++;\r
2889                 }\r
2890 \r
2891                 tot_len -= MmGetMdlByteCount( p_mdl );\r
2892                 if( !tot_len )\r
2893                         break;\r
2894 \r
2895                 NdisGetNextBuffer( p_mdl, &p_mdl );\r
2896                 if( !p_mdl )\r
2897                 {\r
2898                         IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2899                                 ("Failed to get next buffer.\n") );\r
2900                         return IB_ERROR;\r
2901                 }\r
2902         }\r
2903 \r
2904         /* Set the number of data segments. */\r
2905         p_desc->wr.num_ds = j;\r
2906 \r
2907         IPOIB_EXIT( IPOIB_DBG_SEND );\r
2908         return IB_SUCCESS;\r
2909 }\r
2910 \r
2911 #else\r
2912 \r
2913 static NDIS_STATUS\r
2914 __send_gen(\r
2915         IN                              ipoib_port_t* const                     p_port,\r
2916         IN                              ipoib_send_desc_t* const        p_desc )\r
2917 {\r
2918         ib_api_status_t                 status;\r
2919         SCATTER_GATHER_LIST             *p_sgl;\r
2920         uint32_t                                i, j = 1;\r
2921         uint32_t                                offset = sizeof(eth_hdr_t);\r
2922         PERF_DECLARE( SendCopy );\r
2923 \r
2924         IPOIB_ENTER( IPOIB_DBG_SEND );\r
2925 \r
2926         p_sgl = NDIS_PER_PACKET_INFO_FROM_PACKET( p_desc->p_pkt,\r
2927                 ScatterGatherListPacketInfo );\r
2928         if( !p_sgl )\r
2929         {\r
2930                 ASSERT( p_sgl );\r
2931                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2932                         ("Failed to get SGL from packet.\n") );\r
2933                 return NDIS_STATUS_FAILURE;\r
2934         }\r
2935 \r
2936         /* Remember that one of the DS entries is reserved for the IPoIB header. */\r
2937         if( ( p_sgl->NumberOfElements >= MAX_SEND_SGE &&\r
2938                 p_sgl->Elements[0].Length > sizeof(eth_hdr_t)) ||\r
2939                 ( p_sgl->NumberOfElements > MAX_SEND_SGE &&\r
2940                 p_sgl->Elements[0].Length <= sizeof(eth_hdr_t)) )\r
2941         {\r
2942                 IPOIB_PRINT(TRACE_LEVEL_INFORMATION, IPOIB_DBG_SEND,\r
2943                         ("Too many buffers to fit in WR ds_array.  Copying data.\n") );\r
2944                 cl_perf_start( SendCopy );\r
2945                 status = __send_copy( p_port, p_desc );\r
2946                 cl_perf_stop( &p_port->p_adapter->perf, SendCopy );\r
2947                 IPOIB_EXIT( IPOIB_DBG_SEND );\r
2948                 return status;\r
2949         }\r
2950 \r
2951         /*\r
2952          * Skip the ethernet header.  It is either the first element,\r
2953          * or part of it.\r
2954          */\r
2955         i = 0;\r
2956         while( offset )\r
2957         {\r
2958                 if( p_sgl->Elements[i].Length <= sizeof(eth_hdr_t) )\r
2959                 {\r
2960                         offset -= p_sgl->Elements[i++].Length;\r
2961                 }\r
2962                 else\r
2963                 {\r
2964                         p_desc->local_ds[j].vaddr =\r
2965                                 p_sgl->Elements[i].Address.QuadPart + offset;\r
2966                         p_desc->local_ds[j].length =\r
2967                                 p_sgl->Elements[i].Length - offset;\r
2968                         p_desc->local_ds[j].lkey = p_port->ib_mgr.lkey;\r
2969                         i++;\r
2970                         j++;\r
2971                         break;\r
2972                 }\r
2973         }\r
2974         /* Now fill in the rest of the local data segments. */\r
2975         while( i < p_sgl->NumberOfElements )\r
2976         {\r
2977                 p_desc->local_ds[j].vaddr = p_sgl->Elements[i].Address.QuadPart;\r
2978                 p_desc->local_ds[j].length = p_sgl->Elements[i].Length;\r
2979                 p_desc->local_ds[j].lkey = p_port->ib_mgr.lkey;\r
2980                 i++;\r
2981                 j++;\r
2982         }\r
2983 \r
2984         /* Set the number of data segments. */\r
2985         p_desc->wr.num_ds = j;\r
2986 \r
2987         IPOIB_EXIT( IPOIB_DBG_SEND );\r
2988         return NDIS_STATUS_SUCCESS;\r
2989 }\r
2990 #endif\r
2991 \r
2992 \r
2993 static NDIS_STATUS\r
2994 __send_mgr_filter_ip(\r
2995         IN                              ipoib_port_t* const                     p_port,\r
2996         IN              const   eth_hdr_t* const                        p_eth_hdr,\r
2997         IN                              NDIS_BUFFER*                            p_buf,\r
2998         IN                              size_t                                          buf_len,\r
2999         IN      OUT                     ipoib_send_desc_t* const        p_desc )\r
3000 {\r
3001         NDIS_STATUS             status;\r
3002         ip_hdr_t                *p_ip_hdr;\r
3003 \r
3004         PERF_DECLARE( QueryIp );\r
3005         PERF_DECLARE( SendTcp );\r
3006         PERF_DECLARE( FilterUdp );\r
3007 \r
3008         IPOIB_ENTER( IPOIB_DBG_SEND );\r
3009 \r
3010         if( !buf_len )\r
3011         {\r
3012                 cl_perf_start( QueryIp );\r
3013                 NdisGetNextBuffer( p_buf, &p_buf );\r
3014                 if( !p_buf )\r
3015                 {\r
3016                         IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r