[IPoIB] Add WPP tracing, enabled by default in release builds.
[mirror/winof/.git] / ulp / ipoib / kernel / ipoib_port.c
1 /*\r
2  * Copyright (c) 2005 SilverStorm Technologies.  All rights reserved.\r
3  *\r
4  * This software is available to you under the OpenIB.org BSD license\r
5  * below:\r
6  *\r
7  *     Redistribution and use in source and binary forms, with or\r
8  *     without modification, are permitted provided that the following\r
9  *     conditions are met:\r
10  *\r
11  *      - Redistributions of source code must retain the above\r
12  *        copyright notice, this list of conditions and the following\r
13  *        disclaimer.\r
14  *\r
15  *      - Redistributions in binary form must reproduce the above\r
16  *        copyright notice, this list of conditions and the following\r
17  *        disclaimer in the documentation and/or other materials\r
18  *        provided with the distribution.\r
19  *\r
20  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,\r
21  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF\r
22  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND\r
23  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS\r
24  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN\r
25  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN\r
26  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\r
27  * SOFTWARE.\r
28  *\r
29  * $Id$\r
30  */\r
31 \r
32 \r
33 \r
34 #include "ipoib_port.h"\r
35 #include "ipoib_adapter.h"\r
36 #include "ipoib_debug.h"\r
37 #if defined(EVENT_TRACING)\r
38 #ifdef offsetof\r
39 #undef offsetof\r
40 #endif\r
41 #include "ipoib_port.tmh"\r
42 #endif\r
43 \r
44 /*\r
45  * PR 102492 relates to the SA not accepting wildcard values for MCMemberRecord\r
46  * Sets used to create/join Multicast groups.  Defining this keyword causes all\r
47  * member record fields to be hard coded to well known values.\r
48  */\r
49 //#define PR_102492\r
50 \r
51 /*\r
52  * PR 102801 relates to the SA ignoring the MGID when doing a GET_TABLE\r
53  * request that specifies an MGID.  The ESM returns every McMemberRecord\r
54  * instead.\r
55  */\r
56 //#define PR_102801\r
57 \r
58 \r
59 /* Amount of physical memory to register. */\r
60 #define MEM_REG_SIZE    0xFFFFFFFFFFFFFFFF\r
61 \r
62 /* Number of work completions to chain for send and receive polling. */\r
63 #define MAX_SEND_WC             8\r
64 #define MAX_RECV_WC             16\r
65 \r
66 \r
67 ib_gid_t        bcast_mgid_template = {\r
68         0xff,                                                           /* multicast field */\r
69         0x12,                                                           /* scope (to be filled in) */\r
70         0x40, 0x1b,                                                     /* IPv4 signature */\r
71         0xff, 0xff,                                                     /* 16 bits of P_Key (to be filled in) */\r
72         0x00, 0x00, 0x00, 0x00, 0x00, 0x00,     /* 48 bits of zeros */\r
73         0xff, 0xff, 0xff, 0xff,                         /* 32 bit IPv4 broadcast address */\r
74 };\r
75 \r
76 \r
77 #ifdef _DEBUG_\r
78 /* Handy pointer for debug use. */\r
79 ipoib_port_t    *gp_ipoib_port;\r
80 #endif\r
81 \r
82 \r
83 /******************************************************************************\r
84 *\r
85 * Declarations\r
86 *\r
87 ******************************************************************************/\r
88 static void\r
89 __port_construct(\r
90         IN                              ipoib_port_t* const                     p_port );\r
91 \r
92 static ib_api_status_t\r
93 __port_init(\r
94         IN                              ipoib_port_t* const                     p_port,\r
95         IN                              ipoib_adapter_t* const          p_adapter,\r
96         IN                              ib_pnp_port_rec_t* const        p_pnp_rec );\r
97 \r
98 static void\r
99 __port_destroying(\r
100         IN                              cl_obj_t* const                         p_obj );\r
101 \r
102 static void\r
103 __port_cleanup(\r
104         IN                              cl_obj_t* const                         p_obj );\r
105 \r
106 static void\r
107 __port_free(\r
108         IN                              cl_obj_t* const                         p_obj );\r
109 \r
110 \r
111 /******************************************************************************\r
112 *\r
113 * IB resource manager operations\r
114 *\r
115 ******************************************************************************/\r
116 static void\r
117 __ib_mgr_construct(\r
118         IN                              ipoib_port_t* const                     p_port );\r
119 \r
120 static ib_api_status_t\r
121 __ib_mgr_init(\r
122         IN                              ipoib_port_t* const                     p_port );\r
123 \r
124 static void\r
125 __ib_mgr_destroy(\r
126         IN                              ipoib_port_t* const                     p_port );\r
127 \r
128 static void\r
129 __qp_event(\r
130         IN                              ib_async_event_rec_t            *p_event_rec );\r
131 \r
132 static void\r
133 __cq_event(\r
134         IN                              ib_async_event_rec_t            *p_event_rec );\r
135 \r
136 static ib_api_status_t\r
137 __ib_mgr_activate(\r
138         IN                              ipoib_port_t* const                     p_port );\r
139 \r
140 /******************************************************************************\r
141 *\r
142 * Buffer manager operations.\r
143 *\r
144 ******************************************************************************/\r
145 static void\r
146 __buf_mgr_construct(\r
147         IN                              ipoib_port_t* const                     p_port );\r
148 \r
149 static ib_api_status_t\r
150 __buf_mgr_init(\r
151         IN                              ipoib_port_t* const                     p_port );\r
152 \r
153 static void\r
154 __buf_mgr_destroy(\r
155         IN                              ipoib_port_t* const                     p_port );\r
156 \r
157 static cl_status_t\r
158 __recv_ctor(\r
159         IN                              void* const                                     p_object,\r
160         IN                              void*                                           context,\r
161                 OUT                     cl_pool_item_t** const          pp_pool_item );\r
162 \r
163 #if !IPOIB_INLINE_RECV\r
164 static void\r
165 __recv_dtor(\r
166         IN              const   cl_pool_item_t* const           p_pool_item,\r
167         IN                              void                                            *context );\r
168 #endif  /* IPOIB_INLINE_RECV */\r
169 \r
170 static inline ipoib_send_desc_t*\r
171 __buf_mgr_get_send(\r
172         IN                              ipoib_port_t* const                     p_port );\r
173 \r
174 static inline void\r
175 __buf_mgr_put_send(\r
176         IN                              ipoib_port_t* const                     p_port,\r
177         IN                              ipoib_send_desc_t* const        p_desc );\r
178 \r
179 static inline ipoib_recv_desc_t*\r
180 __buf_mgr_get_recv(\r
181         IN                              ipoib_port_t* const                     p_port );\r
182 \r
183 static inline void\r
184 __buf_mgr_put_recv(\r
185         IN                              ipoib_port_t* const                     p_port,\r
186         IN                              ipoib_recv_desc_t* const        p_desc,\r
187         IN                              NDIS_PACKET* const                      p_packet OPTIONAL );\r
188 \r
189 static inline void\r
190 __buf_mgr_put_recv_list(\r
191         IN                              ipoib_port_t* const                     p_port,\r
192         IN                              cl_qlist_t* const                       p_list );\r
193 \r
194 static inline NDIS_PACKET*\r
195 __buf_mgr_get_ndis_pkt(\r
196         IN                              ipoib_port_t* const                     p_port,\r
197         IN                              ipoib_recv_desc_t* const        p_desc );\r
198 \r
199 \r
200 /******************************************************************************\r
201 *\r
202 * Receive manager operations.\r
203 *\r
204 ******************************************************************************/\r
205 static void\r
206 __recv_mgr_construct(\r
207         IN                              ipoib_port_t* const                     p_port );\r
208 \r
209 static ib_api_status_t\r
210 __recv_mgr_init(\r
211         IN                              ipoib_port_t* const                     p_port );\r
212 \r
213 static void\r
214 __recv_mgr_destroy(\r
215         IN                              ipoib_port_t* const                     p_port );\r
216 \r
217 /* Posts receive buffers to the receive queue. */\r
218 static ib_api_status_t\r
219 __recv_mgr_repost(\r
220         IN                              ipoib_port_t* const                     p_port,\r
221         IN                              uint32_t                                        num_wrs );\r
222 \r
223 static void\r
224 __recv_cb(\r
225         IN              const   ib_cq_handle_t                          h_cq,\r
226         IN                              void                                            *cq_context );\r
227 \r
228 static void\r
229 __recv_get_endpts(\r
230         IN                              ipoib_port_t* const                     p_port,\r
231         IN                              ipoib_recv_desc_t* const        p_desc,\r
232         IN                              ib_wc_t* const                          p_wc,\r
233                 OUT                     ipoib_endpt_t** const           pp_src,\r
234                 OUT                     ipoib_endpt_t** const           pp_dst );\r
235 \r
236 static uint32_t\r
237 __recv_mgr_filter(\r
238         IN                              ipoib_port_t* const                     p_port,\r
239         IN                              ib_wc_t* const                          p_done_wc_list,\r
240                 OUT                     cl_qlist_t* const                       p_done_list,\r
241                 OUT                     cl_qlist_t* const                       p_bad_list );\r
242 \r
243 static ib_api_status_t\r
244 __recv_gen(\r
245         IN              const   ipoib_pkt_t* const                      p_ipoib,\r
246                 OUT                     eth_pkt_t* const                        p_eth,\r
247         IN                              ipoib_endpt_t* const            p_src,\r
248         IN                              ipoib_endpt_t* const            p_dst );\r
249 \r
250 static ib_api_status_t\r
251 __recv_dhcp(\r
252         IN                              ipoib_port_t* const                     p_port,\r
253         IN              const   ipoib_pkt_t* const                      p_ipoib,\r
254                 OUT                     eth_pkt_t* const                        p_eth,\r
255         IN                              ipoib_endpt_t* const            p_src,\r
256         IN                              ipoib_endpt_t* const            p_dst );\r
257 \r
258 static ib_api_status_t\r
259 __recv_arp(\r
260         IN                              ipoib_port_t* const                     p_port,\r
261         IN                              ib_wc_t* const                          p_wc,\r
262         IN              const   ipoib_pkt_t* const                      p_ipoib,\r
263                 OUT                     eth_pkt_t* const                        p_eth,\r
264         IN                              ipoib_endpt_t** const           p_src,\r
265         IN                              ipoib_endpt_t* const            p_dst );\r
266 \r
267 static ib_api_status_t\r
268 __recv_mgr_prepare_pkt(\r
269         IN                              ipoib_port_t* const                     p_port,\r
270         IN                              ipoib_recv_desc_t* const        p_desc,\r
271                 OUT                     NDIS_PACKET** const                     pp_packet );\r
272 \r
273 static uint32_t\r
274 __recv_mgr_build_pkt_array(\r
275         IN                              ipoib_port_t* const                     p_port,\r
276                 OUT                     cl_qlist_t* const                       p_done_list );\r
277 \r
278 /******************************************************************************\r
279 *\r
280 * Send manager operations.\r
281 *\r
282 ******************************************************************************/\r
283 static void\r
284 __send_mgr_construct(\r
285         IN                              ipoib_port_t* const                     p_port );\r
286 \r
287 static void\r
288 __send_mgr_destroy(\r
289         IN                              ipoib_port_t* const                     p_port );\r
290 \r
291 static NDIS_STATUS\r
292 __send_gen(\r
293         IN                              ipoib_port_t* const                     p_port,\r
294         IN                              ipoib_send_desc_t* const        p_desc );\r
295 \r
296 static NDIS_STATUS\r
297 __send_mgr_filter_ip(\r
298         IN                              ipoib_port_t* const                     p_port,\r
299         IN              const   eth_hdr_t* const                        p_eth_hdr,\r
300         IN                              NDIS_BUFFER*                            p_buf,\r
301         IN                              size_t                                          buf_len,\r
302         IN      OUT                     ipoib_send_desc_t* const        p_desc );\r
303 \r
304 static NDIS_STATUS\r
305 __send_mgr_filter_udp(\r
306         IN                              ipoib_port_t* const                     p_port,\r
307         IN              const   ip_hdr_t* const                         p_ip_hdr,\r
308         IN                              NDIS_BUFFER*                            p_buf,\r
309         IN                              size_t                                          buf_len,\r
310         IN      OUT                     ipoib_send_desc_t* const        p_desc );\r
311 \r
312 static NDIS_STATUS\r
313 __send_mgr_filter_dhcp(\r
314         IN                              ipoib_port_t* const                     p_port,\r
315         IN              const   udp_hdr_t* const                        p_udp_hdr,\r
316         IN                              NDIS_BUFFER*                            p_buf,\r
317         IN                              size_t                                          buf_len,\r
318         IN      OUT                     ipoib_send_desc_t* const        p_desc );\r
319 \r
320 static NDIS_STATUS\r
321 __send_mgr_filter_arp(\r
322         IN                              ipoib_port_t* const                     p_port,\r
323         IN              const   eth_hdr_t* const                        p_eth_hdr,\r
324         IN                              NDIS_BUFFER*                            p_buf,\r
325         IN                              size_t                                          buf_len,\r
326         IN      OUT                     ipoib_send_desc_t* const        p_desc );\r
327 \r
328 static void\r
329 __process_failed_send(\r
330         IN                              ipoib_port_t* const                     p_port,\r
331         IN                              ipoib_send_desc_t* const        p_desc,\r
332         IN              const   NDIS_STATUS                                     status );\r
333 \r
334 static void\r
335 __send_cb(\r
336         IN              const   ib_cq_handle_t                          h_cq,\r
337         IN                              void                                            *cq_context );\r
338 \r
339 \r
340 /******************************************************************************\r
341 *\r
342 * Endpoint manager operations\r
343 *\r
344 ******************************************************************************/\r
345 static void\r
346 __endpt_mgr_construct(\r
347         IN                              ipoib_port_t* const                     p_port );\r
348 \r
349 static ib_api_status_t\r
350 __endpt_mgr_init(\r
351         IN                              ipoib_port_t* const                     p_port );\r
352 \r
353 static void\r
354 __endpt_mgr_destroy(\r
355         IN                              ipoib_port_t* const                     p_port );\r
356 \r
357 /****f* IPoIB/__endpt_mgr_remove_all\r
358 * NAME\r
359 *       __endpt_mgr_remove_all\r
360 *\r
361 * DESCRIPTION\r
362 *       Removes all enpoints from the port, dereferencing them to initiate\r
363 *       destruction.\r
364 *\r
365 * SYNOPSIS\r
366 */\r
367 static void\r
368 __endpt_mgr_remove_all(\r
369         IN                              ipoib_port_t* const                     p_port );\r
370 /*\r
371 ********/\r
372 \r
373 static void\r
374 __endpt_mgr_remove(\r
375         IN                              ipoib_port_t* const                     p_port,\r
376         IN                              ipoib_endpt_t* const            p_endpt );\r
377 \r
378 static void\r
379 __endpt_mgr_reset_all(\r
380         IN                              ipoib_port_t* const                     p_port );\r
381 \r
382 static inline NDIS_STATUS\r
383 __endpt_mgr_ref(\r
384         IN                              ipoib_port_t* const                     p_port,\r
385         IN              const   mac_addr_t                                      mac,\r
386                 OUT                     ipoib_endpt_t** const           pp_endpt );\r
387 \r
388 static inline NDIS_STATUS\r
389 __endpt_mgr_get_gid_qpn(\r
390         IN                              ipoib_port_t* const                     p_port,\r
391         IN              const   mac_addr_t                                      mac,\r
392                 OUT                     ib_gid_t* const                         p_gid,\r
393                 OUT                     UNALIGNED net32_t* const        p_qpn );\r
394 \r
395 static inline ipoib_endpt_t*\r
396 __endpt_mgr_get_by_gid(\r
397         IN                              ipoib_port_t* const                     p_port,\r
398         IN              const   ib_gid_t* const                         p_gid );\r
399 \r
400 static inline ipoib_endpt_t*\r
401 __endpt_mgr_get_by_lid(\r
402         IN                              ipoib_port_t* const                     p_port,\r
403         IN              const   net16_t                                         lid );\r
404 \r
405 static inline void\r
406 __endpt_mgr_insert_locked(\r
407         IN                              ipoib_port_t* const                     p_port,\r
408         IN              const   mac_addr_t                                      mac,\r
409         IN                              ipoib_endpt_t* const            p_endpt );\r
410 \r
411 static inline void\r
412 __endpt_mgr_insert(\r
413         IN                              ipoib_port_t* const                     p_port,\r
414         IN              const   mac_addr_t                                      mac,\r
415         IN                              ipoib_endpt_t* const            p_endpt );\r
416 \r
417 static ib_api_status_t\r
418 __endpt_mgr_add_local(\r
419         IN                              ipoib_port_t* const                     p_port,\r
420         IN                              ib_port_info_t* const           p_port_info );\r
421 \r
422 static ib_api_status_t\r
423 __endpt_mgr_add_bcast(\r
424         IN                              ipoib_port_t* const                     p_port,\r
425         IN                              ib_mcast_rec_t                          *p_mcast_rec );\r
426 \r
427 /******************************************************************************\r
428 *\r
429 * MCast operations.\r
430 *\r
431 ******************************************************************************/\r
432 static void\r
433 __port_get_mcast(\r
434         IN                              ipoib_port_t* const                     p_port );\r
435 \r
436 static void\r
437 __port_join_bcast(\r
438         IN                              ipoib_port_t* const                     p_port,\r
439         IN                              ib_member_rec_t* const          p_member_rec );\r
440 \r
441 static void\r
442 __port_create_bcast(\r
443         IN                              ipoib_port_t* const                     p_port );\r
444 \r
445 static void\r
446 __port_info_cb(\r
447         IN                              ib_query_rec_t                          *p_query_rec );\r
448 \r
449 \r
450 static void\r
451 __bcast_get_cb(\r
452         IN                              ib_query_rec_t                          *p_query_rec );\r
453 \r
454 \r
455 static void\r
456 __bcast_cb(\r
457         IN                              ib_mcast_rec_t                          *p_mcast_rec );\r
458 \r
459 \r
460 static void\r
461 __mcast_cb(\r
462         IN                              ib_mcast_rec_t                          *p_mcast_rec );\r
463 \r
464 \r
465 static intn_t\r
466 __gid_cmp(\r
467         IN              const   void* const                                     p_key1,\r
468         IN              const   void* const                                     p_key2 )\r
469 {\r
470         return cl_memcmp( p_key1, p_key2, sizeof(ib_gid_t) );\r
471 }\r
472 \r
473 \r
474 /******************************************************************************\r
475 *\r
476 * Implementation\r
477 *\r
478 ******************************************************************************/\r
479 ib_api_status_t\r
480 ipoib_create_port(\r
481         IN                              ipoib_adapter_t* const          p_adapter,\r
482         IN                              ib_pnp_port_rec_t* const        p_pnp_rec,\r
483                 OUT                     ipoib_port_t** const            pp_port )\r
484 {\r
485         ib_api_status_t         status;\r
486         ipoib_port_t            *p_port;\r
487 \r
488         IPOIB_ENTER( IPOIB_DBG_INIT );\r
489 \r
490         CL_ASSERT( !p_adapter->p_port );\r
491 \r
492         p_port = cl_zalloc( sizeof(ipoib_port_t) +\r
493                 (sizeof(ipoib_hdr_t) * (p_adapter->params.sq_depth - 1)) );\r
494         if( !p_port )\r
495         {\r
496                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
497                         ("Failed to allocate ipoib_port_t (%d bytes)\n",\r
498                         sizeof(ipoib_port_t)) );\r
499                 return IB_INSUFFICIENT_MEMORY;\r
500         }\r
501 \r
502 #ifdef _DEBUG_\r
503         gp_ipoib_port = p_port;\r
504 #endif\r
505 \r
506         __port_construct( p_port );\r
507 \r
508         status = __port_init( p_port, p_adapter, p_pnp_rec );\r
509         if( status != IB_SUCCESS )\r
510         {\r
511                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
512                         ("ipoib_port_init returned %s.\n", \r
513                         p_adapter->p_ifc->get_err_str( status )) );\r
514                 __port_cleanup( &p_port->obj );\r
515                 __port_free( &p_port->obj );\r
516                 return status;\r
517         }\r
518 \r
519         *pp_port = p_port;\r
520         IPOIB_EXIT( IPOIB_DBG_INIT );\r
521         return IB_SUCCESS;\r
522 }\r
523 \r
524 \r
525 void\r
526 ipoib_port_destroy(\r
527         IN                              ipoib_port_t* const                     p_port )\r
528 {\r
529         IPOIB_ENTER( IPOIB_DBG_INIT );\r
530 \r
531         CL_ASSERT( p_port );\r
532         CL_ASSERT( p_port->p_adapter );\r
533         CL_ASSERT( !p_port->p_adapter->p_port );\r
534 \r
535         cl_obj_destroy( &p_port->obj );\r
536 \r
537         IPOIB_EXIT( IPOIB_DBG_INIT );\r
538 }\r
539 \r
540 \r
541 static void\r
542 __port_construct(\r
543         IN                              ipoib_port_t* const                     p_port )\r
544 {\r
545         IPOIB_ENTER( IPOIB_DBG_INIT );\r
546 \r
547         p_port->state = IB_QPS_RESET;\r
548 \r
549         cl_obj_construct( &p_port->obj, IPOIB_OBJ_PORT );\r
550         cl_spinlock_construct( &p_port->send_lock );\r
551         cl_spinlock_construct( &p_port->recv_lock );\r
552         __ib_mgr_construct( p_port );\r
553         __buf_mgr_construct( p_port );\r
554 \r
555         __recv_mgr_construct( p_port );\r
556         __send_mgr_construct( p_port );\r
557 \r
558         __endpt_mgr_construct( p_port );\r
559 \r
560         IPOIB_EXIT( IPOIB_DBG_INIT );\r
561 }\r
562 \r
563 \r
564 static ib_api_status_t\r
565 __port_init(\r
566         IN                              ipoib_port_t* const                     p_port,\r
567         IN                              ipoib_adapter_t* const          p_adapter,\r
568         IN                              ib_pnp_port_rec_t* const        p_pnp_rec )\r
569 {\r
570         cl_status_t                     cl_status;\r
571         ib_api_status_t         status;\r
572 \r
573         IPOIB_ENTER( IPOIB_DBG_INIT );\r
574 \r
575         p_port->port_num = p_pnp_rec->p_port_attr->port_num;\r
576         p_port->p_adapter = p_adapter;\r
577 \r
578         cl_status = cl_spinlock_init( &p_port->send_lock );\r
579         if( cl_status != CL_SUCCESS )\r
580         {\r
581                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
582                         ("cl_spinlock_init returned %s\n", cl_status_text[cl_status]) );\r
583                 return IB_ERROR;\r
584         }\r
585 \r
586         cl_status = cl_spinlock_init( &p_port->recv_lock );\r
587         if( cl_status != CL_SUCCESS )\r
588         {\r
589                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
590                         ("cl_spinlock_init returned %s\n", cl_status_text[cl_status]) );\r
591                 return IB_ERROR;\r
592         }\r
593 \r
594         /* Initialize the IB resource manager. */\r
595         status = __ib_mgr_init( p_port );\r
596         if( status != IB_SUCCESS )\r
597         {\r
598                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
599                         ("__ib_mgr_init returned %s\n", \r
600                         p_adapter->p_ifc->get_err_str( status )) );\r
601                 return status;\r
602         }\r
603 \r
604         /* Initialize the buffer manager. */\r
605         status = __buf_mgr_init( p_port );\r
606         if( status != IB_SUCCESS )\r
607         {\r
608                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
609                         ("__buf_mgr_init returned %s\n", \r
610                         p_adapter->p_ifc->get_err_str( status )) );\r
611                 return status;\r
612         }\r
613 \r
614         /* Initialize the receive manager. */\r
615         status = __recv_mgr_init( p_port );\r
616         if( status != IB_SUCCESS )\r
617         {\r
618                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
619                         ("__recv_mgr_init returned %s\n", \r
620                         p_adapter->p_ifc->get_err_str( status )) );\r
621                 return status;\r
622         }\r
623 \r
624         /* Initialize the endpoint manager. */\r
625         status = __endpt_mgr_init( p_port );\r
626         if( status != IB_SUCCESS )\r
627         {\r
628                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
629                         ("__endpt_mgr_init returned %s\n", \r
630                         p_adapter->p_ifc->get_err_str( status )) );\r
631                 return status;\r
632         }\r
633 \r
634         /* We only ever destroy from the PnP callback thread. */\r
635         cl_status = cl_obj_init( &p_port->obj, CL_DESTROY_SYNC,\r
636                 __port_destroying, __port_cleanup, __port_free );\r
637         if( cl_status != CL_SUCCESS )\r
638         {\r
639                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
640                         ("cl_obj_init returned %s\n", cl_status_text[cl_status]) );\r
641                 return IB_ERROR;\r
642         }\r
643 \r
644         cl_obj_insert_rel( &p_port->rel, &p_adapter->obj, &p_port->obj );\r
645 \r
646         IPOIB_EXIT( IPOIB_DBG_INIT );\r
647         return IB_SUCCESS;\r
648 }\r
649 \r
650 \r
651 static void\r
652 __port_destroying(\r
653         IN                              cl_obj_t* const                         p_obj )\r
654 {\r
655         ipoib_port_t    *p_port;\r
656 \r
657         IPOIB_ENTER( IPOIB_DBG_INIT );\r
658 \r
659         CL_ASSERT( p_obj );\r
660 \r
661         p_port = PARENT_STRUCT( p_obj, ipoib_port_t, obj );\r
662 \r
663         ipoib_port_down( p_port );\r
664 \r
665         __endpt_mgr_remove_all( p_port );\r
666 \r
667         IPOIB_EXIT( IPOIB_DBG_INIT );\r
668 }\r
669 \r
670 \r
671 static void\r
672 __port_cleanup(\r
673         IN                              cl_obj_t* const                         p_obj )\r
674 {\r
675         ipoib_port_t    *p_port;\r
676 \r
677         IPOIB_ENTER( IPOIB_DBG_INIT );\r
678 \r
679         CL_ASSERT( p_obj );\r
680 \r
681         p_port = PARENT_STRUCT( p_obj, ipoib_port_t, obj );\r
682 \r
683         /* Wait for all sends and receives to get flushed. */\r
684         while( p_port->send_mgr.depth || p_port->recv_mgr.depth )\r
685                 cl_thread_suspend( 0 );\r
686 \r
687         /* Destroy the send and receive managers before closing the CA. */\r
688         __ib_mgr_destroy( p_port );\r
689 \r
690         IPOIB_EXIT( IPOIB_DBG_INIT );\r
691 }\r
692 \r
693 \r
694 static void\r
695 __port_free(\r
696         IN                              cl_obj_t* const                         p_obj )\r
697 {\r
698         ipoib_port_t    *p_port;\r
699 \r
700         IPOIB_ENTER( IPOIB_DBG_INIT );\r
701 \r
702         CL_ASSERT( p_obj );\r
703 \r
704         p_port = PARENT_STRUCT( p_obj, ipoib_port_t, obj );\r
705 \r
706         __endpt_mgr_destroy( p_port );\r
707         __recv_mgr_destroy( p_port );\r
708         __send_mgr_destroy( p_port );\r
709         __buf_mgr_destroy( p_port );\r
710 \r
711         cl_spinlock_destroy( &p_port->send_lock );\r
712         cl_spinlock_destroy( &p_port->recv_lock );\r
713 \r
714         cl_obj_deinit( p_obj );\r
715 \r
716         cl_free( p_port );\r
717 \r
718         IPOIB_EXIT( IPOIB_DBG_INIT );\r
719 }\r
720 \r
721 \r
722 \r
723 /******************************************************************************\r
724 *\r
725 * IB resource manager implementation.\r
726 *\r
727 ******************************************************************************/\r
728 static void\r
729 __ib_mgr_construct(\r
730         IN                              ipoib_port_t* const                     p_port )\r
731 {\r
732         IPOIB_ENTER( IPOIB_DBG_INIT );\r
733 \r
734         cl_memclr( &p_port->ib_mgr, sizeof(ipoib_ib_mgr_t) );\r
735 \r
736         IPOIB_EXIT( IPOIB_DBG_INIT );\r
737 }\r
738 \r
739 \r
740 static ib_api_status_t\r
741 __ib_mgr_init(\r
742         IN                              ipoib_port_t* const                     p_port )\r
743 {\r
744         ib_api_status_t         status;\r
745         ib_cq_create_t          cq_create;\r
746         ib_qp_create_t          qp_create;\r
747         ib_phys_create_t        phys_create;\r
748         ib_phys_range_t         phys_range;\r
749         uint64_t                        vaddr;\r
750         net32_t                         rkey;\r
751         ib_qp_attr_t            qp_attr;\r
752 \r
753         IPOIB_ENTER( IPOIB_DBG_INIT );\r
754 \r
755         /* Open the CA. */\r
756         status = p_port->p_adapter->p_ifc->open_ca(\r
757                 p_port->p_adapter->h_al, p_port->p_adapter->guids.ca_guid,\r
758                 NULL, p_port, &p_port->ib_mgr.h_ca );\r
759         if( status != IB_SUCCESS )\r
760         {\r
761                 NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
762                         EVENT_IPOIB_OPEN_CA, 1, status );\r
763                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
764                         ("ib_open_ca returned %s\n", \r
765                         p_port->p_adapter->p_ifc->get_err_str( status )) );\r
766                 return status;\r
767         }\r
768 \r
769         /* Allocate the PD. */\r
770         status = p_port->p_adapter->p_ifc->alloc_pd(\r
771                 p_port->ib_mgr.h_ca, IB_PDT_UD, p_port, &p_port->ib_mgr.h_pd );\r
772         if( status != IB_SUCCESS )\r
773         {\r
774                 NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
775                         EVENT_IPOIB_ALLOC_PD, 1, status );\r
776                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
777                         ("ib_alloc_pd returned %s\n", \r
778                         p_port->p_adapter->p_ifc->get_err_str( status )) );\r
779                 return status;\r
780         }\r
781 \r
782         /* Allocate receive CQ. */\r
783         cq_create.size = p_port->p_adapter->params.rq_depth;\r
784         cq_create.pfn_comp_cb = __recv_cb;\r
785         cq_create.h_wait_obj = NULL;\r
786 \r
787         status = p_port->p_adapter->p_ifc->create_cq(\r
788                 p_port->ib_mgr.h_ca, &cq_create, p_port,\r
789                 __cq_event, &p_port->ib_mgr.h_recv_cq );\r
790         if( status != IB_SUCCESS )\r
791         {\r
792                 NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
793                         EVENT_IPOIB_CREATE_RECV_CQ, 1, status );\r
794                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
795                         ("ib_create_cq returned %s.\n", \r
796                         p_port->p_adapter->p_ifc->get_err_str( status )) );\r
797                 return status;\r
798         }\r
799 \r
800         /* Allocate send CQ. */\r
801         cq_create.size = p_port->p_adapter->params.sq_depth;\r
802         cq_create.pfn_comp_cb = __send_cb;\r
803 \r
804         status = p_port->p_adapter->p_ifc->create_cq(\r
805                 p_port->ib_mgr.h_ca, &cq_create, p_port,\r
806                 __cq_event, &p_port->ib_mgr.h_send_cq );\r
807         if( status != IB_SUCCESS )\r
808         {\r
809                 NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
810                         EVENT_IPOIB_CREATE_SEND_CQ, 1, status );\r
811                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
812                         ("ib_create_cq returned %s.\n", \r
813                         p_port->p_adapter->p_ifc->get_err_str( status )) );\r
814                 return status;\r
815         }\r
816 \r
817         /* Allocate the QP. */\r
818         cl_memclr( &qp_create, sizeof(qp_create) );\r
819         qp_create.qp_type = IB_QPT_UNRELIABLE_DGRM;\r
820         qp_create.rq_depth = p_port->p_adapter->params.rq_depth;\r
821         qp_create.rq_sge = 2;   /* To support buffers spanning pages. */\r
822         qp_create.h_rq_cq = p_port->ib_mgr.h_recv_cq;\r
823         qp_create.sq_depth = p_port->p_adapter->params.sq_depth;\r
824         //TODO: Figure out the right number of SGE entries for sends.\r
825         qp_create.sq_sge = MAX_SEND_SGE;\r
826         qp_create.h_sq_cq = p_port->ib_mgr.h_send_cq;\r
827         qp_create.sq_signaled = TRUE;\r
828         status = p_port->p_adapter->p_ifc->create_qp(\r
829                 p_port->ib_mgr.h_pd, &qp_create, p_port,\r
830                 __qp_event, &p_port->ib_mgr.h_qp );\r
831         if( status != IB_SUCCESS )\r
832         {\r
833                 NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
834                         EVENT_IPOIB_CREATE_QP, 1, status );\r
835                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
836                         ("ib_create_qp returned %s\n", \r
837                         p_port->p_adapter->p_ifc->get_err_str( status )) );\r
838                 return status;\r
839         }\r
840         /* Query the QP so we can get our QPN. */\r
841         status = p_port->p_adapter->p_ifc->query_qp(\r
842                 p_port->ib_mgr.h_qp, &qp_attr );\r
843         if( status != IB_SUCCESS )\r
844         {\r
845                 NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
846                         EVENT_IPOIB_QUERY_QP, 1, status );\r
847                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
848                         ("ib_query_qp returned %s\n", \r
849                         p_port->p_adapter->p_ifc->get_err_str( status )) );\r
850                 return status;\r
851         }\r
852         p_port->ib_mgr.qpn = qp_attr.num;\r
853 \r
854         /* Register all of physical memory */\r
855         phys_create.length = MEM_REG_SIZE;\r
856         phys_create.num_ranges = 1;\r
857         phys_create.range_array = &phys_range;\r
858         phys_create.buf_offset = 0;\r
859         phys_create.hca_page_size = PAGE_SIZE;\r
860         phys_create.access_ctrl = IB_AC_LOCAL_WRITE;\r
861         phys_range.base_addr = 0;\r
862         phys_range.size = MEM_REG_SIZE;\r
863         vaddr = 0;\r
864         status = p_port->p_adapter->p_ifc->reg_phys(\r
865                 p_port->ib_mgr.h_pd, &phys_create, &vaddr,\r
866                 &p_port->ib_mgr.lkey, &rkey, &p_port->ib_mgr.h_mr );\r
867         if( status != IB_SUCCESS )\r
868         {\r
869                 NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
870                         EVENT_IPOIB_REG_PHYS, 1, status );\r
871                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
872                         ("ib_reg_phys returned %s\n", \r
873                         p_port->p_adapter->p_ifc->get_err_str( status )) );\r
874                 return status;\r
875         }\r
876 \r
877         IPOIB_EXIT( IPOIB_DBG_INIT );\r
878         return IB_SUCCESS;\r
879 }\r
880 \r
881 \r
882 static void\r
883 __ib_mgr_destroy(\r
884         IN                              ipoib_port_t* const                     p_port )\r
885 {\r
886         ib_api_status_t status;\r
887 \r
888         IPOIB_ENTER( IPOIB_DBG_INIT );\r
889 \r
890         if( p_port->ib_mgr.h_ca )\r
891         {\r
892                 status =\r
893                         p_port->p_adapter->p_ifc->close_ca( p_port->ib_mgr.h_ca, NULL );\r
894                 CL_ASSERT( status == IB_SUCCESS );\r
895                 p_port->ib_mgr.h_ca = NULL;\r
896         }\r
897 \r
898         IPOIB_EXIT( IPOIB_DBG_INIT );\r
899 }\r
900 \r
901 \r
902 \r
903 /******************************************************************************\r
904 *\r
905 * Buffer manager implementation.\r
906 *\r
907 ******************************************************************************/\r
908 static void\r
909 __buf_mgr_construct(\r
910         IN                              ipoib_port_t* const                     p_port )\r
911 {\r
912         IPOIB_ENTER( IPOIB_DBG_INIT );\r
913 \r
914         cl_qpool_construct( &p_port->buf_mgr.recv_pool );\r
915 \r
916         p_port->buf_mgr.h_packet_pool = NULL;\r
917         p_port->buf_mgr.h_buffer_pool = NULL;\r
918 \r
919         ExInitializeNPagedLookasideList( &p_port->buf_mgr.send_buf_list,\r
920                 NULL, NULL, 0, XFER_BLOCK_SIZE, 'bipi', 0 );\r
921 \r
922         p_port->buf_mgr.h_send_pkt_pool = NULL;\r
923         p_port->buf_mgr.h_send_buf_pool = NULL;\r
924 \r
925         IPOIB_EXIT( IPOIB_DBG_INIT );\r
926 }\r
927 \r
928 \r
929 static ib_api_status_t\r
930 __buf_mgr_init(\r
931         IN                              ipoib_port_t* const                     p_port )\r
932 {\r
933         cl_status_t             cl_status;\r
934         NDIS_STATUS             ndis_status;\r
935         ipoib_params_t  *p_params;\r
936 \r
937         IPOIB_ENTER(IPOIB_DBG_INIT );\r
938 \r
939         CL_ASSERT( p_port );\r
940         CL_ASSERT( p_port->p_adapter );\r
941 \r
942         p_params = &p_port->p_adapter->params;\r
943 \r
944         /* Allocate the receive descriptor pool */\r
945         cl_status = cl_qpool_init( &p_port->buf_mgr.recv_pool,\r
946                 p_params->rq_depth * p_params->recv_pool_ratio,\r
947 #if IPOIB_INLINE_RECV\r
948                 0, 0, sizeof(ipoib_recv_desc_t), __recv_ctor, NULL, p_port );\r
949 #else   /* IPOIB_INLINE_RECV */\r
950                 0, 0, sizeof(ipoib_recv_desc_t), __recv_ctor, __recv_dtor, p_port );\r
951 #endif  /* IPOIB_INLINE_RECV */\r
952         if( cl_status != CL_SUCCESS )\r
953         {\r
954                 NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
955                         EVENT_IPOIB_RECV_POOL, 1, cl_status );\r
956                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
957                         ("cl_qpool_init for recvs returned %s\n",\r
958                         cl_status_text[cl_status]) );\r
959                 return IB_INSUFFICIENT_MEMORY;\r
960         }\r
961 \r
962         /* Allocate the NDIS buffer and packet pools for receive indication. */\r
963         NdisAllocatePacketPool( &ndis_status, &p_port->buf_mgr.h_packet_pool,\r
964                 p_params->rq_depth, PROTOCOL_RESERVED_SIZE_IN_PACKET );\r
965         if( ndis_status != NDIS_STATUS_SUCCESS )\r
966         {\r
967                 NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
968                         EVENT_IPOIB_RECV_PKT_POOL, 1, ndis_status );\r
969                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
970                         ("NdisAllocatePacketPool returned %08X\n", ndis_status) );\r
971                 return IB_INSUFFICIENT_RESOURCES;\r
972         }\r
973 \r
974         NdisAllocateBufferPool( &ndis_status, &p_port->buf_mgr.h_buffer_pool,\r
975                 p_params->rq_depth );\r
976         if( ndis_status != NDIS_STATUS_SUCCESS )\r
977         {\r
978                 NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
979                         EVENT_IPOIB_RECV_BUF_POOL, 1, ndis_status );\r
980                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
981                         ("NdisAllocateBufferPool returned %08X\n", ndis_status) );\r
982                 return IB_INSUFFICIENT_RESOURCES;\r
983         }\r
984 \r
985         /* Allocate the NDIS buffer and packet pools for send formatting. */\r
986         NdisAllocatePacketPool( &ndis_status, &p_port->buf_mgr.h_send_pkt_pool,\r
987                 1, PROTOCOL_RESERVED_SIZE_IN_PACKET );\r
988         if( ndis_status != NDIS_STATUS_SUCCESS )\r
989         {\r
990                 NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
991                         EVENT_IPOIB_SEND_PKT_POOL, 1, ndis_status );\r
992                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
993                         ("NdisAllocatePacketPool returned %08X\n", ndis_status) );\r
994                 return IB_INSUFFICIENT_RESOURCES;\r
995         }\r
996 \r
997         NdisAllocateBufferPool( &ndis_status,\r
998                 &p_port->buf_mgr.h_send_buf_pool, 1 );\r
999         if( ndis_status != NDIS_STATUS_SUCCESS )\r
1000         {\r
1001                 NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
1002                         EVENT_IPOIB_SEND_BUF_POOL, 1, ndis_status );\r
1003                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1004                         ("NdisAllocateBufferPool returned %08X\n", ndis_status) );\r
1005                 return IB_INSUFFICIENT_RESOURCES;\r
1006         }\r
1007 \r
1008         IPOIB_EXIT( IPOIB_DBG_INIT );\r
1009         return IB_SUCCESS;\r
1010 }\r
1011 \r
1012 \r
1013 static void\r
1014 __buf_mgr_destroy(\r
1015         IN                              ipoib_port_t* const                     p_port )\r
1016 {\r
1017         IPOIB_ENTER(IPOIB_DBG_INIT );\r
1018 \r
1019         CL_ASSERT( p_port );\r
1020 \r
1021         /* Destroy the send packet and buffer pools. */\r
1022         if( p_port->buf_mgr.h_send_buf_pool )\r
1023                 NdisFreeBufferPool( p_port->buf_mgr.h_send_buf_pool );\r
1024         if( p_port->buf_mgr.h_send_pkt_pool )\r
1025                 NdisFreePacketPool( p_port->buf_mgr.h_send_pkt_pool );\r
1026 \r
1027         /* Destroy the receive packet and buffer pools. */\r
1028         if( p_port->buf_mgr.h_buffer_pool )\r
1029                 NdisFreeBufferPool( p_port->buf_mgr.h_buffer_pool );\r
1030         if( p_port->buf_mgr.h_packet_pool )\r
1031                 NdisFreePacketPool( p_port->buf_mgr.h_packet_pool );\r
1032 \r
1033         /* Free the receive and send descriptors. */\r
1034         cl_qpool_destroy( &p_port->buf_mgr.recv_pool );\r
1035 \r
1036         /* Free the lookaside list of scratch buffers. */\r
1037         ExDeleteNPagedLookasideList( &p_port->buf_mgr.send_buf_list );\r
1038 \r
1039         IPOIB_EXIT(  IPOIB_DBG_INIT );\r
1040 }\r
1041 \r
1042 \r
1043 static cl_status_t\r
1044 __recv_ctor(\r
1045         IN                              void* const                                     p_object,\r
1046         IN                              void*                                           context,\r
1047                 OUT                     cl_pool_item_t** const          pp_pool_item )\r
1048 {\r
1049         ipoib_recv_desc_t       *p_desc;\r
1050         ipoib_port_t            *p_port;\r
1051         uint32_t                        ds0_len;\r
1052 \r
1053         IPOIB_ENTER( IPOIB_DBG_ALLOC );\r
1054 \r
1055         CL_ASSERT( p_object );\r
1056         CL_ASSERT( context );\r
1057 \r
1058         p_desc = (ipoib_recv_desc_t*)p_object;\r
1059         p_port = (ipoib_port_t*)context;\r
1060 \r
1061         /* Setup the work request. */\r
1062         p_desc->wr.ds_array = p_desc->local_ds;\r
1063         p_desc->wr.wr_id = (uintn_t)p_desc;\r
1064 \r
1065 #if IPOIB_INLINE_RECV\r
1066         /* Sanity check on the receive buffer layout */\r
1067         CL_ASSERT( (void*)&p_desc->buf.eth.pkt.type ==\r
1068                 (void*)&p_desc->buf.ib.pkt.type );\r
1069         CL_ASSERT( sizeof(recv_buf_t) == sizeof(ipoib_pkt_t) + sizeof(ib_grh_t) );\r
1070 \r
1071         /* Setup the local data segment. */\r
1072         p_desc->local_ds[0].vaddr = cl_get_physaddr( &p_desc->buf );\r
1073         p_desc->local_ds[0].lkey = p_port->ib_mgr.lkey;\r
1074         ds0_len =\r
1075                 PAGE_SIZE - ((uint32_t)p_desc->local_ds[0].vaddr & (PAGE_SIZE - 1));\r
1076         if( ds0_len >= sizeof(recv_buf_t) )\r
1077         {\r
1078                 /* The whole buffer is within a page. */\r
1079                 p_desc->local_ds[0].length = ds0_len;\r
1080                 p_desc->wr.num_ds = 1;\r
1081         }\r
1082         else\r
1083         {\r
1084                 /* The buffer crosses page boundaries. */\r
1085                 p_desc->local_ds[0].length = ds0_len;\r
1086                 p_desc->local_ds[1].vaddr = cl_get_physaddr( \r
1087                         ((uint8_t*)&p_desc->buf) + ds0_len );\r
1088                 p_desc->local_ds[1].lkey = p_port->ib_mgr.lkey;\r
1089                 p_desc->local_ds[1].length = sizeof(recv_buf_t) - ds0_len;\r
1090                 p_desc->wr.num_ds = 2;\r
1091         }\r
1092 #else   /* IPOIB_INLINE_RECV */\r
1093         /* Allocate the receive buffer. */\r
1094         p_desc->p_buf = (recv_buf_t*)cl_zalloc( sizeof(recv_buf_t) );\r
1095         if( !p_desc->p_buf )\r
1096         {\r
1097                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1098                         ("Failed to allocate receive buffer.\n") );\r
1099                 return CL_INSUFFICIENT_MEMORY;\r
1100         }\r
1101 \r
1102         /* Sanity check on the receive buffer layout */\r
1103         CL_ASSERT( (void*)&p_desc->p_buf->eth.pkt.type ==\r
1104                 (void*)&p_desc->p_buf->ib.pkt.type );\r
1105 \r
1106         /* Setup the local data segment. */\r
1107         p_desc->local_ds[0].vaddr = cl_get_physaddr( p_desc->p_buf );\r
1108         p_desc->local_ds[0].length = sizeof(ipoib_pkt_t) + sizeof(ib_grh_t);\r
1109         p_desc->local_ds[0].lkey = p_port->ib_mgr.lkey;\r
1110 #endif  /* IPOIB_INLINE_RECV */\r
1111 \r
1112         *pp_pool_item = &p_desc->item;\r
1113 \r
1114         IPOIB_EXIT( IPOIB_DBG_ALLOC );\r
1115         return CL_SUCCESS;\r
1116 }\r
1117 \r
1118 \r
1119 #if !IPOIB_INLINE_RECV\r
1120 static void\r
1121 __recv_dtor(\r
1122         IN              const   cl_pool_item_t* const           p_pool_item,\r
1123         IN                              void                                            *context )\r
1124 {\r
1125         ipoib_recv_desc_t       *p_desc;\r
1126 \r
1127         IPOIB_ENTER(  IPOIB_DBG_ALLOC );\r
1128 \r
1129         UNUSED_PARAM( context );\r
1130 \r
1131         p_desc = PARENT_STRUCT( p_pool_item, ipoib_recv_desc_t, item );\r
1132 \r
1133         if( p_desc->p_buf )\r
1134                 cl_free( p_desc->p_buf );\r
1135 \r
1136         IPOIB_EXIT( IPOIB_DBG_ALLOC );\r
1137 }\r
1138 #endif\r
1139 \r
1140 \r
1141 static inline ipoib_recv_desc_t*\r
1142 __buf_mgr_get_recv(\r
1143         IN                              ipoib_port_t* const                     p_port )\r
1144 {\r
1145         ipoib_recv_desc_t       *p_desc;\r
1146         IPOIB_ENTER( IPOIB_DBG_RECV );\r
1147         p_desc = (ipoib_recv_desc_t*)cl_qpool_get( &p_port->buf_mgr.recv_pool );\r
1148         /* Reference the port object for the send. */\r
1149         if( p_desc )\r
1150         {\r
1151                 cl_obj_ref( &p_port->obj );\r
1152                 CL_ASSERT( p_desc->wr.wr_id == (uintn_t)p_desc );\r
1153 #if IPOIB_INLINE_RECV\r
1154                 CL_ASSERT( p_desc->local_ds[0].vaddr ==\r
1155                         cl_get_physaddr( &p_desc->buf ) );\r
1156 #else   /* IPOIB_INLINE_RECV */\r
1157                 CL_ASSERT( p_desc->local_ds[0].vaddr ==\r
1158                         cl_get_physaddr( p_desc->p_buf ) );\r
1159                 CL_ASSERT( p_desc->local_ds[0].length ==\r
1160                         (sizeof(ipoib_pkt_t) + sizeof(ib_grh_t)) );\r
1161 #endif  /* IPOIB_INLINE_RECV */\r
1162                 CL_ASSERT( p_desc->local_ds[0].lkey == p_port->ib_mgr.lkey );\r
1163         }\r
1164         IPOIB_EXIT( IPOIB_DBG_RECV );\r
1165         return p_desc;\r
1166 }\r
1167 \r
1168 \r
1169 static inline void\r
1170 __buf_mgr_put_recv(\r
1171         IN                              ipoib_port_t* const                     p_port,\r
1172         IN                              ipoib_recv_desc_t* const        p_desc,\r
1173         IN                              NDIS_PACKET* const                      p_packet OPTIONAL )\r
1174 {\r
1175         NDIS_BUFFER             *p_buf;\r
1176 \r
1177         IPOIB_ENTER(IPOIB_DBG_RECV );\r
1178 \r
1179         if( p_packet )\r
1180         {\r
1181                 /* Unchain the NDIS buffer. */\r
1182                 NdisUnchainBufferAtFront( p_packet, &p_buf );\r
1183                 CL_ASSERT( p_buf );\r
1184                 /* Return the NDIS packet and NDIS buffer to their pools. */\r
1185                 NdisDprFreePacketNonInterlocked( p_packet );\r
1186                 NdisFreeBuffer( p_buf );\r
1187         }\r
1188 \r
1189         /* Return the descriptor to its pools. */\r
1190         cl_qpool_put( &p_port->buf_mgr.recv_pool, &p_desc->item );\r
1191 \r
1192         /*\r
1193          * Dereference the port object since the receive is no longer outstanding.\r
1194          */\r
1195         cl_obj_deref( &p_port->obj );\r
1196 \r
1197         IPOIB_EXIT(  IPOIB_DBG_RECV );\r
1198 }\r
1199 \r
1200 \r
1201 static inline void\r
1202 __buf_mgr_put_recv_list(\r
1203         IN                              ipoib_port_t* const                     p_port,\r
1204         IN                              cl_qlist_t* const                       p_list )\r
1205 {\r
1206         IPOIB_ENTER(  IPOIB_DBG_RECV );\r
1207         cl_qpool_put_list( &p_port->buf_mgr.recv_pool, p_list );\r
1208         IPOIB_EXIT(  IPOIB_DBG_RECV );\r
1209 }\r
1210 \r
1211 \r
1212 static inline NDIS_PACKET*\r
1213 __buf_mgr_get_ndis_pkt(\r
1214         IN                              ipoib_port_t* const                     p_port,\r
1215         IN                              ipoib_recv_desc_t* const        p_desc )\r
1216 {\r
1217         NDIS_STATUS                             status;\r
1218         NDIS_PACKET                             *p_packet;\r
1219         NDIS_BUFFER                             *p_buffer;\r
1220 \r
1221         IPOIB_ENTER(  IPOIB_DBG_RECV );\r
1222 \r
1223         NdisDprAllocatePacketNonInterlocked( &status, &p_packet,\r
1224                 p_port->buf_mgr.h_packet_pool );\r
1225         if( status != NDIS_STATUS_SUCCESS )\r
1226         {\r
1227                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1228                         ("Failed to allocate NDIS_PACKET: %08x\n", status) );\r
1229                 return NULL;\r
1230         }\r
1231 \r
1232         IPOIB_PORT_FROM_PACKET( p_packet ) = p_port;\r
1233         IPOIB_RECV_FROM_PACKET( p_packet ) = p_desc;\r
1234 \r
1235         NdisAllocateBuffer( &status, &p_buffer,\r
1236 #if IPOIB_INLINE_RECV\r
1237                 p_port->buf_mgr.h_buffer_pool, &p_desc->buf.eth.pkt, p_desc->len );\r
1238 #else   /* IPOIB_INLINE_RECV */\r
1239                 p_port->buf_mgr.h_buffer_pool, &p_desc->p_buf->eth.pkt, p_desc->len );\r
1240 #endif  /* IPOIB_INLINE_RECV */\r
1241         if( status != NDIS_STATUS_SUCCESS )\r
1242         {\r
1243                 IPOIB_PRINT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1244                         ("Failed to allocate NDIS_BUFFER: %08x\n", status) );\r
1245                 NdisDprFreePacketNonInterlocked( p_packet );\r
1246                 return NULL;\r
1247         }\r
1248 \r
1249         NdisChainBufferAtFront( p_packet, p_buffer );\r
1250         NDIS_SET_PACKET_HEADER_SIZE( p_packet, sizeof(eth_hdr_t) );\r
1251         NDIS_SET_PACKET_STATUS( p_packet, NDIS_STATUS_SUCCESS );\r
1252 \r
1253         IPOIB_EXIT(  IPOIB_DBG_RECV );\r
1254         return p_packet;\r
1255 }\r
1256 \r
1257 /******************************************************************************\r
1258 *\r
1259 * Receive manager implementation.\r
1260 *\r
1261 ******************************************************************************/\r
1262 static void\r
1263 __recv_mgr_construct(\r
1264         IN                              ipoib_port_t* const                     p_port )\r
1265 {\r
1266         IPOIB_ENTER( IPOIB_DBG_INIT );\r
1267 \r
1268         cl_qlist_init( &p_port->recv_mgr.done_list );\r
1269 \r
1270         p_port->recv_mgr.recv_pkt_array = NULL;\r
1271 \r
1272         IPOIB_EXIT( IPOIB_DBG_INIT );\r
1273 }\r
1274 \r
1275 \r
1276 static ib_api_status_t\r
1277 __recv_mgr_init(\r
1278         IN                              ipoib_port_t* const                     p_port )\r
1279 {\r
1280         IPOIB_ENTER( IPOIB_DBG_INIT );\r
1281 \r
1282         /* Allocate the NDIS_PACKET pointer array for indicating receives. */\r
1283         p_port->recv_mgr.recv_pkt_array = cl_malloc(\r
1284                 sizeof(NDIS_PACKET*) * p_port->p_adapter->params.rq_depth );\r
1285         if( !p_port->recv_mgr.recv_pkt_array )\r
1286         {\r
1287                 NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,\r
1288                         EVENT_IPOIB_RECV_PKT_ARRAY, 0 );\r
1289                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1290                         ("cl_malloc for PNDIS_PACKET array failed.\n") );\r
1291                 return IB_INSUFFICIENT_MEMORY;\r
1292         }\r
1293 \r
1294         IPOIB_EXIT( IPOIB_DBG_INIT );\r
1295         return IB_SUCCESS;\r
1296 }\r
1297 \r
1298 \r
1299 static void\r
1300 __recv_mgr_destroy(\r
1301         IN                              ipoib_port_t* const                     p_port )\r
1302 {\r
1303         IPOIB_ENTER( IPOIB_DBG_INIT );\r
1304 \r
1305         CL_ASSERT( cl_is_qlist_empty( &p_port->recv_mgr.done_list ) );\r
1306         CL_ASSERT( !p_port->recv_mgr.depth );\r
1307 \r
1308         if( p_port->recv_mgr.recv_pkt_array )\r
1309                 cl_free( p_port->recv_mgr.recv_pkt_array );\r
1310 \r
1311         IPOIB_EXIT( IPOIB_DBG_INIT );\r
1312 }\r
1313 \r
1314 \r
1315 /* Posts receive buffers to the receive queue. */\r
1316 static ib_api_status_t\r
1317 __recv_mgr_repost(\r
1318         IN                              ipoib_port_t* const                     p_port,\r
1319         IN                              uint32_t                                        num_wrs )\r
1320 {\r
1321         ipoib_recv_desc_t       *p_head = NULL, *p_tail = NULL, *p_next;\r
1322         ib_api_status_t         status;\r
1323         ib_recv_wr_t            *p_failed;\r
1324         PERF_DECLARE( GetRecv );\r
1325         PERF_DECLARE( PostRecv );\r
1326 \r
1327         IPOIB_ENTER( IPOIB_DBG_RECV );\r
1328 \r
1329         CL_ASSERT( p_port );\r
1330         CL_ASSERT( num_wrs );\r
1331         cl_obj_lock( &p_port->obj );\r
1332         if( p_port->state == IB_QPS_ERROR )\r
1333         {\r
1334                 cl_obj_unlock( &p_port->obj );\r
1335                 IPOIB_PRINT_EXIT( TRACE_LEVEL_INFORMATION, IPOIB_DBG_RECV,\r
1336                         ("Port in invalid state.  Not reposting.\n") );\r
1337                 return IB_SUCCESS;\r
1338         }\r
1339         cl_obj_ref( &p_port->obj );\r
1340         cl_obj_unlock( &p_port->obj );\r
1341 \r
1342         while( num_wrs-- )\r
1343         {\r
1344                 if( p_port->recv_mgr.depth == p_port->p_adapter->params.rq_depth )\r
1345                 {\r
1346                         IPOIB_PRINT(TRACE_LEVEL_INFORMATION, IPOIB_DBG_RECV,\r
1347                                 ("Receive queue already full\n") );\r
1348                         break;\r
1349                 }\r
1350 \r
1351                 /* Pull receives out of the pool and chain them up. */\r
1352                 cl_perf_start( GetRecv );\r
1353                 p_next = __buf_mgr_get_recv( p_port );\r
1354                 cl_perf_stop( &p_port->p_adapter->perf, GetRecv );\r
1355                 if( !p_next )\r
1356                 {\r
1357                         IPOIB_PRINT(TRACE_LEVEL_INFORMATION, IPOIB_DBG_RECV,\r
1358                                 ("Out of receive descriptors!\n") );\r
1359                         break;\r
1360                 }\r
1361 \r
1362                 if( !p_tail )\r
1363                 {\r
1364                         p_tail = p_next;\r
1365                         p_next->wr.p_next = NULL;\r
1366                 }\r
1367                 else\r
1368                 {\r
1369                         p_next->wr.p_next = &p_head->wr;\r
1370                 }\r
1371 \r
1372                 p_head = p_next;\r
1373 \r
1374                 cl_atomic_inc( &p_port->recv_mgr.depth );\r
1375         }\r
1376 \r
1377         if( !p_head )\r
1378         {\r
1379                 cl_obj_deref( &p_port->obj );\r
1380                 IPOIB_EXIT( IPOIB_DBG_RECV );\r
1381                 return IB_SUCCESS;\r
1382         }\r
1383 \r
1384         cl_perf_start( PostRecv );\r
1385         status = p_port->p_adapter->p_ifc->post_recv(\r
1386                 p_port->ib_mgr.h_qp, &p_head->wr, &p_failed );\r
1387         cl_perf_stop( &p_port->p_adapter->perf, PostRecv );\r
1388 \r
1389         /*\r
1390          * If we failed, fix up the work completion list and return those\r
1391          * buffers to the pool\r
1392          */\r
1393         if( status != IB_SUCCESS )\r
1394         {\r
1395                 IPOIB_PRINT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1396                         ("ip_post_recv returned %s\n", \r
1397                         p_port->p_adapter->p_ifc->get_err_str( status )) );\r
1398                 /* return the descriptors to the pool */\r
1399                 while( p_failed )\r
1400                 {\r
1401                         p_head = PARENT_STRUCT( p_failed, ipoib_recv_desc_t, wr );\r
1402                         p_failed = p_failed->p_next;\r
1403 \r
1404                         __buf_mgr_put_recv( p_port, p_head, NULL );\r
1405                         cl_atomic_dec( &p_port->recv_mgr.depth );\r
1406                 }\r
1407         }\r
1408 \r
1409         cl_obj_deref( &p_port->obj );\r
1410 \r
1411         IPOIB_EXIT( IPOIB_DBG_RECV );\r
1412         return status;\r
1413 }\r
1414 \r
1415 \r
1416 void\r
1417 ipoib_return_packet(\r
1418         IN                              NDIS_HANDLE                                     adapter_context,\r
1419         IN                              NDIS_PACKET                                     *p_packet )\r
1420 {\r
1421         cl_list_item_t          *p_item;\r
1422         ipoib_port_t            *p_port;\r
1423         ipoib_recv_desc_t       *p_desc;\r
1424         ib_api_status_t         status = IB_NOT_DONE;\r
1425         PERF_DECLARE( ReturnPacket );\r
1426         PERF_DECLARE( ReturnPutRecv );\r
1427         PERF_DECLARE( ReturnRepostRecv );\r
1428         PERF_DECLARE( ReturnPreparePkt );\r
1429         PERF_DECLARE( ReturnNdisIndicate );\r
1430 \r
1431         IPOIB_ENTER( IPOIB_DBG_RECV );\r
1432 \r
1433         UNUSED_PARAM( adapter_context );\r
1434         CL_ASSERT( p_packet );\r
1435 \r
1436         cl_perf_start( ReturnPacket );\r
1437 \r
1438         /* Get the manager and descriptor from the packet. */\r
1439         p_port = IPOIB_PORT_FROM_PACKET( p_packet );\r
1440         p_desc = IPOIB_RECV_FROM_PACKET( p_packet );\r
1441 \r
1442         cl_spinlock_acquire( &p_port->recv_lock );\r
1443         cl_perf_start( ReturnPutRecv );\r
1444         __buf_mgr_put_recv( p_port, p_desc, p_packet );\r
1445         cl_perf_stop( &p_port->p_adapter->perf, ReturnPutRecv );\r
1446 \r
1447         /* Repost buffers. */\r
1448         cl_perf_start( ReturnRepostRecv );\r
1449         __recv_mgr_repost( p_port, 0xFFFF );\r
1450         cl_perf_stop( &p_port->p_adapter->perf, ReturnRepostRecv );\r
1451 \r
1452         /* Complete any additional receives waiting for a packet. */\r
1453         p_item = cl_qlist_remove_head( &p_port->recv_mgr.done_list );\r
1454         do\r
1455         {\r
1456                 if( p_item == cl_qlist_end( &p_port->recv_mgr.done_list ) )\r
1457                 {\r
1458                         cl_spinlock_release( &p_port->recv_lock );\r
1459                         break;\r
1460                 }\r
1461 \r
1462                 p_desc = (ipoib_recv_desc_t*)p_item;\r
1463 \r
1464                 cl_perf_start( ReturnPreparePkt );\r
1465                 status = __recv_mgr_prepare_pkt( p_port, p_desc, &p_packet );\r
1466                 cl_perf_stop( &p_port->p_adapter->perf, ReturnPreparePkt );\r
1467                 if( status == IB_SUCCESS )\r
1468                 {\r
1469                         cl_spinlock_release( &p_port->recv_lock );\r
1470                         cl_perf_start( ReturnNdisIndicate );\r
1471                         NdisMIndicateReceivePacket( p_port->p_adapter->h_adapter,\r
1472                                 &p_packet, 1 );\r
1473                         cl_perf_stop( &p_port->p_adapter->perf, ReturnNdisIndicate );\r
1474                 }\r
1475                 else if( status != IB_NOT_DONE )\r
1476                 {\r
1477                         IPOIB_PRINT(TRACE_LEVEL_INFORMATION, IPOIB_DBG_RECV,\r
1478                                 ("__recv_mgr_prepare_pkt returned %s\n",\r
1479                                 p_port->p_adapter->p_ifc->get_err_str( status )) );\r
1480                         /* Return the item to the head of the list. */\r
1481                         cl_qlist_insert_head( &p_port->recv_mgr.done_list, p_item );\r
1482                         cl_spinlock_release( &p_port->recv_lock );\r
1483                 }\r
1484                 else\r
1485                 {\r
1486                         p_item = cl_qlist_remove_head( &p_port->recv_mgr.done_list );\r
1487                 }\r
1488 \r
1489         } while( status == IB_NOT_DONE );\r
1490         cl_perf_stop( &p_port->p_adapter->perf, ReturnPacket );\r
1491 \r
1492         IPOIB_EXIT( IPOIB_DBG_RECV );\r
1493 }\r
1494 \r
1495 \r
1496 static void\r
1497 __recv_cb(\r
1498         IN              const   ib_cq_handle_t                          h_cq,\r
1499         IN                              void                                            *cq_context )\r
1500 {\r
1501         ipoib_port_t            *p_port;\r
1502         ib_api_status_t         status;\r
1503         ib_wc_t                         wc[MAX_RECV_WC], *p_free, *p_wc;\r
1504         uint32_t                        pkt_cnt, recv_cnt = 0;\r
1505         cl_qlist_t                      done_list, bad_list;\r
1506         size_t                          i;\r
1507         PERF_DECLARE( RecvCompBundle );\r
1508         PERF_DECLARE( RecvCb );\r
1509         PERF_DECLARE( PollRecv );\r
1510         PERF_DECLARE( RepostRecv );\r
1511         PERF_DECLARE( FilterRecv );\r
1512         PERF_DECLARE( BuildPktArray );\r
1513         PERF_DECLARE( RecvNdisIndicate );\r
1514         PERF_DECLARE( RearmRecv );\r
1515         PERF_DECLARE( PutRecvList );\r
1516 \r
1517         IPOIB_ENTER( IPOIB_DBG_RECV );\r
1518 \r
1519         cl_perf_clr( RecvCompBundle );\r
1520 \r
1521         cl_perf_start( RecvCb );\r
1522 \r
1523         UNUSED_PARAM( h_cq );\r
1524 \r
1525         p_port = (ipoib_port_t*)cq_context;\r
1526 \r
1527         cl_qlist_init( &done_list );\r
1528         cl_qlist_init( &bad_list );\r
1529 \r
1530         cl_obj_ref( &p_port->obj );\r
1531 \r
1532         for( i = 0; i < MAX_RECV_WC; i++ )\r
1533                 wc[i].p_next = &wc[i + 1];\r
1534         wc[MAX_RECV_WC - 1].p_next = NULL;\r
1535 \r
1536         /*\r
1537          * We'll be accessing the endpoint map so take a reference\r
1538          * on it to prevent modifications.\r
1539          */\r
1540         cl_obj_lock( &p_port->obj );\r
1541         cl_atomic_inc( &p_port->endpt_rdr );\r
1542         cl_obj_unlock( &p_port->obj );\r
1543 \r
1544         do\r
1545         {\r
1546                 /* If we get here, then the list of WCs is intact. */\r
1547                 p_free = wc;\r
1548 \r
1549                 cl_perf_start( PollRecv );\r
1550                 status = p_port->p_adapter->p_ifc->poll_cq(\r
1551                         p_port->ib_mgr.h_recv_cq, &p_free, &p_wc );\r
1552                 cl_perf_stop( &p_port->p_adapter->perf, PollRecv );\r
1553                 CL_ASSERT( status == IB_SUCCESS || status == IB_NOT_FOUND );\r
1554 \r
1555                 /* Look at the payload now and filter ARP and DHCP packets. */\r
1556                 cl_perf_start( FilterRecv );\r
1557                 recv_cnt += __recv_mgr_filter( p_port, p_wc, &done_list, &bad_list );\r
1558                 cl_perf_stop( &p_port->p_adapter->perf, FilterRecv );\r
1559 \r
1560         } while( !p_free );\r
1561 \r
1562         /* We're done looking at the endpoint map, release the reference. */\r
1563         cl_atomic_dec( &p_port->endpt_rdr );\r
1564 \r
1565         /* Update our posted depth. */\r
1566         cl_atomic_sub( &p_port->recv_mgr.depth, recv_cnt );\r
1567 \r
1568         cl_perf_start( BuildPktArray );\r
1569         /* Notify NDIS of any and all possible receive buffers. */\r
1570         pkt_cnt = __recv_mgr_build_pkt_array( p_port, &done_list );\r
1571         cl_perf_stop( &p_port->p_adapter->perf, BuildPktArray );\r
1572 \r
1573         /* Only indicate receives if we actually had any. */\r
1574         if( pkt_cnt )\r
1575         {\r
1576                 cl_perf_start( RecvNdisIndicate );\r
1577                 NdisMIndicateReceivePacket( p_port->p_adapter->h_adapter,\r
1578                         p_port->recv_mgr.recv_pkt_array, pkt_cnt );\r
1579                 cl_perf_stop( &p_port->p_adapter->perf, RecvNdisIndicate );\r
1580         }\r
1581 \r
1582         cl_spinlock_acquire( &p_port->recv_lock );\r
1583 \r
1584         /* Return any discarded receives to the pool */\r
1585         cl_perf_start( PutRecvList );\r
1586         __buf_mgr_put_recv_list( p_port, &bad_list );\r
1587         cl_perf_stop( &p_port->p_adapter->perf, PutRecvList );\r
1588 \r
1589         /* Repost receives. */\r
1590         cl_perf_start( RepostRecv );\r
1591         __recv_mgr_repost( p_port, recv_cnt );\r
1592         cl_perf_stop( &p_port->p_adapter->perf, RepostRecv );\r
1593 \r
1594         cl_spinlock_release( &p_port->recv_lock );\r
1595 \r
1596         /*\r
1597          * Rearm after filtering to prevent contention on the enpoint maps\r
1598          * and eliminate the possibility of having a call to\r
1599          * __endpt_mgr_insert find a duplicate.\r
1600          */\r
1601         cl_perf_start( RearmRecv );\r
1602         status = p_port->p_adapter->p_ifc->rearm_cq(\r
1603                 p_port->ib_mgr.h_recv_cq, FALSE );\r
1604         cl_perf_stop( &p_port->p_adapter->perf, RearmRecv );\r
1605         CL_ASSERT( status == IB_SUCCESS );\r
1606 \r
1607         cl_obj_deref( &p_port->obj );\r
1608 \r
1609         cl_perf_stop( &p_port->p_adapter->perf, RecvCb );\r
1610         cl_perf_log( &p_port->p_adapter->perf, RecvCompBundle, recv_cnt );\r
1611 \r
1612         IPOIB_EXIT( IPOIB_DBG_RECV );\r
1613 }\r
1614 \r
1615 \r
1616 static void\r
1617 __recv_get_endpts(\r
1618         IN                              ipoib_port_t* const                     p_port,\r
1619         IN                              ipoib_recv_desc_t* const        p_desc,\r
1620         IN                              ib_wc_t* const                          p_wc,\r
1621                 OUT                     ipoib_endpt_t** const           pp_src,\r
1622                 OUT                     ipoib_endpt_t** const           pp_dst )\r
1623 {\r
1624         ib_api_status_t         status;\r
1625         mac_addr_t                      mac;\r
1626         PERF_DECLARE( GetEndptByGid );\r
1627         PERF_DECLARE( GetEndptByLid );\r
1628         PERF_DECLARE( EndptInsert );\r
1629 \r
1630         IPOIB_ENTER( IPOIB_DBG_RECV );\r
1631 \r
1632         /* Setup our shortcut pointers based on whether GRH is valid. */\r
1633         if( p_wc->recv.ud.recv_opt & IB_RECV_OPT_GRH_VALID )\r
1634         {\r
1635                 /* Lookup the source endpoints based on GID. */\r
1636                 cl_perf_start( GetEndptByGid );\r
1637                 *pp_src =\r
1638 #if IPOIB_INLINE_RECV\r
1639                         __endpt_mgr_get_by_gid( p_port, &p_desc->buf.ib.grh.src_gid );\r
1640 #else   /* IPOIB_INLINE_RECV */\r
1641                         __endpt_mgr_get_by_gid( p_port, &p_desc->p_buf->ib.grh.src_gid );\r
1642 #endif  /* IPOIB_INLINE_RECV */\r
1643                 cl_perf_stop( &p_port->p_adapter->perf, GetEndptByGid );\r
1644 \r
1645                 /*\r
1646                  * Lookup the destination endpoint based on GID.\r
1647                  * This is used along with the packet filter to determine\r
1648                  * whether to report this to NDIS.\r
1649                  */\r
1650                 cl_perf_start( GetEndptByGid );\r
1651                 *pp_dst =\r
1652 #if IPOIB_INLINE_RECV\r
1653                         __endpt_mgr_get_by_gid( p_port, &p_desc->buf.ib.grh.dest_gid );\r
1654 #else   /* IPOIB_INLINE_RECV */\r
1655                         __endpt_mgr_get_by_gid( p_port, &p_desc->p_buf->ib.grh.dest_gid );\r
1656 #endif  /* IPOIB_INLINE_RECV */\r
1657                 cl_perf_stop( &p_port->p_adapter->perf, GetEndptByGid );\r
1658 \r
1659                 /*\r
1660                  * Create the source endpoint if it does not exist.  Note that we\r
1661                  * can only do this for globally routed traffic since we need the\r
1662                  * information from the GRH to generate the MAC.\r
1663                  */\r
1664                 if( !*pp_src )\r
1665                 {\r
1666                         status = ipoib_mac_from_guid(\r
1667 #if IPOIB_INLINE_RECV\r
1668                                 p_desc->buf.ib.grh.src_gid.unicast.interface_id, &mac );\r
1669 #else   /* IPOIB_INLINE_RECV */\r
1670                                 p_desc->p_buf->ib.grh.src_gid.unicast.interface_id, &mac );\r
1671 #endif  /* IPOIB_INLINE_RECV */\r
1672                         if( status != IB_SUCCESS )\r
1673                         {\r
1674                                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1675                                         ("ipoib_mac_from_guid returned %s\n",\r
1676                                         p_port->p_adapter->p_ifc->get_err_str( status )) );\r
1677                                 return;\r
1678                         }\r
1679 \r
1680                         /* Create the endpoint. */\r
1681 #if IPOIB_INLINE_RECV\r
1682                         *pp_src = ipoib_endpt_create( &p_desc->buf.ib.grh.src_gid,\r
1683 #else   /* IPOIB_INLINE_RECV */\r
1684                         *pp_src = ipoib_endpt_create( &p_desc->p_buf->ib.grh.src_gid,\r
1685 #endif  /* IPOIB_INLINE_RECV */\r
1686                                 p_wc->recv.ud.remote_lid, p_wc->recv.ud.remote_qp );\r
1687                         if( !*pp_src )\r
1688                         {\r
1689                                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1690                                         ("ipoib_endpt_create returned %s\n",\r
1691                                         p_port->p_adapter->p_ifc->get_err_str( status )) );\r
1692                                 return;\r
1693                         }\r
1694                         cl_perf_start( EndptInsert );\r
1695                         cl_obj_lock( &p_port->obj );\r
1696                         __endpt_mgr_insert( p_port, mac, *pp_src );\r
1697                         cl_obj_unlock( &p_port->obj );\r
1698                         cl_perf_stop( &p_port->p_adapter->perf, EndptInsert );\r
1699                 }\r
1700         }\r
1701         else\r
1702         {\r
1703                 /*\r
1704                  * Lookup the remote endpoint based on LID.  Note that only\r
1705                  * unicast traffic can be LID routed.\r
1706                  */\r
1707                 cl_perf_start( GetEndptByLid );\r
1708                 *pp_src = __endpt_mgr_get_by_lid( p_port, p_wc->recv.ud.remote_lid );\r
1709                 cl_perf_stop( &p_port->p_adapter->perf, GetEndptByLid );\r
1710                 *pp_dst = p_port->p_local_endpt;\r
1711                 CL_ASSERT( *pp_dst );\r
1712         }\r
1713 \r
1714         if( *pp_src && (*pp_src)->qpn != p_wc->recv.ud.remote_qp )\r
1715         {\r
1716                 /* Update the QPN for the endpoint. */\r
1717                 IPOIB_PRINT(TRACE_LEVEL_INFORMATION, IPOIB_DBG_RECV,\r
1718                         ("Updating QPN for MAC: %02X-%02X-%02X-%02X-%02X-%02X\n",\r
1719                         (*pp_src )->mac.addr[0], (*pp_src )->mac.addr[1],\r
1720                         (*pp_src )->mac.addr[2], (*pp_src )->mac.addr[3],\r
1721                         (*pp_src )->mac.addr[4], (*pp_src )->mac.addr[5]) );\r
1722                 (*pp_src)->qpn = p_wc->recv.ud.remote_qp;\r
1723         }\r
1724 \r
1725         if( *pp_src && *pp_dst )\r
1726         {\r
1727                 IPOIB_PRINT(TRACE_LEVEL_INFORMATION, IPOIB_DBG_RECV,\r
1728                         ("Recv:\n"\r
1729                         "\tsrc MAC: %02X-%02X-%02X-%02X-%02X-%02X\n"\r
1730                         "\tdst MAC: %02X-%02X-%02X-%02X-%02X-%02X\n",\r
1731                         (*pp_src )->mac.addr[0], (*pp_src )->mac.addr[1],\r
1732                         (*pp_src )->mac.addr[2], (*pp_src )->mac.addr[3],\r
1733                         (*pp_src )->mac.addr[4], (*pp_src )->mac.addr[5],\r
1734                         (*pp_dst )->mac.addr[0], (*pp_dst )->mac.addr[1],\r
1735                         (*pp_dst )->mac.addr[2], (*pp_dst )->mac.addr[3],\r
1736                         (*pp_dst )->mac.addr[4], (*pp_dst )->mac.addr[5]) );\r
1737         }\r
1738 \r
1739         IPOIB_EXIT( IPOIB_DBG_RECV );\r
1740 }\r
1741 \r
1742 \r
1743 static uint32_t\r
1744 __recv_mgr_filter(\r
1745         IN                              ipoib_port_t* const                     p_port,\r
1746         IN                              ib_wc_t* const                          p_done_wc_list,\r
1747                 OUT                     cl_qlist_t* const                       p_done_list,\r
1748                 OUT                     cl_qlist_t* const                       p_bad_list )\r
1749 {\r
1750         ipoib_recv_desc_t               *p_desc;\r
1751         ib_wc_t                                 *p_wc;\r
1752         ipoib_pkt_t                             *p_ipoib;\r
1753         eth_pkt_t                               *p_eth;\r
1754         ipoib_endpt_t                   *p_src, *p_dst;\r
1755         ib_api_status_t                 status;\r
1756         uint32_t                                len, recv_cnt = 0;\r
1757         PERF_DECLARE( GetRecvEndpts );\r
1758         PERF_DECLARE( RecvGen );\r
1759         PERF_DECLARE( RecvTcp );\r
1760         PERF_DECLARE( RecvUdp );\r
1761         PERF_DECLARE( RecvDhcp );\r
1762         PERF_DECLARE( RecvArp );\r
1763 \r
1764         IPOIB_ENTER( IPOIB_DBG_RECV );\r
1765 \r
1766         for( p_wc = p_done_wc_list; p_wc; p_wc = p_wc->p_next )\r
1767         {\r
1768                 CL_ASSERT( p_wc->wc_type == IB_WC_RECV );\r
1769                 p_desc = (ipoib_recv_desc_t*)(uintn_t)p_wc->wr_id;\r
1770                 recv_cnt++;\r
1771 \r
1772                 if( p_wc->status != IB_WCS_SUCCESS )\r
1773                 {\r
1774                         if( p_wc->status != IB_WCS_WR_FLUSHED_ERR )\r
1775                         {\r
1776                                 IPOIB_PRINT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1777                                         ("Failed completion %s\n",\r
1778                                         p_port->p_adapter->p_ifc->get_wc_status_str( p_wc->status )) );\r
1779                                 ipoib_inc_recv_stat( p_port->p_adapter, IP_STAT_ERROR, 0 );\r
1780                         }\r
1781                         else\r
1782                         {\r
1783                                 IPOIB_PRINT(TRACE_LEVEL_INFORMATION, IPOIB_DBG_RECV,\r
1784                                         ("Flushed completion %s\n",\r
1785                                         p_port->p_adapter->p_ifc->get_wc_status_str( p_wc->status )) );\r
1786                                 ipoib_inc_recv_stat( p_port->p_adapter, IP_STAT_DROPPED, 0 );\r
1787                         }\r
1788                         cl_qlist_insert_tail( p_bad_list, &p_desc->item.list_item );\r
1789                         /* Dereference the port object on behalf of the failed receive. */\r
1790                         cl_obj_deref( &p_port->obj );\r
1791                         continue;\r
1792                 }\r
1793 \r
1794                 len = p_wc->length - sizeof(ib_grh_t);\r
1795 \r
1796                 if( len < sizeof(ipoib_hdr_t) )\r
1797                 {\r
1798                         IPOIB_PRINT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1799                                 ("Received ETH packet < min size\n") );\r
1800                         ipoib_inc_recv_stat( p_port->p_adapter, IP_STAT_ERROR, 0 );\r
1801                         cl_qlist_insert_tail( p_bad_list, &p_desc->item.list_item );\r
1802                         cl_obj_deref( &p_port->obj );\r
1803                         continue;\r
1804                 }\r
1805 \r
1806                 /* Successful completion.  Get the receive information. */\r
1807                 cl_perf_start( GetRecvEndpts );\r
1808                 __recv_get_endpts( p_port, p_desc, p_wc, &p_src, &p_dst );\r
1809                 cl_perf_stop( &p_port->p_adapter->perf, GetRecvEndpts );\r
1810 \r
1811 #if IPOIB_INLINE_RECV\r
1812                 p_ipoib = &p_desc->buf.ib.pkt;\r
1813                 p_eth = &p_desc->buf.eth.pkt;\r
1814 #else   /* IPOIB_INLINE_RECV */\r
1815                 p_ipoib = &p_desc->p_buf->ib.pkt;\r
1816                 p_eth = &p_desc->p_buf->eth.pkt;\r
1817 #endif  /*IPOIB_INLINE_RECV */\r
1818 \r
1819                 if( p_src )\r
1820                 {\r
1821                         /* Don't report loopback traffic - we requested SW loopback. */\r
1822                         if( !cl_memcmp( &p_port->p_adapter->mac, &p_src->mac,\r
1823                                 sizeof(p_port->p_adapter->mac) ) )\r
1824                         {\r
1825                                 /*\r
1826                                  * "This is not the packet you're looking for" - don't update\r
1827                                  * receive statistics, the packet never happened.\r
1828                                  */\r
1829                                 cl_qlist_insert_tail( p_bad_list, &p_desc->item.list_item );\r
1830                                 /* Dereference the port object on behalf of the failed recv. */\r
1831                                 cl_obj_deref( &p_port->obj );\r
1832                                 continue;\r
1833                         }\r
1834                 }\r
1835 \r
1836                 switch( p_ipoib->hdr.type )\r
1837                 {\r
1838                 case ETH_PROT_TYPE_IP:\r
1839                         if( len < (sizeof(ipoib_hdr_t) + sizeof(ip_hdr_t)) )\r
1840                         {\r
1841                                 IPOIB_PRINT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1842                                         ("Received IP packet < min size\n") );\r
1843                                 status = IB_INVALID_SETTING;\r
1844                                 break;\r
1845                         }\r
1846 \r
1847                         if( p_ipoib->type.ip.hdr.offset ||\r
1848                                 p_ipoib->type.ip.hdr.prot != IP_PROT_UDP )\r
1849                         {\r
1850                                 /* Unfiltered.  Setup the ethernet header and report. */\r
1851                                 cl_perf_start( RecvTcp );\r
1852                                 status = __recv_gen( p_ipoib, p_eth, p_src, p_dst );\r
1853                                 cl_perf_stop( &p_port->p_adapter->perf, RecvTcp );\r
1854                                 break;\r
1855                         }\r
1856 \r
1857                         /* First packet of a UDP transfer. */\r
1858                         if( len <\r
1859                                 (sizeof(ipoib_hdr_t) + sizeof(ip_hdr_t) + sizeof(udp_hdr_t)) )\r
1860                         {\r
1861                                 IPOIB_PRINT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1862                                         ("Received UDP packet < min size\n") );\r
1863                                 status = IB_INVALID_SETTING;\r
1864                                 break;\r
1865                         }\r
1866 \r
1867                         /* Check if DHCP conversion is required. */\r
1868                         if( (p_ipoib->type.ip.prot.udp.hdr.dst_port == DHCP_PORT_SERVER &&\r
1869                                 p_ipoib->type.ip.prot.udp.hdr.src_port == DHCP_PORT_CLIENT) ||\r
1870                                 (p_ipoib->type.ip.prot.udp.hdr.dst_port == DHCP_PORT_CLIENT &&\r
1871                                 p_ipoib->type.ip.prot.udp.hdr.src_port == DHCP_PORT_SERVER) )\r
1872                         {\r
1873                                 if( len < (sizeof(ipoib_hdr_t) + sizeof(ip_hdr_t) +\r
1874                                         sizeof(udp_hdr_t) + DHCP_MIN_SIZE) )\r
1875                                 {\r
1876                                         IPOIB_PRINT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1877                                                 ("Received DHCP < min size\n") );\r
1878                                         status = IB_INVALID_SETTING;\r
1879                                         break;\r
1880                                 }\r
1881                                 /* UDP packet with BOOTP ports in src/dst port numbers. */\r
1882                                 cl_perf_start( RecvDhcp );\r
1883                                 status = __recv_dhcp( p_port, p_ipoib, p_eth, p_src, p_dst );\r
1884                                 cl_perf_stop( &p_port->p_adapter->perf, RecvDhcp );\r
1885                         }\r
1886                         else\r
1887                         {\r
1888                                 /* Unfiltered.  Setup the ethernet header and report. */\r
1889                                 cl_perf_start( RecvUdp );\r
1890                                 status = __recv_gen( p_ipoib, p_eth, p_src, p_dst );\r
1891                                 cl_perf_stop( &p_port->p_adapter->perf, RecvUdp );\r
1892                         }\r
1893                         break;\r
1894 \r
1895                 case ETH_PROT_TYPE_ARP:\r
1896                         if( len < (sizeof(ipoib_hdr_t) + sizeof(ipoib_arp_pkt_t)) )\r
1897                         {\r
1898                                 IPOIB_PRINT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1899                                         ("Received ARP < min size\n") );\r
1900                                 status = IB_INVALID_SETTING;\r
1901                                 break;\r
1902                         }\r
1903                         cl_perf_start( RecvArp );\r
1904                         status = __recv_arp( p_port, p_wc, p_ipoib, p_eth, &p_src, p_dst );\r
1905                         cl_perf_stop( &p_port->p_adapter->perf, RecvArp );\r
1906                         len = sizeof(ipoib_hdr_t) + sizeof(arp_pkt_t);\r
1907                         break;\r
1908 \r
1909                 default:\r
1910                         /* Unfiltered.  Setup the ethernet header and report. */\r
1911                         cl_perf_start( RecvGen );\r
1912                         status = __recv_gen( p_ipoib, p_eth, p_src, p_dst );\r
1913                         cl_perf_stop( &p_port->p_adapter->perf, RecvGen );\r
1914                 }\r
1915 \r
1916                 if( status != IB_SUCCESS )\r
1917                 {\r
1918                         /* Update stats. */\r
1919                         ipoib_inc_recv_stat( p_port->p_adapter, IP_STAT_ERROR, 0 );\r
1920                         cl_qlist_insert_tail( p_bad_list, &p_desc->item.list_item );\r
1921                         /* Dereference the port object on behalf of the failed receive. */\r
1922                         cl_obj_deref( &p_port->obj );\r
1923                 }\r
1924                 else\r
1925                 {\r
1926                         p_desc->len =\r
1927                                 len + sizeof(eth_hdr_t) - sizeof(ipoib_hdr_t);\r
1928                         if( p_dst->h_mcast)\r
1929                         {\r
1930                                 if( p_dst->dgid.multicast.raw_group_id[10] == 0xFF &&\r
1931                                         p_dst->dgid.multicast.raw_group_id[11] == 0xFF &&\r
1932                                         p_dst->dgid.multicast.raw_group_id[12] == 0xFF &&\r
1933                                         p_dst->dgid.multicast.raw_group_id[13] == 0xFF )\r
1934                                 {\r
1935                                         p_desc->type = PKT_TYPE_BCAST;\r
1936                                 }\r
1937                                 else\r
1938                                 {\r
1939                                         p_desc->type = PKT_TYPE_MCAST;\r
1940                                 }\r
1941                         }\r
1942                         else\r
1943                         {\r
1944                                 p_desc->type = PKT_TYPE_UCAST;\r
1945                         }\r
1946                         cl_qlist_insert_tail( p_done_list, &p_desc->item.list_item );\r
1947                 }\r
1948         }\r
1949 \r
1950         IPOIB_EXIT( IPOIB_DBG_RECV );\r
1951         return recv_cnt;\r
1952 }\r
1953 \r
1954 \r
1955 static ib_api_status_t\r
1956 __recv_gen(\r
1957         IN              const   ipoib_pkt_t* const                      p_ipoib,\r
1958                 OUT                     eth_pkt_t* const                        p_eth,\r
1959         IN                              ipoib_endpt_t* const            p_src,\r
1960         IN                              ipoib_endpt_t* const            p_dst )\r
1961 {\r
1962         IPOIB_ENTER( IPOIB_DBG_RECV );\r
1963 \r
1964         if( !p_src || !p_dst )\r
1965         {\r
1966                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
1967                         ("Received packet with no matching endpoints.\n") );\r
1968                 return IB_NOT_DONE;\r
1969         }\r
1970 \r
1971         /*\r
1972          * Fill in the ethernet header.  Note that doing so will overwrite\r
1973          * the IPoIB header, so start by moving the information from the IPoIB\r
1974          * header.\r
1975          */\r
1976         p_eth->hdr.type = p_ipoib->hdr.type;\r
1977         p_eth->hdr.src = p_src->mac;\r
1978         p_eth->hdr.dst = p_dst->mac;\r
1979 \r
1980         IPOIB_EXIT( IPOIB_DBG_RECV );\r
1981         return IB_SUCCESS;\r
1982 }\r
1983 \r
1984 \r
1985 static ib_api_status_t\r
1986 __recv_dhcp(\r
1987         IN                              ipoib_port_t* const                     p_port,\r
1988         IN              const   ipoib_pkt_t* const                      p_ipoib,\r
1989                 OUT                     eth_pkt_t* const                        p_eth,\r
1990         IN                              ipoib_endpt_t* const            p_src,\r
1991         IN                              ipoib_endpt_t* const            p_dst )\r
1992 {\r
1993         ib_api_status_t         status;\r
1994         dhcp_pkt_t                      *p_dhcp;\r
1995         uint8_t                         *p_option;\r
1996         uint8_t                         *p_cid = NULL;\r
1997         ib_gid_t                        gid;\r
1998         uint8_t                         msg = 0;\r
1999 \r
2000         IPOIB_ENTER( IPOIB_DBG_RECV );\r
2001 \r
2002         UNUSED_PARAM( p_port );\r
2003 \r
2004         /* Create the ethernet header. */\r
2005         status = __recv_gen( p_ipoib, p_eth, p_src, p_dst );\r
2006         if( status != IB_SUCCESS )\r
2007         {\r
2008                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2009                         ("__recv_gen returned %s.\n", \r
2010                         p_port->p_adapter->p_ifc->get_err_str( status )) );\r
2011                 return status;\r
2012         }\r
2013 \r
2014         /* Fixup the payload. */\r
2015         p_dhcp = &p_eth->type.ip.prot.udp.dhcp;\r
2016         if( p_dhcp->op != DHCP_REQUEST && p_dhcp->op != DHCP_REPLY )\r
2017         {\r
2018                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2019                         ("Invalid DHCP op code.\n") );\r
2020                 return IB_INVALID_SETTING;\r
2021         }\r
2022 \r
2023         /*\r
2024          * Find the client identifier option, making sure to skip\r
2025          * the "magic cookie".\r
2026          */\r
2027         p_option = &p_dhcp->options[0];\r
2028         if ( *(uint32_t *)p_option != DHCP_COOKIE )\r
2029         {\r
2030                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2031                         ("DHCP cookie corrupted.\n") );\r
2032                 return IB_INVALID_PARAMETER;\r
2033         }\r
2034 \r
2035         p_option = &p_dhcp->options[4];\r
2036         while( *p_option != DHCP_OPT_END )\r
2037         {\r
2038                 switch( *p_option )\r
2039                 {\r
2040                 case DHCP_OPT_PAD:\r
2041                         p_option++;\r
2042                         break;\r
2043 \r
2044                 case DHCP_OPT_MSG:\r
2045                         msg = p_option[2];\r
2046                         p_option += 3;\r
2047                         break;\r
2048 \r
2049                 case DHCP_OPT_CLIENT_ID:\r
2050                         p_cid = p_option;\r
2051                         /* Fall through. */\r
2052 \r
2053                 default:\r
2054                         /*\r
2055                          * All other options have a length byte following the option code.\r
2056                          * Offset by the length to get to the next option.\r
2057                          */\r
2058                         p_option += (p_option[1] + 2);\r
2059                 }\r
2060         }\r
2061 \r
2062         switch( msg )\r
2063         {\r
2064         /* message from client */\r
2065         case DHCPDISCOVER:\r
2066         case DHCPREQUEST:\r
2067         case DHCPDECLINE:\r
2068         case DHCPRELEASE:\r
2069         case DHCPINFORM:\r
2070                 if( !p_cid )\r
2071                 {\r
2072                         IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2073                                 ("Failed to find required Client-identifier option.\n") );\r
2074                         return IB_INVALID_SETTING;\r
2075                 }\r
2076                 if( p_dhcp->htype != DHCP_HW_TYPE_IB )\r
2077                 {\r
2078                         IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2079                                 ("Invalid hardware address type.\n") );\r
2080                         return IB_INVALID_SETTING;\r
2081                 }\r
2082                 break;\r
2083         /* message from DHCP server */\r
2084         case DHCPOFFER:\r
2085         case DHCPACK:\r
2086         case DHCPNAK:\r
2087                 break;\r
2088 \r
2089         default:\r
2090                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2091                         ("Invalide message type.\n") );\r
2092                 return IB_INVALID_PARAMETER;\r
2093         }\r
2094         p_eth->type.ip.prot.udp.hdr.chksum = 0;\r
2095         p_dhcp->htype = DHCP_HW_TYPE_ETH;\r
2096         p_dhcp->hlen = HW_ADDR_LEN;\r
2097 \r
2098         if( p_cid ) /* from client */\r
2099         {\r
2100                 /* Validate that the length and type of the option is as required. */\r
2101                 if( p_cid[1] != 21 )\r
2102                 {\r
2103                         IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2104                                 ("Client-identifier length not 21 as required.\n") );\r
2105                         return IB_INVALID_SETTING;\r
2106                 }\r
2107                 if( p_cid[2] != DHCP_HW_TYPE_IB )\r
2108                 {\r
2109                         IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2110                                 ("Client-identifier type is wrong.\n") );\r
2111                         return IB_INVALID_SETTING;\r
2112                 }\r
2113                 /*\r
2114                  * Copy the GID value from the option so that we can make aligned\r
2115                  * accesses to the contents.\r
2116                  * Recover CID to standard type.\r
2117                  */\r
2118                 cl_memcpy( &gid, &p_cid[7], sizeof(ib_gid_t) );\r
2119                 p_cid[1] =  HW_ADDR_LEN +1;// CID length \r
2120                 p_cid[2] =  DHCP_HW_TYPE_ETH;// CID type \r
2121                 status = ipoib_mac_from_guid( gid.unicast.interface_id, (mac_addr_t*)&p_cid[3] );\r
2122                 p_cid[HW_ADDR_LEN + 3] = DHCP_OPT_END; //terminate tag\r
2123         }\r
2124         IPOIB_EXIT( IPOIB_DBG_RECV );\r
2125         return status;\r
2126 }\r
2127 \r
2128 \r
2129 static ib_api_status_t\r
2130 __recv_arp(\r
2131         IN                              ipoib_port_t* const                     p_port,\r
2132         IN                              ib_wc_t* const                          p_wc,\r
2133         IN              const   ipoib_pkt_t* const                      p_ipoib,\r
2134                 OUT                     eth_pkt_t* const                        p_eth,\r
2135         IN                              ipoib_endpt_t** const           pp_src,\r
2136         IN                              ipoib_endpt_t* const            p_dst )\r
2137 {\r
2138         ib_api_status_t                 status;\r
2139         arp_pkt_t                               *p_arp;\r
2140         const ipoib_arp_pkt_t   *p_ib_arp;\r
2141         ib_gid_t                                gid;\r
2142         mac_addr_t                              mac;\r
2143         ipoib_hw_addr_t                 null_hw = {0};\r
2144 \r
2145         IPOIB_ENTER( IPOIB_DBG_RECV );\r
2146 \r
2147         if( !p_dst )\r
2148         {\r
2149                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2150                         ("Unknown destination endpoint\n") );\r
2151                 return IB_INVALID_SETTING;\r
2152         }\r
2153 \r
2154         p_ib_arp = &p_ipoib->type.arp;\r
2155         p_arp = &p_eth->type.arp;\r
2156 \r
2157         if( p_ib_arp->hw_type != ARP_HW_TYPE_IB )\r
2158         {\r
2159                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2160                         ("ARP hardware type is not IB\n") );\r
2161                 return IB_INVALID_SETTING;\r
2162         }\r
2163 \r
2164         if( p_ib_arp->hw_size != sizeof(ipoib_hw_addr_t) )\r
2165         {\r
2166                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2167                         ("ARP hardware address size is not sizeof(ipoib_hw_addr_t)\n") );\r
2168                 return IB_INVALID_SETTING;\r
2169         }\r
2170 \r
2171         if( p_ib_arp->prot_type != ETH_PROT_TYPE_IP )\r
2172         {\r
2173                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2174                         ("ARP protocal type not IP\n") );\r
2175                 return IB_INVALID_SETTING;\r
2176         }\r
2177 \r
2178         /*\r
2179          * If we don't have a source, lookup the endpoint specified in the payload.\r
2180          */\r
2181         if( !*pp_src )\r
2182                 *pp_src = __endpt_mgr_get_by_gid( p_port, &p_ib_arp->src_hw.gid );\r
2183 \r
2184         /*\r
2185          * If the endpoint exists for the GID, make sure\r
2186          * the dlid and qpn match the arp.\r
2187          */\r
2188         if( *pp_src )\r
2189         {\r
2190                 if( cl_memcmp( &(*pp_src)->dgid, &p_ib_arp->src_hw.gid,\r
2191                         sizeof(ib_gid_t) ) )\r
2192                 {\r
2193                         /*\r
2194                          * GIDs for the endpoint are different.  The ARP must\r
2195                          * have been proxied.  Dereference it.\r
2196                          */\r
2197                         *pp_src = NULL;\r
2198                 }\r
2199                 else if( ((*pp_src)->dlid != p_wc->recv.ud.remote_lid ||\r
2200                         (*pp_src)->qpn != p_wc->recv.ud.remote_qp) )\r
2201                 {\r
2202                         /* Out of date!  Destroy the endpoint and replace it. */\r
2203                         __endpt_mgr_remove( p_port, *pp_src );\r
2204                         *pp_src = NULL;\r
2205                 }\r
2206         }\r
2207 \r
2208         /* Do we need to create an endpoint for this GID? */\r
2209         if( !*pp_src )\r
2210         {\r
2211                 /* Copy the src GID to allow aligned access */\r
2212                 cl_memcpy( &gid, &p_ib_arp->src_hw.gid, sizeof(ib_gid_t) );\r
2213                 status = ipoib_mac_from_guid( gid.unicast.interface_id, &mac );\r
2214                 if( status != IB_SUCCESS )\r
2215                 {\r
2216                         IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2217                                 ("ipoib_mac_from_guid returned %s\n",\r
2218                                 p_port->p_adapter->p_ifc->get_err_str( status )) );\r
2219                         return status;\r
2220                 }\r
2221                 /*\r
2222                  * Create the endpoint.  Note that the LID is left blank and will be\r
2223                  * resolved by a path query as needed.  This is done because the\r
2224                  * remote LID/GID from the work completion may not be the original\r
2225                  * initiator.\r
2226                  */\r
2227                 *pp_src = ipoib_endpt_create( &p_ib_arp->src_hw.gid,\r
2228                         0, (p_ib_arp->src_hw.flags_qpn & CL_HTON32(0x00FFFFFF)) );\r
2229                 if( !*pp_src )\r
2230                 {\r
2231                         IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2232                                 ("ipoib_endpt_create returned %s\n",\r
2233                                 p_port->p_adapter->p_ifc->get_err_str( status )) );\r
2234                         return status;\r
2235                 }\r
2236 \r
2237                 cl_obj_lock( &p_port->obj );\r
2238                 __endpt_mgr_insert( p_port, mac, *pp_src );\r
2239                 cl_obj_unlock( &p_port->obj );\r
2240         }\r
2241 \r
2242         CL_ASSERT( !cl_memcmp(\r
2243                 &(*pp_src)->dgid, &p_ib_arp->src_hw.gid, sizeof(ib_gid_t) ) );\r
2244         CL_ASSERT( (*pp_src)->qpn ==\r
2245                 (p_ib_arp->src_hw.flags_qpn & CL_HTON32(0x00FFFFFF)) );\r
2246         /* Now swizzle the data. */\r
2247         p_arp->hw_type = ARP_HW_TYPE_ETH;\r
2248         p_arp->hw_size = sizeof(mac_addr_t);\r
2249         p_arp->src_hw = (*pp_src)->mac;\r
2250         p_arp->src_ip = p_ib_arp->src_ip;\r
2251 \r
2252         if( cl_memcmp( &p_ib_arp->dst_hw, &null_hw, sizeof(ipoib_hw_addr_t) ) )\r
2253         {\r
2254 \r
2255                 if( cl_memcmp(&p_dst->dgid, &p_ib_arp->dst_hw.gid, sizeof(ib_gid_t) ) )\r
2256                 {\r
2257                         /*\r
2258                          * We received bcast ARP packet that means\r
2259                          * remote port lets everyone know it was changed IP/MAC\r
2260                          * or just activated\r
2261                          */\r
2262                         if ( !ib_gid_is_multicast((const ib_gid_t *)&p_dst->dgid ) )\r
2263                         {\r
2264                                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2265                                         ("ARP: is not ARP MCAST\n") );\r
2266                                 return IB_INVALID_SETTING;\r
2267                         }\r
2268 \r
2269                         p_arp->dst_hw = p_port->p_local_endpt->mac;\r
2270                         p_dst->mac = p_port->p_local_endpt->mac;\r
2271                         /*\r
2272                          * we don't care what receiver ip addr is,\r
2273                          * as long as OS' ARP table is global  ???\r
2274                          */\r
2275                         p_arp->dst_ip = (net32_t)0;\r
2276 \r
2277                 }\r
2278                 else /* we've got reply to our ARP request */\r
2279                 {\r
2280                         p_arp->dst_hw = p_dst->mac;\r
2281                         p_arp->dst_ip = p_ib_arp->dst_ip;\r
2282                         CL_ASSERT( p_dst->qpn == \r
2283                                 (p_ib_arp->dst_hw.flags_qpn & CL_HTON32(0x00FFFFFF)) );\r
2284                 }\r
2285         }\r
2286         else /* we got ARP reqeust */\r
2287         {\r
2288                 cl_memclr( &p_arp->dst_hw, sizeof(mac_addr_t) );\r
2289                 p_arp->dst_ip = p_ib_arp->dst_ip;\r
2290         }\r
2291 \r
2292         /*\r
2293          * Create the ethernet header.  Note that this is done last so that\r
2294          * we have a chance to create a new endpoint.\r
2295          */\r
2296         status = __recv_gen( p_ipoib, p_eth, *pp_src, p_dst );\r
2297         if( status != IB_SUCCESS )\r
2298         {\r
2299                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2300                         ("__recv_gen returned %s.\n", \r
2301                         p_port->p_adapter->p_ifc->get_err_str( status )) );\r
2302                 return status;\r
2303         }\r
2304 \r
2305         IPOIB_EXIT( IPOIB_DBG_RECV );\r
2306         return IB_SUCCESS;\r
2307 }\r
2308 \r
2309 \r
2310 static ib_api_status_t\r
2311 __recv_mgr_prepare_pkt(\r
2312         IN                              ipoib_port_t* const                     p_port,\r
2313         IN                              ipoib_recv_desc_t* const        p_desc,\r
2314                 OUT                     NDIS_PACKET** const                     pp_packet )\r
2315 {\r
2316         NDIS_STATUS                                                     status;\r
2317         uint32_t                                                        pkt_filter;\r
2318         ip_stat_sel_t                                           type;\r
2319         NDIS_TCP_IP_CHECKSUM_PACKET_INFO        chksum;\r
2320         PERF_DECLARE( GetNdisPkt );\r
2321 \r
2322         IPOIB_ENTER( IPOIB_DBG_RECV );\r
2323 \r
2324         pkt_filter = p_port->p_adapter->packet_filter;\r
2325         /* Check the packet filter. */\r
2326         switch( p_desc->type )\r
2327         {\r
2328         default:\r
2329         case PKT_TYPE_UCAST:\r
2330                 if( pkt_filter & NDIS_PACKET_TYPE_PROMISCUOUS ||\r
2331                         pkt_filter & NDIS_PACKET_TYPE_ALL_FUNCTIONAL ||\r
2332                         pkt_filter & NDIS_PACKET_TYPE_SOURCE_ROUTING ||\r
2333                         pkt_filter & NDIS_PACKET_TYPE_DIRECTED )\r
2334                 {\r
2335                         /* OK to report. */\r
2336                         type = IP_STAT_UCAST_BYTES;\r
2337                         status = NDIS_STATUS_SUCCESS;\r
2338                 }\r
2339                 else\r
2340                 {\r
2341                         type = IP_STAT_DROPPED;\r
2342                         status = NDIS_STATUS_FAILURE;\r
2343                 }\r
2344                 break;\r
2345         case PKT_TYPE_BCAST:\r
2346                 if( pkt_filter & NDIS_PACKET_TYPE_PROMISCUOUS ||\r
2347                         pkt_filter & NDIS_PACKET_TYPE_BROADCAST )\r
2348                 {\r
2349                         /* OK to report. */\r
2350                         type = IP_STAT_BCAST_BYTES;\r
2351                         status = NDIS_STATUS_SUCCESS;\r
2352                 }\r
2353                 else\r
2354                 {\r
2355                         type = IP_STAT_DROPPED;\r
2356                         status = NDIS_STATUS_FAILURE;\r
2357                 }\r
2358                 break;\r
2359         case PKT_TYPE_MCAST:\r
2360                 if( pkt_filter & NDIS_PACKET_TYPE_PROMISCUOUS ||\r
2361                         pkt_filter & NDIS_PACKET_TYPE_ALL_MULTICAST ||\r
2362                         pkt_filter & NDIS_PACKET_TYPE_MULTICAST )\r
2363                 {\r
2364                         /* OK to report. */\r
2365                         type = IP_STAT_MCAST_BYTES;\r
2366                         status = NDIS_STATUS_SUCCESS;\r
2367                 }\r
2368                 else\r
2369                 {\r
2370                         type = IP_STAT_DROPPED;\r
2371                         status = NDIS_STATUS_FAILURE;\r
2372                 }\r
2373                 break;\r
2374         }\r
2375 \r
2376         if( status != NDIS_STATUS_SUCCESS )\r
2377         {\r
2378                 ipoib_inc_recv_stat( p_port->p_adapter, type, 0 );\r
2379                 /* Return the receive descriptor to the pool. */\r
2380                 __buf_mgr_put_recv( p_port, p_desc, NULL );\r
2381                 IPOIB_PRINT_EXIT( TRACE_LEVEL_INFORMATION, IPOIB_DBG_RECV,\r
2382                         ("Packet filter doesn't match receive.  Dropping.\n") );\r
2383                 /*\r
2384                  * Return IB_NOT_DONE since the packet has been completed,\r
2385                  * but has not consumed an array entry.\r
2386                  */\r
2387                 return IB_NOT_DONE;\r
2388         }\r
2389 \r
2390         cl_perf_start( GetNdisPkt );\r
2391         *pp_packet = __buf_mgr_get_ndis_pkt( p_port, p_desc );\r
2392         cl_perf_stop( &p_port->p_adapter->perf, GetNdisPkt );\r
2393         if( !*pp_packet )\r
2394         {\r
2395                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2396                         ("__buf_mgr_get_ndis_pkt failed\n") );\r
2397                 return IB_INSUFFICIENT_RESOURCES;\r
2398         }\r
2399 \r
2400         /* Flag the checksums as having been calculated. */\r
2401         chksum.Value = 0;\r
2402         chksum.Receive.NdisPacketTcpChecksumSucceeded = TRUE;\r
2403         chksum.Receive.NdisPacketUdpChecksumSucceeded = TRUE;\r
2404         chksum.Receive.NdisPacketIpChecksumSucceeded = TRUE;\r
2405         NDIS_PER_PACKET_INFO_FROM_PACKET( *pp_packet, TcpIpChecksumPacketInfo ) =\r
2406                 (void*)(uintn_t)chksum.Value;\r
2407 \r
2408         ipoib_inc_recv_stat( p_port->p_adapter, type, p_desc->len );\r
2409 \r
2410         IPOIB_EXIT( IPOIB_DBG_RECV );\r
2411         return IB_SUCCESS;\r
2412 }\r
2413 \r
2414 \r
2415 static uint32_t\r
2416 __recv_mgr_build_pkt_array(\r
2417         IN                              ipoib_port_t* const                     p_port,\r
2418                 OUT                     cl_qlist_t* const                       p_done_list )\r
2419 {\r
2420         cl_list_item_t                  *p_item;\r
2421         ipoib_recv_desc_t               *p_desc;\r
2422         uint32_t                                i = 0;\r
2423         ib_api_status_t                 status;\r
2424         PERF_DECLARE( PreparePkt );\r
2425 \r
2426         IPOIB_ENTER( IPOIB_DBG_RECV );\r
2427 \r
2428         cl_spinlock_acquire( &p_port->recv_lock );\r
2429 \r
2430         /* Move any existing receives to the head to preserve ordering. */\r
2431         cl_qlist_insert_list_head( p_done_list, &p_port->recv_mgr.done_list );\r
2432         p_item = cl_qlist_remove_head( p_done_list );\r
2433         while( p_item != cl_qlist_end( p_done_list ) )\r
2434         {\r
2435                 p_desc = (ipoib_recv_desc_t*)p_item;\r
2436 \r
2437                 cl_perf_start( PreparePkt );\r
2438                 status = __recv_mgr_prepare_pkt( p_port, p_desc,\r
2439                         &p_port->recv_mgr.recv_pkt_array[i] );\r
2440                 cl_perf_stop( &p_port->p_adapter->perf, PreparePkt );\r
2441                 if( status == IB_SUCCESS )\r
2442                 {\r
2443                         CL_ASSERT( p_port->recv_mgr.recv_pkt_array[i] );\r
2444                         i++;\r
2445                 }\r
2446                 else if( status != IB_NOT_DONE )\r
2447                 {\r
2448                         IPOIB_PRINT(TRACE_LEVEL_INFORMATION, IPOIB_DBG_RECV,\r
2449                                 ("__recv_mgr_prepare_pkt returned %s\n",\r
2450                                 p_port->p_adapter->p_ifc->get_err_str( status )) );\r
2451                         /* Put all completed receives on the port's done list. */\r
2452                         cl_qlist_insert_tail( &p_port->recv_mgr.done_list, p_item );\r
2453                         cl_qlist_insert_list_tail( &p_port->recv_mgr.done_list, p_done_list );\r
2454                         break;\r
2455                 }\r
2456 \r
2457                 p_item = cl_qlist_remove_head( p_done_list );\r
2458         }\r
2459 \r
2460         cl_spinlock_release( &p_port->recv_lock );\r
2461 \r
2462         IPOIB_EXIT( IPOIB_DBG_RECV );\r
2463         return i;\r
2464 }\r
2465 \r
2466 \r
2467 \r
2468 \r
2469 /******************************************************************************\r
2470 *\r
2471 * Send manager implementation.\r
2472 *\r
2473 ******************************************************************************/\r
2474 static void\r
2475 __send_mgr_construct(\r
2476         IN                              ipoib_port_t* const                     p_port )\r
2477 {\r
2478         IPOIB_ENTER( IPOIB_DBG_SEND );\r
2479         p_port->send_mgr.depth = 0;\r
2480         cl_qlist_init( &p_port->send_mgr.pending_list );\r
2481         IPOIB_EXIT( IPOIB_DBG_SEND );\r
2482 }\r
2483 \r
2484 \r
2485 static void\r
2486 __send_mgr_destroy(\r
2487         IN                              ipoib_port_t* const                     p_port )\r
2488 {\r
2489         cl_list_item_t  *p_item;\r
2490         NDIS_PACKET             *p_packet;\r
2491 \r
2492         IPOIB_ENTER( IPOIB_DBG_SEND );\r
2493 \r
2494         /* Complete any pending packets. */\r
2495         for( p_item = cl_qlist_remove_head( &p_port->send_mgr.pending_list );\r
2496                 p_item != cl_qlist_end( &p_port->send_mgr.pending_list );\r
2497                 p_item = cl_qlist_remove_head( &p_port->send_mgr.pending_list ) )\r
2498         {\r
2499                 p_packet = IPOIB_PACKET_FROM_LIST_ITEM( p_item );\r
2500                 NdisMSendComplete( p_port->p_adapter->h_adapter, p_packet,\r
2501                         NDIS_STATUS_RESET_IN_PROGRESS );\r
2502         }\r
2503 \r
2504         IPOIB_EXIT( IPOIB_DBG_SEND );\r
2505 }\r
2506 \r
2507 \r
2508 static NDIS_STATUS\r
2509 __send_mgr_filter(\r
2510         IN                              ipoib_port_t* const                     p_port,\r
2511         IN              const   eth_hdr_t* const                        p_eth_hdr,\r
2512         IN                              NDIS_BUFFER* const                      p_buf,\r
2513         IN                              size_t                                          buf_len,\r
2514         IN      OUT                     ipoib_send_desc_t* const        p_desc )\r
2515 {\r
2516         NDIS_STATUS             status;\r
2517 \r
2518         PERF_DECLARE( FilterIp );\r
2519         PERF_DECLARE( FilterArp );\r
2520         PERF_DECLARE( SendGen );\r
2521 \r
2522         IPOIB_ENTER( IPOIB_DBG_SEND );\r
2523 \r
2524         /*\r
2525          * We already checked the ethernet header length, so we know it's safe\r
2526          * to decrement the buf_len without underflowing.\r
2527          */\r
2528         buf_len -= sizeof(eth_hdr_t);\r
2529 \r
2530         switch( p_eth_hdr->type )\r
2531         {\r
2532         case ETH_PROT_TYPE_IP:\r
2533                 cl_perf_start( FilterIp );\r
2534                 status = __send_mgr_filter_ip(\r
2535                         p_port, p_eth_hdr, p_buf, buf_len, p_desc );\r
2536                 cl_perf_stop( &p_port->p_adapter->perf, FilterIp );\r
2537                 break;\r
2538 \r
2539         case ETH_PROT_TYPE_ARP:\r
2540                 cl_perf_start( FilterArp );\r
2541                 status = __send_mgr_filter_arp(\r
2542                         p_port, p_eth_hdr, p_buf, buf_len, p_desc );\r
2543                 cl_perf_stop( &p_port->p_adapter->perf, FilterArp );\r
2544                 break;\r
2545 \r
2546         default:\r
2547                 /*\r
2548                  * The IPoIB spec doesn't define how to send non IP or ARP packets.\r
2549                  * Just send the payload and hope for the best.\r
2550                  */\r
2551                 cl_perf_start( SendGen );\r
2552                 status = __send_gen( p_port, p_desc );\r
2553                 cl_perf_stop( &p_port->p_adapter->perf, SendGen );\r
2554                 break;\r
2555         }\r
2556 \r
2557         IPOIB_EXIT( IPOIB_DBG_SEND );\r
2558         return status;\r
2559 }\r
2560 \r
2561 \r
2562 static NDIS_STATUS\r
2563 __send_copy(\r
2564         IN                              ipoib_port_t* const                     p_port,\r
2565         IN                              ipoib_send_desc_t* const        p_desc )\r
2566 {\r
2567         NDIS_PACKET                             *p_packet;\r
2568         NDIS_BUFFER                             *p_buf;\r
2569         NDIS_STATUS                             status;\r
2570         UINT                                    tot_len, bytes_copied = 0;\r
2571 \r
2572         IPOIB_ENTER( IPOIB_DBG_SEND );\r
2573 \r
2574         p_desc->p_buf = \r
2575                 ExAllocateFromNPagedLookasideList( &p_port->buf_mgr.send_buf_list );\r
2576         if( !p_desc->p_buf )\r
2577         {\r
2578                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2579                         ("Failed to allocate buffer for packet copy.\n") );\r
2580                 return NDIS_STATUS_RESOURCES;\r
2581         }\r
2582 \r
2583         NdisAllocatePacket( &status, &p_packet, p_port->buf_mgr.h_send_pkt_pool );\r
2584         if( status != NDIS_STATUS_SUCCESS )\r
2585         {\r
2586                 IPOIB_PRINT_EXIT( TRACE_LEVEL_WARNING, IPOIB_DBG_SEND,\r
2587                         ("Failed to allocate NDIS_PACKET for copy.\n") );\r
2588                 return status;\r
2589         }\r
2590 \r
2591         NdisAllocateBuffer( &status, &p_buf, p_port->buf_mgr.h_send_buf_pool,\r
2592                 p_desc->p_buf, XFER_BLOCK_SIZE );\r
2593         if( status != NDIS_STATUS_SUCCESS )\r
2594         {\r
2595                 NdisFreePacket( p_packet );\r
2596                 IPOIB_PRINT_EXIT( TRACE_LEVEL_WARNING, IPOIB_DBG_SEND,\r
2597                         ("Failed to allocate NDIS_BUFFER for copy.\n") );\r
2598                 return status;\r
2599         }\r
2600 \r
2601         NdisChainBufferAtFront( p_packet, p_buf );\r
2602 \r
2603         NdisQueryPacketLength( p_desc->p_pkt, &tot_len );\r
2604 \r
2605         /* Setup the work request. */\r
2606         p_desc->local_ds[1].vaddr = cl_get_physaddr(\r
2607                 ((uint8_t*)p_desc->p_buf) + sizeof(eth_hdr_t) );\r
2608         p_desc->local_ds[1].length = tot_len - sizeof(eth_hdr_t);\r
2609         p_desc->local_ds[1].lkey = p_port->ib_mgr.lkey;\r
2610         p_desc->wr.num_ds = 2;\r
2611 \r
2612         /* Copy the packet. */\r
2613         NdisCopyFromPacketToPacketSafe( p_packet, bytes_copied, tot_len,\r
2614                 p_desc->p_pkt, bytes_copied, &bytes_copied,\r
2615                 NormalPagePriority );\r
2616 \r
2617         /* Free our temp packet now that the data is copied. */\r
2618         NdisUnchainBufferAtFront( p_packet, &p_buf );\r
2619         NdisFreeBuffer( p_buf );\r
2620         NdisFreePacket( p_packet );\r
2621 \r
2622         if( bytes_copied != tot_len )\r
2623         {\r
2624                 /* Something went wrong.  Drop the packet. */\r
2625                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2626                         ("Failed to copy full packet: %d of %d bytes copied.\n",\r
2627                         bytes_copied, tot_len) );\r
2628                 return NDIS_STATUS_RESOURCES;\r
2629         }\r
2630 \r
2631         IPOIB_EXIT( IPOIB_DBG_SEND );\r
2632         return NDIS_STATUS_SUCCESS;\r
2633 }\r
2634 \r
2635 \r
2636 #if !IPOIB_USE_DMA\r
2637 /* Send using the MDL's page information rather than the SGL. */\r
2638 static ib_api_status_t\r
2639 __send_gen(\r
2640         IN                              ipoib_port_t* const                     p_port,\r
2641         IN                              ipoib_send_desc_t* const        p_desc )\r
2642 {\r
2643         uint32_t                                i, j = 1;\r
2644         ULONG                                   offset;\r
2645         MDL                                             *p_mdl;\r
2646         UINT                                    num_pages, tot_len;\r
2647         ULONG                                   buf_len;\r
2648         PPFN_NUMBER                             page_array;\r
2649         boolean_t                               hdr_done = FALSE;\r
2650         ib_api_status_t                 status;\r
2651 \r
2652         IPOIB_ENTER( IPOIB_DBG_SEND );\r
2653 \r
2654         NdisQueryPacket( p_desc->p_pkt, &num_pages, NULL, &p_mdl,\r
2655                 &tot_len );\r
2656 \r
2657         if( !p_mdl )\r
2658         {\r
2659                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2660                         ("No buffers associated with packet.\n") );\r
2661                 return IB_ERROR;\r
2662         }\r
2663 \r
2664         /* Remember that one of the DS entries is reserved for the IPoIB header. */\r
2665         if( num_pages >= MAX_SEND_SGE )\r
2666         {\r
2667                 IPOIB_PRINT(TRACE_LEVEL_INFORMATION, IPOIB_DBG_SEND,\r
2668                         ("Too many buffers to fit in WR ds_array.  Copying data.\n") );\r
2669                 status = __send_copy( p_port, p_desc );\r
2670                 IPOIB_EXIT( IPOIB_DBG_SEND );\r
2671                 return status;\r
2672         }\r
2673 \r
2674         CL_ASSERT( tot_len > sizeof(eth_hdr_t) );\r
2675         CL_ASSERT( tot_len <= XFER_BLOCK_SIZE );\r
2676         /*\r
2677          * Assume that the ethernet header is always fully contained\r
2678          * in the first page of the first MDL.  This makes for much\r
2679          * simpler code.\r
2680          */\r
2681         offset = MmGetMdlByteOffset( p_mdl ) + sizeof(eth_hdr_t);\r
2682         CL_ASSERT( offset <= PAGE_SIZE );\r
2683 \r
2684         while( tot_len )\r
2685         {\r
2686                 buf_len = MmGetMdlByteCount( p_mdl );\r
2687                 page_array = MmGetMdlPfnArray( p_mdl );\r
2688                 CL_ASSERT( page_array );\r
2689                 i = 0;\r
2690                 if( !hdr_done )\r
2691                 {\r
2692                         CL_ASSERT( buf_len >= sizeof(eth_hdr_t) );\r
2693                         /* Skip the ethernet header. */\r
2694                         buf_len -= sizeof(eth_hdr_t);\r
2695                         CL_ASSERT( buf_len <= PAYLOAD_MTU );\r
2696                         if( buf_len )\r
2697                         {\r
2698                                 /* The ethernet header is a subset of this MDL. */\r
2699                                 CL_ASSERT( i == 0 );\r
2700                                 if( offset < PAGE_SIZE )\r
2701                                 {\r
2702                                         p_desc->local_ds[j].lkey = p_port->ib_mgr.lkey;\r
2703                                         p_desc->local_ds[j].vaddr = (page_array[i] << PAGE_SHIFT);\r
2704                                         /* Add the byte offset since we're on the 1st page. */\r
2705                                         p_desc->local_ds[j].vaddr += offset;\r
2706                                         if( offset + buf_len > PAGE_SIZE )\r
2707                                         {\r
2708                                                 p_desc->local_ds[j].length = PAGE_SIZE - offset;\r
2709                                                 buf_len -= p_desc->local_ds[j].length;\r
2710                                         }\r
2711                                         else\r
2712                                         {\r
2713                                                 p_desc->local_ds[j].length = buf_len;\r
2714                                                 buf_len = 0;\r
2715                                         }\r
2716                                         /* This data segment is done.  Move to the next. */\r
2717                                         j++;\r
2718                                 }\r
2719                                 /* This page is done.  Move to the next. */\r
2720                                 i++;\r
2721                         }\r
2722                         /* Done handling the ethernet header. */\r
2723                         hdr_done = TRUE;\r
2724                 }\r
2725 \r
2726                 /* Finish this MDL */\r
2727                 while( buf_len )\r
2728                 {\r
2729                         p_desc->local_ds[j].lkey = p_port->ib_mgr.lkey;\r
2730                         p_desc->local_ds[j].vaddr = (page_array[i] << PAGE_SHIFT);\r
2731                         /* Add the first page's offset if we're on the first page. */\r
2732                         if( i == 0 )\r
2733                                 p_desc->local_ds[j].vaddr += MmGetMdlByteOffset( p_mdl );\r
2734 \r
2735                         if( i == 0 && (MmGetMdlByteOffset( p_mdl ) + buf_len) > PAGE_SIZE )\r
2736                         {\r
2737                                 /* Buffers spans pages. */\r
2738                                 p_desc->local_ds[j].length =\r
2739                                         PAGE_SIZE - MmGetMdlByteOffset( p_mdl );\r
2740                                 buf_len -= p_desc->local_ds[j].length;\r
2741                                 /* This page is done.  Move to the next. */\r
2742                                 i++;\r
2743                         }\r
2744                         else\r
2745                         {\r
2746                                 /* Last page of the buffer. */\r
2747                                 p_desc->local_ds[j].length = buf_len;\r
2748                                 buf_len = 0;\r
2749                         }\r
2750                         /* This data segment is done.  Move to the next. */\r
2751                         j++;\r
2752                 }\r
2753 \r
2754                 tot_len -= MmGetMdlByteCount( p_mdl );\r
2755                 if( !tot_len )\r
2756                         break;\r
2757 \r
2758                 NdisGetNextBuffer( p_mdl, &p_mdl );\r
2759                 if( !p_mdl )\r
2760                 {\r
2761                         IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2762                                 ("Failed to get next buffer.\n") );\r
2763                         return IB_ERROR;\r
2764                 }\r
2765         }\r
2766 \r
2767         /* Set the number of data segments. */\r
2768         p_desc->wr.num_ds = j;\r
2769 \r
2770         IPOIB_EXIT( IPOIB_DBG_SEND );\r
2771         return IB_SUCCESS;\r
2772 }\r
2773 \r
2774 #else\r
2775 \r
2776 static NDIS_STATUS\r
2777 __send_gen(\r
2778         IN                              ipoib_port_t* const                     p_port,\r
2779         IN                              ipoib_send_desc_t* const        p_desc )\r
2780 {\r
2781         ib_api_status_t                 status;\r
2782         SCATTER_GATHER_LIST             *p_sgl;\r
2783         uint32_t                                i, j = 1;\r
2784         uint32_t                                offset = sizeof(eth_hdr_t);\r
2785         PERF_DECLARE( SendCopy );\r
2786 \r
2787         IPOIB_ENTER( IPOIB_DBG_SEND );\r
2788 \r
2789         p_sgl = NDIS_PER_PACKET_INFO_FROM_PACKET( p_desc->p_pkt,\r
2790                 ScatterGatherListPacketInfo );\r
2791         if( !p_sgl )\r
2792         {\r
2793                 ASSERT( p_sgl );\r
2794                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2795                         ("Failed to get SGL from packet.\n") );\r
2796                 return NDIS_STATUS_FAILURE;\r
2797         }\r
2798 \r
2799         /* Remember that one of the DS entries is reserved for the IPoIB header. */\r
2800         if( ( p_sgl->NumberOfElements >= MAX_SEND_SGE &&\r
2801                 p_sgl->Elements[0].Length > sizeof(eth_hdr_t)) ||\r
2802                 ( p_sgl->NumberOfElements > MAX_SEND_SGE &&\r
2803                 p_sgl->Elements[0].Length <= sizeof(eth_hdr_t)) )\r
2804         {\r
2805                 IPOIB_PRINT(TRACE_LEVEL_INFORMATION, IPOIB_DBG_SEND,\r
2806                         ("Too many buffers to fit in WR ds_array.  Copying data.\n") );\r
2807                 cl_perf_start( SendCopy );\r
2808                 status = __send_copy( p_port, p_desc );\r
2809                 cl_perf_stop( &p_port->p_adapter->perf, SendCopy );\r
2810                 IPOIB_EXIT( IPOIB_DBG_SEND );\r
2811                 return status;\r
2812         }\r
2813 \r
2814         /*\r
2815          * Skip the ethernet header.  It is either the first element,\r
2816          * or part of it.\r
2817          */\r
2818         i = 0;\r
2819         while( offset )\r
2820         {\r
2821                 if( p_sgl->Elements[i].Length <= sizeof(eth_hdr_t) )\r
2822                 {\r
2823                         offset -= p_sgl->Elements[i++].Length;\r
2824                 }\r
2825                 else\r
2826                 {\r
2827                         p_desc->local_ds[j].vaddr =\r
2828                                 p_sgl->Elements[i].Address.QuadPart + offset;\r
2829                         p_desc->local_ds[j].length =\r
2830                                 p_sgl->Elements[i].Length - offset;\r
2831                         p_desc->local_ds[j].lkey = p_port->ib_mgr.lkey;\r
2832                         i++;\r
2833                         j++;\r
2834                         break;\r
2835                 }\r
2836         }\r
2837         /* Now fill in the rest of the local data segments. */\r
2838         while( i < p_sgl->NumberOfElements )\r
2839         {\r
2840                 p_desc->local_ds[j].vaddr = p_sgl->Elements[i].Address.QuadPart;\r
2841                 p_desc->local_ds[j].length = p_sgl->Elements[i].Length;\r
2842                 p_desc->local_ds[j].lkey = p_port->ib_mgr.lkey;\r
2843                 i++;\r
2844                 j++;\r
2845         }\r
2846 \r
2847         /* Set the number of data segments. */\r
2848         p_desc->wr.num_ds = j;\r
2849 \r
2850         IPOIB_EXIT( IPOIB_DBG_SEND );\r
2851         return NDIS_STATUS_SUCCESS;\r
2852 }\r
2853 #endif\r
2854 \r
2855 \r
2856 static NDIS_STATUS\r
2857 __send_mgr_filter_ip(\r
2858         IN                              ipoib_port_t* const                     p_port,\r
2859         IN              const   eth_hdr_t* const                        p_eth_hdr,\r
2860         IN                              NDIS_BUFFER*                            p_buf,\r
2861         IN                              size_t                                          buf_len,\r
2862         IN      OUT                     ipoib_send_desc_t* const        p_desc )\r
2863 {\r
2864         NDIS_STATUS             status;\r
2865         ip_hdr_t                *p_ip_hdr;\r
2866 \r
2867         PERF_DECLARE( QueryIp );\r
2868         PERF_DECLARE( SendTcp );\r
2869         PERF_DECLARE( FilterUdp );\r
2870 \r
2871         IPOIB_ENTER( IPOIB_DBG_SEND );\r
2872 \r
2873         if( !buf_len )\r
2874         {\r
2875                 cl_perf_start( QueryIp );\r
2876                 NdisGetNextBuffer( p_buf, &p_buf );\r
2877                 if( !p_buf )\r
2878                 {\r
2879                         IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2880                                 ("Failed to get IP header buffer.\n") );\r
2881                         return NDIS_STATUS_FAILURE;\r
2882                 }\r
2883                 NdisQueryBufferSafe( p_buf, &p_ip_hdr, &buf_len, NormalPagePriority );\r
2884                 if( !p_ip_hdr )\r
2885                 {\r
2886                         IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2887                                 ("Failed to query IP header buffer.\n") );\r
2888                         return NDIS_STATUS_FAILURE;\r
2889                 }\r
2890                 cl_perf_stop( &p_port->p_adapter->perf, QueryIp );\r
2891         }\r
2892         else\r
2893         {\r
2894                 p_ip_hdr = (ip_hdr_t*)(p_eth_hdr + 1);\r
2895         }\r
2896         if( buf_len < sizeof(ip_hdr_t) )\r
2897         {\r
2898                 /* This buffer is done for.  Get the next buffer. */\r
2899                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2900                         ("Buffer too small for IP packet.\n") );\r
2901                 return NDIS_STATUS_BUFFER_TOO_SHORT;\r
2902         }\r
2903 \r
2904         if( p_ip_hdr->offset ||\r
2905                 p_ip_hdr->prot != IP_PROT_UDP )\r
2906         {\r
2907                 /* Not a UDP packet. */\r
2908                 cl_perf_start( SendTcp );\r
2909                 status = __send_gen( p_port, p_desc );\r
2910                 cl_perf_stop( &p_port->p_adapter->perf, SendTcp );\r
2911                 IPOIB_EXIT( IPOIB_DBG_SEND );\r
2912                 return status;\r
2913         }\r
2914 \r
2915         buf_len -= sizeof(ip_hdr_t);\r
2916 \r
2917         cl_perf_start( FilterUdp );\r
2918         status = __send_mgr_filter_udp(\r
2919                 p_port, p_ip_hdr, p_buf, buf_len, p_desc );\r
2920         cl_perf_stop( &p_port->p_adapter->perf, FilterUdp );\r
2921 \r
2922         IPOIB_EXIT( IPOIB_DBG_SEND );\r
2923         return status;\r
2924 }\r
2925 \r
2926 \r
2927 static NDIS_STATUS\r
2928 __send_mgr_filter_udp(\r
2929         IN                              ipoib_port_t* const                     p_port,\r
2930         IN              const   ip_hdr_t* const                         p_ip_hdr,\r
2931         IN                              NDIS_BUFFER*                            p_buf,\r
2932         IN                              size_t                                          buf_len,\r
2933         IN      OUT                     ipoib_send_desc_t* const        p_desc )\r
2934 {\r
2935         ib_api_status_t         status;\r
2936         udp_hdr_t                       *p_udp_hdr;\r
2937         PERF_DECLARE( QueryUdp );\r
2938         PERF_DECLARE( SendUdp );\r
2939         PERF_DECLARE( FilterDhcp );\r
2940 \r
2941         IPOIB_ENTER( IPOIB_DBG_SEND );\r
2942 \r
2943         if( !buf_len )\r
2944         {\r
2945                 cl_perf_start( QueryUdp );\r
2946                 NdisGetNextBuffer( p_buf, &p_buf );\r
2947                 if( !p_buf )\r
2948                 {\r
2949                         IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2950                                 ("Failed to get UDP header buffer.\n") );\r
2951                         return NDIS_STATUS_FAILURE;\r
2952                 }\r
2953                 NdisQueryBufferSafe( p_buf, &p_udp_hdr, &buf_len, NormalPagePriority );\r
2954                 if( !p_udp_hdr )\r
2955                 {\r
2956                         IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2957                                 ("Failed to query UDP header buffer.\n") );\r
2958                         return NDIS_STATUS_FAILURE;\r
2959                 }\r
2960                 cl_perf_stop( &p_port->p_adapter->perf, QueryUdp );\r
2961         }\r
2962         else\r
2963         {\r
2964                 p_udp_hdr = (udp_hdr_t*)(p_ip_hdr + 1);\r
2965         }\r
2966         /* Get the UDP header and check the destination port numbers. */\r
2967         if( buf_len < sizeof(udp_hdr_t) )\r
2968         {\r
2969                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2970                         ("Buffer not large enough for UDP packet.\n") );\r
2971                 return NDIS_STATUS_BUFFER_TOO_SHORT;\r
2972         }\r
2973 \r
2974         if( (p_udp_hdr->src_port != DHCP_PORT_CLIENT ||\r
2975                 p_udp_hdr->dst_port != DHCP_PORT_SERVER) &&\r
2976                 (p_udp_hdr->src_port != DHCP_PORT_SERVER ||\r
2977                 p_udp_hdr->dst_port != DHCP_PORT_CLIENT) )\r
2978         {\r
2979                 /* Not a DHCP packet. */\r
2980                 cl_perf_start( SendUdp );\r
2981                 status = __send_gen( p_port, p_desc );\r
2982                 cl_perf_stop( &p_port->p_adapter->perf, SendUdp );\r
2983                 IPOIB_EXIT( IPOIB_DBG_SEND );\r
2984                 return status;\r
2985         }\r
2986 \r
2987         buf_len -= sizeof(udp_hdr_t);\r
2988 \r
2989         /* Allocate our scratch buffer. */\r
2990         p_desc->p_buf = (send_buf_t*)\r
2991                 ExAllocateFromNPagedLookasideList( &p_port->buf_mgr.send_buf_list );\r
2992         if( !p_desc->p_buf )\r
2993         {\r
2994                 IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,\r
2995                         ("Failed to query DHCP packet buffer.\n") );\r
2996                 return NDIS_STATUS_RESOURCES;\r
2997         }\r
2998         /* Copy the IP and UDP headers. */\r
2999         cl_memcpy( &p_desc->p_buf->ip.hdr, p_ip_hdr , sizeof(ip_hdr_t) );\r
3000         cl_memcpy(\r
3001                 &p_desc->p_buf->ip.prot.udp.hdr, p_udp_hdr, sizeof(udp_hdr_t) );\r
3002 \r
3003         cl_perf_start( FilterDhcp );\r
3004         status = __send_mgr_filter_dhcp(\r
3005                 p_port, p_udp_hdr, p_buf, buf_len, p_desc );\r
3006         cl_perf_stop( &p_port->p_adapter->perf, FilterDhcp );\r
3007 \r
3008         IPOIB_EXIT( IPOIB_DBG_SEND );\r
3009         return status;\r
3010 }\r
3011 \r
3012 \r
3013 static NDIS_STATUS\r
3014 __send_mgr_filter_dhcp(\r
3015         IN                              ipoib_port_t* const                     p_port,\r
3016         IN              const   udp_hdr_t* const                        p_udp_hdr,\r
3017         IN                              NDIS_BUFFER*                            p_buf,\r
3018         IN                              size_t                                          buf_len,\r
3019         IN      OUT                     ipoib_send_desc_t* const        p_desc )\r
3020 {\r
3021         dhcp_pkt_t                      *p_dhcp;\r
3022         dhcp_pkt_t                      *p_ib_dhcp;\r
3023         uint8_t                         *p_option, *p_cid = NULL;\r
3024         uint8_t                         msg = 0;\r
3025         size_t                          len;\r
3026         ib_gid_t                        gid;\r
3027 \r
3028         IPOIB_ENTER( IPOIB_DBG_SEND );\r
3029 \r
3030         if( !buf_len )\r
3031         {\r
3032                 NdisGetNextBuffer( p_buf, &p_buf );\r
3033                 if( !p_buf )\r
3034                 {\r
3035                    &